diff --git a/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md b/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md
index 465ee182c497..79809179cf13 100644
--- a/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md
+++ b/sys/contrib/openzfs/.github/PULL_REQUEST_TEMPLATE.md
@@ -1,42 +1,43 @@
 <!--- Please fill out the following template, which will help other contributors review your Pull Request. -->
 
 <!--- Provide a general summary of your changes in the Title above -->
 
 <!---
 Documentation on ZFS Buildbot options can be found at
 https://openzfs.github.io/openzfs-docs/Developer%20Resources/Buildbot%20Options.html
 -->
 
 ### Motivation and Context
 <!--- Why is this change required? What problem does it solve? -->
 <!--- If it fixes an open issue, please link to the issue here. -->
 
 ### Description
 <!--- Describe your changes in detail -->
 
 ### How Has This Been Tested?
 <!--- Please describe in detail how you tested your changes. -->
 <!--- Include details of your testing environment, and the tests you ran to -->
 <!--- see how your change affects other areas of the code, etc. -->
 <!--- If your change is a performance enhancement, please provide benchmarks here. -->
 <!--- Please think about using the draft PR feature if appropriate -->
 
 ### Types of changes
 <!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Performance enhancement (non-breaking change which improves efficiency)
 - [ ] Code cleanup (non-breaking change which makes code smaller or more readable)
+- [ ] Quality assurance (non-breaking change which makes the code more robust against bugs)
 - [ ] Breaking change (fix or feature that would cause existing functionality to change)
 - [ ] Library ABI change (libzfs, libzfs\_core, libnvpair, libuutil and libzfsbootenv)
 - [ ] Documentation (a change to man pages or other documentation)
 
 ### Checklist:
 <!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
 <!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
 - [ ] My code follows the OpenZFS [code style requirements](https://github.com/openzfs/zfs/blob/master/.github/CONTRIBUTING.md#coding-conventions).
 - [ ] I have updated the documentation accordingly.
 - [ ] I have read the [**contributing** document](https://github.com/openzfs/zfs/blob/master/.github/CONTRIBUTING.md).
 - [ ] I have added [tests](https://github.com/openzfs/zfs/tree/master/tests) to cover my changes.
 - [ ] I have run the ZFS Test Suite with this change applied.
 - [ ] All commit messages are properly formatted and contain [`Signed-off-by`](https://github.com/openzfs/zfs/blob/master/.github/CONTRIBUTING.md#signed-off-by).
diff --git a/sys/contrib/openzfs/.github/workflows/checkstyle.yaml b/sys/contrib/openzfs/.github/workflows/checkstyle.yaml
index b34ca1302873..a01a4fe8587c 100644
--- a/sys/contrib/openzfs/.github/workflows/checkstyle.yaml
+++ b/sys/contrib/openzfs/.github/workflows/checkstyle.yaml
@@ -1,64 +1,64 @@
 name: checkstyle
 
 on:
   push:
   pull_request:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   checkstyle:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Install dependencies
       run: |
         # for x in lxd core20 snapd; do sudo snap remove $x; done
         sudo apt-get purge -y snapd google-chrome-stable firefox
-        ONLY_DEPS=1 .github/workflows/scripts/qemu-3-deps.sh ubuntu22
+        ONLY_DEPS=1 .github/workflows/scripts/qemu-3-deps-vm.sh ubuntu22
         sudo apt-get install -y cppcheck devscripts mandoc pax-utils shellcheck
         sudo python -m pipx install --quiet flake8
         # confirm that the tools are installed
         # the build system doesn't fail when they are not
         checkbashisms --version
         cppcheck --version
         flake8 --version
         scanelf --version
         shellcheck --version
     - name: Prepare
       run: |
         sed -i '/DEBUG_CFLAGS="-Werror"/s/^/#/' config/zfs-build.m4
         ./autogen.sh
     - name: Configure
       run: |
         ./configure
     - name: Make
       run: |
         make -j$(nproc) --no-print-directory --silent
     - name: Checkstyle
       run: |
         make -j$(nproc) --no-print-directory --silent checkstyle
     - name: Lint
       run: |
         make -j$(nproc) --no-print-directory --silent lint
     - name: CheckABI
       id: CheckABI
       run: |
         docker run -v $PWD:/source ghcr.io/openzfs/libabigail make -j$(nproc) --no-print-directory --silent checkabi
     - name: StoreABI
       if: failure() && steps.CheckABI.outcome == 'failure'
       run: |
         docker run -v $PWD:/source ghcr.io/openzfs/libabigail make -j$(nproc) --no-print-directory --silent storeabi
     - name: Prepare artifacts
       if: failure() && steps.CheckABI.outcome == 'failure'
       run: |
         find -name *.abi | tar -cf abi_files.tar -T -
     - uses: actions/upload-artifact@v4
       if: failure() && steps.CheckABI.outcome == 'failure'
       with:
         name: New ABI files (use only if you're sure about interface changes)
         path: abi_files.tar
diff --git a/sys/contrib/openzfs/.github/workflows/labels.yml b/sys/contrib/openzfs/.github/workflows/labels.yml
new file mode 100644
index 000000000000..6193c8afeae9
--- /dev/null
+++ b/sys/contrib/openzfs/.github/workflows/labels.yml
@@ -0,0 +1,49 @@
+name: labels
+
+on:
+  pull_request_target:
+    types: [ opened, synchronize, reopened, converted_to_draft, ready_for_review ]
+
+permissions:
+  pull-requests: write
+
+jobs:
+  open:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'opened' && github.event.pull_request.draft }}
+    steps:
+      - env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ISSUE: ${{ github.event.pull_request.html_url }}
+        run: |
+          gh pr edit $ISSUE --add-label "Status: Work in Progress"
+
+  push:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'synchronize' || github.event.action == 'reopened' }}
+    steps:
+      - env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ISSUE: ${{ github.event.pull_request.html_url }}
+        run: |
+          gh pr edit $ISSUE --remove-label "Status: Accepted,Status: Inactive,Status: Revision Needed,Status: Stale"
+
+  draft:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'converted_to_draft' }}
+    steps:
+      - env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ISSUE: ${{ github.event.pull_request.html_url }}
+        run: |
+          gh pr edit $ISSUE --remove-label "Status: Accepted,Status: Code Review Needed,Status: Inactive,Status: Revision Needed,Status: Stale" --add-label "Status: Work in Progress"
+
+  rfr:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'ready_for_review' }}
+    steps:
+      - env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ISSUE: ${{ github.event.pull_request.html_url }}
+        run: |
+          gh pr edit $ISSUE --remove-label "Status: Accepted,Status: Inactive,Status: Revision Needed,Status: Stale,Status: Work in Progress" --add-label "Status: Code Review Needed"
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/generate-ci-type.py b/sys/contrib/openzfs/.github/workflows/scripts/generate-ci-type.py
index 943aae254469..b49255e8381d 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/generate-ci-type.py
+++ b/sys/contrib/openzfs/.github/workflows/scripts/generate-ci-type.py
@@ -1,107 +1,108 @@
 #!/usr/bin/env python3
 
 """
 Determine the CI type based on the change list and commit message.
 
 Prints "quick" if (explicity required by user):
 - the *last* commit message contains 'ZFS-CI-Type: quick'
 or if (heuristics):
 - the files changed are not in the list of specified directories, and
 - all commit messages do not contain 'ZFS-CI-Type: full'
 
 Otherwise prints "full".
 """
 
 import sys
 import subprocess
 import re
 
 """
 Patterns of files that are not considered to trigger full CI.
 Note: not using pathlib.Path.match() because it does not support '**'
 """
 FULL_RUN_IGNORE_REGEX = list(map(re.compile, [
     r'.*\.md',
     r'.*\.gitignore'
 ]))
 
 """
 Patterns of files that are considered to trigger full CI.
 """
 FULL_RUN_REGEX = list(map(re.compile, [
+    r'\.github/workflows/scripts/.*',
     r'cmd.*',
     r'configs/.*',
     r'META',
     r'.*\.am',
     r'.*\.m4',
     r'autogen\.sh',
     r'configure\.ac',
     r'copy-builtin',
     r'contrib',
     r'etc',
     r'include',
     r'lib/.*',
     r'module/.*',
     r'scripts/.*',
     r'tests/.*',
     r'udev/.*'
 ]))
 
 if __name__ == '__main__':
 
     prog = sys.argv[0]
 
     if len(sys.argv) != 3:
         print(f'Usage: {prog} <head_ref> <base_ref>')
         sys.exit(1)
 
     head, base = sys.argv[1:3]
 
     def output_type(type, reason):
         print(f'{prog}: will run {type} CI: {reason}', file=sys.stderr)
         print(type)
         sys.exit(0)
 
     # check last (HEAD) commit message
     last_commit_message_raw = subprocess.run([
         'git', 'show', '-s', '--format=%B', 'HEAD'
     ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
     for line in last_commit_message_raw.stdout.decode().splitlines():
         if line.strip().lower() == 'zfs-ci-type: quick':
             output_type('quick', f'explicitly requested by HEAD commit {head}')
 
     # check all commit messages
     all_commit_message_raw = subprocess.run([
         'git', 'show', '-s',
         '--format=ZFS-CI-Commit: %H%n%B', f'{head}...{base}'
     ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     all_commit_message = all_commit_message_raw.stdout.decode().splitlines()
 
     commit_ref = head
     for line in all_commit_message:
         if line.startswith('ZFS-CI-Commit:'):
             commit_ref = line.lstrip('ZFS-CI-Commit:').rstrip()
         if line.strip().lower() == 'zfs-ci-type: full':
             output_type('full', f'explicitly requested by commit {commit_ref}')
 
     # check changed files
     changed_files_raw = subprocess.run([
         'git', 'diff', '--name-only', head, base
     ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     changed_files = changed_files_raw.stdout.decode().splitlines()
 
     for f in changed_files:
         for r in FULL_RUN_IGNORE_REGEX:
             if r.match(f):
                 break
         else:
             for r in FULL_RUN_REGEX:
                 if r.match(f):
                     output_type(
                         'full',
                         f'changed file "{f}" matches pattern "{r.pattern}"'
                         )
 
     # catch-all
     output_type('quick', 'no changed file matches full CI patterns')
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-1-setup.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-1-setup.sh
index f838da34efff..de29ad1f57b6 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-1-setup.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-1-setup.sh
@@ -1,93 +1,67 @@
 #!/usr/bin/env bash
 
 ######################################################################
 # 1) setup qemu instance on action runner
 ######################################################################
 
 set -eu
 
 # install needed packages
 export DEBIAN_FRONTEND="noninteractive"
 sudo apt-get -y update
 sudo apt-get install -y axel cloud-image-utils daemonize guestfs-tools \
-  ksmtuned virt-manager linux-modules-extra-$(uname -r) zfsutils-linux
+  virt-manager linux-modules-extra-$(uname -r) zfsutils-linux
 
 # generate ssh keys
 rm -f ~/.ssh/id_ed25519
 ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""
 
-# we expect RAM shortage
-cat << EOF | sudo tee /etc/ksmtuned.conf > /dev/null
-# /etc/ksmtuned.conf - Configuration file for ksmtuned
-# https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/virtualization_tuning_and_optimization_guide/chap-ksm
-KSM_MONITOR_INTERVAL=60
-
-# Millisecond sleep between ksm scans for 16Gb server.
-# Smaller servers sleep more, bigger sleep less.
-KSM_SLEEP_MSEC=30
-
-KSM_NPAGES_BOOST=0
-KSM_NPAGES_DECAY=0
-KSM_NPAGES_MIN=1000
-KSM_NPAGES_MAX=25000
-
-KSM_THRES_COEF=80
-KSM_THRES_CONST=8192
-
-LOGFILE=/var/log/ksmtuned.log
-DEBUG=1
-EOF
-sudo systemctl restart ksm
-sudo systemctl restart ksmtuned
-
 # not needed
 sudo systemctl stop docker.socket
 sudo systemctl stop multipathd.socket
 
 # remove default swapfile and /mnt
 sudo swapoff -a
 sudo umount -l /mnt
 DISK="/dev/disk/cloud/azure_resource-part1"
 sudo sed -e "s|^$DISK.*||g" -i /etc/fstab
 sudo wipefs -aq $DISK
 sudo systemctl daemon-reload
 
 sudo modprobe loop
 sudo modprobe zfs
 
 # partition the disk as needed
 DISK="/dev/disk/cloud/azure_resource"
 sudo sgdisk --zap-all $DISK
 sudo sgdisk -p \
  -n 1:0:+16G -c 1:"swap" \
  -n 2:0:0    -c 2:"tests" \
 $DISK
 sync
 sleep 1
 
-# swap with same size as RAM
+# swap with same size as RAM (16GiB)
 sudo mkswap $DISK-part1
 sudo swapon $DISK-part1
 
-# 60GB data disk
+# JBOD 2xdisk for OpenZFS storage (test vm's)
 SSD1="$DISK-part2"
-
-# 10GB data disk on ext4
-sudo fallocate -l 10G /test.ssd1
-SSD2=$(sudo losetup -b 4096 -f /test.ssd1 --show)
+sudo fallocate -l 12G /test.ssd2
+SSD2=$(sudo losetup -b 4096 -f /test.ssd2 --show)
 
 # adjust zfs module parameter and create pool
 exec 1>/dev/null
 ARC_MIN=$((1024*1024*256))
 ARC_MAX=$((1024*1024*512))
 echo $ARC_MIN | sudo tee /sys/module/zfs/parameters/zfs_arc_min
 echo $ARC_MAX | sudo tee /sys/module/zfs/parameters/zfs_arc_max
 echo 1 | sudo tee /sys/module/zfs/parameters/zvol_use_blk_mq
-sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 \
-  -O relatime=off -O atime=off -O xattr=sa -O compression=lz4 \
-  -O mountpoint=/mnt/tests
+sudo zpool create -f -o ashift=12 zpool $SSD1 $SSD2 -O relatime=off \
+  -O atime=off -O xattr=sa -O compression=lz4 -O sync=disabled \
+  -O redundant_metadata=none -O mountpoint=/mnt/tests
 
 # no need for some scheduler
 for i in /sys/block/s*/queue/scheduler; do
-  echo "none" | sudo tee $i > /dev/null
+  echo "none" | sudo tee $i
 done
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh
index 39ac92107b71..28da6700e541 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-2-start.sh
@@ -1,225 +1,254 @@
 #!/usr/bin/env bash
 
 ######################################################################
 # 2) start qemu with some operating system, init via cloud-init
 ######################################################################
 
 set -eu
 
 # short name used in zfs-qemu.yml
 OS="$1"
 
 # OS variant (virt-install --os-variant list)
 OSv=$OS
 
 # compressed with .zst extension
 REPO="https://github.com/mcmilk/openzfs-freebsd-images"
-FREEBSD="$REPO/releases/download/v2024-10-05"
+FREEBSD="$REPO/releases/download/v2025-04-13"
 URLzs=""
 
 # Ubuntu mirrors
-#UBMIRROR="https://cloud-images.ubuntu.com"
+UBMIRROR="https://cloud-images.ubuntu.com"
 #UBMIRROR="https://mirrors.cloud.tencent.com/ubuntu-cloud-images"
-UBMIRROR="https://mirror.citrahost.com/ubuntu-cloud-images"
+#UBMIRROR="https://mirror.citrahost.com/ubuntu-cloud-images"
 
 # default nic model for vm's
 NIC="virtio"
 
 case "$OS" in
   almalinux8)
     OSNAME="AlmaLinux 8"
     URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
     ;;
   almalinux9)
     OSNAME="AlmaLinux 9"
     URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
     ;;
+  almalinux10)
+    OSNAME="AlmaLinux 10"
+    OSv="almalinux9"
+    URL="https://repo.almalinux.org/almalinux/10/cloud/x86_64/images/AlmaLinux-10-GenericCloud-latest.x86_64.qcow2"
+    ;;
   archlinux)
     OSNAME="Archlinux"
     URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
-    # dns sometimes fails with that url  :/
-    echo "89.187.191.12  geo.mirror.pkgbuild.com" | sudo tee /etc/hosts > /dev/null
+    ;;
+  centos-stream10)
+    OSNAME="CentOS Stream 10"
+    # TODO: #16903 Overwrite OSv to stream9 for virt-install until it's added to osinfo
+    OSv="centos-stream9"
+    URL="https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2"
     ;;
   centos-stream9)
     OSNAME="CentOS Stream 9"
     URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2"
     ;;
   debian11)
     OSNAME="Debian 11"
     URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2"
     ;;
   debian12)
     OSNAME="Debian 12"
     URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
     ;;
-  fedora40)
-    OSNAME="Fedora 40"
-    OSv="fedora-unknown"
-    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2"
-    ;;
   fedora41)
     OSNAME="Fedora 41"
     OSv="fedora-unknown"
     URL="https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2"
     ;;
-  freebsd13-3r)
-    OSNAME="FreeBSD 13.3-RELEASE"
-    OSv="freebsd13.0"
-    URLzs="$FREEBSD/amd64-freebsd-13.3-RELEASE.qcow2.zst"
-    BASH="/usr/local/bin/bash"
-    NIC="rtl8139"
+  fedora42)
+    OSNAME="Fedora 42"
+    OSv="fedora-unknown"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/42/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-42-1.1.x86_64.qcow2"
     ;;
   freebsd13-4r)
     OSNAME="FreeBSD 13.4-RELEASE"
     OSv="freebsd13.0"
     URLzs="$FREEBSD/amd64-freebsd-13.4-RELEASE.qcow2.zst"
     BASH="/usr/local/bin/bash"
     NIC="rtl8139"
     ;;
-  freebsd14-0r)
-    OSNAME="FreeBSD 14.0-RELEASE"
-    OSv="freebsd14.0"
-    URLzs="$FREEBSD/amd64-freebsd-14.0-RELEASE.qcow2.zst"
+  freebsd13-5r)
+    OSNAME="FreeBSD 13.5-RELEASE"
+    OSv="freebsd13.0"
+    URLzs="$FREEBSD/amd64-freebsd-13.5-RELEASE.qcow2.zst"
     BASH="/usr/local/bin/bash"
+    NIC="rtl8139"
     ;;
   freebsd14-1r)
     OSNAME="FreeBSD 14.1-RELEASE"
     OSv="freebsd14.0"
     URLzs="$FREEBSD/amd64-freebsd-14.1-RELEASE.qcow2.zst"
     BASH="/usr/local/bin/bash"
     ;;
-  freebsd13-4s)
-    OSNAME="FreeBSD 13.4-STABLE"
+  freebsd14-2r)
+    OSNAME="FreeBSD 14.2-RELEASE"
+    OSv="freebsd14.0"
+    URLzs="$FREEBSD/amd64-freebsd-14.2-RELEASE.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd13-5s)
+    OSNAME="FreeBSD 13.5-STABLE"
     OSv="freebsd13.0"
-    URLzs="$FREEBSD/amd64-freebsd-13.4-STABLE.qcow2.zst"
+    URLzs="$FREEBSD/amd64-freebsd-13.5-STABLE.qcow2.zst"
     BASH="/usr/local/bin/bash"
+    NIC="rtl8139"
     ;;
-  freebsd14-1s)
-    OSNAME="FreeBSD 14.1-STABLE"
+  freebsd14-2s)
+    OSNAME="FreeBSD 14.2-STABLE"
     OSv="freebsd14.0"
-    URLzs="$FREEBSD/amd64-freebsd-14.1-STABLE.qcow2.zst"
+    URLzs="$FREEBSD/amd64-freebsd-14.2-STABLE.qcow2.zst"
     BASH="/usr/local/bin/bash"
     ;;
   freebsd15-0c)
     OSNAME="FreeBSD 15.0-CURRENT"
     OSv="freebsd14.0"
     URLzs="$FREEBSD/amd64-freebsd-15.0-CURRENT.qcow2.zst"
     BASH="/usr/local/bin/bash"
     ;;
   tumbleweed)
     OSNAME="openSUSE Tumbleweed"
     OSv="opensusetumbleweed"
     MIRROR="http://opensuse-mirror-gce-us.susecloud.net"
     URL="$MIRROR/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2"
     ;;
-  ubuntu20)
-    OSNAME="Ubuntu 20.04"
-    OSv="ubuntu20.04"
-    URL="$UBMIRROR/focal/current/focal-server-cloudimg-amd64.img"
-    ;;
   ubuntu22)
     OSNAME="Ubuntu 22.04"
     OSv="ubuntu22.04"
     URL="$UBMIRROR/jammy/current/jammy-server-cloudimg-amd64.img"
     ;;
   ubuntu24)
     OSNAME="Ubuntu 24.04"
     OSv="ubuntu24.04"
     URL="$UBMIRROR/noble/current/noble-server-cloudimg-amd64.img"
     ;;
   *)
     echo "Wrong value for OS variable!"
     exit 111
     ;;
 esac
 
 # environment file
 ENV="/var/tmp/env.txt"
 echo "ENV=$ENV" >> $ENV
 
 # result path
 echo 'RESPATH="/var/tmp/test_results"' >> $ENV
 
-# FreeBSD 13 has problems with: e1000+virtio
+# FreeBSD 13 has problems with: e1000 and virtio
 echo "NIC=$NIC" >> $ENV
 
 # freebsd15 -> used in zfs-qemu.yml
 echo "OS=$OS" >> $ENV
 
 # freebsd14.0 -> used for virt-install
 echo "OSv=\"$OSv\"" >> $ENV
 
 # FreeBSD 15 (Current) -> used for summary
 echo "OSNAME=\"$OSNAME\"" >> $ENV
 
+# default vm count for testings
+VMs=2
+echo "VMs=\"$VMs\"" >> $ENV
+
+# default cpu count for testing vm's
+CPU=2
+echo "CPU=\"$CPU\"" >> $ENV
+
 sudo mkdir -p "/mnt/tests"
 sudo chown -R $(whoami) /mnt/tests
 
 # we are downloading via axel, curl and wget are mostly slower and
 # require more return value checking
 IMG="/mnt/tests/cloudimg.qcow2"
 if [ ! -z "$URLzs" ]; then
   echo "Loading image $URLzs ..."
   time axel -q -o "$IMG.zst" "$URLzs"
   zstd -q -d --rm "$IMG.zst"
 else
   echo "Loading image $URL ..."
   time axel -q -o "$IMG" "$URL"
 fi
 
 DISK="/dev/zvol/zpool/openzfs"
 FORMAT="raw"
 sudo zfs create -ps -b 64k -V 80g zpool/openzfs
 while true; do test -b $DISK && break; sleep 1; done
 echo "Importing VM image to zvol..."
 sudo qemu-img dd -f qcow2 -O raw if=$IMG of=$DISK bs=4M
 rm -f $IMG
 
 PUBKEY=$(cat ~/.ssh/id_ed25519.pub)
 cat <<EOF > /tmp/user-data
 #cloud-config
 
 fqdn: $OS
 
 users:
 - name: root
   shell: $BASH
 - name: zfs
   sudo: ALL=(ALL) NOPASSWD:ALL
   shell: $BASH
   ssh_authorized_keys:
     - $PUBKEY
 
 growpart:
   mode: auto
   devices: ['/']
   ignore_growroot_disabled: false
 EOF
 
 sudo virsh net-update default add ip-dhcp-host \
   "<host mac='52:54:00:83:79:00' ip='192.168.122.10'/>" --live --config
 
 sudo virt-install \
   --os-variant $OSv \
   --name "openzfs" \
   --cpu host-passthrough \
   --virt-type=kvm --hvm \
   --vcpus=4,sockets=1 \
   --memory $((1024*12)) \
   --memballoon model=virtio \
   --graphics none \
   --network bridge=virbr0,model=$NIC,mac='52:54:00:83:79:00' \
   --cloud-init user-data=/tmp/user-data \
   --disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
   --import --noautoconsole >/dev/null
 
+# enable KSM on Linux
+if [ ${OS:0:7} != "freebsd" ]; then
+  sudo virsh dommemstat --domain "openzfs" --period 5
+  sudo virsh node-memory-tune 100 50 1
+  echo 1 | sudo tee /sys/kernel/mm/ksm/run > /dev/null
+fi
+
+# Give the VMs hostnames so we don't have to refer to them with
+# hardcoded IP addresses.
+#
+# vm0:          Initial VM we install dependencies and build ZFS on.
+# vm1..2        Testing VMs
+for ((i=0; i<=VMs; i++)); do
+  echo "192.168.122.1$i vm$i" | sudo tee -a /etc/hosts
+done
+
 # in case the directory isn't there already
 mkdir -p $HOME/.ssh
 
 cat <<EOF >> $HOME/.ssh/config
 # no questions please
 StrictHostKeyChecking no
 
 # small timeout, used in while loops later
 ConnectTimeout 1
 EOF
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh
similarity index 83%
copy from sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps.sh
copy to sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh
index 96979cd02e09..a581b13c2f58 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps-vm.sh
@@ -1,229 +1,260 @@
 #!/usr/bin/env bash
 
 ######################################################################
 # 3) install dependencies for compiling and loading
+#
+# $1: OS name (like 'fedora41')
+# $2: (optional) Experimental Fedora kernel version, like "6.14" to
+#     install instead of Fedora defaults.
 ######################################################################
 
 set -eu
 
 function archlinux() {
   echo "##[group]Running pacman -Syu"
   sudo btrfs filesystem resize max /
   sudo pacman -Syu --noconfirm
   echo "##[endgroup]"
 
   echo "##[group]Install Development Tools"
   sudo pacman -Sy --noconfirm base-devel bc cpio cryptsetup dhclient dkms \
     fakeroot fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils \
     parted pax perf python-packaging python-setuptools qemu-guest-agent ksh \
     samba sysstat rng-tools rsync wget xxhash
   echo "##[endgroup]"
 }
 
 function debian() {
   export DEBIAN_FRONTEND="noninteractive"
 
   echo "##[group]Running apt-get update+upgrade"
   sudo apt-get update -y
   sudo apt-get upgrade -y
   echo "##[endgroup]"
 
   echo "##[group]Install Development Tools"
   sudo apt-get install -y \
     acl alien attr autoconf bc cpio cryptsetup curl dbench dh-python dkms \
     fakeroot fio gdb gdebi git ksh lcov isc-dhcp-client jq libacl1-dev \
     libaio-dev libattr1-dev libblkid-dev libcurl4-openssl-dev libdevmapper-dev \
     libelf-dev libffi-dev libmount-dev libpam0g-dev libselinux-dev libssl-dev \
     libtool libtool-bin libudev-dev libunwind-dev linux-headers-$(uname -r) \
     lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \
     python3-cffi python3-dev python3-distlib python3-packaging \
     python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \
     rsync samba sysstat uuid-dev watchdog wget xfslibs-dev  xxhash zlib1g-dev
   echo "##[endgroup]"
 }
 
 function freebsd() {
   export ASSUME_ALWAYS_YES="YES"
 
   echo "##[group]Install Development Tools"
   sudo pkg install -y autoconf automake autotools base64 checkbashisms fio \
     gdb gettext gettext-runtime git gmake gsed jq ksh93 lcov libtool lscpu \
     pkgconf python python3 pamtester pamtester qemu-guest-agent rsync xxhash
   sudo pkg install -xy \
     '^samba4[[:digit:]]+$' \
     '^py3[[:digit:]]+-cffi$' \
     '^py3[[:digit:]]+-sysctl$' \
+    '^py3[[:digit:]]+-setuptools$' \
     '^py3[[:digit:]]+-packaging$'
   echo "##[endgroup]"
 }
 
 # common packages for: almalinux, centos, redhat
 function rhel() {
   echo "##[group]Running dnf update"
   echo "max_parallel_downloads=10" | sudo -E tee -a /etc/dnf/dnf.conf
   sudo dnf clean all
   sudo dnf update -y --setopt=fastestmirror=1 --refresh
   echo "##[endgroup]"
 
   echo "##[group]Install Development Tools"
 
   # Alma wants "Development Tools", Fedora 41 wants "development-tools"
   if ! sudo dnf group install -y "Development Tools" ; then
     echo "Trying 'development-tools' instead of 'Development Tools'"
     sudo dnf group install -y development-tools
   fi
 
   sudo dnf install -y \
     acl attr bc bzip2 cryptsetup curl dbench dkms elfutils-libelf-devel fio \
     gdb git jq kernel-rpm-macros ksh libacl-devel libaio-devel \
     libargon2-devel libattr-devel libblkid-devel libcurl-devel libffi-devel \
     ncompress libselinux-devel libtirpc-devel libtool libudev-devel \
     libuuid-devel lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester \
     parted perf python3 python3-cffi python3-devel python3-packaging \
     kernel-devel python3-setuptools qemu-guest-agent rng-tools rpcgen \
     rpm-build rsync samba sysstat systemd watchdog wget xfsprogs-devel xxhash \
     zlib-devel
   echo "##[endgroup]"
 }
 
 function tumbleweed() {
   echo "##[group]Running zypper is TODO!"
   sleep 23456
   echo "##[endgroup]"
 }
 
+# $1: Kernel version to install (like '6.14rc7')
+function install_fedora_experimental_kernel {
+
+  our_version="$1"
+  sudo dnf -y copr enable @kernel-vanilla/stable
+  sudo dnf -y copr enable @kernel-vanilla/mainline
+  all="$(sudo dnf list --showduplicates kernel-*)"
+  echo "Available versions:"
+  echo "$all"
+
+  # You can have a bunch of minor variants of the version we want '6.14'.
+  # Pick the newest variant (sorted by version number).
+  specific_version=$(echo "$all" | grep $our_version | awk '{print $2}' | sort -V | tail -n 1)
+  list="$(echo "$all" | grep $specific_version | grep -Ev 'kernel-rt|kernel-selftests|kernel-debuginfo' | sed 's/.x86_64//g' | awk '{print $1"-"$2}')"
+  sudo dnf install -y $list
+  sudo dnf -y copr disable @kernel-vanilla/stable
+  sudo dnf -y copr disable @kernel-vanilla/mainline
+}
+
 # Install dependencies
 case "$1" in
   almalinux8)
     echo "##[group]Enable epel and powertools repositories"
     sudo dnf config-manager -y --set-enabled powertools
     sudo dnf install -y epel-release
     echo "##[endgroup]"
     rhel
     echo "##[group]Install kernel-abi-whitelists"
     sudo dnf install -y kernel-abi-whitelists
     echo "##[endgroup]"
     ;;
-  almalinux9|centos-stream9)
+  almalinux9|almalinux10|centos-stream9|centos-stream10)
     echo "##[group]Enable epel and crb repositories"
     sudo dnf config-manager -y --set-enabled crb
     sudo dnf install -y epel-release
     echo "##[endgroup]"
     rhel
     echo "##[group]Install kernel-abi-stablelists"
     sudo dnf install -y kernel-abi-stablelists
     echo "##[endgroup]"
     ;;
   archlinux)
     archlinux
     ;;
   debian*)
     echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections
     debian
     echo "##[group]Install Debian specific"
     sudo apt-get install -yq linux-perf dh-sequence-dkms
     echo "##[endgroup]"
     ;;
   fedora*)
     rhel
+    sudo dnf install -y libunwind-devel
+
+    # Fedora 42+ moves /usr/bin/script from 'util-linux' to 'util-linux-script'
+    sudo dnf install -y util-linux-script || true
+
+    # Optional: Install an experimental kernel ($2 = kernel version)
+    if [ -n "${2:-}" ] ; then
+      install_fedora_experimental_kernel "$2"
+    fi
     ;;
   freebsd*)
     freebsd
     ;;
   tumbleweed)
     tumbleweed
     ;;
   ubuntu*)
     debian
     echo "##[group]Install Ubuntu specific"
     sudo apt-get install -yq linux-tools-common libtirpc-dev \
       linux-modules-extra-$(uname -r)
-    if [ "$1" != "ubuntu20" ]; then
-      sudo apt-get install -yq dh-sequence-dkms
-    fi
+    sudo apt-get install -yq dh-sequence-dkms
     echo "##[endgroup]"
     echo "##[group]Delete Ubuntu OpenZFS modules"
     for i in $(find /lib/modules -name zfs -type d); do sudo rm -rvf $i; done
     echo "##[endgroup]"
     ;;
 esac
 
 # This script is used for checkstyle + zloop deps also.
 # Install only the needed packages and exit - when used this way.
 test -z "${ONLY_DEPS:-}" || exit 0
 
 # Start services
 echo "##[group]Enable services"
 case "$1" in
   freebsd*)
     # add virtio things
     echo 'virtio_load="YES"' | sudo -E tee -a /boot/loader.conf
     for i in balloon blk console random scsi; do
       echo "virtio_${i}_load=\"YES\"" | sudo -E tee -a /boot/loader.conf
     done
     echo "fdescfs /dev/fd fdescfs rw 0 0" | sudo -E tee -a /etc/fstab
     sudo -E mount /dev/fd
     sudo -E touch /etc/zfs/exports
     sudo -E sysrc mountd_flags="/etc/zfs/exports"
     echo '[global]' | sudo -E tee /usr/local/etc/smb4.conf >/dev/null
     sudo -E service nfsd enable
     sudo -E service qemu-guest-agent enable
     sudo -E service samba_server enable
     ;;
   debian*|ubuntu*)
     sudo -E systemctl enable nfs-kernel-server
     sudo -E systemctl enable qemu-guest-agent
     sudo -E systemctl enable smbd
     ;;
   *)
     # All other linux distros
     sudo -E systemctl enable nfs-server
     sudo -E systemctl enable qemu-guest-agent
     sudo -E systemctl enable smb
     ;;
 esac
 echo "##[endgroup]"
 
 # Setup Kernel cmdline
 CMDLINE="console=tty0 console=ttyS0,115200n8"
 CMDLINE="$CMDLINE selinux=0"
 CMDLINE="$CMDLINE random.trust_cpu=on"
 CMDLINE="$CMDLINE no_timer_check"
 case "$1" in
   almalinux*|centos*|fedora*)
     GRUB_CFG="/boot/grub2/grub.cfg"
     GRUB_MKCONFIG="grub2-mkconfig"
     CMDLINE="$CMDLINE biosdevname=0 net.ifnames=0"
     echo 'GRUB_SERIAL_COMMAND="serial --speed=115200"' \
       | sudo tee -a /etc/default/grub >/dev/null
     ;;
   ubuntu24)
     GRUB_CFG="/boot/grub/grub.cfg"
     GRUB_MKCONFIG="grub-mkconfig"
     echo 'GRUB_DISABLE_OS_PROBER="false"' \
       | sudo tee -a /etc/default/grub >/dev/null
     ;;
   *)
     GRUB_CFG="/boot/grub/grub.cfg"
     GRUB_MKCONFIG="grub-mkconfig"
     ;;
 esac
 
 case "$1" in
   archlinux|freebsd*)
     true
     ;;
   *)
     echo "##[group]Edit kernel cmdline"
     sudo sed -i -e '/^GRUB_CMDLINE_LINUX/d' /etc/default/grub || true
     echo "GRUB_CMDLINE_LINUX=\"$CMDLINE\"" \
       | sudo tee -a /etc/default/grub >/dev/null
     sudo $GRUB_MKCONFIG -o $GRUB_CFG
     echo "##[endgroup]"
     ;;
 esac
 
 # reset cloud-init configuration and poweroff
 sudo cloud-init clean --logs
 sleep 2 && sudo poweroff &
 exit 0
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps.sh
index 96979cd02e09..267ae4ad3c7b 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-3-deps.sh
@@ -1,229 +1,28 @@
-#!/usr/bin/env bash
-
 ######################################################################
-# 3) install dependencies for compiling and loading
+# 3) Wait for VM to boot from previous step and launch dependencies
+#    script on it.
+#
+# $1: OS name (like 'fedora41')
+# $2: (optional) Experimental kernel version to install on fedora,
+#     like "6.14".
 ######################################################################
 
-set -eu
-
-function archlinux() {
-  echo "##[group]Running pacman -Syu"
-  sudo btrfs filesystem resize max /
-  sudo pacman -Syu --noconfirm
-  echo "##[endgroup]"
-
-  echo "##[group]Install Development Tools"
-  sudo pacman -Sy --noconfirm base-devel bc cpio cryptsetup dhclient dkms \
-    fakeroot fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils \
-    parted pax perf python-packaging python-setuptools qemu-guest-agent ksh \
-    samba sysstat rng-tools rsync wget xxhash
-  echo "##[endgroup]"
-}
-
-function debian() {
-  export DEBIAN_FRONTEND="noninteractive"
-
-  echo "##[group]Running apt-get update+upgrade"
-  sudo apt-get update -y
-  sudo apt-get upgrade -y
-  echo "##[endgroup]"
-
-  echo "##[group]Install Development Tools"
-  sudo apt-get install -y \
-    acl alien attr autoconf bc cpio cryptsetup curl dbench dh-python dkms \
-    fakeroot fio gdb gdebi git ksh lcov isc-dhcp-client jq libacl1-dev \
-    libaio-dev libattr1-dev libblkid-dev libcurl4-openssl-dev libdevmapper-dev \
-    libelf-dev libffi-dev libmount-dev libpam0g-dev libselinux-dev libssl-dev \
-    libtool libtool-bin libudev-dev libunwind-dev linux-headers-$(uname -r) \
-    lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \
-    python3-cffi python3-dev python3-distlib python3-packaging \
-    python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \
-    rsync samba sysstat uuid-dev watchdog wget xfslibs-dev  xxhash zlib1g-dev
-  echo "##[endgroup]"
-}
-
-function freebsd() {
-  export ASSUME_ALWAYS_YES="YES"
-
-  echo "##[group]Install Development Tools"
-  sudo pkg install -y autoconf automake autotools base64 checkbashisms fio \
-    gdb gettext gettext-runtime git gmake gsed jq ksh93 lcov libtool lscpu \
-    pkgconf python python3 pamtester pamtester qemu-guest-agent rsync xxhash
-  sudo pkg install -xy \
-    '^samba4[[:digit:]]+$' \
-    '^py3[[:digit:]]+-cffi$' \
-    '^py3[[:digit:]]+-sysctl$' \
-    '^py3[[:digit:]]+-packaging$'
-  echo "##[endgroup]"
-}
-
-# common packages for: almalinux, centos, redhat
-function rhel() {
-  echo "##[group]Running dnf update"
-  echo "max_parallel_downloads=10" | sudo -E tee -a /etc/dnf/dnf.conf
-  sudo dnf clean all
-  sudo dnf update -y --setopt=fastestmirror=1 --refresh
-  echo "##[endgroup]"
-
-  echo "##[group]Install Development Tools"
-
-  # Alma wants "Development Tools", Fedora 41 wants "development-tools"
-  if ! sudo dnf group install -y "Development Tools" ; then
-    echo "Trying 'development-tools' instead of 'Development Tools'"
-    sudo dnf group install -y development-tools
-  fi
-
-  sudo dnf install -y \
-    acl attr bc bzip2 cryptsetup curl dbench dkms elfutils-libelf-devel fio \
-    gdb git jq kernel-rpm-macros ksh libacl-devel libaio-devel \
-    libargon2-devel libattr-devel libblkid-devel libcurl-devel libffi-devel \
-    ncompress libselinux-devel libtirpc-devel libtool libudev-devel \
-    libuuid-devel lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester \
-    parted perf python3 python3-cffi python3-devel python3-packaging \
-    kernel-devel python3-setuptools qemu-guest-agent rng-tools rpcgen \
-    rpm-build rsync samba sysstat systemd watchdog wget xfsprogs-devel xxhash \
-    zlib-devel
-  echo "##[endgroup]"
-}
-
-function tumbleweed() {
-  echo "##[group]Running zypper is TODO!"
-  sleep 23456
-  echo "##[endgroup]"
-}
-
-# Install dependencies
-case "$1" in
-  almalinux8)
-    echo "##[group]Enable epel and powertools repositories"
-    sudo dnf config-manager -y --set-enabled powertools
-    sudo dnf install -y epel-release
-    echo "##[endgroup]"
-    rhel
-    echo "##[group]Install kernel-abi-whitelists"
-    sudo dnf install -y kernel-abi-whitelists
-    echo "##[endgroup]"
-    ;;
-  almalinux9|centos-stream9)
-    echo "##[group]Enable epel and crb repositories"
-    sudo dnf config-manager -y --set-enabled crb
-    sudo dnf install -y epel-release
-    echo "##[endgroup]"
-    rhel
-    echo "##[group]Install kernel-abi-stablelists"
-    sudo dnf install -y kernel-abi-stablelists
-    echo "##[endgroup]"
-    ;;
-  archlinux)
-    archlinux
-    ;;
-  debian*)
-    echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections
-    debian
-    echo "##[group]Install Debian specific"
-    sudo apt-get install -yq linux-perf dh-sequence-dkms
-    echo "##[endgroup]"
-    ;;
-  fedora*)
-    rhel
-    ;;
-  freebsd*)
-    freebsd
-    ;;
-  tumbleweed)
-    tumbleweed
-    ;;
-  ubuntu*)
-    debian
-    echo "##[group]Install Ubuntu specific"
-    sudo apt-get install -yq linux-tools-common libtirpc-dev \
-      linux-modules-extra-$(uname -r)
-    if [ "$1" != "ubuntu20" ]; then
-      sudo apt-get install -yq dh-sequence-dkms
-    fi
-    echo "##[endgroup]"
-    echo "##[group]Delete Ubuntu OpenZFS modules"
-    for i in $(find /lib/modules -name zfs -type d); do sudo rm -rvf $i; done
-    echo "##[endgroup]"
-    ;;
-esac
-
-# This script is used for checkstyle + zloop deps also.
-# Install only the needed packages and exit - when used this way.
-test -z "${ONLY_DEPS:-}" || exit 0
-
-# Start services
-echo "##[group]Enable services"
-case "$1" in
-  freebsd*)
-    # add virtio things
-    echo 'virtio_load="YES"' | sudo -E tee -a /boot/loader.conf
-    for i in balloon blk console random scsi; do
-      echo "virtio_${i}_load=\"YES\"" | sudo -E tee -a /boot/loader.conf
-    done
-    echo "fdescfs /dev/fd fdescfs rw 0 0" | sudo -E tee -a /etc/fstab
-    sudo -E mount /dev/fd
-    sudo -E touch /etc/zfs/exports
-    sudo -E sysrc mountd_flags="/etc/zfs/exports"
-    echo '[global]' | sudo -E tee /usr/local/etc/smb4.conf >/dev/null
-    sudo -E service nfsd enable
-    sudo -E service qemu-guest-agent enable
-    sudo -E service samba_server enable
-    ;;
-  debian*|ubuntu*)
-    sudo -E systemctl enable nfs-kernel-server
-    sudo -E systemctl enable qemu-guest-agent
-    sudo -E systemctl enable smbd
-    ;;
-  *)
-    # All other linux distros
-    sudo -E systemctl enable nfs-server
-    sudo -E systemctl enable qemu-guest-agent
-    sudo -E systemctl enable smb
-    ;;
-esac
-echo "##[endgroup]"
-
-# Setup Kernel cmdline
-CMDLINE="console=tty0 console=ttyS0,115200n8"
-CMDLINE="$CMDLINE selinux=0"
-CMDLINE="$CMDLINE random.trust_cpu=on"
-CMDLINE="$CMDLINE no_timer_check"
-case "$1" in
-  almalinux*|centos*|fedora*)
-    GRUB_CFG="/boot/grub2/grub.cfg"
-    GRUB_MKCONFIG="grub2-mkconfig"
-    CMDLINE="$CMDLINE biosdevname=0 net.ifnames=0"
-    echo 'GRUB_SERIAL_COMMAND="serial --speed=115200"' \
-      | sudo tee -a /etc/default/grub >/dev/null
-    ;;
-  ubuntu24)
-    GRUB_CFG="/boot/grub/grub.cfg"
-    GRUB_MKCONFIG="grub-mkconfig"
-    echo 'GRUB_DISABLE_OS_PROBER="false"' \
-      | sudo tee -a /etc/default/grub >/dev/null
-    ;;
-  *)
-    GRUB_CFG="/boot/grub/grub.cfg"
-    GRUB_MKCONFIG="grub-mkconfig"
-    ;;
-esac
-
-case "$1" in
-  archlinux|freebsd*)
-    true
-    ;;
-  *)
-    echo "##[group]Edit kernel cmdline"
-    sudo sed -i -e '/^GRUB_CMDLINE_LINUX/d' /etc/default/grub || true
-    echo "GRUB_CMDLINE_LINUX=\"$CMDLINE\"" \
-      | sudo tee -a /etc/default/grub >/dev/null
-    sudo $GRUB_MKCONFIG -o $GRUB_CFG
-    echo "##[endgroup]"
-    ;;
-esac
-
-# reset cloud-init configuration and poweroff
-sudo cloud-init clean --logs
-sleep 2 && sudo poweroff &
-exit 0
+.github/workflows/scripts/qemu-wait-for-vm.sh vm0
+
+# SPECIAL CASE:
+#
+# If the user passed in an experimental kernel version to test on Fedora,
+# we need to update the kernel version in zfs's META file to allow the
+# build to happen.  We update our local copy of META here, since we know
+# it will be rsync'd up in the next step.
+if [ -n "${2:-}" ] ; then
+  sed -i -E 's/Linux-Maximum: .+/Linux-Maximum: 99.99/g' META
+fi
+
+scp .github/workflows/scripts/qemu-3-deps-vm.sh zfs@vm0:qemu-3-deps-vm.sh
+PID=`pidof /usr/bin/qemu-system-x86_64`
+ssh zfs@vm0 '$HOME/qemu-3-deps-vm.sh' "$@"
+# wait for poweroff to succeed
+tail --pid=$PID -f /dev/null
+sleep 5 # avoid this: "error: Domain is already active"
+rm -f $HOME/.ssh/known_hosts
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh
new file mode 100755
index 000000000000..17e976ebcc39
--- /dev/null
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build-vm.sh
@@ -0,0 +1,379 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 4) configure and build openzfs modules.  This is run on the VMs.
+#
+# Usage:
+#
+#       qemu-4-build-vm.sh OS [--enable-debug][--dkms][--poweroff]
+#           [--release][--repo][--tarball]
+#
+# OS:           OS name like 'fedora41'
+# --enable-debug:  Build RPMs with '--enable-debug' (for testing)
+# --dkms:       Build DKMS RPMs as well
+# --poweroff:   Power-off the VM after building
+# --release     Build zfs-release*.rpm as well
+# --repo        After building everything, copy RPMs into /tmp/repo
+#               in the ZFS RPM repository file structure.  Also
+#               copy tarballs if they were built.
+# --tarball:    Also build a tarball of ZFS source
+######################################################################
+
+ENABLE_DEBUG=""
+DKMS=""
+POWEROFF=""
+RELEASE=""
+REPO=""
+TARBALL=""
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --enable-debug)
+      ENABLE_DEBUG=1
+      shift
+      ;;
+    --dkms)
+      DKMS=1
+      shift
+      ;;
+    --poweroff)
+      POWEROFF=1
+      shift
+      ;;
+    --release)
+      RELEASE=1
+      shift
+      ;;
+    --repo)
+      REPO=1
+      shift
+      ;;
+    --tarball)
+      TARBALL=1
+      shift
+      ;;
+    *)
+      OS=$1
+      shift
+      ;;
+  esac
+done
+
+set -eu
+
+function run() {
+  LOG="/var/tmp/build-stderr.txt"
+  echo "****************************************************"
+  echo "$(date) ($*)"
+  echo "****************************************************"
+  ($@ || echo $? > /tmp/rv) 3>&1 1>&2 2>&3 | stdbuf -eL -oL tee -a $LOG
+  if [ -f /tmp/rv ]; then
+    RV=$(cat /tmp/rv)
+    echo "****************************************************"
+    echo "exit with value=$RV ($*)"
+    echo "****************************************************"
+    echo 1 > /var/tmp/build-exitcode.txt
+    exit $RV
+  fi
+}
+
+# Look at the RPMs in the current directory and copy/move them to
+# /tmp/repo, using the directory structure we use for the ZFS RPM repos.
+#
+# For example:
+# /tmp/repo/epel-testing/9.5
+# /tmp/repo/epel-testing/9.5/SRPMS
+# /tmp/repo/epel-testing/9.5/SRPMS/zfs-2.3.99-1.el9.src.rpm
+# /tmp/repo/epel-testing/9.5/SRPMS/zfs-kmod-2.3.99-1.el9.src.rpm
+# /tmp/repo/epel-testing/9.5/kmod
+# /tmp/repo/epel-testing/9.5/kmod/x86_64
+# /tmp/repo/epel-testing/9.5/kmod/x86_64/debug
+# /tmp/repo/epel-testing/9.5/kmod/x86_64/debug/kmod-zfs-debuginfo-2.3.99-1.el9.x86_64.rpm
+# /tmp/repo/epel-testing/9.5/kmod/x86_64/debug/libnvpair3-debuginfo-2.3.99-1.el9.x86_64.rpm
+# /tmp/repo/epel-testing/9.5/kmod/x86_64/debug/libuutil3-debuginfo-2.3.99-1.el9.x86_64.rpm
+# ...
+function copy_rpms_to_repo {
+  # Pick a RPM to query. It doesn't matter which one - we just want to extract
+  # the 'Build Host' value from it.
+  rpm=$(ls zfs-*.rpm | head -n 1)
+
+  # Get zfs version '2.2.99'
+  zfs_ver=$(rpm -qpi $rpm | awk '/Version/{print $3}')
+
+  # Get "2.1" or "2.2"
+  zfs_major=$(echo $zfs_ver | grep -Eo [0-9]+\.[0-9]+)
+
+  # Get 'almalinux9.5' or 'fedora41' type string
+  build_host=$(rpm -qpi $rpm | awk '/Build Host/{print $4}')
+
+  # Get '9.5' or '41' OS version
+  os_ver=$(echo $build_host | grep -Eo '[0-9\.]+$')
+
+  # Our ZFS version and OS name will determine which repo the RPMs
+  # will go in (regular or testing).  Fedora always gets the newest
+  # releases, and Alma gets the older releases.
+  case $build_host in
+  almalinux*)
+    case $zfs_major in
+    2.2)
+      d="epel"
+      ;;
+    *)
+      d="epel-testing"
+      ;;
+    esac
+    ;;
+  fedora*)
+    d="fedora"
+    ;;
+  esac
+
+  prefix=/tmp/repo
+  dst="$prefix/$d/$os_ver"
+
+  # Special case: move zfs-release*.rpm out of the way first (if we built them).
+  # This will make filtering the other RPMs easier.
+  mkdir -p $dst
+  mv zfs-release*.rpm $dst || true
+
+  # Copy source RPMs
+  mkdir -p $dst/SRPMS
+  cp $(ls *.src.rpm) $dst/SRPMS/
+
+  if [[ "$build_host" =~ "almalinux" ]] ; then
+    # Copy kmods+userspace
+    mkdir -p $dst/kmod/x86_64/debug
+    cp $(ls *.rpm | grep -Ev 'src.rpm|dkms|debuginfo') $dst/kmod/x86_64
+    cp *debuginfo*.rpm $dst/kmod/x86_64/debug
+  fi
+
+  if [ -n "$DKMS" ] ; then
+    # Copy dkms+userspace
+    mkdir -p $dst/x86_64
+    cp $(ls *.rpm | grep -Ev 'src.rpm|kmod|debuginfo') $dst/x86_64
+  fi
+
+  # Copy debug
+  mkdir -p $dst/x86_64/debug
+  cp $(ls *debuginfo*.rpm | grep -v kmod) $dst/x86_64/debug
+}
+
+function freebsd() {
+  extra="${1:-}"
+
+  export MAKE="gmake"
+  echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure \
+    --prefix=/usr/local \
+    --with-libintl-prefix=/usr/local \
+    --enable-pyzfs \
+    --enable-debuginfo $extra
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run gmake -j$(sysctl -n hw.ncpu)
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  run sudo gmake install
+  echo "##[endgroup]"
+}
+
+function linux() {
+  extra="${1:-}"
+
+  echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure \
+    --prefix=/usr \
+    --enable-pyzfs \
+    --enable-debuginfo $extra
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run make -j$(nproc)
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  run sudo make install
+  echo "##[endgroup]"
+}
+
+function rpm_build_and_install() {
+  extra="${1:-}"
+
+  # Build RPMs with XZ compression by default (since gzip decompression is slow)
+  echo "%_binary_payload w7.xzdio" >> ~/.rpmmacros
+
+  echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure --enable-debuginfo $extra
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run make pkg-kmod pkg-utils
+  echo "##[endgroup]"
+
+  if [ -n "$DKMS" ] ; then
+    echo "##[group]DKMS"
+    make rpm-dkms
+    echo "##[endgroup]"
+  fi
+
+  if [ -n "$REPO" ] ; then
+    echo "Skipping install since we're only building RPMs and nothing else"
+  else
+    echo "##[group]Install"
+    run sudo dnf -y --nobest install $(ls *.rpm | grep -Ev 'dkms|src.rpm')
+    echo "##[endgroup]"
+  fi
+
+  # Optionally build the zfs-release.*.rpm
+  if [ -n "$RELEASE" ] ; then
+    echo "##[group]Release"
+    pushd ~
+    sudo dnf -y install rpm-build || true
+    # Check out a sparse copy of zfsonlinux.github.com.git so we don't get
+    # all the binaries.  We just need a few kilobytes of files to build RPMs.
+    git clone --depth 1 --no-checkout \
+      https://github.com/zfsonlinux/zfsonlinux.github.com.git
+
+    cd zfsonlinux.github.com
+    git sparse-checkout set zfs-release
+    git checkout
+    cd zfs-release
+
+    mkdir -p ~/rpmbuild/{BUILDROOT,SPECS,RPMS,SRPMS,SOURCES,BUILD}
+    cp RPM-GPG-KEY-openzfs* *.repo ~/rpmbuild/SOURCES
+    cp zfs-release.spec ~/rpmbuild/SPECS/
+    rpmbuild -ba ~/rpmbuild/SPECS/zfs-release.spec
+
+    # ZFS release RPMs are built.  Copy them to the ~/zfs directory just to
+    # keep all the RPMs in the same place.
+    cp ~/rpmbuild/RPMS/noarch/*.rpm ~/zfs
+    cp ~/rpmbuild/SRPMS/*.rpm ~/zfs
+
+    popd
+    rm -fr ~/rpmbuild
+    echo "##[endgroup]"
+  fi
+
+  if [ -n "$REPO" ] ; then
+    echo "##[group]Repo"
+    copy_rpms_to_repo
+    echo "##[endgroup]"
+  fi
+}
+
+function deb_build_and_install() {
+  extra="${1:-}"
+
+  echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure \
+    --prefix=/usr \
+    --enable-pyzfs \
+    --enable-debuginfo $extra
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run make native-deb-kmod native-deb-utils
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  # Do kmod install.  Note that when you build the native debs, the
+  # packages themselves are placed in parent directory '../' rather than
+  # in the source directory like the rpms are.
+  run sudo apt-get -y install $(find ../ | grep -E '\.deb$' \
+    | grep -Ev 'dkms|dracut')
+  echo "##[endgroup]"
+}
+
+function build_tarball {
+  if [ -n "$REPO" ] ; then
+    ./autogen.sh
+    ./configure --with-config=srpm
+    make dist
+    mkdir -p /tmp/repo/releases
+    # The tarball name is based off of 'Version' field in the META file.
+    mv *.tar.gz /tmp/repo/releases/
+  fi
+}
+
+# Debug: show kernel cmdline
+if [ -f /proc/cmdline ] ; then
+  cat /proc/cmdline || true
+fi
+
+# Set our hostname to our OS name and version number.  Specifically, we set the
+# major and minor number so that when we query the Build Host field in the RPMs
+# we build, we can see what specific version of Fedora/Almalinux we were using
+# to build them.  This is helpful for matching up KMOD versions.
+#
+# Examples:
+#
+# rhel8.10
+# almalinux9.5
+# fedora42
+source /etc/os-release
+sudo hostname "$ID$VERSION_ID"
+
+# save some sysinfo
+uname -a > /var/tmp/uname.txt
+
+cd $HOME/zfs
+export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"
+
+extra=""
+if [ -n "$ENABLE_DEBUG" ] ; then
+  extra="--enable-debug"
+fi
+
+# build
+case "$OS" in
+  freebsd*)
+    freebsd "$extra"
+    ;;
+  alma*|centos*)
+    rpm_build_and_install "--with-spec=redhat $extra"
+    ;;
+  fedora*)
+    rpm_build_and_install "$extra"
+
+    # Historically, we've always built the release tarballs on Fedora, since
+    # there was one instance long ago where we built them on CentOS 7, and they
+    # didn't work correctly for everyone.
+    if [ -n "$TARBALL" ] ; then
+        build_tarball
+    fi
+    ;;
+  debian*|ubuntu*)
+    deb_build_and_install "$extra"
+    ;;
+  *)
+    linux "$extra"
+    ;;
+esac
+
+
+# building the zfs module was ok
+echo 0 > /var/tmp/build-exitcode.txt
+
+# reset cloud-init configuration and poweroff
+if [ -n "$POWEROFF" ] ; then
+        sudo cloud-init clean --logs
+        sync && sleep 2 && sudo poweroff &
+fi
+exit 0
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build.sh
index 955f605f5bce..63c9bccaa446 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-4-build.sh
@@ -1,153 +1,11 @@
 #!/usr/bin/env bash
 
 ######################################################################
 # 4) configure and build openzfs modules
 ######################################################################
+echo "Build modules in QEMU machine"
 
-set -eu
+# Bring our VM back up and copy over ZFS source
+.github/workflows/scripts/qemu-prepare-for-build.sh
 
-function run() {
-  LOG="/var/tmp/build-stderr.txt"
-  echo "****************************************************"
-  echo "$(date) ($*)"
-  echo "****************************************************"
-  ($@ || echo $? > /tmp/rv) 3>&1 1>&2 2>&3 | stdbuf -eL -oL tee -a $LOG
-  if [ -f /tmp/rv ]; then
-    RV=$(cat /tmp/rv)
-    echo "****************************************************"
-    echo "exit with value=$RV ($*)"
-    echo "****************************************************"
-    echo 1 > /var/tmp/build-exitcode.txt
-    exit $RV
-  fi
-}
-
-function freebsd() {
-  export MAKE="gmake"
-  echo "##[group]Autogen.sh"
-  run ./autogen.sh
-  echo "##[endgroup]"
-
-  echo "##[group]Configure"
-  run ./configure \
-    --prefix=/usr/local \
-    --with-libintl-prefix=/usr/local \
-    --enable-pyzfs \
-    --enable-debug \
-    --enable-debuginfo
-  echo "##[endgroup]"
-
-  echo "##[group]Build"
-  run gmake -j$(sysctl -n hw.ncpu)
-  echo "##[endgroup]"
-
-  echo "##[group]Install"
-  run sudo gmake install
-  echo "##[endgroup]"
-}
-
-function linux() {
-  echo "##[group]Autogen.sh"
-  run ./autogen.sh
-  echo "##[endgroup]"
-
-  echo "##[group]Configure"
-  run ./configure \
-    --prefix=/usr \
-    --enable-pyzfs \
-    --enable-debug \
-    --enable-debuginfo
-  echo "##[endgroup]"
-
-  echo "##[group]Build"
-  run make -j$(nproc)
-  echo "##[endgroup]"
-
-  echo "##[group]Install"
-  run sudo make install
-  echo "##[endgroup]"
-}
-
-function rpm_build_and_install() {
-  EXTRA_CONFIG="${1:-}"
-  echo "##[group]Autogen.sh"
-  run ./autogen.sh
-  echo "##[endgroup]"
-
-  echo "##[group]Configure"
-  run ./configure --enable-debug --enable-debuginfo $EXTRA_CONFIG
-  echo "##[endgroup]"
-
-  echo "##[group]Build"
-  run make pkg-kmod pkg-utils
-  echo "##[endgroup]"
-
-  echo "##[group]Install"
-  run sudo dnf -y --nobest install $(ls *.rpm | grep -v src.rpm)
-  echo "##[endgroup]"
-
-}
-
-function deb_build_and_install() {
-echo "##[group]Autogen.sh"
-  run ./autogen.sh
-  echo "##[endgroup]"
-
-  echo "##[group]Configure"
-  run ./configure \
-    --prefix=/usr \
-    --enable-pyzfs \
-    --enable-debug \
-    --enable-debuginfo
-  echo "##[endgroup]"
-
-  echo "##[group]Build"
-  run make native-deb-kmod native-deb-utils
-  echo "##[endgroup]"
-
-  echo "##[group]Install"
-  # Do kmod install.  Note that when you build the native debs, the
-  # packages themselves are placed in parent directory '../' rather than
-  # in the source directory like the rpms are.
-  run sudo apt-get -y install $(find ../ | grep -E '\.deb$' \
-    | grep -Ev 'dkms|dracut')
-  echo "##[endgroup]"
-}
-
-# Debug: show kernel cmdline
-if [ -f /proc/cmdline ] ; then
-  cat /proc/cmdline || true
-fi
-
-# save some sysinfo
-uname -a > /var/tmp/uname.txt
-
-cd $HOME/zfs
-export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"
-
-# build
-case "$1" in
-  freebsd*)
-    freebsd
-    ;;
-  alma*|centos*)
-    rpm_build_and_install "--with-spec=redhat"
-    ;;
-  fedora*)
-    rpm_build_and_install
-    ;;
-  debian*|ubuntu*)
-    deb_build_and_install
-    ;;
-  *)
-    linux
-    ;;
-esac
-
-# building the zfs module was ok
-echo 0 > /var/tmp/build-exitcode.txt
-
-# reset cloud-init configuration and poweroff
-sudo cloud-init clean --logs
-sync && sleep 2 && sudo poweroff &
-exit 0
+ssh zfs@vm0 '$HOME/zfs/.github/workflows/scripts/qemu-4-build-vm.sh' $@
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh
index bc40e8894b22..6bf10024a1a6 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-5-setup.sh
@@ -1,126 +1,116 @@
 #!/usr/bin/env bash
 
 ######################################################################
 # 5) start test machines and load openzfs module
 ######################################################################
 
 set -eu
 
 # read our defined variables
 source /var/tmp/env.txt
 
 # wait for poweroff to succeed
 PID=$(pidof /usr/bin/qemu-system-x86_64)
 tail --pid=$PID -f /dev/null
 sudo virsh undefine openzfs
 
-# default values per test vm:
-VMs=2
-CPU=2
-
 # cpu pinning
 CPUSET=("0,1" "2,3")
 
 case "$OS" in
   freebsd*)
-    # FreeBSD can't be optimized via ksmtuned
+    # FreeBSD needs only 6GiB
     RAM=6
     ;;
   *)
-    # Linux can be optimized via ksmtuned
+    # Linux needs more memory, but can be optimized to share it via KSM
     RAM=8
     ;;
 esac
 
-# this can be different for each distro
-echo "VMs=$VMs" >> $ENV
-
 # create snapshot we can clone later
 sudo zfs snapshot zpool/openzfs@now
 
 # setup the testing vm's
 PUBKEY=$(cat ~/.ssh/id_ed25519.pub)
-for i in $(seq 1 $VMs); do
 
+# start testing VMs
+for ((i=1; i<=VMs; i++)); do
   echo "Creating disk for vm$i..."
   DISK="/dev/zvol/zpool/vm$i"
   FORMAT="raw"
-  sudo zfs clone zpool/openzfs@now zpool/vm$i
-  sudo zfs create -ps -b 64k -V 80g zpool/vm$i-2
+  sudo zfs clone zpool/openzfs@now zpool/vm$i-system
+  sudo zfs create -ps -b 64k -V 64g zpool/vm$i-tests
 
   cat <<EOF > /tmp/user-data
 #cloud-config
 
 fqdn: vm$i
 
 users:
 - name: root
   shell: $BASH
 - name: zfs
   sudo: ALL=(ALL) NOPASSWD:ALL
   shell: $BASH
   ssh_authorized_keys:
     - $PUBKEY
 
 growpart:
   mode: auto
   devices: ['/']
   ignore_growroot_disabled: false
 EOF
 
   sudo virsh net-update default add ip-dhcp-host \
     "<host mac='52:54:00:83:79:0$i' ip='192.168.122.1$i'/>" --live --config
 
   sudo virt-install \
     --os-variant $OSv \
     --name "vm$i" \
     --cpu host-passthrough \
     --virt-type=kvm --hvm \
     --vcpus=$CPU,sockets=1 \
     --cpuset=${CPUSET[$((i-1))]} \
     --memory $((1024*RAM)) \
     --memballoon model=virtio \
     --graphics none \
     --cloud-init user-data=/tmp/user-data \
     --network bridge=virbr0,model=$NIC,mac="52:54:00:83:79:0$i" \
-    --disk $DISK,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
-    --disk $DISK-2,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
+    --disk $DISK-system,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
+    --disk $DISK-tests,bus=virtio,cache=none,format=$FORMAT,driver.discard=unmap \
     --import --noautoconsole >/dev/null
 done
 
-# check the memory state from time to time
+# generate some memory stats
 cat <<EOF > cronjob.sh
-# $OS
 exec 1>>/var/tmp/stats.txt
 exec 2>&1
-echo "*******************************************************"
-date
+echo "********************************************************************************"
 uptime
 free -m
-df -h /mnt/tests
 zfs list
 EOF
+
 sudo chmod +x cronjob.sh
 sudo mv -f cronjob.sh /root/cronjob.sh
 echo '*/5 * * * *  /root/cronjob.sh' > crontab.txt
 sudo crontab crontab.txt
 rm crontab.txt
 
 # check if the machines are okay
 echo "Waiting for vm's to come up...  (${VMs}x CPU=$CPU RAM=$RAM)"
-for i in $(seq 1 $VMs); do
-  while true; do
-    ssh 2>/dev/null zfs@192.168.122.1$i "uname -a" && break
-  done
+for ((i=1; i<=VMs; i++)); do
+  .github/workflows/scripts/qemu-wait-for-vm.sh vm$i
 done
 echo "All $VMs VMs are up now."
 
 # Save the VM's serial output (ttyS0) to /var/tmp/console.txt
 # - ttyS0 on the VM corresponds to a local /dev/pty/N entry
 # - use 'virsh ttyconsole' to lookup the /dev/pty/N entry
-for i in $(seq 1 $VMs); do
+for ((i=1; i<=VMs; i++)); do
   mkdir -p $RESPATH/vm$i
   read "pty" <<< $(sudo virsh ttyconsole vm$i)
   sudo nohup bash -c "cat $pty > $RESPATH/vm$i/console.txt" &
 done
 echo "Console logging for ${VMs}x $OS started."
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh
index 2f023198bbf6..e8e6adecd62f 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-6-tests.sh
@@ -1,105 +1,117 @@
 #!/usr/bin/env bash
 
 ######################################################################
 # 6) load openzfs module and run the tests
 #
 # called on runner:  qemu-6-tests.sh
 # called on qemu-vm: qemu-6-tests.sh $OS $2/$3
 ######################################################################
 
 set -eu
 
 function prefix() {
   ID="$1"
   LINE="$2"
   CURRENT=$(date +%s)
   TSSTART=$(cat /tmp/tsstart)
   DIFF=$((CURRENT-TSSTART))
   H=$((DIFF/3600))
   DIFF=$((DIFF-(H*3600)))
   M=$((DIFF/60))
   S=$((DIFF-(M*60)))
 
   CTR=$(cat /tmp/ctr)
   echo $LINE| grep -q "^Test[: ]" && CTR=$((CTR+1)) && echo $CTR > /tmp/ctr
 
   BASE="$HOME/work/zfs/zfs"
   COLOR="$BASE/scripts/zfs-tests-color.sh"
   CLINE=$(echo $LINE| grep "^Test[ :]" | sed -e 's|/usr/local|/usr|g' \
     | sed -e 's| /usr/share/zfs/zfs-tests/tests/| |g' | $COLOR)
   if [ -z "$CLINE" ]; then
     printf "vm${ID}: %s\n" "$LINE"
   else
     # [vm2: 00:15:54  256] Test: functional/checksum/setup (run as root) [00:00] [PASS]
     printf "[vm${ID}: %02d:%02d:%02d %4d] %s\n" \
       "$H" "$M" "$S" "$CTR" "$CLINE"
   fi
 }
 
 # called directly on the runner
 if [ -z ${1:-} ]; then
   cd "/var/tmp"
   source env.txt
   SSH=$(which ssh)
   TESTS='$HOME/zfs/.github/workflows/scripts/qemu-6-tests.sh'
   echo 0 > /tmp/ctr
   date "+%s" > /tmp/tsstart
 
-  for i in $(seq 1 $VMs); do
+  for ((i=1; i<=VMs; i++)); do
     IP="192.168.122.1$i"
     daemonize -c /var/tmp -p vm${i}.pid -o vm${i}log.txt -- \
       $SSH zfs@$IP $TESTS $OS $i $VMs $CI_TYPE
     # handly line by line and add info prefix
     stdbuf -oL tail -fq vm${i}log.txt \
       | while read -r line; do prefix "$i" "$line"; done &
     echo $! > vm${i}log.pid
     # don't mix up the initial --- Configuration --- part
     sleep 0.13
   done
 
   # wait for all vm's to finish
-  for i in $(seq 1 $VMs); do
+  for ((i=1; i<=VMs; i++)); do
     tail --pid=$(cat vm${i}.pid) -f /dev/null
     pid=$(cat vm${i}log.pid)
     rm -f vm${i}log.pid
     kill $pid
   done
 
   exit 0
 fi
 
 # this part runs inside qemu vm
 export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/sbin:/usr/local/bin"
 case "$1" in
   freebsd*)
+    TDIR="/usr/local/share/zfs"
     sudo kldstat -n zfs 2>/dev/null && sudo kldunload zfs
     sudo -E ./zfs/scripts/zfs.sh
-    TDIR="/usr/local/share/zfs"
+    sudo mv -f /var/tmp/*.txt /tmp
+    sudo newfs -U -t -L tmp /dev/vtbd1 >/dev/null
+    sudo mount -o noatime /dev/vtbd1 /var/tmp
+    sudo chmod 1777 /var/tmp
+    sudo mv -f /tmp/*.txt /var/tmp
     ;;
   *)
     # use xfs @ /var/tmp for all distros
+    TDIR="/usr/share/zfs"
+    sudo -E modprobe zfs
     sudo mv -f /var/tmp/*.txt /tmp
     sudo mkfs.xfs -fq /dev/vdb
     sudo mount -o noatime /dev/vdb /var/tmp
     sudo chmod 1777 /var/tmp
     sudo mv -f /tmp/*.txt /var/tmp
-    sudo -E modprobe zfs
-    TDIR="/usr/share/zfs"
+    ;;
+esac
+
+# enable io_uring on el9/el10
+case "$1" in
+  almalinux9|almalinux10|centos-stream*)
+    sudo sysctl kernel.io_uring_disabled=0 > /dev/null
     ;;
 esac
 
 # run functional testings and save exitcode
 cd /var/tmp
 TAGS=$2/$3
 if [ "$4" == "quick" ]; then
   export RUNFILES="sanity.run"
 fi
 sudo dmesg -c > dmesg-prerun.txt
 mount > mount.txt
 df -h > df-prerun.txt
 $TDIR/zfs-tests.sh -vK -s 3GB -T $TAGS
 RV=$?
 df -h > df-postrun.txt
 echo $RV > tests-exitcode.txt
 sync
 exit 0
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-7-prepare.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-7-prepare.sh
index a5fbd7213161..98a5c24c2521 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-7-prepare.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-7-prepare.sh
@@ -1,123 +1,124 @@
 #!/usr/bin/env bash
 
 ######################################################################
 # 7) prepare output of the results
 # - this script pre-creates all needed logfiles for later summary
 ######################################################################
 
 set -eu
 
 # read our defined variables
 cd /var/tmp
 source env.txt
 
 mkdir -p $RESPATH
 
 # check if building the module has failed
 if [ -z ${VMs:-} ]; then
   cd $RESPATH
   echo ":exclamation: ZFS module didn't build successfully :exclamation:" \
     | tee summary.txt | tee /tmp/summary.txt
   cp /var/tmp/*.txt .
   tar cf /tmp/qemu-$OS.tar -C $RESPATH -h . || true
   exit 0
 fi
 
 # build was okay
 BASE="$HOME/work/zfs/zfs"
 MERGE="$BASE/.github/workflows/scripts/merge_summary.awk"
 
 # catch result files of testings (vm's should be there)
-for i in $(seq 1 $VMs); do
-  rsync -arL zfs@192.168.122.1$i:$RESPATH/current $RESPATH/vm$i || true
-  scp zfs@192.168.122.1$i:"/var/tmp/*.txt" $RESPATH/vm$i || true
+for ((i=1; i<=VMs; i++)); do
+  rsync -arL zfs@vm$i:$RESPATH/current $RESPATH/vm$i || true
+  scp zfs@vm$i:"/var/tmp/*.txt" $RESPATH/vm$i || true
+  scp zfs@vm$i:"/var/tmp/*.rpm" $RESPATH/vm$i || true
 done
 cp -f /var/tmp/*.txt $RESPATH || true
 cd $RESPATH
 
 # prepare result files for summary
-for i in $(seq 1 $VMs); do
+for ((i=1; i<=VMs; i++)); do
   file="vm$i/build-stderr.txt"
   test -s $file && mv -f $file build-stderr.txt
 
   file="vm$i/build-exitcode.txt"
   test -s $file && mv -f $file build-exitcode.txt
 
   file="vm$i/uname.txt"
   test -s $file && mv -f $file uname.txt
 
   file="vm$i/tests-exitcode.txt"
   if [ ! -s $file ]; then
     # XXX - add some tests for kernel panic's here
     # tail -n 80 vm$i/console.txt | grep XYZ
     echo 1 > $file
   fi
   rv=$(cat vm$i/tests-exitcode.txt)
   test $rv != 0 && touch /tmp/have_failed_tests
 
   file="vm$i/current/log"
   if [ -s $file ]; then
     cat $file >> log
     awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }; \
       /\[SKIP\]|\[PASS\]/{ show=0; } show' \
       $file > /tmp/vm${i}dbg.txt
   fi
 
   file="vm${i}log.txt"
   fileC="/tmp/vm${i}log.txt"
   if [ -s $file ]; then
     cat $file >> summary
     cat $file | $BASE/scripts/zfs-tests-color.sh > $fileC
   fi
 done
 
 # create summary of tests
 if [ -s summary ]; then
   $MERGE summary | grep -v '^/' > summary.txt
   $MERGE summary | $BASE/scripts/zfs-tests-color.sh > /tmp/summary.txt
   rm -f summary
 else
   touch summary.txt /tmp/summary.txt
 fi
 
 # create file for debugging
 if [ -s log ]; then
   awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }; \
     /\[SKIP\]|\[PASS\]/{ show=0; } show' \
     log > summary-failure-logs.txt
   rm -f log
 else
   touch summary-failure-logs.txt
 fi
 
 # create debug overview for failed tests
 cat summary.txt \
   | awk '/\(expected PASS\)/{ if ($1!="SKIP") print $2; next; } show' \
   | while read t; do
   cat summary-failure-logs.txt \
     | awk '$0~/Test[: ]/{ show=0; } $0~v{ show=1; } show' v="$t" \
     > /tmp/fail.txt
   SIZE=$(stat --printf="%s" /tmp/fail.txt)
   SIZE=$((SIZE/1024))
   # Test Summary:
   echo "##[group]$t ($SIZE KiB)" >> /tmp/failed.txt
   cat /tmp/fail.txt | $BASE/scripts/zfs-tests-color.sh >> /tmp/failed.txt
   echo "##[endgroup]" >> /tmp/failed.txt
   # Job Summary:
   echo -e "\n<details>\n<summary>$t ($SIZE KiB)</summary><pre>" >> failed.txt
   cat /tmp/fail.txt >> failed.txt
   echo "</pre></details>" >> failed.txt
 done
 
 if [ -e /tmp/have_failed_tests ]; then
   echo ":warning: Some tests failed!" >> failed.txt
 else
   echo ":thumbsup: All tests passed." >> failed.txt
 fi
 
 if [ ! -s uname.txt ]; then
   echo ":interrobang: Panic - where is my uname.txt?" > uname.txt
 fi
 
 # artifact ready now
 tar cf /tmp/qemu-$OS.tar -C $RESPATH -h . || true
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-8-summary.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-8-summary.sh
index 26dbab28323b..7d1e16567ab4 100755
--- a/sys/contrib/openzfs/.github/workflows/scripts/qemu-8-summary.sh
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-8-summary.sh
@@ -1,71 +1,71 @@
 #!/usr/bin/env bash
 
 ######################################################################
 # 8) show colored output of results
 ######################################################################
 
 set -eu
 
 # read our defined variables
 source /var/tmp/env.txt
 cd $RESPATH
 
 # helper function for showing some content with headline
 function showfile() {
   content=$(dd if=$1 bs=1024 count=400k 2>/dev/null)
   if [ -z "$2" ]; then
     group1=""
     group2=""
   else
     SIZE=$(stat --printf="%s" "$file")
     SIZE=$((SIZE/1024))
     group1="##[group]$2 ($SIZE KiB)"
     group2="##[endgroup]"
   fi
 cat <<EOF > tmp$$
 $group1
 $content
 $group2
 EOF
   cat tmp$$
   rm -f tmp$$
 }
 
 # overview
 cat /tmp/summary.txt
 echo ""
 
 if [ -f /tmp/have_failed_tests -a -s /tmp/failed.txt ]; then
   echo "Debuginfo of failed tests:"
   cat /tmp/failed.txt
   echo ""
   cat /tmp/summary.txt | grep -v '^/'
   echo ""
 fi
 
 echo -e "\nFull logs for download:\n    $1\n"
 
-for i in $(seq 1 $VMs); do
+for ((i=1; i<=VMs; i++)); do
   rv=$(cat vm$i/tests-exitcode.txt)
 
   if [ $rv = 0 ]; then
     vm="vm$i"
   else
     vm="vm$i"
   fi
 
   file="vm$i/dmesg-prerun.txt"
   test -s "$file" && showfile "$file" "$vm: dmesg kernel"
 
   file="/tmp/vm${i}log.txt"
   test -s "$file" && showfile "$file" "$vm: test results"
 
   file="vm$i/console.txt"
   test -s "$file" && showfile "$file" "$vm: serial console"
 
   file="/tmp/vm${i}dbg.txt"
   test -s "$file" && showfile "$file" "$vm: failure logfile"
 done
 
 test -f /tmp/have_failed_tests && exit 1
 exit 0
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-prepare-for-build.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-prepare-for-build.sh
new file mode 100755
index 000000000000..a5a9e422ba6e
--- /dev/null
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-prepare-for-build.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+# Helper script to run after installing dependencies.  This brings the VM back
+# up and copies over the zfs source directory.
+echo "Build modules in QEMU machine"
+sudo virsh start openzfs
+.github/workflows/scripts/qemu-wait-for-vm.sh vm0
+rsync -ar $HOME/work/zfs/zfs zfs@vm0:./
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-test-repo-vm.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-test-repo-vm.sh
new file mode 100755
index 000000000000..e3cafcbb67cc
--- /dev/null
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-test-repo-vm.sh
@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# Do a test install of ZFS from an external repository.
+#
+# USAGE:
+#
+# 	./qemu-test-repo-vm [URL]
+#
+# URL:		URL to use instead of http://download.zfsonlinux.org
+#		If blank, use the default repo from zfs-release RPM.
+
+set -e
+
+source /etc/os-release
+OS="$ID"
+VERSION="$VERSION_ID"
+
+ALTHOST=""
+if [ -n "$1" ] ; then
+	ALTHOST="$1"
+fi
+
+# Write summary to /tmp/repo so our artifacts scripts pick it up
+mkdir /tmp/repo
+SUMMARY=/tmp/repo/$OS-$VERSION-summary.txt
+
+# $1: Repo 'zfs' 'zfs-kmod' 'zfs-testing' 'zfs-testing-kmod'
+# $2: (optional) Alternate host than 'http://download.zfsonlinux.org' to
+#     install from.  Blank means use default from zfs-release RPM.
+function test_install {
+	repo=$1
+	host=""
+	if [ -n "$2" ] ; then
+		host=$2
+	fi
+
+	args="--disablerepo=zfs --enablerepo=$repo"
+
+	# If we supplied an alternate repo URL, and have not already edited
+	# zfs.repo, then update the repo file.
+	if [ -n "$host" ] && ! grep -q $host /etc/yum.repos.d/zfs.repo ; then
+		sudo sed -i "s;baseurl=http://download.zfsonlinux.org;baseurl=$host;g" /etc/yum.repos.d/zfs.repo
+	fi
+
+	sudo dnf -y install $args zfs zfs-test
+
+	# Load modules and create a simple pool as a sanity test.
+	sudo /usr/share/zfs/zfs.sh -r
+	truncate -s 100M /tmp/file
+	sudo zpool create tank /tmp/file
+	sudo zpool status
+
+	# Print out repo name, rpm installed (kmod or dkms), and repo URL
+	baseurl=$(grep -A 5 "\[$repo\]" /etc/yum.repos.d/zfs.repo  | awk -F'=' '/baseurl=/{print $2; exit}')
+	package=$(sudo rpm -qa | grep zfs | grep -E 'kmod|dkms')
+
+	echo "$repo $package $baseurl" >> $SUMMARY
+
+	sudo zpool destroy tank
+	sudo rm /tmp/file
+	sudo dnf -y remove zfs
+}
+
+echo "##[group]Installing from repo"
+# The openzfs docs are the authoritative instructions for the install.  Use
+# the specific version of zfs-release RPM it recommends.
+case $OS in
+almalinux*)
+	url='https://raw.githubusercontent.com/openzfs/openzfs-docs/refs/heads/master/docs/Getting%20Started/RHEL-based%20distro/index.rst'
+	name=$(curl -Ls $url | grep 'dnf install' | grep -Eo 'zfs-release-[0-9]+-[0-9]+')
+	sudo dnf -y install https://zfsonlinux.org/epel/$name$(rpm --eval "%{dist}").noarch.rpm 2>&1
+	sudo rpm -qi zfs-release
+	test_install zfs $ALTHOST
+	test_install zfs-kmod $ALTHOST
+	test_install zfs-testing $ALTHOST
+	test_install zfs-testing-kmod $ALTHOST
+	;;
+fedora*)
+	url='https://raw.githubusercontent.com/openzfs/openzfs-docs/refs/heads/master/docs/Getting%20Started/Fedora/index.rst'
+	name=$(curl -Ls $url | grep 'dnf install' | grep -Eo 'zfs-release-[0-9]+-[0-9]+')
+	sudo dnf -y install -y https://zfsonlinux.org/fedora/$name$(rpm --eval "%{dist}").noarch.rpm
+	test_install zfs $ALTHOST
+	;;
+esac
+echo "##[endgroup]"
+
+# Write out a simple version of the summary here. Later on we will collate all
+# the summaries and put them into a nice table in the workflow Summary page.
+echo "Summary: "
+cat $SUMMARY
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/qemu-wait-for-vm.sh b/sys/contrib/openzfs/.github/workflows/scripts/qemu-wait-for-vm.sh
new file mode 100755
index 000000000000..e8afdb3f7b98
--- /dev/null
+++ b/sys/contrib/openzfs/.github/workflows/scripts/qemu-wait-for-vm.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+#
+# Wait for a VM to boot up and become active.  This is used in a number of our
+# scripts.
+#
+# $1: VM hostname or IP address
+
+while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
+  ssh 2>/dev/null zfs@$1 "uname -a" && break
+done
diff --git a/sys/contrib/openzfs/.github/workflows/scripts/replace-dupes-with-symlinks.sh b/sys/contrib/openzfs/.github/workflows/scripts/replace-dupes-with-symlinks.sh
new file mode 100755
index 000000000000..5412c954ad2f
--- /dev/null
+++ b/sys/contrib/openzfs/.github/workflows/scripts/replace-dupes-with-symlinks.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Recursively go though a directory structure and replace duplicate files with
+# symlinks.  This cuts down our RPM repo size by ~25%.
+#
+#	replace-dupes-with-symlinks.sh [DIR]
+#
+# DIR: Directory to traverse.  Defaults to current directory if not specified.
+#
+
+src="$1"
+if [ -z "$src" ] ; then
+	src="."
+fi
+
+declare -A db
+
+pushd "$src"
+while read line ; do
+	bn="$(basename $line)"
+	if [ -z "${db[$bn]}" ] ; then
+		# First time this file has been seen
+		db[$bn]="$line"
+	else
+		if diff -b "$line" "${db[$bn]}" &>/dev/null ; then
+			# Files are the same, make a symlink
+			rm "$line"
+			ln -sr "${db[$bn]}" "$line"
+		fi
+	fi
+done <<< "$(find . -type f)"
+popd
diff --git a/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml b/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml
new file mode 100644
index 000000000000..5b5afe746859
--- /dev/null
+++ b/sys/contrib/openzfs/.github/workflows/zfs-qemu-packages.yml
@@ -0,0 +1,140 @@
+# This workflow is used to build and test RPM packages.  It is a
+# 'workflow_dispatch' workflow, which means it gets run manually.
+#
+# The workflow has a dropdown menu with two options:
+#
+# Build RPMs - Build release RPMs and tarballs and put them into an artifact
+#              ZIP file.  The directory structure used in the ZIP file mirrors
+#              the ZFS yum repo.
+#
+# Test repo -  Test install the ZFS RPMs from the ZFS repo.  On EL distos, this
+#              will do a DKMS and KMOD test install from both the regular and
+#              testing repos.  On Fedora, it will do a DKMS install from the
+#              regular repo.  All test install results will be displayed in the
+#              Summary page.  Note that the workflow provides an optional text
+#              text box where you can specify the full URL to an alternate repo.
+#              If left blank, it will install from the default repo from the
+#              zfs-release RPM (http://download.zfsonlinux.org).
+#
+# Most users will never need to use this workflow.  It will be used primary by
+# ZFS admins for building and testing releases.
+#
+name: zfs-qemu-packages
+
+on:
+  workflow_dispatch:
+    inputs:
+      test_type:
+        type: choice
+        required: false
+        default: "Build RPMs"
+        description: "Build RPMs or test the repo?"
+        options:
+        - "Build RPMs"
+        - "Test repo"
+      repo_url:
+        type: string
+        required: false
+        default: ""
+        description: "(optional) repo URL (blank: use http://download.zfsonlinux.org)"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  zfs-qemu-packages-jobs:
+    name: qemu-VMs
+    strategy:
+      fail-fast: false
+      matrix:
+        os: ['almalinux8', 'almalinux9', 'almalinux10', 'fedora41', 'fedora42']
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Setup QEMU
+      timeout-minutes: 10
+      run: .github/workflows/scripts/qemu-1-setup.sh
+
+    - name: Start build machine
+      timeout-minutes: 10
+      run: .github/workflows/scripts/qemu-2-start.sh ${{ matrix.os }}
+
+    - name: Install dependencies
+      timeout-minutes: 20
+      run: |
+        .github/workflows/scripts/qemu-3-deps.sh ${{ matrix.os }}
+
+    - name: Build modules or Test repo
+      timeout-minutes: 30
+      run: |
+        set -e
+        if [ "${{ github.event.inputs.test_type }}" == "Test repo" ] ; then
+                # Bring VM back up and copy over zfs source
+                .github/workflows/scripts/qemu-prepare-for-build.sh
+
+                mkdir -p /tmp/repo
+                ssh zfs@vm0 '$HOME/zfs/.github/workflows/scripts/qemu-test-repo-vm.sh' ${{ github.event.inputs.repo_url }}
+        else
+                .github/workflows/scripts/qemu-4-build.sh --repo --release --dkms --tarball ${{ matrix.os }}
+        fi
+
+    - name: Prepare artifacts
+      if: always()
+      timeout-minutes: 10
+      run: |
+        rsync -a zfs@vm0:/tmp/repo /tmp || true
+        .github/workflows/scripts/replace-dupes-with-symlinks.sh /tmp/repo
+        tar -cf ${{ matrix.os }}-repo.tar -C /tmp repo
+
+    - uses: actions/upload-artifact@v4
+      id: artifact-upload
+      if: always()
+      with:
+        name: ${{ matrix.os }}-repo
+        path: ${{ matrix.os }}-repo.tar
+        compression-level: 0
+        retention-days: 2
+        if-no-files-found: ignore
+
+  combine_repos:
+    if: always()
+    needs: [zfs-qemu-packages-jobs]
+    name: "Results"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/download-artifact@v4
+      id: artifact-download
+      if: always()
+    - name: Test Summary
+      if: always()
+      run: |
+        for i in $(find . -type f -iname "*.tar") ; do
+          tar -xf $i -C /tmp
+        done
+        tar -cf all-repo.tar -C /tmp repo
+
+        # If we're installing from a repo, print out the summary of the versions
+        # that got installed using Markdown.
+        if [ "${{ github.event.inputs.test_type }}" == "Test repo" ] ; then
+          cd /tmp/repo
+          for i in $(ls *.txt) ; do
+            nicename="$(echo $i | sed 's/.txt//g; s/-/ /g')"
+            echo "### $nicename" >> $GITHUB_STEP_SUMMARY
+            echo "|repo|RPM|URL|" >> $GITHUB_STEP_SUMMARY
+            echo "|:---|:---|:---|" >> $GITHUB_STEP_SUMMARY
+            awk '{print "|"$1"|"$2"|"$3"|"}' $i >> $GITHUB_STEP_SUMMARY
+          done
+        fi
+
+    - uses: actions/upload-artifact@v4
+      id: artifact-upload2
+      if: always()
+      with:
+        name: all-repo
+        path: all-repo.tar
+        compression-level: 0
+        retention-days: 5
+        if-no-files-found: ignore
diff --git a/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml b/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml
index e90030f4c02e..1d9899ae895f 100644
--- a/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml
+++ b/sys/contrib/openzfs/.github/workflows/zfs-qemu.yml
@@ -1,177 +1,193 @@
 name: zfs-qemu
 
 on:
   push:
   pull_request:
+  workflow_dispatch:
+    inputs:
+      include_stream9:
+        type: boolean
+        required: false
+        default: false
+        description: 'Test on CentOS 9 stream'
+      include_stream10:
+        type: boolean
+        required: false
+        default: false
+        description: 'Test on CentOS 10 stream'
+      fedora_kernel_ver:
+        type: string
+        required: false
+        default: ""
+        description: "(optional) Experimental kernel version to install on Fedora (like '6.14' or '6.13.3-0.rc3')"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   test-config:
     name: Setup
     runs-on: ubuntu-24.04
     outputs:
       test_os: ${{ steps.os.outputs.os }}
       ci_type: ${{ steps.os.outputs.ci_type }}
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - name: Generate OS config and CI type
         id: os
         run: |
-          FULL_OS='["almalinux8", "almalinux9", "centos-stream9", "debian11", "debian12", "fedora40", "fedora41", "freebsd13-4r", "freebsd14-0r", "freebsd14-1s", "ubuntu20", "ubuntu22", "ubuntu24"]'
-          QUICK_OS='["almalinux8", "almalinux9", "debian12", "fedora41", "freebsd13-3r", "freebsd14-1r", "ubuntu24"]'
+          FULL_OS='["almalinux8", "almalinux9", "almalinux10", "debian11", "debian12", "fedora41", "fedora42", "freebsd13-4r", "freebsd14-2s", "freebsd15-0c", "ubuntu22", "ubuntu24"]'
+          QUICK_OS='["almalinux8", "almalinux9", "almalinux10", "debian12", "fedora42", "freebsd14-2r", "ubuntu24"]'
           # determine CI type when running on PR
           ci_type="full"
           if ${{ github.event_name == 'pull_request' }}; then
             head=${{ github.event.pull_request.head.sha }}
             base=${{ github.event.pull_request.base.sha }}
             ci_type=$(python3 .github/workflows/scripts/generate-ci-type.py $head $base)
           fi
           if [ "$ci_type" == "quick" ]; then
             os_selection="$QUICK_OS"
           else
             os_selection="$FULL_OS"
           fi
-          os_json=$(echo ${os_selection} | jq -c)
+
+          if [ ${{ github.event.inputs.fedora_kernel_ver }} != "" ] ; then
+              # They specified a custom kernel version for Fedora.  Use only
+              # Fedora runners.
+              os_json=$(echo ${os_selection} | jq -c '[.[] | select(startswith("fedora"))]')
+          else
+              # Normal case
+              os_json=$(echo ${os_selection} | jq -c)
+          fi
+
+          # Add optional runners
+          if [ "${{ github.event.inputs.include_stream9 }}" == 'true' ]; then
+            os_json=$(echo $os_json | jq -c '. += ["centos-stream9"]')
+          fi
+          if [ "${{ github.event.inputs.include_stream10 }}" == 'true' ]; then
+            os_json=$(echo $os_json | jq -c '. += ["centos-stream10"]')
+          fi
+
+          echo $os_json
           echo "os=$os_json" >> $GITHUB_OUTPUT
           echo "ci_type=$ci_type" >> $GITHUB_OUTPUT
 
   qemu-vm:
     name: qemu-x86
     needs: [ test-config ]
     strategy:
       fail-fast: false
       matrix:
-        # rhl:     almalinux8, almalinux9, centos-stream9, fedora40, fedora41
-        # debian:  debian11, debian12, ubuntu20, ubuntu22, ubuntu24
+        # rhl:     almalinux8, almalinux9, centos-stream9, fedora41
+        # debian:  debian11, debian12, ubuntu22, ubuntu24
         # misc:    archlinux, tumbleweed
-        # FreeBSD Release: freebsd13-3r, freebsd13-4r, freebsd14-0r, freebsd14-1r
-        # FreeBSD Stable:  freebsd13-4s, freebsd14-1s
+        # FreeBSD variants of 2024-12:
+        # FreeBSD Release: freebsd13-4r, freebsd14-2r
+        # FreeBSD Stable:  freebsd13-4s, freebsd14-2s
         # FreeBSD Current: freebsd15-0c
         os: ${{ fromJson(needs.test-config.outputs.test_os) }}
     runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
 
     - name: Setup QEMU
       timeout-minutes: 10
       run: .github/workflows/scripts/qemu-1-setup.sh
 
     - name: Start build machine
       timeout-minutes: 10
       run: .github/workflows/scripts/qemu-2-start.sh ${{ matrix.os }}
 
     - name: Install dependencies
       timeout-minutes: 20
-      run: |
-        echo "Install dependencies in QEMU machine"
-        IP=192.168.122.10
-        while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
-          ssh 2>/dev/null zfs@$IP "uname -a" && break
-        done
-        scp .github/workflows/scripts/qemu-3-deps.sh zfs@$IP:qemu-3-deps.sh
-        PID=`pidof /usr/bin/qemu-system-x86_64`
-        ssh zfs@$IP '$HOME/qemu-3-deps.sh' ${{ matrix.os }}
-        # wait for poweroff to succeed
-        tail --pid=$PID -f /dev/null
-        sleep 5 # avoid this: "error: Domain is already active"
-        rm -f $HOME/.ssh/known_hosts
+      run: .github/workflows/scripts/qemu-3-deps.sh ${{ matrix.os }} ${{ github.event.inputs.fedora_kernel_ver }}
 
     - name: Build modules
       timeout-minutes: 30
-      run: |
-        echo "Build modules in QEMU machine"
-        sudo virsh start openzfs
-        IP=192.168.122.10
-        while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
-          ssh 2>/dev/null zfs@$IP "uname -a" && break
-        done
-        rsync -ar $HOME/work/zfs/zfs zfs@$IP:./
-        ssh zfs@$IP '$HOME/zfs/.github/workflows/scripts/qemu-4-build.sh' ${{ matrix.os }}
+      run: .github/workflows/scripts/qemu-4-build.sh --poweroff --enable-debug ${{ matrix.os }}
 
     - name: Setup testing machines
       timeout-minutes: 5
       run: .github/workflows/scripts/qemu-5-setup.sh
 
     - name: Run tests
       timeout-minutes: 270
       run: .github/workflows/scripts/qemu-6-tests.sh
       env:
         CI_TYPE: ${{ needs.test-config.outputs.ci_type }}
 
     - name: Prepare artifacts
       if: always()
       timeout-minutes: 10
       run: .github/workflows/scripts/qemu-7-prepare.sh
 
     - uses: actions/upload-artifact@v4
       id: artifact-upload
       if: always()
       with:
         name: Logs-functional-${{ matrix.os }}
         path: /tmp/qemu-${{ matrix.os }}.tar
         if-no-files-found: ignore
 
     - name: Test Summary
       if: always()
       run: .github/workflows/scripts/qemu-8-summary.sh '${{ steps.artifact-upload.outputs.artifact-url }}'
 
   cleanup:
     if: always()
     name: Cleanup
     runs-on: ubuntu-latest
     needs: [ qemu-vm ]
 
     steps:
     - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - uses: actions/download-artifact@v4
     - name: Generating summary
       run: .github/workflows/scripts/qemu-9-summary-page.sh
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 2
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 3
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 4
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 5
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 6
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 7
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 8
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 9
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 10
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 11
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 12
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 13
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 14
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 15
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 16
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 17
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 18
     - name: Generating summary...
       run: .github/workflows/scripts/qemu-9-summary-page.sh 19
     - uses: actions/upload-artifact@v4
       with:
         name: Summary Files
         path: out-*
diff --git a/sys/contrib/openzfs/.github/workflows/zloop.yml b/sys/contrib/openzfs/.github/workflows/zloop.yml
index 90d93c48e4bd..7b3bf49d90d5 100644
--- a/sys/contrib/openzfs/.github/workflows/zloop.yml
+++ b/sys/contrib/openzfs/.github/workflows/zloop.yml
@@ -1,77 +1,77 @@
 name: zloop
 
 on:
   push:
   pull_request:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
   zloop:
     runs-on: ubuntu-24.04
     env:
       TEST_DIR: /var/tmp/zloop
     steps:
     - uses: actions/checkout@v4
       with:
         ref: ${{ github.event.pull_request.head.sha }}
     - name: Install dependencies
       run: |
         sudo apt-get purge -y snapd google-chrome-stable firefox
-        ONLY_DEPS=1 .github/workflows/scripts/qemu-3-deps.sh ubuntu24
+        ONLY_DEPS=1 .github/workflows/scripts/qemu-3-deps-vm.sh ubuntu24
     - name: Autogen.sh
       run: |
         sed -i '/DEBUG_CFLAGS="-Werror"/s/^/#/' config/zfs-build.m4
         ./autogen.sh
     - name: Configure
       run: |
         ./configure --prefix=/usr --enable-debug --enable-debuginfo \
            --enable-asan --enable-ubsan \
            --enable-debug-kmem --enable-debug-kmem-tracking
     - name: Make
       run: |
         make -j$(nproc)
     - name: Install
       run: |
         sudo make install
         sudo depmod
         sudo modprobe zfs
     - name: Tests
       run: |
         sudo mkdir -p $TEST_DIR
         # run for 10 minutes or at most 6 iterations for a maximum runner
         # time of 60 minutes.
         sudo /usr/share/zfs/zloop.sh -t 600 -I 6 -l -m 1 -- -T 120 -P 60
     - name: Prepare artifacts
       if: failure()
       run: |
         sudo chmod +r -R $TEST_DIR/
     - name: Ztest log
       if: failure()
       run: |
         grep -B10 -A1000 'ASSERT' $TEST_DIR/*/ztest.out || tail -n 1000 $TEST_DIR/*/ztest.out
     - name: Gdb log
       if: failure()
       run: |
         sed -n '/Backtraces (full)/q;p' $TEST_DIR/*/ztest.gdb
     - name: Zdb log
       if: failure()
       run: |
         cat $TEST_DIR/*/ztest.zdb
     - uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: Logs
         path: |
           /var/tmp/zloop/*/
           !/var/tmp/zloop/*/vdev/
         if-no-files-found: ignore
     - uses: actions/upload-artifact@v4
       if: failure()
       with:
         name: Pool files
         path: |
           /var/tmp/zloop/*/vdev/
         if-no-files-found: ignore
diff --git a/sys/contrib/openzfs/META b/sys/contrib/openzfs/META
index 0e852d300e8f..9e971a564912 100644
--- a/sys/contrib/openzfs/META
+++ b/sys/contrib/openzfs/META
@@ -1,10 +1,10 @@
 Meta:          1
 Name:          zfs
 Branch:        1.0
-Version:       2.2.7
+Version:       2.2.8
 Release:       1
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS
-Linux-Maximum: 6.12
+Linux-Maximum: 6.15
 Linux-Minimum: 4.18
diff --git a/sys/contrib/openzfs/cmd/zpool/os/linux/zpool_vdev_os.c b/sys/contrib/openzfs/cmd/zpool/os/linux/zpool_vdev_os.c
index f194d28c55a9..862b1e61967b 100644
--- a/sys/contrib/openzfs/cmd/zpool/os/linux/zpool_vdev_os.c
+++ b/sys/contrib/openzfs/cmd/zpool/os/linux/zpool_vdev_os.c
@@ -1,673 +1,674 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013, 2018 by Delphix. All rights reserved.
  * Copyright (c) 2016, 2017 Intel Corporation.
  * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
  */
 
 /*
  * Functions to convert between a list of vdevs and an nvlist representing the
  * configuration.  Each entry in the list can be one of:
  *
  * 	Device vdevs
  * 		disk=(path=..., devid=...)
  * 		file=(path=...)
  *
  * 	Group vdevs
  * 		raidz[1|2]=(...)
  * 		mirror=(...)
  *
  * 	Hot spares
  *
  * While the underlying implementation supports it, group vdevs cannot contain
  * other group vdevs.  All userland verification of devices is contained within
  * this file.  If successful, the nvlist returned can be passed directly to the
  * kernel; we've done as much verification as possible in userland.
  *
  * Hot spares are a special case, and passed down as an array of disk vdevs, at
  * the same level as the root of the vdev tree.
  *
  * The only function exported by this file is 'make_root_vdev'.  The
  * function performs several passes:
  *
  * 	1. Construct the vdev specification.  Performs syntax validation and
  *         makes sure each device is valid.
  * 	2. Check for devices in use.  Using libblkid to make sure that no
  *         devices are also in use.  Some can be overridden using the 'force'
  *         flag, others cannot.
  * 	3. Check for replication errors if the 'force' flag is not specified.
  *         validates that the replication level is consistent across the
  *         entire pool.
  * 	4. Call libzfs to label any whole disks with an EFI label.
  */
 
 #include <assert.h>
 #include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <libintl.h>
 #include <libnvpair.h>
 #include <libzutil.h>
 #include <limits.h>
 #include <sys/spa.h>
 #include <stdio.h>
 #include <string.h>
 #include <unistd.h>
 #include "zpool_util.h"
 #include <sys/zfs_context.h>
 
 #include <scsi/scsi.h>
 #include <scsi/sg.h>
 #include <sys/efi_partition.h>
 #include <sys/stat.h>
 #include <sys/mntent.h>
 #include <uuid/uuid.h>
 #include <blkid/blkid.h>
 
 typedef struct vdev_disk_db_entry
 {
-	char id[24];
+	/* 24 byte name + 1 byte NULL terminator to make GCC happy */
+	char id[25];
 	int sector_size;
 } vdev_disk_db_entry_t;
 
 /*
  * Database of block devices that lie about physical sector sizes.  The
  * identification string must be precisely 24 characters to avoid false
  * negatives
  */
 static vdev_disk_db_entry_t vdev_disk_database[] = {
 	{"ATA     ADATA SSD S396 3", 8192},
 	{"ATA     APPLE SSD SM128E", 8192},
 	{"ATA     APPLE SSD SM256E", 8192},
 	{"ATA     APPLE SSD SM512E", 8192},
 	{"ATA     APPLE SSD SM768E", 8192},
 	{"ATA     C400-MTFDDAC064M", 8192},
 	{"ATA     C400-MTFDDAC128M", 8192},
 	{"ATA     C400-MTFDDAC256M", 8192},
 	{"ATA     C400-MTFDDAC512M", 8192},
 	{"ATA     Corsair Force 3 ", 8192},
 	{"ATA     Corsair Force GS", 8192},
 	{"ATA     INTEL SSDSA2CT04", 8192},
 	{"ATA     INTEL SSDSA2BZ10", 8192},
 	{"ATA     INTEL SSDSA2BZ20", 8192},
 	{"ATA     INTEL SSDSA2BZ30", 8192},
 	{"ATA     INTEL SSDSA2CW04", 8192},
 	{"ATA     INTEL SSDSA2CW08", 8192},
 	{"ATA     INTEL SSDSA2CW12", 8192},
 	{"ATA     INTEL SSDSA2CW16", 8192},
 	{"ATA     INTEL SSDSA2CW30", 8192},
 	{"ATA     INTEL SSDSA2CW60", 8192},
 	{"ATA     INTEL SSDSC2CT06", 8192},
 	{"ATA     INTEL SSDSC2CT12", 8192},
 	{"ATA     INTEL SSDSC2CT18", 8192},
 	{"ATA     INTEL SSDSC2CT24", 8192},
 	{"ATA     INTEL SSDSC2CW06", 8192},
 	{"ATA     INTEL SSDSC2CW12", 8192},
 	{"ATA     INTEL SSDSC2CW18", 8192},
 	{"ATA     INTEL SSDSC2CW24", 8192},
 	{"ATA     INTEL SSDSC2CW48", 8192},
 	{"ATA     KINGSTON SH100S3", 8192},
 	{"ATA     KINGSTON SH103S3", 8192},
 	{"ATA     M4-CT064M4SSD2  ", 8192},
 	{"ATA     M4-CT128M4SSD2  ", 8192},
 	{"ATA     M4-CT256M4SSD2  ", 8192},
 	{"ATA     M4-CT512M4SSD2  ", 8192},
 	{"ATA     OCZ-AGILITY2    ", 8192},
 	{"ATA     OCZ-AGILITY3    ", 8192},
 	{"ATA     OCZ-VERTEX2 3.5 ", 8192},
 	{"ATA     OCZ-VERTEX3     ", 8192},
 	{"ATA     OCZ-VERTEX3 LT  ", 8192},
 	{"ATA     OCZ-VERTEX3 MI  ", 8192},
 	{"ATA     OCZ-VERTEX4     ", 8192},
 	{"ATA     SAMSUNG MZ7WD120", 8192},
 	{"ATA     SAMSUNG MZ7WD240", 8192},
 	{"ATA     SAMSUNG MZ7WD480", 8192},
 	{"ATA     SAMSUNG MZ7WD960", 8192},
 	{"ATA     SAMSUNG SSD 830 ", 8192},
 	{"ATA     Samsung SSD 840 ", 8192},
 	{"ATA     SanDisk SSD U100", 8192},
 	{"ATA     TOSHIBA THNSNH06", 8192},
 	{"ATA     TOSHIBA THNSNH12", 8192},
 	{"ATA     TOSHIBA THNSNH25", 8192},
 	{"ATA     TOSHIBA THNSNH51", 8192},
 	{"ATA     APPLE SSD TS064C", 4096},
 	{"ATA     APPLE SSD TS128C", 4096},
 	{"ATA     APPLE SSD TS256C", 4096},
 	{"ATA     APPLE SSD TS512C", 4096},
 	{"ATA     INTEL SSDSA2M040", 4096},
 	{"ATA     INTEL SSDSA2M080", 4096},
 	{"ATA     INTEL SSDSA2M160", 4096},
 	{"ATA     INTEL SSDSC2MH12", 4096},
 	{"ATA     INTEL SSDSC2MH25", 4096},
 	{"ATA     OCZ CORE_SSD    ", 4096},
 	{"ATA     OCZ-VERTEX      ", 4096},
 	{"ATA     SAMSUNG MCCOE32G", 4096},
 	{"ATA     SAMSUNG MCCOE64G", 4096},
 	{"ATA     SAMSUNG SSD PM80", 4096},
 	/* Flash drives optimized for 4KB IOs on larger pages */
 	{"ATA     INTEL SSDSC2BA10", 4096},
 	{"ATA     INTEL SSDSC2BA20", 4096},
 	{"ATA     INTEL SSDSC2BA40", 4096},
 	{"ATA     INTEL SSDSC2BA80", 4096},
 	{"ATA     INTEL SSDSC2BB08", 4096},
 	{"ATA     INTEL SSDSC2BB12", 4096},
 	{"ATA     INTEL SSDSC2BB16", 4096},
 	{"ATA     INTEL SSDSC2BB24", 4096},
 	{"ATA     INTEL SSDSC2BB30", 4096},
 	{"ATA     INTEL SSDSC2BB40", 4096},
 	{"ATA     INTEL SSDSC2BB48", 4096},
 	{"ATA     INTEL SSDSC2BB60", 4096},
 	{"ATA     INTEL SSDSC2BB80", 4096},
 	{"ATA     INTEL SSDSC2BW24", 4096},
 	{"ATA     INTEL SSDSC2BW48", 4096},
 	{"ATA     INTEL SSDSC2BP24", 4096},
 	{"ATA     INTEL SSDSC2BP48", 4096},
 	{"NA      SmrtStorSDLKAE9W", 4096},
 	{"NVMe    Amazon EC2 NVMe ", 4096},
 	/* Imported from Open Solaris */
 	{"ATA     MARVELL SD88SA02", 4096},
 	/* Advanced format Hard drives */
 	{"ATA     Hitachi HDS5C303", 4096},
 	{"ATA     SAMSUNG HD204UI ", 4096},
 	{"ATA     ST2000DL004 HD20", 4096},
 	{"ATA     WDC WD10EARS-00M", 4096},
 	{"ATA     WDC WD10EARS-00S", 4096},
 	{"ATA     WDC WD10EARS-00Z", 4096},
 	{"ATA     WDC WD15EARS-00M", 4096},
 	{"ATA     WDC WD15EARS-00S", 4096},
 	{"ATA     WDC WD15EARS-00Z", 4096},
 	{"ATA     WDC WD20EARS-00M", 4096},
 	{"ATA     WDC WD20EARS-00S", 4096},
 	{"ATA     WDC WD20EARS-00Z", 4096},
 	{"ATA     WDC WD1600BEVT-0", 4096},
 	{"ATA     WDC WD2500BEVT-0", 4096},
 	{"ATA     WDC WD3200BEVT-0", 4096},
 	{"ATA     WDC WD5000BEVT-0", 4096},
 };
 
 
 #define	INQ_REPLY_LEN	96
 #define	INQ_CMD_LEN	6
 
 static const int vdev_disk_database_size =
 	sizeof (vdev_disk_database) / sizeof (vdev_disk_database[0]);
 
 boolean_t
 check_sector_size_database(char *path, int *sector_size)
 {
 	unsigned char inq_buff[INQ_REPLY_LEN];
 	unsigned char sense_buffer[32];
 	unsigned char inq_cmd_blk[INQ_CMD_LEN] =
 	    {INQUIRY, 0, 0, 0, INQ_REPLY_LEN, 0};
 	sg_io_hdr_t io_hdr;
 	int error;
 	int fd;
 	int i;
 
 	/* Prepare INQUIRY command */
 	memset(&io_hdr, 0, sizeof (sg_io_hdr_t));
 	io_hdr.interface_id = 'S';
 	io_hdr.cmd_len = sizeof (inq_cmd_blk);
 	io_hdr.mx_sb_len = sizeof (sense_buffer);
 	io_hdr.dxfer_direction = SG_DXFER_FROM_DEV;
 	io_hdr.dxfer_len = INQ_REPLY_LEN;
 	io_hdr.dxferp = inq_buff;
 	io_hdr.cmdp = inq_cmd_blk;
 	io_hdr.sbp = sense_buffer;
 	io_hdr.timeout = 10;		/* 10 milliseconds is ample time */
 
 	if ((fd = open(path, O_RDONLY|O_DIRECT)) < 0)
 		return (B_FALSE);
 
 	error = ioctl(fd, SG_IO, (unsigned long) &io_hdr);
 
 	(void) close(fd);
 
 	if (error < 0)
 		return (B_FALSE);
 
 	if ((io_hdr.info & SG_INFO_OK_MASK) != SG_INFO_OK)
 		return (B_FALSE);
 
 	for (i = 0; i < vdev_disk_database_size; i++) {
 		if (memcmp(inq_buff + 8, vdev_disk_database[i].id, 24))
 			continue;
 
 		*sector_size = vdev_disk_database[i].sector_size;
 		return (B_TRUE);
 	}
 
 	return (B_FALSE);
 }
 
 static int
 check_slice(const char *path, blkid_cache cache, int force, boolean_t isspare)
 {
 	int err;
 	char *value;
 
 	/* No valid type detected device is safe to use */
 	value = blkid_get_tag_value(cache, "TYPE", path);
 	if (value == NULL)
 		return (0);
 
 	/*
 	 * If libblkid detects a ZFS device, we check the device
 	 * using check_file() to see if it's safe.  The one safe
 	 * case is a spare device shared between multiple pools.
 	 */
 	if (strcmp(value, "zfs_member") == 0) {
 		err = check_file(path, force, isspare);
 	} else {
 		if (force) {
 			err = 0;
 		} else {
 			err = -1;
 			vdev_error(gettext("%s contains a filesystem of "
 			    "type '%s'\n"), path, value);
 		}
 	}
 
 	free(value);
 
 	return (err);
 }
 
 /*
  * Validate that a disk including all partitions are safe to use.
  *
  * For EFI labeled disks this can done relatively easily with the libefi
  * library.  The partition numbers are extracted from the label and used
  * to generate the expected /dev/ paths.  Each partition can then be
  * checked for conflicts.
  *
  * For non-EFI labeled disks (MBR/EBR/etc) the same process is possible
  * but due to the lack of a readily available libraries this scanning is
  * not implemented.  Instead only the device path as given is checked.
  */
 static int
 check_disk(const char *path, blkid_cache cache, int force,
     boolean_t isspare, boolean_t iswholedisk)
 {
 	struct dk_gpt *vtoc;
 	char slice_path[MAXPATHLEN];
 	int err = 0;
 	int fd, i;
 	int flags = O_RDONLY|O_DIRECT;
 
 	if (!iswholedisk)
 		return (check_slice(path, cache, force, isspare));
 
 	/* only spares can be shared, other devices require exclusive access */
 	if (!isspare)
 		flags |= O_EXCL;
 
 	if ((fd = open(path, flags)) < 0) {
 		char *value = blkid_get_tag_value(cache, "TYPE", path);
 		(void) fprintf(stderr, gettext("%s is in use and contains "
 		    "a %s filesystem.\n"), path, value ? value : "unknown");
 		free(value);
 		return (-1);
 	}
 
 	/*
 	 * Expected to fail for non-EFI labeled disks.  Just check the device
 	 * as given and do not attempt to detect and scan partitions.
 	 */
 	err = efi_alloc_and_read(fd, &vtoc);
 	if (err) {
 		(void) close(fd);
 		return (check_slice(path, cache, force, isspare));
 	}
 
 	/*
 	 * The primary efi partition label is damaged however the secondary
 	 * label at the end of the device is intact.  Rather than use this
 	 * label we should play it safe and treat this as a non efi device.
 	 */
 	if (vtoc->efi_flags & EFI_GPT_PRIMARY_CORRUPT) {
 		efi_free(vtoc);
 		(void) close(fd);
 
 		if (force) {
 			/* Partitions will now be created using the backup */
 			return (0);
 		} else {
 			vdev_error(gettext("%s contains a corrupt primary "
 			    "EFI label.\n"), path);
 			return (-1);
 		}
 	}
 
 	for (i = 0; i < vtoc->efi_nparts; i++) {
 
 		if (vtoc->efi_parts[i].p_tag == V_UNASSIGNED ||
 		    uuid_is_null((uchar_t *)&vtoc->efi_parts[i].p_guid))
 			continue;
 
 		if (strncmp(path, UDISK_ROOT, strlen(UDISK_ROOT)) == 0)
 			(void) snprintf(slice_path, sizeof (slice_path),
 			    "%s%s%d", path, "-part", i+1);
 		else
 			(void) snprintf(slice_path, sizeof (slice_path),
 			    "%s%s%d", path, isdigit(path[strlen(path)-1]) ?
 			    "p" : "", i+1);
 
 		err = check_slice(slice_path, cache, force, isspare);
 		if (err)
 			break;
 	}
 
 	efi_free(vtoc);
 	(void) close(fd);
 
 	return (err);
 }
 
 int
 check_device(const char *path, boolean_t force,
     boolean_t isspare, boolean_t iswholedisk)
 {
 	blkid_cache cache;
 	int error;
 
 	error = blkid_get_cache(&cache, NULL);
 	if (error != 0) {
 		(void) fprintf(stderr, gettext("unable to access the blkid "
 		    "cache.\n"));
 		return (-1);
 	}
 
 	error = check_disk(path, cache, force, isspare, iswholedisk);
 	blkid_put_cache(cache);
 
 	return (error);
 }
 
 void
 after_zpool_upgrade(zpool_handle_t *zhp)
 {
 	(void) zhp;
 }
 
 int
 check_file(const char *file, boolean_t force, boolean_t isspare)
 {
 	return (check_file_generic(file, force, isspare));
 }
 
 /*
  * Read from a sysfs file and return an allocated string.  Removes
  * the newline from the end of the string if there is one.
  *
  * Returns a string on success (which must be freed), or NULL on error.
  */
 static char *zpool_sysfs_gets(char *path)
 {
 	int fd;
 	struct stat statbuf;
 	char *buf = NULL;
 	ssize_t count = 0;
 	fd = open(path, O_RDONLY);
 	if (fd < 0)
 		return (NULL);
 
 	if (fstat(fd, &statbuf) != 0) {
 		close(fd);
 		return (NULL);
 	}
 
 	buf = calloc(statbuf.st_size + 1, sizeof (*buf));
 	if (buf == NULL) {
 		close(fd);
 		return (NULL);
 	}
 
 	/*
 	 * Note, we can read less bytes than st_size, and that's ok.  Sysfs
 	 * files will report their size is 4k even if they only return a small
 	 * string.
 	 */
 	count = read(fd, buf, statbuf.st_size);
 	if (count < 0) {
 		/* Error doing read() or we overran the buffer */
 		close(fd);
 		free(buf);
 		return (NULL);
 	}
 
 	/* Remove trailing newline */
 	if (count > 0 && buf[count - 1] == '\n')
 		buf[count - 1] = 0;
 
 	close(fd);
 
 	return (buf);
 }
 
 /*
  * Write a string to a sysfs file.
  *
  * Returns 0 on success, non-zero otherwise.
  */
 static int zpool_sysfs_puts(char *path, char *str)
 {
 	FILE *file;
 
 	file = fopen(path, "w");
 	if (!file) {
 		return (-1);
 	}
 
 	if (fputs(str, file) < 0) {
 		fclose(file);
 		return (-2);
 	}
 	fclose(file);
 	return (0);
 }
 
 /* Given a vdev nvlist_t, rescan its enclosure sysfs path */
 static void
 rescan_vdev_config_dev_sysfs_path(nvlist_t *vdev_nv)
 {
 	update_vdev_config_dev_sysfs_path(vdev_nv,
 	    fnvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_PATH),
 	    ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH);
 }
 
 /*
  * Given a power string: "on", "off", "1", or "0", return 0 if it's an
  * off value, 1 if it's an on value, and -1 if the value is unrecognized.
  */
 static int zpool_power_parse_value(char *str)
 {
 	if ((strcmp(str, "off") == 0) || (strcmp(str, "0") == 0))
 		return (0);
 
 	if ((strcmp(str, "on") == 0) || (strcmp(str, "1") == 0))
 		return (1);
 
 	return (-1);
 }
 
 /*
  * Given a vdev string return an allocated string containing the sysfs path to
  * its power control file.  Also do a check if the power control file really
  * exists and has correct permissions.
  *
  * Example returned strings:
  *
  * /sys/class/enclosure/0:0:122:0/10/power_status
  * /sys/bus/pci/slots/10/power
  *
  * Returns allocated string on success (which must be freed), NULL on failure.
  */
 static char *
 zpool_power_sysfs_path(zpool_handle_t *zhp, char *vdev)
 {
 	const char *enc_sysfs_dir = NULL;
 	char *path = NULL;
 	nvlist_t *vdev_nv = zpool_find_vdev(zhp, vdev, NULL, NULL, NULL);
 
 	if (vdev_nv == NULL) {
 		return (NULL);
 	}
 
 	/* Make sure we're getting the updated enclosure sysfs path */
 	rescan_vdev_config_dev_sysfs_path(vdev_nv);
 
 	if (nvlist_lookup_string(vdev_nv, ZPOOL_CONFIG_VDEV_ENC_SYSFS_PATH,
 	    &enc_sysfs_dir) != 0) {
 		return (NULL);
 	}
 
 	if (asprintf(&path, "%s/power_status", enc_sysfs_dir) == -1)
 		return (NULL);
 
 	if (access(path, W_OK) != 0) {
 		free(path);
 		path = NULL;
 		/* No HDD 'power_control' file, maybe it's NVMe? */
 		if (asprintf(&path, "%s/power", enc_sysfs_dir) == -1) {
 			return (NULL);
 		}
 
 		if (access(path, R_OK | W_OK) != 0) {
 			/* Not NVMe either */
 			free(path);
 			return (NULL);
 		}
 	}
 
 	return (path);
 }
 
 /*
  * Given a path to a sysfs power control file, return B_TRUE if you should use
  * "on/off" words to control it, or B_FALSE otherwise ("0/1" to control).
  */
 static boolean_t
 zpool_power_use_word(char *sysfs_path)
 {
 	if (strcmp(&sysfs_path[strlen(sysfs_path) - strlen("power_status")],
 	    "power_status") == 0) {
 		return (B_TRUE);
 	}
 	return (B_FALSE);
 }
 
 /*
  * Check the sysfs power control value for a vdev.
  *
  * Returns:
  *  0 - Power is off
  *  1 - Power is on
  * -1 - Error or unsupported
  */
 int
 zpool_power_current_state(zpool_handle_t *zhp, char *vdev)
 {
 	char *val;
 	int rc;
 
 	char *path = zpool_power_sysfs_path(zhp, vdev);
 	if (path == NULL)
 		return (-1);
 
 	val = zpool_sysfs_gets(path);
 	if (val == NULL) {
 		free(path);
 		return (-1);
 	}
 
 	rc = zpool_power_parse_value(val);
 	free(val);
 	free(path);
 	return (rc);
 }
 
 /*
  * Turn on or off the slot to a device
  *
  * Device path is the full path to the device (like /dev/sda or /dev/sda1).
  *
  * Return code:
  * 0:		Success
  * ENOTSUP:	Power control not supported for OS
  * EBADSLT:	Couldn't read current power state
  * ENOENT:	No sysfs path to power control
  * EIO:	Couldn't write sysfs power value
  * EBADE:	Sysfs power value didn't change
  */
 int
 zpool_power(zpool_handle_t *zhp, char *vdev, boolean_t turn_on)
 {
 	char *sysfs_path;
 	const char *val;
 	int rc;
 	int timeout_ms;
 
 	rc = zpool_power_current_state(zhp, vdev);
 	if (rc == -1) {
 		return (EBADSLT);
 	}
 
 	/* Already correct value? */
 	if (rc == (int)turn_on)
 		return (0);
 
 	sysfs_path = zpool_power_sysfs_path(zhp, vdev);
 	if (sysfs_path == NULL)
 		return (ENOENT);
 
 	if (zpool_power_use_word(sysfs_path)) {
 		val = turn_on ? "on" : "off";
 	} else {
 		val = turn_on ? "1" : "0";
 	}
 
 	rc = zpool_sysfs_puts(sysfs_path, (char *)val);
 
 	free(sysfs_path);
 	if (rc != 0) {
 		return (EIO);
 	}
 
 	/*
 	 * Wait up to 30 seconds for sysfs power value to change after
 	 * writing it.
 	 */
 	timeout_ms = zpool_getenv_int("ZPOOL_POWER_ON_SLOT_TIMEOUT_MS", 30000);
 	for (int i = 0; i < MAX(1, timeout_ms / 200); i++) {
 		rc = zpool_power_current_state(zhp, vdev);
 		if (rc == (int)turn_on)
 			return (0);	/* success */
 
 		fsleep(0.200);	/* 200ms */
 	}
 
 	/* sysfs value never changed */
 	return (EBADE);
 }
diff --git a/sys/contrib/openzfs/config/kernel-automount.m4 b/sys/contrib/openzfs/config/kernel-automount.m4
index 52f1931b748e..b5f1392d0fcd 100644
--- a/sys/contrib/openzfs/config/kernel-automount.m4
+++ b/sys/contrib/openzfs/config/kernel-automount.m4
@@ -1,25 +1,62 @@
 dnl #
 dnl # 2.6.37 API change
 dnl # The dops->d_automount() dentry operation was added as a clean
 dnl # solution to handling automounts.  Prior to this cifs/nfs clients
 dnl # which required automount support would abuse the follow_link()
 dnl # operation on directories for this purpose.
 dnl #
-AC_DEFUN([ZFS_AC_KERNEL_SRC_AUTOMOUNT], [
+AC_DEFUN([ZFS_AC_KERNEL_SRC_D_AUTOMOUNT], [
 	ZFS_LINUX_TEST_SRC([dentry_operations_d_automount], [
 		#include <linux/dcache.h>
 		static struct vfsmount *d_automount(struct path *p) { return NULL; }
 		struct dentry_operations dops __attribute__ ((unused)) = {
 			.d_automount = d_automount,
 		};
 	])
 ])
 
-AC_DEFUN([ZFS_AC_KERNEL_AUTOMOUNT], [
+AC_DEFUN([ZFS_AC_KERNEL_D_AUTOMOUNT], [
 	AC_MSG_CHECKING([whether dops->d_automount() exists])
 	ZFS_LINUX_TEST_RESULT([dentry_operations_d_automount], [
 		AC_MSG_RESULT(yes)
 	],[
 		ZFS_LINUX_TEST_ERROR([dops->d_automount()])
 	])
 ])
+
+dnl #
+dnl # 6.14 API change
+dnl # dops->d_revalidate now has four args.
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_D_REVALIDATE_4ARGS], [
+	ZFS_LINUX_TEST_SRC([dentry_operations_d_revalidate_4args], [
+		#include <linux/dcache.h>
+		static int d_revalidate(struct inode *dir,
+		    const struct qstr *name, struct dentry *dentry,
+		    unsigned int fl) { return 0; }
+		struct dentry_operations dops __attribute__ ((unused)) = {
+			.d_revalidate = d_revalidate,
+		};
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_D_REVALIDATE_4ARGS], [
+	AC_MSG_CHECKING([whether dops->d_revalidate() takes 4 args])
+	ZFS_LINUX_TEST_RESULT([dentry_operations_d_revalidate_4args], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_D_REVALIDATE_4ARGS, 1,
+		    [dops->d_revalidate() takes 4 args])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SRC_AUTOMOUNT], [
+	ZFS_AC_KERNEL_SRC_D_AUTOMOUNT
+	ZFS_AC_KERNEL_SRC_D_REVALIDATE_4ARGS
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_AUTOMOUNT], [
+	ZFS_AC_KERNEL_D_AUTOMOUNT
+	ZFS_AC_KERNEL_D_REVALIDATE_4ARGS
+])
diff --git a/sys/contrib/openzfs/config/kernel-kthread.m4 b/sys/contrib/openzfs/config/kernel-kthread.m4
index 4d580efead6b..607953146323 100644
--- a/sys/contrib/openzfs/config/kernel-kthread.m4
+++ b/sys/contrib/openzfs/config/kernel-kthread.m4
@@ -1,91 +1,98 @@
 AC_DEFUN([ZFS_AC_KERNEL_KTHREAD_COMPLETE_AND_EXIT], [
 	dnl #
 	dnl # 5.17 API,
 	dnl # cead18552660702a4a46f58e65188fe5f36e9dfe ("exit: Rename complete_and_exit to kthread_complete_and_exit")
 	dnl #
 	dnl # Also moves the definition from include/linux/kernel.h to include/linux/kthread.h
 	dnl #
 	AC_MSG_CHECKING([whether kthread_complete_and_exit() is available])
 	ZFS_LINUX_TEST_RESULT([kthread_complete_and_exit], [
 		AC_MSG_RESULT(yes)
 		AC_DEFINE(SPL_KTHREAD_COMPLETE_AND_EXIT, kthread_complete_and_exit, [kthread_complete_and_exit() available])
 	], [
 		AC_MSG_RESULT(no)
 		AC_DEFINE(SPL_KTHREAD_COMPLETE_AND_EXIT, complete_and_exit, [using complete_and_exit() instead])
 	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_KTHREAD_DEQUEUE_SIGNAL], [
 	dnl #
-	dnl # 5.17 API: enum pid_type * as new 4th dequeue_signal() argument,
-	dnl # 5768d8906bc23d512b1a736c1e198aa833a6daa4 ("signal: Requeue signals in the appropriate queue")
+	dnl # prehistory:
+	dnl #     int dequeue_signal(struct task_struct *task, sigset_t *mask,
+	dnl #         siginfo_t *info)
 	dnl #
-	dnl # int dequeue_signal(struct task_struct *task, sigset_t *mask, kernel_siginfo_t *info);
-	dnl # int dequeue_signal(struct task_struct *task, sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type);
+	dnl # 4.20: kernel_siginfo_t introduced, replaces siginfo_t
+	dnl #     int dequeue_signal(struct task_struct *task, sigset_t *mask,
+	dnl           kernel_siginfo_t *info)
 	dnl #
-	dnl # 6.12 API: first arg struct_task* removed
-	dnl # int dequeue_signal(sigset_t *mask, kernel_siginfo_t *info, enum pid_type *type);
+	dnl # 5.17: enum pid_type introduced as 4th arg
+	dnl #     int dequeue_signal(struct task_struct *task, sigset_t *mask,
+	dnl #         kernel_siginfo_t *info, enum pid_type *type)
+	dnl #
+	dnl # 6.12: first arg struct_task* removed
+	dnl #     int dequeue_signal(sigset_t *mask, kernel_siginfo_t *info,
+	dnl #         enum pid_type *type)
 	dnl #
 	AC_MSG_CHECKING([whether dequeue_signal() takes 4 arguments])
 	ZFS_LINUX_TEST_RESULT([kthread_dequeue_signal_4arg], [
 		AC_MSG_RESULT(yes)
 		AC_DEFINE(HAVE_DEQUEUE_SIGNAL_4ARG, 1,
 		    [dequeue_signal() takes 4 arguments])
 	], [
 		AC_MSG_RESULT(no)
-		AC_MSG_CHECKING([whether dequeue_signal() a task argument])
-		ZFS_LINUX_TEST_RESULT([kthread_dequeue_signal_3arg_task], [
+		AC_MSG_CHECKING([whether 3-arg dequeue_signal() takes a type argument])
+		ZFS_LINUX_TEST_RESULT([kthread_dequeue_signal_3arg_type], [
 			AC_MSG_RESULT(yes)
-			AC_DEFINE(HAVE_DEQUEUE_SIGNAL_3ARG_TASK, 1,
-			    [dequeue_signal() takes a task argument])
+			AC_DEFINE(HAVE_DEQUEUE_SIGNAL_3ARG_TYPE, 1,
+			    [3-arg dequeue_signal() takes a type argument])
 		], [
 			AC_MSG_RESULT(no)
 		])
 	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD_COMPLETE_AND_EXIT], [
 	ZFS_LINUX_TEST_SRC([kthread_complete_and_exit], [
 		#include <linux/kthread.h>
 	], [
 		struct completion *completion = NULL;
 		long code = 0;
 
 		kthread_complete_and_exit(completion, code);
 	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD_DEQUEUE_SIGNAL], [
-	ZFS_LINUX_TEST_SRC([kthread_dequeue_signal_3arg_task], [
+	ZFS_LINUX_TEST_SRC([kthread_dequeue_signal_4arg], [
 		#include <linux/sched/signal.h>
 	], [
 		struct task_struct *task = NULL;
 		sigset_t *mask = NULL;
 		kernel_siginfo_t *info = NULL;
+		enum pid_type *type = NULL;
 		int error __attribute__ ((unused));
 
-		error = dequeue_signal(task, mask, info);
+		error = dequeue_signal(task, mask, info, type);
 	])
 
-	ZFS_LINUX_TEST_SRC([kthread_dequeue_signal_4arg], [
+	ZFS_LINUX_TEST_SRC([kthread_dequeue_signal_3arg_type], [
 		#include <linux/sched/signal.h>
 	], [
-		struct task_struct *task = NULL;
 		sigset_t *mask = NULL;
 		kernel_siginfo_t *info = NULL;
 		enum pid_type *type = NULL;
 		int error __attribute__ ((unused));
 
-		error = dequeue_signal(task, mask, info, type);
+		error = dequeue_signal(mask, info, type);
 	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_KTHREAD], [
 	ZFS_AC_KERNEL_KTHREAD_COMPLETE_AND_EXIT
 	ZFS_AC_KERNEL_KTHREAD_DEQUEUE_SIGNAL
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_SRC_KTHREAD], [
 	ZFS_AC_KERNEL_SRC_KTHREAD_COMPLETE_AND_EXIT
 	ZFS_AC_KERNEL_SRC_KTHREAD_DEQUEUE_SIGNAL
 ])
diff --git a/sys/contrib/openzfs/config/kernel-mkdir.m4 b/sys/contrib/openzfs/config/kernel-mkdir.m4
index 8e084443c7b4..c1aebc387abe 100644
--- a/sys/contrib/openzfs/config/kernel-mkdir.m4
+++ b/sys/contrib/openzfs/config/kernel-mkdir.m4
@@ -1,87 +1,114 @@
 dnl #
 dnl # Supported mkdir() interfaces checked newest to oldest.
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_MKDIR], [
+	dnl #
+	dnl # 6.15 API change
+	dnl # mkdir() returns struct dentry *
+	dnl #
+	ZFS_LINUX_TEST_SRC([mkdir_return_dentry], [
+		#include <linux/fs.h>
+
+		static struct dentry *mkdir(struct mnt_idmap *idmap,
+			struct inode *inode, struct dentry *dentry,
+			umode_t umode) { return dentry; }
+		static const struct inode_operations
+		    iops __attribute__ ((unused)) = {
+			.mkdir = mkdir,
+		};
+	],[])
+
 	dnl #
 	dnl # 6.3 API change
 	dnl # mkdir() takes struct mnt_idmap * as the first arg
 	dnl #
 	ZFS_LINUX_TEST_SRC([mkdir_mnt_idmap], [
 		#include <linux/fs.h>
 
 		static int mkdir(struct mnt_idmap *idmap,
 			struct inode *inode, struct dentry *dentry,
 			umode_t umode) { return 0; }
 		static const struct inode_operations
 		    iops __attribute__ ((unused)) = {
 			.mkdir = mkdir,
 		};
 	],[])
 
 	dnl #
 	dnl # 5.12 API change
 	dnl # The struct user_namespace arg was added as the first argument to
 	dnl # mkdir()
 	dnl #
 	ZFS_LINUX_TEST_SRC([mkdir_user_namespace], [
 		#include <linux/fs.h>
 
 		static int mkdir(struct user_namespace *userns,
 			struct inode *inode, struct dentry *dentry,
 		    umode_t umode) { return 0; }
 
 		static const struct inode_operations
 		    iops __attribute__ ((unused)) = {
 			.mkdir = mkdir,
 		};
 	],[])
 
 	dnl #
 	dnl # 3.3 API change
 	dnl # The VFS .create, .mkdir and .mknod callbacks were updated to take a
 	dnl # umode_t type rather than an int.  The expectation is that any backport
 	dnl # would also change all three prototypes.  However, if it turns out that
 	dnl # some distribution doesn't backport the whole thing this could be
 	dnl # broken apart into three separate checks.
 	dnl #
 	ZFS_LINUX_TEST_SRC([inode_operations_mkdir], [
 		#include <linux/fs.h>
 
 		static int mkdir(struct inode *inode, struct dentry *dentry,
 		    umode_t umode) { return 0; }
 
 		static const struct inode_operations
 		    iops __attribute__ ((unused)) = {
 			.mkdir = mkdir,
 		};
 	],[])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_MKDIR], [
 	dnl #
-	dnl # 6.3 API change
-	dnl # mkdir() takes struct mnt_idmap * as the first arg
+	dnl # 6.15 API change
+	dnl # mkdir() returns struct dentry *
 	dnl #
-	AC_MSG_CHECKING([whether iops->mkdir() takes struct mnt_idmap*])
-	ZFS_LINUX_TEST_RESULT([mkdir_mnt_idmap], [
+	AC_MSG_CHECKING([whether iops->mkdir() returns struct dentry*])
+	ZFS_LINUX_TEST_RESULT([mkdir_return_dentry], [
 		AC_MSG_RESULT(yes)
-		AC_DEFINE(HAVE_IOPS_MKDIR_IDMAP, 1,
-		    [iops->mkdir() takes struct mnt_idmap*])
+		AC_DEFINE(HAVE_IOPS_MKDIR_DENTRY, 1,
+		    [iops->mkdir() returns struct dentry*])
 	],[
-		AC_MSG_RESULT(no)
-
 		dnl #
-		dnl # 5.12 API change
-		dnl # The struct user_namespace arg was added as the first argument to
-		dnl # mkdir() of the iops structure.
+		dnl # 6.3 API change
+		dnl # mkdir() takes struct mnt_idmap * as the first arg
 		dnl #
-		AC_MSG_CHECKING([whether iops->mkdir() takes struct user_namespace*])
-		ZFS_LINUX_TEST_RESULT([mkdir_user_namespace], [
+		AC_MSG_CHECKING([whether iops->mkdir() takes struct mnt_idmap*])
+		ZFS_LINUX_TEST_RESULT([mkdir_mnt_idmap], [
 			AC_MSG_RESULT(yes)
-			AC_DEFINE(HAVE_IOPS_MKDIR_USERNS, 1,
-			    [iops->mkdir() takes struct user_namespace*])
+			AC_DEFINE(HAVE_IOPS_MKDIR_IDMAP, 1,
+			    [iops->mkdir() takes struct mnt_idmap*])
 		],[
 			AC_MSG_RESULT(no)
+
+			dnl #
+			dnl # 5.12 API change
+			dnl # The struct user_namespace arg was added as the first argument to
+			dnl # mkdir() of the iops structure.
+			dnl #
+			AC_MSG_CHECKING([whether iops->mkdir() takes struct user_namespace*])
+			ZFS_LINUX_TEST_RESULT([mkdir_user_namespace], [
+				AC_MSG_RESULT(yes)
+				AC_DEFINE(HAVE_IOPS_MKDIR_USERNS, 1,
+				    [iops->mkdir() takes struct user_namespace*])
+			],[
+				AC_MSG_RESULT(no)
+			])
 		])
 	])
 ])
diff --git a/sys/contrib/openzfs/config/kernel-objtool.m4 b/sys/contrib/openzfs/config/kernel-objtool.m4
index f9f9d657d805..e616ccebcbc0 100644
--- a/sys/contrib/openzfs/config/kernel-objtool.m4
+++ b/sys/contrib/openzfs/config/kernel-objtool.m4
@@ -1,71 +1,90 @@
 dnl #
 dnl # Detect objtool functionality.
 dnl #
 
 dnl #
 dnl # Kernel 5.10: linux/frame.h was renamed linux/objtool.h
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_OBJTOOL_HEADER], [
 	AC_MSG_CHECKING([whether objtool header is available])
 	ZFS_LINUX_TRY_COMPILE([
 		#include <linux/objtool.h>
 	],[
 	],[
+		objtool_header=$LINUX/include/linux/objtool.h
 		AC_DEFINE(HAVE_KERNEL_OBJTOOL_HEADER, 1,
 		    [kernel has linux/objtool.h])
 		AC_MSG_RESULT(linux/objtool.h)
 	],[
+		objtool_header=$LINUX/include/linux/frame.h
 		AC_MSG_RESULT(linux/frame.h)
 	])
 ])
 
 dnl #
 dnl # Check for objtool support.
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_SRC_OBJTOOL], [
 
 	dnl # 4.6 API for compile-time stack validation
 	ZFS_LINUX_TEST_SRC([objtool], [
 		#undef __ASSEMBLY__
 		#include <asm/ptrace.h>
 		#include <asm/frame.h>
 	],[
 		#if !defined(FRAME_BEGIN)
 		#error "FRAME_BEGIN is not defined"
 		#endif
 	])
 
 	dnl # 4.6 API added STACK_FRAME_NON_STANDARD macro
 	ZFS_LINUX_TEST_SRC([stack_frame_non_standard], [
 		#ifdef HAVE_KERNEL_OBJTOOL_HEADER
 		#include <linux/objtool.h>
 		#else
 		#include <linux/frame.h>
 		#endif
 	],[
 		#if !defined(STACK_FRAME_NON_STANDARD)
 		#error "STACK_FRAME_NON_STANDARD is not defined."
 		#endif
 	])
 ])
 
 AC_DEFUN([ZFS_AC_KERNEL_OBJTOOL], [
 	AC_MSG_CHECKING(
 	    [whether compile-time stack validation (objtool) is available])
 	ZFS_LINUX_TEST_RESULT([objtool], [
 		AC_MSG_RESULT(yes)
 		AC_DEFINE(HAVE_KERNEL_OBJTOOL, 1,
 		    [kernel does stack verification])
 
 		AC_MSG_CHECKING([whether STACK_FRAME_NON_STANDARD is defined])
 		ZFS_LINUX_TEST_RESULT([stack_frame_non_standard], [
 			AC_MSG_RESULT(yes)
 			AC_DEFINE(HAVE_STACK_FRAME_NON_STANDARD, 1,
 			   [STACK_FRAME_NON_STANDARD is defined])
+
+			dnl # Needed for kernels missing the asm macro. We grep
+			dnl # for it in the header file since there is currently
+			dnl # no test to check the result of assembling a file.
+			AC_MSG_CHECKING(
+			    [whether STACK_FRAME_NON_STANDARD asm macro is defined])
+			dnl # Escape square brackets.
+			sp='@<:@@<:@:space:@:>@@:>@'
+			dotmacro='@<:@.@:>@macro'
+			regexp="^$sp*$dotmacro$sp+STACK_FRAME_NON_STANDARD$sp"
+			AS_IF([$EGREP -s -q "$regexp" $objtool_header],[
+				AC_MSG_RESULT(yes)
+				AC_DEFINE(HAVE_STACK_FRAME_NON_STANDARD_ASM, 1,
+				   [STACK_FRAME_NON_STANDARD asm macro is defined])
+			],[
+				AC_MSG_RESULT(no)
+			])
 		],[
 			AC_MSG_RESULT(no)
 		])
 	],[
 		AC_MSG_RESULT(no)
 	])
 ])
diff --git a/sys/contrib/openzfs/config/kernel-sb-dying.m4 b/sys/contrib/openzfs/config/kernel-sb-dying.m4
new file mode 100644
index 000000000000..882f3e542357
--- /dev/null
+++ b/sys/contrib/openzfs/config/kernel-sb-dying.m4
@@ -0,0 +1,19 @@
+dnl #
+dnl # SB_DYING exists since Linux 6.6
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_SB_DYING], [
+	ZFS_LINUX_TEST_SRC([sb_dying], [
+		#include <linux/fs.h>
+	],[
+		(void) SB_DYING;
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SB_DYING], [
+	AC_MSG_CHECKING([whether SB_DYING is defined])
+	ZFS_LINUX_TEST_RESULT([sb_dying], [
+		AC_MSG_RESULT(yes)
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
diff --git a/sys/contrib/openzfs/config/kernel-timer.m4 b/sys/contrib/openzfs/config/kernel-timer.m4
new file mode 100644
index 000000000000..c89ea204e83d
--- /dev/null
+++ b/sys/contrib/openzfs/config/kernel-timer.m4
@@ -0,0 +1,32 @@
+dnl #
+dnl # 6.2: timer_delete_sync introduced, del_timer_sync deprecated and made
+dnl #      into a simple wrapper
+dnl # 6.15: del_timer_sync removed
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_TIMER_DELETE_SYNC], [
+	ZFS_LINUX_TEST_SRC([timer_delete_sync], [
+		#include <linux/timer.h>
+	],[
+		struct timer_list *timer __attribute__((unused)) = NULL;
+		timer_delete_sync(timer);
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_TIMER_DELETE_SYNC], [
+	AC_MSG_CHECKING([whether timer_delete_sync() is available])
+	ZFS_LINUX_TEST_RESULT([timer_delete_sync], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_TIMER_DELETE_SYNC, 1,
+		    [timer_delete_sync is available])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_SRC_TIMER], [
+	ZFS_AC_KERNEL_SRC_TIMER_DELETE_SYNC
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_TIMER], [
+	ZFS_AC_KERNEL_TIMER_DELETE_SYNC
+])
diff --git a/sys/contrib/openzfs/config/kernel-vfs-migrate_folio.m4 b/sys/contrib/openzfs/config/kernel-vfs-migrate_folio.m4
new file mode 100644
index 000000000000..186cd0581a17
--- /dev/null
+++ b/sys/contrib/openzfs/config/kernel-vfs-migrate_folio.m4
@@ -0,0 +1,27 @@
+dnl #
+dnl # Linux 6.0 uses migrate_folio in lieu of migrate_page
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_MIGRATE_FOLIO], [
+	ZFS_LINUX_TEST_SRC([vfs_has_migrate_folio], [
+		#include <linux/fs.h>
+		#include <linux/migrate.h>
+
+		static const struct address_space_operations
+		    aops __attribute__ ((unused)) = {
+			.migrate_folio	= migrate_folio,
+		};
+	],[])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_VFS_MIGRATE_FOLIO], [
+	dnl #
+	dnl # Linux 6.0 uses migrate_folio in lieu of migrate_page
+	dnl #
+	AC_MSG_CHECKING([whether migrate_folio exists])
+	ZFS_LINUX_TEST_RESULT([vfs_has_migrate_folio], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE(HAVE_VFS_MIGRATE_FOLIO, 1, [migrate_folio exists])
+	],[
+		AC_MSG_RESULT([no])
+	])
+])
diff --git a/sys/contrib/openzfs/config/kernel-vfs-migratepage.m4 b/sys/contrib/openzfs/config/kernel-vfs-migratepage.m4
new file mode 100644
index 000000000000..05db3af511eb
--- /dev/null
+++ b/sys/contrib/openzfs/config/kernel-vfs-migratepage.m4
@@ -0,0 +1,27 @@
+dnl #
+dnl # Linux 6.0 gets rid of address_space_operations.migratepage
+dnl #
+AC_DEFUN([ZFS_AC_KERNEL_SRC_VFS_MIGRATEPAGE], [
+	ZFS_LINUX_TEST_SRC([vfs_has_migratepage], [
+		#include <linux/fs.h>
+		#include <linux/migrate.h>
+
+		static const struct address_space_operations
+		    aops __attribute__ ((unused)) = {
+			.migratepage	= migrate_page,
+		};
+	],[])
+])
+
+AC_DEFUN([ZFS_AC_KERNEL_VFS_MIGRATEPAGE], [
+	dnl #
+	dnl # Linux 6.0 gets rid of address_space_operations.migratepage
+	dnl #
+	AC_MSG_CHECKING([whether migratepage exists])
+	ZFS_LINUX_TEST_RESULT([vfs_has_migratepage], [
+		AC_MSG_RESULT([yes])
+		AC_DEFINE(HAVE_VFS_MIGRATEPAGE, 1, [migratepage exists])
+	],[
+		AC_MSG_RESULT([no])
+	])
+])
diff --git a/sys/contrib/openzfs/config/kernel.m4 b/sys/contrib/openzfs/config/kernel.m4
index df3bf5293529..f0a4dc0fe430 100644
--- a/sys/contrib/openzfs/config/kernel.m4
+++ b/sys/contrib/openzfs/config/kernel.m4
@@ -1,966 +1,979 @@
 dnl #
 dnl # Default ZFS kernel configuration
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_KERNEL], [
 	AM_COND_IF([BUILD_LINUX], [
 		dnl # Setup the kernel build environment.
 		ZFS_AC_KERNEL
 		ZFS_AC_QAT
 
 		dnl # Sanity checks for module building and CONFIG_* defines
 		ZFS_AC_KERNEL_CONFIG_DEFINED
 		ZFS_AC_MODULE_SYMVERS
 
 		dnl # Sequential ZFS_LINUX_TRY_COMPILE tests
 		ZFS_AC_KERNEL_FPU_HEADER
 		ZFS_AC_KERNEL_OBJTOOL_HEADER
 		ZFS_AC_KERNEL_MISC_MINOR
 		ZFS_AC_KERNEL_DECLARE_EVENT_CLASS
 
 		dnl # Parallel ZFS_LINUX_TEST_SRC / ZFS_LINUX_TEST_RESULT tests
 		ZFS_AC_KERNEL_TEST_SRC
 		ZFS_AC_KERNEL_TEST_RESULT
 
 		AS_IF([test "$LINUX_OBJ" != "$LINUX"], [
 			KERNEL_MAKE="$KERNEL_MAKE O=$LINUX_OBJ"
 		])
 
 		AC_SUBST(KERNEL_MAKE)
 	])
 ])
 
 dnl #
 dnl # Generate and compile all of the kernel API test cases to determine
 dnl # which interfaces are available.  By invoking the kernel build system
 dnl # only once the compilation can be done in parallel significantly
 dnl # speeding up the process.
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_TEST_SRC], [
 	ZFS_AC_KERNEL_SRC_TYPES
 	ZFS_AC_KERNEL_SRC_OBJTOOL
 	ZFS_AC_KERNEL_SRC_ACCESS_OK_TYPE
 	ZFS_AC_KERNEL_SRC_PDE_DATA
 	ZFS_AC_KERNEL_SRC_GENERIC_FADVISE
 	ZFS_AC_KERNEL_SRC_SCHED
 	ZFS_AC_KERNEL_SRC_USLEEP_RANGE
 	ZFS_AC_KERNEL_SRC_VMALLOC_PAGE_KERNEL
 	ZFS_AC_KERNEL_SRC_INODE_TIMES
 	ZFS_AC_KERNEL_SRC_PROC_OPERATIONS
 	ZFS_AC_KERNEL_SRC_BLOCK_DEVICE_OPERATIONS
 	ZFS_AC_KERNEL_SRC_BIO
 	ZFS_AC_KERNEL_SRC_BLKDEV
 	ZFS_AC_KERNEL_SRC_BLK_QUEUE
 	ZFS_AC_KERNEL_SRC_GENHD_FLAGS
 	ZFS_AC_KERNEL_SRC_REVALIDATE_DISK
 	ZFS_AC_KERNEL_SRC_GET_DISK_RO
 	ZFS_AC_KERNEL_SRC_DISCARD_GRANULARITY
 	ZFS_AC_KERNEL_SRC_INODE_OWNER_OR_CAPABLE
 	ZFS_AC_KERNEL_SRC_XATTR
 	ZFS_AC_KERNEL_SRC_ACL
 	ZFS_AC_KERNEL_SRC_INODE_SETATTR
 	ZFS_AC_KERNEL_SRC_INODE_GETATTR
 	ZFS_AC_KERNEL_SRC_SHOW_OPTIONS
 	ZFS_AC_KERNEL_SRC_SHRINKER
 	ZFS_AC_KERNEL_SRC_MKDIR
 	ZFS_AC_KERNEL_SRC_LOOKUP_FLAGS
 	ZFS_AC_KERNEL_SRC_CREATE
 	ZFS_AC_KERNEL_SRC_PERMISSION
 	ZFS_AC_KERNEL_SRC_TMPFILE
 	ZFS_AC_KERNEL_SRC_AUTOMOUNT
 	ZFS_AC_KERNEL_SRC_COMMIT_METADATA
 	ZFS_AC_KERNEL_SRC_SETATTR_PREPARE
 	ZFS_AC_KERNEL_SRC_INSERT_INODE_LOCKED
 	ZFS_AC_KERNEL_SRC_TRUNCATE_SETSIZE
 	ZFS_AC_KERNEL_SRC_SECURITY_INODE
 	ZFS_AC_KERNEL_SRC_FST_MOUNT
+	ZFS_AC_KERNEL_SRC_SB_DYING
 	ZFS_AC_KERNEL_SRC_SET_NLINK
 	ZFS_AC_KERNEL_SRC_SGET
 	ZFS_AC_KERNEL_SRC_VFS_FILEMAP_DIRTY_FOLIO
 	ZFS_AC_KERNEL_SRC_VFS_READ_FOLIO
+	ZFS_AC_KERNEL_SRC_VFS_MIGRATE_FOLIO
+	ZFS_AC_KERNEL_SRC_VFS_MIGRATEPAGE
 	ZFS_AC_KERNEL_SRC_VFS_FSYNC_2ARGS
 	ZFS_AC_KERNEL_SRC_VFS_DIRECT_IO
 	ZFS_AC_KERNEL_SRC_VFS_READPAGES
 	ZFS_AC_KERNEL_SRC_VFS_SET_PAGE_DIRTY_NOBUFFERS
 	ZFS_AC_KERNEL_SRC_VFS_IOV_ITER
 	ZFS_AC_KERNEL_SRC_VFS_GENERIC_COPY_FILE_RANGE
 	ZFS_AC_KERNEL_SRC_VFS_SPLICE_COPY_FILE_RANGE
 	ZFS_AC_KERNEL_SRC_VFS_REMAP_FILE_RANGE
 	ZFS_AC_KERNEL_SRC_VFS_CLONE_FILE_RANGE
 	ZFS_AC_KERNEL_SRC_VFS_DEDUPE_FILE_RANGE
 	ZFS_AC_KERNEL_SRC_KMAP_ATOMIC_ARGS
 	ZFS_AC_KERNEL_SRC_KMAP_LOCAL_PAGE
 	ZFS_AC_KERNEL_SRC_FOLLOW_DOWN_ONE
 	ZFS_AC_KERNEL_SRC_MAKE_REQUEST_FN
 	ZFS_AC_KERNEL_SRC_GENERIC_IO_ACCT
 	ZFS_AC_KERNEL_SRC_FPU
 	ZFS_AC_KERNEL_SRC_FMODE_T
 	ZFS_AC_KERNEL_SRC_KUIDGID_T
 	ZFS_AC_KERNEL_SRC_KUID_HELPERS
 	ZFS_AC_KERNEL_SRC_RENAME
 	ZFS_AC_KERNEL_SRC_TOTALRAM_PAGES_FUNC
 	ZFS_AC_KERNEL_SRC_TOTALHIGH_PAGES
 	ZFS_AC_KERNEL_SRC_PERCPU
 	ZFS_AC_KERNEL_SRC_GENERIC_FILLATTR
 	ZFS_AC_KERNEL_SRC_MKNOD
 	ZFS_AC_KERNEL_SRC_SYMLINK
 	ZFS_AC_KERNEL_SRC_BIO_MAX_SEGS
 	ZFS_AC_KERNEL_SRC_SIGINFO
 	ZFS_AC_KERNEL_SRC_SYSFS
 	ZFS_AC_KERNEL_SRC_STANDALONE_LINUX_STDARG
 	ZFS_AC_KERNEL_SRC_STRLCPY
 	ZFS_AC_KERNEL_SRC_PAGEMAP_FOLIO_WAIT_BIT
 	ZFS_AC_KERNEL_SRC_ADD_DISK
 	ZFS_AC_KERNEL_SRC_KTHREAD
 	ZFS_AC_KERNEL_SRC_ZERO_PAGE
 	ZFS_AC_KERNEL_SRC___COPY_FROM_USER_INATOMIC
 	ZFS_AC_KERNEL_SRC_IDMAP_MNT_API
 	ZFS_AC_KERNEL_SRC_IDMAP_NO_USERNS
 	ZFS_AC_KERNEL_SRC_IATTR_VFSID
 	ZFS_AC_KERNEL_SRC_WRITEPAGE_T
 	ZFS_AC_KERNEL_SRC_RECLAIMED
 	ZFS_AC_KERNEL_SRC_REGISTER_SYSCTL_TABLE
 	ZFS_AC_KERNEL_SRC_REGISTER_SYSCTL_SZ
 	ZFS_AC_KERNEL_SRC_PROC_HANDLER_CTL_TABLE_CONST
 	ZFS_AC_KERNEL_SRC_COPY_SPLICE_READ
 	ZFS_AC_KERNEL_SRC_SYNC_BDEV
 	ZFS_AC_KERNEL_SRC_MM_PAGE_FLAGS
 	ZFS_AC_KERNEL_SRC_MM_PAGE_SIZE
 	ZFS_AC_KERNEL_SRC_MM_PAGE_MAPPING
 	ZFS_AC_KERNEL_SRC_FILE
+	ZFS_AC_KERNEL_SRC_TIMER
 	case "$host_cpu" in
 		powerpc*)
 			ZFS_AC_KERNEL_SRC_CPU_HAS_FEATURE
 			ZFS_AC_KERNEL_SRC_FLUSH_DCACHE_PAGE
 			;;
 		riscv*)
 			ZFS_AC_KERNEL_SRC_FLUSH_DCACHE_PAGE
 			;;
 	esac
 
 	AC_MSG_CHECKING([for available kernel interfaces])
 	ZFS_LINUX_TEST_COMPILE_ALL([kabi])
 	AC_MSG_RESULT([done])
 ])
 
 dnl #
 dnl # Check results of kernel interface tests.
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL_TEST_RESULT], [
 	ZFS_AC_KERNEL_TYPES
 	ZFS_AC_KERNEL_ACCESS_OK_TYPE
 	ZFS_AC_KERNEL_OBJTOOL
 	ZFS_AC_KERNEL_PDE_DATA
 	ZFS_AC_KERNEL_GENERIC_FADVISE
 	ZFS_AC_KERNEL_SCHED
 	ZFS_AC_KERNEL_USLEEP_RANGE
 	ZFS_AC_KERNEL_VMALLOC_PAGE_KERNEL
 	ZFS_AC_KERNEL_INODE_TIMES
 	ZFS_AC_KERNEL_PROC_OPERATIONS
 	ZFS_AC_KERNEL_BLOCK_DEVICE_OPERATIONS
 	ZFS_AC_KERNEL_BIO
 	ZFS_AC_KERNEL_BLKDEV
 	ZFS_AC_KERNEL_BLK_QUEUE
 	ZFS_AC_KERNEL_GENHD_FLAGS
 	ZFS_AC_KERNEL_REVALIDATE_DISK
 	ZFS_AC_KERNEL_GET_DISK_RO
 	ZFS_AC_KERNEL_DISCARD_GRANULARITY
 	ZFS_AC_KERNEL_INODE_OWNER_OR_CAPABLE
 	ZFS_AC_KERNEL_XATTR
 	ZFS_AC_KERNEL_ACL
 	ZFS_AC_KERNEL_INODE_SETATTR
 	ZFS_AC_KERNEL_INODE_GETATTR
 	ZFS_AC_KERNEL_SHOW_OPTIONS
 	ZFS_AC_KERNEL_SHRINKER
 	ZFS_AC_KERNEL_MKDIR
 	ZFS_AC_KERNEL_LOOKUP_FLAGS
 	ZFS_AC_KERNEL_CREATE
 	ZFS_AC_KERNEL_PERMISSION
 	ZFS_AC_KERNEL_TMPFILE
 	ZFS_AC_KERNEL_AUTOMOUNT
 	ZFS_AC_KERNEL_COMMIT_METADATA
 	ZFS_AC_KERNEL_SETATTR_PREPARE
 	ZFS_AC_KERNEL_INSERT_INODE_LOCKED
 	ZFS_AC_KERNEL_TRUNCATE_SETSIZE
 	ZFS_AC_KERNEL_SECURITY_INODE
 	ZFS_AC_KERNEL_FST_MOUNT
+	ZFS_AC_KERNEL_SB_DYING
 	ZFS_AC_KERNEL_SET_NLINK
 	ZFS_AC_KERNEL_SGET
 	ZFS_AC_KERNEL_VFS_FILEMAP_DIRTY_FOLIO
 	ZFS_AC_KERNEL_VFS_READ_FOLIO
+	ZFS_AC_KERNEL_VFS_MIGRATE_FOLIO
+	ZFS_AC_KERNEL_VFS_MIGRATEPAGE
 	ZFS_AC_KERNEL_VFS_FSYNC_2ARGS
 	ZFS_AC_KERNEL_VFS_DIRECT_IO
 	ZFS_AC_KERNEL_VFS_READPAGES
 	ZFS_AC_KERNEL_VFS_SET_PAGE_DIRTY_NOBUFFERS
 	ZFS_AC_KERNEL_VFS_IOV_ITER
 	ZFS_AC_KERNEL_VFS_GENERIC_COPY_FILE_RANGE
 	ZFS_AC_KERNEL_VFS_SPLICE_COPY_FILE_RANGE
 	ZFS_AC_KERNEL_VFS_REMAP_FILE_RANGE
 	ZFS_AC_KERNEL_VFS_CLONE_FILE_RANGE
 	ZFS_AC_KERNEL_VFS_DEDUPE_FILE_RANGE
 	ZFS_AC_KERNEL_KMAP_ATOMIC_ARGS
 	ZFS_AC_KERNEL_KMAP_LOCAL_PAGE
 	ZFS_AC_KERNEL_FOLLOW_DOWN_ONE
 	ZFS_AC_KERNEL_MAKE_REQUEST_FN
 	ZFS_AC_KERNEL_GENERIC_IO_ACCT
 	ZFS_AC_KERNEL_FPU
 	ZFS_AC_KERNEL_FMODE_T
 	ZFS_AC_KERNEL_KUIDGID_T
 	ZFS_AC_KERNEL_KUID_HELPERS
 	ZFS_AC_KERNEL_RENAME
 	ZFS_AC_KERNEL_TOTALRAM_PAGES_FUNC
 	ZFS_AC_KERNEL_TOTALHIGH_PAGES
 	ZFS_AC_KERNEL_PERCPU
 	ZFS_AC_KERNEL_GENERIC_FILLATTR
 	ZFS_AC_KERNEL_MKNOD
 	ZFS_AC_KERNEL_SYMLINK
 	ZFS_AC_KERNEL_BIO_MAX_SEGS
 	ZFS_AC_KERNEL_SIGINFO
 	ZFS_AC_KERNEL_SYSFS
 	ZFS_AC_KERNEL_STANDALONE_LINUX_STDARG
 	ZFS_AC_KERNEL_STRLCPY
 	ZFS_AC_KERNEL_PAGEMAP_FOLIO_WAIT_BIT
 	ZFS_AC_KERNEL_ADD_DISK
 	ZFS_AC_KERNEL_KTHREAD
 	ZFS_AC_KERNEL_ZERO_PAGE
 	ZFS_AC_KERNEL___COPY_FROM_USER_INATOMIC
 	ZFS_AC_KERNEL_IDMAP_MNT_API
 	ZFS_AC_KERNEL_IDMAP_NO_USERNS
 	ZFS_AC_KERNEL_IATTR_VFSID
 	ZFS_AC_KERNEL_WRITEPAGE_T
 	ZFS_AC_KERNEL_RECLAIMED
 	ZFS_AC_KERNEL_REGISTER_SYSCTL_TABLE
 	ZFS_AC_KERNEL_REGISTER_SYSCTL_SZ
 	ZFS_AC_KERNEL_PROC_HANDLER_CTL_TABLE_CONST
 	ZFS_AC_KERNEL_COPY_SPLICE_READ
 	ZFS_AC_KERNEL_SYNC_BDEV
 	ZFS_AC_KERNEL_MM_PAGE_FLAGS
 	ZFS_AC_KERNEL_MM_PAGE_SIZE
 	ZFS_AC_KERNEL_MM_PAGE_MAPPING
 	ZFS_AC_KERNEL_1ARG_ASSIGN_STR
 	ZFS_AC_KERNEL_FILE
+	ZFS_AC_KERNEL_TIMER
 	case "$host_cpu" in
 		powerpc*)
 			ZFS_AC_KERNEL_CPU_HAS_FEATURE
 			ZFS_AC_KERNEL_FLUSH_DCACHE_PAGE
 			;;
 		riscv*)
 			ZFS_AC_KERNEL_FLUSH_DCACHE_PAGE
 			;;
 	esac
 ])
 
 dnl #
 dnl # Detect name used for Module.symvers file in kernel
 dnl #
 AC_DEFUN([ZFS_AC_MODULE_SYMVERS], [
 	modpost=$LINUX/scripts/Makefile.modpost
 	AC_MSG_CHECKING([kernel file name for module symbols])
 	AS_IF([test "x$enable_linux_builtin" != xyes -a -f "$modpost"], [
 		AS_IF([grep -q Modules.symvers $modpost], [
 			LINUX_SYMBOLS=Modules.symvers
 		], [
 			LINUX_SYMBOLS=Module.symvers
 		])
 
 		AS_IF([test ! -f "$LINUX_OBJ/$LINUX_SYMBOLS"], [
 			AC_MSG_ERROR([
 	*** Please make sure the kernel devel package for your distribution
 	*** is installed.  If you are building with a custom kernel, make sure
 	*** the kernel is configured, built, and the '--with-linux=PATH'
 	*** configure option refers to the location of the kernel source.
 			])
 		])
 	], [
 		LINUX_SYMBOLS=NONE
 	])
 	AC_MSG_RESULT($LINUX_SYMBOLS)
 	AC_SUBST(LINUX_SYMBOLS)
 ])
 
 dnl #
 dnl # Detect the kernel to be built against
 dnl #
 dnl # Most modern Linux distributions have separate locations for bare
 dnl # source (source) and prebuilt (build) files. Additionally, there are
 dnl # `source` and `build` symlinks in `/lib/modules/$(KERNEL_VERSION)`
 dnl # pointing to them. The directory search order is now:
 dnl # 
 dnl # - `configure` command line values if both `--with-linux` and
 dnl #   `--with-linux-obj` were defined
 dnl # 
 dnl # - If only `--with-linux` was defined, `--with-linux-obj` is assumed
 dnl #   to have the same value as `--with-linux`
 dnl # 
 dnl # - If neither `--with-linux` nor `--with-linux-obj` were defined
 dnl #   autodetection is used:
 dnl # 
 dnl #   - `/lib/modules/$(uname -r)/{source,build}` respectively, if exist.
 dnl # 
 dnl #   - If only `/lib/modules/$(uname -r)/build` exists, it is assumed
 dnl #     to be both source and build directory.
 dnl # 
 dnl #   - The first directory in `/lib/modules` with the highest version
 dnl #     number according to `sort -V` which contains both `source` and
 dnl #     `build` symlinks/directories. If module directory contains only
 dnl #     `build` component, it is assumed to be both source and build
 dnl #     directory.
 dnl # 
 dnl #   - Last resort: the first directory matching `/usr/src/kernels/*`
 dnl #     and `/usr/src/linux-*` with the highest version number according
 dnl #     to `sort -V` is assumed to be both source and build directory.
 dnl #
 AC_DEFUN([ZFS_AC_KERNEL], [
 	AC_ARG_WITH([linux],
 		AS_HELP_STRING([--with-linux=PATH],
 		[Path to kernel source]),
 		[kernelsrc="$withval"])
 
 	AC_ARG_WITH(linux-obj,
 		AS_HELP_STRING([--with-linux-obj=PATH],
 		[Path to kernel build objects]),
 		[kernelbuild="$withval"])
 
 	AC_MSG_CHECKING([kernel source and build directories])
 	AS_IF([test -n "$kernelsrc" && test -z "$kernelbuild"], [
 		kernelbuild="$kernelsrc"
 	], [test -z "$kernelsrc"], [
 		AS_IF([test -e "/lib/modules/$(uname -r)/source" && \
 		       test -e "/lib/modules/$(uname -r)/build"], [
 			src="/lib/modules/$(uname -r)/source"
 			build="/lib/modules/$(uname -r)/build"
 		], [test -e "/lib/modules/$(uname -r)/build"], [
 			build="/lib/modules/$(uname -r)/build"
 			src="$build"
 		], [
 			src=
 
 			for d in $(ls -1d /lib/modules/* 2>/dev/null | sort -Vr); do
 				if test -e "$d/source" && test -e "$d/build"; then
 					src="$d/source"
 					build="$d/build"
 					break
 				fi
 
 				if test -e "$d/build"; then
 					src="$d/build"
 					build="$d/build"
 					break
 				fi
 			done
 
 			# the least reliable method
 			if test -z "$src"; then
 				src=$(ls -1d /usr/src/kernels/* /usr/src/linux-* \
 				      2>/dev/null | grep -v obj | sort -Vr | head -1)
 				build="$src"
 			fi
 		])
 
 		AS_IF([test -n "$src" && test -e "$src"], [
 			kernelsrc=$(readlink -e "$src")
 		], [
 			kernelsrc="[Not found]"
 		])
 		AS_IF([test -n "$build" && test -e "$build"], [
 			kernelbuild=$(readlink -e "$build")
 		], [
 			kernelbuild="[Not found]"
 		])
 	], [
 		AS_IF([test "$kernelsrc" = "NONE"], [
 			kernsrcver=NONE
 		])
 		withlinux=yes
 	])
 
 	AC_MSG_RESULT([done])
 	AC_MSG_CHECKING([kernel source directory])
 	AC_MSG_RESULT([$kernelsrc])
 	AC_MSG_CHECKING([kernel build directory])
 	AC_MSG_RESULT([$kernelbuild])
 	AS_IF([test ! -d "$kernelsrc" || test ! -d "$kernelbuild"], [
 		AC_MSG_ERROR([
 	*** Please make sure the kernel devel package for your distribution
 	*** is installed and then try again.  If that fails, you can specify the
 	*** location of the kernel source and build with the '--with-linux=PATH' and
 	*** '--with-linux-obj=PATH' options respectively.])
 	])
 
 	AC_MSG_CHECKING([kernel source version])
 	utsrelease1=$kernelbuild/include/linux/version.h
 	utsrelease2=$kernelbuild/include/linux/utsrelease.h
 	utsrelease3=$kernelbuild/include/generated/utsrelease.h
 	AS_IF([test -r $utsrelease1 && grep -qF UTS_RELEASE $utsrelease1], [
 		utsrelease=$utsrelease1
 	], [test -r $utsrelease2 && grep -qF UTS_RELEASE $utsrelease2], [
 		utsrelease=$utsrelease2
 	], [test -r $utsrelease3 && grep -qF UTS_RELEASE $utsrelease3], [
 		utsrelease=$utsrelease3
 	])
 
 	AS_IF([test -n "$utsrelease"], [
 		kernsrcver=$($AWK '/UTS_RELEASE/ { gsub(/"/, "", $[3]); print $[3] }' $utsrelease)
 		AS_IF([test -z "$kernsrcver"], [
 			AC_MSG_RESULT([Not found])
 			AC_MSG_ERROR([
 	*** Cannot determine kernel version.
 			])
 		])
 	], [
 		AC_MSG_RESULT([Not found])
 		if test "x$enable_linux_builtin" != xyes; then
 			AC_MSG_ERROR([
 	*** Cannot find UTS_RELEASE definition.
 			])
 		else
 			AC_MSG_ERROR([
 	*** Cannot find UTS_RELEASE definition.
 	*** Please run 'make prepare' inside the kernel source tree.])
 		fi
 	])
 
 	AC_MSG_RESULT([$kernsrcver])
 
 	AS_VERSION_COMPARE([$kernsrcver], [$ZFS_META_KVER_MIN], [
 		 AC_MSG_ERROR([
 	*** Cannot build against kernel version $kernsrcver.
 	*** The minimum supported kernel version is $ZFS_META_KVER_MIN.
 		])
 	])
 
 	LINUX=${kernelsrc}
 	LINUX_OBJ=${kernelbuild}
 	LINUX_VERSION=${kernsrcver}
 
 	AC_SUBST(LINUX)
 	AC_SUBST(LINUX_OBJ)
 	AC_SUBST(LINUX_VERSION)
 ])
 
 dnl #
 dnl # Detect the QAT module to be built against, QAT provides hardware
 dnl # acceleration for data compression:
 dnl #
 dnl # https://01.org/intel-quickassist-technology
 dnl #
 dnl # 1) Download and install QAT driver from the above link
 dnl # 2) Start QAT driver in your system:
 dnl # 	 service qat_service start
 dnl # 3) Enable QAT in ZFS, e.g.:
 dnl # 	 ./configure --with-qat=<qat-driver-path>/QAT1.6
 dnl # 	 make
 dnl # 4) Set GZIP compression in ZFS dataset:
 dnl # 	 zfs set compression = gzip <dataset>
 dnl #
 dnl # Then the data written to this ZFS pool is compressed by QAT accelerator
 dnl # automatically, and de-compressed by QAT when read from the pool.
 dnl #
 dnl # 1) Get QAT hardware statistics with:
 dnl #	 cat /proc/icp_dh895xcc_dev/qat
 dnl # 2) To disable QAT:
 dnl # 	 insmod zfs.ko zfs_qat_disable=1
 dnl #
 AC_DEFUN([ZFS_AC_QAT], [
 	AC_ARG_WITH([qat],
 		AS_HELP_STRING([--with-qat=PATH],
 		[Path to qat source]),
 		AS_IF([test "$withval" = "yes"],
 			AC_MSG_ERROR([--with-qat=PATH requires a PATH]),
 			[qatsrc="$withval"]))
 
 	AC_ARG_WITH([qat-obj],
 		AS_HELP_STRING([--with-qat-obj=PATH],
 		[Path to qat build objects]),
 		[qatbuild="$withval"])
 
 	AS_IF([test ! -z "${qatsrc}"], [
 		AC_MSG_CHECKING([qat source directory])
 		AC_MSG_RESULT([$qatsrc])
 		QAT_SRC="${qatsrc}/quickassist"
 		AS_IF([ test ! -e "$QAT_SRC/include/cpa.h"], [
 			AC_MSG_ERROR([
 	*** Please make sure the qat driver package is installed
 	*** and specify the location of the qat source with the
 	*** '--with-qat=PATH' option then try again. Failed to
 	*** find cpa.h in:
 	${QAT_SRC}/include])
 		])
 	])
 
 	AS_IF([test ! -z "${qatsrc}"], [
 		AC_MSG_CHECKING([qat build directory])
 		AS_IF([test -z "$qatbuild"], [
 			qatbuild="${qatsrc}/build"
 		])
 
 		AC_MSG_RESULT([$qatbuild])
 		QAT_OBJ=${qatbuild}
 		AS_IF([ ! test -e "$QAT_OBJ/icp_qa_al.ko" && ! test -e "$QAT_OBJ/qat_api.ko"], [
 			AC_MSG_ERROR([
 	*** Please make sure the qat driver is installed then try again.
 	*** Failed to find icp_qa_al.ko or qat_api.ko in:
 	$QAT_OBJ])
 		])
 
 		AC_SUBST(QAT_SRC)
 		AC_SUBST(QAT_OBJ)
 
 		AC_DEFINE(HAVE_QAT, 1,
 		[qat is enabled and existed])
 	])
 
 	dnl #
 	dnl # Detect the name used for the QAT Module.symvers file.
 	dnl #
 	AS_IF([test ! -z "${qatsrc}"], [
 		AC_MSG_CHECKING([qat file for module symbols])
 		QAT_SYMBOLS=$QAT_SRC/lookaside/access_layer/src/Module.symvers
 
 		AS_IF([test -r $QAT_SYMBOLS], [
 			AC_MSG_RESULT([$QAT_SYMBOLS])
 			AC_SUBST(QAT_SYMBOLS)
 		],[
 			AC_MSG_ERROR([
 	*** Please make sure the qat driver is installed then try again.
 	*** Failed to find Module.symvers in:
 	$QAT_SYMBOLS
 			])
 		])
 	])
 ])
 
 dnl #
 dnl # ZFS_LINUX_CONFTEST_H
 dnl #
 AC_DEFUN([ZFS_LINUX_CONFTEST_H], [
 test -d build/$2 || mkdir -p build/$2
 cat - <<_ACEOF >build/$2/$2.h
 $1
 _ACEOF
 ])
 
 dnl #
 dnl # ZFS_LINUX_CONFTEST_C
 dnl #
 AC_DEFUN([ZFS_LINUX_CONFTEST_C], [
 test -d build/$2 || mkdir -p build/$2
 cat confdefs.h - <<_ACEOF >build/$2/$2.c
 $1
 _ACEOF
 ])
 
 dnl #
 dnl # ZFS_LINUX_CONFTEST_MAKEFILE
 dnl #
 dnl # $1 - test case name
 dnl # $2 - add to top-level Makefile
 dnl # $3 - additional build flags
 dnl #
 AC_DEFUN([ZFS_LINUX_CONFTEST_MAKEFILE], [
 	test -d build || mkdir -p build
 	test -d build/$1 || mkdir -p build/$1
 
 	file=build/$1/Makefile
 
 	dnl # Example command line to manually build source.
 	cat - <<_ACEOF >$file
 # Example command line to manually build source
 # make modules -C $LINUX_OBJ $ARCH_UM M=$PWD/build/$1
 
 ccflags-y := -Werror $FRAME_LARGER_THAN
 _ACEOF
 
 	dnl # Additional custom CFLAGS as requested.
 	m4_ifval($3, [echo "ccflags-y += $3" >>$file], [])
 
 	dnl # Test case source
 	echo "obj-m := $1.o" >>$file
 
 	AS_IF([test "x$2" = "xyes"], [echo "obj-m += $1/" >>build/Makefile], [])
 ])
 
 dnl #
 dnl # ZFS_LINUX_TEST_PROGRAM(C)([PROLOGUE], [BODY])
 dnl #
 m4_define([ZFS_LINUX_TEST_PROGRAM], [
 #include <linux/module.h>
 $1
 
 int
 main (void)
 {
 $2
 	;
 	return 0;
 }
 
 MODULE_DESCRIPTION("conftest");
 MODULE_AUTHOR(ZFS_META_AUTHOR);
 MODULE_VERSION(ZFS_META_VERSION "-" ZFS_META_RELEASE);
 MODULE_LICENSE($3);
 ])
 
 dnl #
 dnl # ZFS_LINUX_TEST_REMOVE
 dnl #
 dnl # Removes the specified test source and results.
 dnl #
 AC_DEFUN([ZFS_LINUX_TEST_REMOVE], [
 	test -d build/$1 && rm -Rf build/$1
 	test -f build/Makefile && sed '/$1/d' build/Makefile
 ])
 
 dnl #
 dnl # ZFS_LINUX_COMPILE
 dnl #
 dnl # $1 - build dir
 dnl # $2 - test command
 dnl # $3 - pass command
 dnl # $4 - fail command
 dnl # $5 - set KBUILD_MODPOST_NOFINAL='yes'
 dnl # $6 - set KBUILD_MODPOST_WARN='yes'
 dnl #
 dnl # Used internally by ZFS_LINUX_TEST_{COMPILE,MODPOST}
 dnl #
 AC_DEFUN([ZFS_LINUX_COMPILE], [
 	AC_ARG_VAR([KERNEL_CC], [C compiler for
 		building kernel modules])
 	AC_ARG_VAR([KERNEL_LD], [Linker for
 		building kernel modules])
 	AC_ARG_VAR([KERNEL_LLVM], [Binary option to
 		build kernel modules with LLVM/CLANG toolchain])
+	AC_ARG_VAR([KERNEL_CROSS_COMPILE], [Cross compile prefix
+		for kernel module builds])
+	AC_ARG_VAR([KERNEL_ARCH], [Architecture to build kernel modules for])
 	AC_TRY_COMMAND([
 	    KBUILD_MODPOST_NOFINAL="$5" KBUILD_MODPOST_WARN="$6"
 	    make modules -k -j$TEST_JOBS ${KERNEL_CC:+CC=$KERNEL_CC}
 	    ${KERNEL_LD:+LD=$KERNEL_LD} ${KERNEL_LLVM:+LLVM=$KERNEL_LLVM}
 	    CONFIG_MODULES=y CFLAGS_MODULE=-DCONFIG_MODULES
+	    ${KERNEL_CROSS_COMPILE:+CROSS_COMPILE=$KERNEL_CROSS_COMPILE}
+	    ${KERNEL_ARCH:+ARCH=$KERNEL_ARCH}
 	    -C $LINUX_OBJ $ARCH_UM M=$PWD/$1 >$1/build.log 2>&1])
 	AS_IF([AC_TRY_COMMAND([$2])], [$3], [$4])
 ])
 
 dnl #
 dnl # ZFS_LINUX_TEST_COMPILE
 dnl #
 dnl # Perform a full compile excluding the final modpost phase.
 dnl #
 AC_DEFUN([ZFS_LINUX_TEST_COMPILE], [
 	ZFS_LINUX_COMPILE([$2], [test -f $2/build.log], [
 		mv $2/Makefile $2/Makefile.compile.$1
 		mv $2/build.log $2/build.log.$1
 	],[
 	        AC_MSG_ERROR([
         *** Unable to compile test source to determine kernel interfaces.])
 	], [yes], [])
 ])
 
 dnl #
 dnl # ZFS_LINUX_TEST_MODPOST
 dnl #
 dnl # Perform a full compile including the modpost phase.  This may
 dnl # be an incremental build if the objects have already been built.
 dnl #
 AC_DEFUN([ZFS_LINUX_TEST_MODPOST], [
 	ZFS_LINUX_COMPILE([$2], [test -f $2/build.log], [
 		mv $2/Makefile $2/Makefile.modpost.$1
 		cat $2/build.log >>build/build.log.$1
 	],[
 	        AC_MSG_ERROR([
         *** Unable to modpost test source to determine kernel interfaces.])
 	], [], [yes])
 ])
 
 dnl #
 dnl # Perform the compilation of the test cases in two phases.
 dnl #
 dnl # Phase 1) attempt to build the object files for all of the tests
 dnl #          defined by the ZFS_LINUX_TEST_SRC macro.  But do not
 dnl #          perform the final modpost stage.
 dnl #
 dnl # Phase 2) disable all tests which failed the initial compilation,
 dnl #          then invoke the final modpost step for the remaining tests.
 dnl #
 dnl # This allows us efficiently build the test cases in parallel while
 dnl # remaining resilient to build failures which are expected when
 dnl # detecting the available kernel interfaces.
 dnl #
 dnl # The maximum allowed parallelism can be controlled by setting the
 dnl # TEST_JOBS environment variable.  Otherwise, it default to $(nproc).
 dnl #
 AC_DEFUN([ZFS_LINUX_TEST_COMPILE_ALL], [
 	dnl # Phase 1 - Compilation only, final linking is skipped.
 	ZFS_LINUX_TEST_COMPILE([$1], [build])
 
 	dnl #
 	dnl # Phase 2 - When building external modules disable test cases
 	dnl # which failed to compile and invoke modpost to verify the
 	dnl # final linking.
 	dnl #
 	dnl # Test names suffixed with '_license' call modpost independently
 	dnl # to ensure that a single incompatibility does not result in the
 	dnl # modpost phase exiting early.  This check is not performed on
 	dnl # every symbol since the majority are compatible and doing so
 	dnl # would significantly slow down this phase.
 	dnl #
 	dnl # When configuring for builtin (--enable-linux-builtin)
 	dnl # fake the linking step artificially create the expected .ko
 	dnl # files for tests which did compile.  This is required for
 	dnl # kernels which do not have loadable module support or have
 	dnl # not yet been built.
 	dnl #
 	AS_IF([test "x$enable_linux_builtin" = "xno"], [
 		for dir in $(awk '/^obj-m/ { print [$]3 }' \
 		    build/Makefile.compile.$1); do
 			name=${dir%/}
 			AS_IF([test -f build/$name/$name.o], [
 				AS_IF([test "${name##*_}" = "license"], [
 					ZFS_LINUX_TEST_MODPOST([$1],
 					    [build/$name])
 					echo "obj-n += $dir" >>build/Makefile
 				], [
 					echo "obj-m += $dir" >>build/Makefile
 				])
 			], [
 				echo "obj-n += $dir" >>build/Makefile
 			])
 		done
 
 		ZFS_LINUX_TEST_MODPOST([$1], [build])
 	], [
 		for dir in $(awk '/^obj-m/ { print [$]3 }' \
 		    build/Makefile.compile.$1); do
 			name=${dir%/}
 			AS_IF([test -f build/$name/$name.o], [
 				touch build/$name/$name.ko
 			])
 		done
 	])
 ])
 
 dnl #
 dnl # ZFS_LINUX_TEST_SRC
 dnl #
 dnl # $1 - name
 dnl # $2 - global
 dnl # $3 - source
 dnl # $4 - extra cflags
 dnl # $5 - check license-compatibility
 dnl #
 dnl # Check if the test source is buildable at all and then if it is
 dnl # license compatible.
 dnl #
 dnl # N.B because all of the test cases are compiled in parallel they
 dnl # must never depend on the results of previous tests.  Each test
 dnl # needs to be entirely independent.
 dnl #
 AC_DEFUN([ZFS_LINUX_TEST_SRC], [
 	ZFS_LINUX_CONFTEST_C([ZFS_LINUX_TEST_PROGRAM([[$2]], [[$3]],
 	    [["Dual BSD/GPL"]])], [$1])
 	ZFS_LINUX_CONFTEST_MAKEFILE([$1], [yes], [$4])
 
 	AS_IF([ test -n "$5" ], [
 		ZFS_LINUX_CONFTEST_C([ZFS_LINUX_TEST_PROGRAM(
 		    [[$2]], [[$3]], [[$5]])], [$1_license])
 		ZFS_LINUX_CONFTEST_MAKEFILE([$1_license], [yes], [$4])
 	])
 ])
 
 dnl #
 dnl # ZFS_LINUX_TEST_RESULT
 dnl #
 dnl # $1 - name of a test source (ZFS_LINUX_TEST_SRC)
 dnl # $2 - run on success (valid .ko generated)
 dnl # $3 - run on failure (unable to compile)
 dnl #
 AC_DEFUN([ZFS_LINUX_TEST_RESULT], [
 	AS_IF([test -d build/$1], [
 		AS_IF([test -f build/$1/$1.ko], [$2], [$3])
 	], [
 		AC_MSG_ERROR([
 	*** No matching source for the "$1" test, check that
 	*** both the test source and result macros refer to the same name.
 		])
 	])
 ])
 
 dnl #
 dnl # ZFS_LINUX_TEST_ERROR
 dnl #
 dnl # Generic error message which can be used when none of the expected
 dnl # kernel interfaces were detected.
 dnl #
 AC_DEFUN([ZFS_LINUX_TEST_ERROR], [
 	AC_MSG_ERROR([
 	*** None of the expected "$1" interfaces were detected.
 	*** This may be because your kernel version is newer than what is
 	*** supported, or you are using a patched custom kernel with
 	*** incompatible modifications.
 	***
 	*** ZFS Version: $ZFS_META_ALIAS
 	*** Compatible Kernels: $ZFS_META_KVER_MIN - $ZFS_META_KVER_MAX
 	])
 ])
 
 dnl #
 dnl # ZFS_LINUX_TEST_RESULT_SYMBOL
 dnl #
 dnl # Like ZFS_LINUX_TEST_RESULT except ZFS_CHECK_SYMBOL_EXPORT is called to
 dnl # verify symbol exports, unless --enable-linux-builtin was provided to
 dnl # configure.
 dnl #
 AC_DEFUN([ZFS_LINUX_TEST_RESULT_SYMBOL], [
 	AS_IF([ ! test -f build/$1/$1.ko], [
 		$5
 	], [
 		AS_IF([test "x$enable_linux_builtin" != "xyes"], [
 			ZFS_CHECK_SYMBOL_EXPORT([$2], [$3], [$4], [$5])
 		], [
 			$4
 		])
 	])
 ])
 
 dnl #
 dnl # ZFS_LINUX_COMPILE_IFELSE
 dnl #
 AC_DEFUN([ZFS_LINUX_COMPILE_IFELSE], [
 	ZFS_LINUX_TEST_REMOVE([conftest])
 
 	m4_ifvaln([$1], [ZFS_LINUX_CONFTEST_C([$1], [conftest])])
 	m4_ifvaln([$5], [ZFS_LINUX_CONFTEST_H([$5], [conftest])],
 	    [ZFS_LINUX_CONFTEST_H([], [conftest])])
 
 	ZFS_LINUX_CONFTEST_MAKEFILE([conftest], [no],
 	    [m4_ifvaln([$5], [-I$PWD/build/conftest], [])])
 	ZFS_LINUX_COMPILE([build/conftest], [$2], [$3], [$4], [], [])
 ])
 
 dnl #
 dnl # ZFS_LINUX_TRY_COMPILE
 dnl #
 dnl # $1 - global
 dnl # $2 - source
 dnl # $3 - run on success (valid .ko generated)
 dnl # $4 - run on failure (unable to compile)
 dnl #
 dnl # When configuring as builtin (--enable-linux-builtin) for kernels
 dnl # without loadable module support (CONFIG_MODULES=n) only the object
 dnl # file is created.  See ZFS_LINUX_TEST_COMPILE_ALL for details.
 dnl #
 AC_DEFUN([ZFS_LINUX_TRY_COMPILE], [
 	AS_IF([test "x$enable_linux_builtin" = "xyes"], [
 		ZFS_LINUX_COMPILE_IFELSE(
 		    [ZFS_LINUX_TEST_PROGRAM([[$1]], [[$2]],
 		    [[ZFS_META_LICENSE]])],
 		    [test -f build/conftest/conftest.o], [$3], [$4])
 	], [
 		ZFS_LINUX_COMPILE_IFELSE(
 		    [ZFS_LINUX_TEST_PROGRAM([[$1]], [[$2]],
 		    [[ZFS_META_LICENSE]])],
 		    [test -f build/conftest/conftest.ko], [$3], [$4])
 	])
 ])
 
 dnl #
 dnl # ZFS_CHECK_SYMBOL_EXPORT
 dnl #
 dnl # Check if a symbol is exported on not by consulting the symbols
 dnl # file, or optionally the source code.
 dnl #
 AC_DEFUN([ZFS_CHECK_SYMBOL_EXPORT], [
 	grep -q -E '[[[:space:]]]$1[[[:space:]]]' \
 		$LINUX_OBJ/$LINUX_SYMBOLS 2>/dev/null
 	rc=$?
 	if test $rc -ne 0; then
 		export=0
 		for file in $2; do
 			grep -q -E "EXPORT_SYMBOL.*($1)" \
 				"$LINUX/$file" 2>/dev/null
 			rc=$?
 			if test $rc -eq 0; then
 				export=1
 				break;
 			fi
 		done
 		if test $export -eq 0; then :
 			$4
 		else :
 			$3
 		fi
 	else :
 		$3
 	fi
 ])
 
 dnl #
 dnl # ZFS_LINUX_TRY_COMPILE_SYMBOL
 dnl #
 dnl # Like ZFS_LINUX_TRY_COMPILER except ZFS_CHECK_SYMBOL_EXPORT is called
 dnl # to verify symbol exports, unless --enable-linux-builtin was provided
 dnl # to configure.
 dnl #
 AC_DEFUN([ZFS_LINUX_TRY_COMPILE_SYMBOL], [
 	ZFS_LINUX_TRY_COMPILE([$1], [$2], [rc=0], [rc=1])
 	if test $rc -ne 0; then :
 		$6
 	else
 		if test "x$enable_linux_builtin" != xyes; then
 			ZFS_CHECK_SYMBOL_EXPORT([$3], [$4], [rc=0], [rc=1])
 		fi
 		if test $rc -ne 0; then :
 			$6
 		else :
 			$5
 		fi
 	fi
 ])
 
 dnl #
 dnl # ZFS_LINUX_TRY_COMPILE_HEADER
 dnl # like ZFS_LINUX_TRY_COMPILE, except the contents conftest.h are
 dnl # provided via the fifth parameter
 dnl #
 AC_DEFUN([ZFS_LINUX_TRY_COMPILE_HEADER], [
 	AS_IF([test "x$enable_linux_builtin" = "xyes"], [
 		ZFS_LINUX_COMPILE_IFELSE(
 		    [ZFS_LINUX_TEST_PROGRAM([[$1]], [[$2]],
 		    [[ZFS_META_LICENSE]])],
 		    [test -f build/conftest/conftest.o], [$3], [$4], [$5])
 	], [
 		ZFS_LINUX_COMPILE_IFELSE(
 		    [ZFS_LINUX_TEST_PROGRAM([[$1]], [[$2]],
 		    [[ZFS_META_LICENSE]])],
 		    [test -f build/conftest/conftest.ko], [$3], [$4], [$5])
 	])
 ])
 
 dnl #
 dnl # AS_VERSION_COMPARE_LE
 dnl # like AS_VERSION_COMPARE_LE, but runs $3 if (and only if) $1 <= $2
 dnl # AS_VERSION_COMPARE_LE (version-1, version-2, [action-if-less-or-equal], [action-if-greater])
 dnl #
 AC_DEFUN([AS_VERSION_COMPARE_LE], [
 	AS_VERSION_COMPARE([$1], [$2], [$3], [$3], [$4])
 ])
 
 dnl #
 dnl # ZFS_LINUX_REQUIRE_API
 dnl # like ZFS_LINUX_TEST_ERROR, except only fails if the kernel is
 dnl # at least some specified version.
 dnl #
 AC_DEFUN([ZFS_LINUX_REQUIRE_API], [
 	AS_VERSION_COMPARE_LE([$2], [$kernsrcver], [
 		AC_MSG_ERROR([
 		*** None of the expected "$1" interfaces were detected. This
 		*** interface is expected for kernels version "$2" and above.
 		*** This may be because your kernel version is newer than what is
 		*** supported, or you are using a patched custom kernel with
 		*** incompatible modifications.  Newer kernels may have incompatible
 		*** APIs.
 		***
 		*** ZFS Version: $ZFS_META_ALIAS
 		*** Compatible Kernels: $ZFS_META_KVER_MIN - $ZFS_META_KVER_MAX
 		])
 	], [
 		AC_MSG_RESULT(no)
 	])
 ])
diff --git a/sys/contrib/openzfs/config/zfs-build.m4 b/sys/contrib/openzfs/config/zfs-build.m4
index bb5a85d815d1..57582b9d18f5 100644
--- a/sys/contrib/openzfs/config/zfs-build.m4
+++ b/sys/contrib/openzfs/config/zfs-build.m4
@@ -1,649 +1,651 @@
 AC_DEFUN([ZFS_AC_LICENSE], [
 	AC_MSG_CHECKING([zfs author])
 	AC_MSG_RESULT([$ZFS_META_AUTHOR])
 
 	AC_MSG_CHECKING([zfs license])
 	AC_MSG_RESULT([$ZFS_META_LICENSE])
 ])
 
 AC_DEFUN([ZFS_AC_DEBUG_ENABLE], [
 	DEBUG_CFLAGS="-Werror"
 	DEBUG_CPPFLAGS="-DDEBUG -UNDEBUG"
 	DEBUG_LDFLAGS=""
 	DEBUG_ZFS="_with_debug"
 	WITH_DEBUG="true"
 	AC_DEFINE(ZFS_DEBUG, 1, [zfs debugging enabled])
 
 	KERNEL_DEBUG_CFLAGS="-Werror"
 	KERNEL_DEBUG_CPPFLAGS="-DDEBUG -UNDEBUG"
 ])
 
 AC_DEFUN([ZFS_AC_DEBUG_DISABLE], [
 	DEBUG_CFLAGS=""
 	DEBUG_CPPFLAGS="-UDEBUG -DNDEBUG"
 	DEBUG_LDFLAGS=""
 	DEBUG_ZFS="_without_debug"
 	WITH_DEBUG=""
 
 	KERNEL_DEBUG_CFLAGS=""
 	KERNEL_DEBUG_CPPFLAGS="-UDEBUG -DNDEBUG"
 ])
 
 dnl #
 dnl # When debugging is enabled:
 dnl # - Enable all ASSERTs (-DDEBUG)
 dnl # - Promote all compiler warnings to errors (-Werror)
 dnl #
 dnl # (If INVARIANTS is detected, we need to force DEBUG, or strange panics
 dnl # can ensue.)
 dnl #
 AC_DEFUN([ZFS_AC_DEBUG], [
 	AC_MSG_CHECKING([whether assertion support will be enabled])
 	AC_ARG_ENABLE([debug],
 		[AS_HELP_STRING([--enable-debug],
 		[Enable compiler and code assertions @<:@default=no@:>@])],
 		[],
 		[enable_debug=no])
 
 	AS_CASE(["x$enable_debug"],
 		["xyes"],
 		[ZFS_AC_DEBUG_ENABLE],
 		["xno"],
 		[ZFS_AC_DEBUG_DISABLE],
 		[AC_MSG_ERROR([Unknown option $enable_debug])])
 
 	AS_CASE(["x$enable_invariants"],
 		["xyes"],
 		[],
 		["xno"],
 		[],
 		[ZFS_AC_DEBUG_INVARIANTS_DETECT])
 
 	AS_CASE(["x$enable_invariants"],
 		["xyes"],
 		[ZFS_AC_DEBUG_ENABLE],
 		["xno"],
 		[],
 		[AC_MSG_ERROR([Unknown option $enable_invariants])])
 
 	AC_SUBST(DEBUG_CFLAGS)
 	AC_SUBST(DEBUG_CPPFLAGS)
 	AC_SUBST(DEBUG_LDFLAGS)
 	AC_SUBST(DEBUG_ZFS)
 	AC_SUBST(WITH_DEBUG)
 
 	AC_SUBST(KERNEL_DEBUG_CFLAGS)
 	AC_SUBST(KERNEL_DEBUG_CPPFLAGS)
 
 	AC_MSG_RESULT([$enable_debug])
 ])
 
 AC_DEFUN([ZFS_AC_DEBUGINFO_ENABLE], [
 	DEBUG_CFLAGS="$DEBUG_CFLAGS -g -fno-inline $NO_IPA_SRA"
 
 	KERNEL_DEBUG_CFLAGS="$KERNEL_DEBUG_CFLAGS -fno-inline $KERNEL_NO_IPA_SRA"
 	KERNEL_MAKE="$KERNEL_MAKE CONFIG_DEBUG_INFO=y"
 
 	DEBUGINFO_ZFS="_with_debuginfo"
 ])
 
 AC_DEFUN([ZFS_AC_DEBUGINFO_DISABLE], [
 	DEBUGINFO_ZFS="_without_debuginfo"
 ])
 
 AC_DEFUN([ZFS_AC_DEBUGINFO], [
 	AC_MSG_CHECKING([whether debuginfo support will be forced])
 	AC_ARG_ENABLE([debuginfo],
 		[AS_HELP_STRING([--enable-debuginfo],
 		[Force generation of debuginfo @<:@default=no@:>@])],
 		[],
 		[enable_debuginfo=no])
 
 	AS_CASE(["x$enable_debuginfo"],
 		["xyes"],
 		[ZFS_AC_DEBUGINFO_ENABLE],
 		["xno"],
 		[ZFS_AC_DEBUGINFO_DISABLE],
 		[AC_MSG_ERROR([Unknown option $enable_debuginfo])])
 
 	AC_SUBST(DEBUG_CFLAGS)
 	AC_SUBST(DEBUGINFO_ZFS)
 
 	AC_SUBST(KERNEL_DEBUG_CFLAGS)
 	AC_SUBST(KERNEL_MAKE)
 
 	AC_MSG_RESULT([$enable_debuginfo])
 ])
 
 dnl #
 dnl # Disabled by default, provides basic memory tracking.  Track the total
 dnl # number of bytes allocated with kmem_alloc() and freed with kmem_free().
 dnl # Then at module unload time if any bytes were leaked it will be reported
 dnl # on the console.
 dnl #
 AC_DEFUN([ZFS_AC_DEBUG_KMEM], [
 	AC_MSG_CHECKING([whether basic kmem accounting is enabled])
 	AC_ARG_ENABLE([debug-kmem],
 		[AS_HELP_STRING([--enable-debug-kmem],
 		[Enable basic kmem accounting @<:@default=no@:>@])],
 		[],
 		[enable_debug_kmem=no])
 
 	AS_IF([test "x$enable_debug_kmem" = xyes], [
 		KERNEL_DEBUG_CPPFLAGS="${KERNEL_DEBUG_CPPFLAGS} -DDEBUG_KMEM"
 		DEBUG_KMEM_ZFS="_with_debug_kmem"
 	], [
 		DEBUG_KMEM_ZFS="_without_debug_kmem"
 	])
 
 	AC_SUBST(KERNEL_DEBUG_CPPFLAGS)
 	AC_SUBST(DEBUG_KMEM_ZFS)
 
 	AC_MSG_RESULT([$enable_debug_kmem])
 ])
 
 dnl #
 dnl # Disabled by default, provides detailed memory tracking.  This feature
 dnl # also requires --enable-debug-kmem to be set.  When enabled not only will
 dnl # total bytes be tracked but also the location of every kmem_alloc() and
 dnl # kmem_free().  When the module is unloaded a list of all leaked addresses
 dnl # and where they were allocated will be dumped to the console.  Enabling
 dnl # this feature has a significant impact on performance but it makes finding
 dnl # memory leaks straight forward.
 dnl #
 AC_DEFUN([ZFS_AC_DEBUG_KMEM_TRACKING], [
 	AC_MSG_CHECKING([whether detailed kmem tracking is enabled])
 	AC_ARG_ENABLE([debug-kmem-tracking],
 		[AS_HELP_STRING([--enable-debug-kmem-tracking],
 		[Enable detailed kmem tracking  @<:@default=no@:>@])],
 		[],
 		[enable_debug_kmem_tracking=no])
 
 	AS_IF([test "x$enable_debug_kmem_tracking" = xyes], [
 		KERNEL_DEBUG_CPPFLAGS="${KERNEL_DEBUG_CPPFLAGS} -DDEBUG_KMEM_TRACKING"
 		DEBUG_KMEM_TRACKING_ZFS="_with_debug_kmem_tracking"
 	], [
 		DEBUG_KMEM_TRACKING_ZFS="_without_debug_kmem_tracking"
 	])
 
 	AC_SUBST(KERNEL_DEBUG_CPPFLAGS)
 	AC_SUBST(DEBUG_KMEM_TRACKING_ZFS)
 
 	AC_MSG_RESULT([$enable_debug_kmem_tracking])
 ])
 
 AC_DEFUN([ZFS_AC_DEBUG_INVARIANTS_DETECT_FREEBSD], [
 	AS_IF([sysctl -n kern.conftxt | grep -Fqx $'options\tINVARIANTS'],
 		[enable_invariants="yes"],
 		[enable_invariants="no"])
 ])
 
 AC_DEFUN([ZFS_AC_DEBUG_INVARIANTS_DETECT], [
 	AM_COND_IF([BUILD_FREEBSD],
 		[ZFS_AC_DEBUG_INVARIANTS_DETECT_FREEBSD],
 		[enable_invariants="no"])
 ])
 
 dnl #
 dnl # Detected for the running kernel by default, enables INVARIANTS features
 dnl # in the FreeBSD kernel module.  This feature must be used when building
 dnl # for a FreeBSD kernel with "options INVARIANTS" in the KERNCONF and must
 dnl # not be used when the INVARIANTS option is absent.
 dnl #
 AC_DEFUN([ZFS_AC_DEBUG_INVARIANTS], [
 	AC_MSG_CHECKING([whether FreeBSD kernel INVARIANTS checks are enabled])
 	AC_ARG_ENABLE([invariants],
 		[AS_HELP_STRING([--enable-invariants],
 		[Enable FreeBSD kernel INVARIANTS checks [[default: detect]]])],
 		[], [ZFS_AC_DEBUG_INVARIANTS_DETECT])
 
 	AS_IF([test "x$enable_invariants" = xyes],
 		[WITH_INVARIANTS="true"],
 		[WITH_INVARIANTS=""])
 	AC_SUBST(WITH_INVARIANTS)
 
 	AC_MSG_RESULT([$enable_invariants])
 ])
 
 AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [
 	AX_COUNT_CPUS([])
 	AC_SUBST(CPU_COUNT)
 
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_CLOBBERED
 	ZFS_AC_CONFIG_ALWAYS_CC_INFINITE_RECURSION
 	ZFS_AC_CONFIG_ALWAYS_KERNEL_CC_INFINITE_RECURSION
 	ZFS_AC_CONFIG_ALWAYS_CC_IMPLICIT_FALLTHROUGH
 	ZFS_AC_CONFIG_ALWAYS_CC_FRAME_LARGER_THAN
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_ZERO_LENGTH
 	ZFS_AC_CONFIG_ALWAYS_CC_FORMAT_OVERFLOW
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_OMIT_FRAME_POINTER
 	ZFS_AC_CONFIG_ALWAYS_CC_NO_IPA_SRA
 	ZFS_AC_CONFIG_ALWAYS_KERNEL_CC_NO_IPA_SRA
 	ZFS_AC_CONFIG_ALWAYS_CC_ASAN
 	ZFS_AC_CONFIG_ALWAYS_CC_UBSAN
 	ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD
 	ZFS_AC_CONFIG_ALWAYS_SYSTEM
 	ZFS_AC_CONFIG_ALWAYS_ARCH
 	ZFS_AC_CONFIG_ALWAYS_PYTHON
 	ZFS_AC_CONFIG_ALWAYS_PYZFS
 	ZFS_AC_CONFIG_ALWAYS_SED
 	ZFS_AC_CONFIG_ALWAYS_CPPCHECK
 	ZFS_AC_CONFIG_ALWAYS_SHELLCHECK
 	ZFS_AC_CONFIG_ALWAYS_PARALLEL
 ])
 
 AC_DEFUN([ZFS_AC_CONFIG], [
 
         dnl # Remove the previous build test directory.
         rm -Rf build
 
 	ZFS_CONFIG=all
 	AC_ARG_WITH([config],
 		AS_HELP_STRING([--with-config=CONFIG],
 		[Config file 'kernel|user|all|srpm']),
 		[ZFS_CONFIG="$withval"])
 	AC_ARG_ENABLE([linux-builtin],
 		[AS_HELP_STRING([--enable-linux-builtin],
 		[Configure for builtin in-tree kernel modules @<:@default=no@:>@])],
 		[],
 		[enable_linux_builtin=no])
 
 	AC_MSG_CHECKING([zfs config])
 	AC_MSG_RESULT([$ZFS_CONFIG]);
 	AC_SUBST(ZFS_CONFIG)
 
 	ZFS_AC_CONFIG_ALWAYS
 
 	AM_COND_IF([BUILD_LINUX], [
 		AC_ARG_VAR([TEST_JOBS], [simultaneous jobs during configure])
 		if test "x$ac_cv_env_TEST_JOBS_set" != "xset"; then
 			TEST_JOBS=$CPU_COUNT
 		fi
 		AC_SUBST(TEST_JOBS)
 	])
 
 	ZFS_INIT_SYSV=
 	ZFS_INIT_SYSTEMD=
 	ZFS_WANT_MODULES_LOAD_D=
 
 	case "$ZFS_CONFIG" in
 		kernel) ZFS_AC_CONFIG_KERNEL ;;
 		user)	ZFS_AC_CONFIG_USER   ;;
 		all)    ZFS_AC_CONFIG_USER
 			ZFS_AC_CONFIG_KERNEL ;;
 		dist)                        ;;
 		srpm)                        ;;
 		*)
 		AC_MSG_RESULT([Error!])
 		AC_MSG_ERROR([Bad value "$ZFS_CONFIG" for --with-config,
 		              user kernel|user|all|srpm]) ;;
 	esac
 
 	AM_CONDITIONAL([INIT_SYSV],           [test "x$ZFS_INIT_SYSV" = "xyes"])
 	AM_CONDITIONAL([INIT_SYSTEMD],        [test "x$ZFS_INIT_SYSTEMD" = "xyes"])
 	AM_CONDITIONAL([WANT_MODULES_LOAD_D], [test "x$ZFS_WANT_MODULES_LOAD_D" = "xyes"])
 
 	AM_CONDITIONAL([CONFIG_USER],
 	    [test "$ZFS_CONFIG" = user -o "$ZFS_CONFIG" = all])
 	AM_CONDITIONAL([CONFIG_KERNEL],
 	    [test "$ZFS_CONFIG" = kernel -o "$ZFS_CONFIG" = all] &&
 	    [test "x$enable_linux_builtin" != xyes ])
 	AM_CONDITIONAL([CONFIG_QAT],
 	    [test "$ZFS_CONFIG" = kernel -o "$ZFS_CONFIG" = all] &&
 	    [test "x$qatsrc" != x ])
 	AM_CONDITIONAL([WANT_DEVNAME2DEVID], [test "x$user_libudev" = xyes ])
 	AM_CONDITIONAL([WANT_MMAP_LIBAIO], [test "x$user_libaio" = xyes ])
 	AM_CONDITIONAL([PAM_ZFS_ENABLED], [test "x$enable_pam" = xyes])
 ])
 
 dnl #
 dnl # Check for rpm+rpmbuild to build RPM packages.  If these tools
 dnl # are missing it is non-fatal but you will not be able to build
 dnl # RPM packages and will be warned if you try too.
 dnl #
 dnl # By default the generic spec file will be used because it requires
 dnl # minimal dependencies.  Distribution specific spec files can be
 dnl # placed under the 'rpm/<distribution>' directory and enabled using
 dnl # the --with-spec=<distribution> configure option.
 dnl #
 AC_DEFUN([ZFS_AC_RPM], [
 	RPM=rpm
 	RPMBUILD=rpmbuild
 
 	AC_MSG_CHECKING([whether $RPM is available])
 	AS_IF([tmp=$($RPM --version 2>/dev/null)], [
 		RPM_VERSION=$(echo $tmp | $AWK '/RPM/ { print $[3] }')
 		HAVE_RPM=yes
 		AC_MSG_RESULT([$HAVE_RPM ($RPM_VERSION)])
 	],[
 		HAVE_RPM=no
 		AC_MSG_RESULT([$HAVE_RPM])
 	])
 
 	AC_MSG_CHECKING([whether $RPMBUILD is available])
 	AS_IF([tmp=$($RPMBUILD --version 2>/dev/null)], [
 		RPMBUILD_VERSION=$(echo $tmp | $AWK '/RPM/ { print $[3] }')
 		HAVE_RPMBUILD=yes
 		AC_MSG_RESULT([$HAVE_RPMBUILD ($RPMBUILD_VERSION)])
 	],[
 		HAVE_RPMBUILD=no
 		AC_MSG_RESULT([$HAVE_RPMBUILD])
 	])
 
 	RPM_DEFINE_COMMON='--define "$(DEBUG_ZFS) 1"'
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(DEBUGINFO_ZFS) 1"'
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(DEBUG_KMEM_ZFS) 1"'
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(DEBUG_KMEM_TRACKING_ZFS) 1"'
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(ASAN_ZFS) 1"'
 	RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "$(UBSAN_ZFS) 1"'
 
 	AS_IF([test "x$enable_debuginfo" = xyes], [
 		RPM_DEFINE_COMMON=${RPM_DEFINE_COMMON}' --define "__strip /bin/true"'
 	])
 
 	RPM_DEFINE_UTIL=' --define "_initconfdir $(initconfdir)"'
 
 	dnl # Make the next three RPM_DEFINE_UTIL additions conditional, since
 	dnl # their values may not be set when running:
 	dnl #
 	dnl #	./configure --with-config=srpm
 	dnl #
 	AS_IF([test -n "$dracutdir" ], [
 		RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' --define "_dracutdir $(dracutdir)"'
 	])
 	AS_IF([test -n "$udevdir" ], [
 		RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' --define "_udevdir $(udevdir)"'
 	])
 	AS_IF([test -n "$udevruledir" ], [
 		RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' --define "_udevruledir $(udevruledir)"'
 	])
 	AS_IF([test -n "$bashcompletiondir" ], [
 		RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' --define "_bashcompletiondir $(bashcompletiondir)"'
 	])
 	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' $(DEFINE_SYSTEMD)'
 	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' $(DEFINE_PYZFS)'
 	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' $(DEFINE_PAM)'
 	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' $(DEFINE_PYTHON_VERSION)'
 	RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' $(DEFINE_PYTHON_PKG_VERSION)'
 
 	dnl # Override default lib directory on Debian/Ubuntu systems.  The
 	dnl # provided /usr/lib/rpm/platform/<arch>/macros files do not
 	dnl # specify the correct path for multiarch systems as described
 	dnl # by the packaging guidelines.
 	dnl #
 	dnl # https://wiki.ubuntu.com/MultiarchSpec
 	dnl # https://wiki.debian.org/Multiarch/Implementation
 	dnl #
 	AS_IF([test "$DEFAULT_PACKAGE" = "deb"], [
 		MULTIARCH_LIBDIR="lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)"
 		RPM_DEFINE_UTIL=${RPM_DEFINE_UTIL}' --define "_lib $(MULTIARCH_LIBDIR)"'
 		AC_SUBST(MULTIARCH_LIBDIR)
 	])
 
 	dnl # Make RPM_DEFINE_KMOD additions conditional on CONFIG_KERNEL,
 	dnl # since the values will not be set otherwise. The spec files
 	dnl # provide defaults for them.
 	dnl #
 	RPM_DEFINE_KMOD='--define "_wrong_version_format_terminate_build 0"'
 	AM_COND_IF([CONFIG_KERNEL], [
 		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernels $(LINUX_VERSION)"'
 		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "ksrc $(LINUX)"'
 		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kobj $(LINUX_OBJ)"'
 		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_cc KERNEL_CC=$(KERNEL_CC)"'
 		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_ld KERNEL_LD=$(KERNEL_LD)"'
 		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_llvm KERNEL_LLVM=$(KERNEL_LLVM)"'
+		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_cross_compile KERNEL_CROSS_COMPILE=$(KERNEL_CROSS_COMPILE)"'
+		RPM_DEFINE_KMOD=${RPM_DEFINE_KMOD}' --define "kernel_arch KERNEL_ARCH=$(KERNEL_ARCH)"'
 	])
 
 	RPM_DEFINE_DKMS=''
 
 	SRPM_DEFINE_COMMON='--define "build_src_rpm 1"'
 	SRPM_DEFINE_UTIL=
 	SRPM_DEFINE_KMOD=
 	SRPM_DEFINE_DKMS=
 
 	RPM_SPEC_DIR="rpm/generic"
 	AC_ARG_WITH([spec],
 		AS_HELP_STRING([--with-spec=SPEC],
 		[Spec files 'generic|redhat']),
 		[RPM_SPEC_DIR="rpm/$withval"])
 
 	AC_MSG_CHECKING([whether spec files are available])
 	AC_MSG_RESULT([yes ($RPM_SPEC_DIR/*.spec.in)])
 
 	AC_SUBST(HAVE_RPM)
 	AC_SUBST(RPM)
 	AC_SUBST(RPM_VERSION)
 
 	AC_SUBST(HAVE_RPMBUILD)
 	AC_SUBST(RPMBUILD)
 	AC_SUBST(RPMBUILD_VERSION)
 
 	AC_SUBST(RPM_SPEC_DIR)
 	AC_SUBST(RPM_DEFINE_UTIL)
 	AC_SUBST(RPM_DEFINE_KMOD)
 	AC_SUBST(RPM_DEFINE_DKMS)
 	AC_SUBST(RPM_DEFINE_COMMON)
 	AC_SUBST(SRPM_DEFINE_UTIL)
 	AC_SUBST(SRPM_DEFINE_KMOD)
 	AC_SUBST(SRPM_DEFINE_DKMS)
 	AC_SUBST(SRPM_DEFINE_COMMON)
 ])
 
 dnl #
 dnl # Check for dpkg+dpkg-buildpackage to build DEB packages.  If these
 dnl # tools are missing it is non-fatal but you will not be able to build
 dnl # DEB packages and will be warned if you try too.
 dnl #
 AC_DEFUN([ZFS_AC_DPKG], [
 	DPKG=dpkg
 	DPKGBUILD=dpkg-buildpackage
 
 	AC_MSG_CHECKING([whether $DPKG is available])
 	AS_IF([tmp=$($DPKG --version 2>/dev/null)], [
 		DPKG_VERSION=$(echo $tmp | $AWK '/Debian/ { print $[7] }')
 		HAVE_DPKG=yes
 		AC_MSG_RESULT([$HAVE_DPKG ($DPKG_VERSION)])
 	],[
 		HAVE_DPKG=no
 		AC_MSG_RESULT([$HAVE_DPKG])
 	])
 
 	AC_MSG_CHECKING([whether $DPKGBUILD is available])
 	AS_IF([tmp=$($DPKGBUILD --version 2>/dev/null)], [
 		DPKGBUILD_VERSION=$(echo $tmp | \
 		    $AWK '/Debian/ { print $[4] }' | cut -f-4 -d'.')
 		HAVE_DPKGBUILD=yes
 		AC_MSG_RESULT([$HAVE_DPKGBUILD ($DPKGBUILD_VERSION)])
 	],[
 		HAVE_DPKGBUILD=no
 		AC_MSG_RESULT([$HAVE_DPKGBUILD])
 	])
 
 	AC_SUBST(HAVE_DPKG)
 	AC_SUBST(DPKG)
 	AC_SUBST(DPKG_VERSION)
 
 	AC_SUBST(HAVE_DPKGBUILD)
 	AC_SUBST(DPKGBUILD)
 	AC_SUBST(DPKGBUILD_VERSION)
 	AC_SUBST([CFGOPTS], ["$CFGOPTS"])
 ])
 
 dnl #
 dnl # Until native packaging for various different packing systems
 dnl # can be added the least we can do is attempt to use alien to
 dnl # convert the RPM packages to the needed package type.  This is
 dnl # a hack but so far it has worked reasonable well.
 dnl #
 AC_DEFUN([ZFS_AC_ALIEN], [
 	ALIEN=alien
 
 	AC_MSG_CHECKING([whether $ALIEN is available])
 	AS_IF([tmp=$($ALIEN --version 2>/dev/null)], [
 		ALIEN_VERSION=$(echo $tmp | $AWK '{ print $[3] }')
 		ALIEN_MAJOR=$(echo ${ALIEN_VERSION} | $AWK -F'.' '{ print $[1] }')
 		ALIEN_MINOR=$(echo ${ALIEN_VERSION} | $AWK -F'.' '{ print $[2] }')
 		ALIEN_POINT=$(echo ${ALIEN_VERSION} | $AWK -F'.' '{ print $[3] }')
 		HAVE_ALIEN=yes
 		AC_MSG_RESULT([$HAVE_ALIEN ($ALIEN_VERSION)])
 	],[
 		HAVE_ALIEN=no
 		AC_MSG_RESULT([$HAVE_ALIEN])
 	])
 
 	AC_SUBST(HAVE_ALIEN)
 	AC_SUBST(ALIEN)
 	AC_SUBST(ALIEN_VERSION)
 	AC_SUBST(ALIEN_MAJOR)
 	AC_SUBST(ALIEN_MINOR)
 	AC_SUBST(ALIEN_POINT)
 ])
 
 dnl #
 dnl # Using the VENDOR tag from config.guess set the default
 dnl # package type for 'make pkg': (rpm | deb | tgz)
 dnl #
 AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [
 	AC_MSG_CHECKING([os distribution])
 	AC_ARG_WITH([vendor],
 		[AS_HELP_STRING([--with-vendor],
 			[Distribution vendor @<:@default=check@:>@])],
 		[with_vendor=$withval],
 		[with_vendor=check])
 	AS_IF([test "x$with_vendor" = "xcheck"],[
 		if test -f /etc/toss-release ; then
 			VENDOR=toss ;
 		elif test -f /etc/fedora-release ; then
 			VENDOR=fedora ;
 		elif test -f /etc/redhat-release ; then
 			VENDOR=redhat ;
 		elif test -f /etc/gentoo-release ; then
 			VENDOR=gentoo ;
 		elif test -f /etc/arch-release ; then
 			VENDOR=arch ;
 		elif test -f /etc/SuSE-release ; then
 			VENDOR=sles ;
 		elif test -f /etc/slackware-version ; then
 			VENDOR=slackware ;
 		elif test -f /etc/lunar.release ; then
 			VENDOR=lunar ;
 		elif test -f /etc/lsb-release ; then
 			VENDOR=ubuntu ;
 		elif test -f /etc/debian_version ; then
 			VENDOR=debian ;
 		elif test -f /etc/alpine-release ; then
 			VENDOR=alpine ;
 		elif test -f /bin/freebsd-version ; then
 			VENDOR=freebsd ;
 		elif test -f /etc/openEuler-release ; then
 			VENDOR=openeuler ;
 		else
 			VENDOR= ;
 		fi],
 		[ test "x${with_vendor}" != x],[
 			VENDOR="$with_vendor" ],
 		[ VENDOR= ; ]
 	)
 	AC_MSG_RESULT([$VENDOR])
 	AC_SUBST(VENDOR)
 
 	AC_MSG_CHECKING([default package type])
 	case "$VENDOR" in
 		toss)       DEFAULT_PACKAGE=rpm  ;;
 		redhat)     DEFAULT_PACKAGE=rpm  ;;
 		fedora)     DEFAULT_PACKAGE=rpm  ;;
 		gentoo)     DEFAULT_PACKAGE=tgz  ;;
 		alpine)     DEFAULT_PACKAGE=tgz  ;;
 		arch)       DEFAULT_PACKAGE=tgz  ;;
 		sles)       DEFAULT_PACKAGE=rpm  ;;
 		slackware)  DEFAULT_PACKAGE=tgz  ;;
 		lunar)      DEFAULT_PACKAGE=tgz  ;;
 		ubuntu)     DEFAULT_PACKAGE=deb  ;;
 		debian)     DEFAULT_PACKAGE=deb  ;;
 		freebsd)    DEFAULT_PACKAGE=pkg  ;;
 		openeuler)  DEFAULT_PACKAGE=rpm  ;;
 		*)          DEFAULT_PACKAGE=rpm  ;;
 	esac
 	AC_MSG_RESULT([$DEFAULT_PACKAGE])
 	AC_SUBST(DEFAULT_PACKAGE)
 
 	AC_MSG_CHECKING([default init directory])
 	case "$VENDOR" in
 		freebsd)    initdir=$sysconfdir/rc.d  ;;
 		*)          initdir=$sysconfdir/init.d;;
 	esac
 	AC_MSG_RESULT([$initdir])
 	AC_SUBST(initdir)
 
 	AC_MSG_CHECKING([default shell])
 	case "$VENDOR" in
 		gentoo|alpine)	DEFAULT_INIT_SHELL=/sbin/openrc-run
 				IS_SYSV_RC=false	;;
 		*)		DEFAULT_INIT_SHELL=/bin/sh
 				IS_SYSV_RC=true		;;
 	esac
 
 	AC_MSG_RESULT([$DEFAULT_INIT_SHELL])
 	AC_SUBST(DEFAULT_INIT_SHELL)
 	AC_SUBST(IS_SYSV_RC)
 
 	AC_MSG_CHECKING([default nfs server init script])
 	AS_IF([test "$VENDOR" = "debian"],
 		[DEFAULT_INIT_NFS_SERVER="nfs-kernel-server"],
 		[DEFAULT_INIT_NFS_SERVER="nfs"]
 	)
 	AC_MSG_RESULT([$DEFAULT_INIT_NFS_SERVER])
 	AC_SUBST(DEFAULT_INIT_NFS_SERVER)
 
 	AC_MSG_CHECKING([default init config directory])
 	case "$VENDOR" in
 		alpine)     initconfdir=/etc/conf.d    ;;
 		gentoo)     initconfdir=/etc/conf.d    ;;
 		toss)       initconfdir=/etc/sysconfig ;;
 		redhat)     initconfdir=/etc/sysconfig ;;
 		fedora)     initconfdir=/etc/sysconfig ;;
 		sles)       initconfdir=/etc/sysconfig ;;
 		openeuler)  initconfdir=/etc/sysconfig ;;
 		ubuntu)     initconfdir=/etc/default   ;;
 		debian)     initconfdir=/etc/default   ;;
 		freebsd)    initconfdir=$sysconfdir/rc.conf.d;;
 		*)          initconfdir=/etc/default   ;;
 	esac
 	AC_MSG_RESULT([$initconfdir])
 	AC_SUBST(initconfdir)
 
 	AC_MSG_CHECKING([whether initramfs-tools is available])
 	if test -d /usr/share/initramfs-tools ; then
 		RPM_DEFINE_INITRAMFS='--define "_initramfs 1"'
 		AC_MSG_RESULT([yes])
 	else
 		RPM_DEFINE_INITRAMFS=''
 		AC_MSG_RESULT([no])
 	fi
 	AC_SUBST(RPM_DEFINE_INITRAMFS)
 
 	AC_MSG_CHECKING([default bash completion directory])
 	case "$VENDOR" in
 		ubuntu)     bashcompletiondir=/usr/share/bash-completion/completions   ;;
 		debian)     bashcompletiondir=/usr/share/bash-completion/completions   ;;
 		freebsd)    bashcompletiondir=$sysconfdir/bash_completion.d;;
 		gentoo)     bashcompletiondir=/usr/share/bash-completion/completions   ;;
 		*)          bashcompletiondir=/etc/bash_completion.d   ;;
 	esac
 	AC_MSG_RESULT([$bashcompletiondir])
 	AC_SUBST(bashcompletiondir)
 
 ])
 
 dnl #
 dnl # Default ZFS package configuration
 dnl #
 AC_DEFUN([ZFS_AC_PACKAGE], [
 	ZFS_AC_DEFAULT_PACKAGE
 	AS_IF([test x$VENDOR != xfreebsd], [
 		ZFS_AC_RPM
 		ZFS_AC_DPKG
 		ZFS_AC_ALIEN
 	])
 ])
diff --git a/sys/contrib/openzfs/contrib/debian/control b/sys/contrib/openzfs/contrib/debian/control
index e56fbf0f1c93..010273397bf9 100644
--- a/sys/contrib/openzfs/contrib/debian/control
+++ b/sys/contrib/openzfs/contrib/debian/control
@@ -1,326 +1,327 @@
 Source: openzfs-linux
 Section: contrib/kernel
 Priority: optional
 Maintainer: ZFS on Linux specific mailing list <zfs-discuss@list.zfsonlinux.org>
 Build-Depends: debhelper-compat (= 12),
                dh-python,
                dh-sequence-dkms | dkms (>> 2.1.1.2-5),
                libaio-dev,
                libblkid-dev,
                libcurl4-openssl-dev,
                libelf-dev,
                libpam0g-dev,
                libssl-dev | libssl1.0-dev,
+               libtirpc-dev,
                libtool,
                libudev-dev,
                lsb-release,
                po-debconf,
                python3-all-dev,
                python3-cffi,
                python3-setuptools,
                python3-sphinx,
                uuid-dev,
                zlib1g-dev
 Standards-Version: 4.5.1
 Homepage: https://openzfs.org/
 Vcs-Git: https://github.com/openzfs/zfs.git
 Vcs-Browser: https://github.com/openzfs/zfs
 Rules-Requires-Root: no
 XS-Autobuild: yes
 
 Package: openzfs-libnvpair3
 Section: contrib/libs
 Architecture: linux-any
 Depends: ${misc:Depends}, ${shlibs:Depends}
 Breaks: libnvpair1, libnvpair3
 Replaces: libnvpair1, libnvpair3, libnvpair3linux
 Conflicts: libnvpair3linux
 Description: Solaris name-value library for Linux
  This library provides routines for packing and unpacking nv pairs for
  transporting data across process boundaries, transporting between
  kernel and userland, and possibly saving onto disk files.
 
 Package: openzfs-libpam-zfs
 Section: contrib/admin
 Architecture: linux-any
 Depends: libpam-runtime, ${misc:Depends}, ${shlibs:Depends}
 Replaces: libpam-zfs
 Conflicts: libpam-zfs
 Description: PAM module for managing encryption keys for ZFS
  OpenZFS is a storage platform that encompasses the functionality of
  traditional filesystems and volume managers. It supports data checksums,
  compression, encryption, snapshots, and more.
  .
  This provides a Pluggable Authentication Module (PAM) that automatically
  unlocks encrypted ZFS datasets upon login.
 
 Package: openzfs-libuutil3
 Section: contrib/libs
 Architecture: linux-any
 Depends: ${misc:Depends}, ${shlibs:Depends}
 Breaks: libuutil1, libuutil3
 Replaces: libuutil1, libuutil3, libuutil3linux
 Conflicts: libuutil3linux
 Description: Solaris userland utility library for Linux
  This library provides a variety of glue functions for ZFS on Linux:
   * libspl: The Solaris Porting Layer userland library, which provides APIs
     that make it possible to run Solaris user code in a Linux environment
     with relatively minimal modification.
   * libavl: The Adelson-Velskii Landis balanced binary tree manipulation
     library.
   * libefi: The Extensible Firmware Interface library for GUID disk
     partitioning.
   * libshare: NFS, SMB, and iSCSI service integration for ZFS.
 
 Package: openzfs-libzfs-dev
 Section: contrib/libdevel
 Architecture: linux-any
 Depends: libssl-dev | libssl1.0-dev,
          openzfs-libnvpair3 (= ${binary:Version}),
          openzfs-libuutil3 (= ${binary:Version}),
          openzfs-libzfs4 (= ${binary:Version}),
          openzfs-libzfsbootenv1 (= ${binary:Version}),
          openzfs-libzpool5 (= ${binary:Version}),
          ${misc:Depends}
 Replaces: libzfslinux-dev
 Conflicts: libzfslinux-dev
 Provides: libnvpair-dev, libuutil-dev
 Description: OpenZFS filesystem development files for Linux
  Header files and static libraries for compiling software against
  libraries of OpenZFS filesystem.
  .
  This package includes the development files of libnvpair3, libuutil3,
  libzpool5 and libzfs4.
 
 Package: openzfs-libzfs4
 Section: contrib/libs
 Architecture: linux-any
 Depends: ${misc:Depends}, ${shlibs:Depends}
 # The libcurl4 is loaded through dlopen("libcurl.so.4").
 # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=988521
 Recommends: libcurl4
 Breaks: libzfs2, libzfs4
 Replaces: libzfs2, libzfs4, libzfs4linux
 Conflicts: libzfs4linux
 Description: OpenZFS filesystem library for Linux - general support
  OpenZFS is a storage platform that encompasses the functionality of
  traditional filesystems and volume managers. It supports data checksums,
  compression, encryption, snapshots, and more.
  .
  The OpenZFS library provides support for managing OpenZFS filesystems.
 
 Package: openzfs-libzfsbootenv1
 Section: contrib/libs
 Architecture: linux-any
 Depends: ${misc:Depends}, ${shlibs:Depends}
 Breaks: libzfs2, libzfs4
 Replaces: libzfs2, libzfs4, libzfsbootenv1linux
 Conflicts: libzfsbootenv1linux
 Description: OpenZFS filesystem library for Linux - label info support
  OpenZFS is a storage platform that encompasses the functionality of
  traditional filesystems and volume managers. It supports data checksums,
  compression, encryption, snapshots, and more.
  .
  The zfsbootenv library provides support for modifying ZFS label information.
 
 Package: openzfs-libzpool5
 Section: contrib/libs
 Architecture: linux-any
 Depends: ${misc:Depends}, ${shlibs:Depends}
 Breaks: libzpool2, libzpool5
 Replaces: libzpool2, libzpool5, libzpool5linux
 Conflicts: libzpool5linux
 Description: OpenZFS pool library for Linux
  OpenZFS is a storage platform that encompasses the functionality of
  traditional filesystems and volume managers. It supports data checksums,
  compression, encryption, snapshots, and more.
  .
  This zpool library provides support for managing zpools.
 
 Package: openzfs-python3-pyzfs
 Section: contrib/python
 Architecture: linux-any
 Depends: python3-cffi,
          openzfs-zfsutils (= ${binary:Version}),
          ${misc:Depends},
          ${python3:Depends}
 Replaces: python3-pyzfs
 Conflicts: python3-pyzfs
 Description: wrapper for libzfs_core C library
  libzfs_core is intended to be a stable interface for programmatic
  administration of ZFS. This wrapper provides one-to-one wrappers for
  libzfs_core API functions, but the signatures and types are more natural to
  Python.
  .
  nvlists are wrapped as dictionaries or lists depending on their usage.
  Some parameters have default values depending on typical use for
  increased convenience. Enumerations and bit flags become strings and lists
  of strings in Python. Errors are reported as exceptions rather than integer
  errno-style error codes.  The wrapper takes care to provide one-to-many
  mapping of the error codes to the exceptions by interpreting a context
  in which the error code is produced.
 
 Package: openzfs-pyzfs-doc
 Section: contrib/doc
 Architecture: all
 Depends: ${misc:Depends}, ${sphinxdoc:Depends}
 Recommends: openzfs-python3-pyzfs
 Replaces: pyzfs-doc
 Conflicts: pyzfs-doc
 Description: wrapper for libzfs_core C library (documentation)
  libzfs_core is intended to be a stable interface for programmatic
  administration of ZFS. This wrapper provides one-to-one wrappers for
  libzfs_core API functions, but the signatures and types are more natural to
  Python.
  .
  nvlists are wrapped as dictionaries or lists depending on their usage.
  Some parameters have default values depending on typical use for
  increased convenience. Enumerations and bit flags become strings and lists
  of strings in Python. Errors are reported as exceptions rather than integer
  errno-style error codes.  The wrapper takes care to provide one-to-many
  mapping of the error codes to the exceptions by interpreting a context
  in which the error code is produced.
  .
  This package contains the documentation.
 
 Package: openzfs-zfs-dkms
 Architecture: all
 Depends: dkms (>> 2.1.1.2-5),
          file,
          libc6-dev | libc-dev,
          lsb-release,
          python3 (>> 3.12) | python3-distutils | libpython3-stdlib (<< 3.6.4),
          ${misc:Depends},
          ${perl:Depends}
 Recommends: openzfs-zfs-zed, openzfs-zfsutils (>= ${source:Version}), ${linux:Recommends}
 # suggests debhelper because e.g. `dkms mkdeb -m zfs -v 0.8.2` needs dh_testdir (#909183)
 Suggests: debhelper
 Breaks: spl-dkms (<< 0.8.0~rc1)
 Replaces: spl-dkms, zfs-dkms
 Provides: openzfs-zfs-modules
 Description: OpenZFS filesystem kernel modules for Linux
  OpenZFS is a storage platform that encompasses the functionality of
  traditional filesystems and volume managers. It supports data checksums,
  compression, encryption, snapshots, and more.
  .
  This DKMS package includes the SPA, DMU, ZVOL, and ZPL components of
  OpenZFS.
 
 Package: openzfs-zfs-initramfs
 Architecture: all
 Depends: busybox-initramfs | busybox-static | busybox,
          initramfs-tools,
          openzfs-zfs-modules | openzfs-zfs-dkms,
          openzfs-zfsutils (>= ${source:Version}),
          ${misc:Depends}
 Breaks: zfsutils-linux (<= 0.7.11-2)
 Replaces: zfsutils-linux (<= 0.7.11-2), zfs-initramfs
 Conflicts: zfs-initramfs
 Description: OpenZFS root filesystem capabilities for Linux - initramfs
  OpenZFS is a storage platform that encompasses the functionality of
  traditional filesystems and volume managers. It supports data checksums,
  compression, encryption, snapshots, and more.
  .
  This package adds OpenZFS to the system initramfs with a hook
  for the initramfs-tools infrastructure.
 
 Package: openzfs-zfs-dracut
 Architecture: all
 Depends: dracut,
          openzfs-zfs-modules | openzfs-zfs-dkms,
          openzfs-zfsutils (>= ${source:Version}),
          ${misc:Depends}
 Conflicts: zfs-dracut
 Replaces: zfs-dracut
 Description: OpenZFS root filesystem capabilities for Linux - dracut
  OpenZFS is a storage platform that encompasses the functionality of
  traditional filesystems and volume managers. It supports data checksums,
  compression, encryption, snapshots, and more.
  .
  This package adds OpenZFS to the system initramfs with a hook
  for the dracut infrastructure.
 
 Package: openzfs-zfsutils
 Section: contrib/admin
 Architecture: linux-any
 Pre-Depends: ${misc:Pre-Depends}
 Depends: openzfs-libnvpair3 (= ${binary:Version}),
          openzfs-libuutil3 (= ${binary:Version}),
          openzfs-libzfs4 (= ${binary:Version}),
          openzfs-libzpool5 (= ${binary:Version}),
          python3,
          ${misc:Depends},
          ${shlibs:Depends}
 Recommends: lsb-base, openzfs-zfs-modules | openzfs-zfs-dkms, openzfs-zfs-zed
 Breaks: openrc,
         spl (<< 0.7.9-2),
         spl-dkms (<< 0.8.0~rc1),
         openzfs-zfs-dkms (<< ${source:Version}),
         openzfs-zfs-dkms (>> ${source:Version}...)
 Replaces: spl (<< 0.7.9-2), spl-dkms, zfsutils-linux
 Conflicts: zfs, zfs-fuse, zfsutils-linux
 Suggests: nfs-kernel-server,
           samba-common-bin (>= 3.0.23),
           openzfs-zfs-initramfs | openzfs-zfs-dracut
 Provides: openzfsutils
 Description: command-line tools to manage OpenZFS filesystems
  OpenZFS is a storage platform that encompasses the functionality of
  traditional filesystems and volume managers. It supports data checksums,
  compression, encryption, snapshots, and more.
  .
  This package provides the zfs and zpool commands to create and administer
  OpenZFS filesystems.
 
 Package: openzfs-zfs-zed
 Section: contrib/admin
 Architecture: linux-any
 Pre-Depends: ${misc:Pre-Depends}
 Depends: openzfs-zfs-modules | openzfs-zfs-dkms,
          openzfs-zfsutils (>= ${binary:Version}),
          ${misc:Depends},
          ${shlibs:Depends}
 Conflicts: zfs, zfs-zed
 Replaces: zfs-zed
 Description: OpenZFS Event Daemon
  OpenZFS is a storage platform that encompasses the functionality of
  traditional filesystems and volume managers. It supports data checksums,
  compression, encryption, snapshots, and more.
  .
  ZED (ZFS Event Daemon) monitors events generated by the ZFS kernel
  module. When a zevent (ZFS Event) is posted, ZED will run any ZEDLETs
  (ZFS Event Daemon Linkage for Executable Tasks) that have been enabled
  for the corresponding zevent class.
  .
  This package provides the OpenZFS Event Daemon (zed).
 
 Package: openzfs-zfs-test
 Section: contrib/admin
 Architecture: linux-any
 Depends: acl,
          attr,
          bc,
          fio,
          ksh,
          lsscsi,
          mdadm,
          parted,
          python3,
          openzfs-python3-pyzfs,
          sudo,
          sysstat,
          openzfs-zfs-modules | openzfs-zfs-dkms,
          openzfs-zfsutils (>=${binary:Version}),
          ${misc:Depends},
          ${shlibs:Depends}
 Recommends: nfs-kernel-server
 Breaks: zfsutils-linux (<= 0.7.9-2)
 Replaces: zfsutils-linux (<= 0.7.9-2), zfs-test
 Conflicts: zutils, zfs-test
 Description: OpenZFS test infrastructure and support scripts
  OpenZFS is a storage platform that encompasses the functionality of
  traditional filesystems and volume managers. It supports data checksums,
  compression, encryption, snapshots, and more.
  .
  This package provides the OpenZFS test infrastructure for destructively
  testing and validating a system using OpenZFS. It is entirely optional
  and should only be installed and used in test environments.
diff --git a/sys/contrib/openzfs/contrib/initramfs/scripts/local-top/zfs b/sys/contrib/openzfs/contrib/initramfs/scripts/local-top/zfs
index 6b80e9f43607..fc455077ec94 100755
--- a/sys/contrib/openzfs/contrib/initramfs/scripts/local-top/zfs
+++ b/sys/contrib/openzfs/contrib/initramfs/scripts/local-top/zfs
@@ -1,53 +1,53 @@
 #!/bin/sh
 # shellcheck disable=SC2154
 
 
 if [ "$1" = "prereqs" ]; then
         echo mdadm mdrun multipath
         exit 0
 fi
 
 
 #
 # Helper functions
 #
 message()
 {
         if plymouth --ping 2>/dev/null; then
                 plymouth message --text="$*"
         else
                 echo "$*" >&2
         fi
         return 0
 }
 
 udev_settle()
 {
         # Wait for udev to be ready, see https://launchpad.net/bugs/85640
         if [ -x /sbin/udevadm ]; then
                 /sbin/udevadm settle --timeout=30
         elif [ -x /sbin/udevsettle ]; then
                 /sbin/udevsettle --timeout=30
         fi
         return 0
 }
 
 
 activate_vg()
 {
         # Sanity checks
         if [ ! -x /sbin/lvm ]; then
                 [ "$quiet" != "y" ] && message "lvm is not available"
                 return 1
         fi
 
-        # Detect and activate available volume groups
+        # Detect and auto-activate available volume groups
         /sbin/lvm vgscan
-        /sbin/lvm vgchange -a y --sysinit
+        /sbin/lvm vgchange -aay --sysinit
         return $?
 }
 
 udev_settle
 activate_vg
 
 exit 0
diff --git a/sys/contrib/openzfs/include/os/freebsd/spl/sys/policy.h b/sys/contrib/openzfs/include/os/freebsd/spl/sys/policy.h
index 32c10bdca90e..e1747bb14821 100644
--- a/sys/contrib/openzfs/include/os/freebsd/spl/sys/policy.h
+++ b/sys/contrib/openzfs/include/os/freebsd/spl/sys/policy.h
@@ -1,73 +1,72 @@
 /*
  * Copyright (c) 2007 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
  * $ $FreeBSD$
  */
 
 #ifndef _OPENSOLARIS_SYS_POLICY_H_
 #define	_OPENSOLARIS_SYS_POLICY_H_
 
 #include <sys/param.h>
 #include <sys/xvattr.h>
 #include <sys/vnode.h>
 struct mount;
 struct vattr;
 struct znode;
 
 int	secpolicy_nfs(cred_t *cr);
 int	secpolicy_zfs(cred_t *crd);
-int	secpolicy_zfs_proc(cred_t *cr, proc_t *proc);
 int	secpolicy_sys_config(cred_t *cr, int checkonly);
 int	secpolicy_zinject(cred_t *cr);
 int	secpolicy_fs_unmount(cred_t *cr, struct mount *vfsp);
 int	secpolicy_basic_link(vnode_t *vp, cred_t *cr);
 int	secpolicy_vnode_owner(vnode_t *vp, cred_t *cr, uid_t owner);
 int	secpolicy_vnode_chown(vnode_t *vp, cred_t *cr, uid_t owner);
 int	secpolicy_vnode_stky_modify(cred_t *cr);
 int	secpolicy_vnode_remove(vnode_t *vp, cred_t *cr);
 int	secpolicy_vnode_access(cred_t *cr, vnode_t *vp, uid_t owner,
 	    accmode_t accmode);
 int	secpolicy_vnode_access2(cred_t *cr, vnode_t *vp, uid_t owner,
 	    accmode_t curmode, accmode_t wantmode);
 int	secpolicy_vnode_any_access(cred_t *cr, vnode_t *vp, uid_t owner);
 int	secpolicy_vnode_setdac(vnode_t *vp, cred_t *cr, uid_t owner);
 int	secpolicy_vnode_setattr(cred_t *cr, vnode_t *vp, struct vattr *vap,
 	    const struct vattr *ovap, int flags,
 	    int unlocked_access(void *, int, cred_t *), void *node);
 int	secpolicy_vnode_create_gid(cred_t *cr);
 int	secpolicy_vnode_setids_setgids(vnode_t *vp, cred_t *cr, gid_t gid);
 int	secpolicy_vnode_setid_retain(struct znode *zp, cred_t *cr,
 	    boolean_t issuidroot);
 void	secpolicy_setid_clear(struct vattr *vap, vnode_t *vp, cred_t *cr);
 int	secpolicy_setid_setsticky_clear(vnode_t *vp, struct vattr *vap,
 	    const struct vattr *ovap, cred_t *cr);
 int	secpolicy_fs_owner(struct mount *vfsp, cred_t *cr);
 int	secpolicy_fs_mount(cred_t *cr, vnode_t *mvp, struct mount *vfsp);
 void	secpolicy_fs_mount_clearopts(cred_t *cr, struct mount *vfsp);
 int	secpolicy_xvattr(vnode_t *vp, xvattr_t *xvap, uid_t owner, cred_t *cr,
 	    vtype_t vtype);
 int	secpolicy_smb(cred_t *cr);
 
 #endif	/* _OPENSOLARIS_SYS_POLICY_H_ */
diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h
index c0d3770748ea..26e7b0b2a34a 100644
--- a/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h
+++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/blkdev_compat.h
@@ -1,591 +1,591 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
  * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
  * LLNL-CODE-403049.
  */
 
 #ifndef _ZFS_BLKDEV_H
 #define	_ZFS_BLKDEV_H
 
 #include <linux/blkdev.h>
 #include <linux/backing-dev.h>
 #include <linux/hdreg.h>
 #include <linux/major.h>
 #include <linux/msdos_fs.h>	/* for SECTOR_* */
 #include <linux/bio.h>
 #include <linux/blk-mq.h>
 
 /*
  * 6.11 API
  * Setting the flush flags directly is no longer possible; flush flags are set
  * on the queue_limits structure and passed to blk_disk_alloc(). In this case
  * we remove this function entirely.
  */
 #if !defined(HAVE_BLK_ALLOC_DISK_2ARG) || \
 	!defined(HAVE_BLKDEV_QUEUE_LIMITS_FEATURES)
 static inline void
 blk_queue_set_write_cache(struct request_queue *q, bool on)
 {
 	if (on) {
 		blk_queue_flag_set(QUEUE_FLAG_WC, q);
 		blk_queue_flag_set(QUEUE_FLAG_FUA, q);
 	} else {
 		blk_queue_flag_clear(QUEUE_FLAG_WC, q);
 		blk_queue_flag_clear(QUEUE_FLAG_FUA, q);
 	}
 }
 #endif /* !HAVE_BLK_ALLOC_DISK_2ARG || !HAVE_BLKDEV_QUEUE_LIMITS_FEATURES */
 
 static inline void
 blk_queue_set_read_ahead(struct request_queue *q, unsigned long ra_pages)
 {
 #if !defined(HAVE_BLK_QUEUE_UPDATE_READAHEAD) && \
 	!defined(HAVE_DISK_UPDATE_READAHEAD)
 #if defined(HAVE_BLK_QUEUE_BDI_DYNAMIC)
 	q->backing_dev_info->ra_pages = ra_pages;
 #elif defined(HAVE_BLK_QUEUE_DISK_BDI)
 	q->disk->bdi->ra_pages = ra_pages;
 #else
 	q->backing_dev_info.ra_pages = ra_pages;
 #endif
 #endif
 }
 
 #define	BIO_BI_SECTOR(bio)	(bio)->bi_iter.bi_sector
 #define	BIO_BI_SIZE(bio)	(bio)->bi_iter.bi_size
 #define	BIO_BI_IDX(bio)		(bio)->bi_iter.bi_idx
 #define	BIO_BI_SKIP(bio)	(bio)->bi_iter.bi_bvec_done
 #define	bio_for_each_segment4(bv, bvp, b, i)	\
 	bio_for_each_segment((bv), (b), (i))
 typedef struct bvec_iter bvec_iterator_t;
 
 static inline void
 bio_set_flags_failfast(struct block_device *bdev, int *flags, bool dev,
     bool transport, bool driver)
 {
 #ifdef CONFIG_BUG
 	/*
 	 * Disable FAILFAST for loopback devices because of the
 	 * following incorrect BUG_ON() in loop_make_request().
 	 * This support is also disabled for md devices because the
 	 * test suite layers md devices on top of loopback devices.
 	 * This may be removed when the loopback driver is fixed.
 	 *
 	 *   BUG_ON(!lo || (rw != READ && rw != WRITE));
 	 */
 	if ((MAJOR(bdev->bd_dev) == LOOP_MAJOR) ||
 	    (MAJOR(bdev->bd_dev) == MD_MAJOR))
 		return;
 
 #ifdef BLOCK_EXT_MAJOR
 	if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
 		return;
 #endif /* BLOCK_EXT_MAJOR */
 #endif /* CONFIG_BUG */
 
 	if (dev)
 		*flags |= REQ_FAILFAST_DEV;
 	if (transport)
 		*flags |= REQ_FAILFAST_TRANSPORT;
 	if (driver)
 		*flags |= REQ_FAILFAST_DRIVER;
 }
 
 /*
  * Maximum disk label length, it may be undefined for some kernels.
  */
 #if !defined(DISK_NAME_LEN)
 #define	DISK_NAME_LEN	32
 #endif /* DISK_NAME_LEN */
 
 static inline int
 bi_status_to_errno(blk_status_t status)
 {
 	switch (status)	{
 	case BLK_STS_OK:
 		return (0);
 	case BLK_STS_NOTSUPP:
 		return (EOPNOTSUPP);
 	case BLK_STS_TIMEOUT:
 		return (ETIMEDOUT);
 	case BLK_STS_NOSPC:
 		return (ENOSPC);
 	case BLK_STS_TRANSPORT:
 		return (ENOLINK);
 	case BLK_STS_TARGET:
 		return (EREMOTEIO);
 #ifdef HAVE_BLK_STS_RESV_CONFLICT
 	case BLK_STS_RESV_CONFLICT:
 #else
 	case BLK_STS_NEXUS:
 #endif
 		return (EBADE);
 	case BLK_STS_MEDIUM:
 		return (ENODATA);
 	case BLK_STS_PROTECTION:
 		return (EILSEQ);
 	case BLK_STS_RESOURCE:
 		return (ENOMEM);
 	case BLK_STS_AGAIN:
 		return (EAGAIN);
 	case BLK_STS_IOERR:
 		return (EIO);
 	default:
 		return (EIO);
 	}
 }
 
 static inline blk_status_t
 errno_to_bi_status(int error)
 {
 	switch (error) {
 	case 0:
 		return (BLK_STS_OK);
 	case EOPNOTSUPP:
 		return (BLK_STS_NOTSUPP);
 	case ETIMEDOUT:
 		return (BLK_STS_TIMEOUT);
 	case ENOSPC:
 		return (BLK_STS_NOSPC);
 	case ENOLINK:
 		return (BLK_STS_TRANSPORT);
 	case EREMOTEIO:
 		return (BLK_STS_TARGET);
 	case EBADE:
 #ifdef HAVE_BLK_STS_RESV_CONFLICT
 		return (BLK_STS_RESV_CONFLICT);
 #else
 		return (BLK_STS_NEXUS);
 #endif
 	case ENODATA:
 		return (BLK_STS_MEDIUM);
 	case EILSEQ:
 		return (BLK_STS_PROTECTION);
 	case ENOMEM:
 		return (BLK_STS_RESOURCE);
 	case EAGAIN:
 		return (BLK_STS_AGAIN);
 	case EIO:
 		return (BLK_STS_IOERR);
 	default:
 		return (BLK_STS_IOERR);
 	}
 }
 
 /*
  * 5.15 MACRO,
  *   GD_DEAD
  *
  * 2.6.36 - 5.14 MACRO,
  *   GENHD_FL_UP
  *
  * Check the disk status and return B_TRUE if alive
  * otherwise B_FALSE
  */
 static inline boolean_t
 zfs_check_disk_status(struct block_device *bdev)
 {
 #if defined(GENHD_FL_UP)
 	return (!!(bdev->bd_disk->flags & GENHD_FL_UP));
 #elif defined(GD_DEAD)
 	return (!test_bit(GD_DEAD, &bdev->bd_disk->state));
 #else
 /*
  * This is encountered if neither GENHD_FL_UP nor GD_DEAD is available in
  * the kernel - likely due to an MACRO change that needs to be chased down.
  */
 #error "Unsupported kernel: no usable disk status check"
 #endif
 }
 
 /*
  * 5.17 API change
  *
  * GENHD_FL_EXT_DEVT flag removed
  * GENHD_FL_NO_PART_SCAN renamed GENHD_FL_NO_PART
  */
 #ifndef HAVE_GENHD_FL_EXT_DEVT
 #define	GENHD_FL_EXT_DEVT	(0)
 #endif
 #ifndef HAVE_GENHD_FL_NO_PART
 #define	GENHD_FL_NO_PART	(GENHD_FL_NO_PART_SCAN)
 #endif
 
 /*
  * 4.1 API,
  * 3.10.0 CentOS 7.x API,
  *   blkdev_reread_part()
  *
  * For older kernels trigger a re-reading of the partition table by calling
  * check_disk_change() which calls flush_disk() to invalidate the device.
  *
  * For newer kernels (as of 5.10), bdev_check_media_change is used, in favor of
  * check_disk_change(), with the modification that invalidation is no longer
  * forced.
  */
 #ifdef HAVE_CHECK_DISK_CHANGE
 #define	zfs_check_media_change(bdev)	check_disk_change(bdev)
 #ifdef HAVE_BLKDEV_REREAD_PART
 #define	vdev_bdev_reread_part(bdev)	blkdev_reread_part(bdev)
 #else
 #define	vdev_bdev_reread_part(bdev)	check_disk_change(bdev)
 #endif /* HAVE_BLKDEV_REREAD_PART */
 #else
 #ifdef HAVE_BDEV_CHECK_MEDIA_CHANGE
 static inline int
 zfs_check_media_change(struct block_device *bdev)
 {
 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
 	struct gendisk *gd = bdev->bd_disk;
 	const struct block_device_operations *bdo = gd->fops;
 #endif
 
 	if (!bdev_check_media_change(bdev))
 		return (0);
 
 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
 	/*
 	 * Force revalidation, to mimic the old behavior of
 	 * check_disk_change()
 	 */
 	if (bdo->revalidate_disk)
 		bdo->revalidate_disk(gd);
 #endif
 
 	return (0);
 }
 #define	vdev_bdev_reread_part(bdev)	zfs_check_media_change(bdev)
 #elif defined(HAVE_DISK_CHECK_MEDIA_CHANGE)
 #define	vdev_bdev_reread_part(bdev)	disk_check_media_change(bdev->bd_disk)
 #define	zfs_check_media_change(bdev)	disk_check_media_change(bdev->bd_disk)
 #else
 /*
  * This is encountered if check_disk_change() and bdev_check_media_change()
  * are not available in the kernel - likely due to an API change that needs
  * to be chased down.
  */
 #error "Unsupported kernel: no usable disk change check"
 #endif /* HAVE_BDEV_CHECK_MEDIA_CHANGE */
 #endif /* HAVE_CHECK_DISK_CHANGE */
 
 /*
  * 2.6.27 API change
  * The function was exported for use, prior to this it existed but the
  * symbol was not exported.
  *
  * 5.11 API change
  * Changed to take a dev_t argument which is set on success and return a
  * non-zero error code on failure.
  */
 static inline int
 vdev_lookup_bdev(const char *path, dev_t *dev)
 {
 #if defined(HAVE_DEVT_LOOKUP_BDEV)
 	return (lookup_bdev(path, dev));
 #elif defined(HAVE_1ARG_LOOKUP_BDEV)
 	struct block_device *bdev = lookup_bdev(path);
 	if (IS_ERR(bdev))
 		return (PTR_ERR(bdev));
 
 	*dev = bdev->bd_dev;
 	bdput(bdev);
 
 	return (0);
 #else
 #error "Unsupported kernel"
 #endif
 }
 
 #if defined(HAVE_BLK_MODE_T)
 #define	blk_mode_is_open_write(flag)	((flag) & BLK_OPEN_WRITE)
 #else
 #define	blk_mode_is_open_write(flag)	((flag) & FMODE_WRITE)
 #endif
 
 /*
  * Kernels without bio_set_op_attrs use bi_rw for the bio flags.
  */
 #if !defined(HAVE_BIO_SET_OP_ATTRS)
 static inline void
 bio_set_op_attrs(struct bio *bio, unsigned rw, unsigned flags)
 {
 	bio->bi_opf = rw | flags;
 }
 #endif
 
 /*
  * bio_set_flush - Set the appropriate flags in a bio to guarantee
  * data are on non-volatile media on completion.
  */
 static inline void
 bio_set_flush(struct bio *bio)
 {
 	bio_set_op_attrs(bio, 0, REQ_PREFLUSH | REQ_OP_WRITE);
 }
 
 /*
  * 4.8 API,
  *   REQ_OP_FLUSH
  *
  * in all cases but may have a performance impact for some kernels.  It
  * has the advantage of minimizing kernel specific changes in the zvol code.
  *
  */
 static inline boolean_t
 bio_is_flush(struct bio *bio)
 {
-	return (bio_op(bio) == REQ_OP_FLUSH);
+	return (bio_op(bio) == REQ_OP_FLUSH || op_is_flush(bio->bi_opf));
 }
 
 /*
  * 4.8 API,
  *   REQ_FUA flag moved to bio->bi_opf
  */
 static inline boolean_t
 bio_is_fua(struct bio *bio)
 {
 	return (bio->bi_opf & REQ_FUA);
 }
 
 /*
  * 4.8 API,
  *   REQ_OP_DISCARD
  *
  * In all cases the normal I/O path is used for discards.  The only
  * difference is how the kernel tags individual I/Os as discards.
  */
 static inline boolean_t
 bio_is_discard(struct bio *bio)
 {
 	return (bio_op(bio) == REQ_OP_DISCARD);
 }
 
 /*
  * 4.8 API,
  *   REQ_OP_SECURE_ERASE
  */
 static inline boolean_t
 bio_is_secure_erase(struct bio *bio)
 {
 	return (bio_op(bio) == REQ_OP_SECURE_ERASE);
 }
 
 /*
  * 2.6.33 API change
  * Discard granularity and alignment restrictions may now be set.  For
  * older kernels which do not support this it is safe to skip it.
  */
 static inline void
 blk_queue_discard_granularity(struct request_queue *q, unsigned int dg)
 {
 	q->limits.discard_granularity = dg;
 }
 
 /*
  * 5.19 API,
  *   bdev_max_discard_sectors()
  *
  * 2.6.32 API,
  *   blk_queue_discard()
  */
 static inline boolean_t
 bdev_discard_supported(struct block_device *bdev)
 {
 #if defined(HAVE_BDEV_MAX_DISCARD_SECTORS)
 	return (bdev_max_discard_sectors(bdev) > 0 &&
 	    bdev_discard_granularity(bdev) > 0);
 #elif defined(HAVE_BLK_QUEUE_DISCARD)
 	return (blk_queue_discard(bdev_get_queue(bdev)) > 0 &&
 	    bdev_get_queue(bdev)->limits.discard_granularity > 0);
 #else
 #error "Unsupported kernel"
 #endif
 }
 
 /*
  * 5.19 API,
  *   bdev_max_secure_erase_sectors()
  *
  * 4.8 API,
  *   blk_queue_secure_erase()
  */
 static inline boolean_t
 bdev_secure_discard_supported(struct block_device *bdev)
 {
 #if defined(HAVE_BDEV_MAX_SECURE_ERASE_SECTORS)
 	return (!!bdev_max_secure_erase_sectors(bdev));
 #elif defined(HAVE_BLK_QUEUE_SECURE_ERASE)
 	return (!!blk_queue_secure_erase(bdev_get_queue(bdev)));
 #else
 #error "Unsupported kernel"
 #endif
 }
 
 /*
  * A common holder for vdev_bdev_open() is used to relax the exclusive open
  * semantics slightly.  Internal vdev disk callers may pass VDEV_HOLDER to
  * allow them to open the device multiple times.  Other kernel callers and
  * user space processes which don't pass this value will get EBUSY.  This is
  * currently required for the correct operation of hot spares.
  */
 #define	VDEV_HOLDER			((void *)0x2401de7)
 
 static inline unsigned long
 blk_generic_start_io_acct(struct request_queue *q __attribute__((unused)),
     struct gendisk *disk __attribute__((unused)),
     int rw __attribute__((unused)), struct bio *bio)
 {
 #if defined(HAVE_BDEV_IO_ACCT_63)
 	return (bdev_start_io_acct(bio->bi_bdev, bio_op(bio),
 	    jiffies));
 #elif defined(HAVE_BDEV_IO_ACCT_OLD)
 	return (bdev_start_io_acct(bio->bi_bdev, bio_sectors(bio),
 	    bio_op(bio), jiffies));
 #elif defined(HAVE_DISK_IO_ACCT)
 	return (disk_start_io_acct(disk, bio_sectors(bio), bio_op(bio)));
 #elif defined(HAVE_BIO_IO_ACCT)
 	return (bio_start_io_acct(bio));
 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
 	unsigned long start_time = jiffies;
 	generic_start_io_acct(q, rw, bio_sectors(bio), &disk->part0);
 	return (start_time);
 #else
 	/* Unsupported */
 	return (0);
 #endif
 }
 
 static inline void
 blk_generic_end_io_acct(struct request_queue *q __attribute__((unused)),
     struct gendisk *disk __attribute__((unused)),
     int rw __attribute__((unused)), struct bio *bio, unsigned long start_time)
 {
 #if defined(HAVE_BDEV_IO_ACCT_63)
 	bdev_end_io_acct(bio->bi_bdev, bio_op(bio), bio_sectors(bio),
 	    start_time);
 #elif defined(HAVE_BDEV_IO_ACCT_OLD)
 	bdev_end_io_acct(bio->bi_bdev, bio_op(bio), start_time);
 #elif defined(HAVE_DISK_IO_ACCT)
 	disk_end_io_acct(disk, bio_op(bio), start_time);
 #elif defined(HAVE_BIO_IO_ACCT)
 	bio_end_io_acct(bio, start_time);
 #elif defined(HAVE_GENERIC_IO_ACCT_4ARG)
 	generic_end_io_acct(q, rw, &disk->part0, start_time);
 #endif
 }
 
 #ifndef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
 static inline struct request_queue *
 blk_generic_alloc_queue(make_request_fn make_request, int node_id)
 {
 #if defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN)
 	return (blk_alloc_queue(make_request, node_id));
 #elif defined(HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH)
 	return (blk_alloc_queue_rh(make_request, node_id));
 #else
 	struct request_queue *q = blk_alloc_queue(GFP_KERNEL);
 	if (q != NULL)
 		blk_queue_make_request(q, make_request);
 
 	return (q);
 #endif
 }
 #endif /* !HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
 
 /*
  * All the io_*() helper functions below can operate on a bio, or a rq, but
  * not both.  The older submit_bio() codepath will pass a bio, and the
  * newer blk-mq codepath will pass a rq.
  */
 static inline int
 io_data_dir(struct bio *bio, struct request *rq)
 {
 	if (rq != NULL) {
 		if (op_is_write(req_op(rq))) {
 			return (WRITE);
 		} else {
 			return (READ);
 		}
 	}
 	return (bio_data_dir(bio));
 }
 
 static inline int
 io_is_flush(struct bio *bio, struct request *rq)
 {
 	if (rq != NULL)
 		return (req_op(rq) == REQ_OP_FLUSH);
 	return (bio_is_flush(bio));
 }
 
 static inline int
 io_is_discard(struct bio *bio, struct request *rq)
 {
 	if (rq != NULL)
 		return (req_op(rq) == REQ_OP_DISCARD);
 	return (bio_is_discard(bio));
 }
 
 static inline int
 io_is_secure_erase(struct bio *bio, struct request *rq)
 {
 	if (rq != NULL)
 		return (req_op(rq) == REQ_OP_SECURE_ERASE);
 	return (bio_is_secure_erase(bio));
 }
 
 static inline int
 io_is_fua(struct bio *bio, struct request *rq)
 {
 	if (rq != NULL)
 		return (rq->cmd_flags & REQ_FUA);
 	return (bio_is_fua(bio));
 }
 
 
 static inline uint64_t
 io_offset(struct bio *bio, struct request *rq)
 {
 	if (rq != NULL)
 		return (blk_rq_pos(rq) << 9);
 	return (BIO_BI_SECTOR(bio) << 9);
 }
 
 static inline uint64_t
 io_size(struct bio *bio, struct request *rq)
 {
 	if (rq != NULL)
 		return (blk_rq_bytes(rq));
 	return (BIO_BI_SIZE(bio));
 }
 
 static inline int
 io_has_data(struct bio *bio, struct request *rq)
 {
 	if (rq != NULL)
 		return (bio_has_data(rq->bio));
 	return (bio_has_data(bio));
 }
 #endif /* _ZFS_BLKDEV_H */
diff --git a/sys/contrib/openzfs/include/os/linux/kernel/linux/page_compat.h b/sys/contrib/openzfs/include/os/linux/kernel/linux/page_compat.h
index 963b96ba6351..7dcf53bbea47 100644
--- a/sys/contrib/openzfs/include/os/linux/kernel/linux/page_compat.h
+++ b/sys/contrib/openzfs/include/os/linux/kernel/linux/page_compat.h
@@ -1,11 +1,11 @@
 #ifndef	_ZFS_PAGE_COMPAT_H
 #define	_ZFS_PAGE_COMPAT_H
 
 /*
  * Create our own accessor functions to follow the Linux API changes
  */
-#define	nr_file_pages() global_node_page_state(NR_FILE_PAGES)
-#define	nr_inactive_anon_pages() global_node_page_state(NR_INACTIVE_ANON)
+#define	nr_file_pages() (global_node_page_state(NR_ACTIVE_FILE) + \
+			global_node_page_state(NR_INACTIVE_FILE))
 #define	nr_inactive_file_pages() global_node_page_state(NR_INACTIVE_FILE)
 
 #endif /* _ZFS_PAGE_COMPAT_H */
diff --git a/sys/contrib/openzfs/include/os/linux/spl/sys/taskq.h b/sys/contrib/openzfs/include/os/linux/spl/sys/taskq.h
index 8c4fee5299ff..0c3149b1efc5 100644
--- a/sys/contrib/openzfs/include/os/linux/spl/sys/taskq.h
+++ b/sys/contrib/openzfs/include/os/linux/spl/sys/taskq.h
@@ -1,170 +1,171 @@
 /*
  *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
  *  Copyright (C) 2007 The Regents of the University of California.
  *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
  *  UCRL-CODE-235197
  *
  *  This file is part of the SPL, Solaris Porting Layer.
  *
  *  The SPL is free software; you can redistribute it and/or modify it
  *  under the terms of the GNU General Public License as published by the
  *  Free Software Foundation; either version 2 of the License, or (at your
  *  option) any later version.
  *
  *  The SPL is distributed in the hope that it will be useful, but WITHOUT
  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  *  for more details.
  *
  *  You should have received a copy of the GNU General Public License along
  *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef _SPL_TASKQ_H
 #define	_SPL_TASKQ_H
 
 #include <linux/module.h>
 #include <linux/gfp.h>
 #include <linux/slab.h>
 #include <linux/interrupt.h>
 #include <linux/kthread.h>
 #include <sys/types.h>
 #include <sys/thread.h>
 #include <sys/rwlock.h>
 #include <sys/wait.h>
+#include <sys/kstat.h>
 
 #define	TASKQ_NAMELEN		31
 
 #define	TASKQ_PREPOPULATE	0x00000001
 #define	TASKQ_CPR_SAFE		0x00000002
 #define	TASKQ_DYNAMIC		0x00000004
 #define	TASKQ_THREADS_CPU_PCT	0x00000008
 #define	TASKQ_DC_BATCH		0x00000010
 #define	TASKQ_ACTIVE		0x80000000
 
 /*
  * Flags for taskq_dispatch. TQ_SLEEP/TQ_NOSLEEP should be same as
  * KM_SLEEP/KM_NOSLEEP.  TQ_NOQUEUE/TQ_NOALLOC are set particularly
  * large so as not to conflict with already used GFP_* defines.
  */
 #define	TQ_SLEEP		0x00000000
 #define	TQ_NOSLEEP		0x00000001
 #define	TQ_PUSHPAGE		0x00000002
 #define	TQ_NOQUEUE		0x01000000
 #define	TQ_NOALLOC		0x02000000
 #define	TQ_NEW			0x04000000
 #define	TQ_FRONT		0x08000000
 
 /*
  * Reserved taskqid values.
  */
 #define	TASKQID_INVALID		((taskqid_t)0)
 #define	TASKQID_INITIAL		((taskqid_t)1)
 
 /*
  * spin_lock(lock) and spin_lock_nested(lock,0) are equivalent,
  * so TQ_LOCK_DYNAMIC must not evaluate to 0
  */
 typedef enum tq_lock_role {
 	TQ_LOCK_GENERAL =	0,
 	TQ_LOCK_DYNAMIC =	1,
 } tq_lock_role_t;
 
 typedef unsigned long taskqid_t;
 typedef void (task_func_t)(void *);
 
 typedef struct taskq {
 	spinlock_t		tq_lock;	/* protects taskq_t */
 	char			*tq_name;	/* taskq name */
 	int			tq_instance;	/* instance of tq_name */
 	struct list_head	tq_thread_list;	/* list of all threads */
 	struct list_head	tq_active_list;	/* list of active threads */
 	int			tq_nactive;	/* # of active threads */
 	int			tq_nthreads;	/* # of existing threads */
 	int			tq_nspawn;	/* # of threads being spawned */
 	int			tq_maxthreads;	/* # of threads maximum */
 	/* If PERCPU flag is set, percent of NCPUs to have as threads */
 	int			tq_cpu_pct;
 	int			tq_pri;		/* priority */
 	int			tq_minalloc;	/* min taskq_ent_t pool size */
 	int			tq_maxalloc;	/* max taskq_ent_t pool size */
 	int			tq_nalloc;	/* cur taskq_ent_t pool size */
 	uint_t			tq_flags;	/* flags */
 	taskqid_t		tq_next_id;	/* next pend/work id */
 	taskqid_t		tq_lowest_id;	/* lowest pend/work id */
 	struct list_head	tq_free_list;	/* free taskq_ent_t's */
 	struct list_head	tq_pend_list;	/* pending taskq_ent_t's */
 	struct list_head	tq_prio_list;	/* priority taskq_ent_t's */
 	struct list_head	tq_delay_list;	/* delayed taskq_ent_t's */
 	struct list_head	tq_taskqs;	/* all taskq_t's */
 	wait_queue_head_t	tq_work_waitq;	/* new work waitq */
 	wait_queue_head_t	tq_wait_waitq;	/* wait waitq */
 	tq_lock_role_t		tq_lock_class;	/* class when taking tq_lock */
 	/* list node for the cpu hotplug callback */
 	struct hlist_node	tq_hp_cb_node;
 	boolean_t		tq_hp_support;
 	unsigned long		lastspawnstop;	/* when to purge dynamic */
 } taskq_t;
 
 typedef struct taskq_ent {
 	spinlock_t		tqent_lock;
 	wait_queue_head_t	tqent_waitq;
 	struct timer_list	tqent_timer;
 	struct list_head	tqent_list;
 	taskqid_t		tqent_id;
 	task_func_t		*tqent_func;
 	void			*tqent_arg;
 	taskq_t			*tqent_taskq;
 	uintptr_t		tqent_flags;
 	unsigned long		tqent_birth;
 } taskq_ent_t;
 
 #define	TQENT_FLAG_PREALLOC	0x1
 #define	TQENT_FLAG_CANCEL	0x2
 
 typedef struct taskq_thread {
 	struct list_head	tqt_thread_list;
 	struct list_head	tqt_active_list;
 	struct task_struct	*tqt_thread;
 	taskq_t			*tqt_tq;
 	taskqid_t		tqt_id;
 	taskq_ent_t		*tqt_task;
 	uintptr_t		tqt_flags;
 } taskq_thread_t;
 
 /* Global system-wide dynamic task queue available for all consumers */
 extern taskq_t *system_taskq;
 /* Global dynamic task queue for long delay */
 extern taskq_t *system_delay_taskq;
 
 /* List of all taskqs */
 extern struct list_head tq_list;
 extern struct rw_semaphore tq_list_sem;
 
 extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t);
 extern taskqid_t taskq_dispatch_delay(taskq_t *, task_func_t, void *,
     uint_t, clock_t);
 extern void taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t,
     taskq_ent_t *);
 extern int taskq_empty_ent(taskq_ent_t *);
 extern void taskq_init_ent(taskq_ent_t *);
 extern taskq_t *taskq_create(const char *, int, pri_t, int, int, uint_t);
 extern void taskq_destroy(taskq_t *);
 extern void taskq_wait_id(taskq_t *, taskqid_t);
 extern void taskq_wait_outstanding(taskq_t *, taskqid_t);
 extern void taskq_wait(taskq_t *);
 extern int taskq_cancel_id(taskq_t *, taskqid_t);
 extern int taskq_member(taskq_t *, kthread_t *);
 extern taskq_t *taskq_of_curthread(void);
 
 #define	taskq_create_proc(name, nthreads, pri, min, max, proc, flags) \
     taskq_create(name, nthreads, pri, min, max, flags)
 #define	taskq_create_sysdc(name, nthreads, min, max, proc, dc, flags) \
 	((void) sizeof (dc), \
 	    taskq_create(name, nthreads, maxclsyspri, min, max, flags))
 
 int spl_taskq_init(void);
 void spl_taskq_fini(void);
 
 #endif  /* _SPL_TASKQ_H */
diff --git a/sys/contrib/openzfs/include/os/linux/spl/sys/uio.h b/sys/contrib/openzfs/include/os/linux/spl/sys/uio.h
index 5e6ea8d3c221..82a227d76ca0 100644
--- a/sys/contrib/openzfs/include/os/linux/spl/sys/uio.h
+++ b/sys/contrib/openzfs/include/os/linux/spl/sys/uio.h
@@ -1,180 +1,180 @@
 /*
  *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
  *  Copyright (C) 2007 The Regents of the University of California.
  *  Copyright (c) 2015 by Chunwei Chen. All rights reserved.
  *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
  *  UCRL-CODE-235197
  *
  *  This file is part of the SPL, Solaris Porting Layer.
  *
  *  The SPL is free software; you can redistribute it and/or modify it
  *  under the terms of the GNU General Public License as published by the
  *  Free Software Foundation; either version 2 of the License, or (at your
  *  option) any later version.
  *
  *  The SPL is distributed in the hope that it will be useful, but WITHOUT
  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  *  for more details.
  *
  *  You should have received a copy of the GNU General Public License along
  *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
  */
 
 #ifndef _SPL_UIO_H
 #define	_SPL_UIO_H
 
 #include <sys/debug.h>
 #include <linux/uio.h>
 #include <linux/blkdev.h>
 #include <linux/blkdev_compat.h>
 #include <linux/mm.h>
 #include <linux/bio.h>
 #include <asm/uaccess.h>
 #include <sys/types.h>
 
 #if defined(HAVE_VFS_IOV_ITER) && defined(HAVE_FAULT_IN_IOV_ITER_READABLE)
 #define	iov_iter_fault_in_readable(a, b)	fault_in_iov_iter_readable(a, b)
 #endif
 
 typedef struct iovec iovec_t;
 
 typedef enum zfs_uio_rw {
 	UIO_READ =		0,
 	UIO_WRITE =		1,
 } zfs_uio_rw_t;
 
 typedef enum zfs_uio_seg {
 	UIO_USERSPACE =		0,
 	UIO_SYSSPACE =		1,
 	UIO_BVEC =		2,
 #if defined(HAVE_VFS_IOV_ITER)
 	UIO_ITER =		3,
 #endif
 } zfs_uio_seg_t;
 
 typedef struct zfs_uio {
 	union {
 		const struct iovec	*uio_iov;
 		const struct bio_vec	*uio_bvec;
 #if defined(HAVE_VFS_IOV_ITER)
 		struct iov_iter		*uio_iter;
 #endif
 	};
 	int		uio_iovcnt;
 	offset_t	uio_loffset;
 	zfs_uio_seg_t	uio_segflg;
 	boolean_t	uio_fault_disable;
 	uint16_t	uio_fmode;
 	uint16_t	uio_extflg;
 	ssize_t		uio_resid;
 
 	size_t		uio_skip;
 
 	struct request	*rq;
 } zfs_uio_t;
 
 
 #define	zfs_uio_segflg(u)		(u)->uio_segflg
 #define	zfs_uio_offset(u)		(u)->uio_loffset
 #define	zfs_uio_resid(u)		(u)->uio_resid
 #define	zfs_uio_iovcnt(u)		(u)->uio_iovcnt
 #define	zfs_uio_iovlen(u, idx)		(u)->uio_iov[(idx)].iov_len
 #define	zfs_uio_iovbase(u, idx)		(u)->uio_iov[(idx)].iov_base
 #define	zfs_uio_fault_disable(u, set)	(u)->uio_fault_disable = set
 #define	zfs_uio_rlimit_fsize(z, u)	(0)
 #define	zfs_uio_fault_move(p, n, rw, u)	zfs_uiomove((p), (n), (rw), (u))
 
 extern int zfs_uio_prefaultpages(ssize_t, zfs_uio_t *);
 
 static inline void
 zfs_uio_setoffset(zfs_uio_t *uio, offset_t off)
 {
 	uio->uio_loffset = off;
 }
 
 static inline void
 zfs_uio_advance(zfs_uio_t *uio, ssize_t size)
 {
 	uio->uio_resid -= size;
 	uio->uio_loffset += size;
 }
 
 static inline void
 zfs_uio_iovec_init(zfs_uio_t *uio, const struct iovec *iov,
     unsigned long nr_segs, offset_t offset, zfs_uio_seg_t seg, ssize_t resid,
     size_t skip)
 {
 	ASSERT(seg == UIO_USERSPACE || seg == UIO_SYSSPACE);
 
 	uio->uio_iov = iov;
 	uio->uio_iovcnt = nr_segs;
 	uio->uio_loffset = offset;
 	uio->uio_segflg = seg;
 	uio->uio_fault_disable = B_FALSE;
 	uio->uio_fmode = 0;
 	uio->uio_extflg = 0;
 	uio->uio_resid = resid;
 	uio->uio_skip = skip;
 }
 
 static inline void
 zfs_uio_bvec_init(zfs_uio_t *uio, struct bio *bio, struct request *rq)
 {
 	/* Either bio or rq will be set, but not both */
 	ASSERT3P(uio, !=, bio);
 
 	if (bio) {
 		uio->uio_iovcnt = bio->bi_vcnt - BIO_BI_IDX(bio);
 		uio->uio_bvec = &bio->bi_io_vec[BIO_BI_IDX(bio)];
 	} else {
 		uio->uio_bvec = NULL;
 		uio->uio_iovcnt = 0;
 	}
 
 	uio->uio_loffset = io_offset(bio, rq);
 	uio->uio_segflg = UIO_BVEC;
 	uio->uio_fault_disable = B_FALSE;
 	uio->uio_fmode = 0;
 	uio->uio_extflg = 0;
 	uio->uio_resid = io_size(bio, rq);
 	if (bio) {
 		uio->uio_skip = BIO_BI_SKIP(bio);
 	} else {
 		uio->uio_skip = 0;
 	}
 
 	uio->rq = rq;
 }
 
 #if defined(HAVE_VFS_IOV_ITER)
 static inline void
 zfs_uio_iov_iter_init(zfs_uio_t *uio, struct iov_iter *iter, offset_t offset,
-    ssize_t resid, size_t skip)
+    ssize_t resid)
 {
 	uio->uio_iter = iter;
 	uio->uio_iovcnt = iter->nr_segs;
 	uio->uio_loffset = offset;
 	uio->uio_segflg = UIO_ITER;
 	uio->uio_fault_disable = B_FALSE;
 	uio->uio_fmode = 0;
 	uio->uio_extflg = 0;
 	uio->uio_resid = resid;
-	uio->uio_skip = skip;
+	uio->uio_skip = 0;
 }
 #endif
 
 #if defined(HAVE_ITER_IOV)
 #define	zfs_uio_iter_iov(iter)	iter_iov((iter))
 #else
 #define	zfs_uio_iter_iov(iter)	(iter)->iov
 #endif
 
 #if defined(HAVE_IOV_ITER_TYPE)
 #define	zfs_uio_iov_iter_type(iter)	iov_iter_type((iter))
 #else
 #define	zfs_uio_iov_iter_type(iter)	(iter)->type
 #endif
 
 #endif /* SPL_UIO_H */
diff --git a/sys/contrib/openzfs/include/os/linux/zfs/sys/abd_os.h b/sys/contrib/openzfs/include/os/linux/zfs/sys/abd_os.h
index ce4f5a2bdf9b..e38d8b81b20b 100644
--- a/sys/contrib/openzfs/include/os/linux/zfs/sys/abd_os.h
+++ b/sys/contrib/openzfs/include/os/linux/zfs/sys/abd_os.h
@@ -1,62 +1,62 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
  * Copyright (c) 2016, 2019 by Delphix. All rights reserved.
  */
 
 #ifndef _ABD_OS_H
 #define	_ABD_OS_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
+struct abd;
+
 struct abd_scatter {
 	uint_t		abd_offset;
 	uint_t		abd_nents;
 	struct scatterlist *abd_sgl;
 };
 
 struct abd_linear {
 	void		*abd_buf;
 	struct scatterlist *abd_sgl; /* for LINEAR_PAGE */
 };
 
-typedef struct abd abd_t;
-
 typedef int abd_iter_page_func_t(struct page *, size_t, size_t, void *);
-int abd_iterate_page_func(abd_t *, size_t, size_t, abd_iter_page_func_t *,
+int abd_iterate_page_func(struct abd *, size_t, size_t, abd_iter_page_func_t *,
     void *);
 
 /*
  * Linux ABD bio functions
  * Note: these are only needed to support vdev_classic. See comment in
  * vdev_disk.c.
  */
-unsigned int abd_bio_map_off(struct bio *, abd_t *, unsigned int, size_t);
-unsigned long abd_nr_pages_off(abd_t *, unsigned int, size_t);
+unsigned int abd_bio_map_off(struct bio *, struct abd *, unsigned int, size_t);
+unsigned long abd_nr_pages_off(struct abd *, unsigned int, size_t);
 
 #ifdef __cplusplus
 }
 #endif
 
 #endif	/* _ABD_H */
diff --git a/sys/contrib/openzfs/include/os/linux/zfs/sys/policy.h b/sys/contrib/openzfs/include/os/linux/zfs/sys/policy.h
index 0c265db78591..03069b9947d5 100644
--- a/sys/contrib/openzfs/include/os/linux/zfs/sys/policy.h
+++ b/sys/contrib/openzfs/include/os/linux/zfs/sys/policy.h
@@ -1,64 +1,63 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2015, Joyent, Inc. All rights reserved.
  * Copyright (c) 2016, Lawrence Livermore National Security, LLC.
  */
 
 #ifndef _SYS_POLICY_H
 #define	_SYS_POLICY_H
 
 #ifdef _KERNEL
 
 #include <sys/cred.h>
 #include <sys/types.h>
 #include <sys/xvattr.h>
 #include <sys/zpl.h>
 
 struct znode;
 
 int secpolicy_nfs(const cred_t *);
 int secpolicy_sys_config(const cred_t *, boolean_t);
 int secpolicy_vnode_access2(const cred_t *, struct inode *,
     uid_t, mode_t, mode_t);
 int secpolicy_vnode_any_access(const cred_t *, struct inode *, uid_t);
 int secpolicy_vnode_chown(const cred_t *, uid_t);
 int secpolicy_vnode_create_gid(const cred_t *);
 int secpolicy_vnode_remove(const cred_t *);
 int secpolicy_vnode_setdac(const cred_t *, uid_t);
 int secpolicy_vnode_setid_retain(struct znode *, const cred_t *, boolean_t);
 int secpolicy_vnode_setids_setgids(const cred_t *, gid_t, zidmap_t *,
     struct user_namespace *);
 int secpolicy_zinject(const cred_t *);
 int secpolicy_zfs(const cred_t *);
-int secpolicy_zfs_proc(const cred_t *, proc_t *);
 void secpolicy_setid_clear(vattr_t *, cred_t *);
 int secpolicy_setid_setsticky_clear(struct inode *, vattr_t *,
     const vattr_t *, cred_t *, zidmap_t *, struct user_namespace *);
 int secpolicy_xvattr(xvattr_t *, uid_t, cred_t *, mode_t);
 int secpolicy_vnode_setattr(cred_t *, struct inode *, struct vattr *,
     const struct vattr *, int, int (void *, int, cred_t *), void *);
 int secpolicy_basic_link(const cred_t *);
 
 #endif /* _KERNEL */
 #endif /* _SYS_POLICY_H */
diff --git a/sys/contrib/openzfs/include/sys/dmu_recv.h b/sys/contrib/openzfs/include/sys/dmu_recv.h
index 3390ca1089f8..3ca73f7ea05c 100644
--- a/sys/contrib/openzfs/include/sys/dmu_recv.h
+++ b/sys/contrib/openzfs/include/sys/dmu_recv.h
@@ -1,89 +1,88 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2019 Datto Inc.
  */
 
 #ifndef _DMU_RECV_H
 #define	_DMU_RECV_H
 
 #include <sys/inttypes.h>
 #include <sys/dsl_bookmark.h>
 #include <sys/dsl_dataset.h>
 #include <sys/spa.h>
 #include <sys/objlist.h>
 
 extern const char *const recv_clone_name;
 
 typedef struct dmu_recv_cookie {
 	struct dsl_dataset *drc_ds;
 	struct dmu_replay_record *drc_drr_begin;
 	struct drr_begin *drc_drrb;
 	const char *drc_tofs;
 	const char *drc_tosnap;
 	boolean_t drc_newfs;
 	boolean_t drc_byteswap;
 	uint64_t drc_featureflags;
 	boolean_t drc_force;
 	boolean_t drc_heal;
 	boolean_t drc_resumable;
 	boolean_t drc_should_save;
 	boolean_t drc_raw;
 	boolean_t drc_clone;
 	boolean_t drc_spill;
 	nvlist_t *drc_keynvl;
 	uint64_t drc_fromsnapobj;
 	uint64_t drc_ivset_guid;
 	void *drc_owner;
 	cred_t *drc_cred;
-	proc_t *drc_proc;
 	nvlist_t *drc_begin_nvl;
 
 	objset_t *drc_os;
 	zfs_file_t *drc_fp; /* The file to read the stream from */
 	uint64_t drc_voff; /* The current offset in the stream */
 	uint64_t drc_bytes_read;
 	/*
 	 * A record that has had its payload read in, but hasn't yet been handed
 	 * off to the worker thread.
 	 */
 	struct receive_record_arg *drc_rrd;
 	/* A record that has had its header read in, but not its payload. */
 	struct receive_record_arg *drc_next_rrd;
 	zio_cksum_t drc_cksum;
 	zio_cksum_t drc_prev_cksum;
 	/* Sorted list of objects not to issue prefetches for. */
 	objlist_t *drc_ignore_objlist;
 } dmu_recv_cookie_t;
 
 int dmu_recv_begin(const char *, const char *, dmu_replay_record_t *,
     boolean_t, boolean_t, boolean_t, nvlist_t *, nvlist_t *, const char *,
     dmu_recv_cookie_t *, zfs_file_t *, offset_t *);
 int dmu_recv_stream(dmu_recv_cookie_t *, offset_t *);
 int dmu_recv_end(dmu_recv_cookie_t *, void *);
 boolean_t dmu_objset_is_receiving(objset_t *);
 
 #endif /* _DMU_RECV_H */
diff --git a/sys/contrib/openzfs/include/sys/dsl_dataset.h b/sys/contrib/openzfs/include/sys/dsl_dataset.h
index 3450527af7e0..047c062386a7 100644
--- a/sys/contrib/openzfs/include/sys/dsl_dataset.h
+++ b/sys/contrib/openzfs/include/sys/dsl_dataset.h
@@ -1,523 +1,521 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_DSL_DATASET_H
 #define	_SYS_DSL_DATASET_H
 
 #include <sys/dmu.h>
 #include <sys/spa.h>
 #include <sys/txg.h>
 #include <sys/zio.h>
 #include <sys/bplist.h>
 #include <sys/dsl_synctask.h>
 #include <sys/zfs_context.h>
 #include <sys/dsl_deadlist.h>
 #include <sys/zfs_refcount.h>
 #include <sys/rrwlock.h>
 #include <sys/dsl_crypt.h>
 #include <zfeature_common.h>
 
 #ifdef	__cplusplus
 extern "C" {
 #endif
 
 struct dsl_dataset;
 struct dsl_dir;
 struct dsl_pool;
 struct dsl_crypto_params;
 struct dsl_key_mapping;
 struct zfs_bookmark_phys;
 
 #define	DS_FLAG_INCONSISTENT	(1ULL<<0)
 #define	DS_IS_INCONSISTENT(ds)	\
 	(dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT)
 
 /*
  * Do not allow this dataset to be promoted.
  */
 #define	DS_FLAG_NOPROMOTE	(1ULL<<1)
 
 /*
  * DS_FLAG_UNIQUE_ACCURATE is set if ds_unique_bytes has been correctly
  * calculated for head datasets (starting with SPA_VERSION_UNIQUE_ACCURATE,
  * refquota/refreservations).
  */
 #define	DS_FLAG_UNIQUE_ACCURATE	(1ULL<<2)
 
 /*
  * DS_FLAG_DEFER_DESTROY is set after 'zfs destroy -d' has been called
  * on a dataset. This allows the dataset to be destroyed using 'zfs release'.
  */
 #define	DS_FLAG_DEFER_DESTROY	(1ULL<<3)
 #define	DS_IS_DEFER_DESTROY(ds)	\
 	(dsl_dataset_phys(ds)->ds_flags & DS_FLAG_DEFER_DESTROY)
 
 /*
  * DS_FIELD_* are strings that are used in the "extensified" dataset zap object.
  * They should be of the format <reverse-dns>:<field>.
  */
 
 /*
  * This field's value is the object ID of a zap object which contains the
  * bookmarks of this dataset.  If it is present, then this dataset is counted
  * in the refcount of the SPA_FEATURES_BOOKMARKS feature.
  */
 #define	DS_FIELD_BOOKMARK_NAMES "com.delphix:bookmarks"
 
 /*
  * This field is present (with value=0) if this dataset may contain large
  * dnodes (>512B).  If it is present, then this dataset is counted in the
  * refcount of the SPA_FEATURE_LARGE_DNODE feature.
  */
 #define	DS_FIELD_LARGE_DNODE "org.zfsonlinux:large_dnode"
 
 /*
  * These fields are set on datasets that are in the middle of a resumable
  * receive, and allow the sender to resume the send if it is interrupted.
  */
 #define	DS_FIELD_RESUME_FROMGUID "com.delphix:resume_fromguid"
 #define	DS_FIELD_RESUME_TONAME "com.delphix:resume_toname"
 #define	DS_FIELD_RESUME_TOGUID "com.delphix:resume_toguid"
 #define	DS_FIELD_RESUME_OBJECT "com.delphix:resume_object"
 #define	DS_FIELD_RESUME_OFFSET "com.delphix:resume_offset"
 #define	DS_FIELD_RESUME_BYTES "com.delphix:resume_bytes"
 #define	DS_FIELD_RESUME_LARGEBLOCK "com.delphix:resume_largeblockok"
 #define	DS_FIELD_RESUME_EMBEDOK "com.delphix:resume_embedok"
 #define	DS_FIELD_RESUME_COMPRESSOK "com.delphix:resume_compressok"
 #define	DS_FIELD_RESUME_RAWOK "com.datto:resume_rawok"
 
 /*
  * This field is set to the object number of the remap deadlist if one exists.
  */
 #define	DS_FIELD_REMAP_DEADLIST	"com.delphix:remap_deadlist"
 
 /*
  * We were receiving an incremental from a redaction bookmark, and these are the
  * guids of its snapshots.
  */
 #define	DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS \
 	"com.delphix:resume_redact_book_snaps"
 
 /*
  * This field is set to the ivset guid for encrypted snapshots. This is used
  * for validating raw receives.
  */
 #define	DS_FIELD_IVSET_GUID	"com.datto:ivset_guid"
 
 /*
  * DS_FLAG_CI_DATASET is set if the dataset contains a file system whose
  * name lookups should be performed case-insensitively.
  */
 #define	DS_FLAG_CI_DATASET	(1ULL<<16)
 
 #define	DS_CREATE_FLAG_NODIRTY	(1ULL<<24)
 
 typedef struct dsl_dataset_phys {
 	uint64_t ds_dir_obj;		/* DMU_OT_DSL_DIR */
 	uint64_t ds_prev_snap_obj;	/* DMU_OT_DSL_DATASET */
 	uint64_t ds_prev_snap_txg;
 	uint64_t ds_next_snap_obj;	/* DMU_OT_DSL_DATASET */
 	uint64_t ds_snapnames_zapobj;	/* DMU_OT_DSL_DS_SNAP_MAP 0 for snaps */
 	uint64_t ds_num_children;	/* clone/snap children; ==0 for head */
 	uint64_t ds_creation_time;	/* seconds since 1970 */
 	uint64_t ds_creation_txg;
 	uint64_t ds_deadlist_obj;	/* DMU_OT_DEADLIST */
 	/*
 	 * ds_referenced_bytes, ds_compressed_bytes, and ds_uncompressed_bytes
 	 * include all blocks referenced by this dataset, including those
 	 * shared with any other datasets.
 	 */
 	uint64_t ds_referenced_bytes;
 	uint64_t ds_compressed_bytes;
 	uint64_t ds_uncompressed_bytes;
 	uint64_t ds_unique_bytes;	/* only relevant to snapshots */
 	/*
 	 * The ds_fsid_guid is a 56-bit ID that can change to avoid
 	 * collisions.  The ds_guid is a 64-bit ID that will never
 	 * change, so there is a small probability that it will collide.
 	 */
 	uint64_t ds_fsid_guid;
 	uint64_t ds_guid;
 	uint64_t ds_flags;		/* DS_FLAG_* */
 	blkptr_t ds_bp;
 	uint64_t ds_next_clones_obj;	/* DMU_OT_DSL_CLONES */
 	uint64_t ds_props_obj;		/* DMU_OT_DSL_PROPS for snaps */
 	uint64_t ds_userrefs_obj;	/* DMU_OT_USERREFS */
 	uint64_t ds_pad[5]; /* pad out to 320 bytes for good measure */
 } dsl_dataset_phys_t;
 
 typedef struct dsl_dataset {
 	dmu_buf_user_t ds_dbu;
 	rrwlock_t ds_bp_rwlock; /* Protects ds_phys->ds_bp */
 
 	/* Immutable: */
 	struct dsl_dir *ds_dir;
 	dmu_buf_t *ds_dbuf;
 	uint64_t ds_object;
 	uint64_t ds_fsid_guid;
 	boolean_t ds_is_snapshot;
 	struct dsl_key_mapping *ds_key_mapping;
 
 	/* only used in syncing context, only valid for non-snapshots: */
 	struct dsl_dataset *ds_prev;
 	uint64_t ds_bookmarks_obj;  /* DMU_OTN_ZAP_METADATA */
 	avl_tree_t ds_bookmarks; /* dsl_bookmark_node_t */
 
 	/* has internal locking: */
 	dsl_deadlist_t ds_deadlist;
 	bplist_t ds_pending_deadlist;
 
 	/*
 	 * The remap deadlist contains blocks (DVA's, really) that are
 	 * referenced by the previous snapshot and point to indirect vdevs,
 	 * but in this dataset they have been remapped to point to concrete
 	 * (or at least, less-indirect) vdevs.  In other words, the
 	 * physical DVA is referenced by the previous snapshot but not by
 	 * this dataset.  Logically, the DVA continues to be referenced,
 	 * but we are using a different (less indirect) physical DVA.
 	 * This deadlist is used to determine when physical DVAs that
 	 * point to indirect vdevs are no longer referenced anywhere,
 	 * and thus should be marked obsolete.
 	 *
 	 * This is only used if SPA_FEATURE_OBSOLETE_COUNTS is enabled.
 	 */
 	dsl_deadlist_t ds_remap_deadlist;
 	/* protects creation of the ds_remap_deadlist */
 	kmutex_t ds_remap_deadlist_lock;
 
 	/* protected by lock on pool's dp_dirty_datasets list */
 	txg_node_t ds_dirty_link;
 	list_node_t ds_synced_link;
 
 	/*
 	 * ds_phys->ds_<accounting> is also protected by ds_lock.
 	 * Protected by ds_lock:
 	 */
 	kmutex_t ds_lock;
 	objset_t *ds_objset;
 	uint64_t ds_userrefs;
 	const void *ds_owner;
 
 	/*
 	 * Long holds prevent the ds from being destroyed; they allow the
 	 * ds to remain held even after dropping the dp_config_rwlock.
 	 * Owning counts as a long hold.  See the comments above
 	 * dsl_pool_hold() for details.
 	 */
 	zfs_refcount_t ds_longholds;
 
 	/* no locking; only for making guesses */
 	uint64_t ds_trysnap_txg;
 
 	/* for objset_open() */
 	kmutex_t ds_opening_lock;
 
 	uint64_t ds_reserved;	/* cached refreservation */
 	uint64_t ds_quota;	/* cached refquota */
 
 	kmutex_t ds_sendstream_lock;
 	list_t ds_sendstreams;
 
 	/*
 	 * When in the middle of a resumable receive, tracks how much
 	 * progress we have made.
 	 */
 	uint64_t ds_resume_object[TXG_SIZE];
 	uint64_t ds_resume_offset[TXG_SIZE];
 	uint64_t ds_resume_bytes[TXG_SIZE];
 
 	/* Protected by our dsl_dir's dd_lock */
 	list_t ds_prop_cbs;
 
 	/*
 	 * For ZFEATURE_FLAG_PER_DATASET features, set if this dataset
 	 * uses this feature.
 	 */
 	void *ds_feature[SPA_FEATURES];
 
 	/*
 	 * Set if we need to activate the feature on this dataset this txg
 	 * (used only in syncing context).
 	 */
 	void *ds_feature_activation[SPA_FEATURES];
 
 	/* Protected by ds_lock; keep at end of struct for better locality */
 	char ds_snapname[ZFS_MAX_DATASET_NAME_LEN];
 } dsl_dataset_t;
 
 static inline dsl_dataset_phys_t *
 dsl_dataset_phys(dsl_dataset_t *ds)
 {
 	return ((dsl_dataset_phys_t *)ds->ds_dbuf->db_data);
 }
 
 typedef struct dsl_dataset_promote_arg {
 	const char *ddpa_clonename;
 	dsl_dataset_t *ddpa_clone;
 	list_t shared_snaps, origin_snaps, clone_snaps;
 	dsl_dataset_t *origin_origin; /* origin of the origin */
 	uint64_t used, comp, uncomp, unique, cloneusedsnap, originusedsnap;
 	nvlist_t *err_ds;
 	cred_t *cr;
-	proc_t *proc;
 } dsl_dataset_promote_arg_t;
 
 typedef struct dsl_dataset_rollback_arg {
 	const char *ddra_fsname;
 	const char *ddra_tosnap;
 	void *ddra_owner;
 	nvlist_t *ddra_result;
 } dsl_dataset_rollback_arg_t;
 
 typedef struct dsl_dataset_snapshot_arg {
 	nvlist_t *ddsa_snaps;
 	nvlist_t *ddsa_props;
 	nvlist_t *ddsa_errors;
 	cred_t *ddsa_cr;
-	proc_t *ddsa_proc;
 } dsl_dataset_snapshot_arg_t;
 
 typedef struct dsl_dataset_rename_snapshot_arg {
 	const char *ddrsa_fsname;
 	const char *ddrsa_oldsnapname;
 	const char *ddrsa_newsnapname;
 	boolean_t ddrsa_recursive;
 	dmu_tx_t *ddrsa_tx;
 } dsl_dataset_rename_snapshot_arg_t;
 
 /*
  * The max length of a temporary tag prefix is the number of hex digits
  * required to express UINT64_MAX plus one for the hyphen.
  */
 #define	MAX_TAG_PREFIX_LEN	17
 
 #define	dsl_dataset_is_snapshot(ds) \
 	(dsl_dataset_phys(ds)->ds_num_children != 0)
 
 #define	DS_UNIQUE_IS_ACCURATE(ds)	\
 	((dsl_dataset_phys(ds)->ds_flags & DS_FLAG_UNIQUE_ACCURATE) != 0)
 
 /* flags for holding the dataset */
 typedef enum ds_hold_flags {
 	DS_HOLD_FLAG_NONE	= 0 << 0,
 	DS_HOLD_FLAG_DECRYPT	= 1 << 0 /* needs access to encrypted data */
 } ds_hold_flags_t;
 
 int dsl_dataset_hold(struct dsl_pool *dp, const char *name, const void *tag,
     dsl_dataset_t **dsp);
 int dsl_dataset_hold_flags(struct dsl_pool *dp, const char *name,
     ds_hold_flags_t flags, const void *tag, dsl_dataset_t **dsp);
 boolean_t dsl_dataset_try_add_ref(struct dsl_pool *dp, dsl_dataset_t *ds,
     const void *tag);
 int dsl_dataset_create_key_mapping(dsl_dataset_t *ds);
 int dsl_dataset_hold_obj_flags(struct dsl_pool *dp, uint64_t dsobj,
     ds_hold_flags_t flags, const void *tag, dsl_dataset_t **);
 void dsl_dataset_remove_key_mapping(dsl_dataset_t *ds);
 int dsl_dataset_hold_obj(struct dsl_pool *dp, uint64_t dsobj,
     const void *tag, dsl_dataset_t **);
 void dsl_dataset_rele_flags(dsl_dataset_t *ds, ds_hold_flags_t flags,
     const void *tag);
 void dsl_dataset_rele(dsl_dataset_t *ds, const void *tag);
 int dsl_dataset_own(struct dsl_pool *dp, const char *name,
     ds_hold_flags_t flags, const void *tag, dsl_dataset_t **dsp);
 int dsl_dataset_own_force(struct dsl_pool *dp, const char *name,
     ds_hold_flags_t flags, const void *tag, dsl_dataset_t **dsp);
 int dsl_dataset_own_obj(struct dsl_pool *dp, uint64_t dsobj,
     ds_hold_flags_t flags, const void *tag, dsl_dataset_t **dsp);
 int dsl_dataset_own_obj_force(struct dsl_pool *dp, uint64_t dsobj,
     ds_hold_flags_t flags, const void *tag, dsl_dataset_t **dsp);
 void dsl_dataset_disown(dsl_dataset_t *ds, ds_hold_flags_t flags,
     const void *tag);
 void dsl_dataset_name(dsl_dataset_t *ds, char *name);
 boolean_t dsl_dataset_tryown(dsl_dataset_t *ds, const void *tag,
     boolean_t override);
 int dsl_dataset_namelen(dsl_dataset_t *ds);
 boolean_t dsl_dataset_has_owner(dsl_dataset_t *ds);
 uint64_t dsl_dataset_create_sync(dsl_dir_t *pds, const char *lastname,
     dsl_dataset_t *origin, uint64_t flags, cred_t *,
     struct dsl_crypto_params *, dmu_tx_t *);
 uint64_t dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
     struct dsl_crypto_params *dcp, uint64_t flags, dmu_tx_t *tx);
 void dsl_dataset_snapshot_sync(void *arg, dmu_tx_t *tx);
 int dsl_dataset_snapshot_check(void *arg, dmu_tx_t *tx);
 int dsl_dataset_snapshot(nvlist_t *snaps, nvlist_t *props, nvlist_t *errors);
 void dsl_dataset_promote_sync(void *arg, dmu_tx_t *tx);
 int dsl_dataset_promote_check(void *arg, dmu_tx_t *tx);
 int dsl_dataset_promote(const char *name, char *conflsnap);
 int dsl_dataset_rename_snapshot(const char *fsname,
     const char *oldsnapname, const char *newsnapname, boolean_t recursive);
 int dsl_dataset_snapshot_tmp(const char *fsname, const char *snapname,
     minor_t cleanup_minor, const char *htag);
 
 blkptr_t *dsl_dataset_get_blkptr(dsl_dataset_t *ds);
 
 spa_t *dsl_dataset_get_spa(dsl_dataset_t *ds);
 
 boolean_t dsl_dataset_modified_since_snap(dsl_dataset_t *ds,
     dsl_dataset_t *snap);
 
 void dsl_dataset_sync(dsl_dataset_t *ds, zio_t *zio, dmu_tx_t *tx);
 void dsl_dataset_sync_done(dsl_dataset_t *ds, dmu_tx_t *tx);
 
 void dsl_dataset_block_born(dsl_dataset_t *ds, const blkptr_t *bp,
     dmu_tx_t *tx);
 int dsl_dataset_block_kill(dsl_dataset_t *ds, const blkptr_t *bp,
     dmu_tx_t *tx, boolean_t async);
 void dsl_dataset_block_remapped(dsl_dataset_t *ds, uint64_t vdev,
     uint64_t offset, uint64_t size, uint64_t birth, dmu_tx_t *tx);
 int dsl_dataset_snap_lookup(dsl_dataset_t *ds, const char *name,
     uint64_t *value);
 
 void dsl_dataset_dirty(dsl_dataset_t *ds, dmu_tx_t *tx);
 
 int get_clones_stat_impl(dsl_dataset_t *ds, nvlist_t *val);
 char *get_receive_resume_token(dsl_dataset_t *ds);
 uint64_t dsl_get_refratio(dsl_dataset_t *ds);
 uint64_t dsl_get_logicalreferenced(dsl_dataset_t *ds);
 uint64_t dsl_get_compressratio(dsl_dataset_t *ds);
 uint64_t dsl_get_used(dsl_dataset_t *ds);
 uint64_t dsl_get_creation(dsl_dataset_t *ds);
 uint64_t dsl_get_creationtxg(dsl_dataset_t *ds);
 uint64_t dsl_get_refquota(dsl_dataset_t *ds);
 uint64_t dsl_get_refreservation(dsl_dataset_t *ds);
 uint64_t dsl_get_guid(dsl_dataset_t *ds);
 uint64_t dsl_get_unique(dsl_dataset_t *ds);
 uint64_t dsl_get_objsetid(dsl_dataset_t *ds);
 uint64_t dsl_get_userrefs(dsl_dataset_t *ds);
 uint64_t dsl_get_defer_destroy(dsl_dataset_t *ds);
 uint64_t dsl_get_referenced(dsl_dataset_t *ds);
 uint64_t dsl_get_numclones(dsl_dataset_t *ds);
 uint64_t dsl_get_inconsistent(dsl_dataset_t *ds);
 uint64_t dsl_get_redacted(dsl_dataset_t *ds);
 uint64_t dsl_get_available(dsl_dataset_t *ds);
 int dsl_get_written(dsl_dataset_t *ds, uint64_t *written);
 int dsl_get_prev_snap(dsl_dataset_t *ds, char *snap);
 void dsl_get_redact_snaps(dsl_dataset_t *ds, nvlist_t *propval);
 int dsl_get_mountpoint(dsl_dataset_t *ds, const char *dsname, char *value,
     char *source);
 
 void get_clones_stat(dsl_dataset_t *ds, nvlist_t *nv);
 void dsl_dataset_stats(dsl_dataset_t *os, nvlist_t *nv);
 
 void dsl_dataset_fast_stat(dsl_dataset_t *ds, dmu_objset_stats_t *stat);
 void dsl_dataset_space(dsl_dataset_t *ds,
     uint64_t *refdbytesp, uint64_t *availbytesp,
     uint64_t *usedobjsp, uint64_t *availobjsp);
 uint64_t dsl_dataset_fsid_guid(dsl_dataset_t *ds);
 int dsl_dataset_space_written(dsl_dataset_t *oldsnap, dsl_dataset_t *newds,
     uint64_t *usedp, uint64_t *compp, uint64_t *uncompp);
 int dsl_dataset_space_written_bookmark(struct zfs_bookmark_phys *bmp,
     dsl_dataset_t *newds, uint64_t *usedp, uint64_t *compp, uint64_t *uncompp);
 int dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap, dsl_dataset_t *last,
     uint64_t *usedp, uint64_t *compp, uint64_t *uncompp);
 
 int dsl_dsobj_to_dsname(char *pname, uint64_t obj, char *buf);
 
 int dsl_dataset_check_quota(dsl_dataset_t *ds, boolean_t check_quota,
     uint64_t asize, uint64_t inflight, uint64_t *used,
     uint64_t *ref_rsrv);
 int dsl_dataset_set_refquota(const char *dsname, zprop_source_t source,
     uint64_t quota);
 int dsl_dataset_set_refreservation(const char *dsname, zprop_source_t source,
     uint64_t reservation);
 int dsl_dataset_set_compression(const char *dsname, zprop_source_t source,
     uint64_t compression);
 
 boolean_t dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
     uint64_t earlier_txg);
 void dsl_dataset_long_hold(dsl_dataset_t *ds, const void *tag);
 void dsl_dataset_long_rele(dsl_dataset_t *ds, const void *tag);
 boolean_t dsl_dataset_long_held(dsl_dataset_t *ds);
 
 int dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone,
     dsl_dataset_t *origin_head, boolean_t force, void *owner, dmu_tx_t *tx);
 void dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
     dsl_dataset_t *origin_head, dmu_tx_t *tx);
 int dsl_dataset_snapshot_check_impl(dsl_dataset_t *ds, const char *snapname,
-    dmu_tx_t *tx, boolean_t recv, uint64_t cnt, cred_t *cr, proc_t *proc);
+    dmu_tx_t *tx, boolean_t recv, uint64_t cnt, cred_t *cr);
 void dsl_dataset_snapshot_sync_impl(dsl_dataset_t *ds, const char *snapname,
     dmu_tx_t *tx);
 
 void dsl_dataset_remove_from_next_clones(dsl_dataset_t *ds, uint64_t obj,
     dmu_tx_t *tx);
 void dsl_dataset_recalc_head_uniq(dsl_dataset_t *ds);
 int dsl_dataset_get_snapname(dsl_dataset_t *ds);
 int dsl_dataset_snap_lookup(dsl_dataset_t *ds, const char *name,
     uint64_t *value);
 int dsl_dataset_snap_remove(dsl_dataset_t *ds, const char *name, dmu_tx_t *tx,
     boolean_t adj_cnt);
 void dsl_dataset_set_refreservation_sync_impl(dsl_dataset_t *ds,
     zprop_source_t source, uint64_t value, dmu_tx_t *tx);
 void dsl_dataset_zapify(dsl_dataset_t *ds, dmu_tx_t *tx);
 boolean_t dsl_dataset_is_zapified(dsl_dataset_t *ds);
 boolean_t dsl_dataset_has_resume_receive_state(dsl_dataset_t *ds);
 
 int dsl_dataset_rollback_check(void *arg, dmu_tx_t *tx);
 void dsl_dataset_rollback_sync(void *arg, dmu_tx_t *tx);
 int dsl_dataset_rollback(const char *fsname, const char *tosnap, void *owner,
     nvlist_t *result);
 
 int dsl_dataset_rename_snapshot_check(void *arg, dmu_tx_t *tx);
 void dsl_dataset_rename_snapshot_sync(void *arg, dmu_tx_t *tx);
 
 uint64_t dsl_dataset_get_remap_deadlist_object(dsl_dataset_t *ds);
 void dsl_dataset_create_remap_deadlist(dsl_dataset_t *ds, dmu_tx_t *tx);
 boolean_t dsl_dataset_remap_deadlist_exists(dsl_dataset_t *ds);
 void dsl_dataset_destroy_remap_deadlist(dsl_dataset_t *ds, dmu_tx_t *tx);
 
 void dsl_dataset_activate_feature(uint64_t dsobj, spa_feature_t f, void *arg,
     dmu_tx_t *tx);
 void dsl_dataset_deactivate_feature(dsl_dataset_t *ds, spa_feature_t f,
     dmu_tx_t *tx);
 boolean_t dsl_dataset_feature_is_active(dsl_dataset_t *ds, spa_feature_t f);
 boolean_t dsl_dataset_get_uint64_array_feature(dsl_dataset_t *ds,
     spa_feature_t f, uint64_t *outlength, uint64_t **outp);
 
 void dsl_dataset_activate_redaction(dsl_dataset_t *ds, uint64_t *redact_snaps,
     uint64_t num_redact_snaps, dmu_tx_t *tx);
 
 int dsl_dataset_oldest_snapshot(spa_t *spa, uint64_t head_ds, uint64_t min_txg,
     uint64_t *oldest_dsobj);
 
 #ifdef ZFS_DEBUG
 #define	dprintf_ds(ds, fmt, ...) do { \
 	if (zfs_flags & ZFS_DEBUG_DPRINTF) { \
 	char *__ds_name = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); \
 	dsl_dataset_name(ds, __ds_name); \
 	dprintf("ds=%s " fmt, __ds_name, __VA_ARGS__); \
 	kmem_free(__ds_name, ZFS_MAX_DATASET_NAME_LEN); \
 	} \
 } while (0)
 #else
 #define	dprintf_ds(dd, fmt, ...)
 #endif
 
 #ifdef	__cplusplus
 }
 #endif
 
 #endif /* _SYS_DSL_DATASET_H */
diff --git a/sys/contrib/openzfs/include/sys/dsl_dir.h b/sys/contrib/openzfs/include/sys/dsl_dir.h
index f7c0d9acd10d..a338615e0e88 100644
--- a/sys/contrib/openzfs/include/sys/dsl_dir.h
+++ b/sys/contrib/openzfs/include/sys/dsl_dir.h
@@ -1,231 +1,231 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_DSL_DIR_H
 #define	_SYS_DSL_DIR_H
 
 #include <sys/dmu.h>
 #include <sys/dsl_deadlist.h>
 #include <sys/dsl_pool.h>
 #include <sys/dsl_synctask.h>
 #include <sys/zfs_refcount.h>
 #include <sys/zfs_context.h>
 #include <sys/dsl_crypt.h>
 #include <sys/bplist.h>
 
 #ifdef	__cplusplus
 extern "C" {
 #endif
 
 struct dsl_dataset;
 struct zthr;
 /*
  * DD_FIELD_* are strings that are used in the "extensified" dsl_dir zap object.
  * They should be of the format <reverse-dns>:<field>.
  */
 
 #define	DD_FIELD_FILESYSTEM_COUNT	"com.joyent:filesystem_count"
 #define	DD_FIELD_SNAPSHOT_COUNT		"com.joyent:snapshot_count"
 #define	DD_FIELD_CRYPTO_KEY_OBJ		"com.datto:crypto_key_obj"
 #define	DD_FIELD_LIVELIST		"com.delphix:livelist"
 #define	DD_FIELD_SNAPSHOTS_CHANGED	"com.ixsystems:snapshots_changed"
 
 typedef enum dd_used {
 	DD_USED_HEAD,
 	DD_USED_SNAP,
 	DD_USED_CHILD,
 	DD_USED_CHILD_RSRV,
 	DD_USED_REFRSRV,
 	DD_USED_NUM
 } dd_used_t;
 
 #define	DD_FLAG_USED_BREAKDOWN (1<<0)
 
 typedef struct dsl_dir_phys {
 	uint64_t dd_creation_time; /* not actually used */
 	uint64_t dd_head_dataset_obj;
 	uint64_t dd_parent_obj;
 	uint64_t dd_origin_obj;
 	uint64_t dd_child_dir_zapobj;
 	/*
 	 * how much space our children are accounting for; for leaf
 	 * datasets, == physical space used by fs + snaps
 	 */
 	uint64_t dd_used_bytes;
 	uint64_t dd_compressed_bytes;
 	uint64_t dd_uncompressed_bytes;
 	/* Administrative quota setting */
 	uint64_t dd_quota;
 	/* Administrative reservation setting */
 	uint64_t dd_reserved;
 	uint64_t dd_props_zapobj;
 	uint64_t dd_deleg_zapobj; /* dataset delegation permissions */
 	uint64_t dd_flags;
 	uint64_t dd_used_breakdown[DD_USED_NUM];
 	uint64_t dd_clones; /* dsl_dir objects */
 	uint64_t dd_pad[13]; /* pad out to 256 bytes for good measure */
 } dsl_dir_phys_t;
 
 struct dsl_dir {
 	dmu_buf_user_t dd_dbu;
 
 	/* These are immutable; no lock needed: */
 	uint64_t dd_object;
 	uint64_t dd_crypto_obj;
 	dsl_pool_t *dd_pool;
 
 	/* Stable until user eviction; no lock needed: */
 	dmu_buf_t *dd_dbuf;
 
 	/* protected by lock on pool's dp_dirty_dirs list */
 	txg_node_t dd_dirty_link;
 
 	/* protected by dp_config_rwlock */
 	dsl_dir_t *dd_parent;
 
 	/* Protected by dd_lock */
 	kmutex_t dd_lock;
 	list_t dd_props; /* list of dsl_prop_record_t's */
 	inode_timespec_t dd_snap_cmtime; /* last snapshot namespace change */
 	uint64_t dd_origin_txg;
 
 	/* gross estimate of space used by in-flight tx's */
 	uint64_t dd_tempreserved[TXG_SIZE];
 	/* amount of space we expect to write; == amount of dirty data */
 	uint64_t dd_space_towrite[TXG_SIZE];
 
 	dsl_deadlist_t dd_livelist;
 	bplist_t dd_pending_frees;
 	bplist_t dd_pending_allocs;
 
 	kmutex_t dd_activity_lock;
 	kcondvar_t dd_activity_cv;
 	boolean_t dd_activity_cancelled;
 	uint64_t dd_activity_waiters;
 
 	/* protected by dd_lock; keep at end of struct for better locality */
 	char dd_myname[ZFS_MAX_DATASET_NAME_LEN];
 };
 
 static inline dsl_dir_phys_t *
 dsl_dir_phys(dsl_dir_t *dd)
 {
 	return (dd->dd_dbuf->db_data);
 }
 
 void dsl_dir_rele(dsl_dir_t *dd, const void *tag);
 void dsl_dir_async_rele(dsl_dir_t *dd, const void *tag);
 int dsl_dir_hold(dsl_pool_t *dp, const char *name, const void *tag,
     dsl_dir_t **, const char **tail);
 int dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
     const char *tail, const void *tag, dsl_dir_t **);
 void dsl_dir_name(dsl_dir_t *dd, char *buf);
 int dsl_dir_namelen(dsl_dir_t *dd);
 uint64_t dsl_dir_create_sync(dsl_pool_t *dp, dsl_dir_t *pds,
     const char *name, dmu_tx_t *tx);
 
 uint64_t dsl_dir_get_used(dsl_dir_t *dd);
 uint64_t dsl_dir_get_compressed(dsl_dir_t *dd);
 uint64_t dsl_dir_get_quota(dsl_dir_t *dd);
 uint64_t dsl_dir_get_reservation(dsl_dir_t *dd);
 uint64_t dsl_dir_get_compressratio(dsl_dir_t *dd);
 uint64_t dsl_dir_get_logicalused(dsl_dir_t *dd);
 uint64_t dsl_dir_get_usedsnap(dsl_dir_t *dd);
 uint64_t dsl_dir_get_usedds(dsl_dir_t *dd);
 uint64_t dsl_dir_get_usedrefreserv(dsl_dir_t *dd);
 uint64_t dsl_dir_get_usedchild(dsl_dir_t *dd);
 void dsl_dir_get_origin(dsl_dir_t *dd, char *buf);
 int dsl_dir_get_filesystem_count(dsl_dir_t *dd, uint64_t *count);
 int dsl_dir_get_snapshot_count(dsl_dir_t *dd, uint64_t *count);
 
 void dsl_dir_stats(dsl_dir_t *dd, nvlist_t *nv);
 uint64_t dsl_dir_space_available(dsl_dir_t *dd,
     dsl_dir_t *ancestor, int64_t delta, int ondiskonly);
 void dsl_dir_dirty(dsl_dir_t *dd, dmu_tx_t *tx);
 void dsl_dir_sync(dsl_dir_t *dd, dmu_tx_t *tx);
 int dsl_dir_tempreserve_space(dsl_dir_t *dd, uint64_t mem,
     uint64_t asize, boolean_t netfree, void **tr_cookiep, dmu_tx_t *tx);
 void dsl_dir_tempreserve_clear(void *tr_cookie, dmu_tx_t *tx);
 void dsl_dir_willuse_space(dsl_dir_t *dd, int64_t space, dmu_tx_t *tx);
 void dsl_dir_diduse_space(dsl_dir_t *dd, dd_used_t type,
     int64_t used, int64_t compressed, int64_t uncompressed, dmu_tx_t *tx);
 void dsl_dir_transfer_space(dsl_dir_t *dd, int64_t delta,
     dd_used_t oldtype, dd_used_t newtype, dmu_tx_t *tx);
 void dsl_dir_diduse_transfer_space(dsl_dir_t *dd, int64_t used,
     int64_t compressed, int64_t uncompressed, int64_t tonew,
     dd_used_t oldtype, dd_used_t newtype, dmu_tx_t *tx);
 int dsl_dir_set_quota(const char *ddname, zprop_source_t source,
     uint64_t quota);
 int dsl_dir_set_reservation(const char *ddname, zprop_source_t source,
     uint64_t reservation);
 int dsl_dir_activate_fs_ss_limit(const char *);
 int dsl_fs_ss_limit_check(dsl_dir_t *, uint64_t, zfs_prop_t, dsl_dir_t *,
-    cred_t *, proc_t *);
+    cred_t *);
 void dsl_fs_ss_count_adjust(dsl_dir_t *, int64_t, const char *, dmu_tx_t *);
 int dsl_dir_rename(const char *oldname, const char *newname);
 int dsl_dir_transfer_possible(dsl_dir_t *sdd, dsl_dir_t *tdd,
-    uint64_t fs_cnt, uint64_t ss_cnt, uint64_t space, cred_t *, proc_t *);
+    uint64_t fs_cnt, uint64_t ss_cnt, uint64_t space, cred_t *);
 boolean_t dsl_dir_is_clone(dsl_dir_t *dd);
 void dsl_dir_new_refreservation(dsl_dir_t *dd, struct dsl_dataset *ds,
     uint64_t reservation, cred_t *cr, dmu_tx_t *tx);
 void dsl_dir_snap_cmtime_update(dsl_dir_t *dd, dmu_tx_t *tx);
 inode_timespec_t dsl_dir_snap_cmtime(dsl_dir_t *dd);
 void dsl_dir_set_reservation_sync_impl(dsl_dir_t *dd, uint64_t value,
     dmu_tx_t *tx);
 void dsl_dir_zapify(dsl_dir_t *dd, dmu_tx_t *tx);
 boolean_t dsl_dir_is_zapified(dsl_dir_t *dd);
 void dsl_dir_livelist_open(dsl_dir_t *dd, uint64_t obj);
 void dsl_dir_livelist_close(dsl_dir_t *dd);
 void dsl_dir_remove_livelist(dsl_dir_t *dd, dmu_tx_t *tx, boolean_t total);
 int dsl_dir_wait(dsl_dir_t *dd, dsl_dataset_t *ds, zfs_wait_activity_t activity,
     boolean_t *waited);
 void dsl_dir_cancel_waiters(dsl_dir_t *dd);
 
 /* internal reserved dir name */
 #define	MOS_DIR_NAME "$MOS"
 #define	ORIGIN_DIR_NAME "$ORIGIN"
 #define	FREE_DIR_NAME "$FREE"
 #define	LEAK_DIR_NAME "$LEAK"
 
 #ifdef ZFS_DEBUG
 #define	dprintf_dd(dd, fmt, ...) do { \
 	if (zfs_flags & ZFS_DEBUG_DPRINTF) { \
 	char *__ds_name = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP); \
 	dsl_dir_name(dd, __ds_name); \
 	dprintf("dd=%s " fmt, __ds_name, __VA_ARGS__); \
 	kmem_free(__ds_name, ZFS_MAX_DATASET_NAME_LEN); \
 	} \
 } while (0)
 #else
 #define	dprintf_dd(dd, fmt, ...)
 #endif
 
 #ifdef	__cplusplus
 }
 #endif
 
 #endif /* _SYS_DSL_DIR_H */
diff --git a/sys/contrib/openzfs/include/sys/frame.h b/sys/contrib/openzfs/include/sys/frame.h
index caae851421d8..0250f47b61e0 100644
--- a/sys/contrib/openzfs/include/sys/frame.h
+++ b/sys/contrib/openzfs/include/sys/frame.h
@@ -1,41 +1,49 @@
 /*
  * CDDL HEADER START
  *
  * This file and its contents are supplied under the terms of the
  * Common Development and Distribution License ("CDDL"), version 1.0.
  * You may only use this file in accordance with the terms of version
  * 1.0 of the CDDL.
  *
  * A full copy of the text of the CDDL should have accompanied this
  * source.  A copy of the CDDL is also available via the Internet at
  * http://www.illumos.org/license/CDDL.
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (C) 2017 by Lawrence Livermore National Security, LLC.
  */
 
 #ifndef _SYS_FRAME_H
 #define	_SYS_FRAME_H
 
 #ifdef	__cplusplus
 extern "C" {
 #endif
 
 #if defined(__KERNEL__) && defined(HAVE_KERNEL_OBJTOOL) && \
     defined(HAVE_STACK_FRAME_NON_STANDARD)
 #if defined(HAVE_KERNEL_OBJTOOL_HEADER)
 #include <linux/objtool.h>
 #else
 #include <linux/frame.h>
 #endif
+#if defined(_ASM) && ! defined(HAVE_STACK_FRAME_NON_STANDARD_ASM)
+.macro STACK_FRAME_NON_STANDARD func:req
+.endm
+#endif
 #else
 #define	STACK_FRAME_NON_STANDARD(func)
+#if defined(_ASM)
+.macro STACK_FRAME_NON_STANDARD func:req
+.endm
+#endif
 #endif
 
 #ifdef	__cplusplus
 }
 #endif
 
 #endif	/* _SYS_FRAME_H */
diff --git a/sys/contrib/openzfs/include/sys/zcp.h b/sys/contrib/openzfs/include/sys/zcp.h
index 6301cc08e7ea..dafbc741690a 100644
--- a/sys/contrib/openzfs/include/sys/zcp.h
+++ b/sys/contrib/openzfs/include/sys/zcp.h
@@ -1,194 +1,193 @@
 /*
  * CDDL HEADER START
  *
  * This file and its contents are supplied under the terms of the
  * Common Development and Distribution License ("CDDL"), version 1.0.
  * You may only use this file in accordance with the terms of version
  * 1.0 of the CDDL.
  *
  * A full copy of the text of the CDDL should have accompanied this
  * source.  A copy of the CDDL is also available via the Internet at
  * http://www.illumos.org/license/CDDL.
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2016, 2018 by Delphix. All rights reserved.
  */
 
 #ifndef _SYS_ZCP_H
 #define	_SYS_ZCP_H
 
 #include <sys/dmu_tx.h>
 #include <sys/dsl_pool.h>
 
 #include <sys/lua/lua.h>
 #include <sys/lua/lualib.h>
 #include <sys/lua/lauxlib.h>
 
 #ifdef	__cplusplus
 extern "C" {
 #endif
 
 #define	ZCP_RUN_INFO_KEY "runinfo"
 
 extern uint64_t zfs_lua_max_instrlimit;
 extern uint64_t zfs_lua_max_memlimit;
 
 int zcp_argerror(lua_State *, int, const char *, ...);
 
 int zcp_eval(const char *, const char *, boolean_t, uint64_t, uint64_t,
     nvpair_t *, nvlist_t *);
 
 int zcp_load_list_lib(lua_State *);
 
 int zcp_load_synctask_lib(lua_State *, boolean_t);
 
 typedef void (zcp_cleanup_t)(void *);
 typedef struct zcp_cleanup_handler {
 	zcp_cleanup_t *zch_cleanup_func;
 	void *zch_cleanup_arg;
 	list_node_t zch_node;
 } zcp_cleanup_handler_t;
 
 typedef struct zcp_alloc_arg {
 	boolean_t	aa_must_succeed;
 	int64_t		aa_alloc_remaining;
 	int64_t		aa_alloc_limit;
 } zcp_alloc_arg_t;
 
 typedef struct zcp_run_info {
 	dsl_pool_t	*zri_pool;
 
 	/*
 	 * An estimate of the total amount of space consumed by all
 	 * synctasks we have successfully performed so far in this
 	 * channel program. Used to generate ENOSPC errors for syncfuncs.
 	 */
 	int		zri_space_used;
 
 	/*
 	 * The credentials of the thread which originally invoked the channel
 	 * program. Since channel programs are always invoked from the synctask
 	 * thread they should always do permissions checks against this cred
 	 * rather than the 'current' thread's.
 	 */
 	cred_t		*zri_cred;
-	proc_t		*zri_proc;
 
 	/*
 	 * The tx in which this channel program is running.
 	 */
 	dmu_tx_t	*zri_tx;
 
 	/*
 	 * The maximum number of Lua instructions the channel program is allowed
 	 * to execute. If it takes longer than this it will time out. A value
 	 * of 0 indicates no instruction limit.
 	 */
 	uint64_t	zri_maxinstrs;
 
 	/*
 	 * The number of Lua instructions the channel program has executed.
 	 */
 	uint64_t	zri_curinstrs;
 
 	/*
 	 * Boolean indicating whether or not the channel program exited
 	 * because it timed out.
 	 */
 	boolean_t	zri_timed_out;
 
 	/*
 	 * Channel program was canceled by user
 	 */
 	boolean_t	zri_canceled;
 
 	/*
 	 * Boolean indicating whether or not we are running in syncing
 	 * context.
 	 */
 	boolean_t	zri_sync;
 
 	/*
 	 * List of currently registered cleanup handlers, which will be
 	 * triggered in the event of a fatal error.
 	 */
 	list_t		zri_cleanup_handlers;
 
 	/*
 	 * The Lua state context of our channel program.
 	 */
 	lua_State	*zri_state;
 
 	/*
 	 * Lua memory allocator arguments.
 	 */
 	zcp_alloc_arg_t	*zri_allocargs;
 
 	/*
 	 * Contains output values from zcp script or error string.
 	 */
 	nvlist_t	*zri_outnvl;
 
 	/*
 	 * The keys of this nvlist are datasets which may be zvols and may need
 	 * to have device minor nodes created.  This information is passed from
 	 * syncing context (where the zvol is created) to open context (where we
 	 * create the minor nodes).
 	 */
 	nvlist_t	*zri_new_zvols;
 
 	/*
 	 * The errno number returned to caller of zcp_eval().
 	 */
 	int		zri_result;
 } zcp_run_info_t;
 
 zcp_run_info_t *zcp_run_info(lua_State *);
 zcp_cleanup_handler_t *zcp_register_cleanup(lua_State *, zcp_cleanup_t, void *);
 void zcp_deregister_cleanup(lua_State *, zcp_cleanup_handler_t *);
 void zcp_cleanup(lua_State *);
 
 /*
  * Argument parsing routines for channel program callback functions.
  */
 typedef struct zcp_arg {
 	/*
 	 * The name of this argument. For keyword arguments this is the name
 	 * functions will use to set the argument. For positional arguments
 	 * the name has no programmatic meaning, but will appear in error
 	 * messages and help output.
 	 */
 	const char *za_name;
 
 	/*
 	 * The Lua type this argument should have (e.g. LUA_TSTRING,
 	 * LUA_TBOOLEAN) see the lua_type() function documentation for a
 	 * complete list. Calling a function with an argument that does
 	 * not match the expected type will result in the program terminating.
 	 */
 	const int za_lua_type;
 } zcp_arg_t;
 
 void zcp_parse_args(lua_State *, const char *, const zcp_arg_t *,
     const zcp_arg_t *);
 int zcp_nvlist_to_lua(lua_State *, nvlist_t *, char *, int);
 int zcp_dataset_hold_error(lua_State *, dsl_pool_t *, const char *, int);
 struct dsl_dataset *zcp_dataset_hold(lua_State *, dsl_pool_t *,
     const char *, const void *);
 
 typedef int (zcp_lib_func_t)(lua_State *);
 typedef struct zcp_lib_info {
 	const char *name;
 	zcp_lib_func_t *func;
 	const zcp_arg_t pargs[4];
 	const zcp_arg_t kwargs[2];
 } zcp_lib_info_t;
 
 #ifdef	__cplusplus
 }
 #endif
 
 #endif	/* _SYS_ZCP_H */
diff --git a/sys/contrib/openzfs/include/sys/zfs_context.h b/sys/contrib/openzfs/include/sys/zfs_context.h
index bd123435cdb1..5dedb14c7fb5 100644
--- a/sys/contrib/openzfs/include/sys/zfs_context.h
+++ b/sys/contrib/openzfs/include/sys/zfs_context.h
@@ -1,790 +1,792 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  */
 
 #ifndef _SYS_ZFS_CONTEXT_H
 #define	_SYS_ZFS_CONTEXT_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 /*
  * This code compiles in three different contexts. When __KERNEL__ is defined,
  * the code uses "unix-like" kernel interfaces. When _STANDALONE is defined, the
  * code is running in a reduced capacity environment of the boot loader which is
  * generally a subset of both POSIX and kernel interfaces (with a few unique
  * interfaces too). When neither are defined, it's in a userland POSIX or
  * similar environment.
  */
 #if defined(__KERNEL__) || defined(_STANDALONE)
 #include <sys/types.h>
 #include <sys/atomic.h>
 #include <sys/sysmacros.h>
 #include <sys/vmsystm.h>
 #include <sys/condvar.h>
 #include <sys/cmn_err.h>
 #include <sys/kmem.h>
 #include <sys/kmem_cache.h>
 #include <sys/vmem.h>
 #include <sys/misc.h>
 #include <sys/taskq.h>
 #include <sys/param.h>
 #include <sys/disp.h>
 #include <sys/debug.h>
 #include <sys/random.h>
 #include <sys/string.h>
 #include <sys/byteorder.h>
 #include <sys/list.h>
 #include <sys/time.h>
 #include <sys/zone.h>
 #include <sys/kstat.h>
 #include <sys/zfs_debug.h>
 #include <sys/sysevent.h>
 #include <sys/sysevent/eventdefs.h>
 #include <sys/zfs_delay.h>
 #include <sys/sunddi.h>
 #include <sys/ctype.h>
 #include <sys/disp.h>
 #include <sys/trace.h>
 #include <sys/procfs_list.h>
 #include <sys/mod.h>
 #include <sys/uio_impl.h>
 #include <sys/zfs_context_os.h>
 #else /* _KERNEL || _STANDALONE */
 
 #define	_SYS_MUTEX_H
 #define	_SYS_RWLOCK_H
 #define	_SYS_CONDVAR_H
 #define	_SYS_VNODE_H
 #define	_SYS_VFS_H
 #define	_SYS_SUNDDI_H
 #define	_SYS_CALLB_H
 
 #include <stdio.h>
 #include <stdlib.h>
 #include <stddef.h>
 #include <stdarg.h>
 #include <fcntl.h>
 #include <unistd.h>
 #include <errno.h>
 #include <string.h>
 #include <pthread.h>
 #include <setjmp.h>
 #include <assert.h>
 #include <umem.h>
 #include <limits.h>
 #include <atomic.h>
 #include <dirent.h>
 #include <time.h>
 #include <ctype.h>
 #include <signal.h>
 #include <sys/mman.h>
 #include <sys/types.h>
 #include <sys/cred.h>
 #include <sys/sysmacros.h>
 #include <sys/resource.h>
 #include <sys/byteorder.h>
 #include <sys/list.h>
 #include <sys/mod.h>
 #include <sys/uio.h>
 #include <sys/zfs_debug.h>
 #include <sys/kstat.h>
 #include <sys/u8_textprep.h>
 #include <sys/sysevent.h>
 #include <sys/sysevent/eventdefs.h>
 #include <sys/sunddi.h>
 #include <sys/debug.h>
 #include <sys/utsname.h>
 #include <sys/trace_zfs.h>
 
 #include <sys/zfs_context_os.h>
 
 /*
  * Stack
  */
 
 #define	noinline	__attribute__((noinline))
 #define	likely(x)	__builtin_expect((x), 1)
 #define	unlikely(x)	__builtin_expect((x), 0)
 
 /*
  * Debugging
  */
 
 /*
  * Note that we are not using the debugging levels.
  */
 
 #define	CE_CONT		0	/* continuation		*/
 #define	CE_NOTE		1	/* notice		*/
 #define	CE_WARN		2	/* warning		*/
 #define	CE_PANIC	3	/* panic		*/
 #define	CE_IGNORE	4	/* print nothing	*/
 
 /*
  * ZFS debugging
  */
 
 extern void dprintf_setup(int *argc, char **argv);
 
 extern void cmn_err(int, const char *, ...)
     __attribute__((format(printf, 2, 3)));
 extern void vcmn_err(int, const char *, va_list)
     __attribute__((format(printf, 2, 0)));
 extern void panic(const char *, ...)
     __attribute__((format(printf, 1, 2), noreturn));
 extern void vpanic(const char *, va_list)
     __attribute__((format(printf, 1, 0), noreturn));
 
 #define	fm_panic	panic
 
 /*
  * DTrace SDT probes have different signatures in userland than they do in
  * the kernel.  If they're being used in kernel code, re-define them out of
  * existence for their counterparts in libzpool.
  *
  * Here's an example of how to use the set-error probes in userland:
  * zfs$target:::set-error /arg0 == EBUSY/ {stack();}
  *
  * Here's an example of how to use DTRACE_PROBE probes in userland:
  * If there is a probe declared as follows:
  * DTRACE_PROBE2(zfs__probe_name, uint64_t, blkid, dnode_t *, dn);
  * Then you can use it as follows:
  * zfs$target:::probe2 /copyinstr(arg0) == "zfs__probe_name"/
  *     {printf("%u %p\n", arg1, arg2);}
  */
 
 #ifdef DTRACE_PROBE
 #undef	DTRACE_PROBE
 #endif	/* DTRACE_PROBE */
 #define	DTRACE_PROBE(a)
 
 #ifdef DTRACE_PROBE1
 #undef	DTRACE_PROBE1
 #endif	/* DTRACE_PROBE1 */
 #define	DTRACE_PROBE1(a, b, c)
 
 #ifdef DTRACE_PROBE2
 #undef	DTRACE_PROBE2
 #endif	/* DTRACE_PROBE2 */
 #define	DTRACE_PROBE2(a, b, c, d, e)
 
 #ifdef DTRACE_PROBE3
 #undef	DTRACE_PROBE3
 #endif	/* DTRACE_PROBE3 */
 #define	DTRACE_PROBE3(a, b, c, d, e, f, g)
 
 #ifdef DTRACE_PROBE4
 #undef	DTRACE_PROBE4
 #endif	/* DTRACE_PROBE4 */
 #define	DTRACE_PROBE4(a, b, c, d, e, f, g, h, i)
 
 /*
  * Tunables.
  */
 typedef struct zfs_kernel_param {
 	const char *name;	/* unused stub */
 } zfs_kernel_param_t;
 
 #define	ZFS_MODULE_PARAM(scope_prefix, name_prefix, name, type, perm, desc)
 #define	ZFS_MODULE_PARAM_ARGS void
 #define	ZFS_MODULE_PARAM_CALL(scope_prefix, name_prefix, name, setfunc, \
 	getfunc, perm, desc)
 
 /*
  * Threads.
  */
 typedef pthread_t	kthread_t;
 
 #define	TS_RUN		0x00000002
 #define	TS_JOINABLE	0x00000004
 
 #define	curthread	((void *)(uintptr_t)pthread_self())
 #define	getcomm()	"unknown"
 
 #define	thread_create_named(name, stk, stksize, func, arg, len, \
     pp, state, pri)	\
 	zk_thread_create(func, arg, stksize, state)
 #define	thread_create(stk, stksize, func, arg, len, pp, state, pri)	\
 	zk_thread_create(func, arg, stksize, state)
 #define	thread_exit()	pthread_exit(NULL)
 #define	thread_join(t)	pthread_join((pthread_t)(t), NULL)
 
 #define	newproc(f, a, cid, pri, ctp, pid)	(ENOSYS)
 
 /* in libzpool, p0 exists only to have its address taken */
 typedef struct proc {
 	uintptr_t	this_is_never_used_dont_dereference_it;
 } proc_t;
 
 extern struct proc p0;
 #define	curproc		(&p0)
 
 #define	PS_NONE		-1
 
 extern kthread_t *zk_thread_create(void (*func)(void *), void *arg,
     size_t stksize, int state);
 
 #define	issig()		(FALSE)
 
 #define	KPREEMPT_SYNC		(-1)
 
 #define	kpreempt(x)		sched_yield()
 #define	kpreempt_disable()	((void)0)
 #define	kpreempt_enable()	((void)0)
 
 /*
  * Mutexes
  */
 typedef struct kmutex {
 	pthread_mutex_t		m_lock;
 	pthread_t		m_owner;
 } kmutex_t;
 
 #define	MUTEX_DEFAULT		0
 #define	MUTEX_NOLOCKDEP		MUTEX_DEFAULT
 #define	MUTEX_HELD(mp)		pthread_equal((mp)->m_owner, pthread_self())
 #define	MUTEX_NOT_HELD(mp)	!MUTEX_HELD(mp)
 
 extern void mutex_init(kmutex_t *mp, char *name, int type, void *cookie);
 extern void mutex_destroy(kmutex_t *mp);
 extern void mutex_enter(kmutex_t *mp);
 extern int mutex_enter_check_return(kmutex_t *mp);
 extern void mutex_exit(kmutex_t *mp);
 extern int mutex_tryenter(kmutex_t *mp);
 
 #define	NESTED_SINGLE 1
 #define	mutex_enter_nested(mp, class) mutex_enter(mp)
 #define	mutex_enter_interruptible(mp) mutex_enter_check_return(mp)
 /*
  * RW locks
  */
 typedef struct krwlock {
 	pthread_rwlock_t	rw_lock;
 	pthread_t		rw_owner;
 	uint_t			rw_readers;
 } krwlock_t;
 
 typedef int krw_t;
 
 #define	RW_READER		0
 #define	RW_WRITER		1
 #define	RW_DEFAULT		RW_READER
 #define	RW_NOLOCKDEP		RW_READER
 
 #define	RW_READ_HELD(rw)	((rw)->rw_readers > 0)
 #define	RW_WRITE_HELD(rw)	pthread_equal((rw)->rw_owner, pthread_self())
 #define	RW_LOCK_HELD(rw)	(RW_READ_HELD(rw) || RW_WRITE_HELD(rw))
 
 extern void rw_init(krwlock_t *rwlp, char *name, int type, void *arg);
 extern void rw_destroy(krwlock_t *rwlp);
 extern void rw_enter(krwlock_t *rwlp, krw_t rw);
 extern int rw_tryenter(krwlock_t *rwlp, krw_t rw);
 extern int rw_tryupgrade(krwlock_t *rwlp);
 extern void rw_exit(krwlock_t *rwlp);
 #define	rw_downgrade(rwlp) do { } while (0)
 
 /*
  * Credentials
  */
 extern uid_t crgetuid(cred_t *cr);
 extern uid_t crgetruid(cred_t *cr);
 extern gid_t crgetgid(cred_t *cr);
 extern int crgetngroups(cred_t *cr);
 extern gid_t *crgetgroups(cred_t *cr);
 
 /*
  * Condition variables
  */
 typedef pthread_cond_t		kcondvar_t;
 
 #define	CV_DEFAULT		0
 #define	CALLOUT_FLAG_ABSOLUTE	0x2
 
 extern void cv_init(kcondvar_t *cv, char *name, int type, void *arg);
 extern void cv_destroy(kcondvar_t *cv);
 extern void cv_wait(kcondvar_t *cv, kmutex_t *mp);
 extern int cv_wait_sig(kcondvar_t *cv, kmutex_t *mp);
 extern int cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime);
 extern int cv_timedwait_hires(kcondvar_t *cvp, kmutex_t *mp, hrtime_t tim,
     hrtime_t res, int flag);
 extern void cv_signal(kcondvar_t *cv);
 extern void cv_broadcast(kcondvar_t *cv);
 
 #define	cv_timedwait_io(cv, mp, at)		cv_timedwait(cv, mp, at)
 #define	cv_timedwait_idle(cv, mp, at)		cv_timedwait(cv, mp, at)
 #define	cv_timedwait_sig(cv, mp, at)		cv_timedwait(cv, mp, at)
 #define	cv_wait_io(cv, mp)			cv_wait(cv, mp)
 #define	cv_wait_idle(cv, mp)			cv_wait(cv, mp)
 #define	cv_wait_io_sig(cv, mp)			cv_wait_sig(cv, mp)
 #define	cv_timedwait_sig_hires(cv, mp, t, r, f) \
 	cv_timedwait_hires(cv, mp, t, r, f)
 #define	cv_timedwait_idle_hires(cv, mp, t, r, f) \
 	cv_timedwait_hires(cv, mp, t, r, f)
 
 /*
  * Thread-specific data
  */
 #define	tsd_get(k) pthread_getspecific(k)
 #define	tsd_set(k, v) pthread_setspecific(k, v)
 #define	tsd_create(kp, d) pthread_key_create((pthread_key_t *)kp, d)
 #define	tsd_destroy(kp) /* nothing */
 #ifdef __FreeBSD__
 typedef off_t loff_t;
 #endif
 
 /*
  * kstat creation, installation and deletion
  */
 extern kstat_t *kstat_create(const char *, int,
     const char *, const char *, uchar_t, ulong_t, uchar_t);
 extern void kstat_install(kstat_t *);
 extern void kstat_delete(kstat_t *);
 extern void kstat_set_raw_ops(kstat_t *ksp,
     int (*headers)(char *buf, size_t size),
     int (*data)(char *buf, size_t size, void *data),
     void *(*addr)(kstat_t *ksp, loff_t index));
 
 /*
  * procfs list manipulation
  */
 
 typedef struct procfs_list {
 	void		*pl_private;
 	kmutex_t	pl_lock;
 	list_t		pl_list;
 	uint64_t	pl_next_id;
 	size_t		pl_node_offset;
 } procfs_list_t;
 
 #ifndef __cplusplus
 struct seq_file { };
 void seq_printf(struct seq_file *m, const char *fmt, ...);
 
 typedef struct procfs_list_node {
 	list_node_t	pln_link;
 	uint64_t	pln_id;
 } procfs_list_node_t;
 
 void procfs_list_install(const char *module,
     const char *submodule,
     const char *name,
     mode_t mode,
     procfs_list_t *procfs_list,
     int (*show)(struct seq_file *f, void *p),
     int (*show_header)(struct seq_file *f),
     int (*clear)(procfs_list_t *procfs_list),
     size_t procfs_list_node_off);
 void procfs_list_uninstall(procfs_list_t *procfs_list);
 void procfs_list_destroy(procfs_list_t *procfs_list);
 void procfs_list_add(procfs_list_t *procfs_list, void *p);
 #endif
 
 /*
  * Kernel memory
  */
 #define	KM_SLEEP		UMEM_NOFAIL
 #define	KM_PUSHPAGE		KM_SLEEP
 #define	KM_NOSLEEP		UMEM_DEFAULT
 #define	KM_NORMALPRI		0	/* not needed with UMEM_DEFAULT */
 #define	KMC_NODEBUG		UMC_NODEBUG
 #define	KMC_KVMEM		0x0
 #define	KMC_RECLAIMABLE		0x0
 #define	kmem_alloc(_s, _f)	umem_alloc(_s, _f)
 #define	kmem_zalloc(_s, _f)	umem_zalloc(_s, _f)
 #define	kmem_free(_b, _s)	umem_free(_b, _s)
 #define	vmem_alloc(_s, _f)	kmem_alloc(_s, _f)
 #define	vmem_zalloc(_s, _f)	kmem_zalloc(_s, _f)
 #define	vmem_free(_b, _s)	kmem_free(_b, _s)
 #define	kmem_cache_create(_a, _b, _c, _d, _e, _f, _g, _h, _i) \
 	umem_cache_create(_a, _b, _c, _d, _e, _f, _g, _h, _i)
 #define	kmem_cache_destroy(_c)	umem_cache_destroy(_c)
 #define	kmem_cache_alloc(_c, _f) umem_cache_alloc(_c, _f)
 #define	kmem_cache_free(_c, _b)	umem_cache_free(_c, _b)
 #define	kmem_debugging()	0
 #define	kmem_cache_reap_now(_c)	umem_cache_reap_now(_c);
 #define	kmem_cache_set_move(_c, _cb)	/* nothing */
 #define	POINTER_INVALIDATE(_pp)		/* nothing */
 #define	POINTER_IS_VALID(_p)	0
 
 typedef umem_cache_t kmem_cache_t;
 
 typedef enum kmem_cbrc {
 	KMEM_CBRC_YES,
 	KMEM_CBRC_NO,
 	KMEM_CBRC_LATER,
 	KMEM_CBRC_DONT_NEED,
 	KMEM_CBRC_DONT_KNOW
 } kmem_cbrc_t;
 
 /*
  * Task queues
  */
 
 #define	TASKQ_NAMELEN	31
 
 typedef uintptr_t taskqid_t;
 typedef void (task_func_t)(void *);
 
 typedef struct taskq_ent {
 	struct taskq_ent	*tqent_next;
 	struct taskq_ent	*tqent_prev;
 	task_func_t		*tqent_func;
 	void			*tqent_arg;
 	uintptr_t		tqent_flags;
 } taskq_ent_t;
 
 typedef struct taskq {
 	char		tq_name[TASKQ_NAMELEN + 1];
 	kmutex_t	tq_lock;
 	krwlock_t	tq_threadlock;
 	kcondvar_t	tq_dispatch_cv;
 	kcondvar_t	tq_wait_cv;
 	kthread_t	**tq_threadlist;
 	int		tq_flags;
 	int		tq_active;
 	int		tq_nthreads;
 	int		tq_nalloc;
 	int		tq_minalloc;
 	int		tq_maxalloc;
 	kcondvar_t	tq_maxalloc_cv;
 	int		tq_maxalloc_wait;
 	taskq_ent_t	*tq_freelist;
 	taskq_ent_t	tq_task;
 } taskq_t;
 
 #define	TQENT_FLAG_PREALLOC	0x1	/* taskq_dispatch_ent used */
 
 #define	TASKQ_PREPOPULATE	0x0001
 #define	TASKQ_CPR_SAFE		0x0002	/* Use CPR safe protocol */
 #define	TASKQ_DYNAMIC		0x0004	/* Use dynamic thread scheduling */
 #define	TASKQ_THREADS_CPU_PCT	0x0008	/* Scale # threads by # cpus */
 #define	TASKQ_DC_BATCH		0x0010	/* Mark threads as batch */
 
 #define	TQ_SLEEP	KM_SLEEP	/* Can block for memory */
 #define	TQ_NOSLEEP	KM_NOSLEEP	/* cannot block for memory; may fail */
 #define	TQ_NOQUEUE	0x02		/* Do not enqueue if can't dispatch */
 #define	TQ_FRONT	0x08		/* Queue in front */
 
 #define	TASKQID_INVALID		((taskqid_t)0)
 
 extern taskq_t *system_taskq;
 extern taskq_t *system_delay_taskq;
 
 extern taskq_t	*taskq_create(const char *, int, pri_t, int, int, uint_t);
 #define	taskq_create_proc(a, b, c, d, e, p, f) \
 	    (taskq_create(a, b, c, d, e, f))
 #define	taskq_create_sysdc(a, b, d, e, p, dc, f) \
 	    ((void) sizeof (dc), taskq_create(a, b, maxclsyspri, d, e, f))
 extern taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t);
 extern taskqid_t taskq_dispatch_delay(taskq_t *, task_func_t, void *, uint_t,
     clock_t);
 extern void	taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t,
     taskq_ent_t *);
 extern int	taskq_empty_ent(taskq_ent_t *);
 extern void	taskq_init_ent(taskq_ent_t *);
 extern void	taskq_destroy(taskq_t *);
 extern void	taskq_wait(taskq_t *);
 extern void	taskq_wait_id(taskq_t *, taskqid_t);
 extern void	taskq_wait_outstanding(taskq_t *, taskqid_t);
 extern int	taskq_member(taskq_t *, kthread_t *);
 extern taskq_t	*taskq_of_curthread(void);
 extern int	taskq_cancel_id(taskq_t *, taskqid_t);
 extern void	system_taskq_init(void);
 extern void	system_taskq_fini(void);
 
 #define	XVA_MAPSIZE	3
 #define	XVA_MAGIC	0x78766174
 
 extern char *vn_dumpdir;
 #define	AV_SCANSTAMP_SZ	32		/* length of anti-virus scanstamp */
 
 typedef struct xoptattr {
 	inode_timespec_t xoa_createtime;	/* Create time of file */
 	uint8_t		xoa_archive;
 	uint8_t		xoa_system;
 	uint8_t		xoa_readonly;
 	uint8_t		xoa_hidden;
 	uint8_t		xoa_nounlink;
 	uint8_t		xoa_immutable;
 	uint8_t		xoa_appendonly;
 	uint8_t		xoa_nodump;
 	uint8_t		xoa_settable;
 	uint8_t		xoa_opaque;
 	uint8_t		xoa_av_quarantined;
 	uint8_t		xoa_av_modified;
 	uint8_t		xoa_av_scanstamp[AV_SCANSTAMP_SZ];
 	uint8_t		xoa_reparse;
 	uint8_t		xoa_offline;
 	uint8_t		xoa_sparse;
 } xoptattr_t;
 
 typedef struct vattr {
 	uint_t		va_mask;	/* bit-mask of attributes */
 	u_offset_t	va_size;	/* file size in bytes */
 } vattr_t;
 
 
 typedef struct xvattr {
 	vattr_t		xva_vattr;	/* Embedded vattr structure */
 	uint32_t	xva_magic;	/* Magic Number */
 	uint32_t	xva_mapsize;	/* Size of attr bitmap (32-bit words) */
 	uint32_t	*xva_rtnattrmapp;	/* Ptr to xva_rtnattrmap[] */
 	uint32_t	xva_reqattrmap[XVA_MAPSIZE];	/* Requested attrs */
 	uint32_t	xva_rtnattrmap[XVA_MAPSIZE];	/* Returned attrs */
 	xoptattr_t	xva_xoptattrs;	/* Optional attributes */
 } xvattr_t;
 
 typedef struct vsecattr {
 	uint_t		vsa_mask;	/* See below */
 	int		vsa_aclcnt;	/* ACL entry count */
 	void		*vsa_aclentp;	/* pointer to ACL entries */
 	int		vsa_dfaclcnt;	/* default ACL entry count */
 	void		*vsa_dfaclentp;	/* pointer to default ACL entries */
 	size_t		vsa_aclentsz;	/* ACE size in bytes of vsa_aclentp */
 } vsecattr_t;
 
 #define	AT_MODE		0x00002
 #define	AT_UID		0x00004
 #define	AT_GID		0x00008
 #define	AT_FSID		0x00010
 #define	AT_NODEID	0x00020
 #define	AT_NLINK	0x00040
 #define	AT_SIZE		0x00080
 #define	AT_ATIME	0x00100
 #define	AT_MTIME	0x00200
 #define	AT_CTIME	0x00400
 #define	AT_RDEV		0x00800
 #define	AT_BLKSIZE	0x01000
 #define	AT_NBLOCKS	0x02000
 #define	AT_SEQ		0x08000
 #define	AT_XVATTR	0x10000
 
 #define	CRCREAT		0
 
 #define	F_FREESP	11
 #define	FIGNORECASE	0x80000 /* request case-insensitive lookups */
 
 /*
  * Random stuff
  */
 #define	ddi_get_lbolt()		(gethrtime() >> 23)
 #define	ddi_get_lbolt64()	(gethrtime() >> 23)
 #define	hz	119	/* frequency when using gethrtime() >> 23 for lbolt */
 
 #define	ddi_time_before(a, b)		(a < b)
 #define	ddi_time_after(a, b)		ddi_time_before(b, a)
 #define	ddi_time_before_eq(a, b)	(!ddi_time_after(a, b))
 #define	ddi_time_after_eq(a, b)		ddi_time_before_eq(b, a)
 
 #define	ddi_time_before64(a, b)		(a < b)
 #define	ddi_time_after64(a, b)		ddi_time_before64(b, a)
 #define	ddi_time_before_eq64(a, b)	(!ddi_time_after64(a, b))
 #define	ddi_time_after_eq64(a, b)	ddi_time_before_eq64(b, a)
 
 extern void delay(clock_t ticks);
 
 #define	SEC_TO_TICK(sec)	((sec) * hz)
 #define	MSEC_TO_TICK(msec)	(howmany((hrtime_t)(msec) * hz, MILLISEC))
 #define	USEC_TO_TICK(usec)	(howmany((hrtime_t)(usec) * hz, MICROSEC))
 #define	NSEC_TO_TICK(nsec)	(howmany((hrtime_t)(nsec) * hz, NANOSEC))
 
 #define	max_ncpus	64
 #define	boot_ncpus	(sysconf(_SC_NPROCESSORS_ONLN))
 
 /*
  * Process priorities as defined by setpriority(2) and getpriority(2).
  */
 #define	minclsyspri	19
 #define	maxclsyspri	-20
 #define	defclsyspri	0
 
 #define	CPU_SEQID	((uintptr_t)pthread_self() & (max_ncpus - 1))
 #define	CPU_SEQID_UNSTABLE	CPU_SEQID
 
 #define	kcred		NULL
 #define	CRED()		NULL
 
+#define	crhold(cr)	((void)cr)
+#define	crfree(cr)	((void)cr)
+
 #define	ptob(x)		((x) * PAGESIZE)
 
 #define	NN_DIVISOR_1000	(1U << 0)
 #define	NN_NUMBUF_SZ	(6)
 
 extern uint64_t physmem;
 extern const char *random_path;
 extern const char *urandom_path;
 
 extern int highbit64(uint64_t i);
 extern int lowbit64(uint64_t i);
 extern int random_get_bytes(uint8_t *ptr, size_t len);
 extern int random_get_pseudo_bytes(uint8_t *ptr, size_t len);
 
 static __inline__ uint32_t
 random_in_range(uint32_t range)
 {
 	uint32_t r;
 
 	ASSERT(range != 0);
 
 	if (range == 1)
 		return (0);
 
 	(void) random_get_pseudo_bytes((uint8_t *)&r, sizeof (r));
 
 	return (r % range);
 }
 
 extern void kernel_init(int mode);
 extern void kernel_fini(void);
 extern void random_init(void);
 extern void random_fini(void);
 
 struct spa;
 extern void show_pool_stats(struct spa *);
 extern int set_global_var(char const *arg);
 
 typedef struct callb_cpr {
 	kmutex_t	*cc_lockp;
 } callb_cpr_t;
 
 #define	CALLB_CPR_INIT(cp, lockp, func, name)	{		\
 	(cp)->cc_lockp = lockp;					\
 }
 
 #define	CALLB_CPR_SAFE_BEGIN(cp) {				\
 	ASSERT(MUTEX_HELD((cp)->cc_lockp));			\
 }
 
 #define	CALLB_CPR_SAFE_END(cp, lockp) {				\
 	ASSERT(MUTEX_HELD((cp)->cc_lockp));			\
 }
 
 #define	CALLB_CPR_EXIT(cp) {					\
 	ASSERT(MUTEX_HELD((cp)->cc_lockp));			\
 	mutex_exit((cp)->cc_lockp);				\
 }
 
 #define	zone_dataset_visible(x, y)	(1)
 #define	INGLOBALZONE(z)			(1)
 extern uint32_t zone_get_hostid(void *zonep);
 
 extern char *kmem_vasprintf(const char *fmt, va_list adx);
 extern char *kmem_asprintf(const char *fmt, ...);
 #define	kmem_strfree(str) kmem_free((str), strlen(str) + 1)
 #define	kmem_strdup(s)  strdup(s)
 
 #ifndef __cplusplus
 extern int kmem_scnprintf(char *restrict str, size_t size,
     const char *restrict fmt, ...);
 #endif
 
 /*
  * Hostname information
  */
 extern int ddi_strtoull(const char *str, char **nptr, int base,
     u_longlong_t *result);
 
 typedef struct utsname	utsname_t;
 extern utsname_t *utsname(void);
 
 /* ZFS Boot Related stuff. */
 
 struct _buf {
 	intptr_t	_fd;
 };
 
 struct bootstat {
 	uint64_t st_size;
 };
 
 typedef struct ace_object {
 	uid_t		a_who;
 	uint32_t	a_access_mask;
 	uint16_t	a_flags;
 	uint16_t	a_type;
 	uint8_t		a_obj_type[16];
 	uint8_t		a_inherit_obj_type[16];
 } ace_object_t;
 
 
 #define	ACE_ACCESS_ALLOWED_OBJECT_ACE_TYPE	0x05
 #define	ACE_ACCESS_DENIED_OBJECT_ACE_TYPE	0x06
 #define	ACE_SYSTEM_AUDIT_OBJECT_ACE_TYPE	0x07
 #define	ACE_SYSTEM_ALARM_OBJECT_ACE_TYPE	0x08
 
 extern int zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr);
 extern int zfs_secpolicy_rename_perms(const char *from, const char *to,
     cred_t *cr);
 extern int zfs_secpolicy_destroy_perms(const char *name, cred_t *cr);
 extern int secpolicy_zfs(const cred_t *cr);
-extern int secpolicy_zfs_proc(const cred_t *cr, proc_t *proc);
 extern zoneid_t getzoneid(void);
 
 /* SID stuff */
 typedef struct ksiddomain {
 	uint_t	kd_ref;
 	uint_t	kd_len;
 	char	*kd_name;
 } ksiddomain_t;
 
 ksiddomain_t *ksid_lookupdomain(const char *);
 void ksiddomain_rele(ksiddomain_t *);
 
 #define	DDI_SLEEP	KM_SLEEP
 #define	ddi_log_sysevent(_a, _b, _c, _d, _e, _f, _g) \
 	sysevent_post_event(_c, _d, _b, "libzpool", _e, _f)
 
 #define	zfs_sleep_until(wakeup)						\
 	do {								\
 		hrtime_t delta = wakeup - gethrtime();			\
 		struct timespec ts;					\
 		ts.tv_sec = delta / NANOSEC;				\
 		ts.tv_nsec = delta % NANOSEC;				\
 		(void) nanosleep(&ts, NULL);				\
 	} while (0)
 
 typedef int fstrans_cookie_t;
 
 extern fstrans_cookie_t spl_fstrans_mark(void);
 extern void spl_fstrans_unmark(fstrans_cookie_t);
 extern int __spl_pf_fstrans_check(void);
 extern int kmem_cache_reap_active(void);
 
 
 /*
  * Kernel modules
  */
 #define	__init
 #define	__exit
 
 #endif  /* _KERNEL || _STANDALONE */
 
 #ifdef __cplusplus
 };
 #endif
 
 #endif	/* _SYS_ZFS_CONTEXT_H */
diff --git a/sys/contrib/openzfs/lib/libspl/os/linux/zone.c b/sys/contrib/openzfs/lib/libspl/os/linux/zone.c
index 622d04cbc14a..f8a10bfa167a 100644
--- a/sys/contrib/openzfs/lib/libspl/os/linux/zone.c
+++ b/sys/contrib/openzfs/lib/libspl/os/linux/zone.c
@@ -1,62 +1,62 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright 2006 Ricardo Correia.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #include <unistd.h>
 #include <stdio.h>
 #include <errno.h>
 #include <stdlib.h>
 #include <limits.h>
 #include <string.h>
 
 #include <zone.h>
 
 zoneid_t
 getzoneid(void)
 {
 	char path[PATH_MAX];
 	char buf[128] = { '\0' };
 	char *cp;
 
 	int c = snprintf(path, sizeof (path), "/proc/self/ns/user");
 	/* This API doesn't have any error checking... */
 	if (c < 0 || c >= sizeof (path))
-		return (0);
+		return (GLOBAL_ZONEID);
 
 	ssize_t r = readlink(path, buf, sizeof (buf) - 1);
 	if (r < 0)
-		return (0);
+		return (GLOBAL_ZONEID);
 
 	cp = strchr(buf, '[');
 	if (cp == NULL)
-		return (0);
+		return (GLOBAL_ZONEID);
 	cp++;
 
 	unsigned long n = strtoul(cp, NULL, 10);
 	if (n == ULONG_MAX && errno == ERANGE)
-		return (0);
+		return (GLOBAL_ZONEID);
 	zoneid_t z = (zoneid_t)n;
 
 	return (z);
 }
diff --git a/sys/contrib/openzfs/lib/libzpool/kernel.c b/sys/contrib/openzfs/lib/libzpool/kernel.c
index a02bee72b8df..148c765753dd 100644
--- a/sys/contrib/openzfs/lib/libzpool/kernel.c
+++ b/sys/contrib/openzfs/lib/libzpool/kernel.c
@@ -1,1474 +1,1467 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
  */
 
 #include <assert.h>
 #include <fcntl.h>
 #include <libgen.h>
 #include <poll.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <limits.h>
 #include <libzutil.h>
 #include <sys/crypto/icp.h>
 #include <sys/processor.h>
 #include <sys/rrwlock.h>
 #include <sys/spa.h>
 #include <sys/stat.h>
 #include <sys/systeminfo.h>
 #include <sys/time.h>
 #include <sys/utsname.h>
 #include <sys/zfs_context.h>
 #include <sys/zfs_onexit.h>
 #include <sys/zfs_vfsops.h>
 #include <sys/zstd/zstd.h>
 #include <sys/zvol.h>
 #include <zfs_fletcher.h>
 #include <zlib.h>
 
 /*
  * Emulation of kernel services in userland.
  */
 
 uint64_t physmem;
 uint32_t hostid;
 struct utsname hw_utsname;
 
 /* If set, all blocks read will be copied to the specified directory. */
 char *vn_dumpdir = NULL;
 
 /* this only exists to have its address taken */
 struct proc p0;
 
 /*
  * =========================================================================
  * threads
  * =========================================================================
  *
  * TS_STACK_MIN is dictated by the minimum allowed pthread stack size.  While
  * TS_STACK_MAX is somewhat arbitrary, it was selected to be large enough for
  * the expected stack depth while small enough to avoid exhausting address
  * space with high thread counts.
  */
 #define	TS_STACK_MIN	MAX(PTHREAD_STACK_MIN, 32768)
 #define	TS_STACK_MAX	(256 * 1024)
 
 struct zk_thread_wrapper {
 	void (*func)(void *);
 	void *arg;
 };
 
 static void *
 zk_thread_wrapper(void *arg)
 {
 	struct zk_thread_wrapper ztw;
 	memcpy(&ztw, arg, sizeof (ztw));
 	free(arg);
 	ztw.func(ztw.arg);
 	return (NULL);
 }
 
 kthread_t *
 zk_thread_create(void (*func)(void *), void *arg, size_t stksize, int state)
 {
 	pthread_attr_t attr;
 	pthread_t tid;
 	char *stkstr;
 	struct zk_thread_wrapper *ztw;
 	int detachstate = PTHREAD_CREATE_DETACHED;
 
 	VERIFY0(pthread_attr_init(&attr));
 
 	if (state & TS_JOINABLE)
 		detachstate = PTHREAD_CREATE_JOINABLE;
 
 	VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
 
 	/*
 	 * We allow the default stack size in user space to be specified by
 	 * setting the ZFS_STACK_SIZE environment variable.  This allows us
 	 * the convenience of observing and debugging stack overruns in
 	 * user space.  Explicitly specified stack sizes will be honored.
 	 * The usage of ZFS_STACK_SIZE is discussed further in the
 	 * ENVIRONMENT VARIABLES sections of the ztest(1) man page.
 	 */
 	if (stksize == 0) {
 		stkstr = getenv("ZFS_STACK_SIZE");
 
 		if (stkstr == NULL)
 			stksize = TS_STACK_MAX;
 		else
 			stksize = MAX(atoi(stkstr), TS_STACK_MIN);
 	}
 
 	VERIFY3S(stksize, >, 0);
 	stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
 
 	/*
 	 * If this ever fails, it may be because the stack size is not a
 	 * multiple of system page size.
 	 */
 	VERIFY0(pthread_attr_setstacksize(&attr, stksize));
 	VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
 
 	VERIFY(ztw = malloc(sizeof (*ztw)));
 	ztw->func = func;
 	ztw->arg = arg;
 	VERIFY0(pthread_create(&tid, &attr, zk_thread_wrapper, ztw));
 	VERIFY0(pthread_attr_destroy(&attr));
 
 	return ((void *)(uintptr_t)tid);
 }
 
 /*
  * =========================================================================
  * kstats
  * =========================================================================
  */
 kstat_t *
 kstat_create(const char *module, int instance, const char *name,
     const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag)
 {
 	(void) module, (void) instance, (void) name, (void) class, (void) type,
 	    (void) ndata, (void) ks_flag;
 	return (NULL);
 }
 
 void
 kstat_install(kstat_t *ksp)
 {
 	(void) ksp;
 }
 
 void
 kstat_delete(kstat_t *ksp)
 {
 	(void) ksp;
 }
 
 void
 kstat_set_raw_ops(kstat_t *ksp,
     int (*headers)(char *buf, size_t size),
     int (*data)(char *buf, size_t size, void *data),
     void *(*addr)(kstat_t *ksp, loff_t index))
 {
 	(void) ksp, (void) headers, (void) data, (void) addr;
 }
 
 /*
  * =========================================================================
  * mutexes
  * =========================================================================
  */
 
 void
 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
 {
 	(void) name, (void) type, (void) cookie;
 	VERIFY0(pthread_mutex_init(&mp->m_lock, NULL));
 	memset(&mp->m_owner, 0, sizeof (pthread_t));
 }
 
 void
 mutex_destroy(kmutex_t *mp)
 {
 	VERIFY0(pthread_mutex_destroy(&mp->m_lock));
 }
 
 void
 mutex_enter(kmutex_t *mp)
 {
 	VERIFY0(pthread_mutex_lock(&mp->m_lock));
 	mp->m_owner = pthread_self();
 }
 
 int
 mutex_enter_check_return(kmutex_t *mp)
 {
 	int error = pthread_mutex_lock(&mp->m_lock);
 	if (error == 0)
 		mp->m_owner = pthread_self();
 	return (error);
 }
 
 int
 mutex_tryenter(kmutex_t *mp)
 {
 	int error = pthread_mutex_trylock(&mp->m_lock);
 	if (error == 0) {
 		mp->m_owner = pthread_self();
 		return (1);
 	} else {
 		VERIFY3S(error, ==, EBUSY);
 		return (0);
 	}
 }
 
 void
 mutex_exit(kmutex_t *mp)
 {
 	memset(&mp->m_owner, 0, sizeof (pthread_t));
 	VERIFY0(pthread_mutex_unlock(&mp->m_lock));
 }
 
 /*
  * =========================================================================
  * rwlocks
  * =========================================================================
  */
 
 void
 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
 {
 	(void) name, (void) type, (void) arg;
 	VERIFY0(pthread_rwlock_init(&rwlp->rw_lock, NULL));
 	rwlp->rw_readers = 0;
 	rwlp->rw_owner = 0;
 }
 
 void
 rw_destroy(krwlock_t *rwlp)
 {
 	VERIFY0(pthread_rwlock_destroy(&rwlp->rw_lock));
 }
 
 void
 rw_enter(krwlock_t *rwlp, krw_t rw)
 {
 	if (rw == RW_READER) {
 		VERIFY0(pthread_rwlock_rdlock(&rwlp->rw_lock));
 		atomic_inc_uint(&rwlp->rw_readers);
 	} else {
 		VERIFY0(pthread_rwlock_wrlock(&rwlp->rw_lock));
 		rwlp->rw_owner = pthread_self();
 	}
 }
 
 void
 rw_exit(krwlock_t *rwlp)
 {
 	if (RW_READ_HELD(rwlp))
 		atomic_dec_uint(&rwlp->rw_readers);
 	else
 		rwlp->rw_owner = 0;
 
 	VERIFY0(pthread_rwlock_unlock(&rwlp->rw_lock));
 }
 
 int
 rw_tryenter(krwlock_t *rwlp, krw_t rw)
 {
 	int error;
 
 	if (rw == RW_READER)
 		error = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
 	else
 		error = pthread_rwlock_trywrlock(&rwlp->rw_lock);
 
 	if (error == 0) {
 		if (rw == RW_READER)
 			atomic_inc_uint(&rwlp->rw_readers);
 		else
 			rwlp->rw_owner = pthread_self();
 
 		return (1);
 	}
 
 	VERIFY3S(error, ==, EBUSY);
 
 	return (0);
 }
 
 uint32_t
 zone_get_hostid(void *zonep)
 {
 	/*
 	 * We're emulating the system's hostid in userland.
 	 */
 	(void) zonep;
 	return (hostid);
 }
 
 int
 rw_tryupgrade(krwlock_t *rwlp)
 {
 	(void) rwlp;
 	return (0);
 }
 
 /*
  * =========================================================================
  * condition variables
  * =========================================================================
  */
 
 void
 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
 {
 	(void) name, (void) type, (void) arg;
 	VERIFY0(pthread_cond_init(cv, NULL));
 }
 
 void
 cv_destroy(kcondvar_t *cv)
 {
 	VERIFY0(pthread_cond_destroy(cv));
 }
 
 void
 cv_wait(kcondvar_t *cv, kmutex_t *mp)
 {
 	memset(&mp->m_owner, 0, sizeof (pthread_t));
 	VERIFY0(pthread_cond_wait(cv, &mp->m_lock));
 	mp->m_owner = pthread_self();
 }
 
 int
 cv_wait_sig(kcondvar_t *cv, kmutex_t *mp)
 {
 	cv_wait(cv, mp);
 	return (1);
 }
 
 int
 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
 {
 	int error;
 	struct timeval tv;
 	struct timespec ts;
 	clock_t delta;
 
 	delta = abstime - ddi_get_lbolt();
 	if (delta <= 0)
 		return (-1);
 
 	VERIFY(gettimeofday(&tv, NULL) == 0);
 
 	ts.tv_sec = tv.tv_sec + delta / hz;
 	ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % hz) * (NANOSEC / hz);
 	if (ts.tv_nsec >= NANOSEC) {
 		ts.tv_sec++;
 		ts.tv_nsec -= NANOSEC;
 	}
 
 	memset(&mp->m_owner, 0, sizeof (pthread_t));
 	error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
 	mp->m_owner = pthread_self();
 
 	if (error == ETIMEDOUT)
 		return (-1);
 
 	VERIFY0(error);
 
 	return (1);
 }
 
 int
 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
     int flag)
 {
 	(void) res;
 	int error;
 	struct timeval tv;
 	struct timespec ts;
 	hrtime_t delta;
 
 	ASSERT(flag == 0 || flag == CALLOUT_FLAG_ABSOLUTE);
 
 	delta = tim;
 	if (flag & CALLOUT_FLAG_ABSOLUTE)
 		delta -= gethrtime();
 
 	if (delta <= 0)
 		return (-1);
 
 	VERIFY0(gettimeofday(&tv, NULL));
 
 	ts.tv_sec = tv.tv_sec + delta / NANOSEC;
 	ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % NANOSEC);
 	if (ts.tv_nsec >= NANOSEC) {
 		ts.tv_sec++;
 		ts.tv_nsec -= NANOSEC;
 	}
 
 	memset(&mp->m_owner, 0, sizeof (pthread_t));
 	error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
 	mp->m_owner = pthread_self();
 
 	if (error == ETIMEDOUT)
 		return (-1);
 
 	VERIFY0(error);
 
 	return (1);
 }
 
 void
 cv_signal(kcondvar_t *cv)
 {
 	VERIFY0(pthread_cond_signal(cv));
 }
 
 void
 cv_broadcast(kcondvar_t *cv)
 {
 	VERIFY0(pthread_cond_broadcast(cv));
 }
 
 /*
  * =========================================================================
  * procfs list
  * =========================================================================
  */
 
 void
 seq_printf(struct seq_file *m, const char *fmt, ...)
 {
 	(void) m, (void) fmt;
 }
 
 void
 procfs_list_install(const char *module,
     const char *submodule,
     const char *name,
     mode_t mode,
     procfs_list_t *procfs_list,
     int (*show)(struct seq_file *f, void *p),
     int (*show_header)(struct seq_file *f),
     int (*clear)(procfs_list_t *procfs_list),
     size_t procfs_list_node_off)
 {
 	(void) module, (void) submodule, (void) name, (void) mode, (void) show,
 	    (void) show_header, (void) clear;
 	mutex_init(&procfs_list->pl_lock, NULL, MUTEX_DEFAULT, NULL);
 	list_create(&procfs_list->pl_list,
 	    procfs_list_node_off + sizeof (procfs_list_node_t),
 	    procfs_list_node_off + offsetof(procfs_list_node_t, pln_link));
 	procfs_list->pl_next_id = 1;
 	procfs_list->pl_node_offset = procfs_list_node_off;
 }
 
 void
 procfs_list_uninstall(procfs_list_t *procfs_list)
 {
 	(void) procfs_list;
 }
 
 void
 procfs_list_destroy(procfs_list_t *procfs_list)
 {
 	ASSERT(list_is_empty(&procfs_list->pl_list));
 	list_destroy(&procfs_list->pl_list);
 	mutex_destroy(&procfs_list->pl_lock);
 }
 
 #define	NODE_ID(procfs_list, obj) \
 		(((procfs_list_node_t *)(((char *)obj) + \
 		(procfs_list)->pl_node_offset))->pln_id)
 
 void
 procfs_list_add(procfs_list_t *procfs_list, void *p)
 {
 	ASSERT(MUTEX_HELD(&procfs_list->pl_lock));
 	NODE_ID(procfs_list, p) = procfs_list->pl_next_id++;
 	list_insert_tail(&procfs_list->pl_list, p);
 }
 
 /*
  * =========================================================================
  * vnode operations
  * =========================================================================
  */
 
 /*
  * =========================================================================
  * Figure out which debugging statements to print
  * =========================================================================
  */
 
 static char *dprintf_string;
 static int dprintf_print_all;
 
 int
 dprintf_find_string(const char *string)
 {
 	char *tmp_str = dprintf_string;
 	int len = strlen(string);
 
 	/*
 	 * Find out if this is a string we want to print.
 	 * String format: file1.c,function_name1,file2.c,file3.c
 	 */
 
 	while (tmp_str != NULL) {
 		if (strncmp(tmp_str, string, len) == 0 &&
 		    (tmp_str[len] == ',' || tmp_str[len] == '\0'))
 			return (1);
 		tmp_str = strchr(tmp_str, ',');
 		if (tmp_str != NULL)
 			tmp_str++; /* Get rid of , */
 	}
 	return (0);
 }
 
 void
 dprintf_setup(int *argc, char **argv)
 {
 	int i, j;
 
 	/*
 	 * Debugging can be specified two ways: by setting the
 	 * environment variable ZFS_DEBUG, or by including a
 	 * "debug=..."  argument on the command line.  The command
 	 * line setting overrides the environment variable.
 	 */
 
 	for (i = 1; i < *argc; i++) {
 		int len = strlen("debug=");
 		/* First look for a command line argument */
 		if (strncmp("debug=", argv[i], len) == 0) {
 			dprintf_string = argv[i] + len;
 			/* Remove from args */
 			for (j = i; j < *argc; j++)
 				argv[j] = argv[j+1];
 			argv[j] = NULL;
 			(*argc)--;
 		}
 	}
 
 	if (dprintf_string == NULL) {
 		/* Look for ZFS_DEBUG environment variable */
 		dprintf_string = getenv("ZFS_DEBUG");
 	}
 
 	/*
 	 * Are we just turning on all debugging?
 	 */
 	if (dprintf_find_string("on"))
 		dprintf_print_all = 1;
 
 	if (dprintf_string != NULL)
 		zfs_flags |= ZFS_DEBUG_DPRINTF;
 }
 
 /*
  * =========================================================================
  * debug printfs
  * =========================================================================
  */
 void
 __dprintf(boolean_t dprint, const char *file, const char *func,
     int line, const char *fmt, ...)
 {
 	/* Get rid of annoying "../common/" prefix to filename. */
 	const char *newfile = zfs_basename(file);
 
 	va_list adx;
 	if (dprint) {
 		/* dprintf messages are printed immediately */
 
 		if (!dprintf_print_all &&
 		    !dprintf_find_string(newfile) &&
 		    !dprintf_find_string(func))
 			return;
 
 		/* Print out just the function name if requested */
 		flockfile(stdout);
 		if (dprintf_find_string("pid"))
 			(void) printf("%d ", getpid());
 		if (dprintf_find_string("tid"))
 			(void) printf("%ju ",
 			    (uintmax_t)(uintptr_t)pthread_self());
 		if (dprintf_find_string("cpu"))
 			(void) printf("%u ", getcpuid());
 		if (dprintf_find_string("time"))
 			(void) printf("%llu ", gethrtime());
 		if (dprintf_find_string("long"))
 			(void) printf("%s, line %d: ", newfile, line);
 		(void) printf("dprintf: %s: ", func);
 		va_start(adx, fmt);
 		(void) vprintf(fmt, adx);
 		va_end(adx);
 		funlockfile(stdout);
 	} else {
 		/* zfs_dbgmsg is logged for dumping later */
 		size_t size;
 		char *buf;
 		int i;
 
 		size = 1024;
 		buf = umem_alloc(size, UMEM_NOFAIL);
 		i = snprintf(buf, size, "%s:%d:%s(): ", newfile, line, func);
 
 		if (i < size) {
 			va_start(adx, fmt);
 			(void) vsnprintf(buf + i, size - i, fmt, adx);
 			va_end(adx);
 		}
 
 		__zfs_dbgmsg(buf);
 
 		umem_free(buf, size);
 	}
 }
 
 /*
  * =========================================================================
  * cmn_err() and panic()
  * =========================================================================
  */
 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
 
 __attribute__((noreturn)) void
 vpanic(const char *fmt, va_list adx)
 {
 	(void) fprintf(stderr, "error: ");
 	(void) vfprintf(stderr, fmt, adx);
 	(void) fprintf(stderr, "\n");
 
 	abort();	/* think of it as a "user-level crash dump" */
 }
 
 __attribute__((noreturn)) void
 panic(const char *fmt, ...)
 {
 	va_list adx;
 
 	va_start(adx, fmt);
 	vpanic(fmt, adx);
 	va_end(adx);
 }
 
 void
 vcmn_err(int ce, const char *fmt, va_list adx)
 {
 	if (ce == CE_PANIC)
 		vpanic(fmt, adx);
 	if (ce != CE_NOTE) {	/* suppress noise in userland stress testing */
 		(void) fprintf(stderr, "%s", ce_prefix[ce]);
 		(void) vfprintf(stderr, fmt, adx);
 		(void) fprintf(stderr, "%s", ce_suffix[ce]);
 	}
 }
 
 void
 cmn_err(int ce, const char *fmt, ...)
 {
 	va_list adx;
 
 	va_start(adx, fmt);
 	vcmn_err(ce, fmt, adx);
 	va_end(adx);
 }
 
 /*
  * =========================================================================
  * misc routines
  * =========================================================================
  */
 
 void
 delay(clock_t ticks)
 {
 	(void) poll(0, 0, ticks * (1000 / hz));
 }
 
 /*
  * Find highest one bit set.
  * Returns bit number + 1 of highest bit that is set, otherwise returns 0.
  * The __builtin_clzll() function is supported by both GCC and Clang.
  */
 int
 highbit64(uint64_t i)
 {
 	if (i == 0)
 	return (0);
 
 	return (NBBY * sizeof (uint64_t) - __builtin_clzll(i));
 }
 
 /*
  * Find lowest one bit set.
  * Returns bit number + 1 of lowest bit that is set, otherwise returns 0.
  * The __builtin_ffsll() function is supported by both GCC and Clang.
  */
 int
 lowbit64(uint64_t i)
 {
 	if (i == 0)
 		return (0);
 
 	return (__builtin_ffsll(i));
 }
 
 const char *random_path = "/dev/random";
 const char *urandom_path = "/dev/urandom";
 static int random_fd = -1, urandom_fd = -1;
 
 void
 random_init(void)
 {
 	VERIFY((random_fd = open(random_path, O_RDONLY | O_CLOEXEC)) != -1);
 	VERIFY((urandom_fd = open(urandom_path, O_RDONLY | O_CLOEXEC)) != -1);
 }
 
 void
 random_fini(void)
 {
 	close(random_fd);
 	close(urandom_fd);
 
 	random_fd = -1;
 	urandom_fd = -1;
 }
 
 static int
 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
 {
 	size_t resid = len;
 	ssize_t bytes;
 
 	ASSERT(fd != -1);
 
 	while (resid != 0) {
 		bytes = read(fd, ptr, resid);
 		ASSERT3S(bytes, >=, 0);
 		ptr += bytes;
 		resid -= bytes;
 	}
 
 	return (0);
 }
 
 int
 random_get_bytes(uint8_t *ptr, size_t len)
 {
 	return (random_get_bytes_common(ptr, len, random_fd));
 }
 
 int
 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
 {
 	return (random_get_bytes_common(ptr, len, urandom_fd));
 }
 
 int
 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result)
 {
 	errno = 0;
 	*result = strtoull(str, nptr, base);
 	if (*result == 0)
 		return (errno);
 	return (0);
 }
 
 utsname_t *
 utsname(void)
 {
 	return (&hw_utsname);
 }
 
 /*
  * =========================================================================
  * kernel emulation setup & teardown
  * =========================================================================
  */
 static int
 umem_out_of_memory(void)
 {
 	char errmsg[] = "out of memory -- generating core dump\n";
 
 	(void) fprintf(stderr, "%s", errmsg);
 	abort();
 	return (0);
 }
 
 void
 kernel_init(int mode)
 {
 	extern uint_t rrw_tsd_key;
 
 	umem_nofail_callback(umem_out_of_memory);
 
 	physmem = sysconf(_SC_PHYS_PAGES);
 
 	dprintf("physmem = %llu pages (%.2f GB)\n", (u_longlong_t)physmem,
 	    (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
 
 	hostid = (mode & SPA_MODE_WRITE) ? get_system_hostid() : 0;
 
 	random_init();
 
 	VERIFY0(uname(&hw_utsname));
 
 	system_taskq_init();
 	icp_init();
 
 	zstd_init();
 
 	spa_init((spa_mode_t)mode);
 
 	fletcher_4_init();
 
 	tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
 }
 
 void
 kernel_fini(void)
 {
 	fletcher_4_fini();
 	spa_fini();
 
 	zstd_fini();
 
 	icp_fini();
 	system_taskq_fini();
 
 	random_fini();
 }
 
 uid_t
 crgetuid(cred_t *cr)
 {
 	(void) cr;
 	return (0);
 }
 
 uid_t
 crgetruid(cred_t *cr)
 {
 	(void) cr;
 	return (0);
 }
 
 gid_t
 crgetgid(cred_t *cr)
 {
 	(void) cr;
 	return (0);
 }
 
 int
 crgetngroups(cred_t *cr)
 {
 	(void) cr;
 	return (0);
 }
 
 gid_t *
 crgetgroups(cred_t *cr)
 {
 	(void) cr;
 	return (NULL);
 }
 
 int
 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr)
 {
 	(void) name, (void) cr;
 	return (0);
 }
 
 int
 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr)
 {
 	(void) from, (void) to, (void) cr;
 	return (0);
 }
 
 int
 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr)
 {
 	(void) name, (void) cr;
 	return (0);
 }
 
 int
 secpolicy_zfs(const cred_t *cr)
 {
 	(void) cr;
 	return (0);
 }
 
-int
-secpolicy_zfs_proc(const cred_t *cr, proc_t *proc)
-{
-	(void) cr, (void) proc;
-	return (0);
-}
-
 ksiddomain_t *
 ksid_lookupdomain(const char *dom)
 {
 	ksiddomain_t *kd;
 
 	kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL);
 	kd->kd_name = spa_strdup(dom);
 	return (kd);
 }
 
 void
 ksiddomain_rele(ksiddomain_t *ksid)
 {
 	spa_strfree(ksid->kd_name);
 	umem_free(ksid, sizeof (ksiddomain_t));
 }
 
 char *
 kmem_vasprintf(const char *fmt, va_list adx)
 {
 	char *buf = NULL;
 	va_list adx_copy;
 
 	va_copy(adx_copy, adx);
 	VERIFY(vasprintf(&buf, fmt, adx_copy) != -1);
 	va_end(adx_copy);
 
 	return (buf);
 }
 
 char *
 kmem_asprintf(const char *fmt, ...)
 {
 	char *buf = NULL;
 	va_list adx;
 
 	va_start(adx, fmt);
 	VERIFY(vasprintf(&buf, fmt, adx) != -1);
 	va_end(adx);
 
 	return (buf);
 }
 
 /*
  * kmem_scnprintf() will return the number of characters that it would have
  * printed whenever it is limited by value of the size variable, rather than
  * the number of characters that it did print. This can cause misbehavior on
  * subsequent uses of the return value, so we define a safe version that will
  * return the number of characters actually printed, minus the NULL format
  * character.  Subsequent use of this by the safe string functions is safe
  * whether it is snprintf(), strlcat() or strlcpy().
  */
 int
 kmem_scnprintf(char *restrict str, size_t size, const char *restrict fmt, ...)
 {
 	int n;
 	va_list ap;
 
 	/* Make the 0 case a no-op so that we do not return -1 */
 	if (size == 0)
 		return (0);
 
 	va_start(ap, fmt);
 	n = vsnprintf(str, size, fmt, ap);
 	va_end(ap);
 
 	if (n >= size)
 		n = size - 1;
 
 	return (n);
 }
 
 zfs_file_t *
 zfs_onexit_fd_hold(int fd, minor_t *minorp)
 {
 	(void) fd;
 	*minorp = 0;
 	return (NULL);
 }
 
 void
 zfs_onexit_fd_rele(zfs_file_t *fp)
 {
 	(void) fp;
 }
 
 int
 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
     uintptr_t *action_handle)
 {
 	(void) minor, (void) func, (void) data, (void) action_handle;
 	return (0);
 }
 
 fstrans_cookie_t
 spl_fstrans_mark(void)
 {
 	return ((fstrans_cookie_t)0);
 }
 
 void
 spl_fstrans_unmark(fstrans_cookie_t cookie)
 {
 	(void) cookie;
 }
 
 int
 __spl_pf_fstrans_check(void)
 {
 	return (0);
 }
 
 int
 kmem_cache_reap_active(void)
 {
 	return (0);
 }
 
 void
 zvol_create_minor(const char *name)
 {
 	(void) name;
 }
 
 void
 zvol_create_minors_recursive(const char *name)
 {
 	(void) name;
 }
 
 void
 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async)
 {
 	(void) spa, (void) name, (void) async;
 }
 
 void
 zvol_rename_minors(spa_t *spa, const char *oldname, const char *newname,
     boolean_t async)
 {
 	(void) spa, (void) oldname, (void) newname, (void) async;
 }
 
 /*
  * Open file
  *
  * path - fully qualified path to file
  * flags - file attributes O_READ / O_WRITE / O_EXCL
  * fpp - pointer to return file pointer
  *
  * Returns 0 on success underlying error on failure.
  */
 int
 zfs_file_open(const char *path, int flags, int mode, zfs_file_t **fpp)
 {
 	int fd = -1;
 	int dump_fd = -1;
 	int err;
 	int old_umask = 0;
 	zfs_file_t *fp;
 	struct stat64 st;
 
 	if (!(flags & O_CREAT) && stat64(path, &st) == -1)
 		return (errno);
 
 	if (!(flags & O_CREAT) && S_ISBLK(st.st_mode))
 		flags |= O_DIRECT;
 
 	if (flags & O_CREAT)
 		old_umask = umask(0);
 
 	fd = open64(path, flags, mode);
 	if (fd == -1)
 		return (errno);
 
 	if (flags & O_CREAT)
 		(void) umask(old_umask);
 
 	if (vn_dumpdir != NULL) {
 		char *dumppath = umem_zalloc(MAXPATHLEN, UMEM_NOFAIL);
 		const char *inpath = zfs_basename(path);
 
 		(void) snprintf(dumppath, MAXPATHLEN,
 		    "%s/%s", vn_dumpdir, inpath);
 		dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666);
 		umem_free(dumppath, MAXPATHLEN);
 		if (dump_fd == -1) {
 			err = errno;
 			close(fd);
 			return (err);
 		}
 	} else {
 		dump_fd = -1;
 	}
 
 	(void) fcntl(fd, F_SETFD, FD_CLOEXEC);
 
 	fp = umem_zalloc(sizeof (zfs_file_t), UMEM_NOFAIL);
 	fp->f_fd = fd;
 	fp->f_dump_fd = dump_fd;
 	*fpp = fp;
 
 	return (0);
 }
 
 void
 zfs_file_close(zfs_file_t *fp)
 {
 	close(fp->f_fd);
 	if (fp->f_dump_fd != -1)
 		close(fp->f_dump_fd);
 
 	umem_free(fp, sizeof (zfs_file_t));
 }
 
 /*
  * Stateful write - use os internal file pointer to determine where to
  * write and update on successful completion.
  *
  * fp -  pointer to file (pipe, socket, etc) to write to
  * buf - buffer to write
  * count - # of bytes to write
  * resid -  pointer to count of unwritten bytes  (if short write)
  *
  * Returns 0 on success errno on failure.
  */
 int
 zfs_file_write(zfs_file_t *fp, const void *buf, size_t count, ssize_t *resid)
 {
 	ssize_t rc;
 
 	rc = write(fp->f_fd, buf, count);
 	if (rc < 0)
 		return (errno);
 
 	if (resid) {
 		*resid = count - rc;
 	} else if (rc != count) {
 		return (EIO);
 	}
 
 	return (0);
 }
 
 /*
  * Stateless write - os internal file pointer is not updated.
  *
  * fp -  pointer to file (pipe, socket, etc) to write to
  * buf - buffer to write
  * count - # of bytes to write
  * off - file offset to write to (only valid for seekable types)
  * resid -  pointer to count of unwritten bytes
  *
  * Returns 0 on success errno on failure.
  */
 int
 zfs_file_pwrite(zfs_file_t *fp, const void *buf,
     size_t count, loff_t pos, ssize_t *resid)
 {
 	ssize_t rc, split, done;
 	int sectors;
 
 	/*
 	 * To simulate partial disk writes, we split writes into two
 	 * system calls so that the process can be killed in between.
 	 * This is used by ztest to simulate realistic failure modes.
 	 */
 	sectors = count >> SPA_MINBLOCKSHIFT;
 	split = (sectors > 0 ? rand() % sectors : 0) << SPA_MINBLOCKSHIFT;
 	rc = pwrite64(fp->f_fd, buf, split, pos);
 	if (rc != -1) {
 		done = rc;
 		rc = pwrite64(fp->f_fd, (char *)buf + split,
 		    count - split, pos + split);
 	}
 #ifdef __linux__
 	if (rc == -1 && errno == EINVAL) {
 		/*
 		 * Under Linux, this most likely means an alignment issue
 		 * (memory or disk) due to O_DIRECT, so we abort() in order
 		 * to catch the offender.
 		 */
 		abort();
 	}
 #endif
 
 	if (rc < 0)
 		return (errno);
 
 	done += rc;
 
 	if (resid) {
 		*resid = count - done;
 	} else if (done != count) {
 		return (EIO);
 	}
 
 	return (0);
 }
 
 /*
  * Stateful read - use os internal file pointer to determine where to
  * read and update on successful completion.
  *
  * fp -  pointer to file (pipe, socket, etc) to read from
  * buf - buffer to write
  * count - # of bytes to read
  * resid -  pointer to count of unread bytes (if short read)
  *
  * Returns 0 on success errno on failure.
  */
 int
 zfs_file_read(zfs_file_t *fp, void *buf, size_t count, ssize_t *resid)
 {
 	int rc;
 
 	rc = read(fp->f_fd, buf, count);
 	if (rc < 0)
 		return (errno);
 
 	if (resid) {
 		*resid = count - rc;
 	} else if (rc != count) {
 		return (EIO);
 	}
 
 	return (0);
 }
 
 /*
  * Stateless read - os internal file pointer is not updated.
  *
  * fp -  pointer to file (pipe, socket, etc) to read from
  * buf - buffer to write
  * count - # of bytes to write
  * off - file offset to read from (only valid for seekable types)
  * resid -  pointer to count of unwritten bytes (if short write)
  *
  * Returns 0 on success errno on failure.
  */
 int
 zfs_file_pread(zfs_file_t *fp, void *buf, size_t count, loff_t off,
     ssize_t *resid)
 {
 	ssize_t rc;
 
 	rc = pread64(fp->f_fd, buf, count, off);
 	if (rc < 0) {
 #ifdef __linux__
 		/*
 		 * Under Linux, this most likely means an alignment issue
 		 * (memory or disk) due to O_DIRECT, so we abort() in order to
 		 * catch the offender.
 		 */
 		if (errno == EINVAL)
 			abort();
 #endif
 		return (errno);
 	}
 
 	if (fp->f_dump_fd != -1) {
 		int status;
 
 		status = pwrite64(fp->f_dump_fd, buf, rc, off);
 		ASSERT(status != -1);
 	}
 
 	if (resid) {
 		*resid = count - rc;
 	} else if (rc != count) {
 		return (EIO);
 	}
 
 	return (0);
 }
 
 /*
  * lseek - set / get file pointer
  *
  * fp -  pointer to file (pipe, socket, etc) to read from
  * offp - value to seek to, returns current value plus passed offset
  * whence - see man pages for standard lseek whence values
  *
  * Returns 0 on success errno on failure (ESPIPE for non seekable types)
  */
 int
 zfs_file_seek(zfs_file_t *fp, loff_t *offp, int whence)
 {
 	loff_t rc;
 
 	rc = lseek(fp->f_fd, *offp, whence);
 	if (rc < 0)
 		return (errno);
 
 	*offp = rc;
 
 	return (0);
 }
 
 /*
  * Get file attributes
  *
  * filp - file pointer
  * zfattr - pointer to file attr structure
  *
  * Currently only used for fetching size and file mode
  *
  * Returns 0 on success or error code of underlying getattr call on failure.
  */
 int
 zfs_file_getattr(zfs_file_t *fp, zfs_file_attr_t *zfattr)
 {
 	struct stat64 st;
 
 	if (fstat64_blk(fp->f_fd, &st) == -1)
 		return (errno);
 
 	zfattr->zfa_size = st.st_size;
 	zfattr->zfa_mode = st.st_mode;
 
 	return (0);
 }
 
 /*
  * Sync file to disk
  *
  * filp - file pointer
  * flags - O_SYNC and or O_DSYNC
  *
  * Returns 0 on success or error code of underlying sync call on failure.
  */
 int
 zfs_file_fsync(zfs_file_t *fp, int flags)
 {
 	(void) flags;
 
 	if (fsync(fp->f_fd) < 0)
 		return (errno);
 
 	return (0);
 }
 
 /*
  * deallocate - zero and/or deallocate file storage
  *
  * fp - file pointer
  * offset - offset to start zeroing or deallocating
  * len - length to zero or deallocate
  */
 int
 zfs_file_deallocate(zfs_file_t *fp, loff_t offset, loff_t len)
 {
 	int rc;
 #if defined(__linux__)
 	rc = fallocate(fp->f_fd,
 	    FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, offset, len);
 #else
 	(void) fp, (void) offset, (void) len;
 	rc = EOPNOTSUPP;
 #endif
 	if (rc)
 		return (SET_ERROR(rc));
 	return (0);
 }
 
 /*
  * Request current file pointer offset
  *
  * fp - pointer to file
  *
  * Returns current file offset.
  */
 loff_t
 zfs_file_off(zfs_file_t *fp)
 {
 	return (lseek(fp->f_fd, SEEK_CUR, 0));
 }
 
 /*
  * unlink file
  *
  * path - fully qualified file path
  *
  * Returns 0 on success.
  *
  * OPTIONAL
  */
 int
 zfs_file_unlink(const char *path)
 {
 	return (remove(path));
 }
 
 /*
  * Get reference to file pointer
  *
  * fd - input file descriptor
  *
  * Returns pointer to file struct or NULL.
  * Unsupported in user space.
  */
 zfs_file_t *
 zfs_file_get(int fd)
 {
 	(void) fd;
 	abort();
 	return (NULL);
 }
 /*
  * Drop reference to file pointer
  *
  * fp - pointer to file struct
  *
  * Unsupported in user space.
  */
 void
 zfs_file_put(zfs_file_t *fp)
 {
 	abort();
 	(void) fp;
 }
 
 void
 zfsvfs_update_fromname(const char *oldname, const char *newname)
 {
 	(void) oldname, (void) newname;
 }
 
 void
 spa_import_os(spa_t *spa)
 {
 	(void) spa;
 }
 
 void
 spa_export_os(spa_t *spa)
 {
 	(void) spa;
 }
 
 void
 spa_activate_os(spa_t *spa)
 {
 	(void) spa;
 }
 
 void
 spa_deactivate_os(spa_t *spa)
 {
 	(void) spa;
 }
diff --git a/sys/contrib/openzfs/man/man4/zfs.4 b/sys/contrib/openzfs/man/man4/zfs.4
index 9d3fc1055732..1d734d865a16 100644
--- a/sys/contrib/openzfs/man/man4/zfs.4
+++ b/sys/contrib/openzfs/man/man4/zfs.4
@@ -1,2701 +1,2690 @@
 .\"
 .\" Copyright (c) 2013 by Turbo Fredriksson <turbo@bayour.com>. All rights reserved.
 .\" Copyright (c) 2019, 2021 by Delphix. All rights reserved.
 .\" Copyright (c) 2019 Datto Inc.
 .\" Copyright (c) 2023, 2024 Klara, Inc.
 .\" The contents of this file are subject to the terms of the Common Development
 .\" and Distribution License (the "License").  You may not use this file except
 .\" in compliance with the License. You can obtain a copy of the license at
 .\" usr/src/OPENSOLARIS.LICENSE or https://opensource.org/licenses/CDDL-1.0.
 .\"
 .\" See the License for the specific language governing permissions and
 .\" limitations under the License. When distributing Covered Code, include this
 .\" CDDL HEADER in each file and include the License file at
 .\" usr/src/OPENSOLARIS.LICENSE.  If applicable, add the following below this
 .\" CDDL HEADER, with the fields enclosed by brackets "[]" replaced with your
 .\" own identifying information:
 .\" Portions Copyright [yyyy] [name of copyright owner]
 .\"
 .Dd January 9, 2024
 .Dt ZFS 4
 .Os
 .
 .Sh NAME
 .Nm zfs
 .Nd tuning of the ZFS kernel module
 .
 .Sh DESCRIPTION
 The ZFS module supports these parameters:
 .Bl -tag -width Ds
 .It Sy dbuf_cache_max_bytes Ns = Ns Sy UINT64_MAX Ns B Pq u64
 Maximum size in bytes of the dbuf cache.
 The target size is determined by the MIN versus
 .No 1/2^ Ns Sy dbuf_cache_shift Pq 1/32nd
 of the target ARC size.
 The behavior of the dbuf cache and its associated settings
 can be observed via the
 .Pa /proc/spl/kstat/zfs/dbufstats
 kstat.
 .
 .It Sy dbuf_metadata_cache_max_bytes Ns = Ns Sy UINT64_MAX Ns B Pq u64
 Maximum size in bytes of the metadata dbuf cache.
 The target size is determined by the MIN versus
 .No 1/2^ Ns Sy dbuf_metadata_cache_shift Pq 1/64th
 of the target ARC size.
 The behavior of the metadata dbuf cache and its associated settings
 can be observed via the
 .Pa /proc/spl/kstat/zfs/dbufstats
 kstat.
 .
 .It Sy dbuf_cache_hiwater_pct Ns = Ns Sy 10 Ns % Pq uint
 The percentage over
 .Sy dbuf_cache_max_bytes
 when dbufs must be evicted directly.
 .
 .It Sy dbuf_cache_lowater_pct Ns = Ns Sy 10 Ns % Pq uint
 The percentage below
 .Sy dbuf_cache_max_bytes
 when the evict thread stops evicting dbufs.
 .
 .It Sy dbuf_cache_shift Ns = Ns Sy 5 Pq uint
 Set the size of the dbuf cache
 .Pq Sy dbuf_cache_max_bytes
 to a log2 fraction of the target ARC size.
 .
 .It Sy dbuf_metadata_cache_shift Ns = Ns Sy 6 Pq uint
 Set the size of the dbuf metadata cache
 .Pq Sy dbuf_metadata_cache_max_bytes
 to a log2 fraction of the target ARC size.
 .
 .It Sy dbuf_mutex_cache_shift Ns = Ns Sy 0 Pq uint
 Set the size of the mutex array for the dbuf cache.
 When set to
 .Sy 0
 the array is dynamically sized based on total system memory.
 .
 .It Sy dmu_object_alloc_chunk_shift Ns = Ns Sy 7 Po 128 Pc Pq uint
 dnode slots allocated in a single operation as a power of 2.
 The default value minimizes lock contention for the bulk operation performed.
 .
 .It Sy dmu_prefetch_max Ns = Ns Sy 134217728 Ns B Po 128 MiB Pc Pq uint
 Limit the amount we can prefetch with one call to this amount in bytes.
 This helps to limit the amount of memory that can be used by prefetching.
 .
 .It Sy ignore_hole_birth Pq int
 Alias for
 .Sy send_holes_without_birth_time .
 .
 .It Sy l2arc_feed_again Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Turbo L2ARC warm-up.
 When the L2ARC is cold the fill interval will be set as fast as possible.
 .
 .It Sy l2arc_feed_min_ms Ns = Ns Sy 200 Pq u64
 Min feed interval in milliseconds.
 Requires
 .Sy l2arc_feed_again Ns = Ns Ar 1
 and only applicable in related situations.
 .
 .It Sy l2arc_feed_secs Ns = Ns Sy 1 Pq u64
 Seconds between L2ARC writing.
 .
 .It Sy l2arc_headroom Ns = Ns Sy 8 Pq u64
 How far through the ARC lists to search for L2ARC cacheable content,
 expressed as a multiplier of
 .Sy l2arc_write_max .
 ARC persistence across reboots can be achieved with persistent L2ARC
 by setting this parameter to
 .Sy 0 ,
 allowing the full length of ARC lists to be searched for cacheable content.
 .
 .It Sy l2arc_headroom_boost Ns = Ns Sy 200 Ns % Pq u64
 Scales
 .Sy l2arc_headroom
 by this percentage when L2ARC contents are being successfully compressed
 before writing.
 A value of
 .Sy 100
 disables this feature.
 .
 .It Sy l2arc_exclude_special Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Controls whether buffers present on special vdevs are eligible for caching
 into L2ARC.
 If set to 1, exclude dbufs on special vdevs from being cached to L2ARC.
 .
 .It Sy l2arc_mfuonly Ns = Ns Sy 0 Ns | Ns 1 Ns | Ns 2 Pq int
 Controls whether only MFU metadata and data are cached from ARC into L2ARC.
 This may be desired to avoid wasting space on L2ARC when reading/writing large
 amounts of data that are not expected to be accessed more than once.
 .Pp
 The default is 0,
 meaning both MRU and MFU data and metadata are cached.
 When turning off this feature (setting it to 0), some MRU buffers will
 still be present in ARC and eventually cached on L2ARC.
 .No If Sy l2arc_noprefetch Ns = Ns Sy 0 ,
 some prefetched buffers will be cached to L2ARC, and those might later
 transition to MRU, in which case the
 .Sy l2arc_mru_asize No arcstat will not be Sy 0 .
 .Pp
 Setting it to 1 means to L2 cache only MFU data and metadata.
 .Pp
 Setting it to 2 means to L2 cache all metadata (MRU+MFU) but
 only MFU data (ie: MRU data are not cached). This can be the right setting
 to cache as much metadata as possible even when having high data turnover.
 .Pp
 Regardless of
 .Sy l2arc_noprefetch ,
 some MFU buffers might be evicted from ARC,
 accessed later on as prefetches and transition to MRU as prefetches.
 If accessed again they are counted as MRU and the
 .Sy l2arc_mru_asize No arcstat will not be Sy 0 .
 .Pp
 The ARC status of L2ARC buffers when they were first cached in
 L2ARC can be seen in the
 .Sy l2arc_mru_asize , Sy l2arc_mfu_asize , No and Sy l2arc_prefetch_asize
 arcstats when importing the pool or onlining a cache
 device if persistent L2ARC is enabled.
 .Pp
 The
 .Sy evict_l2_eligible_mru
 arcstat does not take into account if this option is enabled as the information
 provided by the
 .Sy evict_l2_eligible_m[rf]u
 arcstats can be used to decide if toggling this option is appropriate
 for the current workload.
 .
 .It Sy l2arc_meta_percent Ns = Ns Sy 33 Ns % Pq uint
 Percent of ARC size allowed for L2ARC-only headers.
 Since L2ARC buffers are not evicted on memory pressure,
 too many headers on a system with an irrationally large L2ARC
 can render it slow or unusable.
 This parameter limits L2ARC writes and rebuilds to achieve the target.
 .
 .It Sy l2arc_trim_ahead Ns = Ns Sy 0 Ns % Pq u64
 Trims ahead of the current write size
 .Pq Sy l2arc_write_max
 on L2ARC devices by this percentage of write size if we have filled the device.
 If set to
 .Sy 100
 we TRIM twice the space required to accommodate upcoming writes.
 A minimum of
 .Sy 64 MiB
 will be trimmed.
 It also enables TRIM of the whole L2ARC device upon creation
 or addition to an existing pool or if the header of the device is
 invalid upon importing a pool or onlining a cache device.
 A value of
 .Sy 0
 disables TRIM on L2ARC altogether and is the default as it can put significant
 stress on the underlying storage devices.
 This will vary depending of how well the specific device handles these commands.
 .
 .It Sy l2arc_noprefetch Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Do not write buffers to L2ARC if they were prefetched but not used by
 applications.
 In case there are prefetched buffers in L2ARC and this option
 is later set, we do not read the prefetched buffers from L2ARC.
 Unsetting this option is useful for caching sequential reads from the
 disks to L2ARC and serve those reads from L2ARC later on.
 This may be beneficial in case the L2ARC device is significantly faster
 in sequential reads than the disks of the pool.
 .Pp
 Use
 .Sy 1
 to disable and
 .Sy 0
 to enable caching/reading prefetches to/from L2ARC.
 .
 .It Sy l2arc_norw Ns = Ns Sy 0 Ns | Ns 1 Pq int
 No reads during writes.
 .
 .It Sy l2arc_write_boost Ns = Ns Sy 33554432 Ns B Po 32 MiB Pc Pq u64
 Cold L2ARC devices will have
 .Sy l2arc_write_max
 increased by this amount while they remain cold.
 .
 .It Sy l2arc_write_max Ns = Ns Sy 33554432 Ns B Po 32 MiB Pc Pq u64
 Max write bytes per interval.
 .
 .It Sy l2arc_rebuild_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Rebuild the L2ARC when importing a pool (persistent L2ARC).
 This can be disabled if there are problems importing a pool
 or attaching an L2ARC device (e.g. the L2ARC device is slow
 in reading stored log metadata, or the metadata
 has become somehow fragmented/unusable).
 .
 .It Sy l2arc_rebuild_blocks_min_l2size Ns = Ns Sy 1073741824 Ns B Po 1 GiB Pc Pq u64
 Mininum size of an L2ARC device required in order to write log blocks in it.
 The log blocks are used upon importing the pool to rebuild the persistent L2ARC.
 .Pp
 For L2ARC devices less than 1 GiB, the amount of data
 .Fn l2arc_evict
 evicts is significant compared to the amount of restored L2ARC data.
 In this case, do not write log blocks in L2ARC in order not to waste space.
 .
 .It Sy metaslab_aliquot Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
 Metaslab granularity, in bytes.
 This is roughly similar to what would be referred to as the "stripe size"
 in traditional RAID arrays.
 In normal operation, ZFS will try to write this amount of data to each disk
 before moving on to the next top-level vdev.
 .
 .It Sy metaslab_bias_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Enable metaslab group biasing based on their vdevs' over- or under-utilization
 relative to the pool.
 .
 .It Sy metaslab_force_ganging Ns = Ns Sy 16777217 Ns B Po 16 MiB + 1 B Pc Pq u64
 Make some blocks above a certain size be gang blocks.
 This option is used by the test suite to facilitate testing.
 .
 .It Sy metaslab_force_ganging_pct Ns = Ns Sy 3 Ns % Pq uint
 For blocks that could be forced to be a gang block (due to
 .Sy metaslab_force_ganging ) ,
 force this many of them to be gang blocks.
 .
 .It Sy brt_zap_prefetch Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Controls prefetching BRT records for blocks which are going to be cloned.
 .
 .It Sy brt_zap_default_bs Ns = Ns Sy 12 Po 4 KiB Pc Pq int
 Default BRT ZAP data block size as a power of 2. Note that changing this after
 creating a BRT on the pool will not affect existing BRTs, only newly created
 ones.
 .
 .It Sy brt_zap_default_ibs Ns = Ns Sy 12 Po 4 KiB Pc Pq int
 Default BRT ZAP indirect block size as a power of 2. Note that changing this
 after creating a BRT on the pool will not affect existing BRTs, only newly
 created ones.
 .
 .It Sy ddt_zap_default_bs Ns = Ns Sy 15 Po 32 KiB Pc Pq int
 Default DDT ZAP data block size as a power of 2. Note that changing this after
 creating a DDT on the pool will not affect existing DDTs, only newly created
 ones.
 .
 .It Sy ddt_zap_default_ibs Ns = Ns Sy 15 Po 32 KiB Pc Pq int
 Default DDT ZAP indirect block size as a power of 2. Note that changing this
 after creating a DDT on the pool will not affect existing DDTs, only newly
 created ones.
 .
 .It Sy zfs_default_bs Ns = Ns Sy 9 Po 512 B Pc Pq int
 Default dnode block size as a power of 2.
 .
 .It Sy zfs_default_ibs Ns = Ns Sy 17 Po 128 KiB Pc Pq int
 Default dnode indirect block size as a power of 2.
 .
 .It Sy zfs_history_output_max Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
 When attempting to log an output nvlist of an ioctl in the on-disk history,
 the output will not be stored if it is larger than this size (in bytes).
 This must be less than
 .Sy DMU_MAX_ACCESS Pq 64 MiB .
 This applies primarily to
 .Fn zfs_ioc_channel_program Pq cf. Xr zfs-program 8 .
 .
 .It Sy zfs_keep_log_spacemaps_at_export Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Prevent log spacemaps from being destroyed during pool exports and destroys.
 .
 .It Sy zfs_metaslab_segment_weight_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Enable/disable segment-based metaslab selection.
 .
 .It Sy zfs_metaslab_switch_threshold Ns = Ns Sy 2 Pq int
 When using segment-based metaslab selection, continue allocating
 from the active metaslab until this option's
 worth of buckets have been exhausted.
 .
 .It Sy metaslab_debug_load Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Load all metaslabs during pool import.
 .
 .It Sy metaslab_debug_unload Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Prevent metaslabs from being unloaded.
 .
 .It Sy metaslab_fragmentation_factor_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Enable use of the fragmentation metric in computing metaslab weights.
 .
 .It Sy metaslab_df_max_search Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq uint
 Maximum distance to search forward from the last offset.
 Without this limit, fragmented pools can see
 .Em >100`000
 iterations and
 .Fn metaslab_block_picker
 becomes the performance limiting factor on high-performance storage.
 .Pp
 With the default setting of
 .Sy 16 MiB ,
 we typically see less than
 .Em 500
 iterations, even with very fragmented
 .Sy ashift Ns = Ns Sy 9
 pools.
 The maximum number of iterations possible is
 .Sy metaslab_df_max_search / 2^(ashift+1) .
 With the default setting of
 .Sy 16 MiB
 this is
 .Em 16*1024 Pq with Sy ashift Ns = Ns Sy 9
 or
 .Em 2*1024 Pq with Sy ashift Ns = Ns Sy 12 .
 .
 .It Sy metaslab_df_use_largest_segment Ns = Ns Sy 0 Ns | Ns 1 Pq int
 If not searching forward (due to
 .Sy metaslab_df_max_search , metaslab_df_free_pct ,
 .No or Sy metaslab_df_alloc_threshold ) ,
 this tunable controls which segment is used.
 If set, we will use the largest free segment.
 If unset, we will use a segment of at least the requested size.
 .
 .It Sy zfs_metaslab_max_size_cache_sec Ns = Ns Sy 3600 Ns s Po 1 hour Pc Pq u64
 When we unload a metaslab, we cache the size of the largest free chunk.
 We use that cached size to determine whether or not to load a metaslab
 for a given allocation.
 As more frees accumulate in that metaslab while it's unloaded,
 the cached max size becomes less and less accurate.
 After a number of seconds controlled by this tunable,
 we stop considering the cached max size and start
 considering only the histogram instead.
 .
 .It Sy zfs_metaslab_mem_limit Ns = Ns Sy 25 Ns % Pq uint
 When we are loading a new metaslab, we check the amount of memory being used
 to store metaslab range trees.
 If it is over a threshold, we attempt to unload the least recently used metaslab
 to prevent the system from clogging all of its memory with range trees.
 This tunable sets the percentage of total system memory that is the threshold.
 .
 .It Sy zfs_metaslab_try_hard_before_gang Ns = Ns Sy 0 Ns | Ns 1 Pq int
 .Bl -item -compact
 .It
 If unset, we will first try normal allocation.
 .It
 If that fails then we will do a gang allocation.
 .It
 If that fails then we will do a "try hard" gang allocation.
 .It
 If that fails then we will have a multi-layer gang block.
 .El
 .Pp
 .Bl -item -compact
 .It
 If set, we will first try normal allocation.
 .It
 If that fails then we will do a "try hard" allocation.
 .It
 If that fails we will do a gang allocation.
 .It
 If that fails we will do a "try hard" gang allocation.
 .It
 If that fails then we will have a multi-layer gang block.
 .El
 .
 .It Sy zfs_metaslab_find_max_tries Ns = Ns Sy 100 Pq uint
 When not trying hard, we only consider this number of the best metaslabs.
 This improves performance, especially when there are many metaslabs per vdev
 and the allocation can't actually be satisfied
 (so we would otherwise iterate all metaslabs).
 .
 .It Sy zfs_vdev_default_ms_count Ns = Ns Sy 200 Pq uint
 When a vdev is added, target this number of metaslabs per top-level vdev.
 .
 .It Sy zfs_vdev_default_ms_shift Ns = Ns Sy 29 Po 512 MiB Pc Pq uint
 Default lower limit for metaslab size.
 .
 .It Sy zfs_vdev_max_ms_shift Ns = Ns Sy 34 Po 16 GiB Pc Pq uint
 Default upper limit for metaslab size.
 .
 .It Sy zfs_vdev_max_auto_ashift Ns = Ns Sy 14 Pq uint
 Maximum ashift used when optimizing for logical \[->] physical sector size on
 new
 top-level vdevs.
 May be increased up to
 .Sy ASHIFT_MAX Po 16 Pc ,
 but this may negatively impact pool space efficiency.
 .
 .It Sy zfs_vdev_min_auto_ashift Ns = Ns Sy ASHIFT_MIN Po 9 Pc Pq uint
 Minimum ashift used when creating new top-level vdevs.
 .
 .It Sy zfs_vdev_min_ms_count Ns = Ns Sy 16 Pq uint
 Minimum number of metaslabs to create in a top-level vdev.
 .
 .It Sy vdev_validate_skip Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Skip label validation steps during pool import.
 Changing is not recommended unless you know what you're doing
 and are recovering a damaged label.
 .
 .It Sy zfs_vdev_ms_count_limit Ns = Ns Sy 131072 Po 128k Pc Pq uint
 Practical upper limit of total metaslabs per top-level vdev.
 .
 .It Sy metaslab_preload_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Enable metaslab group preloading.
 .
 .It Sy metaslab_preload_limit Ns = Ns Sy 10 Pq uint
 Maximum number of metaslabs per group to preload
 .
 .It Sy metaslab_preload_pct Ns = Ns Sy 50 Pq uint
 Percentage of CPUs to run a metaslab preload taskq
 .
 .It Sy metaslab_lba_weighting_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Give more weight to metaslabs with lower LBAs,
 assuming they have greater bandwidth,
 as is typically the case on a modern constant angular velocity disk drive.
 .
 .It Sy metaslab_unload_delay Ns = Ns Sy 32 Pq uint
 After a metaslab is used, we keep it loaded for this many TXGs, to attempt to
 reduce unnecessary reloading.
 Note that both this many TXGs and
 .Sy metaslab_unload_delay_ms
 milliseconds must pass before unloading will occur.
 .
 .It Sy metaslab_unload_delay_ms Ns = Ns Sy 600000 Ns ms Po 10 min Pc Pq uint
 After a metaslab is used, we keep it loaded for this many milliseconds,
 to attempt to reduce unnecessary reloading.
 Note, that both this many milliseconds and
 .Sy metaslab_unload_delay
 TXGs must pass before unloading will occur.
 .
 .It Sy reference_history Ns = Ns Sy 3 Pq uint
 Maximum reference holders being tracked when reference_tracking_enable is
 active.
 .
 .It Sy reference_tracking_enable Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Track reference holders to
 .Sy refcount_t
 objects (debug builds only).
 .
 .It Sy send_holes_without_birth_time Ns = Ns Sy 1 Ns | Ns 0 Pq int
 When set, the
 .Sy hole_birth
 optimization will not be used, and all holes will always be sent during a
 .Nm zfs Cm send .
 This is useful if you suspect your datasets are affected by a bug in
 .Sy hole_birth .
 .
 .It Sy spa_config_path Ns = Ns Pa /etc/zfs/zpool.cache Pq charp
 SPA config file.
 .
 .It Sy spa_asize_inflation Ns = Ns Sy 24 Pq uint
 Multiplication factor used to estimate actual disk consumption from the
 size of data being written.
 The default value is a worst case estimate,
 but lower values may be valid for a given pool depending on its configuration.
 Pool administrators who understand the factors involved
 may wish to specify a more realistic inflation factor,
 particularly if they operate close to quota or capacity limits.
 .
 .It Sy spa_load_print_vdev_tree Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Whether to print the vdev tree in the debugging message buffer during pool
 import.
 .
 .It Sy spa_load_verify_data Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Whether to traverse data blocks during an "extreme rewind"
 .Pq Fl X
 import.
 .Pp
 An extreme rewind import normally performs a full traversal of all
 blocks in the pool for verification.
 If this parameter is unset, the traversal skips non-metadata blocks.
 It can be toggled once the
 import has started to stop or start the traversal of non-metadata blocks.
 .
 .It Sy spa_load_verify_metadata  Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Whether to traverse blocks during an "extreme rewind"
 .Pq Fl X
 pool import.
 .Pp
 An extreme rewind import normally performs a full traversal of all
 blocks in the pool for verification.
 If this parameter is unset, the traversal is not performed.
 It can be toggled once the import has started to stop or start the traversal.
 .
 .It Sy spa_load_verify_shift Ns = Ns Sy 4 Po 1/16th Pc Pq uint
 Sets the maximum number of bytes to consume during pool import to the log2
 fraction of the target ARC size.
 .
 .It Sy spa_slop_shift Ns = Ns Sy 5 Po 1/32nd Pc Pq int
 Normally, we don't allow the last
 .Sy 3.2% Pq Sy 1/2^spa_slop_shift
 of space in the pool to be consumed.
 This ensures that we don't run the pool completely out of space,
 due to unaccounted changes (e.g. to the MOS).
 It also limits the worst-case time to allocate space.
 If we have less than this amount of free space,
 most ZPL operations (e.g. write, create) will return
 .Sy ENOSPC .
 .
 .It Sy spa_upgrade_errlog_limit Ns = Ns Sy 0 Pq uint
 Limits the number of on-disk error log entries that will be converted to the
 new format when enabling the
 .Sy head_errlog
 feature.
 The default is to convert all log entries.
 .
 .It Sy vdev_removal_max_span Ns = Ns Sy 32768 Ns B Po 32 KiB Pc Pq uint
 During top-level vdev removal, chunks of data are copied from the vdev
 which may include free space in order to trade bandwidth for IOPS.
 This parameter determines the maximum span of free space, in bytes,
 which will be included as "unnecessary" data in a chunk of copied data.
 .Pp
 The default value here was chosen to align with
 .Sy zfs_vdev_read_gap_limit ,
 which is a similar concept when doing
 regular reads (but there's no reason it has to be the same).
 .
 .It Sy vdev_file_logical_ashift Ns = Ns Sy 9 Po 512 B Pc Pq u64
 Logical ashift for file-based devices.
 .
 .It Sy vdev_file_physical_ashift Ns = Ns Sy 9 Po 512 B Pc Pq u64
 Physical ashift for file-based devices.
 .
 .It Sy zap_iterate_prefetch Ns = Ns Sy 1 Ns | Ns 0 Pq int
 If set, when we start iterating over a ZAP object,
 prefetch the entire object (all leaf blocks).
 However, this is limited by
 .Sy dmu_prefetch_max .
 .
 .It Sy zap_micro_max_size Ns = Ns Sy 131072 Ns B Po 128 KiB Pc Pq int
 Maximum micro ZAP size.
 A micro ZAP is upgraded to a fat ZAP, once it grows beyond the specified size.
 .
 .It Sy zfetch_hole_shift Ns = Ns Sy 2 Pq uint
 Log2 fraction of holes in speculative prefetch stream allowed for it to
 proceed.
 .
 .It Sy zfetch_min_distance Ns = Ns Sy 4194304 Ns B Po 4 MiB Pc Pq uint
 Min bytes to prefetch per stream.
 Prefetch distance starts from the demand access size and quickly grows to
 this value, doubling on each hit.
 After that it may grow further by 1/8 per hit, but only if some prefetch
 since last time haven't completed in time to satisfy demand request, i.e.
 prefetch depth didn't cover the read latency or the pool got saturated.
 .
 .It Sy zfetch_max_distance Ns = Ns Sy 67108864 Ns B Po 64 MiB Pc Pq uint
 Max bytes to prefetch per stream.
 .
 .It Sy zfetch_max_idistance Ns = Ns Sy 67108864 Ns B Po 64 MiB Pc Pq uint
 Max bytes to prefetch indirects for per stream.
 .
 .It Sy zfetch_max_reorder Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq uint
 Requests within this byte distance from the current prefetch stream position
 are considered parts of the stream, reordered due to parallel processing.
 Such requests do not advance the stream position immediately unless
 .Sy zfetch_hole_shift
 fill threshold is reached, but saved to fill holes in the stream later.
 .
 .It Sy zfetch_max_streams Ns = Ns Sy 8 Pq uint
 Max number of streams per zfetch (prefetch streams per file).
 .
 .It Sy zfetch_min_sec_reap Ns = Ns Sy 1 Pq uint
 Min time before inactive prefetch stream can be reclaimed
 .
 .It Sy zfetch_max_sec_reap Ns = Ns Sy 2 Pq uint
 Max time before inactive prefetch stream can be deleted
 .
 .It Sy zfs_abd_scatter_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Enables ARC from using scatter/gather lists and forces all allocations to be
 linear in kernel memory.
 Disabling can improve performance in some code paths
 at the expense of fragmented kernel memory.
 .
 .It Sy zfs_abd_scatter_max_order Ns = Ns Sy MAX_ORDER\-1 Pq uint
 Maximum number of consecutive memory pages allocated in a single block for
 scatter/gather lists.
 .Pp
 The value of
 .Sy MAX_ORDER
 depends on kernel configuration.
 .
 .It Sy zfs_abd_scatter_min_size Ns = Ns Sy 1536 Ns B Po 1.5 KiB Pc Pq uint
 This is the minimum allocation size that will use scatter (page-based) ABDs.
 Smaller allocations will use linear ABDs.
 .
 .It Sy zfs_arc_dnode_limit Ns = Ns Sy 0 Ns B Pq u64
 When the number of bytes consumed by dnodes in the ARC exceeds this number of
 bytes, try to unpin some of it in response to demand for non-metadata.
 This value acts as a ceiling to the amount of dnode metadata, and defaults to
 .Sy 0 ,
 which indicates that a percent which is based on
 .Sy zfs_arc_dnode_limit_percent
 of the ARC meta buffers that may be used for dnodes.
 .It Sy zfs_arc_dnode_limit_percent Ns = Ns Sy 10 Ns % Pq u64
 Percentage that can be consumed by dnodes of ARC meta buffers.
 .Pp
 See also
 .Sy zfs_arc_dnode_limit ,
 which serves a similar purpose but has a higher priority if nonzero.
 .
 .It Sy zfs_arc_dnode_reduce_percent Ns = Ns Sy 10 Ns % Pq u64
 Percentage of ARC dnodes to try to scan in response to demand for non-metadata
 when the number of bytes consumed by dnodes exceeds
 .Sy zfs_arc_dnode_limit .
 .
 .It Sy zfs_arc_average_blocksize Ns = Ns Sy 8192 Ns B Po 8 KiB Pc Pq uint
 The ARC's buffer hash table is sized based on the assumption of an average
 block size of this value.
 This works out to roughly 1 MiB of hash table per 1 GiB of physical memory
 with 8-byte pointers.
 For configurations with a known larger average block size,
 this value can be increased to reduce the memory footprint.
 .
 .It Sy zfs_arc_eviction_pct Ns = Ns Sy 200 Ns % Pq uint
 When
 .Fn arc_is_overflowing ,
 .Fn arc_get_data_impl
 waits for this percent of the requested amount of data to be evicted.
 For example, by default, for every
 .Em 2 KiB
 that's evicted,
 .Em 1 KiB
 of it may be "reused" by a new allocation.
 Since this is above
 .Sy 100 Ns % ,
 it ensures that progress is made towards getting
 .Sy arc_size No under Sy arc_c .
 Since this is finite, it ensures that allocations can still happen,
 even during the potentially long time that
 .Sy arc_size No is more than Sy arc_c .
 .
 .It Sy zfs_arc_evict_batch_limit Ns = Ns Sy 10 Pq uint
 Number ARC headers to evict per sub-list before proceeding to another sub-list.
 This batch-style operation prevents entire sub-lists from being evicted at once
 but comes at a cost of additional unlocking and locking.
 .
 .It Sy zfs_arc_grow_retry Ns = Ns Sy 0 Ns s Pq uint
 If set to a non zero value, it will replace the
 .Sy arc_grow_retry
 value with this value.
 The
 .Sy arc_grow_retry
 .No value Pq default Sy 5 Ns s
 is the number of seconds the ARC will wait before
 trying to resume growth after a memory pressure event.
 .
 .It Sy zfs_arc_lotsfree_percent Ns = Ns Sy 10 Ns % Pq int
 Throttle I/O when free system memory drops below this percentage of total
 system memory.
 Setting this value to
 .Sy 0
 will disable the throttle.
 .
 .It Sy zfs_arc_max Ns = Ns Sy 0 Ns B Pq u64
 Max size of ARC in bytes.
 If
 .Sy 0 ,
 then the max size of ARC is determined by the amount of system memory installed.
 Under Linux, half of system memory will be used as the limit.
 Under
 .Fx ,
 the larger of
 .Sy all_system_memory No \- Sy 1 GiB
 and
 .Sy 5/8 No \(mu Sy all_system_memory
 will be used as the limit.
 This value must be at least
 .Sy 67108864 Ns B Pq 64 MiB .
 .Pp
 This value can be changed dynamically, with some caveats.
 It cannot be set back to
 .Sy 0
 while running, and reducing it below the current ARC size will not cause
 the ARC to shrink without memory pressure to induce shrinking.
 .
 .It Sy zfs_arc_meta_balance Ns = Ns Sy 500 Pq uint
 Balance between metadata and data on ghost hits.
 Values above 100 increase metadata caching by proportionally reducing effect
 of ghost data hits on target data/metadata rate.
 .
 .It Sy zfs_arc_min Ns = Ns Sy 0 Ns B Pq u64
 Min size of ARC in bytes.
 .No If set to Sy 0 , arc_c_min
 will default to consuming the larger of
 .Sy 32 MiB
 and
 .Sy all_system_memory No / Sy 32 .
 .
 .It Sy zfs_arc_min_prefetch_ms Ns = Ns Sy 0 Ns ms Ns Po Ns ≡ Ns 1s Pc Pq uint
 Minimum time prefetched blocks are locked in the ARC.
 .
 .It Sy zfs_arc_min_prescient_prefetch_ms Ns = Ns Sy 0 Ns ms Ns Po Ns ≡ Ns 6s Pc Pq uint
 Minimum time "prescient prefetched" blocks are locked in the ARC.
 These blocks are meant to be prefetched fairly aggressively ahead of
 the code that may use them.
 .
 .It Sy zfs_arc_prune_task_threads Ns = Ns Sy 1 Pq int
 Number of arc_prune threads.
 .Fx
 does not need more than one.
 Linux may theoretically use one per mount point up to number of CPUs,
 but that was not proven to be useful.
 .
 .It Sy zfs_max_missing_tvds Ns = Ns Sy 0 Pq int
 Number of missing top-level vdevs which will be allowed during
 pool import (only in read-only mode).
 .
 .It Sy zfs_max_nvlist_src_size Ns = Sy 0 Pq u64
 Maximum size in bytes allowed to be passed as
 .Sy zc_nvlist_src_size
 for ioctls on
 .Pa /dev/zfs .
 This prevents a user from causing the kernel to allocate
 an excessive amount of memory.
 When the limit is exceeded, the ioctl fails with
 .Sy EINVAL
 and a description of the error is sent to the
 .Pa zfs-dbgmsg
 log.
 This parameter should not need to be touched under normal circumstances.
 If
 .Sy 0 ,
 equivalent to a quarter of the user-wired memory limit under
 .Fx
 and to
 .Sy 134217728 Ns B Pq 128 MiB
 under Linux.
 .
 .It Sy zfs_multilist_num_sublists Ns = Ns Sy 0 Pq uint
 To allow more fine-grained locking, each ARC state contains a series
 of lists for both data and metadata objects.
 Locking is performed at the level of these "sub-lists".
 This parameters controls the number of sub-lists per ARC state,
 and also applies to other uses of the multilist data structure.
 .Pp
 If
 .Sy 0 ,
 equivalent to the greater of the number of online CPUs and
 .Sy 4 .
 .
 .It Sy zfs_arc_overflow_shift Ns = Ns Sy 8 Pq int
 The ARC size is considered to be overflowing if it exceeds the current
 ARC target size
 .Pq Sy arc_c
 by thresholds determined by this parameter.
 Exceeding by
 .Sy ( arc_c No >> Sy zfs_arc_overflow_shift ) No / Sy 2
 starts ARC reclamation process.
 If that appears insufficient, exceeding by
 .Sy ( arc_c No >> Sy zfs_arc_overflow_shift ) No \(mu Sy 1.5
 blocks new buffer allocation until the reclaim thread catches up.
 Started reclamation process continues till ARC size returns below the
 target size.
 .Pp
 The default value of
 .Sy 8
 causes the ARC to start reclamation if it exceeds the target size by
 .Em 0.2%
 of the target size, and block allocations by
 .Em 0.6% .
 .
 .It Sy zfs_arc_shrink_shift Ns = Ns Sy 0 Pq uint
 If nonzero, this will update
 .Sy arc_shrink_shift Pq default Sy 7
 with the new value.
 .
 .It Sy zfs_arc_pc_percent Ns = Ns Sy 0 Ns % Po off Pc Pq uint
 Percent of pagecache to reclaim ARC to.
 .Pp
 This tunable allows the ZFS ARC to play more nicely
 with the kernel's LRU pagecache.
 It can guarantee that the ARC size won't collapse under scanning
 pressure on the pagecache, yet still allows the ARC to be reclaimed down to
 .Sy zfs_arc_min
 if necessary.
 This value is specified as percent of pagecache size (as measured by
-.Sy NR_FILE_PAGES ) ,
+.Sy NR_ACTIVE_FILE
++
+.Sy NR_INACTIVE_FILE ) ,
 where that percent may exceed
 .Sy 100 .
 This
 only operates during memory pressure/reclaim.
 .
 .It Sy zfs_arc_shrinker_limit Ns = Ns Sy 10000 Pq int
 This is a limit on how many pages the ARC shrinker makes available for
 eviction in response to one page allocation attempt.
 Note that in practice, the kernel's shrinker can ask us to evict
 up to about four times this for one allocation attempt.
 .Pp
 The default limit of
 .Sy 10000 Pq in practice, Em 160 MiB No per allocation attempt with 4 KiB pages
 limits the amount of time spent attempting to reclaim ARC memory to
 less than 100 ms per allocation attempt,
 even with a small average compressed block size of ~8 KiB.
 .Pp
 The parameter can be set to 0 (zero) to disable the limit,
 and only applies on Linux.
 .
 .It Sy zfs_arc_sys_free Ns = Ns Sy 0 Ns B Pq u64
 The target number of bytes the ARC should leave as free memory on the system.
 If zero, equivalent to the bigger of
 .Sy 512 KiB No and Sy all_system_memory/64 .
 .
 .It Sy zfs_autoimport_disable Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Disable pool import at module load by ignoring the cache file
 .Pq Sy spa_config_path .
 .
 .It Sy zfs_checksum_events_per_second Ns = Ns Sy 20 Ns /s Pq uint
 Rate limit checksum events to this many per second.
 Note that this should not be set below the ZED thresholds
 (currently 10 checksums over 10 seconds)
 or else the daemon may not trigger any action.
 .
 .It Sy zfs_commit_timeout_pct Ns = Ns Sy 10 Ns % Pq uint
 This controls the amount of time that a ZIL block (lwb) will remain "open"
 when it isn't "full", and it has a thread waiting for it to be committed to
 stable storage.
 The timeout is scaled based on a percentage of the last lwb
 latency to avoid significantly impacting the latency of each individual
 transaction record (itx).
 .
 .It Sy zfs_condense_indirect_commit_entry_delay_ms Ns = Ns Sy 0 Ns ms Pq int
 Vdev indirection layer (used for device removal) sleeps for this many
 milliseconds during mapping generation.
 Intended for use with the test suite to throttle vdev removal speed.
 .
 .It Sy zfs_condense_indirect_obsolete_pct Ns = Ns Sy 25 Ns % Pq uint
 Minimum percent of obsolete bytes in vdev mapping required to attempt to
 condense
 .Pq see Sy zfs_condense_indirect_vdevs_enable .
 Intended for use with the test suite
 to facilitate triggering condensing as needed.
 .
 .It Sy zfs_condense_indirect_vdevs_enable Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Enable condensing indirect vdev mappings.
 When set, attempt to condense indirect vdev mappings
 if the mapping uses more than
 .Sy zfs_condense_min_mapping_bytes
 bytes of memory and if the obsolete space map object uses more than
 .Sy zfs_condense_max_obsolete_bytes
 bytes on-disk.
 The condensing process is an attempt to save memory by removing obsolete
 mappings.
 .
 .It Sy zfs_condense_max_obsolete_bytes Ns = Ns Sy 1073741824 Ns B Po 1 GiB Pc Pq u64
 Only attempt to condense indirect vdev mappings if the on-disk size
 of the obsolete space map object is greater than this number of bytes
 .Pq see Sy zfs_condense_indirect_vdevs_enable .
 .
 .It Sy zfs_condense_min_mapping_bytes Ns = Ns Sy 131072 Ns B Po 128 KiB Pc Pq u64
 Minimum size vdev mapping to attempt to condense
 .Pq see Sy zfs_condense_indirect_vdevs_enable .
 .
 .It Sy zfs_dbgmsg_enable Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Internally ZFS keeps a small log to facilitate debugging.
 The log is enabled by default, and can be disabled by unsetting this option.
 The contents of the log can be accessed by reading
 .Pa /proc/spl/kstat/zfs/dbgmsg .
 Writing
 .Sy 0
 to the file clears the log.
 .Pp
 This setting does not influence debug prints due to
 .Sy zfs_flags .
 .
 .It Sy zfs_dbgmsg_maxsize Ns = Ns Sy 4194304 Ns B Po 4 MiB Pc Pq uint
 Maximum size of the internal ZFS debug log.
 .
 .It Sy zfs_dbuf_state_index Ns = Ns Sy 0 Pq int
 Historically used for controlling what reporting was available under
 .Pa /proc/spl/kstat/zfs .
 No effect.
 .
 .It Sy zfs_deadman_checktime_ms Ns = Ns Sy 60000 Ns ms Po 1 min Pc Pq u64
 Check time in milliseconds.
 This defines the frequency at which we check for hung I/O requests
 and potentially invoke the
 .Sy zfs_deadman_failmode
 behavior.
 .
 .It Sy zfs_deadman_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 When a pool sync operation takes longer than
 .Sy zfs_deadman_synctime_ms ,
 or when an individual I/O operation takes longer than
 .Sy zfs_deadman_ziotime_ms ,
 then the operation is considered to be "hung".
 If
 .Sy zfs_deadman_enabled
 is set, then the deadman behavior is invoked as described by
 .Sy zfs_deadman_failmode .
 By default, the deadman is enabled and set to
 .Sy wait
 which results in "hung" I/O operations only being logged.
 The deadman is automatically disabled when a pool gets suspended.
 .
 .It Sy zfs_deadman_events_per_second Ns = Ns Sy 1 Ns /s Pq int
 Rate limit deadman zevents (which report hung I/O operations) to this many per
 second.
 .
 .It Sy zfs_deadman_failmode Ns = Ns Sy wait Pq charp
 Controls the failure behavior when the deadman detects a "hung" I/O operation.
 Valid values are:
 .Bl -tag -compact -offset 4n -width "continue"
 .It Sy wait
 Wait for a "hung" operation to complete.
 For each "hung" operation a "deadman" event will be posted
 describing that operation.
 .It Sy continue
 Attempt to recover from a "hung" operation by re-dispatching it
 to the I/O pipeline if possible.
 .It Sy panic
 Panic the system.
 This can be used to facilitate automatic fail-over
 to a properly configured fail-over partner.
 .El
 .
 .It Sy zfs_deadman_synctime_ms Ns = Ns Sy 600000 Ns ms Po 10 min Pc Pq u64
 Interval in milliseconds after which the deadman is triggered and also
 the interval after which a pool sync operation is considered to be "hung".
 Once this limit is exceeded the deadman will be invoked every
 .Sy zfs_deadman_checktime_ms
 milliseconds until the pool sync completes.
 .
 .It Sy zfs_deadman_ziotime_ms Ns = Ns Sy 300000 Ns ms Po 5 min Pc Pq u64
 Interval in milliseconds after which the deadman is triggered and an
 individual I/O operation is considered to be "hung".
 As long as the operation remains "hung",
 the deadman will be invoked every
 .Sy zfs_deadman_checktime_ms
 milliseconds until the operation completes.
 .
 .It Sy zfs_dedup_prefetch Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Enable prefetching dedup-ed blocks which are going to be freed.
 .
 .It Sy zfs_delay_min_dirty_percent Ns = Ns Sy 60 Ns % Pq uint
 Start to delay each transaction once there is this amount of dirty data,
 expressed as a percentage of
 .Sy zfs_dirty_data_max .
 This value should be at least
 .Sy zfs_vdev_async_write_active_max_dirty_percent .
 .No See Sx ZFS TRANSACTION DELAY .
 .
 .It Sy zfs_delay_scale Ns = Ns Sy 500000 Pq int
 This controls how quickly the transaction delay approaches infinity.
 Larger values cause longer delays for a given amount of dirty data.
 .Pp
 For the smoothest delay, this value should be about 1 billion divided
 by the maximum number of operations per second.
 This will smoothly handle between ten times and a tenth of this number.
 .No See Sx ZFS TRANSACTION DELAY .
 .Pp
 .Sy zfs_delay_scale No \(mu Sy zfs_dirty_data_max Em must No be smaller than Sy 2^64 .
 .
 .It Sy zfs_disable_ivset_guid_check Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Disables requirement for IVset GUIDs to be present and match when doing a raw
 receive of encrypted datasets.
 Intended for users whose pools were created with
 OpenZFS pre-release versions and now have compatibility issues.
 .
 .It Sy zfs_key_max_salt_uses Ns = Ns Sy 400000000 Po 4*10^8 Pc Pq ulong
 Maximum number of uses of a single salt value before generating a new one for
 encrypted datasets.
 The default value is also the maximum.
 .
 .It Sy zfs_object_mutex_size Ns = Ns Sy 64 Pq uint
 Size of the znode hashtable used for holds.
 .Pp
 Due to the need to hold locks on objects that may not exist yet, kernel mutexes
 are not created per-object and instead a hashtable is used where collisions
 will result in objects waiting when there is not actually contention on the
 same object.
 .
 .It Sy zfs_slow_io_events_per_second Ns = Ns Sy 20 Ns /s Pq int
 Rate limit delay zevents (which report slow I/O operations) to this many per
 second.
 .
 .It Sy zfs_unflushed_max_mem_amt Ns = Ns Sy 1073741824 Ns B Po 1 GiB Pc Pq u64
 Upper-bound limit for unflushed metadata changes to be held by the
 log spacemap in memory, in bytes.
 .
 .It Sy zfs_unflushed_max_mem_ppm Ns = Ns Sy 1000 Ns ppm Po 0.1% Pc Pq u64
 Part of overall system memory that ZFS allows to be used
 for unflushed metadata changes by the log spacemap, in millionths.
 .
 .It Sy zfs_unflushed_log_block_max Ns = Ns Sy 131072 Po 128k Pc Pq u64
 Describes the maximum number of log spacemap blocks allowed for each pool.
 The default value means that the space in all the log spacemaps
 can add up to no more than
 .Sy 131072
 blocks (which means
 .Em 16 GiB
 of logical space before compression and ditto blocks,
 assuming that blocksize is
 .Em 128 KiB ) .
 .Pp
 This tunable is important because it involves a trade-off between import
 time after an unclean export and the frequency of flushing metaslabs.
 The higher this number is, the more log blocks we allow when the pool is
 active which means that we flush metaslabs less often and thus decrease
 the number of I/O operations for spacemap updates per TXG.
 At the same time though, that means that in the event of an unclean export,
 there will be more log spacemap blocks for us to read, inducing overhead
 in the import time of the pool.
 The lower the number, the amount of flushing increases, destroying log
 blocks quicker as they become obsolete faster, which leaves less blocks
 to be read during import time after a crash.
 .Pp
 Each log spacemap block existing during pool import leads to approximately
 one extra logical I/O issued.
 This is the reason why this tunable is exposed in terms of blocks rather
 than space used.
 .
 .It Sy zfs_unflushed_log_block_min Ns = Ns Sy 1000 Pq u64
 If the number of metaslabs is small and our incoming rate is high,
 we could get into a situation that we are flushing all our metaslabs every TXG.
 Thus we always allow at least this many log blocks.
 .
 .It Sy zfs_unflushed_log_block_pct Ns = Ns Sy 400 Ns % Pq u64
 Tunable used to determine the number of blocks that can be used for
 the spacemap log, expressed as a percentage of the total number of
 unflushed metaslabs in the pool.
 .
 .It Sy zfs_unflushed_log_txg_max Ns = Ns Sy 1000 Pq u64
 Tunable limiting maximum time in TXGs any metaslab may remain unflushed.
 It effectively limits maximum number of unflushed per-TXG spacemap logs
 that need to be read after unclean pool export.
 .
 .It Sy zfs_unlink_suspend_progress Ns = Ns Sy 0 Ns | Ns 1 Pq uint
 When enabled, files will not be asynchronously removed from the list of pending
 unlinks and the space they consume will be leaked.
 Once this option has been disabled and the dataset is remounted,
 the pending unlinks will be processed and the freed space returned to the pool.
 This option is used by the test suite.
 .
 .It Sy zfs_delete_blocks Ns = Ns Sy 20480 Pq ulong
 This is the used to define a large file for the purposes of deletion.
 Files containing more than
 .Sy zfs_delete_blocks
 will be deleted asynchronously, while smaller files are deleted synchronously.
 Decreasing this value will reduce the time spent in an
 .Xr unlink 2
 system call, at the expense of a longer delay before the freed space is
 available.
 This only applies on Linux.
 .
 .It Sy zfs_dirty_data_max Ns = Pq int
 Determines the dirty space limit in bytes.
 Once this limit is exceeded, new writes are halted until space frees up.
 This parameter takes precedence over
 .Sy zfs_dirty_data_max_percent .
 .No See Sx ZFS TRANSACTION DELAY .
 .Pp
 Defaults to
 .Sy physical_ram/10 ,
 capped at
 .Sy zfs_dirty_data_max_max .
 .
 .It Sy zfs_dirty_data_max_max Ns = Pq int
 Maximum allowable value of
 .Sy zfs_dirty_data_max ,
 expressed in bytes.
 This limit is only enforced at module load time, and will be ignored if
 .Sy zfs_dirty_data_max
 is later changed.
 This parameter takes precedence over
 .Sy zfs_dirty_data_max_max_percent .
 .No See Sx ZFS TRANSACTION DELAY .
 .Pp
 Defaults to
 .Sy min(physical_ram/4, 4GiB) ,
 or
 .Sy min(physical_ram/4, 1GiB)
 for 32-bit systems.
 .
 .It Sy zfs_dirty_data_max_max_percent Ns = Ns Sy 25 Ns % Pq uint
 Maximum allowable value of
 .Sy zfs_dirty_data_max ,
 expressed as a percentage of physical RAM.
 This limit is only enforced at module load time, and will be ignored if
 .Sy zfs_dirty_data_max
 is later changed.
 The parameter
 .Sy zfs_dirty_data_max_max
 takes precedence over this one.
 .No See Sx ZFS TRANSACTION DELAY .
 .
 .It Sy zfs_dirty_data_max_percent Ns = Ns Sy 10 Ns % Pq uint
 Determines the dirty space limit, expressed as a percentage of all memory.
 Once this limit is exceeded, new writes are halted until space frees up.
 The parameter
 .Sy zfs_dirty_data_max
 takes precedence over this one.
 .No See Sx ZFS TRANSACTION DELAY .
 .Pp
 Subject to
 .Sy zfs_dirty_data_max_max .
 .
 .It Sy zfs_dirty_data_sync_percent Ns = Ns Sy 20 Ns % Pq uint
 Start syncing out a transaction group if there's at least this much dirty data
 .Pq as a percentage of Sy zfs_dirty_data_max .
 This should be less than
 .Sy zfs_vdev_async_write_active_min_dirty_percent .
 .
 .It Sy zfs_wrlog_data_max Ns = Pq int
 The upper limit of write-transaction zil log data size in bytes.
 Write operations are throttled when approaching the limit until log data is
 cleared out after transaction group sync.
 Because of some overhead, it should be set at least 2 times the size of
 .Sy zfs_dirty_data_max
 .No to prevent harming normal write throughput .
 It also should be smaller than the size of the slog device if slog is present.
 .Pp
 Defaults to
 .Sy zfs_dirty_data_max*2
 .
 .It Sy zfs_fallocate_reserve_percent Ns = Ns Sy 110 Ns % Pq uint
 Since ZFS is a copy-on-write filesystem with snapshots, blocks cannot be
 preallocated for a file in order to guarantee that later writes will not
 run out of space.
 Instead,
 .Xr fallocate 2
 space preallocation only checks that sufficient space is currently available
 in the pool or the user's project quota allocation,
 and then creates a sparse file of the requested size.
 The requested space is multiplied by
 .Sy zfs_fallocate_reserve_percent
 to allow additional space for indirect blocks and other internal metadata.
 Setting this to
 .Sy 0
 disables support for
 .Xr fallocate 2
 and causes it to return
 .Sy EOPNOTSUPP .
 .
 .It Sy zfs_fletcher_4_impl Ns = Ns Sy fastest Pq string
 Select a fletcher 4 implementation.
 .Pp
 Supported selectors are:
 .Sy fastest , scalar , sse2 , ssse3 , avx2 , avx512f , avx512bw ,
 .No and Sy aarch64_neon .
 All except
 .Sy fastest No and Sy scalar
 require instruction set extensions to be available,
 and will only appear if ZFS detects that they are present at runtime.
 If multiple implementations of fletcher 4 are available, the
 .Sy fastest
 will be chosen using a micro benchmark.
 Selecting
 .Sy scalar
 results in the original CPU-based calculation being used.
 Selecting any option other than
 .Sy fastest No or Sy scalar
 results in vector instructions
 from the respective CPU instruction set being used.
 .
 .It Sy zfs_bclone_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Enable the experimental block cloning feature.
 If this setting is 0, then even if feature@block_cloning is enabled,
 attempts to clone blocks will act as though the feature is disabled.
 .
 .It Sy zfs_bclone_wait_dirty Ns = Ns Sy 0 Ns | Ns 1 Pq int
 When set to 1 the FICLONE and FICLONERANGE ioctls wait for dirty data to be
 written to disk.
 This allows the clone operation to reliably succeed when a file is
 modified and then immediately cloned.
 For small files this may be slower than making a copy of the file.
 Therefore, this setting defaults to 0 which causes a clone operation to
 immediately fail when encountering a dirty block.
 .
 .It Sy zfs_blake3_impl Ns = Ns Sy fastest Pq string
 Select a BLAKE3 implementation.
 .Pp
 Supported selectors are:
 .Sy cycle , fastest , generic , sse2 , sse41 , avx2 , avx512 .
 All except
 .Sy cycle , fastest No and Sy generic
 require instruction set extensions to be available,
 and will only appear if ZFS detects that they are present at runtime.
 If multiple implementations of BLAKE3 are available, the
 .Sy fastest will be chosen using a micro benchmark. You can see the
 benchmark results by reading this kstat file:
 .Pa /proc/spl/kstat/zfs/chksum_bench .
 .
 .It Sy zfs_free_bpobj_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Enable/disable the processing of the free_bpobj object.
 .
 .It Sy zfs_async_block_max_blocks Ns = Ns Sy UINT64_MAX Po unlimited Pc Pq u64
 Maximum number of blocks freed in a single TXG.
 .
 .It Sy zfs_max_async_dedup_frees Ns = Ns Sy 100000 Po 10^5 Pc Pq u64
 Maximum number of dedup blocks freed in a single TXG.
 .
 .It Sy zfs_vdev_async_read_max_active Ns = Ns Sy 3 Pq uint
 Maximum asynchronous read I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_async_read_min_active Ns = Ns Sy 1 Pq uint
 Minimum asynchronous read I/O operation active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_async_write_active_max_dirty_percent Ns = Ns Sy 60 Ns % Pq uint
 When the pool has more than this much dirty data, use
 .Sy zfs_vdev_async_write_max_active
 to limit active async writes.
 If the dirty data is between the minimum and maximum,
 the active I/O limit is linearly interpolated.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_async_write_active_min_dirty_percent Ns = Ns Sy 30 Ns % Pq uint
 When the pool has less than this much dirty data, use
 .Sy zfs_vdev_async_write_min_active
 to limit active async writes.
 If the dirty data is between the minimum and maximum,
 the active I/O limit is linearly
 interpolated.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_async_write_max_active Ns = Ns Sy 10 Pq uint
 Maximum asynchronous write I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_async_write_min_active Ns = Ns Sy 2 Pq uint
 Minimum asynchronous write I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .Pp
 Lower values are associated with better latency on rotational media but poorer
 resilver performance.
 The default value of
 .Sy 2
 was chosen as a compromise.
 A value of
 .Sy 3
 has been shown to improve resilver performance further at a cost of
 further increasing latency.
 .
 .It Sy zfs_vdev_initializing_max_active Ns = Ns Sy 1 Pq uint
 Maximum initializing I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_initializing_min_active Ns = Ns Sy 1 Pq uint
 Minimum initializing I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_max_active Ns = Ns Sy 1000 Pq uint
 The maximum number of I/O operations active to each device.
 Ideally, this will be at least the sum of each queue's
 .Sy max_active .
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_open_timeout_ms Ns = Ns Sy 1000 Pq uint
 Timeout value to wait before determining a device is missing
 during import.
 This is helpful for transient missing paths due
 to links being briefly removed and recreated in response to
 udev events.
 .
 .It Sy zfs_vdev_rebuild_max_active Ns = Ns Sy 3 Pq uint
 Maximum sequential resilver I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_rebuild_min_active Ns = Ns Sy 1 Pq uint
 Minimum sequential resilver I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_removal_max_active Ns = Ns Sy 2 Pq uint
 Maximum removal I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_removal_min_active Ns = Ns Sy 1 Pq uint
 Minimum removal I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_scrub_max_active Ns = Ns Sy 2 Pq uint
 Maximum scrub I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_scrub_min_active Ns = Ns Sy 1 Pq uint
 Minimum scrub I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_sync_read_max_active Ns = Ns Sy 10 Pq uint
 Maximum synchronous read I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_sync_read_min_active Ns = Ns Sy 10 Pq uint
 Minimum synchronous read I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_sync_write_max_active Ns = Ns Sy 10 Pq uint
 Maximum synchronous write I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_sync_write_min_active Ns = Ns Sy 10 Pq uint
 Minimum synchronous write I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_trim_max_active Ns = Ns Sy 2 Pq uint
 Maximum trim/discard I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_trim_min_active Ns = Ns Sy 1 Pq uint
 Minimum trim/discard I/O operations active to each device.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_nia_delay Ns = Ns Sy 5 Pq uint
 For non-interactive I/O (scrub, resilver, removal, initialize and rebuild),
 the number of concurrently-active I/O operations is limited to
 .Sy zfs_*_min_active ,
 unless the vdev is "idle".
 When there are no interactive I/O operations active (synchronous or otherwise),
 and
 .Sy zfs_vdev_nia_delay
 operations have completed since the last interactive operation,
 then the vdev is considered to be "idle",
 and the number of concurrently-active non-interactive operations is increased to
 .Sy zfs_*_max_active .
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_nia_credit Ns = Ns Sy 5 Pq uint
 Some HDDs tend to prioritize sequential I/O so strongly, that concurrent
 random I/O latency reaches several seconds.
 On some HDDs this happens even if sequential I/O operations
 are submitted one at a time, and so setting
 .Sy zfs_*_max_active Ns = Sy 1
 does not help.
 To prevent non-interactive I/O, like scrub,
 from monopolizing the device, no more than
 .Sy zfs_vdev_nia_credit operations can be sent
 while there are outstanding incomplete interactive operations.
 This enforced wait ensures the HDD services the interactive I/O
 within a reasonable amount of time.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_queue_depth_pct Ns = Ns Sy 1000 Ns % Pq uint
 Maximum number of queued allocations per top-level vdev expressed as
 a percentage of
 .Sy zfs_vdev_async_write_max_active ,
 which allows the system to detect devices that are more capable
 of handling allocations and to allocate more blocks to those devices.
 This allows for dynamic allocation distribution when devices are imbalanced,
 as fuller devices will tend to be slower than empty devices.
 .Pp
 Also see
 .Sy zio_dva_throttle_enabled .
 .
 .It Sy zfs_vdev_def_queue_depth Ns = Ns Sy 32 Pq uint
 Default queue depth for each vdev IO allocator.
 Higher values allow for better coalescing of sequential writes before sending
 them to the disk, but can increase transaction commit times.
 .
 .It Sy zfs_vdev_failfast_mask Ns = Ns Sy 1 Pq uint
 Defines if the driver should retire on a given error type.
 The following options may be bitwise-ored together:
 .TS
 box;
 lbz r l l .
 	Value	Name	Description
 _
 	1	Device	No driver retries on device errors
 	2	Transport	No driver retries on transport errors.
 	4	Driver	No driver retries on driver errors.
 .TE
 .
 .It Sy zfs_vdev_disk_max_segs Ns = Ns Sy 0 Pq uint
 Maximum number of segments to add to a BIO (min 4).
 If this is higher than the maximum allowed by the device queue or the kernel
 itself, it will be clamped.
 Setting it to zero will cause the kernel's ideal size to be used.
 This parameter only applies on Linux.
 This parameter is ignored if
 .Sy zfs_vdev_disk_classic Ns = Ns Sy 1 .
 .
-.It Sy zfs_vdev_disk_classic Ns = Ns 0 Ns | Ns Sy 1 Pq uint
-Controls the method used to submit IO to the Linux block layer
-(default
-.Sy 1 "classic" Ns
-)
-.Pp
-If set to 1, the "classic" method is used.
-This is the method that has been in use since the earliest versions of
-ZFS-on-Linux.
-It has known issues with highly fragmented IO requests and is less efficient on
-many workloads, but it well known and well understood.
-.Pp
-If set to 0, the "new" method is used.
-This method is available since 2.2.4 and should resolve all known issues and be
-far more efficient, but has not had as much testing.
-In the 2.2.x series, this parameter defaults to 1, to use the "classic" method.
-.Pp
-It is not recommended that you change it except on advice from the OpenZFS
-developers.
-If you do change it, please also open a bug report describing why you did so,
+.It Sy zfs_vdev_disk_classic Ns = Ns Sy 0 Ns | Ns 1 Pq uint
+If set to 1, OpenZFS will submit IO to Linux using the method it used in 2.2
+and earlier.
+This "classic" method has known issues with highly fragmented IO requests and
+is slower on many workloads, but it has been in use for many years and is known
+to be very stable.
+If you set this parameter, please also open a bug report why you did so,
 including the workload involved and any error messages.
 .Pp
-This parameter and the "classic" submission method will be removed in a future
-release of OpenZFS once we have total confidence in the new method.
+This parameter and the classic submission method will be removed once we have
+total confidence in the new method.
 .Pp
 This parameter only applies on Linux, and can only be set at module load time.
 .
 .It Sy zfs_expire_snapshot Ns = Ns Sy 300 Ns s Pq int
 Time before expiring
 .Pa .zfs/snapshot .
 .
 .It Sy zfs_admin_snapshot Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Allow the creation, removal, or renaming of entries in the
 .Sy .zfs/snapshot
 directory to cause the creation, destruction, or renaming of snapshots.
 When enabled, this functionality works both locally and over NFS exports
 which have the
 .Em no_root_squash
 option set.
 .
 .It Sy zfs_flags Ns = Ns Sy 0 Pq int
 Set additional debugging flags.
 The following flags may be bitwise-ored together:
 .TS
 box;
 lbz r l l .
 	Value	Name	Description
 _
 	1	ZFS_DEBUG_DPRINTF	Enable dprintf entries in the debug log.
 *	2	ZFS_DEBUG_DBUF_VERIFY	Enable extra dbuf verifications.
 *	4	ZFS_DEBUG_DNODE_VERIFY	Enable extra dnode verifications.
 	8	ZFS_DEBUG_SNAPNAMES	Enable snapshot name verification.
 *	16	ZFS_DEBUG_MODIFY	Check for illegally modified ARC buffers.
 	64	ZFS_DEBUG_ZIO_FREE	Enable verification of block frees.
 	128	ZFS_DEBUG_HISTOGRAM_VERIFY	Enable extra spacemap histogram verifications.
 	256	ZFS_DEBUG_METASLAB_VERIFY	Verify space accounting on disk matches in-memory \fBrange_trees\fP.
 	512	ZFS_DEBUG_SET_ERROR	Enable \fBSET_ERROR\fP and dprintf entries in the debug log.
 	1024	ZFS_DEBUG_INDIRECT_REMAP	Verify split blocks created by device removal.
 	2048	ZFS_DEBUG_TRIM	Verify TRIM ranges are always within the allocatable range tree.
 	4096	ZFS_DEBUG_LOG_SPACEMAP	Verify that the log summary is consistent with the spacemap log
 			       and enable \fBzfs_dbgmsgs\fP for metaslab loading and flushing.
 .TE
 .Sy \& * No Requires debug build .
 .
 .It Sy zfs_btree_verify_intensity Ns = Ns Sy 0 Pq uint
 Enables btree verification.
 The following settings are culminative:
 .TS
 box;
 lbz r l l .
 	Value	Description
 
 	1	Verify height.
 	2	Verify pointers from children to parent.
 	3	Verify element counts.
 	4	Verify element order. (expensive)
 *	5	Verify unused memory is poisoned. (expensive)
 .TE
 .Sy \& * No Requires debug build .
 .
 .It Sy zfs_free_leak_on_eio Ns = Ns Sy 0 Ns | Ns 1 Pq int
 If destroy encounters an
 .Sy EIO
 while reading metadata (e.g. indirect blocks),
 space referenced by the missing metadata can not be freed.
 Normally this causes the background destroy to become "stalled",
 as it is unable to make forward progress.
 While in this stalled state, all remaining space to free
 from the error-encountering filesystem is "temporarily leaked".
 Set this flag to cause it to ignore the
 .Sy EIO ,
 permanently leak the space from indirect blocks that can not be read,
 and continue to free everything else that it can.
 .Pp
 The default "stalling" behavior is useful if the storage partially
 fails (i.e. some but not all I/O operations fail), and then later recovers.
 In this case, we will be able to continue pool operations while it is
 partially failed, and when it recovers, we can continue to free the
 space, with no leaks.
 Note, however, that this case is actually fairly rare.
 .Pp
 Typically pools either
 .Bl -enum -compact -offset 4n -width "1."
 .It
 fail completely (but perhaps temporarily,
 e.g. due to a top-level vdev going offline), or
 .It
 have localized, permanent errors (e.g. disk returns the wrong data
 due to bit flip or firmware bug).
 .El
 In the former case, this setting does not matter because the
 pool will be suspended and the sync thread will not be able to make
 forward progress regardless.
 In the latter, because the error is permanent, the best we can do
 is leak the minimum amount of space,
 which is what setting this flag will do.
 It is therefore reasonable for this flag to normally be set,
 but we chose the more conservative approach of not setting it,
 so that there is no possibility of
 leaking space in the "partial temporary" failure case.
 .
 .It Sy zfs_free_min_time_ms Ns = Ns Sy 1000 Ns ms Po 1s Pc Pq uint
 During a
 .Nm zfs Cm destroy
 operation using the
 .Sy async_destroy
 feature,
 a minimum of this much time will be spent working on freeing blocks per TXG.
 .
 .It Sy zfs_obsolete_min_time_ms Ns = Ns Sy 500 Ns ms Pq uint
 Similar to
 .Sy zfs_free_min_time_ms ,
 but for cleanup of old indirection records for removed vdevs.
 .
 .It Sy zfs_immediate_write_sz Ns = Ns Sy 32768 Ns B Po 32 KiB Pc Pq s64
 Largest data block to write to the ZIL.
 Larger blocks will be treated as if the dataset being written to had the
 .Sy logbias Ns = Ns Sy throughput
 property set.
 .
 .It Sy zfs_initialize_value Ns = Ns Sy 16045690984833335022 Po 0xDEADBEEFDEADBEEE Pc Pq u64
 Pattern written to vdev free space by
 .Xr zpool-initialize 8 .
 .
 .It Sy zfs_initialize_chunk_size Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
 Size of writes used by
 .Xr zpool-initialize 8 .
 This option is used by the test suite.
 .
 .It Sy zfs_livelist_max_entries Ns = Ns Sy 500000 Po 5*10^5 Pc Pq u64
 The threshold size (in block pointers) at which we create a new sub-livelist.
 Larger sublists are more costly from a memory perspective but the fewer
 sublists there are, the lower the cost of insertion.
 .
 .It Sy zfs_livelist_min_percent_shared Ns = Ns Sy 75 Ns % Pq int
 If the amount of shared space between a snapshot and its clone drops below
 this threshold, the clone turns off the livelist and reverts to the old
 deletion method.
 This is in place because livelists no long give us a benefit
 once a clone has been overwritten enough.
 .
 .It Sy zfs_livelist_condense_new_alloc Ns = Ns Sy 0 Pq int
 Incremented each time an extra ALLOC blkptr is added to a livelist entry while
 it is being condensed.
 This option is used by the test suite to track race conditions.
 .
 .It Sy zfs_livelist_condense_sync_cancel Ns = Ns Sy 0 Pq int
 Incremented each time livelist condensing is canceled while in
 .Fn spa_livelist_condense_sync .
 This option is used by the test suite to track race conditions.
 .
 .It Sy zfs_livelist_condense_sync_pause Ns = Ns Sy 0 Ns | Ns 1 Pq int
 When set, the livelist condense process pauses indefinitely before
 executing the synctask \(em
 .Fn spa_livelist_condense_sync .
 This option is used by the test suite to trigger race conditions.
 .
 .It Sy zfs_livelist_condense_zthr_cancel Ns = Ns Sy 0 Pq int
 Incremented each time livelist condensing is canceled while in
 .Fn spa_livelist_condense_cb .
 This option is used by the test suite to track race conditions.
 .
 .It Sy zfs_livelist_condense_zthr_pause Ns = Ns Sy 0 Ns | Ns 1 Pq int
 When set, the livelist condense process pauses indefinitely before
 executing the open context condensing work in
 .Fn spa_livelist_condense_cb .
 This option is used by the test suite to trigger race conditions.
 .
 .It Sy zfs_lua_max_instrlimit Ns = Ns Sy 100000000 Po 10^8 Pc Pq u64
 The maximum execution time limit that can be set for a ZFS channel program,
 specified as a number of Lua instructions.
 .
 .It Sy zfs_lua_max_memlimit Ns = Ns Sy 104857600 Po 100 MiB Pc Pq u64
 The maximum memory limit that can be set for a ZFS channel program, specified
 in bytes.
 .
 .It Sy zfs_max_dataset_nesting Ns = Ns Sy 50 Pq int
 The maximum depth of nested datasets.
 This value can be tuned temporarily to
 fix existing datasets that exceed the predefined limit.
 .
 .It Sy zfs_max_log_walking Ns = Ns Sy 5 Pq u64
 The number of past TXGs that the flushing algorithm of the log spacemap
 feature uses to estimate incoming log blocks.
 .
 .It Sy zfs_max_logsm_summary_length Ns = Ns Sy 10 Pq u64
 Maximum number of rows allowed in the summary of the spacemap log.
 .
 .It Sy zfs_max_recordsize Ns = Ns Sy 16777216 Po 16 MiB Pc Pq uint
 We currently support block sizes from
 .Em 512 Po 512 B Pc No to Em 16777216 Po 16 MiB Pc .
 The benefits of larger blocks, and thus larger I/O,
 need to be weighed against the cost of COWing a giant block to modify one byte.
 Additionally, very large blocks can have an impact on I/O latency,
 and also potentially on the memory allocator.
 Therefore, we formerly forbade creating blocks larger than 1M.
 Larger blocks could be created by changing it,
 and pools with larger blocks can always be imported and used,
 regardless of this setting.
 .Pp
 Note that it is still limited by default to
 .Ar 1 MiB
 on x86_32, because Linux's
 3/1 memory split doesn't leave much room for 16M chunks.
 .
 .It Sy zfs_allow_redacted_dataset_mount Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Allow datasets received with redacted send/receive to be mounted.
 Normally disabled because these datasets may be missing key data.
 .
 .It Sy zfs_min_metaslabs_to_flush Ns = Ns Sy 1 Pq u64
 Minimum number of metaslabs to flush per dirty TXG.
 .
 .It Sy zfs_metaslab_fragmentation_threshold Ns = Ns Sy 70 Ns % Pq uint
 Allow metaslabs to keep their active state as long as their fragmentation
 percentage is no more than this value.
 An active metaslab that exceeds this threshold
 will no longer keep its active status allowing better metaslabs to be selected.
 .
 .It Sy zfs_mg_fragmentation_threshold Ns = Ns Sy 95 Ns % Pq uint
 Metaslab groups are considered eligible for allocations if their
 fragmentation metric (measured as a percentage) is less than or equal to
 this value.
 If a metaslab group exceeds this threshold then it will be
 skipped unless all metaslab groups within the metaslab class have also
 crossed this threshold.
 .
 .It Sy zfs_mg_noalloc_threshold Ns = Ns Sy 0 Ns % Pq uint
 Defines a threshold at which metaslab groups should be eligible for allocations.
 The value is expressed as a percentage of free space
 beyond which a metaslab group is always eligible for allocations.
 If a metaslab group's free space is less than or equal to the
 threshold, the allocator will avoid allocating to that group
 unless all groups in the pool have reached the threshold.
 Once all groups have reached the threshold, all groups are allowed to accept
 allocations.
 The default value of
 .Sy 0
 disables the feature and causes all metaslab groups to be eligible for
 allocations.
 .Pp
 This parameter allows one to deal with pools having heavily imbalanced
 vdevs such as would be the case when a new vdev has been added.
 Setting the threshold to a non-zero percentage will stop allocations
 from being made to vdevs that aren't filled to the specified percentage
 and allow lesser filled vdevs to acquire more allocations than they
 otherwise would under the old
 .Sy zfs_mg_alloc_failures
 facility.
 .
 .It Sy zfs_ddt_data_is_special Ns = Ns Sy 1 Ns | Ns 0 Pq int
 If enabled, ZFS will place DDT data into the special allocation class.
 .
 .It Sy zfs_user_indirect_is_special Ns = Ns Sy 1 Ns | Ns 0 Pq int
 If enabled, ZFS will place user data indirect blocks
 into the special allocation class.
 .
 .It Sy zfs_multihost_history Ns = Ns Sy 0 Pq uint
 Historical statistics for this many latest multihost updates will be available
 in
 .Pa /proc/spl/kstat/zfs/ Ns Ao Ar pool Ac Ns Pa /multihost .
 .
 .It Sy zfs_multihost_interval Ns = Ns Sy 1000 Ns ms Po 1 s Pc Pq u64
 Used to control the frequency of multihost writes which are performed when the
 .Sy multihost
 pool property is on.
 This is one of the factors used to determine the
 length of the activity check during import.
 .Pp
 The multihost write period is
 .Sy zfs_multihost_interval No / Sy leaf-vdevs .
 On average a multihost write will be issued for each leaf vdev
 every
 .Sy zfs_multihost_interval
 milliseconds.
 In practice, the observed period can vary with the I/O load
 and this observed value is the delay which is stored in the uberblock.
 .
 .It Sy zfs_multihost_import_intervals Ns = Ns Sy 20 Pq uint
 Used to control the duration of the activity test on import.
 Smaller values of
 .Sy zfs_multihost_import_intervals
 will reduce the import time but increase
 the risk of failing to detect an active pool.
 The total activity check time is never allowed to drop below one second.
 .Pp
 On import the activity check waits a minimum amount of time determined by
 .Sy zfs_multihost_interval No \(mu Sy zfs_multihost_import_intervals ,
 or the same product computed on the host which last had the pool imported,
 whichever is greater.
 The activity check time may be further extended if the value of MMP
 delay found in the best uberblock indicates actual multihost updates happened
 at longer intervals than
 .Sy zfs_multihost_interval .
 A minimum of
 .Em 100 ms
 is enforced.
 .Pp
 .Sy 0 No is equivalent to Sy 1 .
 .
 .It Sy zfs_multihost_fail_intervals Ns = Ns Sy 10 Pq uint
 Controls the behavior of the pool when multihost write failures or delays are
 detected.
 .Pp
 When
 .Sy 0 ,
 multihost write failures or delays are ignored.
 The failures will still be reported to the ZED which depending on
 its configuration may take action such as suspending the pool or offlining a
 device.
 .Pp
 Otherwise, the pool will be suspended if
 .Sy zfs_multihost_fail_intervals No \(mu Sy zfs_multihost_interval
 milliseconds pass without a successful MMP write.
 This guarantees the activity test will see MMP writes if the pool is imported.
 .Sy 1 No is equivalent to Sy 2 ;
 this is necessary to prevent the pool from being suspended
 due to normal, small I/O latency variations.
 .
 .It Sy zfs_no_scrub_io Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Set to disable scrub I/O.
 This results in scrubs not actually scrubbing data and
 simply doing a metadata crawl of the pool instead.
 .
 .It Sy zfs_no_scrub_prefetch Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Set to disable block prefetching for scrubs.
 .
 .It Sy zfs_nocacheflush Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Disable cache flush operations on disks when writing.
 Setting this will cause pool corruption on power loss
 if a volatile out-of-order write cache is enabled.
 .
 .It Sy zfs_nopwrite_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Allow no-operation writes.
 The occurrence of nopwrites will further depend on other pool properties
 .Pq i.a. the checksumming and compression algorithms .
 .
 .It Sy zfs_dmu_offset_next_sync Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Enable forcing TXG sync to find holes.
 When enabled forces ZFS to sync data when
 .Sy SEEK_HOLE No or Sy SEEK_DATA
 flags are used allowing holes in a file to be accurately reported.
 When disabled holes will not be reported in recently dirtied files.
 .
 .It Sy zfs_pd_bytes_max Ns = Ns Sy 52428800 Ns B Po 50 MiB Pc Pq int
 The number of bytes which should be prefetched during a pool traversal, like
 .Nm zfs Cm send
 or other data crawling operations.
 .
 .It Sy zfs_traverse_indirect_prefetch_limit Ns = Ns Sy 32 Pq uint
 The number of blocks pointed by indirect (non-L0) block which should be
 prefetched during a pool traversal, like
 .Nm zfs Cm send
 or other data crawling operations.
 .
 .It Sy zfs_per_txg_dirty_frees_percent Ns = Ns Sy 30 Ns % Pq u64
 Control percentage of dirtied indirect blocks from frees allowed into one TXG.
 After this threshold is crossed, additional frees will wait until the next TXG.
 .Sy 0 No disables this throttle .
 .
 .It Sy zfs_prefetch_disable Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Disable predictive prefetch.
 Note that it leaves "prescient" prefetch
 .Pq for, e.g., Nm zfs Cm send
 intact.
 Unlike predictive prefetch, prescient prefetch never issues I/O
 that ends up not being needed, so it can't hurt performance.
 .
 .It Sy zfs_qat_checksum_disable Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Disable QAT hardware acceleration for SHA256 checksums.
 May be unset after the ZFS modules have been loaded to initialize the QAT
 hardware as long as support is compiled in and the QAT driver is present.
 .
 .It Sy zfs_qat_compress_disable Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Disable QAT hardware acceleration for gzip compression.
 May be unset after the ZFS modules have been loaded to initialize the QAT
 hardware as long as support is compiled in and the QAT driver is present.
 .
 .It Sy zfs_qat_encrypt_disable Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Disable QAT hardware acceleration for AES-GCM encryption.
 May be unset after the ZFS modules have been loaded to initialize the QAT
 hardware as long as support is compiled in and the QAT driver is present.
 .
 .It Sy zfs_vnops_read_chunk_size Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
 Bytes to read per chunk.
 .
 .It Sy zfs_read_history Ns = Ns Sy 0 Pq uint
 Historical statistics for this many latest reads will be available in
 .Pa /proc/spl/kstat/zfs/ Ns Ao Ar pool Ac Ns Pa /reads .
 .
 .It Sy zfs_read_history_hits Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Include cache hits in read history
 .
 .It Sy zfs_rebuild_max_segment Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq u64
 Maximum read segment size to issue when sequentially resilvering a
 top-level vdev.
 .
 .It Sy zfs_rebuild_scrub_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Automatically start a pool scrub when the last active sequential resilver
 completes in order to verify the checksums of all blocks which have been
 resilvered.
 This is enabled by default and strongly recommended.
 .
 .It Sy zfs_rebuild_vdev_limit Ns = Ns Sy 67108864 Ns B Po 64 MiB Pc Pq u64
 Maximum amount of I/O that can be concurrently issued for a sequential
 resilver per leaf device, given in bytes.
 .
 .It Sy zfs_reconstruct_indirect_combinations_max Ns = Ns Sy 4096 Pq int
 If an indirect split block contains more than this many possible unique
 combinations when being reconstructed, consider it too computationally
 expensive to check them all.
 Instead, try at most this many randomly selected
 combinations each time the block is accessed.
 This allows all segment copies to participate fairly
 in the reconstruction when all combinations
 cannot be checked and prevents repeated use of one bad copy.
 .
 .It Sy zfs_recover Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Set to attempt to recover from fatal errors.
 This should only be used as a last resort,
 as it typically results in leaked space, or worse.
 .
 .It Sy zfs_removal_ignore_errors Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Ignore hard I/O errors during device removal.
 When set, if a device encounters a hard I/O error during the removal process
 the removal will not be cancelled.
 This can result in a normally recoverable block becoming permanently damaged
 and is hence not recommended.
 This should only be used as a last resort when the
 pool cannot be returned to a healthy state prior to removing the device.
 .
 .It Sy zfs_removal_suspend_progress Ns = Ns Sy 0 Ns | Ns 1 Pq uint
 This is used by the test suite so that it can ensure that certain actions
 happen while in the middle of a removal.
 .
 .It Sy zfs_remove_max_segment Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq uint
 The largest contiguous segment that we will attempt to allocate when removing
 a device.
 If there is a performance problem with attempting to allocate large blocks,
 consider decreasing this.
 The default value is also the maximum.
 .
 .It Sy zfs_resilver_disable_defer Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Ignore the
 .Sy resilver_defer
 feature, causing an operation that would start a resilver to
 immediately restart the one in progress.
 .
 .It Sy zfs_resilver_min_time_ms Ns = Ns Sy 3000 Ns ms Po 3 s Pc Pq uint
 Resilvers are processed by the sync thread.
 While resilvering, it will spend at least this much time
 working on a resilver between TXG flushes.
 .
 .It Sy zfs_scan_ignore_errors Ns = Ns Sy 0 Ns | Ns 1 Pq int
 If set, remove the DTL (dirty time list) upon completion of a pool scan (scrub),
 even if there were unrepairable errors.
 Intended to be used during pool repair or recovery to
 stop resilvering when the pool is next imported.
 .
 .It Sy zfs_scrub_min_time_ms Ns = Ns Sy 1000 Ns ms Po 1 s Pc Pq uint
 Scrubs are processed by the sync thread.
 While scrubbing, it will spend at least this much time
 working on a scrub between TXG flushes.
 .
 .It Sy zfs_scrub_error_blocks_per_txg Ns = Ns Sy 4096 Pq uint
 Error blocks to be scrubbed in one txg.
 .
 .It Sy zfs_scan_checkpoint_intval Ns = Ns Sy 7200 Ns s Po 2 hour Pc Pq uint
 To preserve progress across reboots, the sequential scan algorithm periodically
 needs to stop metadata scanning and issue all the verification I/O to disk.
 The frequency of this flushing is determined by this tunable.
 .
 .It Sy zfs_scan_fill_weight Ns = Ns Sy 3 Pq uint
 This tunable affects how scrub and resilver I/O segments are ordered.
 A higher number indicates that we care more about how filled in a segment is,
 while a lower number indicates we care more about the size of the extent without
 considering the gaps within a segment.
 This value is only tunable upon module insertion.
 Changing the value afterwards will have no effect on scrub or resilver
 performance.
 .
 .It Sy zfs_scan_issue_strategy Ns = Ns Sy 0 Pq uint
 Determines the order that data will be verified while scrubbing or resilvering:
 .Bl -tag -compact -offset 4n -width "a"
 .It Sy 1
 Data will be verified as sequentially as possible, given the
 amount of memory reserved for scrubbing
 .Pq see Sy zfs_scan_mem_lim_fact .
 This may improve scrub performance if the pool's data is very fragmented.
 .It Sy 2
 The largest mostly-contiguous chunk of found data will be verified first.
 By deferring scrubbing of small segments, we may later find adjacent data
 to coalesce and increase the segment size.
 .It Sy 0
 .No Use strategy Sy 1 No during normal verification
 .No and strategy Sy 2 No while taking a checkpoint .
 .El
 .
 .It Sy zfs_scan_legacy Ns = Ns Sy 0 Ns | Ns 1 Pq int
 If unset, indicates that scrubs and resilvers will gather metadata in
 memory before issuing sequential I/O.
 Otherwise indicates that the legacy algorithm will be used,
 where I/O is initiated as soon as it is discovered.
 Unsetting will not affect scrubs or resilvers that are already in progress.
 .
 .It Sy zfs_scan_max_ext_gap Ns = Ns Sy 2097152 Ns B Po 2 MiB Pc Pq int
 Sets the largest gap in bytes between scrub/resilver I/O operations
 that will still be considered sequential for sorting purposes.
 Changing this value will not
 affect scrubs or resilvers that are already in progress.
 .
 .It Sy zfs_scan_mem_lim_fact Ns = Ns Sy 20 Ns ^-1 Pq uint
 Maximum fraction of RAM used for I/O sorting by sequential scan algorithm.
 This tunable determines the hard limit for I/O sorting memory usage.
 When the hard limit is reached we stop scanning metadata and start issuing
 data verification I/O.
 This is done until we get below the soft limit.
 .
 .It Sy zfs_scan_mem_lim_soft_fact Ns = Ns Sy 20 Ns ^-1 Pq uint
 The fraction of the hard limit used to determined the soft limit for I/O sorting
 by the sequential scan algorithm.
 When we cross this limit from below no action is taken.
 When we cross this limit from above it is because we are issuing verification
 I/O.
 In this case (unless the metadata scan is done) we stop issuing verification I/O
 and start scanning metadata again until we get to the hard limit.
 .
 .It Sy zfs_scan_report_txgs Ns = Ns Sy 0 Ns | Ns 1 Pq uint
 When reporting resilver throughput and estimated completion time use the
 performance observed over roughly the last
 .Sy zfs_scan_report_txgs
 TXGs.
 When set to zero performance is calculated over the time between checkpoints.
 .
 .It Sy zfs_scan_strict_mem_lim Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Enforce tight memory limits on pool scans when a sequential scan is in progress.
 When disabled, the memory limit may be exceeded by fast disks.
 .
 .It Sy zfs_scan_suspend_progress Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Freezes a scrub/resilver in progress without actually pausing it.
 Intended for testing/debugging.
 .
 .It Sy zfs_scan_vdev_limit Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq int
 Maximum amount of data that can be concurrently issued at once for scrubs and
 resilvers per leaf device, given in bytes.
 .
 .It Sy zfs_send_corrupt_data Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Allow sending of corrupt data (ignore read/checksum errors when sending).
 .
 .It Sy zfs_send_unmodified_spill_blocks Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Include unmodified spill blocks in the send stream.
 Under certain circumstances, previous versions of ZFS could incorrectly
 remove the spill block from an existing object.
 Including unmodified copies of the spill blocks creates a backwards-compatible
 stream which will recreate a spill block if it was incorrectly removed.
 .
 .It Sy zfs_send_no_prefetch_queue_ff Ns = Ns Sy 20 Ns ^\-1 Pq uint
 The fill fraction of the
 .Nm zfs Cm send
 internal queues.
 The fill fraction controls the timing with which internal threads are woken up.
 .
 .It Sy zfs_send_no_prefetch_queue_length Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq uint
 The maximum number of bytes allowed in
 .Nm zfs Cm send Ns 's
 internal queues.
 .
 .It Sy zfs_send_queue_ff Ns = Ns Sy 20 Ns ^\-1 Pq uint
 The fill fraction of the
 .Nm zfs Cm send
 prefetch queue.
 The fill fraction controls the timing with which internal threads are woken up.
 .
 .It Sy zfs_send_queue_length Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq uint
 The maximum number of bytes allowed that will be prefetched by
 .Nm zfs Cm send .
 This value must be at least twice the maximum block size in use.
 .
 .It Sy zfs_recv_queue_ff Ns = Ns Sy 20 Ns ^\-1 Pq uint
 The fill fraction of the
 .Nm zfs Cm receive
 queue.
 The fill fraction controls the timing with which internal threads are woken up.
 .
 .It Sy zfs_recv_queue_length Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq uint
 The maximum number of bytes allowed in the
 .Nm zfs Cm receive
 queue.
 This value must be at least twice the maximum block size in use.
 .
 .It Sy zfs_recv_write_batch_size Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq uint
 The maximum amount of data, in bytes, that
 .Nm zfs Cm receive
 will write in one DMU transaction.
 This is the uncompressed size, even when receiving a compressed send stream.
 This setting will not reduce the write size below a single block.
 Capped at a maximum of
 .Sy 32 MiB .
 .
 .It Sy zfs_recv_best_effort_corrective Ns = Ns Sy 0 Pq int
 When this variable is set to non-zero a corrective receive:
 .Bl -enum -compact -offset 4n -width "1."
 .It
 Does not enforce the restriction of source & destination snapshot GUIDs
 matching.
 .It
 If there is an error during healing, the healing receive is not
 terminated instead it moves on to the next record.
 .El
 .
 .It Sy zfs_override_estimate_recordsize Ns = Ns Sy 0 Ns | Ns 1 Pq uint
 Setting this variable overrides the default logic for estimating block
 sizes when doing a
 .Nm zfs Cm send .
 The default heuristic is that the average block size
 will be the current recordsize.
 Override this value if most data in your dataset is not of that size
 and you require accurate zfs send size estimates.
 .
 .It Sy zfs_sync_pass_deferred_free Ns = Ns Sy 2 Pq uint
 Flushing of data to disk is done in passes.
 Defer frees starting in this pass.
 .
 .It Sy zfs_spa_discard_memory_limit Ns = Ns Sy 16777216 Ns B Po 16 MiB Pc Pq int
 Maximum memory used for prefetching a checkpoint's space map on each
 vdev while discarding the checkpoint.
 .
 .It Sy zfs_special_class_metadata_reserve_pct Ns = Ns Sy 25 Ns % Pq uint
 Only allow small data blocks to be allocated on the special and dedup vdev
 types when the available free space percentage on these vdevs exceeds this
 value.
 This ensures reserved space is available for pool metadata as the
 special vdevs approach capacity.
 .
 .It Sy zfs_sync_pass_dont_compress Ns = Ns Sy 8 Pq uint
 Starting in this sync pass, disable compression (including of metadata).
 With the default setting, in practice, we don't have this many sync passes,
 so this has no effect.
 .Pp
 The original intent was that disabling compression would help the sync passes
 to converge.
 However, in practice, disabling compression increases
 the average number of sync passes; because when we turn compression off,
 many blocks' size will change, and thus we have to re-allocate
 (not overwrite) them.
 It also increases the number of
 .Em 128 KiB
 allocations (e.g. for indirect blocks and spacemaps)
 because these will not be compressed.
 The
 .Em 128 KiB
 allocations are especially detrimental to performance
 on highly fragmented systems, which may have very few free segments of this
 size,
 and may need to load new metaslabs to satisfy these allocations.
 .
 .It Sy zfs_sync_pass_rewrite Ns = Ns Sy 2 Pq uint
 Rewrite new block pointers starting in this pass.
 .
 .It Sy zfs_sync_taskq_batch_pct Ns = Ns Sy 75 Ns % Pq int
 This controls the number of threads used by
 .Sy dp_sync_taskq .
 The default value of
 .Sy 75%
 will create a maximum of one thread per CPU.
 .
 .It Sy zfs_trim_extent_bytes_max Ns = Ns Sy 134217728 Ns B Po 128 MiB Pc Pq uint
 Maximum size of TRIM command.
 Larger ranges will be split into chunks no larger than this value before
 issuing.
 .
 .It Sy zfs_trim_extent_bytes_min Ns = Ns Sy 32768 Ns B Po 32 KiB Pc Pq uint
 Minimum size of TRIM commands.
 TRIM ranges smaller than this will be skipped,
 unless they're part of a larger range which was chunked.
 This is done because it's common for these small TRIMs
 to negatively impact overall performance.
 .
 .It Sy zfs_trim_metaslab_skip Ns = Ns Sy 0 Ns | Ns 1 Pq uint
 Skip uninitialized metaslabs during the TRIM process.
 This option is useful for pools constructed from large thinly-provisioned
 devices
 where TRIM operations are slow.
 As a pool ages, an increasing fraction of the pool's metaslabs
 will be initialized, progressively degrading the usefulness of this option.
 This setting is stored when starting a manual TRIM and will
 persist for the duration of the requested TRIM.
 .
 .It Sy zfs_trim_queue_limit Ns = Ns Sy 10 Pq uint
 Maximum number of queued TRIMs outstanding per leaf vdev.
 The number of concurrent TRIM commands issued to the device is controlled by
 .Sy zfs_vdev_trim_min_active No and Sy zfs_vdev_trim_max_active .
 .
 .It Sy zfs_trim_txg_batch Ns = Ns Sy 32 Pq uint
 The number of transaction groups' worth of frees which should be aggregated
 before TRIM operations are issued to the device.
 This setting represents a trade-off between issuing larger,
 more efficient TRIM operations and the delay
 before the recently trimmed space is available for use by the device.
 .Pp
 Increasing this value will allow frees to be aggregated for a longer time.
 This will result is larger TRIM operations and potentially increased memory
 usage.
 Decreasing this value will have the opposite effect.
 The default of
 .Sy 32
 was determined to be a reasonable compromise.
 .
 .It Sy zfs_txg_history Ns = Ns Sy 0 Pq uint
 Historical statistics for this many latest TXGs will be available in
 .Pa /proc/spl/kstat/zfs/ Ns Ao Ar pool Ac Ns Pa /TXGs .
 .
 .It Sy zfs_txg_timeout Ns = Ns Sy 5 Ns s Pq uint
 Flush dirty data to disk at least every this many seconds (maximum TXG
 duration).
 .
 .It Sy zfs_vdev_aggregation_limit Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq uint
 Max vdev I/O aggregation size.
 .
 .It Sy zfs_vdev_aggregation_limit_non_rotating Ns = Ns Sy 131072 Ns B Po 128 KiB Pc Pq uint
 Max vdev I/O aggregation size for non-rotating media.
 .
 .It Sy zfs_vdev_mirror_rotating_inc Ns = Ns Sy 0 Pq int
 A number by which the balancing algorithm increments the load calculation for
 the purpose of selecting the least busy mirror member when an I/O operation
 immediately follows its predecessor on rotational vdevs
 for the purpose of making decisions based on load.
 .
 .It Sy zfs_vdev_mirror_rotating_seek_inc Ns = Ns Sy 5 Pq int
 A number by which the balancing algorithm increments the load calculation for
 the purpose of selecting the least busy mirror member when an I/O operation
 lacks locality as defined by
 .Sy zfs_vdev_mirror_rotating_seek_offset .
 Operations within this that are not immediately following the previous operation
 are incremented by half.
 .
 .It Sy zfs_vdev_mirror_rotating_seek_offset Ns = Ns Sy 1048576 Ns B Po 1 MiB Pc Pq int
 The maximum distance for the last queued I/O operation in which
 the balancing algorithm considers an operation to have locality.
 .No See Sx ZFS I/O SCHEDULER .
 .
 .It Sy zfs_vdev_mirror_non_rotating_inc Ns = Ns Sy 0 Pq int
 A number by which the balancing algorithm increments the load calculation for
 the purpose of selecting the least busy mirror member on non-rotational vdevs
 when I/O operations do not immediately follow one another.
 .
 .It Sy zfs_vdev_mirror_non_rotating_seek_inc Ns = Ns Sy 1 Pq int
 A number by which the balancing algorithm increments the load calculation for
 the purpose of selecting the least busy mirror member when an I/O operation
 lacks
 locality as defined by the
 .Sy zfs_vdev_mirror_rotating_seek_offset .
 Operations within this that are not immediately following the previous operation
 are incremented by half.
 .
 .It Sy zfs_vdev_read_gap_limit Ns = Ns Sy 32768 Ns B Po 32 KiB Pc Pq uint
 Aggregate read I/O operations if the on-disk gap between them is within this
 threshold.
 .
 .It Sy zfs_vdev_write_gap_limit Ns = Ns Sy 4096 Ns B Po 4 KiB Pc Pq uint
 Aggregate write I/O operations if the on-disk gap between them is within this
 threshold.
 .
 .It Sy zfs_vdev_raidz_impl Ns = Ns Sy fastest Pq string
 Select the raidz parity implementation to use.
 .Pp
 Variants that don't depend on CPU-specific features
 may be selected on module load, as they are supported on all systems.
 The remaining options may only be set after the module is loaded,
 as they are available only if the implementations are compiled in
 and supported on the running system.
 .Pp
 Once the module is loaded,
 .Pa /sys/module/zfs/parameters/zfs_vdev_raidz_impl
 will show the available options,
 with the currently selected one enclosed in square brackets.
 .Pp
 .TS
 lb l l .
 fastest	selected by built-in benchmark
 original	original implementation
 scalar	scalar implementation
 sse2	SSE2 instruction set	64-bit x86
 ssse3	SSSE3 instruction set	64-bit x86
 avx2	AVX2 instruction set	64-bit x86
 avx512f	AVX512F instruction set	64-bit x86
 avx512bw	AVX512F & AVX512BW instruction sets	64-bit x86
 aarch64_neon	NEON	Aarch64/64-bit ARMv8
 aarch64_neonx2	NEON with more unrolling	Aarch64/64-bit ARMv8
 powerpc_altivec	Altivec	PowerPC
 .TE
 .
 .It Sy zfs_vdev_scheduler Pq charp
 .Sy DEPRECATED .
 Prints warning to kernel log for compatibility.
 .
 .It Sy zfs_zevent_len_max Ns = Ns Sy 512 Pq uint
 Max event queue length.
 Events in the queue can be viewed with
 .Xr zpool-events 8 .
 .
 .It Sy zfs_zevent_retain_max Ns = Ns Sy 2000 Pq int
 Maximum recent zevent records to retain for duplicate checking.
 Setting this to
 .Sy 0
 disables duplicate detection.
 .
 .It Sy zfs_zevent_retain_expire_secs Ns = Ns Sy 900 Ns s Po 15 min Pc Pq int
 Lifespan for a recent ereport that was retained for duplicate checking.
 .
 .It Sy zfs_zil_clean_taskq_maxalloc Ns = Ns Sy 1048576 Pq int
 The maximum number of taskq entries that are allowed to be cached.
 When this limit is exceeded transaction records (itxs)
 will be cleaned synchronously.
 .
 .It Sy zfs_zil_clean_taskq_minalloc Ns = Ns Sy 1024 Pq int
 The number of taskq entries that are pre-populated when the taskq is first
 created and are immediately available for use.
 .
 .It Sy zfs_zil_clean_taskq_nthr_pct Ns = Ns Sy 100 Ns % Pq int
 This controls the number of threads used by
 .Sy dp_zil_clean_taskq .
 The default value of
 .Sy 100%
 will create a maximum of one thread per cpu.
 .
 .It Sy zil_maxblocksize Ns = Ns Sy 131072 Ns B Po 128 KiB Pc Pq uint
 This sets the maximum block size used by the ZIL.
 On very fragmented pools, lowering this
 .Pq typically to Sy 36 KiB
 can improve performance.
 .
 .It Sy zil_maxcopied Ns = Ns Sy 7680 Ns B Po 7.5 KiB Pc Pq uint
 This sets the maximum number of write bytes logged via WR_COPIED.
 It tunes a tradeoff between additional memory copy and possibly worse log
 space efficiency vs additional range lock/unlock.
 .
 .It Sy zil_nocacheflush Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Disable the cache flush commands that are normally sent to disk by
 the ZIL after an LWB write has completed.
 Setting this will cause ZIL corruption on power loss
 if a volatile out-of-order write cache is enabled.
 .
 .It Sy zil_replay_disable Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Disable intent logging replay.
 Can be disabled for recovery from corrupted ZIL.
 .
 .It Sy zil_slog_bulk Ns = Ns Sy 67108864 Ns B Po 64 MiB Pc Pq u64
 Limit SLOG write size per commit executed with synchronous priority.
 Any writes above that will be executed with lower (asynchronous) priority
 to limit potential SLOG device abuse by single active ZIL writer.
 .
 .It Sy zfs_zil_saxattr Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Setting this tunable to zero disables ZIL logging of new
 .Sy xattr Ns = Ns Sy sa
 records if the
 .Sy org.openzfs:zilsaxattr
 feature is enabled on the pool.
 This would only be necessary to work around bugs in the ZIL logging or replay
 code for this record type.
 The tunable has no effect if the feature is disabled.
 .
 .It Sy zfs_embedded_slog_min_ms Ns = Ns Sy 64 Pq uint
 Usually, one metaslab from each normal-class vdev is dedicated for use by
 the ZIL to log synchronous writes.
 However, if there are fewer than
 .Sy zfs_embedded_slog_min_ms
 metaslabs in the vdev, this functionality is disabled.
 This ensures that we don't set aside an unreasonable amount of space for the
 ZIL.
 .
 .It Sy zstd_earlyabort_pass Ns = Ns Sy 1 Pq uint
 Whether heuristic for detection of incompressible data with zstd levels >= 3
 using LZ4 and zstd-1 passes is enabled.
 .
 .It Sy zstd_abort_size Ns = Ns Sy 131072 Pq uint
 Minimal uncompressed size (inclusive) of a record before the early abort
 heuristic will be attempted.
 .
 .It Sy zio_deadman_log_all Ns = Ns Sy 0 Ns | Ns 1 Pq int
 If non-zero, the zio deadman will produce debugging messages
 .Pq see Sy zfs_dbgmsg_enable
 for all zios, rather than only for leaf zios possessing a vdev.
 This is meant to be used by developers to gain
 diagnostic information for hang conditions which don't involve a mutex
 or other locking primitive: typically conditions in which a thread in
 the zio pipeline is looping indefinitely.
 .
 .It Sy zio_slow_io_ms Ns = Ns Sy 30000 Ns ms Po 30 s Pc Pq int
 When an I/O operation takes more than this much time to complete,
 it's marked as slow.
 Each slow operation causes a delay zevent.
 Slow I/O counters can be seen with
 .Nm zpool Cm status Fl s .
 .
 .It Sy zio_dva_throttle_enabled Ns = Ns Sy 1 Ns | Ns 0 Pq int
 Throttle block allocations in the I/O pipeline.
 This allows for dynamic allocation distribution when devices are imbalanced.
 When enabled, the maximum number of pending allocations per top-level vdev
 is limited by
 .Sy zfs_vdev_queue_depth_pct .
 .
 .It Sy zfs_xattr_compat Ns = Ns 0 Ns | Ns 1 Pq int
 Control the naming scheme used when setting new xattrs in the user namespace.
 If
 .Sy 0
 .Pq the default on Linux ,
 user namespace xattr names are prefixed with the namespace, to be backwards
 compatible with previous versions of ZFS on Linux.
 If
 .Sy 1
 .Pq the default on Fx ,
 user namespace xattr names are not prefixed, to be backwards compatible with
 previous versions of ZFS on illumos and
 .Fx .
 .Pp
 Either naming scheme can be read on this and future versions of ZFS, regardless
 of this tunable, but legacy ZFS on illumos or
 .Fx
 are unable to read user namespace xattrs written in the Linux format, and
 legacy versions of ZFS on Linux are unable to read user namespace xattrs written
 in the legacy ZFS format.
 .Pp
 An existing xattr with the alternate naming scheme is removed when overwriting
 the xattr so as to not accumulate duplicates.
 .
 .It Sy zio_requeue_io_start_cut_in_line Ns = Ns Sy 0 Ns | Ns 1 Pq int
 Prioritize requeued I/O.
 .
 .It Sy zio_taskq_batch_pct Ns = Ns Sy 80 Ns % Pq uint
 Percentage of online CPUs which will run a worker thread for I/O.
 These workers are responsible for I/O work such as compression, encryption,
 checksum and parity calculations.
 Fractional number of CPUs will be rounded down.
 .Pp
 The default value of
 .Sy 80%
 was chosen to avoid using all CPUs which can result in
 latency issues and inconsistent application performance,
 especially when slower compression and/or checksumming is enabled.
 Set value only applies to pools imported/created after that.
 .
 .It Sy zio_taskq_batch_tpq Ns = Ns Sy 0 Pq uint
 Number of worker threads per taskq.
 Lower values improve I/O ordering and CPU utilization,
 while higher reduces lock contention.
 .Pp
 If
 .Sy 0 ,
 generate a system-dependent value close to 6 threads per taskq.
 Set value only applies to pools imported/created after that.
 .
 .It Sy zio_taskq_read Ns = Ns Sy fixed,1,8 null scale null Pq charp
 Set the queue and thread configuration for the IO read queues.
 This is an advanced debugging parameter.
 Don't change this unless you understand what it does.
 Set values only apply to pools imported/created after that.
 .
 .It Sy zio_taskq_write Ns = Ns Sy batch fixed,1,5 scale fixed,1,5 Pq charp
 Set the queue and thread configuration for the IO write queues.
 This is an advanced debugging parameter.
 Don't change this unless you understand what it does.
 Set values only apply to pools imported/created after that.
 .
 .It Sy zvol_inhibit_dev Ns = Ns Sy 0 Ns | Ns 1 Pq uint
 Do not create zvol device nodes.
 This may slightly improve startup time on
 systems with a very large number of zvols.
 .
 .It Sy zvol_major Ns = Ns Sy 230 Pq uint
 Major number for zvol block devices.
 .
 .It Sy zvol_max_discard_blocks Ns = Ns Sy 16384 Pq long
 Discard (TRIM) operations done on zvols will be done in batches of this
 many blocks, where block size is determined by the
 .Sy volblocksize
 property of a zvol.
 .
 .It Sy zvol_prefetch_bytes Ns = Ns Sy 131072 Ns B Po 128 KiB Pc Pq uint
 When adding a zvol to the system, prefetch this many bytes
 from the start and end of the volume.
 Prefetching these regions of the volume is desirable,
 because they are likely to be accessed immediately by
 .Xr blkid 8
 or the kernel partitioner.
 .
 .It Sy zvol_request_sync Ns = Ns Sy 0 Ns | Ns 1 Pq uint
 When processing I/O requests for a zvol, submit them synchronously.
 This effectively limits the queue depth to
 .Em 1
 for each I/O submitter.
 When unset, requests are handled asynchronously by a thread pool.
 The number of requests which can be handled concurrently is controlled by
 .Sy zvol_threads .
 .Sy zvol_request_sync
 is ignored when running on a kernel that supports block multiqueue
 .Pq Li blk-mq .
 .
 .It Sy zvol_num_taskqs Ns = Ns Sy 0 Pq uint
 Number of zvol taskqs.
 If
 .Sy 0
 (the default) then scaling is done internally to prefer 6 threads per taskq.
 This only applies on Linux.
 .
 .It Sy zvol_threads Ns = Ns Sy 0 Pq uint
 The number of system wide threads to use for processing zvol block IOs.
 If
 .Sy 0
 (the default) then internally set
 .Sy zvol_threads
 to the number of CPUs present or 32 (whichever is greater).
 .
 .It Sy zvol_blk_mq_threads Ns = Ns Sy 0 Pq uint
 The number of threads per zvol to use for queuing IO requests.
 This parameter will only appear if your kernel supports
 .Li blk-mq
 and is only read and assigned to a zvol at zvol load time.
 If
 .Sy 0
 (the default) then internally set
 .Sy zvol_blk_mq_threads
 to the number of CPUs present.
 .
 .It Sy zvol_use_blk_mq Ns = Ns Sy 0 Ns | Ns 1 Pq uint
 Set to
 .Sy 1
 to use the
 .Li blk-mq
 API for zvols.
 Set to
 .Sy 0
 (the default) to use the legacy zvol APIs.
 This setting can give better or worse zvol performance depending on
 the workload.
 This parameter will only appear if your kernel supports
 .Li blk-mq
 and is only read and assigned to a zvol at zvol load time.
 .
 .It Sy zvol_blk_mq_blocks_per_thread Ns = Ns Sy 8 Pq uint
 If
 .Sy zvol_use_blk_mq
 is enabled, then process this number of
 .Sy volblocksize Ns -sized blocks per zvol thread.
 This tunable can be use to favor better performance for zvol reads (lower
 values) or writes (higher values).
 If set to
 .Sy 0 ,
 then the zvol layer will process the maximum number of blocks
 per thread that it can.
 This parameter will only appear if your kernel supports
 .Li blk-mq
 and is only applied at each zvol's load time.
 .
 .It Sy zvol_blk_mq_queue_depth Ns = Ns Sy 0 Pq uint
 The queue_depth value for the zvol
 .Li blk-mq
 interface.
 This parameter will only appear if your kernel supports
 .Li blk-mq
 and is only applied at each zvol's load time.
 If
 .Sy 0
 (the default) then use the kernel's default queue depth.
 Values are clamped to the kernel's
 .Dv BLKDEV_MIN_RQ
 and
 .Dv BLKDEV_MAX_RQ Ns / Ns Dv BLKDEV_DEFAULT_RQ
 limits.
 .
 .It Sy zvol_volmode Ns = Ns Sy 1 Pq uint
 Defines zvol block devices behaviour when
 .Sy volmode Ns = Ns Sy default :
 .Bl -tag -compact -offset 4n -width "a"
 .It Sy 1
 .No equivalent to Sy full
 .It Sy 2
 .No equivalent to Sy dev
 .It Sy 3
 .No equivalent to Sy none
 .El
 .
 .It Sy zvol_enforce_quotas Ns = Ns Sy 0 Ns | Ns 1 Pq uint
 Enable strict ZVOL quota enforcement.
 The strict quota enforcement may have a performance impact.
 .El
 .
 .Sh ZFS I/O SCHEDULER
 ZFS issues I/O operations to leaf vdevs to satisfy and complete I/O operations.
 The scheduler determines when and in what order those operations are issued.
 The scheduler divides operations into five I/O classes,
 prioritized in the following order: sync read, sync write, async read,
 async write, and scrub/resilver.
 Each queue defines the minimum and maximum number of concurrent operations
 that may be issued to the device.
 In addition, the device has an aggregate maximum,
 .Sy zfs_vdev_max_active .
 Note that the sum of the per-queue minima must not exceed the aggregate maximum.
 If the sum of the per-queue maxima exceeds the aggregate maximum,
 then the number of active operations may reach
 .Sy zfs_vdev_max_active ,
 in which case no further operations will be issued,
 regardless of whether all per-queue minima have been met.
 .Pp
 For many physical devices, throughput increases with the number of
 concurrent operations, but latency typically suffers.
 Furthermore, physical devices typically have a limit
 at which more concurrent operations have no
 effect on throughput or can actually cause it to decrease.
 .Pp
 The scheduler selects the next operation to issue by first looking for an
 I/O class whose minimum has not been satisfied.
 Once all are satisfied and the aggregate maximum has not been hit,
 the scheduler looks for classes whose maximum has not been satisfied.
 Iteration through the I/O classes is done in the order specified above.
 No further operations are issued
 if the aggregate maximum number of concurrent operations has been hit,
 or if there are no operations queued for an I/O class that has not hit its
 maximum.
 Every time an I/O operation is queued or an operation completes,
 the scheduler looks for new operations to issue.
 .Pp
 In general, smaller
 .Sy max_active Ns s
 will lead to lower latency of synchronous operations.
 Larger
 .Sy max_active Ns s
 may lead to higher overall throughput, depending on underlying storage.
 .Pp
 The ratio of the queues'
 .Sy max_active Ns s
 determines the balance of performance between reads, writes, and scrubs.
 For example, increasing
 .Sy zfs_vdev_scrub_max_active
 will cause the scrub or resilver to complete more quickly,
 but reads and writes to have higher latency and lower throughput.
 .Pp
 All I/O classes have a fixed maximum number of outstanding operations,
 except for the async write class.
 Asynchronous writes represent the data that is committed to stable storage
 during the syncing stage for transaction groups.
 Transaction groups enter the syncing state periodically,
 so the number of queued async writes will quickly burst up
 and then bleed down to zero.
 Rather than servicing them as quickly as possible,
 the I/O scheduler changes the maximum number of active async write operations
 according to the amount of dirty data in the pool.
 Since both throughput and latency typically increase with the number of
 concurrent operations issued to physical devices, reducing the
 burstiness in the number of simultaneous operations also stabilizes the
 response time of operations from other queues, in particular synchronous ones.
 In broad strokes, the I/O scheduler will issue more concurrent operations
 from the async write queue as there is more dirty data in the pool.
 .
 .Ss Async Writes
 The number of concurrent operations issued for the async write I/O class
 follows a piece-wise linear function defined by a few adjustable points:
 .Bd -literal
        |              o---------| <-- \fBzfs_vdev_async_write_max_active\fP
   ^    |             /^         |
   |    |            / |         |
 active |           /  |         |
  I/O   |          /   |         |
 count  |         /    |         |
        |        /     |         |
        |-------o      |         | <-- \fBzfs_vdev_async_write_min_active\fP
       0|_______^______|_________|
        0%      |      |       100% of \fBzfs_dirty_data_max\fP
                |      |
                |      `-- \fBzfs_vdev_async_write_active_max_dirty_percent\fP
                `--------- \fBzfs_vdev_async_write_active_min_dirty_percent\fP
 .Ed
 .Pp
 Until the amount of dirty data exceeds a minimum percentage of the dirty
 data allowed in the pool, the I/O scheduler will limit the number of
 concurrent operations to the minimum.
 As that threshold is crossed, the number of concurrent operations issued
 increases linearly to the maximum at the specified maximum percentage
 of the dirty data allowed in the pool.
 .Pp
 Ideally, the amount of dirty data on a busy pool will stay in the sloped
 part of the function between
 .Sy zfs_vdev_async_write_active_min_dirty_percent
 and
 .Sy zfs_vdev_async_write_active_max_dirty_percent .
 If it exceeds the maximum percentage,
 this indicates that the rate of incoming data is
 greater than the rate that the backend storage can handle.
 In this case, we must further throttle incoming writes,
 as described in the next section.
 .
 .Sh ZFS TRANSACTION DELAY
 We delay transactions when we've determined that the backend storage
 isn't able to accommodate the rate of incoming writes.
 .Pp
 If there is already a transaction waiting, we delay relative to when
 that transaction will finish waiting.
 This way the calculated delay time
 is independent of the number of threads concurrently executing transactions.
 .Pp
 If we are the only waiter, wait relative to when the transaction started,
 rather than the current time.
 This credits the transaction for "time already served",
 e.g. reading indirect blocks.
 .Pp
 The minimum time for a transaction to take is calculated as
 .D1 min_time = min( Ns Sy zfs_delay_scale No \(mu Po Sy dirty No \- Sy min Pc / Po Sy max No \- Sy dirty Pc , 100ms)
 .Pp
 The delay has two degrees of freedom that can be adjusted via tunables.
 The percentage of dirty data at which we start to delay is defined by
 .Sy zfs_delay_min_dirty_percent .
 This should typically be at or above
 .Sy zfs_vdev_async_write_active_max_dirty_percent ,
 so that we only start to delay after writing at full speed
 has failed to keep up with the incoming write rate.
 The scale of the curve is defined by
 .Sy zfs_delay_scale .
 Roughly speaking, this variable determines the amount of delay at the midpoint
 of the curve.
 .Bd -literal
 delay
  10ms +-------------------------------------------------------------*+
       |                                                             *|
   9ms +                                                             *+
       |                                                             *|
   8ms +                                                             *+
       |                                                            * |
   7ms +                                                            * +
       |                                                            * |
   6ms +                                                            * +
       |                                                            * |
   5ms +                                                           *  +
       |                                                           *  |
   4ms +                                                           *  +
       |                                                           *  |
   3ms +                                                          *   +
       |                                                          *   |
   2ms +                                              (midpoint) *    +
       |                                                  |    **     |
   1ms +                                                  v ***       +
       |             \fBzfs_delay_scale\fP ---------->     ********         |
     0 +-------------------------------------*********----------------+
       0%                    <- \fBzfs_dirty_data_max\fP ->               100%
 .Ed
 .Pp
 Note, that since the delay is added to the outstanding time remaining on the
 most recent transaction it's effectively the inverse of IOPS.
 Here, the midpoint of
 .Em 500 us
 translates to
 .Em 2000 IOPS .
 The shape of the curve
 was chosen such that small changes in the amount of accumulated dirty data
 in the first three quarters of the curve yield relatively small differences
 in the amount of delay.
 .Pp
 The effects can be easier to understand when the amount of delay is
 represented on a logarithmic scale:
 .Bd -literal
 delay
 100ms +-------------------------------------------------------------++
       +                                                              +
       |                                                              |
       +                                                             *+
  10ms +                                                             *+
       +                                                           ** +
       |                                              (midpoint)  **  |
       +                                                  |     **    +
   1ms +                                                  v ****      +
       +             \fBzfs_delay_scale\fP ---------->        *****         +
       |                                             ****             |
       +                                          ****                +
 100us +                                        **                    +
       +                                       *                      +
       |                                      *                       |
       +                                     *                        +
  10us +                                     *                        +
       +                                                              +
       |                                                              |
       +                                                              +
       +--------------------------------------------------------------+
       0%                    <- \fBzfs_dirty_data_max\fP ->               100%
 .Ed
 .Pp
 Note here that only as the amount of dirty data approaches its limit does
 the delay start to increase rapidly.
 The goal of a properly tuned system should be to keep the amount of dirty data
 out of that range by first ensuring that the appropriate limits are set
 for the I/O scheduler to reach optimal throughput on the back-end storage,
 and then by changing the value of
 .Sy zfs_delay_scale
 to increase the steepness of the curve.
diff --git a/sys/contrib/openzfs/module/Kbuild.in b/sys/contrib/openzfs/module/Kbuild.in
index 4afd064930a3..e7790c6b920d 100644
--- a/sys/contrib/openzfs/module/Kbuild.in
+++ b/sys/contrib/openzfs/module/Kbuild.in
@@ -1,508 +1,495 @@
 # When integrated in to a monolithic kernel the spl module must appear
 # first.  This ensures its module initialization function is run before
 # any of the other module initialization functions which depend on it.
 
 ZFS_MODULE_CFLAGS += -std=gnu99 -Wno-declaration-after-statement
 ZFS_MODULE_CFLAGS += -Wmissing-prototypes
 ZFS_MODULE_CFLAGS += @KERNEL_DEBUG_CFLAGS@  @NO_FORMAT_ZERO_LENGTH@
 
 ifneq ($(KBUILD_EXTMOD),)
 zfs_include = @abs_top_srcdir@/include
 icp_include = @abs_srcdir@/icp/include
 zstd_include = @abs_srcdir@/zstd/include
 ZFS_MODULE_CFLAGS += -include @abs_top_builddir@/zfs_config.h
 ZFS_MODULE_CFLAGS += -I@abs_top_builddir@/include
 src = @abs_srcdir@
 obj = @abs_builddir@
 else
 zfs_include = $(srctree)/include/zfs
 icp_include = $(src)/icp/include
 zstd_include = $(src)/zstd/include
 ZFS_MODULE_CFLAGS += -include $(zfs_include)/zfs_config.h
 endif
 
 ZFS_MODULE_CFLAGS += -I$(zfs_include)/os/linux/kernel
 ZFS_MODULE_CFLAGS += -I$(zfs_include)/os/linux/spl
 ZFS_MODULE_CFLAGS += -I$(zfs_include)/os/linux/zfs
 ZFS_MODULE_CFLAGS += -I$(zfs_include)
 ZFS_MODULE_CPPFLAGS += -D_KERNEL
 ZFS_MODULE_CPPFLAGS += @KERNEL_DEBUG_CPPFLAGS@
 
 # KASAN enables -Werror=frame-larger-than=1024, which
 # breaks oh so many parts of our build.
 ifeq ($(CONFIG_KASAN),y)
 ZFS_MODULE_CFLAGS += -Wno-error=frame-larger-than=
 endif
 
 # Generated binary search code is particularly bad with this optimization.
 # Oddly, range_tree.c is not affected when unrolling is not done and dsl_scan.c
 # is not affected when unrolling is done.
 # Disable it until the following upstream issue is resolved:
 # https://github.com/llvm/llvm-project/issues/62790
 ifeq ($(CONFIG_X86),y)
 ifeq ($(CONFIG_CC_IS_CLANG),y)
 CFLAGS_zfs/dsl_scan.o += -mllvm -x86-cmov-converter=false
 CFLAGS_zfs/metaslab.o += -mllvm -x86-cmov-converter=false
 CFLAGS_zfs/range_tree.o += -mllvm -x86-cmov-converter=false
 CFLAGS_zfs/zap_micro.o += -mllvm -x86-cmov-converter=false
 endif
 endif
 
 ifneq ($(KBUILD_EXTMOD),)
 @CONFIG_QAT_TRUE@ZFS_MODULE_CFLAGS += -I@QAT_SRC@/include
 @CONFIG_QAT_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@
 endif
 
 asflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
 ccflags-y := $(ZFS_MODULE_CFLAGS) $(ZFS_MODULE_CPPFLAGS)
 
 ifeq ($(CONFIG_ARM64),y)
 CFLAGS_REMOVE_zcommon/zfs_fletcher_aarch64_neon.o += -mgeneral-regs-only
 CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neon.o += -mgeneral-regs-only
 CFLAGS_REMOVE_zfs/vdev_raidz_math_aarch64_neonx2.o += -mgeneral-regs-only
 endif
 
 # Suppress unused-value warnings in sparc64 architecture headers
 ccflags-$(CONFIG_SPARC64) += -Wno-unused-value
 
 
 obj-$(CONFIG_ZFS) := spl.o zfs.o
 
 SPL_OBJS := \
 	spl-atomic.o \
 	spl-condvar.o \
 	spl-cred.o \
 	spl-err.o \
 	spl-generic.o \
 	spl-kmem-cache.o \
 	spl-kmem.o \
 	spl-kstat.o \
 	spl-proc.o \
 	spl-procfs-list.o \
 	spl-shrinker.o \
 	spl-taskq.o \
 	spl-thread.o \
 	spl-trace.o \
 	spl-tsd.o \
 	spl-vmem.o \
 	spl-xdr.o \
 	spl-zlib.o \
 	spl-zone.o
 
 spl-objs += $(addprefix os/linux/spl/,$(SPL_OBJS))
 
 zfs-objs += avl/avl.o
 
 ICP_OBJS := \
 	algs/aes/aes_impl.o \
 	algs/aes/aes_impl_generic.o \
 	algs/aes/aes_modes.o \
 	algs/blake3/blake3.o \
 	algs/blake3/blake3_generic.o \
 	algs/blake3/blake3_impl.o \
 	algs/edonr/edonr.o \
 	algs/modes/cbc.o \
 	algs/modes/ccm.o \
 	algs/modes/ctr.o \
 	algs/modes/ecb.o \
 	algs/modes/gcm.o \
 	algs/modes/gcm_generic.o \
 	algs/modes/modes.o \
 	algs/sha2/sha2_generic.o \
 	algs/sha2/sha256_impl.o \
 	algs/sha2/sha512_impl.o \
 	algs/skein/skein.o \
 	algs/skein/skein_block.o \
 	algs/skein/skein_iv.o \
 	api/kcf_cipher.o \
 	api/kcf_ctxops.o \
 	api/kcf_mac.o \
 	core/kcf_callprov.o \
 	core/kcf_mech_tabs.o \
 	core/kcf_prov_lib.o \
 	core/kcf_prov_tabs.o \
 	core/kcf_sched.o \
 	illumos-crypto.o \
 	io/aes.o \
 	io/sha2_mod.o \
 	io/skein_mod.o \
 	spi/kcf_spi.o
 
 ICP_OBJS_X86_64 := \
 	asm-x86_64/aes/aes_aesni.o \
 	asm-x86_64/aes/aes_amd64.o \
 	asm-x86_64/aes/aeskey.o \
 	asm-x86_64/blake3/blake3_avx2.o \
 	asm-x86_64/blake3/blake3_avx512.o \
 	asm-x86_64/blake3/blake3_sse2.o \
 	asm-x86_64/blake3/blake3_sse41.o \
 	asm-x86_64/sha2/sha256-x86_64.o \
 	asm-x86_64/sha2/sha512-x86_64.o \
 	asm-x86_64/modes/aesni-gcm-x86_64.o \
 	asm-x86_64/modes/gcm_pclmulqdq.o \
 	asm-x86_64/modes/ghash-x86_64.o
 
 ICP_OBJS_X86 := \
 	algs/aes/aes_impl_aesni.o \
 	algs/aes/aes_impl_x86-64.o \
 	algs/modes/gcm_pclmulqdq.o
 
 ICP_OBJS_ARM := \
 	asm-arm/sha2/sha256-armv7.o \
 	asm-arm/sha2/sha512-armv7.o
 
 ICP_OBJS_ARM64 := \
 	asm-aarch64/blake3/b3_aarch64_sse2.o \
 	asm-aarch64/blake3/b3_aarch64_sse41.o \
 	asm-aarch64/sha2/sha256-armv8.o \
 	asm-aarch64/sha2/sha512-armv8.o
 
 ICP_OBJS_PPC_PPC64 := \
 	asm-ppc64/blake3/b3_ppc64le_sse2.o \
 	asm-ppc64/blake3/b3_ppc64le_sse41.o \
 	asm-ppc64/sha2/sha256-p8.o \
 	asm-ppc64/sha2/sha512-p8.o \
 	asm-ppc64/sha2/sha256-ppc.o \
 	asm-ppc64/sha2/sha512-ppc.o
 
 zfs-objs             += $(addprefix icp/,$(ICP_OBJS))
 zfs-$(CONFIG_X86)    += $(addprefix icp/,$(ICP_OBJS_X86))
 zfs-$(CONFIG_UML_X86)+= $(addprefix icp/,$(ICP_OBJS_X86))
 zfs-$(CONFIG_X86_64) += $(addprefix icp/,$(ICP_OBJS_X86_64))
 zfs-$(CONFIG_ARM)    += $(addprefix icp/,$(ICP_OBJS_ARM))
 zfs-$(CONFIG_ARM64)  += $(addprefix icp/,$(ICP_OBJS_ARM64))
 zfs-$(CONFIG_PPC)    += $(addprefix icp/,$(ICP_OBJS_PPC_PPC64))
 zfs-$(CONFIG_PPC64)  += $(addprefix icp/,$(ICP_OBJS_PPC_PPC64))
 
 $(addprefix $(obj)/icp/,$(ICP_OBJS) $(ICP_OBJS_X86) $(ICP_OBJS_X86_64) \
 	$(ICP_OBJS_ARM64) $(ICP_OBJS_PPC_PPC64)) : asflags-y += -I$(icp_include) -I$(zfs_include)/os/linux/spl -I$(zfs_include)
 
 $(addprefix $(obj)/icp/,$(ICP_OBJS) $(ICP_OBJS_X86) $(ICP_OBJS_X86_64) \
 	$(ICP_OBJS_ARM64) $(ICP_OBJS_PPC_PPC64)) : ccflags-y += -I$(icp_include) -I$(zfs_include)/os/linux/spl -I$(zfs_include)
 
-# Suppress objtool "return with modified stack frame" warnings.
-OBJECT_FILES_NON_STANDARD_aesni-gcm-x86_64.o := y
-
-# Suppress objtool "unsupported stack pointer realignment" warnings.
-# See #6950 for the reasoning.
-OBJECT_FILES_NON_STANDARD_sha256-x86_64.o := y
-OBJECT_FILES_NON_STANDARD_sha512-x86_64.o := y
-
 LUA_OBJS := \
 	lapi.o \
 	lauxlib.o \
 	lbaselib.o \
 	lcode.o \
 	lcompat.o \
 	lcorolib.o \
 	lctype.o \
 	ldebug.o \
 	ldo.o \
 	lfunc.o \
 	lgc.o \
 	llex.o \
 	lmem.o \
 	lobject.o \
 	lopcodes.o \
 	lparser.o \
 	lstate.o \
 	lstring.o \
 	lstrlib.o \
 	ltable.o \
 	ltablib.o \
 	ltm.o \
 	lvm.o \
 	lzio.o \
 	setjmp/setjmp.o
 
 zfs-objs += $(addprefix lua/,$(LUA_OBJS))
 
 
 NVPAIR_OBJS := \
 	fnvpair.o \
 	nvpair.o \
 	nvpair_alloc_fixed.o \
 	nvpair_alloc_spl.o
 
 zfs-objs += $(addprefix nvpair/,$(NVPAIR_OBJS))
 
 
 UNICODE_OBJS := \
 	u8_textprep.o \
 	uconv.o
 
 zfs-objs += $(addprefix unicode/,$(UNICODE_OBJS))
 
 
 ZCOMMON_OBJS := \
 	cityhash.o \
 	simd_stat.o \
 	zfeature_common.o \
 	zfs_comutil.o \
 	zfs_deleg.o \
 	zfs_fletcher.o \
 	zfs_fletcher_superscalar.o \
 	zfs_fletcher_superscalar4.o \
 	zfs_namecheck.o \
 	zfs_prop.o \
 	zfs_valstr.o \
 	zpool_prop.o \
 	zprop_common.o
 
 ZCOMMON_OBJS_X86 := \
 	zfs_fletcher_avx512.o \
 	zfs_fletcher_intel.o \
 	zfs_fletcher_sse.o
 
 ZCOMMON_OBJS_ARM64 := \
 	zfs_fletcher_aarch64_neon.o
 
 zfs-objs            += $(addprefix zcommon/,$(ZCOMMON_OBJS))
 zfs-$(CONFIG_X86)   += $(addprefix zcommon/,$(ZCOMMON_OBJS_X86))
 zfs-$(CONFIG_UML_X86)+= $(addprefix zcommon/,$(ZCOMMON_OBJS_X86))
 zfs-$(CONFIG_ARM64) += $(addprefix zcommon/,$(ZCOMMON_OBJS_ARM64))
 
 
 # Zstd uses -O3 by default, so we should follow
 ZFS_ZSTD_FLAGS := -O3
 
 # -fno-tree-vectorize gets set for gcc in zstd/common/compiler.h
 # Set it for other compilers, too.
 ZFS_ZSTD_FLAGS += -fno-tree-vectorize
 
 # SSE register return with SSE disabled if -march=znverX is passed
 ZFS_ZSTD_FLAGS += -U__BMI__
 
 # Quiet warnings about frame size due to unused code in unmodified zstd lib
 ZFS_ZSTD_FLAGS += -Wframe-larger-than=20480
 
 ZSTD_OBJS := \
 	zfs_zstd.o \
 	zstd_sparc.o
 
 ZSTD_UPSTREAM_OBJS := \
 	lib/common/entropy_common.o \
 	lib/common/error_private.o \
 	lib/common/fse_decompress.o \
 	lib/common/pool.o \
 	lib/common/zstd_common.o \
 	lib/compress/fse_compress.o \
 	lib/compress/hist.o \
 	lib/compress/huf_compress.o \
 	lib/compress/zstd_compress.o \
 	lib/compress/zstd_compress_literals.o \
 	lib/compress/zstd_compress_sequences.o \
 	lib/compress/zstd_compress_superblock.o \
 	lib/compress/zstd_double_fast.o \
 	lib/compress/zstd_fast.o \
 	lib/compress/zstd_lazy.o \
 	lib/compress/zstd_ldm.o \
 	lib/compress/zstd_opt.o \
 	lib/decompress/huf_decompress.o \
 	lib/decompress/zstd_ddict.o \
 	lib/decompress/zstd_decompress.o \
 	lib/decompress/zstd_decompress_block.o
 
 zfs-objs += $(addprefix zstd/,$(ZSTD_OBJS) $(ZSTD_UPSTREAM_OBJS))
 
 # Disable aarch64 neon SIMD instructions for kernel mode
 $(addprefix $(obj)/zstd/,$(ZSTD_OBJS) $(ZSTD_UPSTREAM_OBJS)) : ccflags-y += -I$(zstd_include) $(ZFS_ZSTD_FLAGS)
 $(addprefix $(obj)/zstd/,$(ZSTD_OBJS) $(ZSTD_UPSTREAM_OBJS)) : asflags-y += -I$(zstd_include)
 $(addprefix $(obj)/zstd/,$(ZSTD_UPSTREAM_OBJS)) : ccflags-y += -include $(zstd_include)/aarch64_compat.h -include $(zstd_include)/zstd_compat_wrapper.h -Wp,-w
 $(obj)/zstd/zfs_zstd.o : ccflags-y += -include $(zstd_include)/zstd_compat_wrapper.h
 
 
 ZFS_OBJS := \
 	abd.o \
 	aggsum.o \
 	arc.o \
 	blake3_zfs.o \
 	blkptr.o \
 	bplist.o \
 	bpobj.o \
 	bptree.o \
 	bqueue.o \
 	brt.o \
 	btree.o \
 	dataset_kstats.o \
 	dbuf.o \
 	dbuf_stats.o \
 	ddt.o \
 	ddt_zap.o \
 	dmu.o \
 	dmu_diff.o \
 	dmu_object.o \
 	dmu_objset.o \
 	dmu_recv.o \
 	dmu_redact.o \
 	dmu_send.o \
 	dmu_traverse.o \
 	dmu_tx.o \
 	dmu_zfetch.o \
 	dnode.o \
 	dnode_sync.o \
 	dsl_bookmark.o \
 	dsl_crypt.o \
 	dsl_dataset.o \
 	dsl_deadlist.o \
 	dsl_deleg.o \
 	dsl_destroy.o \
 	dsl_dir.o \
 	dsl_pool.o \
 	dsl_prop.o \
 	dsl_scan.o \
 	dsl_synctask.o \
 	dsl_userhold.o \
 	edonr_zfs.o \
 	fm.o \
 	gzip.o \
 	hkdf.o \
 	lz4.o \
 	lz4_zfs.o \
 	lzjb.o \
 	metaslab.o \
 	mmp.o \
 	multilist.o \
 	objlist.o \
 	pathname.o \
 	range_tree.o \
 	refcount.o \
 	rrwlock.o \
 	sa.o \
 	sha2_zfs.o \
 	skein_zfs.o \
 	spa.o \
 	spa_checkpoint.o \
 	spa_config.o \
 	spa_errlog.o \
 	spa_history.o \
 	spa_log_spacemap.o \
 	spa_misc.o \
 	spa_stats.o \
 	space_map.o \
 	space_reftree.o \
 	txg.o \
 	uberblock.o \
 	unique.o \
 	vdev.o \
 	vdev_draid.o \
 	vdev_draid_rand.o \
 	vdev_indirect.o \
 	vdev_indirect_births.o \
 	vdev_indirect_mapping.o \
 	vdev_initialize.o \
 	vdev_label.o \
 	vdev_mirror.o \
 	vdev_missing.o \
 	vdev_queue.o \
 	vdev_raidz.o \
 	vdev_raidz_math.o \
 	vdev_raidz_math_scalar.o \
 	vdev_rebuild.o \
 	vdev_removal.o \
 	vdev_root.o \
 	vdev_trim.o \
 	zap.o \
 	zap_leaf.o \
 	zap_micro.o \
 	zcp.o \
 	zcp_get.o \
 	zcp_global.o \
 	zcp_iter.o \
 	zcp_set.o \
 	zcp_synctask.o \
 	zfeature.o \
 	zfs_byteswap.o \
 	zfs_chksum.o \
 	zfs_fm.o \
 	zfs_fuid.o \
 	zfs_impl.o \
 	zfs_ioctl.o \
 	zfs_log.o \
 	zfs_onexit.o \
 	zfs_quota.o \
 	zfs_ratelimit.o \
 	zfs_replay.o \
 	zfs_rlock.o \
 	zfs_sa.o \
 	zfs_vnops.o \
 	zfs_znode.o \
 	zil.o \
 	zio.o \
 	zio_checksum.o \
 	zio_compress.o \
 	zio_inject.o \
 	zle.o \
 	zrlock.o \
 	zthr.o \
 	zvol.o
 
 ZFS_OBJS_OS := \
 	abd_os.o \
 	arc_os.o \
 	mmp_os.o \
 	policy.o \
 	qat.o \
 	qat_compress.o \
 	qat_crypt.o \
 	spa_misc_os.o \
 	trace.o \
 	vdev_disk.o \
 	vdev_file.o \
 	zfs_acl.o \
 	zfs_ctldir.o \
 	zfs_debug.o \
 	zfs_dir.o \
 	zfs_file_os.o \
 	zfs_ioctl_os.o \
 	zfs_racct.o \
 	zfs_sysfs.o \
 	zfs_uio.o \
 	zfs_vfsops.o \
 	zfs_vnops_os.o \
 	zfs_znode_os.o \
 	zio_crypt.o \
 	zpl_ctldir.o \
 	zpl_export.o \
 	zpl_file.o \
 	zpl_file_range.o \
 	zpl_inode.o \
 	zpl_super.o \
 	zpl_xattr.o \
 	zvol_os.o
 
 ZFS_OBJS_X86 := \
 	vdev_raidz_math_avx2.o \
 	vdev_raidz_math_avx512bw.o \
 	vdev_raidz_math_avx512f.o \
 	vdev_raidz_math_sse2.o \
 	vdev_raidz_math_ssse3.o
 
 ZFS_OBJS_ARM64 := \
 	vdev_raidz_math_aarch64_neon.o \
 	vdev_raidz_math_aarch64_neonx2.o
 
 ZFS_OBJS_PPC_PPC64 := \
 	vdev_raidz_math_powerpc_altivec.o
 
 zfs-objs            += $(addprefix zfs/,$(ZFS_OBJS)) $(addprefix os/linux/zfs/,$(ZFS_OBJS_OS))
 zfs-$(CONFIG_X86)   += $(addprefix zfs/,$(ZFS_OBJS_X86))
 zfs-$(CONFIG_UML_X86)+= $(addprefix zfs/,$(ZFS_OBJS_X86))
 zfs-$(CONFIG_ARM64) += $(addprefix zfs/,$(ZFS_OBJS_ARM64))
 zfs-$(CONFIG_PPC)   += $(addprefix zfs/,$(ZFS_OBJS_PPC_PPC64))
 zfs-$(CONFIG_PPC64) += $(addprefix zfs/,$(ZFS_OBJS_PPC_PPC64))
 
 UBSAN_SANITIZE_zap_leaf.o := n
 UBSAN_SANITIZE_zap_micro.o := n
 UBSAN_SANITIZE_sa.o := n
 UBSAN_SANITIZE_zfs/zap_micro.o := n
 UBSAN_SANITIZE_zfs/sa.o := n
 
-# Suppress incorrect warnings from versions of objtool which are not
-# aware of x86 EVEX prefix instructions used for AVX512.
-OBJECT_FILES_NON_STANDARD_vdev_raidz_math_avx512bw.o := y
-OBJECT_FILES_NON_STANDARD_vdev_raidz_math_avx512f.o := y
-
 ifeq ($(CONFIG_ALTIVEC),y)
 $(obj)/zfs/vdev_raidz_math_powerpc_altivec.o : c_flags += -maltivec
 endif
diff --git a/sys/contrib/openzfs/module/Makefile.in b/sys/contrib/openzfs/module/Makefile.in
index 9b34b3dfaec7..529ab81dcec5 100644
--- a/sys/contrib/openzfs/module/Makefile.in
+++ b/sys/contrib/openzfs/module/Makefile.in
@@ -1,171 +1,173 @@
 include Kbuild
 
 INSTALL_MOD_DIR ?= extra
 INSTALL_MOD_PATH ?= $(DESTDIR)
 
 all: modules
 distclean maintainer-clean: clean
 install: modules_install data_install
 uninstall: modules_uninstall data_uninstall
 check:
 
 .PHONY: all distclean maintainer-clean install uninstall check distdir \
 	modules modules-Linux modules-FreeBSD modules-unknown \
 	clean clean-Linux clean-FreeBSD \
 	modules_install modules_install-Linux modules_install-FreeBSD \
 	data_install data_install-Linux data_install-FreeBSD \
 	modules_uninstall modules_uninstall-Linux modules_uninstall-FreeBSD \
 	data_uninstall data_uninstall-Linux data_uninstall-FreeBSD \
 	cppcheck cppcheck-Linux cppcheck-FreeBSD
 
 # For FreeBSD, use debug options from ./configure if not overridden.
 export WITH_DEBUG ?= @WITH_DEBUG@
 export WITH_INVARIANTS ?= @WITH_INVARIANTS@
 
 # Filter out options that FreeBSD make doesn't understand
 getflags = ( \
 set -- \
   $(filter-out --%,$(firstword $(MFLAGS))) \
   $(filter -I%,$(MFLAGS)) \
   $(filter -j%,$(MFLAGS)); \
 fmakeflags=""; \
 while getopts :deiI:j:knqrstw flag; do \
   case $$flag in \
     \?) :;; \
     :) if [ $$OPTARG = "j" ]; then \
 	 ncpus=$$(sysctl -n kern.smp.cpus 2>/dev/null || :); \
 	 if [ -n "$$ncpus" ]; then fmakeflags="$$fmakeflags -j$$ncpus"; fi; \
        fi;; \
     d) fmakeflags="$$fmakeflags -dA";; \
     *) fmakeflags="$$fmakeflags -$$flag$$OPTARG";; \
   esac; \
 done; \
 echo $$fmakeflags \
 )
 FMAKEFLAGS = -C @abs_srcdir@ -f Makefile.bsd $(shell $(getflags))
 
 ifneq (@abs_srcdir@,@abs_builddir@)
 FMAKEFLAGS += MAKEOBJDIR=@abs_builddir@
 endif
 
 FMAKE = env -u MAKEFLAGS make $(FMAKEFLAGS)
 
 modules-Linux:
 	mkdir -p $(sort $(dir $(spl-objs) $(spl-)))
 	mkdir -p $(sort $(dir $(zfs-objs) $(zfs-)))
 	$(MAKE) -C @LINUX_OBJ@ $(if @KERNEL_CC@,CC=@KERNEL_CC@) \
 		$(if @KERNEL_LD@,LD=@KERNEL_LD@) $(if @KERNEL_LLVM@,LLVM=@KERNEL_LLVM@) \
+		$(if @KERNEL_CROSS_COMPILE@,CROSS_COMPILE=@KERNEL_CROSS_COMPILE@) \
+		$(if @KERNEL_ARCH@,ARCH=@KERNEL_ARCH@) \
 		M="$$PWD" @KERNEL_MAKE@ CONFIG_ZFS=m modules
 
 modules-FreeBSD:
 	+$(FMAKE)
 
 modules-unknown:
 	@true
 
 modules: modules-@ac_system@
 
 clean-Linux:
 	@# Only cleanup the kernel build directories when CONFIG_KERNEL
 	@# is defined.  This indicates that kernel modules should be built.
 @CONFIG_KERNEL_TRUE@	$(MAKE) -C @LINUX_OBJ@ M="$$PWD" @KERNEL_MAKE@ clean
 
 	$(RM) @LINUX_SYMBOLS@ Module.markers
 	find . -name '*.ur-safe' -type f -delete
 
 clean-FreeBSD:
 	+$(FMAKE) clean
 
 clean: clean-@ac_system@
 
 .PHONY: modules_uninstall-Linux-legacy
 modules_uninstall-Linux-legacy:
 	$(RM) -r $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,spl/ avl/ icp/ lua/ nvpair/ unicode/ zcommon/ zfs/ zstd/)
 
 KMODDIR := $(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@
 modules_install-Linux: modules_uninstall-Linux-legacy
 	@# Install the kernel modules
 	$(MAKE) -C @LINUX_OBJ@ M="$$PWD" modules_install \
 		INSTALL_MOD_PATH=$(INSTALL_MOD_PATH) \
 		INSTALL_MOD_DIR=$(INSTALL_MOD_DIR) \
 		KERNELRELEASE=@LINUX_VERSION@
 	@# Remove extraneous build products when packaging
 	if [ -n "$(DESTDIR)" ]; then \
 		find $(KMODDIR) -name 'modules.*' -delete; \
 	fi
 	@# Debian ships tiny fake System.map files that are
 	@# syntactically valid but just say
 	@# "if you want system.map go install this package"
 	@# Naturally, depmod is less than amused by this.
 	@# So if we find it missing or with one of these present,
 	@# we check for the alternate path for the System.map
 	sysmap=$(INSTALL_MOD_PATH)/boot/System.map-@LINUX_VERSION@; \
 	{ [ -f "$$sysmap" ] && [ $$(wc -l < "$$sysmap") -ge 100 ]; } || \
 		sysmap=$(INSTALL_MOD_PATH)/usr/lib/debug/boot/System.map-@LINUX_VERSION@; \
 	if [ -f $$sysmap ]; then \
 		depmod -ae -F $$sysmap @LINUX_VERSION@; \
 	fi
 
 modules_install-FreeBSD:
 	@# Install the kernel modules
 	+$(FMAKE) install
 
 modules_install: modules_install-@ac_system@
 
 data_install-Linux:
 	@mkdir -p $(DESTDIR)/@prefix@/src/zfs-@VERSION@/@LINUX_VERSION@
 	cp ../zfs.release ../zfs_config.h @LINUX_SYMBOLS@ $(DESTDIR)/@prefix@/src/zfs-@VERSION@/@LINUX_VERSION@
 
 data_install-FreeBSD:
 	@
 
 data_install: data_install-@ac_system@
 
 modules_uninstall-Linux: modules_uninstall-Linux-legacy
 	@# Uninstall the kernel modules
 	$(RM) $(addprefix $(KMODDIR)/$(INSTALL_MOD_DIR)/,zfs.ko spl.ko)
 
 modules_uninstall-FreeBSD:
 	@false
 
 modules_uninstall: modules_uninstall-@ac_system@
 
 data_uninstall-Linux:
 	$(RM) $(addprefix $(DESTDIR)/@prefix@/src/zfs-@VERSION@/@LINUX_VERSION@/,zfs.release zfs_config.h @LINUX_SYMBOLS@)
 
 data_uninstall-FreeBSD:
 	@
 
 data_uninstall: data_uninstall-@ac_system@
 
 cppcheck-Linux:
 	@CPPCHECK@ -j@CPU_COUNT@ --std=c99 --quiet --force --error-exitcode=2 \
 		--inline-suppr \
 		--suppress=unmatchedSuppression \
 		--suppress=noValidConfiguration \
 		--enable=warning,information -D_KERNEL \
 		--include=@LINUX_OBJ@/include/generated/autoconf.h \
 		--include=@top_builddir@/zfs_config.h \
 		--config-exclude=@LINUX_OBJ@/include \
 		-i zstd/lib \
 		-I @LINUX_OBJ@/include \
 		-I @top_srcdir@/include/os/linux/kernel \
 		-I @top_srcdir@/include/os/linux/spl \
 		-I @top_srcdir@/include/os/linux/zfs \
 		-I @top_srcdir@/include \
 		avl icp lua nvpair unicode zcommon zfs zstd os/linux
 
 cppcheck-FreeBSD:
 	@true
 
 cppcheck: cppcheck-@ac_system@
 
 distdir:
 	cd @srcdir@ && find . -name '*.[chS]' -exec sh -c 'for f; do mkdir -p $$distdir/$${f%/*}; cp @srcdir@/$$f $$distdir/$$f; done' _ {} +
 	cp @srcdir@/Makefile.bsd $$distdir/Makefile.bsd
 
 gen-zstd-symbols:
 	for obj in $(addprefix zstd/,$(ZSTD_UPSTREAM_OBJS)); do echo; echo "/* $${obj#zstd/}: */"; @OBJDUMP@ -t $$obj | awk '$$2 == "g" && !/ zfs_/ {print "#define\t" $$6 " zfs_" $$6}' | sort; done >> zstd/include/zstd_compat_wrapper.h
 
 check-zstd-symbols:
 	@OBJDUMP@ -t $(addprefix zstd/,$(ZSTD_UPSTREAM_OBJS)) | awk '/file format/ {print}  $$2 == "g" && (!/ zfs_/ && !/ __pfx_zfs_/) {++ret; print}  END {exit ret}'
diff --git a/sys/contrib/openzfs/module/icp/algs/modes/modes.c b/sys/contrib/openzfs/module/icp/algs/modes/modes.c
index 6f6649b3b58b..87b4e3e16979 100644
--- a/sys/contrib/openzfs/module/icp/algs/modes/modes.c
+++ b/sys/contrib/openzfs/module/icp/algs/modes/modes.c
@@ -1,196 +1,196 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 #include <sys/zfs_context.h>
 #include <modes/modes.h>
 #include <sys/crypto/common.h>
 #include <sys/crypto/impl.h>
 
 /*
  * Initialize by setting iov_or_mp to point to the current iovec or mp,
  * and by setting current_offset to an offset within the current iovec or mp.
  */
 void
 crypto_init_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset)
 {
 	offset_t offset;
 
 	switch (out->cd_format) {
 	case CRYPTO_DATA_RAW:
 		*current_offset = out->cd_offset;
 		break;
 
 	case CRYPTO_DATA_UIO: {
 		zfs_uio_t *uiop = out->cd_uio;
 		uint_t vec_idx;
 
 		offset = out->cd_offset;
 		offset = zfs_uio_index_at_offset(uiop, offset, &vec_idx);
 
 		*current_offset = offset;
 		*iov_or_mp = (void *)(uintptr_t)vec_idx;
 		break;
 	}
 	} /* end switch */
 }
 
 /*
  * Get pointers for where in the output to copy a block of encrypted or
  * decrypted data.  The iov_or_mp argument stores a pointer to the current
  * iovec or mp, and offset stores an offset into the current iovec or mp.
  */
 void
 crypto_get_ptrs(crypto_data_t *out, void **iov_or_mp, offset_t *current_offset,
     uint8_t **out_data_1, size_t *out_data_1_len, uint8_t **out_data_2,
     size_t amt)
 {
 	offset_t offset;
 
 	switch (out->cd_format) {
 	case CRYPTO_DATA_RAW: {
 		iovec_t *iov;
 
 		offset = *current_offset;
 		iov = &out->cd_raw;
 		if ((offset + amt) <= iov->iov_len) {
 			/* one block fits */
 			*out_data_1 = (uint8_t *)iov->iov_base + offset;
 			*out_data_1_len = amt;
 			*out_data_2 = NULL;
 			*current_offset = offset + amt;
 		}
 		break;
 	}
 
 	case CRYPTO_DATA_UIO: {
 		zfs_uio_t *uio = out->cd_uio;
 		offset_t offset;
 		uint_t vec_idx;
 		uint8_t *p;
 		uint64_t iov_len;
 		void *iov_base;
 
 		offset = *current_offset;
 		vec_idx = (uintptr_t)(*iov_or_mp);
 		zfs_uio_iov_at_index(uio, vec_idx, &iov_base, &iov_len);
 		p = (uint8_t *)iov_base + offset;
 		*out_data_1 = p;
 
 		if (offset + amt <= iov_len) {
 			/* can fit one block into this iov */
 			*out_data_1_len = amt;
 			*out_data_2 = NULL;
 			*current_offset = offset + amt;
 		} else {
 			/* one block spans two iovecs */
 			*out_data_1_len = iov_len - offset;
 			if (vec_idx == zfs_uio_iovcnt(uio)) {
 				*out_data_2 = NULL;
 				return;
 			}
 			vec_idx++;
 			zfs_uio_iov_at_index(uio, vec_idx, &iov_base, &iov_len);
 			*out_data_2 = (uint8_t *)iov_base;
 			*current_offset = amt - *out_data_1_len;
 		}
 		*iov_or_mp = (void *)(uintptr_t)vec_idx;
 		break;
 	}
 	} /* end switch */
 }
 
 void
 crypto_free_mode_ctx(void *ctx)
 {
 	common_ctx_t *common_ctx = (common_ctx_t *)ctx;
 
 	switch (common_ctx->cc_flags &
 	    (ECB_MODE|CBC_MODE|CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) {
 	case ECB_MODE:
 		kmem_free(common_ctx, sizeof (ecb_ctx_t));
 		break;
 
 	case CBC_MODE:
 		kmem_free(common_ctx, sizeof (cbc_ctx_t));
 		break;
 
 	case CTR_MODE:
 		kmem_free(common_ctx, sizeof (ctr_ctx_t));
 		break;
 
 	case CCM_MODE:
 		if (((ccm_ctx_t *)ctx)->ccm_pt_buf != NULL)
 			vmem_free(((ccm_ctx_t *)ctx)->ccm_pt_buf,
 			    ((ccm_ctx_t *)ctx)->ccm_data_len);
 
 		kmem_free(ctx, sizeof (ccm_ctx_t));
 		break;
 
 	case GCM_MODE:
 	case GMAC_MODE:
 		gcm_clear_ctx((gcm_ctx_t *)ctx);
 		kmem_free(ctx, sizeof (gcm_ctx_t));
 	}
 }
 
 static void *
 explicit_memset(void *s, int c, size_t n)
 {
 	memset(s, c, n);
 	__asm__ __volatile__("" :: "r"(s) : "memory");
 	return (s);
 }
 
 /*
  * Clear sensitive data in the context and free allocated memory.
  *
  * ctx->gcm_remainder may contain a plaintext remainder. ctx->gcm_H and
  * ctx->gcm_Htable contain the hash sub key which protects authentication.
  * ctx->gcm_pt_buf contains the plaintext result of decryption.
  *
  * Although extremely unlikely, ctx->gcm_J0 and ctx->gcm_tmp could be used for
  * a known plaintext attack, they consist of the IV and the first and last
  * counter respectively. If they should be cleared is debatable.
  */
 void
 gcm_clear_ctx(gcm_ctx_t *ctx)
 {
 	explicit_memset(ctx->gcm_remainder, 0, sizeof (ctx->gcm_remainder));
 	explicit_memset(ctx->gcm_H, 0, sizeof (ctx->gcm_H));
 #if defined(CAN_USE_GCM_ASM)
 	if (ctx->gcm_use_avx == B_TRUE) {
 		ASSERT3P(ctx->gcm_Htable, !=, NULL);
-		memset(ctx->gcm_Htable, 0, ctx->gcm_htab_len);
+		explicit_memset(ctx->gcm_Htable, 0, ctx->gcm_htab_len);
 		kmem_free(ctx->gcm_Htable, ctx->gcm_htab_len);
 	}
 #endif
 	if (ctx->gcm_pt_buf != NULL) {
-		memset(ctx->gcm_pt_buf, 0, ctx->gcm_pt_buf_len);
+		explicit_memset(ctx->gcm_pt_buf, 0, ctx->gcm_pt_buf_len);
 		vmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len);
 	}
 	/* Optional */
 	explicit_memset(ctx->gcm_J0, 0, sizeof (ctx->gcm_J0));
 	explicit_memset(ctx->gcm_tmp, 0, sizeof (ctx->gcm_tmp));
 }
diff --git a/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
index 909b2147dff9..cb75043a49a7 100644
--- a/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
+++ b/sys/contrib/openzfs/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
@@ -1,1264 +1,1281 @@
 # Copyright 2013-2016 The OpenSSL Project Authors. All Rights Reserved.
 #
 # Licensed under the Apache License 2.0 (the "License").  You may not use
 # this file except in compliance with the License.  You can obtain a copy
 # in the file LICENSE in the source distribution or at
 # https://www.openssl.org/source/license.html
 
 #
 # ====================================================================
 # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
 # project. The module is, however, dual licensed under OpenSSL and
 # CRYPTOGAMS licenses depending on where you obtain it. For further
 # details see http://www.openssl.org/~appro/cryptogams/.
 # ====================================================================
 #
 #
 # AES-NI-CTR+GHASH stitch.
 #
 # February 2013
 #
 # OpenSSL GCM implementation is organized in such way that its
 # performance is rather close to the sum of its streamed components,
 # in the context parallelized AES-NI CTR and modulo-scheduled
 # PCLMULQDQ-enabled GHASH. Unfortunately, as no stitch implementation
 # was observed to perform significantly better than the sum of the
 # components on contemporary CPUs, the effort was deemed impossible to
 # justify. This module is based on combination of Intel submissions,
 # [1] and [2], with MOVBE twist suggested by Ilya Albrekht and Max
 # Locktyukhin of Intel Corp. who verified that it reduces shuffles
 # pressure with notable relative improvement, achieving 1.0 cycle per
 # byte processed with 128-bit key on Haswell processor, 0.74 - on
 # Broadwell, 0.63 - on Skylake... [Mentioned results are raw profiled
 # measurements for favourable packet size, one divisible by 96.
 # Applications using the EVP interface will observe a few percent
 # worse performance.]
 #
 # Knights Landing processes 1 byte in 1.25 cycles (measured with EVP).
 #
 # [1] http://rt.openssl.org/Ticket/Display.html?id=2900&user=guest&pass=guest
 # [2] http://www.intel.com/content/dam/www/public/us/en/documents/software-support/enabling-high-performance-gcm.pdf
 
 # Generated once from
 # https://github.com/openssl/openssl/blob/5ffc3324/crypto/modes/asm/aesni-gcm-x86_64.pl
 # and modified for ICP. Modification are kept at a bare minimum to ease later
 # upstream merges.
 
 #if defined(__x86_64__) && defined(HAVE_AVX) && \
     defined(HAVE_AES) && defined(HAVE_PCLMULQDQ)
 
 #define _ASM
 #include <sys/asm_linkage.h>
+#include <sys/frame.h>
 
 /* Windows userland links with OpenSSL */
 #if !defined (_WIN32) || defined (_KERNEL)
 
 /* Apple needs _ */
 #if defined (__APPLE__)
 #define	gcm_avx_can_use_movbe _gcm_avx_can_use_movbe
 #endif
 
 .extern gcm_avx_can_use_movbe
 
 .text
 
 #ifdef HAVE_MOVBE
 .balign 32
 FUNCTION(_aesni_ctr32_ghash_6x)
 .cfi_startproc
 	ENDBR
 	vmovdqu	32(%r11),%xmm2
 	subq	$6,%rdx
 	vpxor	%xmm4,%xmm4,%xmm4
 	vmovdqu	0-128(%rcx),%xmm15
 	vpaddb	%xmm2,%xmm1,%xmm10
 	vpaddb	%xmm2,%xmm10,%xmm11
 	vpaddb	%xmm2,%xmm11,%xmm12
 	vpaddb	%xmm2,%xmm12,%xmm13
 	vpaddb	%xmm2,%xmm13,%xmm14
 	vpxor	%xmm15,%xmm1,%xmm9
 	vmovdqu	%xmm4,16+8(%rsp)
 	jmp	.Loop6x
 
 .balign	32
 .Loop6x:
 	addl	$100663296,%ebx
 	jc	.Lhandle_ctr32
 	vmovdqu	0-32(%r9),%xmm3
 	vpaddb	%xmm2,%xmm14,%xmm1
 	vpxor	%xmm15,%xmm10,%xmm10
 	vpxor	%xmm15,%xmm11,%xmm11
 
 .Lresume_ctr32:
 	vmovdqu	%xmm1,(%r8)
 	vpclmulqdq	$0x10,%xmm3,%xmm7,%xmm5
 	vpxor	%xmm15,%xmm12,%xmm12
 	vmovups	16-128(%rcx),%xmm2
 	vpclmulqdq	$0x01,%xmm3,%xmm7,%xmm6
 	xorq	%r12,%r12
 	cmpq	%r14,%r15
 
 	vaesenc	%xmm2,%xmm9,%xmm9
 	vmovdqu	48+8(%rsp),%xmm0
 	vpxor	%xmm15,%xmm13,%xmm13
 	vpclmulqdq	$0x00,%xmm3,%xmm7,%xmm1
 	vaesenc	%xmm2,%xmm10,%xmm10
 	vpxor	%xmm15,%xmm14,%xmm14
 	setnc	%r12b
 	vpclmulqdq	$0x11,%xmm3,%xmm7,%xmm7
 	vaesenc	%xmm2,%xmm11,%xmm11
 	vmovdqu	16-32(%r9),%xmm3
 	negq	%r12
 	vaesenc	%xmm2,%xmm12,%xmm12
 	vpxor	%xmm5,%xmm6,%xmm6
 	vpclmulqdq	$0x00,%xmm3,%xmm0,%xmm5
 	vpxor	%xmm4,%xmm8,%xmm8
 	vaesenc	%xmm2,%xmm13,%xmm13
 	vpxor	%xmm5,%xmm1,%xmm4
 	andq	$0x60,%r12
 	vmovups	32-128(%rcx),%xmm15
 	vpclmulqdq	$0x10,%xmm3,%xmm0,%xmm1
 	vaesenc	%xmm2,%xmm14,%xmm14
 
 	vpclmulqdq	$0x01,%xmm3,%xmm0,%xmm2
 	leaq	(%r14,%r12,1),%r14
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	16+8(%rsp),%xmm8,%xmm8
 	vpclmulqdq	$0x11,%xmm3,%xmm0,%xmm3
 	vmovdqu	64+8(%rsp),%xmm0
 	vaesenc	%xmm15,%xmm10,%xmm10
 	movbeq	88(%r14),%r13
 	vaesenc	%xmm15,%xmm11,%xmm11
 	movbeq	80(%r14),%r12
 	vaesenc	%xmm15,%xmm12,%xmm12
 	movq	%r13,32+8(%rsp)
 	vaesenc	%xmm15,%xmm13,%xmm13
 	movq	%r12,40+8(%rsp)
 	vmovdqu	48-32(%r9),%xmm5
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vmovups	48-128(%rcx),%xmm15
 	vpxor	%xmm1,%xmm6,%xmm6
 	vpclmulqdq	$0x00,%xmm5,%xmm0,%xmm1
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	%xmm2,%xmm6,%xmm6
 	vpclmulqdq	$0x10,%xmm5,%xmm0,%xmm2
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vpxor	%xmm3,%xmm7,%xmm7
 	vpclmulqdq	$0x01,%xmm5,%xmm0,%xmm3
 	vaesenc	%xmm15,%xmm11,%xmm11
 	vpclmulqdq	$0x11,%xmm5,%xmm0,%xmm5
 	vmovdqu	80+8(%rsp),%xmm0
 	vaesenc	%xmm15,%xmm12,%xmm12
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vpxor	%xmm1,%xmm4,%xmm4
 	vmovdqu	64-32(%r9),%xmm1
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vmovups	64-128(%rcx),%xmm15
 	vpxor	%xmm2,%xmm6,%xmm6
 	vpclmulqdq	$0x00,%xmm1,%xmm0,%xmm2
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	%xmm3,%xmm6,%xmm6
 	vpclmulqdq	$0x10,%xmm1,%xmm0,%xmm3
 	vaesenc	%xmm15,%xmm10,%xmm10
 	movbeq	72(%r14),%r13
 	vpxor	%xmm5,%xmm7,%xmm7
 	vpclmulqdq	$0x01,%xmm1,%xmm0,%xmm5
 	vaesenc	%xmm15,%xmm11,%xmm11
 	movbeq	64(%r14),%r12
 	vpclmulqdq	$0x11,%xmm1,%xmm0,%xmm1
 	vmovdqu	96+8(%rsp),%xmm0
 	vaesenc	%xmm15,%xmm12,%xmm12
 	movq	%r13,48+8(%rsp)
 	vaesenc	%xmm15,%xmm13,%xmm13
 	movq	%r12,56+8(%rsp)
 	vpxor	%xmm2,%xmm4,%xmm4
 	vmovdqu	96-32(%r9),%xmm2
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vmovups	80-128(%rcx),%xmm15
 	vpxor	%xmm3,%xmm6,%xmm6
 	vpclmulqdq	$0x00,%xmm2,%xmm0,%xmm3
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	%xmm5,%xmm6,%xmm6
 	vpclmulqdq	$0x10,%xmm2,%xmm0,%xmm5
 	vaesenc	%xmm15,%xmm10,%xmm10
 	movbeq	56(%r14),%r13
 	vpxor	%xmm1,%xmm7,%xmm7
 	vpclmulqdq	$0x01,%xmm2,%xmm0,%xmm1
 	vpxor	112+8(%rsp),%xmm8,%xmm8
 	vaesenc	%xmm15,%xmm11,%xmm11
 	movbeq	48(%r14),%r12
 	vpclmulqdq	$0x11,%xmm2,%xmm0,%xmm2
 	vaesenc	%xmm15,%xmm12,%xmm12
 	movq	%r13,64+8(%rsp)
 	vaesenc	%xmm15,%xmm13,%xmm13
 	movq	%r12,72+8(%rsp)
 	vpxor	%xmm3,%xmm4,%xmm4
 	vmovdqu	112-32(%r9),%xmm3
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vmovups	96-128(%rcx),%xmm15
 	vpxor	%xmm5,%xmm6,%xmm6
 	vpclmulqdq	$0x10,%xmm3,%xmm8,%xmm5
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	%xmm1,%xmm6,%xmm6
 	vpclmulqdq	$0x01,%xmm3,%xmm8,%xmm1
 	vaesenc	%xmm15,%xmm10,%xmm10
 	movbeq	40(%r14),%r13
 	vpxor	%xmm2,%xmm7,%xmm7
 	vpclmulqdq	$0x00,%xmm3,%xmm8,%xmm2
 	vaesenc	%xmm15,%xmm11,%xmm11
 	movbeq	32(%r14),%r12
 	vpclmulqdq	$0x11,%xmm3,%xmm8,%xmm8
 	vaesenc	%xmm15,%xmm12,%xmm12
 	movq	%r13,80+8(%rsp)
 	vaesenc	%xmm15,%xmm13,%xmm13
 	movq	%r12,88+8(%rsp)
 	vpxor	%xmm5,%xmm6,%xmm6
 	vaesenc	%xmm15,%xmm14,%xmm14
 	vpxor	%xmm1,%xmm6,%xmm6
 
 	vmovups	112-128(%rcx),%xmm15
 	vpslldq	$8,%xmm6,%xmm5
 	vpxor	%xmm2,%xmm4,%xmm4
 	vmovdqu	16(%r11),%xmm3
 
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	%xmm8,%xmm7,%xmm7
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vpxor	%xmm5,%xmm4,%xmm4
 	movbeq	24(%r14),%r13
 	vaesenc	%xmm15,%xmm11,%xmm11
 	movbeq	16(%r14),%r12
 	vpalignr	$8,%xmm4,%xmm4,%xmm0
 	vpclmulqdq	$0x10,%xmm3,%xmm4,%xmm4
 	movq	%r13,96+8(%rsp)
 	vaesenc	%xmm15,%xmm12,%xmm12
 	movq	%r12,104+8(%rsp)
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vmovups	128-128(%rcx),%xmm1
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vaesenc	%xmm1,%xmm9,%xmm9
 	vmovups	144-128(%rcx),%xmm15
 	vaesenc	%xmm1,%xmm10,%xmm10
 	vpsrldq	$8,%xmm6,%xmm6
 	vaesenc	%xmm1,%xmm11,%xmm11
 	vpxor	%xmm6,%xmm7,%xmm7
 	vaesenc	%xmm1,%xmm12,%xmm12
 	vpxor	%xmm0,%xmm4,%xmm4
 	movbeq	8(%r14),%r13
 	vaesenc	%xmm1,%xmm13,%xmm13
 	movbeq	0(%r14),%r12
 	vaesenc	%xmm1,%xmm14,%xmm14
 	vmovups	160-128(%rcx),%xmm1
 	cmpl	$12,%ebp	// ICP uses 10,12,14 not 9,11,13 for rounds.
 	jb	.Lenc_tail
 
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vaesenc	%xmm15,%xmm11,%xmm11
 	vaesenc	%xmm15,%xmm12,%xmm12
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vaesenc	%xmm1,%xmm9,%xmm9
 	vaesenc	%xmm1,%xmm10,%xmm10
 	vaesenc	%xmm1,%xmm11,%xmm11
 	vaesenc	%xmm1,%xmm12,%xmm12
 	vaesenc	%xmm1,%xmm13,%xmm13
 	vmovups	176-128(%rcx),%xmm15
 	vaesenc	%xmm1,%xmm14,%xmm14
 	vmovups	192-128(%rcx),%xmm1
 	cmpl	$14,%ebp	// ICP does not zero key schedule.
 	jb	.Lenc_tail
 
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vaesenc	%xmm15,%xmm11,%xmm11
 	vaesenc	%xmm15,%xmm12,%xmm12
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vaesenc	%xmm1,%xmm9,%xmm9
 	vaesenc	%xmm1,%xmm10,%xmm10
 	vaesenc	%xmm1,%xmm11,%xmm11
 	vaesenc	%xmm1,%xmm12,%xmm12
 	vaesenc	%xmm1,%xmm13,%xmm13
 	vmovups	208-128(%rcx),%xmm15
 	vaesenc	%xmm1,%xmm14,%xmm14
 	vmovups	224-128(%rcx),%xmm1
 	jmp	.Lenc_tail
 
 .balign	32
 .Lhandle_ctr32:
 	vmovdqu	(%r11),%xmm0
 	vpshufb	%xmm0,%xmm1,%xmm6
 	vmovdqu	48(%r11),%xmm5
 	vpaddd	64(%r11),%xmm6,%xmm10
 	vpaddd	%xmm5,%xmm6,%xmm11
 	vmovdqu	0-32(%r9),%xmm3
 	vpaddd	%xmm5,%xmm10,%xmm12
 	vpshufb	%xmm0,%xmm10,%xmm10
 	vpaddd	%xmm5,%xmm11,%xmm13
 	vpshufb	%xmm0,%xmm11,%xmm11
 	vpxor	%xmm15,%xmm10,%xmm10
 	vpaddd	%xmm5,%xmm12,%xmm14
 	vpshufb	%xmm0,%xmm12,%xmm12
 	vpxor	%xmm15,%xmm11,%xmm11
 	vpaddd	%xmm5,%xmm13,%xmm1
 	vpshufb	%xmm0,%xmm13,%xmm13
 	vpshufb	%xmm0,%xmm14,%xmm14
 	vpshufb	%xmm0,%xmm1,%xmm1
 	jmp	.Lresume_ctr32
 
 .balign	32
 .Lenc_tail:
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vmovdqu	%xmm7,16+8(%rsp)
 	vpalignr	$8,%xmm4,%xmm4,%xmm8
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vpclmulqdq	$0x10,%xmm3,%xmm4,%xmm4
 	vpxor	0(%rdi),%xmm1,%xmm2
 	vaesenc	%xmm15,%xmm11,%xmm11
 	vpxor	16(%rdi),%xmm1,%xmm0
 	vaesenc	%xmm15,%xmm12,%xmm12
 	vpxor	32(%rdi),%xmm1,%xmm5
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vpxor	48(%rdi),%xmm1,%xmm6
 	vaesenc	%xmm15,%xmm14,%xmm14
 	vpxor	64(%rdi),%xmm1,%xmm7
 	vpxor	80(%rdi),%xmm1,%xmm3
 	vmovdqu	(%r8),%xmm1
 
 	vaesenclast	%xmm2,%xmm9,%xmm9
 	vmovdqu	32(%r11),%xmm2
 	vaesenclast	%xmm0,%xmm10,%xmm10
 	vpaddb	%xmm2,%xmm1,%xmm0
 	movq	%r13,112+8(%rsp)
 	leaq	96(%rdi),%rdi
 	vaesenclast	%xmm5,%xmm11,%xmm11
 	vpaddb	%xmm2,%xmm0,%xmm5
 	movq	%r12,120+8(%rsp)
 	leaq	96(%rsi),%rsi
 	vmovdqu	0-128(%rcx),%xmm15
 	vaesenclast	%xmm6,%xmm12,%xmm12
 	vpaddb	%xmm2,%xmm5,%xmm6
 	vaesenclast	%xmm7,%xmm13,%xmm13
 	vpaddb	%xmm2,%xmm6,%xmm7
 	vaesenclast	%xmm3,%xmm14,%xmm14
 	vpaddb	%xmm2,%xmm7,%xmm3
 
 	addq	$0x60,%r10
 	subq	$0x6,%rdx
 	jc	.L6x_done
 
 	vmovups	%xmm9,-96(%rsi)
 	vpxor	%xmm15,%xmm1,%xmm9
 	vmovups	%xmm10,-80(%rsi)
 	vmovdqa	%xmm0,%xmm10
 	vmovups	%xmm11,-64(%rsi)
 	vmovdqa	%xmm5,%xmm11
 	vmovups	%xmm12,-48(%rsi)
 	vmovdqa	%xmm6,%xmm12
 	vmovups	%xmm13,-32(%rsi)
 	vmovdqa	%xmm7,%xmm13
 	vmovups	%xmm14,-16(%rsi)
 	vmovdqa	%xmm3,%xmm14
 	vmovdqu	32+8(%rsp),%xmm7
 	jmp	.Loop6x
 
 .L6x_done:
 	vpxor	16+8(%rsp),%xmm8,%xmm8
 	vpxor	%xmm4,%xmm8,%xmm8
 
 	RET
 .cfi_endproc
 SET_SIZE(_aesni_ctr32_ghash_6x)
+STACK_FRAME_NON_STANDARD _aesni_ctr32_ghash_6x
 #endif /* ifdef HAVE_MOVBE */
 
 .balign 32
 FUNCTION(_aesni_ctr32_ghash_no_movbe_6x)
 .cfi_startproc
 	ENDBR
 	vmovdqu	32(%r11),%xmm2
 	subq	$6,%rdx
 	vpxor	%xmm4,%xmm4,%xmm4
 	vmovdqu	0-128(%rcx),%xmm15
 	vpaddb	%xmm2,%xmm1,%xmm10
 	vpaddb	%xmm2,%xmm10,%xmm11
 	vpaddb	%xmm2,%xmm11,%xmm12
 	vpaddb	%xmm2,%xmm12,%xmm13
 	vpaddb	%xmm2,%xmm13,%xmm14
 	vpxor	%xmm15,%xmm1,%xmm9
 	vmovdqu	%xmm4,16+8(%rsp)
 	jmp	.Loop6x_nmb
 
 .balign	32
 .Loop6x_nmb:
 	addl	$100663296,%ebx
 	jc	.Lhandle_ctr32_nmb
 	vmovdqu	0-32(%r9),%xmm3
 	vpaddb	%xmm2,%xmm14,%xmm1
 	vpxor	%xmm15,%xmm10,%xmm10
 	vpxor	%xmm15,%xmm11,%xmm11
 
 .Lresume_ctr32_nmb:
 	vmovdqu	%xmm1,(%r8)
 	vpclmulqdq	$0x10,%xmm3,%xmm7,%xmm5
 	vpxor	%xmm15,%xmm12,%xmm12
 	vmovups	16-128(%rcx),%xmm2
 	vpclmulqdq	$0x01,%xmm3,%xmm7,%xmm6
 	xorq	%r12,%r12
 	cmpq	%r14,%r15
 
 	vaesenc	%xmm2,%xmm9,%xmm9
 	vmovdqu	48+8(%rsp),%xmm0
 	vpxor	%xmm15,%xmm13,%xmm13
 	vpclmulqdq	$0x00,%xmm3,%xmm7,%xmm1
 	vaesenc	%xmm2,%xmm10,%xmm10
 	vpxor	%xmm15,%xmm14,%xmm14
 	setnc	%r12b
 	vpclmulqdq	$0x11,%xmm3,%xmm7,%xmm7
 	vaesenc	%xmm2,%xmm11,%xmm11
 	vmovdqu	16-32(%r9),%xmm3
 	negq	%r12
 	vaesenc	%xmm2,%xmm12,%xmm12
 	vpxor	%xmm5,%xmm6,%xmm6
 	vpclmulqdq	$0x00,%xmm3,%xmm0,%xmm5
 	vpxor	%xmm4,%xmm8,%xmm8
 	vaesenc	%xmm2,%xmm13,%xmm13
 	vpxor	%xmm5,%xmm1,%xmm4
 	andq	$0x60,%r12
 	vmovups	32-128(%rcx),%xmm15
 	vpclmulqdq	$0x10,%xmm3,%xmm0,%xmm1
 	vaesenc	%xmm2,%xmm14,%xmm14
 
 	vpclmulqdq	$0x01,%xmm3,%xmm0,%xmm2
 	leaq	(%r14,%r12,1),%r14
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	16+8(%rsp),%xmm8,%xmm8
 	vpclmulqdq	$0x11,%xmm3,%xmm0,%xmm3
 	vmovdqu	64+8(%rsp),%xmm0
 	vaesenc	%xmm15,%xmm10,%xmm10
 	movq	88(%r14),%r13
 	bswapq	%r13
 	vaesenc	%xmm15,%xmm11,%xmm11
 	movq	80(%r14),%r12
 	bswapq	%r12
 	vaesenc	%xmm15,%xmm12,%xmm12
 	movq	%r13,32+8(%rsp)
 	vaesenc	%xmm15,%xmm13,%xmm13
 	movq	%r12,40+8(%rsp)
 	vmovdqu	48-32(%r9),%xmm5
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vmovups	48-128(%rcx),%xmm15
 	vpxor	%xmm1,%xmm6,%xmm6
 	vpclmulqdq	$0x00,%xmm5,%xmm0,%xmm1
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	%xmm2,%xmm6,%xmm6
 	vpclmulqdq	$0x10,%xmm5,%xmm0,%xmm2
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vpxor	%xmm3,%xmm7,%xmm7
 	vpclmulqdq	$0x01,%xmm5,%xmm0,%xmm3
 	vaesenc	%xmm15,%xmm11,%xmm11
 	vpclmulqdq	$0x11,%xmm5,%xmm0,%xmm5
 	vmovdqu	80+8(%rsp),%xmm0
 	vaesenc	%xmm15,%xmm12,%xmm12
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vpxor	%xmm1,%xmm4,%xmm4
 	vmovdqu	64-32(%r9),%xmm1
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vmovups	64-128(%rcx),%xmm15
 	vpxor	%xmm2,%xmm6,%xmm6
 	vpclmulqdq	$0x00,%xmm1,%xmm0,%xmm2
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	%xmm3,%xmm6,%xmm6
 	vpclmulqdq	$0x10,%xmm1,%xmm0,%xmm3
 	vaesenc	%xmm15,%xmm10,%xmm10
 	movq	72(%r14),%r13
 	bswapq	%r13
 	vpxor	%xmm5,%xmm7,%xmm7
 	vpclmulqdq	$0x01,%xmm1,%xmm0,%xmm5
 	vaesenc	%xmm15,%xmm11,%xmm11
 	movq	64(%r14),%r12
 	bswapq	%r12
 	vpclmulqdq	$0x11,%xmm1,%xmm0,%xmm1
 	vmovdqu	96+8(%rsp),%xmm0
 	vaesenc	%xmm15,%xmm12,%xmm12
 	movq	%r13,48+8(%rsp)
 	vaesenc	%xmm15,%xmm13,%xmm13
 	movq	%r12,56+8(%rsp)
 	vpxor	%xmm2,%xmm4,%xmm4
 	vmovdqu	96-32(%r9),%xmm2
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vmovups	80-128(%rcx),%xmm15
 	vpxor	%xmm3,%xmm6,%xmm6
 	vpclmulqdq	$0x00,%xmm2,%xmm0,%xmm3
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	%xmm5,%xmm6,%xmm6
 	vpclmulqdq	$0x10,%xmm2,%xmm0,%xmm5
 	vaesenc	%xmm15,%xmm10,%xmm10
 	movq	56(%r14),%r13
 	bswapq	%r13
 	vpxor	%xmm1,%xmm7,%xmm7
 	vpclmulqdq	$0x01,%xmm2,%xmm0,%xmm1
 	vpxor	112+8(%rsp),%xmm8,%xmm8
 	vaesenc	%xmm15,%xmm11,%xmm11
 	movq	48(%r14),%r12
 	bswapq	%r12
 	vpclmulqdq	$0x11,%xmm2,%xmm0,%xmm2
 	vaesenc	%xmm15,%xmm12,%xmm12
 	movq	%r13,64+8(%rsp)
 	vaesenc	%xmm15,%xmm13,%xmm13
 	movq	%r12,72+8(%rsp)
 	vpxor	%xmm3,%xmm4,%xmm4
 	vmovdqu	112-32(%r9),%xmm3
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vmovups	96-128(%rcx),%xmm15
 	vpxor	%xmm5,%xmm6,%xmm6
 	vpclmulqdq	$0x10,%xmm3,%xmm8,%xmm5
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	%xmm1,%xmm6,%xmm6
 	vpclmulqdq	$0x01,%xmm3,%xmm8,%xmm1
 	vaesenc	%xmm15,%xmm10,%xmm10
 	movq	40(%r14),%r13
 	bswapq	%r13
 	vpxor	%xmm2,%xmm7,%xmm7
 	vpclmulqdq	$0x00,%xmm3,%xmm8,%xmm2
 	vaesenc	%xmm15,%xmm11,%xmm11
 	movq	32(%r14),%r12
 	bswapq	%r12
 	vpclmulqdq	$0x11,%xmm3,%xmm8,%xmm8
 	vaesenc	%xmm15,%xmm12,%xmm12
 	movq	%r13,80+8(%rsp)
 	vaesenc	%xmm15,%xmm13,%xmm13
 	movq	%r12,88+8(%rsp)
 	vpxor	%xmm5,%xmm6,%xmm6
 	vaesenc	%xmm15,%xmm14,%xmm14
 	vpxor	%xmm1,%xmm6,%xmm6
 
 	vmovups	112-128(%rcx),%xmm15
 	vpslldq	$8,%xmm6,%xmm5
 	vpxor	%xmm2,%xmm4,%xmm4
 	vmovdqu	16(%r11),%xmm3
 
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	%xmm8,%xmm7,%xmm7
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vpxor	%xmm5,%xmm4,%xmm4
 	movq	24(%r14),%r13
 	bswapq	%r13
 	vaesenc	%xmm15,%xmm11,%xmm11
 	movq	16(%r14),%r12
 	bswapq	%r12
 	vpalignr	$8,%xmm4,%xmm4,%xmm0
 	vpclmulqdq	$0x10,%xmm3,%xmm4,%xmm4
 	movq	%r13,96+8(%rsp)
 	vaesenc	%xmm15,%xmm12,%xmm12
 	movq	%r12,104+8(%rsp)
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vmovups	128-128(%rcx),%xmm1
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vaesenc	%xmm1,%xmm9,%xmm9
 	vmovups	144-128(%rcx),%xmm15
 	vaesenc	%xmm1,%xmm10,%xmm10
 	vpsrldq	$8,%xmm6,%xmm6
 	vaesenc	%xmm1,%xmm11,%xmm11
 	vpxor	%xmm6,%xmm7,%xmm7
 	vaesenc	%xmm1,%xmm12,%xmm12
 	vpxor	%xmm0,%xmm4,%xmm4
 	movq	8(%r14),%r13
 	bswapq	%r13
 	vaesenc	%xmm1,%xmm13,%xmm13
 	movq	0(%r14),%r12
 	bswapq	%r12
 	vaesenc	%xmm1,%xmm14,%xmm14
 	vmovups	160-128(%rcx),%xmm1
 	cmpl	$12,%ebp	// ICP uses 10,12,14 not 9,11,13 for rounds.
 	jb	.Lenc_tail_nmb
 
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vaesenc	%xmm15,%xmm11,%xmm11
 	vaesenc	%xmm15,%xmm12,%xmm12
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vaesenc	%xmm1,%xmm9,%xmm9
 	vaesenc	%xmm1,%xmm10,%xmm10
 	vaesenc	%xmm1,%xmm11,%xmm11
 	vaesenc	%xmm1,%xmm12,%xmm12
 	vaesenc	%xmm1,%xmm13,%xmm13
 	vmovups	176-128(%rcx),%xmm15
 	vaesenc	%xmm1,%xmm14,%xmm14
 	vmovups	192-128(%rcx),%xmm1
 	cmpl	$14,%ebp	// ICP does not zero key schedule.
 	jb	.Lenc_tail_nmb
 
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vaesenc	%xmm15,%xmm11,%xmm11
 	vaesenc	%xmm15,%xmm12,%xmm12
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vaesenc	%xmm15,%xmm14,%xmm14
 
 	vaesenc	%xmm1,%xmm9,%xmm9
 	vaesenc	%xmm1,%xmm10,%xmm10
 	vaesenc	%xmm1,%xmm11,%xmm11
 	vaesenc	%xmm1,%xmm12,%xmm12
 	vaesenc	%xmm1,%xmm13,%xmm13
 	vmovups	208-128(%rcx),%xmm15
 	vaesenc	%xmm1,%xmm14,%xmm14
 	vmovups	224-128(%rcx),%xmm1
 	jmp	.Lenc_tail_nmb
 
 .balign	32
 .Lhandle_ctr32_nmb:
 	vmovdqu	(%r11),%xmm0
 	vpshufb	%xmm0,%xmm1,%xmm6
 	vmovdqu	48(%r11),%xmm5
 	vpaddd	64(%r11),%xmm6,%xmm10
 	vpaddd	%xmm5,%xmm6,%xmm11
 	vmovdqu	0-32(%r9),%xmm3
 	vpaddd	%xmm5,%xmm10,%xmm12
 	vpshufb	%xmm0,%xmm10,%xmm10
 	vpaddd	%xmm5,%xmm11,%xmm13
 	vpshufb	%xmm0,%xmm11,%xmm11
 	vpxor	%xmm15,%xmm10,%xmm10
 	vpaddd	%xmm5,%xmm12,%xmm14
 	vpshufb	%xmm0,%xmm12,%xmm12
 	vpxor	%xmm15,%xmm11,%xmm11
 	vpaddd	%xmm5,%xmm13,%xmm1
 	vpshufb	%xmm0,%xmm13,%xmm13
 	vpshufb	%xmm0,%xmm14,%xmm14
 	vpshufb	%xmm0,%xmm1,%xmm1
 	jmp	.Lresume_ctr32_nmb
 
 .balign	32
 .Lenc_tail_nmb:
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vmovdqu	%xmm7,16+8(%rsp)
 	vpalignr	$8,%xmm4,%xmm4,%xmm8
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vpclmulqdq	$0x10,%xmm3,%xmm4,%xmm4
 	vpxor	0(%rdi),%xmm1,%xmm2
 	vaesenc	%xmm15,%xmm11,%xmm11
 	vpxor	16(%rdi),%xmm1,%xmm0
 	vaesenc	%xmm15,%xmm12,%xmm12
 	vpxor	32(%rdi),%xmm1,%xmm5
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vpxor	48(%rdi),%xmm1,%xmm6
 	vaesenc	%xmm15,%xmm14,%xmm14
 	vpxor	64(%rdi),%xmm1,%xmm7
 	vpxor	80(%rdi),%xmm1,%xmm3
 	vmovdqu	(%r8),%xmm1
 
 	vaesenclast	%xmm2,%xmm9,%xmm9
 	vmovdqu	32(%r11),%xmm2
 	vaesenclast	%xmm0,%xmm10,%xmm10
 	vpaddb	%xmm2,%xmm1,%xmm0
 	movq	%r13,112+8(%rsp)
 	leaq	96(%rdi),%rdi
 	vaesenclast	%xmm5,%xmm11,%xmm11
 	vpaddb	%xmm2,%xmm0,%xmm5
 	movq	%r12,120+8(%rsp)
 	leaq	96(%rsi),%rsi
 	vmovdqu	0-128(%rcx),%xmm15
 	vaesenclast	%xmm6,%xmm12,%xmm12
 	vpaddb	%xmm2,%xmm5,%xmm6
 	vaesenclast	%xmm7,%xmm13,%xmm13
 	vpaddb	%xmm2,%xmm6,%xmm7
 	vaesenclast	%xmm3,%xmm14,%xmm14
 	vpaddb	%xmm2,%xmm7,%xmm3
 
 	addq	$0x60,%r10
 	subq	$0x6,%rdx
 	jc	.L6x_done_nmb
 
 	vmovups	%xmm9,-96(%rsi)
 	vpxor	%xmm15,%xmm1,%xmm9
 	vmovups	%xmm10,-80(%rsi)
 	vmovdqa	%xmm0,%xmm10
 	vmovups	%xmm11,-64(%rsi)
 	vmovdqa	%xmm5,%xmm11
 	vmovups	%xmm12,-48(%rsi)
 	vmovdqa	%xmm6,%xmm12
 	vmovups	%xmm13,-32(%rsi)
 	vmovdqa	%xmm7,%xmm13
 	vmovups	%xmm14,-16(%rsi)
 	vmovdqa	%xmm3,%xmm14
 	vmovdqu	32+8(%rsp),%xmm7
 	jmp	.Loop6x_nmb
 
 .L6x_done_nmb:
 	vpxor	16+8(%rsp),%xmm8,%xmm8
 	vpxor	%xmm4,%xmm8,%xmm8
 
 	RET
 .cfi_endproc
 SET_SIZE(_aesni_ctr32_ghash_no_movbe_6x)
+STACK_FRAME_NON_STANDARD _aesni_ctr32_ghash_no_movbe_6x
 
 ENTRY_ALIGN(aesni_gcm_decrypt, 32)
 .cfi_startproc
 	ENDBR
 	xorq	%r10,%r10
 	cmpq	$0x60,%rdx
 	jb	.Lgcm_dec_abort
 
 	leaq	(%rsp),%rax
 .cfi_def_cfa_register	%rax
 	pushq	%rbx
 .cfi_offset	%rbx,-16
 	pushq	%rbp
 .cfi_offset	%rbp,-24
 	pushq	%r12
 .cfi_offset	%r12,-32
 	pushq	%r13
 .cfi_offset	%r13,-40
 	pushq	%r14
 .cfi_offset	%r14,-48
 	pushq	%r15
 .cfi_offset	%r15,-56
 	pushq	%r9
 .cfi_offset	%r9,-64
 	vzeroupper
 
 	vmovdqu	(%r8),%xmm1
 	addq	$-128,%rsp
 	movl	12(%r8),%ebx
 	leaq	.Lbswap_mask(%rip),%r11
 	leaq	-128(%rcx),%r14
 	movq	$0xf80,%r15
 	vmovdqu	(%r9),%xmm8
 	andq	$-128,%rsp
 	vmovdqu	(%r11),%xmm0
 	leaq	128(%rcx),%rcx
 	movq	32(%r9),%r9
 	leaq	32(%r9),%r9
 	movl	504-128(%rcx),%ebp	// ICP has a larger offset for rounds.
 	vpshufb	%xmm0,%xmm8,%xmm8
 
 	andq	%r15,%r14
 	andq	%rsp,%r15
 	subq	%r14,%r15
 	jc	.Ldec_no_key_aliasing
 	cmpq	$768,%r15
 	jnc	.Ldec_no_key_aliasing
 	subq	%r15,%rsp
 .Ldec_no_key_aliasing:
 
 	vmovdqu	80(%rdi),%xmm7
 	leaq	(%rdi),%r14
 	vmovdqu	64(%rdi),%xmm4
 	leaq	-192(%rdi,%rdx,1),%r15
 	vmovdqu	48(%rdi),%xmm5
 	shrq	$4,%rdx
 	xorq	%r10,%r10
 	vmovdqu	32(%rdi),%xmm6
 	vpshufb	%xmm0,%xmm7,%xmm7
 	vmovdqu	16(%rdi),%xmm2
 	vpshufb	%xmm0,%xmm4,%xmm4
 	vmovdqu	(%rdi),%xmm3
 	vpshufb	%xmm0,%xmm5,%xmm5
 	vmovdqu	%xmm4,48(%rsp)
 	vpshufb	%xmm0,%xmm6,%xmm6
 	vmovdqu	%xmm5,64(%rsp)
 	vpshufb	%xmm0,%xmm2,%xmm2
 	vmovdqu	%xmm6,80(%rsp)
 	vpshufb	%xmm0,%xmm3,%xmm3
 	vmovdqu	%xmm2,96(%rsp)
 	vmovdqu	%xmm3,112(%rsp)
 
 #ifdef HAVE_MOVBE
 #ifdef _KERNEL
 	testl	$1,gcm_avx_can_use_movbe(%rip)
 #else
 	testl	$1,gcm_avx_can_use_movbe@GOTPCREL(%rip)
 #endif
 	jz	1f
 	call	_aesni_ctr32_ghash_6x
 	jmp	2f
 1:
 #endif
 	call	_aesni_ctr32_ghash_no_movbe_6x
 2:
 	vmovups	%xmm9,-96(%rsi)
 	vmovups	%xmm10,-80(%rsi)
 	vmovups	%xmm11,-64(%rsi)
 	vmovups	%xmm12,-48(%rsi)
 	vmovups	%xmm13,-32(%rsi)
 	vmovups	%xmm14,-16(%rsi)
 
 	vpshufb	(%r11),%xmm8,%xmm8
 	movq	-56(%rax),%r9
 .cfi_restore	%r9
 	vmovdqu	%xmm8,(%r9)
 
 	vzeroupper
 	movq	-48(%rax),%r15
 .cfi_restore	%r15
 	movq	-40(%rax),%r14
 .cfi_restore	%r14
 	movq	-32(%rax),%r13
 .cfi_restore	%r13
 	movq	-24(%rax),%r12
 .cfi_restore	%r12
 	movq	-16(%rax),%rbp
 .cfi_restore	%rbp
 	movq	-8(%rax),%rbx
 .cfi_restore	%rbx
 	leaq	(%rax),%rsp
 .cfi_def_cfa_register	%rsp
 .Lgcm_dec_abort:
 	movq	%r10,%rax
 	RET
 .cfi_endproc
 SET_SIZE(aesni_gcm_decrypt)
+STACK_FRAME_NON_STANDARD aesni_gcm_decrypt
 
 .balign 32
 FUNCTION(_aesni_ctr32_6x)
 .cfi_startproc
 	ENDBR
 	vmovdqu	0-128(%rcx),%xmm4
 	vmovdqu	32(%r11),%xmm2
 	leaq	-2(%rbp),%r13	// ICP uses 10,12,14 not 9,11,13 for rounds.
 	vmovups	16-128(%rcx),%xmm15
 	leaq	32-128(%rcx),%r12
 	vpxor	%xmm4,%xmm1,%xmm9
 	addl	$100663296,%ebx
 	jc	.Lhandle_ctr32_2
 	vpaddb	%xmm2,%xmm1,%xmm10
 	vpaddb	%xmm2,%xmm10,%xmm11
 	vpxor	%xmm4,%xmm10,%xmm10
 	vpaddb	%xmm2,%xmm11,%xmm12
 	vpxor	%xmm4,%xmm11,%xmm11
 	vpaddb	%xmm2,%xmm12,%xmm13
 	vpxor	%xmm4,%xmm12,%xmm12
 	vpaddb	%xmm2,%xmm13,%xmm14
 	vpxor	%xmm4,%xmm13,%xmm13
 	vpaddb	%xmm2,%xmm14,%xmm1
 	vpxor	%xmm4,%xmm14,%xmm14
 	jmp	.Loop_ctr32
 
 .balign	16
 .Loop_ctr32:
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vaesenc	%xmm15,%xmm11,%xmm11
 	vaesenc	%xmm15,%xmm12,%xmm12
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vaesenc	%xmm15,%xmm14,%xmm14
 	vmovups	(%r12),%xmm15
 	leaq	16(%r12),%r12
 	decl	%r13d
 	jnz	.Loop_ctr32
 
 	vmovdqu	(%r12),%xmm3
 	vaesenc	%xmm15,%xmm9,%xmm9
 	vpxor	0(%rdi),%xmm3,%xmm4
 	vaesenc	%xmm15,%xmm10,%xmm10
 	vpxor	16(%rdi),%xmm3,%xmm5
 	vaesenc	%xmm15,%xmm11,%xmm11
 	vpxor	32(%rdi),%xmm3,%xmm6
 	vaesenc	%xmm15,%xmm12,%xmm12
 	vpxor	48(%rdi),%xmm3,%xmm8
 	vaesenc	%xmm15,%xmm13,%xmm13
 	vpxor	64(%rdi),%xmm3,%xmm2
 	vaesenc	%xmm15,%xmm14,%xmm14
 	vpxor	80(%rdi),%xmm3,%xmm3
 	leaq	96(%rdi),%rdi
 
 	vaesenclast	%xmm4,%xmm9,%xmm9
 	vaesenclast	%xmm5,%xmm10,%xmm10
 	vaesenclast	%xmm6,%xmm11,%xmm11
 	vaesenclast	%xmm8,%xmm12,%xmm12
 	vaesenclast	%xmm2,%xmm13,%xmm13
 	vaesenclast	%xmm3,%xmm14,%xmm14
 	vmovups	%xmm9,0(%rsi)
 	vmovups	%xmm10,16(%rsi)
 	vmovups	%xmm11,32(%rsi)
 	vmovups	%xmm12,48(%rsi)
 	vmovups	%xmm13,64(%rsi)
 	vmovups	%xmm14,80(%rsi)
 	leaq	96(%rsi),%rsi
 
 	RET
 .balign	32
 .Lhandle_ctr32_2:
 	vpshufb	%xmm0,%xmm1,%xmm6
 	vmovdqu	48(%r11),%xmm5
 	vpaddd	64(%r11),%xmm6,%xmm10
 	vpaddd	%xmm5,%xmm6,%xmm11
 	vpaddd	%xmm5,%xmm10,%xmm12
 	vpshufb	%xmm0,%xmm10,%xmm10
 	vpaddd	%xmm5,%xmm11,%xmm13
 	vpshufb	%xmm0,%xmm11,%xmm11
 	vpxor	%xmm4,%xmm10,%xmm10
 	vpaddd	%xmm5,%xmm12,%xmm14
 	vpshufb	%xmm0,%xmm12,%xmm12
 	vpxor	%xmm4,%xmm11,%xmm11
 	vpaddd	%xmm5,%xmm13,%xmm1
 	vpshufb	%xmm0,%xmm13,%xmm13
 	vpxor	%xmm4,%xmm12,%xmm12
 	vpshufb	%xmm0,%xmm14,%xmm14
 	vpxor	%xmm4,%xmm13,%xmm13
 	vpshufb	%xmm0,%xmm1,%xmm1
 	vpxor	%xmm4,%xmm14,%xmm14
 	jmp	.Loop_ctr32
 .cfi_endproc
 SET_SIZE(_aesni_ctr32_6x)
 
 ENTRY_ALIGN(aesni_gcm_encrypt, 32)
 .cfi_startproc
 	ENDBR
 	xorq	%r10,%r10
 	cmpq	$288,%rdx
 	jb	.Lgcm_enc_abort
 
 	leaq	(%rsp),%rax
 .cfi_def_cfa_register	%rax
 	pushq	%rbx
 .cfi_offset	%rbx,-16
 	pushq	%rbp
 .cfi_offset	%rbp,-24
 	pushq	%r12
 .cfi_offset	%r12,-32
 	pushq	%r13
 .cfi_offset	%r13,-40
 	pushq	%r14
 .cfi_offset	%r14,-48
 	pushq	%r15
 .cfi_offset	%r15,-56
 	pushq	%r9
 .cfi_offset	%r9,-64
 	vzeroupper
 
 	vmovdqu	(%r8),%xmm1
 	addq	$-128,%rsp
 	movl	12(%r8),%ebx
 	leaq	.Lbswap_mask(%rip),%r11
 	leaq	-128(%rcx),%r14
 	movq	$0xf80,%r15
 	leaq	128(%rcx),%rcx
 	vmovdqu	(%r11),%xmm0
 	andq	$-128,%rsp
 	movl	504-128(%rcx),%ebp	// ICP has an larger offset for rounds.
 
 	andq	%r15,%r14
 	andq	%rsp,%r15
 	subq	%r14,%r15
 	jc	.Lenc_no_key_aliasing
 	cmpq	$768,%r15
 	jnc	.Lenc_no_key_aliasing
 	subq	%r15,%rsp
 .Lenc_no_key_aliasing:
 
 	leaq	(%rsi),%r14
 	leaq	-192(%rsi,%rdx,1),%r15
 	shrq	$4,%rdx
 
 	call	_aesni_ctr32_6x
 	vpshufb	%xmm0,%xmm9,%xmm8
 	vpshufb	%xmm0,%xmm10,%xmm2
 	vmovdqu	%xmm8,112(%rsp)
 	vpshufb	%xmm0,%xmm11,%xmm4
 	vmovdqu	%xmm2,96(%rsp)
 	vpshufb	%xmm0,%xmm12,%xmm5
 	vmovdqu	%xmm4,80(%rsp)
 	vpshufb	%xmm0,%xmm13,%xmm6
 	vmovdqu	%xmm5,64(%rsp)
 	vpshufb	%xmm0,%xmm14,%xmm7
 	vmovdqu	%xmm6,48(%rsp)
 
 	call	_aesni_ctr32_6x
 
 	vmovdqu	(%r9),%xmm8
 	movq	32(%r9),%r9
 	leaq	32(%r9),%r9
 	subq	$12,%rdx
 	movq	$192,%r10
 	vpshufb	%xmm0,%xmm8,%xmm8
 
 #ifdef HAVE_MOVBE
 #ifdef _KERNEL
 	testl	$1,gcm_avx_can_use_movbe(%rip)
 #else
 	testl	$1,gcm_avx_can_use_movbe@GOTPCREL(%rip)
 #endif
 	jz	1f
 	call	_aesni_ctr32_ghash_6x
 	jmp	2f
 1:
 #endif
 	call	_aesni_ctr32_ghash_no_movbe_6x
 2:
 	vmovdqu	32(%rsp),%xmm7
 	vmovdqu	(%r11),%xmm0
 	vmovdqu	0-32(%r9),%xmm3
 	vpunpckhqdq	%xmm7,%xmm7,%xmm1
 	vmovdqu	32-32(%r9),%xmm15
 	vmovups	%xmm9,-96(%rsi)
 	vpshufb	%xmm0,%xmm9,%xmm9
 	vpxor	%xmm7,%xmm1,%xmm1
 	vmovups	%xmm10,-80(%rsi)
 	vpshufb	%xmm0,%xmm10,%xmm10
 	vmovups	%xmm11,-64(%rsi)
 	vpshufb	%xmm0,%xmm11,%xmm11
 	vmovups	%xmm12,-48(%rsi)
 	vpshufb	%xmm0,%xmm12,%xmm12
 	vmovups	%xmm13,-32(%rsi)
 	vpshufb	%xmm0,%xmm13,%xmm13
 	vmovups	%xmm14,-16(%rsi)
 	vpshufb	%xmm0,%xmm14,%xmm14
 	vmovdqu	%xmm9,16(%rsp)
 	vmovdqu	48(%rsp),%xmm6
 	vmovdqu	16-32(%r9),%xmm0
 	vpunpckhqdq	%xmm6,%xmm6,%xmm2
 	vpclmulqdq	$0x00,%xmm3,%xmm7,%xmm5
 	vpxor	%xmm6,%xmm2,%xmm2
 	vpclmulqdq	$0x11,%xmm3,%xmm7,%xmm7
 	vpclmulqdq	$0x00,%xmm15,%xmm1,%xmm1
 
 	vmovdqu	64(%rsp),%xmm9
 	vpclmulqdq	$0x00,%xmm0,%xmm6,%xmm4
 	vmovdqu	48-32(%r9),%xmm3
 	vpxor	%xmm5,%xmm4,%xmm4
 	vpunpckhqdq	%xmm9,%xmm9,%xmm5
 	vpclmulqdq	$0x11,%xmm0,%xmm6,%xmm6
 	vpxor	%xmm9,%xmm5,%xmm5
 	vpxor	%xmm7,%xmm6,%xmm6
 	vpclmulqdq	$0x10,%xmm15,%xmm2,%xmm2
 	vmovdqu	80-32(%r9),%xmm15
 	vpxor	%xmm1,%xmm2,%xmm2
 
 	vmovdqu	80(%rsp),%xmm1
 	vpclmulqdq	$0x00,%xmm3,%xmm9,%xmm7
 	vmovdqu	64-32(%r9),%xmm0
 	vpxor	%xmm4,%xmm7,%xmm7
 	vpunpckhqdq	%xmm1,%xmm1,%xmm4
 	vpclmulqdq	$0x11,%xmm3,%xmm9,%xmm9
 	vpxor	%xmm1,%xmm4,%xmm4
 	vpxor	%xmm6,%xmm9,%xmm9
 	vpclmulqdq	$0x00,%xmm15,%xmm5,%xmm5
 	vpxor	%xmm2,%xmm5,%xmm5
 
 	vmovdqu	96(%rsp),%xmm2
 	vpclmulqdq	$0x00,%xmm0,%xmm1,%xmm6
 	vmovdqu	96-32(%r9),%xmm3
 	vpxor	%xmm7,%xmm6,%xmm6
 	vpunpckhqdq	%xmm2,%xmm2,%xmm7
 	vpclmulqdq	$0x11,%xmm0,%xmm1,%xmm1
 	vpxor	%xmm2,%xmm7,%xmm7
 	vpxor	%xmm9,%xmm1,%xmm1
 	vpclmulqdq	$0x10,%xmm15,%xmm4,%xmm4
 	vmovdqu	128-32(%r9),%xmm15
 	vpxor	%xmm5,%xmm4,%xmm4
 
 	vpxor	112(%rsp),%xmm8,%xmm8
 	vpclmulqdq	$0x00,%xmm3,%xmm2,%xmm5
 	vmovdqu	112-32(%r9),%xmm0
 	vpunpckhqdq	%xmm8,%xmm8,%xmm9
 	vpxor	%xmm6,%xmm5,%xmm5
 	vpclmulqdq	$0x11,%xmm3,%xmm2,%xmm2
 	vpxor	%xmm8,%xmm9,%xmm9
 	vpxor	%xmm1,%xmm2,%xmm2
 	vpclmulqdq	$0x00,%xmm15,%xmm7,%xmm7
 	vpxor	%xmm4,%xmm7,%xmm4
 
 	vpclmulqdq	$0x00,%xmm0,%xmm8,%xmm6
 	vmovdqu	0-32(%r9),%xmm3
 	vpunpckhqdq	%xmm14,%xmm14,%xmm1
 	vpclmulqdq	$0x11,%xmm0,%xmm8,%xmm8
 	vpxor	%xmm14,%xmm1,%xmm1
 	vpxor	%xmm5,%xmm6,%xmm5
 	vpclmulqdq	$0x10,%xmm15,%xmm9,%xmm9
 	vmovdqu	32-32(%r9),%xmm15
 	vpxor	%xmm2,%xmm8,%xmm7
 	vpxor	%xmm4,%xmm9,%xmm6
 
 	vmovdqu	16-32(%r9),%xmm0
 	vpxor	%xmm5,%xmm7,%xmm9
 	vpclmulqdq	$0x00,%xmm3,%xmm14,%xmm4
 	vpxor	%xmm9,%xmm6,%xmm6
 	vpunpckhqdq	%xmm13,%xmm13,%xmm2
 	vpclmulqdq	$0x11,%xmm3,%xmm14,%xmm14
 	vpxor	%xmm13,%xmm2,%xmm2
 	vpslldq	$8,%xmm6,%xmm9
 	vpclmulqdq	$0x00,%xmm15,%xmm1,%xmm1
 	vpxor	%xmm9,%xmm5,%xmm8
 	vpsrldq	$8,%xmm6,%xmm6
 	vpxor	%xmm6,%xmm7,%xmm7
 
 	vpclmulqdq	$0x00,%xmm0,%xmm13,%xmm5
 	vmovdqu	48-32(%r9),%xmm3
 	vpxor	%xmm4,%xmm5,%xmm5
 	vpunpckhqdq	%xmm12,%xmm12,%xmm9
 	vpclmulqdq	$0x11,%xmm0,%xmm13,%xmm13
 	vpxor	%xmm12,%xmm9,%xmm9
 	vpxor	%xmm14,%xmm13,%xmm13
 	vpalignr	$8,%xmm8,%xmm8,%xmm14
 	vpclmulqdq	$0x10,%xmm15,%xmm2,%xmm2
 	vmovdqu	80-32(%r9),%xmm15
 	vpxor	%xmm1,%xmm2,%xmm2
 
 	vpclmulqdq	$0x00,%xmm3,%xmm12,%xmm4
 	vmovdqu	64-32(%r9),%xmm0
 	vpxor	%xmm5,%xmm4,%xmm4
 	vpunpckhqdq	%xmm11,%xmm11,%xmm1
 	vpclmulqdq	$0x11,%xmm3,%xmm12,%xmm12
 	vpxor	%xmm11,%xmm1,%xmm1
 	vpxor	%xmm13,%xmm12,%xmm12
 	vxorps	16(%rsp),%xmm7,%xmm7
 	vpclmulqdq	$0x00,%xmm15,%xmm9,%xmm9
 	vpxor	%xmm2,%xmm9,%xmm9
 
 	vpclmulqdq	$0x10,16(%r11),%xmm8,%xmm8
 	vxorps	%xmm14,%xmm8,%xmm8
 
 	vpclmulqdq	$0x00,%xmm0,%xmm11,%xmm5
 	vmovdqu	96-32(%r9),%xmm3
 	vpxor	%xmm4,%xmm5,%xmm5
 	vpunpckhqdq	%xmm10,%xmm10,%xmm2
 	vpclmulqdq	$0x11,%xmm0,%xmm11,%xmm11
 	vpxor	%xmm10,%xmm2,%xmm2
 	vpalignr	$8,%xmm8,%xmm8,%xmm14
 	vpxor	%xmm12,%xmm11,%xmm11
 	vpclmulqdq	$0x10,%xmm15,%xmm1,%xmm1
 	vmovdqu	128-32(%r9),%xmm15
 	vpxor	%xmm9,%xmm1,%xmm1
 
 	vxorps	%xmm7,%xmm14,%xmm14
 	vpclmulqdq	$0x10,16(%r11),%xmm8,%xmm8
 	vxorps	%xmm14,%xmm8,%xmm8
 
 	vpclmulqdq	$0x00,%xmm3,%xmm10,%xmm4
 	vmovdqu	112-32(%r9),%xmm0
 	vpxor	%xmm5,%xmm4,%xmm4
 	vpunpckhqdq	%xmm8,%xmm8,%xmm9
 	vpclmulqdq	$0x11,%xmm3,%xmm10,%xmm10
 	vpxor	%xmm8,%xmm9,%xmm9
 	vpxor	%xmm11,%xmm10,%xmm10
 	vpclmulqdq	$0x00,%xmm15,%xmm2,%xmm2
 	vpxor	%xmm1,%xmm2,%xmm2
 
 	vpclmulqdq	$0x00,%xmm0,%xmm8,%xmm5
 	vpclmulqdq	$0x11,%xmm0,%xmm8,%xmm7
 	vpxor	%xmm4,%xmm5,%xmm5
 	vpclmulqdq	$0x10,%xmm15,%xmm9,%xmm6
 	vpxor	%xmm10,%xmm7,%xmm7
 	vpxor	%xmm2,%xmm6,%xmm6
 
 	vpxor	%xmm5,%xmm7,%xmm4
 	vpxor	%xmm4,%xmm6,%xmm6
 	vpslldq	$8,%xmm6,%xmm1
 	vmovdqu	16(%r11),%xmm3
 	vpsrldq	$8,%xmm6,%xmm6
 	vpxor	%xmm1,%xmm5,%xmm8
 	vpxor	%xmm6,%xmm7,%xmm7
 
 	vpalignr	$8,%xmm8,%xmm8,%xmm2
 	vpclmulqdq	$0x10,%xmm3,%xmm8,%xmm8
 	vpxor	%xmm2,%xmm8,%xmm8
 
 	vpalignr	$8,%xmm8,%xmm8,%xmm2
 	vpclmulqdq	$0x10,%xmm3,%xmm8,%xmm8
 	vpxor	%xmm7,%xmm2,%xmm2
 	vpxor	%xmm2,%xmm8,%xmm8
 	vpshufb	(%r11),%xmm8,%xmm8
 	movq	-56(%rax),%r9
 .cfi_restore	%r9
 	vmovdqu	%xmm8,(%r9)
 
 	vzeroupper
 	movq	-48(%rax),%r15
 .cfi_restore	%r15
 	movq	-40(%rax),%r14
 .cfi_restore	%r14
 	movq	-32(%rax),%r13
 .cfi_restore	%r13
 	movq	-24(%rax),%r12
 .cfi_restore	%r12
 	movq	-16(%rax),%rbp
 .cfi_restore	%rbp
 	movq	-8(%rax),%rbx
 .cfi_restore	%rbx
 	leaq	(%rax),%rsp
 .cfi_def_cfa_register	%rsp
 .Lgcm_enc_abort:
 	movq	%r10,%rax
 	RET
 .cfi_endproc
 SET_SIZE(aesni_gcm_encrypt)
+STACK_FRAME_NON_STANDARD aesni_gcm_encrypt
 
 #endif /* !_WIN32 || _KERNEL */
 
 /* Some utility routines */
 
 /*
  * clear all fpu registers
  * void clear_fpu_regs_avx(void);
  */
 ENTRY_ALIGN(clear_fpu_regs_avx, 32)
 	vzeroall
 	RET
 SET_SIZE(clear_fpu_regs_avx)
 
 /*
  * void gcm_xor_avx(const uint8_t *src, uint8_t *dst);
  *
  * XORs one pair of unaligned 128-bit blocks from `src' and `dst' and
  * stores the result at `dst'. The XOR is performed using FPU registers,
  * so make sure FPU state is saved when running this in the kernel.
  */
 ENTRY_ALIGN(gcm_xor_avx, 32)
 	movdqu  (%rdi), %xmm0
 	movdqu  (%rsi), %xmm1
 	pxor    %xmm1, %xmm0
 	movdqu  %xmm0, (%rsi)
 	RET
 SET_SIZE(gcm_xor_avx)
 
 /*
  * Toggle a boolean_t value atomically and return the new value.
  * boolean_t atomic_toggle_boolean_nv(volatile boolean_t *);
  */
 ENTRY_ALIGN(atomic_toggle_boolean_nv, 32)
 	xorl	%eax, %eax
 	lock
 	xorl	$1, (%rdi)
 	jz	1f
 	movl	$1, %eax
 1:
 	RET
 SET_SIZE(atomic_toggle_boolean_nv)
 
 SECTION_STATIC
 
 .balign	64
 .Lbswap_mask:
 .byte	15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0
 .Lpoly:
 .byte	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2
 .Lone_msb:
 .byte	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1
 .Ltwo_lsb:
 .byte	2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
 .Lone_lsb:
 .byte	1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
 .byte	65,69,83,45,78,73,32,71,67,77,32,109,111,100,117,108,101,32,102,111,114,32,120,56,54,95,54,52,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0
 .balign	64
 
+/* Workaround for missing asm macro in RHEL 8. */
+#if defined(__linux__) && defined(HAVE_STACK_FRAME_NON_STANDARD) && \
+    ! defined(HAVE_STACK_FRAME_NON_STANDARD_ASM)
+.section .discard.func_stack_frame_non_standard, "aw"
+#ifdef HAVE_MOVBE
+	.long _aesni_ctr32_ghash_6x  - .
+#endif
+	.long _aesni_ctr32_ghash_no_movbe_6x - .
+	.long aesni_gcm_decrypt - .
+	.long aesni_gcm_encrypt - .
+#endif
+
 /* Mark the stack non-executable. */
 #if defined(__linux__) && defined(__ELF__)
 .section .note.GNU-stack,"",%progbits
 #endif
 
 #endif /* defined(__x86_64__) && defined(HAVE_AVX) && defined(HAVE_AES) ... */
diff --git a/sys/contrib/openzfs/module/icp/asm-x86_64/sha2/sha256-x86_64.S b/sys/contrib/openzfs/module/icp/asm-x86_64/sha2/sha256-x86_64.S
index d3e5e3f0d080..edd68979a5f4 100644
--- a/sys/contrib/openzfs/module/icp/asm-x86_64/sha2/sha256-x86_64.S
+++ b/sys/contrib/openzfs/module/icp/asm-x86_64/sha2/sha256-x86_64.S
@@ -1,5104 +1,5121 @@
 /*
  * Copyright 2004-2022 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     https://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 
 /*
  * Portions Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
  * - modified assembly to fit into OpenZFS
  */
 
 #if defined(__x86_64)
 
 #define _ASM
 #include <sys/asm_linkage.h>
+#include <sys/frame.h>
 
 SECTION_STATIC
 
 .balign	64
 SET_OBJ(K256)
 K256:
 .long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
 .long	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
 .long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
 .long	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
 .long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
 .long	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
 .long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
 .long	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
 .long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
 .long	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
 .long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
 .long	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
 .long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
 .long	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
 .long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
 .long	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
 .long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
 .long	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
 .long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
 .long	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
 .long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
 .long	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
 .long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
 .long	0xd192e819,0xd6990624,0xf40e3585,0x106aa070
 .long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
 .long	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
 .long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
 .long	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
 .long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
 .long	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
 .long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
 .long	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
 
 .long	0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f
 .long	0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f
 .long	0x03020100,0x0b0a0908,0xffffffff,0xffffffff
 .long	0x03020100,0x0b0a0908,0xffffffff,0xffffffff
 .long	0xffffffff,0xffffffff,0x03020100,0x0b0a0908
 .long	0xffffffff,0xffffffff,0x03020100,0x0b0a0908
 
 ENTRY_ALIGN(zfs_sha256_transform_x64, 16)
 .cfi_startproc
 	ENDBR
 	movq	%rsp,%rax
 .cfi_def_cfa_register	%rax
 	pushq	%rbx
 .cfi_offset	%rbx,-16
 	pushq	%rbp
 .cfi_offset	%rbp,-24
 	pushq	%r12
 .cfi_offset	%r12,-32
 	pushq	%r13
 .cfi_offset	%r13,-40
 	pushq	%r14
 .cfi_offset	%r14,-48
 	pushq	%r15
 .cfi_offset	%r15,-56
 	shlq	$4,%rdx
 	subq	$64+32,%rsp
 	leaq	(%rsi,%rdx,4),%rdx
 	andq	$-64,%rsp
 	movq	%rdi,64+0(%rsp)
 	movq	%rsi,64+8(%rsp)
 	movq	%rdx,64+16(%rsp)
 	movq	%rax,88(%rsp)
 .cfi_escape	0x0f,0x06,0x77,0xd8,0x00,0x06,0x23,0x08
 .Lprologue:
 	movl	0(%rdi),%eax
 	movl	4(%rdi),%ebx
 	movl	8(%rdi),%ecx
 	movl	12(%rdi),%edx
 	movl	16(%rdi),%r8d
 	movl	20(%rdi),%r9d
 	movl	24(%rdi),%r10d
 	movl	28(%rdi),%r11d
 	jmp	.Lloop
 .balign	16
 .Lloop:
 	movl	%ebx,%edi
 	leaq	K256(%rip),%rbp
 	xorl	%ecx,%edi
 	movl	0(%rsi),%r12d
 	movl	%r8d,%r13d
 	movl	%eax,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%r9d,%r15d
 	xorl	%r8d,%r13d
 	rorl	$9,%r14d
 	xorl	%r10d,%r15d
 	movl	%r12d,0(%rsp)
 	xorl	%eax,%r14d
 	andl	%r8d,%r15d
 	rorl	$5,%r13d
 	addl	%r11d,%r12d
 	xorl	%r10d,%r15d
 	rorl	$11,%r14d
 	xorl	%r8d,%r13d
 	addl	%r15d,%r12d
 	movl	%eax,%r15d
 	addl	(%rbp),%r12d
 	xorl	%eax,%r14d
 	xorl	%ebx,%r15d
 	rorl	$6,%r13d
 	movl	%ebx,%r11d
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%r11d
 	addl	%r12d,%edx
 	addl	%r12d,%r11d
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%r11d
 	movl	4(%rsi),%r12d
 	movl	%edx,%r13d
 	movl	%r11d,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%r8d,%edi
 	xorl	%edx,%r13d
 	rorl	$9,%r14d
 	xorl	%r9d,%edi
 	movl	%r12d,4(%rsp)
 	xorl	%r11d,%r14d
 	andl	%edx,%edi
 	rorl	$5,%r13d
 	addl	%r10d,%r12d
 	xorl	%r9d,%edi
 	rorl	$11,%r14d
 	xorl	%edx,%r13d
 	addl	%edi,%r12d
 	movl	%r11d,%edi
 	addl	(%rbp),%r12d
 	xorl	%r11d,%r14d
 	xorl	%eax,%edi
 	rorl	$6,%r13d
 	movl	%eax,%r10d
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%r10d
 	addl	%r12d,%ecx
 	addl	%r12d,%r10d
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%r10d
 	movl	8(%rsi),%r12d
 	movl	%ecx,%r13d
 	movl	%r10d,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%edx,%r15d
 	xorl	%ecx,%r13d
 	rorl	$9,%r14d
 	xorl	%r8d,%r15d
 	movl	%r12d,8(%rsp)
 	xorl	%r10d,%r14d
 	andl	%ecx,%r15d
 	rorl	$5,%r13d
 	addl	%r9d,%r12d
 	xorl	%r8d,%r15d
 	rorl	$11,%r14d
 	xorl	%ecx,%r13d
 	addl	%r15d,%r12d
 	movl	%r10d,%r15d
 	addl	(%rbp),%r12d
 	xorl	%r10d,%r14d
 	xorl	%r11d,%r15d
 	rorl	$6,%r13d
 	movl	%r11d,%r9d
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%r9d
 	addl	%r12d,%ebx
 	addl	%r12d,%r9d
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%r9d
 	movl	12(%rsi),%r12d
 	movl	%ebx,%r13d
 	movl	%r9d,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%ecx,%edi
 	xorl	%ebx,%r13d
 	rorl	$9,%r14d
 	xorl	%edx,%edi
 	movl	%r12d,12(%rsp)
 	xorl	%r9d,%r14d
 	andl	%ebx,%edi
 	rorl	$5,%r13d
 	addl	%r8d,%r12d
 	xorl	%edx,%edi
 	rorl	$11,%r14d
 	xorl	%ebx,%r13d
 	addl	%edi,%r12d
 	movl	%r9d,%edi
 	addl	(%rbp),%r12d
 	xorl	%r9d,%r14d
 	xorl	%r10d,%edi
 	rorl	$6,%r13d
 	movl	%r10d,%r8d
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%r8d
 	addl	%r12d,%eax
 	addl	%r12d,%r8d
 	leaq	20(%rbp),%rbp
 	addl	%r14d,%r8d
 	movl	16(%rsi),%r12d
 	movl	%eax,%r13d
 	movl	%r8d,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%ebx,%r15d
 	xorl	%eax,%r13d
 	rorl	$9,%r14d
 	xorl	%ecx,%r15d
 	movl	%r12d,16(%rsp)
 	xorl	%r8d,%r14d
 	andl	%eax,%r15d
 	rorl	$5,%r13d
 	addl	%edx,%r12d
 	xorl	%ecx,%r15d
 	rorl	$11,%r14d
 	xorl	%eax,%r13d
 	addl	%r15d,%r12d
 	movl	%r8d,%r15d
 	addl	(%rbp),%r12d
 	xorl	%r8d,%r14d
 	xorl	%r9d,%r15d
 	rorl	$6,%r13d
 	movl	%r9d,%edx
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%edx
 	addl	%r12d,%r11d
 	addl	%r12d,%edx
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%edx
 	movl	20(%rsi),%r12d
 	movl	%r11d,%r13d
 	movl	%edx,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%eax,%edi
 	xorl	%r11d,%r13d
 	rorl	$9,%r14d
 	xorl	%ebx,%edi
 	movl	%r12d,20(%rsp)
 	xorl	%edx,%r14d
 	andl	%r11d,%edi
 	rorl	$5,%r13d
 	addl	%ecx,%r12d
 	xorl	%ebx,%edi
 	rorl	$11,%r14d
 	xorl	%r11d,%r13d
 	addl	%edi,%r12d
 	movl	%edx,%edi
 	addl	(%rbp),%r12d
 	xorl	%edx,%r14d
 	xorl	%r8d,%edi
 	rorl	$6,%r13d
 	movl	%r8d,%ecx
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%ecx
 	addl	%r12d,%r10d
 	addl	%r12d,%ecx
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%ecx
 	movl	24(%rsi),%r12d
 	movl	%r10d,%r13d
 	movl	%ecx,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%r11d,%r15d
 	xorl	%r10d,%r13d
 	rorl	$9,%r14d
 	xorl	%eax,%r15d
 	movl	%r12d,24(%rsp)
 	xorl	%ecx,%r14d
 	andl	%r10d,%r15d
 	rorl	$5,%r13d
 	addl	%ebx,%r12d
 	xorl	%eax,%r15d
 	rorl	$11,%r14d
 	xorl	%r10d,%r13d
 	addl	%r15d,%r12d
 	movl	%ecx,%r15d
 	addl	(%rbp),%r12d
 	xorl	%ecx,%r14d
 	xorl	%edx,%r15d
 	rorl	$6,%r13d
 	movl	%edx,%ebx
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%ebx
 	addl	%r12d,%r9d
 	addl	%r12d,%ebx
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%ebx
 	movl	28(%rsi),%r12d
 	movl	%r9d,%r13d
 	movl	%ebx,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%r10d,%edi
 	xorl	%r9d,%r13d
 	rorl	$9,%r14d
 	xorl	%r11d,%edi
 	movl	%r12d,28(%rsp)
 	xorl	%ebx,%r14d
 	andl	%r9d,%edi
 	rorl	$5,%r13d
 	addl	%eax,%r12d
 	xorl	%r11d,%edi
 	rorl	$11,%r14d
 	xorl	%r9d,%r13d
 	addl	%edi,%r12d
 	movl	%ebx,%edi
 	addl	(%rbp),%r12d
 	xorl	%ebx,%r14d
 	xorl	%ecx,%edi
 	rorl	$6,%r13d
 	movl	%ecx,%eax
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%eax
 	addl	%r12d,%r8d
 	addl	%r12d,%eax
 	leaq	20(%rbp),%rbp
 	addl	%r14d,%eax
 	movl	32(%rsi),%r12d
 	movl	%r8d,%r13d
 	movl	%eax,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%r9d,%r15d
 	xorl	%r8d,%r13d
 	rorl	$9,%r14d
 	xorl	%r10d,%r15d
 	movl	%r12d,32(%rsp)
 	xorl	%eax,%r14d
 	andl	%r8d,%r15d
 	rorl	$5,%r13d
 	addl	%r11d,%r12d
 	xorl	%r10d,%r15d
 	rorl	$11,%r14d
 	xorl	%r8d,%r13d
 	addl	%r15d,%r12d
 	movl	%eax,%r15d
 	addl	(%rbp),%r12d
 	xorl	%eax,%r14d
 	xorl	%ebx,%r15d
 	rorl	$6,%r13d
 	movl	%ebx,%r11d
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%r11d
 	addl	%r12d,%edx
 	addl	%r12d,%r11d
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%r11d
 	movl	36(%rsi),%r12d
 	movl	%edx,%r13d
 	movl	%r11d,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%r8d,%edi
 	xorl	%edx,%r13d
 	rorl	$9,%r14d
 	xorl	%r9d,%edi
 	movl	%r12d,36(%rsp)
 	xorl	%r11d,%r14d
 	andl	%edx,%edi
 	rorl	$5,%r13d
 	addl	%r10d,%r12d
 	xorl	%r9d,%edi
 	rorl	$11,%r14d
 	xorl	%edx,%r13d
 	addl	%edi,%r12d
 	movl	%r11d,%edi
 	addl	(%rbp),%r12d
 	xorl	%r11d,%r14d
 	xorl	%eax,%edi
 	rorl	$6,%r13d
 	movl	%eax,%r10d
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%r10d
 	addl	%r12d,%ecx
 	addl	%r12d,%r10d
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%r10d
 	movl	40(%rsi),%r12d
 	movl	%ecx,%r13d
 	movl	%r10d,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%edx,%r15d
 	xorl	%ecx,%r13d
 	rorl	$9,%r14d
 	xorl	%r8d,%r15d
 	movl	%r12d,40(%rsp)
 	xorl	%r10d,%r14d
 	andl	%ecx,%r15d
 	rorl	$5,%r13d
 	addl	%r9d,%r12d
 	xorl	%r8d,%r15d
 	rorl	$11,%r14d
 	xorl	%ecx,%r13d
 	addl	%r15d,%r12d
 	movl	%r10d,%r15d
 	addl	(%rbp),%r12d
 	xorl	%r10d,%r14d
 	xorl	%r11d,%r15d
 	rorl	$6,%r13d
 	movl	%r11d,%r9d
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%r9d
 	addl	%r12d,%ebx
 	addl	%r12d,%r9d
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%r9d
 	movl	44(%rsi),%r12d
 	movl	%ebx,%r13d
 	movl	%r9d,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%ecx,%edi
 	xorl	%ebx,%r13d
 	rorl	$9,%r14d
 	xorl	%edx,%edi
 	movl	%r12d,44(%rsp)
 	xorl	%r9d,%r14d
 	andl	%ebx,%edi
 	rorl	$5,%r13d
 	addl	%r8d,%r12d
 	xorl	%edx,%edi
 	rorl	$11,%r14d
 	xorl	%ebx,%r13d
 	addl	%edi,%r12d
 	movl	%r9d,%edi
 	addl	(%rbp),%r12d
 	xorl	%r9d,%r14d
 	xorl	%r10d,%edi
 	rorl	$6,%r13d
 	movl	%r10d,%r8d
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%r8d
 	addl	%r12d,%eax
 	addl	%r12d,%r8d
 	leaq	20(%rbp),%rbp
 	addl	%r14d,%r8d
 	movl	48(%rsi),%r12d
 	movl	%eax,%r13d
 	movl	%r8d,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%ebx,%r15d
 	xorl	%eax,%r13d
 	rorl	$9,%r14d
 	xorl	%ecx,%r15d
 	movl	%r12d,48(%rsp)
 	xorl	%r8d,%r14d
 	andl	%eax,%r15d
 	rorl	$5,%r13d
 	addl	%edx,%r12d
 	xorl	%ecx,%r15d
 	rorl	$11,%r14d
 	xorl	%eax,%r13d
 	addl	%r15d,%r12d
 	movl	%r8d,%r15d
 	addl	(%rbp),%r12d
 	xorl	%r8d,%r14d
 	xorl	%r9d,%r15d
 	rorl	$6,%r13d
 	movl	%r9d,%edx
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%edx
 	addl	%r12d,%r11d
 	addl	%r12d,%edx
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%edx
 	movl	52(%rsi),%r12d
 	movl	%r11d,%r13d
 	movl	%edx,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%eax,%edi
 	xorl	%r11d,%r13d
 	rorl	$9,%r14d
 	xorl	%ebx,%edi
 	movl	%r12d,52(%rsp)
 	xorl	%edx,%r14d
 	andl	%r11d,%edi
 	rorl	$5,%r13d
 	addl	%ecx,%r12d
 	xorl	%ebx,%edi
 	rorl	$11,%r14d
 	xorl	%r11d,%r13d
 	addl	%edi,%r12d
 	movl	%edx,%edi
 	addl	(%rbp),%r12d
 	xorl	%edx,%r14d
 	xorl	%r8d,%edi
 	rorl	$6,%r13d
 	movl	%r8d,%ecx
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%ecx
 	addl	%r12d,%r10d
 	addl	%r12d,%ecx
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%ecx
 	movl	56(%rsi),%r12d
 	movl	%r10d,%r13d
 	movl	%ecx,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%r11d,%r15d
 	xorl	%r10d,%r13d
 	rorl	$9,%r14d
 	xorl	%eax,%r15d
 	movl	%r12d,56(%rsp)
 	xorl	%ecx,%r14d
 	andl	%r10d,%r15d
 	rorl	$5,%r13d
 	addl	%ebx,%r12d
 	xorl	%eax,%r15d
 	rorl	$11,%r14d
 	xorl	%r10d,%r13d
 	addl	%r15d,%r12d
 	movl	%ecx,%r15d
 	addl	(%rbp),%r12d
 	xorl	%ecx,%r14d
 	xorl	%edx,%r15d
 	rorl	$6,%r13d
 	movl	%edx,%ebx
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%ebx
 	addl	%r12d,%r9d
 	addl	%r12d,%ebx
 	leaq	4(%rbp),%rbp
 	addl	%r14d,%ebx
 	movl	60(%rsi),%r12d
 	movl	%r9d,%r13d
 	movl	%ebx,%r14d
 	bswapl	%r12d
 	rorl	$14,%r13d
 	movl	%r10d,%edi
 	xorl	%r9d,%r13d
 	rorl	$9,%r14d
 	xorl	%r11d,%edi
 	movl	%r12d,60(%rsp)
 	xorl	%ebx,%r14d
 	andl	%r9d,%edi
 	rorl	$5,%r13d
 	addl	%eax,%r12d
 	xorl	%r11d,%edi
 	rorl	$11,%r14d
 	xorl	%r9d,%r13d
 	addl	%edi,%r12d
 	movl	%ebx,%edi
 	addl	(%rbp),%r12d
 	xorl	%ebx,%r14d
 	xorl	%ecx,%edi
 	rorl	$6,%r13d
 	movl	%ecx,%eax
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%eax
 	addl	%r12d,%r8d
 	addl	%r12d,%eax
 	leaq	20(%rbp),%rbp
 	jmp	.Lrounds_16_xx
 .balign	16
 .Lrounds_16_xx:
 	movl	4(%rsp),%r13d
 	movl	56(%rsp),%r15d
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%eax
 	movl	%r15d,%r14d
 	rorl	$2,%r15d
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%r15d
 	shrl	$10,%r14d
 	rorl	$17,%r15d
 	xorl	%r13d,%r12d
 	xorl	%r14d,%r15d
 	addl	36(%rsp),%r12d
 	addl	0(%rsp),%r12d
 	movl	%r8d,%r13d
 	addl	%r15d,%r12d
 	movl	%eax,%r14d
 	rorl	$14,%r13d
 	movl	%r9d,%r15d
 	xorl	%r8d,%r13d
 	rorl	$9,%r14d
 	xorl	%r10d,%r15d
 	movl	%r12d,0(%rsp)
 	xorl	%eax,%r14d
 	andl	%r8d,%r15d
 	rorl	$5,%r13d
 	addl	%r11d,%r12d
 	xorl	%r10d,%r15d
 	rorl	$11,%r14d
 	xorl	%r8d,%r13d
 	addl	%r15d,%r12d
 	movl	%eax,%r15d
 	addl	(%rbp),%r12d
 	xorl	%eax,%r14d
 	xorl	%ebx,%r15d
 	rorl	$6,%r13d
 	movl	%ebx,%r11d
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%r11d
 	addl	%r12d,%edx
 	addl	%r12d,%r11d
 	leaq	4(%rbp),%rbp
 	movl	8(%rsp),%r13d
 	movl	60(%rsp),%edi
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%r11d
 	movl	%edi,%r14d
 	rorl	$2,%edi
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%edi
 	shrl	$10,%r14d
 	rorl	$17,%edi
 	xorl	%r13d,%r12d
 	xorl	%r14d,%edi
 	addl	40(%rsp),%r12d
 	addl	4(%rsp),%r12d
 	movl	%edx,%r13d
 	addl	%edi,%r12d
 	movl	%r11d,%r14d
 	rorl	$14,%r13d
 	movl	%r8d,%edi
 	xorl	%edx,%r13d
 	rorl	$9,%r14d
 	xorl	%r9d,%edi
 	movl	%r12d,4(%rsp)
 	xorl	%r11d,%r14d
 	andl	%edx,%edi
 	rorl	$5,%r13d
 	addl	%r10d,%r12d
 	xorl	%r9d,%edi
 	rorl	$11,%r14d
 	xorl	%edx,%r13d
 	addl	%edi,%r12d
 	movl	%r11d,%edi
 	addl	(%rbp),%r12d
 	xorl	%r11d,%r14d
 	xorl	%eax,%edi
 	rorl	$6,%r13d
 	movl	%eax,%r10d
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%r10d
 	addl	%r12d,%ecx
 	addl	%r12d,%r10d
 	leaq	4(%rbp),%rbp
 	movl	12(%rsp),%r13d
 	movl	0(%rsp),%r15d
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%r10d
 	movl	%r15d,%r14d
 	rorl	$2,%r15d
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%r15d
 	shrl	$10,%r14d
 	rorl	$17,%r15d
 	xorl	%r13d,%r12d
 	xorl	%r14d,%r15d
 	addl	44(%rsp),%r12d
 	addl	8(%rsp),%r12d
 	movl	%ecx,%r13d
 	addl	%r15d,%r12d
 	movl	%r10d,%r14d
 	rorl	$14,%r13d
 	movl	%edx,%r15d
 	xorl	%ecx,%r13d
 	rorl	$9,%r14d
 	xorl	%r8d,%r15d
 	movl	%r12d,8(%rsp)
 	xorl	%r10d,%r14d
 	andl	%ecx,%r15d
 	rorl	$5,%r13d
 	addl	%r9d,%r12d
 	xorl	%r8d,%r15d
 	rorl	$11,%r14d
 	xorl	%ecx,%r13d
 	addl	%r15d,%r12d
 	movl	%r10d,%r15d
 	addl	(%rbp),%r12d
 	xorl	%r10d,%r14d
 	xorl	%r11d,%r15d
 	rorl	$6,%r13d
 	movl	%r11d,%r9d
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%r9d
 	addl	%r12d,%ebx
 	addl	%r12d,%r9d
 	leaq	4(%rbp),%rbp
 	movl	16(%rsp),%r13d
 	movl	4(%rsp),%edi
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%r9d
 	movl	%edi,%r14d
 	rorl	$2,%edi
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%edi
 	shrl	$10,%r14d
 	rorl	$17,%edi
 	xorl	%r13d,%r12d
 	xorl	%r14d,%edi
 	addl	48(%rsp),%r12d
 	addl	12(%rsp),%r12d
 	movl	%ebx,%r13d
 	addl	%edi,%r12d
 	movl	%r9d,%r14d
 	rorl	$14,%r13d
 	movl	%ecx,%edi
 	xorl	%ebx,%r13d
 	rorl	$9,%r14d
 	xorl	%edx,%edi
 	movl	%r12d,12(%rsp)
 	xorl	%r9d,%r14d
 	andl	%ebx,%edi
 	rorl	$5,%r13d
 	addl	%r8d,%r12d
 	xorl	%edx,%edi
 	rorl	$11,%r14d
 	xorl	%ebx,%r13d
 	addl	%edi,%r12d
 	movl	%r9d,%edi
 	addl	(%rbp),%r12d
 	xorl	%r9d,%r14d
 	xorl	%r10d,%edi
 	rorl	$6,%r13d
 	movl	%r10d,%r8d
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%r8d
 	addl	%r12d,%eax
 	addl	%r12d,%r8d
 	leaq	20(%rbp),%rbp
 	movl	20(%rsp),%r13d
 	movl	8(%rsp),%r15d
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%r8d
 	movl	%r15d,%r14d
 	rorl	$2,%r15d
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%r15d
 	shrl	$10,%r14d
 	rorl	$17,%r15d
 	xorl	%r13d,%r12d
 	xorl	%r14d,%r15d
 	addl	52(%rsp),%r12d
 	addl	16(%rsp),%r12d
 	movl	%eax,%r13d
 	addl	%r15d,%r12d
 	movl	%r8d,%r14d
 	rorl	$14,%r13d
 	movl	%ebx,%r15d
 	xorl	%eax,%r13d
 	rorl	$9,%r14d
 	xorl	%ecx,%r15d
 	movl	%r12d,16(%rsp)
 	xorl	%r8d,%r14d
 	andl	%eax,%r15d
 	rorl	$5,%r13d
 	addl	%edx,%r12d
 	xorl	%ecx,%r15d
 	rorl	$11,%r14d
 	xorl	%eax,%r13d
 	addl	%r15d,%r12d
 	movl	%r8d,%r15d
 	addl	(%rbp),%r12d
 	xorl	%r8d,%r14d
 	xorl	%r9d,%r15d
 	rorl	$6,%r13d
 	movl	%r9d,%edx
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%edx
 	addl	%r12d,%r11d
 	addl	%r12d,%edx
 	leaq	4(%rbp),%rbp
 	movl	24(%rsp),%r13d
 	movl	12(%rsp),%edi
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%edx
 	movl	%edi,%r14d
 	rorl	$2,%edi
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%edi
 	shrl	$10,%r14d
 	rorl	$17,%edi
 	xorl	%r13d,%r12d
 	xorl	%r14d,%edi
 	addl	56(%rsp),%r12d
 	addl	20(%rsp),%r12d
 	movl	%r11d,%r13d
 	addl	%edi,%r12d
 	movl	%edx,%r14d
 	rorl	$14,%r13d
 	movl	%eax,%edi
 	xorl	%r11d,%r13d
 	rorl	$9,%r14d
 	xorl	%ebx,%edi
 	movl	%r12d,20(%rsp)
 	xorl	%edx,%r14d
 	andl	%r11d,%edi
 	rorl	$5,%r13d
 	addl	%ecx,%r12d
 	xorl	%ebx,%edi
 	rorl	$11,%r14d
 	xorl	%r11d,%r13d
 	addl	%edi,%r12d
 	movl	%edx,%edi
 	addl	(%rbp),%r12d
 	xorl	%edx,%r14d
 	xorl	%r8d,%edi
 	rorl	$6,%r13d
 	movl	%r8d,%ecx
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%ecx
 	addl	%r12d,%r10d
 	addl	%r12d,%ecx
 	leaq	4(%rbp),%rbp
 	movl	28(%rsp),%r13d
 	movl	16(%rsp),%r15d
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%ecx
 	movl	%r15d,%r14d
 	rorl	$2,%r15d
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%r15d
 	shrl	$10,%r14d
 	rorl	$17,%r15d
 	xorl	%r13d,%r12d
 	xorl	%r14d,%r15d
 	addl	60(%rsp),%r12d
 	addl	24(%rsp),%r12d
 	movl	%r10d,%r13d
 	addl	%r15d,%r12d
 	movl	%ecx,%r14d
 	rorl	$14,%r13d
 	movl	%r11d,%r15d
 	xorl	%r10d,%r13d
 	rorl	$9,%r14d
 	xorl	%eax,%r15d
 	movl	%r12d,24(%rsp)
 	xorl	%ecx,%r14d
 	andl	%r10d,%r15d
 	rorl	$5,%r13d
 	addl	%ebx,%r12d
 	xorl	%eax,%r15d
 	rorl	$11,%r14d
 	xorl	%r10d,%r13d
 	addl	%r15d,%r12d
 	movl	%ecx,%r15d
 	addl	(%rbp),%r12d
 	xorl	%ecx,%r14d
 	xorl	%edx,%r15d
 	rorl	$6,%r13d
 	movl	%edx,%ebx
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%ebx
 	addl	%r12d,%r9d
 	addl	%r12d,%ebx
 	leaq	4(%rbp),%rbp
 	movl	32(%rsp),%r13d
 	movl	20(%rsp),%edi
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%ebx
 	movl	%edi,%r14d
 	rorl	$2,%edi
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%edi
 	shrl	$10,%r14d
 	rorl	$17,%edi
 	xorl	%r13d,%r12d
 	xorl	%r14d,%edi
 	addl	0(%rsp),%r12d
 	addl	28(%rsp),%r12d
 	movl	%r9d,%r13d
 	addl	%edi,%r12d
 	movl	%ebx,%r14d
 	rorl	$14,%r13d
 	movl	%r10d,%edi
 	xorl	%r9d,%r13d
 	rorl	$9,%r14d
 	xorl	%r11d,%edi
 	movl	%r12d,28(%rsp)
 	xorl	%ebx,%r14d
 	andl	%r9d,%edi
 	rorl	$5,%r13d
 	addl	%eax,%r12d
 	xorl	%r11d,%edi
 	rorl	$11,%r14d
 	xorl	%r9d,%r13d
 	addl	%edi,%r12d
 	movl	%ebx,%edi
 	addl	(%rbp),%r12d
 	xorl	%ebx,%r14d
 	xorl	%ecx,%edi
 	rorl	$6,%r13d
 	movl	%ecx,%eax
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%eax
 	addl	%r12d,%r8d
 	addl	%r12d,%eax
 	leaq	20(%rbp),%rbp
 	movl	36(%rsp),%r13d
 	movl	24(%rsp),%r15d
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%eax
 	movl	%r15d,%r14d
 	rorl	$2,%r15d
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%r15d
 	shrl	$10,%r14d
 	rorl	$17,%r15d
 	xorl	%r13d,%r12d
 	xorl	%r14d,%r15d
 	addl	4(%rsp),%r12d
 	addl	32(%rsp),%r12d
 	movl	%r8d,%r13d
 	addl	%r15d,%r12d
 	movl	%eax,%r14d
 	rorl	$14,%r13d
 	movl	%r9d,%r15d
 	xorl	%r8d,%r13d
 	rorl	$9,%r14d
 	xorl	%r10d,%r15d
 	movl	%r12d,32(%rsp)
 	xorl	%eax,%r14d
 	andl	%r8d,%r15d
 	rorl	$5,%r13d
 	addl	%r11d,%r12d
 	xorl	%r10d,%r15d
 	rorl	$11,%r14d
 	xorl	%r8d,%r13d
 	addl	%r15d,%r12d
 	movl	%eax,%r15d
 	addl	(%rbp),%r12d
 	xorl	%eax,%r14d
 	xorl	%ebx,%r15d
 	rorl	$6,%r13d
 	movl	%ebx,%r11d
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%r11d
 	addl	%r12d,%edx
 	addl	%r12d,%r11d
 	leaq	4(%rbp),%rbp
 	movl	40(%rsp),%r13d
 	movl	28(%rsp),%edi
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%r11d
 	movl	%edi,%r14d
 	rorl	$2,%edi
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%edi
 	shrl	$10,%r14d
 	rorl	$17,%edi
 	xorl	%r13d,%r12d
 	xorl	%r14d,%edi
 	addl	8(%rsp),%r12d
 	addl	36(%rsp),%r12d
 	movl	%edx,%r13d
 	addl	%edi,%r12d
 	movl	%r11d,%r14d
 	rorl	$14,%r13d
 	movl	%r8d,%edi
 	xorl	%edx,%r13d
 	rorl	$9,%r14d
 	xorl	%r9d,%edi
 	movl	%r12d,36(%rsp)
 	xorl	%r11d,%r14d
 	andl	%edx,%edi
 	rorl	$5,%r13d
 	addl	%r10d,%r12d
 	xorl	%r9d,%edi
 	rorl	$11,%r14d
 	xorl	%edx,%r13d
 	addl	%edi,%r12d
 	movl	%r11d,%edi
 	addl	(%rbp),%r12d
 	xorl	%r11d,%r14d
 	xorl	%eax,%edi
 	rorl	$6,%r13d
 	movl	%eax,%r10d
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%r10d
 	addl	%r12d,%ecx
 	addl	%r12d,%r10d
 	leaq	4(%rbp),%rbp
 	movl	44(%rsp),%r13d
 	movl	32(%rsp),%r15d
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%r10d
 	movl	%r15d,%r14d
 	rorl	$2,%r15d
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%r15d
 	shrl	$10,%r14d
 	rorl	$17,%r15d
 	xorl	%r13d,%r12d
 	xorl	%r14d,%r15d
 	addl	12(%rsp),%r12d
 	addl	40(%rsp),%r12d
 	movl	%ecx,%r13d
 	addl	%r15d,%r12d
 	movl	%r10d,%r14d
 	rorl	$14,%r13d
 	movl	%edx,%r15d
 	xorl	%ecx,%r13d
 	rorl	$9,%r14d
 	xorl	%r8d,%r15d
 	movl	%r12d,40(%rsp)
 	xorl	%r10d,%r14d
 	andl	%ecx,%r15d
 	rorl	$5,%r13d
 	addl	%r9d,%r12d
 	xorl	%r8d,%r15d
 	rorl	$11,%r14d
 	xorl	%ecx,%r13d
 	addl	%r15d,%r12d
 	movl	%r10d,%r15d
 	addl	(%rbp),%r12d
 	xorl	%r10d,%r14d
 	xorl	%r11d,%r15d
 	rorl	$6,%r13d
 	movl	%r11d,%r9d
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%r9d
 	addl	%r12d,%ebx
 	addl	%r12d,%r9d
 	leaq	4(%rbp),%rbp
 	movl	48(%rsp),%r13d
 	movl	36(%rsp),%edi
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%r9d
 	movl	%edi,%r14d
 	rorl	$2,%edi
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%edi
 	shrl	$10,%r14d
 	rorl	$17,%edi
 	xorl	%r13d,%r12d
 	xorl	%r14d,%edi
 	addl	16(%rsp),%r12d
 	addl	44(%rsp),%r12d
 	movl	%ebx,%r13d
 	addl	%edi,%r12d
 	movl	%r9d,%r14d
 	rorl	$14,%r13d
 	movl	%ecx,%edi
 	xorl	%ebx,%r13d
 	rorl	$9,%r14d
 	xorl	%edx,%edi
 	movl	%r12d,44(%rsp)
 	xorl	%r9d,%r14d
 	andl	%ebx,%edi
 	rorl	$5,%r13d
 	addl	%r8d,%r12d
 	xorl	%edx,%edi
 	rorl	$11,%r14d
 	xorl	%ebx,%r13d
 	addl	%edi,%r12d
 	movl	%r9d,%edi
 	addl	(%rbp),%r12d
 	xorl	%r9d,%r14d
 	xorl	%r10d,%edi
 	rorl	$6,%r13d
 	movl	%r10d,%r8d
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%r8d
 	addl	%r12d,%eax
 	addl	%r12d,%r8d
 	leaq	20(%rbp),%rbp
 	movl	52(%rsp),%r13d
 	movl	40(%rsp),%r15d
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%r8d
 	movl	%r15d,%r14d
 	rorl	$2,%r15d
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%r15d
 	shrl	$10,%r14d
 	rorl	$17,%r15d
 	xorl	%r13d,%r12d
 	xorl	%r14d,%r15d
 	addl	20(%rsp),%r12d
 	addl	48(%rsp),%r12d
 	movl	%eax,%r13d
 	addl	%r15d,%r12d
 	movl	%r8d,%r14d
 	rorl	$14,%r13d
 	movl	%ebx,%r15d
 	xorl	%eax,%r13d
 	rorl	$9,%r14d
 	xorl	%ecx,%r15d
 	movl	%r12d,48(%rsp)
 	xorl	%r8d,%r14d
 	andl	%eax,%r15d
 	rorl	$5,%r13d
 	addl	%edx,%r12d
 	xorl	%ecx,%r15d
 	rorl	$11,%r14d
 	xorl	%eax,%r13d
 	addl	%r15d,%r12d
 	movl	%r8d,%r15d
 	addl	(%rbp),%r12d
 	xorl	%r8d,%r14d
 	xorl	%r9d,%r15d
 	rorl	$6,%r13d
 	movl	%r9d,%edx
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%edx
 	addl	%r12d,%r11d
 	addl	%r12d,%edx
 	leaq	4(%rbp),%rbp
 	movl	56(%rsp),%r13d
 	movl	44(%rsp),%edi
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%edx
 	movl	%edi,%r14d
 	rorl	$2,%edi
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%edi
 	shrl	$10,%r14d
 	rorl	$17,%edi
 	xorl	%r13d,%r12d
 	xorl	%r14d,%edi
 	addl	24(%rsp),%r12d
 	addl	52(%rsp),%r12d
 	movl	%r11d,%r13d
 	addl	%edi,%r12d
 	movl	%edx,%r14d
 	rorl	$14,%r13d
 	movl	%eax,%edi
 	xorl	%r11d,%r13d
 	rorl	$9,%r14d
 	xorl	%ebx,%edi
 	movl	%r12d,52(%rsp)
 	xorl	%edx,%r14d
 	andl	%r11d,%edi
 	rorl	$5,%r13d
 	addl	%ecx,%r12d
 	xorl	%ebx,%edi
 	rorl	$11,%r14d
 	xorl	%r11d,%r13d
 	addl	%edi,%r12d
 	movl	%edx,%edi
 	addl	(%rbp),%r12d
 	xorl	%edx,%r14d
 	xorl	%r8d,%edi
 	rorl	$6,%r13d
 	movl	%r8d,%ecx
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%ecx
 	addl	%r12d,%r10d
 	addl	%r12d,%ecx
 	leaq	4(%rbp),%rbp
 	movl	60(%rsp),%r13d
 	movl	48(%rsp),%r15d
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%ecx
 	movl	%r15d,%r14d
 	rorl	$2,%r15d
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%r15d
 	shrl	$10,%r14d
 	rorl	$17,%r15d
 	xorl	%r13d,%r12d
 	xorl	%r14d,%r15d
 	addl	28(%rsp),%r12d
 	addl	56(%rsp),%r12d
 	movl	%r10d,%r13d
 	addl	%r15d,%r12d
 	movl	%ecx,%r14d
 	rorl	$14,%r13d
 	movl	%r11d,%r15d
 	xorl	%r10d,%r13d
 	rorl	$9,%r14d
 	xorl	%eax,%r15d
 	movl	%r12d,56(%rsp)
 	xorl	%ecx,%r14d
 	andl	%r10d,%r15d
 	rorl	$5,%r13d
 	addl	%ebx,%r12d
 	xorl	%eax,%r15d
 	rorl	$11,%r14d
 	xorl	%r10d,%r13d
 	addl	%r15d,%r12d
 	movl	%ecx,%r15d
 	addl	(%rbp),%r12d
 	xorl	%ecx,%r14d
 	xorl	%edx,%r15d
 	rorl	$6,%r13d
 	movl	%edx,%ebx
 	andl	%r15d,%edi
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%edi,%ebx
 	addl	%r12d,%r9d
 	addl	%r12d,%ebx
 	leaq	4(%rbp),%rbp
 	movl	0(%rsp),%r13d
 	movl	52(%rsp),%edi
 	movl	%r13d,%r12d
 	rorl	$11,%r13d
 	addl	%r14d,%ebx
 	movl	%edi,%r14d
 	rorl	$2,%edi
 	xorl	%r12d,%r13d
 	shrl	$3,%r12d
 	rorl	$7,%r13d
 	xorl	%r14d,%edi
 	shrl	$10,%r14d
 	rorl	$17,%edi
 	xorl	%r13d,%r12d
 	xorl	%r14d,%edi
 	addl	32(%rsp),%r12d
 	addl	60(%rsp),%r12d
 	movl	%r9d,%r13d
 	addl	%edi,%r12d
 	movl	%ebx,%r14d
 	rorl	$14,%r13d
 	movl	%r10d,%edi
 	xorl	%r9d,%r13d
 	rorl	$9,%r14d
 	xorl	%r11d,%edi
 	movl	%r12d,60(%rsp)
 	xorl	%ebx,%r14d
 	andl	%r9d,%edi
 	rorl	$5,%r13d
 	addl	%eax,%r12d
 	xorl	%r11d,%edi
 	rorl	$11,%r14d
 	xorl	%r9d,%r13d
 	addl	%edi,%r12d
 	movl	%ebx,%edi
 	addl	(%rbp),%r12d
 	xorl	%ebx,%r14d
 	xorl	%ecx,%edi
 	rorl	$6,%r13d
 	movl	%ecx,%eax
 	andl	%edi,%r15d
 	rorl	$2,%r14d
 	addl	%r13d,%r12d
 	xorl	%r15d,%eax
 	addl	%r12d,%r8d
 	addl	%r12d,%eax
 	leaq	20(%rbp),%rbp
 	cmpb	$0,3(%rbp)
 	jnz	.Lrounds_16_xx
 	movq	64+0(%rsp),%rdi
 	addl	%r14d,%eax
 	leaq	64(%rsi),%rsi
 	addl	0(%rdi),%eax
 	addl	4(%rdi),%ebx
 	addl	8(%rdi),%ecx
 	addl	12(%rdi),%edx
 	addl	16(%rdi),%r8d
 	addl	20(%rdi),%r9d
 	addl	24(%rdi),%r10d
 	addl	28(%rdi),%r11d
 	cmpq	64+16(%rsp),%rsi
 	movl	%eax,0(%rdi)
 	movl	%ebx,4(%rdi)
 	movl	%ecx,8(%rdi)
 	movl	%edx,12(%rdi)
 	movl	%r8d,16(%rdi)
 	movl	%r9d,20(%rdi)
 	movl	%r10d,24(%rdi)
 	movl	%r11d,28(%rdi)
 	jb	.Lloop
 	movq	88(%rsp),%rsi
 .cfi_def_cfa	%rsi,8
 	movq	-48(%rsi),%r15
 .cfi_restore	%r15
 	movq	-40(%rsi),%r14
 .cfi_restore	%r14
 	movq	-32(%rsi),%r13
 .cfi_restore	%r13
 	movq	-24(%rsi),%r12
 .cfi_restore	%r12
 	movq	-16(%rsi),%rbp
 .cfi_restore	%rbp
 	movq	-8(%rsi),%rbx
 .cfi_restore	%rbx
 	leaq	(%rsi),%rsp
 .cfi_def_cfa_register	%rsp
 .Lepilogue:
 	RET
 .cfi_endproc
 SET_SIZE(zfs_sha256_transform_x64)
+STACK_FRAME_NON_STANDARD zfs_sha256_transform_x64
 
 ENTRY_ALIGN(zfs_sha256_transform_shani, 64)
 .cfi_startproc
 	ENDBR
 	leaq	K256+128(%rip),%rcx
 	movdqu	(%rdi),%xmm1
 	movdqu	16(%rdi),%xmm2
 	movdqa	512-128(%rcx),%xmm7
 
 	pshufd	$0x1b,%xmm1,%xmm0
 	pshufd	$0xb1,%xmm1,%xmm1
 	pshufd	$0x1b,%xmm2,%xmm2
 	movdqa	%xmm7,%xmm8
 .byte	102,15,58,15,202,8
 	punpcklqdq	%xmm0,%xmm2
 	jmp	.Loop_shani
 
 .balign	16
 .Loop_shani:
 	movdqu	(%rsi),%xmm3
 	movdqu	16(%rsi),%xmm4
 	movdqu	32(%rsi),%xmm5
 .byte	102,15,56,0,223
 	movdqu	48(%rsi),%xmm6
 
 	movdqa	0-128(%rcx),%xmm0
 	paddd	%xmm3,%xmm0
 .byte	102,15,56,0,231
 	movdqa	%xmm2,%xmm10
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	nop
 	movdqa	%xmm1,%xmm9
 .byte	15,56,203,202
 
 	movdqa	32-128(%rcx),%xmm0
 	paddd	%xmm4,%xmm0
 .byte	102,15,56,0,239
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	leaq	64(%rsi),%rsi
 .byte	15,56,204,220
 .byte	15,56,203,202
 
 	movdqa	64-128(%rcx),%xmm0
 	paddd	%xmm5,%xmm0
 .byte	102,15,56,0,247
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm6,%xmm7
 .byte	102,15,58,15,253,4
 	nop
 	paddd	%xmm7,%xmm3
 .byte	15,56,204,229
 .byte	15,56,203,202
 
 	movdqa	96-128(%rcx),%xmm0
 	paddd	%xmm6,%xmm0
 .byte	15,56,205,222
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm3,%xmm7
 .byte	102,15,58,15,254,4
 	nop
 	paddd	%xmm7,%xmm4
 .byte	15,56,204,238
 .byte	15,56,203,202
 	movdqa	128-128(%rcx),%xmm0
 	paddd	%xmm3,%xmm0
 .byte	15,56,205,227
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm4,%xmm7
 .byte	102,15,58,15,251,4
 	nop
 	paddd	%xmm7,%xmm5
 .byte	15,56,204,243
 .byte	15,56,203,202
 	movdqa	160-128(%rcx),%xmm0
 	paddd	%xmm4,%xmm0
 .byte	15,56,205,236
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm5,%xmm7
 .byte	102,15,58,15,252,4
 	nop
 	paddd	%xmm7,%xmm6
 .byte	15,56,204,220
 .byte	15,56,203,202
 	movdqa	192-128(%rcx),%xmm0
 	paddd	%xmm5,%xmm0
 .byte	15,56,205,245
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm6,%xmm7
 .byte	102,15,58,15,253,4
 	nop
 	paddd	%xmm7,%xmm3
 .byte	15,56,204,229
 .byte	15,56,203,202
 	movdqa	224-128(%rcx),%xmm0
 	paddd	%xmm6,%xmm0
 .byte	15,56,205,222
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm3,%xmm7
 .byte	102,15,58,15,254,4
 	nop
 	paddd	%xmm7,%xmm4
 .byte	15,56,204,238
 .byte	15,56,203,202
 	movdqa	256-128(%rcx),%xmm0
 	paddd	%xmm3,%xmm0
 .byte	15,56,205,227
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm4,%xmm7
 .byte	102,15,58,15,251,4
 	nop
 	paddd	%xmm7,%xmm5
 .byte	15,56,204,243
 .byte	15,56,203,202
 	movdqa	288-128(%rcx),%xmm0
 	paddd	%xmm4,%xmm0
 .byte	15,56,205,236
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm5,%xmm7
 .byte	102,15,58,15,252,4
 	nop
 	paddd	%xmm7,%xmm6
 .byte	15,56,204,220
 .byte	15,56,203,202
 	movdqa	320-128(%rcx),%xmm0
 	paddd	%xmm5,%xmm0
 .byte	15,56,205,245
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm6,%xmm7
 .byte	102,15,58,15,253,4
 	nop
 	paddd	%xmm7,%xmm3
 .byte	15,56,204,229
 .byte	15,56,203,202
 	movdqa	352-128(%rcx),%xmm0
 	paddd	%xmm6,%xmm0
 .byte	15,56,205,222
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm3,%xmm7
 .byte	102,15,58,15,254,4
 	nop
 	paddd	%xmm7,%xmm4
 .byte	15,56,204,238
 .byte	15,56,203,202
 	movdqa	384-128(%rcx),%xmm0
 	paddd	%xmm3,%xmm0
 .byte	15,56,205,227
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm4,%xmm7
 .byte	102,15,58,15,251,4
 	nop
 	paddd	%xmm7,%xmm5
 .byte	15,56,204,243
 .byte	15,56,203,202
 	movdqa	416-128(%rcx),%xmm0
 	paddd	%xmm4,%xmm0
 .byte	15,56,205,236
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	movdqa	%xmm5,%xmm7
 .byte	102,15,58,15,252,4
 .byte	15,56,203,202
 	paddd	%xmm7,%xmm6
 
 	movdqa	448-128(%rcx),%xmm0
 	paddd	%xmm5,%xmm0
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 .byte	15,56,205,245
 	movdqa	%xmm8,%xmm7
 .byte	15,56,203,202
 
 	movdqa	480-128(%rcx),%xmm0
 	paddd	%xmm6,%xmm0
 	nop
 .byte	15,56,203,209
 	pshufd	$0x0e,%xmm0,%xmm0
 	decq	%rdx
 	nop
 .byte	15,56,203,202
 
 	paddd	%xmm10,%xmm2
 	paddd	%xmm9,%xmm1
 	jnz	.Loop_shani
 
 	pshufd	$0xb1,%xmm2,%xmm2
 	pshufd	$0x1b,%xmm1,%xmm7
 	pshufd	$0xb1,%xmm1,%xmm1
 	punpckhqdq	%xmm2,%xmm1
 .byte	102,15,58,15,215,8
 
 	movdqu	%xmm1,(%rdi)
 	movdqu	%xmm2,16(%rdi)
 	RET
 .cfi_endproc
 SET_SIZE(zfs_sha256_transform_shani)
+STACK_FRAME_NON_STANDARD zfs_sha256_transform_shani
 
 ENTRY_ALIGN(zfs_sha256_transform_ssse3, 64)
 .cfi_startproc
 	ENDBR
 	movq	%rsp,%rax
 .cfi_def_cfa_register	%rax
 	pushq	%rbx
 .cfi_offset	%rbx,-16
 	pushq	%rbp
 .cfi_offset	%rbp,-24
 	pushq	%r12
 .cfi_offset	%r12,-32
 	pushq	%r13
 .cfi_offset	%r13,-40
 	pushq	%r14
 .cfi_offset	%r14,-48
 	pushq	%r15
 .cfi_offset	%r15,-56
 	shlq	$4,%rdx
 	subq	$96,%rsp
 	leaq	(%rsi,%rdx,4),%rdx
 	andq	$-64,%rsp
 	movq	%rdi,64+0(%rsp)
 	movq	%rsi,64+8(%rsp)
 	movq	%rdx,64+16(%rsp)
 	movq	%rax,88(%rsp)
 .cfi_escape	0x0f,0x06,0x77,0xd8,0x00,0x06,0x23,0x08
 .Lprologue_ssse3:
 
 	movl	0(%rdi),%eax
 	movl	4(%rdi),%ebx
 	movl	8(%rdi),%ecx
 	movl	12(%rdi),%edx
 	movl	16(%rdi),%r8d
 	movl	20(%rdi),%r9d
 	movl	24(%rdi),%r10d
 	movl	28(%rdi),%r11d
 
 	jmp	.Lloop_ssse3
 .balign	16
 .Lloop_ssse3:
 	movdqa	K256+512(%rip),%xmm7
 	movdqu	0(%rsi),%xmm0
 	movdqu	16(%rsi),%xmm1
 	movdqu	32(%rsi),%xmm2
 .byte	102,15,56,0,199
 	movdqu	48(%rsi),%xmm3
 	leaq	K256(%rip),%rbp
 .byte	102,15,56,0,207
 	movdqa	0(%rbp),%xmm4
 	movdqa	32(%rbp),%xmm5
 .byte	102,15,56,0,215
 	paddd	%xmm0,%xmm4
 	movdqa	64(%rbp),%xmm6
 .byte	102,15,56,0,223
 	movdqa	96(%rbp),%xmm7
 	paddd	%xmm1,%xmm5
 	paddd	%xmm2,%xmm6
 	paddd	%xmm3,%xmm7
 	movdqa	%xmm4,0(%rsp)
 	movl	%eax,%r14d
 	movdqa	%xmm5,16(%rsp)
 	movl	%ebx,%edi
 	movdqa	%xmm6,32(%rsp)
 	xorl	%ecx,%edi
 	movdqa	%xmm7,48(%rsp)
 	movl	%r8d,%r13d
 	jmp	.Lssse3_00_47
 
 .balign	16
 .Lssse3_00_47:
 	subq	$-128,%rbp
 	rorl	$14,%r13d
 	movdqa	%xmm1,%xmm4
 	movl	%r14d,%eax
 	movl	%r9d,%r12d
 	movdqa	%xmm3,%xmm7
 	rorl	$9,%r14d
 	xorl	%r8d,%r13d
 	xorl	%r10d,%r12d
 	rorl	$5,%r13d
 	xorl	%eax,%r14d
 .byte	102,15,58,15,224,4
 	andl	%r8d,%r12d
 	xorl	%r8d,%r13d
 .byte	102,15,58,15,250,4
 	addl	0(%rsp),%r11d
 	movl	%eax,%r15d
 	xorl	%r10d,%r12d
 	rorl	$11,%r14d
 	movdqa	%xmm4,%xmm5
 	xorl	%ebx,%r15d
 	addl	%r12d,%r11d
 	movdqa	%xmm4,%xmm6
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	psrld	$3,%xmm4
 	xorl	%eax,%r14d
 	addl	%r13d,%r11d
 	xorl	%ebx,%edi
 	paddd	%xmm7,%xmm0
 	rorl	$2,%r14d
 	addl	%r11d,%edx
 	psrld	$7,%xmm6
 	addl	%edi,%r11d
 	movl	%edx,%r13d
 	pshufd	$250,%xmm3,%xmm7
 	addl	%r11d,%r14d
 	rorl	$14,%r13d
 	pslld	$14,%xmm5
 	movl	%r14d,%r11d
 	movl	%r8d,%r12d
 	pxor	%xmm6,%xmm4
 	rorl	$9,%r14d
 	xorl	%edx,%r13d
 	xorl	%r9d,%r12d
 	rorl	$5,%r13d
 	psrld	$11,%xmm6
 	xorl	%r11d,%r14d
 	pxor	%xmm5,%xmm4
 	andl	%edx,%r12d
 	xorl	%edx,%r13d
 	pslld	$11,%xmm5
 	addl	4(%rsp),%r10d
 	movl	%r11d,%edi
 	pxor	%xmm6,%xmm4
 	xorl	%r9d,%r12d
 	rorl	$11,%r14d
 	movdqa	%xmm7,%xmm6
 	xorl	%eax,%edi
 	addl	%r12d,%r10d
 	pxor	%xmm5,%xmm4
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%r11d,%r14d
 	psrld	$10,%xmm7
 	addl	%r13d,%r10d
 	xorl	%eax,%r15d
 	paddd	%xmm4,%xmm0
 	rorl	$2,%r14d
 	addl	%r10d,%ecx
 	psrlq	$17,%xmm6
 	addl	%r15d,%r10d
 	movl	%ecx,%r13d
 	addl	%r10d,%r14d
 	pxor	%xmm6,%xmm7
 	rorl	$14,%r13d
 	movl	%r14d,%r10d
 	movl	%edx,%r12d
 	rorl	$9,%r14d
 	psrlq	$2,%xmm6
 	xorl	%ecx,%r13d
 	xorl	%r8d,%r12d
 	pxor	%xmm6,%xmm7
 	rorl	$5,%r13d
 	xorl	%r10d,%r14d
 	andl	%ecx,%r12d
 	pshufd	$128,%xmm7,%xmm7
 	xorl	%ecx,%r13d
 	addl	8(%rsp),%r9d
 	movl	%r10d,%r15d
 	psrldq	$8,%xmm7
 	xorl	%r8d,%r12d
 	rorl	$11,%r14d
 	xorl	%r11d,%r15d
 	addl	%r12d,%r9d
 	rorl	$6,%r13d
 	paddd	%xmm7,%xmm0
 	andl	%r15d,%edi
 	xorl	%r10d,%r14d
 	addl	%r13d,%r9d
 	pshufd	$80,%xmm0,%xmm7
 	xorl	%r11d,%edi
 	rorl	$2,%r14d
 	addl	%r9d,%ebx
 	movdqa	%xmm7,%xmm6
 	addl	%edi,%r9d
 	movl	%ebx,%r13d
 	psrld	$10,%xmm7
 	addl	%r9d,%r14d
 	rorl	$14,%r13d
 	psrlq	$17,%xmm6
 	movl	%r14d,%r9d
 	movl	%ecx,%r12d
 	pxor	%xmm6,%xmm7
 	rorl	$9,%r14d
 	xorl	%ebx,%r13d
 	xorl	%edx,%r12d
 	rorl	$5,%r13d
 	xorl	%r9d,%r14d
 	psrlq	$2,%xmm6
 	andl	%ebx,%r12d
 	xorl	%ebx,%r13d
 	addl	12(%rsp),%r8d
 	pxor	%xmm6,%xmm7
 	movl	%r9d,%edi
 	xorl	%edx,%r12d
 	rorl	$11,%r14d
 	pshufd	$8,%xmm7,%xmm7
 	xorl	%r10d,%edi
 	addl	%r12d,%r8d
 	movdqa	0(%rbp),%xmm6
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	pslldq	$8,%xmm7
 	xorl	%r9d,%r14d
 	addl	%r13d,%r8d
 	xorl	%r10d,%r15d
 	paddd	%xmm7,%xmm0
 	rorl	$2,%r14d
 	addl	%r8d,%eax
 	addl	%r15d,%r8d
 	paddd	%xmm0,%xmm6
 	movl	%eax,%r13d
 	addl	%r8d,%r14d
 	movdqa	%xmm6,0(%rsp)
 	rorl	$14,%r13d
 	movdqa	%xmm2,%xmm4
 	movl	%r14d,%r8d
 	movl	%ebx,%r12d
 	movdqa	%xmm0,%xmm7
 	rorl	$9,%r14d
 	xorl	%eax,%r13d
 	xorl	%ecx,%r12d
 	rorl	$5,%r13d
 	xorl	%r8d,%r14d
 .byte	102,15,58,15,225,4
 	andl	%eax,%r12d
 	xorl	%eax,%r13d
 .byte	102,15,58,15,251,4
 	addl	16(%rsp),%edx
 	movl	%r8d,%r15d
 	xorl	%ecx,%r12d
 	rorl	$11,%r14d
 	movdqa	%xmm4,%xmm5
 	xorl	%r9d,%r15d
 	addl	%r12d,%edx
 	movdqa	%xmm4,%xmm6
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	psrld	$3,%xmm4
 	xorl	%r8d,%r14d
 	addl	%r13d,%edx
 	xorl	%r9d,%edi
 	paddd	%xmm7,%xmm1
 	rorl	$2,%r14d
 	addl	%edx,%r11d
 	psrld	$7,%xmm6
 	addl	%edi,%edx
 	movl	%r11d,%r13d
 	pshufd	$250,%xmm0,%xmm7
 	addl	%edx,%r14d
 	rorl	$14,%r13d
 	pslld	$14,%xmm5
 	movl	%r14d,%edx
 	movl	%eax,%r12d
 	pxor	%xmm6,%xmm4
 	rorl	$9,%r14d
 	xorl	%r11d,%r13d
 	xorl	%ebx,%r12d
 	rorl	$5,%r13d
 	psrld	$11,%xmm6
 	xorl	%edx,%r14d
 	pxor	%xmm5,%xmm4
 	andl	%r11d,%r12d
 	xorl	%r11d,%r13d
 	pslld	$11,%xmm5
 	addl	20(%rsp),%ecx
 	movl	%edx,%edi
 	pxor	%xmm6,%xmm4
 	xorl	%ebx,%r12d
 	rorl	$11,%r14d
 	movdqa	%xmm7,%xmm6
 	xorl	%r8d,%edi
 	addl	%r12d,%ecx
 	pxor	%xmm5,%xmm4
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%edx,%r14d
 	psrld	$10,%xmm7
 	addl	%r13d,%ecx
 	xorl	%r8d,%r15d
 	paddd	%xmm4,%xmm1
 	rorl	$2,%r14d
 	addl	%ecx,%r10d
 	psrlq	$17,%xmm6
 	addl	%r15d,%ecx
 	movl	%r10d,%r13d
 	addl	%ecx,%r14d
 	pxor	%xmm6,%xmm7
 	rorl	$14,%r13d
 	movl	%r14d,%ecx
 	movl	%r11d,%r12d
 	rorl	$9,%r14d
 	psrlq	$2,%xmm6
 	xorl	%r10d,%r13d
 	xorl	%eax,%r12d
 	pxor	%xmm6,%xmm7
 	rorl	$5,%r13d
 	xorl	%ecx,%r14d
 	andl	%r10d,%r12d
 	pshufd	$128,%xmm7,%xmm7
 	xorl	%r10d,%r13d
 	addl	24(%rsp),%ebx
 	movl	%ecx,%r15d
 	psrldq	$8,%xmm7
 	xorl	%eax,%r12d
 	rorl	$11,%r14d
 	xorl	%edx,%r15d
 	addl	%r12d,%ebx
 	rorl	$6,%r13d
 	paddd	%xmm7,%xmm1
 	andl	%r15d,%edi
 	xorl	%ecx,%r14d
 	addl	%r13d,%ebx
 	pshufd	$80,%xmm1,%xmm7
 	xorl	%edx,%edi
 	rorl	$2,%r14d
 	addl	%ebx,%r9d
 	movdqa	%xmm7,%xmm6
 	addl	%edi,%ebx
 	movl	%r9d,%r13d
 	psrld	$10,%xmm7
 	addl	%ebx,%r14d
 	rorl	$14,%r13d
 	psrlq	$17,%xmm6
 	movl	%r14d,%ebx
 	movl	%r10d,%r12d
 	pxor	%xmm6,%xmm7
 	rorl	$9,%r14d
 	xorl	%r9d,%r13d
 	xorl	%r11d,%r12d
 	rorl	$5,%r13d
 	xorl	%ebx,%r14d
 	psrlq	$2,%xmm6
 	andl	%r9d,%r12d
 	xorl	%r9d,%r13d
 	addl	28(%rsp),%eax
 	pxor	%xmm6,%xmm7
 	movl	%ebx,%edi
 	xorl	%r11d,%r12d
 	rorl	$11,%r14d
 	pshufd	$8,%xmm7,%xmm7
 	xorl	%ecx,%edi
 	addl	%r12d,%eax
 	movdqa	32(%rbp),%xmm6
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	pslldq	$8,%xmm7
 	xorl	%ebx,%r14d
 	addl	%r13d,%eax
 	xorl	%ecx,%r15d
 	paddd	%xmm7,%xmm1
 	rorl	$2,%r14d
 	addl	%eax,%r8d
 	addl	%r15d,%eax
 	paddd	%xmm1,%xmm6
 	movl	%r8d,%r13d
 	addl	%eax,%r14d
 	movdqa	%xmm6,16(%rsp)
 	rorl	$14,%r13d
 	movdqa	%xmm3,%xmm4
 	movl	%r14d,%eax
 	movl	%r9d,%r12d
 	movdqa	%xmm1,%xmm7
 	rorl	$9,%r14d
 	xorl	%r8d,%r13d
 	xorl	%r10d,%r12d
 	rorl	$5,%r13d
 	xorl	%eax,%r14d
 .byte	102,15,58,15,226,4
 	andl	%r8d,%r12d
 	xorl	%r8d,%r13d
 .byte	102,15,58,15,248,4
 	addl	32(%rsp),%r11d
 	movl	%eax,%r15d
 	xorl	%r10d,%r12d
 	rorl	$11,%r14d
 	movdqa	%xmm4,%xmm5
 	xorl	%ebx,%r15d
 	addl	%r12d,%r11d
 	movdqa	%xmm4,%xmm6
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	psrld	$3,%xmm4
 	xorl	%eax,%r14d
 	addl	%r13d,%r11d
 	xorl	%ebx,%edi
 	paddd	%xmm7,%xmm2
 	rorl	$2,%r14d
 	addl	%r11d,%edx
 	psrld	$7,%xmm6
 	addl	%edi,%r11d
 	movl	%edx,%r13d
 	pshufd	$250,%xmm1,%xmm7
 	addl	%r11d,%r14d
 	rorl	$14,%r13d
 	pslld	$14,%xmm5
 	movl	%r14d,%r11d
 	movl	%r8d,%r12d
 	pxor	%xmm6,%xmm4
 	rorl	$9,%r14d
 	xorl	%edx,%r13d
 	xorl	%r9d,%r12d
 	rorl	$5,%r13d
 	psrld	$11,%xmm6
 	xorl	%r11d,%r14d
 	pxor	%xmm5,%xmm4
 	andl	%edx,%r12d
 	xorl	%edx,%r13d
 	pslld	$11,%xmm5
 	addl	36(%rsp),%r10d
 	movl	%r11d,%edi
 	pxor	%xmm6,%xmm4
 	xorl	%r9d,%r12d
 	rorl	$11,%r14d
 	movdqa	%xmm7,%xmm6
 	xorl	%eax,%edi
 	addl	%r12d,%r10d
 	pxor	%xmm5,%xmm4
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%r11d,%r14d
 	psrld	$10,%xmm7
 	addl	%r13d,%r10d
 	xorl	%eax,%r15d
 	paddd	%xmm4,%xmm2
 	rorl	$2,%r14d
 	addl	%r10d,%ecx
 	psrlq	$17,%xmm6
 	addl	%r15d,%r10d
 	movl	%ecx,%r13d
 	addl	%r10d,%r14d
 	pxor	%xmm6,%xmm7
 	rorl	$14,%r13d
 	movl	%r14d,%r10d
 	movl	%edx,%r12d
 	rorl	$9,%r14d
 	psrlq	$2,%xmm6
 	xorl	%ecx,%r13d
 	xorl	%r8d,%r12d
 	pxor	%xmm6,%xmm7
 	rorl	$5,%r13d
 	xorl	%r10d,%r14d
 	andl	%ecx,%r12d
 	pshufd	$128,%xmm7,%xmm7
 	xorl	%ecx,%r13d
 	addl	40(%rsp),%r9d
 	movl	%r10d,%r15d
 	psrldq	$8,%xmm7
 	xorl	%r8d,%r12d
 	rorl	$11,%r14d
 	xorl	%r11d,%r15d
 	addl	%r12d,%r9d
 	rorl	$6,%r13d
 	paddd	%xmm7,%xmm2
 	andl	%r15d,%edi
 	xorl	%r10d,%r14d
 	addl	%r13d,%r9d
 	pshufd	$80,%xmm2,%xmm7
 	xorl	%r11d,%edi
 	rorl	$2,%r14d
 	addl	%r9d,%ebx
 	movdqa	%xmm7,%xmm6
 	addl	%edi,%r9d
 	movl	%ebx,%r13d
 	psrld	$10,%xmm7
 	addl	%r9d,%r14d
 	rorl	$14,%r13d
 	psrlq	$17,%xmm6
 	movl	%r14d,%r9d
 	movl	%ecx,%r12d
 	pxor	%xmm6,%xmm7
 	rorl	$9,%r14d
 	xorl	%ebx,%r13d
 	xorl	%edx,%r12d
 	rorl	$5,%r13d
 	xorl	%r9d,%r14d
 	psrlq	$2,%xmm6
 	andl	%ebx,%r12d
 	xorl	%ebx,%r13d
 	addl	44(%rsp),%r8d
 	pxor	%xmm6,%xmm7
 	movl	%r9d,%edi
 	xorl	%edx,%r12d
 	rorl	$11,%r14d
 	pshufd	$8,%xmm7,%xmm7
 	xorl	%r10d,%edi
 	addl	%r12d,%r8d
 	movdqa	64(%rbp),%xmm6
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	pslldq	$8,%xmm7
 	xorl	%r9d,%r14d
 	addl	%r13d,%r8d
 	xorl	%r10d,%r15d
 	paddd	%xmm7,%xmm2
 	rorl	$2,%r14d
 	addl	%r8d,%eax
 	addl	%r15d,%r8d
 	paddd	%xmm2,%xmm6
 	movl	%eax,%r13d
 	addl	%r8d,%r14d
 	movdqa	%xmm6,32(%rsp)
 	rorl	$14,%r13d
 	movdqa	%xmm0,%xmm4
 	movl	%r14d,%r8d
 	movl	%ebx,%r12d
 	movdqa	%xmm2,%xmm7
 	rorl	$9,%r14d
 	xorl	%eax,%r13d
 	xorl	%ecx,%r12d
 	rorl	$5,%r13d
 	xorl	%r8d,%r14d
 .byte	102,15,58,15,227,4
 	andl	%eax,%r12d
 	xorl	%eax,%r13d
 .byte	102,15,58,15,249,4
 	addl	48(%rsp),%edx
 	movl	%r8d,%r15d
 	xorl	%ecx,%r12d
 	rorl	$11,%r14d
 	movdqa	%xmm4,%xmm5
 	xorl	%r9d,%r15d
 	addl	%r12d,%edx
 	movdqa	%xmm4,%xmm6
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	psrld	$3,%xmm4
 	xorl	%r8d,%r14d
 	addl	%r13d,%edx
 	xorl	%r9d,%edi
 	paddd	%xmm7,%xmm3
 	rorl	$2,%r14d
 	addl	%edx,%r11d
 	psrld	$7,%xmm6
 	addl	%edi,%edx
 	movl	%r11d,%r13d
 	pshufd	$250,%xmm2,%xmm7
 	addl	%edx,%r14d
 	rorl	$14,%r13d
 	pslld	$14,%xmm5
 	movl	%r14d,%edx
 	movl	%eax,%r12d
 	pxor	%xmm6,%xmm4
 	rorl	$9,%r14d
 	xorl	%r11d,%r13d
 	xorl	%ebx,%r12d
 	rorl	$5,%r13d
 	psrld	$11,%xmm6
 	xorl	%edx,%r14d
 	pxor	%xmm5,%xmm4
 	andl	%r11d,%r12d
 	xorl	%r11d,%r13d
 	pslld	$11,%xmm5
 	addl	52(%rsp),%ecx
 	movl	%edx,%edi
 	pxor	%xmm6,%xmm4
 	xorl	%ebx,%r12d
 	rorl	$11,%r14d
 	movdqa	%xmm7,%xmm6
 	xorl	%r8d,%edi
 	addl	%r12d,%ecx
 	pxor	%xmm5,%xmm4
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%edx,%r14d
 	psrld	$10,%xmm7
 	addl	%r13d,%ecx
 	xorl	%r8d,%r15d
 	paddd	%xmm4,%xmm3
 	rorl	$2,%r14d
 	addl	%ecx,%r10d
 	psrlq	$17,%xmm6
 	addl	%r15d,%ecx
 	movl	%r10d,%r13d
 	addl	%ecx,%r14d
 	pxor	%xmm6,%xmm7
 	rorl	$14,%r13d
 	movl	%r14d,%ecx
 	movl	%r11d,%r12d
 	rorl	$9,%r14d
 	psrlq	$2,%xmm6
 	xorl	%r10d,%r13d
 	xorl	%eax,%r12d
 	pxor	%xmm6,%xmm7
 	rorl	$5,%r13d
 	xorl	%ecx,%r14d
 	andl	%r10d,%r12d
 	pshufd	$128,%xmm7,%xmm7
 	xorl	%r10d,%r13d
 	addl	56(%rsp),%ebx
 	movl	%ecx,%r15d
 	psrldq	$8,%xmm7
 	xorl	%eax,%r12d
 	rorl	$11,%r14d
 	xorl	%edx,%r15d
 	addl	%r12d,%ebx
 	rorl	$6,%r13d
 	paddd	%xmm7,%xmm3
 	andl	%r15d,%edi
 	xorl	%ecx,%r14d
 	addl	%r13d,%ebx
 	pshufd	$80,%xmm3,%xmm7
 	xorl	%edx,%edi
 	rorl	$2,%r14d
 	addl	%ebx,%r9d
 	movdqa	%xmm7,%xmm6
 	addl	%edi,%ebx
 	movl	%r9d,%r13d
 	psrld	$10,%xmm7
 	addl	%ebx,%r14d
 	rorl	$14,%r13d
 	psrlq	$17,%xmm6
 	movl	%r14d,%ebx
 	movl	%r10d,%r12d
 	pxor	%xmm6,%xmm7
 	rorl	$9,%r14d
 	xorl	%r9d,%r13d
 	xorl	%r11d,%r12d
 	rorl	$5,%r13d
 	xorl	%ebx,%r14d
 	psrlq	$2,%xmm6
 	andl	%r9d,%r12d
 	xorl	%r9d,%r13d
 	addl	60(%rsp),%eax
 	pxor	%xmm6,%xmm7
 	movl	%ebx,%edi
 	xorl	%r11d,%r12d
 	rorl	$11,%r14d
 	pshufd	$8,%xmm7,%xmm7
 	xorl	%ecx,%edi
 	addl	%r12d,%eax
 	movdqa	96(%rbp),%xmm6
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	pslldq	$8,%xmm7
 	xorl	%ebx,%r14d
 	addl	%r13d,%eax
 	xorl	%ecx,%r15d
 	paddd	%xmm7,%xmm3
 	rorl	$2,%r14d
 	addl	%eax,%r8d
 	addl	%r15d,%eax
 	paddd	%xmm3,%xmm6
 	movl	%r8d,%r13d
 	addl	%eax,%r14d
 	movdqa	%xmm6,48(%rsp)
 	cmpb	$0,131(%rbp)
 	jne	.Lssse3_00_47
 	rorl	$14,%r13d
 	movl	%r14d,%eax
 	movl	%r9d,%r12d
 	rorl	$9,%r14d
 	xorl	%r8d,%r13d
 	xorl	%r10d,%r12d
 	rorl	$5,%r13d
 	xorl	%eax,%r14d
 	andl	%r8d,%r12d
 	xorl	%r8d,%r13d
 	addl	0(%rsp),%r11d
 	movl	%eax,%r15d
 	xorl	%r10d,%r12d
 	rorl	$11,%r14d
 	xorl	%ebx,%r15d
 	addl	%r12d,%r11d
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	xorl	%eax,%r14d
 	addl	%r13d,%r11d
 	xorl	%ebx,%edi
 	rorl	$2,%r14d
 	addl	%r11d,%edx
 	addl	%edi,%r11d
 	movl	%edx,%r13d
 	addl	%r11d,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%r11d
 	movl	%r8d,%r12d
 	rorl	$9,%r14d
 	xorl	%edx,%r13d
 	xorl	%r9d,%r12d
 	rorl	$5,%r13d
 	xorl	%r11d,%r14d
 	andl	%edx,%r12d
 	xorl	%edx,%r13d
 	addl	4(%rsp),%r10d
 	movl	%r11d,%edi
 	xorl	%r9d,%r12d
 	rorl	$11,%r14d
 	xorl	%eax,%edi
 	addl	%r12d,%r10d
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%r11d,%r14d
 	addl	%r13d,%r10d
 	xorl	%eax,%r15d
 	rorl	$2,%r14d
 	addl	%r10d,%ecx
 	addl	%r15d,%r10d
 	movl	%ecx,%r13d
 	addl	%r10d,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%r10d
 	movl	%edx,%r12d
 	rorl	$9,%r14d
 	xorl	%ecx,%r13d
 	xorl	%r8d,%r12d
 	rorl	$5,%r13d
 	xorl	%r10d,%r14d
 	andl	%ecx,%r12d
 	xorl	%ecx,%r13d
 	addl	8(%rsp),%r9d
 	movl	%r10d,%r15d
 	xorl	%r8d,%r12d
 	rorl	$11,%r14d
 	xorl	%r11d,%r15d
 	addl	%r12d,%r9d
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	xorl	%r10d,%r14d
 	addl	%r13d,%r9d
 	xorl	%r11d,%edi
 	rorl	$2,%r14d
 	addl	%r9d,%ebx
 	addl	%edi,%r9d
 	movl	%ebx,%r13d
 	addl	%r9d,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%r9d
 	movl	%ecx,%r12d
 	rorl	$9,%r14d
 	xorl	%ebx,%r13d
 	xorl	%edx,%r12d
 	rorl	$5,%r13d
 	xorl	%r9d,%r14d
 	andl	%ebx,%r12d
 	xorl	%ebx,%r13d
 	addl	12(%rsp),%r8d
 	movl	%r9d,%edi
 	xorl	%edx,%r12d
 	rorl	$11,%r14d
 	xorl	%r10d,%edi
 	addl	%r12d,%r8d
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%r9d,%r14d
 	addl	%r13d,%r8d
 	xorl	%r10d,%r15d
 	rorl	$2,%r14d
 	addl	%r8d,%eax
 	addl	%r15d,%r8d
 	movl	%eax,%r13d
 	addl	%r8d,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%r8d
 	movl	%ebx,%r12d
 	rorl	$9,%r14d
 	xorl	%eax,%r13d
 	xorl	%ecx,%r12d
 	rorl	$5,%r13d
 	xorl	%r8d,%r14d
 	andl	%eax,%r12d
 	xorl	%eax,%r13d
 	addl	16(%rsp),%edx
 	movl	%r8d,%r15d
 	xorl	%ecx,%r12d
 	rorl	$11,%r14d
 	xorl	%r9d,%r15d
 	addl	%r12d,%edx
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	xorl	%r8d,%r14d
 	addl	%r13d,%edx
 	xorl	%r9d,%edi
 	rorl	$2,%r14d
 	addl	%edx,%r11d
 	addl	%edi,%edx
 	movl	%r11d,%r13d
 	addl	%edx,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%edx
 	movl	%eax,%r12d
 	rorl	$9,%r14d
 	xorl	%r11d,%r13d
 	xorl	%ebx,%r12d
 	rorl	$5,%r13d
 	xorl	%edx,%r14d
 	andl	%r11d,%r12d
 	xorl	%r11d,%r13d
 	addl	20(%rsp),%ecx
 	movl	%edx,%edi
 	xorl	%ebx,%r12d
 	rorl	$11,%r14d
 	xorl	%r8d,%edi
 	addl	%r12d,%ecx
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%edx,%r14d
 	addl	%r13d,%ecx
 	xorl	%r8d,%r15d
 	rorl	$2,%r14d
 	addl	%ecx,%r10d
 	addl	%r15d,%ecx
 	movl	%r10d,%r13d
 	addl	%ecx,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%ecx
 	movl	%r11d,%r12d
 	rorl	$9,%r14d
 	xorl	%r10d,%r13d
 	xorl	%eax,%r12d
 	rorl	$5,%r13d
 	xorl	%ecx,%r14d
 	andl	%r10d,%r12d
 	xorl	%r10d,%r13d
 	addl	24(%rsp),%ebx
 	movl	%ecx,%r15d
 	xorl	%eax,%r12d
 	rorl	$11,%r14d
 	xorl	%edx,%r15d
 	addl	%r12d,%ebx
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	xorl	%ecx,%r14d
 	addl	%r13d,%ebx
 	xorl	%edx,%edi
 	rorl	$2,%r14d
 	addl	%ebx,%r9d
 	addl	%edi,%ebx
 	movl	%r9d,%r13d
 	addl	%ebx,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%ebx
 	movl	%r10d,%r12d
 	rorl	$9,%r14d
 	xorl	%r9d,%r13d
 	xorl	%r11d,%r12d
 	rorl	$5,%r13d
 	xorl	%ebx,%r14d
 	andl	%r9d,%r12d
 	xorl	%r9d,%r13d
 	addl	28(%rsp),%eax
 	movl	%ebx,%edi
 	xorl	%r11d,%r12d
 	rorl	$11,%r14d
 	xorl	%ecx,%edi
 	addl	%r12d,%eax
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%ebx,%r14d
 	addl	%r13d,%eax
 	xorl	%ecx,%r15d
 	rorl	$2,%r14d
 	addl	%eax,%r8d
 	addl	%r15d,%eax
 	movl	%r8d,%r13d
 	addl	%eax,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%eax
 	movl	%r9d,%r12d
 	rorl	$9,%r14d
 	xorl	%r8d,%r13d
 	xorl	%r10d,%r12d
 	rorl	$5,%r13d
 	xorl	%eax,%r14d
 	andl	%r8d,%r12d
 	xorl	%r8d,%r13d
 	addl	32(%rsp),%r11d
 	movl	%eax,%r15d
 	xorl	%r10d,%r12d
 	rorl	$11,%r14d
 	xorl	%ebx,%r15d
 	addl	%r12d,%r11d
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	xorl	%eax,%r14d
 	addl	%r13d,%r11d
 	xorl	%ebx,%edi
 	rorl	$2,%r14d
 	addl	%r11d,%edx
 	addl	%edi,%r11d
 	movl	%edx,%r13d
 	addl	%r11d,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%r11d
 	movl	%r8d,%r12d
 	rorl	$9,%r14d
 	xorl	%edx,%r13d
 	xorl	%r9d,%r12d
 	rorl	$5,%r13d
 	xorl	%r11d,%r14d
 	andl	%edx,%r12d
 	xorl	%edx,%r13d
 	addl	36(%rsp),%r10d
 	movl	%r11d,%edi
 	xorl	%r9d,%r12d
 	rorl	$11,%r14d
 	xorl	%eax,%edi
 	addl	%r12d,%r10d
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%r11d,%r14d
 	addl	%r13d,%r10d
 	xorl	%eax,%r15d
 	rorl	$2,%r14d
 	addl	%r10d,%ecx
 	addl	%r15d,%r10d
 	movl	%ecx,%r13d
 	addl	%r10d,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%r10d
 	movl	%edx,%r12d
 	rorl	$9,%r14d
 	xorl	%ecx,%r13d
 	xorl	%r8d,%r12d
 	rorl	$5,%r13d
 	xorl	%r10d,%r14d
 	andl	%ecx,%r12d
 	xorl	%ecx,%r13d
 	addl	40(%rsp),%r9d
 	movl	%r10d,%r15d
 	xorl	%r8d,%r12d
 	rorl	$11,%r14d
 	xorl	%r11d,%r15d
 	addl	%r12d,%r9d
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	xorl	%r10d,%r14d
 	addl	%r13d,%r9d
 	xorl	%r11d,%edi
 	rorl	$2,%r14d
 	addl	%r9d,%ebx
 	addl	%edi,%r9d
 	movl	%ebx,%r13d
 	addl	%r9d,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%r9d
 	movl	%ecx,%r12d
 	rorl	$9,%r14d
 	xorl	%ebx,%r13d
 	xorl	%edx,%r12d
 	rorl	$5,%r13d
 	xorl	%r9d,%r14d
 	andl	%ebx,%r12d
 	xorl	%ebx,%r13d
 	addl	44(%rsp),%r8d
 	movl	%r9d,%edi
 	xorl	%edx,%r12d
 	rorl	$11,%r14d
 	xorl	%r10d,%edi
 	addl	%r12d,%r8d
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%r9d,%r14d
 	addl	%r13d,%r8d
 	xorl	%r10d,%r15d
 	rorl	$2,%r14d
 	addl	%r8d,%eax
 	addl	%r15d,%r8d
 	movl	%eax,%r13d
 	addl	%r8d,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%r8d
 	movl	%ebx,%r12d
 	rorl	$9,%r14d
 	xorl	%eax,%r13d
 	xorl	%ecx,%r12d
 	rorl	$5,%r13d
 	xorl	%r8d,%r14d
 	andl	%eax,%r12d
 	xorl	%eax,%r13d
 	addl	48(%rsp),%edx
 	movl	%r8d,%r15d
 	xorl	%ecx,%r12d
 	rorl	$11,%r14d
 	xorl	%r9d,%r15d
 	addl	%r12d,%edx
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	xorl	%r8d,%r14d
 	addl	%r13d,%edx
 	xorl	%r9d,%edi
 	rorl	$2,%r14d
 	addl	%edx,%r11d
 	addl	%edi,%edx
 	movl	%r11d,%r13d
 	addl	%edx,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%edx
 	movl	%eax,%r12d
 	rorl	$9,%r14d
 	xorl	%r11d,%r13d
 	xorl	%ebx,%r12d
 	rorl	$5,%r13d
 	xorl	%edx,%r14d
 	andl	%r11d,%r12d
 	xorl	%r11d,%r13d
 	addl	52(%rsp),%ecx
 	movl	%edx,%edi
 	xorl	%ebx,%r12d
 	rorl	$11,%r14d
 	xorl	%r8d,%edi
 	addl	%r12d,%ecx
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%edx,%r14d
 	addl	%r13d,%ecx
 	xorl	%r8d,%r15d
 	rorl	$2,%r14d
 	addl	%ecx,%r10d
 	addl	%r15d,%ecx
 	movl	%r10d,%r13d
 	addl	%ecx,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%ecx
 	movl	%r11d,%r12d
 	rorl	$9,%r14d
 	xorl	%r10d,%r13d
 	xorl	%eax,%r12d
 	rorl	$5,%r13d
 	xorl	%ecx,%r14d
 	andl	%r10d,%r12d
 	xorl	%r10d,%r13d
 	addl	56(%rsp),%ebx
 	movl	%ecx,%r15d
 	xorl	%eax,%r12d
 	rorl	$11,%r14d
 	xorl	%edx,%r15d
 	addl	%r12d,%ebx
 	rorl	$6,%r13d
 	andl	%r15d,%edi
 	xorl	%ecx,%r14d
 	addl	%r13d,%ebx
 	xorl	%edx,%edi
 	rorl	$2,%r14d
 	addl	%ebx,%r9d
 	addl	%edi,%ebx
 	movl	%r9d,%r13d
 	addl	%ebx,%r14d
 	rorl	$14,%r13d
 	movl	%r14d,%ebx
 	movl	%r10d,%r12d
 	rorl	$9,%r14d
 	xorl	%r9d,%r13d
 	xorl	%r11d,%r12d
 	rorl	$5,%r13d
 	xorl	%ebx,%r14d
 	andl	%r9d,%r12d
 	xorl	%r9d,%r13d
 	addl	60(%rsp),%eax
 	movl	%ebx,%edi
 	xorl	%r11d,%r12d
 	rorl	$11,%r14d
 	xorl	%ecx,%edi
 	addl	%r12d,%eax
 	rorl	$6,%r13d
 	andl	%edi,%r15d
 	xorl	%ebx,%r14d
 	addl	%r13d,%eax
 	xorl	%ecx,%r15d
 	rorl	$2,%r14d
 	addl	%eax,%r8d
 	addl	%r15d,%eax
 	movl	%r8d,%r13d
 	addl	%eax,%r14d
 	movq	64+0(%rsp),%rdi
 	movl	%r14d,%eax
 
 	addl	0(%rdi),%eax
 	leaq	64(%rsi),%rsi
 	addl	4(%rdi),%ebx
 	addl	8(%rdi),%ecx
 	addl	12(%rdi),%edx
 	addl	16(%rdi),%r8d
 	addl	20(%rdi),%r9d
 	addl	24(%rdi),%r10d
 	addl	28(%rdi),%r11d
 
 	cmpq	64+16(%rsp),%rsi
 
 	movl	%eax,0(%rdi)
 	movl	%ebx,4(%rdi)
 	movl	%ecx,8(%rdi)
 	movl	%edx,12(%rdi)
 	movl	%r8d,16(%rdi)
 	movl	%r9d,20(%rdi)
 	movl	%r10d,24(%rdi)
 	movl	%r11d,28(%rdi)
 	jb	.Lloop_ssse3
 
 	movq	88(%rsp),%rsi
 .cfi_def_cfa	%rsi,8
 	movq	-48(%rsi),%r15
 .cfi_restore	%r15
 	movq	-40(%rsi),%r14
 .cfi_restore	%r14
 	movq	-32(%rsi),%r13
 .cfi_restore	%r13
 	movq	-24(%rsi),%r12
 .cfi_restore	%r12
 	movq	-16(%rsi),%rbp
 .cfi_restore	%rbp
 	movq	-8(%rsi),%rbx
 .cfi_restore	%rbx
 	leaq	(%rsi),%rsp
 .cfi_def_cfa_register	%rsp
 .Lepilogue_ssse3:
 	RET
 .cfi_endproc
 SET_SIZE(zfs_sha256_transform_ssse3)
+STACK_FRAME_NON_STANDARD zfs_sha256_transform_ssse3
 
 ENTRY_ALIGN(zfs_sha256_transform_avx, 64)
 .cfi_startproc
 	ENDBR
 	movq	%rsp,%rax
 .cfi_def_cfa_register	%rax
 	pushq	%rbx
 .cfi_offset	%rbx,-16
 	pushq	%rbp
 .cfi_offset	%rbp,-24
 	pushq	%r12
 .cfi_offset	%r12,-32
 	pushq	%r13
 .cfi_offset	%r13,-40
 	pushq	%r14
 .cfi_offset	%r14,-48
 	pushq	%r15
 .cfi_offset	%r15,-56
 	shlq	$4,%rdx
 	subq	$96,%rsp
 	leaq	(%rsi,%rdx,4),%rdx
 	andq	$-64,%rsp
 	movq	%rdi,64+0(%rsp)
 	movq	%rsi,64+8(%rsp)
 	movq	%rdx,64+16(%rsp)
 	movq	%rax,88(%rsp)
 .cfi_escape	0x0f,0x06,0x77,0xd8,0x00,0x06,0x23,0x08
 .Lprologue_avx:
 
 	vzeroupper
 	movl	0(%rdi),%eax
 	movl	4(%rdi),%ebx
 	movl	8(%rdi),%ecx
 	movl	12(%rdi),%edx
 	movl	16(%rdi),%r8d
 	movl	20(%rdi),%r9d
 	movl	24(%rdi),%r10d
 	movl	28(%rdi),%r11d
 	vmovdqa	K256+512+32(%rip),%xmm8
 	vmovdqa	K256+512+64(%rip),%xmm9
 	jmp	.Lloop_avx
 .balign	16
 .Lloop_avx:
 	vmovdqa	K256+512(%rip),%xmm7
 	vmovdqu	0(%rsi),%xmm0
 	vmovdqu	16(%rsi),%xmm1
 	vmovdqu	32(%rsi),%xmm2
 	vmovdqu	48(%rsi),%xmm3
 	vpshufb	%xmm7,%xmm0,%xmm0
 	leaq	K256(%rip),%rbp
 	vpshufb	%xmm7,%xmm1,%xmm1
 	vpshufb	%xmm7,%xmm2,%xmm2
 	vpaddd	0(%rbp),%xmm0,%xmm4
 	vpshufb	%xmm7,%xmm3,%xmm3
 	vpaddd	32(%rbp),%xmm1,%xmm5
 	vpaddd	64(%rbp),%xmm2,%xmm6
 	vpaddd	96(%rbp),%xmm3,%xmm7
 	vmovdqa	%xmm4,0(%rsp)
 	movl	%eax,%r14d
 	vmovdqa	%xmm5,16(%rsp)
 	movl	%ebx,%edi
 	vmovdqa	%xmm6,32(%rsp)
 	xorl	%ecx,%edi
 	vmovdqa	%xmm7,48(%rsp)
 	movl	%r8d,%r13d
 	jmp	.Lavx_00_47
 
 .balign	16
 .Lavx_00_47:
 	subq	$-128,%rbp
 	vpalignr	$4,%xmm0,%xmm1,%xmm4
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%eax
 	movl	%r9d,%r12d
 	vpalignr	$4,%xmm2,%xmm3,%xmm7
 	shrdl	$9,%r14d,%r14d
 	xorl	%r8d,%r13d
 	xorl	%r10d,%r12d
 	vpsrld	$7,%xmm4,%xmm6
 	shrdl	$5,%r13d,%r13d
 	xorl	%eax,%r14d
 	andl	%r8d,%r12d
 	vpaddd	%xmm7,%xmm0,%xmm0
 	xorl	%r8d,%r13d
 	addl	0(%rsp),%r11d
 	movl	%eax,%r15d
 	vpsrld	$3,%xmm4,%xmm7
 	xorl	%r10d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%ebx,%r15d
 	vpslld	$14,%xmm4,%xmm5
 	addl	%r12d,%r11d
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	vpxor	%xmm6,%xmm7,%xmm4
 	xorl	%eax,%r14d
 	addl	%r13d,%r11d
 	xorl	%ebx,%edi
 	vpshufd	$250,%xmm3,%xmm7
 	shrdl	$2,%r14d,%r14d
 	addl	%r11d,%edx
 	addl	%edi,%r11d
 	vpsrld	$11,%xmm6,%xmm6
 	movl	%edx,%r13d
 	addl	%r11d,%r14d
 	shrdl	$14,%r13d,%r13d
 	vpxor	%xmm5,%xmm4,%xmm4
 	movl	%r14d,%r11d
 	movl	%r8d,%r12d
 	shrdl	$9,%r14d,%r14d
 	vpslld	$11,%xmm5,%xmm5
 	xorl	%edx,%r13d
 	xorl	%r9d,%r12d
 	shrdl	$5,%r13d,%r13d
 	vpxor	%xmm6,%xmm4,%xmm4
 	xorl	%r11d,%r14d
 	andl	%edx,%r12d
 	xorl	%edx,%r13d
 	vpsrld	$10,%xmm7,%xmm6
 	addl	4(%rsp),%r10d
 	movl	%r11d,%edi
 	xorl	%r9d,%r12d
 	vpxor	%xmm5,%xmm4,%xmm4
 	shrdl	$11,%r14d,%r14d
 	xorl	%eax,%edi
 	addl	%r12d,%r10d
 	vpsrlq	$17,%xmm7,%xmm7
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%r11d,%r14d
 	vpaddd	%xmm4,%xmm0,%xmm0
 	addl	%r13d,%r10d
 	xorl	%eax,%r15d
 	shrdl	$2,%r14d,%r14d
 	vpxor	%xmm7,%xmm6,%xmm6
 	addl	%r10d,%ecx
 	addl	%r15d,%r10d
 	movl	%ecx,%r13d
 	vpsrlq	$2,%xmm7,%xmm7
 	addl	%r10d,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r10d
 	vpxor	%xmm7,%xmm6,%xmm6
 	movl	%edx,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%ecx,%r13d
 	vpshufb	%xmm8,%xmm6,%xmm6
 	xorl	%r8d,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%r10d,%r14d
 	vpaddd	%xmm6,%xmm0,%xmm0
 	andl	%ecx,%r12d
 	xorl	%ecx,%r13d
 	addl	8(%rsp),%r9d
 	vpshufd	$80,%xmm0,%xmm7
 	movl	%r10d,%r15d
 	xorl	%r8d,%r12d
 	shrdl	$11,%r14d,%r14d
 	vpsrld	$10,%xmm7,%xmm6
 	xorl	%r11d,%r15d
 	addl	%r12d,%r9d
 	shrdl	$6,%r13d,%r13d
 	vpsrlq	$17,%xmm7,%xmm7
 	andl	%r15d,%edi
 	xorl	%r10d,%r14d
 	addl	%r13d,%r9d
 	vpxor	%xmm7,%xmm6,%xmm6
 	xorl	%r11d,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%r9d,%ebx
 	vpsrlq	$2,%xmm7,%xmm7
 	addl	%edi,%r9d
 	movl	%ebx,%r13d
 	addl	%r9d,%r14d
 	vpxor	%xmm7,%xmm6,%xmm6
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r9d
 	movl	%ecx,%r12d
 	vpshufb	%xmm9,%xmm6,%xmm6
 	shrdl	$9,%r14d,%r14d
 	xorl	%ebx,%r13d
 	xorl	%edx,%r12d
 	vpaddd	%xmm6,%xmm0,%xmm0
 	shrdl	$5,%r13d,%r13d
 	xorl	%r9d,%r14d
 	andl	%ebx,%r12d
 	vpaddd	0(%rbp),%xmm0,%xmm6
 	xorl	%ebx,%r13d
 	addl	12(%rsp),%r8d
 	movl	%r9d,%edi
 	xorl	%edx,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r10d,%edi
 	addl	%r12d,%r8d
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%r9d,%r14d
 	addl	%r13d,%r8d
 	xorl	%r10d,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%r8d,%eax
 	addl	%r15d,%r8d
 	movl	%eax,%r13d
 	addl	%r8d,%r14d
 	vmovdqa	%xmm6,0(%rsp)
 	vpalignr	$4,%xmm1,%xmm2,%xmm4
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r8d
 	movl	%ebx,%r12d
 	vpalignr	$4,%xmm3,%xmm0,%xmm7
 	shrdl	$9,%r14d,%r14d
 	xorl	%eax,%r13d
 	xorl	%ecx,%r12d
 	vpsrld	$7,%xmm4,%xmm6
 	shrdl	$5,%r13d,%r13d
 	xorl	%r8d,%r14d
 	andl	%eax,%r12d
 	vpaddd	%xmm7,%xmm1,%xmm1
 	xorl	%eax,%r13d
 	addl	16(%rsp),%edx
 	movl	%r8d,%r15d
 	vpsrld	$3,%xmm4,%xmm7
 	xorl	%ecx,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r9d,%r15d
 	vpslld	$14,%xmm4,%xmm5
 	addl	%r12d,%edx
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	vpxor	%xmm6,%xmm7,%xmm4
 	xorl	%r8d,%r14d
 	addl	%r13d,%edx
 	xorl	%r9d,%edi
 	vpshufd	$250,%xmm0,%xmm7
 	shrdl	$2,%r14d,%r14d
 	addl	%edx,%r11d
 	addl	%edi,%edx
 	vpsrld	$11,%xmm6,%xmm6
 	movl	%r11d,%r13d
 	addl	%edx,%r14d
 	shrdl	$14,%r13d,%r13d
 	vpxor	%xmm5,%xmm4,%xmm4
 	movl	%r14d,%edx
 	movl	%eax,%r12d
 	shrdl	$9,%r14d,%r14d
 	vpslld	$11,%xmm5,%xmm5
 	xorl	%r11d,%r13d
 	xorl	%ebx,%r12d
 	shrdl	$5,%r13d,%r13d
 	vpxor	%xmm6,%xmm4,%xmm4
 	xorl	%edx,%r14d
 	andl	%r11d,%r12d
 	xorl	%r11d,%r13d
 	vpsrld	$10,%xmm7,%xmm6
 	addl	20(%rsp),%ecx
 	movl	%edx,%edi
 	xorl	%ebx,%r12d
 	vpxor	%xmm5,%xmm4,%xmm4
 	shrdl	$11,%r14d,%r14d
 	xorl	%r8d,%edi
 	addl	%r12d,%ecx
 	vpsrlq	$17,%xmm7,%xmm7
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%edx,%r14d
 	vpaddd	%xmm4,%xmm1,%xmm1
 	addl	%r13d,%ecx
 	xorl	%r8d,%r15d
 	shrdl	$2,%r14d,%r14d
 	vpxor	%xmm7,%xmm6,%xmm6
 	addl	%ecx,%r10d
 	addl	%r15d,%ecx
 	movl	%r10d,%r13d
 	vpsrlq	$2,%xmm7,%xmm7
 	addl	%ecx,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%ecx
 	vpxor	%xmm7,%xmm6,%xmm6
 	movl	%r11d,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%r10d,%r13d
 	vpshufb	%xmm8,%xmm6,%xmm6
 	xorl	%eax,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%ecx,%r14d
 	vpaddd	%xmm6,%xmm1,%xmm1
 	andl	%r10d,%r12d
 	xorl	%r10d,%r13d
 	addl	24(%rsp),%ebx
 	vpshufd	$80,%xmm1,%xmm7
 	movl	%ecx,%r15d
 	xorl	%eax,%r12d
 	shrdl	$11,%r14d,%r14d
 	vpsrld	$10,%xmm7,%xmm6
 	xorl	%edx,%r15d
 	addl	%r12d,%ebx
 	shrdl	$6,%r13d,%r13d
 	vpsrlq	$17,%xmm7,%xmm7
 	andl	%r15d,%edi
 	xorl	%ecx,%r14d
 	addl	%r13d,%ebx
 	vpxor	%xmm7,%xmm6,%xmm6
 	xorl	%edx,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%ebx,%r9d
 	vpsrlq	$2,%xmm7,%xmm7
 	addl	%edi,%ebx
 	movl	%r9d,%r13d
 	addl	%ebx,%r14d
 	vpxor	%xmm7,%xmm6,%xmm6
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%ebx
 	movl	%r10d,%r12d
 	vpshufb	%xmm9,%xmm6,%xmm6
 	shrdl	$9,%r14d,%r14d
 	xorl	%r9d,%r13d
 	xorl	%r11d,%r12d
 	vpaddd	%xmm6,%xmm1,%xmm1
 	shrdl	$5,%r13d,%r13d
 	xorl	%ebx,%r14d
 	andl	%r9d,%r12d
 	vpaddd	32(%rbp),%xmm1,%xmm6
 	xorl	%r9d,%r13d
 	addl	28(%rsp),%eax
 	movl	%ebx,%edi
 	xorl	%r11d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%ecx,%edi
 	addl	%r12d,%eax
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%ebx,%r14d
 	addl	%r13d,%eax
 	xorl	%ecx,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%eax,%r8d
 	addl	%r15d,%eax
 	movl	%r8d,%r13d
 	addl	%eax,%r14d
 	vmovdqa	%xmm6,16(%rsp)
 	vpalignr	$4,%xmm2,%xmm3,%xmm4
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%eax
 	movl	%r9d,%r12d
 	vpalignr	$4,%xmm0,%xmm1,%xmm7
 	shrdl	$9,%r14d,%r14d
 	xorl	%r8d,%r13d
 	xorl	%r10d,%r12d
 	vpsrld	$7,%xmm4,%xmm6
 	shrdl	$5,%r13d,%r13d
 	xorl	%eax,%r14d
 	andl	%r8d,%r12d
 	vpaddd	%xmm7,%xmm2,%xmm2
 	xorl	%r8d,%r13d
 	addl	32(%rsp),%r11d
 	movl	%eax,%r15d
 	vpsrld	$3,%xmm4,%xmm7
 	xorl	%r10d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%ebx,%r15d
 	vpslld	$14,%xmm4,%xmm5
 	addl	%r12d,%r11d
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	vpxor	%xmm6,%xmm7,%xmm4
 	xorl	%eax,%r14d
 	addl	%r13d,%r11d
 	xorl	%ebx,%edi
 	vpshufd	$250,%xmm1,%xmm7
 	shrdl	$2,%r14d,%r14d
 	addl	%r11d,%edx
 	addl	%edi,%r11d
 	vpsrld	$11,%xmm6,%xmm6
 	movl	%edx,%r13d
 	addl	%r11d,%r14d
 	shrdl	$14,%r13d,%r13d
 	vpxor	%xmm5,%xmm4,%xmm4
 	movl	%r14d,%r11d
 	movl	%r8d,%r12d
 	shrdl	$9,%r14d,%r14d
 	vpslld	$11,%xmm5,%xmm5
 	xorl	%edx,%r13d
 	xorl	%r9d,%r12d
 	shrdl	$5,%r13d,%r13d
 	vpxor	%xmm6,%xmm4,%xmm4
 	xorl	%r11d,%r14d
 	andl	%edx,%r12d
 	xorl	%edx,%r13d
 	vpsrld	$10,%xmm7,%xmm6
 	addl	36(%rsp),%r10d
 	movl	%r11d,%edi
 	xorl	%r9d,%r12d
 	vpxor	%xmm5,%xmm4,%xmm4
 	shrdl	$11,%r14d,%r14d
 	xorl	%eax,%edi
 	addl	%r12d,%r10d
 	vpsrlq	$17,%xmm7,%xmm7
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%r11d,%r14d
 	vpaddd	%xmm4,%xmm2,%xmm2
 	addl	%r13d,%r10d
 	xorl	%eax,%r15d
 	shrdl	$2,%r14d,%r14d
 	vpxor	%xmm7,%xmm6,%xmm6
 	addl	%r10d,%ecx
 	addl	%r15d,%r10d
 	movl	%ecx,%r13d
 	vpsrlq	$2,%xmm7,%xmm7
 	addl	%r10d,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r10d
 	vpxor	%xmm7,%xmm6,%xmm6
 	movl	%edx,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%ecx,%r13d
 	vpshufb	%xmm8,%xmm6,%xmm6
 	xorl	%r8d,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%r10d,%r14d
 	vpaddd	%xmm6,%xmm2,%xmm2
 	andl	%ecx,%r12d
 	xorl	%ecx,%r13d
 	addl	40(%rsp),%r9d
 	vpshufd	$80,%xmm2,%xmm7
 	movl	%r10d,%r15d
 	xorl	%r8d,%r12d
 	shrdl	$11,%r14d,%r14d
 	vpsrld	$10,%xmm7,%xmm6
 	xorl	%r11d,%r15d
 	addl	%r12d,%r9d
 	shrdl	$6,%r13d,%r13d
 	vpsrlq	$17,%xmm7,%xmm7
 	andl	%r15d,%edi
 	xorl	%r10d,%r14d
 	addl	%r13d,%r9d
 	vpxor	%xmm7,%xmm6,%xmm6
 	xorl	%r11d,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%r9d,%ebx
 	vpsrlq	$2,%xmm7,%xmm7
 	addl	%edi,%r9d
 	movl	%ebx,%r13d
 	addl	%r9d,%r14d
 	vpxor	%xmm7,%xmm6,%xmm6
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r9d
 	movl	%ecx,%r12d
 	vpshufb	%xmm9,%xmm6,%xmm6
 	shrdl	$9,%r14d,%r14d
 	xorl	%ebx,%r13d
 	xorl	%edx,%r12d
 	vpaddd	%xmm6,%xmm2,%xmm2
 	shrdl	$5,%r13d,%r13d
 	xorl	%r9d,%r14d
 	andl	%ebx,%r12d
 	vpaddd	64(%rbp),%xmm2,%xmm6
 	xorl	%ebx,%r13d
 	addl	44(%rsp),%r8d
 	movl	%r9d,%edi
 	xorl	%edx,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r10d,%edi
 	addl	%r12d,%r8d
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%r9d,%r14d
 	addl	%r13d,%r8d
 	xorl	%r10d,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%r8d,%eax
 	addl	%r15d,%r8d
 	movl	%eax,%r13d
 	addl	%r8d,%r14d
 	vmovdqa	%xmm6,32(%rsp)
 	vpalignr	$4,%xmm3,%xmm0,%xmm4
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r8d
 	movl	%ebx,%r12d
 	vpalignr	$4,%xmm1,%xmm2,%xmm7
 	shrdl	$9,%r14d,%r14d
 	xorl	%eax,%r13d
 	xorl	%ecx,%r12d
 	vpsrld	$7,%xmm4,%xmm6
 	shrdl	$5,%r13d,%r13d
 	xorl	%r8d,%r14d
 	andl	%eax,%r12d
 	vpaddd	%xmm7,%xmm3,%xmm3
 	xorl	%eax,%r13d
 	addl	48(%rsp),%edx
 	movl	%r8d,%r15d
 	vpsrld	$3,%xmm4,%xmm7
 	xorl	%ecx,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r9d,%r15d
 	vpslld	$14,%xmm4,%xmm5
 	addl	%r12d,%edx
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	vpxor	%xmm6,%xmm7,%xmm4
 	xorl	%r8d,%r14d
 	addl	%r13d,%edx
 	xorl	%r9d,%edi
 	vpshufd	$250,%xmm2,%xmm7
 	shrdl	$2,%r14d,%r14d
 	addl	%edx,%r11d
 	addl	%edi,%edx
 	vpsrld	$11,%xmm6,%xmm6
 	movl	%r11d,%r13d
 	addl	%edx,%r14d
 	shrdl	$14,%r13d,%r13d
 	vpxor	%xmm5,%xmm4,%xmm4
 	movl	%r14d,%edx
 	movl	%eax,%r12d
 	shrdl	$9,%r14d,%r14d
 	vpslld	$11,%xmm5,%xmm5
 	xorl	%r11d,%r13d
 	xorl	%ebx,%r12d
 	shrdl	$5,%r13d,%r13d
 	vpxor	%xmm6,%xmm4,%xmm4
 	xorl	%edx,%r14d
 	andl	%r11d,%r12d
 	xorl	%r11d,%r13d
 	vpsrld	$10,%xmm7,%xmm6
 	addl	52(%rsp),%ecx
 	movl	%edx,%edi
 	xorl	%ebx,%r12d
 	vpxor	%xmm5,%xmm4,%xmm4
 	shrdl	$11,%r14d,%r14d
 	xorl	%r8d,%edi
 	addl	%r12d,%ecx
 	vpsrlq	$17,%xmm7,%xmm7
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%edx,%r14d
 	vpaddd	%xmm4,%xmm3,%xmm3
 	addl	%r13d,%ecx
 	xorl	%r8d,%r15d
 	shrdl	$2,%r14d,%r14d
 	vpxor	%xmm7,%xmm6,%xmm6
 	addl	%ecx,%r10d
 	addl	%r15d,%ecx
 	movl	%r10d,%r13d
 	vpsrlq	$2,%xmm7,%xmm7
 	addl	%ecx,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%ecx
 	vpxor	%xmm7,%xmm6,%xmm6
 	movl	%r11d,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%r10d,%r13d
 	vpshufb	%xmm8,%xmm6,%xmm6
 	xorl	%eax,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%ecx,%r14d
 	vpaddd	%xmm6,%xmm3,%xmm3
 	andl	%r10d,%r12d
 	xorl	%r10d,%r13d
 	addl	56(%rsp),%ebx
 	vpshufd	$80,%xmm3,%xmm7
 	movl	%ecx,%r15d
 	xorl	%eax,%r12d
 	shrdl	$11,%r14d,%r14d
 	vpsrld	$10,%xmm7,%xmm6
 	xorl	%edx,%r15d
 	addl	%r12d,%ebx
 	shrdl	$6,%r13d,%r13d
 	vpsrlq	$17,%xmm7,%xmm7
 	andl	%r15d,%edi
 	xorl	%ecx,%r14d
 	addl	%r13d,%ebx
 	vpxor	%xmm7,%xmm6,%xmm6
 	xorl	%edx,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%ebx,%r9d
 	vpsrlq	$2,%xmm7,%xmm7
 	addl	%edi,%ebx
 	movl	%r9d,%r13d
 	addl	%ebx,%r14d
 	vpxor	%xmm7,%xmm6,%xmm6
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%ebx
 	movl	%r10d,%r12d
 	vpshufb	%xmm9,%xmm6,%xmm6
 	shrdl	$9,%r14d,%r14d
 	xorl	%r9d,%r13d
 	xorl	%r11d,%r12d
 	vpaddd	%xmm6,%xmm3,%xmm3
 	shrdl	$5,%r13d,%r13d
 	xorl	%ebx,%r14d
 	andl	%r9d,%r12d
 	vpaddd	96(%rbp),%xmm3,%xmm6
 	xorl	%r9d,%r13d
 	addl	60(%rsp),%eax
 	movl	%ebx,%edi
 	xorl	%r11d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%ecx,%edi
 	addl	%r12d,%eax
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%ebx,%r14d
 	addl	%r13d,%eax
 	xorl	%ecx,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%eax,%r8d
 	addl	%r15d,%eax
 	movl	%r8d,%r13d
 	addl	%eax,%r14d
 	vmovdqa	%xmm6,48(%rsp)
 	cmpb	$0,131(%rbp)
 	jne	.Lavx_00_47
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%eax
 	movl	%r9d,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%r8d,%r13d
 	xorl	%r10d,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%eax,%r14d
 	andl	%r8d,%r12d
 	xorl	%r8d,%r13d
 	addl	0(%rsp),%r11d
 	movl	%eax,%r15d
 	xorl	%r10d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%ebx,%r15d
 	addl	%r12d,%r11d
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	xorl	%eax,%r14d
 	addl	%r13d,%r11d
 	xorl	%ebx,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%r11d,%edx
 	addl	%edi,%r11d
 	movl	%edx,%r13d
 	addl	%r11d,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r11d
 	movl	%r8d,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%edx,%r13d
 	xorl	%r9d,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%r11d,%r14d
 	andl	%edx,%r12d
 	xorl	%edx,%r13d
 	addl	4(%rsp),%r10d
 	movl	%r11d,%edi
 	xorl	%r9d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%eax,%edi
 	addl	%r12d,%r10d
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%r11d,%r14d
 	addl	%r13d,%r10d
 	xorl	%eax,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%r10d,%ecx
 	addl	%r15d,%r10d
 	movl	%ecx,%r13d
 	addl	%r10d,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r10d
 	movl	%edx,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%ecx,%r13d
 	xorl	%r8d,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%r10d,%r14d
 	andl	%ecx,%r12d
 	xorl	%ecx,%r13d
 	addl	8(%rsp),%r9d
 	movl	%r10d,%r15d
 	xorl	%r8d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r11d,%r15d
 	addl	%r12d,%r9d
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	xorl	%r10d,%r14d
 	addl	%r13d,%r9d
 	xorl	%r11d,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%r9d,%ebx
 	addl	%edi,%r9d
 	movl	%ebx,%r13d
 	addl	%r9d,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r9d
 	movl	%ecx,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%ebx,%r13d
 	xorl	%edx,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%r9d,%r14d
 	andl	%ebx,%r12d
 	xorl	%ebx,%r13d
 	addl	12(%rsp),%r8d
 	movl	%r9d,%edi
 	xorl	%edx,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r10d,%edi
 	addl	%r12d,%r8d
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%r9d,%r14d
 	addl	%r13d,%r8d
 	xorl	%r10d,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%r8d,%eax
 	addl	%r15d,%r8d
 	movl	%eax,%r13d
 	addl	%r8d,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r8d
 	movl	%ebx,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%eax,%r13d
 	xorl	%ecx,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%r8d,%r14d
 	andl	%eax,%r12d
 	xorl	%eax,%r13d
 	addl	16(%rsp),%edx
 	movl	%r8d,%r15d
 	xorl	%ecx,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r9d,%r15d
 	addl	%r12d,%edx
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	xorl	%r8d,%r14d
 	addl	%r13d,%edx
 	xorl	%r9d,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%edx,%r11d
 	addl	%edi,%edx
 	movl	%r11d,%r13d
 	addl	%edx,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%edx
 	movl	%eax,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%r11d,%r13d
 	xorl	%ebx,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%edx,%r14d
 	andl	%r11d,%r12d
 	xorl	%r11d,%r13d
 	addl	20(%rsp),%ecx
 	movl	%edx,%edi
 	xorl	%ebx,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r8d,%edi
 	addl	%r12d,%ecx
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%edx,%r14d
 	addl	%r13d,%ecx
 	xorl	%r8d,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%ecx,%r10d
 	addl	%r15d,%ecx
 	movl	%r10d,%r13d
 	addl	%ecx,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%ecx
 	movl	%r11d,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%r10d,%r13d
 	xorl	%eax,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%ecx,%r14d
 	andl	%r10d,%r12d
 	xorl	%r10d,%r13d
 	addl	24(%rsp),%ebx
 	movl	%ecx,%r15d
 	xorl	%eax,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%edx,%r15d
 	addl	%r12d,%ebx
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	xorl	%ecx,%r14d
 	addl	%r13d,%ebx
 	xorl	%edx,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%ebx,%r9d
 	addl	%edi,%ebx
 	movl	%r9d,%r13d
 	addl	%ebx,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%ebx
 	movl	%r10d,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%r9d,%r13d
 	xorl	%r11d,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%ebx,%r14d
 	andl	%r9d,%r12d
 	xorl	%r9d,%r13d
 	addl	28(%rsp),%eax
 	movl	%ebx,%edi
 	xorl	%r11d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%ecx,%edi
 	addl	%r12d,%eax
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%ebx,%r14d
 	addl	%r13d,%eax
 	xorl	%ecx,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%eax,%r8d
 	addl	%r15d,%eax
 	movl	%r8d,%r13d
 	addl	%eax,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%eax
 	movl	%r9d,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%r8d,%r13d
 	xorl	%r10d,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%eax,%r14d
 	andl	%r8d,%r12d
 	xorl	%r8d,%r13d
 	addl	32(%rsp),%r11d
 	movl	%eax,%r15d
 	xorl	%r10d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%ebx,%r15d
 	addl	%r12d,%r11d
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	xorl	%eax,%r14d
 	addl	%r13d,%r11d
 	xorl	%ebx,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%r11d,%edx
 	addl	%edi,%r11d
 	movl	%edx,%r13d
 	addl	%r11d,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r11d
 	movl	%r8d,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%edx,%r13d
 	xorl	%r9d,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%r11d,%r14d
 	andl	%edx,%r12d
 	xorl	%edx,%r13d
 	addl	36(%rsp),%r10d
 	movl	%r11d,%edi
 	xorl	%r9d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%eax,%edi
 	addl	%r12d,%r10d
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%r11d,%r14d
 	addl	%r13d,%r10d
 	xorl	%eax,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%r10d,%ecx
 	addl	%r15d,%r10d
 	movl	%ecx,%r13d
 	addl	%r10d,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r10d
 	movl	%edx,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%ecx,%r13d
 	xorl	%r8d,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%r10d,%r14d
 	andl	%ecx,%r12d
 	xorl	%ecx,%r13d
 	addl	40(%rsp),%r9d
 	movl	%r10d,%r15d
 	xorl	%r8d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r11d,%r15d
 	addl	%r12d,%r9d
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	xorl	%r10d,%r14d
 	addl	%r13d,%r9d
 	xorl	%r11d,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%r9d,%ebx
 	addl	%edi,%r9d
 	movl	%ebx,%r13d
 	addl	%r9d,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r9d
 	movl	%ecx,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%ebx,%r13d
 	xorl	%edx,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%r9d,%r14d
 	andl	%ebx,%r12d
 	xorl	%ebx,%r13d
 	addl	44(%rsp),%r8d
 	movl	%r9d,%edi
 	xorl	%edx,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r10d,%edi
 	addl	%r12d,%r8d
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%r9d,%r14d
 	addl	%r13d,%r8d
 	xorl	%r10d,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%r8d,%eax
 	addl	%r15d,%r8d
 	movl	%eax,%r13d
 	addl	%r8d,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%r8d
 	movl	%ebx,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%eax,%r13d
 	xorl	%ecx,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%r8d,%r14d
 	andl	%eax,%r12d
 	xorl	%eax,%r13d
 	addl	48(%rsp),%edx
 	movl	%r8d,%r15d
 	xorl	%ecx,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r9d,%r15d
 	addl	%r12d,%edx
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	xorl	%r8d,%r14d
 	addl	%r13d,%edx
 	xorl	%r9d,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%edx,%r11d
 	addl	%edi,%edx
 	movl	%r11d,%r13d
 	addl	%edx,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%edx
 	movl	%eax,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%r11d,%r13d
 	xorl	%ebx,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%edx,%r14d
 	andl	%r11d,%r12d
 	xorl	%r11d,%r13d
 	addl	52(%rsp),%ecx
 	movl	%edx,%edi
 	xorl	%ebx,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%r8d,%edi
 	addl	%r12d,%ecx
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%edx,%r14d
 	addl	%r13d,%ecx
 	xorl	%r8d,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%ecx,%r10d
 	addl	%r15d,%ecx
 	movl	%r10d,%r13d
 	addl	%ecx,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%ecx
 	movl	%r11d,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%r10d,%r13d
 	xorl	%eax,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%ecx,%r14d
 	andl	%r10d,%r12d
 	xorl	%r10d,%r13d
 	addl	56(%rsp),%ebx
 	movl	%ecx,%r15d
 	xorl	%eax,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%edx,%r15d
 	addl	%r12d,%ebx
 	shrdl	$6,%r13d,%r13d
 	andl	%r15d,%edi
 	xorl	%ecx,%r14d
 	addl	%r13d,%ebx
 	xorl	%edx,%edi
 	shrdl	$2,%r14d,%r14d
 	addl	%ebx,%r9d
 	addl	%edi,%ebx
 	movl	%r9d,%r13d
 	addl	%ebx,%r14d
 	shrdl	$14,%r13d,%r13d
 	movl	%r14d,%ebx
 	movl	%r10d,%r12d
 	shrdl	$9,%r14d,%r14d
 	xorl	%r9d,%r13d
 	xorl	%r11d,%r12d
 	shrdl	$5,%r13d,%r13d
 	xorl	%ebx,%r14d
 	andl	%r9d,%r12d
 	xorl	%r9d,%r13d
 	addl	60(%rsp),%eax
 	movl	%ebx,%edi
 	xorl	%r11d,%r12d
 	shrdl	$11,%r14d,%r14d
 	xorl	%ecx,%edi
 	addl	%r12d,%eax
 	shrdl	$6,%r13d,%r13d
 	andl	%edi,%r15d
 	xorl	%ebx,%r14d
 	addl	%r13d,%eax
 	xorl	%ecx,%r15d
 	shrdl	$2,%r14d,%r14d
 	addl	%eax,%r8d
 	addl	%r15d,%eax
 	movl	%r8d,%r13d
 	addl	%eax,%r14d
 	movq	64+0(%rsp),%rdi
 	movl	%r14d,%eax
 
 	addl	0(%rdi),%eax
 	leaq	64(%rsi),%rsi
 	addl	4(%rdi),%ebx
 	addl	8(%rdi),%ecx
 	addl	12(%rdi),%edx
 	addl	16(%rdi),%r8d
 	addl	20(%rdi),%r9d
 	addl	24(%rdi),%r10d
 	addl	28(%rdi),%r11d
 
 	cmpq	64+16(%rsp),%rsi
 
 	movl	%eax,0(%rdi)
 	movl	%ebx,4(%rdi)
 	movl	%ecx,8(%rdi)
 	movl	%edx,12(%rdi)
 	movl	%r8d,16(%rdi)
 	movl	%r9d,20(%rdi)
 	movl	%r10d,24(%rdi)
 	movl	%r11d,28(%rdi)
 	jb	.Lloop_avx
 
 	movq	88(%rsp),%rsi
 .cfi_def_cfa	%rsi,8
 	vzeroupper
 	movq	-48(%rsi),%r15
 .cfi_restore	%r15
 	movq	-40(%rsi),%r14
 .cfi_restore	%r14
 	movq	-32(%rsi),%r13
 .cfi_restore	%r13
 	movq	-24(%rsi),%r12
 .cfi_restore	%r12
 	movq	-16(%rsi),%rbp
 .cfi_restore	%rbp
 	movq	-8(%rsi),%rbx
 .cfi_restore	%rbx
 	leaq	(%rsi),%rsp
 .cfi_def_cfa_register	%rsp
 .Lepilogue_avx:
 	RET
 .cfi_endproc
 SET_SIZE(zfs_sha256_transform_avx)
+STACK_FRAME_NON_STANDARD zfs_sha256_transform_avx
 
 ENTRY_ALIGN(zfs_sha256_transform_avx2, 64)
 .cfi_startproc
 	ENDBR
 	movq	%rsp,%rax
 .cfi_def_cfa_register	%rax
 	pushq	%rbx
 .cfi_offset	%rbx,-16
 	pushq	%rbp
 .cfi_offset	%rbp,-24
 	pushq	%r12
 .cfi_offset	%r12,-32
 	pushq	%r13
 .cfi_offset	%r13,-40
 	pushq	%r14
 .cfi_offset	%r14,-48
 	pushq	%r15
 .cfi_offset	%r15,-56
 	subq	$544,%rsp
 	shlq	$4,%rdx
 	andq	$-1024,%rsp
 	leaq	(%rsi,%rdx,4),%rdx
 	addq	$448,%rsp
 	movq	%rdi,64+0(%rsp)
 	movq	%rsi,64+8(%rsp)
 	movq	%rdx,64+16(%rsp)
 	movq	%rax,88(%rsp)
 .cfi_escape	0x0f,0x06,0x77,0xd8,0x00,0x06,0x23,0x08
 .Lprologue_avx2:
 
 	vzeroupper
 	subq	$-64,%rsi
 	movl	0(%rdi),%eax
 	movq	%rsi,%r12
 	movl	4(%rdi),%ebx
 	cmpq	%rdx,%rsi
 	movl	8(%rdi),%ecx
 	cmoveq	%rsp,%r12
 	movl	12(%rdi),%edx
 	movl	16(%rdi),%r8d
 	movl	20(%rdi),%r9d
 	movl	24(%rdi),%r10d
 	movl	28(%rdi),%r11d
 	vmovdqa	K256+512+32(%rip),%ymm8
 	vmovdqa	K256+512+64(%rip),%ymm9
 	jmp	.Loop_avx2
 .balign	16
 .Loop_avx2:
 	vmovdqa	K256+512(%rip),%ymm7
 	vmovdqu	-64+0(%rsi),%xmm0
 	vmovdqu	-64+16(%rsi),%xmm1
 	vmovdqu	-64+32(%rsi),%xmm2
 	vmovdqu	-64+48(%rsi),%xmm3
 
 	vinserti128	$1,(%r12),%ymm0,%ymm0
 	vinserti128	$1,16(%r12),%ymm1,%ymm1
 	vpshufb	%ymm7,%ymm0,%ymm0
 	vinserti128	$1,32(%r12),%ymm2,%ymm2
 	vpshufb	%ymm7,%ymm1,%ymm1
 	vinserti128	$1,48(%r12),%ymm3,%ymm3
 
 	leaq	K256(%rip),%rbp
 	vpshufb	%ymm7,%ymm2,%ymm2
 	vpaddd	0(%rbp),%ymm0,%ymm4
 	vpshufb	%ymm7,%ymm3,%ymm3
 	vpaddd	32(%rbp),%ymm1,%ymm5
 	vpaddd	64(%rbp),%ymm2,%ymm6
 	vpaddd	96(%rbp),%ymm3,%ymm7
 	vmovdqa	%ymm4,0(%rsp)
 	xorl	%r14d,%r14d
 	vmovdqa	%ymm5,32(%rsp)
 
 	movq	88(%rsp),%rdi
 .cfi_def_cfa	%rdi,8
 	leaq	-64(%rsp),%rsp
 
 
 
 	movq	%rdi,-8(%rsp)
 .cfi_escape	0x0f,0x05,0x77,0x78,0x06,0x23,0x08
 	movl	%ebx,%edi
 	vmovdqa	%ymm6,0(%rsp)
 	xorl	%ecx,%edi
 	vmovdqa	%ymm7,32(%rsp)
 	movl	%r9d,%r12d
 	subq	$-32*4,%rbp
 	jmp	.Lavx2_00_47
 
 .balign	16
 .Lavx2_00_47:
 	leaq	-64(%rsp),%rsp
 .cfi_escape	0x0f,0x05,0x77,0x38,0x06,0x23,0x08
 
 	pushq	64-8(%rsp)
 .cfi_escape	0x0f,0x05,0x77,0x00,0x06,0x23,0x08
 	leaq	8(%rsp),%rsp
 .cfi_escape	0x0f,0x05,0x77,0x78,0x06,0x23,0x08
 	vpalignr	$4,%ymm0,%ymm1,%ymm4
 	addl	0+128(%rsp),%r11d
 	andl	%r8d,%r12d
 	rorxl	$25,%r8d,%r13d
 	vpalignr	$4,%ymm2,%ymm3,%ymm7
 	rorxl	$11,%r8d,%r15d
 	leal	(%rax,%r14,1),%eax
 	leal	(%r11,%r12,1),%r11d
 	vpsrld	$7,%ymm4,%ymm6
 	andnl	%r10d,%r8d,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%r8d,%r14d
 	vpaddd	%ymm7,%ymm0,%ymm0
 	leal	(%r11,%r12,1),%r11d
 	xorl	%r14d,%r13d
 	movl	%eax,%r15d
 	vpsrld	$3,%ymm4,%ymm7
 	rorxl	$22,%eax,%r12d
 	leal	(%r11,%r13,1),%r11d
 	xorl	%ebx,%r15d
 	vpslld	$14,%ymm4,%ymm5
 	rorxl	$13,%eax,%r14d
 	rorxl	$2,%eax,%r13d
 	leal	(%rdx,%r11,1),%edx
 	vpxor	%ymm6,%ymm7,%ymm4
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%ebx,%edi
 	vpshufd	$250,%ymm3,%ymm7
 	xorl	%r13d,%r14d
 	leal	(%r11,%rdi,1),%r11d
 	movl	%r8d,%r12d
 	vpsrld	$11,%ymm6,%ymm6
 	addl	4+128(%rsp),%r10d
 	andl	%edx,%r12d
 	rorxl	$25,%edx,%r13d
 	vpxor	%ymm5,%ymm4,%ymm4
 	rorxl	$11,%edx,%edi
 	leal	(%r11,%r14,1),%r11d
 	leal	(%r10,%r12,1),%r10d
 	vpslld	$11,%ymm5,%ymm5
 	andnl	%r9d,%edx,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%edx,%r14d
 	vpxor	%ymm6,%ymm4,%ymm4
 	leal	(%r10,%r12,1),%r10d
 	xorl	%r14d,%r13d
 	movl	%r11d,%edi
 	vpsrld	$10,%ymm7,%ymm6
 	rorxl	$22,%r11d,%r12d
 	leal	(%r10,%r13,1),%r10d
 	xorl	%eax,%edi
 	vpxor	%ymm5,%ymm4,%ymm4
 	rorxl	$13,%r11d,%r14d
 	rorxl	$2,%r11d,%r13d
 	leal	(%rcx,%r10,1),%ecx
 	vpsrlq	$17,%ymm7,%ymm7
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%eax,%r15d
 	vpaddd	%ymm4,%ymm0,%ymm0
 	xorl	%r13d,%r14d
 	leal	(%r10,%r15,1),%r10d
 	movl	%edx,%r12d
 	vpxor	%ymm7,%ymm6,%ymm6
 	addl	8+128(%rsp),%r9d
 	andl	%ecx,%r12d
 	rorxl	$25,%ecx,%r13d
 	vpsrlq	$2,%ymm7,%ymm7
 	rorxl	$11,%ecx,%r15d
 	leal	(%r10,%r14,1),%r10d
 	leal	(%r9,%r12,1),%r9d
 	vpxor	%ymm7,%ymm6,%ymm6
 	andnl	%r8d,%ecx,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%ecx,%r14d
 	vpshufb	%ymm8,%ymm6,%ymm6
 	leal	(%r9,%r12,1),%r9d
 	xorl	%r14d,%r13d
 	movl	%r10d,%r15d
 	vpaddd	%ymm6,%ymm0,%ymm0
 	rorxl	$22,%r10d,%r12d
 	leal	(%r9,%r13,1),%r9d
 	xorl	%r11d,%r15d
 	vpshufd	$80,%ymm0,%ymm7
 	rorxl	$13,%r10d,%r14d
 	rorxl	$2,%r10d,%r13d
 	leal	(%rbx,%r9,1),%ebx
 	vpsrld	$10,%ymm7,%ymm6
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%r11d,%edi
 	vpsrlq	$17,%ymm7,%ymm7
 	xorl	%r13d,%r14d
 	leal	(%r9,%rdi,1),%r9d
 	movl	%ecx,%r12d
 	vpxor	%ymm7,%ymm6,%ymm6
 	addl	12+128(%rsp),%r8d
 	andl	%ebx,%r12d
 	rorxl	$25,%ebx,%r13d
 	vpsrlq	$2,%ymm7,%ymm7
 	rorxl	$11,%ebx,%edi
 	leal	(%r9,%r14,1),%r9d
 	leal	(%r8,%r12,1),%r8d
 	vpxor	%ymm7,%ymm6,%ymm6
 	andnl	%edx,%ebx,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%ebx,%r14d
 	vpshufb	%ymm9,%ymm6,%ymm6
 	leal	(%r8,%r12,1),%r8d
 	xorl	%r14d,%r13d
 	movl	%r9d,%edi
 	vpaddd	%ymm6,%ymm0,%ymm0
 	rorxl	$22,%r9d,%r12d
 	leal	(%r8,%r13,1),%r8d
 	xorl	%r10d,%edi
 	vpaddd	0(%rbp),%ymm0,%ymm6
 	rorxl	$13,%r9d,%r14d
 	rorxl	$2,%r9d,%r13d
 	leal	(%rax,%r8,1),%eax
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%r10d,%r15d
 	xorl	%r13d,%r14d
 	leal	(%r8,%r15,1),%r8d
 	movl	%ebx,%r12d
 	vmovdqa	%ymm6,0(%rsp)
 	vpalignr	$4,%ymm1,%ymm2,%ymm4
 	addl	32+128(%rsp),%edx
 	andl	%eax,%r12d
 	rorxl	$25,%eax,%r13d
 	vpalignr	$4,%ymm3,%ymm0,%ymm7
 	rorxl	$11,%eax,%r15d
 	leal	(%r8,%r14,1),%r8d
 	leal	(%rdx,%r12,1),%edx
 	vpsrld	$7,%ymm4,%ymm6
 	andnl	%ecx,%eax,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%eax,%r14d
 	vpaddd	%ymm7,%ymm1,%ymm1
 	leal	(%rdx,%r12,1),%edx
 	xorl	%r14d,%r13d
 	movl	%r8d,%r15d
 	vpsrld	$3,%ymm4,%ymm7
 	rorxl	$22,%r8d,%r12d
 	leal	(%rdx,%r13,1),%edx
 	xorl	%r9d,%r15d
 	vpslld	$14,%ymm4,%ymm5
 	rorxl	$13,%r8d,%r14d
 	rorxl	$2,%r8d,%r13d
 	leal	(%r11,%rdx,1),%r11d
 	vpxor	%ymm6,%ymm7,%ymm4
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%r9d,%edi
 	vpshufd	$250,%ymm0,%ymm7
 	xorl	%r13d,%r14d
 	leal	(%rdx,%rdi,1),%edx
 	movl	%eax,%r12d
 	vpsrld	$11,%ymm6,%ymm6
 	addl	36+128(%rsp),%ecx
 	andl	%r11d,%r12d
 	rorxl	$25,%r11d,%r13d
 	vpxor	%ymm5,%ymm4,%ymm4
 	rorxl	$11,%r11d,%edi
 	leal	(%rdx,%r14,1),%edx
 	leal	(%rcx,%r12,1),%ecx
 	vpslld	$11,%ymm5,%ymm5
 	andnl	%ebx,%r11d,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%r11d,%r14d
 	vpxor	%ymm6,%ymm4,%ymm4
 	leal	(%rcx,%r12,1),%ecx
 	xorl	%r14d,%r13d
 	movl	%edx,%edi
 	vpsrld	$10,%ymm7,%ymm6
 	rorxl	$22,%edx,%r12d
 	leal	(%rcx,%r13,1),%ecx
 	xorl	%r8d,%edi
 	vpxor	%ymm5,%ymm4,%ymm4
 	rorxl	$13,%edx,%r14d
 	rorxl	$2,%edx,%r13d
 	leal	(%r10,%rcx,1),%r10d
 	vpsrlq	$17,%ymm7,%ymm7
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%r8d,%r15d
 	vpaddd	%ymm4,%ymm1,%ymm1
 	xorl	%r13d,%r14d
 	leal	(%rcx,%r15,1),%ecx
 	movl	%r11d,%r12d
 	vpxor	%ymm7,%ymm6,%ymm6
 	addl	40+128(%rsp),%ebx
 	andl	%r10d,%r12d
 	rorxl	$25,%r10d,%r13d
 	vpsrlq	$2,%ymm7,%ymm7
 	rorxl	$11,%r10d,%r15d
 	leal	(%rcx,%r14,1),%ecx
 	leal	(%rbx,%r12,1),%ebx
 	vpxor	%ymm7,%ymm6,%ymm6
 	andnl	%eax,%r10d,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%r10d,%r14d
 	vpshufb	%ymm8,%ymm6,%ymm6
 	leal	(%rbx,%r12,1),%ebx
 	xorl	%r14d,%r13d
 	movl	%ecx,%r15d
 	vpaddd	%ymm6,%ymm1,%ymm1
 	rorxl	$22,%ecx,%r12d
 	leal	(%rbx,%r13,1),%ebx
 	xorl	%edx,%r15d
 	vpshufd	$80,%ymm1,%ymm7
 	rorxl	$13,%ecx,%r14d
 	rorxl	$2,%ecx,%r13d
 	leal	(%r9,%rbx,1),%r9d
 	vpsrld	$10,%ymm7,%ymm6
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%edx,%edi
 	vpsrlq	$17,%ymm7,%ymm7
 	xorl	%r13d,%r14d
 	leal	(%rbx,%rdi,1),%ebx
 	movl	%r10d,%r12d
 	vpxor	%ymm7,%ymm6,%ymm6
 	addl	44+128(%rsp),%eax
 	andl	%r9d,%r12d
 	rorxl	$25,%r9d,%r13d
 	vpsrlq	$2,%ymm7,%ymm7
 	rorxl	$11,%r9d,%edi
 	leal	(%rbx,%r14,1),%ebx
 	leal	(%rax,%r12,1),%eax
 	vpxor	%ymm7,%ymm6,%ymm6
 	andnl	%r11d,%r9d,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%r9d,%r14d
 	vpshufb	%ymm9,%ymm6,%ymm6
 	leal	(%rax,%r12,1),%eax
 	xorl	%r14d,%r13d
 	movl	%ebx,%edi
 	vpaddd	%ymm6,%ymm1,%ymm1
 	rorxl	$22,%ebx,%r12d
 	leal	(%rax,%r13,1),%eax
 	xorl	%ecx,%edi
 	vpaddd	32(%rbp),%ymm1,%ymm6
 	rorxl	$13,%ebx,%r14d
 	rorxl	$2,%ebx,%r13d
 	leal	(%r8,%rax,1),%r8d
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%ecx,%r15d
 	xorl	%r13d,%r14d
 	leal	(%rax,%r15,1),%eax
 	movl	%r9d,%r12d
 	vmovdqa	%ymm6,32(%rsp)
 	leaq	-64(%rsp),%rsp
 .cfi_escape	0x0f,0x05,0x77,0x38,0x06,0x23,0x08
 
 	pushq	64-8(%rsp)
 .cfi_escape	0x0f,0x05,0x77,0x00,0x06,0x23,0x08
 	leaq	8(%rsp),%rsp
 .cfi_escape	0x0f,0x05,0x77,0x78,0x06,0x23,0x08
 	vpalignr	$4,%ymm2,%ymm3,%ymm4
 	addl	0+128(%rsp),%r11d
 	andl	%r8d,%r12d
 	rorxl	$25,%r8d,%r13d
 	vpalignr	$4,%ymm0,%ymm1,%ymm7
 	rorxl	$11,%r8d,%r15d
 	leal	(%rax,%r14,1),%eax
 	leal	(%r11,%r12,1),%r11d
 	vpsrld	$7,%ymm4,%ymm6
 	andnl	%r10d,%r8d,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%r8d,%r14d
 	vpaddd	%ymm7,%ymm2,%ymm2
 	leal	(%r11,%r12,1),%r11d
 	xorl	%r14d,%r13d
 	movl	%eax,%r15d
 	vpsrld	$3,%ymm4,%ymm7
 	rorxl	$22,%eax,%r12d
 	leal	(%r11,%r13,1),%r11d
 	xorl	%ebx,%r15d
 	vpslld	$14,%ymm4,%ymm5
 	rorxl	$13,%eax,%r14d
 	rorxl	$2,%eax,%r13d
 	leal	(%rdx,%r11,1),%edx
 	vpxor	%ymm6,%ymm7,%ymm4
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%ebx,%edi
 	vpshufd	$250,%ymm1,%ymm7
 	xorl	%r13d,%r14d
 	leal	(%r11,%rdi,1),%r11d
 	movl	%r8d,%r12d
 	vpsrld	$11,%ymm6,%ymm6
 	addl	4+128(%rsp),%r10d
 	andl	%edx,%r12d
 	rorxl	$25,%edx,%r13d
 	vpxor	%ymm5,%ymm4,%ymm4
 	rorxl	$11,%edx,%edi
 	leal	(%r11,%r14,1),%r11d
 	leal	(%r10,%r12,1),%r10d
 	vpslld	$11,%ymm5,%ymm5
 	andnl	%r9d,%edx,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%edx,%r14d
 	vpxor	%ymm6,%ymm4,%ymm4
 	leal	(%r10,%r12,1),%r10d
 	xorl	%r14d,%r13d
 	movl	%r11d,%edi
 	vpsrld	$10,%ymm7,%ymm6
 	rorxl	$22,%r11d,%r12d
 	leal	(%r10,%r13,1),%r10d
 	xorl	%eax,%edi
 	vpxor	%ymm5,%ymm4,%ymm4
 	rorxl	$13,%r11d,%r14d
 	rorxl	$2,%r11d,%r13d
 	leal	(%rcx,%r10,1),%ecx
 	vpsrlq	$17,%ymm7,%ymm7
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%eax,%r15d
 	vpaddd	%ymm4,%ymm2,%ymm2
 	xorl	%r13d,%r14d
 	leal	(%r10,%r15,1),%r10d
 	movl	%edx,%r12d
 	vpxor	%ymm7,%ymm6,%ymm6
 	addl	8+128(%rsp),%r9d
 	andl	%ecx,%r12d
 	rorxl	$25,%ecx,%r13d
 	vpsrlq	$2,%ymm7,%ymm7
 	rorxl	$11,%ecx,%r15d
 	leal	(%r10,%r14,1),%r10d
 	leal	(%r9,%r12,1),%r9d
 	vpxor	%ymm7,%ymm6,%ymm6
 	andnl	%r8d,%ecx,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%ecx,%r14d
 	vpshufb	%ymm8,%ymm6,%ymm6
 	leal	(%r9,%r12,1),%r9d
 	xorl	%r14d,%r13d
 	movl	%r10d,%r15d
 	vpaddd	%ymm6,%ymm2,%ymm2
 	rorxl	$22,%r10d,%r12d
 	leal	(%r9,%r13,1),%r9d
 	xorl	%r11d,%r15d
 	vpshufd	$80,%ymm2,%ymm7
 	rorxl	$13,%r10d,%r14d
 	rorxl	$2,%r10d,%r13d
 	leal	(%rbx,%r9,1),%ebx
 	vpsrld	$10,%ymm7,%ymm6
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%r11d,%edi
 	vpsrlq	$17,%ymm7,%ymm7
 	xorl	%r13d,%r14d
 	leal	(%r9,%rdi,1),%r9d
 	movl	%ecx,%r12d
 	vpxor	%ymm7,%ymm6,%ymm6
 	addl	12+128(%rsp),%r8d
 	andl	%ebx,%r12d
 	rorxl	$25,%ebx,%r13d
 	vpsrlq	$2,%ymm7,%ymm7
 	rorxl	$11,%ebx,%edi
 	leal	(%r9,%r14,1),%r9d
 	leal	(%r8,%r12,1),%r8d
 	vpxor	%ymm7,%ymm6,%ymm6
 	andnl	%edx,%ebx,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%ebx,%r14d
 	vpshufb	%ymm9,%ymm6,%ymm6
 	leal	(%r8,%r12,1),%r8d
 	xorl	%r14d,%r13d
 	movl	%r9d,%edi
 	vpaddd	%ymm6,%ymm2,%ymm2
 	rorxl	$22,%r9d,%r12d
 	leal	(%r8,%r13,1),%r8d
 	xorl	%r10d,%edi
 	vpaddd	64(%rbp),%ymm2,%ymm6
 	rorxl	$13,%r9d,%r14d
 	rorxl	$2,%r9d,%r13d
 	leal	(%rax,%r8,1),%eax
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%r10d,%r15d
 	xorl	%r13d,%r14d
 	leal	(%r8,%r15,1),%r8d
 	movl	%ebx,%r12d
 	vmovdqa	%ymm6,0(%rsp)
 	vpalignr	$4,%ymm3,%ymm0,%ymm4
 	addl	32+128(%rsp),%edx
 	andl	%eax,%r12d
 	rorxl	$25,%eax,%r13d
 	vpalignr	$4,%ymm1,%ymm2,%ymm7
 	rorxl	$11,%eax,%r15d
 	leal	(%r8,%r14,1),%r8d
 	leal	(%rdx,%r12,1),%edx
 	vpsrld	$7,%ymm4,%ymm6
 	andnl	%ecx,%eax,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%eax,%r14d
 	vpaddd	%ymm7,%ymm3,%ymm3
 	leal	(%rdx,%r12,1),%edx
 	xorl	%r14d,%r13d
 	movl	%r8d,%r15d
 	vpsrld	$3,%ymm4,%ymm7
 	rorxl	$22,%r8d,%r12d
 	leal	(%rdx,%r13,1),%edx
 	xorl	%r9d,%r15d
 	vpslld	$14,%ymm4,%ymm5
 	rorxl	$13,%r8d,%r14d
 	rorxl	$2,%r8d,%r13d
 	leal	(%r11,%rdx,1),%r11d
 	vpxor	%ymm6,%ymm7,%ymm4
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%r9d,%edi
 	vpshufd	$250,%ymm2,%ymm7
 	xorl	%r13d,%r14d
 	leal	(%rdx,%rdi,1),%edx
 	movl	%eax,%r12d
 	vpsrld	$11,%ymm6,%ymm6
 	addl	36+128(%rsp),%ecx
 	andl	%r11d,%r12d
 	rorxl	$25,%r11d,%r13d
 	vpxor	%ymm5,%ymm4,%ymm4
 	rorxl	$11,%r11d,%edi
 	leal	(%rdx,%r14,1),%edx
 	leal	(%rcx,%r12,1),%ecx
 	vpslld	$11,%ymm5,%ymm5
 	andnl	%ebx,%r11d,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%r11d,%r14d
 	vpxor	%ymm6,%ymm4,%ymm4
 	leal	(%rcx,%r12,1),%ecx
 	xorl	%r14d,%r13d
 	movl	%edx,%edi
 	vpsrld	$10,%ymm7,%ymm6
 	rorxl	$22,%edx,%r12d
 	leal	(%rcx,%r13,1),%ecx
 	xorl	%r8d,%edi
 	vpxor	%ymm5,%ymm4,%ymm4
 	rorxl	$13,%edx,%r14d
 	rorxl	$2,%edx,%r13d
 	leal	(%r10,%rcx,1),%r10d
 	vpsrlq	$17,%ymm7,%ymm7
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%r8d,%r15d
 	vpaddd	%ymm4,%ymm3,%ymm3
 	xorl	%r13d,%r14d
 	leal	(%rcx,%r15,1),%ecx
 	movl	%r11d,%r12d
 	vpxor	%ymm7,%ymm6,%ymm6
 	addl	40+128(%rsp),%ebx
 	andl	%r10d,%r12d
 	rorxl	$25,%r10d,%r13d
 	vpsrlq	$2,%ymm7,%ymm7
 	rorxl	$11,%r10d,%r15d
 	leal	(%rcx,%r14,1),%ecx
 	leal	(%rbx,%r12,1),%ebx
 	vpxor	%ymm7,%ymm6,%ymm6
 	andnl	%eax,%r10d,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%r10d,%r14d
 	vpshufb	%ymm8,%ymm6,%ymm6
 	leal	(%rbx,%r12,1),%ebx
 	xorl	%r14d,%r13d
 	movl	%ecx,%r15d
 	vpaddd	%ymm6,%ymm3,%ymm3
 	rorxl	$22,%ecx,%r12d
 	leal	(%rbx,%r13,1),%ebx
 	xorl	%edx,%r15d
 	vpshufd	$80,%ymm3,%ymm7
 	rorxl	$13,%ecx,%r14d
 	rorxl	$2,%ecx,%r13d
 	leal	(%r9,%rbx,1),%r9d
 	vpsrld	$10,%ymm7,%ymm6
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%edx,%edi
 	vpsrlq	$17,%ymm7,%ymm7
 	xorl	%r13d,%r14d
 	leal	(%rbx,%rdi,1),%ebx
 	movl	%r10d,%r12d
 	vpxor	%ymm7,%ymm6,%ymm6
 	addl	44+128(%rsp),%eax
 	andl	%r9d,%r12d
 	rorxl	$25,%r9d,%r13d
 	vpsrlq	$2,%ymm7,%ymm7
 	rorxl	$11,%r9d,%edi
 	leal	(%rbx,%r14,1),%ebx
 	leal	(%rax,%r12,1),%eax
 	vpxor	%ymm7,%ymm6,%ymm6
 	andnl	%r11d,%r9d,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%r9d,%r14d
 	vpshufb	%ymm9,%ymm6,%ymm6
 	leal	(%rax,%r12,1),%eax
 	xorl	%r14d,%r13d
 	movl	%ebx,%edi
 	vpaddd	%ymm6,%ymm3,%ymm3
 	rorxl	$22,%ebx,%r12d
 	leal	(%rax,%r13,1),%eax
 	xorl	%ecx,%edi
 	vpaddd	96(%rbp),%ymm3,%ymm6
 	rorxl	$13,%ebx,%r14d
 	rorxl	$2,%ebx,%r13d
 	leal	(%r8,%rax,1),%r8d
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%ecx,%r15d
 	xorl	%r13d,%r14d
 	leal	(%rax,%r15,1),%eax
 	movl	%r9d,%r12d
 	vmovdqa	%ymm6,32(%rsp)
 	leaq	128(%rbp),%rbp
 	cmpb	$0,3(%rbp)
 	jne	.Lavx2_00_47
 	addl	0+64(%rsp),%r11d
 	andl	%r8d,%r12d
 	rorxl	$25,%r8d,%r13d
 	rorxl	$11,%r8d,%r15d
 	leal	(%rax,%r14,1),%eax
 	leal	(%r11,%r12,1),%r11d
 	andnl	%r10d,%r8d,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%r8d,%r14d
 	leal	(%r11,%r12,1),%r11d
 	xorl	%r14d,%r13d
 	movl	%eax,%r15d
 	rorxl	$22,%eax,%r12d
 	leal	(%r11,%r13,1),%r11d
 	xorl	%ebx,%r15d
 	rorxl	$13,%eax,%r14d
 	rorxl	$2,%eax,%r13d
 	leal	(%rdx,%r11,1),%edx
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%ebx,%edi
 	xorl	%r13d,%r14d
 	leal	(%r11,%rdi,1),%r11d
 	movl	%r8d,%r12d
 	addl	4+64(%rsp),%r10d
 	andl	%edx,%r12d
 	rorxl	$25,%edx,%r13d
 	rorxl	$11,%edx,%edi
 	leal	(%r11,%r14,1),%r11d
 	leal	(%r10,%r12,1),%r10d
 	andnl	%r9d,%edx,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%edx,%r14d
 	leal	(%r10,%r12,1),%r10d
 	xorl	%r14d,%r13d
 	movl	%r11d,%edi
 	rorxl	$22,%r11d,%r12d
 	leal	(%r10,%r13,1),%r10d
 	xorl	%eax,%edi
 	rorxl	$13,%r11d,%r14d
 	rorxl	$2,%r11d,%r13d
 	leal	(%rcx,%r10,1),%ecx
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%eax,%r15d
 	xorl	%r13d,%r14d
 	leal	(%r10,%r15,1),%r10d
 	movl	%edx,%r12d
 	addl	8+64(%rsp),%r9d
 	andl	%ecx,%r12d
 	rorxl	$25,%ecx,%r13d
 	rorxl	$11,%ecx,%r15d
 	leal	(%r10,%r14,1),%r10d
 	leal	(%r9,%r12,1),%r9d
 	andnl	%r8d,%ecx,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%ecx,%r14d
 	leal	(%r9,%r12,1),%r9d
 	xorl	%r14d,%r13d
 	movl	%r10d,%r15d
 	rorxl	$22,%r10d,%r12d
 	leal	(%r9,%r13,1),%r9d
 	xorl	%r11d,%r15d
 	rorxl	$13,%r10d,%r14d
 	rorxl	$2,%r10d,%r13d
 	leal	(%rbx,%r9,1),%ebx
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%r11d,%edi
 	xorl	%r13d,%r14d
 	leal	(%r9,%rdi,1),%r9d
 	movl	%ecx,%r12d
 	addl	12+64(%rsp),%r8d
 	andl	%ebx,%r12d
 	rorxl	$25,%ebx,%r13d
 	rorxl	$11,%ebx,%edi
 	leal	(%r9,%r14,1),%r9d
 	leal	(%r8,%r12,1),%r8d
 	andnl	%edx,%ebx,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%ebx,%r14d
 	leal	(%r8,%r12,1),%r8d
 	xorl	%r14d,%r13d
 	movl	%r9d,%edi
 	rorxl	$22,%r9d,%r12d
 	leal	(%r8,%r13,1),%r8d
 	xorl	%r10d,%edi
 	rorxl	$13,%r9d,%r14d
 	rorxl	$2,%r9d,%r13d
 	leal	(%rax,%r8,1),%eax
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%r10d,%r15d
 	xorl	%r13d,%r14d
 	leal	(%r8,%r15,1),%r8d
 	movl	%ebx,%r12d
 	addl	32+64(%rsp),%edx
 	andl	%eax,%r12d
 	rorxl	$25,%eax,%r13d
 	rorxl	$11,%eax,%r15d
 	leal	(%r8,%r14,1),%r8d
 	leal	(%rdx,%r12,1),%edx
 	andnl	%ecx,%eax,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%eax,%r14d
 	leal	(%rdx,%r12,1),%edx
 	xorl	%r14d,%r13d
 	movl	%r8d,%r15d
 	rorxl	$22,%r8d,%r12d
 	leal	(%rdx,%r13,1),%edx
 	xorl	%r9d,%r15d
 	rorxl	$13,%r8d,%r14d
 	rorxl	$2,%r8d,%r13d
 	leal	(%r11,%rdx,1),%r11d
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%r9d,%edi
 	xorl	%r13d,%r14d
 	leal	(%rdx,%rdi,1),%edx
 	movl	%eax,%r12d
 	addl	36+64(%rsp),%ecx
 	andl	%r11d,%r12d
 	rorxl	$25,%r11d,%r13d
 	rorxl	$11,%r11d,%edi
 	leal	(%rdx,%r14,1),%edx
 	leal	(%rcx,%r12,1),%ecx
 	andnl	%ebx,%r11d,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%r11d,%r14d
 	leal	(%rcx,%r12,1),%ecx
 	xorl	%r14d,%r13d
 	movl	%edx,%edi
 	rorxl	$22,%edx,%r12d
 	leal	(%rcx,%r13,1),%ecx
 	xorl	%r8d,%edi
 	rorxl	$13,%edx,%r14d
 	rorxl	$2,%edx,%r13d
 	leal	(%r10,%rcx,1),%r10d
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%r8d,%r15d
 	xorl	%r13d,%r14d
 	leal	(%rcx,%r15,1),%ecx
 	movl	%r11d,%r12d
 	addl	40+64(%rsp),%ebx
 	andl	%r10d,%r12d
 	rorxl	$25,%r10d,%r13d
 	rorxl	$11,%r10d,%r15d
 	leal	(%rcx,%r14,1),%ecx
 	leal	(%rbx,%r12,1),%ebx
 	andnl	%eax,%r10d,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%r10d,%r14d
 	leal	(%rbx,%r12,1),%ebx
 	xorl	%r14d,%r13d
 	movl	%ecx,%r15d
 	rorxl	$22,%ecx,%r12d
 	leal	(%rbx,%r13,1),%ebx
 	xorl	%edx,%r15d
 	rorxl	$13,%ecx,%r14d
 	rorxl	$2,%ecx,%r13d
 	leal	(%r9,%rbx,1),%r9d
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%edx,%edi
 	xorl	%r13d,%r14d
 	leal	(%rbx,%rdi,1),%ebx
 	movl	%r10d,%r12d
 	addl	44+64(%rsp),%eax
 	andl	%r9d,%r12d
 	rorxl	$25,%r9d,%r13d
 	rorxl	$11,%r9d,%edi
 	leal	(%rbx,%r14,1),%ebx
 	leal	(%rax,%r12,1),%eax
 	andnl	%r11d,%r9d,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%r9d,%r14d
 	leal	(%rax,%r12,1),%eax
 	xorl	%r14d,%r13d
 	movl	%ebx,%edi
 	rorxl	$22,%ebx,%r12d
 	leal	(%rax,%r13,1),%eax
 	xorl	%ecx,%edi
 	rorxl	$13,%ebx,%r14d
 	rorxl	$2,%ebx,%r13d
 	leal	(%r8,%rax,1),%r8d
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%ecx,%r15d
 	xorl	%r13d,%r14d
 	leal	(%rax,%r15,1),%eax
 	movl	%r9d,%r12d
 	addl	0(%rsp),%r11d
 	andl	%r8d,%r12d
 	rorxl	$25,%r8d,%r13d
 	rorxl	$11,%r8d,%r15d
 	leal	(%rax,%r14,1),%eax
 	leal	(%r11,%r12,1),%r11d
 	andnl	%r10d,%r8d,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%r8d,%r14d
 	leal	(%r11,%r12,1),%r11d
 	xorl	%r14d,%r13d
 	movl	%eax,%r15d
 	rorxl	$22,%eax,%r12d
 	leal	(%r11,%r13,1),%r11d
 	xorl	%ebx,%r15d
 	rorxl	$13,%eax,%r14d
 	rorxl	$2,%eax,%r13d
 	leal	(%rdx,%r11,1),%edx
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%ebx,%edi
 	xorl	%r13d,%r14d
 	leal	(%r11,%rdi,1),%r11d
 	movl	%r8d,%r12d
 	addl	4(%rsp),%r10d
 	andl	%edx,%r12d
 	rorxl	$25,%edx,%r13d
 	rorxl	$11,%edx,%edi
 	leal	(%r11,%r14,1),%r11d
 	leal	(%r10,%r12,1),%r10d
 	andnl	%r9d,%edx,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%edx,%r14d
 	leal	(%r10,%r12,1),%r10d
 	xorl	%r14d,%r13d
 	movl	%r11d,%edi
 	rorxl	$22,%r11d,%r12d
 	leal	(%r10,%r13,1),%r10d
 	xorl	%eax,%edi
 	rorxl	$13,%r11d,%r14d
 	rorxl	$2,%r11d,%r13d
 	leal	(%rcx,%r10,1),%ecx
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%eax,%r15d
 	xorl	%r13d,%r14d
 	leal	(%r10,%r15,1),%r10d
 	movl	%edx,%r12d
 	addl	8(%rsp),%r9d
 	andl	%ecx,%r12d
 	rorxl	$25,%ecx,%r13d
 	rorxl	$11,%ecx,%r15d
 	leal	(%r10,%r14,1),%r10d
 	leal	(%r9,%r12,1),%r9d
 	andnl	%r8d,%ecx,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%ecx,%r14d
 	leal	(%r9,%r12,1),%r9d
 	xorl	%r14d,%r13d
 	movl	%r10d,%r15d
 	rorxl	$22,%r10d,%r12d
 	leal	(%r9,%r13,1),%r9d
 	xorl	%r11d,%r15d
 	rorxl	$13,%r10d,%r14d
 	rorxl	$2,%r10d,%r13d
 	leal	(%rbx,%r9,1),%ebx
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%r11d,%edi
 	xorl	%r13d,%r14d
 	leal	(%r9,%rdi,1),%r9d
 	movl	%ecx,%r12d
 	addl	12(%rsp),%r8d
 	andl	%ebx,%r12d
 	rorxl	$25,%ebx,%r13d
 	rorxl	$11,%ebx,%edi
 	leal	(%r9,%r14,1),%r9d
 	leal	(%r8,%r12,1),%r8d
 	andnl	%edx,%ebx,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%ebx,%r14d
 	leal	(%r8,%r12,1),%r8d
 	xorl	%r14d,%r13d
 	movl	%r9d,%edi
 	rorxl	$22,%r9d,%r12d
 	leal	(%r8,%r13,1),%r8d
 	xorl	%r10d,%edi
 	rorxl	$13,%r9d,%r14d
 	rorxl	$2,%r9d,%r13d
 	leal	(%rax,%r8,1),%eax
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%r10d,%r15d
 	xorl	%r13d,%r14d
 	leal	(%r8,%r15,1),%r8d
 	movl	%ebx,%r12d
 	addl	32(%rsp),%edx
 	andl	%eax,%r12d
 	rorxl	$25,%eax,%r13d
 	rorxl	$11,%eax,%r15d
 	leal	(%r8,%r14,1),%r8d
 	leal	(%rdx,%r12,1),%edx
 	andnl	%ecx,%eax,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%eax,%r14d
 	leal	(%rdx,%r12,1),%edx
 	xorl	%r14d,%r13d
 	movl	%r8d,%r15d
 	rorxl	$22,%r8d,%r12d
 	leal	(%rdx,%r13,1),%edx
 	xorl	%r9d,%r15d
 	rorxl	$13,%r8d,%r14d
 	rorxl	$2,%r8d,%r13d
 	leal	(%r11,%rdx,1),%r11d
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%r9d,%edi
 	xorl	%r13d,%r14d
 	leal	(%rdx,%rdi,1),%edx
 	movl	%eax,%r12d
 	addl	36(%rsp),%ecx
 	andl	%r11d,%r12d
 	rorxl	$25,%r11d,%r13d
 	rorxl	$11,%r11d,%edi
 	leal	(%rdx,%r14,1),%edx
 	leal	(%rcx,%r12,1),%ecx
 	andnl	%ebx,%r11d,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%r11d,%r14d
 	leal	(%rcx,%r12,1),%ecx
 	xorl	%r14d,%r13d
 	movl	%edx,%edi
 	rorxl	$22,%edx,%r12d
 	leal	(%rcx,%r13,1),%ecx
 	xorl	%r8d,%edi
 	rorxl	$13,%edx,%r14d
 	rorxl	$2,%edx,%r13d
 	leal	(%r10,%rcx,1),%r10d
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%r8d,%r15d
 	xorl	%r13d,%r14d
 	leal	(%rcx,%r15,1),%ecx
 	movl	%r11d,%r12d
 	addl	40(%rsp),%ebx
 	andl	%r10d,%r12d
 	rorxl	$25,%r10d,%r13d
 	rorxl	$11,%r10d,%r15d
 	leal	(%rcx,%r14,1),%ecx
 	leal	(%rbx,%r12,1),%ebx
 	andnl	%eax,%r10d,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%r10d,%r14d
 	leal	(%rbx,%r12,1),%ebx
 	xorl	%r14d,%r13d
 	movl	%ecx,%r15d
 	rorxl	$22,%ecx,%r12d
 	leal	(%rbx,%r13,1),%ebx
 	xorl	%edx,%r15d
 	rorxl	$13,%ecx,%r14d
 	rorxl	$2,%ecx,%r13d
 	leal	(%r9,%rbx,1),%r9d
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%edx,%edi
 	xorl	%r13d,%r14d
 	leal	(%rbx,%rdi,1),%ebx
 	movl	%r10d,%r12d
 	addl	44(%rsp),%eax
 	andl	%r9d,%r12d
 	rorxl	$25,%r9d,%r13d
 	rorxl	$11,%r9d,%edi
 	leal	(%rbx,%r14,1),%ebx
 	leal	(%rax,%r12,1),%eax
 	andnl	%r11d,%r9d,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%r9d,%r14d
 	leal	(%rax,%r12,1),%eax
 	xorl	%r14d,%r13d
 	movl	%ebx,%edi
 	rorxl	$22,%ebx,%r12d
 	leal	(%rax,%r13,1),%eax
 	xorl	%ecx,%edi
 	rorxl	$13,%ebx,%r14d
 	rorxl	$2,%ebx,%r13d
 	leal	(%r8,%rax,1),%r8d
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%ecx,%r15d
 	xorl	%r13d,%r14d
 	leal	(%rax,%r15,1),%eax
 	movl	%r9d,%r12d
 	movq	512(%rsp),%rdi
 	addl	%r14d,%eax
 
 	leaq	448(%rsp),%rbp
 
 	addl	0(%rdi),%eax
 	addl	4(%rdi),%ebx
 	addl	8(%rdi),%ecx
 	addl	12(%rdi),%edx
 	addl	16(%rdi),%r8d
 	addl	20(%rdi),%r9d
 	addl	24(%rdi),%r10d
 	addl	28(%rdi),%r11d
 
 	movl	%eax,0(%rdi)
 	movl	%ebx,4(%rdi)
 	movl	%ecx,8(%rdi)
 	movl	%edx,12(%rdi)
 	movl	%r8d,16(%rdi)
 	movl	%r9d,20(%rdi)
 	movl	%r10d,24(%rdi)
 	movl	%r11d,28(%rdi)
 
 	cmpq	80(%rbp),%rsi
 	je	.Ldone_avx2
 
 	xorl	%r14d,%r14d
 	movl	%ebx,%edi
 	xorl	%ecx,%edi
 	movl	%r9d,%r12d
 	jmp	.Lower_avx2
 .balign	16
 .Lower_avx2:
 	addl	0+16(%rbp),%r11d
 	andl	%r8d,%r12d
 	rorxl	$25,%r8d,%r13d
 	rorxl	$11,%r8d,%r15d
 	leal	(%rax,%r14,1),%eax
 	leal	(%r11,%r12,1),%r11d
 	andnl	%r10d,%r8d,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%r8d,%r14d
 	leal	(%r11,%r12,1),%r11d
 	xorl	%r14d,%r13d
 	movl	%eax,%r15d
 	rorxl	$22,%eax,%r12d
 	leal	(%r11,%r13,1),%r11d
 	xorl	%ebx,%r15d
 	rorxl	$13,%eax,%r14d
 	rorxl	$2,%eax,%r13d
 	leal	(%rdx,%r11,1),%edx
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%ebx,%edi
 	xorl	%r13d,%r14d
 	leal	(%r11,%rdi,1),%r11d
 	movl	%r8d,%r12d
 	addl	4+16(%rbp),%r10d
 	andl	%edx,%r12d
 	rorxl	$25,%edx,%r13d
 	rorxl	$11,%edx,%edi
 	leal	(%r11,%r14,1),%r11d
 	leal	(%r10,%r12,1),%r10d
 	andnl	%r9d,%edx,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%edx,%r14d
 	leal	(%r10,%r12,1),%r10d
 	xorl	%r14d,%r13d
 	movl	%r11d,%edi
 	rorxl	$22,%r11d,%r12d
 	leal	(%r10,%r13,1),%r10d
 	xorl	%eax,%edi
 	rorxl	$13,%r11d,%r14d
 	rorxl	$2,%r11d,%r13d
 	leal	(%rcx,%r10,1),%ecx
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%eax,%r15d
 	xorl	%r13d,%r14d
 	leal	(%r10,%r15,1),%r10d
 	movl	%edx,%r12d
 	addl	8+16(%rbp),%r9d
 	andl	%ecx,%r12d
 	rorxl	$25,%ecx,%r13d
 	rorxl	$11,%ecx,%r15d
 	leal	(%r10,%r14,1),%r10d
 	leal	(%r9,%r12,1),%r9d
 	andnl	%r8d,%ecx,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%ecx,%r14d
 	leal	(%r9,%r12,1),%r9d
 	xorl	%r14d,%r13d
 	movl	%r10d,%r15d
 	rorxl	$22,%r10d,%r12d
 	leal	(%r9,%r13,1),%r9d
 	xorl	%r11d,%r15d
 	rorxl	$13,%r10d,%r14d
 	rorxl	$2,%r10d,%r13d
 	leal	(%rbx,%r9,1),%ebx
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%r11d,%edi
 	xorl	%r13d,%r14d
 	leal	(%r9,%rdi,1),%r9d
 	movl	%ecx,%r12d
 	addl	12+16(%rbp),%r8d
 	andl	%ebx,%r12d
 	rorxl	$25,%ebx,%r13d
 	rorxl	$11,%ebx,%edi
 	leal	(%r9,%r14,1),%r9d
 	leal	(%r8,%r12,1),%r8d
 	andnl	%edx,%ebx,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%ebx,%r14d
 	leal	(%r8,%r12,1),%r8d
 	xorl	%r14d,%r13d
 	movl	%r9d,%edi
 	rorxl	$22,%r9d,%r12d
 	leal	(%r8,%r13,1),%r8d
 	xorl	%r10d,%edi
 	rorxl	$13,%r9d,%r14d
 	rorxl	$2,%r9d,%r13d
 	leal	(%rax,%r8,1),%eax
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%r10d,%r15d
 	xorl	%r13d,%r14d
 	leal	(%r8,%r15,1),%r8d
 	movl	%ebx,%r12d
 	addl	32+16(%rbp),%edx
 	andl	%eax,%r12d
 	rorxl	$25,%eax,%r13d
 	rorxl	$11,%eax,%r15d
 	leal	(%r8,%r14,1),%r8d
 	leal	(%rdx,%r12,1),%edx
 	andnl	%ecx,%eax,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%eax,%r14d
 	leal	(%rdx,%r12,1),%edx
 	xorl	%r14d,%r13d
 	movl	%r8d,%r15d
 	rorxl	$22,%r8d,%r12d
 	leal	(%rdx,%r13,1),%edx
 	xorl	%r9d,%r15d
 	rorxl	$13,%r8d,%r14d
 	rorxl	$2,%r8d,%r13d
 	leal	(%r11,%rdx,1),%r11d
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%r9d,%edi
 	xorl	%r13d,%r14d
 	leal	(%rdx,%rdi,1),%edx
 	movl	%eax,%r12d
 	addl	36+16(%rbp),%ecx
 	andl	%r11d,%r12d
 	rorxl	$25,%r11d,%r13d
 	rorxl	$11,%r11d,%edi
 	leal	(%rdx,%r14,1),%edx
 	leal	(%rcx,%r12,1),%ecx
 	andnl	%ebx,%r11d,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%r11d,%r14d
 	leal	(%rcx,%r12,1),%ecx
 	xorl	%r14d,%r13d
 	movl	%edx,%edi
 	rorxl	$22,%edx,%r12d
 	leal	(%rcx,%r13,1),%ecx
 	xorl	%r8d,%edi
 	rorxl	$13,%edx,%r14d
 	rorxl	$2,%edx,%r13d
 	leal	(%r10,%rcx,1),%r10d
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%r8d,%r15d
 	xorl	%r13d,%r14d
 	leal	(%rcx,%r15,1),%ecx
 	movl	%r11d,%r12d
 	addl	40+16(%rbp),%ebx
 	andl	%r10d,%r12d
 	rorxl	$25,%r10d,%r13d
 	rorxl	$11,%r10d,%r15d
 	leal	(%rcx,%r14,1),%ecx
 	leal	(%rbx,%r12,1),%ebx
 	andnl	%eax,%r10d,%r12d
 	xorl	%r15d,%r13d
 	rorxl	$6,%r10d,%r14d
 	leal	(%rbx,%r12,1),%ebx
 	xorl	%r14d,%r13d
 	movl	%ecx,%r15d
 	rorxl	$22,%ecx,%r12d
 	leal	(%rbx,%r13,1),%ebx
 	xorl	%edx,%r15d
 	rorxl	$13,%ecx,%r14d
 	rorxl	$2,%ecx,%r13d
 	leal	(%r9,%rbx,1),%r9d
 	andl	%r15d,%edi
 	xorl	%r12d,%r14d
 	xorl	%edx,%edi
 	xorl	%r13d,%r14d
 	leal	(%rbx,%rdi,1),%ebx
 	movl	%r10d,%r12d
 	addl	44+16(%rbp),%eax
 	andl	%r9d,%r12d
 	rorxl	$25,%r9d,%r13d
 	rorxl	$11,%r9d,%edi
 	leal	(%rbx,%r14,1),%ebx
 	leal	(%rax,%r12,1),%eax
 	andnl	%r11d,%r9d,%r12d
 	xorl	%edi,%r13d
 	rorxl	$6,%r9d,%r14d
 	leal	(%rax,%r12,1),%eax
 	xorl	%r14d,%r13d
 	movl	%ebx,%edi
 	rorxl	$22,%ebx,%r12d
 	leal	(%rax,%r13,1),%eax
 	xorl	%ecx,%edi
 	rorxl	$13,%ebx,%r14d
 	rorxl	$2,%ebx,%r13d
 	leal	(%r8,%rax,1),%r8d
 	andl	%edi,%r15d
 	xorl	%r12d,%r14d
 	xorl	%ecx,%r15d
 	xorl	%r13d,%r14d
 	leal	(%rax,%r15,1),%eax
 	movl	%r9d,%r12d
 	leaq	-64(%rbp),%rbp
 	cmpq	%rsp,%rbp
 	jae	.Lower_avx2
 
 	movq	512(%rsp),%rdi
 	addl	%r14d,%eax
 
 	leaq	448(%rsp),%rsp
 
 .cfi_escape	0x0f,0x06,0x77,0xd8,0x00,0x06,0x23,0x08
 
 	addl	0(%rdi),%eax
 	addl	4(%rdi),%ebx
 	addl	8(%rdi),%ecx
 	addl	12(%rdi),%edx
 	addl	16(%rdi),%r8d
 	addl	20(%rdi),%r9d
 	leaq	128(%rsi),%rsi
 	addl	24(%rdi),%r10d
 	movq	%rsi,%r12
 	addl	28(%rdi),%r11d
 	cmpq	64+16(%rsp),%rsi
 
 	movl	%eax,0(%rdi)
 	cmoveq	%rsp,%r12
 	movl	%ebx,4(%rdi)
 	movl	%ecx,8(%rdi)
 	movl	%edx,12(%rdi)
 	movl	%r8d,16(%rdi)
 	movl	%r9d,20(%rdi)
 	movl	%r10d,24(%rdi)
 	movl	%r11d,28(%rdi)
 
 	jbe	.Loop_avx2
 	leaq	(%rsp),%rbp
 
 
 .cfi_escape	0x0f,0x06,0x76,0xd8,0x00,0x06,0x23,0x08
 
 .Ldone_avx2:
 	movq	88(%rbp),%rsi
 .cfi_def_cfa	%rsi,8
 	vzeroupper
 	movq	-48(%rsi),%r15
 .cfi_restore	%r15
 	movq	-40(%rsi),%r14
 .cfi_restore	%r14
 	movq	-32(%rsi),%r13
 .cfi_restore	%r13
 	movq	-24(%rsi),%r12
 .cfi_restore	%r12
 	movq	-16(%rsi),%rbp
 .cfi_restore	%rbp
 	movq	-8(%rsi),%rbx
 .cfi_restore	%rbx
 	leaq	(%rsi),%rsp
 .cfi_def_cfa_register	%rsp
 .Lepilogue_avx2:
 	RET
 .cfi_endproc
 SET_SIZE(zfs_sha256_transform_avx2)
+STACK_FRAME_NON_STANDARD zfs_sha256_transform_avx2
+
+/* Workaround for missing asm macro in RHEL 8. */
+#if defined(__linux__) && defined(HAVE_STACK_FRAME_NON_STANDARD) && \
+    ! defined(HAVE_STACK_FRAME_NON_STANDARD_ASM)
+.section .discard.func_stack_frame_non_standard, "aw"
+	.long zfs_sha256_transform_x64 - .
+	.long zfs_sha256_transform_shani - .
+	.long zfs_sha256_transform_ssse3 - .
+	.long zfs_sha256_transform_avx - .
+	.long zfs_sha256_transform_avx2 - .
+#endif
 
 #if defined(__ELF__)
 	.section .note.GNU-stack,"",%progbits
 #endif
 #endif
diff --git a/sys/contrib/openzfs/module/icp/asm-x86_64/sha2/sha512-x86_64.S b/sys/contrib/openzfs/module/icp/asm-x86_64/sha2/sha512-x86_64.S
index fbbcca650d10..4af77a3decbc 100644
--- a/sys/contrib/openzfs/module/icp/asm-x86_64/sha2/sha512-x86_64.S
+++ b/sys/contrib/openzfs/module/icp/asm-x86_64/sha2/sha512-x86_64.S
@@ -1,4011 +1,4024 @@
 /*
  * Copyright 2004-2022 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  *     https://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
 
 /*
  * Portions Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
  * - modified assembly to fit into OpenZFS
  */
 
 #if defined(__x86_64)
 
 #define _ASM
 #include <sys/asm_linkage.h>
+#include <sys/frame.h>
 
 SECTION_STATIC
 
 .balign	64
 SET_OBJ(K512)
 K512:
 .quad	0x428a2f98d728ae22,0x7137449123ef65cd
 .quad	0x428a2f98d728ae22,0x7137449123ef65cd
 .quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
 .quad	0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
 .quad	0x3956c25bf348b538,0x59f111f1b605d019
 .quad	0x3956c25bf348b538,0x59f111f1b605d019
 .quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
 .quad	0x923f82a4af194f9b,0xab1c5ed5da6d8118
 .quad	0xd807aa98a3030242,0x12835b0145706fbe
 .quad	0xd807aa98a3030242,0x12835b0145706fbe
 .quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
 .quad	0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
 .quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
 .quad	0x72be5d74f27b896f,0x80deb1fe3b1696b1
 .quad	0x9bdc06a725c71235,0xc19bf174cf692694
 .quad	0x9bdc06a725c71235,0xc19bf174cf692694
 .quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
 .quad	0xe49b69c19ef14ad2,0xefbe4786384f25e3
 .quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
 .quad	0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
 .quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
 .quad	0x2de92c6f592b0275,0x4a7484aa6ea6e483
 .quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
 .quad	0x5cb0a9dcbd41fbd4,0x76f988da831153b5
 .quad	0x983e5152ee66dfab,0xa831c66d2db43210
 .quad	0x983e5152ee66dfab,0xa831c66d2db43210
 .quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
 .quad	0xb00327c898fb213f,0xbf597fc7beef0ee4
 .quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
 .quad	0xc6e00bf33da88fc2,0xd5a79147930aa725
 .quad	0x06ca6351e003826f,0x142929670a0e6e70
 .quad	0x06ca6351e003826f,0x142929670a0e6e70
 .quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
 .quad	0x27b70a8546d22ffc,0x2e1b21385c26c926
 .quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
 .quad	0x4d2c6dfc5ac42aed,0x53380d139d95b3df
 .quad	0x650a73548baf63de,0x766a0abb3c77b2a8
 .quad	0x650a73548baf63de,0x766a0abb3c77b2a8
 .quad	0x81c2c92e47edaee6,0x92722c851482353b
 .quad	0x81c2c92e47edaee6,0x92722c851482353b
 .quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
 .quad	0xa2bfe8a14cf10364,0xa81a664bbc423001
 .quad	0xc24b8b70d0f89791,0xc76c51a30654be30
 .quad	0xc24b8b70d0f89791,0xc76c51a30654be30
 .quad	0xd192e819d6ef5218,0xd69906245565a910
 .quad	0xd192e819d6ef5218,0xd69906245565a910
 .quad	0xf40e35855771202a,0x106aa07032bbd1b8
 .quad	0xf40e35855771202a,0x106aa07032bbd1b8
 .quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
 .quad	0x19a4c116b8d2d0c8,0x1e376c085141ab53
 .quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
 .quad	0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
 .quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
 .quad	0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
 .quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
 .quad	0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
 .quad	0x748f82ee5defb2fc,0x78a5636f43172f60
 .quad	0x748f82ee5defb2fc,0x78a5636f43172f60
 .quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
 .quad	0x84c87814a1f0ab72,0x8cc702081a6439ec
 .quad	0x90befffa23631e28,0xa4506cebde82bde9
 .quad	0x90befffa23631e28,0xa4506cebde82bde9
 .quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
 .quad	0xbef9a3f7b2c67915,0xc67178f2e372532b
 .quad	0xca273eceea26619c,0xd186b8c721c0c207
 .quad	0xca273eceea26619c,0xd186b8c721c0c207
 .quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
 .quad	0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
 .quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
 .quad	0x06f067aa72176fba,0x0a637dc5a2c898a6
 .quad	0x113f9804bef90dae,0x1b710b35131c471b
 .quad	0x113f9804bef90dae,0x1b710b35131c471b
 .quad	0x28db77f523047d84,0x32caab7b40c72493
 .quad	0x28db77f523047d84,0x32caab7b40c72493
 .quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
 .quad	0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
 .quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
 .quad	0x4cc5d4becb3e42b6,0x597f299cfc657e2a
 .quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
 .quad	0x5fcb6fab3ad6faec,0x6c44198c4a475817
 .quad	0x0001020304050607,0x08090a0b0c0d0e0f
 .quad	0x0001020304050607,0x08090a0b0c0d0e0f
 
 ENTRY_ALIGN(zfs_sha512_transform_x64, 16)
 .cfi_startproc
 	ENDBR
 	movq	%rsp,%rax
 .cfi_def_cfa_register	%rax
 	pushq	%rbx
 .cfi_offset	%rbx,-16
 	pushq	%rbp
 .cfi_offset	%rbp,-24
 	pushq	%r12
 .cfi_offset	%r12,-32
 	pushq	%r13
 .cfi_offset	%r13,-40
 	pushq	%r14
 .cfi_offset	%r14,-48
 	pushq	%r15
 .cfi_offset	%r15,-56
 	shlq	$4,%rdx
 	subq	$128+32,%rsp
 	leaq	(%rsi,%rdx,8),%rdx
 	andq	$-64,%rsp
 	movq	%rdi,128+0(%rsp)
 	movq	%rsi,128+8(%rsp)
 	movq	%rdx,128+16(%rsp)
 	movq	%rax,152(%rsp)
 .cfi_escape	0x0f,0x06,0x77,0x98,0x01,0x06,0x23,0x08
 .Lprologue:
 	movq	0(%rdi),%rax
 	movq	8(%rdi),%rbx
 	movq	16(%rdi),%rcx
 	movq	24(%rdi),%rdx
 	movq	32(%rdi),%r8
 	movq	40(%rdi),%r9
 	movq	48(%rdi),%r10
 	movq	56(%rdi),%r11
 	jmp	.Lloop
 .balign	16
 .Lloop:
 	movq	%rbx,%rdi
 	leaq	K512(%rip),%rbp
 	xorq	%rcx,%rdi
 	movq	0(%rsi),%r12
 	movq	%r8,%r13
 	movq	%rax,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%r9,%r15
 	xorq	%r8,%r13
 	rorq	$5,%r14
 	xorq	%r10,%r15
 	movq	%r12,0(%rsp)
 	xorq	%rax,%r14
 	andq	%r8,%r15
 	rorq	$4,%r13
 	addq	%r11,%r12
 	xorq	%r10,%r15
 	rorq	$6,%r14
 	xorq	%r8,%r13
 	addq	%r15,%r12
 	movq	%rax,%r15
 	addq	(%rbp),%r12
 	xorq	%rax,%r14
 	xorq	%rbx,%r15
 	rorq	$14,%r13
 	movq	%rbx,%r11
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%r11
 	addq	%r12,%rdx
 	addq	%r12,%r11
 	leaq	8(%rbp),%rbp
 	addq	%r14,%r11
 	movq	8(%rsi),%r12
 	movq	%rdx,%r13
 	movq	%r11,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%r8,%rdi
 	xorq	%rdx,%r13
 	rorq	$5,%r14
 	xorq	%r9,%rdi
 	movq	%r12,8(%rsp)
 	xorq	%r11,%r14
 	andq	%rdx,%rdi
 	rorq	$4,%r13
 	addq	%r10,%r12
 	xorq	%r9,%rdi
 	rorq	$6,%r14
 	xorq	%rdx,%r13
 	addq	%rdi,%r12
 	movq	%r11,%rdi
 	addq	(%rbp),%r12
 	xorq	%r11,%r14
 	xorq	%rax,%rdi
 	rorq	$14,%r13
 	movq	%rax,%r10
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%r10
 	addq	%r12,%rcx
 	addq	%r12,%r10
 	leaq	24(%rbp),%rbp
 	addq	%r14,%r10
 	movq	16(%rsi),%r12
 	movq	%rcx,%r13
 	movq	%r10,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%rdx,%r15
 	xorq	%rcx,%r13
 	rorq	$5,%r14
 	xorq	%r8,%r15
 	movq	%r12,16(%rsp)
 	xorq	%r10,%r14
 	andq	%rcx,%r15
 	rorq	$4,%r13
 	addq	%r9,%r12
 	xorq	%r8,%r15
 	rorq	$6,%r14
 	xorq	%rcx,%r13
 	addq	%r15,%r12
 	movq	%r10,%r15
 	addq	(%rbp),%r12
 	xorq	%r10,%r14
 	xorq	%r11,%r15
 	rorq	$14,%r13
 	movq	%r11,%r9
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%r9
 	addq	%r12,%rbx
 	addq	%r12,%r9
 	leaq	8(%rbp),%rbp
 	addq	%r14,%r9
 	movq	24(%rsi),%r12
 	movq	%rbx,%r13
 	movq	%r9,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%rcx,%rdi
 	xorq	%rbx,%r13
 	rorq	$5,%r14
 	xorq	%rdx,%rdi
 	movq	%r12,24(%rsp)
 	xorq	%r9,%r14
 	andq	%rbx,%rdi
 	rorq	$4,%r13
 	addq	%r8,%r12
 	xorq	%rdx,%rdi
 	rorq	$6,%r14
 	xorq	%rbx,%r13
 	addq	%rdi,%r12
 	movq	%r9,%rdi
 	addq	(%rbp),%r12
 	xorq	%r9,%r14
 	xorq	%r10,%rdi
 	rorq	$14,%r13
 	movq	%r10,%r8
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%r8
 	addq	%r12,%rax
 	addq	%r12,%r8
 	leaq	24(%rbp),%rbp
 	addq	%r14,%r8
 	movq	32(%rsi),%r12
 	movq	%rax,%r13
 	movq	%r8,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%rbx,%r15
 	xorq	%rax,%r13
 	rorq	$5,%r14
 	xorq	%rcx,%r15
 	movq	%r12,32(%rsp)
 	xorq	%r8,%r14
 	andq	%rax,%r15
 	rorq	$4,%r13
 	addq	%rdx,%r12
 	xorq	%rcx,%r15
 	rorq	$6,%r14
 	xorq	%rax,%r13
 	addq	%r15,%r12
 	movq	%r8,%r15
 	addq	(%rbp),%r12
 	xorq	%r8,%r14
 	xorq	%r9,%r15
 	rorq	$14,%r13
 	movq	%r9,%rdx
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%rdx
 	addq	%r12,%r11
 	addq	%r12,%rdx
 	leaq	8(%rbp),%rbp
 	addq	%r14,%rdx
 	movq	40(%rsi),%r12
 	movq	%r11,%r13
 	movq	%rdx,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%rax,%rdi
 	xorq	%r11,%r13
 	rorq	$5,%r14
 	xorq	%rbx,%rdi
 	movq	%r12,40(%rsp)
 	xorq	%rdx,%r14
 	andq	%r11,%rdi
 	rorq	$4,%r13
 	addq	%rcx,%r12
 	xorq	%rbx,%rdi
 	rorq	$6,%r14
 	xorq	%r11,%r13
 	addq	%rdi,%r12
 	movq	%rdx,%rdi
 	addq	(%rbp),%r12
 	xorq	%rdx,%r14
 	xorq	%r8,%rdi
 	rorq	$14,%r13
 	movq	%r8,%rcx
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%rcx
 	addq	%r12,%r10
 	addq	%r12,%rcx
 	leaq	24(%rbp),%rbp
 	addq	%r14,%rcx
 	movq	48(%rsi),%r12
 	movq	%r10,%r13
 	movq	%rcx,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%r11,%r15
 	xorq	%r10,%r13
 	rorq	$5,%r14
 	xorq	%rax,%r15
 	movq	%r12,48(%rsp)
 	xorq	%rcx,%r14
 	andq	%r10,%r15
 	rorq	$4,%r13
 	addq	%rbx,%r12
 	xorq	%rax,%r15
 	rorq	$6,%r14
 	xorq	%r10,%r13
 	addq	%r15,%r12
 	movq	%rcx,%r15
 	addq	(%rbp),%r12
 	xorq	%rcx,%r14
 	xorq	%rdx,%r15
 	rorq	$14,%r13
 	movq	%rdx,%rbx
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%rbx
 	addq	%r12,%r9
 	addq	%r12,%rbx
 	leaq	8(%rbp),%rbp
 	addq	%r14,%rbx
 	movq	56(%rsi),%r12
 	movq	%r9,%r13
 	movq	%rbx,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%r10,%rdi
 	xorq	%r9,%r13
 	rorq	$5,%r14
 	xorq	%r11,%rdi
 	movq	%r12,56(%rsp)
 	xorq	%rbx,%r14
 	andq	%r9,%rdi
 	rorq	$4,%r13
 	addq	%rax,%r12
 	xorq	%r11,%rdi
 	rorq	$6,%r14
 	xorq	%r9,%r13
 	addq	%rdi,%r12
 	movq	%rbx,%rdi
 	addq	(%rbp),%r12
 	xorq	%rbx,%r14
 	xorq	%rcx,%rdi
 	rorq	$14,%r13
 	movq	%rcx,%rax
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%rax
 	addq	%r12,%r8
 	addq	%r12,%rax
 	leaq	24(%rbp),%rbp
 	addq	%r14,%rax
 	movq	64(%rsi),%r12
 	movq	%r8,%r13
 	movq	%rax,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%r9,%r15
 	xorq	%r8,%r13
 	rorq	$5,%r14
 	xorq	%r10,%r15
 	movq	%r12,64(%rsp)
 	xorq	%rax,%r14
 	andq	%r8,%r15
 	rorq	$4,%r13
 	addq	%r11,%r12
 	xorq	%r10,%r15
 	rorq	$6,%r14
 	xorq	%r8,%r13
 	addq	%r15,%r12
 	movq	%rax,%r15
 	addq	(%rbp),%r12
 	xorq	%rax,%r14
 	xorq	%rbx,%r15
 	rorq	$14,%r13
 	movq	%rbx,%r11
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%r11
 	addq	%r12,%rdx
 	addq	%r12,%r11
 	leaq	8(%rbp),%rbp
 	addq	%r14,%r11
 	movq	72(%rsi),%r12
 	movq	%rdx,%r13
 	movq	%r11,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%r8,%rdi
 	xorq	%rdx,%r13
 	rorq	$5,%r14
 	xorq	%r9,%rdi
 	movq	%r12,72(%rsp)
 	xorq	%r11,%r14
 	andq	%rdx,%rdi
 	rorq	$4,%r13
 	addq	%r10,%r12
 	xorq	%r9,%rdi
 	rorq	$6,%r14
 	xorq	%rdx,%r13
 	addq	%rdi,%r12
 	movq	%r11,%rdi
 	addq	(%rbp),%r12
 	xorq	%r11,%r14
 	xorq	%rax,%rdi
 	rorq	$14,%r13
 	movq	%rax,%r10
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%r10
 	addq	%r12,%rcx
 	addq	%r12,%r10
 	leaq	24(%rbp),%rbp
 	addq	%r14,%r10
 	movq	80(%rsi),%r12
 	movq	%rcx,%r13
 	movq	%r10,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%rdx,%r15
 	xorq	%rcx,%r13
 	rorq	$5,%r14
 	xorq	%r8,%r15
 	movq	%r12,80(%rsp)
 	xorq	%r10,%r14
 	andq	%rcx,%r15
 	rorq	$4,%r13
 	addq	%r9,%r12
 	xorq	%r8,%r15
 	rorq	$6,%r14
 	xorq	%rcx,%r13
 	addq	%r15,%r12
 	movq	%r10,%r15
 	addq	(%rbp),%r12
 	xorq	%r10,%r14
 	xorq	%r11,%r15
 	rorq	$14,%r13
 	movq	%r11,%r9
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%r9
 	addq	%r12,%rbx
 	addq	%r12,%r9
 	leaq	8(%rbp),%rbp
 	addq	%r14,%r9
 	movq	88(%rsi),%r12
 	movq	%rbx,%r13
 	movq	%r9,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%rcx,%rdi
 	xorq	%rbx,%r13
 	rorq	$5,%r14
 	xorq	%rdx,%rdi
 	movq	%r12,88(%rsp)
 	xorq	%r9,%r14
 	andq	%rbx,%rdi
 	rorq	$4,%r13
 	addq	%r8,%r12
 	xorq	%rdx,%rdi
 	rorq	$6,%r14
 	xorq	%rbx,%r13
 	addq	%rdi,%r12
 	movq	%r9,%rdi
 	addq	(%rbp),%r12
 	xorq	%r9,%r14
 	xorq	%r10,%rdi
 	rorq	$14,%r13
 	movq	%r10,%r8
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%r8
 	addq	%r12,%rax
 	addq	%r12,%r8
 	leaq	24(%rbp),%rbp
 	addq	%r14,%r8
 	movq	96(%rsi),%r12
 	movq	%rax,%r13
 	movq	%r8,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%rbx,%r15
 	xorq	%rax,%r13
 	rorq	$5,%r14
 	xorq	%rcx,%r15
 	movq	%r12,96(%rsp)
 	xorq	%r8,%r14
 	andq	%rax,%r15
 	rorq	$4,%r13
 	addq	%rdx,%r12
 	xorq	%rcx,%r15
 	rorq	$6,%r14
 	xorq	%rax,%r13
 	addq	%r15,%r12
 	movq	%r8,%r15
 	addq	(%rbp),%r12
 	xorq	%r8,%r14
 	xorq	%r9,%r15
 	rorq	$14,%r13
 	movq	%r9,%rdx
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%rdx
 	addq	%r12,%r11
 	addq	%r12,%rdx
 	leaq	8(%rbp),%rbp
 	addq	%r14,%rdx
 	movq	104(%rsi),%r12
 	movq	%r11,%r13
 	movq	%rdx,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%rax,%rdi
 	xorq	%r11,%r13
 	rorq	$5,%r14
 	xorq	%rbx,%rdi
 	movq	%r12,104(%rsp)
 	xorq	%rdx,%r14
 	andq	%r11,%rdi
 	rorq	$4,%r13
 	addq	%rcx,%r12
 	xorq	%rbx,%rdi
 	rorq	$6,%r14
 	xorq	%r11,%r13
 	addq	%rdi,%r12
 	movq	%rdx,%rdi
 	addq	(%rbp),%r12
 	xorq	%rdx,%r14
 	xorq	%r8,%rdi
 	rorq	$14,%r13
 	movq	%r8,%rcx
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%rcx
 	addq	%r12,%r10
 	addq	%r12,%rcx
 	leaq	24(%rbp),%rbp
 	addq	%r14,%rcx
 	movq	112(%rsi),%r12
 	movq	%r10,%r13
 	movq	%rcx,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%r11,%r15
 	xorq	%r10,%r13
 	rorq	$5,%r14
 	xorq	%rax,%r15
 	movq	%r12,112(%rsp)
 	xorq	%rcx,%r14
 	andq	%r10,%r15
 	rorq	$4,%r13
 	addq	%rbx,%r12
 	xorq	%rax,%r15
 	rorq	$6,%r14
 	xorq	%r10,%r13
 	addq	%r15,%r12
 	movq	%rcx,%r15
 	addq	(%rbp),%r12
 	xorq	%rcx,%r14
 	xorq	%rdx,%r15
 	rorq	$14,%r13
 	movq	%rdx,%rbx
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%rbx
 	addq	%r12,%r9
 	addq	%r12,%rbx
 	leaq	8(%rbp),%rbp
 	addq	%r14,%rbx
 	movq	120(%rsi),%r12
 	movq	%r9,%r13
 	movq	%rbx,%r14
 	bswapq	%r12
 	rorq	$23,%r13
 	movq	%r10,%rdi
 	xorq	%r9,%r13
 	rorq	$5,%r14
 	xorq	%r11,%rdi
 	movq	%r12,120(%rsp)
 	xorq	%rbx,%r14
 	andq	%r9,%rdi
 	rorq	$4,%r13
 	addq	%rax,%r12
 	xorq	%r11,%rdi
 	rorq	$6,%r14
 	xorq	%r9,%r13
 	addq	%rdi,%r12
 	movq	%rbx,%rdi
 	addq	(%rbp),%r12
 	xorq	%rbx,%r14
 	xorq	%rcx,%rdi
 	rorq	$14,%r13
 	movq	%rcx,%rax
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%rax
 	addq	%r12,%r8
 	addq	%r12,%rax
 	leaq	24(%rbp),%rbp
 	jmp	.Lrounds_16_xx
 .balign	16
 .Lrounds_16_xx:
 	movq	8(%rsp),%r13
 	movq	112(%rsp),%r15
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%rax
 	movq	%r15,%r14
 	rorq	$42,%r15
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%r15
 	shrq	$6,%r14
 	rorq	$19,%r15
 	xorq	%r13,%r12
 	xorq	%r14,%r15
 	addq	72(%rsp),%r12
 	addq	0(%rsp),%r12
 	movq	%r8,%r13
 	addq	%r15,%r12
 	movq	%rax,%r14
 	rorq	$23,%r13
 	movq	%r9,%r15
 	xorq	%r8,%r13
 	rorq	$5,%r14
 	xorq	%r10,%r15
 	movq	%r12,0(%rsp)
 	xorq	%rax,%r14
 	andq	%r8,%r15
 	rorq	$4,%r13
 	addq	%r11,%r12
 	xorq	%r10,%r15
 	rorq	$6,%r14
 	xorq	%r8,%r13
 	addq	%r15,%r12
 	movq	%rax,%r15
 	addq	(%rbp),%r12
 	xorq	%rax,%r14
 	xorq	%rbx,%r15
 	rorq	$14,%r13
 	movq	%rbx,%r11
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%r11
 	addq	%r12,%rdx
 	addq	%r12,%r11
 	leaq	8(%rbp),%rbp
 	movq	16(%rsp),%r13
 	movq	120(%rsp),%rdi
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%r11
 	movq	%rdi,%r14
 	rorq	$42,%rdi
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%rdi
 	shrq	$6,%r14
 	rorq	$19,%rdi
 	xorq	%r13,%r12
 	xorq	%r14,%rdi
 	addq	80(%rsp),%r12
 	addq	8(%rsp),%r12
 	movq	%rdx,%r13
 	addq	%rdi,%r12
 	movq	%r11,%r14
 	rorq	$23,%r13
 	movq	%r8,%rdi
 	xorq	%rdx,%r13
 	rorq	$5,%r14
 	xorq	%r9,%rdi
 	movq	%r12,8(%rsp)
 	xorq	%r11,%r14
 	andq	%rdx,%rdi
 	rorq	$4,%r13
 	addq	%r10,%r12
 	xorq	%r9,%rdi
 	rorq	$6,%r14
 	xorq	%rdx,%r13
 	addq	%rdi,%r12
 	movq	%r11,%rdi
 	addq	(%rbp),%r12
 	xorq	%r11,%r14
 	xorq	%rax,%rdi
 	rorq	$14,%r13
 	movq	%rax,%r10
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%r10
 	addq	%r12,%rcx
 	addq	%r12,%r10
 	leaq	24(%rbp),%rbp
 	movq	24(%rsp),%r13
 	movq	0(%rsp),%r15
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%r10
 	movq	%r15,%r14
 	rorq	$42,%r15
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%r15
 	shrq	$6,%r14
 	rorq	$19,%r15
 	xorq	%r13,%r12
 	xorq	%r14,%r15
 	addq	88(%rsp),%r12
 	addq	16(%rsp),%r12
 	movq	%rcx,%r13
 	addq	%r15,%r12
 	movq	%r10,%r14
 	rorq	$23,%r13
 	movq	%rdx,%r15
 	xorq	%rcx,%r13
 	rorq	$5,%r14
 	xorq	%r8,%r15
 	movq	%r12,16(%rsp)
 	xorq	%r10,%r14
 	andq	%rcx,%r15
 	rorq	$4,%r13
 	addq	%r9,%r12
 	xorq	%r8,%r15
 	rorq	$6,%r14
 	xorq	%rcx,%r13
 	addq	%r15,%r12
 	movq	%r10,%r15
 	addq	(%rbp),%r12
 	xorq	%r10,%r14
 	xorq	%r11,%r15
 	rorq	$14,%r13
 	movq	%r11,%r9
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%r9
 	addq	%r12,%rbx
 	addq	%r12,%r9
 	leaq	8(%rbp),%rbp
 	movq	32(%rsp),%r13
 	movq	8(%rsp),%rdi
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%r9
 	movq	%rdi,%r14
 	rorq	$42,%rdi
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%rdi
 	shrq	$6,%r14
 	rorq	$19,%rdi
 	xorq	%r13,%r12
 	xorq	%r14,%rdi
 	addq	96(%rsp),%r12
 	addq	24(%rsp),%r12
 	movq	%rbx,%r13
 	addq	%rdi,%r12
 	movq	%r9,%r14
 	rorq	$23,%r13
 	movq	%rcx,%rdi
 	xorq	%rbx,%r13
 	rorq	$5,%r14
 	xorq	%rdx,%rdi
 	movq	%r12,24(%rsp)
 	xorq	%r9,%r14
 	andq	%rbx,%rdi
 	rorq	$4,%r13
 	addq	%r8,%r12
 	xorq	%rdx,%rdi
 	rorq	$6,%r14
 	xorq	%rbx,%r13
 	addq	%rdi,%r12
 	movq	%r9,%rdi
 	addq	(%rbp),%r12
 	xorq	%r9,%r14
 	xorq	%r10,%rdi
 	rorq	$14,%r13
 	movq	%r10,%r8
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%r8
 	addq	%r12,%rax
 	addq	%r12,%r8
 	leaq	24(%rbp),%rbp
 	movq	40(%rsp),%r13
 	movq	16(%rsp),%r15
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%r8
 	movq	%r15,%r14
 	rorq	$42,%r15
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%r15
 	shrq	$6,%r14
 	rorq	$19,%r15
 	xorq	%r13,%r12
 	xorq	%r14,%r15
 	addq	104(%rsp),%r12
 	addq	32(%rsp),%r12
 	movq	%rax,%r13
 	addq	%r15,%r12
 	movq	%r8,%r14
 	rorq	$23,%r13
 	movq	%rbx,%r15
 	xorq	%rax,%r13
 	rorq	$5,%r14
 	xorq	%rcx,%r15
 	movq	%r12,32(%rsp)
 	xorq	%r8,%r14
 	andq	%rax,%r15
 	rorq	$4,%r13
 	addq	%rdx,%r12
 	xorq	%rcx,%r15
 	rorq	$6,%r14
 	xorq	%rax,%r13
 	addq	%r15,%r12
 	movq	%r8,%r15
 	addq	(%rbp),%r12
 	xorq	%r8,%r14
 	xorq	%r9,%r15
 	rorq	$14,%r13
 	movq	%r9,%rdx
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%rdx
 	addq	%r12,%r11
 	addq	%r12,%rdx
 	leaq	8(%rbp),%rbp
 	movq	48(%rsp),%r13
 	movq	24(%rsp),%rdi
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%rdx
 	movq	%rdi,%r14
 	rorq	$42,%rdi
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%rdi
 	shrq	$6,%r14
 	rorq	$19,%rdi
 	xorq	%r13,%r12
 	xorq	%r14,%rdi
 	addq	112(%rsp),%r12
 	addq	40(%rsp),%r12
 	movq	%r11,%r13
 	addq	%rdi,%r12
 	movq	%rdx,%r14
 	rorq	$23,%r13
 	movq	%rax,%rdi
 	xorq	%r11,%r13
 	rorq	$5,%r14
 	xorq	%rbx,%rdi
 	movq	%r12,40(%rsp)
 	xorq	%rdx,%r14
 	andq	%r11,%rdi
 	rorq	$4,%r13
 	addq	%rcx,%r12
 	xorq	%rbx,%rdi
 	rorq	$6,%r14
 	xorq	%r11,%r13
 	addq	%rdi,%r12
 	movq	%rdx,%rdi
 	addq	(%rbp),%r12
 	xorq	%rdx,%r14
 	xorq	%r8,%rdi
 	rorq	$14,%r13
 	movq	%r8,%rcx
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%rcx
 	addq	%r12,%r10
 	addq	%r12,%rcx
 	leaq	24(%rbp),%rbp
 	movq	56(%rsp),%r13
 	movq	32(%rsp),%r15
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%rcx
 	movq	%r15,%r14
 	rorq	$42,%r15
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%r15
 	shrq	$6,%r14
 	rorq	$19,%r15
 	xorq	%r13,%r12
 	xorq	%r14,%r15
 	addq	120(%rsp),%r12
 	addq	48(%rsp),%r12
 	movq	%r10,%r13
 	addq	%r15,%r12
 	movq	%rcx,%r14
 	rorq	$23,%r13
 	movq	%r11,%r15
 	xorq	%r10,%r13
 	rorq	$5,%r14
 	xorq	%rax,%r15
 	movq	%r12,48(%rsp)
 	xorq	%rcx,%r14
 	andq	%r10,%r15
 	rorq	$4,%r13
 	addq	%rbx,%r12
 	xorq	%rax,%r15
 	rorq	$6,%r14
 	xorq	%r10,%r13
 	addq	%r15,%r12
 	movq	%rcx,%r15
 	addq	(%rbp),%r12
 	xorq	%rcx,%r14
 	xorq	%rdx,%r15
 	rorq	$14,%r13
 	movq	%rdx,%rbx
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%rbx
 	addq	%r12,%r9
 	addq	%r12,%rbx
 	leaq	8(%rbp),%rbp
 	movq	64(%rsp),%r13
 	movq	40(%rsp),%rdi
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%rbx
 	movq	%rdi,%r14
 	rorq	$42,%rdi
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%rdi
 	shrq	$6,%r14
 	rorq	$19,%rdi
 	xorq	%r13,%r12
 	xorq	%r14,%rdi
 	addq	0(%rsp),%r12
 	addq	56(%rsp),%r12
 	movq	%r9,%r13
 	addq	%rdi,%r12
 	movq	%rbx,%r14
 	rorq	$23,%r13
 	movq	%r10,%rdi
 	xorq	%r9,%r13
 	rorq	$5,%r14
 	xorq	%r11,%rdi
 	movq	%r12,56(%rsp)
 	xorq	%rbx,%r14
 	andq	%r9,%rdi
 	rorq	$4,%r13
 	addq	%rax,%r12
 	xorq	%r11,%rdi
 	rorq	$6,%r14
 	xorq	%r9,%r13
 	addq	%rdi,%r12
 	movq	%rbx,%rdi
 	addq	(%rbp),%r12
 	xorq	%rbx,%r14
 	xorq	%rcx,%rdi
 	rorq	$14,%r13
 	movq	%rcx,%rax
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%rax
 	addq	%r12,%r8
 	addq	%r12,%rax
 	leaq	24(%rbp),%rbp
 	movq	72(%rsp),%r13
 	movq	48(%rsp),%r15
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%rax
 	movq	%r15,%r14
 	rorq	$42,%r15
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%r15
 	shrq	$6,%r14
 	rorq	$19,%r15
 	xorq	%r13,%r12
 	xorq	%r14,%r15
 	addq	8(%rsp),%r12
 	addq	64(%rsp),%r12
 	movq	%r8,%r13
 	addq	%r15,%r12
 	movq	%rax,%r14
 	rorq	$23,%r13
 	movq	%r9,%r15
 	xorq	%r8,%r13
 	rorq	$5,%r14
 	xorq	%r10,%r15
 	movq	%r12,64(%rsp)
 	xorq	%rax,%r14
 	andq	%r8,%r15
 	rorq	$4,%r13
 	addq	%r11,%r12
 	xorq	%r10,%r15
 	rorq	$6,%r14
 	xorq	%r8,%r13
 	addq	%r15,%r12
 	movq	%rax,%r15
 	addq	(%rbp),%r12
 	xorq	%rax,%r14
 	xorq	%rbx,%r15
 	rorq	$14,%r13
 	movq	%rbx,%r11
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%r11
 	addq	%r12,%rdx
 	addq	%r12,%r11
 	leaq	8(%rbp),%rbp
 	movq	80(%rsp),%r13
 	movq	56(%rsp),%rdi
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%r11
 	movq	%rdi,%r14
 	rorq	$42,%rdi
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%rdi
 	shrq	$6,%r14
 	rorq	$19,%rdi
 	xorq	%r13,%r12
 	xorq	%r14,%rdi
 	addq	16(%rsp),%r12
 	addq	72(%rsp),%r12
 	movq	%rdx,%r13
 	addq	%rdi,%r12
 	movq	%r11,%r14
 	rorq	$23,%r13
 	movq	%r8,%rdi
 	xorq	%rdx,%r13
 	rorq	$5,%r14
 	xorq	%r9,%rdi
 	movq	%r12,72(%rsp)
 	xorq	%r11,%r14
 	andq	%rdx,%rdi
 	rorq	$4,%r13
 	addq	%r10,%r12
 	xorq	%r9,%rdi
 	rorq	$6,%r14
 	xorq	%rdx,%r13
 	addq	%rdi,%r12
 	movq	%r11,%rdi
 	addq	(%rbp),%r12
 	xorq	%r11,%r14
 	xorq	%rax,%rdi
 	rorq	$14,%r13
 	movq	%rax,%r10
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%r10
 	addq	%r12,%rcx
 	addq	%r12,%r10
 	leaq	24(%rbp),%rbp
 	movq	88(%rsp),%r13
 	movq	64(%rsp),%r15
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%r10
 	movq	%r15,%r14
 	rorq	$42,%r15
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%r15
 	shrq	$6,%r14
 	rorq	$19,%r15
 	xorq	%r13,%r12
 	xorq	%r14,%r15
 	addq	24(%rsp),%r12
 	addq	80(%rsp),%r12
 	movq	%rcx,%r13
 	addq	%r15,%r12
 	movq	%r10,%r14
 	rorq	$23,%r13
 	movq	%rdx,%r15
 	xorq	%rcx,%r13
 	rorq	$5,%r14
 	xorq	%r8,%r15
 	movq	%r12,80(%rsp)
 	xorq	%r10,%r14
 	andq	%rcx,%r15
 	rorq	$4,%r13
 	addq	%r9,%r12
 	xorq	%r8,%r15
 	rorq	$6,%r14
 	xorq	%rcx,%r13
 	addq	%r15,%r12
 	movq	%r10,%r15
 	addq	(%rbp),%r12
 	xorq	%r10,%r14
 	xorq	%r11,%r15
 	rorq	$14,%r13
 	movq	%r11,%r9
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%r9
 	addq	%r12,%rbx
 	addq	%r12,%r9
 	leaq	8(%rbp),%rbp
 	movq	96(%rsp),%r13
 	movq	72(%rsp),%rdi
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%r9
 	movq	%rdi,%r14
 	rorq	$42,%rdi
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%rdi
 	shrq	$6,%r14
 	rorq	$19,%rdi
 	xorq	%r13,%r12
 	xorq	%r14,%rdi
 	addq	32(%rsp),%r12
 	addq	88(%rsp),%r12
 	movq	%rbx,%r13
 	addq	%rdi,%r12
 	movq	%r9,%r14
 	rorq	$23,%r13
 	movq	%rcx,%rdi
 	xorq	%rbx,%r13
 	rorq	$5,%r14
 	xorq	%rdx,%rdi
 	movq	%r12,88(%rsp)
 	xorq	%r9,%r14
 	andq	%rbx,%rdi
 	rorq	$4,%r13
 	addq	%r8,%r12
 	xorq	%rdx,%rdi
 	rorq	$6,%r14
 	xorq	%rbx,%r13
 	addq	%rdi,%r12
 	movq	%r9,%rdi
 	addq	(%rbp),%r12
 	xorq	%r9,%r14
 	xorq	%r10,%rdi
 	rorq	$14,%r13
 	movq	%r10,%r8
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%r8
 	addq	%r12,%rax
 	addq	%r12,%r8
 	leaq	24(%rbp),%rbp
 	movq	104(%rsp),%r13
 	movq	80(%rsp),%r15
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%r8
 	movq	%r15,%r14
 	rorq	$42,%r15
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%r15
 	shrq	$6,%r14
 	rorq	$19,%r15
 	xorq	%r13,%r12
 	xorq	%r14,%r15
 	addq	40(%rsp),%r12
 	addq	96(%rsp),%r12
 	movq	%rax,%r13
 	addq	%r15,%r12
 	movq	%r8,%r14
 	rorq	$23,%r13
 	movq	%rbx,%r15
 	xorq	%rax,%r13
 	rorq	$5,%r14
 	xorq	%rcx,%r15
 	movq	%r12,96(%rsp)
 	xorq	%r8,%r14
 	andq	%rax,%r15
 	rorq	$4,%r13
 	addq	%rdx,%r12
 	xorq	%rcx,%r15
 	rorq	$6,%r14
 	xorq	%rax,%r13
 	addq	%r15,%r12
 	movq	%r8,%r15
 	addq	(%rbp),%r12
 	xorq	%r8,%r14
 	xorq	%r9,%r15
 	rorq	$14,%r13
 	movq	%r9,%rdx
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%rdx
 	addq	%r12,%r11
 	addq	%r12,%rdx
 	leaq	8(%rbp),%rbp
 	movq	112(%rsp),%r13
 	movq	88(%rsp),%rdi
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%rdx
 	movq	%rdi,%r14
 	rorq	$42,%rdi
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%rdi
 	shrq	$6,%r14
 	rorq	$19,%rdi
 	xorq	%r13,%r12
 	xorq	%r14,%rdi
 	addq	48(%rsp),%r12
 	addq	104(%rsp),%r12
 	movq	%r11,%r13
 	addq	%rdi,%r12
 	movq	%rdx,%r14
 	rorq	$23,%r13
 	movq	%rax,%rdi
 	xorq	%r11,%r13
 	rorq	$5,%r14
 	xorq	%rbx,%rdi
 	movq	%r12,104(%rsp)
 	xorq	%rdx,%r14
 	andq	%r11,%rdi
 	rorq	$4,%r13
 	addq	%rcx,%r12
 	xorq	%rbx,%rdi
 	rorq	$6,%r14
 	xorq	%r11,%r13
 	addq	%rdi,%r12
 	movq	%rdx,%rdi
 	addq	(%rbp),%r12
 	xorq	%rdx,%r14
 	xorq	%r8,%rdi
 	rorq	$14,%r13
 	movq	%r8,%rcx
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%rcx
 	addq	%r12,%r10
 	addq	%r12,%rcx
 	leaq	24(%rbp),%rbp
 	movq	120(%rsp),%r13
 	movq	96(%rsp),%r15
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%rcx
 	movq	%r15,%r14
 	rorq	$42,%r15
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%r15
 	shrq	$6,%r14
 	rorq	$19,%r15
 	xorq	%r13,%r12
 	xorq	%r14,%r15
 	addq	56(%rsp),%r12
 	addq	112(%rsp),%r12
 	movq	%r10,%r13
 	addq	%r15,%r12
 	movq	%rcx,%r14
 	rorq	$23,%r13
 	movq	%r11,%r15
 	xorq	%r10,%r13
 	rorq	$5,%r14
 	xorq	%rax,%r15
 	movq	%r12,112(%rsp)
 	xorq	%rcx,%r14
 	andq	%r10,%r15
 	rorq	$4,%r13
 	addq	%rbx,%r12
 	xorq	%rax,%r15
 	rorq	$6,%r14
 	xorq	%r10,%r13
 	addq	%r15,%r12
 	movq	%rcx,%r15
 	addq	(%rbp),%r12
 	xorq	%rcx,%r14
 	xorq	%rdx,%r15
 	rorq	$14,%r13
 	movq	%rdx,%rbx
 	andq	%r15,%rdi
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%rdi,%rbx
 	addq	%r12,%r9
 	addq	%r12,%rbx
 	leaq	8(%rbp),%rbp
 	movq	0(%rsp),%r13
 	movq	104(%rsp),%rdi
 	movq	%r13,%r12
 	rorq	$7,%r13
 	addq	%r14,%rbx
 	movq	%rdi,%r14
 	rorq	$42,%rdi
 	xorq	%r12,%r13
 	shrq	$7,%r12
 	rorq	$1,%r13
 	xorq	%r14,%rdi
 	shrq	$6,%r14
 	rorq	$19,%rdi
 	xorq	%r13,%r12
 	xorq	%r14,%rdi
 	addq	64(%rsp),%r12
 	addq	120(%rsp),%r12
 	movq	%r9,%r13
 	addq	%rdi,%r12
 	movq	%rbx,%r14
 	rorq	$23,%r13
 	movq	%r10,%rdi
 	xorq	%r9,%r13
 	rorq	$5,%r14
 	xorq	%r11,%rdi
 	movq	%r12,120(%rsp)
 	xorq	%rbx,%r14
 	andq	%r9,%rdi
 	rorq	$4,%r13
 	addq	%rax,%r12
 	xorq	%r11,%rdi
 	rorq	$6,%r14
 	xorq	%r9,%r13
 	addq	%rdi,%r12
 	movq	%rbx,%rdi
 	addq	(%rbp),%r12
 	xorq	%rbx,%r14
 	xorq	%rcx,%rdi
 	rorq	$14,%r13
 	movq	%rcx,%rax
 	andq	%rdi,%r15
 	rorq	$28,%r14
 	addq	%r13,%r12
 	xorq	%r15,%rax
 	addq	%r12,%r8
 	addq	%r12,%rax
 	leaq	24(%rbp),%rbp
 	cmpb	$0,7(%rbp)
 	jnz	.Lrounds_16_xx
 	movq	128+0(%rsp),%rdi
 	addq	%r14,%rax
 	leaq	128(%rsi),%rsi
 	addq	0(%rdi),%rax
 	addq	8(%rdi),%rbx
 	addq	16(%rdi),%rcx
 	addq	24(%rdi),%rdx
 	addq	32(%rdi),%r8
 	addq	40(%rdi),%r9
 	addq	48(%rdi),%r10
 	addq	56(%rdi),%r11
 	cmpq	128+16(%rsp),%rsi
 	movq	%rax,0(%rdi)
 	movq	%rbx,8(%rdi)
 	movq	%rcx,16(%rdi)
 	movq	%rdx,24(%rdi)
 	movq	%r8,32(%rdi)
 	movq	%r9,40(%rdi)
 	movq	%r10,48(%rdi)
 	movq	%r11,56(%rdi)
 	jb	.Lloop
 	movq	152(%rsp),%rsi
 .cfi_def_cfa	%rsi,8
 	movq	-48(%rsi),%r15
 .cfi_restore	%r15
 	movq	-40(%rsi),%r14
 .cfi_restore	%r14
 	movq	-32(%rsi),%r13
 .cfi_restore	%r13
 	movq	-24(%rsi),%r12
 .cfi_restore	%r12
 	movq	-16(%rsi),%rbp
 .cfi_restore	%rbp
 	movq	-8(%rsi),%rbx
 .cfi_restore	%rbx
 	leaq	(%rsi),%rsp
 .cfi_def_cfa_register	%rsp
 .Lepilogue:
 	RET
 .cfi_endproc
 SET_SIZE(zfs_sha512_transform_x64)
+STACK_FRAME_NON_STANDARD zfs_sha512_transform_x64
 
 ENTRY_ALIGN(zfs_sha512_transform_avx, 64)
 .cfi_startproc
 	ENDBR
 	movq	%rsp,%rax
 .cfi_def_cfa_register	%rax
 	pushq	%rbx
 .cfi_offset	%rbx,-16
 	pushq	%rbp
 .cfi_offset	%rbp,-24
 	pushq	%r12
 .cfi_offset	%r12,-32
 	pushq	%r13
 .cfi_offset	%r13,-40
 	pushq	%r14
 .cfi_offset	%r14,-48
 	pushq	%r15
 .cfi_offset	%r15,-56
 	shlq	$4,%rdx
 	subq	$160,%rsp
 	leaq	(%rsi,%rdx,8),%rdx
 	andq	$-64,%rsp
 	movq	%rdi,128+0(%rsp)
 	movq	%rsi,128+8(%rsp)
 	movq	%rdx,128+16(%rsp)
 	movq	%rax,152(%rsp)
 .cfi_escape	0x0f,0x06,0x77,0x98,0x01,0x06,0x23,0x08
 .Lprologue_avx:
 
 	vzeroupper
 	movq	0(%rdi),%rax
 	movq	8(%rdi),%rbx
 	movq	16(%rdi),%rcx
 	movq	24(%rdi),%rdx
 	movq	32(%rdi),%r8
 	movq	40(%rdi),%r9
 	movq	48(%rdi),%r10
 	movq	56(%rdi),%r11
 	jmp	.Lloop_avx
 .balign	16
 .Lloop_avx:
 	vmovdqa	K512+1280(%rip),%xmm11
 	vmovdqu	0(%rsi),%xmm0
 	leaq	K512+128(%rip),%rbp
 	vmovdqu	16(%rsi),%xmm1
 	vmovdqu	32(%rsi),%xmm2
 	vpshufb	%xmm11,%xmm0,%xmm0
 	vmovdqu	48(%rsi),%xmm3
 	vpshufb	%xmm11,%xmm1,%xmm1
 	vmovdqu	64(%rsi),%xmm4
 	vpshufb	%xmm11,%xmm2,%xmm2
 	vmovdqu	80(%rsi),%xmm5
 	vpshufb	%xmm11,%xmm3,%xmm3
 	vmovdqu	96(%rsi),%xmm6
 	vpshufb	%xmm11,%xmm4,%xmm4
 	vmovdqu	112(%rsi),%xmm7
 	vpshufb	%xmm11,%xmm5,%xmm5
 	vpaddq	-128(%rbp),%xmm0,%xmm8
 	vpshufb	%xmm11,%xmm6,%xmm6
 	vpaddq	-96(%rbp),%xmm1,%xmm9
 	vpshufb	%xmm11,%xmm7,%xmm7
 	vpaddq	-64(%rbp),%xmm2,%xmm10
 	vpaddq	-32(%rbp),%xmm3,%xmm11
 	vmovdqa	%xmm8,0(%rsp)
 	vpaddq	0(%rbp),%xmm4,%xmm8
 	vmovdqa	%xmm9,16(%rsp)
 	vpaddq	32(%rbp),%xmm5,%xmm9
 	vmovdqa	%xmm10,32(%rsp)
 	vpaddq	64(%rbp),%xmm6,%xmm10
 	vmovdqa	%xmm11,48(%rsp)
 	vpaddq	96(%rbp),%xmm7,%xmm11
 	vmovdqa	%xmm8,64(%rsp)
 	movq	%rax,%r14
 	vmovdqa	%xmm9,80(%rsp)
 	movq	%rbx,%rdi
 	vmovdqa	%xmm10,96(%rsp)
 	xorq	%rcx,%rdi
 	vmovdqa	%xmm11,112(%rsp)
 	movq	%r8,%r13
 	jmp	.Lavx_00_47
 
 .balign	16
 .Lavx_00_47:
 	addq	$256,%rbp
 	vpalignr	$8,%xmm0,%xmm1,%xmm8
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rax
 	vpalignr	$8,%xmm4,%xmm5,%xmm11
 	movq	%r9,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$1,%xmm8,%xmm10
 	xorq	%r8,%r13
 	xorq	%r10,%r12
 	vpaddq	%xmm11,%xmm0,%xmm0
 	shrdq	$4,%r13,%r13
 	xorq	%rax,%r14
 	vpsrlq	$7,%xmm8,%xmm11
 	andq	%r8,%r12
 	xorq	%r8,%r13
 	vpsllq	$56,%xmm8,%xmm9
 	addq	0(%rsp),%r11
 	movq	%rax,%r15
 	vpxor	%xmm10,%xmm11,%xmm8
 	xorq	%r10,%r12
 	shrdq	$6,%r14,%r14
 	vpsrlq	$7,%xmm10,%xmm10
 	xorq	%rbx,%r15
 	addq	%r12,%r11
 	vpxor	%xmm9,%xmm8,%xmm8
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	vpsllq	$7,%xmm9,%xmm9
 	xorq	%rax,%r14
 	addq	%r13,%r11
 	vpxor	%xmm10,%xmm8,%xmm8
 	xorq	%rbx,%rdi
 	shrdq	$28,%r14,%r14
 	vpsrlq	$6,%xmm7,%xmm11
 	addq	%r11,%rdx
 	addq	%rdi,%r11
 	vpxor	%xmm9,%xmm8,%xmm8
 	movq	%rdx,%r13
 	addq	%r11,%r14
 	vpsllq	$3,%xmm7,%xmm10
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r11
 	vpaddq	%xmm8,%xmm0,%xmm0
 	movq	%r8,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$19,%xmm7,%xmm9
 	xorq	%rdx,%r13
 	xorq	%r9,%r12
 	vpxor	%xmm10,%xmm11,%xmm11
 	shrdq	$4,%r13,%r13
 	xorq	%r11,%r14
 	vpsllq	$42,%xmm10,%xmm10
 	andq	%rdx,%r12
 	xorq	%rdx,%r13
 	vpxor	%xmm9,%xmm11,%xmm11
 	addq	8(%rsp),%r10
 	movq	%r11,%rdi
 	vpsrlq	$42,%xmm9,%xmm9
 	xorq	%r9,%r12
 	shrdq	$6,%r14,%r14
 	vpxor	%xmm10,%xmm11,%xmm11
 	xorq	%rax,%rdi
 	addq	%r12,%r10
 	vpxor	%xmm9,%xmm11,%xmm11
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	vpaddq	%xmm11,%xmm0,%xmm0
 	xorq	%r11,%r14
 	addq	%r13,%r10
 	vpaddq	-128(%rbp),%xmm0,%xmm10
 	xorq	%rax,%r15
 	shrdq	$28,%r14,%r14
 	addq	%r10,%rcx
 	addq	%r15,%r10
 	movq	%rcx,%r13
 	addq	%r10,%r14
 	vmovdqa	%xmm10,0(%rsp)
 	vpalignr	$8,%xmm1,%xmm2,%xmm8
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r10
 	vpalignr	$8,%xmm5,%xmm6,%xmm11
 	movq	%rdx,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$1,%xmm8,%xmm10
 	xorq	%rcx,%r13
 	xorq	%r8,%r12
 	vpaddq	%xmm11,%xmm1,%xmm1
 	shrdq	$4,%r13,%r13
 	xorq	%r10,%r14
 	vpsrlq	$7,%xmm8,%xmm11
 	andq	%rcx,%r12
 	xorq	%rcx,%r13
 	vpsllq	$56,%xmm8,%xmm9
 	addq	16(%rsp),%r9
 	movq	%r10,%r15
 	vpxor	%xmm10,%xmm11,%xmm8
 	xorq	%r8,%r12
 	shrdq	$6,%r14,%r14
 	vpsrlq	$7,%xmm10,%xmm10
 	xorq	%r11,%r15
 	addq	%r12,%r9
 	vpxor	%xmm9,%xmm8,%xmm8
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	vpsllq	$7,%xmm9,%xmm9
 	xorq	%r10,%r14
 	addq	%r13,%r9
 	vpxor	%xmm10,%xmm8,%xmm8
 	xorq	%r11,%rdi
 	shrdq	$28,%r14,%r14
 	vpsrlq	$6,%xmm0,%xmm11
 	addq	%r9,%rbx
 	addq	%rdi,%r9
 	vpxor	%xmm9,%xmm8,%xmm8
 	movq	%rbx,%r13
 	addq	%r9,%r14
 	vpsllq	$3,%xmm0,%xmm10
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r9
 	vpaddq	%xmm8,%xmm1,%xmm1
 	movq	%rcx,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$19,%xmm0,%xmm9
 	xorq	%rbx,%r13
 	xorq	%rdx,%r12
 	vpxor	%xmm10,%xmm11,%xmm11
 	shrdq	$4,%r13,%r13
 	xorq	%r9,%r14
 	vpsllq	$42,%xmm10,%xmm10
 	andq	%rbx,%r12
 	xorq	%rbx,%r13
 	vpxor	%xmm9,%xmm11,%xmm11
 	addq	24(%rsp),%r8
 	movq	%r9,%rdi
 	vpsrlq	$42,%xmm9,%xmm9
 	xorq	%rdx,%r12
 	shrdq	$6,%r14,%r14
 	vpxor	%xmm10,%xmm11,%xmm11
 	xorq	%r10,%rdi
 	addq	%r12,%r8
 	vpxor	%xmm9,%xmm11,%xmm11
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	vpaddq	%xmm11,%xmm1,%xmm1
 	xorq	%r9,%r14
 	addq	%r13,%r8
 	vpaddq	-96(%rbp),%xmm1,%xmm10
 	xorq	%r10,%r15
 	shrdq	$28,%r14,%r14
 	addq	%r8,%rax
 	addq	%r15,%r8
 	movq	%rax,%r13
 	addq	%r8,%r14
 	vmovdqa	%xmm10,16(%rsp)
 	vpalignr	$8,%xmm2,%xmm3,%xmm8
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r8
 	vpalignr	$8,%xmm6,%xmm7,%xmm11
 	movq	%rbx,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$1,%xmm8,%xmm10
 	xorq	%rax,%r13
 	xorq	%rcx,%r12
 	vpaddq	%xmm11,%xmm2,%xmm2
 	shrdq	$4,%r13,%r13
 	xorq	%r8,%r14
 	vpsrlq	$7,%xmm8,%xmm11
 	andq	%rax,%r12
 	xorq	%rax,%r13
 	vpsllq	$56,%xmm8,%xmm9
 	addq	32(%rsp),%rdx
 	movq	%r8,%r15
 	vpxor	%xmm10,%xmm11,%xmm8
 	xorq	%rcx,%r12
 	shrdq	$6,%r14,%r14
 	vpsrlq	$7,%xmm10,%xmm10
 	xorq	%r9,%r15
 	addq	%r12,%rdx
 	vpxor	%xmm9,%xmm8,%xmm8
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	vpsllq	$7,%xmm9,%xmm9
 	xorq	%r8,%r14
 	addq	%r13,%rdx
 	vpxor	%xmm10,%xmm8,%xmm8
 	xorq	%r9,%rdi
 	shrdq	$28,%r14,%r14
 	vpsrlq	$6,%xmm1,%xmm11
 	addq	%rdx,%r11
 	addq	%rdi,%rdx
 	vpxor	%xmm9,%xmm8,%xmm8
 	movq	%r11,%r13
 	addq	%rdx,%r14
 	vpsllq	$3,%xmm1,%xmm10
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rdx
 	vpaddq	%xmm8,%xmm2,%xmm2
 	movq	%rax,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$19,%xmm1,%xmm9
 	xorq	%r11,%r13
 	xorq	%rbx,%r12
 	vpxor	%xmm10,%xmm11,%xmm11
 	shrdq	$4,%r13,%r13
 	xorq	%rdx,%r14
 	vpsllq	$42,%xmm10,%xmm10
 	andq	%r11,%r12
 	xorq	%r11,%r13
 	vpxor	%xmm9,%xmm11,%xmm11
 	addq	40(%rsp),%rcx
 	movq	%rdx,%rdi
 	vpsrlq	$42,%xmm9,%xmm9
 	xorq	%rbx,%r12
 	shrdq	$6,%r14,%r14
 	vpxor	%xmm10,%xmm11,%xmm11
 	xorq	%r8,%rdi
 	addq	%r12,%rcx
 	vpxor	%xmm9,%xmm11,%xmm11
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	vpaddq	%xmm11,%xmm2,%xmm2
 	xorq	%rdx,%r14
 	addq	%r13,%rcx
 	vpaddq	-64(%rbp),%xmm2,%xmm10
 	xorq	%r8,%r15
 	shrdq	$28,%r14,%r14
 	addq	%rcx,%r10
 	addq	%r15,%rcx
 	movq	%r10,%r13
 	addq	%rcx,%r14
 	vmovdqa	%xmm10,32(%rsp)
 	vpalignr	$8,%xmm3,%xmm4,%xmm8
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rcx
 	vpalignr	$8,%xmm7,%xmm0,%xmm11
 	movq	%r11,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$1,%xmm8,%xmm10
 	xorq	%r10,%r13
 	xorq	%rax,%r12
 	vpaddq	%xmm11,%xmm3,%xmm3
 	shrdq	$4,%r13,%r13
 	xorq	%rcx,%r14
 	vpsrlq	$7,%xmm8,%xmm11
 	andq	%r10,%r12
 	xorq	%r10,%r13
 	vpsllq	$56,%xmm8,%xmm9
 	addq	48(%rsp),%rbx
 	movq	%rcx,%r15
 	vpxor	%xmm10,%xmm11,%xmm8
 	xorq	%rax,%r12
 	shrdq	$6,%r14,%r14
 	vpsrlq	$7,%xmm10,%xmm10
 	xorq	%rdx,%r15
 	addq	%r12,%rbx
 	vpxor	%xmm9,%xmm8,%xmm8
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	vpsllq	$7,%xmm9,%xmm9
 	xorq	%rcx,%r14
 	addq	%r13,%rbx
 	vpxor	%xmm10,%xmm8,%xmm8
 	xorq	%rdx,%rdi
 	shrdq	$28,%r14,%r14
 	vpsrlq	$6,%xmm2,%xmm11
 	addq	%rbx,%r9
 	addq	%rdi,%rbx
 	vpxor	%xmm9,%xmm8,%xmm8
 	movq	%r9,%r13
 	addq	%rbx,%r14
 	vpsllq	$3,%xmm2,%xmm10
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rbx
 	vpaddq	%xmm8,%xmm3,%xmm3
 	movq	%r10,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$19,%xmm2,%xmm9
 	xorq	%r9,%r13
 	xorq	%r11,%r12
 	vpxor	%xmm10,%xmm11,%xmm11
 	shrdq	$4,%r13,%r13
 	xorq	%rbx,%r14
 	vpsllq	$42,%xmm10,%xmm10
 	andq	%r9,%r12
 	xorq	%r9,%r13
 	vpxor	%xmm9,%xmm11,%xmm11
 	addq	56(%rsp),%rax
 	movq	%rbx,%rdi
 	vpsrlq	$42,%xmm9,%xmm9
 	xorq	%r11,%r12
 	shrdq	$6,%r14,%r14
 	vpxor	%xmm10,%xmm11,%xmm11
 	xorq	%rcx,%rdi
 	addq	%r12,%rax
 	vpxor	%xmm9,%xmm11,%xmm11
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	vpaddq	%xmm11,%xmm3,%xmm3
 	xorq	%rbx,%r14
 	addq	%r13,%rax
 	vpaddq	-32(%rbp),%xmm3,%xmm10
 	xorq	%rcx,%r15
 	shrdq	$28,%r14,%r14
 	addq	%rax,%r8
 	addq	%r15,%rax
 	movq	%r8,%r13
 	addq	%rax,%r14
 	vmovdqa	%xmm10,48(%rsp)
 	vpalignr	$8,%xmm4,%xmm5,%xmm8
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rax
 	vpalignr	$8,%xmm0,%xmm1,%xmm11
 	movq	%r9,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$1,%xmm8,%xmm10
 	xorq	%r8,%r13
 	xorq	%r10,%r12
 	vpaddq	%xmm11,%xmm4,%xmm4
 	shrdq	$4,%r13,%r13
 	xorq	%rax,%r14
 	vpsrlq	$7,%xmm8,%xmm11
 	andq	%r8,%r12
 	xorq	%r8,%r13
 	vpsllq	$56,%xmm8,%xmm9
 	addq	64(%rsp),%r11
 	movq	%rax,%r15
 	vpxor	%xmm10,%xmm11,%xmm8
 	xorq	%r10,%r12
 	shrdq	$6,%r14,%r14
 	vpsrlq	$7,%xmm10,%xmm10
 	xorq	%rbx,%r15
 	addq	%r12,%r11
 	vpxor	%xmm9,%xmm8,%xmm8
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	vpsllq	$7,%xmm9,%xmm9
 	xorq	%rax,%r14
 	addq	%r13,%r11
 	vpxor	%xmm10,%xmm8,%xmm8
 	xorq	%rbx,%rdi
 	shrdq	$28,%r14,%r14
 	vpsrlq	$6,%xmm3,%xmm11
 	addq	%r11,%rdx
 	addq	%rdi,%r11
 	vpxor	%xmm9,%xmm8,%xmm8
 	movq	%rdx,%r13
 	addq	%r11,%r14
 	vpsllq	$3,%xmm3,%xmm10
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r11
 	vpaddq	%xmm8,%xmm4,%xmm4
 	movq	%r8,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$19,%xmm3,%xmm9
 	xorq	%rdx,%r13
 	xorq	%r9,%r12
 	vpxor	%xmm10,%xmm11,%xmm11
 	shrdq	$4,%r13,%r13
 	xorq	%r11,%r14
 	vpsllq	$42,%xmm10,%xmm10
 	andq	%rdx,%r12
 	xorq	%rdx,%r13
 	vpxor	%xmm9,%xmm11,%xmm11
 	addq	72(%rsp),%r10
 	movq	%r11,%rdi
 	vpsrlq	$42,%xmm9,%xmm9
 	xorq	%r9,%r12
 	shrdq	$6,%r14,%r14
 	vpxor	%xmm10,%xmm11,%xmm11
 	xorq	%rax,%rdi
 	addq	%r12,%r10
 	vpxor	%xmm9,%xmm11,%xmm11
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	vpaddq	%xmm11,%xmm4,%xmm4
 	xorq	%r11,%r14
 	addq	%r13,%r10
 	vpaddq	0(%rbp),%xmm4,%xmm10
 	xorq	%rax,%r15
 	shrdq	$28,%r14,%r14
 	addq	%r10,%rcx
 	addq	%r15,%r10
 	movq	%rcx,%r13
 	addq	%r10,%r14
 	vmovdqa	%xmm10,64(%rsp)
 	vpalignr	$8,%xmm5,%xmm6,%xmm8
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r10
 	vpalignr	$8,%xmm1,%xmm2,%xmm11
 	movq	%rdx,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$1,%xmm8,%xmm10
 	xorq	%rcx,%r13
 	xorq	%r8,%r12
 	vpaddq	%xmm11,%xmm5,%xmm5
 	shrdq	$4,%r13,%r13
 	xorq	%r10,%r14
 	vpsrlq	$7,%xmm8,%xmm11
 	andq	%rcx,%r12
 	xorq	%rcx,%r13
 	vpsllq	$56,%xmm8,%xmm9
 	addq	80(%rsp),%r9
 	movq	%r10,%r15
 	vpxor	%xmm10,%xmm11,%xmm8
 	xorq	%r8,%r12
 	shrdq	$6,%r14,%r14
 	vpsrlq	$7,%xmm10,%xmm10
 	xorq	%r11,%r15
 	addq	%r12,%r9
 	vpxor	%xmm9,%xmm8,%xmm8
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	vpsllq	$7,%xmm9,%xmm9
 	xorq	%r10,%r14
 	addq	%r13,%r9
 	vpxor	%xmm10,%xmm8,%xmm8
 	xorq	%r11,%rdi
 	shrdq	$28,%r14,%r14
 	vpsrlq	$6,%xmm4,%xmm11
 	addq	%r9,%rbx
 	addq	%rdi,%r9
 	vpxor	%xmm9,%xmm8,%xmm8
 	movq	%rbx,%r13
 	addq	%r9,%r14
 	vpsllq	$3,%xmm4,%xmm10
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r9
 	vpaddq	%xmm8,%xmm5,%xmm5
 	movq	%rcx,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$19,%xmm4,%xmm9
 	xorq	%rbx,%r13
 	xorq	%rdx,%r12
 	vpxor	%xmm10,%xmm11,%xmm11
 	shrdq	$4,%r13,%r13
 	xorq	%r9,%r14
 	vpsllq	$42,%xmm10,%xmm10
 	andq	%rbx,%r12
 	xorq	%rbx,%r13
 	vpxor	%xmm9,%xmm11,%xmm11
 	addq	88(%rsp),%r8
 	movq	%r9,%rdi
 	vpsrlq	$42,%xmm9,%xmm9
 	xorq	%rdx,%r12
 	shrdq	$6,%r14,%r14
 	vpxor	%xmm10,%xmm11,%xmm11
 	xorq	%r10,%rdi
 	addq	%r12,%r8
 	vpxor	%xmm9,%xmm11,%xmm11
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	vpaddq	%xmm11,%xmm5,%xmm5
 	xorq	%r9,%r14
 	addq	%r13,%r8
 	vpaddq	32(%rbp),%xmm5,%xmm10
 	xorq	%r10,%r15
 	shrdq	$28,%r14,%r14
 	addq	%r8,%rax
 	addq	%r15,%r8
 	movq	%rax,%r13
 	addq	%r8,%r14
 	vmovdqa	%xmm10,80(%rsp)
 	vpalignr	$8,%xmm6,%xmm7,%xmm8
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r8
 	vpalignr	$8,%xmm2,%xmm3,%xmm11
 	movq	%rbx,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$1,%xmm8,%xmm10
 	xorq	%rax,%r13
 	xorq	%rcx,%r12
 	vpaddq	%xmm11,%xmm6,%xmm6
 	shrdq	$4,%r13,%r13
 	xorq	%r8,%r14
 	vpsrlq	$7,%xmm8,%xmm11
 	andq	%rax,%r12
 	xorq	%rax,%r13
 	vpsllq	$56,%xmm8,%xmm9
 	addq	96(%rsp),%rdx
 	movq	%r8,%r15
 	vpxor	%xmm10,%xmm11,%xmm8
 	xorq	%rcx,%r12
 	shrdq	$6,%r14,%r14
 	vpsrlq	$7,%xmm10,%xmm10
 	xorq	%r9,%r15
 	addq	%r12,%rdx
 	vpxor	%xmm9,%xmm8,%xmm8
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	vpsllq	$7,%xmm9,%xmm9
 	xorq	%r8,%r14
 	addq	%r13,%rdx
 	vpxor	%xmm10,%xmm8,%xmm8
 	xorq	%r9,%rdi
 	shrdq	$28,%r14,%r14
 	vpsrlq	$6,%xmm5,%xmm11
 	addq	%rdx,%r11
 	addq	%rdi,%rdx
 	vpxor	%xmm9,%xmm8,%xmm8
 	movq	%r11,%r13
 	addq	%rdx,%r14
 	vpsllq	$3,%xmm5,%xmm10
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rdx
 	vpaddq	%xmm8,%xmm6,%xmm6
 	movq	%rax,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$19,%xmm5,%xmm9
 	xorq	%r11,%r13
 	xorq	%rbx,%r12
 	vpxor	%xmm10,%xmm11,%xmm11
 	shrdq	$4,%r13,%r13
 	xorq	%rdx,%r14
 	vpsllq	$42,%xmm10,%xmm10
 	andq	%r11,%r12
 	xorq	%r11,%r13
 	vpxor	%xmm9,%xmm11,%xmm11
 	addq	104(%rsp),%rcx
 	movq	%rdx,%rdi
 	vpsrlq	$42,%xmm9,%xmm9
 	xorq	%rbx,%r12
 	shrdq	$6,%r14,%r14
 	vpxor	%xmm10,%xmm11,%xmm11
 	xorq	%r8,%rdi
 	addq	%r12,%rcx
 	vpxor	%xmm9,%xmm11,%xmm11
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	vpaddq	%xmm11,%xmm6,%xmm6
 	xorq	%rdx,%r14
 	addq	%r13,%rcx
 	vpaddq	64(%rbp),%xmm6,%xmm10
 	xorq	%r8,%r15
 	shrdq	$28,%r14,%r14
 	addq	%rcx,%r10
 	addq	%r15,%rcx
 	movq	%r10,%r13
 	addq	%rcx,%r14
 	vmovdqa	%xmm10,96(%rsp)
 	vpalignr	$8,%xmm7,%xmm0,%xmm8
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rcx
 	vpalignr	$8,%xmm3,%xmm4,%xmm11
 	movq	%r11,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$1,%xmm8,%xmm10
 	xorq	%r10,%r13
 	xorq	%rax,%r12
 	vpaddq	%xmm11,%xmm7,%xmm7
 	shrdq	$4,%r13,%r13
 	xorq	%rcx,%r14
 	vpsrlq	$7,%xmm8,%xmm11
 	andq	%r10,%r12
 	xorq	%r10,%r13
 	vpsllq	$56,%xmm8,%xmm9
 	addq	112(%rsp),%rbx
 	movq	%rcx,%r15
 	vpxor	%xmm10,%xmm11,%xmm8
 	xorq	%rax,%r12
 	shrdq	$6,%r14,%r14
 	vpsrlq	$7,%xmm10,%xmm10
 	xorq	%rdx,%r15
 	addq	%r12,%rbx
 	vpxor	%xmm9,%xmm8,%xmm8
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	vpsllq	$7,%xmm9,%xmm9
 	xorq	%rcx,%r14
 	addq	%r13,%rbx
 	vpxor	%xmm10,%xmm8,%xmm8
 	xorq	%rdx,%rdi
 	shrdq	$28,%r14,%r14
 	vpsrlq	$6,%xmm6,%xmm11
 	addq	%rbx,%r9
 	addq	%rdi,%rbx
 	vpxor	%xmm9,%xmm8,%xmm8
 	movq	%r9,%r13
 	addq	%rbx,%r14
 	vpsllq	$3,%xmm6,%xmm10
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rbx
 	vpaddq	%xmm8,%xmm7,%xmm7
 	movq	%r10,%r12
 	shrdq	$5,%r14,%r14
 	vpsrlq	$19,%xmm6,%xmm9
 	xorq	%r9,%r13
 	xorq	%r11,%r12
 	vpxor	%xmm10,%xmm11,%xmm11
 	shrdq	$4,%r13,%r13
 	xorq	%rbx,%r14
 	vpsllq	$42,%xmm10,%xmm10
 	andq	%r9,%r12
 	xorq	%r9,%r13
 	vpxor	%xmm9,%xmm11,%xmm11
 	addq	120(%rsp),%rax
 	movq	%rbx,%rdi
 	vpsrlq	$42,%xmm9,%xmm9
 	xorq	%r11,%r12
 	shrdq	$6,%r14,%r14
 	vpxor	%xmm10,%xmm11,%xmm11
 	xorq	%rcx,%rdi
 	addq	%r12,%rax
 	vpxor	%xmm9,%xmm11,%xmm11
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	vpaddq	%xmm11,%xmm7,%xmm7
 	xorq	%rbx,%r14
 	addq	%r13,%rax
 	vpaddq	96(%rbp),%xmm7,%xmm10
 	xorq	%rcx,%r15
 	shrdq	$28,%r14,%r14
 	addq	%rax,%r8
 	addq	%r15,%rax
 	movq	%r8,%r13
 	addq	%rax,%r14
 	vmovdqa	%xmm10,112(%rsp)
 	cmpb	$0,135(%rbp)
 	jne	.Lavx_00_47
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rax
 	movq	%r9,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%r8,%r13
 	xorq	%r10,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%rax,%r14
 	andq	%r8,%r12
 	xorq	%r8,%r13
 	addq	0(%rsp),%r11
 	movq	%rax,%r15
 	xorq	%r10,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%rbx,%r15
 	addq	%r12,%r11
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	xorq	%rax,%r14
 	addq	%r13,%r11
 	xorq	%rbx,%rdi
 	shrdq	$28,%r14,%r14
 	addq	%r11,%rdx
 	addq	%rdi,%r11
 	movq	%rdx,%r13
 	addq	%r11,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r11
 	movq	%r8,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%rdx,%r13
 	xorq	%r9,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%r11,%r14
 	andq	%rdx,%r12
 	xorq	%rdx,%r13
 	addq	8(%rsp),%r10
 	movq	%r11,%rdi
 	xorq	%r9,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%rax,%rdi
 	addq	%r12,%r10
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	xorq	%r11,%r14
 	addq	%r13,%r10
 	xorq	%rax,%r15
 	shrdq	$28,%r14,%r14
 	addq	%r10,%rcx
 	addq	%r15,%r10
 	movq	%rcx,%r13
 	addq	%r10,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r10
 	movq	%rdx,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%rcx,%r13
 	xorq	%r8,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%r10,%r14
 	andq	%rcx,%r12
 	xorq	%rcx,%r13
 	addq	16(%rsp),%r9
 	movq	%r10,%r15
 	xorq	%r8,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%r11,%r15
 	addq	%r12,%r9
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	xorq	%r10,%r14
 	addq	%r13,%r9
 	xorq	%r11,%rdi
 	shrdq	$28,%r14,%r14
 	addq	%r9,%rbx
 	addq	%rdi,%r9
 	movq	%rbx,%r13
 	addq	%r9,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r9
 	movq	%rcx,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%rbx,%r13
 	xorq	%rdx,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%r9,%r14
 	andq	%rbx,%r12
 	xorq	%rbx,%r13
 	addq	24(%rsp),%r8
 	movq	%r9,%rdi
 	xorq	%rdx,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%r10,%rdi
 	addq	%r12,%r8
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	xorq	%r9,%r14
 	addq	%r13,%r8
 	xorq	%r10,%r15
 	shrdq	$28,%r14,%r14
 	addq	%r8,%rax
 	addq	%r15,%r8
 	movq	%rax,%r13
 	addq	%r8,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r8
 	movq	%rbx,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%rax,%r13
 	xorq	%rcx,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%r8,%r14
 	andq	%rax,%r12
 	xorq	%rax,%r13
 	addq	32(%rsp),%rdx
 	movq	%r8,%r15
 	xorq	%rcx,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%r9,%r15
 	addq	%r12,%rdx
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	xorq	%r8,%r14
 	addq	%r13,%rdx
 	xorq	%r9,%rdi
 	shrdq	$28,%r14,%r14
 	addq	%rdx,%r11
 	addq	%rdi,%rdx
 	movq	%r11,%r13
 	addq	%rdx,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rdx
 	movq	%rax,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%r11,%r13
 	xorq	%rbx,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%rdx,%r14
 	andq	%r11,%r12
 	xorq	%r11,%r13
 	addq	40(%rsp),%rcx
 	movq	%rdx,%rdi
 	xorq	%rbx,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%r8,%rdi
 	addq	%r12,%rcx
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	xorq	%rdx,%r14
 	addq	%r13,%rcx
 	xorq	%r8,%r15
 	shrdq	$28,%r14,%r14
 	addq	%rcx,%r10
 	addq	%r15,%rcx
 	movq	%r10,%r13
 	addq	%rcx,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rcx
 	movq	%r11,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%r10,%r13
 	xorq	%rax,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%rcx,%r14
 	andq	%r10,%r12
 	xorq	%r10,%r13
 	addq	48(%rsp),%rbx
 	movq	%rcx,%r15
 	xorq	%rax,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%rdx,%r15
 	addq	%r12,%rbx
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	xorq	%rcx,%r14
 	addq	%r13,%rbx
 	xorq	%rdx,%rdi
 	shrdq	$28,%r14,%r14
 	addq	%rbx,%r9
 	addq	%rdi,%rbx
 	movq	%r9,%r13
 	addq	%rbx,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rbx
 	movq	%r10,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%r9,%r13
 	xorq	%r11,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%rbx,%r14
 	andq	%r9,%r12
 	xorq	%r9,%r13
 	addq	56(%rsp),%rax
 	movq	%rbx,%rdi
 	xorq	%r11,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%rcx,%rdi
 	addq	%r12,%rax
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	xorq	%rbx,%r14
 	addq	%r13,%rax
 	xorq	%rcx,%r15
 	shrdq	$28,%r14,%r14
 	addq	%rax,%r8
 	addq	%r15,%rax
 	movq	%r8,%r13
 	addq	%rax,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rax
 	movq	%r9,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%r8,%r13
 	xorq	%r10,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%rax,%r14
 	andq	%r8,%r12
 	xorq	%r8,%r13
 	addq	64(%rsp),%r11
 	movq	%rax,%r15
 	xorq	%r10,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%rbx,%r15
 	addq	%r12,%r11
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	xorq	%rax,%r14
 	addq	%r13,%r11
 	xorq	%rbx,%rdi
 	shrdq	$28,%r14,%r14
 	addq	%r11,%rdx
 	addq	%rdi,%r11
 	movq	%rdx,%r13
 	addq	%r11,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r11
 	movq	%r8,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%rdx,%r13
 	xorq	%r9,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%r11,%r14
 	andq	%rdx,%r12
 	xorq	%rdx,%r13
 	addq	72(%rsp),%r10
 	movq	%r11,%rdi
 	xorq	%r9,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%rax,%rdi
 	addq	%r12,%r10
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	xorq	%r11,%r14
 	addq	%r13,%r10
 	xorq	%rax,%r15
 	shrdq	$28,%r14,%r14
 	addq	%r10,%rcx
 	addq	%r15,%r10
 	movq	%rcx,%r13
 	addq	%r10,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r10
 	movq	%rdx,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%rcx,%r13
 	xorq	%r8,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%r10,%r14
 	andq	%rcx,%r12
 	xorq	%rcx,%r13
 	addq	80(%rsp),%r9
 	movq	%r10,%r15
 	xorq	%r8,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%r11,%r15
 	addq	%r12,%r9
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	xorq	%r10,%r14
 	addq	%r13,%r9
 	xorq	%r11,%rdi
 	shrdq	$28,%r14,%r14
 	addq	%r9,%rbx
 	addq	%rdi,%r9
 	movq	%rbx,%r13
 	addq	%r9,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r9
 	movq	%rcx,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%rbx,%r13
 	xorq	%rdx,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%r9,%r14
 	andq	%rbx,%r12
 	xorq	%rbx,%r13
 	addq	88(%rsp),%r8
 	movq	%r9,%rdi
 	xorq	%rdx,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%r10,%rdi
 	addq	%r12,%r8
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	xorq	%r9,%r14
 	addq	%r13,%r8
 	xorq	%r10,%r15
 	shrdq	$28,%r14,%r14
 	addq	%r8,%rax
 	addq	%r15,%r8
 	movq	%rax,%r13
 	addq	%r8,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%r8
 	movq	%rbx,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%rax,%r13
 	xorq	%rcx,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%r8,%r14
 	andq	%rax,%r12
 	xorq	%rax,%r13
 	addq	96(%rsp),%rdx
 	movq	%r8,%r15
 	xorq	%rcx,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%r9,%r15
 	addq	%r12,%rdx
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	xorq	%r8,%r14
 	addq	%r13,%rdx
 	xorq	%r9,%rdi
 	shrdq	$28,%r14,%r14
 	addq	%rdx,%r11
 	addq	%rdi,%rdx
 	movq	%r11,%r13
 	addq	%rdx,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rdx
 	movq	%rax,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%r11,%r13
 	xorq	%rbx,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%rdx,%r14
 	andq	%r11,%r12
 	xorq	%r11,%r13
 	addq	104(%rsp),%rcx
 	movq	%rdx,%rdi
 	xorq	%rbx,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%r8,%rdi
 	addq	%r12,%rcx
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	xorq	%rdx,%r14
 	addq	%r13,%rcx
 	xorq	%r8,%r15
 	shrdq	$28,%r14,%r14
 	addq	%rcx,%r10
 	addq	%r15,%rcx
 	movq	%r10,%r13
 	addq	%rcx,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rcx
 	movq	%r11,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%r10,%r13
 	xorq	%rax,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%rcx,%r14
 	andq	%r10,%r12
 	xorq	%r10,%r13
 	addq	112(%rsp),%rbx
 	movq	%rcx,%r15
 	xorq	%rax,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%rdx,%r15
 	addq	%r12,%rbx
 	shrdq	$14,%r13,%r13
 	andq	%r15,%rdi
 	xorq	%rcx,%r14
 	addq	%r13,%rbx
 	xorq	%rdx,%rdi
 	shrdq	$28,%r14,%r14
 	addq	%rbx,%r9
 	addq	%rdi,%rbx
 	movq	%r9,%r13
 	addq	%rbx,%r14
 	shrdq	$23,%r13,%r13
 	movq	%r14,%rbx
 	movq	%r10,%r12
 	shrdq	$5,%r14,%r14
 	xorq	%r9,%r13
 	xorq	%r11,%r12
 	shrdq	$4,%r13,%r13
 	xorq	%rbx,%r14
 	andq	%r9,%r12
 	xorq	%r9,%r13
 	addq	120(%rsp),%rax
 	movq	%rbx,%rdi
 	xorq	%r11,%r12
 	shrdq	$6,%r14,%r14
 	xorq	%rcx,%rdi
 	addq	%r12,%rax
 	shrdq	$14,%r13,%r13
 	andq	%rdi,%r15
 	xorq	%rbx,%r14
 	addq	%r13,%rax
 	xorq	%rcx,%r15
 	shrdq	$28,%r14,%r14
 	addq	%rax,%r8
 	addq	%r15,%rax
 	movq	%r8,%r13
 	addq	%rax,%r14
 	movq	128+0(%rsp),%rdi
 	movq	%r14,%rax
 
 	addq	0(%rdi),%rax
 	leaq	128(%rsi),%rsi
 	addq	8(%rdi),%rbx
 	addq	16(%rdi),%rcx
 	addq	24(%rdi),%rdx
 	addq	32(%rdi),%r8
 	addq	40(%rdi),%r9
 	addq	48(%rdi),%r10
 	addq	56(%rdi),%r11
 
 	cmpq	128+16(%rsp),%rsi
 
 	movq	%rax,0(%rdi)
 	movq	%rbx,8(%rdi)
 	movq	%rcx,16(%rdi)
 	movq	%rdx,24(%rdi)
 	movq	%r8,32(%rdi)
 	movq	%r9,40(%rdi)
 	movq	%r10,48(%rdi)
 	movq	%r11,56(%rdi)
 	jb	.Lloop_avx
 
 	movq	152(%rsp),%rsi
 .cfi_def_cfa	%rsi,8
 	vzeroupper
 	movq	-48(%rsi),%r15
 .cfi_restore	%r15
 	movq	-40(%rsi),%r14
 .cfi_restore	%r14
 	movq	-32(%rsi),%r13
 .cfi_restore	%r13
 	movq	-24(%rsi),%r12
 .cfi_restore	%r12
 	movq	-16(%rsi),%rbp
 .cfi_restore	%rbp
 	movq	-8(%rsi),%rbx
 .cfi_restore	%rbx
 	leaq	(%rsi),%rsp
 .cfi_def_cfa_register	%rsp
 .Lepilogue_avx:
 	RET
 .cfi_endproc
 SET_SIZE(zfs_sha512_transform_avx)
+STACK_FRAME_NON_STANDARD zfs_sha512_transform_avx
 
 ENTRY_ALIGN(zfs_sha512_transform_avx2, 64)
 .cfi_startproc
 	ENDBR
 	movq	%rsp,%rax
 .cfi_def_cfa_register	%rax
 	pushq	%rbx
 .cfi_offset	%rbx,-16
 	pushq	%rbp
 .cfi_offset	%rbp,-24
 	pushq	%r12
 .cfi_offset	%r12,-32
 	pushq	%r13
 .cfi_offset	%r13,-40
 	pushq	%r14
 .cfi_offset	%r14,-48
 	pushq	%r15
 .cfi_offset	%r15,-56
 	subq	$1312,%rsp
 	shlq	$4,%rdx
 	andq	$-2048,%rsp
 	leaq	(%rsi,%rdx,8),%rdx
 	addq	$1152,%rsp
 	movq	%rdi,128+0(%rsp)
 	movq	%rsi,128+8(%rsp)
 	movq	%rdx,128+16(%rsp)
 	movq	%rax,152(%rsp)
 .cfi_escape	0x0f,0x06,0x77,0x98,0x01,0x06,0x23,0x08
 .Lprologue_avx2:
 
 	vzeroupper
 	subq	$-128,%rsi
 	movq	0(%rdi),%rax
 	movq	%rsi,%r12
 	movq	8(%rdi),%rbx
 	cmpq	%rdx,%rsi
 	movq	16(%rdi),%rcx
 	cmoveq	%rsp,%r12
 	movq	24(%rdi),%rdx
 	movq	32(%rdi),%r8
 	movq	40(%rdi),%r9
 	movq	48(%rdi),%r10
 	movq	56(%rdi),%r11
 	jmp	.Loop_avx2
 .balign	16
 .Loop_avx2:
 	vmovdqu	-128(%rsi),%xmm0
 	vmovdqu	-128+16(%rsi),%xmm1
 	vmovdqu	-128+32(%rsi),%xmm2
 	leaq	K512+128(%rip),%rbp
 	vmovdqu	-128+48(%rsi),%xmm3
 	vmovdqu	-128+64(%rsi),%xmm4
 	vmovdqu	-128+80(%rsi),%xmm5
 	vmovdqu	-128+96(%rsi),%xmm6
 	vmovdqu	-128+112(%rsi),%xmm7
 
 	vmovdqa	1152(%rbp),%ymm10
 	vinserti128	$1,(%r12),%ymm0,%ymm0
 	vinserti128	$1,16(%r12),%ymm1,%ymm1
 	vpshufb	%ymm10,%ymm0,%ymm0
 	vinserti128	$1,32(%r12),%ymm2,%ymm2
 	vpshufb	%ymm10,%ymm1,%ymm1
 	vinserti128	$1,48(%r12),%ymm3,%ymm3
 	vpshufb	%ymm10,%ymm2,%ymm2
 	vinserti128	$1,64(%r12),%ymm4,%ymm4
 	vpshufb	%ymm10,%ymm3,%ymm3
 	vinserti128	$1,80(%r12),%ymm5,%ymm5
 	vpshufb	%ymm10,%ymm4,%ymm4
 	vinserti128	$1,96(%r12),%ymm6,%ymm6
 	vpshufb	%ymm10,%ymm5,%ymm5
 	vinserti128	$1,112(%r12),%ymm7,%ymm7
 
 	vpaddq	-128(%rbp),%ymm0,%ymm8
 	vpshufb	%ymm10,%ymm6,%ymm6
 	vpaddq	-96(%rbp),%ymm1,%ymm9
 	vpshufb	%ymm10,%ymm7,%ymm7
 	vpaddq	-64(%rbp),%ymm2,%ymm10
 	vpaddq	-32(%rbp),%ymm3,%ymm11
 	vmovdqa	%ymm8,0(%rsp)
 	vpaddq	0(%rbp),%ymm4,%ymm8
 	vmovdqa	%ymm9,32(%rsp)
 	vpaddq	32(%rbp),%ymm5,%ymm9
 	vmovdqa	%ymm10,64(%rsp)
 	vpaddq	64(%rbp),%ymm6,%ymm10
 	vmovdqa	%ymm11,96(%rsp)
 
 	movq	152(%rsp),%rdi
 .cfi_def_cfa	%rdi,8
 	leaq	-128(%rsp),%rsp
 
 
 
 	movq	%rdi,-8(%rsp)
 .cfi_escape	0x0f,0x05,0x77,0x78,0x06,0x23,0x08
 	vpaddq	96(%rbp),%ymm7,%ymm11
 	vmovdqa	%ymm8,0(%rsp)
 	xorq	%r14,%r14
 	vmovdqa	%ymm9,32(%rsp)
 	movq	%rbx,%rdi
 	vmovdqa	%ymm10,64(%rsp)
 	xorq	%rcx,%rdi
 	vmovdqa	%ymm11,96(%rsp)
 	movq	%r9,%r12
 	addq	$32*8,%rbp
 	jmp	.Lavx2_00_47
 
 .balign	16
 .Lavx2_00_47:
 	leaq	-128(%rsp),%rsp
 .cfi_escape	0x0f,0x06,0x77,0xf8,0x00,0x06,0x23,0x08
 
 	pushq	128-8(%rsp)
 .cfi_escape	0x0f,0x05,0x77,0x00,0x06,0x23,0x08
 	leaq	8(%rsp),%rsp
 .cfi_escape	0x0f,0x05,0x77,0x78,0x06,0x23,0x08
 	vpalignr	$8,%ymm0,%ymm1,%ymm8
 	addq	0+256(%rsp),%r11
 	andq	%r8,%r12
 	rorxq	$41,%r8,%r13
 	vpalignr	$8,%ymm4,%ymm5,%ymm11
 	rorxq	$18,%r8,%r15
 	leaq	(%rax,%r14,1),%rax
 	leaq	(%r11,%r12,1),%r11
 	vpsrlq	$1,%ymm8,%ymm10
 	andnq	%r10,%r8,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%r8,%r14
 	vpaddq	%ymm11,%ymm0,%ymm0
 	vpsrlq	$7,%ymm8,%ymm11
 	leaq	(%r11,%r12,1),%r11
 	xorq	%r14,%r13
 	movq	%rax,%r15
 	vpsllq	$56,%ymm8,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm8
 	rorxq	$39,%rax,%r12
 	leaq	(%r11,%r13,1),%r11
 	xorq	%rbx,%r15
 	vpsrlq	$7,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm8,%ymm8
 	rorxq	$34,%rax,%r14
 	rorxq	$28,%rax,%r13
 	leaq	(%rdx,%r11,1),%rdx
 	vpsllq	$7,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm8,%ymm8
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%rbx,%rdi
 	vpsrlq	$6,%ymm7,%ymm11
 	vpxor	%ymm9,%ymm8,%ymm8
 	xorq	%r13,%r14
 	leaq	(%r11,%rdi,1),%r11
 	movq	%r8,%r12
 	vpsllq	$3,%ymm7,%ymm10
 	vpaddq	%ymm8,%ymm0,%ymm0
 	addq	8+256(%rsp),%r10
 	andq	%rdx,%r12
 	rorxq	$41,%rdx,%r13
 	vpsrlq	$19,%ymm7,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	rorxq	$18,%rdx,%rdi
 	leaq	(%r11,%r14,1),%r11
 	leaq	(%r10,%r12,1),%r10
 	vpsllq	$42,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm11,%ymm11
 	andnq	%r9,%rdx,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%rdx,%r14
 	vpsrlq	$42,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	leaq	(%r10,%r12,1),%r10
 	xorq	%r14,%r13
 	movq	%r11,%rdi
 	vpxor	%ymm9,%ymm11,%ymm11
 	rorxq	$39,%r11,%r12
 	leaq	(%r10,%r13,1),%r10
 	xorq	%rax,%rdi
 	vpaddq	%ymm11,%ymm0,%ymm0
 	rorxq	$34,%r11,%r14
 	rorxq	$28,%r11,%r13
 	leaq	(%rcx,%r10,1),%rcx
 	vpaddq	-128(%rbp),%ymm0,%ymm10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%rax,%r15
 	xorq	%r13,%r14
 	leaq	(%r10,%r15,1),%r10
 	movq	%rdx,%r12
 	vmovdqa	%ymm10,0(%rsp)
 	vpalignr	$8,%ymm1,%ymm2,%ymm8
 	addq	32+256(%rsp),%r9
 	andq	%rcx,%r12
 	rorxq	$41,%rcx,%r13
 	vpalignr	$8,%ymm5,%ymm6,%ymm11
 	rorxq	$18,%rcx,%r15
 	leaq	(%r10,%r14,1),%r10
 	leaq	(%r9,%r12,1),%r9
 	vpsrlq	$1,%ymm8,%ymm10
 	andnq	%r8,%rcx,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%rcx,%r14
 	vpaddq	%ymm11,%ymm1,%ymm1
 	vpsrlq	$7,%ymm8,%ymm11
 	leaq	(%r9,%r12,1),%r9
 	xorq	%r14,%r13
 	movq	%r10,%r15
 	vpsllq	$56,%ymm8,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm8
 	rorxq	$39,%r10,%r12
 	leaq	(%r9,%r13,1),%r9
 	xorq	%r11,%r15
 	vpsrlq	$7,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm8,%ymm8
 	rorxq	$34,%r10,%r14
 	rorxq	$28,%r10,%r13
 	leaq	(%rbx,%r9,1),%rbx
 	vpsllq	$7,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm8,%ymm8
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%r11,%rdi
 	vpsrlq	$6,%ymm0,%ymm11
 	vpxor	%ymm9,%ymm8,%ymm8
 	xorq	%r13,%r14
 	leaq	(%r9,%rdi,1),%r9
 	movq	%rcx,%r12
 	vpsllq	$3,%ymm0,%ymm10
 	vpaddq	%ymm8,%ymm1,%ymm1
 	addq	40+256(%rsp),%r8
 	andq	%rbx,%r12
 	rorxq	$41,%rbx,%r13
 	vpsrlq	$19,%ymm0,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	rorxq	$18,%rbx,%rdi
 	leaq	(%r9,%r14,1),%r9
 	leaq	(%r8,%r12,1),%r8
 	vpsllq	$42,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm11,%ymm11
 	andnq	%rdx,%rbx,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%rbx,%r14
 	vpsrlq	$42,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	leaq	(%r8,%r12,1),%r8
 	xorq	%r14,%r13
 	movq	%r9,%rdi
 	vpxor	%ymm9,%ymm11,%ymm11
 	rorxq	$39,%r9,%r12
 	leaq	(%r8,%r13,1),%r8
 	xorq	%r10,%rdi
 	vpaddq	%ymm11,%ymm1,%ymm1
 	rorxq	$34,%r9,%r14
 	rorxq	$28,%r9,%r13
 	leaq	(%rax,%r8,1),%rax
 	vpaddq	-96(%rbp),%ymm1,%ymm10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%r10,%r15
 	xorq	%r13,%r14
 	leaq	(%r8,%r15,1),%r8
 	movq	%rbx,%r12
 	vmovdqa	%ymm10,32(%rsp)
 	vpalignr	$8,%ymm2,%ymm3,%ymm8
 	addq	64+256(%rsp),%rdx
 	andq	%rax,%r12
 	rorxq	$41,%rax,%r13
 	vpalignr	$8,%ymm6,%ymm7,%ymm11
 	rorxq	$18,%rax,%r15
 	leaq	(%r8,%r14,1),%r8
 	leaq	(%rdx,%r12,1),%rdx
 	vpsrlq	$1,%ymm8,%ymm10
 	andnq	%rcx,%rax,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%rax,%r14
 	vpaddq	%ymm11,%ymm2,%ymm2
 	vpsrlq	$7,%ymm8,%ymm11
 	leaq	(%rdx,%r12,1),%rdx
 	xorq	%r14,%r13
 	movq	%r8,%r15
 	vpsllq	$56,%ymm8,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm8
 	rorxq	$39,%r8,%r12
 	leaq	(%rdx,%r13,1),%rdx
 	xorq	%r9,%r15
 	vpsrlq	$7,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm8,%ymm8
 	rorxq	$34,%r8,%r14
 	rorxq	$28,%r8,%r13
 	leaq	(%r11,%rdx,1),%r11
 	vpsllq	$7,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm8,%ymm8
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%r9,%rdi
 	vpsrlq	$6,%ymm1,%ymm11
 	vpxor	%ymm9,%ymm8,%ymm8
 	xorq	%r13,%r14
 	leaq	(%rdx,%rdi,1),%rdx
 	movq	%rax,%r12
 	vpsllq	$3,%ymm1,%ymm10
 	vpaddq	%ymm8,%ymm2,%ymm2
 	addq	72+256(%rsp),%rcx
 	andq	%r11,%r12
 	rorxq	$41,%r11,%r13
 	vpsrlq	$19,%ymm1,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	rorxq	$18,%r11,%rdi
 	leaq	(%rdx,%r14,1),%rdx
 	leaq	(%rcx,%r12,1),%rcx
 	vpsllq	$42,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm11,%ymm11
 	andnq	%rbx,%r11,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%r11,%r14
 	vpsrlq	$42,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	leaq	(%rcx,%r12,1),%rcx
 	xorq	%r14,%r13
 	movq	%rdx,%rdi
 	vpxor	%ymm9,%ymm11,%ymm11
 	rorxq	$39,%rdx,%r12
 	leaq	(%rcx,%r13,1),%rcx
 	xorq	%r8,%rdi
 	vpaddq	%ymm11,%ymm2,%ymm2
 	rorxq	$34,%rdx,%r14
 	rorxq	$28,%rdx,%r13
 	leaq	(%r10,%rcx,1),%r10
 	vpaddq	-64(%rbp),%ymm2,%ymm10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%r8,%r15
 	xorq	%r13,%r14
 	leaq	(%rcx,%r15,1),%rcx
 	movq	%r11,%r12
 	vmovdqa	%ymm10,64(%rsp)
 	vpalignr	$8,%ymm3,%ymm4,%ymm8
 	addq	96+256(%rsp),%rbx
 	andq	%r10,%r12
 	rorxq	$41,%r10,%r13
 	vpalignr	$8,%ymm7,%ymm0,%ymm11
 	rorxq	$18,%r10,%r15
 	leaq	(%rcx,%r14,1),%rcx
 	leaq	(%rbx,%r12,1),%rbx
 	vpsrlq	$1,%ymm8,%ymm10
 	andnq	%rax,%r10,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%r10,%r14
 	vpaddq	%ymm11,%ymm3,%ymm3
 	vpsrlq	$7,%ymm8,%ymm11
 	leaq	(%rbx,%r12,1),%rbx
 	xorq	%r14,%r13
 	movq	%rcx,%r15
 	vpsllq	$56,%ymm8,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm8
 	rorxq	$39,%rcx,%r12
 	leaq	(%rbx,%r13,1),%rbx
 	xorq	%rdx,%r15
 	vpsrlq	$7,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm8,%ymm8
 	rorxq	$34,%rcx,%r14
 	rorxq	$28,%rcx,%r13
 	leaq	(%r9,%rbx,1),%r9
 	vpsllq	$7,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm8,%ymm8
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%rdx,%rdi
 	vpsrlq	$6,%ymm2,%ymm11
 	vpxor	%ymm9,%ymm8,%ymm8
 	xorq	%r13,%r14
 	leaq	(%rbx,%rdi,1),%rbx
 	movq	%r10,%r12
 	vpsllq	$3,%ymm2,%ymm10
 	vpaddq	%ymm8,%ymm3,%ymm3
 	addq	104+256(%rsp),%rax
 	andq	%r9,%r12
 	rorxq	$41,%r9,%r13
 	vpsrlq	$19,%ymm2,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	rorxq	$18,%r9,%rdi
 	leaq	(%rbx,%r14,1),%rbx
 	leaq	(%rax,%r12,1),%rax
 	vpsllq	$42,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm11,%ymm11
 	andnq	%r11,%r9,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%r9,%r14
 	vpsrlq	$42,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	leaq	(%rax,%r12,1),%rax
 	xorq	%r14,%r13
 	movq	%rbx,%rdi
 	vpxor	%ymm9,%ymm11,%ymm11
 	rorxq	$39,%rbx,%r12
 	leaq	(%rax,%r13,1),%rax
 	xorq	%rcx,%rdi
 	vpaddq	%ymm11,%ymm3,%ymm3
 	rorxq	$34,%rbx,%r14
 	rorxq	$28,%rbx,%r13
 	leaq	(%r8,%rax,1),%r8
 	vpaddq	-32(%rbp),%ymm3,%ymm10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%rcx,%r15
 	xorq	%r13,%r14
 	leaq	(%rax,%r15,1),%rax
 	movq	%r9,%r12
 	vmovdqa	%ymm10,96(%rsp)
 	leaq	-128(%rsp),%rsp
 .cfi_escape	0x0f,0x06,0x77,0xf8,0x00,0x06,0x23,0x08
 
 	pushq	128-8(%rsp)
 .cfi_escape	0x0f,0x05,0x77,0x00,0x06,0x23,0x08
 	leaq	8(%rsp),%rsp
 .cfi_escape	0x0f,0x05,0x77,0x78,0x06,0x23,0x08
 	vpalignr	$8,%ymm4,%ymm5,%ymm8
 	addq	0+256(%rsp),%r11
 	andq	%r8,%r12
 	rorxq	$41,%r8,%r13
 	vpalignr	$8,%ymm0,%ymm1,%ymm11
 	rorxq	$18,%r8,%r15
 	leaq	(%rax,%r14,1),%rax
 	leaq	(%r11,%r12,1),%r11
 	vpsrlq	$1,%ymm8,%ymm10
 	andnq	%r10,%r8,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%r8,%r14
 	vpaddq	%ymm11,%ymm4,%ymm4
 	vpsrlq	$7,%ymm8,%ymm11
 	leaq	(%r11,%r12,1),%r11
 	xorq	%r14,%r13
 	movq	%rax,%r15
 	vpsllq	$56,%ymm8,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm8
 	rorxq	$39,%rax,%r12
 	leaq	(%r11,%r13,1),%r11
 	xorq	%rbx,%r15
 	vpsrlq	$7,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm8,%ymm8
 	rorxq	$34,%rax,%r14
 	rorxq	$28,%rax,%r13
 	leaq	(%rdx,%r11,1),%rdx
 	vpsllq	$7,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm8,%ymm8
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%rbx,%rdi
 	vpsrlq	$6,%ymm3,%ymm11
 	vpxor	%ymm9,%ymm8,%ymm8
 	xorq	%r13,%r14
 	leaq	(%r11,%rdi,1),%r11
 	movq	%r8,%r12
 	vpsllq	$3,%ymm3,%ymm10
 	vpaddq	%ymm8,%ymm4,%ymm4
 	addq	8+256(%rsp),%r10
 	andq	%rdx,%r12
 	rorxq	$41,%rdx,%r13
 	vpsrlq	$19,%ymm3,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	rorxq	$18,%rdx,%rdi
 	leaq	(%r11,%r14,1),%r11
 	leaq	(%r10,%r12,1),%r10
 	vpsllq	$42,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm11,%ymm11
 	andnq	%r9,%rdx,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%rdx,%r14
 	vpsrlq	$42,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	leaq	(%r10,%r12,1),%r10
 	xorq	%r14,%r13
 	movq	%r11,%rdi
 	vpxor	%ymm9,%ymm11,%ymm11
 	rorxq	$39,%r11,%r12
 	leaq	(%r10,%r13,1),%r10
 	xorq	%rax,%rdi
 	vpaddq	%ymm11,%ymm4,%ymm4
 	rorxq	$34,%r11,%r14
 	rorxq	$28,%r11,%r13
 	leaq	(%rcx,%r10,1),%rcx
 	vpaddq	0(%rbp),%ymm4,%ymm10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%rax,%r15
 	xorq	%r13,%r14
 	leaq	(%r10,%r15,1),%r10
 	movq	%rdx,%r12
 	vmovdqa	%ymm10,0(%rsp)
 	vpalignr	$8,%ymm5,%ymm6,%ymm8
 	addq	32+256(%rsp),%r9
 	andq	%rcx,%r12
 	rorxq	$41,%rcx,%r13
 	vpalignr	$8,%ymm1,%ymm2,%ymm11
 	rorxq	$18,%rcx,%r15
 	leaq	(%r10,%r14,1),%r10
 	leaq	(%r9,%r12,1),%r9
 	vpsrlq	$1,%ymm8,%ymm10
 	andnq	%r8,%rcx,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%rcx,%r14
 	vpaddq	%ymm11,%ymm5,%ymm5
 	vpsrlq	$7,%ymm8,%ymm11
 	leaq	(%r9,%r12,1),%r9
 	xorq	%r14,%r13
 	movq	%r10,%r15
 	vpsllq	$56,%ymm8,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm8
 	rorxq	$39,%r10,%r12
 	leaq	(%r9,%r13,1),%r9
 	xorq	%r11,%r15
 	vpsrlq	$7,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm8,%ymm8
 	rorxq	$34,%r10,%r14
 	rorxq	$28,%r10,%r13
 	leaq	(%rbx,%r9,1),%rbx
 	vpsllq	$7,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm8,%ymm8
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%r11,%rdi
 	vpsrlq	$6,%ymm4,%ymm11
 	vpxor	%ymm9,%ymm8,%ymm8
 	xorq	%r13,%r14
 	leaq	(%r9,%rdi,1),%r9
 	movq	%rcx,%r12
 	vpsllq	$3,%ymm4,%ymm10
 	vpaddq	%ymm8,%ymm5,%ymm5
 	addq	40+256(%rsp),%r8
 	andq	%rbx,%r12
 	rorxq	$41,%rbx,%r13
 	vpsrlq	$19,%ymm4,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	rorxq	$18,%rbx,%rdi
 	leaq	(%r9,%r14,1),%r9
 	leaq	(%r8,%r12,1),%r8
 	vpsllq	$42,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm11,%ymm11
 	andnq	%rdx,%rbx,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%rbx,%r14
 	vpsrlq	$42,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	leaq	(%r8,%r12,1),%r8
 	xorq	%r14,%r13
 	movq	%r9,%rdi
 	vpxor	%ymm9,%ymm11,%ymm11
 	rorxq	$39,%r9,%r12
 	leaq	(%r8,%r13,1),%r8
 	xorq	%r10,%rdi
 	vpaddq	%ymm11,%ymm5,%ymm5
 	rorxq	$34,%r9,%r14
 	rorxq	$28,%r9,%r13
 	leaq	(%rax,%r8,1),%rax
 	vpaddq	32(%rbp),%ymm5,%ymm10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%r10,%r15
 	xorq	%r13,%r14
 	leaq	(%r8,%r15,1),%r8
 	movq	%rbx,%r12
 	vmovdqa	%ymm10,32(%rsp)
 	vpalignr	$8,%ymm6,%ymm7,%ymm8
 	addq	64+256(%rsp),%rdx
 	andq	%rax,%r12
 	rorxq	$41,%rax,%r13
 	vpalignr	$8,%ymm2,%ymm3,%ymm11
 	rorxq	$18,%rax,%r15
 	leaq	(%r8,%r14,1),%r8
 	leaq	(%rdx,%r12,1),%rdx
 	vpsrlq	$1,%ymm8,%ymm10
 	andnq	%rcx,%rax,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%rax,%r14
 	vpaddq	%ymm11,%ymm6,%ymm6
 	vpsrlq	$7,%ymm8,%ymm11
 	leaq	(%rdx,%r12,1),%rdx
 	xorq	%r14,%r13
 	movq	%r8,%r15
 	vpsllq	$56,%ymm8,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm8
 	rorxq	$39,%r8,%r12
 	leaq	(%rdx,%r13,1),%rdx
 	xorq	%r9,%r15
 	vpsrlq	$7,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm8,%ymm8
 	rorxq	$34,%r8,%r14
 	rorxq	$28,%r8,%r13
 	leaq	(%r11,%rdx,1),%r11
 	vpsllq	$7,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm8,%ymm8
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%r9,%rdi
 	vpsrlq	$6,%ymm5,%ymm11
 	vpxor	%ymm9,%ymm8,%ymm8
 	xorq	%r13,%r14
 	leaq	(%rdx,%rdi,1),%rdx
 	movq	%rax,%r12
 	vpsllq	$3,%ymm5,%ymm10
 	vpaddq	%ymm8,%ymm6,%ymm6
 	addq	72+256(%rsp),%rcx
 	andq	%r11,%r12
 	rorxq	$41,%r11,%r13
 	vpsrlq	$19,%ymm5,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	rorxq	$18,%r11,%rdi
 	leaq	(%rdx,%r14,1),%rdx
 	leaq	(%rcx,%r12,1),%rcx
 	vpsllq	$42,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm11,%ymm11
 	andnq	%rbx,%r11,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%r11,%r14
 	vpsrlq	$42,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	leaq	(%rcx,%r12,1),%rcx
 	xorq	%r14,%r13
 	movq	%rdx,%rdi
 	vpxor	%ymm9,%ymm11,%ymm11
 	rorxq	$39,%rdx,%r12
 	leaq	(%rcx,%r13,1),%rcx
 	xorq	%r8,%rdi
 	vpaddq	%ymm11,%ymm6,%ymm6
 	rorxq	$34,%rdx,%r14
 	rorxq	$28,%rdx,%r13
 	leaq	(%r10,%rcx,1),%r10
 	vpaddq	64(%rbp),%ymm6,%ymm10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%r8,%r15
 	xorq	%r13,%r14
 	leaq	(%rcx,%r15,1),%rcx
 	movq	%r11,%r12
 	vmovdqa	%ymm10,64(%rsp)
 	vpalignr	$8,%ymm7,%ymm0,%ymm8
 	addq	96+256(%rsp),%rbx
 	andq	%r10,%r12
 	rorxq	$41,%r10,%r13
 	vpalignr	$8,%ymm3,%ymm4,%ymm11
 	rorxq	$18,%r10,%r15
 	leaq	(%rcx,%r14,1),%rcx
 	leaq	(%rbx,%r12,1),%rbx
 	vpsrlq	$1,%ymm8,%ymm10
 	andnq	%rax,%r10,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%r10,%r14
 	vpaddq	%ymm11,%ymm7,%ymm7
 	vpsrlq	$7,%ymm8,%ymm11
 	leaq	(%rbx,%r12,1),%rbx
 	xorq	%r14,%r13
 	movq	%rcx,%r15
 	vpsllq	$56,%ymm8,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm8
 	rorxq	$39,%rcx,%r12
 	leaq	(%rbx,%r13,1),%rbx
 	xorq	%rdx,%r15
 	vpsrlq	$7,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm8,%ymm8
 	rorxq	$34,%rcx,%r14
 	rorxq	$28,%rcx,%r13
 	leaq	(%r9,%rbx,1),%r9
 	vpsllq	$7,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm8,%ymm8
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%rdx,%rdi
 	vpsrlq	$6,%ymm6,%ymm11
 	vpxor	%ymm9,%ymm8,%ymm8
 	xorq	%r13,%r14
 	leaq	(%rbx,%rdi,1),%rbx
 	movq	%r10,%r12
 	vpsllq	$3,%ymm6,%ymm10
 	vpaddq	%ymm8,%ymm7,%ymm7
 	addq	104+256(%rsp),%rax
 	andq	%r9,%r12
 	rorxq	$41,%r9,%r13
 	vpsrlq	$19,%ymm6,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	rorxq	$18,%r9,%rdi
 	leaq	(%rbx,%r14,1),%rbx
 	leaq	(%rax,%r12,1),%rax
 	vpsllq	$42,%ymm10,%ymm10
 	vpxor	%ymm9,%ymm11,%ymm11
 	andnq	%r11,%r9,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%r9,%r14
 	vpsrlq	$42,%ymm9,%ymm9
 	vpxor	%ymm10,%ymm11,%ymm11
 	leaq	(%rax,%r12,1),%rax
 	xorq	%r14,%r13
 	movq	%rbx,%rdi
 	vpxor	%ymm9,%ymm11,%ymm11
 	rorxq	$39,%rbx,%r12
 	leaq	(%rax,%r13,1),%rax
 	xorq	%rcx,%rdi
 	vpaddq	%ymm11,%ymm7,%ymm7
 	rorxq	$34,%rbx,%r14
 	rorxq	$28,%rbx,%r13
 	leaq	(%r8,%rax,1),%r8
 	vpaddq	96(%rbp),%ymm7,%ymm10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%rcx,%r15
 	xorq	%r13,%r14
 	leaq	(%rax,%r15,1),%rax
 	movq	%r9,%r12
 	vmovdqa	%ymm10,96(%rsp)
 	leaq	256(%rbp),%rbp
 	cmpb	$0,-121(%rbp)
 	jne	.Lavx2_00_47
 	addq	0+128(%rsp),%r11
 	andq	%r8,%r12
 	rorxq	$41,%r8,%r13
 	rorxq	$18,%r8,%r15
 	leaq	(%rax,%r14,1),%rax
 	leaq	(%r11,%r12,1),%r11
 	andnq	%r10,%r8,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%r8,%r14
 	leaq	(%r11,%r12,1),%r11
 	xorq	%r14,%r13
 	movq	%rax,%r15
 	rorxq	$39,%rax,%r12
 	leaq	(%r11,%r13,1),%r11
 	xorq	%rbx,%r15
 	rorxq	$34,%rax,%r14
 	rorxq	$28,%rax,%r13
 	leaq	(%rdx,%r11,1),%rdx
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%rbx,%rdi
 	xorq	%r13,%r14
 	leaq	(%r11,%rdi,1),%r11
 	movq	%r8,%r12
 	addq	8+128(%rsp),%r10
 	andq	%rdx,%r12
 	rorxq	$41,%rdx,%r13
 	rorxq	$18,%rdx,%rdi
 	leaq	(%r11,%r14,1),%r11
 	leaq	(%r10,%r12,1),%r10
 	andnq	%r9,%rdx,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%rdx,%r14
 	leaq	(%r10,%r12,1),%r10
 	xorq	%r14,%r13
 	movq	%r11,%rdi
 	rorxq	$39,%r11,%r12
 	leaq	(%r10,%r13,1),%r10
 	xorq	%rax,%rdi
 	rorxq	$34,%r11,%r14
 	rorxq	$28,%r11,%r13
 	leaq	(%rcx,%r10,1),%rcx
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%rax,%r15
 	xorq	%r13,%r14
 	leaq	(%r10,%r15,1),%r10
 	movq	%rdx,%r12
 	addq	32+128(%rsp),%r9
 	andq	%rcx,%r12
 	rorxq	$41,%rcx,%r13
 	rorxq	$18,%rcx,%r15
 	leaq	(%r10,%r14,1),%r10
 	leaq	(%r9,%r12,1),%r9
 	andnq	%r8,%rcx,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%rcx,%r14
 	leaq	(%r9,%r12,1),%r9
 	xorq	%r14,%r13
 	movq	%r10,%r15
 	rorxq	$39,%r10,%r12
 	leaq	(%r9,%r13,1),%r9
 	xorq	%r11,%r15
 	rorxq	$34,%r10,%r14
 	rorxq	$28,%r10,%r13
 	leaq	(%rbx,%r9,1),%rbx
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%r11,%rdi
 	xorq	%r13,%r14
 	leaq	(%r9,%rdi,1),%r9
 	movq	%rcx,%r12
 	addq	40+128(%rsp),%r8
 	andq	%rbx,%r12
 	rorxq	$41,%rbx,%r13
 	rorxq	$18,%rbx,%rdi
 	leaq	(%r9,%r14,1),%r9
 	leaq	(%r8,%r12,1),%r8
 	andnq	%rdx,%rbx,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%rbx,%r14
 	leaq	(%r8,%r12,1),%r8
 	xorq	%r14,%r13
 	movq	%r9,%rdi
 	rorxq	$39,%r9,%r12
 	leaq	(%r8,%r13,1),%r8
 	xorq	%r10,%rdi
 	rorxq	$34,%r9,%r14
 	rorxq	$28,%r9,%r13
 	leaq	(%rax,%r8,1),%rax
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%r10,%r15
 	xorq	%r13,%r14
 	leaq	(%r8,%r15,1),%r8
 	movq	%rbx,%r12
 	addq	64+128(%rsp),%rdx
 	andq	%rax,%r12
 	rorxq	$41,%rax,%r13
 	rorxq	$18,%rax,%r15
 	leaq	(%r8,%r14,1),%r8
 	leaq	(%rdx,%r12,1),%rdx
 	andnq	%rcx,%rax,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%rax,%r14
 	leaq	(%rdx,%r12,1),%rdx
 	xorq	%r14,%r13
 	movq	%r8,%r15
 	rorxq	$39,%r8,%r12
 	leaq	(%rdx,%r13,1),%rdx
 	xorq	%r9,%r15
 	rorxq	$34,%r8,%r14
 	rorxq	$28,%r8,%r13
 	leaq	(%r11,%rdx,1),%r11
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%r9,%rdi
 	xorq	%r13,%r14
 	leaq	(%rdx,%rdi,1),%rdx
 	movq	%rax,%r12
 	addq	72+128(%rsp),%rcx
 	andq	%r11,%r12
 	rorxq	$41,%r11,%r13
 	rorxq	$18,%r11,%rdi
 	leaq	(%rdx,%r14,1),%rdx
 	leaq	(%rcx,%r12,1),%rcx
 	andnq	%rbx,%r11,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%r11,%r14
 	leaq	(%rcx,%r12,1),%rcx
 	xorq	%r14,%r13
 	movq	%rdx,%rdi
 	rorxq	$39,%rdx,%r12
 	leaq	(%rcx,%r13,1),%rcx
 	xorq	%r8,%rdi
 	rorxq	$34,%rdx,%r14
 	rorxq	$28,%rdx,%r13
 	leaq	(%r10,%rcx,1),%r10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%r8,%r15
 	xorq	%r13,%r14
 	leaq	(%rcx,%r15,1),%rcx
 	movq	%r11,%r12
 	addq	96+128(%rsp),%rbx
 	andq	%r10,%r12
 	rorxq	$41,%r10,%r13
 	rorxq	$18,%r10,%r15
 	leaq	(%rcx,%r14,1),%rcx
 	leaq	(%rbx,%r12,1),%rbx
 	andnq	%rax,%r10,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%r10,%r14
 	leaq	(%rbx,%r12,1),%rbx
 	xorq	%r14,%r13
 	movq	%rcx,%r15
 	rorxq	$39,%rcx,%r12
 	leaq	(%rbx,%r13,1),%rbx
 	xorq	%rdx,%r15
 	rorxq	$34,%rcx,%r14
 	rorxq	$28,%rcx,%r13
 	leaq	(%r9,%rbx,1),%r9
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%rdx,%rdi
 	xorq	%r13,%r14
 	leaq	(%rbx,%rdi,1),%rbx
 	movq	%r10,%r12
 	addq	104+128(%rsp),%rax
 	andq	%r9,%r12
 	rorxq	$41,%r9,%r13
 	rorxq	$18,%r9,%rdi
 	leaq	(%rbx,%r14,1),%rbx
 	leaq	(%rax,%r12,1),%rax
 	andnq	%r11,%r9,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%r9,%r14
 	leaq	(%rax,%r12,1),%rax
 	xorq	%r14,%r13
 	movq	%rbx,%rdi
 	rorxq	$39,%rbx,%r12
 	leaq	(%rax,%r13,1),%rax
 	xorq	%rcx,%rdi
 	rorxq	$34,%rbx,%r14
 	rorxq	$28,%rbx,%r13
 	leaq	(%r8,%rax,1),%r8
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%rcx,%r15
 	xorq	%r13,%r14
 	leaq	(%rax,%r15,1),%rax
 	movq	%r9,%r12
 	addq	0(%rsp),%r11
 	andq	%r8,%r12
 	rorxq	$41,%r8,%r13
 	rorxq	$18,%r8,%r15
 	leaq	(%rax,%r14,1),%rax
 	leaq	(%r11,%r12,1),%r11
 	andnq	%r10,%r8,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%r8,%r14
 	leaq	(%r11,%r12,1),%r11
 	xorq	%r14,%r13
 	movq	%rax,%r15
 	rorxq	$39,%rax,%r12
 	leaq	(%r11,%r13,1),%r11
 	xorq	%rbx,%r15
 	rorxq	$34,%rax,%r14
 	rorxq	$28,%rax,%r13
 	leaq	(%rdx,%r11,1),%rdx
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%rbx,%rdi
 	xorq	%r13,%r14
 	leaq	(%r11,%rdi,1),%r11
 	movq	%r8,%r12
 	addq	8(%rsp),%r10
 	andq	%rdx,%r12
 	rorxq	$41,%rdx,%r13
 	rorxq	$18,%rdx,%rdi
 	leaq	(%r11,%r14,1),%r11
 	leaq	(%r10,%r12,1),%r10
 	andnq	%r9,%rdx,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%rdx,%r14
 	leaq	(%r10,%r12,1),%r10
 	xorq	%r14,%r13
 	movq	%r11,%rdi
 	rorxq	$39,%r11,%r12
 	leaq	(%r10,%r13,1),%r10
 	xorq	%rax,%rdi
 	rorxq	$34,%r11,%r14
 	rorxq	$28,%r11,%r13
 	leaq	(%rcx,%r10,1),%rcx
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%rax,%r15
 	xorq	%r13,%r14
 	leaq	(%r10,%r15,1),%r10
 	movq	%rdx,%r12
 	addq	32(%rsp),%r9
 	andq	%rcx,%r12
 	rorxq	$41,%rcx,%r13
 	rorxq	$18,%rcx,%r15
 	leaq	(%r10,%r14,1),%r10
 	leaq	(%r9,%r12,1),%r9
 	andnq	%r8,%rcx,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%rcx,%r14
 	leaq	(%r9,%r12,1),%r9
 	xorq	%r14,%r13
 	movq	%r10,%r15
 	rorxq	$39,%r10,%r12
 	leaq	(%r9,%r13,1),%r9
 	xorq	%r11,%r15
 	rorxq	$34,%r10,%r14
 	rorxq	$28,%r10,%r13
 	leaq	(%rbx,%r9,1),%rbx
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%r11,%rdi
 	xorq	%r13,%r14
 	leaq	(%r9,%rdi,1),%r9
 	movq	%rcx,%r12
 	addq	40(%rsp),%r8
 	andq	%rbx,%r12
 	rorxq	$41,%rbx,%r13
 	rorxq	$18,%rbx,%rdi
 	leaq	(%r9,%r14,1),%r9
 	leaq	(%r8,%r12,1),%r8
 	andnq	%rdx,%rbx,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%rbx,%r14
 	leaq	(%r8,%r12,1),%r8
 	xorq	%r14,%r13
 	movq	%r9,%rdi
 	rorxq	$39,%r9,%r12
 	leaq	(%r8,%r13,1),%r8
 	xorq	%r10,%rdi
 	rorxq	$34,%r9,%r14
 	rorxq	$28,%r9,%r13
 	leaq	(%rax,%r8,1),%rax
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%r10,%r15
 	xorq	%r13,%r14
 	leaq	(%r8,%r15,1),%r8
 	movq	%rbx,%r12
 	addq	64(%rsp),%rdx
 	andq	%rax,%r12
 	rorxq	$41,%rax,%r13
 	rorxq	$18,%rax,%r15
 	leaq	(%r8,%r14,1),%r8
 	leaq	(%rdx,%r12,1),%rdx
 	andnq	%rcx,%rax,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%rax,%r14
 	leaq	(%rdx,%r12,1),%rdx
 	xorq	%r14,%r13
 	movq	%r8,%r15
 	rorxq	$39,%r8,%r12
 	leaq	(%rdx,%r13,1),%rdx
 	xorq	%r9,%r15
 	rorxq	$34,%r8,%r14
 	rorxq	$28,%r8,%r13
 	leaq	(%r11,%rdx,1),%r11
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%r9,%rdi
 	xorq	%r13,%r14
 	leaq	(%rdx,%rdi,1),%rdx
 	movq	%rax,%r12
 	addq	72(%rsp),%rcx
 	andq	%r11,%r12
 	rorxq	$41,%r11,%r13
 	rorxq	$18,%r11,%rdi
 	leaq	(%rdx,%r14,1),%rdx
 	leaq	(%rcx,%r12,1),%rcx
 	andnq	%rbx,%r11,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%r11,%r14
 	leaq	(%rcx,%r12,1),%rcx
 	xorq	%r14,%r13
 	movq	%rdx,%rdi
 	rorxq	$39,%rdx,%r12
 	leaq	(%rcx,%r13,1),%rcx
 	xorq	%r8,%rdi
 	rorxq	$34,%rdx,%r14
 	rorxq	$28,%rdx,%r13
 	leaq	(%r10,%rcx,1),%r10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%r8,%r15
 	xorq	%r13,%r14
 	leaq	(%rcx,%r15,1),%rcx
 	movq	%r11,%r12
 	addq	96(%rsp),%rbx
 	andq	%r10,%r12
 	rorxq	$41,%r10,%r13
 	rorxq	$18,%r10,%r15
 	leaq	(%rcx,%r14,1),%rcx
 	leaq	(%rbx,%r12,1),%rbx
 	andnq	%rax,%r10,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%r10,%r14
 	leaq	(%rbx,%r12,1),%rbx
 	xorq	%r14,%r13
 	movq	%rcx,%r15
 	rorxq	$39,%rcx,%r12
 	leaq	(%rbx,%r13,1),%rbx
 	xorq	%rdx,%r15
 	rorxq	$34,%rcx,%r14
 	rorxq	$28,%rcx,%r13
 	leaq	(%r9,%rbx,1),%r9
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%rdx,%rdi
 	xorq	%r13,%r14
 	leaq	(%rbx,%rdi,1),%rbx
 	movq	%r10,%r12
 	addq	104(%rsp),%rax
 	andq	%r9,%r12
 	rorxq	$41,%r9,%r13
 	rorxq	$18,%r9,%rdi
 	leaq	(%rbx,%r14,1),%rbx
 	leaq	(%rax,%r12,1),%rax
 	andnq	%r11,%r9,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%r9,%r14
 	leaq	(%rax,%r12,1),%rax
 	xorq	%r14,%r13
 	movq	%rbx,%rdi
 	rorxq	$39,%rbx,%r12
 	leaq	(%rax,%r13,1),%rax
 	xorq	%rcx,%rdi
 	rorxq	$34,%rbx,%r14
 	rorxq	$28,%rbx,%r13
 	leaq	(%r8,%rax,1),%r8
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%rcx,%r15
 	xorq	%r13,%r14
 	leaq	(%rax,%r15,1),%rax
 	movq	%r9,%r12
 	movq	1280(%rsp),%rdi
 	addq	%r14,%rax
 
 	leaq	1152(%rsp),%rbp
 
 	addq	0(%rdi),%rax
 	addq	8(%rdi),%rbx
 	addq	16(%rdi),%rcx
 	addq	24(%rdi),%rdx
 	addq	32(%rdi),%r8
 	addq	40(%rdi),%r9
 	addq	48(%rdi),%r10
 	addq	56(%rdi),%r11
 
 	movq	%rax,0(%rdi)
 	movq	%rbx,8(%rdi)
 	movq	%rcx,16(%rdi)
 	movq	%rdx,24(%rdi)
 	movq	%r8,32(%rdi)
 	movq	%r9,40(%rdi)
 	movq	%r10,48(%rdi)
 	movq	%r11,56(%rdi)
 
 	cmpq	144(%rbp),%rsi
 	je	.Ldone_avx2
 
 	xorq	%r14,%r14
 	movq	%rbx,%rdi
 	xorq	%rcx,%rdi
 	movq	%r9,%r12
 	jmp	.Lower_avx2
 .balign	16
 .Lower_avx2:
 	addq	0+16(%rbp),%r11
 	andq	%r8,%r12
 	rorxq	$41,%r8,%r13
 	rorxq	$18,%r8,%r15
 	leaq	(%rax,%r14,1),%rax
 	leaq	(%r11,%r12,1),%r11
 	andnq	%r10,%r8,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%r8,%r14
 	leaq	(%r11,%r12,1),%r11
 	xorq	%r14,%r13
 	movq	%rax,%r15
 	rorxq	$39,%rax,%r12
 	leaq	(%r11,%r13,1),%r11
 	xorq	%rbx,%r15
 	rorxq	$34,%rax,%r14
 	rorxq	$28,%rax,%r13
 	leaq	(%rdx,%r11,1),%rdx
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%rbx,%rdi
 	xorq	%r13,%r14
 	leaq	(%r11,%rdi,1),%r11
 	movq	%r8,%r12
 	addq	8+16(%rbp),%r10
 	andq	%rdx,%r12
 	rorxq	$41,%rdx,%r13
 	rorxq	$18,%rdx,%rdi
 	leaq	(%r11,%r14,1),%r11
 	leaq	(%r10,%r12,1),%r10
 	andnq	%r9,%rdx,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%rdx,%r14
 	leaq	(%r10,%r12,1),%r10
 	xorq	%r14,%r13
 	movq	%r11,%rdi
 	rorxq	$39,%r11,%r12
 	leaq	(%r10,%r13,1),%r10
 	xorq	%rax,%rdi
 	rorxq	$34,%r11,%r14
 	rorxq	$28,%r11,%r13
 	leaq	(%rcx,%r10,1),%rcx
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%rax,%r15
 	xorq	%r13,%r14
 	leaq	(%r10,%r15,1),%r10
 	movq	%rdx,%r12
 	addq	32+16(%rbp),%r9
 	andq	%rcx,%r12
 	rorxq	$41,%rcx,%r13
 	rorxq	$18,%rcx,%r15
 	leaq	(%r10,%r14,1),%r10
 	leaq	(%r9,%r12,1),%r9
 	andnq	%r8,%rcx,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%rcx,%r14
 	leaq	(%r9,%r12,1),%r9
 	xorq	%r14,%r13
 	movq	%r10,%r15
 	rorxq	$39,%r10,%r12
 	leaq	(%r9,%r13,1),%r9
 	xorq	%r11,%r15
 	rorxq	$34,%r10,%r14
 	rorxq	$28,%r10,%r13
 	leaq	(%rbx,%r9,1),%rbx
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%r11,%rdi
 	xorq	%r13,%r14
 	leaq	(%r9,%rdi,1),%r9
 	movq	%rcx,%r12
 	addq	40+16(%rbp),%r8
 	andq	%rbx,%r12
 	rorxq	$41,%rbx,%r13
 	rorxq	$18,%rbx,%rdi
 	leaq	(%r9,%r14,1),%r9
 	leaq	(%r8,%r12,1),%r8
 	andnq	%rdx,%rbx,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%rbx,%r14
 	leaq	(%r8,%r12,1),%r8
 	xorq	%r14,%r13
 	movq	%r9,%rdi
 	rorxq	$39,%r9,%r12
 	leaq	(%r8,%r13,1),%r8
 	xorq	%r10,%rdi
 	rorxq	$34,%r9,%r14
 	rorxq	$28,%r9,%r13
 	leaq	(%rax,%r8,1),%rax
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%r10,%r15
 	xorq	%r13,%r14
 	leaq	(%r8,%r15,1),%r8
 	movq	%rbx,%r12
 	addq	64+16(%rbp),%rdx
 	andq	%rax,%r12
 	rorxq	$41,%rax,%r13
 	rorxq	$18,%rax,%r15
 	leaq	(%r8,%r14,1),%r8
 	leaq	(%rdx,%r12,1),%rdx
 	andnq	%rcx,%rax,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%rax,%r14
 	leaq	(%rdx,%r12,1),%rdx
 	xorq	%r14,%r13
 	movq	%r8,%r15
 	rorxq	$39,%r8,%r12
 	leaq	(%rdx,%r13,1),%rdx
 	xorq	%r9,%r15
 	rorxq	$34,%r8,%r14
 	rorxq	$28,%r8,%r13
 	leaq	(%r11,%rdx,1),%r11
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%r9,%rdi
 	xorq	%r13,%r14
 	leaq	(%rdx,%rdi,1),%rdx
 	movq	%rax,%r12
 	addq	72+16(%rbp),%rcx
 	andq	%r11,%r12
 	rorxq	$41,%r11,%r13
 	rorxq	$18,%r11,%rdi
 	leaq	(%rdx,%r14,1),%rdx
 	leaq	(%rcx,%r12,1),%rcx
 	andnq	%rbx,%r11,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%r11,%r14
 	leaq	(%rcx,%r12,1),%rcx
 	xorq	%r14,%r13
 	movq	%rdx,%rdi
 	rorxq	$39,%rdx,%r12
 	leaq	(%rcx,%r13,1),%rcx
 	xorq	%r8,%rdi
 	rorxq	$34,%rdx,%r14
 	rorxq	$28,%rdx,%r13
 	leaq	(%r10,%rcx,1),%r10
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%r8,%r15
 	xorq	%r13,%r14
 	leaq	(%rcx,%r15,1),%rcx
 	movq	%r11,%r12
 	addq	96+16(%rbp),%rbx
 	andq	%r10,%r12
 	rorxq	$41,%r10,%r13
 	rorxq	$18,%r10,%r15
 	leaq	(%rcx,%r14,1),%rcx
 	leaq	(%rbx,%r12,1),%rbx
 	andnq	%rax,%r10,%r12
 	xorq	%r15,%r13
 	rorxq	$14,%r10,%r14
 	leaq	(%rbx,%r12,1),%rbx
 	xorq	%r14,%r13
 	movq	%rcx,%r15
 	rorxq	$39,%rcx,%r12
 	leaq	(%rbx,%r13,1),%rbx
 	xorq	%rdx,%r15
 	rorxq	$34,%rcx,%r14
 	rorxq	$28,%rcx,%r13
 	leaq	(%r9,%rbx,1),%r9
 	andq	%r15,%rdi
 	xorq	%r12,%r14
 	xorq	%rdx,%rdi
 	xorq	%r13,%r14
 	leaq	(%rbx,%rdi,1),%rbx
 	movq	%r10,%r12
 	addq	104+16(%rbp),%rax
 	andq	%r9,%r12
 	rorxq	$41,%r9,%r13
 	rorxq	$18,%r9,%rdi
 	leaq	(%rbx,%r14,1),%rbx
 	leaq	(%rax,%r12,1),%rax
 	andnq	%r11,%r9,%r12
 	xorq	%rdi,%r13
 	rorxq	$14,%r9,%r14
 	leaq	(%rax,%r12,1),%rax
 	xorq	%r14,%r13
 	movq	%rbx,%rdi
 	rorxq	$39,%rbx,%r12
 	leaq	(%rax,%r13,1),%rax
 	xorq	%rcx,%rdi
 	rorxq	$34,%rbx,%r14
 	rorxq	$28,%rbx,%r13
 	leaq	(%r8,%rax,1),%r8
 	andq	%rdi,%r15
 	xorq	%r12,%r14
 	xorq	%rcx,%r15
 	xorq	%r13,%r14
 	leaq	(%rax,%r15,1),%rax
 	movq	%r9,%r12
 	leaq	-128(%rbp),%rbp
 	cmpq	%rsp,%rbp
 	jae	.Lower_avx2
 
 	movq	1280(%rsp),%rdi
 	addq	%r14,%rax
 
 	leaq	1152(%rsp),%rsp
 
 .cfi_escape	0x0f,0x06,0x77,0x98,0x01,0x06,0x23,0x08
 
 	addq	0(%rdi),%rax
 	addq	8(%rdi),%rbx
 	addq	16(%rdi),%rcx
 	addq	24(%rdi),%rdx
 	addq	32(%rdi),%r8
 	addq	40(%rdi),%r9
 	leaq	256(%rsi),%rsi
 	addq	48(%rdi),%r10
 	movq	%rsi,%r12
 	addq	56(%rdi),%r11
 	cmpq	128+16(%rsp),%rsi
 
 	movq	%rax,0(%rdi)
 	cmoveq	%rsp,%r12
 	movq	%rbx,8(%rdi)
 	movq	%rcx,16(%rdi)
 	movq	%rdx,24(%rdi)
 	movq	%r8,32(%rdi)
 	movq	%r9,40(%rdi)
 	movq	%r10,48(%rdi)
 	movq	%r11,56(%rdi)
 
 	jbe	.Loop_avx2
 	leaq	(%rsp),%rbp
 
 .cfi_escape	0x0f,0x06,0x76,0x98,0x01,0x06,0x23,0x08
 
 .Ldone_avx2:
 	movq	152(%rbp),%rsi
 .cfi_def_cfa	%rsi,8
 	vzeroupper
 	movq	-48(%rsi),%r15
 .cfi_restore	%r15
 	movq	-40(%rsi),%r14
 .cfi_restore	%r14
 	movq	-32(%rsi),%r13
 .cfi_restore	%r13
 	movq	-24(%rsi),%r12
 .cfi_restore	%r12
 	movq	-16(%rsi),%rbp
 .cfi_restore	%rbp
 	movq	-8(%rsi),%rbx
 .cfi_restore	%rbx
 	leaq	(%rsi),%rsp
 .cfi_def_cfa_register	%rsp
 .Lepilogue_avx2:
 	RET
 .cfi_endproc
 SET_SIZE(zfs_sha512_transform_avx2)
+STACK_FRAME_NON_STANDARD zfs_sha512_transform_avx2
+
+/* Workaround for missing asm macro in RHEL 8. */
+#if defined(__linux__) && defined(HAVE_STACK_FRAME_NON_STANDARD) && \
+    ! defined(HAVE_STACK_FRAME_NON_STANDARD_ASM)
+.section .discard.func_stack_frame_non_standard, "aw"
+	.long zfs_sha512_transform_x64 - .
+	.long zfs_sha512_transform_avx - .
+	.long zfs_sha512_transform_avx2 - .
+#endif
 
 #if defined(__ELF__)
 	.section .note.GNU-stack,"",%progbits
 #endif
 #endif
diff --git a/sys/contrib/openzfs/module/os/freebsd/spl/spl_policy.c b/sys/contrib/openzfs/module/os/freebsd/spl/spl_policy.c
index 42a693b073d1..5f59934f22e8 100644
--- a/sys/contrib/openzfs/module/os/freebsd/spl/spl_policy.c
+++ b/sys/contrib/openzfs/module/os/freebsd/spl/spl_policy.c
@@ -1,435 +1,428 @@
 /*
  * Copyright (c) 2007 Pawel Jakub Dawidek <pjd@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
  * 1. Redistributions of source code must retain the above copyright
  *    notice, this list of conditions and the following disclaimer.
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
  *
  * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  */
 
 #include <sys/types.h>
 #include <sys/param.h>
 #include <sys/priv.h>
 #include <sys/vnode.h>
 #include <sys/mntent.h>
 #include <sys/mount.h>
 #include <sys/stat.h>
 #include <sys/jail.h>
 #include <sys/policy.h>
 #include <sys/zfs_vfsops.h>
 #include <sys/zfs_znode.h>
 
 
 int
 secpolicy_nfs(cred_t *cr)
 {
 
 	return (priv_check_cred(cr, PRIV_NFS_DAEMON));
 }
 
 int
 secpolicy_zfs(cred_t *cr)
 {
 
 	return (priv_check_cred(cr, PRIV_VFS_MOUNT));
 }
 
-int
-secpolicy_zfs_proc(cred_t *cr, proc_t *proc)
-{
-
-	return (priv_check_cred(cr, PRIV_VFS_MOUNT));
-}
-
 int
 secpolicy_sys_config(cred_t *cr, int checkonly __unused)
 {
 
 	return (priv_check_cred(cr, PRIV_ZFS_POOL_CONFIG));
 }
 
 int
 secpolicy_zinject(cred_t *cr)
 {
 
 	return (priv_check_cred(cr, PRIV_ZFS_INJECT));
 }
 
 int
 secpolicy_fs_unmount(cred_t *cr, struct mount *vfsp __unused)
 {
 
 	return (priv_check_cred(cr, PRIV_VFS_UNMOUNT));
 }
 
 int
 secpolicy_fs_owner(struct mount *mp, cred_t *cr)
 {
 
 	if (zfs_super_owner) {
 		if (cr->cr_uid == mp->mnt_cred->cr_uid &&
 		    cr->cr_prison == mp->mnt_cred->cr_prison) {
 			return (0);
 		}
 	}
 	return (EPERM);
 }
 
 /*
  * This check is done in kern_link(), so we could just return 0 here.
  */
 extern int hardlink_check_uid;
 int
 secpolicy_basic_link(vnode_t *vp, cred_t *cr)
 {
 
 	if (!hardlink_check_uid)
 		return (0);
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return (0);
 	return (priv_check_cred(cr, PRIV_VFS_LINK));
 }
 
 int
 secpolicy_vnode_stky_modify(cred_t *cr)
 {
 
 	return (EPERM);
 }
 
 int
 secpolicy_vnode_remove(vnode_t *vp, cred_t *cr)
 {
 
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return (0);
 	return (priv_check_cred(cr, PRIV_VFS_ADMIN));
 }
 
 int
 secpolicy_vnode_access(cred_t *cr, vnode_t *vp, uid_t owner, accmode_t accmode)
 {
 
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return (0);
 
 	if ((accmode & VREAD) && priv_check_cred(cr, PRIV_VFS_READ) != 0)
 		return (EACCES);
 	if ((accmode & VWRITE) &&
 	    priv_check_cred(cr, PRIV_VFS_WRITE) != 0) {
 		return (EACCES);
 	}
 	if (accmode & VEXEC) {
 		if (vp->v_type == VDIR) {
 			if (priv_check_cred(cr, PRIV_VFS_LOOKUP) != 0)
 				return (EACCES);
 		} else {
 			if (priv_check_cred(cr, PRIV_VFS_EXEC) != 0)
 				return (EACCES);
 		}
 	}
 	return (0);
 }
 
 /*
  * Like secpolicy_vnode_access() but we get the actual wanted mode and the
  * current mode of the file, not the missing bits.
  */
 int
 secpolicy_vnode_access2(cred_t *cr, vnode_t *vp, uid_t owner,
     accmode_t curmode, accmode_t wantmode)
 {
 	accmode_t mode;
 
 	mode = ~curmode & wantmode;
 
 	if (mode == 0)
 		return (0);
 
 	return (secpolicy_vnode_access(cr, vp, owner, mode));
 }
 
 int
 secpolicy_vnode_any_access(cred_t *cr, vnode_t *vp, uid_t owner)
 {
 	static int privs[] = {
 	    PRIV_VFS_ADMIN,
 	    PRIV_VFS_READ,
 	    PRIV_VFS_WRITE,
 	    PRIV_VFS_EXEC,
 	    PRIV_VFS_LOOKUP
 	};
 	int i;
 
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return (0);
 
 	/* Same as secpolicy_vnode_setdac */
 	if (owner == cr->cr_uid)
 		return (0);
 
 	for (i = 0; i < sizeof (privs)/sizeof (int); i++) {
 		int priv;
 
 		switch (priv = privs[i]) {
 		case PRIV_VFS_EXEC:
 			if (vp->v_type == VDIR)
 				continue;
 			break;
 		case PRIV_VFS_LOOKUP:
 			if (vp->v_type != VDIR)
 				continue;
 			break;
 		}
 		if (priv_check_cred(cr, priv) == 0)
 			return (0);
 	}
 	return (EPERM);
 }
 
 int
 secpolicy_vnode_setdac(vnode_t *vp, cred_t *cr, uid_t owner)
 {
 
 	if (owner == cr->cr_uid)
 		return (0);
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return (0);
 	return (priv_check_cred(cr, PRIV_VFS_ADMIN));
 }
 
 int
 secpolicy_vnode_setattr(cred_t *cr, vnode_t *vp, struct vattr *vap,
     const struct vattr *ovap, int flags,
     int unlocked_access(void *, int, cred_t *), void *node)
 {
 	int mask = vap->va_mask;
 	int error;
 
 	if (mask & AT_SIZE) {
 		if (vp->v_type == VDIR)
 			return (EISDIR);
 		error = unlocked_access(node, VWRITE, cr);
 		if (error)
 			return (error);
 	}
 	if (mask & AT_MODE) {
 		/*
 		 * If not the owner of the file then check privilege
 		 * for two things: the privilege to set the mode at all
 		 * and, if we're setting setuid, we also need permissions
 		 * to add the set-uid bit, if we're not the owner.
 		 * In the specific case of creating a set-uid root
 		 * file, we need even more permissions.
 		 */
 		error = secpolicy_vnode_setdac(vp, cr, ovap->va_uid);
 		if (error)
 			return (error);
 		error = secpolicy_setid_setsticky_clear(vp, vap, ovap, cr);
 		if (error)
 			return (error);
 	} else {
 		vap->va_mode = ovap->va_mode;
 	}
 	if (mask & (AT_UID | AT_GID)) {
 		error = secpolicy_vnode_setdac(vp, cr, ovap->va_uid);
 		if (error)
 			return (error);
 
 		/*
 		 * To change the owner of a file, or change the group of
 		 * a file to a group of which we are not a member, the
 		 * caller must have privilege.
 		 */
 		if (((mask & AT_UID) && vap->va_uid != ovap->va_uid) ||
 		    ((mask & AT_GID) && vap->va_gid != ovap->va_gid &&
 		    !groupmember(vap->va_gid, cr))) {
 			if (secpolicy_fs_owner(vp->v_mount, cr) != 0) {
 				error = priv_check_cred(cr, PRIV_VFS_CHOWN);
 				if (error)
 					return (error);
 			}
 		}
 
 		if (((mask & AT_UID) && vap->va_uid != ovap->va_uid) ||
 		    ((mask & AT_GID) && vap->va_gid != ovap->va_gid)) {
 			secpolicy_setid_clear(vap, vp, cr);
 		}
 	}
 	if (mask & (AT_ATIME | AT_MTIME)) {
 		/*
 		 * From utimes(2):
 		 * If times is NULL, ... The caller must be the owner of
 		 * the file, have permission to write the file, or be the
 		 * super-user.
 		 * If times is non-NULL, ... The caller must be the owner of
 		 * the file or be the super-user.
 		 */
 		error = secpolicy_vnode_setdac(vp, cr, ovap->va_uid);
 		if (error && (vap->va_vaflags & VA_UTIMES_NULL))
 			error = unlocked_access(node, VWRITE, cr);
 		if (error)
 			return (error);
 	}
 	return (0);
 }
 
 int
 secpolicy_vnode_create_gid(cred_t *cr)
 {
 
 	return (EPERM);
 }
 
 int
 secpolicy_vnode_setids_setgids(vnode_t *vp, cred_t *cr, gid_t gid)
 {
 
 	if (groupmember(gid, cr))
 		return (0);
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return (0);
 	return (priv_check_cred(cr, PRIV_VFS_SETGID));
 }
 
 int
 secpolicy_vnode_setid_retain(znode_t *zp, cred_t *cr,
     boolean_t issuidroot __unused)
 {
 
 	if (secpolicy_fs_owner(ZTOV(zp)->v_mount, cr) == 0)
 		return (0);
 	return (priv_check_cred(cr, PRIV_VFS_RETAINSUGID));
 }
 
 void
 secpolicy_setid_clear(struct vattr *vap, vnode_t *vp, cred_t *cr)
 {
 
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return;
 
 	if ((vap->va_mode & (S_ISUID | S_ISGID)) != 0) {
 		if (priv_check_cred(cr, PRIV_VFS_RETAINSUGID)) {
 			vap->va_mask |= AT_MODE;
 			vap->va_mode &= ~(S_ISUID|S_ISGID);
 		}
 	}
 }
 
 int
 secpolicy_setid_setsticky_clear(vnode_t *vp, struct vattr *vap,
     const struct vattr *ovap, cred_t *cr)
 {
 	int error;
 
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return (0);
 
 	/*
 	 * Privileged processes may set the sticky bit on non-directories,
 	 * as well as set the setgid bit on a file with a group that the process
 	 * is not a member of. Both of these are allowed in jail(8).
 	 */
 	if (vp->v_type != VDIR && (vap->va_mode & S_ISTXT)) {
 		if (priv_check_cred(cr, PRIV_VFS_STICKYFILE))
 			return (EFTYPE);
 	}
 	/*
 	 * Check for privilege if attempting to set the
 	 * group-id bit.
 	 */
 	if ((vap->va_mode & S_ISGID) != 0) {
 		error = secpolicy_vnode_setids_setgids(vp, cr, ovap->va_gid);
 		if (error)
 			return (error);
 	}
 	/*
 	 * Deny setting setuid if we are not the file owner.
 	 */
 	if ((vap->va_mode & S_ISUID) && ovap->va_uid != cr->cr_uid) {
 		error = priv_check_cred(cr, PRIV_VFS_ADMIN);
 		if (error)
 			return (error);
 	}
 	return (0);
 }
 
 int
 secpolicy_fs_mount(cred_t *cr, vnode_t *mvp, struct mount *vfsp)
 {
 
 	return (priv_check_cred(cr, PRIV_VFS_MOUNT));
 }
 
 int
 secpolicy_vnode_owner(vnode_t *vp, cred_t *cr, uid_t owner)
 {
 
 	if (owner == cr->cr_uid)
 		return (0);
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return (0);
 
 	/* XXX: vfs_suser()? */
 	return (priv_check_cred(cr, PRIV_VFS_MOUNT_OWNER));
 }
 
 int
 secpolicy_vnode_chown(vnode_t *vp, cred_t *cr, uid_t owner)
 {
 
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return (0);
 	return (priv_check_cred(cr, PRIV_VFS_CHOWN));
 }
 
 void
 secpolicy_fs_mount_clearopts(cred_t *cr, struct mount *vfsp)
 {
 
 	if (priv_check_cred(cr, PRIV_VFS_MOUNT_NONUSER) != 0) {
 		MNT_ILOCK(vfsp);
 		vfsp->vfs_flag |= VFS_NOSETUID | MNT_USER;
 		vfs_clearmntopt(vfsp, MNTOPT_SETUID);
 		vfs_setmntopt(vfsp, MNTOPT_NOSETUID, NULL, 0);
 		MNT_IUNLOCK(vfsp);
 	}
 }
 
 /*
  * Check privileges for setting xvattr attributes
  */
 int
 secpolicy_xvattr(vnode_t *vp, xvattr_t *xvap, uid_t owner, cred_t *cr,
     vtype_t vtype)
 {
 
 	if (secpolicy_fs_owner(vp->v_mount, cr) == 0)
 		return (0);
 	return (priv_check_cred(cr, PRIV_VFS_SYSFLAGS));
 }
 
 int
 secpolicy_smb(cred_t *cr)
 {
 
 	return (priv_check_cred(cr, PRIV_NETSMB));
 }
diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-taskq.c b/sys/contrib/openzfs/module/os/linux/spl/spl-taskq.c
index 7f6780909973..5ebd7a6256f2 100644
--- a/sys/contrib/openzfs/module/os/linux/spl/spl-taskq.c
+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-taskq.c
@@ -1,1423 +1,1428 @@
 /*
  *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
  *  Copyright (C) 2007 The Regents of the University of California.
  *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
  *  UCRL-CODE-235197
  *
  *  This file is part of the SPL, Solaris Porting Layer.
  *
  *  The SPL is free software; you can redistribute it and/or modify it
  *  under the terms of the GNU General Public License as published by the
  *  Free Software Foundation; either version 2 of the License, or (at your
  *  option) any later version.
  *
  *  The SPL is distributed in the hope that it will be useful, but WITHOUT
  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  *  for more details.
  *
  *  You should have received a copy of the GNU General Public License along
  *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
  *
  *  Solaris Porting Layer (SPL) Task Queue Implementation.
  */
 
 #include <sys/timer.h>
 #include <sys/taskq.h>
 #include <sys/kmem.h>
 #include <sys/tsd.h>
 #include <sys/trace_spl.h>
 #include <linux/cpuhotplug.h>
 
+/* Linux 6.2 renamed timer_delete_sync(); point it at its old name for those. */
+#ifndef HAVE_TIMER_DELETE_SYNC
+#define	timer_delete_sync(t)	del_timer_sync(t)
+#endif
+
 static int spl_taskq_thread_bind = 0;
 module_param(spl_taskq_thread_bind, int, 0644);
 MODULE_PARM_DESC(spl_taskq_thread_bind, "Bind taskq thread to CPU by default");
 
 static uint_t spl_taskq_thread_timeout_ms = 5000;
 /* BEGIN CSTYLED */
 module_param(spl_taskq_thread_timeout_ms, uint, 0644);
 /* END CSTYLED */
 MODULE_PARM_DESC(spl_taskq_thread_timeout_ms,
 	"Minimum idle threads exit interval for dynamic taskqs");
 
 static int spl_taskq_thread_dynamic = 1;
 module_param(spl_taskq_thread_dynamic, int, 0444);
 MODULE_PARM_DESC(spl_taskq_thread_dynamic, "Allow dynamic taskq threads");
 
 static int spl_taskq_thread_priority = 1;
 module_param(spl_taskq_thread_priority, int, 0644);
 MODULE_PARM_DESC(spl_taskq_thread_priority,
 	"Allow non-default priority for taskq threads");
 
 static uint_t spl_taskq_thread_sequential = 4;
 /* BEGIN CSTYLED */
 module_param(spl_taskq_thread_sequential, uint, 0644);
 /* END CSTYLED */
 MODULE_PARM_DESC(spl_taskq_thread_sequential,
 	"Create new taskq threads after N sequential tasks");
 
 /*
  * Global system-wide dynamic task queue available for all consumers. This
  * taskq is not intended for long-running tasks; instead, a dedicated taskq
  * should be created.
  */
 taskq_t *system_taskq;
 EXPORT_SYMBOL(system_taskq);
 /* Global dynamic task queue for long delay */
 taskq_t *system_delay_taskq;
 EXPORT_SYMBOL(system_delay_taskq);
 
 /* Private dedicated taskq for creating new taskq threads on demand. */
 static taskq_t *dynamic_taskq;
 static taskq_thread_t *taskq_thread_create(taskq_t *);
 
 /* Multi-callback id for cpu hotplugging. */
 static int spl_taskq_cpuhp_state;
 
 /* List of all taskqs */
 LIST_HEAD(tq_list);
 struct rw_semaphore tq_list_sem;
 static uint_t taskq_tsd;
 
 static int
 task_km_flags(uint_t flags)
 {
 	if (flags & TQ_NOSLEEP)
 		return (KM_NOSLEEP);
 
 	if (flags & TQ_PUSHPAGE)
 		return (KM_PUSHPAGE);
 
 	return (KM_SLEEP);
 }
 
 /*
  * taskq_find_by_name - Find the largest instance number of a named taskq.
  */
 static int
 taskq_find_by_name(const char *name)
 {
 	struct list_head *tql = NULL;
 	taskq_t *tq;
 
 	list_for_each_prev(tql, &tq_list) {
 		tq = list_entry(tql, taskq_t, tq_taskqs);
 		if (strcmp(name, tq->tq_name) == 0)
 			return (tq->tq_instance);
 	}
 	return (-1);
 }
 
 /*
  * NOTE: Must be called with tq->tq_lock held, returns a list_t which
  * is not attached to the free, work, or pending taskq lists.
  */
 static taskq_ent_t *
 task_alloc(taskq_t *tq, uint_t flags, unsigned long *irqflags)
 {
 	taskq_ent_t *t;
 	int count = 0;
 
 	ASSERT(tq);
 retry:
 	/* Acquire taskq_ent_t's from free list if available */
 	if (!list_empty(&tq->tq_free_list) && !(flags & TQ_NEW)) {
 		t = list_entry(tq->tq_free_list.next, taskq_ent_t, tqent_list);
 
 		ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));
 		ASSERT(!(t->tqent_flags & TQENT_FLAG_CANCEL));
 		ASSERT(!timer_pending(&t->tqent_timer));
 
 		list_del_init(&t->tqent_list);
 		return (t);
 	}
 
 	/* Free list is empty and memory allocations are prohibited */
 	if (flags & TQ_NOALLOC)
 		return (NULL);
 
 	/* Hit maximum taskq_ent_t pool size */
 	if (tq->tq_nalloc >= tq->tq_maxalloc) {
 		if (flags & TQ_NOSLEEP)
 			return (NULL);
 
 		/*
 		 * Sleep periodically polling the free list for an available
 		 * taskq_ent_t. Dispatching with TQ_SLEEP should always succeed
 		 * but we cannot block forever waiting for an taskq_ent_t to
 		 * show up in the free list, otherwise a deadlock can happen.
 		 *
 		 * Therefore, we need to allocate a new task even if the number
 		 * of allocated tasks is above tq->tq_maxalloc, but we still
 		 * end up delaying the task allocation by one second, thereby
 		 * throttling the task dispatch rate.
 		 */
 		spin_unlock_irqrestore(&tq->tq_lock, *irqflags);
 		schedule_timeout_interruptible(HZ / 100);
 		spin_lock_irqsave_nested(&tq->tq_lock, *irqflags,
 		    tq->tq_lock_class);
 		if (count < 100) {
 			count++;
 			goto retry;
 		}
 	}
 
 	spin_unlock_irqrestore(&tq->tq_lock, *irqflags);
 	t = kmem_alloc(sizeof (taskq_ent_t), task_km_flags(flags));
 	spin_lock_irqsave_nested(&tq->tq_lock, *irqflags, tq->tq_lock_class);
 
 	if (t) {
 		taskq_init_ent(t);
 		tq->tq_nalloc++;
 	}
 
 	return (t);
 }
 
 /*
  * NOTE: Must be called with tq->tq_lock held, expects the taskq_ent_t
  * to already be removed from the free, work, or pending taskq lists.
  */
 static void
 task_free(taskq_t *tq, taskq_ent_t *t)
 {
 	ASSERT(tq);
 	ASSERT(t);
 	ASSERT(list_empty(&t->tqent_list));
 	ASSERT(!timer_pending(&t->tqent_timer));
 
 	kmem_free(t, sizeof (taskq_ent_t));
 	tq->tq_nalloc--;
 }
 
 /*
  * NOTE: Must be called with tq->tq_lock held, either destroys the
  * taskq_ent_t if too many exist or moves it to the free list for later use.
  */
 static void
 task_done(taskq_t *tq, taskq_ent_t *t)
 {
 	ASSERT(tq);
 	ASSERT(t);
 
 	/* Wake tasks blocked in taskq_wait_id() */
 	wake_up_all(&t->tqent_waitq);
 
 	list_del_init(&t->tqent_list);
 
 	if (tq->tq_nalloc <= tq->tq_minalloc) {
 		t->tqent_id = TASKQID_INVALID;
 		t->tqent_func = NULL;
 		t->tqent_arg = NULL;
 		t->tqent_flags = 0;
 
 		list_add_tail(&t->tqent_list, &tq->tq_free_list);
 	} else {
 		task_free(tq, t);
 	}
 }
 
 /*
  * When a delayed task timer expires remove it from the delay list and
  * add it to the priority list in order for immediate processing.
  */
 static void
 task_expire_impl(taskq_ent_t *t)
 {
 	taskq_ent_t *w;
 	taskq_t *tq = t->tqent_taskq;
 	struct list_head *l = NULL;
 	unsigned long flags;
 
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 
 	if (t->tqent_flags & TQENT_FLAG_CANCEL) {
 		ASSERT(list_empty(&t->tqent_list));
 		spin_unlock_irqrestore(&tq->tq_lock, flags);
 		return;
 	}
 
 	t->tqent_birth = jiffies;
 	DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t);
 
 	/*
 	 * The priority list must be maintained in strict task id order
 	 * from lowest to highest for lowest_id to be easily calculable.
 	 */
 	list_del(&t->tqent_list);
 	list_for_each_prev(l, &tq->tq_prio_list) {
 		w = list_entry(l, taskq_ent_t, tqent_list);
 		if (w->tqent_id < t->tqent_id) {
 			list_add(&t->tqent_list, l);
 			break;
 		}
 	}
 	if (l == &tq->tq_prio_list)
 		list_add(&t->tqent_list, &tq->tq_prio_list);
 
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 	wake_up(&tq->tq_work_waitq);
 }
 
 static void
 task_expire(struct timer_list *tl)
 {
 	struct timer_list *tmr = (struct timer_list *)tl;
 	taskq_ent_t *t = from_timer(t, tmr, tqent_timer);
 	task_expire_impl(t);
 }
 
 /*
  * Returns the lowest incomplete taskqid_t.  The taskqid_t may
  * be queued on the pending list, on the priority list, on the
  * delay list, or on the work list currently being handled, but
  * it is not 100% complete yet.
  */
 static taskqid_t
 taskq_lowest_id(taskq_t *tq)
 {
 	taskqid_t lowest_id = tq->tq_next_id;
 	taskq_ent_t *t;
 	taskq_thread_t *tqt;
 
 	if (!list_empty(&tq->tq_pend_list)) {
 		t = list_entry(tq->tq_pend_list.next, taskq_ent_t, tqent_list);
 		lowest_id = MIN(lowest_id, t->tqent_id);
 	}
 
 	if (!list_empty(&tq->tq_prio_list)) {
 		t = list_entry(tq->tq_prio_list.next, taskq_ent_t, tqent_list);
 		lowest_id = MIN(lowest_id, t->tqent_id);
 	}
 
 	if (!list_empty(&tq->tq_delay_list)) {
 		t = list_entry(tq->tq_delay_list.next, taskq_ent_t, tqent_list);
 		lowest_id = MIN(lowest_id, t->tqent_id);
 	}
 
 	if (!list_empty(&tq->tq_active_list)) {
 		tqt = list_entry(tq->tq_active_list.next, taskq_thread_t,
 		    tqt_active_list);
 		ASSERT(tqt->tqt_id != TASKQID_INVALID);
 		lowest_id = MIN(lowest_id, tqt->tqt_id);
 	}
 
 	return (lowest_id);
 }
 
 /*
  * Insert a task into a list keeping the list sorted by increasing taskqid.
  */
 static void
 taskq_insert_in_order(taskq_t *tq, taskq_thread_t *tqt)
 {
 	taskq_thread_t *w;
 	struct list_head *l = NULL;
 
 	ASSERT(tq);
 	ASSERT(tqt);
 
 	list_for_each_prev(l, &tq->tq_active_list) {
 		w = list_entry(l, taskq_thread_t, tqt_active_list);
 		if (w->tqt_id < tqt->tqt_id) {
 			list_add(&tqt->tqt_active_list, l);
 			break;
 		}
 	}
 	if (l == &tq->tq_active_list)
 		list_add(&tqt->tqt_active_list, &tq->tq_active_list);
 }
 
 /*
  * Find and return a task from the given list if it exists.  The list
  * must be in lowest to highest task id order.
  */
 static taskq_ent_t *
 taskq_find_list(taskq_t *tq, struct list_head *lh, taskqid_t id)
 {
 	struct list_head *l = NULL;
 	taskq_ent_t *t;
 
 	list_for_each(l, lh) {
 		t = list_entry(l, taskq_ent_t, tqent_list);
 
 		if (t->tqent_id == id)
 			return (t);
 
 		if (t->tqent_id > id)
 			break;
 	}
 
 	return (NULL);
 }
 
 /*
  * Find an already dispatched task given the task id regardless of what
  * state it is in.  If a task is still pending it will be returned.
  * If a task is executing, then -EBUSY will be returned instead.
  * If the task has already been run then NULL is returned.
  */
 static taskq_ent_t *
 taskq_find(taskq_t *tq, taskqid_t id)
 {
 	taskq_thread_t *tqt;
 	struct list_head *l = NULL;
 	taskq_ent_t *t;
 
 	t = taskq_find_list(tq, &tq->tq_delay_list, id);
 	if (t)
 		return (t);
 
 	t = taskq_find_list(tq, &tq->tq_prio_list, id);
 	if (t)
 		return (t);
 
 	t = taskq_find_list(tq, &tq->tq_pend_list, id);
 	if (t)
 		return (t);
 
 	list_for_each(l, &tq->tq_active_list) {
 		tqt = list_entry(l, taskq_thread_t, tqt_active_list);
 		if (tqt->tqt_id == id) {
 			/*
 			 * Instead of returning tqt_task, we just return a non
 			 * NULL value to prevent misuse, since tqt_task only
 			 * has two valid fields.
 			 */
 			return (ERR_PTR(-EBUSY));
 		}
 	}
 
 	return (NULL);
 }
 
 /*
  * Theory for the taskq_wait_id(), taskq_wait_outstanding(), and
  * taskq_wait() functions below.
  *
  * Taskq waiting is accomplished by tracking the lowest outstanding task
  * id and the next available task id.  As tasks are dispatched they are
  * added to the tail of the pending, priority, or delay lists.  As worker
  * threads become available the tasks are removed from the heads of these
  * lists and linked to the worker threads.  This ensures the lists are
  * kept sorted by lowest to highest task id.
  *
  * Therefore the lowest outstanding task id can be quickly determined by
  * checking the head item from all of these lists.  This value is stored
  * with the taskq as the lowest id.  It only needs to be recalculated when
  * either the task with the current lowest id completes or is canceled.
  *
  * By blocking until the lowest task id exceeds the passed task id the
  * taskq_wait_outstanding() function can be easily implemented.  Similarly,
  * by blocking until the lowest task id matches the next task id taskq_wait()
  * can be implemented.
  *
  * Callers should be aware that when there are multiple worked threads it
  * is possible for larger task ids to complete before smaller ones.  Also
  * when the taskq contains delay tasks with small task ids callers may
  * block for a considerable length of time waiting for them to expire and
  * execute.
  */
 static int
 taskq_wait_id_check(taskq_t *tq, taskqid_t id)
 {
 	int rc;
 	unsigned long flags;
 
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 	rc = (taskq_find(tq, id) == NULL);
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 	return (rc);
 }
 
 /*
  * The taskq_wait_id() function blocks until the passed task id completes.
  * This does not guarantee that all lower task ids have completed.
  */
 void
 taskq_wait_id(taskq_t *tq, taskqid_t id)
 {
 	wait_event(tq->tq_wait_waitq, taskq_wait_id_check(tq, id));
 }
 EXPORT_SYMBOL(taskq_wait_id);
 
 static int
 taskq_wait_outstanding_check(taskq_t *tq, taskqid_t id)
 {
 	int rc;
 	unsigned long flags;
 
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 	rc = (id < tq->tq_lowest_id);
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 	return (rc);
 }
 
 /*
  * The taskq_wait_outstanding() function will block until all tasks with a
  * lower taskqid than the passed 'id' have been completed.  Note that all
  * task id's are assigned monotonically at dispatch time.  Zero may be
  * passed for the id to indicate all tasks dispatch up to this point,
  * but not after, should be waited for.
  */
 void
 taskq_wait_outstanding(taskq_t *tq, taskqid_t id)
 {
 	id = id ? id : tq->tq_next_id - 1;
 	wait_event(tq->tq_wait_waitq, taskq_wait_outstanding_check(tq, id));
 }
 EXPORT_SYMBOL(taskq_wait_outstanding);
 
 static int
 taskq_wait_check(taskq_t *tq)
 {
 	int rc;
 	unsigned long flags;
 
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 	rc = (tq->tq_lowest_id == tq->tq_next_id);
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 	return (rc);
 }
 
 /*
  * The taskq_wait() function will block until the taskq is empty.
  * This means that if a taskq re-dispatches work to itself taskq_wait()
  * callers will block indefinitely.
  */
 void
 taskq_wait(taskq_t *tq)
 {
 	wait_event(tq->tq_wait_waitq, taskq_wait_check(tq));
 }
 EXPORT_SYMBOL(taskq_wait);
 
 int
 taskq_member(taskq_t *tq, kthread_t *t)
 {
 	return (tq == (taskq_t *)tsd_get_by_thread(taskq_tsd, t));
 }
 EXPORT_SYMBOL(taskq_member);
 
 taskq_t *
 taskq_of_curthread(void)
 {
 	return (tsd_get(taskq_tsd));
 }
 EXPORT_SYMBOL(taskq_of_curthread);
 
 /*
  * Cancel an already dispatched task given the task id.  Still pending tasks
  * will be immediately canceled, and if the task is active the function will
  * block until it completes.  Preallocated tasks which are canceled must be
  * freed by the caller.
  */
 int
 taskq_cancel_id(taskq_t *tq, taskqid_t id)
 {
 	taskq_ent_t *t;
 	int rc = ENOENT;
 	unsigned long flags;
 
 	ASSERT(tq);
 
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 	t = taskq_find(tq, id);
 	if (t && t != ERR_PTR(-EBUSY)) {
 		list_del_init(&t->tqent_list);
 		t->tqent_flags |= TQENT_FLAG_CANCEL;
 
 		/*
 		 * When canceling the lowest outstanding task id we
 		 * must recalculate the new lowest outstanding id.
 		 */
 		if (tq->tq_lowest_id == t->tqent_id) {
 			tq->tq_lowest_id = taskq_lowest_id(tq);
 			ASSERT3S(tq->tq_lowest_id, >, t->tqent_id);
 		}
 
 		/*
 		 * The task_expire() function takes the tq->tq_lock so drop
 		 * drop the lock before synchronously cancelling the timer.
 		 */
 		if (timer_pending(&t->tqent_timer)) {
 			spin_unlock_irqrestore(&tq->tq_lock, flags);
-			del_timer_sync(&t->tqent_timer);
+			timer_delete_sync(&t->tqent_timer);
 			spin_lock_irqsave_nested(&tq->tq_lock, flags,
 			    tq->tq_lock_class);
 		}
 
 		if (!(t->tqent_flags & TQENT_FLAG_PREALLOC))
 			task_done(tq, t);
 
 		rc = 0;
 	}
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 	if (t == ERR_PTR(-EBUSY)) {
 		taskq_wait_id(tq, id);
 		rc = EBUSY;
 	}
 
 	return (rc);
 }
 EXPORT_SYMBOL(taskq_cancel_id);
 
 static int taskq_thread_spawn(taskq_t *tq);
 
 taskqid_t
 taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
 {
 	taskq_ent_t *t;
 	taskqid_t rc = TASKQID_INVALID;
 	unsigned long irqflags;
 
 	ASSERT(tq);
 	ASSERT(func);
 
 	spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class);
 
 	/* Taskq being destroyed and all tasks drained */
 	if (!(tq->tq_flags & TASKQ_ACTIVE))
 		goto out;
 
 	/* Do not queue the task unless there is idle thread for it */
 	ASSERT(tq->tq_nactive <= tq->tq_nthreads);
 	if ((flags & TQ_NOQUEUE) && (tq->tq_nactive == tq->tq_nthreads)) {
 		/* Dynamic taskq may be able to spawn another thread */
 		if (taskq_thread_spawn(tq) == 0)
 			goto out;
 	}
 
 	if ((t = task_alloc(tq, flags, &irqflags)) == NULL)
 		goto out;
 
 	spin_lock(&t->tqent_lock);
 
 	/* Queue to the front of the list to enforce TQ_NOQUEUE semantics */
 	if (flags & TQ_NOQUEUE)
 		list_add(&t->tqent_list, &tq->tq_prio_list);
 	/* Queue to the priority list instead of the pending list */
 	else if (flags & TQ_FRONT)
 		list_add_tail(&t->tqent_list, &tq->tq_prio_list);
 	else
 		list_add_tail(&t->tqent_list, &tq->tq_pend_list);
 
 	t->tqent_id = rc = tq->tq_next_id;
 	tq->tq_next_id++;
 	t->tqent_func = func;
 	t->tqent_arg = arg;
 	t->tqent_taskq = tq;
 	t->tqent_timer.function = NULL;
 	t->tqent_timer.expires = 0;
 
 	t->tqent_birth = jiffies;
 	DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t);
 
 	ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));
 
 	spin_unlock(&t->tqent_lock);
 
 	wake_up(&tq->tq_work_waitq);
 
 	/* Spawn additional taskq threads if required. */
 	if (!(flags & TQ_NOQUEUE) && tq->tq_nactive == tq->tq_nthreads)
 		(void) taskq_thread_spawn(tq);
 out:
 	spin_unlock_irqrestore(&tq->tq_lock, irqflags);
 	return (rc);
 }
 EXPORT_SYMBOL(taskq_dispatch);
 
 taskqid_t
 taskq_dispatch_delay(taskq_t *tq, task_func_t func, void *arg,
     uint_t flags, clock_t expire_time)
 {
 	taskqid_t rc = TASKQID_INVALID;
 	taskq_ent_t *t;
 	unsigned long irqflags;
 
 	ASSERT(tq);
 	ASSERT(func);
 
 	spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class);
 
 	/* Taskq being destroyed and all tasks drained */
 	if (!(tq->tq_flags & TASKQ_ACTIVE))
 		goto out;
 
 	if ((t = task_alloc(tq, flags, &irqflags)) == NULL)
 		goto out;
 
 	spin_lock(&t->tqent_lock);
 
 	/* Queue to the delay list for subsequent execution */
 	list_add_tail(&t->tqent_list, &tq->tq_delay_list);
 
 	t->tqent_id = rc = tq->tq_next_id;
 	tq->tq_next_id++;
 	t->tqent_func = func;
 	t->tqent_arg = arg;
 	t->tqent_taskq = tq;
 	t->tqent_timer.function = task_expire;
 	t->tqent_timer.expires = (unsigned long)expire_time;
 	add_timer(&t->tqent_timer);
 
 	ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));
 
 	spin_unlock(&t->tqent_lock);
 
 	/* Spawn additional taskq threads if required. */
 	if (tq->tq_nactive == tq->tq_nthreads)
 		(void) taskq_thread_spawn(tq);
 out:
 	spin_unlock_irqrestore(&tq->tq_lock, irqflags);
 	return (rc);
 }
 EXPORT_SYMBOL(taskq_dispatch_delay);
 
 void
 taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint_t flags,
     taskq_ent_t *t)
 {
 	unsigned long irqflags;
 	ASSERT(tq);
 	ASSERT(func);
 
 	spin_lock_irqsave_nested(&tq->tq_lock, irqflags,
 	    tq->tq_lock_class);
 
 	/* Taskq being destroyed and all tasks drained */
 	if (!(tq->tq_flags & TASKQ_ACTIVE)) {
 		t->tqent_id = TASKQID_INVALID;
 		goto out;
 	}
 
 	if ((flags & TQ_NOQUEUE) && (tq->tq_nactive == tq->tq_nthreads)) {
 		/* Dynamic taskq may be able to spawn another thread */
 		if (taskq_thread_spawn(tq) == 0)
 			goto out;
 		flags |= TQ_FRONT;
 	}
 
 	spin_lock(&t->tqent_lock);
 
 	/*
 	 * Make sure the entry is not on some other taskq; it is important to
 	 * ASSERT() under lock
 	 */
 	ASSERT(taskq_empty_ent(t));
 
 	/*
 	 * Mark it as a prealloc'd task.  This is important
 	 * to ensure that we don't free it later.
 	 */
 	t->tqent_flags |= TQENT_FLAG_PREALLOC;
 
 	/* Queue to the priority list instead of the pending list */
 	if (flags & TQ_FRONT)
 		list_add_tail(&t->tqent_list, &tq->tq_prio_list);
 	else
 		list_add_tail(&t->tqent_list, &tq->tq_pend_list);
 
 	t->tqent_id = tq->tq_next_id;
 	tq->tq_next_id++;
 	t->tqent_func = func;
 	t->tqent_arg = arg;
 	t->tqent_taskq = tq;
 
 	t->tqent_birth = jiffies;
 	DTRACE_PROBE1(taskq_ent__birth, taskq_ent_t *, t);
 
 	spin_unlock(&t->tqent_lock);
 
 	wake_up(&tq->tq_work_waitq);
 
 	/* Spawn additional taskq threads if required. */
 	if (tq->tq_nactive == tq->tq_nthreads)
 		(void) taskq_thread_spawn(tq);
 out:
 	spin_unlock_irqrestore(&tq->tq_lock, irqflags);
 }
 EXPORT_SYMBOL(taskq_dispatch_ent);
 
 int
 taskq_empty_ent(taskq_ent_t *t)
 {
 	return (list_empty(&t->tqent_list));
 }
 EXPORT_SYMBOL(taskq_empty_ent);
 
 void
 taskq_init_ent(taskq_ent_t *t)
 {
 	spin_lock_init(&t->tqent_lock);
 	init_waitqueue_head(&t->tqent_waitq);
 	timer_setup(&t->tqent_timer, NULL, 0);
 	INIT_LIST_HEAD(&t->tqent_list);
 	t->tqent_id = 0;
 	t->tqent_func = NULL;
 	t->tqent_arg = NULL;
 	t->tqent_flags = 0;
 	t->tqent_taskq = NULL;
 }
 EXPORT_SYMBOL(taskq_init_ent);
 
 /*
  * Return the next pending task, preference is given to tasks on the
  * priority list which were dispatched with TQ_FRONT.
  */
 static taskq_ent_t *
 taskq_next_ent(taskq_t *tq)
 {
 	struct list_head *list;
 
 	if (!list_empty(&tq->tq_prio_list))
 		list = &tq->tq_prio_list;
 	else if (!list_empty(&tq->tq_pend_list))
 		list = &tq->tq_pend_list;
 	else
 		return (NULL);
 
 	return (list_entry(list->next, taskq_ent_t, tqent_list));
 }
 
 /*
  * Spawns a new thread for the specified taskq.
  */
 static void
 taskq_thread_spawn_task(void *arg)
 {
 	taskq_t *tq = (taskq_t *)arg;
 	unsigned long flags;
 
 	if (taskq_thread_create(tq) == NULL) {
 		/* restore spawning count if failed */
 		spin_lock_irqsave_nested(&tq->tq_lock, flags,
 		    tq->tq_lock_class);
 		tq->tq_nspawn--;
 		spin_unlock_irqrestore(&tq->tq_lock, flags);
 	}
 }
 
 /*
  * Spawn addition threads for dynamic taskqs (TASKQ_DYNAMIC) the current
  * number of threads is insufficient to handle the pending tasks.  These
  * new threads must be created by the dedicated dynamic_taskq to avoid
  * deadlocks between thread creation and memory reclaim.  The system_taskq
  * which is also a dynamic taskq cannot be safely used for this.
  */
 static int
 taskq_thread_spawn(taskq_t *tq)
 {
 	int spawning = 0;
 
 	if (!(tq->tq_flags & TASKQ_DYNAMIC))
 		return (0);
 
 	tq->lastspawnstop = jiffies;
 	if ((tq->tq_nthreads + tq->tq_nspawn < tq->tq_maxthreads) &&
 	    (tq->tq_flags & TASKQ_ACTIVE)) {
 		spawning = (++tq->tq_nspawn);
 		taskq_dispatch(dynamic_taskq, taskq_thread_spawn_task,
 		    tq, TQ_NOSLEEP);
 	}
 
 	return (spawning);
 }
 
 /*
  * Threads in a dynamic taskq may exit once there is no more work to do.
  * To prevent threads from being created and destroyed too often limit
  * the exit rate to one per spl_taskq_thread_timeout_ms.
  *
  * The first thread is the thread list is treated as the primary thread.
  * There is nothing special about the primary thread but in order to avoid
  * all the taskq pids from changing we opt to make it long running.
  */
 static int
 taskq_thread_should_stop(taskq_t *tq, taskq_thread_t *tqt)
 {
 	ASSERT(!taskq_next_ent(tq));
 	if (!(tq->tq_flags & TASKQ_DYNAMIC) || !spl_taskq_thread_dynamic)
 		return (0);
 	if (!(tq->tq_flags & TASKQ_ACTIVE))
 		return (1);
 	if (list_first_entry(&(tq->tq_thread_list), taskq_thread_t,
 	    tqt_thread_list) == tqt)
 		return (0);
 	ASSERT3U(tq->tq_nthreads, >, 1);
 	if (tq->tq_nspawn != 0)
 		return (0);
 	if (time_before(jiffies, tq->lastspawnstop +
 	    msecs_to_jiffies(spl_taskq_thread_timeout_ms)))
 		return (0);
 	tq->lastspawnstop = jiffies;
 	return (1);
 }
 
 static int
 taskq_thread(void *args)
 {
 	DECLARE_WAITQUEUE(wait, current);
 	sigset_t blocked;
 	taskq_thread_t *tqt = args;
 	taskq_t *tq;
 	taskq_ent_t *t;
 	int seq_tasks = 0;
 	unsigned long flags;
 	taskq_ent_t dup_task = {};
 
 	ASSERT(tqt);
 	ASSERT(tqt->tqt_tq);
 	tq = tqt->tqt_tq;
 	current->flags |= PF_NOFREEZE;
 
 	(void) spl_fstrans_mark();
 
 	sigfillset(&blocked);
 	sigprocmask(SIG_BLOCK, &blocked, NULL);
 	flush_signals(current);
 
 	tsd_set(taskq_tsd, tq);
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 	/*
 	 * If we are dynamically spawned, decrease spawning count. Note that
 	 * we could be created during taskq_create, in which case we shouldn't
 	 * do the decrement. But it's fine because taskq_create will reset
 	 * tq_nspawn later.
 	 */
 	if (tq->tq_flags & TASKQ_DYNAMIC)
 		tq->tq_nspawn--;
 
 	/* Immediately exit if more threads than allowed were created. */
 	if (tq->tq_nthreads >= tq->tq_maxthreads)
 		goto error;
 
 	tq->tq_nthreads++;
 	list_add_tail(&tqt->tqt_thread_list, &tq->tq_thread_list);
 	wake_up(&tq->tq_wait_waitq);
 	set_current_state(TASK_INTERRUPTIBLE);
 
 	while (!kthread_should_stop()) {
 
 		if (list_empty(&tq->tq_pend_list) &&
 		    list_empty(&tq->tq_prio_list)) {
 
 			if (taskq_thread_should_stop(tq, tqt))
 				break;
 
 			add_wait_queue_exclusive(&tq->tq_work_waitq, &wait);
 			spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 			schedule();
 			seq_tasks = 0;
 
 			spin_lock_irqsave_nested(&tq->tq_lock, flags,
 			    tq->tq_lock_class);
 			remove_wait_queue(&tq->tq_work_waitq, &wait);
 		} else {
 			__set_current_state(TASK_RUNNING);
 		}
 
 		if ((t = taskq_next_ent(tq)) != NULL) {
 			list_del_init(&t->tqent_list);
 
 			/*
 			 * A TQENT_FLAG_PREALLOC task may be reused or freed
 			 * during the task function call. Store tqent_id and
 			 * tqent_flags here.
 			 *
 			 * Also use an on stack taskq_ent_t for tqt_task
 			 * assignment in this case; we want to make sure
 			 * to duplicate all fields, so the values are
 			 * correct when it's accessed via DTRACE_PROBE*.
 			 */
 			tqt->tqt_id = t->tqent_id;
 			tqt->tqt_flags = t->tqent_flags;
 
 			if (t->tqent_flags & TQENT_FLAG_PREALLOC) {
 				dup_task = *t;
 				t = &dup_task;
 			}
 			tqt->tqt_task = t;
 
 			taskq_insert_in_order(tq, tqt);
 			tq->tq_nactive++;
 			spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 			DTRACE_PROBE1(taskq_ent__start, taskq_ent_t *, t);
 
 			/* Perform the requested task */
 			t->tqent_func(t->tqent_arg);
 
 			DTRACE_PROBE1(taskq_ent__finish, taskq_ent_t *, t);
 
 			spin_lock_irqsave_nested(&tq->tq_lock, flags,
 			    tq->tq_lock_class);
 			tq->tq_nactive--;
 			list_del_init(&tqt->tqt_active_list);
 			tqt->tqt_task = NULL;
 
 			/* For prealloc'd tasks, we don't free anything. */
 			if (!(tqt->tqt_flags & TQENT_FLAG_PREALLOC))
 				task_done(tq, t);
 
 			/*
 			 * When the current lowest outstanding taskqid is
 			 * done calculate the new lowest outstanding id
 			 */
 			if (tq->tq_lowest_id == tqt->tqt_id) {
 				tq->tq_lowest_id = taskq_lowest_id(tq);
 				ASSERT3S(tq->tq_lowest_id, >, tqt->tqt_id);
 			}
 
 			/* Spawn additional taskq threads if required. */
 			if ((++seq_tasks) > spl_taskq_thread_sequential &&
 			    taskq_thread_spawn(tq))
 				seq_tasks = 0;
 
 			tqt->tqt_id = TASKQID_INVALID;
 			tqt->tqt_flags = 0;
 			wake_up_all(&tq->tq_wait_waitq);
 		}
 
 		set_current_state(TASK_INTERRUPTIBLE);
 
 	}
 
 	__set_current_state(TASK_RUNNING);
 	tq->tq_nthreads--;
 	list_del_init(&tqt->tqt_thread_list);
 error:
 	kmem_free(tqt, sizeof (taskq_thread_t));
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 	tsd_set(taskq_tsd, NULL);
 	thread_exit();
 
 	return (0);
 }
 
 static taskq_thread_t *
 taskq_thread_create(taskq_t *tq)
 {
 	static int last_used_cpu = 0;
 	taskq_thread_t *tqt;
 
 	tqt = kmem_alloc(sizeof (*tqt), KM_PUSHPAGE);
 	INIT_LIST_HEAD(&tqt->tqt_thread_list);
 	INIT_LIST_HEAD(&tqt->tqt_active_list);
 	tqt->tqt_tq = tq;
 	tqt->tqt_id = TASKQID_INVALID;
 
 	tqt->tqt_thread = spl_kthread_create(taskq_thread, tqt,
 	    "%s", tq->tq_name);
 	if (tqt->tqt_thread == NULL) {
 		kmem_free(tqt, sizeof (taskq_thread_t));
 		return (NULL);
 	}
 
 	if (spl_taskq_thread_bind) {
 		last_used_cpu = (last_used_cpu + 1) % num_online_cpus();
 		kthread_bind(tqt->tqt_thread, last_used_cpu);
 	}
 
 	if (spl_taskq_thread_priority)
 		set_user_nice(tqt->tqt_thread, PRIO_TO_NICE(tq->tq_pri));
 
 	wake_up_process(tqt->tqt_thread);
 
 	return (tqt);
 }
 
 taskq_t *
 taskq_create(const char *name, int threads_arg, pri_t pri,
     int minalloc, int maxalloc, uint_t flags)
 {
 	taskq_t *tq;
 	taskq_thread_t *tqt;
 	int count = 0, rc = 0, i;
 	unsigned long irqflags;
 	int nthreads = threads_arg;
 
 	ASSERT(name != NULL);
 	ASSERT(minalloc >= 0);
 	ASSERT(!(flags & (TASKQ_CPR_SAFE))); /* Unsupported */
 
 	/* Scale the number of threads using nthreads as a percentage */
 	if (flags & TASKQ_THREADS_CPU_PCT) {
 		ASSERT(nthreads <= 100);
 		ASSERT(nthreads >= 0);
 		nthreads = MIN(threads_arg, 100);
 		nthreads = MAX(nthreads, 0);
 		nthreads = MAX((num_online_cpus() * nthreads) /100, 1);
 	}
 
 	tq = kmem_alloc(sizeof (*tq), KM_PUSHPAGE);
 	if (tq == NULL)
 		return (NULL);
 
 	tq->tq_hp_support = B_FALSE;
 
 	if (flags & TASKQ_THREADS_CPU_PCT) {
 		tq->tq_hp_support = B_TRUE;
 		if (cpuhp_state_add_instance_nocalls(spl_taskq_cpuhp_state,
 		    &tq->tq_hp_cb_node) != 0) {
 			kmem_free(tq, sizeof (*tq));
 			return (NULL);
 		}
 	}
 
 	spin_lock_init(&tq->tq_lock);
 	INIT_LIST_HEAD(&tq->tq_thread_list);
 	INIT_LIST_HEAD(&tq->tq_active_list);
 	tq->tq_name = kmem_strdup(name);
 	tq->tq_nactive = 0;
 	tq->tq_nthreads = 0;
 	tq->tq_nspawn = 0;
 	tq->tq_maxthreads = nthreads;
 	tq->tq_cpu_pct = threads_arg;
 	tq->tq_pri = pri;
 	tq->tq_minalloc = minalloc;
 	tq->tq_maxalloc = maxalloc;
 	tq->tq_nalloc = 0;
 	tq->tq_flags = (flags | TASKQ_ACTIVE);
 	tq->tq_next_id = TASKQID_INITIAL;
 	tq->tq_lowest_id = TASKQID_INITIAL;
 	tq->lastspawnstop = jiffies;
 	INIT_LIST_HEAD(&tq->tq_free_list);
 	INIT_LIST_HEAD(&tq->tq_pend_list);
 	INIT_LIST_HEAD(&tq->tq_prio_list);
 	INIT_LIST_HEAD(&tq->tq_delay_list);
 	init_waitqueue_head(&tq->tq_work_waitq);
 	init_waitqueue_head(&tq->tq_wait_waitq);
 	tq->tq_lock_class = TQ_LOCK_GENERAL;
 	INIT_LIST_HEAD(&tq->tq_taskqs);
 
 	if (flags & TASKQ_PREPOPULATE) {
 		spin_lock_irqsave_nested(&tq->tq_lock, irqflags,
 		    tq->tq_lock_class);
 
 		for (i = 0; i < minalloc; i++)
 			task_done(tq, task_alloc(tq, TQ_PUSHPAGE | TQ_NEW,
 			    &irqflags));
 
 		spin_unlock_irqrestore(&tq->tq_lock, irqflags);
 	}
 
 	if ((flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic)
 		nthreads = 1;
 
 	for (i = 0; i < nthreads; i++) {
 		tqt = taskq_thread_create(tq);
 		if (tqt == NULL)
 			rc = 1;
 		else
 			count++;
 	}
 
 	/* Wait for all threads to be started before potential destroy */
 	wait_event(tq->tq_wait_waitq, tq->tq_nthreads == count);
 	/*
 	 * taskq_thread might have touched nspawn, but we don't want them to
 	 * because they're not dynamically spawned. So we reset it to 0
 	 */
 	tq->tq_nspawn = 0;
 
 	if (rc) {
 		taskq_destroy(tq);
 		tq = NULL;
 	} else {
 		down_write(&tq_list_sem);
 		tq->tq_instance = taskq_find_by_name(name) + 1;
 		list_add_tail(&tq->tq_taskqs, &tq_list);
 		up_write(&tq_list_sem);
 	}
 
 	return (tq);
 }
 EXPORT_SYMBOL(taskq_create);
 
 void
 taskq_destroy(taskq_t *tq)
 {
 	struct task_struct *thread;
 	taskq_thread_t *tqt;
 	taskq_ent_t *t;
 	unsigned long flags;
 
 	ASSERT(tq);
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 	tq->tq_flags &= ~TASKQ_ACTIVE;
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 	if (tq->tq_hp_support) {
 		VERIFY0(cpuhp_state_remove_instance_nocalls(
 		    spl_taskq_cpuhp_state, &tq->tq_hp_cb_node));
 	}
 
 	/*
 	 * When TASKQ_ACTIVE is clear new tasks may not be added nor may
 	 * new worker threads be spawned for dynamic taskq.
 	 */
 	if (dynamic_taskq != NULL)
 		taskq_wait_outstanding(dynamic_taskq, 0);
 
 	taskq_wait(tq);
 
 	/* remove taskq from global list used by the kstats */
 	down_write(&tq_list_sem);
 	list_del(&tq->tq_taskqs);
 	up_write(&tq_list_sem);
 
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 	/* wait for spawning threads to insert themselves to the list */
 	while (tq->tq_nspawn) {
 		spin_unlock_irqrestore(&tq->tq_lock, flags);
 		schedule_timeout_interruptible(1);
 		spin_lock_irqsave_nested(&tq->tq_lock, flags,
 		    tq->tq_lock_class);
 	}
 
 	/*
 	 * Signal each thread to exit and block until it does.  Each thread
 	 * is responsible for removing itself from the list and freeing its
 	 * taskq_thread_t.  This allows for idle threads to opt to remove
 	 * themselves from the taskq.  They can be recreated as needed.
 	 */
 	while (!list_empty(&tq->tq_thread_list)) {
 		tqt = list_entry(tq->tq_thread_list.next,
 		    taskq_thread_t, tqt_thread_list);
 		thread = tqt->tqt_thread;
 		spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 		kthread_stop(thread);
 
 		spin_lock_irqsave_nested(&tq->tq_lock, flags,
 		    tq->tq_lock_class);
 	}
 
 	while (!list_empty(&tq->tq_free_list)) {
 		t = list_entry(tq->tq_free_list.next, taskq_ent_t, tqent_list);
 
 		ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC));
 
 		list_del_init(&t->tqent_list);
 		task_free(tq, t);
 	}
 
 	ASSERT0(tq->tq_nthreads);
 	ASSERT0(tq->tq_nalloc);
 	ASSERT0(tq->tq_nspawn);
 	ASSERT(list_empty(&tq->tq_thread_list));
 	ASSERT(list_empty(&tq->tq_active_list));
 	ASSERT(list_empty(&tq->tq_free_list));
 	ASSERT(list_empty(&tq->tq_pend_list));
 	ASSERT(list_empty(&tq->tq_prio_list));
 	ASSERT(list_empty(&tq->tq_delay_list));
 
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 	kmem_strfree(tq->tq_name);
 	kmem_free(tq, sizeof (taskq_t));
 }
 EXPORT_SYMBOL(taskq_destroy);
 
 static unsigned int spl_taskq_kick = 0;
 
 /*
  * 2.6.36 API Change
  * module_param_cb is introduced to take kernel_param_ops and
  * module_param_call is marked as obsolete. Also set and get operations
  * were changed to take a 'const struct kernel_param *'.
  */
 static int
 #ifdef module_param_cb
 param_set_taskq_kick(const char *val, const struct kernel_param *kp)
 #else
 param_set_taskq_kick(const char *val, struct kernel_param *kp)
 #endif
 {
 	int ret;
 	taskq_t *tq = NULL;
 	taskq_ent_t *t;
 	unsigned long flags;
 
 	ret = param_set_uint(val, kp);
 	if (ret < 0 || !spl_taskq_kick)
 		return (ret);
 	/* reset value */
 	spl_taskq_kick = 0;
 
 	down_read(&tq_list_sem);
 	list_for_each_entry(tq, &tq_list, tq_taskqs) {
 		spin_lock_irqsave_nested(&tq->tq_lock, flags,
 		    tq->tq_lock_class);
 		/* Check if the first pending is older than 5 seconds */
 		t = taskq_next_ent(tq);
 		if (t && time_after(jiffies, t->tqent_birth + 5*HZ)) {
 			(void) taskq_thread_spawn(tq);
 			printk(KERN_INFO "spl: Kicked taskq %s/%d\n",
 			    tq->tq_name, tq->tq_instance);
 		}
 		spin_unlock_irqrestore(&tq->tq_lock, flags);
 	}
 	up_read(&tq_list_sem);
 	return (ret);
 }
 
 #ifdef module_param_cb
 static const struct kernel_param_ops param_ops_taskq_kick = {
 	.set = param_set_taskq_kick,
 	.get = param_get_uint,
 };
 module_param_cb(spl_taskq_kick, &param_ops_taskq_kick, &spl_taskq_kick, 0644);
 #else
 module_param_call(spl_taskq_kick, param_set_taskq_kick, param_get_uint,
 	&spl_taskq_kick, 0644);
 #endif
 MODULE_PARM_DESC(spl_taskq_kick,
 	"Write nonzero to kick stuck taskqs to spawn more threads");
 
 /*
  * This callback will be called exactly once for each core that comes online,
  * for each dynamic taskq. We attempt to expand taskqs that have
  * TASKQ_THREADS_CPU_PCT set. We need to redo the percentage calculation every
  * time, to correctly determine whether or not to add a thread.
  */
 static int
 spl_taskq_expand(unsigned int cpu, struct hlist_node *node)
 {
 	taskq_t *tq = list_entry(node, taskq_t, tq_hp_cb_node);
 	unsigned long flags;
 	int err = 0;
 
 	ASSERT(tq);
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 
 	if (!(tq->tq_flags & TASKQ_ACTIVE)) {
 		spin_unlock_irqrestore(&tq->tq_lock, flags);
 		return (err);
 	}
 
 	ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT);
 	int nthreads = MIN(tq->tq_cpu_pct, 100);
 	nthreads = MAX(((num_online_cpus() + 1) * nthreads) / 100, 1);
 	tq->tq_maxthreads = nthreads;
 
 	if (!((tq->tq_flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic) &&
 	    tq->tq_maxthreads > tq->tq_nthreads) {
 		spin_unlock_irqrestore(&tq->tq_lock, flags);
 		taskq_thread_t *tqt = taskq_thread_create(tq);
 		if (tqt == NULL)
 			err = -1;
 		return (err);
 	}
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 	return (err);
 }
 
 /*
  * While we don't support offlining CPUs, it is possible that CPUs will fail
  * to online successfully. We do need to be able to handle this case
  * gracefully.
  */
 static int
 spl_taskq_prepare_down(unsigned int cpu, struct hlist_node *node)
 {
 	taskq_t *tq = list_entry(node, taskq_t, tq_hp_cb_node);
 	unsigned long flags;
 
 	ASSERT(tq);
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 
 	if (!(tq->tq_flags & TASKQ_ACTIVE))
 		goto out;
 
 	ASSERT(tq->tq_flags & TASKQ_THREADS_CPU_PCT);
 	int nthreads = MIN(tq->tq_cpu_pct, 100);
 	nthreads = MAX(((num_online_cpus()) * nthreads) / 100, 1);
 	tq->tq_maxthreads = nthreads;
 
 	if (!((tq->tq_flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic) &&
 	    tq->tq_maxthreads < tq->tq_nthreads) {
 		ASSERT3U(tq->tq_maxthreads, ==, tq->tq_nthreads - 1);
 		taskq_thread_t *tqt = list_entry(tq->tq_thread_list.next,
 		    taskq_thread_t, tqt_thread_list);
 		struct task_struct *thread = tqt->tqt_thread;
 		spin_unlock_irqrestore(&tq->tq_lock, flags);
 
 		kthread_stop(thread);
 
 		return (0);
 	}
 
 out:
 	spin_unlock_irqrestore(&tq->tq_lock, flags);
 	return (0);
 }
 
 int
 spl_taskq_init(void)
 {
 	init_rwsem(&tq_list_sem);
 	tsd_create(&taskq_tsd, NULL);
 
 	spl_taskq_cpuhp_state = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 	    "fs/spl_taskq:online", spl_taskq_expand, spl_taskq_prepare_down);
 
 	system_taskq = taskq_create("spl_system_taskq", MAX(boot_ncpus, 64),
 	    maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC);
 	if (system_taskq == NULL)
 		return (-ENOMEM);
 
 	system_delay_taskq = taskq_create("spl_delay_taskq", MAX(boot_ncpus, 4),
 	    maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC);
 	if (system_delay_taskq == NULL) {
 		cpuhp_remove_multi_state(spl_taskq_cpuhp_state);
 		taskq_destroy(system_taskq);
 		return (-ENOMEM);
 	}
 
 	dynamic_taskq = taskq_create("spl_dynamic_taskq", 1,
 	    maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE);
 	if (dynamic_taskq == NULL) {
 		cpuhp_remove_multi_state(spl_taskq_cpuhp_state);
 		taskq_destroy(system_taskq);
 		taskq_destroy(system_delay_taskq);
 		return (-ENOMEM);
 	}
 
 	/*
 	 * This is used to annotate tq_lock, so
 	 *   taskq_dispatch -> taskq_thread_spawn -> taskq_dispatch
 	 * does not trigger a lockdep warning re: possible recursive locking
 	 */
 	dynamic_taskq->tq_lock_class = TQ_LOCK_DYNAMIC;
 
 	return (0);
 }
 
 void
 spl_taskq_fini(void)
 {
 	taskq_destroy(dynamic_taskq);
 	dynamic_taskq = NULL;
 
 	taskq_destroy(system_delay_taskq);
 	system_delay_taskq = NULL;
 
 	taskq_destroy(system_taskq);
 	system_taskq = NULL;
 
 	tsd_destroy(&taskq_tsd);
 
 	cpuhp_remove_multi_state(spl_taskq_cpuhp_state);
 	spl_taskq_cpuhp_state = 0;
 }
diff --git a/sys/contrib/openzfs/module/os/linux/spl/spl-thread.c b/sys/contrib/openzfs/module/os/linux/spl/spl-thread.c
index 7f74d44f91ff..7b0ce30c7884 100644
--- a/sys/contrib/openzfs/module/os/linux/spl/spl-thread.c
+++ b/sys/contrib/openzfs/module/os/linux/spl/spl-thread.c
@@ -1,198 +1,198 @@
 /*
  *  Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
  *  Copyright (C) 2007 The Regents of the University of California.
  *  Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  *  Written by Brian Behlendorf <behlendorf1@llnl.gov>.
  *  UCRL-CODE-235197
  *
  *  This file is part of the SPL, Solaris Porting Layer.
  *
  *  The SPL is free software; you can redistribute it and/or modify it
  *  under the terms of the GNU General Public License as published by the
  *  Free Software Foundation; either version 2 of the License, or (at your
  *  option) any later version.
  *
  *  The SPL is distributed in the hope that it will be useful, but WITHOUT
  *  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  *  FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  *  for more details.
  *
  *  You should have received a copy of the GNU General Public License along
  *  with the SPL.  If not, see <http://www.gnu.org/licenses/>.
  *
  *  Solaris Porting Layer (SPL) Thread Implementation.
  */
 
 #include <sys/thread.h>
 #include <sys/kmem.h>
 #include <sys/tsd.h>
 #include <sys/string.h>
 
 /*
  * Thread interfaces
  */
 typedef struct thread_priv_s {
 	unsigned long tp_magic;		/* Magic */
 	int tp_name_size;		/* Name size */
 	char *tp_name;			/* Name (without _thread suffix) */
 	void (*tp_func)(void *);	/* Registered function */
 	void *tp_args;			/* Args to be passed to function */
 	size_t tp_len;			/* Len to be passed to function */
 	int tp_state;			/* State to start thread at */
 	pri_t tp_pri;			/* Priority to start threat at */
 } thread_priv_t;
 
 static int
 thread_generic_wrapper(void *arg)
 {
 	thread_priv_t *tp = (thread_priv_t *)arg;
 	void (*func)(void *);
 	void *args;
 
 	ASSERT(tp->tp_magic == TP_MAGIC);
 	func = tp->tp_func;
 	args = tp->tp_args;
 	set_current_state(tp->tp_state);
 	set_user_nice((kthread_t *)current, PRIO_TO_NICE(tp->tp_pri));
 	kmem_free(tp->tp_name, tp->tp_name_size);
 	kmem_free(tp, sizeof (thread_priv_t));
 
 	if (func)
 		func(args);
 
 	return (0);
 }
 
 /*
  * thread_create() may block forever if it cannot create a thread or
  * allocate memory.  This is preferable to returning a NULL which Solaris
  * style callers likely never check for... since it can't fail.
  */
 kthread_t *
 __thread_create(caddr_t stk, size_t  stksize, thread_func_t func,
     const char *name, void *args, size_t len, proc_t *pp, int state, pri_t pri)
 {
 	thread_priv_t *tp;
 	struct task_struct *tsk;
 	char *p;
 
 	/* Option pp is simply ignored */
 	/* Variable stack size unsupported */
 	ASSERT(stk == NULL);
 
 	tp = kmem_alloc(sizeof (thread_priv_t), KM_PUSHPAGE);
 	if (tp == NULL)
 		return (NULL);
 
 	tp->tp_magic = TP_MAGIC;
 	tp->tp_name_size = strlen(name) + 1;
 
 	tp->tp_name = kmem_alloc(tp->tp_name_size, KM_PUSHPAGE);
 	if (tp->tp_name == NULL) {
 		kmem_free(tp, sizeof (thread_priv_t));
 		return (NULL);
 	}
 
 	strlcpy(tp->tp_name, name, tp->tp_name_size);
 
 	/*
 	 * Strip trailing "_thread" from passed name which will be the func
 	 * name since the exposed API has no parameter for passing a name.
 	 */
 	p = strstr(tp->tp_name, "_thread");
 	if (p)
 		p[0] = '\0';
 
 	tp->tp_func  = func;
 	tp->tp_args  = args;
 	tp->tp_len   = len;
 	tp->tp_state = state;
 	tp->tp_pri   = pri;
 
 	tsk = spl_kthread_create(thread_generic_wrapper, (void *)tp,
 	    "%s", tp->tp_name);
 	if (IS_ERR(tsk))
 		return (NULL);
 
 	wake_up_process(tsk);
 	return ((kthread_t *)tsk);
 }
 EXPORT_SYMBOL(__thread_create);
 
 /*
  * spl_kthread_create - Wrapper providing pre-3.13 semantics for
  * kthread_create() in which it is not killable and less likely
  * to return -ENOMEM.
  */
 struct task_struct *
 spl_kthread_create(int (*func)(void *), void *data, const char namefmt[], ...)
 {
 	struct task_struct *tsk;
 	va_list args;
 	char name[TASK_COMM_LEN];
 
 	va_start(args, namefmt);
 	vsnprintf(name, sizeof (name), namefmt, args);
 	va_end(args);
 	do {
 		tsk = kthread_create(func, data, "%s", name);
 		if (IS_ERR(tsk)) {
 			if (signal_pending(current)) {
 				clear_thread_flag(TIF_SIGPENDING);
 				continue;
 			}
 			if (PTR_ERR(tsk) == -ENOMEM)
 				continue;
 			return (NULL);
 		} else {
 			return (tsk);
 		}
 	} while (1);
 }
 EXPORT_SYMBOL(spl_kthread_create);
 
 /*
  * Extract the next pending signal from p_sig into p_cursig; stop the process
  * if a stop has been requested or if a traced signal is pending.
  */
 int
 issig(void)
 {
 
 	if (!signal_pending(current))
 		return (0);
 
 	spl_kernel_siginfo_t __info;
 	sigset_t set;
 	siginitsetinv(&set, 1ULL << (SIGSTOP - 1) | 1ULL << (SIGTSTP - 1));
 	sigorsets(&set, &current->blocked, &set);
 
 	spin_lock_irq(&current->sighand->siglock);
 #if defined(HAVE_DEQUEUE_SIGNAL_4ARG)
 	enum pid_type __type;
 	if (dequeue_signal(current, &set, &__info, &__type) != 0) {
-#elif defined(HAVE_DEQUEUE_SIGNAL_3ARG_TASK)
-	if (dequeue_signal(current, &set, &__info) != 0) {
-#else
+#elif defined(HAVE_DEQUEUE_SIGNAL_3ARG_TYPE)
 	enum pid_type __type;
 	if (dequeue_signal(&set, &__info, &__type) != 0) {
+#else
+	if (dequeue_signal(current, &set, &__info) != 0) {
 #endif
 		spin_unlock_irq(&current->sighand->siglock);
 		kernel_signal_stop();
 
 		/*
 		 * Dequeued SIGSTOP/SIGTSTP.
 		 * Check if process has other singal pending.
 		 */
 		if (signal_pending(current))
 			return (1);
 
 		return (0);
 	}
 
 	spin_unlock_irq(&current->sighand->siglock);
 
 	return (1);
 }
 
 EXPORT_SYMBOL(issig);
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/policy.c b/sys/contrib/openzfs/module/os/linux/zfs/policy.c
index d21bc667ba69..b2916f4217c4 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/policy.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/policy.c
@@ -1,373 +1,365 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2013, Joyent, Inc. All rights reserved.
  * Copyright (C) 2016 Lawrence Livermore National Security, LLC.
+ * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
  *
  * For Linux the vast majority of this enforcement is already handled via
  * the standard Linux VFS permission checks.  However certain administrative
  * commands which bypass the standard mechanisms may need to make use of
  * this functionality.
  */
 
 #include <sys/policy.h>
 #include <linux/security.h>
 #include <linux/vfs_compat.h>
 
-/*
- * The passed credentials cannot be directly verified because Linux only
- * provides and interface to check the *current* process credentials.  In
- * order to handle this the capable() test is only run when the passed
- * credentials match the current process credentials or the kcred.  In
- * all other cases this function must fail and return the passed err.
- */
 static int
 priv_policy_ns(const cred_t *cr, int capability, int err,
     struct user_namespace *ns)
 {
-	if (cr != CRED() && (cr != kcred))
-		return (err);
+	/*
+	 * The passed credentials cannot be directly verified because Linux
+	 * only provides an interface to check the *current* process
+	 * credentials.  In order to handle this we check if the passed in
+	 * creds match the current process credentials or the kcred.  If not,
+	 * we swap the passed credentials into the current task, perform the
+	 * check, and then revert it before returning.
+	 */
+	const cred_t *old =
+	    (cr != CRED() && cr != kcred) ? override_creds(cr) : NULL;
 
 #if defined(CONFIG_USER_NS)
-	if (!(ns ? ns_capable(ns, capability) : capable(capability)))
+	if (ns ? ns_capable(ns, capability) : capable(capability))
 #else
-	if (!capable(capability))
+	if (capable(capability))
 #endif
-		return (err);
+		err = 0;
 
-	return (0);
+	if (old)
+		revert_creds(old);
+
+	return (err);
 }
 
 static int
 priv_policy(const cred_t *cr, int capability, int err)
 {
 	return (priv_policy_ns(cr, capability, err, cr->user_ns));
 }
 
 static int
 priv_policy_user(const cred_t *cr, int capability, int err)
 {
 	/*
 	 * All priv_policy_user checks are preceded by kuid/kgid_has_mapping()
 	 * checks. If we cannot do them, we shouldn't be using ns_capable()
 	 * since we don't know whether the affected files are valid in our
 	 * namespace.
 	 */
 #if defined(CONFIG_USER_NS)
 	return (priv_policy_ns(cr, capability, err, cr->user_ns));
 #else
 	return (priv_policy_ns(cr, capability, err, NULL));
 #endif
 }
 
 /*
  * Checks for operations that are either client-only or are used by
  * both clients and servers.
  */
 int
 secpolicy_nfs(const cred_t *cr)
 {
 	return (priv_policy(cr, CAP_SYS_ADMIN, EPERM));
 }
 
 /*
  * Catch all system configuration.
  */
 int
 secpolicy_sys_config(const cred_t *cr, boolean_t checkonly)
 {
 	return (priv_policy(cr, CAP_SYS_ADMIN, EPERM));
 }
 
 /*
  * Like secpolicy_vnode_access() but we get the actual wanted mode and the
  * current mode of the file, not the missing bits.
  *
  * Enforced in the Linux VFS.
  */
 int
 secpolicy_vnode_access2(const cred_t *cr, struct inode *ip, uid_t owner,
     mode_t curmode, mode_t wantmode)
 {
 	return (0);
 }
 
 /*
  * This is a special routine for ZFS; it is used to determine whether
  * any of the privileges in effect allow any form of access to the
  * file.  There's no reason to audit this or any reason to record
  * this.  More work is needed to do the "KPLD" stuff.
  */
 int
 secpolicy_vnode_any_access(const cred_t *cr, struct inode *ip, uid_t owner)
 {
 	if (crgetuid(cr) == owner)
 		return (0);
 
 	if (zpl_inode_owner_or_capable(zfs_init_idmap, ip))
 		return (0);
 
 #if defined(CONFIG_USER_NS)
 	if (!kuid_has_mapping(cr->user_ns, SUID_TO_KUID(owner)))
 		return (EPERM);
 #endif
 
 	if (priv_policy_user(cr, CAP_DAC_OVERRIDE, EPERM) == 0)
 		return (0);
 
 	if (priv_policy_user(cr, CAP_DAC_READ_SEARCH, EPERM) == 0)
 		return (0);
 
 	return (EPERM);
 }
 
 /*
  * Determine if subject can chown owner of a file.
  */
 int
 secpolicy_vnode_chown(const cred_t *cr, uid_t owner)
 {
 	if (crgetuid(cr) == owner)
 		return (0);
 
 #if defined(CONFIG_USER_NS)
 	if (!kuid_has_mapping(cr->user_ns, SUID_TO_KUID(owner)))
 		return (EPERM);
 #endif
 
 	return (priv_policy_user(cr, CAP_FOWNER, EPERM));
 }
 
 /*
  * Determine if subject can change group ownership of a file.
  */
 int
 secpolicy_vnode_create_gid(const cred_t *cr)
 {
 	return (priv_policy(cr, CAP_SETGID, EPERM));
 }
 
 /*
  * Policy determines whether we can remove an entry from a directory,
  * regardless of permission bits.
  */
 int
 secpolicy_vnode_remove(const cred_t *cr)
 {
 	return (priv_policy(cr, CAP_FOWNER, EPERM));
 }
 
 /*
  * Determine that subject can modify the mode of a file.  allzone privilege
  * needed when modifying root owned object.
  */
 int
 secpolicy_vnode_setdac(const cred_t *cr, uid_t owner)
 {
 	if (crgetuid(cr) == owner)
 		return (0);
 
 #if defined(CONFIG_USER_NS)
 	if (!kuid_has_mapping(cr->user_ns, SUID_TO_KUID(owner)))
 		return (EPERM);
 #endif
 
 	return (priv_policy_user(cr, CAP_FOWNER, EPERM));
 }
 
 /*
  * Are we allowed to retain the set-uid/set-gid bits when
  * changing ownership or when writing to a file?
  * "issuid" should be true when set-uid; only in that case
  * root ownership is checked (setgid is assumed).
  *
  * Enforced in the Linux VFS.
  */
 int
 secpolicy_vnode_setid_retain(struct znode *zp __maybe_unused, const cred_t *cr,
     boolean_t issuidroot)
 {
 	return (priv_policy_user(cr, CAP_FSETID, EPERM));
 }
 
 /*
  * Determine that subject can set the file setgid flag.
  */
 int
 secpolicy_vnode_setids_setgids(const cred_t *cr, gid_t gid, zidmap_t *mnt_ns,
     struct user_namespace *fs_ns)
 {
 	gid = zfs_gid_to_vfsgid(mnt_ns, fs_ns, gid);
 #if defined(CONFIG_USER_NS)
 	if (!kgid_has_mapping(cr->user_ns, SGID_TO_KGID(gid)))
 		return (EPERM);
 #endif
 	if (crgetgid(cr) != gid && !groupmember(gid, cr))
 		return (priv_policy_user(cr, CAP_FSETID, EPERM));
 
 	return (0);
 }
 
 /*
  * Determine if the subject can inject faults in the ZFS fault injection
  * framework.  Requires all privileges.
  */
 int
 secpolicy_zinject(const cred_t *cr)
 {
 	return (priv_policy(cr, CAP_SYS_ADMIN, EACCES));
 }
 
 /*
  * Determine if the subject has permission to manipulate ZFS datasets
  * (not pools).  Equivalent to the SYS_MOUNT privilege.
  */
 int
 secpolicy_zfs(const cred_t *cr)
 {
 	return (priv_policy(cr, CAP_SYS_ADMIN, EACCES));
 }
 
-/*
- * Equivalent to secpolicy_zfs(), but works even if the cred_t is not that of
- * the current process.  Takes both cred_t and proc_t so that this can work
- * easily on all platforms.
- */
-int
-secpolicy_zfs_proc(const cred_t *cr, proc_t *proc)
-{
-	if (!has_capability(proc, CAP_SYS_ADMIN))
-		return (EACCES);
-	return (0);
-}
-
 void
 secpolicy_setid_clear(vattr_t *vap, cred_t *cr)
 {
 	if ((vap->va_mode & (S_ISUID | S_ISGID)) != 0 &&
 	    secpolicy_vnode_setid_retain(NULL, cr,
 	    (vap->va_mode & S_ISUID) != 0 &&
 	    (vap->va_mask & AT_UID) != 0 && vap->va_uid == 0) != 0) {
 		vap->va_mask |= AT_MODE;
 		vap->va_mode &= ~(S_ISUID|S_ISGID);
 	}
 }
 
 /*
  * Determine that subject can set the file setid flags.
  */
 static int
 secpolicy_vnode_setid_modify(const cred_t *cr, uid_t owner, zidmap_t *mnt_ns,
     struct user_namespace *fs_ns)
 {
 	owner = zfs_uid_to_vfsuid(mnt_ns, fs_ns, owner);
 
 	if (crgetuid(cr) == owner)
 		return (0);
 
 #if defined(CONFIG_USER_NS)
 	if (!kuid_has_mapping(cr->user_ns, SUID_TO_KUID(owner)))
 		return (EPERM);
 #endif
 
 	return (priv_policy_user(cr, CAP_FSETID, EPERM));
 }
 
 /*
  * Determine that subject can make a file a "sticky".
  *
  * Enforced in the Linux VFS.
  */
 static int
 secpolicy_vnode_stky_modify(const cred_t *cr)
 {
 	return (0);
 }
 
 int
 secpolicy_setid_setsticky_clear(struct inode *ip, vattr_t *vap,
     const vattr_t *ovap, cred_t *cr, zidmap_t *mnt_ns,
     struct user_namespace *fs_ns)
 {
 	int error;
 
 	if ((vap->va_mode & S_ISUID) != 0 &&
 	    (error = secpolicy_vnode_setid_modify(cr,
 	    ovap->va_uid, mnt_ns, fs_ns)) != 0) {
 		return (error);
 	}
 
 	/*
 	 * Check privilege if attempting to set the
 	 * sticky bit on a non-directory.
 	 */
 	if (!S_ISDIR(ip->i_mode) && (vap->va_mode & S_ISVTX) != 0 &&
 	    secpolicy_vnode_stky_modify(cr) != 0) {
 		vap->va_mode &= ~S_ISVTX;
 	}
 
 	/*
 	 * Check for privilege if attempting to set the
 	 * group-id bit.
 	 */
 	if ((vap->va_mode & S_ISGID) != 0 &&
 	    secpolicy_vnode_setids_setgids(cr, ovap->va_gid,
 	    mnt_ns, fs_ns) != 0) {
 		vap->va_mode &= ~S_ISGID;
 	}
 
 	return (0);
 }
 
 /*
  * Check privileges for setting xvattr attributes
  */
 int
 secpolicy_xvattr(xvattr_t *xvap, uid_t owner, cred_t *cr, mode_t type)
 {
 	return (secpolicy_vnode_chown(cr, owner));
 }
 
 /*
  * Check privileges for setattr attributes.
  *
  * Enforced in the Linux VFS.
  */
 int
 secpolicy_vnode_setattr(cred_t *cr, struct inode *ip, struct vattr *vap,
     const struct vattr *ovap, int flags,
     int unlocked_access(void *, int, cred_t *), void *node)
 {
 	return (0);
 }
 
 /*
  * Check privileges for links.
  *
  * Enforced in the Linux VFS.
  */
 int
 secpolicy_basic_link(const cred_t *cr)
 {
 	return (0);
 }
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c b/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c
index a1d03188d826..e27d7346a665 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/vdev_disk.c
@@ -1,1656 +1,1654 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
  * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>.
  * LLNL-CODE-403049.
  * Copyright (c) 2012, 2019 by Delphix. All rights reserved.
  * Copyright (c) 2023, 2024, Klara Inc.
  */
 
 #include <sys/zfs_context.h>
 #include <sys/spa_impl.h>
 #include <sys/vdev_disk.h>
 #include <sys/vdev_impl.h>
 #include <sys/vdev_trim.h>
 #include <sys/abd.h>
 #include <sys/fs/zfs.h>
 #include <sys/zio.h>
 #include <linux/blkpg.h>
 #include <linux/msdos_fs.h>
 #include <linux/vfs_compat.h>
 #include <linux/blk-cgroup.h>
 
 /*
  * Linux 6.8.x uses a bdev_handle as an instance/refcount for an underlying
  * block_device. Since it carries the block_device inside, its convenient to
  * just use the handle as a proxy.
  *
  * Linux 6.9.x uses a file for the same purpose.
  *
  * For pre-6.8, we just emulate this with a cast, since we don't need any of
  * the other fields inside the handle.
  */
 #if defined(HAVE_BDEV_OPEN_BY_PATH)
 typedef struct bdev_handle zfs_bdev_handle_t;
 #define	BDH_BDEV(bdh)		((bdh)->bdev)
 #define	BDH_IS_ERR(bdh)		(IS_ERR(bdh))
 #define	BDH_PTR_ERR(bdh)	(PTR_ERR(bdh))
 #define	BDH_ERR_PTR(err)	(ERR_PTR(err))
 #elif defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
 typedef struct file zfs_bdev_handle_t;
 #define	BDH_BDEV(bdh)		(file_bdev(bdh))
 #define	BDH_IS_ERR(bdh)		(IS_ERR(bdh))
 #define	BDH_PTR_ERR(bdh)	(PTR_ERR(bdh))
 #define	BDH_ERR_PTR(err)	(ERR_PTR(err))
 #else
 typedef void zfs_bdev_handle_t;
 #define	BDH_BDEV(bdh)		((struct block_device *)bdh)
 #define	BDH_IS_ERR(bdh)		(IS_ERR(BDH_BDEV(bdh)))
 #define	BDH_PTR_ERR(bdh)	(PTR_ERR(BDH_BDEV(bdh)))
 #define	BDH_ERR_PTR(err)	(ERR_PTR(err))
 #endif
 
 typedef struct vdev_disk {
 	zfs_bdev_handle_t		*vd_bdh;
 	krwlock_t			vd_lock;
 } vdev_disk_t;
 
 /*
  * Maximum number of segments to add to a bio (min 4). If this is higher than
  * the maximum allowed by the device queue or the kernel itself, it will be
  * clamped. Setting it to zero will cause the kernel's ideal size to be used.
  */
 uint_t zfs_vdev_disk_max_segs = 0;
 
 /*
  * Unique identifier for the exclusive vdev holder.
  */
 static void *zfs_vdev_holder = VDEV_HOLDER;
 
 /*
  * Wait up to zfs_vdev_open_timeout_ms milliseconds before determining the
  * device is missing. The missing path may be transient since the links
  * can be briefly removed and recreated in response to udev events.
  */
 static uint_t zfs_vdev_open_timeout_ms = 1000;
 
 /*
  * Size of the "reserved" partition, in blocks.
  */
 #define	EFI_MIN_RESV_SIZE	(16 * 1024)
 
 /*
  * BIO request failfast mask.
  */
 
 static unsigned int zfs_vdev_failfast_mask = 1;
 
 /*
  * Convert SPA mode flags into bdev open mode flags.
  */
 #ifdef HAVE_BLK_MODE_T
 typedef blk_mode_t vdev_bdev_mode_t;
 #define	VDEV_BDEV_MODE_READ	BLK_OPEN_READ
 #define	VDEV_BDEV_MODE_WRITE	BLK_OPEN_WRITE
 #define	VDEV_BDEV_MODE_EXCL	BLK_OPEN_EXCL
 #define	VDEV_BDEV_MODE_MASK	(BLK_OPEN_READ|BLK_OPEN_WRITE|BLK_OPEN_EXCL)
 #else
 typedef fmode_t vdev_bdev_mode_t;
 #define	VDEV_BDEV_MODE_READ	FMODE_READ
 #define	VDEV_BDEV_MODE_WRITE	FMODE_WRITE
 #define	VDEV_BDEV_MODE_EXCL	FMODE_EXCL
 #define	VDEV_BDEV_MODE_MASK	(FMODE_READ|FMODE_WRITE|FMODE_EXCL)
 #endif
 
 static vdev_bdev_mode_t
 vdev_bdev_mode(spa_mode_t smode)
 {
 	ASSERT3U(smode, !=, SPA_MODE_UNINIT);
 	ASSERT0(smode & ~(SPA_MODE_READ|SPA_MODE_WRITE));
 
 	vdev_bdev_mode_t bmode = VDEV_BDEV_MODE_EXCL;
 
 	if (smode & SPA_MODE_READ)
 		bmode |= VDEV_BDEV_MODE_READ;
 
 	if (smode & SPA_MODE_WRITE)
 		bmode |= VDEV_BDEV_MODE_WRITE;
 
 	ASSERT(bmode & VDEV_BDEV_MODE_MASK);
 	ASSERT0(bmode & ~VDEV_BDEV_MODE_MASK);
 
 	return (bmode);
 }
 
 /*
  * Returns the usable capacity (in bytes) for the partition or disk.
  */
 static uint64_t
 bdev_capacity(struct block_device *bdev)
 {
 #ifdef HAVE_BDEV_NR_BYTES
 	return (bdev_nr_bytes(bdev));
 #else
 	return (i_size_read(bdev->bd_inode));
 #endif
 }
 
 #if !defined(HAVE_BDEV_WHOLE)
 static inline struct block_device *
 bdev_whole(struct block_device *bdev)
 {
 	return (bdev->bd_contains);
 }
 #endif
 
 #if defined(HAVE_BDEVNAME)
 #define	vdev_bdevname(bdev, name)	bdevname(bdev, name)
 #else
 static inline void
 vdev_bdevname(struct block_device *bdev, char *name)
 {
 	snprintf(name, BDEVNAME_SIZE, "%pg", bdev);
 }
 #endif
 
 /*
  * Returns the maximum expansion capacity of the block device (in bytes).
  *
  * It is possible to expand a vdev when it has been created as a wholedisk
  * and the containing block device has increased in capacity.  Or when the
  * partition containing the pool has been manually increased in size.
  *
  * This function is only responsible for calculating the potential expansion
  * size so it can be reported by 'zpool list'.  The efi_use_whole_disk() is
  * responsible for verifying the expected partition layout in the wholedisk
  * case, and updating the partition table if appropriate.  Once the partition
  * size has been increased the additional capacity will be visible using
  * bdev_capacity().
  *
  * The returned maximum expansion capacity is always expected to be larger, or
  * at the very least equal, to its usable capacity to prevent overestimating
  * the pool expandsize.
  */
 static uint64_t
 bdev_max_capacity(struct block_device *bdev, uint64_t wholedisk)
 {
 	uint64_t psize;
 	int64_t available;
 
 	if (wholedisk && bdev != bdev_whole(bdev)) {
 		/*
 		 * When reporting maximum expansion capacity for a wholedisk
 		 * deduct any capacity which is expected to be lost due to
 		 * alignment restrictions.  Over reporting this value isn't
 		 * harmful and would only result in slightly less capacity
 		 * than expected post expansion.
 		 * The estimated available space may be slightly smaller than
 		 * bdev_capacity() for devices where the number of sectors is
 		 * not a multiple of the alignment size and the partition layout
 		 * is keeping less than PARTITION_END_ALIGNMENT bytes after the
 		 * "reserved" EFI partition: in such cases return the device
 		 * usable capacity.
 		 */
 		available = bdev_capacity(bdev_whole(bdev)) -
 		    ((EFI_MIN_RESV_SIZE + NEW_START_BLOCK +
 		    PARTITION_END_ALIGNMENT) << SECTOR_BITS);
 		psize = MAX(available, bdev_capacity(bdev));
 	} else {
 		psize = bdev_capacity(bdev);
 	}
 
 	return (psize);
 }
 
 static void
 vdev_disk_error(zio_t *zio)
 {
 	/*
 	 * This function can be called in interrupt context, for instance while
 	 * handling IRQs coming from a misbehaving disk device; use printk()
 	 * which is safe from any context.
 	 */
 	printk(KERN_WARNING "zio pool=%s vdev=%s error=%d type=%d "
 	    "offset=%llu size=%llu flags=%llu\n", spa_name(zio->io_spa),
 	    zio->io_vd->vdev_path, zio->io_error, zio->io_type,
 	    (u_longlong_t)zio->io_offset, (u_longlong_t)zio->io_size,
 	    zio->io_flags);
 }
 
 static void
 vdev_disk_kobj_evt_post(vdev_t *v)
 {
 	vdev_disk_t *vd = v->vdev_tsd;
 	if (vd && vd->vd_bdh) {
 		spl_signal_kobj_evt(BDH_BDEV(vd->vd_bdh));
 	} else {
 		vdev_dbgmsg(v, "vdev_disk_t is NULL for VDEV:%s\n",
 		    v->vdev_path);
 	}
 }
 
 static zfs_bdev_handle_t *
 vdev_blkdev_get_by_path(const char *path, spa_mode_t smode, void *holder)
 {
 	vdev_bdev_mode_t bmode = vdev_bdev_mode(smode);
 
 #if defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
 	return (bdev_file_open_by_path(path, bmode, holder, NULL));
 #elif defined(HAVE_BDEV_OPEN_BY_PATH)
 	return (bdev_open_by_path(path, bmode, holder, NULL));
 #elif defined(HAVE_BLKDEV_GET_BY_PATH_4ARG)
 	return (blkdev_get_by_path(path, bmode, holder, NULL));
 #else
 	return (blkdev_get_by_path(path, bmode, holder));
 #endif
 }
 
 static void
 vdev_blkdev_put(zfs_bdev_handle_t *bdh, spa_mode_t smode, void *holder)
 {
 #if defined(HAVE_BDEV_RELEASE)
 	return (bdev_release(bdh));
 #elif defined(HAVE_BLKDEV_PUT_HOLDER)
 	return (blkdev_put(BDH_BDEV(bdh), holder));
 #elif defined(HAVE_BLKDEV_PUT)
 	return (blkdev_put(BDH_BDEV(bdh), vdev_bdev_mode(smode)));
 #else
 	fput(bdh);
 #endif
 }
 
 static int
 vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *max_psize,
     uint64_t *logical_ashift, uint64_t *physical_ashift)
 {
 	zfs_bdev_handle_t *bdh;
 	spa_mode_t smode = spa_mode(v->vdev_spa);
 	hrtime_t timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms);
 	vdev_disk_t *vd;
 
 	/* Must have a pathname and it must be absolute. */
 	if (v->vdev_path == NULL || v->vdev_path[0] != '/') {
 		v->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
 		vdev_dbgmsg(v, "invalid vdev_path");
 		return (SET_ERROR(EINVAL));
 	}
 
 	/*
 	 * Reopen the device if it is currently open.  When expanding a
 	 * partition force re-scanning the partition table if userland
 	 * did not take care of this already. We need to do this while closed
 	 * in order to get an accurate updated block device size.  Then
 	 * since udev may need to recreate the device links increase the
 	 * open retry timeout before reporting the device as unavailable.
 	 */
 	vd = v->vdev_tsd;
 	if (vd) {
 		char disk_name[BDEVNAME_SIZE + 6] = "/dev/";
 		boolean_t reread_part = B_FALSE;
 
 		rw_enter(&vd->vd_lock, RW_WRITER);
 		bdh = vd->vd_bdh;
 		vd->vd_bdh = NULL;
 
 		if (bdh) {
 			struct block_device *bdev = BDH_BDEV(bdh);
 			if (v->vdev_expanding && bdev != bdev_whole(bdev)) {
 				vdev_bdevname(bdev_whole(bdev), disk_name + 5);
 				/*
 				 * If userland has BLKPG_RESIZE_PARTITION,
 				 * then it should have updated the partition
 				 * table already. We can detect this by
 				 * comparing our current physical size
 				 * with that of the device. If they are
 				 * the same, then we must not have
 				 * BLKPG_RESIZE_PARTITION or it failed to
 				 * update the partition table online. We
 				 * fallback to rescanning the partition
 				 * table from the kernel below. However,
 				 * if the capacity already reflects the
 				 * updated partition, then we skip
 				 * rescanning the partition table here.
 				 */
 				if (v->vdev_psize == bdev_capacity(bdev))
 					reread_part = B_TRUE;
 			}
 
 			vdev_blkdev_put(bdh, smode, zfs_vdev_holder);
 		}
 
 		if (reread_part) {
 			bdh = vdev_blkdev_get_by_path(disk_name, smode,
 			    zfs_vdev_holder);
 			if (!BDH_IS_ERR(bdh)) {
 				int error =
 				    vdev_bdev_reread_part(BDH_BDEV(bdh));
 				vdev_blkdev_put(bdh, smode, zfs_vdev_holder);
 				if (error == 0) {
 					timeout = MSEC2NSEC(
 					    zfs_vdev_open_timeout_ms * 2);
 				}
 			}
 		}
 	} else {
 		vd = kmem_zalloc(sizeof (vdev_disk_t), KM_SLEEP);
 
 		rw_init(&vd->vd_lock, NULL, RW_DEFAULT, NULL);
 		rw_enter(&vd->vd_lock, RW_WRITER);
 	}
 
 	/*
 	 * Devices are always opened by the path provided at configuration
 	 * time.  This means that if the provided path is a udev by-id path
 	 * then drives may be re-cabled without an issue.  If the provided
 	 * path is a udev by-path path, then the physical location information
 	 * will be preserved.  This can be critical for more complicated
 	 * configurations where drives are located in specific physical
 	 * locations to maximize the systems tolerance to component failure.
 	 *
 	 * Alternatively, you can provide your own udev rule to flexibly map
 	 * the drives as you see fit.  It is not advised that you use the
 	 * /dev/[hd]d devices which may be reordered due to probing order.
 	 * Devices in the wrong locations will be detected by the higher
 	 * level vdev validation.
 	 *
 	 * The specified paths may be briefly removed and recreated in
 	 * response to udev events.  This should be exceptionally unlikely
 	 * because the zpool command makes every effort to verify these paths
 	 * have already settled prior to reaching this point.  Therefore,
 	 * a ENOENT failure at this point is highly likely to be transient
 	 * and it is reasonable to sleep and retry before giving up.  In
 	 * practice delays have been observed to be on the order of 100ms.
 	 *
 	 * When ERESTARTSYS is returned it indicates the block device is
 	 * a zvol which could not be opened due to the deadlock detection
 	 * logic in zvol_open().  Extend the timeout and retry the open
 	 * subsequent attempts are expected to eventually succeed.
 	 */
 	hrtime_t start = gethrtime();
 	bdh = BDH_ERR_PTR(-ENXIO);
 	while (BDH_IS_ERR(bdh) && ((gethrtime() - start) < timeout)) {
 		bdh = vdev_blkdev_get_by_path(v->vdev_path, smode,
 		    zfs_vdev_holder);
 		if (unlikely(BDH_PTR_ERR(bdh) == -ENOENT)) {
 			/*
 			 * There is no point of waiting since device is removed
 			 * explicitly
 			 */
 			if (v->vdev_removed)
 				break;
 
 			schedule_timeout_interruptible(MSEC_TO_TICK(10));
 		} else if (unlikely(BDH_PTR_ERR(bdh) == -ERESTARTSYS)) {
 			timeout = MSEC2NSEC(zfs_vdev_open_timeout_ms * 10);
 			continue;
 		} else if (BDH_IS_ERR(bdh)) {
 			break;
 		}
 	}
 
 	if (BDH_IS_ERR(bdh)) {
 		int error = -BDH_PTR_ERR(bdh);
 		vdev_dbgmsg(v, "open error=%d timeout=%llu/%llu", error,
 		    (u_longlong_t)(gethrtime() - start),
 		    (u_longlong_t)timeout);
 		vd->vd_bdh = NULL;
 		v->vdev_tsd = vd;
 		rw_exit(&vd->vd_lock);
 		return (SET_ERROR(error));
 	} else {
 		vd->vd_bdh = bdh;
 		v->vdev_tsd = vd;
 		rw_exit(&vd->vd_lock);
 	}
 
 	struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
 
 	/*  Determine the physical block size */
 	int physical_block_size = bdev_physical_block_size(bdev);
 
 	/*  Determine the logical block size */
 	int logical_block_size = bdev_logical_block_size(bdev);
 
 	/* Clear the nowritecache bit, causes vdev_reopen() to try again. */
 	v->vdev_nowritecache = B_FALSE;
 
 	/* Set when device reports it supports TRIM. */
 	v->vdev_has_trim = bdev_discard_supported(bdev);
 
 	/* Set when device reports it supports secure TRIM. */
 	v->vdev_has_securetrim = bdev_secure_discard_supported(bdev);
 
 	/* Inform the ZIO pipeline that we are non-rotational */
 	v->vdev_nonrot = blk_queue_nonrot(bdev_get_queue(bdev));
 
 	/* Physical volume size in bytes for the partition */
 	*psize = bdev_capacity(bdev);
 
 	/* Physical volume size in bytes including possible expansion space */
 	*max_psize = bdev_max_capacity(bdev, v->vdev_wholedisk);
 
 	/* Based on the minimum sector size set the block size */
 	*physical_ashift = highbit64(MAX(physical_block_size,
 	    SPA_MINBLOCKSIZE)) - 1;
 
 	*logical_ashift = highbit64(MAX(logical_block_size,
 	    SPA_MINBLOCKSIZE)) - 1;
 
 	return (0);
 }
 
 static void
 vdev_disk_close(vdev_t *v)
 {
 	vdev_disk_t *vd = v->vdev_tsd;
 
 	if (v->vdev_reopening || vd == NULL)
 		return;
 
 	if (vd->vd_bdh != NULL)
 		vdev_blkdev_put(vd->vd_bdh, spa_mode(v->vdev_spa),
 		    zfs_vdev_holder);
 
 	rw_destroy(&vd->vd_lock);
 	kmem_free(vd, sizeof (vdev_disk_t));
 	v->vdev_tsd = NULL;
 }
 
 /*
  * preempt_schedule_notrace is GPL-only which breaks the ZFS build, so
  * replace it with preempt_schedule under the following condition:
  */
 #if defined(CONFIG_ARM64) && \
     defined(CONFIG_PREEMPTION) && \
     defined(CONFIG_BLK_CGROUP)
 #define	preempt_schedule_notrace(x) preempt_schedule(x)
 #endif
 
 /*
  * As for the Linux 5.18 kernel bio_alloc() expects a block_device struct
  * as an argument removing the need to set it with bio_set_dev().  This
  * removes the need for all of the following compatibility code.
  */
 #if !defined(HAVE_BIO_ALLOC_4ARG)
 
 #if defined(CONFIG_BLK_CGROUP) && defined(HAVE_BIO_SET_DEV_GPL_ONLY)
 /*
  * The Linux 5.5 kernel updated percpu_ref_tryget() which is inlined by
  * blkg_tryget() to use rcu_read_lock() instead of rcu_read_lock_sched().
  * As a side effect the function was converted to GPL-only.  Define our
  * own version when needed which uses rcu_read_lock_sched().
  *
  * The Linux 5.17 kernel split linux/blk-cgroup.h into a private and a public
  * part, moving blkg_tryget into the private one. Define our own version.
  */
 #if defined(HAVE_BLKG_TRYGET_GPL_ONLY) || !defined(HAVE_BLKG_TRYGET)
 static inline bool
 vdev_blkg_tryget(struct blkcg_gq *blkg)
 {
 	struct percpu_ref *ref = &blkg->refcnt;
 	unsigned long __percpu *count;
 	bool rc;
 
 	rcu_read_lock_sched();
 
 	if (__ref_is_percpu(ref, &count)) {
 		this_cpu_inc(*count);
 		rc = true;
 	} else {
 #ifdef ZFS_PERCPU_REF_COUNT_IN_DATA
 		rc = atomic_long_inc_not_zero(&ref->data->count);
 #else
 		rc = atomic_long_inc_not_zero(&ref->count);
 #endif
 	}
 
 	rcu_read_unlock_sched();
 
 	return (rc);
 }
 #else
 #define	vdev_blkg_tryget(bg)	blkg_tryget(bg)
 #endif
 #ifdef HAVE_BIO_SET_DEV_MACRO
 /*
  * The Linux 5.0 kernel updated the bio_set_dev() macro so it calls the
  * GPL-only bio_associate_blkg() symbol thus inadvertently converting
  * the entire macro.  Provide a minimal version which always assigns the
  * request queue's root_blkg to the bio.
  */
 static inline void
 vdev_bio_associate_blkg(struct bio *bio)
 {
 #if defined(HAVE_BIO_BDEV_DISK)
 	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
 #else
 	struct request_queue *q = bio->bi_disk->queue;
 #endif
 
 	ASSERT3P(q, !=, NULL);
 	ASSERT3P(bio->bi_blkg, ==, NULL);
 
 	if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
 		bio->bi_blkg = q->root_blkg;
 }
 
 #define	bio_associate_blkg vdev_bio_associate_blkg
 #else
 static inline void
 vdev_bio_set_dev(struct bio *bio, struct block_device *bdev)
 {
 #if defined(HAVE_BIO_BDEV_DISK)
 	struct request_queue *q = bdev->bd_disk->queue;
 #else
 	struct request_queue *q = bio->bi_disk->queue;
 #endif
 	bio_clear_flag(bio, BIO_REMAPPED);
 	if (bio->bi_bdev != bdev)
 		bio_clear_flag(bio, BIO_THROTTLED);
 	bio->bi_bdev = bdev;
 
 	ASSERT3P(q, !=, NULL);
 	ASSERT3P(bio->bi_blkg, ==, NULL);
 
 	if (q->root_blkg && vdev_blkg_tryget(q->root_blkg))
 		bio->bi_blkg = q->root_blkg;
 }
 #define	bio_set_dev		vdev_bio_set_dev
 #endif
 #endif
 #endif /* !HAVE_BIO_ALLOC_4ARG */
 
 static inline void
 vdev_submit_bio(struct bio *bio)
 {
 	struct bio_list *bio_list = current->bio_list;
 	current->bio_list = NULL;
 	(void) submit_bio(bio);
 	current->bio_list = bio_list;
 }
 
 static inline struct bio *
 vdev_bio_alloc(struct block_device *bdev, gfp_t gfp_mask,
     unsigned short nr_vecs)
 {
 	struct bio *bio;
 
 #ifdef HAVE_BIO_ALLOC_4ARG
 	bio = bio_alloc(bdev, nr_vecs, 0, gfp_mask);
 #else
 	bio = bio_alloc(gfp_mask, nr_vecs);
 	if (likely(bio != NULL))
 		bio_set_dev(bio, bdev);
 #endif
 
 	return (bio);
 }
 
 static inline uint_t
 vdev_bio_max_segs(struct block_device *bdev)
 {
 	/*
 	 * Smallest of the device max segs and the tuneable max segs. Minimum
 	 * 4, so there's room to finish split pages if they come up.
 	 */
 	const uint_t dev_max_segs = queue_max_segments(bdev_get_queue(bdev));
 	const uint_t tune_max_segs = (zfs_vdev_disk_max_segs > 0) ?
 	    MAX(4, zfs_vdev_disk_max_segs) : dev_max_segs;
 	const uint_t max_segs = MIN(tune_max_segs, dev_max_segs);
 
 #ifdef HAVE_BIO_MAX_SEGS
 	return (bio_max_segs(max_segs));
 #else
 	return (MIN(max_segs, BIO_MAX_PAGES));
 #endif
 }
 
 static inline uint_t
 vdev_bio_max_bytes(struct block_device *bdev)
 {
 	return (queue_max_sectors(bdev_get_queue(bdev)) << 9);
 }
 
 
 /*
  * Virtual block IO object (VBIO)
  *
  * Linux block IO (BIO) objects have a limit on how many data segments (pages)
  * they can hold. Depending on how they're allocated and structured, a large
  * ZIO can require more than one BIO to be submitted to the kernel, which then
  * all have to complete before we can return the completed ZIO back to ZFS.
  *
  * A VBIO is a wrapper around multiple BIOs, carrying everything needed to
  * translate a ZIO down into the kernel block layer and back again.
  *
  * Note that these are only used for data ZIOs (read/write). Meta-operations
  * (flush/trim) don't need multiple BIOs and so can just make the call
  * directly.
  */
 typedef struct {
 	zio_t		*vbio_zio;	/* parent zio */
 
 	struct block_device *vbio_bdev;	/* blockdev to submit bios to */
 
 	abd_t		*vbio_abd;	/* abd carrying borrowed linear buf */
 
 	uint_t		vbio_max_segs;	/* max segs per bio */
 
 	uint_t		vbio_max_bytes;	/* max bytes per bio */
 	uint_t		vbio_lbs_mask;	/* logical block size mask */
 
 	uint64_t	vbio_offset;	/* start offset of next bio */
 
 	struct bio	*vbio_bio;	/* pointer to the current bio */
 	int		vbio_flags;	/* bio flags */
 } vbio_t;
 
 static vbio_t *
 vbio_alloc(zio_t *zio, struct block_device *bdev, int flags)
 {
 	vbio_t *vbio = kmem_zalloc(sizeof (vbio_t), KM_SLEEP);
 
 	vbio->vbio_zio = zio;
 	vbio->vbio_bdev = bdev;
 	vbio->vbio_abd = NULL;
 	vbio->vbio_max_segs = vdev_bio_max_segs(bdev);
 	vbio->vbio_max_bytes = vdev_bio_max_bytes(bdev);
 	vbio->vbio_lbs_mask = ~(bdev_logical_block_size(bdev)-1);
 	vbio->vbio_offset = zio->io_offset;
 	vbio->vbio_bio = NULL;
 	vbio->vbio_flags = flags;
 
 	return (vbio);
 }
 
 static void vbio_completion(struct bio *bio);
 
 static int
 vbio_add_page(vbio_t *vbio, struct page *page, uint_t size, uint_t offset)
 {
 	struct bio *bio = vbio->vbio_bio;
 	uint_t ssize;
 
 	while (size > 0) {
 		if (bio == NULL) {
 			/* New BIO, allocate and set up */
 			bio = vdev_bio_alloc(vbio->vbio_bdev, GFP_NOIO,
 			    vbio->vbio_max_segs);
 			VERIFY(bio);
 
 			BIO_BI_SECTOR(bio) = vbio->vbio_offset >> 9;
 			bio_set_op_attrs(bio,
 			    vbio->vbio_zio->io_type == ZIO_TYPE_WRITE ?
 			    WRITE : READ, vbio->vbio_flags);
 
 			if (vbio->vbio_bio) {
 				bio_chain(vbio->vbio_bio, bio);
 				vdev_submit_bio(vbio->vbio_bio);
 			}
 			vbio->vbio_bio = bio;
 		}
 
 		/*
 		 * Only load as much of the current page data as will fit in
 		 * the space left in the BIO, respecting lbs alignment. Older
 		 * kernels will error if we try to overfill the BIO, while
 		 * newer ones will accept it and split the BIO. This ensures
 		 * everything works on older kernels, and avoids an additional
 		 * overhead on the new.
 		 */
 		ssize = MIN(size, (vbio->vbio_max_bytes - BIO_BI_SIZE(bio)) &
 		    vbio->vbio_lbs_mask);
 		if (ssize > 0 &&
 		    bio_add_page(bio, page, ssize, offset) == ssize) {
 			/* Accepted, adjust and load any remaining. */
 			size -= ssize;
 			offset += ssize;
 			continue;
 		}
 
 		/* No room, set up for a new BIO and loop */
 		vbio->vbio_offset += BIO_BI_SIZE(bio);
 
 		/* Signal new BIO allocation wanted */
 		bio = NULL;
 	}
 
 	return (0);
 }
 
 /* Iterator callback to submit ABD pages to the vbio. */
 static int
 vbio_fill_cb(struct page *page, size_t off, size_t len, void *priv)
 {
 	vbio_t *vbio = priv;
 	return (vbio_add_page(vbio, page, len, off));
 }
 
 /* Create some BIOs, fill them with data and submit them */
 static void
 vbio_submit(vbio_t *vbio, abd_t *abd, uint64_t size)
 {
 	/*
 	 * We plug so we can submit the BIOs as we go and only unplug them when
 	 * they are fully created and submitted. This is important; if we don't
 	 * plug, then the kernel may start executing earlier BIOs while we're
 	 * still creating and executing later ones, and if the device goes
 	 * away while that's happening, older kernels can get confused and
 	 * trample memory.
 	 */
 	struct blk_plug plug;
 	blk_start_plug(&plug);
 
 	(void) abd_iterate_page_func(abd, 0, size, vbio_fill_cb, vbio);
 	ASSERT(vbio->vbio_bio);
 
 	vbio->vbio_bio->bi_end_io = vbio_completion;
 	vbio->vbio_bio->bi_private = vbio;
 
 	/*
 	 * Once submitted, vbio_bio now owns vbio (through bi_private) and we
 	 * can't touch it again. The bio may complete and vbio_completion() be
 	 * called and free the vbio before this task is run again, so we must
 	 * consider it invalid from this point.
 	 */
 	vdev_submit_bio(vbio->vbio_bio);
 
 	blk_finish_plug(&plug);
 }
 
 /* IO completion callback */
 static void
 vbio_completion(struct bio *bio)
 {
 	vbio_t *vbio = bio->bi_private;
 	zio_t *zio = vbio->vbio_zio;
 
 	ASSERT(zio);
 
 	/* Capture and log any errors */
 	zio->io_error = bi_status_to_errno(bio->bi_status);
 	ASSERT3U(zio->io_error, >=, 0);
 
 	if (zio->io_error)
 		vdev_disk_error(zio);
 
 	/* Return the BIO to the kernel */
 	bio_put(bio);
 
 	/*
 	 * We're likely in an interrupt context so we can't do ABD/memory work
 	 * here; instead we stash vbio on the zio and take care of it in the
 	 * done callback.
 	 */
 	ASSERT3P(zio->io_bio, ==, NULL);
 	zio->io_bio = vbio;
 
 	zio_delay_interrupt(zio);
 }
 
 /*
  * Iterator callback to count ABD pages and check their size & alignment.
  *
  * On Linux, each BIO segment can take a page pointer, and an offset+length of
  * the data within that page. A page can be arbitrarily large ("compound"
  * pages) but we still have to ensure the data portion is correctly sized and
  * aligned to the logical block size, to ensure that if the kernel wants to
  * split the BIO, the two halves will still be properly aligned.
  */
 typedef struct {
 	size_t	blocksize;
 	int	seen_first;
 	int	seen_last;
 } vdev_disk_check_alignment_t;
 
 static int
 vdev_disk_check_alignment_cb(struct page *page, size_t off, size_t len,
     void *priv)
 {
 	(void) page;
 	vdev_disk_check_alignment_t *s = priv;
 
 	/*
 	 * The cardinal rule: a single on-disk block must never cross an
 	 * physical (order-0) page boundary, as the kernel expects to be able
 	 * to split at both LBS and page boundaries.
 	 *
 	 * This implies various alignment rules for the blocks in this
 	 * (possibly compound) page, which we can check for.
 	 */
 
 	/*
 	 * If the previous page did not end on a page boundary, then we
 	 * can't proceed without creating a hole.
 	 */
 	if (s->seen_last)
 		return (1);
 
 	/* This page must contain only whole LBS-sized blocks. */
 	if (!IS_P2ALIGNED(len, s->blocksize))
 		return (1);
 
 	/*
 	 * If this is not the first page in the ABD, then the data must start
 	 * on a page-aligned boundary (so the kernel can split on page
 	 * boundaries without having to deal with a hole). If it is, then
 	 * it can start on LBS-alignment.
 	 */
 	if (s->seen_first) {
 		if (!IS_P2ALIGNED(off, PAGESIZE))
 			return (1);
 	} else {
 		if (!IS_P2ALIGNED(off, s->blocksize))
 			return (1);
 		s->seen_first = 1;
 	}
 
 	/*
 	 * If this data does not end on a page-aligned boundary, then this
 	 * must be the last page in the ABD, for the same reason.
 	 */
 	s->seen_last = !IS_P2ALIGNED(off+len, PAGESIZE);
 
 	return (0);
 }
 
 /*
  * Check if we can submit the pages in this ABD to the kernel as-is. Returns
  * the number of pages, or 0 if it can't be submitted like this.
  */
 static boolean_t
 vdev_disk_check_alignment(abd_t *abd, uint64_t size, struct block_device *bdev)
 {
 	vdev_disk_check_alignment_t s = {
 	    .blocksize = bdev_logical_block_size(bdev),
 	};
 
 	if (abd_iterate_page_func(abd, 0, size,
 	    vdev_disk_check_alignment_cb, &s))
 		return (B_FALSE);
 
 	return (B_TRUE);
 }
 
 static int
 vdev_disk_io_rw(zio_t *zio)
 {
 	vdev_t *v = zio->io_vd;
 	vdev_disk_t *vd = v->vdev_tsd;
 	struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
 	int flags = 0;
 
 	/*
 	 * Accessing outside the block device is never allowed.
 	 */
 	if (zio->io_offset + zio->io_size > bdev_capacity(bdev)) {
 		vdev_dbgmsg(zio->io_vd,
 		    "Illegal access %llu size %llu, device size %llu",
 		    (u_longlong_t)zio->io_offset,
 		    (u_longlong_t)zio->io_size,
 		    (u_longlong_t)bdev_capacity(bdev));
 		return (SET_ERROR(EIO));
 	}
 
 	if (!(zio->io_flags & (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)) &&
 	    v->vdev_failfast == B_TRUE) {
 		bio_set_flags_failfast(bdev, &flags, zfs_vdev_failfast_mask & 1,
 		    zfs_vdev_failfast_mask & 2, zfs_vdev_failfast_mask & 4);
 	}
 
 	/*
 	 * Check alignment of the incoming ABD. If any part of it would require
 	 * submitting a page that is not aligned to both the logical block size
 	 * and the page size, then we take a copy into a new memory region with
 	 * correct alignment.  This should be impossible on a 512b LBS. On
 	 * larger blocks, this can happen at least when a small number of
 	 * blocks (usually 1) are allocated from a shared slab, or when
 	 * abnormally-small data regions (eg gang headers) are mixed into the
 	 * same ABD as larger allocations (eg aggregations).
 	 */
 	abd_t *abd = zio->io_abd;
 	if (!vdev_disk_check_alignment(abd, zio->io_size, bdev)) {
 		/* Allocate a new memory region with guaranteed alignment */
 		abd = abd_alloc_for_io(zio->io_size,
 		    zio->io_abd->abd_flags & ABD_FLAG_META);
 
 		/* If we're writing copy our data into it */
 		if (zio->io_type == ZIO_TYPE_WRITE)
 			abd_copy(abd, zio->io_abd, zio->io_size);
 
 		/*
 		 * False here would mean the new allocation has an invalid
 		 * alignment too, which would mean that abd_alloc() is not
 		 * guaranteeing this, or our logic in
 		 * vdev_disk_check_alignment() is wrong. In either case,
 		 * something in seriously wrong and its not safe to continue.
 		 */
 		VERIFY(vdev_disk_check_alignment(abd, zio->io_size, bdev));
 	}
 
 	/* Allocate vbio, with a pointer to the borrowed ABD if necessary */
 	vbio_t *vbio = vbio_alloc(zio, bdev, flags);
 	if (abd != zio->io_abd)
 		vbio->vbio_abd = abd;
 
 	/* Fill it with data pages and submit it to the kernel */
 	vbio_submit(vbio, abd, zio->io_size);
 	return (0);
 }
 
 /* ========== */
 
 /*
  * This is the classic, battle-tested BIO submission code. Until we're totally
  * sure that the new code is safe and correct in all cases, this will remain
- * available.
- *
- * It is enabled by setting zfs_vdev_disk_classic=1 at module load time. It is
- * enabled (=1) by default since 2.2.4, and disabled by default (=0) on master.
+ * available and can be enabled by setting zfs_vdev_disk_classic=1 at module
+ * load time.
  *
  * These functions have been renamed to vdev_classic_* to make it clear what
  * they belong to, but their implementations are unchanged.
  */
 
 /*
  * Virtual device vector for disks.
  */
 typedef struct dio_request {
 	zio_t			*dr_zio;	/* Parent ZIO */
 	atomic_t		dr_ref;		/* References */
 	int			dr_error;	/* Bio error */
 	int			dr_bio_count;	/* Count of bio's */
 	struct bio		*dr_bio[];	/* Attached bio's */
 } dio_request_t;
 
 static dio_request_t *
 vdev_classic_dio_alloc(int bio_count)
 {
 	dio_request_t *dr = kmem_zalloc(sizeof (dio_request_t) +
 	    sizeof (struct bio *) * bio_count, KM_SLEEP);
 	atomic_set(&dr->dr_ref, 0);
 	dr->dr_bio_count = bio_count;
 	dr->dr_error = 0;
 
 	for (int i = 0; i < dr->dr_bio_count; i++)
 		dr->dr_bio[i] = NULL;
 
 	return (dr);
 }
 
 static void
 vdev_classic_dio_free(dio_request_t *dr)
 {
 	int i;
 
 	for (i = 0; i < dr->dr_bio_count; i++)
 		if (dr->dr_bio[i])
 			bio_put(dr->dr_bio[i]);
 
 	kmem_free(dr, sizeof (dio_request_t) +
 	    sizeof (struct bio *) * dr->dr_bio_count);
 }
 
 static void
 vdev_classic_dio_get(dio_request_t *dr)
 {
 	atomic_inc(&dr->dr_ref);
 }
 
 static void
 vdev_classic_dio_put(dio_request_t *dr)
 {
 	int rc = atomic_dec_return(&dr->dr_ref);
 
 	/*
 	 * Free the dio_request when the last reference is dropped and
 	 * ensure zio_interpret is called only once with the correct zio
 	 */
 	if (rc == 0) {
 		zio_t *zio = dr->dr_zio;
 		int error = dr->dr_error;
 
 		vdev_classic_dio_free(dr);
 
 		if (zio) {
 			zio->io_error = error;
 			ASSERT3S(zio->io_error, >=, 0);
 			if (zio->io_error)
 				vdev_disk_error(zio);
 
 			zio_delay_interrupt(zio);
 		}
 	}
 }
 
 static void
 vdev_classic_physio_completion(struct bio *bio)
 {
 	dio_request_t *dr = bio->bi_private;
 
 	if (dr->dr_error == 0) {
 		dr->dr_error = bi_status_to_errno(bio->bi_status);
 	}
 
 	/* Drop reference acquired by vdev_classic_physio */
 	vdev_classic_dio_put(dr);
 }
 
 static inline unsigned int
 vdev_classic_bio_max_segs(zio_t *zio, int bio_size, uint64_t abd_offset)
 {
 	unsigned long nr_segs = abd_nr_pages_off(zio->io_abd,
 	    bio_size, abd_offset);
 
 #ifdef HAVE_BIO_MAX_SEGS
 	return (bio_max_segs(nr_segs));
 #else
 	return (MIN(nr_segs, BIO_MAX_PAGES));
 #endif
 }
 
 static int
 vdev_classic_physio(zio_t *zio)
 {
 	vdev_t *v = zio->io_vd;
 	vdev_disk_t *vd = v->vdev_tsd;
 	struct block_device *bdev = BDH_BDEV(vd->vd_bdh);
 	size_t io_size = zio->io_size;
 	uint64_t io_offset = zio->io_offset;
 	int rw = zio->io_type == ZIO_TYPE_READ ? READ : WRITE;
 	int flags = 0;
 
 	dio_request_t *dr;
 	uint64_t abd_offset;
 	uint64_t bio_offset;
 	int bio_size;
 	int bio_count = 16;
 	int error = 0;
 	struct blk_plug plug;
 	unsigned short nr_vecs;
 
 	/*
 	 * Accessing outside the block device is never allowed.
 	 */
 	if (io_offset + io_size > bdev_capacity(bdev)) {
 		vdev_dbgmsg(zio->io_vd,
 		    "Illegal access %llu size %llu, device size %llu",
 		    (u_longlong_t)io_offset,
 		    (u_longlong_t)io_size,
 		    (u_longlong_t)bdev_capacity(bdev));
 		return (SET_ERROR(EIO));
 	}
 
 retry:
 	dr = vdev_classic_dio_alloc(bio_count);
 
 	if (!(zio->io_flags & (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)) &&
 	    zio->io_vd->vdev_failfast == B_TRUE) {
 		bio_set_flags_failfast(bdev, &flags, zfs_vdev_failfast_mask & 1,
 		    zfs_vdev_failfast_mask & 2, zfs_vdev_failfast_mask & 4);
 	}
 
 	dr->dr_zio = zio;
 
 	/*
 	 * Since bio's can have up to BIO_MAX_PAGES=256 iovec's, each of which
 	 * is at least 512 bytes and at most PAGESIZE (typically 4K), one bio
 	 * can cover at least 128KB and at most 1MB.  When the required number
 	 * of iovec's exceeds this, we are forced to break the IO in multiple
 	 * bio's and wait for them all to complete.  This is likely if the
 	 * recordsize property is increased beyond 1MB.  The default
 	 * bio_count=16 should typically accommodate the maximum-size zio of
 	 * 16MB.
 	 */
 
 	abd_offset = 0;
 	bio_offset = io_offset;
 	bio_size = io_size;
 	for (int i = 0; i <= dr->dr_bio_count; i++) {
 
 		/* Finished constructing bio's for given buffer */
 		if (bio_size <= 0)
 			break;
 
 		/*
 		 * If additional bio's are required, we have to retry, but
 		 * this should be rare - see the comment above.
 		 */
 		if (dr->dr_bio_count == i) {
 			vdev_classic_dio_free(dr);
 			bio_count *= 2;
 			goto retry;
 		}
 
 		nr_vecs = vdev_classic_bio_max_segs(zio, bio_size, abd_offset);
 		dr->dr_bio[i] = vdev_bio_alloc(bdev, GFP_NOIO, nr_vecs);
 		if (unlikely(dr->dr_bio[i] == NULL)) {
 			vdev_classic_dio_free(dr);
 			return (SET_ERROR(ENOMEM));
 		}
 
 		/* Matching put called by vdev_classic_physio_completion */
 		vdev_classic_dio_get(dr);
 
 		BIO_BI_SECTOR(dr->dr_bio[i]) = bio_offset >> 9;
 		dr->dr_bio[i]->bi_end_io = vdev_classic_physio_completion;
 		dr->dr_bio[i]->bi_private = dr;
 		bio_set_op_attrs(dr->dr_bio[i], rw, flags);
 
 		/* Remaining size is returned to become the new size */
 		bio_size = abd_bio_map_off(dr->dr_bio[i], zio->io_abd,
 		    bio_size, abd_offset);
 
 		/* Advance in buffer and construct another bio if needed */
 		abd_offset += BIO_BI_SIZE(dr->dr_bio[i]);
 		bio_offset += BIO_BI_SIZE(dr->dr_bio[i]);
 	}
 
 	/* Extra reference to protect dio_request during vdev_submit_bio */
 	vdev_classic_dio_get(dr);
 
 	if (dr->dr_bio_count > 1)
 		blk_start_plug(&plug);
 
 	/* Submit all bio's associated with this dio */
 	for (int i = 0; i < dr->dr_bio_count; i++) {
 		if (dr->dr_bio[i])
 			vdev_submit_bio(dr->dr_bio[i]);
 	}
 
 	if (dr->dr_bio_count > 1)
 		blk_finish_plug(&plug);
 
 	vdev_classic_dio_put(dr);
 
 	return (error);
 }
 
 /* ========== */
 
 static void
 vdev_disk_io_flush_completion(struct bio *bio)
 {
 	zio_t *zio = bio->bi_private;
 	zio->io_error = bi_status_to_errno(bio->bi_status);
 
 	if (zio->io_error && (zio->io_error == EOPNOTSUPP))
 		zio->io_vd->vdev_nowritecache = B_TRUE;
 
 	bio_put(bio);
 	ASSERT3S(zio->io_error, >=, 0);
 	if (zio->io_error)
 		vdev_disk_error(zio);
 	zio_interrupt(zio);
 }
 
 static int
 vdev_disk_io_flush(struct block_device *bdev, zio_t *zio)
 {
 	struct request_queue *q;
 	struct bio *bio;
 
 	q = bdev_get_queue(bdev);
 	if (!q)
 		return (SET_ERROR(ENXIO));
 
 	bio = vdev_bio_alloc(bdev, GFP_NOIO, 0);
 	if (unlikely(bio == NULL))
 		return (SET_ERROR(ENOMEM));
 
 	bio->bi_end_io = vdev_disk_io_flush_completion;
 	bio->bi_private = zio;
 	bio_set_flush(bio);
 	vdev_submit_bio(bio);
 	invalidate_bdev(bdev);
 
 	return (0);
 }
 
 static void
 vdev_disk_discard_end_io(struct bio *bio)
 {
 	zio_t *zio = bio->bi_private;
 	zio->io_error = bi_status_to_errno(bio->bi_status);
 
 	bio_put(bio);
 	if (zio->io_error)
 		vdev_disk_error(zio);
 	zio_interrupt(zio);
 }
 
 /*
  * Wrappers for the different secure erase and discard APIs. We use async
  * when available; in this case, *biop is set to the last bio in the chain.
  */
 static int
 vdev_bdev_issue_secure_erase(zfs_bdev_handle_t *bdh, sector_t sector,
     sector_t nsect, struct bio **biop)
 {
 	*biop = NULL;
 	int error;
 
 #if defined(HAVE_BLKDEV_ISSUE_SECURE_ERASE)
 	error = blkdev_issue_secure_erase(BDH_BDEV(bdh),
 	    sector, nsect, GFP_NOFS);
 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
 	error = __blkdev_issue_discard(BDH_BDEV(bdh),
 	    sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE, biop);
 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
 	error = blkdev_issue_discard(BDH_BDEV(bdh),
 	    sector, nsect, GFP_NOFS, BLKDEV_DISCARD_SECURE);
 #else
 #error "unsupported kernel"
 #endif
 
 	return (error);
 }
 
 static int
 vdev_bdev_issue_discard(zfs_bdev_handle_t *bdh, sector_t sector,
     sector_t nsect, struct bio **biop)
 {
 	*biop = NULL;
 	int error;
 
 #if defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS)
 	error = __blkdev_issue_discard(BDH_BDEV(bdh),
 	    sector, nsect, GFP_NOFS, 0, biop);
 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS)
 	error = __blkdev_issue_discard(BDH_BDEV(bdh),
 	    sector, nsect, GFP_NOFS, biop);
 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_FLAGS)
 	error = blkdev_issue_discard(BDH_BDEV(bdh),
 	    sector, nsect, GFP_NOFS, 0);
 #elif defined(HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS)
 	error = blkdev_issue_discard(BDH_BDEV(bdh),
 	    sector, nsect, GFP_NOFS);
 #else
 #error "unsupported kernel"
 #endif
 
 	return (error);
 }
 
 /*
  * Entry point for TRIM ops. This calls the right wrapper for secure erase or
  * discard, and then does the appropriate finishing work for error vs success
  * and async vs sync.
  */
 static int
 vdev_disk_io_trim(zio_t *zio)
 {
 	int error;
 	struct bio *bio;
 
 	zfs_bdev_handle_t *bdh = ((vdev_disk_t *)zio->io_vd->vdev_tsd)->vd_bdh;
 	sector_t sector = zio->io_offset >> 9;
 	sector_t nsects = zio->io_size >> 9;
 
 	if (zio->io_trim_flags & ZIO_TRIM_SECURE)
 		error = vdev_bdev_issue_secure_erase(bdh, sector, nsects, &bio);
 	else
 		error = vdev_bdev_issue_discard(bdh, sector, nsects, &bio);
 
 	if (error != 0)
 		return (SET_ERROR(-error));
 
 	if (bio == NULL) {
 		/*
 		 * This was a synchronous op that completed successfully, so
 		 * return it to ZFS immediately.
 		 */
 		zio_interrupt(zio);
 	} else {
 		/*
 		 * This was an asynchronous op; set up completion callback and
 		 * issue it.
 		 */
 		bio->bi_private = zio;
 		bio->bi_end_io = vdev_disk_discard_end_io;
 		vdev_submit_bio(bio);
 	}
 
 	return (0);
 }
 
 int (*vdev_disk_io_rw_fn)(zio_t *zio) = NULL;
 
 static void
 vdev_disk_io_start(zio_t *zio)
 {
 	vdev_t *v = zio->io_vd;
 	vdev_disk_t *vd = v->vdev_tsd;
 	int error;
 
 	/*
 	 * If the vdev is closed, it's likely in the REMOVED or FAULTED state.
 	 * Nothing to be done here but return failure.
 	 */
 	if (vd == NULL) {
 		zio->io_error = ENXIO;
 		zio_interrupt(zio);
 		return;
 	}
 
 	rw_enter(&vd->vd_lock, RW_READER);
 
 	/*
 	 * If the vdev is closed, it's likely due to a failed reopen and is
 	 * in the UNAVAIL state.  Nothing to be done here but return failure.
 	 */
 	if (vd->vd_bdh == NULL) {
 		rw_exit(&vd->vd_lock);
 		zio->io_error = ENXIO;
 		zio_interrupt(zio);
 		return;
 	}
 
 	switch (zio->io_type) {
 	case ZIO_TYPE_IOCTL:
 
 		if (!vdev_readable(v)) {
 			rw_exit(&vd->vd_lock);
 			zio->io_error = SET_ERROR(ENXIO);
 			zio_interrupt(zio);
 			return;
 		}
 
 		switch (zio->io_cmd) {
 		case DKIOCFLUSHWRITECACHE:
 
 			if (zfs_nocacheflush)
 				break;
 
 			if (v->vdev_nowritecache) {
 				zio->io_error = SET_ERROR(ENOTSUP);
 				break;
 			}
 
 			error = vdev_disk_io_flush(BDH_BDEV(vd->vd_bdh), zio);
 			if (error == 0) {
 				rw_exit(&vd->vd_lock);
 				return;
 			}
 
 			zio->io_error = error;
 
 			break;
 
 		default:
 			zio->io_error = SET_ERROR(ENOTSUP);
 		}
 
 		rw_exit(&vd->vd_lock);
 		zio_execute(zio);
 		return;
 
 	case ZIO_TYPE_TRIM:
 		error = vdev_disk_io_trim(zio);
 		rw_exit(&vd->vd_lock);
 		if (error) {
 			zio->io_error = error;
 			zio_execute(zio);
 		}
 		return;
 
 	case ZIO_TYPE_READ:
 	case ZIO_TYPE_WRITE:
 		zio->io_target_timestamp = zio_handle_io_delay(zio);
 		error = vdev_disk_io_rw_fn(zio);
 		rw_exit(&vd->vd_lock);
 		if (error) {
 			zio->io_error = error;
 			zio_interrupt(zio);
 		}
 		return;
 
 	default:
 		/*
 		 * Getting here means our parent vdev has made a very strange
 		 * request of us, and shouldn't happen. Assert here to force a
 		 * crash in dev builds, but in production return the IO
 		 * unhandled. The pool will likely suspend anyway but that's
 		 * nicer than crashing the kernel.
 		 */
 		ASSERT3S(zio->io_type, ==, -1);
 
 		rw_exit(&vd->vd_lock);
 		zio->io_error = SET_ERROR(ENOTSUP);
 		zio_interrupt(zio);
 		return;
 	}
 
 	__builtin_unreachable();
 }
 
 static void
 vdev_disk_io_done(zio_t *zio)
 {
 	/* If this was a read or write, we need to clean up the vbio */
 	if (zio->io_bio != NULL) {
 		vbio_t *vbio = zio->io_bio;
 		zio->io_bio = NULL;
 
 		/*
 		 * If we copied the ABD before issuing it, clean up and return
 		 * the copy to the ADB, with changes if appropriate.
 		 */
 		if (vbio->vbio_abd != NULL) {
 			if (zio->io_type == ZIO_TYPE_READ)
 				abd_copy(zio->io_abd, vbio->vbio_abd,
 				    zio->io_size);
 
 			abd_free(vbio->vbio_abd);
 			vbio->vbio_abd = NULL;
 		}
 
 		/* Final cleanup */
 		kmem_free(vbio, sizeof (vbio_t));
 	}
 
 	/*
 	 * If the device returned EIO, we revalidate the media.  If it is
 	 * determined the media has changed this triggers the asynchronous
 	 * removal of the device from the configuration.
 	 */
 	if (zio->io_error == EIO) {
 		vdev_t *v = zio->io_vd;
 		vdev_disk_t *vd = v->vdev_tsd;
 
 		if (!zfs_check_disk_status(BDH_BDEV(vd->vd_bdh))) {
 			invalidate_bdev(BDH_BDEV(vd->vd_bdh));
 			v->vdev_remove_wanted = B_TRUE;
 			spa_async_request(zio->io_spa, SPA_ASYNC_REMOVE);
 		}
 	}
 }
 
 static void
 vdev_disk_hold(vdev_t *vd)
 {
 	ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
 
 	/* We must have a pathname, and it must be absolute. */
 	if (vd->vdev_path == NULL || vd->vdev_path[0] != '/')
 		return;
 
 	/*
 	 * Only prefetch path and devid info if the device has
 	 * never been opened.
 	 */
 	if (vd->vdev_tsd != NULL)
 		return;
 
 }
 
 static void
 vdev_disk_rele(vdev_t *vd)
 {
 	ASSERT(spa_config_held(vd->vdev_spa, SCL_STATE, RW_WRITER));
 
 	/* XXX: Implement me as a vnode rele for the device */
 }
 
 /*
  * BIO submission method. See comment above about vdev_classic.
  * Set zfs_vdev_disk_classic=0 for new, =1 for classic
  */
-static uint_t zfs_vdev_disk_classic = 1;	/* default classic */
+static uint_t zfs_vdev_disk_classic = 0;	/* default new */
 
 /* Set submission function from module parameter */
 static int
 vdev_disk_param_set_classic(const char *buf, zfs_kernel_param_t *kp)
 {
 	int err = param_set_uint(buf, kp);
 	if (err < 0)
 		return (SET_ERROR(err));
 
 	vdev_disk_io_rw_fn =
 	    zfs_vdev_disk_classic ? vdev_classic_physio : vdev_disk_io_rw;
 
 	printk(KERN_INFO "ZFS: forcing %s BIO submission\n",
 	    zfs_vdev_disk_classic ? "classic" : "new");
 
 	return (0);
 }
 
 /*
  * At first use vdev use, set the submission function from the default value if
  * it hasn't been set already.
  */
 static int
 vdev_disk_init(spa_t *spa, nvlist_t *nv, void **tsd)
 {
 	(void) spa;
 	(void) nv;
 	(void) tsd;
 
 	if (vdev_disk_io_rw_fn == NULL)
 		vdev_disk_io_rw_fn = zfs_vdev_disk_classic ?
 		    vdev_classic_physio : vdev_disk_io_rw;
 
 	return (0);
 }
 
 vdev_ops_t vdev_disk_ops = {
 	.vdev_op_init = vdev_disk_init,
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_disk_open,
 	.vdev_op_close = vdev_disk_close,
 	.vdev_op_asize = vdev_default_asize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_disk_io_start,
 	.vdev_op_io_done = vdev_disk_io_done,
 	.vdev_op_state_change = NULL,
 	.vdev_op_need_resilver = NULL,
 	.vdev_op_hold = vdev_disk_hold,
 	.vdev_op_rele = vdev_disk_rele,
 	.vdev_op_remap = NULL,
 	.vdev_op_xlate = vdev_default_xlate,
 	.vdev_op_rebuild_asize = NULL,
 	.vdev_op_metaslab_init = NULL,
 	.vdev_op_config_generate = NULL,
 	.vdev_op_nparity = NULL,
 	.vdev_op_ndisks = NULL,
 	.vdev_op_type = VDEV_TYPE_DISK,		/* name of this vdev type */
 	.vdev_op_leaf = B_TRUE,			/* leaf vdev */
 	.vdev_op_kobj_evt_post = vdev_disk_kobj_evt_post
 };
 
 /*
  * The zfs_vdev_scheduler module option has been deprecated. Setting this
  * value no longer has any effect.  It has not yet been entirely removed
  * to allow the module to be loaded if this option is specified in the
  * /etc/modprobe.d/zfs.conf file.  The following warning will be logged.
  */
 static int
 param_set_vdev_scheduler(const char *val, zfs_kernel_param_t *kp)
 {
 	int error = param_set_charp(val, kp);
 	if (error == 0) {
 		printk(KERN_INFO "The 'zfs_vdev_scheduler' module option "
 		    "is not supported.\n");
 	}
 
 	return (error);
 }
 
 static const char *zfs_vdev_scheduler = "unused";
 module_param_call(zfs_vdev_scheduler, param_set_vdev_scheduler,
     param_get_charp, &zfs_vdev_scheduler, 0644);
 MODULE_PARM_DESC(zfs_vdev_scheduler, "I/O scheduler");
 
 int
 param_set_min_auto_ashift(const char *buf, zfs_kernel_param_t *kp)
 {
 	uint_t val;
 	int error;
 
 	error = kstrtouint(buf, 0, &val);
 	if (error < 0)
 		return (SET_ERROR(error));
 
 	if (val < ASHIFT_MIN || val > zfs_vdev_max_auto_ashift)
 		return (SET_ERROR(-EINVAL));
 
 	error = param_set_uint(buf, kp);
 	if (error < 0)
 		return (SET_ERROR(error));
 
 	return (0);
 }
 
 int
 param_set_max_auto_ashift(const char *buf, zfs_kernel_param_t *kp)
 {
 	uint_t val;
 	int error;
 
 	error = kstrtouint(buf, 0, &val);
 	if (error < 0)
 		return (SET_ERROR(error));
 
 	if (val > ASHIFT_MAX || val < zfs_vdev_min_auto_ashift)
 		return (SET_ERROR(-EINVAL));
 
 	error = param_set_uint(buf, kp);
 	if (error < 0)
 		return (SET_ERROR(error));
 
 	return (0);
 }
 
 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, open_timeout_ms, UINT, ZMOD_RW,
 	"Timeout before determining that a device is missing");
 
 ZFS_MODULE_PARAM(zfs_vdev, zfs_vdev_, failfast_mask, UINT, ZMOD_RW,
 	"Defines failfast mask: 1 - device, 2 - transport, 4 - driver");
 
 ZFS_MODULE_PARAM(zfs_vdev_disk, zfs_vdev_disk_, max_segs, UINT, ZMOD_RW,
 	"Maximum number of data segments to add to an IO request (min 4)");
 
 ZFS_MODULE_PARAM_CALL(zfs_vdev_disk, zfs_vdev_disk_, classic,
     vdev_disk_param_set_classic, param_get_uint, ZMOD_RD,
 	"Use classic BIO submission method");
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/vdev_file.c b/sys/contrib/openzfs/module/os/linux/zfs/vdev_file.c
index 6d5841a2f0c9..8fec5970aad5 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/vdev_file.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/vdev_file.c
@@ -1,377 +1,379 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
 #include <sys/spa.h>
 #include <sys/spa_impl.h>
 #include <sys/vdev_file.h>
 #include <sys/vdev_impl.h>
 #include <sys/vdev_trim.h>
 #include <sys/zio.h>
 #include <sys/fs/zfs.h>
 #include <sys/fm/fs/zfs.h>
 #include <sys/abd.h>
-#include <sys/fcntl.h>
 #include <sys/vnode.h>
 #include <sys/zfs_file.h>
 #ifdef _KERNEL
 #include <linux/falloc.h>
+#include <sys/fcntl.h>
+#else
+#include <fcntl.h>
 #endif
 /*
  * Virtual device vector for files.
  */
 
 static taskq_t *vdev_file_taskq;
 
 /*
  * By default, the logical/physical ashift for file vdevs is set to
  * SPA_MINBLOCKSHIFT (9). This allows all file vdevs to use 512B (1 << 9)
  * blocksizes. Users may opt to change one or both of these for testing
  * or performance reasons. Care should be taken as these values will
  * impact the vdev_ashift setting which can only be set at vdev creation
  * time.
  */
 static uint_t vdev_file_logical_ashift = SPA_MINBLOCKSHIFT;
 static uint_t vdev_file_physical_ashift = SPA_MINBLOCKSHIFT;
 
 static void
 vdev_file_hold(vdev_t *vd)
 {
 	ASSERT(vd->vdev_path != NULL);
 }
 
 static void
 vdev_file_rele(vdev_t *vd)
 {
 	ASSERT(vd->vdev_path != NULL);
 }
 
 static mode_t
 vdev_file_open_mode(spa_mode_t spa_mode)
 {
 	mode_t mode = 0;
 
 	if ((spa_mode & SPA_MODE_READ) && (spa_mode & SPA_MODE_WRITE)) {
 		mode = O_RDWR;
 	} else if (spa_mode & SPA_MODE_READ) {
 		mode = O_RDONLY;
 	} else if (spa_mode & SPA_MODE_WRITE) {
 		mode = O_WRONLY;
 	}
 
 	return (mode | O_LARGEFILE);
 }
 
 static int
 vdev_file_open(vdev_t *vd, uint64_t *psize, uint64_t *max_psize,
     uint64_t *logical_ashift, uint64_t *physical_ashift)
 {
 	vdev_file_t *vf;
 	zfs_file_t *fp;
 	zfs_file_attr_t zfa;
 	int error;
 
 	/*
 	 * Rotational optimizations only make sense on block devices.
 	 */
 	vd->vdev_nonrot = B_TRUE;
 
 	/*
 	 * Allow TRIM on file based vdevs.  This may not always be supported,
 	 * since it depends on your kernel version and underlying filesystem
 	 * type but it is always safe to attempt.
 	 */
 	vd->vdev_has_trim = B_TRUE;
 
 	/*
 	 * Disable secure TRIM on file based vdevs.  There is no way to
 	 * request this behavior from the underlying filesystem.
 	 */
 	vd->vdev_has_securetrim = B_FALSE;
 
 	/*
 	 * We must have a pathname, and it must be absolute.
 	 */
 	if (vd->vdev_path == NULL || vd->vdev_path[0] != '/') {
 		vd->vdev_stat.vs_aux = VDEV_AUX_BAD_LABEL;
 		return (SET_ERROR(EINVAL));
 	}
 
 	/*
 	 * Reopen the device if it's not currently open.  Otherwise,
 	 * just update the physical size of the device.
 	 */
 	if (vd->vdev_tsd != NULL) {
 		ASSERT(vd->vdev_reopening);
 		vf = vd->vdev_tsd;
 		goto skip_open;
 	}
 
 	vf = vd->vdev_tsd = kmem_zalloc(sizeof (vdev_file_t), KM_SLEEP);
 
 	/*
 	 * We always open the files from the root of the global zone, even if
 	 * we're in a local zone.  If the user has gotten to this point, the
 	 * administrator has already decided that the pool should be available
 	 * to local zone users, so the underlying devices should be as well.
 	 */
 	ASSERT(vd->vdev_path != NULL && vd->vdev_path[0] == '/');
 
 	error = zfs_file_open(vd->vdev_path,
 	    vdev_file_open_mode(spa_mode(vd->vdev_spa)), 0, &fp);
 	if (error) {
 		vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED;
 		return (error);
 	}
 
 	vf->vf_file = fp;
 
 #ifdef _KERNEL
 	/*
 	 * Make sure it's a regular file.
 	 */
 	if (zfs_file_getattr(fp, &zfa)) {
 		return (SET_ERROR(ENODEV));
 	}
 	if (!S_ISREG(zfa.zfa_mode)) {
 		vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED;
 		return (SET_ERROR(ENODEV));
 	}
 #endif
 
 skip_open:
 
 	error =  zfs_file_getattr(vf->vf_file, &zfa);
 	if (error) {
 		vd->vdev_stat.vs_aux = VDEV_AUX_OPEN_FAILED;
 		return (error);
 	}
 
 	*max_psize = *psize = zfa.zfa_size;
 	*logical_ashift = vdev_file_logical_ashift;
 	*physical_ashift = vdev_file_physical_ashift;
 
 	return (0);
 }
 
 static void
 vdev_file_close(vdev_t *vd)
 {
 	vdev_file_t *vf = vd->vdev_tsd;
 
 	if (vd->vdev_reopening || vf == NULL)
 		return;
 
 	if (vf->vf_file != NULL) {
 		(void) zfs_file_close(vf->vf_file);
 	}
 
 	vd->vdev_delayed_close = B_FALSE;
 	kmem_free(vf, sizeof (vdev_file_t));
 	vd->vdev_tsd = NULL;
 }
 
 static void
 vdev_file_io_strategy(void *arg)
 {
 	zio_t *zio = (zio_t *)arg;
 	vdev_t *vd = zio->io_vd;
 	vdev_file_t *vf = vd->vdev_tsd;
 	ssize_t resid;
 	void *buf;
 	loff_t off;
 	ssize_t size;
 	int err;
 
 	off = zio->io_offset;
 	size = zio->io_size;
 	resid = 0;
 
 	if (zio->io_type == ZIO_TYPE_READ) {
 		buf = abd_borrow_buf(zio->io_abd, zio->io_size);
 		err = zfs_file_pread(vf->vf_file, buf, size, off, &resid);
 		abd_return_buf_copy(zio->io_abd, buf, size);
 	} else {
 		buf = abd_borrow_buf_copy(zio->io_abd, zio->io_size);
 		err = zfs_file_pwrite(vf->vf_file, buf, size, off, &resid);
 		abd_return_buf(zio->io_abd, buf, size);
 	}
 	zio->io_error = err;
 	if (resid != 0 && zio->io_error == 0)
 		zio->io_error = SET_ERROR(ENOSPC);
 
 	zio_delay_interrupt(zio);
 }
 
 static void
 vdev_file_io_fsync(void *arg)
 {
 	zio_t *zio = (zio_t *)arg;
 	vdev_file_t *vf = zio->io_vd->vdev_tsd;
 
 	zio->io_error = zfs_file_fsync(vf->vf_file, O_SYNC | O_DSYNC);
 
 	zio_interrupt(zio);
 }
 
 static void
 vdev_file_io_start(zio_t *zio)
 {
 	vdev_t *vd = zio->io_vd;
 	vdev_file_t *vf = vd->vdev_tsd;
 
 	if (zio->io_type == ZIO_TYPE_IOCTL) {
 		/* XXPOLICY */
 		if (!vdev_readable(vd)) {
 			zio->io_error = SET_ERROR(ENXIO);
 			zio_interrupt(zio);
 			return;
 		}
 
 		switch (zio->io_cmd) {
 		case DKIOCFLUSHWRITECACHE:
 
 			if (zfs_nocacheflush)
 				break;
 
 			/*
 			 * We cannot safely call vfs_fsync() when PF_FSTRANS
 			 * is set in the current context.  Filesystems like
 			 * XFS include sanity checks to verify it is not
 			 * already set, see xfs_vm_writepage().  Therefore
 			 * the sync must be dispatched to a different context.
 			 */
 			if (__spl_pf_fstrans_check()) {
 				VERIFY3U(taskq_dispatch(vdev_file_taskq,
 				    vdev_file_io_fsync, zio, TQ_SLEEP), !=,
 				    TASKQID_INVALID);
 				return;
 			}
 
 			zio->io_error = zfs_file_fsync(vf->vf_file,
 			    O_SYNC | O_DSYNC);
 			break;
 		default:
 			zio->io_error = SET_ERROR(ENOTSUP);
 		}
 
 		zio_execute(zio);
 		return;
 	} else if (zio->io_type == ZIO_TYPE_TRIM) {
 		ASSERT3U(zio->io_size, !=, 0);
 		zio->io_error = zfs_file_deallocate(vf->vf_file,
 		    zio->io_offset, zio->io_size);
 		zio_execute(zio);
 		return;
 	}
 
 	zio->io_target_timestamp = zio_handle_io_delay(zio);
 
 	VERIFY3U(taskq_dispatch(vdev_file_taskq, vdev_file_io_strategy, zio,
 	    TQ_SLEEP), !=, TASKQID_INVALID);
 }
 
 static void
 vdev_file_io_done(zio_t *zio)
 {
 	(void) zio;
 }
 
 vdev_ops_t vdev_file_ops = {
 	.vdev_op_init = NULL,
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_file_open,
 	.vdev_op_close = vdev_file_close,
 	.vdev_op_asize = vdev_default_asize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_file_io_start,
 	.vdev_op_io_done = vdev_file_io_done,
 	.vdev_op_state_change = NULL,
 	.vdev_op_need_resilver = NULL,
 	.vdev_op_hold = vdev_file_hold,
 	.vdev_op_rele = vdev_file_rele,
 	.vdev_op_remap = NULL,
 	.vdev_op_xlate = vdev_default_xlate,
 	.vdev_op_rebuild_asize = NULL,
 	.vdev_op_metaslab_init = NULL,
 	.vdev_op_config_generate = NULL,
 	.vdev_op_nparity = NULL,
 	.vdev_op_ndisks = NULL,
 	.vdev_op_type = VDEV_TYPE_FILE,		/* name of this vdev type */
 	.vdev_op_leaf = B_TRUE			/* leaf vdev */
 };
 
 void
 vdev_file_init(void)
 {
 	vdev_file_taskq = taskq_create("z_vdev_file", MAX(boot_ncpus, 16),
 	    minclsyspri, boot_ncpus, INT_MAX, TASKQ_DYNAMIC);
 
 	VERIFY(vdev_file_taskq);
 }
 
 void
 vdev_file_fini(void)
 {
 	taskq_destroy(vdev_file_taskq);
 }
 
 /*
  * From userland we access disks just like files.
  */
 #ifndef _KERNEL
 
 vdev_ops_t vdev_disk_ops = {
 	.vdev_op_init = NULL,
 	.vdev_op_fini = NULL,
 	.vdev_op_open = vdev_file_open,
 	.vdev_op_close = vdev_file_close,
 	.vdev_op_asize = vdev_default_asize,
 	.vdev_op_min_asize = vdev_default_min_asize,
 	.vdev_op_min_alloc = NULL,
 	.vdev_op_io_start = vdev_file_io_start,
 	.vdev_op_io_done = vdev_file_io_done,
 	.vdev_op_state_change = NULL,
 	.vdev_op_need_resilver = NULL,
 	.vdev_op_hold = vdev_file_hold,
 	.vdev_op_rele = vdev_file_rele,
 	.vdev_op_remap = NULL,
 	.vdev_op_xlate = vdev_default_xlate,
 	.vdev_op_rebuild_asize = NULL,
 	.vdev_op_metaslab_init = NULL,
 	.vdev_op_config_generate = NULL,
 	.vdev_op_nparity = NULL,
 	.vdev_op_ndisks = NULL,
 	.vdev_op_type = VDEV_TYPE_DISK,		/* name of this vdev type */
 	.vdev_op_leaf = B_TRUE			/* leaf vdev */
 };
 
 #endif
 
 ZFS_MODULE_PARAM(zfs_vdev_file, vdev_file_, logical_ashift, UINT, ZMOD_RW,
 	"Logical ashift for file-based devices");
 ZFS_MODULE_PARAM(zfs_vdev_file, vdev_file_, physical_ashift, UINT, ZMOD_RW,
 	"Physical ashift for file-based devices");
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zfs_uio.c b/sys/contrib/openzfs/module/os/linux/zfs/zfs_uio.c
index feba18fdaf4d..5f8d3efdc902 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zfs_uio.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zfs_uio.c
@@ -1,439 +1,436 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
 /*	  All Rights Reserved	*/
 
 /*
  * University Copyright- Copyright (c) 1982, 1986, 1988
  * The Regents of the University of California
  * All Rights Reserved
  *
  * University Acknowledgment- Portions of this document are derived from
  * software developed by the University of California, Berkeley, and its
  * contributors.
  */
 /*
  * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
  */
 
 #ifdef _KERNEL
 
 #include <sys/types.h>
 #include <sys/uio_impl.h>
 #include <sys/sysmacros.h>
 #include <sys/string.h>
 #include <linux/kmap_compat.h>
 #include <linux/uaccess.h>
 
 /*
  * Move "n" bytes at byte address "p"; "rw" indicates the direction
  * of the move, and the I/O parameters are provided in "uio", which is
  * update to reflect the data which was moved.  Returns 0 on success or
  * a non-zero errno on failure.
  */
 static int
 zfs_uiomove_iov(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
 {
 	const struct iovec *iov = uio->uio_iov;
 	size_t skip = uio->uio_skip;
 	ulong_t cnt;
 
 	while (n && uio->uio_resid) {
 		cnt = MIN(iov->iov_len - skip, n);
 		switch (uio->uio_segflg) {
 		case UIO_USERSPACE:
 			/*
 			 * p = kernel data pointer
 			 * iov->iov_base = user data pointer
 			 */
 			if (rw == UIO_READ) {
 				if (copy_to_user(iov->iov_base+skip, p, cnt))
 					return (EFAULT);
 			} else {
 				unsigned long b_left = 0;
 				if (uio->uio_fault_disable) {
 					if (!zfs_access_ok(VERIFY_READ,
 					    (iov->iov_base + skip), cnt)) {
 						return (EFAULT);
 					}
 					pagefault_disable();
 					b_left =
 					    __copy_from_user_inatomic(p,
 					    (iov->iov_base + skip), cnt);
 					pagefault_enable();
 				} else {
 					b_left =
 					    copy_from_user(p,
 					    (iov->iov_base + skip), cnt);
 				}
 				if (b_left > 0) {
 					unsigned long c_bytes =
 					    cnt - b_left;
 					uio->uio_skip += c_bytes;
 					ASSERT3U(uio->uio_skip, <,
 					    iov->iov_len);
 					uio->uio_resid -= c_bytes;
 					uio->uio_loffset += c_bytes;
 					return (EFAULT);
 				}
 			}
 			break;
 		case UIO_SYSSPACE:
 			if (rw == UIO_READ)
 				memcpy(iov->iov_base + skip, p, cnt);
 			else
 				memcpy(p, iov->iov_base + skip, cnt);
 			break;
 		default:
 			ASSERT(0);
 		}
 		skip += cnt;
 		if (skip == iov->iov_len) {
 			skip = 0;
 			uio->uio_iov = (++iov);
 			uio->uio_iovcnt--;
 		}
 		uio->uio_skip = skip;
 		uio->uio_resid -= cnt;
 		uio->uio_loffset += cnt;
 		p = (caddr_t)p + cnt;
 		n -= cnt;
 	}
 	return (0);
 }
 
 static int
 zfs_uiomove_bvec_impl(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
 {
 	const struct bio_vec *bv = uio->uio_bvec;
 	size_t skip = uio->uio_skip;
 	ulong_t cnt;
 
 	while (n && uio->uio_resid) {
 		void *paddr;
 		cnt = MIN(bv->bv_len - skip, n);
 
 		paddr = zfs_kmap_local(bv->bv_page);
 		if (rw == UIO_READ) {
 			/* Copy from buffer 'p' to the bvec data */
 			memcpy(paddr + bv->bv_offset + skip, p, cnt);
 		} else {
 			/* Copy from bvec data to buffer 'p' */
 			memcpy(p, paddr + bv->bv_offset + skip, cnt);
 		}
 		zfs_kunmap_local(paddr);
 
 		skip += cnt;
 		if (skip == bv->bv_len) {
 			skip = 0;
 			uio->uio_bvec = (++bv);
 			uio->uio_iovcnt--;
 		}
 		uio->uio_skip = skip;
 		uio->uio_resid -= cnt;
 		uio->uio_loffset += cnt;
 		p = (caddr_t)p + cnt;
 		n -= cnt;
 	}
 	return (0);
 }
 
 static void
 zfs_copy_bvec(void *p, size_t skip, size_t cnt, zfs_uio_rw_t rw,
     struct bio_vec *bv)
 {
 	void *paddr;
 
 	paddr = zfs_kmap_local(bv->bv_page);
 	if (rw == UIO_READ) {
 		/* Copy from buffer 'p' to the bvec data */
 		memcpy(paddr + bv->bv_offset + skip, p, cnt);
 	} else {
 		/* Copy from bvec data to buffer 'p' */
 		memcpy(p, paddr + bv->bv_offset + skip, cnt);
 	}
 	zfs_kunmap_local(paddr);
 }
 
 /*
  * Copy 'n' bytes of data between the buffer p[] and the data represented
  * by the request in the uio.
  */
 static int
 zfs_uiomove_bvec_rq(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
 {
 	struct request *rq = uio->rq;
 	struct bio_vec bv;
 	struct req_iterator iter;
 	size_t this_seg_start;	/* logical offset */
 	size_t this_seg_end;		/* logical offset */
 	size_t skip_in_seg;
 	size_t copy_from_seg;
 	size_t orig_loffset;
 	int copied = 0;
 
 	/*
 	 * Get the original logical offset of this entire request (because
 	 * uio->uio_loffset will be modified over time).
 	 */
 	orig_loffset = io_offset(NULL, rq);
 	this_seg_start = orig_loffset;
 
 	rq_for_each_segment(bv, rq, iter) {
 		/*
 		 * Lookup what the logical offset of the last byte of this
 		 * segment is.
 		 */
 		this_seg_end = this_seg_start + bv.bv_len - 1;
 
 		/*
 		 * We only need to operate on segments that have data we're
 		 * copying.
 		 */
 		if (uio->uio_loffset >= this_seg_start &&
 		    uio->uio_loffset <= this_seg_end) {
 			/*
 			 * Some, or all, of the data in this segment needs to be
 			 * copied.
 			 */
 
 			/*
 			 * We may be not be copying from the first byte in the
 			 * segment.  Figure out how many bytes to skip copying
 			 * from the beginning of this segment.
 			 */
 			skip_in_seg = uio->uio_loffset - this_seg_start;
 
 			/*
 			 * Calculate the total number of bytes from this
 			 * segment that we will be copying.
 			 */
 			copy_from_seg = MIN(bv.bv_len - skip_in_seg, n);
 
 			/* Copy the bytes */
 			zfs_copy_bvec(p, skip_in_seg, copy_from_seg, rw, &bv);
 			p = ((char *)p) + copy_from_seg;
 
 			n -= copy_from_seg;
 			uio->uio_resid -= copy_from_seg;
 			uio->uio_loffset += copy_from_seg;
 			copied = 1;	/* We copied some data */
 		}
 
 		this_seg_start = this_seg_end + 1;
 	}
 
 	if (!copied) {
 		/* Didn't copy anything */
 		uio->uio_resid = 0;
 	}
 	return (0);
 }
 
 static int
 zfs_uiomove_bvec(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
 {
 	if (uio->rq != NULL)
 		return (zfs_uiomove_bvec_rq(p, n, rw, uio));
 	return (zfs_uiomove_bvec_impl(p, n, rw, uio));
 }
 
 #if defined(HAVE_VFS_IOV_ITER)
 static int
 zfs_uiomove_iter(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio,
     boolean_t revert)
 {
 	size_t cnt = MIN(n, uio->uio_resid);
 
-	if (uio->uio_skip)
-		iov_iter_advance(uio->uio_iter, uio->uio_skip);
-
 	if (rw == UIO_READ)
 		cnt = copy_to_iter(p, cnt, uio->uio_iter);
 	else
 		cnt = copy_from_iter(p, cnt, uio->uio_iter);
 
 	/*
 	 * When operating on a full pipe no bytes are processed.
 	 * In which case return EFAULT which is converted to EAGAIN
 	 * by the kernel's generic_file_splice_read() function.
 	 */
 	if (cnt == 0)
 		return (EFAULT);
 
 	/*
 	 * Revert advancing the uio_iter.  This is set by zfs_uiocopy()
 	 * to avoid consuming the uio and its iov_iter structure.
 	 */
 	if (revert)
 		iov_iter_revert(uio->uio_iter, cnt);
 
 	uio->uio_resid -= cnt;
 	uio->uio_loffset += cnt;
 
 	return (0);
 }
 #endif
 
 int
 zfs_uiomove(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio)
 {
 	if (uio->uio_segflg == UIO_BVEC)
 		return (zfs_uiomove_bvec(p, n, rw, uio));
 #if defined(HAVE_VFS_IOV_ITER)
 	else if (uio->uio_segflg == UIO_ITER)
 		return (zfs_uiomove_iter(p, n, rw, uio, B_FALSE));
 #endif
 	else
 		return (zfs_uiomove_iov(p, n, rw, uio));
 }
 EXPORT_SYMBOL(zfs_uiomove);
 
 /*
  * Fault in the pages of the first n bytes specified by the uio structure.
  * 1 byte in each page is touched and the uio struct is unmodified. Any
  * error will terminate the process as this is only a best attempt to get
  * the pages resident.
  */
 int
 zfs_uio_prefaultpages(ssize_t n, zfs_uio_t *uio)
 {
 	if (uio->uio_segflg == UIO_SYSSPACE || uio->uio_segflg == UIO_BVEC) {
 		/* There's never a need to fault in kernel pages */
 		return (0);
 #if defined(HAVE_VFS_IOV_ITER)
 	} else if (uio->uio_segflg == UIO_ITER) {
 		/*
 		 * At least a Linux 4.9 kernel, iov_iter_fault_in_readable()
 		 * can be relied on to fault in user pages when referenced.
 		 */
 		if (iov_iter_fault_in_readable(uio->uio_iter, n))
 			return (EFAULT);
 #endif
 	} else {
 		/* Fault in all user pages */
 		ASSERT3S(uio->uio_segflg, ==, UIO_USERSPACE);
 		const struct iovec *iov = uio->uio_iov;
 		int iovcnt = uio->uio_iovcnt;
 		size_t skip = uio->uio_skip;
 		uint8_t tmp;
 		caddr_t p;
 
 		for (; n > 0 && iovcnt > 0; iov++, iovcnt--, skip = 0) {
 			ulong_t cnt = MIN(iov->iov_len - skip, n);
 			/* empty iov */
 			if (cnt == 0)
 				continue;
 			n -= cnt;
 			/* touch each page in this segment. */
 			p = iov->iov_base + skip;
 			while (cnt) {
 				if (copy_from_user(&tmp, p, 1))
 					return (EFAULT);
 				ulong_t incr = MIN(cnt, PAGESIZE);
 				p += incr;
 				cnt -= incr;
 			}
 			/* touch the last byte in case it straddles a page. */
 			p--;
 			if (copy_from_user(&tmp, p, 1))
 				return (EFAULT);
 		}
 	}
 
 	return (0);
 }
 EXPORT_SYMBOL(zfs_uio_prefaultpages);
 
 /*
  * The same as zfs_uiomove() but doesn't modify uio structure.
  * return in cbytes how many bytes were copied.
  */
 int
 zfs_uiocopy(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, size_t *cbytes)
 {
 	zfs_uio_t uio_copy;
 	int ret;
 
 	memcpy(&uio_copy, uio, sizeof (zfs_uio_t));
 
 	if (uio->uio_segflg == UIO_BVEC)
 		ret = zfs_uiomove_bvec(p, n, rw, &uio_copy);
 #if defined(HAVE_VFS_IOV_ITER)
 	else if (uio->uio_segflg == UIO_ITER)
 		ret = zfs_uiomove_iter(p, n, rw, &uio_copy, B_TRUE);
 #endif
 	else
 		ret = zfs_uiomove_iov(p, n, rw, &uio_copy);
 
 	*cbytes = uio->uio_resid - uio_copy.uio_resid;
 
 	return (ret);
 }
 EXPORT_SYMBOL(zfs_uiocopy);
 
 /*
  * Drop the next n chars out of *uio.
  */
 void
 zfs_uioskip(zfs_uio_t *uio, size_t n)
 {
 	if (n > uio->uio_resid)
 		return;
 	/*
 	 * When using a uio with a struct request, we simply
 	 * use uio_loffset as a pointer to the next logical byte to
 	 * copy in the request.  We don't have to do any fancy
 	 * accounting with uio_bvec/uio_iovcnt since we don't use
 	 * them.
 	 */
 	if (uio->uio_segflg == UIO_BVEC && uio->rq == NULL) {
 		uio->uio_skip += n;
 		while (uio->uio_iovcnt &&
 		    uio->uio_skip >= uio->uio_bvec->bv_len) {
 			uio->uio_skip -= uio->uio_bvec->bv_len;
 			uio->uio_bvec++;
 			uio->uio_iovcnt--;
 		}
 #if defined(HAVE_VFS_IOV_ITER)
 	} else if (uio->uio_segflg == UIO_ITER) {
 		iov_iter_advance(uio->uio_iter, n);
 #endif
 	} else {
 		uio->uio_skip += n;
 		while (uio->uio_iovcnt &&
 		    uio->uio_skip >= uio->uio_iov->iov_len) {
 			uio->uio_skip -= uio->uio_iov->iov_len;
 			uio->uio_iov++;
 			uio->uio_iovcnt--;
 		}
 	}
 	uio->uio_loffset += n;
 	uio->uio_resid -= n;
 }
 EXPORT_SYMBOL(zfs_uioskip);
 
 #endif /* _KERNEL */
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_ctldir.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_ctldir.c
index 56a30be5110c..a7fdb8f28009 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_ctldir.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_ctldir.c
@@ -1,605 +1,621 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (C) 2011 Lawrence Livermore National Security, LLC.
  * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
  * LLNL-CODE-403049.
  * Rewritten for Linux by:
  *   Rohan Puri <rohan.puri15@gmail.com>
  *   Brian Behlendorf <behlendorf1@llnl.gov>
  */
 
 #include <sys/zfs_znode.h>
 #include <sys/zfs_vfsops.h>
 #include <sys/zfs_vnops.h>
 #include <sys/zfs_ctldir.h>
 #include <sys/zpl.h>
 #include <sys/dmu.h>
 #include <sys/dsl_dataset.h>
 #include <sys/zap.h>
 
 /*
  * Common open routine.  Disallow any write access.
  */
 static int
 zpl_common_open(struct inode *ip, struct file *filp)
 {
 	if (blk_mode_is_open_write(filp->f_mode))
 		return (-EACCES);
 
 	return (generic_file_open(ip, filp));
 }
 
 /*
  * Get root directory contents.
  */
 static int
 zpl_root_iterate(struct file *filp, struct dir_context *ctx)
 {
 	zfsvfs_t *zfsvfs = ITOZSB(file_inode(filp));
 	int error = 0;
 
 	if ((error = zpl_enter(zfsvfs, FTAG)) != 0)
 		return (error);
 
 	if (!dir_emit_dots(filp, ctx))
 		goto out;
 
 	if (ctx->pos == 2) {
 		if (!dir_emit(ctx, ZFS_SNAPDIR_NAME,
 		    strlen(ZFS_SNAPDIR_NAME), ZFSCTL_INO_SNAPDIR, DT_DIR))
 			goto out;
 
 		ctx->pos++;
 	}
 
 	if (ctx->pos == 3) {
 		if (!dir_emit(ctx, ZFS_SHAREDIR_NAME,
 		    strlen(ZFS_SHAREDIR_NAME), ZFSCTL_INO_SHARES, DT_DIR))
 			goto out;
 
 		ctx->pos++;
 	}
 out:
 	zpl_exit(zfsvfs, FTAG);
 
 	return (error);
 }
 
 /*
  * Get root directory attributes.
  */
 static int
 #ifdef HAVE_IDMAP_IOPS_GETATTR
 zpl_root_getattr_impl(struct mnt_idmap *user_ns,
     const struct path *path, struct kstat *stat, u32 request_mask,
     unsigned int query_flags)
 #elif defined(HAVE_USERNS_IOPS_GETATTR)
 zpl_root_getattr_impl(struct user_namespace *user_ns,
     const struct path *path, struct kstat *stat, u32 request_mask,
     unsigned int query_flags)
 #else
 zpl_root_getattr_impl(const struct path *path, struct kstat *stat,
     u32 request_mask, unsigned int query_flags)
 #endif
 {
 	(void) request_mask, (void) query_flags;
 	struct inode *ip = path->dentry->d_inode;
 
 #if (defined(HAVE_USERNS_IOPS_GETATTR) || defined(HAVE_IDMAP_IOPS_GETATTR))
 #ifdef HAVE_GENERIC_FILLATTR_USERNS
 	generic_fillattr(user_ns, ip, stat);
 #elif defined(HAVE_GENERIC_FILLATTR_IDMAP)
 	generic_fillattr(user_ns, ip, stat);
 #elif defined(HAVE_GENERIC_FILLATTR_IDMAP_REQMASK)
 	generic_fillattr(user_ns, request_mask, ip, stat);
 #else
 	(void) user_ns;
 #endif
 #else
 	generic_fillattr(ip, stat);
 #endif
 	stat->atime = current_time(ip);
 
 	return (0);
 }
 ZPL_GETATTR_WRAPPER(zpl_root_getattr);
 
 static struct dentry *
 zpl_root_lookup(struct inode *dip, struct dentry *dentry, unsigned int flags)
 {
 	cred_t *cr = CRED();
 	struct inode *ip;
 	int error;
 
 	crhold(cr);
 	error = -zfsctl_root_lookup(dip, dname(dentry), &ip, 0, cr, NULL, NULL);
 	ASSERT3S(error, <=, 0);
 	crfree(cr);
 
 	if (error) {
 		if (error == -ENOENT)
 			return (d_splice_alias(NULL, dentry));
 		else
 			return (ERR_PTR(error));
 	}
 
 	return (d_splice_alias(ip, dentry));
 }
 
 /*
  * The '.zfs' control directory file and inode operations.
  */
 const struct file_operations zpl_fops_root = {
 	.open		= zpl_common_open,
 	.llseek		= generic_file_llseek,
 	.read		= generic_read_dir,
 	.iterate_shared	= zpl_root_iterate,
 };
 
 const struct inode_operations zpl_ops_root = {
 	.lookup		= zpl_root_lookup,
 	.getattr	= zpl_root_getattr,
 };
 
 static struct vfsmount *
 zpl_snapdir_automount(struct path *path)
 {
 	int error;
 
 	error = -zfsctl_snapshot_mount(path, 0);
 	if (error)
 		return (ERR_PTR(error));
 
 	/*
 	 * Rather than returning the new vfsmount for the snapshot we must
 	 * return NULL to indicate a mount collision.  This is done because
 	 * the user space mount calls do_add_mount() which adds the vfsmount
 	 * to the name space.  If we returned the new mount here it would be
 	 * added again to the vfsmount list resulting in list corruption.
 	 */
 	return (NULL);
 }
 
 /*
  * Negative dentries must always be revalidated so newly created snapshots
  * can be detected and automounted.  Normal dentries should be kept because
  * as of the 3.18 kernel revaliding the mountpoint dentry will result in
  * the snapshot being immediately unmounted.
  */
+#ifdef HAVE_D_REVALIDATE_4ARGS
+static int
+zpl_snapdir_revalidate(struct inode *dir, const struct qstr *name,
+    struct dentry *dentry, unsigned int flags)
+#else
 static int
 zpl_snapdir_revalidate(struct dentry *dentry, unsigned int flags)
+#endif
 {
 	return (!!dentry->d_inode);
 }
 
 static dentry_operations_t zpl_dops_snapdirs = {
 /*
  * Auto mounting of snapshots is only supported for 2.6.37 and
  * newer kernels.  Prior to this kernel the ops->follow_link()
  * callback was used as a hack to trigger the mount.  The
  * resulting vfsmount was then explicitly grafted in to the
  * name space.  While it might be possible to add compatibility
  * code to accomplish this it would require considerable care.
  */
 	.d_automount	= zpl_snapdir_automount,
 	.d_revalidate	= zpl_snapdir_revalidate,
 };
 
 static struct dentry *
 zpl_snapdir_lookup(struct inode *dip, struct dentry *dentry,
     unsigned int flags)
 {
 	fstrans_cookie_t cookie;
 	cred_t *cr = CRED();
 	struct inode *ip = NULL;
 	int error;
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	error = -zfsctl_snapdir_lookup(dip, dname(dentry), &ip,
 	    0, cr, NULL, NULL);
 	ASSERT3S(error, <=, 0);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 
 	if (error && error != -ENOENT)
 		return (ERR_PTR(error));
 
 	ASSERT(error == 0 || ip == NULL);
 	d_clear_d_op(dentry);
 	d_set_d_op(dentry, &zpl_dops_snapdirs);
 	dentry->d_flags |= DCACHE_NEED_AUTOMOUNT;
 
 	return (d_splice_alias(ip, dentry));
 }
 
 static int
 zpl_snapdir_iterate(struct file *filp, struct dir_context *ctx)
 {
 	zfsvfs_t *zfsvfs = ITOZSB(file_inode(filp));
 	fstrans_cookie_t cookie;
 	char snapname[MAXNAMELEN];
 	boolean_t case_conflict;
 	uint64_t id, pos;
 	int error = 0;
 
 	if ((error = zpl_enter(zfsvfs, FTAG)) != 0)
 		return (error);
 	cookie = spl_fstrans_mark();
 
 	if (!dir_emit_dots(filp, ctx))
 		goto out;
 
 	/* Start the position at 0 if it already emitted . and .. */
 	pos = (ctx->pos == 2 ? 0 : ctx->pos);
 	while (error == 0) {
 		dsl_pool_config_enter(dmu_objset_pool(zfsvfs->z_os), FTAG);
 		error = -dmu_snapshot_list_next(zfsvfs->z_os, MAXNAMELEN,
 		    snapname, &id, &pos, &case_conflict);
 		dsl_pool_config_exit(dmu_objset_pool(zfsvfs->z_os), FTAG);
 		if (error)
 			goto out;
 
 		if (!dir_emit(ctx, snapname, strlen(snapname),
 		    ZFSCTL_INO_SHARES - id, DT_DIR))
 			goto out;
 
 		ctx->pos = pos;
 	}
 out:
 	spl_fstrans_unmark(cookie);
 	zpl_exit(zfsvfs, FTAG);
 
 	if (error == -ENOENT)
 		return (0);
 
 	return (error);
 }
 
 static int
 #ifdef HAVE_IOPS_RENAME_USERNS
 zpl_snapdir_rename2(struct user_namespace *user_ns, struct inode *sdip,
     struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry,
     unsigned int flags)
 #elif defined(HAVE_IOPS_RENAME_IDMAP)
 zpl_snapdir_rename2(struct mnt_idmap *user_ns, struct inode *sdip,
     struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry,
     unsigned int flags)
 #else
 zpl_snapdir_rename2(struct inode *sdip, struct dentry *sdentry,
     struct inode *tdip, struct dentry *tdentry, unsigned int flags)
 #endif
 {
 	cred_t *cr = CRED();
 	int error;
 
 	/* We probably don't want to support renameat2(2) in ctldir */
 	if (flags)
 		return (-EINVAL);
 
 	crhold(cr);
 	error = -zfsctl_snapdir_rename(sdip, dname(sdentry),
 	    tdip, dname(tdentry), cr, 0);
 	ASSERT3S(error, <=, 0);
 	crfree(cr);
 
 	return (error);
 }
 
 #if (!defined(HAVE_RENAME_WANTS_FLAGS) && \
 	!defined(HAVE_IOPS_RENAME_USERNS) && \
 	!defined(HAVE_IOPS_RENAME_IDMAP))
 static int
 zpl_snapdir_rename(struct inode *sdip, struct dentry *sdentry,
     struct inode *tdip, struct dentry *tdentry)
 {
 	return (zpl_snapdir_rename2(sdip, sdentry, tdip, tdentry, 0));
 }
 #endif
 
 static int
 zpl_snapdir_rmdir(struct inode *dip, struct dentry *dentry)
 {
 	cred_t *cr = CRED();
 	int error;
 
 	crhold(cr);
 	error = -zfsctl_snapdir_remove(dip, dname(dentry), cr, 0);
 	ASSERT3S(error, <=, 0);
 	crfree(cr);
 
 	return (error);
 }
 
+#if defined(HAVE_IOPS_MKDIR_USERNS)
 static int
-#ifdef HAVE_IOPS_MKDIR_USERNS
 zpl_snapdir_mkdir(struct user_namespace *user_ns, struct inode *dip,
     struct dentry *dentry, umode_t mode)
 #elif defined(HAVE_IOPS_MKDIR_IDMAP)
+static int
+zpl_snapdir_mkdir(struct mnt_idmap *user_ns, struct inode *dip,
+    struct dentry *dentry, umode_t mode)
+#elif defined(HAVE_IOPS_MKDIR_DENTRY)
+static struct dentry *
 zpl_snapdir_mkdir(struct mnt_idmap *user_ns, struct inode *dip,
     struct dentry *dentry, umode_t mode)
 #else
+static int
 zpl_snapdir_mkdir(struct inode *dip, struct dentry *dentry, umode_t mode)
 #endif
 {
 	cred_t *cr = CRED();
 	vattr_t *vap;
 	struct inode *ip;
 	int error;
 
 	crhold(cr);
 	vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
 #if (defined(HAVE_IOPS_MKDIR_USERNS) || defined(HAVE_IOPS_MKDIR_IDMAP))
 	zpl_vap_init(vap, dip, mode | S_IFDIR, cr, user_ns);
 #else
 	zpl_vap_init(vap, dip, mode | S_IFDIR, cr, zfs_init_idmap);
 #endif
 
 	error = -zfsctl_snapdir_mkdir(dip, dname(dentry), vap, &ip, cr, 0);
 	if (error == 0) {
 		d_clear_d_op(dentry);
 		d_set_d_op(dentry, &zpl_dops_snapdirs);
 		d_instantiate(dentry, ip);
 	}
 
 	kmem_free(vap, sizeof (vattr_t));
 	ASSERT3S(error, <=, 0);
 	crfree(cr);
 
+#if defined(HAVE_IOPS_MKDIR_DENTRY)
+	return (ERR_PTR(error));
+#else
 	return (error);
+#endif
 }
 
 /*
  * Get snapshot directory attributes.
  */
 static int
 #ifdef HAVE_IDMAP_IOPS_GETATTR
 zpl_snapdir_getattr_impl(struct mnt_idmap *user_ns,
     const struct path *path, struct kstat *stat, u32 request_mask,
     unsigned int query_flags)
 #elif defined(HAVE_USERNS_IOPS_GETATTR)
 zpl_snapdir_getattr_impl(struct user_namespace *user_ns,
     const struct path *path, struct kstat *stat, u32 request_mask,
     unsigned int query_flags)
 #else
 zpl_snapdir_getattr_impl(const struct path *path, struct kstat *stat,
     u32 request_mask, unsigned int query_flags)
 #endif
 {
 	(void) request_mask, (void) query_flags;
 	struct inode *ip = path->dentry->d_inode;
 	zfsvfs_t *zfsvfs = ITOZSB(ip);
 	int error;
 
 	if ((error = zpl_enter(zfsvfs, FTAG)) != 0)
 		return (error);
 #if (defined(HAVE_USERNS_IOPS_GETATTR) || defined(HAVE_IDMAP_IOPS_GETATTR))
 #ifdef HAVE_GENERIC_FILLATTR_USERNS
 	generic_fillattr(user_ns, ip, stat);
 #elif defined(HAVE_GENERIC_FILLATTR_IDMAP)
 	generic_fillattr(user_ns, ip, stat);
 #elif defined(HAVE_GENERIC_FILLATTR_IDMAP_REQMASK)
 	generic_fillattr(user_ns, request_mask, ip, stat);
 #else
 	(void) user_ns;
 #endif
 #else
 	generic_fillattr(ip, stat);
 #endif
 
 	stat->nlink = stat->size = 2;
 
 	dsl_dataset_t *ds = dmu_objset_ds(zfsvfs->z_os);
 	if (dsl_dataset_phys(ds)->ds_snapnames_zapobj != 0) {
 		uint64_t snap_count;
 		int err = zap_count(
 		    dmu_objset_pool(ds->ds_objset)->dp_meta_objset,
 		    dsl_dataset_phys(ds)->ds_snapnames_zapobj, &snap_count);
 		if (err != 0) {
 			zpl_exit(zfsvfs, FTAG);
 			return (-err);
 		}
 		stat->nlink += snap_count;
 	}
 
 	stat->ctime = stat->mtime = dmu_objset_snap_cmtime(zfsvfs->z_os);
 	stat->atime = current_time(ip);
 	zpl_exit(zfsvfs, FTAG);
 
 	return (0);
 }
 ZPL_GETATTR_WRAPPER(zpl_snapdir_getattr);
 
 /*
  * The '.zfs/snapshot' directory file operations.  These mainly control
  * generating the list of available snapshots when doing an 'ls' in the
  * directory.  See zpl_snapdir_readdir().
  */
 const struct file_operations zpl_fops_snapdir = {
 	.open		= zpl_common_open,
 	.llseek		= generic_file_llseek,
 	.read		= generic_read_dir,
 	.iterate_shared	= zpl_snapdir_iterate,
 
 };
 
 /*
  * The '.zfs/snapshot' directory inode operations.  These mainly control
  * creating an inode for a snapshot directory and initializing the needed
  * infrastructure to automount the snapshot.  See zpl_snapdir_lookup().
  */
 const struct inode_operations zpl_ops_snapdir = {
 	.lookup		= zpl_snapdir_lookup,
 	.getattr	= zpl_snapdir_getattr,
 #if (defined(HAVE_RENAME_WANTS_FLAGS) || \
 	defined(HAVE_IOPS_RENAME_USERNS) || \
 	defined(HAVE_IOPS_RENAME_IDMAP))
 	.rename		= zpl_snapdir_rename2,
 #else
 	.rename		= zpl_snapdir_rename,
 #endif
 	.rmdir		= zpl_snapdir_rmdir,
 	.mkdir		= zpl_snapdir_mkdir,
 };
 
 static struct dentry *
 zpl_shares_lookup(struct inode *dip, struct dentry *dentry,
     unsigned int flags)
 {
 	fstrans_cookie_t cookie;
 	cred_t *cr = CRED();
 	struct inode *ip = NULL;
 	int error;
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	error = -zfsctl_shares_lookup(dip, dname(dentry), &ip,
 	    0, cr, NULL, NULL);
 	ASSERT3S(error, <=, 0);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 
 	if (error) {
 		if (error == -ENOENT)
 			return (d_splice_alias(NULL, dentry));
 		else
 			return (ERR_PTR(error));
 	}
 
 	return (d_splice_alias(ip, dentry));
 }
 
 static int
 zpl_shares_iterate(struct file *filp, struct dir_context *ctx)
 {
 	fstrans_cookie_t cookie;
 	cred_t *cr = CRED();
 	zfsvfs_t *zfsvfs = ITOZSB(file_inode(filp));
 	znode_t *dzp;
 	int error = 0;
 
 	if ((error = zpl_enter(zfsvfs, FTAG)) != 0)
 		return (error);
 	cookie = spl_fstrans_mark();
 
 	if (zfsvfs->z_shares_dir == 0) {
 		dir_emit_dots(filp, ctx);
 		goto out;
 	}
 
 	error = -zfs_zget(zfsvfs, zfsvfs->z_shares_dir, &dzp);
 	if (error)
 		goto out;
 
 	crhold(cr);
 	error = -zfs_readdir(ZTOI(dzp), ctx, cr);
 	crfree(cr);
 
 	iput(ZTOI(dzp));
 out:
 	spl_fstrans_unmark(cookie);
 	zpl_exit(zfsvfs, FTAG);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 #ifdef HAVE_USERNS_IOPS_GETATTR
 zpl_shares_getattr_impl(struct user_namespace *user_ns,
     const struct path *path, struct kstat *stat, u32 request_mask,
     unsigned int query_flags)
 #elif defined(HAVE_IDMAP_IOPS_GETATTR)
 zpl_shares_getattr_impl(struct mnt_idmap *user_ns,
     const struct path *path, struct kstat *stat, u32 request_mask,
     unsigned int query_flags)
 #else
 zpl_shares_getattr_impl(const struct path *path, struct kstat *stat,
     u32 request_mask, unsigned int query_flags)
 #endif
 {
 	(void) request_mask, (void) query_flags;
 	struct inode *ip = path->dentry->d_inode;
 	zfsvfs_t *zfsvfs = ITOZSB(ip);
 	znode_t *dzp;
 	int error;
 
 	if ((error = zpl_enter(zfsvfs, FTAG)) != 0)
 		return (error);
 
 	if (zfsvfs->z_shares_dir == 0) {
 #if (defined(HAVE_USERNS_IOPS_GETATTR) || defined(HAVE_IDMAP_IOPS_GETATTR))
 #ifdef HAVE_GENERIC_FILLATTR_USERNS
 		generic_fillattr(user_ns, path->dentry->d_inode, stat);
 #elif defined(HAVE_GENERIC_FILLATTR_IDMAP)
 		generic_fillattr(user_ns, path->dentry->d_inode, stat);
 #elif defined(HAVE_GENERIC_FILLATTR_IDMAP_REQMASK)
 	generic_fillattr(user_ns, request_mask, ip, stat);
 #else
 		(void) user_ns;
 #endif
 #else
 		generic_fillattr(path->dentry->d_inode, stat);
 #endif
 		stat->nlink = stat->size = 2;
 		stat->atime = current_time(ip);
 		zpl_exit(zfsvfs, FTAG);
 		return (0);
 	}
 
 	error = -zfs_zget(zfsvfs, zfsvfs->z_shares_dir, &dzp);
 	if (error == 0) {
 #ifdef HAVE_GENERIC_FILLATTR_IDMAP_REQMASK
 		error = -zfs_getattr_fast(user_ns, request_mask, ZTOI(dzp),
 		    stat);
 #elif (defined(HAVE_USERNS_IOPS_GETATTR) || defined(HAVE_IDMAP_IOPS_GETATTR))
 		error = -zfs_getattr_fast(user_ns, ZTOI(dzp), stat);
 #else
 		error = -zfs_getattr_fast(kcred->user_ns, ZTOI(dzp), stat);
 #endif
 		iput(ZTOI(dzp));
 	}
 
 	zpl_exit(zfsvfs, FTAG);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 ZPL_GETATTR_WRAPPER(zpl_shares_getattr);
 
 /*
  * The '.zfs/shares' directory file operations.
  */
 const struct file_operations zpl_fops_shares = {
 	.open		= zpl_common_open,
 	.llseek		= generic_file_llseek,
 	.read		= generic_read_dir,
 	.iterate_shared	= zpl_shares_iterate,
 };
 
 /*
  * The '.zfs/shares' directory inode operations.
  */
 const struct inode_operations zpl_ops_shares = {
 	.lookup		= zpl_shares_lookup,
 	.getattr	= zpl_shares_getattr,
 };
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c
index 4d1bf1d5477f..7a1e7eee79de 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_file.c
@@ -1,1143 +1,1150 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2011, Lawrence Livermore National Security, LLC.
  * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
  */
 
 
 #ifdef CONFIG_COMPAT
 #include <linux/compat.h>
 #endif
 #include <linux/fs.h>
+#include <linux/migrate.h>
 #include <sys/file.h>
 #include <sys/dmu_objset.h>
 #include <sys/zfs_znode.h>
 #include <sys/zfs_vfsops.h>
 #include <sys/zfs_vnops.h>
 #include <sys/zfs_project.h>
 #if defined(HAVE_VFS_SET_PAGE_DIRTY_NOBUFFERS) || \
     defined(HAVE_VFS_FILEMAP_DIRTY_FOLIO)
 #include <linux/pagemap.h>
 #endif
 #include <linux/fadvise.h>
 #ifdef HAVE_VFS_FILEMAP_DIRTY_FOLIO
 #include <linux/writeback.h>
 #endif
 
 /*
  * When using fallocate(2) to preallocate space, inflate the requested
  * capacity check by 10% to account for the required metadata blocks.
  */
 static unsigned int zfs_fallocate_reserve_percent = 110;
 
 static int
 zpl_open(struct inode *ip, struct file *filp)
 {
 	cred_t *cr = CRED();
 	int error;
 	fstrans_cookie_t cookie;
 
 	error = generic_file_open(ip, filp);
 	if (error)
 		return (error);
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	error = -zfs_open(ip, filp->f_mode, filp->f_flags, cr);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 zpl_release(struct inode *ip, struct file *filp)
 {
 	cred_t *cr = CRED();
 	int error;
 	fstrans_cookie_t cookie;
 
 	cookie = spl_fstrans_mark();
 	if (ITOZ(ip)->z_atime_dirty)
 		zfs_mark_inode_dirty(ip);
 
 	crhold(cr);
 	error = -zfs_close(ip, filp->f_flags, cr);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 zpl_iterate(struct file *filp, struct dir_context *ctx)
 {
 	cred_t *cr = CRED();
 	int error;
 	fstrans_cookie_t cookie;
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	error = -zfs_readdir(file_inode(filp), ctx, cr);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 zpl_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
 {
 	struct inode *inode = filp->f_mapping->host;
 	znode_t *zp = ITOZ(inode);
 	zfsvfs_t *zfsvfs = ITOZSB(inode);
 	cred_t *cr = CRED();
 	int error;
 	fstrans_cookie_t cookie;
 
 	/*
 	 * The variables z_sync_writes_cnt and z_async_writes_cnt work in
 	 * tandem so that sync writes can detect if there are any non-sync
 	 * writes going on and vice-versa. The "vice-versa" part to this logic
 	 * is located in zfs_putpage() where non-sync writes check if there are
 	 * any ongoing sync writes. If any sync and non-sync writes overlap,
 	 * we do a commit to complete the non-sync writes since the latter can
 	 * potentially take several seconds to complete and thus block sync
 	 * writes in the upcoming call to filemap_write_and_wait_range().
 	 */
 	atomic_inc_32(&zp->z_sync_writes_cnt);
 	/*
 	 * If the following check does not detect an overlapping non-sync write
 	 * (say because it's just about to start), then it is guaranteed that
 	 * the non-sync write will detect this sync write. This is because we
 	 * always increment z_sync_writes_cnt / z_async_writes_cnt before doing
 	 * the check on z_async_writes_cnt / z_sync_writes_cnt here and in
 	 * zfs_putpage() respectively.
 	 */
 	if (atomic_load_32(&zp->z_async_writes_cnt) > 0) {
 		if ((error = zpl_enter(zfsvfs, FTAG)) != 0) {
 			atomic_dec_32(&zp->z_sync_writes_cnt);
 			return (error);
 		}
 		zil_commit(zfsvfs->z_log, zp->z_id);
 		zpl_exit(zfsvfs, FTAG);
 	}
 
 	error = filemap_write_and_wait_range(inode->i_mapping, start, end);
 
 	/*
 	 * The sync write is not complete yet but we decrement
 	 * z_sync_writes_cnt since zfs_fsync() increments and decrements
 	 * it internally. If a non-sync write starts just after the decrement
 	 * operation but before we call zfs_fsync(), it may not detect this
 	 * overlapping sync write but it does not matter since we have already
 	 * gone past filemap_write_and_wait_range() and we won't block due to
 	 * the non-sync write.
 	 */
 	atomic_dec_32(&zp->z_sync_writes_cnt);
 
 	if (error)
 		return (error);
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	error = -zfs_fsync(zp, datasync, cr);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static inline int
 zfs_io_flags(struct kiocb *kiocb)
 {
 	int flags = 0;
 
 #if defined(IOCB_DSYNC)
 	if (kiocb->ki_flags & IOCB_DSYNC)
 		flags |= O_DSYNC;
 #endif
 #if defined(IOCB_SYNC)
 	if (kiocb->ki_flags & IOCB_SYNC)
 		flags |= O_SYNC;
 #endif
 #if defined(IOCB_APPEND)
 	if (kiocb->ki_flags & IOCB_APPEND)
 		flags |= O_APPEND;
 #endif
 #if defined(IOCB_DIRECT)
 	if (kiocb->ki_flags & IOCB_DIRECT)
 		flags |= O_DIRECT;
 #endif
 	return (flags);
 }
 
 /*
  * If relatime is enabled, call file_accessed() if zfs_relatime_need_update()
  * is true.  This is needed since datasets with inherited "relatime" property
  * aren't necessarily mounted with the MNT_RELATIME flag (e.g. after
  * `zfs set relatime=...`), which is what relatime test in VFS by
  * relatime_need_update() is based on.
  */
 static inline void
 zpl_file_accessed(struct file *filp)
 {
 	struct inode *ip = filp->f_mapping->host;
 
 	if (!IS_NOATIME(ip) && ITOZSB(ip)->z_relatime) {
 		if (zfs_relatime_need_update(ip))
 			file_accessed(filp);
 	} else {
 		file_accessed(filp);
 	}
 }
 
 /*
  * When HAVE_VFS_IOV_ITER is defined the iov_iter structure supports
  * iovecs, kvevs, bvecs and pipes, plus all the required interfaces to
  * manipulate the iov_iter are available.  In which case the full iov_iter
  * can be attached to the uio and correctly handled in the lower layers.
  * Otherwise, for older kernels extract the iovec and pass it instead.
  */
 static void
 zpl_uio_init(zfs_uio_t *uio, struct kiocb *kiocb, struct iov_iter *to,
     loff_t pos, ssize_t count, size_t skip)
 {
 #if defined(HAVE_VFS_IOV_ITER)
-	zfs_uio_iov_iter_init(uio, to, pos, count, skip);
+	(void) skip;
+	zfs_uio_iov_iter_init(uio, to, pos, count);
 #else
 	zfs_uio_iovec_init(uio, zfs_uio_iter_iov(to), to->nr_segs, pos,
 	    zfs_uio_iov_iter_type(to) & ITER_KVEC ?
 	    UIO_SYSSPACE : UIO_USERSPACE,
 	    count, skip);
 #endif
 }
 
 static ssize_t
 zpl_iter_read(struct kiocb *kiocb, struct iov_iter *to)
 {
 	cred_t *cr = CRED();
 	fstrans_cookie_t cookie;
 	struct file *filp = kiocb->ki_filp;
 	ssize_t count = iov_iter_count(to);
 	zfs_uio_t uio;
 
 	zpl_uio_init(&uio, kiocb, to, kiocb->ki_pos, count, 0);
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 
 	ssize_t ret = -zfs_read(ITOZ(filp->f_mapping->host), &uio,
 	    filp->f_flags | zfs_io_flags(kiocb), cr);
 
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 
 	if (ret < 0)
 		return (ret);
 
 	ssize_t read = count - uio.uio_resid;
 	kiocb->ki_pos += read;
 
 	zpl_file_accessed(filp);
 
 	return (read);
 }
 
 static inline ssize_t
 zpl_generic_write_checks(struct kiocb *kiocb, struct iov_iter *from,
     size_t *countp)
 {
 	ssize_t ret = generic_write_checks(kiocb, from);
 	if (ret <= 0)
 		return (ret);
 
 	*countp = ret;
 
 	return (0);
 }
 
 static ssize_t
 zpl_iter_write(struct kiocb *kiocb, struct iov_iter *from)
 {
 	cred_t *cr = CRED();
 	fstrans_cookie_t cookie;
 	struct file *filp = kiocb->ki_filp;
 	struct inode *ip = filp->f_mapping->host;
 	zfs_uio_t uio;
 	size_t count = 0;
 	ssize_t ret;
 
 	ret = zpl_generic_write_checks(kiocb, from, &count);
 	if (ret)
 		return (ret);
 
 	zpl_uio_init(&uio, kiocb, from, kiocb->ki_pos, count, from->iov_offset);
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 
 	ret = -zfs_write(ITOZ(ip), &uio,
 	    filp->f_flags | zfs_io_flags(kiocb), cr);
 
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 
 	if (ret < 0)
 		return (ret);
 
 	ssize_t wrote = count - uio.uio_resid;
 	kiocb->ki_pos += wrote;
 
 	return (wrote);
 }
 
 static ssize_t
 zpl_direct_IO_impl(void)
 {
 	/*
 	 * All O_DIRECT requests should be handled by
 	 * zpl_{iter/aio}_{write/read}(). There is no way kernel generic code
 	 * should call the direct_IO address_space_operations function. We set
 	 * this code path to be fatal if it is executed.
 	 */
 	PANIC(0);
 	return (0);
 }
 
 #if defined(HAVE_VFS_DIRECT_IO_ITER)
 static ssize_t
 zpl_direct_IO(struct kiocb *kiocb, struct iov_iter *iter)
 {
 	return (zpl_direct_IO_impl());
 }
 #elif defined(HAVE_VFS_DIRECT_IO_ITER_OFFSET)
 static ssize_t
 zpl_direct_IO(struct kiocb *kiocb, struct iov_iter *iter, loff_t pos)
 {
 	return (zpl_direct_IO_impl());
 }
 #else
 #error "Unknown Direct I/O interface"
 #endif
 
 static loff_t
 zpl_llseek(struct file *filp, loff_t offset, int whence)
 {
 #if defined(SEEK_HOLE) && defined(SEEK_DATA)
 	fstrans_cookie_t cookie;
 
 	if (whence == SEEK_DATA || whence == SEEK_HOLE) {
 		struct inode *ip = filp->f_mapping->host;
 		loff_t maxbytes = ip->i_sb->s_maxbytes;
 		loff_t error;
 
 		spl_inode_lock_shared(ip);
 		cookie = spl_fstrans_mark();
 		error = -zfs_holey(ITOZ(ip), whence, &offset);
 		spl_fstrans_unmark(cookie);
 		if (error == 0)
 			error = lseek_execute(filp, ip, offset, maxbytes);
 		spl_inode_unlock_shared(ip);
 
 		return (error);
 	}
 #endif /* SEEK_HOLE && SEEK_DATA */
 
 	return (generic_file_llseek(filp, offset, whence));
 }
 
 /*
  * It's worth taking a moment to describe how mmap is implemented
  * for zfs because it differs considerably from other Linux filesystems.
  * However, this issue is handled the same way under OpenSolaris.
  *
  * The issue is that by design zfs bypasses the Linux page cache and
  * leaves all caching up to the ARC.  This has been shown to work
  * well for the common read(2)/write(2) case.  However, mmap(2)
  * is problem because it relies on being tightly integrated with the
  * page cache.  To handle this we cache mmap'ed files twice, once in
  * the ARC and a second time in the page cache.  The code is careful
  * to keep both copies synchronized.
  *
  * When a file with an mmap'ed region is written to using write(2)
  * both the data in the ARC and existing pages in the page cache
  * are updated.  For a read(2) data will be read first from the page
  * cache then the ARC if needed.  Neither a write(2) or read(2) will
  * will ever result in new pages being added to the page cache.
  *
  * New pages are added to the page cache only via .readpage() which
  * is called when the vfs needs to read a page off disk to back the
  * virtual memory region.  These pages may be modified without
  * notifying the ARC and will be written out periodically via
  * .writepage().  This will occur due to either a sync or the usual
  * page aging behavior.  Note because a read(2) of a mmap'ed file
  * will always check the page cache first even when the ARC is out
  * of date correct data will still be returned.
  *
  * While this implementation ensures correct behavior it does have
  * have some drawbacks.  The most obvious of which is that it
  * increases the required memory footprint when access mmap'ed
  * files.  It also adds additional complexity to the code keeping
  * both caches synchronized.
  *
  * Longer term it may be possible to cleanly resolve this wart by
  * mapping page cache pages directly on to the ARC buffers.  The
  * Linux address space operations are flexible enough to allow
  * selection of which pages back a particular index.  The trick
  * would be working out the details of which subsystem is in
  * charge, the ARC, the page cache, or both.  It may also prove
  * helpful to move the ARC buffers to a scatter-gather lists
  * rather than a vmalloc'ed region.
  */
 static int
 zpl_mmap(struct file *filp, struct vm_area_struct *vma)
 {
 	struct inode *ip = filp->f_mapping->host;
 	int error;
 	fstrans_cookie_t cookie;
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_map(ip, vma->vm_pgoff, (caddr_t *)vma->vm_start,
 	    (size_t)(vma->vm_end - vma->vm_start), vma->vm_flags);
 	spl_fstrans_unmark(cookie);
 
 	if (error)
 		return (error);
 
 	error = generic_file_mmap(filp, vma);
 	if (error)
 		return (error);
 
 	return (error);
 }
 
 /*
  * Populate a page with data for the Linux page cache.  This function is
  * only used to support mmap(2).  There will be an identical copy of the
  * data in the ARC which is kept up to date via .write() and .writepage().
  */
 static inline int
 zpl_readpage_common(struct page *pp)
 {
 	fstrans_cookie_t cookie;
 
 	ASSERT(PageLocked(pp));
 
 	cookie = spl_fstrans_mark();
 	int error = -zfs_getpage(pp->mapping->host, pp);
 	spl_fstrans_unmark(cookie);
 
 	unlock_page(pp);
 
 	return (error);
 }
 
 #ifdef HAVE_VFS_READ_FOLIO
 static int
 zpl_read_folio(struct file *filp, struct folio *folio)
 {
 	return (zpl_readpage_common(&folio->page));
 }
 #else
 static int
 zpl_readpage(struct file *filp, struct page *pp)
 {
 	return (zpl_readpage_common(pp));
 }
 #endif
 
 static int
 zpl_readpage_filler(void *data, struct page *pp)
 {
 	return (zpl_readpage_common(pp));
 }
 
 /*
  * Populate a set of pages with data for the Linux page cache.  This
  * function will only be called for read ahead and never for demand
  * paging.  For simplicity, the code relies on read_cache_pages() to
  * correctly lock each page for IO and call zpl_readpage().
  */
 #ifdef HAVE_VFS_READPAGES
 static int
 zpl_readpages(struct file *filp, struct address_space *mapping,
     struct list_head *pages, unsigned nr_pages)
 {
 	return (read_cache_pages(mapping, pages, zpl_readpage_filler, NULL));
 }
 #else
 static void
 zpl_readahead(struct readahead_control *ractl)
 {
 	struct page *page;
 
 	while ((page = readahead_page(ractl)) != NULL) {
 		int ret;
 
 		ret = zpl_readpage_filler(NULL, page);
 		put_page(page);
 		if (ret)
 			break;
 	}
 }
 #endif
 
 static int
 zpl_putpage(struct page *pp, struct writeback_control *wbc, void *data)
 {
 	boolean_t *for_sync = data;
 	fstrans_cookie_t cookie;
 	int ret;
 
 	ASSERT(PageLocked(pp));
 	ASSERT(!PageWriteback(pp));
 
 	cookie = spl_fstrans_mark();
 	ret = zfs_putpage(pp->mapping->host, pp, wbc, *for_sync);
 	spl_fstrans_unmark(cookie);
 
 	return (ret);
 }
 
 #ifdef HAVE_WRITEPAGE_T_FOLIO
 static int
 zpl_putfolio(struct folio *pp, struct writeback_control *wbc, void *data)
 {
 	return (zpl_putpage(&pp->page, wbc, data));
 }
 #endif
 
 static inline int
 zpl_write_cache_pages(struct address_space *mapping,
     struct writeback_control *wbc, void *data)
 {
 	int result;
 
 #ifdef HAVE_WRITEPAGE_T_FOLIO
 	result = write_cache_pages(mapping, wbc, zpl_putfolio, data);
 #else
 	result = write_cache_pages(mapping, wbc, zpl_putpage, data);
 #endif
 	return (result);
 }
 
 static int
 zpl_writepages(struct address_space *mapping, struct writeback_control *wbc)
 {
 	znode_t		*zp = ITOZ(mapping->host);
 	zfsvfs_t	*zfsvfs = ITOZSB(mapping->host);
 	enum writeback_sync_modes sync_mode;
 	int result;
 
 	if ((result = zpl_enter(zfsvfs, FTAG)) != 0)
 		return (result);
 	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
 		wbc->sync_mode = WB_SYNC_ALL;
 	zpl_exit(zfsvfs, FTAG);
 	sync_mode = wbc->sync_mode;
 
 	/*
 	 * We don't want to run write_cache_pages() in SYNC mode here, because
 	 * that would make putpage() wait for a single page to be committed to
 	 * disk every single time, resulting in atrocious performance. Instead
 	 * we run it once in non-SYNC mode so that the ZIL gets all the data,
 	 * and then we commit it all in one go.
 	 */
 	boolean_t for_sync = (sync_mode == WB_SYNC_ALL);
 	wbc->sync_mode = WB_SYNC_NONE;
 	result = zpl_write_cache_pages(mapping, wbc, &for_sync);
 	if (sync_mode != wbc->sync_mode) {
 		if ((result = zpl_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
 			return (result);
 		if (zfsvfs->z_log != NULL)
 			zil_commit(zfsvfs->z_log, zp->z_id);
 		zpl_exit(zfsvfs, FTAG);
 
 		/*
 		 * We need to call write_cache_pages() again (we can't just
 		 * return after the commit) because the previous call in
 		 * non-SYNC mode does not guarantee that we got all the dirty
 		 * pages (see the implementation of write_cache_pages() for
 		 * details). That being said, this is a no-op in most cases.
 		 */
 		wbc->sync_mode = sync_mode;
 		result = zpl_write_cache_pages(mapping, wbc, &for_sync);
 	}
 	return (result);
 }
 
 /*
  * Write out dirty pages to the ARC, this function is only required to
  * support mmap(2).  Mapped pages may be dirtied by memory operations
  * which never call .write().  These dirty pages are kept in sync with
  * the ARC buffers via this hook.
  */
 static int
 zpl_writepage(struct page *pp, struct writeback_control *wbc)
 {
 	if (ITOZSB(pp->mapping->host)->z_os->os_sync == ZFS_SYNC_ALWAYS)
 		wbc->sync_mode = WB_SYNC_ALL;
 
 	boolean_t for_sync = (wbc->sync_mode == WB_SYNC_ALL);
 
 	return (zpl_putpage(pp, wbc, &for_sync));
 }
 
 /*
  * The flag combination which matches the behavior of zfs_space() is
  * FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE.  The FALLOC_FL_PUNCH_HOLE
  * flag was introduced in the 2.6.38 kernel.
  *
  * The original mode=0 (allocate space) behavior can be reasonably emulated
  * by checking if enough space exists and creating a sparse file, as real
  * persistent space reservation is not possible due to COW, snapshots, etc.
  */
 static long
 zpl_fallocate_common(struct inode *ip, int mode, loff_t offset, loff_t len)
 {
 	cred_t *cr = CRED();
 	loff_t olen;
 	fstrans_cookie_t cookie;
 	int error = 0;
 
 	int test_mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE;
 
 	if ((mode & ~(FALLOC_FL_KEEP_SIZE | test_mode)) != 0)
 		return (-EOPNOTSUPP);
 
 	if (offset < 0 || len <= 0)
 		return (-EINVAL);
 
 	spl_inode_lock(ip);
 	olen = i_size_read(ip);
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	if (mode & (test_mode)) {
 		flock64_t bf;
 
 		if (mode & FALLOC_FL_KEEP_SIZE) {
 			if (offset > olen)
 				goto out_unmark;
 
 			if (offset + len > olen)
 				len = olen - offset;
 		}
 		bf.l_type = F_WRLCK;
 		bf.l_whence = SEEK_SET;
 		bf.l_start = offset;
 		bf.l_len = len;
 		bf.l_pid = 0;
 
 		error = -zfs_space(ITOZ(ip), F_FREESP, &bf, O_RDWR, offset, cr);
 	} else if ((mode & ~FALLOC_FL_KEEP_SIZE) == 0) {
 		unsigned int percent = zfs_fallocate_reserve_percent;
 		struct kstatfs statfs;
 
 		/* Legacy mode, disable fallocate compatibility. */
 		if (percent == 0) {
 			error = -EOPNOTSUPP;
 			goto out_unmark;
 		}
 
 		/*
 		 * Use zfs_statvfs() instead of dmu_objset_space() since it
 		 * also checks project quota limits, which are relevant here.
 		 */
 		error = zfs_statvfs(ip, &statfs);
 		if (error)
 			goto out_unmark;
 
 		/*
 		 * Shrink available space a bit to account for overhead/races.
 		 * We know the product previously fit into availbytes from
 		 * dmu_objset_space(), so the smaller product will also fit.
 		 */
 		if (len > statfs.f_bavail * (statfs.f_bsize * 100 / percent)) {
 			error = -ENOSPC;
 			goto out_unmark;
 		}
 		if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > olen)
 			error = zfs_freesp(ITOZ(ip), offset + len, 0, 0, FALSE);
 	}
 out_unmark:
 	spl_fstrans_unmark(cookie);
 	spl_inode_unlock(ip);
 
 	crfree(cr);
 
 	return (error);
 }
 
 static long
 zpl_fallocate(struct file *filp, int mode, loff_t offset, loff_t len)
 {
 	return zpl_fallocate_common(file_inode(filp),
 	    mode, offset, len);
 }
 
 static int
 zpl_ioctl_getversion(struct file *filp, void __user *arg)
 {
 	uint32_t generation = file_inode(filp)->i_generation;
 
 	return (copy_to_user(arg, &generation, sizeof (generation)));
 }
 
 static int
 zpl_fadvise(struct file *filp, loff_t offset, loff_t len, int advice)
 {
 	struct inode *ip = file_inode(filp);
 	znode_t *zp = ITOZ(ip);
 	zfsvfs_t *zfsvfs = ITOZSB(ip);
 	objset_t *os = zfsvfs->z_os;
 	int error = 0;
 
 	if (S_ISFIFO(ip->i_mode))
 		return (-ESPIPE);
 
 	if (offset < 0 || len < 0)
 		return (-EINVAL);
 
 	if ((error = zpl_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
 		return (error);
 
 	switch (advice) {
 	case POSIX_FADV_SEQUENTIAL:
 	case POSIX_FADV_WILLNEED:
 #ifdef HAVE_GENERIC_FADVISE
 		if (zn_has_cached_data(zp, offset, offset + len - 1))
 			error = generic_fadvise(filp, offset, len, advice);
 #endif
 		/*
 		 * Pass on the caller's size directly, but note that
 		 * dmu_prefetch_max will effectively cap it.  If there
 		 * really is a larger sequential access pattern, perhaps
 		 * dmu_zfetch will detect it.
 		 */
 		if (len == 0)
 			len = i_size_read(ip) - offset;
 
 		dmu_prefetch(os, zp->z_id, 0, offset, len,
 		    ZIO_PRIORITY_ASYNC_READ);
 		break;
 	case POSIX_FADV_NORMAL:
 	case POSIX_FADV_RANDOM:
 	case POSIX_FADV_DONTNEED:
 	case POSIX_FADV_NOREUSE:
 		/* ignored for now */
 		break;
 	default:
 		error = -EINVAL;
 		break;
 	}
 
 	zfs_exit(zfsvfs, FTAG);
 
 	return (error);
 }
 
 #define	ZFS_FL_USER_VISIBLE	(FS_FL_USER_VISIBLE | ZFS_PROJINHERIT_FL)
 #define	ZFS_FL_USER_MODIFIABLE	(FS_FL_USER_MODIFIABLE | ZFS_PROJINHERIT_FL)
 
 static uint32_t
 __zpl_ioctl_getflags(struct inode *ip)
 {
 	uint64_t zfs_flags = ITOZ(ip)->z_pflags;
 	uint32_t ioctl_flags = 0;
 
 	if (zfs_flags & ZFS_IMMUTABLE)
 		ioctl_flags |= FS_IMMUTABLE_FL;
 
 	if (zfs_flags & ZFS_APPENDONLY)
 		ioctl_flags |= FS_APPEND_FL;
 
 	if (zfs_flags & ZFS_NODUMP)
 		ioctl_flags |= FS_NODUMP_FL;
 
 	if (zfs_flags & ZFS_PROJINHERIT)
 		ioctl_flags |= ZFS_PROJINHERIT_FL;
 
 	return (ioctl_flags & ZFS_FL_USER_VISIBLE);
 }
 
 /*
  * Map zfs file z_pflags (xvattr_t) to linux file attributes. Only file
  * attributes common to both Linux and Solaris are mapped.
  */
 static int
 zpl_ioctl_getflags(struct file *filp, void __user *arg)
 {
 	uint32_t flags;
 	int err;
 
 	flags = __zpl_ioctl_getflags(file_inode(filp));
 	err = copy_to_user(arg, &flags, sizeof (flags));
 
 	return (err);
 }
 
 /*
  * fchange() is a helper macro to detect if we have been asked to change a
  * flag. This is ugly, but the requirement that we do this is a consequence of
  * how the Linux file attribute interface was designed. Another consequence is
  * that concurrent modification of files suffers from a TOCTOU race. Neither
  * are things we can fix without modifying the kernel-userland interface, which
  * is outside of our jurisdiction.
  */
 
 #define	fchange(f0, f1, b0, b1) (!((f0) & (b0)) != !((f1) & (b1)))
 
 static int
 __zpl_ioctl_setflags(struct inode *ip, uint32_t ioctl_flags, xvattr_t *xva)
 {
 	uint64_t zfs_flags = ITOZ(ip)->z_pflags;
 	xoptattr_t *xoap;
 
 	if (ioctl_flags & ~(FS_IMMUTABLE_FL | FS_APPEND_FL | FS_NODUMP_FL |
 	    ZFS_PROJINHERIT_FL))
 		return (-EOPNOTSUPP);
 
 	if (ioctl_flags & ~ZFS_FL_USER_MODIFIABLE)
 		return (-EACCES);
 
 	if ((fchange(ioctl_flags, zfs_flags, FS_IMMUTABLE_FL, ZFS_IMMUTABLE) ||
 	    fchange(ioctl_flags, zfs_flags, FS_APPEND_FL, ZFS_APPENDONLY)) &&
 	    !capable(CAP_LINUX_IMMUTABLE))
 		return (-EPERM);
 
 	if (!zpl_inode_owner_or_capable(zfs_init_idmap, ip))
 		return (-EACCES);
 
 	xva_init(xva);
 	xoap = xva_getxoptattr(xva);
 
 #define	FLAG_CHANGE(iflag, zflag, xflag, xfield)	do {	\
 	if (((ioctl_flags & (iflag)) && !(zfs_flags & (zflag))) ||	\
 	    ((zfs_flags & (zflag)) && !(ioctl_flags & (iflag)))) {	\
 		XVA_SET_REQ(xva, (xflag));	\
 		(xfield) = ((ioctl_flags & (iflag)) != 0);	\
 	}	\
 } while (0)
 
 	FLAG_CHANGE(FS_IMMUTABLE_FL, ZFS_IMMUTABLE, XAT_IMMUTABLE,
 	    xoap->xoa_immutable);
 	FLAG_CHANGE(FS_APPEND_FL, ZFS_APPENDONLY, XAT_APPENDONLY,
 	    xoap->xoa_appendonly);
 	FLAG_CHANGE(FS_NODUMP_FL, ZFS_NODUMP, XAT_NODUMP,
 	    xoap->xoa_nodump);
 	FLAG_CHANGE(ZFS_PROJINHERIT_FL, ZFS_PROJINHERIT, XAT_PROJINHERIT,
 	    xoap->xoa_projinherit);
 
 #undef	FLAG_CHANGE
 
 	return (0);
 }
 
 static int
 zpl_ioctl_setflags(struct file *filp, void __user *arg)
 {
 	struct inode *ip = file_inode(filp);
 	uint32_t flags;
 	cred_t *cr = CRED();
 	xvattr_t xva;
 	int err;
 	fstrans_cookie_t cookie;
 
 	if (copy_from_user(&flags, arg, sizeof (flags)))
 		return (-EFAULT);
 
 	err = __zpl_ioctl_setflags(ip, flags, &xva);
 	if (err)
 		return (err);
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 
 	return (err);
 }
 
 static int
 zpl_ioctl_getxattr(struct file *filp, void __user *arg)
 {
 	zfsxattr_t fsx = { 0 };
 	struct inode *ip = file_inode(filp);
 	int err;
 
 	fsx.fsx_xflags = __zpl_ioctl_getflags(ip);
 	fsx.fsx_projid = ITOZ(ip)->z_projid;
 	err = copy_to_user(arg, &fsx, sizeof (fsx));
 
 	return (err);
 }
 
 static int
 zpl_ioctl_setxattr(struct file *filp, void __user *arg)
 {
 	struct inode *ip = file_inode(filp);
 	zfsxattr_t fsx;
 	cred_t *cr = CRED();
 	xvattr_t xva;
 	xoptattr_t *xoap;
 	int err;
 	fstrans_cookie_t cookie;
 
 	if (copy_from_user(&fsx, arg, sizeof (fsx)))
 		return (-EFAULT);
 
 	if (!zpl_is_valid_projid(fsx.fsx_projid))
 		return (-EINVAL);
 
 	err = __zpl_ioctl_setflags(ip, fsx.fsx_xflags, &xva);
 	if (err)
 		return (err);
 
 	xoap = xva_getxoptattr(&xva);
 	XVA_SET_REQ(&xva, XAT_PROJID);
 	xoap->xoa_projid = fsx.fsx_projid;
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 
 	return (err);
 }
 
 /*
  * Expose Additional File Level Attributes of ZFS.
  */
 static int
 zpl_ioctl_getdosflags(struct file *filp, void __user *arg)
 {
 	struct inode *ip = file_inode(filp);
 	uint64_t dosflags = ITOZ(ip)->z_pflags;
 	dosflags &= ZFS_DOS_FL_USER_VISIBLE;
 	int err = copy_to_user(arg, &dosflags, sizeof (dosflags));
 
 	return (err);
 }
 
 static int
 __zpl_ioctl_setdosflags(struct inode *ip, uint64_t ioctl_flags, xvattr_t *xva)
 {
 	uint64_t zfs_flags = ITOZ(ip)->z_pflags;
 	xoptattr_t *xoap;
 
 	if (ioctl_flags & (~ZFS_DOS_FL_USER_VISIBLE))
 		return (-EOPNOTSUPP);
 
 	if ((fchange(ioctl_flags, zfs_flags, ZFS_IMMUTABLE, ZFS_IMMUTABLE) ||
 	    fchange(ioctl_flags, zfs_flags, ZFS_APPENDONLY, ZFS_APPENDONLY)) &&
 	    !capable(CAP_LINUX_IMMUTABLE))
 		return (-EPERM);
 
 	if (!zpl_inode_owner_or_capable(zfs_init_idmap, ip))
 		return (-EACCES);
 
 	xva_init(xva);
 	xoap = xva_getxoptattr(xva);
 
 #define	FLAG_CHANGE(iflag, xflag, xfield)	do {	\
 	if (((ioctl_flags & (iflag)) && !(zfs_flags & (iflag))) ||	\
 	    ((zfs_flags & (iflag)) && !(ioctl_flags & (iflag)))) {	\
 		XVA_SET_REQ(xva, (xflag));	\
 		(xfield) = ((ioctl_flags & (iflag)) != 0);	\
 	}	\
 } while (0)
 
 	FLAG_CHANGE(ZFS_IMMUTABLE, XAT_IMMUTABLE, xoap->xoa_immutable);
 	FLAG_CHANGE(ZFS_APPENDONLY, XAT_APPENDONLY, xoap->xoa_appendonly);
 	FLAG_CHANGE(ZFS_NODUMP, XAT_NODUMP, xoap->xoa_nodump);
 	FLAG_CHANGE(ZFS_READONLY, XAT_READONLY, xoap->xoa_readonly);
 	FLAG_CHANGE(ZFS_HIDDEN, XAT_HIDDEN, xoap->xoa_hidden);
 	FLAG_CHANGE(ZFS_SYSTEM, XAT_SYSTEM, xoap->xoa_system);
 	FLAG_CHANGE(ZFS_ARCHIVE, XAT_ARCHIVE, xoap->xoa_archive);
 	FLAG_CHANGE(ZFS_NOUNLINK, XAT_NOUNLINK, xoap->xoa_nounlink);
 	FLAG_CHANGE(ZFS_REPARSE, XAT_REPARSE, xoap->xoa_reparse);
 	FLAG_CHANGE(ZFS_OFFLINE, XAT_OFFLINE, xoap->xoa_offline);
 	FLAG_CHANGE(ZFS_SPARSE, XAT_SPARSE, xoap->xoa_sparse);
 
 #undef	FLAG_CHANGE
 
 	return (0);
 }
 
 /*
  * Set Additional File Level Attributes of ZFS.
  */
 static int
 zpl_ioctl_setdosflags(struct file *filp, void __user *arg)
 {
 	struct inode *ip = file_inode(filp);
 	uint64_t dosflags;
 	cred_t *cr = CRED();
 	xvattr_t xva;
 	int err;
 	fstrans_cookie_t cookie;
 
 	if (copy_from_user(&dosflags, arg, sizeof (dosflags)))
 		return (-EFAULT);
 
 	err = __zpl_ioctl_setdosflags(ip, dosflags, &xva);
 	if (err)
 		return (err);
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	err = -zfs_setattr(ITOZ(ip), (vattr_t *)&xva, 0, cr, zfs_init_idmap);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 
 	return (err);
 }
 
 static long
 zpl_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 {
 	switch (cmd) {
 	case FS_IOC_GETVERSION:
 		return (zpl_ioctl_getversion(filp, (void *)arg));
 	case FS_IOC_GETFLAGS:
 		return (zpl_ioctl_getflags(filp, (void *)arg));
 	case FS_IOC_SETFLAGS:
 		return (zpl_ioctl_setflags(filp, (void *)arg));
 	case ZFS_IOC_FSGETXATTR:
 		return (zpl_ioctl_getxattr(filp, (void *)arg));
 	case ZFS_IOC_FSSETXATTR:
 		return (zpl_ioctl_setxattr(filp, (void *)arg));
 	case ZFS_IOC_GETDOSFLAGS:
 		return (zpl_ioctl_getdosflags(filp, (void *)arg));
 	case ZFS_IOC_SETDOSFLAGS:
 		return (zpl_ioctl_setdosflags(filp, (void *)arg));
 	case ZFS_IOC_COMPAT_FICLONE:
 		return (zpl_ioctl_ficlone(filp, (void *)arg));
 	case ZFS_IOC_COMPAT_FICLONERANGE:
 		return (zpl_ioctl_ficlonerange(filp, (void *)arg));
 	case ZFS_IOC_COMPAT_FIDEDUPERANGE:
 		return (zpl_ioctl_fideduperange(filp, (void *)arg));
 	default:
 		return (-ENOTTY);
 	}
 }
 
 #ifdef CONFIG_COMPAT
 static long
 zpl_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 {
 	switch (cmd) {
 	case FS_IOC32_GETVERSION:
 		cmd = FS_IOC_GETVERSION;
 		break;
 	case FS_IOC32_GETFLAGS:
 		cmd = FS_IOC_GETFLAGS;
 		break;
 	case FS_IOC32_SETFLAGS:
 		cmd = FS_IOC_SETFLAGS;
 		break;
 	default:
 		return (-ENOTTY);
 	}
 	return (zpl_ioctl(filp, cmd, (unsigned long)compat_ptr(arg)));
 }
 #endif /* CONFIG_COMPAT */
 
 const struct address_space_operations zpl_address_space_operations = {
 #ifdef HAVE_VFS_READPAGES
 	.readpages	= zpl_readpages,
 #else
 	.readahead	= zpl_readahead,
 #endif
 #ifdef HAVE_VFS_READ_FOLIO
 	.read_folio	= zpl_read_folio,
 #else
 	.readpage	= zpl_readpage,
 #endif
 	.writepage	= zpl_writepage,
 	.writepages	= zpl_writepages,
 	.direct_IO	= zpl_direct_IO,
 #ifdef HAVE_VFS_SET_PAGE_DIRTY_NOBUFFERS
 	.set_page_dirty = __set_page_dirty_nobuffers,
 #endif
 #ifdef HAVE_VFS_FILEMAP_DIRTY_FOLIO
 	.dirty_folio	= filemap_dirty_folio,
 #endif
+#ifdef HAVE_VFS_MIGRATE_FOLIO
+	.migrate_folio	= migrate_folio,
+#elif defined(HAVE_VFS_MIGRATEPAGE)
+	.migratepage	= migrate_page,
+#endif
 };
 
 const struct file_operations zpl_file_operations = {
 	.open		= zpl_open,
 	.release	= zpl_release,
 	.llseek		= zpl_llseek,
 	.read_iter	= zpl_iter_read,
 	.write_iter	= zpl_iter_write,
 #ifdef HAVE_VFS_IOV_ITER
 #ifdef HAVE_COPY_SPLICE_READ
 	.splice_read	= copy_splice_read,
 #else
 	.splice_read	= generic_file_splice_read,
 #endif
 	.splice_write	= iter_file_splice_write,
 #endif
 	.mmap		= zpl_mmap,
 	.fsync		= zpl_fsync,
 	.fallocate	= zpl_fallocate,
 	.copy_file_range	= zpl_copy_file_range,
 #ifdef HAVE_VFS_CLONE_FILE_RANGE
 	.clone_file_range	= zpl_clone_file_range,
 #endif
 #ifdef HAVE_VFS_REMAP_FILE_RANGE
 	.remap_file_range	= zpl_remap_file_range,
 #endif
 #ifdef HAVE_VFS_DEDUPE_FILE_RANGE
 	.dedupe_file_range	= zpl_dedupe_file_range,
 #endif
 	.fadvise	= zpl_fadvise,
 	.unlocked_ioctl	= zpl_ioctl,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl	= zpl_compat_ioctl,
 #endif
 };
 
 const struct file_operations zpl_dir_file_operations = {
 	.llseek		= generic_file_llseek,
 	.read		= generic_read_dir,
 	.iterate_shared	= zpl_iterate,
 	.fsync		= zpl_fsync,
 	.unlocked_ioctl = zpl_ioctl,
 #ifdef CONFIG_COMPAT
 	.compat_ioctl   = zpl_compat_ioctl,
 #endif
 };
 
 /* CSTYLED */
 module_param(zfs_fallocate_reserve_percent, uint, 0644);
 MODULE_PARM_DESC(zfs_fallocate_reserve_percent,
 	"Percentage of length to use for the available capacity check");
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c
index 8386fc2ae0ce..56ef3a7d1212 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_inode.c
@@ -1,797 +1,808 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2011, Lawrence Livermore National Security, LLC.
  * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
  */
 
 
 #include <sys/sysmacros.h>
 #include <sys/zfs_ctldir.h>
 #include <sys/zfs_vfsops.h>
 #include <sys/zfs_vnops.h>
 #include <sys/zfs_znode.h>
 #include <sys/dmu_objset.h>
 #include <sys/vfs.h>
 #include <sys/zpl.h>
 #include <sys/file.h>
 
 static struct dentry *
 zpl_lookup(struct inode *dir, struct dentry *dentry, unsigned int flags)
 {
 	cred_t *cr = CRED();
 	struct inode *ip;
 	znode_t *zp;
 	int error;
 	fstrans_cookie_t cookie;
 	pathname_t *ppn = NULL;
 	pathname_t pn;
 	int zfs_flags = 0;
 	zfsvfs_t *zfsvfs = dentry->d_sb->s_fs_info;
 
 	if (dlen(dentry) >= ZAP_MAXNAMELEN)
 		return (ERR_PTR(-ENAMETOOLONG));
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 
 	/* If we are a case insensitive fs, we need the real name */
 	if (zfsvfs->z_case == ZFS_CASE_INSENSITIVE) {
 		zfs_flags = FIGNORECASE;
 		pn_alloc(&pn);
 		ppn = &pn;
 	}
 
 	error = -zfs_lookup(ITOZ(dir), dname(dentry), &zp,
 	    zfs_flags, cr, NULL, ppn);
 	spl_fstrans_unmark(cookie);
 	ASSERT3S(error, <=, 0);
 	crfree(cr);
 
 	spin_lock(&dentry->d_lock);
 	dentry->d_time = jiffies;
 	spin_unlock(&dentry->d_lock);
 
 	if (error) {
 		/*
 		 * If we have a case sensitive fs, we do not want to
 		 * insert negative entries, so return NULL for ENOENT.
 		 * Fall through if the error is not ENOENT. Also free memory.
 		 */
 		if (ppn) {
 			pn_free(ppn);
 			if (error == -ENOENT)
 				return (NULL);
 		}
 
 		if (error == -ENOENT)
 			return (d_splice_alias(NULL, dentry));
 		else
 			return (ERR_PTR(error));
 	}
 	ip = ZTOI(zp);
 
 	/*
 	 * If we are case insensitive, call the correct function
 	 * to install the name.
 	 */
 	if (ppn) {
 		struct dentry *new_dentry;
 		struct qstr ci_name;
 
 		if (strcmp(dname(dentry), pn.pn_buf) == 0) {
 			new_dentry = d_splice_alias(ip,  dentry);
 		} else {
 			ci_name.name = pn.pn_buf;
 			ci_name.len = strlen(pn.pn_buf);
 			new_dentry = d_add_ci(dentry, ip, &ci_name);
 		}
 		pn_free(ppn);
 		return (new_dentry);
 	} else {
 		return (d_splice_alias(ip, dentry));
 	}
 }
 
 void
 zpl_vap_init(vattr_t *vap, struct inode *dir, umode_t mode, cred_t *cr,
     zidmap_t *mnt_ns)
 {
 	vap->va_mask = ATTR_MODE;
 	vap->va_mode = mode;
 
 	vap->va_uid = zfs_vfsuid_to_uid(mnt_ns,
 	    zfs_i_user_ns(dir), crgetuid(cr));
 
 	if (dir->i_mode & S_ISGID) {
 		vap->va_gid = KGID_TO_SGID(dir->i_gid);
 		if (S_ISDIR(mode))
 			vap->va_mode |= S_ISGID;
 	} else {
 		vap->va_gid = zfs_vfsgid_to_gid(mnt_ns,
 		    zfs_i_user_ns(dir), crgetgid(cr));
 	}
 }
 
 static int
 #ifdef HAVE_IOPS_CREATE_USERNS
 zpl_create(struct user_namespace *user_ns, struct inode *dir,
     struct dentry *dentry, umode_t mode, bool flag)
 #elif defined(HAVE_IOPS_CREATE_IDMAP)
 zpl_create(struct mnt_idmap *user_ns, struct inode *dir,
     struct dentry *dentry, umode_t mode, bool flag)
 #else
 zpl_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool flag)
 #endif
 {
 	cred_t *cr = CRED();
 	znode_t *zp;
 	vattr_t *vap;
 	int error;
 	fstrans_cookie_t cookie;
 #if !(defined(HAVE_IOPS_CREATE_USERNS) || defined(HAVE_IOPS_CREATE_IDMAP))
 	zidmap_t *user_ns = kcred->user_ns;
 #endif
 
 	crhold(cr);
 	vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
 	zpl_vap_init(vap, dir, mode, cr, user_ns);
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_create(ITOZ(dir), dname(dentry), vap, 0,
 	    mode, &zp, cr, 0, NULL, user_ns);
 	if (error == 0) {
 		error = zpl_xattr_security_init(ZTOI(zp), dir, &dentry->d_name);
 		if (error == 0)
 			error = zpl_init_acl(ZTOI(zp), dir);
 
 		if (error) {
 			(void) zfs_remove(ITOZ(dir), dname(dentry), cr, 0);
 			remove_inode_hash(ZTOI(zp));
 			iput(ZTOI(zp));
 		} else {
 			d_instantiate(dentry, ZTOI(zp));
 		}
 	}
 
 	spl_fstrans_unmark(cookie);
 	kmem_free(vap, sizeof (vattr_t));
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 #ifdef HAVE_IOPS_MKNOD_USERNS
 zpl_mknod(struct user_namespace *user_ns, struct inode *dir,
     struct dentry *dentry, umode_t mode,
 #elif defined(HAVE_IOPS_MKNOD_IDMAP)
 zpl_mknod(struct mnt_idmap *user_ns, struct inode *dir,
     struct dentry *dentry, umode_t mode,
 #else
 zpl_mknod(struct inode *dir, struct dentry *dentry, umode_t mode,
 #endif
     dev_t rdev)
 {
 	cred_t *cr = CRED();
 	znode_t *zp;
 	vattr_t *vap;
 	int error;
 	fstrans_cookie_t cookie;
 #if !(defined(HAVE_IOPS_MKNOD_USERNS) || defined(HAVE_IOPS_MKNOD_IDMAP))
 	zidmap_t *user_ns = kcred->user_ns;
 #endif
 
 	/*
 	 * We currently expect Linux to supply rdev=0 for all sockets
 	 * and fifos, but we want to know if this behavior ever changes.
 	 */
 	if (S_ISSOCK(mode) || S_ISFIFO(mode))
 		ASSERT(rdev == 0);
 
 	crhold(cr);
 	vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
 	zpl_vap_init(vap, dir, mode, cr, user_ns);
 	vap->va_rdev = rdev;
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_create(ITOZ(dir), dname(dentry), vap, 0,
 	    mode, &zp, cr, 0, NULL, user_ns);
 	if (error == 0) {
 		error = zpl_xattr_security_init(ZTOI(zp), dir, &dentry->d_name);
 		if (error == 0)
 			error = zpl_init_acl(ZTOI(zp), dir);
 
 		if (error) {
 			(void) zfs_remove(ITOZ(dir), dname(dentry), cr, 0);
 			remove_inode_hash(ZTOI(zp));
 			iput(ZTOI(zp));
 		} else {
 			d_instantiate(dentry, ZTOI(zp));
 		}
 	}
 
 	spl_fstrans_unmark(cookie);
 	kmem_free(vap, sizeof (vattr_t));
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 #ifdef HAVE_TMPFILE_IDMAP
 zpl_tmpfile(struct mnt_idmap *userns, struct inode *dir,
     struct file *file, umode_t mode)
 #elif !defined(HAVE_TMPFILE_DENTRY)
 zpl_tmpfile(struct user_namespace *userns, struct inode *dir,
     struct file *file, umode_t mode)
 #else
 #ifdef HAVE_TMPFILE_USERNS
 zpl_tmpfile(struct user_namespace *userns, struct inode *dir,
     struct dentry *dentry, umode_t mode)
 #else
 zpl_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
 #endif
 #endif
 {
 	cred_t *cr = CRED();
 	struct inode *ip;
 	vattr_t *vap;
 	int error;
 	fstrans_cookie_t cookie;
 #if !(defined(HAVE_TMPFILE_USERNS) || defined(HAVE_TMPFILE_IDMAP))
 	zidmap_t *userns = kcred->user_ns;
 #endif
 
 	crhold(cr);
 	vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
 	/*
 	 * The VFS does not apply the umask, therefore it is applied here
 	 * when POSIX ACLs are not enabled.
 	 */
 	if (!IS_POSIXACL(dir))
 		mode &= ~current_umask();
 	zpl_vap_init(vap, dir, mode, cr, userns);
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_tmpfile(dir, vap, 0, mode, &ip, cr, 0, NULL, userns);
 	if (error == 0) {
 		/* d_tmpfile will do drop_nlink, so we should set it first */
 		set_nlink(ip, 1);
 #ifndef HAVE_TMPFILE_DENTRY
 		d_tmpfile(file, ip);
 
 		error = zpl_xattr_security_init(ip, dir,
 		    &file->f_path.dentry->d_name);
 #else
 		d_tmpfile(dentry, ip);
 
 		error = zpl_xattr_security_init(ip, dir, &dentry->d_name);
 #endif
 		if (error == 0)
 			error = zpl_init_acl(ip, dir);
 #ifndef HAVE_TMPFILE_DENTRY
 		error = finish_open_simple(file, error);
 #endif
 		/*
 		 * don't need to handle error here, file is already in
 		 * unlinked set.
 		 */
 	}
 
 	spl_fstrans_unmark(cookie);
 	kmem_free(vap, sizeof (vattr_t));
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 zpl_unlink(struct inode *dir, struct dentry *dentry)
 {
 	cred_t *cr = CRED();
 	int error;
 	fstrans_cookie_t cookie;
 	zfsvfs_t *zfsvfs = dentry->d_sb->s_fs_info;
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	error = -zfs_remove(ITOZ(dir), dname(dentry), cr, 0);
 
 	/*
 	 * For a CI FS we must invalidate the dentry to prevent the
 	 * creation of negative entries.
 	 */
 	if (error == 0 && zfsvfs->z_case == ZFS_CASE_INSENSITIVE)
 		d_invalidate(dentry);
 
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
+#if defined(HAVE_IOPS_MKDIR_USERNS)
 static int
-#ifdef HAVE_IOPS_MKDIR_USERNS
 zpl_mkdir(struct user_namespace *user_ns, struct inode *dir,
     struct dentry *dentry, umode_t mode)
 #elif defined(HAVE_IOPS_MKDIR_IDMAP)
+static int
+zpl_mkdir(struct mnt_idmap *user_ns, struct inode *dir,
+    struct dentry *dentry, umode_t mode)
+#elif defined(HAVE_IOPS_MKDIR_DENTRY)
+static struct dentry *
 zpl_mkdir(struct mnt_idmap *user_ns, struct inode *dir,
     struct dentry *dentry, umode_t mode)
 #else
+static int
 zpl_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
 #endif
 {
 	cred_t *cr = CRED();
 	vattr_t *vap;
 	znode_t *zp;
 	int error;
 	fstrans_cookie_t cookie;
-#if !(defined(HAVE_IOPS_MKDIR_USERNS) || defined(HAVE_IOPS_MKDIR_IDMAP))
+#if !(defined(HAVE_IOPS_MKDIR_USERNS) || \
+	defined(HAVE_IOPS_MKDIR_IDMAP) || defined(HAVE_IOPS_MKDIR_DENTRY))
 	zidmap_t *user_ns = kcred->user_ns;
 #endif
 
 	crhold(cr);
 	vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
 	zpl_vap_init(vap, dir, mode | S_IFDIR, cr, user_ns);
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_mkdir(ITOZ(dir), dname(dentry), vap, &zp, cr, 0, NULL,
 	    user_ns);
 	if (error == 0) {
 		error = zpl_xattr_security_init(ZTOI(zp), dir, &dentry->d_name);
 		if (error == 0)
 			error = zpl_init_acl(ZTOI(zp), dir);
 
 		if (error) {
 			(void) zfs_rmdir(ITOZ(dir), dname(dentry), NULL, cr, 0);
 			remove_inode_hash(ZTOI(zp));
 			iput(ZTOI(zp));
 		} else {
 			d_instantiate(dentry, ZTOI(zp));
 		}
 	}
 
 	spl_fstrans_unmark(cookie);
 	kmem_free(vap, sizeof (vattr_t));
 	crfree(cr);
-	ASSERT3S(error, <=, 0);
 
+	ASSERT3S(error, <=, 0);
+#if defined(HAVE_IOPS_MKDIR_DENTRY)
+	return (error != 0 ? ERR_PTR(error) : NULL);
+#else
 	return (error);
+#endif
 }
 
 static int
 zpl_rmdir(struct inode *dir, struct dentry *dentry)
 {
 	cred_t *cr = CRED();
 	int error;
 	fstrans_cookie_t cookie;
 	zfsvfs_t *zfsvfs = dentry->d_sb->s_fs_info;
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	error = -zfs_rmdir(ITOZ(dir), dname(dentry), NULL, cr, 0);
 
 	/*
 	 * For a CI FS we must invalidate the dentry to prevent the
 	 * creation of negative entries.
 	 */
 	if (error == 0 && zfsvfs->z_case == ZFS_CASE_INSENSITIVE)
 		d_invalidate(dentry);
 
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 #ifdef HAVE_USERNS_IOPS_GETATTR
 zpl_getattr_impl(struct user_namespace *user_ns,
     const struct path *path, struct kstat *stat, u32 request_mask,
     unsigned int query_flags)
 #elif defined(HAVE_IDMAP_IOPS_GETATTR)
 zpl_getattr_impl(struct mnt_idmap *user_ns,
     const struct path *path, struct kstat *stat, u32 request_mask,
     unsigned int query_flags)
 #else
 zpl_getattr_impl(const struct path *path, struct kstat *stat, u32 request_mask,
     unsigned int query_flags)
 #endif
 {
 	int error;
 	fstrans_cookie_t cookie;
 	struct inode *ip = path->dentry->d_inode;
 	znode_t *zp __maybe_unused = ITOZ(ip);
 
 	cookie = spl_fstrans_mark();
 
 	/*
 	 * XXX query_flags currently ignored.
 	 */
 
 #ifdef HAVE_GENERIC_FILLATTR_IDMAP_REQMASK
 	error = -zfs_getattr_fast(user_ns, request_mask, ip, stat);
 #elif (defined(HAVE_USERNS_IOPS_GETATTR) || defined(HAVE_IDMAP_IOPS_GETATTR))
 	error = -zfs_getattr_fast(user_ns, ip, stat);
 #else
 	error = -zfs_getattr_fast(kcred->user_ns, ip, stat);
 #endif
 
 #ifdef STATX_BTIME
 	if (request_mask & STATX_BTIME) {
 		stat->btime = zp->z_btime;
 		stat->result_mask |= STATX_BTIME;
 	}
 #endif
 
 #ifdef STATX_ATTR_IMMUTABLE
 	if (zp->z_pflags & ZFS_IMMUTABLE)
 		stat->attributes |= STATX_ATTR_IMMUTABLE;
 	stat->attributes_mask |= STATX_ATTR_IMMUTABLE;
 #endif
 
 #ifdef STATX_ATTR_APPEND
 	if (zp->z_pflags & ZFS_APPENDONLY)
 		stat->attributes |= STATX_ATTR_APPEND;
 	stat->attributes_mask |= STATX_ATTR_APPEND;
 #endif
 
 #ifdef STATX_ATTR_NODUMP
 	if (zp->z_pflags & ZFS_NODUMP)
 		stat->attributes |= STATX_ATTR_NODUMP;
 	stat->attributes_mask |= STATX_ATTR_NODUMP;
 #endif
 
 	spl_fstrans_unmark(cookie);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 ZPL_GETATTR_WRAPPER(zpl_getattr);
 
 static int
 #ifdef HAVE_USERNS_IOPS_SETATTR
 zpl_setattr(struct user_namespace *user_ns, struct dentry *dentry,
     struct iattr *ia)
 #elif defined(HAVE_IDMAP_IOPS_SETATTR)
 zpl_setattr(struct mnt_idmap *user_ns, struct dentry *dentry,
     struct iattr *ia)
 #else
 zpl_setattr(struct dentry *dentry, struct iattr *ia)
 #endif
 {
 	struct inode *ip = dentry->d_inode;
 	cred_t *cr = CRED();
 	vattr_t *vap;
 	int error;
 	fstrans_cookie_t cookie;
 
 #ifdef HAVE_SETATTR_PREPARE_USERNS
 	error = zpl_setattr_prepare(user_ns, dentry, ia);
 #elif defined(HAVE_SETATTR_PREPARE_IDMAP)
 	error = zpl_setattr_prepare(user_ns, dentry, ia);
 #else
 	error = zpl_setattr_prepare(zfs_init_idmap, dentry, ia);
 #endif
 	if (error)
 		return (error);
 
 	crhold(cr);
 	vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
 	vap->va_mask = ia->ia_valid & ATTR_IATTR_MASK;
 	vap->va_mode = ia->ia_mode;
 	if (ia->ia_valid & ATTR_UID)
 #ifdef HAVE_IATTR_VFSID
 		vap->va_uid = zfs_vfsuid_to_uid(user_ns, zfs_i_user_ns(ip),
 		    __vfsuid_val(ia->ia_vfsuid));
 #else
 		vap->va_uid = KUID_TO_SUID(ia->ia_uid);
 #endif
 	if (ia->ia_valid & ATTR_GID)
 #ifdef HAVE_IATTR_VFSID
 		vap->va_gid = zfs_vfsgid_to_gid(user_ns, zfs_i_user_ns(ip),
 		    __vfsgid_val(ia->ia_vfsgid));
 #else
 		vap->va_gid = KGID_TO_SGID(ia->ia_gid);
 #endif
 	vap->va_size = ia->ia_size;
 	vap->va_atime = ia->ia_atime;
 	vap->va_mtime = ia->ia_mtime;
 	vap->va_ctime = ia->ia_ctime;
 
 	if (vap->va_mask & ATTR_ATIME)
 		zpl_inode_set_atime_to_ts(ip,
 		    zpl_inode_timestamp_truncate(ia->ia_atime, ip));
 
 	cookie = spl_fstrans_mark();
 #ifdef HAVE_USERNS_IOPS_SETATTR
 	error = -zfs_setattr(ITOZ(ip), vap, 0, cr, user_ns);
 #elif defined(HAVE_IDMAP_IOPS_SETATTR)
 	error = -zfs_setattr(ITOZ(ip), vap, 0, cr, user_ns);
 #else
 	error = -zfs_setattr(ITOZ(ip), vap, 0, cr, zfs_init_idmap);
 #endif
 	if (!error && (ia->ia_valid & ATTR_MODE))
 		error = zpl_chmod_acl(ip);
 
 	spl_fstrans_unmark(cookie);
 	kmem_free(vap, sizeof (vattr_t));
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 #ifdef HAVE_IOPS_RENAME_USERNS
 zpl_rename2(struct user_namespace *user_ns, struct inode *sdip,
     struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry,
     unsigned int rflags)
 #elif defined(HAVE_IOPS_RENAME_IDMAP)
 zpl_rename2(struct mnt_idmap *user_ns, struct inode *sdip,
     struct dentry *sdentry, struct inode *tdip, struct dentry *tdentry,
     unsigned int rflags)
 #else
 zpl_rename2(struct inode *sdip, struct dentry *sdentry,
     struct inode *tdip, struct dentry *tdentry, unsigned int rflags)
 #endif
 {
 	cred_t *cr = CRED();
 	vattr_t *wo_vap = NULL;
 	int error;
 	fstrans_cookie_t cookie;
 #if !(defined(HAVE_IOPS_RENAME_USERNS) || defined(HAVE_IOPS_RENAME_IDMAP))
 	zidmap_t *user_ns = kcred->user_ns;
 #endif
 
 	crhold(cr);
 	if (rflags & RENAME_WHITEOUT) {
 		wo_vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
 		zpl_vap_init(wo_vap, sdip, S_IFCHR, cr, user_ns);
 		wo_vap->va_rdev = makedevice(0, 0);
 	}
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_rename(ITOZ(sdip), dname(sdentry), ITOZ(tdip),
 	    dname(tdentry), cr, 0, rflags, wo_vap, user_ns);
 	spl_fstrans_unmark(cookie);
 	if (wo_vap)
 		kmem_free(wo_vap, sizeof (vattr_t));
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 #if !defined(HAVE_IOPS_RENAME_USERNS) && \
 	!defined(HAVE_RENAME_WANTS_FLAGS) && \
 	!defined(HAVE_IOPS_RENAME_IDMAP)
 static int
 zpl_rename(struct inode *sdip, struct dentry *sdentry,
     struct inode *tdip, struct dentry *tdentry)
 {
 	return (zpl_rename2(sdip, sdentry, tdip, tdentry, 0));
 }
 #endif
 
 static int
 #ifdef HAVE_IOPS_SYMLINK_USERNS
 zpl_symlink(struct user_namespace *user_ns, struct inode *dir,
     struct dentry *dentry, const char *name)
 #elif defined(HAVE_IOPS_SYMLINK_IDMAP)
 zpl_symlink(struct mnt_idmap *user_ns, struct inode *dir,
     struct dentry *dentry, const char *name)
 #else
 zpl_symlink(struct inode *dir, struct dentry *dentry, const char *name)
 #endif
 {
 	cred_t *cr = CRED();
 	vattr_t *vap;
 	znode_t *zp;
 	int error;
 	fstrans_cookie_t cookie;
 #if !(defined(HAVE_IOPS_SYMLINK_USERNS) || defined(HAVE_IOPS_SYMLINK_IDMAP))
 	zidmap_t *user_ns = kcred->user_ns;
 #endif
 
 	crhold(cr);
 	vap = kmem_zalloc(sizeof (vattr_t), KM_SLEEP);
 	zpl_vap_init(vap, dir, S_IFLNK | S_IRWXUGO, cr, user_ns);
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_symlink(ITOZ(dir), dname(dentry), vap,
 	    (char *)name, &zp, cr, 0, user_ns);
 	if (error == 0) {
 		error = zpl_xattr_security_init(ZTOI(zp), dir, &dentry->d_name);
 		if (error) {
 			(void) zfs_remove(ITOZ(dir), dname(dentry), cr, 0);
 			remove_inode_hash(ZTOI(zp));
 			iput(ZTOI(zp));
 		} else {
 			d_instantiate(dentry, ZTOI(zp));
 		}
 	}
 
 	spl_fstrans_unmark(cookie);
 	kmem_free(vap, sizeof (vattr_t));
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static void
 zpl_put_link(void *ptr)
 {
 	kmem_free(ptr, MAXPATHLEN);
 }
 
 static int
 zpl_get_link_common(struct dentry *dentry, struct inode *ip, char **link)
 {
 	fstrans_cookie_t cookie;
 	cred_t *cr = CRED();
 	int error;
 
 	crhold(cr);
 	*link = NULL;
 
 	struct iovec iov;
 	iov.iov_len = MAXPATHLEN;
 	iov.iov_base = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
 
 	zfs_uio_t uio;
 	zfs_uio_iovec_init(&uio, &iov, 1, 0, UIO_SYSSPACE, MAXPATHLEN - 1, 0);
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_readlink(ip, &uio, cr);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 
 	if (error)
 		kmem_free(iov.iov_base, MAXPATHLEN);
 	else
 		*link = iov.iov_base;
 
 	return (error);
 }
 
 static const char *
 zpl_get_link(struct dentry *dentry, struct inode *inode,
     struct delayed_call *done)
 {
 	char *link = NULL;
 	int error;
 
 	if (!dentry)
 		return (ERR_PTR(-ECHILD));
 
 	error = zpl_get_link_common(dentry, inode, &link);
 	if (error)
 		return (ERR_PTR(error));
 
 	set_delayed_call(done, zpl_put_link, link);
 
 	return (link);
 }
 
 static int
 zpl_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
 {
 	cred_t *cr = CRED();
 	struct inode *ip = old_dentry->d_inode;
 	int error;
 	fstrans_cookie_t cookie;
 
 	if (ip->i_nlink >= ZFS_LINK_MAX)
 		return (-EMLINK);
 
 	crhold(cr);
 	zpl_inode_set_ctime_to_ts(ip, current_time(ip));
 	/* Must have an existing ref, so igrab() cannot return NULL */
 	VERIFY3P(igrab(ip), !=, NULL);
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_link(ITOZ(dir), ITOZ(ip), dname(dentry), cr, 0);
 	if (error) {
 		iput(ip);
 		goto out;
 	}
 
 	d_instantiate(dentry, ip);
 out:
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 const struct inode_operations zpl_inode_operations = {
 	.setattr	= zpl_setattr,
 	.getattr	= zpl_getattr,
 	.listxattr	= zpl_xattr_list,
 #if defined(CONFIG_FS_POSIX_ACL)
 	.set_acl	= zpl_set_acl,
 #if defined(HAVE_GET_INODE_ACL)
 	.get_inode_acl	= zpl_get_acl,
 #else
 	.get_acl	= zpl_get_acl,
 #endif /* HAVE_GET_INODE_ACL */
 #endif /* CONFIG_FS_POSIX_ACL */
 };
 
 const struct inode_operations zpl_dir_inode_operations = {
 	.create		= zpl_create,
 	.lookup		= zpl_lookup,
 	.link		= zpl_link,
 	.unlink		= zpl_unlink,
 	.symlink	= zpl_symlink,
 	.mkdir		= zpl_mkdir,
 	.rmdir		= zpl_rmdir,
 	.mknod		= zpl_mknod,
 #if defined(HAVE_RENAME_WANTS_FLAGS) || defined(HAVE_IOPS_RENAME_USERNS)
 	.rename		= zpl_rename2,
 #elif defined(HAVE_IOPS_RENAME_IDMAP)
 	.rename		= zpl_rename2,
 #else
 	.rename		= zpl_rename,
 #endif
 	.tmpfile	= zpl_tmpfile,
 	.setattr	= zpl_setattr,
 	.getattr	= zpl_getattr,
 	.listxattr	= zpl_xattr_list,
 #if defined(CONFIG_FS_POSIX_ACL)
 	.set_acl	= zpl_set_acl,
 #if defined(HAVE_GET_INODE_ACL)
 	.get_inode_acl	= zpl_get_acl,
 #else
 	.get_acl	= zpl_get_acl,
 #endif /* HAVE_GET_INODE_ACL */
 #endif /* CONFIG_FS_POSIX_ACL */
 };
 
 const struct inode_operations zpl_symlink_inode_operations = {
 	.get_link	= zpl_get_link,
 	.setattr	= zpl_setattr,
 	.getattr	= zpl_getattr,
 	.listxattr	= zpl_xattr_list,
 };
 
 const struct inode_operations zpl_special_inode_operations = {
 	.setattr	= zpl_setattr,
 	.getattr	= zpl_getattr,
 	.listxattr	= zpl_xattr_list,
 #if defined(CONFIG_FS_POSIX_ACL)
 	.set_acl	= zpl_set_acl,
 #if defined(HAVE_GET_INODE_ACL)
 	.get_inode_acl	= zpl_get_acl,
 #else
 	.get_acl	= zpl_get_acl,
 #endif /* HAVE_GET_INODE_ACL */
 #endif /* CONFIG_FS_POSIX_ACL */
 };
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zpl_super.c b/sys/contrib/openzfs/module/os/linux/zfs/zpl_super.c
index 19a9ab9c7e3f..4c05b14c0f7e 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zpl_super.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zpl_super.c
@@ -1,411 +1,419 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2011, Lawrence Livermore National Security, LLC.
  * Copyright (c) 2023, Datto Inc. All rights reserved.
  */
 
 
 #include <sys/zfs_znode.h>
 #include <sys/zfs_vfsops.h>
 #include <sys/zfs_vnops.h>
 #include <sys/zfs_ctldir.h>
 #include <sys/zpl.h>
 #include <linux/iversion.h>
 
 
 static struct inode *
 zpl_inode_alloc(struct super_block *sb)
 {
 	struct inode *ip;
 
 	VERIFY3S(zfs_inode_alloc(sb, &ip), ==, 0);
 	inode_set_iversion(ip, 1);
 
 	return (ip);
 }
 
 static void
 zpl_inode_destroy(struct inode *ip)
 {
 	ASSERT(atomic_read(&ip->i_count) == 0);
 	zfs_inode_destroy(ip);
 }
 
 /*
  * Called from __mark_inode_dirty() to reflect that something in the
  * inode has changed.  We use it to ensure the znode system attributes
  * are always strictly update to date with respect to the inode.
  */
 static void
 zpl_dirty_inode(struct inode *ip, int flags)
 {
 	fstrans_cookie_t cookie;
 
 	cookie = spl_fstrans_mark();
 	zfs_dirty_inode(ip, flags);
 	spl_fstrans_unmark(cookie);
 }
 
 /*
  * When ->drop_inode() is called its return value indicates if the
  * inode should be evicted from the inode cache.  If the inode is
  * unhashed and has no links the default policy is to evict it
  * immediately.
  *
  * The ->evict_inode() callback must minimally truncate the inode pages,
  * and call clear_inode().  For 2.6.35 and later kernels this will
  * simply update the inode state, with the sync occurring before the
  * truncate in evict().  For earlier kernels clear_inode() maps to
  * end_writeback() which is responsible for completing all outstanding
  * write back.  In either case, once this is done it is safe to cleanup
  * any remaining inode specific data via zfs_inactive().
  * remaining filesystem specific data.
  */
 static void
 zpl_evict_inode(struct inode *ip)
 {
 	fstrans_cookie_t cookie;
 
 	cookie = spl_fstrans_mark();
 	truncate_setsize(ip, 0);
 	clear_inode(ip);
 	zfs_inactive(ip);
 	spl_fstrans_unmark(cookie);
 }
 
 static void
 zpl_put_super(struct super_block *sb)
 {
 	fstrans_cookie_t cookie;
 	int error;
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_umount(sb);
 	spl_fstrans_unmark(cookie);
 	ASSERT3S(error, <=, 0);
 }
 
 static int
 zpl_sync_fs(struct super_block *sb, int wait)
 {
 	fstrans_cookie_t cookie;
 	cred_t *cr = CRED();
 	int error;
 
 	crhold(cr);
 	cookie = spl_fstrans_mark();
 	error = -zfs_sync(sb, wait, cr);
 	spl_fstrans_unmark(cookie);
 	crfree(cr);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 zpl_statfs(struct dentry *dentry, struct kstatfs *statp)
 {
 	fstrans_cookie_t cookie;
 	int error;
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_statvfs(dentry->d_inode, statp);
 	spl_fstrans_unmark(cookie);
 	ASSERT3S(error, <=, 0);
 
 	/*
 	 * If required by a 32-bit system call, dynamically scale the
 	 * block size up to 16MiB and decrease the block counts.  This
 	 * allows for a maximum size of 64EiB to be reported.  The file
 	 * counts must be artificially capped at 2^32-1.
 	 */
 	if (unlikely(zpl_is_32bit_api())) {
 		while (statp->f_blocks > UINT32_MAX &&
 		    statp->f_bsize < SPA_MAXBLOCKSIZE) {
 			statp->f_frsize <<= 1;
 			statp->f_bsize <<= 1;
 
 			statp->f_blocks >>= 1;
 			statp->f_bfree >>= 1;
 			statp->f_bavail >>= 1;
 		}
 
 		uint64_t usedobjs = statp->f_files - statp->f_ffree;
 		statp->f_ffree = MIN(statp->f_ffree, UINT32_MAX - usedobjs);
 		statp->f_files = statp->f_ffree + usedobjs;
 	}
 
 	return (error);
 }
 
 static int
 zpl_remount_fs(struct super_block *sb, int *flags, char *data)
 {
 	zfs_mnt_t zm = { .mnt_osname = NULL, .mnt_data = data };
 	fstrans_cookie_t cookie;
 	int error;
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_remount(sb, flags, &zm);
 	spl_fstrans_unmark(cookie);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 __zpl_show_devname(struct seq_file *seq, zfsvfs_t *zfsvfs)
 {
 	int error;
 	if ((error = zpl_enter(zfsvfs, FTAG)) != 0)
 		return (error);
 
 	char *fsname = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
 	dmu_objset_name(zfsvfs->z_os, fsname);
 
 	for (int i = 0; fsname[i] != 0; i++) {
 		/*
 		 * Spaces in the dataset name must be converted to their
 		 * octal escape sequence for getmntent(3) to correctly
 		 * parse then fsname portion of /proc/self/mounts.
 		 */
 		if (fsname[i] == ' ') {
 			seq_puts(seq, "\\040");
 		} else {
 			seq_putc(seq, fsname[i]);
 		}
 	}
 
 	kmem_free(fsname, ZFS_MAX_DATASET_NAME_LEN);
 
 	zpl_exit(zfsvfs, FTAG);
 
 	return (0);
 }
 
 static int
 zpl_show_devname(struct seq_file *seq, struct dentry *root)
 {
 	return (__zpl_show_devname(seq, root->d_sb->s_fs_info));
 }
 
 static int
 __zpl_show_options(struct seq_file *seq, zfsvfs_t *zfsvfs)
 {
 	seq_printf(seq, ",%s",
 	    zfsvfs->z_flags & ZSB_XATTR ? "xattr" : "noxattr");
 
 #ifdef CONFIG_FS_POSIX_ACL
 	switch (zfsvfs->z_acl_type) {
 	case ZFS_ACLTYPE_POSIX:
 		seq_puts(seq, ",posixacl");
 		break;
 	default:
 		seq_puts(seq, ",noacl");
 		break;
 	}
 #endif /* CONFIG_FS_POSIX_ACL */
 
 	switch (zfsvfs->z_case) {
 	case ZFS_CASE_SENSITIVE:
 		seq_puts(seq, ",casesensitive");
 		break;
 	case ZFS_CASE_INSENSITIVE:
 		seq_puts(seq, ",caseinsensitive");
 		break;
 	default:
 		seq_puts(seq, ",casemixed");
 		break;
 	}
 
 	return (0);
 }
 
 static int
 zpl_show_options(struct seq_file *seq, struct dentry *root)
 {
 	return (__zpl_show_options(seq, root->d_sb->s_fs_info));
 }
 
 static int
 zpl_fill_super(struct super_block *sb, void *data, int silent)
 {
 	zfs_mnt_t *zm = (zfs_mnt_t *)data;
 	fstrans_cookie_t cookie;
 	int error;
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_domount(sb, zm, silent);
 	spl_fstrans_unmark(cookie);
 	ASSERT3S(error, <=, 0);
 
 	return (error);
 }
 
 static int
 zpl_test_super(struct super_block *s, void *data)
 {
 	zfsvfs_t *zfsvfs = s->s_fs_info;
 	objset_t *os = data;
 	/*
 	 * If the os doesn't match the z_os in the super_block, assume it is
 	 * not a match. Matching would imply a multimount of a dataset. It is
 	 * possible that during a multimount, there is a simultaneous operation
 	 * that changes the z_os, e.g., rollback, where the match will be
 	 * missed, but in that case the user will get an EBUSY.
 	 */
 	return (zfsvfs != NULL && os == zfsvfs->z_os);
 }
 
 static struct super_block *
 zpl_mount_impl(struct file_system_type *fs_type, int flags, zfs_mnt_t *zm)
 {
 	struct super_block *s;
 	objset_t *os;
 	int err;
 
 	err = dmu_objset_hold(zm->mnt_osname, FTAG, &os);
 	if (err)
 		return (ERR_PTR(-err));
 
 	/*
 	 * The dsl pool lock must be released prior to calling sget().
 	 * It is possible sget() may block on the lock in grab_super()
 	 * while deactivate_super() holds that same lock and waits for
 	 * a txg sync.  If the dsl_pool lock is held over sget()
 	 * this can prevent the pool sync and cause a deadlock.
 	 */
 	dsl_dataset_long_hold(dmu_objset_ds(os), FTAG);
 	dsl_pool_rele(dmu_objset_pool(os), FTAG);
 
 	s = sget(fs_type, zpl_test_super, set_anon_super, flags, os);
 
 	/*
 	 * Recheck with the lock held to prevent mounting the wrong dataset
 	 * since z_os can be stale when the teardown lock is held.
 	 *
 	 * We can't do this in zpl_test_super in since it's under spinlock and
 	 * also s_umount lock is not held there so it would race with
 	 * zfs_umount and zfsvfs can be freed.
 	 */
 	if (!IS_ERR(s) && s->s_fs_info != NULL) {
 		zfsvfs_t *zfsvfs = s->s_fs_info;
 		if (zpl_enter(zfsvfs, FTAG) == 0) {
 			if (os != zfsvfs->z_os)
 				err = -SET_ERROR(EBUSY);
 			zpl_exit(zfsvfs, FTAG);
 		} else {
 			err = -SET_ERROR(EBUSY);
 		}
 	}
 	dsl_dataset_long_rele(dmu_objset_ds(os), FTAG);
 	dsl_dataset_rele(dmu_objset_ds(os), FTAG);
 
 	if (IS_ERR(s))
 		return (ERR_CAST(s));
 
 	if (err) {
 		deactivate_locked_super(s);
 		return (ERR_PTR(err));
 	}
 
 	if (s->s_root == NULL) {
 		err = zpl_fill_super(s, zm, flags & SB_SILENT ? 1 : 0);
 		if (err) {
 			deactivate_locked_super(s);
 			return (ERR_PTR(err));
 		}
 		s->s_flags |= SB_ACTIVE;
 	} else if ((flags ^ s->s_flags) & SB_RDONLY) {
 		deactivate_locked_super(s);
 		return (ERR_PTR(-EBUSY));
 	}
 
 	return (s);
 }
 
 static struct dentry *
 zpl_mount(struct file_system_type *fs_type, int flags,
     const char *osname, void *data)
 {
 	zfs_mnt_t zm = { .mnt_osname = osname, .mnt_data = data };
 
 	struct super_block *sb = zpl_mount_impl(fs_type, flags, &zm);
 	if (IS_ERR(sb))
 		return (ERR_CAST(sb));
 
 	return (dget(sb->s_root));
 }
 
 static void
 zpl_kill_sb(struct super_block *sb)
 {
 	zfs_preumount(sb);
 	kill_anon_super(sb);
 }
 
 void
 zpl_prune_sb(uint64_t nr_to_scan, void *arg)
 {
 	struct super_block *sb = (struct super_block *)arg;
 	int objects = 0;
 
 	/*
-	 * deactivate_locked_super calls shrinker_free and only then
-	 * sops->kill_sb cb, resulting in UAF on umount when trying to reach
-	 * for the shrinker functions in zpl_prune_sb of in-umount dataset.
-	 * Increment if s_active is not zero, but don't prune if it is -
-	 * umount could be underway.
+	 * Ensure the superblock is not in the process of being torn down.
 	 */
-	if (atomic_inc_not_zero(&sb->s_active)) {
-		(void) -zfs_prune(sb, nr_to_scan, &objects);
-		atomic_dec(&sb->s_active);
+#ifdef HAVE_SB_DYING
+	if (down_read_trylock(&sb->s_umount)) {
+		if (!(sb->s_flags & SB_DYING) && sb->s_root &&
+		    (sb->s_flags & SB_BORN)) {
+			(void) zfs_prune(sb, nr_to_scan, &objects);
+		}
+		up_read(&sb->s_umount);
 	}
-
+#else
+	if (down_read_trylock(&sb->s_umount)) {
+		if (!hlist_unhashed(&sb->s_instances) &&
+		    sb->s_root && (sb->s_flags & SB_BORN)) {
+			(void) zfs_prune(sb, nr_to_scan, &objects);
+		}
+		up_read(&sb->s_umount);
+	}
+#endif
 }
 
 const struct super_operations zpl_super_operations = {
 	.alloc_inode		= zpl_inode_alloc,
 	.destroy_inode		= zpl_inode_destroy,
 	.dirty_inode		= zpl_dirty_inode,
 	.write_inode		= NULL,
 	.evict_inode		= zpl_evict_inode,
 	.put_super		= zpl_put_super,
 	.sync_fs		= zpl_sync_fs,
 	.statfs			= zpl_statfs,
 	.remount_fs		= zpl_remount_fs,
 	.show_devname		= zpl_show_devname,
 	.show_options		= zpl_show_options,
 	.show_stats		= NULL,
 };
 
 struct file_system_type zpl_fs_type = {
 	.owner			= THIS_MODULE,
 	.name			= ZFS_DRIVER,
 #if defined(HAVE_IDMAP_MNT_API)
 	.fs_flags		= FS_USERNS_MOUNT | FS_ALLOW_IDMAP,
 #else
 	.fs_flags		= FS_USERNS_MOUNT,
 #endif
 	.mount			= zpl_mount,
 	.kill_sb		= zpl_kill_sb,
 };
diff --git a/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c b/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c
index 01f812b8e814..790babd3888e 100644
--- a/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c
+++ b/sys/contrib/openzfs/module/os/linux/zfs/zvol_os.c
@@ -1,1935 +1,1946 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2024, Rob Norris <robn@despairlabs.com>
  * Copyright (c) 2024, Klara, Inc.
  */
 
 #include <sys/dataset_kstats.h>
 #include <sys/dbuf.h>
 #include <sys/dmu_traverse.h>
 #include <sys/dsl_dataset.h>
 #include <sys/dsl_prop.h>
 #include <sys/dsl_dir.h>
 #include <sys/zap.h>
 #include <sys/zfeature.h>
 #include <sys/zil_impl.h>
 #include <sys/dmu_tx.h>
 #include <sys/zio.h>
 #include <sys/zfs_rlock.h>
 #include <sys/spa_impl.h>
 #include <sys/zvol.h>
 #include <sys/zvol_impl.h>
 #include <cityhash.h>
 
 #include <linux/blkdev_compat.h>
 #include <linux/task_io_accounting_ops.h>
 #include <linux/workqueue.h>
 #include <linux/blk-mq.h>
 
 static void zvol_request_impl(zvol_state_t *zv, struct bio *bio,
     struct request *rq, boolean_t force_sync);
 
 static unsigned int zvol_major = ZVOL_MAJOR;
 static unsigned int zvol_request_sync = 0;
 static unsigned int zvol_prefetch_bytes = (128 * 1024);
 static unsigned long zvol_max_discard_blocks = 16384;
 
 /*
  * Switch taskq at multiple of 512 MB offset. This can be set to a lower value
  * to utilize more threads for small files but may affect prefetch hits.
  */
 #define	ZVOL_TASKQ_OFFSET_SHIFT 29
 
 #ifndef HAVE_BLKDEV_GET_ERESTARTSYS
 static unsigned int zvol_open_timeout_ms = 1000;
 #endif
 
 static unsigned int zvol_threads = 0;
 static unsigned int zvol_blk_mq_threads = 0;
 static unsigned int zvol_blk_mq_actual_threads;
 static boolean_t zvol_use_blk_mq = B_FALSE;
 
 /*
  * The maximum number of volblocksize blocks to process per thread.  Typically,
  * write heavy workloads preform better with higher values here, and read
  * heavy workloads preform better with lower values, but that's not a hard
  * and fast rule.  It's basically a knob to tune between "less overhead with
  * less parallelism" and "more overhead, but more parallelism".
  *
  * '8' was chosen as a reasonable, balanced, default based off of sequential
  * read and write tests to a zvol in an NVMe pool (with 16 CPUs).
  */
 static unsigned int zvol_blk_mq_blocks_per_thread = 8;
 
 static unsigned int zvol_num_taskqs = 0;
 
 #ifndef	BLKDEV_DEFAULT_RQ
 /* BLKDEV_MAX_RQ was renamed to BLKDEV_DEFAULT_RQ in the 5.16 kernel */
 #define	BLKDEV_DEFAULT_RQ BLKDEV_MAX_RQ
 #endif
 
 /*
  * Finalize our BIO or request.
  */
 static inline void
 zvol_end_io(struct bio *bio, struct request *rq, int error)
 {
 	if (bio) {
 		bio->bi_status = errno_to_bi_status(-error);
 		bio_endio(bio);
 	} else {
 		blk_mq_end_request(rq, errno_to_bi_status(error));
 	}
 }
 
 static unsigned int zvol_blk_mq_queue_depth = BLKDEV_DEFAULT_RQ;
 static unsigned int zvol_actual_blk_mq_queue_depth;
 
 struct zvol_state_os {
 	struct gendisk		*zvo_disk;	/* generic disk */
 	struct request_queue	*zvo_queue;	/* request queue */
 	dev_t			zvo_dev;	/* device id */
 
 	struct blk_mq_tag_set tag_set;
 
 	/* Set from the global 'zvol_use_blk_mq' at zvol load */
 	boolean_t use_blk_mq;
 };
 
 typedef struct zv_taskq {
 	uint_t tqs_cnt;
 	taskq_t **tqs_taskq;
 } zv_taskq_t;
 static zv_taskq_t zvol_taskqs;
 static struct ida zvol_ida;
 
 typedef struct zv_request_stack {
 	zvol_state_t	*zv;
 	struct bio	*bio;
 	struct request *rq;
 } zv_request_t;
 
 typedef struct zv_work {
 	struct request  *rq;
 	struct work_struct work;
 } zv_work_t;
 
 typedef struct zv_request_task {
 	zv_request_t zvr;
 	taskq_ent_t	ent;
 } zv_request_task_t;
 
 static zv_request_task_t *
 zv_request_task_create(zv_request_t zvr)
 {
 	zv_request_task_t *task;
 	task = kmem_alloc(sizeof (zv_request_task_t), KM_SLEEP);
 	taskq_init_ent(&task->ent);
 	task->zvr = zvr;
 	return (task);
 }
 
 static void
 zv_request_task_free(zv_request_task_t *task)
 {
 	kmem_free(task, sizeof (*task));
 }
 
 /*
  * This is called when a new block multiqueue request comes in.  A request
  * contains one or more BIOs.
  */
 static blk_status_t zvol_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
     const struct blk_mq_queue_data *bd)
 {
 	struct request *rq = bd->rq;
 	zvol_state_t *zv = rq->q->queuedata;
 
 	/* Tell the kernel that we are starting to process this request */
 	blk_mq_start_request(rq);
 
 	if (blk_rq_is_passthrough(rq)) {
 		/* Skip non filesystem request */
 		blk_mq_end_request(rq, BLK_STS_IOERR);
 		return (BLK_STS_IOERR);
 	}
 
 	zvol_request_impl(zv, NULL, rq, 0);
 
 	/* Acknowledge to the kernel that we got this request */
 	return (BLK_STS_OK);
 }
 
 static struct blk_mq_ops zvol_blk_mq_queue_ops = {
 	.queue_rq = zvol_mq_queue_rq,
 };
 
 /* Initialize our blk-mq struct */
 static int zvol_blk_mq_alloc_tag_set(zvol_state_t *zv)
 {
 	struct zvol_state_os *zso = zv->zv_zso;
 
 	memset(&zso->tag_set, 0, sizeof (zso->tag_set));
 
 	/* Initialize tag set. */
 	zso->tag_set.ops = &zvol_blk_mq_queue_ops;
 	zso->tag_set.nr_hw_queues = zvol_blk_mq_actual_threads;
 	zso->tag_set.queue_depth = zvol_actual_blk_mq_queue_depth;
 	zso->tag_set.numa_node = NUMA_NO_NODE;
 	zso->tag_set.cmd_size = 0;
 
 	/*
 	 * We need BLK_MQ_F_BLOCKING here since we do blocking calls in
 	 * zvol_request_impl()
 	 */
-	zso->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;
+	zso->tag_set.flags = BLK_MQ_F_BLOCKING;
+
+#ifdef BLK_MQ_F_SHOULD_MERGE
+	/*
+	 * Linux 6.14 removed BLK_MQ_F_SHOULD_MERGE and made it implicit.
+	 * For older kernels, we set it.
+	 */
+	zso->tag_set.flags |= BLK_MQ_F_SHOULD_MERGE;
+#endif
+
 	zso->tag_set.driver_data = zv;
 
 	return (blk_mq_alloc_tag_set(&zso->tag_set));
 }
 
 /*
  * Given a path, return TRUE if path is a ZVOL.
  */
 boolean_t
 zvol_os_is_zvol(const char *path)
 {
 	dev_t dev = 0;
 
 	if (vdev_lookup_bdev(path, &dev) != 0)
 		return (B_FALSE);
 
 	if (MAJOR(dev) == zvol_major)
 		return (B_TRUE);
 
 	return (B_FALSE);
 }
 
 static void
 zvol_write(zv_request_t *zvr)
 {
 	struct bio *bio = zvr->bio;
 	struct request *rq = zvr->rq;
 	int error = 0;
 	zfs_uio_t uio;
 	zvol_state_t *zv = zvr->zv;
 	struct request_queue *q;
 	struct gendisk *disk;
 	unsigned long start_time = 0;
 	boolean_t acct = B_FALSE;
 
 	ASSERT3P(zv, !=, NULL);
 	ASSERT3U(zv->zv_open_count, >, 0);
 	ASSERT3P(zv->zv_zilog, !=, NULL);
 
 	q = zv->zv_zso->zvo_queue;
 	disk = zv->zv_zso->zvo_disk;
 
 	/* bio marked as FLUSH need to flush before write */
 	if (io_is_flush(bio, rq))
 		zil_commit(zv->zv_zilog, ZVOL_OBJ);
 
 	/* Some requests are just for flush and nothing else. */
 	if (io_size(bio, rq) == 0) {
 		rw_exit(&zv->zv_suspend_lock);
 		zvol_end_io(bio, rq, 0);
 		return;
 	}
 
 	zfs_uio_bvec_init(&uio, bio, rq);
 
 	ssize_t start_resid = uio.uio_resid;
 
 	/*
 	 * With use_blk_mq, accounting is done by blk_mq_start_request()
 	 * and blk_mq_end_request(), so we can skip it here.
 	 */
 	if (bio) {
 		acct = blk_queue_io_stat(q);
 		if (acct) {
 			start_time = blk_generic_start_io_acct(q, disk, WRITE,
 			    bio);
 		}
 	}
 
 	boolean_t sync =
 	    io_is_fua(bio, rq) || zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS;
 
 	zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
 	    uio.uio_loffset, uio.uio_resid, RL_WRITER);
 
 	uint64_t volsize = zv->zv_volsize;
 	while (uio.uio_resid > 0 && uio.uio_loffset < volsize) {
 		uint64_t bytes = MIN(uio.uio_resid, DMU_MAX_ACCESS >> 1);
 		uint64_t off = uio.uio_loffset;
 		dmu_tx_t *tx = dmu_tx_create(zv->zv_objset);
 
 		if (bytes > volsize - off)	/* don't write past the end */
 			bytes = volsize - off;
 
 		dmu_tx_hold_write_by_dnode(tx, zv->zv_dn, off, bytes);
 
 		/* This will only fail for ENOSPC */
 		error = dmu_tx_assign(tx, TXG_WAIT);
 		if (error) {
 			dmu_tx_abort(tx);
 			break;
 		}
 		error = dmu_write_uio_dnode(zv->zv_dn, &uio, bytes, tx);
 		if (error == 0) {
 			zvol_log_write(zv, tx, off, bytes, sync);
 		}
 		dmu_tx_commit(tx);
 
 		if (error)
 			break;
 	}
 	zfs_rangelock_exit(lr);
 
 	int64_t nwritten = start_resid - uio.uio_resid;
 	dataset_kstats_update_write_kstats(&zv->zv_kstat, nwritten);
 	task_io_account_write(nwritten);
 
 	if (sync)
 		zil_commit(zv->zv_zilog, ZVOL_OBJ);
 
 	rw_exit(&zv->zv_suspend_lock);
 
 	if (bio && acct) {
 		blk_generic_end_io_acct(q, disk, WRITE, bio, start_time);
 	}
 
 	zvol_end_io(bio, rq, -error);
 }
 
 static void
 zvol_write_task(void *arg)
 {
 	zv_request_task_t *task = arg;
 	zvol_write(&task->zvr);
 	zv_request_task_free(task);
 }
 
 static void
 zvol_discard(zv_request_t *zvr)
 {
 	struct bio *bio = zvr->bio;
 	struct request *rq = zvr->rq;
 	zvol_state_t *zv = zvr->zv;
 	uint64_t start = io_offset(bio, rq);
 	uint64_t size = io_size(bio, rq);
 	uint64_t end = start + size;
 	boolean_t sync;
 	int error = 0;
 	dmu_tx_t *tx;
 	struct request_queue *q = zv->zv_zso->zvo_queue;
 	struct gendisk *disk = zv->zv_zso->zvo_disk;
 	unsigned long start_time = 0;
 	boolean_t acct = B_FALSE;
 
 	ASSERT3P(zv, !=, NULL);
 	ASSERT3U(zv->zv_open_count, >, 0);
 	ASSERT3P(zv->zv_zilog, !=, NULL);
 
 	if (bio) {
 		acct = blk_queue_io_stat(q);
 		if (acct) {
 			start_time = blk_generic_start_io_acct(q, disk, WRITE,
 			    bio);
 		}
 	}
 
 	sync = io_is_fua(bio, rq) || zv->zv_objset->os_sync == ZFS_SYNC_ALWAYS;
 
 	if (end > zv->zv_volsize) {
 		error = SET_ERROR(EIO);
 		goto unlock;
 	}
 
 	/*
 	 * Align the request to volume block boundaries when a secure erase is
 	 * not required.  This will prevent dnode_free_range() from zeroing out
 	 * the unaligned parts which is slow (read-modify-write) and useless
 	 * since we are not freeing any space by doing so.
 	 */
 	if (!io_is_secure_erase(bio, rq)) {
 		start = P2ROUNDUP(start, zv->zv_volblocksize);
 		end = P2ALIGN_TYPED(end, zv->zv_volblocksize, uint64_t);
 		size = end - start;
 	}
 
 	if (start >= end)
 		goto unlock;
 
 	zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
 	    start, size, RL_WRITER);
 
 	tx = dmu_tx_create(zv->zv_objset);
 	dmu_tx_mark_netfree(tx);
 	error = dmu_tx_assign(tx, TXG_WAIT);
 	if (error != 0) {
 		dmu_tx_abort(tx);
 	} else {
 		zvol_log_truncate(zv, tx, start, size, B_TRUE);
 		dmu_tx_commit(tx);
 		error = dmu_free_long_range(zv->zv_objset,
 		    ZVOL_OBJ, start, size);
 	}
 	zfs_rangelock_exit(lr);
 
 	if (error == 0 && sync)
 		zil_commit(zv->zv_zilog, ZVOL_OBJ);
 
 unlock:
 	rw_exit(&zv->zv_suspend_lock);
 
 	if (bio && acct) {
 		blk_generic_end_io_acct(q, disk, WRITE, bio,
 		    start_time);
 	}
 
 	zvol_end_io(bio, rq, -error);
 }
 
 static void
 zvol_discard_task(void *arg)
 {
 	zv_request_task_t *task = arg;
 	zvol_discard(&task->zvr);
 	zv_request_task_free(task);
 }
 
 static void
 zvol_read(zv_request_t *zvr)
 {
 	struct bio *bio = zvr->bio;
 	struct request *rq = zvr->rq;
 	int error = 0;
 	zfs_uio_t uio;
 	boolean_t acct = B_FALSE;
 	zvol_state_t *zv = zvr->zv;
 	struct request_queue *q;
 	struct gendisk *disk;
 	unsigned long start_time = 0;
 
 	ASSERT3P(zv, !=, NULL);
 	ASSERT3U(zv->zv_open_count, >, 0);
 
 	zfs_uio_bvec_init(&uio, bio, rq);
 
 	q = zv->zv_zso->zvo_queue;
 	disk = zv->zv_zso->zvo_disk;
 
 	ssize_t start_resid = uio.uio_resid;
 
 	/*
 	 * When blk-mq is being used, accounting is done by
 	 * blk_mq_start_request() and blk_mq_end_request().
 	 */
 	if (bio) {
 		acct = blk_queue_io_stat(q);
 		if (acct)
 			start_time = blk_generic_start_io_acct(q, disk, READ,
 			    bio);
 	}
 
 	zfs_locked_range_t *lr = zfs_rangelock_enter(&zv->zv_rangelock,
 	    uio.uio_loffset, uio.uio_resid, RL_READER);
 
 	uint64_t volsize = zv->zv_volsize;
 
 	while (uio.uio_resid > 0 && uio.uio_loffset < volsize) {
 		uint64_t bytes = MIN(uio.uio_resid, DMU_MAX_ACCESS >> 1);
 
 		/* don't read past the end */
 		if (bytes > volsize - uio.uio_loffset)
 			bytes = volsize - uio.uio_loffset;
 
 		error = dmu_read_uio_dnode(zv->zv_dn, &uio, bytes);
 		if (error) {
 			/* convert checksum errors into IO errors */
 			if (error == ECKSUM)
 				error = SET_ERROR(EIO);
 			break;
 		}
 	}
 	zfs_rangelock_exit(lr);
 
 	int64_t nread = start_resid - uio.uio_resid;
 	dataset_kstats_update_read_kstats(&zv->zv_kstat, nread);
 	task_io_account_read(nread);
 
 	rw_exit(&zv->zv_suspend_lock);
 
 	if (bio && acct) {
 		blk_generic_end_io_acct(q, disk, READ, bio, start_time);
 	}
 
 	zvol_end_io(bio, rq, -error);
 }
 
 static void
 zvol_read_task(void *arg)
 {
 	zv_request_task_t *task = arg;
 	zvol_read(&task->zvr);
 	zv_request_task_free(task);
 }
 
 
 /*
  * Process a BIO or request
  *
  * Either 'bio' or 'rq' should be set depending on if we are processing a
  * bio or a request (both should not be set).
  *
  * force_sync:	Set to 0 to defer processing to a background taskq
  *			Set to 1 to process data synchronously
  */
 static void
 zvol_request_impl(zvol_state_t *zv, struct bio *bio, struct request *rq,
     boolean_t force_sync)
 {
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 	uint64_t offset = io_offset(bio, rq);
 	uint64_t size = io_size(bio, rq);
 	int rw = io_data_dir(bio, rq);
 
 	if (unlikely(zv->zv_flags & ZVOL_REMOVING)) {
 		zvol_end_io(bio, rq, -SET_ERROR(ENXIO));
 		goto out;
 	}
 
 	if (zvol_request_sync)
 		force_sync = 1;
 
 	zv_request_t zvr = {
 		.zv = zv,
 		.bio = bio,
 		.rq = rq,
 	};
 
 	if (io_has_data(bio, rq) && offset + size > zv->zv_volsize) {
 		printk(KERN_INFO "%s: bad access: offset=%llu, size=%lu\n",
 		    zv->zv_zso->zvo_disk->disk_name,
 		    (long long unsigned)offset,
 		    (long unsigned)size);
 
 		zvol_end_io(bio, rq, -SET_ERROR(EIO));
 		goto out;
 	}
 
 	zv_request_task_t *task;
 	zv_taskq_t *ztqs = &zvol_taskqs;
 	uint_t blk_mq_hw_queue = 0;
 	uint_t tq_idx;
 	uint_t taskq_hash;
 	if (rq)
 #ifdef HAVE_BLK_MQ_RQ_HCTX
 		blk_mq_hw_queue = rq->mq_hctx->queue_num;
 #else
 		blk_mq_hw_queue =
 		    rq->q->queue_hw_ctx[rq->q->mq_map[rq->cpu]]->queue_num;
 #endif
 	taskq_hash = cityhash4((uintptr_t)zv, offset >> ZVOL_TASKQ_OFFSET_SHIFT,
 	    blk_mq_hw_queue, 0);
 	tq_idx = taskq_hash % ztqs->tqs_cnt;
 
 	if (rw == WRITE) {
 		if (unlikely(zv->zv_flags & ZVOL_RDONLY)) {
 			zvol_end_io(bio, rq, -SET_ERROR(EROFS));
 			goto out;
 		}
 
 		/*
 		 * Prevents the zvol from being suspended, or the ZIL being
 		 * concurrently opened.  Will be released after the i/o
 		 * completes.
 		 */
 		rw_enter(&zv->zv_suspend_lock, RW_READER);
 
 		/*
 		 * Open a ZIL if this is the first time we have written to this
 		 * zvol. We protect zv->zv_zilog with zv_suspend_lock rather
 		 * than zv_state_lock so that we don't need to acquire an
 		 * additional lock in this path.
 		 */
 		if (zv->zv_zilog == NULL) {
 			rw_exit(&zv->zv_suspend_lock);
 			rw_enter(&zv->zv_suspend_lock, RW_WRITER);
 			if (zv->zv_zilog == NULL) {
 				zv->zv_zilog = zil_open(zv->zv_objset,
 				    zvol_get_data, &zv->zv_kstat.dk_zil_sums);
 				zv->zv_flags |= ZVOL_WRITTEN_TO;
 				/* replay / destroy done in zvol_create_minor */
 				VERIFY0((zv->zv_zilog->zl_header->zh_flags &
 				    ZIL_REPLAY_NEEDED));
 			}
 			rw_downgrade(&zv->zv_suspend_lock);
 		}
 
 		/*
 		 * We don't want this thread to be blocked waiting for i/o to
 		 * complete, so we instead wait from a taskq callback. The
 		 * i/o may be a ZIL write (via zil_commit()), or a read of an
 		 * indirect block, or a read of a data block (if this is a
 		 * partial-block write).  We will indicate that the i/o is
 		 * complete by calling END_IO() from the taskq callback.
 		 *
 		 * This design allows the calling thread to continue and
 		 * initiate more concurrent operations by calling
 		 * zvol_request() again. There are typically only a small
 		 * number of threads available to call zvol_request() (e.g.
 		 * one per iSCSI target), so keeping the latency of
 		 * zvol_request() low is important for performance.
 		 *
 		 * The zvol_request_sync module parameter allows this
 		 * behavior to be altered, for performance evaluation
 		 * purposes.  If the callback blocks, setting
 		 * zvol_request_sync=1 will result in much worse performance.
 		 *
 		 * We can have up to zvol_threads concurrent i/o's being
 		 * processed for all zvols on the system.  This is typically
 		 * a vast improvement over the zvol_request_sync=1 behavior
 		 * of one i/o at a time per zvol.  However, an even better
 		 * design would be for zvol_request() to initiate the zio
 		 * directly, and then be notified by the zio_done callback,
 		 * which would call END_IO().  Unfortunately, the DMU/ZIL
 		 * interfaces lack this functionality (they block waiting for
 		 * the i/o to complete).
 		 */
 		if (io_is_discard(bio, rq) || io_is_secure_erase(bio, rq)) {
 			if (force_sync) {
 				zvol_discard(&zvr);
 			} else {
 				task = zv_request_task_create(zvr);
 				taskq_dispatch_ent(ztqs->tqs_taskq[tq_idx],
 				    zvol_discard_task, task, 0, &task->ent);
 			}
 		} else {
 			if (force_sync) {
 				zvol_write(&zvr);
 			} else {
 				task = zv_request_task_create(zvr);
 				taskq_dispatch_ent(ztqs->tqs_taskq[tq_idx],
 				    zvol_write_task, task, 0, &task->ent);
 			}
 		}
 	} else {
 		/*
 		 * The SCST driver, and possibly others, may issue READ I/Os
 		 * with a length of zero bytes.  These empty I/Os contain no
 		 * data and require no additional handling.
 		 */
 		if (size == 0) {
 			zvol_end_io(bio, rq, 0);
 			goto out;
 		}
 
 		rw_enter(&zv->zv_suspend_lock, RW_READER);
 
 		/* See comment in WRITE case above. */
 		if (force_sync) {
 			zvol_read(&zvr);
 		} else {
 			task = zv_request_task_create(zvr);
 			taskq_dispatch_ent(ztqs->tqs_taskq[tq_idx],
 			    zvol_read_task, task, 0, &task->ent);
 		}
 	}
 
 out:
 	spl_fstrans_unmark(cookie);
 }
 
 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
 #ifdef HAVE_BDEV_SUBMIT_BIO_RETURNS_VOID
 static void
 zvol_submit_bio(struct bio *bio)
 #else
 static blk_qc_t
 zvol_submit_bio(struct bio *bio)
 #endif
 #else
 static MAKE_REQUEST_FN_RET
 zvol_request(struct request_queue *q, struct bio *bio)
 #endif
 {
 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
 #if defined(HAVE_BIO_BDEV_DISK)
 	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
 #else
 	struct request_queue *q = bio->bi_disk->queue;
 #endif
 #endif
 	zvol_state_t *zv = q->queuedata;
 
 	zvol_request_impl(zv, bio, NULL, 0);
 #if defined(HAVE_MAKE_REQUEST_FN_RET_QC) || \
 	defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS) && \
 	!defined(HAVE_BDEV_SUBMIT_BIO_RETURNS_VOID)
 	return (BLK_QC_T_NONE);
 #endif
 }
 
 static int
 #ifdef HAVE_BLK_MODE_T
 zvol_open(struct gendisk *disk, blk_mode_t flag)
 #else
 zvol_open(struct block_device *bdev, fmode_t flag)
 #endif
 {
 	zvol_state_t *zv;
 	int error = 0;
 	boolean_t drop_suspend = B_FALSE;
 #ifndef HAVE_BLKDEV_GET_ERESTARTSYS
 	hrtime_t timeout = MSEC2NSEC(zvol_open_timeout_ms);
 	hrtime_t start = gethrtime();
 
 retry:
 #endif
 	rw_enter(&zvol_state_lock, RW_READER);
 	/*
 	 * Obtain a copy of private_data under the zvol_state_lock to make
 	 * sure that either the result of zvol free code path setting
 	 * disk->private_data to NULL is observed, or zvol_os_free()
 	 * is not called on this zv because of the positive zv_open_count.
 	 */
 #ifdef HAVE_BLK_MODE_T
 	zv = disk->private_data;
 #else
 	zv = bdev->bd_disk->private_data;
 #endif
 	if (zv == NULL) {
 		rw_exit(&zvol_state_lock);
 		return (-SET_ERROR(ENXIO));
 	}
 
 	mutex_enter(&zv->zv_state_lock);
 
 	if (unlikely(zv->zv_flags & ZVOL_REMOVING)) {
 		mutex_exit(&zv->zv_state_lock);
 		rw_exit(&zvol_state_lock);
 		return (-SET_ERROR(ENXIO));
 	}
 
 	/*
 	 * Make sure zvol is not suspended during first open
 	 * (hold zv_suspend_lock) and respect proper lock acquisition
 	 * ordering - zv_suspend_lock before zv_state_lock
 	 */
 	if (zv->zv_open_count == 0) {
 		if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) {
 			mutex_exit(&zv->zv_state_lock);
 			rw_enter(&zv->zv_suspend_lock, RW_READER);
 			mutex_enter(&zv->zv_state_lock);
 			/* check to see if zv_suspend_lock is needed */
 			if (zv->zv_open_count != 0) {
 				rw_exit(&zv->zv_suspend_lock);
 			} else {
 				drop_suspend = B_TRUE;
 			}
 		} else {
 			drop_suspend = B_TRUE;
 		}
 	}
 	rw_exit(&zvol_state_lock);
 
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 
 	if (zv->zv_open_count == 0) {
 		boolean_t drop_namespace = B_FALSE;
 
 		ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
 
 		/*
 		 * In all other call paths the spa_namespace_lock is taken
 		 * before the bdev->bd_mutex lock.  However, on open(2)
 		 * the __blkdev_get() function calls fops->open() with the
 		 * bdev->bd_mutex lock held.  This can result in a deadlock
 		 * when zvols from one pool are used as vdevs in another.
 		 *
 		 * To prevent a lock inversion deadlock we preemptively
 		 * take the spa_namespace_lock.  Normally the lock will not
 		 * be contended and this is safe because spa_open_common()
 		 * handles the case where the caller already holds the
 		 * spa_namespace_lock.
 		 *
 		 * When the lock cannot be aquired after multiple retries
 		 * this must be the vdev on zvol deadlock case and we have
 		 * no choice but to return an error.  For 5.12 and older
 		 * kernels returning -ERESTARTSYS will result in the
 		 * bdev->bd_mutex being dropped, then reacquired, and
 		 * fops->open() being called again.  This process can be
 		 * repeated safely until both locks are acquired.  For 5.13
 		 * and newer the -ERESTARTSYS retry logic was removed from
 		 * the kernel so the only option is to return the error for
 		 * the caller to handle it.
 		 */
 		if (!mutex_owned(&spa_namespace_lock)) {
 			if (!mutex_tryenter(&spa_namespace_lock)) {
 				mutex_exit(&zv->zv_state_lock);
 				rw_exit(&zv->zv_suspend_lock);
 
 #ifdef HAVE_BLKDEV_GET_ERESTARTSYS
 				schedule();
 				return (-SET_ERROR(ERESTARTSYS));
 #else
 				if ((gethrtime() - start) > timeout)
 					return (-SET_ERROR(ERESTARTSYS));
 
 				schedule_timeout_interruptible(
 					MSEC_TO_TICK(10));
 				goto retry;
 #endif
 			} else {
 				drop_namespace = B_TRUE;
 			}
 		}
 
 		error = -zvol_first_open(zv, !(blk_mode_is_open_write(flag)));
 
 		if (drop_namespace)
 			mutex_exit(&spa_namespace_lock);
 	}
 
 	if (error == 0) {
 		if ((blk_mode_is_open_write(flag)) &&
 		    (zv->zv_flags & ZVOL_RDONLY)) {
 			if (zv->zv_open_count == 0)
 				zvol_last_close(zv);
 
 			error = -SET_ERROR(EROFS);
 		} else {
 			zv->zv_open_count++;
 		}
 	}
 
 	mutex_exit(&zv->zv_state_lock);
 	if (drop_suspend)
 		rw_exit(&zv->zv_suspend_lock);
 
 	if (error == 0)
 #ifdef HAVE_BLK_MODE_T
 		disk_check_media_change(disk);
 #else
 		zfs_check_media_change(bdev);
 #endif
 
 	return (error);
 }
 
 static void
 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_RELEASE_1ARG
 zvol_release(struct gendisk *disk)
 #else
 zvol_release(struct gendisk *disk, fmode_t unused)
 #endif
 {
 #if !defined(HAVE_BLOCK_DEVICE_OPERATIONS_RELEASE_1ARG)
 	(void) unused;
 #endif
 	zvol_state_t *zv;
 	boolean_t drop_suspend = B_TRUE;
 
 	rw_enter(&zvol_state_lock, RW_READER);
 	zv = disk->private_data;
 
 	mutex_enter(&zv->zv_state_lock);
 	ASSERT3U(zv->zv_open_count, >, 0);
 	/*
 	 * make sure zvol is not suspended during last close
 	 * (hold zv_suspend_lock) and respect proper lock acquisition
 	 * ordering - zv_suspend_lock before zv_state_lock
 	 */
 	if (zv->zv_open_count == 1) {
 		if (!rw_tryenter(&zv->zv_suspend_lock, RW_READER)) {
 			mutex_exit(&zv->zv_state_lock);
 			rw_enter(&zv->zv_suspend_lock, RW_READER);
 			mutex_enter(&zv->zv_state_lock);
 			/* check to see if zv_suspend_lock is needed */
 			if (zv->zv_open_count != 1) {
 				rw_exit(&zv->zv_suspend_lock);
 				drop_suspend = B_FALSE;
 			}
 		}
 	} else {
 		drop_suspend = B_FALSE;
 	}
 	rw_exit(&zvol_state_lock);
 
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 
 	zv->zv_open_count--;
 	if (zv->zv_open_count == 0) {
 		ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
 		zvol_last_close(zv);
 	}
 
 	mutex_exit(&zv->zv_state_lock);
 
 	if (drop_suspend)
 		rw_exit(&zv->zv_suspend_lock);
 }
 
 static int
 zvol_ioctl(struct block_device *bdev, fmode_t mode,
     unsigned int cmd, unsigned long arg)
 {
 	zvol_state_t *zv = bdev->bd_disk->private_data;
 	int error = 0;
 
 	ASSERT3U(zv->zv_open_count, >, 0);
 
 	switch (cmd) {
 	case BLKFLSBUF:
 #ifdef HAVE_FSYNC_BDEV
 		fsync_bdev(bdev);
 #elif defined(HAVE_SYNC_BLOCKDEV)
 		sync_blockdev(bdev);
 #else
 #error "Neither fsync_bdev() nor sync_blockdev() found"
 #endif
 		invalidate_bdev(bdev);
 		rw_enter(&zv->zv_suspend_lock, RW_READER);
 
 		if (!(zv->zv_flags & ZVOL_RDONLY))
 			txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0);
 
 		rw_exit(&zv->zv_suspend_lock);
 		break;
 
 	case BLKZNAME:
 		mutex_enter(&zv->zv_state_lock);
 		error = copy_to_user((void *)arg, zv->zv_name, MAXNAMELEN);
 		mutex_exit(&zv->zv_state_lock);
 		break;
 
 	default:
 		error = -ENOTTY;
 		break;
 	}
 
 	return (SET_ERROR(error));
 }
 
 #ifdef CONFIG_COMPAT
 static int
 zvol_compat_ioctl(struct block_device *bdev, fmode_t mode,
     unsigned cmd, unsigned long arg)
 {
 	return (zvol_ioctl(bdev, mode, cmd, arg));
 }
 #else
 #define	zvol_compat_ioctl	NULL
 #endif
 
 static unsigned int
 zvol_check_events(struct gendisk *disk, unsigned int clearing)
 {
 	unsigned int mask = 0;
 
 	rw_enter(&zvol_state_lock, RW_READER);
 
 	zvol_state_t *zv = disk->private_data;
 	if (zv != NULL) {
 		mutex_enter(&zv->zv_state_lock);
 		mask = zv->zv_changed ? DISK_EVENT_MEDIA_CHANGE : 0;
 		zv->zv_changed = 0;
 		mutex_exit(&zv->zv_state_lock);
 	}
 
 	rw_exit(&zvol_state_lock);
 
 	return (mask);
 }
 
 static int
 zvol_revalidate_disk(struct gendisk *disk)
 {
 	rw_enter(&zvol_state_lock, RW_READER);
 
 	zvol_state_t *zv = disk->private_data;
 	if (zv != NULL) {
 		mutex_enter(&zv->zv_state_lock);
 		set_capacity(zv->zv_zso->zvo_disk,
 		    zv->zv_volsize >> SECTOR_BITS);
 		mutex_exit(&zv->zv_state_lock);
 	}
 
 	rw_exit(&zvol_state_lock);
 
 	return (0);
 }
 
 int
 zvol_os_update_volsize(zvol_state_t *zv, uint64_t volsize)
 {
 	struct gendisk *disk = zv->zv_zso->zvo_disk;
 
 #if defined(HAVE_REVALIDATE_DISK_SIZE)
 	revalidate_disk_size(disk, zvol_revalidate_disk(disk) == 0);
 #elif defined(HAVE_REVALIDATE_DISK)
 	revalidate_disk(disk);
 #else
 	zvol_revalidate_disk(disk);
 #endif
 	return (0);
 }
 
 void
 zvol_os_clear_private(zvol_state_t *zv)
 {
 	/*
 	 * Cleared while holding zvol_state_lock as a writer
 	 * which will prevent zvol_open() from opening it.
 	 */
 	zv->zv_zso->zvo_disk->private_data = NULL;
 }
 
 /*
  * Provide a simple virtual geometry for legacy compatibility.  For devices
  * smaller than 1 MiB a small head and sector count is used to allow very
  * tiny devices.  For devices over 1 Mib a standard head and sector count
  * is used to keep the cylinders count reasonable.
  */
 static int
 zvol_getgeo(struct block_device *bdev, struct hd_geometry *geo)
 {
 	zvol_state_t *zv = bdev->bd_disk->private_data;
 	sector_t sectors;
 
 	ASSERT3U(zv->zv_open_count, >, 0);
 
 	sectors = get_capacity(zv->zv_zso->zvo_disk);
 
 	if (sectors > 2048) {
 		geo->heads = 16;
 		geo->sectors = 63;
 	} else {
 		geo->heads = 2;
 		geo->sectors = 4;
 	}
 
 	geo->start = 0;
 	geo->cylinders = sectors / (geo->heads * geo->sectors);
 
 	return (0);
 }
 
 /*
  * Why have two separate block_device_operations structs?
  *
  * Normally we'd just have one, and assign 'submit_bio' as needed.  However,
  * it's possible the user's kernel is built with CONSTIFY_PLUGIN, meaning we
  * can't just change submit_bio dynamically at runtime.  So just create two
  * separate structs to get around this.
  */
 static const struct block_device_operations zvol_ops_blk_mq = {
 	.open			= zvol_open,
 	.release		= zvol_release,
 	.ioctl			= zvol_ioctl,
 	.compat_ioctl		= zvol_compat_ioctl,
 	.check_events		= zvol_check_events,
 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
 	.revalidate_disk	= zvol_revalidate_disk,
 #endif
 	.getgeo			= zvol_getgeo,
 	.owner			= THIS_MODULE,
 };
 
 static const struct block_device_operations zvol_ops = {
 	.open			= zvol_open,
 	.release		= zvol_release,
 	.ioctl			= zvol_ioctl,
 	.compat_ioctl		= zvol_compat_ioctl,
 	.check_events		= zvol_check_events,
 #ifdef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK
 	.revalidate_disk	= zvol_revalidate_disk,
 #endif
 	.getgeo			= zvol_getgeo,
 	.owner			= THIS_MODULE,
 #ifdef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS
 	.submit_bio		= zvol_submit_bio,
 #endif
 };
 
 /*
  * Since 6.9, Linux has been removing queue limit setters in favour of an
  * initial queue_limits struct applied when the device is open. Since 6.11,
  * queue_limits is being extended to allow more things to be applied when the
  * device is open. Setters are also being removed for this.
  *
  * For OpenZFS, this means that depending on kernel version, some options may
  * be set up before the device is open, and some applied to an open device
  * (queue) after the fact.
  *
  * We manage this complexity by having our own limits struct,
  * zvol_queue_limits_t, in which we carry any queue config that we're
  * interested in setting. This structure is the same on all kernels.
  *
  * These limits are then applied to the queue at device open time by the most
  * appropriate method for the kernel.
  *
  * zvol_queue_limits_convert() is used on 6.9+ (where the two-arg form of
  * blk_alloc_disk() exists). This converts our limits struct to a proper Linux
  * struct queue_limits, and passes it in. Any fields added in later kernels are
  * (obviously) not set up here.
  *
  * zvol_queue_limits_apply() is called on all kernel versions after the queue
  * is created, and applies any remaining config. Before 6.9 that will be
  * everything, via setter methods. After 6.9 that will be whatever couldn't be
  * put into struct queue_limits. (This implies that zvol_queue_limits_apply()
  * will always be a no-op on the latest kernel we support).
  */
 typedef struct zvol_queue_limits {
 	unsigned int	zql_max_hw_sectors;
 	unsigned short	zql_max_segments;
 	unsigned int	zql_max_segment_size;
 	unsigned int	zql_io_opt;
 	unsigned int	zql_physical_block_size;
 	unsigned int	zql_max_discard_sectors;
 	unsigned int	zql_discard_granularity;
 } zvol_queue_limits_t;
 
 static void
 zvol_queue_limits_init(zvol_queue_limits_t *limits, zvol_state_t *zv,
     boolean_t use_blk_mq)
 {
 	limits->zql_max_hw_sectors = (DMU_MAX_ACCESS / 4) >> 9;
 
 	if (use_blk_mq) {
 		/*
 		 * IO requests can be really big (1MB).  When an IO request
 		 * comes in, it is passed off to zvol_read() or zvol_write()
 		 * in a new thread, where it is chunked up into 'volblocksize'
 		 * sized pieces and processed.  So for example, if the request
 		 * is a 1MB write and your volblocksize is 128k, one zvol_write
 		 * thread will take that request and sequentially do ten 128k
 		 * IOs.  This is due to the fact that the thread needs to lock
 		 * each volblocksize sized block.  So you might be wondering:
 		 * "instead of passing the whole 1MB request to one thread,
 		 * why not pass ten individual 128k chunks to ten threads and
 		 * process the whole write in parallel?"  The short answer is
 		 * that there's a sweet spot number of chunks that balances
 		 * the greater parallelism with the added overhead of more
 		 * threads. The sweet spot can be different depending on if you
 		 * have a read or write  heavy workload.  Writes typically want
 		 * high chunk counts while reads typically want lower ones.  On
 		 * a test pool with 6 NVMe drives in a 3x 2-disk mirror
 		 * configuration, with volblocksize=8k, the sweet spot for good
 		 * sequential reads and writes was at 8 chunks.
 		 */
 
 		/*
 		 * Below we tell the kernel how big we want our requests
 		 * to be.  You would think that blk_queue_io_opt() would be
 		 * used to do this since it is used to "set optimal request
 		 * size for the queue", but that doesn't seem to do
 		 * anything - the kernel still gives you huge requests
 		 * with tons of little PAGE_SIZE segments contained within it.
 		 *
 		 * Knowing that the kernel will just give you PAGE_SIZE segments
 		 * no matter what, you can say "ok, I want PAGE_SIZE byte
 		 * segments, and I want 'N' of them per request", where N is
 		 * the correct number of segments for the volblocksize and
 		 * number of chunks you want.
 		 */
 		if (zvol_blk_mq_blocks_per_thread != 0) {
 			unsigned int chunks;
 			chunks = MIN(zvol_blk_mq_blocks_per_thread, UINT16_MAX);
 
 			limits->zql_max_segment_size = PAGE_SIZE;
 			limits->zql_max_segments =
 			    (zv->zv_volblocksize * chunks) / PAGE_SIZE;
 		} else {
 			/*
 			 * Special case: zvol_blk_mq_blocks_per_thread = 0
 			 * Max everything out.
 			 */
 			limits->zql_max_segments = UINT16_MAX;
 			limits->zql_max_segment_size = UINT_MAX;
 		}
 	} else {
 		limits->zql_max_segments = UINT16_MAX;
 		limits->zql_max_segment_size = UINT_MAX;
 	}
 
 	limits->zql_io_opt = zv->zv_volblocksize;
 
 	limits->zql_physical_block_size = zv->zv_volblocksize;
 	limits->zql_max_discard_sectors =
 	    (zvol_max_discard_blocks * zv->zv_volblocksize) >> 9;
 	limits->zql_discard_granularity = zv->zv_volblocksize;
 }
 
 #ifdef HAVE_BLK_ALLOC_DISK_2ARG
 static void
 zvol_queue_limits_convert(zvol_queue_limits_t *limits,
     struct queue_limits *qlimits)
 {
 	memset(qlimits, 0, sizeof (struct queue_limits));
 	qlimits->max_hw_sectors = limits->zql_max_hw_sectors;
 	qlimits->max_segments = limits->zql_max_segments;
 	qlimits->max_segment_size = limits->zql_max_segment_size;
 	qlimits->io_opt = limits->zql_io_opt;
 	qlimits->physical_block_size = limits->zql_physical_block_size;
 	qlimits->max_discard_sectors = limits->zql_max_discard_sectors;
 	qlimits->max_hw_discard_sectors = limits->zql_max_discard_sectors;
 	qlimits->discard_granularity = limits->zql_discard_granularity;
 #ifdef HAVE_BLKDEV_QUEUE_LIMITS_FEATURES
 	qlimits->features =
 	    BLK_FEAT_WRITE_CACHE | BLK_FEAT_FUA | BLK_FEAT_IO_STAT;
 #endif
 }
 #endif
 
 static void
 zvol_queue_limits_apply(zvol_queue_limits_t *limits,
     struct request_queue *queue)
 {
 #ifndef HAVE_BLK_ALLOC_DISK_2ARG
 	blk_queue_max_hw_sectors(queue, limits->zql_max_hw_sectors);
 	blk_queue_max_segments(queue, limits->zql_max_segments);
 	blk_queue_max_segment_size(queue, limits->zql_max_segment_size);
 	blk_queue_io_opt(queue, limits->zql_io_opt);
 	blk_queue_physical_block_size(queue, limits->zql_physical_block_size);
 	blk_queue_max_discard_sectors(queue, limits->zql_max_discard_sectors);
 	blk_queue_discard_granularity(queue, limits->zql_discard_granularity);
 #endif
 #ifndef HAVE_BLKDEV_QUEUE_LIMITS_FEATURES
 	blk_queue_set_write_cache(queue, B_TRUE);
 	blk_queue_flag_set(QUEUE_FLAG_IO_STAT, queue);
 #endif
 }
 
 static int
 zvol_alloc_non_blk_mq(struct zvol_state_os *zso, zvol_queue_limits_t *limits)
 {
 #if defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS)
 #if defined(HAVE_BLK_ALLOC_DISK)
 	zso->zvo_disk = blk_alloc_disk(NUMA_NO_NODE);
 	if (zso->zvo_disk == NULL)
 		return (1);
 
 	zso->zvo_disk->minors = ZVOL_MINORS;
 	zso->zvo_queue = zso->zvo_disk->queue;
 #elif defined(HAVE_BLK_ALLOC_DISK_2ARG)
 	struct queue_limits qlimits;
 	zvol_queue_limits_convert(limits, &qlimits);
 	struct gendisk *disk = blk_alloc_disk(&qlimits, NUMA_NO_NODE);
 	if (IS_ERR(disk)) {
 		zso->zvo_disk = NULL;
 		return (1);
 	}
 
 	zso->zvo_disk = disk;
 	zso->zvo_disk->minors = ZVOL_MINORS;
 	zso->zvo_queue = zso->zvo_disk->queue;
 
 #else
 	zso->zvo_queue = blk_alloc_queue(NUMA_NO_NODE);
 	if (zso->zvo_queue == NULL)
 		return (1);
 
 	zso->zvo_disk = alloc_disk(ZVOL_MINORS);
 	if (zso->zvo_disk == NULL) {
 		blk_cleanup_queue(zso->zvo_queue);
 		return (1);
 	}
 
 	zso->zvo_disk->queue = zso->zvo_queue;
 #endif /* HAVE_BLK_ALLOC_DISK */
 #else
 	zso->zvo_queue = blk_generic_alloc_queue(zvol_request, NUMA_NO_NODE);
 	if (zso->zvo_queue == NULL)
 		return (1);
 
 	zso->zvo_disk = alloc_disk(ZVOL_MINORS);
 	if (zso->zvo_disk == NULL) {
 		blk_cleanup_queue(zso->zvo_queue);
 		return (1);
 	}
 
 	zso->zvo_disk->queue = zso->zvo_queue;
 #endif /* HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
 
 	zvol_queue_limits_apply(limits, zso->zvo_queue);
 
 	return (0);
 
 }
 
 static int
 zvol_alloc_blk_mq(zvol_state_t *zv, zvol_queue_limits_t *limits)
 {
 	struct zvol_state_os *zso = zv->zv_zso;
 
 	/* Allocate our blk-mq tag_set */
 	if (zvol_blk_mq_alloc_tag_set(zv) != 0)
 		return (1);
 
 #if defined(HAVE_BLK_ALLOC_DISK)
 	zso->zvo_disk = blk_mq_alloc_disk(&zso->tag_set, zv);
 	if (zso->zvo_disk == NULL) {
 		blk_mq_free_tag_set(&zso->tag_set);
 		return (1);
 	}
 	zso->zvo_queue = zso->zvo_disk->queue;
 	zso->zvo_disk->minors = ZVOL_MINORS;
 #elif defined(HAVE_BLK_ALLOC_DISK_2ARG)
 	struct queue_limits qlimits;
 	zvol_queue_limits_convert(limits, &qlimits);
 	struct gendisk *disk = blk_mq_alloc_disk(&zso->tag_set, &qlimits, zv);
 	if (IS_ERR(disk)) {
 		zso->zvo_disk = NULL;
 		blk_mq_free_tag_set(&zso->tag_set);
 		return (1);
 	}
 
 	zso->zvo_disk = disk;
 	zso->zvo_queue = zso->zvo_disk->queue;
 	zso->zvo_disk->minors = ZVOL_MINORS;
 #else
 	zso->zvo_disk = alloc_disk(ZVOL_MINORS);
 	if (zso->zvo_disk == NULL) {
 		blk_cleanup_queue(zso->zvo_queue);
 		blk_mq_free_tag_set(&zso->tag_set);
 		return (1);
 	}
 	/* Allocate queue */
 	zso->zvo_queue = blk_mq_init_queue(&zso->tag_set);
 	if (IS_ERR(zso->zvo_queue)) {
 		blk_mq_free_tag_set(&zso->tag_set);
 		return (1);
 	}
 
 	/* Our queue is now created, assign it to our disk */
 	zso->zvo_disk->queue = zso->zvo_queue;
 #endif
 
 	zvol_queue_limits_apply(limits, zso->zvo_queue);
 
 	return (0);
 }
 
 /*
  * Allocate memory for a new zvol_state_t and setup the required
  * request queue and generic disk structures for the block device.
  */
 static zvol_state_t *
 zvol_alloc(dev_t dev, const char *name, uint64_t volblocksize)
 {
 	zvol_state_t *zv;
 	struct zvol_state_os *zso;
 	uint64_t volmode;
 	int ret;
 
 	if (dsl_prop_get_integer(name, "volmode", &volmode, NULL) != 0)
 		return (NULL);
 
 	if (volmode == ZFS_VOLMODE_DEFAULT)
 		volmode = zvol_volmode;
 
 	if (volmode == ZFS_VOLMODE_NONE)
 		return (NULL);
 
 	zv = kmem_zalloc(sizeof (zvol_state_t), KM_SLEEP);
 	zso = kmem_zalloc(sizeof (struct zvol_state_os), KM_SLEEP);
 	zv->zv_zso = zso;
 	zv->zv_volmode = volmode;
 	zv->zv_volblocksize = volblocksize;
 
 	list_link_init(&zv->zv_next);
 	mutex_init(&zv->zv_state_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&zv->zv_removing_cv, NULL, CV_DEFAULT, NULL);
 
 	zv->zv_zso->use_blk_mq = zvol_use_blk_mq;
 
 	zvol_queue_limits_t limits;
 	zvol_queue_limits_init(&limits, zv, zv->zv_zso->use_blk_mq);
 
 	/*
 	 * The block layer has 3 interfaces for getting BIOs:
 	 *
 	 * 1. blk-mq request queues (new)
 	 * 2. submit_bio() (oldest)
 	 * 3. regular request queues (old).
 	 *
 	 * Each of those interfaces has two permutations:
 	 *
 	 * a) We have blk_alloc_disk()/blk_mq_alloc_disk(), which allocates
 	 *    both the disk and its queue (5.14 kernel or newer)
 	 *
 	 * b) We don't have blk_*alloc_disk(), and have to allocate the
 	 *    disk and the queue separately. (5.13 kernel or older)
 	 */
 	if (zv->zv_zso->use_blk_mq) {
 		ret = zvol_alloc_blk_mq(zv, &limits);
+		if (ret != 0)
+			goto out_kmem;
 		zso->zvo_disk->fops = &zvol_ops_blk_mq;
 	} else {
 		ret = zvol_alloc_non_blk_mq(zso, &limits);
+		if (ret != 0)
+			goto out_kmem;
 		zso->zvo_disk->fops = &zvol_ops;
 	}
-	if (ret != 0)
-		goto out_kmem;
 
 	/* Limit read-ahead to a single page to prevent over-prefetching. */
 	blk_queue_set_read_ahead(zso->zvo_queue, 1);
 
 	if (!zv->zv_zso->use_blk_mq) {
 		/* Disable write merging in favor of the ZIO pipeline. */
 		blk_queue_flag_set(QUEUE_FLAG_NOMERGES, zso->zvo_queue);
 	}
 
 	zso->zvo_queue->queuedata = zv;
 	zso->zvo_dev = dev;
 	zv->zv_open_count = 0;
 	strlcpy(zv->zv_name, name, MAXNAMELEN);
 
 	zfs_rangelock_init(&zv->zv_rangelock, NULL, NULL);
 	rw_init(&zv->zv_suspend_lock, NULL, RW_DEFAULT, NULL);
 
 	zso->zvo_disk->major = zvol_major;
 	zso->zvo_disk->events = DISK_EVENT_MEDIA_CHANGE;
 
 	/*
 	 * Setting ZFS_VOLMODE_DEV disables partitioning on ZVOL devices.
 	 * This is accomplished by limiting the number of minors for the
 	 * device to one and explicitly disabling partition scanning.
 	 */
 	if (volmode == ZFS_VOLMODE_DEV) {
 		zso->zvo_disk->minors = 1;
 		zso->zvo_disk->flags &= ~GENHD_FL_EXT_DEVT;
 		zso->zvo_disk->flags |= GENHD_FL_NO_PART;
 	}
 
 	zso->zvo_disk->first_minor = (dev & MINORMASK);
 	zso->zvo_disk->private_data = zv;
 	snprintf(zso->zvo_disk->disk_name, DISK_NAME_LEN, "%s%d",
 	    ZVOL_DEV_NAME, (dev & MINORMASK));
 
 	return (zv);
 
 out_kmem:
 	kmem_free(zso, sizeof (struct zvol_state_os));
 	kmem_free(zv, sizeof (zvol_state_t));
 	return (NULL);
 }
 
 /*
  * Cleanup then free a zvol_state_t which was created by zvol_alloc().
  * At this time, the structure is not opened by anyone, is taken off
  * the zvol_state_list, and has its private data set to NULL.
  * The zvol_state_lock is dropped.
  *
  * This function may take many milliseconds to complete (e.g. we've seen
  * it take over 256ms), due to the calls to "blk_cleanup_queue" and
  * "del_gendisk". Thus, consumers need to be careful to account for this
  * latency when calling this function.
  */
 void
 zvol_os_free(zvol_state_t *zv)
 {
 
 	ASSERT(!RW_LOCK_HELD(&zv->zv_suspend_lock));
 	ASSERT(!MUTEX_HELD(&zv->zv_state_lock));
 	ASSERT0(zv->zv_open_count);
 	ASSERT3P(zv->zv_zso->zvo_disk->private_data, ==, NULL);
 
 	rw_destroy(&zv->zv_suspend_lock);
 	zfs_rangelock_fini(&zv->zv_rangelock);
 
 	del_gendisk(zv->zv_zso->zvo_disk);
 #if defined(HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS) && \
 	(defined(HAVE_BLK_ALLOC_DISK) || defined(HAVE_BLK_ALLOC_DISK_2ARG))
 #if defined(HAVE_BLK_CLEANUP_DISK)
 	blk_cleanup_disk(zv->zv_zso->zvo_disk);
 #else
 	put_disk(zv->zv_zso->zvo_disk);
 #endif
 #else
 	blk_cleanup_queue(zv->zv_zso->zvo_queue);
 	put_disk(zv->zv_zso->zvo_disk);
 #endif
 
 	if (zv->zv_zso->use_blk_mq)
 		blk_mq_free_tag_set(&zv->zv_zso->tag_set);
 
 	ida_simple_remove(&zvol_ida,
 	    MINOR(zv->zv_zso->zvo_dev) >> ZVOL_MINOR_BITS);
 
 	cv_destroy(&zv->zv_removing_cv);
 	mutex_destroy(&zv->zv_state_lock);
 	dataset_kstats_destroy(&zv->zv_kstat);
 
 	kmem_free(zv->zv_zso, sizeof (struct zvol_state_os));
 	kmem_free(zv, sizeof (zvol_state_t));
 }
 
 void
 zvol_wait_close(zvol_state_t *zv)
 {
 }
 
 struct add_disk_work {
 	struct delayed_work work;
 	struct gendisk *disk;
 	int error;
 };
 
 static int
 __zvol_os_add_disk(struct gendisk *disk)
 {
 	int error = 0;
 #ifdef HAVE_ADD_DISK_RET
 	error = add_disk(disk);
 #else
 	add_disk(disk);
 #endif
 	return (error);
 }
 
 #if defined(HAVE_BDEV_FILE_OPEN_BY_PATH)
 static void
 zvol_os_add_disk_work(struct work_struct *work)
 {
 	struct add_disk_work *add_disk_work;
 	add_disk_work = container_of(work, struct add_disk_work, work.work);
 	add_disk_work->error = __zvol_os_add_disk(add_disk_work->disk);
 }
 #endif
 
 /*
  * SPECIAL CASE:
  *
  * This function basically calls add_disk() from a workqueue.   You may be
  * thinking: why not just call add_disk() directly?
  *
  * When you call add_disk(), the zvol appears to the world.  When this happens,
  * the kernel calls disk_scan_partitions() on the zvol, which behaves
  * differently on the 6.9+ kernels:
  *
  * - 6.8 and older kernels -
  * disk_scan_partitions()
  *	handle = bdev_open_by_dev(
  *		zvol_open()
  *	bdev_release(handle);
  *		zvol_release()
  *
  *
  * - 6.9+ kernels -
  * disk_scan_partitions()
  * 	file = bdev_file_open_by_dev()
  *		zvol_open()
  *	fput(file)
  *	< wait for return to userspace >
  *		zvol_release()
  *
  * The difference is that the bdev_release() from the 6.8 kernel is synchronous
  * while the fput() from the 6.9 kernel is async.  Or more specifically it's
  * async that has to wait until we return to userspace (since it adds the fput
  * into the caller's work queue with the TWA_RESUME flag set).  This is not the
  * behavior we want, since we want do things like create+destroy a zvol within
  * a single ZFS_IOC_CREATE ioctl, and the "create" part needs to release the
  * reference to the zvol while we're in the IOCTL, which can't wait until we
  * return to userspace.
  *
  * We can get around this since fput() has a special codepath for when it's
  * running in a kernel thread or interrupt.  In those cases, it just puts the
  * fput into the system workqueue, which we can force to run with
  * __flush_workqueue().  That is why we call add_disk() from a workqueue - so it
  * run from a kernel thread and "tricks" the fput() codepaths.
  *
  * Note that __flush_workqueue() is slowly getting deprecated.  This may be ok
  * though, since our IOCTL will spin on EBUSY waiting for the zvol release (via
  * fput) to happen, which it eventually, naturally, will from the system_wq
  * without us explicitly calling __flush_workqueue().
  */
 static int
 zvol_os_add_disk(struct gendisk *disk)
 {
 #if defined(HAVE_BDEV_FILE_OPEN_BY_PATH)	/* 6.9+ kernel */
 	struct add_disk_work add_disk_work;
 
 	INIT_DELAYED_WORK(&add_disk_work.work, zvol_os_add_disk_work);
 	add_disk_work.disk = disk;
 	add_disk_work.error = 0;
 
 	/* Use *_delayed_work functions since they're not GPL'd */
 	schedule_delayed_work(&add_disk_work.work, 0);
 	flush_delayed_work(&add_disk_work.work);
 
 	__flush_workqueue(system_wq);
 	return (add_disk_work.error);
 #else	/* <= 6.8 kernel */
 	return (__zvol_os_add_disk(disk));
 #endif
 }
 
 /*
  * Create a block device minor node and setup the linkage between it
  * and the specified volume.  Once this function returns the block
  * device is live and ready for use.
  */
 int
 zvol_os_create_minor(const char *name)
 {
 	zvol_state_t *zv;
 	objset_t *os;
 	dmu_object_info_t *doi;
 	uint64_t volsize;
 	uint64_t len;
 	unsigned minor = 0;
 	int error = 0;
 	int idx;
 	uint64_t hash = zvol_name_hash(name);
 	bool replayed_zil = B_FALSE;
 
 	if (zvol_inhibit_dev)
 		return (0);
 
 	idx = ida_simple_get(&zvol_ida, 0, 0, kmem_flags_convert(KM_SLEEP));
 	if (idx < 0)
 		return (SET_ERROR(-idx));
 	minor = idx << ZVOL_MINOR_BITS;
 	if (MINOR(minor) != minor) {
 		/* too many partitions can cause an overflow */
 		zfs_dbgmsg("zvol: create minor overflow: %s, minor %u/%u",
 		    name, minor, MINOR(minor));
 		ida_simple_remove(&zvol_ida, idx);
 		return (SET_ERROR(EINVAL));
 	}
 
 	zv = zvol_find_by_name_hash(name, hash, RW_NONE);
 	if (zv) {
 		ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 		mutex_exit(&zv->zv_state_lock);
 		ida_simple_remove(&zvol_ida, idx);
 		return (SET_ERROR(EEXIST));
 	}
 
 	doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP);
 
 	error = dmu_objset_own(name, DMU_OST_ZVOL, B_TRUE, B_TRUE, FTAG, &os);
 	if (error)
 		goto out_doi;
 
 	error = dmu_object_info(os, ZVOL_OBJ, doi);
 	if (error)
 		goto out_dmu_objset_disown;
 
 	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
 	if (error)
 		goto out_dmu_objset_disown;
 
 	zv = zvol_alloc(MKDEV(zvol_major, minor), name,
 	    doi->doi_data_block_size);
 	if (zv == NULL) {
 		error = SET_ERROR(EAGAIN);
 		goto out_dmu_objset_disown;
 	}
 	zv->zv_hash = hash;
 
 	if (dmu_objset_is_snapshot(os))
 		zv->zv_flags |= ZVOL_RDONLY;
 
 	zv->zv_volsize = volsize;
 	zv->zv_objset = os;
 
 	set_capacity(zv->zv_zso->zvo_disk, zv->zv_volsize >> 9);
 
 #ifdef QUEUE_FLAG_DISCARD
 	blk_queue_flag_set(QUEUE_FLAG_DISCARD, zv->zv_zso->zvo_queue);
 #endif
 #ifdef QUEUE_FLAG_NONROT
 	blk_queue_flag_set(QUEUE_FLAG_NONROT, zv->zv_zso->zvo_queue);
 #endif
 #ifdef QUEUE_FLAG_ADD_RANDOM
 	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, zv->zv_zso->zvo_queue);
 #endif
 	/* This flag was introduced in kernel version 4.12. */
 #ifdef QUEUE_FLAG_SCSI_PASSTHROUGH
 	blk_queue_flag_set(QUEUE_FLAG_SCSI_PASSTHROUGH, zv->zv_zso->zvo_queue);
 #endif
 
 	ASSERT3P(zv->zv_kstat.dk_kstats, ==, NULL);
 	error = dataset_kstats_create(&zv->zv_kstat, zv->zv_objset);
 	if (error)
 		goto out_dmu_objset_disown;
 	ASSERT3P(zv->zv_zilog, ==, NULL);
 	zv->zv_zilog = zil_open(os, zvol_get_data, &zv->zv_kstat.dk_zil_sums);
 	if (spa_writeable(dmu_objset_spa(os))) {
 		if (zil_replay_disable)
 			replayed_zil = zil_destroy(zv->zv_zilog, B_FALSE);
 		else
 			replayed_zil = zil_replay(os, zv, zvol_replay_vector);
 	}
 	if (replayed_zil)
 		zil_close(zv->zv_zilog);
 	zv->zv_zilog = NULL;
 
 	/*
 	 * When udev detects the addition of the device it will immediately
 	 * invoke blkid(8) to determine the type of content on the device.
 	 * Prefetching the blocks commonly scanned by blkid(8) will speed
 	 * up this process.
 	 */
 	len = MIN(zvol_prefetch_bytes, SPA_MAXBLOCKSIZE);
 	if (len > 0) {
 		dmu_prefetch(os, ZVOL_OBJ, 0, 0, len, ZIO_PRIORITY_SYNC_READ);
 		dmu_prefetch(os, ZVOL_OBJ, 0, volsize - len, len,
 		    ZIO_PRIORITY_SYNC_READ);
 	}
 
 	zv->zv_objset = NULL;
 out_dmu_objset_disown:
 	dmu_objset_disown(os, B_TRUE, FTAG);
 out_doi:
 	kmem_free(doi, sizeof (dmu_object_info_t));
 
 	/*
 	 * Keep in mind that once add_disk() is called, the zvol is
 	 * announced to the world, and zvol_open()/zvol_release() can
 	 * be called at any time. Incidentally, add_disk() itself calls
 	 * zvol_open()->zvol_first_open() and zvol_release()->zvol_last_close()
 	 * directly as well.
 	 */
 	if (error == 0) {
 		rw_enter(&zvol_state_lock, RW_WRITER);
 		zvol_insert(zv);
 		rw_exit(&zvol_state_lock);
 		error = zvol_os_add_disk(zv->zv_zso->zvo_disk);
 	} else {
 		ida_simple_remove(&zvol_ida, idx);
 	}
 
 	return (error);
 }
 
 void
 zvol_os_rename_minor(zvol_state_t *zv, const char *newname)
 {
 	int readonly = get_disk_ro(zv->zv_zso->zvo_disk);
 
 	ASSERT(RW_LOCK_HELD(&zvol_state_lock));
 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
 
 	strlcpy(zv->zv_name, newname, sizeof (zv->zv_name));
 
 	/* move to new hashtable entry  */
 	zv->zv_hash = zvol_name_hash(newname);
 	hlist_del(&zv->zv_hlink);
 	hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash));
 
 	/*
 	 * The block device's read-only state is briefly changed causing
 	 * a KOBJ_CHANGE uevent to be issued.  This ensures udev detects
 	 * the name change and fixes the symlinks.  This does not change
 	 * ZVOL_RDONLY in zv->zv_flags so the actual read-only state never
 	 * changes.  This would normally be done using kobject_uevent() but
 	 * that is a GPL-only symbol which is why we need this workaround.
 	 */
 	set_disk_ro(zv->zv_zso->zvo_disk, !readonly);
 	set_disk_ro(zv->zv_zso->zvo_disk, readonly);
 
 	dataset_kstats_rename(&zv->zv_kstat, newname);
 }
 
 void
 zvol_os_set_disk_ro(zvol_state_t *zv, int flags)
 {
 
 	set_disk_ro(zv->zv_zso->zvo_disk, flags);
 }
 
 void
 zvol_os_set_capacity(zvol_state_t *zv, uint64_t capacity)
 {
 
 	set_capacity(zv->zv_zso->zvo_disk, capacity);
 }
 
 int
 zvol_init(void)
 {
 	int error;
 
 	/*
 	 * zvol_threads is the module param the user passes in.
 	 *
 	 * zvol_actual_threads is what we use internally, since the user can
 	 * pass zvol_thread = 0 to mean "use all the CPUs" (the default).
 	 */
 	static unsigned int zvol_actual_threads;
 
 	if (zvol_threads == 0) {
 		/*
 		 * See dde9380a1 for why 32 was chosen here.  This should
 		 * probably be refined to be some multiple of the number
 		 * of CPUs.
 		 */
 		zvol_actual_threads = MAX(num_online_cpus(), 32);
 	} else {
 		zvol_actual_threads = MIN(MAX(zvol_threads, 1), 1024);
 	}
 
 	/*
 	 * Use atleast 32 zvol_threads but for many core system,
 	 * prefer 6 threads per taskq, but no more taskqs
 	 * than threads in them on large systems.
 	 *
 	 *                 taskq   total
 	 * cpus    taskqs  threads threads
 	 * ------- ------- ------- -------
 	 * 1       1       32       32
 	 * 2       1       32       32
 	 * 4       1       32       32
 	 * 8       2       16       32
 	 * 16      3       11       33
 	 * 32      5       7        35
 	 * 64      8       8        64
 	 * 128     11      12       132
 	 * 256     16      16       256
 	 */
 	zv_taskq_t *ztqs = &zvol_taskqs;
 	uint_t num_tqs = MIN(num_online_cpus(), zvol_num_taskqs);
 	if (num_tqs == 0) {
 		num_tqs = 1 + num_online_cpus() / 6;
 		while (num_tqs * num_tqs > zvol_actual_threads)
 			num_tqs--;
 	}
 	uint_t per_tq_thread = zvol_actual_threads / num_tqs;
 	if (per_tq_thread * num_tqs < zvol_actual_threads)
 		per_tq_thread++;
 	ztqs->tqs_cnt = num_tqs;
 	ztqs->tqs_taskq = kmem_alloc(num_tqs * sizeof (taskq_t *), KM_SLEEP);
 	error = register_blkdev(zvol_major, ZVOL_DRIVER);
 	if (error) {
 		kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * sizeof (taskq_t *));
 		ztqs->tqs_taskq = NULL;
 		printk(KERN_INFO "ZFS: register_blkdev() failed %d\n", error);
 		return (error);
 	}
 
 	if (zvol_blk_mq_queue_depth == 0) {
 		zvol_actual_blk_mq_queue_depth = BLKDEV_DEFAULT_RQ;
 	} else {
 		zvol_actual_blk_mq_queue_depth =
 		    MAX(zvol_blk_mq_queue_depth, BLKDEV_MIN_RQ);
 	}
 
 	if (zvol_blk_mq_threads == 0) {
 		zvol_blk_mq_actual_threads = num_online_cpus();
 	} else {
 		zvol_blk_mq_actual_threads = MIN(MAX(zvol_blk_mq_threads, 1),
 		    1024);
 	}
 
 	for (uint_t i = 0; i < num_tqs; i++) {
 		char name[32];
 		(void) snprintf(name, sizeof (name), "%s_tq-%u",
 		    ZVOL_DRIVER, i);
 		ztqs->tqs_taskq[i] = taskq_create(name, per_tq_thread,
 		    maxclsyspri, per_tq_thread, INT_MAX,
 		    TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
 		if (ztqs->tqs_taskq[i] == NULL) {
 			for (int j = i - 1; j >= 0; j--)
 				taskq_destroy(ztqs->tqs_taskq[j]);
 			unregister_blkdev(zvol_major, ZVOL_DRIVER);
 			kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt *
 			    sizeof (taskq_t *));
 			ztqs->tqs_taskq = NULL;
 			return (-ENOMEM);
 		}
 	}
 
 	zvol_init_impl();
 	ida_init(&zvol_ida);
 	return (0);
 }
 
 void
 zvol_fini(void)
 {
 	zv_taskq_t *ztqs = &zvol_taskqs;
 	zvol_fini_impl();
 	unregister_blkdev(zvol_major, ZVOL_DRIVER);
 
 	if (ztqs->tqs_taskq == NULL) {
 		ASSERT3U(ztqs->tqs_cnt, ==, 0);
 	} else {
 		for (uint_t i = 0; i < ztqs->tqs_cnt; i++) {
 			ASSERT3P(ztqs->tqs_taskq[i], !=, NULL);
 			taskq_destroy(ztqs->tqs_taskq[i]);
 		}
 		kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt *
 		    sizeof (taskq_t *));
 		ztqs->tqs_taskq = NULL;
 	}
 
 	ida_destroy(&zvol_ida);
 }
 
 /* BEGIN CSTYLED */
 module_param(zvol_inhibit_dev, uint, 0644);
 MODULE_PARM_DESC(zvol_inhibit_dev, "Do not create zvol device nodes");
 
 module_param(zvol_major, uint, 0444);
 MODULE_PARM_DESC(zvol_major, "Major number for zvol device");
 
 module_param(zvol_threads, uint, 0444);
 MODULE_PARM_DESC(zvol_threads, "Number of threads to handle I/O requests. Set"
     "to 0 to use all active CPUs");
 
 module_param(zvol_request_sync, uint, 0644);
 MODULE_PARM_DESC(zvol_request_sync, "Synchronously handle bio requests");
 
 module_param(zvol_max_discard_blocks, ulong, 0444);
 MODULE_PARM_DESC(zvol_max_discard_blocks, "Max number of blocks to discard");
 
 module_param(zvol_num_taskqs, uint, 0444);
 MODULE_PARM_DESC(zvol_num_taskqs, "Number of zvol taskqs");
 
 module_param(zvol_prefetch_bytes, uint, 0644);
 MODULE_PARM_DESC(zvol_prefetch_bytes, "Prefetch N bytes at zvol start+end");
 
 module_param(zvol_volmode, uint, 0644);
 MODULE_PARM_DESC(zvol_volmode, "Default volmode property value");
 
 module_param(zvol_blk_mq_queue_depth, uint, 0644);
 MODULE_PARM_DESC(zvol_blk_mq_queue_depth, "Default blk-mq queue depth");
 
 module_param(zvol_use_blk_mq, uint, 0644);
 MODULE_PARM_DESC(zvol_use_blk_mq, "Use the blk-mq API for zvols");
 
 module_param(zvol_blk_mq_blocks_per_thread, uint, 0644);
 MODULE_PARM_DESC(zvol_blk_mq_blocks_per_thread,
     "Process volblocksize blocks per thread");
 
 #ifndef HAVE_BLKDEV_GET_ERESTARTSYS
 module_param(zvol_open_timeout_ms, uint, 0644);
 MODULE_PARM_DESC(zvol_open_timeout_ms, "Timeout for ZVOL open retries");
 #endif
 
 /* END CSTYLED */
diff --git a/sys/contrib/openzfs/module/zcommon/zfs_valstr.c b/sys/contrib/openzfs/module/zcommon/zfs_valstr.c
index e2d4d1aefefb..f810d206cfab 100644
--- a/sys/contrib/openzfs/module/zcommon/zfs_valstr.c
+++ b/sys/contrib/openzfs/module/zcommon/zfs_valstr.c
@@ -1,277 +1,279 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2024, Klara Inc.
  */
 
 #include <sys/fs/zfs.h>
 #include <sys/types.h>
 #include <sys/sysmacros.h>
 #include <sys/string.h>
 #include <sys/debug.h>
 #include "zfs_valstr.h"
 
 /*
  * Each bit in a bitfield has three possible string representations:
  * - single char
  * - two-char pair
  * - full name
  */
 typedef struct {
 	const char	vb_bit;
-	const char	vb_pair[2];
+
+	/* 2 byte name + 1 byte NULL terminator to make GCC happy */
+	const char	vb_pair[3];
 	const char	*vb_name;
 } valstr_bit_t;
 
 /*
  * Emits a character for each bit in `bits`, up to the number of elements
  * in the table. Set bits get the character in vb_bit, clear bits get a
  * space. This results in all strings having the same width, for easier
  * visual comparison.
  */
 static size_t
 valstr_bitfield_bits(const valstr_bit_t *table, const size_t nelems,
     uint64_t bits, char *out, size_t outlen)
 {
 	ASSERT(out);
 	size_t n = 0;
 	for (int b = 0; b < nelems; b++) {
 		if (n == outlen)
 			break;
 		uint64_t mask = (1ULL << b);
 		out[n++] = (bits & mask) ? table[b].vb_bit : ' ';
 	}
 	if (n < outlen)
 		out[n++] = '\0';
 	return (n);
 }
 
 /*
  * Emits a two-char pair for each bit set in `bits`, taken from vb_pair, and
  * separated by a `|` character. This gives a concise representation of the
  * whole value.
  */
 static size_t
 valstr_bitfield_pairs(const valstr_bit_t *table, const size_t nelems,
     uint64_t bits, char *out, size_t outlen)
 {
 	ASSERT(out);
 	size_t n = 0;
 	for (int b = 0; b < nelems; b++) {
 		ASSERT3U(n, <=, outlen);
 		if (n == outlen)
 			break;
 		uint64_t mask = (1ULL << b);
 		if (bits & mask) {
 			size_t len = (n > 0) ? 3 : 2;
 			if (n > outlen-len)
 				break;
 			if (n > 0)
 				out[n++] = '|';
 			out[n++] = table[b].vb_pair[0];
 			out[n++] = table[b].vb_pair[1];
 		}
 	}
 	if (n < outlen)
 		out[n++] = '\0';
 	return (n);
 }
 
 /*
  * Emits the full name for each bit set in `bits`, taken from vb_name, and
  * separated by a space. This unambiguously shows the entire set of bits, but
  * can get very long.
  */
 static size_t
 valstr_bitfield_str(const valstr_bit_t *table, const size_t nelems,
     uint64_t bits, char *out, size_t outlen)
 {
 	ASSERT(out);
 	size_t n = 0;
 	for (int b = 0; b < nelems; b++) {
 		ASSERT3U(n, <=, outlen);
 		if (n == outlen)
 			break;
 		uint64_t mask = (1ULL << b);
 		if (bits & mask) {
 			size_t len = strlen(table[b].vb_name);
 			if (n > 0)
 				len++;
 			if (n > outlen-len)
 				break;
 			if (n > 0) {
 				out[n++] = ' ';
 				len--;
 			}
 			memcpy(&out[n], table[b].vb_name, len);
 			n += len;
 		}
 	}
 	if (n < outlen)
 		out[n++] = '\0';
 	return (n);
 }
 
 /*
  * Emits the name of the given enum value in the table.
  */
 static size_t
 valstr_enum_str(const char **table, const size_t nelems,
     int v, char *out, size_t outlen)
 {
 	ASSERT(out);
 	ASSERT3U(v, <, nelems);
 	if (v >= nelems)
 		return (0);
 	return (MIN(strlcpy(out, table[v], outlen), outlen));
 }
 
 /*
  * These macros create the string tables for the given name, and implement
  * the public functions described in zfs_valstr.h.
  */
 #define	_VALSTR_BITFIELD_IMPL(name, ...)				\
 static const valstr_bit_t valstr_ ## name ## _table[] = { __VA_ARGS__ };\
 size_t									\
 zfs_valstr_ ## name ## _bits(uint64_t bits, char *out, size_t outlen)	\
 {									\
 	return (valstr_bitfield_bits(valstr_ ## name ## _table,		\
 	    ARRAY_SIZE(valstr_ ## name ## _table), bits, out, outlen));	\
 }									\
 									\
 size_t									\
 zfs_valstr_ ## name ## _pairs(uint64_t bits, char *out, size_t outlen)	\
 {									\
 	return (valstr_bitfield_pairs(valstr_ ## name ## _table,	\
 	    ARRAY_SIZE(valstr_ ## name ## _table), bits, out, outlen));	\
 }									\
 									\
 size_t									\
 zfs_valstr_ ## name(uint64_t bits, char *out, size_t outlen)		\
 {									\
 	return (valstr_bitfield_str(valstr_ ## name ## _table,		\
 	    ARRAY_SIZE(valstr_ ## name ## _table), bits, out, outlen));	\
 }									\
 
 #define	_VALSTR_ENUM_IMPL(name, ...)					\
 static const char *valstr_ ## name ## _table[] = { __VA_ARGS__ };	\
 size_t									\
 zfs_valstr_ ## name(int v, char *out, size_t outlen)			\
 {									\
 	return (valstr_enum_str(valstr_ ## name ## _table,		\
 	    ARRAY_SIZE(valstr_ ## name ## _table), v, out, outlen));	\
 }									\
 
 
 /* String tables */
 
 /* ZIO flags: zio_flag_t, typically zio->io_flags */
 /* BEGIN CSTYLED */
 _VALSTR_BITFIELD_IMPL(zio_flag,
 	{ '.', "DA", "DONT_AGGREGATE" },
 	{ '.', "RP", "IO_REPAIR" },
 	{ '.', "SH", "SELF_HEAL" },
 	{ '.', "RS", "RESILVER" },
 	{ '.', "SC", "SCRUB" },
 	{ '.', "ST", "SCAN_THREAD" },
 	{ '.', "PH", "PHYSICAL" },
 	{ '.', "CF", "CANFAIL" },
 	{ '.', "SP", "SPECULATIVE" },
 	{ '.', "CW", "CONFIG_WRITER" },
 	{ '.', "DR", "DONT_RETRY" },
 	{ '?', "??", "[UNUSED 11]" },
 	{ '.', "ND", "NODATA" },
 	{ '.', "ID", "INDUCE_DAMAGE" },
 	{ '.', "AL", "IO_ALLOCATING" },
 	{ '.', "RE", "IO_RETRY" },
 	{ '.', "PR", "PROBE" },
 	{ '.', "TH", "TRYHARD" },
 	{ '.', "OP", "OPTIONAL" },
 	{ '.', "DQ", "DONT_QUEUE" },
 	{ '.', "DP", "DONT_PROPAGATE" },
 	{ '.', "BY", "IO_BYPASS" },
 	{ '.', "RW", "IO_REWRITE" },
 	{ '.', "CM", "RAW_COMPRESS" },
 	{ '.', "EN", "RAW_ENCRYPT" },
 	{ '.', "GG", "GANG_CHILD" },
 	{ '.', "DD", "DDT_CHILD" },
 	{ '.', "GF", "GODFATHER" },
 	{ '.', "NP", "NOPWRITE" },
 	{ '.', "EX", "REEXECUTED" },
 	{ '.', "DG", "DELEGATED" },
 )
 /* END CSTYLED */
 
 /*
  * ZIO pipeline stage(s): enum zio_stage, typically zio->io_stage or
  *                        zio->io_pipeline.
  */
 /* BEGIN CSTYLED */
 _VALSTR_BITFIELD_IMPL(zio_stage,
 	{ 'O', "O ", "OPEN" },
 	{ 'I', "RI", "READ_BP_INIT" },
 	{ 'I', "WI", "WRITE_BP_INIT" },
 	{ 'I', "FI", "FREE_BP_INIT" },
 	{ 'A', "IA", "ISSUE_ASYNC" },
 	{ 'W', "WC", "WRITE_COMPRESS" },
 	{ 'E', "EN", "ENCRYPT" },
 	{ 'C', "CG", "CHECKSUM_GENERATE" },
 	{ 'N', "NW", "NOP_WRITE" },
 	{ 'B', "BF", "BRT_FREE" },
 	{ 'd', "dS", "DDT_READ_START" },
 	{ 'd', "dD", "DDT_READ_DONE" },
 	{ 'd', "dW", "DDT_WRITE" },
 	{ 'd', "dF", "DDT_FREE" },
 	{ 'G', "GA", "GANG_ASSEMBLE" },
 	{ 'G', "GI", "GANG_ISSUE" },
 	{ 'D', "DT", "DVA_THROTTLE" },
 	{ 'D', "DA", "DVA_ALLOCATE" },
 	{ 'D', "DF", "DVA_FREE" },
 	{ 'D', "DC", "DVA_CLAIM" },
 	{ 'R', "R ", "READY" },
 	{ 'V', "VS", "VDEV_IO_START" },
 	{ 'V', "VD", "VDEV_IO_DONE" },
 	{ 'V', "VA", "VDEV_IO_ASSESS" },
 	{ 'C', "CV", "CHECKSUM_VERIFY" },
 	{ 'X', "X ", "DONE" },
 )
 /* END CSTYLED */
 
 /* ZIO priority: zio_priority_t, typically zio->io_priority */
 /* BEGIN CSTYLED */
 _VALSTR_ENUM_IMPL(zio_priority,
 	"SYNC_READ",
 	"SYNC_WRITE",
 	"ASYNC_READ",
 	"ASYNC_WRITE",
 	"SCRUB",
 	"REMOVAL",
 	"INITIALIZING",
 	"TRIM",
 	"REBUILD",
 	"[NUM_QUEUEABLE]",
 	"NOW",
 )
 /* END CSTYLED */
 
 #undef _VALSTR_BITFIELD_IMPL
 #undef _VALSTR_ENUM_IMPL
diff --git a/sys/contrib/openzfs/module/zfs/arc.c b/sys/contrib/openzfs/module/zfs/arc.c
index 4013b2d90fd5..5c6e92f0f8b3 100644
--- a/sys/contrib/openzfs/module/zfs/arc.c
+++ b/sys/contrib/openzfs/module/zfs/arc.c
@@ -1,10751 +1,10756 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2018, Joyent, Inc.
  * Copyright (c) 2011, 2020, Delphix. All rights reserved.
  * Copyright (c) 2014, Saso Kiselkov. All rights reserved.
  * Copyright (c) 2017, Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
  * Copyright (c) 2020, George Amanakis. All rights reserved.
  * Copyright (c) 2019, Klara Inc.
  * Copyright (c) 2019, Allan Jude
  * Copyright (c) 2020, The FreeBSD Foundation [1]
  *
  * [1] Portions of this software were developed by Allan Jude
  *     under sponsorship from the FreeBSD Foundation.
  */
 
 /*
  * DVA-based Adjustable Replacement Cache
  *
  * While much of the theory of operation used here is
  * based on the self-tuning, low overhead replacement cache
  * presented by Megiddo and Modha at FAST 2003, there are some
  * significant differences:
  *
  * 1. The Megiddo and Modha model assumes any page is evictable.
  * Pages in its cache cannot be "locked" into memory.  This makes
  * the eviction algorithm simple: evict the last page in the list.
  * This also make the performance characteristics easy to reason
  * about.  Our cache is not so simple.  At any given moment, some
  * subset of the blocks in the cache are un-evictable because we
  * have handed out a reference to them.  Blocks are only evictable
  * when there are no external references active.  This makes
  * eviction far more problematic:  we choose to evict the evictable
  * blocks that are the "lowest" in the list.
  *
  * There are times when it is not possible to evict the requested
  * space.  In these circumstances we are unable to adjust the cache
  * size.  To prevent the cache growing unbounded at these times we
  * implement a "cache throttle" that slows the flow of new data
  * into the cache until we can make space available.
  *
  * 2. The Megiddo and Modha model assumes a fixed cache size.
  * Pages are evicted when the cache is full and there is a cache
  * miss.  Our model has a variable sized cache.  It grows with
  * high use, but also tries to react to memory pressure from the
  * operating system: decreasing its size when system memory is
  * tight.
  *
  * 3. The Megiddo and Modha model assumes a fixed page size. All
  * elements of the cache are therefore exactly the same size.  So
  * when adjusting the cache size following a cache miss, its simply
  * a matter of choosing a single page to evict.  In our model, we
  * have variable sized cache blocks (ranging from 512 bytes to
  * 128K bytes).  We therefore choose a set of blocks to evict to make
  * space for a cache miss that approximates as closely as possible
  * the space used by the new block.
  *
  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
  * by N. Megiddo & D. Modha, FAST 2003
  */
 
 /*
  * The locking model:
  *
  * A new reference to a cache buffer can be obtained in two
  * ways: 1) via a hash table lookup using the DVA as a key,
  * or 2) via one of the ARC lists.  The arc_read() interface
  * uses method 1, while the internal ARC algorithms for
  * adjusting the cache use method 2.  We therefore provide two
  * types of locks: 1) the hash table lock array, and 2) the
  * ARC list locks.
  *
  * Buffers do not have their own mutexes, rather they rely on the
  * hash table mutexes for the bulk of their protection (i.e. most
  * fields in the arc_buf_hdr_t are protected by these mutexes).
  *
  * buf_hash_find() returns the appropriate mutex (held) when it
  * locates the requested buffer in the hash table.  It returns
  * NULL for the mutex if the buffer was not in the table.
  *
  * buf_hash_remove() expects the appropriate hash mutex to be
  * already held before it is invoked.
  *
  * Each ARC state also has a mutex which is used to protect the
  * buffer list associated with the state.  When attempting to
  * obtain a hash table lock while holding an ARC list lock you
  * must use: mutex_tryenter() to avoid deadlock.  Also note that
  * the active state mutex must be held before the ghost state mutex.
  *
  * It as also possible to register a callback which is run when the
  * metadata limit is reached and no buffers can be safely evicted.  In
  * this case the arc user should drop a reference on some arc buffers so
  * they can be reclaimed.  For example, when using the ZPL each dentry
  * holds a references on a znode.  These dentries must be pruned before
  * the arc buffer holding the znode can be safely evicted.
  *
  * Note that the majority of the performance stats are manipulated
  * with atomic operations.
  *
  * The L2ARC uses the l2ad_mtx on each vdev for the following:
  *
  *	- L2ARC buflist creation
  *	- L2ARC buflist eviction
  *	- L2ARC write completion, which walks L2ARC buflists
  *	- ARC header destruction, as it removes from L2ARC buflists
  *	- ARC header release, as it removes from L2ARC buflists
  */
 
 /*
  * ARC operation:
  *
  * Every block that is in the ARC is tracked by an arc_buf_hdr_t structure.
  * This structure can point either to a block that is still in the cache or to
  * one that is only accessible in an L2 ARC device, or it can provide
  * information about a block that was recently evicted. If a block is
  * only accessible in the L2ARC, then the arc_buf_hdr_t only has enough
  * information to retrieve it from the L2ARC device. This information is
  * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block
  * that is in this state cannot access the data directly.
  *
  * Blocks that are actively being referenced or have not been evicted
  * are cached in the L1ARC. The L1ARC (l1arc_buf_hdr_t) is a structure within
  * the arc_buf_hdr_t that will point to the data block in memory. A block can
  * only be read by a consumer if it has an l1arc_buf_hdr_t. The L1ARC
  * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and
  * also in the arc_buf_hdr_t's private physical data block pointer (b_pabd).
  *
  * The L1ARC's data pointer may or may not be uncompressed. The ARC has the
  * ability to store the physical data (b_pabd) associated with the DVA of the
  * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block,
  * it will match its on-disk compression characteristics. This behavior can be
  * disabled by setting 'zfs_compressed_arc_enabled' to B_FALSE. When the
  * compressed ARC functionality is disabled, the b_pabd will point to an
  * uncompressed version of the on-disk data.
  *
  * Data in the L1ARC is not accessed by consumers of the ARC directly. Each
  * arc_buf_hdr_t can have multiple ARC buffers (arc_buf_t) which reference it.
  * Each ARC buffer (arc_buf_t) is being actively accessed by a specific ARC
  * consumer. The ARC will provide references to this data and will keep it
  * cached until it is no longer in use. The ARC caches only the L1ARC's physical
  * data block and will evict any arc_buf_t that is no longer referenced. The
  * amount of memory consumed by the arc_buf_ts' data buffers can be seen via the
  * "overhead_size" kstat.
  *
  * Depending on the consumer, an arc_buf_t can be requested in uncompressed or
  * compressed form. The typical case is that consumers will want uncompressed
  * data, and when that happens a new data buffer is allocated where the data is
  * decompressed for them to use. Currently the only consumer who wants
  * compressed arc_buf_t's is "zfs send", when it streams data exactly as it
  * exists on disk. When this happens, the arc_buf_t's data buffer is shared
  * with the arc_buf_hdr_t.
  *
  * Here is a diagram showing an arc_buf_hdr_t referenced by two arc_buf_t's. The
  * first one is owned by a compressed send consumer (and therefore references
  * the same compressed data buffer as the arc_buf_hdr_t) and the second could be
  * used by any other consumer (and has its own uncompressed copy of the data
  * buffer).
  *
  *   arc_buf_hdr_t
  *   +-----------+
  *   | fields    |
  *   | common to |
  *   | L1- and   |
  *   | L2ARC     |
  *   +-----------+
  *   | l2arc_buf_hdr_t
  *   |           |
  *   +-----------+
  *   | l1arc_buf_hdr_t
  *   |           |              arc_buf_t
  *   | b_buf     +------------>+-----------+      arc_buf_t
  *   | b_pabd    +-+           |b_next     +---->+-----------+
  *   +-----------+ |           |-----------|     |b_next     +-->NULL
  *                 |           |b_comp = T |     +-----------+
  *                 |           |b_data     +-+   |b_comp = F |
  *                 |           +-----------+ |   |b_data     +-+
  *                 +->+------+               |   +-----------+ |
  *        compressed  |      |               |                 |
  *           data     |      |<--------------+                 | uncompressed
  *                    +------+          compressed,            |     data
  *                                        shared               +-->+------+
  *                                         data                    |      |
  *                                                                 |      |
  *                                                                 +------+
  *
  * When a consumer reads a block, the ARC must first look to see if the
  * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
  * arc_buf_t and either copies uncompressed data into a new data buffer from an
  * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
  * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
  * hdr is compressed and the desired compression characteristics of the
  * arc_buf_t consumer. If the arc_buf_t ends up sharing data with the
  * arc_buf_hdr_t and both of them are uncompressed then the arc_buf_t must be
  * the last buffer in the hdr's b_buf list, however a shared compressed buf can
  * be anywhere in the hdr's list.
  *
  * The diagram below shows an example of an uncompressed ARC hdr that is
  * sharing its data with an arc_buf_t (note that the shared uncompressed buf is
  * the last element in the buf list):
  *
  *                arc_buf_hdr_t
  *                +-----------+
  *                |           |
  *                |           |
  *                |           |
  *                +-----------+
  * l2arc_buf_hdr_t|           |
  *                |           |
  *                +-----------+
  * l1arc_buf_hdr_t|           |
  *                |           |                 arc_buf_t    (shared)
  *                |    b_buf  +------------>+---------+      arc_buf_t
  *                |           |             |b_next   +---->+---------+
  *                |  b_pabd   +-+           |---------|     |b_next   +-->NULL
  *                +-----------+ |           |         |     +---------+
  *                              |           |b_data   +-+   |         |
  *                              |           +---------+ |   |b_data   +-+
  *                              +->+------+             |   +---------+ |
  *                                 |      |             |               |
  *                   uncompressed  |      |             |               |
  *                        data     +------+             |               |
  *                                    ^                 +->+------+     |
  *                                    |       uncompressed |      |     |
  *                                    |           data     |      |     |
  *                                    |                    +------+     |
  *                                    +---------------------------------+
  *
  * Writing to the ARC requires that the ARC first discard the hdr's b_pabd
  * since the physical block is about to be rewritten. The new data contents
  * will be contained in the arc_buf_t. As the I/O pipeline performs the write,
  * it may compress the data before writing it to disk. The ARC will be called
  * with the transformed data and will memcpy the transformed on-disk block into
  * a newly allocated b_pabd. Writes are always done into buffers which have
  * either been loaned (and hence are new and don't have other readers) or
  * buffers which have been released (and hence have their own hdr, if there
  * were originally other readers of the buf's original hdr). This ensures that
  * the ARC only needs to update a single buf and its hdr after a write occurs.
  *
  * When the L2ARC is in use, it will also take advantage of the b_pabd. The
  * L2ARC will always write the contents of b_pabd to the L2ARC. This means
  * that when compressed ARC is enabled that the L2ARC blocks are identical
  * to the on-disk block in the main data pool. This provides a significant
  * advantage since the ARC can leverage the bp's checksum when reading from the
  * L2ARC to determine if the contents are valid. However, if the compressed
  * ARC is disabled, then the L2ARC's block must be transformed to look
  * like the physical block in the main data pool before comparing the
  * checksum and determining its validity.
  *
  * The L1ARC has a slightly different system for storing encrypted data.
  * Raw (encrypted + possibly compressed) data has a few subtle differences from
  * data that is just compressed. The biggest difference is that it is not
  * possible to decrypt encrypted data (or vice-versa) if the keys aren't loaded.
  * The other difference is that encryption cannot be treated as a suggestion.
  * If a caller would prefer compressed data, but they actually wind up with
  * uncompressed data the worst thing that could happen is there might be a
  * performance hit. If the caller requests encrypted data, however, we must be
  * sure they actually get it or else secret information could be leaked. Raw
  * data is stored in hdr->b_crypt_hdr.b_rabd. An encrypted header, therefore,
  * may have both an encrypted version and a decrypted version of its data at
  * once. When a caller needs a raw arc_buf_t, it is allocated and the data is
  * copied out of this header. To avoid complications with b_pabd, raw buffers
  * cannot be shared.
  */
 
 #include <sys/spa.h>
 #include <sys/zio.h>
 #include <sys/spa_impl.h>
 #include <sys/zio_compress.h>
 #include <sys/zio_checksum.h>
 #include <sys/zfs_context.h>
 #include <sys/arc.h>
 #include <sys/zfs_refcount.h>
 #include <sys/vdev.h>
 #include <sys/vdev_impl.h>
 #include <sys/dsl_pool.h>
 #include <sys/multilist.h>
 #include <sys/abd.h>
 #include <sys/zil.h>
 #include <sys/fm/fs/zfs.h>
 #include <sys/callb.h>
 #include <sys/kstat.h>
 #include <sys/zthr.h>
 #include <zfs_fletcher.h>
 #include <sys/arc_impl.h>
 #include <sys/trace_zfs.h>
 #include <sys/aggsum.h>
 #include <sys/wmsum.h>
 #include <cityhash.h>
 #include <sys/vdev_trim.h>
 #include <sys/zfs_racct.h>
 #include <sys/zstd/zstd.h>
 
 #ifndef _KERNEL
 /* set with ZFS_DEBUG=watch, to enable watchpoints on frozen buffers */
 boolean_t arc_watch = B_FALSE;
 #endif
 
 /*
  * This thread's job is to keep enough free memory in the system, by
  * calling arc_kmem_reap_soon() plus arc_reduce_target_size(), which improves
  * arc_available_memory().
  */
 static zthr_t *arc_reap_zthr;
 
 /*
  * This thread's job is to keep arc_size under arc_c, by calling
  * arc_evict(), which improves arc_is_overflowing().
  */
 static zthr_t *arc_evict_zthr;
 static arc_buf_hdr_t **arc_state_evict_markers;
 static int arc_state_evict_marker_count;
 
 static kmutex_t arc_evict_lock;
 static boolean_t arc_evict_needed = B_FALSE;
 static clock_t arc_last_uncached_flush;
 
 /*
  * Count of bytes evicted since boot.
  */
 static uint64_t arc_evict_count;
 
 /*
  * List of arc_evict_waiter_t's, representing threads waiting for the
  * arc_evict_count to reach specific values.
  */
 static list_t arc_evict_waiters;
 
 /*
  * When arc_is_overflowing(), arc_get_data_impl() waits for this percent of
  * the requested amount of data to be evicted.  For example, by default for
  * every 2KB that's evicted, 1KB of it may be "reused" by a new allocation.
  * Since this is above 100%, it ensures that progress is made towards getting
  * arc_size under arc_c.  Since this is finite, it ensures that allocations
  * can still happen, even during the potentially long time that arc_size is
  * more than arc_c.
  */
 static uint_t zfs_arc_eviction_pct = 200;
 
 /*
  * The number of headers to evict in arc_evict_state_impl() before
  * dropping the sublist lock and evicting from another sublist. A lower
  * value means we're more likely to evict the "correct" header (i.e. the
  * oldest header in the arc state), but comes with higher overhead
  * (i.e. more invocations of arc_evict_state_impl()).
  */
 static uint_t zfs_arc_evict_batch_limit = 10;
 
 /* number of seconds before growing cache again */
 uint_t arc_grow_retry = 5;
 
 /*
  * Minimum time between calls to arc_kmem_reap_soon().
  */
 static const int arc_kmem_cache_reap_retry_ms = 1000;
 
 /* shift of arc_c for calculating overflow limit in arc_get_data_impl */
 static int zfs_arc_overflow_shift = 8;
 
 /* log2(fraction of arc to reclaim) */
 uint_t arc_shrink_shift = 7;
 
 /* percent of pagecache to reclaim arc to */
 #ifdef _KERNEL
 uint_t zfs_arc_pc_percent = 0;
 #endif
 
 /*
  * log2(fraction of ARC which must be free to allow growing).
  * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
  * when reading a new block into the ARC, we will evict an equal-sized block
  * from the ARC.
  *
  * This must be less than arc_shrink_shift, so that when we shrink the ARC,
  * we will still not allow it to grow.
  */
 uint_t		arc_no_grow_shift = 5;
 
 
 /*
  * minimum lifespan of a prefetch block in clock ticks
  * (initialized in arc_init())
  */
 static uint_t		arc_min_prefetch_ms;
 static uint_t		arc_min_prescient_prefetch_ms;
 
 /*
  * If this percent of memory is free, don't throttle.
  */
 uint_t arc_lotsfree_percent = 10;
 
 /*
  * The arc has filled available memory and has now warmed up.
  */
 boolean_t arc_warm;
 
 /*
  * These tunables are for performance analysis.
  */
 uint64_t zfs_arc_max = 0;
 uint64_t zfs_arc_min = 0;
 static uint64_t zfs_arc_dnode_limit = 0;
 static uint_t zfs_arc_dnode_reduce_percent = 10;
 static uint_t zfs_arc_grow_retry = 0;
 static uint_t zfs_arc_shrink_shift = 0;
 uint_t zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
 
 /*
  * ARC dirty data constraints for arc_tempreserve_space() throttle:
  * * total dirty data limit
  * * anon block dirty limit
  * * each pool's anon allowance
  */
 static const unsigned long zfs_arc_dirty_limit_percent = 50;
 static const unsigned long zfs_arc_anon_limit_percent = 25;
 static const unsigned long zfs_arc_pool_dirty_percent = 20;
 
 /*
  * Enable or disable compressed arc buffers.
  */
 int zfs_compressed_arc_enabled = B_TRUE;
 
 /*
  * Balance between metadata and data on ghost hits.  Values above 100
  * increase metadata caching by proportionally reducing effect of ghost
  * data hits on target data/metadata rate.
  */
 static uint_t zfs_arc_meta_balance = 500;
 
 /*
  * Percentage that can be consumed by dnodes of ARC meta buffers.
  */
 static uint_t zfs_arc_dnode_limit_percent = 10;
 
 /*
  * These tunables are Linux-specific
  */
 static uint64_t zfs_arc_sys_free = 0;
 static uint_t zfs_arc_min_prefetch_ms = 0;
 static uint_t zfs_arc_min_prescient_prefetch_ms = 0;
 static uint_t zfs_arc_lotsfree_percent = 10;
 
 /*
  * Number of arc_prune threads
  */
 static int zfs_arc_prune_task_threads = 1;
 
 /* The 7 states: */
 arc_state_t ARC_anon;
 arc_state_t ARC_mru;
 arc_state_t ARC_mru_ghost;
 arc_state_t ARC_mfu;
 arc_state_t ARC_mfu_ghost;
 arc_state_t ARC_l2c_only;
 arc_state_t ARC_uncached;
 
 arc_stats_t arc_stats = {
 	{ "hits",			KSTAT_DATA_UINT64 },
 	{ "iohits",			KSTAT_DATA_UINT64 },
 	{ "misses",			KSTAT_DATA_UINT64 },
 	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
 	{ "demand_data_iohits",		KSTAT_DATA_UINT64 },
 	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
 	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
 	{ "demand_metadata_iohits",	KSTAT_DATA_UINT64 },
 	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
 	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
 	{ "prefetch_data_iohits",	KSTAT_DATA_UINT64 },
 	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
 	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
 	{ "prefetch_metadata_iohits",	KSTAT_DATA_UINT64 },
 	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
 	{ "mru_hits",			KSTAT_DATA_UINT64 },
 	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
 	{ "uncached_hits",		KSTAT_DATA_UINT64 },
 	{ "deleted",			KSTAT_DATA_UINT64 },
 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
 	{ "access_skip",		KSTAT_DATA_UINT64 },
 	{ "evict_skip",			KSTAT_DATA_UINT64 },
 	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
 	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
 	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
 	{ "evict_l2_eligible_mfu",	KSTAT_DATA_UINT64 },
 	{ "evict_l2_eligible_mru",	KSTAT_DATA_UINT64 },
 	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
 	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
 	{ "hash_elements",		KSTAT_DATA_UINT64 },
 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
 	{ "hash_chains",		KSTAT_DATA_UINT64 },
 	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
 	{ "meta",			KSTAT_DATA_UINT64 },
 	{ "pd",				KSTAT_DATA_UINT64 },
 	{ "pm",				KSTAT_DATA_UINT64 },
 	{ "c",				KSTAT_DATA_UINT64 },
 	{ "c_min",			KSTAT_DATA_UINT64 },
 	{ "c_max",			KSTAT_DATA_UINT64 },
 	{ "size",			KSTAT_DATA_UINT64 },
 	{ "compressed_size",		KSTAT_DATA_UINT64 },
 	{ "uncompressed_size",		KSTAT_DATA_UINT64 },
 	{ "overhead_size",		KSTAT_DATA_UINT64 },
 	{ "hdr_size",			KSTAT_DATA_UINT64 },
 	{ "data_size",			KSTAT_DATA_UINT64 },
 	{ "metadata_size",		KSTAT_DATA_UINT64 },
 	{ "dbuf_size",			KSTAT_DATA_UINT64 },
 	{ "dnode_size",			KSTAT_DATA_UINT64 },
 	{ "bonus_size",			KSTAT_DATA_UINT64 },
 #if defined(COMPAT_FREEBSD11)
 	{ "other_size",			KSTAT_DATA_UINT64 },
 #endif
 	{ "anon_size",			KSTAT_DATA_UINT64 },
 	{ "anon_data",			KSTAT_DATA_UINT64 },
 	{ "anon_metadata",		KSTAT_DATA_UINT64 },
 	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
 	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
 	{ "mru_size",			KSTAT_DATA_UINT64 },
 	{ "mru_data",			KSTAT_DATA_UINT64 },
 	{ "mru_metadata",		KSTAT_DATA_UINT64 },
 	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
 	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
 	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
 	{ "mru_ghost_data",		KSTAT_DATA_UINT64 },
 	{ "mru_ghost_metadata",		KSTAT_DATA_UINT64 },
 	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
 	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
 	{ "mfu_size",			KSTAT_DATA_UINT64 },
 	{ "mfu_data",			KSTAT_DATA_UINT64 },
 	{ "mfu_metadata",		KSTAT_DATA_UINT64 },
 	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
 	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
 	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
 	{ "mfu_ghost_data",		KSTAT_DATA_UINT64 },
 	{ "mfu_ghost_metadata",		KSTAT_DATA_UINT64 },
 	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
 	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
 	{ "uncached_size",		KSTAT_DATA_UINT64 },
 	{ "uncached_data",		KSTAT_DATA_UINT64 },
 	{ "uncached_metadata",		KSTAT_DATA_UINT64 },
 	{ "uncached_evictable_data",	KSTAT_DATA_UINT64 },
 	{ "uncached_evictable_metadata", KSTAT_DATA_UINT64 },
 	{ "l2_hits",			KSTAT_DATA_UINT64 },
 	{ "l2_misses",			KSTAT_DATA_UINT64 },
 	{ "l2_prefetch_asize",		KSTAT_DATA_UINT64 },
 	{ "l2_mru_asize",		KSTAT_DATA_UINT64 },
 	{ "l2_mfu_asize",		KSTAT_DATA_UINT64 },
 	{ "l2_bufc_data_asize",		KSTAT_DATA_UINT64 },
 	{ "l2_bufc_metadata_asize",	KSTAT_DATA_UINT64 },
 	{ "l2_feeds",			KSTAT_DATA_UINT64 },
 	{ "l2_rw_clash",		KSTAT_DATA_UINT64 },
 	{ "l2_read_bytes",		KSTAT_DATA_UINT64 },
 	{ "l2_write_bytes",		KSTAT_DATA_UINT64 },
 	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
 	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
 	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
 	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
 	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
 	{ "l2_io_error",		KSTAT_DATA_UINT64 },
 	{ "l2_size",			KSTAT_DATA_UINT64 },
 	{ "l2_asize",			KSTAT_DATA_UINT64 },
 	{ "l2_hdr_size",		KSTAT_DATA_UINT64 },
 	{ "l2_log_blk_writes",		KSTAT_DATA_UINT64 },
 	{ "l2_log_blk_avg_asize",	KSTAT_DATA_UINT64 },
 	{ "l2_log_blk_asize",		KSTAT_DATA_UINT64 },
 	{ "l2_log_blk_count",		KSTAT_DATA_UINT64 },
 	{ "l2_data_to_meta_ratio",	KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_success",		KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_unsupported",	KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_io_errors",	KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_dh_errors",	KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_cksum_lb_errors",	KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_lowmem",		KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_size",		KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_asize",		KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_bufs",		KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_bufs_precached",	KSTAT_DATA_UINT64 },
 	{ "l2_rebuild_log_blks",	KSTAT_DATA_UINT64 },
 	{ "memory_throttle_count",	KSTAT_DATA_UINT64 },
 	{ "memory_direct_count",	KSTAT_DATA_UINT64 },
 	{ "memory_indirect_count",	KSTAT_DATA_UINT64 },
 	{ "memory_all_bytes",		KSTAT_DATA_UINT64 },
 	{ "memory_free_bytes",		KSTAT_DATA_UINT64 },
 	{ "memory_available_bytes",	KSTAT_DATA_INT64 },
 	{ "arc_no_grow",		KSTAT_DATA_UINT64 },
 	{ "arc_tempreserve",		KSTAT_DATA_UINT64 },
 	{ "arc_loaned_bytes",		KSTAT_DATA_UINT64 },
 	{ "arc_prune",			KSTAT_DATA_UINT64 },
 	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
 	{ "arc_dnode_limit",		KSTAT_DATA_UINT64 },
 	{ "async_upgrade_sync",		KSTAT_DATA_UINT64 },
 	{ "predictive_prefetch", KSTAT_DATA_UINT64 },
 	{ "demand_hit_predictive_prefetch", KSTAT_DATA_UINT64 },
 	{ "demand_iohit_predictive_prefetch", KSTAT_DATA_UINT64 },
 	{ "prescient_prefetch", KSTAT_DATA_UINT64 },
 	{ "demand_hit_prescient_prefetch", KSTAT_DATA_UINT64 },
 	{ "demand_iohit_prescient_prefetch", KSTAT_DATA_UINT64 },
 	{ "arc_need_free",		KSTAT_DATA_UINT64 },
 	{ "arc_sys_free",		KSTAT_DATA_UINT64 },
 	{ "arc_raw_size",		KSTAT_DATA_UINT64 },
 	{ "cached_only_in_progress",	KSTAT_DATA_UINT64 },
 	{ "abd_chunk_waste_size",	KSTAT_DATA_UINT64 },
 };
 
 arc_sums_t arc_sums;
 
 #define	ARCSTAT_MAX(stat, val) {					\
 	uint64_t m;							\
 	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
 	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
 		continue;						\
 }
 
 /*
  * We define a macro to allow ARC hits/misses to be easily broken down by
  * two separate conditions, giving a total of four different subtypes for
  * each of hits and misses (so eight statistics total).
  */
 #define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
 	if (cond1) {							\
 		if (cond2) {						\
 			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
 		} else {						\
 			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
 		}							\
 	} else {							\
 		if (cond2) {						\
 			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
 		} else {						\
 			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
 		}							\
 	}
 
 /*
  * This macro allows us to use kstats as floating averages. Each time we
  * update this kstat, we first factor it and the update value by
  * ARCSTAT_AVG_FACTOR to shrink the new value's contribution to the overall
  * average. This macro assumes that integer loads and stores are atomic, but
  * is not safe for multiple writers updating the kstat in parallel (only the
  * last writer's update will remain).
  */
 #define	ARCSTAT_F_AVG_FACTOR	3
 #define	ARCSTAT_F_AVG(stat, value) \
 	do { \
 		uint64_t x = ARCSTAT(stat); \
 		x = x - x / ARCSTAT_F_AVG_FACTOR + \
 		    (value) / ARCSTAT_F_AVG_FACTOR; \
 		ARCSTAT(stat) = x; \
 	} while (0)
 
 static kstat_t			*arc_ksp;
 
 /*
  * There are several ARC variables that are critical to export as kstats --
  * but we don't want to have to grovel around in the kstat whenever we wish to
  * manipulate them.  For these variables, we therefore define them to be in
  * terms of the statistic variable.  This assures that we are not introducing
  * the possibility of inconsistency by having shadow copies of the variables,
  * while still allowing the code to be readable.
  */
 #define	arc_tempreserve	ARCSTAT(arcstat_tempreserve)
 #define	arc_loaned_bytes	ARCSTAT(arcstat_loaned_bytes)
 #define	arc_dnode_limit	ARCSTAT(arcstat_dnode_limit) /* max size for dnodes */
 #define	arc_need_free	ARCSTAT(arcstat_need_free) /* waiting to be evicted */
 
 hrtime_t arc_growtime;
 list_t arc_prune_list;
 kmutex_t arc_prune_mtx;
 taskq_t *arc_prune_taskq;
 
 #define	GHOST_STATE(state)	\
 	((state) == arc_mru_ghost || (state) == arc_mfu_ghost ||	\
 	(state) == arc_l2c_only)
 
 #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
 #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
 #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_FLAG_IO_ERROR)
 #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
 #define	HDR_PRESCIENT_PREFETCH(hdr)	\
 	((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
 #define	HDR_COMPRESSION_ENABLED(hdr)	\
 	((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
 
 #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
 #define	HDR_UNCACHED(hdr)	((hdr)->b_flags & ARC_FLAG_UNCACHED)
 #define	HDR_L2_READING(hdr)	\
 	(((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
 	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
 #define	HDR_PROTECTED(hdr)	((hdr)->b_flags & ARC_FLAG_PROTECTED)
 #define	HDR_NOAUTH(hdr)		((hdr)->b_flags & ARC_FLAG_NOAUTH)
 #define	HDR_SHARED_DATA(hdr)	((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
 
 #define	HDR_ISTYPE_METADATA(hdr)	\
 	((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
 #define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
 
 #define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
 #define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
 #define	HDR_HAS_RABD(hdr)	\
 	(HDR_HAS_L1HDR(hdr) && HDR_PROTECTED(hdr) &&	\
 	(hdr)->b_crypt_hdr.b_rabd != NULL)
 #define	HDR_ENCRYPTED(hdr)	\
 	(HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
 #define	HDR_AUTHENTICATED(hdr)	\
 	(HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
 
 /* For storing compression mode in b_flags */
 #define	HDR_COMPRESS_OFFSET	(highbit64(ARC_FLAG_COMPRESS_0) - 1)
 
 #define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET((hdr)->b_flags, \
 	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS))
 #define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
 	HDR_COMPRESS_OFFSET, SPA_COMPRESSBITS, (cmp));
 
 #define	ARC_BUF_LAST(buf)	((buf)->b_next == NULL)
 #define	ARC_BUF_SHARED(buf)	((buf)->b_flags & ARC_BUF_FLAG_SHARED)
 #define	ARC_BUF_COMPRESSED(buf)	((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED)
 #define	ARC_BUF_ENCRYPTED(buf)	((buf)->b_flags & ARC_BUF_FLAG_ENCRYPTED)
 
 /*
  * Other sizes
  */
 
 #define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
 #define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
 
 /*
  * Hash table routines
  */
 
 #define	BUF_LOCKS 2048
 typedef struct buf_hash_table {
 	uint64_t ht_mask;
 	arc_buf_hdr_t **ht_table;
 	kmutex_t ht_locks[BUF_LOCKS] ____cacheline_aligned;
 } buf_hash_table_t;
 
 static buf_hash_table_t buf_hash_table;
 
 #define	BUF_HASH_INDEX(spa, dva, birth) \
 	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
 #define	BUF_HASH_LOCK(idx)	(&buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
 #define	HDR_LOCK(hdr) \
 	(BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
 
 uint64_t zfs_crc64_table[256];
 
 /*
  * Level 2 ARC
  */
 
 #define	L2ARC_WRITE_SIZE	(32 * 1024 * 1024)	/* initial write max */
 #define	L2ARC_HEADROOM		8			/* num of writes */
 
 /*
  * If we discover during ARC scan any buffers to be compressed, we boost
  * our headroom for the next scanning cycle by this percentage multiple.
  */
 #define	L2ARC_HEADROOM_BOOST	200
 #define	L2ARC_FEED_SECS		1		/* caching interval secs */
 #define	L2ARC_FEED_MIN_MS	200		/* min caching interval ms */
 
 /*
  * We can feed L2ARC from two states of ARC buffers, mru and mfu,
  * and each of the state has two types: data and metadata.
  */
 #define	L2ARC_FEED_TYPES	4
 
 /* L2ARC Performance Tunables */
 uint64_t l2arc_write_max = L2ARC_WRITE_SIZE;	/* def max write size */
 uint64_t l2arc_write_boost = L2ARC_WRITE_SIZE;	/* extra warmup write */
 uint64_t l2arc_headroom = L2ARC_HEADROOM;	/* # of dev writes */
 uint64_t l2arc_headroom_boost = L2ARC_HEADROOM_BOOST;
 uint64_t l2arc_feed_secs = L2ARC_FEED_SECS;	/* interval seconds */
 uint64_t l2arc_feed_min_ms = L2ARC_FEED_MIN_MS;	/* min interval msecs */
 int l2arc_noprefetch = B_TRUE;			/* don't cache prefetch bufs */
 int l2arc_feed_again = B_TRUE;			/* turbo warmup */
 int l2arc_norw = B_FALSE;			/* no reads during writes */
 static uint_t l2arc_meta_percent = 33;	/* limit on headers size */
 
 /*
  * L2ARC Internals
  */
 static list_t L2ARC_dev_list;			/* device list */
 static list_t *l2arc_dev_list;			/* device list pointer */
 static kmutex_t l2arc_dev_mtx;			/* device list mutex */
 static l2arc_dev_t *l2arc_dev_last;		/* last device used */
 static list_t L2ARC_free_on_write;		/* free after write buf list */
 static list_t *l2arc_free_on_write;		/* free after write list ptr */
 static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
 static uint64_t l2arc_ndev;			/* number of devices */
 
 typedef struct l2arc_read_callback {
 	arc_buf_hdr_t		*l2rcb_hdr;		/* read header */
 	blkptr_t		l2rcb_bp;		/* original blkptr */
 	zbookmark_phys_t	l2rcb_zb;		/* original bookmark */
 	int			l2rcb_flags;		/* original flags */
 	abd_t			*l2rcb_abd;		/* temporary buffer */
 } l2arc_read_callback_t;
 
 typedef struct l2arc_data_free {
 	/* protected by l2arc_free_on_write_mtx */
 	abd_t		*l2df_abd;
 	size_t		l2df_size;
 	arc_buf_contents_t l2df_type;
 	list_node_t	l2df_list_node;
 } l2arc_data_free_t;
 
 typedef enum arc_fill_flags {
 	ARC_FILL_LOCKED		= 1 << 0, /* hdr lock is held */
 	ARC_FILL_COMPRESSED	= 1 << 1, /* fill with compressed data */
 	ARC_FILL_ENCRYPTED	= 1 << 2, /* fill with encrypted data */
 	ARC_FILL_NOAUTH		= 1 << 3, /* don't attempt to authenticate */
 	ARC_FILL_IN_PLACE	= 1 << 4  /* fill in place (special case) */
 } arc_fill_flags_t;
 
 typedef enum arc_ovf_level {
 	ARC_OVF_NONE,			/* ARC within target size. */
 	ARC_OVF_SOME,			/* ARC is slightly overflowed. */
 	ARC_OVF_SEVERE			/* ARC is severely overflowed. */
 } arc_ovf_level_t;
 
 static kmutex_t l2arc_feed_thr_lock;
 static kcondvar_t l2arc_feed_thr_cv;
 static uint8_t l2arc_thread_exit;
 
 static kmutex_t l2arc_rebuild_thr_lock;
 static kcondvar_t l2arc_rebuild_thr_cv;
 
 enum arc_hdr_alloc_flags {
 	ARC_HDR_ALLOC_RDATA = 0x1,
 	ARC_HDR_USE_RESERVE = 0x4,
 	ARC_HDR_ALLOC_LINEAR = 0x8,
 };
 
 
 static abd_t *arc_get_data_abd(arc_buf_hdr_t *, uint64_t, const void *, int);
 static void *arc_get_data_buf(arc_buf_hdr_t *, uint64_t, const void *);
 static void arc_get_data_impl(arc_buf_hdr_t *, uint64_t, const void *, int);
 static void arc_free_data_abd(arc_buf_hdr_t *, abd_t *, uint64_t, const void *);
 static void arc_free_data_buf(arc_buf_hdr_t *, void *, uint64_t, const void *);
 static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size,
     const void *tag);
 static void arc_hdr_free_abd(arc_buf_hdr_t *, boolean_t);
 static void arc_hdr_alloc_abd(arc_buf_hdr_t *, int);
 static void arc_hdr_destroy(arc_buf_hdr_t *);
 static void arc_access(arc_buf_hdr_t *, arc_flags_t, boolean_t);
 static void arc_buf_watch(arc_buf_t *);
 static void arc_change_state(arc_state_t *, arc_buf_hdr_t *);
 
 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
 static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
 
 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
 static void l2arc_read_done(zio_t *);
 static void l2arc_do_free_on_write(void);
 static void l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
     boolean_t state_only);
 
 static void arc_prune_async(uint64_t adjust);
 
 #define	l2arc_hdr_arcstats_increment(hdr) \
 	l2arc_hdr_arcstats_update((hdr), B_TRUE, B_FALSE)
 #define	l2arc_hdr_arcstats_decrement(hdr) \
 	l2arc_hdr_arcstats_update((hdr), B_FALSE, B_FALSE)
 #define	l2arc_hdr_arcstats_increment_state(hdr) \
 	l2arc_hdr_arcstats_update((hdr), B_TRUE, B_TRUE)
 #define	l2arc_hdr_arcstats_decrement_state(hdr) \
 	l2arc_hdr_arcstats_update((hdr), B_FALSE, B_TRUE)
 
 /*
  * l2arc_exclude_special : A zfs module parameter that controls whether buffers
  * 		present on special vdevs are eligibile for caching in L2ARC. If
  * 		set to 1, exclude dbufs on special vdevs from being cached to
  * 		L2ARC.
  */
 int l2arc_exclude_special = 0;
 
 /*
  * l2arc_mfuonly : A ZFS module parameter that controls whether only MFU
  * 		metadata and data are cached from ARC into L2ARC.
  */
 static int l2arc_mfuonly = 0;
 
 /*
  * L2ARC TRIM
  * l2arc_trim_ahead : A ZFS module parameter that controls how much ahead of
  * 		the current write size (l2arc_write_max) we should TRIM if we
  * 		have filled the device. It is defined as a percentage of the
  * 		write size. If set to 100 we trim twice the space required to
  * 		accommodate upcoming writes. A minimum of 64MB will be trimmed.
  * 		It also enables TRIM of the whole L2ARC device upon creation or
  * 		addition to an existing pool or if the header of the device is
  * 		invalid upon importing a pool or onlining a cache device. The
  * 		default is 0, which disables TRIM on L2ARC altogether as it can
  * 		put significant stress on the underlying storage devices. This
  * 		will vary depending of how well the specific device handles
  * 		these commands.
  */
 static uint64_t l2arc_trim_ahead = 0;
 
 /*
  * Performance tuning of L2ARC persistence:
  *
  * l2arc_rebuild_enabled : A ZFS module parameter that controls whether adding
  * 		an L2ARC device (either at pool import or later) will attempt
  * 		to rebuild L2ARC buffer contents.
  * l2arc_rebuild_blocks_min_l2size : A ZFS module parameter that controls
  * 		whether log blocks are written to the L2ARC device. If the L2ARC
  * 		device is less than 1GB, the amount of data l2arc_evict()
  * 		evicts is significant compared to the amount of restored L2ARC
  * 		data. In this case do not write log blocks in L2ARC in order
  * 		not to waste space.
  */
 static int l2arc_rebuild_enabled = B_TRUE;
 static uint64_t l2arc_rebuild_blocks_min_l2size = 1024 * 1024 * 1024;
 
 /* L2ARC persistence rebuild control routines. */
 void l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen);
 static __attribute__((noreturn)) void l2arc_dev_rebuild_thread(void *arg);
 static int l2arc_rebuild(l2arc_dev_t *dev);
 
 /* L2ARC persistence read I/O routines. */
 static int l2arc_dev_hdr_read(l2arc_dev_t *dev);
 static int l2arc_log_blk_read(l2arc_dev_t *dev,
     const l2arc_log_blkptr_t *this_lp, const l2arc_log_blkptr_t *next_lp,
     l2arc_log_blk_phys_t *this_lb, l2arc_log_blk_phys_t *next_lb,
     zio_t *this_io, zio_t **next_io);
 static zio_t *l2arc_log_blk_fetch(vdev_t *vd,
     const l2arc_log_blkptr_t *lp, l2arc_log_blk_phys_t *lb);
 static void l2arc_log_blk_fetch_abort(zio_t *zio);
 
 /* L2ARC persistence block restoration routines. */
 static void l2arc_log_blk_restore(l2arc_dev_t *dev,
     const l2arc_log_blk_phys_t *lb, uint64_t lb_asize);
 static void l2arc_hdr_restore(const l2arc_log_ent_phys_t *le,
     l2arc_dev_t *dev);
 
 /* L2ARC persistence write I/O routines. */
 static uint64_t l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio,
     l2arc_write_callback_t *cb);
 
 /* L2ARC persistence auxiliary routines. */
 boolean_t l2arc_log_blkptr_valid(l2arc_dev_t *dev,
     const l2arc_log_blkptr_t *lbp);
 static boolean_t l2arc_log_blk_insert(l2arc_dev_t *dev,
     const arc_buf_hdr_t *ab);
 boolean_t l2arc_range_check_overlap(uint64_t bottom,
     uint64_t top, uint64_t check);
 static void l2arc_blk_fetch_done(zio_t *zio);
 static inline uint64_t
     l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev);
 
 /*
  * We use Cityhash for this. It's fast, and has good hash properties without
  * requiring any large static buffers.
  */
 static uint64_t
 buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
 {
 	return (cityhash4(spa, dva->dva_word[0], dva->dva_word[1], birth));
 }
 
 #define	HDR_EMPTY(hdr)						\
 	((hdr)->b_dva.dva_word[0] == 0 &&			\
 	(hdr)->b_dva.dva_word[1] == 0)
 
 #define	HDR_EMPTY_OR_LOCKED(hdr)				\
 	(HDR_EMPTY(hdr) || MUTEX_HELD(HDR_LOCK(hdr)))
 
 #define	HDR_EQUAL(spa, dva, birth, hdr)				\
 	((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
 	((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
 	((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
 
 static void
 buf_discard_identity(arc_buf_hdr_t *hdr)
 {
 	hdr->b_dva.dva_word[0] = 0;
 	hdr->b_dva.dva_word[1] = 0;
 	hdr->b_birth = 0;
 }
 
 static arc_buf_hdr_t *
 buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
 {
 	const dva_t *dva = BP_IDENTITY(bp);
 	uint64_t birth = BP_PHYSICAL_BIRTH(bp);
 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
 	arc_buf_hdr_t *hdr;
 
 	mutex_enter(hash_lock);
 	for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
 	    hdr = hdr->b_hash_next) {
 		if (HDR_EQUAL(spa, dva, birth, hdr)) {
 			*lockp = hash_lock;
 			return (hdr);
 		}
 	}
 	mutex_exit(hash_lock);
 	*lockp = NULL;
 	return (NULL);
 }
 
 /*
  * Insert an entry into the hash table.  If there is already an element
  * equal to elem in the hash table, then the already existing element
  * will be returned and the new element will not be inserted.
  * Otherwise returns NULL.
  * If lockp == NULL, the caller is assumed to already hold the hash lock.
  */
 static arc_buf_hdr_t *
 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
 {
 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
 	arc_buf_hdr_t *fhdr;
 	uint32_t i;
 
 	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
 	ASSERT(hdr->b_birth != 0);
 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
 
 	if (lockp != NULL) {
 		*lockp = hash_lock;
 		mutex_enter(hash_lock);
 	} else {
 		ASSERT(MUTEX_HELD(hash_lock));
 	}
 
 	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
 	    fhdr = fhdr->b_hash_next, i++) {
 		if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
 			return (fhdr);
 	}
 
 	hdr->b_hash_next = buf_hash_table.ht_table[idx];
 	buf_hash_table.ht_table[idx] = hdr;
 	arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
 
 	/* collect some hash table performance data */
 	if (i > 0) {
 		ARCSTAT_BUMP(arcstat_hash_collisions);
 		if (i == 1)
 			ARCSTAT_BUMP(arcstat_hash_chains);
 
 		ARCSTAT_MAX(arcstat_hash_chain_max, i);
 	}
 	uint64_t he = atomic_inc_64_nv(
 	    &arc_stats.arcstat_hash_elements.value.ui64);
 	ARCSTAT_MAX(arcstat_hash_elements_max, he);
 
 	return (NULL);
 }
 
 static void
 buf_hash_remove(arc_buf_hdr_t *hdr)
 {
 	arc_buf_hdr_t *fhdr, **hdrp;
 	uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
 
 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
 	ASSERT(HDR_IN_HASH_TABLE(hdr));
 
 	hdrp = &buf_hash_table.ht_table[idx];
 	while ((fhdr = *hdrp) != hdr) {
 		ASSERT3P(fhdr, !=, NULL);
 		hdrp = &fhdr->b_hash_next;
 	}
 	*hdrp = hdr->b_hash_next;
 	hdr->b_hash_next = NULL;
 	arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
 
 	/* collect some hash table performance data */
 	atomic_dec_64(&arc_stats.arcstat_hash_elements.value.ui64);
 
 	if (buf_hash_table.ht_table[idx] &&
 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
 		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
 }
 
 /*
  * Global data structures and functions for the buf kmem cache.
  */
 
 static kmem_cache_t *hdr_full_cache;
 static kmem_cache_t *hdr_l2only_cache;
 static kmem_cache_t *buf_cache;
 
 static void
 buf_fini(void)
 {
 #if defined(_KERNEL)
 	/*
 	 * Large allocations which do not require contiguous pages
 	 * should be using vmem_free() in the linux kernel\
 	 */
 	vmem_free(buf_hash_table.ht_table,
 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
 #else
 	kmem_free(buf_hash_table.ht_table,
 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
 #endif
 	for (int i = 0; i < BUF_LOCKS; i++)
 		mutex_destroy(BUF_HASH_LOCK(i));
 	kmem_cache_destroy(hdr_full_cache);
 	kmem_cache_destroy(hdr_l2only_cache);
 	kmem_cache_destroy(buf_cache);
 }
 
 /*
  * Constructor callback - called when the cache is empty
  * and a new buf is requested.
  */
 static int
 hdr_full_cons(void *vbuf, void *unused, int kmflag)
 {
 	(void) unused, (void) kmflag;
 	arc_buf_hdr_t *hdr = vbuf;
 
 	memset(hdr, 0, HDR_FULL_SIZE);
 	hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
 	zfs_refcount_create(&hdr->b_l1hdr.b_refcnt);
 #ifdef ZFS_DEBUG
 	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
 #endif
 	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
 	list_link_init(&hdr->b_l2hdr.b_l2node);
 	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
 
 	return (0);
 }
 
 static int
 hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
 {
 	(void) unused, (void) kmflag;
 	arc_buf_hdr_t *hdr = vbuf;
 
 	memset(hdr, 0, HDR_L2ONLY_SIZE);
 	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
 
 	return (0);
 }
 
 static int
 buf_cons(void *vbuf, void *unused, int kmflag)
 {
 	(void) unused, (void) kmflag;
 	arc_buf_t *buf = vbuf;
 
 	memset(buf, 0, sizeof (arc_buf_t));
 	arc_space_consume(sizeof (arc_buf_t), ARC_SPACE_HDRS);
 
 	return (0);
 }
 
 /*
  * Destructor callback - called when a cached buf is
  * no longer required.
  */
 static void
 hdr_full_dest(void *vbuf, void *unused)
 {
 	(void) unused;
 	arc_buf_hdr_t *hdr = vbuf;
 
 	ASSERT(HDR_EMPTY(hdr));
 	zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt);
 #ifdef ZFS_DEBUG
 	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
 #endif
 	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
 }
 
 static void
 hdr_l2only_dest(void *vbuf, void *unused)
 {
 	(void) unused;
 	arc_buf_hdr_t *hdr = vbuf;
 
 	ASSERT(HDR_EMPTY(hdr));
 	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
 }
 
 static void
 buf_dest(void *vbuf, void *unused)
 {
 	(void) unused;
 	(void) vbuf;
 
 	arc_space_return(sizeof (arc_buf_t), ARC_SPACE_HDRS);
 }
 
 static void
 buf_init(void)
 {
 	uint64_t *ct = NULL;
 	uint64_t hsize = 1ULL << 12;
 	int i, j;
 
 	/*
 	 * The hash table is big enough to fill all of physical memory
 	 * with an average block size of zfs_arc_average_blocksize (default 8K).
 	 * By default, the table will take up
 	 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
 	 */
 	while (hsize * zfs_arc_average_blocksize < arc_all_memory())
 		hsize <<= 1;
 retry:
 	buf_hash_table.ht_mask = hsize - 1;
 #if defined(_KERNEL)
 	/*
 	 * Large allocations which do not require contiguous pages
 	 * should be using vmem_alloc() in the linux kernel
 	 */
 	buf_hash_table.ht_table =
 	    vmem_zalloc(hsize * sizeof (void*), KM_SLEEP);
 #else
 	buf_hash_table.ht_table =
 	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
 #endif
 	if (buf_hash_table.ht_table == NULL) {
 		ASSERT(hsize > (1ULL << 8));
 		hsize >>= 1;
 		goto retry;
 	}
 
 	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
 	    0, hdr_full_cons, hdr_full_dest, NULL, NULL, NULL, KMC_RECLAIMABLE);
 	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
 	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, NULL,
 	    NULL, NULL, 0);
 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
 
 	for (i = 0; i < 256; i++)
 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
 			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
 
 	for (i = 0; i < BUF_LOCKS; i++)
 		mutex_init(BUF_HASH_LOCK(i), NULL, MUTEX_DEFAULT, NULL);
 }
 
 #define	ARC_MINTIME	(hz>>4) /* 62 ms */
 
 /*
  * This is the size that the buf occupies in memory. If the buf is compressed,
  * it will correspond to the compressed size. You should use this method of
  * getting the buf size unless you explicitly need the logical size.
  */
 uint64_t
 arc_buf_size(arc_buf_t *buf)
 {
 	return (ARC_BUF_COMPRESSED(buf) ?
 	    HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr));
 }
 
 uint64_t
 arc_buf_lsize(arc_buf_t *buf)
 {
 	return (HDR_GET_LSIZE(buf->b_hdr));
 }
 
 /*
  * This function will return B_TRUE if the buffer is encrypted in memory.
  * This buffer can be decrypted by calling arc_untransform().
  */
 boolean_t
 arc_is_encrypted(arc_buf_t *buf)
 {
 	return (ARC_BUF_ENCRYPTED(buf) != 0);
 }
 
 /*
  * Returns B_TRUE if the buffer represents data that has not had its MAC
  * verified yet.
  */
 boolean_t
 arc_is_unauthenticated(arc_buf_t *buf)
 {
 	return (HDR_NOAUTH(buf->b_hdr) != 0);
 }
 
 void
 arc_get_raw_params(arc_buf_t *buf, boolean_t *byteorder, uint8_t *salt,
     uint8_t *iv, uint8_t *mac)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT(HDR_PROTECTED(hdr));
 
 	memcpy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN);
 	memcpy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN);
 	memcpy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN);
 	*byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
 	    ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
 }
 
 /*
  * Indicates how this buffer is compressed in memory. If it is not compressed
  * the value will be ZIO_COMPRESS_OFF. It can be made normally readable with
  * arc_untransform() as long as it is also unencrypted.
  */
 enum zio_compress
 arc_get_compression(arc_buf_t *buf)
 {
 	return (ARC_BUF_COMPRESSED(buf) ?
 	    HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF);
 }
 
 /*
  * Return the compression algorithm used to store this data in the ARC. If ARC
  * compression is enabled or this is an encrypted block, this will be the same
  * as what's used to store it on-disk. Otherwise, this will be ZIO_COMPRESS_OFF.
  */
 static inline enum zio_compress
 arc_hdr_get_compress(arc_buf_hdr_t *hdr)
 {
 	return (HDR_COMPRESSION_ENABLED(hdr) ?
 	    HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF);
 }
 
 uint8_t
 arc_get_complevel(arc_buf_t *buf)
 {
 	return (buf->b_hdr->b_complevel);
 }
 
 static inline boolean_t
 arc_buf_is_shared(arc_buf_t *buf)
 {
 	boolean_t shared = (buf->b_data != NULL &&
 	    buf->b_hdr->b_l1hdr.b_pabd != NULL &&
 	    abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
 	    buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
 	IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr));
 	EQUIV(shared, ARC_BUF_SHARED(buf));
 	IMPLY(shared, ARC_BUF_COMPRESSED(buf) || ARC_BUF_LAST(buf));
 
 	/*
 	 * It would be nice to assert arc_can_share() too, but the "hdr isn't
 	 * already being shared" requirement prevents us from doing that.
 	 */
 
 	return (shared);
 }
 
 /*
  * Free the checksum associated with this header. If there is no checksum, this
  * is a no-op.
  */
 static inline void
 arc_cksum_free(arc_buf_hdr_t *hdr)
 {
 #ifdef ZFS_DEBUG
 	ASSERT(HDR_HAS_L1HDR(hdr));
 
 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
 	if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
 		kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
 		hdr->b_l1hdr.b_freeze_cksum = NULL;
 	}
 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
 #endif
 }
 
 /*
  * Return true iff at least one of the bufs on hdr is not compressed.
  * Encrypted buffers count as compressed.
  */
 static boolean_t
 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
 {
 	ASSERT(hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY_OR_LOCKED(hdr));
 
 	for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
 		if (!ARC_BUF_COMPRESSED(b)) {
 			return (B_TRUE);
 		}
 	}
 	return (B_FALSE);
 }
 
 
 /*
  * If we've turned on the ZFS_DEBUG_MODIFY flag, verify that the buf's data
  * matches the checksum that is stored in the hdr. If there is no checksum,
  * or if the buf is compressed, this is a no-op.
  */
 static void
 arc_cksum_verify(arc_buf_t *buf)
 {
 #ifdef ZFS_DEBUG
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	zio_cksum_t zc;
 
 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
 		return;
 
 	if (ARC_BUF_COMPRESSED(buf))
 		return;
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 
 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
 
 	if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
 		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
 		return;
 	}
 
 	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc);
 	if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
 		panic("buffer modified while frozen!");
 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
 #endif
 }
 
 /*
  * This function makes the assumption that data stored in the L2ARC
  * will be transformed exactly as it is in the main pool. Because of
  * this we can verify the checksum against the reading process's bp.
  */
 static boolean_t
 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
 {
 	ASSERT(!BP_IS_EMBEDDED(zio->io_bp));
 	VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
 
 	/*
 	 * Block pointers always store the checksum for the logical data.
 	 * If the block pointer has the gang bit set, then the checksum
 	 * it represents is for the reconstituted data and not for an
 	 * individual gang member. The zio pipeline, however, must be able to
 	 * determine the checksum of each of the gang constituents so it
 	 * treats the checksum comparison differently than what we need
 	 * for l2arc blocks. This prevents us from using the
 	 * zio_checksum_error() interface directly. Instead we must call the
 	 * zio_checksum_error_impl() so that we can ensure the checksum is
 	 * generated using the correct checksum algorithm and accounts for the
 	 * logical I/O size and not just a gang fragment.
 	 */
 	return (zio_checksum_error_impl(zio->io_spa, zio->io_bp,
 	    BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,
 	    zio->io_offset, NULL) == 0);
 }
 
 /*
  * Given a buf full of data, if ZFS_DEBUG_MODIFY is enabled this computes a
  * checksum and attaches it to the buf's hdr so that we can ensure that the buf
  * isn't modified later on. If buf is compressed or there is already a checksum
  * on the hdr, this is a no-op (we only checksum uncompressed bufs).
  */
 static void
 arc_cksum_compute(arc_buf_t *buf)
 {
 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
 		return;
 
 #ifdef ZFS_DEBUG
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
 	if (hdr->b_l1hdr.b_freeze_cksum != NULL || ARC_BUF_COMPRESSED(buf)) {
 		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
 		return;
 	}
 
 	ASSERT(!ARC_BUF_ENCRYPTED(buf));
 	ASSERT(!ARC_BUF_COMPRESSED(buf));
 	hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
 	    KM_SLEEP);
 	fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL,
 	    hdr->b_l1hdr.b_freeze_cksum);
 	mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
 #endif
 	arc_buf_watch(buf);
 }
 
 #ifndef _KERNEL
 void
 arc_buf_sigsegv(int sig, siginfo_t *si, void *unused)
 {
 	(void) sig, (void) unused;
 	panic("Got SIGSEGV at address: 0x%lx\n", (long)si->si_addr);
 }
 #endif
 
 static void
 arc_buf_unwatch(arc_buf_t *buf)
 {
 #ifndef _KERNEL
 	if (arc_watch) {
 		ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
 		    PROT_READ | PROT_WRITE));
 	}
 #else
 	(void) buf;
 #endif
 }
 
 static void
 arc_buf_watch(arc_buf_t *buf)
 {
 #ifndef _KERNEL
 	if (arc_watch)
 		ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
 		    PROT_READ));
 #else
 	(void) buf;
 #endif
 }
 
 static arc_buf_contents_t
 arc_buf_type(arc_buf_hdr_t *hdr)
 {
 	arc_buf_contents_t type;
 	if (HDR_ISTYPE_METADATA(hdr)) {
 		type = ARC_BUFC_METADATA;
 	} else {
 		type = ARC_BUFC_DATA;
 	}
 	VERIFY3U(hdr->b_type, ==, type);
 	return (type);
 }
 
 boolean_t
 arc_is_metadata(arc_buf_t *buf)
 {
 	return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0);
 }
 
 static uint32_t
 arc_bufc_to_flags(arc_buf_contents_t type)
 {
 	switch (type) {
 	case ARC_BUFC_DATA:
 		/* metadata field is 0 if buffer contains normal data */
 		return (0);
 	case ARC_BUFC_METADATA:
 		return (ARC_FLAG_BUFC_METADATA);
 	default:
 		break;
 	}
 	panic("undefined ARC buffer type!");
 	return ((uint32_t)-1);
 }
 
 void
 arc_buf_thaw(arc_buf_t *buf)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 
 	arc_cksum_verify(buf);
 
 	/*
 	 * Compressed buffers do not manipulate the b_freeze_cksum.
 	 */
 	if (ARC_BUF_COMPRESSED(buf))
 		return;
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	arc_cksum_free(hdr);
 	arc_buf_unwatch(buf);
 }
 
 void
 arc_buf_freeze(arc_buf_t *buf)
 {
 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
 		return;
 
 	if (ARC_BUF_COMPRESSED(buf))
 		return;
 
 	ASSERT(HDR_HAS_L1HDR(buf->b_hdr));
 	arc_cksum_compute(buf);
 }
 
 /*
  * The arc_buf_hdr_t's b_flags should never be modified directly. Instead,
  * the following functions should be used to ensure that the flags are
  * updated in a thread-safe way. When manipulating the flags either
  * the hash_lock must be held or the hdr must be undiscoverable. This
  * ensures that we're not racing with any other threads when updating
  * the flags.
  */
 static inline void
 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
 {
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 	hdr->b_flags |= flags;
 }
 
 static inline void
 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
 {
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 	hdr->b_flags &= ~flags;
 }
 
 /*
  * Setting the compression bits in the arc_buf_hdr_t's b_flags is
  * done in a special way since we have to clear and set bits
  * at the same time. Consumers that wish to set the compression bits
  * must use this function to ensure that the flags are updated in
  * thread-safe manner.
  */
 static void
 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
 {
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 
 	/*
 	 * Holes and embedded blocks will always have a psize = 0 so
 	 * we ignore the compression of the blkptr and set the
 	 * want to uncompress them. Mark them as uncompressed.
 	 */
 	if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
 		arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
 		ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
 	} else {
 		arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
 		ASSERT(HDR_COMPRESSION_ENABLED(hdr));
 	}
 
 	HDR_SET_COMPRESS(hdr, cmp);
 	ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
 }
 
 /*
  * Looks for another buf on the same hdr which has the data decompressed, copies
  * from it, and returns true. If no such buf exists, returns false.
  */
 static boolean_t
 arc_buf_try_copy_decompressed_data(arc_buf_t *buf)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	boolean_t copied = B_FALSE;
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT3P(buf->b_data, !=, NULL);
 	ASSERT(!ARC_BUF_COMPRESSED(buf));
 
 	for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
 	    from = from->b_next) {
 		/* can't use our own data buffer */
 		if (from == buf) {
 			continue;
 		}
 
 		if (!ARC_BUF_COMPRESSED(from)) {
 			memcpy(buf->b_data, from->b_data, arc_buf_size(buf));
 			copied = B_TRUE;
 			break;
 		}
 	}
 
 #ifdef ZFS_DEBUG
 	/*
 	 * There were no decompressed bufs, so there should not be a
 	 * checksum on the hdr either.
 	 */
 	if (zfs_flags & ZFS_DEBUG_MODIFY)
 		EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
 #endif
 
 	return (copied);
 }
 
 /*
  * Allocates an ARC buf header that's in an evicted & L2-cached state.
  * This is used during l2arc reconstruction to make empty ARC buffers
  * which circumvent the regular disk->arc->l2arc path and instead come
  * into being in the reverse order, i.e. l2arc->arc.
  */
 static arc_buf_hdr_t *
 arc_buf_alloc_l2only(size_t size, arc_buf_contents_t type, l2arc_dev_t *dev,
     dva_t dva, uint64_t daddr, int32_t psize, uint64_t birth,
     enum zio_compress compress, uint8_t complevel, boolean_t protected,
     boolean_t prefetch, arc_state_type_t arcs_state)
 {
 	arc_buf_hdr_t	*hdr;
 
 	ASSERT(size != 0);
 	hdr = kmem_cache_alloc(hdr_l2only_cache, KM_SLEEP);
 	hdr->b_birth = birth;
 	hdr->b_type = type;
 	hdr->b_flags = 0;
 	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L2HDR);
 	HDR_SET_LSIZE(hdr, size);
 	HDR_SET_PSIZE(hdr, psize);
 	arc_hdr_set_compress(hdr, compress);
 	hdr->b_complevel = complevel;
 	if (protected)
 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
 	if (prefetch)
 		arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
 	hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa);
 
 	hdr->b_dva = dva;
 
 	hdr->b_l2hdr.b_dev = dev;
 	hdr->b_l2hdr.b_daddr = daddr;
 	hdr->b_l2hdr.b_arcs_state = arcs_state;
 
 	return (hdr);
 }
 
 /*
  * Return the size of the block, b_pabd, that is stored in the arc_buf_hdr_t.
  */
 static uint64_t
 arc_hdr_size(arc_buf_hdr_t *hdr)
 {
 	uint64_t size;
 
 	if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
 	    HDR_GET_PSIZE(hdr) > 0) {
 		size = HDR_GET_PSIZE(hdr);
 	} else {
 		ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
 		size = HDR_GET_LSIZE(hdr);
 	}
 	return (size);
 }
 
 static int
 arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
 {
 	int ret;
 	uint64_t csize;
 	uint64_t lsize = HDR_GET_LSIZE(hdr);
 	uint64_t psize = HDR_GET_PSIZE(hdr);
 	void *tmpbuf = NULL;
 	abd_t *abd = hdr->b_l1hdr.b_pabd;
 
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 	ASSERT(HDR_AUTHENTICATED(hdr));
 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
 
 	/*
 	 * The MAC is calculated on the compressed data that is stored on disk.
 	 * However, if compressed arc is disabled we will only have the
 	 * decompressed data available to us now. Compress it into a temporary
 	 * abd so we can verify the MAC. The performance overhead of this will
 	 * be relatively low, since most objects in an encrypted objset will
 	 * be encrypted (instead of authenticated) anyway.
 	 */
 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
 	    !HDR_COMPRESSION_ENABLED(hdr)) {
 
 		csize = zio_compress_data(HDR_GET_COMPRESS(hdr),
 		    hdr->b_l1hdr.b_pabd, &tmpbuf, lsize, hdr->b_complevel);
 		ASSERT3P(tmpbuf, !=, NULL);
 		ASSERT3U(csize, <=, psize);
 		abd = abd_get_from_buf(tmpbuf, lsize);
 		abd_take_ownership_of_buf(abd, B_TRUE);
 		abd_zero_off(abd, csize, psize - csize);
 	}
 
 	/*
 	 * Authentication is best effort. We authenticate whenever the key is
 	 * available. If we succeed we clear ARC_FLAG_NOAUTH.
 	 */
 	if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) {
 		ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
 		ASSERT3U(lsize, ==, psize);
 		ret = spa_do_crypt_objset_mac_abd(B_FALSE, spa, dsobj, abd,
 		    psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
 	} else {
 		ret = spa_do_crypt_mac_abd(B_FALSE, spa, dsobj, abd, psize,
 		    hdr->b_crypt_hdr.b_mac);
 	}
 
 	if (ret == 0)
 		arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH);
 	else if (ret != ENOENT)
 		goto error;
 
 	if (tmpbuf != NULL)
 		abd_free(abd);
 
 	return (0);
 
 error:
 	if (tmpbuf != NULL)
 		abd_free(abd);
 
 	return (ret);
 }
 
 /*
  * This function will take a header that only has raw encrypted data in
  * b_crypt_hdr.b_rabd and decrypt it into a new buffer which is stored in
  * b_l1hdr.b_pabd. If designated in the header flags, this function will
  * also decompress the data.
  */
 static int
 arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)
 {
 	int ret;
 	abd_t *cabd = NULL;
 	void *tmp = NULL;
 	boolean_t no_crypt = B_FALSE;
 	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
 
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 	ASSERT(HDR_ENCRYPTED(hdr));
 
 	arc_hdr_alloc_abd(hdr, 0);
 
 	ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot,
 	    B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv,
 	    hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd,
 	    hdr->b_crypt_hdr.b_rabd, &no_crypt);
 	if (ret != 0)
 		goto error;
 
 	if (no_crypt) {
 		abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd,
 		    HDR_GET_PSIZE(hdr));
 	}
 
 	/*
 	 * If this header has disabled arc compression but the b_pabd is
 	 * compressed after decrypting it, we need to decompress the newly
 	 * decrypted data.
 	 */
 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
 	    !HDR_COMPRESSION_ENABLED(hdr)) {
 		/*
 		 * We want to make sure that we are correctly honoring the
 		 * zfs_abd_scatter_enabled setting, so we allocate an abd here
 		 * and then loan a buffer from it, rather than allocating a
 		 * linear buffer and wrapping it in an abd later.
 		 */
 		cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0);
 		tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr));
 
 		ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
 		    hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
 		    HDR_GET_LSIZE(hdr), &hdr->b_complevel);
 		if (ret != 0) {
 			abd_return_buf(cabd, tmp, arc_hdr_size(hdr));
 			goto error;
 		}
 
 		abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
 		    arc_hdr_size(hdr), hdr);
 		hdr->b_l1hdr.b_pabd = cabd;
 	}
 
 	return (0);
 
 error:
 	arc_hdr_free_abd(hdr, B_FALSE);
 	if (cabd != NULL)
 		arc_free_data_buf(hdr, cabd, arc_hdr_size(hdr), hdr);
 
 	return (ret);
 }
 
 /*
  * This function is called during arc_buf_fill() to prepare the header's
  * abd plaintext pointer for use. This involves authenticated protected
  * data and decrypting encrypted data into the plaintext abd.
  */
 static int
 arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa,
     const zbookmark_phys_t *zb, boolean_t noauth)
 {
 	int ret;
 
 	ASSERT(HDR_PROTECTED(hdr));
 
 	if (hash_lock != NULL)
 		mutex_enter(hash_lock);
 
 	if (HDR_NOAUTH(hdr) && !noauth) {
 		/*
 		 * The caller requested authenticated data but our data has
 		 * not been authenticated yet. Verify the MAC now if we can.
 		 */
 		ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset);
 		if (ret != 0)
 			goto error;
 	} else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) {
 		/*
 		 * If we only have the encrypted version of the data, but the
 		 * unencrypted version was requested we take this opportunity
 		 * to store the decrypted version in the header for future use.
 		 */
 		ret = arc_hdr_decrypt(hdr, spa, zb);
 		if (ret != 0)
 			goto error;
 	}
 
 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
 
 	if (hash_lock != NULL)
 		mutex_exit(hash_lock);
 
 	return (0);
 
 error:
 	if (hash_lock != NULL)
 		mutex_exit(hash_lock);
 
 	return (ret);
 }
 
 /*
  * This function is used by the dbuf code to decrypt bonus buffers in place.
  * The dbuf code itself doesn't have any locking for decrypting a shared dnode
  * block, so we use the hash lock here to protect against concurrent calls to
  * arc_buf_fill().
  */
 static void
 arc_buf_untransform_in_place(arc_buf_t *buf)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT(HDR_ENCRYPTED(hdr));
 	ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 	ASSERT3PF(hdr->b_l1hdr.b_pabd, !=, NULL, "hdr %px buf %px", hdr, buf);
 
 	zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data,
 	    arc_buf_size(buf));
 	buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
 	buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
 }
 
 /*
  * Given a buf that has a data buffer attached to it, this function will
  * efficiently fill the buf with data of the specified compression setting from
  * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
  * are already sharing a data buf, no copy is performed.
  *
  * If the buf is marked as compressed but uncompressed data was requested, this
  * will allocate a new data buffer for the buf, remove that flag, and fill the
  * buf with uncompressed data. You can't request a compressed buf on a hdr with
  * uncompressed data, and (since we haven't added support for it yet) if you
  * want compressed data your buf must already be marked as compressed and have
  * the correct-sized data buffer.
  */
 static int
 arc_buf_fill(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
     arc_fill_flags_t flags)
 {
 	int error = 0;
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	boolean_t hdr_compressed =
 	    (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
 	boolean_t compressed = (flags & ARC_FILL_COMPRESSED) != 0;
 	boolean_t encrypted = (flags & ARC_FILL_ENCRYPTED) != 0;
 	dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
 	kmutex_t *hash_lock = (flags & ARC_FILL_LOCKED) ? NULL : HDR_LOCK(hdr);
 
 	ASSERT3P(buf->b_data, !=, NULL);
 	IMPLY(compressed, hdr_compressed || ARC_BUF_ENCRYPTED(buf));
 	IMPLY(compressed, ARC_BUF_COMPRESSED(buf));
 	IMPLY(encrypted, HDR_ENCRYPTED(hdr));
 	IMPLY(encrypted, ARC_BUF_ENCRYPTED(buf));
 	IMPLY(encrypted, ARC_BUF_COMPRESSED(buf));
 	IMPLY(encrypted, !arc_buf_is_shared(buf));
 
 	/*
 	 * If the caller wanted encrypted data we just need to copy it from
 	 * b_rabd and potentially byteswap it. We won't be able to do any
 	 * further transforms on it.
 	 */
 	if (encrypted) {
 		ASSERT(HDR_HAS_RABD(hdr));
 		abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd,
 		    HDR_GET_PSIZE(hdr));
 		goto byteswap;
 	}
 
 	/*
 	 * Adjust encrypted and authenticated headers to accommodate
 	 * the request if needed. Dnode blocks (ARC_FILL_IN_PLACE) are
 	 * allowed to fail decryption due to keys not being loaded
 	 * without being marked as an IO error.
 	 */
 	if (HDR_PROTECTED(hdr)) {
 		error = arc_fill_hdr_crypt(hdr, hash_lock, spa,
 		    zb, !!(flags & ARC_FILL_NOAUTH));
 		if (error == EACCES && (flags & ARC_FILL_IN_PLACE) != 0) {
 			return (error);
 		} else if (error != 0) {
 			if (hash_lock != NULL)
 				mutex_enter(hash_lock);
 			arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
 			if (hash_lock != NULL)
 				mutex_exit(hash_lock);
 			return (error);
 		}
 	}
 
 	/*
 	 * There is a special case here for dnode blocks which are
 	 * decrypting their bonus buffers. These blocks may request to
 	 * be decrypted in-place. This is necessary because there may
 	 * be many dnodes pointing into this buffer and there is
 	 * currently no method to synchronize replacing the backing
 	 * b_data buffer and updating all of the pointers. Here we use
 	 * the hash lock to ensure there are no races. If the need
 	 * arises for other types to be decrypted in-place, they must
 	 * add handling here as well.
 	 */
 	if ((flags & ARC_FILL_IN_PLACE) != 0) {
 		ASSERT(!hdr_compressed);
 		ASSERT(!compressed);
 		ASSERT(!encrypted);
 
 		if (HDR_ENCRYPTED(hdr) && ARC_BUF_ENCRYPTED(buf)) {
 			ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
 
 			if (hash_lock != NULL)
 				mutex_enter(hash_lock);
 			arc_buf_untransform_in_place(buf);
 			if (hash_lock != NULL)
 				mutex_exit(hash_lock);
 
 			/* Compute the hdr's checksum if necessary */
 			arc_cksum_compute(buf);
 		}
 
 		return (0);
 	}
 
 	if (hdr_compressed == compressed) {
 		if (ARC_BUF_SHARED(buf)) {
 			ASSERT(arc_buf_is_shared(buf));
 		} else {
 			abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
 			    arc_buf_size(buf));
 		}
 	} else {
 		ASSERT(hdr_compressed);
 		ASSERT(!compressed);
 
 		/*
 		 * If the buf is sharing its data with the hdr, unlink it and
 		 * allocate a new data buffer for the buf.
 		 */
 		if (ARC_BUF_SHARED(buf)) {
 			ASSERTF(ARC_BUF_COMPRESSED(buf),
 			"buf %p was uncompressed", buf);
 
 			/* We need to give the buf its own b_data */
 			buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
 			buf->b_data =
 			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
 			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
 
 			/* Previously overhead was 0; just add new overhead */
 			ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
 		} else if (ARC_BUF_COMPRESSED(buf)) {
 			ASSERT(!arc_buf_is_shared(buf));
 
 			/* We need to reallocate the buf's b_data */
 			arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
 			    buf);
 			buf->b_data =
 			    arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
 
 			/* We increased the size of b_data; update overhead */
 			ARCSTAT_INCR(arcstat_overhead_size,
 			    HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
 		}
 
 		/*
 		 * Regardless of the buf's previous compression settings, it
 		 * should not be compressed at the end of this function.
 		 */
 		buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
 
 		/*
 		 * Try copying the data from another buf which already has a
 		 * decompressed version. If that's not possible, it's time to
 		 * bite the bullet and decompress the data from the hdr.
 		 */
 		if (arc_buf_try_copy_decompressed_data(buf)) {
 			/* Skip byteswapping and checksumming (already done) */
 			return (0);
 		} else {
 			error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
 			    hdr->b_l1hdr.b_pabd, buf->b_data,
 			    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr),
 			    &hdr->b_complevel);
 
 			/*
 			 * Absent hardware errors or software bugs, this should
 			 * be impossible, but log it anyway so we can debug it.
 			 */
 			if (error != 0) {
 				zfs_dbgmsg(
 				    "hdr %px, compress %d, psize %d, lsize %d",
 				    hdr, arc_hdr_get_compress(hdr),
 				    HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
 				if (hash_lock != NULL)
 					mutex_enter(hash_lock);
 				arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
 				if (hash_lock != NULL)
 					mutex_exit(hash_lock);
 				return (SET_ERROR(EIO));
 			}
 		}
 	}
 
 byteswap:
 	/* Byteswap the buf's data if necessary */
 	if (bswap != DMU_BSWAP_NUMFUNCS) {
 		ASSERT(!HDR_SHARED_DATA(hdr));
 		ASSERT3U(bswap, <, DMU_BSWAP_NUMFUNCS);
 		dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
 	}
 
 	/* Compute the hdr's checksum if necessary */
 	arc_cksum_compute(buf);
 
 	return (0);
 }
 
 /*
  * If this function is being called to decrypt an encrypted buffer or verify an
  * authenticated one, the key must be loaded and a mapping must be made
  * available in the keystore via spa_keystore_create_mapping() or one of its
  * callers.
  */
 int
 arc_untransform(arc_buf_t *buf, spa_t *spa, const zbookmark_phys_t *zb,
     boolean_t in_place)
 {
 	int ret;
 	arc_fill_flags_t flags = 0;
 
 	if (in_place)
 		flags |= ARC_FILL_IN_PLACE;
 
 	ret = arc_buf_fill(buf, spa, zb, flags);
 	if (ret == ECKSUM) {
 		/*
 		 * Convert authentication and decryption errors to EIO
 		 * (and generate an ereport) before leaving the ARC.
 		 */
 		ret = SET_ERROR(EIO);
 		spa_log_error(spa, zb, &buf->b_hdr->b_birth);
 		(void) zfs_ereport_post(FM_EREPORT_ZFS_AUTHENTICATION,
 		    spa, NULL, zb, NULL, 0);
 	}
 
 	return (ret);
 }
 
 /*
  * Increment the amount of evictable space in the arc_state_t's refcount.
  * We account for the space used by the hdr and the arc buf individually
  * so that we can add and remove them from the refcount individually.
  */
 static void
 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
 {
 	arc_buf_contents_t type = arc_buf_type(hdr);
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 
 	if (GHOST_STATE(state)) {
 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
 		ASSERT(!HDR_HAS_RABD(hdr));
 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
 		    HDR_GET_LSIZE(hdr), hdr);
 		return;
 	}
 
 	if (hdr->b_l1hdr.b_pabd != NULL) {
 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
 		    arc_hdr_size(hdr), hdr);
 	}
 	if (HDR_HAS_RABD(hdr)) {
 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
 		    HDR_GET_PSIZE(hdr), hdr);
 	}
 
 	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
 	    buf = buf->b_next) {
 		if (ARC_BUF_SHARED(buf))
 			continue;
 		(void) zfs_refcount_add_many(&state->arcs_esize[type],
 		    arc_buf_size(buf), buf);
 	}
 }
 
 /*
  * Decrement the amount of evictable space in the arc_state_t's refcount.
  * We account for the space used by the hdr and the arc buf individually
  * so that we can add and remove them from the refcount individually.
  */
 static void
 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
 {
 	arc_buf_contents_t type = arc_buf_type(hdr);
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 
 	if (GHOST_STATE(state)) {
 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
 		ASSERT(!HDR_HAS_RABD(hdr));
 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
 		    HDR_GET_LSIZE(hdr), hdr);
 		return;
 	}
 
 	if (hdr->b_l1hdr.b_pabd != NULL) {
 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
 		    arc_hdr_size(hdr), hdr);
 	}
 	if (HDR_HAS_RABD(hdr)) {
 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
 		    HDR_GET_PSIZE(hdr), hdr);
 	}
 
 	for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
 	    buf = buf->b_next) {
 		if (ARC_BUF_SHARED(buf))
 			continue;
 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
 		    arc_buf_size(buf), buf);
 	}
 }
 
 /*
  * Add a reference to this hdr indicating that someone is actively
  * referencing that memory. When the refcount transitions from 0 to 1,
  * we remove it from the respective arc_state_t list to indicate that
  * it is not evictable.
  */
 static void
 add_reference(arc_buf_hdr_t *hdr, const void *tag)
 {
 	arc_state_t *state = hdr->b_l1hdr.b_state;
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	if (!HDR_EMPTY(hdr) && !MUTEX_HELD(HDR_LOCK(hdr))) {
 		ASSERT(state == arc_anon);
 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
 	}
 
 	if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
 	    state != arc_anon && state != arc_l2c_only) {
 		/* We don't use the L2-only state list. */
 		multilist_remove(&state->arcs_list[arc_buf_type(hdr)], hdr);
 		arc_evictable_space_decrement(hdr, state);
 	}
 }
 
 /*
  * Remove a reference from this hdr. When the reference transitions from
  * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
  * list making it eligible for eviction.
  */
 static int
 remove_reference(arc_buf_hdr_t *hdr, const void *tag)
 {
 	int cnt;
 	arc_state_t *state = hdr->b_l1hdr.b_state;
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(state == arc_anon || MUTEX_HELD(HDR_LOCK(hdr)));
 	ASSERT(!GHOST_STATE(state));	/* arc_l2c_only counts as a ghost. */
 
 	if ((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) != 0)
 		return (cnt);
 
 	if (state == arc_anon) {
 		arc_hdr_destroy(hdr);
 		return (0);
 	}
 	if (state == arc_uncached && !HDR_PREFETCH(hdr)) {
 		arc_change_state(arc_anon, hdr);
 		arc_hdr_destroy(hdr);
 		return (0);
 	}
 	multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
 	arc_evictable_space_increment(hdr, state);
 	return (0);
 }
 
 /*
  * Returns detailed information about a specific arc buffer.  When the
  * state_index argument is set the function will calculate the arc header
  * list position for its arc state.  Since this requires a linear traversal
  * callers are strongly encourage not to do this.  However, it can be helpful
  * for targeted analysis so the functionality is provided.
  */
 void
 arc_buf_info(arc_buf_t *ab, arc_buf_info_t *abi, int state_index)
 {
 	(void) state_index;
 	arc_buf_hdr_t *hdr = ab->b_hdr;
 	l1arc_buf_hdr_t *l1hdr = NULL;
 	l2arc_buf_hdr_t *l2hdr = NULL;
 	arc_state_t *state = NULL;
 
 	memset(abi, 0, sizeof (arc_buf_info_t));
 
 	if (hdr == NULL)
 		return;
 
 	abi->abi_flags = hdr->b_flags;
 
 	if (HDR_HAS_L1HDR(hdr)) {
 		l1hdr = &hdr->b_l1hdr;
 		state = l1hdr->b_state;
 	}
 	if (HDR_HAS_L2HDR(hdr))
 		l2hdr = &hdr->b_l2hdr;
 
 	if (l1hdr) {
 		abi->abi_bufcnt = 0;
 		for (arc_buf_t *buf = l1hdr->b_buf; buf; buf = buf->b_next)
 			abi->abi_bufcnt++;
 		abi->abi_access = l1hdr->b_arc_access;
 		abi->abi_mru_hits = l1hdr->b_mru_hits;
 		abi->abi_mru_ghost_hits = l1hdr->b_mru_ghost_hits;
 		abi->abi_mfu_hits = l1hdr->b_mfu_hits;
 		abi->abi_mfu_ghost_hits = l1hdr->b_mfu_ghost_hits;
 		abi->abi_holds = zfs_refcount_count(&l1hdr->b_refcnt);
 	}
 
 	if (l2hdr) {
 		abi->abi_l2arc_dattr = l2hdr->b_daddr;
 		abi->abi_l2arc_hits = l2hdr->b_hits;
 	}
 
 	abi->abi_state_type = state ? state->arcs_state : ARC_STATE_ANON;
 	abi->abi_state_contents = arc_buf_type(hdr);
 	abi->abi_size = arc_hdr_size(hdr);
 }
 
 /*
  * Move the supplied buffer to the indicated state. The hash lock
  * for the buffer must be held by the caller.
  */
 static void
 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
 {
 	arc_state_t *old_state;
 	int64_t refcnt;
 	boolean_t update_old, update_new;
 	arc_buf_contents_t type = arc_buf_type(hdr);
 
 	/*
 	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
 	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
 	 * L1 hdr doesn't always exist when we change state to arc_anon before
 	 * destroying a header, in which case reallocating to add the L1 hdr is
 	 * pointless.
 	 */
 	if (HDR_HAS_L1HDR(hdr)) {
 		old_state = hdr->b_l1hdr.b_state;
 		refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt);
 		update_old = (hdr->b_l1hdr.b_buf != NULL ||
 		    hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
 
 		IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL);
 		IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL);
 		IMPLY(old_state == arc_anon, hdr->b_l1hdr.b_buf == NULL ||
 		    ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
 	} else {
 		old_state = arc_l2c_only;
 		refcnt = 0;
 		update_old = B_FALSE;
 	}
 	update_new = update_old;
 	if (GHOST_STATE(old_state))
 		update_old = B_TRUE;
 	if (GHOST_STATE(new_state))
 		update_new = B_TRUE;
 
 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
 	ASSERT3P(new_state, !=, old_state);
 
 	/*
 	 * If this buffer is evictable, transfer it from the
 	 * old state list to the new state list.
 	 */
 	if (refcnt == 0) {
 		if (old_state != arc_anon && old_state != arc_l2c_only) {
 			ASSERT(HDR_HAS_L1HDR(hdr));
 			/* remove_reference() saves on insert. */
 			if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
 				multilist_remove(&old_state->arcs_list[type],
 				    hdr);
 				arc_evictable_space_decrement(hdr, old_state);
 			}
 		}
 		if (new_state != arc_anon && new_state != arc_l2c_only) {
 			/*
 			 * An L1 header always exists here, since if we're
 			 * moving to some L1-cached state (i.e. not l2c_only or
 			 * anonymous), we realloc the header to add an L1hdr
 			 * beforehand.
 			 */
 			ASSERT(HDR_HAS_L1HDR(hdr));
 			multilist_insert(&new_state->arcs_list[type], hdr);
 			arc_evictable_space_increment(hdr, new_state);
 		}
 	}
 
 	ASSERT(!HDR_EMPTY(hdr));
 	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
 		buf_hash_remove(hdr);
 
 	/* adjust state sizes (ignore arc_l2c_only) */
 
 	if (update_new && new_state != arc_l2c_only) {
 		ASSERT(HDR_HAS_L1HDR(hdr));
 		if (GHOST_STATE(new_state)) {
 
 			/*
 			 * When moving a header to a ghost state, we first
 			 * remove all arc buffers. Thus, we'll have no arc
 			 * buffer to use for the reference. As a result, we
 			 * use the arc header pointer for the reference.
 			 */
 			(void) zfs_refcount_add_many(
 			    &new_state->arcs_size[type],
 			    HDR_GET_LSIZE(hdr), hdr);
 			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
 			ASSERT(!HDR_HAS_RABD(hdr));
 		} else {
 
 			/*
 			 * Each individual buffer holds a unique reference,
 			 * thus we must remove each of these references one
 			 * at a time.
 			 */
 			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
 			    buf = buf->b_next) {
 
 				/*
 				 * When the arc_buf_t is sharing the data
 				 * block with the hdr, the owner of the
 				 * reference belongs to the hdr. Only
 				 * add to the refcount if the arc_buf_t is
 				 * not shared.
 				 */
 				if (ARC_BUF_SHARED(buf))
 					continue;
 
 				(void) zfs_refcount_add_many(
 				    &new_state->arcs_size[type],
 				    arc_buf_size(buf), buf);
 			}
 
 			if (hdr->b_l1hdr.b_pabd != NULL) {
 				(void) zfs_refcount_add_many(
 				    &new_state->arcs_size[type],
 				    arc_hdr_size(hdr), hdr);
 			}
 
 			if (HDR_HAS_RABD(hdr)) {
 				(void) zfs_refcount_add_many(
 				    &new_state->arcs_size[type],
 				    HDR_GET_PSIZE(hdr), hdr);
 			}
 		}
 	}
 
 	if (update_old && old_state != arc_l2c_only) {
 		ASSERT(HDR_HAS_L1HDR(hdr));
 		if (GHOST_STATE(old_state)) {
 			ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
 			ASSERT(!HDR_HAS_RABD(hdr));
 
 			/*
 			 * When moving a header off of a ghost state,
 			 * the header will not contain any arc buffers.
 			 * We use the arc header pointer for the reference
 			 * which is exactly what we did when we put the
 			 * header on the ghost state.
 			 */
 
 			(void) zfs_refcount_remove_many(
 			    &old_state->arcs_size[type],
 			    HDR_GET_LSIZE(hdr), hdr);
 		} else {
 
 			/*
 			 * Each individual buffer holds a unique reference,
 			 * thus we must remove each of these references one
 			 * at a time.
 			 */
 			for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
 			    buf = buf->b_next) {
 
 				/*
 				 * When the arc_buf_t is sharing the data
 				 * block with the hdr, the owner of the
 				 * reference belongs to the hdr. Only
 				 * add to the refcount if the arc_buf_t is
 				 * not shared.
 				 */
 				if (ARC_BUF_SHARED(buf))
 					continue;
 
 				(void) zfs_refcount_remove_many(
 				    &old_state->arcs_size[type],
 				    arc_buf_size(buf), buf);
 			}
 			ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
 			    HDR_HAS_RABD(hdr));
 
 			if (hdr->b_l1hdr.b_pabd != NULL) {
 				(void) zfs_refcount_remove_many(
 				    &old_state->arcs_size[type],
 				    arc_hdr_size(hdr), hdr);
 			}
 
 			if (HDR_HAS_RABD(hdr)) {
 				(void) zfs_refcount_remove_many(
 				    &old_state->arcs_size[type],
 				    HDR_GET_PSIZE(hdr), hdr);
 			}
 		}
 	}
 
 	if (HDR_HAS_L1HDR(hdr)) {
 		hdr->b_l1hdr.b_state = new_state;
 
 		if (HDR_HAS_L2HDR(hdr) && new_state != arc_l2c_only) {
 			l2arc_hdr_arcstats_decrement_state(hdr);
 			hdr->b_l2hdr.b_arcs_state = new_state->arcs_state;
 			l2arc_hdr_arcstats_increment_state(hdr);
 		}
 	}
 }
 
 void
 arc_space_consume(uint64_t space, arc_space_type_t type)
 {
 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
 
 	switch (type) {
 	default:
 		break;
 	case ARC_SPACE_DATA:
 		ARCSTAT_INCR(arcstat_data_size, space);
 		break;
 	case ARC_SPACE_META:
 		ARCSTAT_INCR(arcstat_metadata_size, space);
 		break;
 	case ARC_SPACE_BONUS:
 		ARCSTAT_INCR(arcstat_bonus_size, space);
 		break;
 	case ARC_SPACE_DNODE:
 		ARCSTAT_INCR(arcstat_dnode_size, space);
 		break;
 	case ARC_SPACE_DBUF:
 		ARCSTAT_INCR(arcstat_dbuf_size, space);
 		break;
 	case ARC_SPACE_HDRS:
 		ARCSTAT_INCR(arcstat_hdr_size, space);
 		break;
 	case ARC_SPACE_L2HDRS:
 		aggsum_add(&arc_sums.arcstat_l2_hdr_size, space);
 		break;
 	case ARC_SPACE_ABD_CHUNK_WASTE:
 		/*
 		 * Note: this includes space wasted by all scatter ABD's, not
 		 * just those allocated by the ARC.  But the vast majority of
 		 * scatter ABD's come from the ARC, because other users are
 		 * very short-lived.
 		 */
 		ARCSTAT_INCR(arcstat_abd_chunk_waste_size, space);
 		break;
 	}
 
 	if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
 		ARCSTAT_INCR(arcstat_meta_used, space);
 
 	aggsum_add(&arc_sums.arcstat_size, space);
 }
 
 void
 arc_space_return(uint64_t space, arc_space_type_t type)
 {
 	ASSERT(type >= 0 && type < ARC_SPACE_NUMTYPES);
 
 	switch (type) {
 	default:
 		break;
 	case ARC_SPACE_DATA:
 		ARCSTAT_INCR(arcstat_data_size, -space);
 		break;
 	case ARC_SPACE_META:
 		ARCSTAT_INCR(arcstat_metadata_size, -space);
 		break;
 	case ARC_SPACE_BONUS:
 		ARCSTAT_INCR(arcstat_bonus_size, -space);
 		break;
 	case ARC_SPACE_DNODE:
 		ARCSTAT_INCR(arcstat_dnode_size, -space);
 		break;
 	case ARC_SPACE_DBUF:
 		ARCSTAT_INCR(arcstat_dbuf_size, -space);
 		break;
 	case ARC_SPACE_HDRS:
 		ARCSTAT_INCR(arcstat_hdr_size, -space);
 		break;
 	case ARC_SPACE_L2HDRS:
 		aggsum_add(&arc_sums.arcstat_l2_hdr_size, -space);
 		break;
 	case ARC_SPACE_ABD_CHUNK_WASTE:
 		ARCSTAT_INCR(arcstat_abd_chunk_waste_size, -space);
 		break;
 	}
 
 	if (type != ARC_SPACE_DATA && type != ARC_SPACE_ABD_CHUNK_WASTE)
 		ARCSTAT_INCR(arcstat_meta_used, -space);
 
 	ASSERT(aggsum_compare(&arc_sums.arcstat_size, space) >= 0);
 	aggsum_add(&arc_sums.arcstat_size, -space);
 }
 
 /*
  * Given a hdr and a buf, returns whether that buf can share its b_data buffer
  * with the hdr's b_pabd.
  */
 static boolean_t
 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
 {
 	/*
 	 * The criteria for sharing a hdr's data are:
 	 * 1. the buffer is not encrypted
 	 * 2. the hdr's compression matches the buf's compression
 	 * 3. the hdr doesn't need to be byteswapped
 	 * 4. the hdr isn't already being shared
 	 * 5. the buf is either compressed or it is the last buf in the hdr list
 	 *
 	 * Criterion #5 maintains the invariant that shared uncompressed
 	 * bufs must be the final buf in the hdr's b_buf list. Reading this, you
 	 * might ask, "if a compressed buf is allocated first, won't that be the
 	 * last thing in the list?", but in that case it's impossible to create
 	 * a shared uncompressed buf anyway (because the hdr must be compressed
 	 * to have the compressed buf). You might also think that #3 is
 	 * sufficient to make this guarantee, however it's possible
 	 * (specifically in the rare L2ARC write race mentioned in
 	 * arc_buf_alloc_impl()) there will be an existing uncompressed buf that
 	 * is shareable, but wasn't at the time of its allocation. Rather than
 	 * allow a new shared uncompressed buf to be created and then shuffle
 	 * the list around to make it the last element, this simply disallows
 	 * sharing if the new buf isn't the first to be added.
 	 */
 	ASSERT3P(buf->b_hdr, ==, hdr);
 	boolean_t hdr_compressed =
 	    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF;
 	boolean_t buf_compressed = ARC_BUF_COMPRESSED(buf) != 0;
 	return (!ARC_BUF_ENCRYPTED(buf) &&
 	    buf_compressed == hdr_compressed &&
 	    hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
 	    !HDR_SHARED_DATA(hdr) &&
 	    (ARC_BUF_LAST(buf) || ARC_BUF_COMPRESSED(buf)));
 }
 
 /*
  * Allocate a buf for this hdr. If you care about the data that's in the hdr,
  * or if you want a compressed buffer, pass those flags in. Returns 0 if the
  * copy was made successfully, or an error code otherwise.
  */
 static int
 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb,
     const void *tag, boolean_t encrypted, boolean_t compressed,
     boolean_t noauth, boolean_t fill, arc_buf_t **ret)
 {
 	arc_buf_t *buf;
 	arc_fill_flags_t flags = ARC_FILL_LOCKED;
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
 	VERIFY(hdr->b_type == ARC_BUFC_DATA ||
 	    hdr->b_type == ARC_BUFC_METADATA);
 	ASSERT3P(ret, !=, NULL);
 	ASSERT3P(*ret, ==, NULL);
 	IMPLY(encrypted, compressed);
 
 	buf = *ret = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
 	buf->b_hdr = hdr;
 	buf->b_data = NULL;
 	buf->b_next = hdr->b_l1hdr.b_buf;
 	buf->b_flags = 0;
 
 	add_reference(hdr, tag);
 
 	/*
 	 * We're about to change the hdr's b_flags. We must either
 	 * hold the hash_lock or be undiscoverable.
 	 */
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 
 	/*
 	 * Only honor requests for compressed bufs if the hdr is actually
 	 * compressed. This must be overridden if the buffer is encrypted since
 	 * encrypted buffers cannot be decompressed.
 	 */
 	if (encrypted) {
 		buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
 		buf->b_flags |= ARC_BUF_FLAG_ENCRYPTED;
 		flags |= ARC_FILL_COMPRESSED | ARC_FILL_ENCRYPTED;
 	} else if (compressed &&
 	    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
 		buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
 		flags |= ARC_FILL_COMPRESSED;
 	}
 
 	if (noauth) {
 		ASSERT0(encrypted);
 		flags |= ARC_FILL_NOAUTH;
 	}
 
 	/*
 	 * If the hdr's data can be shared then we share the data buffer and
 	 * set the appropriate bit in the hdr's b_flags to indicate the hdr is
 	 * sharing it's b_pabd with the arc_buf_t. Otherwise, we allocate a new
 	 * buffer to store the buf's data.
 	 *
 	 * There are two additional restrictions here because we're sharing
 	 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
 	 * actively involved in an L2ARC write, because if this buf is used by
 	 * an arc_write() then the hdr's data buffer will be released when the
 	 * write completes, even though the L2ARC write might still be using it.
 	 * Second, the hdr's ABD must be linear so that the buf's user doesn't
 	 * need to be ABD-aware.  It must be allocated via
 	 * zio_[data_]buf_alloc(), not as a page, because we need to be able
 	 * to abd_release_ownership_of_buf(), which isn't allowed on "linear
 	 * page" buffers because the ABD code needs to handle freeing them
 	 * specially.
 	 */
 	boolean_t can_share = arc_can_share(hdr, buf) &&
 	    !HDR_L2_WRITING(hdr) &&
 	    hdr->b_l1hdr.b_pabd != NULL &&
 	    abd_is_linear(hdr->b_l1hdr.b_pabd) &&
 	    !abd_is_linear_page(hdr->b_l1hdr.b_pabd);
 
 	/* Set up b_data and sharing */
 	if (can_share) {
 		buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
 		buf->b_flags |= ARC_BUF_FLAG_SHARED;
 		arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
 	} else {
 		buf->b_data =
 		    arc_get_data_buf(hdr, arc_buf_size(buf), buf);
 		ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
 	}
 	VERIFY3P(buf->b_data, !=, NULL);
 
 	hdr->b_l1hdr.b_buf = buf;
 
 	/*
 	 * If the user wants the data from the hdr, we need to either copy or
 	 * decompress the data.
 	 */
 	if (fill) {
 		ASSERT3P(zb, !=, NULL);
 		return (arc_buf_fill(buf, spa, zb, flags));
 	}
 
 	return (0);
 }
 
 static const char *arc_onloan_tag = "onloan";
 
 static inline void
 arc_loaned_bytes_update(int64_t delta)
 {
 	atomic_add_64(&arc_loaned_bytes, delta);
 
 	/* assert that it did not wrap around */
 	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
 }
 
 /*
  * Loan out an anonymous arc buffer. Loaned buffers are not counted as in
  * flight data by arc_tempreserve_space() until they are "returned". Loaned
  * buffers must be returned to the arc before they can be used by the DMU or
  * freed.
  */
 arc_buf_t *
 arc_loan_buf(spa_t *spa, boolean_t is_metadata, int size)
 {
 	arc_buf_t *buf = arc_alloc_buf(spa, arc_onloan_tag,
 	    is_metadata ? ARC_BUFC_METADATA : ARC_BUFC_DATA, size);
 
 	arc_loaned_bytes_update(arc_buf_size(buf));
 
 	return (buf);
 }
 
 arc_buf_t *
 arc_loan_compressed_buf(spa_t *spa, uint64_t psize, uint64_t lsize,
     enum zio_compress compression_type, uint8_t complevel)
 {
 	arc_buf_t *buf = arc_alloc_compressed_buf(spa, arc_onloan_tag,
 	    psize, lsize, compression_type, complevel);
 
 	arc_loaned_bytes_update(arc_buf_size(buf));
 
 	return (buf);
 }
 
 arc_buf_t *
 arc_loan_raw_buf(spa_t *spa, uint64_t dsobj, boolean_t byteorder,
     const uint8_t *salt, const uint8_t *iv, const uint8_t *mac,
     dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
     enum zio_compress compression_type, uint8_t complevel)
 {
 	arc_buf_t *buf = arc_alloc_raw_buf(spa, arc_onloan_tag, dsobj,
 	    byteorder, salt, iv, mac, ot, psize, lsize, compression_type,
 	    complevel);
 
 	atomic_add_64(&arc_loaned_bytes, psize);
 	return (buf);
 }
 
 
 /*
  * Return a loaned arc buffer to the arc.
  */
 void
 arc_return_buf(arc_buf_t *buf, const void *tag)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT3P(buf->b_data, !=, NULL);
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
 	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
 
 	arc_loaned_bytes_update(-arc_buf_size(buf));
 }
 
 /* Detach an arc_buf from a dbuf (tag) */
 void
 arc_loan_inuse_buf(arc_buf_t *buf, const void *tag)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT3P(buf->b_data, !=, NULL);
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	(void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
 	(void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
 
 	arc_loaned_bytes_update(arc_buf_size(buf));
 }
 
 static void
 l2arc_free_abd_on_write(abd_t *abd, size_t size, arc_buf_contents_t type)
 {
 	l2arc_data_free_t *df = kmem_alloc(sizeof (*df), KM_SLEEP);
 
 	df->l2df_abd = abd;
 	df->l2df_size = size;
 	df->l2df_type = type;
 	mutex_enter(&l2arc_free_on_write_mtx);
 	list_insert_head(l2arc_free_on_write, df);
 	mutex_exit(&l2arc_free_on_write_mtx);
 }
 
 static void
 arc_hdr_free_on_write(arc_buf_hdr_t *hdr, boolean_t free_rdata)
 {
 	arc_state_t *state = hdr->b_l1hdr.b_state;
 	arc_buf_contents_t type = arc_buf_type(hdr);
 	uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);
 
 	/* protected by hash lock, if in the hash table */
 	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 		ASSERT(state != arc_anon && state != arc_l2c_only);
 
 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
 		    size, hdr);
 	}
 	(void) zfs_refcount_remove_many(&state->arcs_size[type], size, hdr);
 	if (type == ARC_BUFC_METADATA) {
 		arc_space_return(size, ARC_SPACE_META);
 	} else {
 		ASSERT(type == ARC_BUFC_DATA);
 		arc_space_return(size, ARC_SPACE_DATA);
 	}
 
 	if (free_rdata) {
 		l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type);
 	} else {
 		l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
 	}
 }
 
 /*
  * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
  * data buffer, we transfer the refcount ownership to the hdr and update
  * the appropriate kstats.
  */
 static void
 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
 {
 	ASSERT(arc_can_share(hdr, buf));
 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
 	ASSERT(!ARC_BUF_ENCRYPTED(buf));
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 
 	/*
 	 * Start sharing the data buffer. We transfer the
 	 * refcount ownership to the hdr since it always owns
 	 * the refcount whenever an arc_buf_t is shared.
 	 */
 	zfs_refcount_transfer_ownership_many(
 	    &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
 	    arc_hdr_size(hdr), buf, hdr);
 	hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
 	abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
 	    HDR_ISTYPE_METADATA(hdr));
 	arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
 	buf->b_flags |= ARC_BUF_FLAG_SHARED;
 
 	/*
 	 * Since we've transferred ownership to the hdr we need
 	 * to increment its compressed and uncompressed kstats and
 	 * decrement the overhead size.
 	 */
 	ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
 	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
 	ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf));
 }
 
 static void
 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
 {
 	ASSERT(arc_buf_is_shared(buf));
 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 
 	/*
 	 * We are no longer sharing this buffer so we need
 	 * to transfer its ownership to the rightful owner.
 	 */
 	zfs_refcount_transfer_ownership_many(
 	    &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
 	    arc_hdr_size(hdr), hdr, buf);
 	arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
 	abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
 	abd_free(hdr->b_l1hdr.b_pabd);
 	hdr->b_l1hdr.b_pabd = NULL;
 	buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
 
 	/*
 	 * Since the buffer is no longer shared between
 	 * the arc buf and the hdr, count it as overhead.
 	 */
 	ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
 	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
 	ARCSTAT_INCR(arcstat_overhead_size, arc_buf_size(buf));
 }
 
 /*
  * Remove an arc_buf_t from the hdr's buf list and return the last
  * arc_buf_t on the list. If no buffers remain on the list then return
  * NULL.
  */
 static arc_buf_t *
 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
 {
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 
 	arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
 	arc_buf_t *lastbuf = NULL;
 
 	/*
 	 * Remove the buf from the hdr list and locate the last
 	 * remaining buffer on the list.
 	 */
 	while (*bufp != NULL) {
 		if (*bufp == buf)
 			*bufp = buf->b_next;
 
 		/*
 		 * If we've removed a buffer in the middle of
 		 * the list then update the lastbuf and update
 		 * bufp.
 		 */
 		if (*bufp != NULL) {
 			lastbuf = *bufp;
 			bufp = &(*bufp)->b_next;
 		}
 	}
 	buf->b_next = NULL;
 	ASSERT3P(lastbuf, !=, buf);
 	IMPLY(lastbuf != NULL, ARC_BUF_LAST(lastbuf));
 
 	return (lastbuf);
 }
 
 /*
  * Free up buf->b_data and pull the arc_buf_t off of the arc_buf_hdr_t's
  * list and free it.
  */
 static void
 arc_buf_destroy_impl(arc_buf_t *buf)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	/*
 	 * Free up the data associated with the buf but only if we're not
 	 * sharing this with the hdr. If we are sharing it with the hdr, the
 	 * hdr is responsible for doing the free.
 	 */
 	if (buf->b_data != NULL) {
 		/*
 		 * We're about to change the hdr's b_flags. We must either
 		 * hold the hash_lock or be undiscoverable.
 		 */
 		ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
 
 		arc_cksum_verify(buf);
 		arc_buf_unwatch(buf);
 
 		if (ARC_BUF_SHARED(buf)) {
 			arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
 		} else {
 			ASSERT(!arc_buf_is_shared(buf));
 			uint64_t size = arc_buf_size(buf);
 			arc_free_data_buf(hdr, buf->b_data, size, buf);
 			ARCSTAT_INCR(arcstat_overhead_size, -size);
 		}
 		buf->b_data = NULL;
 
 		/*
 		 * If we have no more encrypted buffers and we've already
 		 * gotten a copy of the decrypted data we can free b_rabd
 		 * to save some space.
 		 */
 		if (ARC_BUF_ENCRYPTED(buf) && HDR_HAS_RABD(hdr) &&
 		    hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {
 			arc_buf_t *b;
 			for (b = hdr->b_l1hdr.b_buf; b; b = b->b_next) {
 				if (b != buf && ARC_BUF_ENCRYPTED(b))
 					break;
 			}
 			if (b == NULL)
 				arc_hdr_free_abd(hdr, B_TRUE);
 		}
 	}
 
 	arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
 
 	if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
 		/*
 		 * If the current arc_buf_t is sharing its data buffer with the
 		 * hdr, then reassign the hdr's b_pabd to share it with the new
 		 * buffer at the end of the list. The shared buffer is always
 		 * the last one on the hdr's buffer list.
 		 *
 		 * There is an equivalent case for compressed bufs, but since
 		 * they aren't guaranteed to be the last buf in the list and
 		 * that is an exceedingly rare case, we just allow that space be
 		 * wasted temporarily. We must also be careful not to share
 		 * encrypted buffers, since they cannot be shared.
 		 */
 		if (lastbuf != NULL && !ARC_BUF_ENCRYPTED(lastbuf)) {
 			/* Only one buf can be shared at once */
 			ASSERT(!arc_buf_is_shared(lastbuf));
 			/* hdr is uncompressed so can't have compressed buf */
 			ASSERT(!ARC_BUF_COMPRESSED(lastbuf));
 
 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
 			arc_hdr_free_abd(hdr, B_FALSE);
 
 			/*
 			 * We must setup a new shared block between the
 			 * last buffer and the hdr. The data would have
 			 * been allocated by the arc buf so we need to transfer
 			 * ownership to the hdr since it's now being shared.
 			 */
 			arc_share_buf(hdr, lastbuf);
 		}
 	} else if (HDR_SHARED_DATA(hdr)) {
 		/*
 		 * Uncompressed shared buffers are always at the end
 		 * of the list. Compressed buffers don't have the
 		 * same requirements. This makes it hard to
 		 * simply assert that the lastbuf is shared so
 		 * we rely on the hdr's compression flags to determine
 		 * if we have a compressed, shared buffer.
 		 */
 		ASSERT3P(lastbuf, !=, NULL);
 		ASSERT(arc_buf_is_shared(lastbuf) ||
 		    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
 	}
 
 	/*
 	 * Free the checksum if we're removing the last uncompressed buf from
 	 * this hdr.
 	 */
 	if (!arc_hdr_has_uncompressed_buf(hdr)) {
 		arc_cksum_free(hdr);
 	}
 
 	/* clean up the buf */
 	buf->b_hdr = NULL;
 	kmem_cache_free(buf_cache, buf);
 }
 
 static void
 arc_hdr_alloc_abd(arc_buf_hdr_t *hdr, int alloc_flags)
 {
 	uint64_t size;
 	boolean_t alloc_rdata = ((alloc_flags & ARC_HDR_ALLOC_RDATA) != 0);
 
 	ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(!HDR_SHARED_DATA(hdr) || alloc_rdata);
 	IMPLY(alloc_rdata, HDR_PROTECTED(hdr));
 
 	if (alloc_rdata) {
 		size = HDR_GET_PSIZE(hdr);
 		ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL);
 		hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr,
 		    alloc_flags);
 		ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL);
 		ARCSTAT_INCR(arcstat_raw_size, size);
 	} else {
 		size = arc_hdr_size(hdr);
 		ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
 		hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr,
 		    alloc_flags);
 		ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
 	}
 
 	ARCSTAT_INCR(arcstat_compressed_size, size);
 	ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
 }
 
 static void
 arc_hdr_free_abd(arc_buf_hdr_t *hdr, boolean_t free_rdata)
 {
 	uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
 	IMPLY(free_rdata, HDR_HAS_RABD(hdr));
 
 	/*
 	 * If the hdr is currently being written to the l2arc then
 	 * we defer freeing the data by adding it to the l2arc_free_on_write
 	 * list. The l2arc will free the data once it's finished
 	 * writing it to the l2arc device.
 	 */
 	if (HDR_L2_WRITING(hdr)) {
 		arc_hdr_free_on_write(hdr, free_rdata);
 		ARCSTAT_BUMP(arcstat_l2_free_on_write);
 	} else if (free_rdata) {
 		arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr);
 	} else {
 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, size, hdr);
 	}
 
 	if (free_rdata) {
 		hdr->b_crypt_hdr.b_rabd = NULL;
 		ARCSTAT_INCR(arcstat_raw_size, -size);
 	} else {
 		hdr->b_l1hdr.b_pabd = NULL;
 	}
 
 	if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr))
 		hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
 
 	ARCSTAT_INCR(arcstat_compressed_size, -size);
 	ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
 }
 
 /*
  * Allocate empty anonymous ARC header.  The header will get its identity
  * assigned and buffers attached later as part of read or write operations.
  *
  * In case of read arc_read() assigns header its identify (b_dva + b_birth),
  * inserts it into ARC hash to become globally visible and allocates physical
  * (b_pabd) or raw (b_rabd) ABD buffer to read into from disk.  On disk read
  * completion arc_read_done() allocates ARC buffer(s) as needed, potentially
  * sharing one of them with the physical ABD buffer.
  *
  * In case of write arc_alloc_buf() allocates ARC buffer to be filled with
  * data.  Then after compression and/or encryption arc_write_ready() allocates
  * and fills (or potentially shares) physical (b_pabd) or raw (b_rabd) ABD
  * buffer.  On disk write completion arc_write_done() assigns the header its
  * new identity (b_dva + b_birth) and inserts into ARC hash.
  *
  * In case of partial overwrite the old data is read first as described. Then
  * arc_release() either allocates new anonymous ARC header and moves the ARC
  * buffer to it, or reuses the old ARC header by discarding its identity and
  * removing it from ARC hash.  After buffer modification normal write process
  * follows as described.
  */
 static arc_buf_hdr_t *
 arc_hdr_alloc(uint64_t spa, int32_t psize, int32_t lsize,
     boolean_t protected, enum zio_compress compression_type, uint8_t complevel,
     arc_buf_contents_t type)
 {
 	arc_buf_hdr_t *hdr;
 
 	VERIFY(type == ARC_BUFC_DATA || type == ARC_BUFC_METADATA);
 	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
 
 	ASSERT(HDR_EMPTY(hdr));
 #ifdef ZFS_DEBUG
 	ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
 #endif
 	HDR_SET_PSIZE(hdr, psize);
 	HDR_SET_LSIZE(hdr, lsize);
 	hdr->b_spa = spa;
 	hdr->b_type = type;
 	hdr->b_flags = 0;
 	arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
 	arc_hdr_set_compress(hdr, compression_type);
 	hdr->b_complevel = complevel;
 	if (protected)
 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
 
 	hdr->b_l1hdr.b_state = arc_anon;
 	hdr->b_l1hdr.b_arc_access = 0;
 	hdr->b_l1hdr.b_mru_hits = 0;
 	hdr->b_l1hdr.b_mru_ghost_hits = 0;
 	hdr->b_l1hdr.b_mfu_hits = 0;
 	hdr->b_l1hdr.b_mfu_ghost_hits = 0;
 	hdr->b_l1hdr.b_buf = NULL;
 
 	ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 
 	return (hdr);
 }
 
 /*
  * Transition between the two allocation states for the arc_buf_hdr struct.
  * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
  * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
  * version is used when a cache buffer is only in the L2ARC in order to reduce
  * memory usage.
  */
 static arc_buf_hdr_t *
 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
 {
 	ASSERT(HDR_HAS_L2HDR(hdr));
 
 	arc_buf_hdr_t *nhdr;
 	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
 
 	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
 	    (old == hdr_l2only_cache && new == hdr_full_cache));
 
 	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
 
 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
 	buf_hash_remove(hdr);
 
 	memcpy(nhdr, hdr, HDR_L2ONLY_SIZE);
 
 	if (new == hdr_full_cache) {
 		arc_hdr_set_flags(nhdr, ARC_FLAG_HAS_L1HDR);
 		/*
 		 * arc_access and arc_change_state need to be aware that a
 		 * header has just come out of L2ARC, so we set its state to
 		 * l2c_only even though it's about to change.
 		 */
 		nhdr->b_l1hdr.b_state = arc_l2c_only;
 
 		/* Verify previous threads set to NULL before freeing */
 		ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
 		ASSERT(!HDR_HAS_RABD(hdr));
 	} else {
 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
 #ifdef ZFS_DEBUG
 		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
 #endif
 
 		/*
 		 * If we've reached here, We must have been called from
 		 * arc_evict_hdr(), as such we should have already been
 		 * removed from any ghost list we were previously on
 		 * (which protects us from racing with arc_evict_state),
 		 * thus no locking is needed during this check.
 		 */
 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 
 		/*
 		 * A buffer must not be moved into the arc_l2c_only
 		 * state if it's not finished being written out to the
 		 * l2arc device. Otherwise, the b_l1hdr.b_pabd field
 		 * might try to be accessed, even though it was removed.
 		 */
 		VERIFY(!HDR_L2_WRITING(hdr));
 		VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
 		ASSERT(!HDR_HAS_RABD(hdr));
 
 		arc_hdr_clear_flags(nhdr, ARC_FLAG_HAS_L1HDR);
 	}
 	/*
 	 * The header has been reallocated so we need to re-insert it into any
 	 * lists it was on.
 	 */
 	(void) buf_hash_insert(nhdr, NULL);
 
 	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
 
 	mutex_enter(&dev->l2ad_mtx);
 
 	/*
 	 * We must place the realloc'ed header back into the list at
 	 * the same spot. Otherwise, if it's placed earlier in the list,
 	 * l2arc_write_buffers() could find it during the function's
 	 * write phase, and try to write it out to the l2arc.
 	 */
 	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
 	list_remove(&dev->l2ad_buflist, hdr);
 
 	mutex_exit(&dev->l2ad_mtx);
 
 	/*
 	 * Since we're using the pointer address as the tag when
 	 * incrementing and decrementing the l2ad_alloc refcount, we
 	 * must remove the old pointer (that we're about to destroy) and
 	 * add the new pointer to the refcount. Otherwise we'd remove
 	 * the wrong pointer address when calling arc_hdr_destroy() later.
 	 */
 
 	(void) zfs_refcount_remove_many(&dev->l2ad_alloc,
 	    arc_hdr_size(hdr), hdr);
 	(void) zfs_refcount_add_many(&dev->l2ad_alloc,
 	    arc_hdr_size(nhdr), nhdr);
 
 	buf_discard_identity(hdr);
 	kmem_cache_free(old, hdr);
 
 	return (nhdr);
 }
 
 /*
  * This function is used by the send / receive code to convert a newly
  * allocated arc_buf_t to one that is suitable for a raw encrypted write. It
  * is also used to allow the root objset block to be updated without altering
  * its embedded MACs. Both block types will always be uncompressed so we do not
  * have to worry about compression type or psize.
  */
 void
 arc_convert_to_raw(arc_buf_t *buf, uint64_t dsobj, boolean_t byteorder,
     dmu_object_type_t ot, const uint8_t *salt, const uint8_t *iv,
     const uint8_t *mac)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT(ot == DMU_OT_DNODE || ot == DMU_OT_OBJSET);
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
 
 	buf->b_flags |= (ARC_BUF_FLAG_COMPRESSED | ARC_BUF_FLAG_ENCRYPTED);
 	arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
 	hdr->b_crypt_hdr.b_dsobj = dsobj;
 	hdr->b_crypt_hdr.b_ot = ot;
 	hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
 	    DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot);
 	if (!arc_hdr_has_uncompressed_buf(hdr))
 		arc_cksum_free(hdr);
 
 	if (salt != NULL)
 		memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
 	if (iv != NULL)
 		memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
 	if (mac != NULL)
 		memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
 }
 
 /*
  * Allocate a new arc_buf_hdr_t and arc_buf_t and return the buf to the caller.
  * The buf is returned thawed since we expect the consumer to modify it.
  */
 arc_buf_t *
 arc_alloc_buf(spa_t *spa, const void *tag, arc_buf_contents_t type,
     int32_t size)
 {
 	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
 	    B_FALSE, ZIO_COMPRESS_OFF, 0, type);
 
 	arc_buf_t *buf = NULL;
 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE,
 	    B_FALSE, B_FALSE, &buf));
 	arc_buf_thaw(buf);
 
 	return (buf);
 }
 
 /*
  * Allocate a compressed buf in the same manner as arc_alloc_buf. Don't use this
  * for bufs containing metadata.
  */
 arc_buf_t *
 arc_alloc_compressed_buf(spa_t *spa, const void *tag, uint64_t psize,
     uint64_t lsize, enum zio_compress compression_type, uint8_t complevel)
 {
 	ASSERT3U(lsize, >, 0);
 	ASSERT3U(lsize, >=, psize);
 	ASSERT3U(compression_type, >, ZIO_COMPRESS_OFF);
 	ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);
 
 	arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
 	    B_FALSE, compression_type, complevel, ARC_BUFC_DATA);
 
 	arc_buf_t *buf = NULL;
 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE,
 	    B_TRUE, B_FALSE, B_FALSE, &buf));
 	arc_buf_thaw(buf);
 
 	/*
 	 * To ensure that the hdr has the correct data in it if we call
 	 * arc_untransform() on this buf before it's been written to disk,
 	 * it's easiest if we just set up sharing between the buf and the hdr.
 	 */
 	arc_share_buf(hdr, buf);
 
 	return (buf);
 }
 
 arc_buf_t *
 arc_alloc_raw_buf(spa_t *spa, const void *tag, uint64_t dsobj,
     boolean_t byteorder, const uint8_t *salt, const uint8_t *iv,
     const uint8_t *mac, dmu_object_type_t ot, uint64_t psize, uint64_t lsize,
     enum zio_compress compression_type, uint8_t complevel)
 {
 	arc_buf_hdr_t *hdr;
 	arc_buf_t *buf;
 	arc_buf_contents_t type = DMU_OT_IS_METADATA(ot) ?
 	    ARC_BUFC_METADATA : ARC_BUFC_DATA;
 
 	ASSERT3U(lsize, >, 0);
 	ASSERT3U(lsize, >=, psize);
 	ASSERT3U(compression_type, >=, ZIO_COMPRESS_OFF);
 	ASSERT3U(compression_type, <, ZIO_COMPRESS_FUNCTIONS);
 
 	hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE,
 	    compression_type, complevel, type);
 
 	hdr->b_crypt_hdr.b_dsobj = dsobj;
 	hdr->b_crypt_hdr.b_ot = ot;
 	hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
 	    DMU_BSWAP_NUMFUNCS : DMU_OT_BYTESWAP(ot);
 	memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
 	memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
 	memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
 
 	/*
 	 * This buffer will be considered encrypted even if the ot is not an
 	 * encrypted type. It will become authenticated instead in
 	 * arc_write_ready().
 	 */
 	buf = NULL;
 	VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE,
 	    B_FALSE, B_FALSE, &buf));
 	arc_buf_thaw(buf);
 
 	return (buf);
 }
 
 static void
 l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
     boolean_t state_only)
 {
 	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
 	l2arc_dev_t *dev = l2hdr->b_dev;
 	uint64_t lsize = HDR_GET_LSIZE(hdr);
 	uint64_t psize = HDR_GET_PSIZE(hdr);
 	uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
 	arc_buf_contents_t type = hdr->b_type;
 	int64_t lsize_s;
 	int64_t psize_s;
 	int64_t asize_s;
 
 	if (incr) {
 		lsize_s = lsize;
 		psize_s = psize;
 		asize_s = asize;
 	} else {
 		lsize_s = -lsize;
 		psize_s = -psize;
 		asize_s = -asize;
 	}
 
 	/* If the buffer is a prefetch, count it as such. */
 	if (HDR_PREFETCH(hdr)) {
 		ARCSTAT_INCR(arcstat_l2_prefetch_asize, asize_s);
 	} else {
 		/*
 		 * We use the value stored in the L2 header upon initial
 		 * caching in L2ARC. This value will be updated in case
 		 * an MRU/MRU_ghost buffer transitions to MFU but the L2ARC
 		 * metadata (log entry) cannot currently be updated. Having
 		 * the ARC state in the L2 header solves the problem of a
 		 * possibly absent L1 header (apparent in buffers restored
 		 * from persistent L2ARC).
 		 */
 		switch (hdr->b_l2hdr.b_arcs_state) {
 			case ARC_STATE_MRU_GHOST:
 			case ARC_STATE_MRU:
 				ARCSTAT_INCR(arcstat_l2_mru_asize, asize_s);
 				break;
 			case ARC_STATE_MFU_GHOST:
 			case ARC_STATE_MFU:
 				ARCSTAT_INCR(arcstat_l2_mfu_asize, asize_s);
 				break;
 			default:
 				break;
 		}
 	}
 
 	if (state_only)
 		return;
 
 	ARCSTAT_INCR(arcstat_l2_psize, psize_s);
 	ARCSTAT_INCR(arcstat_l2_lsize, lsize_s);
 
 	switch (type) {
 		case ARC_BUFC_DATA:
 			ARCSTAT_INCR(arcstat_l2_bufc_data_asize, asize_s);
 			break;
 		case ARC_BUFC_METADATA:
 			ARCSTAT_INCR(arcstat_l2_bufc_metadata_asize, asize_s);
 			break;
 		default:
 			break;
 	}
 }
 
 
 static void
 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
 {
 	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
 	l2arc_dev_t *dev = l2hdr->b_dev;
 	uint64_t psize = HDR_GET_PSIZE(hdr);
 	uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
 
 	ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
 	ASSERT(HDR_HAS_L2HDR(hdr));
 
 	list_remove(&dev->l2ad_buflist, hdr);
 
 	l2arc_hdr_arcstats_decrement(hdr);
 	vdev_space_update(dev->l2ad_vdev, -asize, 0, 0);
 
 	(void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr),
 	    hdr);
 	arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
 }
 
 static void
 arc_hdr_destroy(arc_buf_hdr_t *hdr)
 {
 	if (HDR_HAS_L1HDR(hdr)) {
 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
 	}
 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
 
 	if (HDR_HAS_L2HDR(hdr)) {
 		l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
 		boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
 
 		if (!buflist_held)
 			mutex_enter(&dev->l2ad_mtx);
 
 		/*
 		 * Even though we checked this conditional above, we
 		 * need to check this again now that we have the
 		 * l2ad_mtx. This is because we could be racing with
 		 * another thread calling l2arc_evict() which might have
 		 * destroyed this header's L2 portion as we were waiting
 		 * to acquire the l2ad_mtx. If that happens, we don't
 		 * want to re-destroy the header's L2 portion.
 		 */
 		if (HDR_HAS_L2HDR(hdr)) {
 
 			if (!HDR_EMPTY(hdr))
 				buf_discard_identity(hdr);
 
 			arc_hdr_l2hdr_destroy(hdr);
 		}
 
 		if (!buflist_held)
 			mutex_exit(&dev->l2ad_mtx);
 	}
 
 	/*
 	 * The header's identify can only be safely discarded once it is no
 	 * longer discoverable.  This requires removing it from the hash table
 	 * and the l2arc header list.  After this point the hash lock can not
 	 * be used to protect the header.
 	 */
 	if (!HDR_EMPTY(hdr))
 		buf_discard_identity(hdr);
 
 	if (HDR_HAS_L1HDR(hdr)) {
 		arc_cksum_free(hdr);
 
 		while (hdr->b_l1hdr.b_buf != NULL)
 			arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
 
 		if (hdr->b_l1hdr.b_pabd != NULL)
 			arc_hdr_free_abd(hdr, B_FALSE);
 
 		if (HDR_HAS_RABD(hdr))
 			arc_hdr_free_abd(hdr, B_TRUE);
 	}
 
 	ASSERT3P(hdr->b_hash_next, ==, NULL);
 	if (HDR_HAS_L1HDR(hdr)) {
 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
 #ifdef ZFS_DEBUG
 		ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
 #endif
 		kmem_cache_free(hdr_full_cache, hdr);
 	} else {
 		kmem_cache_free(hdr_l2only_cache, hdr);
 	}
 }
 
 void
 arc_buf_destroy(arc_buf_t *buf, const void *tag)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	if (hdr->b_l1hdr.b_state == arc_anon) {
 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
 		ASSERT(ARC_BUF_LAST(buf));
 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 		VERIFY0(remove_reference(hdr, tag));
 		return;
 	}
 
 	kmutex_t *hash_lock = HDR_LOCK(hdr);
 	mutex_enter(hash_lock);
 
 	ASSERT3P(hdr, ==, buf->b_hdr);
 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
 	ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
 	ASSERT3P(buf->b_data, !=, NULL);
 
 	arc_buf_destroy_impl(buf);
 	(void) remove_reference(hdr, tag);
 	mutex_exit(hash_lock);
 }
 
 /*
  * Evict the arc_buf_hdr that is provided as a parameter. The resultant
  * state of the header is dependent on its state prior to entering this
  * function. The following transitions are possible:
  *
  *    - arc_mru -> arc_mru_ghost
  *    - arc_mfu -> arc_mfu_ghost
  *    - arc_mru_ghost -> arc_l2c_only
  *    - arc_mru_ghost -> deleted
  *    - arc_mfu_ghost -> arc_l2c_only
  *    - arc_mfu_ghost -> deleted
  *    - arc_uncached -> deleted
  *
  * Return total size of evicted data buffers for eviction progress tracking.
  * When evicting from ghost states return logical buffer size to make eviction
  * progress at the same (or at least comparable) rate as from non-ghost states.
  *
  * Return *real_evicted for actual ARC size reduction to wake up threads
  * waiting for it.  For non-ghost states it includes size of evicted data
  * buffers (the headers are not freed there).  For ghost states it includes
  * only the evicted headers size.
  */
 static int64_t
 arc_evict_hdr(arc_buf_hdr_t *hdr, uint64_t *real_evicted)
 {
 	arc_state_t *evicted_state, *state;
 	int64_t bytes_evicted = 0;
 	uint_t min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ?
 	    arc_min_prescient_prefetch_ms : arc_min_prefetch_ms;
 
 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 	ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
 	ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));
 
 	*real_evicted = 0;
 	state = hdr->b_l1hdr.b_state;
 	if (GHOST_STATE(state)) {
 
 		/*
 		 * l2arc_write_buffers() relies on a header's L1 portion
 		 * (i.e. its b_pabd field) during it's write phase.
 		 * Thus, we cannot push a header onto the arc_l2c_only
 		 * state (removing its L1 piece) until the header is
 		 * done being written to the l2arc.
 		 */
 		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
 			ARCSTAT_BUMP(arcstat_evict_l2_skip);
 			return (bytes_evicted);
 		}
 
 		ARCSTAT_BUMP(arcstat_deleted);
 		bytes_evicted += HDR_GET_LSIZE(hdr);
 
 		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
 
 		if (HDR_HAS_L2HDR(hdr)) {
 			ASSERT(hdr->b_l1hdr.b_pabd == NULL);
 			ASSERT(!HDR_HAS_RABD(hdr));
 			/*
 			 * This buffer is cached on the 2nd Level ARC;
 			 * don't destroy the header.
 			 */
 			arc_change_state(arc_l2c_only, hdr);
 			/*
 			 * dropping from L1+L2 cached to L2-only,
 			 * realloc to remove the L1 header.
 			 */
 			(void) arc_hdr_realloc(hdr, hdr_full_cache,
 			    hdr_l2only_cache);
 			*real_evicted += HDR_FULL_SIZE - HDR_L2ONLY_SIZE;
 		} else {
 			arc_change_state(arc_anon, hdr);
 			arc_hdr_destroy(hdr);
 			*real_evicted += HDR_FULL_SIZE;
 		}
 		return (bytes_evicted);
 	}
 
 	ASSERT(state == arc_mru || state == arc_mfu || state == arc_uncached);
 	evicted_state = (state == arc_uncached) ? arc_anon :
 	    ((state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost);
 
 	/* prefetch buffers have a minimum lifespan */
 	if ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
 	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
 	    MSEC_TO_TICK(min_lifetime)) {
 		ARCSTAT_BUMP(arcstat_evict_skip);
 		return (bytes_evicted);
 	}
 
 	if (HDR_HAS_L2HDR(hdr)) {
 		ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
 	} else {
 		if (l2arc_write_eligible(hdr->b_spa, hdr)) {
 			ARCSTAT_INCR(arcstat_evict_l2_eligible,
 			    HDR_GET_LSIZE(hdr));
 
 			switch (state->arcs_state) {
 				case ARC_STATE_MRU:
 					ARCSTAT_INCR(
 					    arcstat_evict_l2_eligible_mru,
 					    HDR_GET_LSIZE(hdr));
 					break;
 				case ARC_STATE_MFU:
 					ARCSTAT_INCR(
 					    arcstat_evict_l2_eligible_mfu,
 					    HDR_GET_LSIZE(hdr));
 					break;
 				default:
 					break;
 			}
 		} else {
 			ARCSTAT_INCR(arcstat_evict_l2_ineligible,
 			    HDR_GET_LSIZE(hdr));
 		}
 	}
 
 	bytes_evicted += arc_hdr_size(hdr);
 	*real_evicted += arc_hdr_size(hdr);
 
 	/*
 	 * If this hdr is being evicted and has a compressed buffer then we
 	 * discard it here before we change states.  This ensures that the
 	 * accounting is updated correctly in arc_free_data_impl().
 	 */
 	if (hdr->b_l1hdr.b_pabd != NULL)
 		arc_hdr_free_abd(hdr, B_FALSE);
 
 	if (HDR_HAS_RABD(hdr))
 		arc_hdr_free_abd(hdr, B_TRUE);
 
 	arc_change_state(evicted_state, hdr);
 	DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
 	if (evicted_state == arc_anon) {
 		arc_hdr_destroy(hdr);
 		*real_evicted += HDR_FULL_SIZE;
 	} else {
 		ASSERT(HDR_IN_HASH_TABLE(hdr));
 	}
 
 	return (bytes_evicted);
 }
 
 static void
 arc_set_need_free(void)
 {
 	ASSERT(MUTEX_HELD(&arc_evict_lock));
 	int64_t remaining = arc_free_memory() - arc_sys_free / 2;
 	arc_evict_waiter_t *aw = list_tail(&arc_evict_waiters);
 	if (aw == NULL) {
 		arc_need_free = MAX(-remaining, 0);
 	} else {
 		arc_need_free =
 		    MAX(-remaining, (int64_t)(aw->aew_count - arc_evict_count));
 	}
 }
 
 static uint64_t
 arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
     uint64_t spa, uint64_t bytes)
 {
 	multilist_sublist_t *mls;
 	uint64_t bytes_evicted = 0, real_evicted = 0;
 	arc_buf_hdr_t *hdr;
 	kmutex_t *hash_lock;
 	uint_t evict_count = zfs_arc_evict_batch_limit;
 
 	ASSERT3P(marker, !=, NULL);
 
 	mls = multilist_sublist_lock_idx(ml, idx);
 
 	for (hdr = multilist_sublist_prev(mls, marker); likely(hdr != NULL);
 	    hdr = multilist_sublist_prev(mls, marker)) {
 		if ((evict_count == 0) || (bytes_evicted >= bytes))
 			break;
 
 		/*
 		 * To keep our iteration location, move the marker
 		 * forward. Since we're not holding hdr's hash lock, we
 		 * must be very careful and not remove 'hdr' from the
 		 * sublist. Otherwise, other consumers might mistake the
 		 * 'hdr' as not being on a sublist when they call the
 		 * multilist_link_active() function (they all rely on
 		 * the hash lock protecting concurrent insertions and
 		 * removals). multilist_sublist_move_forward() was
 		 * specifically implemented to ensure this is the case
 		 * (only 'marker' will be removed and re-inserted).
 		 */
 		multilist_sublist_move_forward(mls, marker);
 
 		/*
 		 * The only case where the b_spa field should ever be
 		 * zero, is the marker headers inserted by
 		 * arc_evict_state(). It's possible for multiple threads
 		 * to be calling arc_evict_state() concurrently (e.g.
 		 * dsl_pool_close() and zio_inject_fault()), so we must
 		 * skip any markers we see from these other threads.
 		 */
 		if (hdr->b_spa == 0)
 			continue;
 
 		/* we're only interested in evicting buffers of a certain spa */
 		if (spa != 0 && hdr->b_spa != spa) {
 			ARCSTAT_BUMP(arcstat_evict_skip);
 			continue;
 		}
 
 		hash_lock = HDR_LOCK(hdr);
 
 		/*
 		 * We aren't calling this function from any code path
 		 * that would already be holding a hash lock, so we're
 		 * asserting on this assumption to be defensive in case
 		 * this ever changes. Without this check, it would be
 		 * possible to incorrectly increment arcstat_mutex_miss
 		 * below (e.g. if the code changed such that we called
 		 * this function with a hash lock held).
 		 */
 		ASSERT(!MUTEX_HELD(hash_lock));
 
 		if (mutex_tryenter(hash_lock)) {
 			uint64_t revicted;
 			uint64_t evicted = arc_evict_hdr(hdr, &revicted);
 			mutex_exit(hash_lock);
 
 			bytes_evicted += evicted;
 			real_evicted += revicted;
 
 			/*
 			 * If evicted is zero, arc_evict_hdr() must have
 			 * decided to skip this header, don't increment
 			 * evict_count in this case.
 			 */
 			if (evicted != 0)
 				evict_count--;
 
 		} else {
 			ARCSTAT_BUMP(arcstat_mutex_miss);
 		}
 	}
 
 	multilist_sublist_unlock(mls);
 
 	/*
 	 * Increment the count of evicted bytes, and wake up any threads that
 	 * are waiting for the count to reach this value.  Since the list is
 	 * ordered by ascending aew_count, we pop off the beginning of the
 	 * list until we reach the end, or a waiter that's past the current
 	 * "count".  Doing this outside the loop reduces the number of times
 	 * we need to acquire the global arc_evict_lock.
 	 *
 	 * Only wake when there's sufficient free memory in the system
 	 * (specifically, arc_sys_free/2, which by default is a bit more than
 	 * 1/64th of RAM).  See the comments in arc_wait_for_eviction().
 	 */
 	mutex_enter(&arc_evict_lock);
 	arc_evict_count += real_evicted;
 
 	if (arc_free_memory() > arc_sys_free / 2) {
 		arc_evict_waiter_t *aw;
 		while ((aw = list_head(&arc_evict_waiters)) != NULL &&
 		    aw->aew_count <= arc_evict_count) {
 			list_remove(&arc_evict_waiters, aw);
 			cv_broadcast(&aw->aew_cv);
 		}
 	}
 	arc_set_need_free();
 	mutex_exit(&arc_evict_lock);
 
 	/*
 	 * If the ARC size is reduced from arc_c_max to arc_c_min (especially
 	 * if the average cached block is small), eviction can be on-CPU for
 	 * many seconds.  To ensure that other threads that may be bound to
 	 * this CPU are able to make progress, make a voluntary preemption
 	 * call here.
 	 */
 	kpreempt(KPREEMPT_SYNC);
 
 	return (bytes_evicted);
 }
 
 static arc_buf_hdr_t *
 arc_state_alloc_marker(void)
 {
 	arc_buf_hdr_t *marker = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
 
 	/*
 	 * A b_spa of 0 is used to indicate that this header is
 	 * a marker. This fact is used in arc_evict_state_impl().
 	 */
 	marker->b_spa = 0;
 
 	return (marker);
 }
 
 static void
 arc_state_free_marker(arc_buf_hdr_t *marker)
 {
 	kmem_cache_free(hdr_full_cache, marker);
 }
 
 /*
  * Allocate an array of buffer headers used as placeholders during arc state
  * eviction.
  */
 static arc_buf_hdr_t **
 arc_state_alloc_markers(int count)
 {
 	arc_buf_hdr_t **markers;
 
 	markers = kmem_zalloc(sizeof (*markers) * count, KM_SLEEP);
 	for (int i = 0; i < count; i++)
 		markers[i] = arc_state_alloc_marker();
 	return (markers);
 }
 
 static void
 arc_state_free_markers(arc_buf_hdr_t **markers, int count)
 {
 	for (int i = 0; i < count; i++)
 		arc_state_free_marker(markers[i]);
 	kmem_free(markers, sizeof (*markers) * count);
 }
 
 /*
  * Evict buffers from the given arc state, until we've removed the
  * specified number of bytes. Move the removed buffers to the
  * appropriate evict state.
  *
  * This function makes a "best effort". It skips over any buffers
  * it can't get a hash_lock on, and so, may not catch all candidates.
  * It may also return without evicting as much space as requested.
  *
  * If bytes is specified using the special value ARC_EVICT_ALL, this
  * will evict all available (i.e. unlocked and evictable) buffers from
  * the given arc state; which is used by arc_flush().
  */
 static uint64_t
 arc_evict_state(arc_state_t *state, arc_buf_contents_t type, uint64_t spa,
     uint64_t bytes)
 {
 	uint64_t total_evicted = 0;
 	multilist_t *ml = &state->arcs_list[type];
 	int num_sublists;
 	arc_buf_hdr_t **markers;
 
 	num_sublists = multilist_get_num_sublists(ml);
 
 	/*
 	 * If we've tried to evict from each sublist, made some
 	 * progress, but still have not hit the target number of bytes
 	 * to evict, we want to keep trying. The markers allow us to
 	 * pick up where we left off for each individual sublist, rather
 	 * than starting from the tail each time.
 	 */
 	if (zthr_iscurthread(arc_evict_zthr)) {
 		markers = arc_state_evict_markers;
 		ASSERT3S(num_sublists, <=, arc_state_evict_marker_count);
 	} else {
 		markers = arc_state_alloc_markers(num_sublists);
 	}
 	for (int i = 0; i < num_sublists; i++) {
 		multilist_sublist_t *mls;
 
 		mls = multilist_sublist_lock_idx(ml, i);
 		multilist_sublist_insert_tail(mls, markers[i]);
 		multilist_sublist_unlock(mls);
 	}
 
 	/*
 	 * While we haven't hit our target number of bytes to evict, or
 	 * we're evicting all available buffers.
 	 */
 	while (total_evicted < bytes) {
 		int sublist_idx = multilist_get_random_index(ml);
 		uint64_t scan_evicted = 0;
 
 		/*
 		 * Start eviction using a randomly selected sublist,
 		 * this is to try and evenly balance eviction across all
 		 * sublists. Always starting at the same sublist
 		 * (e.g. index 0) would cause evictions to favor certain
 		 * sublists over others.
 		 */
 		for (int i = 0; i < num_sublists; i++) {
 			uint64_t bytes_remaining;
 			uint64_t bytes_evicted;
 
 			if (total_evicted < bytes)
 				bytes_remaining = bytes - total_evicted;
 			else
 				break;
 
 			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
 			    markers[sublist_idx], spa, bytes_remaining);
 
 			scan_evicted += bytes_evicted;
 			total_evicted += bytes_evicted;
 
 			/* we've reached the end, wrap to the beginning */
 			if (++sublist_idx >= num_sublists)
 				sublist_idx = 0;
 		}
 
 		/*
 		 * If we didn't evict anything during this scan, we have
 		 * no reason to believe we'll evict more during another
 		 * scan, so break the loop.
 		 */
 		if (scan_evicted == 0) {
 			/* This isn't possible, let's make that obvious */
 			ASSERT3S(bytes, !=, 0);
 
 			/*
 			 * When bytes is ARC_EVICT_ALL, the only way to
 			 * break the loop is when scan_evicted is zero.
 			 * In that case, we actually have evicted enough,
 			 * so we don't want to increment the kstat.
 			 */
 			if (bytes != ARC_EVICT_ALL) {
 				ASSERT3S(total_evicted, <, bytes);
 				ARCSTAT_BUMP(arcstat_evict_not_enough);
 			}
 
 			break;
 		}
 	}
 
 	for (int i = 0; i < num_sublists; i++) {
 		multilist_sublist_t *mls = multilist_sublist_lock_idx(ml, i);
 		multilist_sublist_remove(mls, markers[i]);
 		multilist_sublist_unlock(mls);
 	}
 	if (markers != arc_state_evict_markers)
 		arc_state_free_markers(markers, num_sublists);
 
 	return (total_evicted);
 }
 
 /*
  * Flush all "evictable" data of the given type from the arc state
  * specified. This will not evict any "active" buffers (i.e. referenced).
  *
  * When 'retry' is set to B_FALSE, the function will make a single pass
  * over the state and evict any buffers that it can. Since it doesn't
  * continually retry the eviction, it might end up leaving some buffers
  * in the ARC due to lock misses.
  *
  * When 'retry' is set to B_TRUE, the function will continually retry the
  * eviction until *all* evictable buffers have been removed from the
  * state. As a result, if concurrent insertions into the state are
  * allowed (e.g. if the ARC isn't shutting down), this function might
  * wind up in an infinite loop, continually trying to evict buffers.
  */
 static uint64_t
 arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
     boolean_t retry)
 {
 	uint64_t evicted = 0;
 
 	while (zfs_refcount_count(&state->arcs_esize[type]) != 0) {
 		evicted += arc_evict_state(state, type, spa, ARC_EVICT_ALL);
 
 		if (!retry)
 			break;
 	}
 
 	return (evicted);
 }
 
 /*
  * Evict the specified number of bytes from the state specified. This
  * function prevents us from trying to evict more from a state's list
  * than is "evictable", and to skip evicting altogether when passed a
  * negative value for "bytes". In contrast, arc_evict_state() will
  * evict everything it can, when passed a negative value for "bytes".
  */
 static uint64_t
 arc_evict_impl(arc_state_t *state, arc_buf_contents_t type, int64_t bytes)
 {
 	uint64_t delta;
 
 	if (bytes > 0 && zfs_refcount_count(&state->arcs_esize[type]) > 0) {
 		delta = MIN(zfs_refcount_count(&state->arcs_esize[type]),
 		    bytes);
 		return (arc_evict_state(state, type, 0, delta));
 	}
 
 	return (0);
 }
 
 /*
  * Adjust specified fraction, taking into account initial ghost state(s) size,
  * ghost hit bytes towards increasing the fraction, ghost hit bytes towards
  * decreasing it, plus a balance factor, controlling the decrease rate, used
  * to balance metadata vs data.
  */
 static uint64_t
 arc_evict_adj(uint64_t frac, uint64_t total, uint64_t up, uint64_t down,
     uint_t balance)
 {
-	if (total < 8 || up + down == 0)
+	if (total < 32 || up + down == 0)
 		return (frac);
 
 	/*
-	 * We should not have more ghost hits than ghost size, but they
-	 * may get close.  Restrict maximum adjustment in that case.
+	 * We should not have more ghost hits than ghost size, but they may
+	 * get close.  To avoid overflows below up/down should not be bigger
+	 * than 1/5 of total.  But to limit maximum adjustment speed restrict
+	 * it some more.
 	 */
-	if (up + down >= total / 4) {
-		uint64_t scale = (up + down) / (total / 8);
+	if (up + down >= total / 16) {
+		uint64_t scale = (up + down) / (total / 32);
 		up /= scale;
 		down /= scale;
 	}
 
 	/* Get maximal dynamic range by choosing optimal shifts. */
 	int s = highbit64(total);
 	s = MIN(64 - s, 32);
 
+	ASSERT3U(frac, <=, 1ULL << 32);
 	uint64_t ofrac = (1ULL << 32) - frac;
 
 	if (frac >= 4 * ofrac)
 		up /= frac / (2 * ofrac + 1);
 	up = (up << s) / (total >> (32 - s));
 	if (ofrac >= 4 * frac)
 		down /= ofrac / (2 * frac + 1);
 	down = (down << s) / (total >> (32 - s));
 	down = down * 100 / balance;
 
+	ASSERT3U(up, <=, (1ULL << 32) - frac);
+	ASSERT3U(down, <=, frac);
 	return (frac + up - down);
 }
 
 /*
  * Evict buffers from the cache, such that arcstat_size is capped by arc_c.
  */
 static uint64_t
 arc_evict(void)
 {
 	uint64_t bytes, total_evicted = 0;
 	int64_t e, mrud, mrum, mfud, mfum, w;
 	static uint64_t ogrd, ogrm, ogfd, ogfm;
 	static uint64_t gsrd, gsrm, gsfd, gsfm;
 	uint64_t ngrd, ngrm, ngfd, ngfm;
 
 	/* Get current size of ARC states we can evict from. */
 	mrud = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_DATA]) +
 	    zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]);
 	mrum = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA]) +
 	    zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
 	mfud = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
 	mfum = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
 	uint64_t d = mrud + mfud;
 	uint64_t m = mrum + mfum;
 	uint64_t t = d + m;
 
 	/* Get ARC ghost hits since last eviction. */
 	ngrd = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
 	uint64_t grd = ngrd - ogrd;
 	ogrd = ngrd;
 	ngrm = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
 	uint64_t grm = ngrm - ogrm;
 	ogrm = ngrm;
 	ngfd = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
 	uint64_t gfd = ngfd - ogfd;
 	ogfd = ngfd;
 	ngfm = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
 	uint64_t gfm = ngfm - ogfm;
 	ogfm = ngfm;
 
 	/* Adjust ARC states balance based on ghost hits. */
 	arc_meta = arc_evict_adj(arc_meta, gsrd + gsrm + gsfd + gsfm,
 	    grm + gfm, grd + gfd, zfs_arc_meta_balance);
 	arc_pd = arc_evict_adj(arc_pd, gsrd + gsfd, grd, gfd, 100);
 	arc_pm = arc_evict_adj(arc_pm, gsrm + gsfm, grm, gfm, 100);
 
 	uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
 	uint64_t ac = arc_c;
 	int64_t wt = t - (asize - ac);
 
 	/*
 	 * Try to reduce pinned dnodes if more than 3/4 of wanted metadata
 	 * target is not evictable or if they go over arc_dnode_limit.
 	 */
 	int64_t prune = 0;
 	int64_t dn = wmsum_value(&arc_sums.arcstat_dnode_size);
 	w = wt * (int64_t)(arc_meta >> 16) >> 16;
 	if (zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA]) +
 	    zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA]) -
 	    zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA]) -
 	    zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]) >
 	    w * 3 / 4) {
 		prune = dn / sizeof (dnode_t) *
 		    zfs_arc_dnode_reduce_percent / 100;
 	} else if (dn > arc_dnode_limit) {
 		prune = (dn - arc_dnode_limit) / sizeof (dnode_t) *
 		    zfs_arc_dnode_reduce_percent / 100;
 	}
 	if (prune > 0)
 		arc_prune_async(prune);
 
 	/* Evict MRU metadata. */
 	w = wt * (int64_t)(arc_meta * arc_pm >> 48) >> 16;
 	e = MIN((int64_t)(asize - ac), (int64_t)(mrum - w));
 	bytes = arc_evict_impl(arc_mru, ARC_BUFC_METADATA, e);
 	total_evicted += bytes;
 	mrum -= bytes;
 	asize -= bytes;
 
 	/* Evict MFU metadata. */
 	w = wt * (int64_t)(arc_meta >> 16) >> 16;
 	e = MIN((int64_t)(asize - ac), (int64_t)(m - bytes - w));
 	bytes = arc_evict_impl(arc_mfu, ARC_BUFC_METADATA, e);
 	total_evicted += bytes;
 	mfum -= bytes;
 	asize -= bytes;
 
 	/* Evict MRU data. */
 	wt -= m - total_evicted;
 	w = wt * (int64_t)(arc_pd >> 16) >> 16;
 	e = MIN((int64_t)(asize - ac), (int64_t)(mrud - w));
 	bytes = arc_evict_impl(arc_mru, ARC_BUFC_DATA, e);
 	total_evicted += bytes;
 	mrud -= bytes;
 	asize -= bytes;
 
 	/* Evict MFU data. */
 	e = asize - ac;
 	bytes = arc_evict_impl(arc_mfu, ARC_BUFC_DATA, e);
 	mfud -= bytes;
 	total_evicted += bytes;
 
 	/*
 	 * Evict ghost lists
 	 *
 	 * Size of each state's ghost list represents how much that state
 	 * may grow by shrinking the other states.  Would it need to shrink
 	 * other states to zero (that is unlikely), its ghost size would be
 	 * equal to sum of other three state sizes.  But excessive ghost
 	 * size may result in false ghost hits (too far back), that may
 	 * never result in real cache hits if several states are competing.
 	 * So choose some arbitraty point of 1/2 of other state sizes.
 	 */
 	gsrd = (mrum + mfud + mfum) / 2;
 	e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]) -
 	    gsrd;
 	(void) arc_evict_impl(arc_mru_ghost, ARC_BUFC_DATA, e);
 
 	gsrm = (mrud + mfud + mfum) / 2;
 	e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]) -
 	    gsrm;
 	(void) arc_evict_impl(arc_mru_ghost, ARC_BUFC_METADATA, e);
 
 	gsfd = (mrud + mrum + mfum) / 2;
 	e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]) -
 	    gsfd;
 	(void) arc_evict_impl(arc_mfu_ghost, ARC_BUFC_DATA, e);
 
 	gsfm = (mrud + mrum + mfud) / 2;
 	e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]) -
 	    gsfm;
 	(void) arc_evict_impl(arc_mfu_ghost, ARC_BUFC_METADATA, e);
 
 	return (total_evicted);
 }
 
 void
 arc_flush(spa_t *spa, boolean_t retry)
 {
 	uint64_t guid = 0;
 
 	/*
 	 * If retry is B_TRUE, a spa must not be specified since we have
 	 * no good way to determine if all of a spa's buffers have been
 	 * evicted from an arc state.
 	 */
 	ASSERT(!retry || spa == NULL);
 
 	if (spa != NULL)
 		guid = spa_load_guid(spa);
 
 	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
 	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
 
 	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
 	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
 
 	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
 	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
 
 	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
 	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
 
 	(void) arc_flush_state(arc_uncached, guid, ARC_BUFC_DATA, retry);
 	(void) arc_flush_state(arc_uncached, guid, ARC_BUFC_METADATA, retry);
 }
 
 void
 arc_reduce_target_size(int64_t to_free)
 {
 	uint64_t c = arc_c;
 
 	if (c <= arc_c_min)
 		return;
 
 	/*
 	 * All callers want the ARC to actually evict (at least) this much
 	 * memory.  Therefore we reduce from the lower of the current size and
 	 * the target size.  This way, even if arc_c is much higher than
 	 * arc_size (as can be the case after many calls to arc_freed(), we will
 	 * immediately have arc_c < arc_size and therefore the arc_evict_zthr
 	 * will evict.
 	 */
 	uint64_t asize = aggsum_value(&arc_sums.arcstat_size);
 	if (asize < c)
 		to_free += c - asize;
 	arc_c = MAX((int64_t)c - to_free, (int64_t)arc_c_min);
 
 	/* See comment in arc_evict_cb_check() on why lock+flag */
 	mutex_enter(&arc_evict_lock);
 	arc_evict_needed = B_TRUE;
 	mutex_exit(&arc_evict_lock);
 	zthr_wakeup(arc_evict_zthr);
 }
 
 /*
  * Determine if the system is under memory pressure and is asking
  * to reclaim memory. A return value of B_TRUE indicates that the system
  * is under memory pressure and that the arc should adjust accordingly.
  */
 boolean_t
 arc_reclaim_needed(void)
 {
 	return (arc_available_memory() < 0);
 }
 
 void
 arc_kmem_reap_soon(void)
 {
 	size_t			i;
 	kmem_cache_t		*prev_cache = NULL;
 	kmem_cache_t		*prev_data_cache = NULL;
 
 #ifdef _KERNEL
 #if defined(_ILP32)
 	/*
 	 * Reclaim unused memory from all kmem caches.
 	 */
 	kmem_reap();
 #endif
 #endif
 
 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
 #if defined(_ILP32)
 		/* reach upper limit of cache size on 32-bit */
 		if (zio_buf_cache[i] == NULL)
 			break;
 #endif
 		if (zio_buf_cache[i] != prev_cache) {
 			prev_cache = zio_buf_cache[i];
 			kmem_cache_reap_now(zio_buf_cache[i]);
 		}
 		if (zio_data_buf_cache[i] != prev_data_cache) {
 			prev_data_cache = zio_data_buf_cache[i];
 			kmem_cache_reap_now(zio_data_buf_cache[i]);
 		}
 	}
 	kmem_cache_reap_now(buf_cache);
 	kmem_cache_reap_now(hdr_full_cache);
 	kmem_cache_reap_now(hdr_l2only_cache);
 	kmem_cache_reap_now(zfs_btree_leaf_cache);
 	abd_cache_reap_now();
 }
 
 static boolean_t
 arc_evict_cb_check(void *arg, zthr_t *zthr)
 {
 	(void) arg, (void) zthr;
 
 #ifdef ZFS_DEBUG
 	/*
 	 * This is necessary in order to keep the kstat information
 	 * up to date for tools that display kstat data such as the
 	 * mdb ::arc dcmd and the Linux crash utility.  These tools
 	 * typically do not call kstat's update function, but simply
 	 * dump out stats from the most recent update.  Without
 	 * this call, these commands may show stale stats for the
 	 * anon, mru, mru_ghost, mfu, and mfu_ghost lists.  Even
 	 * with this call, the data might be out of date if the
 	 * evict thread hasn't been woken recently; but that should
 	 * suffice.  The arc_state_t structures can be queried
 	 * directly if more accurate information is needed.
 	 */
 	if (arc_ksp != NULL)
 		arc_ksp->ks_update(arc_ksp, KSTAT_READ);
 #endif
 
 	/*
 	 * We have to rely on arc_wait_for_eviction() to tell us when to
 	 * evict, rather than checking if we are overflowing here, so that we
 	 * are sure to not leave arc_wait_for_eviction() waiting on aew_cv.
 	 * If we have become "not overflowing" since arc_wait_for_eviction()
 	 * checked, we need to wake it up.  We could broadcast the CV here,
 	 * but arc_wait_for_eviction() may have not yet gone to sleep.  We
 	 * would need to use a mutex to ensure that this function doesn't
 	 * broadcast until arc_wait_for_eviction() has gone to sleep (e.g.
 	 * the arc_evict_lock).  However, the lock ordering of such a lock
 	 * would necessarily be incorrect with respect to the zthr_lock,
 	 * which is held before this function is called, and is held by
 	 * arc_wait_for_eviction() when it calls zthr_wakeup().
 	 */
 	if (arc_evict_needed)
 		return (B_TRUE);
 
 	/*
 	 * If we have buffers in uncached state, evict them periodically.
 	 */
 	return ((zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_DATA]) +
 	    zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]) &&
 	    ddi_get_lbolt() - arc_last_uncached_flush >
 	    MSEC_TO_TICK(arc_min_prefetch_ms / 2)));
 }
 
 /*
  * Keep arc_size under arc_c by running arc_evict which evicts data
  * from the ARC.
  */
 static void
 arc_evict_cb(void *arg, zthr_t *zthr)
 {
 	(void) arg, (void) zthr;
 
 	uint64_t evicted = 0;
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 
 	/* Always try to evict from uncached state. */
 	arc_last_uncached_flush = ddi_get_lbolt();
 	evicted += arc_flush_state(arc_uncached, 0, ARC_BUFC_DATA, B_FALSE);
 	evicted += arc_flush_state(arc_uncached, 0, ARC_BUFC_METADATA, B_FALSE);
 
 	/* Evict from other states only if told to. */
 	if (arc_evict_needed)
 		evicted += arc_evict();
 
 	/*
 	 * If evicted is zero, we couldn't evict anything
 	 * via arc_evict(). This could be due to hash lock
 	 * collisions, but more likely due to the majority of
 	 * arc buffers being unevictable. Therefore, even if
 	 * arc_size is above arc_c, another pass is unlikely to
 	 * be helpful and could potentially cause us to enter an
 	 * infinite loop.  Additionally, zthr_iscancelled() is
 	 * checked here so that if the arc is shutting down, the
 	 * broadcast will wake any remaining arc evict waiters.
 	 */
 	mutex_enter(&arc_evict_lock);
 	arc_evict_needed = !zthr_iscancelled(arc_evict_zthr) &&
 	    evicted > 0 && aggsum_compare(&arc_sums.arcstat_size, arc_c) > 0;
 	if (!arc_evict_needed) {
 		/*
 		 * We're either no longer overflowing, or we
 		 * can't evict anything more, so we should wake
 		 * arc_get_data_impl() sooner.
 		 */
 		arc_evict_waiter_t *aw;
 		while ((aw = list_remove_head(&arc_evict_waiters)) != NULL) {
 			cv_broadcast(&aw->aew_cv);
 		}
 		arc_set_need_free();
 	}
 	mutex_exit(&arc_evict_lock);
 	spl_fstrans_unmark(cookie);
 }
 
 static boolean_t
 arc_reap_cb_check(void *arg, zthr_t *zthr)
 {
 	(void) arg, (void) zthr;
 
 	int64_t free_memory = arc_available_memory();
 	static int reap_cb_check_counter = 0;
 
 	/*
 	 * If a kmem reap is already active, don't schedule more.  We must
 	 * check for this because kmem_cache_reap_soon() won't actually
 	 * block on the cache being reaped (this is to prevent callers from
 	 * becoming implicitly blocked by a system-wide kmem reap -- which,
 	 * on a system with many, many full magazines, can take minutes).
 	 */
 	if (!kmem_cache_reap_active() && free_memory < 0) {
 
 		arc_no_grow = B_TRUE;
 		arc_warm = B_TRUE;
 		/*
 		 * Wait at least zfs_grow_retry (default 5) seconds
 		 * before considering growing.
 		 */
 		arc_growtime = gethrtime() + SEC2NSEC(arc_grow_retry);
 		return (B_TRUE);
 	} else if (free_memory < arc_c >> arc_no_grow_shift) {
 		arc_no_grow = B_TRUE;
 	} else if (gethrtime() >= arc_growtime) {
 		arc_no_grow = B_FALSE;
 	}
 
 	/*
 	 * Called unconditionally every 60 seconds to reclaim unused
 	 * zstd compression and decompression context. This is done
 	 * here to avoid the need for an independent thread.
 	 */
 	if (!((reap_cb_check_counter++) % 60))
 		zfs_zstd_cache_reap_now();
 
 	return (B_FALSE);
 }
 
 /*
  * Keep enough free memory in the system by reaping the ARC's kmem
  * caches.  To cause more slabs to be reapable, we may reduce the
  * target size of the cache (arc_c), causing the arc_evict_cb()
  * to free more buffers.
  */
 static void
 arc_reap_cb(void *arg, zthr_t *zthr)
 {
 	(void) arg, (void) zthr;
 
 	int64_t free_memory;
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 
 	/*
 	 * Kick off asynchronous kmem_reap()'s of all our caches.
 	 */
 	arc_kmem_reap_soon();
 
 	/*
 	 * Wait at least arc_kmem_cache_reap_retry_ms between
 	 * arc_kmem_reap_soon() calls. Without this check it is possible to
 	 * end up in a situation where we spend lots of time reaping
 	 * caches, while we're near arc_c_min.  Waiting here also gives the
 	 * subsequent free memory check a chance of finding that the
 	 * asynchronous reap has already freed enough memory, and we don't
 	 * need to call arc_reduce_target_size().
 	 */
 	delay((hz * arc_kmem_cache_reap_retry_ms + 999) / 1000);
 
 	/*
 	 * Reduce the target size as needed to maintain the amount of free
 	 * memory in the system at a fraction of the arc_size (1/128th by
 	 * default).  If oversubscribed (free_memory < 0) then reduce the
 	 * target arc_size by the deficit amount plus the fractional
 	 * amount.  If free memory is positive but less than the fractional
 	 * amount, reduce by what is needed to hit the fractional amount.
 	 */
 	free_memory = arc_available_memory();
 
 	int64_t can_free = arc_c - arc_c_min;
 	if (can_free > 0) {
 		int64_t to_free = (can_free >> arc_shrink_shift) - free_memory;
 		if (to_free > 0)
 			arc_reduce_target_size(to_free);
 	}
 	spl_fstrans_unmark(cookie);
 }
 
 #ifdef _KERNEL
 /*
  * Determine the amount of memory eligible for eviction contained in the
  * ARC. All clean data reported by the ghost lists can always be safely
  * evicted. Due to arc_c_min, the same does not hold for all clean data
  * contained by the regular mru and mfu lists.
  *
  * In the case of the regular mru and mfu lists, we need to report as
  * much clean data as possible, such that evicting that same reported
  * data will not bring arc_size below arc_c_min. Thus, in certain
  * circumstances, the total amount of clean data in the mru and mfu
  * lists might not actually be evictable.
  *
  * The following two distinct cases are accounted for:
  *
  * 1. The sum of the amount of dirty data contained by both the mru and
  *    mfu lists, plus the ARC's other accounting (e.g. the anon list),
  *    is greater than or equal to arc_c_min.
  *    (i.e. amount of dirty data >= arc_c_min)
  *
  *    This is the easy case; all clean data contained by the mru and mfu
  *    lists is evictable. Evicting all clean data can only drop arc_size
  *    to the amount of dirty data, which is greater than arc_c_min.
  *
  * 2. The sum of the amount of dirty data contained by both the mru and
  *    mfu lists, plus the ARC's other accounting (e.g. the anon list),
  *    is less than arc_c_min.
  *    (i.e. arc_c_min > amount of dirty data)
  *
  *    2.1. arc_size is greater than or equal arc_c_min.
  *         (i.e. arc_size >= arc_c_min > amount of dirty data)
  *
  *         In this case, not all clean data from the regular mru and mfu
  *         lists is actually evictable; we must leave enough clean data
  *         to keep arc_size above arc_c_min. Thus, the maximum amount of
  *         evictable data from the two lists combined, is exactly the
  *         difference between arc_size and arc_c_min.
  *
  *    2.2. arc_size is less than arc_c_min
  *         (i.e. arc_c_min > arc_size > amount of dirty data)
  *
  *         In this case, none of the data contained in the mru and mfu
  *         lists is evictable, even if it's clean. Since arc_size is
  *         already below arc_c_min, evicting any more would only
  *         increase this negative difference.
  */
 
 #endif /* _KERNEL */
 
 /*
  * Adapt arc info given the number of bytes we are trying to add and
  * the state that we are coming from.  This function is only called
  * when we are adding new content to the cache.
  */
 static void
 arc_adapt(uint64_t bytes)
 {
 	/*
 	 * Wake reap thread if we do not have any available memory
 	 */
 	if (arc_reclaim_needed()) {
 		zthr_wakeup(arc_reap_zthr);
 		return;
 	}
 
 	if (arc_no_grow)
 		return;
 
 	if (arc_c >= arc_c_max)
 		return;
 
 	/*
 	 * If we're within (2 * maxblocksize) bytes of the target
 	 * cache size, increment the target cache size
 	 */
 	if (aggsum_upper_bound(&arc_sums.arcstat_size) +
 	    2 * SPA_MAXBLOCKSIZE >= arc_c) {
 		uint64_t dc = MAX(bytes, SPA_OLD_MAXBLOCKSIZE);
 		if (atomic_add_64_nv(&arc_c, dc) > arc_c_max)
 			arc_c = arc_c_max;
 	}
 }
 
 /*
  * Check if arc_size has grown past our upper threshold, determined by
  * zfs_arc_overflow_shift.
  */
 static arc_ovf_level_t
 arc_is_overflowing(boolean_t use_reserve)
 {
 	/* Always allow at least one block of overflow */
 	int64_t overflow = MAX(SPA_MAXBLOCKSIZE,
 	    arc_c >> zfs_arc_overflow_shift);
 
 	/*
 	 * We just compare the lower bound here for performance reasons. Our
 	 * primary goals are to make sure that the arc never grows without
 	 * bound, and that it can reach its maximum size. This check
 	 * accomplishes both goals. The maximum amount we could run over by is
 	 * 2 * aggsum_borrow_multiplier * NUM_CPUS * the average size of a block
 	 * in the ARC. In practice, that's in the tens of MB, which is low
 	 * enough to be safe.
 	 */
 	int64_t over = aggsum_lower_bound(&arc_sums.arcstat_size) -
 	    arc_c - overflow / 2;
 	if (!use_reserve)
 		overflow /= 2;
 	return (over < 0 ? ARC_OVF_NONE :
 	    over < overflow ? ARC_OVF_SOME : ARC_OVF_SEVERE);
 }
 
 static abd_t *
 arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,
     int alloc_flags)
 {
 	arc_buf_contents_t type = arc_buf_type(hdr);
 
 	arc_get_data_impl(hdr, size, tag, alloc_flags);
 	if (alloc_flags & ARC_HDR_ALLOC_LINEAR)
 		return (abd_alloc_linear(size, type == ARC_BUFC_METADATA));
 	else
 		return (abd_alloc(size, type == ARC_BUFC_METADATA));
 }
 
 static void *
 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)
 {
 	arc_buf_contents_t type = arc_buf_type(hdr);
 
 	arc_get_data_impl(hdr, size, tag, 0);
 	if (type == ARC_BUFC_METADATA) {
 		return (zio_buf_alloc(size));
 	} else {
 		ASSERT(type == ARC_BUFC_DATA);
 		return (zio_data_buf_alloc(size));
 	}
 }
 
 /*
  * Wait for the specified amount of data (in bytes) to be evicted from the
  * ARC, and for there to be sufficient free memory in the system.  Waiting for
  * eviction ensures that the memory used by the ARC decreases.  Waiting for
  * free memory ensures that the system won't run out of free pages, regardless
  * of ARC behavior and settings.  See arc_lowmem_init().
  */
 void
 arc_wait_for_eviction(uint64_t amount, boolean_t use_reserve)
 {
 	switch (arc_is_overflowing(use_reserve)) {
 	case ARC_OVF_NONE:
 		return;
 	case ARC_OVF_SOME:
 		/*
 		 * This is a bit racy without taking arc_evict_lock, but the
 		 * worst that can happen is we either call zthr_wakeup() extra
 		 * time due to race with other thread here, or the set flag
 		 * get cleared by arc_evict_cb(), which is unlikely due to
 		 * big hysteresis, but also not important since at this level
 		 * of overflow the eviction is purely advisory.  Same time
 		 * taking the global lock here every time without waiting for
 		 * the actual eviction creates a significant lock contention.
 		 */
 		if (!arc_evict_needed) {
 			arc_evict_needed = B_TRUE;
 			zthr_wakeup(arc_evict_zthr);
 		}
 		return;
 	case ARC_OVF_SEVERE:
 	default:
 	{
 		arc_evict_waiter_t aw;
 		list_link_init(&aw.aew_node);
 		cv_init(&aw.aew_cv, NULL, CV_DEFAULT, NULL);
 
 		uint64_t last_count = 0;
 		mutex_enter(&arc_evict_lock);
 		if (!list_is_empty(&arc_evict_waiters)) {
 			arc_evict_waiter_t *last =
 			    list_tail(&arc_evict_waiters);
 			last_count = last->aew_count;
 		} else if (!arc_evict_needed) {
 			arc_evict_needed = B_TRUE;
 			zthr_wakeup(arc_evict_zthr);
 		}
 		/*
 		 * Note, the last waiter's count may be less than
 		 * arc_evict_count if we are low on memory in which
 		 * case arc_evict_state_impl() may have deferred
 		 * wakeups (but still incremented arc_evict_count).
 		 */
 		aw.aew_count = MAX(last_count, arc_evict_count) + amount;
 
 		list_insert_tail(&arc_evict_waiters, &aw);
 
 		arc_set_need_free();
 
 		DTRACE_PROBE3(arc__wait__for__eviction,
 		    uint64_t, amount,
 		    uint64_t, arc_evict_count,
 		    uint64_t, aw.aew_count);
 
 		/*
 		 * We will be woken up either when arc_evict_count reaches
 		 * aew_count, or when the ARC is no longer overflowing and
 		 * eviction completes.
 		 * In case of "false" wakeup, we will still be on the list.
 		 */
 		do {
 			cv_wait(&aw.aew_cv, &arc_evict_lock);
 		} while (list_link_active(&aw.aew_node));
 		mutex_exit(&arc_evict_lock);
 
 		cv_destroy(&aw.aew_cv);
 	}
 	}
 }
 
 /*
  * Allocate a block and return it to the caller. If we are hitting the
  * hard limit for the cache size, we must sleep, waiting for the eviction
  * thread to catch up. If we're past the target size but below the hard
  * limit, we'll only signal the reclaim thread and continue on.
  */
 static void
 arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,
     int alloc_flags)
 {
 	arc_adapt(size);
 
 	/*
 	 * If arc_size is currently overflowing, we must be adding data
 	 * faster than we are evicting.  To ensure we don't compound the
 	 * problem by adding more data and forcing arc_size to grow even
 	 * further past it's target size, we wait for the eviction thread to
 	 * make some progress.  We also wait for there to be sufficient free
 	 * memory in the system, as measured by arc_free_memory().
 	 *
 	 * Specifically, we wait for zfs_arc_eviction_pct percent of the
 	 * requested size to be evicted.  This should be more than 100%, to
 	 * ensure that that progress is also made towards getting arc_size
 	 * under arc_c.  See the comment above zfs_arc_eviction_pct.
 	 */
 	arc_wait_for_eviction(size * zfs_arc_eviction_pct / 100,
 	    alloc_flags & ARC_HDR_USE_RESERVE);
 
 	arc_buf_contents_t type = arc_buf_type(hdr);
 	if (type == ARC_BUFC_METADATA) {
 		arc_space_consume(size, ARC_SPACE_META);
 	} else {
 		arc_space_consume(size, ARC_SPACE_DATA);
 	}
 
 	/*
 	 * Update the state size.  Note that ghost states have a
 	 * "ghost size" and so don't need to be updated.
 	 */
 	arc_state_t *state = hdr->b_l1hdr.b_state;
 	if (!GHOST_STATE(state)) {
 
 		(void) zfs_refcount_add_many(&state->arcs_size[type], size,
 		    tag);
 
 		/*
 		 * If this is reached via arc_read, the link is
 		 * protected by the hash lock. If reached via
 		 * arc_buf_alloc, the header should not be accessed by
 		 * any other thread. And, if reached via arc_read_done,
 		 * the hash lock will protect it if it's found in the
 		 * hash table; otherwise no other thread should be
 		 * trying to [add|remove]_reference it.
 		 */
 		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
 			ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 			(void) zfs_refcount_add_many(&state->arcs_esize[type],
 			    size, tag);
 		}
 	}
 }
 
 static void
 arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size,
     const void *tag)
 {
 	arc_free_data_impl(hdr, size, tag);
 	abd_free(abd);
 }
 
 static void
 arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, const void *tag)
 {
 	arc_buf_contents_t type = arc_buf_type(hdr);
 
 	arc_free_data_impl(hdr, size, tag);
 	if (type == ARC_BUFC_METADATA) {
 		zio_buf_free(buf, size);
 	} else {
 		ASSERT(type == ARC_BUFC_DATA);
 		zio_data_buf_free(buf, size);
 	}
 }
 
 /*
  * Free the arc data buffer.
  */
 static void
 arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)
 {
 	arc_state_t *state = hdr->b_l1hdr.b_state;
 	arc_buf_contents_t type = arc_buf_type(hdr);
 
 	/* protected by hash lock, if in the hash table */
 	if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
 		ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 		ASSERT(state != arc_anon && state != arc_l2c_only);
 
 		(void) zfs_refcount_remove_many(&state->arcs_esize[type],
 		    size, tag);
 	}
 	(void) zfs_refcount_remove_many(&state->arcs_size[type], size, tag);
 
 	VERIFY3U(hdr->b_type, ==, type);
 	if (type == ARC_BUFC_METADATA) {
 		arc_space_return(size, ARC_SPACE_META);
 	} else {
 		ASSERT(type == ARC_BUFC_DATA);
 		arc_space_return(size, ARC_SPACE_DATA);
 	}
 }
 
 /*
  * This routine is called whenever a buffer is accessed.
  */
 static void
 arc_access(arc_buf_hdr_t *hdr, arc_flags_t arc_flags, boolean_t hit)
 {
 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
 	ASSERT(HDR_HAS_L1HDR(hdr));
 
 	/*
 	 * Update buffer prefetch status.
 	 */
 	boolean_t was_prefetch = HDR_PREFETCH(hdr);
 	boolean_t now_prefetch = arc_flags & ARC_FLAG_PREFETCH;
 	if (was_prefetch != now_prefetch) {
 		if (was_prefetch) {
 			ARCSTAT_CONDSTAT(hit, demand_hit, demand_iohit,
 			    HDR_PRESCIENT_PREFETCH(hdr), prescient, predictive,
 			    prefetch);
 		}
 		if (HDR_HAS_L2HDR(hdr))
 			l2arc_hdr_arcstats_decrement_state(hdr);
 		if (was_prefetch) {
 			arc_hdr_clear_flags(hdr,
 			    ARC_FLAG_PREFETCH | ARC_FLAG_PRESCIENT_PREFETCH);
 		} else {
 			arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
 		}
 		if (HDR_HAS_L2HDR(hdr))
 			l2arc_hdr_arcstats_increment_state(hdr);
 	}
 	if (now_prefetch) {
 		if (arc_flags & ARC_FLAG_PRESCIENT_PREFETCH) {
 			arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
 			ARCSTAT_BUMP(arcstat_prescient_prefetch);
 		} else {
 			ARCSTAT_BUMP(arcstat_predictive_prefetch);
 		}
 	}
 	if (arc_flags & ARC_FLAG_L2CACHE)
 		arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
 
 	clock_t now = ddi_get_lbolt();
 	if (hdr->b_l1hdr.b_state == arc_anon) {
 		arc_state_t	*new_state;
 		/*
 		 * This buffer is not in the cache, and does not appear in
 		 * our "ghost" lists.  Add it to the MRU or uncached state.
 		 */
 		ASSERT0(hdr->b_l1hdr.b_arc_access);
 		hdr->b_l1hdr.b_arc_access = now;
 		if (HDR_UNCACHED(hdr)) {
 			new_state = arc_uncached;
 			DTRACE_PROBE1(new_state__uncached, arc_buf_hdr_t *,
 			    hdr);
 		} else {
 			new_state = arc_mru;
 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
 		}
 		arc_change_state(new_state, hdr);
 	} else if (hdr->b_l1hdr.b_state == arc_mru) {
 		/*
 		 * This buffer has been accessed once recently and either
 		 * its read is still in progress or it is in the cache.
 		 */
 		if (HDR_IO_IN_PROGRESS(hdr)) {
 			hdr->b_l1hdr.b_arc_access = now;
 			return;
 		}
 		hdr->b_l1hdr.b_mru_hits++;
 		ARCSTAT_BUMP(arcstat_mru_hits);
 
 		/*
 		 * If the previous access was a prefetch, then it already
 		 * handled possible promotion, so nothing more to do for now.
 		 */
 		if (was_prefetch) {
 			hdr->b_l1hdr.b_arc_access = now;
 			return;
 		}
 
 		/*
 		 * If more than ARC_MINTIME have passed from the previous
 		 * hit, promote the buffer to the MFU state.
 		 */
 		if (ddi_time_after(now, hdr->b_l1hdr.b_arc_access +
 		    ARC_MINTIME)) {
 			hdr->b_l1hdr.b_arc_access = now;
 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 			arc_change_state(arc_mfu, hdr);
 		}
 	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
 		arc_state_t	*new_state;
 		/*
 		 * This buffer has been accessed once recently, but was
 		 * evicted from the cache.  Would we have bigger MRU, it
 		 * would be an MRU hit, so handle it the same way, except
 		 * we don't need to check the previous access time.
 		 */
 		hdr->b_l1hdr.b_mru_ghost_hits++;
 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
 		hdr->b_l1hdr.b_arc_access = now;
 		wmsum_add(&arc_mru_ghost->arcs_hits[arc_buf_type(hdr)],
 		    arc_hdr_size(hdr));
 		if (was_prefetch) {
 			new_state = arc_mru;
 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
 		} else {
 			new_state = arc_mfu;
 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 		}
 		arc_change_state(new_state, hdr);
 	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
 		/*
 		 * This buffer has been accessed more than once and either
 		 * still in the cache or being restored from one of ghosts.
 		 */
 		if (!HDR_IO_IN_PROGRESS(hdr)) {
 			hdr->b_l1hdr.b_mfu_hits++;
 			ARCSTAT_BUMP(arcstat_mfu_hits);
 		}
 		hdr->b_l1hdr.b_arc_access = now;
 	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
 		/*
 		 * This buffer has been accessed more than once recently, but
 		 * has been evicted from the cache.  Would we have bigger MFU
 		 * it would stay in cache, so move it back to MFU state.
 		 */
 		hdr->b_l1hdr.b_mfu_ghost_hits++;
 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
 		hdr->b_l1hdr.b_arc_access = now;
 		wmsum_add(&arc_mfu_ghost->arcs_hits[arc_buf_type(hdr)],
 		    arc_hdr_size(hdr));
 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 		arc_change_state(arc_mfu, hdr);
 	} else if (hdr->b_l1hdr.b_state == arc_uncached) {
 		/*
 		 * This buffer is uncacheable, but we got a hit.  Probably
 		 * a demand read after prefetch.  Nothing more to do here.
 		 */
 		if (!HDR_IO_IN_PROGRESS(hdr))
 			ARCSTAT_BUMP(arcstat_uncached_hits);
 		hdr->b_l1hdr.b_arc_access = now;
 	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
 		/*
 		 * This buffer is on the 2nd Level ARC and was not accessed
 		 * for a long time, so treat it as new and put into MRU.
 		 */
 		hdr->b_l1hdr.b_arc_access = now;
 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
 		arc_change_state(arc_mru, hdr);
 	} else {
 		cmn_err(CE_PANIC, "invalid arc state 0x%p",
 		    hdr->b_l1hdr.b_state);
 	}
 }
 
 /*
  * This routine is called by dbuf_hold() to update the arc_access() state
  * which otherwise would be skipped for entries in the dbuf cache.
  */
 void
 arc_buf_access(arc_buf_t *buf)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	/*
 	 * Avoid taking the hash_lock when possible as an optimization.
 	 * The header must be checked again under the hash_lock in order
 	 * to handle the case where it is concurrently being released.
 	 */
 	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr))
 		return;
 
 	kmutex_t *hash_lock = HDR_LOCK(hdr);
 	mutex_enter(hash_lock);
 
 	if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
 		mutex_exit(hash_lock);
 		ARCSTAT_BUMP(arcstat_access_skip);
 		return;
 	}
 
 	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
 	    hdr->b_l1hdr.b_state == arc_mfu ||
 	    hdr->b_l1hdr.b_state == arc_uncached);
 
 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
 	arc_access(hdr, 0, B_TRUE);
 	mutex_exit(hash_lock);
 
 	ARCSTAT_BUMP(arcstat_hits);
 	ARCSTAT_CONDSTAT(B_TRUE /* demand */, demand, prefetch,
 	    !HDR_ISTYPE_METADATA(hdr), data, metadata, hits);
 }
 
 /* a generic arc_read_done_func_t which you can use */
 void
 arc_bcopy_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
     arc_buf_t *buf, void *arg)
 {
 	(void) zio, (void) zb, (void) bp;
 
 	if (buf == NULL)
 		return;
 
 	memcpy(arg, buf->b_data, arc_buf_size(buf));
 	arc_buf_destroy(buf, arg);
 }
 
 /* a generic arc_read_done_func_t */
 void
 arc_getbuf_func(zio_t *zio, const zbookmark_phys_t *zb, const blkptr_t *bp,
     arc_buf_t *buf, void *arg)
 {
 	(void) zb, (void) bp;
 	arc_buf_t **bufp = arg;
 
 	if (buf == NULL) {
 		ASSERT(zio == NULL || zio->io_error != 0);
 		*bufp = NULL;
 	} else {
 		ASSERT(zio == NULL || zio->io_error == 0);
 		*bufp = buf;
 		ASSERT(buf->b_data != NULL);
 	}
 }
 
 static void
 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
 {
 	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
 		ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
 		ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF);
 	} else {
 		if (HDR_COMPRESSION_ENABLED(hdr)) {
 			ASSERT3U(arc_hdr_get_compress(hdr), ==,
 			    BP_GET_COMPRESS(bp));
 		}
 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
 		ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
 		ASSERT3U(!!HDR_PROTECTED(hdr), ==, BP_IS_PROTECTED(bp));
 	}
 }
 
 static void
 arc_read_done(zio_t *zio)
 {
 	blkptr_t 	*bp = zio->io_bp;
 	arc_buf_hdr_t	*hdr = zio->io_private;
 	kmutex_t	*hash_lock = NULL;
 	arc_callback_t	*callback_list;
 	arc_callback_t	*acb;
 
 	/*
 	 * The hdr was inserted into hash-table and removed from lists
 	 * prior to starting I/O.  We should find this header, since
 	 * it's in the hash table, and it should be legit since it's
 	 * not possible to evict it during the I/O.  The only possible
 	 * reason for it not to be found is if we were freed during the
 	 * read.
 	 */
 	if (HDR_IN_HASH_TABLE(hdr)) {
 		arc_buf_hdr_t *found;
 
 		ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp));
 		ASSERT3U(hdr->b_dva.dva_word[0], ==,
 		    BP_IDENTITY(zio->io_bp)->dva_word[0]);
 		ASSERT3U(hdr->b_dva.dva_word[1], ==,
 		    BP_IDENTITY(zio->io_bp)->dva_word[1]);
 
 		found = buf_hash_find(hdr->b_spa, zio->io_bp, &hash_lock);
 
 		ASSERT((found == hdr &&
 		    DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
 		    (found == hdr && HDR_L2_READING(hdr)));
 		ASSERT3P(hash_lock, !=, NULL);
 	}
 
 	if (BP_IS_PROTECTED(bp)) {
 		hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
 		hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
 		zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
 		    hdr->b_crypt_hdr.b_iv);
 
 		if (zio->io_error == 0) {
 			if (BP_GET_TYPE(bp) == DMU_OT_INTENT_LOG) {
 				void *tmpbuf;
 
 				tmpbuf = abd_borrow_buf_copy(zio->io_abd,
 				    sizeof (zil_chain_t));
 				zio_crypt_decode_mac_zil(tmpbuf,
 				    hdr->b_crypt_hdr.b_mac);
 				abd_return_buf(zio->io_abd, tmpbuf,
 				    sizeof (zil_chain_t));
 			} else {
 				zio_crypt_decode_mac_bp(bp,
 				    hdr->b_crypt_hdr.b_mac);
 			}
 		}
 	}
 
 	if (zio->io_error == 0) {
 		/* byteswap if necessary */
 		if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
 			if (BP_GET_LEVEL(zio->io_bp) > 0) {
 				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
 			} else {
 				hdr->b_l1hdr.b_byteswap =
 				    DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
 			}
 		} else {
 			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
 		}
 		if (!HDR_L2_READING(hdr)) {
 			hdr->b_complevel = zio->io_prop.zp_complevel;
 		}
 	}
 
 	arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
 	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
 		arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
 
 	callback_list = hdr->b_l1hdr.b_acb;
 	ASSERT3P(callback_list, !=, NULL);
 	hdr->b_l1hdr.b_acb = NULL;
 
 	/*
 	 * If a read request has a callback (i.e. acb_done is not NULL), then we
 	 * make a buf containing the data according to the parameters which were
 	 * passed in. The implementation of arc_buf_alloc_impl() ensures that we
 	 * aren't needlessly decompressing the data multiple times.
 	 */
 	int callback_cnt = 0;
 	for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
 
 		/* We need the last one to call below in original order. */
 		callback_list = acb;
 
 		if (!acb->acb_done || acb->acb_nobuf)
 			continue;
 
 		callback_cnt++;
 
 		if (zio->io_error != 0)
 			continue;
 
 		int error = arc_buf_alloc_impl(hdr, zio->io_spa,
 		    &acb->acb_zb, acb->acb_private, acb->acb_encrypted,
 		    acb->acb_compressed, acb->acb_noauth, B_TRUE,
 		    &acb->acb_buf);
 
 		/*
 		 * Assert non-speculative zios didn't fail because an
 		 * encryption key wasn't loaded
 		 */
 		ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) ||
 		    error != EACCES);
 
 		/*
 		 * If we failed to decrypt, report an error now (as the zio
 		 * layer would have done if it had done the transforms).
 		 */
 		if (error == ECKSUM) {
 			ASSERT(BP_IS_PROTECTED(bp));
 			error = SET_ERROR(EIO);
 			if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
 				spa_log_error(zio->io_spa, &acb->acb_zb,
 				    &zio->io_bp->blk_birth);
 				(void) zfs_ereport_post(
 				    FM_EREPORT_ZFS_AUTHENTICATION,
 				    zio->io_spa, NULL, &acb->acb_zb, zio, 0);
 			}
 		}
 
 		if (error != 0) {
 			/*
 			 * Decompression or decryption failed.  Set
 			 * io_error so that when we call acb_done
 			 * (below), we will indicate that the read
 			 * failed. Note that in the unusual case
 			 * where one callback is compressed and another
 			 * uncompressed, we will mark all of them
 			 * as failed, even though the uncompressed
 			 * one can't actually fail.  In this case,
 			 * the hdr will not be anonymous, because
 			 * if there are multiple callbacks, it's
 			 * because multiple threads found the same
 			 * arc buf in the hash table.
 			 */
 			zio->io_error = error;
 		}
 	}
 
 	/*
 	 * If there are multiple callbacks, we must have the hash lock,
 	 * because the only way for multiple threads to find this hdr is
 	 * in the hash table.  This ensures that if there are multiple
 	 * callbacks, the hdr is not anonymous.  If it were anonymous,
 	 * we couldn't use arc_buf_destroy() in the error case below.
 	 */
 	ASSERT(callback_cnt < 2 || hash_lock != NULL);
 
 	if (zio->io_error == 0) {
 		arc_hdr_verify(hdr, zio->io_bp);
 	} else {
 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
 		if (hdr->b_l1hdr.b_state != arc_anon)
 			arc_change_state(arc_anon, hdr);
 		if (HDR_IN_HASH_TABLE(hdr))
 			buf_hash_remove(hdr);
 	}
 
 	arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
 	(void) remove_reference(hdr, hdr);
 
 	if (hash_lock != NULL)
 		mutex_exit(hash_lock);
 
 	/* execute each callback and free its structure */
 	while ((acb = callback_list) != NULL) {
 		if (acb->acb_done != NULL) {
 			if (zio->io_error != 0 && acb->acb_buf != NULL) {
 				/*
 				 * If arc_buf_alloc_impl() fails during
 				 * decompression, the buf will still be
 				 * allocated, and needs to be freed here.
 				 */
 				arc_buf_destroy(acb->acb_buf,
 				    acb->acb_private);
 				acb->acb_buf = NULL;
 			}
 			acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
 			    acb->acb_buf, acb->acb_private);
 		}
 
 		if (acb->acb_zio_dummy != NULL) {
 			acb->acb_zio_dummy->io_error = zio->io_error;
 			zio_nowait(acb->acb_zio_dummy);
 		}
 
 		callback_list = acb->acb_prev;
 		if (acb->acb_wait) {
 			mutex_enter(&acb->acb_wait_lock);
 			acb->acb_wait_error = zio->io_error;
 			acb->acb_wait = B_FALSE;
 			cv_signal(&acb->acb_wait_cv);
 			mutex_exit(&acb->acb_wait_lock);
 			/* acb will be freed by the waiting thread. */
 		} else {
 			kmem_free(acb, sizeof (arc_callback_t));
 		}
 	}
 }
 
 /*
  * "Read" the block at the specified DVA (in bp) via the
  * cache.  If the block is found in the cache, invoke the provided
  * callback immediately and return.  Note that the `zio' parameter
  * in the callback will be NULL in this case, since no IO was
  * required.  If the block is not in the cache pass the read request
  * on to the spa with a substitute callback function, so that the
  * requested block will be added to the cache.
  *
  * If a read request arrives for a block that has a read in-progress,
  * either wait for the in-progress read to complete (and return the
  * results); or, if this is a read with a "done" func, add a record
  * to the read to invoke the "done" func when the read completes,
  * and return; or just return.
  *
  * arc_read_done() will invoke all the requested "done" functions
  * for readers of this block.
  */
 int
 arc_read(zio_t *pio, spa_t *spa, const blkptr_t *bp,
     arc_read_done_func_t *done, void *private, zio_priority_t priority,
     int zio_flags, arc_flags_t *arc_flags, const zbookmark_phys_t *zb)
 {
 	arc_buf_hdr_t *hdr = NULL;
 	kmutex_t *hash_lock = NULL;
 	zio_t *rzio;
 	uint64_t guid = spa_load_guid(spa);
 	boolean_t compressed_read = (zio_flags & ZIO_FLAG_RAW_COMPRESS) != 0;
 	boolean_t encrypted_read = BP_IS_ENCRYPTED(bp) &&
 	    (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
 	boolean_t noauth_read = BP_IS_AUTHENTICATED(bp) &&
 	    (zio_flags & ZIO_FLAG_RAW_ENCRYPT) != 0;
 	boolean_t embedded_bp = !!BP_IS_EMBEDDED(bp);
 	boolean_t no_buf = *arc_flags & ARC_FLAG_NO_BUF;
 	arc_buf_t *buf = NULL;
 	int rc = 0;
 
 	ASSERT(!embedded_bp ||
 	    BPE_GET_ETYPE(bp) == BP_EMBEDDED_TYPE_DATA);
 	ASSERT(!BP_IS_HOLE(bp));
 	ASSERT(!BP_IS_REDACTED(bp));
 
 	/*
 	 * Normally SPL_FSTRANS will already be set since kernel threads which
 	 * expect to call the DMU interfaces will set it when created.  System
 	 * calls are similarly handled by setting/cleaning the bit in the
 	 * registered callback (module/os/.../zfs/zpl_*).
 	 *
 	 * External consumers such as Lustre which call the exported DMU
 	 * interfaces may not have set SPL_FSTRANS.  To avoid a deadlock
 	 * on the hash_lock always set and clear the bit.
 	 */
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 top:
 	/*
 	 * Verify the block pointer contents are reasonable.  This should
 	 * always be the case since the blkptr is protected by a checksum.
 	 * However, if there is damage it's desirable to detect this early
 	 * and treat it as a checksum error.  This allows an alternate blkptr
 	 * to be tried when one is available (e.g. ditto blocks).
 	 */
 	if (!zfs_blkptr_verify(spa, bp, (zio_flags & ZIO_FLAG_CONFIG_WRITER) ?
 	    BLK_CONFIG_HELD : BLK_CONFIG_NEEDED, BLK_VERIFY_LOG)) {
 		rc = SET_ERROR(ECKSUM);
 		goto done;
 	}
 
 	if (!embedded_bp) {
 		/*
 		 * Embedded BP's have no DVA and require no I/O to "read".
 		 * Create an anonymous arc buf to back it.
 		 */
 		hdr = buf_hash_find(guid, bp, &hash_lock);
 	}
 
 	/*
 	 * Determine if we have an L1 cache hit or a cache miss. For simplicity
 	 * we maintain encrypted data separately from compressed / uncompressed
 	 * data. If the user is requesting raw encrypted data and we don't have
 	 * that in the header we will read from disk to guarantee that we can
 	 * get it even if the encryption keys aren't loaded.
 	 */
 	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) ||
 	    (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) {
 		boolean_t is_data = !HDR_ISTYPE_METADATA(hdr);
 
 		if (HDR_IO_IN_PROGRESS(hdr)) {
 			if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
 				mutex_exit(hash_lock);
 				ARCSTAT_BUMP(arcstat_cached_only_in_progress);
 				rc = SET_ERROR(ENOENT);
 				goto done;
 			}
 
 			zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
 			ASSERT3P(head_zio, !=, NULL);
 			if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
 			    priority == ZIO_PRIORITY_SYNC_READ) {
 				/*
 				 * This is a sync read that needs to wait for
 				 * an in-flight async read. Request that the
 				 * zio have its priority upgraded.
 				 */
 				zio_change_priority(head_zio, priority);
 				DTRACE_PROBE1(arc__async__upgrade__sync,
 				    arc_buf_hdr_t *, hdr);
 				ARCSTAT_BUMP(arcstat_async_upgrade_sync);
 			}
 
 			DTRACE_PROBE1(arc__iohit, arc_buf_hdr_t *, hdr);
 			arc_access(hdr, *arc_flags, B_FALSE);
 
 			/*
 			 * If there are multiple threads reading the same block
 			 * and that block is not yet in the ARC, then only one
 			 * thread will do the physical I/O and all other
 			 * threads will wait until that I/O completes.
 			 * Synchronous reads use the acb_wait_cv whereas nowait
 			 * reads register a callback. Both are signalled/called
 			 * in arc_read_done.
 			 *
 			 * Errors of the physical I/O may need to be propagated.
 			 * Synchronous read errors are returned here from
 			 * arc_read_done via acb_wait_error.  Nowait reads
 			 * attach the acb_zio_dummy zio to pio and
 			 * arc_read_done propagates the physical I/O's io_error
 			 * to acb_zio_dummy, and thereby to pio.
 			 */
 			arc_callback_t *acb = NULL;
 			if (done || pio || *arc_flags & ARC_FLAG_WAIT) {
 				acb = kmem_zalloc(sizeof (arc_callback_t),
 				    KM_SLEEP);
 				acb->acb_done = done;
 				acb->acb_private = private;
 				acb->acb_compressed = compressed_read;
 				acb->acb_encrypted = encrypted_read;
 				acb->acb_noauth = noauth_read;
 				acb->acb_nobuf = no_buf;
 				if (*arc_flags & ARC_FLAG_WAIT) {
 					acb->acb_wait = B_TRUE;
 					mutex_init(&acb->acb_wait_lock, NULL,
 					    MUTEX_DEFAULT, NULL);
 					cv_init(&acb->acb_wait_cv, NULL,
 					    CV_DEFAULT, NULL);
 				}
 				acb->acb_zb = *zb;
 				if (pio != NULL) {
 					acb->acb_zio_dummy = zio_null(pio,
 					    spa, NULL, NULL, NULL, zio_flags);
 				}
 				acb->acb_zio_head = head_zio;
 				acb->acb_next = hdr->b_l1hdr.b_acb;
 				hdr->b_l1hdr.b_acb->acb_prev = acb;
 				hdr->b_l1hdr.b_acb = acb;
 			}
 			mutex_exit(hash_lock);
 
 			ARCSTAT_BUMP(arcstat_iohits);
 			ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
 			    demand, prefetch, is_data, data, metadata, iohits);
 
 			if (*arc_flags & ARC_FLAG_WAIT) {
 				mutex_enter(&acb->acb_wait_lock);
 				while (acb->acb_wait) {
 					cv_wait(&acb->acb_wait_cv,
 					    &acb->acb_wait_lock);
 				}
 				rc = acb->acb_wait_error;
 				mutex_exit(&acb->acb_wait_lock);
 				mutex_destroy(&acb->acb_wait_lock);
 				cv_destroy(&acb->acb_wait_cv);
 				kmem_free(acb, sizeof (arc_callback_t));
 			}
 			goto out;
 		}
 
 		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
 		    hdr->b_l1hdr.b_state == arc_mfu ||
 		    hdr->b_l1hdr.b_state == arc_uncached);
 
 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
 		arc_access(hdr, *arc_flags, B_TRUE);
 
 		if (done && !no_buf) {
 			ASSERT(!embedded_bp || !BP_IS_HOLE(bp));
 
 			/* Get a buf with the desired data in it. */
 			rc = arc_buf_alloc_impl(hdr, spa, zb, private,
 			    encrypted_read, compressed_read, noauth_read,
 			    B_TRUE, &buf);
 			if (rc == ECKSUM) {
 				/*
 				 * Convert authentication and decryption errors
 				 * to EIO (and generate an ereport if needed)
 				 * before leaving the ARC.
 				 */
 				rc = SET_ERROR(EIO);
 				if ((zio_flags & ZIO_FLAG_SPECULATIVE) == 0) {
 					spa_log_error(spa, zb, &hdr->b_birth);
 					(void) zfs_ereport_post(
 					    FM_EREPORT_ZFS_AUTHENTICATION,
 					    spa, NULL, zb, NULL, 0);
 				}
 			}
 			if (rc != 0) {
 				arc_buf_destroy_impl(buf);
 				buf = NULL;
 				(void) remove_reference(hdr, private);
 			}
 
 			/* assert any errors weren't due to unloaded keys */
 			ASSERT((zio_flags & ZIO_FLAG_SPECULATIVE) ||
 			    rc != EACCES);
 		}
 		mutex_exit(hash_lock);
 		ARCSTAT_BUMP(arcstat_hits);
 		ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
 		    demand, prefetch, is_data, data, metadata, hits);
 		*arc_flags |= ARC_FLAG_CACHED;
 		goto done;
 	} else {
 		uint64_t lsize = BP_GET_LSIZE(bp);
 		uint64_t psize = BP_GET_PSIZE(bp);
 		arc_callback_t *acb;
 		vdev_t *vd = NULL;
 		uint64_t addr = 0;
 		boolean_t devw = B_FALSE;
 		uint64_t size;
 		abd_t *hdr_abd;
 		int alloc_flags = encrypted_read ? ARC_HDR_ALLOC_RDATA : 0;
 		arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
 
 		if (*arc_flags & ARC_FLAG_CACHED_ONLY) {
 			if (hash_lock != NULL)
 				mutex_exit(hash_lock);
 			rc = SET_ERROR(ENOENT);
 			goto done;
 		}
 
 		if (hdr == NULL) {
 			/*
 			 * This block is not in the cache or it has
 			 * embedded data.
 			 */
 			arc_buf_hdr_t *exists = NULL;
 			hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
 			    BP_IS_PROTECTED(bp), BP_GET_COMPRESS(bp), 0, type);
 
 			if (!embedded_bp) {
 				hdr->b_dva = *BP_IDENTITY(bp);
 				hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
 				exists = buf_hash_insert(hdr, &hash_lock);
 			}
 			if (exists != NULL) {
 				/* somebody beat us to the hash insert */
 				mutex_exit(hash_lock);
 				buf_discard_identity(hdr);
 				arc_hdr_destroy(hdr);
 				goto top; /* restart the IO request */
 			}
 		} else {
 			/*
 			 * This block is in the ghost cache or encrypted data
 			 * was requested and we didn't have it. If it was
 			 * L2-only (and thus didn't have an L1 hdr),
 			 * we realloc the header to add an L1 hdr.
 			 */
 			if (!HDR_HAS_L1HDR(hdr)) {
 				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
 				    hdr_full_cache);
 			}
 
 			if (GHOST_STATE(hdr->b_l1hdr.b_state)) {
 				ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
 				ASSERT(!HDR_HAS_RABD(hdr));
 				ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 				ASSERT0(zfs_refcount_count(
 				    &hdr->b_l1hdr.b_refcnt));
 				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
 #ifdef ZFS_DEBUG
 				ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
 #endif
 			} else if (HDR_IO_IN_PROGRESS(hdr)) {
 				/*
 				 * If this header already had an IO in progress
 				 * and we are performing another IO to fetch
 				 * encrypted data we must wait until the first
 				 * IO completes so as not to confuse
 				 * arc_read_done(). This should be very rare
 				 * and so the performance impact shouldn't
 				 * matter.
 				 */
 				arc_callback_t *acb = kmem_zalloc(
 				    sizeof (arc_callback_t), KM_SLEEP);
 				acb->acb_wait = B_TRUE;
 				mutex_init(&acb->acb_wait_lock, NULL,
 				    MUTEX_DEFAULT, NULL);
 				cv_init(&acb->acb_wait_cv, NULL, CV_DEFAULT,
 				    NULL);
 				acb->acb_zio_head =
 				    hdr->b_l1hdr.b_acb->acb_zio_head;
 				acb->acb_next = hdr->b_l1hdr.b_acb;
 				hdr->b_l1hdr.b_acb->acb_prev = acb;
 				hdr->b_l1hdr.b_acb = acb;
 				mutex_exit(hash_lock);
 				mutex_enter(&acb->acb_wait_lock);
 				while (acb->acb_wait) {
 					cv_wait(&acb->acb_wait_cv,
 					    &acb->acb_wait_lock);
 				}
 				mutex_exit(&acb->acb_wait_lock);
 				mutex_destroy(&acb->acb_wait_lock);
 				cv_destroy(&acb->acb_wait_cv);
 				kmem_free(acb, sizeof (arc_callback_t));
 				goto top;
 			}
 		}
 		if (*arc_flags & ARC_FLAG_UNCACHED) {
 			arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
 			if (!encrypted_read)
 				alloc_flags |= ARC_HDR_ALLOC_LINEAR;
 		}
 
 		/*
 		 * Take additional reference for IO_IN_PROGRESS.  It stops
 		 * arc_access() from putting this header without any buffers
 		 * and so other references but obviously nonevictable onto
 		 * the evictable list of MRU or MFU state.
 		 */
 		add_reference(hdr, hdr);
 		if (!embedded_bp)
 			arc_access(hdr, *arc_flags, B_FALSE);
 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
 		arc_hdr_alloc_abd(hdr, alloc_flags);
 		if (encrypted_read) {
 			ASSERT(HDR_HAS_RABD(hdr));
 			size = HDR_GET_PSIZE(hdr);
 			hdr_abd = hdr->b_crypt_hdr.b_rabd;
 			zio_flags |= ZIO_FLAG_RAW;
 		} else {
 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
 			size = arc_hdr_size(hdr);
 			hdr_abd = hdr->b_l1hdr.b_pabd;
 
 			if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
 				zio_flags |= ZIO_FLAG_RAW_COMPRESS;
 			}
 
 			/*
 			 * For authenticated bp's, we do not ask the ZIO layer
 			 * to authenticate them since this will cause the entire
 			 * IO to fail if the key isn't loaded. Instead, we
 			 * defer authentication until arc_buf_fill(), which will
 			 * verify the data when the key is available.
 			 */
 			if (BP_IS_AUTHENTICATED(bp))
 				zio_flags |= ZIO_FLAG_RAW_ENCRYPT;
 		}
 
 		if (BP_IS_AUTHENTICATED(bp))
 			arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
 		if (BP_GET_LEVEL(bp) > 0)
 			arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
 		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
 
 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
 		acb->acb_done = done;
 		acb->acb_private = private;
 		acb->acb_compressed = compressed_read;
 		acb->acb_encrypted = encrypted_read;
 		acb->acb_noauth = noauth_read;
 		acb->acb_zb = *zb;
 
 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
 		hdr->b_l1hdr.b_acb = acb;
 
 		if (HDR_HAS_L2HDR(hdr) &&
 		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
 			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
 			addr = hdr->b_l2hdr.b_daddr;
 			/*
 			 * Lock out L2ARC device removal.
 			 */
 			if (vdev_is_dead(vd) ||
 			    !spa_config_tryenter(spa, SCL_L2ARC, vd, RW_READER))
 				vd = NULL;
 		}
 
 		/*
 		 * We count both async reads and scrub IOs as asynchronous so
 		 * that both can be upgraded in the event of a cache hit while
 		 * the read IO is still in-flight.
 		 */
 		if (priority == ZIO_PRIORITY_ASYNC_READ ||
 		    priority == ZIO_PRIORITY_SCRUB)
 			arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
 		else
 			arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
 
 		/*
 		 * At this point, we have a level 1 cache miss or a blkptr
 		 * with embedded data.  Try again in L2ARC if possible.
 		 */
 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
 
 		/*
 		 * Skip ARC stat bump for block pointers with embedded
 		 * data. The data are read from the blkptr itself via
 		 * decode_embedded_bp_compressed().
 		 */
 		if (!embedded_bp) {
 			DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr,
 			    blkptr_t *, bp, uint64_t, lsize,
 			    zbookmark_phys_t *, zb);
 			ARCSTAT_BUMP(arcstat_misses);
 			ARCSTAT_CONDSTAT(!(*arc_flags & ARC_FLAG_PREFETCH),
 			    demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data,
 			    metadata, misses);
 			zfs_racct_read(size, 1);
 		}
 
 		/* Check if the spa even has l2 configured */
 		const boolean_t spa_has_l2 = l2arc_ndev != 0 &&
 		    spa->spa_l2cache.sav_count > 0;
 
 		if (vd != NULL && spa_has_l2 && !(l2arc_norw && devw)) {
 			/*
 			 * Read from the L2ARC if the following are true:
 			 * 1. The L2ARC vdev was previously cached.
 			 * 2. This buffer still has L2ARC metadata.
 			 * 3. This buffer isn't currently writing to the L2ARC.
 			 * 4. The L2ARC entry wasn't evicted, which may
 			 *    also have invalidated the vdev.
 			 */
 			if (HDR_HAS_L2HDR(hdr) &&
 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr)) {
 				l2arc_read_callback_t *cb;
 				abd_t *abd;
 				uint64_t asize;
 
 				DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
 				ARCSTAT_BUMP(arcstat_l2_hits);
 				hdr->b_l2hdr.b_hits++;
 
 				cb = kmem_zalloc(sizeof (l2arc_read_callback_t),
 				    KM_SLEEP);
 				cb->l2rcb_hdr = hdr;
 				cb->l2rcb_bp = *bp;
 				cb->l2rcb_zb = *zb;
 				cb->l2rcb_flags = zio_flags;
 
 				/*
 				 * When Compressed ARC is disabled, but the
 				 * L2ARC block is compressed, arc_hdr_size()
 				 * will have returned LSIZE rather than PSIZE.
 				 */
 				if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
 				    !HDR_COMPRESSION_ENABLED(hdr) &&
 				    HDR_GET_PSIZE(hdr) != 0) {
 					size = HDR_GET_PSIZE(hdr);
 				}
 
 				asize = vdev_psize_to_asize(vd, size);
 				if (asize != size) {
 					abd = abd_alloc_for_io(asize,
 					    HDR_ISTYPE_METADATA(hdr));
 					cb->l2rcb_abd = abd;
 				} else {
 					abd = hdr_abd;
 				}
 
 				ASSERT(addr >= VDEV_LABEL_START_SIZE &&
 				    addr + asize <= vd->vdev_psize -
 				    VDEV_LABEL_END_SIZE);
 
 				/*
 				 * l2arc read.  The SCL_L2ARC lock will be
 				 * released by l2arc_read_done().
 				 * Issue a null zio if the underlying buffer
 				 * was squashed to zero size by compression.
 				 */
 				ASSERT3U(arc_hdr_get_compress(hdr), !=,
 				    ZIO_COMPRESS_EMPTY);
 				rzio = zio_read_phys(pio, vd, addr,
 				    asize, abd,
 				    ZIO_CHECKSUM_OFF,
 				    l2arc_read_done, cb, priority,
 				    zio_flags | ZIO_FLAG_CANFAIL |
 				    ZIO_FLAG_DONT_PROPAGATE |
 				    ZIO_FLAG_DONT_RETRY, B_FALSE);
 				acb->acb_zio_head = rzio;
 
 				if (hash_lock != NULL)
 					mutex_exit(hash_lock);
 
 				DTRACE_PROBE2(l2arc__read, vdev_t *, vd,
 				    zio_t *, rzio);
 				ARCSTAT_INCR(arcstat_l2_read_bytes,
 				    HDR_GET_PSIZE(hdr));
 
 				if (*arc_flags & ARC_FLAG_NOWAIT) {
 					zio_nowait(rzio);
 					goto out;
 				}
 
 				ASSERT(*arc_flags & ARC_FLAG_WAIT);
 				if (zio_wait(rzio) == 0)
 					goto out;
 
 				/* l2arc read error; goto zio_read() */
 				if (hash_lock != NULL)
 					mutex_enter(hash_lock);
 			} else {
 				DTRACE_PROBE1(l2arc__miss,
 				    arc_buf_hdr_t *, hdr);
 				ARCSTAT_BUMP(arcstat_l2_misses);
 				if (HDR_L2_WRITING(hdr))
 					ARCSTAT_BUMP(arcstat_l2_rw_clash);
 				spa_config_exit(spa, SCL_L2ARC, vd);
 			}
 		} else {
 			if (vd != NULL)
 				spa_config_exit(spa, SCL_L2ARC, vd);
 
 			/*
 			 * Only a spa with l2 should contribute to l2
 			 * miss stats.  (Including the case of having a
 			 * faulted cache device - that's also a miss.)
 			 */
 			if (spa_has_l2) {
 				/*
 				 * Skip ARC stat bump for block pointers with
 				 * embedded data. The data are read from the
 				 * blkptr itself via
 				 * decode_embedded_bp_compressed().
 				 */
 				if (!embedded_bp) {
 					DTRACE_PROBE1(l2arc__miss,
 					    arc_buf_hdr_t *, hdr);
 					ARCSTAT_BUMP(arcstat_l2_misses);
 				}
 			}
 		}
 
 		rzio = zio_read(pio, spa, bp, hdr_abd, size,
 		    arc_read_done, hdr, priority, zio_flags, zb);
 		acb->acb_zio_head = rzio;
 
 		if (hash_lock != NULL)
 			mutex_exit(hash_lock);
 
 		if (*arc_flags & ARC_FLAG_WAIT) {
 			rc = zio_wait(rzio);
 			goto out;
 		}
 
 		ASSERT(*arc_flags & ARC_FLAG_NOWAIT);
 		zio_nowait(rzio);
 	}
 
 out:
 	/* embedded bps don't actually go to disk */
 	if (!embedded_bp)
 		spa_read_history_add(spa, zb, *arc_flags);
 	spl_fstrans_unmark(cookie);
 	return (rc);
 
 done:
 	if (done)
 		done(NULL, zb, bp, buf, private);
 	if (pio && rc != 0) {
 		zio_t *zio = zio_null(pio, spa, NULL, NULL, NULL, zio_flags);
 		zio->io_error = rc;
 		zio_nowait(zio);
 	}
 	goto out;
 }
 
 arc_prune_t *
 arc_add_prune_callback(arc_prune_func_t *func, void *private)
 {
 	arc_prune_t *p;
 
 	p = kmem_alloc(sizeof (*p), KM_SLEEP);
 	p->p_pfunc = func;
 	p->p_private = private;
 	list_link_init(&p->p_node);
 	zfs_refcount_create(&p->p_refcnt);
 
 	mutex_enter(&arc_prune_mtx);
 	zfs_refcount_add(&p->p_refcnt, &arc_prune_list);
 	list_insert_head(&arc_prune_list, p);
 	mutex_exit(&arc_prune_mtx);
 
 	return (p);
 }
 
 void
 arc_remove_prune_callback(arc_prune_t *p)
 {
 	boolean_t wait = B_FALSE;
 	mutex_enter(&arc_prune_mtx);
 	list_remove(&arc_prune_list, p);
 	if (zfs_refcount_remove(&p->p_refcnt, &arc_prune_list) > 0)
 		wait = B_TRUE;
 	mutex_exit(&arc_prune_mtx);
 
 	/* wait for arc_prune_task to finish */
 	if (wait)
 		taskq_wait_outstanding(arc_prune_taskq, 0);
 	ASSERT0(zfs_refcount_count(&p->p_refcnt));
 	zfs_refcount_destroy(&p->p_refcnt);
 	kmem_free(p, sizeof (*p));
 }
 
 /*
  * Helper function for arc_prune_async() it is responsible for safely
  * handling the execution of a registered arc_prune_func_t.
  */
 static void
 arc_prune_task(void *ptr)
 {
 	arc_prune_t *ap = (arc_prune_t *)ptr;
 	arc_prune_func_t *func = ap->p_pfunc;
 
 	if (func != NULL)
 		func(ap->p_adjust, ap->p_private);
 
 	zfs_refcount_remove(&ap->p_refcnt, func);
 }
 
 /*
  * Notify registered consumers they must drop holds on a portion of the ARC
  * buffers they reference.  This provides a mechanism to ensure the ARC can
  * honor the metadata limit and reclaim otherwise pinned ARC buffers.
  *
  * This operation is performed asynchronously so it may be safely called
  * in the context of the arc_reclaim_thread().  A reference is taken here
  * for each registered arc_prune_t and the arc_prune_task() is responsible
  * for releasing it once the registered arc_prune_func_t has completed.
  */
 static void
 arc_prune_async(uint64_t adjust)
 {
 	arc_prune_t *ap;
 
 	mutex_enter(&arc_prune_mtx);
 	for (ap = list_head(&arc_prune_list); ap != NULL;
 	    ap = list_next(&arc_prune_list, ap)) {
 
 		if (zfs_refcount_count(&ap->p_refcnt) >= 2)
 			continue;
 
 		zfs_refcount_add(&ap->p_refcnt, ap->p_pfunc);
 		ap->p_adjust = adjust;
 		if (taskq_dispatch(arc_prune_taskq, arc_prune_task,
 		    ap, TQ_SLEEP) == TASKQID_INVALID) {
 			zfs_refcount_remove(&ap->p_refcnt, ap->p_pfunc);
 			continue;
 		}
 		ARCSTAT_BUMP(arcstat_prune);
 	}
 	mutex_exit(&arc_prune_mtx);
 }
 
 /*
  * Notify the arc that a block was freed, and thus will never be used again.
  */
 void
 arc_freed(spa_t *spa, const blkptr_t *bp)
 {
 	arc_buf_hdr_t *hdr;
 	kmutex_t *hash_lock;
 	uint64_t guid = spa_load_guid(spa);
 
 	ASSERT(!BP_IS_EMBEDDED(bp));
 
 	hdr = buf_hash_find(guid, bp, &hash_lock);
 	if (hdr == NULL)
 		return;
 
 	/*
 	 * We might be trying to free a block that is still doing I/O
 	 * (i.e. prefetch) or has some other reference (i.e. a dedup-ed,
 	 * dmu_sync-ed block). A block may also have a reference if it is
 	 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would
 	 * have written the new block to its final resting place on disk but
 	 * without the dedup flag set. This would have left the hdr in the MRU
 	 * state and discoverable. When the txg finally syncs it detects that
 	 * the block was overridden in open context and issues an override I/O.
 	 * Since this is a dedup block, the override I/O will determine if the
 	 * block is already in the DDT. If so, then it will replace the io_bp
 	 * with the bp from the DDT and allow the I/O to finish. When the I/O
 	 * reaches the done callback, dbuf_write_override_done, it will
 	 * check to see if the io_bp and io_bp_override are identical.
 	 * If they are not, then it indicates that the bp was replaced with
 	 * the bp in the DDT and the override bp is freed. This allows
 	 * us to arrive here with a reference on a block that is being
 	 * freed. So if we have an I/O in progress, or a reference to
 	 * this hdr, then we don't destroy the hdr.
 	 */
 	if (!HDR_HAS_L1HDR(hdr) ||
 	    zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
 		arc_change_state(arc_anon, hdr);
 		arc_hdr_destroy(hdr);
 		mutex_exit(hash_lock);
 	} else {
 		mutex_exit(hash_lock);
 	}
 
 }
 
 /*
  * Release this buffer from the cache, making it an anonymous buffer.  This
  * must be done after a read and prior to modifying the buffer contents.
  * If the buffer has more than one reference, we must make
  * a new hdr for the buffer.
  */
 void
 arc_release(arc_buf_t *buf, const void *tag)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	/*
 	 * It would be nice to assert that if its DMU metadata (level >
 	 * 0 || it's the dnode file), then it must be syncing context.
 	 * But we don't know that information at this level.
 	 */
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 
 	/*
 	 * We don't grab the hash lock prior to this check, because if
 	 * the buffer's header is in the arc_anon state, it won't be
 	 * linked into the hash table.
 	 */
 	if (hdr->b_l1hdr.b_state == arc_anon) {
 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 		ASSERT(!HDR_IN_HASH_TABLE(hdr));
 		ASSERT(!HDR_HAS_L2HDR(hdr));
 
 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
 		ASSERT(ARC_BUF_LAST(buf));
 		ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 
 		hdr->b_l1hdr.b_arc_access = 0;
 
 		/*
 		 * If the buf is being overridden then it may already
 		 * have a hdr that is not empty.
 		 */
 		buf_discard_identity(hdr);
 		arc_buf_thaw(buf);
 
 		return;
 	}
 
 	kmutex_t *hash_lock = HDR_LOCK(hdr);
 	mutex_enter(hash_lock);
 
 	/*
 	 * This assignment is only valid as long as the hash_lock is
 	 * held, we must be careful not to reference state or the
 	 * b_state field after dropping the lock.
 	 */
 	arc_state_t *state = hdr->b_l1hdr.b_state;
 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
 	ASSERT3P(state, !=, arc_anon);
 
 	/* this buffer is not on any list */
 	ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
 
 	if (HDR_HAS_L2HDR(hdr)) {
 		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
 
 		/*
 		 * We have to recheck this conditional again now that
 		 * we're holding the l2ad_mtx to prevent a race with
 		 * another thread which might be concurrently calling
 		 * l2arc_evict(). In that case, l2arc_evict() might have
 		 * destroyed the header's L2 portion as we were waiting
 		 * to acquire the l2ad_mtx.
 		 */
 		if (HDR_HAS_L2HDR(hdr))
 			arc_hdr_l2hdr_destroy(hdr);
 
 		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
 	}
 
 	/*
 	 * Do we have more than one buf?
 	 */
 	if (hdr->b_l1hdr.b_buf != buf || !ARC_BUF_LAST(buf)) {
 		arc_buf_hdr_t *nhdr;
 		uint64_t spa = hdr->b_spa;
 		uint64_t psize = HDR_GET_PSIZE(hdr);
 		uint64_t lsize = HDR_GET_LSIZE(hdr);
 		boolean_t protected = HDR_PROTECTED(hdr);
 		enum zio_compress compress = arc_hdr_get_compress(hdr);
 		arc_buf_contents_t type = arc_buf_type(hdr);
 		VERIFY3U(hdr->b_type, ==, type);
 
 		ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
 		VERIFY3S(remove_reference(hdr, tag), >, 0);
 
 		if (ARC_BUF_SHARED(buf) && !ARC_BUF_COMPRESSED(buf)) {
 			ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
 			ASSERT(ARC_BUF_LAST(buf));
 		}
 
 		/*
 		 * Pull the data off of this hdr and attach it to
 		 * a new anonymous hdr. Also find the last buffer
 		 * in the hdr's buffer list.
 		 */
 		arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
 		ASSERT3P(lastbuf, !=, NULL);
 
 		/*
 		 * If the current arc_buf_t and the hdr are sharing their data
 		 * buffer, then we must stop sharing that block.
 		 */
 		if (ARC_BUF_SHARED(buf)) {
 			ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
 			ASSERT(!arc_buf_is_shared(lastbuf));
 
 			/*
 			 * First, sever the block sharing relationship between
 			 * buf and the arc_buf_hdr_t.
 			 */
 			arc_unshare_buf(hdr, buf);
 
 			/*
 			 * Now we need to recreate the hdr's b_pabd. Since we
 			 * have lastbuf handy, we try to share with it, but if
 			 * we can't then we allocate a new b_pabd and copy the
 			 * data from buf into it.
 			 */
 			if (arc_can_share(hdr, lastbuf)) {
 				arc_share_buf(hdr, lastbuf);
 			} else {
 				arc_hdr_alloc_abd(hdr, 0);
 				abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
 				    buf->b_data, psize);
 			}
 			VERIFY3P(lastbuf->b_data, !=, NULL);
 		} else if (HDR_SHARED_DATA(hdr)) {
 			/*
 			 * Uncompressed shared buffers are always at the end
 			 * of the list. Compressed buffers don't have the
 			 * same requirements. This makes it hard to
 			 * simply assert that the lastbuf is shared so
 			 * we rely on the hdr's compression flags to determine
 			 * if we have a compressed, shared buffer.
 			 */
 			ASSERT(arc_buf_is_shared(lastbuf) ||
 			    arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
 			ASSERT(!arc_buf_is_shared(buf));
 		}
 
 		ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
 		ASSERT3P(state, !=, arc_l2c_only);
 
 		(void) zfs_refcount_remove_many(&state->arcs_size[type],
 		    arc_buf_size(buf), buf);
 
 		if (zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
 			ASSERT3P(state, !=, arc_l2c_only);
 			(void) zfs_refcount_remove_many(
 			    &state->arcs_esize[type],
 			    arc_buf_size(buf), buf);
 		}
 
 		arc_cksum_verify(buf);
 		arc_buf_unwatch(buf);
 
 		/* if this is the last uncompressed buf free the checksum */
 		if (!arc_hdr_has_uncompressed_buf(hdr))
 			arc_cksum_free(hdr);
 
 		mutex_exit(hash_lock);
 
 		nhdr = arc_hdr_alloc(spa, psize, lsize, protected,
 		    compress, hdr->b_complevel, type);
 		ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
 		ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt));
 		VERIFY3U(nhdr->b_type, ==, type);
 		ASSERT(!HDR_SHARED_DATA(nhdr));
 
 		nhdr->b_l1hdr.b_buf = buf;
 		(void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
 		buf->b_hdr = nhdr;
 
 		(void) zfs_refcount_add_many(&arc_anon->arcs_size[type],
 		    arc_buf_size(buf), buf);
 	} else {
 		ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
 		/* protected by hash lock, or hdr is on arc_anon */
 		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 		hdr->b_l1hdr.b_mru_hits = 0;
 		hdr->b_l1hdr.b_mru_ghost_hits = 0;
 		hdr->b_l1hdr.b_mfu_hits = 0;
 		hdr->b_l1hdr.b_mfu_ghost_hits = 0;
 		arc_change_state(arc_anon, hdr);
 		hdr->b_l1hdr.b_arc_access = 0;
 
 		mutex_exit(hash_lock);
 		buf_discard_identity(hdr);
 		arc_buf_thaw(buf);
 	}
 }
 
 int
 arc_released(arc_buf_t *buf)
 {
 	return (buf->b_data != NULL &&
 	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
 }
 
 #ifdef ZFS_DEBUG
 int
 arc_referenced(arc_buf_t *buf)
 {
 	return (zfs_refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
 }
 #endif
 
 static void
 arc_write_ready(zio_t *zio)
 {
 	arc_write_callback_t *callback = zio->io_private;
 	arc_buf_t *buf = callback->awcb_buf;
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	blkptr_t *bp = zio->io_bp;
 	uint64_t psize = BP_IS_HOLE(bp) ? 0 : BP_GET_PSIZE(bp);
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 
 	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(!zfs_refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
 
 	/*
 	 * If we're reexecuting this zio because the pool suspended, then
 	 * cleanup any state that was previously set the first time the
 	 * callback was invoked.
 	 */
 	if (zio->io_flags & ZIO_FLAG_REEXECUTED) {
 		arc_cksum_free(hdr);
 		arc_buf_unwatch(buf);
 		if (hdr->b_l1hdr.b_pabd != NULL) {
 			if (ARC_BUF_SHARED(buf)) {
 				arc_unshare_buf(hdr, buf);
 			} else {
 				ASSERT(!arc_buf_is_shared(buf));
 				arc_hdr_free_abd(hdr, B_FALSE);
 			}
 		}
 
 		if (HDR_HAS_RABD(hdr))
 			arc_hdr_free_abd(hdr, B_TRUE);
 	}
 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
 	ASSERT(!HDR_HAS_RABD(hdr));
 	ASSERT(!HDR_SHARED_DATA(hdr));
 	ASSERT(!arc_buf_is_shared(buf));
 
 	callback->awcb_ready(zio, buf, callback->awcb_private);
 
 	if (HDR_IO_IN_PROGRESS(hdr)) {
 		ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);
 	} else {
 		arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
 		add_reference(hdr, hdr); /* For IO_IN_PROGRESS. */
 	}
 
 	if (BP_IS_PROTECTED(bp)) {
 		/* ZIL blocks are written through zio_rewrite */
 		ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
 
 		if (BP_SHOULD_BYTESWAP(bp)) {
 			if (BP_GET_LEVEL(bp) > 0) {
 				hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
 			} else {
 				hdr->b_l1hdr.b_byteswap =
 				    DMU_OT_BYTESWAP(BP_GET_TYPE(bp));
 			}
 		} else {
 			hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
 		}
 
 		arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
 		hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
 		hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
 		zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
 		    hdr->b_crypt_hdr.b_iv);
 		zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac);
 	} else {
 		arc_hdr_clear_flags(hdr, ARC_FLAG_PROTECTED);
 	}
 
 	/*
 	 * If this block was written for raw encryption but the zio layer
 	 * ended up only authenticating it, adjust the buffer flags now.
 	 */
 	if (BP_IS_AUTHENTICATED(bp) && ARC_BUF_ENCRYPTED(buf)) {
 		arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
 		buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
 		if (BP_GET_COMPRESS(bp) == ZIO_COMPRESS_OFF)
 			buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
 	} else if (BP_IS_HOLE(bp) && ARC_BUF_ENCRYPTED(buf)) {
 		buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
 		buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
 	}
 
 	/* this must be done after the buffer flags are adjusted */
 	arc_cksum_compute(buf);
 
 	enum zio_compress compress;
 	if (BP_IS_HOLE(bp) || BP_IS_EMBEDDED(bp)) {
 		compress = ZIO_COMPRESS_OFF;
 	} else {
 		ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
 		compress = BP_GET_COMPRESS(bp);
 	}
 	HDR_SET_PSIZE(hdr, psize);
 	arc_hdr_set_compress(hdr, compress);
 	hdr->b_complevel = zio->io_prop.zp_complevel;
 
 	if (zio->io_error != 0 || psize == 0)
 		goto out;
 
 	/*
 	 * Fill the hdr with data. If the buffer is encrypted we have no choice
 	 * but to copy the data into b_radb. If the hdr is compressed, the data
 	 * we want is available from the zio, otherwise we can take it from
 	 * the buf.
 	 *
 	 * We might be able to share the buf's data with the hdr here. However,
 	 * doing so would cause the ARC to be full of linear ABDs if we write a
 	 * lot of shareable data. As a compromise, we check whether scattered
 	 * ABDs are allowed, and assume that if they are then the user wants
 	 * the ARC to be primarily filled with them regardless of the data being
 	 * written. Therefore, if they're allowed then we allocate one and copy
 	 * the data into it; otherwise, we share the data directly if we can.
 	 */
 	if (ARC_BUF_ENCRYPTED(buf)) {
 		ASSERT3U(psize, >, 0);
 		ASSERT(ARC_BUF_COMPRESSED(buf));
 		arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |
 		    ARC_HDR_USE_RESERVE);
 		abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
 	} else if (!(HDR_UNCACHED(hdr) ||
 	    abd_size_alloc_linear(arc_buf_size(buf))) ||
 	    !arc_can_share(hdr, buf)) {
 		/*
 		 * Ideally, we would always copy the io_abd into b_pabd, but the
 		 * user may have disabled compressed ARC, thus we must check the
 		 * hdr's compression setting rather than the io_bp's.
 		 */
 		if (BP_IS_ENCRYPTED(bp)) {
 			ASSERT3U(psize, >, 0);
 			arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |
 			    ARC_HDR_USE_RESERVE);
 			abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
 		} else if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
 		    !ARC_BUF_COMPRESSED(buf)) {
 			ASSERT3U(psize, >, 0);
 			arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
 			abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
 		} else {
 			ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));
 			arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
 			abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
 			    arc_buf_size(buf));
 		}
 	} else {
 		ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));
 		ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));
 		ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
 		ASSERT(ARC_BUF_LAST(buf));
 
 		arc_share_buf(hdr, buf);
 	}
 
 out:
 	arc_hdr_verify(hdr, bp);
 	spl_fstrans_unmark(cookie);
 }
 
 static void
 arc_write_children_ready(zio_t *zio)
 {
 	arc_write_callback_t *callback = zio->io_private;
 	arc_buf_t *buf = callback->awcb_buf;
 
 	callback->awcb_children_ready(zio, buf, callback->awcb_private);
 }
 
 static void
 arc_write_done(zio_t *zio)
 {
 	arc_write_callback_t *callback = zio->io_private;
 	arc_buf_t *buf = callback->awcb_buf;
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
 
 	if (zio->io_error == 0) {
 		arc_hdr_verify(hdr, zio->io_bp);
 
 		if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
 			buf_discard_identity(hdr);
 		} else {
 			hdr->b_dva = *BP_IDENTITY(zio->io_bp);
 			hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
 		}
 	} else {
 		ASSERT(HDR_EMPTY(hdr));
 	}
 
 	/*
 	 * If the block to be written was all-zero or compressed enough to be
 	 * embedded in the BP, no write was performed so there will be no
 	 * dva/birth/checksum.  The buffer must therefore remain anonymous
 	 * (and uncached).
 	 */
 	if (!HDR_EMPTY(hdr)) {
 		arc_buf_hdr_t *exists;
 		kmutex_t *hash_lock;
 
 		ASSERT3U(zio->io_error, ==, 0);
 
 		arc_cksum_verify(buf);
 
 		exists = buf_hash_insert(hdr, &hash_lock);
 		if (exists != NULL) {
 			/*
 			 * This can only happen if we overwrite for
 			 * sync-to-convergence, because we remove
 			 * buffers from the hash table when we arc_free().
 			 */
 			if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
 					panic("bad overwrite, hdr=%p exists=%p",
 					    (void *)hdr, (void *)exists);
 				ASSERT(zfs_refcount_is_zero(
 				    &exists->b_l1hdr.b_refcnt));
 				arc_change_state(arc_anon, exists);
 				arc_hdr_destroy(exists);
 				mutex_exit(hash_lock);
 				exists = buf_hash_insert(hdr, &hash_lock);
 				ASSERT3P(exists, ==, NULL);
 			} else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
 				/* nopwrite */
 				ASSERT(zio->io_prop.zp_nopwrite);
 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
 					panic("bad nopwrite, hdr=%p exists=%p",
 					    (void *)hdr, (void *)exists);
 			} else {
 				/* Dedup */
 				ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
 				ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
 				ASSERT(hdr->b_l1hdr.b_state == arc_anon);
 				ASSERT(BP_GET_DEDUP(zio->io_bp));
 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
 			}
 		}
 		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
 		VERIFY3S(remove_reference(hdr, hdr), >, 0);
 		/* if it's not anon, we are doing a scrub */
 		if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
 			arc_access(hdr, 0, B_FALSE);
 		mutex_exit(hash_lock);
 	} else {
 		arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
 		VERIFY3S(remove_reference(hdr, hdr), >, 0);
 	}
 
 	callback->awcb_done(zio, buf, callback->awcb_private);
 
 	abd_free(zio->io_abd);
 	kmem_free(callback, sizeof (arc_write_callback_t));
 }
 
 zio_t *
 arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
     blkptr_t *bp, arc_buf_t *buf, boolean_t uncached, boolean_t l2arc,
     const zio_prop_t *zp, arc_write_done_func_t *ready,
     arc_write_done_func_t *children_ready, arc_write_done_func_t *done,
     void *private, zio_priority_t priority, int zio_flags,
     const zbookmark_phys_t *zb)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 	arc_write_callback_t *callback;
 	zio_t *zio;
 	zio_prop_t localprop = *zp;
 
 	ASSERT3P(ready, !=, NULL);
 	ASSERT3P(done, !=, NULL);
 	ASSERT(!HDR_IO_ERROR(hdr));
 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 	ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
 	ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
 	if (uncached)
 		arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
 	else if (l2arc)
 		arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
 
 	if (ARC_BUF_ENCRYPTED(buf)) {
 		ASSERT(ARC_BUF_COMPRESSED(buf));
 		localprop.zp_encrypt = B_TRUE;
 		localprop.zp_compress = HDR_GET_COMPRESS(hdr);
 		localprop.zp_complevel = hdr->b_complevel;
 		localprop.zp_byteorder =
 		    (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
 		    ZFS_HOST_BYTEORDER : !ZFS_HOST_BYTEORDER;
 		memcpy(localprop.zp_salt, hdr->b_crypt_hdr.b_salt,
 		    ZIO_DATA_SALT_LEN);
 		memcpy(localprop.zp_iv, hdr->b_crypt_hdr.b_iv,
 		    ZIO_DATA_IV_LEN);
 		memcpy(localprop.zp_mac, hdr->b_crypt_hdr.b_mac,
 		    ZIO_DATA_MAC_LEN);
 		if (DMU_OT_IS_ENCRYPTED(localprop.zp_type)) {
 			localprop.zp_nopwrite = B_FALSE;
 			localprop.zp_copies =
 			    MIN(localprop.zp_copies, SPA_DVAS_PER_BP - 1);
 		}
 		zio_flags |= ZIO_FLAG_RAW;
 	} else if (ARC_BUF_COMPRESSED(buf)) {
 		ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));
 		localprop.zp_compress = HDR_GET_COMPRESS(hdr);
 		localprop.zp_complevel = hdr->b_complevel;
 		zio_flags |= ZIO_FLAG_RAW_COMPRESS;
 	}
 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
 	callback->awcb_ready = ready;
 	callback->awcb_children_ready = children_ready;
 	callback->awcb_done = done;
 	callback->awcb_private = private;
 	callback->awcb_buf = buf;
 
 	/*
 	 * The hdr's b_pabd is now stale, free it now. A new data block
 	 * will be allocated when the zio pipeline calls arc_write_ready().
 	 */
 	if (hdr->b_l1hdr.b_pabd != NULL) {
 		/*
 		 * If the buf is currently sharing the data block with
 		 * the hdr then we need to break that relationship here.
 		 * The hdr will remain with a NULL data pointer and the
 		 * buf will take sole ownership of the block.
 		 */
 		if (ARC_BUF_SHARED(buf)) {
 			arc_unshare_buf(hdr, buf);
 		} else {
 			ASSERT(!arc_buf_is_shared(buf));
 			arc_hdr_free_abd(hdr, B_FALSE);
 		}
 		VERIFY3P(buf->b_data, !=, NULL);
 	}
 
 	if (HDR_HAS_RABD(hdr))
 		arc_hdr_free_abd(hdr, B_TRUE);
 
 	if (!(zio_flags & ZIO_FLAG_RAW))
 		arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);
 
 	ASSERT(!arc_buf_is_shared(buf));
 	ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
 
 	zio = zio_write(pio, spa, txg, bp,
 	    abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),
 	    HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready,
 	    (children_ready != NULL) ? arc_write_children_ready : NULL,
 	    arc_write_done, callback, priority, zio_flags, zb);
 
 	return (zio);
 }
 
 void
 arc_tempreserve_clear(uint64_t reserve)
 {
 	atomic_add_64(&arc_tempreserve, -reserve);
 	ASSERT((int64_t)arc_tempreserve >= 0);
 }
 
 int
 arc_tempreserve_space(spa_t *spa, uint64_t reserve, uint64_t txg)
 {
 	int error;
 	uint64_t anon_size;
 
 	if (!arc_no_grow &&
 	    reserve > arc_c/4 &&
 	    reserve * 4 > (2ULL << SPA_MAXBLOCKSHIFT))
 		arc_c = MIN(arc_c_max, reserve * 4);
 
 	/*
 	 * Throttle when the calculated memory footprint for the TXG
 	 * exceeds the target ARC size.
 	 */
 	if (reserve > arc_c) {
 		DMU_TX_STAT_BUMP(dmu_tx_memory_reserve);
 		return (SET_ERROR(ERESTART));
 	}
 
 	/*
 	 * Don't count loaned bufs as in flight dirty data to prevent long
 	 * network delays from blocking transactions that are ready to be
 	 * assigned to a txg.
 	 */
 
 	/* assert that it has not wrapped around */
 	ASSERT3S(atomic_add_64_nv(&arc_loaned_bytes, 0), >=, 0);
 
 	anon_size = MAX((int64_t)
 	    (zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]) +
 	    zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]) -
 	    arc_loaned_bytes), 0);
 
 	/*
 	 * Writes will, almost always, require additional memory allocations
 	 * in order to compress/encrypt/etc the data.  We therefore need to
 	 * make sure that there is sufficient available memory for this.
 	 */
 	error = arc_memory_throttle(spa, reserve, txg);
 	if (error != 0)
 		return (error);
 
 	/*
 	 * Throttle writes when the amount of dirty data in the cache
 	 * gets too large.  We try to keep the cache less than half full
 	 * of dirty blocks so that our sync times don't grow too large.
 	 *
 	 * In the case of one pool being built on another pool, we want
 	 * to make sure we don't end up throttling the lower (backing)
 	 * pool when the upper pool is the majority contributor to dirty
 	 * data. To insure we make forward progress during throttling, we
 	 * also check the current pool's net dirty data and only throttle
 	 * if it exceeds zfs_arc_pool_dirty_percent of the anonymous dirty
 	 * data in the cache.
 	 *
 	 * Note: if two requests come in concurrently, we might let them
 	 * both succeed, when one of them should fail.  Not a huge deal.
 	 */
 	uint64_t total_dirty = reserve + arc_tempreserve + anon_size;
 	uint64_t spa_dirty_anon = spa_dirty_data(spa);
 	uint64_t rarc_c = arc_warm ? arc_c : arc_c_max;
 	if (total_dirty > rarc_c * zfs_arc_dirty_limit_percent / 100 &&
 	    anon_size > rarc_c * zfs_arc_anon_limit_percent / 100 &&
 	    spa_dirty_anon > anon_size * zfs_arc_pool_dirty_percent / 100) {
 #ifdef ZFS_DEBUG
 		uint64_t meta_esize = zfs_refcount_count(
 		    &arc_anon->arcs_esize[ARC_BUFC_METADATA]);
 		uint64_t data_esize =
 		    zfs_refcount_count(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
 		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
 		    "anon_data=%lluK tempreserve=%lluK rarc_c=%lluK\n",
 		    (u_longlong_t)arc_tempreserve >> 10,
 		    (u_longlong_t)meta_esize >> 10,
 		    (u_longlong_t)data_esize >> 10,
 		    (u_longlong_t)reserve >> 10,
 		    (u_longlong_t)rarc_c >> 10);
 #endif
 		DMU_TX_STAT_BUMP(dmu_tx_dirty_throttle);
 		return (SET_ERROR(ERESTART));
 	}
 	atomic_add_64(&arc_tempreserve, reserve);
 	return (0);
 }
 
 static void
 arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
     kstat_named_t *data, kstat_named_t *metadata,
     kstat_named_t *evict_data, kstat_named_t *evict_metadata)
 {
 	data->value.ui64 =
 	    zfs_refcount_count(&state->arcs_size[ARC_BUFC_DATA]);
 	metadata->value.ui64 =
 	    zfs_refcount_count(&state->arcs_size[ARC_BUFC_METADATA]);
 	size->value.ui64 = data->value.ui64 + metadata->value.ui64;
 	evict_data->value.ui64 =
 	    zfs_refcount_count(&state->arcs_esize[ARC_BUFC_DATA]);
 	evict_metadata->value.ui64 =
 	    zfs_refcount_count(&state->arcs_esize[ARC_BUFC_METADATA]);
 }
 
 static int
 arc_kstat_update(kstat_t *ksp, int rw)
 {
 	arc_stats_t *as = ksp->ks_data;
 
 	if (rw == KSTAT_WRITE)
 		return (SET_ERROR(EACCES));
 
 	as->arcstat_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_hits);
 	as->arcstat_iohits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_iohits);
 	as->arcstat_misses.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_misses);
 	as->arcstat_demand_data_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_demand_data_hits);
 	as->arcstat_demand_data_iohits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_demand_data_iohits);
 	as->arcstat_demand_data_misses.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_demand_data_misses);
 	as->arcstat_demand_metadata_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_demand_metadata_hits);
 	as->arcstat_demand_metadata_iohits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_demand_metadata_iohits);
 	as->arcstat_demand_metadata_misses.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_demand_metadata_misses);
 	as->arcstat_prefetch_data_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_prefetch_data_hits);
 	as->arcstat_prefetch_data_iohits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_prefetch_data_iohits);
 	as->arcstat_prefetch_data_misses.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_prefetch_data_misses);
 	as->arcstat_prefetch_metadata_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_hits);
 	as->arcstat_prefetch_metadata_iohits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_iohits);
 	as->arcstat_prefetch_metadata_misses.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_prefetch_metadata_misses);
 	as->arcstat_mru_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_mru_hits);
 	as->arcstat_mru_ghost_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_mru_ghost_hits);
 	as->arcstat_mfu_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_mfu_hits);
 	as->arcstat_mfu_ghost_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_mfu_ghost_hits);
 	as->arcstat_uncached_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_uncached_hits);
 	as->arcstat_deleted.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_deleted);
 	as->arcstat_mutex_miss.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_mutex_miss);
 	as->arcstat_access_skip.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_access_skip);
 	as->arcstat_evict_skip.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_evict_skip);
 	as->arcstat_evict_not_enough.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_evict_not_enough);
 	as->arcstat_evict_l2_cached.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_evict_l2_cached);
 	as->arcstat_evict_l2_eligible.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible);
 	as->arcstat_evict_l2_eligible_mfu.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mfu);
 	as->arcstat_evict_l2_eligible_mru.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_evict_l2_eligible_mru);
 	as->arcstat_evict_l2_ineligible.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_evict_l2_ineligible);
 	as->arcstat_evict_l2_skip.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_evict_l2_skip);
 	as->arcstat_hash_collisions.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_hash_collisions);
 	as->arcstat_hash_chains.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_hash_chains);
 	as->arcstat_size.value.ui64 =
 	    aggsum_value(&arc_sums.arcstat_size);
 	as->arcstat_compressed_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_compressed_size);
 	as->arcstat_uncompressed_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_uncompressed_size);
 	as->arcstat_overhead_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_overhead_size);
 	as->arcstat_hdr_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_hdr_size);
 	as->arcstat_data_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_data_size);
 	as->arcstat_metadata_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_metadata_size);
 	as->arcstat_dbuf_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_dbuf_size);
 #if defined(COMPAT_FREEBSD11)
 	as->arcstat_other_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_bonus_size) +
 	    wmsum_value(&arc_sums.arcstat_dnode_size) +
 	    wmsum_value(&arc_sums.arcstat_dbuf_size);
 #endif
 
 	arc_kstat_update_state(arc_anon,
 	    &as->arcstat_anon_size,
 	    &as->arcstat_anon_data,
 	    &as->arcstat_anon_metadata,
 	    &as->arcstat_anon_evictable_data,
 	    &as->arcstat_anon_evictable_metadata);
 	arc_kstat_update_state(arc_mru,
 	    &as->arcstat_mru_size,
 	    &as->arcstat_mru_data,
 	    &as->arcstat_mru_metadata,
 	    &as->arcstat_mru_evictable_data,
 	    &as->arcstat_mru_evictable_metadata);
 	arc_kstat_update_state(arc_mru_ghost,
 	    &as->arcstat_mru_ghost_size,
 	    &as->arcstat_mru_ghost_data,
 	    &as->arcstat_mru_ghost_metadata,
 	    &as->arcstat_mru_ghost_evictable_data,
 	    &as->arcstat_mru_ghost_evictable_metadata);
 	arc_kstat_update_state(arc_mfu,
 	    &as->arcstat_mfu_size,
 	    &as->arcstat_mfu_data,
 	    &as->arcstat_mfu_metadata,
 	    &as->arcstat_mfu_evictable_data,
 	    &as->arcstat_mfu_evictable_metadata);
 	arc_kstat_update_state(arc_mfu_ghost,
 	    &as->arcstat_mfu_ghost_size,
 	    &as->arcstat_mfu_ghost_data,
 	    &as->arcstat_mfu_ghost_metadata,
 	    &as->arcstat_mfu_ghost_evictable_data,
 	    &as->arcstat_mfu_ghost_evictable_metadata);
 	arc_kstat_update_state(arc_uncached,
 	    &as->arcstat_uncached_size,
 	    &as->arcstat_uncached_data,
 	    &as->arcstat_uncached_metadata,
 	    &as->arcstat_uncached_evictable_data,
 	    &as->arcstat_uncached_evictable_metadata);
 
 	as->arcstat_dnode_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_dnode_size);
 	as->arcstat_bonus_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_bonus_size);
 	as->arcstat_l2_hits.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_hits);
 	as->arcstat_l2_misses.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_misses);
 	as->arcstat_l2_prefetch_asize.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_prefetch_asize);
 	as->arcstat_l2_mru_asize.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_mru_asize);
 	as->arcstat_l2_mfu_asize.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_mfu_asize);
 	as->arcstat_l2_bufc_data_asize.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_bufc_data_asize);
 	as->arcstat_l2_bufc_metadata_asize.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_bufc_metadata_asize);
 	as->arcstat_l2_feeds.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_feeds);
 	as->arcstat_l2_rw_clash.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rw_clash);
 	as->arcstat_l2_read_bytes.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_read_bytes);
 	as->arcstat_l2_write_bytes.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_write_bytes);
 	as->arcstat_l2_writes_sent.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_writes_sent);
 	as->arcstat_l2_writes_done.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_writes_done);
 	as->arcstat_l2_writes_error.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_writes_error);
 	as->arcstat_l2_writes_lock_retry.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_writes_lock_retry);
 	as->arcstat_l2_evict_lock_retry.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_evict_lock_retry);
 	as->arcstat_l2_evict_reading.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_evict_reading);
 	as->arcstat_l2_evict_l1cached.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_evict_l1cached);
 	as->arcstat_l2_free_on_write.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_free_on_write);
 	as->arcstat_l2_abort_lowmem.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_abort_lowmem);
 	as->arcstat_l2_cksum_bad.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_cksum_bad);
 	as->arcstat_l2_io_error.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_io_error);
 	as->arcstat_l2_lsize.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_lsize);
 	as->arcstat_l2_psize.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_psize);
 	as->arcstat_l2_hdr_size.value.ui64 =
 	    aggsum_value(&arc_sums.arcstat_l2_hdr_size);
 	as->arcstat_l2_log_blk_writes.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_log_blk_writes);
 	as->arcstat_l2_log_blk_asize.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_log_blk_asize);
 	as->arcstat_l2_log_blk_count.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_log_blk_count);
 	as->arcstat_l2_rebuild_success.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_success);
 	as->arcstat_l2_rebuild_abort_unsupported.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
 	as->arcstat_l2_rebuild_abort_io_errors.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
 	as->arcstat_l2_rebuild_abort_dh_errors.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
 	as->arcstat_l2_rebuild_abort_cksum_lb_errors.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
 	as->arcstat_l2_rebuild_abort_lowmem.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
 	as->arcstat_l2_rebuild_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_size);
 	as->arcstat_l2_rebuild_asize.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_asize);
 	as->arcstat_l2_rebuild_bufs.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs);
 	as->arcstat_l2_rebuild_bufs_precached.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_bufs_precached);
 	as->arcstat_l2_rebuild_log_blks.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_l2_rebuild_log_blks);
 	as->arcstat_memory_throttle_count.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_memory_throttle_count);
 	as->arcstat_memory_direct_count.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_memory_direct_count);
 	as->arcstat_memory_indirect_count.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_memory_indirect_count);
 
 	as->arcstat_memory_all_bytes.value.ui64 =
 	    arc_all_memory();
 	as->arcstat_memory_free_bytes.value.ui64 =
 	    arc_free_memory();
 	as->arcstat_memory_available_bytes.value.i64 =
 	    arc_available_memory();
 
 	as->arcstat_prune.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_prune);
 	as->arcstat_meta_used.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_meta_used);
 	as->arcstat_async_upgrade_sync.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_async_upgrade_sync);
 	as->arcstat_predictive_prefetch.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_predictive_prefetch);
 	as->arcstat_demand_hit_predictive_prefetch.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_demand_hit_predictive_prefetch);
 	as->arcstat_demand_iohit_predictive_prefetch.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_demand_iohit_predictive_prefetch);
 	as->arcstat_prescient_prefetch.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_prescient_prefetch);
 	as->arcstat_demand_hit_prescient_prefetch.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_demand_hit_prescient_prefetch);
 	as->arcstat_demand_iohit_prescient_prefetch.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_demand_iohit_prescient_prefetch);
 	as->arcstat_raw_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_raw_size);
 	as->arcstat_cached_only_in_progress.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_cached_only_in_progress);
 	as->arcstat_abd_chunk_waste_size.value.ui64 =
 	    wmsum_value(&arc_sums.arcstat_abd_chunk_waste_size);
 
 	return (0);
 }
 
 /*
  * This function *must* return indices evenly distributed between all
  * sublists of the multilist. This is needed due to how the ARC eviction
  * code is laid out; arc_evict_state() assumes ARC buffers are evenly
  * distributed between all sublists and uses this assumption when
  * deciding which sublist to evict from and how much to evict from it.
  */
 static unsigned int
 arc_state_multilist_index_func(multilist_t *ml, void *obj)
 {
 	arc_buf_hdr_t *hdr = obj;
 
 	/*
 	 * We rely on b_dva to generate evenly distributed index
 	 * numbers using buf_hash below. So, as an added precaution,
 	 * let's make sure we never add empty buffers to the arc lists.
 	 */
 	ASSERT(!HDR_EMPTY(hdr));
 
 	/*
 	 * The assumption here, is the hash value for a given
 	 * arc_buf_hdr_t will remain constant throughout its lifetime
 	 * (i.e. its b_spa, b_dva, and b_birth fields don't change).
 	 * Thus, we don't need to store the header's sublist index
 	 * on insertion, as this index can be recalculated on removal.
 	 *
 	 * Also, the low order bits of the hash value are thought to be
 	 * distributed evenly. Otherwise, in the case that the multilist
 	 * has a power of two number of sublists, each sublists' usage
 	 * would not be evenly distributed. In this context full 64bit
 	 * division would be a waste of time, so limit it to 32 bits.
 	 */
 	return ((unsigned int)buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
 	    multilist_get_num_sublists(ml));
 }
 
 static unsigned int
 arc_state_l2c_multilist_index_func(multilist_t *ml, void *obj)
 {
 	panic("Header %p insert into arc_l2c_only %p", obj, ml);
 }
 
 #define	WARN_IF_TUNING_IGNORED(tuning, value, do_warn) do {	\
 	if ((do_warn) && (tuning) && ((tuning) != (value))) {	\
 		cmn_err(CE_WARN,				\
 		    "ignoring tunable %s (using %llu instead)",	\
 		    (#tuning), (u_longlong_t)(value));	\
 	}							\
 } while (0)
 
 /*
  * Called during module initialization and periodically thereafter to
  * apply reasonable changes to the exposed performance tunings.  Can also be
  * called explicitly by param_set_arc_*() functions when ARC tunables are
  * updated manually.  Non-zero zfs_* values which differ from the currently set
  * values will be applied.
  */
 void
 arc_tuning_update(boolean_t verbose)
 {
 	uint64_t allmem = arc_all_memory();
 
 	/* Valid range: 32M - <arc_c_max> */
 	if ((zfs_arc_min) && (zfs_arc_min != arc_c_min) &&
 	    (zfs_arc_min >= 2ULL << SPA_MAXBLOCKSHIFT) &&
 	    (zfs_arc_min <= arc_c_max)) {
 		arc_c_min = zfs_arc_min;
 		arc_c = MAX(arc_c, arc_c_min);
 	}
 	WARN_IF_TUNING_IGNORED(zfs_arc_min, arc_c_min, verbose);
 
 	/* Valid range: 64M - <all physical memory> */
 	if ((zfs_arc_max) && (zfs_arc_max != arc_c_max) &&
 	    (zfs_arc_max >= MIN_ARC_MAX) && (zfs_arc_max < allmem) &&
 	    (zfs_arc_max > arc_c_min)) {
 		arc_c_max = zfs_arc_max;
 		arc_c = MIN(arc_c, arc_c_max);
 		if (arc_dnode_limit > arc_c_max)
 			arc_dnode_limit = arc_c_max;
 	}
 	WARN_IF_TUNING_IGNORED(zfs_arc_max, arc_c_max, verbose);
 
 	/* Valid range: 0 - <all physical memory> */
 	arc_dnode_limit = zfs_arc_dnode_limit ? zfs_arc_dnode_limit :
 	    MIN(zfs_arc_dnode_limit_percent, 100) * arc_c_max / 100;
 	WARN_IF_TUNING_IGNORED(zfs_arc_dnode_limit, arc_dnode_limit, verbose);
 
 	/* Valid range: 1 - N */
 	if (zfs_arc_grow_retry)
 		arc_grow_retry = zfs_arc_grow_retry;
 
 	/* Valid range: 1 - N */
 	if (zfs_arc_shrink_shift) {
 		arc_shrink_shift = zfs_arc_shrink_shift;
 		arc_no_grow_shift = MIN(arc_no_grow_shift, arc_shrink_shift -1);
 	}
 
 	/* Valid range: 1 - N ms */
 	if (zfs_arc_min_prefetch_ms)
 		arc_min_prefetch_ms = zfs_arc_min_prefetch_ms;
 
 	/* Valid range: 1 - N ms */
 	if (zfs_arc_min_prescient_prefetch_ms) {
 		arc_min_prescient_prefetch_ms =
 		    zfs_arc_min_prescient_prefetch_ms;
 	}
 
 	/* Valid range: 0 - 100 */
 	if (zfs_arc_lotsfree_percent <= 100)
 		arc_lotsfree_percent = zfs_arc_lotsfree_percent;
 	WARN_IF_TUNING_IGNORED(zfs_arc_lotsfree_percent, arc_lotsfree_percent,
 	    verbose);
 
 	/* Valid range: 0 - <all physical memory> */
 	if ((zfs_arc_sys_free) && (zfs_arc_sys_free != arc_sys_free))
 		arc_sys_free = MIN(zfs_arc_sys_free, allmem);
 	WARN_IF_TUNING_IGNORED(zfs_arc_sys_free, arc_sys_free, verbose);
 }
 
 static void
 arc_state_multilist_init(multilist_t *ml,
     multilist_sublist_index_func_t *index_func, int *maxcountp)
 {
 	multilist_create(ml, sizeof (arc_buf_hdr_t),
 	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node), index_func);
 	*maxcountp = MAX(*maxcountp, multilist_get_num_sublists(ml));
 }
 
 static void
 arc_state_init(void)
 {
 	int num_sublists = 0;
 
 	arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_METADATA],
 	    arc_state_multilist_index_func, &num_sublists);
 	arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_DATA],
 	    arc_state_multilist_index_func, &num_sublists);
 	arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
 	    arc_state_multilist_index_func, &num_sublists);
 	arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
 	    arc_state_multilist_index_func, &num_sublists);
 	arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
 	    arc_state_multilist_index_func, &num_sublists);
 	arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_DATA],
 	    arc_state_multilist_index_func, &num_sublists);
 	arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
 	    arc_state_multilist_index_func, &num_sublists);
 	arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
 	    arc_state_multilist_index_func, &num_sublists);
 	arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_METADATA],
 	    arc_state_multilist_index_func, &num_sublists);
 	arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_DATA],
 	    arc_state_multilist_index_func, &num_sublists);
 
 	/*
 	 * L2 headers should never be on the L2 state list since they don't
 	 * have L1 headers allocated.  Special index function asserts that.
 	 */
 	arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
 	    arc_state_l2c_multilist_index_func, &num_sublists);
 	arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
 	    arc_state_l2c_multilist_index_func, &num_sublists);
 
 	/*
 	 * Keep track of the number of markers needed to reclaim buffers from
 	 * any ARC state.  The markers will be pre-allocated so as to minimize
 	 * the number of memory allocations performed by the eviction thread.
 	 */
 	arc_state_evict_marker_count = num_sublists;
 
 	zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);
 
 	zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);
 
 	wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA], 0);
 	wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
 	wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA], 0);
 	wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
 
 	wmsum_init(&arc_sums.arcstat_hits, 0);
 	wmsum_init(&arc_sums.arcstat_iohits, 0);
 	wmsum_init(&arc_sums.arcstat_misses, 0);
 	wmsum_init(&arc_sums.arcstat_demand_data_hits, 0);
 	wmsum_init(&arc_sums.arcstat_demand_data_iohits, 0);
 	wmsum_init(&arc_sums.arcstat_demand_data_misses, 0);
 	wmsum_init(&arc_sums.arcstat_demand_metadata_hits, 0);
 	wmsum_init(&arc_sums.arcstat_demand_metadata_iohits, 0);
 	wmsum_init(&arc_sums.arcstat_demand_metadata_misses, 0);
 	wmsum_init(&arc_sums.arcstat_prefetch_data_hits, 0);
 	wmsum_init(&arc_sums.arcstat_prefetch_data_iohits, 0);
 	wmsum_init(&arc_sums.arcstat_prefetch_data_misses, 0);
 	wmsum_init(&arc_sums.arcstat_prefetch_metadata_hits, 0);
 	wmsum_init(&arc_sums.arcstat_prefetch_metadata_iohits, 0);
 	wmsum_init(&arc_sums.arcstat_prefetch_metadata_misses, 0);
 	wmsum_init(&arc_sums.arcstat_mru_hits, 0);
 	wmsum_init(&arc_sums.arcstat_mru_ghost_hits, 0);
 	wmsum_init(&arc_sums.arcstat_mfu_hits, 0);
 	wmsum_init(&arc_sums.arcstat_mfu_ghost_hits, 0);
 	wmsum_init(&arc_sums.arcstat_uncached_hits, 0);
 	wmsum_init(&arc_sums.arcstat_deleted, 0);
 	wmsum_init(&arc_sums.arcstat_mutex_miss, 0);
 	wmsum_init(&arc_sums.arcstat_access_skip, 0);
 	wmsum_init(&arc_sums.arcstat_evict_skip, 0);
 	wmsum_init(&arc_sums.arcstat_evict_not_enough, 0);
 	wmsum_init(&arc_sums.arcstat_evict_l2_cached, 0);
 	wmsum_init(&arc_sums.arcstat_evict_l2_eligible, 0);
 	wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mfu, 0);
 	wmsum_init(&arc_sums.arcstat_evict_l2_eligible_mru, 0);
 	wmsum_init(&arc_sums.arcstat_evict_l2_ineligible, 0);
 	wmsum_init(&arc_sums.arcstat_evict_l2_skip, 0);
 	wmsum_init(&arc_sums.arcstat_hash_collisions, 0);
 	wmsum_init(&arc_sums.arcstat_hash_chains, 0);
 	aggsum_init(&arc_sums.arcstat_size, 0);
 	wmsum_init(&arc_sums.arcstat_compressed_size, 0);
 	wmsum_init(&arc_sums.arcstat_uncompressed_size, 0);
 	wmsum_init(&arc_sums.arcstat_overhead_size, 0);
 	wmsum_init(&arc_sums.arcstat_hdr_size, 0);
 	wmsum_init(&arc_sums.arcstat_data_size, 0);
 	wmsum_init(&arc_sums.arcstat_metadata_size, 0);
 	wmsum_init(&arc_sums.arcstat_dbuf_size, 0);
 	wmsum_init(&arc_sums.arcstat_dnode_size, 0);
 	wmsum_init(&arc_sums.arcstat_bonus_size, 0);
 	wmsum_init(&arc_sums.arcstat_l2_hits, 0);
 	wmsum_init(&arc_sums.arcstat_l2_misses, 0);
 	wmsum_init(&arc_sums.arcstat_l2_prefetch_asize, 0);
 	wmsum_init(&arc_sums.arcstat_l2_mru_asize, 0);
 	wmsum_init(&arc_sums.arcstat_l2_mfu_asize, 0);
 	wmsum_init(&arc_sums.arcstat_l2_bufc_data_asize, 0);
 	wmsum_init(&arc_sums.arcstat_l2_bufc_metadata_asize, 0);
 	wmsum_init(&arc_sums.arcstat_l2_feeds, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rw_clash, 0);
 	wmsum_init(&arc_sums.arcstat_l2_read_bytes, 0);
 	wmsum_init(&arc_sums.arcstat_l2_write_bytes, 0);
 	wmsum_init(&arc_sums.arcstat_l2_writes_sent, 0);
 	wmsum_init(&arc_sums.arcstat_l2_writes_done, 0);
 	wmsum_init(&arc_sums.arcstat_l2_writes_error, 0);
 	wmsum_init(&arc_sums.arcstat_l2_writes_lock_retry, 0);
 	wmsum_init(&arc_sums.arcstat_l2_evict_lock_retry, 0);
 	wmsum_init(&arc_sums.arcstat_l2_evict_reading, 0);
 	wmsum_init(&arc_sums.arcstat_l2_evict_l1cached, 0);
 	wmsum_init(&arc_sums.arcstat_l2_free_on_write, 0);
 	wmsum_init(&arc_sums.arcstat_l2_abort_lowmem, 0);
 	wmsum_init(&arc_sums.arcstat_l2_cksum_bad, 0);
 	wmsum_init(&arc_sums.arcstat_l2_io_error, 0);
 	wmsum_init(&arc_sums.arcstat_l2_lsize, 0);
 	wmsum_init(&arc_sums.arcstat_l2_psize, 0);
 	aggsum_init(&arc_sums.arcstat_l2_hdr_size, 0);
 	wmsum_init(&arc_sums.arcstat_l2_log_blk_writes, 0);
 	wmsum_init(&arc_sums.arcstat_l2_log_blk_asize, 0);
 	wmsum_init(&arc_sums.arcstat_l2_log_blk_count, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_success, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_unsupported, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_io_errors, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_dh_errors, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_abort_lowmem, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_size, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_asize, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_bufs_precached, 0);
 	wmsum_init(&arc_sums.arcstat_l2_rebuild_log_blks, 0);
 	wmsum_init(&arc_sums.arcstat_memory_throttle_count, 0);
 	wmsum_init(&arc_sums.arcstat_memory_direct_count, 0);
 	wmsum_init(&arc_sums.arcstat_memory_indirect_count, 0);
 	wmsum_init(&arc_sums.arcstat_prune, 0);
 	wmsum_init(&arc_sums.arcstat_meta_used, 0);
 	wmsum_init(&arc_sums.arcstat_async_upgrade_sync, 0);
 	wmsum_init(&arc_sums.arcstat_predictive_prefetch, 0);
 	wmsum_init(&arc_sums.arcstat_demand_hit_predictive_prefetch, 0);
 	wmsum_init(&arc_sums.arcstat_demand_iohit_predictive_prefetch, 0);
 	wmsum_init(&arc_sums.arcstat_prescient_prefetch, 0);
 	wmsum_init(&arc_sums.arcstat_demand_hit_prescient_prefetch, 0);
 	wmsum_init(&arc_sums.arcstat_demand_iohit_prescient_prefetch, 0);
 	wmsum_init(&arc_sums.arcstat_raw_size, 0);
 	wmsum_init(&arc_sums.arcstat_cached_only_in_progress, 0);
 	wmsum_init(&arc_sums.arcstat_abd_chunk_waste_size, 0);
 
 	arc_anon->arcs_state = ARC_STATE_ANON;
 	arc_mru->arcs_state = ARC_STATE_MRU;
 	arc_mru_ghost->arcs_state = ARC_STATE_MRU_GHOST;
 	arc_mfu->arcs_state = ARC_STATE_MFU;
 	arc_mfu_ghost->arcs_state = ARC_STATE_MFU_GHOST;
 	arc_l2c_only->arcs_state = ARC_STATE_L2C_ONLY;
 	arc_uncached->arcs_state = ARC_STATE_UNCACHED;
 }
 
 static void
 arc_state_fini(void)
 {
 	zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);
 
 	zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
 	zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
 	zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);
 
 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
 	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
 	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
 	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
 	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
 	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
 	multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
 	multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
 	multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_METADATA]);
 	multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_DATA]);
 
 	wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
 	wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
 	wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
 	wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
 
 	wmsum_fini(&arc_sums.arcstat_hits);
 	wmsum_fini(&arc_sums.arcstat_iohits);
 	wmsum_fini(&arc_sums.arcstat_misses);
 	wmsum_fini(&arc_sums.arcstat_demand_data_hits);
 	wmsum_fini(&arc_sums.arcstat_demand_data_iohits);
 	wmsum_fini(&arc_sums.arcstat_demand_data_misses);
 	wmsum_fini(&arc_sums.arcstat_demand_metadata_hits);
 	wmsum_fini(&arc_sums.arcstat_demand_metadata_iohits);
 	wmsum_fini(&arc_sums.arcstat_demand_metadata_misses);
 	wmsum_fini(&arc_sums.arcstat_prefetch_data_hits);
 	wmsum_fini(&arc_sums.arcstat_prefetch_data_iohits);
 	wmsum_fini(&arc_sums.arcstat_prefetch_data_misses);
 	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_hits);
 	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_iohits);
 	wmsum_fini(&arc_sums.arcstat_prefetch_metadata_misses);
 	wmsum_fini(&arc_sums.arcstat_mru_hits);
 	wmsum_fini(&arc_sums.arcstat_mru_ghost_hits);
 	wmsum_fini(&arc_sums.arcstat_mfu_hits);
 	wmsum_fini(&arc_sums.arcstat_mfu_ghost_hits);
 	wmsum_fini(&arc_sums.arcstat_uncached_hits);
 	wmsum_fini(&arc_sums.arcstat_deleted);
 	wmsum_fini(&arc_sums.arcstat_mutex_miss);
 	wmsum_fini(&arc_sums.arcstat_access_skip);
 	wmsum_fini(&arc_sums.arcstat_evict_skip);
 	wmsum_fini(&arc_sums.arcstat_evict_not_enough);
 	wmsum_fini(&arc_sums.arcstat_evict_l2_cached);
 	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible);
 	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mfu);
 	wmsum_fini(&arc_sums.arcstat_evict_l2_eligible_mru);
 	wmsum_fini(&arc_sums.arcstat_evict_l2_ineligible);
 	wmsum_fini(&arc_sums.arcstat_evict_l2_skip);
 	wmsum_fini(&arc_sums.arcstat_hash_collisions);
 	wmsum_fini(&arc_sums.arcstat_hash_chains);
 	aggsum_fini(&arc_sums.arcstat_size);
 	wmsum_fini(&arc_sums.arcstat_compressed_size);
 	wmsum_fini(&arc_sums.arcstat_uncompressed_size);
 	wmsum_fini(&arc_sums.arcstat_overhead_size);
 	wmsum_fini(&arc_sums.arcstat_hdr_size);
 	wmsum_fini(&arc_sums.arcstat_data_size);
 	wmsum_fini(&arc_sums.arcstat_metadata_size);
 	wmsum_fini(&arc_sums.arcstat_dbuf_size);
 	wmsum_fini(&arc_sums.arcstat_dnode_size);
 	wmsum_fini(&arc_sums.arcstat_bonus_size);
 	wmsum_fini(&arc_sums.arcstat_l2_hits);
 	wmsum_fini(&arc_sums.arcstat_l2_misses);
 	wmsum_fini(&arc_sums.arcstat_l2_prefetch_asize);
 	wmsum_fini(&arc_sums.arcstat_l2_mru_asize);
 	wmsum_fini(&arc_sums.arcstat_l2_mfu_asize);
 	wmsum_fini(&arc_sums.arcstat_l2_bufc_data_asize);
 	wmsum_fini(&arc_sums.arcstat_l2_bufc_metadata_asize);
 	wmsum_fini(&arc_sums.arcstat_l2_feeds);
 	wmsum_fini(&arc_sums.arcstat_l2_rw_clash);
 	wmsum_fini(&arc_sums.arcstat_l2_read_bytes);
 	wmsum_fini(&arc_sums.arcstat_l2_write_bytes);
 	wmsum_fini(&arc_sums.arcstat_l2_writes_sent);
 	wmsum_fini(&arc_sums.arcstat_l2_writes_done);
 	wmsum_fini(&arc_sums.arcstat_l2_writes_error);
 	wmsum_fini(&arc_sums.arcstat_l2_writes_lock_retry);
 	wmsum_fini(&arc_sums.arcstat_l2_evict_lock_retry);
 	wmsum_fini(&arc_sums.arcstat_l2_evict_reading);
 	wmsum_fini(&arc_sums.arcstat_l2_evict_l1cached);
 	wmsum_fini(&arc_sums.arcstat_l2_free_on_write);
 	wmsum_fini(&arc_sums.arcstat_l2_abort_lowmem);
 	wmsum_fini(&arc_sums.arcstat_l2_cksum_bad);
 	wmsum_fini(&arc_sums.arcstat_l2_io_error);
 	wmsum_fini(&arc_sums.arcstat_l2_lsize);
 	wmsum_fini(&arc_sums.arcstat_l2_psize);
 	aggsum_fini(&arc_sums.arcstat_l2_hdr_size);
 	wmsum_fini(&arc_sums.arcstat_l2_log_blk_writes);
 	wmsum_fini(&arc_sums.arcstat_l2_log_blk_asize);
 	wmsum_fini(&arc_sums.arcstat_l2_log_blk_count);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_success);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_unsupported);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_io_errors);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_dh_errors);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_cksum_lb_errors);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_abort_lowmem);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_size);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_asize);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_bufs_precached);
 	wmsum_fini(&arc_sums.arcstat_l2_rebuild_log_blks);
 	wmsum_fini(&arc_sums.arcstat_memory_throttle_count);
 	wmsum_fini(&arc_sums.arcstat_memory_direct_count);
 	wmsum_fini(&arc_sums.arcstat_memory_indirect_count);
 	wmsum_fini(&arc_sums.arcstat_prune);
 	wmsum_fini(&arc_sums.arcstat_meta_used);
 	wmsum_fini(&arc_sums.arcstat_async_upgrade_sync);
 	wmsum_fini(&arc_sums.arcstat_predictive_prefetch);
 	wmsum_fini(&arc_sums.arcstat_demand_hit_predictive_prefetch);
 	wmsum_fini(&arc_sums.arcstat_demand_iohit_predictive_prefetch);
 	wmsum_fini(&arc_sums.arcstat_prescient_prefetch);
 	wmsum_fini(&arc_sums.arcstat_demand_hit_prescient_prefetch);
 	wmsum_fini(&arc_sums.arcstat_demand_iohit_prescient_prefetch);
 	wmsum_fini(&arc_sums.arcstat_raw_size);
 	wmsum_fini(&arc_sums.arcstat_cached_only_in_progress);
 	wmsum_fini(&arc_sums.arcstat_abd_chunk_waste_size);
 }
 
 uint64_t
 arc_target_bytes(void)
 {
 	return (arc_c);
 }
 
 void
 arc_set_limits(uint64_t allmem)
 {
 	/* Set min cache to 1/32 of all memory, or 32MB, whichever is more. */
 	arc_c_min = MAX(allmem / 32, 2ULL << SPA_MAXBLOCKSHIFT);
 
 	/* How to set default max varies by platform. */
 	arc_c_max = arc_default_max(arc_c_min, allmem);
 }
 void
 arc_init(void)
 {
 	uint64_t percent, allmem = arc_all_memory();
 	mutex_init(&arc_evict_lock, NULL, MUTEX_DEFAULT, NULL);
 	list_create(&arc_evict_waiters, sizeof (arc_evict_waiter_t),
 	    offsetof(arc_evict_waiter_t, aew_node));
 
 	arc_min_prefetch_ms = 1000;
 	arc_min_prescient_prefetch_ms = 6000;
 
 #if defined(_KERNEL)
 	arc_lowmem_init();
 #endif
 
 	arc_set_limits(allmem);
 
 #ifdef _KERNEL
 	/*
 	 * If zfs_arc_max is non-zero at init, meaning it was set in the kernel
 	 * environment before the module was loaded, don't block setting the
 	 * maximum because it is less than arc_c_min, instead, reset arc_c_min
 	 * to a lower value.
 	 * zfs_arc_min will be handled by arc_tuning_update().
 	 */
 	if (zfs_arc_max != 0 && zfs_arc_max >= MIN_ARC_MAX &&
 	    zfs_arc_max < allmem) {
 		arc_c_max = zfs_arc_max;
 		if (arc_c_min >= arc_c_max) {
 			arc_c_min = MAX(zfs_arc_max / 2,
 			    2ULL << SPA_MAXBLOCKSHIFT);
 		}
 	}
 #else
 	/*
 	 * In userland, there's only the memory pressure that we artificially
 	 * create (see arc_available_memory()).  Don't let arc_c get too
 	 * small, because it can cause transactions to be larger than
 	 * arc_c, causing arc_tempreserve_space() to fail.
 	 */
 	arc_c_min = MAX(arc_c_max / 2, 2ULL << SPA_MAXBLOCKSHIFT);
 #endif
 
 	arc_c = arc_c_min;
 	/*
 	 * 32-bit fixed point fractions of metadata from total ARC size,
 	 * MRU data from all data and MRU metadata from all metadata.
 	 */
 	arc_meta = (1ULL << 32) / 4;	/* Metadata is 25% of arc_c. */
 	arc_pd = (1ULL << 32) / 2;	/* Data MRU is 50% of data. */
 	arc_pm = (1ULL << 32) / 2;	/* Metadata MRU is 50% of metadata. */
 
 	percent = MIN(zfs_arc_dnode_limit_percent, 100);
 	arc_dnode_limit = arc_c_max * percent / 100;
 
 	/* Apply user specified tunings */
 	arc_tuning_update(B_TRUE);
 
 	/* if kmem_flags are set, lets try to use less memory */
 	if (kmem_debugging())
 		arc_c = arc_c / 2;
 	if (arc_c < arc_c_min)
 		arc_c = arc_c_min;
 
 	arc_register_hotplug();
 
 	arc_state_init();
 
 	buf_init();
 
 	list_create(&arc_prune_list, sizeof (arc_prune_t),
 	    offsetof(arc_prune_t, p_node));
 	mutex_init(&arc_prune_mtx, NULL, MUTEX_DEFAULT, NULL);
 
 	arc_prune_taskq = taskq_create("arc_prune", zfs_arc_prune_task_threads,
 	    defclsyspri, 100, INT_MAX, TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
 
 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
 	    sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
 
 	if (arc_ksp != NULL) {
 		arc_ksp->ks_data = &arc_stats;
 		arc_ksp->ks_update = arc_kstat_update;
 		kstat_install(arc_ksp);
 	}
 
 	arc_state_evict_markers =
 	    arc_state_alloc_markers(arc_state_evict_marker_count);
 	arc_evict_zthr = zthr_create_timer("arc_evict",
 	    arc_evict_cb_check, arc_evict_cb, NULL, SEC2NSEC(1), defclsyspri);
 	arc_reap_zthr = zthr_create_timer("arc_reap",
 	    arc_reap_cb_check, arc_reap_cb, NULL, SEC2NSEC(1), minclsyspri);
 
 	arc_warm = B_FALSE;
 
 	/*
 	 * Calculate maximum amount of dirty data per pool.
 	 *
 	 * If it has been set by a module parameter, take that.
 	 * Otherwise, use a percentage of physical memory defined by
 	 * zfs_dirty_data_max_percent (default 10%) with a cap at
 	 * zfs_dirty_data_max_max (default 4G or 25% of physical memory).
 	 */
 #ifdef __LP64__
 	if (zfs_dirty_data_max_max == 0)
 		zfs_dirty_data_max_max = MIN(4ULL * 1024 * 1024 * 1024,
 		    allmem * zfs_dirty_data_max_max_percent / 100);
 #else
 	if (zfs_dirty_data_max_max == 0)
 		zfs_dirty_data_max_max = MIN(1ULL * 1024 * 1024 * 1024,
 		    allmem * zfs_dirty_data_max_max_percent / 100);
 #endif
 
 	if (zfs_dirty_data_max == 0) {
 		zfs_dirty_data_max = allmem *
 		    zfs_dirty_data_max_percent / 100;
 		zfs_dirty_data_max = MIN(zfs_dirty_data_max,
 		    zfs_dirty_data_max_max);
 	}
 
 	if (zfs_wrlog_data_max == 0) {
 
 		/*
 		 * dp_wrlog_total is reduced for each txg at the end of
 		 * spa_sync(). However, dp_dirty_total is reduced every time
 		 * a block is written out. Thus under normal operation,
 		 * dp_wrlog_total could grow 2 times as big as
 		 * zfs_dirty_data_max.
 		 */
 		zfs_wrlog_data_max = zfs_dirty_data_max * 2;
 	}
 }
 
 void
 arc_fini(void)
 {
 	arc_prune_t *p;
 
 #ifdef _KERNEL
 	arc_lowmem_fini();
 #endif /* _KERNEL */
 
 	/* Use B_TRUE to ensure *all* buffers are evicted */
 	arc_flush(NULL, B_TRUE);
 
 	if (arc_ksp != NULL) {
 		kstat_delete(arc_ksp);
 		arc_ksp = NULL;
 	}
 
 	taskq_wait(arc_prune_taskq);
 	taskq_destroy(arc_prune_taskq);
 
 	mutex_enter(&arc_prune_mtx);
 	while ((p = list_remove_head(&arc_prune_list)) != NULL) {
 		zfs_refcount_remove(&p->p_refcnt, &arc_prune_list);
 		zfs_refcount_destroy(&p->p_refcnt);
 		kmem_free(p, sizeof (*p));
 	}
 	mutex_exit(&arc_prune_mtx);
 
 	list_destroy(&arc_prune_list);
 	mutex_destroy(&arc_prune_mtx);
 
 	(void) zthr_cancel(arc_evict_zthr);
 	(void) zthr_cancel(arc_reap_zthr);
 	arc_state_free_markers(arc_state_evict_markers,
 	    arc_state_evict_marker_count);
 
 	mutex_destroy(&arc_evict_lock);
 	list_destroy(&arc_evict_waiters);
 
 	/*
 	 * Free any buffers that were tagged for destruction.  This needs
 	 * to occur before arc_state_fini() runs and destroys the aggsum
 	 * values which are updated when freeing scatter ABDs.
 	 */
 	l2arc_do_free_on_write();
 
 	/*
 	 * buf_fini() must proceed arc_state_fini() because buf_fin() may
 	 * trigger the release of kmem magazines, which can callback to
 	 * arc_space_return() which accesses aggsums freed in act_state_fini().
 	 */
 	buf_fini();
 	arc_state_fini();
 
 	arc_unregister_hotplug();
 
 	/*
 	 * We destroy the zthrs after all the ARC state has been
 	 * torn down to avoid the case of them receiving any
 	 * wakeup() signals after they are destroyed.
 	 */
 	zthr_destroy(arc_evict_zthr);
 	zthr_destroy(arc_reap_zthr);
 
 	ASSERT0(arc_loaned_bytes);
 }
 
 /*
  * Level 2 ARC
  *
  * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
  * It uses dedicated storage devices to hold cached data, which are populated
  * using large infrequent writes.  The main role of this cache is to boost
  * the performance of random read workloads.  The intended L2ARC devices
  * include short-stroked disks, solid state disks, and other media with
  * substantially faster read latency than disk.
  *
  *                 +-----------------------+
  *                 |         ARC           |
  *                 +-----------------------+
  *                    |         ^     ^
  *                    |         |     |
  *      l2arc_feed_thread()    arc_read()
  *                    |         |     |
  *                    |  l2arc read   |
  *                    V         |     |
  *               +---------------+    |
  *               |     L2ARC     |    |
  *               +---------------+    |
  *                   |    ^           |
  *          l2arc_write() |           |
  *                   |    |           |
  *                   V    |           |
  *                 +-------+      +-------+
  *                 | vdev  |      | vdev  |
  *                 | cache |      | cache |
  *                 +-------+      +-------+
  *                 +=========+     .-----.
  *                 :  L2ARC  :    |-_____-|
  *                 : devices :    | Disks |
  *                 +=========+    `-_____-'
  *
  * Read requests are satisfied from the following sources, in order:
  *
  *	1) ARC
  *	2) vdev cache of L2ARC devices
  *	3) L2ARC devices
  *	4) vdev cache of disks
  *	5) disks
  *
  * Some L2ARC device types exhibit extremely slow write performance.
  * To accommodate for this there are some significant differences between
  * the L2ARC and traditional cache design:
  *
  * 1. There is no eviction path from the ARC to the L2ARC.  Evictions from
  * the ARC behave as usual, freeing buffers and placing headers on ghost
  * lists.  The ARC does not send buffers to the L2ARC during eviction as
  * this would add inflated write latencies for all ARC memory pressure.
  *
  * 2. The L2ARC attempts to cache data from the ARC before it is evicted.
  * It does this by periodically scanning buffers from the eviction-end of
  * the MFU and MRU ARC lists, copying them to the L2ARC devices if they are
  * not already there. It scans until a headroom of buffers is satisfied,
  * which itself is a buffer for ARC eviction. If a compressible buffer is
  * found during scanning and selected for writing to an L2ARC device, we
  * temporarily boost scanning headroom during the next scan cycle to make
  * sure we adapt to compression effects (which might significantly reduce
  * the data volume we write to L2ARC). The thread that does this is
  * l2arc_feed_thread(), illustrated below; example sizes are included to
  * provide a better sense of ratio than this diagram:
  *
  *	       head -->                        tail
  *	        +---------------------+----------+
  *	ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->.   # already on L2ARC
  *	        +---------------------+----------+   |   o L2ARC eligible
  *	ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->|   : ARC buffer
  *	        +---------------------+----------+   |
  *	             15.9 Gbytes      ^ 32 Mbytes    |
  *	                           headroom          |
  *	                                      l2arc_feed_thread()
  *	                                             |
  *	                 l2arc write hand <--[oooo]--'
  *	                         |           8 Mbyte
  *	                         |          write max
  *	                         V
  *		  +==============================+
  *	L2ARC dev |####|#|###|###|    |####| ... |
  *	          +==============================+
  *	                     32 Gbytes
  *
  * 3. If an ARC buffer is copied to the L2ARC but then hit instead of
  * evicted, then the L2ARC has cached a buffer much sooner than it probably
  * needed to, potentially wasting L2ARC device bandwidth and storage.  It is
  * safe to say that this is an uncommon case, since buffers at the end of
  * the ARC lists have moved there due to inactivity.
  *
  * 4. If the ARC evicts faster than the L2ARC can maintain a headroom,
  * then the L2ARC simply misses copying some buffers.  This serves as a
  * pressure valve to prevent heavy read workloads from both stalling the ARC
  * with waits and clogging the L2ARC with writes.  This also helps prevent
  * the potential for the L2ARC to churn if it attempts to cache content too
  * quickly, such as during backups of the entire pool.
  *
  * 5. After system boot and before the ARC has filled main memory, there are
  * no evictions from the ARC and so the tails of the ARC_mfu and ARC_mru
  * lists can remain mostly static.  Instead of searching from tail of these
  * lists as pictured, the l2arc_feed_thread() will search from the list heads
  * for eligible buffers, greatly increasing its chance of finding them.
  *
  * The L2ARC device write speed is also boosted during this time so that
  * the L2ARC warms up faster.  Since there have been no ARC evictions yet,
  * there are no L2ARC reads, and no fear of degrading read performance
  * through increased writes.
  *
  * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
  * the vdev queue can aggregate them into larger and fewer writes.  Each
  * device is written to in a rotor fashion, sweeping writes through
  * available space then repeating.
  *
  * 7. The L2ARC does not store dirty content.  It never needs to flush
  * write buffers back to disk based storage.
  *
  * 8. If an ARC buffer is written (and dirtied) which also exists in the
  * L2ARC, the now stale L2ARC buffer is immediately dropped.
  *
  * The performance of the L2ARC can be tweaked by a number of tunables, which
  * may be necessary for different workloads:
  *
  *	l2arc_write_max		max write bytes per interval
  *	l2arc_write_boost	extra write bytes during device warmup
  *	l2arc_noprefetch	skip caching prefetched buffers
  *	l2arc_headroom		number of max device writes to precache
  *	l2arc_headroom_boost	when we find compressed buffers during ARC
  *				scanning, we multiply headroom by this
  *				percentage factor for the next scan cycle,
  *				since more compressed buffers are likely to
  *				be present
  *	l2arc_feed_secs		seconds between L2ARC writing
  *
  * Tunables may be removed or added as future performance improvements are
  * integrated, and also may become zpool properties.
  *
  * There are three key functions that control how the L2ARC warms up:
  *
  *	l2arc_write_eligible()	check if a buffer is eligible to cache
  *	l2arc_write_size()	calculate how much to write
  *	l2arc_write_interval()	calculate sleep delay between writes
  *
  * These three functions determine what to write, how much, and how quickly
  * to send writes.
  *
  * L2ARC persistence:
  *
  * When writing buffers to L2ARC, we periodically add some metadata to
  * make sure we can pick them up after reboot, thus dramatically reducing
  * the impact that any downtime has on the performance of storage systems
  * with large caches.
  *
  * The implementation works fairly simply by integrating the following two
  * modifications:
  *
  * *) When writing to the L2ARC, we occasionally write a "l2arc log block",
  *    which is an additional piece of metadata which describes what's been
  *    written. This allows us to rebuild the arc_buf_hdr_t structures of the
  *    main ARC buffers. There are 2 linked-lists of log blocks headed by
  *    dh_start_lbps[2]. We alternate which chain we append to, so they are
  *    time-wise and offset-wise interleaved, but that is an optimization rather
  *    than for correctness. The log block also includes a pointer to the
  *    previous block in its chain.
  *
  * *) We reserve SPA_MINBLOCKSIZE of space at the start of each L2ARC device
  *    for our header bookkeeping purposes. This contains a device header,
  *    which contains our top-level reference structures. We update it each
  *    time we write a new log block, so that we're able to locate it in the
  *    L2ARC device. If this write results in an inconsistent device header
  *    (e.g. due to power failure), we detect this by verifying the header's
  *    checksum and simply fail to reconstruct the L2ARC after reboot.
  *
  * Implementation diagram:
  *
  * +=== L2ARC device (not to scale) ======================================+
  * |       ___two newest log block pointers__.__________                  |
  * |      /                                   \dh_start_lbps[1]           |
  * |	 /				       \         \dh_start_lbps[0]|
  * |.___/__.                                    V         V               |
  * ||L2 dev|....|lb |bufs |lb |bufs |lb |bufs |lb |bufs |lb |---(empty)---|
  * ||   hdr|      ^         /^       /^        /         /                |
  * |+------+  ...--\-------/  \-----/--\------/         /                 |
  * |                \--------------/    \--------------/                  |
  * +======================================================================+
  *
  * As can be seen on the diagram, rather than using a simple linked list,
  * we use a pair of linked lists with alternating elements. This is a
  * performance enhancement due to the fact that we only find out the
  * address of the next log block access once the current block has been
  * completely read in. Obviously, this hurts performance, because we'd be
  * keeping the device's I/O queue at only a 1 operation deep, thus
  * incurring a large amount of I/O round-trip latency. Having two lists
  * allows us to fetch two log blocks ahead of where we are currently
  * rebuilding L2ARC buffers.
  *
  * On-device data structures:
  *
  * L2ARC device header:	l2arc_dev_hdr_phys_t
  * L2ARC log block:	l2arc_log_blk_phys_t
  *
  * L2ARC reconstruction:
  *
  * When writing data, we simply write in the standard rotary fashion,
  * evicting buffers as we go and simply writing new data over them (writing
  * a new log block every now and then). This obviously means that once we
  * loop around the end of the device, we will start cutting into an already
  * committed log block (and its referenced data buffers), like so:
  *
  *    current write head__       __old tail
  *                        \     /
  *                        V    V
  * <--|bufs |lb |bufs |lb |    |bufs |lb |bufs |lb |-->
  *                         ^    ^^^^^^^^^___________________________________
  *                         |                                                \
  *                   <<nextwrite>> may overwrite this blk and/or its bufs --'
  *
  * When importing the pool, we detect this situation and use it to stop
  * our scanning process (see l2arc_rebuild).
  *
  * There is one significant caveat to consider when rebuilding ARC contents
  * from an L2ARC device: what about invalidated buffers? Given the above
  * construction, we cannot update blocks which we've already written to amend
  * them to remove buffers which were invalidated. Thus, during reconstruction,
  * we might be populating the cache with buffers for data that's not on the
  * main pool anymore, or may have been overwritten!
  *
  * As it turns out, this isn't a problem. Every arc_read request includes
  * both the DVA and, crucially, the birth TXG of the BP the caller is
  * looking for. So even if the cache were populated by completely rotten
  * blocks for data that had been long deleted and/or overwritten, we'll
  * never actually return bad data from the cache, since the DVA with the
  * birth TXG uniquely identify a block in space and time - once created,
  * a block is immutable on disk. The worst thing we have done is wasted
  * some time and memory at l2arc rebuild to reconstruct outdated ARC
  * entries that will get dropped from the l2arc as it is being updated
  * with new blocks.
  *
  * L2ARC buffers that have been evicted by l2arc_evict() ahead of the write
  * hand are not restored. This is done by saving the offset (in bytes)
  * l2arc_evict() has evicted to in the L2ARC device header and taking it
  * into account when restoring buffers.
  */
 
 static boolean_t
 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
 {
 	/*
 	 * A buffer is *not* eligible for the L2ARC if it:
 	 * 1. belongs to a different spa.
 	 * 2. is already cached on the L2ARC.
 	 * 3. has an I/O in progress (it may be an incomplete read).
 	 * 4. is flagged not eligible (zfs property).
 	 */
 	if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
 	    HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
 		return (B_FALSE);
 
 	return (B_TRUE);
 }
 
 static uint64_t
 l2arc_write_size(l2arc_dev_t *dev)
 {
 	uint64_t size;
 
 	/*
 	 * Make sure our globals have meaningful values in case the user
 	 * altered them.
 	 */
 	size = l2arc_write_max;
 	if (size == 0) {
 		cmn_err(CE_NOTE, "l2arc_write_max must be greater than zero, "
 		    "resetting it to the default (%d)", L2ARC_WRITE_SIZE);
 		size = l2arc_write_max = L2ARC_WRITE_SIZE;
 	}
 
 	if (arc_warm == B_FALSE)
 		size += l2arc_write_boost;
 
 	/* We need to add in the worst case scenario of log block overhead. */
 	size += l2arc_log_blk_overhead(size, dev);
 	if (dev->l2ad_vdev->vdev_has_trim && l2arc_trim_ahead > 0) {
 		/*
 		 * Trim ahead of the write size 64MB or (l2arc_trim_ahead/100)
 		 * times the writesize, whichever is greater.
 		 */
 		size += MAX(64 * 1024 * 1024,
 		    (size * l2arc_trim_ahead) / 100);
 	}
 
 	/*
 	 * Make sure the write size does not exceed the size of the cache
 	 * device. This is important in l2arc_evict(), otherwise infinite
 	 * iteration can occur.
 	 */
 	size = MIN(size, (dev->l2ad_end - dev->l2ad_start) / 4);
 
 	size = P2ROUNDUP(size, 1ULL << dev->l2ad_vdev->vdev_ashift);
 
 	return (size);
 
 }
 
 static clock_t
 l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
 {
 	clock_t interval, next, now;
 
 	/*
 	 * If the ARC lists are busy, increase our write rate; if the
 	 * lists are stale, idle back.  This is achieved by checking
 	 * how much we previously wrote - if it was more than half of
 	 * what we wanted, schedule the next write much sooner.
 	 */
 	if (l2arc_feed_again && wrote > (wanted / 2))
 		interval = (hz * l2arc_feed_min_ms) / 1000;
 	else
 		interval = hz * l2arc_feed_secs;
 
 	now = ddi_get_lbolt();
 	next = MAX(now, MIN(now + interval, began + interval));
 
 	return (next);
 }
 
 /*
  * Cycle through L2ARC devices.  This is how L2ARC load balances.
  * If a device is returned, this also returns holding the spa config lock.
  */
 static l2arc_dev_t *
 l2arc_dev_get_next(void)
 {
 	l2arc_dev_t *first, *next = NULL;
 
 	/*
 	 * Lock out the removal of spas (spa_namespace_lock), then removal
 	 * of cache devices (l2arc_dev_mtx).  Once a device has been selected,
 	 * both locks will be dropped and a spa config lock held instead.
 	 */
 	mutex_enter(&spa_namespace_lock);
 	mutex_enter(&l2arc_dev_mtx);
 
 	/* if there are no vdevs, there is nothing to do */
 	if (l2arc_ndev == 0)
 		goto out;
 
 	first = NULL;
 	next = l2arc_dev_last;
 	do {
 		/* loop around the list looking for a non-faulted vdev */
 		if (next == NULL) {
 			next = list_head(l2arc_dev_list);
 		} else {
 			next = list_next(l2arc_dev_list, next);
 			if (next == NULL)
 				next = list_head(l2arc_dev_list);
 		}
 
 		/* if we have come back to the start, bail out */
 		if (first == NULL)
 			first = next;
 		else if (next == first)
 			break;
 
 		ASSERT3P(next, !=, NULL);
 	} while (vdev_is_dead(next->l2ad_vdev) || next->l2ad_rebuild ||
 	    next->l2ad_trim_all);
 
 	/* if we were unable to find any usable vdevs, return NULL */
 	if (vdev_is_dead(next->l2ad_vdev) || next->l2ad_rebuild ||
 	    next->l2ad_trim_all)
 		next = NULL;
 
 	l2arc_dev_last = next;
 
 out:
 	mutex_exit(&l2arc_dev_mtx);
 
 	/*
 	 * Grab the config lock to prevent the 'next' device from being
 	 * removed while we are writing to it.
 	 */
 	if (next != NULL)
 		spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
 	mutex_exit(&spa_namespace_lock);
 
 	return (next);
 }
 
 /*
  * Free buffers that were tagged for destruction.
  */
 static void
 l2arc_do_free_on_write(void)
 {
 	l2arc_data_free_t *df;
 
 	mutex_enter(&l2arc_free_on_write_mtx);
 	while ((df = list_remove_head(l2arc_free_on_write)) != NULL) {
 		ASSERT3P(df->l2df_abd, !=, NULL);
 		abd_free(df->l2df_abd);
 		kmem_free(df, sizeof (l2arc_data_free_t));
 	}
 	mutex_exit(&l2arc_free_on_write_mtx);
 }
 
 /*
  * A write to a cache device has completed.  Update all headers to allow
  * reads from these buffers to begin.
  */
 static void
 l2arc_write_done(zio_t *zio)
 {
 	l2arc_write_callback_t	*cb;
 	l2arc_lb_abd_buf_t	*abd_buf;
 	l2arc_lb_ptr_buf_t	*lb_ptr_buf;
 	l2arc_dev_t		*dev;
 	l2arc_dev_hdr_phys_t	*l2dhdr;
 	list_t			*buflist;
 	arc_buf_hdr_t		*head, *hdr, *hdr_prev;
 	kmutex_t		*hash_lock;
 	int64_t			bytes_dropped = 0;
 
 	cb = zio->io_private;
 	ASSERT3P(cb, !=, NULL);
 	dev = cb->l2wcb_dev;
 	l2dhdr = dev->l2ad_dev_hdr;
 	ASSERT3P(dev, !=, NULL);
 	head = cb->l2wcb_head;
 	ASSERT3P(head, !=, NULL);
 	buflist = &dev->l2ad_buflist;
 	ASSERT3P(buflist, !=, NULL);
 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
 	    l2arc_write_callback_t *, cb);
 
 	/*
 	 * All writes completed, or an error was hit.
 	 */
 top:
 	mutex_enter(&dev->l2ad_mtx);
 	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
 		hdr_prev = list_prev(buflist, hdr);
 
 		hash_lock = HDR_LOCK(hdr);
 
 		/*
 		 * We cannot use mutex_enter or else we can deadlock
 		 * with l2arc_write_buffers (due to swapping the order
 		 * the hash lock and l2ad_mtx are taken).
 		 */
 		if (!mutex_tryenter(hash_lock)) {
 			/*
 			 * Missed the hash lock. We must retry so we
 			 * don't leave the ARC_FLAG_L2_WRITING bit set.
 			 */
 			ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);
 
 			/*
 			 * We don't want to rescan the headers we've
 			 * already marked as having been written out, so
 			 * we reinsert the head node so we can pick up
 			 * where we left off.
 			 */
 			list_remove(buflist, head);
 			list_insert_after(buflist, hdr, head);
 
 			mutex_exit(&dev->l2ad_mtx);
 
 			/*
 			 * We wait for the hash lock to become available
 			 * to try and prevent busy waiting, and increase
 			 * the chance we'll be able to acquire the lock
 			 * the next time around.
 			 */
 			mutex_enter(hash_lock);
 			mutex_exit(hash_lock);
 			goto top;
 		}
 
 		/*
 		 * We could not have been moved into the arc_l2c_only
 		 * state while in-flight due to our ARC_FLAG_L2_WRITING
 		 * bit being set. Let's just ensure that's being enforced.
 		 */
 		ASSERT(HDR_HAS_L1HDR(hdr));
 
 		/*
 		 * Skipped - drop L2ARC entry and mark the header as no
 		 * longer L2 eligibile.
 		 */
 		if (zio->io_error != 0) {
 			/*
 			 * Error - drop L2ARC entry.
 			 */
 			list_remove(buflist, hdr);
 			arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
 
 			uint64_t psize = HDR_GET_PSIZE(hdr);
 			l2arc_hdr_arcstats_decrement(hdr);
 
 			bytes_dropped +=
 			    vdev_psize_to_asize(dev->l2ad_vdev, psize);
 			(void) zfs_refcount_remove_many(&dev->l2ad_alloc,
 			    arc_hdr_size(hdr), hdr);
 		}
 
 		/*
 		 * Allow ARC to begin reads and ghost list evictions to
 		 * this L2ARC entry.
 		 */
 		arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING);
 
 		mutex_exit(hash_lock);
 	}
 
 	/*
 	 * Free the allocated abd buffers for writing the log blocks.
 	 * If the zio failed reclaim the allocated space and remove the
 	 * pointers to these log blocks from the log block pointer list
 	 * of the L2ARC device.
 	 */
 	while ((abd_buf = list_remove_tail(&cb->l2wcb_abd_list)) != NULL) {
 		abd_free(abd_buf->abd);
 		zio_buf_free(abd_buf, sizeof (*abd_buf));
 		if (zio->io_error != 0) {
 			lb_ptr_buf = list_remove_head(&dev->l2ad_lbptr_list);
 			/*
 			 * L2BLK_GET_PSIZE returns aligned size for log
 			 * blocks.
 			 */
 			uint64_t asize =
 			    L2BLK_GET_PSIZE((lb_ptr_buf->lb_ptr)->lbp_prop);
 			bytes_dropped += asize;
 			ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
 			ARCSTAT_BUMPDOWN(arcstat_l2_log_blk_count);
 			zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
 			    lb_ptr_buf);
 			zfs_refcount_remove(&dev->l2ad_lb_count, lb_ptr_buf);
 			kmem_free(lb_ptr_buf->lb_ptr,
 			    sizeof (l2arc_log_blkptr_t));
 			kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t));
 		}
 	}
 	list_destroy(&cb->l2wcb_abd_list);
 
 	if (zio->io_error != 0) {
 		ARCSTAT_BUMP(arcstat_l2_writes_error);
 
 		/*
 		 * Restore the lbps array in the header to its previous state.
 		 * If the list of log block pointers is empty, zero out the
 		 * log block pointers in the device header.
 		 */
 		lb_ptr_buf = list_head(&dev->l2ad_lbptr_list);
 		for (int i = 0; i < 2; i++) {
 			if (lb_ptr_buf == NULL) {
 				/*
 				 * If the list is empty zero out the device
 				 * header. Otherwise zero out the second log
 				 * block pointer in the header.
 				 */
 				if (i == 0) {
 					memset(l2dhdr, 0,
 					    dev->l2ad_dev_hdr_asize);
 				} else {
 					memset(&l2dhdr->dh_start_lbps[i], 0,
 					    sizeof (l2arc_log_blkptr_t));
 				}
 				break;
 			}
 			memcpy(&l2dhdr->dh_start_lbps[i], lb_ptr_buf->lb_ptr,
 			    sizeof (l2arc_log_blkptr_t));
 			lb_ptr_buf = list_next(&dev->l2ad_lbptr_list,
 			    lb_ptr_buf);
 		}
 	}
 
 	ARCSTAT_BUMP(arcstat_l2_writes_done);
 	list_remove(buflist, head);
 	ASSERT(!HDR_HAS_L1HDR(head));
 	kmem_cache_free(hdr_l2only_cache, head);
 	mutex_exit(&dev->l2ad_mtx);
 
 	ASSERT(dev->l2ad_vdev != NULL);
 	vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
 
 	l2arc_do_free_on_write();
 
 	kmem_free(cb, sizeof (l2arc_write_callback_t));
 }
 
 static int
 l2arc_untransform(zio_t *zio, l2arc_read_callback_t *cb)
 {
 	int ret;
 	spa_t *spa = zio->io_spa;
 	arc_buf_hdr_t *hdr = cb->l2rcb_hdr;
 	blkptr_t *bp = zio->io_bp;
 	uint8_t salt[ZIO_DATA_SALT_LEN];
 	uint8_t iv[ZIO_DATA_IV_LEN];
 	uint8_t mac[ZIO_DATA_MAC_LEN];
 	boolean_t no_crypt = B_FALSE;
 
 	/*
 	 * ZIL data is never be written to the L2ARC, so we don't need
 	 * special handling for its unique MAC storage.
 	 */
 	ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_INTENT_LOG);
 	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
 	ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
 
 	/*
 	 * If the data was encrypted, decrypt it now. Note that
 	 * we must check the bp here and not the hdr, since the
 	 * hdr does not have its encryption parameters updated
 	 * until arc_read_done().
 	 */
 	if (BP_IS_ENCRYPTED(bp)) {
 		abd_t *eabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
 		    ARC_HDR_USE_RESERVE);
 
 		zio_crypt_decode_params_bp(bp, salt, iv);
 		zio_crypt_decode_mac_bp(bp, mac);
 
 		ret = spa_do_crypt_abd(B_FALSE, spa, &cb->l2rcb_zb,
 		    BP_GET_TYPE(bp), BP_GET_DEDUP(bp), BP_SHOULD_BYTESWAP(bp),
 		    salt, iv, mac, HDR_GET_PSIZE(hdr), eabd,
 		    hdr->b_l1hdr.b_pabd, &no_crypt);
 		if (ret != 0) {
 			arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
 			goto error;
 		}
 
 		/*
 		 * If we actually performed decryption, replace b_pabd
 		 * with the decrypted data. Otherwise we can just throw
 		 * our decryption buffer away.
 		 */
 		if (!no_crypt) {
 			arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
 			    arc_hdr_size(hdr), hdr);
 			hdr->b_l1hdr.b_pabd = eabd;
 			zio->io_abd = eabd;
 		} else {
 			arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
 		}
 	}
 
 	/*
 	 * If the L2ARC block was compressed, but ARC compression
 	 * is disabled we decompress the data into a new buffer and
 	 * replace the existing data.
 	 */
 	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
 	    !HDR_COMPRESSION_ENABLED(hdr)) {
 		abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
 		    ARC_HDR_USE_RESERVE);
 		void *tmp = abd_borrow_buf(cabd, arc_hdr_size(hdr));
 
 		ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
 		    hdr->b_l1hdr.b_pabd, tmp, HDR_GET_PSIZE(hdr),
 		    HDR_GET_LSIZE(hdr), &hdr->b_complevel);
 		if (ret != 0) {
 			abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
 			arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);
 			goto error;
 		}
 
 		abd_return_buf_copy(cabd, tmp, arc_hdr_size(hdr));
 		arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
 		    arc_hdr_size(hdr), hdr);
 		hdr->b_l1hdr.b_pabd = cabd;
 		zio->io_abd = cabd;
 		zio->io_size = HDR_GET_LSIZE(hdr);
 	}
 
 	return (0);
 
 error:
 	return (ret);
 }
 
 
 /*
  * A read to a cache device completed.  Validate buffer contents before
  * handing over to the regular ARC routines.
  */
 static void
 l2arc_read_done(zio_t *zio)
 {
 	int tfm_error = 0;
 	l2arc_read_callback_t *cb = zio->io_private;
 	arc_buf_hdr_t *hdr;
 	kmutex_t *hash_lock;
 	boolean_t valid_cksum;
 	boolean_t using_rdata = (BP_IS_ENCRYPTED(&cb->l2rcb_bp) &&
 	    (cb->l2rcb_flags & ZIO_FLAG_RAW_ENCRYPT));
 
 	ASSERT3P(zio->io_vd, !=, NULL);
 	ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
 
 	spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
 
 	ASSERT3P(cb, !=, NULL);
 	hdr = cb->l2rcb_hdr;
 	ASSERT3P(hdr, !=, NULL);
 
 	hash_lock = HDR_LOCK(hdr);
 	mutex_enter(hash_lock);
 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
 
 	/*
 	 * If the data was read into a temporary buffer,
 	 * move it and free the buffer.
 	 */
 	if (cb->l2rcb_abd != NULL) {
 		ASSERT3U(arc_hdr_size(hdr), <, zio->io_size);
 		if (zio->io_error == 0) {
 			if (using_rdata) {
 				abd_copy(hdr->b_crypt_hdr.b_rabd,
 				    cb->l2rcb_abd, arc_hdr_size(hdr));
 			} else {
 				abd_copy(hdr->b_l1hdr.b_pabd,
 				    cb->l2rcb_abd, arc_hdr_size(hdr));
 			}
 		}
 
 		/*
 		 * The following must be done regardless of whether
 		 * there was an error:
 		 * - free the temporary buffer
 		 * - point zio to the real ARC buffer
 		 * - set zio size accordingly
 		 * These are required because zio is either re-used for
 		 * an I/O of the block in the case of the error
 		 * or the zio is passed to arc_read_done() and it
 		 * needs real data.
 		 */
 		abd_free(cb->l2rcb_abd);
 		zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);
 
 		if (using_rdata) {
 			ASSERT(HDR_HAS_RABD(hdr));
 			zio->io_abd = zio->io_orig_abd =
 			    hdr->b_crypt_hdr.b_rabd;
 		} else {
 			ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
 			zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
 		}
 	}
 
 	ASSERT3P(zio->io_abd, !=, NULL);
 
 	/*
 	 * Check this survived the L2ARC journey.
 	 */
 	ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd ||
 	    (HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd));
 	zio->io_bp_copy = cb->l2rcb_bp;	/* XXX fix in L2ARC 2.0	*/
 	zio->io_bp = &zio->io_bp_copy;	/* XXX fix in L2ARC 2.0	*/
 	zio->io_prop.zp_complevel = hdr->b_complevel;
 
 	valid_cksum = arc_cksum_is_equal(hdr, zio);
 
 	/*
 	 * b_rabd will always match the data as it exists on disk if it is
 	 * being used. Therefore if we are reading into b_rabd we do not
 	 * attempt to untransform the data.
 	 */
 	if (valid_cksum && !using_rdata)
 		tfm_error = l2arc_untransform(zio, cb);
 
 	if (valid_cksum && tfm_error == 0 && zio->io_error == 0 &&
 	    !HDR_L2_EVICTED(hdr)) {
 		mutex_exit(hash_lock);
 		zio->io_private = hdr;
 		arc_read_done(zio);
 	} else {
 		/*
 		 * Buffer didn't survive caching.  Increment stats and
 		 * reissue to the original storage device.
 		 */
 		if (zio->io_error != 0) {
 			ARCSTAT_BUMP(arcstat_l2_io_error);
 		} else {
 			zio->io_error = SET_ERROR(EIO);
 		}
 		if (!valid_cksum || tfm_error != 0)
 			ARCSTAT_BUMP(arcstat_l2_cksum_bad);
 
 		/*
 		 * If there's no waiter, issue an async i/o to the primary
 		 * storage now.  If there *is* a waiter, the caller must
 		 * issue the i/o in a context where it's OK to block.
 		 */
 		if (zio->io_waiter == NULL) {
 			zio_t *pio = zio_unique_parent(zio);
 			void *abd = (using_rdata) ?
 			    hdr->b_crypt_hdr.b_rabd : hdr->b_l1hdr.b_pabd;
 
 			ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
 
 			zio = zio_read(pio, zio->io_spa, zio->io_bp,
 			    abd, zio->io_size, arc_read_done,
 			    hdr, zio->io_priority, cb->l2rcb_flags,
 			    &cb->l2rcb_zb);
 
 			/*
 			 * Original ZIO will be freed, so we need to update
 			 * ARC header with the new ZIO pointer to be used
 			 * by zio_change_priority() in arc_read().
 			 */
 			for (struct arc_callback *acb = hdr->b_l1hdr.b_acb;
 			    acb != NULL; acb = acb->acb_next)
 				acb->acb_zio_head = zio;
 
 			mutex_exit(hash_lock);
 			zio_nowait(zio);
 		} else {
 			mutex_exit(hash_lock);
 		}
 	}
 
 	kmem_free(cb, sizeof (l2arc_read_callback_t));
 }
 
 /*
  * This is the list priority from which the L2ARC will search for pages to
  * cache.  This is used within loops (0..3) to cycle through lists in the
  * desired order.  This order can have a significant effect on cache
  * performance.
  *
  * Currently the metadata lists are hit first, MFU then MRU, followed by
  * the data lists.  This function returns a locked list, and also returns
  * the lock pointer.
  */
 static multilist_sublist_t *
 l2arc_sublist_lock(int list_num)
 {
 	multilist_t *ml = NULL;
 	unsigned int idx;
 
 	ASSERT(list_num >= 0 && list_num < L2ARC_FEED_TYPES);
 
 	switch (list_num) {
 	case 0:
 		ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
 		break;
 	case 1:
 		ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
 		break;
 	case 2:
 		ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
 		break;
 	case 3:
 		ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
 		break;
 	default:
 		return (NULL);
 	}
 
 	/*
 	 * Return a randomly-selected sublist. This is acceptable
 	 * because the caller feeds only a little bit of data for each
 	 * call (8MB). Subsequent calls will result in different
 	 * sublists being selected.
 	 */
 	idx = multilist_get_random_index(ml);
 	return (multilist_sublist_lock_idx(ml, idx));
 }
 
 /*
  * Calculates the maximum overhead of L2ARC metadata log blocks for a given
  * L2ARC write size. l2arc_evict and l2arc_write_size need to include this
  * overhead in processing to make sure there is enough headroom available
  * when writing buffers.
  */
 static inline uint64_t
 l2arc_log_blk_overhead(uint64_t write_sz, l2arc_dev_t *dev)
 {
 	if (dev->l2ad_log_entries == 0) {
 		return (0);
 	} else {
 		uint64_t log_entries = write_sz >> SPA_MINBLOCKSHIFT;
 
 		uint64_t log_blocks = (log_entries +
 		    dev->l2ad_log_entries - 1) /
 		    dev->l2ad_log_entries;
 
 		return (vdev_psize_to_asize(dev->l2ad_vdev,
 		    sizeof (l2arc_log_blk_phys_t)) * log_blocks);
 	}
 }
 
 /*
  * Evict buffers from the device write hand to the distance specified in
  * bytes. This distance may span populated buffers, it may span nothing.
  * This is clearing a region on the L2ARC device ready for writing.
  * If the 'all' boolean is set, every buffer is evicted.
  */
 static void
 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 {
 	list_t *buflist;
 	arc_buf_hdr_t *hdr, *hdr_prev;
 	kmutex_t *hash_lock;
 	uint64_t taddr;
 	l2arc_lb_ptr_buf_t *lb_ptr_buf, *lb_ptr_buf_prev;
 	vdev_t *vd = dev->l2ad_vdev;
 	boolean_t rerun;
 
 	buflist = &dev->l2ad_buflist;
 
 top:
 	rerun = B_FALSE;
 	if (dev->l2ad_hand + distance > dev->l2ad_end) {
 		/*
 		 * When there is no space to accommodate upcoming writes,
 		 * evict to the end. Then bump the write and evict hands
 		 * to the start and iterate. This iteration does not
 		 * happen indefinitely as we make sure in
 		 * l2arc_write_size() that when the write hand is reset,
 		 * the write size does not exceed the end of the device.
 		 */
 		rerun = B_TRUE;
 		taddr = dev->l2ad_end;
 	} else {
 		taddr = dev->l2ad_hand + distance;
 	}
 	DTRACE_PROBE4(l2arc__evict, l2arc_dev_t *, dev, list_t *, buflist,
 	    uint64_t, taddr, boolean_t, all);
 
 	if (!all) {
 		/*
 		 * This check has to be placed after deciding whether to
 		 * iterate (rerun).
 		 */
 		if (dev->l2ad_first) {
 			/*
 			 * This is the first sweep through the device. There is
 			 * nothing to evict. We have already trimmmed the
 			 * whole device.
 			 */
 			goto out;
 		} else {
 			/*
 			 * Trim the space to be evicted.
 			 */
 			if (vd->vdev_has_trim && dev->l2ad_evict < taddr &&
 			    l2arc_trim_ahead > 0) {
 				/*
 				 * We have to drop the spa_config lock because
 				 * vdev_trim_range() will acquire it.
 				 * l2ad_evict already accounts for the label
 				 * size. To prevent vdev_trim_ranges() from
 				 * adding it again, we subtract it from
 				 * l2ad_evict.
 				 */
 				spa_config_exit(dev->l2ad_spa, SCL_L2ARC, dev);
 				vdev_trim_simple(vd,
 				    dev->l2ad_evict - VDEV_LABEL_START_SIZE,
 				    taddr - dev->l2ad_evict);
 				spa_config_enter(dev->l2ad_spa, SCL_L2ARC, dev,
 				    RW_READER);
 			}
 
 			/*
 			 * When rebuilding L2ARC we retrieve the evict hand
 			 * from the header of the device. Of note, l2arc_evict()
 			 * does not actually delete buffers from the cache
 			 * device, but trimming may do so depending on the
 			 * hardware implementation. Thus keeping track of the
 			 * evict hand is useful.
 			 */
 			dev->l2ad_evict = MAX(dev->l2ad_evict, taddr);
 		}
 	}
 
 retry:
 	mutex_enter(&dev->l2ad_mtx);
 	/*
 	 * We have to account for evicted log blocks. Run vdev_space_update()
 	 * on log blocks whose offset (in bytes) is before the evicted offset
 	 * (in bytes) by searching in the list of pointers to log blocks
 	 * present in the L2ARC device.
 	 */
 	for (lb_ptr_buf = list_tail(&dev->l2ad_lbptr_list); lb_ptr_buf;
 	    lb_ptr_buf = lb_ptr_buf_prev) {
 
 		lb_ptr_buf_prev = list_prev(&dev->l2ad_lbptr_list, lb_ptr_buf);
 
 		/* L2BLK_GET_PSIZE returns aligned size for log blocks */
 		uint64_t asize = L2BLK_GET_PSIZE(
 		    (lb_ptr_buf->lb_ptr)->lbp_prop);
 
 		/*
 		 * We don't worry about log blocks left behind (ie
 		 * lbp_payload_start < l2ad_hand) because l2arc_write_buffers()
 		 * will never write more than l2arc_evict() evicts.
 		 */
 		if (!all && l2arc_log_blkptr_valid(dev, lb_ptr_buf->lb_ptr)) {
 			break;
 		} else {
 			vdev_space_update(vd, -asize, 0, 0);
 			ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
 			ARCSTAT_BUMPDOWN(arcstat_l2_log_blk_count);
 			zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
 			    lb_ptr_buf);
 			zfs_refcount_remove(&dev->l2ad_lb_count, lb_ptr_buf);
 			list_remove(&dev->l2ad_lbptr_list, lb_ptr_buf);
 			kmem_free(lb_ptr_buf->lb_ptr,
 			    sizeof (l2arc_log_blkptr_t));
 			kmem_free(lb_ptr_buf, sizeof (l2arc_lb_ptr_buf_t));
 		}
 	}
 
 	for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
 		hdr_prev = list_prev(buflist, hdr);
 
 		ASSERT(!HDR_EMPTY(hdr));
 		hash_lock = HDR_LOCK(hdr);
 
 		/*
 		 * We cannot use mutex_enter or else we can deadlock
 		 * with l2arc_write_buffers (due to swapping the order
 		 * the hash lock and l2ad_mtx are taken).
 		 */
 		if (!mutex_tryenter(hash_lock)) {
 			/*
 			 * Missed the hash lock.  Retry.
 			 */
 			ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
 			mutex_exit(&dev->l2ad_mtx);
 			mutex_enter(hash_lock);
 			mutex_exit(hash_lock);
 			goto retry;
 		}
 
 		/*
 		 * A header can't be on this list if it doesn't have L2 header.
 		 */
 		ASSERT(HDR_HAS_L2HDR(hdr));
 
 		/* Ensure this header has finished being written. */
 		ASSERT(!HDR_L2_WRITING(hdr));
 		ASSERT(!HDR_L2_WRITE_HEAD(hdr));
 
 		if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict ||
 		    hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
 			/*
 			 * We've evicted to the target address,
 			 * or the end of the device.
 			 */
 			mutex_exit(hash_lock);
 			break;
 		}
 
 		if (!HDR_HAS_L1HDR(hdr)) {
 			ASSERT(!HDR_L2_READING(hdr));
 			/*
 			 * This doesn't exist in the ARC.  Destroy.
 			 * arc_hdr_destroy() will call list_remove()
 			 * and decrement arcstat_l2_lsize.
 			 */
 			arc_change_state(arc_anon, hdr);
 			arc_hdr_destroy(hdr);
 		} else {
 			ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
 			ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
 			/*
 			 * Invalidate issued or about to be issued
 			 * reads, since we may be about to write
 			 * over this location.
 			 */
 			if (HDR_L2_READING(hdr)) {
 				ARCSTAT_BUMP(arcstat_l2_evict_reading);
 				arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED);
 			}
 
 			arc_hdr_l2hdr_destroy(hdr);
 		}
 		mutex_exit(hash_lock);
 	}
 	mutex_exit(&dev->l2ad_mtx);
 
 out:
 	/*
 	 * We need to check if we evict all buffers, otherwise we may iterate
 	 * unnecessarily.
 	 */
 	if (!all && rerun) {
 		/*
 		 * Bump device hand to the device start if it is approaching the
 		 * end. l2arc_evict() has already evicted ahead for this case.
 		 */
 		dev->l2ad_hand = dev->l2ad_start;
 		dev->l2ad_evict = dev->l2ad_start;
 		dev->l2ad_first = B_FALSE;
 		goto top;
 	}
 
 	if (!all) {
 		/*
 		 * In case of cache device removal (all) the following
 		 * assertions may be violated without functional consequences
 		 * as the device is about to be removed.
 		 */
 		ASSERT3U(dev->l2ad_hand + distance, <=, dev->l2ad_end);
 		if (!dev->l2ad_first)
 			ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
 	}
 }
 
 /*
  * Handle any abd transforms that might be required for writing to the L2ARC.
  * If successful, this function will always return an abd with the data
  * transformed as it is on disk in a new abd of asize bytes.
  */
 static int
 l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
     abd_t **abd_out)
 {
 	int ret;
 	abd_t *cabd = NULL, *eabd = NULL, *to_write = hdr->b_l1hdr.b_pabd;
 	enum zio_compress compress = HDR_GET_COMPRESS(hdr);
 	uint64_t psize = HDR_GET_PSIZE(hdr);
 	uint64_t size = arc_hdr_size(hdr);
 	boolean_t ismd = HDR_ISTYPE_METADATA(hdr);
 	boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
 	dsl_crypto_key_t *dck = NULL;
 	uint8_t mac[ZIO_DATA_MAC_LEN] = { 0 };
 	boolean_t no_crypt = B_FALSE;
 
 	ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
 	    !HDR_COMPRESSION_ENABLED(hdr)) ||
 	    HDR_ENCRYPTED(hdr) || HDR_SHARED_DATA(hdr) || psize != asize);
 	ASSERT3U(psize, <=, asize);
 
 	/*
 	 * If this data simply needs its own buffer, we simply allocate it
 	 * and copy the data. This may be done to eliminate a dependency on a
 	 * shared buffer or to reallocate the buffer to match asize.
 	 */
 	if (HDR_HAS_RABD(hdr)) {
 		ASSERT3U(asize, >, psize);
 		to_write = abd_alloc_for_io(asize, ismd);
 		abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, psize);
 		abd_zero_off(to_write, psize, asize - psize);
 		goto out;
 	}
 
 	if ((compress == ZIO_COMPRESS_OFF || HDR_COMPRESSION_ENABLED(hdr)) &&
 	    !HDR_ENCRYPTED(hdr)) {
 		ASSERT3U(size, ==, psize);
 		to_write = abd_alloc_for_io(asize, ismd);
 		abd_copy(to_write, hdr->b_l1hdr.b_pabd, size);
 		if (asize > size)
 			abd_zero_off(to_write, size, asize - size);
 		goto out;
 	}
 
 	if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) {
 		size_t bufsize = MAX(size, asize);
 		void *buf = zio_buf_alloc(bufsize);
 		uint64_t csize = zio_compress_data(compress, to_write, &buf,
 		    size, hdr->b_complevel);
 		if (csize > psize) {
 			/*
 			 * We can't re-compress the block into the original
 			 * psize.  Even if it fits into asize, it does not
 			 * matter, since checksum will never match on read.
 			 */
 			zio_buf_free(buf, bufsize);
 			return (SET_ERROR(EIO));
 		}
 		if (asize > csize)
 			memset((char *)buf + csize, 0, asize - csize);
 		to_write = cabd = abd_get_from_buf(buf, bufsize);
 		abd_take_ownership_of_buf(cabd, B_TRUE);
 	}
 
 	if (HDR_ENCRYPTED(hdr)) {
 		eabd = abd_alloc_for_io(asize, ismd);
 
 		/*
 		 * If the dataset was disowned before the buffer
 		 * made it to this point, the key to re-encrypt
 		 * it won't be available. In this case we simply
 		 * won't write the buffer to the L2ARC.
 		 */
 		ret = spa_keystore_lookup_key(spa, hdr->b_crypt_hdr.b_dsobj,
 		    FTAG, &dck);
 		if (ret != 0)
 			goto error;
 
 		ret = zio_do_crypt_abd(B_TRUE, &dck->dck_key,
 		    hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt,
 		    hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd,
 		    &no_crypt);
 		if (ret != 0)
 			goto error;
 
 		if (no_crypt)
 			abd_copy(eabd, to_write, psize);
 
 		if (psize != asize)
 			abd_zero_off(eabd, psize, asize - psize);
 
 		/* assert that the MAC we got here matches the one we saved */
 		ASSERT0(memcmp(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN));
 		spa_keystore_dsl_key_rele(spa, dck, FTAG);
 
 		if (to_write == cabd)
 			abd_free(cabd);
 
 		to_write = eabd;
 	}
 
 out:
 	ASSERT3P(to_write, !=, hdr->b_l1hdr.b_pabd);
 	*abd_out = to_write;
 	return (0);
 
 error:
 	if (dck != NULL)
 		spa_keystore_dsl_key_rele(spa, dck, FTAG);
 	if (cabd != NULL)
 		abd_free(cabd);
 	if (eabd != NULL)
 		abd_free(eabd);
 
 	*abd_out = NULL;
 	return (ret);
 }
 
 static void
 l2arc_blk_fetch_done(zio_t *zio)
 {
 	l2arc_read_callback_t *cb;
 
 	cb = zio->io_private;
 	if (cb->l2rcb_abd != NULL)
 		abd_free(cb->l2rcb_abd);
 	kmem_free(cb, sizeof (l2arc_read_callback_t));
 }
 
 /*
  * Find and write ARC buffers to the L2ARC device.
  *
  * An ARC_FLAG_L2_WRITING flag is set so that the L2ARC buffers are not valid
  * for reading until they have completed writing.
  * The headroom_boost is an in-out parameter used to maintain headroom boost
  * state between calls to this function.
  *
  * Returns the number of bytes actually written (which may be smaller than
  * the delta by which the device hand has changed due to alignment and the
  * writing of log blocks).
  */
 static uint64_t
 l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz)
 {
 	arc_buf_hdr_t 		*hdr, *head, *marker;
 	uint64_t 		write_asize, write_psize, headroom;
 	boolean_t		full, from_head = !arc_warm;
 	l2arc_write_callback_t	*cb = NULL;
 	zio_t 			*pio, *wzio;
 	uint64_t 		guid = spa_load_guid(spa);
 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
 
 	ASSERT3P(dev->l2ad_vdev, !=, NULL);
 
 	pio = NULL;
 	write_asize = write_psize = 0;
 	full = B_FALSE;
 	head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
 	arc_hdr_set_flags(head, ARC_FLAG_L2_WRITE_HEAD | ARC_FLAG_HAS_L2HDR);
 	marker = arc_state_alloc_marker();
 
 	/*
 	 * Copy buffers for L2ARC writing.
 	 */
 	for (int pass = 0; pass < L2ARC_FEED_TYPES; pass++) {
 		/*
 		 * pass == 0: MFU meta
 		 * pass == 1: MRU meta
 		 * pass == 2: MFU data
 		 * pass == 3: MRU data
 		 */
 		if (l2arc_mfuonly == 1) {
 			if (pass == 1 || pass == 3)
 				continue;
 		} else if (l2arc_mfuonly > 1) {
 			if (pass == 3)
 				continue;
 		}
 
 		uint64_t passed_sz = 0;
 		headroom = target_sz * l2arc_headroom;
 		if (zfs_compressed_arc_enabled)
 			headroom = (headroom * l2arc_headroom_boost) / 100;
 
 		/*
 		 * Until the ARC is warm and starts to evict, read from the
 		 * head of the ARC lists rather than the tail.
 		 */
 		multilist_sublist_t *mls = l2arc_sublist_lock(pass);
 		ASSERT3P(mls, !=, NULL);
 		if (from_head)
 			hdr = multilist_sublist_head(mls);
 		else
 			hdr = multilist_sublist_tail(mls);
 
 		while (hdr != NULL) {
 			kmutex_t *hash_lock;
 			abd_t *to_write = NULL;
 
 			hash_lock = HDR_LOCK(hdr);
 			if (!mutex_tryenter(hash_lock)) {
 skip:
 				/* Skip this buffer rather than waiting. */
 				if (from_head)
 					hdr = multilist_sublist_next(mls, hdr);
 				else
 					hdr = multilist_sublist_prev(mls, hdr);
 				continue;
 			}
 
 			passed_sz += HDR_GET_LSIZE(hdr);
 			if (l2arc_headroom != 0 && passed_sz > headroom) {
 				/*
 				 * Searched too far.
 				 */
 				mutex_exit(hash_lock);
 				break;
 			}
 
 			if (!l2arc_write_eligible(guid, hdr)) {
 				mutex_exit(hash_lock);
 				goto skip;
 			}
 
 			ASSERT(HDR_HAS_L1HDR(hdr));
 			ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);
 			ASSERT3U(arc_hdr_size(hdr), >, 0);
 			ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
 			    HDR_HAS_RABD(hdr));
 			uint64_t psize = HDR_GET_PSIZE(hdr);
 			uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev,
 			    psize);
 
 			/*
 			 * If the allocated size of this buffer plus the max
 			 * size for the pending log block exceeds the evicted
 			 * target size, terminate writing buffers for this run.
 			 */
 			if (write_asize + asize +
 			    sizeof (l2arc_log_blk_phys_t) > target_sz) {
 				full = B_TRUE;
 				mutex_exit(hash_lock);
 				break;
 			}
 
 			/*
 			 * We should not sleep with sublist lock held or it
 			 * may block ARC eviction.  Insert a marker to save
 			 * the position and drop the lock.
 			 */
 			if (from_head) {
 				multilist_sublist_insert_after(mls, hdr,
 				    marker);
 			} else {
 				multilist_sublist_insert_before(mls, hdr,
 				    marker);
 			}
 			multilist_sublist_unlock(mls);
 
 			/*
 			 * If this header has b_rabd, we can use this since it
 			 * must always match the data exactly as it exists on
 			 * disk. Otherwise, the L2ARC can normally use the
 			 * hdr's data, but if we're sharing data between the
 			 * hdr and one of its bufs, L2ARC needs its own copy of
 			 * the data so that the ZIO below can't race with the
 			 * buf consumer. To ensure that this copy will be
 			 * available for the lifetime of the ZIO and be cleaned
 			 * up afterwards, we add it to the l2arc_free_on_write
 			 * queue. If we need to apply any transforms to the
 			 * data (compression, encryption) we will also need the
 			 * extra buffer.
 			 */
 			if (HDR_HAS_RABD(hdr) && psize == asize) {
 				to_write = hdr->b_crypt_hdr.b_rabd;
 			} else if ((HDR_COMPRESSION_ENABLED(hdr) ||
 			    HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) &&
 			    !HDR_ENCRYPTED(hdr) && !HDR_SHARED_DATA(hdr) &&
 			    psize == asize) {
 				to_write = hdr->b_l1hdr.b_pabd;
 			} else {
 				int ret;
 				arc_buf_contents_t type = arc_buf_type(hdr);
 
 				ret = l2arc_apply_transforms(spa, hdr, asize,
 				    &to_write);
 				if (ret != 0) {
 					arc_hdr_clear_flags(hdr,
 					    ARC_FLAG_L2CACHE);
 					mutex_exit(hash_lock);
 					goto next;
 				}
 
 				l2arc_free_abd_on_write(to_write, asize, type);
 			}
 
 			hdr->b_l2hdr.b_dev = dev;
 			hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
 			hdr->b_l2hdr.b_hits = 0;
 			hdr->b_l2hdr.b_arcs_state =
 			    hdr->b_l1hdr.b_state->arcs_state;
 			mutex_enter(&dev->l2ad_mtx);
 			if (pio == NULL) {
 				/*
 				 * Insert a dummy header on the buflist so
 				 * l2arc_write_done() can find where the
 				 * write buffers begin without searching.
 				 */
 				list_insert_head(&dev->l2ad_buflist, head);
 			}
 			list_insert_head(&dev->l2ad_buflist, hdr);
 			mutex_exit(&dev->l2ad_mtx);
 			arc_hdr_set_flags(hdr, ARC_FLAG_HAS_L2HDR |
 			    ARC_FLAG_L2_WRITING);
 
 			(void) zfs_refcount_add_many(&dev->l2ad_alloc,
 			    arc_hdr_size(hdr), hdr);
 			l2arc_hdr_arcstats_increment(hdr);
 
 			boolean_t commit = l2arc_log_blk_insert(dev, hdr);
 			mutex_exit(hash_lock);
 
 			if (pio == NULL) {
 				cb = kmem_alloc(
 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
 				cb->l2wcb_dev = dev;
 				cb->l2wcb_head = head;
 				list_create(&cb->l2wcb_abd_list,
 				    sizeof (l2arc_lb_abd_buf_t),
 				    offsetof(l2arc_lb_abd_buf_t, node));
 				pio = zio_root(spa, l2arc_write_done, cb,
 				    ZIO_FLAG_CANFAIL);
 			}
 
 			wzio = zio_write_phys(pio, dev->l2ad_vdev,
 			    dev->l2ad_hand, asize, to_write,
 			    ZIO_CHECKSUM_OFF, NULL, hdr,
 			    ZIO_PRIORITY_ASYNC_WRITE,
 			    ZIO_FLAG_CANFAIL, B_FALSE);
 
 			DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
 			    zio_t *, wzio);
 			zio_nowait(wzio);
 
 			write_psize += psize;
 			write_asize += asize;
 			dev->l2ad_hand += asize;
 			vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
 
 			if (commit) {
 				/* l2ad_hand will be adjusted inside. */
 				write_asize +=
 				    l2arc_log_blk_commit(dev, pio, cb);
 			}
 
 next:
 			multilist_sublist_lock(mls);
 			if (from_head)
 				hdr = multilist_sublist_next(mls, marker);
 			else
 				hdr = multilist_sublist_prev(mls, marker);
 			multilist_sublist_remove(mls, marker);
 		}
 
 		multilist_sublist_unlock(mls);
 
 		if (full == B_TRUE)
 			break;
 	}
 
 	arc_state_free_marker(marker);
 
 	/* No buffers selected for writing? */
 	if (pio == NULL) {
 		ASSERT0(write_psize);
 		ASSERT(!HDR_HAS_L1HDR(head));
 		kmem_cache_free(hdr_l2only_cache, head);
 
 		/*
 		 * Although we did not write any buffers l2ad_evict may
 		 * have advanced.
 		 */
 		if (dev->l2ad_evict != l2dhdr->dh_evict)
 			l2arc_dev_hdr_update(dev);
 
 		return (0);
 	}
 
 	if (!dev->l2ad_first)
 		ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
 
 	ASSERT3U(write_asize, <=, target_sz);
 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
 	ARCSTAT_INCR(arcstat_l2_write_bytes, write_psize);
 
 	dev->l2ad_writing = B_TRUE;
 	(void) zio_wait(pio);
 	dev->l2ad_writing = B_FALSE;
 
 	/*
 	 * Update the device header after the zio completes as
 	 * l2arc_write_done() may have updated the memory holding the log block
 	 * pointers in the device header.
 	 */
 	l2arc_dev_hdr_update(dev);
 
 	return (write_asize);
 }
 
 static boolean_t
 l2arc_hdr_limit_reached(void)
 {
 	int64_t s = aggsum_upper_bound(&arc_sums.arcstat_l2_hdr_size);
 
 	return (arc_reclaim_needed() ||
 	    (s > (arc_warm ? arc_c : arc_c_max) * l2arc_meta_percent / 100));
 }
 
 /*
  * This thread feeds the L2ARC at regular intervals.  This is the beating
  * heart of the L2ARC.
  */
 static  __attribute__((noreturn)) void
 l2arc_feed_thread(void *unused)
 {
 	(void) unused;
 	callb_cpr_t cpr;
 	l2arc_dev_t *dev;
 	spa_t *spa;
 	uint64_t size, wrote;
 	clock_t begin, next = ddi_get_lbolt();
 	fstrans_cookie_t cookie;
 
 	CALLB_CPR_INIT(&cpr, &l2arc_feed_thr_lock, callb_generic_cpr, FTAG);
 
 	mutex_enter(&l2arc_feed_thr_lock);
 
 	cookie = spl_fstrans_mark();
 	while (l2arc_thread_exit == 0) {
 		CALLB_CPR_SAFE_BEGIN(&cpr);
 		(void) cv_timedwait_idle(&l2arc_feed_thr_cv,
 		    &l2arc_feed_thr_lock, next);
 		CALLB_CPR_SAFE_END(&cpr, &l2arc_feed_thr_lock);
 		next = ddi_get_lbolt() + hz;
 
 		/*
 		 * Quick check for L2ARC devices.
 		 */
 		mutex_enter(&l2arc_dev_mtx);
 		if (l2arc_ndev == 0) {
 			mutex_exit(&l2arc_dev_mtx);
 			continue;
 		}
 		mutex_exit(&l2arc_dev_mtx);
 		begin = ddi_get_lbolt();
 
 		/*
 		 * This selects the next l2arc device to write to, and in
 		 * doing so the next spa to feed from: dev->l2ad_spa.   This
 		 * will return NULL if there are now no l2arc devices or if
 		 * they are all faulted.
 		 *
 		 * If a device is returned, its spa's config lock is also
 		 * held to prevent device removal.  l2arc_dev_get_next()
 		 * will grab and release l2arc_dev_mtx.
 		 */
 		if ((dev = l2arc_dev_get_next()) == NULL)
 			continue;
 
 		spa = dev->l2ad_spa;
 		ASSERT3P(spa, !=, NULL);
 
 		/*
 		 * If the pool is read-only then force the feed thread to
 		 * sleep a little longer.
 		 */
 		if (!spa_writeable(spa)) {
 			next = ddi_get_lbolt() + 5 * l2arc_feed_secs * hz;
 			spa_config_exit(spa, SCL_L2ARC, dev);
 			continue;
 		}
 
 		/*
 		 * Avoid contributing to memory pressure.
 		 */
 		if (l2arc_hdr_limit_reached()) {
 			ARCSTAT_BUMP(arcstat_l2_abort_lowmem);
 			spa_config_exit(spa, SCL_L2ARC, dev);
 			continue;
 		}
 
 		ARCSTAT_BUMP(arcstat_l2_feeds);
 
 		size = l2arc_write_size(dev);
 
 		/*
 		 * Evict L2ARC buffers that will be overwritten.
 		 */
 		l2arc_evict(dev, size, B_FALSE);
 
 		/*
 		 * Write ARC buffers.
 		 */
 		wrote = l2arc_write_buffers(spa, dev, size);
 
 		/*
 		 * Calculate interval between writes.
 		 */
 		next = l2arc_write_interval(begin, size, wrote);
 		spa_config_exit(spa, SCL_L2ARC, dev);
 	}
 	spl_fstrans_unmark(cookie);
 
 	l2arc_thread_exit = 0;
 	cv_broadcast(&l2arc_feed_thr_cv);
 	CALLB_CPR_EXIT(&cpr);		/* drops l2arc_feed_thr_lock */
 	thread_exit();
 }
 
 boolean_t
 l2arc_vdev_present(vdev_t *vd)
 {
 	return (l2arc_vdev_get(vd) != NULL);
 }
 
 /*
  * Returns the l2arc_dev_t associated with a particular vdev_t or NULL if
  * the vdev_t isn't an L2ARC device.
  */
 l2arc_dev_t *
 l2arc_vdev_get(vdev_t *vd)
 {
 	l2arc_dev_t	*dev;
 
 	mutex_enter(&l2arc_dev_mtx);
 	for (dev = list_head(l2arc_dev_list); dev != NULL;
 	    dev = list_next(l2arc_dev_list, dev)) {
 		if (dev->l2ad_vdev == vd)
 			break;
 	}
 	mutex_exit(&l2arc_dev_mtx);
 
 	return (dev);
 }
 
 static void
 l2arc_rebuild_dev(l2arc_dev_t *dev, boolean_t reopen)
 {
 	l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
 	uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
 	spa_t *spa = dev->l2ad_spa;
 
 	/*
 	 * The L2ARC has to hold at least the payload of one log block for
 	 * them to be restored (persistent L2ARC). The payload of a log block
 	 * depends on the amount of its log entries. We always write log blocks
 	 * with 1022 entries. How many of them are committed or restored depends
 	 * on the size of the L2ARC device. Thus the maximum payload of
 	 * one log block is 1022 * SPA_MAXBLOCKSIZE = 16GB. If the L2ARC device
 	 * is less than that, we reduce the amount of committed and restored
 	 * log entries per block so as to enable persistence.
 	 */
 	if (dev->l2ad_end < l2arc_rebuild_blocks_min_l2size) {
 		dev->l2ad_log_entries = 0;
 	} else {
 		dev->l2ad_log_entries = MIN((dev->l2ad_end -
 		    dev->l2ad_start) >> SPA_MAXBLOCKSHIFT,
 		    L2ARC_LOG_BLK_MAX_ENTRIES);
 	}
 
 	/*
 	 * Read the device header, if an error is returned do not rebuild L2ARC.
 	 */
 	if (l2arc_dev_hdr_read(dev) == 0 && dev->l2ad_log_entries > 0) {
 		/*
 		 * If we are onlining a cache device (vdev_reopen) that was
 		 * still present (l2arc_vdev_present()) and rebuild is enabled,
 		 * we should evict all ARC buffers and pointers to log blocks
 		 * and reclaim their space before restoring its contents to
 		 * L2ARC.
 		 */
 		if (reopen) {
 			if (!l2arc_rebuild_enabled) {
 				return;
 			} else {
 				l2arc_evict(dev, 0, B_TRUE);
 				/* start a new log block */
 				dev->l2ad_log_ent_idx = 0;
 				dev->l2ad_log_blk_payload_asize = 0;
 				dev->l2ad_log_blk_payload_start = 0;
 			}
 		}
 		/*
 		 * Just mark the device as pending for a rebuild. We won't
 		 * be starting a rebuild in line here as it would block pool
 		 * import. Instead spa_load_impl will hand that off to an
 		 * async task which will call l2arc_spa_rebuild_start.
 		 */
 		dev->l2ad_rebuild = B_TRUE;
 	} else if (spa_writeable(spa)) {
 		/*
 		 * In this case TRIM the whole device if l2arc_trim_ahead > 0,
 		 * otherwise create a new header. We zero out the memory holding
 		 * the header to reset dh_start_lbps. If we TRIM the whole
 		 * device the new header will be written by
 		 * vdev_trim_l2arc_thread() at the end of the TRIM to update the
 		 * trim_state in the header too. When reading the header, if
 		 * trim_state is not VDEV_TRIM_COMPLETE and l2arc_trim_ahead > 0
 		 * we opt to TRIM the whole device again.
 		 */
 		if (l2arc_trim_ahead > 0) {
 			dev->l2ad_trim_all = B_TRUE;
 		} else {
 			memset(l2dhdr, 0, l2dhdr_asize);
 			l2arc_dev_hdr_update(dev);
 		}
 	}
 }
 
 /*
  * Add a vdev for use by the L2ARC.  By this point the spa has already
  * validated the vdev and opened it.
  */
 void
 l2arc_add_vdev(spa_t *spa, vdev_t *vd)
 {
 	l2arc_dev_t		*adddev;
 	uint64_t		l2dhdr_asize;
 
 	ASSERT(!l2arc_vdev_present(vd));
 
 	/*
 	 * Create a new l2arc device entry.
 	 */
 	adddev = vmem_zalloc(sizeof (l2arc_dev_t), KM_SLEEP);
 	adddev->l2ad_spa = spa;
 	adddev->l2ad_vdev = vd;
 	/* leave extra size for an l2arc device header */
 	l2dhdr_asize = adddev->l2ad_dev_hdr_asize =
 	    MAX(sizeof (*adddev->l2ad_dev_hdr), 1 << vd->vdev_ashift);
 	adddev->l2ad_start = VDEV_LABEL_START_SIZE + l2dhdr_asize;
 	adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
 	ASSERT3U(adddev->l2ad_start, <, adddev->l2ad_end);
 	adddev->l2ad_hand = adddev->l2ad_start;
 	adddev->l2ad_evict = adddev->l2ad_start;
 	adddev->l2ad_first = B_TRUE;
 	adddev->l2ad_writing = B_FALSE;
 	adddev->l2ad_trim_all = B_FALSE;
 	list_link_init(&adddev->l2ad_node);
 	adddev->l2ad_dev_hdr = kmem_zalloc(l2dhdr_asize, KM_SLEEP);
 
 	mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
 	/*
 	 * This is a list of all ARC buffers that are still valid on the
 	 * device.
 	 */
 	list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
 	    offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
 
 	/*
 	 * This is a list of pointers to log blocks that are still present
 	 * on the device.
 	 */
 	list_create(&adddev->l2ad_lbptr_list, sizeof (l2arc_lb_ptr_buf_t),
 	    offsetof(l2arc_lb_ptr_buf_t, node));
 
 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
 	zfs_refcount_create(&adddev->l2ad_alloc);
 	zfs_refcount_create(&adddev->l2ad_lb_asize);
 	zfs_refcount_create(&adddev->l2ad_lb_count);
 
 	/*
 	 * Decide if dev is eligible for L2ARC rebuild or whole device
 	 * trimming. This has to happen before the device is added in the
 	 * cache device list and l2arc_dev_mtx is released. Otherwise
 	 * l2arc_feed_thread() might already start writing on the
 	 * device.
 	 */
 	l2arc_rebuild_dev(adddev, B_FALSE);
 
 	/*
 	 * Add device to global list
 	 */
 	mutex_enter(&l2arc_dev_mtx);
 	list_insert_head(l2arc_dev_list, adddev);
 	atomic_inc_64(&l2arc_ndev);
 	mutex_exit(&l2arc_dev_mtx);
 }
 
 /*
  * Decide if a vdev is eligible for L2ARC rebuild, called from vdev_reopen()
  * in case of onlining a cache device.
  */
 void
 l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen)
 {
 	l2arc_dev_t		*dev = NULL;
 
 	dev = l2arc_vdev_get(vd);
 	ASSERT3P(dev, !=, NULL);
 
 	/*
 	 * In contrast to l2arc_add_vdev() we do not have to worry about
 	 * l2arc_feed_thread() invalidating previous content when onlining a
 	 * cache device. The device parameters (l2ad*) are not cleared when
 	 * offlining the device and writing new buffers will not invalidate
 	 * all previous content. In worst case only buffers that have not had
 	 * their log block written to the device will be lost.
 	 * When onlining the cache device (ie offline->online without exporting
 	 * the pool in between) this happens:
 	 * vdev_reopen() -> vdev_open() -> l2arc_rebuild_vdev()
 	 * 			|			|
 	 * 		vdev_is_dead() = B_FALSE	l2ad_rebuild = B_TRUE
 	 * During the time where vdev_is_dead = B_FALSE and until l2ad_rebuild
 	 * is set to B_TRUE we might write additional buffers to the device.
 	 */
 	l2arc_rebuild_dev(dev, reopen);
 }
 
 /*
  * Remove a vdev from the L2ARC.
  */
 void
 l2arc_remove_vdev(vdev_t *vd)
 {
 	l2arc_dev_t *remdev = NULL;
 
 	/*
 	 * Find the device by vdev
 	 */
 	remdev = l2arc_vdev_get(vd);
 	ASSERT3P(remdev, !=, NULL);
 
 	/*
 	 * Cancel any ongoing or scheduled rebuild.
 	 */
 	mutex_enter(&l2arc_rebuild_thr_lock);
 	if (remdev->l2ad_rebuild_began == B_TRUE) {
 		remdev->l2ad_rebuild_cancel = B_TRUE;
 		while (remdev->l2ad_rebuild == B_TRUE)
 			cv_wait(&l2arc_rebuild_thr_cv, &l2arc_rebuild_thr_lock);
 	}
 	mutex_exit(&l2arc_rebuild_thr_lock);
 
 	/*
 	 * Remove device from global list
 	 */
 	mutex_enter(&l2arc_dev_mtx);
 	list_remove(l2arc_dev_list, remdev);
 	l2arc_dev_last = NULL;		/* may have been invalidated */
 	atomic_dec_64(&l2arc_ndev);
 	mutex_exit(&l2arc_dev_mtx);
 
 	/*
 	 * Clear all buflists and ARC references.  L2ARC device flush.
 	 */
 	l2arc_evict(remdev, 0, B_TRUE);
 	list_destroy(&remdev->l2ad_buflist);
 	ASSERT(list_is_empty(&remdev->l2ad_lbptr_list));
 	list_destroy(&remdev->l2ad_lbptr_list);
 	mutex_destroy(&remdev->l2ad_mtx);
 	zfs_refcount_destroy(&remdev->l2ad_alloc);
 	zfs_refcount_destroy(&remdev->l2ad_lb_asize);
 	zfs_refcount_destroy(&remdev->l2ad_lb_count);
 	kmem_free(remdev->l2ad_dev_hdr, remdev->l2ad_dev_hdr_asize);
 	vmem_free(remdev, sizeof (l2arc_dev_t));
 }
 
 void
 l2arc_init(void)
 {
 	l2arc_thread_exit = 0;
 	l2arc_ndev = 0;
 
 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
 	mutex_init(&l2arc_rebuild_thr_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&l2arc_rebuild_thr_cv, NULL, CV_DEFAULT, NULL);
 	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
 
 	l2arc_dev_list = &L2ARC_dev_list;
 	l2arc_free_on_write = &L2ARC_free_on_write;
 	list_create(l2arc_dev_list, sizeof (l2arc_dev_t),
 	    offsetof(l2arc_dev_t, l2ad_node));
 	list_create(l2arc_free_on_write, sizeof (l2arc_data_free_t),
 	    offsetof(l2arc_data_free_t, l2df_list_node));
 }
 
 void
 l2arc_fini(void)
 {
 	mutex_destroy(&l2arc_feed_thr_lock);
 	cv_destroy(&l2arc_feed_thr_cv);
 	mutex_destroy(&l2arc_rebuild_thr_lock);
 	cv_destroy(&l2arc_rebuild_thr_cv);
 	mutex_destroy(&l2arc_dev_mtx);
 	mutex_destroy(&l2arc_free_on_write_mtx);
 
 	list_destroy(l2arc_dev_list);
 	list_destroy(l2arc_free_on_write);
 }
 
 void
 l2arc_start(void)
 {
 	if (!(spa_mode_global & SPA_MODE_WRITE))
 		return;
 
 	(void) thread_create(NULL, 0, l2arc_feed_thread, NULL, 0, &p0,
 	    TS_RUN, defclsyspri);
 }
 
 void
 l2arc_stop(void)
 {
 	if (!(spa_mode_global & SPA_MODE_WRITE))
 		return;
 
 	mutex_enter(&l2arc_feed_thr_lock);
 	cv_signal(&l2arc_feed_thr_cv);	/* kick thread out of startup */
 	l2arc_thread_exit = 1;
 	while (l2arc_thread_exit != 0)
 		cv_wait(&l2arc_feed_thr_cv, &l2arc_feed_thr_lock);
 	mutex_exit(&l2arc_feed_thr_lock);
 }
 
 /*
  * Punches out rebuild threads for the L2ARC devices in a spa. This should
  * be called after pool import from the spa async thread, since starting
  * these threads directly from spa_import() will make them part of the
  * "zpool import" context and delay process exit (and thus pool import).
  */
 void
 l2arc_spa_rebuild_start(spa_t *spa)
 {
 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
 
 	/*
 	 * Locate the spa's l2arc devices and kick off rebuild threads.
 	 */
 	for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
 		l2arc_dev_t *dev =
 		    l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
 		if (dev == NULL) {
 			/* Don't attempt a rebuild if the vdev is UNAVAIL */
 			continue;
 		}
 		mutex_enter(&l2arc_rebuild_thr_lock);
 		if (dev->l2ad_rebuild && !dev->l2ad_rebuild_cancel) {
 			dev->l2ad_rebuild_began = B_TRUE;
 			(void) thread_create(NULL, 0, l2arc_dev_rebuild_thread,
 			    dev, 0, &p0, TS_RUN, minclsyspri);
 		}
 		mutex_exit(&l2arc_rebuild_thr_lock);
 	}
 }
 
 /*
  * Main entry point for L2ARC rebuilding.
  */
 static __attribute__((noreturn)) void
 l2arc_dev_rebuild_thread(void *arg)
 {
 	l2arc_dev_t *dev = arg;
 
 	VERIFY(!dev->l2ad_rebuild_cancel);
 	VERIFY(dev->l2ad_rebuild);
 	(void) l2arc_rebuild(dev);
 	mutex_enter(&l2arc_rebuild_thr_lock);
 	dev->l2ad_rebuild_began = B_FALSE;
 	dev->l2ad_rebuild = B_FALSE;
 	mutex_exit(&l2arc_rebuild_thr_lock);
 
 	thread_exit();
 }
 
 /*
  * This function implements the actual L2ARC metadata rebuild. It:
  * starts reading the log block chain and restores each block's contents
  * to memory (reconstructing arc_buf_hdr_t's).
  *
  * Operation stops under any of the following conditions:
  *
  * 1) We reach the end of the log block chain.
  * 2) We encounter *any* error condition (cksum errors, io errors)
  */
 static int
 l2arc_rebuild(l2arc_dev_t *dev)
 {
 	vdev_t			*vd = dev->l2ad_vdev;
 	spa_t			*spa = vd->vdev_spa;
 	int			err = 0;
 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
 	l2arc_log_blk_phys_t	*this_lb, *next_lb;
 	zio_t			*this_io = NULL, *next_io = NULL;
 	l2arc_log_blkptr_t	lbps[2];
 	l2arc_lb_ptr_buf_t	*lb_ptr_buf;
 	boolean_t		lock_held;
 
 	this_lb = vmem_zalloc(sizeof (*this_lb), KM_SLEEP);
 	next_lb = vmem_zalloc(sizeof (*next_lb), KM_SLEEP);
 
 	/*
 	 * We prevent device removal while issuing reads to the device,
 	 * then during the rebuilding phases we drop this lock again so
 	 * that a spa_unload or device remove can be initiated - this is
 	 * safe, because the spa will signal us to stop before removing
 	 * our device and wait for us to stop.
 	 */
 	spa_config_enter(spa, SCL_L2ARC, vd, RW_READER);
 	lock_held = B_TRUE;
 
 	/*
 	 * Retrieve the persistent L2ARC device state.
 	 * L2BLK_GET_PSIZE returns aligned size for log blocks.
 	 */
 	dev->l2ad_evict = MAX(l2dhdr->dh_evict, dev->l2ad_start);
 	dev->l2ad_hand = MAX(l2dhdr->dh_start_lbps[0].lbp_daddr +
 	    L2BLK_GET_PSIZE((&l2dhdr->dh_start_lbps[0])->lbp_prop),
 	    dev->l2ad_start);
 	dev->l2ad_first = !!(l2dhdr->dh_flags & L2ARC_DEV_HDR_EVICT_FIRST);
 
 	vd->vdev_trim_action_time = l2dhdr->dh_trim_action_time;
 	vd->vdev_trim_state = l2dhdr->dh_trim_state;
 
 	/*
 	 * In case the zfs module parameter l2arc_rebuild_enabled is false
 	 * we do not start the rebuild process.
 	 */
 	if (!l2arc_rebuild_enabled)
 		goto out;
 
 	/* Prepare the rebuild process */
 	memcpy(lbps, l2dhdr->dh_start_lbps, sizeof (lbps));
 
 	/* Start the rebuild process */
 	for (;;) {
 		if (!l2arc_log_blkptr_valid(dev, &lbps[0]))
 			break;
 
 		if ((err = l2arc_log_blk_read(dev, &lbps[0], &lbps[1],
 		    this_lb, next_lb, this_io, &next_io)) != 0)
 			goto out;
 
 		/*
 		 * Our memory pressure valve. If the system is running low
 		 * on memory, rather than swamping memory with new ARC buf
 		 * hdrs, we opt not to rebuild the L2ARC. At this point,
 		 * however, we have already set up our L2ARC dev to chain in
 		 * new metadata log blocks, so the user may choose to offline/
 		 * online the L2ARC dev at a later time (or re-import the pool)
 		 * to reconstruct it (when there's less memory pressure).
 		 */
 		if (l2arc_hdr_limit_reached()) {
 			ARCSTAT_BUMP(arcstat_l2_rebuild_abort_lowmem);
 			cmn_err(CE_NOTE, "System running low on memory, "
 			    "aborting L2ARC rebuild.");
 			err = SET_ERROR(ENOMEM);
 			goto out;
 		}
 
 		spa_config_exit(spa, SCL_L2ARC, vd);
 		lock_held = B_FALSE;
 
 		/*
 		 * Now that we know that the next_lb checks out alright, we
 		 * can start reconstruction from this log block.
 		 * L2BLK_GET_PSIZE returns aligned size for log blocks.
 		 */
 		uint64_t asize = L2BLK_GET_PSIZE((&lbps[0])->lbp_prop);
 		l2arc_log_blk_restore(dev, this_lb, asize);
 
 		/*
 		 * log block restored, include its pointer in the list of
 		 * pointers to log blocks present in the L2ARC device.
 		 */
 		lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP);
 		lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t),
 		    KM_SLEEP);
 		memcpy(lb_ptr_buf->lb_ptr, &lbps[0],
 		    sizeof (l2arc_log_blkptr_t));
 		mutex_enter(&dev->l2ad_mtx);
 		list_insert_tail(&dev->l2ad_lbptr_list, lb_ptr_buf);
 		ARCSTAT_INCR(arcstat_l2_log_blk_asize, asize);
 		ARCSTAT_BUMP(arcstat_l2_log_blk_count);
 		zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
 		zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
 		mutex_exit(&dev->l2ad_mtx);
 		vdev_space_update(vd, asize, 0, 0);
 
 		/*
 		 * Protection against loops of log blocks:
 		 *
 		 *				       l2ad_hand  l2ad_evict
 		 *                                         V	      V
 		 * l2ad_start |=======================================| l2ad_end
 		 *             -----|||----|||---|||----|||
 		 *                  (3)    (2)   (1)    (0)
 		 *             ---|||---|||----|||---|||
 		 *		  (7)   (6)    (5)   (4)
 		 *
 		 * In this situation the pointer of log block (4) passes
 		 * l2arc_log_blkptr_valid() but the log block should not be
 		 * restored as it is overwritten by the payload of log block
 		 * (0). Only log blocks (0)-(3) should be restored. We check
 		 * whether l2ad_evict lies in between the payload starting
 		 * offset of the next log block (lbps[1].lbp_payload_start)
 		 * and the payload starting offset of the present log block
 		 * (lbps[0].lbp_payload_start). If true and this isn't the
 		 * first pass, we are looping from the beginning and we should
 		 * stop.
 		 */
 		if (l2arc_range_check_overlap(lbps[1].lbp_payload_start,
 		    lbps[0].lbp_payload_start, dev->l2ad_evict) &&
 		    !dev->l2ad_first)
 			goto out;
 
 		kpreempt(KPREEMPT_SYNC);
 		for (;;) {
 			mutex_enter(&l2arc_rebuild_thr_lock);
 			if (dev->l2ad_rebuild_cancel) {
 				dev->l2ad_rebuild = B_FALSE;
 				cv_signal(&l2arc_rebuild_thr_cv);
 				mutex_exit(&l2arc_rebuild_thr_lock);
 				err = SET_ERROR(ECANCELED);
 				goto out;
 			}
 			mutex_exit(&l2arc_rebuild_thr_lock);
 			if (spa_config_tryenter(spa, SCL_L2ARC, vd,
 			    RW_READER)) {
 				lock_held = B_TRUE;
 				break;
 			}
 			/*
 			 * L2ARC config lock held by somebody in writer,
 			 * possibly due to them trying to remove us. They'll
 			 * likely to want us to shut down, so after a little
 			 * delay, we check l2ad_rebuild_cancel and retry
 			 * the lock again.
 			 */
 			delay(1);
 		}
 
 		/*
 		 * Continue with the next log block.
 		 */
 		lbps[0] = lbps[1];
 		lbps[1] = this_lb->lb_prev_lbp;
 		PTR_SWAP(this_lb, next_lb);
 		this_io = next_io;
 		next_io = NULL;
 	}
 
 	if (this_io != NULL)
 		l2arc_log_blk_fetch_abort(this_io);
 out:
 	if (next_io != NULL)
 		l2arc_log_blk_fetch_abort(next_io);
 	vmem_free(this_lb, sizeof (*this_lb));
 	vmem_free(next_lb, sizeof (*next_lb));
 
 	if (!l2arc_rebuild_enabled) {
 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
 		    "disabled");
 	} else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) > 0) {
 		ARCSTAT_BUMP(arcstat_l2_rebuild_success);
 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
 		    "successful, restored %llu blocks",
 		    (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
 	} else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) == 0) {
 		/*
 		 * No error but also nothing restored, meaning the lbps array
 		 * in the device header points to invalid/non-present log
 		 * blocks. Reset the header.
 		 */
 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
 		    "no valid log blocks");
 		memset(l2dhdr, 0, dev->l2ad_dev_hdr_asize);
 		l2arc_dev_hdr_update(dev);
 	} else if (err == ECANCELED) {
 		/*
 		 * In case the rebuild was canceled do not log to spa history
 		 * log as the pool may be in the process of being removed.
 		 */
 		zfs_dbgmsg("L2ARC rebuild aborted, restored %llu blocks",
 		    (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
 	} else if (err != 0) {
 		spa_history_log_internal(spa, "L2ARC rebuild", NULL,
 		    "aborted, restored %llu blocks",
 		    (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
 	}
 
 	if (lock_held)
 		spa_config_exit(spa, SCL_L2ARC, vd);
 
 	return (err);
 }
 
 /*
  * Attempts to read the device header on the provided L2ARC device and writes
  * it to `hdr'. On success, this function returns 0, otherwise the appropriate
  * error code is returned.
  */
 static int
 l2arc_dev_hdr_read(l2arc_dev_t *dev)
 {
 	int			err;
 	uint64_t		guid;
 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
 	const uint64_t		l2dhdr_asize = dev->l2ad_dev_hdr_asize;
 	abd_t 			*abd;
 
 	guid = spa_guid(dev->l2ad_vdev->vdev_spa);
 
 	abd = abd_get_from_buf(l2dhdr, l2dhdr_asize);
 
 	err = zio_wait(zio_read_phys(NULL, dev->l2ad_vdev,
 	    VDEV_LABEL_START_SIZE, l2dhdr_asize, abd,
 	    ZIO_CHECKSUM_LABEL, NULL, NULL, ZIO_PRIORITY_SYNC_READ,
 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY |
 	    ZIO_FLAG_SPECULATIVE, B_FALSE));
 
 	abd_free(abd);
 
 	if (err != 0) {
 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_dh_errors);
 		zfs_dbgmsg("L2ARC IO error (%d) while reading device header, "
 		    "vdev guid: %llu", err,
 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid);
 		return (err);
 	}
 
 	if (l2dhdr->dh_magic == BSWAP_64(L2ARC_DEV_HDR_MAGIC))
 		byteswap_uint64_array(l2dhdr, sizeof (*l2dhdr));
 
 	if (l2dhdr->dh_magic != L2ARC_DEV_HDR_MAGIC ||
 	    l2dhdr->dh_spa_guid != guid ||
 	    l2dhdr->dh_vdev_guid != dev->l2ad_vdev->vdev_guid ||
 	    l2dhdr->dh_version != L2ARC_PERSISTENT_VERSION ||
 	    l2dhdr->dh_log_entries != dev->l2ad_log_entries ||
 	    l2dhdr->dh_end != dev->l2ad_end ||
 	    !l2arc_range_check_overlap(dev->l2ad_start, dev->l2ad_end,
 	    l2dhdr->dh_evict) ||
 	    (l2dhdr->dh_trim_state != VDEV_TRIM_COMPLETE &&
 	    l2arc_trim_ahead > 0)) {
 		/*
 		 * Attempt to rebuild a device containing no actual dev hdr
 		 * or containing a header from some other pool or from another
 		 * version of persistent L2ARC.
 		 */
 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_unsupported);
 		return (SET_ERROR(ENOTSUP));
 	}
 
 	return (0);
 }
 
 /*
  * Reads L2ARC log blocks from storage and validates their contents.
  *
  * This function implements a simple fetcher to make sure that while
  * we're processing one buffer the L2ARC is already fetching the next
  * one in the chain.
  *
  * The arguments this_lp and next_lp point to the current and next log block
  * address in the block chain. Similarly, this_lb and next_lb hold the
  * l2arc_log_blk_phys_t's of the current and next L2ARC blk.
  *
  * The `this_io' and `next_io' arguments are used for block fetching.
  * When issuing the first blk IO during rebuild, you should pass NULL for
  * `this_io'. This function will then issue a sync IO to read the block and
  * also issue an async IO to fetch the next block in the block chain. The
  * fetched IO is returned in `next_io'. On subsequent calls to this
  * function, pass the value returned in `next_io' from the previous call
  * as `this_io' and a fresh `next_io' pointer to hold the next fetch IO.
  * Prior to the call, you should initialize your `next_io' pointer to be
  * NULL. If no fetch IO was issued, the pointer is left set at NULL.
  *
  * On success, this function returns 0, otherwise it returns an appropriate
  * error code. On error the fetching IO is aborted and cleared before
  * returning from this function. Therefore, if we return `success', the
  * caller can assume that we have taken care of cleanup of fetch IOs.
  */
 static int
 l2arc_log_blk_read(l2arc_dev_t *dev,
     const l2arc_log_blkptr_t *this_lbp, const l2arc_log_blkptr_t *next_lbp,
     l2arc_log_blk_phys_t *this_lb, l2arc_log_blk_phys_t *next_lb,
     zio_t *this_io, zio_t **next_io)
 {
 	int		err = 0;
 	zio_cksum_t	cksum;
 	abd_t		*abd = NULL;
 	uint64_t	asize;
 
 	ASSERT(this_lbp != NULL && next_lbp != NULL);
 	ASSERT(this_lb != NULL && next_lb != NULL);
 	ASSERT(next_io != NULL && *next_io == NULL);
 	ASSERT(l2arc_log_blkptr_valid(dev, this_lbp));
 
 	/*
 	 * Check to see if we have issued the IO for this log block in a
 	 * previous run. If not, this is the first call, so issue it now.
 	 */
 	if (this_io == NULL) {
 		this_io = l2arc_log_blk_fetch(dev->l2ad_vdev, this_lbp,
 		    this_lb);
 	}
 
 	/*
 	 * Peek to see if we can start issuing the next IO immediately.
 	 */
 	if (l2arc_log_blkptr_valid(dev, next_lbp)) {
 		/*
 		 * Start issuing IO for the next log block early - this
 		 * should help keep the L2ARC device busy while we
 		 * decompress and restore this log block.
 		 */
 		*next_io = l2arc_log_blk_fetch(dev->l2ad_vdev, next_lbp,
 		    next_lb);
 	}
 
 	/* Wait for the IO to read this log block to complete */
 	if ((err = zio_wait(this_io)) != 0) {
 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_io_errors);
 		zfs_dbgmsg("L2ARC IO error (%d) while reading log block, "
 		    "offset: %llu, vdev guid: %llu", err,
 		    (u_longlong_t)this_lbp->lbp_daddr,
 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid);
 		goto cleanup;
 	}
 
 	/*
 	 * Make sure the buffer checks out.
 	 * L2BLK_GET_PSIZE returns aligned size for log blocks.
 	 */
 	asize = L2BLK_GET_PSIZE((this_lbp)->lbp_prop);
 	fletcher_4_native(this_lb, asize, NULL, &cksum);
 	if (!ZIO_CHECKSUM_EQUAL(cksum, this_lbp->lbp_cksum)) {
 		ARCSTAT_BUMP(arcstat_l2_rebuild_abort_cksum_lb_errors);
 		zfs_dbgmsg("L2ARC log block cksum failed, offset: %llu, "
 		    "vdev guid: %llu, l2ad_hand: %llu, l2ad_evict: %llu",
 		    (u_longlong_t)this_lbp->lbp_daddr,
 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid,
 		    (u_longlong_t)dev->l2ad_hand,
 		    (u_longlong_t)dev->l2ad_evict);
 		err = SET_ERROR(ECKSUM);
 		goto cleanup;
 	}
 
 	/* Now we can take our time decoding this buffer */
 	switch (L2BLK_GET_COMPRESS((this_lbp)->lbp_prop)) {
 	case ZIO_COMPRESS_OFF:
 		break;
 	case ZIO_COMPRESS_LZ4:
 		abd = abd_alloc_for_io(asize, B_TRUE);
 		abd_copy_from_buf_off(abd, this_lb, 0, asize);
 		if ((err = zio_decompress_data(
 		    L2BLK_GET_COMPRESS((this_lbp)->lbp_prop),
 		    abd, this_lb, asize, sizeof (*this_lb), NULL)) != 0) {
 			err = SET_ERROR(EINVAL);
 			goto cleanup;
 		}
 		break;
 	default:
 		err = SET_ERROR(EINVAL);
 		goto cleanup;
 	}
 	if (this_lb->lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC))
 		byteswap_uint64_array(this_lb, sizeof (*this_lb));
 	if (this_lb->lb_magic != L2ARC_LOG_BLK_MAGIC) {
 		err = SET_ERROR(EINVAL);
 		goto cleanup;
 	}
 cleanup:
 	/* Abort an in-flight fetch I/O in case of error */
 	if (err != 0 && *next_io != NULL) {
 		l2arc_log_blk_fetch_abort(*next_io);
 		*next_io = NULL;
 	}
 	if (abd != NULL)
 		abd_free(abd);
 	return (err);
 }
 
 /*
  * Restores the payload of a log block to ARC. This creates empty ARC hdr
  * entries which only contain an l2arc hdr, essentially restoring the
  * buffers to their L2ARC evicted state. This function also updates space
  * usage on the L2ARC vdev to make sure it tracks restored buffers.
  */
 static void
 l2arc_log_blk_restore(l2arc_dev_t *dev, const l2arc_log_blk_phys_t *lb,
     uint64_t lb_asize)
 {
 	uint64_t	size = 0, asize = 0;
 	uint64_t	log_entries = dev->l2ad_log_entries;
 
 	/*
 	 * Usually arc_adapt() is called only for data, not headers, but
 	 * since we may allocate significant amount of memory here, let ARC
 	 * grow its arc_c.
 	 */
 	arc_adapt(log_entries * HDR_L2ONLY_SIZE);
 
 	for (int i = log_entries - 1; i >= 0; i--) {
 		/*
 		 * Restore goes in the reverse temporal direction to preserve
 		 * correct temporal ordering of buffers in the l2ad_buflist.
 		 * l2arc_hdr_restore also does a list_insert_tail instead of
 		 * list_insert_head on the l2ad_buflist:
 		 *
 		 *		LIST	l2ad_buflist		LIST
 		 *		HEAD  <------ (time) ------	TAIL
 		 * direction	+-----+-----+-----+-----+-----+    direction
 		 * of l2arc <== | buf | buf | buf | buf | buf | ===> of rebuild
 		 * fill		+-----+-----+-----+-----+-----+
 		 *		^				^
 		 *		|				|
 		 *		|				|
 		 *	l2arc_feed_thread		l2arc_rebuild
 		 *	will place new bufs here	restores bufs here
 		 *
 		 * During l2arc_rebuild() the device is not used by
 		 * l2arc_feed_thread() as dev->l2ad_rebuild is set to true.
 		 */
 		size += L2BLK_GET_LSIZE((&lb->lb_entries[i])->le_prop);
 		asize += vdev_psize_to_asize(dev->l2ad_vdev,
 		    L2BLK_GET_PSIZE((&lb->lb_entries[i])->le_prop));
 		l2arc_hdr_restore(&lb->lb_entries[i], dev);
 	}
 
 	/*
 	 * Record rebuild stats:
 	 *	size		Logical size of restored buffers in the L2ARC
 	 *	asize		Aligned size of restored buffers in the L2ARC
 	 */
 	ARCSTAT_INCR(arcstat_l2_rebuild_size, size);
 	ARCSTAT_INCR(arcstat_l2_rebuild_asize, asize);
 	ARCSTAT_INCR(arcstat_l2_rebuild_bufs, log_entries);
 	ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_asize, lb_asize);
 	ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio, asize / lb_asize);
 	ARCSTAT_BUMP(arcstat_l2_rebuild_log_blks);
 }
 
 /*
  * Restores a single ARC buf hdr from a log entry. The ARC buffer is put
  * into a state indicating that it has been evicted to L2ARC.
  */
 static void
 l2arc_hdr_restore(const l2arc_log_ent_phys_t *le, l2arc_dev_t *dev)
 {
 	arc_buf_hdr_t		*hdr, *exists;
 	kmutex_t		*hash_lock;
 	arc_buf_contents_t	type = L2BLK_GET_TYPE((le)->le_prop);
 	uint64_t		asize;
 
 	/*
 	 * Do all the allocation before grabbing any locks, this lets us
 	 * sleep if memory is full and we don't have to deal with failed
 	 * allocations.
 	 */
 	hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type,
 	    dev, le->le_dva, le->le_daddr,
 	    L2BLK_GET_PSIZE((le)->le_prop), le->le_birth,
 	    L2BLK_GET_COMPRESS((le)->le_prop), le->le_complevel,
 	    L2BLK_GET_PROTECTED((le)->le_prop),
 	    L2BLK_GET_PREFETCH((le)->le_prop),
 	    L2BLK_GET_STATE((le)->le_prop));
 	asize = vdev_psize_to_asize(dev->l2ad_vdev,
 	    L2BLK_GET_PSIZE((le)->le_prop));
 
 	/*
 	 * vdev_space_update() has to be called before arc_hdr_destroy() to
 	 * avoid underflow since the latter also calls vdev_space_update().
 	 */
 	l2arc_hdr_arcstats_increment(hdr);
 	vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
 
 	mutex_enter(&dev->l2ad_mtx);
 	list_insert_tail(&dev->l2ad_buflist, hdr);
 	(void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
 	mutex_exit(&dev->l2ad_mtx);
 
 	exists = buf_hash_insert(hdr, &hash_lock);
 	if (exists) {
 		/* Buffer was already cached, no need to restore it. */
 		arc_hdr_destroy(hdr);
 		/*
 		 * If the buffer is already cached, check whether it has
 		 * L2ARC metadata. If not, enter them and update the flag.
 		 * This is important is case of onlining a cache device, since
 		 * we previously evicted all L2ARC metadata from ARC.
 		 */
 		if (!HDR_HAS_L2HDR(exists)) {
 			arc_hdr_set_flags(exists, ARC_FLAG_HAS_L2HDR);
 			exists->b_l2hdr.b_dev = dev;
 			exists->b_l2hdr.b_daddr = le->le_daddr;
 			exists->b_l2hdr.b_arcs_state =
 			    L2BLK_GET_STATE((le)->le_prop);
 			mutex_enter(&dev->l2ad_mtx);
 			list_insert_tail(&dev->l2ad_buflist, exists);
 			(void) zfs_refcount_add_many(&dev->l2ad_alloc,
 			    arc_hdr_size(exists), exists);
 			mutex_exit(&dev->l2ad_mtx);
 			l2arc_hdr_arcstats_increment(exists);
 			vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
 		}
 		ARCSTAT_BUMP(arcstat_l2_rebuild_bufs_precached);
 	}
 
 	mutex_exit(hash_lock);
 }
 
 /*
  * Starts an asynchronous read IO to read a log block. This is used in log
  * block reconstruction to start reading the next block before we are done
  * decoding and reconstructing the current block, to keep the l2arc device
  * nice and hot with read IO to process.
  * The returned zio will contain a newly allocated memory buffers for the IO
  * data which should then be freed by the caller once the zio is no longer
  * needed (i.e. due to it having completed). If you wish to abort this
  * zio, you should do so using l2arc_log_blk_fetch_abort, which takes
  * care of disposing of the allocated buffers correctly.
  */
 static zio_t *
 l2arc_log_blk_fetch(vdev_t *vd, const l2arc_log_blkptr_t *lbp,
     l2arc_log_blk_phys_t *lb)
 {
 	uint32_t		asize;
 	zio_t			*pio;
 	l2arc_read_callback_t	*cb;
 
 	/* L2BLK_GET_PSIZE returns aligned size for log blocks */
 	asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
 	ASSERT(asize <= sizeof (l2arc_log_blk_phys_t));
 
 	cb = kmem_zalloc(sizeof (l2arc_read_callback_t), KM_SLEEP);
 	cb->l2rcb_abd = abd_get_from_buf(lb, asize);
 	pio = zio_root(vd->vdev_spa, l2arc_blk_fetch_done, cb,
 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY);
 	(void) zio_nowait(zio_read_phys(pio, vd, lbp->lbp_daddr, asize,
 	    cb->l2rcb_abd, ZIO_CHECKSUM_OFF, NULL, NULL,
 	    ZIO_PRIORITY_ASYNC_READ, ZIO_FLAG_CANFAIL |
 	    ZIO_FLAG_DONT_PROPAGATE | ZIO_FLAG_DONT_RETRY, B_FALSE));
 
 	return (pio);
 }
 
 /*
  * Aborts a zio returned from l2arc_log_blk_fetch and frees the data
  * buffers allocated for it.
  */
 static void
 l2arc_log_blk_fetch_abort(zio_t *zio)
 {
 	(void) zio_wait(zio);
 }
 
 /*
  * Creates a zio to update the device header on an l2arc device.
  */
 void
 l2arc_dev_hdr_update(l2arc_dev_t *dev)
 {
 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
 	const uint64_t		l2dhdr_asize = dev->l2ad_dev_hdr_asize;
 	abd_t			*abd;
 	int			err;
 
 	VERIFY(spa_config_held(dev->l2ad_spa, SCL_STATE_ALL, RW_READER));
 
 	l2dhdr->dh_magic = L2ARC_DEV_HDR_MAGIC;
 	l2dhdr->dh_version = L2ARC_PERSISTENT_VERSION;
 	l2dhdr->dh_spa_guid = spa_guid(dev->l2ad_vdev->vdev_spa);
 	l2dhdr->dh_vdev_guid = dev->l2ad_vdev->vdev_guid;
 	l2dhdr->dh_log_entries = dev->l2ad_log_entries;
 	l2dhdr->dh_evict = dev->l2ad_evict;
 	l2dhdr->dh_start = dev->l2ad_start;
 	l2dhdr->dh_end = dev->l2ad_end;
 	l2dhdr->dh_lb_asize = zfs_refcount_count(&dev->l2ad_lb_asize);
 	l2dhdr->dh_lb_count = zfs_refcount_count(&dev->l2ad_lb_count);
 	l2dhdr->dh_flags = 0;
 	l2dhdr->dh_trim_action_time = dev->l2ad_vdev->vdev_trim_action_time;
 	l2dhdr->dh_trim_state = dev->l2ad_vdev->vdev_trim_state;
 	if (dev->l2ad_first)
 		l2dhdr->dh_flags |= L2ARC_DEV_HDR_EVICT_FIRST;
 
 	abd = abd_get_from_buf(l2dhdr, l2dhdr_asize);
 
 	err = zio_wait(zio_write_phys(NULL, dev->l2ad_vdev,
 	    VDEV_LABEL_START_SIZE, l2dhdr_asize, abd, ZIO_CHECKSUM_LABEL, NULL,
 	    NULL, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE));
 
 	abd_free(abd);
 
 	if (err != 0) {
 		zfs_dbgmsg("L2ARC IO error (%d) while writing device header, "
 		    "vdev guid: %llu", err,
 		    (u_longlong_t)dev->l2ad_vdev->vdev_guid);
 	}
 }
 
 /*
  * Commits a log block to the L2ARC device. This routine is invoked from
  * l2arc_write_buffers when the log block fills up.
  * This function allocates some memory to temporarily hold the serialized
  * buffer to be written. This is then released in l2arc_write_done.
  */
 static uint64_t
 l2arc_log_blk_commit(l2arc_dev_t *dev, zio_t *pio, l2arc_write_callback_t *cb)
 {
 	l2arc_log_blk_phys_t	*lb = &dev->l2ad_log_blk;
 	l2arc_dev_hdr_phys_t	*l2dhdr = dev->l2ad_dev_hdr;
 	uint64_t		psize, asize;
 	zio_t			*wzio;
 	l2arc_lb_abd_buf_t	*abd_buf;
 	uint8_t			*tmpbuf = NULL;
 	l2arc_lb_ptr_buf_t	*lb_ptr_buf;
 
 	VERIFY3S(dev->l2ad_log_ent_idx, ==, dev->l2ad_log_entries);
 
 	abd_buf = zio_buf_alloc(sizeof (*abd_buf));
 	abd_buf->abd = abd_get_from_buf(lb, sizeof (*lb));
 	lb_ptr_buf = kmem_zalloc(sizeof (l2arc_lb_ptr_buf_t), KM_SLEEP);
 	lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t), KM_SLEEP);
 
 	/* link the buffer into the block chain */
 	lb->lb_prev_lbp = l2dhdr->dh_start_lbps[1];
 	lb->lb_magic = L2ARC_LOG_BLK_MAGIC;
 
 	/*
 	 * l2arc_log_blk_commit() may be called multiple times during a single
 	 * l2arc_write_buffers() call. Save the allocated abd buffers in a list
 	 * so we can free them in l2arc_write_done() later on.
 	 */
 	list_insert_tail(&cb->l2wcb_abd_list, abd_buf);
 
 	/* try to compress the buffer */
 	psize = zio_compress_data(ZIO_COMPRESS_LZ4,
 	    abd_buf->abd, (void **) &tmpbuf, sizeof (*lb), 0);
 
 	/* a log block is never entirely zero */
 	ASSERT(psize != 0);
 	asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
 	ASSERT(asize <= sizeof (*lb));
 
 	/*
 	 * Update the start log block pointer in the device header to point
 	 * to the log block we're about to write.
 	 */
 	l2dhdr->dh_start_lbps[1] = l2dhdr->dh_start_lbps[0];
 	l2dhdr->dh_start_lbps[0].lbp_daddr = dev->l2ad_hand;
 	l2dhdr->dh_start_lbps[0].lbp_payload_asize =
 	    dev->l2ad_log_blk_payload_asize;
 	l2dhdr->dh_start_lbps[0].lbp_payload_start =
 	    dev->l2ad_log_blk_payload_start;
 	L2BLK_SET_LSIZE(
 	    (&l2dhdr->dh_start_lbps[0])->lbp_prop, sizeof (*lb));
 	L2BLK_SET_PSIZE(
 	    (&l2dhdr->dh_start_lbps[0])->lbp_prop, asize);
 	L2BLK_SET_CHECKSUM(
 	    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
 	    ZIO_CHECKSUM_FLETCHER_4);
 	if (asize < sizeof (*lb)) {
 		/* compression succeeded */
 		memset(tmpbuf + psize, 0, asize - psize);
 		L2BLK_SET_COMPRESS(
 		    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
 		    ZIO_COMPRESS_LZ4);
 	} else {
 		/* compression failed */
 		memcpy(tmpbuf, lb, sizeof (*lb));
 		L2BLK_SET_COMPRESS(
 		    (&l2dhdr->dh_start_lbps[0])->lbp_prop,
 		    ZIO_COMPRESS_OFF);
 	}
 
 	/* checksum what we're about to write */
 	fletcher_4_native(tmpbuf, asize, NULL,
 	    &l2dhdr->dh_start_lbps[0].lbp_cksum);
 
 	abd_free(abd_buf->abd);
 
 	/* perform the write itself */
 	abd_buf->abd = abd_get_from_buf(tmpbuf, sizeof (*lb));
 	abd_take_ownership_of_buf(abd_buf->abd, B_TRUE);
 	wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand,
 	    asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL,
 	    ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_CANFAIL, B_FALSE);
 	DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, zio_t *, wzio);
 	(void) zio_nowait(wzio);
 
 	dev->l2ad_hand += asize;
 	/*
 	 * Include the committed log block's pointer  in the list of pointers
 	 * to log blocks present in the L2ARC device.
 	 */
 	memcpy(lb_ptr_buf->lb_ptr, &l2dhdr->dh_start_lbps[0],
 	    sizeof (l2arc_log_blkptr_t));
 	mutex_enter(&dev->l2ad_mtx);
 	list_insert_head(&dev->l2ad_lbptr_list, lb_ptr_buf);
 	ARCSTAT_INCR(arcstat_l2_log_blk_asize, asize);
 	ARCSTAT_BUMP(arcstat_l2_log_blk_count);
 	zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
 	zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
 	mutex_exit(&dev->l2ad_mtx);
 	vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
 
 	/* bump the kstats */
 	ARCSTAT_INCR(arcstat_l2_write_bytes, asize);
 	ARCSTAT_BUMP(arcstat_l2_log_blk_writes);
 	ARCSTAT_F_AVG(arcstat_l2_log_blk_avg_asize, asize);
 	ARCSTAT_F_AVG(arcstat_l2_data_to_meta_ratio,
 	    dev->l2ad_log_blk_payload_asize / asize);
 
 	/* start a new log block */
 	dev->l2ad_log_ent_idx = 0;
 	dev->l2ad_log_blk_payload_asize = 0;
 	dev->l2ad_log_blk_payload_start = 0;
 
 	return (asize);
 }
 
 /*
  * Validates an L2ARC log block address to make sure that it can be read
  * from the provided L2ARC device.
  */
 boolean_t
 l2arc_log_blkptr_valid(l2arc_dev_t *dev, const l2arc_log_blkptr_t *lbp)
 {
 	/* L2BLK_GET_PSIZE returns aligned size for log blocks */
 	uint64_t asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
 	uint64_t end = lbp->lbp_daddr + asize - 1;
 	uint64_t start = lbp->lbp_payload_start;
 	boolean_t evicted = B_FALSE;
 
 	/*
 	 * A log block is valid if all of the following conditions are true:
 	 * - it fits entirely (including its payload) between l2ad_start and
 	 *   l2ad_end
 	 * - it has a valid size
 	 * - neither the log block itself nor part of its payload was evicted
 	 *   by l2arc_evict():
 	 *
 	 *		l2ad_hand          l2ad_evict
 	 *		|			 |	lbp_daddr
 	 *		|     start		 |	|  end
 	 *		|     |			 |	|  |
 	 *		V     V		         V	V  V
 	 *   l2ad_start ============================================ l2ad_end
 	 *                    --------------------------||||
 	 *				^		 ^
 	 *				|		log block
 	 *				payload
 	 */
 
 	evicted =
 	    l2arc_range_check_overlap(start, end, dev->l2ad_hand) ||
 	    l2arc_range_check_overlap(start, end, dev->l2ad_evict) ||
 	    l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, start) ||
 	    l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, end);
 
 	return (start >= dev->l2ad_start && end <= dev->l2ad_end &&
 	    asize > 0 && asize <= sizeof (l2arc_log_blk_phys_t) &&
 	    (!evicted || dev->l2ad_first));
 }
 
 /*
  * Inserts ARC buffer header `hdr' into the current L2ARC log block on
  * the device. The buffer being inserted must be present in L2ARC.
  * Returns B_TRUE if the L2ARC log block is full and needs to be committed
  * to L2ARC, or B_FALSE if it still has room for more ARC buffers.
  */
 static boolean_t
 l2arc_log_blk_insert(l2arc_dev_t *dev, const arc_buf_hdr_t *hdr)
 {
 	l2arc_log_blk_phys_t	*lb = &dev->l2ad_log_blk;
 	l2arc_log_ent_phys_t	*le;
 
 	if (dev->l2ad_log_entries == 0)
 		return (B_FALSE);
 
 	int index = dev->l2ad_log_ent_idx++;
 
 	ASSERT3S(index, <, dev->l2ad_log_entries);
 	ASSERT(HDR_HAS_L2HDR(hdr));
 
 	le = &lb->lb_entries[index];
 	memset(le, 0, sizeof (*le));
 	le->le_dva = hdr->b_dva;
 	le->le_birth = hdr->b_birth;
 	le->le_daddr = hdr->b_l2hdr.b_daddr;
 	if (index == 0)
 		dev->l2ad_log_blk_payload_start = le->le_daddr;
 	L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr));
 	L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr));
 	L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr));
 	le->le_complevel = hdr->b_complevel;
 	L2BLK_SET_TYPE((le)->le_prop, hdr->b_type);
 	L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr)));
 	L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr)));
 	L2BLK_SET_STATE((le)->le_prop, hdr->b_l2hdr.b_arcs_state);
 
 	dev->l2ad_log_blk_payload_asize += vdev_psize_to_asize(dev->l2ad_vdev,
 	    HDR_GET_PSIZE(hdr));
 
 	return (dev->l2ad_log_ent_idx == dev->l2ad_log_entries);
 }
 
 /*
  * Checks whether a given L2ARC device address sits in a time-sequential
  * range. The trick here is that the L2ARC is a rotary buffer, so we can't
  * just do a range comparison, we need to handle the situation in which the
  * range wraps around the end of the L2ARC device. Arguments:
  *	bottom -- Lower end of the range to check (written to earlier).
  *	top    -- Upper end of the range to check (written to later).
  *	check  -- The address for which we want to determine if it sits in
  *		  between the top and bottom.
  *
  * The 3-way conditional below represents the following cases:
  *
  *	bottom < top : Sequentially ordered case:
  *	  <check>--------+-------------------+
  *	                 |  (overlap here?)  |
  *	 L2ARC dev       V                   V
  *	 |---------------<bottom>============<top>--------------|
  *
  *	bottom > top: Looped-around case:
  *	                      <check>--------+------------------+
  *	                                     |  (overlap here?) |
  *	 L2ARC dev                           V                  V
  *	 |===============<top>---------------<bottom>===========|
  *	 ^               ^
  *	 |  (or here?)   |
  *	 +---------------+---------<check>
  *
  *	top == bottom : Just a single address comparison.
  */
 boolean_t
 l2arc_range_check_overlap(uint64_t bottom, uint64_t top, uint64_t check)
 {
 	if (bottom < top)
 		return (bottom <= check && check <= top);
 	else if (bottom > top)
 		return (check <= top || bottom <= check);
 	else
 		return (check == top);
 }
 
 EXPORT_SYMBOL(arc_buf_size);
 EXPORT_SYMBOL(arc_write);
 EXPORT_SYMBOL(arc_read);
 EXPORT_SYMBOL(arc_buf_info);
 EXPORT_SYMBOL(arc_getbuf_func);
 EXPORT_SYMBOL(arc_add_prune_callback);
 EXPORT_SYMBOL(arc_remove_prune_callback);
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min, param_set_arc_min,
 	spl_param_get_u64, ZMOD_RW, "Minimum ARC size in bytes");
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, max, param_set_arc_max,
 	spl_param_get_u64, ZMOD_RW, "Maximum ARC size in bytes");
 
 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, meta_balance, UINT, ZMOD_RW,
 	"Balance between metadata and data on ghost hits.");
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, grow_retry, param_set_arc_int,
 	param_get_uint, ZMOD_RW, "Seconds before growing ARC size");
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, shrink_shift, param_set_arc_int,
 	param_get_uint, ZMOD_RW, "log2(fraction of ARC to reclaim)");
 
 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, pc_percent, UINT, ZMOD_RW,
 	"Percent of pagecache to reclaim ARC to");
 
 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, average_blocksize, UINT, ZMOD_RD,
 	"Target average block size");
 
 ZFS_MODULE_PARAM(zfs, zfs_, compressed_arc_enabled, INT, ZMOD_RW,
 	"Disable compressed ARC buffers");
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min_prefetch_ms, param_set_arc_int,
 	param_get_uint, ZMOD_RW, "Min life of prefetch block in ms");
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min_prescient_prefetch_ms,
     param_set_arc_int, param_get_uint, ZMOD_RW,
 	"Min life of prescient prefetched block in ms");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, write_max, U64, ZMOD_RW,
 	"Max write bytes per interval");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, write_boost, U64, ZMOD_RW,
 	"Extra write bytes during device warmup");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, headroom, U64, ZMOD_RW,
 	"Number of max device writes to precache");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, headroom_boost, U64, ZMOD_RW,
 	"Compressed l2arc_headroom multiplier");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, trim_ahead, U64, ZMOD_RW,
 	"TRIM ahead L2ARC write size multiplier");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_secs, U64, ZMOD_RW,
 	"Seconds between L2ARC writing");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_min_ms, U64, ZMOD_RW,
 	"Min feed interval in milliseconds");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, noprefetch, INT, ZMOD_RW,
 	"Skip caching prefetched buffers");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, feed_again, INT, ZMOD_RW,
 	"Turbo L2ARC warmup");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, norw, INT, ZMOD_RW,
 	"No reads during writes");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, meta_percent, UINT, ZMOD_RW,
 	"Percent of ARC size allowed for L2ARC-only headers");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_enabled, INT, ZMOD_RW,
 	"Rebuild the L2ARC when importing a pool");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, rebuild_blocks_min_l2size, U64, ZMOD_RW,
 	"Min size in bytes to write rebuild log blocks in L2ARC");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, mfuonly, INT, ZMOD_RW,
 	"Cache only MFU data from ARC into L2ARC");
 
 ZFS_MODULE_PARAM(zfs_l2arc, l2arc_, exclude_special, INT, ZMOD_RW,
 	"Exclude dbufs on special vdevs from being cached to L2ARC if set.");
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, lotsfree_percent, param_set_arc_int,
 	param_get_uint, ZMOD_RW, "System free memory I/O throttle in bytes");
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, sys_free, param_set_arc_u64,
 	spl_param_get_u64, ZMOD_RW, "System free memory target size in bytes");
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, dnode_limit, param_set_arc_u64,
 	spl_param_get_u64, ZMOD_RW, "Minimum bytes of dnodes in ARC");
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, dnode_limit_percent,
     param_set_arc_int, param_get_uint, ZMOD_RW,
 	"Percent of ARC meta buffers for dnodes");
 
 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, dnode_reduce_percent, UINT, ZMOD_RW,
 	"Percentage of excess dnodes to try to unpin");
 
 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, eviction_pct, UINT, ZMOD_RW,
 	"When full, ARC allocation waits for eviction of this % of alloc size");
 
 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, evict_batch_limit, UINT, ZMOD_RW,
 	"The number of headers to evict per sublist before moving to the next");
 
 ZFS_MODULE_PARAM(zfs_arc, zfs_arc_, prune_task_threads, INT, ZMOD_RW,
 	"Number of arc_prune threads");
diff --git a/sys/contrib/openzfs/module/zfs/ddt.c b/sys/contrib/openzfs/module/zfs/ddt.c
index 1fb198219904..770a96785c6b 100644
--- a/sys/contrib/openzfs/module/zfs/ddt.c
+++ b/sys/contrib/openzfs/module/zfs/ddt.c
@@ -1,1248 +1,1249 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2016 by Delphix. All rights reserved.
  * Copyright (c) 2022 by Pawel Jakub Dawidek
  */
 
 #include <sys/zfs_context.h>
 #include <sys/spa.h>
 #include <sys/spa_impl.h>
 #include <sys/zio.h>
 #include <sys/ddt.h>
 #include <sys/zap.h>
 #include <sys/dmu_tx.h>
 #include <sys/arc.h>
 #include <sys/dsl_pool.h>
 #include <sys/zio_checksum.h>
 #include <sys/zio_compress.h>
 #include <sys/dsl_scan.h>
 #include <sys/abd.h>
 
 static kmem_cache_t *ddt_cache;
 static kmem_cache_t *ddt_entry_cache;
 
 /*
  * Enable/disable prefetching of dedup-ed blocks which are going to be freed.
  */
 int zfs_dedup_prefetch = 0;
 
 static const ddt_ops_t *const ddt_ops[DDT_TYPES] = {
 	&ddt_zap_ops,
 };
 
 static const char *const ddt_class_name[DDT_CLASSES] = {
 	"ditto",
 	"duplicate",
 	"unique",
 };
 
 static void
 ddt_object_create(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     dmu_tx_t *tx)
 {
 	spa_t *spa = ddt->ddt_spa;
 	objset_t *os = ddt->ddt_os;
 	uint64_t *objectp = &ddt->ddt_object[type][class];
 	boolean_t prehash = zio_checksum_table[ddt->ddt_checksum].ci_flags &
 	    ZCHECKSUM_FLAG_DEDUP;
 	char name[DDT_NAMELEN];
 
 	ddt_object_name(ddt, type, class, name);
 
 	ASSERT(*objectp == 0);
 	VERIFY(ddt_ops[type]->ddt_op_create(os, objectp, tx, prehash) == 0);
 	ASSERT(*objectp != 0);
 
 	VERIFY(zap_add(os, DMU_POOL_DIRECTORY_OBJECT, name,
 	    sizeof (uint64_t), 1, objectp, tx) == 0);
 
 	VERIFY(zap_add(os, spa->spa_ddt_stat_object, name,
 	    sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t),
 	    &ddt->ddt_histogram[type][class], tx) == 0);
 }
 
 static void
 ddt_object_destroy(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     dmu_tx_t *tx)
 {
 	spa_t *spa = ddt->ddt_spa;
 	objset_t *os = ddt->ddt_os;
 	uint64_t *objectp = &ddt->ddt_object[type][class];
 	uint64_t count;
 	char name[DDT_NAMELEN];
 
 	ddt_object_name(ddt, type, class, name);
 
 	ASSERT(*objectp != 0);
 	ASSERT(ddt_histogram_empty(&ddt->ddt_histogram[type][class]));
 	VERIFY(ddt_object_count(ddt, type, class, &count) == 0 && count == 0);
 	VERIFY(zap_remove(os, DMU_POOL_DIRECTORY_OBJECT, name, tx) == 0);
 	VERIFY(zap_remove(os, spa->spa_ddt_stat_object, name, tx) == 0);
 	VERIFY(ddt_ops[type]->ddt_op_destroy(os, *objectp, tx) == 0);
 	memset(&ddt->ddt_object_stats[type][class], 0, sizeof (ddt_object_t));
 
 	*objectp = 0;
 }
 
 static int
 ddt_object_load(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
 {
 	ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];
 	dmu_object_info_t doi;
 	uint64_t count;
 	char name[DDT_NAMELEN];
 	int error;
 
 	ddt_object_name(ddt, type, class, name);
 
 	error = zap_lookup(ddt->ddt_os, DMU_POOL_DIRECTORY_OBJECT, name,
 	    sizeof (uint64_t), 1, &ddt->ddt_object[type][class]);
 	if (error != 0)
 		return (error);
 
 	error = zap_lookup(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name,
 	    sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t),
 	    &ddt->ddt_histogram[type][class]);
 	if (error != 0)
 		return (error);
 
 	/*
 	 * Seed the cached statistics.
 	 */
 	error = ddt_object_info(ddt, type, class, &doi);
 	if (error)
 		return (error);
 
 	error = ddt_object_count(ddt, type, class, &count);
 	if (error)
 		return (error);
 
 	ddo->ddo_count = count;
 	ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9;
 	ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size;
 
 	return (0);
 }
 
 static void
 ddt_object_sync(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     dmu_tx_t *tx)
 {
 	ddt_object_t *ddo = &ddt->ddt_object_stats[type][class];
 	dmu_object_info_t doi;
 	uint64_t count;
 	char name[DDT_NAMELEN];
 
 	ddt_object_name(ddt, type, class, name);
 
 	VERIFY(zap_update(ddt->ddt_os, ddt->ddt_spa->spa_ddt_stat_object, name,
 	    sizeof (uint64_t), sizeof (ddt_histogram_t) / sizeof (uint64_t),
 	    &ddt->ddt_histogram[type][class], tx) == 0);
 
 	/*
 	 * Cache DDT statistics; this is the only time they'll change.
 	 */
 	VERIFY(ddt_object_info(ddt, type, class, &doi) == 0);
 	VERIFY(ddt_object_count(ddt, type, class, &count) == 0);
 
 	ddo->ddo_count = count;
 	ddo->ddo_dspace = doi.doi_physical_blocks_512 << 9;
 	ddo->ddo_mspace = doi.doi_fill_count * doi.doi_data_block_size;
 }
 
 static int
 ddt_object_lookup(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     ddt_entry_t *dde)
 {
 	if (!ddt_object_exists(ddt, type, class))
 		return (SET_ERROR(ENOENT));
 
 	return (ddt_ops[type]->ddt_op_lookup(ddt->ddt_os,
 	    ddt->ddt_object[type][class], dde));
 }
 
 static void
 ddt_object_prefetch(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     ddt_entry_t *dde)
 {
 	if (!ddt_object_exists(ddt, type, class))
 		return;
 
 	ddt_ops[type]->ddt_op_prefetch(ddt->ddt_os,
 	    ddt->ddt_object[type][class], dde);
 }
 
 int
 ddt_object_update(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     ddt_entry_t *dde, dmu_tx_t *tx)
 {
 	ASSERT(ddt_object_exists(ddt, type, class));
 
 	return (ddt_ops[type]->ddt_op_update(ddt->ddt_os,
 	    ddt->ddt_object[type][class], dde, tx));
 }
 
 static int
 ddt_object_remove(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     ddt_entry_t *dde, dmu_tx_t *tx)
 {
 	ASSERT(ddt_object_exists(ddt, type, class));
 
 	return (ddt_ops[type]->ddt_op_remove(ddt->ddt_os,
 	    ddt->ddt_object[type][class], dde, tx));
 }
 
 int
 ddt_object_walk(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     uint64_t *walk, ddt_entry_t *dde)
 {
 	ASSERT(ddt_object_exists(ddt, type, class));
 
 	return (ddt_ops[type]->ddt_op_walk(ddt->ddt_os,
 	    ddt->ddt_object[type][class], dde, walk));
 }
 
 int
 ddt_object_count(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     uint64_t *count)
 {
 	ASSERT(ddt_object_exists(ddt, type, class));
 
 	return (ddt_ops[type]->ddt_op_count(ddt->ddt_os,
 	    ddt->ddt_object[type][class], count));
 }
 
 int
 ddt_object_info(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     dmu_object_info_t *doi)
 {
 	if (!ddt_object_exists(ddt, type, class))
 		return (SET_ERROR(ENOENT));
 
 	return (dmu_object_info(ddt->ddt_os, ddt->ddt_object[type][class],
 	    doi));
 }
 
 boolean_t
 ddt_object_exists(ddt_t *ddt, enum ddt_type type, enum ddt_class class)
 {
 	return (!!ddt->ddt_object[type][class]);
 }
 
 void
 ddt_object_name(ddt_t *ddt, enum ddt_type type, enum ddt_class class,
     char *name)
 {
 	(void) snprintf(name, DDT_NAMELEN, DMU_POOL_DDT,
 	    zio_checksum_table[ddt->ddt_checksum].ci_name,
 	    ddt_ops[type]->ddt_op_name, ddt_class_name[class]);
 }
 
 void
 ddt_bp_fill(const ddt_phys_t *ddp, blkptr_t *bp, uint64_t txg)
 {
 	ASSERT(txg != 0);
 
 	for (int d = 0; d < SPA_DVAS_PER_BP; d++)
 		bp->blk_dva[d] = ddp->ddp_dva[d];
 	BP_SET_BIRTH(bp, txg, ddp->ddp_phys_birth);
 }
 
 /*
  * The bp created via this function may be used for repairs and scrub, but it
  * will be missing the salt / IV required to do a full decrypting read.
  */
 void
 ddt_bp_create(enum zio_checksum checksum,
     const ddt_key_t *ddk, const ddt_phys_t *ddp, blkptr_t *bp)
 {
 	BP_ZERO(bp);
 
 	if (ddp != NULL)
 		ddt_bp_fill(ddp, bp, ddp->ddp_phys_birth);
 
 	bp->blk_cksum = ddk->ddk_cksum;
 
 	BP_SET_LSIZE(bp, DDK_GET_LSIZE(ddk));
 	BP_SET_PSIZE(bp, DDK_GET_PSIZE(ddk));
 	BP_SET_COMPRESS(bp, DDK_GET_COMPRESS(ddk));
 	BP_SET_CRYPT(bp, DDK_GET_CRYPT(ddk));
 	BP_SET_FILL(bp, 1);
 	BP_SET_CHECKSUM(bp, checksum);
 	BP_SET_TYPE(bp, DMU_OT_DEDUP);
 	BP_SET_LEVEL(bp, 0);
 	BP_SET_DEDUP(bp, 1);
 	BP_SET_BYTEORDER(bp, ZFS_HOST_BYTEORDER);
 }
 
 void
 ddt_key_fill(ddt_key_t *ddk, const blkptr_t *bp)
 {
 	ddk->ddk_cksum = bp->blk_cksum;
 	ddk->ddk_prop = 0;
 
 	ASSERT(BP_IS_ENCRYPTED(bp) || !BP_USES_CRYPT(bp));
 
 	DDK_SET_LSIZE(ddk, BP_GET_LSIZE(bp));
 	DDK_SET_PSIZE(ddk, BP_GET_PSIZE(bp));
 	DDK_SET_COMPRESS(ddk, BP_GET_COMPRESS(bp));
 	DDK_SET_CRYPT(ddk, BP_USES_CRYPT(bp));
 }
 
 void
 ddt_phys_fill(ddt_phys_t *ddp, const blkptr_t *bp)
 {
 	ASSERT(ddp->ddp_phys_birth == 0);
 
 	for (int d = 0; d < SPA_DVAS_PER_BP; d++)
 		ddp->ddp_dva[d] = bp->blk_dva[d];
 	ddp->ddp_phys_birth = BP_PHYSICAL_BIRTH(bp);
 }
 
 void
 ddt_phys_clear(ddt_phys_t *ddp)
 {
 	memset(ddp, 0, sizeof (*ddp));
 }
 
 void
 ddt_phys_addref(ddt_phys_t *ddp)
 {
 	ddp->ddp_refcnt++;
 }
 
 void
 ddt_phys_decref(ddt_phys_t *ddp)
 {
 	if (ddp) {
 		ASSERT(ddp->ddp_refcnt > 0);
 		ddp->ddp_refcnt--;
 	}
 }
 
 void
 ddt_phys_free(ddt_t *ddt, ddt_key_t *ddk, ddt_phys_t *ddp, uint64_t txg)
 {
 	blkptr_t blk;
 
 	ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk);
 
 	/*
 	 * We clear the dedup bit so that zio_free() will actually free the
 	 * space, rather than just decrementing the refcount in the DDT.
 	 */
 	BP_SET_DEDUP(&blk, 0);
 
 	ddt_phys_clear(ddp);
 	zio_free(ddt->ddt_spa, txg, &blk);
 }
 
 ddt_phys_t *
 ddt_phys_select(const ddt_entry_t *dde, const blkptr_t *bp)
 {
 	ddt_phys_t *ddp = (ddt_phys_t *)dde->dde_phys;
 
 	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
 		if (DVA_EQUAL(BP_IDENTITY(bp), &ddp->ddp_dva[0]) &&
 		    BP_PHYSICAL_BIRTH(bp) == ddp->ddp_phys_birth)
 			return (ddp);
 	}
 	return (NULL);
 }
 
 uint64_t
 ddt_phys_total_refcnt(const ddt_entry_t *dde)
 {
 	uint64_t refcnt = 0;
 
 	for (int p = DDT_PHYS_SINGLE; p <= DDT_PHYS_TRIPLE; p++)
 		refcnt += dde->dde_phys[p].ddp_refcnt;
 
 	return (refcnt);
 }
 
 static void
 ddt_stat_generate(ddt_t *ddt, ddt_entry_t *dde, ddt_stat_t *dds)
 {
 	spa_t *spa = ddt->ddt_spa;
 	ddt_phys_t *ddp = dde->dde_phys;
 	ddt_key_t *ddk = &dde->dde_key;
 	uint64_t lsize = DDK_GET_LSIZE(ddk);
 	uint64_t psize = DDK_GET_PSIZE(ddk);
 
 	memset(dds, 0, sizeof (*dds));
 
 	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
 		uint64_t dsize = 0;
 		uint64_t refcnt = ddp->ddp_refcnt;
 
 		if (ddp->ddp_phys_birth == 0)
 			continue;
 
 		for (int d = 0; d < DDE_GET_NDVAS(dde); d++)
 			dsize += dva_get_dsize_sync(spa, &ddp->ddp_dva[d]);
 
 		dds->dds_blocks += 1;
 		dds->dds_lsize += lsize;
 		dds->dds_psize += psize;
 		dds->dds_dsize += dsize;
 
 		dds->dds_ref_blocks += refcnt;
 		dds->dds_ref_lsize += lsize * refcnt;
 		dds->dds_ref_psize += psize * refcnt;
 		dds->dds_ref_dsize += dsize * refcnt;
 	}
 }
 
 void
 ddt_stat_add(ddt_stat_t *dst, const ddt_stat_t *src, uint64_t neg)
 {
 	const uint64_t *s = (const uint64_t *)src;
 	uint64_t *d = (uint64_t *)dst;
 	uint64_t *d_end = (uint64_t *)(dst + 1);
 
 	ASSERT(neg == 0 || neg == -1ULL);	/* add or subtract */
 
 	for (int i = 0; i < d_end - d; i++)
 		d[i] += (s[i] ^ neg) - neg;
 }
 
 static void
 ddt_stat_update(ddt_t *ddt, ddt_entry_t *dde, uint64_t neg)
 {
 	ddt_stat_t dds;
 	ddt_histogram_t *ddh;
 	int bucket;
 
 	ddt_stat_generate(ddt, dde, &dds);
 
 	bucket = highbit64(dds.dds_ref_blocks) - 1;
-	ASSERT(bucket >= 0);
+	if (unlikely(bucket >= 0))	/* if() needed for GCC bounds check */
+		ASSERT(bucket >= 0);
 
 	ddh = &ddt->ddt_histogram[dde->dde_type][dde->dde_class];
 
 	ddt_stat_add(&ddh->ddh_stat[bucket], &dds, neg);
 }
 
 void
 ddt_histogram_add(ddt_histogram_t *dst, const ddt_histogram_t *src)
 {
 	for (int h = 0; h < 64; h++)
 		ddt_stat_add(&dst->ddh_stat[h], &src->ddh_stat[h], 0);
 }
 
 void
 ddt_histogram_stat(ddt_stat_t *dds, const ddt_histogram_t *ddh)
 {
 	memset(dds, 0, sizeof (*dds));
 
 	for (int h = 0; h < 64; h++)
 		ddt_stat_add(dds, &ddh->ddh_stat[h], 0);
 }
 
 boolean_t
 ddt_histogram_empty(const ddt_histogram_t *ddh)
 {
 	const uint64_t *s = (const uint64_t *)ddh;
 	const uint64_t *s_end = (const uint64_t *)(ddh + 1);
 
 	while (s < s_end)
 		if (*s++ != 0)
 			return (B_FALSE);
 
 	return (B_TRUE);
 }
 
 void
 ddt_get_dedup_object_stats(spa_t *spa, ddt_object_t *ddo_total)
 {
 	/* Sum the statistics we cached in ddt_object_sync(). */
 	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
 		ddt_t *ddt = spa->spa_ddt[c];
 		for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
 			for (enum ddt_class class = 0; class < DDT_CLASSES;
 			    class++) {
 				ddt_object_t *ddo =
 				    &ddt->ddt_object_stats[type][class];
 				ddo_total->ddo_count += ddo->ddo_count;
 				ddo_total->ddo_dspace += ddo->ddo_dspace;
 				ddo_total->ddo_mspace += ddo->ddo_mspace;
 			}
 		}
 	}
 
 	/* ... and compute the averages. */
 	if (ddo_total->ddo_count != 0) {
 		ddo_total->ddo_dspace /= ddo_total->ddo_count;
 		ddo_total->ddo_mspace /= ddo_total->ddo_count;
 	}
 }
 
 void
 ddt_get_dedup_histogram(spa_t *spa, ddt_histogram_t *ddh)
 {
 	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
 		ddt_t *ddt = spa->spa_ddt[c];
 		for (enum ddt_type type = 0; type < DDT_TYPES && ddt; type++) {
 			for (enum ddt_class class = 0; class < DDT_CLASSES;
 			    class++) {
 				ddt_histogram_add(ddh,
 				    &ddt->ddt_histogram_cache[type][class]);
 			}
 		}
 	}
 }
 
 void
 ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total)
 {
 	ddt_histogram_t *ddh_total;
 
 	ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP);
 	ddt_get_dedup_histogram(spa, ddh_total);
 	ddt_histogram_stat(dds_total, ddh_total);
 	kmem_free(ddh_total, sizeof (ddt_histogram_t));
 }
 
 uint64_t
 ddt_get_dedup_dspace(spa_t *spa)
 {
 	ddt_stat_t dds_total;
 
 	if (spa->spa_dedup_dspace != ~0ULL)
 		return (spa->spa_dedup_dspace);
 
 	memset(&dds_total, 0, sizeof (ddt_stat_t));
 
 	/* Calculate and cache the stats */
 	ddt_get_dedup_stats(spa, &dds_total);
 	spa->spa_dedup_dspace = dds_total.dds_ref_dsize - dds_total.dds_dsize;
 	return (spa->spa_dedup_dspace);
 }
 
 uint64_t
 ddt_get_pool_dedup_ratio(spa_t *spa)
 {
 	ddt_stat_t dds_total = { 0 };
 
 	ddt_get_dedup_stats(spa, &dds_total);
 	if (dds_total.dds_dsize == 0)
 		return (100);
 
 	return (dds_total.dds_ref_dsize * 100 / dds_total.dds_dsize);
 }
 
 size_t
 ddt_compress(void *src, uchar_t *dst, size_t s_len, size_t d_len)
 {
 	uchar_t *version = dst++;
 	int cpfunc = ZIO_COMPRESS_ZLE;
 	zio_compress_info_t *ci = &zio_compress_table[cpfunc];
 	size_t c_len;
 
 	ASSERT(d_len >= s_len + 1);	/* no compression plus version byte */
 
 	c_len = ci->ci_compress(src, dst, s_len, d_len - 1, ci->ci_level);
 
 	if (c_len == s_len) {
 		cpfunc = ZIO_COMPRESS_OFF;
 		memcpy(dst, src, s_len);
 	}
 
 	*version = cpfunc;
 	if (ZFS_HOST_BYTEORDER)
 		*version |= DDT_COMPRESS_BYTEORDER_MASK;
 
 	return (c_len + 1);
 }
 
 void
 ddt_decompress(uchar_t *src, void *dst, size_t s_len, size_t d_len)
 {
 	uchar_t version = *src++;
 	int cpfunc = version & DDT_COMPRESS_FUNCTION_MASK;
 	zio_compress_info_t *ci = &zio_compress_table[cpfunc];
 
 	if (ci->ci_decompress != NULL)
 		(void) ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level);
 	else
 		memcpy(dst, src, d_len);
 
 	if (((version & DDT_COMPRESS_BYTEORDER_MASK) != 0) !=
 	    (ZFS_HOST_BYTEORDER != 0))
 		byteswap_uint64_array(dst, d_len);
 }
 
 ddt_t *
 ddt_select(spa_t *spa, const blkptr_t *bp)
 {
 	return (spa->spa_ddt[BP_GET_CHECKSUM(bp)]);
 }
 
 void
 ddt_enter(ddt_t *ddt)
 {
 	mutex_enter(&ddt->ddt_lock);
 }
 
 void
 ddt_exit(ddt_t *ddt)
 {
 	mutex_exit(&ddt->ddt_lock);
 }
 
 void
 ddt_init(void)
 {
 	ddt_cache = kmem_cache_create("ddt_cache",
 	    sizeof (ddt_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
 	ddt_entry_cache = kmem_cache_create("ddt_entry_cache",
 	    sizeof (ddt_entry_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
 }
 
 void
 ddt_fini(void)
 {
 	kmem_cache_destroy(ddt_entry_cache);
 	kmem_cache_destroy(ddt_cache);
 }
 
 static ddt_entry_t *
 ddt_alloc(const ddt_key_t *ddk)
 {
 	ddt_entry_t *dde;
 
 	dde = kmem_cache_alloc(ddt_entry_cache, KM_SLEEP);
 	memset(dde, 0, sizeof (ddt_entry_t));
 	cv_init(&dde->dde_cv, NULL, CV_DEFAULT, NULL);
 
 	dde->dde_key = *ddk;
 
 	return (dde);
 }
 
 static void
 ddt_free(ddt_entry_t *dde)
 {
 	ASSERT(!dde->dde_loading);
 
 	for (int p = 0; p < DDT_PHYS_TYPES; p++)
 		ASSERT(dde->dde_lead_zio[p] == NULL);
 
 	if (dde->dde_repair_abd != NULL)
 		abd_free(dde->dde_repair_abd);
 
 	cv_destroy(&dde->dde_cv);
 	kmem_cache_free(ddt_entry_cache, dde);
 }
 
 void
 ddt_remove(ddt_t *ddt, ddt_entry_t *dde)
 {
 	ASSERT(MUTEX_HELD(&ddt->ddt_lock));
 
 	avl_remove(&ddt->ddt_tree, dde);
 	ddt_free(dde);
 }
 
 ddt_entry_t *
 ddt_lookup(ddt_t *ddt, const blkptr_t *bp, boolean_t add)
 {
 	ddt_entry_t *dde, dde_search;
 	enum ddt_type type;
 	enum ddt_class class;
 	avl_index_t where;
 	int error;
 
 	ASSERT(MUTEX_HELD(&ddt->ddt_lock));
 
 	ddt_key_fill(&dde_search.dde_key, bp);
 
 	dde = avl_find(&ddt->ddt_tree, &dde_search, &where);
 	if (dde == NULL) {
 		if (!add)
 			return (NULL);
 		dde = ddt_alloc(&dde_search.dde_key);
 		avl_insert(&ddt->ddt_tree, dde, where);
 	}
 
 	while (dde->dde_loading)
 		cv_wait(&dde->dde_cv, &ddt->ddt_lock);
 
 	if (dde->dde_loaded)
 		return (dde);
 
 	dde->dde_loading = B_TRUE;
 
 	ddt_exit(ddt);
 
 	error = ENOENT;
 
 	for (type = 0; type < DDT_TYPES; type++) {
 		for (class = 0; class < DDT_CLASSES; class++) {
 			error = ddt_object_lookup(ddt, type, class, dde);
 			if (error != ENOENT) {
 				ASSERT0(error);
 				break;
 			}
 		}
 		if (error != ENOENT)
 			break;
 	}
 
 	ddt_enter(ddt);
 
 	ASSERT(dde->dde_loaded == B_FALSE);
 	ASSERT(dde->dde_loading == B_TRUE);
 
 	dde->dde_type = type;	/* will be DDT_TYPES if no entry found */
 	dde->dde_class = class;	/* will be DDT_CLASSES if no entry found */
 	dde->dde_loaded = B_TRUE;
 	dde->dde_loading = B_FALSE;
 
 	if (error == 0)
 		ddt_stat_update(ddt, dde, -1ULL);
 
 	cv_broadcast(&dde->dde_cv);
 
 	return (dde);
 }
 
 void
 ddt_prefetch(spa_t *spa, const blkptr_t *bp)
 {
 	ddt_t *ddt;
 	ddt_entry_t dde;
 
 	if (!zfs_dedup_prefetch || bp == NULL || !BP_GET_DEDUP(bp))
 		return;
 
 	/*
 	 * We only remove the DDT once all tables are empty and only
 	 * prefetch dedup blocks when there are entries in the DDT.
 	 * Thus no locking is required as the DDT can't disappear on us.
 	 */
 	ddt = ddt_select(spa, bp);
 	ddt_key_fill(&dde.dde_key, bp);
 
 	for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
 		for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
 			ddt_object_prefetch(ddt, type, class, &dde);
 		}
 	}
 }
 
 /*
  * Opaque struct used for ddt_key comparison
  */
 #define	DDT_KEY_CMP_LEN	(sizeof (ddt_key_t) / sizeof (uint16_t))
 
 typedef struct ddt_key_cmp {
 	uint16_t	u16[DDT_KEY_CMP_LEN];
 } ddt_key_cmp_t;
 
 int
 ddt_entry_compare(const void *x1, const void *x2)
 {
 	const ddt_entry_t *dde1 = x1;
 	const ddt_entry_t *dde2 = x2;
 	const ddt_key_cmp_t *k1 = (const ddt_key_cmp_t *)&dde1->dde_key;
 	const ddt_key_cmp_t *k2 = (const ddt_key_cmp_t *)&dde2->dde_key;
 	int32_t cmp = 0;
 
 	for (int i = 0; i < DDT_KEY_CMP_LEN; i++) {
 		cmp = (int32_t)k1->u16[i] - (int32_t)k2->u16[i];
 		if (likely(cmp))
 			break;
 	}
 
 	return (TREE_ISIGN(cmp));
 }
 
 static ddt_t *
 ddt_table_alloc(spa_t *spa, enum zio_checksum c)
 {
 	ddt_t *ddt;
 
 	ddt = kmem_cache_alloc(ddt_cache, KM_SLEEP);
 	memset(ddt, 0, sizeof (ddt_t));
 
 	mutex_init(&ddt->ddt_lock, NULL, MUTEX_DEFAULT, NULL);
 	avl_create(&ddt->ddt_tree, ddt_entry_compare,
 	    sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node));
 	avl_create(&ddt->ddt_repair_tree, ddt_entry_compare,
 	    sizeof (ddt_entry_t), offsetof(ddt_entry_t, dde_node));
 	ddt->ddt_checksum = c;
 	ddt->ddt_spa = spa;
 	ddt->ddt_os = spa->spa_meta_objset;
 
 	return (ddt);
 }
 
 static void
 ddt_table_free(ddt_t *ddt)
 {
 	ASSERT(avl_numnodes(&ddt->ddt_tree) == 0);
 	ASSERT(avl_numnodes(&ddt->ddt_repair_tree) == 0);
 	avl_destroy(&ddt->ddt_tree);
 	avl_destroy(&ddt->ddt_repair_tree);
 	mutex_destroy(&ddt->ddt_lock);
 	kmem_cache_free(ddt_cache, ddt);
 }
 
 void
 ddt_create(spa_t *spa)
 {
 	spa->spa_dedup_checksum = ZIO_DEDUPCHECKSUM;
 
 	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++)
 		spa->spa_ddt[c] = ddt_table_alloc(spa, c);
 }
 
 int
 ddt_load(spa_t *spa)
 {
 	int error;
 
 	ddt_create(spa);
 
 	error = zap_lookup(spa->spa_meta_objset, DMU_POOL_DIRECTORY_OBJECT,
 	    DMU_POOL_DDT_STATS, sizeof (uint64_t), 1,
 	    &spa->spa_ddt_stat_object);
 
 	if (error)
 		return (error == ENOENT ? 0 : error);
 
 	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
 		ddt_t *ddt = spa->spa_ddt[c];
 		for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
 			for (enum ddt_class class = 0; class < DDT_CLASSES;
 			    class++) {
 				error = ddt_object_load(ddt, type, class);
 				if (error != 0 && error != ENOENT)
 					return (error);
 			}
 		}
 
 		/*
 		 * Seed the cached histograms.
 		 */
 		memcpy(&ddt->ddt_histogram_cache, ddt->ddt_histogram,
 		    sizeof (ddt->ddt_histogram));
 		spa->spa_dedup_dspace = ~0ULL;
 	}
 
 	return (0);
 }
 
 void
 ddt_unload(spa_t *spa)
 {
 	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
 		if (spa->spa_ddt[c]) {
 			ddt_table_free(spa->spa_ddt[c]);
 			spa->spa_ddt[c] = NULL;
 		}
 	}
 }
 
 boolean_t
 ddt_class_contains(spa_t *spa, enum ddt_class max_class, const blkptr_t *bp)
 {
 	ddt_t *ddt;
 	ddt_entry_t *dde;
 
 	if (!BP_GET_DEDUP(bp))
 		return (B_FALSE);
 
 	if (max_class == DDT_CLASS_UNIQUE)
 		return (B_TRUE);
 
 	ddt = spa->spa_ddt[BP_GET_CHECKSUM(bp)];
 	dde = kmem_cache_alloc(ddt_entry_cache, KM_SLEEP);
 
 	ddt_key_fill(&(dde->dde_key), bp);
 
 	for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
 		for (enum ddt_class class = 0; class <= max_class; class++) {
 			if (ddt_object_lookup(ddt, type, class, dde) == 0) {
 				kmem_cache_free(ddt_entry_cache, dde);
 				return (B_TRUE);
 			}
 		}
 	}
 
 	kmem_cache_free(ddt_entry_cache, dde);
 	return (B_FALSE);
 }
 
 ddt_entry_t *
 ddt_repair_start(ddt_t *ddt, const blkptr_t *bp)
 {
 	ddt_key_t ddk;
 	ddt_entry_t *dde;
 
 	ddt_key_fill(&ddk, bp);
 
 	dde = ddt_alloc(&ddk);
 
 	for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
 		for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
 			/*
 			 * We can only do repair if there are multiple copies
 			 * of the block.  For anything in the UNIQUE class,
 			 * there's definitely only one copy, so don't even try.
 			 */
 			if (class != DDT_CLASS_UNIQUE &&
 			    ddt_object_lookup(ddt, type, class, dde) == 0)
 				return (dde);
 		}
 	}
 
 	memset(dde->dde_phys, 0, sizeof (dde->dde_phys));
 
 	return (dde);
 }
 
 void
 ddt_repair_done(ddt_t *ddt, ddt_entry_t *dde)
 {
 	avl_index_t where;
 
 	ddt_enter(ddt);
 
 	if (dde->dde_repair_abd != NULL && spa_writeable(ddt->ddt_spa) &&
 	    avl_find(&ddt->ddt_repair_tree, dde, &where) == NULL)
 		avl_insert(&ddt->ddt_repair_tree, dde, where);
 	else
 		ddt_free(dde);
 
 	ddt_exit(ddt);
 }
 
 static void
 ddt_repair_entry_done(zio_t *zio)
 {
 	ddt_entry_t *rdde = zio->io_private;
 
 	ddt_free(rdde);
 }
 
 static void
 ddt_repair_entry(ddt_t *ddt, ddt_entry_t *dde, ddt_entry_t *rdde, zio_t *rio)
 {
 	ddt_phys_t *ddp = dde->dde_phys;
 	ddt_phys_t *rddp = rdde->dde_phys;
 	ddt_key_t *ddk = &dde->dde_key;
 	ddt_key_t *rddk = &rdde->dde_key;
 	zio_t *zio;
 	blkptr_t blk;
 
 	zio = zio_null(rio, rio->io_spa, NULL,
 	    ddt_repair_entry_done, rdde, rio->io_flags);
 
 	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++, rddp++) {
 		if (ddp->ddp_phys_birth == 0 ||
 		    ddp->ddp_phys_birth != rddp->ddp_phys_birth ||
 		    memcmp(ddp->ddp_dva, rddp->ddp_dva, sizeof (ddp->ddp_dva)))
 			continue;
 		ddt_bp_create(ddt->ddt_checksum, ddk, ddp, &blk);
 		zio_nowait(zio_rewrite(zio, zio->io_spa, 0, &blk,
 		    rdde->dde_repair_abd, DDK_GET_PSIZE(rddk), NULL, NULL,
 		    ZIO_PRIORITY_SYNC_WRITE, ZIO_DDT_CHILD_FLAGS(zio), NULL));
 	}
 
 	zio_nowait(zio);
 }
 
 static void
 ddt_repair_table(ddt_t *ddt, zio_t *rio)
 {
 	spa_t *spa = ddt->ddt_spa;
 	ddt_entry_t *dde, *rdde_next, *rdde;
 	avl_tree_t *t = &ddt->ddt_repair_tree;
 	blkptr_t blk;
 
 	if (spa_sync_pass(spa) > 1)
 		return;
 
 	ddt_enter(ddt);
 	for (rdde = avl_first(t); rdde != NULL; rdde = rdde_next) {
 		rdde_next = AVL_NEXT(t, rdde);
 		avl_remove(&ddt->ddt_repair_tree, rdde);
 		ddt_exit(ddt);
 		ddt_bp_create(ddt->ddt_checksum, &rdde->dde_key, NULL, &blk);
 		dde = ddt_repair_start(ddt, &blk);
 		ddt_repair_entry(ddt, dde, rdde, rio);
 		ddt_repair_done(ddt, dde);
 		ddt_enter(ddt);
 	}
 	ddt_exit(ddt);
 }
 
 static void
 ddt_sync_entry(ddt_t *ddt, ddt_entry_t *dde, dmu_tx_t *tx, uint64_t txg)
 {
 	dsl_pool_t *dp = ddt->ddt_spa->spa_dsl_pool;
 	ddt_phys_t *ddp = dde->dde_phys;
 	ddt_key_t *ddk = &dde->dde_key;
 	enum ddt_type otype = dde->dde_type;
 	enum ddt_type ntype = DDT_TYPE_CURRENT;
 	enum ddt_class oclass = dde->dde_class;
 	enum ddt_class nclass;
 	uint64_t total_refcnt = 0;
 
 	ASSERT(dde->dde_loaded);
 	ASSERT(!dde->dde_loading);
 
 	for (int p = 0; p < DDT_PHYS_TYPES; p++, ddp++) {
 		ASSERT(dde->dde_lead_zio[p] == NULL);
 		if (ddp->ddp_phys_birth == 0) {
 			ASSERT(ddp->ddp_refcnt == 0);
 			continue;
 		}
 		if (p == DDT_PHYS_DITTO) {
 			/*
 			 * Note, we no longer create DDT-DITTO blocks, but we
 			 * don't want to leak any written by older software.
 			 */
 			ddt_phys_free(ddt, ddk, ddp, txg);
 			continue;
 		}
 		if (ddp->ddp_refcnt == 0)
 			ddt_phys_free(ddt, ddk, ddp, txg);
 		total_refcnt += ddp->ddp_refcnt;
 	}
 
 	/* We do not create new DDT-DITTO blocks. */
 	ASSERT0(dde->dde_phys[DDT_PHYS_DITTO].ddp_phys_birth);
 	if (total_refcnt > 1)
 		nclass = DDT_CLASS_DUPLICATE;
 	else
 		nclass = DDT_CLASS_UNIQUE;
 
 	if (otype != DDT_TYPES &&
 	    (otype != ntype || oclass != nclass || total_refcnt == 0)) {
 		VERIFY(ddt_object_remove(ddt, otype, oclass, dde, tx) == 0);
 		ASSERT(ddt_object_lookup(ddt, otype, oclass, dde) == ENOENT);
 	}
 
 	if (total_refcnt != 0) {
 		dde->dde_type = ntype;
 		dde->dde_class = nclass;
 		ddt_stat_update(ddt, dde, 0);
 		if (!ddt_object_exists(ddt, ntype, nclass))
 			ddt_object_create(ddt, ntype, nclass, tx);
 		VERIFY(ddt_object_update(ddt, ntype, nclass, dde, tx) == 0);
 
 		/*
 		 * If the class changes, the order that we scan this bp
 		 * changes.  If it decreases, we could miss it, so
 		 * scan it right now.  (This covers both class changing
 		 * while we are doing ddt_walk(), and when we are
 		 * traversing.)
 		 */
 		if (nclass < oclass) {
 			dsl_scan_ddt_entry(dp->dp_scan,
 			    ddt->ddt_checksum, dde, tx);
 		}
 	}
 }
 
 static void
 ddt_sync_table(ddt_t *ddt, dmu_tx_t *tx, uint64_t txg)
 {
 	spa_t *spa = ddt->ddt_spa;
 	ddt_entry_t *dde;
 	void *cookie = NULL;
 
 	if (avl_numnodes(&ddt->ddt_tree) == 0)
 		return;
 
 	ASSERT(spa->spa_uberblock.ub_version >= SPA_VERSION_DEDUP);
 
 	if (spa->spa_ddt_stat_object == 0) {
 		spa->spa_ddt_stat_object = zap_create_link(ddt->ddt_os,
 		    DMU_OT_DDT_STATS, DMU_POOL_DIRECTORY_OBJECT,
 		    DMU_POOL_DDT_STATS, tx);
 	}
 
 	while ((dde = avl_destroy_nodes(&ddt->ddt_tree, &cookie)) != NULL) {
 		ddt_sync_entry(ddt, dde, tx, txg);
 		ddt_free(dde);
 	}
 
 	for (enum ddt_type type = 0; type < DDT_TYPES; type++) {
 		uint64_t add, count = 0;
 		for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
 			if (ddt_object_exists(ddt, type, class)) {
 				ddt_object_sync(ddt, type, class, tx);
 				VERIFY(ddt_object_count(ddt, type, class,
 				    &add) == 0);
 				count += add;
 			}
 		}
 		for (enum ddt_class class = 0; class < DDT_CLASSES; class++) {
 			if (count == 0 && ddt_object_exists(ddt, type, class))
 				ddt_object_destroy(ddt, type, class, tx);
 		}
 	}
 
 	memcpy(&ddt->ddt_histogram_cache, ddt->ddt_histogram,
 	    sizeof (ddt->ddt_histogram));
 	spa->spa_dedup_dspace = ~0ULL;
 }
 
 void
 ddt_sync(spa_t *spa, uint64_t txg)
 {
 	dsl_scan_t *scn = spa->spa_dsl_pool->dp_scan;
 	dmu_tx_t *tx;
 	zio_t *rio;
 
 	ASSERT(spa_syncing_txg(spa) == txg);
 
 	tx = dmu_tx_create_assigned(spa->spa_dsl_pool, txg);
 
 	rio = zio_root(spa, NULL, NULL,
 	    ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SELF_HEAL);
 
 	/*
 	 * This function may cause an immediate scan of ddt blocks (see
 	 * the comment above dsl_scan_ddt() for details). We set the
 	 * scan's root zio here so that we can wait for any scan IOs in
 	 * addition to the regular ddt IOs.
 	 */
 	ASSERT3P(scn->scn_zio_root, ==, NULL);
 	scn->scn_zio_root = rio;
 
 	for (enum zio_checksum c = 0; c < ZIO_CHECKSUM_FUNCTIONS; c++) {
 		ddt_t *ddt = spa->spa_ddt[c];
 		if (ddt == NULL)
 			continue;
 		ddt_sync_table(ddt, tx, txg);
 		ddt_repair_table(ddt, rio);
 	}
 
 	(void) zio_wait(rio);
 	scn->scn_zio_root = NULL;
 
 	dmu_tx_commit(tx);
 }
 
 int
 ddt_walk(spa_t *spa, ddt_bookmark_t *ddb, ddt_entry_t *dde)
 {
 	do {
 		do {
 			do {
 				ddt_t *ddt = spa->spa_ddt[ddb->ddb_checksum];
 				int error = ENOENT;
 				if (ddt_object_exists(ddt, ddb->ddb_type,
 				    ddb->ddb_class)) {
 					error = ddt_object_walk(ddt,
 					    ddb->ddb_type, ddb->ddb_class,
 					    &ddb->ddb_cursor, dde);
 				}
 				dde->dde_type = ddb->ddb_type;
 				dde->dde_class = ddb->ddb_class;
 				if (error == 0)
 					return (0);
 				if (error != ENOENT)
 					return (error);
 				ddb->ddb_cursor = 0;
 			} while (++ddb->ddb_checksum < ZIO_CHECKSUM_FUNCTIONS);
 			ddb->ddb_checksum = 0;
 		} while (++ddb->ddb_type < DDT_TYPES);
 		ddb->ddb_type = 0;
 	} while (++ddb->ddb_class < DDT_CLASSES);
 
 	return (SET_ERROR(ENOENT));
 }
 
 /*
  * This function is used by Block Cloning (brt.c) to increase reference
  * counter for the DDT entry if the block is already in DDT.
  *
  * Return false if the block, despite having the D bit set, is not present
  * in the DDT. Currently this is not possible but might be in the future.
  * See the comment below.
  */
 boolean_t
 ddt_addref(spa_t *spa, const blkptr_t *bp)
 {
 	ddt_t *ddt;
 	ddt_entry_t *dde;
 	boolean_t result;
 
 	spa_config_enter(spa, SCL_ZIO, FTAG, RW_READER);
 	ddt = ddt_select(spa, bp);
 	ddt_enter(ddt);
 
 	dde = ddt_lookup(ddt, bp, B_TRUE);
 	ASSERT(dde != NULL);
 
 	if (dde->dde_type < DDT_TYPES) {
 		ddt_phys_t *ddp;
 
 		ASSERT3S(dde->dde_class, <, DDT_CLASSES);
 
 		ddp = &dde->dde_phys[BP_GET_NDVAS(bp)];
 
 		/*
 		 * This entry already existed (dde_type is real), so it must
 		 * have refcnt >0 at the start of this txg. We are called from
 		 * brt_pending_apply(), before frees are issued, so the refcnt
 		 * can't be lowered yet. Therefore, it must be >0. We assert
 		 * this because if the order of BRT and DDT interactions were
 		 * ever to change and the refcnt was ever zero here, then
 		 * likely further action is required to fill out the DDT entry,
 		 * and this is a place that is likely to be missed in testing.
 		 */
 		ASSERT3U(ddp->ddp_refcnt, >, 0);
 
 		ddt_phys_addref(ddp);
 		result = B_TRUE;
 	} else {
 		/*
 		 * At the time of implementating this if the block has the
 		 * DEDUP flag set it must exist in the DEDUP table, but
 		 * there are many advocates that want ability to remove
 		 * entries from DDT with refcnt=1. If this will happen,
 		 * we may have a block with the DEDUP set, but which doesn't
 		 * have a corresponding entry in the DDT. Be ready.
 		 */
 		ASSERT3S(dde->dde_class, ==, DDT_CLASSES);
 		ddt_remove(ddt, dde);
 		result = B_FALSE;
 	}
 
 	ddt_exit(ddt);
 	spa_config_exit(spa, SCL_ZIO, FTAG);
 
 	return (result);
 }
 
 ZFS_MODULE_PARAM(zfs_dedup, zfs_dedup_, prefetch, INT, ZMOD_RW,
 	"Enable prefetching dedup-ed blks");
diff --git a/sys/contrib/openzfs/module/zfs/dmu_objset.c b/sys/contrib/openzfs/module/zfs/dmu_objset.c
index 5ea2fca9db1e..ff3af5721a04 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_objset.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_objset.c
@@ -1,3102 +1,3111 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  * Copyright (c) 2015, STRATO AG, Inc. All rights reserved.
  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
  * Copyright 2017 Nexenta Systems, Inc.
  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
  * Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
  * Copyright (c) 2019, Klara Inc.
  * Copyright (c) 2019, Allan Jude
  * Copyright (c) 2022 Hewlett Packard Enterprise Development LP.
+ * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
 
 #include <sys/cred.h>
 #include <sys/zfs_context.h>
 #include <sys/dmu_objset.h>
 #include <sys/dsl_dir.h>
 #include <sys/dsl_dataset.h>
 #include <sys/dsl_prop.h>
 #include <sys/dsl_pool.h>
 #include <sys/dsl_synctask.h>
 #include <sys/dsl_deleg.h>
 #include <sys/dnode.h>
 #include <sys/dbuf.h>
 #include <sys/zvol.h>
 #include <sys/dmu_tx.h>
 #include <sys/zap.h>
 #include <sys/zil.h>
 #include <sys/dmu_impl.h>
 #include <sys/zfs_ioctl.h>
 #include <sys/sa.h>
 #include <sys/zfs_onexit.h>
 #include <sys/dsl_destroy.h>
 #include <sys/vdev.h>
 #include <sys/zfeature.h>
 #include <sys/policy.h>
 #include <sys/spa_impl.h>
 #include <sys/dmu_recv.h>
 #include <sys/zfs_project.h>
 #include "zfs_namecheck.h"
 #include <sys/vdev_impl.h>
 #include <sys/arc.h>
+#include <sys/cred.h>
 
 /*
  * Needed to close a window in dnode_move() that allows the objset to be freed
  * before it can be safely accessed.
  */
 krwlock_t os_lock;
 
 /*
  * Tunable to overwrite the maximum number of threads for the parallelization
  * of dmu_objset_find_dp, needed to speed up the import of pools with many
  * datasets.
  * Default is 4 times the number of leaf vdevs.
  */
 static const int dmu_find_threads = 0;
 
 /*
  * Backfill lower metadnode objects after this many have been freed.
  * Backfilling negatively impacts object creation rates, so only do it
  * if there are enough holes to fill.
  */
 static const int dmu_rescan_dnode_threshold = 1 << DN_MAX_INDBLKSHIFT;
 
 static const char *upgrade_tag = "upgrade_tag";
 
 static void dmu_objset_find_dp_cb(void *arg);
 
 static void dmu_objset_upgrade(objset_t *os, dmu_objset_upgrade_cb_t cb);
 static void dmu_objset_upgrade_stop(objset_t *os);
 
 void
 dmu_objset_init(void)
 {
 	rw_init(&os_lock, NULL, RW_DEFAULT, NULL);
 }
 
 void
 dmu_objset_fini(void)
 {
 	rw_destroy(&os_lock);
 }
 
 spa_t *
 dmu_objset_spa(objset_t *os)
 {
 	return (os->os_spa);
 }
 
 zilog_t *
 dmu_objset_zil(objset_t *os)
 {
 	return (os->os_zil);
 }
 
 dsl_pool_t *
 dmu_objset_pool(objset_t *os)
 {
 	dsl_dataset_t *ds;
 
 	if ((ds = os->os_dsl_dataset) != NULL && ds->ds_dir)
 		return (ds->ds_dir->dd_pool);
 	else
 		return (spa_get_dsl(os->os_spa));
 }
 
 dsl_dataset_t *
 dmu_objset_ds(objset_t *os)
 {
 	return (os->os_dsl_dataset);
 }
 
 dmu_objset_type_t
 dmu_objset_type(objset_t *os)
 {
 	return (os->os_phys->os_type);
 }
 
 void
 dmu_objset_name(objset_t *os, char *buf)
 {
 	dsl_dataset_name(os->os_dsl_dataset, buf);
 }
 
 uint64_t
 dmu_objset_id(objset_t *os)
 {
 	dsl_dataset_t *ds = os->os_dsl_dataset;
 
 	return (ds ? ds->ds_object : 0);
 }
 
 uint64_t
 dmu_objset_dnodesize(objset_t *os)
 {
 	return (os->os_dnodesize);
 }
 
 zfs_sync_type_t
 dmu_objset_syncprop(objset_t *os)
 {
 	return (os->os_sync);
 }
 
 zfs_logbias_op_t
 dmu_objset_logbias(objset_t *os)
 {
 	return (os->os_logbias);
 }
 
 static void
 checksum_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	/*
 	 * Inheritance should have been done by now.
 	 */
 	ASSERT(newval != ZIO_CHECKSUM_INHERIT);
 
 	os->os_checksum = zio_checksum_select(newval, ZIO_CHECKSUM_ON_VALUE);
 }
 
 static void
 compression_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	/*
 	 * Inheritance and range checking should have been done by now.
 	 */
 	ASSERT(newval != ZIO_COMPRESS_INHERIT);
 
 	os->os_compress = zio_compress_select(os->os_spa,
 	    ZIO_COMPRESS_ALGO(newval), ZIO_COMPRESS_ON);
 	os->os_complevel = zio_complevel_select(os->os_spa, os->os_compress,
 	    ZIO_COMPRESS_LEVEL(newval), ZIO_COMPLEVEL_DEFAULT);
 }
 
 static void
 copies_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	/*
 	 * Inheritance and range checking should have been done by now.
 	 */
 	ASSERT(newval > 0);
 	ASSERT(newval <= spa_max_replication(os->os_spa));
 
 	os->os_copies = newval;
 }
 
 static void
 dedup_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 	spa_t *spa = os->os_spa;
 	enum zio_checksum checksum;
 
 	/*
 	 * Inheritance should have been done by now.
 	 */
 	ASSERT(newval != ZIO_CHECKSUM_INHERIT);
 
 	checksum = zio_checksum_dedup_select(spa, newval, ZIO_CHECKSUM_OFF);
 
 	os->os_dedup_checksum = checksum & ZIO_CHECKSUM_MASK;
 	os->os_dedup_verify = !!(checksum & ZIO_CHECKSUM_VERIFY);
 }
 
 static void
 primary_cache_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	/*
 	 * Inheritance and range checking should have been done by now.
 	 */
 	ASSERT(newval == ZFS_CACHE_ALL || newval == ZFS_CACHE_NONE ||
 	    newval == ZFS_CACHE_METADATA);
 
 	os->os_primary_cache = newval;
 }
 
 static void
 secondary_cache_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	/*
 	 * Inheritance and range checking should have been done by now.
 	 */
 	ASSERT(newval == ZFS_CACHE_ALL || newval == ZFS_CACHE_NONE ||
 	    newval == ZFS_CACHE_METADATA);
 
 	os->os_secondary_cache = newval;
 }
 
 static void
 prefetch_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	/*
 	 * Inheritance should have been done by now.
 	 */
 	ASSERT(newval == ZFS_PREFETCH_ALL || newval == ZFS_PREFETCH_NONE ||
 	    newval == ZFS_PREFETCH_METADATA);
 	os->os_prefetch = newval;
 }
 
 static void
 sync_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	/*
 	 * Inheritance and range checking should have been done by now.
 	 */
 	ASSERT(newval == ZFS_SYNC_STANDARD || newval == ZFS_SYNC_ALWAYS ||
 	    newval == ZFS_SYNC_DISABLED);
 
 	os->os_sync = newval;
 	if (os->os_zil)
 		zil_set_sync(os->os_zil, newval);
 }
 
 static void
 redundant_metadata_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	/*
 	 * Inheritance and range checking should have been done by now.
 	 */
 	ASSERT(newval == ZFS_REDUNDANT_METADATA_ALL ||
 	    newval == ZFS_REDUNDANT_METADATA_MOST ||
 	    newval == ZFS_REDUNDANT_METADATA_SOME ||
 	    newval == ZFS_REDUNDANT_METADATA_NONE);
 
 	os->os_redundant_metadata = newval;
 }
 
 static void
 dnodesize_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	switch (newval) {
 	case ZFS_DNSIZE_LEGACY:
 		os->os_dnodesize = DNODE_MIN_SIZE;
 		break;
 	case ZFS_DNSIZE_AUTO:
 		/*
 		 * Choose a dnode size that will work well for most
 		 * workloads if the user specified "auto". Future code
 		 * improvements could dynamically select a dnode size
 		 * based on observed workload patterns.
 		 */
 		os->os_dnodesize = DNODE_MIN_SIZE * 2;
 		break;
 	case ZFS_DNSIZE_1K:
 	case ZFS_DNSIZE_2K:
 	case ZFS_DNSIZE_4K:
 	case ZFS_DNSIZE_8K:
 	case ZFS_DNSIZE_16K:
 		os->os_dnodesize = newval;
 		break;
 	}
 }
 
 static void
 smallblk_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	/*
 	 * Inheritance and range checking should have been done by now.
 	 */
 	ASSERT(newval <= SPA_MAXBLOCKSIZE);
 	ASSERT(ISP2(newval));
 
 	os->os_zpl_special_smallblock = newval;
 }
 
 static void
 logbias_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	ASSERT(newval == ZFS_LOGBIAS_LATENCY ||
 	    newval == ZFS_LOGBIAS_THROUGHPUT);
 	os->os_logbias = newval;
 	if (os->os_zil)
 		zil_set_logbias(os->os_zil, newval);
 }
 
 static void
 recordsize_changed_cb(void *arg, uint64_t newval)
 {
 	objset_t *os = arg;
 
 	os->os_recordsize = newval;
 }
 
 void
 dmu_objset_byteswap(void *buf, size_t size)
 {
 	objset_phys_t *osp = buf;
 
 	ASSERT(size == OBJSET_PHYS_SIZE_V1 || size == OBJSET_PHYS_SIZE_V2 ||
 	    size == sizeof (objset_phys_t));
 	dnode_byteswap(&osp->os_meta_dnode);
 	byteswap_uint64_array(&osp->os_zil_header, sizeof (zil_header_t));
 	osp->os_type = BSWAP_64(osp->os_type);
 	osp->os_flags = BSWAP_64(osp->os_flags);
 	if (size >= OBJSET_PHYS_SIZE_V2) {
 		dnode_byteswap(&osp->os_userused_dnode);
 		dnode_byteswap(&osp->os_groupused_dnode);
 		if (size >= sizeof (objset_phys_t))
 			dnode_byteswap(&osp->os_projectused_dnode);
 	}
 }
 
 /*
  * The hash is a CRC-based hash of the objset_t pointer and the object number.
  */
 static uint64_t
 dnode_hash(const objset_t *os, uint64_t obj)
 {
 	uintptr_t osv = (uintptr_t)os;
 	uint64_t crc = -1ULL;
 
 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
 	/*
 	 * The lower 11 bits of the pointer don't have much entropy, because
 	 * the objset_t is more than 1KB long and so likely aligned to 2KB.
 	 */
 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 11)) & 0xFF];
 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF];
 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF];
 	crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 16)) & 0xFF];
 
 	crc ^= (osv>>14) ^ (obj>>24);
 
 	return (crc);
 }
 
 static unsigned int
 dnode_multilist_index_func(multilist_t *ml, void *obj)
 {
 	dnode_t *dn = obj;
 
 	/*
 	 * The low order bits of the hash value are thought to be
 	 * distributed evenly. Otherwise, in the case that the multilist
 	 * has a power of two number of sublists, each sublists' usage
 	 * would not be evenly distributed. In this context full 64bit
 	 * division would be a waste of time, so limit it to 32 bits.
 	 */
 	return ((unsigned int)dnode_hash(dn->dn_objset, dn->dn_object) %
 	    multilist_get_num_sublists(ml));
 }
 
 static inline boolean_t
 dmu_os_is_l2cacheable(objset_t *os)
 {
 	if (os->os_secondary_cache == ZFS_CACHE_ALL ||
 	    os->os_secondary_cache == ZFS_CACHE_METADATA) {
 		if (l2arc_exclude_special == 0)
 			return (B_TRUE);
 
 		blkptr_t *bp = os->os_rootbp;
 		if (bp == NULL || BP_IS_HOLE(bp))
 			return (B_FALSE);
 		uint64_t vdev = DVA_GET_VDEV(bp->blk_dva);
 		vdev_t *rvd = os->os_spa->spa_root_vdev;
 		vdev_t *vd = NULL;
 
 		if (vdev < rvd->vdev_children)
 			vd = rvd->vdev_child[vdev];
 
 		if (vd == NULL)
 			return (B_TRUE);
 
 		if (vd->vdev_alloc_bias != VDEV_BIAS_SPECIAL &&
 		    vd->vdev_alloc_bias != VDEV_BIAS_DEDUP)
 			return (B_TRUE);
 	}
 	return (B_FALSE);
 }
 
 /*
  * Instantiates the objset_t in-memory structure corresponding to the
  * objset_phys_t that's pointed to by the specified blkptr_t.
  */
 int
 dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
     objset_t **osp)
 {
 	objset_t *os;
 	int i, err;
 
 	ASSERT(ds == NULL || MUTEX_HELD(&ds->ds_opening_lock));
 	ASSERT(!BP_IS_REDACTED(bp));
 
 	/*
 	 * We need the pool config lock to get properties.
 	 */
 	ASSERT(ds == NULL || dsl_pool_config_held(ds->ds_dir->dd_pool));
 
 	/*
 	 * The $ORIGIN dataset (if it exists) doesn't have an associated
 	 * objset, so there's no reason to open it. The $ORIGIN dataset
 	 * will not exist on pools older than SPA_VERSION_ORIGIN.
 	 */
 	if (ds != NULL && spa_get_dsl(spa) != NULL &&
 	    spa_get_dsl(spa)->dp_origin_snap != NULL) {
 		ASSERT3P(ds->ds_dir, !=,
 		    spa_get_dsl(spa)->dp_origin_snap->ds_dir);
 	}
 
 	os = kmem_zalloc(sizeof (objset_t), KM_SLEEP);
 	os->os_dsl_dataset = ds;
 	os->os_spa = spa;
 	os->os_rootbp = bp;
 	if (!BP_IS_HOLE(os->os_rootbp)) {
 		arc_flags_t aflags = ARC_FLAG_WAIT;
 		zbookmark_phys_t zb;
 		int size;
 		zio_flag_t zio_flags = ZIO_FLAG_CANFAIL;
 		SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET,
 		    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
 
 		if (dmu_os_is_l2cacheable(os))
 			aflags |= ARC_FLAG_L2CACHE;
 
 		if (ds != NULL && ds->ds_dir->dd_crypto_obj != 0) {
 			ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF);
 			ASSERT(BP_IS_AUTHENTICATED(bp));
 			zio_flags |= ZIO_FLAG_RAW;
 		}
 
 		dprintf_bp(os->os_rootbp, "reading %s", "");
 		err = arc_read(NULL, spa, os->os_rootbp,
 		    arc_getbuf_func, &os->os_phys_buf,
 		    ZIO_PRIORITY_SYNC_READ, zio_flags, &aflags, &zb);
 		if (err != 0) {
 			kmem_free(os, sizeof (objset_t));
 			/* convert checksum errors into IO errors */
 			if (err == ECKSUM)
 				err = SET_ERROR(EIO);
 			return (err);
 		}
 
 		if (spa_version(spa) < SPA_VERSION_USERSPACE)
 			size = OBJSET_PHYS_SIZE_V1;
 		else if (!spa_feature_is_enabled(spa,
 		    SPA_FEATURE_PROJECT_QUOTA))
 			size = OBJSET_PHYS_SIZE_V2;
 		else
 			size = sizeof (objset_phys_t);
 
 		/* Increase the blocksize if we are permitted. */
 		if (arc_buf_size(os->os_phys_buf) < size) {
 			arc_buf_t *buf = arc_alloc_buf(spa, &os->os_phys_buf,
 			    ARC_BUFC_METADATA, size);
 			memset(buf->b_data, 0, size);
 			memcpy(buf->b_data, os->os_phys_buf->b_data,
 			    arc_buf_size(os->os_phys_buf));
 			arc_buf_destroy(os->os_phys_buf, &os->os_phys_buf);
 			os->os_phys_buf = buf;
 		}
 
 		os->os_phys = os->os_phys_buf->b_data;
 		os->os_flags = os->os_phys->os_flags;
 	} else {
 		int size = spa_version(spa) >= SPA_VERSION_USERSPACE ?
 		    sizeof (objset_phys_t) : OBJSET_PHYS_SIZE_V1;
 		os->os_phys_buf = arc_alloc_buf(spa, &os->os_phys_buf,
 		    ARC_BUFC_METADATA, size);
 		os->os_phys = os->os_phys_buf->b_data;
 		memset(os->os_phys, 0, size);
 	}
 	/*
 	 * These properties will be filled in by the logic in zfs_get_zplprop()
 	 * when they are queried for the first time.
 	 */
 	os->os_version = OBJSET_PROP_UNINITIALIZED;
 	os->os_normalization = OBJSET_PROP_UNINITIALIZED;
 	os->os_utf8only = OBJSET_PROP_UNINITIALIZED;
 	os->os_casesensitivity = OBJSET_PROP_UNINITIALIZED;
 
 	/*
 	 * Note: the changed_cb will be called once before the register
 	 * func returns, thus changing the checksum/compression from the
 	 * default (fletcher2/off).  Snapshots don't need to know about
 	 * checksum/compression/copies.
 	 */
 	if (ds != NULL) {
 		os->os_encrypted = (ds->ds_dir->dd_crypto_obj != 0);
 
 		err = dsl_prop_register(ds,
 		    zfs_prop_to_name(ZFS_PROP_PRIMARYCACHE),
 		    primary_cache_changed_cb, os);
 		if (err == 0) {
 			err = dsl_prop_register(ds,
 			    zfs_prop_to_name(ZFS_PROP_SECONDARYCACHE),
 			    secondary_cache_changed_cb, os);
 		}
 		if (err == 0) {
 			err = dsl_prop_register(ds,
 			    zfs_prop_to_name(ZFS_PROP_PREFETCH),
 			    prefetch_changed_cb, os);
 		}
 		if (!ds->ds_is_snapshot) {
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(ZFS_PROP_CHECKSUM),
 				    checksum_changed_cb, os);
 			}
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(ZFS_PROP_COMPRESSION),
 				    compression_changed_cb, os);
 			}
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(ZFS_PROP_COPIES),
 				    copies_changed_cb, os);
 			}
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(ZFS_PROP_DEDUP),
 				    dedup_changed_cb, os);
 			}
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(ZFS_PROP_LOGBIAS),
 				    logbias_changed_cb, os);
 			}
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(ZFS_PROP_SYNC),
 				    sync_changed_cb, os);
 			}
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(
 				    ZFS_PROP_REDUNDANT_METADATA),
 				    redundant_metadata_changed_cb, os);
 			}
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(ZFS_PROP_RECORDSIZE),
 				    recordsize_changed_cb, os);
 			}
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(ZFS_PROP_DNODESIZE),
 				    dnodesize_changed_cb, os);
 			}
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(
 				    ZFS_PROP_SPECIAL_SMALL_BLOCKS),
 				    smallblk_changed_cb, os);
 			}
 		}
 		if (err != 0) {
 			arc_buf_destroy(os->os_phys_buf, &os->os_phys_buf);
 			kmem_free(os, sizeof (objset_t));
 			return (err);
 		}
 	} else {
 		/* It's the meta-objset. */
 		os->os_checksum = ZIO_CHECKSUM_FLETCHER_4;
 		os->os_compress = ZIO_COMPRESS_ON;
 		os->os_complevel = ZIO_COMPLEVEL_DEFAULT;
 		os->os_encrypted = B_FALSE;
 		os->os_copies = spa_max_replication(spa);
 		os->os_dedup_checksum = ZIO_CHECKSUM_OFF;
 		os->os_dedup_verify = B_FALSE;
 		os->os_logbias = ZFS_LOGBIAS_LATENCY;
 		os->os_sync = ZFS_SYNC_STANDARD;
 		os->os_primary_cache = ZFS_CACHE_ALL;
 		os->os_secondary_cache = ZFS_CACHE_ALL;
 		os->os_dnodesize = DNODE_MIN_SIZE;
 		os->os_prefetch = ZFS_PREFETCH_ALL;
 	}
 
 	if (ds == NULL || !ds->ds_is_snapshot)
 		os->os_zil_header = os->os_phys->os_zil_header;
 	os->os_zil = zil_alloc(os, &os->os_zil_header);
 
 	for (i = 0; i < TXG_SIZE; i++) {
 		multilist_create(&os->os_dirty_dnodes[i], sizeof (dnode_t),
 		    offsetof(dnode_t, dn_dirty_link[i]),
 		    dnode_multilist_index_func);
 	}
 	list_create(&os->os_dnodes, sizeof (dnode_t),
 	    offsetof(dnode_t, dn_link));
 	list_create(&os->os_downgraded_dbufs, sizeof (dmu_buf_impl_t),
 	    offsetof(dmu_buf_impl_t, db_link));
 
 	list_link_init(&os->os_evicting_node);
 
 	mutex_init(&os->os_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&os->os_userused_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&os->os_obj_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&os->os_user_ptr_lock, NULL, MUTEX_DEFAULT, NULL);
 	os->os_obj_next_percpu_len = boot_ncpus;
 	os->os_obj_next_percpu = kmem_zalloc(os->os_obj_next_percpu_len *
 	    sizeof (os->os_obj_next_percpu[0]), KM_SLEEP);
 
 	dnode_special_open(os, &os->os_phys->os_meta_dnode,
 	    DMU_META_DNODE_OBJECT, &os->os_meta_dnode);
 	if (OBJSET_BUF_HAS_USERUSED(os->os_phys_buf)) {
 		dnode_special_open(os, &os->os_phys->os_userused_dnode,
 		    DMU_USERUSED_OBJECT, &os->os_userused_dnode);
 		dnode_special_open(os, &os->os_phys->os_groupused_dnode,
 		    DMU_GROUPUSED_OBJECT, &os->os_groupused_dnode);
 		if (OBJSET_BUF_HAS_PROJECTUSED(os->os_phys_buf))
 			dnode_special_open(os,
 			    &os->os_phys->os_projectused_dnode,
 			    DMU_PROJECTUSED_OBJECT, &os->os_projectused_dnode);
 	}
 
 	mutex_init(&os->os_upgrade_lock, NULL, MUTEX_DEFAULT, NULL);
 
 	*osp = os;
 	return (0);
 }
 
 int
 dmu_objset_from_ds(dsl_dataset_t *ds, objset_t **osp)
 {
 	int err = 0;
 
 	/*
 	 * We need the pool_config lock to manipulate the dsl_dataset_t.
 	 * Even if the dataset is long-held, we need the pool_config lock
 	 * to open the objset, as it needs to get properties.
 	 */
 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
 
 	mutex_enter(&ds->ds_opening_lock);
 	if (ds->ds_objset == NULL) {
 		objset_t *os;
 		rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
 		err = dmu_objset_open_impl(dsl_dataset_get_spa(ds),
 		    ds, dsl_dataset_get_blkptr(ds), &os);
 		rrw_exit(&ds->ds_bp_rwlock, FTAG);
 
 		if (err == 0) {
 			mutex_enter(&ds->ds_lock);
 			ASSERT(ds->ds_objset == NULL);
 			ds->ds_objset = os;
 			mutex_exit(&ds->ds_lock);
 		}
 	}
 	*osp = ds->ds_objset;
 	mutex_exit(&ds->ds_opening_lock);
 	return (err);
 }
 
 /*
  * Holds the pool while the objset is held.  Therefore only one objset
  * can be held at a time.
  */
 int
 dmu_objset_hold_flags(const char *name, boolean_t decrypt, const void *tag,
     objset_t **osp)
 {
 	dsl_pool_t *dp;
 	dsl_dataset_t *ds;
 	int err;
 	ds_hold_flags_t flags;
 
 	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
 	err = dsl_pool_hold(name, tag, &dp);
 	if (err != 0)
 		return (err);
 	err = dsl_dataset_hold_flags(dp, name, flags, tag, &ds);
 	if (err != 0) {
 		dsl_pool_rele(dp, tag);
 		return (err);
 	}
 
 	err = dmu_objset_from_ds(ds, osp);
 	if (err != 0) {
-		dsl_dataset_rele(ds, tag);
+		dsl_dataset_rele_flags(ds, flags, tag);
 		dsl_pool_rele(dp, tag);
 	}
 
 	return (err);
 }
 
 int
 dmu_objset_hold(const char *name, const void *tag, objset_t **osp)
 {
 	return (dmu_objset_hold_flags(name, B_FALSE, tag, osp));
 }
 
 static int
 dmu_objset_own_impl(dsl_dataset_t *ds, dmu_objset_type_t type,
     boolean_t readonly, boolean_t decrypt, const void *tag, objset_t **osp)
 {
 	(void) tag;
 
 	int err = dmu_objset_from_ds(ds, osp);
 	if (err != 0) {
 		return (err);
 	} else if (type != DMU_OST_ANY && type != (*osp)->os_phys->os_type) {
 		return (SET_ERROR(EINVAL));
 	} else if (!readonly && dsl_dataset_is_snapshot(ds)) {
 		return (SET_ERROR(EROFS));
 	} else if (!readonly && decrypt &&
 	    dsl_dir_incompatible_encryption_version(ds->ds_dir)) {
 		return (SET_ERROR(EROFS));
 	}
 
 	/* if we are decrypting, we can now check MACs in os->os_phys_buf */
 	if (decrypt && arc_is_unauthenticated((*osp)->os_phys_buf)) {
 		zbookmark_phys_t zb;
 
 		SET_BOOKMARK(&zb, ds->ds_object, ZB_ROOT_OBJECT,
 		    ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
 		err = arc_untransform((*osp)->os_phys_buf, (*osp)->os_spa,
 		    &zb, B_FALSE);
 		if (err != 0)
 			return (err);
 
 		ASSERT0(arc_is_unauthenticated((*osp)->os_phys_buf));
 	}
 
 	return (0);
 }
 
 /*
  * dsl_pool must not be held when this is called.
  * Upon successful return, there will be a longhold on the dataset,
  * and the dsl_pool will not be held.
  */
 int
 dmu_objset_own(const char *name, dmu_objset_type_t type,
     boolean_t readonly, boolean_t decrypt, const void *tag, objset_t **osp)
 {
 	dsl_pool_t *dp;
 	dsl_dataset_t *ds;
 	int err;
 	ds_hold_flags_t flags;
 
 	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
 	err = dsl_pool_hold(name, FTAG, &dp);
 	if (err != 0)
 		return (err);
 	err = dsl_dataset_own(dp, name, flags, tag, &ds);
 	if (err != 0) {
 		dsl_pool_rele(dp, FTAG);
 		return (err);
 	}
 	err = dmu_objset_own_impl(ds, type, readonly, decrypt, tag, osp);
 	if (err != 0) {
 		dsl_dataset_disown(ds, flags, tag);
 		dsl_pool_rele(dp, FTAG);
 		return (err);
 	}
 
 	/*
 	 * User accounting requires the dataset to be decrypted and rw.
 	 * We also don't begin user accounting during claiming to help
 	 * speed up pool import times and to keep this txg reserved
 	 * completely for recovery work.
 	 */
 	if (!readonly && !dp->dp_spa->spa_claiming &&
 	    (ds->ds_dir->dd_crypto_obj == 0 || decrypt)) {
 		if (dmu_objset_userobjspace_upgradable(*osp) ||
 		    dmu_objset_projectquota_upgradable(*osp)) {
 			dmu_objset_id_quota_upgrade(*osp);
 		} else if (dmu_objset_userused_enabled(*osp)) {
 			dmu_objset_userspace_upgrade(*osp);
 		}
 	}
 
 	dsl_pool_rele(dp, FTAG);
 	return (0);
 }
 
 int
 dmu_objset_own_obj(dsl_pool_t *dp, uint64_t obj, dmu_objset_type_t type,
     boolean_t readonly, boolean_t decrypt, const void *tag, objset_t **osp)
 {
 	dsl_dataset_t *ds;
 	int err;
 	ds_hold_flags_t flags;
 
 	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
 	err = dsl_dataset_own_obj(dp, obj, flags, tag, &ds);
 	if (err != 0)
 		return (err);
 
 	err = dmu_objset_own_impl(ds, type, readonly, decrypt, tag, osp);
 	if (err != 0) {
 		dsl_dataset_disown(ds, flags, tag);
 		return (err);
 	}
 
 	return (0);
 }
 
 void
 dmu_objset_rele_flags(objset_t *os, boolean_t decrypt, const void *tag)
 {
 	ds_hold_flags_t flags;
 	dsl_pool_t *dp = dmu_objset_pool(os);
 
 	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
 	dsl_dataset_rele_flags(os->os_dsl_dataset, flags, tag);
 	dsl_pool_rele(dp, tag);
 }
 
 void
 dmu_objset_rele(objset_t *os, const void *tag)
 {
 	dmu_objset_rele_flags(os, B_FALSE, tag);
 }
 
 /*
  * When we are called, os MUST refer to an objset associated with a dataset
  * that is owned by 'tag'; that is, is held and long held by 'tag' and ds_owner
  * == tag.  We will then release and reacquire ownership of the dataset while
  * holding the pool config_rwlock to avoid intervening namespace or ownership
  * changes may occur.
  *
  * This exists solely to accommodate zfs_ioc_userspace_upgrade()'s desire to
  * release the hold on its dataset and acquire a new one on the dataset of the
  * same name so that it can be partially torn down and reconstructed.
  */
 void
 dmu_objset_refresh_ownership(dsl_dataset_t *ds, dsl_dataset_t **newds,
     boolean_t decrypt, const void *tag)
 {
 	dsl_pool_t *dp;
 	char name[ZFS_MAX_DATASET_NAME_LEN];
 	ds_hold_flags_t flags;
 
 	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
 	VERIFY3P(ds, !=, NULL);
 	VERIFY3P(ds->ds_owner, ==, tag);
 	VERIFY(dsl_dataset_long_held(ds));
 
 	dsl_dataset_name(ds, name);
 	dp = ds->ds_dir->dd_pool;
 	dsl_pool_config_enter(dp, FTAG);
 	dsl_dataset_disown(ds, flags, tag);
 	VERIFY0(dsl_dataset_own(dp, name, flags, tag, newds));
 	dsl_pool_config_exit(dp, FTAG);
 }
 
 void
 dmu_objset_disown(objset_t *os, boolean_t decrypt, const void *tag)
 {
 	ds_hold_flags_t flags;
 
 	flags = (decrypt) ? DS_HOLD_FLAG_DECRYPT : DS_HOLD_FLAG_NONE;
 	/*
 	 * Stop upgrading thread
 	 */
 	dmu_objset_upgrade_stop(os);
 	dsl_dataset_disown(os->os_dsl_dataset, flags, tag);
 }
 
 void
 dmu_objset_evict_dbufs(objset_t *os)
 {
 	dnode_t *dn_marker;
 	dnode_t *dn;
 
 	dn_marker = kmem_alloc(sizeof (dnode_t), KM_SLEEP);
 
 	mutex_enter(&os->os_lock);
 	dn = list_head(&os->os_dnodes);
 	while (dn != NULL) {
 		/*
 		 * Skip dnodes without holds.  We have to do this dance
 		 * because dnode_add_ref() only works if there is already a
 		 * hold.  If the dnode has no holds, then it has no dbufs.
 		 */
 		if (dnode_add_ref(dn, FTAG)) {
 			list_insert_after(&os->os_dnodes, dn, dn_marker);
 			mutex_exit(&os->os_lock);
 
 			dnode_evict_dbufs(dn);
 			dnode_rele(dn, FTAG);
 
 			mutex_enter(&os->os_lock);
 			dn = list_next(&os->os_dnodes, dn_marker);
 			list_remove(&os->os_dnodes, dn_marker);
 		} else {
 			dn = list_next(&os->os_dnodes, dn);
 		}
 	}
 	mutex_exit(&os->os_lock);
 
 	kmem_free(dn_marker, sizeof (dnode_t));
 
 	if (DMU_USERUSED_DNODE(os) != NULL) {
 		if (DMU_PROJECTUSED_DNODE(os) != NULL)
 			dnode_evict_dbufs(DMU_PROJECTUSED_DNODE(os));
 		dnode_evict_dbufs(DMU_GROUPUSED_DNODE(os));
 		dnode_evict_dbufs(DMU_USERUSED_DNODE(os));
 	}
 	dnode_evict_dbufs(DMU_META_DNODE(os));
 }
 
 /*
  * Objset eviction processing is split into into two pieces.
  * The first marks the objset as evicting, evicts any dbufs that
  * have a refcount of zero, and then queues up the objset for the
  * second phase of eviction.  Once os->os_dnodes has been cleared by
  * dnode_buf_pageout()->dnode_destroy(), the second phase is executed.
  * The second phase closes the special dnodes, dequeues the objset from
  * the list of those undergoing eviction, and finally frees the objset.
  *
  * NOTE: Due to asynchronous eviction processing (invocation of
  *       dnode_buf_pageout()), it is possible for the meta dnode for the
  *       objset to have no holds even though os->os_dnodes is not empty.
  */
 void
 dmu_objset_evict(objset_t *os)
 {
 	dsl_dataset_t *ds = os->os_dsl_dataset;
 
 	for (int t = 0; t < TXG_SIZE; t++)
 		ASSERT(!dmu_objset_is_dirty(os, t));
 
 	if (ds)
 		dsl_prop_unregister_all(ds, os);
 
 	if (os->os_sa)
 		sa_tear_down(os);
 
 	dmu_objset_evict_dbufs(os);
 
 	mutex_enter(&os->os_lock);
 	spa_evicting_os_register(os->os_spa, os);
 	if (list_is_empty(&os->os_dnodes)) {
 		mutex_exit(&os->os_lock);
 		dmu_objset_evict_done(os);
 	} else {
 		mutex_exit(&os->os_lock);
 	}
 
 
 }
 
 void
 dmu_objset_evict_done(objset_t *os)
 {
 	ASSERT3P(list_head(&os->os_dnodes), ==, NULL);
 
 	dnode_special_close(&os->os_meta_dnode);
 	if (DMU_USERUSED_DNODE(os)) {
 		if (DMU_PROJECTUSED_DNODE(os))
 			dnode_special_close(&os->os_projectused_dnode);
 		dnode_special_close(&os->os_userused_dnode);
 		dnode_special_close(&os->os_groupused_dnode);
 	}
 	zil_free(os->os_zil);
 
 	arc_buf_destroy(os->os_phys_buf, &os->os_phys_buf);
 
 	/*
 	 * This is a barrier to prevent the objset from going away in
 	 * dnode_move() until we can safely ensure that the objset is still in
 	 * use. We consider the objset valid before the barrier and invalid
 	 * after the barrier.
 	 */
 	rw_enter(&os_lock, RW_READER);
 	rw_exit(&os_lock);
 
 	kmem_free(os->os_obj_next_percpu,
 	    os->os_obj_next_percpu_len * sizeof (os->os_obj_next_percpu[0]));
 
 	mutex_destroy(&os->os_lock);
 	mutex_destroy(&os->os_userused_lock);
 	mutex_destroy(&os->os_obj_lock);
 	mutex_destroy(&os->os_user_ptr_lock);
 	mutex_destroy(&os->os_upgrade_lock);
 	for (int i = 0; i < TXG_SIZE; i++)
 		multilist_destroy(&os->os_dirty_dnodes[i]);
 	spa_evicting_os_deregister(os->os_spa, os);
 	kmem_free(os, sizeof (objset_t));
 }
 
 inode_timespec_t
 dmu_objset_snap_cmtime(objset_t *os)
 {
 	return (dsl_dir_snap_cmtime(os->os_dsl_dataset->ds_dir));
 }
 
 objset_t *
 dmu_objset_create_impl_dnstats(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
     dmu_objset_type_t type, int levels, int blksz, int ibs, dmu_tx_t *tx)
 {
 	objset_t *os;
 	dnode_t *mdn;
 
 	ASSERT(dmu_tx_is_syncing(tx));
 
 	if (blksz == 0)
 		blksz = DNODE_BLOCK_SIZE;
 	if (ibs == 0)
 		ibs = DN_MAX_INDBLKSHIFT;
 
 	if (ds != NULL)
 		VERIFY0(dmu_objset_from_ds(ds, &os));
 	else
 		VERIFY0(dmu_objset_open_impl(spa, NULL, bp, &os));
 
 	mdn = DMU_META_DNODE(os);
 
 	dnode_allocate(mdn, DMU_OT_DNODE, blksz, ibs, DMU_OT_NONE, 0,
 	    DNODE_MIN_SLOTS, tx);
 
 	/*
 	 * We don't want to have to increase the meta-dnode's nlevels
 	 * later, because then we could do it in quiescing context while
 	 * we are also accessing it in open context.
 	 *
 	 * This precaution is not necessary for the MOS (ds == NULL),
 	 * because the MOS is only updated in syncing context.
 	 * This is most fortunate: the MOS is the only objset that
 	 * needs to be synced multiple times as spa_sync() iterates
 	 * to convergence, so minimizing its dn_nlevels matters.
 	 */
 	if (ds != NULL) {
 		if (levels == 0) {
 			levels = 1;
 
 			/*
 			 * Determine the number of levels necessary for the
 			 * meta-dnode to contain DN_MAX_OBJECT dnodes.  Note
 			 * that in order to ensure that we do not overflow
 			 * 64 bits, there has to be a nlevels that gives us a
 			 * number of blocks > DN_MAX_OBJECT but < 2^64.
 			 * Therefore, (mdn->dn_indblkshift - SPA_BLKPTRSHIFT)
 			 * (10) must be less than (64 - log2(DN_MAX_OBJECT))
 			 * (16).
 			 */
 			while ((uint64_t)mdn->dn_nblkptr <<
 			    (mdn->dn_datablkshift - DNODE_SHIFT + (levels - 1) *
 			    (mdn->dn_indblkshift - SPA_BLKPTRSHIFT)) <
 			    DN_MAX_OBJECT)
 				levels++;
 		}
 
 		mdn->dn_next_nlevels[tx->tx_txg & TXG_MASK] =
 		    mdn->dn_nlevels = levels;
 	}
 
 	ASSERT(type != DMU_OST_NONE);
 	ASSERT(type != DMU_OST_ANY);
 	ASSERT(type < DMU_OST_NUMTYPES);
 	os->os_phys->os_type = type;
 
 	/*
 	 * Enable user accounting if it is enabled and this is not an
 	 * encrypted receive.
 	 */
 	if (dmu_objset_userused_enabled(os) &&
 	    (!os->os_encrypted || !dmu_objset_is_receiving(os))) {
 		os->os_phys->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
 		if (dmu_objset_userobjused_enabled(os)) {
 			ASSERT3P(ds, !=, NULL);
 			ds->ds_feature_activation[
 			    SPA_FEATURE_USEROBJ_ACCOUNTING] = (void *)B_TRUE;
 			os->os_phys->os_flags |=
 			    OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE;
 		}
 		if (dmu_objset_projectquota_enabled(os)) {
 			ASSERT3P(ds, !=, NULL);
 			ds->ds_feature_activation[
 			    SPA_FEATURE_PROJECT_QUOTA] = (void *)B_TRUE;
 			os->os_phys->os_flags |=
 			    OBJSET_FLAG_PROJECTQUOTA_COMPLETE;
 		}
 		os->os_flags = os->os_phys->os_flags;
 	}
 
 	dsl_dataset_dirty(ds, tx);
 
 	return (os);
 }
 
 /* called from dsl for meta-objset */
 objset_t *
 dmu_objset_create_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
     dmu_objset_type_t type, dmu_tx_t *tx)
 {
 	return (dmu_objset_create_impl_dnstats(spa, ds, bp, type, 0, 0, 0, tx));
 }
 
 typedef struct dmu_objset_create_arg {
 	const char *doca_name;
 	cred_t *doca_cred;
-	proc_t *doca_proc;
 	void (*doca_userfunc)(objset_t *os, void *arg,
 	    cred_t *cr, dmu_tx_t *tx);
 	void *doca_userarg;
 	dmu_objset_type_t doca_type;
 	uint64_t doca_flags;
 	dsl_crypto_params_t *doca_dcp;
 } dmu_objset_create_arg_t;
 
 static int
 dmu_objset_create_check(void *arg, dmu_tx_t *tx)
 {
 	dmu_objset_create_arg_t *doca = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dir_t *pdd;
 	dsl_dataset_t *parentds;
 	objset_t *parentos;
 	const char *tail;
 	int error;
 
 	if (strchr(doca->doca_name, '@') != NULL)
 		return (SET_ERROR(EINVAL));
 
 	if (strlen(doca->doca_name) >= ZFS_MAX_DATASET_NAME_LEN)
 		return (SET_ERROR(ENAMETOOLONG));
 
 	if (dataset_nestcheck(doca->doca_name) != 0)
 		return (SET_ERROR(ENAMETOOLONG));
 
 	error = dsl_dir_hold(dp, doca->doca_name, FTAG, &pdd, &tail);
 	if (error != 0)
 		return (error);
 	if (tail == NULL) {
 		dsl_dir_rele(pdd, FTAG);
 		return (SET_ERROR(EEXIST));
 	}
 
 	error = dmu_objset_create_crypt_check(pdd, doca->doca_dcp, NULL);
 	if (error != 0) {
 		dsl_dir_rele(pdd, FTAG);
 		return (error);
 	}
 
 	error = dsl_fs_ss_limit_check(pdd, 1, ZFS_PROP_FILESYSTEM_LIMIT, NULL,
-	    doca->doca_cred, doca->doca_proc);
+	    doca->doca_cred);
 	if (error != 0) {
 		dsl_dir_rele(pdd, FTAG);
 		return (error);
 	}
 
 	/* can't create below anything but filesystems (eg. no ZVOLs) */
 	error = dsl_dataset_hold_obj(pdd->dd_pool,
 	    dsl_dir_phys(pdd)->dd_head_dataset_obj, FTAG, &parentds);
 	if (error != 0) {
 		dsl_dir_rele(pdd, FTAG);
 		return (error);
 	}
 	error = dmu_objset_from_ds(parentds, &parentos);
 	if (error != 0) {
 		dsl_dataset_rele(parentds, FTAG);
 		dsl_dir_rele(pdd, FTAG);
 		return (error);
 	}
 	if (dmu_objset_type(parentos) != DMU_OST_ZFS) {
 		dsl_dataset_rele(parentds, FTAG);
 		dsl_dir_rele(pdd, FTAG);
 		return (SET_ERROR(ZFS_ERR_WRONG_PARENT));
 	}
 	dsl_dataset_rele(parentds, FTAG);
 	dsl_dir_rele(pdd, FTAG);
 
 	return (error);
 }
 
 static void
 dmu_objset_create_sync(void *arg, dmu_tx_t *tx)
 {
 	dmu_objset_create_arg_t *doca = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	spa_t *spa = dp->dp_spa;
 	dsl_dir_t *pdd;
 	const char *tail;
 	dsl_dataset_t *ds;
 	uint64_t obj;
 	blkptr_t *bp;
 	objset_t *os;
 	zio_t *rzio;
 
 	VERIFY0(dsl_dir_hold(dp, doca->doca_name, FTAG, &pdd, &tail));
 
 	obj = dsl_dataset_create_sync(pdd, tail, NULL, doca->doca_flags,
 	    doca->doca_cred, doca->doca_dcp, tx);
 
 	VERIFY0(dsl_dataset_hold_obj_flags(pdd->dd_pool, obj,
 	    DS_HOLD_FLAG_DECRYPT, FTAG, &ds));
 	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
 	bp = dsl_dataset_get_blkptr(ds);
 	os = dmu_objset_create_impl(spa, ds, bp, doca->doca_type, tx);
 	rrw_exit(&ds->ds_bp_rwlock, FTAG);
 
 	if (doca->doca_userfunc != NULL) {
 		doca->doca_userfunc(os, doca->doca_userarg,
 		    doca->doca_cred, tx);
 	}
 
 	/*
 	 * The doca_userfunc() may write out some data that needs to be
 	 * encrypted if the dataset is encrypted (specifically the root
 	 * directory).  This data must be written out before the encryption
 	 * key mapping is removed by dsl_dataset_rele_flags().  Force the
 	 * I/O to occur immediately by invoking the relevant sections of
 	 * dsl_pool_sync().
 	 */
 	if (os->os_encrypted) {
 		dsl_dataset_t *tmpds = NULL;
 		boolean_t need_sync_done = B_FALSE;
 
 		mutex_enter(&ds->ds_lock);
 		ds->ds_owner = FTAG;
 		mutex_exit(&ds->ds_lock);
 
 		rzio = zio_root(spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
 		tmpds = txg_list_remove_this(&dp->dp_dirty_datasets, ds,
 		    tx->tx_txg);
 		if (tmpds != NULL) {
 			dsl_dataset_sync(ds, rzio, tx);
 			need_sync_done = B_TRUE;
 		}
 		VERIFY0(zio_wait(rzio));
 
 		dmu_objset_sync_done(os, tx);
 		taskq_wait(dp->dp_sync_taskq);
 		if (txg_list_member(&dp->dp_dirty_datasets, ds, tx->tx_txg)) {
 			ASSERT3P(ds->ds_key_mapping, !=, NULL);
 			key_mapping_rele(spa, ds->ds_key_mapping, ds);
 		}
 
 		rzio = zio_root(spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
 		tmpds = txg_list_remove_this(&dp->dp_dirty_datasets, ds,
 		    tx->tx_txg);
 		if (tmpds != NULL) {
 			dmu_buf_rele(ds->ds_dbuf, ds);
 			dsl_dataset_sync(ds, rzio, tx);
 		}
 		VERIFY0(zio_wait(rzio));
 
 		if (need_sync_done) {
 			ASSERT3P(ds->ds_key_mapping, !=, NULL);
 			key_mapping_rele(spa, ds->ds_key_mapping, ds);
 			dsl_dataset_sync_done(ds, tx);
 			dmu_buf_rele(ds->ds_dbuf, ds);
 		}
 
 		mutex_enter(&ds->ds_lock);
 		ds->ds_owner = NULL;
 		mutex_exit(&ds->ds_lock);
 	}
 
 	spa_history_log_internal_ds(ds, "create", tx, " ");
 
 	dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT, FTAG);
 	dsl_dir_rele(pdd, FTAG);
 }
 
 int
 dmu_objset_create(const char *name, dmu_objset_type_t type, uint64_t flags,
     dsl_crypto_params_t *dcp, dmu_objset_create_sync_func_t func, void *arg)
 {
 	dmu_objset_create_arg_t doca;
 	dsl_crypto_params_t tmp_dcp = { 0 };
 
+	cred_t *cr = CRED();
+	crhold(cr);
+
 	doca.doca_name = name;
-	doca.doca_cred = CRED();
-	doca.doca_proc = curproc;
+	doca.doca_cred = cr;
 	doca.doca_flags = flags;
 	doca.doca_userfunc = func;
 	doca.doca_userarg = arg;
 	doca.doca_type = type;
 
 	/*
 	 * Some callers (mostly for testing) do not provide a dcp on their
 	 * own but various code inside the sync task will require it to be
 	 * allocated. Rather than adding NULL checks throughout this code
 	 * or adding dummy dcp's to all of the callers we simply create a
 	 * dummy one here and use that. This zero dcp will have the same
 	 * effect as asking for inheritance of all encryption params.
 	 */
 	doca.doca_dcp = (dcp != NULL) ? dcp : &tmp_dcp;
 
 	int rv = dsl_sync_task(name,
 	    dmu_objset_create_check, dmu_objset_create_sync, &doca,
 	    6, ZFS_SPACE_CHECK_NORMAL);
 
 	if (rv == 0)
 		zvol_create_minor(name);
+
+	crfree(cr);
+
 	return (rv);
 }
 
 typedef struct dmu_objset_clone_arg {
 	const char *doca_clone;
 	const char *doca_origin;
 	cred_t *doca_cred;
-	proc_t *doca_proc;
 } dmu_objset_clone_arg_t;
 
 static int
 dmu_objset_clone_check(void *arg, dmu_tx_t *tx)
 {
 	dmu_objset_clone_arg_t *doca = arg;
 	dsl_dir_t *pdd;
 	const char *tail;
 	int error;
 	dsl_dataset_t *origin;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 
 	if (strchr(doca->doca_clone, '@') != NULL)
 		return (SET_ERROR(EINVAL));
 
 	if (strlen(doca->doca_clone) >= ZFS_MAX_DATASET_NAME_LEN)
 		return (SET_ERROR(ENAMETOOLONG));
 
 	error = dsl_dir_hold(dp, doca->doca_clone, FTAG, &pdd, &tail);
 	if (error != 0)
 		return (error);
 	if (tail == NULL) {
 		dsl_dir_rele(pdd, FTAG);
 		return (SET_ERROR(EEXIST));
 	}
 
 	error = dsl_fs_ss_limit_check(pdd, 1, ZFS_PROP_FILESYSTEM_LIMIT, NULL,
-	    doca->doca_cred, doca->doca_proc);
+	    doca->doca_cred);
 	if (error != 0) {
 		dsl_dir_rele(pdd, FTAG);
 		return (SET_ERROR(EDQUOT));
 	}
 
 	error = dsl_dataset_hold(dp, doca->doca_origin, FTAG, &origin);
 	if (error != 0) {
 		dsl_dir_rele(pdd, FTAG);
 		return (error);
 	}
 
 	/* You can only clone snapshots, not the head datasets. */
 	if (!origin->ds_is_snapshot) {
 		dsl_dataset_rele(origin, FTAG);
 		dsl_dir_rele(pdd, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
 
 	dsl_dataset_rele(origin, FTAG);
 	dsl_dir_rele(pdd, FTAG);
 
 	return (0);
 }
 
 static void
 dmu_objset_clone_sync(void *arg, dmu_tx_t *tx)
 {
 	dmu_objset_clone_arg_t *doca = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dir_t *pdd;
 	const char *tail;
 	dsl_dataset_t *origin, *ds;
 	uint64_t obj;
 	char namebuf[ZFS_MAX_DATASET_NAME_LEN];
 
 	VERIFY0(dsl_dir_hold(dp, doca->doca_clone, FTAG, &pdd, &tail));
 	VERIFY0(dsl_dataset_hold(dp, doca->doca_origin, FTAG, &origin));
 
 	obj = dsl_dataset_create_sync(pdd, tail, origin, 0,
 	    doca->doca_cred, NULL, tx);
 
 	VERIFY0(dsl_dataset_hold_obj(pdd->dd_pool, obj, FTAG, &ds));
 	dsl_dataset_name(origin, namebuf);
 	spa_history_log_internal_ds(ds, "clone", tx,
 	    "origin=%s (%llu)", namebuf, (u_longlong_t)origin->ds_object);
 	dsl_dataset_rele(ds, FTAG);
 	dsl_dataset_rele(origin, FTAG);
 	dsl_dir_rele(pdd, FTAG);
 }
 
 int
 dmu_objset_clone(const char *clone, const char *origin)
 {
 	dmu_objset_clone_arg_t doca;
 
+	cred_t *cr = CRED();
+	crhold(cr);
+
 	doca.doca_clone = clone;
 	doca.doca_origin = origin;
-	doca.doca_cred = CRED();
-	doca.doca_proc = curproc;
+	doca.doca_cred = cr;
 
 	int rv = dsl_sync_task(clone,
 	    dmu_objset_clone_check, dmu_objset_clone_sync, &doca,
 	    6, ZFS_SPACE_CHECK_NORMAL);
 
 	if (rv == 0)
 		zvol_create_minor(clone);
 
+	crfree(cr);
+
 	return (rv);
 }
 
 int
 dmu_objset_snapshot_one(const char *fsname, const char *snapname)
 {
 	int err;
 	char *longsnap = kmem_asprintf("%s@%s", fsname, snapname);
 	nvlist_t *snaps = fnvlist_alloc();
 
 	fnvlist_add_boolean(snaps, longsnap);
 	kmem_strfree(longsnap);
 	err = dsl_dataset_snapshot(snaps, NULL, NULL);
 	fnvlist_free(snaps);
 	return (err);
 }
 
 static void
 dmu_objset_upgrade_task_cb(void *data)
 {
 	objset_t *os = data;
 
 	mutex_enter(&os->os_upgrade_lock);
 	os->os_upgrade_status = EINTR;
 	if (!os->os_upgrade_exit) {
 		int status;
 
 		mutex_exit(&os->os_upgrade_lock);
 
 		status = os->os_upgrade_cb(os);
 
 		mutex_enter(&os->os_upgrade_lock);
 
 		os->os_upgrade_status = status;
 	}
 	os->os_upgrade_exit = B_TRUE;
 	os->os_upgrade_id = 0;
 	mutex_exit(&os->os_upgrade_lock);
 	dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);
 }
 
 static void
 dmu_objset_upgrade(objset_t *os, dmu_objset_upgrade_cb_t cb)
 {
 	if (os->os_upgrade_id != 0)
 		return;
 
 	ASSERT(dsl_pool_config_held(dmu_objset_pool(os)));
 	dsl_dataset_long_hold(dmu_objset_ds(os), upgrade_tag);
 
 	mutex_enter(&os->os_upgrade_lock);
 	if (os->os_upgrade_id == 0 && os->os_upgrade_status == 0) {
 		os->os_upgrade_exit = B_FALSE;
 		os->os_upgrade_cb = cb;
 		os->os_upgrade_id = taskq_dispatch(
 		    os->os_spa->spa_upgrade_taskq,
 		    dmu_objset_upgrade_task_cb, os, TQ_SLEEP);
 		if (os->os_upgrade_id == TASKQID_INVALID) {
 			dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);
 			os->os_upgrade_status = ENOMEM;
 		}
 	} else {
 		dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);
 	}
 	mutex_exit(&os->os_upgrade_lock);
 }
 
 static void
 dmu_objset_upgrade_stop(objset_t *os)
 {
 	mutex_enter(&os->os_upgrade_lock);
 	os->os_upgrade_exit = B_TRUE;
 	if (os->os_upgrade_id != 0) {
 		taskqid_t id = os->os_upgrade_id;
 
 		os->os_upgrade_id = 0;
 		mutex_exit(&os->os_upgrade_lock);
 
 		if ((taskq_cancel_id(os->os_spa->spa_upgrade_taskq, id)) == 0) {
 			dsl_dataset_long_rele(dmu_objset_ds(os), upgrade_tag);
 		}
 		txg_wait_synced(os->os_spa->spa_dsl_pool, 0);
 	} else {
 		mutex_exit(&os->os_upgrade_lock);
 	}
 }
 
 static void
 dmu_objset_sync_dnodes(multilist_sublist_t *list, dmu_tx_t *tx)
 {
 	dnode_t *dn;
 
 	while ((dn = multilist_sublist_head(list)) != NULL) {
 		ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT);
 		ASSERT(dn->dn_dbuf->db_data_pending);
 		/*
 		 * Initialize dn_zio outside dnode_sync() because the
 		 * meta-dnode needs to set it outside dnode_sync().
 		 */
 		dn->dn_zio = dn->dn_dbuf->db_data_pending->dr_zio;
 		ASSERT(dn->dn_zio);
 
 		ASSERT3U(dn->dn_nlevels, <=, DN_MAX_LEVELS);
 		multilist_sublist_remove(list, dn);
 
 		/*
 		 * See the comment above dnode_rele_task() for an explanation
 		 * of why this dnode hold is always needed (even when not
 		 * doing user accounting).
 		 */
 		multilist_t *newlist = &dn->dn_objset->os_synced_dnodes;
 		(void) dnode_add_ref(dn, newlist);
 		multilist_insert(newlist, dn);
 
 		dnode_sync(dn, tx);
 	}
 }
 
 static void
 dmu_objset_write_ready(zio_t *zio, arc_buf_t *abuf, void *arg)
 {
 	(void) abuf;
 	blkptr_t *bp = zio->io_bp;
 	objset_t *os = arg;
 	dnode_phys_t *dnp = &os->os_phys->os_meta_dnode;
 	uint64_t fill = 0;
 
 	ASSERT(!BP_IS_EMBEDDED(bp));
 	ASSERT3U(BP_GET_TYPE(bp), ==, DMU_OT_OBJSET);
 	ASSERT0(BP_GET_LEVEL(bp));
 
 	/*
 	 * Update rootbp fill count: it should be the number of objects
 	 * allocated in the object set (not counting the "special"
 	 * objects that are stored in the objset_phys_t -- the meta
 	 * dnode and user/group/project accounting objects).
 	 */
 	for (int i = 0; i < dnp->dn_nblkptr; i++)
 		fill += BP_GET_FILL(&dnp->dn_blkptr[i]);
 
 	BP_SET_FILL(bp, fill);
 
 	if (os->os_dsl_dataset != NULL)
 		rrw_enter(&os->os_dsl_dataset->ds_bp_rwlock, RW_WRITER, FTAG);
 	*os->os_rootbp = *bp;
 	if (os->os_dsl_dataset != NULL)
 		rrw_exit(&os->os_dsl_dataset->ds_bp_rwlock, FTAG);
 }
 
 static void
 dmu_objset_write_done(zio_t *zio, arc_buf_t *abuf, void *arg)
 {
 	(void) abuf;
 	blkptr_t *bp = zio->io_bp;
 	blkptr_t *bp_orig = &zio->io_bp_orig;
 	objset_t *os = arg;
 
 	if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
 		ASSERT(BP_EQUAL(bp, bp_orig));
 	} else {
 		dsl_dataset_t *ds = os->os_dsl_dataset;
 		dmu_tx_t *tx = os->os_synctx;
 
 		(void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE);
 		dsl_dataset_block_born(ds, bp, tx);
 	}
 	kmem_free(bp, sizeof (*bp));
 }
 
 typedef struct sync_dnodes_arg {
 	multilist_t *sda_list;
 	int sda_sublist_idx;
 	multilist_t *sda_newlist;
 	dmu_tx_t *sda_tx;
 } sync_dnodes_arg_t;
 
 static void
 sync_dnodes_task(void *arg)
 {
 	sync_dnodes_arg_t *sda = arg;
 
 	multilist_sublist_t *ms =
 	    multilist_sublist_lock_idx(sda->sda_list, sda->sda_sublist_idx);
 
 	dmu_objset_sync_dnodes(ms, sda->sda_tx);
 
 	multilist_sublist_unlock(ms);
 
 	kmem_free(sda, sizeof (*sda));
 }
 
 
 /* called from dsl */
 void
 dmu_objset_sync(objset_t *os, zio_t *pio, dmu_tx_t *tx)
 {
 	int txgoff;
 	zbookmark_phys_t zb;
 	zio_prop_t zp;
 	zio_t *zio;
 	list_t *list;
 	dbuf_dirty_record_t *dr;
 	int num_sublists;
 	multilist_t *ml;
 	blkptr_t *blkptr_copy = kmem_alloc(sizeof (*os->os_rootbp), KM_SLEEP);
 	*blkptr_copy = *os->os_rootbp;
 
 	dprintf_ds(os->os_dsl_dataset, "txg=%llu\n", (u_longlong_t)tx->tx_txg);
 
 	ASSERT(dmu_tx_is_syncing(tx));
 	/* XXX the write_done callback should really give us the tx... */
 	os->os_synctx = tx;
 
 	if (os->os_dsl_dataset == NULL) {
 		/*
 		 * This is the MOS.  If we have upgraded,
 		 * spa_max_replication() could change, so reset
 		 * os_copies here.
 		 */
 		os->os_copies = spa_max_replication(os->os_spa);
 	}
 
 	/*
 	 * Create the root block IO
 	 */
 	SET_BOOKMARK(&zb, os->os_dsl_dataset ?
 	    os->os_dsl_dataset->ds_object : DMU_META_OBJSET,
 	    ZB_ROOT_OBJECT, ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
 	arc_release(os->os_phys_buf, &os->os_phys_buf);
 
 	dmu_write_policy(os, NULL, 0, 0, &zp);
 
 	/*
 	 * If we are either claiming the ZIL or doing a raw receive, write
 	 * out the os_phys_buf raw. Neither of these actions will effect the
 	 * MAC at this point.
 	 */
 	if (os->os_raw_receive ||
 	    os->os_next_write_raw[tx->tx_txg & TXG_MASK]) {
 		ASSERT(os->os_encrypted);
 		arc_convert_to_raw(os->os_phys_buf,
 		    os->os_dsl_dataset->ds_object, ZFS_HOST_BYTEORDER,
 		    DMU_OT_OBJSET, NULL, NULL, NULL);
 	}
 
 	zio = arc_write(pio, os->os_spa, tx->tx_txg,
 	    blkptr_copy, os->os_phys_buf, B_FALSE, dmu_os_is_l2cacheable(os),
 	    &zp, dmu_objset_write_ready, NULL, dmu_objset_write_done,
 	    os, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb);
 
 	/*
 	 * Sync special dnodes - the parent IO for the sync is the root block
 	 */
 	DMU_META_DNODE(os)->dn_zio = zio;
 	dnode_sync(DMU_META_DNODE(os), tx);
 
 	os->os_phys->os_flags = os->os_flags;
 
 	if (DMU_USERUSED_DNODE(os) &&
 	    DMU_USERUSED_DNODE(os)->dn_type != DMU_OT_NONE) {
 		DMU_USERUSED_DNODE(os)->dn_zio = zio;
 		dnode_sync(DMU_USERUSED_DNODE(os), tx);
 		DMU_GROUPUSED_DNODE(os)->dn_zio = zio;
 		dnode_sync(DMU_GROUPUSED_DNODE(os), tx);
 	}
 
 	if (DMU_PROJECTUSED_DNODE(os) &&
 	    DMU_PROJECTUSED_DNODE(os)->dn_type != DMU_OT_NONE) {
 		DMU_PROJECTUSED_DNODE(os)->dn_zio = zio;
 		dnode_sync(DMU_PROJECTUSED_DNODE(os), tx);
 	}
 
 	txgoff = tx->tx_txg & TXG_MASK;
 
 	/*
 	 * We must create the list here because it uses the
 	 * dn_dirty_link[] of this txg.  But it may already
 	 * exist because we call dsl_dataset_sync() twice per txg.
 	 */
 	if (os->os_synced_dnodes.ml_sublists == NULL) {
 		multilist_create(&os->os_synced_dnodes, sizeof (dnode_t),
 		    offsetof(dnode_t, dn_dirty_link[txgoff]),
 		    dnode_multilist_index_func);
 	} else {
 		ASSERT3U(os->os_synced_dnodes.ml_offset, ==,
 		    offsetof(dnode_t, dn_dirty_link[txgoff]));
 	}
 
 	ml = &os->os_dirty_dnodes[txgoff];
 	num_sublists = multilist_get_num_sublists(ml);
 	for (int i = 0; i < num_sublists; i++) {
 		if (multilist_sublist_is_empty_idx(ml, i))
 			continue;
 		sync_dnodes_arg_t *sda = kmem_alloc(sizeof (*sda), KM_SLEEP);
 		sda->sda_list = ml;
 		sda->sda_sublist_idx = i;
 		sda->sda_tx = tx;
 		(void) taskq_dispatch(dmu_objset_pool(os)->dp_sync_taskq,
 		    sync_dnodes_task, sda, 0);
 		/* callback frees sda */
 	}
 	taskq_wait(dmu_objset_pool(os)->dp_sync_taskq);
 
 	list = &DMU_META_DNODE(os)->dn_dirty_records[txgoff];
 	while ((dr = list_remove_head(list)) != NULL) {
 		ASSERT0(dr->dr_dbuf->db_level);
 		zio_nowait(dr->dr_zio);
 	}
 
 	/* Enable dnode backfill if enough objects have been freed. */
 	if (os->os_freed_dnodes >= dmu_rescan_dnode_threshold) {
 		os->os_rescan_dnodes = B_TRUE;
 		os->os_freed_dnodes = 0;
 	}
 
 	/*
 	 * Free intent log blocks up to this tx.
 	 */
 	zil_sync(os->os_zil, tx);
 	os->os_phys->os_zil_header = os->os_zil_header;
 	zio_nowait(zio);
 }
 
 boolean_t
 dmu_objset_is_dirty(objset_t *os, uint64_t txg)
 {
 	return (!multilist_is_empty(&os->os_dirty_dnodes[txg & TXG_MASK]));
 }
 
 static file_info_cb_t *file_cbs[DMU_OST_NUMTYPES];
 
 void
 dmu_objset_register_type(dmu_objset_type_t ost, file_info_cb_t *cb)
 {
 	file_cbs[ost] = cb;
 }
 
 int
 dmu_get_file_info(objset_t *os, dmu_object_type_t bonustype, const void *data,
     zfs_file_info_t *zfi)
 {
 	file_info_cb_t *cb = file_cbs[os->os_phys->os_type];
 	if (cb == NULL)
 		return (EINVAL);
 	return (cb(bonustype, data, zfi));
 }
 
 boolean_t
 dmu_objset_userused_enabled(objset_t *os)
 {
 	return (spa_version(os->os_spa) >= SPA_VERSION_USERSPACE &&
 	    file_cbs[os->os_phys->os_type] != NULL &&
 	    DMU_USERUSED_DNODE(os) != NULL);
 }
 
 boolean_t
 dmu_objset_userobjused_enabled(objset_t *os)
 {
 	return (dmu_objset_userused_enabled(os) &&
 	    spa_feature_is_enabled(os->os_spa, SPA_FEATURE_USEROBJ_ACCOUNTING));
 }
 
 boolean_t
 dmu_objset_projectquota_enabled(objset_t *os)
 {
 	return (file_cbs[os->os_phys->os_type] != NULL &&
 	    DMU_PROJECTUSED_DNODE(os) != NULL &&
 	    spa_feature_is_enabled(os->os_spa, SPA_FEATURE_PROJECT_QUOTA));
 }
 
 typedef struct userquota_node {
 	/* must be in the first filed, see userquota_update_cache() */
 	char		uqn_id[20 + DMU_OBJACCT_PREFIX_LEN];
 	int64_t		uqn_delta;
 	avl_node_t	uqn_node;
 } userquota_node_t;
 
 typedef struct userquota_cache {
 	avl_tree_t uqc_user_deltas;
 	avl_tree_t uqc_group_deltas;
 	avl_tree_t uqc_project_deltas;
 } userquota_cache_t;
 
 static int
 userquota_compare(const void *l, const void *r)
 {
 	const userquota_node_t *luqn = l;
 	const userquota_node_t *ruqn = r;
 	int rv;
 
 	/*
 	 * NB: can only access uqn_id because userquota_update_cache() doesn't
 	 * pass in an entire userquota_node_t.
 	 */
 	rv = strcmp(luqn->uqn_id, ruqn->uqn_id);
 
 	return (TREE_ISIGN(rv));
 }
 
 static void
 do_userquota_cacheflush(objset_t *os, userquota_cache_t *cache, dmu_tx_t *tx)
 {
 	void *cookie;
 	userquota_node_t *uqn;
 
 	ASSERT(dmu_tx_is_syncing(tx));
 
 	cookie = NULL;
 	while ((uqn = avl_destroy_nodes(&cache->uqc_user_deltas,
 	    &cookie)) != NULL) {
 		/*
 		 * os_userused_lock protects against concurrent calls to
 		 * zap_increment_int().  It's needed because zap_increment_int()
 		 * is not thread-safe (i.e. not atomic).
 		 */
 		mutex_enter(&os->os_userused_lock);
 		VERIFY0(zap_increment(os, DMU_USERUSED_OBJECT,
 		    uqn->uqn_id, uqn->uqn_delta, tx));
 		mutex_exit(&os->os_userused_lock);
 		kmem_free(uqn, sizeof (*uqn));
 	}
 	avl_destroy(&cache->uqc_user_deltas);
 
 	cookie = NULL;
 	while ((uqn = avl_destroy_nodes(&cache->uqc_group_deltas,
 	    &cookie)) != NULL) {
 		mutex_enter(&os->os_userused_lock);
 		VERIFY0(zap_increment(os, DMU_GROUPUSED_OBJECT,
 		    uqn->uqn_id, uqn->uqn_delta, tx));
 		mutex_exit(&os->os_userused_lock);
 		kmem_free(uqn, sizeof (*uqn));
 	}
 	avl_destroy(&cache->uqc_group_deltas);
 
 	if (dmu_objset_projectquota_enabled(os)) {
 		cookie = NULL;
 		while ((uqn = avl_destroy_nodes(&cache->uqc_project_deltas,
 		    &cookie)) != NULL) {
 			mutex_enter(&os->os_userused_lock);
 			VERIFY0(zap_increment(os, DMU_PROJECTUSED_OBJECT,
 			    uqn->uqn_id, uqn->uqn_delta, tx));
 			mutex_exit(&os->os_userused_lock);
 			kmem_free(uqn, sizeof (*uqn));
 		}
 		avl_destroy(&cache->uqc_project_deltas);
 	}
 }
 
 static void
 userquota_update_cache(avl_tree_t *avl, const char *id, int64_t delta)
 {
 	userquota_node_t *uqn;
 	avl_index_t idx;
 
 	ASSERT(strlen(id) < sizeof (uqn->uqn_id));
 	/*
 	 * Use id directly for searching because uqn_id is the first field of
 	 * userquota_node_t and fields after uqn_id won't be accessed in
 	 * avl_find().
 	 */
 	uqn = avl_find(avl, (const void *)id, &idx);
 	if (uqn == NULL) {
 		uqn = kmem_zalloc(sizeof (*uqn), KM_SLEEP);
 		strlcpy(uqn->uqn_id, id, sizeof (uqn->uqn_id));
 		avl_insert(avl, uqn, idx);
 	}
 	uqn->uqn_delta += delta;
 }
 
 static void
 do_userquota_update(objset_t *os, userquota_cache_t *cache, uint64_t used,
     uint64_t flags, uint64_t user, uint64_t group, uint64_t project,
     boolean_t subtract)
 {
 	if (flags & DNODE_FLAG_USERUSED_ACCOUNTED) {
 		int64_t delta = DNODE_MIN_SIZE + used;
 		char name[20];
 
 		if (subtract)
 			delta = -delta;
 
 		(void) snprintf(name, sizeof (name), "%llx", (longlong_t)user);
 		userquota_update_cache(&cache->uqc_user_deltas, name, delta);
 
 		(void) snprintf(name, sizeof (name), "%llx", (longlong_t)group);
 		userquota_update_cache(&cache->uqc_group_deltas, name, delta);
 
 		if (dmu_objset_projectquota_enabled(os)) {
 			(void) snprintf(name, sizeof (name), "%llx",
 			    (longlong_t)project);
 			userquota_update_cache(&cache->uqc_project_deltas,
 			    name, delta);
 		}
 	}
 }
 
 static void
 do_userobjquota_update(objset_t *os, userquota_cache_t *cache, uint64_t flags,
     uint64_t user, uint64_t group, uint64_t project, boolean_t subtract)
 {
 	if (flags & DNODE_FLAG_USEROBJUSED_ACCOUNTED) {
 		char name[20 + DMU_OBJACCT_PREFIX_LEN];
 		int delta = subtract ? -1 : 1;
 
 		(void) snprintf(name, sizeof (name), DMU_OBJACCT_PREFIX "%llx",
 		    (longlong_t)user);
 		userquota_update_cache(&cache->uqc_user_deltas, name, delta);
 
 		(void) snprintf(name, sizeof (name), DMU_OBJACCT_PREFIX "%llx",
 		    (longlong_t)group);
 		userquota_update_cache(&cache->uqc_group_deltas, name, delta);
 
 		if (dmu_objset_projectquota_enabled(os)) {
 			(void) snprintf(name, sizeof (name),
 			    DMU_OBJACCT_PREFIX "%llx", (longlong_t)project);
 			userquota_update_cache(&cache->uqc_project_deltas,
 			    name, delta);
 		}
 	}
 }
 
 typedef struct userquota_updates_arg {
 	objset_t *uua_os;
 	int uua_sublist_idx;
 	dmu_tx_t *uua_tx;
 } userquota_updates_arg_t;
 
 static void
 userquota_updates_task(void *arg)
 {
 	userquota_updates_arg_t *uua = arg;
 	objset_t *os = uua->uua_os;
 	dmu_tx_t *tx = uua->uua_tx;
 	dnode_t *dn;
 	userquota_cache_t cache = { { 0 } };
 
 	multilist_sublist_t *list = multilist_sublist_lock_idx(
 	    &os->os_synced_dnodes, uua->uua_sublist_idx);
 
 	ASSERT(multilist_sublist_head(list) == NULL ||
 	    dmu_objset_userused_enabled(os));
 	avl_create(&cache.uqc_user_deltas, userquota_compare,
 	    sizeof (userquota_node_t), offsetof(userquota_node_t, uqn_node));
 	avl_create(&cache.uqc_group_deltas, userquota_compare,
 	    sizeof (userquota_node_t), offsetof(userquota_node_t, uqn_node));
 	if (dmu_objset_projectquota_enabled(os))
 		avl_create(&cache.uqc_project_deltas, userquota_compare,
 		    sizeof (userquota_node_t), offsetof(userquota_node_t,
 		    uqn_node));
 
 	while ((dn = multilist_sublist_head(list)) != NULL) {
 		int flags;
 		ASSERT(!DMU_OBJECT_IS_SPECIAL(dn->dn_object));
 		ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE ||
 		    dn->dn_phys->dn_flags &
 		    DNODE_FLAG_USERUSED_ACCOUNTED);
 
 		flags = dn->dn_id_flags;
 		ASSERT(flags);
 		if (flags & DN_ID_OLD_EXIST)  {
 			do_userquota_update(os, &cache, dn->dn_oldused,
 			    dn->dn_oldflags, dn->dn_olduid, dn->dn_oldgid,
 			    dn->dn_oldprojid, B_TRUE);
 			do_userobjquota_update(os, &cache, dn->dn_oldflags,
 			    dn->dn_olduid, dn->dn_oldgid,
 			    dn->dn_oldprojid, B_TRUE);
 		}
 		if (flags & DN_ID_NEW_EXIST) {
 			do_userquota_update(os, &cache,
 			    DN_USED_BYTES(dn->dn_phys), dn->dn_phys->dn_flags,
 			    dn->dn_newuid, dn->dn_newgid,
 			    dn->dn_newprojid, B_FALSE);
 			do_userobjquota_update(os, &cache,
 			    dn->dn_phys->dn_flags, dn->dn_newuid, dn->dn_newgid,
 			    dn->dn_newprojid, B_FALSE);
 		}
 
 		mutex_enter(&dn->dn_mtx);
 		dn->dn_oldused = 0;
 		dn->dn_oldflags = 0;
 		if (dn->dn_id_flags & DN_ID_NEW_EXIST) {
 			dn->dn_olduid = dn->dn_newuid;
 			dn->dn_oldgid = dn->dn_newgid;
 			dn->dn_oldprojid = dn->dn_newprojid;
 			dn->dn_id_flags |= DN_ID_OLD_EXIST;
 			if (dn->dn_bonuslen == 0)
 				dn->dn_id_flags |= DN_ID_CHKED_SPILL;
 			else
 				dn->dn_id_flags |= DN_ID_CHKED_BONUS;
 		}
 		dn->dn_id_flags &= ~(DN_ID_NEW_EXIST);
 		mutex_exit(&dn->dn_mtx);
 
 		multilist_sublist_remove(list, dn);
 		dnode_rele(dn, &os->os_synced_dnodes);
 	}
 	do_userquota_cacheflush(os, &cache, tx);
 	multilist_sublist_unlock(list);
 	kmem_free(uua, sizeof (*uua));
 }
 
 /*
  * Release dnode holds from dmu_objset_sync_dnodes().  When the dnode is being
  * synced (i.e. we have issued the zio's for blocks in the dnode), it can't be
  * evicted because the block containing the dnode can't be evicted until it is
  * written out.  However, this hold is necessary to prevent the dnode_t from
  * being moved (via dnode_move()) while it's still referenced by
  * dbuf_dirty_record_t:dr_dnode.  And dr_dnode is needed for
  * dirty_lightweight_leaf-type dirty records.
  *
  * If we are doing user-object accounting, the dnode_rele() happens from
  * userquota_updates_task() instead.
  */
 static void
 dnode_rele_task(void *arg)
 {
 	userquota_updates_arg_t *uua = arg;
 	objset_t *os = uua->uua_os;
 
 	multilist_sublist_t *list = multilist_sublist_lock_idx(
 	    &os->os_synced_dnodes, uua->uua_sublist_idx);
 
 	dnode_t *dn;
 	while ((dn = multilist_sublist_head(list)) != NULL) {
 		multilist_sublist_remove(list, dn);
 		dnode_rele(dn, &os->os_synced_dnodes);
 	}
 	multilist_sublist_unlock(list);
 	kmem_free(uua, sizeof (*uua));
 }
 
 /*
  * Return TRUE if userquota updates are needed.
  */
 static boolean_t
 dmu_objset_do_userquota_updates_prep(objset_t *os, dmu_tx_t *tx)
 {
 	if (!dmu_objset_userused_enabled(os))
 		return (B_FALSE);
 
 	/*
 	 * If this is a raw receive just return and handle accounting
 	 * later when we have the keys loaded. We also don't do user
 	 * accounting during claiming since the datasets are not owned
 	 * for the duration of claiming and this txg should only be
 	 * used for recovery.
 	 */
 	if (os->os_encrypted && dmu_objset_is_receiving(os))
 		return (B_FALSE);
 
 	if (tx->tx_txg <= os->os_spa->spa_claim_max_txg)
 		return (B_FALSE);
 
 	/* Allocate the user/group/project used objects if necessary. */
 	if (DMU_USERUSED_DNODE(os)->dn_type == DMU_OT_NONE) {
 		VERIFY0(zap_create_claim(os,
 		    DMU_USERUSED_OBJECT,
 		    DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx));
 		VERIFY0(zap_create_claim(os,
 		    DMU_GROUPUSED_OBJECT,
 		    DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx));
 	}
 
 	if (dmu_objset_projectquota_enabled(os) &&
 	    DMU_PROJECTUSED_DNODE(os)->dn_type == DMU_OT_NONE) {
 		VERIFY0(zap_create_claim(os, DMU_PROJECTUSED_OBJECT,
 		    DMU_OT_USERGROUP_USED, DMU_OT_NONE, 0, tx));
 	}
 	return (B_TRUE);
 }
 
 /*
  * Dispatch taskq tasks to dp_sync_taskq to update the user accounting, and
  * also release the holds on the dnodes from dmu_objset_sync_dnodes().
  * The caller must taskq_wait(dp_sync_taskq).
  */
 void
 dmu_objset_sync_done(objset_t *os, dmu_tx_t *tx)
 {
 	boolean_t need_userquota = dmu_objset_do_userquota_updates_prep(os, tx);
 
 	int num_sublists = multilist_get_num_sublists(&os->os_synced_dnodes);
 	for (int i = 0; i < num_sublists; i++) {
 		userquota_updates_arg_t *uua =
 		    kmem_alloc(sizeof (*uua), KM_SLEEP);
 		uua->uua_os = os;
 		uua->uua_sublist_idx = i;
 		uua->uua_tx = tx;
 
 		/*
 		 * If we don't need to update userquotas, use
 		 * dnode_rele_task() to call dnode_rele()
 		 */
 		(void) taskq_dispatch(dmu_objset_pool(os)->dp_sync_taskq,
 		    need_userquota ? userquota_updates_task : dnode_rele_task,
 		    uua, 0);
 		/* callback frees uua */
 	}
 }
 
 
 /*
  * Returns a pointer to data to find uid/gid from
  *
  * If a dirty record for transaction group that is syncing can't
  * be found then NULL is returned.  In the NULL case it is assumed
  * the uid/gid aren't changing.
  */
 static void *
 dmu_objset_userquota_find_data(dmu_buf_impl_t *db, dmu_tx_t *tx)
 {
 	dbuf_dirty_record_t *dr;
 	void *data;
 
 	if (db->db_dirtycnt == 0)
 		return (db->db.db_data);  /* Nothing is changing */
 
 	dr = dbuf_find_dirty_eq(db, tx->tx_txg);
 
 	if (dr == NULL) {
 		data = NULL;
 	} else {
 		if (dr->dr_dnode->dn_bonuslen == 0 &&
 		    dr->dr_dbuf->db_blkid == DMU_SPILL_BLKID)
 			data = dr->dt.dl.dr_data->b_data;
 		else
 			data = dr->dt.dl.dr_data;
 	}
 
 	return (data);
 }
 
 void
 dmu_objset_userquota_get_ids(dnode_t *dn, boolean_t before, dmu_tx_t *tx)
 {
 	objset_t *os = dn->dn_objset;
 	void *data = NULL;
 	dmu_buf_impl_t *db = NULL;
 	int flags = dn->dn_id_flags;
 	int error;
 	boolean_t have_spill = B_FALSE;
 
 	if (!dmu_objset_userused_enabled(dn->dn_objset))
 		return;
 
 	/*
 	 * Raw receives introduce a problem with user accounting. Raw
 	 * receives cannot update the user accounting info because the
 	 * user ids and the sizes are encrypted. To guarantee that we
 	 * never end up with bad user accounting, we simply disable it
 	 * during raw receives. We also disable this for normal receives
 	 * so that an incremental raw receive may be done on top of an
 	 * existing non-raw receive.
 	 */
 	if (os->os_encrypted && dmu_objset_is_receiving(os))
 		return;
 
 	if (before && (flags & (DN_ID_CHKED_BONUS|DN_ID_OLD_EXIST|
 	    DN_ID_CHKED_SPILL)))
 		return;
 
 	if (before && dn->dn_bonuslen != 0)
 		data = DN_BONUS(dn->dn_phys);
 	else if (!before && dn->dn_bonuslen != 0) {
 		if (dn->dn_bonus) {
 			db = dn->dn_bonus;
 			mutex_enter(&db->db_mtx);
 			data = dmu_objset_userquota_find_data(db, tx);
 		} else {
 			data = DN_BONUS(dn->dn_phys);
 		}
 	} else if (dn->dn_bonuslen == 0 && dn->dn_bonustype == DMU_OT_SA) {
 			int rf = 0;
 
 			if (RW_WRITE_HELD(&dn->dn_struct_rwlock))
 				rf |= DB_RF_HAVESTRUCT;
 			error = dmu_spill_hold_by_dnode(dn,
 			    rf | DB_RF_MUST_SUCCEED,
 			    FTAG, (dmu_buf_t **)&db);
 			ASSERT(error == 0);
 			mutex_enter(&db->db_mtx);
 			data = (before) ? db->db.db_data :
 			    dmu_objset_userquota_find_data(db, tx);
 			have_spill = B_TRUE;
 	} else {
 		mutex_enter(&dn->dn_mtx);
 		dn->dn_id_flags |= DN_ID_CHKED_BONUS;
 		mutex_exit(&dn->dn_mtx);
 		return;
 	}
 
 	/*
 	 * Must always call the callback in case the object
 	 * type has changed and that type isn't an object type to track
 	 */
 	zfs_file_info_t zfi;
 	error = file_cbs[os->os_phys->os_type](dn->dn_bonustype, data, &zfi);
 
 	if (before) {
 		ASSERT(data);
 		dn->dn_olduid = zfi.zfi_user;
 		dn->dn_oldgid = zfi.zfi_group;
 		dn->dn_oldprojid = zfi.zfi_project;
 	} else if (data) {
 		dn->dn_newuid = zfi.zfi_user;
 		dn->dn_newgid = zfi.zfi_group;
 		dn->dn_newprojid = zfi.zfi_project;
 	}
 
 	/*
 	 * Preserve existing uid/gid when the callback can't determine
 	 * what the new uid/gid are and the callback returned EEXIST.
 	 * The EEXIST error tells us to just use the existing uid/gid.
 	 * If we don't know what the old values are then just assign
 	 * them to 0, since that is a new file  being created.
 	 */
 	if (!before && data == NULL && error == EEXIST) {
 		if (flags & DN_ID_OLD_EXIST) {
 			dn->dn_newuid = dn->dn_olduid;
 			dn->dn_newgid = dn->dn_oldgid;
 			dn->dn_newprojid = dn->dn_oldprojid;
 		} else {
 			dn->dn_newuid = 0;
 			dn->dn_newgid = 0;
 			dn->dn_newprojid = ZFS_DEFAULT_PROJID;
 		}
 		error = 0;
 	}
 
 	if (db)
 		mutex_exit(&db->db_mtx);
 
 	mutex_enter(&dn->dn_mtx);
 	if (error == 0 && before)
 		dn->dn_id_flags |= DN_ID_OLD_EXIST;
 	if (error == 0 && !before)
 		dn->dn_id_flags |= DN_ID_NEW_EXIST;
 
 	if (have_spill) {
 		dn->dn_id_flags |= DN_ID_CHKED_SPILL;
 	} else {
 		dn->dn_id_flags |= DN_ID_CHKED_BONUS;
 	}
 	mutex_exit(&dn->dn_mtx);
 	if (have_spill)
 		dmu_buf_rele((dmu_buf_t *)db, FTAG);
 }
 
 boolean_t
 dmu_objset_userspace_present(objset_t *os)
 {
 	return (os->os_phys->os_flags &
 	    OBJSET_FLAG_USERACCOUNTING_COMPLETE);
 }
 
 boolean_t
 dmu_objset_userobjspace_present(objset_t *os)
 {
 	return (os->os_phys->os_flags &
 	    OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE);
 }
 
 boolean_t
 dmu_objset_projectquota_present(objset_t *os)
 {
 	return (os->os_phys->os_flags &
 	    OBJSET_FLAG_PROJECTQUOTA_COMPLETE);
 }
 
 static int
 dmu_objset_space_upgrade(objset_t *os)
 {
 	uint64_t obj;
 	int err = 0;
 
 	/*
 	 * We simply need to mark every object dirty, so that it will be
 	 * synced out and now accounted.  If this is called
 	 * concurrently, or if we already did some work before crashing,
 	 * that's fine, since we track each object's accounted state
 	 * independently.
 	 */
 
 	for (obj = 0; err == 0; err = dmu_object_next(os, &obj, FALSE, 0)) {
 		dmu_tx_t *tx;
 		dmu_buf_t *db;
 		int objerr;
 
 		mutex_enter(&os->os_upgrade_lock);
 		if (os->os_upgrade_exit)
 			err = SET_ERROR(EINTR);
 		mutex_exit(&os->os_upgrade_lock);
 		if (err != 0)
 			return (err);
 
 		if (issig())
 			return (SET_ERROR(EINTR));
 
 		objerr = dmu_bonus_hold(os, obj, FTAG, &db);
 		if (objerr != 0)
 			continue;
 		tx = dmu_tx_create(os);
 		dmu_tx_hold_bonus(tx, obj);
 		objerr = dmu_tx_assign(tx, TXG_WAIT);
 		if (objerr != 0) {
 			dmu_buf_rele(db, FTAG);
 			dmu_tx_abort(tx);
 			continue;
 		}
 		dmu_buf_will_dirty(db, tx);
 		dmu_buf_rele(db, FTAG);
 		dmu_tx_commit(tx);
 	}
 	return (0);
 }
 
 static int
 dmu_objset_userspace_upgrade_cb(objset_t *os)
 {
 	int err = 0;
 
 	if (dmu_objset_userspace_present(os))
 		return (0);
 	if (dmu_objset_is_snapshot(os))
 		return (SET_ERROR(EINVAL));
 	if (!dmu_objset_userused_enabled(os))
 		return (SET_ERROR(ENOTSUP));
 
 	err = dmu_objset_space_upgrade(os);
 	if (err)
 		return (err);
 
 	os->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
 	txg_wait_synced(dmu_objset_pool(os), 0);
 	return (0);
 }
 
 void
 dmu_objset_userspace_upgrade(objset_t *os)
 {
 	dmu_objset_upgrade(os, dmu_objset_userspace_upgrade_cb);
 }
 
 static int
 dmu_objset_id_quota_upgrade_cb(objset_t *os)
 {
 	int err = 0;
 
 	if (dmu_objset_userobjspace_present(os) &&
 	    dmu_objset_projectquota_present(os))
 		return (0);
 	if (dmu_objset_is_snapshot(os))
 		return (SET_ERROR(EINVAL));
 	if (!dmu_objset_userused_enabled(os))
 		return (SET_ERROR(ENOTSUP));
 	if (!dmu_objset_projectquota_enabled(os) &&
 	    dmu_objset_userobjspace_present(os))
 		return (SET_ERROR(ENOTSUP));
 
 	if (dmu_objset_userobjused_enabled(os))
 		dmu_objset_ds(os)->ds_feature_activation[
 		    SPA_FEATURE_USEROBJ_ACCOUNTING] = (void *)B_TRUE;
 	if (dmu_objset_projectquota_enabled(os))
 		dmu_objset_ds(os)->ds_feature_activation[
 		    SPA_FEATURE_PROJECT_QUOTA] = (void *)B_TRUE;
 
 	err = dmu_objset_space_upgrade(os);
 	if (err)
 		return (err);
 
 	os->os_flags |= OBJSET_FLAG_USERACCOUNTING_COMPLETE;
 	if (dmu_objset_userobjused_enabled(os))
 		os->os_flags |= OBJSET_FLAG_USEROBJACCOUNTING_COMPLETE;
 	if (dmu_objset_projectquota_enabled(os))
 		os->os_flags |= OBJSET_FLAG_PROJECTQUOTA_COMPLETE;
 
 	txg_wait_synced(dmu_objset_pool(os), 0);
 	return (0);
 }
 
 void
 dmu_objset_id_quota_upgrade(objset_t *os)
 {
 	dmu_objset_upgrade(os, dmu_objset_id_quota_upgrade_cb);
 }
 
 boolean_t
 dmu_objset_userobjspace_upgradable(objset_t *os)
 {
 	return (dmu_objset_type(os) == DMU_OST_ZFS &&
 	    !dmu_objset_is_snapshot(os) &&
 	    dmu_objset_userobjused_enabled(os) &&
 	    !dmu_objset_userobjspace_present(os) &&
 	    spa_writeable(dmu_objset_spa(os)));
 }
 
 boolean_t
 dmu_objset_projectquota_upgradable(objset_t *os)
 {
 	return (dmu_objset_type(os) == DMU_OST_ZFS &&
 	    !dmu_objset_is_snapshot(os) &&
 	    dmu_objset_projectquota_enabled(os) &&
 	    !dmu_objset_projectquota_present(os) &&
 	    spa_writeable(dmu_objset_spa(os)));
 }
 
 void
 dmu_objset_space(objset_t *os, uint64_t *refdbytesp, uint64_t *availbytesp,
     uint64_t *usedobjsp, uint64_t *availobjsp)
 {
 	dsl_dataset_space(os->os_dsl_dataset, refdbytesp, availbytesp,
 	    usedobjsp, availobjsp);
 }
 
 uint64_t
 dmu_objset_fsid_guid(objset_t *os)
 {
 	return (dsl_dataset_fsid_guid(os->os_dsl_dataset));
 }
 
 void
 dmu_objset_fast_stat(objset_t *os, dmu_objset_stats_t *stat)
 {
 	stat->dds_type = os->os_phys->os_type;
 	if (os->os_dsl_dataset)
 		dsl_dataset_fast_stat(os->os_dsl_dataset, stat);
 }
 
 void
 dmu_objset_stats(objset_t *os, nvlist_t *nv)
 {
 	ASSERT(os->os_dsl_dataset ||
 	    os->os_phys->os_type == DMU_OST_META);
 
 	if (os->os_dsl_dataset != NULL)
 		dsl_dataset_stats(os->os_dsl_dataset, nv);
 
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_TYPE,
 	    os->os_phys->os_type);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USERACCOUNTING,
 	    dmu_objset_userspace_present(os));
 }
 
 int
 dmu_objset_is_snapshot(objset_t *os)
 {
 	if (os->os_dsl_dataset != NULL)
 		return (os->os_dsl_dataset->ds_is_snapshot);
 	else
 		return (B_FALSE);
 }
 
 int
 dmu_snapshot_realname(objset_t *os, const char *name, char *real, int maxlen,
     boolean_t *conflict)
 {
 	dsl_dataset_t *ds = os->os_dsl_dataset;
 	uint64_t ignored;
 
 	if (dsl_dataset_phys(ds)->ds_snapnames_zapobj == 0)
 		return (SET_ERROR(ENOENT));
 
 	return (zap_lookup_norm(ds->ds_dir->dd_pool->dp_meta_objset,
 	    dsl_dataset_phys(ds)->ds_snapnames_zapobj, name, 8, 1, &ignored,
 	    MT_NORMALIZE, real, maxlen, conflict));
 }
 
 int
 dmu_snapshot_list_next(objset_t *os, int namelen, char *name,
     uint64_t *idp, uint64_t *offp, boolean_t *case_conflict)
 {
 	dsl_dataset_t *ds = os->os_dsl_dataset;
 	zap_cursor_t cursor;
 	zap_attribute_t attr;
 
 	ASSERT(dsl_pool_config_held(dmu_objset_pool(os)));
 
 	if (dsl_dataset_phys(ds)->ds_snapnames_zapobj == 0)
 		return (SET_ERROR(ENOENT));
 
 	zap_cursor_init_serialized(&cursor,
 	    ds->ds_dir->dd_pool->dp_meta_objset,
 	    dsl_dataset_phys(ds)->ds_snapnames_zapobj, *offp);
 
 	if (zap_cursor_retrieve(&cursor, &attr) != 0) {
 		zap_cursor_fini(&cursor);
 		return (SET_ERROR(ENOENT));
 	}
 
 	if (strlen(attr.za_name) + 1 > namelen) {
 		zap_cursor_fini(&cursor);
 		return (SET_ERROR(ENAMETOOLONG));
 	}
 
 	(void) strlcpy(name, attr.za_name, namelen);
 	if (idp)
 		*idp = attr.za_first_integer;
 	if (case_conflict)
 		*case_conflict = attr.za_normalization_conflict;
 	zap_cursor_advance(&cursor);
 	*offp = zap_cursor_serialize(&cursor);
 	zap_cursor_fini(&cursor);
 
 	return (0);
 }
 
 int
 dmu_snapshot_lookup(objset_t *os, const char *name, uint64_t *value)
 {
 	return (dsl_dataset_snap_lookup(os->os_dsl_dataset, name, value));
 }
 
 int
 dmu_dir_list_next(objset_t *os, int namelen, char *name,
     uint64_t *idp, uint64_t *offp)
 {
 	dsl_dir_t *dd = os->os_dsl_dataset->ds_dir;
 	zap_cursor_t cursor;
 	zap_attribute_t attr;
 
 	/* there is no next dir on a snapshot! */
 	if (os->os_dsl_dataset->ds_object !=
 	    dsl_dir_phys(dd)->dd_head_dataset_obj)
 		return (SET_ERROR(ENOENT));
 
 	zap_cursor_init_serialized(&cursor,
 	    dd->dd_pool->dp_meta_objset,
 	    dsl_dir_phys(dd)->dd_child_dir_zapobj, *offp);
 
 	if (zap_cursor_retrieve(&cursor, &attr) != 0) {
 		zap_cursor_fini(&cursor);
 		return (SET_ERROR(ENOENT));
 	}
 
 	if (strlen(attr.za_name) + 1 > namelen) {
 		zap_cursor_fini(&cursor);
 		return (SET_ERROR(ENAMETOOLONG));
 	}
 
 	(void) strlcpy(name, attr.za_name, namelen);
 	if (idp)
 		*idp = attr.za_first_integer;
 	zap_cursor_advance(&cursor);
 	*offp = zap_cursor_serialize(&cursor);
 	zap_cursor_fini(&cursor);
 
 	return (0);
 }
 
 typedef struct dmu_objset_find_ctx {
 	taskq_t		*dc_tq;
 	dsl_pool_t	*dc_dp;
 	uint64_t	dc_ddobj;
 	char		*dc_ddname; /* last component of ddobj's name */
 	int		(*dc_func)(dsl_pool_t *, dsl_dataset_t *, void *);
 	void		*dc_arg;
 	int		dc_flags;
 	kmutex_t	*dc_error_lock;
 	int		*dc_error;
 } dmu_objset_find_ctx_t;
 
 static void
 dmu_objset_find_dp_impl(dmu_objset_find_ctx_t *dcp)
 {
 	dsl_pool_t *dp = dcp->dc_dp;
 	dsl_dir_t *dd;
 	dsl_dataset_t *ds;
 	zap_cursor_t zc;
 	zap_attribute_t *attr;
 	uint64_t thisobj;
 	int err = 0;
 
 	/* don't process if there already was an error */
 	if (*dcp->dc_error != 0)
 		goto out;
 
 	/*
 	 * Note: passing the name (dc_ddname) here is optional, but it
 	 * improves performance because we don't need to call
 	 * zap_value_search() to determine the name.
 	 */
 	err = dsl_dir_hold_obj(dp, dcp->dc_ddobj, dcp->dc_ddname, FTAG, &dd);
 	if (err != 0)
 		goto out;
 
 	/* Don't visit hidden ($MOS & $ORIGIN) objsets. */
 	if (dd->dd_myname[0] == '$') {
 		dsl_dir_rele(dd, FTAG);
 		goto out;
 	}
 
 	thisobj = dsl_dir_phys(dd)->dd_head_dataset_obj;
 	attr = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
 
 	/*
 	 * Iterate over all children.
 	 */
 	if (dcp->dc_flags & DS_FIND_CHILDREN) {
 		for (zap_cursor_init(&zc, dp->dp_meta_objset,
 		    dsl_dir_phys(dd)->dd_child_dir_zapobj);
 		    zap_cursor_retrieve(&zc, attr) == 0;
 		    (void) zap_cursor_advance(&zc)) {
 			ASSERT3U(attr->za_integer_length, ==,
 			    sizeof (uint64_t));
 			ASSERT3U(attr->za_num_integers, ==, 1);
 
 			dmu_objset_find_ctx_t *child_dcp =
 			    kmem_alloc(sizeof (*child_dcp), KM_SLEEP);
 			*child_dcp = *dcp;
 			child_dcp->dc_ddobj = attr->za_first_integer;
 			child_dcp->dc_ddname = spa_strdup(attr->za_name);
 			if (dcp->dc_tq != NULL)
 				(void) taskq_dispatch(dcp->dc_tq,
 				    dmu_objset_find_dp_cb, child_dcp, TQ_SLEEP);
 			else
 				dmu_objset_find_dp_impl(child_dcp);
 		}
 		zap_cursor_fini(&zc);
 	}
 
 	/*
 	 * Iterate over all snapshots.
 	 */
 	if (dcp->dc_flags & DS_FIND_SNAPSHOTS) {
 		dsl_dataset_t *ds;
 		err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds);
 
 		if (err == 0) {
 			uint64_t snapobj;
 
 			snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
 			dsl_dataset_rele(ds, FTAG);
 
 			for (zap_cursor_init(&zc, dp->dp_meta_objset, snapobj);
 			    zap_cursor_retrieve(&zc, attr) == 0;
 			    (void) zap_cursor_advance(&zc)) {
 				ASSERT3U(attr->za_integer_length, ==,
 				    sizeof (uint64_t));
 				ASSERT3U(attr->za_num_integers, ==, 1);
 
 				err = dsl_dataset_hold_obj(dp,
 				    attr->za_first_integer, FTAG, &ds);
 				if (err != 0)
 					break;
 				err = dcp->dc_func(dp, ds, dcp->dc_arg);
 				dsl_dataset_rele(ds, FTAG);
 				if (err != 0)
 					break;
 			}
 			zap_cursor_fini(&zc);
 		}
 	}
 
 	kmem_free(attr, sizeof (zap_attribute_t));
 
 	if (err != 0) {
 		dsl_dir_rele(dd, FTAG);
 		goto out;
 	}
 
 	/*
 	 * Apply to self.
 	 */
 	err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds);
 
 	/*
 	 * Note: we hold the dir while calling dsl_dataset_hold_obj() so
 	 * that the dir will remain cached, and we won't have to re-instantiate
 	 * it (which could be expensive due to finding its name via
 	 * zap_value_search()).
 	 */
 	dsl_dir_rele(dd, FTAG);
 	if (err != 0)
 		goto out;
 	err = dcp->dc_func(dp, ds, dcp->dc_arg);
 	dsl_dataset_rele(ds, FTAG);
 
 out:
 	if (err != 0) {
 		mutex_enter(dcp->dc_error_lock);
 		/* only keep first error */
 		if (*dcp->dc_error == 0)
 			*dcp->dc_error = err;
 		mutex_exit(dcp->dc_error_lock);
 	}
 
 	if (dcp->dc_ddname != NULL)
 		spa_strfree(dcp->dc_ddname);
 	kmem_free(dcp, sizeof (*dcp));
 }
 
 static void
 dmu_objset_find_dp_cb(void *arg)
 {
 	dmu_objset_find_ctx_t *dcp = arg;
 	dsl_pool_t *dp = dcp->dc_dp;
 
 	/*
 	 * We need to get a pool_config_lock here, as there are several
 	 * assert(pool_config_held) down the stack. Getting a lock via
 	 * dsl_pool_config_enter is risky, as it might be stalled by a
 	 * pending writer. This would deadlock, as the write lock can
 	 * only be granted when our parent thread gives up the lock.
 	 * The _prio interface gives us priority over a pending writer.
 	 */
 	dsl_pool_config_enter_prio(dp, FTAG);
 
 	dmu_objset_find_dp_impl(dcp);
 
 	dsl_pool_config_exit(dp, FTAG);
 }
 
 /*
  * Find objsets under and including ddobj, call func(ds) on each.
  * The order for the enumeration is completely undefined.
  * func is called with dsl_pool_config held.
  */
 int
 dmu_objset_find_dp(dsl_pool_t *dp, uint64_t ddobj,
     int func(dsl_pool_t *, dsl_dataset_t *, void *), void *arg, int flags)
 {
 	int error = 0;
 	taskq_t *tq = NULL;
 	int ntasks;
 	dmu_objset_find_ctx_t *dcp;
 	kmutex_t err_lock;
 
 	mutex_init(&err_lock, NULL, MUTEX_DEFAULT, NULL);
 	dcp = kmem_alloc(sizeof (*dcp), KM_SLEEP);
 	dcp->dc_tq = NULL;
 	dcp->dc_dp = dp;
 	dcp->dc_ddobj = ddobj;
 	dcp->dc_ddname = NULL;
 	dcp->dc_func = func;
 	dcp->dc_arg = arg;
 	dcp->dc_flags = flags;
 	dcp->dc_error_lock = &err_lock;
 	dcp->dc_error = &error;
 
 	if ((flags & DS_FIND_SERIALIZE) || dsl_pool_config_held_writer(dp)) {
 		/*
 		 * In case a write lock is held we can't make use of
 		 * parallelism, as down the stack of the worker threads
 		 * the lock is asserted via dsl_pool_config_held.
 		 * In case of a read lock this is solved by getting a read
 		 * lock in each worker thread, which isn't possible in case
 		 * of a writer lock. So we fall back to the synchronous path
 		 * here.
 		 * In the future it might be possible to get some magic into
 		 * dsl_pool_config_held in a way that it returns true for
 		 * the worker threads so that a single lock held from this
 		 * thread suffices. For now, stay single threaded.
 		 */
 		dmu_objset_find_dp_impl(dcp);
 		mutex_destroy(&err_lock);
 
 		return (error);
 	}
 
 	ntasks = dmu_find_threads;
 	if (ntasks == 0)
 		ntasks = vdev_count_leaves(dp->dp_spa) * 4;
 	tq = taskq_create("dmu_objset_find", ntasks, maxclsyspri, ntasks,
 	    INT_MAX, 0);
 	if (tq == NULL) {
 		kmem_free(dcp, sizeof (*dcp));
 		mutex_destroy(&err_lock);
 
 		return (SET_ERROR(ENOMEM));
 	}
 	dcp->dc_tq = tq;
 
 	/* dcp will be freed by task */
 	(void) taskq_dispatch(tq, dmu_objset_find_dp_cb, dcp, TQ_SLEEP);
 
 	/*
 	 * PORTING: this code relies on the property of taskq_wait to wait
 	 * until no more tasks are queued and no more tasks are active. As
 	 * we always queue new tasks from within other tasks, task_wait
 	 * reliably waits for the full recursion to finish, even though we
 	 * enqueue new tasks after taskq_wait has been called.
 	 * On platforms other than illumos, taskq_wait may not have this
 	 * property.
 	 */
 	taskq_wait(tq);
 	taskq_destroy(tq);
 	mutex_destroy(&err_lock);
 
 	return (error);
 }
 
 /*
  * Find all objsets under name, and for each, call 'func(child_name, arg)'.
  * The dp_config_rwlock must not be held when this is called, and it
  * will not be held when the callback is called.
  * Therefore this function should only be used when the pool is not changing
  * (e.g. in syncing context), or the callback can deal with the possible races.
  */
 static int
 dmu_objset_find_impl(spa_t *spa, const char *name,
     int func(const char *, void *), void *arg, int flags)
 {
 	dsl_dir_t *dd;
 	dsl_pool_t *dp = spa_get_dsl(spa);
 	dsl_dataset_t *ds;
 	zap_cursor_t zc;
 	zap_attribute_t *attr;
 	char *child;
 	uint64_t thisobj;
 	int err;
 
 	dsl_pool_config_enter(dp, FTAG);
 
 	err = dsl_dir_hold(dp, name, FTAG, &dd, NULL);
 	if (err != 0) {
 		dsl_pool_config_exit(dp, FTAG);
 		return (err);
 	}
 
 	/* Don't visit hidden ($MOS & $ORIGIN) objsets. */
 	if (dd->dd_myname[0] == '$') {
 		dsl_dir_rele(dd, FTAG);
 		dsl_pool_config_exit(dp, FTAG);
 		return (0);
 	}
 
 	thisobj = dsl_dir_phys(dd)->dd_head_dataset_obj;
 	attr = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
 
 	/*
 	 * Iterate over all children.
 	 */
 	if (flags & DS_FIND_CHILDREN) {
 		for (zap_cursor_init(&zc, dp->dp_meta_objset,
 		    dsl_dir_phys(dd)->dd_child_dir_zapobj);
 		    zap_cursor_retrieve(&zc, attr) == 0;
 		    (void) zap_cursor_advance(&zc)) {
 			ASSERT3U(attr->za_integer_length, ==,
 			    sizeof (uint64_t));
 			ASSERT3U(attr->za_num_integers, ==, 1);
 
 			child = kmem_asprintf("%s/%s", name, attr->za_name);
 			dsl_pool_config_exit(dp, FTAG);
 			err = dmu_objset_find_impl(spa, child,
 			    func, arg, flags);
 			dsl_pool_config_enter(dp, FTAG);
 			kmem_strfree(child);
 			if (err != 0)
 				break;
 		}
 		zap_cursor_fini(&zc);
 
 		if (err != 0) {
 			dsl_dir_rele(dd, FTAG);
 			dsl_pool_config_exit(dp, FTAG);
 			kmem_free(attr, sizeof (zap_attribute_t));
 			return (err);
 		}
 	}
 
 	/*
 	 * Iterate over all snapshots.
 	 */
 	if (flags & DS_FIND_SNAPSHOTS) {
 		err = dsl_dataset_hold_obj(dp, thisobj, FTAG, &ds);
 
 		if (err == 0) {
 			uint64_t snapobj;
 
 			snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
 			dsl_dataset_rele(ds, FTAG);
 
 			for (zap_cursor_init(&zc, dp->dp_meta_objset, snapobj);
 			    zap_cursor_retrieve(&zc, attr) == 0;
 			    (void) zap_cursor_advance(&zc)) {
 				ASSERT3U(attr->za_integer_length, ==,
 				    sizeof (uint64_t));
 				ASSERT3U(attr->za_num_integers, ==, 1);
 
 				child = kmem_asprintf("%s@%s",
 				    name, attr->za_name);
 				dsl_pool_config_exit(dp, FTAG);
 				err = func(child, arg);
 				dsl_pool_config_enter(dp, FTAG);
 				kmem_strfree(child);
 				if (err != 0)
 					break;
 			}
 			zap_cursor_fini(&zc);
 		}
 	}
 
 	dsl_dir_rele(dd, FTAG);
 	kmem_free(attr, sizeof (zap_attribute_t));
 	dsl_pool_config_exit(dp, FTAG);
 
 	if (err != 0)
 		return (err);
 
 	/* Apply to self. */
 	return (func(name, arg));
 }
 
 /*
  * See comment above dmu_objset_find_impl().
  */
 int
 dmu_objset_find(const char *name, int func(const char *, void *), void *arg,
     int flags)
 {
 	spa_t *spa;
 	int error;
 
 	error = spa_open(name, &spa, FTAG);
 	if (error != 0)
 		return (error);
 	error = dmu_objset_find_impl(spa, name, func, arg, flags);
 	spa_close(spa, FTAG);
 	return (error);
 }
 
 boolean_t
 dmu_objset_incompatible_encryption_version(objset_t *os)
 {
 	return (dsl_dir_incompatible_encryption_version(
 	    os->os_dsl_dataset->ds_dir));
 }
 
 void
 dmu_objset_set_user(objset_t *os, void *user_ptr)
 {
 	ASSERT(MUTEX_HELD(&os->os_user_ptr_lock));
 	os->os_user_ptr = user_ptr;
 }
 
 void *
 dmu_objset_get_user(objset_t *os)
 {
 	ASSERT(MUTEX_HELD(&os->os_user_ptr_lock));
 	return (os->os_user_ptr);
 }
 
 /*
  * Determine name of filesystem, given name of snapshot.
  * buf must be at least ZFS_MAX_DATASET_NAME_LEN bytes
  */
 int
 dmu_fsname(const char *snapname, char *buf)
 {
 	char *atp = strchr(snapname, '@');
 	if (atp == NULL)
 		return (SET_ERROR(EINVAL));
 	if (atp - snapname >= ZFS_MAX_DATASET_NAME_LEN)
 		return (SET_ERROR(ENAMETOOLONG));
 	(void) strlcpy(buf, snapname, atp - snapname + 1);
 	return (0);
 }
 
 /*
  * Call when we think we're going to write/free space in open context
  * to track the amount of dirty data in the open txg, which is also the
  * amount of memory that can not be evicted until this txg syncs.
  *
  * Note that there are two conditions where this can be called from
  * syncing context:
  *
  * [1] When we just created the dataset, in which case we go on with
  *     updating any accounting of dirty data as usual.
  * [2] When we are dirtying MOS data, in which case we only update the
  *     pool's accounting of dirty data.
  */
 void
 dmu_objset_willuse_space(objset_t *os, int64_t space, dmu_tx_t *tx)
 {
 	dsl_dataset_t *ds = os->os_dsl_dataset;
 	int64_t aspace = spa_get_worst_case_asize(os->os_spa, space);
 
 	if (ds != NULL) {
 		dsl_dir_willuse_space(ds->ds_dir, aspace, tx);
 	}
 
 	dsl_pool_dirty_space(dmu_tx_pool(tx), space, tx);
 }
 
 #if defined(_KERNEL)
 EXPORT_SYMBOL(dmu_objset_zil);
 EXPORT_SYMBOL(dmu_objset_pool);
 EXPORT_SYMBOL(dmu_objset_ds);
 EXPORT_SYMBOL(dmu_objset_type);
 EXPORT_SYMBOL(dmu_objset_name);
 EXPORT_SYMBOL(dmu_objset_hold);
 EXPORT_SYMBOL(dmu_objset_hold_flags);
 EXPORT_SYMBOL(dmu_objset_own);
 EXPORT_SYMBOL(dmu_objset_rele);
 EXPORT_SYMBOL(dmu_objset_rele_flags);
 EXPORT_SYMBOL(dmu_objset_disown);
 EXPORT_SYMBOL(dmu_objset_from_ds);
 EXPORT_SYMBOL(dmu_objset_create);
 EXPORT_SYMBOL(dmu_objset_clone);
 EXPORT_SYMBOL(dmu_objset_stats);
 EXPORT_SYMBOL(dmu_objset_fast_stat);
 EXPORT_SYMBOL(dmu_objset_spa);
 EXPORT_SYMBOL(dmu_objset_space);
 EXPORT_SYMBOL(dmu_objset_fsid_guid);
 EXPORT_SYMBOL(dmu_objset_find);
 EXPORT_SYMBOL(dmu_objset_byteswap);
 EXPORT_SYMBOL(dmu_objset_evict_dbufs);
 EXPORT_SYMBOL(dmu_objset_snap_cmtime);
 EXPORT_SYMBOL(dmu_objset_dnodesize);
 
 EXPORT_SYMBOL(dmu_objset_sync);
 EXPORT_SYMBOL(dmu_objset_is_dirty);
 EXPORT_SYMBOL(dmu_objset_create_impl_dnstats);
 EXPORT_SYMBOL(dmu_objset_create_impl);
 EXPORT_SYMBOL(dmu_objset_open_impl);
 EXPORT_SYMBOL(dmu_objset_evict);
 EXPORT_SYMBOL(dmu_objset_register_type);
 EXPORT_SYMBOL(dmu_objset_sync_done);
 EXPORT_SYMBOL(dmu_objset_userquota_get_ids);
 EXPORT_SYMBOL(dmu_objset_userused_enabled);
 EXPORT_SYMBOL(dmu_objset_userspace_upgrade);
 EXPORT_SYMBOL(dmu_objset_userspace_present);
 EXPORT_SYMBOL(dmu_objset_userobjused_enabled);
 EXPORT_SYMBOL(dmu_objset_userobjspace_upgradable);
 EXPORT_SYMBOL(dmu_objset_userobjspace_present);
 EXPORT_SYMBOL(dmu_objset_projectquota_enabled);
 EXPORT_SYMBOL(dmu_objset_projectquota_present);
 EXPORT_SYMBOL(dmu_objset_projectquota_upgradable);
 EXPORT_SYMBOL(dmu_objset_id_quota_upgrade);
 #endif
diff --git a/sys/contrib/openzfs/module/zfs/dmu_recv.c b/sys/contrib/openzfs/module/zfs/dmu_recv.c
index 883ee192f07f..41b279d48464 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_recv.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_recv.c
@@ -1,3806 +1,3820 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright 2014 HybridCluster. All rights reserved.
  * Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
  * Copyright (c) 2019, Klara Inc.
  * Copyright (c) 2019, Allan Jude
  * Copyright (c) 2019 Datto Inc.
  * Copyright (c) 2022 Axcient.
+ * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
  */
 
 #include <sys/arc.h>
 #include <sys/spa_impl.h>
 #include <sys/dmu.h>
 #include <sys/dmu_impl.h>
 #include <sys/dmu_send.h>
 #include <sys/dmu_recv.h>
 #include <sys/dmu_tx.h>
 #include <sys/dbuf.h>
 #include <sys/dnode.h>
 #include <sys/zfs_context.h>
 #include <sys/dmu_objset.h>
 #include <sys/dmu_traverse.h>
 #include <sys/dsl_dataset.h>
 #include <sys/dsl_dir.h>
 #include <sys/dsl_prop.h>
 #include <sys/dsl_pool.h>
 #include <sys/dsl_synctask.h>
 #include <sys/zfs_ioctl.h>
 #include <sys/zap.h>
 #include <sys/zvol.h>
 #include <sys/zio_checksum.h>
 #include <sys/zfs_znode.h>
 #include <zfs_fletcher.h>
 #include <sys/avl.h>
 #include <sys/ddt.h>
 #include <sys/zfs_onexit.h>
 #include <sys/dsl_destroy.h>
 #include <sys/blkptr.h>
 #include <sys/dsl_bookmark.h>
 #include <sys/zfeature.h>
 #include <sys/bqueue.h>
 #include <sys/objlist.h>
 #ifdef _KERNEL
 #include <sys/zfs_vfsops.h>
 #endif
 #include <sys/zfs_file.h>
+#include <sys/cred.h>
 
 static uint_t zfs_recv_queue_length = SPA_MAXBLOCKSIZE;
 static uint_t zfs_recv_queue_ff = 20;
 static uint_t zfs_recv_write_batch_size = 1024 * 1024;
 static int zfs_recv_best_effort_corrective = 0;
 
 static const void *const dmu_recv_tag = "dmu_recv_tag";
 const char *const recv_clone_name = "%recv";
 
 typedef enum {
 	ORNS_NO,
 	ORNS_YES,
 	ORNS_MAYBE
 } or_need_sync_t;
 
 static int receive_read_payload_and_next_header(dmu_recv_cookie_t *ra, int len,
     void *buf);
 
 struct receive_record_arg {
 	dmu_replay_record_t header;
 	void *payload; /* Pointer to a buffer containing the payload */
 	/*
 	 * If the record is a WRITE or SPILL, pointer to the abd containing the
 	 * payload.
 	 */
 	abd_t *abd;
 	int payload_size;
 	uint64_t bytes_read; /* bytes read from stream when record created */
 	boolean_t eos_marker; /* Marks the end of the stream */
 	bqueue_node_t node;
 };
 
 struct receive_writer_arg {
 	objset_t *os;
 	boolean_t byteswap;
 	bqueue_t q;
 
 	/*
 	 * These three members are used to signal to the main thread when
 	 * we're done.
 	 */
 	kmutex_t mutex;
 	kcondvar_t cv;
 	boolean_t done;
 
 	int err;
 	const char *tofs;
 	boolean_t heal;
 	boolean_t resumable;
 	boolean_t raw;   /* DMU_BACKUP_FEATURE_RAW set */
 	boolean_t spill; /* DRR_FLAG_SPILL_BLOCK set */
 	boolean_t full;  /* this is a full send stream */
 	uint64_t last_object;
 	uint64_t last_offset;
 	uint64_t max_object; /* highest object ID referenced in stream */
 	uint64_t bytes_read; /* bytes read when current record created */
 
 	list_t write_batch;
 
 	/* Encryption parameters for the last received DRR_OBJECT_RANGE */
 	boolean_t or_crypt_params_present;
 	uint64_t or_firstobj;
 	uint64_t or_numslots;
 	uint8_t or_salt[ZIO_DATA_SALT_LEN];
 	uint8_t or_iv[ZIO_DATA_IV_LEN];
 	uint8_t or_mac[ZIO_DATA_MAC_LEN];
 	boolean_t or_byteorder;
 	zio_t *heal_pio;
 
 	/* Keep track of DRR_FREEOBJECTS right after DRR_OBJECT_RANGE */
 	or_need_sync_t or_need_sync;
 };
 
 typedef struct dmu_recv_begin_arg {
 	const char *drba_origin;
 	dmu_recv_cookie_t *drba_cookie;
 	cred_t *drba_cred;
-	proc_t *drba_proc;
 	dsl_crypto_params_t *drba_dcp;
 } dmu_recv_begin_arg_t;
 
 static void
 byteswap_record(dmu_replay_record_t *drr)
 {
 #define	DO64(X) (drr->drr_u.X = BSWAP_64(drr->drr_u.X))
 #define	DO32(X) (drr->drr_u.X = BSWAP_32(drr->drr_u.X))
 	drr->drr_type = BSWAP_32(drr->drr_type);
 	drr->drr_payloadlen = BSWAP_32(drr->drr_payloadlen);
 
 	switch (drr->drr_type) {
 	case DRR_BEGIN:
 		DO64(drr_begin.drr_magic);
 		DO64(drr_begin.drr_versioninfo);
 		DO64(drr_begin.drr_creation_time);
 		DO32(drr_begin.drr_type);
 		DO32(drr_begin.drr_flags);
 		DO64(drr_begin.drr_toguid);
 		DO64(drr_begin.drr_fromguid);
 		break;
 	case DRR_OBJECT:
 		DO64(drr_object.drr_object);
 		DO32(drr_object.drr_type);
 		DO32(drr_object.drr_bonustype);
 		DO32(drr_object.drr_blksz);
 		DO32(drr_object.drr_bonuslen);
 		DO32(drr_object.drr_raw_bonuslen);
 		DO64(drr_object.drr_toguid);
 		DO64(drr_object.drr_maxblkid);
 		break;
 	case DRR_FREEOBJECTS:
 		DO64(drr_freeobjects.drr_firstobj);
 		DO64(drr_freeobjects.drr_numobjs);
 		DO64(drr_freeobjects.drr_toguid);
 		break;
 	case DRR_WRITE:
 		DO64(drr_write.drr_object);
 		DO32(drr_write.drr_type);
 		DO64(drr_write.drr_offset);
 		DO64(drr_write.drr_logical_size);
 		DO64(drr_write.drr_toguid);
 		ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_write.drr_key.ddk_cksum);
 		DO64(drr_write.drr_key.ddk_prop);
 		DO64(drr_write.drr_compressed_size);
 		break;
 	case DRR_WRITE_EMBEDDED:
 		DO64(drr_write_embedded.drr_object);
 		DO64(drr_write_embedded.drr_offset);
 		DO64(drr_write_embedded.drr_length);
 		DO64(drr_write_embedded.drr_toguid);
 		DO32(drr_write_embedded.drr_lsize);
 		DO32(drr_write_embedded.drr_psize);
 		break;
 	case DRR_FREE:
 		DO64(drr_free.drr_object);
 		DO64(drr_free.drr_offset);
 		DO64(drr_free.drr_length);
 		DO64(drr_free.drr_toguid);
 		break;
 	case DRR_SPILL:
 		DO64(drr_spill.drr_object);
 		DO64(drr_spill.drr_length);
 		DO64(drr_spill.drr_toguid);
 		DO64(drr_spill.drr_compressed_size);
 		DO32(drr_spill.drr_type);
 		break;
 	case DRR_OBJECT_RANGE:
 		DO64(drr_object_range.drr_firstobj);
 		DO64(drr_object_range.drr_numslots);
 		DO64(drr_object_range.drr_toguid);
 		break;
 	case DRR_REDACT:
 		DO64(drr_redact.drr_object);
 		DO64(drr_redact.drr_offset);
 		DO64(drr_redact.drr_length);
 		DO64(drr_redact.drr_toguid);
 		break;
 	case DRR_END:
 		DO64(drr_end.drr_toguid);
 		ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_end.drr_checksum);
 		break;
 	default:
 		break;
 	}
 
 	if (drr->drr_type != DRR_BEGIN) {
 		ZIO_CHECKSUM_BSWAP(&drr->drr_u.drr_checksum.drr_checksum);
 	}
 
 #undef DO64
 #undef DO32
 }
 
 static boolean_t
 redact_snaps_contains(uint64_t *snaps, uint64_t num_snaps, uint64_t guid)
 {
 	for (int i = 0; i < num_snaps; i++) {
 		if (snaps[i] == guid)
 			return (B_TRUE);
 	}
 	return (B_FALSE);
 }
 
 /*
  * Check that the new stream we're trying to receive is redacted with respect to
  * a subset of the snapshots that the origin was redacted with respect to.  For
  * the reasons behind this, see the man page on redacted zfs sends and receives.
  */
 static boolean_t
 compatible_redact_snaps(uint64_t *origin_snaps, uint64_t origin_num_snaps,
     uint64_t *redact_snaps, uint64_t num_redact_snaps)
 {
 	/*
 	 * Short circuit the comparison; if we are redacted with respect to
 	 * more snapshots than the origin, we can't be redacted with respect
 	 * to a subset.
 	 */
 	if (num_redact_snaps > origin_num_snaps) {
 		return (B_FALSE);
 	}
 
 	for (int i = 0; i < num_redact_snaps; i++) {
 		if (!redact_snaps_contains(origin_snaps, origin_num_snaps,
 		    redact_snaps[i])) {
 			return (B_FALSE);
 		}
 	}
 	return (B_TRUE);
 }
 
 static boolean_t
 redact_check(dmu_recv_begin_arg_t *drba, dsl_dataset_t *origin)
 {
 	uint64_t *origin_snaps;
 	uint64_t origin_num_snaps;
 	dmu_recv_cookie_t *drc = drba->drba_cookie;
 	struct drr_begin *drrb = drc->drc_drrb;
 	int featureflags = DMU_GET_FEATUREFLAGS(drrb->drr_versioninfo);
 	int err = 0;
 	boolean_t ret = B_TRUE;
 	uint64_t *redact_snaps;
 	uint_t numredactsnaps;
 
 	/*
 	 * If this is a full send stream, we're safe no matter what.
 	 */
 	if (drrb->drr_fromguid == 0)
 		return (ret);
 
 	VERIFY(dsl_dataset_get_uint64_array_feature(origin,
 	    SPA_FEATURE_REDACTED_DATASETS, &origin_num_snaps, &origin_snaps));
 
 	if (nvlist_lookup_uint64_array(drc->drc_begin_nvl,
 	    BEGINNV_REDACT_FROM_SNAPS, &redact_snaps, &numredactsnaps) ==
 	    0) {
 		/*
 		 * If the send stream was sent from the redaction bookmark or
 		 * the redacted version of the dataset, then we're safe.  Verify
 		 * that this is from the a compatible redaction bookmark or
 		 * redacted dataset.
 		 */
 		if (!compatible_redact_snaps(origin_snaps, origin_num_snaps,
 		    redact_snaps, numredactsnaps)) {
 			err = EINVAL;
 		}
 	} else if (featureflags & DMU_BACKUP_FEATURE_REDACTED) {
 		/*
 		 * If the stream is redacted, it must be redacted with respect
 		 * to a subset of what the origin is redacted with respect to.
 		 * See case number 2 in the zfs man page section on redacted zfs
 		 * send.
 		 */
 		err = nvlist_lookup_uint64_array(drc->drc_begin_nvl,
 		    BEGINNV_REDACT_SNAPS, &redact_snaps, &numredactsnaps);
 
 		if (err != 0 || !compatible_redact_snaps(origin_snaps,
 		    origin_num_snaps, redact_snaps, numredactsnaps)) {
 			err = EINVAL;
 		}
 	} else if (!redact_snaps_contains(origin_snaps, origin_num_snaps,
 	    drrb->drr_toguid)) {
 		/*
 		 * If the stream isn't redacted but the origin is, this must be
 		 * one of the snapshots the origin is redacted with respect to.
 		 * See case number 1 in the zfs man page section on redacted zfs
 		 * send.
 		 */
 		err = EINVAL;
 	}
 
 	if (err != 0)
 		ret = B_FALSE;
 	return (ret);
 }
 
 /*
  * If we previously received a stream with --large-block, we don't support
  * receiving an incremental on top of it without --large-block.  This avoids
  * forcing a read-modify-write or trying to re-aggregate a string of WRITE
  * records.
  */
 static int
 recv_check_large_blocks(dsl_dataset_t *ds, uint64_t featureflags)
 {
 	if (dsl_dataset_feature_is_active(ds, SPA_FEATURE_LARGE_BLOCKS) &&
 	    !(featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS))
 		return (SET_ERROR(ZFS_ERR_STREAM_LARGE_BLOCK_MISMATCH));
 	return (0);
 }
 
 static int
 recv_begin_check_existing_impl(dmu_recv_begin_arg_t *drba, dsl_dataset_t *ds,
     uint64_t fromguid, uint64_t featureflags)
 {
 	uint64_t obj;
 	uint64_t children;
 	int error;
 	dsl_dataset_t *snap;
 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
 	boolean_t encrypted = ds->ds_dir->dd_crypto_obj != 0;
 	boolean_t raw = (featureflags & DMU_BACKUP_FEATURE_RAW) != 0;
 	boolean_t embed = (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) != 0;
 
 	/* Temporary clone name must not exist. */
 	error = zap_lookup(dp->dp_meta_objset,
 	    dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, recv_clone_name,
 	    8, 1, &obj);
 	if (error != ENOENT)
 		return (error == 0 ? SET_ERROR(EBUSY) : error);
 
 	/* Resume state must not be set. */
 	if (dsl_dataset_has_resume_receive_state(ds))
 		return (SET_ERROR(EBUSY));
 
 	/* New snapshot name must not exist if we're not healing it. */
 	error = zap_lookup(dp->dp_meta_objset,
 	    dsl_dataset_phys(ds)->ds_snapnames_zapobj,
 	    drba->drba_cookie->drc_tosnap, 8, 1, &obj);
 	if (drba->drba_cookie->drc_heal) {
 		if (error != 0)
 			return (error);
 	} else if (error != ENOENT) {
 		return (error == 0 ? SET_ERROR(EEXIST) : error);
 	}
 
 	/* Must not have children if receiving a ZVOL. */
 	error = zap_count(dp->dp_meta_objset,
 	    dsl_dir_phys(ds->ds_dir)->dd_child_dir_zapobj, &children);
 	if (error != 0)
 		return (error);
 	if (drba->drba_cookie->drc_drrb->drr_type != DMU_OST_ZFS &&
 	    children > 0)
 		return (SET_ERROR(ZFS_ERR_WRONG_PARENT));
 
 	/*
 	 * Check snapshot limit before receiving. We'll recheck again at the
 	 * end, but might as well abort before receiving if we're already over
 	 * the limit.
 	 *
 	 * Note that we do not check the file system limit with
 	 * dsl_dir_fscount_check because the temporary %clones don't count
 	 * against that limit.
 	 */
 	error = dsl_fs_ss_limit_check(ds->ds_dir, 1, ZFS_PROP_SNAPSHOT_LIMIT,
-	    NULL, drba->drba_cred, drba->drba_proc);
+	    NULL, drba->drba_cred);
 	if (error != 0)
 		return (error);
 
 	if (drba->drba_cookie->drc_heal) {
 		/* Encryption is incompatible with embedded data. */
 		if (encrypted && embed)
 			return (SET_ERROR(EINVAL));
 
 		/* Healing is not supported when in 'force' mode. */
 		if (drba->drba_cookie->drc_force)
 			return (SET_ERROR(EINVAL));
 
 		/* Must have keys loaded if doing encrypted non-raw recv. */
 		if (encrypted && !raw) {
 			if (spa_keystore_lookup_key(dp->dp_spa, ds->ds_object,
 			    NULL, NULL) != 0)
 				return (SET_ERROR(EACCES));
 		}
 
 		error = dsl_dataset_hold_obj(dp, obj, FTAG, &snap);
 		if (error != 0)
 			return (error);
 
 		/*
 		 * When not doing best effort corrective recv healing can only
 		 * be done if the send stream is for the same snapshot as the
 		 * one we are trying to heal.
 		 */
 		if (zfs_recv_best_effort_corrective == 0 &&
 		    drba->drba_cookie->drc_drrb->drr_toguid !=
 		    dsl_dataset_phys(snap)->ds_guid) {
 			dsl_dataset_rele(snap, FTAG);
 			return (SET_ERROR(ENOTSUP));
 		}
 		dsl_dataset_rele(snap, FTAG);
 	} else if (fromguid != 0) {
 		/* Sanity check the incremental recv */
 		uint64_t obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 
 		/* Can't perform a raw receive on top of a non-raw receive */
 		if (!encrypted && raw)
 			return (SET_ERROR(EINVAL));
 
 		/* Encryption is incompatible with embedded data */
 		if (encrypted && embed)
 			return (SET_ERROR(EINVAL));
 
 		/* Find snapshot in this dir that matches fromguid. */
 		while (obj != 0) {
 			error = dsl_dataset_hold_obj(dp, obj, FTAG,
 			    &snap);
 			if (error != 0)
 				return (SET_ERROR(ENODEV));
 			if (snap->ds_dir != ds->ds_dir) {
 				dsl_dataset_rele(snap, FTAG);
 				return (SET_ERROR(ENODEV));
 			}
 			if (dsl_dataset_phys(snap)->ds_guid == fromguid)
 				break;
 			obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
 			dsl_dataset_rele(snap, FTAG);
 		}
 		if (obj == 0)
 			return (SET_ERROR(ENODEV));
 
 		if (drba->drba_cookie->drc_force) {
 			drba->drba_cookie->drc_fromsnapobj = obj;
 		} else {
 			/*
 			 * If we are not forcing, there must be no
 			 * changes since fromsnap. Raw sends have an
 			 * additional constraint that requires that
 			 * no "noop" snapshots exist between fromsnap
 			 * and tosnap for the IVset checking code to
 			 * work properly.
 			 */
 			if (dsl_dataset_modified_since_snap(ds, snap) ||
 			    (raw &&
 			    dsl_dataset_phys(ds)->ds_prev_snap_obj !=
 			    snap->ds_object)) {
 				dsl_dataset_rele(snap, FTAG);
 				return (SET_ERROR(ETXTBSY));
 			}
 			drba->drba_cookie->drc_fromsnapobj =
 			    ds->ds_prev->ds_object;
 		}
 
 		if (dsl_dataset_feature_is_active(snap,
 		    SPA_FEATURE_REDACTED_DATASETS) && !redact_check(drba,
 		    snap)) {
 			dsl_dataset_rele(snap, FTAG);
 			return (SET_ERROR(EINVAL));
 		}
 
 		error = recv_check_large_blocks(snap, featureflags);
 		if (error != 0) {
 			dsl_dataset_rele(snap, FTAG);
 			return (error);
 		}
 
 		dsl_dataset_rele(snap, FTAG);
 	} else {
 		/* If full and not healing then must be forced. */
 		if (!drba->drba_cookie->drc_force)
 			return (SET_ERROR(EEXIST));
 
 		/*
 		 * We don't support using zfs recv -F to blow away
 		 * encrypted filesystems. This would require the
 		 * dsl dir to point to the old encryption key and
 		 * the new one at the same time during the receive.
 		 */
 		if ((!encrypted && raw) || encrypted)
 			return (SET_ERROR(EINVAL));
 
 		/*
 		 * Perform the same encryption checks we would if
 		 * we were creating a new dataset from scratch.
 		 */
 		if (!raw) {
 			boolean_t will_encrypt;
 
 			error = dmu_objset_create_crypt_check(
 			    ds->ds_dir->dd_parent, drba->drba_dcp,
 			    &will_encrypt);
 			if (error != 0)
 				return (error);
 
 			if (will_encrypt && embed)
 				return (SET_ERROR(EINVAL));
 		}
 	}
 
 	return (0);
 }
 
 /*
  * Check that any feature flags used in the data stream we're receiving are
  * supported by the pool we are receiving into.
  *
  * Note that some of the features we explicitly check here have additional
  * (implicit) features they depend on, but those dependencies are enforced
  * through the zfeature_register() calls declaring the features that we
  * explicitly check.
  */
 static int
 recv_begin_check_feature_flags_impl(uint64_t featureflags, spa_t *spa)
 {
 	/*
 	 * Check if there are any unsupported feature flags.
 	 */
 	if (!DMU_STREAM_SUPPORTED(featureflags)) {
 		return (SET_ERROR(ZFS_ERR_UNKNOWN_SEND_STREAM_FEATURE));
 	}
 
 	/* Verify pool version supports SA if SA_SPILL feature set */
 	if ((featureflags & DMU_BACKUP_FEATURE_SA_SPILL) &&
 	    spa_version(spa) < SPA_VERSION_SA)
 		return (SET_ERROR(ENOTSUP));
 
 	/*
 	 * LZ4 compressed, ZSTD compressed, embedded, mooched, large blocks,
 	 * and large_dnodes in the stream can only be used if those pool
 	 * features are enabled because we don't attempt to decompress /
 	 * un-embed / un-mooch / split up the blocks / dnodes during the
 	 * receive process.
 	 */
 	if ((featureflags & DMU_BACKUP_FEATURE_LZ4) &&
 	    !spa_feature_is_enabled(spa, SPA_FEATURE_LZ4_COMPRESS))
 		return (SET_ERROR(ENOTSUP));
 	if ((featureflags & DMU_BACKUP_FEATURE_ZSTD) &&
 	    !spa_feature_is_enabled(spa, SPA_FEATURE_ZSTD_COMPRESS))
 		return (SET_ERROR(ENOTSUP));
 	if ((featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) &&
 	    !spa_feature_is_enabled(spa, SPA_FEATURE_EMBEDDED_DATA))
 		return (SET_ERROR(ENOTSUP));
 	if ((featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
 	    !spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS))
 		return (SET_ERROR(ENOTSUP));
 	if ((featureflags & DMU_BACKUP_FEATURE_LARGE_DNODE) &&
 	    !spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_DNODE))
 		return (SET_ERROR(ENOTSUP));
 
 	/*
 	 * Receiving redacted streams requires that redacted datasets are
 	 * enabled.
 	 */
 	if ((featureflags & DMU_BACKUP_FEATURE_REDACTED) &&
 	    !spa_feature_is_enabled(spa, SPA_FEATURE_REDACTED_DATASETS))
 		return (SET_ERROR(ENOTSUP));
 
 	return (0);
 }
 
 static int
 dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
 {
 	dmu_recv_begin_arg_t *drba = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	struct drr_begin *drrb = drba->drba_cookie->drc_drrb;
 	uint64_t fromguid = drrb->drr_fromguid;
 	int flags = drrb->drr_flags;
 	ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE;
 	int error;
 	uint64_t featureflags = drba->drba_cookie->drc_featureflags;
 	dsl_dataset_t *ds;
 	const char *tofs = drba->drba_cookie->drc_tofs;
 
 	/* already checked */
 	ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
 	ASSERT(!(featureflags & DMU_BACKUP_FEATURE_RESUMING));
 
 	if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
 	    DMU_COMPOUNDSTREAM ||
 	    drrb->drr_type >= DMU_OST_NUMTYPES ||
 	    ((flags & DRR_FLAG_CLONE) && drba->drba_origin == NULL))
 		return (SET_ERROR(EINVAL));
 
 	error = recv_begin_check_feature_flags_impl(featureflags, dp->dp_spa);
 	if (error != 0)
 		return (error);
 
 	/* Resumable receives require extensible datasets */
 	if (drba->drba_cookie->drc_resumable &&
 	    !spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_EXTENSIBLE_DATASET))
 		return (SET_ERROR(ENOTSUP));
 
 	if (featureflags & DMU_BACKUP_FEATURE_RAW) {
 		/* raw receives require the encryption feature */
 		if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_ENCRYPTION))
 			return (SET_ERROR(ENOTSUP));
 
 		/* embedded data is incompatible with encryption and raw recv */
 		if (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)
 			return (SET_ERROR(EINVAL));
 
 		/* raw receives require spill block allocation flag */
 		if (!(flags & DRR_FLAG_SPILL_BLOCK))
 			return (SET_ERROR(ZFS_ERR_SPILL_BLOCK_FLAG_MISSING));
 	} else {
 		/*
 		 * We support unencrypted datasets below encrypted ones now,
 		 * so add the DS_HOLD_FLAG_DECRYPT flag only if we are dealing
 		 * with a dataset we may encrypt.
 		 */
 		if (drba->drba_dcp == NULL ||
 		    drba->drba_dcp->cp_crypt != ZIO_CRYPT_OFF) {
 			dsflags |= DS_HOLD_FLAG_DECRYPT;
 		}
 	}
 
 	error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds);
 	if (error == 0) {
 		/* target fs already exists; recv into temp clone */
 
 		/* Can't recv a clone into an existing fs */
 		if (flags & DRR_FLAG_CLONE || drba->drba_origin) {
 			dsl_dataset_rele_flags(ds, dsflags, FTAG);
 			return (SET_ERROR(EINVAL));
 		}
 
 		error = recv_begin_check_existing_impl(drba, ds, fromguid,
 		    featureflags);
 		dsl_dataset_rele_flags(ds, dsflags, FTAG);
 	} else if (error == ENOENT) {
 		/* target fs does not exist; must be a full backup or clone */
 		char buf[ZFS_MAX_DATASET_NAME_LEN];
 		objset_t *os;
 
 		/* healing recv must be done "into" an existing snapshot */
 		if (drba->drba_cookie->drc_heal == B_TRUE)
 			return (SET_ERROR(ENOTSUP));
 
 		/*
 		 * If it's a non-clone incremental, we are missing the
 		 * target fs, so fail the recv.
 		 */
 		if (fromguid != 0 && !((flags & DRR_FLAG_CLONE) ||
 		    drba->drba_origin))
 			return (SET_ERROR(ENOENT));
 
 		/*
 		 * If we're receiving a full send as a clone, and it doesn't
 		 * contain all the necessary free records and freeobject
 		 * records, reject it.
 		 */
 		if (fromguid == 0 && drba->drba_origin != NULL &&
 		    !(flags & DRR_FLAG_FREERECORDS))
 			return (SET_ERROR(EINVAL));
 
 		/* Open the parent of tofs */
 		ASSERT3U(strlen(tofs), <, sizeof (buf));
 		(void) strlcpy(buf, tofs, strrchr(tofs, '/') - tofs + 1);
 		error = dsl_dataset_hold(dp, buf, FTAG, &ds);
 		if (error != 0)
 			return (error);
 
 		if ((featureflags & DMU_BACKUP_FEATURE_RAW) == 0 &&
 		    drba->drba_origin == NULL) {
 			boolean_t will_encrypt;
 
 			/*
 			 * Check that we aren't breaking any encryption rules
 			 * and that we have all the parameters we need to
 			 * create an encrypted dataset if necessary. If we are
 			 * making an encrypted dataset the stream can't have
 			 * embedded data.
 			 */
 			error = dmu_objset_create_crypt_check(ds->ds_dir,
 			    drba->drba_dcp, &will_encrypt);
 			if (error != 0) {
 				dsl_dataset_rele(ds, FTAG);
 				return (error);
 			}
 
 			if (will_encrypt &&
 			    (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)) {
 				dsl_dataset_rele(ds, FTAG);
 				return (SET_ERROR(EINVAL));
 			}
 		}
 
 		/*
 		 * Check filesystem and snapshot limits before receiving. We'll
 		 * recheck snapshot limits again at the end (we create the
 		 * filesystems and increment those counts during begin_sync).
 		 */
 		error = dsl_fs_ss_limit_check(ds->ds_dir, 1,
-		    ZFS_PROP_FILESYSTEM_LIMIT, NULL,
-		    drba->drba_cred, drba->drba_proc);
+		    ZFS_PROP_FILESYSTEM_LIMIT, NULL, drba->drba_cred);
 		if (error != 0) {
 			dsl_dataset_rele(ds, FTAG);
 			return (error);
 		}
 
 		error = dsl_fs_ss_limit_check(ds->ds_dir, 1,
-		    ZFS_PROP_SNAPSHOT_LIMIT, NULL,
-		    drba->drba_cred, drba->drba_proc);
+		    ZFS_PROP_SNAPSHOT_LIMIT, NULL, drba->drba_cred);
 		if (error != 0) {
 			dsl_dataset_rele(ds, FTAG);
 			return (error);
 		}
 
 		/* can't recv below anything but filesystems (eg. no ZVOLs) */
 		error = dmu_objset_from_ds(ds, &os);
 		if (error != 0) {
 			dsl_dataset_rele(ds, FTAG);
 			return (error);
 		}
 		if (dmu_objset_type(os) != DMU_OST_ZFS) {
 			dsl_dataset_rele(ds, FTAG);
 			return (SET_ERROR(ZFS_ERR_WRONG_PARENT));
 		}
 
 		if (drba->drba_origin != NULL) {
 			dsl_dataset_t *origin;
 			error = dsl_dataset_hold_flags(dp, drba->drba_origin,
 			    dsflags, FTAG, &origin);
 			if (error != 0) {
 				dsl_dataset_rele(ds, FTAG);
 				return (error);
 			}
 			if (!origin->ds_is_snapshot) {
 				dsl_dataset_rele_flags(origin, dsflags, FTAG);
 				dsl_dataset_rele(ds, FTAG);
 				return (SET_ERROR(EINVAL));
 			}
 			if (dsl_dataset_phys(origin)->ds_guid != fromguid &&
 			    fromguid != 0) {
 				dsl_dataset_rele_flags(origin, dsflags, FTAG);
 				dsl_dataset_rele(ds, FTAG);
 				return (SET_ERROR(ENODEV));
 			}
 
 			if (origin->ds_dir->dd_crypto_obj != 0 &&
 			    (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)) {
 				dsl_dataset_rele_flags(origin, dsflags, FTAG);
 				dsl_dataset_rele(ds, FTAG);
 				return (SET_ERROR(EINVAL));
 			}
 
 			/*
 			 * If the origin is redacted we need to verify that this
 			 * send stream can safely be received on top of the
 			 * origin.
 			 */
 			if (dsl_dataset_feature_is_active(origin,
 			    SPA_FEATURE_REDACTED_DATASETS)) {
 				if (!redact_check(drba, origin)) {
 					dsl_dataset_rele_flags(origin, dsflags,
 					    FTAG);
 					dsl_dataset_rele_flags(ds, dsflags,
 					    FTAG);
 					return (SET_ERROR(EINVAL));
 				}
 			}
 
 			error = recv_check_large_blocks(ds, featureflags);
 			if (error != 0) {
 				dsl_dataset_rele_flags(origin, dsflags, FTAG);
 				dsl_dataset_rele_flags(ds, dsflags, FTAG);
 				return (error);
 			}
 
 			dsl_dataset_rele_flags(origin, dsflags, FTAG);
 		}
 
 		dsl_dataset_rele(ds, FTAG);
 		error = 0;
 	}
 	return (error);
 }
 
 static void
 dmu_recv_begin_sync(void *arg, dmu_tx_t *tx)
 {
 	dmu_recv_begin_arg_t *drba = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	objset_t *mos = dp->dp_meta_objset;
 	dmu_recv_cookie_t *drc = drba->drba_cookie;
 	struct drr_begin *drrb = drc->drc_drrb;
 	const char *tofs = drc->drc_tofs;
 	uint64_t featureflags = drc->drc_featureflags;
 	dsl_dataset_t *ds, *newds;
 	objset_t *os;
 	uint64_t dsobj;
 	ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE;
 	int error;
 	uint64_t crflags = 0;
 	dsl_crypto_params_t dummy_dcp = { 0 };
 	dsl_crypto_params_t *dcp = drba->drba_dcp;
 
 	if (drrb->drr_flags & DRR_FLAG_CI_DATA)
 		crflags |= DS_FLAG_CI_DATASET;
 
 	if ((featureflags & DMU_BACKUP_FEATURE_RAW) == 0)
 		dsflags |= DS_HOLD_FLAG_DECRYPT;
 
 	/*
 	 * Raw, non-incremental recvs always use a dummy dcp with
 	 * the raw cmd set. Raw incremental recvs do not use a dcp
 	 * since the encryption parameters are already set in stone.
 	 */
 	if (dcp == NULL && drrb->drr_fromguid == 0 &&
 	    drba->drba_origin == NULL) {
 		ASSERT3P(dcp, ==, NULL);
 		dcp = &dummy_dcp;
 
 		if (featureflags & DMU_BACKUP_FEATURE_RAW)
 			dcp->cp_cmd = DCP_CMD_RAW_RECV;
 	}
 
 	error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds);
 	if (error == 0) {
 		/* Create temporary clone unless we're doing corrective recv */
 		dsl_dataset_t *snap = NULL;
 
 		if (drba->drba_cookie->drc_fromsnapobj != 0) {
 			VERIFY0(dsl_dataset_hold_obj(dp,
 			    drba->drba_cookie->drc_fromsnapobj, FTAG, &snap));
 			ASSERT3P(dcp, ==, NULL);
 		}
 		if (drc->drc_heal) {
 			/* When healing we want to use the provided snapshot */
 			VERIFY0(dsl_dataset_snap_lookup(ds, drc->drc_tosnap,
 			    &dsobj));
 		} else {
 			dsobj = dsl_dataset_create_sync(ds->ds_dir,
 			    recv_clone_name, snap, crflags, drba->drba_cred,
 			    dcp, tx);
 		}
 		if (drba->drba_cookie->drc_fromsnapobj != 0)
 			dsl_dataset_rele(snap, FTAG);
 		dsl_dataset_rele_flags(ds, dsflags, FTAG);
 	} else {
 		dsl_dir_t *dd;
 		const char *tail;
 		dsl_dataset_t *origin = NULL;
 
 		VERIFY0(dsl_dir_hold(dp, tofs, FTAG, &dd, &tail));
 
 		if (drba->drba_origin != NULL) {
 			VERIFY0(dsl_dataset_hold(dp, drba->drba_origin,
 			    FTAG, &origin));
 			ASSERT3P(dcp, ==, NULL);
 		}
 
 		/* Create new dataset. */
 		dsobj = dsl_dataset_create_sync(dd, strrchr(tofs, '/') + 1,
 		    origin, crflags, drba->drba_cred, dcp, tx);
 		if (origin != NULL)
 			dsl_dataset_rele(origin, FTAG);
 		dsl_dir_rele(dd, FTAG);
 		drc->drc_newfs = B_TRUE;
 	}
 	VERIFY0(dsl_dataset_own_obj_force(dp, dsobj, dsflags, dmu_recv_tag,
 	    &newds));
 	if (dsl_dataset_feature_is_active(newds,
 	    SPA_FEATURE_REDACTED_DATASETS)) {
 		/*
 		 * If the origin dataset is redacted, the child will be redacted
 		 * when we create it.  We clear the new dataset's
 		 * redaction info; if it should be redacted, we'll fill
 		 * in its information later.
 		 */
 		dsl_dataset_deactivate_feature(newds,
 		    SPA_FEATURE_REDACTED_DATASETS, tx);
 	}
 	VERIFY0(dmu_objset_from_ds(newds, &os));
 
 	if (drc->drc_resumable) {
 		dsl_dataset_zapify(newds, tx);
 		if (drrb->drr_fromguid != 0) {
 			VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_FROMGUID,
 			    8, 1, &drrb->drr_fromguid, tx));
 		}
 		VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TOGUID,
 		    8, 1, &drrb->drr_toguid, tx));
 		VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_TONAME,
 		    1, strlen(drrb->drr_toname) + 1, drrb->drr_toname, tx));
 		uint64_t one = 1;
 		uint64_t zero = 0;
 		VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OBJECT,
 		    8, 1, &one, tx));
 		VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_OFFSET,
 		    8, 1, &zero, tx));
 		VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_BYTES,
 		    8, 1, &zero, tx));
 		if (featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) {
 			VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_LARGEBLOCK,
 			    8, 1, &one, tx));
 		}
 		if (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA) {
 			VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_EMBEDOK,
 			    8, 1, &one, tx));
 		}
 		if (featureflags & DMU_BACKUP_FEATURE_COMPRESSED) {
 			VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_COMPRESSOK,
 			    8, 1, &one, tx));
 		}
 		if (featureflags & DMU_BACKUP_FEATURE_RAW) {
 			VERIFY0(zap_add(mos, dsobj, DS_FIELD_RESUME_RAWOK,
 			    8, 1, &one, tx));
 		}
 
 		uint64_t *redact_snaps;
 		uint_t numredactsnaps;
 		if (nvlist_lookup_uint64_array(drc->drc_begin_nvl,
 		    BEGINNV_REDACT_FROM_SNAPS, &redact_snaps,
 		    &numredactsnaps) == 0) {
 			VERIFY0(zap_add(mos, dsobj,
 			    DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS,
 			    sizeof (*redact_snaps), numredactsnaps,
 			    redact_snaps, tx));
 		}
 	}
 
 	/*
 	 * Usually the os->os_encrypted value is tied to the presence of a
 	 * DSL Crypto Key object in the dd. However, that will not be received
 	 * until dmu_recv_stream(), so we set the value manually for now.
 	 */
 	if (featureflags & DMU_BACKUP_FEATURE_RAW) {
 		os->os_encrypted = B_TRUE;
 		drba->drba_cookie->drc_raw = B_TRUE;
 	}
 
 	if (featureflags & DMU_BACKUP_FEATURE_REDACTED) {
 		uint64_t *redact_snaps;
 		uint_t numredactsnaps;
 		VERIFY0(nvlist_lookup_uint64_array(drc->drc_begin_nvl,
 		    BEGINNV_REDACT_SNAPS, &redact_snaps, &numredactsnaps));
 		dsl_dataset_activate_redaction(newds, redact_snaps,
 		    numredactsnaps, tx);
 	}
 
 	dmu_buf_will_dirty(newds->ds_dbuf, tx);
 	dsl_dataset_phys(newds)->ds_flags |= DS_FLAG_INCONSISTENT;
 
 	/*
 	 * If we actually created a non-clone, we need to create the objset
 	 * in our new dataset. If this is a raw send we postpone this until
 	 * dmu_recv_stream() so that we can allocate the metadnode with the
 	 * properties from the DRR_BEGIN payload.
 	 */
 	rrw_enter(&newds->ds_bp_rwlock, RW_READER, FTAG);
 	if (BP_IS_HOLE(dsl_dataset_get_blkptr(newds)) &&
 	    (featureflags & DMU_BACKUP_FEATURE_RAW) == 0 &&
 	    !drc->drc_heal) {
 		(void) dmu_objset_create_impl(dp->dp_spa,
 		    newds, dsl_dataset_get_blkptr(newds), drrb->drr_type, tx);
 	}
 	rrw_exit(&newds->ds_bp_rwlock, FTAG);
 
 	drba->drba_cookie->drc_ds = newds;
 	drba->drba_cookie->drc_os = os;
 
 	spa_history_log_internal_ds(newds, "receive", tx, " ");
 }
 
 static int
 dmu_recv_resume_begin_check(void *arg, dmu_tx_t *tx)
 {
 	dmu_recv_begin_arg_t *drba = arg;
 	dmu_recv_cookie_t *drc = drba->drba_cookie;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	struct drr_begin *drrb = drc->drc_drrb;
 	int error;
 	ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE;
 	dsl_dataset_t *ds;
 	const char *tofs = drc->drc_tofs;
 
 	/* already checked */
 	ASSERT3U(drrb->drr_magic, ==, DMU_BACKUP_MAGIC);
 	ASSERT(drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING);
 
 	if (DMU_GET_STREAM_HDRTYPE(drrb->drr_versioninfo) ==
 	    DMU_COMPOUNDSTREAM ||
 	    drrb->drr_type >= DMU_OST_NUMTYPES)
 		return (SET_ERROR(EINVAL));
 
 	/*
 	 * This is mostly a sanity check since we should have already done these
 	 * checks during a previous attempt to receive the data.
 	 */
 	error = recv_begin_check_feature_flags_impl(drc->drc_featureflags,
 	    dp->dp_spa);
 	if (error != 0)
 		return (error);
 
 	/* 6 extra bytes for /%recv */
 	char recvname[ZFS_MAX_DATASET_NAME_LEN + 6];
 
 	(void) snprintf(recvname, sizeof (recvname), "%s/%s",
 	    tofs, recv_clone_name);
 
 	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) {
 		/* raw receives require spill block allocation flag */
 		if (!(drrb->drr_flags & DRR_FLAG_SPILL_BLOCK))
 			return (SET_ERROR(ZFS_ERR_SPILL_BLOCK_FLAG_MISSING));
 	} else {
 		dsflags |= DS_HOLD_FLAG_DECRYPT;
 	}
 
 	boolean_t recvexist = B_TRUE;
 	if (dsl_dataset_hold_flags(dp, recvname, dsflags, FTAG, &ds) != 0) {
 		/* %recv does not exist; continue in tofs */
 		recvexist = B_FALSE;
 		error = dsl_dataset_hold_flags(dp, tofs, dsflags, FTAG, &ds);
 		if (error != 0)
 			return (error);
 	}
 
 	/*
 	 * Resume of full/newfs recv on existing dataset should be done with
 	 * force flag
 	 */
 	if (recvexist && drrb->drr_fromguid == 0 && !drc->drc_force) {
 		dsl_dataset_rele_flags(ds, dsflags, FTAG);
 		return (SET_ERROR(ZFS_ERR_RESUME_EXISTS));
 	}
 
 	/* check that ds is marked inconsistent */
 	if (!DS_IS_INCONSISTENT(ds)) {
 		dsl_dataset_rele_flags(ds, dsflags, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
 
 	/* check that there is resuming data, and that the toguid matches */
 	if (!dsl_dataset_is_zapified(ds)) {
 		dsl_dataset_rele_flags(ds, dsflags, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
 	uint64_t val;
 	error = zap_lookup(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val);
 	if (error != 0 || drrb->drr_toguid != val) {
 		dsl_dataset_rele_flags(ds, dsflags, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
 
 	/*
 	 * Check if the receive is still running.  If so, it will be owned.
 	 * Note that nothing else can own the dataset (e.g. after the receive
 	 * fails) because it will be marked inconsistent.
 	 */
 	if (dsl_dataset_has_owner(ds)) {
 		dsl_dataset_rele_flags(ds, dsflags, FTAG);
 		return (SET_ERROR(EBUSY));
 	}
 
 	/* There should not be any snapshots of this fs yet. */
 	if (ds->ds_prev != NULL && ds->ds_prev->ds_dir == ds->ds_dir) {
 		dsl_dataset_rele_flags(ds, dsflags, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
 
 	/*
 	 * Note: resume point will be checked when we process the first WRITE
 	 * record.
 	 */
 
 	/* check that the origin matches */
 	val = 0;
 	(void) zap_lookup(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val);
 	if (drrb->drr_fromguid != val) {
 		dsl_dataset_rele_flags(ds, dsflags, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
 
 	if (ds->ds_prev != NULL && drrb->drr_fromguid != 0)
 		drc->drc_fromsnapobj = ds->ds_prev->ds_object;
 
 	/*
 	 * If we're resuming, and the send is redacted, then the original send
 	 * must have been redacted, and must have been redacted with respect to
 	 * the same snapshots.
 	 */
 	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_REDACTED) {
 		uint64_t num_ds_redact_snaps;
 		uint64_t *ds_redact_snaps;
 
 		uint_t num_stream_redact_snaps;
 		uint64_t *stream_redact_snaps;
 
 		if (nvlist_lookup_uint64_array(drc->drc_begin_nvl,
 		    BEGINNV_REDACT_SNAPS, &stream_redact_snaps,
 		    &num_stream_redact_snaps) != 0) {
 			dsl_dataset_rele_flags(ds, dsflags, FTAG);
 			return (SET_ERROR(EINVAL));
 		}
 
 		if (!dsl_dataset_get_uint64_array_feature(ds,
 		    SPA_FEATURE_REDACTED_DATASETS, &num_ds_redact_snaps,
 		    &ds_redact_snaps)) {
 			dsl_dataset_rele_flags(ds, dsflags, FTAG);
 			return (SET_ERROR(EINVAL));
 		}
 
 		for (int i = 0; i < num_ds_redact_snaps; i++) {
 			if (!redact_snaps_contains(ds_redact_snaps,
 			    num_ds_redact_snaps, stream_redact_snaps[i])) {
 				dsl_dataset_rele_flags(ds, dsflags, FTAG);
 				return (SET_ERROR(EINVAL));
 			}
 		}
 	}
 
 	error = recv_check_large_blocks(ds, drc->drc_featureflags);
 	if (error != 0) {
 		dsl_dataset_rele_flags(ds, dsflags, FTAG);
 		return (error);
 	}
 
 	dsl_dataset_rele_flags(ds, dsflags, FTAG);
 	return (0);
 }
 
 static void
 dmu_recv_resume_begin_sync(void *arg, dmu_tx_t *tx)
 {
 	dmu_recv_begin_arg_t *drba = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	const char *tofs = drba->drba_cookie->drc_tofs;
 	uint64_t featureflags = drba->drba_cookie->drc_featureflags;
 	dsl_dataset_t *ds;
 	ds_hold_flags_t dsflags = DS_HOLD_FLAG_NONE;
 	/* 6 extra bytes for /%recv */
 	char recvname[ZFS_MAX_DATASET_NAME_LEN + 6];
 
 	(void) snprintf(recvname, sizeof (recvname), "%s/%s", tofs,
 	    recv_clone_name);
 
 	if (featureflags & DMU_BACKUP_FEATURE_RAW) {
 		drba->drba_cookie->drc_raw = B_TRUE;
 	} else {
 		dsflags |= DS_HOLD_FLAG_DECRYPT;
 	}
 
 	if (dsl_dataset_own_force(dp, recvname, dsflags, dmu_recv_tag, &ds)
 	    != 0) {
 		/* %recv does not exist; continue in tofs */
 		VERIFY0(dsl_dataset_own_force(dp, tofs, dsflags, dmu_recv_tag,
 		    &ds));
 		drba->drba_cookie->drc_newfs = B_TRUE;
 	}
 
 	ASSERT(DS_IS_INCONSISTENT(ds));
 	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
 	ASSERT(!BP_IS_HOLE(dsl_dataset_get_blkptr(ds)) ||
 	    drba->drba_cookie->drc_raw);
 	rrw_exit(&ds->ds_bp_rwlock, FTAG);
 
 	drba->drba_cookie->drc_ds = ds;
 	VERIFY0(dmu_objset_from_ds(ds, &drba->drba_cookie->drc_os));
 	drba->drba_cookie->drc_should_save = B_TRUE;
 
 	spa_history_log_internal_ds(ds, "resume receive", tx, " ");
 }
 
 /*
  * NB: callers *MUST* call dmu_recv_stream() if dmu_recv_begin()
  * succeeds; otherwise we will leak the holds on the datasets.
  */
 int
 dmu_recv_begin(const char *tofs, const char *tosnap,
     dmu_replay_record_t *drr_begin, boolean_t force, boolean_t heal,
     boolean_t resumable, nvlist_t *localprops, nvlist_t *hidden_args,
     const char *origin, dmu_recv_cookie_t *drc, zfs_file_t *fp,
     offset_t *voffp)
 {
 	dmu_recv_begin_arg_t drba = { 0 };
 	int err = 0;
 
+	cred_t *cr = CRED();
+	crhold(cr);
+
 	memset(drc, 0, sizeof (dmu_recv_cookie_t));
 	drc->drc_drr_begin = drr_begin;
 	drc->drc_drrb = &drr_begin->drr_u.drr_begin;
 	drc->drc_tosnap = tosnap;
 	drc->drc_tofs = tofs;
 	drc->drc_force = force;
 	drc->drc_heal = heal;
 	drc->drc_resumable = resumable;
-	drc->drc_cred = CRED();
-	drc->drc_proc = curproc;
+	drc->drc_cred = cr;
 	drc->drc_clone = (origin != NULL);
 
 	if (drc->drc_drrb->drr_magic == BSWAP_64(DMU_BACKUP_MAGIC)) {
 		drc->drc_byteswap = B_TRUE;
 		(void) fletcher_4_incremental_byteswap(drr_begin,
 		    sizeof (dmu_replay_record_t), &drc->drc_cksum);
 		byteswap_record(drr_begin);
 	} else if (drc->drc_drrb->drr_magic == DMU_BACKUP_MAGIC) {
 		(void) fletcher_4_incremental_native(drr_begin,
 		    sizeof (dmu_replay_record_t), &drc->drc_cksum);
 	} else {
+		crfree(cr);
+		drc->drc_cred = NULL;
 		return (SET_ERROR(EINVAL));
 	}
 
 	drc->drc_fp = fp;
 	drc->drc_voff = *voffp;
 	drc->drc_featureflags =
 	    DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo);
 
 	uint32_t payloadlen = drc->drc_drr_begin->drr_payloadlen;
 
 	/*
 	 * Since OpenZFS 2.0.0, we have enforced a 64MB limit in userspace
 	 * configurable via ZFS_SENDRECV_MAX_NVLIST. We enforce 256MB as a hard
 	 * upper limit. Systems with less than 1GB of RAM will see a lower
 	 * limit from `arc_all_memory() / 4`.
 	 */
-	if (payloadlen > (MIN((1U << 28), arc_all_memory() / 4)))
-		return (E2BIG);
-
+	if (payloadlen > (MIN((1U << 28), arc_all_memory() / 4))) {
+		crfree(cr);
+		drc->drc_cred = NULL;
+		return (SET_ERROR(E2BIG));
+	}
 
 	if (payloadlen != 0) {
 		void *payload = vmem_alloc(payloadlen, KM_SLEEP);
 		/*
 		 * For compatibility with recursive send streams, we don't do
 		 * this here if the stream could be part of a package. Instead,
 		 * we'll do it in dmu_recv_stream. If we pull the next header
 		 * too early, and it's the END record, we break the `recv_skip`
 		 * logic.
 		 */
 
 		err = receive_read_payload_and_next_header(drc, payloadlen,
 		    payload);
 		if (err != 0) {
 			vmem_free(payload, payloadlen);
+			crfree(cr);
+			drc->drc_cred = NULL;
 			return (err);
 		}
 		err = nvlist_unpack(payload, payloadlen, &drc->drc_begin_nvl,
 		    KM_SLEEP);
 		vmem_free(payload, payloadlen);
 		if (err != 0) {
 			kmem_free(drc->drc_next_rrd,
 			    sizeof (*drc->drc_next_rrd));
+			crfree(cr);
+			drc->drc_cred = NULL;
 			return (err);
 		}
 	}
 
 	if (drc->drc_drrb->drr_flags & DRR_FLAG_SPILL_BLOCK)
 		drc->drc_spill = B_TRUE;
 
 	drba.drba_origin = origin;
 	drba.drba_cookie = drc;
-	drba.drba_cred = CRED();
-	drba.drba_proc = curproc;
+	drba.drba_cred = drc->drc_cred;
 
 	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING) {
 		err = dsl_sync_task(tofs,
 		    dmu_recv_resume_begin_check, dmu_recv_resume_begin_sync,
 		    &drba, 5, ZFS_SPACE_CHECK_NORMAL);
 	} else {
 		/*
 		 * For non-raw, non-incremental, non-resuming receives the
 		 * user can specify encryption parameters on the command line
 		 * with "zfs recv -o". For these receives we create a dcp and
 		 * pass it to the sync task. Creating the dcp will implicitly
 		 * remove the encryption params from the localprops nvlist,
 		 * which avoids errors when trying to set these normally
 		 * read-only properties. Any other kind of receive that
 		 * attempts to set these properties will fail as a result.
 		 */
 		if ((DMU_GET_FEATUREFLAGS(drc->drc_drrb->drr_versioninfo) &
 		    DMU_BACKUP_FEATURE_RAW) == 0 &&
 		    origin == NULL && drc->drc_drrb->drr_fromguid == 0) {
 			err = dsl_crypto_params_create_nvlist(DCP_CMD_NONE,
 			    localprops, hidden_args, &drba.drba_dcp);
 		}
 
 		if (err == 0) {
 			err = dsl_sync_task(tofs,
 			    dmu_recv_begin_check, dmu_recv_begin_sync,
 			    &drba, 5, ZFS_SPACE_CHECK_NORMAL);
 			dsl_crypto_params_free(drba.drba_dcp, !!err);
 		}
 	}
 
 	if (err != 0) {
 		kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd));
 		nvlist_free(drc->drc_begin_nvl);
+		crfree(cr);
+		drc->drc_cred = NULL;
 	}
 	return (err);
 }
 
 /*
  * Holds data need for corrective recv callback
  */
 typedef struct cr_cb_data {
 	uint64_t size;
 	zbookmark_phys_t zb;
 	spa_t *spa;
 } cr_cb_data_t;
 
 static void
 corrective_read_done(zio_t *zio)
 {
 	cr_cb_data_t *data = zio->io_private;
 	/* Corruption corrected; update error log if needed */
 	if (zio->io_error == 0)
 		spa_remove_error(data->spa, &data->zb, &zio->io_bp->blk_birth);
 	kmem_free(data, sizeof (cr_cb_data_t));
 	abd_free(zio->io_abd);
 }
 
 /*
  * zio_rewrite the data pointed to by bp with the data from the rrd's abd.
  */
 static int
 do_corrective_recv(struct receive_writer_arg *rwa, struct drr_write *drrw,
     struct receive_record_arg *rrd, blkptr_t *bp)
 {
 	int err;
 	zio_t *io;
 	zbookmark_phys_t zb;
 	dnode_t *dn;
 	abd_t *abd = rrd->abd;
 	zio_cksum_t bp_cksum = bp->blk_cksum;
 	zio_flag_t flags = ZIO_FLAG_SPECULATIVE | ZIO_FLAG_DONT_RETRY |
 	    ZIO_FLAG_CANFAIL;
 
 	if (rwa->raw)
 		flags |= ZIO_FLAG_RAW;
 
 	err = dnode_hold(rwa->os, drrw->drr_object, FTAG, &dn);
 	if (err != 0)
 		return (err);
 	SET_BOOKMARK(&zb, dmu_objset_id(rwa->os), drrw->drr_object, 0,
 	    dbuf_whichblock(dn, 0, drrw->drr_offset));
 	dnode_rele(dn, FTAG);
 
 	if (!rwa->raw && DRR_WRITE_COMPRESSED(drrw)) {
 		/* Decompress the stream data */
 		abd_t *dabd = abd_alloc_linear(
 		    drrw->drr_logical_size, B_FALSE);
 		err = zio_decompress_data(drrw->drr_compressiontype,
 		    abd, abd_to_buf(dabd), abd_get_size(abd),
 		    abd_get_size(dabd), NULL);
 
 		if (err != 0) {
 			abd_free(dabd);
 			return (err);
 		}
 		/* Swap in the newly decompressed data into the abd */
 		abd_free(abd);
 		abd = dabd;
 	}
 
 	if (!rwa->raw && BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF) {
 		/* Recompress the data */
 		abd_t *cabd = abd_alloc_linear(BP_GET_PSIZE(bp),
 		    B_FALSE);
 		void *buf = abd_to_buf(cabd);
 		uint64_t csize = zio_compress_data(BP_GET_COMPRESS(bp),
 		    abd, &buf, abd_get_size(abd),
 		    rwa->os->os_complevel);
 		abd_zero_off(cabd, csize, BP_GET_PSIZE(bp) - csize);
 		/* Swap in newly compressed data into the abd */
 		abd_free(abd);
 		abd = cabd;
 		flags |= ZIO_FLAG_RAW_COMPRESS;
 	}
 
 	/*
 	 * The stream is not encrypted but the data on-disk is.
 	 * We need to re-encrypt the buf using the same
 	 * encryption type, salt, iv, and mac that was used to encrypt
 	 * the block previosly.
 	 */
 	if (!rwa->raw && BP_USES_CRYPT(bp)) {
 		dsl_dataset_t *ds;
 		dsl_crypto_key_t *dck = NULL;
 		uint8_t salt[ZIO_DATA_SALT_LEN];
 		uint8_t iv[ZIO_DATA_IV_LEN];
 		uint8_t mac[ZIO_DATA_MAC_LEN];
 		boolean_t no_crypt = B_FALSE;
 		dsl_pool_t *dp = dmu_objset_pool(rwa->os);
 		abd_t *eabd = abd_alloc_linear(BP_GET_PSIZE(bp), B_FALSE);
 
 		zio_crypt_decode_params_bp(bp, salt, iv);
 		zio_crypt_decode_mac_bp(bp, mac);
 
 		dsl_pool_config_enter(dp, FTAG);
 		err = dsl_dataset_hold_flags(dp, rwa->tofs,
 		    DS_HOLD_FLAG_DECRYPT, FTAG, &ds);
 		if (err != 0) {
 			dsl_pool_config_exit(dp, FTAG);
 			abd_free(eabd);
 			return (SET_ERROR(EACCES));
 		}
 
 		/* Look up the key from the spa's keystore */
 		err = spa_keystore_lookup_key(rwa->os->os_spa,
 		    zb.zb_objset, FTAG, &dck);
 		if (err != 0) {
 			dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT,
 			    FTAG);
 			dsl_pool_config_exit(dp, FTAG);
 			abd_free(eabd);
 			return (SET_ERROR(EACCES));
 		}
 
 		err = zio_do_crypt_abd(B_TRUE, &dck->dck_key,
 		    BP_GET_TYPE(bp), BP_SHOULD_BYTESWAP(bp), salt, iv,
 		    mac, abd_get_size(abd), abd, eabd, &no_crypt);
 
 		spa_keystore_dsl_key_rele(rwa->os->os_spa, dck, FTAG);
 		dsl_dataset_rele_flags(ds, DS_HOLD_FLAG_DECRYPT, FTAG);
 		dsl_pool_config_exit(dp, FTAG);
 
 		ASSERT0(no_crypt);
 		if (err != 0) {
 			abd_free(eabd);
 			return (err);
 		}
 		/* Swap in the newly encrypted data into the abd */
 		abd_free(abd);
 		abd = eabd;
 
 		/*
 		 * We want to prevent zio_rewrite() from trying to
 		 * encrypt the data again
 		 */
 		flags |= ZIO_FLAG_RAW_ENCRYPT;
 	}
 	rrd->abd = abd;
 
 	io = zio_rewrite(NULL, rwa->os->os_spa, bp->blk_birth, bp, abd,
 	    BP_GET_PSIZE(bp), NULL, NULL, ZIO_PRIORITY_SYNC_WRITE, flags, &zb);
 
 	ASSERT(abd_get_size(abd) == BP_GET_LSIZE(bp) ||
 	    abd_get_size(abd) == BP_GET_PSIZE(bp));
 
 	/* compute new bp checksum value and make sure it matches the old one */
 	zio_checksum_compute(io, BP_GET_CHECKSUM(bp), abd, abd_get_size(abd));
 	if (!ZIO_CHECKSUM_EQUAL(bp_cksum, io->io_bp->blk_cksum)) {
 		zio_destroy(io);
 		if (zfs_recv_best_effort_corrective != 0)
 			return (0);
 		return (SET_ERROR(ECKSUM));
 	}
 
 	/* Correct the corruption in place */
 	err = zio_wait(io);
 	if (err == 0) {
 		cr_cb_data_t *cb_data =
 		    kmem_alloc(sizeof (cr_cb_data_t), KM_SLEEP);
 		cb_data->spa = rwa->os->os_spa;
 		cb_data->size = drrw->drr_logical_size;
 		cb_data->zb = zb;
 		/* Test if healing worked by re-reading the bp */
 		err = zio_wait(zio_read(rwa->heal_pio, rwa->os->os_spa, bp,
 		    abd_alloc_for_io(drrw->drr_logical_size, B_FALSE),
 		    drrw->drr_logical_size, corrective_read_done,
 		    cb_data, ZIO_PRIORITY_ASYNC_READ, flags, NULL));
 	}
 	if (err != 0 && zfs_recv_best_effort_corrective != 0)
 		err = 0;
 
 	return (err);
 }
 
 static int
 receive_read(dmu_recv_cookie_t *drc, int len, void *buf)
 {
 	int done = 0;
 
 	/*
 	 * The code doesn't rely on this (lengths being multiples of 8).  See
 	 * comment in dump_bytes.
 	 */
 	ASSERT(len % 8 == 0 ||
 	    (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) != 0);
 
 	while (done < len) {
 		ssize_t resid = len - done;
 		zfs_file_t *fp = drc->drc_fp;
 		int err = zfs_file_read(fp, (char *)buf + done,
 		    len - done, &resid);
 		if (err == 0 && resid == len - done) {
 			/*
 			 * Note: ECKSUM or ZFS_ERR_STREAM_TRUNCATED indicates
 			 * that the receive was interrupted and can
 			 * potentially be resumed.
 			 */
 			err = SET_ERROR(ZFS_ERR_STREAM_TRUNCATED);
 		}
 		drc->drc_voff += len - done - resid;
 		done = len - resid;
 		if (err != 0)
 			return (err);
 	}
 
 	drc->drc_bytes_read += len;
 
 	ASSERT3U(done, ==, len);
 	return (0);
 }
 
 static inline uint8_t
 deduce_nblkptr(dmu_object_type_t bonus_type, uint64_t bonus_size)
 {
 	if (bonus_type == DMU_OT_SA) {
 		return (1);
 	} else {
 		return (1 +
 		    ((DN_OLD_MAX_BONUSLEN -
 		    MIN(DN_OLD_MAX_BONUSLEN, bonus_size)) >> SPA_BLKPTRSHIFT));
 	}
 }
 
 static void
 save_resume_state(struct receive_writer_arg *rwa,
     uint64_t object, uint64_t offset, dmu_tx_t *tx)
 {
 	int txgoff = dmu_tx_get_txg(tx) & TXG_MASK;
 
 	if (!rwa->resumable)
 		return;
 
 	/*
 	 * We use ds_resume_bytes[] != 0 to indicate that we need to
 	 * update this on disk, so it must not be 0.
 	 */
 	ASSERT(rwa->bytes_read != 0);
 
 	/*
 	 * We only resume from write records, which have a valid
 	 * (non-meta-dnode) object number.
 	 */
 	ASSERT(object != 0);
 
 	/*
 	 * For resuming to work correctly, we must receive records in order,
 	 * sorted by object,offset.  This is checked by the callers, but
 	 * assert it here for good measure.
 	 */
 	ASSERT3U(object, >=, rwa->os->os_dsl_dataset->ds_resume_object[txgoff]);
 	ASSERT(object != rwa->os->os_dsl_dataset->ds_resume_object[txgoff] ||
 	    offset >= rwa->os->os_dsl_dataset->ds_resume_offset[txgoff]);
 	ASSERT3U(rwa->bytes_read, >=,
 	    rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff]);
 
 	rwa->os->os_dsl_dataset->ds_resume_object[txgoff] = object;
 	rwa->os->os_dsl_dataset->ds_resume_offset[txgoff] = offset;
 	rwa->os->os_dsl_dataset->ds_resume_bytes[txgoff] = rwa->bytes_read;
 }
 
 static int
 receive_object_is_same_generation(objset_t *os, uint64_t object,
     dmu_object_type_t old_bonus_type, dmu_object_type_t new_bonus_type,
     const void *new_bonus, boolean_t *samegenp)
 {
 	zfs_file_info_t zoi;
 	int err;
 
 	dmu_buf_t *old_bonus_dbuf;
 	err = dmu_bonus_hold(os, object, FTAG, &old_bonus_dbuf);
 	if (err != 0)
 		return (err);
 	err = dmu_get_file_info(os, old_bonus_type, old_bonus_dbuf->db_data,
 	    &zoi);
 	dmu_buf_rele(old_bonus_dbuf, FTAG);
 	if (err != 0)
 		return (err);
 	uint64_t old_gen = zoi.zfi_generation;
 
 	err = dmu_get_file_info(os, new_bonus_type, new_bonus, &zoi);
 	if (err != 0)
 		return (err);
 	uint64_t new_gen = zoi.zfi_generation;
 
 	*samegenp = (old_gen == new_gen);
 	return (0);
 }
 
 static int
 receive_handle_existing_object(const struct receive_writer_arg *rwa,
     const struct drr_object *drro, const dmu_object_info_t *doi,
     const void *bonus_data,
     uint64_t *object_to_hold, uint32_t *new_blksz)
 {
 	uint32_t indblksz = drro->drr_indblkshift ?
 	    1ULL << drro->drr_indblkshift : 0;
 	int nblkptr = deduce_nblkptr(drro->drr_bonustype,
 	    drro->drr_bonuslen);
 	uint8_t dn_slots = drro->drr_dn_slots != 0 ?
 	    drro->drr_dn_slots : DNODE_MIN_SLOTS;
 	boolean_t do_free_range = B_FALSE;
 	int err;
 
 	*object_to_hold = drro->drr_object;
 
 	/* nblkptr should be bounded by the bonus size and type */
 	if (rwa->raw && nblkptr != drro->drr_nblkptr)
 		return (SET_ERROR(EINVAL));
 
 	/*
 	 * After the previous send stream, the sending system may
 	 * have freed this object, and then happened to re-allocate
 	 * this object number in a later txg. In this case, we are
 	 * receiving a different logical file, and the block size may
 	 * appear to be different.  i.e. we may have a different
 	 * block size for this object than what the send stream says.
 	 * In this case we need to remove the object's contents,
 	 * so that its structure can be changed and then its contents
 	 * entirely replaced by subsequent WRITE records.
 	 *
 	 * If this is a -L (--large-block) incremental stream, and
 	 * the previous stream was not -L, the block size may appear
 	 * to increase.  i.e. we may have a smaller block size for
 	 * this object than what the send stream says.  In this case
 	 * we need to keep the object's contents and block size
 	 * intact, so that we don't lose parts of the object's
 	 * contents that are not changed by this incremental send
 	 * stream.
 	 *
 	 * We can distinguish between the two above cases by using
 	 * the ZPL's generation number (see
 	 * receive_object_is_same_generation()).  However, we only
 	 * want to rely on the generation number when absolutely
 	 * necessary, because with raw receives, the generation is
 	 * encrypted.  We also want to minimize dependence on the
 	 * ZPL, so that other types of datasets can also be received
 	 * (e.g. ZVOLs, although note that ZVOLS currently do not
 	 * reallocate their objects or change their structure).
 	 * Therefore, we check a number of different cases where we
 	 * know it is safe to discard the object's contents, before
 	 * using the ZPL's generation number to make the above
 	 * distinction.
 	 */
 	if (drro->drr_blksz != doi->doi_data_block_size) {
 		if (rwa->raw) {
 			/*
 			 * RAW streams always have large blocks, so
 			 * we are sure that the data is not needed
 			 * due to changing --large-block to be on.
 			 * Which is fortunate since the bonus buffer
 			 * (which contains the ZPL generation) is
 			 * encrypted, and the key might not be
 			 * loaded.
 			 */
 			do_free_range = B_TRUE;
 		} else if (rwa->full) {
 			/*
 			 * This is a full send stream, so it always
 			 * replaces what we have.  Even if the
 			 * generation numbers happen to match, this
 			 * can not actually be the same logical file.
 			 * This is relevant when receiving a full
 			 * send as a clone.
 			 */
 			do_free_range = B_TRUE;
 		} else if (drro->drr_type !=
 		    DMU_OT_PLAIN_FILE_CONTENTS ||
 		    doi->doi_type != DMU_OT_PLAIN_FILE_CONTENTS) {
 			/*
 			 * PLAIN_FILE_CONTENTS are the only type of
 			 * objects that have ever been stored with
 			 * large blocks, so we don't need the special
 			 * logic below.  ZAP blocks can shrink (when
 			 * there's only one block), so we don't want
 			 * to hit the error below about block size
 			 * only increasing.
 			 */
 			do_free_range = B_TRUE;
 		} else if (doi->doi_max_offset <=
 		    doi->doi_data_block_size) {
 			/*
 			 * There is only one block.  We can free it,
 			 * because its contents will be replaced by a
 			 * WRITE record.  This can not be the no-L ->
 			 * -L case, because the no-L case would have
 			 * resulted in multiple blocks.  If we
 			 * supported -L -> no-L, it would not be safe
 			 * to free the file's contents.  Fortunately,
 			 * that is not allowed (see
 			 * recv_check_large_blocks()).
 			 */
 			do_free_range = B_TRUE;
 		} else {
 			boolean_t is_same_gen;
 			err = receive_object_is_same_generation(rwa->os,
 			    drro->drr_object, doi->doi_bonus_type,
 			    drro->drr_bonustype, bonus_data, &is_same_gen);
 			if (err != 0)
 				return (SET_ERROR(EINVAL));
 
 			if (is_same_gen) {
 				/*
 				 * This is the same logical file, and
 				 * the block size must be increasing.
 				 * It could only decrease if
 				 * --large-block was changed to be
 				 * off, which is checked in
 				 * recv_check_large_blocks().
 				 */
 				if (drro->drr_blksz <=
 				    doi->doi_data_block_size)
 					return (SET_ERROR(EINVAL));
 				/*
 				 * We keep the existing blocksize and
 				 * contents.
 				 */
 				*new_blksz =
 				    doi->doi_data_block_size;
 			} else {
 				do_free_range = B_TRUE;
 			}
 		}
 	}
 
 	/* nblkptr can only decrease if the object was reallocated */
 	if (nblkptr < doi->doi_nblkptr)
 		do_free_range = B_TRUE;
 
 	/* number of slots can only change on reallocation */
 	if (dn_slots != doi->doi_dnodesize >> DNODE_SHIFT)
 		do_free_range = B_TRUE;
 
 	/*
 	 * For raw sends we also check a few other fields to
 	 * ensure we are preserving the objset structure exactly
 	 * as it was on the receive side:
 	 *     - A changed indirect block size
 	 *     - A smaller nlevels
 	 */
 	if (rwa->raw) {
 		if (indblksz != doi->doi_metadata_block_size)
 			do_free_range = B_TRUE;
 		if (drro->drr_nlevels < doi->doi_indirection)
 			do_free_range = B_TRUE;
 	}
 
 	if (do_free_range) {
 		err = dmu_free_long_range(rwa->os, drro->drr_object,
 		    0, DMU_OBJECT_END);
 		if (err != 0)
 			return (SET_ERROR(EINVAL));
 	}
 
 	/*
 	 * The dmu does not currently support decreasing nlevels or changing
 	 * indirect block size if there is already one, same as changing the
 	 * number of of dnode slots on an object.  For non-raw sends this
 	 * does not matter and the new object can just use the previous one's
 	 * parameters.  For raw sends, however, the structure of the received
 	 * dnode (including indirects and dnode slots) must match that of the
 	 * send side.  Therefore, instead of using dmu_object_reclaim(), we
 	 * must free the object completely and call dmu_object_claim_dnsize()
 	 * instead.
 	 */
 	if ((rwa->raw && ((doi->doi_indirection > 1 &&
 	    indblksz != doi->doi_metadata_block_size) ||
 	    drro->drr_nlevels < doi->doi_indirection)) ||
 	    dn_slots != doi->doi_dnodesize >> DNODE_SHIFT) {
 		err = dmu_free_long_object(rwa->os, drro->drr_object);
 		if (err != 0)
 			return (SET_ERROR(EINVAL));
 
 		txg_wait_synced(dmu_objset_pool(rwa->os), 0);
 		*object_to_hold = DMU_NEW_OBJECT;
 	}
 
 	/*
 	 * For raw receives, free everything beyond the new incoming
 	 * maxblkid. Normally this would be done with a DRR_FREE
 	 * record that would come after this DRR_OBJECT record is
 	 * processed. However, for raw receives we manually set the
 	 * maxblkid from the drr_maxblkid and so we must first free
 	 * everything above that blkid to ensure the DMU is always
 	 * consistent with itself. We will never free the first block
 	 * of the object here because a maxblkid of 0 could indicate
 	 * an object with a single block or one with no blocks. This
 	 * free may be skipped when dmu_free_long_range() was called
 	 * above since it covers the entire object's contents.
 	 */
 	if (rwa->raw && *object_to_hold != DMU_NEW_OBJECT && !do_free_range) {
 		err = dmu_free_long_range(rwa->os, drro->drr_object,
 		    (drro->drr_maxblkid + 1) * doi->doi_data_block_size,
 		    DMU_OBJECT_END);
 		if (err != 0)
 			return (SET_ERROR(EINVAL));
 	}
 	return (0);
 }
 
 noinline static int
 receive_object(struct receive_writer_arg *rwa, struct drr_object *drro,
     void *data)
 {
 	dmu_object_info_t doi;
 	dmu_tx_t *tx;
 	int err;
 	uint32_t new_blksz = drro->drr_blksz;
 	uint8_t dn_slots = drro->drr_dn_slots != 0 ?
 	    drro->drr_dn_slots : DNODE_MIN_SLOTS;
 
 	if (drro->drr_type == DMU_OT_NONE ||
 	    !DMU_OT_IS_VALID(drro->drr_type) ||
 	    !DMU_OT_IS_VALID(drro->drr_bonustype) ||
 	    drro->drr_checksumtype >= ZIO_CHECKSUM_FUNCTIONS ||
 	    drro->drr_compress >= ZIO_COMPRESS_FUNCTIONS ||
 	    P2PHASE(drro->drr_blksz, SPA_MINBLOCKSIZE) ||
 	    drro->drr_blksz < SPA_MINBLOCKSIZE ||
 	    drro->drr_blksz > spa_maxblocksize(dmu_objset_spa(rwa->os)) ||
 	    drro->drr_bonuslen >
 	    DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(rwa->os))) ||
 	    dn_slots >
 	    (spa_maxdnodesize(dmu_objset_spa(rwa->os)) >> DNODE_SHIFT)) {
 		return (SET_ERROR(EINVAL));
 	}
 
 	if (rwa->raw) {
 		/*
 		 * We should have received a DRR_OBJECT_RANGE record
 		 * containing this block and stored it in rwa.
 		 */
 		if (drro->drr_object < rwa->or_firstobj ||
 		    drro->drr_object >= rwa->or_firstobj + rwa->or_numslots ||
 		    drro->drr_raw_bonuslen < drro->drr_bonuslen ||
 		    drro->drr_indblkshift > SPA_MAXBLOCKSHIFT ||
 		    drro->drr_nlevels > DN_MAX_LEVELS ||
 		    drro->drr_nblkptr > DN_MAX_NBLKPTR ||
 		    DN_SLOTS_TO_BONUSLEN(dn_slots) <
 		    drro->drr_raw_bonuslen)
 			return (SET_ERROR(EINVAL));
 	} else {
 		/*
 		 * The DRR_OBJECT_SPILL flag is valid when the DRR_BEGIN
 		 * record indicates this by setting DRR_FLAG_SPILL_BLOCK.
 		 */
 		if (((drro->drr_flags & ~(DRR_OBJECT_SPILL))) ||
 		    (!rwa->spill && DRR_OBJECT_HAS_SPILL(drro->drr_flags))) {
 			return (SET_ERROR(EINVAL));
 		}
 
 		if (drro->drr_raw_bonuslen != 0 || drro->drr_nblkptr != 0 ||
 		    drro->drr_indblkshift != 0 || drro->drr_nlevels != 0) {
 			return (SET_ERROR(EINVAL));
 		}
 	}
 
 	err = dmu_object_info(rwa->os, drro->drr_object, &doi);
 
 	if (err != 0 && err != ENOENT && err != EEXIST)
 		return (SET_ERROR(EINVAL));
 
 	if (drro->drr_object > rwa->max_object)
 		rwa->max_object = drro->drr_object;
 
 	/*
 	 * If we are losing blkptrs or changing the block size this must
 	 * be a new file instance.  We must clear out the previous file
 	 * contents before we can change this type of metadata in the dnode.
 	 * Raw receives will also check that the indirect structure of the
 	 * dnode hasn't changed.
 	 */
 	uint64_t object_to_hold;
 	if (err == 0) {
 		err = receive_handle_existing_object(rwa, drro, &doi, data,
 		    &object_to_hold, &new_blksz);
 		if (err != 0)
 			return (err);
 	} else if (err == EEXIST) {
 		/*
 		 * The object requested is currently an interior slot of a
 		 * multi-slot dnode. This will be resolved when the next txg
 		 * is synced out, since the send stream will have told us
 		 * to free this slot when we freed the associated dnode
 		 * earlier in the stream.
 		 */
 		txg_wait_synced(dmu_objset_pool(rwa->os), 0);
 
 		if (dmu_object_info(rwa->os, drro->drr_object, NULL) != ENOENT)
 			return (SET_ERROR(EINVAL));
 
 		/* object was freed and we are about to allocate a new one */
 		object_to_hold = DMU_NEW_OBJECT;
 	} else {
 		/*
 		 * If the only record in this range so far was DRR_FREEOBJECTS
 		 * with at least one actually freed object, it's possible that
 		 * the block will now be converted to a hole. We need to wait
 		 * for the txg to sync to prevent races.
 		 */
 		if (rwa->or_need_sync == ORNS_YES)
 			txg_wait_synced(dmu_objset_pool(rwa->os), 0);
 
 		/* object is free and we are about to allocate a new one */
 		object_to_hold = DMU_NEW_OBJECT;
 	}
 
 	/* Only relevant for the first object in the range */
 	rwa->or_need_sync = ORNS_NO;
 
 	/*
 	 * If this is a multi-slot dnode there is a chance that this
 	 * object will expand into a slot that is already used by
 	 * another object from the previous snapshot. We must free
 	 * these objects before we attempt to allocate the new dnode.
 	 */
 	if (dn_slots > 1) {
 		boolean_t need_sync = B_FALSE;
 
 		for (uint64_t slot = drro->drr_object + 1;
 		    slot < drro->drr_object + dn_slots;
 		    slot++) {
 			dmu_object_info_t slot_doi;
 
 			err = dmu_object_info(rwa->os, slot, &slot_doi);
 			if (err == ENOENT || err == EEXIST)
 				continue;
 			else if (err != 0)
 				return (err);
 
 			err = dmu_free_long_object(rwa->os, slot);
 			if (err != 0)
 				return (err);
 
 			need_sync = B_TRUE;
 		}
 
 		if (need_sync)
 			txg_wait_synced(dmu_objset_pool(rwa->os), 0);
 	}
 
 	tx = dmu_tx_create(rwa->os);
 	dmu_tx_hold_bonus(tx, object_to_hold);
 	dmu_tx_hold_write(tx, object_to_hold, 0, 0);
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err != 0) {
 		dmu_tx_abort(tx);
 		return (err);
 	}
 
 	if (object_to_hold == DMU_NEW_OBJECT) {
 		/* Currently free, wants to be allocated */
 		err = dmu_object_claim_dnsize(rwa->os, drro->drr_object,
 		    drro->drr_type, new_blksz,
 		    drro->drr_bonustype, drro->drr_bonuslen,
 		    dn_slots << DNODE_SHIFT, tx);
 	} else if (drro->drr_type != doi.doi_type ||
 	    new_blksz != doi.doi_data_block_size ||
 	    drro->drr_bonustype != doi.doi_bonus_type ||
 	    drro->drr_bonuslen != doi.doi_bonus_size) {
 		/* Currently allocated, but with different properties */
 		err = dmu_object_reclaim_dnsize(rwa->os, drro->drr_object,
 		    drro->drr_type, new_blksz,
 		    drro->drr_bonustype, drro->drr_bonuslen,
 		    dn_slots << DNODE_SHIFT, rwa->spill ?
 		    DRR_OBJECT_HAS_SPILL(drro->drr_flags) : B_FALSE, tx);
 	} else if (rwa->spill && !DRR_OBJECT_HAS_SPILL(drro->drr_flags)) {
 		/*
 		 * Currently allocated, the existing version of this object
 		 * may reference a spill block that is no longer allocated
 		 * at the source and needs to be freed.
 		 */
 		err = dmu_object_rm_spill(rwa->os, drro->drr_object, tx);
 	}
 
 	if (err != 0) {
 		dmu_tx_commit(tx);
 		return (SET_ERROR(EINVAL));
 	}
 
 	if (rwa->or_crypt_params_present) {
 		/*
 		 * Set the crypt params for the buffer associated with this
 		 * range of dnodes.  This causes the blkptr_t to have the
 		 * same crypt params (byteorder, salt, iv, mac) as on the
 		 * sending side.
 		 *
 		 * Since we are committing this tx now, it is possible for
 		 * the dnode block to end up on-disk with the incorrect MAC,
 		 * if subsequent objects in this block are received in a
 		 * different txg.  However, since the dataset is marked as
 		 * inconsistent, no code paths will do a non-raw read (or
 		 * decrypt the block / verify the MAC). The receive code and
 		 * scrub code can safely do raw reads and verify the
 		 * checksum.  They don't need to verify the MAC.
 		 */
 		dmu_buf_t *db = NULL;
 		uint64_t offset = rwa->or_firstobj * DNODE_MIN_SIZE;
 
 		err = dmu_buf_hold_by_dnode(DMU_META_DNODE(rwa->os),
 		    offset, FTAG, &db, DMU_READ_PREFETCH | DMU_READ_NO_DECRYPT);
 		if (err != 0) {
 			dmu_tx_commit(tx);
 			return (SET_ERROR(EINVAL));
 		}
 
 		dmu_buf_set_crypt_params(db, rwa->or_byteorder,
 		    rwa->or_salt, rwa->or_iv, rwa->or_mac, tx);
 
 		dmu_buf_rele(db, FTAG);
 
 		rwa->or_crypt_params_present = B_FALSE;
 	}
 
 	dmu_object_set_checksum(rwa->os, drro->drr_object,
 	    drro->drr_checksumtype, tx);
 	dmu_object_set_compress(rwa->os, drro->drr_object,
 	    drro->drr_compress, tx);
 
 	/* handle more restrictive dnode structuring for raw recvs */
 	if (rwa->raw) {
 		/*
 		 * Set the indirect block size, block shift, nlevels.
 		 * This will not fail because we ensured all of the
 		 * blocks were freed earlier if this is a new object.
 		 * For non-new objects block size and indirect block
 		 * shift cannot change and nlevels can only increase.
 		 */
 		ASSERT3U(new_blksz, ==, drro->drr_blksz);
 		VERIFY0(dmu_object_set_blocksize(rwa->os, drro->drr_object,
 		    drro->drr_blksz, drro->drr_indblkshift, tx));
 		VERIFY0(dmu_object_set_nlevels(rwa->os, drro->drr_object,
 		    drro->drr_nlevels, tx));
 
 		/*
 		 * Set the maxblkid. This will always succeed because
 		 * we freed all blocks beyond the new maxblkid above.
 		 */
 		VERIFY0(dmu_object_set_maxblkid(rwa->os, drro->drr_object,
 		    drro->drr_maxblkid, tx));
 	}
 
 	if (data != NULL) {
 		dmu_buf_t *db;
 		dnode_t *dn;
 		uint32_t flags = DMU_READ_NO_PREFETCH;
 
 		if (rwa->raw)
 			flags |= DMU_READ_NO_DECRYPT;
 
 		VERIFY0(dnode_hold(rwa->os, drro->drr_object, FTAG, &dn));
 		VERIFY0(dmu_bonus_hold_by_dnode(dn, FTAG, &db, flags));
 
 		dmu_buf_will_dirty(db, tx);
 
 		ASSERT3U(db->db_size, >=, drro->drr_bonuslen);
 		memcpy(db->db_data, data, DRR_OBJECT_PAYLOAD_SIZE(drro));
 
 		/*
 		 * Raw bonus buffers have their byteorder determined by the
 		 * DRR_OBJECT_RANGE record.
 		 */
 		if (rwa->byteswap && !rwa->raw) {
 			dmu_object_byteswap_t byteswap =
 			    DMU_OT_BYTESWAP(drro->drr_bonustype);
 			dmu_ot_byteswap[byteswap].ob_func(db->db_data,
 			    DRR_OBJECT_PAYLOAD_SIZE(drro));
 		}
 		dmu_buf_rele(db, FTAG);
 		dnode_rele(dn, FTAG);
 	}
 
 	/*
 	 * If the receive fails, we want the resume stream to start with the
 	 * same record that we last successfully received. There is no way to
 	 * request resume from the object record, but we can benefit from the
 	 * fact that sender always sends object record before anything else,
 	 * after which it will "resend" data at offset 0 and resume normally.
 	 */
 	save_resume_state(rwa, drro->drr_object, 0, tx);
 
 	dmu_tx_commit(tx);
 
 	return (0);
 }
 
 noinline static int
 receive_freeobjects(struct receive_writer_arg *rwa,
     struct drr_freeobjects *drrfo)
 {
 	uint64_t obj;
 	int next_err = 0;
 
 	if (drrfo->drr_firstobj + drrfo->drr_numobjs < drrfo->drr_firstobj)
 		return (SET_ERROR(EINVAL));
 
 	for (obj = drrfo->drr_firstobj == 0 ? 1 : drrfo->drr_firstobj;
 	    obj < drrfo->drr_firstobj + drrfo->drr_numobjs &&
 	    obj < DN_MAX_OBJECT && next_err == 0;
 	    next_err = dmu_object_next(rwa->os, &obj, FALSE, 0)) {
 		dmu_object_info_t doi;
 		int err;
 
 		err = dmu_object_info(rwa->os, obj, &doi);
 		if (err == ENOENT)
 			continue;
 		else if (err != 0)
 			return (err);
 
 		err = dmu_free_long_object(rwa->os, obj);
 
 		if (err != 0)
 			return (err);
 
 		if (rwa->or_need_sync == ORNS_MAYBE)
 			rwa->or_need_sync = ORNS_YES;
 	}
 	if (next_err != ESRCH)
 		return (next_err);
 	return (0);
 }
 
 /*
  * Note: if this fails, the caller will clean up any records left on the
  * rwa->write_batch list.
  */
 static int
 flush_write_batch_impl(struct receive_writer_arg *rwa)
 {
 	dnode_t *dn;
 	int err;
 
 	if (dnode_hold(rwa->os, rwa->last_object, FTAG, &dn) != 0)
 		return (SET_ERROR(EINVAL));
 
 	struct receive_record_arg *last_rrd = list_tail(&rwa->write_batch);
 	struct drr_write *last_drrw = &last_rrd->header.drr_u.drr_write;
 
 	struct receive_record_arg *first_rrd = list_head(&rwa->write_batch);
 	struct drr_write *first_drrw = &first_rrd->header.drr_u.drr_write;
 
 	ASSERT3U(rwa->last_object, ==, last_drrw->drr_object);
 	ASSERT3U(rwa->last_offset, ==, last_drrw->drr_offset);
 
 	dmu_tx_t *tx = dmu_tx_create(rwa->os);
 	dmu_tx_hold_write_by_dnode(tx, dn, first_drrw->drr_offset,
 	    last_drrw->drr_offset - first_drrw->drr_offset +
 	    last_drrw->drr_logical_size);
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err != 0) {
 		dmu_tx_abort(tx);
 		dnode_rele(dn, FTAG);
 		return (err);
 	}
 
 	struct receive_record_arg *rrd;
 	while ((rrd = list_head(&rwa->write_batch)) != NULL) {
 		struct drr_write *drrw = &rrd->header.drr_u.drr_write;
 		abd_t *abd = rrd->abd;
 
 		ASSERT3U(drrw->drr_object, ==, rwa->last_object);
 
 		if (drrw->drr_logical_size != dn->dn_datablksz) {
 			/*
 			 * The WRITE record is larger than the object's block
 			 * size.  We must be receiving an incremental
 			 * large-block stream into a dataset that previously did
 			 * a non-large-block receive.  Lightweight writes must
 			 * be exactly one block, so we need to decompress the
 			 * data (if compressed) and do a normal dmu_write().
 			 */
 			ASSERT3U(drrw->drr_logical_size, >, dn->dn_datablksz);
 			if (DRR_WRITE_COMPRESSED(drrw)) {
 				abd_t *decomp_abd =
 				    abd_alloc_linear(drrw->drr_logical_size,
 				    B_FALSE);
 
 				err = zio_decompress_data(
 				    drrw->drr_compressiontype,
 				    abd, abd_to_buf(decomp_abd),
 				    abd_get_size(abd),
 				    abd_get_size(decomp_abd), NULL);
 
 				if (err == 0) {
 					dmu_write_by_dnode(dn,
 					    drrw->drr_offset,
 					    drrw->drr_logical_size,
 					    abd_to_buf(decomp_abd), tx);
 				}
 				abd_free(decomp_abd);
 			} else {
 				dmu_write_by_dnode(dn,
 				    drrw->drr_offset,
 				    drrw->drr_logical_size,
 				    abd_to_buf(abd), tx);
 			}
 			if (err == 0)
 				abd_free(abd);
 		} else {
 			zio_prop_t zp = {0};
 			dmu_write_policy(rwa->os, dn, 0, 0, &zp);
 
 			zio_flag_t zio_flags = 0;
 
 			if (rwa->raw) {
 				zp.zp_encrypt = B_TRUE;
 				zp.zp_compress = drrw->drr_compressiontype;
 				zp.zp_byteorder = ZFS_HOST_BYTEORDER ^
 				    !!DRR_IS_RAW_BYTESWAPPED(drrw->drr_flags) ^
 				    rwa->byteswap;
 				memcpy(zp.zp_salt, drrw->drr_salt,
 				    ZIO_DATA_SALT_LEN);
 				memcpy(zp.zp_iv, drrw->drr_iv,
 				    ZIO_DATA_IV_LEN);
 				memcpy(zp.zp_mac, drrw->drr_mac,
 				    ZIO_DATA_MAC_LEN);
 				if (DMU_OT_IS_ENCRYPTED(zp.zp_type)) {
 					zp.zp_nopwrite = B_FALSE;
 					zp.zp_copies = MIN(zp.zp_copies,
 					    SPA_DVAS_PER_BP - 1);
 				}
 				zio_flags |= ZIO_FLAG_RAW;
 			} else if (DRR_WRITE_COMPRESSED(drrw)) {
 				ASSERT3U(drrw->drr_compressed_size, >, 0);
 				ASSERT3U(drrw->drr_logical_size, >=,
 				    drrw->drr_compressed_size);
 				zp.zp_compress = drrw->drr_compressiontype;
 				zio_flags |= ZIO_FLAG_RAW_COMPRESS;
 			} else if (rwa->byteswap) {
 				/*
 				 * Note: compressed blocks never need to be
 				 * byteswapped, because WRITE records for
 				 * metadata blocks are never compressed. The
 				 * exception is raw streams, which are written
 				 * in the original byteorder, and the byteorder
 				 * bit is preserved in the BP by setting
 				 * zp_byteorder above.
 				 */
 				dmu_object_byteswap_t byteswap =
 				    DMU_OT_BYTESWAP(drrw->drr_type);
 				dmu_ot_byteswap[byteswap].ob_func(
 				    abd_to_buf(abd),
 				    DRR_WRITE_PAYLOAD_SIZE(drrw));
 			}
 
 			/*
 			 * Since this data can't be read until the receive
 			 * completes, we can do a "lightweight" write for
 			 * improved performance.
 			 */
 			err = dmu_lightweight_write_by_dnode(dn,
 			    drrw->drr_offset, abd, &zp, zio_flags, tx);
 		}
 
 		if (err != 0) {
 			/*
 			 * This rrd is left on the list, so the caller will
 			 * free it (and the abd).
 			 */
 			break;
 		}
 
 		/*
 		 * Note: If the receive fails, we want the resume stream to
 		 * start with the same record that we last successfully
 		 * received (as opposed to the next record), so that we can
 		 * verify that we are resuming from the correct location.
 		 */
 		save_resume_state(rwa, drrw->drr_object, drrw->drr_offset, tx);
 
 		list_remove(&rwa->write_batch, rrd);
 		kmem_free(rrd, sizeof (*rrd));
 	}
 
 	dmu_tx_commit(tx);
 	dnode_rele(dn, FTAG);
 	return (err);
 }
 
 noinline static int
 flush_write_batch(struct receive_writer_arg *rwa)
 {
 	if (list_is_empty(&rwa->write_batch))
 		return (0);
 	int err = rwa->err;
 	if (err == 0)
 		err = flush_write_batch_impl(rwa);
 	if (err != 0) {
 		struct receive_record_arg *rrd;
 		while ((rrd = list_remove_head(&rwa->write_batch)) != NULL) {
 			abd_free(rrd->abd);
 			kmem_free(rrd, sizeof (*rrd));
 		}
 	}
 	ASSERT(list_is_empty(&rwa->write_batch));
 	return (err);
 }
 
 noinline static int
 receive_process_write_record(struct receive_writer_arg *rwa,
     struct receive_record_arg *rrd)
 {
 	int err = 0;
 
 	ASSERT3U(rrd->header.drr_type, ==, DRR_WRITE);
 	struct drr_write *drrw = &rrd->header.drr_u.drr_write;
 
 	if (drrw->drr_offset + drrw->drr_logical_size < drrw->drr_offset ||
 	    !DMU_OT_IS_VALID(drrw->drr_type))
 		return (SET_ERROR(EINVAL));
 
 	if (rwa->heal) {
 		blkptr_t *bp;
 		dmu_buf_t *dbp;
 		int flags = DB_RF_CANFAIL;
 
 		if (rwa->raw)
 			flags |= DB_RF_NO_DECRYPT;
 
 		if (rwa->byteswap) {
 			dmu_object_byteswap_t byteswap =
 			    DMU_OT_BYTESWAP(drrw->drr_type);
 			dmu_ot_byteswap[byteswap].ob_func(abd_to_buf(rrd->abd),
 			    DRR_WRITE_PAYLOAD_SIZE(drrw));
 		}
 
 		err = dmu_buf_hold_noread(rwa->os, drrw->drr_object,
 		    drrw->drr_offset, FTAG, &dbp);
 		if (err != 0)
 			return (err);
 
 		/* Try to read the object to see if it needs healing */
 		err = dbuf_read((dmu_buf_impl_t *)dbp, NULL, flags);
 		/*
 		 * We only try to heal when dbuf_read() returns a ECKSUMs.
 		 * Other errors (even EIO) get returned to caller.
 		 * EIO indicates that the device is not present/accessible,
 		 * so writing to it will likely fail.
 		 * If the block is healthy, we don't want to overwrite it
 		 * unnecessarily.
 		 */
 		if (err != ECKSUM) {
 			dmu_buf_rele(dbp, FTAG);
 			return (err);
 		}
 		/* Make sure the on-disk block and recv record sizes match */
 		if (drrw->drr_logical_size != dbp->db_size) {
 			err = ENOTSUP;
 			dmu_buf_rele(dbp, FTAG);
 			return (err);
 		}
 		/* Get the block pointer for the corrupted block */
 		bp = dmu_buf_get_blkptr(dbp);
 		err = do_corrective_recv(rwa, drrw, rrd, bp);
 		dmu_buf_rele(dbp, FTAG);
 		return (err);
 	}
 
 	/*
 	 * For resuming to work, records must be in increasing order
 	 * by (object, offset).
 	 */
 	if (drrw->drr_object < rwa->last_object ||
 	    (drrw->drr_object == rwa->last_object &&
 	    drrw->drr_offset < rwa->last_offset)) {
 		return (SET_ERROR(EINVAL));
 	}
 
 	struct receive_record_arg *first_rrd = list_head(&rwa->write_batch);
 	struct drr_write *first_drrw = &first_rrd->header.drr_u.drr_write;
 	uint64_t batch_size =
 	    MIN(zfs_recv_write_batch_size, DMU_MAX_ACCESS / 2);
 	if (first_rrd != NULL &&
 	    (drrw->drr_object != first_drrw->drr_object ||
 	    drrw->drr_offset >= first_drrw->drr_offset + batch_size)) {
 		err = flush_write_batch(rwa);
 		if (err != 0)
 			return (err);
 	}
 
 	rwa->last_object = drrw->drr_object;
 	rwa->last_offset = drrw->drr_offset;
 
 	if (rwa->last_object > rwa->max_object)
 		rwa->max_object = rwa->last_object;
 
 	list_insert_tail(&rwa->write_batch, rrd);
 	/*
 	 * Return EAGAIN to indicate that we will use this rrd again,
 	 * so the caller should not free it
 	 */
 	return (EAGAIN);
 }
 
 static int
 receive_write_embedded(struct receive_writer_arg *rwa,
     struct drr_write_embedded *drrwe, void *data)
 {
 	dmu_tx_t *tx;
 	int err;
 
 	if (drrwe->drr_offset + drrwe->drr_length < drrwe->drr_offset)
 		return (SET_ERROR(EINVAL));
 
 	if (drrwe->drr_psize > BPE_PAYLOAD_SIZE)
 		return (SET_ERROR(EINVAL));
 
 	if (drrwe->drr_etype >= NUM_BP_EMBEDDED_TYPES)
 		return (SET_ERROR(EINVAL));
 	if (drrwe->drr_compression >= ZIO_COMPRESS_FUNCTIONS)
 		return (SET_ERROR(EINVAL));
 	if (rwa->raw)
 		return (SET_ERROR(EINVAL));
 
 	if (drrwe->drr_object > rwa->max_object)
 		rwa->max_object = drrwe->drr_object;
 
 	tx = dmu_tx_create(rwa->os);
 
 	dmu_tx_hold_write(tx, drrwe->drr_object,
 	    drrwe->drr_offset, drrwe->drr_length);
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err != 0) {
 		dmu_tx_abort(tx);
 		return (err);
 	}
 
 	dmu_write_embedded(rwa->os, drrwe->drr_object,
 	    drrwe->drr_offset, data, drrwe->drr_etype,
 	    drrwe->drr_compression, drrwe->drr_lsize, drrwe->drr_psize,
 	    rwa->byteswap ^ ZFS_HOST_BYTEORDER, tx);
 
 	/* See comment in restore_write. */
 	save_resume_state(rwa, drrwe->drr_object, drrwe->drr_offset, tx);
 	dmu_tx_commit(tx);
 	return (0);
 }
 
 static int
 receive_spill(struct receive_writer_arg *rwa, struct drr_spill *drrs,
     abd_t *abd)
 {
 	dmu_buf_t *db, *db_spill;
 	int err;
 
 	if (drrs->drr_length < SPA_MINBLOCKSIZE ||
 	    drrs->drr_length > spa_maxblocksize(dmu_objset_spa(rwa->os)))
 		return (SET_ERROR(EINVAL));
 
 	/*
 	 * This is an unmodified spill block which was added to the stream
 	 * to resolve an issue with incorrectly removing spill blocks.  It
 	 * should be ignored by current versions of the code which support
 	 * the DRR_FLAG_SPILL_BLOCK flag.
 	 */
 	if (rwa->spill && DRR_SPILL_IS_UNMODIFIED(drrs->drr_flags)) {
 		abd_free(abd);
 		return (0);
 	}
 
 	if (rwa->raw) {
 		if (!DMU_OT_IS_VALID(drrs->drr_type) ||
 		    drrs->drr_compressiontype >= ZIO_COMPRESS_FUNCTIONS ||
 		    drrs->drr_compressed_size == 0)
 			return (SET_ERROR(EINVAL));
 	}
 
 	if (dmu_object_info(rwa->os, drrs->drr_object, NULL) != 0)
 		return (SET_ERROR(EINVAL));
 
 	if (drrs->drr_object > rwa->max_object)
 		rwa->max_object = drrs->drr_object;
 
 	VERIFY0(dmu_bonus_hold(rwa->os, drrs->drr_object, FTAG, &db));
 	if ((err = dmu_spill_hold_by_bonus(db, DMU_READ_NO_DECRYPT, FTAG,
 	    &db_spill)) != 0) {
 		dmu_buf_rele(db, FTAG);
 		return (err);
 	}
 
 	dmu_tx_t *tx = dmu_tx_create(rwa->os);
 
 	dmu_tx_hold_spill(tx, db->db_object);
 
 	err = dmu_tx_assign(tx, TXG_WAIT);
 	if (err != 0) {
 		dmu_buf_rele(db, FTAG);
 		dmu_buf_rele(db_spill, FTAG);
 		dmu_tx_abort(tx);
 		return (err);
 	}
 
 	/*
 	 * Spill blocks may both grow and shrink.  When a change in size
 	 * occurs any existing dbuf must be updated to match the logical
 	 * size of the provided arc_buf_t.
 	 */
 	if (db_spill->db_size != drrs->drr_length) {
 		dmu_buf_will_fill(db_spill, tx, B_FALSE);
 		VERIFY0(dbuf_spill_set_blksz(db_spill,
 		    drrs->drr_length, tx));
 	}
 
 	arc_buf_t *abuf;
 	if (rwa->raw) {
 		boolean_t byteorder = ZFS_HOST_BYTEORDER ^
 		    !!DRR_IS_RAW_BYTESWAPPED(drrs->drr_flags) ^
 		    rwa->byteswap;
 
 		abuf = arc_loan_raw_buf(dmu_objset_spa(rwa->os),
 		    drrs->drr_object, byteorder, drrs->drr_salt,
 		    drrs->drr_iv, drrs->drr_mac, drrs->drr_type,
 		    drrs->drr_compressed_size, drrs->drr_length,
 		    drrs->drr_compressiontype, 0);
 	} else {
 		abuf = arc_loan_buf(dmu_objset_spa(rwa->os),
 		    DMU_OT_IS_METADATA(drrs->drr_type),
 		    drrs->drr_length);
 		if (rwa->byteswap) {
 			dmu_object_byteswap_t byteswap =
 			    DMU_OT_BYTESWAP(drrs->drr_type);
 			dmu_ot_byteswap[byteswap].ob_func(abd_to_buf(abd),
 			    DRR_SPILL_PAYLOAD_SIZE(drrs));
 		}
 	}
 
 	memcpy(abuf->b_data, abd_to_buf(abd), DRR_SPILL_PAYLOAD_SIZE(drrs));
 	abd_free(abd);
 	dbuf_assign_arcbuf((dmu_buf_impl_t *)db_spill, abuf, tx);
 
 	dmu_buf_rele(db, FTAG);
 	dmu_buf_rele(db_spill, FTAG);
 
 	dmu_tx_commit(tx);
 	return (0);
 }
 
 noinline static int
 receive_free(struct receive_writer_arg *rwa, struct drr_free *drrf)
 {
 	int err;
 
 	if (drrf->drr_length != -1ULL &&
 	    drrf->drr_offset + drrf->drr_length < drrf->drr_offset)
 		return (SET_ERROR(EINVAL));
 
 	if (dmu_object_info(rwa->os, drrf->drr_object, NULL) != 0)
 		return (SET_ERROR(EINVAL));
 
 	if (drrf->drr_object > rwa->max_object)
 		rwa->max_object = drrf->drr_object;
 
 	err = dmu_free_long_range(rwa->os, drrf->drr_object,
 	    drrf->drr_offset, drrf->drr_length);
 
 	return (err);
 }
 
 static int
 receive_object_range(struct receive_writer_arg *rwa,
     struct drr_object_range *drror)
 {
 	/*
 	 * By default, we assume this block is in our native format
 	 * (ZFS_HOST_BYTEORDER). We then take into account whether
 	 * the send stream is byteswapped (rwa->byteswap). Finally,
 	 * we need to byteswap again if this particular block was
 	 * in non-native format on the send side.
 	 */
 	boolean_t byteorder = ZFS_HOST_BYTEORDER ^ rwa->byteswap ^
 	    !!DRR_IS_RAW_BYTESWAPPED(drror->drr_flags);
 
 	/*
 	 * Since dnode block sizes are constant, we should not need to worry
 	 * about making sure that the dnode block size is the same on the
 	 * sending and receiving sides for the time being. For non-raw sends,
 	 * this does not matter (and in fact we do not send a DRR_OBJECT_RANGE
 	 * record at all). Raw sends require this record type because the
 	 * encryption parameters are used to protect an entire block of bonus
 	 * buffers. If the size of dnode blocks ever becomes variable,
 	 * handling will need to be added to ensure that dnode block sizes
 	 * match on the sending and receiving side.
 	 */
 	if (drror->drr_numslots != DNODES_PER_BLOCK ||
 	    P2PHASE(drror->drr_firstobj, DNODES_PER_BLOCK) != 0 ||
 	    !rwa->raw)
 		return (SET_ERROR(EINVAL));
 
 	if (drror->drr_firstobj > rwa->max_object)
 		rwa->max_object = drror->drr_firstobj;
 
 	/*
 	 * The DRR_OBJECT_RANGE handling must be deferred to receive_object()
 	 * so that the block of dnodes is not written out when it's empty,
 	 * and converted to a HOLE BP.
 	 */
 	rwa->or_crypt_params_present = B_TRUE;
 	rwa->or_firstobj = drror->drr_firstobj;
 	rwa->or_numslots = drror->drr_numslots;
 	memcpy(rwa->or_salt, drror->drr_salt, ZIO_DATA_SALT_LEN);
 	memcpy(rwa->or_iv, drror->drr_iv, ZIO_DATA_IV_LEN);
 	memcpy(rwa->or_mac, drror->drr_mac, ZIO_DATA_MAC_LEN);
 	rwa->or_byteorder = byteorder;
 
 	rwa->or_need_sync = ORNS_MAYBE;
 
 	return (0);
 }
 
 /*
  * Until we have the ability to redact large ranges of data efficiently, we
  * process these records as frees.
  */
 noinline static int
 receive_redact(struct receive_writer_arg *rwa, struct drr_redact *drrr)
 {
 	struct drr_free drrf = {0};
 	drrf.drr_length = drrr->drr_length;
 	drrf.drr_object = drrr->drr_object;
 	drrf.drr_offset = drrr->drr_offset;
 	drrf.drr_toguid = drrr->drr_toguid;
 	return (receive_free(rwa, &drrf));
 }
 
 /* used to destroy the drc_ds on error */
 static void
 dmu_recv_cleanup_ds(dmu_recv_cookie_t *drc)
 {
 	dsl_dataset_t *ds = drc->drc_ds;
 	ds_hold_flags_t dsflags;
 
 	dsflags = (drc->drc_raw) ? DS_HOLD_FLAG_NONE : DS_HOLD_FLAG_DECRYPT;
 	/*
 	 * Wait for the txg sync before cleaning up the receive. For
 	 * resumable receives, this ensures that our resume state has
 	 * been written out to disk. For raw receives, this ensures
 	 * that the user accounting code will not attempt to do anything
 	 * after we stopped receiving the dataset.
 	 */
 	txg_wait_synced(ds->ds_dir->dd_pool, 0);
 	ds->ds_objset->os_raw_receive = B_FALSE;
 
 	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
 	if (drc->drc_resumable && drc->drc_should_save &&
 	    !BP_IS_HOLE(dsl_dataset_get_blkptr(ds))) {
 		rrw_exit(&ds->ds_bp_rwlock, FTAG);
 		dsl_dataset_disown(ds, dsflags, dmu_recv_tag);
 	} else {
 		char name[ZFS_MAX_DATASET_NAME_LEN];
 		rrw_exit(&ds->ds_bp_rwlock, FTAG);
 		dsl_dataset_name(ds, name);
 		dsl_dataset_disown(ds, dsflags, dmu_recv_tag);
 		if (!drc->drc_heal)
 			(void) dsl_destroy_head(name);
 	}
 }
 
 static void
 receive_cksum(dmu_recv_cookie_t *drc, int len, void *buf)
 {
 	if (drc->drc_byteswap) {
 		(void) fletcher_4_incremental_byteswap(buf, len,
 		    &drc->drc_cksum);
 	} else {
 		(void) fletcher_4_incremental_native(buf, len, &drc->drc_cksum);
 	}
 }
 
 /*
  * Read the payload into a buffer of size len, and update the current record's
  * payload field.
  * Allocate drc->drc_next_rrd and read the next record's header into
  * drc->drc_next_rrd->header.
  * Verify checksum of payload and next record.
  */
 static int
 receive_read_payload_and_next_header(dmu_recv_cookie_t *drc, int len, void *buf)
 {
 	int err;
 
 	if (len != 0) {
 		ASSERT3U(len, <=, SPA_MAXBLOCKSIZE);
 		err = receive_read(drc, len, buf);
 		if (err != 0)
 			return (err);
 		receive_cksum(drc, len, buf);
 
 		/* note: rrd is NULL when reading the begin record's payload */
 		if (drc->drc_rrd != NULL) {
 			drc->drc_rrd->payload = buf;
 			drc->drc_rrd->payload_size = len;
 			drc->drc_rrd->bytes_read = drc->drc_bytes_read;
 		}
 	} else {
 		ASSERT3P(buf, ==, NULL);
 	}
 
 	drc->drc_prev_cksum = drc->drc_cksum;
 
 	drc->drc_next_rrd = kmem_zalloc(sizeof (*drc->drc_next_rrd), KM_SLEEP);
 	err = receive_read(drc, sizeof (drc->drc_next_rrd->header),
 	    &drc->drc_next_rrd->header);
 	drc->drc_next_rrd->bytes_read = drc->drc_bytes_read;
 
 	if (err != 0) {
 		kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd));
 		drc->drc_next_rrd = NULL;
 		return (err);
 	}
 	if (drc->drc_next_rrd->header.drr_type == DRR_BEGIN) {
 		kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd));
 		drc->drc_next_rrd = NULL;
 		return (SET_ERROR(EINVAL));
 	}
 
 	/*
 	 * Note: checksum is of everything up to but not including the
 	 * checksum itself.
 	 */
 	ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
 	    ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
 	receive_cksum(drc,
 	    offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
 	    &drc->drc_next_rrd->header);
 
 	zio_cksum_t cksum_orig =
 	    drc->drc_next_rrd->header.drr_u.drr_checksum.drr_checksum;
 	zio_cksum_t *cksump =
 	    &drc->drc_next_rrd->header.drr_u.drr_checksum.drr_checksum;
 
 	if (drc->drc_byteswap)
 		byteswap_record(&drc->drc_next_rrd->header);
 
 	if ((!ZIO_CHECKSUM_IS_ZERO(cksump)) &&
 	    !ZIO_CHECKSUM_EQUAL(drc->drc_cksum, *cksump)) {
 		kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd));
 		drc->drc_next_rrd = NULL;
 		return (SET_ERROR(ECKSUM));
 	}
 
 	receive_cksum(drc, sizeof (cksum_orig), &cksum_orig);
 
 	return (0);
 }
 
 /*
  * Issue the prefetch reads for any necessary indirect blocks.
  *
  * We use the object ignore list to tell us whether or not to issue prefetches
  * for a given object.  We do this for both correctness (in case the blocksize
  * of an object has changed) and performance (if the object doesn't exist, don't
  * needlessly try to issue prefetches).  We also trim the list as we go through
  * the stream to prevent it from growing to an unbounded size.
  *
  * The object numbers within will always be in sorted order, and any write
  * records we see will also be in sorted order, but they're not sorted with
  * respect to each other (i.e. we can get several object records before
  * receiving each object's write records).  As a result, once we've reached a
  * given object number, we can safely remove any reference to lower object
  * numbers in the ignore list. In practice, we receive up to 32 object records
  * before receiving write records, so the list can have up to 32 nodes in it.
  */
 static void
 receive_read_prefetch(dmu_recv_cookie_t *drc, uint64_t object, uint64_t offset,
     uint64_t length)
 {
 	if (!objlist_exists(drc->drc_ignore_objlist, object)) {
 		dmu_prefetch(drc->drc_os, object, 1, offset, length,
 		    ZIO_PRIORITY_SYNC_READ);
 	}
 }
 
 /*
  * Read records off the stream, issuing any necessary prefetches.
  */
 static int
 receive_read_record(dmu_recv_cookie_t *drc)
 {
 	int err;
 
 	switch (drc->drc_rrd->header.drr_type) {
 	case DRR_OBJECT:
 	{
 		struct drr_object *drro =
 		    &drc->drc_rrd->header.drr_u.drr_object;
 		uint32_t size = DRR_OBJECT_PAYLOAD_SIZE(drro);
 		void *buf = NULL;
 		dmu_object_info_t doi;
 
 		if (size != 0)
 			buf = kmem_zalloc(size, KM_SLEEP);
 
 		err = receive_read_payload_and_next_header(drc, size, buf);
 		if (err != 0) {
 			kmem_free(buf, size);
 			return (err);
 		}
 		err = dmu_object_info(drc->drc_os, drro->drr_object, &doi);
 		/*
 		 * See receive_read_prefetch for an explanation why we're
 		 * storing this object in the ignore_obj_list.
 		 */
 		if (err == ENOENT || err == EEXIST ||
 		    (err == 0 && doi.doi_data_block_size != drro->drr_blksz)) {
 			objlist_insert(drc->drc_ignore_objlist,
 			    drro->drr_object);
 			err = 0;
 		}
 		return (err);
 	}
 	case DRR_FREEOBJECTS:
 	{
 		err = receive_read_payload_and_next_header(drc, 0, NULL);
 		return (err);
 	}
 	case DRR_WRITE:
 	{
 		struct drr_write *drrw = &drc->drc_rrd->header.drr_u.drr_write;
 		int size = DRR_WRITE_PAYLOAD_SIZE(drrw);
 		abd_t *abd = abd_alloc_linear(size, B_FALSE);
 		err = receive_read_payload_and_next_header(drc, size,
 		    abd_to_buf(abd));
 		if (err != 0) {
 			abd_free(abd);
 			return (err);
 		}
 		drc->drc_rrd->abd = abd;
 		receive_read_prefetch(drc, drrw->drr_object, drrw->drr_offset,
 		    drrw->drr_logical_size);
 		return (err);
 	}
 	case DRR_WRITE_EMBEDDED:
 	{
 		struct drr_write_embedded *drrwe =
 		    &drc->drc_rrd->header.drr_u.drr_write_embedded;
 		uint32_t size = P2ROUNDUP(drrwe->drr_psize, 8);
 		void *buf = kmem_zalloc(size, KM_SLEEP);
 
 		err = receive_read_payload_and_next_header(drc, size, buf);
 		if (err != 0) {
 			kmem_free(buf, size);
 			return (err);
 		}
 
 		receive_read_prefetch(drc, drrwe->drr_object, drrwe->drr_offset,
 		    drrwe->drr_length);
 		return (err);
 	}
 	case DRR_FREE:
 	case DRR_REDACT:
 	{
 		/*
 		 * It might be beneficial to prefetch indirect blocks here, but
 		 * we don't really have the data to decide for sure.
 		 */
 		err = receive_read_payload_and_next_header(drc, 0, NULL);
 		return (err);
 	}
 	case DRR_END:
 	{
 		struct drr_end *drre = &drc->drc_rrd->header.drr_u.drr_end;
 		if (!ZIO_CHECKSUM_EQUAL(drc->drc_prev_cksum,
 		    drre->drr_checksum))
 			return (SET_ERROR(ECKSUM));
 		return (0);
 	}
 	case DRR_SPILL:
 	{
 		struct drr_spill *drrs = &drc->drc_rrd->header.drr_u.drr_spill;
 		int size = DRR_SPILL_PAYLOAD_SIZE(drrs);
 		abd_t *abd = abd_alloc_linear(size, B_FALSE);
 		err = receive_read_payload_and_next_header(drc, size,
 		    abd_to_buf(abd));
 		if (err != 0)
 			abd_free(abd);
 		else
 			drc->drc_rrd->abd = abd;
 		return (err);
 	}
 	case DRR_OBJECT_RANGE:
 	{
 		err = receive_read_payload_and_next_header(drc, 0, NULL);
 		return (err);
 
 	}
 	default:
 		return (SET_ERROR(EINVAL));
 	}
 }
 
 
 
 static void
 dprintf_drr(struct receive_record_arg *rrd, int err)
 {
 #ifdef ZFS_DEBUG
 	switch (rrd->header.drr_type) {
 	case DRR_OBJECT:
 	{
 		struct drr_object *drro = &rrd->header.drr_u.drr_object;
 		dprintf("drr_type = OBJECT obj = %llu type = %u "
 		    "bonustype = %u blksz = %u bonuslen = %u cksumtype = %u "
 		    "compress = %u dn_slots = %u err = %d\n",
 		    (u_longlong_t)drro->drr_object, drro->drr_type,
 		    drro->drr_bonustype, drro->drr_blksz, drro->drr_bonuslen,
 		    drro->drr_checksumtype, drro->drr_compress,
 		    drro->drr_dn_slots, err);
 		break;
 	}
 	case DRR_FREEOBJECTS:
 	{
 		struct drr_freeobjects *drrfo =
 		    &rrd->header.drr_u.drr_freeobjects;
 		dprintf("drr_type = FREEOBJECTS firstobj = %llu "
 		    "numobjs = %llu err = %d\n",
 		    (u_longlong_t)drrfo->drr_firstobj,
 		    (u_longlong_t)drrfo->drr_numobjs, err);
 		break;
 	}
 	case DRR_WRITE:
 	{
 		struct drr_write *drrw = &rrd->header.drr_u.drr_write;
 		dprintf("drr_type = WRITE obj = %llu type = %u offset = %llu "
 		    "lsize = %llu cksumtype = %u flags = %u "
 		    "compress = %u psize = %llu err = %d\n",
 		    (u_longlong_t)drrw->drr_object, drrw->drr_type,
 		    (u_longlong_t)drrw->drr_offset,
 		    (u_longlong_t)drrw->drr_logical_size,
 		    drrw->drr_checksumtype, drrw->drr_flags,
 		    drrw->drr_compressiontype,
 		    (u_longlong_t)drrw->drr_compressed_size, err);
 		break;
 	}
 	case DRR_WRITE_BYREF:
 	{
 		struct drr_write_byref *drrwbr =
 		    &rrd->header.drr_u.drr_write_byref;
 		dprintf("drr_type = WRITE_BYREF obj = %llu offset = %llu "
 		    "length = %llu toguid = %llx refguid = %llx "
 		    "refobject = %llu refoffset = %llu cksumtype = %u "
 		    "flags = %u err = %d\n",
 		    (u_longlong_t)drrwbr->drr_object,
 		    (u_longlong_t)drrwbr->drr_offset,
 		    (u_longlong_t)drrwbr->drr_length,
 		    (u_longlong_t)drrwbr->drr_toguid,
 		    (u_longlong_t)drrwbr->drr_refguid,
 		    (u_longlong_t)drrwbr->drr_refobject,
 		    (u_longlong_t)drrwbr->drr_refoffset,
 		    drrwbr->drr_checksumtype, drrwbr->drr_flags, err);
 		break;
 	}
 	case DRR_WRITE_EMBEDDED:
 	{
 		struct drr_write_embedded *drrwe =
 		    &rrd->header.drr_u.drr_write_embedded;
 		dprintf("drr_type = WRITE_EMBEDDED obj = %llu offset = %llu "
 		    "length = %llu compress = %u etype = %u lsize = %u "
 		    "psize = %u err = %d\n",
 		    (u_longlong_t)drrwe->drr_object,
 		    (u_longlong_t)drrwe->drr_offset,
 		    (u_longlong_t)drrwe->drr_length,
 		    drrwe->drr_compression, drrwe->drr_etype,
 		    drrwe->drr_lsize, drrwe->drr_psize, err);
 		break;
 	}
 	case DRR_FREE:
 	{
 		struct drr_free *drrf = &rrd->header.drr_u.drr_free;
 		dprintf("drr_type = FREE obj = %llu offset = %llu "
 		    "length = %lld err = %d\n",
 		    (u_longlong_t)drrf->drr_object,
 		    (u_longlong_t)drrf->drr_offset,
 		    (longlong_t)drrf->drr_length,
 		    err);
 		break;
 	}
 	case DRR_SPILL:
 	{
 		struct drr_spill *drrs = &rrd->header.drr_u.drr_spill;
 		dprintf("drr_type = SPILL obj = %llu length = %llu "
 		    "err = %d\n", (u_longlong_t)drrs->drr_object,
 		    (u_longlong_t)drrs->drr_length, err);
 		break;
 	}
 	case DRR_OBJECT_RANGE:
 	{
 		struct drr_object_range *drror =
 		    &rrd->header.drr_u.drr_object_range;
 		dprintf("drr_type = OBJECT_RANGE firstobj = %llu "
 		    "numslots = %llu flags = %u err = %d\n",
 		    (u_longlong_t)drror->drr_firstobj,
 		    (u_longlong_t)drror->drr_numslots,
 		    drror->drr_flags, err);
 		break;
 	}
 	default:
 		return;
 	}
 #endif
 }
 
 /*
  * Commit the records to the pool.
  */
 static int
 receive_process_record(struct receive_writer_arg *rwa,
     struct receive_record_arg *rrd)
 {
 	int err;
 
 	/* Processing in order, therefore bytes_read should be increasing. */
 	ASSERT3U(rrd->bytes_read, >=, rwa->bytes_read);
 	rwa->bytes_read = rrd->bytes_read;
 
 	/* We can only heal write records; other ones get ignored */
 	if (rwa->heal && rrd->header.drr_type != DRR_WRITE) {
 		if (rrd->abd != NULL) {
 			abd_free(rrd->abd);
 			rrd->abd = NULL;
 		} else if (rrd->payload != NULL) {
 			kmem_free(rrd->payload, rrd->payload_size);
 			rrd->payload = NULL;
 		}
 		return (0);
 	}
 
 	if (!rwa->heal && rrd->header.drr_type != DRR_WRITE) {
 		err = flush_write_batch(rwa);
 		if (err != 0) {
 			if (rrd->abd != NULL) {
 				abd_free(rrd->abd);
 				rrd->abd = NULL;
 				rrd->payload = NULL;
 			} else if (rrd->payload != NULL) {
 				kmem_free(rrd->payload, rrd->payload_size);
 				rrd->payload = NULL;
 			}
 
 			return (err);
 		}
 	}
 
 	switch (rrd->header.drr_type) {
 	case DRR_OBJECT:
 	{
 		struct drr_object *drro = &rrd->header.drr_u.drr_object;
 		err = receive_object(rwa, drro, rrd->payload);
 		kmem_free(rrd->payload, rrd->payload_size);
 		rrd->payload = NULL;
 		break;
 	}
 	case DRR_FREEOBJECTS:
 	{
 		struct drr_freeobjects *drrfo =
 		    &rrd->header.drr_u.drr_freeobjects;
 		err = receive_freeobjects(rwa, drrfo);
 		break;
 	}
 	case DRR_WRITE:
 	{
 		err = receive_process_write_record(rwa, rrd);
 		if (rwa->heal) {
 			/*
 			 * If healing - always free the abd after processing
 			 */
 			abd_free(rrd->abd);
 			rrd->abd = NULL;
 		} else if (err != EAGAIN) {
 			/*
 			 * On success, a non-healing
 			 * receive_process_write_record() returns
 			 * EAGAIN to indicate that we do not want to free
 			 * the rrd or arc_buf.
 			 */
 			ASSERT(err != 0);
 			abd_free(rrd->abd);
 			rrd->abd = NULL;
 		}
 		break;
 	}
 	case DRR_WRITE_EMBEDDED:
 	{
 		struct drr_write_embedded *drrwe =
 		    &rrd->header.drr_u.drr_write_embedded;
 		err = receive_write_embedded(rwa, drrwe, rrd->payload);
 		kmem_free(rrd->payload, rrd->payload_size);
 		rrd->payload = NULL;
 		break;
 	}
 	case DRR_FREE:
 	{
 		struct drr_free *drrf = &rrd->header.drr_u.drr_free;
 		err = receive_free(rwa, drrf);
 		break;
 	}
 	case DRR_SPILL:
 	{
 		struct drr_spill *drrs = &rrd->header.drr_u.drr_spill;
 		err = receive_spill(rwa, drrs, rrd->abd);
 		if (err != 0)
 			abd_free(rrd->abd);
 		rrd->abd = NULL;
 		rrd->payload = NULL;
 		break;
 	}
 	case DRR_OBJECT_RANGE:
 	{
 		struct drr_object_range *drror =
 		    &rrd->header.drr_u.drr_object_range;
 		err = receive_object_range(rwa, drror);
 		break;
 	}
 	case DRR_REDACT:
 	{
 		struct drr_redact *drrr = &rrd->header.drr_u.drr_redact;
 		err = receive_redact(rwa, drrr);
 		break;
 	}
 	default:
 		err = (SET_ERROR(EINVAL));
 	}
 
 	if (err != 0)
 		dprintf_drr(rrd, err);
 
 	return (err);
 }
 
 /*
  * dmu_recv_stream's worker thread; pull records off the queue, and then call
  * receive_process_record  When we're done, signal the main thread and exit.
  */
 static __attribute__((noreturn)) void
 receive_writer_thread(void *arg)
 {
 	struct receive_writer_arg *rwa = arg;
 	struct receive_record_arg *rrd;
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 
 	for (rrd = bqueue_dequeue(&rwa->q); !rrd->eos_marker;
 	    rrd = bqueue_dequeue(&rwa->q)) {
 		/*
 		 * If there's an error, the main thread will stop putting things
 		 * on the queue, but we need to clear everything in it before we
 		 * can exit.
 		 */
 		int err = 0;
 		if (rwa->err == 0) {
 			err = receive_process_record(rwa, rrd);
 		} else if (rrd->abd != NULL) {
 			abd_free(rrd->abd);
 			rrd->abd = NULL;
 			rrd->payload = NULL;
 		} else if (rrd->payload != NULL) {
 			kmem_free(rrd->payload, rrd->payload_size);
 			rrd->payload = NULL;
 		}
 		/*
 		 * EAGAIN indicates that this record has been saved (on
 		 * raw->write_batch), and will be used again, so we don't
 		 * free it.
 		 * When healing data we always need to free the record.
 		 */
 		if (err != EAGAIN || rwa->heal) {
 			if (rwa->err == 0)
 				rwa->err = err;
 			kmem_free(rrd, sizeof (*rrd));
 		}
 	}
 	kmem_free(rrd, sizeof (*rrd));
 
 	if (rwa->heal) {
 		zio_wait(rwa->heal_pio);
 	} else {
 		int err = flush_write_batch(rwa);
 		if (rwa->err == 0)
 			rwa->err = err;
 	}
 	mutex_enter(&rwa->mutex);
 	rwa->done = B_TRUE;
 	cv_signal(&rwa->cv);
 	mutex_exit(&rwa->mutex);
 	spl_fstrans_unmark(cookie);
 	thread_exit();
 }
 
 static int
 resume_check(dmu_recv_cookie_t *drc, nvlist_t *begin_nvl)
 {
 	uint64_t val;
 	objset_t *mos = dmu_objset_pool(drc->drc_os)->dp_meta_objset;
 	uint64_t dsobj = dmu_objset_id(drc->drc_os);
 	uint64_t resume_obj, resume_off;
 
 	if (nvlist_lookup_uint64(begin_nvl,
 	    "resume_object", &resume_obj) != 0 ||
 	    nvlist_lookup_uint64(begin_nvl,
 	    "resume_offset", &resume_off) != 0) {
 		return (SET_ERROR(EINVAL));
 	}
 	VERIFY0(zap_lookup(mos, dsobj,
 	    DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val));
 	if (resume_obj != val)
 		return (SET_ERROR(EINVAL));
 	VERIFY0(zap_lookup(mos, dsobj,
 	    DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val));
 	if (resume_off != val)
 		return (SET_ERROR(EINVAL));
 
 	return (0);
 }
 
 /*
  * Read in the stream's records, one by one, and apply them to the pool.  There
  * are two threads involved; the thread that calls this function will spin up a
  * worker thread, read the records off the stream one by one, and issue
  * prefetches for any necessary indirect blocks.  It will then push the records
  * onto an internal blocking queue.  The worker thread will pull the records off
  * the queue, and actually write the data into the DMU.  This way, the worker
  * thread doesn't have to wait for reads to complete, since everything it needs
  * (the indirect blocks) will be prefetched.
  *
  * NB: callers *must* call dmu_recv_end() if this succeeds.
  */
 int
 dmu_recv_stream(dmu_recv_cookie_t *drc, offset_t *voffp)
 {
 	int err = 0;
 	struct receive_writer_arg *rwa = kmem_zalloc(sizeof (*rwa), KM_SLEEP);
 
 	if (dsl_dataset_has_resume_receive_state(drc->drc_ds)) {
 		uint64_t bytes = 0;
 		(void) zap_lookup(drc->drc_ds->ds_dir->dd_pool->dp_meta_objset,
 		    drc->drc_ds->ds_object, DS_FIELD_RESUME_BYTES,
 		    sizeof (bytes), 1, &bytes);
 		drc->drc_bytes_read += bytes;
 	}
 
 	drc->drc_ignore_objlist = objlist_create();
 
 	/* these were verified in dmu_recv_begin */
 	ASSERT3U(DMU_GET_STREAM_HDRTYPE(drc->drc_drrb->drr_versioninfo), ==,
 	    DMU_SUBSTREAM);
 	ASSERT3U(drc->drc_drrb->drr_type, <, DMU_OST_NUMTYPES);
 
 	ASSERT(dsl_dataset_phys(drc->drc_ds)->ds_flags & DS_FLAG_INCONSISTENT);
 	ASSERT0(drc->drc_os->os_encrypted &&
 	    (drc->drc_featureflags & DMU_BACKUP_FEATURE_EMBED_DATA));
 
 	/* handle DSL encryption key payload */
 	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RAW) {
 		nvlist_t *keynvl = NULL;
 
 		ASSERT(drc->drc_os->os_encrypted);
 		ASSERT(drc->drc_raw);
 
 		err = nvlist_lookup_nvlist(drc->drc_begin_nvl, "crypt_keydata",
 		    &keynvl);
 		if (err != 0)
 			goto out;
 
 		if (!drc->drc_heal) {
 			/*
 			 * If this is a new dataset we set the key immediately.
 			 * Otherwise we don't want to change the key until we
 			 * are sure the rest of the receive succeeded so we
 			 * stash the keynvl away until then.
 			 */
 			err = dsl_crypto_recv_raw(spa_name(drc->drc_os->os_spa),
 			    drc->drc_ds->ds_object, drc->drc_fromsnapobj,
 			    drc->drc_drrb->drr_type, keynvl, drc->drc_newfs);
 			if (err != 0)
 				goto out;
 		}
 
 		/* see comment in dmu_recv_end_sync() */
 		drc->drc_ivset_guid = 0;
 		(void) nvlist_lookup_uint64(keynvl, "to_ivset_guid",
 		    &drc->drc_ivset_guid);
 
 		if (!drc->drc_newfs)
 			drc->drc_keynvl = fnvlist_dup(keynvl);
 	}
 
 	if (drc->drc_featureflags & DMU_BACKUP_FEATURE_RESUMING) {
 		err = resume_check(drc, drc->drc_begin_nvl);
 		if (err != 0)
 			goto out;
 	}
 
 	/*
 	 * For compatibility with recursive send streams, we do this here,
 	 * rather than in dmu_recv_begin. If we pull the next header too
 	 * early, and it's the END record, we break the `recv_skip` logic.
 	 */
 	if (drc->drc_drr_begin->drr_payloadlen == 0) {
 		err = receive_read_payload_and_next_header(drc, 0, NULL);
 		if (err != 0)
 			goto out;
 	}
 
 	/*
 	 * If we failed before this point we will clean up any new resume
 	 * state that was created. Now that we've gotten past the initial
 	 * checks we are ok to retain that resume state.
 	 */
 	drc->drc_should_save = B_TRUE;
 
 	(void) bqueue_init(&rwa->q, zfs_recv_queue_ff,
 	    MAX(zfs_recv_queue_length, 2 * zfs_max_recordsize),
 	    offsetof(struct receive_record_arg, node));
 	cv_init(&rwa->cv, NULL, CV_DEFAULT, NULL);
 	mutex_init(&rwa->mutex, NULL, MUTEX_DEFAULT, NULL);
 	rwa->os = drc->drc_os;
 	rwa->byteswap = drc->drc_byteswap;
 	rwa->heal = drc->drc_heal;
 	rwa->tofs = drc->drc_tofs;
 	rwa->resumable = drc->drc_resumable;
 	rwa->raw = drc->drc_raw;
 	rwa->spill = drc->drc_spill;
 	rwa->full = (drc->drc_drr_begin->drr_u.drr_begin.drr_fromguid == 0);
 	rwa->os->os_raw_receive = drc->drc_raw;
 	if (drc->drc_heal) {
 		rwa->heal_pio = zio_root(drc->drc_os->os_spa, NULL, NULL,
 		    ZIO_FLAG_GODFATHER);
 	}
 	list_create(&rwa->write_batch, sizeof (struct receive_record_arg),
 	    offsetof(struct receive_record_arg, node.bqn_node));
 
 	(void) thread_create(NULL, 0, receive_writer_thread, rwa, 0, curproc,
 	    TS_RUN, minclsyspri);
 	/*
 	 * We're reading rwa->err without locks, which is safe since we are the
 	 * only reader, and the worker thread is the only writer.  It's ok if we
 	 * miss a write for an iteration or two of the loop, since the writer
 	 * thread will keep freeing records we send it until we send it an eos
 	 * marker.
 	 *
 	 * We can leave this loop in 3 ways:  First, if rwa->err is
 	 * non-zero.  In that case, the writer thread will free the rrd we just
 	 * pushed.  Second, if  we're interrupted; in that case, either it's the
 	 * first loop and drc->drc_rrd was never allocated, or it's later, and
 	 * drc->drc_rrd has been handed off to the writer thread who will free
 	 * it.  Finally, if receive_read_record fails or we're at the end of the
 	 * stream, then we free drc->drc_rrd and exit.
 	 */
 	while (rwa->err == 0) {
 		if (issig()) {
 			err = SET_ERROR(EINTR);
 			break;
 		}
 
 		ASSERT3P(drc->drc_rrd, ==, NULL);
 		drc->drc_rrd = drc->drc_next_rrd;
 		drc->drc_next_rrd = NULL;
 		/* Allocates and loads header into drc->drc_next_rrd */
 		err = receive_read_record(drc);
 
 		if (drc->drc_rrd->header.drr_type == DRR_END || err != 0) {
 			kmem_free(drc->drc_rrd, sizeof (*drc->drc_rrd));
 			drc->drc_rrd = NULL;
 			break;
 		}
 
 		bqueue_enqueue(&rwa->q, drc->drc_rrd,
 		    sizeof (struct receive_record_arg) +
 		    drc->drc_rrd->payload_size);
 		drc->drc_rrd = NULL;
 	}
 
 	ASSERT3P(drc->drc_rrd, ==, NULL);
 	drc->drc_rrd = kmem_zalloc(sizeof (*drc->drc_rrd), KM_SLEEP);
 	drc->drc_rrd->eos_marker = B_TRUE;
 	bqueue_enqueue_flush(&rwa->q, drc->drc_rrd, 1);
 
 	mutex_enter(&rwa->mutex);
 	while (!rwa->done) {
 		/*
 		 * We need to use cv_wait_sig() so that any process that may
 		 * be sleeping here can still fork.
 		 */
 		(void) cv_wait_sig(&rwa->cv, &rwa->mutex);
 	}
 	mutex_exit(&rwa->mutex);
 
 	/*
 	 * If we are receiving a full stream as a clone, all object IDs which
 	 * are greater than the maximum ID referenced in the stream are
 	 * by definition unused and must be freed.
 	 */
 	if (drc->drc_clone && drc->drc_drrb->drr_fromguid == 0) {
 		uint64_t obj = rwa->max_object + 1;
 		int free_err = 0;
 		int next_err = 0;
 
 		while (next_err == 0) {
 			free_err = dmu_free_long_object(rwa->os, obj);
 			if (free_err != 0 && free_err != ENOENT)
 				break;
 
 			next_err = dmu_object_next(rwa->os, &obj, FALSE, 0);
 		}
 
 		if (err == 0) {
 			if (free_err != 0 && free_err != ENOENT)
 				err = free_err;
 			else if (next_err != ESRCH)
 				err = next_err;
 		}
 	}
 
 	cv_destroy(&rwa->cv);
 	mutex_destroy(&rwa->mutex);
 	bqueue_destroy(&rwa->q);
 	list_destroy(&rwa->write_batch);
 	if (err == 0)
 		err = rwa->err;
 
 out:
 	/*
 	 * If we hit an error before we started the receive_writer_thread
 	 * we need to clean up the next_rrd we create by processing the
 	 * DRR_BEGIN record.
 	 */
 	if (drc->drc_next_rrd != NULL)
 		kmem_free(drc->drc_next_rrd, sizeof (*drc->drc_next_rrd));
 
 	/*
 	 * The objset will be invalidated by dmu_recv_end() when we do
 	 * dsl_dataset_clone_swap_sync_impl().
 	 */
 	drc->drc_os = NULL;
 
 	kmem_free(rwa, sizeof (*rwa));
 	nvlist_free(drc->drc_begin_nvl);
 
 	if (err != 0) {
 		/*
 		 * Clean up references. If receive is not resumable,
 		 * destroy what we created, so we don't leave it in
 		 * the inconsistent state.
 		 */
 		dmu_recv_cleanup_ds(drc);
 		nvlist_free(drc->drc_keynvl);
+		crfree(drc->drc_cred);
+		drc->drc_cred = NULL;
 	}
 
 	objlist_destroy(drc->drc_ignore_objlist);
 	drc->drc_ignore_objlist = NULL;
 	*voffp = drc->drc_voff;
 	return (err);
 }
 
 static int
 dmu_recv_end_check(void *arg, dmu_tx_t *tx)
 {
 	dmu_recv_cookie_t *drc = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	int error;
 
 	ASSERT3P(drc->drc_ds->ds_owner, ==, dmu_recv_tag);
 
 	if (drc->drc_heal) {
 		error = 0;
 	} else if (!drc->drc_newfs) {
 		dsl_dataset_t *origin_head;
 
 		error = dsl_dataset_hold(dp, drc->drc_tofs, FTAG, &origin_head);
 		if (error != 0)
 			return (error);
 		if (drc->drc_force) {
 			/*
 			 * We will destroy any snapshots in tofs (i.e. before
 			 * origin_head) that are after the origin (which is
 			 * the snap before drc_ds, because drc_ds can not
 			 * have any snaps of its own).
 			 */
 			uint64_t obj;
 
 			obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
 			while (obj !=
 			    dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) {
 				dsl_dataset_t *snap;
 				error = dsl_dataset_hold_obj(dp, obj, FTAG,
 				    &snap);
 				if (error != 0)
 					break;
 				if (snap->ds_dir != origin_head->ds_dir)
 					error = SET_ERROR(EINVAL);
 				if (error == 0)  {
 					error = dsl_destroy_snapshot_check_impl(
 					    snap, B_FALSE);
 				}
 				obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
 				dsl_dataset_rele(snap, FTAG);
 				if (error != 0)
 					break;
 			}
 			if (error != 0) {
 				dsl_dataset_rele(origin_head, FTAG);
 				return (error);
 			}
 		}
 		if (drc->drc_keynvl != NULL) {
 			error = dsl_crypto_recv_raw_key_check(drc->drc_ds,
 			    drc->drc_keynvl, tx);
 			if (error != 0) {
 				dsl_dataset_rele(origin_head, FTAG);
 				return (error);
 			}
 		}
 
 		error = dsl_dataset_clone_swap_check_impl(drc->drc_ds,
 		    origin_head, drc->drc_force, drc->drc_owner, tx);
 		if (error != 0) {
 			dsl_dataset_rele(origin_head, FTAG);
 			return (error);
 		}
 		error = dsl_dataset_snapshot_check_impl(origin_head,
-		    drc->drc_tosnap, tx, B_TRUE, 1,
-		    drc->drc_cred, drc->drc_proc);
+		    drc->drc_tosnap, tx, B_TRUE, 1, drc->drc_cred);
 		dsl_dataset_rele(origin_head, FTAG);
 		if (error != 0)
 			return (error);
 
 		error = dsl_destroy_head_check_impl(drc->drc_ds, 1);
 	} else {
 		error = dsl_dataset_snapshot_check_impl(drc->drc_ds,
-		    drc->drc_tosnap, tx, B_TRUE, 1,
-		    drc->drc_cred, drc->drc_proc);
+		    drc->drc_tosnap, tx, B_TRUE, 1, drc->drc_cred);
 	}
 	return (error);
 }
 
 static void
 dmu_recv_end_sync(void *arg, dmu_tx_t *tx)
 {
 	dmu_recv_cookie_t *drc = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	boolean_t encrypted = drc->drc_ds->ds_dir->dd_crypto_obj != 0;
 	uint64_t newsnapobj = 0;
 
 	spa_history_log_internal_ds(drc->drc_ds, "finish receiving",
 	    tx, "snap=%s", drc->drc_tosnap);
 	drc->drc_ds->ds_objset->os_raw_receive = B_FALSE;
 
 	if (drc->drc_heal) {
 		if (drc->drc_keynvl != NULL) {
 			nvlist_free(drc->drc_keynvl);
 			drc->drc_keynvl = NULL;
 		}
 	} else if (!drc->drc_newfs) {
 		dsl_dataset_t *origin_head;
 
 		VERIFY0(dsl_dataset_hold(dp, drc->drc_tofs, FTAG,
 		    &origin_head));
 
 		if (drc->drc_force) {
 			/*
 			 * Destroy any snapshots of drc_tofs (origin_head)
 			 * after the origin (the snap before drc_ds).
 			 */
 			uint64_t obj;
 
 			obj = dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
 			while (obj !=
 			    dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj) {
 				dsl_dataset_t *snap;
 				VERIFY0(dsl_dataset_hold_obj(dp, obj, FTAG,
 				    &snap));
 				ASSERT3P(snap->ds_dir, ==, origin_head->ds_dir);
 				obj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
 				dsl_destroy_snapshot_sync_impl(snap,
 				    B_FALSE, tx);
 				dsl_dataset_rele(snap, FTAG);
 			}
 		}
 		if (drc->drc_keynvl != NULL) {
 			dsl_crypto_recv_raw_key_sync(drc->drc_ds,
 			    drc->drc_keynvl, tx);
 			nvlist_free(drc->drc_keynvl);
 			drc->drc_keynvl = NULL;
 		}
 
 		VERIFY3P(drc->drc_ds->ds_prev, ==,
 		    origin_head->ds_prev);
 
 		dsl_dataset_clone_swap_sync_impl(drc->drc_ds,
 		    origin_head, tx);
 		/*
 		 * The objset was evicted by dsl_dataset_clone_swap_sync_impl,
 		 * so drc_os is no longer valid.
 		 */
 		drc->drc_os = NULL;
 
 		dsl_dataset_snapshot_sync_impl(origin_head,
 		    drc->drc_tosnap, tx);
 
 		/* set snapshot's creation time and guid */
 		dmu_buf_will_dirty(origin_head->ds_prev->ds_dbuf, tx);
 		dsl_dataset_phys(origin_head->ds_prev)->ds_creation_time =
 		    drc->drc_drrb->drr_creation_time;
 		dsl_dataset_phys(origin_head->ds_prev)->ds_guid =
 		    drc->drc_drrb->drr_toguid;
 		dsl_dataset_phys(origin_head->ds_prev)->ds_flags &=
 		    ~DS_FLAG_INCONSISTENT;
 
 		dmu_buf_will_dirty(origin_head->ds_dbuf, tx);
 		dsl_dataset_phys(origin_head)->ds_flags &=
 		    ~DS_FLAG_INCONSISTENT;
 
 		newsnapobj =
 		    dsl_dataset_phys(origin_head)->ds_prev_snap_obj;
 
 		dsl_dataset_rele(origin_head, FTAG);
 		dsl_destroy_head_sync_impl(drc->drc_ds, tx);
 
 		if (drc->drc_owner != NULL)
 			VERIFY3P(origin_head->ds_owner, ==, drc->drc_owner);
 	} else {
 		dsl_dataset_t *ds = drc->drc_ds;
 
 		dsl_dataset_snapshot_sync_impl(ds, drc->drc_tosnap, tx);
 
 		/* set snapshot's creation time and guid */
 		dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
 		dsl_dataset_phys(ds->ds_prev)->ds_creation_time =
 		    drc->drc_drrb->drr_creation_time;
 		dsl_dataset_phys(ds->ds_prev)->ds_guid =
 		    drc->drc_drrb->drr_toguid;
 		dsl_dataset_phys(ds->ds_prev)->ds_flags &=
 		    ~DS_FLAG_INCONSISTENT;
 
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
 		dsl_dataset_phys(ds)->ds_flags &= ~DS_FLAG_INCONSISTENT;
 		if (dsl_dataset_has_resume_receive_state(ds)) {
 			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
 			    DS_FIELD_RESUME_FROMGUID, tx);
 			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
 			    DS_FIELD_RESUME_OBJECT, tx);
 			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
 			    DS_FIELD_RESUME_OFFSET, tx);
 			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
 			    DS_FIELD_RESUME_BYTES, tx);
 			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
 			    DS_FIELD_RESUME_TOGUID, tx);
 			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
 			    DS_FIELD_RESUME_TONAME, tx);
 			(void) zap_remove(dp->dp_meta_objset, ds->ds_object,
 			    DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, tx);
 		}
 		newsnapobj =
 		    dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
 	}
 
 	/*
 	 * If this is a raw receive, the crypt_keydata nvlist will include
 	 * a to_ivset_guid for us to set on the new snapshot. This value
 	 * will override the value generated by the snapshot code. However,
 	 * this value may not be present, because older implementations of
 	 * the raw send code did not include this value, and we are still
 	 * allowed to receive them if the zfs_disable_ivset_guid_check
 	 * tunable is set, in which case we will leave the newly-generated
 	 * value.
 	 */
 	if (!drc->drc_heal && drc->drc_raw && drc->drc_ivset_guid != 0) {
 		dmu_object_zapify(dp->dp_meta_objset, newsnapobj,
 		    DMU_OT_DSL_DATASET, tx);
 		VERIFY0(zap_update(dp->dp_meta_objset, newsnapobj,
 		    DS_FIELD_IVSET_GUID, sizeof (uint64_t), 1,
 		    &drc->drc_ivset_guid, tx));
 	}
 
 	/*
 	 * Release the hold from dmu_recv_begin.  This must be done before
 	 * we return to open context, so that when we free the dataset's dnode
 	 * we can evict its bonus buffer. Since the dataset may be destroyed
 	 * at this point (and therefore won't have a valid pointer to the spa)
 	 * we release the key mapping manually here while we do have a valid
 	 * pointer, if it exists.
 	 */
 	if (!drc->drc_raw && encrypted) {
 		(void) spa_keystore_remove_mapping(dmu_tx_pool(tx)->dp_spa,
 		    drc->drc_ds->ds_object, drc->drc_ds);
 	}
 	dsl_dataset_disown(drc->drc_ds, 0, dmu_recv_tag);
 	drc->drc_ds = NULL;
 }
 
 static int dmu_recv_end_modified_blocks = 3;
 
 static int
 dmu_recv_existing_end(dmu_recv_cookie_t *drc)
 {
 #ifdef _KERNEL
 	/*
 	 * We will be destroying the ds; make sure its origin is unmounted if
 	 * necessary.
 	 */
 	char name[ZFS_MAX_DATASET_NAME_LEN];
 	dsl_dataset_name(drc->drc_ds, name);
 	zfs_destroy_unmount_origin(name);
 #endif
 
 	return (dsl_sync_task(drc->drc_tofs,
 	    dmu_recv_end_check, dmu_recv_end_sync, drc,
 	    dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL));
 }
 
 static int
 dmu_recv_new_end(dmu_recv_cookie_t *drc)
 {
 	return (dsl_sync_task(drc->drc_tofs,
 	    dmu_recv_end_check, dmu_recv_end_sync, drc,
 	    dmu_recv_end_modified_blocks, ZFS_SPACE_CHECK_NORMAL));
 }
 
 int
 dmu_recv_end(dmu_recv_cookie_t *drc, void *owner)
 {
 	int error;
 
 	drc->drc_owner = owner;
 
 	if (drc->drc_newfs)
 		error = dmu_recv_new_end(drc);
 	else
 		error = dmu_recv_existing_end(drc);
 
 	if (error != 0) {
 		dmu_recv_cleanup_ds(drc);
 		nvlist_free(drc->drc_keynvl);
 	} else if (!drc->drc_heal) {
 		if (drc->drc_newfs) {
 			zvol_create_minor(drc->drc_tofs);
 		}
 		char *snapname = kmem_asprintf("%s@%s",
 		    drc->drc_tofs, drc->drc_tosnap);
 		zvol_create_minor(snapname);
 		kmem_strfree(snapname);
 	}
+
+	crfree(drc->drc_cred);
+	drc->drc_cred = NULL;
+
 	return (error);
 }
 
 /*
  * Return TRUE if this objset is currently being received into.
  */
 boolean_t
 dmu_objset_is_receiving(objset_t *os)
 {
 	return (os->os_dsl_dataset != NULL &&
 	    os->os_dsl_dataset->ds_owner == dmu_recv_tag);
 }
 
 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, queue_length, UINT, ZMOD_RW,
 	"Maximum receive queue length");
 
 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, queue_ff, UINT, ZMOD_RW,
 	"Receive queue fill fraction");
 
 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, write_batch_size, UINT, ZMOD_RW,
 	"Maximum amount of writes to batch into one transaction");
 
 ZFS_MODULE_PARAM(zfs_recv, zfs_recv_, best_effort_corrective, INT, ZMOD_RW,
 	"Ignore errors during corrective receive");
 /* END CSTYLED */
diff --git a/sys/contrib/openzfs/module/zfs/dmu_send.c b/sys/contrib/openzfs/module/zfs/dmu_send.c
index 30d2ca5d6aa2..716b1b1f778c 100644
--- a/sys/contrib/openzfs/module/zfs/dmu_send.c
+++ b/sys/contrib/openzfs/module/zfs/dmu_send.c
@@ -1,3122 +1,3122 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2011, 2018 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright 2014 HybridCluster. All rights reserved.
  * Copyright 2016 RackTop Systems.
  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
  * Copyright (c) 2019, Klara Inc.
  * Copyright (c) 2019, Allan Jude
  */
 
 #include <sys/dmu.h>
 #include <sys/dmu_impl.h>
 #include <sys/dmu_tx.h>
 #include <sys/dbuf.h>
 #include <sys/dnode.h>
 #include <sys/zfs_context.h>
 #include <sys/dmu_objset.h>
 #include <sys/dmu_traverse.h>
 #include <sys/dsl_dataset.h>
 #include <sys/dsl_dir.h>
 #include <sys/dsl_prop.h>
 #include <sys/dsl_pool.h>
 #include <sys/dsl_synctask.h>
 #include <sys/spa_impl.h>
 #include <sys/zfs_ioctl.h>
 #include <sys/zap.h>
 #include <sys/zio_checksum.h>
 #include <sys/zfs_znode.h>
 #include <zfs_fletcher.h>
 #include <sys/avl.h>
 #include <sys/ddt.h>
 #include <sys/zfs_onexit.h>
 #include <sys/dmu_send.h>
 #include <sys/dmu_recv.h>
 #include <sys/dsl_destroy.h>
 #include <sys/blkptr.h>
 #include <sys/dsl_bookmark.h>
 #include <sys/zfeature.h>
 #include <sys/bqueue.h>
 #include <sys/zvol.h>
 #include <sys/policy.h>
 #include <sys/objlist.h>
 #ifdef _KERNEL
 #include <sys/zfs_vfsops.h>
 #endif
 
 /* Set this tunable to TRUE to replace corrupt data with 0x2f5baddb10c */
 static int zfs_send_corrupt_data = B_FALSE;
 /*
  * This tunable controls the amount of data (measured in bytes) that will be
  * prefetched by zfs send.  If the main thread is blocking on reads that haven't
  * completed, this variable might need to be increased.  If instead the main
  * thread is issuing new reads because the prefetches have fallen out of the
  * cache, this may need to be decreased.
  */
 static uint_t zfs_send_queue_length = SPA_MAXBLOCKSIZE;
 /*
  * This tunable controls the length of the queues that zfs send worker threads
  * use to communicate.  If the send_main_thread is blocking on these queues,
  * this variable may need to be increased.  If there is a significant slowdown
  * at the start of a send as these threads consume all the available IO
  * resources, this variable may need to be decreased.
  */
 static uint_t zfs_send_no_prefetch_queue_length = 1024 * 1024;
 /*
  * These tunables control the fill fraction of the queues by zfs send.  The fill
  * fraction controls the frequency with which threads have to be cv_signaled.
  * If a lot of cpu time is being spent on cv_signal, then these should be tuned
  * down.  If the queues empty before the signalled thread can catch up, then
  * these should be tuned up.
  */
 static uint_t zfs_send_queue_ff = 20;
 static uint_t zfs_send_no_prefetch_queue_ff = 20;
 
 /*
  * Use this to override the recordsize calculation for fast zfs send estimates.
  */
 static uint_t zfs_override_estimate_recordsize = 0;
 
 /* Set this tunable to FALSE to disable setting of DRR_FLAG_FREERECORDS */
 static const boolean_t zfs_send_set_freerecords_bit = B_TRUE;
 
 /* Set this tunable to FALSE is disable sending unmodified spill blocks. */
 static int zfs_send_unmodified_spill_blocks = B_TRUE;
 
 static inline boolean_t
 overflow_multiply(uint64_t a, uint64_t b, uint64_t *c)
 {
 	uint64_t temp = a * b;
 	if (b != 0 && temp / b != a)
 		return (B_FALSE);
 	*c = temp;
 	return (B_TRUE);
 }
 
 struct send_thread_arg {
 	bqueue_t	q;
 	objset_t	*os;		/* Objset to traverse */
 	uint64_t	fromtxg;	/* Traverse from this txg */
 	int		flags;		/* flags to pass to traverse_dataset */
 	int		error_code;
 	boolean_t	cancel;
 	zbookmark_phys_t resume;
 	uint64_t	*num_blocks_visited;
 };
 
 struct redact_list_thread_arg {
 	boolean_t		cancel;
 	bqueue_t		q;
 	zbookmark_phys_t	resume;
 	redaction_list_t	*rl;
 	boolean_t		mark_redact;
 	int			error_code;
 	uint64_t		*num_blocks_visited;
 };
 
 struct send_merge_thread_arg {
 	bqueue_t			q;
 	objset_t			*os;
 	struct redact_list_thread_arg	*from_arg;
 	struct send_thread_arg		*to_arg;
 	struct redact_list_thread_arg	*redact_arg;
 	int				error;
 	boolean_t			cancel;
 };
 
 struct send_range {
 	boolean_t		eos_marker; /* Marks the end of the stream */
 	uint64_t		object;
 	uint64_t		start_blkid;
 	uint64_t		end_blkid;
 	bqueue_node_t		ln;
 	enum type {DATA, HOLE, OBJECT, OBJECT_RANGE, REDACT,
 	    PREVIOUSLY_REDACTED} type;
 	union {
 		struct srd {
 			dmu_object_type_t	obj_type;
 			uint32_t		datablksz; // logical size
 			uint32_t		datasz; // payload size
 			blkptr_t		bp;
 			arc_buf_t		*abuf;
 			abd_t			*abd;
 			kmutex_t		lock;
 			kcondvar_t		cv;
 			boolean_t		io_outstanding;
 			boolean_t		io_compressed;
 			int			io_err;
 		} data;
 		struct srh {
 			uint32_t		datablksz;
 		} hole;
 		struct sro {
 			/*
 			 * This is a pointer because embedding it in the
 			 * struct causes these structures to be massively larger
 			 * for all range types; this makes the code much less
 			 * memory efficient.
 			 */
 			dnode_phys_t		*dnp;
 			blkptr_t		bp;
 		} object;
 		struct srr {
 			uint32_t		datablksz;
 		} redact;
 		struct sror {
 			blkptr_t		bp;
 		} object_range;
 	} sru;
 };
 
 /*
  * The list of data whose inclusion in a send stream can be pending from
  * one call to backup_cb to another.  Multiple calls to dump_free(),
  * dump_freeobjects(), and dump_redact() can be aggregated into a single
  * DRR_FREE, DRR_FREEOBJECTS, or DRR_REDACT replay record.
  */
 typedef enum {
 	PENDING_NONE,
 	PENDING_FREE,
 	PENDING_FREEOBJECTS,
 	PENDING_REDACT
 } dmu_pendop_t;
 
 typedef struct dmu_send_cookie {
 	dmu_replay_record_t *dsc_drr;
 	dmu_send_outparams_t *dsc_dso;
 	offset_t *dsc_off;
 	objset_t *dsc_os;
 	zio_cksum_t dsc_zc;
 	uint64_t dsc_toguid;
 	uint64_t dsc_fromtxg;
 	int dsc_err;
 	dmu_pendop_t dsc_pending_op;
 	uint64_t dsc_featureflags;
 	uint64_t dsc_last_data_object;
 	uint64_t dsc_last_data_offset;
 	uint64_t dsc_resume_object;
 	uint64_t dsc_resume_offset;
 	boolean_t dsc_sent_begin;
 	boolean_t dsc_sent_end;
 } dmu_send_cookie_t;
 
 static int do_dump(dmu_send_cookie_t *dscp, struct send_range *range);
 
 static void
 range_free(struct send_range *range)
 {
 	if (range->type == OBJECT) {
 		size_t size = sizeof (dnode_phys_t) *
 		    (range->sru.object.dnp->dn_extra_slots + 1);
 		kmem_free(range->sru.object.dnp, size);
 	} else if (range->type == DATA) {
 		mutex_enter(&range->sru.data.lock);
 		while (range->sru.data.io_outstanding)
 			cv_wait(&range->sru.data.cv, &range->sru.data.lock);
 		if (range->sru.data.abd != NULL)
 			abd_free(range->sru.data.abd);
 		if (range->sru.data.abuf != NULL) {
 			arc_buf_destroy(range->sru.data.abuf,
 			    &range->sru.data.abuf);
 		}
 		mutex_exit(&range->sru.data.lock);
 
 		cv_destroy(&range->sru.data.cv);
 		mutex_destroy(&range->sru.data.lock);
 	}
 	kmem_free(range, sizeof (*range));
 }
 
 /*
  * For all record types except BEGIN, fill in the checksum (overlaid in
  * drr_u.drr_checksum.drr_checksum).  The checksum verifies everything
  * up to the start of the checksum itself.
  */
 static int
 dump_record(dmu_send_cookie_t *dscp, void *payload, int payload_len)
 {
 	dmu_send_outparams_t *dso = dscp->dsc_dso;
 	ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
 	    ==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
 	(void) fletcher_4_incremental_native(dscp->dsc_drr,
 	    offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
 	    &dscp->dsc_zc);
 	if (dscp->dsc_drr->drr_type == DRR_BEGIN) {
 		dscp->dsc_sent_begin = B_TRUE;
 	} else {
 		ASSERT(ZIO_CHECKSUM_IS_ZERO(&dscp->dsc_drr->drr_u.
 		    drr_checksum.drr_checksum));
 		dscp->dsc_drr->drr_u.drr_checksum.drr_checksum = dscp->dsc_zc;
 	}
 	if (dscp->dsc_drr->drr_type == DRR_END) {
 		dscp->dsc_sent_end = B_TRUE;
 	}
 	(void) fletcher_4_incremental_native(&dscp->dsc_drr->
 	    drr_u.drr_checksum.drr_checksum,
 	    sizeof (zio_cksum_t), &dscp->dsc_zc);
 	*dscp->dsc_off += sizeof (dmu_replay_record_t);
 	dscp->dsc_err = dso->dso_outfunc(dscp->dsc_os, dscp->dsc_drr,
 	    sizeof (dmu_replay_record_t), dso->dso_arg);
 	if (dscp->dsc_err != 0)
 		return (SET_ERROR(EINTR));
 	if (payload_len != 0) {
 		*dscp->dsc_off += payload_len;
 		/*
 		 * payload is null when dso_dryrun == B_TRUE (i.e. when we're
 		 * doing a send size calculation)
 		 */
 		if (payload != NULL) {
 			(void) fletcher_4_incremental_native(
 			    payload, payload_len, &dscp->dsc_zc);
 		}
 
 		/*
 		 * The code does not rely on this (len being a multiple of 8).
 		 * We keep this assertion because of the corresponding assertion
 		 * in receive_read().  Keeping this assertion ensures that we do
 		 * not inadvertently break backwards compatibility (causing the
 		 * assertion in receive_read() to trigger on old software).
 		 *
 		 * Raw sends cannot be received on old software, and so can
 		 * bypass this assertion.
 		 */
 
 		ASSERT((payload_len % 8 == 0) ||
 		    (dscp->dsc_featureflags & DMU_BACKUP_FEATURE_RAW));
 
 		dscp->dsc_err = dso->dso_outfunc(dscp->dsc_os, payload,
 		    payload_len, dso->dso_arg);
 		if (dscp->dsc_err != 0)
 			return (SET_ERROR(EINTR));
 	}
 	return (0);
 }
 
 /*
  * Fill in the drr_free struct, or perform aggregation if the previous record is
  * also a free record, and the two are adjacent.
  *
  * Note that we send free records even for a full send, because we want to be
  * able to receive a full send as a clone, which requires a list of all the free
  * and freeobject records that were generated on the source.
  */
 static int
 dump_free(dmu_send_cookie_t *dscp, uint64_t object, uint64_t offset,
     uint64_t length)
 {
 	struct drr_free *drrf = &(dscp->dsc_drr->drr_u.drr_free);
 
 	/*
 	 * When we receive a free record, dbuf_free_range() assumes
 	 * that the receiving system doesn't have any dbufs in the range
 	 * being freed.  This is always true because there is a one-record
 	 * constraint: we only send one WRITE record for any given
 	 * object,offset.  We know that the one-record constraint is
 	 * true because we always send data in increasing order by
 	 * object,offset.
 	 *
 	 * If the increasing-order constraint ever changes, we should find
 	 * another way to assert that the one-record constraint is still
 	 * satisfied.
 	 */
 	ASSERT(object > dscp->dsc_last_data_object ||
 	    (object == dscp->dsc_last_data_object &&
 	    offset > dscp->dsc_last_data_offset));
 
 	/*
 	 * If there is a pending op, but it's not PENDING_FREE, push it out,
 	 * since free block aggregation can only be done for blocks of the
 	 * same type (i.e., DRR_FREE records can only be aggregated with
 	 * other DRR_FREE records.  DRR_FREEOBJECTS records can only be
 	 * aggregated with other DRR_FREEOBJECTS records).
 	 */
 	if (dscp->dsc_pending_op != PENDING_NONE &&
 	    dscp->dsc_pending_op != PENDING_FREE) {
 		if (dump_record(dscp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dscp->dsc_pending_op = PENDING_NONE;
 	}
 
 	if (dscp->dsc_pending_op == PENDING_FREE) {
 		/*
 		 * Check to see whether this free block can be aggregated
 		 * with pending one.
 		 */
 		if (drrf->drr_object == object && drrf->drr_offset +
 		    drrf->drr_length == offset) {
 			if (offset + length < offset || length == UINT64_MAX)
 				drrf->drr_length = UINT64_MAX;
 			else
 				drrf->drr_length += length;
 			return (0);
 		} else {
 			/* not a continuation.  Push out pending record */
 			if (dump_record(dscp, NULL, 0) != 0)
 				return (SET_ERROR(EINTR));
 			dscp->dsc_pending_op = PENDING_NONE;
 		}
 	}
 	/* create a FREE record and make it pending */
 	memset(dscp->dsc_drr, 0, sizeof (dmu_replay_record_t));
 	dscp->dsc_drr->drr_type = DRR_FREE;
 	drrf->drr_object = object;
 	drrf->drr_offset = offset;
 	if (offset + length < offset)
 		drrf->drr_length = DMU_OBJECT_END;
 	else
 		drrf->drr_length = length;
 	drrf->drr_toguid = dscp->dsc_toguid;
 	if (length == DMU_OBJECT_END) {
 		if (dump_record(dscp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 	} else {
 		dscp->dsc_pending_op = PENDING_FREE;
 	}
 
 	return (0);
 }
 
 /*
  * Fill in the drr_redact struct, or perform aggregation if the previous record
  * is also a redaction record, and the two are adjacent.
  */
 static int
 dump_redact(dmu_send_cookie_t *dscp, uint64_t object, uint64_t offset,
     uint64_t length)
 {
 	struct drr_redact *drrr = &dscp->dsc_drr->drr_u.drr_redact;
 
 	/*
 	 * If there is a pending op, but it's not PENDING_REDACT, push it out,
 	 * since free block aggregation can only be done for blocks of the
 	 * same type (i.e., DRR_REDACT records can only be aggregated with
 	 * other DRR_REDACT records).
 	 */
 	if (dscp->dsc_pending_op != PENDING_NONE &&
 	    dscp->dsc_pending_op != PENDING_REDACT) {
 		if (dump_record(dscp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dscp->dsc_pending_op = PENDING_NONE;
 	}
 
 	if (dscp->dsc_pending_op == PENDING_REDACT) {
 		/*
 		 * Check to see whether this redacted block can be aggregated
 		 * with pending one.
 		 */
 		if (drrr->drr_object == object && drrr->drr_offset +
 		    drrr->drr_length == offset) {
 			drrr->drr_length += length;
 			return (0);
 		} else {
 			/* not a continuation.  Push out pending record */
 			if (dump_record(dscp, NULL, 0) != 0)
 				return (SET_ERROR(EINTR));
 			dscp->dsc_pending_op = PENDING_NONE;
 		}
 	}
 	/* create a REDACT record and make it pending */
 	memset(dscp->dsc_drr, 0, sizeof (dmu_replay_record_t));
 	dscp->dsc_drr->drr_type = DRR_REDACT;
 	drrr->drr_object = object;
 	drrr->drr_offset = offset;
 	drrr->drr_length = length;
 	drrr->drr_toguid = dscp->dsc_toguid;
 	dscp->dsc_pending_op = PENDING_REDACT;
 
 	return (0);
 }
 
 static int
 dmu_dump_write(dmu_send_cookie_t *dscp, dmu_object_type_t type, uint64_t object,
     uint64_t offset, int lsize, int psize, const blkptr_t *bp,
     boolean_t io_compressed, void *data)
 {
 	uint64_t payload_size;
 	boolean_t raw = (dscp->dsc_featureflags & DMU_BACKUP_FEATURE_RAW);
 	struct drr_write *drrw = &(dscp->dsc_drr->drr_u.drr_write);
 
 	/*
 	 * We send data in increasing object, offset order.
 	 * See comment in dump_free() for details.
 	 */
 	ASSERT(object > dscp->dsc_last_data_object ||
 	    (object == dscp->dsc_last_data_object &&
 	    offset > dscp->dsc_last_data_offset));
 	dscp->dsc_last_data_object = object;
 	dscp->dsc_last_data_offset = offset + lsize - 1;
 
 	/*
 	 * If there is any kind of pending aggregation (currently either
 	 * a grouping of free objects or free blocks), push it out to
 	 * the stream, since aggregation can't be done across operations
 	 * of different types.
 	 */
 	if (dscp->dsc_pending_op != PENDING_NONE) {
 		if (dump_record(dscp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dscp->dsc_pending_op = PENDING_NONE;
 	}
 	/* write a WRITE record */
 	memset(dscp->dsc_drr, 0, sizeof (dmu_replay_record_t));
 	dscp->dsc_drr->drr_type = DRR_WRITE;
 	drrw->drr_object = object;
 	drrw->drr_type = type;
 	drrw->drr_offset = offset;
 	drrw->drr_toguid = dscp->dsc_toguid;
 	drrw->drr_logical_size = lsize;
 
 	/* only set the compression fields if the buf is compressed or raw */
 	boolean_t compressed =
 	    (bp != NULL ? BP_GET_COMPRESS(bp) != ZIO_COMPRESS_OFF &&
 	    io_compressed : lsize != psize);
 	if (raw || compressed) {
 		ASSERT(bp != NULL);
 		ASSERT(raw || dscp->dsc_featureflags &
 		    DMU_BACKUP_FEATURE_COMPRESSED);
 		ASSERT(!BP_IS_EMBEDDED(bp));
 		ASSERT3S(psize, >, 0);
 
 		if (raw) {
 			ASSERT(BP_IS_PROTECTED(bp));
 
 			/*
 			 * This is a raw protected block so we need to pass
 			 * along everything the receiving side will need to
 			 * interpret this block, including the byteswap, salt,
 			 * IV, and MAC.
 			 */
 			if (BP_SHOULD_BYTESWAP(bp))
 				drrw->drr_flags |= DRR_RAW_BYTESWAP;
 			zio_crypt_decode_params_bp(bp, drrw->drr_salt,
 			    drrw->drr_iv);
 			zio_crypt_decode_mac_bp(bp, drrw->drr_mac);
 		} else {
 			/* this is a compressed block */
 			ASSERT(dscp->dsc_featureflags &
 			    DMU_BACKUP_FEATURE_COMPRESSED);
 			ASSERT(!BP_SHOULD_BYTESWAP(bp));
 			ASSERT(!DMU_OT_IS_METADATA(BP_GET_TYPE(bp)));
 			ASSERT3U(BP_GET_COMPRESS(bp), !=, ZIO_COMPRESS_OFF);
 			ASSERT3S(lsize, >=, psize);
 		}
 
 		/* set fields common to compressed and raw sends */
 		drrw->drr_compressiontype = BP_GET_COMPRESS(bp);
 		drrw->drr_compressed_size = psize;
 		payload_size = drrw->drr_compressed_size;
 	} else {
 		payload_size = drrw->drr_logical_size;
 	}
 
 	if (bp == NULL || BP_IS_EMBEDDED(bp) || (BP_IS_PROTECTED(bp) && !raw)) {
 		/*
 		 * There's no pre-computed checksum for partial-block writes,
 		 * embedded BP's, or encrypted BP's that are being sent as
 		 * plaintext, so (like fletcher4-checksummed blocks) userland
 		 * will have to compute a dedup-capable checksum itself.
 		 */
 		drrw->drr_checksumtype = ZIO_CHECKSUM_OFF;
 	} else {
 		drrw->drr_checksumtype = BP_GET_CHECKSUM(bp);
 		if (zio_checksum_table[drrw->drr_checksumtype].ci_flags &
 		    ZCHECKSUM_FLAG_DEDUP)
 			drrw->drr_flags |= DRR_CHECKSUM_DEDUP;
 		DDK_SET_LSIZE(&drrw->drr_key, BP_GET_LSIZE(bp));
 		DDK_SET_PSIZE(&drrw->drr_key, BP_GET_PSIZE(bp));
 		DDK_SET_COMPRESS(&drrw->drr_key, BP_GET_COMPRESS(bp));
 		DDK_SET_CRYPT(&drrw->drr_key, BP_IS_PROTECTED(bp));
 		drrw->drr_key.ddk_cksum = bp->blk_cksum;
 	}
 
 	if (dump_record(dscp, data, payload_size) != 0)
 		return (SET_ERROR(EINTR));
 	return (0);
 }
 
 static int
 dump_write_embedded(dmu_send_cookie_t *dscp, uint64_t object, uint64_t offset,
     int blksz, const blkptr_t *bp)
 {
 	char buf[BPE_PAYLOAD_SIZE];
 	struct drr_write_embedded *drrw =
 	    &(dscp->dsc_drr->drr_u.drr_write_embedded);
 
 	if (dscp->dsc_pending_op != PENDING_NONE) {
 		if (dump_record(dscp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dscp->dsc_pending_op = PENDING_NONE;
 	}
 
 	ASSERT(BP_IS_EMBEDDED(bp));
 
 	memset(dscp->dsc_drr, 0, sizeof (dmu_replay_record_t));
 	dscp->dsc_drr->drr_type = DRR_WRITE_EMBEDDED;
 	drrw->drr_object = object;
 	drrw->drr_offset = offset;
 	drrw->drr_length = blksz;
 	drrw->drr_toguid = dscp->dsc_toguid;
 	drrw->drr_compression = BP_GET_COMPRESS(bp);
 	drrw->drr_etype = BPE_GET_ETYPE(bp);
 	drrw->drr_lsize = BPE_GET_LSIZE(bp);
 	drrw->drr_psize = BPE_GET_PSIZE(bp);
 
 	decode_embedded_bp_compressed(bp, buf);
 
 	uint32_t psize = drrw->drr_psize;
 	uint32_t rsize = P2ROUNDUP(psize, 8);
 
 	if (psize != rsize)
 		memset(buf + psize, 0, rsize - psize);
 
 	if (dump_record(dscp, buf, rsize) != 0)
 		return (SET_ERROR(EINTR));
 	return (0);
 }
 
 static int
 dump_spill(dmu_send_cookie_t *dscp, const blkptr_t *bp, uint64_t object,
     void *data)
 {
 	struct drr_spill *drrs = &(dscp->dsc_drr->drr_u.drr_spill);
 	uint64_t blksz = BP_GET_LSIZE(bp);
 	uint64_t payload_size = blksz;
 
 	if (dscp->dsc_pending_op != PENDING_NONE) {
 		if (dump_record(dscp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dscp->dsc_pending_op = PENDING_NONE;
 	}
 
 	/* write a SPILL record */
 	memset(dscp->dsc_drr, 0, sizeof (dmu_replay_record_t));
 	dscp->dsc_drr->drr_type = DRR_SPILL;
 	drrs->drr_object = object;
 	drrs->drr_length = blksz;
 	drrs->drr_toguid = dscp->dsc_toguid;
 
 	/* See comment in dump_dnode() for full details */
 	if (zfs_send_unmodified_spill_blocks &&
 	    (bp->blk_birth <= dscp->dsc_fromtxg)) {
 		drrs->drr_flags |= DRR_SPILL_UNMODIFIED;
 	}
 
 	/* handle raw send fields */
 	if (dscp->dsc_featureflags & DMU_BACKUP_FEATURE_RAW) {
 		ASSERT(BP_IS_PROTECTED(bp));
 
 		if (BP_SHOULD_BYTESWAP(bp))
 			drrs->drr_flags |= DRR_RAW_BYTESWAP;
 		drrs->drr_compressiontype = BP_GET_COMPRESS(bp);
 		drrs->drr_compressed_size = BP_GET_PSIZE(bp);
 		zio_crypt_decode_params_bp(bp, drrs->drr_salt, drrs->drr_iv);
 		zio_crypt_decode_mac_bp(bp, drrs->drr_mac);
 		payload_size = drrs->drr_compressed_size;
 	}
 
 	if (dump_record(dscp, data, payload_size) != 0)
 		return (SET_ERROR(EINTR));
 	return (0);
 }
 
 static int
 dump_freeobjects(dmu_send_cookie_t *dscp, uint64_t firstobj, uint64_t numobjs)
 {
 	struct drr_freeobjects *drrfo = &(dscp->dsc_drr->drr_u.drr_freeobjects);
 	uint64_t maxobj = DNODES_PER_BLOCK *
 	    (DMU_META_DNODE(dscp->dsc_os)->dn_maxblkid + 1);
 
 	/*
 	 * ZoL < 0.7 does not handle large FREEOBJECTS records correctly,
 	 * leading to zfs recv never completing. to avoid this issue, don't
 	 * send FREEOBJECTS records for object IDs which cannot exist on the
 	 * receiving side.
 	 */
 	if (maxobj > 0) {
 		if (maxobj <= firstobj)
 			return (0);
 
 		if (maxobj < firstobj + numobjs)
 			numobjs = maxobj - firstobj;
 	}
 
 	/*
 	 * If there is a pending op, but it's not PENDING_FREEOBJECTS,
 	 * push it out, since free block aggregation can only be done for
 	 * blocks of the same type (i.e., DRR_FREE records can only be
 	 * aggregated with other DRR_FREE records.  DRR_FREEOBJECTS records
 	 * can only be aggregated with other DRR_FREEOBJECTS records).
 	 */
 	if (dscp->dsc_pending_op != PENDING_NONE &&
 	    dscp->dsc_pending_op != PENDING_FREEOBJECTS) {
 		if (dump_record(dscp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dscp->dsc_pending_op = PENDING_NONE;
 	}
 
 	if (dscp->dsc_pending_op == PENDING_FREEOBJECTS) {
 		/*
 		 * See whether this free object array can be aggregated
 		 * with pending one
 		 */
 		if (drrfo->drr_firstobj + drrfo->drr_numobjs == firstobj) {
 			drrfo->drr_numobjs += numobjs;
 			return (0);
 		} else {
 			/* can't be aggregated.  Push out pending record */
 			if (dump_record(dscp, NULL, 0) != 0)
 				return (SET_ERROR(EINTR));
 			dscp->dsc_pending_op = PENDING_NONE;
 		}
 	}
 
 	/* write a FREEOBJECTS record */
 	memset(dscp->dsc_drr, 0, sizeof (dmu_replay_record_t));
 	dscp->dsc_drr->drr_type = DRR_FREEOBJECTS;
 	drrfo->drr_firstobj = firstobj;
 	drrfo->drr_numobjs = numobjs;
 	drrfo->drr_toguid = dscp->dsc_toguid;
 
 	dscp->dsc_pending_op = PENDING_FREEOBJECTS;
 
 	return (0);
 }
 
 static int
 dump_dnode(dmu_send_cookie_t *dscp, const blkptr_t *bp, uint64_t object,
     dnode_phys_t *dnp)
 {
 	struct drr_object *drro = &(dscp->dsc_drr->drr_u.drr_object);
 	int bonuslen;
 
 	if (object < dscp->dsc_resume_object) {
 		/*
 		 * Note: when resuming, we will visit all the dnodes in
 		 * the block of dnodes that we are resuming from.  In
 		 * this case it's unnecessary to send the dnodes prior to
 		 * the one we are resuming from.  We should be at most one
 		 * block's worth of dnodes behind the resume point.
 		 */
 		ASSERT3U(dscp->dsc_resume_object - object, <,
 		    1 << (DNODE_BLOCK_SHIFT - DNODE_SHIFT));
 		return (0);
 	}
 
 	if (dnp == NULL || dnp->dn_type == DMU_OT_NONE)
 		return (dump_freeobjects(dscp, object, 1));
 
 	if (dscp->dsc_pending_op != PENDING_NONE) {
 		if (dump_record(dscp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dscp->dsc_pending_op = PENDING_NONE;
 	}
 
 	/* write an OBJECT record */
 	memset(dscp->dsc_drr, 0, sizeof (dmu_replay_record_t));
 	dscp->dsc_drr->drr_type = DRR_OBJECT;
 	drro->drr_object = object;
 	drro->drr_type = dnp->dn_type;
 	drro->drr_bonustype = dnp->dn_bonustype;
 	drro->drr_blksz = dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT;
 	drro->drr_bonuslen = dnp->dn_bonuslen;
 	drro->drr_dn_slots = dnp->dn_extra_slots + 1;
 	drro->drr_checksumtype = dnp->dn_checksum;
 	drro->drr_compress = dnp->dn_compress;
 	drro->drr_toguid = dscp->dsc_toguid;
 
 	if (!(dscp->dsc_featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS) &&
 	    drro->drr_blksz > SPA_OLD_MAXBLOCKSIZE)
 		drro->drr_blksz = SPA_OLD_MAXBLOCKSIZE;
 
 	bonuslen = P2ROUNDUP(dnp->dn_bonuslen, 8);
 
 	if ((dscp->dsc_featureflags & DMU_BACKUP_FEATURE_RAW)) {
 		ASSERT(BP_IS_ENCRYPTED(bp));
 
 		if (BP_SHOULD_BYTESWAP(bp))
 			drro->drr_flags |= DRR_RAW_BYTESWAP;
 
 		/* needed for reconstructing dnp on recv side */
 		drro->drr_maxblkid = dnp->dn_maxblkid;
 		drro->drr_indblkshift = dnp->dn_indblkshift;
 		drro->drr_nlevels = dnp->dn_nlevels;
 		drro->drr_nblkptr = dnp->dn_nblkptr;
 
 		/*
 		 * Since we encrypt the entire bonus area, the (raw) part
 		 * beyond the bonuslen is actually nonzero, so we need
 		 * to send it.
 		 */
 		if (bonuslen != 0) {
 			if (drro->drr_bonuslen > DN_MAX_BONUS_LEN(dnp))
 				return (SET_ERROR(EINVAL));
 			drro->drr_raw_bonuslen = DN_MAX_BONUS_LEN(dnp);
 			bonuslen = drro->drr_raw_bonuslen;
 		}
 	}
 
 	/*
 	 * DRR_OBJECT_SPILL is set for every dnode which references a
 	 * spill block.	 This allows the receiving pool to definitively
 	 * determine when a spill block should be kept or freed.
 	 */
 	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)
 		drro->drr_flags |= DRR_OBJECT_SPILL;
 
 	if (dump_record(dscp, DN_BONUS(dnp), bonuslen) != 0)
 		return (SET_ERROR(EINTR));
 
 	/* Free anything past the end of the file. */
 	if (dump_free(dscp, object, (dnp->dn_maxblkid + 1) *
 	    (dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT), DMU_OBJECT_END) != 0)
 		return (SET_ERROR(EINTR));
 
 	/*
 	 * Send DRR_SPILL records for unmodified spill blocks.	This is useful
 	 * because changing certain attributes of the object (e.g. blocksize)
 	 * can cause old versions of ZFS to incorrectly remove a spill block.
 	 * Including these records in the stream forces an up to date version
 	 * to always be written ensuring they're never lost.  Current versions
 	 * of the code which understand the DRR_FLAG_SPILL_BLOCK feature can
 	 * ignore these unmodified spill blocks.
 	 */
 	if (zfs_send_unmodified_spill_blocks &&
 	    (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) &&
 	    (DN_SPILL_BLKPTR(dnp)->blk_birth <= dscp->dsc_fromtxg)) {
 		struct send_range record;
 		blkptr_t *bp = DN_SPILL_BLKPTR(dnp);
 
 		memset(&record, 0, sizeof (struct send_range));
 		record.type = DATA;
 		record.object = object;
 		record.eos_marker = B_FALSE;
 		record.start_blkid = DMU_SPILL_BLKID;
 		record.end_blkid = record.start_blkid + 1;
 		record.sru.data.bp = *bp;
 		record.sru.data.obj_type = dnp->dn_type;
 		record.sru.data.datablksz = BP_GET_LSIZE(bp);
 
 		if (do_dump(dscp, &record) != 0)
 			return (SET_ERROR(EINTR));
 	}
 
 	if (dscp->dsc_err != 0)
 		return (SET_ERROR(EINTR));
 
 	return (0);
 }
 
 static int
 dump_object_range(dmu_send_cookie_t *dscp, const blkptr_t *bp,
     uint64_t firstobj, uint64_t numslots)
 {
 	struct drr_object_range *drror =
 	    &(dscp->dsc_drr->drr_u.drr_object_range);
 
 	/* we only use this record type for raw sends */
 	ASSERT(BP_IS_PROTECTED(bp));
 	ASSERT(dscp->dsc_featureflags & DMU_BACKUP_FEATURE_RAW);
 	ASSERT3U(BP_GET_COMPRESS(bp), ==, ZIO_COMPRESS_OFF);
 	ASSERT3U(BP_GET_TYPE(bp), ==, DMU_OT_DNODE);
 	ASSERT0(BP_GET_LEVEL(bp));
 
 	if (dscp->dsc_pending_op != PENDING_NONE) {
 		if (dump_record(dscp, NULL, 0) != 0)
 			return (SET_ERROR(EINTR));
 		dscp->dsc_pending_op = PENDING_NONE;
 	}
 
 	memset(dscp->dsc_drr, 0, sizeof (dmu_replay_record_t));
 	dscp->dsc_drr->drr_type = DRR_OBJECT_RANGE;
 	drror->drr_firstobj = firstobj;
 	drror->drr_numslots = numslots;
 	drror->drr_toguid = dscp->dsc_toguid;
 	if (BP_SHOULD_BYTESWAP(bp))
 		drror->drr_flags |= DRR_RAW_BYTESWAP;
 	zio_crypt_decode_params_bp(bp, drror->drr_salt, drror->drr_iv);
 	zio_crypt_decode_mac_bp(bp, drror->drr_mac);
 
 	if (dump_record(dscp, NULL, 0) != 0)
 		return (SET_ERROR(EINTR));
 	return (0);
 }
 
 static boolean_t
 send_do_embed(const blkptr_t *bp, uint64_t featureflags)
 {
 	if (!BP_IS_EMBEDDED(bp))
 		return (B_FALSE);
 
 	/*
 	 * Compression function must be legacy, or explicitly enabled.
 	 */
 	if ((BP_GET_COMPRESS(bp) >= ZIO_COMPRESS_LEGACY_FUNCTIONS &&
 	    !(featureflags & DMU_BACKUP_FEATURE_LZ4)))
 		return (B_FALSE);
 
 	/*
 	 * If we have not set the ZSTD feature flag, we can't send ZSTD
 	 * compressed embedded blocks, as the receiver may not support them.
 	 */
 	if ((BP_GET_COMPRESS(bp) == ZIO_COMPRESS_ZSTD &&
 	    !(featureflags & DMU_BACKUP_FEATURE_ZSTD)))
 		return (B_FALSE);
 
 	/*
 	 * Embed type must be explicitly enabled.
 	 */
 	switch (BPE_GET_ETYPE(bp)) {
 	case BP_EMBEDDED_TYPE_DATA:
 		if (featureflags & DMU_BACKUP_FEATURE_EMBED_DATA)
 			return (B_TRUE);
 		break;
 	default:
 		return (B_FALSE);
 	}
 	return (B_FALSE);
 }
 
 /*
  * This function actually handles figuring out what kind of record needs to be
  * dumped, and calling the appropriate helper function.  In most cases,
  * the data has already been read by send_reader_thread().
  */
 static int
 do_dump(dmu_send_cookie_t *dscp, struct send_range *range)
 {
 	int err = 0;
 	switch (range->type) {
 	case OBJECT:
 		err = dump_dnode(dscp, &range->sru.object.bp, range->object,
 		    range->sru.object.dnp);
 		return (err);
 	case OBJECT_RANGE: {
 		ASSERT3U(range->start_blkid + 1, ==, range->end_blkid);
 		if (!(dscp->dsc_featureflags & DMU_BACKUP_FEATURE_RAW)) {
 			return (0);
 		}
 		uint64_t epb = BP_GET_LSIZE(&range->sru.object_range.bp) >>
 		    DNODE_SHIFT;
 		uint64_t firstobj = range->start_blkid * epb;
 		err = dump_object_range(dscp, &range->sru.object_range.bp,
 		    firstobj, epb);
 		break;
 	}
 	case REDACT: {
 		struct srr *srrp = &range->sru.redact;
 		err = dump_redact(dscp, range->object, range->start_blkid *
 		    srrp->datablksz, (range->end_blkid - range->start_blkid) *
 		    srrp->datablksz);
 		return (err);
 	}
 	case DATA: {
 		struct srd *srdp = &range->sru.data;
 		blkptr_t *bp = &srdp->bp;
 		spa_t *spa =
 		    dmu_objset_spa(dscp->dsc_os);
 
 		ASSERT3U(srdp->datablksz, ==, BP_GET_LSIZE(bp));
 		ASSERT3U(range->start_blkid + 1, ==, range->end_blkid);
 		if (BP_GET_TYPE(bp) == DMU_OT_SA) {
 			arc_flags_t aflags = ARC_FLAG_WAIT;
 			zio_flag_t zioflags = ZIO_FLAG_CANFAIL;
 
 			if (dscp->dsc_featureflags & DMU_BACKUP_FEATURE_RAW) {
 				ASSERT(BP_IS_PROTECTED(bp));
 				zioflags |= ZIO_FLAG_RAW;
 			}
 
 			zbookmark_phys_t zb;
 			ASSERT3U(range->start_blkid, ==, DMU_SPILL_BLKID);
 			zb.zb_objset = dmu_objset_id(dscp->dsc_os);
 			zb.zb_object = range->object;
 			zb.zb_level = 0;
 			zb.zb_blkid = range->start_blkid;
 
 			arc_buf_t *abuf = NULL;
 			if (!dscp->dsc_dso->dso_dryrun && arc_read(NULL, spa,
 			    bp, arc_getbuf_func, &abuf, ZIO_PRIORITY_ASYNC_READ,
 			    zioflags, &aflags, &zb) != 0)
 				return (SET_ERROR(EIO));
 
 			err = dump_spill(dscp, bp, zb.zb_object,
 			    (abuf == NULL ? NULL : abuf->b_data));
 			if (abuf != NULL)
 				arc_buf_destroy(abuf, &abuf);
 			return (err);
 		}
 		if (send_do_embed(bp, dscp->dsc_featureflags)) {
 			err = dump_write_embedded(dscp, range->object,
 			    range->start_blkid * srdp->datablksz,
 			    srdp->datablksz, bp);
 			return (err);
 		}
 		ASSERT(range->object > dscp->dsc_resume_object ||
 		    (range->object == dscp->dsc_resume_object &&
 		    range->start_blkid * srdp->datablksz >=
 		    dscp->dsc_resume_offset));
 		/* it's a level-0 block of a regular object */
 
 		mutex_enter(&srdp->lock);
 		while (srdp->io_outstanding)
 			cv_wait(&srdp->cv, &srdp->lock);
 		err = srdp->io_err;
 		mutex_exit(&srdp->lock);
 
 		if (err != 0) {
 			if (zfs_send_corrupt_data &&
 			    !dscp->dsc_dso->dso_dryrun) {
 				/*
 				 * Send a block filled with 0x"zfs badd bloc"
 				 */
 				srdp->abuf = arc_alloc_buf(spa, &srdp->abuf,
 				    ARC_BUFC_DATA, srdp->datablksz);
 				uint64_t *ptr;
 				for (ptr = srdp->abuf->b_data;
 				    (char *)ptr < (char *)srdp->abuf->b_data +
 				    srdp->datablksz; ptr++)
 					*ptr = 0x2f5baddb10cULL;
 			} else {
 				return (SET_ERROR(EIO));
 			}
 		}
 
 		ASSERT(dscp->dsc_dso->dso_dryrun ||
 		    srdp->abuf != NULL || srdp->abd != NULL);
 
 		uint64_t offset = range->start_blkid * srdp->datablksz;
 
 		char *data = NULL;
 		if (srdp->abd != NULL) {
 			data = abd_to_buf(srdp->abd);
 			ASSERT3P(srdp->abuf, ==, NULL);
 		} else if (srdp->abuf != NULL) {
 			data = srdp->abuf->b_data;
 		}
 
 		/*
 		 * If we have large blocks stored on disk but the send flags
 		 * don't allow us to send large blocks, we split the data from
 		 * the arc buf into chunks.
 		 */
 		if (srdp->datablksz > SPA_OLD_MAXBLOCKSIZE &&
 		    !(dscp->dsc_featureflags &
 		    DMU_BACKUP_FEATURE_LARGE_BLOCKS)) {
 			while (srdp->datablksz > 0 && err == 0) {
 				int n = MIN(srdp->datablksz,
 				    SPA_OLD_MAXBLOCKSIZE);
 				err = dmu_dump_write(dscp, srdp->obj_type,
 				    range->object, offset, n, n, NULL, B_FALSE,
 				    data);
 				offset += n;
 				/*
 				 * When doing dry run, data==NULL is used as a
 				 * sentinel value by
 				 * dmu_dump_write()->dump_record().
 				 */
 				if (data != NULL)
 					data += n;
 				srdp->datablksz -= n;
 			}
 		} else {
 			err = dmu_dump_write(dscp, srdp->obj_type,
 			    range->object, offset,
 			    srdp->datablksz, srdp->datasz, bp,
 			    srdp->io_compressed, data);
 		}
 		return (err);
 	}
 	case HOLE: {
 		struct srh *srhp = &range->sru.hole;
 		if (range->object == DMU_META_DNODE_OBJECT) {
 			uint32_t span = srhp->datablksz >> DNODE_SHIFT;
 			uint64_t first_obj = range->start_blkid * span;
 			uint64_t numobj = range->end_blkid * span - first_obj;
 			return (dump_freeobjects(dscp, first_obj, numobj));
 		}
 		uint64_t offset = 0;
 
 		/*
 		 * If this multiply overflows, we don't need to send this block.
 		 * Even if it has a birth time, it can never not be a hole, so
 		 * we don't need to send records for it.
 		 */
 		if (!overflow_multiply(range->start_blkid, srhp->datablksz,
 		    &offset)) {
 			return (0);
 		}
 		uint64_t len = 0;
 
 		if (!overflow_multiply(range->end_blkid, srhp->datablksz, &len))
 			len = UINT64_MAX;
 		len = len - offset;
 		return (dump_free(dscp, range->object, offset, len));
 	}
 	default:
 		panic("Invalid range type in do_dump: %d", range->type);
 	}
 	return (err);
 }
 
 static struct send_range *
 range_alloc(enum type type, uint64_t object, uint64_t start_blkid,
     uint64_t end_blkid, boolean_t eos)
 {
 	struct send_range *range = kmem_alloc(sizeof (*range), KM_SLEEP);
 	range->type = type;
 	range->object = object;
 	range->start_blkid = start_blkid;
 	range->end_blkid = end_blkid;
 	range->eos_marker = eos;
 	if (type == DATA) {
 		range->sru.data.abd = NULL;
 		range->sru.data.abuf = NULL;
 		mutex_init(&range->sru.data.lock, NULL, MUTEX_DEFAULT, NULL);
 		cv_init(&range->sru.data.cv, NULL, CV_DEFAULT, NULL);
 		range->sru.data.io_outstanding = 0;
 		range->sru.data.io_err = 0;
 		range->sru.data.io_compressed = B_FALSE;
 	}
 	return (range);
 }
 
 /*
  * This is the callback function to traverse_dataset that acts as a worker
  * thread for dmu_send_impl.
  */
 static int
 send_cb(spa_t *spa, zilog_t *zilog, const blkptr_t *bp,
     const zbookmark_phys_t *zb, const struct dnode_phys *dnp, void *arg)
 {
 	(void) zilog;
 	struct send_thread_arg *sta = arg;
 	struct send_range *record;
 
 	ASSERT(zb->zb_object == DMU_META_DNODE_OBJECT ||
 	    zb->zb_object >= sta->resume.zb_object);
 
 	/*
 	 * All bps of an encrypted os should have the encryption bit set.
 	 * If this is not true it indicates tampering and we report an error.
 	 */
 	if (sta->os->os_encrypted &&
 	    !BP_IS_HOLE(bp) && !BP_USES_CRYPT(bp)) {
 		spa_log_error(spa, zb, &bp->blk_birth);
 		return (SET_ERROR(EIO));
 	}
 
 	if (sta->cancel)
 		return (SET_ERROR(EINTR));
 	if (zb->zb_object != DMU_META_DNODE_OBJECT &&
 	    DMU_OBJECT_IS_SPECIAL(zb->zb_object))
 		return (0);
 	atomic_inc_64(sta->num_blocks_visited);
 
 	if (zb->zb_level == ZB_DNODE_LEVEL) {
 		if (zb->zb_object == DMU_META_DNODE_OBJECT)
 			return (0);
 		record = range_alloc(OBJECT, zb->zb_object, 0, 0, B_FALSE);
 		record->sru.object.bp = *bp;
 		size_t size  = sizeof (*dnp) * (dnp->dn_extra_slots + 1);
 		record->sru.object.dnp = kmem_alloc(size, KM_SLEEP);
 		memcpy(record->sru.object.dnp, dnp, size);
 		bqueue_enqueue(&sta->q, record, sizeof (*record));
 		return (0);
 	}
 	if (zb->zb_level == 0 && zb->zb_object == DMU_META_DNODE_OBJECT &&
 	    !BP_IS_HOLE(bp)) {
 		record = range_alloc(OBJECT_RANGE, 0, zb->zb_blkid,
 		    zb->zb_blkid + 1, B_FALSE);
 		record->sru.object_range.bp = *bp;
 		bqueue_enqueue(&sta->q, record, sizeof (*record));
 		return (0);
 	}
 	if (zb->zb_level < 0 || (zb->zb_level > 0 && !BP_IS_HOLE(bp)))
 		return (0);
 	if (zb->zb_object == DMU_META_DNODE_OBJECT && !BP_IS_HOLE(bp))
 		return (0);
 
 	uint64_t span = bp_span_in_blocks(dnp->dn_indblkshift, zb->zb_level);
 	uint64_t start;
 
 	/*
 	 * If this multiply overflows, we don't need to send this block.
 	 * Even if it has a birth time, it can never not be a hole, so
 	 * we don't need to send records for it.
 	 */
 	if (!overflow_multiply(span, zb->zb_blkid, &start) || (!(zb->zb_blkid ==
 	    DMU_SPILL_BLKID || DMU_OT_IS_METADATA(dnp->dn_type)) &&
 	    span * zb->zb_blkid > dnp->dn_maxblkid)) {
 		ASSERT(BP_IS_HOLE(bp));
 		return (0);
 	}
 
 	if (zb->zb_blkid == DMU_SPILL_BLKID)
 		ASSERT3U(BP_GET_TYPE(bp), ==, DMU_OT_SA);
 
 	enum type record_type = DATA;
 	if (BP_IS_HOLE(bp))
 		record_type = HOLE;
 	else if (BP_IS_REDACTED(bp))
 		record_type = REDACT;
 	else
 		record_type = DATA;
 
 	record = range_alloc(record_type, zb->zb_object, start,
 	    (start + span < start ? 0 : start + span), B_FALSE);
 
 	uint64_t datablksz = (zb->zb_blkid == DMU_SPILL_BLKID ?
 	    BP_GET_LSIZE(bp) : dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
 
 	if (BP_IS_HOLE(bp)) {
 		record->sru.hole.datablksz = datablksz;
 	} else if (BP_IS_REDACTED(bp)) {
 		record->sru.redact.datablksz = datablksz;
 	} else {
 		record->sru.data.datablksz = datablksz;
 		record->sru.data.obj_type = dnp->dn_type;
 		record->sru.data.bp = *bp;
 	}
 
 	bqueue_enqueue(&sta->q, record, sizeof (*record));
 	return (0);
 }
 
 struct redact_list_cb_arg {
 	uint64_t *num_blocks_visited;
 	bqueue_t *q;
 	boolean_t *cancel;
 	boolean_t mark_redact;
 };
 
 static int
 redact_list_cb(redact_block_phys_t *rb, void *arg)
 {
 	struct redact_list_cb_arg *rlcap = arg;
 
 	atomic_inc_64(rlcap->num_blocks_visited);
 	if (*rlcap->cancel)
 		return (-1);
 
 	struct send_range *data = range_alloc(REDACT, rb->rbp_object,
 	    rb->rbp_blkid, rb->rbp_blkid + redact_block_get_count(rb), B_FALSE);
 	ASSERT3U(data->end_blkid, >, rb->rbp_blkid);
 	if (rlcap->mark_redact) {
 		data->type = REDACT;
 		data->sru.redact.datablksz = redact_block_get_size(rb);
 	} else {
 		data->type = PREVIOUSLY_REDACTED;
 	}
 	bqueue_enqueue(rlcap->q, data, sizeof (*data));
 
 	return (0);
 }
 
 /*
  * This function kicks off the traverse_dataset.  It also handles setting the
  * error code of the thread in case something goes wrong, and pushes the End of
  * Stream record when the traverse_dataset call has finished.
  */
 static __attribute__((noreturn)) void
 send_traverse_thread(void *arg)
 {
 	struct send_thread_arg *st_arg = arg;
 	int err = 0;
 	struct send_range *data;
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 
 	err = traverse_dataset_resume(st_arg->os->os_dsl_dataset,
 	    st_arg->fromtxg, &st_arg->resume,
 	    st_arg->flags, send_cb, st_arg);
 
 	if (err != EINTR)
 		st_arg->error_code = err;
 	data = range_alloc(DATA, 0, 0, 0, B_TRUE);
 	bqueue_enqueue_flush(&st_arg->q, data, sizeof (*data));
 	spl_fstrans_unmark(cookie);
 	thread_exit();
 }
 
 /*
  * Utility function that causes End of Stream records to compare after of all
  * others, so that other threads' comparison logic can stay simple.
  */
 static int __attribute__((unused))
 send_range_after(const struct send_range *from, const struct send_range *to)
 {
 	if (from->eos_marker == B_TRUE)
 		return (1);
 	if (to->eos_marker == B_TRUE)
 		return (-1);
 
 	uint64_t from_obj = from->object;
 	uint64_t from_end_obj = from->object + 1;
 	uint64_t to_obj = to->object;
 	uint64_t to_end_obj = to->object + 1;
 	if (from_obj == 0) {
 		ASSERT(from->type == HOLE || from->type == OBJECT_RANGE);
 		from_obj = from->start_blkid << DNODES_PER_BLOCK_SHIFT;
 		from_end_obj = from->end_blkid << DNODES_PER_BLOCK_SHIFT;
 	}
 	if (to_obj == 0) {
 		ASSERT(to->type == HOLE || to->type == OBJECT_RANGE);
 		to_obj = to->start_blkid << DNODES_PER_BLOCK_SHIFT;
 		to_end_obj = to->end_blkid << DNODES_PER_BLOCK_SHIFT;
 	}
 
 	if (from_end_obj <= to_obj)
 		return (-1);
 	if (from_obj >= to_end_obj)
 		return (1);
 	int64_t cmp = TREE_CMP(to->type == OBJECT_RANGE, from->type ==
 	    OBJECT_RANGE);
 	if (unlikely(cmp))
 		return (cmp);
 	cmp = TREE_CMP(to->type == OBJECT, from->type == OBJECT);
 	if (unlikely(cmp))
 		return (cmp);
 	if (from->end_blkid <= to->start_blkid)
 		return (-1);
 	if (from->start_blkid >= to->end_blkid)
 		return (1);
 	return (0);
 }
 
 /*
  * Pop the new data off the queue, check that the records we receive are in
  * the right order, but do not free the old data.  This is used so that the
  * records can be sent on to the main thread without copying the data.
  */
 static struct send_range *
 get_next_range_nofree(bqueue_t *bq, struct send_range *prev)
 {
 	struct send_range *next = bqueue_dequeue(bq);
 	ASSERT3S(send_range_after(prev, next), ==, -1);
 	return (next);
 }
 
 /*
  * Pop the new data off the queue, check that the records we receive are in
  * the right order, and free the old data.
  */
 static struct send_range *
 get_next_range(bqueue_t *bq, struct send_range *prev)
 {
 	struct send_range *next = get_next_range_nofree(bq, prev);
 	range_free(prev);
 	return (next);
 }
 
 static __attribute__((noreturn)) void
 redact_list_thread(void *arg)
 {
 	struct redact_list_thread_arg *rlt_arg = arg;
 	struct send_range *record;
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 	if (rlt_arg->rl != NULL) {
 		struct redact_list_cb_arg rlcba = {0};
 		rlcba.cancel = &rlt_arg->cancel;
 		rlcba.q = &rlt_arg->q;
 		rlcba.num_blocks_visited = rlt_arg->num_blocks_visited;
 		rlcba.mark_redact = rlt_arg->mark_redact;
 		int err = dsl_redaction_list_traverse(rlt_arg->rl,
 		    &rlt_arg->resume, redact_list_cb, &rlcba);
 		if (err != EINTR)
 			rlt_arg->error_code = err;
 	}
 	record = range_alloc(DATA, 0, 0, 0, B_TRUE);
 	bqueue_enqueue_flush(&rlt_arg->q, record, sizeof (*record));
 	spl_fstrans_unmark(cookie);
 
 	thread_exit();
 }
 
 /*
  * Compare the start point of the two provided ranges. End of stream ranges
  * compare last, objects compare before any data or hole inside that object and
  * multi-object holes that start at the same object.
  */
 static int
 send_range_start_compare(struct send_range *r1, struct send_range *r2)
 {
 	uint64_t r1_objequiv = r1->object;
 	uint64_t r1_l0equiv = r1->start_blkid;
 	uint64_t r2_objequiv = r2->object;
 	uint64_t r2_l0equiv = r2->start_blkid;
 	int64_t cmp = TREE_CMP(r1->eos_marker, r2->eos_marker);
 	if (unlikely(cmp))
 		return (cmp);
 	if (r1->object == 0) {
 		r1_objequiv = r1->start_blkid * DNODES_PER_BLOCK;
 		r1_l0equiv = 0;
 	}
 	if (r2->object == 0) {
 		r2_objequiv = r2->start_blkid * DNODES_PER_BLOCK;
 		r2_l0equiv = 0;
 	}
 
 	cmp = TREE_CMP(r1_objequiv, r2_objequiv);
 	if (likely(cmp))
 		return (cmp);
 	cmp = TREE_CMP(r2->type == OBJECT_RANGE, r1->type == OBJECT_RANGE);
 	if (unlikely(cmp))
 		return (cmp);
 	cmp = TREE_CMP(r2->type == OBJECT, r1->type == OBJECT);
 	if (unlikely(cmp))
 		return (cmp);
 
 	return (TREE_CMP(r1_l0equiv, r2_l0equiv));
 }
 
 enum q_idx {
 	REDACT_IDX = 0,
 	TO_IDX,
 	FROM_IDX,
 	NUM_THREADS
 };
 
 /*
  * This function returns the next range the send_merge_thread should operate on.
  * The inputs are two arrays; the first one stores the range at the front of the
  * queues stored in the second one.  The ranges are sorted in descending
  * priority order; the metadata from earlier ranges overrules metadata from
  * later ranges.  out_mask is used to return which threads the ranges came from;
  * bit i is set if ranges[i] started at the same place as the returned range.
  *
  * This code is not hardcoded to compare a specific number of threads; it could
  * be used with any number, just by changing the q_idx enum.
  *
  * The "next range" is the one with the earliest start; if two starts are equal,
  * the highest-priority range is the next to operate on.  If a higher-priority
  * range starts in the middle of the first range, then the first range will be
  * truncated to end where the higher-priority range starts, and we will operate
  * on that one next time.   In this way, we make sure that each block covered by
  * some range gets covered by a returned range, and each block covered is
  * returned using the metadata of the highest-priority range it appears in.
  *
  * For example, if the three ranges at the front of the queues were [2,4),
  * [3,5), and [1,3), then the ranges returned would be [1,2) with the metadata
  * from the third range, [2,4) with the metadata from the first range, and then
  * [4,5) with the metadata from the second.
  */
 static struct send_range *
 find_next_range(struct send_range **ranges, bqueue_t **qs, uint64_t *out_mask)
 {
 	int idx = 0; // index of the range with the earliest start
 	int i;
 	uint64_t bmask = 0;
 	for (i = 1; i < NUM_THREADS; i++) {
 		if (send_range_start_compare(ranges[i], ranges[idx]) < 0)
 			idx = i;
 	}
 	if (ranges[idx]->eos_marker) {
 		struct send_range *ret = range_alloc(DATA, 0, 0, 0, B_TRUE);
 		*out_mask = 0;
 		return (ret);
 	}
 	/*
 	 * Find all the ranges that start at that same point.
 	 */
 	for (i = 0; i < NUM_THREADS; i++) {
 		if (send_range_start_compare(ranges[i], ranges[idx]) == 0)
 			bmask |= 1 << i;
 	}
 	*out_mask = bmask;
 	/*
 	 * OBJECT_RANGE records only come from the TO thread, and should always
 	 * be treated as overlapping with nothing and sent on immediately.  They
 	 * are only used in raw sends, and are never redacted.
 	 */
 	if (ranges[idx]->type == OBJECT_RANGE) {
 		ASSERT3U(idx, ==, TO_IDX);
 		ASSERT3U(*out_mask, ==, 1 << TO_IDX);
 		struct send_range *ret = ranges[idx];
 		ranges[idx] = get_next_range_nofree(qs[idx], ranges[idx]);
 		return (ret);
 	}
 	/*
 	 * Find the first start or end point after the start of the first range.
 	 */
 	uint64_t first_change = ranges[idx]->end_blkid;
 	for (i = 0; i < NUM_THREADS; i++) {
 		if (i == idx || ranges[i]->eos_marker ||
 		    ranges[i]->object > ranges[idx]->object ||
 		    ranges[i]->object == DMU_META_DNODE_OBJECT)
 			continue;
 		ASSERT3U(ranges[i]->object, ==, ranges[idx]->object);
 		if (first_change > ranges[i]->start_blkid &&
 		    (bmask & (1 << i)) == 0)
 			first_change = ranges[i]->start_blkid;
 		else if (first_change > ranges[i]->end_blkid)
 			first_change = ranges[i]->end_blkid;
 	}
 	/*
 	 * Update all ranges to no longer overlap with the range we're
 	 * returning. All such ranges must start at the same place as the range
 	 * being returned, and end at or after first_change. Thus we update
 	 * their start to first_change. If that makes them size 0, then free
 	 * them and pull a new range from that thread.
 	 */
 	for (i = 0; i < NUM_THREADS; i++) {
 		if (i == idx || (bmask & (1 << i)) == 0)
 			continue;
 		ASSERT3U(first_change, >, ranges[i]->start_blkid);
 		ranges[i]->start_blkid = first_change;
 		ASSERT3U(ranges[i]->start_blkid, <=, ranges[i]->end_blkid);
 		if (ranges[i]->start_blkid == ranges[i]->end_blkid)
 			ranges[i] = get_next_range(qs[i], ranges[i]);
 	}
 	/*
 	 * Short-circuit the simple case; if the range doesn't overlap with
 	 * anything else, or it only overlaps with things that start at the same
 	 * place and are longer, send it on.
 	 */
 	if (first_change == ranges[idx]->end_blkid) {
 		struct send_range *ret = ranges[idx];
 		ranges[idx] = get_next_range_nofree(qs[idx], ranges[idx]);
 		return (ret);
 	}
 
 	/*
 	 * Otherwise, return a truncated copy of ranges[idx] and move the start
 	 * of ranges[idx] back to first_change.
 	 */
 	struct send_range *ret = kmem_alloc(sizeof (*ret), KM_SLEEP);
 	*ret = *ranges[idx];
 	ret->end_blkid = first_change;
 	ranges[idx]->start_blkid = first_change;
 	return (ret);
 }
 
 #define	FROM_AND_REDACT_BITS ((1 << REDACT_IDX) | (1 << FROM_IDX))
 
 /*
  * Merge the results from the from thread and the to thread, and then hand the
  * records off to send_prefetch_thread to prefetch them.  If this is not a
  * send from a redaction bookmark, the from thread will push an end of stream
  * record and stop, and we'll just send everything that was changed in the
  * to_ds since the ancestor's creation txg. If it is, then since
  * traverse_dataset has a canonical order, we can compare each change as
  * they're pulled off the queues.  That will give us a stream that is
  * appropriately sorted, and covers all records.  In addition, we pull the
  * data from the redact_list_thread and use that to determine which blocks
  * should be redacted.
  */
 static __attribute__((noreturn)) void
 send_merge_thread(void *arg)
 {
 	struct send_merge_thread_arg *smt_arg = arg;
 	struct send_range *front_ranges[NUM_THREADS];
 	bqueue_t *queues[NUM_THREADS];
 	int err = 0;
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 
 	if (smt_arg->redact_arg == NULL) {
 		front_ranges[REDACT_IDX] =
 		    kmem_zalloc(sizeof (struct send_range), KM_SLEEP);
 		front_ranges[REDACT_IDX]->eos_marker = B_TRUE;
 		front_ranges[REDACT_IDX]->type = REDACT;
 		queues[REDACT_IDX] = NULL;
 	} else {
 		front_ranges[REDACT_IDX] =
 		    bqueue_dequeue(&smt_arg->redact_arg->q);
 		queues[REDACT_IDX] = &smt_arg->redact_arg->q;
 	}
 	front_ranges[TO_IDX] = bqueue_dequeue(&smt_arg->to_arg->q);
 	queues[TO_IDX] = &smt_arg->to_arg->q;
 	front_ranges[FROM_IDX] = bqueue_dequeue(&smt_arg->from_arg->q);
 	queues[FROM_IDX] = &smt_arg->from_arg->q;
 	uint64_t mask = 0;
 	struct send_range *range;
 	for (range = find_next_range(front_ranges, queues, &mask);
 	    !range->eos_marker && err == 0 && !smt_arg->cancel;
 	    range = find_next_range(front_ranges, queues, &mask)) {
 		/*
 		 * If the range in question was in both the from redact bookmark
 		 * and the bookmark we're using to redact, then don't send it.
 		 * It's already redacted on the receiving system, so a redaction
 		 * record would be redundant.
 		 */
 		if ((mask & FROM_AND_REDACT_BITS) == FROM_AND_REDACT_BITS) {
 			ASSERT3U(range->type, ==, REDACT);
 			range_free(range);
 			continue;
 		}
 		bqueue_enqueue(&smt_arg->q, range, sizeof (*range));
 
 		if (smt_arg->to_arg->error_code != 0) {
 			err = smt_arg->to_arg->error_code;
 		} else if (smt_arg->from_arg->error_code != 0) {
 			err = smt_arg->from_arg->error_code;
 		} else if (smt_arg->redact_arg != NULL &&
 		    smt_arg->redact_arg->error_code != 0) {
 			err = smt_arg->redact_arg->error_code;
 		}
 	}
 	if (smt_arg->cancel && err == 0)
 		err = SET_ERROR(EINTR);
 	smt_arg->error = err;
 	if (smt_arg->error != 0) {
 		smt_arg->to_arg->cancel = B_TRUE;
 		smt_arg->from_arg->cancel = B_TRUE;
 		if (smt_arg->redact_arg != NULL)
 			smt_arg->redact_arg->cancel = B_TRUE;
 	}
 	for (int i = 0; i < NUM_THREADS; i++) {
 		while (!front_ranges[i]->eos_marker) {
 			front_ranges[i] = get_next_range(queues[i],
 			    front_ranges[i]);
 		}
 		range_free(front_ranges[i]);
 	}
 	range->eos_marker = B_TRUE;
 	bqueue_enqueue_flush(&smt_arg->q, range, 1);
 	spl_fstrans_unmark(cookie);
 	thread_exit();
 }
 
 struct send_reader_thread_arg {
 	struct send_merge_thread_arg *smta;
 	bqueue_t q;
 	boolean_t cancel;
 	boolean_t issue_reads;
 	uint64_t featureflags;
 	int error;
 };
 
 static void
 dmu_send_read_done(zio_t *zio)
 {
 	struct send_range *range = zio->io_private;
 
 	mutex_enter(&range->sru.data.lock);
 	if (zio->io_error != 0) {
 		abd_free(range->sru.data.abd);
 		range->sru.data.abd = NULL;
 		range->sru.data.io_err = zio->io_error;
 	}
 
 	ASSERT(range->sru.data.io_outstanding);
 	range->sru.data.io_outstanding = B_FALSE;
 	cv_broadcast(&range->sru.data.cv);
 	mutex_exit(&range->sru.data.lock);
 }
 
 static void
 issue_data_read(struct send_reader_thread_arg *srta, struct send_range *range)
 {
 	struct srd *srdp = &range->sru.data;
 	blkptr_t *bp = &srdp->bp;
 	objset_t *os = srta->smta->os;
 
 	ASSERT3U(range->type, ==, DATA);
 	ASSERT3U(range->start_blkid + 1, ==, range->end_blkid);
 	/*
 	 * If we have large blocks stored on disk but
 	 * the send flags don't allow us to send large
 	 * blocks, we split the data from the arc buf
 	 * into chunks.
 	 */
 	boolean_t split_large_blocks =
 	    srdp->datablksz > SPA_OLD_MAXBLOCKSIZE &&
 	    !(srta->featureflags & DMU_BACKUP_FEATURE_LARGE_BLOCKS);
 	/*
 	 * We should only request compressed data from the ARC if all
 	 * the following are true:
 	 *  - stream compression was requested
 	 *  - we aren't splitting large blocks into smaller chunks
 	 *  - the data won't need to be byteswapped before sending
 	 *  - this isn't an embedded block
 	 *  - this isn't metadata (if receiving on a different endian
 	 *    system it can be byteswapped more easily)
 	 */
 	boolean_t request_compressed =
 	    (srta->featureflags & DMU_BACKUP_FEATURE_COMPRESSED) &&
 	    !split_large_blocks && !BP_SHOULD_BYTESWAP(bp) &&
 	    !BP_IS_EMBEDDED(bp) && !DMU_OT_IS_METADATA(BP_GET_TYPE(bp));
 
 	zio_flag_t zioflags = ZIO_FLAG_CANFAIL;
 
 	if (srta->featureflags & DMU_BACKUP_FEATURE_RAW) {
 		zioflags |= ZIO_FLAG_RAW;
 		srdp->io_compressed = B_TRUE;
 	} else if (request_compressed) {
 		zioflags |= ZIO_FLAG_RAW_COMPRESS;
 		srdp->io_compressed = B_TRUE;
 	}
 
 	srdp->datasz = (zioflags & ZIO_FLAG_RAW_COMPRESS) ?
 	    BP_GET_PSIZE(bp) : BP_GET_LSIZE(bp);
 
 	if (!srta->issue_reads)
 		return;
 	if (BP_IS_REDACTED(bp))
 		return;
 	if (send_do_embed(bp, srta->featureflags))
 		return;
 
 	zbookmark_phys_t zb = {
 	    .zb_objset = dmu_objset_id(os),
 	    .zb_object = range->object,
 	    .zb_level = 0,
 	    .zb_blkid = range->start_blkid,
 	};
 
 	arc_flags_t aflags = ARC_FLAG_CACHED_ONLY;
 
 	int arc_err = arc_read(NULL, os->os_spa, bp,
 	    arc_getbuf_func, &srdp->abuf, ZIO_PRIORITY_ASYNC_READ,
 	    zioflags, &aflags, &zb);
 	/*
 	 * If the data is not already cached in the ARC, we read directly
 	 * from zio.  This avoids the performance overhead of adding a new
 	 * entry to the ARC, and we also avoid polluting the ARC cache with
 	 * data that is not likely to be used in the future.
 	 */
 	if (arc_err != 0) {
 		srdp->abd = abd_alloc_linear(srdp->datasz, B_FALSE);
 		srdp->io_outstanding = B_TRUE;
 		zio_nowait(zio_read(NULL, os->os_spa, bp, srdp->abd,
 		    srdp->datasz, dmu_send_read_done, range,
 		    ZIO_PRIORITY_ASYNC_READ, zioflags, &zb));
 	}
 }
 
 /*
  * Create a new record with the given values.
  */
 static void
 enqueue_range(struct send_reader_thread_arg *srta, bqueue_t *q, dnode_t *dn,
     uint64_t blkid, uint64_t count, const blkptr_t *bp, uint32_t datablksz)
 {
 	enum type range_type = (bp == NULL || BP_IS_HOLE(bp) ? HOLE :
 	    (BP_IS_REDACTED(bp) ? REDACT : DATA));
 
 	struct send_range *range = range_alloc(range_type, dn->dn_object,
 	    blkid, blkid + count, B_FALSE);
 
 	if (blkid == DMU_SPILL_BLKID) {
 		ASSERT3P(bp, !=, NULL);
 		ASSERT3U(BP_GET_TYPE(bp), ==, DMU_OT_SA);
 	}
 
 	switch (range_type) {
 	case HOLE:
 		range->sru.hole.datablksz = datablksz;
 		break;
 	case DATA:
 		ASSERT3U(count, ==, 1);
 		range->sru.data.datablksz = datablksz;
 		range->sru.data.obj_type = dn->dn_type;
 		range->sru.data.bp = *bp;
 		issue_data_read(srta, range);
 		break;
 	case REDACT:
 		range->sru.redact.datablksz = datablksz;
 		break;
 	default:
 		break;
 	}
 	bqueue_enqueue(q, range, datablksz);
 }
 
 /*
  * This thread is responsible for two things: First, it retrieves the correct
  * blkptr in the to ds if we need to send the data because of something from
  * the from thread.  As a result of this, we're the first ones to discover that
  * some indirect blocks can be discarded because they're not holes. Second,
  * it issues prefetches for the data we need to send.
  */
 static __attribute__((noreturn)) void
 send_reader_thread(void *arg)
 {
 	struct send_reader_thread_arg *srta = arg;
 	struct send_merge_thread_arg *smta = srta->smta;
 	bqueue_t *inq = &smta->q;
 	bqueue_t *outq = &srta->q;
 	objset_t *os = smta->os;
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 	struct send_range *range = bqueue_dequeue(inq);
 	int err = 0;
 
 	/*
 	 * If the record we're analyzing is from a redaction bookmark from the
 	 * fromds, then we need to know whether or not it exists in the tods so
 	 * we know whether to create records for it or not. If it does, we need
 	 * the datablksz so we can generate an appropriate record for it.
 	 * Finally, if it isn't redacted, we need the blkptr so that we can send
 	 * a WRITE record containing the actual data.
 	 */
 	uint64_t last_obj = UINT64_MAX;
 	uint64_t last_obj_exists = B_TRUE;
 	while (!range->eos_marker && !srta->cancel && smta->error == 0 &&
 	    err == 0) {
 		switch (range->type) {
 		case DATA:
 			issue_data_read(srta, range);
 			bqueue_enqueue(outq, range, range->sru.data.datablksz);
 			range = get_next_range_nofree(inq, range);
 			break;
 		case HOLE:
 		case OBJECT:
 		case OBJECT_RANGE:
 		case REDACT: // Redacted blocks must exist
 			bqueue_enqueue(outq, range, sizeof (*range));
 			range = get_next_range_nofree(inq, range);
 			break;
 		case PREVIOUSLY_REDACTED: {
 			/*
 			 * This entry came from the "from bookmark" when
 			 * sending from a bookmark that has a redaction
 			 * list.  We need to check if this object/blkid
 			 * exists in the target ("to") dataset, and if
 			 * not then we drop this entry.  We also need
 			 * to fill in the block pointer so that we know
 			 * what to prefetch.
 			 *
 			 * To accomplish the above, we first cache whether or
 			 * not the last object we examined exists.  If it
 			 * doesn't, we can drop this record. If it does, we hold
 			 * the dnode and use it to call dbuf_dnode_findbp. We do
 			 * this instead of dbuf_bookmark_findbp because we will
 			 * often operate on large ranges, and holding the dnode
 			 * once is more efficient.
 			 */
 			boolean_t object_exists = B_TRUE;
 			/*
 			 * If the data is redacted, we only care if it exists,
 			 * so that we don't send records for objects that have
 			 * been deleted.
 			 */
 			dnode_t *dn;
 			if (range->object == last_obj && !last_obj_exists) {
 				/*
 				 * If we're still examining the same object as
 				 * previously, and it doesn't exist, we don't
 				 * need to call dbuf_bookmark_findbp.
 				 */
 				object_exists = B_FALSE;
 			} else {
 				err = dnode_hold(os, range->object, FTAG, &dn);
 				if (err == ENOENT) {
 					object_exists = B_FALSE;
 					err = 0;
 				}
 				last_obj = range->object;
 				last_obj_exists = object_exists;
 			}
 
 			if (err != 0) {
 				break;
 			} else if (!object_exists) {
 				/*
 				 * The block was modified, but doesn't
 				 * exist in the to dataset; if it was
 				 * deleted in the to dataset, then we'll
 				 * visit the hole bp for it at some point.
 				 */
 				range = get_next_range(inq, range);
 				continue;
 			}
 			uint64_t file_max =
 			    MIN(dn->dn_maxblkid, range->end_blkid);
 			/*
 			 * The object exists, so we need to try to find the
 			 * blkptr for each block in the range we're processing.
 			 */
 			rw_enter(&dn->dn_struct_rwlock, RW_READER);
 			for (uint64_t blkid = range->start_blkid;
 			    blkid < file_max; blkid++) {
 				blkptr_t bp;
 				uint32_t datablksz =
 				    dn->dn_phys->dn_datablkszsec <<
 				    SPA_MINBLOCKSHIFT;
 				uint64_t offset = blkid * datablksz;
 				/*
 				 * This call finds the next non-hole block in
 				 * the object. This is to prevent a
 				 * performance problem where we're unredacting
 				 * a large hole. Using dnode_next_offset to
 				 * skip over the large hole avoids iterating
 				 * over every block in it.
 				 */
 				err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK,
 				    &offset, 1, 1, 0);
 				if (err == ESRCH) {
 					offset = UINT64_MAX;
 					err = 0;
 				} else if (err != 0) {
 					break;
 				}
 				if (offset != blkid * datablksz) {
 					/*
 					 * if there is a hole from here
 					 * (blkid) to offset
 					 */
 					offset = MIN(offset, file_max *
 					    datablksz);
 					uint64_t nblks = (offset / datablksz) -
 					    blkid;
 					enqueue_range(srta, outq, dn, blkid,
 					    nblks, NULL, datablksz);
 					blkid += nblks;
 				}
 				if (blkid >= file_max)
 					break;
 				err = dbuf_dnode_findbp(dn, 0, blkid, &bp,
 				    NULL, NULL);
 				if (err != 0)
 					break;
 				ASSERT(!BP_IS_HOLE(&bp));
 				enqueue_range(srta, outq, dn, blkid, 1, &bp,
 				    datablksz);
 			}
 			rw_exit(&dn->dn_struct_rwlock);
 			dnode_rele(dn, FTAG);
 			range = get_next_range(inq, range);
 		}
 		}
 	}
 	if (srta->cancel || err != 0) {
 		smta->cancel = B_TRUE;
 		srta->error = err;
 	} else if (smta->error != 0) {
 		srta->error = smta->error;
 	}
 	while (!range->eos_marker)
 		range = get_next_range(inq, range);
 
 	bqueue_enqueue_flush(outq, range, 1);
 	spl_fstrans_unmark(cookie);
 	thread_exit();
 }
 
 #define	NUM_SNAPS_NOT_REDACTED UINT64_MAX
 
 struct dmu_send_params {
 	/* Pool args */
 	const void *tag; // Tag dp was held with, will be used to release dp.
 	dsl_pool_t *dp;
 	/* To snapshot args */
 	const char *tosnap;
 	dsl_dataset_t *to_ds;
 	/* From snapshot args */
 	zfs_bookmark_phys_t ancestor_zb;
 	uint64_t *fromredactsnaps;
 	/* NUM_SNAPS_NOT_REDACTED if not sending from redaction bookmark */
 	uint64_t numfromredactsnaps;
 	/* Stream params */
 	boolean_t is_clone;
 	boolean_t embedok;
 	boolean_t large_block_ok;
 	boolean_t compressok;
 	boolean_t rawok;
 	boolean_t savedok;
 	uint64_t resumeobj;
 	uint64_t resumeoff;
 	uint64_t saved_guid;
 	zfs_bookmark_phys_t *redactbook;
 	/* Stream output params */
 	dmu_send_outparams_t *dso;
 
 	/* Stream progress params */
 	offset_t *off;
 	int outfd;
 	char saved_toname[MAXNAMELEN];
 };
 
 static int
 setup_featureflags(struct dmu_send_params *dspp, objset_t *os,
     uint64_t *featureflags)
 {
 	dsl_dataset_t *to_ds = dspp->to_ds;
 	dsl_pool_t *dp = dspp->dp;
 
 	if (dmu_objset_type(os) == DMU_OST_ZFS) {
 		uint64_t version;
 		if (zfs_get_zplprop(os, ZFS_PROP_VERSION, &version) != 0)
 			return (SET_ERROR(EINVAL));
 
 		if (version >= ZPL_VERSION_SA)
 			*featureflags |= DMU_BACKUP_FEATURE_SA_SPILL;
 	}
 
 	/* raw sends imply large_block_ok */
 	if ((dspp->rawok || dspp->large_block_ok) &&
 	    dsl_dataset_feature_is_active(to_ds, SPA_FEATURE_LARGE_BLOCKS)) {
 		*featureflags |= DMU_BACKUP_FEATURE_LARGE_BLOCKS;
 	}
 
 	/* encrypted datasets will not have embedded blocks */
 	if ((dspp->embedok || dspp->rawok) && !os->os_encrypted &&
 	    spa_feature_is_active(dp->dp_spa, SPA_FEATURE_EMBEDDED_DATA)) {
 		*featureflags |= DMU_BACKUP_FEATURE_EMBED_DATA;
 	}
 
 	/* raw send implies compressok */
 	if (dspp->compressok || dspp->rawok)
 		*featureflags |= DMU_BACKUP_FEATURE_COMPRESSED;
 
 	if (dspp->rawok && os->os_encrypted)
 		*featureflags |= DMU_BACKUP_FEATURE_RAW;
 
 	if ((*featureflags &
 	    (DMU_BACKUP_FEATURE_EMBED_DATA | DMU_BACKUP_FEATURE_COMPRESSED |
 	    DMU_BACKUP_FEATURE_RAW)) != 0 &&
 	    spa_feature_is_active(dp->dp_spa, SPA_FEATURE_LZ4_COMPRESS)) {
 		*featureflags |= DMU_BACKUP_FEATURE_LZ4;
 	}
 
 	/*
 	 * We specifically do not include DMU_BACKUP_FEATURE_EMBED_DATA here to
 	 * allow sending ZSTD compressed datasets to a receiver that does not
 	 * support ZSTD
 	 */
 	if ((*featureflags &
 	    (DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_RAW)) != 0 &&
 	    dsl_dataset_feature_is_active(to_ds, SPA_FEATURE_ZSTD_COMPRESS)) {
 		*featureflags |= DMU_BACKUP_FEATURE_ZSTD;
 	}
 
 	if (dspp->resumeobj != 0 || dspp->resumeoff != 0) {
 		*featureflags |= DMU_BACKUP_FEATURE_RESUMING;
 	}
 
 	if (dspp->redactbook != NULL) {
 		*featureflags |= DMU_BACKUP_FEATURE_REDACTED;
 	}
 
 	if (dsl_dataset_feature_is_active(to_ds, SPA_FEATURE_LARGE_DNODE)) {
 		*featureflags |= DMU_BACKUP_FEATURE_LARGE_DNODE;
 	}
 	return (0);
 }
 
 static dmu_replay_record_t *
 create_begin_record(struct dmu_send_params *dspp, objset_t *os,
     uint64_t featureflags)
 {
 	dmu_replay_record_t *drr = kmem_zalloc(sizeof (dmu_replay_record_t),
 	    KM_SLEEP);
 	drr->drr_type = DRR_BEGIN;
 
 	struct drr_begin *drrb = &drr->drr_u.drr_begin;
 	dsl_dataset_t *to_ds = dspp->to_ds;
 
 	drrb->drr_magic = DMU_BACKUP_MAGIC;
 	drrb->drr_creation_time = dsl_dataset_phys(to_ds)->ds_creation_time;
 	drrb->drr_type = dmu_objset_type(os);
 	drrb->drr_toguid = dsl_dataset_phys(to_ds)->ds_guid;
 	drrb->drr_fromguid = dspp->ancestor_zb.zbm_guid;
 
 	DMU_SET_STREAM_HDRTYPE(drrb->drr_versioninfo, DMU_SUBSTREAM);
 	DMU_SET_FEATUREFLAGS(drrb->drr_versioninfo, featureflags);
 
 	if (dspp->is_clone)
 		drrb->drr_flags |= DRR_FLAG_CLONE;
 	if (dsl_dataset_phys(dspp->to_ds)->ds_flags & DS_FLAG_CI_DATASET)
 		drrb->drr_flags |= DRR_FLAG_CI_DATA;
 	if (zfs_send_set_freerecords_bit)
 		drrb->drr_flags |= DRR_FLAG_FREERECORDS;
 	drr->drr_u.drr_begin.drr_flags |= DRR_FLAG_SPILL_BLOCK;
 
 	if (dspp->savedok) {
 		drrb->drr_toguid = dspp->saved_guid;
 		strlcpy(drrb->drr_toname, dspp->saved_toname,
 		    sizeof (drrb->drr_toname));
 	} else {
 		dsl_dataset_name(to_ds, drrb->drr_toname);
 		if (!to_ds->ds_is_snapshot) {
 			(void) strlcat(drrb->drr_toname, "@--head--",
 			    sizeof (drrb->drr_toname));
 		}
 	}
 	return (drr);
 }
 
 static void
 setup_to_thread(struct send_thread_arg *to_arg, objset_t *to_os,
     dmu_sendstatus_t *dssp, uint64_t fromtxg, boolean_t rawok)
 {
 	VERIFY0(bqueue_init(&to_arg->q, zfs_send_no_prefetch_queue_ff,
 	    MAX(zfs_send_no_prefetch_queue_length, 2 * zfs_max_recordsize),
 	    offsetof(struct send_range, ln)));
 	to_arg->error_code = 0;
 	to_arg->cancel = B_FALSE;
 	to_arg->os = to_os;
 	to_arg->fromtxg = fromtxg;
 	to_arg->flags = TRAVERSE_PRE | TRAVERSE_PREFETCH_METADATA;
 	if (rawok)
 		to_arg->flags |= TRAVERSE_NO_DECRYPT;
 	if (zfs_send_corrupt_data)
 		to_arg->flags |= TRAVERSE_HARD;
 	to_arg->num_blocks_visited = &dssp->dss_blocks;
 	(void) thread_create(NULL, 0, send_traverse_thread, to_arg, 0,
 	    curproc, TS_RUN, minclsyspri);
 }
 
 static void
 setup_from_thread(struct redact_list_thread_arg *from_arg,
     redaction_list_t *from_rl, dmu_sendstatus_t *dssp)
 {
 	VERIFY0(bqueue_init(&from_arg->q, zfs_send_no_prefetch_queue_ff,
 	    MAX(zfs_send_no_prefetch_queue_length, 2 * zfs_max_recordsize),
 	    offsetof(struct send_range, ln)));
 	from_arg->error_code = 0;
 	from_arg->cancel = B_FALSE;
 	from_arg->rl = from_rl;
 	from_arg->mark_redact = B_FALSE;
 	from_arg->num_blocks_visited = &dssp->dss_blocks;
 	/*
 	 * If from_ds is null, send_traverse_thread just returns success and
 	 * enqueues an eos marker.
 	 */
 	(void) thread_create(NULL, 0, redact_list_thread, from_arg, 0,
 	    curproc, TS_RUN, minclsyspri);
 }
 
 static void
 setup_redact_list_thread(struct redact_list_thread_arg *rlt_arg,
     struct dmu_send_params *dspp, redaction_list_t *rl, dmu_sendstatus_t *dssp)
 {
 	if (dspp->redactbook == NULL)
 		return;
 
 	rlt_arg->cancel = B_FALSE;
 	VERIFY0(bqueue_init(&rlt_arg->q, zfs_send_no_prefetch_queue_ff,
 	    MAX(zfs_send_no_prefetch_queue_length, 2 * zfs_max_recordsize),
 	    offsetof(struct send_range, ln)));
 	rlt_arg->error_code = 0;
 	rlt_arg->mark_redact = B_TRUE;
 	rlt_arg->rl = rl;
 	rlt_arg->num_blocks_visited = &dssp->dss_blocks;
 
 	(void) thread_create(NULL, 0, redact_list_thread, rlt_arg, 0,
 	    curproc, TS_RUN, minclsyspri);
 }
 
 static void
 setup_merge_thread(struct send_merge_thread_arg *smt_arg,
     struct dmu_send_params *dspp, struct redact_list_thread_arg *from_arg,
     struct send_thread_arg *to_arg, struct redact_list_thread_arg *rlt_arg,
     objset_t *os)
 {
 	VERIFY0(bqueue_init(&smt_arg->q, zfs_send_no_prefetch_queue_ff,
 	    MAX(zfs_send_no_prefetch_queue_length, 2 * zfs_max_recordsize),
 	    offsetof(struct send_range, ln)));
 	smt_arg->cancel = B_FALSE;
 	smt_arg->error = 0;
 	smt_arg->from_arg = from_arg;
 	smt_arg->to_arg = to_arg;
 	if (dspp->redactbook != NULL)
 		smt_arg->redact_arg = rlt_arg;
 
 	smt_arg->os = os;
 	(void) thread_create(NULL, 0, send_merge_thread, smt_arg, 0, curproc,
 	    TS_RUN, minclsyspri);
 }
 
 static void
 setup_reader_thread(struct send_reader_thread_arg *srt_arg,
     struct dmu_send_params *dspp, struct send_merge_thread_arg *smt_arg,
     uint64_t featureflags)
 {
 	VERIFY0(bqueue_init(&srt_arg->q, zfs_send_queue_ff,
 	    MAX(zfs_send_queue_length, 2 * zfs_max_recordsize),
 	    offsetof(struct send_range, ln)));
 	srt_arg->smta = smt_arg;
 	srt_arg->issue_reads = !dspp->dso->dso_dryrun;
 	srt_arg->featureflags = featureflags;
 	(void) thread_create(NULL, 0, send_reader_thread, srt_arg, 0,
 	    curproc, TS_RUN, minclsyspri);
 }
 
 static int
 setup_resume_points(struct dmu_send_params *dspp,
     struct send_thread_arg *to_arg, struct redact_list_thread_arg *from_arg,
     struct redact_list_thread_arg *rlt_arg,
     struct send_merge_thread_arg *smt_arg, boolean_t resuming, objset_t *os,
     redaction_list_t *redact_rl, nvlist_t *nvl)
 {
 	(void) smt_arg;
 	dsl_dataset_t *to_ds = dspp->to_ds;
 	int err = 0;
 
 	uint64_t obj = 0;
 	uint64_t blkid = 0;
 	if (resuming) {
 		obj = dspp->resumeobj;
 		dmu_object_info_t to_doi;
 		err = dmu_object_info(os, obj, &to_doi);
 		if (err != 0)
 			return (err);
 
 		blkid = dspp->resumeoff / to_doi.doi_data_block_size;
 	}
 	/*
 	 * If we're resuming a redacted send, we can skip to the appropriate
 	 * point in the redaction bookmark by binary searching through it.
 	 */
 	if (redact_rl != NULL) {
 		SET_BOOKMARK(&rlt_arg->resume, to_ds->ds_object, obj, 0, blkid);
 	}
 
 	SET_BOOKMARK(&to_arg->resume, to_ds->ds_object, obj, 0, blkid);
 	if (nvlist_exists(nvl, BEGINNV_REDACT_FROM_SNAPS)) {
 		uint64_t objset = dspp->ancestor_zb.zbm_redaction_obj;
 		/*
 		 * Note: If the resume point is in an object whose
 		 * blocksize is different in the from vs to snapshots,
 		 * we will have divided by the "wrong" blocksize.
 		 * However, in this case fromsnap's send_cb() will
 		 * detect that the blocksize has changed and therefore
 		 * ignore this object.
 		 *
 		 * If we're resuming a send from a redaction bookmark,
 		 * we still cannot accidentally suggest blocks behind
 		 * the to_ds.  In addition, we know that any blocks in
 		 * the object in the to_ds will have to be sent, since
 		 * the size changed.  Therefore, we can't cause any harm
 		 * this way either.
 		 */
 		SET_BOOKMARK(&from_arg->resume, objset, obj, 0, blkid);
 	}
 	if (resuming) {
 		fnvlist_add_uint64(nvl, BEGINNV_RESUME_OBJECT, dspp->resumeobj);
 		fnvlist_add_uint64(nvl, BEGINNV_RESUME_OFFSET, dspp->resumeoff);
 	}
 	return (0);
 }
 
 static dmu_sendstatus_t *
 setup_send_progress(struct dmu_send_params *dspp)
 {
 	dmu_sendstatus_t *dssp = kmem_zalloc(sizeof (*dssp), KM_SLEEP);
 	dssp->dss_outfd = dspp->outfd;
 	dssp->dss_off = dspp->off;
 	dssp->dss_proc = curproc;
 	mutex_enter(&dspp->to_ds->ds_sendstream_lock);
 	list_insert_head(&dspp->to_ds->ds_sendstreams, dssp);
 	mutex_exit(&dspp->to_ds->ds_sendstream_lock);
 	return (dssp);
 }
 
 /*
  * Actually do the bulk of the work in a zfs send.
  *
  * The idea is that we want to do a send from ancestor_zb to to_ds.  We also
  * want to not send any data that has been modified by all the datasets in
  * redactsnaparr, and store the list of blocks that are redacted in this way in
  * a bookmark named redactbook, created on the to_ds.  We do this by creating
  * several worker threads, whose function is described below.
  *
  * There are three cases.
  * The first case is a redacted zfs send.  In this case there are 5 threads.
  * The first thread is the to_ds traversal thread: it calls dataset_traverse on
  * the to_ds and finds all the blocks that have changed since ancestor_zb (if
  * it's a full send, that's all blocks in the dataset).  It then sends those
  * blocks on to the send merge thread. The redact list thread takes the data
  * from the redaction bookmark and sends those blocks on to the send merge
  * thread.  The send merge thread takes the data from the to_ds traversal
  * thread, and combines it with the redaction records from the redact list
  * thread.  If a block appears in both the to_ds's data and the redaction data,
  * the send merge thread will mark it as redacted and send it on to the prefetch
  * thread.  Otherwise, the send merge thread will send the block on to the
  * prefetch thread unchanged. The prefetch thread will issue prefetch reads for
  * any data that isn't redacted, and then send the data on to the main thread.
  * The main thread behaves the same as in a normal send case, issuing demand
  * reads for data blocks and sending out records over the network
  *
  * The graphic below diagrams the flow of data in the case of a redacted zfs
  * send.  Each box represents a thread, and each line represents the flow of
  * data.
  *
  *             Records from the |
  *           redaction bookmark |
  * +--------------------+       |  +---------------------------+
  * |                    |       v  | Send Merge Thread         |
  * | Redact List Thread +----------> Apply redaction marks to  |
  * |                    |          | records as specified by   |
  * +--------------------+          | redaction ranges          |
  *                                 +----^---------------+------+
  *                                      |               | Merged data
  *                                      |               |
  *                                      |  +------------v--------+
  *                                      |  | Prefetch Thread     |
  * +--------------------+               |  | Issues prefetch     |
  * | to_ds Traversal    |               |  | reads of data blocks|
  * | Thread (finds      +---------------+  +------------+--------+
  * | candidate blocks)  |  Blocks modified              | Prefetched data
  * +--------------------+  by to_ds since               |
  *                         ancestor_zb     +------------v----+
  *                                         | Main Thread     |  File Descriptor
  *                                         | Sends data over +->(to zfs receive)
  *                                         | wire            |
  *                                         +-----------------+
  *
  * The second case is an incremental send from a redaction bookmark.  The to_ds
  * traversal thread and the main thread behave the same as in the redacted
  * send case.  The new thread is the from bookmark traversal thread.  It
  * iterates over the redaction list in the redaction bookmark, and enqueues
  * records for each block that was redacted in the original send.  The send
  * merge thread now has to merge the data from the two threads.  For details
  * about that process, see the header comment of send_merge_thread().  Any data
  * it decides to send on will be prefetched by the prefetch thread.  Note that
  * you can perform a redacted send from a redaction bookmark; in that case,
  * the data flow behaves very similarly to the flow in the redacted send case,
  * except with the addition of the bookmark traversal thread iterating over the
  * redaction bookmark.  The send_merge_thread also has to take on the
  * responsibility of merging the redact list thread's records, the bookmark
  * traversal thread's records, and the to_ds records.
  *
  * +---------------------+
  * |                     |
  * | Redact List Thread  +--------------+
  * |                     |              |
  * +---------------------+              |
  *        Blocks in redaction list      | Ranges modified by every secure snap
  *        of from bookmark              | (or EOS if not readcted)
  *                                      |
  * +---------------------+   |     +----v----------------------+
  * | bookmark Traversal  |   v     | Send Merge Thread         |
  * | Thread (finds       +---------> Merges bookmark, rlt, and |
  * | candidate blocks)   |         | to_ds send records        |
  * +---------------------+         +----^---------------+------+
  *                                      |               | Merged data
  *                                      |  +------------v--------+
  *                                      |  | Prefetch Thread     |
  * +--------------------+               |  | Issues prefetch     |
  * | to_ds Traversal    |               |  | reads of data blocks|
  * | Thread (finds      +---------------+  +------------+--------+
  * | candidate blocks)  |  Blocks modified              | Prefetched data
  * +--------------------+  by to_ds since  +------------v----+
  *                         ancestor_zb     | Main Thread     |  File Descriptor
  *                                         | Sends data over +->(to zfs receive)
  *                                         | wire            |
  *                                         +-----------------+
  *
  * The final case is a simple zfs full or incremental send.  The to_ds traversal
  * thread behaves the same as always. The redact list thread is never started.
  * The send merge thread takes all the blocks that the to_ds traversal thread
  * sends it, prefetches the data, and sends the blocks on to the main thread.
  * The main thread sends the data over the wire.
  *
  * To keep performance acceptable, we want to prefetch the data in the worker
  * threads.  While the to_ds thread could simply use the TRAVERSE_PREFETCH
  * feature built into traverse_dataset, the combining and deletion of records
  * due to redaction and sends from redaction bookmarks mean that we could
  * issue many unnecessary prefetches.  As a result, we only prefetch data
  * after we've determined that the record is not going to be redacted.  To
  * prevent the prefetching from getting too far ahead of the main thread, the
  * blocking queues that are used for communication are capped not by the
  * number of entries in the queue, but by the sum of the size of the
  * prefetches associated with them.  The limit on the amount of data that the
  * thread can prefetch beyond what the main thread has reached is controlled
  * by the global variable zfs_send_queue_length.  In addition, to prevent poor
  * performance in the beginning of a send, we also limit the distance ahead
  * that the traversal threads can be.  That distance is controlled by the
  * zfs_send_no_prefetch_queue_length tunable.
  *
  * Note: Releases dp using the specified tag.
  */
 static int
 dmu_send_impl(struct dmu_send_params *dspp)
 {
 	objset_t *os;
 	dmu_replay_record_t *drr;
 	dmu_sendstatus_t *dssp;
 	dmu_send_cookie_t dsc = {0};
 	int err;
 	uint64_t fromtxg = dspp->ancestor_zb.zbm_creation_txg;
 	uint64_t featureflags = 0;
 	struct redact_list_thread_arg *from_arg;
 	struct send_thread_arg *to_arg;
 	struct redact_list_thread_arg *rlt_arg;
 	struct send_merge_thread_arg *smt_arg;
 	struct send_reader_thread_arg *srt_arg;
 	struct send_range *range;
 	redaction_list_t *from_rl = NULL;
 	redaction_list_t *redact_rl = NULL;
 	boolean_t resuming = (dspp->resumeobj != 0 || dspp->resumeoff != 0);
 	boolean_t book_resuming = resuming;
 
 	dsl_dataset_t *to_ds = dspp->to_ds;
 	zfs_bookmark_phys_t *ancestor_zb = &dspp->ancestor_zb;
 	dsl_pool_t *dp = dspp->dp;
 	const void *tag = dspp->tag;
 
 	err = dmu_objset_from_ds(to_ds, &os);
 	if (err != 0) {
 		dsl_pool_rele(dp, tag);
 		return (err);
 	}
 
 	/*
 	 * If this is a non-raw send of an encrypted ds, we can ensure that
 	 * the objset_phys_t is authenticated. This is safe because this is
 	 * either a snapshot or we have owned the dataset, ensuring that
 	 * it can't be modified.
 	 */
 	if (!dspp->rawok && os->os_encrypted &&
 	    arc_is_unauthenticated(os->os_phys_buf)) {
 		zbookmark_phys_t zb;
 
 		SET_BOOKMARK(&zb, to_ds->ds_object, ZB_ROOT_OBJECT,
 		    ZB_ROOT_LEVEL, ZB_ROOT_BLKID);
 		err = arc_untransform(os->os_phys_buf, os->os_spa,
 		    &zb, B_FALSE);
 		if (err != 0) {
 			dsl_pool_rele(dp, tag);
 			return (err);
 		}
 
 		ASSERT0(arc_is_unauthenticated(os->os_phys_buf));
 	}
 
 	if ((err = setup_featureflags(dspp, os, &featureflags)) != 0) {
 		dsl_pool_rele(dp, tag);
 		return (err);
 	}
 
 	/*
 	 * If we're doing a redacted send, hold the bookmark's redaction list.
 	 */
 	if (dspp->redactbook != NULL) {
 		err = dsl_redaction_list_hold_obj(dp,
 		    dspp->redactbook->zbm_redaction_obj, FTAG,
 		    &redact_rl);
 		if (err != 0) {
 			dsl_pool_rele(dp, tag);
 			return (SET_ERROR(EINVAL));
 		}
 		dsl_redaction_list_long_hold(dp, redact_rl, FTAG);
 	}
 
 	/*
 	 * If we're sending from a redaction bookmark, hold the redaction list
 	 * so that we can consider sending the redacted blocks.
 	 */
 	if (ancestor_zb->zbm_redaction_obj != 0) {
 		err = dsl_redaction_list_hold_obj(dp,
 		    ancestor_zb->zbm_redaction_obj, FTAG, &from_rl);
 		if (err != 0) {
 			if (redact_rl != NULL) {
 				dsl_redaction_list_long_rele(redact_rl, FTAG);
 				dsl_redaction_list_rele(redact_rl, FTAG);
 			}
 			dsl_pool_rele(dp, tag);
 			return (SET_ERROR(EINVAL));
 		}
 		dsl_redaction_list_long_hold(dp, from_rl, FTAG);
 	}
 
 	dsl_dataset_long_hold(to_ds, FTAG);
 
 	from_arg = kmem_zalloc(sizeof (*from_arg), KM_SLEEP);
 	to_arg = kmem_zalloc(sizeof (*to_arg), KM_SLEEP);
 	rlt_arg = kmem_zalloc(sizeof (*rlt_arg), KM_SLEEP);
 	smt_arg = kmem_zalloc(sizeof (*smt_arg), KM_SLEEP);
 	srt_arg = kmem_zalloc(sizeof (*srt_arg), KM_SLEEP);
 
 	drr = create_begin_record(dspp, os, featureflags);
 	dssp = setup_send_progress(dspp);
 
 	dsc.dsc_drr = drr;
 	dsc.dsc_dso = dspp->dso;
 	dsc.dsc_os = os;
 	dsc.dsc_off = dspp->off;
 	dsc.dsc_toguid = dsl_dataset_phys(to_ds)->ds_guid;
 	dsc.dsc_fromtxg = fromtxg;
 	dsc.dsc_pending_op = PENDING_NONE;
 	dsc.dsc_featureflags = featureflags;
 	dsc.dsc_resume_object = dspp->resumeobj;
 	dsc.dsc_resume_offset = dspp->resumeoff;
 
 	dsl_pool_rele(dp, tag);
 
 	void *payload = NULL;
 	size_t payload_len = 0;
 	nvlist_t *nvl = fnvlist_alloc();
 
 	/*
 	 * If we're doing a redacted send, we include the snapshots we're
 	 * redacted with respect to so that the target system knows what send
 	 * streams can be correctly received on top of this dataset. If we're
 	 * instead sending a redacted dataset, we include the snapshots that the
 	 * dataset was created with respect to.
 	 */
 	if (dspp->redactbook != NULL) {
 		fnvlist_add_uint64_array(nvl, BEGINNV_REDACT_SNAPS,
 		    redact_rl->rl_phys->rlp_snaps,
 		    redact_rl->rl_phys->rlp_num_snaps);
 	} else if (dsl_dataset_feature_is_active(to_ds,
 	    SPA_FEATURE_REDACTED_DATASETS)) {
 		uint64_t *tods_guids;
 		uint64_t length;
 		VERIFY(dsl_dataset_get_uint64_array_feature(to_ds,
 		    SPA_FEATURE_REDACTED_DATASETS, &length, &tods_guids));
 		fnvlist_add_uint64_array(nvl, BEGINNV_REDACT_SNAPS, tods_guids,
 		    length);
 	}
 
 	/*
 	 * If we're sending from a redaction bookmark, then we should retrieve
 	 * the guids of that bookmark so we can send them over the wire.
 	 */
 	if (from_rl != NULL) {
 		fnvlist_add_uint64_array(nvl, BEGINNV_REDACT_FROM_SNAPS,
 		    from_rl->rl_phys->rlp_snaps,
 		    from_rl->rl_phys->rlp_num_snaps);
 	}
 
 	/*
 	 * If the snapshot we're sending from is redacted, include the redaction
 	 * list in the stream.
 	 */
 	if (dspp->numfromredactsnaps != NUM_SNAPS_NOT_REDACTED) {
 		ASSERT3P(from_rl, ==, NULL);
 		fnvlist_add_uint64_array(nvl, BEGINNV_REDACT_FROM_SNAPS,
 		    dspp->fromredactsnaps, (uint_t)dspp->numfromredactsnaps);
 		if (dspp->numfromredactsnaps > 0) {
 			kmem_free(dspp->fromredactsnaps,
 			    dspp->numfromredactsnaps * sizeof (uint64_t));
 			dspp->fromredactsnaps = NULL;
 		}
 	}
 
 	if (resuming || book_resuming) {
 		err = setup_resume_points(dspp, to_arg, from_arg,
 		    rlt_arg, smt_arg, resuming, os, redact_rl, nvl);
 		if (err != 0)
 			goto out;
 	}
 
 	if (featureflags & DMU_BACKUP_FEATURE_RAW) {
 		uint64_t ivset_guid = ancestor_zb->zbm_ivset_guid;
 		nvlist_t *keynvl = NULL;
 		ASSERT(os->os_encrypted);
 
 		err = dsl_crypto_populate_key_nvlist(os, ivset_guid,
 		    &keynvl);
 		if (err != 0) {
 			fnvlist_free(nvl);
 			goto out;
 		}
 
 		fnvlist_add_nvlist(nvl, "crypt_keydata", keynvl);
 		fnvlist_free(keynvl);
 	}
 
 	if (!nvlist_empty(nvl)) {
 		payload = fnvlist_pack(nvl, &payload_len);
 		drr->drr_payloadlen = payload_len;
 	}
 
 	fnvlist_free(nvl);
 	err = dump_record(&dsc, payload, payload_len);
 	fnvlist_pack_free(payload, payload_len);
 	if (err != 0) {
 		err = dsc.dsc_err;
 		goto out;
 	}
 
 	setup_to_thread(to_arg, os, dssp, fromtxg, dspp->rawok);
 	setup_from_thread(from_arg, from_rl, dssp);
 	setup_redact_list_thread(rlt_arg, dspp, redact_rl, dssp);
 	setup_merge_thread(smt_arg, dspp, from_arg, to_arg, rlt_arg, os);
 	setup_reader_thread(srt_arg, dspp, smt_arg, featureflags);
 
 	range = bqueue_dequeue(&srt_arg->q);
 	while (err == 0 && !range->eos_marker) {
 		err = do_dump(&dsc, range);
 		range = get_next_range(&srt_arg->q, range);
 		if (issig())
 			err = SET_ERROR(EINTR);
 	}
 
 	/*
 	 * If we hit an error or are interrupted, cancel our worker threads and
 	 * clear the queue of any pending records.  The threads will pass the
 	 * cancel up the tree of worker threads, and each one will clean up any
 	 * pending records before exiting.
 	 */
 	if (err != 0) {
 		srt_arg->cancel = B_TRUE;
 		while (!range->eos_marker) {
 			range = get_next_range(&srt_arg->q, range);
 		}
 	}
 	range_free(range);
 
 	bqueue_destroy(&srt_arg->q);
 	bqueue_destroy(&smt_arg->q);
 	if (dspp->redactbook != NULL)
 		bqueue_destroy(&rlt_arg->q);
 	bqueue_destroy(&to_arg->q);
 	bqueue_destroy(&from_arg->q);
 
 	if (err == 0 && srt_arg->error != 0)
 		err = srt_arg->error;
 
 	if (err != 0)
 		goto out;
 
 	if (dsc.dsc_pending_op != PENDING_NONE)
 		if (dump_record(&dsc, NULL, 0) != 0)
 			err = SET_ERROR(EINTR);
 
 	if (err != 0) {
 		if (err == EINTR && dsc.dsc_err != 0)
 			err = dsc.dsc_err;
 		goto out;
 	}
 
 	/*
 	 * Send the DRR_END record if this is not a saved stream.
 	 * Otherwise, the omitted DRR_END record will signal to
 	 * the receive side that the stream is incomplete.
 	 */
 	if (!dspp->savedok) {
 		memset(drr, 0, sizeof (dmu_replay_record_t));
 		drr->drr_type = DRR_END;
 		drr->drr_u.drr_end.drr_checksum = dsc.dsc_zc;
 		drr->drr_u.drr_end.drr_toguid = dsc.dsc_toguid;
 
 		if (dump_record(&dsc, NULL, 0) != 0)
 			err = dsc.dsc_err;
 	}
 out:
 	mutex_enter(&to_ds->ds_sendstream_lock);
 	list_remove(&to_ds->ds_sendstreams, dssp);
 	mutex_exit(&to_ds->ds_sendstream_lock);
 
 	VERIFY(err != 0 || (dsc.dsc_sent_begin &&
 	    (dsc.dsc_sent_end || dspp->savedok)));
 
 	kmem_free(drr, sizeof (dmu_replay_record_t));
 	kmem_free(dssp, sizeof (dmu_sendstatus_t));
 	kmem_free(from_arg, sizeof (*from_arg));
 	kmem_free(to_arg, sizeof (*to_arg));
 	kmem_free(rlt_arg, sizeof (*rlt_arg));
 	kmem_free(smt_arg, sizeof (*smt_arg));
 	kmem_free(srt_arg, sizeof (*srt_arg));
 
 	dsl_dataset_long_rele(to_ds, FTAG);
 	if (from_rl != NULL) {
 		dsl_redaction_list_long_rele(from_rl, FTAG);
 		dsl_redaction_list_rele(from_rl, FTAG);
 	}
 	if (redact_rl != NULL) {
 		dsl_redaction_list_long_rele(redact_rl, FTAG);
 		dsl_redaction_list_rele(redact_rl, FTAG);
 	}
 
 	return (err);
 }
 
 int
 dmu_send_obj(const char *pool, uint64_t tosnap, uint64_t fromsnap,
     boolean_t embedok, boolean_t large_block_ok, boolean_t compressok,
     boolean_t rawok, boolean_t savedok, int outfd, offset_t *off,
     dmu_send_outparams_t *dsop)
 {
 	int err;
 	dsl_dataset_t *fromds;
 	ds_hold_flags_t dsflags;
 	struct dmu_send_params dspp = {0};
 	dspp.embedok = embedok;
 	dspp.large_block_ok = large_block_ok;
 	dspp.compressok = compressok;
 	dspp.outfd = outfd;
 	dspp.off = off;
 	dspp.dso = dsop;
 	dspp.tag = FTAG;
 	dspp.rawok = rawok;
 	dspp.savedok = savedok;
 
 	dsflags = (rawok) ? DS_HOLD_FLAG_NONE : DS_HOLD_FLAG_DECRYPT;
 	err = dsl_pool_hold(pool, FTAG, &dspp.dp);
 	if (err != 0)
 		return (err);
 
 	err = dsl_dataset_hold_obj_flags(dspp.dp, tosnap, dsflags, FTAG,
 	    &dspp.to_ds);
 	if (err != 0) {
 		dsl_pool_rele(dspp.dp, FTAG);
 		return (err);
 	}
 
 	if (fromsnap != 0) {
-		err = dsl_dataset_hold_obj_flags(dspp.dp, fromsnap, dsflags,
-		    FTAG, &fromds);
+		err = dsl_dataset_hold_obj(dspp.dp, fromsnap, FTAG, &fromds);
+
 		if (err != 0) {
 			dsl_dataset_rele_flags(dspp.to_ds, dsflags, FTAG);
 			dsl_pool_rele(dspp.dp, FTAG);
 			return (err);
 		}
 		dspp.ancestor_zb.zbm_guid = dsl_dataset_phys(fromds)->ds_guid;
 		dspp.ancestor_zb.zbm_creation_txg =
 		    dsl_dataset_phys(fromds)->ds_creation_txg;
 		dspp.ancestor_zb.zbm_creation_time =
 		    dsl_dataset_phys(fromds)->ds_creation_time;
 
 		if (dsl_dataset_is_zapified(fromds)) {
 			(void) zap_lookup(dspp.dp->dp_meta_objset,
 			    fromds->ds_object, DS_FIELD_IVSET_GUID, 8, 1,
 			    &dspp.ancestor_zb.zbm_ivset_guid);
 		}
 
 		/* See dmu_send for the reasons behind this. */
 		uint64_t *fromredact;
 
 		if (!dsl_dataset_get_uint64_array_feature(fromds,
 		    SPA_FEATURE_REDACTED_DATASETS,
 		    &dspp.numfromredactsnaps,
 		    &fromredact)) {
 			dspp.numfromredactsnaps = NUM_SNAPS_NOT_REDACTED;
 		} else if (dspp.numfromredactsnaps > 0) {
 			uint64_t size = dspp.numfromredactsnaps *
 			    sizeof (uint64_t);
 			dspp.fromredactsnaps = kmem_zalloc(size, KM_SLEEP);
 			memcpy(dspp.fromredactsnaps, fromredact, size);
 		}
 
 		boolean_t is_before =
 		    dsl_dataset_is_before(dspp.to_ds, fromds, 0);
 		dspp.is_clone = (dspp.to_ds->ds_dir !=
 		    fromds->ds_dir);
 		dsl_dataset_rele(fromds, FTAG);
 		if (!is_before) {
 			dsl_pool_rele(dspp.dp, FTAG);
 			err = SET_ERROR(EXDEV);
 		} else {
 			err = dmu_send_impl(&dspp);
 		}
 	} else {
 		dspp.numfromredactsnaps = NUM_SNAPS_NOT_REDACTED;
 		err = dmu_send_impl(&dspp);
 	}
 	if (dspp.fromredactsnaps)
 		kmem_free(dspp.fromredactsnaps,
 		    dspp.numfromredactsnaps * sizeof (uint64_t));
 
-	dsl_dataset_rele(dspp.to_ds, FTAG);
+	dsl_dataset_rele_flags(dspp.to_ds, dsflags, FTAG);
 	return (err);
 }
 
 int
 dmu_send(const char *tosnap, const char *fromsnap, boolean_t embedok,
     boolean_t large_block_ok, boolean_t compressok, boolean_t rawok,
     boolean_t savedok, uint64_t resumeobj, uint64_t resumeoff,
     const char *redactbook, int outfd, offset_t *off,
     dmu_send_outparams_t *dsop)
 {
 	int err = 0;
 	ds_hold_flags_t dsflags;
 	boolean_t owned = B_FALSE;
 	dsl_dataset_t *fromds = NULL;
 	zfs_bookmark_phys_t book = {0};
 	struct dmu_send_params dspp = {0};
 
 	dsflags = (rawok) ? DS_HOLD_FLAG_NONE : DS_HOLD_FLAG_DECRYPT;
 	dspp.tosnap = tosnap;
 	dspp.embedok = embedok;
 	dspp.large_block_ok = large_block_ok;
 	dspp.compressok = compressok;
 	dspp.outfd = outfd;
 	dspp.off = off;
 	dspp.dso = dsop;
 	dspp.tag = FTAG;
 	dspp.resumeobj = resumeobj;
 	dspp.resumeoff = resumeoff;
 	dspp.rawok = rawok;
 	dspp.savedok = savedok;
 
 	if (fromsnap != NULL && strpbrk(fromsnap, "@#") == NULL)
 		return (SET_ERROR(EINVAL));
 
 	err = dsl_pool_hold(tosnap, FTAG, &dspp.dp);
 	if (err != 0)
 		return (err);
 
 	if (strchr(tosnap, '@') == NULL && spa_writeable(dspp.dp->dp_spa)) {
 		/*
 		 * We are sending a filesystem or volume.  Ensure
 		 * that it doesn't change by owning the dataset.
 		 */
 
 		if (savedok) {
 			/*
 			 * We are looking for the dataset that represents the
 			 * partially received send stream. If this stream was
 			 * received as a new snapshot of an existing dataset,
 			 * this will be saved in a hidden clone named
 			 * "<pool>/<dataset>/%recv". Otherwise, the stream
 			 * will be saved in the live dataset itself. In
 			 * either case we need to use dsl_dataset_own_force()
 			 * because the stream is marked as inconsistent,
 			 * which would normally make it unavailable to be
 			 * owned.
 			 */
 			char *name = kmem_asprintf("%s/%s", tosnap,
 			    recv_clone_name);
 			err = dsl_dataset_own_force(dspp.dp, name, dsflags,
 			    FTAG, &dspp.to_ds);
 			if (err == ENOENT) {
 				err = dsl_dataset_own_force(dspp.dp, tosnap,
 				    dsflags, FTAG, &dspp.to_ds);
 			}
 
 			if (err == 0) {
 				owned = B_TRUE;
 				err = zap_lookup(dspp.dp->dp_meta_objset,
 				    dspp.to_ds->ds_object,
 				    DS_FIELD_RESUME_TOGUID, 8, 1,
 				    &dspp.saved_guid);
 			}
 
 			if (err == 0) {
 				err = zap_lookup(dspp.dp->dp_meta_objset,
 				    dspp.to_ds->ds_object,
 				    DS_FIELD_RESUME_TONAME, 1,
 				    sizeof (dspp.saved_toname),
 				    dspp.saved_toname);
 			}
 			/* Only disown if there was an error in the lookups */
 			if (owned && (err != 0))
 				dsl_dataset_disown(dspp.to_ds, dsflags, FTAG);
 
 			kmem_strfree(name);
 		} else {
 			err = dsl_dataset_own(dspp.dp, tosnap, dsflags,
 			    FTAG, &dspp.to_ds);
 			if (err == 0)
 				owned = B_TRUE;
 		}
 	} else {
 		err = dsl_dataset_hold_flags(dspp.dp, tosnap, dsflags, FTAG,
 		    &dspp.to_ds);
 	}
 
 	if (err != 0) {
 		/* Note: dsl dataset is not owned at this point */
 		dsl_pool_rele(dspp.dp, FTAG);
 		return (err);
 	}
 
 	if (redactbook != NULL) {
 		char path[ZFS_MAX_DATASET_NAME_LEN];
 		(void) strlcpy(path, tosnap, sizeof (path));
 		char *at = strchr(path, '@');
 		if (at == NULL) {
 			err = EINVAL;
 		} else {
 			(void) snprintf(at, sizeof (path) - (at - path), "#%s",
 			    redactbook);
 			err = dsl_bookmark_lookup(dspp.dp, path,
 			    NULL, &book);
 			dspp.redactbook = &book;
 		}
 	}
 
 	if (err != 0) {
 		dsl_pool_rele(dspp.dp, FTAG);
 		if (owned)
 			dsl_dataset_disown(dspp.to_ds, dsflags, FTAG);
 		else
 			dsl_dataset_rele_flags(dspp.to_ds, dsflags, FTAG);
 		return (err);
 	}
 
 	if (fromsnap != NULL) {
 		zfs_bookmark_phys_t *zb = &dspp.ancestor_zb;
 		int fsnamelen;
 		if (strpbrk(tosnap, "@#") != NULL)
 			fsnamelen = strpbrk(tosnap, "@#") - tosnap;
 		else
 			fsnamelen = strlen(tosnap);
 
 		/*
 		 * If the fromsnap is in a different filesystem, then
 		 * mark the send stream as a clone.
 		 */
 		if (strncmp(tosnap, fromsnap, fsnamelen) != 0 ||
 		    (fromsnap[fsnamelen] != '@' &&
 		    fromsnap[fsnamelen] != '#')) {
 			dspp.is_clone = B_TRUE;
 		}
 
 		if (strchr(fromsnap, '@') != NULL) {
 			err = dsl_dataset_hold(dspp.dp, fromsnap, FTAG,
 			    &fromds);
 
 			if (err != 0) {
 				ASSERT3P(fromds, ==, NULL);
 			} else {
 				/*
 				 * We need to make a deep copy of the redact
 				 * snapshots of the from snapshot, because the
 				 * array will be freed when we evict from_ds.
 				 */
 				uint64_t *fromredact;
 				if (!dsl_dataset_get_uint64_array_feature(
 				    fromds, SPA_FEATURE_REDACTED_DATASETS,
 				    &dspp.numfromredactsnaps,
 				    &fromredact)) {
 					dspp.numfromredactsnaps =
 					    NUM_SNAPS_NOT_REDACTED;
 				} else if (dspp.numfromredactsnaps > 0) {
 					uint64_t size =
 					    dspp.numfromredactsnaps *
 					    sizeof (uint64_t);
 					dspp.fromredactsnaps = kmem_zalloc(size,
 					    KM_SLEEP);
 					memcpy(dspp.fromredactsnaps, fromredact,
 					    size);
 				}
 				if (!dsl_dataset_is_before(dspp.to_ds, fromds,
 				    0)) {
 					err = SET_ERROR(EXDEV);
 				} else {
 					zb->zbm_creation_txg =
 					    dsl_dataset_phys(fromds)->
 					    ds_creation_txg;
 					zb->zbm_creation_time =
 					    dsl_dataset_phys(fromds)->
 					    ds_creation_time;
 					zb->zbm_guid =
 					    dsl_dataset_phys(fromds)->ds_guid;
 					zb->zbm_redaction_obj = 0;
 
 					if (dsl_dataset_is_zapified(fromds)) {
 						(void) zap_lookup(
 						    dspp.dp->dp_meta_objset,
 						    fromds->ds_object,
 						    DS_FIELD_IVSET_GUID, 8, 1,
 						    &zb->zbm_ivset_guid);
 					}
 				}
 				dsl_dataset_rele(fromds, FTAG);
 			}
 		} else {
 			dspp.numfromredactsnaps = NUM_SNAPS_NOT_REDACTED;
 			err = dsl_bookmark_lookup(dspp.dp, fromsnap, dspp.to_ds,
 			    zb);
 			if (err == EXDEV && zb->zbm_redaction_obj != 0 &&
 			    zb->zbm_guid ==
 			    dsl_dataset_phys(dspp.to_ds)->ds_guid)
 				err = 0;
 		}
 
 		if (err == 0) {
 			/* dmu_send_impl will call dsl_pool_rele for us. */
 			err = dmu_send_impl(&dspp);
 		} else {
 			if (dspp.fromredactsnaps)
 				kmem_free(dspp.fromredactsnaps,
 				    dspp.numfromredactsnaps *
 				    sizeof (uint64_t));
 			dsl_pool_rele(dspp.dp, FTAG);
 		}
 	} else {
 		dspp.numfromredactsnaps = NUM_SNAPS_NOT_REDACTED;
 		err = dmu_send_impl(&dspp);
 	}
 	if (owned)
 		dsl_dataset_disown(dspp.to_ds, dsflags, FTAG);
 	else
 		dsl_dataset_rele_flags(dspp.to_ds, dsflags, FTAG);
 	return (err);
 }
 
 static int
 dmu_adjust_send_estimate_for_indirects(dsl_dataset_t *ds, uint64_t uncompressed,
     uint64_t compressed, boolean_t stream_compressed, uint64_t *sizep)
 {
 	int err = 0;
 	uint64_t size;
 	/*
 	 * Assume that space (both on-disk and in-stream) is dominated by
 	 * data.  We will adjust for indirect blocks and the copies property,
 	 * but ignore per-object space used (eg, dnodes and DRR_OBJECT records).
 	 */
 
 	uint64_t recordsize;
 	uint64_t record_count;
 	objset_t *os;
 	VERIFY0(dmu_objset_from_ds(ds, &os));
 
 	/* Assume all (uncompressed) blocks are recordsize. */
 	if (zfs_override_estimate_recordsize != 0) {
 		recordsize = zfs_override_estimate_recordsize;
 	} else if (os->os_phys->os_type == DMU_OST_ZVOL) {
 		err = dsl_prop_get_int_ds(ds,
 		    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &recordsize);
 	} else {
 		err = dsl_prop_get_int_ds(ds,
 		    zfs_prop_to_name(ZFS_PROP_RECORDSIZE), &recordsize);
 	}
 	if (err != 0)
 		return (err);
 	record_count = uncompressed / recordsize;
 
 	/*
 	 * If we're estimating a send size for a compressed stream, use the
 	 * compressed data size to estimate the stream size. Otherwise, use the
 	 * uncompressed data size.
 	 */
 	size = stream_compressed ? compressed : uncompressed;
 
 	/*
 	 * Subtract out approximate space used by indirect blocks.
 	 * Assume most space is used by data blocks (non-indirect, non-dnode).
 	 * Assume no ditto blocks or internal fragmentation.
 	 *
 	 * Therefore, space used by indirect blocks is sizeof(blkptr_t) per
 	 * block.
 	 */
 	size -= record_count * sizeof (blkptr_t);
 
 	/* Add in the space for the record associated with each block. */
 	size += record_count * sizeof (dmu_replay_record_t);
 
 	*sizep = size;
 
 	return (0);
 }
 
 int
 dmu_send_estimate_fast(dsl_dataset_t *origds, dsl_dataset_t *fromds,
     zfs_bookmark_phys_t *frombook, boolean_t stream_compressed,
     boolean_t saved, uint64_t *sizep)
 {
 	int err;
 	dsl_dataset_t *ds = origds;
 	uint64_t uncomp, comp;
 
 	ASSERT(dsl_pool_config_held(origds->ds_dir->dd_pool));
 	ASSERT(fromds == NULL || frombook == NULL);
 
 	/*
 	 * If this is a saved send we may actually be sending
 	 * from the %recv clone used for resuming.
 	 */
 	if (saved) {
 		objset_t *mos = origds->ds_dir->dd_pool->dp_meta_objset;
 		uint64_t guid;
 		char dsname[ZFS_MAX_DATASET_NAME_LEN + 6];
 
 		dsl_dataset_name(origds, dsname);
 		(void) strcat(dsname, "/");
 		(void) strlcat(dsname, recv_clone_name, sizeof (dsname));
 
 		err = dsl_dataset_hold(origds->ds_dir->dd_pool,
 		    dsname, FTAG, &ds);
 		if (err != ENOENT && err != 0) {
 			return (err);
 		} else if (err == ENOENT) {
 			ds = origds;
 		}
 
 		/* check that this dataset has partially received data */
 		err = zap_lookup(mos, ds->ds_object,
 		    DS_FIELD_RESUME_TOGUID, 8, 1, &guid);
 		if (err != 0) {
 			err = SET_ERROR(err == ENOENT ? EINVAL : err);
 			goto out;
 		}
 
 		err = zap_lookup(mos, ds->ds_object,
 		    DS_FIELD_RESUME_TONAME, 1, sizeof (dsname), dsname);
 		if (err != 0) {
 			err = SET_ERROR(err == ENOENT ? EINVAL : err);
 			goto out;
 		}
 	}
 
 	/* tosnap must be a snapshot or the target of a saved send */
 	if (!ds->ds_is_snapshot && ds == origds)
 		return (SET_ERROR(EINVAL));
 
 	if (fromds != NULL) {
 		uint64_t used;
 		if (!fromds->ds_is_snapshot) {
 			err = SET_ERROR(EINVAL);
 			goto out;
 		}
 
 		if (!dsl_dataset_is_before(ds, fromds, 0)) {
 			err = SET_ERROR(EXDEV);
 			goto out;
 		}
 
 		err = dsl_dataset_space_written(fromds, ds, &used, &comp,
 		    &uncomp);
 		if (err != 0)
 			goto out;
 	} else if (frombook != NULL) {
 		uint64_t used;
 		err = dsl_dataset_space_written_bookmark(frombook, ds, &used,
 		    &comp, &uncomp);
 		if (err != 0)
 			goto out;
 	} else {
 		uncomp = dsl_dataset_phys(ds)->ds_uncompressed_bytes;
 		comp = dsl_dataset_phys(ds)->ds_compressed_bytes;
 	}
 
 	err = dmu_adjust_send_estimate_for_indirects(ds, uncomp, comp,
 	    stream_compressed, sizep);
 	/*
 	 * Add the size of the BEGIN and END records to the estimate.
 	 */
 	*sizep += 2 * sizeof (dmu_replay_record_t);
 
 out:
 	if (ds != origds)
 		dsl_dataset_rele(ds, FTAG);
 	return (err);
 }
 
 ZFS_MODULE_PARAM(zfs_send, zfs_send_, corrupt_data, INT, ZMOD_RW,
 	"Allow sending corrupt data");
 
 ZFS_MODULE_PARAM(zfs_send, zfs_send_, queue_length, UINT, ZMOD_RW,
 	"Maximum send queue length");
 
 ZFS_MODULE_PARAM(zfs_send, zfs_send_, unmodified_spill_blocks, INT, ZMOD_RW,
 	"Send unmodified spill blocks");
 
 ZFS_MODULE_PARAM(zfs_send, zfs_send_, no_prefetch_queue_length, UINT, ZMOD_RW,
 	"Maximum send queue length for non-prefetch queues");
 
 ZFS_MODULE_PARAM(zfs_send, zfs_send_, queue_ff, UINT, ZMOD_RW,
 	"Send queue fill fraction");
 
 ZFS_MODULE_PARAM(zfs_send, zfs_send_, no_prefetch_queue_ff, UINT, ZMOD_RW,
 	"Send queue fill fraction for non-prefetch queues");
 
 ZFS_MODULE_PARAM(zfs_send, zfs_, override_estimate_recordsize, UINT, ZMOD_RW,
 	"Override block size estimate with fixed size");
diff --git a/sys/contrib/openzfs/module/zfs/dsl_dataset.c b/sys/contrib/openzfs/module/zfs/dsl_dataset.c
index 85b598df3f49..c65f7ca03daf 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_dataset.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_dataset.c
@@ -1,5024 +1,5033 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2020 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 RackTop Systems.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
  * Copyright 2016, OmniTI Computer Consulting, Inc. All rights reserved.
  * Copyright 2017 Nexenta Systems, Inc.
  * Copyright (c) 2019, Klara Inc.
  * Copyright (c) 2019, Allan Jude
  * Copyright (c) 2020 The FreeBSD Foundation [1]
+ * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
  *
  * [1] Portions of this software were developed by Allan Jude
  *     under sponsorship from the FreeBSD Foundation.
  */
 
 #include <sys/dmu_objset.h>
 #include <sys/dsl_dataset.h>
 #include <sys/dsl_dir.h>
 #include <sys/dsl_prop.h>
 #include <sys/dsl_synctask.h>
 #include <sys/dmu_traverse.h>
 #include <sys/dmu_impl.h>
 #include <sys/dmu_tx.h>
 #include <sys/arc.h>
 #include <sys/zio.h>
 #include <sys/zap.h>
 #include <sys/zfeature.h>
 #include <sys/unique.h>
 #include <sys/zfs_context.h>
 #include <sys/zfs_ioctl.h>
 #include <sys/spa.h>
 #include <sys/spa_impl.h>
 #include <sys/vdev.h>
 #include <sys/zfs_znode.h>
 #include <sys/zfs_onexit.h>
 #include <sys/zvol.h>
 #include <sys/dsl_scan.h>
 #include <sys/dsl_deadlist.h>
 #include <sys/dsl_destroy.h>
 #include <sys/dsl_userhold.h>
 #include <sys/dsl_bookmark.h>
 #include <sys/policy.h>
 #include <sys/dmu_send.h>
 #include <sys/dmu_recv.h>
 #include <sys/zio_compress.h>
 #include <zfs_fletcher.h>
 #include <sys/zio_checksum.h>
 
 /*
  * The SPA supports block sizes up to 16MB.  However, very large blocks
  * can have an impact on i/o latency (e.g. tying up a spinning disk for
  * ~300ms), and also potentially on the memory allocator.  Therefore,
  * we did not allow the recordsize to be set larger than zfs_max_recordsize
  * (former default: 1MB).  Larger blocks could be created by changing this
  * tunable, and pools with larger blocks could always be imported and used,
  * regardless of this setting.
  *
  * We do, however, still limit it by default to 1M on x86_32, because Linux's
  * 3/1 memory split doesn't leave much room for 16M chunks.
  */
 #ifdef _ILP32
 uint_t zfs_max_recordsize =  1 * 1024 * 1024;
 #else
 uint_t zfs_max_recordsize = 16 * 1024 * 1024;
 #endif
 static int zfs_allow_redacted_dataset_mount = 0;
 
 int zfs_snapshot_history_enabled = 1;
 
 #define	SWITCH64(x, y) \
 	{ \
 		uint64_t __tmp = (x); \
 		(x) = (y); \
 		(y) = __tmp; \
 	}
 
 #define	DS_REF_MAX	(1ULL << 62)
 
 static void dsl_dataset_set_remap_deadlist_object(dsl_dataset_t *ds,
     uint64_t obj, dmu_tx_t *tx);
 static void dsl_dataset_unset_remap_deadlist_object(dsl_dataset_t *ds,
     dmu_tx_t *tx);
 
 static void unload_zfeature(dsl_dataset_t *ds, spa_feature_t f);
 
 extern uint_t spa_asize_inflation;
 
 static zil_header_t zero_zil;
 
 /*
  * Figure out how much of this delta should be propagated to the dsl_dir
  * layer.  If there's a refreservation, that space has already been
  * partially accounted for in our ancestors.
  */
 static int64_t
 parent_delta(dsl_dataset_t *ds, int64_t delta)
 {
 	dsl_dataset_phys_t *ds_phys;
 	uint64_t old_bytes, new_bytes;
 
 	if (ds->ds_reserved == 0)
 		return (delta);
 
 	ds_phys = dsl_dataset_phys(ds);
 	old_bytes = MAX(ds_phys->ds_unique_bytes, ds->ds_reserved);
 	new_bytes = MAX(ds_phys->ds_unique_bytes + delta, ds->ds_reserved);
 
 	ASSERT3U(ABS((int64_t)(new_bytes - old_bytes)), <=, ABS(delta));
 	return (new_bytes - old_bytes);
 }
 
 void
 dsl_dataset_block_born(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx)
 {
 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
 	int used = bp_get_dsize_sync(spa, bp);
 	int compressed = BP_GET_PSIZE(bp);
 	int uncompressed = BP_GET_UCSIZE(bp);
 	int64_t delta;
 	spa_feature_t f;
 
 	dprintf_bp(bp, "ds=%p", ds);
 
 	ASSERT(dmu_tx_is_syncing(tx));
 	/* It could have been compressed away to nothing */
 	if (BP_IS_HOLE(bp) || BP_IS_REDACTED(bp))
 		return;
 	ASSERT(BP_GET_TYPE(bp) != DMU_OT_NONE);
 	ASSERT(DMU_OT_IS_VALID(BP_GET_TYPE(bp)));
 	if (ds == NULL) {
 		dsl_pool_mos_diduse_space(tx->tx_pool,
 		    used, compressed, uncompressed);
 		return;
 	}
 
 	ASSERT3U(bp->blk_birth, >, dsl_dataset_phys(ds)->ds_prev_snap_txg);
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
 	mutex_enter(&ds->ds_lock);
 	delta = parent_delta(ds, used);
 	dsl_dataset_phys(ds)->ds_referenced_bytes += used;
 	dsl_dataset_phys(ds)->ds_compressed_bytes += compressed;
 	dsl_dataset_phys(ds)->ds_uncompressed_bytes += uncompressed;
 	dsl_dataset_phys(ds)->ds_unique_bytes += used;
 
 	if (BP_GET_LSIZE(bp) > SPA_OLD_MAXBLOCKSIZE) {
 		ds->ds_feature_activation[SPA_FEATURE_LARGE_BLOCKS] =
 		    (void *)B_TRUE;
 	}
 
 
 	f = zio_checksum_to_feature(BP_GET_CHECKSUM(bp));
 	if (f != SPA_FEATURE_NONE) {
 		ASSERT3S(spa_feature_table[f].fi_type, ==,
 		    ZFEATURE_TYPE_BOOLEAN);
 		ds->ds_feature_activation[f] = (void *)B_TRUE;
 	}
 
 	f = zio_compress_to_feature(BP_GET_COMPRESS(bp));
 	if (f != SPA_FEATURE_NONE) {
 		ASSERT3S(spa_feature_table[f].fi_type, ==,
 		    ZFEATURE_TYPE_BOOLEAN);
 		ds->ds_feature_activation[f] = (void *)B_TRUE;
 	}
 
 	/*
 	 * Track block for livelist, but ignore embedded blocks because
 	 * they do not need to be freed.
 	 */
 	if (dsl_deadlist_is_open(&ds->ds_dir->dd_livelist) &&
 	    bp->blk_birth > ds->ds_dir->dd_origin_txg &&
 	    !(BP_IS_EMBEDDED(bp))) {
 		ASSERT(dsl_dir_is_clone(ds->ds_dir));
 		ASSERT(spa_feature_is_enabled(spa,
 		    SPA_FEATURE_LIVELIST));
 		bplist_append(&ds->ds_dir->dd_pending_allocs, bp);
 	}
 
 	mutex_exit(&ds->ds_lock);
 	dsl_dir_diduse_transfer_space(ds->ds_dir, delta,
 	    compressed, uncompressed, used,
 	    DD_USED_REFRSRV, DD_USED_HEAD, tx);
 }
 
 /*
  * Called when the specified segment has been remapped, and is thus no
  * longer referenced in the head dataset.  The vdev must be indirect.
  *
  * If the segment is referenced by a snapshot, put it on the remap deadlist.
  * Otherwise, add this segment to the obsolete spacemap.
  */
 void
 dsl_dataset_block_remapped(dsl_dataset_t *ds, uint64_t vdev, uint64_t offset,
     uint64_t size, uint64_t birth, dmu_tx_t *tx)
 {
 	spa_t *spa = ds->ds_dir->dd_pool->dp_spa;
 
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(birth <= tx->tx_txg);
 	ASSERT(!ds->ds_is_snapshot);
 
 	if (birth > dsl_dataset_phys(ds)->ds_prev_snap_txg) {
 		spa_vdev_indirect_mark_obsolete(spa, vdev, offset, size, tx);
 	} else {
 		blkptr_t fakebp;
 		dva_t *dva = &fakebp.blk_dva[0];
 
 		ASSERT(ds != NULL);
 
 		mutex_enter(&ds->ds_remap_deadlist_lock);
 		if (!dsl_dataset_remap_deadlist_exists(ds)) {
 			dsl_dataset_create_remap_deadlist(ds, tx);
 		}
 		mutex_exit(&ds->ds_remap_deadlist_lock);
 
 		BP_ZERO(&fakebp);
 		fakebp.blk_birth = birth;
 		DVA_SET_VDEV(dva, vdev);
 		DVA_SET_OFFSET(dva, offset);
 		DVA_SET_ASIZE(dva, size);
 		dsl_deadlist_insert(&ds->ds_remap_deadlist, &fakebp, B_FALSE,
 		    tx);
 	}
 }
 
 int
 dsl_dataset_block_kill(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx,
     boolean_t async)
 {
 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
 
 	int used = bp_get_dsize_sync(spa, bp);
 	int compressed = BP_GET_PSIZE(bp);
 	int uncompressed = BP_GET_UCSIZE(bp);
 
 	if (BP_IS_HOLE(bp) || BP_IS_REDACTED(bp))
 		return (0);
 
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(bp->blk_birth <= tx->tx_txg);
 
 	if (ds == NULL) {
 		dsl_free(tx->tx_pool, tx->tx_txg, bp);
 		dsl_pool_mos_diduse_space(tx->tx_pool,
 		    -used, -compressed, -uncompressed);
 		return (used);
 	}
 	ASSERT3P(tx->tx_pool, ==, ds->ds_dir->dd_pool);
 
 	ASSERT(!ds->ds_is_snapshot);
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
 
 	/*
 	 * Track block for livelist, but ignore embedded blocks because
 	 * they do not need to be freed.
 	 */
 	if (dsl_deadlist_is_open(&ds->ds_dir->dd_livelist) &&
 	    bp->blk_birth > ds->ds_dir->dd_origin_txg &&
 	    !(BP_IS_EMBEDDED(bp))) {
 		ASSERT(dsl_dir_is_clone(ds->ds_dir));
 		ASSERT(spa_feature_is_enabled(spa,
 		    SPA_FEATURE_LIVELIST));
 		bplist_append(&ds->ds_dir->dd_pending_frees, bp);
 	}
 
 	if (bp->blk_birth > dsl_dataset_phys(ds)->ds_prev_snap_txg) {
 		int64_t delta;
 
 		dprintf_bp(bp, "freeing ds=%llu", (u_longlong_t)ds->ds_object);
 		dsl_free(tx->tx_pool, tx->tx_txg, bp);
 
 		mutex_enter(&ds->ds_lock);
 		ASSERT(dsl_dataset_phys(ds)->ds_unique_bytes >= used ||
 		    !DS_UNIQUE_IS_ACCURATE(ds));
 		delta = parent_delta(ds, -used);
 		dsl_dataset_phys(ds)->ds_unique_bytes -= used;
 		mutex_exit(&ds->ds_lock);
 		dsl_dir_diduse_transfer_space(ds->ds_dir,
 		    delta, -compressed, -uncompressed, -used,
 		    DD_USED_REFRSRV, DD_USED_HEAD, tx);
 	} else {
 		dprintf_bp(bp, "putting on dead list: %s", "");
 		if (async) {
 			/*
 			 * We are here as part of zio's write done callback,
 			 * which means we're a zio interrupt thread.  We can't
 			 * call dsl_deadlist_insert() now because it may block
 			 * waiting for I/O.  Instead, put bp on the deferred
 			 * queue and let dsl_pool_sync() finish the job.
 			 */
 			bplist_append(&ds->ds_pending_deadlist, bp);
 		} else {
 			dsl_deadlist_insert(&ds->ds_deadlist, bp, B_FALSE, tx);
 		}
 		ASSERT3U(ds->ds_prev->ds_object, ==,
 		    dsl_dataset_phys(ds)->ds_prev_snap_obj);
 		ASSERT(dsl_dataset_phys(ds->ds_prev)->ds_num_children > 0);
 		/* if (bp->blk_birth > prev prev snap txg) prev unique += bs */
 		if (dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj ==
 		    ds->ds_object && bp->blk_birth >
 		    dsl_dataset_phys(ds->ds_prev)->ds_prev_snap_txg) {
 			dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
 			mutex_enter(&ds->ds_prev->ds_lock);
 			dsl_dataset_phys(ds->ds_prev)->ds_unique_bytes += used;
 			mutex_exit(&ds->ds_prev->ds_lock);
 		}
 		if (bp->blk_birth > ds->ds_dir->dd_origin_txg) {
 			dsl_dir_transfer_space(ds->ds_dir, used,
 			    DD_USED_HEAD, DD_USED_SNAP, tx);
 		}
 	}
 
 	dsl_bookmark_block_killed(ds, bp, tx);
 
 	mutex_enter(&ds->ds_lock);
 	ASSERT3U(dsl_dataset_phys(ds)->ds_referenced_bytes, >=, used);
 	dsl_dataset_phys(ds)->ds_referenced_bytes -= used;
 	ASSERT3U(dsl_dataset_phys(ds)->ds_compressed_bytes, >=, compressed);
 	dsl_dataset_phys(ds)->ds_compressed_bytes -= compressed;
 	ASSERT3U(dsl_dataset_phys(ds)->ds_uncompressed_bytes, >=, uncompressed);
 	dsl_dataset_phys(ds)->ds_uncompressed_bytes -= uncompressed;
 	mutex_exit(&ds->ds_lock);
 
 	return (used);
 }
 
 struct feature_type_uint64_array_arg {
 	uint64_t length;
 	uint64_t *array;
 };
 
 static void
 unload_zfeature(dsl_dataset_t *ds, spa_feature_t f)
 {
 	switch (spa_feature_table[f].fi_type) {
 	case ZFEATURE_TYPE_BOOLEAN:
 		break;
 	case ZFEATURE_TYPE_UINT64_ARRAY:
 	{
 		struct feature_type_uint64_array_arg *ftuaa = ds->ds_feature[f];
 		kmem_free(ftuaa->array, ftuaa->length * sizeof (uint64_t));
 		kmem_free(ftuaa, sizeof (*ftuaa));
 		break;
 	}
 	default:
 		panic("Invalid zfeature type %d", spa_feature_table[f].fi_type);
 	}
 }
 
 static int
 load_zfeature(objset_t *mos, dsl_dataset_t *ds, spa_feature_t f)
 {
 	int err = 0;
 	switch (spa_feature_table[f].fi_type) {
 	case ZFEATURE_TYPE_BOOLEAN:
 		err = zap_contains(mos, ds->ds_object,
 		    spa_feature_table[f].fi_guid);
 		if (err == 0) {
 			ds->ds_feature[f] = (void *)B_TRUE;
 		} else {
 			ASSERT3U(err, ==, ENOENT);
 			err = 0;
 		}
 		break;
 	case ZFEATURE_TYPE_UINT64_ARRAY:
 	{
 		uint64_t int_size, num_int;
 		uint64_t *data;
 		err = zap_length(mos, ds->ds_object,
 		    spa_feature_table[f].fi_guid, &int_size, &num_int);
 		if (err != 0) {
 			ASSERT3U(err, ==, ENOENT);
 			err = 0;
 			break;
 		}
 		ASSERT3U(int_size, ==, sizeof (uint64_t));
 		data = kmem_alloc(int_size * num_int, KM_SLEEP);
 		VERIFY0(zap_lookup(mos, ds->ds_object,
 		    spa_feature_table[f].fi_guid, int_size, num_int, data));
 		struct feature_type_uint64_array_arg *ftuaa =
 		    kmem_alloc(sizeof (*ftuaa), KM_SLEEP);
 		ftuaa->length = num_int;
 		ftuaa->array = data;
 		ds->ds_feature[f] = ftuaa;
 		break;
 	}
 	default:
 		panic("Invalid zfeature type %d", spa_feature_table[f].fi_type);
 	}
 	return (err);
 }
 
 /*
  * We have to release the fsid synchronously or we risk that a subsequent
  * mount of the same dataset will fail to unique_insert the fsid.  This
  * failure would manifest itself as the fsid of this dataset changing
  * between mounts which makes NFS clients quite unhappy.
  */
 static void
 dsl_dataset_evict_sync(void *dbu)
 {
 	dsl_dataset_t *ds = dbu;
 
 	ASSERT(ds->ds_owner == NULL);
 
 	unique_remove(ds->ds_fsid_guid);
 }
 
 static void
 dsl_dataset_evict_async(void *dbu)
 {
 	dsl_dataset_t *ds = dbu;
 
 	ASSERT(ds->ds_owner == NULL);
 
 	ds->ds_dbuf = NULL;
 
 	if (ds->ds_objset != NULL)
 		dmu_objset_evict(ds->ds_objset);
 
 	if (ds->ds_prev) {
 		dsl_dataset_rele(ds->ds_prev, ds);
 		ds->ds_prev = NULL;
 	}
 
 	dsl_bookmark_fini_ds(ds);
 
 	bplist_destroy(&ds->ds_pending_deadlist);
 	if (dsl_deadlist_is_open(&ds->ds_deadlist))
 		dsl_deadlist_close(&ds->ds_deadlist);
 	if (dsl_deadlist_is_open(&ds->ds_remap_deadlist))
 		dsl_deadlist_close(&ds->ds_remap_deadlist);
 	if (ds->ds_dir)
 		dsl_dir_async_rele(ds->ds_dir, ds);
 
 	ASSERT(!list_link_active(&ds->ds_synced_link));
 
 	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
 		if (dsl_dataset_feature_is_active(ds, f))
 			unload_zfeature(ds, f);
 	}
 
 	list_destroy(&ds->ds_prop_cbs);
 	mutex_destroy(&ds->ds_lock);
 	mutex_destroy(&ds->ds_opening_lock);
 	mutex_destroy(&ds->ds_sendstream_lock);
 	mutex_destroy(&ds->ds_remap_deadlist_lock);
 	zfs_refcount_destroy(&ds->ds_longholds);
 	rrw_destroy(&ds->ds_bp_rwlock);
 
 	kmem_free(ds, sizeof (dsl_dataset_t));
 }
 
 int
 dsl_dataset_get_snapname(dsl_dataset_t *ds)
 {
 	dsl_dataset_phys_t *headphys;
 	int err;
 	dmu_buf_t *headdbuf;
 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
 	objset_t *mos = dp->dp_meta_objset;
 
 	if (ds->ds_snapname[0])
 		return (0);
 	if (dsl_dataset_phys(ds)->ds_next_snap_obj == 0)
 		return (0);
 
 	err = dmu_bonus_hold(mos, dsl_dir_phys(ds->ds_dir)->dd_head_dataset_obj,
 	    FTAG, &headdbuf);
 	if (err != 0)
 		return (err);
 	headphys = headdbuf->db_data;
 	err = zap_value_search(dp->dp_meta_objset,
 	    headphys->ds_snapnames_zapobj, ds->ds_object, 0, ds->ds_snapname);
 	if (err != 0 && zfs_recover == B_TRUE) {
 		err = 0;
 		(void) snprintf(ds->ds_snapname, sizeof (ds->ds_snapname),
 		    "SNAPOBJ=%llu-ERR=%d",
 		    (unsigned long long)ds->ds_object, err);
 	}
 	dmu_buf_rele(headdbuf, FTAG);
 	return (err);
 }
 
 int
 dsl_dataset_snap_lookup(dsl_dataset_t *ds, const char *name, uint64_t *value)
 {
 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
 	uint64_t snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
 	matchtype_t mt = 0;
 	int err;
 
 	if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
 		mt = MT_NORMALIZE;
 
 	err = zap_lookup_norm(mos, snapobj, name, 8, 1,
 	    value, mt, NULL, 0, NULL);
 	if (err == ENOTSUP && (mt & MT_NORMALIZE))
 		err = zap_lookup(mos, snapobj, name, 8, 1, value);
 	return (err);
 }
 
 int
 dsl_dataset_snap_remove(dsl_dataset_t *ds, const char *name, dmu_tx_t *tx,
     boolean_t adj_cnt)
 {
 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
 	uint64_t snapobj = dsl_dataset_phys(ds)->ds_snapnames_zapobj;
 	matchtype_t mt = 0;
 	int err;
 
 	dsl_dir_snap_cmtime_update(ds->ds_dir, tx);
 
 	if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
 		mt = MT_NORMALIZE;
 
 	err = zap_remove_norm(mos, snapobj, name, mt, tx);
 	if (err == ENOTSUP && (mt & MT_NORMALIZE))
 		err = zap_remove(mos, snapobj, name, tx);
 
 	if (err == 0 && adj_cnt)
 		dsl_fs_ss_count_adjust(ds->ds_dir, -1,
 		    DD_FIELD_SNAPSHOT_COUNT, tx);
 
 	return (err);
 }
 
 boolean_t
 dsl_dataset_try_add_ref(dsl_pool_t *dp, dsl_dataset_t *ds, const void *tag)
 {
 	dmu_buf_t *dbuf = ds->ds_dbuf;
 	boolean_t result = B_FALSE;
 
 	if (dbuf != NULL && dmu_buf_try_add_ref(dbuf, dp->dp_meta_objset,
 	    ds->ds_object, DMU_BONUS_BLKID, tag)) {
 
 		if (ds == dmu_buf_get_user(dbuf))
 			result = B_TRUE;
 		else
 			dmu_buf_rele(dbuf, tag);
 	}
 
 	return (result);
 }
 
 int
 dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, const void *tag,
     dsl_dataset_t **dsp)
 {
 	objset_t *mos = dp->dp_meta_objset;
 	dmu_buf_t *dbuf;
 	dsl_dataset_t *ds;
 	int err;
 	dmu_object_info_t doi;
 
 	ASSERT(dsl_pool_config_held(dp));
 
 	err = dmu_bonus_hold(mos, dsobj, tag, &dbuf);
 	if (err != 0)
 		return (err);
 
 	/* Make sure dsobj has the correct object type. */
 	dmu_object_info_from_db(dbuf, &doi);
 	if (doi.doi_bonus_type != DMU_OT_DSL_DATASET) {
 		dmu_buf_rele(dbuf, tag);
 		return (SET_ERROR(EINVAL));
 	}
 
 	ds = dmu_buf_get_user(dbuf);
 	if (ds == NULL) {
 		dsl_dataset_t *winner = NULL;
 
 		ds = kmem_zalloc(sizeof (dsl_dataset_t), KM_SLEEP);
 		ds->ds_dbuf = dbuf;
 		ds->ds_object = dsobj;
 		ds->ds_is_snapshot = dsl_dataset_phys(ds)->ds_num_children != 0;
 		list_link_init(&ds->ds_synced_link);
 
 		err = dsl_dir_hold_obj(dp, dsl_dataset_phys(ds)->ds_dir_obj,
 		    NULL, ds, &ds->ds_dir);
 		if (err != 0) {
 			kmem_free(ds, sizeof (dsl_dataset_t));
 			dmu_buf_rele(dbuf, tag);
 			return (err);
 		}
 
 		mutex_init(&ds->ds_lock, NULL, MUTEX_DEFAULT, NULL);
 		mutex_init(&ds->ds_opening_lock, NULL, MUTEX_DEFAULT, NULL);
 		mutex_init(&ds->ds_sendstream_lock, NULL, MUTEX_DEFAULT, NULL);
 		mutex_init(&ds->ds_remap_deadlist_lock,
 		    NULL, MUTEX_DEFAULT, NULL);
 		rrw_init(&ds->ds_bp_rwlock, B_FALSE);
 		zfs_refcount_create(&ds->ds_longholds);
 
 		bplist_create(&ds->ds_pending_deadlist);
 
 		list_create(&ds->ds_sendstreams, sizeof (dmu_sendstatus_t),
 		    offsetof(dmu_sendstatus_t, dss_link));
 
 		list_create(&ds->ds_prop_cbs, sizeof (dsl_prop_cb_record_t),
 		    offsetof(dsl_prop_cb_record_t, cbr_ds_node));
 
 		if (doi.doi_type == DMU_OTN_ZAP_METADATA) {
 			spa_feature_t f;
 
 			for (f = 0; f < SPA_FEATURES; f++) {
 				if (!(spa_feature_table[f].fi_flags &
 				    ZFEATURE_FLAG_PER_DATASET))
 					continue;
 				err = load_zfeature(mos, ds, f);
 			}
 		}
 
 		if (!ds->ds_is_snapshot) {
 			ds->ds_snapname[0] = '\0';
 			if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 				err = dsl_dataset_hold_obj(dp,
 				    dsl_dataset_phys(ds)->ds_prev_snap_obj,
 				    ds, &ds->ds_prev);
 			}
 			if (err != 0)
 				goto after_dsl_bookmark_fini;
 			err = dsl_bookmark_init_ds(ds);
 		} else {
 			if (zfs_flags & ZFS_DEBUG_SNAPNAMES)
 				err = dsl_dataset_get_snapname(ds);
 			if (err == 0 &&
 			    dsl_dataset_phys(ds)->ds_userrefs_obj != 0) {
 				err = zap_count(
 				    ds->ds_dir->dd_pool->dp_meta_objset,
 				    dsl_dataset_phys(ds)->ds_userrefs_obj,
 				    &ds->ds_userrefs);
 			}
 		}
 
 		if (err == 0 && !ds->ds_is_snapshot) {
 			err = dsl_prop_get_int_ds(ds,
 			    zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
 			    &ds->ds_reserved);
 			if (err == 0) {
 				err = dsl_prop_get_int_ds(ds,
 				    zfs_prop_to_name(ZFS_PROP_REFQUOTA),
 				    &ds->ds_quota);
 			}
 		} else {
 			ds->ds_reserved = ds->ds_quota = 0;
 		}
 
 		if (err == 0 && ds->ds_dir->dd_crypto_obj != 0 &&
 		    ds->ds_is_snapshot &&
 		    zap_contains(mos, dsobj, DS_FIELD_IVSET_GUID) != 0) {
 			dp->dp_spa->spa_errata =
 			    ZPOOL_ERRATA_ZOL_8308_ENCRYPTION;
 		}
 
 		dsl_deadlist_open(&ds->ds_deadlist,
 		    mos, dsl_dataset_phys(ds)->ds_deadlist_obj);
 		uint64_t remap_deadlist_obj =
 		    dsl_dataset_get_remap_deadlist_object(ds);
 		if (remap_deadlist_obj != 0) {
 			dsl_deadlist_open(&ds->ds_remap_deadlist, mos,
 			    remap_deadlist_obj);
 		}
 
 		dmu_buf_init_user(&ds->ds_dbu, dsl_dataset_evict_sync,
 		    dsl_dataset_evict_async, &ds->ds_dbuf);
 		if (err == 0)
 			winner = dmu_buf_set_user_ie(dbuf, &ds->ds_dbu);
 
 		if (err != 0 || winner != NULL) {
 			dsl_deadlist_close(&ds->ds_deadlist);
 			if (dsl_deadlist_is_open(&ds->ds_remap_deadlist))
 				dsl_deadlist_close(&ds->ds_remap_deadlist);
 			dsl_bookmark_fini_ds(ds);
 after_dsl_bookmark_fini:
 			if (ds->ds_prev)
 				dsl_dataset_rele(ds->ds_prev, ds);
 			dsl_dir_rele(ds->ds_dir, ds);
 			for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
 				if (dsl_dataset_feature_is_active(ds, f))
 					unload_zfeature(ds, f);
 			}
 
 			list_destroy(&ds->ds_prop_cbs);
 			list_destroy(&ds->ds_sendstreams);
 			bplist_destroy(&ds->ds_pending_deadlist);
 			mutex_destroy(&ds->ds_lock);
 			mutex_destroy(&ds->ds_opening_lock);
 			mutex_destroy(&ds->ds_sendstream_lock);
 			mutex_destroy(&ds->ds_remap_deadlist_lock);
 			zfs_refcount_destroy(&ds->ds_longholds);
 			rrw_destroy(&ds->ds_bp_rwlock);
 			kmem_free(ds, sizeof (dsl_dataset_t));
 			if (err != 0) {
 				dmu_buf_rele(dbuf, tag);
 				return (err);
 			}
 			ds = winner;
 		} else {
 			ds->ds_fsid_guid =
 			    unique_insert(dsl_dataset_phys(ds)->ds_fsid_guid);
 			if (ds->ds_fsid_guid !=
 			    dsl_dataset_phys(ds)->ds_fsid_guid) {
 				zfs_dbgmsg("ds_fsid_guid changed from "
 				    "%llx to %llx for pool %s dataset id %llu",
 				    (long long)
 				    dsl_dataset_phys(ds)->ds_fsid_guid,
 				    (long long)ds->ds_fsid_guid,
 				    spa_name(dp->dp_spa),
 				    (u_longlong_t)dsobj);
 			}
 		}
 	}
 
 	ASSERT3P(ds->ds_dbuf, ==, dbuf);
 	ASSERT3P(dsl_dataset_phys(ds), ==, dbuf->db_data);
 	ASSERT(dsl_dataset_phys(ds)->ds_prev_snap_obj != 0 ||
 	    spa_version(dp->dp_spa) < SPA_VERSION_ORIGIN ||
 	    dp->dp_origin_snap == NULL || ds == dp->dp_origin_snap);
 	*dsp = ds;
 
 	return (0);
 }
 
 int
 dsl_dataset_create_key_mapping(dsl_dataset_t *ds)
 {
 	dsl_dir_t *dd = ds->ds_dir;
 
 	if (dd->dd_crypto_obj == 0)
 		return (0);
 
 	return (spa_keystore_create_mapping(dd->dd_pool->dp_spa,
 	    ds, ds, &ds->ds_key_mapping));
 }
 
 int
 dsl_dataset_hold_obj_flags(dsl_pool_t *dp, uint64_t dsobj,
     ds_hold_flags_t flags, const void *tag, dsl_dataset_t **dsp)
 {
 	int err;
 
 	err = dsl_dataset_hold_obj(dp, dsobj, tag, dsp);
 	if (err != 0)
 		return (err);
 
 	ASSERT3P(*dsp, !=, NULL);
 
 	if (flags & DS_HOLD_FLAG_DECRYPT) {
 		err = dsl_dataset_create_key_mapping(*dsp);
 		if (err != 0)
 			dsl_dataset_rele(*dsp, tag);
 	}
 
 	return (err);
 }
 
 int
 dsl_dataset_hold_flags(dsl_pool_t *dp, const char *name, ds_hold_flags_t flags,
     const void *tag, dsl_dataset_t **dsp)
 {
 	dsl_dir_t *dd;
 	const char *snapname;
 	uint64_t obj;
 	int err = 0;
 	dsl_dataset_t *ds;
 
 	err = dsl_dir_hold(dp, name, FTAG, &dd, &snapname);
 	if (err != 0)
 		return (err);
 
 	ASSERT(dsl_pool_config_held(dp));
 	obj = dsl_dir_phys(dd)->dd_head_dataset_obj;
 	if (obj != 0)
 		err = dsl_dataset_hold_obj_flags(dp, obj, flags, tag, &ds);
 	else
 		err = SET_ERROR(ENOENT);
 
 	/* we may be looking for a snapshot */
 	if (err == 0 && snapname != NULL) {
 		dsl_dataset_t *snap_ds;
 
 		if (*snapname++ != '@') {
 			dsl_dataset_rele_flags(ds, flags, tag);
 			dsl_dir_rele(dd, FTAG);
 			return (SET_ERROR(ENOENT));
 		}
 
 		dprintf("looking for snapshot '%s'\n", snapname);
 		err = dsl_dataset_snap_lookup(ds, snapname, &obj);
 		if (err == 0) {
 			err = dsl_dataset_hold_obj_flags(dp, obj, flags, tag,
 			    &snap_ds);
 		}
 		dsl_dataset_rele_flags(ds, flags, tag);
 
 		if (err == 0) {
 			mutex_enter(&snap_ds->ds_lock);
 			if (snap_ds->ds_snapname[0] == 0)
 				(void) strlcpy(snap_ds->ds_snapname, snapname,
 				    sizeof (snap_ds->ds_snapname));
 			mutex_exit(&snap_ds->ds_lock);
 			ds = snap_ds;
 		}
 	}
 	if (err == 0)
 		*dsp = ds;
 	dsl_dir_rele(dd, FTAG);
 	return (err);
 }
 
 int
 dsl_dataset_hold(dsl_pool_t *dp, const char *name, const void *tag,
     dsl_dataset_t **dsp)
 {
 	return (dsl_dataset_hold_flags(dp, name, 0, tag, dsp));
 }
 
 static int
 dsl_dataset_own_obj_impl(dsl_pool_t *dp, uint64_t dsobj, ds_hold_flags_t flags,
     const void *tag, boolean_t override, dsl_dataset_t **dsp)
 {
 	int err = dsl_dataset_hold_obj_flags(dp, dsobj, flags, tag, dsp);
 	if (err != 0)
 		return (err);
 	if (!dsl_dataset_tryown(*dsp, tag, override)) {
 		dsl_dataset_rele_flags(*dsp, flags, tag);
 		*dsp = NULL;
 		return (SET_ERROR(EBUSY));
 	}
 	return (0);
 }
 
 
 int
 dsl_dataset_own_obj(dsl_pool_t *dp, uint64_t dsobj, ds_hold_flags_t flags,
     const void *tag, dsl_dataset_t **dsp)
 {
 	return (dsl_dataset_own_obj_impl(dp, dsobj, flags, tag, B_FALSE, dsp));
 }
 
 int
 dsl_dataset_own_obj_force(dsl_pool_t *dp, uint64_t dsobj,
     ds_hold_flags_t flags, const void *tag, dsl_dataset_t **dsp)
 {
 	return (dsl_dataset_own_obj_impl(dp, dsobj, flags, tag, B_TRUE, dsp));
 }
 
 static int
 dsl_dataset_own_impl(dsl_pool_t *dp, const char *name, ds_hold_flags_t flags,
     const void *tag, boolean_t override, dsl_dataset_t **dsp)
 {
 	int err = dsl_dataset_hold_flags(dp, name, flags, tag, dsp);
 	if (err != 0)
 		return (err);
 	if (!dsl_dataset_tryown(*dsp, tag, override)) {
 		dsl_dataset_rele_flags(*dsp, flags, tag);
 		return (SET_ERROR(EBUSY));
 	}
 	return (0);
 }
 
 int
 dsl_dataset_own_force(dsl_pool_t *dp, const char *name, ds_hold_flags_t flags,
     const void *tag, dsl_dataset_t **dsp)
 {
 	return (dsl_dataset_own_impl(dp, name, flags, tag, B_TRUE, dsp));
 }
 
 int
 dsl_dataset_own(dsl_pool_t *dp, const char *name, ds_hold_flags_t flags,
     const void *tag, dsl_dataset_t **dsp)
 {
 	return (dsl_dataset_own_impl(dp, name, flags, tag, B_FALSE, dsp));
 }
 
 /*
  * See the comment above dsl_pool_hold() for details.  In summary, a long
  * hold is used to prevent destruction of a dataset while the pool hold
  * is dropped, allowing other concurrent operations (e.g. spa_sync()).
  *
  * The dataset and pool must be held when this function is called.  After it
  * is called, the pool hold may be released while the dataset is still held
  * and accessed.
  */
 void
 dsl_dataset_long_hold(dsl_dataset_t *ds, const void *tag)
 {
 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
 	(void) zfs_refcount_add(&ds->ds_longholds, tag);
 }
 
 void
 dsl_dataset_long_rele(dsl_dataset_t *ds, const void *tag)
 {
 	(void) zfs_refcount_remove(&ds->ds_longholds, tag);
 }
 
 /* Return B_TRUE if there are any long holds on this dataset. */
 boolean_t
 dsl_dataset_long_held(dsl_dataset_t *ds)
 {
 	return (!zfs_refcount_is_zero(&ds->ds_longholds));
 }
 
 void
 dsl_dataset_name(dsl_dataset_t *ds, char *name)
 {
 	if (ds == NULL) {
 		(void) strlcpy(name, "mos", ZFS_MAX_DATASET_NAME_LEN);
 	} else {
 		dsl_dir_name(ds->ds_dir, name);
 		VERIFY0(dsl_dataset_get_snapname(ds));
 		if (ds->ds_snapname[0]) {
 			VERIFY3U(strlcat(name, "@", ZFS_MAX_DATASET_NAME_LEN),
 			    <, ZFS_MAX_DATASET_NAME_LEN);
 			/*
 			 * We use a "recursive" mutex so that we
 			 * can call dprintf_ds() with ds_lock held.
 			 */
 			if (!MUTEX_HELD(&ds->ds_lock)) {
 				mutex_enter(&ds->ds_lock);
 				VERIFY3U(strlcat(name, ds->ds_snapname,
 				    ZFS_MAX_DATASET_NAME_LEN), <,
 				    ZFS_MAX_DATASET_NAME_LEN);
 				mutex_exit(&ds->ds_lock);
 			} else {
 				VERIFY3U(strlcat(name, ds->ds_snapname,
 				    ZFS_MAX_DATASET_NAME_LEN), <,
 				    ZFS_MAX_DATASET_NAME_LEN);
 			}
 		}
 	}
 }
 
 int
 dsl_dataset_namelen(dsl_dataset_t *ds)
 {
 	VERIFY0(dsl_dataset_get_snapname(ds));
 	mutex_enter(&ds->ds_lock);
 	int len = strlen(ds->ds_snapname);
 	mutex_exit(&ds->ds_lock);
 	/* add '@' if ds is a snap */
 	if (len > 0)
 		len++;
 	len += dsl_dir_namelen(ds->ds_dir);
 	return (len);
 }
 
 void
 dsl_dataset_rele(dsl_dataset_t *ds, const void *tag)
 {
 	dmu_buf_rele(ds->ds_dbuf, tag);
 }
 
 void
 dsl_dataset_remove_key_mapping(dsl_dataset_t *ds)
 {
 	dsl_dir_t *dd = ds->ds_dir;
 
 	if (dd == NULL || dd->dd_crypto_obj == 0)
 		return;
 
 	(void) spa_keystore_remove_mapping(dd->dd_pool->dp_spa,
 	    ds->ds_object, ds);
 }
 
 void
 dsl_dataset_rele_flags(dsl_dataset_t *ds, ds_hold_flags_t flags,
     const void *tag)
 {
 	if (flags & DS_HOLD_FLAG_DECRYPT)
 		dsl_dataset_remove_key_mapping(ds);
 
 	dsl_dataset_rele(ds, tag);
 }
 
 void
 dsl_dataset_disown(dsl_dataset_t *ds, ds_hold_flags_t flags, const void *tag)
 {
 	ASSERT3P(ds->ds_owner, ==, tag);
 	ASSERT(ds->ds_dbuf != NULL);
 
 	mutex_enter(&ds->ds_lock);
 	ds->ds_owner = NULL;
 	mutex_exit(&ds->ds_lock);
 	dsl_dataset_long_rele(ds, tag);
 	dsl_dataset_rele_flags(ds, flags, tag);
 }
 
 boolean_t
 dsl_dataset_tryown(dsl_dataset_t *ds, const void *tag, boolean_t override)
 {
 	boolean_t gotit = FALSE;
 
 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
 	mutex_enter(&ds->ds_lock);
 	if (ds->ds_owner == NULL && (override || !(DS_IS_INCONSISTENT(ds) ||
 	    (dsl_dataset_feature_is_active(ds,
 	    SPA_FEATURE_REDACTED_DATASETS) &&
 	    !zfs_allow_redacted_dataset_mount)))) {
 		ds->ds_owner = tag;
 		dsl_dataset_long_hold(ds, tag);
 		gotit = TRUE;
 	}
 	mutex_exit(&ds->ds_lock);
 	return (gotit);
 }
 
 boolean_t
 dsl_dataset_has_owner(dsl_dataset_t *ds)
 {
 	boolean_t rv;
 	mutex_enter(&ds->ds_lock);
 	rv = (ds->ds_owner != NULL);
 	mutex_exit(&ds->ds_lock);
 	return (rv);
 }
 
 static boolean_t
 zfeature_active(spa_feature_t f, void *arg)
 {
 	switch (spa_feature_table[f].fi_type) {
 	case ZFEATURE_TYPE_BOOLEAN: {
 		boolean_t val = (boolean_t)(uintptr_t)arg;
 		ASSERT(val == B_FALSE || val == B_TRUE);
 		return (val);
 	}
 	case ZFEATURE_TYPE_UINT64_ARRAY:
 		/*
 		 * In this case, arg is a uint64_t array.  The feature is active
 		 * if the array is non-null.
 		 */
 		return (arg != NULL);
 	default:
 		panic("Invalid zfeature type %d", spa_feature_table[f].fi_type);
 		return (B_FALSE);
 	}
 }
 
 boolean_t
 dsl_dataset_feature_is_active(dsl_dataset_t *ds, spa_feature_t f)
 {
 	return (zfeature_active(f, ds->ds_feature[f]));
 }
 
 /*
  * The buffers passed out by this function are references to internal buffers;
  * they should not be freed by callers of this function, and they should not be
  * used after the dataset has been released.
  */
 boolean_t
 dsl_dataset_get_uint64_array_feature(dsl_dataset_t *ds, spa_feature_t f,
     uint64_t *outlength, uint64_t **outp)
 {
 	VERIFY(spa_feature_table[f].fi_type & ZFEATURE_TYPE_UINT64_ARRAY);
 	if (!dsl_dataset_feature_is_active(ds, f)) {
 		return (B_FALSE);
 	}
 	struct feature_type_uint64_array_arg *ftuaa = ds->ds_feature[f];
 	*outp = ftuaa->array;
 	*outlength = ftuaa->length;
 	return (B_TRUE);
 }
 
 void
 dsl_dataset_activate_feature(uint64_t dsobj, spa_feature_t f, void *arg,
     dmu_tx_t *tx)
 {
 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
 	objset_t *mos = dmu_tx_pool(tx)->dp_meta_objset;
 	uint64_t zero = 0;
 
 	VERIFY(spa_feature_table[f].fi_flags & ZFEATURE_FLAG_PER_DATASET);
 
 	spa_feature_incr(spa, f, tx);
 	dmu_object_zapify(mos, dsobj, DMU_OT_DSL_DATASET, tx);
 
 	switch (spa_feature_table[f].fi_type) {
 	case ZFEATURE_TYPE_BOOLEAN:
 		ASSERT3S((boolean_t)(uintptr_t)arg, ==, B_TRUE);
 		VERIFY0(zap_add(mos, dsobj, spa_feature_table[f].fi_guid,
 		    sizeof (zero), 1, &zero, tx));
 		break;
 	case ZFEATURE_TYPE_UINT64_ARRAY:
 	{
 		struct feature_type_uint64_array_arg *ftuaa = arg;
 		VERIFY0(zap_add(mos, dsobj, spa_feature_table[f].fi_guid,
 		    sizeof (uint64_t), ftuaa->length, ftuaa->array, tx));
 		break;
 	}
 	default:
 		panic("Invalid zfeature type %d", spa_feature_table[f].fi_type);
 	}
 }
 
 static void
 dsl_dataset_deactivate_feature_impl(dsl_dataset_t *ds, spa_feature_t f,
     dmu_tx_t *tx)
 {
 	spa_t *spa = dmu_tx_pool(tx)->dp_spa;
 	objset_t *mos = dmu_tx_pool(tx)->dp_meta_objset;
 	uint64_t dsobj = ds->ds_object;
 
 	VERIFY(spa_feature_table[f].fi_flags & ZFEATURE_FLAG_PER_DATASET);
 
 	VERIFY0(zap_remove(mos, dsobj, spa_feature_table[f].fi_guid, tx));
 	spa_feature_decr(spa, f, tx);
 	ds->ds_feature[f] = NULL;
 }
 
 void
 dsl_dataset_deactivate_feature(dsl_dataset_t *ds, spa_feature_t f, dmu_tx_t *tx)
 {
 	unload_zfeature(ds, f);
 	dsl_dataset_deactivate_feature_impl(ds, f, tx);
 }
 
 uint64_t
 dsl_dataset_create_sync_dd(dsl_dir_t *dd, dsl_dataset_t *origin,
     dsl_crypto_params_t *dcp, uint64_t flags, dmu_tx_t *tx)
 {
 	dsl_pool_t *dp = dd->dd_pool;
 	dmu_buf_t *dbuf;
 	dsl_dataset_phys_t *dsphys;
 	uint64_t dsobj;
 	objset_t *mos = dp->dp_meta_objset;
 
 	if (origin == NULL)
 		origin = dp->dp_origin_snap;
 
 	ASSERT(origin == NULL || origin->ds_dir->dd_pool == dp);
 	ASSERT(origin == NULL || dsl_dataset_phys(origin)->ds_num_children > 0);
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(dsl_dir_phys(dd)->dd_head_dataset_obj == 0);
 
 	dsobj = dmu_object_alloc(mos, DMU_OT_DSL_DATASET, 0,
 	    DMU_OT_DSL_DATASET, sizeof (dsl_dataset_phys_t), tx);
 	VERIFY0(dmu_bonus_hold(mos, dsobj, FTAG, &dbuf));
 	dmu_buf_will_dirty(dbuf, tx);
 	dsphys = dbuf->db_data;
 	memset(dsphys, 0, sizeof (dsl_dataset_phys_t));
 	dsphys->ds_dir_obj = dd->dd_object;
 	dsphys->ds_flags = flags;
 	dsphys->ds_fsid_guid = unique_create();
 	(void) random_get_pseudo_bytes((void*)&dsphys->ds_guid,
 	    sizeof (dsphys->ds_guid));
 	dsphys->ds_snapnames_zapobj =
 	    zap_create_norm(mos, U8_TEXTPREP_TOUPPER, DMU_OT_DSL_DS_SNAP_MAP,
 	    DMU_OT_NONE, 0, tx);
 	dsphys->ds_creation_time = gethrestime_sec();
 	dsphys->ds_creation_txg = tx->tx_txg == TXG_INITIAL ? 1 : tx->tx_txg;
 
 	if (origin == NULL) {
 		dsphys->ds_deadlist_obj = dsl_deadlist_alloc(mos, tx);
 	} else {
 		dsl_dataset_t *ohds; /* head of the origin snapshot */
 
 		dsphys->ds_prev_snap_obj = origin->ds_object;
 		dsphys->ds_prev_snap_txg =
 		    dsl_dataset_phys(origin)->ds_creation_txg;
 		dsphys->ds_referenced_bytes =
 		    dsl_dataset_phys(origin)->ds_referenced_bytes;
 		dsphys->ds_compressed_bytes =
 		    dsl_dataset_phys(origin)->ds_compressed_bytes;
 		dsphys->ds_uncompressed_bytes =
 		    dsl_dataset_phys(origin)->ds_uncompressed_bytes;
 		rrw_enter(&origin->ds_bp_rwlock, RW_READER, FTAG);
 		dsphys->ds_bp = dsl_dataset_phys(origin)->ds_bp;
 		rrw_exit(&origin->ds_bp_rwlock, FTAG);
 
 		/*
 		 * Inherit flags that describe the dataset's contents
 		 * (INCONSISTENT) or properties (Case Insensitive).
 		 */
 		dsphys->ds_flags |= dsl_dataset_phys(origin)->ds_flags &
 		    (DS_FLAG_INCONSISTENT | DS_FLAG_CI_DATASET);
 
 		for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
 			if (zfeature_active(f, origin->ds_feature[f])) {
 				dsl_dataset_activate_feature(dsobj, f,
 				    origin->ds_feature[f], tx);
 			}
 		}
 
 		dmu_buf_will_dirty(origin->ds_dbuf, tx);
 		dsl_dataset_phys(origin)->ds_num_children++;
 
 		VERIFY0(dsl_dataset_hold_obj(dp,
 		    dsl_dir_phys(origin->ds_dir)->dd_head_dataset_obj,
 		    FTAG, &ohds));
 		dsphys->ds_deadlist_obj = dsl_deadlist_clone(&ohds->ds_deadlist,
 		    dsphys->ds_prev_snap_txg, dsphys->ds_prev_snap_obj, tx);
 		dsl_dataset_rele(ohds, FTAG);
 
 		if (spa_version(dp->dp_spa) >= SPA_VERSION_NEXT_CLONES) {
 			if (dsl_dataset_phys(origin)->ds_next_clones_obj == 0) {
 				dsl_dataset_phys(origin)->ds_next_clones_obj =
 				    zap_create(mos,
 				    DMU_OT_NEXT_CLONES, DMU_OT_NONE, 0, tx);
 			}
 			VERIFY0(zap_add_int(mos,
 			    dsl_dataset_phys(origin)->ds_next_clones_obj,
 			    dsobj, tx));
 		}
 
 		dmu_buf_will_dirty(dd->dd_dbuf, tx);
 		dsl_dir_phys(dd)->dd_origin_obj = origin->ds_object;
 		if (spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
 			if (dsl_dir_phys(origin->ds_dir)->dd_clones == 0) {
 				dmu_buf_will_dirty(origin->ds_dir->dd_dbuf, tx);
 				dsl_dir_phys(origin->ds_dir)->dd_clones =
 				    zap_create(mos,
 				    DMU_OT_DSL_CLONES, DMU_OT_NONE, 0, tx);
 			}
 			VERIFY0(zap_add_int(mos,
 			    dsl_dir_phys(origin->ds_dir)->dd_clones,
 			    dsobj, tx));
 		}
 	}
 
 	/* handle encryption */
 	dsl_dataset_create_crypt_sync(dsobj, dd, origin, dcp, tx);
 
 	if (spa_version(dp->dp_spa) >= SPA_VERSION_UNIQUE_ACCURATE)
 		dsphys->ds_flags |= DS_FLAG_UNIQUE_ACCURATE;
 
 	dmu_buf_rele(dbuf, FTAG);
 
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 	dsl_dir_phys(dd)->dd_head_dataset_obj = dsobj;
 
 	return (dsobj);
 }
 
 static void
 dsl_dataset_zero_zil(dsl_dataset_t *ds, dmu_tx_t *tx)
 {
 	objset_t *os;
 
 	VERIFY0(dmu_objset_from_ds(ds, &os));
 	if (memcmp(&os->os_zil_header, &zero_zil, sizeof (zero_zil)) != 0) {
 		dsl_pool_t *dp = ds->ds_dir->dd_pool;
 		zio_t *zio;
 
 		memset(&os->os_zil_header, 0, sizeof (os->os_zil_header));
 		if (os->os_encrypted)
 			os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_TRUE;
 
 		zio = zio_root(dp->dp_spa, NULL, NULL, ZIO_FLAG_MUSTSUCCEED);
 		dsl_dataset_sync(ds, zio, tx);
 		VERIFY0(zio_wait(zio));
 		dsl_dataset_sync_done(ds, tx);
 	}
 }
 
 uint64_t
 dsl_dataset_create_sync(dsl_dir_t *pdd, const char *lastname,
     dsl_dataset_t *origin, uint64_t flags, cred_t *cr,
     dsl_crypto_params_t *dcp, dmu_tx_t *tx)
 {
 	dsl_pool_t *dp = pdd->dd_pool;
 	uint64_t dsobj, ddobj;
 	dsl_dir_t *dd;
 
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(lastname[0] != '@');
 	/*
 	 * Filesystems will eventually have their origin set to dp_origin_snap,
 	 * but that's taken care of in dsl_dataset_create_sync_dd. When
 	 * creating a filesystem, this function is called with origin equal to
 	 * NULL.
 	 */
 	if (origin != NULL)
 		ASSERT3P(origin, !=, dp->dp_origin_snap);
 
 	ddobj = dsl_dir_create_sync(dp, pdd, lastname, tx);
 	VERIFY0(dsl_dir_hold_obj(dp, ddobj, lastname, FTAG, &dd));
 
 	dsobj = dsl_dataset_create_sync_dd(dd, origin, dcp,
 	    flags & ~DS_CREATE_FLAG_NODIRTY, tx);
 
 	dsl_deleg_set_create_perms(dd, tx, cr);
 
 	/*
 	 * If we are creating a clone and the livelist feature is enabled,
 	 * add the entry DD_FIELD_LIVELIST to ZAP.
 	 */
 	if (origin != NULL &&
 	    spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_LIVELIST)) {
 		objset_t *mos = dd->dd_pool->dp_meta_objset;
 		dsl_dir_zapify(dd, tx);
 		uint64_t obj = dsl_deadlist_alloc(mos, tx);
 		VERIFY0(zap_add(mos, dd->dd_object, DD_FIELD_LIVELIST,
 		    sizeof (uint64_t), 1, &obj, tx));
 		spa_feature_incr(dp->dp_spa, SPA_FEATURE_LIVELIST, tx);
 	}
 
 	/*
 	 * Since we're creating a new node we know it's a leaf, so we can
 	 * initialize the counts if the limit feature is active.
 	 */
 	if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_FS_SS_LIMIT)) {
 		uint64_t cnt = 0;
 		objset_t *os = dd->dd_pool->dp_meta_objset;
 
 		dsl_dir_zapify(dd, tx);
 		VERIFY0(zap_add(os, dd->dd_object, DD_FIELD_FILESYSTEM_COUNT,
 		    sizeof (cnt), 1, &cnt, tx));
 		VERIFY0(zap_add(os, dd->dd_object, DD_FIELD_SNAPSHOT_COUNT,
 		    sizeof (cnt), 1, &cnt, tx));
 	}
 
 	dsl_dir_rele(dd, FTAG);
 
 	/*
 	 * If we are creating a clone, make sure we zero out any stale
 	 * data from the origin snapshots zil header.
 	 */
 	if (origin != NULL && !(flags & DS_CREATE_FLAG_NODIRTY)) {
 		dsl_dataset_t *ds;
 
 		VERIFY0(dsl_dataset_hold_obj(dp, dsobj, FTAG, &ds));
 		dsl_dataset_zero_zil(ds, tx);
 		dsl_dataset_rele(ds, FTAG);
 	}
 
 	return (dsobj);
 }
 
 /*
  * The unique space in the head dataset can be calculated by subtracting
  * the space used in the most recent snapshot, that is still being used
  * in this file system, from the space currently in use.  To figure out
  * the space in the most recent snapshot still in use, we need to take
  * the total space used in the snapshot and subtract out the space that
  * has been freed up since the snapshot was taken.
  */
 void
 dsl_dataset_recalc_head_uniq(dsl_dataset_t *ds)
 {
 	uint64_t mrs_used;
 	uint64_t dlused, dlcomp, dluncomp;
 
 	ASSERT(!ds->ds_is_snapshot);
 
 	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0)
 		mrs_used = dsl_dataset_phys(ds->ds_prev)->ds_referenced_bytes;
 	else
 		mrs_used = 0;
 
 	dsl_deadlist_space(&ds->ds_deadlist, &dlused, &dlcomp, &dluncomp);
 
 	ASSERT3U(dlused, <=, mrs_used);
 	dsl_dataset_phys(ds)->ds_unique_bytes =
 	    dsl_dataset_phys(ds)->ds_referenced_bytes - (mrs_used - dlused);
 
 	if (spa_version(ds->ds_dir->dd_pool->dp_spa) >=
 	    SPA_VERSION_UNIQUE_ACCURATE)
 		dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_UNIQUE_ACCURATE;
 }
 
 void
 dsl_dataset_remove_from_next_clones(dsl_dataset_t *ds, uint64_t obj,
     dmu_tx_t *tx)
 {
 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
 	uint64_t count __maybe_unused;
 	int err;
 
 	ASSERT(dsl_dataset_phys(ds)->ds_num_children >= 2);
 	err = zap_remove_int(mos, dsl_dataset_phys(ds)->ds_next_clones_obj,
 	    obj, tx);
 	/*
 	 * The err should not be ENOENT, but a bug in a previous version
 	 * of the code could cause upgrade_clones_cb() to not set
 	 * ds_next_snap_obj when it should, leading to a missing entry.
 	 * If we knew that the pool was created after
 	 * SPA_VERSION_NEXT_CLONES, we could assert that it isn't
 	 * ENOENT.  However, at least we can check that we don't have
 	 * too many entries in the next_clones_obj even after failing to
 	 * remove this one.
 	 */
 	if (err != ENOENT)
 		VERIFY0(err);
 	ASSERT0(zap_count(mos, dsl_dataset_phys(ds)->ds_next_clones_obj,
 	    &count));
 	ASSERT3U(count, <=, dsl_dataset_phys(ds)->ds_num_children - 2);
 }
 
 
 blkptr_t *
 dsl_dataset_get_blkptr(dsl_dataset_t *ds)
 {
 	return (&dsl_dataset_phys(ds)->ds_bp);
 }
 
 spa_t *
 dsl_dataset_get_spa(dsl_dataset_t *ds)
 {
 	return (ds->ds_dir->dd_pool->dp_spa);
 }
 
 void
 dsl_dataset_dirty(dsl_dataset_t *ds, dmu_tx_t *tx)
 {
 	dsl_pool_t *dp;
 
 	if (ds == NULL) /* this is the meta-objset */
 		return;
 
 	ASSERT(ds->ds_objset != NULL);
 
 	if (dsl_dataset_phys(ds)->ds_next_snap_obj != 0)
 		panic("dirtying snapshot!");
 
 	/* Must not dirty a dataset in the same txg where it got snapshotted. */
 	ASSERT3U(tx->tx_txg, >, dsl_dataset_phys(ds)->ds_prev_snap_txg);
 
 	dp = ds->ds_dir->dd_pool;
 	if (txg_list_add(&dp->dp_dirty_datasets, ds, tx->tx_txg)) {
 		objset_t *os = ds->ds_objset;
 
 		/* up the hold count until we can be written out */
 		dmu_buf_add_ref(ds->ds_dbuf, ds);
 
 		/* if this dataset is encrypted, grab a reference to the DCK */
 		if (ds->ds_dir->dd_crypto_obj != 0 &&
 		    !os->os_raw_receive &&
 		    !os->os_next_write_raw[tx->tx_txg & TXG_MASK]) {
 			ASSERT3P(ds->ds_key_mapping, !=, NULL);
 			key_mapping_add_ref(ds->ds_key_mapping, ds);
 		}
 	}
 }
 
 static int
 dsl_dataset_snapshot_reserve_space(dsl_dataset_t *ds, dmu_tx_t *tx)
 {
 	uint64_t asize;
 
 	if (!dmu_tx_is_syncing(tx))
 		return (0);
 
 	/*
 	 * If there's an fs-only reservation, any blocks that might become
 	 * owned by the snapshot dataset must be accommodated by space
 	 * outside of the reservation.
 	 */
 	ASSERT(ds->ds_reserved == 0 || DS_UNIQUE_IS_ACCURATE(ds));
 	asize = MIN(dsl_dataset_phys(ds)->ds_unique_bytes, ds->ds_reserved);
 	if (asize > dsl_dir_space_available(ds->ds_dir, NULL, 0, TRUE))
 		return (SET_ERROR(ENOSPC));
 
 	/*
 	 * Propagate any reserved space for this snapshot to other
 	 * snapshot checks in this sync group.
 	 */
 	if (asize > 0)
 		dsl_dir_willuse_space(ds->ds_dir, asize, tx);
 
 	return (0);
 }
 
 int
 dsl_dataset_snapshot_check_impl(dsl_dataset_t *ds, const char *snapname,
-    dmu_tx_t *tx, boolean_t recv, uint64_t cnt, cred_t *cr, proc_t *proc)
+    dmu_tx_t *tx, boolean_t recv, uint64_t cnt, cred_t *cr)
 {
 	int error;
 	uint64_t value;
 
 	ds->ds_trysnap_txg = tx->tx_txg;
 
 	if (!dmu_tx_is_syncing(tx))
 		return (0);
 
 	/*
 	 * We don't allow multiple snapshots of the same txg.  If there
 	 * is already one, try again.
 	 */
 	if (dsl_dataset_phys(ds)->ds_prev_snap_txg >= tx->tx_txg)
 		return (SET_ERROR(EAGAIN));
 
 	/*
 	 * Check for conflicting snapshot name.
 	 */
 	error = dsl_dataset_snap_lookup(ds, snapname, &value);
 	if (error == 0)
 		return (SET_ERROR(EEXIST));
 	if (error != ENOENT)
 		return (error);
 
 	/*
 	 * We don't allow taking snapshots of inconsistent datasets, such as
 	 * those into which we are currently receiving.  However, if we are
 	 * creating this snapshot as part of a receive, this check will be
 	 * executed atomically with respect to the completion of the receive
 	 * itself but prior to the clearing of DS_FLAG_INCONSISTENT; in this
 	 * case we ignore this, knowing it will be fixed up for us shortly in
 	 * dmu_recv_end_sync().
 	 */
 	if (!recv && DS_IS_INCONSISTENT(ds))
 		return (SET_ERROR(EBUSY));
 
 	/*
 	 * Skip the check for temporary snapshots or if we have already checked
 	 * the counts in dsl_dataset_snapshot_check. This means we really only
 	 * check the count here when we're receiving a stream.
 	 */
 	if (cnt != 0 && cr != NULL) {
 		error = dsl_fs_ss_limit_check(ds->ds_dir, cnt,
-		    ZFS_PROP_SNAPSHOT_LIMIT, NULL, cr, proc);
+		    ZFS_PROP_SNAPSHOT_LIMIT, NULL, cr);
 		if (error != 0)
 			return (error);
 	}
 
 	error = dsl_dataset_snapshot_reserve_space(ds, tx);
 	if (error != 0)
 		return (error);
 
 	return (0);
 }
 
 int
 dsl_dataset_snapshot_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_snapshot_arg_t *ddsa = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	nvpair_t *pair;
 	int rv = 0;
 
 	/*
 	 * Pre-compute how many total new snapshots will be created for each
 	 * level in the tree and below. This is needed for validating the
 	 * snapshot limit when either taking a recursive snapshot or when
 	 * taking multiple snapshots.
 	 *
 	 * The problem is that the counts are not actually adjusted when
 	 * we are checking, only when we finally sync. For a single snapshot,
 	 * this is easy, the count will increase by 1 at each node up the tree,
 	 * but its more complicated for the recursive/multiple snapshot case.
 	 *
 	 * The dsl_fs_ss_limit_check function does recursively check the count
 	 * at each level up the tree but since it is validating each snapshot
 	 * independently we need to be sure that we are validating the complete
 	 * count for the entire set of snapshots. We do this by rolling up the
 	 * counts for each component of the name into an nvlist and then
 	 * checking each of those cases with the aggregated count.
 	 *
 	 * This approach properly handles not only the recursive snapshot
 	 * case (where we get all of those on the ddsa_snaps list) but also
 	 * the sibling case (e.g. snapshot a/b and a/c so that we will also
 	 * validate the limit on 'a' using a count of 2).
 	 *
 	 * We validate the snapshot names in the third loop and only report
 	 * name errors once.
 	 */
 	if (dmu_tx_is_syncing(tx)) {
 		char *nm;
 		nvlist_t *cnt_track = NULL;
 		cnt_track = fnvlist_alloc();
 
 		nm = kmem_alloc(MAXPATHLEN, KM_SLEEP);
 
 		/* Rollup aggregated counts into the cnt_track list */
 		for (pair = nvlist_next_nvpair(ddsa->ddsa_snaps, NULL);
 		    pair != NULL;
 		    pair = nvlist_next_nvpair(ddsa->ddsa_snaps, pair)) {
 			char *pdelim;
 			uint64_t val;
 
 			(void) strlcpy(nm, nvpair_name(pair), MAXPATHLEN);
 			pdelim = strchr(nm, '@');
 			if (pdelim == NULL)
 				continue;
 			*pdelim = '\0';
 
 			do {
 				if (nvlist_lookup_uint64(cnt_track, nm,
 				    &val) == 0) {
 					/* update existing entry */
 					fnvlist_add_uint64(cnt_track, nm,
 					    val + 1);
 				} else {
 					/* add to list */
 					fnvlist_add_uint64(cnt_track, nm, 1);
 				}
 
 				pdelim = strrchr(nm, '/');
 				if (pdelim != NULL)
 					*pdelim = '\0';
 			} while (pdelim != NULL);
 		}
 
 		kmem_free(nm, MAXPATHLEN);
 
 		/* Check aggregated counts at each level */
 		for (pair = nvlist_next_nvpair(cnt_track, NULL);
 		    pair != NULL; pair = nvlist_next_nvpair(cnt_track, pair)) {
 			int error = 0;
 			const char *name;
 			uint64_t cnt = 0;
 			dsl_dataset_t *ds;
 
 			name = nvpair_name(pair);
 			cnt = fnvpair_value_uint64(pair);
 			ASSERT(cnt > 0);
 
 			error = dsl_dataset_hold(dp, name, FTAG, &ds);
 			if (error == 0) {
 				error = dsl_fs_ss_limit_check(ds->ds_dir, cnt,
 				    ZFS_PROP_SNAPSHOT_LIMIT, NULL,
-				    ddsa->ddsa_cr, ddsa->ddsa_proc);
+				    ddsa->ddsa_cr);
 				dsl_dataset_rele(ds, FTAG);
 			}
 
 			if (error != 0) {
 				if (ddsa->ddsa_errors != NULL)
 					fnvlist_add_int32(ddsa->ddsa_errors,
 					    name, error);
 				rv = error;
 				/* only report one error for this check */
 				break;
 			}
 		}
 		nvlist_free(cnt_track);
 	}
 
 	for (pair = nvlist_next_nvpair(ddsa->ddsa_snaps, NULL);
 	    pair != NULL; pair = nvlist_next_nvpair(ddsa->ddsa_snaps, pair)) {
 		int error = 0;
 		dsl_dataset_t *ds;
 		const char *name, *atp = NULL;
 		char dsname[ZFS_MAX_DATASET_NAME_LEN];
 
 		name = nvpair_name(pair);
 		if (strlen(name) >= ZFS_MAX_DATASET_NAME_LEN)
 			error = SET_ERROR(ENAMETOOLONG);
 		if (error == 0) {
 			atp = strchr(name, '@');
 			if (atp == NULL)
 				error = SET_ERROR(EINVAL);
 			if (error == 0)
 				(void) strlcpy(dsname, name, atp - name + 1);
 		}
 		if (error == 0)
 			error = dsl_dataset_hold(dp, dsname, FTAG, &ds);
 		if (error == 0) {
 			/* passing 0/NULL skips dsl_fs_ss_limit_check */
 			error = dsl_dataset_snapshot_check_impl(ds,
-			    atp + 1, tx, B_FALSE, 0, NULL, NULL);
+			    atp + 1, tx, B_FALSE, 0, NULL);
 			dsl_dataset_rele(ds, FTAG);
 		}
 
 		if (error != 0) {
 			if (ddsa->ddsa_errors != NULL) {
 				fnvlist_add_int32(ddsa->ddsa_errors,
 				    name, error);
 			}
 			rv = error;
 		}
 	}
 
 	return (rv);
 }
 
 void
 dsl_dataset_snapshot_sync_impl(dsl_dataset_t *ds, const char *snapname,
     dmu_tx_t *tx)
 {
 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
 	dmu_buf_t *dbuf;
 	dsl_dataset_phys_t *dsphys;
 	uint64_t dsobj, crtxg;
 	objset_t *mos = dp->dp_meta_objset;
 	objset_t *os __maybe_unused;
 
 	ASSERT(RRW_WRITE_HELD(&dp->dp_config_rwlock));
 
 	/*
 	 * If we are on an old pool, the zil must not be active, in which
 	 * case it will be zeroed.  Usually zil_suspend() accomplishes this.
 	 */
 	ASSERT(spa_version(dmu_tx_pool(tx)->dp_spa) >= SPA_VERSION_FAST_SNAP ||
 	    dmu_objset_from_ds(ds, &os) != 0 ||
 	    memcmp(&os->os_phys->os_zil_header, &zero_zil,
 	    sizeof (zero_zil)) == 0);
 
 	/* Should not snapshot a dirty dataset. */
 	ASSERT(!txg_list_member(&ds->ds_dir->dd_pool->dp_dirty_datasets,
 	    ds, tx->tx_txg));
 
 	dsl_fs_ss_count_adjust(ds->ds_dir, 1, DD_FIELD_SNAPSHOT_COUNT, tx);
 
 	/*
 	 * The origin's ds_creation_txg has to be < TXG_INITIAL
 	 */
 	if (strcmp(snapname, ORIGIN_DIR_NAME) == 0)
 		crtxg = 1;
 	else
 		crtxg = tx->tx_txg;
 
 	dsobj = dmu_object_alloc(mos, DMU_OT_DSL_DATASET, 0,
 	    DMU_OT_DSL_DATASET, sizeof (dsl_dataset_phys_t), tx);
 	VERIFY0(dmu_bonus_hold(mos, dsobj, FTAG, &dbuf));
 	dmu_buf_will_dirty(dbuf, tx);
 	dsphys = dbuf->db_data;
 	memset(dsphys, 0, sizeof (dsl_dataset_phys_t));
 	dsphys->ds_dir_obj = ds->ds_dir->dd_object;
 	dsphys->ds_fsid_guid = unique_create();
 	(void) random_get_pseudo_bytes((void*)&dsphys->ds_guid,
 	    sizeof (dsphys->ds_guid));
 	dsphys->ds_prev_snap_obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 	dsphys->ds_prev_snap_txg = dsl_dataset_phys(ds)->ds_prev_snap_txg;
 	dsphys->ds_next_snap_obj = ds->ds_object;
 	dsphys->ds_num_children = 1;
 	dsphys->ds_creation_time = gethrestime_sec();
 	dsphys->ds_creation_txg = crtxg;
 	dsphys->ds_deadlist_obj = dsl_dataset_phys(ds)->ds_deadlist_obj;
 	dsphys->ds_referenced_bytes = dsl_dataset_phys(ds)->ds_referenced_bytes;
 	dsphys->ds_compressed_bytes = dsl_dataset_phys(ds)->ds_compressed_bytes;
 	dsphys->ds_uncompressed_bytes =
 	    dsl_dataset_phys(ds)->ds_uncompressed_bytes;
 	dsphys->ds_flags = dsl_dataset_phys(ds)->ds_flags;
 	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
 	dsphys->ds_bp = dsl_dataset_phys(ds)->ds_bp;
 	rrw_exit(&ds->ds_bp_rwlock, FTAG);
 	dmu_buf_rele(dbuf, FTAG);
 
 	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
 		if (zfeature_active(f, ds->ds_feature[f])) {
 			dsl_dataset_activate_feature(dsobj, f,
 			    ds->ds_feature[f], tx);
 		}
 	}
 
 	ASSERT3U(ds->ds_prev != 0, ==,
 	    dsl_dataset_phys(ds)->ds_prev_snap_obj != 0);
 	if (ds->ds_prev) {
 		uint64_t next_clones_obj =
 		    dsl_dataset_phys(ds->ds_prev)->ds_next_clones_obj;
 		ASSERT(dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj ==
 		    ds->ds_object ||
 		    dsl_dataset_phys(ds->ds_prev)->ds_num_children > 1);
 		if (dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj ==
 		    ds->ds_object) {
 			dmu_buf_will_dirty(ds->ds_prev->ds_dbuf, tx);
 			ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_txg, ==,
 			    dsl_dataset_phys(ds->ds_prev)->ds_creation_txg);
 			dsl_dataset_phys(ds->ds_prev)->ds_next_snap_obj = dsobj;
 		} else if (next_clones_obj != 0) {
 			dsl_dataset_remove_from_next_clones(ds->ds_prev,
 			    dsphys->ds_next_snap_obj, tx);
 			VERIFY0(zap_add_int(mos,
 			    next_clones_obj, dsobj, tx));
 		}
 	}
 
 	/*
 	 * If we have a reference-reservation on this dataset, we will
 	 * need to increase the amount of refreservation being charged
 	 * since our unique space is going to zero.
 	 */
 	if (ds->ds_reserved) {
 		int64_t delta;
 		ASSERT(DS_UNIQUE_IS_ACCURATE(ds));
 		delta = MIN(dsl_dataset_phys(ds)->ds_unique_bytes,
 		    ds->ds_reserved);
 		dsl_dir_diduse_space(ds->ds_dir, DD_USED_REFRSRV,
 		    delta, 0, 0, tx);
 	}
 
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
 	dsl_dataset_phys(ds)->ds_deadlist_obj =
 	    dsl_deadlist_clone(&ds->ds_deadlist, UINT64_MAX,
 	    dsl_dataset_phys(ds)->ds_prev_snap_obj, tx);
 	dsl_deadlist_close(&ds->ds_deadlist);
 	dsl_deadlist_open(&ds->ds_deadlist, mos,
 	    dsl_dataset_phys(ds)->ds_deadlist_obj);
 	dsl_deadlist_add_key(&ds->ds_deadlist,
 	    dsl_dataset_phys(ds)->ds_prev_snap_txg, tx);
 	dsl_bookmark_snapshotted(ds, tx);
 
 	if (dsl_dataset_remap_deadlist_exists(ds)) {
 		uint64_t remap_deadlist_obj =
 		    dsl_dataset_get_remap_deadlist_object(ds);
 		/*
 		 * Move the remap_deadlist to the snapshot.  The head
 		 * will create a new remap deadlist on demand, from
 		 * dsl_dataset_block_remapped().
 		 */
 		dsl_dataset_unset_remap_deadlist_object(ds, tx);
 		dsl_deadlist_close(&ds->ds_remap_deadlist);
 
 		dmu_object_zapify(mos, dsobj, DMU_OT_DSL_DATASET, tx);
 		VERIFY0(zap_add(mos, dsobj, DS_FIELD_REMAP_DEADLIST,
 		    sizeof (remap_deadlist_obj), 1, &remap_deadlist_obj, tx));
 	}
 
 	/*
 	 * Create a ivset guid for this snapshot if the dataset is
 	 * encrypted. This may be overridden by a raw receive. A
 	 * previous implementation of this code did not have this
 	 * field as part of the on-disk format for ZFS encryption
 	 * (see errata #4). As part of the remediation for this
 	 * issue, we ask the user to enable the bookmark_v2 feature
 	 * which is now a dependency of the encryption feature. We
 	 * use this as a heuristic to determine when the user has
 	 * elected to correct any datasets created with the old code.
 	 * As a result, we only do this step if the bookmark_v2
 	 * feature is enabled, which limits the number of states a
 	 * given pool / dataset can be in with regards to terms of
 	 * correcting the issue.
 	 */
 	if (ds->ds_dir->dd_crypto_obj != 0 &&
 	    spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_BOOKMARK_V2)) {
 		uint64_t ivset_guid = unique_create();
 
 		dmu_object_zapify(mos, dsobj, DMU_OT_DSL_DATASET, tx);
 		VERIFY0(zap_add(mos, dsobj, DS_FIELD_IVSET_GUID,
 		    sizeof (ivset_guid), 1, &ivset_guid, tx));
 	}
 
 	ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_txg, <, tx->tx_txg);
 	dsl_dataset_phys(ds)->ds_prev_snap_obj = dsobj;
 	dsl_dataset_phys(ds)->ds_prev_snap_txg = crtxg;
 	dsl_dataset_phys(ds)->ds_unique_bytes = 0;
 
 	if (spa_version(dp->dp_spa) >= SPA_VERSION_UNIQUE_ACCURATE)
 		dsl_dataset_phys(ds)->ds_flags |= DS_FLAG_UNIQUE_ACCURATE;
 
 	VERIFY0(zap_add(mos, dsl_dataset_phys(ds)->ds_snapnames_zapobj,
 	    snapname, 8, 1, &dsobj, tx));
 
 	if (ds->ds_prev)
 		dsl_dataset_rele(ds->ds_prev, ds);
 	VERIFY0(dsl_dataset_hold_obj(dp,
 	    dsl_dataset_phys(ds)->ds_prev_snap_obj, ds, &ds->ds_prev));
 
 	dsl_scan_ds_snapshotted(ds, tx);
 
 	dsl_dir_snap_cmtime_update(ds->ds_dir, tx);
 
 	if (zfs_snapshot_history_enabled)
 		spa_history_log_internal_ds(ds->ds_prev, "snapshot", tx, " ");
 }
 
 void
 dsl_dataset_snapshot_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_snapshot_arg_t *ddsa = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	nvpair_t *pair;
 
 	for (pair = nvlist_next_nvpair(ddsa->ddsa_snaps, NULL);
 	    pair != NULL; pair = nvlist_next_nvpair(ddsa->ddsa_snaps, pair)) {
 		dsl_dataset_t *ds;
 		const char *name, *atp;
 		char dsname[ZFS_MAX_DATASET_NAME_LEN];
 
 		name = nvpair_name(pair);
 		atp = strchr(name, '@');
 		(void) strlcpy(dsname, name, atp - name + 1);
 		VERIFY0(dsl_dataset_hold(dp, dsname, FTAG, &ds));
 
 		dsl_dataset_snapshot_sync_impl(ds, atp + 1, tx);
 		if (ddsa->ddsa_props != NULL) {
 			dsl_props_set_sync_impl(ds->ds_prev,
 			    ZPROP_SRC_LOCAL, ddsa->ddsa_props, tx);
 		}
 		dsl_dataset_rele(ds, FTAG);
 	}
 }
 
 /*
  * The snapshots must all be in the same pool.
  * All-or-nothing: if there are any failures, nothing will be modified.
  */
 int
 dsl_dataset_snapshot(nvlist_t *snaps, nvlist_t *props, nvlist_t *errors)
 {
 	dsl_dataset_snapshot_arg_t ddsa;
 	nvpair_t *pair;
 	boolean_t needsuspend;
 	int error;
 	spa_t *spa;
 	const char *firstname;
 	nvlist_t *suspended = NULL;
 
 	pair = nvlist_next_nvpair(snaps, NULL);
 	if (pair == NULL)
 		return (0);
 	firstname = nvpair_name(pair);
 
 	error = spa_open(firstname, &spa, FTAG);
 	if (error != 0)
 		return (error);
 	needsuspend = (spa_version(spa) < SPA_VERSION_FAST_SNAP);
 	spa_close(spa, FTAG);
 
 	if (needsuspend) {
 		suspended = fnvlist_alloc();
 		for (pair = nvlist_next_nvpair(snaps, NULL); pair != NULL;
 		    pair = nvlist_next_nvpair(snaps, pair)) {
 			char fsname[ZFS_MAX_DATASET_NAME_LEN];
 			const char *snapname = nvpair_name(pair);
 			const char *atp;
 			void *cookie;
 
 			atp = strchr(snapname, '@');
 			if (atp == NULL) {
 				error = SET_ERROR(EINVAL);
 				break;
 			}
 			(void) strlcpy(fsname, snapname, atp - snapname + 1);
 
 			error = zil_suspend(fsname, &cookie);
 			if (error != 0)
 				break;
 			fnvlist_add_uint64(suspended, fsname,
 			    (uintptr_t)cookie);
 		}
 	}
 
+	cred_t *cr = CRED();
+	crhold(cr);
+
 	ddsa.ddsa_snaps = snaps;
 	ddsa.ddsa_props = props;
 	ddsa.ddsa_errors = errors;
-	ddsa.ddsa_cr = CRED();
-	ddsa.ddsa_proc = curproc;
+	ddsa.ddsa_cr = cr;
 
 	if (error == 0) {
 		error = dsl_sync_task(firstname, dsl_dataset_snapshot_check,
 		    dsl_dataset_snapshot_sync, &ddsa,
 		    fnvlist_num_pairs(snaps) * 3, ZFS_SPACE_CHECK_NORMAL);
 	}
 
+	crfree(cr);
+
 	if (suspended != NULL) {
 		for (pair = nvlist_next_nvpair(suspended, NULL); pair != NULL;
 		    pair = nvlist_next_nvpair(suspended, pair)) {
 			zil_resume((void *)(uintptr_t)
 			    fnvpair_value_uint64(pair));
 		}
 		fnvlist_free(suspended);
 	}
 
 	if (error == 0) {
 		for (pair = nvlist_next_nvpair(snaps, NULL); pair != NULL;
 		    pair = nvlist_next_nvpair(snaps, pair)) {
 			zvol_create_minor(nvpair_name(pair));
 		}
 	}
 
 	return (error);
 }
 
 typedef struct dsl_dataset_snapshot_tmp_arg {
 	const char *ddsta_fsname;
 	const char *ddsta_snapname;
 	minor_t ddsta_cleanup_minor;
 	const char *ddsta_htag;
 } dsl_dataset_snapshot_tmp_arg_t;
 
 static int
 dsl_dataset_snapshot_tmp_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_snapshot_tmp_arg_t *ddsta = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds;
 	int error;
 
 	error = dsl_dataset_hold(dp, ddsta->ddsta_fsname, FTAG, &ds);
 	if (error != 0)
 		return (error);
 
 	/* NULL cred means no limit check for tmp snapshot */
 	error = dsl_dataset_snapshot_check_impl(ds, ddsta->ddsta_snapname,
-	    tx, B_FALSE, 0, NULL, NULL);
+	    tx, B_FALSE, 0, NULL);
 	if (error != 0) {
 		dsl_dataset_rele(ds, FTAG);
 		return (error);
 	}
 
 	if (spa_version(dp->dp_spa) < SPA_VERSION_USERREFS) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(ENOTSUP));
 	}
 	error = dsl_dataset_user_hold_check_one(NULL, ddsta->ddsta_htag,
 	    B_TRUE, tx);
 	if (error != 0) {
 		dsl_dataset_rele(ds, FTAG);
 		return (error);
 	}
 
 	dsl_dataset_rele(ds, FTAG);
 	return (0);
 }
 
 static void
 dsl_dataset_snapshot_tmp_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_snapshot_tmp_arg_t *ddsta = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds = NULL;
 
 	VERIFY0(dsl_dataset_hold(dp, ddsta->ddsta_fsname, FTAG, &ds));
 
 	dsl_dataset_snapshot_sync_impl(ds, ddsta->ddsta_snapname, tx);
 	dsl_dataset_user_hold_sync_one(ds->ds_prev, ddsta->ddsta_htag,
 	    ddsta->ddsta_cleanup_minor, gethrestime_sec(), tx);
 	dsl_destroy_snapshot_sync_impl(ds->ds_prev, B_TRUE, tx);
 
 	dsl_dataset_rele(ds, FTAG);
 }
 
 int
 dsl_dataset_snapshot_tmp(const char *fsname, const char *snapname,
     minor_t cleanup_minor, const char *htag)
 {
 	dsl_dataset_snapshot_tmp_arg_t ddsta;
 	int error;
 	spa_t *spa;
 	boolean_t needsuspend;
 	void *cookie;
 
 	ddsta.ddsta_fsname = fsname;
 	ddsta.ddsta_snapname = snapname;
 	ddsta.ddsta_cleanup_minor = cleanup_minor;
 	ddsta.ddsta_htag = htag;
 
 	error = spa_open(fsname, &spa, FTAG);
 	if (error != 0)
 		return (error);
 	needsuspend = (spa_version(spa) < SPA_VERSION_FAST_SNAP);
 	spa_close(spa, FTAG);
 
 	if (needsuspend) {
 		error = zil_suspend(fsname, &cookie);
 		if (error != 0)
 			return (error);
 	}
 
 	error = dsl_sync_task(fsname, dsl_dataset_snapshot_tmp_check,
 	    dsl_dataset_snapshot_tmp_sync, &ddsta, 3, ZFS_SPACE_CHECK_RESERVED);
 
 	if (needsuspend)
 		zil_resume(cookie);
 	return (error);
 }
 
 void
 dsl_dataset_sync(dsl_dataset_t *ds, zio_t *zio, dmu_tx_t *tx)
 {
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(ds->ds_objset != NULL);
 	ASSERT(dsl_dataset_phys(ds)->ds_next_snap_obj == 0);
 
 	/*
 	 * in case we had to change ds_fsid_guid when we opened it,
 	 * sync it out now.
 	 */
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
 	dsl_dataset_phys(ds)->ds_fsid_guid = ds->ds_fsid_guid;
 
 	if (ds->ds_resume_bytes[tx->tx_txg & TXG_MASK] != 0) {
 		VERIFY0(zap_update(tx->tx_pool->dp_meta_objset,
 		    ds->ds_object, DS_FIELD_RESUME_OBJECT, 8, 1,
 		    &ds->ds_resume_object[tx->tx_txg & TXG_MASK], tx));
 		VERIFY0(zap_update(tx->tx_pool->dp_meta_objset,
 		    ds->ds_object, DS_FIELD_RESUME_OFFSET, 8, 1,
 		    &ds->ds_resume_offset[tx->tx_txg & TXG_MASK], tx));
 		VERIFY0(zap_update(tx->tx_pool->dp_meta_objset,
 		    ds->ds_object, DS_FIELD_RESUME_BYTES, 8, 1,
 		    &ds->ds_resume_bytes[tx->tx_txg & TXG_MASK], tx));
 		ds->ds_resume_object[tx->tx_txg & TXG_MASK] = 0;
 		ds->ds_resume_offset[tx->tx_txg & TXG_MASK] = 0;
 		ds->ds_resume_bytes[tx->tx_txg & TXG_MASK] = 0;
 	}
 
 	dmu_objset_sync(ds->ds_objset, zio, tx);
 }
 
 /*
  * Check if the percentage of blocks shared between the clone and the
  * snapshot (as opposed to those that are clone only) is below a certain
  * threshold
  */
 static boolean_t
 dsl_livelist_should_disable(dsl_dataset_t *ds)
 {
 	uint64_t used, referenced;
 	int percent_shared;
 
 	used = dsl_dir_get_usedds(ds->ds_dir);
 	referenced = dsl_get_referenced(ds);
 	if (referenced == 0)
 		return (B_FALSE);
 	percent_shared = (100 * (referenced - used)) / referenced;
 	if (percent_shared <= zfs_livelist_min_percent_shared)
 		return (B_TRUE);
 	return (B_FALSE);
 }
 
 /*
  *  Check if it is possible to combine two livelist entries into one.
  *  This is the case if the combined number of 'live' blkptrs (ALLOCs that
  *  don't have a matching FREE) is under the maximum sublist size.
  *  We check this by subtracting twice the total number of frees from the total
  *  number of blkptrs. FREEs are counted twice because each FREE blkptr
  *  will cancel out an ALLOC blkptr when the livelist is processed.
  */
 static boolean_t
 dsl_livelist_should_condense(dsl_deadlist_entry_t *first,
     dsl_deadlist_entry_t *next)
 {
 	uint64_t total_free = first->dle_bpobj.bpo_phys->bpo_num_freed +
 	    next->dle_bpobj.bpo_phys->bpo_num_freed;
 	uint64_t total_entries = first->dle_bpobj.bpo_phys->bpo_num_blkptrs +
 	    next->dle_bpobj.bpo_phys->bpo_num_blkptrs;
 	if ((total_entries - (2 * total_free)) < zfs_livelist_max_entries)
 		return (B_TRUE);
 	return (B_FALSE);
 }
 
 typedef struct try_condense_arg {
 	spa_t *spa;
 	dsl_dataset_t *ds;
 } try_condense_arg_t;
 
 /*
  * Iterate over the livelist entries, searching for a pair to condense.
  * A nonzero return value means stop, 0 means keep looking.
  */
 static int
 dsl_livelist_try_condense(void *arg, dsl_deadlist_entry_t *first)
 {
 	try_condense_arg_t *tca = arg;
 	spa_t *spa = tca->spa;
 	dsl_dataset_t *ds = tca->ds;
 	dsl_deadlist_t *ll = &ds->ds_dir->dd_livelist;
 	dsl_deadlist_entry_t *next;
 
 	/* The condense thread has not yet been created at import */
 	if (spa->spa_livelist_condense_zthr == NULL)
 		return (1);
 
 	/* A condense is already in progress */
 	if (spa->spa_to_condense.ds != NULL)
 		return (1);
 
 	next = AVL_NEXT(&ll->dl_tree, &first->dle_node);
 	/* The livelist has only one entry - don't condense it */
 	if (next == NULL)
 		return (1);
 
 	/* Next is the newest entry - don't condense it */
 	if (AVL_NEXT(&ll->dl_tree, &next->dle_node) == NULL)
 		return (1);
 
 	/* This pair is not ready to condense but keep looking */
 	if (!dsl_livelist_should_condense(first, next))
 		return (0);
 
 	/*
 	 * Add a ref to prevent the dataset from being evicted while
 	 * the condense zthr or synctask are running. Ref will be
 	 * released at the end of the condense synctask
 	 */
 	dmu_buf_add_ref(ds->ds_dbuf, spa);
 
 	spa->spa_to_condense.ds = ds;
 	spa->spa_to_condense.first = first;
 	spa->spa_to_condense.next = next;
 	spa->spa_to_condense.syncing = B_FALSE;
 	spa->spa_to_condense.cancelled = B_FALSE;
 
 	zthr_wakeup(spa->spa_livelist_condense_zthr);
 	return (1);
 }
 
 static void
 dsl_flush_pending_livelist(dsl_dataset_t *ds, dmu_tx_t *tx)
 {
 	dsl_dir_t *dd = ds->ds_dir;
 	spa_t *spa = ds->ds_dir->dd_pool->dp_spa;
 	dsl_deadlist_entry_t *last = dsl_deadlist_last(&dd->dd_livelist);
 
 	/* Check if we need to add a new sub-livelist */
 	if (last == NULL) {
 		/* The livelist is empty */
 		dsl_deadlist_add_key(&dd->dd_livelist,
 		    tx->tx_txg - 1, tx);
 	} else if (spa_sync_pass(spa) == 1) {
 		/*
 		 * Check if the newest entry is full. If it is, make a new one.
 		 * We only do this once per sync because we could overfill a
 		 * sublist in one sync pass and don't want to add another entry
 		 * for a txg that is already represented. This ensures that
 		 * blkptrs born in the same txg are stored in the same sublist.
 		 */
 		bpobj_t bpobj = last->dle_bpobj;
 		uint64_t all = bpobj.bpo_phys->bpo_num_blkptrs;
 		uint64_t free = bpobj.bpo_phys->bpo_num_freed;
 		uint64_t alloc = all - free;
 		if (alloc > zfs_livelist_max_entries) {
 			dsl_deadlist_add_key(&dd->dd_livelist,
 			    tx->tx_txg - 1, tx);
 		}
 	}
 
 	/* Insert each entry into the on-disk livelist */
 	bplist_iterate(&dd->dd_pending_allocs,
 	    dsl_deadlist_insert_alloc_cb, &dd->dd_livelist, tx);
 	bplist_iterate(&dd->dd_pending_frees,
 	    dsl_deadlist_insert_free_cb, &dd->dd_livelist, tx);
 
 	/* Attempt to condense every pair of adjacent entries */
 	try_condense_arg_t arg = {
 	    .spa = spa,
 	    .ds = ds
 	};
 	dsl_deadlist_iterate(&dd->dd_livelist, dsl_livelist_try_condense,
 	    &arg);
 }
 
 void
 dsl_dataset_sync_done(dsl_dataset_t *ds, dmu_tx_t *tx)
 {
 	objset_t *os = ds->ds_objset;
 
 	bplist_iterate(&ds->ds_pending_deadlist,
 	    dsl_deadlist_insert_alloc_cb, &ds->ds_deadlist, tx);
 
 	if (dsl_deadlist_is_open(&ds->ds_dir->dd_livelist)) {
 		dsl_flush_pending_livelist(ds, tx);
 		if (dsl_livelist_should_disable(ds)) {
 			dsl_dir_remove_livelist(ds->ds_dir, tx, B_TRUE);
 		}
 	}
 
 	dsl_bookmark_sync_done(ds, tx);
 
 	multilist_destroy(&os->os_synced_dnodes);
 
 	if (os->os_encrypted)
 		os->os_next_write_raw[tx->tx_txg & TXG_MASK] = B_FALSE;
 	else
 		ASSERT0(os->os_next_write_raw[tx->tx_txg & TXG_MASK]);
 
 	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
 		if (zfeature_active(f,
 		    ds->ds_feature_activation[f])) {
 			if (zfeature_active(f, ds->ds_feature[f]))
 				continue;
 			dsl_dataset_activate_feature(ds->ds_object, f,
 			    ds->ds_feature_activation[f], tx);
 			ds->ds_feature[f] = ds->ds_feature_activation[f];
 		}
 	}
 
 	ASSERT(!dmu_objset_is_dirty(os, dmu_tx_get_txg(tx)));
 }
 
 int
 get_clones_stat_impl(dsl_dataset_t *ds, nvlist_t *val)
 {
 	uint64_t count = 0;
 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
 	zap_cursor_t zc;
 	zap_attribute_t za;
 
 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
 
 	/*
 	 * There may be missing entries in ds_next_clones_obj
 	 * due to a bug in a previous version of the code.
 	 * Only trust it if it has the right number of entries.
 	 */
 	if (dsl_dataset_phys(ds)->ds_next_clones_obj != 0) {
 		VERIFY0(zap_count(mos, dsl_dataset_phys(ds)->ds_next_clones_obj,
 		    &count));
 	}
 	if (count != dsl_dataset_phys(ds)->ds_num_children - 1) {
 		return (SET_ERROR(ENOENT));
 	}
 	for (zap_cursor_init(&zc, mos,
 	    dsl_dataset_phys(ds)->ds_next_clones_obj);
 	    zap_cursor_retrieve(&zc, &za) == 0;
 	    zap_cursor_advance(&zc)) {
 		dsl_dataset_t *clone;
 		char buf[ZFS_MAX_DATASET_NAME_LEN];
 		VERIFY0(dsl_dataset_hold_obj(ds->ds_dir->dd_pool,
 		    za.za_first_integer, FTAG, &clone));
 		dsl_dir_name(clone->ds_dir, buf);
 		fnvlist_add_boolean(val, buf);
 		dsl_dataset_rele(clone, FTAG);
 	}
 	zap_cursor_fini(&zc);
 	return (0);
 }
 
 void
 get_clones_stat(dsl_dataset_t *ds, nvlist_t *nv)
 {
 	nvlist_t *propval = fnvlist_alloc();
 	nvlist_t *val = fnvlist_alloc();
 
 	if (get_clones_stat_impl(ds, val) == 0) {
 		fnvlist_add_nvlist(propval, ZPROP_VALUE, val);
 		fnvlist_add_nvlist(nv, zfs_prop_to_name(ZFS_PROP_CLONES),
 		    propval);
 	}
 
 	nvlist_free(val);
 	nvlist_free(propval);
 }
 
 static char *
 get_receive_resume_token_impl(dsl_dataset_t *ds)
 {
 	if (!dsl_dataset_has_resume_receive_state(ds))
 		return (NULL);
 
 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
 	char *str;
 	void *packed;
 	uint8_t *compressed;
 	uint64_t val;
 	nvlist_t *token_nv = fnvlist_alloc();
 	size_t packed_size, compressed_size;
 
 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_FROMGUID, sizeof (val), 1, &val) == 0) {
 		fnvlist_add_uint64(token_nv, "fromguid", val);
 	}
 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_OBJECT, sizeof (val), 1, &val) == 0) {
 		fnvlist_add_uint64(token_nv, "object", val);
 	}
 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_OFFSET, sizeof (val), 1, &val) == 0) {
 		fnvlist_add_uint64(token_nv, "offset", val);
 	}
 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_BYTES, sizeof (val), 1, &val) == 0) {
 		fnvlist_add_uint64(token_nv, "bytes", val);
 	}
 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_TOGUID, sizeof (val), 1, &val) == 0) {
 		fnvlist_add_uint64(token_nv, "toguid", val);
 	}
 	char buf[MAXNAMELEN];
 	if (zap_lookup(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_TONAME, 1, sizeof (buf), buf) == 0) {
 		fnvlist_add_string(token_nv, "toname", buf);
 	}
 	if (zap_contains(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_LARGEBLOCK) == 0) {
 		fnvlist_add_boolean(token_nv, "largeblockok");
 	}
 	if (zap_contains(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_EMBEDOK) == 0) {
 		fnvlist_add_boolean(token_nv, "embedok");
 	}
 	if (zap_contains(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_COMPRESSOK) == 0) {
 		fnvlist_add_boolean(token_nv, "compressok");
 	}
 	if (zap_contains(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_RAWOK) == 0) {
 		fnvlist_add_boolean(token_nv, "rawok");
 	}
 	if (dsl_dataset_feature_is_active(ds,
 	    SPA_FEATURE_REDACTED_DATASETS)) {
 		uint64_t num_redact_snaps = 0;
 		uint64_t *redact_snaps = NULL;
 		VERIFY3B(dsl_dataset_get_uint64_array_feature(ds,
 		    SPA_FEATURE_REDACTED_DATASETS, &num_redact_snaps,
 		    &redact_snaps), ==, B_TRUE);
 		fnvlist_add_uint64_array(token_nv, "redact_snaps",
 		    redact_snaps, num_redact_snaps);
 	}
 	if (zap_contains(dp->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS) == 0) {
 		uint64_t num_redact_snaps = 0, int_size = 0;
 		uint64_t *redact_snaps = NULL;
 		VERIFY0(zap_length(dp->dp_meta_objset, ds->ds_object,
 		    DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, &int_size,
 		    &num_redact_snaps));
 		ASSERT3U(int_size, ==, sizeof (uint64_t));
 
 		redact_snaps = kmem_alloc(int_size * num_redact_snaps,
 		    KM_SLEEP);
 		VERIFY0(zap_lookup(dp->dp_meta_objset, ds->ds_object,
 		    DS_FIELD_RESUME_REDACT_BOOKMARK_SNAPS, int_size,
 		    num_redact_snaps, redact_snaps));
 		fnvlist_add_uint64_array(token_nv, "book_redact_snaps",
 		    redact_snaps, num_redact_snaps);
 		kmem_free(redact_snaps, int_size * num_redact_snaps);
 	}
 	packed = fnvlist_pack(token_nv, &packed_size);
 	fnvlist_free(token_nv);
 	compressed = kmem_alloc(packed_size, KM_SLEEP);
 
 	compressed_size = gzip_compress(packed, compressed,
 	    packed_size, packed_size, 6);
 
 	zio_cksum_t cksum;
 	fletcher_4_native_varsize(compressed, compressed_size, &cksum);
 
 	size_t alloc_size = compressed_size * 2 + 1;
 	str = kmem_alloc(alloc_size, KM_SLEEP);
 	for (int i = 0; i < compressed_size; i++) {
 		size_t offset = i * 2;
 		(void) snprintf(str + offset, alloc_size - offset,
 	    "%02x", compressed[i]);
 	}
 	str[compressed_size * 2] = '\0';
 	char *propval = kmem_asprintf("%u-%llx-%llx-%s",
 	    ZFS_SEND_RESUME_TOKEN_VERSION,
 	    (longlong_t)cksum.zc_word[0],
 	    (longlong_t)packed_size, str);
 	kmem_free(packed, packed_size);
 	kmem_free(str, alloc_size);
 	kmem_free(compressed, packed_size);
 	return (propval);
 }
 
 /*
  * Returns a string that represents the receive resume state token. It should
  * be freed with strfree(). NULL is returned if no resume state is present.
  */
 char *
 get_receive_resume_token(dsl_dataset_t *ds)
 {
 	/*
 	 * A failed "newfs" (e.g. full) resumable receive leaves
 	 * the stats set on this dataset.  Check here for the prop.
 	 */
 	char *token = get_receive_resume_token_impl(ds);
 	if (token != NULL)
 		return (token);
 	/*
 	 * A failed incremental resumable receive leaves the
 	 * stats set on our child named "%recv".  Check the child
 	 * for the prop.
 	 */
 	/* 6 extra bytes for /%recv */
 	char name[ZFS_MAX_DATASET_NAME_LEN + 6];
 	dsl_dataset_t *recv_ds;
 	dsl_dataset_name(ds, name);
 	if (strlcat(name, "/", sizeof (name)) < sizeof (name) &&
 	    strlcat(name, recv_clone_name, sizeof (name)) < sizeof (name) &&
 	    dsl_dataset_hold(ds->ds_dir->dd_pool, name, FTAG, &recv_ds) == 0) {
 		token = get_receive_resume_token_impl(recv_ds);
 		dsl_dataset_rele(recv_ds, FTAG);
 	}
 	return (token);
 }
 
 uint64_t
 dsl_get_refratio(dsl_dataset_t *ds)
 {
 	uint64_t ratio = dsl_dataset_phys(ds)->ds_compressed_bytes == 0 ? 100 :
 	    (dsl_dataset_phys(ds)->ds_uncompressed_bytes * 100 /
 	    dsl_dataset_phys(ds)->ds_compressed_bytes);
 	return (ratio);
 }
 
 uint64_t
 dsl_get_logicalreferenced(dsl_dataset_t *ds)
 {
 	return (dsl_dataset_phys(ds)->ds_uncompressed_bytes);
 }
 
 uint64_t
 dsl_get_compressratio(dsl_dataset_t *ds)
 {
 	if (ds->ds_is_snapshot) {
 		return (dsl_get_refratio(ds));
 	} else {
 		dsl_dir_t *dd = ds->ds_dir;
 		mutex_enter(&dd->dd_lock);
 		uint64_t val = dsl_dir_get_compressratio(dd);
 		mutex_exit(&dd->dd_lock);
 		return (val);
 	}
 }
 
 uint64_t
 dsl_get_used(dsl_dataset_t *ds)
 {
 	if (ds->ds_is_snapshot) {
 		return (dsl_dataset_phys(ds)->ds_unique_bytes);
 	} else {
 		dsl_dir_t *dd = ds->ds_dir;
 		mutex_enter(&dd->dd_lock);
 		uint64_t val = dsl_dir_get_used(dd);
 		mutex_exit(&dd->dd_lock);
 		return (val);
 	}
 }
 
 uint64_t
 dsl_get_creation(dsl_dataset_t *ds)
 {
 	return (dsl_dataset_phys(ds)->ds_creation_time);
 }
 
 uint64_t
 dsl_get_creationtxg(dsl_dataset_t *ds)
 {
 	return (dsl_dataset_phys(ds)->ds_creation_txg);
 }
 
 uint64_t
 dsl_get_refquota(dsl_dataset_t *ds)
 {
 	return (ds->ds_quota);
 }
 
 uint64_t
 dsl_get_refreservation(dsl_dataset_t *ds)
 {
 	return (ds->ds_reserved);
 }
 
 uint64_t
 dsl_get_guid(dsl_dataset_t *ds)
 {
 	return (dsl_dataset_phys(ds)->ds_guid);
 }
 
 uint64_t
 dsl_get_unique(dsl_dataset_t *ds)
 {
 	return (dsl_dataset_phys(ds)->ds_unique_bytes);
 }
 
 uint64_t
 dsl_get_objsetid(dsl_dataset_t *ds)
 {
 	return (ds->ds_object);
 }
 
 uint64_t
 dsl_get_userrefs(dsl_dataset_t *ds)
 {
 	return (ds->ds_userrefs);
 }
 
 uint64_t
 dsl_get_defer_destroy(dsl_dataset_t *ds)
 {
 	return (DS_IS_DEFER_DESTROY(ds) ? 1 : 0);
 }
 
 uint64_t
 dsl_get_referenced(dsl_dataset_t *ds)
 {
 	return (dsl_dataset_phys(ds)->ds_referenced_bytes);
 }
 
 uint64_t
 dsl_get_numclones(dsl_dataset_t *ds)
 {
 	ASSERT(ds->ds_is_snapshot);
 	return (dsl_dataset_phys(ds)->ds_num_children - 1);
 }
 
 uint64_t
 dsl_get_inconsistent(dsl_dataset_t *ds)
 {
 	return ((dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT) ?
 	    1 : 0);
 }
 
 uint64_t
 dsl_get_redacted(dsl_dataset_t *ds)
 {
 	return (dsl_dataset_feature_is_active(ds,
 	    SPA_FEATURE_REDACTED_DATASETS));
 }
 
 uint64_t
 dsl_get_available(dsl_dataset_t *ds)
 {
 	uint64_t refdbytes = dsl_get_referenced(ds);
 	uint64_t availbytes = dsl_dir_space_available(ds->ds_dir,
 	    NULL, 0, TRUE);
 	if (ds->ds_reserved > dsl_dataset_phys(ds)->ds_unique_bytes) {
 		availbytes +=
 		    ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes;
 	}
 	if (ds->ds_quota != 0) {
 		/*
 		 * Adjust available bytes according to refquota
 		 */
 		if (refdbytes < ds->ds_quota) {
 			availbytes = MIN(availbytes,
 			    ds->ds_quota - refdbytes);
 		} else {
 			availbytes = 0;
 		}
 	}
 	return (availbytes);
 }
 
 int
 dsl_get_written(dsl_dataset_t *ds, uint64_t *written)
 {
 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
 	dsl_dataset_t *prev;
 	int err = dsl_dataset_hold_obj(dp,
 	    dsl_dataset_phys(ds)->ds_prev_snap_obj, FTAG, &prev);
 	if (err == 0) {
 		uint64_t comp, uncomp;
 		err = dsl_dataset_space_written(prev, ds, written,
 		    &comp, &uncomp);
 		dsl_dataset_rele(prev, FTAG);
 	}
 	return (err);
 }
 
 /*
  * 'snap' should be a buffer of size ZFS_MAX_DATASET_NAME_LEN.
  */
 int
 dsl_get_prev_snap(dsl_dataset_t *ds, char *snap)
 {
 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
 	if (ds->ds_prev != NULL && ds->ds_prev != dp->dp_origin_snap) {
 		dsl_dataset_name(ds->ds_prev, snap);
 		return (0);
 	} else {
 		return (SET_ERROR(ENOENT));
 	}
 }
 
 void
 dsl_get_redact_snaps(dsl_dataset_t *ds, nvlist_t *propval)
 {
 	uint64_t nsnaps;
 	uint64_t *snaps;
 	if (dsl_dataset_get_uint64_array_feature(ds,
 	    SPA_FEATURE_REDACTED_DATASETS, &nsnaps, &snaps)) {
 		fnvlist_add_uint64_array(propval, ZPROP_VALUE, snaps,
 		    nsnaps);
 	}
 }
 
 /*
  * Returns the mountpoint property and source for the given dataset in the value
  * and source buffers. The value buffer must be at least as large as MAXPATHLEN
  * and the source buffer as least as large a ZFS_MAX_DATASET_NAME_LEN.
  * Returns 0 on success and an error on failure.
  */
 int
 dsl_get_mountpoint(dsl_dataset_t *ds, const char *dsname, char *value,
     char *source)
 {
 	int error;
 	dsl_pool_t *dp = ds->ds_dir->dd_pool;
 
 	/* Retrieve the mountpoint value stored in the zap object */
 	error = dsl_prop_get_ds(ds, zfs_prop_to_name(ZFS_PROP_MOUNTPOINT), 1,
 	    ZAP_MAXVALUELEN, value, source);
 	if (error != 0) {
 		return (error);
 	}
 
 	/*
 	 * Process the dsname and source to find the full mountpoint string.
 	 * Can be skipped for 'legacy' or 'none'.
 	 */
 	if (value[0] == '/') {
 		char *buf = kmem_alloc(ZAP_MAXVALUELEN, KM_SLEEP);
 		char *root = buf;
 		const char *relpath;
 
 		/*
 		 * If we inherit the mountpoint, even from a dataset
 		 * with a received value, the source will be the path of
 		 * the dataset we inherit from. If source is
 		 * ZPROP_SOURCE_VAL_RECVD, the received value is not
 		 * inherited.
 		 */
 		if (strcmp(source, ZPROP_SOURCE_VAL_RECVD) == 0) {
 			relpath = "";
 		} else {
 			ASSERT0(strncmp(dsname, source, strlen(source)));
 			relpath = dsname + strlen(source);
 			if (relpath[0] == '/')
 				relpath++;
 		}
 
 		spa_altroot(dp->dp_spa, root, ZAP_MAXVALUELEN);
 
 		/*
 		 * Special case an alternate root of '/'. This will
 		 * avoid having multiple leading slashes in the
 		 * mountpoint path.
 		 */
 		if (strcmp(root, "/") == 0)
 			root++;
 
 		/*
 		 * If the mountpoint is '/' then skip over this
 		 * if we are obtaining either an alternate root or
 		 * an inherited mountpoint.
 		 */
 		char *mnt = value;
 		if (value[1] == '\0' && (root[0] != '\0' ||
 		    relpath[0] != '\0'))
 			mnt = value + 1;
 
 		mnt = kmem_strdup(mnt);
 
 		if (relpath[0] == '\0') {
 			(void) snprintf(value, ZAP_MAXVALUELEN, "%s%s",
 			    root, mnt);
 		} else {
 			(void) snprintf(value, ZAP_MAXVALUELEN, "%s%s%s%s",
 			    root, mnt, relpath[0] == '@' ? "" : "/",
 			    relpath);
 		}
 		kmem_free(buf, ZAP_MAXVALUELEN);
 		kmem_strfree(mnt);
 	}
 
 	return (0);
 }
 
 void
 dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv)
 {
 	dsl_pool_t *dp __maybe_unused = ds->ds_dir->dd_pool;
 
 	ASSERT(dsl_pool_config_held(dp));
 
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFRATIO,
 	    dsl_get_refratio(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_LOGICALREFERENCED,
 	    dsl_get_logicalreferenced(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_COMPRESSRATIO,
 	    dsl_get_compressratio(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USED,
 	    dsl_get_used(ds));
 
 	if (ds->ds_is_snapshot) {
 		get_clones_stat(ds, nv);
 	} else {
 		char buf[ZFS_MAX_DATASET_NAME_LEN];
 		if (dsl_get_prev_snap(ds, buf) == 0)
 			dsl_prop_nvlist_add_string(nv, ZFS_PROP_PREV_SNAP,
 			    buf);
 		dsl_dir_stats(ds->ds_dir, nv);
 	}
 
 	nvlist_t *propval = fnvlist_alloc();
 	dsl_get_redact_snaps(ds, propval);
 	fnvlist_add_nvlist(nv, zfs_prop_to_name(ZFS_PROP_REDACT_SNAPS),
 	    propval);
 	nvlist_free(propval);
 
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_AVAILABLE,
 	    dsl_get_available(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFERENCED,
 	    dsl_get_referenced(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_CREATION,
 	    dsl_get_creation(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_CREATETXG,
 	    dsl_get_creationtxg(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFQUOTA,
 	    dsl_get_refquota(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_REFRESERVATION,
 	    dsl_get_refreservation(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_GUID,
 	    dsl_get_guid(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_UNIQUE,
 	    dsl_get_unique(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_OBJSETID,
 	    dsl_get_objsetid(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USERREFS,
 	    dsl_get_userrefs(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_DEFER_DESTROY,
 	    dsl_get_defer_destroy(ds));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_SNAPSHOTS_CHANGED,
 	    dsl_dir_snap_cmtime(ds->ds_dir).tv_sec);
 	dsl_dataset_crypt_stats(ds, nv);
 
 	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 		uint64_t written;
 		if (dsl_get_written(ds, &written) == 0) {
 			dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_WRITTEN,
 			    written);
 		}
 	}
 
 	if (!dsl_dataset_is_snapshot(ds)) {
 		char *token = get_receive_resume_token(ds);
 		if (token != NULL) {
 			dsl_prop_nvlist_add_string(nv,
 			    ZFS_PROP_RECEIVE_RESUME_TOKEN, token);
 			kmem_strfree(token);
 		}
 	}
 }
 
 void
 dsl_dataset_fast_stat(dsl_dataset_t *ds, dmu_objset_stats_t *stat)
 {
 	dsl_pool_t *dp __maybe_unused = ds->ds_dir->dd_pool;
 	ASSERT(dsl_pool_config_held(dp));
 
 	stat->dds_creation_txg = dsl_get_creationtxg(ds);
 	stat->dds_inconsistent = dsl_get_inconsistent(ds);
 	stat->dds_guid = dsl_get_guid(ds);
 	stat->dds_redacted = dsl_get_redacted(ds);
 	stat->dds_origin[0] = '\0';
 	if (ds->ds_is_snapshot) {
 		stat->dds_is_snapshot = B_TRUE;
 		stat->dds_num_clones = dsl_get_numclones(ds);
 	} else {
 		stat->dds_is_snapshot = B_FALSE;
 		stat->dds_num_clones = 0;
 
 		if (dsl_dir_is_clone(ds->ds_dir)) {
 			dsl_dir_get_origin(ds->ds_dir, stat->dds_origin);
 		}
 	}
 }
 
 uint64_t
 dsl_dataset_fsid_guid(dsl_dataset_t *ds)
 {
 	return (ds->ds_fsid_guid);
 }
 
 void
 dsl_dataset_space(dsl_dataset_t *ds,
     uint64_t *refdbytesp, uint64_t *availbytesp,
     uint64_t *usedobjsp, uint64_t *availobjsp)
 {
 	*refdbytesp = dsl_dataset_phys(ds)->ds_referenced_bytes;
 	*availbytesp = dsl_dir_space_available(ds->ds_dir, NULL, 0, TRUE);
 	if (ds->ds_reserved > dsl_dataset_phys(ds)->ds_unique_bytes)
 		*availbytesp +=
 		    ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes;
 	if (ds->ds_quota != 0) {
 		/*
 		 * Adjust available bytes according to refquota
 		 */
 		if (*refdbytesp < ds->ds_quota)
 			*availbytesp = MIN(*availbytesp,
 			    ds->ds_quota - *refdbytesp);
 		else
 			*availbytesp = 0;
 	}
 	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
 	*usedobjsp = BP_GET_FILL(&dsl_dataset_phys(ds)->ds_bp);
 	rrw_exit(&ds->ds_bp_rwlock, FTAG);
 	*availobjsp = DN_MAX_OBJECT - *usedobjsp;
 }
 
 boolean_t
 dsl_dataset_modified_since_snap(dsl_dataset_t *ds, dsl_dataset_t *snap)
 {
 	dsl_pool_t *dp __maybe_unused = ds->ds_dir->dd_pool;
 	uint64_t birth;
 
 	ASSERT(dsl_pool_config_held(dp));
 	if (snap == NULL)
 		return (B_FALSE);
 	rrw_enter(&ds->ds_bp_rwlock, RW_READER, FTAG);
 	birth = dsl_dataset_get_blkptr(ds)->blk_birth;
 	rrw_exit(&ds->ds_bp_rwlock, FTAG);
 	if (birth > dsl_dataset_phys(snap)->ds_creation_txg) {
 		objset_t *os, *os_snap;
 		/*
 		 * It may be that only the ZIL differs, because it was
 		 * reset in the head.  Don't count that as being
 		 * modified.
 		 */
 		if (dmu_objset_from_ds(ds, &os) != 0)
 			return (B_TRUE);
 		if (dmu_objset_from_ds(snap, &os_snap) != 0)
 			return (B_TRUE);
 		return (memcmp(&os->os_phys->os_meta_dnode,
 		    &os_snap->os_phys->os_meta_dnode,
 		    sizeof (os->os_phys->os_meta_dnode)) != 0);
 	}
 	return (B_FALSE);
 }
 
 static int
 dsl_dataset_rename_snapshot_check_impl(dsl_pool_t *dp,
     dsl_dataset_t *hds, void *arg)
 {
 	(void) dp;
 	dsl_dataset_rename_snapshot_arg_t *ddrsa = arg;
 	int error;
 	uint64_t val;
 
 	error = dsl_dataset_snap_lookup(hds, ddrsa->ddrsa_oldsnapname, &val);
 	if (error != 0) {
 		/* ignore nonexistent snapshots */
 		return (error == ENOENT ? 0 : error);
 	}
 
 	/* new name should not exist */
 	error = dsl_dataset_snap_lookup(hds, ddrsa->ddrsa_newsnapname, &val);
 	if (error == 0)
 		error = SET_ERROR(EEXIST);
 	else if (error == ENOENT)
 		error = 0;
 
 	/* dataset name + 1 for the "@" + the new snapshot name must fit */
 	if (dsl_dir_namelen(hds->ds_dir) + 1 +
 	    strlen(ddrsa->ddrsa_newsnapname) >= ZFS_MAX_DATASET_NAME_LEN)
 		error = SET_ERROR(ENAMETOOLONG);
 
 	return (error);
 }
 
 int
 dsl_dataset_rename_snapshot_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_rename_snapshot_arg_t *ddrsa = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *hds;
 	int error;
 
 	error = dsl_dataset_hold(dp, ddrsa->ddrsa_fsname, FTAG, &hds);
 	if (error != 0)
 		return (error);
 
 	if (ddrsa->ddrsa_recursive) {
 		error = dmu_objset_find_dp(dp, hds->ds_dir->dd_object,
 		    dsl_dataset_rename_snapshot_check_impl, ddrsa,
 		    DS_FIND_CHILDREN);
 	} else {
 		error = dsl_dataset_rename_snapshot_check_impl(dp, hds, ddrsa);
 	}
 	dsl_dataset_rele(hds, FTAG);
 	return (error);
 }
 
 static int
 dsl_dataset_rename_snapshot_sync_impl(dsl_pool_t *dp,
     dsl_dataset_t *hds, void *arg)
 {
 	dsl_dataset_rename_snapshot_arg_t *ddrsa = arg;
 	dsl_dataset_t *ds;
 	uint64_t val;
 	dmu_tx_t *tx = ddrsa->ddrsa_tx;
 	char *oldname, *newname;
 	int error;
 
 	error = dsl_dataset_snap_lookup(hds, ddrsa->ddrsa_oldsnapname, &val);
 	ASSERT(error == 0 || error == ENOENT);
 	if (error == ENOENT) {
 		/* ignore nonexistent snapshots */
 		return (0);
 	}
 
 	VERIFY0(dsl_dataset_hold_obj(dp, val, FTAG, &ds));
 
 	/* log before we change the name */
 	spa_history_log_internal_ds(ds, "rename", tx,
 	    "-> @%s", ddrsa->ddrsa_newsnapname);
 
 	VERIFY0(dsl_dataset_snap_remove(hds, ddrsa->ddrsa_oldsnapname, tx,
 	    B_FALSE));
 	mutex_enter(&ds->ds_lock);
 	(void) strlcpy(ds->ds_snapname, ddrsa->ddrsa_newsnapname,
 	    sizeof (ds->ds_snapname));
 	mutex_exit(&ds->ds_lock);
 	VERIFY0(zap_add(dp->dp_meta_objset,
 	    dsl_dataset_phys(hds)->ds_snapnames_zapobj,
 	    ds->ds_snapname, 8, 1, &ds->ds_object, tx));
 
 	oldname = kmem_asprintf("%s@%s", ddrsa->ddrsa_fsname,
 	    ddrsa->ddrsa_oldsnapname);
 	newname = kmem_asprintf("%s@%s", ddrsa->ddrsa_fsname,
 	    ddrsa->ddrsa_newsnapname);
 	zvol_rename_minors(dp->dp_spa, oldname, newname, B_TRUE);
 	kmem_strfree(oldname);
 	kmem_strfree(newname);
 
 	dsl_dataset_rele(ds, FTAG);
 	return (0);
 }
 
 void
 dsl_dataset_rename_snapshot_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_rename_snapshot_arg_t *ddrsa = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *hds = NULL;
 
 	VERIFY0(dsl_dataset_hold(dp, ddrsa->ddrsa_fsname, FTAG, &hds));
 	ddrsa->ddrsa_tx = tx;
 	if (ddrsa->ddrsa_recursive) {
 		VERIFY0(dmu_objset_find_dp(dp, hds->ds_dir->dd_object,
 		    dsl_dataset_rename_snapshot_sync_impl, ddrsa,
 		    DS_FIND_CHILDREN));
 	} else {
 		VERIFY0(dsl_dataset_rename_snapshot_sync_impl(dp, hds, ddrsa));
 	}
 	dsl_dataset_rele(hds, FTAG);
 }
 
 int
 dsl_dataset_rename_snapshot(const char *fsname,
     const char *oldsnapname, const char *newsnapname, boolean_t recursive)
 {
 	dsl_dataset_rename_snapshot_arg_t ddrsa;
 
 	ddrsa.ddrsa_fsname = fsname;
 	ddrsa.ddrsa_oldsnapname = oldsnapname;
 	ddrsa.ddrsa_newsnapname = newsnapname;
 	ddrsa.ddrsa_recursive = recursive;
 
 	return (dsl_sync_task(fsname, dsl_dataset_rename_snapshot_check,
 	    dsl_dataset_rename_snapshot_sync, &ddrsa,
 	    1, ZFS_SPACE_CHECK_RESERVED));
 }
 
 /*
  * If we're doing an ownership handoff, we need to make sure that there is
  * only one long hold on the dataset.  We're not allowed to change anything here
  * so we don't permanently release the long hold or regular hold here.  We want
  * to do this only when syncing to avoid the dataset unexpectedly going away
  * when we release the long hold.
  */
 static int
 dsl_dataset_handoff_check(dsl_dataset_t *ds, void *owner, dmu_tx_t *tx)
 {
 	boolean_t held = B_FALSE;
 
 	if (!dmu_tx_is_syncing(tx))
 		return (0);
 
 	dsl_dir_t *dd = ds->ds_dir;
 	mutex_enter(&dd->dd_activity_lock);
 	uint64_t holds = zfs_refcount_count(&ds->ds_longholds) -
 	    (owner != NULL ? 1 : 0);
 	/*
 	 * The value of dd_activity_waiters can chance as soon as we drop the
 	 * lock, but we're fine with that; new waiters coming in or old
 	 * waiters leaving doesn't cause problems, since we're going to cancel
 	 * waiters later anyway. The goal of this check is to verify that no
 	 * non-waiters have long-holds, and all new long-holds will be
 	 * prevented because we're holding the pool config as writer.
 	 */
 	if (holds != dd->dd_activity_waiters)
 		held = B_TRUE;
 	mutex_exit(&dd->dd_activity_lock);
 
 	if (held)
 		return (SET_ERROR(EBUSY));
 
 	return (0);
 }
 
 int
 dsl_dataset_rollback_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_rollback_arg_t *ddra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds;
 	int64_t unused_refres_delta;
 	int error;
 
 	error = dsl_dataset_hold(dp, ddra->ddra_fsname, FTAG, &ds);
 	if (error != 0)
 		return (error);
 
 	/* must not be a snapshot */
 	if (ds->ds_is_snapshot) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
 
 	/* must have a most recent snapshot */
 	if (dsl_dataset_phys(ds)->ds_prev_snap_txg < TXG_INITIAL) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(ESRCH));
 	}
 
 	/*
 	 * No rollback to a snapshot created in the current txg, because
 	 * the rollback may dirty the dataset and create blocks that are
 	 * not reachable from the rootbp while having a birth txg that
 	 * falls into the snapshot's range.
 	 */
 	if (dmu_tx_is_syncing(tx) &&
 	    dsl_dataset_phys(ds)->ds_prev_snap_txg >= tx->tx_txg) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EAGAIN));
 	}
 
 	/*
 	 * If the expected target snapshot is specified, then check that
 	 * the latest snapshot is it.
 	 */
 	if (ddra->ddra_tosnap != NULL) {
 		dsl_dataset_t *snapds;
 
 		/* Check if the target snapshot exists at all. */
 		error = dsl_dataset_hold(dp, ddra->ddra_tosnap, FTAG, &snapds);
 		if (error != 0) {
 			/*
 			 * ESRCH is used to signal that the target snapshot does
 			 * not exist, while ENOENT is used to report that
 			 * the rolled back dataset does not exist.
 			 * ESRCH is also used to cover other cases where the
 			 * target snapshot is not related to the dataset being
 			 * rolled back such as being in a different pool.
 			 */
 			if (error == ENOENT || error == EXDEV)
 				error = SET_ERROR(ESRCH);
 			dsl_dataset_rele(ds, FTAG);
 			return (error);
 		}
 		ASSERT(snapds->ds_is_snapshot);
 
 		/* Check if the snapshot is the latest snapshot indeed. */
 		if (snapds != ds->ds_prev) {
 			/*
 			 * Distinguish between the case where the only problem
 			 * is intervening snapshots (EEXIST) vs the snapshot
 			 * not being a valid target for rollback (ESRCH).
 			 */
 			if (snapds->ds_dir == ds->ds_dir ||
 			    (dsl_dir_is_clone(ds->ds_dir) &&
 			    dsl_dir_phys(ds->ds_dir)->dd_origin_obj ==
 			    snapds->ds_object)) {
 				error = SET_ERROR(EEXIST);
 			} else {
 				error = SET_ERROR(ESRCH);
 			}
 			dsl_dataset_rele(snapds, FTAG);
 			dsl_dataset_rele(ds, FTAG);
 			return (error);
 		}
 		dsl_dataset_rele(snapds, FTAG);
 	}
 
 	/* must not have any bookmarks after the most recent snapshot */
 	if (dsl_bookmark_latest_txg(ds) >
 	    dsl_dataset_phys(ds)->ds_prev_snap_txg) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EEXIST));
 	}
 
 	error = dsl_dataset_handoff_check(ds, ddra->ddra_owner, tx);
 	if (error != 0) {
 		dsl_dataset_rele(ds, FTAG);
 		return (error);
 	}
 
 	/*
 	 * Check if the snap we are rolling back to uses more than
 	 * the refquota.
 	 */
 	if (ds->ds_quota != 0 &&
 	    dsl_dataset_phys(ds->ds_prev)->ds_referenced_bytes > ds->ds_quota) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EDQUOT));
 	}
 
 	/*
 	 * When we do the clone swap, we will temporarily use more space
 	 * due to the refreservation (the head will no longer have any
 	 * unique space, so the entire amount of the refreservation will need
 	 * to be free).  We will immediately destroy the clone, freeing
 	 * this space, but the freeing happens over many txg's.
 	 */
 	unused_refres_delta = (int64_t)MIN(ds->ds_reserved,
 	    dsl_dataset_phys(ds)->ds_unique_bytes);
 
 	if (unused_refres_delta > 0 &&
 	    unused_refres_delta >
 	    dsl_dir_space_available(ds->ds_dir, NULL, 0, TRUE)) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(ENOSPC));
 	}
 
 	dsl_dataset_rele(ds, FTAG);
 	return (0);
 }
 
 void
 dsl_dataset_rollback_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_rollback_arg_t *ddra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds, *clone;
 	uint64_t cloneobj;
 	char namebuf[ZFS_MAX_DATASET_NAME_LEN];
 
 	VERIFY0(dsl_dataset_hold(dp, ddra->ddra_fsname, FTAG, &ds));
 
 	dsl_dataset_name(ds->ds_prev, namebuf);
 	fnvlist_add_string(ddra->ddra_result, "target", namebuf);
 
 	cloneobj = dsl_dataset_create_sync(ds->ds_dir, "%rollback",
 	    ds->ds_prev, DS_CREATE_FLAG_NODIRTY, kcred, NULL, tx);
 
 	VERIFY0(dsl_dataset_hold_obj(dp, cloneobj, FTAG, &clone));
 
 	dsl_dataset_clone_swap_sync_impl(clone, ds, tx);
 	dsl_dataset_zero_zil(ds, tx);
 
 	dsl_destroy_head_sync_impl(clone, tx);
 
 	dsl_dataset_rele(clone, FTAG);
 	dsl_dataset_rele(ds, FTAG);
 }
 
 /*
  * Rolls back the given filesystem or volume to the most recent snapshot.
  * The name of the most recent snapshot will be returned under key "target"
  * in the result nvlist.
  *
  * If owner != NULL:
  * - The existing dataset MUST be owned by the specified owner at entry
  * - Upon return, dataset will still be held by the same owner, whether we
  *   succeed or not.
  *
  * This mode is required any time the existing filesystem is mounted.  See
  * notes above zfs_suspend_fs() for further details.
  */
 int
 dsl_dataset_rollback(const char *fsname, const char *tosnap, void *owner,
     nvlist_t *result)
 {
 	dsl_dataset_rollback_arg_t ddra;
 
 	ddra.ddra_fsname = fsname;
 	ddra.ddra_tosnap = tosnap;
 	ddra.ddra_owner = owner;
 	ddra.ddra_result = result;
 
 	return (dsl_sync_task(fsname, dsl_dataset_rollback_check,
 	    dsl_dataset_rollback_sync, &ddra,
 	    1, ZFS_SPACE_CHECK_RESERVED));
 }
 
 struct promotenode {
 	list_node_t link;
 	dsl_dataset_t *ds;
 };
 
 static int snaplist_space(list_t *l, uint64_t mintxg, uint64_t *spacep);
 static int promote_hold(dsl_dataset_promote_arg_t *ddpa, dsl_pool_t *dp,
     const void *tag);
 static void promote_rele(dsl_dataset_promote_arg_t *ddpa, const void *tag);
 
 int
 dsl_dataset_promote_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_promote_arg_t *ddpa = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *hds;
 	struct promotenode *snap;
 	int err;
 	uint64_t unused;
 	uint64_t ss_mv_cnt;
 	size_t max_snap_len;
 	boolean_t conflicting_snaps;
 
 	err = promote_hold(ddpa, dp, FTAG);
 	if (err != 0)
 		return (err);
 
 	hds = ddpa->ddpa_clone;
 	max_snap_len = MAXNAMELEN - strlen(ddpa->ddpa_clonename) - 1;
 
 	if (dsl_dataset_phys(hds)->ds_flags & DS_FLAG_NOPROMOTE) {
 		promote_rele(ddpa, FTAG);
 		return (SET_ERROR(EXDEV));
 	}
 
 	snap = list_head(&ddpa->shared_snaps);
 	if (snap == NULL) {
 		err = SET_ERROR(ENOENT);
 		goto out;
 	}
 	dsl_dataset_t *const origin_ds = snap->ds;
 
 	/*
 	 * Encrypted clones share a DSL Crypto Key with their origin's dsl dir.
 	 * When doing a promote we must make sure the encryption root for
 	 * both the target and the target's origin does not change to avoid
 	 * needing to rewrap encryption keys
 	 */
 	err = dsl_dataset_promote_crypt_check(hds->ds_dir, origin_ds->ds_dir);
 	if (err != 0)
 		goto out;
 
 	/*
 	 * Compute and check the amount of space to transfer.  Since this is
 	 * so expensive, don't do the preliminary check.
 	 */
 	if (!dmu_tx_is_syncing(tx)) {
 		promote_rele(ddpa, FTAG);
 		return (0);
 	}
 
 	/* compute origin's new unique space */
 	snap = list_tail(&ddpa->clone_snaps);
 	ASSERT(snap != NULL);
 	ASSERT3U(dsl_dataset_phys(snap->ds)->ds_prev_snap_obj, ==,
 	    origin_ds->ds_object);
 	dsl_deadlist_space_range(&snap->ds->ds_deadlist,
 	    dsl_dataset_phys(origin_ds)->ds_prev_snap_txg, UINT64_MAX,
 	    &ddpa->unique, &unused, &unused);
 
 	/*
 	 * Walk the snapshots that we are moving
 	 *
 	 * Compute space to transfer.  Consider the incremental changes
 	 * to used by each snapshot:
 	 * (my used) = (prev's used) + (blocks born) - (blocks killed)
 	 * So each snapshot gave birth to:
 	 * (blocks born) = (my used) - (prev's used) + (blocks killed)
 	 * So a sequence would look like:
 	 * (uN - u(N-1) + kN) + ... + (u1 - u0 + k1) + (u0 - 0 + k0)
 	 * Which simplifies to:
 	 * uN + kN + kN-1 + ... + k1 + k0
 	 * Note however, if we stop before we reach the ORIGIN we get:
 	 * uN + kN + kN-1 + ... + kM - uM-1
 	 */
 	conflicting_snaps = B_FALSE;
 	ss_mv_cnt = 0;
 	ddpa->used = dsl_dataset_phys(origin_ds)->ds_referenced_bytes;
 	ddpa->comp = dsl_dataset_phys(origin_ds)->ds_compressed_bytes;
 	ddpa->uncomp = dsl_dataset_phys(origin_ds)->ds_uncompressed_bytes;
 	for (snap = list_head(&ddpa->shared_snaps); snap;
 	    snap = list_next(&ddpa->shared_snaps, snap)) {
 		uint64_t val, dlused, dlcomp, dluncomp;
 		dsl_dataset_t *ds = snap->ds;
 
 		ss_mv_cnt++;
 
 		/*
 		 * If there are long holds, we won't be able to evict
 		 * the objset.
 		 */
 		if (dsl_dataset_long_held(ds)) {
 			err = SET_ERROR(EBUSY);
 			goto out;
 		}
 
 		/* Check that the snapshot name does not conflict */
 		VERIFY0(dsl_dataset_get_snapname(ds));
 		if (strlen(ds->ds_snapname) >= max_snap_len) {
 			err = SET_ERROR(ENAMETOOLONG);
 			goto out;
 		}
 		err = dsl_dataset_snap_lookup(hds, ds->ds_snapname, &val);
 		if (err == 0) {
 			fnvlist_add_boolean(ddpa->err_ds,
 			    snap->ds->ds_snapname);
 			conflicting_snaps = B_TRUE;
 		} else if (err != ENOENT) {
 			goto out;
 		}
 
 		/* The very first snapshot does not have a deadlist */
 		if (dsl_dataset_phys(ds)->ds_prev_snap_obj == 0)
 			continue;
 
 		dsl_deadlist_space(&ds->ds_deadlist,
 		    &dlused, &dlcomp, &dluncomp);
 		ddpa->used += dlused;
 		ddpa->comp += dlcomp;
 		ddpa->uncomp += dluncomp;
 	}
 
 	/*
 	 * Check that bookmarks that are being transferred don't have
 	 * name conflicts.
 	 */
 	for (dsl_bookmark_node_t *dbn = avl_first(&origin_ds->ds_bookmarks);
 	    dbn != NULL && dbn->dbn_phys.zbm_creation_txg <=
 	    dsl_dataset_phys(origin_ds)->ds_creation_txg;
 	    dbn = AVL_NEXT(&origin_ds->ds_bookmarks, dbn)) {
 		if (strlen(dbn->dbn_name) >= max_snap_len) {
 			err = SET_ERROR(ENAMETOOLONG);
 			goto out;
 		}
 		zfs_bookmark_phys_t bm;
 		err = dsl_bookmark_lookup_impl(ddpa->ddpa_clone,
 		    dbn->dbn_name, &bm);
 
 		if (err == 0) {
 			fnvlist_add_boolean(ddpa->err_ds, dbn->dbn_name);
 			conflicting_snaps = B_TRUE;
 		} else if (err == ESRCH) {
 			err = 0;
 		}
 		if (err != 0) {
 			goto out;
 		}
 	}
 
 	/*
 	 * In order to return the full list of conflicting snapshots, we check
 	 * whether there was a conflict after traversing all of them.
 	 */
 	if (conflicting_snaps) {
 		err = SET_ERROR(EEXIST);
 		goto out;
 	}
 
 	/*
 	 * If we are a clone of a clone then we never reached ORIGIN,
 	 * so we need to subtract out the clone origin's used space.
 	 */
 	if (ddpa->origin_origin) {
 		ddpa->used -=
 		    dsl_dataset_phys(ddpa->origin_origin)->ds_referenced_bytes;
 		ddpa->comp -=
 		    dsl_dataset_phys(ddpa->origin_origin)->ds_compressed_bytes;
 		ddpa->uncomp -=
 		    dsl_dataset_phys(ddpa->origin_origin)->
 		    ds_uncompressed_bytes;
 	}
 
 	/* Check that there is enough space and limit headroom here */
 	err = dsl_dir_transfer_possible(origin_ds->ds_dir, hds->ds_dir,
-	    0, ss_mv_cnt, ddpa->used, ddpa->cr, ddpa->proc);
+	    0, ss_mv_cnt, ddpa->used, ddpa->cr);
 	if (err != 0)
 		goto out;
 
 	/*
 	 * Compute the amounts of space that will be used by snapshots
 	 * after the promotion (for both origin and clone).  For each,
 	 * it is the amount of space that will be on all of their
 	 * deadlists (that was not born before their new origin).
 	 */
 	if (dsl_dir_phys(hds->ds_dir)->dd_flags & DD_FLAG_USED_BREAKDOWN) {
 		uint64_t space;
 
 		/*
 		 * Note, typically this will not be a clone of a clone,
 		 * so dd_origin_txg will be < TXG_INITIAL, so
 		 * these snaplist_space() -> dsl_deadlist_space_range()
 		 * calls will be fast because they do not have to
 		 * iterate over all bps.
 		 */
 		snap = list_head(&ddpa->origin_snaps);
 		if (snap == NULL) {
 			err = SET_ERROR(ENOENT);
 			goto out;
 		}
 		err = snaplist_space(&ddpa->shared_snaps,
 		    snap->ds->ds_dir->dd_origin_txg, &ddpa->cloneusedsnap);
 		if (err != 0)
 			goto out;
 
 		err = snaplist_space(&ddpa->clone_snaps,
 		    snap->ds->ds_dir->dd_origin_txg, &space);
 		if (err != 0)
 			goto out;
 		ddpa->cloneusedsnap += space;
 	}
 	if (dsl_dir_phys(origin_ds->ds_dir)->dd_flags &
 	    DD_FLAG_USED_BREAKDOWN) {
 		err = snaplist_space(&ddpa->origin_snaps,
 		    dsl_dataset_phys(origin_ds)->ds_creation_txg,
 		    &ddpa->originusedsnap);
 		if (err != 0)
 			goto out;
 	}
 
 out:
 	promote_rele(ddpa, FTAG);
 	return (err);
 }
 
 void
 dsl_dataset_promote_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_promote_arg_t *ddpa = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *hds;
 	struct promotenode *snap;
 	dsl_dataset_t *origin_ds;
 	dsl_dataset_t *origin_head;
 	dsl_dir_t *dd;
 	dsl_dir_t *odd = NULL;
 	uint64_t oldnext_obj;
 	int64_t delta;
 
 	ASSERT(nvlist_empty(ddpa->err_ds));
 
 	VERIFY0(promote_hold(ddpa, dp, FTAG));
 	hds = ddpa->ddpa_clone;
 
 	ASSERT0(dsl_dataset_phys(hds)->ds_flags & DS_FLAG_NOPROMOTE);
 
 	snap = list_head(&ddpa->shared_snaps);
 	origin_ds = snap->ds;
 	dd = hds->ds_dir;
 
 	snap = list_head(&ddpa->origin_snaps);
 	origin_head = snap->ds;
 
 	/*
 	 * We need to explicitly open odd, since origin_ds's dd will be
 	 * changing.
 	 */
 	VERIFY0(dsl_dir_hold_obj(dp, origin_ds->ds_dir->dd_object,
 	    NULL, FTAG, &odd));
 
 	dsl_dataset_promote_crypt_sync(hds->ds_dir, odd, tx);
 
 	/* change origin's next snap */
 	dmu_buf_will_dirty(origin_ds->ds_dbuf, tx);
 	oldnext_obj = dsl_dataset_phys(origin_ds)->ds_next_snap_obj;
 	snap = list_tail(&ddpa->clone_snaps);
 	ASSERT3U(dsl_dataset_phys(snap->ds)->ds_prev_snap_obj, ==,
 	    origin_ds->ds_object);
 	dsl_dataset_phys(origin_ds)->ds_next_snap_obj = snap->ds->ds_object;
 
 	/* change the origin's next clone */
 	if (dsl_dataset_phys(origin_ds)->ds_next_clones_obj) {
 		dsl_dataset_remove_from_next_clones(origin_ds,
 		    snap->ds->ds_object, tx);
 		VERIFY0(zap_add_int(dp->dp_meta_objset,
 		    dsl_dataset_phys(origin_ds)->ds_next_clones_obj,
 		    oldnext_obj, tx));
 	}
 
 	/* change origin */
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 	ASSERT3U(dsl_dir_phys(dd)->dd_origin_obj, ==, origin_ds->ds_object);
 	dsl_dir_phys(dd)->dd_origin_obj = dsl_dir_phys(odd)->dd_origin_obj;
 	dd->dd_origin_txg = origin_head->ds_dir->dd_origin_txg;
 	dmu_buf_will_dirty(odd->dd_dbuf, tx);
 	dsl_dir_phys(odd)->dd_origin_obj = origin_ds->ds_object;
 	origin_head->ds_dir->dd_origin_txg =
 	    dsl_dataset_phys(origin_ds)->ds_creation_txg;
 
 	/* change dd_clone entries */
 	if (spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
 		VERIFY0(zap_remove_int(dp->dp_meta_objset,
 		    dsl_dir_phys(odd)->dd_clones, hds->ds_object, tx));
 		VERIFY0(zap_add_int(dp->dp_meta_objset,
 		    dsl_dir_phys(ddpa->origin_origin->ds_dir)->dd_clones,
 		    hds->ds_object, tx));
 
 		VERIFY0(zap_remove_int(dp->dp_meta_objset,
 		    dsl_dir_phys(ddpa->origin_origin->ds_dir)->dd_clones,
 		    origin_head->ds_object, tx));
 		if (dsl_dir_phys(dd)->dd_clones == 0) {
 			dsl_dir_phys(dd)->dd_clones =
 			    zap_create(dp->dp_meta_objset, DMU_OT_DSL_CLONES,
 			    DMU_OT_NONE, 0, tx);
 		}
 		VERIFY0(zap_add_int(dp->dp_meta_objset,
 		    dsl_dir_phys(dd)->dd_clones, origin_head->ds_object, tx));
 	}
 
 	/*
 	 * Move bookmarks to this dir.
 	 */
 	dsl_bookmark_node_t *dbn_next;
 	for (dsl_bookmark_node_t *dbn = avl_first(&origin_head->ds_bookmarks);
 	    dbn != NULL && dbn->dbn_phys.zbm_creation_txg <=
 	    dsl_dataset_phys(origin_ds)->ds_creation_txg;
 	    dbn = dbn_next) {
 		dbn_next = AVL_NEXT(&origin_head->ds_bookmarks, dbn);
 
 		avl_remove(&origin_head->ds_bookmarks, dbn);
 		VERIFY0(zap_remove(dp->dp_meta_objset,
 		    origin_head->ds_bookmarks_obj, dbn->dbn_name, tx));
 
 		dsl_bookmark_node_add(hds, dbn, tx);
 	}
 
 	dsl_bookmark_next_changed(hds, origin_ds, tx);
 
 	/* move snapshots to this dir */
 	for (snap = list_head(&ddpa->shared_snaps); snap;
 	    snap = list_next(&ddpa->shared_snaps, snap)) {
 		dsl_dataset_t *ds = snap->ds;
 
 		/*
 		 * Property callbacks are registered to a particular
 		 * dsl_dir.  Since ours is changing, evict the objset
 		 * so that they will be unregistered from the old dsl_dir.
 		 */
 		if (ds->ds_objset) {
 			dmu_objset_evict(ds->ds_objset);
 			ds->ds_objset = NULL;
 		}
 
 		/* move snap name entry */
 		VERIFY0(dsl_dataset_get_snapname(ds));
 		VERIFY0(dsl_dataset_snap_remove(origin_head,
 		    ds->ds_snapname, tx, B_TRUE));
 		VERIFY0(zap_add(dp->dp_meta_objset,
 		    dsl_dataset_phys(hds)->ds_snapnames_zapobj, ds->ds_snapname,
 		    8, 1, &ds->ds_object, tx));
 		dsl_fs_ss_count_adjust(hds->ds_dir, 1,
 		    DD_FIELD_SNAPSHOT_COUNT, tx);
 
 		/* change containing dsl_dir */
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
 		ASSERT3U(dsl_dataset_phys(ds)->ds_dir_obj, ==, odd->dd_object);
 		dsl_dataset_phys(ds)->ds_dir_obj = dd->dd_object;
 		ASSERT3P(ds->ds_dir, ==, odd);
 		dsl_dir_rele(ds->ds_dir, ds);
 		VERIFY0(dsl_dir_hold_obj(dp, dd->dd_object,
 		    NULL, ds, &ds->ds_dir));
 
 		/* move any clone references */
 		if (dsl_dataset_phys(ds)->ds_next_clones_obj &&
 		    spa_version(dp->dp_spa) >= SPA_VERSION_DIR_CLONES) {
 			zap_cursor_t zc;
 			zap_attribute_t za;
 
 			for (zap_cursor_init(&zc, dp->dp_meta_objset,
 			    dsl_dataset_phys(ds)->ds_next_clones_obj);
 			    zap_cursor_retrieve(&zc, &za) == 0;
 			    zap_cursor_advance(&zc)) {
 				dsl_dataset_t *cnds;
 				uint64_t o;
 
 				if (za.za_first_integer == oldnext_obj) {
 					/*
 					 * We've already moved the
 					 * origin's reference.
 					 */
 					continue;
 				}
 
 				VERIFY0(dsl_dataset_hold_obj(dp,
 				    za.za_first_integer, FTAG, &cnds));
 				o = dsl_dir_phys(cnds->ds_dir)->
 				    dd_head_dataset_obj;
 
 				VERIFY0(zap_remove_int(dp->dp_meta_objset,
 				    dsl_dir_phys(odd)->dd_clones, o, tx));
 				VERIFY0(zap_add_int(dp->dp_meta_objset,
 				    dsl_dir_phys(dd)->dd_clones, o, tx));
 				dsl_dataset_rele(cnds, FTAG);
 			}
 			zap_cursor_fini(&zc);
 		}
 
 		ASSERT(!dsl_prop_hascb(ds));
 	}
 
 	/*
 	 * Change space accounting.
 	 * Note, pa->*usedsnap and dd_used_breakdown[SNAP] will either
 	 * both be valid, or both be 0 (resulting in delta == 0).  This
 	 * is true for each of {clone,origin} independently.
 	 */
 
 	delta = ddpa->cloneusedsnap -
 	    dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_SNAP];
 	ASSERT3S(delta, >=, 0);
 	ASSERT3U(ddpa->used, >=, delta);
 	dsl_dir_diduse_space(dd, DD_USED_SNAP, delta, 0, 0, tx);
 	dsl_dir_diduse_space(dd, DD_USED_HEAD,
 	    ddpa->used - delta, ddpa->comp, ddpa->uncomp, tx);
 
 	delta = ddpa->originusedsnap -
 	    dsl_dir_phys(odd)->dd_used_breakdown[DD_USED_SNAP];
 	ASSERT3S(delta, <=, 0);
 	ASSERT3U(ddpa->used, >=, -delta);
 	dsl_dir_diduse_space(odd, DD_USED_SNAP, delta, 0, 0, tx);
 	dsl_dir_diduse_space(odd, DD_USED_HEAD,
 	    -ddpa->used - delta, -ddpa->comp, -ddpa->uncomp, tx);
 
 	dsl_dataset_phys(origin_ds)->ds_unique_bytes = ddpa->unique;
 
 	/*
 	 * Since livelists are specific to a clone's origin txg, they
 	 * are no longer accurate. Destroy the livelist from the clone being
 	 * promoted. If the origin dataset is a clone, destroy its livelist
 	 * as well.
 	 */
 	dsl_dir_remove_livelist(dd, tx, B_TRUE);
 	dsl_dir_remove_livelist(odd, tx, B_TRUE);
 
 	/* log history record */
 	spa_history_log_internal_ds(hds, "promote", tx, " ");
 
 	dsl_dir_rele(odd, FTAG);
 
 	/*
 	 * Transfer common error blocks from old head to new head, before
 	 * calling promote_rele() on ddpa since we need to dereference
 	 * origin_head and hds.
 	 */
 	if (spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_HEAD_ERRLOG)) {
 		uint64_t old_head = origin_head->ds_object;
 		uint64_t new_head = hds->ds_object;
 		spa_swap_errlog(dp->dp_spa, new_head, old_head, tx);
 	}
 
 	promote_rele(ddpa, FTAG);
 }
 
 /*
  * Make a list of dsl_dataset_t's for the snapshots between first_obj
  * (exclusive) and last_obj (inclusive).  The list will be in reverse
  * order (last_obj will be the list_head()).  If first_obj == 0, do all
  * snapshots back to this dataset's origin.
  */
 static int
 snaplist_make(dsl_pool_t *dp,
     uint64_t first_obj, uint64_t last_obj, list_t *l, const void *tag)
 {
 	uint64_t obj = last_obj;
 
 	list_create(l, sizeof (struct promotenode),
 	    offsetof(struct promotenode, link));
 
 	while (obj != first_obj) {
 		dsl_dataset_t *ds;
 		struct promotenode *snap;
 		int err;
 
 		err = dsl_dataset_hold_obj(dp, obj, tag, &ds);
 		ASSERT(err != ENOENT);
 		if (err != 0)
 			return (err);
 
 		if (first_obj == 0)
 			first_obj = dsl_dir_phys(ds->ds_dir)->dd_origin_obj;
 
 		snap = kmem_alloc(sizeof (*snap), KM_SLEEP);
 		snap->ds = ds;
 		list_insert_tail(l, snap);
 		obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 	}
 
 	return (0);
 }
 
 static int
 snaplist_space(list_t *l, uint64_t mintxg, uint64_t *spacep)
 {
 	struct promotenode *snap;
 
 	*spacep = 0;
 	for (snap = list_head(l); snap; snap = list_next(l, snap)) {
 		uint64_t used, comp, uncomp;
 		dsl_deadlist_space_range(&snap->ds->ds_deadlist,
 		    mintxg, UINT64_MAX, &used, &comp, &uncomp);
 		*spacep += used;
 	}
 	return (0);
 }
 
 static void
 snaplist_destroy(list_t *l, const void *tag)
 {
 	struct promotenode *snap;
 
 	if (l == NULL || !list_link_active(&l->list_head))
 		return;
 
 	while ((snap = list_remove_tail(l)) != NULL) {
 		dsl_dataset_rele(snap->ds, tag);
 		kmem_free(snap, sizeof (*snap));
 	}
 	list_destroy(l);
 }
 
 static int
 promote_hold(dsl_dataset_promote_arg_t *ddpa, dsl_pool_t *dp, const void *tag)
 {
 	int error;
 	dsl_dir_t *dd;
 	struct promotenode *snap;
 
 	error = dsl_dataset_hold(dp, ddpa->ddpa_clonename, tag,
 	    &ddpa->ddpa_clone);
 	if (error != 0)
 		return (error);
 	dd = ddpa->ddpa_clone->ds_dir;
 
 	if (ddpa->ddpa_clone->ds_is_snapshot ||
 	    !dsl_dir_is_clone(dd)) {
 		dsl_dataset_rele(ddpa->ddpa_clone, tag);
 		return (SET_ERROR(EINVAL));
 	}
 
 	error = snaplist_make(dp, 0, dsl_dir_phys(dd)->dd_origin_obj,
 	    &ddpa->shared_snaps, tag);
 	if (error != 0)
 		goto out;
 
 	error = snaplist_make(dp, 0, ddpa->ddpa_clone->ds_object,
 	    &ddpa->clone_snaps, tag);
 	if (error != 0)
 		goto out;
 
 	snap = list_head(&ddpa->shared_snaps);
 	ASSERT3U(snap->ds->ds_object, ==, dsl_dir_phys(dd)->dd_origin_obj);
 	error = snaplist_make(dp, dsl_dir_phys(dd)->dd_origin_obj,
 	    dsl_dir_phys(snap->ds->ds_dir)->dd_head_dataset_obj,
 	    &ddpa->origin_snaps, tag);
 	if (error != 0)
 		goto out;
 
 	if (dsl_dir_phys(snap->ds->ds_dir)->dd_origin_obj != 0) {
 		error = dsl_dataset_hold_obj(dp,
 		    dsl_dir_phys(snap->ds->ds_dir)->dd_origin_obj,
 		    tag, &ddpa->origin_origin);
 		if (error != 0)
 			goto out;
 	}
 out:
 	if (error != 0)
 		promote_rele(ddpa, tag);
 	return (error);
 }
 
 static void
 promote_rele(dsl_dataset_promote_arg_t *ddpa, const void *tag)
 {
 	snaplist_destroy(&ddpa->shared_snaps, tag);
 	snaplist_destroy(&ddpa->clone_snaps, tag);
 	snaplist_destroy(&ddpa->origin_snaps, tag);
 	if (ddpa->origin_origin != NULL)
 		dsl_dataset_rele(ddpa->origin_origin, tag);
 	dsl_dataset_rele(ddpa->ddpa_clone, tag);
 }
 
 /*
  * Promote a clone.
  *
  * If it fails due to a conflicting snapshot name, "conflsnap" will be filled
  * in with the name.  (It must be at least ZFS_MAX_DATASET_NAME_LEN bytes long.)
  */
 int
 dsl_dataset_promote(const char *name, char *conflsnap)
 {
 	dsl_dataset_promote_arg_t ddpa = { 0 };
 	uint64_t numsnaps;
 	int error;
 	nvpair_t *snap_pair;
 	objset_t *os;
 
 	/*
 	 * We will modify space proportional to the number of
 	 * snapshots.  Compute numsnaps.
 	 */
 	error = dmu_objset_hold(name, FTAG, &os);
 	if (error != 0)
 		return (error);
 	error = zap_count(dmu_objset_pool(os)->dp_meta_objset,
 	    dsl_dataset_phys(dmu_objset_ds(os))->ds_snapnames_zapobj,
 	    &numsnaps);
 	dmu_objset_rele(os, FTAG);
 	if (error != 0)
 		return (error);
 
+	cred_t *cr = CRED();
+	crhold(cr);
+
 	ddpa.ddpa_clonename = name;
 	ddpa.err_ds = fnvlist_alloc();
-	ddpa.cr = CRED();
-	ddpa.proc = curproc;
+	ddpa.cr = cr;
 
 	error = dsl_sync_task(name, dsl_dataset_promote_check,
 	    dsl_dataset_promote_sync, &ddpa,
 	    2 + numsnaps, ZFS_SPACE_CHECK_RESERVED);
 
+	crfree(cr);
+
 	/*
 	 * Return the first conflicting snapshot found.
 	 */
 	snap_pair = nvlist_next_nvpair(ddpa.err_ds, NULL);
 	if (snap_pair != NULL && conflsnap != NULL)
 		(void) strlcpy(conflsnap, nvpair_name(snap_pair),
 		    ZFS_MAX_DATASET_NAME_LEN);
 
 	fnvlist_free(ddpa.err_ds);
 	return (error);
 }
 
 int
 dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone,
     dsl_dataset_t *origin_head, boolean_t force, void *owner, dmu_tx_t *tx)
 {
 	/*
 	 * "slack" factor for received datasets with refquota set on them.
 	 * See the bottom of this function for details on its use.
 	 */
 	uint64_t refquota_slack = (uint64_t)DMU_MAX_ACCESS *
 	    spa_asize_inflation;
 	int64_t unused_refres_delta;
 
 	/* they should both be heads */
 	if (clone->ds_is_snapshot ||
 	    origin_head->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	/* if we are not forcing, the branch point should be just before them */
 	if (!force && clone->ds_prev != origin_head->ds_prev)
 		return (SET_ERROR(EINVAL));
 
 	/* clone should be the clone (unless they are unrelated) */
 	if (clone->ds_prev != NULL &&
 	    clone->ds_prev != clone->ds_dir->dd_pool->dp_origin_snap &&
 	    origin_head->ds_dir != clone->ds_prev->ds_dir)
 		return (SET_ERROR(EINVAL));
 
 	/* the clone should be a child of the origin */
 	if (clone->ds_dir->dd_parent != origin_head->ds_dir)
 		return (SET_ERROR(EINVAL));
 
 	/* origin_head shouldn't be modified unless 'force' */
 	if (!force &&
 	    dsl_dataset_modified_since_snap(origin_head, origin_head->ds_prev))
 		return (SET_ERROR(ETXTBSY));
 
 	/* origin_head should have no long holds (e.g. is not mounted) */
 	if (dsl_dataset_handoff_check(origin_head, owner, tx))
 		return (SET_ERROR(EBUSY));
 
 	/* check amount of any unconsumed refreservation */
 	unused_refres_delta =
 	    (int64_t)MIN(origin_head->ds_reserved,
 	    dsl_dataset_phys(origin_head)->ds_unique_bytes) -
 	    (int64_t)MIN(origin_head->ds_reserved,
 	    dsl_dataset_phys(clone)->ds_unique_bytes);
 
 	if (unused_refres_delta > 0 &&
 	    unused_refres_delta >
 	    dsl_dir_space_available(origin_head->ds_dir, NULL, 0, TRUE))
 		return (SET_ERROR(ENOSPC));
 
 	/*
 	 * The clone can't be too much over the head's refquota.
 	 *
 	 * To ensure that the entire refquota can be used, we allow one
 	 * transaction to exceed the refquota.  Therefore, this check
 	 * needs to also allow for the space referenced to be more than the
 	 * refquota.  The maximum amount of space that one transaction can use
 	 * on disk is DMU_MAX_ACCESS * spa_asize_inflation.  Allowing this
 	 * overage ensures that we are able to receive a filesystem that
 	 * exceeds the refquota on the source system.
 	 *
 	 * So that overage is the refquota_slack we use below.
 	 */
 	if (origin_head->ds_quota != 0 &&
 	    dsl_dataset_phys(clone)->ds_referenced_bytes >
 	    origin_head->ds_quota + refquota_slack)
 		return (SET_ERROR(EDQUOT));
 
 	return (0);
 }
 
 static void
 dsl_dataset_swap_remap_deadlists(dsl_dataset_t *clone,
     dsl_dataset_t *origin, dmu_tx_t *tx)
 {
 	uint64_t clone_remap_dl_obj, origin_remap_dl_obj;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 
 	ASSERT(dsl_pool_sync_context(dp));
 
 	clone_remap_dl_obj = dsl_dataset_get_remap_deadlist_object(clone);
 	origin_remap_dl_obj = dsl_dataset_get_remap_deadlist_object(origin);
 
 	if (clone_remap_dl_obj != 0) {
 		dsl_deadlist_close(&clone->ds_remap_deadlist);
 		dsl_dataset_unset_remap_deadlist_object(clone, tx);
 	}
 	if (origin_remap_dl_obj != 0) {
 		dsl_deadlist_close(&origin->ds_remap_deadlist);
 		dsl_dataset_unset_remap_deadlist_object(origin, tx);
 	}
 
 	if (clone_remap_dl_obj != 0) {
 		dsl_dataset_set_remap_deadlist_object(origin,
 		    clone_remap_dl_obj, tx);
 		dsl_deadlist_open(&origin->ds_remap_deadlist,
 		    dp->dp_meta_objset, clone_remap_dl_obj);
 	}
 	if (origin_remap_dl_obj != 0) {
 		dsl_dataset_set_remap_deadlist_object(clone,
 		    origin_remap_dl_obj, tx);
 		dsl_deadlist_open(&clone->ds_remap_deadlist,
 		    dp->dp_meta_objset, origin_remap_dl_obj);
 	}
 }
 
 void
 dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
     dsl_dataset_t *origin_head, dmu_tx_t *tx)
 {
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	int64_t unused_refres_delta;
 
 	ASSERT(clone->ds_reserved == 0);
 	/*
 	 * NOTE: On DEBUG kernels there could be a race between this and
 	 * the check function if spa_asize_inflation is adjusted...
 	 */
 	ASSERT(origin_head->ds_quota == 0 ||
 	    dsl_dataset_phys(clone)->ds_unique_bytes <= origin_head->ds_quota +
 	    DMU_MAX_ACCESS * spa_asize_inflation);
 	ASSERT3P(clone->ds_prev, ==, origin_head->ds_prev);
 
 	dsl_dir_cancel_waiters(origin_head->ds_dir);
 
 	/*
 	 * Swap per-dataset feature flags.
 	 */
 	for (spa_feature_t f = 0; f < SPA_FEATURES; f++) {
 		if (!(spa_feature_table[f].fi_flags &
 		    ZFEATURE_FLAG_PER_DATASET)) {
 			ASSERT(!dsl_dataset_feature_is_active(clone, f));
 			ASSERT(!dsl_dataset_feature_is_active(origin_head, f));
 			continue;
 		}
 
 		boolean_t clone_inuse = dsl_dataset_feature_is_active(clone, f);
 		void *clone_feature = clone->ds_feature[f];
 		boolean_t origin_head_inuse =
 		    dsl_dataset_feature_is_active(origin_head, f);
 		void *origin_head_feature = origin_head->ds_feature[f];
 
 		if (clone_inuse)
 			dsl_dataset_deactivate_feature_impl(clone, f, tx);
 		if (origin_head_inuse)
 			dsl_dataset_deactivate_feature_impl(origin_head, f, tx);
 
 		if (clone_inuse) {
 			dsl_dataset_activate_feature(origin_head->ds_object, f,
 			    clone_feature, tx);
 			origin_head->ds_feature[f] = clone_feature;
 		}
 		if (origin_head_inuse) {
 			dsl_dataset_activate_feature(clone->ds_object, f,
 			    origin_head_feature, tx);
 			clone->ds_feature[f] = origin_head_feature;
 		}
 	}
 
 	dmu_buf_will_dirty(clone->ds_dbuf, tx);
 	dmu_buf_will_dirty(origin_head->ds_dbuf, tx);
 
 	if (clone->ds_objset != NULL) {
 		dmu_objset_evict(clone->ds_objset);
 		clone->ds_objset = NULL;
 	}
 
 	if (origin_head->ds_objset != NULL) {
 		dmu_objset_evict(origin_head->ds_objset);
 		origin_head->ds_objset = NULL;
 	}
 
 	unused_refres_delta =
 	    (int64_t)MIN(origin_head->ds_reserved,
 	    dsl_dataset_phys(origin_head)->ds_unique_bytes) -
 	    (int64_t)MIN(origin_head->ds_reserved,
 	    dsl_dataset_phys(clone)->ds_unique_bytes);
 
 	/*
 	 * Reset origin's unique bytes.
 	 */
 	{
 		dsl_dataset_t *origin = clone->ds_prev;
 		uint64_t comp, uncomp;
 
 		dmu_buf_will_dirty(origin->ds_dbuf, tx);
 		dsl_deadlist_space_range(&clone->ds_deadlist,
 		    dsl_dataset_phys(origin)->ds_prev_snap_txg, UINT64_MAX,
 		    &dsl_dataset_phys(origin)->ds_unique_bytes, &comp, &uncomp);
 	}
 
 	/* swap blkptrs */
 	{
 		rrw_enter(&clone->ds_bp_rwlock, RW_WRITER, FTAG);
 		rrw_enter(&origin_head->ds_bp_rwlock, RW_WRITER, FTAG);
 		blkptr_t tmp;
 		tmp = dsl_dataset_phys(origin_head)->ds_bp;
 		dsl_dataset_phys(origin_head)->ds_bp =
 		    dsl_dataset_phys(clone)->ds_bp;
 		dsl_dataset_phys(clone)->ds_bp = tmp;
 		rrw_exit(&origin_head->ds_bp_rwlock, FTAG);
 		rrw_exit(&clone->ds_bp_rwlock, FTAG);
 	}
 
 	/* set dd_*_bytes */
 	{
 		int64_t dused, dcomp, duncomp;
 		uint64_t cdl_used, cdl_comp, cdl_uncomp;
 		uint64_t odl_used, odl_comp, odl_uncomp;
 
 		ASSERT3U(dsl_dir_phys(clone->ds_dir)->
 		    dd_used_breakdown[DD_USED_SNAP], ==, 0);
 
 		dsl_deadlist_space(&clone->ds_deadlist,
 		    &cdl_used, &cdl_comp, &cdl_uncomp);
 		dsl_deadlist_space(&origin_head->ds_deadlist,
 		    &odl_used, &odl_comp, &odl_uncomp);
 
 		dused = dsl_dataset_phys(clone)->ds_referenced_bytes +
 		    cdl_used -
 		    (dsl_dataset_phys(origin_head)->ds_referenced_bytes +
 		    odl_used);
 		dcomp = dsl_dataset_phys(clone)->ds_compressed_bytes +
 		    cdl_comp -
 		    (dsl_dataset_phys(origin_head)->ds_compressed_bytes +
 		    odl_comp);
 		duncomp = dsl_dataset_phys(clone)->ds_uncompressed_bytes +
 		    cdl_uncomp -
 		    (dsl_dataset_phys(origin_head)->ds_uncompressed_bytes +
 		    odl_uncomp);
 
 		dsl_dir_diduse_space(origin_head->ds_dir, DD_USED_HEAD,
 		    dused, dcomp, duncomp, tx);
 		dsl_dir_diduse_space(clone->ds_dir, DD_USED_HEAD,
 		    -dused, -dcomp, -duncomp, tx);
 
 		/*
 		 * The difference in the space used by snapshots is the
 		 * difference in snapshot space due to the head's
 		 * deadlist (since that's the only thing that's
 		 * changing that affects the snapused).
 		 */
 		dsl_deadlist_space_range(&clone->ds_deadlist,
 		    origin_head->ds_dir->dd_origin_txg, UINT64_MAX,
 		    &cdl_used, &cdl_comp, &cdl_uncomp);
 		dsl_deadlist_space_range(&origin_head->ds_deadlist,
 		    origin_head->ds_dir->dd_origin_txg, UINT64_MAX,
 		    &odl_used, &odl_comp, &odl_uncomp);
 		dsl_dir_transfer_space(origin_head->ds_dir, cdl_used - odl_used,
 		    DD_USED_HEAD, DD_USED_SNAP, tx);
 	}
 
 	/* swap ds_*_bytes */
 	SWITCH64(dsl_dataset_phys(origin_head)->ds_referenced_bytes,
 	    dsl_dataset_phys(clone)->ds_referenced_bytes);
 	SWITCH64(dsl_dataset_phys(origin_head)->ds_compressed_bytes,
 	    dsl_dataset_phys(clone)->ds_compressed_bytes);
 	SWITCH64(dsl_dataset_phys(origin_head)->ds_uncompressed_bytes,
 	    dsl_dataset_phys(clone)->ds_uncompressed_bytes);
 	SWITCH64(dsl_dataset_phys(origin_head)->ds_unique_bytes,
 	    dsl_dataset_phys(clone)->ds_unique_bytes);
 
 	/* apply any parent delta for change in unconsumed refreservation */
 	dsl_dir_diduse_space(origin_head->ds_dir, DD_USED_REFRSRV,
 	    unused_refres_delta, 0, 0, tx);
 
 	/*
 	 * Swap deadlists.
 	 */
 	dsl_deadlist_close(&clone->ds_deadlist);
 	dsl_deadlist_close(&origin_head->ds_deadlist);
 	SWITCH64(dsl_dataset_phys(origin_head)->ds_deadlist_obj,
 	    dsl_dataset_phys(clone)->ds_deadlist_obj);
 	dsl_deadlist_open(&clone->ds_deadlist, dp->dp_meta_objset,
 	    dsl_dataset_phys(clone)->ds_deadlist_obj);
 	dsl_deadlist_open(&origin_head->ds_deadlist, dp->dp_meta_objset,
 	    dsl_dataset_phys(origin_head)->ds_deadlist_obj);
 	dsl_dataset_swap_remap_deadlists(clone, origin_head, tx);
 
 	/*
 	 * If there is a bookmark at the origin, its "next dataset" is
 	 * changing, so we need to reset its FBN.
 	 */
 	dsl_bookmark_next_changed(origin_head, origin_head->ds_prev, tx);
 
 	dsl_scan_ds_clone_swapped(origin_head, clone, tx);
 
 	/*
 	 * Destroy any livelists associated with the clone or the origin,
 	 * since after the swap the corresponding livelists are no longer
 	 * valid.
 	 */
 	dsl_dir_remove_livelist(clone->ds_dir, tx, B_TRUE);
 	dsl_dir_remove_livelist(origin_head->ds_dir, tx, B_TRUE);
 
 	spa_history_log_internal_ds(clone, "clone swap", tx,
 	    "parent=%s", origin_head->ds_dir->dd_myname);
 }
 
 /*
  * Given a pool name and a dataset object number in that pool,
  * return the name of that dataset.
  */
 int
 dsl_dsobj_to_dsname(char *pname, uint64_t obj, char *buf)
 {
 	dsl_pool_t *dp;
 	dsl_dataset_t *ds;
 	int error;
 
 	error = dsl_pool_hold(pname, FTAG, &dp);
 	if (error != 0)
 		return (error);
 
 	error = dsl_dataset_hold_obj(dp, obj, FTAG, &ds);
 	if (error == 0) {
 		dsl_dataset_name(ds, buf);
 		dsl_dataset_rele(ds, FTAG);
 	}
 	dsl_pool_rele(dp, FTAG);
 
 	return (error);
 }
 
 int
 dsl_dataset_check_quota(dsl_dataset_t *ds, boolean_t check_quota,
     uint64_t asize, uint64_t inflight, uint64_t *used, uint64_t *ref_rsrv)
 {
 	int error = 0;
 
 	ASSERT3S(asize, >, 0);
 
 	/*
 	 * *ref_rsrv is the portion of asize that will come from any
 	 * unconsumed refreservation space.
 	 */
 	*ref_rsrv = 0;
 
 	mutex_enter(&ds->ds_lock);
 	/*
 	 * Make a space adjustment for reserved bytes.
 	 */
 	if (ds->ds_reserved > dsl_dataset_phys(ds)->ds_unique_bytes) {
 		ASSERT3U(*used, >=,
 		    ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes);
 		*used -=
 		    (ds->ds_reserved - dsl_dataset_phys(ds)->ds_unique_bytes);
 		*ref_rsrv =
 		    asize - MIN(asize, parent_delta(ds, asize + inflight));
 	}
 
 	if (!check_quota || ds->ds_quota == 0) {
 		mutex_exit(&ds->ds_lock);
 		return (0);
 	}
 	/*
 	 * If they are requesting more space, and our current estimate
 	 * is over quota, they get to try again unless the actual
 	 * on-disk is over quota and there are no pending changes (which
 	 * may free up space for us).
 	 */
 	if (dsl_dataset_phys(ds)->ds_referenced_bytes + inflight >=
 	    ds->ds_quota) {
 		if (inflight > 0 ||
 		    dsl_dataset_phys(ds)->ds_referenced_bytes < ds->ds_quota)
 			error = SET_ERROR(ERESTART);
 		else
 			error = SET_ERROR(EDQUOT);
 	}
 	mutex_exit(&ds->ds_lock);
 
 	return (error);
 }
 
 typedef struct dsl_dataset_set_qr_arg {
 	const char *ddsqra_name;
 	zprop_source_t ddsqra_source;
 	uint64_t ddsqra_value;
 } dsl_dataset_set_qr_arg_t;
 
 
 static int
 dsl_dataset_set_refquota_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_set_qr_arg_t *ddsqra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds;
 	int error;
 	uint64_t newval;
 
 	if (spa_version(dp->dp_spa) < SPA_VERSION_REFQUOTA)
 		return (SET_ERROR(ENOTSUP));
 
 	error = dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds);
 	if (error != 0)
 		return (error);
 
 	if (ds->ds_is_snapshot) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
 
 	error = dsl_prop_predict(ds->ds_dir,
 	    zfs_prop_to_name(ZFS_PROP_REFQUOTA),
 	    ddsqra->ddsqra_source, ddsqra->ddsqra_value, &newval);
 	if (error != 0) {
 		dsl_dataset_rele(ds, FTAG);
 		return (error);
 	}
 
 	if (newval == 0) {
 		dsl_dataset_rele(ds, FTAG);
 		return (0);
 	}
 
 	if (newval < dsl_dataset_phys(ds)->ds_referenced_bytes ||
 	    newval < ds->ds_reserved) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(ENOSPC));
 	}
 
 	dsl_dataset_rele(ds, FTAG);
 	return (0);
 }
 
 static void
 dsl_dataset_set_refquota_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_set_qr_arg_t *ddsqra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds = NULL;
 	uint64_t newval;
 
 	VERIFY0(dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds));
 
 	dsl_prop_set_sync_impl(ds,
 	    zfs_prop_to_name(ZFS_PROP_REFQUOTA),
 	    ddsqra->ddsqra_source, sizeof (ddsqra->ddsqra_value), 1,
 	    &ddsqra->ddsqra_value, tx);
 
 	VERIFY0(dsl_prop_get_int_ds(ds,
 	    zfs_prop_to_name(ZFS_PROP_REFQUOTA), &newval));
 
 	if (ds->ds_quota != newval) {
 		dmu_buf_will_dirty(ds->ds_dbuf, tx);
 		ds->ds_quota = newval;
 	}
 	dsl_dataset_rele(ds, FTAG);
 }
 
 int
 dsl_dataset_set_refquota(const char *dsname, zprop_source_t source,
     uint64_t refquota)
 {
 	dsl_dataset_set_qr_arg_t ddsqra;
 
 	ddsqra.ddsqra_name = dsname;
 	ddsqra.ddsqra_source = source;
 	ddsqra.ddsqra_value = refquota;
 
 	return (dsl_sync_task(dsname, dsl_dataset_set_refquota_check,
 	    dsl_dataset_set_refquota_sync, &ddsqra, 0,
 	    ZFS_SPACE_CHECK_EXTRA_RESERVED));
 }
 
 static int
 dsl_dataset_set_refreservation_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_set_qr_arg_t *ddsqra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds;
 	int error;
 	uint64_t newval, unique;
 
 	if (spa_version(dp->dp_spa) < SPA_VERSION_REFRESERVATION)
 		return (SET_ERROR(ENOTSUP));
 
 	error = dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds);
 	if (error != 0)
 		return (error);
 
 	if (ds->ds_is_snapshot) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
 
 	error = dsl_prop_predict(ds->ds_dir,
 	    zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
 	    ddsqra->ddsqra_source, ddsqra->ddsqra_value, &newval);
 	if (error != 0) {
 		dsl_dataset_rele(ds, FTAG);
 		return (error);
 	}
 
 	/*
 	 * If we are doing the preliminary check in open context, the
 	 * space estimates may be inaccurate.
 	 */
 	if (!dmu_tx_is_syncing(tx)) {
 		dsl_dataset_rele(ds, FTAG);
 		return (0);
 	}
 
 	mutex_enter(&ds->ds_lock);
 	if (!DS_UNIQUE_IS_ACCURATE(ds))
 		dsl_dataset_recalc_head_uniq(ds);
 	unique = dsl_dataset_phys(ds)->ds_unique_bytes;
 	mutex_exit(&ds->ds_lock);
 
 	if (MAX(unique, newval) > MAX(unique, ds->ds_reserved)) {
 		uint64_t delta = MAX(unique, newval) -
 		    MAX(unique, ds->ds_reserved);
 
 		if (delta >
 		    dsl_dir_space_available(ds->ds_dir, NULL, 0, B_TRUE) ||
 		    (ds->ds_quota > 0 && newval > ds->ds_quota)) {
 			dsl_dataset_rele(ds, FTAG);
 			return (SET_ERROR(ENOSPC));
 		}
 	}
 
 	dsl_dataset_rele(ds, FTAG);
 	return (0);
 }
 
 void
 dsl_dataset_set_refreservation_sync_impl(dsl_dataset_t *ds,
     zprop_source_t source, uint64_t value, dmu_tx_t *tx)
 {
 	uint64_t newval;
 	uint64_t unique;
 	int64_t delta;
 
 	dsl_prop_set_sync_impl(ds, zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
 	    source, sizeof (value), 1, &value, tx);
 
 	VERIFY0(dsl_prop_get_int_ds(ds,
 	    zfs_prop_to_name(ZFS_PROP_REFRESERVATION), &newval));
 
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
 	mutex_enter(&ds->ds_dir->dd_lock);
 	mutex_enter(&ds->ds_lock);
 	ASSERT(DS_UNIQUE_IS_ACCURATE(ds));
 	unique = dsl_dataset_phys(ds)->ds_unique_bytes;
 	delta = MAX(0, (int64_t)(newval - unique)) -
 	    MAX(0, (int64_t)(ds->ds_reserved - unique));
 	ds->ds_reserved = newval;
 	mutex_exit(&ds->ds_lock);
 
 	dsl_dir_diduse_space(ds->ds_dir, DD_USED_REFRSRV, delta, 0, 0, tx);
 	mutex_exit(&ds->ds_dir->dd_lock);
 }
 
 static void
 dsl_dataset_set_refreservation_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_set_qr_arg_t *ddsqra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds = NULL;
 
 	VERIFY0(dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds));
 	dsl_dataset_set_refreservation_sync_impl(ds,
 	    ddsqra->ddsqra_source, ddsqra->ddsqra_value, tx);
 	dsl_dataset_rele(ds, FTAG);
 }
 
 int
 dsl_dataset_set_refreservation(const char *dsname, zprop_source_t source,
     uint64_t refreservation)
 {
 	dsl_dataset_set_qr_arg_t ddsqra;
 
 	ddsqra.ddsqra_name = dsname;
 	ddsqra.ddsqra_source = source;
 	ddsqra.ddsqra_value = refreservation;
 
 	return (dsl_sync_task(dsname, dsl_dataset_set_refreservation_check,
 	    dsl_dataset_set_refreservation_sync, &ddsqra, 0,
 	    ZFS_SPACE_CHECK_EXTRA_RESERVED));
 }
 
 typedef struct dsl_dataset_set_compression_arg {
 	const char *ddsca_name;
 	zprop_source_t ddsca_source;
 	uint64_t ddsca_value;
 } dsl_dataset_set_compression_arg_t;
 
 static int
 dsl_dataset_set_compression_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_set_compression_arg_t *ddsca = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 
 	uint64_t compval = ZIO_COMPRESS_ALGO(ddsca->ddsca_value);
 	spa_feature_t f = zio_compress_to_feature(compval);
 
 	if (f == SPA_FEATURE_NONE)
 		return (SET_ERROR(EINVAL));
 
 	if (!spa_feature_is_enabled(dp->dp_spa, f))
 		return (SET_ERROR(ENOTSUP));
 
 	return (0);
 }
 
 static void
 dsl_dataset_set_compression_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dataset_set_compression_arg_t *ddsca = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds = NULL;
 
 	uint64_t compval = ZIO_COMPRESS_ALGO(ddsca->ddsca_value);
 	spa_feature_t f = zio_compress_to_feature(compval);
 	ASSERT3S(f, !=, SPA_FEATURE_NONE);
 	ASSERT3S(spa_feature_table[f].fi_type, ==, ZFEATURE_TYPE_BOOLEAN);
 
 	VERIFY0(dsl_dataset_hold(dp, ddsca->ddsca_name, FTAG, &ds));
 	if (zfeature_active(f, ds->ds_feature[f]) != B_TRUE) {
 		ds->ds_feature_activation[f] = (void *)B_TRUE;
 		dsl_dataset_activate_feature(ds->ds_object, f,
 		    ds->ds_feature_activation[f], tx);
 		ds->ds_feature[f] = ds->ds_feature_activation[f];
 	}
 	dsl_dataset_rele(ds, FTAG);
 }
 
 int
 dsl_dataset_set_compression(const char *dsname, zprop_source_t source,
     uint64_t compression)
 {
 	dsl_dataset_set_compression_arg_t ddsca;
 
 	/*
 	 * The sync task is only required for zstd in order to activate
 	 * the feature flag when the property is first set.
 	 */
 	if (ZIO_COMPRESS_ALGO(compression) != ZIO_COMPRESS_ZSTD)
 		return (0);
 
 	ddsca.ddsca_name = dsname;
 	ddsca.ddsca_source = source;
 	ddsca.ddsca_value = compression;
 
 	return (dsl_sync_task(dsname, dsl_dataset_set_compression_check,
 	    dsl_dataset_set_compression_sync, &ddsca, 0,
 	    ZFS_SPACE_CHECK_EXTRA_RESERVED));
 }
 
 /*
  * Return (in *usedp) the amount of space referenced by "new" that was not
  * referenced at the time the bookmark corresponds to.  "New" may be a
  * snapshot or a head.  The bookmark must be before new, in
  * new's filesystem (or its origin) -- caller verifies this.
  *
  * The written space is calculated by considering two components:  First, we
  * ignore any freed space, and calculate the written as new's used space
  * minus old's used space.  Next, we add in the amount of space that was freed
  * between the two time points, thus reducing new's used space relative to
  * old's. Specifically, this is the space that was born before
  * zbm_creation_txg, and freed before new (ie. on new's deadlist or a
  * previous deadlist).
  *
  * space freed                         [---------------------]
  * snapshots                       ---O-------O--------O-------O------
  *                                         bookmark           new
  *
  * Note, the bookmark's zbm_*_bytes_refd must be valid, but if the HAS_FBN
  * flag is not set, we will calculate the freed_before_next based on the
  * next snapshot's deadlist, rather than using zbm_*_freed_before_next_snap.
  */
 static int
 dsl_dataset_space_written_impl(zfs_bookmark_phys_t *bmp,
     dsl_dataset_t *new, uint64_t *usedp, uint64_t *compp, uint64_t *uncompp)
 {
 	int err = 0;
 	dsl_pool_t *dp = new->ds_dir->dd_pool;
 
 	ASSERT(dsl_pool_config_held(dp));
 	if (dsl_dataset_is_snapshot(new)) {
 		ASSERT3U(bmp->zbm_creation_txg, <,
 		    dsl_dataset_phys(new)->ds_creation_txg);
 	}
 
 	*usedp = 0;
 	*usedp += dsl_dataset_phys(new)->ds_referenced_bytes;
 	*usedp -= bmp->zbm_referenced_bytes_refd;
 
 	*compp = 0;
 	*compp += dsl_dataset_phys(new)->ds_compressed_bytes;
 	*compp -= bmp->zbm_compressed_bytes_refd;
 
 	*uncompp = 0;
 	*uncompp += dsl_dataset_phys(new)->ds_uncompressed_bytes;
 	*uncompp -= bmp->zbm_uncompressed_bytes_refd;
 
 	dsl_dataset_t *snap = new;
 
 	while (dsl_dataset_phys(snap)->ds_prev_snap_txg >
 	    bmp->zbm_creation_txg) {
 		uint64_t used, comp, uncomp;
 
 		dsl_deadlist_space_range(&snap->ds_deadlist,
 		    0, bmp->zbm_creation_txg,
 		    &used, &comp, &uncomp);
 		*usedp += used;
 		*compp += comp;
 		*uncompp += uncomp;
 
 		uint64_t snapobj = dsl_dataset_phys(snap)->ds_prev_snap_obj;
 		if (snap != new)
 			dsl_dataset_rele(snap, FTAG);
 		err = dsl_dataset_hold_obj(dp, snapobj, FTAG, &snap);
 		if (err != 0)
 			break;
 	}
 
 	/*
 	 * We might not have the FBN if we are calculating written from
 	 * a snapshot (because we didn't know the correct "next" snapshot
 	 * until now).
 	 */
 	if (bmp->zbm_flags & ZBM_FLAG_HAS_FBN) {
 		*usedp += bmp->zbm_referenced_freed_before_next_snap;
 		*compp += bmp->zbm_compressed_freed_before_next_snap;
 		*uncompp += bmp->zbm_uncompressed_freed_before_next_snap;
 	} else {
 		ASSERT3U(dsl_dataset_phys(snap)->ds_prev_snap_txg, ==,
 		    bmp->zbm_creation_txg);
 		uint64_t used, comp, uncomp;
 		dsl_deadlist_space(&snap->ds_deadlist, &used, &comp, &uncomp);
 		*usedp += used;
 		*compp += comp;
 		*uncompp += uncomp;
 	}
 	if (snap != new)
 		dsl_dataset_rele(snap, FTAG);
 	return (err);
 }
 
 /*
  * Return (in *usedp) the amount of space written in new that was not
  * present at the time the bookmark corresponds to.  New may be a
  * snapshot or the head.  Old must be a bookmark before new, in
  * new's filesystem (or its origin) -- caller verifies this.
  */
 int
 dsl_dataset_space_written_bookmark(zfs_bookmark_phys_t *bmp,
     dsl_dataset_t *new, uint64_t *usedp, uint64_t *compp, uint64_t *uncompp)
 {
 	if (!(bmp->zbm_flags & ZBM_FLAG_HAS_FBN))
 		return (SET_ERROR(ENOTSUP));
 	return (dsl_dataset_space_written_impl(bmp, new,
 	    usedp, compp, uncompp));
 }
 
 /*
  * Return (in *usedp) the amount of space written in new that is not
  * present in oldsnap.  New may be a snapshot or the head.  Old must be
  * a snapshot before new, in new's filesystem (or its origin).  If not then
  * fail and return EINVAL.
  */
 int
 dsl_dataset_space_written(dsl_dataset_t *oldsnap, dsl_dataset_t *new,
     uint64_t *usedp, uint64_t *compp, uint64_t *uncompp)
 {
 	if (!dsl_dataset_is_before(new, oldsnap, 0))
 		return (SET_ERROR(EINVAL));
 
 	zfs_bookmark_phys_t zbm = { 0 };
 	dsl_dataset_phys_t *dsp = dsl_dataset_phys(oldsnap);
 	zbm.zbm_guid = dsp->ds_guid;
 	zbm.zbm_creation_txg = dsp->ds_creation_txg;
 	zbm.zbm_creation_time = dsp->ds_creation_time;
 	zbm.zbm_referenced_bytes_refd = dsp->ds_referenced_bytes;
 	zbm.zbm_compressed_bytes_refd = dsp->ds_compressed_bytes;
 	zbm.zbm_uncompressed_bytes_refd = dsp->ds_uncompressed_bytes;
 
 	/*
 	 * If oldsnap is the origin (or origin's origin, ...) of new,
 	 * we can't easily calculate the effective FBN.  Therefore,
 	 * we do not set ZBM_FLAG_HAS_FBN, so that the _impl will calculate
 	 * it relative to the correct "next": the next snapshot towards "new",
 	 * rather than the next snapshot in oldsnap's dsl_dir.
 	 */
 	return (dsl_dataset_space_written_impl(&zbm, new,
 	    usedp, compp, uncompp));
 }
 
 /*
  * Return (in *usedp) the amount of space that will be reclaimed if firstsnap,
  * lastsnap, and all snapshots in between are deleted.
  *
  * blocks that would be freed            [---------------------------]
  * snapshots                       ---O-------O--------O-------O--------O
  *                                        firstsnap        lastsnap
  *
  * This is the set of blocks that were born after the snap before firstsnap,
  * (birth > firstsnap->prev_snap_txg) and died before the snap after the
  * last snap (ie, is on lastsnap->ds_next->ds_deadlist or an earlier deadlist).
  * We calculate this by iterating over the relevant deadlists (from the snap
  * after lastsnap, backward to the snap after firstsnap), summing up the
  * space on the deadlist that was born after the snap before firstsnap.
  */
 int
 dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap,
     dsl_dataset_t *lastsnap,
     uint64_t *usedp, uint64_t *compp, uint64_t *uncompp)
 {
 	int err = 0;
 	uint64_t snapobj;
 	dsl_pool_t *dp = firstsnap->ds_dir->dd_pool;
 
 	ASSERT(firstsnap->ds_is_snapshot);
 	ASSERT(lastsnap->ds_is_snapshot);
 
 	/*
 	 * Check that the snapshots are in the same dsl_dir, and firstsnap
 	 * is before lastsnap.
 	 */
 	if (firstsnap->ds_dir != lastsnap->ds_dir ||
 	    dsl_dataset_phys(firstsnap)->ds_creation_txg >
 	    dsl_dataset_phys(lastsnap)->ds_creation_txg)
 		return (SET_ERROR(EINVAL));
 
 	*usedp = *compp = *uncompp = 0;
 
 	snapobj = dsl_dataset_phys(lastsnap)->ds_next_snap_obj;
 	while (snapobj != firstsnap->ds_object) {
 		dsl_dataset_t *ds;
 		uint64_t used, comp, uncomp;
 
 		err = dsl_dataset_hold_obj(dp, snapobj, FTAG, &ds);
 		if (err != 0)
 			break;
 
 		dsl_deadlist_space_range(&ds->ds_deadlist,
 		    dsl_dataset_phys(firstsnap)->ds_prev_snap_txg, UINT64_MAX,
 		    &used, &comp, &uncomp);
 		*usedp += used;
 		*compp += comp;
 		*uncompp += uncomp;
 
 		snapobj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 		ASSERT3U(snapobj, !=, 0);
 		dsl_dataset_rele(ds, FTAG);
 	}
 	return (err);
 }
 
 /*
  * Return TRUE if 'earlier' is an earlier snapshot in 'later's timeline.
  * For example, they could both be snapshots of the same filesystem, and
  * 'earlier' is before 'later'.  Or 'earlier' could be the origin of
  * 'later's filesystem.  Or 'earlier' could be an older snapshot in the origin's
  * filesystem.  Or 'earlier' could be the origin's origin.
  *
  * If non-zero, earlier_txg is used instead of earlier's ds_creation_txg.
  */
 boolean_t
 dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
     uint64_t earlier_txg)
 {
 	dsl_pool_t *dp = later->ds_dir->dd_pool;
 	int error;
 	boolean_t ret;
 
 	ASSERT(dsl_pool_config_held(dp));
 	ASSERT(earlier->ds_is_snapshot || earlier_txg != 0);
 
 	if (earlier_txg == 0)
 		earlier_txg = dsl_dataset_phys(earlier)->ds_creation_txg;
 
 	if (later->ds_is_snapshot &&
 	    earlier_txg >= dsl_dataset_phys(later)->ds_creation_txg)
 		return (B_FALSE);
 
 	if (later->ds_dir == earlier->ds_dir)
 		return (B_TRUE);
 
 	/*
 	 * We check dd_origin_obj explicitly here rather than using
 	 * dsl_dir_is_clone() so that we will return TRUE if "earlier"
 	 * is $ORIGIN@$ORIGIN.  dsl_dataset_space_written() depends on
 	 * this behavior.
 	 */
 	if (dsl_dir_phys(later->ds_dir)->dd_origin_obj == 0)
 		return (B_FALSE);
 
 	dsl_dataset_t *origin;
 	error = dsl_dataset_hold_obj(dp,
 	    dsl_dir_phys(later->ds_dir)->dd_origin_obj, FTAG, &origin);
 	if (error != 0)
 		return (B_FALSE);
 	if (dsl_dataset_phys(origin)->ds_creation_txg == earlier_txg &&
 	    origin->ds_dir == earlier->ds_dir) {
 		dsl_dataset_rele(origin, FTAG);
 		return (B_TRUE);
 	}
 	ret = dsl_dataset_is_before(origin, earlier, earlier_txg);
 	dsl_dataset_rele(origin, FTAG);
 	return (ret);
 }
 
 void
 dsl_dataset_zapify(dsl_dataset_t *ds, dmu_tx_t *tx)
 {
 	objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
 	dmu_object_zapify(mos, ds->ds_object, DMU_OT_DSL_DATASET, tx);
 }
 
 boolean_t
 dsl_dataset_is_zapified(dsl_dataset_t *ds)
 {
 	dmu_object_info_t doi;
 
 	dmu_object_info_from_db(ds->ds_dbuf, &doi);
 	return (doi.doi_type == DMU_OTN_ZAP_METADATA);
 }
 
 boolean_t
 dsl_dataset_has_resume_receive_state(dsl_dataset_t *ds)
 {
 	return (dsl_dataset_is_zapified(ds) &&
 	    zap_contains(ds->ds_dir->dd_pool->dp_meta_objset,
 	    ds->ds_object, DS_FIELD_RESUME_TOGUID) == 0);
 }
 
 uint64_t
 dsl_dataset_get_remap_deadlist_object(dsl_dataset_t *ds)
 {
 	uint64_t remap_deadlist_obj;
 	int err;
 
 	if (!dsl_dataset_is_zapified(ds))
 		return (0);
 
 	err = zap_lookup(ds->ds_dir->dd_pool->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_REMAP_DEADLIST, sizeof (remap_deadlist_obj), 1,
 	    &remap_deadlist_obj);
 
 	if (err != 0) {
 		VERIFY3S(err, ==, ENOENT);
 		return (0);
 	}
 
 	ASSERT(remap_deadlist_obj != 0);
 	return (remap_deadlist_obj);
 }
 
 boolean_t
 dsl_dataset_remap_deadlist_exists(dsl_dataset_t *ds)
 {
 	EQUIV(dsl_deadlist_is_open(&ds->ds_remap_deadlist),
 	    dsl_dataset_get_remap_deadlist_object(ds) != 0);
 	return (dsl_deadlist_is_open(&ds->ds_remap_deadlist));
 }
 
 static void
 dsl_dataset_set_remap_deadlist_object(dsl_dataset_t *ds, uint64_t obj,
     dmu_tx_t *tx)
 {
 	ASSERT(obj != 0);
 	dsl_dataset_zapify(ds, tx);
 	VERIFY0(zap_add(ds->ds_dir->dd_pool->dp_meta_objset, ds->ds_object,
 	    DS_FIELD_REMAP_DEADLIST, sizeof (obj), 1, &obj, tx));
 }
 
 static void
 dsl_dataset_unset_remap_deadlist_object(dsl_dataset_t *ds, dmu_tx_t *tx)
 {
 	VERIFY0(zap_remove(ds->ds_dir->dd_pool->dp_meta_objset,
 	    ds->ds_object, DS_FIELD_REMAP_DEADLIST, tx));
 }
 
 void
 dsl_dataset_destroy_remap_deadlist(dsl_dataset_t *ds, dmu_tx_t *tx)
 {
 	uint64_t remap_deadlist_object;
 	spa_t *spa = ds->ds_dir->dd_pool->dp_spa;
 
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(dsl_dataset_remap_deadlist_exists(ds));
 
 	remap_deadlist_object = ds->ds_remap_deadlist.dl_object;
 	dsl_deadlist_close(&ds->ds_remap_deadlist);
 	dsl_deadlist_free(spa_meta_objset(spa), remap_deadlist_object, tx);
 	dsl_dataset_unset_remap_deadlist_object(ds, tx);
 	spa_feature_decr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
 }
 
 void
 dsl_dataset_create_remap_deadlist(dsl_dataset_t *ds, dmu_tx_t *tx)
 {
 	uint64_t remap_deadlist_obj;
 	spa_t *spa = ds->ds_dir->dd_pool->dp_spa;
 
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(MUTEX_HELD(&ds->ds_remap_deadlist_lock));
 	/*
 	 * Currently we only create remap deadlists when there are indirect
 	 * vdevs with referenced mappings.
 	 */
 	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_DEVICE_REMOVAL));
 
 	remap_deadlist_obj = dsl_deadlist_clone(
 	    &ds->ds_deadlist, UINT64_MAX,
 	    dsl_dataset_phys(ds)->ds_prev_snap_obj, tx);
 	dsl_dataset_set_remap_deadlist_object(ds,
 	    remap_deadlist_obj, tx);
 	dsl_deadlist_open(&ds->ds_remap_deadlist, spa_meta_objset(spa),
 	    remap_deadlist_obj);
 	spa_feature_incr(spa, SPA_FEATURE_OBSOLETE_COUNTS, tx);
 }
 
 void
 dsl_dataset_activate_redaction(dsl_dataset_t *ds, uint64_t *redact_snaps,
     uint64_t num_redact_snaps, dmu_tx_t *tx)
 {
 	uint64_t dsobj = ds->ds_object;
 	struct feature_type_uint64_array_arg *ftuaa =
 	    kmem_zalloc(sizeof (*ftuaa), KM_SLEEP);
 	ftuaa->length = (int64_t)num_redact_snaps;
 	if (num_redact_snaps > 0) {
 		ftuaa->array = kmem_alloc(num_redact_snaps * sizeof (uint64_t),
 		    KM_SLEEP);
 		memcpy(ftuaa->array, redact_snaps, num_redact_snaps *
 		    sizeof (uint64_t));
 	}
 	dsl_dataset_activate_feature(dsobj, SPA_FEATURE_REDACTED_DATASETS,
 	    ftuaa, tx);
 	ds->ds_feature[SPA_FEATURE_REDACTED_DATASETS] = ftuaa;
 }
 
 /*
  * Find and return (in *oldest_dsobj) the oldest snapshot of the dsobj
  * dataset whose birth time is >= min_txg.
  */
 int
 dsl_dataset_oldest_snapshot(spa_t *spa, uint64_t head_ds, uint64_t min_txg,
     uint64_t *oldest_dsobj)
 {
 	dsl_dataset_t *ds;
 	dsl_pool_t *dp = spa->spa_dsl_pool;
 
 	int error = dsl_dataset_hold_obj(dp, head_ds, FTAG, &ds);
 	if (error != 0)
 		return (error);
 
 	uint64_t prev_obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 	uint64_t prev_obj_txg = dsl_dataset_phys(ds)->ds_prev_snap_txg;
 
 	while (prev_obj != 0 && min_txg < prev_obj_txg) {
 		dsl_dataset_rele(ds, FTAG);
 		if ((error = dsl_dataset_hold_obj(dp, prev_obj,
 		    FTAG, &ds)) != 0)
 			return (error);
 		prev_obj_txg = dsl_dataset_phys(ds)->ds_prev_snap_txg;
 		prev_obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
 	}
 	*oldest_dsobj = ds->ds_object;
 	dsl_dataset_rele(ds, FTAG);
 	return (0);
 }
 
 ZFS_MODULE_PARAM(zfs, zfs_, max_recordsize, UINT, ZMOD_RW,
 	"Max allowed record size");
 
 ZFS_MODULE_PARAM(zfs, zfs_, allow_redacted_dataset_mount, INT, ZMOD_RW,
 	"Allow mounting of redacted datasets");
 
 ZFS_MODULE_PARAM(zfs, zfs_, snapshot_history_enabled, INT, ZMOD_RW,
 	"Include snapshot events in pool history/events");
 
 EXPORT_SYMBOL(dsl_dataset_hold);
 EXPORT_SYMBOL(dsl_dataset_hold_flags);
 EXPORT_SYMBOL(dsl_dataset_hold_obj);
 EXPORT_SYMBOL(dsl_dataset_hold_obj_flags);
 EXPORT_SYMBOL(dsl_dataset_own);
 EXPORT_SYMBOL(dsl_dataset_own_obj);
 EXPORT_SYMBOL(dsl_dataset_name);
 EXPORT_SYMBOL(dsl_dataset_rele);
 EXPORT_SYMBOL(dsl_dataset_rele_flags);
 EXPORT_SYMBOL(dsl_dataset_disown);
 EXPORT_SYMBOL(dsl_dataset_tryown);
 EXPORT_SYMBOL(dsl_dataset_create_sync);
 EXPORT_SYMBOL(dsl_dataset_create_sync_dd);
 EXPORT_SYMBOL(dsl_dataset_snapshot_check);
 EXPORT_SYMBOL(dsl_dataset_snapshot_sync);
 EXPORT_SYMBOL(dsl_dataset_promote);
 EXPORT_SYMBOL(dsl_dataset_user_hold);
 EXPORT_SYMBOL(dsl_dataset_user_release);
 EXPORT_SYMBOL(dsl_dataset_get_holds);
 EXPORT_SYMBOL(dsl_dataset_get_blkptr);
 EXPORT_SYMBOL(dsl_dataset_get_spa);
 EXPORT_SYMBOL(dsl_dataset_modified_since_snap);
 EXPORT_SYMBOL(dsl_dataset_space_written);
 EXPORT_SYMBOL(dsl_dataset_space_wouldfree);
 EXPORT_SYMBOL(dsl_dataset_sync);
 EXPORT_SYMBOL(dsl_dataset_block_born);
 EXPORT_SYMBOL(dsl_dataset_block_kill);
 EXPORT_SYMBOL(dsl_dataset_dirty);
 EXPORT_SYMBOL(dsl_dataset_stats);
 EXPORT_SYMBOL(dsl_dataset_fast_stat);
 EXPORT_SYMBOL(dsl_dataset_space);
 EXPORT_SYMBOL(dsl_dataset_fsid_guid);
 EXPORT_SYMBOL(dsl_dsobj_to_dsname);
 EXPORT_SYMBOL(dsl_dataset_check_quota);
 EXPORT_SYMBOL(dsl_dataset_clone_swap_check_impl);
 EXPORT_SYMBOL(dsl_dataset_clone_swap_sync_impl);
diff --git a/sys/contrib/openzfs/module/zfs/dsl_dir.c b/sys/contrib/openzfs/module/zfs/dsl_dir.c
index baf970121a61..4839fb39f5b3 100644
--- a/sys/contrib/openzfs/module/zfs/dsl_dir.c
+++ b/sys/contrib/openzfs/module/zfs/dsl_dir.c
@@ -1,2493 +1,2489 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
  * Copyright (c) 2013 Martin Matuska. All rights reserved.
  * Copyright (c) 2014 Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
  * Copyright (c) 2018, loli10K <ezomori.nozomu@gmail.com>. All rights reserved.
  * Copyright (c) 2023 Hewlett Packard Enterprise Development LP.
+ * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
  */
 
 #include <sys/dmu.h>
 #include <sys/dmu_objset.h>
 #include <sys/dmu_tx.h>
 #include <sys/dsl_dataset.h>
 #include <sys/dsl_dir.h>
 #include <sys/dsl_prop.h>
 #include <sys/dsl_synctask.h>
 #include <sys/dsl_deleg.h>
 #include <sys/dmu_impl.h>
 #include <sys/spa.h>
 #include <sys/spa_impl.h>
 #include <sys/metaslab.h>
 #include <sys/zap.h>
 #include <sys/zio.h>
 #include <sys/arc.h>
 #include <sys/sunddi.h>
 #include <sys/zfeature.h>
 #include <sys/policy.h>
 #include <sys/zfs_vfsops.h>
 #include <sys/zfs_znode.h>
 #include <sys/zvol.h>
 #include <sys/zthr.h>
 #include "zfs_namecheck.h"
 #include "zfs_prop.h"
 
 /*
  * This controls if we verify the ZVOL quota or not.
  * Currently, quotas are not implemented for ZVOLs.
  * The quota size is the size of the ZVOL.
  * The size of the volume already implies the ZVOL size quota.
  * The quota mechanism can introduce a significant performance drop.
  */
 static int zvol_enforce_quotas = B_TRUE;
 
 /*
  * Filesystem and Snapshot Limits
  * ------------------------------
  *
  * These limits are used to restrict the number of filesystems and/or snapshots
  * that can be created at a given level in the tree or below. A typical
  * use-case is with a delegated dataset where the administrator wants to ensure
  * that a user within the zone is not creating too many additional filesystems
  * or snapshots, even though they're not exceeding their space quota.
  *
  * The filesystem and snapshot counts are stored as extensible properties. This
  * capability is controlled by a feature flag and must be enabled to be used.
  * Once enabled, the feature is not active until the first limit is set. At
  * that point, future operations to create/destroy filesystems or snapshots
  * will validate and update the counts.
  *
  * Because the count properties will not exist before the feature is active,
  * the counts are updated when a limit is first set on an uninitialized
  * dsl_dir node in the tree (The filesystem/snapshot count on a node includes
  * all of the nested filesystems/snapshots. Thus, a new leaf node has a
  * filesystem count of 0 and a snapshot count of 0. Non-existent filesystem and
  * snapshot count properties on a node indicate uninitialized counts on that
  * node.) When first setting a limit on an uninitialized node, the code starts
  * at the filesystem with the new limit and descends into all sub-filesystems
  * to add the count properties.
  *
  * In practice this is lightweight since a limit is typically set when the
  * filesystem is created and thus has no children. Once valid, changing the
  * limit value won't require a re-traversal since the counts are already valid.
  * When recursively fixing the counts, if a node with a limit is encountered
  * during the descent, the counts are known to be valid and there is no need to
  * descend into that filesystem's children. The counts on filesystems above the
  * one with the new limit will still be uninitialized, unless a limit is
  * eventually set on one of those filesystems. The counts are always recursively
  * updated when a limit is set on a dataset, unless there is already a limit.
  * When a new limit value is set on a filesystem with an existing limit, it is
  * possible for the new limit to be less than the current count at that level
  * since a user who can change the limit is also allowed to exceed the limit.
  *
  * Once the feature is active, then whenever a filesystem or snapshot is
  * created, the code recurses up the tree, validating the new count against the
  * limit at each initialized level. In practice, most levels will not have a
  * limit set. If there is a limit at any initialized level up the tree, the
  * check must pass or the creation will fail. Likewise, when a filesystem or
  * snapshot is destroyed, the counts are recursively adjusted all the way up
  * the initialized nodes in the tree. Renaming a filesystem into different point
  * in the tree will first validate, then update the counts on each branch up to
  * the common ancestor. A receive will also validate the counts and then update
  * them.
  *
  * An exception to the above behavior is that the limit is not enforced if the
  * user has permission to modify the limit. This is primarily so that
  * recursive snapshots in the global zone always work. We want to prevent a
  * denial-of-service in which a lower level delegated dataset could max out its
  * limit and thus block recursive snapshots from being taken in the global zone.
  * Because of this, it is possible for the snapshot count to be over the limit
  * and snapshots taken in the global zone could cause a lower level dataset to
  * hit or exceed its limit. The administrator taking the global zone recursive
  * snapshot should be aware of this side-effect and behave accordingly.
  * For consistency, the filesystem limit is also not enforced if the user can
  * modify the limit.
  *
  * The filesystem and snapshot limits are validated by dsl_fs_ss_limit_check()
  * and updated by dsl_fs_ss_count_adjust(). A new limit value is setup in
  * dsl_dir_activate_fs_ss_limit() and the counts are adjusted, if necessary, by
  * dsl_dir_init_fs_ss_count().
  */
 
 static uint64_t dsl_dir_space_towrite(dsl_dir_t *dd);
 
 typedef struct ddulrt_arg {
 	dsl_dir_t	*ddulrta_dd;
 	uint64_t	ddlrta_txg;
 } ddulrt_arg_t;
 
 static void
 dsl_dir_evict_async(void *dbu)
 {
 	dsl_dir_t *dd = dbu;
 	int t;
 	dsl_pool_t *dp __maybe_unused = dd->dd_pool;
 
 	dd->dd_dbuf = NULL;
 
 	for (t = 0; t < TXG_SIZE; t++) {
 		ASSERT(!txg_list_member(&dp->dp_dirty_dirs, dd, t));
 		ASSERT(dd->dd_tempreserved[t] == 0);
 		ASSERT(dd->dd_space_towrite[t] == 0);
 	}
 
 	if (dd->dd_parent)
 		dsl_dir_async_rele(dd->dd_parent, dd);
 
 	spa_async_close(dd->dd_pool->dp_spa, dd);
 
 	if (dsl_deadlist_is_open(&dd->dd_livelist))
 		dsl_dir_livelist_close(dd);
 
 	dsl_prop_fini(dd);
 	cv_destroy(&dd->dd_activity_cv);
 	mutex_destroy(&dd->dd_activity_lock);
 	mutex_destroy(&dd->dd_lock);
 	kmem_free(dd, sizeof (dsl_dir_t));
 }
 
 int
 dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
     const char *tail, const void *tag, dsl_dir_t **ddp)
 {
 	dmu_buf_t *dbuf;
 	dsl_dir_t *dd;
 	dmu_object_info_t doi;
 	int err;
 
 	ASSERT(dsl_pool_config_held(dp));
 
 	err = dmu_bonus_hold(dp->dp_meta_objset, ddobj, tag, &dbuf);
 	if (err != 0)
 		return (err);
 	dd = dmu_buf_get_user(dbuf);
 
 	dmu_object_info_from_db(dbuf, &doi);
 	ASSERT3U(doi.doi_bonus_type, ==, DMU_OT_DSL_DIR);
 	ASSERT3U(doi.doi_bonus_size, >=, sizeof (dsl_dir_phys_t));
 
 	if (dd == NULL) {
 		dsl_dir_t *winner;
 
 		dd = kmem_zalloc(sizeof (dsl_dir_t), KM_SLEEP);
 		dd->dd_object = ddobj;
 		dd->dd_dbuf = dbuf;
 		dd->dd_pool = dp;
 
 		mutex_init(&dd->dd_lock, NULL, MUTEX_DEFAULT, NULL);
 		mutex_init(&dd->dd_activity_lock, NULL, MUTEX_DEFAULT, NULL);
 		cv_init(&dd->dd_activity_cv, NULL, CV_DEFAULT, NULL);
 		dsl_prop_init(dd);
 
 		if (dsl_dir_is_zapified(dd)) {
 			err = zap_lookup(dp->dp_meta_objset,
 			    ddobj, DD_FIELD_CRYPTO_KEY_OBJ,
 			    sizeof (uint64_t), 1, &dd->dd_crypto_obj);
 			if (err == 0) {
 				/* check for on-disk format errata */
 				if (dsl_dir_incompatible_encryption_version(
 				    dd)) {
 					dp->dp_spa->spa_errata =
 					    ZPOOL_ERRATA_ZOL_6845_ENCRYPTION;
 				}
 			} else if (err != ENOENT) {
 				goto errout;
 			}
 		}
 
 		if (dsl_dir_phys(dd)->dd_parent_obj) {
 			err = dsl_dir_hold_obj(dp,
 			    dsl_dir_phys(dd)->dd_parent_obj, NULL, dd,
 			    &dd->dd_parent);
 			if (err != 0)
 				goto errout;
 			if (tail) {
 #ifdef ZFS_DEBUG
 				uint64_t foundobj;
 
 				err = zap_lookup(dp->dp_meta_objset,
 				    dsl_dir_phys(dd->dd_parent)->
 				    dd_child_dir_zapobj, tail,
 				    sizeof (foundobj), 1, &foundobj);
 				ASSERT(err || foundobj == ddobj);
 #endif
 				(void) strlcpy(dd->dd_myname, tail,
 				    sizeof (dd->dd_myname));
 			} else {
 				err = zap_value_search(dp->dp_meta_objset,
 				    dsl_dir_phys(dd->dd_parent)->
 				    dd_child_dir_zapobj,
 				    ddobj, 0, dd->dd_myname);
 			}
 			if (err != 0)
 				goto errout;
 		} else {
 			(void) strlcpy(dd->dd_myname, spa_name(dp->dp_spa),
 			    sizeof (dd->dd_myname));
 		}
 
 		if (dsl_dir_is_clone(dd)) {
 			dmu_buf_t *origin_bonus;
 			dsl_dataset_phys_t *origin_phys;
 
 			/*
 			 * We can't open the origin dataset, because
 			 * that would require opening this dsl_dir.
 			 * Just look at its phys directly instead.
 			 */
 			err = dmu_bonus_hold(dp->dp_meta_objset,
 			    dsl_dir_phys(dd)->dd_origin_obj, FTAG,
 			    &origin_bonus);
 			if (err != 0)
 				goto errout;
 			origin_phys = origin_bonus->db_data;
 			dd->dd_origin_txg =
 			    origin_phys->ds_creation_txg;
 			dmu_buf_rele(origin_bonus, FTAG);
 			if (dsl_dir_is_zapified(dd)) {
 				uint64_t obj;
 				err = zap_lookup(dp->dp_meta_objset,
 				    dd->dd_object, DD_FIELD_LIVELIST,
 				    sizeof (uint64_t), 1, &obj);
 				if (err == 0)
 					dsl_dir_livelist_open(dd, obj);
 				else if (err != ENOENT)
 					goto errout;
 			}
 		}
 
 		if (dsl_dir_is_zapified(dd)) {
 			inode_timespec_t t = {0};
 			(void) zap_lookup(dp->dp_meta_objset, ddobj,
 			    DD_FIELD_SNAPSHOTS_CHANGED,
 			    sizeof (uint64_t),
 			    sizeof (inode_timespec_t) / sizeof (uint64_t),
 			    &t);
 			dd->dd_snap_cmtime = t;
 		}
 
 		dmu_buf_init_user(&dd->dd_dbu, NULL, dsl_dir_evict_async,
 		    &dd->dd_dbuf);
 		winner = dmu_buf_set_user_ie(dbuf, &dd->dd_dbu);
 		if (winner != NULL) {
 			if (dd->dd_parent)
 				dsl_dir_rele(dd->dd_parent, dd);
 			if (dsl_deadlist_is_open(&dd->dd_livelist))
 				dsl_dir_livelist_close(dd);
 			dsl_prop_fini(dd);
 			cv_destroy(&dd->dd_activity_cv);
 			mutex_destroy(&dd->dd_activity_lock);
 			mutex_destroy(&dd->dd_lock);
 			kmem_free(dd, sizeof (dsl_dir_t));
 			dd = winner;
 		} else {
 			spa_open_ref(dp->dp_spa, dd);
 		}
 	}
 
 	/*
 	 * The dsl_dir_t has both open-to-close and instantiate-to-evict
 	 * holds on the spa.  We need the open-to-close holds because
 	 * otherwise the spa_refcnt wouldn't change when we open a
 	 * dir which the spa also has open, so we could incorrectly
 	 * think it was OK to unload/export/destroy the pool.  We need
 	 * the instantiate-to-evict hold because the dsl_dir_t has a
 	 * pointer to the dd_pool, which has a pointer to the spa_t.
 	 */
 	spa_open_ref(dp->dp_spa, tag);
 	ASSERT3P(dd->dd_pool, ==, dp);
 	ASSERT3U(dd->dd_object, ==, ddobj);
 	ASSERT3P(dd->dd_dbuf, ==, dbuf);
 	*ddp = dd;
 	return (0);
 
 errout:
 	if (dd->dd_parent)
 		dsl_dir_rele(dd->dd_parent, dd);
 	if (dsl_deadlist_is_open(&dd->dd_livelist))
 		dsl_dir_livelist_close(dd);
 	dsl_prop_fini(dd);
 	cv_destroy(&dd->dd_activity_cv);
 	mutex_destroy(&dd->dd_activity_lock);
 	mutex_destroy(&dd->dd_lock);
 	kmem_free(dd, sizeof (dsl_dir_t));
 	dmu_buf_rele(dbuf, tag);
 	return (err);
 }
 
 void
 dsl_dir_rele(dsl_dir_t *dd, const void *tag)
 {
 	dprintf_dd(dd, "%s\n", "");
 	spa_close(dd->dd_pool->dp_spa, tag);
 	dmu_buf_rele(dd->dd_dbuf, tag);
 }
 
 /*
  * Remove a reference to the given dsl dir that is being asynchronously
  * released.  Async releases occur from a taskq performing eviction of
  * dsl datasets and dirs.  This process is identical to a normal release
  * with the exception of using the async API for releasing the reference on
  * the spa.
  */
 void
 dsl_dir_async_rele(dsl_dir_t *dd, const void *tag)
 {
 	dprintf_dd(dd, "%s\n", "");
 	spa_async_close(dd->dd_pool->dp_spa, tag);
 	dmu_buf_rele(dd->dd_dbuf, tag);
 }
 
 /* buf must be at least ZFS_MAX_DATASET_NAME_LEN bytes */
 void
 dsl_dir_name(dsl_dir_t *dd, char *buf)
 {
 	if (dd->dd_parent) {
 		dsl_dir_name(dd->dd_parent, buf);
 		VERIFY3U(strlcat(buf, "/", ZFS_MAX_DATASET_NAME_LEN), <,
 		    ZFS_MAX_DATASET_NAME_LEN);
 	} else {
 		buf[0] = '\0';
 	}
 	if (!MUTEX_HELD(&dd->dd_lock)) {
 		/*
 		 * recursive mutex so that we can use
 		 * dprintf_dd() with dd_lock held
 		 */
 		mutex_enter(&dd->dd_lock);
 		VERIFY3U(strlcat(buf, dd->dd_myname, ZFS_MAX_DATASET_NAME_LEN),
 		    <, ZFS_MAX_DATASET_NAME_LEN);
 		mutex_exit(&dd->dd_lock);
 	} else {
 		VERIFY3U(strlcat(buf, dd->dd_myname, ZFS_MAX_DATASET_NAME_LEN),
 		    <, ZFS_MAX_DATASET_NAME_LEN);
 	}
 }
 
 /* Calculate name length, avoiding all the strcat calls of dsl_dir_name */
 int
 dsl_dir_namelen(dsl_dir_t *dd)
 {
 	int result = 0;
 
 	if (dd->dd_parent) {
 		/* parent's name + 1 for the "/" */
 		result = dsl_dir_namelen(dd->dd_parent) + 1;
 	}
 
 	if (!MUTEX_HELD(&dd->dd_lock)) {
 		/* see dsl_dir_name */
 		mutex_enter(&dd->dd_lock);
 		result += strlen(dd->dd_myname);
 		mutex_exit(&dd->dd_lock);
 	} else {
 		result += strlen(dd->dd_myname);
 	}
 
 	return (result);
 }
 
 static int
 getcomponent(const char *path, char *component, const char **nextp)
 {
 	char *p;
 
 	if ((path == NULL) || (path[0] == '\0'))
 		return (SET_ERROR(ENOENT));
 	/* This would be a good place to reserve some namespace... */
 	p = strpbrk(path, "/@");
 	if (p && (p[1] == '/' || p[1] == '@')) {
 		/* two separators in a row */
 		return (SET_ERROR(EINVAL));
 	}
 	if (p == NULL || p == path) {
 		/*
 		 * if the first thing is an @ or /, it had better be an
 		 * @ and it had better not have any more ats or slashes,
 		 * and it had better have something after the @.
 		 */
 		if (p != NULL &&
 		    (p[0] != '@' || strpbrk(path+1, "/@") || p[1] == '\0'))
 			return (SET_ERROR(EINVAL));
 		if (strlen(path) >= ZFS_MAX_DATASET_NAME_LEN)
 			return (SET_ERROR(ENAMETOOLONG));
 		(void) strlcpy(component, path, ZFS_MAX_DATASET_NAME_LEN);
 		p = NULL;
 	} else if (p[0] == '/') {
 		if (p - path >= ZFS_MAX_DATASET_NAME_LEN)
 			return (SET_ERROR(ENAMETOOLONG));
 		(void) strlcpy(component, path, p - path + 1);
 		p++;
 	} else if (p[0] == '@') {
 		/*
 		 * if the next separator is an @, there better not be
 		 * any more slashes.
 		 */
 		if (strchr(path, '/'))
 			return (SET_ERROR(EINVAL));
 		if (p - path >= ZFS_MAX_DATASET_NAME_LEN)
 			return (SET_ERROR(ENAMETOOLONG));
 		(void) strlcpy(component, path, p - path + 1);
 	} else {
 		panic("invalid p=%p", (void *)p);
 	}
 	*nextp = p;
 	return (0);
 }
 
 /*
  * Return the dsl_dir_t, and possibly the last component which couldn't
  * be found in *tail.  The name must be in the specified dsl_pool_t.  This
  * thread must hold the dp_config_rwlock for the pool.  Returns NULL if the
  * path is bogus, or if tail==NULL and we couldn't parse the whole name.
  * (*tail)[0] == '@' means that the last component is a snapshot.
  */
 int
 dsl_dir_hold(dsl_pool_t *dp, const char *name, const void *tag,
     dsl_dir_t **ddp, const char **tailp)
 {
 	char *buf;
 	const char *spaname, *next, *nextnext = NULL;
 	int err;
 	dsl_dir_t *dd;
 	uint64_t ddobj;
 
 	buf = kmem_alloc(ZFS_MAX_DATASET_NAME_LEN, KM_SLEEP);
 	err = getcomponent(name, buf, &next);
 	if (err != 0)
 		goto error;
 
 	/* Make sure the name is in the specified pool. */
 	spaname = spa_name(dp->dp_spa);
 	if (strcmp(buf, spaname) != 0) {
 		err = SET_ERROR(EXDEV);
 		goto error;
 	}
 
 	ASSERT(dsl_pool_config_held(dp));
 
 	err = dsl_dir_hold_obj(dp, dp->dp_root_dir_obj, NULL, tag, &dd);
 	if (err != 0) {
 		goto error;
 	}
 
 	while (next != NULL) {
 		dsl_dir_t *child_dd;
 		err = getcomponent(next, buf, &nextnext);
 		if (err != 0)
 			break;
 		ASSERT(next[0] != '\0');
 		if (next[0] == '@')
 			break;
 		dprintf("looking up %s in obj%lld\n",
 		    buf, (longlong_t)dsl_dir_phys(dd)->dd_child_dir_zapobj);
 
 		err = zap_lookup(dp->dp_meta_objset,
 		    dsl_dir_phys(dd)->dd_child_dir_zapobj,
 		    buf, sizeof (ddobj), 1, &ddobj);
 		if (err != 0) {
 			if (err == ENOENT)
 				err = 0;
 			break;
 		}
 
 		err = dsl_dir_hold_obj(dp, ddobj, buf, tag, &child_dd);
 		if (err != 0)
 			break;
 		dsl_dir_rele(dd, tag);
 		dd = child_dd;
 		next = nextnext;
 	}
 
 	if (err != 0) {
 		dsl_dir_rele(dd, tag);
 		goto error;
 	}
 
 	/*
 	 * It's an error if there's more than one component left, or
 	 * tailp==NULL and there's any component left.
 	 */
 	if (next != NULL &&
 	    (tailp == NULL || (nextnext && nextnext[0] != '\0'))) {
 		/* bad path name */
 		dsl_dir_rele(dd, tag);
 		dprintf("next=%p (%s) tail=%p\n", next, next?next:"", tailp);
 		err = SET_ERROR(ENOENT);
 	}
 	if (tailp != NULL)
 		*tailp = next;
 	if (err == 0)
 		*ddp = dd;
 error:
 	kmem_free(buf, ZFS_MAX_DATASET_NAME_LEN);
 	return (err);
 }
 
 /*
  * If the counts are already initialized for this filesystem and its
  * descendants then do nothing, otherwise initialize the counts.
  *
  * The counts on this filesystem, and those below, may be uninitialized due to
  * either the use of a pre-existing pool which did not support the
  * filesystem/snapshot limit feature, or one in which the feature had not yet
  * been enabled.
  *
  * Recursively descend the filesystem tree and update the filesystem/snapshot
  * counts on each filesystem below, then update the cumulative count on the
  * current filesystem. If the filesystem already has a count set on it,
  * then we know that its counts, and the counts on the filesystems below it,
  * are already correct, so we don't have to update this filesystem.
  */
 static void
 dsl_dir_init_fs_ss_count(dsl_dir_t *dd, dmu_tx_t *tx)
 {
 	uint64_t my_fs_cnt = 0;
 	uint64_t my_ss_cnt = 0;
 	dsl_pool_t *dp = dd->dd_pool;
 	objset_t *os = dp->dp_meta_objset;
 	zap_cursor_t *zc;
 	zap_attribute_t *za;
 	dsl_dataset_t *ds;
 
 	ASSERT(spa_feature_is_active(dp->dp_spa, SPA_FEATURE_FS_SS_LIMIT));
 	ASSERT(dsl_pool_config_held(dp));
 	ASSERT(dmu_tx_is_syncing(tx));
 
 	dsl_dir_zapify(dd, tx);
 
 	/*
 	 * If the filesystem count has already been initialized then we
 	 * don't need to recurse down any further.
 	 */
 	if (zap_contains(os, dd->dd_object, DD_FIELD_FILESYSTEM_COUNT) == 0)
 		return;
 
 	zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP);
 	za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
 
 	/* Iterate my child dirs */
 	for (zap_cursor_init(zc, os, dsl_dir_phys(dd)->dd_child_dir_zapobj);
 	    zap_cursor_retrieve(zc, za) == 0; zap_cursor_advance(zc)) {
 		dsl_dir_t *chld_dd;
 		uint64_t count;
 
 		VERIFY0(dsl_dir_hold_obj(dp, za->za_first_integer, NULL, FTAG,
 		    &chld_dd));
 
 		/*
 		 * Ignore hidden ($FREE, $MOS & $ORIGIN) objsets.
 		 */
 		if (chld_dd->dd_myname[0] == '$') {
 			dsl_dir_rele(chld_dd, FTAG);
 			continue;
 		}
 
 		my_fs_cnt++;	/* count this child */
 
 		dsl_dir_init_fs_ss_count(chld_dd, tx);
 
 		VERIFY0(zap_lookup(os, chld_dd->dd_object,
 		    DD_FIELD_FILESYSTEM_COUNT, sizeof (count), 1, &count));
 		my_fs_cnt += count;
 		VERIFY0(zap_lookup(os, chld_dd->dd_object,
 		    DD_FIELD_SNAPSHOT_COUNT, sizeof (count), 1, &count));
 		my_ss_cnt += count;
 
 		dsl_dir_rele(chld_dd, FTAG);
 	}
 	zap_cursor_fini(zc);
 	/* Count my snapshots (we counted children's snapshots above) */
 	VERIFY0(dsl_dataset_hold_obj(dd->dd_pool,
 	    dsl_dir_phys(dd)->dd_head_dataset_obj, FTAG, &ds));
 
 	for (zap_cursor_init(zc, os, dsl_dataset_phys(ds)->ds_snapnames_zapobj);
 	    zap_cursor_retrieve(zc, za) == 0;
 	    zap_cursor_advance(zc)) {
 		/* Don't count temporary snapshots */
 		if (za->za_name[0] != '%')
 			my_ss_cnt++;
 	}
 	zap_cursor_fini(zc);
 
 	dsl_dataset_rele(ds, FTAG);
 
 	kmem_free(zc, sizeof (zap_cursor_t));
 	kmem_free(za, sizeof (zap_attribute_t));
 
 	/* we're in a sync task, update counts */
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 	VERIFY0(zap_add(os, dd->dd_object, DD_FIELD_FILESYSTEM_COUNT,
 	    sizeof (my_fs_cnt), 1, &my_fs_cnt, tx));
 	VERIFY0(zap_add(os, dd->dd_object, DD_FIELD_SNAPSHOT_COUNT,
 	    sizeof (my_ss_cnt), 1, &my_ss_cnt, tx));
 }
 
 static int
 dsl_dir_actv_fs_ss_limit_check(void *arg, dmu_tx_t *tx)
 {
 	char *ddname = (char *)arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds;
 	dsl_dir_t *dd;
 	int error;
 
 	error = dsl_dataset_hold(dp, ddname, FTAG, &ds);
 	if (error != 0)
 		return (error);
 
 	if (!spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_FS_SS_LIMIT)) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(ENOTSUP));
 	}
 
 	dd = ds->ds_dir;
 	if (spa_feature_is_active(dp->dp_spa, SPA_FEATURE_FS_SS_LIMIT) &&
 	    dsl_dir_is_zapified(dd) &&
 	    zap_contains(dp->dp_meta_objset, dd->dd_object,
 	    DD_FIELD_FILESYSTEM_COUNT) == 0) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EALREADY));
 	}
 
 	dsl_dataset_rele(ds, FTAG);
 	return (0);
 }
 
 static void
 dsl_dir_actv_fs_ss_limit_sync(void *arg, dmu_tx_t *tx)
 {
 	char *ddname = (char *)arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds;
 	spa_t *spa;
 
 	VERIFY0(dsl_dataset_hold(dp, ddname, FTAG, &ds));
 
 	spa = dsl_dataset_get_spa(ds);
 
 	if (!spa_feature_is_active(spa, SPA_FEATURE_FS_SS_LIMIT)) {
 		/*
 		 * Since the feature was not active and we're now setting a
 		 * limit, increment the feature-active counter so that the
 		 * feature becomes active for the first time.
 		 *
 		 * We are already in a sync task so we can update the MOS.
 		 */
 		spa_feature_incr(spa, SPA_FEATURE_FS_SS_LIMIT, tx);
 	}
 
 	/*
 	 * Since we are now setting a non-UINT64_MAX limit on the filesystem,
 	 * we need to ensure the counts are correct. Descend down the tree from
 	 * this point and update all of the counts to be accurate.
 	 */
 	dsl_dir_init_fs_ss_count(ds->ds_dir, tx);
 
 	dsl_dataset_rele(ds, FTAG);
 }
 
 /*
  * Make sure the feature is enabled and activate it if necessary.
  * Since we're setting a limit, ensure the on-disk counts are valid.
  * This is only called by the ioctl path when setting a limit value.
  *
  * We do not need to validate the new limit, since users who can change the
  * limit are also allowed to exceed the limit.
  */
 int
 dsl_dir_activate_fs_ss_limit(const char *ddname)
 {
 	int error;
 
 	error = dsl_sync_task(ddname, dsl_dir_actv_fs_ss_limit_check,
 	    dsl_dir_actv_fs_ss_limit_sync, (void *)ddname, 0,
 	    ZFS_SPACE_CHECK_RESERVED);
 
 	if (error == EALREADY)
 		error = 0;
 
 	return (error);
 }
 
 /*
  * Used to determine if the filesystem_limit or snapshot_limit should be
  * enforced. We allow the limit to be exceeded if the user has permission to
  * write the property value. We pass in the creds that we got in the open
  * context since we will always be the GZ root in syncing context. We also have
  * to handle the case where we are allowed to change the limit on the current
  * dataset, but there may be another limit in the tree above.
  *
  * We can never modify these two properties within a non-global zone. In
  * addition, the other checks are modeled on zfs_secpolicy_write_perms. We
  * can't use that function since we are already holding the dp_config_rwlock.
  * In addition, we already have the dd and dealing with snapshots is simplified
  * in this code.
  */
 
 typedef enum {
 	ENFORCE_ALWAYS,
 	ENFORCE_NEVER,
 	ENFORCE_ABOVE
 } enforce_res_t;
 
 static enforce_res_t
 dsl_enforce_ds_ss_limits(dsl_dir_t *dd, zfs_prop_t prop,
-    cred_t *cr, proc_t *proc)
+    cred_t *cr)
 {
 	enforce_res_t enforce = ENFORCE_ALWAYS;
 	uint64_t obj;
 	dsl_dataset_t *ds;
 	uint64_t zoned;
 	const char *zonedstr;
 
 	ASSERT(prop == ZFS_PROP_FILESYSTEM_LIMIT ||
 	    prop == ZFS_PROP_SNAPSHOT_LIMIT);
 
 #ifdef _KERNEL
 	if (crgetzoneid(cr) != GLOBAL_ZONEID)
 		return (ENFORCE_ALWAYS);
 
-	/*
-	 * We are checking the saved credentials of the user process, which is
-	 * not the current process.  Note that we can't use secpolicy_zfs(),
-	 * because it only works if the cred is that of the current process (on
-	 * Linux).
-	 */
-	if (secpolicy_zfs_proc(cr, proc) == 0)
+	if (secpolicy_zfs(cr) == 0)
 		return (ENFORCE_NEVER);
-#else
-	(void) proc;
 #endif
 
 	if ((obj = dsl_dir_phys(dd)->dd_head_dataset_obj) == 0)
 		return (ENFORCE_ALWAYS);
 
 	ASSERT(dsl_pool_config_held(dd->dd_pool));
 
 	if (dsl_dataset_hold_obj(dd->dd_pool, obj, FTAG, &ds) != 0)
 		return (ENFORCE_ALWAYS);
 
 	zonedstr = zfs_prop_to_name(ZFS_PROP_ZONED);
 	if (dsl_prop_get_ds(ds, zonedstr, 8, 1, &zoned, NULL) || zoned) {
 		/* Only root can access zoned fs's from the GZ */
 		enforce = ENFORCE_ALWAYS;
 	} else {
 		if (dsl_deleg_access_impl(ds, zfs_prop_to_name(prop), cr) == 0)
 			enforce = ENFORCE_ABOVE;
 	}
 
 	dsl_dataset_rele(ds, FTAG);
 	return (enforce);
 }
 
 /*
  * Check if adding additional child filesystem(s) would exceed any filesystem
  * limits or adding additional snapshot(s) would exceed any snapshot limits.
  * The prop argument indicates which limit to check.
  *
  * Note that all filesystem limits up to the root (or the highest
  * initialized) filesystem or the given ancestor must be satisfied.
  */
 int
 dsl_fs_ss_limit_check(dsl_dir_t *dd, uint64_t delta, zfs_prop_t prop,
-    dsl_dir_t *ancestor, cred_t *cr, proc_t *proc)
+    dsl_dir_t *ancestor, cred_t *cr)
 {
 	objset_t *os = dd->dd_pool->dp_meta_objset;
 	uint64_t limit, count;
 	const char *count_prop;
 	enforce_res_t enforce;
 	int err = 0;
 
 	ASSERT(dsl_pool_config_held(dd->dd_pool));
 	ASSERT(prop == ZFS_PROP_FILESYSTEM_LIMIT ||
 	    prop == ZFS_PROP_SNAPSHOT_LIMIT);
 
 	if (prop == ZFS_PROP_SNAPSHOT_LIMIT) {
 		/*
 		 * We don't enforce the limit for temporary snapshots. This is
 		 * indicated by a NULL cred_t argument.
 		 */
 		if (cr == NULL)
 			return (0);
 
 		count_prop = DD_FIELD_SNAPSHOT_COUNT;
 	} else {
 		count_prop = DD_FIELD_FILESYSTEM_COUNT;
 	}
 	/*
 	 * If we're allowed to change the limit, don't enforce the limit
 	 * e.g. this can happen if a snapshot is taken by an administrative
 	 * user in the global zone (i.e. a recursive snapshot by root).
 	 * However, we must handle the case of delegated permissions where we
 	 * are allowed to change the limit on the current dataset, but there
 	 * is another limit in the tree above.
 	 */
-	enforce = dsl_enforce_ds_ss_limits(dd, prop, cr, proc);
+	enforce = dsl_enforce_ds_ss_limits(dd, prop, cr);
 	if (enforce == ENFORCE_NEVER)
 		return (0);
 
 	/*
 	 * e.g. if renaming a dataset with no snapshots, count adjustment
 	 * is 0.
 	 */
 	if (delta == 0)
 		return (0);
 
 	/*
 	 * If an ancestor has been provided, stop checking the limit once we
 	 * hit that dir. We need this during rename so that we don't overcount
 	 * the check once we recurse up to the common ancestor.
 	 */
 	if (ancestor == dd)
 		return (0);
 
 	/*
 	 * If we hit an uninitialized node while recursing up the tree, we can
 	 * stop since we know there is no limit here (or above). The counts are
 	 * not valid on this node and we know we won't touch this node's counts.
 	 */
 	if (!dsl_dir_is_zapified(dd))
 		return (0);
 	err = zap_lookup(os, dd->dd_object,
 	    count_prop, sizeof (count), 1, &count);
 	if (err == ENOENT)
 		return (0);
 	if (err != 0)
 		return (err);
 
 	err = dsl_prop_get_dd(dd, zfs_prop_to_name(prop), 8, 1, &limit, NULL,
 	    B_FALSE);
 	if (err != 0)
 		return (err);
 
 	/* Is there a limit which we've hit? */
 	if (enforce == ENFORCE_ALWAYS && (count + delta) > limit)
 		return (SET_ERROR(EDQUOT));
 
 	if (dd->dd_parent != NULL)
 		err = dsl_fs_ss_limit_check(dd->dd_parent, delta, prop,
-		    ancestor, cr, proc);
+		    ancestor, cr);
 
 	return (err);
 }
 
 /*
  * Adjust the filesystem or snapshot count for the specified dsl_dir_t and all
  * parents. When a new filesystem/snapshot is created, increment the count on
  * all parents, and when a filesystem/snapshot is destroyed, decrement the
  * count.
  */
 void
 dsl_fs_ss_count_adjust(dsl_dir_t *dd, int64_t delta, const char *prop,
     dmu_tx_t *tx)
 {
 	int err;
 	objset_t *os = dd->dd_pool->dp_meta_objset;
 	uint64_t count;
 
 	ASSERT(dsl_pool_config_held(dd->dd_pool));
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(strcmp(prop, DD_FIELD_FILESYSTEM_COUNT) == 0 ||
 	    strcmp(prop, DD_FIELD_SNAPSHOT_COUNT) == 0);
 
 	/*
 	 * We don't do accounting for hidden ($FREE, $MOS & $ORIGIN) objsets.
 	 */
 	if (dd->dd_myname[0] == '$' && strcmp(prop,
 	    DD_FIELD_FILESYSTEM_COUNT) == 0) {
 		return;
 	}
 
 	/*
 	 * e.g. if renaming a dataset with no snapshots, count adjustment is 0
 	 */
 	if (delta == 0)
 		return;
 
 	/*
 	 * If we hit an uninitialized node while recursing up the tree, we can
 	 * stop since we know the counts are not valid on this node and we
 	 * know we shouldn't touch this node's counts. An uninitialized count
 	 * on the node indicates that either the feature has not yet been
 	 * activated or there are no limits on this part of the tree.
 	 */
 	if (!dsl_dir_is_zapified(dd) || (err = zap_lookup(os, dd->dd_object,
 	    prop, sizeof (count), 1, &count)) == ENOENT)
 		return;
 	VERIFY0(err);
 
 	count += delta;
 	/* Use a signed verify to make sure we're not neg. */
 	VERIFY3S(count, >=, 0);
 
 	VERIFY0(zap_update(os, dd->dd_object, prop, sizeof (count), 1, &count,
 	    tx));
 
 	/* Roll up this additional count into our ancestors */
 	if (dd->dd_parent != NULL)
 		dsl_fs_ss_count_adjust(dd->dd_parent, delta, prop, tx);
 }
 
 uint64_t
 dsl_dir_create_sync(dsl_pool_t *dp, dsl_dir_t *pds, const char *name,
     dmu_tx_t *tx)
 {
 	objset_t *mos = dp->dp_meta_objset;
 	uint64_t ddobj;
 	dsl_dir_phys_t *ddphys;
 	dmu_buf_t *dbuf;
 
 	ddobj = dmu_object_alloc(mos, DMU_OT_DSL_DIR, 0,
 	    DMU_OT_DSL_DIR, sizeof (dsl_dir_phys_t), tx);
 	if (pds) {
 		VERIFY0(zap_add(mos, dsl_dir_phys(pds)->dd_child_dir_zapobj,
 		    name, sizeof (uint64_t), 1, &ddobj, tx));
 	} else {
 		/* it's the root dir */
 		VERIFY0(zap_add(mos, DMU_POOL_DIRECTORY_OBJECT,
 		    DMU_POOL_ROOT_DATASET, sizeof (uint64_t), 1, &ddobj, tx));
 	}
 	VERIFY0(dmu_bonus_hold(mos, ddobj, FTAG, &dbuf));
 	dmu_buf_will_dirty(dbuf, tx);
 	ddphys = dbuf->db_data;
 
 	ddphys->dd_creation_time = gethrestime_sec();
 	if (pds) {
 		ddphys->dd_parent_obj = pds->dd_object;
 
 		/* update the filesystem counts */
 		dsl_fs_ss_count_adjust(pds, 1, DD_FIELD_FILESYSTEM_COUNT, tx);
 	}
 	ddphys->dd_props_zapobj = zap_create(mos,
 	    DMU_OT_DSL_PROPS, DMU_OT_NONE, 0, tx);
 	ddphys->dd_child_dir_zapobj = zap_create(mos,
 	    DMU_OT_DSL_DIR_CHILD_MAP, DMU_OT_NONE, 0, tx);
 	if (spa_version(dp->dp_spa) >= SPA_VERSION_USED_BREAKDOWN)
 		ddphys->dd_flags |= DD_FLAG_USED_BREAKDOWN;
 
 	dmu_buf_rele(dbuf, FTAG);
 
 	return (ddobj);
 }
 
 boolean_t
 dsl_dir_is_clone(dsl_dir_t *dd)
 {
 	return (dsl_dir_phys(dd)->dd_origin_obj &&
 	    (dd->dd_pool->dp_origin_snap == NULL ||
 	    dsl_dir_phys(dd)->dd_origin_obj !=
 	    dd->dd_pool->dp_origin_snap->ds_object));
 }
 
 uint64_t
 dsl_dir_get_used(dsl_dir_t *dd)
 {
 	return (dsl_dir_phys(dd)->dd_used_bytes);
 }
 
 uint64_t
 dsl_dir_get_compressed(dsl_dir_t *dd)
 {
 	return (dsl_dir_phys(dd)->dd_compressed_bytes);
 }
 
 uint64_t
 dsl_dir_get_quota(dsl_dir_t *dd)
 {
 	return (dsl_dir_phys(dd)->dd_quota);
 }
 
 uint64_t
 dsl_dir_get_reservation(dsl_dir_t *dd)
 {
 	return (dsl_dir_phys(dd)->dd_reserved);
 }
 
 uint64_t
 dsl_dir_get_compressratio(dsl_dir_t *dd)
 {
 	/* a fixed point number, 100x the ratio */
 	return (dsl_dir_phys(dd)->dd_compressed_bytes == 0 ? 100 :
 	    (dsl_dir_phys(dd)->dd_uncompressed_bytes * 100 /
 	    dsl_dir_phys(dd)->dd_compressed_bytes));
 }
 
 uint64_t
 dsl_dir_get_logicalused(dsl_dir_t *dd)
 {
 	return (dsl_dir_phys(dd)->dd_uncompressed_bytes);
 }
 
 uint64_t
 dsl_dir_get_usedsnap(dsl_dir_t *dd)
 {
 	return (dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_SNAP]);
 }
 
 uint64_t
 dsl_dir_get_usedds(dsl_dir_t *dd)
 {
 	return (dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_HEAD]);
 }
 
 uint64_t
 dsl_dir_get_usedrefreserv(dsl_dir_t *dd)
 {
 	return (dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_REFRSRV]);
 }
 
 uint64_t
 dsl_dir_get_usedchild(dsl_dir_t *dd)
 {
 	return (dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_CHILD] +
 	    dsl_dir_phys(dd)->dd_used_breakdown[DD_USED_CHILD_RSRV]);
 }
 
 void
 dsl_dir_get_origin(dsl_dir_t *dd, char *buf)
 {
 	dsl_dataset_t *ds;
 	VERIFY0(dsl_dataset_hold_obj(dd->dd_pool,
 	    dsl_dir_phys(dd)->dd_origin_obj, FTAG, &ds));
 
 	dsl_dataset_name(ds, buf);
 
 	dsl_dataset_rele(ds, FTAG);
 }
 
 int
 dsl_dir_get_filesystem_count(dsl_dir_t *dd, uint64_t *count)
 {
 	if (dsl_dir_is_zapified(dd)) {
 		objset_t *os = dd->dd_pool->dp_meta_objset;
 		return (zap_lookup(os, dd->dd_object, DD_FIELD_FILESYSTEM_COUNT,
 		    sizeof (*count), 1, count));
 	} else {
 		return (SET_ERROR(ENOENT));
 	}
 }
 
 int
 dsl_dir_get_snapshot_count(dsl_dir_t *dd, uint64_t *count)
 {
 	if (dsl_dir_is_zapified(dd)) {
 		objset_t *os = dd->dd_pool->dp_meta_objset;
 		return (zap_lookup(os, dd->dd_object, DD_FIELD_SNAPSHOT_COUNT,
 		    sizeof (*count), 1, count));
 	} else {
 		return (SET_ERROR(ENOENT));
 	}
 }
 
 void
 dsl_dir_stats(dsl_dir_t *dd, nvlist_t *nv)
 {
 	mutex_enter(&dd->dd_lock);
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_QUOTA,
 	    dsl_dir_get_quota(dd));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_RESERVATION,
 	    dsl_dir_get_reservation(dd));
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_LOGICALUSED,
 	    dsl_dir_get_logicalused(dd));
 	if (dsl_dir_phys(dd)->dd_flags & DD_FLAG_USED_BREAKDOWN) {
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USEDSNAP,
 		    dsl_dir_get_usedsnap(dd));
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USEDDS,
 		    dsl_dir_get_usedds(dd));
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USEDREFRESERV,
 		    dsl_dir_get_usedrefreserv(dd));
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USEDCHILD,
 		    dsl_dir_get_usedchild(dd));
 	}
 	mutex_exit(&dd->dd_lock);
 
 	uint64_t count;
 	if (dsl_dir_get_filesystem_count(dd, &count) == 0) {
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_FILESYSTEM_COUNT,
 		    count);
 	}
 	if (dsl_dir_get_snapshot_count(dd, &count) == 0) {
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_SNAPSHOT_COUNT,
 		    count);
 	}
 
 	if (dsl_dir_is_clone(dd)) {
 		char buf[ZFS_MAX_DATASET_NAME_LEN];
 		dsl_dir_get_origin(dd, buf);
 		dsl_prop_nvlist_add_string(nv, ZFS_PROP_ORIGIN, buf);
 	}
 
 }
 
 void
 dsl_dir_dirty(dsl_dir_t *dd, dmu_tx_t *tx)
 {
 	dsl_pool_t *dp = dd->dd_pool;
 
 	ASSERT(dsl_dir_phys(dd));
 
 	if (txg_list_add(&dp->dp_dirty_dirs, dd, tx->tx_txg)) {
 		/* up the hold count until we can be written out */
 		dmu_buf_add_ref(dd->dd_dbuf, dd);
 	}
 }
 
 static int64_t
 parent_delta(dsl_dir_t *dd, uint64_t used, int64_t delta)
 {
 	uint64_t old_accounted = MAX(used, dsl_dir_phys(dd)->dd_reserved);
 	uint64_t new_accounted =
 	    MAX(used + delta, dsl_dir_phys(dd)->dd_reserved);
 	return (new_accounted - old_accounted);
 }
 
 void
 dsl_dir_sync(dsl_dir_t *dd, dmu_tx_t *tx)
 {
 	ASSERT(dmu_tx_is_syncing(tx));
 
 	mutex_enter(&dd->dd_lock);
 	ASSERT0(dd->dd_tempreserved[tx->tx_txg & TXG_MASK]);
 	dprintf_dd(dd, "txg=%llu towrite=%lluK\n", (u_longlong_t)tx->tx_txg,
 	    (u_longlong_t)dd->dd_space_towrite[tx->tx_txg & TXG_MASK] / 1024);
 	dd->dd_space_towrite[tx->tx_txg & TXG_MASK] = 0;
 	mutex_exit(&dd->dd_lock);
 
 	/* release the hold from dsl_dir_dirty */
 	dmu_buf_rele(dd->dd_dbuf, dd);
 }
 
 static uint64_t
 dsl_dir_space_towrite(dsl_dir_t *dd)
 {
 	uint64_t space = 0;
 
 	ASSERT(MUTEX_HELD(&dd->dd_lock));
 
 	for (int i = 0; i < TXG_SIZE; i++)
 		space += dd->dd_space_towrite[i & TXG_MASK];
 
 	return (space);
 }
 
 /*
  * How much space would dd have available if ancestor had delta applied
  * to it?  If ondiskonly is set, we're only interested in what's
  * on-disk, not estimated pending changes.
  */
 uint64_t
 dsl_dir_space_available(dsl_dir_t *dd,
     dsl_dir_t *ancestor, int64_t delta, int ondiskonly)
 {
 	uint64_t parentspace, myspace, quota, used;
 
 	/*
 	 * If there are no restrictions otherwise, assume we have
 	 * unlimited space available.
 	 */
 	quota = UINT64_MAX;
 	parentspace = UINT64_MAX;
 
 	if (dd->dd_parent != NULL) {
 		parentspace = dsl_dir_space_available(dd->dd_parent,
 		    ancestor, delta, ondiskonly);
 	}
 
 	mutex_enter(&dd->dd_lock);
 	if (dsl_dir_phys(dd)->dd_quota != 0)
 		quota = dsl_dir_phys(dd)->dd_quota;
 	used = dsl_dir_phys(dd)->dd_used_bytes;
 	if (!ondiskonly)
 		used += dsl_dir_space_towrite(dd);
 
 	if (dd->dd_parent == NULL) {
 		uint64_t poolsize = dsl_pool_adjustedsize(dd->dd_pool,
 		    ZFS_SPACE_CHECK_NORMAL);
 		quota = MIN(quota, poolsize);
 	}
 
 	if (dsl_dir_phys(dd)->dd_reserved > used && parentspace != UINT64_MAX) {
 		/*
 		 * We have some space reserved, in addition to what our
 		 * parent gave us.
 		 */
 		parentspace += dsl_dir_phys(dd)->dd_reserved - used;
 	}
 
 	if (dd == ancestor) {
 		ASSERT(delta <= 0);
 		ASSERT(used >= -delta);
 		used += delta;
 		if (parentspace != UINT64_MAX)
 			parentspace -= delta;
 	}
 
 	if (used > quota) {
 		/* over quota */
 		myspace = 0;
 	} else {
 		/*
 		 * the lesser of the space provided by our parent and
 		 * the space left in our quota
 		 */
 		myspace = MIN(parentspace, quota - used);
 	}
 
 	mutex_exit(&dd->dd_lock);
 
 	return (myspace);
 }
 
 struct tempreserve {
 	list_node_t tr_node;
 	dsl_dir_t *tr_ds;
 	uint64_t tr_size;
 };
 
 static int
 dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree,
     boolean_t ignorequota, list_t *tr_list,
     dmu_tx_t *tx, boolean_t first)
 {
 	uint64_t txg;
 	uint64_t quota;
 	struct tempreserve *tr;
 	int retval;
 	uint64_t ext_quota;
 	uint64_t ref_rsrv;
 
 top_of_function:
 	txg = tx->tx_txg;
 	retval = EDQUOT;
 	ref_rsrv = 0;
 
 	ASSERT3U(txg, !=, 0);
 	ASSERT3S(asize, >, 0);
 
 	mutex_enter(&dd->dd_lock);
 
 	/*
 	 * Check against the dsl_dir's quota.  We don't add in the delta
 	 * when checking for over-quota because they get one free hit.
 	 */
 	uint64_t est_inflight = dsl_dir_space_towrite(dd);
 	for (int i = 0; i < TXG_SIZE; i++)
 		est_inflight += dd->dd_tempreserved[i];
 	uint64_t used_on_disk = dsl_dir_phys(dd)->dd_used_bytes;
 
 	/*
 	 * On the first iteration, fetch the dataset's used-on-disk and
 	 * refreservation values. Also, if checkrefquota is set, test if
 	 * allocating this space would exceed the dataset's refquota.
 	 */
 	if (first && tx->tx_objset) {
 		int error;
 		dsl_dataset_t *ds = tx->tx_objset->os_dsl_dataset;
 
 		error = dsl_dataset_check_quota(ds, !netfree,
 		    asize, est_inflight, &used_on_disk, &ref_rsrv);
 		if (error != 0) {
 			mutex_exit(&dd->dd_lock);
 			DMU_TX_STAT_BUMP(dmu_tx_quota);
 			return (error);
 		}
 	}
 
 	/*
 	 * If this transaction will result in a net free of space,
 	 * we want to let it through.
 	 */
 	if (ignorequota || netfree || dsl_dir_phys(dd)->dd_quota == 0 ||
 	    (tx->tx_objset && dmu_objset_type(tx->tx_objset) == DMU_OST_ZVOL &&
 	    zvol_enforce_quotas == B_FALSE))
 		quota = UINT64_MAX;
 	else
 		quota = dsl_dir_phys(dd)->dd_quota;
 
 	/*
 	 * Adjust the quota against the actual pool size at the root
 	 * minus any outstanding deferred frees.
 	 * To ensure that it's possible to remove files from a full
 	 * pool without inducing transient overcommits, we throttle
 	 * netfree transactions against a quota that is slightly larger,
 	 * but still within the pool's allocation slop.  In cases where
 	 * we're very close to full, this will allow a steady trickle of
 	 * removes to get through.
 	 */
 	if (dd->dd_parent == NULL) {
 		uint64_t avail = dsl_pool_unreserved_space(dd->dd_pool,
 		    (netfree) ?
 		    ZFS_SPACE_CHECK_RESERVED : ZFS_SPACE_CHECK_NORMAL);
 
 		if (avail < quota) {
 			quota = avail;
 			retval = SET_ERROR(ENOSPC);
 		}
 	}
 
 	/*
 	 * If they are requesting more space, and our current estimate
 	 * is over quota, they get to try again unless the actual
 	 * on-disk is over quota and there are no pending changes
 	 * or deferred frees (which may free up space for us).
 	 */
 	ext_quota = quota >> 5;
 	if (quota == UINT64_MAX)
 		ext_quota = 0;
 
 	if (used_on_disk >= quota) {
 		if (retval == ENOSPC && (used_on_disk - quota) <
 		    dsl_pool_deferred_space(dd->dd_pool)) {
 			retval = SET_ERROR(ERESTART);
 		}
 		/* Quota exceeded */
 		mutex_exit(&dd->dd_lock);
 		DMU_TX_STAT_BUMP(dmu_tx_quota);
 		return (retval);
 	} else if (used_on_disk + est_inflight >= quota + ext_quota) {
 		dprintf_dd(dd, "failing: used=%lluK inflight = %lluK "
 		    "quota=%lluK tr=%lluK\n",
 		    (u_longlong_t)used_on_disk>>10,
 		    (u_longlong_t)est_inflight>>10,
 		    (u_longlong_t)quota>>10, (u_longlong_t)asize>>10);
 		mutex_exit(&dd->dd_lock);
 		DMU_TX_STAT_BUMP(dmu_tx_quota);
 		return (SET_ERROR(ERESTART));
 	}
 
 	/* We need to up our estimated delta before dropping dd_lock */
 	dd->dd_tempreserved[txg & TXG_MASK] += asize;
 
 	uint64_t parent_rsrv = parent_delta(dd, used_on_disk + est_inflight,
 	    asize - ref_rsrv);
 	mutex_exit(&dd->dd_lock);
 
 	tr = kmem_zalloc(sizeof (struct tempreserve), KM_SLEEP);
 	tr->tr_ds = dd;
 	tr->tr_size = asize;
 	list_insert_tail(tr_list, tr);
 
 	/* see if it's OK with our parent */
 	if (dd->dd_parent != NULL && parent_rsrv != 0) {
 		/*
 		 * Recurse on our parent without recursion. This has been
 		 * observed to be potentially large stack usage even within
 		 * the test suite. Largest seen stack was 7632 bytes on linux.
 		 */
 
 		dd = dd->dd_parent;
 		asize = parent_rsrv;
 		ignorequota = (dsl_dir_phys(dd)->dd_head_dataset_obj == 0);
 		first = B_FALSE;
 		goto top_of_function;
 	}
 
 	return (0);
 }
 
 /*
  * Reserve space in this dsl_dir, to be used in this tx's txg.
  * After the space has been dirtied (and dsl_dir_willuse_space()
  * has been called), the reservation should be canceled, using
  * dsl_dir_tempreserve_clear().
  */
 int
 dsl_dir_tempreserve_space(dsl_dir_t *dd, uint64_t lsize, uint64_t asize,
     boolean_t netfree, void **tr_cookiep, dmu_tx_t *tx)
 {
 	int err;
 	list_t *tr_list;
 
 	if (asize == 0) {
 		*tr_cookiep = NULL;
 		return (0);
 	}
 
 	tr_list = kmem_alloc(sizeof (list_t), KM_SLEEP);
 	list_create(tr_list, sizeof (struct tempreserve),
 	    offsetof(struct tempreserve, tr_node));
 	ASSERT3S(asize, >, 0);
 
 	err = arc_tempreserve_space(dd->dd_pool->dp_spa, lsize, tx->tx_txg);
 	if (err == 0) {
 		struct tempreserve *tr;
 
 		tr = kmem_zalloc(sizeof (struct tempreserve), KM_SLEEP);
 		tr->tr_size = lsize;
 		list_insert_tail(tr_list, tr);
 	} else {
 		if (err == EAGAIN) {
 			/*
 			 * If arc_memory_throttle() detected that pageout
 			 * is running and we are low on memory, we delay new
 			 * non-pageout transactions to give pageout an
 			 * advantage.
 			 *
 			 * It is unfortunate to be delaying while the caller's
 			 * locks are held.
 			 */
 			txg_delay(dd->dd_pool, tx->tx_txg,
 			    MSEC2NSEC(10), MSEC2NSEC(10));
 			err = SET_ERROR(ERESTART);
 		}
 	}
 
 	if (err == 0) {
 		err = dsl_dir_tempreserve_impl(dd, asize, netfree,
 		    B_FALSE, tr_list, tx, B_TRUE);
 	}
 
 	if (err != 0)
 		dsl_dir_tempreserve_clear(tr_list, tx);
 	else
 		*tr_cookiep = tr_list;
 
 	return (err);
 }
 
 /*
  * Clear a temporary reservation that we previously made with
  * dsl_dir_tempreserve_space().
  */
 void
 dsl_dir_tempreserve_clear(void *tr_cookie, dmu_tx_t *tx)
 {
 	int txgidx = tx->tx_txg & TXG_MASK;
 	list_t *tr_list = tr_cookie;
 	struct tempreserve *tr;
 
 	ASSERT3U(tx->tx_txg, !=, 0);
 
 	if (tr_cookie == NULL)
 		return;
 
 	while ((tr = list_remove_head(tr_list)) != NULL) {
 		if (tr->tr_ds) {
 			mutex_enter(&tr->tr_ds->dd_lock);
 			ASSERT3U(tr->tr_ds->dd_tempreserved[txgidx], >=,
 			    tr->tr_size);
 			tr->tr_ds->dd_tempreserved[txgidx] -= tr->tr_size;
 			mutex_exit(&tr->tr_ds->dd_lock);
 		} else {
 			arc_tempreserve_clear(tr->tr_size);
 		}
 		kmem_free(tr, sizeof (struct tempreserve));
 	}
 
 	kmem_free(tr_list, sizeof (list_t));
 }
 
 /*
  * This should be called from open context when we think we're going to write
  * or free space, for example when dirtying data. Be conservative; it's okay
  * to write less space or free more, but we don't want to write more or free
  * less than the amount specified.
  *
  * NOTE: The behavior of this function is identical to the Illumos / FreeBSD
  * version however it has been adjusted to use an iterative rather than
  * recursive algorithm to minimize stack usage.
  */
 void
 dsl_dir_willuse_space(dsl_dir_t *dd, int64_t space, dmu_tx_t *tx)
 {
 	int64_t parent_space;
 	uint64_t est_used;
 
 	do {
 		mutex_enter(&dd->dd_lock);
 		if (space > 0)
 			dd->dd_space_towrite[tx->tx_txg & TXG_MASK] += space;
 
 		est_used = dsl_dir_space_towrite(dd) +
 		    dsl_dir_phys(dd)->dd_used_bytes;
 		parent_space = parent_delta(dd, est_used, space);
 		mutex_exit(&dd->dd_lock);
 
 		/* Make sure that we clean up dd_space_to* */
 		dsl_dir_dirty(dd, tx);
 
 		dd = dd->dd_parent;
 		space = parent_space;
 	} while (space && dd);
 }
 
 /* call from syncing context when we actually write/free space for this dd */
 void
 dsl_dir_diduse_space(dsl_dir_t *dd, dd_used_t type,
     int64_t used, int64_t compressed, int64_t uncompressed, dmu_tx_t *tx)
 {
 	int64_t accounted_delta;
 
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(type < DD_USED_NUM);
 
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 
 	/*
 	 * dsl_dataset_set_refreservation_sync_impl() calls this with
 	 * dd_lock held, so that it can atomically update
 	 * ds->ds_reserved and the dsl_dir accounting, so that
 	 * dsl_dataset_check_quota() can see dataset and dir accounting
 	 * consistently.
 	 */
 	boolean_t needlock = !MUTEX_HELD(&dd->dd_lock);
 	if (needlock)
 		mutex_enter(&dd->dd_lock);
 	dsl_dir_phys_t *ddp = dsl_dir_phys(dd);
 	accounted_delta = parent_delta(dd, ddp->dd_used_bytes, used);
 	ASSERT(used >= 0 || ddp->dd_used_bytes >= -used);
 	ASSERT(compressed >= 0 || ddp->dd_compressed_bytes >= -compressed);
 	ASSERT(uncompressed >= 0 ||
 	    ddp->dd_uncompressed_bytes >= -uncompressed);
 	ddp->dd_used_bytes += used;
 	ddp->dd_uncompressed_bytes += uncompressed;
 	ddp->dd_compressed_bytes += compressed;
 
 	if (ddp->dd_flags & DD_FLAG_USED_BREAKDOWN) {
 		ASSERT(used >= 0 || ddp->dd_used_breakdown[type] >= -used);
 		ddp->dd_used_breakdown[type] += used;
 #ifdef ZFS_DEBUG
 		{
 			dd_used_t t;
 			uint64_t u = 0;
 			for (t = 0; t < DD_USED_NUM; t++)
 				u += ddp->dd_used_breakdown[t];
 			ASSERT3U(u, ==, ddp->dd_used_bytes);
 		}
 #endif
 	}
 	if (needlock)
 		mutex_exit(&dd->dd_lock);
 
 	if (dd->dd_parent != NULL) {
 		dsl_dir_diduse_transfer_space(dd->dd_parent,
 		    accounted_delta, compressed, uncompressed,
 		    used, DD_USED_CHILD_RSRV, DD_USED_CHILD, tx);
 	}
 }
 
 void
 dsl_dir_transfer_space(dsl_dir_t *dd, int64_t delta,
     dd_used_t oldtype, dd_used_t newtype, dmu_tx_t *tx)
 {
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(oldtype < DD_USED_NUM);
 	ASSERT(newtype < DD_USED_NUM);
 
 	dsl_dir_phys_t *ddp = dsl_dir_phys(dd);
 	if (delta == 0 ||
 	    !(ddp->dd_flags & DD_FLAG_USED_BREAKDOWN))
 		return;
 
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 	mutex_enter(&dd->dd_lock);
 	ASSERT(delta > 0 ?
 	    ddp->dd_used_breakdown[oldtype] >= delta :
 	    ddp->dd_used_breakdown[newtype] >= -delta);
 	ASSERT(ddp->dd_used_bytes >= ABS(delta));
 	ddp->dd_used_breakdown[oldtype] -= delta;
 	ddp->dd_used_breakdown[newtype] += delta;
 	mutex_exit(&dd->dd_lock);
 }
 
 void
 dsl_dir_diduse_transfer_space(dsl_dir_t *dd, int64_t used,
     int64_t compressed, int64_t uncompressed, int64_t tonew,
     dd_used_t oldtype, dd_used_t newtype, dmu_tx_t *tx)
 {
 	int64_t accounted_delta;
 
 	ASSERT(dmu_tx_is_syncing(tx));
 	ASSERT(oldtype < DD_USED_NUM);
 	ASSERT(newtype < DD_USED_NUM);
 
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 
 	mutex_enter(&dd->dd_lock);
 	dsl_dir_phys_t *ddp = dsl_dir_phys(dd);
 	accounted_delta = parent_delta(dd, ddp->dd_used_bytes, used);
 	ASSERT(used >= 0 || ddp->dd_used_bytes >= -used);
 	ASSERT(compressed >= 0 || ddp->dd_compressed_bytes >= -compressed);
 	ASSERT(uncompressed >= 0 ||
 	    ddp->dd_uncompressed_bytes >= -uncompressed);
 	ddp->dd_used_bytes += used;
 	ddp->dd_uncompressed_bytes += uncompressed;
 	ddp->dd_compressed_bytes += compressed;
 
 	if (ddp->dd_flags & DD_FLAG_USED_BREAKDOWN) {
 		ASSERT(tonew - used <= 0 ||
 		    ddp->dd_used_breakdown[oldtype] >= tonew - used);
 		ASSERT(tonew >= 0 ||
 		    ddp->dd_used_breakdown[newtype] >= -tonew);
 		ddp->dd_used_breakdown[oldtype] -= tonew - used;
 		ddp->dd_used_breakdown[newtype] += tonew;
 #ifdef ZFS_DEBUG
 		{
 			dd_used_t t;
 			uint64_t u = 0;
 			for (t = 0; t < DD_USED_NUM; t++)
 				u += ddp->dd_used_breakdown[t];
 			ASSERT3U(u, ==, ddp->dd_used_bytes);
 		}
 #endif
 	}
 	mutex_exit(&dd->dd_lock);
 
 	if (dd->dd_parent != NULL) {
 		dsl_dir_diduse_transfer_space(dd->dd_parent,
 		    accounted_delta, compressed, uncompressed,
 		    used, DD_USED_CHILD_RSRV, DD_USED_CHILD, tx);
 	}
 }
 
 typedef struct dsl_dir_set_qr_arg {
 	const char *ddsqra_name;
 	zprop_source_t ddsqra_source;
 	uint64_t ddsqra_value;
 } dsl_dir_set_qr_arg_t;
 
 static int
 dsl_dir_set_quota_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dir_set_qr_arg_t *ddsqra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds;
 	int error;
 	uint64_t towrite, newval;
 
 	error = dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds);
 	if (error != 0)
 		return (error);
 
 	error = dsl_prop_predict(ds->ds_dir, "quota",
 	    ddsqra->ddsqra_source, ddsqra->ddsqra_value, &newval);
 	if (error != 0) {
 		dsl_dataset_rele(ds, FTAG);
 		return (error);
 	}
 
 	if (newval == 0) {
 		dsl_dataset_rele(ds, FTAG);
 		return (0);
 	}
 
 	mutex_enter(&ds->ds_dir->dd_lock);
 	/*
 	 * If we are doing the preliminary check in open context, and
 	 * there are pending changes, then don't fail it, since the
 	 * pending changes could under-estimate the amount of space to be
 	 * freed up.
 	 */
 	towrite = dsl_dir_space_towrite(ds->ds_dir);
 	if ((dmu_tx_is_syncing(tx) || towrite == 0) &&
 	    (newval < dsl_dir_phys(ds->ds_dir)->dd_reserved ||
 	    newval < dsl_dir_phys(ds->ds_dir)->dd_used_bytes + towrite)) {
 		error = SET_ERROR(ENOSPC);
 	}
 	mutex_exit(&ds->ds_dir->dd_lock);
 	dsl_dataset_rele(ds, FTAG);
 	return (error);
 }
 
 static void
 dsl_dir_set_quota_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dir_set_qr_arg_t *ddsqra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds;
 	uint64_t newval;
 
 	VERIFY0(dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds));
 
 	if (spa_version(dp->dp_spa) >= SPA_VERSION_RECVD_PROPS) {
 		dsl_prop_set_sync_impl(ds, zfs_prop_to_name(ZFS_PROP_QUOTA),
 		    ddsqra->ddsqra_source, sizeof (ddsqra->ddsqra_value), 1,
 		    &ddsqra->ddsqra_value, tx);
 
 		VERIFY0(dsl_prop_get_int_ds(ds,
 		    zfs_prop_to_name(ZFS_PROP_QUOTA), &newval));
 	} else {
 		newval = ddsqra->ddsqra_value;
 		spa_history_log_internal_ds(ds, "set", tx, "%s=%lld",
 		    zfs_prop_to_name(ZFS_PROP_QUOTA), (longlong_t)newval);
 	}
 
 	dmu_buf_will_dirty(ds->ds_dir->dd_dbuf, tx);
 	mutex_enter(&ds->ds_dir->dd_lock);
 	dsl_dir_phys(ds->ds_dir)->dd_quota = newval;
 	mutex_exit(&ds->ds_dir->dd_lock);
 	dsl_dataset_rele(ds, FTAG);
 }
 
 int
 dsl_dir_set_quota(const char *ddname, zprop_source_t source, uint64_t quota)
 {
 	dsl_dir_set_qr_arg_t ddsqra;
 
 	ddsqra.ddsqra_name = ddname;
 	ddsqra.ddsqra_source = source;
 	ddsqra.ddsqra_value = quota;
 
 	return (dsl_sync_task(ddname, dsl_dir_set_quota_check,
 	    dsl_dir_set_quota_sync, &ddsqra, 0,
 	    ZFS_SPACE_CHECK_EXTRA_RESERVED));
 }
 
 static int
 dsl_dir_set_reservation_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dir_set_qr_arg_t *ddsqra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds;
 	dsl_dir_t *dd;
 	uint64_t newval, used, avail;
 	int error;
 
 	error = dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds);
 	if (error != 0)
 		return (error);
 	dd = ds->ds_dir;
 
 	/*
 	 * If we are doing the preliminary check in open context, the
 	 * space estimates may be inaccurate.
 	 */
 	if (!dmu_tx_is_syncing(tx)) {
 		dsl_dataset_rele(ds, FTAG);
 		return (0);
 	}
 
 	error = dsl_prop_predict(ds->ds_dir,
 	    zfs_prop_to_name(ZFS_PROP_RESERVATION),
 	    ddsqra->ddsqra_source, ddsqra->ddsqra_value, &newval);
 	if (error != 0) {
 		dsl_dataset_rele(ds, FTAG);
 		return (error);
 	}
 
 	mutex_enter(&dd->dd_lock);
 	used = dsl_dir_phys(dd)->dd_used_bytes;
 	mutex_exit(&dd->dd_lock);
 
 	if (dd->dd_parent) {
 		avail = dsl_dir_space_available(dd->dd_parent,
 		    NULL, 0, FALSE);
 	} else {
 		avail = dsl_pool_adjustedsize(dd->dd_pool,
 		    ZFS_SPACE_CHECK_NORMAL) - used;
 	}
 
 	if (MAX(used, newval) > MAX(used, dsl_dir_phys(dd)->dd_reserved)) {
 		uint64_t delta = MAX(used, newval) -
 		    MAX(used, dsl_dir_phys(dd)->dd_reserved);
 
 		if (delta > avail ||
 		    (dsl_dir_phys(dd)->dd_quota > 0 &&
 		    newval > dsl_dir_phys(dd)->dd_quota))
 			error = SET_ERROR(ENOSPC);
 	}
 
 	dsl_dataset_rele(ds, FTAG);
 	return (error);
 }
 
 void
 dsl_dir_set_reservation_sync_impl(dsl_dir_t *dd, uint64_t value, dmu_tx_t *tx)
 {
 	uint64_t used;
 	int64_t delta;
 
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 
 	mutex_enter(&dd->dd_lock);
 	used = dsl_dir_phys(dd)->dd_used_bytes;
 	delta = MAX(used, value) - MAX(used, dsl_dir_phys(dd)->dd_reserved);
 	dsl_dir_phys(dd)->dd_reserved = value;
 
 	if (dd->dd_parent != NULL) {
 		/* Roll up this additional usage into our ancestors */
 		dsl_dir_diduse_space(dd->dd_parent, DD_USED_CHILD_RSRV,
 		    delta, 0, 0, tx);
 	}
 	mutex_exit(&dd->dd_lock);
 }
 
 static void
 dsl_dir_set_reservation_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dir_set_qr_arg_t *ddsqra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dataset_t *ds;
 	uint64_t newval;
 
 	VERIFY0(dsl_dataset_hold(dp, ddsqra->ddsqra_name, FTAG, &ds));
 
 	if (spa_version(dp->dp_spa) >= SPA_VERSION_RECVD_PROPS) {
 		dsl_prop_set_sync_impl(ds,
 		    zfs_prop_to_name(ZFS_PROP_RESERVATION),
 		    ddsqra->ddsqra_source, sizeof (ddsqra->ddsqra_value), 1,
 		    &ddsqra->ddsqra_value, tx);
 
 		VERIFY0(dsl_prop_get_int_ds(ds,
 		    zfs_prop_to_name(ZFS_PROP_RESERVATION), &newval));
 	} else {
 		newval = ddsqra->ddsqra_value;
 		spa_history_log_internal_ds(ds, "set", tx, "%s=%lld",
 		    zfs_prop_to_name(ZFS_PROP_RESERVATION),
 		    (longlong_t)newval);
 	}
 
 	dsl_dir_set_reservation_sync_impl(ds->ds_dir, newval, tx);
 	dsl_dataset_rele(ds, FTAG);
 }
 
 int
 dsl_dir_set_reservation(const char *ddname, zprop_source_t source,
     uint64_t reservation)
 {
 	dsl_dir_set_qr_arg_t ddsqra;
 
 	ddsqra.ddsqra_name = ddname;
 	ddsqra.ddsqra_source = source;
 	ddsqra.ddsqra_value = reservation;
 
 	return (dsl_sync_task(ddname, dsl_dir_set_reservation_check,
 	    dsl_dir_set_reservation_sync, &ddsqra, 0,
 	    ZFS_SPACE_CHECK_EXTRA_RESERVED));
 }
 
 static dsl_dir_t *
 closest_common_ancestor(dsl_dir_t *ds1, dsl_dir_t *ds2)
 {
 	for (; ds1; ds1 = ds1->dd_parent) {
 		dsl_dir_t *dd;
 		for (dd = ds2; dd; dd = dd->dd_parent) {
 			if (ds1 == dd)
 				return (dd);
 		}
 	}
 	return (NULL);
 }
 
 /*
  * If delta is applied to dd, how much of that delta would be applied to
  * ancestor?  Syncing context only.
  */
 static int64_t
 would_change(dsl_dir_t *dd, int64_t delta, dsl_dir_t *ancestor)
 {
 	if (dd == ancestor)
 		return (delta);
 
 	mutex_enter(&dd->dd_lock);
 	delta = parent_delta(dd, dsl_dir_phys(dd)->dd_used_bytes, delta);
 	mutex_exit(&dd->dd_lock);
 	return (would_change(dd->dd_parent, delta, ancestor));
 }
 
 typedef struct dsl_dir_rename_arg {
 	const char *ddra_oldname;
 	const char *ddra_newname;
 	cred_t *ddra_cred;
-	proc_t *ddra_proc;
 } dsl_dir_rename_arg_t;
 
 typedef struct dsl_valid_rename_arg {
 	int char_delta;
 	int nest_delta;
 } dsl_valid_rename_arg_t;
 
 static int
 dsl_valid_rename(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
 {
 	(void) dp;
 	dsl_valid_rename_arg_t *dvra = arg;
 	char namebuf[ZFS_MAX_DATASET_NAME_LEN];
 
 	dsl_dataset_name(ds, namebuf);
 
 	ASSERT3U(strnlen(namebuf, ZFS_MAX_DATASET_NAME_LEN),
 	    <, ZFS_MAX_DATASET_NAME_LEN);
 	int namelen = strlen(namebuf) + dvra->char_delta;
 	int depth = get_dataset_depth(namebuf) + dvra->nest_delta;
 
 	if (namelen >= ZFS_MAX_DATASET_NAME_LEN)
 		return (SET_ERROR(ENAMETOOLONG));
 	if (dvra->nest_delta > 0 && depth >= zfs_max_dataset_nesting)
 		return (SET_ERROR(ENAMETOOLONG));
 	return (0);
 }
 
 static int
 dsl_dir_rename_check(void *arg, dmu_tx_t *tx)
 {
 	dsl_dir_rename_arg_t *ddra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dir_t *dd, *newparent;
 	dsl_valid_rename_arg_t dvra;
 	dsl_dataset_t *parentds;
 	objset_t *parentos;
 	const char *mynewname;
 	int error;
 
 	/* target dir should exist */
 	error = dsl_dir_hold(dp, ddra->ddra_oldname, FTAG, &dd, NULL);
 	if (error != 0)
 		return (error);
 
 	/* new parent should exist */
 	error = dsl_dir_hold(dp, ddra->ddra_newname, FTAG,
 	    &newparent, &mynewname);
 	if (error != 0) {
 		dsl_dir_rele(dd, FTAG);
 		return (error);
 	}
 
 	/* can't rename to different pool */
 	if (dd->dd_pool != newparent->dd_pool) {
 		dsl_dir_rele(newparent, FTAG);
 		dsl_dir_rele(dd, FTAG);
 		return (SET_ERROR(EXDEV));
 	}
 
 	/* new name should not already exist */
 	if (mynewname == NULL) {
 		dsl_dir_rele(newparent, FTAG);
 		dsl_dir_rele(dd, FTAG);
 		return (SET_ERROR(EEXIST));
 	}
 
 	/* can't rename below anything but filesystems (eg. no ZVOLs) */
 	error = dsl_dataset_hold_obj(newparent->dd_pool,
 	    dsl_dir_phys(newparent)->dd_head_dataset_obj, FTAG, &parentds);
 	if (error != 0) {
 		dsl_dir_rele(newparent, FTAG);
 		dsl_dir_rele(dd, FTAG);
 		return (error);
 	}
 	error = dmu_objset_from_ds(parentds, &parentos);
 	if (error != 0) {
 		dsl_dataset_rele(parentds, FTAG);
 		dsl_dir_rele(newparent, FTAG);
 		dsl_dir_rele(dd, FTAG);
 		return (error);
 	}
 	if (dmu_objset_type(parentos) != DMU_OST_ZFS) {
 		dsl_dataset_rele(parentds, FTAG);
 		dsl_dir_rele(newparent, FTAG);
 		dsl_dir_rele(dd, FTAG);
 		return (SET_ERROR(ZFS_ERR_WRONG_PARENT));
 	}
 	dsl_dataset_rele(parentds, FTAG);
 
 	ASSERT3U(strnlen(ddra->ddra_newname, ZFS_MAX_DATASET_NAME_LEN),
 	    <, ZFS_MAX_DATASET_NAME_LEN);
 	ASSERT3U(strnlen(ddra->ddra_oldname, ZFS_MAX_DATASET_NAME_LEN),
 	    <, ZFS_MAX_DATASET_NAME_LEN);
 	dvra.char_delta = strlen(ddra->ddra_newname)
 	    - strlen(ddra->ddra_oldname);
 	dvra.nest_delta = get_dataset_depth(ddra->ddra_newname)
 	    - get_dataset_depth(ddra->ddra_oldname);
 
 	/* if the name length is growing, validate child name lengths */
 	if (dvra.char_delta > 0 || dvra.nest_delta > 0) {
 		error = dmu_objset_find_dp(dp, dd->dd_object, dsl_valid_rename,
 		    &dvra, DS_FIND_CHILDREN | DS_FIND_SNAPSHOTS);
 		if (error != 0) {
 			dsl_dir_rele(newparent, FTAG);
 			dsl_dir_rele(dd, FTAG);
 			return (error);
 		}
 	}
 
 	if (dmu_tx_is_syncing(tx)) {
 		if (spa_feature_is_active(dp->dp_spa,
 		    SPA_FEATURE_FS_SS_LIMIT)) {
 			/*
 			 * Although this is the check function and we don't
 			 * normally make on-disk changes in check functions,
 			 * we need to do that here.
 			 *
 			 * Ensure this portion of the tree's counts have been
 			 * initialized in case the new parent has limits set.
 			 */
 			dsl_dir_init_fs_ss_count(dd, tx);
 		}
 	}
 
 	if (newparent != dd->dd_parent) {
 		/* is there enough space? */
 		uint64_t myspace =
 		    MAX(dsl_dir_phys(dd)->dd_used_bytes,
 		    dsl_dir_phys(dd)->dd_reserved);
 		objset_t *os = dd->dd_pool->dp_meta_objset;
 		uint64_t fs_cnt = 0;
 		uint64_t ss_cnt = 0;
 
 		if (dsl_dir_is_zapified(dd)) {
 			int err;
 
 			err = zap_lookup(os, dd->dd_object,
 			    DD_FIELD_FILESYSTEM_COUNT, sizeof (fs_cnt), 1,
 			    &fs_cnt);
 			if (err != ENOENT && err != 0) {
 				dsl_dir_rele(newparent, FTAG);
 				dsl_dir_rele(dd, FTAG);
 				return (err);
 			}
 
 			/*
 			 * have to add 1 for the filesystem itself that we're
 			 * moving
 			 */
 			fs_cnt++;
 
 			err = zap_lookup(os, dd->dd_object,
 			    DD_FIELD_SNAPSHOT_COUNT, sizeof (ss_cnt), 1,
 			    &ss_cnt);
 			if (err != ENOENT && err != 0) {
 				dsl_dir_rele(newparent, FTAG);
 				dsl_dir_rele(dd, FTAG);
 				return (err);
 			}
 		}
 
 		/* check for encryption errors */
 		error = dsl_dir_rename_crypt_check(dd, newparent);
 		if (error != 0) {
 			dsl_dir_rele(newparent, FTAG);
 			dsl_dir_rele(dd, FTAG);
 			return (SET_ERROR(EACCES));
 		}
 
 		/* no rename into our descendant */
 		if (closest_common_ancestor(dd, newparent) == dd) {
 			dsl_dir_rele(newparent, FTAG);
 			dsl_dir_rele(dd, FTAG);
 			return (SET_ERROR(EINVAL));
 		}
 
 		error = dsl_dir_transfer_possible(dd->dd_parent,
-		    newparent, fs_cnt, ss_cnt, myspace,
-		    ddra->ddra_cred, ddra->ddra_proc);
+		    newparent, fs_cnt, ss_cnt, myspace, ddra->ddra_cred);
 		if (error != 0) {
 			dsl_dir_rele(newparent, FTAG);
 			dsl_dir_rele(dd, FTAG);
 			return (error);
 		}
 	}
 
 	dsl_dir_rele(newparent, FTAG);
 	dsl_dir_rele(dd, FTAG);
 	return (0);
 }
 
 static void
 dsl_dir_rename_sync(void *arg, dmu_tx_t *tx)
 {
 	dsl_dir_rename_arg_t *ddra = arg;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	dsl_dir_t *dd, *newparent;
 	const char *mynewname;
 	objset_t *mos = dp->dp_meta_objset;
 
 	VERIFY0(dsl_dir_hold(dp, ddra->ddra_oldname, FTAG, &dd, NULL));
 	VERIFY0(dsl_dir_hold(dp, ddra->ddra_newname, FTAG, &newparent,
 	    &mynewname));
 
 	ASSERT3P(mynewname, !=, NULL);
 
 	/* Log this before we change the name. */
 	spa_history_log_internal_dd(dd, "rename", tx,
 	    "-> %s", ddra->ddra_newname);
 
 	if (newparent != dd->dd_parent) {
 		objset_t *os = dd->dd_pool->dp_meta_objset;
 		uint64_t fs_cnt = 0;
 		uint64_t ss_cnt = 0;
 
 		/*
 		 * We already made sure the dd counts were initialized in the
 		 * check function.
 		 */
 		if (spa_feature_is_active(dp->dp_spa,
 		    SPA_FEATURE_FS_SS_LIMIT)) {
 			VERIFY0(zap_lookup(os, dd->dd_object,
 			    DD_FIELD_FILESYSTEM_COUNT, sizeof (fs_cnt), 1,
 			    &fs_cnt));
 			/* add 1 for the filesystem itself that we're moving */
 			fs_cnt++;
 
 			VERIFY0(zap_lookup(os, dd->dd_object,
 			    DD_FIELD_SNAPSHOT_COUNT, sizeof (ss_cnt), 1,
 			    &ss_cnt));
 		}
 
 		dsl_fs_ss_count_adjust(dd->dd_parent, -fs_cnt,
 		    DD_FIELD_FILESYSTEM_COUNT, tx);
 		dsl_fs_ss_count_adjust(newparent, fs_cnt,
 		    DD_FIELD_FILESYSTEM_COUNT, tx);
 
 		dsl_fs_ss_count_adjust(dd->dd_parent, -ss_cnt,
 		    DD_FIELD_SNAPSHOT_COUNT, tx);
 		dsl_fs_ss_count_adjust(newparent, ss_cnt,
 		    DD_FIELD_SNAPSHOT_COUNT, tx);
 
 		dsl_dir_diduse_space(dd->dd_parent, DD_USED_CHILD,
 		    -dsl_dir_phys(dd)->dd_used_bytes,
 		    -dsl_dir_phys(dd)->dd_compressed_bytes,
 		    -dsl_dir_phys(dd)->dd_uncompressed_bytes, tx);
 		dsl_dir_diduse_space(newparent, DD_USED_CHILD,
 		    dsl_dir_phys(dd)->dd_used_bytes,
 		    dsl_dir_phys(dd)->dd_compressed_bytes,
 		    dsl_dir_phys(dd)->dd_uncompressed_bytes, tx);
 
 		if (dsl_dir_phys(dd)->dd_reserved >
 		    dsl_dir_phys(dd)->dd_used_bytes) {
 			uint64_t unused_rsrv = dsl_dir_phys(dd)->dd_reserved -
 			    dsl_dir_phys(dd)->dd_used_bytes;
 
 			dsl_dir_diduse_space(dd->dd_parent, DD_USED_CHILD_RSRV,
 			    -unused_rsrv, 0, 0, tx);
 			dsl_dir_diduse_space(newparent, DD_USED_CHILD_RSRV,
 			    unused_rsrv, 0, 0, tx);
 		}
 	}
 
 	dmu_buf_will_dirty(dd->dd_dbuf, tx);
 
 	/* remove from old parent zapobj */
 	VERIFY0(zap_remove(mos,
 	    dsl_dir_phys(dd->dd_parent)->dd_child_dir_zapobj,
 	    dd->dd_myname, tx));
 
 	(void) strlcpy(dd->dd_myname, mynewname,
 	    sizeof (dd->dd_myname));
 	dsl_dir_rele(dd->dd_parent, dd);
 	dsl_dir_phys(dd)->dd_parent_obj = newparent->dd_object;
 	VERIFY0(dsl_dir_hold_obj(dp,
 	    newparent->dd_object, NULL, dd, &dd->dd_parent));
 
 	/* add to new parent zapobj */
 	VERIFY0(zap_add(mos, dsl_dir_phys(newparent)->dd_child_dir_zapobj,
 	    dd->dd_myname, 8, 1, &dd->dd_object, tx));
 
 	/* TODO: A rename callback to avoid these layering violations. */
 	zfsvfs_update_fromname(ddra->ddra_oldname, ddra->ddra_newname);
 	zvol_rename_minors(dp->dp_spa, ddra->ddra_oldname,
 	    ddra->ddra_newname, B_TRUE);
 
 	dsl_prop_notify_all(dd);
 
 	dsl_dir_rele(newparent, FTAG);
 	dsl_dir_rele(dd, FTAG);
 }
 
 int
 dsl_dir_rename(const char *oldname, const char *newname)
 {
+	cred_t *cr = CRED();
+	crhold(cr);
+
 	dsl_dir_rename_arg_t ddra;
 
 	ddra.ddra_oldname = oldname;
 	ddra.ddra_newname = newname;
-	ddra.ddra_cred = CRED();
-	ddra.ddra_proc = curproc;
+	ddra.ddra_cred = cr;
 
-	return (dsl_sync_task(oldname,
+	int err = dsl_sync_task(oldname,
 	    dsl_dir_rename_check, dsl_dir_rename_sync, &ddra,
-	    3, ZFS_SPACE_CHECK_RESERVED));
+	    3, ZFS_SPACE_CHECK_RESERVED);
+
+	crfree(cr);
+	return (err);
 }
 
 int
 dsl_dir_transfer_possible(dsl_dir_t *sdd, dsl_dir_t *tdd,
     uint64_t fs_cnt, uint64_t ss_cnt, uint64_t space,
-    cred_t *cr, proc_t *proc)
+    cred_t *cr)
 {
 	dsl_dir_t *ancestor;
 	int64_t adelta;
 	uint64_t avail;
 	int err;
 
 	ancestor = closest_common_ancestor(sdd, tdd);
 	adelta = would_change(sdd, -space, ancestor);
 	avail = dsl_dir_space_available(tdd, ancestor, adelta, FALSE);
 	if (avail < space)
 		return (SET_ERROR(ENOSPC));
 
 	err = dsl_fs_ss_limit_check(tdd, fs_cnt, ZFS_PROP_FILESYSTEM_LIMIT,
-	    ancestor, cr, proc);
+	    ancestor, cr);
 	if (err != 0)
 		return (err);
 	err = dsl_fs_ss_limit_check(tdd, ss_cnt, ZFS_PROP_SNAPSHOT_LIMIT,
-	    ancestor, cr, proc);
+	    ancestor, cr);
 	if (err != 0)
 		return (err);
 
 	return (0);
 }
 
 inode_timespec_t
 dsl_dir_snap_cmtime(dsl_dir_t *dd)
 {
 	inode_timespec_t t;
 
 	mutex_enter(&dd->dd_lock);
 	t = dd->dd_snap_cmtime;
 	mutex_exit(&dd->dd_lock);
 
 	return (t);
 }
 
 void
 dsl_dir_snap_cmtime_update(dsl_dir_t *dd, dmu_tx_t *tx)
 {
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	inode_timespec_t t;
 	gethrestime(&t);
 
 	mutex_enter(&dd->dd_lock);
 	dd->dd_snap_cmtime = t;
 	if (spa_feature_is_enabled(dp->dp_spa,
 	    SPA_FEATURE_EXTENSIBLE_DATASET)) {
 		objset_t *mos = dd->dd_pool->dp_meta_objset;
 		uint64_t ddobj = dd->dd_object;
 		dsl_dir_zapify(dd, tx);
 		VERIFY0(zap_update(mos, ddobj,
 		    DD_FIELD_SNAPSHOTS_CHANGED,
 		    sizeof (uint64_t),
 		    sizeof (inode_timespec_t) / sizeof (uint64_t),
 		    &t, tx));
 	}
 	mutex_exit(&dd->dd_lock);
 }
 
 void
 dsl_dir_zapify(dsl_dir_t *dd, dmu_tx_t *tx)
 {
 	objset_t *mos = dd->dd_pool->dp_meta_objset;
 	dmu_object_zapify(mos, dd->dd_object, DMU_OT_DSL_DIR, tx);
 }
 
 boolean_t
 dsl_dir_is_zapified(dsl_dir_t *dd)
 {
 	dmu_object_info_t doi;
 
 	dmu_object_info_from_db(dd->dd_dbuf, &doi);
 	return (doi.doi_type == DMU_OTN_ZAP_METADATA);
 }
 
 void
 dsl_dir_livelist_open(dsl_dir_t *dd, uint64_t obj)
 {
 	objset_t *mos = dd->dd_pool->dp_meta_objset;
 	ASSERT(spa_feature_is_active(dd->dd_pool->dp_spa,
 	    SPA_FEATURE_LIVELIST));
 	dsl_deadlist_open(&dd->dd_livelist, mos, obj);
 	bplist_create(&dd->dd_pending_allocs);
 	bplist_create(&dd->dd_pending_frees);
 }
 
 void
 dsl_dir_livelist_close(dsl_dir_t *dd)
 {
 	dsl_deadlist_close(&dd->dd_livelist);
 	bplist_destroy(&dd->dd_pending_allocs);
 	bplist_destroy(&dd->dd_pending_frees);
 }
 
 void
 dsl_dir_remove_livelist(dsl_dir_t *dd, dmu_tx_t *tx, boolean_t total)
 {
 	uint64_t obj;
 	dsl_pool_t *dp = dmu_tx_pool(tx);
 	spa_t *spa = dp->dp_spa;
 	livelist_condense_entry_t to_condense = spa->spa_to_condense;
 
 	if (!dsl_deadlist_is_open(&dd->dd_livelist))
 		return;
 
 	/*
 	 * If the livelist being removed is set to be condensed, stop the
 	 * condense zthr and indicate the cancellation in the spa_to_condense
 	 * struct in case the condense no-wait synctask has already started
 	 */
 	zthr_t *ll_condense_thread = spa->spa_livelist_condense_zthr;
 	if (ll_condense_thread != NULL &&
 	    (to_condense.ds != NULL) && (to_condense.ds->ds_dir == dd)) {
 		/*
 		 * We use zthr_wait_cycle_done instead of zthr_cancel
 		 * because we don't want to destroy the zthr, just have
 		 * it skip its current task.
 		 */
 		spa->spa_to_condense.cancelled = B_TRUE;
 		zthr_wait_cycle_done(ll_condense_thread);
 		/*
 		 * If we've returned from zthr_wait_cycle_done without
 		 * clearing the to_condense data structure it's either
 		 * because the no-wait synctask has started (which is
 		 * indicated by 'syncing' field of to_condense) and we
 		 * can expect it to clear to_condense on its own.
 		 * Otherwise, we returned before the zthr ran. The
 		 * checkfunc will now fail as cancelled == B_TRUE so we
 		 * can safely NULL out ds, allowing a different dir's
 		 * livelist to be condensed.
 		 *
 		 * We can be sure that the to_condense struct will not
 		 * be repopulated at this stage because both this
 		 * function and dsl_livelist_try_condense execute in
 		 * syncing context.
 		 */
 		if ((spa->spa_to_condense.ds != NULL) &&
 		    !spa->spa_to_condense.syncing) {
 			dmu_buf_rele(spa->spa_to_condense.ds->ds_dbuf,
 			    spa);
 			spa->spa_to_condense.ds = NULL;
 		}
 	}
 
 	dsl_dir_livelist_close(dd);
 	VERIFY0(zap_lookup(dp->dp_meta_objset, dd->dd_object,
 	    DD_FIELD_LIVELIST, sizeof (uint64_t), 1, &obj));
 	VERIFY0(zap_remove(dp->dp_meta_objset, dd->dd_object,
 	    DD_FIELD_LIVELIST, tx));
 	if (total) {
 		dsl_deadlist_free(dp->dp_meta_objset, obj, tx);
 		spa_feature_decr(spa, SPA_FEATURE_LIVELIST, tx);
 	}
 }
 
 static int
 dsl_dir_activity_in_progress(dsl_dir_t *dd, dsl_dataset_t *ds,
     zfs_wait_activity_t activity, boolean_t *in_progress)
 {
 	int error = 0;
 
 	ASSERT(MUTEX_HELD(&dd->dd_activity_lock));
 
 	switch (activity) {
 	case ZFS_WAIT_DELETEQ: {
 #ifdef _KERNEL
 		objset_t *os;
 		error = dmu_objset_from_ds(ds, &os);
 		if (error != 0)
 			break;
 
 		mutex_enter(&os->os_user_ptr_lock);
 		void *user = dmu_objset_get_user(os);
 		mutex_exit(&os->os_user_ptr_lock);
 		if (dmu_objset_type(os) != DMU_OST_ZFS ||
 		    user == NULL || zfs_get_vfs_flag_unmounted(os)) {
 			*in_progress = B_FALSE;
 			return (0);
 		}
 
 		uint64_t readonly = B_FALSE;
 		error = zfs_get_temporary_prop(ds, ZFS_PROP_READONLY, &readonly,
 		    NULL);
 
 		if (error != 0)
 			break;
 
 		if (readonly || !spa_writeable(dd->dd_pool->dp_spa)) {
 			*in_progress = B_FALSE;
 			return (0);
 		}
 
 		uint64_t count, unlinked_obj;
 		error = zap_lookup(os, MASTER_NODE_OBJ, ZFS_UNLINKED_SET, 8, 1,
 		    &unlinked_obj);
 		if (error != 0) {
 			dsl_dataset_rele(ds, FTAG);
 			break;
 		}
 		error = zap_count(os, unlinked_obj, &count);
 
 		if (error == 0)
 			*in_progress = (count != 0);
 		break;
 #else
 		/*
 		 * The delete queue is ZPL specific, and libzpool doesn't have
 		 * it. It doesn't make sense to wait for it.
 		 */
 		(void) ds;
 		*in_progress = B_FALSE;
 		break;
 #endif
 	}
 	default:
 		panic("unrecognized value for activity %d", activity);
 	}
 
 	return (error);
 }
 
 int
 dsl_dir_wait(dsl_dir_t *dd, dsl_dataset_t *ds, zfs_wait_activity_t activity,
     boolean_t *waited)
 {
 	int error = 0;
 	boolean_t in_progress;
 	dsl_pool_t *dp = dd->dd_pool;
 	for (;;) {
 		dsl_pool_config_enter(dp, FTAG);
 		error = dsl_dir_activity_in_progress(dd, ds, activity,
 		    &in_progress);
 		dsl_pool_config_exit(dp, FTAG);
 		if (error != 0 || !in_progress)
 			break;
 
 		*waited = B_TRUE;
 
 		if (cv_wait_sig(&dd->dd_activity_cv, &dd->dd_activity_lock) ==
 		    0 || dd->dd_activity_cancelled) {
 			error = SET_ERROR(EINTR);
 			break;
 		}
 	}
 	return (error);
 }
 
 void
 dsl_dir_cancel_waiters(dsl_dir_t *dd)
 {
 	mutex_enter(&dd->dd_activity_lock);
 	dd->dd_activity_cancelled = B_TRUE;
 	cv_broadcast(&dd->dd_activity_cv);
 	while (dd->dd_activity_waiters > 0)
 		cv_wait(&dd->dd_activity_cv, &dd->dd_activity_lock);
 	mutex_exit(&dd->dd_activity_lock);
 }
 
 #if defined(_KERNEL)
 EXPORT_SYMBOL(dsl_dir_set_quota);
 EXPORT_SYMBOL(dsl_dir_set_reservation);
 #endif
 
 /* CSTYLED */
 ZFS_MODULE_PARAM(zfs, , zvol_enforce_quotas, INT, ZMOD_RW,
 	"Enable strict ZVOL quota enforcment");
diff --git a/sys/contrib/openzfs/module/zfs/metaslab.c b/sys/contrib/openzfs/module/zfs/metaslab.c
index 3876b1014973..7eda653a810e 100644
--- a/sys/contrib/openzfs/module/zfs/metaslab.c
+++ b/sys/contrib/openzfs/module/zfs/metaslab.c
@@ -1,6236 +1,6244 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2019 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2017, Intel Corporation.
  */
 
 #include <sys/zfs_context.h>
+#include <sys/brt.h>
 #include <sys/dmu.h>
 #include <sys/dmu_tx.h>
 #include <sys/space_map.h>
 #include <sys/metaslab_impl.h>
 #include <sys/vdev_impl.h>
 #include <sys/vdev_draid.h>
 #include <sys/zio.h>
 #include <sys/spa_impl.h>
 #include <sys/zfeature.h>
 #include <sys/vdev_indirect_mapping.h>
 #include <sys/zap.h>
 #include <sys/btree.h>
 
 #define	WITH_DF_BLOCK_ALLOCATOR
 
 #define	GANG_ALLOCATION(flags) \
 	((flags) & (METASLAB_GANG_CHILD | METASLAB_GANG_HEADER))
 
 /*
  * Metaslab granularity, in bytes. This is roughly similar to what would be
  * referred to as the "stripe size" in traditional RAID arrays. In normal
  * operation, we will try to write this amount of data to each disk before
  * moving on to the next top-level vdev.
  */
 static uint64_t metaslab_aliquot = 1024 * 1024;
 
 /*
  * For testing, make some blocks above a certain size be gang blocks.
  */
 uint64_t metaslab_force_ganging = SPA_MAXBLOCKSIZE + 1;
 
 /*
  * Of blocks of size >= metaslab_force_ganging, actually gang them this often.
  */
 uint_t metaslab_force_ganging_pct = 3;
 
 /*
  * In pools where the log space map feature is not enabled we touch
  * multiple metaslabs (and their respective space maps) with each
  * transaction group. Thus, we benefit from having a small space map
  * block size since it allows us to issue more I/O operations scattered
  * around the disk. So a sane default for the space map block size
  * is 8~16K.
  */
 int zfs_metaslab_sm_blksz_no_log = (1 << 14);
 
 /*
  * When the log space map feature is enabled, we accumulate a lot of
  * changes per metaslab that are flushed once in a while so we benefit
  * from a bigger block size like 128K for the metaslab space maps.
  */
 int zfs_metaslab_sm_blksz_with_log = (1 << 17);
 
 /*
  * The in-core space map representation is more compact than its on-disk form.
  * The zfs_condense_pct determines how much more compact the in-core
  * space map representation must be before we compact it on-disk.
  * Values should be greater than or equal to 100.
  */
 uint_t zfs_condense_pct = 200;
 
 /*
  * Condensing a metaslab is not guaranteed to actually reduce the amount of
  * space used on disk. In particular, a space map uses data in increments of
  * MAX(1 << ashift, space_map_blksz), so a metaslab might use the
  * same number of blocks after condensing. Since the goal of condensing is to
  * reduce the number of IOPs required to read the space map, we only want to
  * condense when we can be sure we will reduce the number of blocks used by the
  * space map. Unfortunately, we cannot precisely compute whether or not this is
  * the case in metaslab_should_condense since we are holding ms_lock. Instead,
  * we apply the following heuristic: do not condense a spacemap unless the
  * uncondensed size consumes greater than zfs_metaslab_condense_block_threshold
  * blocks.
  */
 static const int zfs_metaslab_condense_block_threshold = 4;
 
 /*
  * The zfs_mg_noalloc_threshold defines which metaslab groups should
  * be eligible for allocation. The value is defined as a percentage of
  * free space. Metaslab groups that have more free space than
  * zfs_mg_noalloc_threshold are always eligible for allocations. Once
  * a metaslab group's free space is less than or equal to the
  * zfs_mg_noalloc_threshold the allocator will avoid allocating to that
  * group unless all groups in the pool have reached zfs_mg_noalloc_threshold.
  * Once all groups in the pool reach zfs_mg_noalloc_threshold then all
  * groups are allowed to accept allocations. Gang blocks are always
  * eligible to allocate on any metaslab group. The default value of 0 means
  * no metaslab group will be excluded based on this criterion.
  */
 static uint_t zfs_mg_noalloc_threshold = 0;
 
 /*
  * Metaslab groups are considered eligible for allocations if their
  * fragmentation metric (measured as a percentage) is less than or
  * equal to zfs_mg_fragmentation_threshold. If a metaslab group
  * exceeds this threshold then it will be skipped unless all metaslab
  * groups within the metaslab class have also crossed this threshold.
  *
  * This tunable was introduced to avoid edge cases where we continue
  * allocating from very fragmented disks in our pool while other, less
  * fragmented disks, exists. On the other hand, if all disks in the
  * pool are uniformly approaching the threshold, the threshold can
  * be a speed bump in performance, where we keep switching the disks
  * that we allocate from (e.g. we allocate some segments from disk A
  * making it bypassing the threshold while freeing segments from disk
  * B getting its fragmentation below the threshold).
  *
  * Empirically, we've seen that our vdev selection for allocations is
  * good enough that fragmentation increases uniformly across all vdevs
  * the majority of the time. Thus we set the threshold percentage high
  * enough to avoid hitting the speed bump on pools that are being pushed
  * to the edge.
  */
 static uint_t zfs_mg_fragmentation_threshold = 95;
 
 /*
  * Allow metaslabs to keep their active state as long as their fragmentation
  * percentage is less than or equal to zfs_metaslab_fragmentation_threshold. An
  * active metaslab that exceeds this threshold will no longer keep its active
  * status allowing better metaslabs to be selected.
  */
 static uint_t zfs_metaslab_fragmentation_threshold = 70;
 
 /*
  * When set will load all metaslabs when pool is first opened.
  */
 int metaslab_debug_load = B_FALSE;
 
 /*
  * When set will prevent metaslabs from being unloaded.
  */
 static int metaslab_debug_unload = B_FALSE;
 
 /*
  * Minimum size which forces the dynamic allocator to change
  * it's allocation strategy.  Once the space map cannot satisfy
  * an allocation of this size then it switches to using more
  * aggressive strategy (i.e search by size rather than offset).
  */
 uint64_t metaslab_df_alloc_threshold = SPA_OLD_MAXBLOCKSIZE;
 
 /*
  * The minimum free space, in percent, which must be available
  * in a space map to continue allocations in a first-fit fashion.
  * Once the space map's free space drops below this level we dynamically
  * switch to using best-fit allocations.
  */
 uint_t metaslab_df_free_pct = 4;
 
 /*
  * Maximum distance to search forward from the last offset. Without this
  * limit, fragmented pools can see >100,000 iterations and
  * metaslab_block_picker() becomes the performance limiting factor on
  * high-performance storage.
  *
  * With the default setting of 16MB, we typically see less than 500
  * iterations, even with very fragmented, ashift=9 pools. The maximum number
  * of iterations possible is:
  *     metaslab_df_max_search / (2 * (1<<ashift))
  * With the default setting of 16MB this is 16*1024 (with ashift=9) or
  * 2048 (with ashift=12).
  */
 static uint_t metaslab_df_max_search = 16 * 1024 * 1024;
 
 /*
  * Forces the metaslab_block_picker function to search for at least this many
  * segments forwards until giving up on finding a segment that the allocation
  * will fit into.
  */
 static const uint32_t metaslab_min_search_count = 100;
 
 /*
  * If we are not searching forward (due to metaslab_df_max_search,
  * metaslab_df_free_pct, or metaslab_df_alloc_threshold), this tunable
  * controls what segment is used.  If it is set, we will use the largest free
  * segment.  If it is not set, we will use a segment of exactly the requested
  * size (or larger).
  */
 static int metaslab_df_use_largest_segment = B_FALSE;
 
 /*
  * These tunables control how long a metaslab will remain loaded after the
  * last allocation from it.  A metaslab can't be unloaded until at least
  * metaslab_unload_delay TXG's and metaslab_unload_delay_ms milliseconds
  * have elapsed.  However, zfs_metaslab_mem_limit may cause it to be
  * unloaded sooner.  These settings are intended to be generous -- to keep
  * metaslabs loaded for a long time, reducing the rate of metaslab loading.
  */
 static uint_t metaslab_unload_delay = 32;
 static uint_t metaslab_unload_delay_ms = 10 * 60 * 1000; /* ten minutes */
 
 /*
  * Max number of metaslabs per group to preload.
  */
 uint_t metaslab_preload_limit = 10;
 
 /*
  * Enable/disable preloading of metaslab.
  */
 static int metaslab_preload_enabled = B_TRUE;
 
 /*
  * Enable/disable fragmentation weighting on metaslabs.
  */
 static int metaslab_fragmentation_factor_enabled = B_TRUE;
 
 /*
  * Enable/disable lba weighting (i.e. outer tracks are given preference).
  */
 static int metaslab_lba_weighting_enabled = B_TRUE;
 
 /*
  * Enable/disable metaslab group biasing.
  */
 static int metaslab_bias_enabled = B_TRUE;
 
 /*
  * Enable/disable remapping of indirect DVAs to their concrete vdevs.
  */
 static const boolean_t zfs_remap_blkptr_enable = B_TRUE;
 
 /*
  * Enable/disable segment-based metaslab selection.
  */
 static int zfs_metaslab_segment_weight_enabled = B_TRUE;
 
 /*
  * When using segment-based metaslab selection, we will continue
  * allocating from the active metaslab until we have exhausted
  * zfs_metaslab_switch_threshold of its buckets.
  */
 static int zfs_metaslab_switch_threshold = 2;
 
 /*
  * Internal switch to enable/disable the metaslab allocation tracing
  * facility.
  */
 static const boolean_t metaslab_trace_enabled = B_FALSE;
 
 /*
  * Maximum entries that the metaslab allocation tracing facility will keep
  * in a given list when running in non-debug mode. We limit the number
  * of entries in non-debug mode to prevent us from using up too much memory.
  * The limit should be sufficiently large that we don't expect any allocation
  * to every exceed this value. In debug mode, the system will panic if this
  * limit is ever reached allowing for further investigation.
  */
 static const uint64_t metaslab_trace_max_entries = 5000;
 
 /*
  * Maximum number of metaslabs per group that can be disabled
  * simultaneously.
  */
 static const int max_disabled_ms = 3;
 
 /*
  * Time (in seconds) to respect ms_max_size when the metaslab is not loaded.
  * To avoid 64-bit overflow, don't set above UINT32_MAX.
  */
 static uint64_t zfs_metaslab_max_size_cache_sec = 1 * 60 * 60; /* 1 hour */
 
 /*
  * Maximum percentage of memory to use on storing loaded metaslabs. If loading
  * a metaslab would take it over this percentage, the oldest selected metaslab
  * is automatically unloaded.
  */
 static uint_t zfs_metaslab_mem_limit = 25;
 
 /*
  * Force the per-metaslab range trees to use 64-bit integers to store
  * segments. Used for debugging purposes.
  */
 static const boolean_t zfs_metaslab_force_large_segs = B_FALSE;
 
 /*
  * By default we only store segments over a certain size in the size-sorted
  * metaslab trees (ms_allocatable_by_size and
  * ms_unflushed_frees_by_size). This dramatically reduces memory usage and
  * improves load and unload times at the cost of causing us to use slightly
  * larger segments than we would otherwise in some cases.
  */
 static const uint32_t metaslab_by_size_min_shift = 14;
 
 /*
  * If not set, we will first try normal allocation.  If that fails then
  * we will do a gang allocation.  If that fails then we will do a "try hard"
  * gang allocation.  If that fails then we will have a multi-layer gang
  * block.
  *
  * If set, we will first try normal allocation.  If that fails then
  * we will do a "try hard" allocation.  If that fails we will do a gang
  * allocation.  If that fails we will do a "try hard" gang allocation.  If
  * that fails then we will have a multi-layer gang block.
  */
 static int zfs_metaslab_try_hard_before_gang = B_FALSE;
 
 /*
  * When not trying hard, we only consider the best zfs_metaslab_find_max_tries
  * metaslabs.  This improves performance, especially when there are many
  * metaslabs per vdev and the allocation can't actually be satisfied (so we
  * would otherwise iterate all the metaslabs).  If there is a metaslab with a
  * worse weight but it can actually satisfy the allocation, we won't find it
  * until trying hard.  This may happen if the worse metaslab is not loaded
  * (and the true weight is better than we have calculated), or due to weight
  * bucketization.  E.g. we are looking for a 60K segment, and the best
  * metaslabs all have free segments in the 32-63K bucket, but the best
  * zfs_metaslab_find_max_tries metaslabs have ms_max_size <60KB, and a
  * subsequent metaslab has ms_max_size >60KB (but fewer segments in this
  * bucket, and therefore a lower weight).
  */
 static uint_t zfs_metaslab_find_max_tries = 100;
 
 static uint64_t metaslab_weight(metaslab_t *, boolean_t);
 static void metaslab_set_fragmentation(metaslab_t *, boolean_t);
 static void metaslab_free_impl(vdev_t *, uint64_t, uint64_t, boolean_t);
 static void metaslab_check_free_impl(vdev_t *, uint64_t, uint64_t);
 
 static void metaslab_passivate(metaslab_t *msp, uint64_t weight);
 static uint64_t metaslab_weight_from_range_tree(metaslab_t *msp);
 static void metaslab_flush_update(metaslab_t *, dmu_tx_t *);
 static unsigned int metaslab_idx_func(multilist_t *, void *);
 static void metaslab_evict(metaslab_t *, uint64_t);
 static void metaslab_rt_add(range_tree_t *rt, range_seg_t *rs, void *arg);
 kmem_cache_t *metaslab_alloc_trace_cache;
 
 typedef struct metaslab_stats {
 	kstat_named_t metaslabstat_trace_over_limit;
 	kstat_named_t metaslabstat_reload_tree;
 	kstat_named_t metaslabstat_too_many_tries;
 	kstat_named_t metaslabstat_try_hard;
 } metaslab_stats_t;
 
 static metaslab_stats_t metaslab_stats = {
 	{ "trace_over_limit",		KSTAT_DATA_UINT64 },
 	{ "reload_tree",		KSTAT_DATA_UINT64 },
 	{ "too_many_tries",		KSTAT_DATA_UINT64 },
 	{ "try_hard",			KSTAT_DATA_UINT64 },
 };
 
 #define	METASLABSTAT_BUMP(stat) \
 	atomic_inc_64(&metaslab_stats.stat.value.ui64);
 
 
 static kstat_t *metaslab_ksp;
 
 void
 metaslab_stat_init(void)
 {
 	ASSERT(metaslab_alloc_trace_cache == NULL);
 	metaslab_alloc_trace_cache = kmem_cache_create(
 	    "metaslab_alloc_trace_cache", sizeof (metaslab_alloc_trace_t),
 	    0, NULL, NULL, NULL, NULL, NULL, 0);
 	metaslab_ksp = kstat_create("zfs", 0, "metaslab_stats",
 	    "misc", KSTAT_TYPE_NAMED, sizeof (metaslab_stats) /
 	    sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
 	if (metaslab_ksp != NULL) {
 		metaslab_ksp->ks_data = &metaslab_stats;
 		kstat_install(metaslab_ksp);
 	}
 }
 
 void
 metaslab_stat_fini(void)
 {
 	if (metaslab_ksp != NULL) {
 		kstat_delete(metaslab_ksp);
 		metaslab_ksp = NULL;
 	}
 
 	kmem_cache_destroy(metaslab_alloc_trace_cache);
 	metaslab_alloc_trace_cache = NULL;
 }
 
 /*
  * ==========================================================================
  * Metaslab classes
  * ==========================================================================
  */
 metaslab_class_t *
 metaslab_class_create(spa_t *spa, const metaslab_ops_t *ops)
 {
 	metaslab_class_t *mc;
 
 	mc = kmem_zalloc(offsetof(metaslab_class_t,
 	    mc_allocator[spa->spa_alloc_count]), KM_SLEEP);
 
 	mc->mc_spa = spa;
 	mc->mc_ops = ops;
 	mutex_init(&mc->mc_lock, NULL, MUTEX_DEFAULT, NULL);
 	multilist_create(&mc->mc_metaslab_txg_list, sizeof (metaslab_t),
 	    offsetof(metaslab_t, ms_class_txg_node), metaslab_idx_func);
 	for (int i = 0; i < spa->spa_alloc_count; i++) {
 		metaslab_class_allocator_t *mca = &mc->mc_allocator[i];
 		mca->mca_rotor = NULL;
 		zfs_refcount_create_tracked(&mca->mca_alloc_slots);
 	}
 
 	return (mc);
 }
 
 void
 metaslab_class_destroy(metaslab_class_t *mc)
 {
 	spa_t *spa = mc->mc_spa;
 
 	ASSERT(mc->mc_alloc == 0);
 	ASSERT(mc->mc_deferred == 0);
 	ASSERT(mc->mc_space == 0);
 	ASSERT(mc->mc_dspace == 0);
 
 	for (int i = 0; i < spa->spa_alloc_count; i++) {
 		metaslab_class_allocator_t *mca = &mc->mc_allocator[i];
 		ASSERT(mca->mca_rotor == NULL);
 		zfs_refcount_destroy(&mca->mca_alloc_slots);
 	}
 	mutex_destroy(&mc->mc_lock);
 	multilist_destroy(&mc->mc_metaslab_txg_list);
 	kmem_free(mc, offsetof(metaslab_class_t,
 	    mc_allocator[spa->spa_alloc_count]));
 }
 
 int
 metaslab_class_validate(metaslab_class_t *mc)
 {
 	metaslab_group_t *mg;
 	vdev_t *vd;
 
 	/*
 	 * Must hold one of the spa_config locks.
 	 */
 	ASSERT(spa_config_held(mc->mc_spa, SCL_ALL, RW_READER) ||
 	    spa_config_held(mc->mc_spa, SCL_ALL, RW_WRITER));
 
 	if ((mg = mc->mc_allocator[0].mca_rotor) == NULL)
 		return (0);
 
 	do {
 		vd = mg->mg_vd;
 		ASSERT(vd->vdev_mg != NULL);
 		ASSERT3P(vd->vdev_top, ==, vd);
 		ASSERT3P(mg->mg_class, ==, mc);
 		ASSERT3P(vd->vdev_ops, !=, &vdev_hole_ops);
 	} while ((mg = mg->mg_next) != mc->mc_allocator[0].mca_rotor);
 
 	return (0);
 }
 
 static void
 metaslab_class_space_update(metaslab_class_t *mc, int64_t alloc_delta,
     int64_t defer_delta, int64_t space_delta, int64_t dspace_delta)
 {
 	atomic_add_64(&mc->mc_alloc, alloc_delta);
 	atomic_add_64(&mc->mc_deferred, defer_delta);
 	atomic_add_64(&mc->mc_space, space_delta);
 	atomic_add_64(&mc->mc_dspace, dspace_delta);
 }
 
 uint64_t
 metaslab_class_get_alloc(metaslab_class_t *mc)
 {
 	return (mc->mc_alloc);
 }
 
 uint64_t
 metaslab_class_get_deferred(metaslab_class_t *mc)
 {
 	return (mc->mc_deferred);
 }
 
 uint64_t
 metaslab_class_get_space(metaslab_class_t *mc)
 {
 	return (mc->mc_space);
 }
 
 uint64_t
 metaslab_class_get_dspace(metaslab_class_t *mc)
 {
 	return (spa_deflate(mc->mc_spa) ? mc->mc_dspace : mc->mc_space);
 }
 
 void
 metaslab_class_histogram_verify(metaslab_class_t *mc)
 {
 	spa_t *spa = mc->mc_spa;
 	vdev_t *rvd = spa->spa_root_vdev;
 	uint64_t *mc_hist;
 	int i;
 
 	if ((zfs_flags & ZFS_DEBUG_HISTOGRAM_VERIFY) == 0)
 		return;
 
 	mc_hist = kmem_zalloc(sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE,
 	    KM_SLEEP);
 
 	mutex_enter(&mc->mc_lock);
 	for (int c = 0; c < rvd->vdev_children; c++) {
 		vdev_t *tvd = rvd->vdev_child[c];
 		metaslab_group_t *mg = vdev_get_mg(tvd, mc);
 
 		/*
 		 * Skip any holes, uninitialized top-levels, or
 		 * vdevs that are not in this metalab class.
 		 */
 		if (!vdev_is_concrete(tvd) || tvd->vdev_ms_shift == 0 ||
 		    mg->mg_class != mc) {
 			continue;
 		}
 
 		IMPLY(mg == mg->mg_vd->vdev_log_mg,
 		    mc == spa_embedded_log_class(mg->mg_vd->vdev_spa));
 
 		for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++)
 			mc_hist[i] += mg->mg_histogram[i];
 	}
 
 	for (i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
 		VERIFY3U(mc_hist[i], ==, mc->mc_histogram[i]);
 	}
 
 	mutex_exit(&mc->mc_lock);
 	kmem_free(mc_hist, sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE);
 }
 
 /*
  * Calculate the metaslab class's fragmentation metric. The metric
  * is weighted based on the space contribution of each metaslab group.
  * The return value will be a number between 0 and 100 (inclusive), or
  * ZFS_FRAG_INVALID if the metric has not been set. See comment above the
  * zfs_frag_table for more information about the metric.
  */
 uint64_t
 metaslab_class_fragmentation(metaslab_class_t *mc)
 {
 	vdev_t *rvd = mc->mc_spa->spa_root_vdev;
 	uint64_t fragmentation = 0;
 
 	spa_config_enter(mc->mc_spa, SCL_VDEV, FTAG, RW_READER);
 
 	for (int c = 0; c < rvd->vdev_children; c++) {
 		vdev_t *tvd = rvd->vdev_child[c];
 		metaslab_group_t *mg = tvd->vdev_mg;
 
 		/*
 		 * Skip any holes, uninitialized top-levels,
 		 * or vdevs that are not in this metalab class.
 		 */
 		if (!vdev_is_concrete(tvd) || tvd->vdev_ms_shift == 0 ||
 		    mg->mg_class != mc) {
 			continue;
 		}
 
 		/*
 		 * If a metaslab group does not contain a fragmentation
 		 * metric then just bail out.
 		 */
 		if (mg->mg_fragmentation == ZFS_FRAG_INVALID) {
 			spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG);
 			return (ZFS_FRAG_INVALID);
 		}
 
 		/*
 		 * Determine how much this metaslab_group is contributing
 		 * to the overall pool fragmentation metric.
 		 */
 		fragmentation += mg->mg_fragmentation *
 		    metaslab_group_get_space(mg);
 	}
 	fragmentation /= metaslab_class_get_space(mc);
 
 	ASSERT3U(fragmentation, <=, 100);
 	spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG);
 	return (fragmentation);
 }
 
 /*
  * Calculate the amount of expandable space that is available in
  * this metaslab class. If a device is expanded then its expandable
  * space will be the amount of allocatable space that is currently not
  * part of this metaslab class.
  */
 uint64_t
 metaslab_class_expandable_space(metaslab_class_t *mc)
 {
 	vdev_t *rvd = mc->mc_spa->spa_root_vdev;
 	uint64_t space = 0;
 
 	spa_config_enter(mc->mc_spa, SCL_VDEV, FTAG, RW_READER);
 	for (int c = 0; c < rvd->vdev_children; c++) {
 		vdev_t *tvd = rvd->vdev_child[c];
 		metaslab_group_t *mg = tvd->vdev_mg;
 
 		if (!vdev_is_concrete(tvd) || tvd->vdev_ms_shift == 0 ||
 		    mg->mg_class != mc) {
 			continue;
 		}
 
 		/*
 		 * Calculate if we have enough space to add additional
 		 * metaslabs. We report the expandable space in terms
 		 * of the metaslab size since that's the unit of expansion.
 		 */
 		space += P2ALIGN_TYPED(tvd->vdev_max_asize - tvd->vdev_asize,
 		    1ULL << tvd->vdev_ms_shift, uint64_t);
 	}
 	spa_config_exit(mc->mc_spa, SCL_VDEV, FTAG);
 	return (space);
 }
 
 void
 metaslab_class_evict_old(metaslab_class_t *mc, uint64_t txg)
 {
 	multilist_t *ml = &mc->mc_metaslab_txg_list;
 	hrtime_t now = gethrtime();
 	for (int i = 0; i < multilist_get_num_sublists(ml); i++) {
 		multilist_sublist_t *mls = multilist_sublist_lock_idx(ml, i);
 		metaslab_t *msp = multilist_sublist_head(mls);
 		multilist_sublist_unlock(mls);
 		while (msp != NULL) {
 			mutex_enter(&msp->ms_lock);
 
 			/*
 			 * If the metaslab has been removed from the list
 			 * (which could happen if we were at the memory limit
 			 * and it was evicted during this loop), then we can't
 			 * proceed and we should restart the sublist.
 			 */
 			if (!multilist_link_active(&msp->ms_class_txg_node)) {
 				mutex_exit(&msp->ms_lock);
 				i--;
 				break;
 			}
 			mls = multilist_sublist_lock_idx(ml, i);
 			metaslab_t *next_msp = multilist_sublist_next(mls, msp);
 			multilist_sublist_unlock(mls);
 			if (txg >
 			    msp->ms_selected_txg + metaslab_unload_delay &&
 			    now > msp->ms_selected_time +
 			    MSEC2NSEC(metaslab_unload_delay_ms) &&
 			    (msp->ms_allocator == -1 ||
 			    !metaslab_preload_enabled)) {
 				metaslab_evict(msp, txg);
 			} else {
 				/*
 				 * Once we've hit a metaslab selected too
 				 * recently to evict, we're done evicting for
 				 * now.
 				 */
 				mutex_exit(&msp->ms_lock);
 				break;
 			}
 			mutex_exit(&msp->ms_lock);
 			msp = next_msp;
 		}
 	}
 }
 
 static int
 metaslab_compare(const void *x1, const void *x2)
 {
 	const metaslab_t *m1 = (const metaslab_t *)x1;
 	const metaslab_t *m2 = (const metaslab_t *)x2;
 
 	int sort1 = 0;
 	int sort2 = 0;
 	if (m1->ms_allocator != -1 && m1->ms_primary)
 		sort1 = 1;
 	else if (m1->ms_allocator != -1 && !m1->ms_primary)
 		sort1 = 2;
 	if (m2->ms_allocator != -1 && m2->ms_primary)
 		sort2 = 1;
 	else if (m2->ms_allocator != -1 && !m2->ms_primary)
 		sort2 = 2;
 
 	/*
 	 * Sort inactive metaslabs first, then primaries, then secondaries. When
 	 * selecting a metaslab to allocate from, an allocator first tries its
 	 * primary, then secondary active metaslab. If it doesn't have active
 	 * metaslabs, or can't allocate from them, it searches for an inactive
 	 * metaslab to activate. If it can't find a suitable one, it will steal
 	 * a primary or secondary metaslab from another allocator.
 	 */
 	if (sort1 < sort2)
 		return (-1);
 	if (sort1 > sort2)
 		return (1);
 
 	int cmp = TREE_CMP(m2->ms_weight, m1->ms_weight);
 	if (likely(cmp))
 		return (cmp);
 
 	IMPLY(TREE_CMP(m1->ms_start, m2->ms_start) == 0, m1 == m2);
 
 	return (TREE_CMP(m1->ms_start, m2->ms_start));
 }
 
 /*
  * ==========================================================================
  * Metaslab groups
  * ==========================================================================
  */
 /*
  * Update the allocatable flag and the metaslab group's capacity.
  * The allocatable flag is set to true if the capacity is below
  * the zfs_mg_noalloc_threshold or has a fragmentation value that is
  * greater than zfs_mg_fragmentation_threshold. If a metaslab group
  * transitions from allocatable to non-allocatable or vice versa then the
  * metaslab group's class is updated to reflect the transition.
  */
 static void
 metaslab_group_alloc_update(metaslab_group_t *mg)
 {
 	vdev_t *vd = mg->mg_vd;
 	metaslab_class_t *mc = mg->mg_class;
 	vdev_stat_t *vs = &vd->vdev_stat;
 	boolean_t was_allocatable;
 	boolean_t was_initialized;
 
 	ASSERT(vd == vd->vdev_top);
 	ASSERT3U(spa_config_held(mc->mc_spa, SCL_ALLOC, RW_READER), ==,
 	    SCL_ALLOC);
 
 	mutex_enter(&mg->mg_lock);
 	was_allocatable = mg->mg_allocatable;
 	was_initialized = mg->mg_initialized;
 
 	mg->mg_free_capacity = ((vs->vs_space - vs->vs_alloc) * 100) /
 	    (vs->vs_space + 1);
 
 	mutex_enter(&mc->mc_lock);
 
 	/*
 	 * If the metaslab group was just added then it won't
 	 * have any space until we finish syncing out this txg.
 	 * At that point we will consider it initialized and available
 	 * for allocations.  We also don't consider non-activated
 	 * metaslab groups (e.g. vdevs that are in the middle of being removed)
 	 * to be initialized, because they can't be used for allocation.
 	 */
 	mg->mg_initialized = metaslab_group_initialized(mg);
 	if (!was_initialized && mg->mg_initialized) {
 		mc->mc_groups++;
 	} else if (was_initialized && !mg->mg_initialized) {
 		ASSERT3U(mc->mc_groups, >, 0);
 		mc->mc_groups--;
 	}
 	if (mg->mg_initialized)
 		mg->mg_no_free_space = B_FALSE;
 
 	/*
 	 * A metaslab group is considered allocatable if it has plenty
 	 * of free space or is not heavily fragmented. We only take
 	 * fragmentation into account if the metaslab group has a valid
 	 * fragmentation metric (i.e. a value between 0 and 100).
 	 */
 	mg->mg_allocatable = (mg->mg_activation_count > 0 &&
 	    mg->mg_free_capacity > zfs_mg_noalloc_threshold &&
 	    (mg->mg_fragmentation == ZFS_FRAG_INVALID ||
 	    mg->mg_fragmentation <= zfs_mg_fragmentation_threshold));
 
 	/*
 	 * The mc_alloc_groups maintains a count of the number of
 	 * groups in this metaslab class that are still above the
 	 * zfs_mg_noalloc_threshold. This is used by the allocating
 	 * threads to determine if they should avoid allocations to
 	 * a given group. The allocator will avoid allocations to a group
 	 * if that group has reached or is below the zfs_mg_noalloc_threshold
 	 * and there are still other groups that are above the threshold.
 	 * When a group transitions from allocatable to non-allocatable or
 	 * vice versa we update the metaslab class to reflect that change.
 	 * When the mc_alloc_groups value drops to 0 that means that all
 	 * groups have reached the zfs_mg_noalloc_threshold making all groups
 	 * eligible for allocations. This effectively means that all devices
 	 * are balanced again.
 	 */
 	if (was_allocatable && !mg->mg_allocatable)
 		mc->mc_alloc_groups--;
 	else if (!was_allocatable && mg->mg_allocatable)
 		mc->mc_alloc_groups++;
 	mutex_exit(&mc->mc_lock);
 
 	mutex_exit(&mg->mg_lock);
 }
 
 int
 metaslab_sort_by_flushed(const void *va, const void *vb)
 {
 	const metaslab_t *a = va;
 	const metaslab_t *b = vb;
 
 	int cmp = TREE_CMP(a->ms_unflushed_txg, b->ms_unflushed_txg);
 	if (likely(cmp))
 		return (cmp);
 
 	uint64_t a_vdev_id = a->ms_group->mg_vd->vdev_id;
 	uint64_t b_vdev_id = b->ms_group->mg_vd->vdev_id;
 	cmp = TREE_CMP(a_vdev_id, b_vdev_id);
 	if (cmp)
 		return (cmp);
 
 	return (TREE_CMP(a->ms_id, b->ms_id));
 }
 
 metaslab_group_t *
 metaslab_group_create(metaslab_class_t *mc, vdev_t *vd, int allocators)
 {
 	metaslab_group_t *mg;
 
 	mg = kmem_zalloc(offsetof(metaslab_group_t,
 	    mg_allocator[allocators]), KM_SLEEP);
 	mutex_init(&mg->mg_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&mg->mg_ms_disabled_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&mg->mg_ms_disabled_cv, NULL, CV_DEFAULT, NULL);
 	avl_create(&mg->mg_metaslab_tree, metaslab_compare,
 	    sizeof (metaslab_t), offsetof(metaslab_t, ms_group_node));
 	mg->mg_vd = vd;
 	mg->mg_class = mc;
 	mg->mg_activation_count = 0;
 	mg->mg_initialized = B_FALSE;
 	mg->mg_no_free_space = B_TRUE;
 	mg->mg_allocators = allocators;
 
 	for (int i = 0; i < allocators; i++) {
 		metaslab_group_allocator_t *mga = &mg->mg_allocator[i];
 		zfs_refcount_create_tracked(&mga->mga_alloc_queue_depth);
 	}
 
 	return (mg);
 }
 
 void
 metaslab_group_destroy(metaslab_group_t *mg)
 {
 	ASSERT(mg->mg_prev == NULL);
 	ASSERT(mg->mg_next == NULL);
 	/*
 	 * We may have gone below zero with the activation count
 	 * either because we never activated in the first place or
 	 * because we're done, and possibly removing the vdev.
 	 */
 	ASSERT(mg->mg_activation_count <= 0);
 
 	avl_destroy(&mg->mg_metaslab_tree);
 	mutex_destroy(&mg->mg_lock);
 	mutex_destroy(&mg->mg_ms_disabled_lock);
 	cv_destroy(&mg->mg_ms_disabled_cv);
 
 	for (int i = 0; i < mg->mg_allocators; i++) {
 		metaslab_group_allocator_t *mga = &mg->mg_allocator[i];
 		zfs_refcount_destroy(&mga->mga_alloc_queue_depth);
 	}
 	kmem_free(mg, offsetof(metaslab_group_t,
 	    mg_allocator[mg->mg_allocators]));
 }
 
 void
 metaslab_group_activate(metaslab_group_t *mg)
 {
 	metaslab_class_t *mc = mg->mg_class;
 	spa_t *spa = mc->mc_spa;
 	metaslab_group_t *mgprev, *mgnext;
 
 	ASSERT3U(spa_config_held(spa, SCL_ALLOC, RW_WRITER), !=, 0);
 
 	ASSERT(mg->mg_prev == NULL);
 	ASSERT(mg->mg_next == NULL);
 	ASSERT(mg->mg_activation_count <= 0);
 
 	if (++mg->mg_activation_count <= 0)
 		return;
 
 	mg->mg_aliquot = metaslab_aliquot * MAX(1,
 	    vdev_get_ndisks(mg->mg_vd) - vdev_get_nparity(mg->mg_vd));
 	metaslab_group_alloc_update(mg);
 
 	if ((mgprev = mc->mc_allocator[0].mca_rotor) == NULL) {
 		mg->mg_prev = mg;
 		mg->mg_next = mg;
 	} else {
 		mgnext = mgprev->mg_next;
 		mg->mg_prev = mgprev;
 		mg->mg_next = mgnext;
 		mgprev->mg_next = mg;
 		mgnext->mg_prev = mg;
 	}
 	for (int i = 0; i < spa->spa_alloc_count; i++) {
 		mc->mc_allocator[i].mca_rotor = mg;
 		mg = mg->mg_next;
 	}
 }
 
 /*
  * Passivate a metaslab group and remove it from the allocation rotor.
  * Callers must hold both the SCL_ALLOC and SCL_ZIO lock prior to passivating
  * a metaslab group. This function will momentarily drop spa_config_locks
  * that are lower than the SCL_ALLOC lock (see comment below).
  */
 void
 metaslab_group_passivate(metaslab_group_t *mg)
 {
 	metaslab_class_t *mc = mg->mg_class;
 	spa_t *spa = mc->mc_spa;
 	metaslab_group_t *mgprev, *mgnext;
 	int locks = spa_config_held(spa, SCL_ALL, RW_WRITER);
 
 	ASSERT3U(spa_config_held(spa, SCL_ALLOC | SCL_ZIO, RW_WRITER), ==,
 	    (SCL_ALLOC | SCL_ZIO));
 
 	if (--mg->mg_activation_count != 0) {
 		for (int i = 0; i < spa->spa_alloc_count; i++)
 			ASSERT(mc->mc_allocator[i].mca_rotor != mg);
 		ASSERT(mg->mg_prev == NULL);
 		ASSERT(mg->mg_next == NULL);
 		ASSERT(mg->mg_activation_count < 0);
 		return;
 	}
 
 	/*
 	 * The spa_config_lock is an array of rwlocks, ordered as
 	 * follows (from highest to lowest):
 	 *	SCL_CONFIG > SCL_STATE > SCL_L2ARC > SCL_ALLOC >
 	 *	SCL_ZIO > SCL_FREE > SCL_VDEV
 	 * (For more information about the spa_config_lock see spa_misc.c)
 	 * The higher the lock, the broader its coverage. When we passivate
 	 * a metaslab group, we must hold both the SCL_ALLOC and the SCL_ZIO
 	 * config locks. However, the metaslab group's taskq might be trying
 	 * to preload metaslabs so we must drop the SCL_ZIO lock and any
 	 * lower locks to allow the I/O to complete. At a minimum,
 	 * we continue to hold the SCL_ALLOC lock, which prevents any future
 	 * allocations from taking place and any changes to the vdev tree.
 	 */
 	spa_config_exit(spa, locks & ~(SCL_ZIO - 1), spa);
 	taskq_wait_outstanding(spa->spa_metaslab_taskq, 0);
 	spa_config_enter(spa, locks & ~(SCL_ZIO - 1), spa, RW_WRITER);
 	metaslab_group_alloc_update(mg);
 	for (int i = 0; i < mg->mg_allocators; i++) {
 		metaslab_group_allocator_t *mga = &mg->mg_allocator[i];
 		metaslab_t *msp = mga->mga_primary;
 		if (msp != NULL) {
 			mutex_enter(&msp->ms_lock);
 			metaslab_passivate(msp,
 			    metaslab_weight_from_range_tree(msp));
 			mutex_exit(&msp->ms_lock);
 		}
 		msp = mga->mga_secondary;
 		if (msp != NULL) {
 			mutex_enter(&msp->ms_lock);
 			metaslab_passivate(msp,
 			    metaslab_weight_from_range_tree(msp));
 			mutex_exit(&msp->ms_lock);
 		}
 	}
 
 	mgprev = mg->mg_prev;
 	mgnext = mg->mg_next;
 
 	if (mg == mgnext) {
 		mgnext = NULL;
 	} else {
 		mgprev->mg_next = mgnext;
 		mgnext->mg_prev = mgprev;
 	}
 	for (int i = 0; i < spa->spa_alloc_count; i++) {
 		if (mc->mc_allocator[i].mca_rotor == mg)
 			mc->mc_allocator[i].mca_rotor = mgnext;
 	}
 
 	mg->mg_prev = NULL;
 	mg->mg_next = NULL;
 }
 
 boolean_t
 metaslab_group_initialized(metaslab_group_t *mg)
 {
 	vdev_t *vd = mg->mg_vd;
 	vdev_stat_t *vs = &vd->vdev_stat;
 
 	return (vs->vs_space != 0 && mg->mg_activation_count > 0);
 }
 
 uint64_t
 metaslab_group_get_space(metaslab_group_t *mg)
 {
 	/*
 	 * Note that the number of nodes in mg_metaslab_tree may be one less
 	 * than vdev_ms_count, due to the embedded log metaslab.
 	 */
 	mutex_enter(&mg->mg_lock);
 	uint64_t ms_count = avl_numnodes(&mg->mg_metaslab_tree);
 	mutex_exit(&mg->mg_lock);
 	return ((1ULL << mg->mg_vd->vdev_ms_shift) * ms_count);
 }
 
 void
 metaslab_group_histogram_verify(metaslab_group_t *mg)
 {
 	uint64_t *mg_hist;
 	avl_tree_t *t = &mg->mg_metaslab_tree;
 	uint64_t ashift = mg->mg_vd->vdev_ashift;
 
 	if ((zfs_flags & ZFS_DEBUG_HISTOGRAM_VERIFY) == 0)
 		return;
 
 	mg_hist = kmem_zalloc(sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE,
 	    KM_SLEEP);
 
 	ASSERT3U(RANGE_TREE_HISTOGRAM_SIZE, >=,
 	    SPACE_MAP_HISTOGRAM_SIZE + ashift);
 
 	mutex_enter(&mg->mg_lock);
 	for (metaslab_t *msp = avl_first(t);
 	    msp != NULL; msp = AVL_NEXT(t, msp)) {
 		VERIFY3P(msp->ms_group, ==, mg);
 		/* skip if not active */
 		if (msp->ms_sm == NULL)
 			continue;
 
 		for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
 			mg_hist[i + ashift] +=
 			    msp->ms_sm->sm_phys->smp_histogram[i];
 		}
 	}
 
 	for (int i = 0; i < RANGE_TREE_HISTOGRAM_SIZE; i ++)
 		VERIFY3U(mg_hist[i], ==, mg->mg_histogram[i]);
 
 	mutex_exit(&mg->mg_lock);
 
 	kmem_free(mg_hist, sizeof (uint64_t) * RANGE_TREE_HISTOGRAM_SIZE);
 }
 
 static void
 metaslab_group_histogram_add(metaslab_group_t *mg, metaslab_t *msp)
 {
 	metaslab_class_t *mc = mg->mg_class;
 	uint64_t ashift = mg->mg_vd->vdev_ashift;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	if (msp->ms_sm == NULL)
 		return;
 
 	mutex_enter(&mg->mg_lock);
 	mutex_enter(&mc->mc_lock);
 	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
 		IMPLY(mg == mg->mg_vd->vdev_log_mg,
 		    mc == spa_embedded_log_class(mg->mg_vd->vdev_spa));
 		mg->mg_histogram[i + ashift] +=
 		    msp->ms_sm->sm_phys->smp_histogram[i];
 		mc->mc_histogram[i + ashift] +=
 		    msp->ms_sm->sm_phys->smp_histogram[i];
 	}
 	mutex_exit(&mc->mc_lock);
 	mutex_exit(&mg->mg_lock);
 }
 
 void
 metaslab_group_histogram_remove(metaslab_group_t *mg, metaslab_t *msp)
 {
 	metaslab_class_t *mc = mg->mg_class;
 	uint64_t ashift = mg->mg_vd->vdev_ashift;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	if (msp->ms_sm == NULL)
 		return;
 
 	mutex_enter(&mg->mg_lock);
 	mutex_enter(&mc->mc_lock);
 	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
 		ASSERT3U(mg->mg_histogram[i + ashift], >=,
 		    msp->ms_sm->sm_phys->smp_histogram[i]);
 		ASSERT3U(mc->mc_histogram[i + ashift], >=,
 		    msp->ms_sm->sm_phys->smp_histogram[i]);
 		IMPLY(mg == mg->mg_vd->vdev_log_mg,
 		    mc == spa_embedded_log_class(mg->mg_vd->vdev_spa));
 
 		mg->mg_histogram[i + ashift] -=
 		    msp->ms_sm->sm_phys->smp_histogram[i];
 		mc->mc_histogram[i + ashift] -=
 		    msp->ms_sm->sm_phys->smp_histogram[i];
 	}
 	mutex_exit(&mc->mc_lock);
 	mutex_exit(&mg->mg_lock);
 }
 
 static void
 metaslab_group_add(metaslab_group_t *mg, metaslab_t *msp)
 {
 	ASSERT(msp->ms_group == NULL);
 	mutex_enter(&mg->mg_lock);
 	msp->ms_group = mg;
 	msp->ms_weight = 0;
 	avl_add(&mg->mg_metaslab_tree, msp);
 	mutex_exit(&mg->mg_lock);
 
 	mutex_enter(&msp->ms_lock);
 	metaslab_group_histogram_add(mg, msp);
 	mutex_exit(&msp->ms_lock);
 }
 
 static void
 metaslab_group_remove(metaslab_group_t *mg, metaslab_t *msp)
 {
 	mutex_enter(&msp->ms_lock);
 	metaslab_group_histogram_remove(mg, msp);
 	mutex_exit(&msp->ms_lock);
 
 	mutex_enter(&mg->mg_lock);
 	ASSERT(msp->ms_group == mg);
 	avl_remove(&mg->mg_metaslab_tree, msp);
 
 	metaslab_class_t *mc = msp->ms_group->mg_class;
 	multilist_sublist_t *mls =
 	    multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
 	if (multilist_link_active(&msp->ms_class_txg_node))
 		multilist_sublist_remove(mls, msp);
 	multilist_sublist_unlock(mls);
 
 	msp->ms_group = NULL;
 	mutex_exit(&mg->mg_lock);
 }
 
 static void
 metaslab_group_sort_impl(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	ASSERT(MUTEX_HELD(&mg->mg_lock));
 	ASSERT(msp->ms_group == mg);
 
 	avl_remove(&mg->mg_metaslab_tree, msp);
 	msp->ms_weight = weight;
 	avl_add(&mg->mg_metaslab_tree, msp);
 
 }
 
 static void
 metaslab_group_sort(metaslab_group_t *mg, metaslab_t *msp, uint64_t weight)
 {
 	/*
 	 * Although in principle the weight can be any value, in
 	 * practice we do not use values in the range [1, 511].
 	 */
 	ASSERT(weight >= SPA_MINBLOCKSIZE || weight == 0);
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	mutex_enter(&mg->mg_lock);
 	metaslab_group_sort_impl(mg, msp, weight);
 	mutex_exit(&mg->mg_lock);
 }
 
 /*
  * Calculate the fragmentation for a given metaslab group. We can use
  * a simple average here since all metaslabs within the group must have
  * the same size. The return value will be a value between 0 and 100
  * (inclusive), or ZFS_FRAG_INVALID if less than half of the metaslab in this
  * group have a fragmentation metric.
  */
 uint64_t
 metaslab_group_fragmentation(metaslab_group_t *mg)
 {
 	vdev_t *vd = mg->mg_vd;
 	uint64_t fragmentation = 0;
 	uint64_t valid_ms = 0;
 
 	for (int m = 0; m < vd->vdev_ms_count; m++) {
 		metaslab_t *msp = vd->vdev_ms[m];
 
 		if (msp->ms_fragmentation == ZFS_FRAG_INVALID)
 			continue;
 		if (msp->ms_group != mg)
 			continue;
 
 		valid_ms++;
 		fragmentation += msp->ms_fragmentation;
 	}
 
 	if (valid_ms <= mg->mg_vd->vdev_ms_count / 2)
 		return (ZFS_FRAG_INVALID);
 
 	fragmentation /= valid_ms;
 	ASSERT3U(fragmentation, <=, 100);
 	return (fragmentation);
 }
 
 /*
  * Determine if a given metaslab group should skip allocations. A metaslab
  * group should avoid allocations if its free capacity is less than the
  * zfs_mg_noalloc_threshold or its fragmentation metric is greater than
  * zfs_mg_fragmentation_threshold and there is at least one metaslab group
  * that can still handle allocations. If the allocation throttle is enabled
  * then we skip allocations to devices that have reached their maximum
  * allocation queue depth unless the selected metaslab group is the only
  * eligible group remaining.
  */
 static boolean_t
 metaslab_group_allocatable(metaslab_group_t *mg, metaslab_group_t *rotor,
     int flags, uint64_t psize, int allocator, int d)
 {
 	spa_t *spa = mg->mg_vd->vdev_spa;
 	metaslab_class_t *mc = mg->mg_class;
 
 	/*
 	 * We can only consider skipping this metaslab group if it's
 	 * in the normal metaslab class and there are other metaslab
 	 * groups to select from. Otherwise, we always consider it eligible
 	 * for allocations.
 	 */
 	if ((mc != spa_normal_class(spa) &&
 	    mc != spa_special_class(spa) &&
 	    mc != spa_dedup_class(spa)) ||
 	    mc->mc_groups <= 1)
 		return (B_TRUE);
 
 	/*
 	 * If the metaslab group's mg_allocatable flag is set (see comments
 	 * in metaslab_group_alloc_update() for more information) and
 	 * the allocation throttle is disabled then allow allocations to this
 	 * device. However, if the allocation throttle is enabled then
 	 * check if we have reached our allocation limit (mga_alloc_queue_depth)
 	 * to determine if we should allow allocations to this metaslab group.
 	 * If all metaslab groups are no longer considered allocatable
 	 * (mc_alloc_groups == 0) or we're trying to allocate the smallest
 	 * gang block size then we allow allocations on this metaslab group
 	 * regardless of the mg_allocatable or throttle settings.
 	 */
 	if (mg->mg_allocatable) {
 		metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
 		int64_t qdepth;
 		uint64_t qmax = mga->mga_cur_max_alloc_queue_depth;
 
 		if (!mc->mc_alloc_throttle_enabled)
 			return (B_TRUE);
 
 		/*
 		 * If this metaslab group does not have any free space, then
 		 * there is no point in looking further.
 		 */
 		if (mg->mg_no_free_space)
 			return (B_FALSE);
 
 		/*
 		 * Some allocations (e.g., those coming from device removal
 		 * where the * allocations are not even counted in the
 		 * metaslab * allocation queues) are allowed to bypass
 		 * the throttle.
 		 */
 		if (flags & METASLAB_DONT_THROTTLE)
 			return (B_TRUE);
 
 		/*
 		 * Relax allocation throttling for ditto blocks.  Due to
 		 * random imbalances in allocation it tends to push copies
 		 * to one vdev, that looks a bit better at the moment.
 		 */
 		qmax = qmax * (4 + d) / 4;
 
 		qdepth = zfs_refcount_count(&mga->mga_alloc_queue_depth);
 
 		/*
 		 * If this metaslab group is below its qmax or it's
 		 * the only allocatable metaslab group, then attempt
 		 * to allocate from it.
 		 */
 		if (qdepth < qmax || mc->mc_alloc_groups == 1)
 			return (B_TRUE);
 		ASSERT3U(mc->mc_alloc_groups, >, 1);
 
 		/*
 		 * Since this metaslab group is at or over its qmax, we
 		 * need to determine if there are metaslab groups after this
 		 * one that might be able to handle this allocation. This is
 		 * racy since we can't hold the locks for all metaslab
 		 * groups at the same time when we make this check.
 		 */
 		for (metaslab_group_t *mgp = mg->mg_next;
 		    mgp != rotor; mgp = mgp->mg_next) {
 			metaslab_group_allocator_t *mgap =
 			    &mgp->mg_allocator[allocator];
 			qmax = mgap->mga_cur_max_alloc_queue_depth;
 			qmax = qmax * (4 + d) / 4;
 			qdepth =
 			    zfs_refcount_count(&mgap->mga_alloc_queue_depth);
 
 			/*
 			 * If there is another metaslab group that
 			 * might be able to handle the allocation, then
 			 * we return false so that we skip this group.
 			 */
 			if (qdepth < qmax && !mgp->mg_no_free_space)
 				return (B_FALSE);
 		}
 
 		/*
 		 * We didn't find another group to handle the allocation
 		 * so we can't skip this metaslab group even though
 		 * we are at or over our qmax.
 		 */
 		return (B_TRUE);
 
 	} else if (mc->mc_alloc_groups == 0 || psize == SPA_MINBLOCKSIZE) {
 		return (B_TRUE);
 	}
 	return (B_FALSE);
 }
 
 /*
  * ==========================================================================
  * Range tree callbacks
  * ==========================================================================
  */
 
 /*
  * Comparison function for the private size-ordered tree using 32-bit
  * ranges. Tree is sorted by size, larger sizes at the end of the tree.
  */
 __attribute__((always_inline)) inline
 static int
 metaslab_rangesize32_compare(const void *x1, const void *x2)
 {
 	const range_seg32_t *r1 = x1;
 	const range_seg32_t *r2 = x2;
 
 	uint64_t rs_size1 = r1->rs_end - r1->rs_start;
 	uint64_t rs_size2 = r2->rs_end - r2->rs_start;
 
 	int cmp = TREE_CMP(rs_size1, rs_size2);
 
 	return (cmp + !cmp * TREE_CMP(r1->rs_start, r2->rs_start));
 }
 
 /*
  * Comparison function for the private size-ordered tree using 64-bit
  * ranges. Tree is sorted by size, larger sizes at the end of the tree.
  */
 __attribute__((always_inline)) inline
 static int
 metaslab_rangesize64_compare(const void *x1, const void *x2)
 {
 	const range_seg64_t *r1 = x1;
 	const range_seg64_t *r2 = x2;
 
 	uint64_t rs_size1 = r1->rs_end - r1->rs_start;
 	uint64_t rs_size2 = r2->rs_end - r2->rs_start;
 
 	int cmp = TREE_CMP(rs_size1, rs_size2);
 
 	return (cmp + !cmp * TREE_CMP(r1->rs_start, r2->rs_start));
 }
 
 typedef struct metaslab_rt_arg {
 	zfs_btree_t *mra_bt;
 	uint32_t mra_floor_shift;
 } metaslab_rt_arg_t;
 
 struct mssa_arg {
 	range_tree_t *rt;
 	metaslab_rt_arg_t *mra;
 };
 
 static void
 metaslab_size_sorted_add(void *arg, uint64_t start, uint64_t size)
 {
 	struct mssa_arg *mssap = arg;
 	range_tree_t *rt = mssap->rt;
 	metaslab_rt_arg_t *mrap = mssap->mra;
 	range_seg_max_t seg = {0};
 	rs_set_start(&seg, rt, start);
 	rs_set_end(&seg, rt, start + size);
 	metaslab_rt_add(rt, &seg, mrap);
 }
 
 static void
 metaslab_size_tree_full_load(range_tree_t *rt)
 {
 	metaslab_rt_arg_t *mrap = rt->rt_arg;
 	METASLABSTAT_BUMP(metaslabstat_reload_tree);
 	ASSERT0(zfs_btree_numnodes(mrap->mra_bt));
 	mrap->mra_floor_shift = 0;
 	struct mssa_arg arg = {0};
 	arg.rt = rt;
 	arg.mra = mrap;
 	range_tree_walk(rt, metaslab_size_sorted_add, &arg);
 }
 
 
 ZFS_BTREE_FIND_IN_BUF_FUNC(metaslab_rt_find_rangesize32_in_buf,
     range_seg32_t, metaslab_rangesize32_compare)
 
 ZFS_BTREE_FIND_IN_BUF_FUNC(metaslab_rt_find_rangesize64_in_buf,
     range_seg64_t, metaslab_rangesize64_compare)
 
 /*
  * Create any block allocator specific components. The current allocators
  * rely on using both a size-ordered range_tree_t and an array of uint64_t's.
  */
 static void
 metaslab_rt_create(range_tree_t *rt, void *arg)
 {
 	metaslab_rt_arg_t *mrap = arg;
 	zfs_btree_t *size_tree = mrap->mra_bt;
 
 	size_t size;
 	int (*compare) (const void *, const void *);
 	bt_find_in_buf_f bt_find;
 	switch (rt->rt_type) {
 	case RANGE_SEG32:
 		size = sizeof (range_seg32_t);
 		compare = metaslab_rangesize32_compare;
 		bt_find = metaslab_rt_find_rangesize32_in_buf;
 		break;
 	case RANGE_SEG64:
 		size = sizeof (range_seg64_t);
 		compare = metaslab_rangesize64_compare;
 		bt_find = metaslab_rt_find_rangesize64_in_buf;
 		break;
 	default:
 		panic("Invalid range seg type %d", rt->rt_type);
 	}
 	zfs_btree_create(size_tree, compare, bt_find, size);
 	mrap->mra_floor_shift = metaslab_by_size_min_shift;
 }
 
 static void
 metaslab_rt_destroy(range_tree_t *rt, void *arg)
 {
 	(void) rt;
 	metaslab_rt_arg_t *mrap = arg;
 	zfs_btree_t *size_tree = mrap->mra_bt;
 
 	zfs_btree_destroy(size_tree);
 	kmem_free(mrap, sizeof (*mrap));
 }
 
 static void
 metaslab_rt_add(range_tree_t *rt, range_seg_t *rs, void *arg)
 {
 	metaslab_rt_arg_t *mrap = arg;
 	zfs_btree_t *size_tree = mrap->mra_bt;
 
 	if (rs_get_end(rs, rt) - rs_get_start(rs, rt) <
 	    (1ULL << mrap->mra_floor_shift))
 		return;
 
 	zfs_btree_add(size_tree, rs);
 }
 
 static void
 metaslab_rt_remove(range_tree_t *rt, range_seg_t *rs, void *arg)
 {
 	metaslab_rt_arg_t *mrap = arg;
 	zfs_btree_t *size_tree = mrap->mra_bt;
 
 	if (rs_get_end(rs, rt) - rs_get_start(rs, rt) < (1ULL <<
 	    mrap->mra_floor_shift))
 		return;
 
 	zfs_btree_remove(size_tree, rs);
 }
 
 static void
 metaslab_rt_vacate(range_tree_t *rt, void *arg)
 {
 	metaslab_rt_arg_t *mrap = arg;
 	zfs_btree_t *size_tree = mrap->mra_bt;
 	zfs_btree_clear(size_tree);
 	zfs_btree_destroy(size_tree);
 
 	metaslab_rt_create(rt, arg);
 }
 
 static const range_tree_ops_t metaslab_rt_ops = {
 	.rtop_create = metaslab_rt_create,
 	.rtop_destroy = metaslab_rt_destroy,
 	.rtop_add = metaslab_rt_add,
 	.rtop_remove = metaslab_rt_remove,
 	.rtop_vacate = metaslab_rt_vacate
 };
 
 /*
  * ==========================================================================
  * Common allocator routines
  * ==========================================================================
  */
 
 /*
  * Return the maximum contiguous segment within the metaslab.
  */
 uint64_t
 metaslab_largest_allocatable(metaslab_t *msp)
 {
 	zfs_btree_t *t = &msp->ms_allocatable_by_size;
 	range_seg_t *rs;
 
 	if (t == NULL)
 		return (0);
 	if (zfs_btree_numnodes(t) == 0)
 		metaslab_size_tree_full_load(msp->ms_allocatable);
 
 	rs = zfs_btree_last(t, NULL);
 	if (rs == NULL)
 		return (0);
 
 	return (rs_get_end(rs, msp->ms_allocatable) - rs_get_start(rs,
 	    msp->ms_allocatable));
 }
 
 /*
  * Return the maximum contiguous segment within the unflushed frees of this
  * metaslab.
  */
 static uint64_t
 metaslab_largest_unflushed_free(metaslab_t *msp)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	if (msp->ms_unflushed_frees == NULL)
 		return (0);
 
 	if (zfs_btree_numnodes(&msp->ms_unflushed_frees_by_size) == 0)
 		metaslab_size_tree_full_load(msp->ms_unflushed_frees);
 	range_seg_t *rs = zfs_btree_last(&msp->ms_unflushed_frees_by_size,
 	    NULL);
 	if (rs == NULL)
 		return (0);
 
 	/*
 	 * When a range is freed from the metaslab, that range is added to
 	 * both the unflushed frees and the deferred frees. While the block
 	 * will eventually be usable, if the metaslab were loaded the range
 	 * would not be added to the ms_allocatable tree until TXG_DEFER_SIZE
 	 * txgs had passed.  As a result, when attempting to estimate an upper
 	 * bound for the largest currently-usable free segment in the
 	 * metaslab, we need to not consider any ranges currently in the defer
 	 * trees. This algorithm approximates the largest available chunk in
 	 * the largest range in the unflushed_frees tree by taking the first
 	 * chunk.  While this may be a poor estimate, it should only remain so
 	 * briefly and should eventually self-correct as frees are no longer
 	 * deferred. Similar logic applies to the ms_freed tree. See
 	 * metaslab_load() for more details.
 	 *
 	 * There are two primary sources of inaccuracy in this estimate. Both
 	 * are tolerated for performance reasons. The first source is that we
 	 * only check the largest segment for overlaps. Smaller segments may
 	 * have more favorable overlaps with the other trees, resulting in
 	 * larger usable chunks.  Second, we only look at the first chunk in
 	 * the largest segment; there may be other usable chunks in the
 	 * largest segment, but we ignore them.
 	 */
 	uint64_t rstart = rs_get_start(rs, msp->ms_unflushed_frees);
 	uint64_t rsize = rs_get_end(rs, msp->ms_unflushed_frees) - rstart;
 	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
 		uint64_t start = 0;
 		uint64_t size = 0;
 		boolean_t found = range_tree_find_in(msp->ms_defer[t], rstart,
 		    rsize, &start, &size);
 		if (found) {
 			if (rstart == start)
 				return (0);
 			rsize = start - rstart;
 		}
 	}
 
 	uint64_t start = 0;
 	uint64_t size = 0;
 	boolean_t found = range_tree_find_in(msp->ms_freed, rstart,
 	    rsize, &start, &size);
 	if (found)
 		rsize = start - rstart;
 
 	return (rsize);
 }
 
 static range_seg_t *
 metaslab_block_find(zfs_btree_t *t, range_tree_t *rt, uint64_t start,
     uint64_t size, zfs_btree_index_t *where)
 {
 	range_seg_t *rs;
 	range_seg_max_t rsearch;
 
 	rs_set_start(&rsearch, rt, start);
 	rs_set_end(&rsearch, rt, start + size);
 
 	rs = zfs_btree_find(t, &rsearch, where);
 	if (rs == NULL) {
 		rs = zfs_btree_next(t, where, where);
 	}
 
 	return (rs);
 }
 
 #if defined(WITH_DF_BLOCK_ALLOCATOR) || \
     defined(WITH_CF_BLOCK_ALLOCATOR)
 
 /*
  * This is a helper function that can be used by the allocator to find a
  * suitable block to allocate. This will search the specified B-tree looking
  * for a block that matches the specified criteria.
  */
 static uint64_t
 metaslab_block_picker(range_tree_t *rt, uint64_t *cursor, uint64_t size,
     uint64_t max_search)
 {
 	if (*cursor == 0)
 		*cursor = rt->rt_start;
 	zfs_btree_t *bt = &rt->rt_root;
 	zfs_btree_index_t where;
 	range_seg_t *rs = metaslab_block_find(bt, rt, *cursor, size, &where);
 	uint64_t first_found;
 	int count_searched = 0;
 
 	if (rs != NULL)
 		first_found = rs_get_start(rs, rt);
 
 	while (rs != NULL && (rs_get_start(rs, rt) - first_found <=
 	    max_search || count_searched < metaslab_min_search_count)) {
 		uint64_t offset = rs_get_start(rs, rt);
 		if (offset + size <= rs_get_end(rs, rt)) {
 			*cursor = offset + size;
 			return (offset);
 		}
 		rs = zfs_btree_next(bt, &where, &where);
 		count_searched++;
 	}
 
 	*cursor = 0;
 	return (-1ULL);
 }
 #endif /* WITH_DF/CF_BLOCK_ALLOCATOR */
 
 #if defined(WITH_DF_BLOCK_ALLOCATOR)
 /*
  * ==========================================================================
  * Dynamic Fit (df) block allocator
  *
  * Search for a free chunk of at least this size, starting from the last
  * offset (for this alignment of block) looking for up to
  * metaslab_df_max_search bytes (16MB).  If a large enough free chunk is not
  * found within 16MB, then return a free chunk of exactly the requested size (or
  * larger).
  *
  * If it seems like searching from the last offset will be unproductive, skip
  * that and just return a free chunk of exactly the requested size (or larger).
  * This is based on metaslab_df_alloc_threshold and metaslab_df_free_pct.  This
  * mechanism is probably not very useful and may be removed in the future.
  *
  * The behavior when not searching can be changed to return the largest free
  * chunk, instead of a free chunk of exactly the requested size, by setting
  * metaslab_df_use_largest_segment.
  * ==========================================================================
  */
 static uint64_t
 metaslab_df_alloc(metaslab_t *msp, uint64_t size)
 {
 	/*
 	 * Find the largest power of 2 block size that evenly divides the
 	 * requested size. This is used to try to allocate blocks with similar
 	 * alignment from the same area of the metaslab (i.e. same cursor
 	 * bucket) but it does not guarantee that other allocations sizes
 	 * may exist in the same region.
 	 */
 	uint64_t align = size & -size;
 	uint64_t *cursor = &msp->ms_lbas[highbit64(align) - 1];
 	range_tree_t *rt = msp->ms_allocatable;
 	uint_t free_pct = range_tree_space(rt) * 100 / msp->ms_size;
 	uint64_t offset;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	/*
 	 * If we're running low on space, find a segment based on size,
 	 * rather than iterating based on offset.
 	 */
 	if (metaslab_largest_allocatable(msp) < metaslab_df_alloc_threshold ||
 	    free_pct < metaslab_df_free_pct) {
 		offset = -1;
 	} else {
 		offset = metaslab_block_picker(rt,
 		    cursor, size, metaslab_df_max_search);
 	}
 
 	if (offset == -1) {
 		range_seg_t *rs;
 		if (zfs_btree_numnodes(&msp->ms_allocatable_by_size) == 0)
 			metaslab_size_tree_full_load(msp->ms_allocatable);
 
 		if (metaslab_df_use_largest_segment) {
 			/* use largest free segment */
 			rs = zfs_btree_last(&msp->ms_allocatable_by_size, NULL);
 		} else {
 			zfs_btree_index_t where;
 			/* use segment of this size, or next largest */
 			rs = metaslab_block_find(&msp->ms_allocatable_by_size,
 			    rt, msp->ms_start, size, &where);
 		}
 		if (rs != NULL && rs_get_start(rs, rt) + size <= rs_get_end(rs,
 		    rt)) {
 			offset = rs_get_start(rs, rt);
 			*cursor = offset + size;
 		}
 	}
 
 	return (offset);
 }
 
 const metaslab_ops_t zfs_metaslab_ops = {
 	metaslab_df_alloc
 };
 #endif /* WITH_DF_BLOCK_ALLOCATOR */
 
 #if defined(WITH_CF_BLOCK_ALLOCATOR)
 /*
  * ==========================================================================
  * Cursor fit block allocator -
  * Select the largest region in the metaslab, set the cursor to the beginning
  * of the range and the cursor_end to the end of the range. As allocations
  * are made advance the cursor. Continue allocating from the cursor until
  * the range is exhausted and then find a new range.
  * ==========================================================================
  */
 static uint64_t
 metaslab_cf_alloc(metaslab_t *msp, uint64_t size)
 {
 	range_tree_t *rt = msp->ms_allocatable;
 	zfs_btree_t *t = &msp->ms_allocatable_by_size;
 	uint64_t *cursor = &msp->ms_lbas[0];
 	uint64_t *cursor_end = &msp->ms_lbas[1];
 	uint64_t offset = 0;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	ASSERT3U(*cursor_end, >=, *cursor);
 
 	if ((*cursor + size) > *cursor_end) {
 		range_seg_t *rs;
 
 		if (zfs_btree_numnodes(t) == 0)
 			metaslab_size_tree_full_load(msp->ms_allocatable);
 		rs = zfs_btree_last(t, NULL);
 		if (rs == NULL || (rs_get_end(rs, rt) - rs_get_start(rs, rt)) <
 		    size)
 			return (-1ULL);
 
 		*cursor = rs_get_start(rs, rt);
 		*cursor_end = rs_get_end(rs, rt);
 	}
 
 	offset = *cursor;
 	*cursor += size;
 
 	return (offset);
 }
 
 const metaslab_ops_t zfs_metaslab_ops = {
 	metaslab_cf_alloc
 };
 #endif /* WITH_CF_BLOCK_ALLOCATOR */
 
 #if defined(WITH_NDF_BLOCK_ALLOCATOR)
 /*
  * ==========================================================================
  * New dynamic fit allocator -
  * Select a region that is large enough to allocate 2^metaslab_ndf_clump_shift
  * contiguous blocks. If no region is found then just use the largest segment
  * that remains.
  * ==========================================================================
  */
 
 /*
  * Determines desired number of contiguous blocks (2^metaslab_ndf_clump_shift)
  * to request from the allocator.
  */
 uint64_t metaslab_ndf_clump_shift = 4;
 
 static uint64_t
 metaslab_ndf_alloc(metaslab_t *msp, uint64_t size)
 {
 	zfs_btree_t *t = &msp->ms_allocatable->rt_root;
 	range_tree_t *rt = msp->ms_allocatable;
 	zfs_btree_index_t where;
 	range_seg_t *rs;
 	range_seg_max_t rsearch;
 	uint64_t hbit = highbit64(size);
 	uint64_t *cursor = &msp->ms_lbas[hbit - 1];
 	uint64_t max_size = metaslab_largest_allocatable(msp);
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	if (max_size < size)
 		return (-1ULL);
 
 	rs_set_start(&rsearch, rt, *cursor);
 	rs_set_end(&rsearch, rt, *cursor + size);
 
 	rs = zfs_btree_find(t, &rsearch, &where);
 	if (rs == NULL || (rs_get_end(rs, rt) - rs_get_start(rs, rt)) < size) {
 		t = &msp->ms_allocatable_by_size;
 
 		rs_set_start(&rsearch, rt, 0);
 		rs_set_end(&rsearch, rt, MIN(max_size, 1ULL << (hbit +
 		    metaslab_ndf_clump_shift)));
 
 		rs = zfs_btree_find(t, &rsearch, &where);
 		if (rs == NULL)
 			rs = zfs_btree_next(t, &where, &where);
 		ASSERT(rs != NULL);
 	}
 
 	if ((rs_get_end(rs, rt) - rs_get_start(rs, rt)) >= size) {
 		*cursor = rs_get_start(rs, rt) + size;
 		return (rs_get_start(rs, rt));
 	}
 	return (-1ULL);
 }
 
 const metaslab_ops_t zfs_metaslab_ops = {
 	metaslab_ndf_alloc
 };
 #endif /* WITH_NDF_BLOCK_ALLOCATOR */
 
 
 /*
  * ==========================================================================
  * Metaslabs
  * ==========================================================================
  */
 
 /*
  * Wait for any in-progress metaslab loads to complete.
  */
 static void
 metaslab_load_wait(metaslab_t *msp)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	while (msp->ms_loading) {
 		ASSERT(!msp->ms_loaded);
 		cv_wait(&msp->ms_load_cv, &msp->ms_lock);
 	}
 }
 
 /*
  * Wait for any in-progress flushing to complete.
  */
 static void
 metaslab_flush_wait(metaslab_t *msp)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	while (msp->ms_flushing)
 		cv_wait(&msp->ms_flush_cv, &msp->ms_lock);
 }
 
 static unsigned int
 metaslab_idx_func(multilist_t *ml, void *arg)
 {
 	metaslab_t *msp = arg;
 
 	/*
 	 * ms_id values are allocated sequentially, so full 64bit
 	 * division would be a waste of time, so limit it to 32 bits.
 	 */
 	return ((unsigned int)msp->ms_id % multilist_get_num_sublists(ml));
 }
 
 uint64_t
 metaslab_allocated_space(metaslab_t *msp)
 {
 	return (msp->ms_allocated_space);
 }
 
 /*
  * Verify that the space accounting on disk matches the in-core range_trees.
  */
 static void
 metaslab_verify_space(metaslab_t *msp, uint64_t txg)
 {
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 	uint64_t allocating = 0;
 	uint64_t sm_free_space, msp_free_space;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	ASSERT(!msp->ms_condensing);
 
 	if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0)
 		return;
 
 	/*
 	 * We can only verify the metaslab space when we're called
 	 * from syncing context with a loaded metaslab that has an
 	 * allocated space map. Calling this in non-syncing context
 	 * does not provide a consistent view of the metaslab since
 	 * we're performing allocations in the future.
 	 */
 	if (txg != spa_syncing_txg(spa) || msp->ms_sm == NULL ||
 	    !msp->ms_loaded)
 		return;
 
 	/*
 	 * Even though the smp_alloc field can get negative,
 	 * when it comes to a metaslab's space map, that should
 	 * never be the case.
 	 */
 	ASSERT3S(space_map_allocated(msp->ms_sm), >=, 0);
 
 	ASSERT3U(space_map_allocated(msp->ms_sm), >=,
 	    range_tree_space(msp->ms_unflushed_frees));
 
 	ASSERT3U(metaslab_allocated_space(msp), ==,
 	    space_map_allocated(msp->ms_sm) +
 	    range_tree_space(msp->ms_unflushed_allocs) -
 	    range_tree_space(msp->ms_unflushed_frees));
 
 	sm_free_space = msp->ms_size - metaslab_allocated_space(msp);
 
 	/*
 	 * Account for future allocations since we would have
 	 * already deducted that space from the ms_allocatable.
 	 */
 	for (int t = 0; t < TXG_CONCURRENT_STATES; t++) {
 		allocating +=
 		    range_tree_space(msp->ms_allocating[(txg + t) & TXG_MASK]);
 	}
 	ASSERT3U(allocating + msp->ms_allocated_this_txg, ==,
 	    msp->ms_allocating_total);
 
 	ASSERT3U(msp->ms_deferspace, ==,
 	    range_tree_space(msp->ms_defer[0]) +
 	    range_tree_space(msp->ms_defer[1]));
 
 	msp_free_space = range_tree_space(msp->ms_allocatable) + allocating +
 	    msp->ms_deferspace + range_tree_space(msp->ms_freed);
 
 	VERIFY3U(sm_free_space, ==, msp_free_space);
 }
 
 static void
 metaslab_aux_histograms_clear(metaslab_t *msp)
 {
 	/*
 	 * Auxiliary histograms are only cleared when resetting them,
 	 * which can only happen while the metaslab is loaded.
 	 */
 	ASSERT(msp->ms_loaded);
 
 	memset(msp->ms_synchist, 0, sizeof (msp->ms_synchist));
 	for (int t = 0; t < TXG_DEFER_SIZE; t++)
 		memset(msp->ms_deferhist[t], 0, sizeof (msp->ms_deferhist[t]));
 }
 
 static void
 metaslab_aux_histogram_add(uint64_t *histogram, uint64_t shift,
     range_tree_t *rt)
 {
 	/*
 	 * This is modeled after space_map_histogram_add(), so refer to that
 	 * function for implementation details. We want this to work like
 	 * the space map histogram, and not the range tree histogram, as we
 	 * are essentially constructing a delta that will be later subtracted
 	 * from the space map histogram.
 	 */
 	int idx = 0;
 	for (int i = shift; i < RANGE_TREE_HISTOGRAM_SIZE; i++) {
 		ASSERT3U(i, >=, idx + shift);
 		histogram[idx] += rt->rt_histogram[i] << (i - idx - shift);
 
 		if (idx < SPACE_MAP_HISTOGRAM_SIZE - 1) {
 			ASSERT3U(idx + shift, ==, i);
 			idx++;
 			ASSERT3U(idx, <, SPACE_MAP_HISTOGRAM_SIZE);
 		}
 	}
 }
 
 /*
  * Called at every sync pass that the metaslab gets synced.
  *
  * The reason is that we want our auxiliary histograms to be updated
  * wherever the metaslab's space map histogram is updated. This way
  * we stay consistent on which parts of the metaslab space map's
  * histogram are currently not available for allocations (e.g because
  * they are in the defer, freed, and freeing trees).
  */
 static void
 metaslab_aux_histograms_update(metaslab_t *msp)
 {
 	space_map_t *sm = msp->ms_sm;
 	ASSERT(sm != NULL);
 
 	/*
 	 * This is similar to the metaslab's space map histogram updates
 	 * that take place in metaslab_sync(). The only difference is that
 	 * we only care about segments that haven't made it into the
 	 * ms_allocatable tree yet.
 	 */
 	if (msp->ms_loaded) {
 		metaslab_aux_histograms_clear(msp);
 
 		metaslab_aux_histogram_add(msp->ms_synchist,
 		    sm->sm_shift, msp->ms_freed);
 
 		for (int t = 0; t < TXG_DEFER_SIZE; t++) {
 			metaslab_aux_histogram_add(msp->ms_deferhist[t],
 			    sm->sm_shift, msp->ms_defer[t]);
 		}
 	}
 
 	metaslab_aux_histogram_add(msp->ms_synchist,
 	    sm->sm_shift, msp->ms_freeing);
 }
 
 /*
  * Called every time we are done syncing (writing to) the metaslab,
  * i.e. at the end of each sync pass.
  * [see the comment in metaslab_impl.h for ms_synchist, ms_deferhist]
  */
 static void
 metaslab_aux_histograms_update_done(metaslab_t *msp, boolean_t defer_allowed)
 {
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 	space_map_t *sm = msp->ms_sm;
 
 	if (sm == NULL) {
 		/*
 		 * We came here from metaslab_init() when creating/opening a
 		 * pool, looking at a metaslab that hasn't had any allocations
 		 * yet.
 		 */
 		return;
 	}
 
 	/*
 	 * This is similar to the actions that we take for the ms_freed
 	 * and ms_defer trees in metaslab_sync_done().
 	 */
 	uint64_t hist_index = spa_syncing_txg(spa) % TXG_DEFER_SIZE;
 	if (defer_allowed) {
 		memcpy(msp->ms_deferhist[hist_index], msp->ms_synchist,
 		    sizeof (msp->ms_synchist));
 	} else {
 		memset(msp->ms_deferhist[hist_index], 0,
 		    sizeof (msp->ms_deferhist[hist_index]));
 	}
 	memset(msp->ms_synchist, 0, sizeof (msp->ms_synchist));
 }
 
 /*
  * Ensure that the metaslab's weight and fragmentation are consistent
  * with the contents of the histogram (either the range tree's histogram
  * or the space map's depending whether the metaslab is loaded).
  */
 static void
 metaslab_verify_weight_and_frag(metaslab_t *msp)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0)
 		return;
 
 	/*
 	 * We can end up here from vdev_remove_complete(), in which case we
 	 * cannot do these assertions because we hold spa config locks and
 	 * thus we are not allowed to read from the DMU.
 	 *
 	 * We check if the metaslab group has been removed and if that's
 	 * the case we return immediately as that would mean that we are
 	 * here from the aforementioned code path.
 	 */
 	if (msp->ms_group == NULL)
 		return;
 
 	/*
 	 * Devices being removed always return a weight of 0 and leave
 	 * fragmentation and ms_max_size as is - there is nothing for
 	 * us to verify here.
 	 */
 	vdev_t *vd = msp->ms_group->mg_vd;
 	if (vd->vdev_removing)
 		return;
 
 	/*
 	 * If the metaslab is dirty it probably means that we've done
 	 * some allocations or frees that have changed our histograms
 	 * and thus the weight.
 	 */
 	for (int t = 0; t < TXG_SIZE; t++) {
 		if (txg_list_member(&vd->vdev_ms_list, msp, t))
 			return;
 	}
 
 	/*
 	 * This verification checks that our in-memory state is consistent
 	 * with what's on disk. If the pool is read-only then there aren't
 	 * any changes and we just have the initially-loaded state.
 	 */
 	if (!spa_writeable(msp->ms_group->mg_vd->vdev_spa))
 		return;
 
 	/* some extra verification for in-core tree if you can */
 	if (msp->ms_loaded) {
 		range_tree_stat_verify(msp->ms_allocatable);
 		VERIFY(space_map_histogram_verify(msp->ms_sm,
 		    msp->ms_allocatable));
 	}
 
 	uint64_t weight = msp->ms_weight;
 	uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK;
 	boolean_t space_based = WEIGHT_IS_SPACEBASED(msp->ms_weight);
 	uint64_t frag = msp->ms_fragmentation;
 	uint64_t max_segsize = msp->ms_max_size;
 
 	msp->ms_weight = 0;
 	msp->ms_fragmentation = 0;
 
 	/*
 	 * This function is used for verification purposes and thus should
 	 * not introduce any side-effects/mutations on the system's state.
 	 *
 	 * Regardless of whether metaslab_weight() thinks this metaslab
 	 * should be active or not, we want to ensure that the actual weight
 	 * (and therefore the value of ms_weight) would be the same if it
 	 * was to be recalculated at this point.
 	 *
 	 * In addition we set the nodirty flag so metaslab_weight() does
 	 * not dirty the metaslab for future TXGs (e.g. when trying to
 	 * force condensing to upgrade the metaslab spacemaps).
 	 */
 	msp->ms_weight = metaslab_weight(msp, B_TRUE) | was_active;
 
 	VERIFY3U(max_segsize, ==, msp->ms_max_size);
 
 	/*
 	 * If the weight type changed then there is no point in doing
 	 * verification. Revert fields to their original values.
 	 */
 	if ((space_based && !WEIGHT_IS_SPACEBASED(msp->ms_weight)) ||
 	    (!space_based && WEIGHT_IS_SPACEBASED(msp->ms_weight))) {
 		msp->ms_fragmentation = frag;
 		msp->ms_weight = weight;
 		return;
 	}
 
 	VERIFY3U(msp->ms_fragmentation, ==, frag);
 	VERIFY3U(msp->ms_weight, ==, weight);
 }
 
 /*
  * If we're over the zfs_metaslab_mem_limit, select the loaded metaslab from
  * this class that was used longest ago, and attempt to unload it.  We don't
  * want to spend too much time in this loop to prevent performance
  * degradation, and we expect that most of the time this operation will
  * succeed. Between that and the normal unloading processing during txg sync,
  * we expect this to keep the metaslab memory usage under control.
  */
 static void
 metaslab_potentially_evict(metaslab_class_t *mc)
 {
 #ifdef _KERNEL
 	uint64_t allmem = arc_all_memory();
 	uint64_t inuse = spl_kmem_cache_inuse(zfs_btree_leaf_cache);
 	uint64_t size =	spl_kmem_cache_entry_size(zfs_btree_leaf_cache);
 	uint_t tries = 0;
 	for (; allmem * zfs_metaslab_mem_limit / 100 < inuse * size &&
 	    tries < multilist_get_num_sublists(&mc->mc_metaslab_txg_list) * 2;
 	    tries++) {
 		unsigned int idx = multilist_get_random_index(
 		    &mc->mc_metaslab_txg_list);
 		multilist_sublist_t *mls =
 		    multilist_sublist_lock_idx(&mc->mc_metaslab_txg_list, idx);
 		metaslab_t *msp = multilist_sublist_head(mls);
 		multilist_sublist_unlock(mls);
 		while (msp != NULL && allmem * zfs_metaslab_mem_limit / 100 <
 		    inuse * size) {
 			VERIFY3P(mls, ==, multilist_sublist_lock_idx(
 			    &mc->mc_metaslab_txg_list, idx));
 			ASSERT3U(idx, ==,
 			    metaslab_idx_func(&mc->mc_metaslab_txg_list, msp));
 
 			if (!multilist_link_active(&msp->ms_class_txg_node)) {
 				multilist_sublist_unlock(mls);
 				break;
 			}
 			metaslab_t *next_msp = multilist_sublist_next(mls, msp);
 			multilist_sublist_unlock(mls);
 			/*
 			 * If the metaslab is currently loading there are two
 			 * cases. If it's the metaslab we're evicting, we
 			 * can't continue on or we'll panic when we attempt to
 			 * recursively lock the mutex. If it's another
 			 * metaslab that's loading, it can be safely skipped,
 			 * since we know it's very new and therefore not a
 			 * good eviction candidate. We check later once the
 			 * lock is held that the metaslab is fully loaded
 			 * before actually unloading it.
 			 */
 			if (msp->ms_loading) {
 				msp = next_msp;
 				inuse =
 				    spl_kmem_cache_inuse(zfs_btree_leaf_cache);
 				continue;
 			}
 			/*
 			 * We can't unload metaslabs with no spacemap because
 			 * they're not ready to be unloaded yet. We can't
 			 * unload metaslabs with outstanding allocations
 			 * because doing so could cause the metaslab's weight
 			 * to decrease while it's unloaded, which violates an
 			 * invariant that we use to prevent unnecessary
 			 * loading. We also don't unload metaslabs that are
 			 * currently active because they are high-weight
 			 * metaslabs that are likely to be used in the near
 			 * future.
 			 */
 			mutex_enter(&msp->ms_lock);
 			if (msp->ms_allocator == -1 && msp->ms_sm != NULL &&
 			    msp->ms_allocating_total == 0) {
 				metaslab_unload(msp);
 			}
 			mutex_exit(&msp->ms_lock);
 			msp = next_msp;
 			inuse = spl_kmem_cache_inuse(zfs_btree_leaf_cache);
 		}
 	}
 #else
 	(void) mc, (void) zfs_metaslab_mem_limit;
 #endif
 }
 
 static int
 metaslab_load_impl(metaslab_t *msp)
 {
 	int error = 0;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	ASSERT(msp->ms_loading);
 	ASSERT(!msp->ms_condensing);
 
 	/*
 	 * We temporarily drop the lock to unblock other operations while we
 	 * are reading the space map. Therefore, metaslab_sync() and
 	 * metaslab_sync_done() can run at the same time as we do.
 	 *
 	 * If we are using the log space maps, metaslab_sync() can't write to
 	 * the metaslab's space map while we are loading as we only write to
 	 * it when we are flushing the metaslab, and that can't happen while
 	 * we are loading it.
 	 *
 	 * If we are not using log space maps though, metaslab_sync() can
 	 * append to the space map while we are loading. Therefore we load
 	 * only entries that existed when we started the load. Additionally,
 	 * metaslab_sync_done() has to wait for the load to complete because
 	 * there are potential races like metaslab_load() loading parts of the
 	 * space map that are currently being appended by metaslab_sync(). If
 	 * we didn't, the ms_allocatable would have entries that
 	 * metaslab_sync_done() would try to re-add later.
 	 *
 	 * That's why before dropping the lock we remember the synced length
 	 * of the metaslab and read up to that point of the space map,
 	 * ignoring entries appended by metaslab_sync() that happen after we
 	 * drop the lock.
 	 */
 	uint64_t length = msp->ms_synced_length;
 	mutex_exit(&msp->ms_lock);
 
 	hrtime_t load_start = gethrtime();
 	metaslab_rt_arg_t *mrap;
 	if (msp->ms_allocatable->rt_arg == NULL) {
 		mrap = kmem_zalloc(sizeof (*mrap), KM_SLEEP);
 	} else {
 		mrap = msp->ms_allocatable->rt_arg;
 		msp->ms_allocatable->rt_ops = NULL;
 		msp->ms_allocatable->rt_arg = NULL;
 	}
 	mrap->mra_bt = &msp->ms_allocatable_by_size;
 	mrap->mra_floor_shift = metaslab_by_size_min_shift;
 
 	if (msp->ms_sm != NULL) {
 		error = space_map_load_length(msp->ms_sm, msp->ms_allocatable,
 		    SM_FREE, length);
 
 		/* Now, populate the size-sorted tree. */
 		metaslab_rt_create(msp->ms_allocatable, mrap);
 		msp->ms_allocatable->rt_ops = &metaslab_rt_ops;
 		msp->ms_allocatable->rt_arg = mrap;
 
 		struct mssa_arg arg = {0};
 		arg.rt = msp->ms_allocatable;
 		arg.mra = mrap;
 		range_tree_walk(msp->ms_allocatable, metaslab_size_sorted_add,
 		    &arg);
 	} else {
 		/*
 		 * Add the size-sorted tree first, since we don't need to load
 		 * the metaslab from the spacemap.
 		 */
 		metaslab_rt_create(msp->ms_allocatable, mrap);
 		msp->ms_allocatable->rt_ops = &metaslab_rt_ops;
 		msp->ms_allocatable->rt_arg = mrap;
 		/*
 		 * The space map has not been allocated yet, so treat
 		 * all the space in the metaslab as free and add it to the
 		 * ms_allocatable tree.
 		 */
 		range_tree_add(msp->ms_allocatable,
 		    msp->ms_start, msp->ms_size);
 
 		if (msp->ms_new) {
 			/*
 			 * If the ms_sm doesn't exist, this means that this
 			 * metaslab hasn't gone through metaslab_sync() and
 			 * thus has never been dirtied. So we shouldn't
 			 * expect any unflushed allocs or frees from previous
 			 * TXGs.
 			 */
 			ASSERT(range_tree_is_empty(msp->ms_unflushed_allocs));
 			ASSERT(range_tree_is_empty(msp->ms_unflushed_frees));
 		}
 	}
 
 	/*
 	 * We need to grab the ms_sync_lock to prevent metaslab_sync() from
 	 * changing the ms_sm (or log_sm) and the metaslab's range trees
 	 * while we are about to use them and populate the ms_allocatable.
 	 * The ms_lock is insufficient for this because metaslab_sync() doesn't
 	 * hold the ms_lock while writing the ms_checkpointing tree to disk.
 	 */
 	mutex_enter(&msp->ms_sync_lock);
 	mutex_enter(&msp->ms_lock);
 
 	ASSERT(!msp->ms_condensing);
 	ASSERT(!msp->ms_flushing);
 
 	if (error != 0) {
 		mutex_exit(&msp->ms_sync_lock);
 		return (error);
 	}
 
 	ASSERT3P(msp->ms_group, !=, NULL);
 	msp->ms_loaded = B_TRUE;
 
 	/*
 	 * Apply all the unflushed changes to ms_allocatable right
 	 * away so any manipulations we do below have a clear view
 	 * of what is allocated and what is free.
 	 */
 	range_tree_walk(msp->ms_unflushed_allocs,
 	    range_tree_remove, msp->ms_allocatable);
 	range_tree_walk(msp->ms_unflushed_frees,
 	    range_tree_add, msp->ms_allocatable);
 
 	ASSERT3P(msp->ms_group, !=, NULL);
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 	if (spa_syncing_log_sm(spa) != NULL) {
 		ASSERT(spa_feature_is_enabled(spa,
 		    SPA_FEATURE_LOG_SPACEMAP));
 
 		/*
 		 * If we use a log space map we add all the segments
 		 * that are in ms_unflushed_frees so they are available
 		 * for allocation.
 		 *
 		 * ms_allocatable needs to contain all free segments
 		 * that are ready for allocations (thus not segments
 		 * from ms_freeing, ms_freed, and the ms_defer trees).
 		 * But if we grab the lock in this code path at a sync
 		 * pass later that 1, then it also contains the
 		 * segments of ms_freed (they were added to it earlier
 		 * in this path through ms_unflushed_frees). So we
 		 * need to remove all the segments that exist in
 		 * ms_freed from ms_allocatable as they will be added
 		 * later in metaslab_sync_done().
 		 *
 		 * When there's no log space map, the ms_allocatable
 		 * correctly doesn't contain any segments that exist
 		 * in ms_freed [see ms_synced_length].
 		 */
 		range_tree_walk(msp->ms_freed,
 		    range_tree_remove, msp->ms_allocatable);
 	}
 
 	/*
 	 * If we are not using the log space map, ms_allocatable
 	 * contains the segments that exist in the ms_defer trees
 	 * [see ms_synced_length]. Thus we need to remove them
 	 * from ms_allocatable as they will be added again in
 	 * metaslab_sync_done().
 	 *
 	 * If we are using the log space map, ms_allocatable still
 	 * contains the segments that exist in the ms_defer trees.
 	 * Not because it read them through the ms_sm though. But
 	 * because these segments are part of ms_unflushed_frees
 	 * whose segments we add to ms_allocatable earlier in this
 	 * code path.
 	 */
 	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
 		range_tree_walk(msp->ms_defer[t],
 		    range_tree_remove, msp->ms_allocatable);
 	}
 
 	/*
 	 * Call metaslab_recalculate_weight_and_sort() now that the
 	 * metaslab is loaded so we get the metaslab's real weight.
 	 *
 	 * Unless this metaslab was created with older software and
 	 * has not yet been converted to use segment-based weight, we
 	 * expect the new weight to be better or equal to the weight
 	 * that the metaslab had while it was not loaded. This is
 	 * because the old weight does not take into account the
 	 * consolidation of adjacent segments between TXGs. [see
 	 * comment for ms_synchist and ms_deferhist[] for more info]
 	 */
 	uint64_t weight = msp->ms_weight;
 	uint64_t max_size = msp->ms_max_size;
 	metaslab_recalculate_weight_and_sort(msp);
 	if (!WEIGHT_IS_SPACEBASED(weight))
 		ASSERT3U(weight, <=, msp->ms_weight);
 	msp->ms_max_size = metaslab_largest_allocatable(msp);
 	ASSERT3U(max_size, <=, msp->ms_max_size);
 	hrtime_t load_end = gethrtime();
 	msp->ms_load_time = load_end;
 	zfs_dbgmsg("metaslab_load: txg %llu, spa %s, vdev_id %llu, "
 	    "ms_id %llu, smp_length %llu, "
 	    "unflushed_allocs %llu, unflushed_frees %llu, "
 	    "freed %llu, defer %llu + %llu, unloaded time %llu ms, "
 	    "loading_time %lld ms, ms_max_size %llu, "
 	    "max size error %lld, "
 	    "old_weight %llx, new_weight %llx",
 	    (u_longlong_t)spa_syncing_txg(spa), spa_name(spa),
 	    (u_longlong_t)msp->ms_group->mg_vd->vdev_id,
 	    (u_longlong_t)msp->ms_id,
 	    (u_longlong_t)space_map_length(msp->ms_sm),
 	    (u_longlong_t)range_tree_space(msp->ms_unflushed_allocs),
 	    (u_longlong_t)range_tree_space(msp->ms_unflushed_frees),
 	    (u_longlong_t)range_tree_space(msp->ms_freed),
 	    (u_longlong_t)range_tree_space(msp->ms_defer[0]),
 	    (u_longlong_t)range_tree_space(msp->ms_defer[1]),
 	    (longlong_t)((load_start - msp->ms_unload_time) / 1000000),
 	    (longlong_t)((load_end - load_start) / 1000000),
 	    (u_longlong_t)msp->ms_max_size,
 	    (u_longlong_t)msp->ms_max_size - max_size,
 	    (u_longlong_t)weight, (u_longlong_t)msp->ms_weight);
 
 	metaslab_verify_space(msp, spa_syncing_txg(spa));
 	mutex_exit(&msp->ms_sync_lock);
 	return (0);
 }
 
 int
 metaslab_load(metaslab_t *msp)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	/*
 	 * There may be another thread loading the same metaslab, if that's
 	 * the case just wait until the other thread is done and return.
 	 */
 	metaslab_load_wait(msp);
 	if (msp->ms_loaded)
 		return (0);
 	VERIFY(!msp->ms_loading);
 	ASSERT(!msp->ms_condensing);
 
 	/*
 	 * We set the loading flag BEFORE potentially dropping the lock to
 	 * wait for an ongoing flush (see ms_flushing below). This way other
 	 * threads know that there is already a thread that is loading this
 	 * metaslab.
 	 */
 	msp->ms_loading = B_TRUE;
 
 	/*
 	 * Wait for any in-progress flushing to finish as we drop the ms_lock
 	 * both here (during space_map_load()) and in metaslab_flush() (when
 	 * we flush our changes to the ms_sm).
 	 */
 	if (msp->ms_flushing)
 		metaslab_flush_wait(msp);
 
 	/*
 	 * In the possibility that we were waiting for the metaslab to be
 	 * flushed (where we temporarily dropped the ms_lock), ensure that
 	 * no one else loaded the metaslab somehow.
 	 */
 	ASSERT(!msp->ms_loaded);
 
 	/*
 	 * If we're loading a metaslab in the normal class, consider evicting
 	 * another one to keep our memory usage under the limit defined by the
 	 * zfs_metaslab_mem_limit tunable.
 	 */
 	if (spa_normal_class(msp->ms_group->mg_class->mc_spa) ==
 	    msp->ms_group->mg_class) {
 		metaslab_potentially_evict(msp->ms_group->mg_class);
 	}
 
 	int error = metaslab_load_impl(msp);
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	msp->ms_loading = B_FALSE;
 	cv_broadcast(&msp->ms_load_cv);
 
 	return (error);
 }
 
 void
 metaslab_unload(metaslab_t *msp)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	/*
 	 * This can happen if a metaslab is selected for eviction (in
 	 * metaslab_potentially_evict) and then unloaded during spa_sync (via
 	 * metaslab_class_evict_old).
 	 */
 	if (!msp->ms_loaded)
 		return;
 
 	range_tree_vacate(msp->ms_allocatable, NULL, NULL);
 	msp->ms_loaded = B_FALSE;
 	msp->ms_unload_time = gethrtime();
 
 	msp->ms_activation_weight = 0;
 	msp->ms_weight &= ~METASLAB_ACTIVE_MASK;
 
 	if (msp->ms_group != NULL) {
 		metaslab_class_t *mc = msp->ms_group->mg_class;
 		multilist_sublist_t *mls =
 		    multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
 		if (multilist_link_active(&msp->ms_class_txg_node))
 			multilist_sublist_remove(mls, msp);
 		multilist_sublist_unlock(mls);
 
 		spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 		zfs_dbgmsg("metaslab_unload: txg %llu, spa %s, vdev_id %llu, "
 		    "ms_id %llu, weight %llx, "
 		    "selected txg %llu (%llu ms ago), alloc_txg %llu, "
 		    "loaded %llu ms ago, max_size %llu",
 		    (u_longlong_t)spa_syncing_txg(spa), spa_name(spa),
 		    (u_longlong_t)msp->ms_group->mg_vd->vdev_id,
 		    (u_longlong_t)msp->ms_id,
 		    (u_longlong_t)msp->ms_weight,
 		    (u_longlong_t)msp->ms_selected_txg,
 		    (u_longlong_t)(msp->ms_unload_time -
 		    msp->ms_selected_time) / 1000 / 1000,
 		    (u_longlong_t)msp->ms_alloc_txg,
 		    (u_longlong_t)(msp->ms_unload_time -
 		    msp->ms_load_time) / 1000 / 1000,
 		    (u_longlong_t)msp->ms_max_size);
 	}
 
 	/*
 	 * We explicitly recalculate the metaslab's weight based on its space
 	 * map (as it is now not loaded). We want unload metaslabs to always
 	 * have their weights calculated from the space map histograms, while
 	 * loaded ones have it calculated from their in-core range tree
 	 * [see metaslab_load()]. This way, the weight reflects the information
 	 * available in-core, whether it is loaded or not.
 	 *
 	 * If ms_group == NULL means that we came here from metaslab_fini(),
 	 * at which point it doesn't make sense for us to do the recalculation
 	 * and the sorting.
 	 */
 	if (msp->ms_group != NULL)
 		metaslab_recalculate_weight_and_sort(msp);
 }
 
 /*
  * We want to optimize the memory use of the per-metaslab range
  * trees. To do this, we store the segments in the range trees in
  * units of sectors, zero-indexing from the start of the metaslab. If
  * the vdev_ms_shift - the vdev_ashift is less than 32, we can store
  * the ranges using two uint32_ts, rather than two uint64_ts.
  */
 range_seg_type_t
 metaslab_calculate_range_tree_type(vdev_t *vdev, metaslab_t *msp,
     uint64_t *start, uint64_t *shift)
 {
 	if (vdev->vdev_ms_shift - vdev->vdev_ashift < 32 &&
 	    !zfs_metaslab_force_large_segs) {
 		*shift = vdev->vdev_ashift;
 		*start = msp->ms_start;
 		return (RANGE_SEG32);
 	} else {
 		*shift = 0;
 		*start = 0;
 		return (RANGE_SEG64);
 	}
 }
 
 void
 metaslab_set_selected_txg(metaslab_t *msp, uint64_t txg)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	metaslab_class_t *mc = msp->ms_group->mg_class;
 	multilist_sublist_t *mls =
 	    multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
 	if (multilist_link_active(&msp->ms_class_txg_node))
 		multilist_sublist_remove(mls, msp);
 	msp->ms_selected_txg = txg;
 	msp->ms_selected_time = gethrtime();
 	multilist_sublist_insert_tail(mls, msp);
 	multilist_sublist_unlock(mls);
 }
 
 void
 metaslab_space_update(vdev_t *vd, metaslab_class_t *mc, int64_t alloc_delta,
     int64_t defer_delta, int64_t space_delta)
 {
 	vdev_space_update(vd, alloc_delta, defer_delta, space_delta);
 
 	ASSERT3P(vd->vdev_spa->spa_root_vdev, ==, vd->vdev_parent);
 	ASSERT(vd->vdev_ms_count != 0);
 
 	metaslab_class_space_update(mc, alloc_delta, defer_delta, space_delta,
 	    vdev_deflated_space(vd, space_delta));
 }
 
 int
 metaslab_init(metaslab_group_t *mg, uint64_t id, uint64_t object,
     uint64_t txg, metaslab_t **msp)
 {
 	vdev_t *vd = mg->mg_vd;
 	spa_t *spa = vd->vdev_spa;
 	objset_t *mos = spa->spa_meta_objset;
 	metaslab_t *ms;
 	int error;
 
 	ms = kmem_zalloc(sizeof (metaslab_t), KM_SLEEP);
 	mutex_init(&ms->ms_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&ms->ms_sync_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&ms->ms_load_cv, NULL, CV_DEFAULT, NULL);
 	cv_init(&ms->ms_flush_cv, NULL, CV_DEFAULT, NULL);
 	multilist_link_init(&ms->ms_class_txg_node);
 
 	ms->ms_id = id;
 	ms->ms_start = id << vd->vdev_ms_shift;
 	ms->ms_size = 1ULL << vd->vdev_ms_shift;
 	ms->ms_allocator = -1;
 	ms->ms_new = B_TRUE;
 
 	vdev_ops_t *ops = vd->vdev_ops;
 	if (ops->vdev_op_metaslab_init != NULL)
 		ops->vdev_op_metaslab_init(vd, &ms->ms_start, &ms->ms_size);
 
 	/*
 	 * We only open space map objects that already exist. All others
 	 * will be opened when we finally allocate an object for it. For
 	 * readonly pools there is no need to open the space map object.
 	 *
 	 * Note:
 	 * When called from vdev_expand(), we can't call into the DMU as
 	 * we are holding the spa_config_lock as a writer and we would
 	 * deadlock [see relevant comment in vdev_metaslab_init()]. in
 	 * that case, the object parameter is zero though, so we won't
 	 * call into the DMU.
 	 */
 	if (object != 0 && !(spa->spa_mode == SPA_MODE_READ &&
 	    !spa->spa_read_spacemaps)) {
 		error = space_map_open(&ms->ms_sm, mos, object, ms->ms_start,
 		    ms->ms_size, vd->vdev_ashift);
 
 		if (error != 0) {
 			kmem_free(ms, sizeof (metaslab_t));
 			return (error);
 		}
 
 		ASSERT(ms->ms_sm != NULL);
 		ms->ms_allocated_space = space_map_allocated(ms->ms_sm);
 	}
 
 	uint64_t shift, start;
 	range_seg_type_t type =
 	    metaslab_calculate_range_tree_type(vd, ms, &start, &shift);
 
 	ms->ms_allocatable = range_tree_create(NULL, type, NULL, start, shift);
 	for (int t = 0; t < TXG_SIZE; t++) {
 		ms->ms_allocating[t] = range_tree_create(NULL, type,
 		    NULL, start, shift);
 	}
 	ms->ms_freeing = range_tree_create(NULL, type, NULL, start, shift);
 	ms->ms_freed = range_tree_create(NULL, type, NULL, start, shift);
 	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
 		ms->ms_defer[t] = range_tree_create(NULL, type, NULL,
 		    start, shift);
 	}
 	ms->ms_checkpointing =
 	    range_tree_create(NULL, type, NULL, start, shift);
 	ms->ms_unflushed_allocs =
 	    range_tree_create(NULL, type, NULL, start, shift);
 
 	metaslab_rt_arg_t *mrap = kmem_zalloc(sizeof (*mrap), KM_SLEEP);
 	mrap->mra_bt = &ms->ms_unflushed_frees_by_size;
 	mrap->mra_floor_shift = metaslab_by_size_min_shift;
 	ms->ms_unflushed_frees = range_tree_create(&metaslab_rt_ops,
 	    type, mrap, start, shift);
 
 	ms->ms_trim = range_tree_create(NULL, type, NULL, start, shift);
 
 	metaslab_group_add(mg, ms);
 	metaslab_set_fragmentation(ms, B_FALSE);
 
 	/*
 	 * If we're opening an existing pool (txg == 0) or creating
 	 * a new one (txg == TXG_INITIAL), all space is available now.
 	 * If we're adding space to an existing pool, the new space
 	 * does not become available until after this txg has synced.
 	 * The metaslab's weight will also be initialized when we sync
 	 * out this txg. This ensures that we don't attempt to allocate
 	 * from it before we have initialized it completely.
 	 */
 	if (txg <= TXG_INITIAL) {
 		metaslab_sync_done(ms, 0);
 		metaslab_space_update(vd, mg->mg_class,
 		    metaslab_allocated_space(ms), 0, 0);
 	}
 
 	if (txg != 0) {
 		vdev_dirty(vd, 0, NULL, txg);
 		vdev_dirty(vd, VDD_METASLAB, ms, txg);
 	}
 
 	*msp = ms;
 
 	return (0);
 }
 
 static void
 metaslab_fini_flush_data(metaslab_t *msp)
 {
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 
 	if (metaslab_unflushed_txg(msp) == 0) {
 		ASSERT3P(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL),
 		    ==, NULL);
 		return;
 	}
 	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));
 
 	mutex_enter(&spa->spa_flushed_ms_lock);
 	avl_remove(&spa->spa_metaslabs_by_flushed, msp);
 	mutex_exit(&spa->spa_flushed_ms_lock);
 
 	spa_log_sm_decrement_mscount(spa, metaslab_unflushed_txg(msp));
 	spa_log_summary_decrement_mscount(spa, metaslab_unflushed_txg(msp),
 	    metaslab_unflushed_dirty(msp));
 }
 
 uint64_t
 metaslab_unflushed_changes_memused(metaslab_t *ms)
 {
 	return ((range_tree_numsegs(ms->ms_unflushed_allocs) +
 	    range_tree_numsegs(ms->ms_unflushed_frees)) *
 	    ms->ms_unflushed_allocs->rt_root.bt_elem_size);
 }
 
 void
 metaslab_fini(metaslab_t *msp)
 {
 	metaslab_group_t *mg = msp->ms_group;
 	vdev_t *vd = mg->mg_vd;
 	spa_t *spa = vd->vdev_spa;
 
 	metaslab_fini_flush_data(msp);
 
 	metaslab_group_remove(mg, msp);
 
 	mutex_enter(&msp->ms_lock);
 	VERIFY(msp->ms_group == NULL);
 
 	/*
 	 * If this metaslab hasn't been through metaslab_sync_done() yet its
 	 * space hasn't been accounted for in its vdev and doesn't need to be
 	 * subtracted.
 	 */
 	if (!msp->ms_new) {
 		metaslab_space_update(vd, mg->mg_class,
 		    -metaslab_allocated_space(msp), 0, -msp->ms_size);
 
 	}
 	space_map_close(msp->ms_sm);
 	msp->ms_sm = NULL;
 
 	metaslab_unload(msp);
 
 	range_tree_destroy(msp->ms_allocatable);
 	range_tree_destroy(msp->ms_freeing);
 	range_tree_destroy(msp->ms_freed);
 
 	ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=,
 	    metaslab_unflushed_changes_memused(msp));
 	spa->spa_unflushed_stats.sus_memused -=
 	    metaslab_unflushed_changes_memused(msp);
 	range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL);
 	range_tree_destroy(msp->ms_unflushed_allocs);
 	range_tree_destroy(msp->ms_checkpointing);
 	range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL);
 	range_tree_destroy(msp->ms_unflushed_frees);
 
 	for (int t = 0; t < TXG_SIZE; t++) {
 		range_tree_destroy(msp->ms_allocating[t]);
 	}
 	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
 		range_tree_destroy(msp->ms_defer[t]);
 	}
 	ASSERT0(msp->ms_deferspace);
 
 	for (int t = 0; t < TXG_SIZE; t++)
 		ASSERT(!txg_list_member(&vd->vdev_ms_list, msp, t));
 
 	range_tree_vacate(msp->ms_trim, NULL, NULL);
 	range_tree_destroy(msp->ms_trim);
 
 	mutex_exit(&msp->ms_lock);
 	cv_destroy(&msp->ms_load_cv);
 	cv_destroy(&msp->ms_flush_cv);
 	mutex_destroy(&msp->ms_lock);
 	mutex_destroy(&msp->ms_sync_lock);
 	ASSERT3U(msp->ms_allocator, ==, -1);
 
 	kmem_free(msp, sizeof (metaslab_t));
 }
 
 #define	FRAGMENTATION_TABLE_SIZE	17
 
 /*
  * This table defines a segment size based fragmentation metric that will
  * allow each metaslab to derive its own fragmentation value. This is done
  * by calculating the space in each bucket of the spacemap histogram and
  * multiplying that by the fragmentation metric in this table. Doing
  * this for all buckets and dividing it by the total amount of free
  * space in this metaslab (i.e. the total free space in all buckets) gives
  * us the fragmentation metric. This means that a high fragmentation metric
  * equates to most of the free space being comprised of small segments.
  * Conversely, if the metric is low, then most of the free space is in
  * large segments. A 10% change in fragmentation equates to approximately
  * double the number of segments.
  *
  * This table defines 0% fragmented space using 16MB segments. Testing has
  * shown that segments that are greater than or equal to 16MB do not suffer
  * from drastic performance problems. Using this value, we derive the rest
  * of the table. Since the fragmentation value is never stored on disk, it
  * is possible to change these calculations in the future.
  */
 static const int zfs_frag_table[FRAGMENTATION_TABLE_SIZE] = {
 	100,	/* 512B	*/
 	100,	/* 1K	*/
 	98,	/* 2K	*/
 	95,	/* 4K	*/
 	90,	/* 8K	*/
 	80,	/* 16K	*/
 	70,	/* 32K	*/
 	60,	/* 64K	*/
 	50,	/* 128K	*/
 	40,	/* 256K	*/
 	30,	/* 512K	*/
 	20,	/* 1M	*/
 	15,	/* 2M	*/
 	10,	/* 4M	*/
 	5,	/* 8M	*/
 	0	/* 16M	*/
 };
 
 /*
  * Calculate the metaslab's fragmentation metric and set ms_fragmentation.
  * Setting this value to ZFS_FRAG_INVALID means that the metaslab has not
  * been upgraded and does not support this metric. Otherwise, the return
  * value should be in the range [0, 100].
  */
 static void
 metaslab_set_fragmentation(metaslab_t *msp, boolean_t nodirty)
 {
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 	uint64_t fragmentation = 0;
 	uint64_t total = 0;
 	boolean_t feature_enabled = spa_feature_is_enabled(spa,
 	    SPA_FEATURE_SPACEMAP_HISTOGRAM);
 
 	if (!feature_enabled) {
 		msp->ms_fragmentation = ZFS_FRAG_INVALID;
 		return;
 	}
 
 	/*
 	 * A null space map means that the entire metaslab is free
 	 * and thus is not fragmented.
 	 */
 	if (msp->ms_sm == NULL) {
 		msp->ms_fragmentation = 0;
 		return;
 	}
 
 	/*
 	 * If this metaslab's space map has not been upgraded, flag it
 	 * so that we upgrade next time we encounter it.
 	 */
 	if (msp->ms_sm->sm_dbuf->db_size != sizeof (space_map_phys_t)) {
 		uint64_t txg = spa_syncing_txg(spa);
 		vdev_t *vd = msp->ms_group->mg_vd;
 
 		/*
 		 * If we've reached the final dirty txg, then we must
 		 * be shutting down the pool. We don't want to dirty
 		 * any data past this point so skip setting the condense
 		 * flag. We can retry this action the next time the pool
 		 * is imported. We also skip marking this metaslab for
 		 * condensing if the caller has explicitly set nodirty.
 		 */
 		if (!nodirty &&
 		    spa_writeable(spa) && txg < spa_final_dirty_txg(spa)) {
 			msp->ms_condense_wanted = B_TRUE;
 			vdev_dirty(vd, VDD_METASLAB, msp, txg + 1);
 			zfs_dbgmsg("txg %llu, requesting force condense: "
 			    "ms_id %llu, vdev_id %llu", (u_longlong_t)txg,
 			    (u_longlong_t)msp->ms_id,
 			    (u_longlong_t)vd->vdev_id);
 		}
 		msp->ms_fragmentation = ZFS_FRAG_INVALID;
 		return;
 	}
 
 	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
 		uint64_t space = 0;
 		uint8_t shift = msp->ms_sm->sm_shift;
 
 		int idx = MIN(shift - SPA_MINBLOCKSHIFT + i,
 		    FRAGMENTATION_TABLE_SIZE - 1);
 
 		if (msp->ms_sm->sm_phys->smp_histogram[i] == 0)
 			continue;
 
 		space = msp->ms_sm->sm_phys->smp_histogram[i] << (i + shift);
 		total += space;
 
 		ASSERT3U(idx, <, FRAGMENTATION_TABLE_SIZE);
 		fragmentation += space * zfs_frag_table[idx];
 	}
 
 	if (total > 0)
 		fragmentation /= total;
 	ASSERT3U(fragmentation, <=, 100);
 
 	msp->ms_fragmentation = fragmentation;
 }
 
 /*
  * Compute a weight -- a selection preference value -- for the given metaslab.
  * This is based on the amount of free space, the level of fragmentation,
  * the LBA range, and whether the metaslab is loaded.
  */
 static uint64_t
 metaslab_space_weight(metaslab_t *msp)
 {
 	metaslab_group_t *mg = msp->ms_group;
 	vdev_t *vd = mg->mg_vd;
 	uint64_t weight, space;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	/*
 	 * The baseline weight is the metaslab's free space.
 	 */
 	space = msp->ms_size - metaslab_allocated_space(msp);
 
 	if (metaslab_fragmentation_factor_enabled &&
 	    msp->ms_fragmentation != ZFS_FRAG_INVALID) {
 		/*
 		 * Use the fragmentation information to inversely scale
 		 * down the baseline weight. We need to ensure that we
 		 * don't exclude this metaslab completely when it's 100%
 		 * fragmented. To avoid this we reduce the fragmented value
 		 * by 1.
 		 */
 		space = (space * (100 - (msp->ms_fragmentation - 1))) / 100;
 
 		/*
 		 * If space < SPA_MINBLOCKSIZE, then we will not allocate from
 		 * this metaslab again. The fragmentation metric may have
 		 * decreased the space to something smaller than
 		 * SPA_MINBLOCKSIZE, so reset the space to SPA_MINBLOCKSIZE
 		 * so that we can consume any remaining space.
 		 */
 		if (space > 0 && space < SPA_MINBLOCKSIZE)
 			space = SPA_MINBLOCKSIZE;
 	}
 	weight = space;
 
 	/*
 	 * Modern disks have uniform bit density and constant angular velocity.
 	 * Therefore, the outer recording zones are faster (higher bandwidth)
 	 * than the inner zones by the ratio of outer to inner track diameter,
 	 * which is typically around 2:1.  We account for this by assigning
 	 * higher weight to lower metaslabs (multiplier ranging from 2x to 1x).
 	 * In effect, this means that we'll select the metaslab with the most
 	 * free bandwidth rather than simply the one with the most free space.
 	 */
 	if (!vd->vdev_nonrot && metaslab_lba_weighting_enabled) {
 		weight = 2 * weight - (msp->ms_id * weight) / vd->vdev_ms_count;
 		ASSERT(weight >= space && weight <= 2 * space);
 	}
 
 	/*
 	 * If this metaslab is one we're actively using, adjust its
 	 * weight to make it preferable to any inactive metaslab so
 	 * we'll polish it off. If the fragmentation on this metaslab
 	 * has exceed our threshold, then don't mark it active.
 	 */
 	if (msp->ms_loaded && msp->ms_fragmentation != ZFS_FRAG_INVALID &&
 	    msp->ms_fragmentation <= zfs_metaslab_fragmentation_threshold) {
 		weight |= (msp->ms_weight & METASLAB_ACTIVE_MASK);
 	}
 
 	WEIGHT_SET_SPACEBASED(weight);
 	return (weight);
 }
 
 /*
  * Return the weight of the specified metaslab, according to the segment-based
  * weighting algorithm. The metaslab must be loaded. This function can
  * be called within a sync pass since it relies only on the metaslab's
  * range tree which is always accurate when the metaslab is loaded.
  */
 static uint64_t
 metaslab_weight_from_range_tree(metaslab_t *msp)
 {
 	uint64_t weight = 0;
 	uint32_t segments = 0;
 
 	ASSERT(msp->ms_loaded);
 
 	for (int i = RANGE_TREE_HISTOGRAM_SIZE - 1; i >= SPA_MINBLOCKSHIFT;
 	    i--) {
 		uint8_t shift = msp->ms_group->mg_vd->vdev_ashift;
 		int max_idx = SPACE_MAP_HISTOGRAM_SIZE + shift - 1;
 
 		segments <<= 1;
 		segments += msp->ms_allocatable->rt_histogram[i];
 
 		/*
 		 * The range tree provides more precision than the space map
 		 * and must be downgraded so that all values fit within the
 		 * space map's histogram. This allows us to compare loaded
 		 * vs. unloaded metaslabs to determine which metaslab is
 		 * considered "best".
 		 */
 		if (i > max_idx)
 			continue;
 
 		if (segments != 0) {
 			WEIGHT_SET_COUNT(weight, segments);
 			WEIGHT_SET_INDEX(weight, i);
 			WEIGHT_SET_ACTIVE(weight, 0);
 			break;
 		}
 	}
 	return (weight);
 }
 
 /*
  * Calculate the weight based on the on-disk histogram. Should be applied
  * only to unloaded metaslabs  (i.e no incoming allocations) in-order to
  * give results consistent with the on-disk state
  */
 static uint64_t
 metaslab_weight_from_spacemap(metaslab_t *msp)
 {
 	space_map_t *sm = msp->ms_sm;
 	ASSERT(!msp->ms_loaded);
 	ASSERT(sm != NULL);
 	ASSERT3U(space_map_object(sm), !=, 0);
 	ASSERT3U(sm->sm_dbuf->db_size, ==, sizeof (space_map_phys_t));
 
 	/*
 	 * Create a joint histogram from all the segments that have made
 	 * it to the metaslab's space map histogram, that are not yet
 	 * available for allocation because they are still in the freeing
 	 * pipeline (e.g. freeing, freed, and defer trees). Then subtract
 	 * these segments from the space map's histogram to get a more
 	 * accurate weight.
 	 */
 	uint64_t deferspace_histogram[SPACE_MAP_HISTOGRAM_SIZE] = {0};
 	for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++)
 		deferspace_histogram[i] += msp->ms_synchist[i];
 	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
 		for (int i = 0; i < SPACE_MAP_HISTOGRAM_SIZE; i++) {
 			deferspace_histogram[i] += msp->ms_deferhist[t][i];
 		}
 	}
 
 	uint64_t weight = 0;
 	for (int i = SPACE_MAP_HISTOGRAM_SIZE - 1; i >= 0; i--) {
 		ASSERT3U(sm->sm_phys->smp_histogram[i], >=,
 		    deferspace_histogram[i]);
 		uint64_t count =
 		    sm->sm_phys->smp_histogram[i] - deferspace_histogram[i];
 		if (count != 0) {
 			WEIGHT_SET_COUNT(weight, count);
 			WEIGHT_SET_INDEX(weight, i + sm->sm_shift);
 			WEIGHT_SET_ACTIVE(weight, 0);
 			break;
 		}
 	}
 	return (weight);
 }
 
 /*
  * Compute a segment-based weight for the specified metaslab. The weight
  * is determined by highest bucket in the histogram. The information
  * for the highest bucket is encoded into the weight value.
  */
 static uint64_t
 metaslab_segment_weight(metaslab_t *msp)
 {
 	metaslab_group_t *mg = msp->ms_group;
 	uint64_t weight = 0;
 	uint8_t shift = mg->mg_vd->vdev_ashift;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	/*
 	 * The metaslab is completely free.
 	 */
 	if (metaslab_allocated_space(msp) == 0) {
 		int idx = highbit64(msp->ms_size) - 1;
 		int max_idx = SPACE_MAP_HISTOGRAM_SIZE + shift - 1;
 
 		if (idx < max_idx) {
 			WEIGHT_SET_COUNT(weight, 1ULL);
 			WEIGHT_SET_INDEX(weight, idx);
 		} else {
 			WEIGHT_SET_COUNT(weight, 1ULL << (idx - max_idx));
 			WEIGHT_SET_INDEX(weight, max_idx);
 		}
 		WEIGHT_SET_ACTIVE(weight, 0);
 		ASSERT(!WEIGHT_IS_SPACEBASED(weight));
 		return (weight);
 	}
 
 	ASSERT3U(msp->ms_sm->sm_dbuf->db_size, ==, sizeof (space_map_phys_t));
 
 	/*
 	 * If the metaslab is fully allocated then just make the weight 0.
 	 */
 	if (metaslab_allocated_space(msp) == msp->ms_size)
 		return (0);
 	/*
 	 * If the metaslab is already loaded, then use the range tree to
 	 * determine the weight. Otherwise, we rely on the space map information
 	 * to generate the weight.
 	 */
 	if (msp->ms_loaded) {
 		weight = metaslab_weight_from_range_tree(msp);
 	} else {
 		weight = metaslab_weight_from_spacemap(msp);
 	}
 
 	/*
 	 * If the metaslab was active the last time we calculated its weight
 	 * then keep it active. We want to consume the entire region that
 	 * is associated with this weight.
 	 */
 	if (msp->ms_activation_weight != 0 && weight != 0)
 		WEIGHT_SET_ACTIVE(weight, WEIGHT_GET_ACTIVE(msp->ms_weight));
 	return (weight);
 }
 
 /*
  * Determine if we should attempt to allocate from this metaslab. If the
  * metaslab is loaded, then we can determine if the desired allocation
  * can be satisfied by looking at the size of the maximum free segment
  * on that metaslab. Otherwise, we make our decision based on the metaslab's
  * weight. For segment-based weighting we can determine the maximum
  * allocation based on the index encoded in its value. For space-based
  * weights we rely on the entire weight (excluding the weight-type bit).
  */
 static boolean_t
 metaslab_should_allocate(metaslab_t *msp, uint64_t asize, boolean_t try_hard)
 {
 	/*
 	 * This case will usually but not always get caught by the checks below;
 	 * metaslabs can be loaded by various means, including the trim and
 	 * initialize code. Once that happens, without this check they are
 	 * allocatable even before they finish their first txg sync.
 	 */
 	if (unlikely(msp->ms_new))
 		return (B_FALSE);
 
 	/*
 	 * If the metaslab is loaded, ms_max_size is definitive and we can use
 	 * the fast check. If it's not, the ms_max_size is a lower bound (once
 	 * set), and we should use the fast check as long as we're not in
 	 * try_hard and it's been less than zfs_metaslab_max_size_cache_sec
 	 * seconds since the metaslab was unloaded.
 	 */
 	if (msp->ms_loaded ||
 	    (msp->ms_max_size != 0 && !try_hard && gethrtime() <
 	    msp->ms_unload_time + SEC2NSEC(zfs_metaslab_max_size_cache_sec)))
 		return (msp->ms_max_size >= asize);
 
 	boolean_t should_allocate;
 	if (!WEIGHT_IS_SPACEBASED(msp->ms_weight)) {
 		/*
 		 * The metaslab segment weight indicates segments in the
 		 * range [2^i, 2^(i+1)), where i is the index in the weight.
 		 * Since the asize might be in the middle of the range, we
 		 * should attempt the allocation if asize < 2^(i+1).
 		 */
 		should_allocate = (asize <
 		    1ULL << (WEIGHT_GET_INDEX(msp->ms_weight) + 1));
 	} else {
 		should_allocate = (asize <=
 		    (msp->ms_weight & ~METASLAB_WEIGHT_TYPE));
 	}
 
 	return (should_allocate);
 }
 
 static uint64_t
 metaslab_weight(metaslab_t *msp, boolean_t nodirty)
 {
 	vdev_t *vd = msp->ms_group->mg_vd;
 	spa_t *spa = vd->vdev_spa;
 	uint64_t weight;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	metaslab_set_fragmentation(msp, nodirty);
 
 	/*
 	 * Update the maximum size. If the metaslab is loaded, this will
 	 * ensure that we get an accurate maximum size if newly freed space
 	 * has been added back into the free tree. If the metaslab is
 	 * unloaded, we check if there's a larger free segment in the
 	 * unflushed frees. This is a lower bound on the largest allocatable
 	 * segment size. Coalescing of adjacent entries may reveal larger
 	 * allocatable segments, but we aren't aware of those until loading
 	 * the space map into a range tree.
 	 */
 	if (msp->ms_loaded) {
 		msp->ms_max_size = metaslab_largest_allocatable(msp);
 	} else {
 		msp->ms_max_size = MAX(msp->ms_max_size,
 		    metaslab_largest_unflushed_free(msp));
 	}
 
 	/*
 	 * Segment-based weighting requires space map histogram support.
 	 */
 	if (zfs_metaslab_segment_weight_enabled &&
 	    spa_feature_is_enabled(spa, SPA_FEATURE_SPACEMAP_HISTOGRAM) &&
 	    (msp->ms_sm == NULL || msp->ms_sm->sm_dbuf->db_size ==
 	    sizeof (space_map_phys_t))) {
 		weight = metaslab_segment_weight(msp);
 	} else {
 		weight = metaslab_space_weight(msp);
 	}
 	return (weight);
 }
 
 void
 metaslab_recalculate_weight_and_sort(metaslab_t *msp)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	/* note: we preserve the mask (e.g. indication of primary, etc..) */
 	uint64_t was_active = msp->ms_weight & METASLAB_ACTIVE_MASK;
 	metaslab_group_sort(msp->ms_group, msp,
 	    metaslab_weight(msp, B_FALSE) | was_active);
 }
 
 static int
 metaslab_activate_allocator(metaslab_group_t *mg, metaslab_t *msp,
     int allocator, uint64_t activation_weight)
 {
 	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	/*
 	 * If we're activating for the claim code, we don't want to actually
 	 * set the metaslab up for a specific allocator.
 	 */
 	if (activation_weight == METASLAB_WEIGHT_CLAIM) {
 		ASSERT0(msp->ms_activation_weight);
 		msp->ms_activation_weight = msp->ms_weight;
 		metaslab_group_sort(mg, msp, msp->ms_weight |
 		    activation_weight);
 		return (0);
 	}
 
 	metaslab_t **mspp = (activation_weight == METASLAB_WEIGHT_PRIMARY ?
 	    &mga->mga_primary : &mga->mga_secondary);
 
 	mutex_enter(&mg->mg_lock);
 	if (*mspp != NULL) {
 		mutex_exit(&mg->mg_lock);
 		return (EEXIST);
 	}
 
 	*mspp = msp;
 	ASSERT3S(msp->ms_allocator, ==, -1);
 	msp->ms_allocator = allocator;
 	msp->ms_primary = (activation_weight == METASLAB_WEIGHT_PRIMARY);
 
 	ASSERT0(msp->ms_activation_weight);
 	msp->ms_activation_weight = msp->ms_weight;
 	metaslab_group_sort_impl(mg, msp,
 	    msp->ms_weight | activation_weight);
 	mutex_exit(&mg->mg_lock);
 
 	return (0);
 }
 
 static int
 metaslab_activate(metaslab_t *msp, int allocator, uint64_t activation_weight)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	/*
 	 * The current metaslab is already activated for us so there
 	 * is nothing to do. Already activated though, doesn't mean
 	 * that this metaslab is activated for our allocator nor our
 	 * requested activation weight. The metaslab could have started
 	 * as an active one for our allocator but changed allocators
 	 * while we were waiting to grab its ms_lock or we stole it
 	 * [see find_valid_metaslab()]. This means that there is a
 	 * possibility of passivating a metaslab of another allocator
 	 * or from a different activation mask, from this thread.
 	 */
 	if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) {
 		ASSERT(msp->ms_loaded);
 		return (0);
 	}
 
 	int error = metaslab_load(msp);
 	if (error != 0) {
 		metaslab_group_sort(msp->ms_group, msp, 0);
 		return (error);
 	}
 
 	/*
 	 * When entering metaslab_load() we may have dropped the
 	 * ms_lock because we were loading this metaslab, or we
 	 * were waiting for another thread to load it for us. In
 	 * that scenario, we recheck the weight of the metaslab
 	 * to see if it was activated by another thread.
 	 *
 	 * If the metaslab was activated for another allocator or
 	 * it was activated with a different activation weight (e.g.
 	 * we wanted to make it a primary but it was activated as
 	 * secondary) we return error (EBUSY).
 	 *
 	 * If the metaslab was activated for the same allocator
 	 * and requested activation mask, skip activating it.
 	 */
 	if ((msp->ms_weight & METASLAB_ACTIVE_MASK) != 0) {
 		if (msp->ms_allocator != allocator)
 			return (EBUSY);
 
 		if ((msp->ms_weight & activation_weight) == 0)
 			return (SET_ERROR(EBUSY));
 
 		EQUIV((activation_weight == METASLAB_WEIGHT_PRIMARY),
 		    msp->ms_primary);
 		return (0);
 	}
 
 	/*
 	 * If the metaslab has literally 0 space, it will have weight 0. In
 	 * that case, don't bother activating it. This can happen if the
 	 * metaslab had space during find_valid_metaslab, but another thread
 	 * loaded it and used all that space while we were waiting to grab the
 	 * lock.
 	 */
 	if (msp->ms_weight == 0) {
 		ASSERT0(range_tree_space(msp->ms_allocatable));
 		return (SET_ERROR(ENOSPC));
 	}
 
 	if ((error = metaslab_activate_allocator(msp->ms_group, msp,
 	    allocator, activation_weight)) != 0) {
 		return (error);
 	}
 
 	ASSERT(msp->ms_loaded);
 	ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK);
 
 	return (0);
 }
 
 static void
 metaslab_passivate_allocator(metaslab_group_t *mg, metaslab_t *msp,
     uint64_t weight)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	ASSERT(msp->ms_loaded);
 
 	if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) {
 		metaslab_group_sort(mg, msp, weight);
 		return;
 	}
 
 	mutex_enter(&mg->mg_lock);
 	ASSERT3P(msp->ms_group, ==, mg);
 	ASSERT3S(0, <=, msp->ms_allocator);
 	ASSERT3U(msp->ms_allocator, <, mg->mg_allocators);
 
 	metaslab_group_allocator_t *mga = &mg->mg_allocator[msp->ms_allocator];
 	if (msp->ms_primary) {
 		ASSERT3P(mga->mga_primary, ==, msp);
 		ASSERT(msp->ms_weight & METASLAB_WEIGHT_PRIMARY);
 		mga->mga_primary = NULL;
 	} else {
 		ASSERT3P(mga->mga_secondary, ==, msp);
 		ASSERT(msp->ms_weight & METASLAB_WEIGHT_SECONDARY);
 		mga->mga_secondary = NULL;
 	}
 	msp->ms_allocator = -1;
 	metaslab_group_sort_impl(mg, msp, weight);
 	mutex_exit(&mg->mg_lock);
 }
 
 static void
 metaslab_passivate(metaslab_t *msp, uint64_t weight)
 {
 	uint64_t size __maybe_unused = weight & ~METASLAB_WEIGHT_TYPE;
 
 	/*
 	 * If size < SPA_MINBLOCKSIZE, then we will not allocate from
 	 * this metaslab again.  In that case, it had better be empty,
 	 * or we would be leaving space on the table.
 	 */
 	ASSERT(!WEIGHT_IS_SPACEBASED(msp->ms_weight) ||
 	    size >= SPA_MINBLOCKSIZE ||
 	    range_tree_space(msp->ms_allocatable) == 0);
 	ASSERT0(weight & METASLAB_ACTIVE_MASK);
 
 	ASSERT(msp->ms_activation_weight != 0);
 	msp->ms_activation_weight = 0;
 	metaslab_passivate_allocator(msp->ms_group, msp, weight);
 	ASSERT0(msp->ms_weight & METASLAB_ACTIVE_MASK);
 }
 
 /*
  * Segment-based metaslabs are activated once and remain active until
  * we either fail an allocation attempt (similar to space-based metaslabs)
  * or have exhausted the free space in zfs_metaslab_switch_threshold
  * buckets since the metaslab was activated. This function checks to see
  * if we've exhausted the zfs_metaslab_switch_threshold buckets in the
  * metaslab and passivates it proactively. This will allow us to select a
  * metaslab with a larger contiguous region, if any, remaining within this
  * metaslab group. If we're in sync pass > 1, then we continue using this
  * metaslab so that we don't dirty more block and cause more sync passes.
  */
 static void
 metaslab_segment_may_passivate(metaslab_t *msp)
 {
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 
 	if (WEIGHT_IS_SPACEBASED(msp->ms_weight) || spa_sync_pass(spa) > 1)
 		return;
 
 	/*
 	 * Since we are in the middle of a sync pass, the most accurate
 	 * information that is accessible to us is the in-core range tree
 	 * histogram; calculate the new weight based on that information.
 	 */
 	uint64_t weight = metaslab_weight_from_range_tree(msp);
 	int activation_idx = WEIGHT_GET_INDEX(msp->ms_activation_weight);
 	int current_idx = WEIGHT_GET_INDEX(weight);
 
 	if (current_idx <= activation_idx - zfs_metaslab_switch_threshold)
 		metaslab_passivate(msp, weight);
 }
 
 static void
 metaslab_preload(void *arg)
 {
 	metaslab_t *msp = arg;
 	metaslab_class_t *mc = msp->ms_group->mg_class;
 	spa_t *spa = mc->mc_spa;
 	fstrans_cookie_t cookie = spl_fstrans_mark();
 
 	ASSERT(!MUTEX_HELD(&msp->ms_group->mg_lock));
 
 	mutex_enter(&msp->ms_lock);
 	(void) metaslab_load(msp);
 	metaslab_set_selected_txg(msp, spa_syncing_txg(spa));
 	mutex_exit(&msp->ms_lock);
 	spl_fstrans_unmark(cookie);
 }
 
 static void
 metaslab_group_preload(metaslab_group_t *mg)
 {
 	spa_t *spa = mg->mg_vd->vdev_spa;
 	metaslab_t *msp;
 	avl_tree_t *t = &mg->mg_metaslab_tree;
 	int m = 0;
 
 	if (spa_shutting_down(spa) || !metaslab_preload_enabled)
 		return;
 
 	mutex_enter(&mg->mg_lock);
 
 	/*
 	 * Load the next potential metaslabs
 	 */
 	for (msp = avl_first(t); msp != NULL; msp = AVL_NEXT(t, msp)) {
 		ASSERT3P(msp->ms_group, ==, mg);
 
 		/*
 		 * We preload only the maximum number of metaslabs specified
 		 * by metaslab_preload_limit. If a metaslab is being forced
 		 * to condense then we preload it too. This will ensure
 		 * that force condensing happens in the next txg.
 		 */
 		if (++m > metaslab_preload_limit && !msp->ms_condense_wanted) {
 			continue;
 		}
 
 		VERIFY(taskq_dispatch(spa->spa_metaslab_taskq, metaslab_preload,
 		    msp, TQ_SLEEP | (m <= mg->mg_allocators ? TQ_FRONT : 0))
 		    != TASKQID_INVALID);
 	}
 	mutex_exit(&mg->mg_lock);
 }
 
 /*
  * Determine if the space map's on-disk footprint is past our tolerance for
  * inefficiency. We would like to use the following criteria to make our
  * decision:
  *
  * 1. Do not condense if the size of the space map object would dramatically
  *    increase as a result of writing out the free space range tree.
  *
  * 2. Condense if the on on-disk space map representation is at least
  *    zfs_condense_pct/100 times the size of the optimal representation
  *    (i.e. zfs_condense_pct = 110 and in-core = 1MB, optimal = 1.1MB).
  *
  * 3. Do not condense if the on-disk size of the space map does not actually
  *    decrease.
  *
  * Unfortunately, we cannot compute the on-disk size of the space map in this
  * context because we cannot accurately compute the effects of compression, etc.
  * Instead, we apply the heuristic described in the block comment for
  * zfs_metaslab_condense_block_threshold - we only condense if the space used
  * is greater than a threshold number of blocks.
  */
 static boolean_t
 metaslab_should_condense(metaslab_t *msp)
 {
 	space_map_t *sm = msp->ms_sm;
 	vdev_t *vd = msp->ms_group->mg_vd;
 	uint64_t vdev_blocksize = 1ULL << vd->vdev_ashift;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	ASSERT(msp->ms_loaded);
 	ASSERT(sm != NULL);
 	ASSERT3U(spa_sync_pass(vd->vdev_spa), ==, 1);
 
 	/*
 	 * We always condense metaslabs that are empty and metaslabs for
 	 * which a condense request has been made.
 	 */
 	if (range_tree_numsegs(msp->ms_allocatable) == 0 ||
 	    msp->ms_condense_wanted)
 		return (B_TRUE);
 
 	uint64_t record_size = MAX(sm->sm_blksz, vdev_blocksize);
 	uint64_t object_size = space_map_length(sm);
 	uint64_t optimal_size = space_map_estimate_optimal_size(sm,
 	    msp->ms_allocatable, SM_NO_VDEVID);
 
 	return (object_size >= (optimal_size * zfs_condense_pct / 100) &&
 	    object_size > zfs_metaslab_condense_block_threshold * record_size);
 }
 
 /*
  * Condense the on-disk space map representation to its minimized form.
  * The minimized form consists of a small number of allocations followed
  * by the entries of the free range tree (ms_allocatable). The condensed
  * spacemap contains all the entries of previous TXGs (including those in
  * the pool-wide log spacemaps; thus this is effectively a superset of
  * metaslab_flush()), but this TXG's entries still need to be written.
  */
 static void
 metaslab_condense(metaslab_t *msp, dmu_tx_t *tx)
 {
 	range_tree_t *condense_tree;
 	space_map_t *sm = msp->ms_sm;
 	uint64_t txg = dmu_tx_get_txg(tx);
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	ASSERT(msp->ms_loaded);
 	ASSERT(msp->ms_sm != NULL);
 
 	/*
 	 * In order to condense the space map, we need to change it so it
 	 * only describes which segments are currently allocated and free.
 	 *
 	 * All the current free space resides in the ms_allocatable, all
 	 * the ms_defer trees, and all the ms_allocating trees. We ignore
 	 * ms_freed because it is empty because we're in sync pass 1. We
 	 * ignore ms_freeing because these changes are not yet reflected
 	 * in the spacemap (they will be written later this txg).
 	 *
 	 * So to truncate the space map to represent all the entries of
 	 * previous TXGs we do the following:
 	 *
 	 * 1] We create a range tree (condense tree) that is 100% empty.
 	 * 2] We add to it all segments found in the ms_defer trees
 	 *    as those segments are marked as free in the original space
 	 *    map. We do the same with the ms_allocating trees for the same
 	 *    reason. Adding these segments should be a relatively
 	 *    inexpensive operation since we expect these trees to have a
 	 *    small number of nodes.
 	 * 3] We vacate any unflushed allocs, since they are not frees we
 	 *    need to add to the condense tree. Then we vacate any
 	 *    unflushed frees as they should already be part of ms_allocatable.
 	 * 4] At this point, we would ideally like to add all segments
 	 *    in the ms_allocatable tree from the condense tree. This way
 	 *    we would write all the entries of the condense tree as the
 	 *    condensed space map, which would only contain freed
 	 *    segments with everything else assumed to be allocated.
 	 *
 	 *    Doing so can be prohibitively expensive as ms_allocatable can
 	 *    be large, and therefore computationally expensive to add to
 	 *    the condense_tree. Instead we first sync out an entry marking
 	 *    everything as allocated, then the condense_tree and then the
 	 *    ms_allocatable, in the condensed space map. While this is not
 	 *    optimal, it is typically close to optimal and more importantly
 	 *    much cheaper to compute.
 	 *
 	 * 5] Finally, as both of the unflushed trees were written to our
 	 *    new and condensed metaslab space map, we basically flushed
 	 *    all the unflushed changes to disk, thus we call
 	 *    metaslab_flush_update().
 	 */
 	ASSERT3U(spa_sync_pass(spa), ==, 1);
 	ASSERT(range_tree_is_empty(msp->ms_freed)); /* since it is pass 1 */
 
 	zfs_dbgmsg("condensing: txg %llu, msp[%llu] %px, vdev id %llu, "
 	    "spa %s, smp size %llu, segments %llu, forcing condense=%s",
 	    (u_longlong_t)txg, (u_longlong_t)msp->ms_id, msp,
 	    (u_longlong_t)msp->ms_group->mg_vd->vdev_id,
 	    spa->spa_name, (u_longlong_t)space_map_length(msp->ms_sm),
 	    (u_longlong_t)range_tree_numsegs(msp->ms_allocatable),
 	    msp->ms_condense_wanted ? "TRUE" : "FALSE");
 
 	msp->ms_condense_wanted = B_FALSE;
 
 	range_seg_type_t type;
 	uint64_t shift, start;
 	type = metaslab_calculate_range_tree_type(msp->ms_group->mg_vd, msp,
 	    &start, &shift);
 
 	condense_tree = range_tree_create(NULL, type, NULL, start, shift);
 
 	for (int t = 0; t < TXG_DEFER_SIZE; t++) {
 		range_tree_walk(msp->ms_defer[t],
 		    range_tree_add, condense_tree);
 	}
 
 	for (int t = 0; t < TXG_CONCURRENT_STATES; t++) {
 		range_tree_walk(msp->ms_allocating[(txg + t) & TXG_MASK],
 		    range_tree_add, condense_tree);
 	}
 
 	ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=,
 	    metaslab_unflushed_changes_memused(msp));
 	spa->spa_unflushed_stats.sus_memused -=
 	    metaslab_unflushed_changes_memused(msp);
 	range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL);
 	range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL);
 
 	/*
 	 * We're about to drop the metaslab's lock thus allowing other
 	 * consumers to change it's content. Set the metaslab's ms_condensing
 	 * flag to ensure that allocations on this metaslab do not occur
 	 * while we're in the middle of committing it to disk. This is only
 	 * critical for ms_allocatable as all other range trees use per TXG
 	 * views of their content.
 	 */
 	msp->ms_condensing = B_TRUE;
 
 	mutex_exit(&msp->ms_lock);
 	uint64_t object = space_map_object(msp->ms_sm);
 	space_map_truncate(sm,
 	    spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP) ?
 	    zfs_metaslab_sm_blksz_with_log : zfs_metaslab_sm_blksz_no_log, tx);
 
 	/*
 	 * space_map_truncate() may have reallocated the spacemap object.
 	 * If so, update the vdev_ms_array.
 	 */
 	if (space_map_object(msp->ms_sm) != object) {
 		object = space_map_object(msp->ms_sm);
 		dmu_write(spa->spa_meta_objset,
 		    msp->ms_group->mg_vd->vdev_ms_array, sizeof (uint64_t) *
 		    msp->ms_id, sizeof (uint64_t), &object, tx);
 	}
 
 	/*
 	 * Note:
 	 * When the log space map feature is enabled, each space map will
 	 * always have ALLOCS followed by FREES for each sync pass. This is
 	 * typically true even when the log space map feature is disabled,
 	 * except from the case where a metaslab goes through metaslab_sync()
 	 * and gets condensed. In that case the metaslab's space map will have
 	 * ALLOCS followed by FREES (due to condensing) followed by ALLOCS
 	 * followed by FREES (due to space_map_write() in metaslab_sync()) for
 	 * sync pass 1.
 	 */
 	range_tree_t *tmp_tree = range_tree_create(NULL, type, NULL, start,
 	    shift);
 	range_tree_add(tmp_tree, msp->ms_start, msp->ms_size);
 	space_map_write(sm, tmp_tree, SM_ALLOC, SM_NO_VDEVID, tx);
 	space_map_write(sm, msp->ms_allocatable, SM_FREE, SM_NO_VDEVID, tx);
 	space_map_write(sm, condense_tree, SM_FREE, SM_NO_VDEVID, tx);
 
 	range_tree_vacate(condense_tree, NULL, NULL);
 	range_tree_destroy(condense_tree);
 	range_tree_vacate(tmp_tree, NULL, NULL);
 	range_tree_destroy(tmp_tree);
 	mutex_enter(&msp->ms_lock);
 
 	msp->ms_condensing = B_FALSE;
 	metaslab_flush_update(msp, tx);
 }
 
 static void
 metaslab_unflushed_add(metaslab_t *msp, dmu_tx_t *tx)
 {
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 	ASSERT(spa_syncing_log_sm(spa) != NULL);
 	ASSERT(msp->ms_sm != NULL);
 	ASSERT(range_tree_is_empty(msp->ms_unflushed_allocs));
 	ASSERT(range_tree_is_empty(msp->ms_unflushed_frees));
 
 	mutex_enter(&spa->spa_flushed_ms_lock);
 	metaslab_set_unflushed_txg(msp, spa_syncing_txg(spa), tx);
 	metaslab_set_unflushed_dirty(msp, B_TRUE);
 	avl_add(&spa->spa_metaslabs_by_flushed, msp);
 	mutex_exit(&spa->spa_flushed_ms_lock);
 
 	spa_log_sm_increment_current_mscount(spa);
 	spa_log_summary_add_flushed_metaslab(spa, B_TRUE);
 }
 
 void
 metaslab_unflushed_bump(metaslab_t *msp, dmu_tx_t *tx, boolean_t dirty)
 {
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 	ASSERT(spa_syncing_log_sm(spa) != NULL);
 	ASSERT(msp->ms_sm != NULL);
 	ASSERT(metaslab_unflushed_txg(msp) != 0);
 	ASSERT3P(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL), ==, msp);
 	ASSERT(range_tree_is_empty(msp->ms_unflushed_allocs));
 	ASSERT(range_tree_is_empty(msp->ms_unflushed_frees));
 
 	VERIFY3U(tx->tx_txg, <=, spa_final_dirty_txg(spa));
 
 	/* update metaslab's position in our flushing tree */
 	uint64_t ms_prev_flushed_txg = metaslab_unflushed_txg(msp);
 	boolean_t ms_prev_flushed_dirty = metaslab_unflushed_dirty(msp);
 	mutex_enter(&spa->spa_flushed_ms_lock);
 	avl_remove(&spa->spa_metaslabs_by_flushed, msp);
 	metaslab_set_unflushed_txg(msp, spa_syncing_txg(spa), tx);
 	metaslab_set_unflushed_dirty(msp, dirty);
 	avl_add(&spa->spa_metaslabs_by_flushed, msp);
 	mutex_exit(&spa->spa_flushed_ms_lock);
 
 	/* update metaslab counts of spa_log_sm_t nodes */
 	spa_log_sm_decrement_mscount(spa, ms_prev_flushed_txg);
 	spa_log_sm_increment_current_mscount(spa);
 
 	/* update log space map summary */
 	spa_log_summary_decrement_mscount(spa, ms_prev_flushed_txg,
 	    ms_prev_flushed_dirty);
 	spa_log_summary_add_flushed_metaslab(spa, dirty);
 
 	/* cleanup obsolete logs if any */
 	spa_cleanup_old_sm_logs(spa, tx);
 }
 
 /*
  * Called when the metaslab has been flushed (its own spacemap now reflects
  * all the contents of the pool-wide spacemap log). Updates the metaslab's
  * metadata and any pool-wide related log space map data (e.g. summary,
  * obsolete logs, etc..) to reflect that.
  */
 static void
 metaslab_flush_update(metaslab_t *msp, dmu_tx_t *tx)
 {
 	metaslab_group_t *mg = msp->ms_group;
 	spa_t *spa = mg->mg_vd->vdev_spa;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	ASSERT3U(spa_sync_pass(spa), ==, 1);
 
 	/*
 	 * Just because a metaslab got flushed, that doesn't mean that
 	 * it will pass through metaslab_sync_done(). Thus, make sure to
 	 * update ms_synced_length here in case it doesn't.
 	 */
 	msp->ms_synced_length = space_map_length(msp->ms_sm);
 
 	/*
 	 * We may end up here from metaslab_condense() without the
 	 * feature being active. In that case this is a no-op.
 	 */
 	if (!spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP) ||
 	    metaslab_unflushed_txg(msp) == 0)
 		return;
 
 	metaslab_unflushed_bump(msp, tx, B_FALSE);
 }
 
 boolean_t
 metaslab_flush(metaslab_t *msp, dmu_tx_t *tx)
 {
 	spa_t *spa = msp->ms_group->mg_vd->vdev_spa;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	ASSERT3U(spa_sync_pass(spa), ==, 1);
 	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));
 
 	ASSERT(msp->ms_sm != NULL);
 	ASSERT(metaslab_unflushed_txg(msp) != 0);
 	ASSERT(avl_find(&spa->spa_metaslabs_by_flushed, msp, NULL) != NULL);
 
 	/*
 	 * There is nothing wrong with flushing the same metaslab twice, as
 	 * this codepath should work on that case. However, the current
 	 * flushing scheme makes sure to avoid this situation as we would be
 	 * making all these calls without having anything meaningful to write
 	 * to disk. We assert this behavior here.
 	 */
 	ASSERT3U(metaslab_unflushed_txg(msp), <, dmu_tx_get_txg(tx));
 
 	/*
 	 * We can not flush while loading, because then we would
 	 * not load the ms_unflushed_{allocs,frees}.
 	 */
 	if (msp->ms_loading)
 		return (B_FALSE);
 
 	metaslab_verify_space(msp, dmu_tx_get_txg(tx));
 	metaslab_verify_weight_and_frag(msp);
 
 	/*
 	 * Metaslab condensing is effectively flushing. Therefore if the
 	 * metaslab can be condensed we can just condense it instead of
 	 * flushing it.
 	 *
 	 * Note that metaslab_condense() does call metaslab_flush_update()
 	 * so we can just return immediately after condensing. We also
 	 * don't need to care about setting ms_flushing or broadcasting
 	 * ms_flush_cv, even if we temporarily drop the ms_lock in
 	 * metaslab_condense(), as the metaslab is already loaded.
 	 */
 	if (msp->ms_loaded && metaslab_should_condense(msp)) {
 		metaslab_group_t *mg = msp->ms_group;
 
 		/*
 		 * For all histogram operations below refer to the
 		 * comments of metaslab_sync() where we follow a
 		 * similar procedure.
 		 */
 		metaslab_group_histogram_verify(mg);
 		metaslab_class_histogram_verify(mg->mg_class);
 		metaslab_group_histogram_remove(mg, msp);
 
 		metaslab_condense(msp, tx);
 
 		space_map_histogram_clear(msp->ms_sm);
 		space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx);
 		ASSERT(range_tree_is_empty(msp->ms_freed));
 		for (int t = 0; t < TXG_DEFER_SIZE; t++) {
 			space_map_histogram_add(msp->ms_sm,
 			    msp->ms_defer[t], tx);
 		}
 		metaslab_aux_histograms_update(msp);
 
 		metaslab_group_histogram_add(mg, msp);
 		metaslab_group_histogram_verify(mg);
 		metaslab_class_histogram_verify(mg->mg_class);
 
 		metaslab_verify_space(msp, dmu_tx_get_txg(tx));
 
 		/*
 		 * Since we recreated the histogram (and potentially
 		 * the ms_sm too while condensing) ensure that the
 		 * weight is updated too because we are not guaranteed
 		 * that this metaslab is dirty and will go through
 		 * metaslab_sync_done().
 		 */
 		metaslab_recalculate_weight_and_sort(msp);
 		return (B_TRUE);
 	}
 
 	msp->ms_flushing = B_TRUE;
 	uint64_t sm_len_before = space_map_length(msp->ms_sm);
 
 	mutex_exit(&msp->ms_lock);
 	space_map_write(msp->ms_sm, msp->ms_unflushed_allocs, SM_ALLOC,
 	    SM_NO_VDEVID, tx);
 	space_map_write(msp->ms_sm, msp->ms_unflushed_frees, SM_FREE,
 	    SM_NO_VDEVID, tx);
 	mutex_enter(&msp->ms_lock);
 
 	uint64_t sm_len_after = space_map_length(msp->ms_sm);
 	if (zfs_flags & ZFS_DEBUG_LOG_SPACEMAP) {
 		zfs_dbgmsg("flushing: txg %llu, spa %s, vdev_id %llu, "
 		    "ms_id %llu, unflushed_allocs %llu, unflushed_frees %llu, "
 		    "appended %llu bytes", (u_longlong_t)dmu_tx_get_txg(tx),
 		    spa_name(spa),
 		    (u_longlong_t)msp->ms_group->mg_vd->vdev_id,
 		    (u_longlong_t)msp->ms_id,
 		    (u_longlong_t)range_tree_space(msp->ms_unflushed_allocs),
 		    (u_longlong_t)range_tree_space(msp->ms_unflushed_frees),
 		    (u_longlong_t)(sm_len_after - sm_len_before));
 	}
 
 	ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=,
 	    metaslab_unflushed_changes_memused(msp));
 	spa->spa_unflushed_stats.sus_memused -=
 	    metaslab_unflushed_changes_memused(msp);
 	range_tree_vacate(msp->ms_unflushed_allocs, NULL, NULL);
 	range_tree_vacate(msp->ms_unflushed_frees, NULL, NULL);
 
 	metaslab_verify_space(msp, dmu_tx_get_txg(tx));
 	metaslab_verify_weight_and_frag(msp);
 
 	metaslab_flush_update(msp, tx);
 
 	metaslab_verify_space(msp, dmu_tx_get_txg(tx));
 	metaslab_verify_weight_and_frag(msp);
 
 	msp->ms_flushing = B_FALSE;
 	cv_broadcast(&msp->ms_flush_cv);
 	return (B_TRUE);
 }
 
 /*
  * Write a metaslab to disk in the context of the specified transaction group.
  */
 void
 metaslab_sync(metaslab_t *msp, uint64_t txg)
 {
 	metaslab_group_t *mg = msp->ms_group;
 	vdev_t *vd = mg->mg_vd;
 	spa_t *spa = vd->vdev_spa;
 	objset_t *mos = spa_meta_objset(spa);
 	range_tree_t *alloctree = msp->ms_allocating[txg & TXG_MASK];
 	dmu_tx_t *tx;
 
 	ASSERT(!vd->vdev_ishole);
 
 	/*
 	 * This metaslab has just been added so there's no work to do now.
 	 */
 	if (msp->ms_new) {
 		ASSERT0(range_tree_space(alloctree));
 		ASSERT0(range_tree_space(msp->ms_freeing));
 		ASSERT0(range_tree_space(msp->ms_freed));
 		ASSERT0(range_tree_space(msp->ms_checkpointing));
 		ASSERT0(range_tree_space(msp->ms_trim));
 		return;
 	}
 
 	/*
 	 * Normally, we don't want to process a metaslab if there are no
 	 * allocations or frees to perform. However, if the metaslab is being
 	 * forced to condense, it's loaded and we're not beyond the final
 	 * dirty txg, we need to let it through. Not condensing beyond the
 	 * final dirty txg prevents an issue where metaslabs that need to be
 	 * condensed but were loaded for other reasons could cause a panic
 	 * here. By only checking the txg in that branch of the conditional,
 	 * we preserve the utility of the VERIFY statements in all other
 	 * cases.
 	 */
 	if (range_tree_is_empty(alloctree) &&
 	    range_tree_is_empty(msp->ms_freeing) &&
 	    range_tree_is_empty(msp->ms_checkpointing) &&
 	    !(msp->ms_loaded && msp->ms_condense_wanted &&
 	    txg <= spa_final_dirty_txg(spa)))
 		return;
 
 
 	VERIFY3U(txg, <=, spa_final_dirty_txg(spa));
 
 	/*
 	 * The only state that can actually be changing concurrently
 	 * with metaslab_sync() is the metaslab's ms_allocatable. No
 	 * other thread can be modifying this txg's alloc, freeing,
 	 * freed, or space_map_phys_t.  We drop ms_lock whenever we
 	 * could call into the DMU, because the DMU can call down to
 	 * us (e.g. via zio_free()) at any time.
 	 *
 	 * The spa_vdev_remove_thread() can be reading metaslab state
 	 * concurrently, and it is locked out by the ms_sync_lock.
 	 * Note that the ms_lock is insufficient for this, because it
 	 * is dropped by space_map_write().
 	 */
 	tx = dmu_tx_create_assigned(spa_get_dsl(spa), txg);
 
 	/*
 	 * Generate a log space map if one doesn't exist already.
 	 */
 	spa_generate_syncing_log_sm(spa, tx);
 
 	if (msp->ms_sm == NULL) {
 		uint64_t new_object = space_map_alloc(mos,
 		    spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP) ?
 		    zfs_metaslab_sm_blksz_with_log :
 		    zfs_metaslab_sm_blksz_no_log, tx);
 		VERIFY3U(new_object, !=, 0);
 
 		dmu_write(mos, vd->vdev_ms_array, sizeof (uint64_t) *
 		    msp->ms_id, sizeof (uint64_t), &new_object, tx);
 
 		VERIFY0(space_map_open(&msp->ms_sm, mos, new_object,
 		    msp->ms_start, msp->ms_size, vd->vdev_ashift));
 		ASSERT(msp->ms_sm != NULL);
 
 		ASSERT(range_tree_is_empty(msp->ms_unflushed_allocs));
 		ASSERT(range_tree_is_empty(msp->ms_unflushed_frees));
 		ASSERT0(metaslab_allocated_space(msp));
 	}
 
 	if (!range_tree_is_empty(msp->ms_checkpointing) &&
 	    vd->vdev_checkpoint_sm == NULL) {
 		ASSERT(spa_has_checkpoint(spa));
 
 		uint64_t new_object = space_map_alloc(mos,
 		    zfs_vdev_standard_sm_blksz, tx);
 		VERIFY3U(new_object, !=, 0);
 
 		VERIFY0(space_map_open(&vd->vdev_checkpoint_sm,
 		    mos, new_object, 0, vd->vdev_asize, vd->vdev_ashift));
 		ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL);
 
 		/*
 		 * We save the space map object as an entry in vdev_top_zap
 		 * so it can be retrieved when the pool is reopened after an
 		 * export or through zdb.
 		 */
 		VERIFY0(zap_add(vd->vdev_spa->spa_meta_objset,
 		    vd->vdev_top_zap, VDEV_TOP_ZAP_POOL_CHECKPOINT_SM,
 		    sizeof (new_object), 1, &new_object, tx));
 	}
 
 	mutex_enter(&msp->ms_sync_lock);
 	mutex_enter(&msp->ms_lock);
 
 	/*
 	 * Note: metaslab_condense() clears the space map's histogram.
 	 * Therefore we must verify and remove this histogram before
 	 * condensing.
 	 */
 	metaslab_group_histogram_verify(mg);
 	metaslab_class_histogram_verify(mg->mg_class);
 	metaslab_group_histogram_remove(mg, msp);
 
 	if (spa->spa_sync_pass == 1 && msp->ms_loaded &&
 	    metaslab_should_condense(msp))
 		metaslab_condense(msp, tx);
 
 	/*
 	 * We'll be going to disk to sync our space accounting, thus we
 	 * drop the ms_lock during that time so allocations coming from
 	 * open-context (ZIL) for future TXGs do not block.
 	 */
 	mutex_exit(&msp->ms_lock);
 	space_map_t *log_sm = spa_syncing_log_sm(spa);
 	if (log_sm != NULL) {
 		ASSERT(spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP));
 		if (metaslab_unflushed_txg(msp) == 0)
 			metaslab_unflushed_add(msp, tx);
 		else if (!metaslab_unflushed_dirty(msp))
 			metaslab_unflushed_bump(msp, tx, B_TRUE);
 
 		space_map_write(log_sm, alloctree, SM_ALLOC,
 		    vd->vdev_id, tx);
 		space_map_write(log_sm, msp->ms_freeing, SM_FREE,
 		    vd->vdev_id, tx);
 		mutex_enter(&msp->ms_lock);
 
 		ASSERT3U(spa->spa_unflushed_stats.sus_memused, >=,
 		    metaslab_unflushed_changes_memused(msp));
 		spa->spa_unflushed_stats.sus_memused -=
 		    metaslab_unflushed_changes_memused(msp);
 		range_tree_remove_xor_add(alloctree,
 		    msp->ms_unflushed_frees, msp->ms_unflushed_allocs);
 		range_tree_remove_xor_add(msp->ms_freeing,
 		    msp->ms_unflushed_allocs, msp->ms_unflushed_frees);
 		spa->spa_unflushed_stats.sus_memused +=
 		    metaslab_unflushed_changes_memused(msp);
 	} else {
 		ASSERT(!spa_feature_is_enabled(spa, SPA_FEATURE_LOG_SPACEMAP));
 
 		space_map_write(msp->ms_sm, alloctree, SM_ALLOC,
 		    SM_NO_VDEVID, tx);
 		space_map_write(msp->ms_sm, msp->ms_freeing, SM_FREE,
 		    SM_NO_VDEVID, tx);
 		mutex_enter(&msp->ms_lock);
 	}
 
 	msp->ms_allocated_space += range_tree_space(alloctree);
 	ASSERT3U(msp->ms_allocated_space, >=,
 	    range_tree_space(msp->ms_freeing));
 	msp->ms_allocated_space -= range_tree_space(msp->ms_freeing);
 
 	if (!range_tree_is_empty(msp->ms_checkpointing)) {
 		ASSERT(spa_has_checkpoint(spa));
 		ASSERT3P(vd->vdev_checkpoint_sm, !=, NULL);
 
 		/*
 		 * Since we are doing writes to disk and the ms_checkpointing
 		 * tree won't be changing during that time, we drop the
 		 * ms_lock while writing to the checkpoint space map, for the
 		 * same reason mentioned above.
 		 */
 		mutex_exit(&msp->ms_lock);
 		space_map_write(vd->vdev_checkpoint_sm,
 		    msp->ms_checkpointing, SM_FREE, SM_NO_VDEVID, tx);
 		mutex_enter(&msp->ms_lock);
 
 		spa->spa_checkpoint_info.sci_dspace +=
 		    range_tree_space(msp->ms_checkpointing);
 		vd->vdev_stat.vs_checkpoint_space +=
 		    range_tree_space(msp->ms_checkpointing);
 		ASSERT3U(vd->vdev_stat.vs_checkpoint_space, ==,
 		    -space_map_allocated(vd->vdev_checkpoint_sm));
 
 		range_tree_vacate(msp->ms_checkpointing, NULL, NULL);
 	}
 
 	if (msp->ms_loaded) {
 		/*
 		 * When the space map is loaded, we have an accurate
 		 * histogram in the range tree. This gives us an opportunity
 		 * to bring the space map's histogram up-to-date so we clear
 		 * it first before updating it.
 		 */
 		space_map_histogram_clear(msp->ms_sm);
 		space_map_histogram_add(msp->ms_sm, msp->ms_allocatable, tx);
 
 		/*
 		 * Since we've cleared the histogram we need to add back
 		 * any free space that has already been processed, plus
 		 * any deferred space. This allows the on-disk histogram
 		 * to accurately reflect all free space even if some space
 		 * is not yet available for allocation (i.e. deferred).
 		 */
 		space_map_histogram_add(msp->ms_sm, msp->ms_freed, tx);
 
 		/*
 		 * Add back any deferred free space that has not been
 		 * added back into the in-core free tree yet. This will
 		 * ensure that we don't end up with a space map histogram
 		 * that is completely empty unless the metaslab is fully
 		 * allocated.
 		 */
 		for (int t = 0; t < TXG_DEFER_SIZE; t++) {
 			space_map_histogram_add(msp->ms_sm,
 			    msp->ms_defer[t], tx);
 		}
 	}
 
 	/*
 	 * Always add the free space from this sync pass to the space
 	 * map histogram. We want to make sure that the on-disk histogram
 	 * accounts for all free space. If the space map is not loaded,
 	 * then we will lose some accuracy but will correct it the next
 	 * time we load the space map.
 	 */
 	space_map_histogram_add(msp->ms_sm, msp->ms_freeing, tx);
 	metaslab_aux_histograms_update(msp);
 
 	metaslab_group_histogram_add(mg, msp);
 	metaslab_group_histogram_verify(mg);
 	metaslab_class_histogram_verify(mg->mg_class);
 
 	/*
 	 * For sync pass 1, we avoid traversing this txg's free range tree
 	 * and instead will just swap the pointers for freeing and freed.
 	 * We can safely do this since the freed_tree is guaranteed to be
 	 * empty on the initial pass.
 	 *
 	 * Keep in mind that even if we are currently using a log spacemap
 	 * we want current frees to end up in the ms_allocatable (but not
 	 * get appended to the ms_sm) so their ranges can be reused as usual.
 	 */
 	if (spa_sync_pass(spa) == 1) {
 		range_tree_swap(&msp->ms_freeing, &msp->ms_freed);
 		ASSERT0(msp->ms_allocated_this_txg);
 	} else {
 		range_tree_vacate(msp->ms_freeing,
 		    range_tree_add, msp->ms_freed);
 	}
 	msp->ms_allocated_this_txg += range_tree_space(alloctree);
 	range_tree_vacate(alloctree, NULL, NULL);
 
 	ASSERT0(range_tree_space(msp->ms_allocating[txg & TXG_MASK]));
 	ASSERT0(range_tree_space(msp->ms_allocating[TXG_CLEAN(txg)
 	    & TXG_MASK]));
 	ASSERT0(range_tree_space(msp->ms_freeing));
 	ASSERT0(range_tree_space(msp->ms_checkpointing));
 
 	mutex_exit(&msp->ms_lock);
 
 	/*
 	 * Verify that the space map object ID has been recorded in the
 	 * vdev_ms_array.
 	 */
 	uint64_t object;
 	VERIFY0(dmu_read(mos, vd->vdev_ms_array,
 	    msp->ms_id * sizeof (uint64_t), sizeof (uint64_t), &object, 0));
 	VERIFY3U(object, ==, space_map_object(msp->ms_sm));
 
 	mutex_exit(&msp->ms_sync_lock);
 	dmu_tx_commit(tx);
 }
 
 static void
 metaslab_evict(metaslab_t *msp, uint64_t txg)
 {
 	if (!msp->ms_loaded || msp->ms_disabled != 0)
 		return;
 
 	for (int t = 1; t < TXG_CONCURRENT_STATES; t++) {
 		VERIFY0(range_tree_space(
 		    msp->ms_allocating[(txg + t) & TXG_MASK]));
 	}
 	if (msp->ms_allocator != -1)
 		metaslab_passivate(msp, msp->ms_weight & ~METASLAB_ACTIVE_MASK);
 
 	if (!metaslab_debug_unload)
 		metaslab_unload(msp);
 }
 
 /*
  * Called after a transaction group has completely synced to mark
  * all of the metaslab's free space as usable.
  */
 void
 metaslab_sync_done(metaslab_t *msp, uint64_t txg)
 {
 	metaslab_group_t *mg = msp->ms_group;
 	vdev_t *vd = mg->mg_vd;
 	spa_t *spa = vd->vdev_spa;
 	range_tree_t **defer_tree;
 	int64_t alloc_delta, defer_delta;
 	boolean_t defer_allowed = B_TRUE;
 
 	ASSERT(!vd->vdev_ishole);
 
 	mutex_enter(&msp->ms_lock);
 
 	if (msp->ms_new) {
 		/* this is a new metaslab, add its capacity to the vdev */
 		metaslab_space_update(vd, mg->mg_class, 0, 0, msp->ms_size);
 
 		/* there should be no allocations nor frees at this point */
 		VERIFY0(msp->ms_allocated_this_txg);
 		VERIFY0(range_tree_space(msp->ms_freed));
 	}
 
 	ASSERT0(range_tree_space(msp->ms_freeing));
 	ASSERT0(range_tree_space(msp->ms_checkpointing));
 
 	defer_tree = &msp->ms_defer[txg % TXG_DEFER_SIZE];
 
 	uint64_t free_space = metaslab_class_get_space(spa_normal_class(spa)) -
 	    metaslab_class_get_alloc(spa_normal_class(spa));
 	if (free_space <= spa_get_slop_space(spa) || vd->vdev_removing) {
 		defer_allowed = B_FALSE;
 	}
 
 	defer_delta = 0;
 	alloc_delta = msp->ms_allocated_this_txg -
 	    range_tree_space(msp->ms_freed);
 
 	if (defer_allowed) {
 		defer_delta = range_tree_space(msp->ms_freed) -
 		    range_tree_space(*defer_tree);
 	} else {
 		defer_delta -= range_tree_space(*defer_tree);
 	}
 	metaslab_space_update(vd, mg->mg_class, alloc_delta + defer_delta,
 	    defer_delta, 0);
 
 	if (spa_syncing_log_sm(spa) == NULL) {
 		/*
 		 * If there's a metaslab_load() in progress and we don't have
 		 * a log space map, it means that we probably wrote to the
 		 * metaslab's space map. If this is the case, we need to
 		 * make sure that we wait for the load to complete so that we
 		 * have a consistent view at the in-core side of the metaslab.
 		 */
 		metaslab_load_wait(msp);
 	} else {
 		ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));
 	}
 
 	/*
 	 * When auto-trimming is enabled, free ranges which are added to
 	 * ms_allocatable are also be added to ms_trim.  The ms_trim tree is
 	 * periodically consumed by the vdev_autotrim_thread() which issues
 	 * trims for all ranges and then vacates the tree.  The ms_trim tree
 	 * can be discarded at any time with the sole consequence of recent
 	 * frees not being trimmed.
 	 */
 	if (spa_get_autotrim(spa) == SPA_AUTOTRIM_ON) {
 		range_tree_walk(*defer_tree, range_tree_add, msp->ms_trim);
 		if (!defer_allowed) {
 			range_tree_walk(msp->ms_freed, range_tree_add,
 			    msp->ms_trim);
 		}
 	} else {
 		range_tree_vacate(msp->ms_trim, NULL, NULL);
 	}
 
 	/*
 	 * Move the frees from the defer_tree back to the free
 	 * range tree (if it's loaded). Swap the freed_tree and
 	 * the defer_tree -- this is safe to do because we've
 	 * just emptied out the defer_tree.
 	 */
 	range_tree_vacate(*defer_tree,
 	    msp->ms_loaded ? range_tree_add : NULL, msp->ms_allocatable);
 	if (defer_allowed) {
 		range_tree_swap(&msp->ms_freed, defer_tree);
 	} else {
 		range_tree_vacate(msp->ms_freed,
 		    msp->ms_loaded ? range_tree_add : NULL,
 		    msp->ms_allocatable);
 	}
 
 	msp->ms_synced_length = space_map_length(msp->ms_sm);
 
 	msp->ms_deferspace += defer_delta;
 	ASSERT3S(msp->ms_deferspace, >=, 0);
 	ASSERT3S(msp->ms_deferspace, <=, msp->ms_size);
 	if (msp->ms_deferspace != 0) {
 		/*
 		 * Keep syncing this metaslab until all deferred frees
 		 * are back in circulation.
 		 */
 		vdev_dirty(vd, VDD_METASLAB, msp, txg + 1);
 	}
 	metaslab_aux_histograms_update_done(msp, defer_allowed);
 
 	if (msp->ms_new) {
 		msp->ms_new = B_FALSE;
 		mutex_enter(&mg->mg_lock);
 		mg->mg_ms_ready++;
 		mutex_exit(&mg->mg_lock);
 	}
 
 	/*
 	 * Re-sort metaslab within its group now that we've adjusted
 	 * its allocatable space.
 	 */
 	metaslab_recalculate_weight_and_sort(msp);
 
 	ASSERT0(range_tree_space(msp->ms_allocating[txg & TXG_MASK]));
 	ASSERT0(range_tree_space(msp->ms_freeing));
 	ASSERT0(range_tree_space(msp->ms_freed));
 	ASSERT0(range_tree_space(msp->ms_checkpointing));
 	msp->ms_allocating_total -= msp->ms_allocated_this_txg;
 	msp->ms_allocated_this_txg = 0;
 	mutex_exit(&msp->ms_lock);
 }
 
 void
 metaslab_sync_reassess(metaslab_group_t *mg)
 {
 	spa_t *spa = mg->mg_class->mc_spa;
 
 	spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER);
 	metaslab_group_alloc_update(mg);
 	mg->mg_fragmentation = metaslab_group_fragmentation(mg);
 
 	/*
 	 * Preload the next potential metaslabs but only on active
 	 * metaslab groups. We can get into a state where the metaslab
 	 * is no longer active since we dirty metaslabs as we remove a
 	 * a device, thus potentially making the metaslab group eligible
 	 * for preloading.
 	 */
 	if (mg->mg_activation_count > 0) {
 		metaslab_group_preload(mg);
 	}
 	spa_config_exit(spa, SCL_ALLOC, FTAG);
 }
 
 /*
  * When writing a ditto block (i.e. more than one DVA for a given BP) on
  * the same vdev as an existing DVA of this BP, then try to allocate it
  * on a different metaslab than existing DVAs (i.e. a unique metaslab).
  */
 static boolean_t
 metaslab_is_unique(metaslab_t *msp, dva_t *dva)
 {
 	uint64_t dva_ms_id;
 
 	if (DVA_GET_ASIZE(dva) == 0)
 		return (B_TRUE);
 
 	if (msp->ms_group->mg_vd->vdev_id != DVA_GET_VDEV(dva))
 		return (B_TRUE);
 
 	dva_ms_id = DVA_GET_OFFSET(dva) >> msp->ms_group->mg_vd->vdev_ms_shift;
 
 	return (msp->ms_id != dva_ms_id);
 }
 
 /*
  * ==========================================================================
  * Metaslab allocation tracing facility
  * ==========================================================================
  */
 
 /*
  * Add an allocation trace element to the allocation tracing list.
  */
 static void
 metaslab_trace_add(zio_alloc_list_t *zal, metaslab_group_t *mg,
     metaslab_t *msp, uint64_t psize, uint32_t dva_id, uint64_t offset,
     int allocator)
 {
 	metaslab_alloc_trace_t *mat;
 
 	if (!metaslab_trace_enabled)
 		return;
 
 	/*
 	 * When the tracing list reaches its maximum we remove
 	 * the second element in the list before adding a new one.
 	 * By removing the second element we preserve the original
 	 * entry as a clue to what allocations steps have already been
 	 * performed.
 	 */
 	if (zal->zal_size == metaslab_trace_max_entries) {
 		metaslab_alloc_trace_t *mat_next;
 #ifdef ZFS_DEBUG
 		panic("too many entries in allocation list");
 #endif
 		METASLABSTAT_BUMP(metaslabstat_trace_over_limit);
 		zal->zal_size--;
 		mat_next = list_next(&zal->zal_list, list_head(&zal->zal_list));
 		list_remove(&zal->zal_list, mat_next);
 		kmem_cache_free(metaslab_alloc_trace_cache, mat_next);
 	}
 
 	mat = kmem_cache_alloc(metaslab_alloc_trace_cache, KM_SLEEP);
 	list_link_init(&mat->mat_list_node);
 	mat->mat_mg = mg;
 	mat->mat_msp = msp;
 	mat->mat_size = psize;
 	mat->mat_dva_id = dva_id;
 	mat->mat_offset = offset;
 	mat->mat_weight = 0;
 	mat->mat_allocator = allocator;
 
 	if (msp != NULL)
 		mat->mat_weight = msp->ms_weight;
 
 	/*
 	 * The list is part of the zio so locking is not required. Only
 	 * a single thread will perform allocations for a given zio.
 	 */
 	list_insert_tail(&zal->zal_list, mat);
 	zal->zal_size++;
 
 	ASSERT3U(zal->zal_size, <=, metaslab_trace_max_entries);
 }
 
 void
 metaslab_trace_init(zio_alloc_list_t *zal)
 {
 	list_create(&zal->zal_list, sizeof (metaslab_alloc_trace_t),
 	    offsetof(metaslab_alloc_trace_t, mat_list_node));
 	zal->zal_size = 0;
 }
 
 void
 metaslab_trace_fini(zio_alloc_list_t *zal)
 {
 	metaslab_alloc_trace_t *mat;
 
 	while ((mat = list_remove_head(&zal->zal_list)) != NULL)
 		kmem_cache_free(metaslab_alloc_trace_cache, mat);
 	list_destroy(&zal->zal_list);
 	zal->zal_size = 0;
 }
 
 /*
  * ==========================================================================
  * Metaslab block operations
  * ==========================================================================
  */
 
 static void
 metaslab_group_alloc_increment(spa_t *spa, uint64_t vdev, const void *tag,
     int flags, int allocator)
 {
 	if (!(flags & METASLAB_ASYNC_ALLOC) ||
 	    (flags & METASLAB_DONT_THROTTLE))
 		return;
 
 	metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg;
 	if (!mg->mg_class->mc_alloc_throttle_enabled)
 		return;
 
 	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
 	(void) zfs_refcount_add(&mga->mga_alloc_queue_depth, tag);
 }
 
 static void
 metaslab_group_increment_qdepth(metaslab_group_t *mg, int allocator)
 {
 	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
 	metaslab_class_allocator_t *mca =
 	    &mg->mg_class->mc_allocator[allocator];
 	uint64_t max = mg->mg_max_alloc_queue_depth;
 	uint64_t cur = mga->mga_cur_max_alloc_queue_depth;
 	while (cur < max) {
 		if (atomic_cas_64(&mga->mga_cur_max_alloc_queue_depth,
 		    cur, cur + 1) == cur) {
 			atomic_inc_64(&mca->mca_alloc_max_slots);
 			return;
 		}
 		cur = mga->mga_cur_max_alloc_queue_depth;
 	}
 }
 
 void
 metaslab_group_alloc_decrement(spa_t *spa, uint64_t vdev, const void *tag,
     int flags, int allocator, boolean_t io_complete)
 {
 	if (!(flags & METASLAB_ASYNC_ALLOC) ||
 	    (flags & METASLAB_DONT_THROTTLE))
 		return;
 
 	metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg;
 	if (!mg->mg_class->mc_alloc_throttle_enabled)
 		return;
 
 	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
 	(void) zfs_refcount_remove(&mga->mga_alloc_queue_depth, tag);
 	if (io_complete)
 		metaslab_group_increment_qdepth(mg, allocator);
 }
 
 void
 metaslab_group_alloc_verify(spa_t *spa, const blkptr_t *bp, const void *tag,
     int allocator)
 {
 #ifdef ZFS_DEBUG
 	const dva_t *dva = bp->blk_dva;
 	int ndvas = BP_GET_NDVAS(bp);
 
 	for (int d = 0; d < ndvas; d++) {
 		uint64_t vdev = DVA_GET_VDEV(&dva[d]);
 		metaslab_group_t *mg = vdev_lookup_top(spa, vdev)->vdev_mg;
 		metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
 		VERIFY(zfs_refcount_not_held(&mga->mga_alloc_queue_depth, tag));
 	}
 #endif
 }
 
 static uint64_t
 metaslab_block_alloc(metaslab_t *msp, uint64_t size, uint64_t txg)
 {
 	uint64_t start;
 	range_tree_t *rt = msp->ms_allocatable;
 	metaslab_class_t *mc = msp->ms_group->mg_class;
 
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 	VERIFY(!msp->ms_condensing);
 	VERIFY0(msp->ms_disabled);
 
 	start = mc->mc_ops->msop_alloc(msp, size);
 	if (start != -1ULL) {
 		metaslab_group_t *mg = msp->ms_group;
 		vdev_t *vd = mg->mg_vd;
 
 		VERIFY0(P2PHASE(start, 1ULL << vd->vdev_ashift));
 		VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift));
 		VERIFY3U(range_tree_space(rt) - size, <=, msp->ms_size);
 		range_tree_remove(rt, start, size);
 		range_tree_clear(msp->ms_trim, start, size);
 
 		if (range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK]))
 			vdev_dirty(mg->mg_vd, VDD_METASLAB, msp, txg);
 
 		range_tree_add(msp->ms_allocating[txg & TXG_MASK], start, size);
 		msp->ms_allocating_total += size;
 
 		/* Track the last successful allocation */
 		msp->ms_alloc_txg = txg;
 		metaslab_verify_space(msp, txg);
 	}
 
 	/*
 	 * Now that we've attempted the allocation we need to update the
 	 * metaslab's maximum block size since it may have changed.
 	 */
 	msp->ms_max_size = metaslab_largest_allocatable(msp);
 	return (start);
 }
 
 /*
  * Find the metaslab with the highest weight that is less than what we've
  * already tried.  In the common case, this means that we will examine each
  * metaslab at most once. Note that concurrent callers could reorder metaslabs
  * by activation/passivation once we have dropped the mg_lock. If a metaslab is
  * activated by another thread, and we fail to allocate from the metaslab we
  * have selected, we may not try the newly-activated metaslab, and instead
  * activate another metaslab.  This is not optimal, but generally does not cause
  * any problems (a possible exception being if every metaslab is completely full
  * except for the newly-activated metaslab which we fail to examine).
  */
 static metaslab_t *
 find_valid_metaslab(metaslab_group_t *mg, uint64_t activation_weight,
     dva_t *dva, int d, boolean_t want_unique, uint64_t asize, int allocator,
     boolean_t try_hard, zio_alloc_list_t *zal, metaslab_t *search,
     boolean_t *was_active)
 {
 	avl_index_t idx;
 	avl_tree_t *t = &mg->mg_metaslab_tree;
 	metaslab_t *msp = avl_find(t, search, &idx);
 	if (msp == NULL)
 		msp = avl_nearest(t, idx, AVL_AFTER);
 
 	uint_t tries = 0;
 	for (; msp != NULL; msp = AVL_NEXT(t, msp)) {
 		int i;
 
 		if (!try_hard && tries > zfs_metaslab_find_max_tries) {
 			METASLABSTAT_BUMP(metaslabstat_too_many_tries);
 			return (NULL);
 		}
 		tries++;
 
 		if (!metaslab_should_allocate(msp, asize, try_hard)) {
 			metaslab_trace_add(zal, mg, msp, asize, d,
 			    TRACE_TOO_SMALL, allocator);
 			continue;
 		}
 
 		/*
 		 * If the selected metaslab is condensing or disabled,
 		 * skip it.
 		 */
 		if (msp->ms_condensing || msp->ms_disabled > 0)
 			continue;
 
 		*was_active = msp->ms_allocator != -1;
 		/*
 		 * If we're activating as primary, this is our first allocation
 		 * from this disk, so we don't need to check how close we are.
 		 * If the metaslab under consideration was already active,
 		 * we're getting desperate enough to steal another allocator's
 		 * metaslab, so we still don't care about distances.
 		 */
 		if (activation_weight == METASLAB_WEIGHT_PRIMARY || *was_active)
 			break;
 
 		for (i = 0; i < d; i++) {
 			if (want_unique &&
 			    !metaslab_is_unique(msp, &dva[i]))
 				break;  /* try another metaslab */
 		}
 		if (i == d)
 			break;
 	}
 
 	if (msp != NULL) {
 		search->ms_weight = msp->ms_weight;
 		search->ms_start = msp->ms_start + 1;
 		search->ms_allocator = msp->ms_allocator;
 		search->ms_primary = msp->ms_primary;
 	}
 	return (msp);
 }
 
 static void
 metaslab_active_mask_verify(metaslab_t *msp)
 {
 	ASSERT(MUTEX_HELD(&msp->ms_lock));
 
 	if ((zfs_flags & ZFS_DEBUG_METASLAB_VERIFY) == 0)
 		return;
 
 	if ((msp->ms_weight & METASLAB_ACTIVE_MASK) == 0)
 		return;
 
 	if (msp->ms_weight & METASLAB_WEIGHT_PRIMARY) {
 		VERIFY0(msp->ms_weight & METASLAB_WEIGHT_SECONDARY);
 		VERIFY0(msp->ms_weight & METASLAB_WEIGHT_CLAIM);
 		VERIFY3S(msp->ms_allocator, !=, -1);
 		VERIFY(msp->ms_primary);
 		return;
 	}
 
 	if (msp->ms_weight & METASLAB_WEIGHT_SECONDARY) {
 		VERIFY0(msp->ms_weight & METASLAB_WEIGHT_PRIMARY);
 		VERIFY0(msp->ms_weight & METASLAB_WEIGHT_CLAIM);
 		VERIFY3S(msp->ms_allocator, !=, -1);
 		VERIFY(!msp->ms_primary);
 		return;
 	}
 
 	if (msp->ms_weight & METASLAB_WEIGHT_CLAIM) {
 		VERIFY0(msp->ms_weight & METASLAB_WEIGHT_PRIMARY);
 		VERIFY0(msp->ms_weight & METASLAB_WEIGHT_SECONDARY);
 		VERIFY3S(msp->ms_allocator, ==, -1);
 		return;
 	}
 }
 
 static uint64_t
 metaslab_group_alloc_normal(metaslab_group_t *mg, zio_alloc_list_t *zal,
     uint64_t asize, uint64_t txg, boolean_t want_unique, dva_t *dva, int d,
     int allocator, boolean_t try_hard)
 {
 	metaslab_t *msp = NULL;
 	uint64_t offset = -1ULL;
 
 	uint64_t activation_weight = METASLAB_WEIGHT_PRIMARY;
 	for (int i = 0; i < d; i++) {
 		if (activation_weight == METASLAB_WEIGHT_PRIMARY &&
 		    DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) {
 			activation_weight = METASLAB_WEIGHT_SECONDARY;
 		} else if (activation_weight == METASLAB_WEIGHT_SECONDARY &&
 		    DVA_GET_VDEV(&dva[i]) == mg->mg_vd->vdev_id) {
 			activation_weight = METASLAB_WEIGHT_CLAIM;
 			break;
 		}
 	}
 
 	/*
 	 * If we don't have enough metaslabs active to fill the entire array, we
 	 * just use the 0th slot.
 	 */
 	if (mg->mg_ms_ready < mg->mg_allocators * 3)
 		allocator = 0;
 	metaslab_group_allocator_t *mga = &mg->mg_allocator[allocator];
 
 	ASSERT3U(mg->mg_vd->vdev_ms_count, >=, 2);
 
 	metaslab_t *search = kmem_alloc(sizeof (*search), KM_SLEEP);
 	search->ms_weight = UINT64_MAX;
 	search->ms_start = 0;
 	/*
 	 * At the end of the metaslab tree are the already-active metaslabs,
 	 * first the primaries, then the secondaries. When we resume searching
 	 * through the tree, we need to consider ms_allocator and ms_primary so
 	 * we start in the location right after where we left off, and don't
 	 * accidentally loop forever considering the same metaslabs.
 	 */
 	search->ms_allocator = -1;
 	search->ms_primary = B_TRUE;
 	for (;;) {
 		boolean_t was_active = B_FALSE;
 
 		mutex_enter(&mg->mg_lock);
 
 		if (activation_weight == METASLAB_WEIGHT_PRIMARY &&
 		    mga->mga_primary != NULL) {
 			msp = mga->mga_primary;
 
 			/*
 			 * Even though we don't hold the ms_lock for the
 			 * primary metaslab, those fields should not
 			 * change while we hold the mg_lock. Thus it is
 			 * safe to make assertions on them.
 			 */
 			ASSERT(msp->ms_primary);
 			ASSERT3S(msp->ms_allocator, ==, allocator);
 			ASSERT(msp->ms_loaded);
 
 			was_active = B_TRUE;
 			ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK);
 		} else if (activation_weight == METASLAB_WEIGHT_SECONDARY &&
 		    mga->mga_secondary != NULL) {
 			msp = mga->mga_secondary;
 
 			/*
 			 * See comment above about the similar assertions
 			 * for the primary metaslab.
 			 */
 			ASSERT(!msp->ms_primary);
 			ASSERT3S(msp->ms_allocator, ==, allocator);
 			ASSERT(msp->ms_loaded);
 
 			was_active = B_TRUE;
 			ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK);
 		} else {
 			msp = find_valid_metaslab(mg, activation_weight, dva, d,
 			    want_unique, asize, allocator, try_hard, zal,
 			    search, &was_active);
 		}
 
 		mutex_exit(&mg->mg_lock);
 		if (msp == NULL) {
 			kmem_free(search, sizeof (*search));
 			return (-1ULL);
 		}
 		mutex_enter(&msp->ms_lock);
 
 		metaslab_active_mask_verify(msp);
 
 		/*
 		 * This code is disabled out because of issues with
 		 * tracepoints in non-gpl kernel modules.
 		 */
 #if 0
 		DTRACE_PROBE3(ms__activation__attempt,
 		    metaslab_t *, msp, uint64_t, activation_weight,
 		    boolean_t, was_active);
 #endif
 
 		/*
 		 * Ensure that the metaslab we have selected is still
 		 * capable of handling our request. It's possible that
 		 * another thread may have changed the weight while we
 		 * were blocked on the metaslab lock. We check the
 		 * active status first to see if we need to set_selected_txg
 		 * a new metaslab.
 		 */
 		if (was_active && !(msp->ms_weight & METASLAB_ACTIVE_MASK)) {
 			ASSERT3S(msp->ms_allocator, ==, -1);
 			mutex_exit(&msp->ms_lock);
 			continue;
 		}
 
 		/*
 		 * If the metaslab was activated for another allocator
 		 * while we were waiting in the ms_lock above, or it's
 		 * a primary and we're seeking a secondary (or vice versa),
 		 * we go back and select a new metaslab.
 		 */
 		if (!was_active && (msp->ms_weight & METASLAB_ACTIVE_MASK) &&
 		    (msp->ms_allocator != -1) &&
 		    (msp->ms_allocator != allocator || ((activation_weight ==
 		    METASLAB_WEIGHT_PRIMARY) != msp->ms_primary))) {
 			ASSERT(msp->ms_loaded);
 			ASSERT((msp->ms_weight & METASLAB_WEIGHT_CLAIM) ||
 			    msp->ms_allocator != -1);
 			mutex_exit(&msp->ms_lock);
 			continue;
 		}
 
 		/*
 		 * This metaslab was used for claiming regions allocated
 		 * by the ZIL during pool import. Once these regions are
 		 * claimed we don't need to keep the CLAIM bit set
 		 * anymore. Passivate this metaslab to zero its activation
 		 * mask.
 		 */
 		if (msp->ms_weight & METASLAB_WEIGHT_CLAIM &&
 		    activation_weight != METASLAB_WEIGHT_CLAIM) {
 			ASSERT(msp->ms_loaded);
 			ASSERT3S(msp->ms_allocator, ==, -1);
 			metaslab_passivate(msp, msp->ms_weight &
 			    ~METASLAB_WEIGHT_CLAIM);
 			mutex_exit(&msp->ms_lock);
 			continue;
 		}
 
 		metaslab_set_selected_txg(msp, txg);
 
 		int activation_error =
 		    metaslab_activate(msp, allocator, activation_weight);
 		metaslab_active_mask_verify(msp);
 
 		/*
 		 * If the metaslab was activated by another thread for
 		 * another allocator or activation_weight (EBUSY), or it
 		 * failed because another metaslab was assigned as primary
 		 * for this allocator (EEXIST) we continue using this
 		 * metaslab for our allocation, rather than going on to a
 		 * worse metaslab (we waited for that metaslab to be loaded
 		 * after all).
 		 *
 		 * If the activation failed due to an I/O error or ENOSPC we
 		 * skip to the next metaslab.
 		 */
 		boolean_t activated;
 		if (activation_error == 0) {
 			activated = B_TRUE;
 		} else if (activation_error == EBUSY ||
 		    activation_error == EEXIST) {
 			activated = B_FALSE;
 		} else {
 			mutex_exit(&msp->ms_lock);
 			continue;
 		}
 		ASSERT(msp->ms_loaded);
 
 		/*
 		 * Now that we have the lock, recheck to see if we should
 		 * continue to use this metaslab for this allocation. The
 		 * the metaslab is now loaded so metaslab_should_allocate()
 		 * can accurately determine if the allocation attempt should
 		 * proceed.
 		 */
 		if (!metaslab_should_allocate(msp, asize, try_hard)) {
 			/* Passivate this metaslab and select a new one. */
 			metaslab_trace_add(zal, mg, msp, asize, d,
 			    TRACE_TOO_SMALL, allocator);
 			goto next;
 		}
 
 		/*
 		 * If this metaslab is currently condensing then pick again
 		 * as we can't manipulate this metaslab until it's committed
 		 * to disk. If this metaslab is being initialized, we shouldn't
 		 * allocate from it since the allocated region might be
 		 * overwritten after allocation.
 		 */
 		if (msp->ms_condensing) {
 			metaslab_trace_add(zal, mg, msp, asize, d,
 			    TRACE_CONDENSING, allocator);
 			if (activated) {
 				metaslab_passivate(msp, msp->ms_weight &
 				    ~METASLAB_ACTIVE_MASK);
 			}
 			mutex_exit(&msp->ms_lock);
 			continue;
 		} else if (msp->ms_disabled > 0) {
 			metaslab_trace_add(zal, mg, msp, asize, d,
 			    TRACE_DISABLED, allocator);
 			if (activated) {
 				metaslab_passivate(msp, msp->ms_weight &
 				    ~METASLAB_ACTIVE_MASK);
 			}
 			mutex_exit(&msp->ms_lock);
 			continue;
 		}
 
 		offset = metaslab_block_alloc(msp, asize, txg);
 		metaslab_trace_add(zal, mg, msp, asize, d, offset, allocator);
 
 		if (offset != -1ULL) {
 			/* Proactively passivate the metaslab, if needed */
 			if (activated)
 				metaslab_segment_may_passivate(msp);
 			break;
 		}
 next:
 		ASSERT(msp->ms_loaded);
 
 		/*
 		 * This code is disabled out because of issues with
 		 * tracepoints in non-gpl kernel modules.
 		 */
 #if 0
 		DTRACE_PROBE2(ms__alloc__failure, metaslab_t *, msp,
 		    uint64_t, asize);
 #endif
 
 		/*
 		 * We were unable to allocate from this metaslab so determine
 		 * a new weight for this metaslab. Now that we have loaded
 		 * the metaslab we can provide a better hint to the metaslab
 		 * selector.
 		 *
 		 * For space-based metaslabs, we use the maximum block size.
 		 * This information is only available when the metaslab
 		 * is loaded and is more accurate than the generic free
 		 * space weight that was calculated by metaslab_weight().
 		 * This information allows us to quickly compare the maximum
 		 * available allocation in the metaslab to the allocation
 		 * size being requested.
 		 *
 		 * For segment-based metaslabs, determine the new weight
 		 * based on the highest bucket in the range tree. We
 		 * explicitly use the loaded segment weight (i.e. the range
 		 * tree histogram) since it contains the space that is
 		 * currently available for allocation and is accurate
 		 * even within a sync pass.
 		 */
 		uint64_t weight;
 		if (WEIGHT_IS_SPACEBASED(msp->ms_weight)) {
 			weight = metaslab_largest_allocatable(msp);
 			WEIGHT_SET_SPACEBASED(weight);
 		} else {
 			weight = metaslab_weight_from_range_tree(msp);
 		}
 
 		if (activated) {
 			metaslab_passivate(msp, weight);
 		} else {
 			/*
 			 * For the case where we use the metaslab that is
 			 * active for another allocator we want to make
 			 * sure that we retain the activation mask.
 			 *
 			 * Note that we could attempt to use something like
 			 * metaslab_recalculate_weight_and_sort() that
 			 * retains the activation mask here. That function
 			 * uses metaslab_weight() to set the weight though
 			 * which is not as accurate as the calculations
 			 * above.
 			 */
 			weight |= msp->ms_weight & METASLAB_ACTIVE_MASK;
 			metaslab_group_sort(mg, msp, weight);
 		}
 		metaslab_active_mask_verify(msp);
 
 		/*
 		 * We have just failed an allocation attempt, check
 		 * that metaslab_should_allocate() agrees. Otherwise,
 		 * we may end up in an infinite loop retrying the same
 		 * metaslab.
 		 */
 		ASSERT(!metaslab_should_allocate(msp, asize, try_hard));
 
 		mutex_exit(&msp->ms_lock);
 	}
 	mutex_exit(&msp->ms_lock);
 	kmem_free(search, sizeof (*search));
 	return (offset);
 }
 
 static uint64_t
 metaslab_group_alloc(metaslab_group_t *mg, zio_alloc_list_t *zal,
     uint64_t asize, uint64_t txg, boolean_t want_unique, dva_t *dva, int d,
     int allocator, boolean_t try_hard)
 {
 	uint64_t offset;
 
 	offset = metaslab_group_alloc_normal(mg, zal, asize, txg, want_unique,
 	    dva, d, allocator, try_hard);
 
 	mutex_enter(&mg->mg_lock);
 	if (offset == -1ULL) {
 		mg->mg_failed_allocations++;
 		metaslab_trace_add(zal, mg, NULL, asize, d,
 		    TRACE_GROUP_FAILURE, allocator);
 		if (asize == SPA_GANGBLOCKSIZE) {
 			/*
 			 * This metaslab group was unable to allocate
 			 * the minimum gang block size so it must be out of
 			 * space. We must notify the allocation throttle
 			 * to start skipping allocation attempts to this
 			 * metaslab group until more space becomes available.
 			 * Note: this failure cannot be caused by the
 			 * allocation throttle since the allocation throttle
 			 * is only responsible for skipping devices and
 			 * not failing block allocations.
 			 */
 			mg->mg_no_free_space = B_TRUE;
 		}
 	}
 	mg->mg_allocations++;
 	mutex_exit(&mg->mg_lock);
 	return (offset);
 }
 
 /*
  * Allocate a block for the specified i/o.
  */
 int
 metaslab_alloc_dva(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
     dva_t *dva, int d, dva_t *hintdva, uint64_t txg, int flags,
     zio_alloc_list_t *zal, int allocator)
 {
 	metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator];
 	metaslab_group_t *mg, *rotor;
 	vdev_t *vd;
 	boolean_t try_hard = B_FALSE;
 
 	ASSERT(!DVA_IS_VALID(&dva[d]));
 
 	/*
 	 * For testing, make some blocks above a certain size be gang blocks.
 	 * This will result in more split blocks when using device removal,
 	 * and a large number of split blocks coupled with ztest-induced
 	 * damage can result in extremely long reconstruction times.  This
 	 * will also test spilling from special to normal.
 	 */
 	if (psize >= metaslab_force_ganging &&
 	    metaslab_force_ganging_pct > 0 &&
 	    (random_in_range(100) < MIN(metaslab_force_ganging_pct, 100))) {
 		metaslab_trace_add(zal, NULL, NULL, psize, d, TRACE_FORCE_GANG,
 		    allocator);
 		return (SET_ERROR(ENOSPC));
 	}
 
 	/*
 	 * Start at the rotor and loop through all mgs until we find something.
 	 * Note that there's no locking on mca_rotor or mca_aliquot because
 	 * nothing actually breaks if we miss a few updates -- we just won't
 	 * allocate quite as evenly.  It all balances out over time.
 	 *
 	 * If we are doing ditto or log blocks, try to spread them across
 	 * consecutive vdevs.  If we're forced to reuse a vdev before we've
 	 * allocated all of our ditto blocks, then try and spread them out on
 	 * that vdev as much as possible.  If it turns out to not be possible,
 	 * gradually lower our standards until anything becomes acceptable.
 	 * Also, allocating on consecutive vdevs (as opposed to random vdevs)
 	 * gives us hope of containing our fault domains to something we're
 	 * able to reason about.  Otherwise, any two top-level vdev failures
 	 * will guarantee the loss of data.  With consecutive allocation,
 	 * only two adjacent top-level vdev failures will result in data loss.
 	 *
 	 * If we are doing gang blocks (hintdva is non-NULL), try to keep
 	 * ourselves on the same vdev as our gang block header.  That
 	 * way, we can hope for locality in vdev_cache, plus it makes our
 	 * fault domains something tractable.
 	 */
 	if (hintdva) {
 		vd = vdev_lookup_top(spa, DVA_GET_VDEV(&hintdva[d]));
 
 		/*
 		 * It's possible the vdev we're using as the hint no
 		 * longer exists or its mg has been closed (e.g. by
 		 * device removal).  Consult the rotor when
 		 * all else fails.
 		 */
 		if (vd != NULL && vd->vdev_mg != NULL) {
 			mg = vdev_get_mg(vd, mc);
 
 			if (flags & METASLAB_HINTBP_AVOID)
 				mg = mg->mg_next;
 		} else {
 			mg = mca->mca_rotor;
 		}
 	} else if (d != 0) {
 		vd = vdev_lookup_top(spa, DVA_GET_VDEV(&dva[d - 1]));
 		mg = vd->vdev_mg->mg_next;
 	} else {
 		ASSERT(mca->mca_rotor != NULL);
 		mg = mca->mca_rotor;
 	}
 
 	/*
 	 * If the hint put us into the wrong metaslab class, or into a
 	 * metaslab group that has been passivated, just follow the rotor.
 	 */
 	if (mg->mg_class != mc || mg->mg_activation_count <= 0)
 		mg = mca->mca_rotor;
 
 	rotor = mg;
 top:
 	do {
 		boolean_t allocatable;
 
 		ASSERT(mg->mg_activation_count == 1);
 		vd = mg->mg_vd;
 
 		/*
 		 * Don't allocate from faulted devices.
 		 */
 		if (try_hard) {
 			spa_config_enter(spa, SCL_ZIO, FTAG, RW_READER);
 			allocatable = vdev_allocatable(vd);
 			spa_config_exit(spa, SCL_ZIO, FTAG);
 		} else {
 			allocatable = vdev_allocatable(vd);
 		}
 
 		/*
 		 * Determine if the selected metaslab group is eligible
 		 * for allocations. If we're ganging then don't allow
 		 * this metaslab group to skip allocations since that would
 		 * inadvertently return ENOSPC and suspend the pool
 		 * even though space is still available.
 		 */
 		if (allocatable && !GANG_ALLOCATION(flags) && !try_hard) {
 			allocatable = metaslab_group_allocatable(mg, rotor,
 			    flags, psize, allocator, d);
 		}
 
 		if (!allocatable) {
 			metaslab_trace_add(zal, mg, NULL, psize, d,
 			    TRACE_NOT_ALLOCATABLE, allocator);
 			goto next;
 		}
 
 		/*
 		 * Avoid writing single-copy data to an unhealthy,
 		 * non-redundant vdev, unless we've already tried all
 		 * other vdevs.
 		 */
 		if (vd->vdev_state < VDEV_STATE_HEALTHY &&
 		    d == 0 && !try_hard && vd->vdev_children == 0) {
 			metaslab_trace_add(zal, mg, NULL, psize, d,
 			    TRACE_VDEV_ERROR, allocator);
 			goto next;
 		}
 
 		ASSERT(mg->mg_class == mc);
 
 		uint64_t asize = vdev_psize_to_asize(vd, psize);
 		ASSERT(P2PHASE(asize, 1ULL << vd->vdev_ashift) == 0);
 
 		/*
 		 * If we don't need to try hard, then require that the
 		 * block be on a different metaslab from any other DVAs
 		 * in this BP (unique=true).  If we are trying hard, then
 		 * allow any metaslab to be used (unique=false).
 		 */
 		uint64_t offset = metaslab_group_alloc(mg, zal, asize, txg,
 		    !try_hard, dva, d, allocator, try_hard);
 
 		if (offset != -1ULL) {
 			/*
 			 * If we've just selected this metaslab group,
 			 * figure out whether the corresponding vdev is
 			 * over- or under-used relative to the pool,
 			 * and set an allocation bias to even it out.
 			 *
 			 * Bias is also used to compensate for unequally
 			 * sized vdevs so that space is allocated fairly.
 			 */
 			if (mca->mca_aliquot == 0 && metaslab_bias_enabled) {
 				vdev_stat_t *vs = &vd->vdev_stat;
 				int64_t vs_free = vs->vs_space - vs->vs_alloc;
 				int64_t mc_free = mc->mc_space - mc->mc_alloc;
 				int64_t ratio;
 
 				/*
 				 * Calculate how much more or less we should
 				 * try to allocate from this device during
 				 * this iteration around the rotor.
 				 *
 				 * This basically introduces a zero-centered
 				 * bias towards the devices with the most
 				 * free space, while compensating for vdev
 				 * size differences.
 				 *
 				 * Examples:
 				 *  vdev V1 = 16M/128M
 				 *  vdev V2 = 16M/128M
 				 *  ratio(V1) = 100% ratio(V2) = 100%
 				 *
 				 *  vdev V1 = 16M/128M
 				 *  vdev V2 = 64M/128M
 				 *  ratio(V1) = 127% ratio(V2) =  72%
 				 *
 				 *  vdev V1 = 16M/128M
 				 *  vdev V2 = 64M/512M
 				 *  ratio(V1) =  40% ratio(V2) = 160%
 				 */
 				ratio = (vs_free * mc->mc_alloc_groups * 100) /
 				    (mc_free + 1);
 				mg->mg_bias = ((ratio - 100) *
 				    (int64_t)mg->mg_aliquot) / 100;
 			} else if (!metaslab_bias_enabled) {
 				mg->mg_bias = 0;
 			}
 
 			if ((flags & METASLAB_ZIL) ||
 			    atomic_add_64_nv(&mca->mca_aliquot, asize) >=
 			    mg->mg_aliquot + mg->mg_bias) {
 				mca->mca_rotor = mg->mg_next;
 				mca->mca_aliquot = 0;
 			}
 
 			DVA_SET_VDEV(&dva[d], vd->vdev_id);
 			DVA_SET_OFFSET(&dva[d], offset);
 			DVA_SET_GANG(&dva[d],
 			    ((flags & METASLAB_GANG_HEADER) ? 1 : 0));
 			DVA_SET_ASIZE(&dva[d], asize);
 
 			return (0);
 		}
 next:
 		mca->mca_rotor = mg->mg_next;
 		mca->mca_aliquot = 0;
 	} while ((mg = mg->mg_next) != rotor);
 
 	/*
 	 * If we haven't tried hard, perhaps do so now.
 	 */
 	if (!try_hard && (zfs_metaslab_try_hard_before_gang ||
 	    GANG_ALLOCATION(flags) || (flags & METASLAB_ZIL) != 0 ||
 	    psize <= 1 << spa->spa_min_ashift)) {
 		METASLABSTAT_BUMP(metaslabstat_try_hard);
 		try_hard = B_TRUE;
 		goto top;
 	}
 
 	memset(&dva[d], 0, sizeof (dva_t));
 
 	metaslab_trace_add(zal, rotor, NULL, psize, d, TRACE_ENOSPC, allocator);
 	return (SET_ERROR(ENOSPC));
 }
 
 void
 metaslab_free_concrete(vdev_t *vd, uint64_t offset, uint64_t asize,
     boolean_t checkpoint)
 {
 	metaslab_t *msp;
 	spa_t *spa = vd->vdev_spa;
 
 	ASSERT(vdev_is_concrete(vd));
 	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);
 	ASSERT3U(offset >> vd->vdev_ms_shift, <, vd->vdev_ms_count);
 
 	msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];
 
 	VERIFY(!msp->ms_condensing);
 	VERIFY3U(offset, >=, msp->ms_start);
 	VERIFY3U(offset + asize, <=, msp->ms_start + msp->ms_size);
 	VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift));
 	VERIFY0(P2PHASE(asize, 1ULL << vd->vdev_ashift));
 
 	metaslab_check_free_impl(vd, offset, asize);
 
 	mutex_enter(&msp->ms_lock);
 	if (range_tree_is_empty(msp->ms_freeing) &&
 	    range_tree_is_empty(msp->ms_checkpointing)) {
 		vdev_dirty(vd, VDD_METASLAB, msp, spa_syncing_txg(spa));
 	}
 
 	if (checkpoint) {
 		ASSERT(spa_has_checkpoint(spa));
 		range_tree_add(msp->ms_checkpointing, offset, asize);
 	} else {
 		range_tree_add(msp->ms_freeing, offset, asize);
 	}
 	mutex_exit(&msp->ms_lock);
 }
 
 void
 metaslab_free_impl_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset,
     uint64_t size, void *arg)
 {
 	(void) inner_offset;
 	boolean_t *checkpoint = arg;
 
 	ASSERT3P(checkpoint, !=, NULL);
 
 	if (vd->vdev_ops->vdev_op_remap != NULL)
 		vdev_indirect_mark_obsolete(vd, offset, size);
 	else
 		metaslab_free_impl(vd, offset, size, *checkpoint);
 }
 
 static void
 metaslab_free_impl(vdev_t *vd, uint64_t offset, uint64_t size,
     boolean_t checkpoint)
 {
 	spa_t *spa = vd->vdev_spa;
 
 	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);
 
 	if (spa_syncing_txg(spa) > spa_freeze_txg(spa))
 		return;
 
 	if (spa->spa_vdev_removal != NULL &&
 	    spa->spa_vdev_removal->svr_vdev_id == vd->vdev_id &&
 	    vdev_is_concrete(vd)) {
 		/*
 		 * Note: we check if the vdev is concrete because when
 		 * we complete the removal, we first change the vdev to be
 		 * an indirect vdev (in open context), and then (in syncing
 		 * context) clear spa_vdev_removal.
 		 */
 		free_from_removing_vdev(vd, offset, size);
 	} else if (vd->vdev_ops->vdev_op_remap != NULL) {
 		vdev_indirect_mark_obsolete(vd, offset, size);
 		vd->vdev_ops->vdev_op_remap(vd, offset, size,
 		    metaslab_free_impl_cb, &checkpoint);
 	} else {
 		metaslab_free_concrete(vd, offset, size, checkpoint);
 	}
 }
 
 typedef struct remap_blkptr_cb_arg {
 	blkptr_t *rbca_bp;
 	spa_remap_cb_t rbca_cb;
 	vdev_t *rbca_remap_vd;
 	uint64_t rbca_remap_offset;
 	void *rbca_cb_arg;
 } remap_blkptr_cb_arg_t;
 
 static void
 remap_blkptr_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset,
     uint64_t size, void *arg)
 {
 	remap_blkptr_cb_arg_t *rbca = arg;
 	blkptr_t *bp = rbca->rbca_bp;
 
 	/* We can not remap split blocks. */
 	if (size != DVA_GET_ASIZE(&bp->blk_dva[0]))
 		return;
 	ASSERT0(inner_offset);
 
 	if (rbca->rbca_cb != NULL) {
 		/*
 		 * At this point we know that we are not handling split
 		 * blocks and we invoke the callback on the previous
 		 * vdev which must be indirect.
 		 */
 		ASSERT3P(rbca->rbca_remap_vd->vdev_ops, ==, &vdev_indirect_ops);
 
 		rbca->rbca_cb(rbca->rbca_remap_vd->vdev_id,
 		    rbca->rbca_remap_offset, size, rbca->rbca_cb_arg);
 
 		/* set up remap_blkptr_cb_arg for the next call */
 		rbca->rbca_remap_vd = vd;
 		rbca->rbca_remap_offset = offset;
 	}
 
 	/*
 	 * The phys birth time is that of dva[0].  This ensures that we know
 	 * when each dva was written, so that resilver can determine which
 	 * blocks need to be scrubbed (i.e. those written during the time
 	 * the vdev was offline).  It also ensures that the key used in
 	 * the ARC hash table is unique (i.e. dva[0] + phys_birth).  If
 	 * we didn't change the phys_birth, a lookup in the ARC for a
 	 * remapped BP could find the data that was previously stored at
 	 * this vdev + offset.
 	 */
 	vdev_t *oldvd = vdev_lookup_top(vd->vdev_spa,
 	    DVA_GET_VDEV(&bp->blk_dva[0]));
 	vdev_indirect_births_t *vib = oldvd->vdev_indirect_births;
 	bp->blk_phys_birth = vdev_indirect_births_physbirth(vib,
 	    DVA_GET_OFFSET(&bp->blk_dva[0]), DVA_GET_ASIZE(&bp->blk_dva[0]));
 
 	DVA_SET_VDEV(&bp->blk_dva[0], vd->vdev_id);
 	DVA_SET_OFFSET(&bp->blk_dva[0], offset);
 }
 
 /*
  * If the block pointer contains any indirect DVAs, modify them to refer to
  * concrete DVAs.  Note that this will sometimes not be possible, leaving
  * the indirect DVA in place.  This happens if the indirect DVA spans multiple
  * segments in the mapping (i.e. it is a "split block").
  *
  * If the BP was remapped, calls the callback on the original dva (note the
  * callback can be called multiple times if the original indirect DVA refers
  * to another indirect DVA, etc).
  *
  * Returns TRUE if the BP was remapped.
  */
 boolean_t
 spa_remap_blkptr(spa_t *spa, blkptr_t *bp, spa_remap_cb_t callback, void *arg)
 {
 	remap_blkptr_cb_arg_t rbca;
 
 	if (!zfs_remap_blkptr_enable)
 		return (B_FALSE);
 
 	if (!spa_feature_is_enabled(spa, SPA_FEATURE_OBSOLETE_COUNTS))
 		return (B_FALSE);
 
 	/*
 	 * Dedup BP's can not be remapped, because ddt_phys_select() depends
 	 * on DVA[0] being the same in the BP as in the DDT (dedup table).
 	 */
 	if (BP_GET_DEDUP(bp))
 		return (B_FALSE);
 
 	/*
 	 * Gang blocks can not be remapped, because
 	 * zio_checksum_gang_verifier() depends on the DVA[0] that's in
 	 * the BP used to read the gang block header (GBH) being the same
 	 * as the DVA[0] that we allocated for the GBH.
 	 */
 	if (BP_IS_GANG(bp))
 		return (B_FALSE);
 
 	/*
 	 * Embedded BP's have no DVA to remap.
 	 */
 	if (BP_GET_NDVAS(bp) < 1)
 		return (B_FALSE);
 
+	/*
+	 * Cloned blocks can not be remapped since BRT depends on specific
+	 * vdev id and offset in the DVA[0] for its reference counting.
+	 */
+	if (!BP_IS_METADATA(bp) && brt_maybe_exists(spa, bp))
+		return (B_FALSE);
+
 	/*
 	 * Note: we only remap dva[0].  If we remapped other dvas, we
 	 * would no longer know what their phys birth txg is.
 	 */
 	dva_t *dva = &bp->blk_dva[0];
 
 	uint64_t offset = DVA_GET_OFFSET(dva);
 	uint64_t size = DVA_GET_ASIZE(dva);
 	vdev_t *vd = vdev_lookup_top(spa, DVA_GET_VDEV(dva));
 
 	if (vd->vdev_ops->vdev_op_remap == NULL)
 		return (B_FALSE);
 
 	rbca.rbca_bp = bp;
 	rbca.rbca_cb = callback;
 	rbca.rbca_remap_vd = vd;
 	rbca.rbca_remap_offset = offset;
 	rbca.rbca_cb_arg = arg;
 
 	/*
 	 * remap_blkptr_cb() will be called in order for each level of
 	 * indirection, until a concrete vdev is reached or a split block is
 	 * encountered. old_vd and old_offset are updated within the callback
 	 * as we go from the one indirect vdev to the next one (either concrete
 	 * or indirect again) in that order.
 	 */
 	vd->vdev_ops->vdev_op_remap(vd, offset, size, remap_blkptr_cb, &rbca);
 
 	/* Check if the DVA wasn't remapped because it is a split block */
 	if (DVA_GET_VDEV(&rbca.rbca_bp->blk_dva[0]) == vd->vdev_id)
 		return (B_FALSE);
 
 	return (B_TRUE);
 }
 
 /*
  * Undo the allocation of a DVA which happened in the given transaction group.
  */
 void
 metaslab_unalloc_dva(spa_t *spa, const dva_t *dva, uint64_t txg)
 {
 	metaslab_t *msp;
 	vdev_t *vd;
 	uint64_t vdev = DVA_GET_VDEV(dva);
 	uint64_t offset = DVA_GET_OFFSET(dva);
 	uint64_t size = DVA_GET_ASIZE(dva);
 
 	ASSERT(DVA_IS_VALID(dva));
 	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);
 
 	if (txg > spa_freeze_txg(spa))
 		return;
 
 	if ((vd = vdev_lookup_top(spa, vdev)) == NULL || !DVA_IS_VALID(dva) ||
 	    (offset >> vd->vdev_ms_shift) >= vd->vdev_ms_count) {
 		zfs_panic_recover("metaslab_free_dva(): bad DVA %llu:%llu:%llu",
 		    (u_longlong_t)vdev, (u_longlong_t)offset,
 		    (u_longlong_t)size);
 		return;
 	}
 
 	ASSERT(!vd->vdev_removing);
 	ASSERT(vdev_is_concrete(vd));
 	ASSERT0(vd->vdev_indirect_config.vic_mapping_object);
 	ASSERT3P(vd->vdev_indirect_mapping, ==, NULL);
 
 	if (DVA_GET_GANG(dva))
 		size = vdev_gang_header_asize(vd);
 
 	msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];
 
 	mutex_enter(&msp->ms_lock);
 	range_tree_remove(msp->ms_allocating[txg & TXG_MASK],
 	    offset, size);
 	msp->ms_allocating_total -= size;
 
 	VERIFY(!msp->ms_condensing);
 	VERIFY3U(offset, >=, msp->ms_start);
 	VERIFY3U(offset + size, <=, msp->ms_start + msp->ms_size);
 	VERIFY3U(range_tree_space(msp->ms_allocatable) + size, <=,
 	    msp->ms_size);
 	VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift));
 	VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift));
 	range_tree_add(msp->ms_allocatable, offset, size);
 	mutex_exit(&msp->ms_lock);
 }
 
 /*
  * Free the block represented by the given DVA.
  */
 void
 metaslab_free_dva(spa_t *spa, const dva_t *dva, boolean_t checkpoint)
 {
 	uint64_t vdev = DVA_GET_VDEV(dva);
 	uint64_t offset = DVA_GET_OFFSET(dva);
 	uint64_t size = DVA_GET_ASIZE(dva);
 	vdev_t *vd = vdev_lookup_top(spa, vdev);
 
 	ASSERT(DVA_IS_VALID(dva));
 	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);
 
 	if (DVA_GET_GANG(dva)) {
 		size = vdev_gang_header_asize(vd);
 	}
 
 	metaslab_free_impl(vd, offset, size, checkpoint);
 }
 
 /*
  * Reserve some allocation slots. The reservation system must be called
  * before we call into the allocator. If there aren't any available slots
  * then the I/O will be throttled until an I/O completes and its slots are
  * freed up. The function returns true if it was successful in placing
  * the reservation.
  */
 boolean_t
 metaslab_class_throttle_reserve(metaslab_class_t *mc, int slots, int allocator,
     zio_t *zio, int flags)
 {
 	metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator];
 	uint64_t max = mca->mca_alloc_max_slots;
 
 	ASSERT(mc->mc_alloc_throttle_enabled);
 	if (GANG_ALLOCATION(flags) || (flags & METASLAB_MUST_RESERVE) ||
 	    zfs_refcount_count(&mca->mca_alloc_slots) + slots <= max) {
 		/*
 		 * The potential race between _count() and _add() is covered
 		 * by the allocator lock in most cases, or irrelevant due to
 		 * GANG_ALLOCATION() or METASLAB_MUST_RESERVE set in others.
 		 * But even if we assume some other non-existing scenario, the
 		 * worst that can happen is few more I/Os get to allocation
 		 * earlier, that is not a problem.
 		 *
 		 * We reserve the slots individually so that we can unreserve
 		 * them individually when an I/O completes.
 		 */
 		zfs_refcount_add_few(&mca->mca_alloc_slots, slots, zio);
 		zio->io_flags |= ZIO_FLAG_IO_ALLOCATING;
 		return (B_TRUE);
 	}
 	return (B_FALSE);
 }
 
 void
 metaslab_class_throttle_unreserve(metaslab_class_t *mc, int slots,
     int allocator, zio_t *zio)
 {
 	metaslab_class_allocator_t *mca = &mc->mc_allocator[allocator];
 
 	ASSERT(mc->mc_alloc_throttle_enabled);
 	zfs_refcount_remove_few(&mca->mca_alloc_slots, slots, zio);
 }
 
 static int
 metaslab_claim_concrete(vdev_t *vd, uint64_t offset, uint64_t size,
     uint64_t txg)
 {
 	metaslab_t *msp;
 	spa_t *spa = vd->vdev_spa;
 	int error = 0;
 
 	if (offset >> vd->vdev_ms_shift >= vd->vdev_ms_count)
 		return (SET_ERROR(ENXIO));
 
 	ASSERT3P(vd->vdev_ms, !=, NULL);
 	msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];
 
 	mutex_enter(&msp->ms_lock);
 
 	if ((txg != 0 && spa_writeable(spa)) || !msp->ms_loaded) {
 		error = metaslab_activate(msp, 0, METASLAB_WEIGHT_CLAIM);
 		if (error == EBUSY) {
 			ASSERT(msp->ms_loaded);
 			ASSERT(msp->ms_weight & METASLAB_ACTIVE_MASK);
 			error = 0;
 		}
 	}
 
 	if (error == 0 &&
 	    !range_tree_contains(msp->ms_allocatable, offset, size))
 		error = SET_ERROR(ENOENT);
 
 	if (error || txg == 0) {	/* txg == 0 indicates dry run */
 		mutex_exit(&msp->ms_lock);
 		return (error);
 	}
 
 	VERIFY(!msp->ms_condensing);
 	VERIFY0(P2PHASE(offset, 1ULL << vd->vdev_ashift));
 	VERIFY0(P2PHASE(size, 1ULL << vd->vdev_ashift));
 	VERIFY3U(range_tree_space(msp->ms_allocatable) - size, <=,
 	    msp->ms_size);
 	range_tree_remove(msp->ms_allocatable, offset, size);
 	range_tree_clear(msp->ms_trim, offset, size);
 
 	if (spa_writeable(spa)) {	/* don't dirty if we're zdb(8) */
 		metaslab_class_t *mc = msp->ms_group->mg_class;
 		multilist_sublist_t *mls =
 		    multilist_sublist_lock_obj(&mc->mc_metaslab_txg_list, msp);
 		if (!multilist_link_active(&msp->ms_class_txg_node)) {
 			msp->ms_selected_txg = txg;
 			multilist_sublist_insert_head(mls, msp);
 		}
 		multilist_sublist_unlock(mls);
 
 		if (range_tree_is_empty(msp->ms_allocating[txg & TXG_MASK]))
 			vdev_dirty(vd, VDD_METASLAB, msp, txg);
 		range_tree_add(msp->ms_allocating[txg & TXG_MASK],
 		    offset, size);
 		msp->ms_allocating_total += size;
 	}
 
 	mutex_exit(&msp->ms_lock);
 
 	return (0);
 }
 
 typedef struct metaslab_claim_cb_arg_t {
 	uint64_t	mcca_txg;
 	int		mcca_error;
 } metaslab_claim_cb_arg_t;
 
 static void
 metaslab_claim_impl_cb(uint64_t inner_offset, vdev_t *vd, uint64_t offset,
     uint64_t size, void *arg)
 {
 	(void) inner_offset;
 	metaslab_claim_cb_arg_t *mcca_arg = arg;
 
 	if (mcca_arg->mcca_error == 0) {
 		mcca_arg->mcca_error = metaslab_claim_concrete(vd, offset,
 		    size, mcca_arg->mcca_txg);
 	}
 }
 
 int
 metaslab_claim_impl(vdev_t *vd, uint64_t offset, uint64_t size, uint64_t txg)
 {
 	if (vd->vdev_ops->vdev_op_remap != NULL) {
 		metaslab_claim_cb_arg_t arg;
 
 		/*
 		 * Only zdb(8) can claim on indirect vdevs.  This is used
 		 * to detect leaks of mapped space (that are not accounted
 		 * for in the obsolete counts, spacemap, or bpobj).
 		 */
 		ASSERT(!spa_writeable(vd->vdev_spa));
 		arg.mcca_error = 0;
 		arg.mcca_txg = txg;
 
 		vd->vdev_ops->vdev_op_remap(vd, offset, size,
 		    metaslab_claim_impl_cb, &arg);
 
 		if (arg.mcca_error == 0) {
 			arg.mcca_error = metaslab_claim_concrete(vd,
 			    offset, size, txg);
 		}
 		return (arg.mcca_error);
 	} else {
 		return (metaslab_claim_concrete(vd, offset, size, txg));
 	}
 }
 
 /*
  * Intent log support: upon opening the pool after a crash, notify the SPA
  * of blocks that the intent log has allocated for immediate write, but
  * which are still considered free by the SPA because the last transaction
  * group didn't commit yet.
  */
 static int
 metaslab_claim_dva(spa_t *spa, const dva_t *dva, uint64_t txg)
 {
 	uint64_t vdev = DVA_GET_VDEV(dva);
 	uint64_t offset = DVA_GET_OFFSET(dva);
 	uint64_t size = DVA_GET_ASIZE(dva);
 	vdev_t *vd;
 
 	if ((vd = vdev_lookup_top(spa, vdev)) == NULL) {
 		return (SET_ERROR(ENXIO));
 	}
 
 	ASSERT(DVA_IS_VALID(dva));
 
 	if (DVA_GET_GANG(dva))
 		size = vdev_gang_header_asize(vd);
 
 	return (metaslab_claim_impl(vd, offset, size, txg));
 }
 
 int
 metaslab_alloc(spa_t *spa, metaslab_class_t *mc, uint64_t psize, blkptr_t *bp,
     int ndvas, uint64_t txg, blkptr_t *hintbp, int flags,
     zio_alloc_list_t *zal, zio_t *zio, int allocator)
 {
 	dva_t *dva = bp->blk_dva;
 	dva_t *hintdva = (hintbp != NULL) ? hintbp->blk_dva : NULL;
 	int error = 0;
 
 	ASSERT(bp->blk_birth == 0);
 	ASSERT(BP_PHYSICAL_BIRTH(bp) == 0);
 
 	spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER);
 
 	if (mc->mc_allocator[allocator].mca_rotor == NULL) {
 		/* no vdevs in this class */
 		spa_config_exit(spa, SCL_ALLOC, FTAG);
 		return (SET_ERROR(ENOSPC));
 	}
 
 	ASSERT(ndvas > 0 && ndvas <= spa_max_replication(spa));
 	ASSERT(BP_GET_NDVAS(bp) == 0);
 	ASSERT(hintbp == NULL || ndvas <= BP_GET_NDVAS(hintbp));
 	ASSERT3P(zal, !=, NULL);
 
 	for (int d = 0; d < ndvas; d++) {
 		error = metaslab_alloc_dva(spa, mc, psize, dva, d, hintdva,
 		    txg, flags, zal, allocator);
 		if (error != 0) {
 			for (d--; d >= 0; d--) {
 				metaslab_unalloc_dva(spa, &dva[d], txg);
 				metaslab_group_alloc_decrement(spa,
 				    DVA_GET_VDEV(&dva[d]), zio, flags,
 				    allocator, B_FALSE);
 				memset(&dva[d], 0, sizeof (dva_t));
 			}
 			spa_config_exit(spa, SCL_ALLOC, FTAG);
 			return (error);
 		} else {
 			/*
 			 * Update the metaslab group's queue depth
 			 * based on the newly allocated dva.
 			 */
 			metaslab_group_alloc_increment(spa,
 			    DVA_GET_VDEV(&dva[d]), zio, flags, allocator);
 		}
 	}
 	ASSERT(error == 0);
 	ASSERT(BP_GET_NDVAS(bp) == ndvas);
 
 	spa_config_exit(spa, SCL_ALLOC, FTAG);
 
 	BP_SET_BIRTH(bp, txg, 0);
 
 	return (0);
 }
 
 void
 metaslab_free(spa_t *spa, const blkptr_t *bp, uint64_t txg, boolean_t now)
 {
 	const dva_t *dva = bp->blk_dva;
 	int ndvas = BP_GET_NDVAS(bp);
 
 	ASSERT(!BP_IS_HOLE(bp));
 	ASSERT(!now || bp->blk_birth >= spa_syncing_txg(spa));
 
 	/*
 	 * If we have a checkpoint for the pool we need to make sure that
 	 * the blocks that we free that are part of the checkpoint won't be
 	 * reused until the checkpoint is discarded or we revert to it.
 	 *
 	 * The checkpoint flag is passed down the metaslab_free code path
 	 * and is set whenever we want to add a block to the checkpoint's
 	 * accounting. That is, we "checkpoint" blocks that existed at the
 	 * time the checkpoint was created and are therefore referenced by
 	 * the checkpointed uberblock.
 	 *
 	 * Note that, we don't checkpoint any blocks if the current
 	 * syncing txg <= spa_checkpoint_txg. We want these frees to sync
 	 * normally as they will be referenced by the checkpointed uberblock.
 	 */
 	boolean_t checkpoint = B_FALSE;
 	if (bp->blk_birth <= spa->spa_checkpoint_txg &&
 	    spa_syncing_txg(spa) > spa->spa_checkpoint_txg) {
 		/*
 		 * At this point, if the block is part of the checkpoint
 		 * there is no way it was created in the current txg.
 		 */
 		ASSERT(!now);
 		ASSERT3U(spa_syncing_txg(spa), ==, txg);
 		checkpoint = B_TRUE;
 	}
 
 	spa_config_enter(spa, SCL_FREE, FTAG, RW_READER);
 
 	for (int d = 0; d < ndvas; d++) {
 		if (now) {
 			metaslab_unalloc_dva(spa, &dva[d], txg);
 		} else {
 			ASSERT3U(txg, ==, spa_syncing_txg(spa));
 			metaslab_free_dva(spa, &dva[d], checkpoint);
 		}
 	}
 
 	spa_config_exit(spa, SCL_FREE, FTAG);
 }
 
 int
 metaslab_claim(spa_t *spa, const blkptr_t *bp, uint64_t txg)
 {
 	const dva_t *dva = bp->blk_dva;
 	int ndvas = BP_GET_NDVAS(bp);
 	int error = 0;
 
 	ASSERT(!BP_IS_HOLE(bp));
 
 	if (txg != 0) {
 		/*
 		 * First do a dry run to make sure all DVAs are claimable,
 		 * so we don't have to unwind from partial failures below.
 		 */
 		if ((error = metaslab_claim(spa, bp, 0)) != 0)
 			return (error);
 	}
 
 	spa_config_enter(spa, SCL_ALLOC, FTAG, RW_READER);
 
 	for (int d = 0; d < ndvas; d++) {
 		error = metaslab_claim_dva(spa, &dva[d], txg);
 		if (error != 0)
 			break;
 	}
 
 	spa_config_exit(spa, SCL_ALLOC, FTAG);
 
 	ASSERT(error == 0 || txg == 0);
 
 	return (error);
 }
 
 static void
 metaslab_check_free_impl_cb(uint64_t inner, vdev_t *vd, uint64_t offset,
     uint64_t size, void *arg)
 {
 	(void) inner, (void) arg;
 
 	if (vd->vdev_ops == &vdev_indirect_ops)
 		return;
 
 	metaslab_check_free_impl(vd, offset, size);
 }
 
 static void
 metaslab_check_free_impl(vdev_t *vd, uint64_t offset, uint64_t size)
 {
 	metaslab_t *msp;
 	spa_t *spa __maybe_unused = vd->vdev_spa;
 
 	if ((zfs_flags & ZFS_DEBUG_ZIO_FREE) == 0)
 		return;
 
 	if (vd->vdev_ops->vdev_op_remap != NULL) {
 		vd->vdev_ops->vdev_op_remap(vd, offset, size,
 		    metaslab_check_free_impl_cb, NULL);
 		return;
 	}
 
 	ASSERT(vdev_is_concrete(vd));
 	ASSERT3U(offset >> vd->vdev_ms_shift, <, vd->vdev_ms_count);
 	ASSERT3U(spa_config_held(spa, SCL_ALL, RW_READER), !=, 0);
 
 	msp = vd->vdev_ms[offset >> vd->vdev_ms_shift];
 
 	mutex_enter(&msp->ms_lock);
 	if (msp->ms_loaded) {
 		range_tree_verify_not_present(msp->ms_allocatable,
 		    offset, size);
 	}
 
 	/*
 	 * Check all segments that currently exist in the freeing pipeline.
 	 *
 	 * It would intuitively make sense to also check the current allocating
 	 * tree since metaslab_unalloc_dva() exists for extents that are
 	 * allocated and freed in the same sync pass within the same txg.
 	 * Unfortunately there are places (e.g. the ZIL) where we allocate a
 	 * segment but then we free part of it within the same txg
 	 * [see zil_sync()]. Thus, we don't call range_tree_verify() in the
 	 * current allocating tree.
 	 */
 	range_tree_verify_not_present(msp->ms_freeing, offset, size);
 	range_tree_verify_not_present(msp->ms_checkpointing, offset, size);
 	range_tree_verify_not_present(msp->ms_freed, offset, size);
 	for (int j = 0; j < TXG_DEFER_SIZE; j++)
 		range_tree_verify_not_present(msp->ms_defer[j], offset, size);
 	range_tree_verify_not_present(msp->ms_trim, offset, size);
 	mutex_exit(&msp->ms_lock);
 }
 
 void
 metaslab_check_free(spa_t *spa, const blkptr_t *bp)
 {
 	if ((zfs_flags & ZFS_DEBUG_ZIO_FREE) == 0)
 		return;
 
 	spa_config_enter(spa, SCL_VDEV, FTAG, RW_READER);
 	for (int i = 0; i < BP_GET_NDVAS(bp); i++) {
 		uint64_t vdev = DVA_GET_VDEV(&bp->blk_dva[i]);
 		vdev_t *vd = vdev_lookup_top(spa, vdev);
 		uint64_t offset = DVA_GET_OFFSET(&bp->blk_dva[i]);
 		uint64_t size = DVA_GET_ASIZE(&bp->blk_dva[i]);
 
 		if (DVA_GET_GANG(&bp->blk_dva[i]))
 			size = vdev_gang_header_asize(vd);
 
 		ASSERT3P(vd, !=, NULL);
 
 		metaslab_check_free_impl(vd, offset, size);
 	}
 	spa_config_exit(spa, SCL_VDEV, FTAG);
 }
 
 static void
 metaslab_group_disable_wait(metaslab_group_t *mg)
 {
 	ASSERT(MUTEX_HELD(&mg->mg_ms_disabled_lock));
 	while (mg->mg_disabled_updating) {
 		cv_wait(&mg->mg_ms_disabled_cv, &mg->mg_ms_disabled_lock);
 	}
 }
 
 static void
 metaslab_group_disabled_increment(metaslab_group_t *mg)
 {
 	ASSERT(MUTEX_HELD(&mg->mg_ms_disabled_lock));
 	ASSERT(mg->mg_disabled_updating);
 
 	while (mg->mg_ms_disabled >= max_disabled_ms) {
 		cv_wait(&mg->mg_ms_disabled_cv, &mg->mg_ms_disabled_lock);
 	}
 	mg->mg_ms_disabled++;
 	ASSERT3U(mg->mg_ms_disabled, <=, max_disabled_ms);
 }
 
 /*
  * Mark the metaslab as disabled to prevent any allocations on this metaslab.
  * We must also track how many metaslabs are currently disabled within a
  * metaslab group and limit them to prevent allocation failures from
  * occurring because all metaslabs are disabled.
  */
 void
 metaslab_disable(metaslab_t *msp)
 {
 	ASSERT(!MUTEX_HELD(&msp->ms_lock));
 	metaslab_group_t *mg = msp->ms_group;
 
 	mutex_enter(&mg->mg_ms_disabled_lock);
 
 	/*
 	 * To keep an accurate count of how many threads have disabled
 	 * a specific metaslab group, we only allow one thread to mark
 	 * the metaslab group at a time. This ensures that the value of
 	 * ms_disabled will be accurate when we decide to mark a metaslab
 	 * group as disabled. To do this we force all other threads
 	 * to wait till the metaslab's mg_disabled_updating flag is no
 	 * longer set.
 	 */
 	metaslab_group_disable_wait(mg);
 	mg->mg_disabled_updating = B_TRUE;
 	if (msp->ms_disabled == 0) {
 		metaslab_group_disabled_increment(mg);
 	}
 	mutex_enter(&msp->ms_lock);
 	msp->ms_disabled++;
 	mutex_exit(&msp->ms_lock);
 
 	mg->mg_disabled_updating = B_FALSE;
 	cv_broadcast(&mg->mg_ms_disabled_cv);
 	mutex_exit(&mg->mg_ms_disabled_lock);
 }
 
 void
 metaslab_enable(metaslab_t *msp, boolean_t sync, boolean_t unload)
 {
 	metaslab_group_t *mg = msp->ms_group;
 	spa_t *spa = mg->mg_vd->vdev_spa;
 
 	/*
 	 * Wait for the outstanding IO to be synced to prevent newly
 	 * allocated blocks from being overwritten.  This used by
 	 * initialize and TRIM which are modifying unallocated space.
 	 */
 	if (sync)
 		txg_wait_synced(spa_get_dsl(spa), 0);
 
 	mutex_enter(&mg->mg_ms_disabled_lock);
 	mutex_enter(&msp->ms_lock);
 	if (--msp->ms_disabled == 0) {
 		mg->mg_ms_disabled--;
 		cv_broadcast(&mg->mg_ms_disabled_cv);
 		if (unload)
 			metaslab_unload(msp);
 	}
 	mutex_exit(&msp->ms_lock);
 	mutex_exit(&mg->mg_ms_disabled_lock);
 }
 
 void
 metaslab_set_unflushed_dirty(metaslab_t *ms, boolean_t dirty)
 {
 	ms->ms_unflushed_dirty = dirty;
 }
 
 static void
 metaslab_update_ondisk_flush_data(metaslab_t *ms, dmu_tx_t *tx)
 {
 	vdev_t *vd = ms->ms_group->mg_vd;
 	spa_t *spa = vd->vdev_spa;
 	objset_t *mos = spa_meta_objset(spa);
 
 	ASSERT(spa_feature_is_active(spa, SPA_FEATURE_LOG_SPACEMAP));
 
 	metaslab_unflushed_phys_t entry = {
 		.msp_unflushed_txg = metaslab_unflushed_txg(ms),
 	};
 	uint64_t entry_size = sizeof (entry);
 	uint64_t entry_offset = ms->ms_id * entry_size;
 
 	uint64_t object = 0;
 	int err = zap_lookup(mos, vd->vdev_top_zap,
 	    VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, sizeof (uint64_t), 1,
 	    &object);
 	if (err == ENOENT) {
 		object = dmu_object_alloc(mos, DMU_OTN_UINT64_METADATA,
 		    SPA_OLD_MAXBLOCKSIZE, DMU_OT_NONE, 0, tx);
 		VERIFY0(zap_add(mos, vd->vdev_top_zap,
 		    VDEV_TOP_ZAP_MS_UNFLUSHED_PHYS_TXGS, sizeof (uint64_t), 1,
 		    &object, tx));
 	} else {
 		VERIFY0(err);
 	}
 
 	dmu_write(spa_meta_objset(spa), object, entry_offset, entry_size,
 	    &entry, tx);
 }
 
 void
 metaslab_set_unflushed_txg(metaslab_t *ms, uint64_t txg, dmu_tx_t *tx)
 {
 	ms->ms_unflushed_txg = txg;
 	metaslab_update_ondisk_flush_data(ms, tx);
 }
 
 boolean_t
 metaslab_unflushed_dirty(metaslab_t *ms)
 {
 	return (ms->ms_unflushed_dirty);
 }
 
 uint64_t
 metaslab_unflushed_txg(metaslab_t *ms)
 {
 	return (ms->ms_unflushed_txg);
 }
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, aliquot, U64, ZMOD_RW,
 	"Allocation granularity (a.k.a. stripe size)");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, debug_load, INT, ZMOD_RW,
 	"Load all metaslabs when pool is first opened");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, debug_unload, INT, ZMOD_RW,
 	"Prevent metaslabs from being unloaded");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_enabled, INT, ZMOD_RW,
 	"Preload potential metaslabs during reassessment");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, preload_limit, UINT, ZMOD_RW,
 	"Max number of metaslabs per group to preload");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, unload_delay, UINT, ZMOD_RW,
 	"Delay in txgs after metaslab was last used before unloading");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, unload_delay_ms, UINT, ZMOD_RW,
 	"Delay in milliseconds after metaslab was last used before unloading");
 
 /* BEGIN CSTYLED */
 ZFS_MODULE_PARAM(zfs_mg, zfs_mg_, noalloc_threshold, UINT, ZMOD_RW,
 	"Percentage of metaslab group size that should be free to make it "
 	"eligible for allocation");
 
 ZFS_MODULE_PARAM(zfs_mg, zfs_mg_, fragmentation_threshold, UINT, ZMOD_RW,
 	"Percentage of metaslab group size that should be considered eligible "
 	"for allocations unless all metaslab groups within the metaslab class "
 	"have also crossed this threshold");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, fragmentation_factor_enabled, INT,
 	ZMOD_RW,
 	"Use the fragmentation metric to prefer less fragmented metaslabs");
 /* END CSTYLED */
 
 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, fragmentation_threshold, UINT,
 	ZMOD_RW, "Fragmentation for metaslab to allow allocation");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, lba_weighting_enabled, INT, ZMOD_RW,
 	"Prefer metaslabs with lower LBAs");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, bias_enabled, INT, ZMOD_RW,
 	"Enable metaslab group biasing");
 
 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, segment_weight_enabled, INT,
 	ZMOD_RW, "Enable segment-based metaslab selection");
 
 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, switch_threshold, INT, ZMOD_RW,
 	"Segment-based metaslab selection maximum buckets before switching");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, force_ganging, U64, ZMOD_RW,
 	"Blocks larger than this size are sometimes forced to be gang blocks");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, force_ganging_pct, UINT, ZMOD_RW,
 	"Percentage of large blocks that will be forced to be gang blocks");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, df_max_search, UINT, ZMOD_RW,
 	"Max distance (bytes) to search forward before using size tree");
 
 ZFS_MODULE_PARAM(zfs_metaslab, metaslab_, df_use_largest_segment, INT, ZMOD_RW,
 	"When looking in size tree, use largest segment instead of exact fit");
 
 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, max_size_cache_sec, U64,
 	ZMOD_RW, "How long to trust the cached max chunk size of a metaslab");
 
 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, mem_limit, UINT, ZMOD_RW,
 	"Percentage of memory that can be used to store metaslab range trees");
 
 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, try_hard_before_gang, INT,
 	ZMOD_RW, "Try hard to allocate before ganging");
 
 ZFS_MODULE_PARAM(zfs_metaslab, zfs_metaslab_, find_max_tries, UINT, ZMOD_RW,
 	"Normally only consider this many of the best metaslabs in each vdev");
diff --git a/sys/contrib/openzfs/module/zfs/zcp.c b/sys/contrib/openzfs/module/zfs/zcp.c
index 7c279162a9d1..c92067fbd9a2 100644
--- a/sys/contrib/openzfs/module/zfs/zcp.c
+++ b/sys/contrib/openzfs/module/zfs/zcp.c
@@ -1,1449 +1,1453 @@
 /*
  * CDDL HEADER START
  *
  * This file and its contents are supplied under the terms of the
  * Common Development and Distribution License ("CDDL"), version 1.0.
  * You may only use this file in accordance with the terms of version
  * 1.0 of the CDDL.
  *
  * A full copy of the text of the CDDL should have accompanied this
  * source.  A copy of the CDDL is also available via the Internet at
  * http://www.illumos.org/license/CDDL.
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2016, 2018 by Delphix. All rights reserved.
  */
 
 /*
  * ZFS Channel Programs (ZCP)
  *
  * The ZCP interface allows various ZFS commands and operations ZFS
  * administrative operations (e.g. creating and destroying snapshots, typically
  * performed via an ioctl to /dev/zfs by the zfs(8) command and
  * libzfs/libzfs_core) to be run * programmatically as a Lua script.  A ZCP
  * script is run as a dsl_sync_task and fully executed during one transaction
  * group sync.  This ensures that no other changes can be written concurrently
  * with a running Lua script.  Combining multiple calls to the exposed ZFS
  * functions into one script gives a number of benefits:
  *
  * 1. Atomicity.  For some compound or iterative operations, it's useful to be
  * able to guarantee that the state of a pool has not changed between calls to
  * ZFS.
  *
  * 2. Performance.  If a large number of changes need to be made (e.g. deleting
  * many filesystems), there can be a significant performance penalty as a
  * result of the need to wait for a transaction group sync to pass for every
  * single operation.  When expressed as a single ZCP script, all these changes
  * can be performed at once in one txg sync.
  *
  * A modified version of the Lua 5.2 interpreter is used to run channel program
  * scripts. The Lua 5.2 manual can be found at:
  *
  *      http://www.lua.org/manual/5.2/
  *
  * If being run by a user (via an ioctl syscall), executing a ZCP script
  * requires root privileges in the global zone.
  *
  * Scripts are passed to zcp_eval() as a string, then run in a synctask by
  * zcp_eval_sync().  Arguments can be passed into the Lua script as an nvlist,
  * which will be converted to a Lua table.  Similarly, values returned from
  * a ZCP script will be converted to an nvlist.  See zcp_lua_to_nvlist_impl()
  * for details on exact allowed types and conversion.
  *
  * ZFS functionality is exposed to a ZCP script as a library of function calls.
  * These calls are sorted into submodules, such as zfs.list and zfs.sync, for
  * iterators and synctasks, respectively.  Each of these submodules resides in
  * its own source file, with a zcp_*_info structure describing each library
  * call in the submodule.
  *
  * Error handling in ZCP scripts is handled by a number of different methods
  * based on severity:
  *
  * 1. Memory and time limits are in place to prevent a channel program from
  * consuming excessive system or running forever.  If one of these limits is
  * hit, the channel program will be stopped immediately and return from
  * zcp_eval() with an error code. No attempt will be made to roll back or undo
  * any changes made by the channel program before the error occurred.
  * Consumers invoking zcp_eval() from elsewhere in the kernel may pass a time
  * limit of 0, disabling the time limit.
  *
  * 2. Internal Lua errors can occur as a result of a syntax error, calling a
  * library function with incorrect arguments, invoking the error() function,
  * failing an assert(), or other runtime errors.  In these cases the channel
  * program will stop executing and return from zcp_eval() with an error code.
  * In place of a return value, an error message will also be returned in the
  * 'result' nvlist containing information about the error. No attempt will be
  * made to roll back or undo any changes made by the channel program before the
  * error occurred.
  *
  * 3. If an error occurs inside a ZFS library call which returns an error code,
  * the error is returned to the Lua script to be handled as desired.
  *
  * In the first two cases, Lua's error-throwing mechanism is used, which
  * longjumps out of the script execution with luaL_error() and returns with the
  * error.
  *
  * See zfs-program(8) for more information on high level usage.
  */
 
 #include <sys/lua/lua.h>
 #include <sys/lua/lualib.h>
 #include <sys/lua/lauxlib.h>
 
 #include <sys/dsl_prop.h>
 #include <sys/dsl_synctask.h>
 #include <sys/dsl_dataset.h>
 #include <sys/zcp.h>
 #include <sys/zcp_iter.h>
 #include <sys/zcp_prop.h>
 #include <sys/zcp_global.h>
 #include <sys/zvol.h>
 
 #ifndef KM_NORMALPRI
 #define	KM_NORMALPRI	0
 #endif
 
 #define	ZCP_NVLIST_MAX_DEPTH 20
 
 static const uint64_t zfs_lua_check_instrlimit_interval = 100;
 uint64_t zfs_lua_max_instrlimit = ZCP_MAX_INSTRLIMIT;
 uint64_t zfs_lua_max_memlimit = ZCP_MAX_MEMLIMIT;
 
 /*
  * Forward declarations for mutually recursive functions
  */
 static int zcp_nvpair_value_to_lua(lua_State *, nvpair_t *, char *, int);
 static int zcp_lua_to_nvlist_impl(lua_State *, int, nvlist_t *, const char *,
     int);
 
 /*
  * The outer-most error callback handler for use with lua_pcall(). On
  * error Lua will call this callback with a single argument that
  * represents the error value. In most cases this will be a string
  * containing an error message, but channel programs can use Lua's
  * error() function to return arbitrary objects as errors. This callback
  * returns (on the Lua stack) the original error object along with a traceback.
  *
  * Fatal Lua errors can occur while resources are held, so we also call any
  * registered cleanup function here.
  */
 static int
 zcp_error_handler(lua_State *state)
 {
 	const char *msg;
 
 	zcp_cleanup(state);
 
 	VERIFY3U(1, ==, lua_gettop(state));
 	msg = lua_tostring(state, 1);
 	luaL_traceback(state, state, msg, 1);
 	return (1);
 }
 
 int
 zcp_argerror(lua_State *state, int narg, const char *msg, ...)
 {
 	va_list alist;
 
 	va_start(alist, msg);
 	const char *buf = lua_pushvfstring(state, msg, alist);
 	va_end(alist);
 
 	return (luaL_argerror(state, narg, buf));
 }
 
 /*
  * Install a new cleanup function, which will be invoked with the given
  * opaque argument if a fatal error causes the Lua interpreter to longjump out
  * of a function call.
  *
  * If an error occurs, the cleanup function will be invoked exactly once and
  * then unregistered.
  *
  * Returns the registered cleanup handler so the caller can deregister it
  * if no error occurs.
  */
 zcp_cleanup_handler_t *
 zcp_register_cleanup(lua_State *state, zcp_cleanup_t cleanfunc, void *cleanarg)
 {
 	zcp_run_info_t *ri = zcp_run_info(state);
 
 	zcp_cleanup_handler_t *zch = kmem_alloc(sizeof (*zch), KM_SLEEP);
 	zch->zch_cleanup_func = cleanfunc;
 	zch->zch_cleanup_arg = cleanarg;
 	list_insert_head(&ri->zri_cleanup_handlers, zch);
 
 	return (zch);
 }
 
 void
 zcp_deregister_cleanup(lua_State *state, zcp_cleanup_handler_t *zch)
 {
 	zcp_run_info_t *ri = zcp_run_info(state);
 	list_remove(&ri->zri_cleanup_handlers, zch);
 	kmem_free(zch, sizeof (*zch));
 }
 
 /*
  * Execute the currently registered cleanup handlers then free them and
  * destroy the handler list.
  */
 void
 zcp_cleanup(lua_State *state)
 {
 	zcp_run_info_t *ri = zcp_run_info(state);
 
 	for (zcp_cleanup_handler_t *zch =
 	    list_remove_head(&ri->zri_cleanup_handlers); zch != NULL;
 	    zch = list_remove_head(&ri->zri_cleanup_handlers)) {
 		zch->zch_cleanup_func(zch->zch_cleanup_arg);
 		kmem_free(zch, sizeof (*zch));
 	}
 }
 
 /*
  * Convert the lua table at the given index on the Lua stack to an nvlist
  * and return it.
  *
  * If the table can not be converted for any reason, NULL is returned and
  * an error message is pushed onto the Lua stack.
  */
 static nvlist_t *
 zcp_table_to_nvlist(lua_State *state, int index, int depth)
 {
 	nvlist_t *nvl;
 	/*
 	 * Converting a Lua table to an nvlist with key uniqueness checking is
 	 * O(n^2) in the number of keys in the nvlist, which can take a long
 	 * time when we return a large table from a channel program.
 	 * Furthermore, Lua's table interface *almost* guarantees unique keys
 	 * on its own (details below). Therefore, we don't use fnvlist_alloc()
 	 * here to avoid the built-in uniqueness checking.
 	 *
 	 * The *almost* is because it's possible to have key collisions between
 	 * e.g. the string "1" and the number 1, or the string "true" and the
 	 * boolean true, so we explicitly check that when we're looking at a
 	 * key which is an integer / boolean or a string that can be parsed as
 	 * one of those types. In the worst case this could still devolve into
 	 * O(n^2), so we only start doing these checks on boolean/integer keys
 	 * once we've seen a string key which fits this weird usage pattern.
 	 *
 	 * Ultimately, we still want callers to know that the keys in this
 	 * nvlist are unique, so before we return this we set the nvlist's
 	 * flags to reflect that.
 	 */
 	VERIFY0(nvlist_alloc(&nvl, 0, KM_SLEEP));
 
 	/*
 	 * Push an empty stack slot where lua_next() will store each
 	 * table key.
 	 */
 	lua_pushnil(state);
 	boolean_t saw_str_could_collide = B_FALSE;
 	while (lua_next(state, index) != 0) {
 		/*
 		 * The next key-value pair from the table at index is
 		 * now on the stack, with the key at stack slot -2 and
 		 * the value at slot -1.
 		 */
 		int err = 0;
 		char buf[32];
 		const char *key = NULL;
 		boolean_t key_could_collide = B_FALSE;
 
 		switch (lua_type(state, -2)) {
 		case LUA_TSTRING:
 			key = lua_tostring(state, -2);
 
 			/* check if this could collide with a number or bool */
 			long long tmp;
 			int parselen;
 			if ((sscanf(key, "%lld%n", &tmp, &parselen) > 0 &&
 			    parselen == strlen(key)) ||
 			    strcmp(key, "true") == 0 ||
 			    strcmp(key, "false") == 0) {
 				key_could_collide = B_TRUE;
 				saw_str_could_collide = B_TRUE;
 			}
 			break;
 		case LUA_TBOOLEAN:
 			key = (lua_toboolean(state, -2) == B_TRUE ?
 			    "true" : "false");
 			if (saw_str_could_collide) {
 				key_could_collide = B_TRUE;
 			}
 			break;
 		case LUA_TNUMBER:
 			(void) snprintf(buf, sizeof (buf), "%lld",
 			    (longlong_t)lua_tonumber(state, -2));
 
 			key = buf;
 			if (saw_str_could_collide) {
 				key_could_collide = B_TRUE;
 			}
 			break;
 		default:
 			fnvlist_free(nvl);
 			(void) lua_pushfstring(state, "Invalid key "
 			    "type '%s' in table",
 			    lua_typename(state, lua_type(state, -2)));
 			return (NULL);
 		}
 		/*
 		 * Check for type-mismatched key collisions, and throw an error.
 		 */
 		if (key_could_collide && nvlist_exists(nvl, key)) {
 			fnvlist_free(nvl);
 			(void) lua_pushfstring(state, "Collision of "
 			    "key '%s' in table", key);
 			return (NULL);
 		}
 		/*
 		 * Recursively convert the table value and insert into
 		 * the new nvlist with the parsed key.  To prevent
 		 * stack overflow on circular or heavily nested tables,
 		 * we track the current nvlist depth.
 		 */
 		if (depth >= ZCP_NVLIST_MAX_DEPTH) {
 			fnvlist_free(nvl);
 			(void) lua_pushfstring(state, "Maximum table "
 			    "depth (%d) exceeded for table",
 			    ZCP_NVLIST_MAX_DEPTH);
 			return (NULL);
 		}
 		err = zcp_lua_to_nvlist_impl(state, -1, nvl, key,
 		    depth + 1);
 		if (err != 0) {
 			fnvlist_free(nvl);
 			/*
 			 * Error message has been pushed to the lua
 			 * stack by the recursive call.
 			 */
 			return (NULL);
 		}
 		/*
 		 * Pop the value pushed by lua_next().
 		 */
 		lua_pop(state, 1);
 	}
 
 	/*
 	 * Mark the nvlist as having unique keys. This is a little ugly, but we
 	 * ensured above that there are no duplicate keys in the nvlist.
 	 */
 	nvl->nvl_nvflag |= NV_UNIQUE_NAME;
 
 	return (nvl);
 }
 
 /*
  * Convert a value from the given index into the lua stack to an nvpair, adding
  * it to an nvlist with the given key.
  *
  * Values are converted as follows:
  *
  *   string -> string
  *   number -> int64
  *   boolean -> boolean
  *   nil -> boolean (no value)
  *
  * Lua tables are converted to nvlists and then inserted. The table's keys
  * are converted to strings then used as keys in the nvlist to store each table
  * element.  Keys are converted as follows:
  *
  *   string -> no change
  *   number -> "%lld"
  *   boolean -> "true" | "false"
  *   nil -> error
  *
  * In the case of a key collision, an error is thrown.
  *
  * If an error is encountered, a nonzero error code is returned, and an error
  * string will be pushed onto the Lua stack.
  */
 static int
 zcp_lua_to_nvlist_impl(lua_State *state, int index, nvlist_t *nvl,
     const char *key, int depth)
 {
 	/*
 	 * Verify that we have enough remaining space in the lua stack to parse
 	 * a key-value pair and push an error.
 	 */
 	if (!lua_checkstack(state, 3)) {
 		(void) lua_pushstring(state, "Lua stack overflow");
 		return (1);
 	}
 
 	index = lua_absindex(state, index);
 
 	switch (lua_type(state, index)) {
 	case LUA_TNIL:
 		fnvlist_add_boolean(nvl, key);
 		break;
 	case LUA_TBOOLEAN:
 		fnvlist_add_boolean_value(nvl, key,
 		    lua_toboolean(state, index));
 		break;
 	case LUA_TNUMBER:
 		fnvlist_add_int64(nvl, key, lua_tonumber(state, index));
 		break;
 	case LUA_TSTRING:
 		fnvlist_add_string(nvl, key, lua_tostring(state, index));
 		break;
 	case LUA_TTABLE: {
 		nvlist_t *value_nvl = zcp_table_to_nvlist(state, index, depth);
 		if (value_nvl == NULL)
 			return (SET_ERROR(EINVAL));
 
 		fnvlist_add_nvlist(nvl, key, value_nvl);
 		fnvlist_free(value_nvl);
 		break;
 	}
 	default:
 		(void) lua_pushfstring(state,
 		    "Invalid value type '%s' for key '%s'",
 		    lua_typename(state, lua_type(state, index)), key);
 		return (SET_ERROR(EINVAL));
 	}
 
 	return (0);
 }
 
 /*
  * Convert a lua value to an nvpair, adding it to an nvlist with the given key.
  */
 static void
 zcp_lua_to_nvlist(lua_State *state, int index, nvlist_t *nvl, const char *key)
 {
 	/*
 	 * On error, zcp_lua_to_nvlist_impl pushes an error string onto the Lua
 	 * stack before returning with a nonzero error code. If an error is
 	 * returned, throw a fatal lua error with the given string.
 	 */
 	if (zcp_lua_to_nvlist_impl(state, index, nvl, key, 0) != 0)
 		(void) lua_error(state);
 }
 
 static int
 zcp_lua_to_nvlist_helper(lua_State *state)
 {
 	nvlist_t *nv = (nvlist_t *)lua_touserdata(state, 2);
 	const char *key = (const char *)lua_touserdata(state, 1);
 	zcp_lua_to_nvlist(state, 3, nv, key);
 	return (0);
 }
 
 static void
 zcp_convert_return_values(lua_State *state, nvlist_t *nvl,
     const char *key, int *result)
 {
 	int err;
 	VERIFY3U(1, ==, lua_gettop(state));
 	lua_pushcfunction(state, zcp_lua_to_nvlist_helper);
 	lua_pushlightuserdata(state, (char *)key);
 	lua_pushlightuserdata(state, nvl);
 	lua_pushvalue(state, 1);
 	lua_remove(state, 1);
 	err = lua_pcall(state, 3, 0, 0); /* zcp_lua_to_nvlist_helper */
 	if (err != 0) {
 		zcp_lua_to_nvlist(state, 1, nvl, ZCP_RET_ERROR);
 		*result = SET_ERROR(ECHRNG);
 	}
 }
 
 /*
  * Push a Lua table representing nvl onto the stack.  If it can't be
  * converted, return EINVAL, fill in errbuf, and push nothing. errbuf may
  * be specified as NULL, in which case no error string will be output.
  *
  * Most nvlists are converted as simple key->value Lua tables, but we make
  * an exception for the case where all nvlist entries are BOOLEANs (a string
  * key without a value). In Lua, a table key pointing to a value of Nil
  * (no value) is equivalent to the key not existing, so a BOOLEAN nvlist
  * entry can't be directly converted to a Lua table entry. Nvlists of entirely
  * BOOLEAN entries are frequently used to pass around lists of datasets, so for
  * convenience we check for this case, and convert it to a simple Lua array of
  * strings.
  */
 int
 zcp_nvlist_to_lua(lua_State *state, nvlist_t *nvl,
     char *errbuf, int errbuf_len)
 {
 	nvpair_t *pair;
 	lua_newtable(state);
 	boolean_t has_values = B_FALSE;
 	/*
 	 * If the list doesn't have any values, just convert it to a string
 	 * array.
 	 */
 	for (pair = nvlist_next_nvpair(nvl, NULL);
 	    pair != NULL; pair = nvlist_next_nvpair(nvl, pair)) {
 		if (nvpair_type(pair) != DATA_TYPE_BOOLEAN) {
 			has_values = B_TRUE;
 			break;
 		}
 	}
 	if (!has_values) {
 		int i = 1;
 		for (pair = nvlist_next_nvpair(nvl, NULL);
 		    pair != NULL; pair = nvlist_next_nvpair(nvl, pair)) {
 			(void) lua_pushinteger(state, i);
 			(void) lua_pushstring(state, nvpair_name(pair));
 			(void) lua_settable(state, -3);
 			i++;
 		}
 	} else {
 		for (pair = nvlist_next_nvpair(nvl, NULL);
 		    pair != NULL; pair = nvlist_next_nvpair(nvl, pair)) {
 			int err = zcp_nvpair_value_to_lua(state, pair,
 			    errbuf, errbuf_len);
 			if (err != 0) {
 				lua_pop(state, 1);
 				return (err);
 			}
 			(void) lua_setfield(state, -2, nvpair_name(pair));
 		}
 	}
 	return (0);
 }
 
 /*
  * Push a Lua object representing the value of "pair" onto the stack.
  *
  * Only understands boolean_value, string, int64, nvlist,
  * string_array, and int64_array type values.  For other
  * types, returns EINVAL, fills in errbuf, and pushes nothing.
  */
 static int
 zcp_nvpair_value_to_lua(lua_State *state, nvpair_t *pair,
     char *errbuf, int errbuf_len)
 {
 	int err = 0;
 
 	if (pair == NULL) {
 		lua_pushnil(state);
 		return (0);
 	}
 
 	switch (nvpair_type(pair)) {
 	case DATA_TYPE_BOOLEAN_VALUE:
 		(void) lua_pushboolean(state,
 		    fnvpair_value_boolean_value(pair));
 		break;
 	case DATA_TYPE_STRING:
 		(void) lua_pushstring(state, fnvpair_value_string(pair));
 		break;
 	case DATA_TYPE_INT64:
 		(void) lua_pushinteger(state, fnvpair_value_int64(pair));
 		break;
 	case DATA_TYPE_NVLIST:
 		err = zcp_nvlist_to_lua(state,
 		    fnvpair_value_nvlist(pair), errbuf, errbuf_len);
 		break;
 	case DATA_TYPE_STRING_ARRAY: {
 		const char **strarr;
 		uint_t nelem;
 		(void) nvpair_value_string_array(pair, &strarr, &nelem);
 		lua_newtable(state);
 		for (int i = 0; i < nelem; i++) {
 			(void) lua_pushinteger(state, i + 1);
 			(void) lua_pushstring(state, strarr[i]);
 			(void) lua_settable(state, -3);
 		}
 		break;
 	}
 	case DATA_TYPE_UINT64_ARRAY: {
 		uint64_t *intarr;
 		uint_t nelem;
 		(void) nvpair_value_uint64_array(pair, &intarr, &nelem);
 		lua_newtable(state);
 		for (int i = 0; i < nelem; i++) {
 			(void) lua_pushinteger(state, i + 1);
 			(void) lua_pushinteger(state, intarr[i]);
 			(void) lua_settable(state, -3);
 		}
 		break;
 	}
 	case DATA_TYPE_INT64_ARRAY: {
 		int64_t *intarr;
 		uint_t nelem;
 		(void) nvpair_value_int64_array(pair, &intarr, &nelem);
 		lua_newtable(state);
 		for (int i = 0; i < nelem; i++) {
 			(void) lua_pushinteger(state, i + 1);
 			(void) lua_pushinteger(state, intarr[i]);
 			(void) lua_settable(state, -3);
 		}
 		break;
 	}
 	default: {
 		if (errbuf != NULL) {
 			(void) snprintf(errbuf, errbuf_len,
 			    "Unhandled nvpair type %d for key '%s'",
 			    nvpair_type(pair), nvpair_name(pair));
 		}
 		return (SET_ERROR(EINVAL));
 	}
 	}
 	return (err);
 }
 
 int
 zcp_dataset_hold_error(lua_State *state, dsl_pool_t *dp, const char *dsname,
     int error)
 {
 	if (error == ENOENT) {
 		(void) zcp_argerror(state, 1, "no such dataset '%s'", dsname);
 		return (0); /* not reached; zcp_argerror will longjmp */
 	} else if (error == EXDEV) {
 		(void) zcp_argerror(state, 1,
 		    "dataset '%s' is not in the target pool '%s'",
 		    dsname, spa_name(dp->dp_spa));
 		return (0); /* not reached; zcp_argerror will longjmp */
 	} else if (error == EIO) {
 		(void) luaL_error(state,
 		    "I/O error while accessing dataset '%s'", dsname);
 		return (0); /* not reached; luaL_error will longjmp */
 	} else if (error != 0) {
 		(void) luaL_error(state,
 		    "unexpected error %d while accessing dataset '%s'",
 		    error, dsname);
 		return (0); /* not reached; luaL_error will longjmp */
 	}
 	return (0);
 }
 
 /*
  * Note: will longjmp (via lua_error()) on error.
  * Assumes that the dsname is argument #1 (for error reporting purposes).
  */
 dsl_dataset_t *
 zcp_dataset_hold(lua_State *state, dsl_pool_t *dp, const char *dsname,
     const void *tag)
 {
 	dsl_dataset_t *ds;
 	int error = dsl_dataset_hold(dp, dsname, tag, &ds);
 	(void) zcp_dataset_hold_error(state, dp, dsname, error);
 	return (ds);
 }
 
 static int zcp_debug(lua_State *);
 static const zcp_lib_info_t zcp_debug_info = {
 	.name = "debug",
 	.func = zcp_debug,
 	.pargs = {
 	    { .za_name = "debug string", .za_lua_type = LUA_TSTRING },
 	    {NULL, 0}
 	},
 	.kwargs = {
 	    {NULL, 0}
 	}
 };
 
 static int
 zcp_debug(lua_State *state)
 {
 	const char *dbgstring;
 	zcp_run_info_t *ri = zcp_run_info(state);
 	const zcp_lib_info_t *libinfo = &zcp_debug_info;
 
 	zcp_parse_args(state, libinfo->name, libinfo->pargs, libinfo->kwargs);
 
 	dbgstring = lua_tostring(state, 1);
 
 	zfs_dbgmsg("txg %lld ZCP: %s", (longlong_t)ri->zri_tx->tx_txg,
 	    dbgstring);
 
 	return (0);
 }
 
 static int zcp_exists(lua_State *);
 static const zcp_lib_info_t zcp_exists_info = {
 	.name = "exists",
 	.func = zcp_exists,
 	.pargs = {
 	    { .za_name = "dataset", .za_lua_type = LUA_TSTRING },
 	    {NULL, 0}
 	},
 	.kwargs = {
 	    {NULL, 0}
 	}
 };
 
 static int
 zcp_exists(lua_State *state)
 {
 	zcp_run_info_t *ri = zcp_run_info(state);
 	dsl_pool_t *dp = ri->zri_pool;
 	const zcp_lib_info_t *libinfo = &zcp_exists_info;
 
 	zcp_parse_args(state, libinfo->name, libinfo->pargs, libinfo->kwargs);
 
 	const char *dsname = lua_tostring(state, 1);
 
 	dsl_dataset_t *ds;
 	int error = dsl_dataset_hold(dp, dsname, FTAG, &ds);
 	if (error == 0) {
 		dsl_dataset_rele(ds, FTAG);
 		lua_pushboolean(state, B_TRUE);
 	} else if (error == ENOENT) {
 		lua_pushboolean(state, B_FALSE);
 	} else if (error == EXDEV) {
 		return (luaL_error(state, "dataset '%s' is not in the "
 		    "target pool", dsname));
 	} else if (error == EIO) {
 		return (luaL_error(state, "I/O error opening dataset '%s'",
 		    dsname));
 	} else if (error != 0) {
 		return (luaL_error(state, "unexpected error %d", error));
 	}
 
 	return (1);
 }
 
 /*
  * Allocate/realloc/free a buffer for the lua interpreter.
  *
  * When nsize is 0, behaves as free() and returns NULL.
  *
  * If ptr is NULL, behaves as malloc() and returns an allocated buffer of size
  * at least nsize.
  *
  * Otherwise, behaves as realloc(), changing the allocation from osize to nsize.
  * Shrinking the buffer size never fails.
  *
  * The original allocated buffer size is stored as a uint64 at the beginning of
  * the buffer to avoid actually reallocating when shrinking a buffer, since lua
  * requires that this operation never fail.
  */
 static void *
 zcp_lua_alloc(void *ud, void *ptr, size_t osize, size_t nsize)
 {
 	zcp_alloc_arg_t *allocargs = ud;
 
 	if (nsize == 0) {
 		if (ptr != NULL) {
 			int64_t *allocbuf = (int64_t *)ptr - 1;
 			int64_t allocsize = *allocbuf;
 			ASSERT3S(allocsize, >, 0);
 			ASSERT3S(allocargs->aa_alloc_remaining + allocsize, <=,
 			    allocargs->aa_alloc_limit);
 			allocargs->aa_alloc_remaining += allocsize;
 			vmem_free(allocbuf, allocsize);
 		}
 		return (NULL);
 	} else if (ptr == NULL) {
 		int64_t *allocbuf;
 		int64_t allocsize = nsize + sizeof (int64_t);
 
 		if (!allocargs->aa_must_succeed &&
 		    (allocsize <= 0 ||
 		    allocsize > allocargs->aa_alloc_remaining)) {
 			return (NULL);
 		}
 
 		allocbuf = vmem_alloc(allocsize, KM_SLEEP);
 		allocargs->aa_alloc_remaining -= allocsize;
 
 		*allocbuf = allocsize;
 		return (allocbuf + 1);
 	} else if (nsize <= osize) {
 		/*
 		 * If shrinking the buffer, lua requires that the reallocation
 		 * never fail.
 		 */
 		return (ptr);
 	} else {
 		ASSERT3U(nsize, >, osize);
 
 		uint64_t *luabuf = zcp_lua_alloc(ud, NULL, 0, nsize);
 		if (luabuf == NULL) {
 			return (NULL);
 		}
 		(void) memcpy(luabuf, ptr, osize);
 		VERIFY3P(zcp_lua_alloc(ud, ptr, osize, 0), ==, NULL);
 		return (luabuf);
 	}
 }
 
 static void
 zcp_lua_counthook(lua_State *state, lua_Debug *ar)
 {
 	(void) ar;
 	lua_getfield(state, LUA_REGISTRYINDEX, ZCP_RUN_INFO_KEY);
 	zcp_run_info_t *ri = lua_touserdata(state, -1);
 
 	/*
 	 * Check if we were canceled while waiting for the
 	 * txg to sync or from our open context thread
 	 */
 	if (ri->zri_canceled || (!ri->zri_sync && issig())) {
 		ri->zri_canceled = B_TRUE;
 		(void) lua_pushstring(state, "Channel program was canceled.");
 		(void) lua_error(state);
 		/* Unreachable */
 	}
 
 	/*
 	 * Check how many instructions the channel program has
 	 * executed so far, and compare against the limit.
 	 */
 	ri->zri_curinstrs += zfs_lua_check_instrlimit_interval;
 	if (ri->zri_maxinstrs != 0 && ri->zri_curinstrs > ri->zri_maxinstrs) {
 		ri->zri_timed_out = B_TRUE;
 		(void) lua_pushstring(state,
 		    "Channel program timed out.");
 		(void) lua_error(state);
 		/* Unreachable */
 	}
 }
 
 static int
 zcp_panic_cb(lua_State *state)
 {
 	panic("unprotected error in call to Lua API (%s)\n",
 	    lua_tostring(state, -1));
 	return (0);
 }
 
 static void
 zcp_eval_impl(dmu_tx_t *tx, zcp_run_info_t *ri)
 {
 	int err;
 	lua_State *state = ri->zri_state;
 
 	VERIFY3U(3, ==, lua_gettop(state));
 
 	/* finish initializing our runtime state */
 	ri->zri_pool = dmu_tx_pool(tx);
 	ri->zri_tx = tx;
 	list_create(&ri->zri_cleanup_handlers, sizeof (zcp_cleanup_handler_t),
 	    offsetof(zcp_cleanup_handler_t, zch_node));
 
 	/*
 	 * Store the zcp_run_info_t struct for this run in the Lua registry.
 	 * Registry entries are not directly accessible by the Lua scripts but
 	 * can be accessed by our callbacks.
 	 */
 	lua_pushlightuserdata(state, ri);
 	lua_setfield(state, LUA_REGISTRYINDEX, ZCP_RUN_INFO_KEY);
 	VERIFY3U(3, ==, lua_gettop(state));
 
 	/*
 	 * Tell the Lua interpreter to call our handler every count
 	 * instructions. Channel programs that execute too many instructions
 	 * should die with ETIME.
 	 */
 	(void) lua_sethook(state, zcp_lua_counthook, LUA_MASKCOUNT,
 	    zfs_lua_check_instrlimit_interval);
 
 	/*
 	 * Tell the Lua memory allocator to stop using KM_SLEEP before handing
 	 * off control to the channel program. Channel programs that use too
 	 * much memory should die with ENOSPC.
 	 */
 	ri->zri_allocargs->aa_must_succeed = B_FALSE;
 
 	/*
 	 * Call the Lua function that open-context passed us. This pops the
 	 * function and its input from the stack and pushes any return
 	 * or error values.
 	 */
 	err = lua_pcall(state, 1, LUA_MULTRET, 1);
 
 	/*
 	 * Let Lua use KM_SLEEP while we interpret the return values.
 	 */
 	ri->zri_allocargs->aa_must_succeed = B_TRUE;
 
 	/*
 	 * Remove the error handler callback from the stack. At this point,
 	 * there shouldn't be any cleanup handler registered in the handler
 	 * list (zri_cleanup_handlers), regardless of whether it ran or not.
 	 */
 	list_destroy(&ri->zri_cleanup_handlers);
 	lua_remove(state, 1);
 
 	switch (err) {
 	case LUA_OK: {
 		/*
 		 * Lua supports returning multiple values in a single return
 		 * statement.  Return values will have been pushed onto the
 		 * stack:
 		 * 1: Return value 1
 		 * 2: Return value 2
 		 * 3: etc...
 		 * To simplify the process of retrieving a return value from a
 		 * channel program, we disallow returning more than one value
 		 * to ZFS from the Lua script, yielding a singleton return
 		 * nvlist of the form { "return": Return value 1 }.
 		 */
 		int return_count = lua_gettop(state);
 
 		if (return_count == 1) {
 			ri->zri_result = 0;
 			zcp_convert_return_values(state, ri->zri_outnvl,
 			    ZCP_RET_RETURN, &ri->zri_result);
 		} else if (return_count > 1) {
 			ri->zri_result = SET_ERROR(ECHRNG);
 			lua_settop(state, 0);
 			(void) lua_pushfstring(state, "Multiple return "
 			    "values not supported");
 			zcp_convert_return_values(state, ri->zri_outnvl,
 			    ZCP_RET_ERROR, &ri->zri_result);
 		}
 		break;
 	}
 	case LUA_ERRRUN:
 	case LUA_ERRGCMM: {
 		/*
 		 * The channel program encountered a fatal error within the
 		 * script, such as failing an assertion, or calling a function
 		 * with incompatible arguments. The error value and the
 		 * traceback generated by zcp_error_handler() should be on the
 		 * stack.
 		 */
 		VERIFY3U(1, ==, lua_gettop(state));
 		if (ri->zri_timed_out) {
 			ri->zri_result = SET_ERROR(ETIME);
 		} else if (ri->zri_canceled) {
 			ri->zri_result = SET_ERROR(EINTR);
 		} else {
 			ri->zri_result = SET_ERROR(ECHRNG);
 		}
 
 		zcp_convert_return_values(state, ri->zri_outnvl,
 		    ZCP_RET_ERROR, &ri->zri_result);
 
 		if (ri->zri_result == ETIME && ri->zri_outnvl != NULL) {
 			(void) nvlist_add_uint64(ri->zri_outnvl,
 			    ZCP_ARG_INSTRLIMIT, ri->zri_curinstrs);
 		}
 		break;
 	}
 	case LUA_ERRERR: {
 		/*
 		 * The channel program encountered a fatal error within the
 		 * script, and we encountered another error while trying to
 		 * compute the traceback in zcp_error_handler(). We can only
 		 * return the error message.
 		 */
 		VERIFY3U(1, ==, lua_gettop(state));
 		if (ri->zri_timed_out) {
 			ri->zri_result = SET_ERROR(ETIME);
 		} else if (ri->zri_canceled) {
 			ri->zri_result = SET_ERROR(EINTR);
 		} else {
 			ri->zri_result = SET_ERROR(ECHRNG);
 		}
 
 		zcp_convert_return_values(state, ri->zri_outnvl,
 		    ZCP_RET_ERROR, &ri->zri_result);
 		break;
 	}
 	case LUA_ERRMEM:
 		/*
 		 * Lua ran out of memory while running the channel program.
 		 * There's not much we can do.
 		 */
 		ri->zri_result = SET_ERROR(ENOSPC);
 		break;
 	default:
 		VERIFY0(err);
 	}
 }
 
 static void
 zcp_pool_error(zcp_run_info_t *ri, const char *poolname, int error)
 {
 	ri->zri_result = SET_ERROR(ECHRNG);
 	lua_settop(ri->zri_state, 0);
 	(void) lua_pushfstring(ri->zri_state, "Could not open pool: %s "
 	    "errno: %d", poolname, error);
 	zcp_convert_return_values(ri->zri_state, ri->zri_outnvl,
 	    ZCP_RET_ERROR, &ri->zri_result);
 
 }
 
 /*
  * This callback is called when txg_wait_synced_sig encountered a signal.
  * The txg_wait_synced_sig will continue to wait for the txg to complete
  * after calling this callback.
  */
 static void
 zcp_eval_sig(void *arg, dmu_tx_t *tx)
 {
 	(void) tx;
 	zcp_run_info_t *ri = arg;
 
 	ri->zri_canceled = B_TRUE;
 }
 
 static void
 zcp_eval_sync(void *arg, dmu_tx_t *tx)
 {
 	zcp_run_info_t *ri = arg;
 
 	/*
 	 * Open context should have setup the stack to contain:
 	 * 1: Error handler callback
 	 * 2: Script to run (converted to a Lua function)
 	 * 3: nvlist input to function (converted to Lua table or nil)
 	 */
 	VERIFY3U(3, ==, lua_gettop(ri->zri_state));
 
 	zcp_eval_impl(tx, ri);
 }
 
 static void
 zcp_eval_open(zcp_run_info_t *ri, const char *poolname)
 {
 	int error;
 	dsl_pool_t *dp;
 	dmu_tx_t *tx;
 
 	/*
 	 * See comment from the same assertion in zcp_eval_sync().
 	 */
 	VERIFY3U(3, ==, lua_gettop(ri->zri_state));
 
 	error = dsl_pool_hold(poolname, FTAG, &dp);
 	if (error != 0) {
 		zcp_pool_error(ri, poolname, error);
 		return;
 	}
 
 	/*
 	 * As we are running in open-context, we have no transaction associated
 	 * with the channel program. At the same time, functions from the
 	 * zfs.check submodule need to be associated with a transaction as
 	 * they are basically dry-runs of their counterparts in the zfs.sync
 	 * submodule. These functions should be able to run in open-context.
 	 * Therefore we create a new transaction that we later abort once
 	 * the channel program has been evaluated.
 	 */
 	tx = dmu_tx_create_dd(dp->dp_mos_dir);
 
 	zcp_eval_impl(tx, ri);
 
 	dmu_tx_abort(tx);
 
 	dsl_pool_rele(dp, FTAG);
 }
 
 int
 zcp_eval(const char *poolname, const char *program, boolean_t sync,
     uint64_t instrlimit, uint64_t memlimit, nvpair_t *nvarg, nvlist_t *outnvl)
 {
 	int err;
 	lua_State *state;
 	zcp_run_info_t runinfo;
 
 	if (instrlimit > zfs_lua_max_instrlimit)
 		return (SET_ERROR(EINVAL));
 	if (memlimit == 0 || memlimit > zfs_lua_max_memlimit)
 		return (SET_ERROR(EINVAL));
 
 	zcp_alloc_arg_t allocargs = {
 		.aa_must_succeed = B_TRUE,
 		.aa_alloc_remaining = (int64_t)memlimit,
 		.aa_alloc_limit = (int64_t)memlimit,
 	};
 
 	/*
 	 * Creates a Lua state with a memory allocator that uses KM_SLEEP.
 	 * This should never fail.
 	 */
 	state = lua_newstate(zcp_lua_alloc, &allocargs);
 	VERIFY(state != NULL);
 	(void) lua_atpanic(state, zcp_panic_cb);
 
 	/*
 	 * Load core Lua libraries we want access to.
 	 */
 	VERIFY3U(1, ==, luaopen_base(state));
 	lua_pop(state, 1);
 	VERIFY3U(1, ==, luaopen_coroutine(state));
 	lua_setglobal(state, LUA_COLIBNAME);
 	VERIFY0(lua_gettop(state));
 	VERIFY3U(1, ==, luaopen_string(state));
 	lua_setglobal(state, LUA_STRLIBNAME);
 	VERIFY0(lua_gettop(state));
 	VERIFY3U(1, ==, luaopen_table(state));
 	lua_setglobal(state, LUA_TABLIBNAME);
 	VERIFY0(lua_gettop(state));
 
 	/*
 	 * Load globally visible variables such as errno aliases.
 	 */
 	zcp_load_globals(state);
 	VERIFY0(lua_gettop(state));
 
 	/*
 	 * Load ZFS-specific modules.
 	 */
 	lua_newtable(state);
 	VERIFY3U(1, ==, zcp_load_list_lib(state));
 	lua_setfield(state, -2, "list");
 	VERIFY3U(1, ==, zcp_load_synctask_lib(state, B_FALSE));
 	lua_setfield(state, -2, "check");
 	VERIFY3U(1, ==, zcp_load_synctask_lib(state, B_TRUE));
 	lua_setfield(state, -2, "sync");
 	VERIFY3U(1, ==, zcp_load_get_lib(state));
 	lua_pushcclosure(state, zcp_debug_info.func, 0);
 	lua_setfield(state, -2, zcp_debug_info.name);
 	lua_pushcclosure(state, zcp_exists_info.func, 0);
 	lua_setfield(state, -2, zcp_exists_info.name);
 	lua_setglobal(state, "zfs");
 	VERIFY0(lua_gettop(state));
 
 	/*
 	 * Push the error-callback that calculates Lua stack traces on
 	 * unexpected failures.
 	 */
 	lua_pushcfunction(state, zcp_error_handler);
 	VERIFY3U(1, ==, lua_gettop(state));
 
 	/*
 	 * Load the actual script as a function onto the stack as text ("t").
 	 * The only valid error condition is a syntax error in the script.
 	 * ERRMEM should not be possible because our allocator is using
 	 * KM_SLEEP.  ERRGCMM should not be possible because we have not added
 	 * any objects with __gc metamethods to the interpreter that could
 	 * fail.
 	 */
 	err = luaL_loadbufferx(state, program, strlen(program),
 	    "channel program", "t");
 	if (err == LUA_ERRSYNTAX) {
 		fnvlist_add_string(outnvl, ZCP_RET_ERROR,
 		    lua_tostring(state, -1));
 		lua_close(state);
 		return (SET_ERROR(EINVAL));
 	}
 	VERIFY0(err);
 	VERIFY3U(2, ==, lua_gettop(state));
 
 	/*
 	 * Convert the input nvlist to a Lua object and put it on top of the
 	 * stack.
 	 */
 	char errmsg[128];
 	err = zcp_nvpair_value_to_lua(state, nvarg,
 	    errmsg, sizeof (errmsg));
 	if (err != 0) {
 		fnvlist_add_string(outnvl, ZCP_RET_ERROR, errmsg);
 		lua_close(state);
 		return (SET_ERROR(EINVAL));
 	}
 	VERIFY3U(3, ==, lua_gettop(state));
 
+	cred_t *cr = CRED();
+	crhold(cr);
+
 	runinfo.zri_state = state;
 	runinfo.zri_allocargs = &allocargs;
 	runinfo.zri_outnvl = outnvl;
 	runinfo.zri_result = 0;
-	runinfo.zri_cred = CRED();
-	runinfo.zri_proc = curproc;
+	runinfo.zri_cred = cr;
 	runinfo.zri_timed_out = B_FALSE;
 	runinfo.zri_canceled = B_FALSE;
 	runinfo.zri_sync = sync;
 	runinfo.zri_space_used = 0;
 	runinfo.zri_curinstrs = 0;
 	runinfo.zri_maxinstrs = instrlimit;
 	runinfo.zri_new_zvols = fnvlist_alloc();
 
 	if (sync) {
 		err = dsl_sync_task_sig(poolname, NULL, zcp_eval_sync,
 		    zcp_eval_sig, &runinfo, 0, ZFS_SPACE_CHECK_ZCP_EVAL);
 		if (err != 0)
 			zcp_pool_error(&runinfo, poolname, err);
 	} else {
 		zcp_eval_open(&runinfo, poolname);
 	}
 	lua_close(state);
 
+	crfree(cr);
+
 	/*
 	 * Create device minor nodes for any new zvols.
 	 */
 	for (nvpair_t *pair = nvlist_next_nvpair(runinfo.zri_new_zvols, NULL);
 	    pair != NULL;
 	    pair = nvlist_next_nvpair(runinfo.zri_new_zvols, pair)) {
 		zvol_create_minor(nvpair_name(pair));
 	}
 	fnvlist_free(runinfo.zri_new_zvols);
 
 	return (runinfo.zri_result);
 }
 
 /*
  * Retrieve metadata about the currently running channel program.
  */
 zcp_run_info_t *
 zcp_run_info(lua_State *state)
 {
 	zcp_run_info_t *ri;
 
 	lua_getfield(state, LUA_REGISTRYINDEX, ZCP_RUN_INFO_KEY);
 	ri = lua_touserdata(state, -1);
 	lua_pop(state, 1);
 	return (ri);
 }
 
 /*
  * Argument Parsing
  * ================
  *
  * The Lua language allows methods to be called with any number
  * of arguments of any type. When calling back into ZFS we need to sanitize
  * arguments from channel programs to make sure unexpected arguments or
  * arguments of the wrong type result in clear error messages. To do this
  * in a uniform way all callbacks from channel programs should use the
  * zcp_parse_args() function to interpret inputs.
  *
  * Positional vs Keyword Arguments
  * ===============================
  *
  * Every callback function takes a fixed set of required positional arguments
  * and optional keyword arguments. For example, the destroy function takes
  * a single positional string argument (the name of the dataset to destroy)
  * and an optional "defer" keyword boolean argument. When calling lua functions
  * with parentheses, only positional arguments can be used:
  *
  *     zfs.sync.snapshot("rpool@snap")
  *
  * To use keyword arguments functions should be called with a single argument
  * that is a lua table containing mappings of integer -> positional arguments
  * and string -> keyword arguments:
  *
  *     zfs.sync.snapshot({1="rpool@snap", defer=true})
  *
  * The lua language allows curly braces to be used in place of parenthesis as
  * syntactic sugar for this calling convention:
  *
  *     zfs.sync.snapshot{"rpool@snap", defer=true}
  */
 
 /*
  * Throw an error and print the given arguments.  If there are too many
  * arguments to fit in the output buffer, only the error format string is
  * output.
  */
 static void
 zcp_args_error(lua_State *state, const char *fname, const zcp_arg_t *pargs,
     const zcp_arg_t *kwargs, const char *fmt, ...)
 {
 	int i;
 	char errmsg[512];
 	size_t len = sizeof (errmsg);
 	size_t msglen = 0;
 	va_list argp;
 
 	va_start(argp, fmt);
 	VERIFY3U(len, >, vsnprintf(errmsg, len, fmt, argp));
 	va_end(argp);
 
 	/*
 	 * Calculate the total length of the final string, including extra
 	 * formatting characters. If the argument dump would be too large,
 	 * only print the error string.
 	 */
 	msglen = strlen(errmsg);
 	msglen += strlen(fname) + 4; /* : + {} + null terminator */
 	for (i = 0; pargs[i].za_name != NULL; i++) {
 		msglen += strlen(pargs[i].za_name);
 		msglen += strlen(lua_typename(state, pargs[i].za_lua_type));
 		if (pargs[i + 1].za_name != NULL || kwargs[0].za_name != NULL)
 			msglen += 5; /* < + ( + )> + , */
 		else
 			msglen += 4; /* < + ( + )> */
 	}
 	for (i = 0; kwargs[i].za_name != NULL; i++) {
 		msglen += strlen(kwargs[i].za_name);
 		msglen += strlen(lua_typename(state, kwargs[i].za_lua_type));
 		if (kwargs[i + 1].za_name != NULL)
 			msglen += 4; /* =( + ) + , */
 		else
 			msglen += 3; /* =( + ) */
 	}
 
 	if (msglen >= len)
 		(void) luaL_error(state, errmsg);
 
 	VERIFY3U(len, >, strlcat(errmsg, ": ", len));
 	VERIFY3U(len, >, strlcat(errmsg, fname, len));
 	VERIFY3U(len, >, strlcat(errmsg, "{", len));
 	for (i = 0; pargs[i].za_name != NULL; i++) {
 		VERIFY3U(len, >, strlcat(errmsg, "<", len));
 		VERIFY3U(len, >, strlcat(errmsg, pargs[i].za_name, len));
 		VERIFY3U(len, >, strlcat(errmsg, "(", len));
 		VERIFY3U(len, >, strlcat(errmsg,
 		    lua_typename(state, pargs[i].za_lua_type), len));
 		VERIFY3U(len, >, strlcat(errmsg, ")>", len));
 		if (pargs[i + 1].za_name != NULL || kwargs[0].za_name != NULL) {
 			VERIFY3U(len, >, strlcat(errmsg, ", ", len));
 		}
 	}
 	for (i = 0; kwargs[i].za_name != NULL; i++) {
 		VERIFY3U(len, >, strlcat(errmsg, kwargs[i].za_name, len));
 		VERIFY3U(len, >, strlcat(errmsg, "=(", len));
 		VERIFY3U(len, >, strlcat(errmsg,
 		    lua_typename(state, kwargs[i].za_lua_type), len));
 		VERIFY3U(len, >, strlcat(errmsg, ")", len));
 		if (kwargs[i + 1].za_name != NULL) {
 			VERIFY3U(len, >, strlcat(errmsg, ", ", len));
 		}
 	}
 	VERIFY3U(len, >, strlcat(errmsg, "}", len));
 
 	(void) luaL_error(state, errmsg);
 	panic("unreachable code");
 }
 
 static void
 zcp_parse_table_args(lua_State *state, const char *fname,
     const zcp_arg_t *pargs, const zcp_arg_t *kwargs)
 {
 	int i;
 	int type;
 
 	for (i = 0; pargs[i].za_name != NULL; i++) {
 		/*
 		 * Check the table for this positional argument, leaving it
 		 * on the top of the stack once we finish validating it.
 		 */
 		lua_pushinteger(state, i + 1);
 		lua_gettable(state, 1);
 
 		type = lua_type(state, -1);
 		if (type == LUA_TNIL) {
 			zcp_args_error(state, fname, pargs, kwargs,
 			    "too few arguments");
 			panic("unreachable code");
 		} else if (type != pargs[i].za_lua_type) {
 			zcp_args_error(state, fname, pargs, kwargs,
 			    "arg %d wrong type (is '%s', expected '%s')",
 			    i + 1, lua_typename(state, type),
 			    lua_typename(state, pargs[i].za_lua_type));
 			panic("unreachable code");
 		}
 
 		/*
 		 * Remove the positional argument from the table.
 		 */
 		lua_pushinteger(state, i + 1);
 		lua_pushnil(state);
 		lua_settable(state, 1);
 	}
 
 	for (i = 0; kwargs[i].za_name != NULL; i++) {
 		/*
 		 * Check the table for this keyword argument, which may be
 		 * nil if it was omitted. Leave the value on the top of
 		 * the stack after validating it.
 		 */
 		lua_getfield(state, 1, kwargs[i].za_name);
 
 		type = lua_type(state, -1);
 		if (type != LUA_TNIL && type != kwargs[i].za_lua_type) {
 			zcp_args_error(state, fname, pargs, kwargs,
 			    "kwarg '%s' wrong type (is '%s', expected '%s')",
 			    kwargs[i].za_name, lua_typename(state, type),
 			    lua_typename(state, kwargs[i].za_lua_type));
 			panic("unreachable code");
 		}
 
 		/*
 		 * Remove the keyword argument from the table.
 		 */
 		lua_pushnil(state);
 		lua_setfield(state, 1, kwargs[i].za_name);
 	}
 
 	/*
 	 * Any entries remaining in the table are invalid inputs, print
 	 * an error message based on what the entry is.
 	 */
 	lua_pushnil(state);
 	if (lua_next(state, 1)) {
 		if (lua_isnumber(state, -2) && lua_tointeger(state, -2) > 0) {
 			zcp_args_error(state, fname, pargs, kwargs,
 			    "too many positional arguments");
 		} else if (lua_isstring(state, -2)) {
 			zcp_args_error(state, fname, pargs, kwargs,
 			    "invalid kwarg '%s'", lua_tostring(state, -2));
 		} else {
 			zcp_args_error(state, fname, pargs, kwargs,
 			    "kwarg keys must be strings");
 		}
 		panic("unreachable code");
 	}
 
 	lua_remove(state, 1);
 }
 
 static void
 zcp_parse_pos_args(lua_State *state, const char *fname, const zcp_arg_t *pargs,
     const zcp_arg_t *kwargs)
 {
 	int i;
 	int type;
 
 	for (i = 0; pargs[i].za_name != NULL; i++) {
 		type = lua_type(state, i + 1);
 		if (type == LUA_TNONE) {
 			zcp_args_error(state, fname, pargs, kwargs,
 			    "too few arguments");
 			panic("unreachable code");
 		} else if (type != pargs[i].za_lua_type) {
 			zcp_args_error(state, fname, pargs, kwargs,
 			    "arg %d wrong type (is '%s', expected '%s')",
 			    i + 1, lua_typename(state, type),
 			    lua_typename(state, pargs[i].za_lua_type));
 			panic("unreachable code");
 		}
 	}
 	if (lua_gettop(state) != i) {
 		zcp_args_error(state, fname, pargs, kwargs,
 		    "too many positional arguments");
 		panic("unreachable code");
 	}
 
 	for (i = 0; kwargs[i].za_name != NULL; i++) {
 		lua_pushnil(state);
 	}
 }
 
 /*
  * Checks the current Lua stack against an expected set of positional and
  * keyword arguments. If the stack does not match the expected arguments
  * aborts the current channel program with a useful error message, otherwise
  * it re-arranges the stack so that it contains the positional arguments
  * followed by the keyword argument values in declaration order. Any missing
  * keyword argument will be represented by a nil value on the stack.
  *
  * If the stack contains exactly one argument of type LUA_TTABLE the curly
  * braces calling convention is assumed, otherwise the stack is parsed for
  * positional arguments only.
  *
  * This function should be used by every function callback. It should be called
  * before the callback manipulates the Lua stack as it assumes the stack
  * represents the function arguments.
  */
 void
 zcp_parse_args(lua_State *state, const char *fname, const zcp_arg_t *pargs,
     const zcp_arg_t *kwargs)
 {
 	if (lua_gettop(state) == 1 && lua_istable(state, 1)) {
 		zcp_parse_table_args(state, fname, pargs, kwargs);
 	} else {
 		zcp_parse_pos_args(state, fname, pargs, kwargs);
 	}
 }
 
 ZFS_MODULE_PARAM(zfs_lua, zfs_lua_, max_instrlimit, U64, ZMOD_RW,
 	"Max instruction limit that can be specified for a channel program");
 
 ZFS_MODULE_PARAM(zfs_lua, zfs_lua_, max_memlimit, U64, ZMOD_RW,
 	"Max memory limit that can be specified for a channel program");
diff --git a/sys/contrib/openzfs/module/zfs/zcp_synctask.c b/sys/contrib/openzfs/module/zfs/zcp_synctask.c
index 058910054d97..af94569e0865 100644
--- a/sys/contrib/openzfs/module/zfs/zcp_synctask.c
+++ b/sys/contrib/openzfs/module/zfs/zcp_synctask.c
@@ -1,586 +1,584 @@
 /*
  * CDDL HEADER START
  *
  * This file and its contents are supplied under the terms of the
  * Common Development and Distribution License ("CDDL"), version 1.0.
  * You may only use this file in accordance with the terms of version
  * 1.0 of the CDDL.
  *
  * A full copy of the text of the CDDL should have accompanied this
  * source.  A copy of the CDDL is also available via the Internet at
  * http://www.illumos.org/license/CDDL.
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright (c) 2016, 2017 by Delphix. All rights reserved.
  * Copyright (c) 2019, 2020 by Christian Schwarz. All rights reserved.
  * Copyright 2020 Joyent, Inc.
  */
 
 #include <sys/lua/lua.h>
 #include <sys/lua/lauxlib.h>
 
 #include <sys/zcp.h>
 #include <sys/zcp_set.h>
 #include <sys/dsl_dir.h>
 #include <sys/dsl_pool.h>
 #include <sys/dsl_prop.h>
 #include <sys/dsl_synctask.h>
 #include <sys/dsl_dataset.h>
 #include <sys/dsl_bookmark.h>
 #include <sys/dsl_destroy.h>
 #include <sys/dmu_objset.h>
 #include <sys/zfs_znode.h>
 #include <sys/zfeature.h>
 #include <sys/metaslab.h>
 
 #define	DST_AVG_BLKSHIFT 14
 
 typedef struct zcp_inherit_prop_arg {
 	lua_State		*zipa_state;
 	const char		*zipa_prop;
 	dsl_props_set_arg_t	zipa_dpsa;
 } zcp_inherit_prop_arg_t;
 
 typedef int (zcp_synctask_func_t)(lua_State *, boolean_t, nvlist_t *);
 typedef struct zcp_synctask_info {
 	const char *name;
 	zcp_synctask_func_t *func;
 	const zcp_arg_t pargs[4];
 	const zcp_arg_t kwargs[2];
 	zfs_space_check_t space_check;
 	int blocks_modified;
 } zcp_synctask_info_t;
 
 static void
 zcp_synctask_cleanup(void *arg)
 {
 	fnvlist_free(arg);
 }
 
 /*
  * Generic synctask interface for channel program syncfuncs.
  *
  * To perform some action in syncing context, we'd generally call
  * dsl_sync_task(), but since the Lua script is already running inside a
  * synctask we need to leave out some actions (such as acquiring the config
  * rwlock and performing space checks).
  *
  * If 'sync' is false, executes a dry run and returns the error code.
  *
  * If we are not running in syncing context and we are not doing a dry run
  * (meaning we are running a zfs.sync function in open-context) then we
  * return a Lua error.
  *
  * This function also handles common fatal error cases for channel program
  * library functions. If a fatal error occurs, err_dsname will be the dataset
  * name reported in error messages, if supplied.
  */
 static int
 zcp_sync_task(lua_State *state, dsl_checkfunc_t *checkfunc,
     dsl_syncfunc_t *syncfunc, void *arg, boolean_t sync, const char *err_dsname)
 {
 	int err;
 	zcp_run_info_t *ri = zcp_run_info(state);
 
 	err = checkfunc(arg, ri->zri_tx);
 	if (!sync)
 		return (err);
 
 	if (!ri->zri_sync) {
 		return (luaL_error(state, "running functions from the zfs.sync "
 		    "submodule requires passing sync=TRUE to "
 		    "lzc_channel_program() (i.e. do not specify the \"-n\" "
 		    "command line argument)"));
 	}
 
 	if (err == 0) {
 		syncfunc(arg, ri->zri_tx);
 	} else if (err == EIO) {
 		if (err_dsname != NULL) {
 			return (luaL_error(state,
 			    "I/O error while accessing dataset '%s'",
 			    err_dsname));
 		} else {
 			return (luaL_error(state,
 			    "I/O error while accessing dataset."));
 		}
 	}
 
 	return (err);
 }
 
 
 static int zcp_synctask_destroy(lua_State *, boolean_t, nvlist_t *);
 static const zcp_synctask_info_t zcp_synctask_destroy_info = {
 	.name = "destroy",
 	.func = zcp_synctask_destroy,
 	.pargs = {
 	    {.za_name = "filesystem | snapshot", .za_lua_type = LUA_TSTRING },
 	    {NULL, 0}
 	},
 	.kwargs = {
 	    {.za_name = "defer", .za_lua_type = LUA_TBOOLEAN },
 	    {NULL, 0}
 	},
 	.space_check = ZFS_SPACE_CHECK_DESTROY,
 	.blocks_modified = 0
 };
 
 static int
 zcp_synctask_destroy(lua_State *state, boolean_t sync, nvlist_t *err_details)
 {
 	(void) err_details;
 	int err;
 	const char *dsname = lua_tostring(state, 1);
 
 	boolean_t issnap = (strchr(dsname, '@') != NULL);
 
 	if (!issnap && !lua_isnil(state, 2)) {
 		return (luaL_error(state,
 		    "'deferred' kwarg only supported for snapshots: %s",
 		    dsname));
 	}
 
 	if (issnap) {
 		dsl_destroy_snapshot_arg_t ddsa = { 0 };
 		ddsa.ddsa_name = dsname;
 		if (!lua_isnil(state, 2)) {
 			ddsa.ddsa_defer = lua_toboolean(state, 2);
 		} else {
 			ddsa.ddsa_defer = B_FALSE;
 		}
 
 		err = zcp_sync_task(state, dsl_destroy_snapshot_check,
 		    dsl_destroy_snapshot_sync, &ddsa, sync, dsname);
 	} else {
 		dsl_destroy_head_arg_t ddha = { 0 };
 		ddha.ddha_name = dsname;
 
 		err = zcp_sync_task(state, dsl_destroy_head_check,
 		    dsl_destroy_head_sync, &ddha, sync, dsname);
 	}
 
 	return (err);
 }
 
 static int zcp_synctask_promote(lua_State *, boolean_t, nvlist_t *);
 static const zcp_synctask_info_t zcp_synctask_promote_info = {
 	.name = "promote",
 	.func = zcp_synctask_promote,
 	.pargs = {
 	    {.za_name = "clone", .za_lua_type = LUA_TSTRING },
 	    {NULL, 0}
 	},
 	.kwargs = {
 	    {NULL, 0}
 	},
 	.space_check = ZFS_SPACE_CHECK_RESERVED,
 	.blocks_modified = 3
 };
 
 static int
 zcp_synctask_promote(lua_State *state, boolean_t sync, nvlist_t *err_details)
 {
 	int err;
 	dsl_dataset_promote_arg_t ddpa = { 0 };
 	const char *dsname = lua_tostring(state, 1);
 	zcp_run_info_t *ri = zcp_run_info(state);
 
 	ddpa.ddpa_clonename = dsname;
 	ddpa.err_ds = err_details;
 	ddpa.cr = ri->zri_cred;
-	ddpa.proc = ri->zri_proc;
 
 	/*
 	 * If there was a snapshot name conflict, then err_ds will be filled
 	 * with a list of conflicting snapshot names.
 	 */
 	err = zcp_sync_task(state, dsl_dataset_promote_check,
 	    dsl_dataset_promote_sync, &ddpa, sync, dsname);
 
 	return (err);
 }
 
 static int zcp_synctask_rollback(lua_State *, boolean_t, nvlist_t *err_details);
 static const zcp_synctask_info_t zcp_synctask_rollback_info = {
 	.name = "rollback",
 	.func = zcp_synctask_rollback,
 	.space_check = ZFS_SPACE_CHECK_RESERVED,
 	.blocks_modified = 1,
 	.pargs = {
 	    {.za_name = "filesystem", .za_lua_type = LUA_TSTRING },
 	    {0, 0}
 	},
 	.kwargs = {
 	    {0, 0}
 	}
 };
 
 static int
 zcp_synctask_rollback(lua_State *state, boolean_t sync, nvlist_t *err_details)
 {
 	int err;
 	const char *dsname = lua_tostring(state, 1);
 	dsl_dataset_rollback_arg_t ddra = { 0 };
 
 	ddra.ddra_fsname = dsname;
 	ddra.ddra_result = err_details;
 
 	err = zcp_sync_task(state, dsl_dataset_rollback_check,
 	    dsl_dataset_rollback_sync, &ddra, sync, dsname);
 
 	return (err);
 }
 
 static int zcp_synctask_snapshot(lua_State *, boolean_t, nvlist_t *);
 static const zcp_synctask_info_t zcp_synctask_snapshot_info = {
 	.name = "snapshot",
 	.func = zcp_synctask_snapshot,
 	.pargs = {
 	    {.za_name = "filesystem@snapname | volume@snapname",
 	    .za_lua_type = LUA_TSTRING },
 	    {NULL, 0}
 	},
 	.kwargs = {
 	    {NULL, 0}
 	},
 	.space_check = ZFS_SPACE_CHECK_NORMAL,
 	.blocks_modified = 3
 };
 
 static int
 zcp_synctask_snapshot(lua_State *state, boolean_t sync, nvlist_t *err_details)
 {
 	(void) err_details;
 	int err;
 	dsl_dataset_snapshot_arg_t ddsa = { 0 };
 	const char *dsname = lua_tostring(state, 1);
 	zcp_run_info_t *ri = zcp_run_info(state);
 
 	/*
 	 * On old pools, the ZIL must not be active when a snapshot is created,
 	 * but we can't suspend the ZIL because we're already in syncing
 	 * context.
 	 */
 	if (spa_version(ri->zri_pool->dp_spa) < SPA_VERSION_FAST_SNAP) {
 		return (SET_ERROR(ENOTSUP));
 	}
 
 	/*
 	 * We only allow for a single snapshot rather than a list, so the
 	 * error list output is unnecessary.
 	 */
 	ddsa.ddsa_errors = NULL;
 	ddsa.ddsa_props = NULL;
 	ddsa.ddsa_cr = ri->zri_cred;
-	ddsa.ddsa_proc = ri->zri_proc;
 	ddsa.ddsa_snaps = fnvlist_alloc();
 	fnvlist_add_boolean(ddsa.ddsa_snaps, dsname);
 
 	zcp_cleanup_handler_t *zch = zcp_register_cleanup(state,
 	    zcp_synctask_cleanup, ddsa.ddsa_snaps);
 
 	err = zcp_sync_task(state, dsl_dataset_snapshot_check,
 	    dsl_dataset_snapshot_sync, &ddsa, sync, dsname);
 
 	if (err == 0) {
 		/*
 		 * We may need to create a new device minor node for this
 		 * dataset (if it is a zvol and the "snapdev" property is set).
 		 * Save it in the nvlist so that it can be processed in open
 		 * context.
 		 */
 		fnvlist_add_boolean(ri->zri_new_zvols, dsname);
 	}
 
 	zcp_deregister_cleanup(state, zch);
 	fnvlist_free(ddsa.ddsa_snaps);
 
 	return (err);
 }
 
 static int zcp_synctask_rename_snapshot(lua_State *, boolean_t, nvlist_t *);
 static const zcp_synctask_info_t zcp_synctask_rename_snapshot_info = {
 	.name = "rename_snapshot",
 	.func = zcp_synctask_rename_snapshot,
 	.pargs = {
 	    {.za_name = "filesystem | volume", .za_lua_type = LUA_TSTRING },
 	    {.za_name = "oldsnapname", .za_lua_type = LUA_TSTRING },
 	    {.za_name = "newsnapname", .za_lua_type = LUA_TSTRING },
 	    {NULL, 0}
 	},
 	.space_check = ZFS_SPACE_CHECK_RESERVED,
 	.blocks_modified = 1
 };
 
 static int
 zcp_synctask_rename_snapshot(lua_State *state, boolean_t sync,
     nvlist_t *err_details)
 {
 	(void) err_details;
 	int err;
 	const char *fsname = lua_tostring(state, 1);
 	const char *oldsnapname = lua_tostring(state, 2);
 	const char *newsnapname = lua_tostring(state, 3);
 
 	struct dsl_dataset_rename_snapshot_arg ddrsa = { 0 };
 	ddrsa.ddrsa_fsname = fsname;
 	ddrsa.ddrsa_oldsnapname = oldsnapname;
 	ddrsa.ddrsa_newsnapname = newsnapname;
 	ddrsa.ddrsa_recursive = B_FALSE;
 
 	err = zcp_sync_task(state, dsl_dataset_rename_snapshot_check,
 	    dsl_dataset_rename_snapshot_sync, &ddrsa, sync, NULL);
 
 	return (err);
 }
 
 static int zcp_synctask_inherit_prop(lua_State *, boolean_t,
     nvlist_t *err_details);
 static const zcp_synctask_info_t zcp_synctask_inherit_prop_info = {
 	.name = "inherit",
 	.func = zcp_synctask_inherit_prop,
 	.space_check = ZFS_SPACE_CHECK_RESERVED,
 	.blocks_modified = 2, /* 2 * numprops */
 	.pargs = {
 		{ .za_name = "dataset", .za_lua_type = LUA_TSTRING },
 		{ .za_name = "property", .za_lua_type = LUA_TSTRING },
 		{ NULL, 0 }
 	},
 	.kwargs = {
 		{ NULL, 0 }
 	},
 };
 
 static int
 zcp_synctask_inherit_prop_check(void *arg, dmu_tx_t *tx)
 {
 	zcp_inherit_prop_arg_t *args = arg;
 	zfs_prop_t prop = zfs_name_to_prop(args->zipa_prop);
 
 	if (prop == ZPROP_USERPROP) {
 		if (zfs_prop_user(args->zipa_prop))
 			return (0);
 
 		return (EINVAL);
 	}
 
 	if (zfs_prop_readonly(prop))
 		return (EINVAL);
 
 	if (!zfs_prop_inheritable(prop))
 		return (EINVAL);
 
 	return (dsl_props_set_check(&args->zipa_dpsa, tx));
 }
 
 static void
 zcp_synctask_inherit_prop_sync(void *arg, dmu_tx_t *tx)
 {
 	zcp_inherit_prop_arg_t *args = arg;
 	dsl_props_set_arg_t *dpsa = &args->zipa_dpsa;
 
 	dsl_props_set_sync(dpsa, tx);
 }
 
 static int
 zcp_synctask_inherit_prop(lua_State *state, boolean_t sync,
     nvlist_t *err_details)
 {
 	(void) err_details;
 	int err;
 	zcp_inherit_prop_arg_t zipa = { 0 };
 	dsl_props_set_arg_t *dpsa = &zipa.zipa_dpsa;
 
 	const char *dsname = lua_tostring(state, 1);
 	const char *prop = lua_tostring(state, 2);
 
 	zipa.zipa_state = state;
 	zipa.zipa_prop = prop;
 	dpsa->dpsa_dsname = dsname;
 	dpsa->dpsa_source = ZPROP_SRC_INHERITED;
 	dpsa->dpsa_props = fnvlist_alloc();
 	fnvlist_add_boolean(dpsa->dpsa_props, prop);
 
 	zcp_cleanup_handler_t *zch = zcp_register_cleanup(state,
 	    zcp_synctask_cleanup, dpsa->dpsa_props);
 
 	err = zcp_sync_task(state, zcp_synctask_inherit_prop_check,
 	    zcp_synctask_inherit_prop_sync, &zipa, sync, dsname);
 
 	zcp_deregister_cleanup(state, zch);
 	fnvlist_free(dpsa->dpsa_props);
 
 	return (err);
 }
 
 static int zcp_synctask_bookmark(lua_State *, boolean_t, nvlist_t *);
 static const zcp_synctask_info_t zcp_synctask_bookmark_info = {
 	.name = "bookmark",
 	.func = zcp_synctask_bookmark,
 	.pargs = {
 	    {.za_name = "snapshot | bookmark", .za_lua_type = LUA_TSTRING },
 	    {.za_name = "bookmark", .za_lua_type = LUA_TSTRING },
 	    {NULL, 0}
 	},
 	.kwargs = {
 	    {NULL, 0}
 	},
 	.space_check = ZFS_SPACE_CHECK_NORMAL,
 	.blocks_modified = 1,
 };
 
 static int
 zcp_synctask_bookmark(lua_State *state, boolean_t sync, nvlist_t *err_details)
 {
 	(void) err_details;
 	int err;
 	const char *source = lua_tostring(state, 1);
 	const char *new = lua_tostring(state, 2);
 
 	nvlist_t *bmarks = fnvlist_alloc();
 	fnvlist_add_string(bmarks, new, source);
 
 	zcp_cleanup_handler_t *zch = zcp_register_cleanup(state,
 	    zcp_synctask_cleanup, bmarks);
 
 	dsl_bookmark_create_arg_t dbca = {
 		.dbca_bmarks = bmarks,
 		.dbca_errors = NULL,
 	};
 	err = zcp_sync_task(state, dsl_bookmark_create_check,
 	    dsl_bookmark_create_sync, &dbca, sync, source);
 
 	zcp_deregister_cleanup(state, zch);
 	fnvlist_free(bmarks);
 
 	return (err);
 }
 
 static int zcp_synctask_set_prop(lua_State *, boolean_t, nvlist_t *err_details);
 static const zcp_synctask_info_t zcp_synctask_set_prop_info = {
 	.name = "set_prop",
 	.func = zcp_synctask_set_prop,
 	.space_check = ZFS_SPACE_CHECK_RESERVED,
 	.blocks_modified = 2,
 	.pargs = {
 		{ .za_name = "dataset", .za_lua_type = LUA_TSTRING },
 		{ .za_name = "property", .za_lua_type =  LUA_TSTRING },
 		{ .za_name = "value", .za_lua_type =  LUA_TSTRING },
 		{ NULL, 0 }
 	},
 	.kwargs = {
 		{ NULL, 0 }
 	}
 };
 
 static int
 zcp_synctask_set_prop(lua_State *state, boolean_t sync, nvlist_t *err_details)
 {
 	(void) err_details;
 	int err;
 	zcp_set_prop_arg_t args = { 0 };
 
 	const char *dsname = lua_tostring(state, 1);
 	const char *prop = lua_tostring(state, 2);
 	const char *val = lua_tostring(state, 3);
 
 	args.state = state;
 	args.dsname = dsname;
 	args.prop = prop;
 	args.val = val;
 
 	err = zcp_sync_task(state, zcp_set_prop_check, zcp_set_prop_sync,
 	    &args, sync, dsname);
 
 	return (err);
 }
 
 static int
 zcp_synctask_wrapper(lua_State *state)
 {
 	int err;
 	zcp_cleanup_handler_t *zch;
 	int num_ret = 1;
 	nvlist_t *err_details = fnvlist_alloc();
 
 	/*
 	 * Make sure err_details is properly freed, even if a fatal error is
 	 * thrown during the synctask.
 	 */
 	zch = zcp_register_cleanup(state, zcp_synctask_cleanup, err_details);
 
 	zcp_synctask_info_t *info = lua_touserdata(state, lua_upvalueindex(1));
 	boolean_t sync = lua_toboolean(state, lua_upvalueindex(2));
 
 	zcp_run_info_t *ri = zcp_run_info(state);
 	dsl_pool_t *dp = ri->zri_pool;
 
 	/* MOS space is triple-dittoed, so we multiply by 3. */
 	uint64_t funcspace =
 	    ((uint64_t)info->blocks_modified << DST_AVG_BLKSHIFT) * 3;
 
 	zcp_parse_args(state, info->name, info->pargs, info->kwargs);
 
 	err = 0;
 	if (info->space_check != ZFS_SPACE_CHECK_NONE) {
 		uint64_t quota = dsl_pool_unreserved_space(dp,
 		    info->space_check);
 		uint64_t used = dsl_dir_phys(dp->dp_root_dir)->dd_used_bytes +
 		    ri->zri_space_used;
 
 		if (used + funcspace > quota) {
 			err = SET_ERROR(ENOSPC);
 		}
 	}
 
 	if (err == 0) {
 		err = info->func(state, sync, err_details);
 	}
 
 	if (err == 0) {
 		ri->zri_space_used += funcspace;
 	}
 
 	lua_pushnumber(state, (lua_Number)err);
 	if (fnvlist_num_pairs(err_details) > 0) {
 		(void) zcp_nvlist_to_lua(state, err_details, NULL, 0);
 		num_ret++;
 	}
 
 	zcp_deregister_cleanup(state, zch);
 	fnvlist_free(err_details);
 
 	return (num_ret);
 }
 
 int
 zcp_load_synctask_lib(lua_State *state, boolean_t sync)
 {
 	const zcp_synctask_info_t *zcp_synctask_funcs[] = {
 		&zcp_synctask_destroy_info,
 		&zcp_synctask_promote_info,
 		&zcp_synctask_rollback_info,
 		&zcp_synctask_snapshot_info,
 		&zcp_synctask_rename_snapshot_info,
 		&zcp_synctask_inherit_prop_info,
 		&zcp_synctask_bookmark_info,
 		&zcp_synctask_set_prop_info,
 		NULL
 	};
 
 	lua_newtable(state);
 
 	for (int i = 0; zcp_synctask_funcs[i] != NULL; i++) {
 		const zcp_synctask_info_t *info = zcp_synctask_funcs[i];
 		lua_pushlightuserdata(state, (void *)(uintptr_t)info);
 		lua_pushboolean(state, sync);
 		lua_pushcclosure(state, &zcp_synctask_wrapper, 2);
 		lua_setfield(state, -2, info->name);
 	}
 
 	return (1);
 }
diff --git a/sys/contrib/openzfs/rpm/generic/zfs-dkms.spec.in b/sys/contrib/openzfs/rpm/generic/zfs-dkms.spec.in
index cd85dd28cf56..fe127025e860 100644
--- a/sys/contrib/openzfs/rpm/generic/zfs-dkms.spec.in
+++ b/sys/contrib/openzfs/rpm/generic/zfs-dkms.spec.in
@@ -1,161 +1,168 @@
 %{?!packager: %define packager Brian Behlendorf <behlendorf1@llnl.gov>}
 
 %if ! 0%{?rhel}%{?fedora}%{?mageia}%{?suse_version}%{?openEuler}
 %define not_rpm 1
 %endif
 
 # Exclude input files from mangling
 %global __brp_mangle_shebangs_exclude_from ^/usr/src/.*$
 
 %define module  @PACKAGE@
 %define mkconf  scripts/dkms.mkconf
 
 Name:           %{module}-dkms
 
 Version:        @VERSION@
 Release:        @RELEASE@%{?dist}
 Summary:        Kernel module(s) (dkms)
 
 Group:          System Environment/Kernel
 License:        @ZFS_META_LICENSE@
 URL:            https://github.com/openzfs/zfs
 Source0:        %{module}-%{version}.tar.gz
 BuildRoot:      %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 BuildArch:      noarch
 
 Requires:       dkms >= 2.2.0.3
 Requires(pre):  dkms >= 2.2.0.3
 Requires(post): dkms >= 2.2.0.3
 Requires(preun): dkms >= 2.2.0.3
 Requires:       gcc, make, perl, diffutils
 Requires(post): gcc, make, perl, diffutils
+
+# Hold back kernel upgrades if kernel is not supported by ZFS
 %if 0%{?rhel}%{?fedora}%{?mageia}%{?suse_version}%{?openEuler}
 Requires:       kernel-devel >= @ZFS_META_KVER_MIN@, kernel-devel <= @ZFS_META_KVER_MAX@.999
 Requires(post): kernel-devel >= @ZFS_META_KVER_MIN@, kernel-devel <= @ZFS_META_KVER_MAX@.999
+Conflicts:      kernel-devel < @ZFS_META_KVER_MIN@, kernel-devel > @ZFS_META_KVER_MAX@.999
+Requires:       kernel-uname-r >= @ZFS_META_KVER_MIN@, kernel-uname-r <= @ZFS_META_KVER_MAX@.999
+Requires(post): kernel-uname-r >= @ZFS_META_KVER_MIN@, kernel-uname-r <= @ZFS_META_KVER_MAX@.999
+Conflicts:      kernel-uname-r < @ZFS_META_KVER_MIN@, kernel-uname-r > @ZFS_META_KVER_MAX@.999
+
 Obsoletes:      spl-dkms <= %{version}
 %endif
 Provides:       %{module}-kmod = %{version}
 AutoReqProv:    no
 
 %if (0%{?fedora}%{?suse_version}%{?openEuler}) || (0%{?rhel} && 0%{?rhel} < 9)
 # We don't directly use it, but if this isn't installed, rpmbuild as root can
 # crash+corrupt rpmdb
 # See issue #12071
 BuildRequires:  ncompress
 %endif
 
 %description
 This package contains the dkms ZFS kernel modules.
 
 %prep
 %setup -q -n %{module}-%{version}
 
 %build
 %{mkconf} -n %{module} -v %{version} -f dkms.conf
 
 %install
 if [ "$RPM_BUILD_ROOT" != "/" ]; then
     rm -rf $RPM_BUILD_ROOT
 fi
 mkdir -p $RPM_BUILD_ROOT/usr/src/
 cp -rf ${RPM_BUILD_DIR}/%{module}-%{version} $RPM_BUILD_ROOT/usr/src/
 
 %clean
 if [ "$RPM_BUILD_ROOT" != "/" ]; then
     rm -rf $RPM_BUILD_ROOT
 fi
 
 %files
 %defattr(-,root,root)
 /usr/src/%{module}-%{version}
 
 %pre
 echo "Running pre installation script: $0. Parameters: $*"
 # We don't want any other versions lingering around in dkms.
 # Tests with 'dnf' showed that in case of reinstall, or upgrade
 #  the preun scriptlet removed the version we are trying to install.
 # Because of this, find all zfs dkms sources in /var/lib/dkms and
 #  remove them, if we find a matching version in dkms.
 
 dkms_root=/var/lib/dkms
 if [ -d ${dkms_root}/%{module} ]; then
     cd ${dkms_root}/%{module}
     for x in [[:digit:]]*; do
         [ -d "$x" ] || continue
         otherver="$x"
         opath="${dkms_root}/%{module}/${otherver}"
         if [ "$otherver" != %{version} ]; then
             # This is a workaround for a broken 'dkms status', we caused in a previous version.
             # One day it might be not needed anymore, but it does not hurt to keep it.
             if dkms status -m %{module} -v "$otherver" 2>&1 | grep "${opath}/source/dkms.conf does not exist"
             then
                 echo "ERROR: dkms status is broken!" >&2
                 if [ -L "${opath}/source" -a ! -d "${opath}/source" ]
                 then
                     echo "Trying to fix it by removing the symlink: ${opath}/source" >&2
                     echo "You should manually remove ${opath}" >&2
                     rm -f "${opath}/source" || echo "Removal failed!" >&2
                 fi
             fi
             if [ `dkms status -m %{module} -v "$otherver" | grep -c %{module}` -gt 0 ]; then
                 echo "Removing old %{module} dkms modules version $otherver from all kernels."
                 dkms remove -m %{module} -v "$otherver" --all ||:
             fi
         fi
     done
     cd ${dkms_root}
 fi
 
 # Uninstall this version of zfs dkms modules before installation of the package.
 if [ `dkms status -m %{module} -v %{version} | grep -c %{module}` -gt 0 ]; then
     echo "Removing %{module} dkms modules version %{version} from all kernels."
     dkms remove -m %{module} -v %{version} --all ||:
 fi
 
 %post
 echo "Running post installation script: $0. Parameters: $*"
 # Add the module to dkms, as reccommended in the dkms man page.
 # This is generally rpm specfic.
 # But this also may help, if we have a broken 'dkms status'.
 # Because, if the sources are available and only the symlink pointing
 #  to them is missing, this will resolve the situation
 echo "Adding %{module} dkms modules version %{version} to dkms."
 dkms add -m %{module} -v %{version} %{!?not_rpm:--rpm_safe_upgrade} ||:
 
 # After installing the package, dkms install this zfs version for the current kernel.
 # Force the overwriting of old modules to avoid diff warnings in dkms status.
 # Or in case of a downgrade to overwrite newer versions.
 # Or if some other backed up versions have been restored before.
 echo "Installing %{module} dkms modules version %{version} for the current kernel."
 dkms install --force -m %{module} -v %{version} ||:
 
 %preun
 dkms_root="/var/lib/dkms/%{module}/%{version}"
 echo "Running pre uninstall script: $0. Parameters: $*"
 # In case of upgrade we do nothing. See above comment in pre hook.
 if [ "$1" = "1" -o "$1" = "upgrade" ] ; then
     echo "This is an upgrade. Skipping pre uninstall action."
     exit 0
 fi
 
 # Check if we uninstall the package. In that case remove the dkms modules.
 # '0' is the value for the first parameter for rpm packages.
 # 'remove' or 'purge' are the possible names for deb packages.
 if [ "$1" = "0" -o "$1" = "remove" -o "$1" = "purge" ] ; then
     if [ `dkms status -m %{module} -v %{version} | grep -c %{module}` -gt 0 ]; then
         echo "Removing %{module} dkms modules version %{version} from all kernels."
         dkms remove -m %{module} -v %{version} --all %{!?not_rpm:--rpm_safe_upgrade} && exit 0
     fi
     # If removing the modules failed, it might be because of the broken 'dkms status'.
     if dkms status -m %{module} -v %{version} 2>&1 | grep "${dkms_root}/source/dkms.conf does not exist"
     then
         echo "ERROR: dkms status is broken!" >&2
         echo "You should manually remove ${dkms_root}" >&2
         echo "WARNING: installed modules in /lib/modules/`uname -r`/extra could not be removed automatically!" >&2
     fi
 else
     echo "Script parameter $1 did not match any removal condition."
 fi
 
 exit 0
 
diff --git a/sys/contrib/openzfs/rpm/generic/zfs-kmod.spec.in b/sys/contrib/openzfs/rpm/generic/zfs-kmod.spec.in
index 30524474d1ac..7ed828bd0c9c 100644
--- a/sys/contrib/openzfs/rpm/generic/zfs-kmod.spec.in
+++ b/sys/contrib/openzfs/rpm/generic/zfs-kmod.spec.in
@@ -1,213 +1,215 @@
 %define module  @PACKAGE@
 
 %if !%{defined ksrc}
 %if 0%{?rhel}%{?fedora}%{?openEuler}
 %define ksrc    ${kernel_version##*___}
 %else
 %define ksrc    "$( \
         if [ -e "/usr/src/linux-${kernel_version%%___*}" ]; then \
             echo "/usr/src/linux-${kernel_version%%___*}"; \
         elif [ -e "/lib/modules/${kernel_version%%___*}/source" ]; then \
             echo "/lib/modules/${kernel_version%%___*}/source"; \
         else \
             echo "/lib/modules/${kernel_version%%___*}/build"; \
         fi)"
 %endif
 %endif
 
 %if !%{defined kobj}
 %if 0%{?rhel}%{?fedora}%{?openEuler}
 %define kobj    ${kernel_version##*___}
 %else
 %define kobj    "$( \
         if [ -e "/usr/src/linux-${kernel_version%%___*}" ]; then \
             echo "/usr/src/linux-${kernel_version%%___*}"; \
         else \
             echo "/lib/modules/${kernel_version%%___*}/build"; \
         fi)"
 %endif
 %endif
 
 #define repo    rpmfusion
 #define repo    chaos
 
 # (un)define the next line to either build for the newest or all current kernels
 %define buildforkernels newest
 #define buildforkernels current
 #define buildforkernels akmod
 
 %bcond_with     debug
 %bcond_with     debuginfo
 
 
 Name:           %{module}-kmod
 
 Version:        @VERSION@
 Release:        @RELEASE@%{?dist}
 Summary:        Kernel module(s)
 
 Group:          System Environment/Kernel
 License:        @ZFS_META_LICENSE@
 URL:            https://github.com/openzfs/zfs
 Source0:        %{module}-%{version}.tar.gz
 Source10:       kmodtool
 BuildRoot:      %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id} -u -n)
 %if 0%{?rhel}%{?fedora}%{?openEuler}
 BuildRequires:  gcc, make
 BuildRequires:  elfutils-libelf-devel
 %endif
 
 %if (0%{?fedora}%{?suse_version}%{?openEuler}) || (0%{?rhel} && 0%{?rhel} < 9)
 # We don't directly use it, but if this isn't installed, rpmbuild as root can
 # crash+corrupt rpmdb
 # See issue #12071
 BuildRequires:  ncompress
 %endif
 
 # The developments headers will conflict with the dkms packages.
 Conflicts:      %{module}-dkms
 
 %if %{defined repo}
 
 # Building for a repository use the proper build-sysbuild package
 # to determine which kernel-devel packages should be installed.
 BuildRequires:  %{_bindir}/kmodtool
 %{!?kernels:BuildRequires: buildsys-build-%{repo}-kerneldevpkgs-%{?buildforkernels:%{buildforkernels}}%{!?buildforkernels:current}-%{_target_cpu}}
 
 %else
 
 # Building local packages attempt to to use the installed kernel.
 %{?rhel:BuildRequires: kernel-devel}
 %{?fedora:BuildRequires: kernel-devel}
 %{?openEuler:BuildRequires: kernel-devel}
 %{?suse_version:BuildRequires: kernel-source}
 
 %if !%{defined kernels} && !%{defined build_src_rpm}
     %if 0%{?rhel}%{?fedora}%{?suse_version}%{?openEuler}
         %define kernels %(ls -1 /usr/src/kernels)
     %else
         %define kernels %(ls -1 /lib/modules)
     %endif
 %endif
 %endif
 
 # LDFLAGS are not sanitized by arch/*/Makefile for these architectures.
 %ifarch ppc ppc64 ppc64le aarch64
 %global __global_ldflags %{nil}
 %endif
 
 # Kmodtool does its magic here.  A patched version of kmodtool is shipped
 # with the source rpm until kmod development packages are supported upstream.
 # https://bugzilla.rpmfusion.org/show_bug.cgi?id=2714
 %{expand:%(bash %{SOURCE10} --target %{_target_cpu} %{?repo:--repo %{?repo}} --kmodname %{name} %{?buildforkernels:--%{buildforkernels}} --devel %{?prefix:--prefix "%{?prefix}"} %{?kernels:--for-kernels "%{?kernels}"} %{?kernelbuildroot:--buildroot "%{?kernelbuildroot}"} 2>/dev/null) }
 
 
 %description
 This package contains the ZFS kernel modules.
 
 %prep
 # Error out if there was something wrong with kmodtool.
 %{?kmodtool_check}
 
 # Print kmodtool output for debugging purposes:
 bash %{SOURCE10}  --target %{_target_cpu} %{?repo:--repo %{?repo}} --kmodname %{name} %{?buildforkernels:--%{buildforkernels}} --devel %{?prefix:--prefix "%{?prefix}"} %{?kernels:--for-kernels "%{?kernels}"} %{?kernelbuildroot:--buildroot "%{?kernelbuildroot}"} 2>/dev/null
 
 %if %{with debug}
     %define debug --enable-debug
 %else
     %define debug --disable-debug
 %endif
 
 %if %{with debuginfo}
     %define debuginfo --enable-debuginfo
 %else
     %define debuginfo --disable-debuginfo
 %endif
 
 # Leverage VPATH from configure to avoid making multiple copies.
 %define _configure ../%{module}-%{version}/configure
 
 %setup -q -c -T -a 0
 
 for kernel_version in %{?kernel_versions}; do
     %{__mkdir} _kmod_build_${kernel_version%%___*}
 done
 
 %build
 for kernel_version in %{?kernel_versions}; do
     cd _kmod_build_${kernel_version%%___*}
     %configure \
         --with-config=kernel \
         --with-linux=%{ksrc} \
         --with-linux-obj=%{kobj} \
         %{debug} \
         %{debuginfo} \
         %{?kernel_cc} \
         %{?kernel_ld} \
-        %{?kernel_llvm}
+        %{?kernel_llvm} \
+        %{?kernel_cross_compile} \
+        %{?kernel_arch}
 
     # Pre-6.10 kernel builds didn't need to copy over the source files to the
     # build directory.  However we do need to do it though post-6.10 due to
     # these commits:
     #
     # b1992c3772e6 kbuild: use $(src) instead of $(srctree)/$(src) for source
     #                      directory
     #
     # 9a0ebe5011f4 kbuild: use $(obj)/ instead of $(src)/ for common pattern
     #                      rules
     #
     # Note that kmodtool actually copies over the source into the build
     # directory, so what we're doing here is normal.  For efficiency reasons
     # though we just use hardlinks instead of copying.
     #
     # See https://github.com/openzfs/zfs/issues/16439 for more info.
     cp -lR ../%{module}-%{version}/module/* module/
 
     make %{?_smp_mflags}
     cd ..
 done
 
 
 # Module signing (modsign)
 #
 # This must be run _after_ find-debuginfo.sh runs, otherwise that will strip
 # the signature off of the modules.
 # (Based on Fedora's kernel.spec workaround)
 %define __modsign_install_post \
     sign_pem="%{ksrc}/certs/signing_key.pem"; \
     sign_x509="%{ksrc}/certs/signing_key.x509"; \
     if [ -f "${sign_x509}" ]\
     then \
         echo "Signing kernel modules ..."; \
         for kmod in $(find ${RPM_BUILD_ROOT}%{kmodinstdir_prefix}/*/extra/ -name \*.ko); do \
             %{ksrc}/scripts/sign-file sha256 ${sign_pem} ${sign_x509} ${kmod}; \
         done \
     fi \
 %{nil}
 
 # hack to ensure signing happens after find-debuginfo.sh runs
 %define __spec_install_post \
     %{?__debug_package:%{__debug_install_post}}\
     %{__arch_install_post}\
     %{__os_install_post}\
     %{__modsign_install_post}
 
 %install
 rm -rf ${RPM_BUILD_ROOT}
 
 # Relies on the kernel 'modules_install' make target.
 for kernel_version in %{?kernel_versions}; do
     cd _kmod_build_${kernel_version%%___*}
     make install \
         DESTDIR=${RPM_BUILD_ROOT} \
         %{?prefix:INSTALL_MOD_PATH=%{?prefix}} \
         INSTALL_MOD_DIR=%{kmodinstdir_postfix}
     cd ..
 done
 # find-debuginfo.sh only considers executables
 chmod u+x ${RPM_BUILD_ROOT}%{kmodinstdir_prefix}/*/extra/*/*
 %{?akmod_install}
 
 
 %clean
 rm -rf $RPM_BUILD_ROOT
diff --git a/sys/contrib/openzfs/rpm/redhat/zfs-kmod.spec.in b/sys/contrib/openzfs/rpm/redhat/zfs-kmod.spec.in
index 876c198c64de..a95bdf20f873 100644
--- a/sys/contrib/openzfs/rpm/redhat/zfs-kmod.spec.in
+++ b/sys/contrib/openzfs/rpm/redhat/zfs-kmod.spec.in
@@ -1,112 +1,114 @@
 %bcond_with     debug
 %bcond_with     debuginfo
 
 Name:           @PACKAGE@-kmod
 Version:        @VERSION@
 Release:        @RELEASE@%{?dist}
 
 Summary:        Kernel module(s)
 Group:          System Environment/Kernel
 License:        @ZFS_META_LICENSE@
 URL:            https://github.com/openzfs/zfs
 BuildRequires:  %kernel_module_package_buildreqs
 Source0:        @PACKAGE@-%{version}.tar.gz
 BuildRoot:      %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
 
 # Additional dependency information for the kmod sub-package must be specified
 # by generating a preamble text file which kmodtool can append to the spec file.
 %(/bin/echo -e "\
 Requires:       @PACKAGE@ = %{version}\n\
 Conflicts:      @PACKAGE@-dkms" > %{_sourcedir}/kmod-preamble)
 
 # LDFLAGS are not sanitized by arch/*/Makefile for these architectures.
 %ifarch ppc ppc64 ppc64le aarch64
 %global __global_ldflags %{nil}
 %endif
 
 %description
 This package contains the ZFS kernel modules.
 
 %define kmod_name @PACKAGE@
 
 %kernel_module_package -n %{kmod_name} -p %{_sourcedir}/kmod-preamble
 
 %define ksrc %{_usrsrc}/kernels/%{kverrel}
 %define kobj %{ksrc}
 
 %package -n kmod-%{kmod_name}-devel
 Summary:        ZFS kernel module(s) devel common
 Group:          System Environment/Kernel
 
 %description -n  kmod-%{kmod_name}-devel
 This package provides the header files and objects to build kernel modules.
 
 %prep
 if ! [ -d "%{ksrc}"  ]; then
         echo "Kernel build directory isn't set properly, cannot continue"
         exit 1
 fi
 
 %if %{with debug}
 %define debug --enable-debug
 %else
 %define debug --disable-debug
 %endif
 
 %if %{with debuginfo}
 %define debuginfo --enable-debuginfo
 %else
 %define debuginfo --disable-debuginfo
 %endif
 
 %setup -n %{kmod_name}-%{version}
 %build
 %configure \
         --with-config=kernel \
         --with-linux=%{ksrc} \
         --with-linux-obj=%{kobj} \
         %{debug} \
         %{debuginfo} \
         %{?kernel_cc} \
         %{?kernel_ld} \
-        %{?kernel_llvm}
+        %{?kernel_llvm} \
+        %{?kernel_cross_compile} \
+        %{?kernel_arch}
 make %{?_smp_mflags}
 
 # Module signing (modsign)
 #
 # This must be run _after_ find-debuginfo.sh runs, otherwise that will strip
 # the signature off of the modules.
 # (Based on Fedora's kernel.spec workaround)
 %define __modsign_install_post \
         sign_pem="%{ksrc}/certs/signing_key.pem"; \
         sign_x509="%{ksrc}/certs/signing_key.x509"; \
         if [ -f "${sign_x509}" ]\
         then \
             echo "Signing kernel modules ..."; \
             for kmod in $(find %{buildroot}/lib/modules/%{kverrel}/extra/ -name \*.ko); do \
                     %{ksrc}/scripts/sign-file sha256 ${sign_pem} ${sign_x509} ${kmod}; \
             done \
         fi \
 %{nil}
 
 # hack to ensure signing happens after find-debuginfo.sh runs
 %define __spec_install_post \
         %{?__debug_package:%{__debug_install_post}}\
         %{__arch_install_post}\
         %{__os_install_post}\
         %{__modsign_install_post}
 
 %install
 make install \
         DESTDIR=${RPM_BUILD_ROOT} \
         INSTALL_MOD_DIR=extra/%{kmod_name}
 %{__rm} -f %{buildroot}/lib/modules/%{kverrel}/modules.*
 
 # find-debuginfo.sh only considers executables
 %{__chmod} u+x  %{buildroot}/lib/modules/%{kverrel}/extra/*/*
 
 %clean
 rm -rf $RPM_BUILD_ROOT
 
 %files -n kmod-%{kmod_name}-devel
 %{_usrsrc}/%{kmod_name}-%{version}
diff --git a/sys/contrib/openzfs/tests/runfiles/common.run b/sys/contrib/openzfs/tests/runfiles/common.run
index f302df81b919..b49e1979bcbc 100644
--- a/sys/contrib/openzfs/tests/runfiles/common.run
+++ b/sys/contrib/openzfs/tests/runfiles/common.run
@@ -1,1020 +1,1020 @@
 #
 # This file and its contents are supplied under the terms of the
 # Common Development and Distribution License ("CDDL"), version 1.0.
 # You may only use this file in accordance with the terms of version
 # 1.0 of the CDDL.
 #
 # A full copy of the text of the CDDL should have accompanied this
 # source.  A copy of the CDDL is also available via the Internet at
 # http://www.illumos.org/license/CDDL.
 #
 # This run file contains all of the common functional tests.  When
 # adding a new test consider also adding it to the sanity.run file
 # if the new test runs to completion in only a few seconds.
 #
 # Approximate run time: 4-5 hours
 #
 
 [DEFAULT]
 pre = setup
 quiet = False
 pre_user = root
 user = root
 timeout = 600
 post_user = root
 post = cleanup
 failsafe_user = root
 failsafe = callbacks/zfs_failsafe
 outputdir = /var/tmp/test_results
 tags = ['functional']
 
 [tests/functional/acl/off]
 tests = ['dosmode', 'posixmode']
 tags = ['functional', 'acl']
 
 [tests/functional/alloc_class]
 tests = ['alloc_class_001_pos', 'alloc_class_002_neg', 'alloc_class_003_pos',
     'alloc_class_004_pos', 'alloc_class_005_pos', 'alloc_class_006_pos',
     'alloc_class_007_pos', 'alloc_class_008_pos', 'alloc_class_009_pos',
     'alloc_class_010_pos', 'alloc_class_011_neg', 'alloc_class_012_pos',
     'alloc_class_013_pos', 'alloc_class_014_neg', 'alloc_class_015_pos']
 tags = ['functional', 'alloc_class']
 
 [tests/functional/append]
 tests = ['file_append', 'threadsappend_001_pos']
 tags = ['functional', 'append']
 
 [tests/functional/arc]
 tests = ['dbufstats_001_pos', 'dbufstats_002_pos', 'dbufstats_003_pos',
     'arcstats_runtime_tuning']
 tags = ['functional', 'arc']
 
 [tests/functional/atime]
 tests = ['atime_001_pos', 'atime_002_neg', 'root_atime_off', 'root_atime_on']
 tags = ['functional', 'atime']
 
 [tests/functional/bclone]
 tests = ['bclone_crossfs_corner_cases_limited',
     'bclone_crossfs_data',
     'bclone_crossfs_embedded',
     'bclone_crossfs_hole',
     'bclone_diffprops_all',
     'bclone_diffprops_checksum',
     'bclone_diffprops_compress',
     'bclone_diffprops_copies',
     'bclone_diffprops_recordsize',
     'bclone_prop_sync',
     'bclone_samefs_corner_cases_limited',
     'bclone_samefs_data',
     'bclone_samefs_embedded',
     'bclone_samefs_hole']
 tags = ['functional', 'bclone']
 timeout = 7200
 
 [tests/functional/block_cloning]
 tests = ['block_cloning_clone_mmap_cached',
     'block_cloning_copyfilerange',
     'block_cloning_copyfilerange_partial',
     'block_cloning_copyfilerange_fallback',
     'block_cloning_disabled_copyfilerange',
     'block_cloning_copyfilerange_cross_dataset',
     'block_cloning_cross_enc_dataset',
     'block_cloning_copyfilerange_fallback_same_txg',
     'block_cloning_replay', 'block_cloning_replay_encrypted',
     'block_cloning_lwb_buffer_overflow', 'block_cloning_clone_mmap_write',
     'block_cloning_rlimit_fsize']
 tags = ['functional', 'block_cloning']
 
 [tests/functional/bootfs]
 tests = ['bootfs_001_pos', 'bootfs_002_neg', 'bootfs_003_pos',
     'bootfs_004_neg', 'bootfs_005_neg', 'bootfs_006_pos', 'bootfs_007_pos',
     'bootfs_008_pos']
 tags = ['functional', 'bootfs']
 
 [tests/functional/btree]
 tests = ['btree_positive', 'btree_negative']
 tags = ['functional', 'btree']
 pre =
 post =
 
 [tests/functional/cache]
 tests = ['cache_001_pos', 'cache_002_pos', 'cache_003_pos', 'cache_004_neg',
     'cache_005_neg', 'cache_006_pos', 'cache_007_neg', 'cache_008_neg',
     'cache_009_pos', 'cache_010_pos', 'cache_011_pos', 'cache_012_pos']
 tags = ['functional', 'cache']
 
 [tests/functional/cachefile]
 tests = ['cachefile_001_pos', 'cachefile_002_pos', 'cachefile_003_pos',
     'cachefile_004_pos']
 tags = ['functional', 'cachefile']
 
 [tests/functional/casenorm]
 tests = ['case_all_values', 'norm_all_values', 'mixed_create_failure',
     'sensitive_none_lookup', 'sensitive_none_delete',
     'sensitive_formd_lookup', 'sensitive_formd_delete',
     'insensitive_none_lookup', 'insensitive_none_delete',
     'insensitive_formd_lookup', 'insensitive_formd_delete',
     'mixed_none_lookup', 'mixed_none_lookup_ci', 'mixed_none_delete',
     'mixed_formd_lookup', 'mixed_formd_lookup_ci', 'mixed_formd_delete']
 tags = ['functional', 'casenorm']
 
 [tests/functional/channel_program/lua_core]
 tests = ['tst.args_to_lua', 'tst.divide_by_zero', 'tst.exists',
     'tst.integer_illegal', 'tst.integer_overflow', 'tst.language_functions_neg',
     'tst.language_functions_pos', 'tst.large_prog', 'tst.libraries',
     'tst.memory_limit', 'tst.nested_neg', 'tst.nested_pos', 'tst.nvlist_to_lua',
     'tst.recursive_neg', 'tst.recursive_pos', 'tst.return_large',
     'tst.return_nvlist_neg', 'tst.return_nvlist_pos',
     'tst.return_recursive_table', 'tst.stack_gsub', 'tst.timeout']
 tags = ['functional', 'channel_program', 'lua_core']
 
 [tests/functional/channel_program/synctask_core]
 tests = ['tst.destroy_fs', 'tst.destroy_snap', 'tst.get_count_and_limit',
     'tst.get_index_props', 'tst.get_mountpoint', 'tst.get_neg',
     'tst.get_number_props', 'tst.get_string_props', 'tst.get_type',
     'tst.get_userquota', 'tst.get_written', 'tst.inherit', 'tst.list_bookmarks',
     'tst.list_children', 'tst.list_clones', 'tst.list_holds',
     'tst.list_snapshots', 'tst.list_system_props',
     'tst.list_user_props', 'tst.parse_args_neg','tst.promote_conflict',
     'tst.promote_multiple', 'tst.promote_simple', 'tst.rollback_mult',
     'tst.rollback_one', 'tst.set_props', 'tst.snapshot_destroy', 'tst.snapshot_neg',
     'tst.snapshot_recursive', 'tst.snapshot_rename', 'tst.snapshot_simple',
     'tst.bookmark.create', 'tst.bookmark.copy',
     'tst.terminate_by_signal'
     ]
 tags = ['functional', 'channel_program', 'synctask_core']
 
 [tests/functional/checksum]
 tests = ['run_edonr_test', 'run_sha2_test', 'run_skein_test', 'run_blake3_test',
     'filetest_001_pos', 'filetest_002_pos']
 tags = ['functional', 'checksum']
 
 [tests/functional/clean_mirror]
 tests = [ 'clean_mirror_001_pos', 'clean_mirror_002_pos',
     'clean_mirror_003_pos', 'clean_mirror_004_pos']
 tags = ['functional', 'clean_mirror']
 
 [tests/functional/cli_root/zdb]
 tests = ['zdb_002_pos', 'zdb_003_pos', 'zdb_004_pos', 'zdb_005_pos',
     'zdb_006_pos', 'zdb_args_neg', 'zdb_args_pos',
     'zdb_block_size_histogram', 'zdb_checksum', 'zdb_decompress',
     'zdb_display_block', 'zdb_encrypted', 'zdb_label_checksum',
     'zdb_object_range_neg', 'zdb_object_range_pos', 'zdb_objset_id',
     'zdb_decompress_zstd', 'zdb_recover', 'zdb_recover_2', 'zdb_backup']
 pre =
 post =
 tags = ['functional', 'cli_root', 'zdb']
 timeout = 1200
 
 [tests/functional/cli_root/zfs]
 tests = ['zfs_001_neg', 'zfs_002_pos']
 tags = ['functional', 'cli_root', 'zfs']
 
 [tests/functional/cli_root/zfs_bookmark]
 tests = ['zfs_bookmark_cliargs']
 tags = ['functional', 'cli_root', 'zfs_bookmark']
 
 [tests/functional/cli_root/zfs_change-key]
 tests = ['zfs_change-key', 'zfs_change-key_child', 'zfs_change-key_format',
     'zfs_change-key_inherit', 'zfs_change-key_load', 'zfs_change-key_location',
     'zfs_change-key_pbkdf2iters', 'zfs_change-key_clones']
 tags = ['functional', 'cli_root', 'zfs_change-key']
 
 [tests/functional/cli_root/zfs_clone]
 tests = ['zfs_clone_001_neg', 'zfs_clone_002_pos', 'zfs_clone_003_pos',
     'zfs_clone_004_pos', 'zfs_clone_005_pos', 'zfs_clone_006_pos',
     'zfs_clone_007_pos', 'zfs_clone_008_neg', 'zfs_clone_009_neg',
     'zfs_clone_010_pos', 'zfs_clone_encrypted', 'zfs_clone_deeply_nested',
     'zfs_clone_rm_nested']
 tags = ['functional', 'cli_root', 'zfs_clone']
 
 [tests/functional/cli_root/zfs_copies]
 tests = ['zfs_copies_001_pos', 'zfs_copies_002_pos', 'zfs_copies_003_pos',
     'zfs_copies_004_neg', 'zfs_copies_005_neg', 'zfs_copies_006_pos']
 tags = ['functional', 'cli_root', 'zfs_copies']
 
 [tests/functional/cli_root/zfs_create]
 tests = ['zfs_create_001_pos', 'zfs_create_002_pos', 'zfs_create_003_pos',
     'zfs_create_004_pos', 'zfs_create_005_pos', 'zfs_create_006_pos',
     'zfs_create_007_pos', 'zfs_create_008_neg', 'zfs_create_009_neg',
     'zfs_create_010_neg', 'zfs_create_011_pos', 'zfs_create_012_pos',
     'zfs_create_013_pos', 'zfs_create_014_pos', 'zfs_create_encrypted',
     'zfs_create_crypt_combos', 'zfs_create_dryrun', 'zfs_create_nomount',
     'zfs_create_verbose']
 tags = ['functional', 'cli_root', 'zfs_create']
 
 [tests/functional/cli_root/zfs_destroy]
 tests = ['zfs_clone_livelist_condense_and_disable',
     'zfs_clone_livelist_condense_races', 'zfs_clone_livelist_dedup',
     'zfs_destroy_001_pos', 'zfs_destroy_002_pos', 'zfs_destroy_003_pos',
     'zfs_destroy_004_pos', 'zfs_destroy_005_neg', 'zfs_destroy_006_neg',
     'zfs_destroy_007_neg', 'zfs_destroy_008_pos', 'zfs_destroy_009_pos',
     'zfs_destroy_010_pos', 'zfs_destroy_011_pos', 'zfs_destroy_012_pos',
     'zfs_destroy_013_neg', 'zfs_destroy_014_pos', 'zfs_destroy_015_pos',
     'zfs_destroy_016_pos', 'zfs_destroy_clone_livelist',
     'zfs_destroy_dev_removal', 'zfs_destroy_dev_removal_condense']
 tags = ['functional', 'cli_root', 'zfs_destroy']
 
 [tests/functional/cli_root/zfs_diff]
 tests = ['zfs_diff_changes', 'zfs_diff_cliargs', 'zfs_diff_timestamp',
     'zfs_diff_types', 'zfs_diff_encrypted', 'zfs_diff_mangle']
 tags = ['functional', 'cli_root', 'zfs_diff']
 
 [tests/functional/cli_root/zfs_get]
 tests = ['zfs_get_001_pos', 'zfs_get_002_pos', 'zfs_get_003_pos',
     'zfs_get_004_pos', 'zfs_get_005_neg', 'zfs_get_006_neg', 'zfs_get_007_neg',
     'zfs_get_008_pos', 'zfs_get_009_pos', 'zfs_get_010_neg']
 tags = ['functional', 'cli_root', 'zfs_get']
 
 [tests/functional/cli_root/zfs_ids_to_path]
 tests = ['zfs_ids_to_path_001_pos']
 tags = ['functional', 'cli_root', 'zfs_ids_to_path']
 
 [tests/functional/cli_root/zfs_inherit]
 tests = ['zfs_inherit_001_neg', 'zfs_inherit_002_neg', 'zfs_inherit_003_pos',
     'zfs_inherit_mountpoint']
 tags = ['functional', 'cli_root', 'zfs_inherit']
 
 [tests/functional/cli_root/zfs_load-key]
 tests = ['zfs_load-key', 'zfs_load-key_all', 'zfs_load-key_file',
     'zfs_load-key_https', 'zfs_load-key_location', 'zfs_load-key_noop',
     'zfs_load-key_recursive']
 tags = ['functional', 'cli_root', 'zfs_load-key']
 
 [tests/functional/cli_root/zfs_mount]
 tests = ['zfs_mount_001_pos', 'zfs_mount_002_pos', 'zfs_mount_003_pos',
     'zfs_mount_004_pos', 'zfs_mount_005_pos', 'zfs_mount_007_pos',
     'zfs_mount_009_neg', 'zfs_mount_010_neg', 'zfs_mount_011_neg',
     'zfs_mount_012_pos', 'zfs_mount_all_001_pos', 'zfs_mount_encrypted',
     'zfs_mount_remount', 'zfs_mount_all_fail', 'zfs_mount_all_mountpoints',
     'zfs_mount_test_race', 'zfs_mount_recursive']
 tags = ['functional', 'cli_root', 'zfs_mount']
 
 [tests/functional/cli_root/zfs_program]
 tests = ['zfs_program_json']
 tags = ['functional', 'cli_root', 'zfs_program']
 
 [tests/functional/cli_root/zfs_promote]
 tests = ['zfs_promote_001_pos', 'zfs_promote_002_pos', 'zfs_promote_003_pos',
     'zfs_promote_004_pos', 'zfs_promote_005_pos', 'zfs_promote_006_neg',
     'zfs_promote_007_neg', 'zfs_promote_008_pos', 'zfs_promote_encryptionroot']
 tags = ['functional', 'cli_root', 'zfs_promote']
 
 [tests/functional/cli_root/zfs_property]
 tests = ['zfs_written_property_001_pos']
 tags = ['functional', 'cli_root', 'zfs_property']
 
 [tests/functional/cli_root/zfs_receive]
 tests = ['zfs_receive_001_pos', 'zfs_receive_002_pos', 'zfs_receive_003_pos',
     'zfs_receive_004_neg', 'zfs_receive_005_neg', 'zfs_receive_006_pos',
     'zfs_receive_007_neg', 'zfs_receive_008_pos', 'zfs_receive_009_neg',
     'zfs_receive_010_pos', 'zfs_receive_011_pos', 'zfs_receive_012_pos',
     'zfs_receive_013_pos', 'zfs_receive_014_pos', 'zfs_receive_015_pos',
     'zfs_receive_016_pos', 'receive-o-x_props_override',
     'receive-o-x_props_aliases',
     'zfs_receive_from_encrypted', 'zfs_receive_to_encrypted',
     'zfs_receive_raw', 'zfs_receive_raw_incremental', 'zfs_receive_-e',
     'zfs_receive_raw_-d', 'zfs_receive_from_zstd', 'zfs_receive_new_props',
     'zfs_receive_-wR-encrypted-mix', 'zfs_receive_corrective',
     'zfs_receive_compressed_corrective', 'zfs_receive_large_block_corrective']
 tags = ['functional', 'cli_root', 'zfs_receive']
 
 [tests/functional/cli_root/zfs_rename]
 tests = ['zfs_rename_001_pos', 'zfs_rename_002_pos', 'zfs_rename_003_pos',
     'zfs_rename_004_neg', 'zfs_rename_005_neg', 'zfs_rename_006_pos',
     'zfs_rename_007_pos', 'zfs_rename_008_pos', 'zfs_rename_009_neg',
     'zfs_rename_010_neg', 'zfs_rename_011_pos', 'zfs_rename_012_neg',
     'zfs_rename_013_pos', 'zfs_rename_014_neg', 'zfs_rename_encrypted_child',
     'zfs_rename_to_encrypted', 'zfs_rename_mountpoint', 'zfs_rename_nounmount']
 tags = ['functional', 'cli_root', 'zfs_rename']
 
 [tests/functional/cli_root/zfs_reservation]
 tests = ['zfs_reservation_001_pos', 'zfs_reservation_002_pos']
 tags = ['functional', 'cli_root', 'zfs_reservation']
 
 [tests/functional/cli_root/zfs_rollback]
 tests = ['zfs_rollback_001_pos', 'zfs_rollback_002_pos',
     'zfs_rollback_003_neg', 'zfs_rollback_004_neg']
 tags = ['functional', 'cli_root', 'zfs_rollback']
 
 [tests/functional/cli_root/zfs_send]
 tests = ['zfs_send_001_pos', 'zfs_send_002_pos', 'zfs_send_003_pos',
     'zfs_send_004_neg', 'zfs_send_005_pos', 'zfs_send_006_pos',
     'zfs_send_007_pos', 'zfs_send_encrypted', 'zfs_send_encrypted_unloaded',
     'zfs_send_raw', 'zfs_send_sparse', 'zfs_send-b', 'zfs_send_skip_missing']
 tags = ['functional', 'cli_root', 'zfs_send']
 
 [tests/functional/cli_root/zfs_set]
 tests = ['cache_001_pos', 'cache_002_neg', 'canmount_001_pos',
     'canmount_002_pos', 'canmount_003_pos', 'canmount_004_pos',
     'checksum_001_pos', 'compression_001_pos', 'mountpoint_001_pos',
     'mountpoint_002_pos', 'reservation_001_neg', 'user_property_002_pos',
     'share_mount_001_neg', 'snapdir_001_pos', 'onoffs_001_pos',
     'user_property_001_pos', 'user_property_003_neg', 'readonly_001_pos',
     'user_property_004_pos', 'version_001_neg', 'zfs_set_001_neg',
     'zfs_set_002_neg', 'zfs_set_003_neg', 'property_alias_001_pos',
     'mountpoint_003_pos', 'ro_props_001_pos', 'zfs_set_keylocation',
     'zfs_set_feature_activation', 'zfs_set_nomount']
 tags = ['functional', 'cli_root', 'zfs_set']
 
 [tests/functional/cli_root/zfs_share]
 tests = ['zfs_share_001_pos', 'zfs_share_002_pos', 'zfs_share_003_pos',
     'zfs_share_004_pos', 'zfs_share_006_pos', 'zfs_share_008_neg',
     'zfs_share_010_neg', 'zfs_share_011_pos', 'zfs_share_concurrent_shares',
     'zfs_share_after_mount']
 tags = ['functional', 'cli_root', 'zfs_share']
 
 [tests/functional/cli_root/zfs_snapshot]
 tests = ['zfs_snapshot_001_neg', 'zfs_snapshot_002_neg',
     'zfs_snapshot_003_neg', 'zfs_snapshot_004_neg', 'zfs_snapshot_005_neg',
     'zfs_snapshot_006_pos', 'zfs_snapshot_007_neg', 'zfs_snapshot_008_neg',
     'zfs_snapshot_009_pos']
 tags = ['functional', 'cli_root', 'zfs_snapshot']
 
 [tests/functional/cli_root/zfs_unload-key]
 tests = ['zfs_unload-key', 'zfs_unload-key_all', 'zfs_unload-key_recursive']
 tags = ['functional', 'cli_root', 'zfs_unload-key']
 
 [tests/functional/cli_root/zfs_unmount]
 tests = ['zfs_unmount_001_pos', 'zfs_unmount_002_pos', 'zfs_unmount_003_pos',
     'zfs_unmount_004_pos', 'zfs_unmount_005_pos', 'zfs_unmount_006_pos',
     'zfs_unmount_007_neg', 'zfs_unmount_008_neg', 'zfs_unmount_009_pos',
     'zfs_unmount_all_001_pos', 'zfs_unmount_nested', 'zfs_unmount_unload_keys']
 tags = ['functional', 'cli_root', 'zfs_unmount']
 
 [tests/functional/cli_root/zfs_unshare]
 tests = ['zfs_unshare_001_pos', 'zfs_unshare_002_pos', 'zfs_unshare_003_pos',
     'zfs_unshare_004_neg', 'zfs_unshare_005_neg', 'zfs_unshare_006_pos',
     'zfs_unshare_007_pos']
 tags = ['functional', 'cli_root', 'zfs_unshare']
 
 [tests/functional/cli_root/zfs_upgrade]
 tests = ['zfs_upgrade_001_pos', 'zfs_upgrade_002_pos', 'zfs_upgrade_003_pos',
     'zfs_upgrade_004_pos', 'zfs_upgrade_005_pos', 'zfs_upgrade_006_neg',
     'zfs_upgrade_007_neg']
 tags = ['functional', 'cli_root', 'zfs_upgrade']
 
 [tests/functional/cli_root/zfs_wait]
 tests = ['zfs_wait_deleteq', 'zfs_wait_getsubopt']
 tags = ['functional', 'cli_root', 'zfs_wait']
 
 [tests/functional/cli_root/zhack]
 tests = ['zhack_label_repair_001', 'zhack_label_repair_002',
     'zhack_label_repair_003', 'zhack_label_repair_004']
 pre =
 post =
 tags = ['functional', 'cli_root', 'zhack']
 
 [tests/functional/cli_root/zpool]
 tests = ['zpool_001_neg', 'zpool_002_pos', 'zpool_003_pos', 'zpool_colors']
 tags = ['functional', 'cli_root', 'zpool']
 
 [tests/functional/cli_root/zpool_add]
 tests = ['zpool_add_001_pos', 'zpool_add_002_pos', 'zpool_add_003_pos',
     'zpool_add_004_pos', 'zpool_add_006_pos', 'zpool_add_007_neg',
     'zpool_add_008_neg', 'zpool_add_009_neg', 'zpool_add_010_pos',
     'add-o_ashift', 'add_prop_ashift', 'zpool_add_dryrun_output',
     'zpool_add--allow-ashift-mismatch']
 tags = ['functional', 'cli_root', 'zpool_add']
 
 [tests/functional/cli_root/zpool_attach]
 tests = ['zpool_attach_001_neg', 'attach-o_ashift']
 tags = ['functional', 'cli_root', 'zpool_attach']
 
 [tests/functional/cli_root/zpool_clear]
 tests = ['zpool_clear_001_pos', 'zpool_clear_002_neg', 'zpool_clear_003_neg',
     'zpool_clear_readonly']
 tags = ['functional', 'cli_root', 'zpool_clear']
 
 [tests/functional/cli_root/zpool_create]
 tests = ['zpool_create_001_pos', 'zpool_create_002_pos',
     'zpool_create_003_pos', 'zpool_create_004_pos', 'zpool_create_005_pos',
     'zpool_create_006_pos', 'zpool_create_007_neg', 'zpool_create_008_pos',
     'zpool_create_009_neg', 'zpool_create_010_neg', 'zpool_create_011_neg',
     'zpool_create_012_neg', 'zpool_create_014_neg', 'zpool_create_015_neg',
     'zpool_create_017_neg', 'zpool_create_018_pos', 'zpool_create_019_pos',
     'zpool_create_020_pos', 'zpool_create_021_pos', 'zpool_create_022_pos',
     'zpool_create_023_neg', 'zpool_create_024_pos',
     'zpool_create_encrypted', 'zpool_create_crypt_combos',
     'zpool_create_draid_001_pos', 'zpool_create_draid_002_pos',
     'zpool_create_draid_003_pos', 'zpool_create_draid_004_pos',
     'zpool_create_features_001_pos', 'zpool_create_features_002_pos',
     'zpool_create_features_003_pos', 'zpool_create_features_004_neg',
     'zpool_create_features_005_pos', 'zpool_create_features_006_pos',
     'zpool_create_features_007_pos', 'zpool_create_features_008_pos',
     'zpool_create_features_009_pos', 'create-o_ashift',
     'zpool_create_tempname', 'zpool_create_dryrun_output']
 tags = ['functional', 'cli_root', 'zpool_create']
 
 [tests/functional/cli_root/zpool_destroy]
 tests = ['zpool_destroy_001_pos', 'zpool_destroy_002_pos',
     'zpool_destroy_003_neg']
 pre =
 post =
 tags = ['functional', 'cli_root', 'zpool_destroy']
 
 [tests/functional/cli_root/zpool_detach]
 tests = ['zpool_detach_001_neg']
 tags = ['functional', 'cli_root', 'zpool_detach']
 
 [tests/functional/cli_root/zpool_events]
 tests = ['zpool_events_clear', 'zpool_events_cliargs', 'zpool_events_follow',
     'zpool_events_poolname', 'zpool_events_errors', 'zpool_events_duplicates',
     'zpool_events_clear_retained']
 tags = ['functional', 'cli_root', 'zpool_events']
 
 [tests/functional/cli_root/zpool_export]
 tests = ['zpool_export_001_pos', 'zpool_export_002_pos',
     'zpool_export_003_neg', 'zpool_export_004_pos']
 tags = ['functional', 'cli_root', 'zpool_export']
 
 [tests/functional/cli_root/zpool_get]
 tests = ['zpool_get_001_pos', 'zpool_get_002_pos', 'zpool_get_003_pos',
     'zpool_get_004_neg', 'zpool_get_005_pos', 'vdev_get_001_pos']
 tags = ['functional', 'cli_root', 'zpool_get']
 
 [tests/functional/cli_root/zpool_history]
 tests = ['zpool_history_001_neg', 'zpool_history_002_pos']
 tags = ['functional', 'cli_root', 'zpool_history']
 
 [tests/functional/cli_root/zpool_import]
 tests = ['zpool_import_001_pos', 'zpool_import_002_pos',
     'zpool_import_003_pos', 'zpool_import_004_pos', 'zpool_import_005_pos',
     'zpool_import_006_pos', 'zpool_import_007_pos', 'zpool_import_008_pos',
     'zpool_import_009_neg', 'zpool_import_010_pos', 'zpool_import_011_neg',
     'zpool_import_012_pos', 'zpool_import_013_neg', 'zpool_import_014_pos',
     'zpool_import_015_pos', 'zpool_import_016_pos', 'zpool_import_017_pos',
     'zpool_import_features_001_pos', 'zpool_import_features_002_neg',
     'zpool_import_features_003_pos', 'zpool_import_missing_001_pos',
     'zpool_import_missing_002_pos', 'zpool_import_missing_003_pos',
     'zpool_import_rename_001_pos', 'zpool_import_all_001_pos',
     'zpool_import_encrypted', 'zpool_import_encrypted_load',
     'zpool_import_errata3', 'zpool_import_errata4',
     'import_cachefile_device_added',
     'import_cachefile_device_removed',
     'import_cachefile_device_replaced',
     'import_cachefile_mirror_attached',
     'import_cachefile_mirror_detached',
     'import_cachefile_paths_changed',
     'import_cachefile_shared_device',
     'import_devices_missing', 'import_log_missing',
     'import_paths_changed',
     'import_rewind_config_changed',
     'import_rewind_device_replaced',
     'zpool_import_status']
 tags = ['functional', 'cli_root', 'zpool_import']
 timeout = 1200
 
 [tests/functional/cli_root/zpool_labelclear]
 tests = ['zpool_labelclear_active', 'zpool_labelclear_exported',
     'zpool_labelclear_removed', 'zpool_labelclear_valid']
 pre =
 post =
 tags = ['functional', 'cli_root', 'zpool_labelclear']
 
 [tests/functional/cli_root/zpool_initialize]
 tests = ['zpool_initialize_attach_detach_add_remove',
     'zpool_initialize_fault_export_import_online',
     'zpool_initialize_import_export',
     'zpool_initialize_offline_export_import_online',
     'zpool_initialize_online_offline',
     'zpool_initialize_split',
     'zpool_initialize_start_and_cancel_neg',
     'zpool_initialize_start_and_cancel_pos',
     'zpool_initialize_suspend_resume',
     'zpool_initialize_uninit',
     'zpool_initialize_unsupported_vdevs',
     'zpool_initialize_verify_checksums',
     'zpool_initialize_verify_initialized']
 pre =
 tags = ['functional', 'cli_root', 'zpool_initialize']
 
 [tests/functional/cli_root/zpool_offline]
 tests = ['zpool_offline_001_pos', 'zpool_offline_002_neg',
     'zpool_offline_003_pos']
 tags = ['functional', 'cli_root', 'zpool_offline']
 
 [tests/functional/cli_root/zpool_online]
 tests = ['zpool_online_001_pos', 'zpool_online_002_neg']
 tags = ['functional', 'cli_root', 'zpool_online']
 
 [tests/functional/cli_root/zpool_remove]
 tests = ['zpool_remove_001_neg', 'zpool_remove_002_pos',
     'zpool_remove_003_pos']
 tags = ['functional', 'cli_root', 'zpool_remove']
 
 [tests/functional/cli_root/zpool_replace]
 tests = ['zpool_replace_001_neg', 'replace-o_ashift', 'replace_prop_ashift']
 tags = ['functional', 'cli_root', 'zpool_replace']
 
 [tests/functional/cli_root/zpool_resilver]
 tests = ['zpool_resilver_bad_args', 'zpool_resilver_restart',
     'zpool_resilver_concurrent']
 tags = ['functional', 'cli_root', 'zpool_resilver']
 
 [tests/functional/cli_root/zpool_scrub]
 tests = ['zpool_scrub_001_neg', 'zpool_scrub_002_pos', 'zpool_scrub_003_pos',
     'zpool_scrub_004_pos', 'zpool_scrub_005_pos',
     'zpool_scrub_encrypted_unloaded', 'zpool_scrub_print_repairing',
     'zpool_scrub_offline_device', 'zpool_scrub_multiple_copies',
     'zpool_error_scrub_001_pos', 'zpool_error_scrub_002_pos',
     'zpool_error_scrub_003_pos', 'zpool_error_scrub_004_pos']
 tags = ['functional', 'cli_root', 'zpool_scrub']
 
 [tests/functional/cli_root/zpool_set]
 tests = ['zpool_set_001_pos', 'zpool_set_002_neg', 'zpool_set_003_neg',
     'zpool_set_ashift', 'zpool_set_features', 'vdev_set_001_pos',
     'user_property_001_pos', 'user_property_002_neg']
 tags = ['functional', 'cli_root', 'zpool_set']
 
 [tests/functional/cli_root/zpool_split]
 tests = ['zpool_split_cliargs', 'zpool_split_devices',
     'zpool_split_encryption', 'zpool_split_props', 'zpool_split_vdevs',
     'zpool_split_resilver', 'zpool_split_indirect',
     'zpool_split_dryrun_output']
 tags = ['functional', 'cli_root', 'zpool_split']
 
 [tests/functional/cli_root/zpool_status]
 tests = ['zpool_status_001_pos', 'zpool_status_002_pos',
     'zpool_status_003_pos', 'zpool_status_004_pos',
     'zpool_status_005_pos', 'zpool_status_006_pos',
     'zpool_status_007_pos', 'zpool_status_008_pos',
     'zpool_status_features_001_pos']
 tags = ['functional', 'cli_root', 'zpool_status']
 
 [tests/functional/cli_root/zpool_sync]
 tests = ['zpool_sync_001_pos', 'zpool_sync_002_neg']
 tags = ['functional', 'cli_root', 'zpool_sync']
 
 [tests/functional/cli_root/zpool_trim]
 tests = ['zpool_trim_attach_detach_add_remove',
     'zpool_trim_fault_export_import_online',
     'zpool_trim_import_export', 'zpool_trim_multiple', 'zpool_trim_neg',
     'zpool_trim_offline_export_import_online', 'zpool_trim_online_offline',
     'zpool_trim_partial', 'zpool_trim_rate', 'zpool_trim_rate_neg',
     'zpool_trim_secure', 'zpool_trim_split', 'zpool_trim_start_and_cancel_neg',
     'zpool_trim_start_and_cancel_pos', 'zpool_trim_suspend_resume',
     'zpool_trim_unsupported_vdevs', 'zpool_trim_verify_checksums',
     'zpool_trim_verify_trimmed']
 tags = ['functional', 'zpool_trim']
 
 [tests/functional/cli_root/zpool_upgrade]
 tests = ['zpool_upgrade_001_pos', 'zpool_upgrade_002_pos',
     'zpool_upgrade_003_pos', 'zpool_upgrade_004_pos',
     'zpool_upgrade_005_neg', 'zpool_upgrade_006_neg',
     'zpool_upgrade_007_pos', 'zpool_upgrade_008_pos',
     'zpool_upgrade_009_neg', 'zpool_upgrade_features_001_pos']
 tags = ['functional', 'cli_root', 'zpool_upgrade']
 
 [tests/functional/cli_root/zpool_wait]
 tests = ['zpool_wait_discard', 'zpool_wait_freeing',
     'zpool_wait_initialize_basic', 'zpool_wait_initialize_cancel',
     'zpool_wait_initialize_flag', 'zpool_wait_multiple',
     'zpool_wait_no_activity', 'zpool_wait_remove', 'zpool_wait_remove_cancel',
     'zpool_wait_trim_basic', 'zpool_wait_trim_cancel', 'zpool_wait_trim_flag',
     'zpool_wait_usage']
 tags = ['functional', 'cli_root', 'zpool_wait']
 
 [tests/functional/cli_root/zpool_wait/scan]
 tests = ['zpool_wait_replace_cancel', 'zpool_wait_rebuild',
     'zpool_wait_resilver', 'zpool_wait_scrub_cancel',
     'zpool_wait_replace', 'zpool_wait_scrub_basic', 'zpool_wait_scrub_flag']
 tags = ['functional', 'cli_root', 'zpool_wait']
 
 [tests/functional/cli_user/misc]
 tests = ['zdb_001_neg', 'zfs_001_neg', 'zfs_allow_001_neg',
     'zfs_clone_001_neg', 'zfs_create_001_neg', 'zfs_destroy_001_neg',
     'zfs_get_001_neg', 'zfs_inherit_001_neg', 'zfs_mount_001_neg',
     'zfs_promote_001_neg', 'zfs_receive_001_neg', 'zfs_rename_001_neg',
     'zfs_rollback_001_neg', 'zfs_send_001_neg', 'zfs_set_001_neg',
     'zfs_share_001_neg', 'zfs_snapshot_001_neg', 'zfs_unallow_001_neg',
     'zfs_unmount_001_neg', 'zfs_unshare_001_neg', 'zfs_upgrade_001_neg',
     'zpool_001_neg', 'zpool_add_001_neg', 'zpool_attach_001_neg',
     'zpool_clear_001_neg', 'zpool_create_001_neg', 'zpool_destroy_001_neg',
     'zpool_detach_001_neg', 'zpool_export_001_neg', 'zpool_get_001_neg',
     'zpool_history_001_neg', 'zpool_import_001_neg', 'zpool_import_002_neg',
     'zpool_offline_001_neg', 'zpool_online_001_neg', 'zpool_remove_001_neg',
     'zpool_replace_001_neg', 'zpool_scrub_001_neg', 'zpool_set_001_neg',
     'zpool_status_001_neg', 'zpool_upgrade_001_neg', 'arcstat_001_pos',
     'arc_summary_001_pos', 'arc_summary_002_neg', 'zpool_wait_privilege',
     'zilstat_001_pos']
 user =
 tags = ['functional', 'cli_user', 'misc']
 
 [tests/functional/cli_user/zfs_list]
 tests = ['zfs_list_001_pos', 'zfs_list_002_pos', 'zfs_list_003_pos',
     'zfs_list_004_neg', 'zfs_list_005_neg', 'zfs_list_007_pos',
     'zfs_list_008_neg']
 user =
 tags = ['functional', 'cli_user', 'zfs_list']
 
 [tests/functional/cli_user/zpool_iostat]
 tests = ['zpool_iostat_001_neg', 'zpool_iostat_002_pos',
     'zpool_iostat_003_neg', 'zpool_iostat_004_pos',
     'zpool_iostat_005_pos', 'zpool_iostat_-c_disable',
     'zpool_iostat_-c_homedir', 'zpool_iostat_-c_searchpath']
 user =
 tags = ['functional', 'cli_user', 'zpool_iostat']
 
 [tests/functional/cli_user/zpool_list]
 tests = ['zpool_list_001_pos', 'zpool_list_002_neg']
 user =
 tags = ['functional', 'cli_user', 'zpool_list']
 
 [tests/functional/cli_user/zpool_status]
 tests = ['zpool_status_003_pos', 'zpool_status_-c_disable',
     'zpool_status_-c_homedir', 'zpool_status_-c_searchpath']
 user =
 tags = ['functional', 'cli_user', 'zpool_status']
 
 [tests/functional/compression]
 tests = ['compress_001_pos', 'compress_002_pos', 'compress_003_pos',
     'l2arc_compressed_arc', 'l2arc_compressed_arc_disabled',
     'l2arc_encrypted', 'l2arc_encrypted_no_compressed_arc']
 tags = ['functional', 'compression']
 
 [tests/functional/cp_files]
 tests = ['cp_files_001_pos', 'cp_files_002_pos', 'cp_stress']
 tags = ['functional', 'cp_files']
 
 [tests/functional/crtime]
 tests = ['crtime_001_pos' ]
 tags = ['functional', 'crtime']
 
 [tests/functional/ctime]
 tests = ['ctime_001_pos' ]
 tags = ['functional', 'ctime']
 
 [tests/functional/deadman]
 tests = ['deadman_ratelimit', 'deadman_sync', 'deadman_zio']
 pre =
 post =
 tags = ['functional', 'deadman']
 
 [tests/functional/delegate]
 tests = ['zfs_allow_001_pos', 'zfs_allow_002_pos', 'zfs_allow_003_pos',
     'zfs_allow_004_pos', 'zfs_allow_005_pos', 'zfs_allow_006_pos',
     'zfs_allow_007_pos', 'zfs_allow_008_pos', 'zfs_allow_009_neg',
     'zfs_allow_010_pos', 'zfs_allow_011_neg', 'zfs_allow_012_neg',
     'zfs_unallow_001_pos', 'zfs_unallow_002_pos', 'zfs_unallow_003_pos',
     'zfs_unallow_004_pos', 'zfs_unallow_005_pos', 'zfs_unallow_006_pos',
     'zfs_unallow_007_neg', 'zfs_unallow_008_neg']
 tags = ['functional', 'delegate']
 
 [tests/functional/exec]
 tests = ['exec_001_pos', 'exec_002_neg']
 tags = ['functional', 'exec']
 
 [tests/functional/fallocate]
 tests = ['fallocate_punch-hole']
 tags = ['functional', 'fallocate']
 
 [tests/functional/features/async_destroy]
 tests = ['async_destroy_001_pos']
 tags = ['functional', 'features', 'async_destroy']
 
 [tests/functional/features/large_dnode]
 tests = ['large_dnode_001_pos', 'large_dnode_003_pos', 'large_dnode_004_neg',
     'large_dnode_005_pos', 'large_dnode_007_neg', 'large_dnode_009_pos']
 tags = ['functional', 'features', 'large_dnode']
 
 [tests/functional/grow]
 pre =
 post =
 tests = ['grow_pool_001_pos', 'grow_replicas_001_pos']
 tags = ['functional', 'grow']
 
 [tests/functional/history]
 tests = ['history_001_pos', 'history_002_pos', 'history_003_pos',
     'history_004_pos', 'history_005_neg', 'history_006_neg',
     'history_007_pos', 'history_008_pos', 'history_009_pos',
     'history_010_pos']
 tags = ['functional', 'history']
 
 [tests/functional/hkdf]
 pre =
 post =
 tests = ['hkdf_test']
 tags = ['functional', 'hkdf']
 
 [tests/functional/inheritance]
 tests = ['inherit_001_pos']
 pre =
 tags = ['functional', 'inheritance']
 
 [tests/functional/io]
 tests = ['sync', 'psync', 'posixaio', 'mmap']
 tags = ['functional', 'io']
 
 [tests/functional/inuse]
 tests = ['inuse_004_pos', 'inuse_005_pos', 'inuse_008_pos', 'inuse_009_pos']
 post =
 tags = ['functional', 'inuse']
 
 [tests/functional/large_files]
 tests = ['large_files_001_pos', 'large_files_002_pos']
 tags = ['functional', 'large_files']
 
 [tests/functional/limits]
 tests = ['filesystem_count', 'filesystem_limit', 'snapshot_count',
     'snapshot_limit']
 tags = ['functional', 'limits']
 
 [tests/functional/link_count]
 tests = ['link_count_001', 'link_count_root_inode']
 tags = ['functional', 'link_count']
 
 [tests/functional/migration]
 tests = ['migration_001_pos', 'migration_002_pos', 'migration_003_pos',
     'migration_004_pos', 'migration_005_pos', 'migration_006_pos',
     'migration_007_pos', 'migration_008_pos', 'migration_009_pos',
     'migration_010_pos', 'migration_011_pos', 'migration_012_pos']
 tags = ['functional', 'migration']
 
 [tests/functional/mmap]
 tests = ['mmap_mixed', 'mmap_read_001_pos', 'mmap_seek_001_pos',
     'mmap_sync_001_pos', 'mmap_write_001_pos']
 tags = ['functional', 'mmap']
 
 [tests/functional/mount]
 tests = ['umount_001', 'umountall_001']
 tags = ['functional', 'mount']
 
 [tests/functional/mv_files]
 tests = ['mv_files_001_pos', 'mv_files_002_pos', 'random_creation']
 tags = ['functional', 'mv_files']
 
 [tests/functional/nestedfs]
 tests = ['nestedfs_001_pos']
 tags = ['functional', 'nestedfs']
 
 [tests/functional/no_space]
 tests = ['enospc_001_pos', 'enospc_002_pos', 'enospc_003_pos',
     'enospc_df', 'enospc_ganging', 'enospc_rm']
 tags = ['functional', 'no_space']
 
 [tests/functional/nopwrite]
 tests = ['nopwrite_copies', 'nopwrite_mtime', 'nopwrite_negative',
     'nopwrite_promoted_clone', 'nopwrite_recsize', 'nopwrite_sync',
     'nopwrite_varying_compression', 'nopwrite_volume']
 tags = ['functional', 'nopwrite']
 
 [tests/functional/online_offline]
 tests = ['online_offline_001_pos', 'online_offline_002_neg',
     'online_offline_003_neg']
 tags = ['functional', 'online_offline']
 
 [tests/functional/pool_checkpoint]
 tests = ['checkpoint_after_rewind', 'checkpoint_big_rewind',
     'checkpoint_capacity', 'checkpoint_conf_change', 'checkpoint_discard',
     'checkpoint_discard_busy', 'checkpoint_discard_many',
     'checkpoint_indirect', 'checkpoint_invalid', 'checkpoint_lun_expsz',
     'checkpoint_open', 'checkpoint_removal', 'checkpoint_rewind',
     'checkpoint_ro_rewind', 'checkpoint_sm_scale', 'checkpoint_twice',
     'checkpoint_vdev_add', 'checkpoint_zdb', 'checkpoint_zhack_feat']
 tags = ['functional', 'pool_checkpoint']
 timeout = 1800
 
 [tests/functional/pool_names]
 tests = ['pool_names_001_pos', 'pool_names_002_neg']
 pre =
 post =
 tags = ['functional', 'pool_names']
 
 [tests/functional/poolversion]
 tests = ['poolversion_001_pos', 'poolversion_002_pos']
 tags = ['functional', 'poolversion']
 
 [tests/functional/pyzfs]
 tests = ['pyzfs_unittest']
 pre =
 post =
 tags = ['functional', 'pyzfs']
 
 [tests/functional/quota]
 tests = ['quota_001_pos', 'quota_002_pos', 'quota_003_pos',
          'quota_004_pos', 'quota_005_pos', 'quota_006_neg']
 tags = ['functional', 'quota']
 
 [tests/functional/redacted_send]
 tests = ['redacted_compressed', 'redacted_contents', 'redacted_deleted',
     'redacted_disabled_feature', 'redacted_embedded', 'redacted_holes',
     'redacted_incrementals', 'redacted_largeblocks', 'redacted_many_clones',
     'redacted_mixed_recsize', 'redacted_mounts', 'redacted_negative',
     'redacted_origin', 'redacted_panic', 'redacted_props', 'redacted_resume',
     'redacted_size', 'redacted_volume']
 tags = ['functional', 'redacted_send']
 
 [tests/functional/raidz]
 tests = ['raidz_001_neg', 'raidz_002_pos', 'raidz_003_pos', 'raidz_004_pos']
 tags = ['functional', 'raidz']
 
 [tests/functional/redundancy]
 tests = ['redundancy_draid', 'redundancy_draid1', 'redundancy_draid2',
     'redundancy_draid3', 'redundancy_draid_damaged1',
     'redundancy_draid_damaged2', 'redundancy_draid_spare1',
     'redundancy_draid_spare2', 'redundancy_draid_spare3', 'redundancy_mirror',
     'redundancy_raidz', 'redundancy_raidz1', 'redundancy_raidz2',
     'redundancy_raidz3', 'redundancy_stripe']
 tags = ['functional', 'redundancy']
 timeout = 1200
 
 [tests/functional/refquota]
 tests = ['refquota_001_pos', 'refquota_002_pos', 'refquota_003_pos',
     'refquota_004_pos', 'refquota_005_pos', 'refquota_006_neg',
     'refquota_007_neg', 'refquota_008_neg']
 tags = ['functional', 'refquota']
 
 [tests/functional/refreserv]
 tests = ['refreserv_001_pos', 'refreserv_002_pos', 'refreserv_003_pos',
     'refreserv_004_pos', 'refreserv_005_pos', 'refreserv_multi_raidz',
     'refreserv_raidz']
 tags = ['functional', 'refreserv']
 
 [tests/functional/removal]
 pre =
 tests = ['removal_all_vdev', 'removal_cancel', 'removal_check_space',
     'removal_condense_export', 'removal_multiple_indirection',
     'removal_nopwrite', 'removal_remap_deadlists',
     'removal_resume_export', 'removal_sanity', 'removal_with_add',
     'removal_with_create_fs', 'removal_with_dedup',
     'removal_with_errors', 'removal_with_export', 'removal_with_indirect',
     'removal_with_ganging', 'removal_with_faulted',
     'removal_with_remove', 'removal_with_scrub', 'removal_with_send',
     'removal_with_send_recv', 'removal_with_snapshot',
     'removal_with_write', 'removal_with_zdb', 'remove_expanded',
     'remove_mirror', 'remove_mirror_sanity', 'remove_raidz',
     'remove_indirect', 'remove_attach_mirror', 'removal_reservation']
 tags = ['functional', 'removal']
 
 [tests/functional/rename_dirs]
 tests = ['rename_dirs_001_pos']
 tags = ['functional', 'rename_dirs']
 
 [tests/functional/replacement]
 tests = ['attach_import', 'attach_multiple', 'attach_rebuild',
     'attach_resilver', 'detach', 'rebuild_disabled_feature',
     'rebuild_multiple', 'rebuild_raidz', 'replace_import', 'replace_rebuild',
     'replace_resilver', 'resilver_restart_001', 'resilver_restart_002',
     'scrub_cancel']
 tags = ['functional', 'replacement']
 
 [tests/functional/reservation]
 tests = ['reservation_001_pos', 'reservation_002_pos', 'reservation_003_pos',
     'reservation_004_pos', 'reservation_005_pos', 'reservation_006_pos',
     'reservation_007_pos', 'reservation_008_pos', 'reservation_009_pos',
     'reservation_010_pos', 'reservation_011_pos', 'reservation_012_pos',
     'reservation_013_pos', 'reservation_014_pos', 'reservation_015_pos',
     'reservation_016_pos', 'reservation_017_pos', 'reservation_018_pos',
     'reservation_019_pos', 'reservation_020_pos', 'reservation_021_neg',
     'reservation_022_pos']
 tags = ['functional', 'reservation']
 
 [tests/functional/rootpool]
 tests = ['rootpool_002_neg', 'rootpool_003_neg', 'rootpool_007_pos']
 tags = ['functional', 'rootpool']
 
 [tests/functional/rsend]
 tests = ['recv_dedup', 'recv_dedup_encrypted_zvol', 'rsend_001_pos',
     'rsend_002_pos', 'rsend_003_pos', 'rsend_004_pos', 'rsend_005_pos',
     'rsend_006_pos', 'rsend_007_pos', 'rsend_008_pos', 'rsend_009_pos',
     'rsend_010_pos', 'rsend_011_pos', 'rsend_012_pos', 'rsend_013_pos',
     'rsend_014_pos', 'rsend_016_neg', 'rsend_019_pos', 'rsend_020_pos',
     'rsend_021_pos', 'rsend_022_pos', 'rsend_024_pos', 'rsend_025_pos',
     'rsend_026_neg', 'rsend_027_pos', 'rsend_028_neg', 'rsend_029_neg',
     'rsend_030_pos', 'rsend_031_pos', 'send-c_verify_ratio',
     'send-c_verify_contents', 'send-c_props', 'send-c_incremental',
     'send-c_volume', 'send-c_zstream_recompress', 'send-c_zstreamdump',
     'send-c_lz4_disabled', 'send-c_recv_lz4_disabled',
     'send-c_mixed_compression', 'send-c_stream_size_estimate',
     'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
     'send-c_recv_dedup', 'send-L_toggle', 'send_encrypted_incremental',
     'send_encrypted_freeobjects', 'send_encrypted_hierarchy',
     'send_encrypted_props', 'send_encrypted_truncated_files',
     'send_freeobjects', 'send_realloc_files', 'send_realloc_encrypted_files',
     'send_spill_block', 'send_holds', 'send_hole_birth', 'send_mixed_raw',
     'send-wR_encrypted_zvol', 'send_partial_dataset', 'send_invalid',
     'send_doall', 'send_raw_spill_block', 'send_raw_ashift',
-    'send_raw_large_blocks']
+    'send_raw_large_blocks', 'send_leak_keymaps']
 tags = ['functional', 'rsend']
 
 [tests/functional/scrub_mirror]
 tests = ['scrub_mirror_001_pos', 'scrub_mirror_002_pos',
     'scrub_mirror_003_pos', 'scrub_mirror_004_pos']
 tags = ['functional', 'scrub_mirror']
 
 [tests/functional/slog]
 tests = ['slog_001_pos', 'slog_002_pos', 'slog_003_pos', 'slog_004_pos',
     'slog_005_pos', 'slog_006_pos', 'slog_007_pos', 'slog_008_neg',
     'slog_009_neg', 'slog_010_neg', 'slog_011_neg', 'slog_012_neg',
     'slog_013_pos', 'slog_014_pos', 'slog_015_neg', 'slog_replay_fs_001',
     'slog_replay_fs_002', 'slog_replay_volume', 'slog_016_pos']
 tags = ['functional', 'slog']
 
 [tests/functional/snapshot]
 tests = ['clone_001_pos', 'rollback_001_pos', 'rollback_002_pos',
     'rollback_003_pos', 'snapshot_001_pos', 'snapshot_002_pos',
     'snapshot_003_pos', 'snapshot_004_pos', 'snapshot_005_pos',
     'snapshot_006_pos', 'snapshot_007_pos', 'snapshot_008_pos',
     'snapshot_009_pos', 'snapshot_010_pos', 'snapshot_011_pos',
     'snapshot_012_pos', 'snapshot_013_pos', 'snapshot_014_pos',
     'snapshot_017_pos', 'snapshot_018_pos']
 tags = ['functional', 'snapshot']
 
 [tests/functional/snapused]
 tests = ['snapused_001_pos', 'snapused_002_pos', 'snapused_003_pos',
     'snapused_004_pos', 'snapused_005_pos']
 tags = ['functional', 'snapused']
 
 [tests/functional/sparse]
 tests = ['sparse_001_pos']
 tags = ['functional', 'sparse']
 
 [tests/functional/stat]
 tests = ['stat_001_pos']
 tags = ['functional', 'stat']
 
 [tests/functional/suid]
 tests = ['suid_write_to_suid', 'suid_write_to_sgid', 'suid_write_to_suid_sgid',
     'suid_write_to_none', 'suid_write_zil_replay']
 tags = ['functional', 'suid']
 
 [tests/functional/trim]
 tests = ['autotrim_integrity', 'autotrim_config', 'autotrim_trim_integrity',
     'trim_integrity', 'trim_config', 'trim_l2arc']
 tags = ['functional', 'trim']
 
 [tests/functional/truncate]
 tests = ['truncate_001_pos', 'truncate_002_pos', 'truncate_timestamps']
 tags = ['functional', 'truncate']
 
 [tests/functional/upgrade]
 tests = ['upgrade_userobj_001_pos', 'upgrade_readonly_pool']
 tags = ['functional', 'upgrade']
 
 [tests/functional/userquota]
 tests = [
     'userquota_001_pos', 'userquota_002_pos', 'userquota_003_pos',
     'userquota_004_pos', 'userquota_005_neg', 'userquota_006_pos',
     'userquota_007_pos', 'userquota_008_pos', 'userquota_009_pos',
     'userquota_010_pos', 'userquota_011_pos', 'userquota_012_neg',
     'userspace_001_pos', 'userspace_002_pos', 'userspace_encrypted',
     'userspace_send_encrypted', 'userspace_encrypted_13709']
 tags = ['functional', 'userquota']
 
 [tests/functional/vdev_zaps]
 tests = ['vdev_zaps_001_pos', 'vdev_zaps_002_pos', 'vdev_zaps_003_pos',
     'vdev_zaps_004_pos', 'vdev_zaps_005_pos', 'vdev_zaps_006_pos',
     'vdev_zaps_007_pos']
 tags = ['functional', 'vdev_zaps']
 
 [tests/functional/write_dirs]
 tests = ['write_dirs_001_pos', 'write_dirs_002_pos']
 tags = ['functional', 'write_dirs']
 
 [tests/functional/xattr]
 tests = ['xattr_001_pos', 'xattr_002_neg', 'xattr_003_neg', 'xattr_004_pos',
     'xattr_005_pos', 'xattr_006_pos', 'xattr_007_neg',
     'xattr_011_pos', 'xattr_012_pos', 'xattr_013_pos', 'xattr_compat']
 tags = ['functional', 'xattr']
 
 [tests/functional/zvol/zvol_ENOSPC]
 tests = ['zvol_ENOSPC_001_pos']
 tags = ['functional', 'zvol', 'zvol_ENOSPC']
 
 [tests/functional/zvol/zvol_cli]
 tests = ['zvol_cli_001_pos', 'zvol_cli_002_pos', 'zvol_cli_003_neg']
 tags = ['functional', 'zvol', 'zvol_cli']
 
 [tests/functional/zvol/zvol_misc]
 tests = ['zvol_misc_002_pos', 'zvol_misc_hierarchy', 'zvol_misc_rename_inuse',
     'zvol_misc_snapdev', 'zvol_misc_trim', 'zvol_misc_volmode', 'zvol_misc_zil']
 tags = ['functional', 'zvol', 'zvol_misc']
 
 [tests/functional/zvol/zvol_stress]
 tests = ['zvol_stress']
 tags = ['functional', 'zvol', 'zvol_stress']
 
 [tests/functional/zvol/zvol_swap]
 tests = ['zvol_swap_001_pos', 'zvol_swap_002_pos', 'zvol_swap_004_pos']
 tags = ['functional', 'zvol', 'zvol_swap']
 
 [tests/functional/libzfs]
 tests = ['many_fds', 'libzfs_input']
 tags = ['functional', 'libzfs']
 
 [tests/functional/log_spacemap]
 tests = ['log_spacemap_import_logs']
 pre =
 post =
 tags = ['functional', 'log_spacemap']
 
 [tests/functional/l2arc]
 tests = ['l2arc_arcstats_pos', 'l2arc_mfuonly_pos', 'l2arc_l2miss_pos',
     'persist_l2arc_001_pos', 'persist_l2arc_002_pos',
     'persist_l2arc_003_neg', 'persist_l2arc_004_pos', 'persist_l2arc_005_pos']
 tags = ['functional', 'l2arc']
 
 [tests/functional/zpool_influxdb]
 tests = ['zpool_influxdb']
 tags = ['functional', 'zpool_influxdb']
diff --git a/sys/contrib/openzfs/tests/zfs-tests/cmd/file/largest_file.c b/sys/contrib/openzfs/tests/zfs-tests/cmd/file/largest_file.c
index d7252556b3cf..4cf04a7681eb 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/cmd/file/largest_file.c
+++ b/sys/contrib/openzfs/tests/zfs-tests/cmd/file/largest_file.c
@@ -1,147 +1,147 @@
 /*
  * CDDL HEADER START
  *
  * The contents of this file are subject to the terms of the
  * Common Development and Distribution License (the "License").
  * You may not use this file except in compliance with the License.
  *
  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  * or https://opensource.org/licenses/CDDL-1.0.
  * See the License for the specific language governing permissions
  * and limitations under the License.
  *
  * When distributing Covered Code, include this CDDL HEADER in each
  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  * If applicable, add the following below this CDDL HEADER, with the
  * fields enclosed by brackets "[]" replaced with your own identifying
  * information: Portions Copyright [yyyy] [name of copyright owner]
  *
  * CDDL HEADER END
  */
 
 /*
  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
 /*
  * Copyright (c) 2012 by Delphix. All rights reserved.
  */
 
 #include "file_common.h"
 #include <sys/param.h>
 #include <signal.h>
 #include <stdio.h>
 #include <string.h>
 #include <sys/stdtypes.h>
 #include <unistd.h>
 
 /*
  * --------------------------------------------------------------
  *
  *	Assertion:
  *		The last byte of the largest file size can be
  *		accessed without any errors.  Also, the writing
  *		beyond the last byte of the largest file size
  *		will produce an errno of EFBIG.
  *
  * --------------------------------------------------------------
  *	If the write() system call below returns a "1",
  *	then the last byte can be accessed.
  * --------------------------------------------------------------
  */
 static void	sigxfsz(int);
 static void	usage(char *);
 
 int
 main(int argc, char **argv)
 {
 	int		fd = 0;
 	offset_t	offset = (MAXOFFSET_T - 1);
 	offset_t	llseek_ret = 0;
 	int		write_ret = 0;
 	int		err = 0;
-	char		mybuf[5] = "aaaa\0";
+	char		mybuf[5] = "aaaa";
 	char		*testfile;
 	mode_t		mode = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
 	struct sigaction sa;
 
 	if (argc != 2) {
 		usage(argv[0]);
 	}
 
 	if (sigemptyset(&sa.sa_mask) == -1)
 		return (errno);
 	sa.sa_flags = 0;
 	sa.sa_handler = sigxfsz;
 	if (sigaction(SIGXFSZ, &sa, NULL) == -1)
 		return (errno);
 
 	testfile = strdup(argv[1]);
 	if (testfile == NULL)
 		return (errno);
 
 	fd = open(testfile, O_CREAT | O_RDWR, mode);
 	if (fd < 0) {
 		err = errno;
 		perror("Failed to create testfile");
 		free(testfile);
 		return (err);
 	}
 
 	llseek_ret = lseek64(fd, offset, SEEK_SET);
 	if (llseek_ret < 0) {
 		err = errno;
 		perror("Failed to seek to end of testfile");
 		goto out;
 	}
 
 	write_ret = write(fd, mybuf, 1);
 	if (write_ret < 0) {
 		err = errno;
 		perror("Failed to write to end of file");
 		goto out;
 	}
 
 	offset = 0;
 	llseek_ret = lseek64(fd, offset, SEEK_CUR);
 	if (llseek_ret < 0) {
 		err = errno;
 		perror("Failed to seek to end of file");
 		goto out;
 	}
 
 	write_ret = write(fd, mybuf, 1);
 	if (write_ret < 0) {
 		if (errno == EFBIG || errno == EINVAL) {
 			(void) printf("write errno=EFBIG|EINVAL: success\n");
 			err = 0;
 		} else {
 			err = errno;
 			perror("Did not receive EFBIG");
 		}
 	} else {
 		(void) printf("write completed successfully, test failed\n");
 		err = 1;
 	}
 
 out:
 	(void) unlink(testfile);
 	free(testfile);
 	close(fd);
 	return (err);
 }
 
 static void
 usage(char *name)
 {
 	(void) printf("%s <testfile>\n", name);
 	exit(1);
 }
 
 static void
 sigxfsz(int signo)
 {
 	(void) signo;
 	(void) printf("\nlargest_file: sigxfsz() caught SIGXFSZ\n");
 }
diff --git a/sys/contrib/openzfs/tests/zfs-tests/cmd/getversion.c b/sys/contrib/openzfs/tests/zfs-tests/cmd/getversion.c
index 62c1c5b6abc0..1e026b92d17d 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/cmd/getversion.c
+++ b/sys/contrib/openzfs/tests/zfs-tests/cmd/getversion.c
@@ -1,48 +1,48 @@
 /*
  * This file and its contents are supplied under the terms of the
  * Common Development and Distribution License ("CDDL"), version 1.0.
  * You may only use this file in accordance with the terms of version
  * 1.0 of the CDDL.
  *
  * A full copy of the text of the CDDL should have accompanied this
  * source.  A copy of the CDDL is also available via the Internet at
  * http://www.illumos.org/license/CDDL.
  */
 
 /*
  * Copyright 2021 iXsystems, Inc.
  */
 
 /*
  * FreeBSD and macOS expose file generation number through stat(2) and stat(1).
  * Linux exposes it instead through an ioctl.
  */
 
 #include <sys/ioctl.h>
-#include <sys/fcntl.h>
 #include <linux/fs.h>
 #include <err.h>
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
 
 int
 main(int argc, const char * const argv[])
 {
 	if (argc != 2)
 		errx(EXIT_FAILURE, "usage: %s filename", argv[0]);
 
 	int fd = open(argv[1], O_RDONLY);
 	if (fd == -1)
 		err(EXIT_FAILURE, "failed to open %s", argv[1]);
 
 	int gen = 0;
 	if (ioctl(fd, FS_IOC_GETVERSION, &gen) == -1)
 		err(EXIT_FAILURE, "FS_IOC_GETVERSION failed");
 
 	(void) close(fd);
 
 	(void) printf("%d\n", gen);
 
 	return (EXIT_SUCCESS);
 }
diff --git a/sys/contrib/openzfs/tests/zfs-tests/include/blkdev.shlib b/sys/contrib/openzfs/tests/zfs-tests/include/blkdev.shlib
index 6b83b10d604d..e9721bd9fc5e 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/include/blkdev.shlib
+++ b/sys/contrib/openzfs/tests/zfs-tests/include/blkdev.shlib
@@ -1,634 +1,623 @@
 #
 # This file and its contents are supplied under the terms of the
 # Common Development and Distribution License ("CDDL"), version 1.0.
 # You may only use this file in accordance with the terms of version
 # 1.0 of the CDDL.
 #
 # A full copy of the text of the CDDL should have accompanied this
 # source.  A copy of the CDDL is also available via the Internet at
 # http://www.illumos.org/license/CDDL.
 #
 
 #
 # Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 # Use is subject to license terms.
 # Copyright (c) 2012, 2019 by Delphix. All rights reserved.
 # Copyright 2016 Nexenta Systems, Inc.
 # Copyright (c) 2016, 2017 by Intel Corporation. All rights reserved.
 # Copyright (c) 2017 Lawrence Livermore National Security, LLC.
 # Copyright (c) 2017 Datto Inc.
 # Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
 # Copyright 2019 Richard Elling
 #
 
 #
 # Returns SCSI host number for the given disk
 #
 function get_scsi_host #disk
 {
 	typeset disk=$1
 	ls /sys/block/${disk}/device/scsi_device | cut -d : -f 1
 }
 
 #
 # Cause a scan of all scsi host adapters by default
 #
 # $1 optional host number
 #
 function scan_scsi_hosts
 {
 	typeset hostnum=${1}
 
 	if is_linux; then
 		if [[ -z $hostnum ]]; then
 			for host in /sys/class/scsi_host/host*; do
 				log_must eval "echo '- - -' > $host/scan"
 			done
 		else
 			log_note "/sys/class/scsi_host/host$hostnum/scan"
 			log_must eval \
 			    "echo '- - -' > /sys/class/scsi_host/host$hostnum/scan"
 		fi
 	fi
 }
 
 #
 # Wait for newly created block devices to have their minors created.
 # Additional arguments can be passed to udevadm trigger, with the expected
 # arguments to typically be a block device pathname. This is useful when
 # checking waiting on a specific device to settle rather than triggering
 # all devices and waiting for them all to settle.
 #
 # The udevadm settle timeout can be 120 or 180 seconds by default for
 # some distros. If a long delay is experienced, it could be due to some
 # strangeness in a malfunctioning device that isn't related to the devices
 # under test. To help debug this condition, a notice is given if settle takes
 # too long.
 #
 # Note: there is no meaningful return code if udevadm fails. Consumers
 # should not expect a return code (do not call as argument to log_must)
 #
 function block_device_wait
 {
 	if is_linux; then
 		udevadm trigger $* 2>/dev/null
 		typeset start=$SECONDS
 		udevadm settle
 		typeset elapsed=$((SECONDS - start))
 		[[ $elapsed > 60 ]] && \
 		    log_note udevadm settle time too long: $elapsed
 	elif is_freebsd; then
 		if [[ ${#@} -eq 0 ]]; then
 			# Do something that has to go through the geom event
 			# queue to complete.
 			sysctl kern.geom.conftxt >/dev/null
 			return
 		fi
 	fi
 	# Poll for the given paths to appear, but give up eventually.
 	typeset -i i
 	for (( i = 0; i < 5; ++i )); do
 		typeset missing=false
 		typeset dev
 		for dev in "${@}"; do
 			if ! [[ -e $dev ]]; then
 				missing=true
 				break
 			fi
 		done
 		if ! $missing; then
 			break
 		fi
 		sleep ${#@}
 	done
 }
 
 #
 # Check if the given device is physical device
 #
 function is_physical_device #device
 {
 	typeset device=${1#$DEV_DSKDIR/}
 	device=${device#$DEV_RDSKDIR/}
 
 	if is_linux; then
 		is_disk_device "$DEV_DSKDIR/$device" && \
 		[ -f /sys/module/loop/parameters/max_part ]
 	elif is_freebsd; then
 		is_disk_device "$DEV_DSKDIR/$device" && \
 		echo $device | grep -qE \
 		    -e '^a?da[0-9]+$' \
 		    -e '^md[0-9]+$' \
 		    -e '^mfid[0-9]+$' \
 		    -e '^nda[0-9]+$' \
 		    -e '^nvd[0-9]+$' \
 		    -e '^vtbd[0-9]+$'
 	else
 		echo $device | grep -qE "^c[0-F]+([td][0-F]+)+$"
 	fi
 }
 
 #
 # Check if the given device is a real device (ie SCSI device)
 #
 function is_real_device #disk
 {
 	typeset disk=$1
 	[[ -z $disk ]] && log_fail "No argument for disk given."
 
 	if is_linux; then
 		lsblk $DEV_RDSKDIR/$disk -o TYPE 2>/dev/null | \
 		    grep -q disk
 	fi
 }
 
 #
 # Check if the given device is a loop device
 #
 function is_loop_device #disk
 {
 	typeset disk=$1
 	[[ -z $disk ]] && log_fail "No argument for disk given."
 
 	if is_linux; then
 		lsblk $DEV_RDSKDIR/$disk -o TYPE 2>/dev/null | \
 		    grep -q loop
 	fi
 }
 
 #
 # Linux:
 # Check if the given device is a multipath device and if there is a symbolic
 # link to a device mapper and to a disk
 # Currently no support for dm devices alone without multipath
 #
 # FreeBSD:
 # Check if the given device is a gmultipath device.
 #
 # Others:
 # No multipath detection.
 #
 function is_mpath_device #disk
 {
 	typeset disk=$1
 	[[ -z $disk ]] && log_fail "No argument for disk given."
 
 	if is_linux; then
 		if lsblk $DEV_MPATHDIR/$disk -o TYPE 2>/dev/null | \
 		   grep -q mpath; then
 			readlink $DEV_MPATHDIR/$disk > /dev/null 2>&1
 		else
 			false
 		fi
 	elif is_freebsd; then
 		is_disk_device $DEV_MPATHDIR/$disk
 	else
 		false
 	fi
 }
 
 #
 # Check if the given path is the appropriate sort of device special node.
 #
 function is_disk_device #path
 {
 	typeset path=$1
 
 	if is_freebsd; then
 		# FreeBSD doesn't have block devices, only character devices.
 		test -c $path
 	else
 		test -b $path
 	fi
 }
 
 # Set the slice prefix for disk partitioning depending
 # on whether the device is a real, multipath, or loop device.
 # Currently all disks have to be of the same type, so only
 # checks first disk to determine slice prefix.
 #
 function set_slice_prefix
 {
 	typeset disk
 	typeset -i i=0
 
 	if is_linux; then
 		while (( i < $DISK_ARRAY_NUM )); do
 			disk="$(echo $DISKS | awk '{print $(i + 1)}')"
 			if is_mpath_device $disk && ! echo $disk | awk 'substr($1,18,1) ~ /^[[:digit:]]+$/ {exit 1}' || is_real_device $disk; then
 				export SLICE_PREFIX=""
 				return 0
 			elif is_mpath_device $disk || is_loop_device $disk; then
 				export SLICE_PREFIX="p"
 				return 0
 			else
 				log_fail "$disk not supported for partitioning."
 			fi
 			(( i = i + 1))
 		done
 	fi
 }
 
 #
 # Set the directory path of the listed devices in $DISK_ARRAY_NUM
 # Currently all disks have to be of the same type, so only
 # checks first disk to determine device directory
 # default = /dev (linux)
 # real disk = /dev (linux)
 # multipath device = /dev/mapper (linux)
 #
 function set_device_dir
 {
 	typeset disk
 	typeset -i i=0
 
 	if is_linux; then
 		while (( i < $DISK_ARRAY_NUM )); do
 			disk="$(echo $DISKS | awk '{print $(i + 1)}')"
 			if is_mpath_device $disk; then
 				export DEV_DSKDIR=$DEV_MPATHDIR
 				return 0
 			else
 				export DEV_DSKDIR=$DEV_RDSKDIR
 				return 0
 			fi
 			(( i = i + 1))
 		done
 	else
 		export DEV_DSKDIR=$DEV_RDSKDIR
 	fi
 }
 
 #
 # Get the directory path of given device
 #
 function get_device_dir #device
 {
 	typeset device=$1
 
 	if ! is_freebsd && ! is_physical_device $device; then
 		if [[ $device != "/" ]]; then
 			device=${device%/*}
 		fi
 		if is_disk_device "$DEV_DSKDIR/$device"; then
 			device="$DEV_DSKDIR"
 		fi
 		echo $device
 	else
 		echo "$DEV_DSKDIR"
 	fi
 }
 
 #
 # Get persistent name for given disk
 #
 function get_persistent_disk_name #device
 {
 	typeset device=$1
 
 	if is_linux; then
 		if is_real_device $device; then
 			udevadm info -q all -n $DEV_DSKDIR/$device \
 			    | awk '/disk\/by-id/ {print $2; exit}' | cut -d/ -f3
 		elif is_mpath_device $device; then
 			udevadm info -q all -n $DEV_DSKDIR/$device \
 			    | awk '/disk\/by-id\/dm-uuid/ {print $2; exit}' \
 			    | cut -d/ -f3
 		else
 			echo $device
 		fi
 	else
 		echo $device
 	fi
 }
 
 #
 # Online or offline a disk on the system
 #
 # First checks state of disk. Test will fail if disk is not properly onlined
 # or offlined. Online is a full rescan of SCSI disks by echoing to every
 # host entry.
 #
 function on_off_disk # disk state{online,offline} host
 {
 	typeset disk=$1
 	typeset state=$2
 	typeset host=$3
 
 	[[ -z $disk ]] || [[ -z $state ]] &&  \
 	    log_fail "Arguments invalid or missing"
 
 	if is_linux; then
 		if [[ $state == "offline" ]] && ( is_mpath_device $disk ); then
 			dm_name="$(readlink $DEV_DSKDIR/$disk | cut -d/ -f2)"
 			dep="$(ls /sys/block/${dm_name}/slaves | awk '{print $1}')"
 			while [[ -n $dep ]]; do
 				#check if disk is online
 				if lsscsi | grep -qF $dep; then
 					dep_dir="/sys/block/${dm_name}"
 					dep_dir+="/slaves/${dep}/device"
 					ss="${dep_dir}/state"
 					sd="${dep_dir}/delete"
 					log_must eval "echo 'offline' > ${ss}"
 					log_must eval "echo '1' > ${sd}"
 					if lsscsi | grep -qF $dep; then
 						log_fail "Offlining $disk failed"
 					fi
 				fi
 				dep="$(ls /sys/block/$dm_name/slaves 2>/dev/null | awk '{print $1}')"
 			done
 		elif [[ $state == "offline" ]] && ( is_real_device $disk ); then
 			#check if disk is online
 			if lsscsi | grep -qF $disk; then
 				dev_state="/sys/block/$disk/device/state"
 				dev_delete="/sys/block/$disk/device/delete"
 				log_must eval "echo 'offline' > ${dev_state}"
 				log_must eval "echo '1' > ${dev_delete}"
 				if lsscsi | grep -qF $disk; then
 					log_fail "Offlining $disk failed"
 				fi
 			else
 				log_note "$disk is already offline"
 			fi
 		elif [[ $state == "online" ]]; then
 			#force a full rescan
 			scan_scsi_hosts $host
 			block_device_wait
 			if is_mpath_device $disk; then
 				dm_name="$(readlink $DEV_DSKDIR/$disk | cut -d/ -f2)"
 				dep="$(ls /sys/block/$dm_name/slaves | awk '{print $1}')"
 				if lsscsi | grep -qF $dep; then
 					log_fail "Onlining $disk failed"
 				fi
 			elif is_real_device $disk; then
 				block_device_wait
 				typeset -i retries=0
 				while ! lsscsi | grep -qF $disk; do
 					if (( $retries > 2 )); then
 						log_fail "Onlining $disk failed"
 						break
 					fi
 					(( ++retries ))
 					sleep 1
 				done
 			else
 				log_fail "$disk is not a real dev"
 			fi
 		else
 			log_fail "$disk failed to $state"
 		fi
 	fi
 }
 
 #
 # Simulate disk removal
 #
 function remove_disk #disk
 {
 	typeset disk=$1
 	on_off_disk $disk "offline"
 	block_device_wait
 }
 
 #
 # Simulate disk insertion for the given SCSI host
 #
 function insert_disk #disk scsi_host
 {
 	typeset disk=$1
 	typeset scsi_host=$2
 	on_off_disk $disk "online" $scsi_host
 	block_device_wait
 }
 
 #
 # Load scsi_debug module with specified parameters
 # $blksz can be either one of: < 512b | 512e | 4Kn >
 #
 function load_scsi_debug # dev_size_mb add_host num_tgts max_luns blksz
 {
 	typeset devsize=$1
 	typeset hosts=$2
 	typeset tgts=$3
 	typeset luns=$4
 	typeset blksz=$5
 
 	[[ -z $devsize ]] || [[ -z $hosts ]] || [[ -z $tgts ]] || \
 	    [[ -z $luns ]] || [[ -z $blksz ]] && \
 	    log_fail "Arguments invalid or missing"
 
 	case "$5" in
 		'512b')
 			typeset sector=512
 			typeset blkexp=0
 		;;
 		'512e')
 			typeset sector=512
 			typeset blkexp=3
 		;;
 		'4Kn')
 			typeset sector=4096
 			typeset blkexp=0
 		;;
 		*) log_fail "Unsupported blksz value: $5" ;;
 	esac
 
 	if is_linux; then
 		modprobe -n scsi_debug ||
 			log_unsupported "Platform does not have scsi_debug module"
 		if lsmod | grep -q scsi_debug; then
 			log_fail "scsi_debug module already installed"
 		else
 			log_must modprobe scsi_debug dev_size_mb=$devsize \
 			    add_host=$hosts num_tgts=$tgts max_luns=$luns \
 			    sector_size=$sector physblk_exp=$blkexp
 			block_device_wait
 			if ! lsscsi | grep -q scsi_debug; then
 				log_fail "scsi_debug module install failed"
 			fi
 		fi
 	fi
 }
 
 #
 # Unload scsi_debug module, if needed.
 #
 function unload_scsi_debug
 {
 	log_must_retry "in use" 5 modprobe -r scsi_debug
 }
 
 #
 # Get scsi_debug device name.
 # Returns basename of scsi_debug device (for example "sdb").
 #
 function get_debug_device
 {
 	for i in {1..10} ; do
 		val=$(lsscsi | awk '/scsi_debug/ {print $6; exit}' | cut -d/ -f3)
 
 		# lsscsi can take time to settle
 		if [ "$val" != "-" ] ; then
 			break
 		fi
 		sleep 1
 	done
 	echo "$val"
 }
 
 #
 # Get actual devices used by the pool (i.e. linux sdb1 not sdb).
 #
 function get_pool_devices #testpool #devdir
 {
 	typeset testpool=$1
 	typeset devdir=$2
 	typeset out=""
 
 	case "$UNAME" in
 	Linux|FreeBSD)
 		zpool status -P $testpool | awk -v d="$devdir" '$1 ~ d {sub(d "/", ""); printf("%s ", $1)}'
 		;;
 	esac
 }
 
 #
 # Write to standard out giving the level, device name, offset and length
 # of all blocks in an input file. The offset and length are in units of
 # 512 byte blocks. In the case of mirrored vdevs, only the first
 # device is listed, as the levels, blocks and offsets will be the same
 # on other devices. Note that this function only works with mirrored
 # or non-redundant pools, not raidz.
 #
 # The output of this function can be used to introduce corruption at
 # varying levels of indirection.
 #
 function list_file_blocks # input_file
 {
 	typeset input_file=$1
 
 	[[ -f $input_file ]] || log_fail "Couldn't find $input_file"
 
 	typeset ds="$(zfs list -H -o name $input_file)"
 	typeset pool="${ds%%/*}"
 	typeset objnum="$(get_objnum $input_file)"
 
 	#
 	# Establish a mapping between vdev ids as shown in a DVA and the
 	# pathnames they correspond to in ${VDEV_MAP[][]}.
 	#
 	# The vdev bits in a DVA refer to the top level vdev id.
 	# ${VDEV_MAP[$id]} is an array of the vdev paths within that vdev.
 	#
 	eval $(zdb -C $pool | awk '
 	    BEGIN { printf "typeset -a VDEV_MAP;" }
 	    function subscript(s) {
 	        # "[#]" is more convenient than the bare "#"
 	        match(s, /\[[0-9]*\]/)
 		return substr(s, RSTART, RLENGTH)
 	    }
 	    id && !/^                / {
 	        # left a top level vdev
 	        id = 0
 	    }
 	    id && $1 ~ /^path:$/ {
 	        # found a vdev path; save it in the map
 	        printf "VDEV_MAP%s%s=%s;", id, child, $2
 	    }
 	    /^            children/ {
 	        # entering a top level vdev
 	        id = subscript($0)
 		child = "[0]" # default in case there is no nested vdev
 		printf "typeset -a VDEV_MAP%s;", id
 	    }
 	    /^                children/ {
 	        # entering a nested vdev (e.g. child of a top level mirror)
 	        child = subscript($0)
 	    }
 	')
 
 	#
 	# The awk below parses the output of zdb, printing out the level
 	# of each block along with vdev id, offset and length. The last
 	# two are converted to decimal in the while loop. 4M is added to
 	# the offset to compensate for the first two labels and boot
 	# block. Lastly, the offset and length are printed in units of
 	# 512B blocks for ease of use with dd.
 	#
 	typeset level vdev path offset length
-	if awk -n '' 2>/dev/null; then
-		# gawk needs -n to decode hex
-		AWK='awk -n'
-	else
-		AWK='awk'
-	fi
 	sync_all_pools true
-	zdb -dddddd $ds $objnum | $AWK -v pad=$((4<<20)) -v bs=512 '
+	zdb -dddddd $ds $objnum | awk '
 	    /^$/ { looking = 0 }
 	    looking {
 	        level = $2
 	        field = 3
 	        while (split($field, dva, ":") == 3) {
-	            # top level vdev id
-	            vdev = int(dva[1])
-	            # offset + 4M label/boot pad in 512B blocks
-	            offset = (int("0x"dva[2]) + pad) / bs
-		    # length in 512B blocks
-		    len = int("0x"dva[3]) / bs
 
-	            print level, vdev, offset, len
+	            print level, int(dva[1]), "0x"dva[2], "0x"dva[3]
 
 	            ++field
 	        }
 	    }
 	    /^Indirect blocks:/ { looking = 1 }
 	' | \
 	while read level vdev offset length; do
 		for path in ${VDEV_MAP[$vdev][@]}; do
-			echo "$level $path $offset $length"
+			echo "$level $path $(( ($offset + (4<<20)) / 512 ))" \
+			    "$(( $length / 512 ))"
 		done
 	done 2>/dev/null
 }
 
 function corrupt_blocks_at_level # input_file corrupt_level
 {
 	typeset input_file=$1
 	typeset corrupt_level="L${2:-0}"
 	typeset level path offset length
 
 	[[ -f $input_file ]] || log_fail "Couldn't find $input_file"
 
 	if is_freebsd; then
 		# Temporarily allow corrupting an inuse device.
 		debugflags=$(sysctl -n kern.geom.debugflags)
 		sysctl kern.geom.debugflags=16
 	fi
 
 	list_file_blocks $input_file | \
 	while read level path offset length; do
 		if [[ $level = $corrupt_level ]]; then
 			log_must dd if=/dev/urandom of=$path bs=512 \
 			    count=$length seek=$offset conv=notrunc
 		fi
 	done
 
 	if is_freebsd; then
 		sysctl kern.geom.debugflags=$debugflags
 	fi
 
 	# This is necessary for pools made of loop devices.
 	sync
 }
 
 function corrupt_label_checksum # label_number vdev_path
 {
 	typeset label_size=$((256*1024))
 	typeset vdev_size=$(stat_size ${2})
 	typeset -a offsets=("$((128*1024 - 32))" \
 	    "$(($label_size + (128*1024 - 32)))" \
 	    "$(($vdev_size - $label_size - (128*1024 + 32)))" \
 	    "$(($vdev_size - (128*1024 + 32)))")
 
 	dd if=/dev/urandom of=${2} seek=${offsets[$1]} bs=1 count=32 \
 	    conv=notrunc
 }
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am
index 2747e24cad9c..f9005769cff2 100644
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/Makefile.am
@@ -1,2113 +1,2114 @@
 CLEANFILES =
 dist_noinst_DATA =
 include $(top_srcdir)/config/Substfiles.am
 
 
 datadir_zfs_tests_testsdir = $(datadir)/$(PACKAGE)/zfs-tests/tests
 nobase_dist_datadir_zfs_tests_tests_DATA = \
 	perf/nfs-sample.cfg \
 	perf/perf.shlib \
 	\
 	perf/fio/mkfiles.fio \
 	perf/fio/random_reads.fio \
 	perf/fio/random_readwrite.fio \
 	perf/fio/random_readwrite_fixed.fio \
 	perf/fio/random_writes.fio \
 	perf/fio/sequential_reads.fio \
 	perf/fio/sequential_readwrite.fio \
 	perf/fio/sequential_writes.fio
 
 nobase_dist_datadir_zfs_tests_tests_SCRIPTS = \
 	perf/regression/random_reads.ksh \
 	perf/regression/random_readwrite.ksh \
 	perf/regression/random_readwrite_fixed.ksh \
 	perf/regression/random_writes.ksh \
 	perf/regression/random_writes_zil.ksh \
 	perf/regression/sequential_reads_arc_cached_clone.ksh \
 	perf/regression/sequential_reads_arc_cached.ksh \
 	perf/regression/sequential_reads_dbuf_cached.ksh \
 	perf/regression/sequential_reads.ksh \
 	perf/regression/sequential_writes.ksh \
 	perf/regression/setup.ksh \
 	\
 	perf/scripts/prefetch_io.sh
 
 # These lists can be regenerated by running make regen-tests at the root, or, on a *clean* source:
 #   find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '*.Po' ! -executable   -name '*.in'                                              | sort | sed 's/\.in$//;s/^/\t/;$!s/$/ \\/'
 #   find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '*.Po'   -executable   -name '*.in'                                              | sort | sed 's/\.in$//;s/^/\t/;$!s/$/ \\/'
 #   find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '*.Po'               ! -name '*.in' ! -name '*.c'  | grep  -Fe /simd -e /tmpfile | sort | sed           's/^/\t/;$!s/$/ \\/'
 #   find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '*.Po' ! -executable ! -name '*.in' ! -name '*.c'  | grep -vFe /simd -e /tmpfile | sort | sed           's/^/\t/;$!s/$/ \\/'
 #   find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '*.Po'   -executable ! -name '*.in' ! -name '*.c'  | grep -vFe /simd -e /tmpfile | sort | sed           's/^/\t/;$!s/$/ \\/'
 #
 # simd and tmpfile are Linux-only and not installed elsewhere
 #
 # C programs are specced in ../Makefile.am above as part of the main Makefile
 
 find_common := find functional/ ! -type d ! -name .gitignore ! -name .dirstamp ! -name '*.Po'
 regen:
 	@$(MAKE) -C $(top_builddir) clean
 	@$(MAKE) clean
 	$(SED) $(ac_inplace) '/^# -- >8 --/q' Makefile.am
 	echo >> Makefile.am
 	echo 'nobase_nodist_datadir_zfs_tests_tests_DATA = \' >> Makefile.am
 	$(find_common) ! -executable   -name '*.in'                                              | sort | sed 's/\.in$$//;s/^/\t/;$$!s/$$/ \\/' >> Makefile.am
 	echo 'nobase_nodist_datadir_zfs_tests_tests_SCRIPTS = \' >> Makefile.am
 	$(find_common)   -executable   -name '*.in'                                              | sort | sed 's/\.in$$//;s/^/\t/;$$!s/$$/ \\/' >> Makefile.am
 	echo >> Makefile.am
 	echo 'SUBSTFILES += $$(nobase_nodist_datadir_zfs_tests_tests_DATA) $$(nobase_nodist_datadir_zfs_tests_tests_SCRIPTS)' >> Makefile.am
 	echo >> Makefile.am
 	echo 'if BUILD_LINUX' >> Makefile.am
 	echo 'nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \' >> Makefile.am
 	$(find_common)               ! -name '*.in' ! -name '*.c'  | grep  -Fe /simd -e /tmpfile | sort | sed           's/^/\t/;$$!s/$$/ \\/' >> Makefile.am
 	echo 'endif' >> Makefile.am
 	echo >> Makefile.am
 	echo 'nobase_dist_datadir_zfs_tests_tests_DATA += \' >> Makefile.am
 	$(find_common) ! -executable ! -name '*.in' ! -name '*.c'  | grep -vFe /simd -e /tmpfile | sort | sed           's/^/\t/;$$!s/$$/ \\/' >> Makefile.am
 	echo >> Makefile.am
 	echo 'nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \' >> Makefile.am
 	$(find_common)   -executable ! -name '*.in' ! -name '*.c'  | grep -vFe /simd -e /tmpfile | sort | sed           's/^/\t/;$$!s/$$/ \\/' >> Makefile.am
 
 # -- >8 --
 
 nobase_nodist_datadir_zfs_tests_tests_DATA = \
 	functional/pam/utilities.kshlib
 nobase_nodist_datadir_zfs_tests_tests_SCRIPTS = \
 	functional/pyzfs/pyzfs_unittest.ksh
 
 SUBSTFILES += $(nobase_nodist_datadir_zfs_tests_tests_DATA) $(nobase_nodist_datadir_zfs_tests_tests_SCRIPTS)
 
 if BUILD_LINUX
 nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/simd/simd_supported.ksh \
 	functional/tmpfile/cleanup.ksh \
 	functional/tmpfile/setup.ksh \
 	functional/luks/luks_sanity.ksh
 endif
 
 nobase_dist_datadir_zfs_tests_tests_DATA += \
 	functional/acl/acl.cfg \
 	functional/acl/acl_common.kshlib \
 	functional/alloc_class/alloc_class.cfg \
 	functional/alloc_class/alloc_class.kshlib \
 	functional/atime/atime.cfg \
 	functional/atime/atime_common.kshlib \
 	functional/bclone/bclone.cfg \
 	functional/bclone/bclone_common.kshlib \
 	functional/bclone/bclone_corner_cases.kshlib \
 	functional/block_cloning/block_cloning.kshlib \
 	functional/cache/cache.cfg \
 	functional/cache/cache.kshlib \
 	functional/cachefile/cachefile.cfg \
 	functional/cachefile/cachefile.kshlib \
 	functional/casenorm/casenorm.cfg \
 	functional/casenorm/casenorm.kshlib \
 	functional/channel_program/channel_common.kshlib \
 	functional/channel_program/lua_core/tst.args_to_lua.out \
 	functional/channel_program/lua_core/tst.args_to_lua.zcp \
 	functional/channel_program/lua_core/tst.divide_by_zero.err \
 	functional/channel_program/lua_core/tst.divide_by_zero.zcp \
 	functional/channel_program/lua_core/tst.exists.zcp \
 	functional/channel_program/lua_core/tst.large_prog.out \
 	functional/channel_program/lua_core/tst.large_prog.zcp \
 	functional/channel_program/lua_core/tst.lib_base.lua \
 	functional/channel_program/lua_core/tst.lib_coroutine.lua \
 	functional/channel_program/lua_core/tst.lib_strings.lua \
 	functional/channel_program/lua_core/tst.lib_table.lua \
 	functional/channel_program/lua_core/tst.nested_neg.zcp \
 	functional/channel_program/lua_core/tst.nested_pos.zcp \
 	functional/channel_program/lua_core/tst.recursive.zcp \
 	functional/channel_program/lua_core/tst.return_large.zcp \
 	functional/channel_program/lua_core/tst.return_recursive_table.zcp \
 	functional/channel_program/lua_core/tst.stack_gsub.err \
 	functional/channel_program/lua_core/tst.stack_gsub.zcp \
 	functional/channel_program/lua_core/tst.timeout.zcp \
 	functional/channel_program/synctask_core/tst.bookmark.copy.zcp \
 	functional/channel_program/synctask_core/tst.bookmark.create.zcp \
 	functional/channel_program/synctask_core/tst.get_index_props.out \
 	functional/channel_program/synctask_core/tst.get_index_props.zcp \
 	functional/channel_program/synctask_core/tst.get_number_props.out \
 	functional/channel_program/synctask_core/tst.get_number_props.zcp \
 	functional/channel_program/synctask_core/tst.get_string_props.out \
 	functional/channel_program/synctask_core/tst.get_string_props.zcp \
 	functional/channel_program/synctask_core/tst.promote_conflict.zcp \
 	functional/channel_program/synctask_core/tst.set_props.zcp \
 	functional/channel_program/synctask_core/tst.snapshot_destroy.zcp \
 	functional/channel_program/synctask_core/tst.snapshot_neg.zcp \
 	functional/channel_program/synctask_core/tst.snapshot_recursive.zcp \
 	functional/channel_program/synctask_core/tst.snapshot_rename.zcp \
 	functional/channel_program/synctask_core/tst.snapshot_simple.zcp \
 	functional/checksum/default.cfg \
 	functional/clean_mirror/clean_mirror_common.kshlib \
 	functional/clean_mirror/default.cfg \
 	functional/cli_root/cli_common.kshlib \
 	functional/cli_root/zfs_copies/zfs_copies.cfg \
 	functional/cli_root/zfs_copies/zfs_copies.kshlib \
 	functional/cli_root/zfs_create/properties.kshlib \
 	functional/cli_root/zfs_create/zfs_create.cfg \
 	functional/cli_root/zfs_create/zfs_create_common.kshlib \
 	functional/cli_root/zfs_destroy/zfs_destroy.cfg \
 	functional/cli_root/zfs_destroy/zfs_destroy_common.kshlib \
 	functional/cli_root/zfs_get/zfs_get_common.kshlib \
 	functional/cli_root/zfs_get/zfs_get_list_d.kshlib \
 	functional/cli_root/zfs_jail/jail.conf \
 	functional/cli_root/zfs_load-key/HEXKEY \
 	functional/cli_root/zfs_load-key/PASSPHRASE \
 	functional/cli_root/zfs_load-key/RAWKEY \
 	functional/cli_root/zfs_load-key/zfs_load-key.cfg \
 	functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib \
 	functional/cli_root/zfs_mount/zfs_mount.cfg \
 	functional/cli_root/zfs_mount/zfs_mount.kshlib \
 	functional/cli_root/zfs_promote/zfs_promote.cfg \
 	functional/cli_root/zfs_receive/zstd_test_data.txt \
 	functional/cli_root/zfs_rename/zfs_rename.cfg \
 	functional/cli_root/zfs_rename/zfs_rename.kshlib \
 	functional/cli_root/zfs_rollback/zfs_rollback.cfg \
 	functional/cli_root/zfs_rollback/zfs_rollback_common.kshlib \
 	functional/cli_root/zfs_send/zfs_send.cfg \
 	functional/cli_root/zfs_set/zfs_set_common.kshlib \
 	functional/cli_root/zfs_share/zfs_share.cfg \
 	functional/cli_root/zfs_snapshot/zfs_snapshot.cfg \
 	functional/cli_root/zfs_unmount/zfs_unmount.cfg \
 	functional/cli_root/zfs_unmount/zfs_unmount.kshlib \
 	functional/cli_root/zfs_upgrade/zfs_upgrade.kshlib \
 	functional/cli_root/zfs_wait/zfs_wait.kshlib \
 	functional/cli_root/zpool_add/zpool_add.cfg \
 	functional/cli_root/zpool_add/zpool_add.kshlib \
 	functional/cli_root/zpool_clear/zpool_clear.cfg \
 	functional/cli_root/zpool_create/draidcfg.gz \
 	functional/cli_root/zpool_create/zpool_create.cfg \
 	functional/cli_root/zpool_create/zpool_create.shlib \
 	functional/cli_root/zpool_destroy/zpool_destroy.cfg \
 	functional/cli_root/zpool_events/zpool_events.cfg \
 	functional/cli_root/zpool_events/zpool_events.kshlib \
 	functional/cli_root/zpool_expand/zpool_expand.cfg \
 	functional/cli_root/zpool_export/zpool_export.cfg \
 	functional/cli_root/zpool_export/zpool_export.kshlib \
 	functional/cli_root/zpool_get/vdev_get.cfg \
 	functional/cli_root/zpool_get/zpool_get.cfg \
 	functional/cli_root/zpool_get/zpool_get_parsable.cfg \
 	functional/cli_root/zpool_import/blockfiles/cryptv0.dat.bz2 \
 	functional/cli_root/zpool_import/blockfiles/missing_ivset.dat.bz2 \
 	functional/cli_root/zpool_import/blockfiles/unclean_export.dat.bz2 \
 	functional/cli_root/zpool_import/zpool_import.cfg \
 	functional/cli_root/zpool_import/zpool_import.kshlib \
 	functional/cli_root/zpool_initialize/zpool_initialize.kshlib \
 	functional/cli_root/zpool_labelclear/labelclear.cfg \
 	functional/cli_root/zpool_remove/zpool_remove.cfg \
 	functional/cli_root/zpool_reopen/zpool_reopen.cfg \
 	functional/cli_root/zpool_reopen/zpool_reopen.shlib \
 	functional/cli_root/zpool_resilver/zpool_resilver.cfg \
 	functional/cli_root/zpool_scrub/zpool_scrub.cfg \
 	functional/cli_root/zpool_split/zpool_split.cfg \
 	functional/cli_root/zpool_trim/zpool_trim.kshlib \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-broken-mirror1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-broken-mirror2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v10.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v11.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v12.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v13.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v14.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v15.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1mirror1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1mirror2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1mirror3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1raidz1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1raidz2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1raidz3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1stripe1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1stripe2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v1stripe3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2mirror1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2mirror2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2mirror3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2raidz1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2raidz2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2raidz3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2stripe1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2stripe2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v2stripe3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3hotspare1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3hotspare2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3hotspare3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3mirror1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3mirror2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3mirror3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz21.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz22.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz23.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3raidz3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3stripe1.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3stripe2.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v3stripe3.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v4.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v5.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v6.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v7.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v8.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v999.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-v9.dat.bz2 \
 	functional/cli_root/zpool_upgrade/blockfiles/zfs-pool-vBROKEN.dat.bz2 \
 	functional/cli_root/zpool_upgrade/zpool_upgrade.cfg \
 	functional/cli_root/zpool_upgrade/zpool_upgrade.kshlib \
 	functional/cli_root/zpool_wait/zpool_wait.kshlib \
 	functional/cli_root/zhack/library.kshlib \
 	functional/cli_user/misc/misc.cfg \
 	functional/cli_user/zfs_list/zfs_list.cfg \
 	functional/cli_user/zfs_list/zfs_list.kshlib \
 	functional/compression/compress.cfg \
 	functional/compression/testpool_zstd.tar.gz \
 	functional/deadman/deadman.cfg \
 	functional/delegate/delegate.cfg \
 	functional/delegate/delegate_common.kshlib \
 	functional/devices/devices.cfg \
 	functional/devices/devices_common.kshlib \
 	functional/events/events.cfg \
 	functional/events/events_common.kshlib \
 	functional/fault/fault.cfg \
 	functional/grow/grow.cfg \
 	functional/history/history.cfg \
 	functional/history/history_common.kshlib \
 	functional/history/i386.migratedpool.DAT.Z \
 	functional/history/i386.orig_history.txt \
 	functional/history/sparc.migratedpool.DAT.Z \
 	functional/history/sparc.orig_history.txt \
 	functional/history/zfs-pool-v4.dat.Z \
 	functional/inheritance/config001.cfg \
 	functional/inheritance/config002.cfg \
 	functional/inheritance/config003.cfg \
 	functional/inheritance/config004.cfg \
 	functional/inheritance/config005.cfg \
 	functional/inheritance/config006.cfg \
 	functional/inheritance/config007.cfg \
 	functional/inheritance/config008.cfg \
 	functional/inheritance/config009.cfg \
 	functional/inheritance/config010.cfg \
 	functional/inheritance/config011.cfg \
 	functional/inheritance/config012.cfg \
 	functional/inheritance/config013.cfg \
 	functional/inheritance/config014.cfg \
 	functional/inheritance/config015.cfg \
 	functional/inheritance/config016.cfg \
 	functional/inheritance/config017.cfg \
 	functional/inheritance/config018.cfg \
 	functional/inheritance/config019.cfg \
 	functional/inheritance/config020.cfg \
 	functional/inheritance/config021.cfg \
 	functional/inheritance/config022.cfg \
 	functional/inheritance/config023.cfg \
 	functional/inheritance/config024.cfg \
 	functional/inheritance/inherit.kshlib \
 	functional/inheritance/README.config \
 	functional/inheritance/README.state \
 	functional/inheritance/state001.cfg \
 	functional/inheritance/state002.cfg \
 	functional/inheritance/state003.cfg \
 	functional/inheritance/state004.cfg \
 	functional/inheritance/state005.cfg \
 	functional/inheritance/state006.cfg \
 	functional/inheritance/state007.cfg \
 	functional/inheritance/state008.cfg \
 	functional/inheritance/state009.cfg \
 	functional/inheritance/state010.cfg \
 	functional/inheritance/state011.cfg \
 	functional/inheritance/state012.cfg \
 	functional/inheritance/state013.cfg \
 	functional/inheritance/state014.cfg \
 	functional/inheritance/state015.cfg \
 	functional/inheritance/state016.cfg \
 	functional/inheritance/state017.cfg \
 	functional/inheritance/state018.cfg \
 	functional/inheritance/state019.cfg \
 	functional/inheritance/state020.cfg \
 	functional/inheritance/state021.cfg \
 	functional/inheritance/state022.cfg \
 	functional/inheritance/state023.cfg \
 	functional/inheritance/state024.cfg \
 	functional/inuse/inuse.cfg \
 	functional/io/io.cfg \
 	functional/l2arc/l2arc.cfg \
 	functional/largest_pool/largest_pool.cfg \
 	functional/migration/migration.cfg \
 	functional/migration/migration.kshlib \
 	functional/mmap/mmap.cfg \
 	functional/mmp/mmp.cfg \
 	functional/mmp/mmp.kshlib \
 	functional/mv_files/mv_files.cfg \
 	functional/mv_files/mv_files_common.kshlib \
 	functional/nopwrite/nopwrite.shlib \
 	functional/no_space/enospc.cfg \
 	functional/online_offline/online_offline.cfg \
 	functional/pool_checkpoint/pool_checkpoint.kshlib \
 	functional/projectquota/projectquota.cfg \
 	functional/projectquota/projectquota_common.kshlib \
 	functional/quota/quota.cfg \
 	functional/quota/quota.kshlib \
 	functional/redacted_send/redacted.cfg \
 	functional/redacted_send/redacted.kshlib \
 	functional/redundancy/redundancy.cfg \
 	functional/redundancy/redundancy.kshlib \
 	functional/refreserv/refreserv.cfg \
 	functional/removal/removal.kshlib \
 	functional/replacement/replacement.cfg \
 	functional/reservation/reservation.cfg \
 	functional/reservation/reservation.shlib \
 	functional/rsend/dedup_encrypted_zvol.bz2 \
 	functional/rsend/dedup_encrypted_zvol.zsend.bz2 \
 	functional/rsend/dedup.zsend.bz2 \
 	functional/rsend/fs.tar.gz \
 	functional/rsend/rsend.cfg \
 	functional/rsend/rsend.kshlib \
 	functional/scrub_mirror/default.cfg \
 	functional/scrub_mirror/scrub_mirror_common.kshlib \
 	functional/slog/slog.cfg \
 	functional/slog/slog.kshlib \
 	functional/snapshot/snapshot.cfg \
 	functional/snapused/snapused.kshlib \
 	functional/sparse/sparse.cfg \
 	functional/trim/trim.cfg \
 	functional/trim/trim.kshlib \
 	functional/truncate/truncate.cfg \
 	functional/upgrade/upgrade_common.kshlib \
 	functional/user_namespace/user_namespace.cfg \
 	functional/user_namespace/user_namespace_common.kshlib \
 	functional/userquota/13709_reproducer.bz2 \
 	functional/userquota/userquota.cfg \
 	functional/userquota/userquota_common.kshlib \
 	functional/vdev_zaps/vdev_zaps.kshlib \
 	functional/xattr/xattr.cfg \
 	functional/xattr/xattr_common.kshlib \
 	functional/zvol/zvol.cfg \
 	functional/zvol/zvol_cli/zvol_cli.cfg \
 	functional/zvol/zvol_common.shlib \
 	functional/zvol/zvol_ENOSPC/zvol_ENOSPC.cfg \
 	functional/zvol/zvol_misc/zvol_misc_common.kshlib \
 	functional/zvol/zvol_swap/zvol_swap.cfg \
 	functional/idmap_mount/idmap_mount.cfg \
 	functional/idmap_mount/idmap_mount_common.kshlib
 
 nobase_dist_datadir_zfs_tests_tests_SCRIPTS += \
 	functional/acl/off/cleanup.ksh \
 	functional/acl/off/dosmode.ksh \
 	functional/acl/off/posixmode.ksh \
 	functional/acl/off/setup.ksh \
 	functional/acl/posix/cleanup.ksh \
 	functional/acl/posix/posix_001_pos.ksh \
 	functional/acl/posix/posix_002_pos.ksh \
 	functional/acl/posix/posix_003_pos.ksh \
 	functional/acl/posix/posix_004_pos.ksh \
 	functional/acl/posix-sa/cleanup.ksh \
 	functional/acl/posix-sa/posix_001_pos.ksh \
 	functional/acl/posix-sa/posix_002_pos.ksh \
 	functional/acl/posix-sa/posix_003_pos.ksh \
 	functional/acl/posix-sa/posix_004_pos.ksh \
 	functional/acl/posix-sa/setup.ksh \
 	functional/acl/posix/setup.ksh \
 	functional/alloc_class/alloc_class_001_pos.ksh \
 	functional/alloc_class/alloc_class_002_neg.ksh \
 	functional/alloc_class/alloc_class_003_pos.ksh \
 	functional/alloc_class/alloc_class_004_pos.ksh \
 	functional/alloc_class/alloc_class_005_pos.ksh \
 	functional/alloc_class/alloc_class_006_pos.ksh \
 	functional/alloc_class/alloc_class_007_pos.ksh \
 	functional/alloc_class/alloc_class_008_pos.ksh \
 	functional/alloc_class/alloc_class_009_pos.ksh \
 	functional/alloc_class/alloc_class_010_pos.ksh \
 	functional/alloc_class/alloc_class_011_neg.ksh \
 	functional/alloc_class/alloc_class_012_pos.ksh \
 	functional/alloc_class/alloc_class_013_pos.ksh \
 	functional/alloc_class/alloc_class_014_neg.ksh \
 	functional/alloc_class/alloc_class_015_pos.ksh \
 	functional/alloc_class/cleanup.ksh \
 	functional/alloc_class/setup.ksh \
 	functional/append/file_append.ksh \
 	functional/append/threadsappend_001_pos.ksh \
 	functional/append/cleanup.ksh \
 	functional/append/setup.ksh \
 	functional/arc/arcstats_runtime_tuning.ksh \
 	functional/arc/cleanup.ksh \
 	functional/arc/dbufstats_001_pos.ksh \
 	functional/arc/dbufstats_002_pos.ksh \
 	functional/arc/dbufstats_003_pos.ksh \
 	functional/arc/setup.ksh \
 	functional/atime/atime_001_pos.ksh \
 	functional/atime/atime_002_neg.ksh \
 	functional/atime/atime_003_pos.ksh \
 	functional/atime/cleanup.ksh \
 	functional/atime/root_atime_off.ksh \
 	functional/atime/root_atime_on.ksh \
 	functional/atime/root_relatime_on.ksh \
 	functional/atime/setup.ksh \
 	functional/bclone/bclone_crossfs_corner_cases.ksh \
 	functional/bclone/bclone_crossfs_corner_cases_limited.ksh \
 	functional/bclone/bclone_crossfs_data.ksh \
 	functional/bclone/bclone_crossfs_embedded.ksh \
 	functional/bclone/bclone_crossfs_hole.ksh \
 	functional/bclone/bclone_diffprops_all.ksh \
 	functional/bclone/bclone_diffprops_checksum.ksh \
 	functional/bclone/bclone_diffprops_compress.ksh \
 	functional/bclone/bclone_diffprops_copies.ksh \
 	functional/bclone/bclone_diffprops_recordsize.ksh \
 	functional/bclone/bclone_prop_sync.ksh \
 	functional/bclone/bclone_samefs_corner_cases.ksh \
 	functional/bclone/bclone_samefs_corner_cases_limited.ksh \
 	functional/bclone/bclone_samefs_data.ksh \
 	functional/bclone/bclone_samefs_embedded.ksh \
 	functional/bclone/bclone_samefs_hole.ksh \
 	functional/bclone/cleanup.ksh \
 	functional/bclone/setup.ksh \
 	functional/block_cloning/cleanup.ksh \
 	functional/block_cloning/setup.ksh \
 	functional/block_cloning/block_cloning_clone_mmap_cached.ksh \
 	functional/block_cloning/block_cloning_clone_mmap_write.ksh \
 	functional/block_cloning/block_cloning_copyfilerange_cross_dataset.ksh \
 	functional/block_cloning/block_cloning_copyfilerange_fallback.ksh \
 	functional/block_cloning/block_cloning_copyfilerange_fallback_same_txg.ksh \
 	functional/block_cloning/block_cloning_copyfilerange.ksh \
 	functional/block_cloning/block_cloning_copyfilerange_partial.ksh \
 	functional/block_cloning/block_cloning_disabled_copyfilerange.ksh \
 	functional/block_cloning/block_cloning_disabled_ficlone.ksh \
 	functional/block_cloning/block_cloning_disabled_ficlonerange.ksh \
 	functional/block_cloning/block_cloning_ficlone.ksh \
 	functional/block_cloning/block_cloning_ficlonerange.ksh \
 	functional/block_cloning/block_cloning_ficlonerange_partial.ksh \
 	functional/block_cloning/block_cloning_cross_enc_dataset.ksh \
 	functional/block_cloning/block_cloning_replay.ksh \
 	functional/block_cloning/block_cloning_replay_encrypted.ksh \
 	functional/block_cloning/block_cloning_lwb_buffer_overflow.ksh \
 	functional/block_cloning/block_cloning_rlimit_fsize.ksh \
 	functional/bootfs/bootfs_001_pos.ksh \
 	functional/bootfs/bootfs_002_neg.ksh \
 	functional/bootfs/bootfs_003_pos.ksh \
 	functional/bootfs/bootfs_004_neg.ksh \
 	functional/bootfs/bootfs_005_neg.ksh \
 	functional/bootfs/bootfs_006_pos.ksh \
 	functional/bootfs/bootfs_007_pos.ksh \
 	functional/bootfs/bootfs_008_pos.ksh \
 	functional/bootfs/cleanup.ksh \
 	functional/bootfs/setup.ksh \
 	functional/btree/btree_negative.ksh \
 	functional/btree/btree_positive.ksh \
 	functional/cache/cache_001_pos.ksh \
 	functional/cache/cache_002_pos.ksh \
 	functional/cache/cache_003_pos.ksh \
 	functional/cache/cache_004_neg.ksh \
 	functional/cache/cache_005_neg.ksh \
 	functional/cache/cache_006_pos.ksh \
 	functional/cache/cache_007_neg.ksh \
 	functional/cache/cache_008_neg.ksh \
 	functional/cache/cache_009_pos.ksh \
 	functional/cache/cache_010_pos.ksh \
 	functional/cache/cache_011_pos.ksh \
 	functional/cache/cache_012_pos.ksh \
 	functional/cache/cleanup.ksh \
 	functional/cachefile/cachefile_001_pos.ksh \
 	functional/cachefile/cachefile_002_pos.ksh \
 	functional/cachefile/cachefile_003_pos.ksh \
 	functional/cachefile/cachefile_004_pos.ksh \
 	functional/cachefile/cleanup.ksh \
 	functional/cachefile/setup.ksh \
 	functional/cache/setup.ksh \
 	functional/casenorm/case_all_values.ksh \
 	functional/casenorm/cleanup.ksh \
 	functional/casenorm/insensitive_formd_delete.ksh \
 	functional/casenorm/insensitive_formd_lookup.ksh \
 	functional/casenorm/insensitive_none_delete.ksh \
 	functional/casenorm/insensitive_none_lookup.ksh \
 	functional/casenorm/mixed_create_failure.ksh \
 	functional/casenorm/mixed_formd_delete.ksh \
 	functional/casenorm/mixed_formd_lookup_ci.ksh \
 	functional/casenorm/mixed_formd_lookup.ksh \
 	functional/casenorm/mixed_none_delete.ksh \
 	functional/casenorm/mixed_none_lookup_ci.ksh \
 	functional/casenorm/mixed_none_lookup.ksh \
 	functional/casenorm/norm_all_values.ksh \
 	functional/casenorm/sensitive_formd_delete.ksh \
 	functional/casenorm/sensitive_formd_lookup.ksh \
 	functional/casenorm/sensitive_none_delete.ksh \
 	functional/casenorm/sensitive_none_lookup.ksh \
 	functional/casenorm/setup.ksh \
 	functional/channel_program/lua_core/cleanup.ksh \
 	functional/channel_program/lua_core/setup.ksh \
 	functional/channel_program/lua_core/tst.args_to_lua.ksh \
 	functional/channel_program/lua_core/tst.divide_by_zero.ksh \
 	functional/channel_program/lua_core/tst.exists.ksh \
 	functional/channel_program/lua_core/tst.integer_illegal.ksh \
 	functional/channel_program/lua_core/tst.integer_overflow.ksh \
 	functional/channel_program/lua_core/tst.language_functions_neg.ksh \
 	functional/channel_program/lua_core/tst.language_functions_pos.ksh \
 	functional/channel_program/lua_core/tst.large_prog.ksh \
 	functional/channel_program/lua_core/tst.libraries.ksh \
 	functional/channel_program/lua_core/tst.memory_limit.ksh \
 	functional/channel_program/lua_core/tst.nested_neg.ksh \
 	functional/channel_program/lua_core/tst.nested_pos.ksh \
 	functional/channel_program/lua_core/tst.nvlist_to_lua.ksh \
 	functional/channel_program/lua_core/tst.recursive_neg.ksh \
 	functional/channel_program/lua_core/tst.recursive_pos.ksh \
 	functional/channel_program/lua_core/tst.return_large.ksh \
 	functional/channel_program/lua_core/tst.return_nvlist_neg.ksh \
 	functional/channel_program/lua_core/tst.return_nvlist_pos.ksh \
 	functional/channel_program/lua_core/tst.return_recursive_table.ksh \
 	functional/channel_program/lua_core/tst.stack_gsub.ksh \
 	functional/channel_program/lua_core/tst.timeout.ksh \
 	functional/channel_program/synctask_core/cleanup.ksh \
 	functional/channel_program/synctask_core/setup.ksh \
 	functional/channel_program/synctask_core/tst.bookmark.copy.ksh \
 	functional/channel_program/synctask_core/tst.bookmark.create.ksh \
 	functional/channel_program/synctask_core/tst.destroy_fs.ksh \
 	functional/channel_program/synctask_core/tst.destroy_snap.ksh \
 	functional/channel_program/synctask_core/tst.get_count_and_limit.ksh \
 	functional/channel_program/synctask_core/tst.get_index_props.ksh \
 	functional/channel_program/synctask_core/tst.get_mountpoint.ksh \
 	functional/channel_program/synctask_core/tst.get_neg.ksh \
 	functional/channel_program/synctask_core/tst.get_number_props.ksh \
 	functional/channel_program/synctask_core/tst.get_string_props.ksh \
 	functional/channel_program/synctask_core/tst.get_type.ksh \
 	functional/channel_program/synctask_core/tst.get_userquota.ksh \
 	functional/channel_program/synctask_core/tst.get_written.ksh \
 	functional/channel_program/synctask_core/tst.inherit.ksh \
 	functional/channel_program/synctask_core/tst.list_bookmarks.ksh \
 	functional/channel_program/synctask_core/tst.list_children.ksh \
 	functional/channel_program/synctask_core/tst.list_clones.ksh \
 	functional/channel_program/synctask_core/tst.list_holds.ksh \
 	functional/channel_program/synctask_core/tst.list_snapshots.ksh \
 	functional/channel_program/synctask_core/tst.list_system_props.ksh \
 	functional/channel_program/synctask_core/tst.list_user_props.ksh \
 	functional/channel_program/synctask_core/tst.parse_args_neg.ksh \
 	functional/channel_program/synctask_core/tst.promote_conflict.ksh \
 	functional/channel_program/synctask_core/tst.promote_multiple.ksh \
 	functional/channel_program/synctask_core/tst.promote_simple.ksh \
 	functional/channel_program/synctask_core/tst.rollback_mult.ksh \
 	functional/channel_program/synctask_core/tst.rollback_one.ksh \
 	functional/channel_program/synctask_core/tst.set_props.ksh \
 	functional/channel_program/synctask_core/tst.snapshot_destroy.ksh \
 	functional/channel_program/synctask_core/tst.snapshot_neg.ksh \
 	functional/channel_program/synctask_core/tst.snapshot_recursive.ksh \
 	functional/channel_program/synctask_core/tst.snapshot_rename.ksh \
 	functional/channel_program/synctask_core/tst.snapshot_simple.ksh \
 	functional/channel_program/synctask_core/tst.terminate_by_signal.ksh \
 	functional/chattr/chattr_001_pos.ksh \
 	functional/chattr/chattr_002_neg.ksh \
 	functional/chattr/cleanup.ksh \
 	functional/chattr/setup.ksh \
 	functional/checksum/cleanup.ksh \
 	functional/checksum/filetest_001_pos.ksh \
 	functional/checksum/filetest_002_pos.ksh \
 	functional/checksum/run_blake3_test.ksh \
 	functional/checksum/run_edonr_test.ksh \
 	functional/checksum/run_sha2_test.ksh \
 	functional/checksum/run_skein_test.ksh \
 	functional/checksum/setup.ksh \
 	functional/clean_mirror/clean_mirror_001_pos.ksh \
 	functional/clean_mirror/clean_mirror_002_pos.ksh \
 	functional/clean_mirror/clean_mirror_003_pos.ksh \
 	functional/clean_mirror/clean_mirror_004_pos.ksh \
 	functional/clean_mirror/cleanup.ksh \
 	functional/clean_mirror/setup.ksh \
 	functional/cli_root/zdb/zdb_002_pos.ksh \
 	functional/cli_root/zdb/zdb_003_pos.ksh \
 	functional/cli_root/zdb/zdb_004_pos.ksh \
 	functional/cli_root/zdb/zdb_005_pos.ksh \
 	functional/cli_root/zdb/zdb_006_pos.ksh \
 	functional/cli_root/zdb/zdb_args_neg.ksh \
 	functional/cli_root/zdb/zdb_args_pos.ksh \
 	functional/cli_root/zdb/zdb_backup.ksh \
 	functional/cli_root/zdb/zdb_block_size_histogram.ksh \
 	functional/cli_root/zdb/zdb_checksum.ksh \
 	functional/cli_root/zdb/zdb_decompress.ksh \
 	functional/cli_root/zdb/zdb_decompress_zstd.ksh \
 	functional/cli_root/zdb/zdb_display_block.ksh \
 	functional/cli_root/zdb/zdb_encrypted.ksh \
 	functional/cli_root/zdb/zdb_label_checksum.ksh \
 	functional/cli_root/zdb/zdb_object_range_neg.ksh \
 	functional/cli_root/zdb/zdb_object_range_pos.ksh \
 	functional/cli_root/zdb/zdb_objset_id.ksh \
 	functional/cli_root/zdb/zdb_recover_2.ksh \
 	functional/cli_root/zdb/zdb_recover.ksh \
 	functional/cli_root/zfs_bookmark/cleanup.ksh \
 	functional/cli_root/zfs_bookmark/setup.ksh \
 	functional/cli_root/zfs_bookmark/zfs_bookmark_cliargs.ksh \
 	functional/cli_root/zfs_change-key/cleanup.ksh \
 	functional/cli_root/zfs_change-key/setup.ksh \
 	functional/cli_root/zfs_change-key/zfs_change-key_child.ksh \
 	functional/cli_root/zfs_change-key/zfs_change-key_clones.ksh \
 	functional/cli_root/zfs_change-key/zfs_change-key_format.ksh \
 	functional/cli_root/zfs_change-key/zfs_change-key_inherit.ksh \
 	functional/cli_root/zfs_change-key/zfs_change-key.ksh \
 	functional/cli_root/zfs_change-key/zfs_change-key_load.ksh \
 	functional/cli_root/zfs_change-key/zfs_change-key_location.ksh \
 	functional/cli_root/zfs_change-key/zfs_change-key_pbkdf2iters.ksh \
 	functional/cli_root/zfs/cleanup.ksh \
 	functional/cli_root/zfs_clone/cleanup.ksh \
 	functional/cli_root/zfs_clone/setup.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_001_neg.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_002_pos.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_003_pos.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_004_pos.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_005_pos.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_006_pos.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_007_pos.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_008_neg.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_009_neg.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_010_pos.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_deeply_nested.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_encrypted.ksh \
 	functional/cli_root/zfs_clone/zfs_clone_rm_nested.ksh \
 	functional/cli_root/zfs_copies/cleanup.ksh \
 	functional/cli_root/zfs_copies/setup.ksh \
 	functional/cli_root/zfs_copies/zfs_copies_001_pos.ksh \
 	functional/cli_root/zfs_copies/zfs_copies_002_pos.ksh \
 	functional/cli_root/zfs_copies/zfs_copies_003_pos.ksh \
 	functional/cli_root/zfs_copies/zfs_copies_004_neg.ksh \
 	functional/cli_root/zfs_copies/zfs_copies_005_neg.ksh \
 	functional/cli_root/zfs_copies/zfs_copies_006_pos.ksh \
 	functional/cli_root/zfs_create/cleanup.ksh \
 	functional/cli_root/zfs_create/setup.ksh \
 	functional/cli_root/zfs_create/zfs_create_001_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_002_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_003_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_004_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_005_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_006_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_007_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_008_neg.ksh \
 	functional/cli_root/zfs_create/zfs_create_009_neg.ksh \
 	functional/cli_root/zfs_create/zfs_create_010_neg.ksh \
 	functional/cli_root/zfs_create/zfs_create_011_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_012_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_013_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_014_pos.ksh \
 	functional/cli_root/zfs_create/zfs_create_crypt_combos.ksh \
 	functional/cli_root/zfs_create/zfs_create_dryrun.ksh \
 	functional/cli_root/zfs_create/zfs_create_encrypted.ksh \
 	functional/cli_root/zfs_create/zfs_create_nomount.ksh \
 	functional/cli_root/zfs_create/zfs_create_verbose.ksh \
 	functional/cli_root/zfs_destroy/cleanup.ksh \
 	functional/cli_root/zfs_destroy/setup.ksh \
 	functional/cli_root/zfs_destroy/zfs_clone_livelist_condense_and_disable.ksh \
 	functional/cli_root/zfs_destroy/zfs_clone_livelist_condense_races.ksh \
 	functional/cli_root/zfs_destroy/zfs_clone_livelist_dedup.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_001_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_002_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_003_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_004_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_005_neg.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_006_neg.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_007_neg.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_008_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_009_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_010_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_011_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_012_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_013_neg.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_014_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_015_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_016_pos.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_clone_livelist.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_dev_removal_condense.ksh \
 	functional/cli_root/zfs_destroy/zfs_destroy_dev_removal.ksh \
 	functional/cli_root/zfs_diff/cleanup.ksh \
 	functional/cli_root/zfs_diff/setup.ksh \
 	functional/cli_root/zfs_diff/zfs_diff_changes.ksh \
 	functional/cli_root/zfs_diff/zfs_diff_cliargs.ksh \
 	functional/cli_root/zfs_diff/zfs_diff_encrypted.ksh \
 	functional/cli_root/zfs_diff/zfs_diff_mangle.ksh \
 	functional/cli_root/zfs_diff/zfs_diff_timestamp.ksh \
 	functional/cli_root/zfs_diff/zfs_diff_types.ksh \
 	functional/cli_root/zfs_get/cleanup.ksh \
 	functional/cli_root/zfs_get/setup.ksh \
 	functional/cli_root/zfs_get/zfs_get_001_pos.ksh \
 	functional/cli_root/zfs_get/zfs_get_002_pos.ksh \
 	functional/cli_root/zfs_get/zfs_get_003_pos.ksh \
 	functional/cli_root/zfs_get/zfs_get_004_pos.ksh \
 	functional/cli_root/zfs_get/zfs_get_005_neg.ksh \
 	functional/cli_root/zfs_get/zfs_get_006_neg.ksh \
 	functional/cli_root/zfs_get/zfs_get_007_neg.ksh \
 	functional/cli_root/zfs_get/zfs_get_008_pos.ksh \
 	functional/cli_root/zfs_get/zfs_get_009_pos.ksh \
 	functional/cli_root/zfs_get/zfs_get_010_neg.ksh \
 	functional/cli_root/zfs_ids_to_path/cleanup.ksh \
 	functional/cli_root/zfs_ids_to_path/setup.ksh \
 	functional/cli_root/zfs_ids_to_path/zfs_ids_to_path_001_pos.ksh \
 	functional/cli_root/zfs_inherit/cleanup.ksh \
 	functional/cli_root/zfs_inherit/setup.ksh \
 	functional/cli_root/zfs_inherit/zfs_inherit_001_neg.ksh \
 	functional/cli_root/zfs_inherit/zfs_inherit_002_neg.ksh \
 	functional/cli_root/zfs_inherit/zfs_inherit_003_pos.ksh \
 	functional/cli_root/zfs_inherit/zfs_inherit_mountpoint.ksh \
 	functional/cli_root/zfs_jail/cleanup.ksh \
 	functional/cli_root/zfs_jail/setup.ksh \
 	functional/cli_root/zfs_jail/zfs_jail_001_pos.ksh \
 	functional/cli_root/zfs_load-key/cleanup.ksh \
 	functional/cli_root/zfs_load-key/setup.ksh \
 	functional/cli_root/zfs_load-key/zfs_load-key_all.ksh \
 	functional/cli_root/zfs_load-key/zfs_load-key_file.ksh \
 	functional/cli_root/zfs_load-key/zfs_load-key_https.ksh \
 	functional/cli_root/zfs_load-key/zfs_load-key.ksh \
 	functional/cli_root/zfs_load-key/zfs_load-key_location.ksh \
 	functional/cli_root/zfs_load-key/zfs_load-key_noop.ksh \
 	functional/cli_root/zfs_load-key/zfs_load-key_recursive.ksh \
 	functional/cli_root/zfs_mount/cleanup.ksh \
 	functional/cli_root/zfs_mount/setup.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_001_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_002_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_003_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_004_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_005_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_006_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_007_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_008_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_009_neg.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_010_neg.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_011_neg.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_012_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_013_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_014_neg.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_all_001_pos.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_all_fail.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_all_mountpoints.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_encrypted.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_recursive.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_remount.ksh \
 	functional/cli_root/zfs_mount/zfs_mount_test_race.ksh \
 	functional/cli_root/zfs_mount/zfs_multi_mount.ksh \
 	functional/cli_root/zfs_program/cleanup.ksh \
 	functional/cli_root/zfs_program/setup.ksh \
 	functional/cli_root/zfs_program/zfs_program_json.ksh \
 	functional/cli_root/zfs_promote/cleanup.ksh \
 	functional/cli_root/zfs_promote/setup.ksh \
 	functional/cli_root/zfs_promote/zfs_promote_001_pos.ksh \
 	functional/cli_root/zfs_promote/zfs_promote_002_pos.ksh \
 	functional/cli_root/zfs_promote/zfs_promote_003_pos.ksh \
 	functional/cli_root/zfs_promote/zfs_promote_004_pos.ksh \
 	functional/cli_root/zfs_promote/zfs_promote_005_pos.ksh \
 	functional/cli_root/zfs_promote/zfs_promote_006_neg.ksh \
 	functional/cli_root/zfs_promote/zfs_promote_007_neg.ksh \
 	functional/cli_root/zfs_promote/zfs_promote_008_pos.ksh \
 	functional/cli_root/zfs_promote/zfs_promote_encryptionroot.ksh \
 	functional/cli_root/zfs_property/cleanup.ksh \
 	functional/cli_root/zfs_property/setup.ksh \
 	functional/cli_root/zfs_property/zfs_written_property_001_pos.ksh \
 	functional/cli_root/zfs_receive/cleanup.ksh \
 	functional/cli_root/zfs_receive/receive-o-x_props_aliases.ksh \
 	functional/cli_root/zfs_receive/receive-o-x_props_override.ksh \
 	functional/cli_root/zfs_receive/setup.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_001_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_002_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_003_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_004_neg.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_005_neg.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_006_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_007_neg.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_008_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_009_neg.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_010_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_011_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_012_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_013_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_014_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_015_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_016_pos.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_-e.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_from_encrypted.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_from_zstd.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_new_props.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_raw_-d.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_raw_incremental.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_raw.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_to_encrypted.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_-wR-encrypted-mix.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_corrective.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_compressed_corrective.ksh \
 	functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh \
 	functional/cli_root/zfs_rename/cleanup.ksh \
 	functional/cli_root/zfs_rename/setup.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_001_pos.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_002_pos.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_003_pos.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_004_neg.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_005_neg.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_006_pos.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_007_pos.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_008_pos.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_009_neg.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_010_neg.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_011_pos.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_012_neg.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_013_pos.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_014_neg.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_encrypted_child.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_mountpoint.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_nounmount.ksh \
 	functional/cli_root/zfs_rename/zfs_rename_to_encrypted.ksh \
 	functional/cli_root/zfs_reservation/cleanup.ksh \
 	functional/cli_root/zfs_reservation/setup.ksh \
 	functional/cli_root/zfs_reservation/zfs_reservation_001_pos.ksh \
 	functional/cli_root/zfs_reservation/zfs_reservation_002_pos.ksh \
 	functional/cli_root/zfs_rollback/cleanup.ksh \
 	functional/cli_root/zfs_rollback/setup.ksh \
 	functional/cli_root/zfs_rollback/zfs_rollback_001_pos.ksh \
 	functional/cli_root/zfs_rollback/zfs_rollback_002_pos.ksh \
 	functional/cli_root/zfs_rollback/zfs_rollback_003_neg.ksh \
 	functional/cli_root/zfs_rollback/zfs_rollback_004_neg.ksh \
 	functional/cli_root/zfs_send/cleanup.ksh \
 	functional/cli_root/zfs_send/setup.ksh \
 	functional/cli_root/zfs_send/zfs_send_001_pos.ksh \
 	functional/cli_root/zfs_send/zfs_send_002_pos.ksh \
 	functional/cli_root/zfs_send/zfs_send_003_pos.ksh \
 	functional/cli_root/zfs_send/zfs_send_004_neg.ksh \
 	functional/cli_root/zfs_send/zfs_send_005_pos.ksh \
 	functional/cli_root/zfs_send/zfs_send_006_pos.ksh \
 	functional/cli_root/zfs_send/zfs_send_007_pos.ksh \
 	functional/cli_root/zfs_send/zfs_send-b.ksh \
 	functional/cli_root/zfs_send/zfs_send_encrypted.ksh \
 	functional/cli_root/zfs_send/zfs_send_encrypted_unloaded.ksh \
 	functional/cli_root/zfs_send/zfs_send_raw.ksh \
 	functional/cli_root/zfs_send/zfs_send_skip_missing.ksh \
 	functional/cli_root/zfs_send/zfs_send_sparse.ksh \
 	functional/cli_root/zfs_set/cache_001_pos.ksh \
 	functional/cli_root/zfs_set/cache_002_neg.ksh \
 	functional/cli_root/zfs_set/canmount_001_pos.ksh \
 	functional/cli_root/zfs_set/canmount_002_pos.ksh \
 	functional/cli_root/zfs_set/canmount_003_pos.ksh \
 	functional/cli_root/zfs_set/canmount_004_pos.ksh \
 	functional/cli_root/zfs_set/checksum_001_pos.ksh \
 	functional/cli_root/zfs_set/cleanup.ksh \
 	functional/cli_root/zfs_set/compression_001_pos.ksh \
 	functional/cli_root/zfs_set/mountpoint_001_pos.ksh \
 	functional/cli_root/zfs_set/mountpoint_002_pos.ksh \
 	functional/cli_root/zfs_set/mountpoint_003_pos.ksh \
 	functional/cli_root/zfs_set/onoffs_001_pos.ksh \
 	functional/cli_root/zfs_set/property_alias_001_pos.ksh \
 	functional/cli_root/zfs_set/readonly_001_pos.ksh \
 	functional/cli_root/zfs_set/reservation_001_neg.ksh \
 	functional/cli_root/zfs_set/ro_props_001_pos.ksh \
 	functional/cli_root/zfs_set/setup.ksh \
 	functional/cli_root/zfs_set/share_mount_001_neg.ksh \
 	functional/cli_root/zfs_set/snapdir_001_pos.ksh \
 	functional/cli_root/zfs/setup.ksh \
 	functional/cli_root/zfs_set/user_property_001_pos.ksh \
 	functional/cli_root/zfs_set/user_property_002_pos.ksh \
 	functional/cli_root/zfs_set/user_property_003_neg.ksh \
 	functional/cli_root/zfs_set/user_property_004_pos.ksh \
 	functional/cli_root/zfs_set/version_001_neg.ksh \
 	functional/cli_root/zfs_set/zfs_set_001_neg.ksh \
 	functional/cli_root/zfs_set/zfs_set_002_neg.ksh \
 	functional/cli_root/zfs_set/zfs_set_003_neg.ksh \
 	functional/cli_root/zfs_set/zfs_set_feature_activation.ksh \
 	functional/cli_root/zfs_set/zfs_set_keylocation.ksh \
 	functional/cli_root/zfs_set/zfs_set_nomount.ksh \
 	functional/cli_root/zfs_share/cleanup.ksh \
 	functional/cli_root/zfs_share/setup.ksh \
 	functional/cli_root/zfs_share/zfs_share_001_pos.ksh \
 	functional/cli_root/zfs_share/zfs_share_002_pos.ksh \
 	functional/cli_root/zfs_share/zfs_share_003_pos.ksh \
 	functional/cli_root/zfs_share/zfs_share_004_pos.ksh \
 	functional/cli_root/zfs_share/zfs_share_005_pos.ksh \
 	functional/cli_root/zfs_share/zfs_share_006_pos.ksh \
 	functional/cli_root/zfs_share/zfs_share_007_neg.ksh \
 	functional/cli_root/zfs_share/zfs_share_008_neg.ksh \
 	functional/cli_root/zfs_share/zfs_share_009_neg.ksh \
 	functional/cli_root/zfs_share/zfs_share_010_neg.ksh \
 	functional/cli_root/zfs_share/zfs_share_011_pos.ksh \
 	functional/cli_root/zfs_share/zfs_share_012_pos.ksh \
 	functional/cli_root/zfs_share/zfs_share_013_pos.ksh \
 	functional/cli_root/zfs_share/zfs_share_concurrent_shares.ksh \
 	functional/cli_root/zfs_share/zfs_share_after_mount.ksh \
 	functional/cli_root/zfs_snapshot/cleanup.ksh \
 	functional/cli_root/zfs_snapshot/setup.ksh \
 	functional/cli_root/zfs_snapshot/zfs_snapshot_001_neg.ksh \
 	functional/cli_root/zfs_snapshot/zfs_snapshot_002_neg.ksh \
 	functional/cli_root/zfs_snapshot/zfs_snapshot_003_neg.ksh \
 	functional/cli_root/zfs_snapshot/zfs_snapshot_004_neg.ksh \
 	functional/cli_root/zfs_snapshot/zfs_snapshot_005_neg.ksh \
 	functional/cli_root/zfs_snapshot/zfs_snapshot_006_pos.ksh \
 	functional/cli_root/zfs_snapshot/zfs_snapshot_007_neg.ksh \
 	functional/cli_root/zfs_snapshot/zfs_snapshot_008_neg.ksh \
 	functional/cli_root/zfs_snapshot/zfs_snapshot_009_pos.ksh \
 	functional/cli_root/zfs_sysfs/cleanup.ksh \
 	functional/cli_root/zfs_sysfs/setup.ksh \
 	functional/cli_root/zfs_sysfs/zfeature_set_unsupported.ksh \
 	functional/cli_root/zfs_sysfs/zfs_get_unsupported.ksh \
 	functional/cli_root/zfs_sysfs/zfs_set_unsupported.ksh \
 	functional/cli_root/zfs_sysfs/zfs_sysfs_live.ksh \
 	functional/cli_root/zfs_sysfs/zpool_get_unsupported.ksh \
 	functional/cli_root/zfs_sysfs/zpool_set_unsupported.ksh \
 	functional/cli_root/zfs_unload-key/cleanup.ksh \
 	functional/cli_root/zfs_unload-key/setup.ksh \
 	functional/cli_root/zfs_unload-key/zfs_unload-key_all.ksh \
 	functional/cli_root/zfs_unload-key/zfs_unload-key.ksh \
 	functional/cli_root/zfs_unload-key/zfs_unload-key_recursive.ksh \
 	functional/cli_root/zfs_unmount/cleanup.ksh \
 	functional/cli_root/zfs_unmount/setup.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_001_pos.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_002_pos.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_003_pos.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_004_pos.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_005_pos.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_006_pos.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_007_neg.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_008_neg.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_009_pos.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_all_001_pos.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_nested.ksh \
 	functional/cli_root/zfs_unmount/zfs_unmount_unload_keys.ksh \
 	functional/cli_root/zfs_unshare/cleanup.ksh \
 	functional/cli_root/zfs_unshare/setup.ksh \
 	functional/cli_root/zfs_unshare/zfs_unshare_001_pos.ksh \
 	functional/cli_root/zfs_unshare/zfs_unshare_002_pos.ksh \
 	functional/cli_root/zfs_unshare/zfs_unshare_003_pos.ksh \
 	functional/cli_root/zfs_unshare/zfs_unshare_004_neg.ksh \
 	functional/cli_root/zfs_unshare/zfs_unshare_005_neg.ksh \
 	functional/cli_root/zfs_unshare/zfs_unshare_006_pos.ksh \
 	functional/cli_root/zfs_unshare/zfs_unshare_007_pos.ksh \
 	functional/cli_root/zfs_unshare/zfs_unshare_008_pos.ksh \
 	functional/cli_root/zfs_upgrade/cleanup.ksh \
 	functional/cli_root/zfs_upgrade/setup.ksh \
 	functional/cli_root/zfs_upgrade/zfs_upgrade_001_pos.ksh \
 	functional/cli_root/zfs_upgrade/zfs_upgrade_002_pos.ksh \
 	functional/cli_root/zfs_upgrade/zfs_upgrade_003_pos.ksh \
 	functional/cli_root/zfs_upgrade/zfs_upgrade_004_pos.ksh \
 	functional/cli_root/zfs_upgrade/zfs_upgrade_005_pos.ksh \
 	functional/cli_root/zfs_upgrade/zfs_upgrade_006_neg.ksh \
 	functional/cli_root/zfs_upgrade/zfs_upgrade_007_neg.ksh \
 	functional/cli_root/zfs_wait/cleanup.ksh \
 	functional/cli_root/zfs_wait/setup.ksh \
 	functional/cli_root/zfs_wait/zfs_wait_deleteq.ksh \
 	functional/cli_root/zfs_wait/zfs_wait_getsubopt.ksh \
 	functional/cli_root/zfs/zfs_001_neg.ksh \
 	functional/cli_root/zfs/zfs_002_pos.ksh \
 	functional/cli_root/zfs/zfs_003_neg.ksh \
 	functional/cli_root/zhack/zhack_label_repair_001.ksh \
 	functional/cli_root/zhack/zhack_label_repair_002.ksh \
 	functional/cli_root/zhack/zhack_label_repair_003.ksh \
 	functional/cli_root/zhack/zhack_label_repair_004.ksh \
 	functional/cli_root/zpool_add/add_nested_replacing_spare.ksh \
 	functional/cli_root/zpool_add/add-o_ashift.ksh \
 	functional/cli_root/zpool_add/add_prop_ashift.ksh \
 	functional/cli_root/zpool_add/cleanup.ksh \
 	functional/cli_root/zpool_add/setup.ksh \
 	functional/cli_root/zpool_add/zpool_add--allow-ashift-mismatch.ksh \
 	functional/cli_root/zpool_add/zpool_add_001_pos.ksh \
 	functional/cli_root/zpool_add/zpool_add_002_pos.ksh \
 	functional/cli_root/zpool_add/zpool_add_003_pos.ksh \
 	functional/cli_root/zpool_add/zpool_add_004_pos.ksh \
 	functional/cli_root/zpool_add/zpool_add_005_pos.ksh \
 	functional/cli_root/zpool_add/zpool_add_006_pos.ksh \
 	functional/cli_root/zpool_add/zpool_add_007_neg.ksh \
 	functional/cli_root/zpool_add/zpool_add_008_neg.ksh \
 	functional/cli_root/zpool_add/zpool_add_009_neg.ksh \
 	functional/cli_root/zpool_add/zpool_add_010_pos.ksh \
 	functional/cli_root/zpool_add/zpool_add_dryrun_output.ksh \
 	functional/cli_root/zpool_attach/attach-o_ashift.ksh \
 	functional/cli_root/zpool_attach/cleanup.ksh \
 	functional/cli_root/zpool_attach/setup.ksh \
 	functional/cli_root/zpool_attach/zpool_attach_001_neg.ksh \
 	functional/cli_root/zpool/cleanup.ksh \
 	functional/cli_root/zpool_clear/cleanup.ksh \
 	functional/cli_root/zpool_clear/setup.ksh \
 	functional/cli_root/zpool_clear/zpool_clear_001_pos.ksh \
 	functional/cli_root/zpool_clear/zpool_clear_002_neg.ksh \
 	functional/cli_root/zpool_clear/zpool_clear_003_neg.ksh \
 	functional/cli_root/zpool_clear/zpool_clear_readonly.ksh \
 	functional/cli_root/zpool_create/cleanup.ksh \
 	functional/cli_root/zpool_create/create-o_ashift.ksh \
 	functional/cli_root/zpool_create/setup.ksh \
 	functional/cli_root/zpool_create/zpool_create_001_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_002_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_003_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_004_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_005_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_006_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_007_neg.ksh \
 	functional/cli_root/zpool_create/zpool_create_008_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_009_neg.ksh \
 	functional/cli_root/zpool_create/zpool_create_010_neg.ksh \
 	functional/cli_root/zpool_create/zpool_create_011_neg.ksh \
 	functional/cli_root/zpool_create/zpool_create_012_neg.ksh \
 	functional/cli_root/zpool_create/zpool_create_014_neg.ksh \
 	functional/cli_root/zpool_create/zpool_create_015_neg.ksh \
 	functional/cli_root/zpool_create/zpool_create_016_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_017_neg.ksh \
 	functional/cli_root/zpool_create/zpool_create_018_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_019_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_020_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_021_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_022_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_023_neg.ksh \
 	functional/cli_root/zpool_create/zpool_create_024_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_crypt_combos.ksh \
 	functional/cli_root/zpool_create/zpool_create_draid_001_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_draid_002_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_draid_003_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_draid_004_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_dryrun_output.ksh \
 	functional/cli_root/zpool_create/zpool_create_encrypted.ksh \
 	functional/cli_root/zpool_create/zpool_create_features_001_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_features_002_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_features_003_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_features_004_neg.ksh \
 	functional/cli_root/zpool_create/zpool_create_features_005_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_features_006_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_features_007_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_features_008_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_features_009_pos.ksh \
 	functional/cli_root/zpool_create/zpool_create_tempname.ksh \
 	functional/cli_root/zpool_destroy/zpool_destroy_001_pos.ksh \
 	functional/cli_root/zpool_destroy/zpool_destroy_002_pos.ksh \
 	functional/cli_root/zpool_destroy/zpool_destroy_003_neg.ksh \
 	functional/cli_root/zpool_detach/cleanup.ksh \
 	functional/cli_root/zpool_detach/setup.ksh \
 	functional/cli_root/zpool_detach/zpool_detach_001_neg.ksh \
 	functional/cli_root/zpool_events/cleanup.ksh \
 	functional/cli_root/zpool_events/setup.ksh \
 	functional/cli_root/zpool_events/zpool_events_clear.ksh \
 	functional/cli_root/zpool_events/zpool_events_clear_retained.ksh \
 	functional/cli_root/zpool_events/zpool_events_cliargs.ksh \
 	functional/cli_root/zpool_events/zpool_events_duplicates.ksh \
 	functional/cli_root/zpool_events/zpool_events_errors.ksh \
 	functional/cli_root/zpool_events/zpool_events_follow.ksh \
 	functional/cli_root/zpool_events/zpool_events_poolname.ksh \
 	functional/cli_root/zpool_expand/cleanup.ksh \
 	functional/cli_root/zpool_expand/setup.ksh \
 	functional/cli_root/zpool_expand/zpool_expand_001_pos.ksh \
 	functional/cli_root/zpool_expand/zpool_expand_002_pos.ksh \
 	functional/cli_root/zpool_expand/zpool_expand_003_neg.ksh \
 	functional/cli_root/zpool_expand/zpool_expand_004_pos.ksh \
 	functional/cli_root/zpool_expand/zpool_expand_005_pos.ksh \
 	functional/cli_root/zpool_export/cleanup.ksh \
 	functional/cli_root/zpool_export/setup.ksh \
 	functional/cli_root/zpool_export/zpool_export_001_pos.ksh \
 	functional/cli_root/zpool_export/zpool_export_002_pos.ksh \
 	functional/cli_root/zpool_export/zpool_export_003_neg.ksh \
 	functional/cli_root/zpool_export/zpool_export_004_pos.ksh \
 	functional/cli_root/zpool_get/cleanup.ksh \
 	functional/cli_root/zpool_get/setup.ksh \
 	functional/cli_root/zpool_get/vdev_get_001_pos.ksh \
 	functional/cli_root/zpool_get/zpool_get_001_pos.ksh \
 	functional/cli_root/zpool_get/zpool_get_002_pos.ksh \
 	functional/cli_root/zpool_get/zpool_get_003_pos.ksh \
 	functional/cli_root/zpool_get/zpool_get_004_neg.ksh \
 	functional/cli_root/zpool_get/zpool_get_005_pos.ksh \
 	functional/cli_root/zpool_history/cleanup.ksh \
 	functional/cli_root/zpool_history/setup.ksh \
 	functional/cli_root/zpool_history/zpool_history_001_neg.ksh \
 	functional/cli_root/zpool_history/zpool_history_002_pos.ksh \
 	functional/cli_root/zpool_import/cleanup.ksh \
 	functional/cli_root/zpool_import/import_cachefile_device_added.ksh \
 	functional/cli_root/zpool_import/import_cachefile_device_removed.ksh \
 	functional/cli_root/zpool_import/import_cachefile_device_replaced.ksh \
 	functional/cli_root/zpool_import/import_cachefile_mirror_attached.ksh \
 	functional/cli_root/zpool_import/import_cachefile_mirror_detached.ksh \
 	functional/cli_root/zpool_import/import_cachefile_paths_changed.ksh \
 	functional/cli_root/zpool_import/import_cachefile_shared_device.ksh \
 	functional/cli_root/zpool_import/import_devices_missing.ksh \
 	functional/cli_root/zpool_import/import_log_missing.ksh \
 	functional/cli_root/zpool_import/import_paths_changed.ksh \
 	functional/cli_root/zpool_import/import_rewind_config_changed.ksh \
 	functional/cli_root/zpool_import/import_rewind_device_replaced.ksh \
 	functional/cli_root/zpool_import/setup.ksh \
 	functional/cli_root/zpool_import/zpool_import_001_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_002_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_003_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_004_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_005_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_006_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_007_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_008_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_009_neg.ksh \
 	functional/cli_root/zpool_import/zpool_import_010_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_011_neg.ksh \
 	functional/cli_root/zpool_import/zpool_import_012_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_013_neg.ksh \
 	functional/cli_root/zpool_import/zpool_import_014_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_015_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_016_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_017_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_all_001_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_encrypted.ksh \
 	functional/cli_root/zpool_import/zpool_import_encrypted_load.ksh \
 	functional/cli_root/zpool_import/zpool_import_errata3.ksh \
 	functional/cli_root/zpool_import/zpool_import_errata4.ksh \
 	functional/cli_root/zpool_import/zpool_import_features_001_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_features_002_neg.ksh \
 	functional/cli_root/zpool_import/zpool_import_features_003_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_hostid_changed.ksh \
 	functional/cli_root/zpool_import/zpool_import_hostid_changed_unclean_export.ksh \
 	functional/cli_root/zpool_import/zpool_import_hostid_changed_cachefile.ksh \
 	functional/cli_root/zpool_import/zpool_import_hostid_changed_cachefile_unclean_export.ksh \
 	functional/cli_root/zpool_import/zpool_import_missing_001_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_missing_002_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_missing_003_pos.ksh \
 	functional/cli_root/zpool_import/zpool_import_rename_001_pos.ksh \
 	functional/cli_root/zpool_initialize/cleanup.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_attach_detach_add_remove.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_fault_export_import_online.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_import_export.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_offline_export_import_online.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_online_offline.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_split.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_start_and_cancel_neg.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_start_and_cancel_pos.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_suspend_resume.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_uninit.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_unsupported_vdevs.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_verify_checksums.ksh \
 	functional/cli_root/zpool_initialize/zpool_initialize_verify_initialized.ksh \
 	functional/cli_root/zpool_labelclear/zpool_labelclear_active.ksh \
 	functional/cli_root/zpool_labelclear/zpool_labelclear_exported.ksh \
 	functional/cli_root/zpool_labelclear/zpool_labelclear_removed.ksh \
 	functional/cli_root/zpool_labelclear/zpool_labelclear_valid.ksh \
 	functional/cli_root/zpool_offline/cleanup.ksh \
 	functional/cli_root/zpool_offline/setup.ksh \
 	functional/cli_root/zpool_offline/zpool_offline_001_pos.ksh \
 	functional/cli_root/zpool_offline/zpool_offline_002_neg.ksh \
 	functional/cli_root/zpool_offline/zpool_offline_003_pos.ksh \
 	functional/cli_root/zpool_online/cleanup.ksh \
 	functional/cli_root/zpool_online/setup.ksh \
 	functional/cli_root/zpool_online/zpool_online_001_pos.ksh \
 	functional/cli_root/zpool_online/zpool_online_002_neg.ksh \
 	functional/cli_root/zpool_remove/cleanup.ksh \
 	functional/cli_root/zpool_remove/setup.ksh \
 	functional/cli_root/zpool_remove/zpool_remove_001_neg.ksh \
 	functional/cli_root/zpool_remove/zpool_remove_002_pos.ksh \
 	functional/cli_root/zpool_remove/zpool_remove_003_pos.ksh \
 	functional/cli_root/zpool_reopen/cleanup.ksh \
 	functional/cli_root/zpool_reopen/setup.ksh \
 	functional/cli_root/zpool_reopen/zpool_reopen_001_pos.ksh \
 	functional/cli_root/zpool_reopen/zpool_reopen_002_pos.ksh \
 	functional/cli_root/zpool_reopen/zpool_reopen_003_pos.ksh \
 	functional/cli_root/zpool_reopen/zpool_reopen_004_pos.ksh \
 	functional/cli_root/zpool_reopen/zpool_reopen_005_pos.ksh \
 	functional/cli_root/zpool_reopen/zpool_reopen_006_neg.ksh \
 	functional/cli_root/zpool_reopen/zpool_reopen_007_pos.ksh \
 	functional/cli_root/zpool_replace/cleanup.ksh \
 	functional/cli_root/zpool_replace/replace-o_ashift.ksh \
 	functional/cli_root/zpool_replace/replace_prop_ashift.ksh \
 	functional/cli_root/zpool_replace/setup.ksh \
 	functional/cli_root/zpool_replace/zpool_replace_001_neg.ksh \
 	functional/cli_root/zpool_resilver/cleanup.ksh \
 	functional/cli_root/zpool_resilver/setup.ksh \
 	functional/cli_root/zpool_resilver/zpool_resilver_bad_args.ksh \
 	functional/cli_root/zpool_resilver/zpool_resilver_restart.ksh \
 	functional/cli_root/zpool_resilver/zpool_resilver_concurrent.ksh \
 	functional/cli_root/zpool_scrub/cleanup.ksh \
 	functional/cli_root/zpool_scrub/setup.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_001_neg.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_002_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_003_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_004_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_005_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_encrypted_unloaded.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_multiple_copies.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_offline_device.ksh \
 	functional/cli_root/zpool_scrub/zpool_scrub_print_repairing.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_001_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_002_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_003_pos.ksh \
 	functional/cli_root/zpool_scrub/zpool_error_scrub_004_pos.ksh \
 	functional/cli_root/zpool_set/cleanup.ksh \
 	functional/cli_root/zpool_set/setup.ksh \
 	functional/cli_root/zpool/setup.ksh \
 	functional/cli_root/zpool_set/vdev_set_001_pos.ksh \
 	functional/cli_root/zpool_set/zpool_set_common.kshlib \
 	functional/cli_root/zpool_set/zpool_set_001_pos.ksh \
 	functional/cli_root/zpool_set/zpool_set_002_neg.ksh \
 	functional/cli_root/zpool_set/zpool_set_003_neg.ksh \
 	functional/cli_root/zpool_set/zpool_set_ashift.ksh \
 	functional/cli_root/zpool_set/user_property_001_pos.ksh \
 	functional/cli_root/zpool_set/user_property_002_neg.ksh \
 	functional/cli_root/zpool_set/zpool_set_features.ksh \
 	functional/cli_root/zpool_split/cleanup.ksh \
 	functional/cli_root/zpool_split/setup.ksh \
 	functional/cli_root/zpool_split/zpool_split_cliargs.ksh \
 	functional/cli_root/zpool_split/zpool_split_devices.ksh \
 	functional/cli_root/zpool_split/zpool_split_dryrun_output.ksh \
 	functional/cli_root/zpool_split/zpool_split_encryption.ksh \
 	functional/cli_root/zpool_split/zpool_split_indirect.ksh \
 	functional/cli_root/zpool_split/zpool_split_props.ksh \
 	functional/cli_root/zpool_split/zpool_split_resilver.ksh \
 	functional/cli_root/zpool_split/zpool_split_vdevs.ksh \
 	functional/cli_root/zpool_split/zpool_split_wholedisk.ksh \
 	functional/cli_root/zpool_status/cleanup.ksh \
 	functional/cli_root/zpool_status/setup.ksh \
 	functional/cli_root/zpool_status/zpool_status_001_pos.ksh \
 	functional/cli_root/zpool_status/zpool_status_002_pos.ksh \
 	functional/cli_root/zpool_status/zpool_status_003_pos.ksh \
 	functional/cli_root/zpool_status/zpool_status_004_pos.ksh \
 	functional/cli_root/zpool_status/zpool_status_005_pos.ksh \
 	functional/cli_root/zpool_status/zpool_status_006_pos.ksh \
 	functional/cli_root/zpool_status/zpool_status_007_pos.ksh \
 	functional/cli_root/zpool_status/zpool_status_008_pos.ksh \
 	functional/cli_root/zpool_status/zpool_status_features_001_pos.ksh \
 	functional/cli_root/zpool_sync/cleanup.ksh \
 	functional/cli_root/zpool_sync/setup.ksh \
 	functional/cli_root/zpool_sync/zpool_sync_001_pos.ksh \
 	functional/cli_root/zpool_sync/zpool_sync_002_neg.ksh \
 	functional/cli_root/zpool_trim/cleanup.ksh \
 	functional/cli_root/zpool_trim/setup.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_attach_detach_add_remove.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_fault_export_import_online.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_import_export.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_multiple.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_neg.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_offline_export_import_online.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_online_offline.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_partial.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_rate.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_rate_neg.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_secure.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_split.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_start_and_cancel_neg.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_start_and_cancel_pos.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_suspend_resume.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_unsupported_vdevs.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_verify_checksums.ksh \
 	functional/cli_root/zpool_trim/zpool_trim_verify_trimmed.ksh \
 	functional/cli_root/zpool_upgrade/cleanup.ksh \
 	functional/cli_root/zpool_upgrade/setup.ksh \
 	functional/cli_root/zpool_upgrade/zpool_upgrade_001_pos.ksh \
 	functional/cli_root/zpool_upgrade/zpool_upgrade_002_pos.ksh \
 	functional/cli_root/zpool_upgrade/zpool_upgrade_003_pos.ksh \
 	functional/cli_root/zpool_upgrade/zpool_upgrade_004_pos.ksh \
 	functional/cli_root/zpool_upgrade/zpool_upgrade_005_neg.ksh \
 	functional/cli_root/zpool_upgrade/zpool_upgrade_006_neg.ksh \
 	functional/cli_root/zpool_upgrade/zpool_upgrade_007_pos.ksh \
 	functional/cli_root/zpool_upgrade/zpool_upgrade_008_pos.ksh \
 	functional/cli_root/zpool_upgrade/zpool_upgrade_009_neg.ksh \
 	functional/cli_root/zpool_upgrade/zpool_upgrade_features_001_pos.ksh \
 	functional/cli_root/zpool_wait/cleanup.ksh \
 	functional/cli_root/zpool_wait/scan/cleanup.ksh \
 	functional/cli_root/zpool_wait/scan/setup.ksh \
 	functional/cli_root/zpool_wait/scan/zpool_wait_rebuild.ksh \
 	functional/cli_root/zpool_wait/scan/zpool_wait_replace_cancel.ksh \
 	functional/cli_root/zpool_wait/scan/zpool_wait_replace.ksh \
 	functional/cli_root/zpool_wait/scan/zpool_wait_resilver.ksh \
 	functional/cli_root/zpool_wait/scan/zpool_wait_scrub_basic.ksh \
 	functional/cli_root/zpool_wait/scan/zpool_wait_scrub_cancel.ksh \
 	functional/cli_root/zpool_wait/scan/zpool_wait_scrub_flag.ksh \
 	functional/cli_root/zpool_wait/setup.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_discard.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_freeing.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_initialize_basic.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_initialize_cancel.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_initialize_flag.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_multiple.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_no_activity.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_remove_cancel.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_remove.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_trim_basic.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_trim_cancel.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_trim_flag.ksh \
 	functional/cli_root/zpool_wait/zpool_wait_usage.ksh \
 	functional/cli_root/zpool/zpool_001_neg.ksh \
 	functional/cli_root/zpool/zpool_002_pos.ksh \
 	functional/cli_root/zpool/zpool_003_pos.ksh \
 	functional/cli_root/zpool/zpool_colors.ksh \
 	functional/cli_user/misc/arcstat_001_pos.ksh \
 	functional/cli_user/misc/arc_summary_001_pos.ksh \
 	functional/cli_user/misc/arc_summary_002_neg.ksh \
 	functional/cli_user/misc/zilstat_001_pos.ksh \
 	functional/cli_user/misc/cleanup.ksh \
 	functional/cli_user/misc/setup.ksh \
 	functional/cli_user/misc/zdb_001_neg.ksh \
 	functional/cli_user/misc/zfs_001_neg.ksh \
 	functional/cli_user/misc/zfs_allow_001_neg.ksh \
 	functional/cli_user/misc/zfs_clone_001_neg.ksh \
 	functional/cli_user/misc/zfs_create_001_neg.ksh \
 	functional/cli_user/misc/zfs_destroy_001_neg.ksh \
 	functional/cli_user/misc/zfs_get_001_neg.ksh \
 	functional/cli_user/misc/zfs_inherit_001_neg.ksh \
 	functional/cli_user/misc/zfs_mount_001_neg.ksh \
 	functional/cli_user/misc/zfs_promote_001_neg.ksh \
 	functional/cli_user/misc/zfs_receive_001_neg.ksh \
 	functional/cli_user/misc/zfs_rename_001_neg.ksh \
 	functional/cli_user/misc/zfs_rollback_001_neg.ksh \
 	functional/cli_user/misc/zfs_send_001_neg.ksh \
 	functional/cli_user/misc/zfs_set_001_neg.ksh \
 	functional/cli_user/misc/zfs_share_001_neg.ksh \
 	functional/cli_user/misc/zfs_snapshot_001_neg.ksh \
 	functional/cli_user/misc/zfs_unallow_001_neg.ksh \
 	functional/cli_user/misc/zfs_unmount_001_neg.ksh \
 	functional/cli_user/misc/zfs_unshare_001_neg.ksh \
 	functional/cli_user/misc/zfs_upgrade_001_neg.ksh \
 	functional/cli_user/misc/zpool_001_neg.ksh \
 	functional/cli_user/misc/zpool_add_001_neg.ksh \
 	functional/cli_user/misc/zpool_attach_001_neg.ksh \
 	functional/cli_user/misc/zpool_clear_001_neg.ksh \
 	functional/cli_user/misc/zpool_create_001_neg.ksh \
 	functional/cli_user/misc/zpool_destroy_001_neg.ksh \
 	functional/cli_user/misc/zpool_detach_001_neg.ksh \
 	functional/cli_user/misc/zpool_export_001_neg.ksh \
 	functional/cli_user/misc/zpool_get_001_neg.ksh \
 	functional/cli_user/misc/zpool_history_001_neg.ksh \
 	functional/cli_user/misc/zpool_import_001_neg.ksh \
 	functional/cli_user/misc/zpool_import_002_neg.ksh \
 	functional/cli_user/misc/zpool_offline_001_neg.ksh \
 	functional/cli_user/misc/zpool_online_001_neg.ksh \
 	functional/cli_user/misc/zpool_remove_001_neg.ksh \
 	functional/cli_user/misc/zpool_replace_001_neg.ksh \
 	functional/cli_user/misc/zpool_scrub_001_neg.ksh \
 	functional/cli_user/misc/zpool_set_001_neg.ksh \
 	functional/cli_user/misc/zpool_status_001_neg.ksh \
 	functional/cli_user/misc/zpool_upgrade_001_neg.ksh \
 	functional/cli_user/misc/zpool_wait_privilege.ksh \
 	functional/cli_user/zfs_list/cleanup.ksh \
 	functional/cli_user/zfs_list/setup.ksh \
 	functional/cli_user/zfs_list/zfs_list_001_pos.ksh \
 	functional/cli_user/zfs_list/zfs_list_002_pos.ksh \
 	functional/cli_user/zfs_list/zfs_list_003_pos.ksh \
 	functional/cli_user/zfs_list/zfs_list_004_neg.ksh \
 	functional/cli_user/zfs_list/zfs_list_005_neg.ksh \
 	functional/cli_user/zfs_list/zfs_list_007_pos.ksh \
 	functional/cli_user/zfs_list/zfs_list_008_neg.ksh \
 	functional/cli_user/zpool_iostat/cleanup.ksh \
 	functional/cli_user/zpool_iostat/setup.ksh \
 	functional/cli_user/zpool_iostat/zpool_iostat_001_neg.ksh \
 	functional/cli_user/zpool_iostat/zpool_iostat_002_pos.ksh \
 	functional/cli_user/zpool_iostat/zpool_iostat_003_neg.ksh \
 	functional/cli_user/zpool_iostat/zpool_iostat_004_pos.ksh \
 	functional/cli_user/zpool_iostat/zpool_iostat_005_pos.ksh \
 	functional/cli_user/zpool_iostat/zpool_iostat_-c_disable.ksh \
 	functional/cli_user/zpool_iostat/zpool_iostat_-c_homedir.ksh \
 	functional/cli_user/zpool_iostat/zpool_iostat_-c_searchpath.ksh \
 	functional/cli_user/zpool_list/cleanup.ksh \
 	functional/cli_user/zpool_list/setup.ksh \
 	functional/cli_user/zpool_list/zpool_list_001_pos.ksh \
 	functional/cli_user/zpool_list/zpool_list_002_neg.ksh \
 	functional/cli_user/zpool_status/cleanup.ksh \
 	functional/cli_user/zpool_status/setup.ksh \
 	functional/cli_user/zpool_status/zpool_status_003_pos.ksh \
 	functional/cli_user/zpool_status/zpool_status_-c_disable.ksh \
 	functional/cli_user/zpool_status/zpool_status_-c_homedir.ksh \
 	functional/cli_user/zpool_status/zpool_status_-c_searchpath.ksh \
 	functional/compression/cleanup.ksh \
 	functional/compression/compress_001_pos.ksh \
 	functional/compression/compress_002_pos.ksh \
 	functional/compression/compress_003_pos.ksh \
 	functional/compression/compress_004_pos.ksh \
 	functional/compression/compress_zstd_bswap.ksh \
 	functional/compression/l2arc_compressed_arc_disabled.ksh \
 	functional/compression/l2arc_compressed_arc.ksh \
 	functional/compression/l2arc_encrypted.ksh \
 	functional/compression/l2arc_encrypted_no_compressed_arc.ksh \
 	functional/compression/setup.ksh \
 	functional/cp_files/cleanup.ksh \
 	functional/cp_files/cp_files_001_pos.ksh \
 	functional/cp_files/cp_files_002_pos.ksh \
 	functional/cp_files/cp_stress.ksh \
 	functional/cp_files/setup.ksh \
 	functional/crtime/cleanup.ksh \
 	functional/crtime/crtime_001_pos.ksh \
 	functional/crtime/setup.ksh \
 	functional/ctime/cleanup.ksh \
 	functional/ctime/ctime_001_pos.ksh \
 	functional/ctime/setup.ksh \
 	functional/deadman/deadman_ratelimit.ksh \
 	functional/deadman/deadman_sync.ksh \
 	functional/deadman/deadman_zio.ksh \
 	functional/delegate/cleanup.ksh \
 	functional/delegate/setup.ksh \
 	functional/delegate/zfs_allow_001_pos.ksh \
 	functional/delegate/zfs_allow_002_pos.ksh \
 	functional/delegate/zfs_allow_003_pos.ksh \
 	functional/delegate/zfs_allow_004_pos.ksh \
 	functional/delegate/zfs_allow_005_pos.ksh \
 	functional/delegate/zfs_allow_006_pos.ksh \
 	functional/delegate/zfs_allow_007_pos.ksh \
 	functional/delegate/zfs_allow_008_pos.ksh \
 	functional/delegate/zfs_allow_009_neg.ksh \
 	functional/delegate/zfs_allow_010_pos.ksh \
 	functional/delegate/zfs_allow_011_neg.ksh \
 	functional/delegate/zfs_allow_012_neg.ksh \
 	functional/delegate/zfs_unallow_001_pos.ksh \
 	functional/delegate/zfs_unallow_002_pos.ksh \
 	functional/delegate/zfs_unallow_003_pos.ksh \
 	functional/delegate/zfs_unallow_004_pos.ksh \
 	functional/delegate/zfs_unallow_005_pos.ksh \
 	functional/delegate/zfs_unallow_006_pos.ksh \
 	functional/delegate/zfs_unallow_007_neg.ksh \
 	functional/delegate/zfs_unallow_008_neg.ksh \
 	functional/devices/cleanup.ksh \
 	functional/devices/devices_001_pos.ksh \
 	functional/devices/devices_002_neg.ksh \
 	functional/devices/devices_003_pos.ksh \
 	functional/devices/setup.ksh \
 	functional/dos_attributes/cleanup.ksh \
 	functional/dos_attributes/read_dos_attrs_001.ksh \
 	functional/dos_attributes/setup.ksh \
 	functional/dos_attributes/write_dos_attrs_001.ksh \
 	functional/events/cleanup.ksh \
 	functional/events/events_001_pos.ksh \
 	functional/events/events_002_pos.ksh \
 	functional/events/setup.ksh \
 	functional/events/zed_cksum_config.ksh \
 	functional/events/zed_cksum_reported.ksh \
 	functional/events/zed_fd_spill.ksh \
 	functional/events/zed_io_config.ksh \
 	functional/events/zed_rc_filter.ksh \
 	functional/events/zed_slow_io.ksh \
 	functional/events/zed_slow_io_many_vdevs.ksh \
 	functional/exec/cleanup.ksh \
 	functional/exec/exec_001_pos.ksh \
 	functional/exec/exec_002_neg.ksh \
 	functional/exec/setup.ksh \
 	functional/fadvise/cleanup.ksh \
 	functional/fadvise/fadvise_sequential.ksh \
 	functional/fadvise/setup.ksh \
 	functional/fallocate/cleanup.ksh \
 	functional/fallocate/fallocate_prealloc.ksh \
 	functional/fallocate/fallocate_punch-hole.ksh \
 	functional/fallocate/fallocate_zero-range.ksh \
 	functional/fallocate/setup.ksh \
 	functional/fault/auto_offline_001_pos.ksh \
 	functional/fault/auto_online_001_pos.ksh \
 	functional/fault/auto_online_002_pos.ksh \
 	functional/fault/auto_replace_001_pos.ksh \
 	functional/fault/auto_replace_002_pos.ksh \
 	functional/fault/auto_spare_001_pos.ksh \
 	functional/fault/auto_spare_002_pos.ksh \
 	functional/fault/auto_spare_ashift.ksh \
 	functional/fault/auto_spare_multiple.ksh \
 	functional/fault/auto_spare_shared.ksh \
 	functional/fault/cleanup.ksh \
 	functional/fault/decompress_fault.ksh \
 	functional/fault/decrypt_fault.ksh \
 	functional/fault/scrub_after_resilver.ksh \
 	functional/fault/suspend_resume_single.ksh \
 	functional/fault/setup.ksh \
 	functional/fault/zpool_status_-s.ksh \
 	functional/features/async_destroy/async_destroy_001_pos.ksh \
 	functional/features/async_destroy/cleanup.ksh \
 	functional/features/async_destroy/setup.ksh \
 	functional/features/large_dnode/cleanup.ksh \
 	functional/features/large_dnode/large_dnode_001_pos.ksh \
 	functional/features/large_dnode/large_dnode_002_pos.ksh \
 	functional/features/large_dnode/large_dnode_003_pos.ksh \
 	functional/features/large_dnode/large_dnode_004_neg.ksh \
 	functional/features/large_dnode/large_dnode_005_pos.ksh \
 	functional/features/large_dnode/large_dnode_006_pos.ksh \
 	functional/features/large_dnode/large_dnode_007_neg.ksh \
 	functional/features/large_dnode/large_dnode_008_pos.ksh \
 	functional/features/large_dnode/large_dnode_009_pos.ksh \
 	functional/features/large_dnode/setup.ksh \
 	functional/grow/grow_pool_001_pos.ksh \
 	functional/grow/grow_replicas_001_pos.ksh \
 	functional/history/cleanup.ksh \
 	functional/history/history_001_pos.ksh \
 	functional/history/history_002_pos.ksh \
 	functional/history/history_003_pos.ksh \
 	functional/history/history_004_pos.ksh \
 	functional/history/history_005_neg.ksh \
 	functional/history/history_006_neg.ksh \
 	functional/history/history_007_pos.ksh \
 	functional/history/history_008_pos.ksh \
 	functional/history/history_009_pos.ksh \
 	functional/history/history_010_pos.ksh \
 	functional/history/setup.ksh \
 	functional/inheritance/cleanup.ksh \
 	functional/inheritance/inherit_001_pos.ksh \
 	functional/inuse/inuse_001_pos.ksh \
 	functional/inuse/inuse_003_pos.ksh \
 	functional/inuse/inuse_004_pos.ksh \
 	functional/inuse/inuse_005_pos.ksh \
 	functional/inuse/inuse_006_pos.ksh \
 	functional/inuse/inuse_007_pos.ksh \
 	functional/inuse/inuse_008_pos.ksh \
 	functional/inuse/inuse_009_pos.ksh \
 	functional/inuse/setup.ksh \
 	functional/io/cleanup.ksh \
 	functional/io/io_uring.ksh \
 	functional/io/libaio.ksh \
 	functional/io/mmap.ksh \
 	functional/io/posixaio.ksh \
 	functional/io/psync.ksh \
 	functional/io/setup.ksh \
 	functional/io/sync.ksh \
 	functional/l2arc/cleanup.ksh \
 	functional/l2arc/l2arc_arcstats_pos.ksh \
 	functional/l2arc/l2arc_l2miss_pos.ksh \
 	functional/l2arc/l2arc_mfuonly_pos.ksh \
 	functional/l2arc/persist_l2arc_001_pos.ksh \
 	functional/l2arc/persist_l2arc_002_pos.ksh \
 	functional/l2arc/persist_l2arc_003_neg.ksh \
 	functional/l2arc/persist_l2arc_004_pos.ksh \
 	functional/l2arc/persist_l2arc_005_pos.ksh \
 	functional/l2arc/setup.ksh \
 	functional/large_files/cleanup.ksh \
 	functional/large_files/large_files_001_pos.ksh \
 	functional/large_files/large_files_002_pos.ksh \
 	functional/large_files/setup.ksh \
 	functional/largest_pool/largest_pool_001_pos.ksh \
 	functional/libzfs/cleanup.ksh \
 	functional/libzfs/libzfs_input.ksh \
 	functional/libzfs/setup.ksh \
 	functional/limits/cleanup.ksh \
 	functional/limits/filesystem_count.ksh \
 	functional/limits/filesystem_limit.ksh \
 	functional/limits/setup.ksh \
 	functional/limits/snapshot_count.ksh \
 	functional/limits/snapshot_limit.ksh \
 	functional/link_count/cleanup.ksh \
 	functional/link_count/link_count_001.ksh \
 	functional/link_count/link_count_root_inode.ksh \
 	functional/link_count/setup.ksh \
 	functional/log_spacemap/log_spacemap_import_logs.ksh \
 	functional/migration/cleanup.ksh \
 	functional/migration/migration_001_pos.ksh \
 	functional/migration/migration_002_pos.ksh \
 	functional/migration/migration_003_pos.ksh \
 	functional/migration/migration_004_pos.ksh \
 	functional/migration/migration_005_pos.ksh \
 	functional/migration/migration_006_pos.ksh \
 	functional/migration/migration_007_pos.ksh \
 	functional/migration/migration_008_pos.ksh \
 	functional/migration/migration_009_pos.ksh \
 	functional/migration/migration_010_pos.ksh \
 	functional/migration/migration_011_pos.ksh \
 	functional/migration/migration_012_pos.ksh \
 	functional/migration/setup.ksh \
 	functional/mmap/cleanup.ksh \
 	functional/mmap/mmap_libaio_001_pos.ksh \
 	functional/mmap/mmap_mixed.ksh \
 	functional/mmap/mmap_read_001_pos.ksh \
 	functional/mmap/mmap_seek_001_pos.ksh \
 	functional/mmap/mmap_sync_001_pos.ksh \
 	functional/mmap/mmap_write_001_pos.ksh \
 	functional/mmap/setup.ksh \
 	functional/mmp/cleanup.ksh \
 	functional/mmp/mmp_active_import.ksh \
 	functional/mmp/mmp_exported_import.ksh \
 	functional/mmp/mmp_hostid.ksh \
 	functional/mmp/mmp_inactive_import.ksh \
 	functional/mmp/mmp_interval.ksh \
 	functional/mmp/mmp_on_off.ksh \
 	functional/mmp/mmp_on_thread.ksh \
 	functional/mmp/mmp_on_uberblocks.ksh \
 	functional/mmp/mmp_on_zdb.ksh \
 	functional/mmp/mmp_reset_interval.ksh \
 	functional/mmp/mmp_write_distribution.ksh \
 	functional/mmp/mmp_write_slow_disk.ksh \
 	functional/mmp/mmp_write_uberblocks.ksh \
 	functional/mmp/multihost_history.ksh \
 	functional/mmp/setup.ksh \
 	functional/mount/cleanup.ksh \
 	functional/mount/setup.ksh \
 	functional/mount/umount_001.ksh \
 	functional/mount/umountall_001.ksh \
 	functional/mount/umount_unlinked_drain.ksh \
 	functional/mv_files/cleanup.ksh \
 	functional/mv_files/mv_files_001_pos.ksh \
 	functional/mv_files/mv_files_002_pos.ksh \
 	functional/mv_files/random_creation.ksh \
 	functional/mv_files/setup.ksh \
 	functional/nestedfs/cleanup.ksh \
 	functional/nestedfs/nestedfs_001_pos.ksh \
 	functional/nestedfs/setup.ksh \
 	functional/nopwrite/cleanup.ksh \
 	functional/nopwrite/nopwrite_copies.ksh \
 	functional/nopwrite/nopwrite_mtime.ksh \
 	functional/nopwrite/nopwrite_negative.ksh \
 	functional/nopwrite/nopwrite_promoted_clone.ksh \
 	functional/nopwrite/nopwrite_recsize.ksh \
 	functional/nopwrite/nopwrite_sync.ksh \
 	functional/nopwrite/nopwrite_varying_compression.ksh \
 	functional/nopwrite/nopwrite_volume.ksh \
 	functional/nopwrite/setup.ksh \
 	functional/no_space/cleanup.ksh \
 	functional/no_space/enospc_001_pos.ksh \
 	functional/no_space/enospc_002_pos.ksh \
 	functional/no_space/enospc_003_pos.ksh \
 	functional/no_space/enospc_df.ksh \
 	functional/no_space/enospc_ganging.ksh \
 	functional/no_space/enospc_rm.ksh \
 	functional/no_space/setup.ksh \
 	functional/online_offline/cleanup.ksh \
 	functional/online_offline/online_offline_001_pos.ksh \
 	functional/online_offline/online_offline_002_neg.ksh \
 	functional/online_offline/online_offline_003_neg.ksh \
 	functional/online_offline/setup.ksh \
 	functional/pam/cleanup.ksh \
 	functional/pam/pam_basic.ksh \
 	functional/pam/pam_change_unmounted.ksh \
 	functional/pam/pam_nounmount.ksh \
 	functional/pam/pam_recursive.ksh \
 	functional/pam/pam_short_password.ksh \
 	functional/pam/setup.ksh \
 	functional/pool_checkpoint/checkpoint_after_rewind.ksh \
 	functional/pool_checkpoint/checkpoint_big_rewind.ksh \
 	functional/pool_checkpoint/checkpoint_capacity.ksh \
 	functional/pool_checkpoint/checkpoint_conf_change.ksh \
 	functional/pool_checkpoint/checkpoint_discard_busy.ksh \
 	functional/pool_checkpoint/checkpoint_discard.ksh \
 	functional/pool_checkpoint/checkpoint_discard_many.ksh \
 	functional/pool_checkpoint/checkpoint_indirect.ksh \
 	functional/pool_checkpoint/checkpoint_invalid.ksh \
 	functional/pool_checkpoint/checkpoint_lun_expsz.ksh \
 	functional/pool_checkpoint/checkpoint_open.ksh \
 	functional/pool_checkpoint/checkpoint_removal.ksh \
 	functional/pool_checkpoint/checkpoint_rewind.ksh \
 	functional/pool_checkpoint/checkpoint_ro_rewind.ksh \
 	functional/pool_checkpoint/checkpoint_sm_scale.ksh \
 	functional/pool_checkpoint/checkpoint_twice.ksh \
 	functional/pool_checkpoint/checkpoint_vdev_add.ksh \
 	functional/pool_checkpoint/checkpoint_zdb.ksh \
 	functional/pool_checkpoint/checkpoint_zhack_feat.ksh \
 	functional/pool_checkpoint/cleanup.ksh \
 	functional/pool_checkpoint/setup.ksh \
 	functional/pool_names/pool_names_001_pos.ksh \
 	functional/pool_names/pool_names_002_neg.ksh \
 	functional/poolversion/cleanup.ksh \
 	functional/poolversion/poolversion_001_pos.ksh \
 	functional/poolversion/poolversion_002_pos.ksh \
 	functional/poolversion/setup.ksh \
 	functional/privilege/cleanup.ksh \
 	functional/privilege/privilege_001_pos.ksh \
 	functional/privilege/privilege_002_pos.ksh \
 	functional/privilege/setup.ksh \
 	functional/procfs/cleanup.ksh \
 	functional/procfs/pool_state.ksh \
 	functional/procfs/procfs_list_basic.ksh \
 	functional/procfs/procfs_list_concurrent_readers.ksh \
 	functional/procfs/procfs_list_stale_read.ksh \
 	functional/procfs/setup.ksh \
 	functional/projectquota/cleanup.ksh \
 	functional/projectquota/projectid_001_pos.ksh \
 	functional/projectquota/projectid_002_pos.ksh \
 	functional/projectquota/projectid_003_pos.ksh \
 	functional/projectquota/projectquota_001_pos.ksh \
 	functional/projectquota/projectquota_002_pos.ksh \
 	functional/projectquota/projectquota_003_pos.ksh \
 	functional/projectquota/projectquota_004_neg.ksh \
 	functional/projectquota/projectquota_005_pos.ksh \
 	functional/projectquota/projectquota_006_pos.ksh \
 	functional/projectquota/projectquota_007_pos.ksh \
 	functional/projectquota/projectquota_008_pos.ksh \
 	functional/projectquota/projectquota_009_pos.ksh \
 	functional/projectquota/projectspace_001_pos.ksh \
 	functional/projectquota/projectspace_002_pos.ksh \
 	functional/projectquota/projectspace_003_pos.ksh \
 	functional/projectquota/projectspace_004_pos.ksh \
 	functional/projectquota/projecttree_001_pos.ksh \
 	functional/projectquota/projecttree_002_pos.ksh \
 	functional/projectquota/projecttree_003_neg.ksh \
 	functional/projectquota/setup.ksh \
 	functional/quota/cleanup.ksh \
 	functional/quota/quota_001_pos.ksh \
 	functional/quota/quota_002_pos.ksh \
 	functional/quota/quota_003_pos.ksh \
 	functional/quota/quota_004_pos.ksh \
 	functional/quota/quota_005_pos.ksh \
 	functional/quota/quota_006_neg.ksh \
 	functional/quota/setup.ksh \
 	functional/raidz/cleanup.ksh \
 	functional/raidz/raidz_001_neg.ksh \
 	functional/raidz/raidz_002_pos.ksh \
 	functional/raidz/raidz_003_pos.ksh \
 	functional/raidz/raidz_004_pos.ksh \
 	functional/raidz/setup.ksh \
 	functional/redacted_send/cleanup.ksh \
 	functional/redacted_send/redacted_compressed.ksh \
 	functional/redacted_send/redacted_contents.ksh \
 	functional/redacted_send/redacted_deleted.ksh \
 	functional/redacted_send/redacted_disabled_feature.ksh \
 	functional/redacted_send/redacted_embedded.ksh \
 	functional/redacted_send/redacted_holes.ksh \
 	functional/redacted_send/redacted_incrementals.ksh \
 	functional/redacted_send/redacted_largeblocks.ksh \
 	functional/redacted_send/redacted_many_clones.ksh \
 	functional/redacted_send/redacted_mixed_recsize.ksh \
 	functional/redacted_send/redacted_mounts.ksh \
 	functional/redacted_send/redacted_negative.ksh \
 	functional/redacted_send/redacted_origin.ksh \
 	functional/redacted_send/redacted_panic.ksh \
 	functional/redacted_send/redacted_props.ksh \
 	functional/redacted_send/redacted_resume.ksh \
 	functional/redacted_send/redacted_size.ksh \
 	functional/redacted_send/redacted_volume.ksh \
 	functional/redacted_send/setup.ksh \
 	functional/redundancy/cleanup.ksh \
 	functional/redundancy/redundancy_draid1.ksh \
 	functional/redundancy/redundancy_draid2.ksh \
 	functional/redundancy/redundancy_draid3.ksh \
 	functional/redundancy/redundancy_draid_damaged1.ksh \
 	functional/redundancy/redundancy_draid_damaged2.ksh \
 	functional/redundancy/redundancy_draid.ksh \
 	functional/redundancy/redundancy_draid_spare1.ksh \
 	functional/redundancy/redundancy_draid_spare2.ksh \
 	functional/redundancy/redundancy_draid_spare3.ksh \
 	functional/redundancy/redundancy_mirror.ksh \
 	functional/redundancy/redundancy_raidz1.ksh \
 	functional/redundancy/redundancy_raidz2.ksh \
 	functional/redundancy/redundancy_raidz3.ksh \
 	functional/redundancy/redundancy_raidz.ksh \
 	functional/redundancy/redundancy_stripe.ksh \
 	functional/redundancy/setup.ksh \
 	functional/refquota/cleanup.ksh \
 	functional/refquota/refquota_001_pos.ksh \
 	functional/refquota/refquota_002_pos.ksh \
 	functional/refquota/refquota_003_pos.ksh \
 	functional/refquota/refquota_004_pos.ksh \
 	functional/refquota/refquota_005_pos.ksh \
 	functional/refquota/refquota_006_neg.ksh \
 	functional/refquota/refquota_007_neg.ksh \
 	functional/refquota/refquota_008_neg.ksh \
 	functional/refquota/setup.ksh \
 	functional/refreserv/cleanup.ksh \
 	functional/refreserv/refreserv_001_pos.ksh \
 	functional/refreserv/refreserv_002_pos.ksh \
 	functional/refreserv/refreserv_003_pos.ksh \
 	functional/refreserv/refreserv_004_pos.ksh \
 	functional/refreserv/refreserv_005_pos.ksh \
 	functional/refreserv/refreserv_multi_raidz.ksh \
 	functional/refreserv/refreserv_raidz.ksh \
 	functional/refreserv/setup.ksh \
 	functional/removal/cleanup.ksh \
 	functional/removal/removal_all_vdev.ksh \
 	functional/removal/removal_cancel.ksh \
 	functional/removal/removal_check_space.ksh \
 	functional/removal/removal_condense_export.ksh \
 	functional/removal/removal_multiple_indirection.ksh \
 	functional/removal/removal_nopwrite.ksh \
 	functional/removal/removal_remap_deadlists.ksh \
 	functional/removal/removal_reservation.ksh \
 	functional/removal/removal_resume_export.ksh \
 	functional/removal/removal_sanity.ksh \
 	functional/removal/removal_with_add.ksh \
 	functional/removal/removal_with_create_fs.ksh \
 	functional/removal/removal_with_dedup.ksh \
 	functional/removal/removal_with_errors.ksh \
 	functional/removal/removal_with_export.ksh \
 	functional/removal/removal_with_faulted.ksh \
 	functional/removal/removal_with_ganging.ksh \
 	functional/removal/removal_with_indirect.ksh \
 	functional/removal/removal_with_remove.ksh \
 	functional/removal/removal_with_scrub.ksh \
 	functional/removal/removal_with_send.ksh \
 	functional/removal/removal_with_send_recv.ksh \
 	functional/removal/removal_with_snapshot.ksh \
 	functional/removal/removal_with_write.ksh \
 	functional/removal/removal_with_zdb.ksh \
 	functional/removal/remove_attach_mirror.ksh \
 	functional/removal/remove_expanded.ksh \
 	functional/removal/remove_indirect.ksh \
 	functional/removal/remove_mirror.ksh \
 	functional/removal/remove_mirror_sanity.ksh \
 	functional/removal/remove_raidz.ksh \
 	functional/rename_dirs/cleanup.ksh \
 	functional/rename_dirs/rename_dirs_001_pos.ksh \
 	functional/rename_dirs/setup.ksh \
 	functional/renameat2/cleanup.ksh \
 	functional/renameat2/setup.ksh \
 	functional/renameat2/renameat2_exchange.ksh \
 	functional/renameat2/renameat2_noreplace.ksh \
 	functional/renameat2/renameat2_whiteout.ksh \
 	functional/replacement/attach_import.ksh \
 	functional/replacement/attach_multiple.ksh \
 	functional/replacement/attach_rebuild.ksh \
 	functional/replacement/attach_resilver.ksh \
 	functional/replacement/cleanup.ksh \
 	functional/replacement/detach.ksh \
 	functional/replacement/rebuild_disabled_feature.ksh \
 	functional/replacement/rebuild_multiple.ksh \
 	functional/replacement/rebuild_raidz.ksh \
 	functional/replacement/replace_import.ksh \
 	functional/replacement/replace_rebuild.ksh \
 	functional/replacement/replace_resilver.ksh \
 	functional/replacement/resilver_restart_001.ksh \
 	functional/replacement/resilver_restart_002.ksh \
 	functional/replacement/scrub_cancel.ksh \
 	functional/replacement/setup.ksh \
 	functional/reservation/cleanup.ksh \
 	functional/reservation/reservation_001_pos.ksh \
 	functional/reservation/reservation_002_pos.ksh \
 	functional/reservation/reservation_003_pos.ksh \
 	functional/reservation/reservation_004_pos.ksh \
 	functional/reservation/reservation_005_pos.ksh \
 	functional/reservation/reservation_006_pos.ksh \
 	functional/reservation/reservation_007_pos.ksh \
 	functional/reservation/reservation_008_pos.ksh \
 	functional/reservation/reservation_009_pos.ksh \
 	functional/reservation/reservation_010_pos.ksh \
 	functional/reservation/reservation_011_pos.ksh \
 	functional/reservation/reservation_012_pos.ksh \
 	functional/reservation/reservation_013_pos.ksh \
 	functional/reservation/reservation_014_pos.ksh \
 	functional/reservation/reservation_015_pos.ksh \
 	functional/reservation/reservation_016_pos.ksh \
 	functional/reservation/reservation_017_pos.ksh \
 	functional/reservation/reservation_018_pos.ksh \
 	functional/reservation/reservation_019_pos.ksh \
 	functional/reservation/reservation_020_pos.ksh \
 	functional/reservation/reservation_021_neg.ksh \
 	functional/reservation/reservation_022_pos.ksh \
 	functional/reservation/setup.ksh \
 	functional/rootpool/cleanup.ksh \
 	functional/rootpool/rootpool_002_neg.ksh \
 	functional/rootpool/rootpool_003_neg.ksh \
 	functional/rootpool/rootpool_007_pos.ksh \
 	functional/rootpool/setup.ksh \
 	functional/rsend/cleanup.ksh \
 	functional/rsend/recv_dedup_encrypted_zvol.ksh \
 	functional/rsend/recv_dedup.ksh \
 	functional/rsend/rsend_001_pos.ksh \
 	functional/rsend/rsend_002_pos.ksh \
 	functional/rsend/rsend_003_pos.ksh \
 	functional/rsend/rsend_004_pos.ksh \
 	functional/rsend/rsend_005_pos.ksh \
 	functional/rsend/rsend_006_pos.ksh \
 	functional/rsend/rsend_007_pos.ksh \
 	functional/rsend/rsend_008_pos.ksh \
 	functional/rsend/rsend_009_pos.ksh \
 	functional/rsend/rsend_010_pos.ksh \
 	functional/rsend/rsend_011_pos.ksh \
 	functional/rsend/rsend_012_pos.ksh \
 	functional/rsend/rsend_013_pos.ksh \
 	functional/rsend/rsend_014_pos.ksh \
 	functional/rsend/rsend_016_neg.ksh \
 	functional/rsend/rsend_019_pos.ksh \
 	functional/rsend/rsend_020_pos.ksh \
 	functional/rsend/rsend_021_pos.ksh \
 	functional/rsend/rsend_022_pos.ksh \
 	functional/rsend/rsend_024_pos.ksh \
 	functional/rsend/rsend_025_pos.ksh \
 	functional/rsend/rsend_026_neg.ksh \
 	functional/rsend/rsend_027_pos.ksh \
 	functional/rsend/rsend_028_neg.ksh \
 	functional/rsend/rsend_029_neg.ksh \
 	functional/rsend/rsend_030_pos.ksh \
 	functional/rsend/rsend_031_pos.ksh \
 	functional/rsend/send-c_embedded_blocks.ksh \
 	functional/rsend/send-c_incremental.ksh \
 	functional/rsend/send-c_lz4_disabled.ksh \
 	functional/rsend/send-c_mixed_compression.ksh \
 	functional/rsend/send-c_props.ksh \
 	functional/rsend/send-c_recv_dedup.ksh \
 	functional/rsend/send-c_recv_lz4_disabled.ksh \
 	functional/rsend/send-c_resume.ksh \
 	functional/rsend/send-c_stream_size_estimate.ksh \
 	functional/rsend/send-c_verify_contents.ksh \
 	functional/rsend/send-c_verify_ratio.ksh \
 	functional/rsend/send-c_volume.ksh \
 	functional/rsend/send-c_zstream_recompress.ksh \
 	functional/rsend/send-c_zstreamdump.ksh \
 	functional/rsend/send-cpL_varied_recsize.ksh \
 	functional/rsend/send_doall.ksh \
 	functional/rsend/send_encrypted_incremental.ksh \
 	functional/rsend/send_encrypted_files.ksh \
 	functional/rsend/send_encrypted_freeobjects.ksh \
 	functional/rsend/send_encrypted_hierarchy.ksh \
 	functional/rsend/send_encrypted_props.ksh \
 	functional/rsend/send_encrypted_truncated_files.ksh \
 	functional/rsend/send_freeobjects.ksh \
 	functional/rsend/send_holds.ksh \
 	functional/rsend/send_hole_birth.ksh \
 	functional/rsend/send_invalid.ksh \
+	functional/rsend/send_leak_keymaps.ksh \
 	functional/rsend/send-L_toggle.ksh \
 	functional/rsend/send_mixed_raw.ksh \
 	functional/rsend/send_partial_dataset.ksh \
 	functional/rsend/send_raw_ashift.ksh \
 	functional/rsend/send_raw_spill_block.ksh \
 	functional/rsend/send_raw_large_blocks.ksh \
 	functional/rsend/send_realloc_dnode_size.ksh \
 	functional/rsend/send_realloc_encrypted_files.ksh \
 	functional/rsend/send_realloc_files.ksh \
 	functional/rsend/send_spill_block.ksh \
 	functional/rsend/send-wR_encrypted_zvol.ksh \
 	functional/rsend/setup.ksh \
 	functional/scrub_mirror/cleanup.ksh \
 	functional/scrub_mirror/scrub_mirror_001_pos.ksh \
 	functional/scrub_mirror/scrub_mirror_002_pos.ksh \
 	functional/scrub_mirror/scrub_mirror_003_pos.ksh \
 	functional/scrub_mirror/scrub_mirror_004_pos.ksh \
 	functional/scrub_mirror/setup.ksh \
 	functional/slog/cleanup.ksh \
 	functional/slog/setup.ksh \
 	functional/slog/slog_001_pos.ksh \
 	functional/slog/slog_002_pos.ksh \
 	functional/slog/slog_003_pos.ksh \
 	functional/slog/slog_004_pos.ksh \
 	functional/slog/slog_005_pos.ksh \
 	functional/slog/slog_006_pos.ksh \
 	functional/slog/slog_007_pos.ksh \
 	functional/slog/slog_008_neg.ksh \
 	functional/slog/slog_009_neg.ksh \
 	functional/slog/slog_010_neg.ksh \
 	functional/slog/slog_011_neg.ksh \
 	functional/slog/slog_012_neg.ksh \
 	functional/slog/slog_013_pos.ksh \
 	functional/slog/slog_014_pos.ksh \
 	functional/slog/slog_015_neg.ksh \
 	functional/slog/slog_016_pos.ksh \
 	functional/slog/slog_replay_fs_001.ksh \
 	functional/slog/slog_replay_fs_002.ksh \
 	functional/slog/slog_replay_volume.ksh \
 	functional/snapshot/cleanup.ksh \
 	functional/snapshot/clone_001_pos.ksh \
 	functional/snapshot/rollback_001_pos.ksh \
 	functional/snapshot/rollback_002_pos.ksh \
 	functional/snapshot/rollback_003_pos.ksh \
 	functional/snapshot/setup.ksh \
 	functional/snapshot/snapshot_001_pos.ksh \
 	functional/snapshot/snapshot_002_pos.ksh \
 	functional/snapshot/snapshot_003_pos.ksh \
 	functional/snapshot/snapshot_004_pos.ksh \
 	functional/snapshot/snapshot_005_pos.ksh \
 	functional/snapshot/snapshot_006_pos.ksh \
 	functional/snapshot/snapshot_007_pos.ksh \
 	functional/snapshot/snapshot_008_pos.ksh \
 	functional/snapshot/snapshot_009_pos.ksh \
 	functional/snapshot/snapshot_010_pos.ksh \
 	functional/snapshot/snapshot_011_pos.ksh \
 	functional/snapshot/snapshot_012_pos.ksh \
 	functional/snapshot/snapshot_013_pos.ksh \
 	functional/snapshot/snapshot_014_pos.ksh \
 	functional/snapshot/snapshot_015_pos.ksh \
 	functional/snapshot/snapshot_016_pos.ksh \
 	functional/snapshot/snapshot_017_pos.ksh \
 	functional/snapshot/snapshot_018_pos.ksh \
 	functional/snapused/cleanup.ksh \
 	functional/snapused/setup.ksh \
 	functional/snapused/snapused_001_pos.ksh \
 	functional/snapused/snapused_002_pos.ksh \
 	functional/snapused/snapused_003_pos.ksh \
 	functional/snapused/snapused_004_pos.ksh \
 	functional/snapused/snapused_005_pos.ksh \
 	functional/sparse/cleanup.ksh \
 	functional/sparse/setup.ksh \
 	functional/sparse/sparse_001_pos.ksh \
 	functional/stat/cleanup.ksh \
 	functional/stat/setup.ksh \
 	functional/stat/stat_001_pos.ksh \
 	functional/suid/cleanup.ksh \
 	functional/suid/setup.ksh \
 	functional/suid/suid_write_to_none.ksh \
 	functional/suid/suid_write_to_sgid.ksh \
 	functional/suid/suid_write_to_suid.ksh \
 	functional/suid/suid_write_to_suid_sgid.ksh \
 	functional/suid/suid_write_zil_replay.ksh \
 	functional/trim/autotrim_config.ksh \
 	functional/trim/autotrim_integrity.ksh \
 	functional/trim/autotrim_trim_integrity.ksh \
 	functional/trim/cleanup.ksh \
 	functional/trim/setup.ksh \
 	functional/trim/trim_config.ksh \
 	functional/trim/trim_integrity.ksh \
 	functional/trim/trim_l2arc.ksh \
 	functional/truncate/cleanup.ksh \
 	functional/truncate/setup.ksh \
 	functional/truncate/truncate_001_pos.ksh \
 	functional/truncate/truncate_002_pos.ksh \
 	functional/truncate/truncate_timestamps.ksh \
 	functional/upgrade/cleanup.ksh \
 	functional/upgrade/setup.ksh \
 	functional/upgrade/upgrade_projectquota_001_pos.ksh \
 	functional/upgrade/upgrade_readonly_pool.ksh \
 	functional/upgrade/upgrade_userobj_001_pos.ksh \
 	functional/user_namespace/cleanup.ksh \
 	functional/user_namespace/setup.ksh \
 	functional/user_namespace/user_namespace_001.ksh \
 	functional/user_namespace/user_namespace_002.ksh \
 	functional/user_namespace/user_namespace_003.ksh \
 	functional/user_namespace/user_namespace_004.ksh \
 	functional/userquota/cleanup.ksh \
 	functional/userquota/groupspace_001_pos.ksh \
 	functional/userquota/groupspace_002_pos.ksh \
 	functional/userquota/groupspace_003_pos.ksh \
 	functional/userquota/setup.ksh \
 	functional/userquota/userquota_001_pos.ksh \
 	functional/userquota/userquota_002_pos.ksh \
 	functional/userquota/userquota_003_pos.ksh \
 	functional/userquota/userquota_004_pos.ksh \
 	functional/userquota/userquota_005_neg.ksh \
 	functional/userquota/userquota_006_pos.ksh \
 	functional/userquota/userquota_007_pos.ksh \
 	functional/userquota/userquota_008_pos.ksh \
 	functional/userquota/userquota_009_pos.ksh \
 	functional/userquota/userquota_010_pos.ksh \
 	functional/userquota/userquota_011_pos.ksh \
 	functional/userquota/userquota_012_neg.ksh \
 	functional/userquota/userquota_013_pos.ksh \
 	functional/userquota/userspace_001_pos.ksh \
 	functional/userquota/userspace_002_pos.ksh \
 	functional/userquota/userspace_003_pos.ksh \
 	functional/userquota/userspace_encrypted.ksh \
 	functional/userquota/userspace_send_encrypted.ksh \
 	functional/userquota/userspace_encrypted_13709.ksh \
 	functional/vdev_zaps/cleanup.ksh \
 	functional/vdev_zaps/setup.ksh \
 	functional/vdev_zaps/vdev_zaps_001_pos.ksh \
 	functional/vdev_zaps/vdev_zaps_002_pos.ksh \
 	functional/vdev_zaps/vdev_zaps_003_pos.ksh \
 	functional/vdev_zaps/vdev_zaps_004_pos.ksh \
 	functional/vdev_zaps/vdev_zaps_005_pos.ksh \
 	functional/vdev_zaps/vdev_zaps_006_pos.ksh \
 	functional/vdev_zaps/vdev_zaps_007_pos.ksh \
 	functional/write_dirs/cleanup.ksh \
 	functional/write_dirs/setup.ksh \
 	functional/write_dirs/write_dirs_001_pos.ksh \
 	functional/write_dirs/write_dirs_002_pos.ksh \
 	functional/xattr/cleanup.ksh \
 	functional/xattr/setup.ksh \
 	functional/xattr/xattr_001_pos.ksh \
 	functional/xattr/xattr_002_neg.ksh \
 	functional/xattr/xattr_003_neg.ksh \
 	functional/xattr/xattr_004_pos.ksh \
 	functional/xattr/xattr_005_pos.ksh \
 	functional/xattr/xattr_006_pos.ksh \
 	functional/xattr/xattr_007_neg.ksh \
 	functional/xattr/xattr_008_pos.ksh \
 	functional/xattr/xattr_009_neg.ksh \
 	functional/xattr/xattr_010_neg.ksh \
 	functional/xattr/xattr_011_pos.ksh \
 	functional/xattr/xattr_012_pos.ksh \
 	functional/xattr/xattr_013_pos.ksh \
 	functional/xattr/xattr_compat.ksh \
 	functional/zpool_influxdb/cleanup.ksh \
 	functional/zpool_influxdb/setup.ksh \
 	functional/zpool_influxdb/zpool_influxdb.ksh \
 	functional/zvol/zvol_cli/cleanup.ksh \
 	functional/zvol/zvol_cli/setup.ksh \
 	functional/zvol/zvol_cli/zvol_cli_001_pos.ksh \
 	functional/zvol/zvol_cli/zvol_cli_002_pos.ksh \
 	functional/zvol/zvol_cli/zvol_cli_003_neg.ksh \
 	functional/zvol/zvol_ENOSPC/cleanup.ksh \
 	functional/zvol/zvol_ENOSPC/setup.ksh \
 	functional/zvol/zvol_ENOSPC/zvol_ENOSPC_001_pos.ksh \
 	functional/zvol/zvol_misc/cleanup.ksh \
 	functional/zvol/zvol_misc/setup.ksh \
 	functional/zvol/zvol_misc/zvol_misc_001_neg.ksh \
 	functional/zvol/zvol_misc/zvol_misc_002_pos.ksh \
 	functional/zvol/zvol_misc/zvol_misc_003_neg.ksh \
 	functional/zvol/zvol_misc/zvol_misc_004_pos.ksh \
 	functional/zvol/zvol_misc/zvol_misc_005_neg.ksh \
 	functional/zvol/zvol_misc/zvol_misc_006_pos.ksh \
 	functional/zvol/zvol_misc/zvol_misc_fua.ksh \
 	functional/zvol/zvol_misc/zvol_misc_hierarchy.ksh \
 	functional/zvol/zvol_misc/zvol_misc_rename_inuse.ksh \
 	functional/zvol/zvol_misc/zvol_misc_snapdev.ksh \
 	functional/zvol/zvol_misc/zvol_misc_trim.ksh \
 	functional/zvol/zvol_misc/zvol_misc_volmode.ksh \
 	functional/zvol/zvol_misc/zvol_misc_zil.ksh \
 	functional/zvol/zvol_stress/cleanup.ksh \
 	functional/zvol/zvol_stress/setup.ksh \
 	functional/zvol/zvol_stress/zvol_stress.ksh \
 	functional/zvol/zvol_swap/cleanup.ksh \
 	functional/zvol/zvol_swap/setup.ksh \
 	functional/zvol/zvol_swap/zvol_swap_001_pos.ksh \
 	functional/zvol/zvol_swap/zvol_swap_002_pos.ksh \
 	functional/zvol/zvol_swap/zvol_swap_003_pos.ksh \
 	functional/zvol/zvol_swap/zvol_swap_004_pos.ksh \
 	functional/zvol/zvol_swap/zvol_swap_005_pos.ksh \
 	functional/zvol/zvol_swap/zvol_swap_006_pos.ksh \
 	functional/idmap_mount/cleanup.ksh \
 	functional/idmap_mount/setup.ksh \
 	functional/idmap_mount/idmap_mount_001.ksh \
 	functional/idmap_mount/idmap_mount_002.ksh \
 	functional/idmap_mount/idmap_mount_003.ksh \
 	functional/idmap_mount/idmap_mount_004.ksh \
 	functional/idmap_mount/idmap_mount_005.ksh
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh
index 52b22dd833f0..dd0084816855 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_003_pos.ksh
@@ -1,75 +1,75 @@
 #!/bin/ksh -p
 #
 # CDDL HEADER START
 #
 # The contents of this file are subject to the terms of the
 # Common Development and Distribution License (the "License").
 # You may not use this file except in compliance with the License.
 #
 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 # or https://opensource.org/licenses/CDDL-1.0.
 # See the License for the specific language governing permissions
 # and limitations under the License.
 #
 # When distributing Covered Code, include this CDDL HEADER in each
 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 # If applicable, add the following below this CDDL HEADER, with the
 # fields enclosed by brackets "[]" replaced with your own identifying
 # information: Portions Copyright [yyyy] [name of copyright owner]
 #
 # CDDL HEADER END
 #
 
 #
 # Copyright (c) 2019, Delphix. All rights reserved.
 # Copyright (c) 2021, George Amanakis. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
 
 #
 # DESCRIPTION:
 # Verify correct output with 'zpool status -v' after corrupting a file
 #
 # STRATEGY:
 # 1. Create a pool and a file
 # 2. zinject checksum errors
 # 3. Read the file
 # 4. Take a snapshot and make a clone
 # 5. Verify we see "snapshot, clone and filesystem" output in 'zpool status -v'
 #      and 'zpool status -ev'
 
 function cleanup
 {
 	log_must zinject -c all
 	datasetexists $TESTPOOL2 && log_must zpool destroy $TESTPOOL2
 	rm -f $TESTDIR/vdev_a
 }
 
 verify_runnable "both"
 
 log_assert "Verify correct 'zpool status -v' output with a corrupted file"
 log_onexit cleanup
 
 truncate -s $MINVDEVSIZE $TESTDIR/vdev_a
 log_must zpool create -f $TESTPOOL2 $TESTDIR/vdev_a
 
 log_must fio --rw=write --name=job --size=10M --filename=/$TESTPOOL2/10m_file
 log_must zinject -t data -e checksum -f 100 -am /$TESTPOOL2/10m_file
 
 # Try to read the 2nd megabyte of 10m_file
-dd if=/$TESTPOOL2/10m_file bs=1M || true
+dd if=/$TESTPOOL2/10m_file bs=1M of=/dev/null || true
 
 log_must zfs snapshot $TESTPOOL2@snap
 log_must zfs clone $TESTPOOL2@snap $TESTPOOL2/clone
 log_must zfs create $TESTPOOL2/$TESTFS1
 
 # Look to see that snapshot, clone and filesystem our files report errors
 log_must zpool status -v $TESTPOOL2
 log_must eval "zpool status -v | grep '$TESTPOOL2@snap:/10m_file'"
 log_must eval "zpool status -v | grep '$TESTPOOL2/clone/10m_file'"
 log_must eval "zpool status -v | grep '$TESTPOOL2/10m_file'"
 log_must eval "zpool status -ev | grep '$TESTPOOL2/10m_file'"
 log_mustnot eval "zpool status -v | grep '$TESTFS1'"
 
 log_pass "'zpool status -v' outputs affected filesystem, snapshot & clone"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_004_pos.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_004_pos.ksh
index 111d598dfb7d..e80835821a83 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_004_pos.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/cli_root/zpool_status/zpool_status_004_pos.ksh
@@ -1,82 +1,82 @@
 #!/bin/ksh -p
 #
 # CDDL HEADER START
 #
 # The contents of this file are subject to the terms of the
 # Common Development and Distribution License (the "License").
 # You may not use this file except in compliance with the License.
 #
 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 # or https://opensource.org/licenses/CDDL-1.0.
 # See the License for the specific language governing permissions
 # and limitations under the License.
 #
 # When distributing Covered Code, include this CDDL HEADER in each
 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 # If applicable, add the following below this CDDL HEADER, with the
 # fields enclosed by brackets "[]" replaced with your own identifying
 # information: Portions Copyright [yyyy] [name of copyright owner]
 #
 # CDDL HEADER END
 #
 
 #
 # Copyright (c) 2019, by Delphix. All rights reserved.
 # Copyright (c) 2021, George Amanakis. All rights reserved.
 #
 
 . $STF_SUITE/include/libtest.shlib
 
 #
 # DESCRIPTION:
 # Verify feature@head_errlog=disabled works.
 #
 # STRATEGY:
 # 1. Create a pool with feature@head_errlog=disabled and a file
 # 2. zinject checksum errors
 # 3. Read the file
 # 4. Take a snapshot and make a clone
 # 5. Verify that zpool status displays the old behaviour.
 
 function cleanup
 {
 	log_must zinject -c all
 	datasetexists $TESTPOOL2 && log_must zpool destroy $TESTPOOL2
 	rm -f $TESTDIR/vdev_a
 }
 
 verify_runnable "both"
 
 log_assert "Verify 'zpool status -v' with feature@head_errlog=disabled works"
 log_onexit cleanup
 
 truncate -s $MINVDEVSIZE $TESTDIR/vdev_a
 log_must zpool create -f -o feature@head_errlog=disabled $TESTPOOL2 $TESTDIR/vdev_a
 
 state=$(zpool list -Ho feature@head_errlog $TESTPOOL2)
 if [[ "$state" != "disabled" ]]; then
 	log_fail "head_errlog has state $state"
 fi
 
 log_must fio --rw=write --name=job --size=10M --filename=/$TESTPOOL2/10m_file
 log_must zinject -t data -e checksum -f 100 -am /$TESTPOOL2/10m_file
 
 # Try to read the file
-dd if=/$TESTPOOL2/10m_file bs=1M || true
+dd if=/$TESTPOOL2/10m_file bs=1M of=/dev/null || true
 
 log_must zfs snapshot $TESTPOOL2@snap
 log_must zfs clone $TESTPOOL2@snap $TESTPOOL2/clone
 
 # Check that snapshot and clone do not report the error.
 log_mustnot eval "zpool status -v | grep '$TESTPOOL2@snap:/10m_file'"
 log_mustnot eval "zpool status -v | grep '$TESTPOOL2/clone/10m_file'"
 log_must eval "zpool status -v | grep '$TESTPOOL2/10m_file'"
 
 # Check that enabling the feature reports the error properly.
 log_must zpool set feature@head_errlog=enabled $TESTPOOL2
 log_must zpool status -v $TESTPOOL2
 log_must eval "zpool status -v | grep '$TESTPOOL2@snap:/10m_file'"
 log_must eval "zpool status -v | grep '$TESTPOOL2/clone/10m_file'"
 log_must eval "zpool status -v | grep '$TESTPOOL2/10m_file'"
 
 log_pass "'zpool status -v' with feature@head_errlog=disabled works"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/io/io_uring.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/io/io_uring.ksh
index f14b9f450826..6db2b3ae554b 100755
--- a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/io/io_uring.ksh
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/io/io_uring.ksh
@@ -1,79 +1,72 @@
 #! /bin/ksh -p
 #
 # CDDL HEADER START
 #
 # The contents of this file are subject to the terms of the
 # Common Development and Distribution License (the "License").
 # You may not use this file except in compliance with the License.
 #
 # You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 # or https://opensource.org/licenses/CDDL-1.0.
 # See the License for the specific language governing permissions
 # and limitations under the License.
 #
 # When distributing Covered Code, include this CDDL HEADER in each
 # file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 # If applicable, add the following below this CDDL HEADER, with the
 # fields enclosed by brackets "[]" replaced with your own identifying
 # information: Portions Copyright [yyyy] [name of copyright owner]
 #
 # CDDL HEADER END
 #
 
 #
 # Copyright (c) 2018 by Lawrence Livermore National Security, LLC.
 #
 
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/io/io.cfg
 
 #
 # DESCRIPTION:
 #	Verify Linux io_uring.
 #
 # STRATEGY:
 #	1. Use fio(1) in verify mode to perform write, read,
 #	   random read, and random write workloads.
 #	2. Repeat the test with additional fio(1) options.
 #
 
 verify_runnable "global"
 
 
 if ! $(grep -q "CONFIG_IO_URING=y" /boot/config-$(uname -r)); then
 	log_unsupported "Requires io_uring support within Kernel"
 fi
 
-if [ -e /etc/os-release ] ; then
-	source /etc/os-release
-	if [ $PLATFORM_ID = "platform:el9" ]; then
-		log_unsupported "Disabled on RHEL 9 variants: fails with 'Operation not permitted'"
-	fi
-fi
-
 fio --ioengine=io_uring --parse-only || log_unsupported "fio io_uring support required"
 
 function cleanup
 {
 	log_must rm -f "$mntpnt/rw*"
 }
 
 log_assert "Verify Linux io_uring"
 
 log_onexit cleanup
 
 ioengine="--ioengine=io_uring"
 mntpnt=$(get_prop mountpoint $TESTPOOL/$TESTFS)
 dir="--directory=$mntpnt"
 
 set -A fio_arg -- "--sync=0" "--sync=1" "--direct=0" "--direct=1"
 
 for arg in "${fio_arg[@]}"; do
 	log_must fio $dir $ioengine $arg $FIO_WRITE_ARGS
 	log_must fio $dir $ioengine $arg $FIO_READ_ARGS
 	log_must fio $dir $ioengine $arg $FIO_RANDWRITE_ARGS
 	log_must fio $dir $ioengine $arg $FIO_RANDREAD_ARGS
 	log_must rm -f "$mntpnt/rw*"
 done
 
 log_pass "Verified Linux io_uring"
diff --git a/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_leak_keymaps.ksh b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_leak_keymaps.ksh
new file mode 100755
index 000000000000..6ab8da6fd740
--- /dev/null
+++ b/sys/contrib/openzfs/tests/zfs-tests/tests/functional/rsend/send_leak_keymaps.ksh
@@ -0,0 +1,82 @@
+#!/bin/ksh -p
+# SPDX-License-Identifier: CDDL-1.0
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2025 by George Amanakis. All rights reserved.
+#
+
+. $STF_SUITE/tests/functional/rsend/rsend.kshlib
+
+#
+# DESCRIPTION:
+# Verify that an incremental non-raw zfs send from an encrypted filesystem
+# does not leak any keys or key mappings.
+#
+# STRATEGY:
+# 1. Create a new encrypted filesystem
+# 2. Write some files and create snapshots.
+# 3. Send to a new filesystem
+# 4. Do an incremental (-I) send and before that access all properties on the
+#	sending filesystem (emulate sanoid)
+# 5. Export and re-import the pool. Upon exporting the pool if any keys/key
+#	mappings leaked a panic will occur.
+#
+
+verify_runnable "both"
+
+function cleanup
+{
+	datasetexists $TESTPOOL/$TESTFS2 && \
+		destroy_dataset $TESTPOOL/$TESTFS2 -r
+	datasetexists $TESTPOOL/recv && \
+		destroy_dataset $TESTPOOL/recv -r
+	[[ -f $keyfile ]] && log_must rm $keyfile
+}
+log_onexit cleanup
+
+log_assert "Verify non-raw send with encryption does not leak any key mappings"
+
+typeset keyfile=/$TESTPOOL/pkey
+
+# Create an encrypted dataset
+log_must eval "echo 'password' > $keyfile"
+log_must zfs create -o encryption=on -o keyformat=passphrase \
+	-o keylocation=file://$keyfile $TESTPOOL/$TESTFS2
+
+log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/testfile bs=128K count=4 \
+	status=none
+
+for i in $(seq 0 20); do
+    log_note "Taking snapshots"
+    log_must zfs snapshot $TESTPOOL/$TESTFS2@snap_$i
+    log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/testfile bs=128K \
+	    count=4 status=none
+done
+
+log_must eval "zfs send $TESTPOOL/$TESTFS2@snap_0 | zfs recv $TESTPOOL/recv"
+
+for i in $(seq 3 3 20); do
+    log_note "Sending incremental snapshot snap_$((i - 3)) -> snap_$i"
+    log_must zfs get -Hpd 1 -t snapshot all $TESTPOOL/$TESTFS2 &>/dev/null
+    log_must eval "zfs send -I $TESTPOOL/$TESTFS2@snap_$((i - 3)) \
+	$TESTPOOL/$TESTFS2@snap_$i | zfs recv $TESTPOOL/recv"
+done
+
+log_must zpool export $TESTPOOL
+log_must zpool import $TESTPOOL
+
+log_pass "Verify non-raw send with encryption does not leak any key mappings"
diff --git a/sys/modules/zfs/zfs_config.h b/sys/modules/zfs/zfs_config.h
index 3c67896111ca..b0fc72c654f3 100644
--- a/sys/modules/zfs/zfs_config.h
+++ b/sys/modules/zfs/zfs_config.h
@@ -1,822 +1,848 @@
 /* zfs_config.h.  Generated from zfs_config.h.in by configure.  */
 /* zfs_config.h.in.  Generated from configure.ac by autoheader.  */
 
 /* Define to 1 if translation of program messages to the user's native
    language is requested. */
 /* #undef ENABLE_NLS */
 
 /* __assign_str() has one arg */
 /* #undef HAVE_1ARG_ASSIGN_STR */
 
 /* lookup_bdev() wants 1 arg */
 /* #undef HAVE_1ARG_LOOKUP_BDEV */
 
 /* kernel has access_ok with 'type' parameter */
 /* #undef HAVE_ACCESS_OK_TYPE */
 
 /* add_disk() returns int */
 /* #undef HAVE_ADD_DISK_RET */
 
 /* Define if host toolchain supports AES */
 #define HAVE_AES 1
 
 /* Define if you have [rt] */
 #define HAVE_AIO_H 1
 
 #ifdef __amd64__
 #ifndef RESCUE
 /* Define if host toolchain supports AVX */
 #define HAVE_AVX 1
 #endif
 
 /* Define if host toolchain supports AVX2 */
 #define HAVE_AVX2 1
 
 /* Define if host toolchain supports AVX512BW */
 #define HAVE_AVX512BW 1
 
 /* Define if host toolchain supports AVX512CD */
 #define HAVE_AVX512CD 1
 
 /* Define if host toolchain supports AVX512DQ */
 #define HAVE_AVX512DQ 1
 
 /* Define if host toolchain supports AVX512ER */
 #define HAVE_AVX512ER 1
 
 /* Define if host toolchain supports AVX512F */
 #define HAVE_AVX512F 1
 
 /* Define if host toolchain supports AVX512IFMA */
 #define HAVE_AVX512IFMA 1
 
 /* Define if host toolchain supports AVX512PF */
 #define HAVE_AVX512PF 1
 
 /* Define if host toolchain supports AVX512VBMI */
 #define HAVE_AVX512VBMI 1
 
 /* Define if host toolchain supports AVX512VL */
 #define HAVE_AVX512VL 1
 #endif
 
 /* backtrace() is available */
 /* #undef HAVE_BACKTRACE */
 
 /* bdevname() is available */
 /* #undef HAVE_BDEVNAME */
 
 /* bdev_check_media_change() exists */
 /* #undef HAVE_BDEV_CHECK_MEDIA_CHANGE */
 
 /* bdev_file_open_by_path() exists */
 /* #undef HAVE_BDEV_FILE_OPEN_BY_PATH */
 
 /* bdev_*_io_acct() available */
 /* #undef HAVE_BDEV_IO_ACCT_63 */
 
 /* bdev_*_io_acct() available */
 /* #undef HAVE_BDEV_IO_ACCT_OLD */
 
 /* bdev_kobj() exists */
 /* #undef HAVE_BDEV_KOBJ */
 
 /* bdev_max_discard_sectors() is available */
 /* #undef HAVE_BDEV_MAX_DISCARD_SECTORS */
 
 /* bdev_max_secure_erase_sectors() is available */
 /* #undef HAVE_BDEV_MAX_SECURE_ERASE_SECTORS */
 
 /* bdev_nr_bytes() is available */
 /* #undef HAVE_BDEV_NR_BYTES */
 
 /* bdev_open_by_path() exists */
 /* #undef HAVE_BDEV_OPEN_BY_PATH */
 
 /* bdev_release() exists */
 /* #undef HAVE_BDEV_RELEASE */
 
 /* block_device_operations->submit_bio() returns void */
 /* #undef HAVE_BDEV_SUBMIT_BIO_RETURNS_VOID */
 
 /* bdev_whole() is available */
 /* #undef HAVE_BDEV_WHOLE */
 
 /* bio_alloc() takes 4 arguments */
 /* #undef HAVE_BIO_ALLOC_4ARG */
 
 /* bio->bi_bdev->bd_disk exists */
 /* #undef HAVE_BIO_BDEV_DISK */
 
 /* bio_*_io_acct() available */
 /* #undef HAVE_BIO_IO_ACCT */
 
 /* bio_max_segs() is implemented */
 /* #undef HAVE_BIO_MAX_SEGS */
 
 /* bio_set_dev() GPL-only */
 /* #undef HAVE_BIO_SET_DEV_GPL_ONLY */
 
 /* bio_set_dev() is a macro */
 /* #undef HAVE_BIO_SET_DEV_MACRO */
 
 /* bio_set_op_attrs is available */
 /* #undef HAVE_BIO_SET_OP_ATTRS */
 
 /* blkdev_get_by_path() exists and takes 4 args */
 /* #undef HAVE_BLKDEV_GET_BY_PATH_4ARG */
 
 /* blkdev_get_by_path() handles ERESTARTSYS */
 /* #undef HAVE_BLKDEV_GET_ERESTARTSYS */
 
 /* __blkdev_issue_discard(flags) is available */
 /* #undef HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_FLAGS */
 
 /* __blkdev_issue_discard() is available */
 /* #undef HAVE_BLKDEV_ISSUE_DISCARD_ASYNC_NOFLAGS */
 
 /* blkdev_issue_discard(flags) is available */
 /* #undef HAVE_BLKDEV_ISSUE_DISCARD_FLAGS */
 
 /* blkdev_issue_discard() is available */
 /* #undef HAVE_BLKDEV_ISSUE_DISCARD_NOFLAGS */
 
 /* blkdev_issue_secure_erase() is available */
 /* #undef HAVE_BLKDEV_ISSUE_SECURE_ERASE */
 
 /* blkdev_put() exists */
 /* #undef HAVE_BLKDEV_PUT */
 
 /* blkdev_put() accepts void* as arg 2 */
 /* #undef HAVE_BLKDEV_PUT_HOLDER */
 
 /* struct queue_limits has a features field */
 /* #undef HAVE_BLKDEV_QUEUE_LIMITS_FEATURES */
 
 /* blkdev_reread_part() exists */
 /* #undef HAVE_BLKDEV_REREAD_PART */
 
 /* blkg_tryget() is available */
 /* #undef HAVE_BLKG_TRYGET */
 
 /* blkg_tryget() GPL-only */
 /* #undef HAVE_BLKG_TRYGET_GPL_ONLY */
 
 /* blk_alloc_disk() exists */
 /* #undef HAVE_BLK_ALLOC_DISK */
 
 /* blk_alloc_disk() exists and takes 2 args */
 /* #undef HAVE_BLK_ALLOC_DISK_2ARG */
 
 /* blk_alloc_queue() expects request function */
 /* #undef HAVE_BLK_ALLOC_QUEUE_REQUEST_FN */
 
 /* blk_alloc_queue_rh() expects request function */
 /* #undef HAVE_BLK_ALLOC_QUEUE_REQUEST_FN_RH */
 
 /* blk_cleanup_disk() exists */
 /* #undef HAVE_BLK_CLEANUP_DISK */
 
 /* blk_mode_t is defined */
 /* #undef HAVE_BLK_MODE_T */
 
 /* block multiqueue hardware context is cached in struct request */
 /* #undef HAVE_BLK_MQ_RQ_HCTX */
 
 /* blk queue backing_dev_info is dynamic */
 /* #undef HAVE_BLK_QUEUE_BDI_DYNAMIC */
 
 /* blk_queue_discard() is available */
 /* #undef HAVE_BLK_QUEUE_DISCARD */
 
 /* backing_dev_info is available through queue gendisk */
 /* #undef HAVE_BLK_QUEUE_DISK_BDI */
 
 /* blk_queue_secure_erase() is available */
 /* #undef HAVE_BLK_QUEUE_SECURE_ERASE */
 
 /* blk_queue_update_readahead() exists */
 /* #undef HAVE_BLK_QUEUE_UPDATE_READAHEAD */
 
 /* BLK_STS_RESV_CONFLICT is defined */
 /* #undef HAVE_BLK_STS_RESV_CONFLICT */
 
 /* Define if release() in block_device_operations takes 1 arg */
 /* #undef HAVE_BLOCK_DEVICE_OPERATIONS_RELEASE_1ARG */
 
 /* Define if revalidate_disk() in block_device_operations */
 /* #undef HAVE_BLOCK_DEVICE_OPERATIONS_REVALIDATE_DISK */
 
+/* Define to 1 if you have the Mac OS X function
+   CFLocaleCopyPreferredLanguages in the CoreFoundation framework. */
+/* #undef HAVE_CFLOCALECOPYPREFERREDLANGUAGES */
+
+/* Define to 1 if you have the Mac OS X function CFPreferencesCopyAppValue in
+   the CoreFoundation framework. */
+/* #undef HAVE_CFPREFERENCESCOPYAPPVALUE */
+
 /* check_disk_change() exists */
 /* #undef HAVE_CHECK_DISK_CHANGE */
 
 /* copy_splice_read exists */
 /* #undef HAVE_COPY_SPLICE_READ */
 
 /* cpu_has_feature() is GPL-only */
 /* #undef HAVE_CPU_HAS_FEATURE_GPL_ONLY */
 
 /* Define if the GNU dcgettext() function is already present or preinstalled.
    */
 /* #undef HAVE_DCGETTEXT */
 
 /* DECLARE_EVENT_CLASS() is available */
 /* #undef HAVE_DECLARE_EVENT_CLASS */
 
-/* dequeue_signal() takes a task argument */
-/* #undef HAVE_DEQUEUE_SIGNAL_3ARG_TASK */
+/* 3-arg dequeue_signal() takes a type argument */
+/* #undef HAVE_DEQUEUE_SIGNAL_3ARG_TYPE */
 
 /* dequeue_signal() takes 4 arguments */
 /* #undef HAVE_DEQUEUE_SIGNAL_4ARG */
 
 /* lookup_bdev() wants dev_t arg */
 /* #undef HAVE_DEVT_LOOKUP_BDEV */
 
 /* disk_check_media_change() exists */
 /* #undef HAVE_DISK_CHECK_MEDIA_CHANGE */
 
 /* disk_*_io_acct() available */
 /* #undef HAVE_DISK_IO_ACCT */
 
 /* disk_update_readahead() exists */
 /* #undef HAVE_DISK_UPDATE_READAHEAD */
 
 /* Define to 1 if you have the <dlfcn.h> header file. */
 #define HAVE_DLFCN_H 1
 
+/* dops->d_revalidate() takes 4 args */
+/* #undef HAVE_D_REVALIDATE_4ARGS */
+
 /* Define to 1 if you have the 'execvpe' function. */
 #define HAVE_EXECVPE 1
 
 /* fault_in_iov_iter_readable() is available */
 /* #undef HAVE_FAULT_IN_IOV_ITER_READABLE */
 
 /* file->f_version exists */
 /* #undef HAVE_FILE_F_VERSION */
 
 /* flush_dcache_page() is GPL-only */
 /* #undef HAVE_FLUSH_DCACHE_PAGE_GPL_ONLY */
 
 /* Define if compiler supports -Wformat-overflow */
 /* #undef HAVE_FORMAT_OVERFLOW */
 
 /* fsync_bdev() is declared in include/blkdev.h */
 /* #undef HAVE_FSYNC_BDEV */
 
 /* yes */
 /* #undef HAVE_GENERIC_FADVISE */
 
 /* generic_fillattr requires struct mnt_idmap* */
 /* #undef HAVE_GENERIC_FILLATTR_IDMAP */
 
 /* generic_fillattr requires struct mnt_idmap* and u32 request_mask */
 /* #undef HAVE_GENERIC_FILLATTR_IDMAP_REQMASK */
 
 /* generic_fillattr requires struct user_namespace* */
 /* #undef HAVE_GENERIC_FILLATTR_USERNS */
 
 /* generic_*_io_acct() 4 arg available */
 /* #undef HAVE_GENERIC_IO_ACCT_4ARG */
 
 /* GENHD_FL_EXT_DEVT flag is available */
 /* #undef HAVE_GENHD_FL_EXT_DEVT */
 
 /* GENHD_FL_NO_PART flag is available */
 /* #undef HAVE_GENHD_FL_NO_PART */
 
 /* Define if the GNU gettext() function is already present or preinstalled. */
 /* #undef HAVE_GETTEXT */
 
 /* Define to 1 if you have the 'gettid' function. */
 /* #undef HAVE_GETTID */
 
 /* iops->get_acl() exists */
 /* #undef HAVE_GET_ACL */
 
 /* iops->get_acl() takes rcu */
 /* #undef HAVE_GET_ACL_RCU */
 
 /* has iops->get_inode_acl() */
 /* #undef HAVE_GET_INODE_ACL */
 
 /* iattr->ia_vfsuid and iattr->ia_vfsgid exist */
 /* #undef HAVE_IATTR_VFSID */
 
 /* Define if you have the iconv() function and it works. */
 #define HAVE_ICONV 1
 
 /* iops->getattr() takes struct mnt_idmap* */
 /* #undef HAVE_IDMAP_IOPS_GETATTR */
 
 /* iops->setattr() takes struct mnt_idmap* */
 /* #undef HAVE_IDMAP_IOPS_SETATTR */
 
 /* APIs for idmapped mount are present */
 /* #undef HAVE_IDMAP_MNT_API */
 
 /* mnt_idmap does not have user_namespace */
 /* #undef HAVE_IDMAP_NO_USERNS */
 
 /* Define if compiler supports -Wimplicit-fallthrough */
 /* #undef HAVE_IMPLICIT_FALLTHROUGH */
 
 /* Define if compiler supports -Winfinite-recursion */
 /* #undef HAVE_INFINITE_RECURSION */
 
 /* inode_get_atime() exists in linux/fs.h */
 /* #undef HAVE_INODE_GET_ATIME */
 
 /* inode_get_ctime() exists in linux/fs.h */
 /* #undef HAVE_INODE_GET_CTIME */
 
 /* inode_get_mtime() exists in linux/fs.h */
 /* #undef HAVE_INODE_GET_MTIME */
 
 /* inode_owner_or_capable() exists */
 /* #undef HAVE_INODE_OWNER_OR_CAPABLE */
 
 /* inode_owner_or_capable() takes mnt_idmap */
 /* #undef HAVE_INODE_OWNER_OR_CAPABLE_IDMAP */
 
 /* inode_owner_or_capable() takes user_ns */
 /* #undef HAVE_INODE_OWNER_OR_CAPABLE_USERNS */
 
 /* inode_set_atime_to_ts() exists in linux/fs.h */
 /* #undef HAVE_INODE_SET_ATIME_TO_TS */
 
 /* inode_set_ctime_to_ts() exists in linux/fs.h */
 /* #undef HAVE_INODE_SET_CTIME_TO_TS */
 
 /* inode_set_mtime_to_ts() exists in linux/fs.h */
 /* #undef HAVE_INODE_SET_MTIME_TO_TS */
 
 /* timestamp_truncate() exists */
 /* #undef HAVE_INODE_TIMESTAMP_TRUNCATE */
 
 /* Define to 1 if you have the <inttypes.h> header file. */
 #define HAVE_INTTYPES_H 1
 
 /* iops->create() takes struct mnt_idmap* */
 /* #undef HAVE_IOPS_CREATE_IDMAP */
 
 /* iops->create() takes struct user_namespace* */
 /* #undef HAVE_IOPS_CREATE_USERNS */
 
+/* iops->mkdir() returns struct dentry* */
+/* #undef HAVE_IOPS_MKDIR_DENTRY */
+
 /* iops->mkdir() takes struct mnt_idmap* */
 /* #undef HAVE_IOPS_MKDIR_IDMAP */
 
 /* iops->mkdir() takes struct user_namespace* */
 /* #undef HAVE_IOPS_MKDIR_USERNS */
 
 /* iops->mknod() takes struct mnt_idmap* */
 /* #undef HAVE_IOPS_MKNOD_IDMAP */
 
 /* iops->mknod() takes struct user_namespace* */
 /* #undef HAVE_IOPS_MKNOD_USERNS */
 
 /* iops->permission() takes struct mnt_idmap* */
 /* #undef HAVE_IOPS_PERMISSION_IDMAP */
 
 /* iops->permission() takes struct user_namespace* */
 /* #undef HAVE_IOPS_PERMISSION_USERNS */
 
 /* iops->rename() takes struct mnt_idmap* */
 /* #undef HAVE_IOPS_RENAME_IDMAP */
 
 /* iops->rename() takes struct user_namespace* */
 /* #undef HAVE_IOPS_RENAME_USERNS */
 
 /* iops->symlink() takes struct mnt_idmap* */
 /* #undef HAVE_IOPS_SYMLINK_IDMAP */
 
 /* iops->symlink() takes struct user_namespace* */
 /* #undef HAVE_IOPS_SYMLINK_USERNS */
 
 /* iov_iter_get_pages() is available */
 /* #undef HAVE_IOV_ITER_GET_PAGES */
 
 /* iov_iter_get_pages2() is available */
 /* #undef HAVE_IOV_ITER_GET_PAGES2 */
 
 /* iov_iter_type() is available */
 /* #undef HAVE_IOV_ITER_TYPE */
 
 /* Define to 1 if you have the 'issetugid' function. */
 #define HAVE_ISSETUGID 1
 
 /* iter_iov() is available */
 /* #undef HAVE_ITER_IOV */
 
 /* kernel has kernel_fpu_* functions */
 /* #undef HAVE_KERNEL_FPU */
 
 /* kernel has asm/fpu/api.h */
 /* #undef HAVE_KERNEL_FPU_API_HEADER */
 
 /* kernel fpu internal */
 /* #undef HAVE_KERNEL_FPU_INTERNAL */
 
 /* kernel has asm/fpu/internal.h */
 /* #undef HAVE_KERNEL_FPU_INTERNAL_HEADER */
 
 /* Define if compiler supports -Winfinite-recursion */
 /* #undef HAVE_KERNEL_INFINITE_RECURSION */
 
 /* kernel defines intptr_t */
 /* #undef HAVE_KERNEL_INTPTR_T */
 
 /* kernel has kernel_neon_* functions */
 /* #undef HAVE_KERNEL_NEON */
 
 /* kernel does stack verification */
 /* #undef HAVE_KERNEL_OBJTOOL */
 
 /* kernel has linux/objtool.h */
 /* #undef HAVE_KERNEL_OBJTOOL_HEADER */
 
 /* strlcpy() exists */
 /* #undef HAVE_KERNEL_STRLCPY */
 
 /* kernel has kmap_local_page */
 /* #undef HAVE_KMAP_LOCAL_PAGE */
 
 /* Define if you have [aio] */
 /* #undef HAVE_LIBAIO */
 
 /* Define if you have [blkid] */
 /* #undef HAVE_LIBBLKID */
 
 /* Define if you have [crypto] */
 #define HAVE_LIBCRYPTO 1
 
 /* Define if you have [tirpc] */
 /* #undef HAVE_LIBTIRPC */
 
 /* Define if you have [udev] */
 /* #undef HAVE_LIBUDEV */
 
 /* Define if you have [unwind] */
 /* #undef HAVE_LIBUNWIND */
 
 /* libunwind has unw_get_elf_filename */
 /* #undef HAVE_LIBUNWIND_ELF */
 
 /* Define if you have [uuid] */
 /* #undef HAVE_LIBUUID */
 
 /* makedev() is declared in sys/mkdev.h */
 /* #undef HAVE_MAKEDEV_IN_MKDEV */
 
 /* makedev() is declared in sys/sysmacros.h */
 /* #undef HAVE_MAKEDEV_IN_SYSMACROS */
 
 /* Noting that make_request_fn() returns blk_qc_t */
 /* #undef HAVE_MAKE_REQUEST_FN_RET_QC */
 
 /* Define to 1 if you have the 'mlockall' function. */
 #define HAVE_MLOCKALL 1
 
 /* PG_error flag is available */
 /* #undef HAVE_MM_PAGE_FLAG_ERROR */
 
 /* page_mapping() is available */
 /* #undef HAVE_MM_PAGE_MAPPING */
 
 /* page_size() is available */
 /* #undef HAVE_MM_PAGE_SIZE */
 
 /* Define if host toolchain supports MOVBE */
 #define HAVE_MOVBE 1
 
 /* folio_wait_bit() exists */
 /* #undef HAVE_PAGEMAP_FOLIO_WAIT_BIT */
 
 /* part_to_dev() exists */
 /* #undef HAVE_PART_TO_DEV */
 
 /* iops->getattr() takes a path */
 /* #undef HAVE_PATH_IOPS_GETATTR */
 
 /* Define if host toolchain supports PCLMULQDQ */
 #define HAVE_PCLMULQDQ 1
 
 /* proc_handler ctl_table arg is const */
 /* #undef HAVE_PROC_HANDLER_CTL_TABLE_CONST */
 
 /* proc_ops structure exists */
 /* #undef HAVE_PROC_OPS_STRUCT */
 
 /* If available, contains the Python version number currently in use. */
 /* #undef HAVE_PYTHON */
 
 /* qat is enabled and existed */
 /* #undef HAVE_QAT */
 
 /* struct reclaim_state has reclaimed */
 /* #undef HAVE_RECLAIM_STATE_RECLAIMED */
 
 /* register_shrinker is vararg */
 /* #undef HAVE_REGISTER_SHRINKER_VARARG */
 
 /* register_sysctl_sz exists */
 /* #undef HAVE_REGISTER_SYSCTL_SZ */
 
 /* register_sysctl_table exists */
 /* #undef HAVE_REGISTER_SYSCTL_TABLE */
 
 /* iops->rename() wants flags */
 /* #undef HAVE_RENAME_WANTS_FLAGS */
 
 /* revalidate_disk() is available */
 /* #undef HAVE_REVALIDATE_DISK */
 
 /* revalidate_disk_size() is available */
 /* #undef HAVE_REVALIDATE_DISK_SIZE */
 
 /* Define to 1 if you have the <security/pam_modules.h> header file. */
 #define HAVE_SECURITY_PAM_MODULES_H 1
 
 /* setattr_prepare() accepts mnt_idmap */
 /* #undef HAVE_SETATTR_PREPARE_IDMAP */
 
 /* setattr_prepare() is available, doesn't accept user_namespace */
 /* #undef HAVE_SETATTR_PREPARE_NO_USERNS */
 
 /* setattr_prepare() accepts user_namespace */
 /* #undef HAVE_SETATTR_PREPARE_USERNS */
 
 /* iops->set_acl() takes 4 args, arg1 is struct mnt_idmap * */
 /* #undef HAVE_SET_ACL_IDMAP_DENTRY */
 
 /* iops->set_acl() takes 4 args */
 /* #undef HAVE_SET_ACL_USERNS */
 
 /* iops->set_acl() takes 4 args, arg2 is struct dentry * */
 /* #undef HAVE_SET_ACL_USERNS_DENTRY_ARG2 */
 
 /* shrinker_register exists */
 /* #undef HAVE_SHRINKER_REGISTER */
 
 /* kernel_siginfo_t exists */
 /* #undef HAVE_SIGINFO */
 
 #if defined(__amd64__) || defined(__i386__)
 /* Define if host toolchain supports SSE */
 #define HAVE_SSE 1
 
 /* Define if host toolchain supports SSE2 */
 #define HAVE_SSE2 1
 
 /* Define if host toolchain supports SSE3 */
 #define HAVE_SSE3 1
 
 /* Define if host toolchain supports SSE4.1 */
 #define HAVE_SSE4_1 1
 
 /* Define if host toolchain supports SSE4.2 */
 #define HAVE_SSE4_2 1
 
 /* Define if host toolchain supports SSSE3 */
 #define HAVE_SSSE3 1
 #endif
 
 /* STACK_FRAME_NON_STANDARD is defined */
 /* #undef HAVE_STACK_FRAME_NON_STANDARD */
 
+/* STACK_FRAME_NON_STANDARD asm macro is defined */
+/* #undef HAVE_STACK_FRAME_NON_STANDARD_ASM */
+
 /* standalone <linux/stdarg.h> exists */
 /* #undef HAVE_STANDALONE_LINUX_STDARG */
 
 /* Define to 1 if you have the <stdint.h> header file. */
 #define HAVE_STDINT_H 1
 
 /* Define to 1 if you have the <stdio.h> header file. */
 #define HAVE_STDIO_H 1
 
 /* Define to 1 if you have the <stdlib.h> header file. */
 #define HAVE_STDLIB_H 1
 
 /* Define to 1 if you have the 'strerror_l' function. */
 #define HAVE_STRERROR_L 1
 
 /* Define to 1 if you have the <strings.h> header file. */
 #define HAVE_STRINGS_H 1
 
 /* Define to 1 if you have the <string.h> header file. */
 #define HAVE_STRING_H 1
 
 /* Define to 1 if you have the 'strlcat' function. */
 #define HAVE_STRLCAT 1
 
 /* Define to 1 if you have the 'strlcpy' function. */
 #define HAVE_STRLCPY 1
 
 /* submit_bio is member of struct block_device_operations */
 /* #undef HAVE_SUBMIT_BIO_IN_BLOCK_DEVICE_OPERATIONS */
 
 /* have super_block s_shrink */
 /* #undef HAVE_SUPER_BLOCK_S_SHRINK */
 
 /* have super_block s_shrink pointer */
 /* #undef HAVE_SUPER_BLOCK_S_SHRINK_PTR */
 
 /* sync_blockdev() is declared in include/blkdev.h */
 /* #undef HAVE_SYNC_BLOCKDEV */
 
 /* struct kobj_type has default_groups */
 /* #undef HAVE_SYSFS_DEFAULT_GROUPS */
 
 /* Define to 1 if you have the <sys/stat.h> header file. */
 #define HAVE_SYS_STAT_H 1
 
 /* Define to 1 if you have the <sys/types.h> header file. */
 #define HAVE_SYS_TYPES_H 1
 
+/* timer_delete_sync is available */
+/* #undef HAVE_TIMER_DELETE_SYNC */
+
 /* i_op->tmpfile() uses old dentry signature */
 /* #undef HAVE_TMPFILE_DENTRY */
 
 /* i_op->tmpfile() has mnt_idmap */
 /* #undef HAVE_TMPFILE_IDMAP */
 
 /* i_op->tmpfile() has userns */
 /* #undef HAVE_TMPFILE_USERNS */
 
 /* totalhigh_pages() exists */
 /* #undef HAVE_TOTALHIGH_PAGES */
 
 /* kernel has totalram_pages() */
 /* #undef HAVE_TOTALRAM_PAGES_FUNC */
 
 /* Define to 1 if you have the 'udev_device_get_is_initialized' function. */
 /* #undef HAVE_UDEV_DEVICE_GET_IS_INITIALIZED */
 
 /* kernel has __kernel_fpu_* functions */
 /* #undef HAVE_UNDERSCORE_KERNEL_FPU */
 
 /* Define to 1 if you have the <unistd.h> header file. */
 #define HAVE_UNISTD_H 1
 
 /* iops->getattr() takes struct user_namespace* */
 /* #undef HAVE_USERNS_IOPS_GETATTR */
 
 /* iops->setattr() takes struct user_namespace* */
 /* #undef HAVE_USERNS_IOPS_SETATTR */
 
 /* fops->clone_file_range() is available */
 /* #undef HAVE_VFS_CLONE_FILE_RANGE */
 
 /* fops->dedupe_file_range() is available */
 /* #undef HAVE_VFS_DEDUPE_FILE_RANGE */
 
 /* aops->direct_IO() uses iov_iter without rw */
 /* #undef HAVE_VFS_DIRECT_IO_ITER */
 
 /* aops->direct_IO() uses iov_iter with offset */
 /* #undef HAVE_VFS_DIRECT_IO_ITER_OFFSET */
 
 /* filemap_dirty_folio exists */
 /* #undef HAVE_VFS_FILEMAP_DIRTY_FOLIO */
 
 /* generic_copy_file_range() is available */
 /* #undef HAVE_VFS_GENERIC_COPY_FILE_RANGE */
 
 /* All required iov_iter interfaces are available */
 /* #undef HAVE_VFS_IOV_ITER */
 
+/* migratepage exists */
+/* #undef HAVE_VFS_MIGRATEPAGE */
+
+/* migrate_folio exists */
+/* #undef HAVE_VFS_MIGRATE_FOLIO */
+
 /* address_space_operations->readpages exists */
 /* #undef HAVE_VFS_READPAGES */
 
 /* read_folio exists */
 /* #undef HAVE_VFS_READ_FOLIO */
 
 /* fops->remap_file_range() is available */
 /* #undef HAVE_VFS_REMAP_FILE_RANGE */
 
 /* __set_page_dirty_nobuffers exists */
 /* #undef HAVE_VFS_SET_PAGE_DIRTY_NOBUFFERS */
 
 /* splice_copy_file_range() is available */
 /* #undef HAVE_VFS_SPLICE_COPY_FILE_RANGE */
 
 /* __vmalloc page flags exists */
 /* #undef HAVE_VMALLOC_PAGE_KERNEL */
 
 /* int (*writepage_t)() takes struct folio* */
 /* #undef HAVE_WRITEPAGE_T_FOLIO */
 
 /* xattr_handler->get() wants dentry and inode and flags */
 /* #undef HAVE_XATTR_GET_DENTRY_INODE_FLAGS */
 
 /* xattr_handler->set() wants both dentry and inode */
 /* #undef HAVE_XATTR_SET_DENTRY_INODE */
 
 /* xattr_handler->set() takes mnt_idmap */
 /* #undef HAVE_XATTR_SET_IDMAP */
 
 /* xattr_handler->set() takes user_namespace */
 /* #undef HAVE_XATTR_SET_USERNS */
 
 /* Define if host toolchain supports XSAVE */
 #define HAVE_XSAVE 1
 
 /* Define if host toolchain supports XSAVEOPT */
 #define HAVE_XSAVEOPT 1
 
 /* Define if host toolchain supports XSAVES */
 #define HAVE_XSAVES 1
 
 /* ZERO_PAGE() is GPL-only */
 /* #undef HAVE_ZERO_PAGE_GPL_ONLY */
 
 /* Define if you have [z] */
 #define HAVE_ZLIB 1
 
 /* kernel exports FPU functions */
 /* #undef KERNEL_EXPORTS_X86_FPU */
 
 /* TBD: fetch(3) support */
 #if 0
 /* whether the chosen libfetch is to be loaded at run-time */
 #define LIBFETCH_DYNAMIC 1
 
 /* libfetch is fetch(3) */
 #define LIBFETCH_IS_FETCH 1
 
 /* libfetch is libcurl */
 #define LIBFETCH_IS_LIBCURL 0
 
 /* soname of chosen libfetch */
 #define LIBFETCH_SONAME "libfetch.so.6"
 #endif
 
 /* Define to the sub-directory where libtool stores uninstalled libraries. */
 #define LT_OBJDIR ".libs/"
 
 /* make_request_fn() return type */
 /* #undef MAKE_REQUEST_FN_RET */
 
 /* using complete_and_exit() instead */
 /* #undef SPL_KTHREAD_COMPLETE_AND_EXIT */
 
 /* Defined for legacy compatibility. */
 #define SPL_META_ALIAS ZFS_META_ALIAS
 
 /* Defined for legacy compatibility. */
 #define SPL_META_RELEASE ZFS_META_RELEASE
 
 /* Defined for legacy compatibility. */
 #define SPL_META_VERSION ZFS_META_VERSION
 
 /* pde_data() is PDE_DATA() */
 /* #undef SPL_PDE_DATA */
 
 /* Define to 1 if all of the C89 standard headers exist (not just the ones
    required in a freestanding environment). This macro is provided for
    backward compatibility; new code need not use it. */
 #define SYSTEM_FREEBSD 1
 
 /* True if ZFS is to be compiled for a Linux system */
 /* #undef SYSTEM_LINUX */
 
 /* Version number of package */
 /* #undef ZFS_DEBUG */
 
 /* /dev/zfs minor */
 /* #undef ZFS_DEVICE_MINOR */
 
 /* Define the project alias string. */
-#define ZFS_META_ALIAS "zfs-2.2.7-FreeBSD_ge269af1b3"
+#define ZFS_META_ALIAS "zfs-2.2.8-FreeBSD_g3e4a3e161"
 
 /* Define the project author. */
 #define ZFS_META_AUTHOR "OpenZFS"
 
 /* Define the project release date. */
 /* #undef ZFS_META_DATA */
 
 /* Define the maximum compatible kernel version. */
-#define ZFS_META_KVER_MAX "6.12"
+#define ZFS_META_KVER_MAX "6.15"
 
 /* Define the minimum compatible kernel version. */
 #define ZFS_META_KVER_MIN "4.18"
 
 /* Define the project license. */
 #define ZFS_META_LICENSE "CDDL"
 
 /* Define the libtool library 'age' version information. */
 /* #undef ZFS_META_LT_AGE */
 
 /* Define the libtool library 'current' version information. */
 /* #undef ZFS_META_LT_CURRENT */
 
 /* Define the libtool library 'revision' version information. */
 /* #undef ZFS_META_LT_REVISION */
 
 /* Define the project name. */
 #define ZFS_META_NAME "zfs"
 
 /* Define the project release. */
-#define ZFS_META_RELEASE "FreeBSD_ge269af1b3"
+#define ZFS_META_RELEASE "FreeBSD_g3e4a3e161"
 
 /* Define the project version. */
-#define ZFS_META_VERSION "2.2.7"
+#define ZFS_META_VERSION "2.2.8"
 
 /* count is located in percpu_ref.data */
 /* #undef ZFS_PERCPU_REF_COUNT_IN_DATA */
diff --git a/sys/modules/zfs/zfs_gitrev.h b/sys/modules/zfs/zfs_gitrev.h
index 9d202e3893f4..7a81682b6bc3 100644
--- a/sys/modules/zfs/zfs_gitrev.h
+++ b/sys/modules/zfs/zfs_gitrev.h
@@ -1 +1 @@
-#define	ZFS_META_GITREV "zfs-2.2.7-0-ge269af1b3"
+#define	ZFS_META_GITREV "zfs-2.2.8-0-g3e4a3e161"