diff --git a/config/always-system.m4 b/config/always-system.m4 new file mode 100644 index 000000000000..3225a52af8ae --- /dev/null +++ b/config/always-system.m4 @@ -0,0 +1,26 @@ +dnl # +dnl # Set the target system +dnl # +AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_SYSTEM], [ + AC_MSG_CHECKING([for system type ($host_os)]) + case $host_os in + *linux*) + AC_DEFINE([SYSTEM_LINUX], [1], + [True if ZFS is to be compiled for a Linux system]) + ac_system="Linux" + ;; + *freebsd*) + AC_DEFINE([SYSTEM_FREEBSD], [1], + [True if ZFS is to be compiled for a FreeBSD system]) + ac_system="FreeBSD" + ;; + *) + ac_system="unknown" + ;; + esac + AC_MSG_RESULT([$ac_system]) + AC_SUBST([ac_system]) + + AM_CONDITIONAL([BUILD_LINUX], [test "x$ac_system" = "xLinux"]) + AM_CONDITIONAL([BUILD_FREEBSD], [test "x$ac_system" = "xFreeBSD"]) +]) diff --git a/config/zfs-build.m4 b/config/zfs-build.m4 index c2e5bb25fe2e..da8cccc39314 100644 --- a/config/zfs-build.m4 +++ b/config/zfs-build.m4 @@ -1,498 +1,499 @@ AC_DEFUN([ZFS_AC_LICENSE], [ AC_MSG_CHECKING([zfs author]) AC_MSG_RESULT([$ZFS_META_AUTHOR]) AC_MSG_CHECKING([zfs license]) AC_MSG_RESULT([$ZFS_META_LICENSE]) ]) AC_DEFUN([ZFS_AC_DEBUG_ENABLE], [ DEBUG_CFLAGS="-Werror" DEBUG_CPPFLAGS="-DDEBUG -UNDEBUG" DEBUG_LDFLAGS="" DEBUG_ZFS="_with_debug" AC_DEFINE(ZFS_DEBUG, 1, [zfs debugging enabled]) KERNEL_DEBUG_CFLAGS="-Werror" KERNEL_DEBUG_CPPFLAGS="-DDEBUG -UNDEBUG" ]) AC_DEFUN([ZFS_AC_DEBUG_DISABLE], [ DEBUG_CFLAGS="" DEBUG_CPPFLAGS="-UDEBUG -DNDEBUG" DEBUG_LDFLAGS="" DEBUG_ZFS="_without_debug" KERNEL_DEBUG_CFLAGS="" KERNEL_DEBUG_CPPFLAGS="-UDEBUG -DNDEBUG" ]) dnl # dnl # When debugging is enabled: dnl # - Enable all ASSERTs (-DDEBUG) dnl # - Promote all compiler warnings to errors (-Werror) dnl # AC_DEFUN([ZFS_AC_DEBUG], [ AC_MSG_CHECKING([whether assertion support will be enabled]) AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug], [Enable compiler and code assertions @<:@default=no@:>@])], [], [enable_debug=no]) AS_CASE(["x$enable_debug"], ["xyes"], [ZFS_AC_DEBUG_ENABLE], ["xno"], [ZFS_AC_DEBUG_DISABLE], [AC_MSG_ERROR([Unknown option $enable_debug])]) AC_SUBST(DEBUG_CFLAGS) AC_SUBST(DEBUG_CPPFLAGS) AC_SUBST(DEBUG_LDFLAGS) AC_SUBST(DEBUG_ZFS) AC_SUBST(KERNEL_DEBUG_CFLAGS) AC_SUBST(KERNEL_DEBUG_CPPFLAGS) AC_MSG_RESULT([$enable_debug]) ]) AC_DEFUN([ZFS_AC_DEBUGINFO_ENABLE], [ DEBUG_CFLAGS="$DEBUG_CFLAGS -g -fno-inline" KERNEL_DEBUG_CFLAGS="$KERNEL_DEBUG_CFLAGS -fno-inline" KERNEL_MAKE="$KERNEL_MAKE CONFIG_DEBUG_INFO=y" DEBUGINFO_ZFS="_with_debuginfo" ]) AC_DEFUN([ZFS_AC_DEBUGINFO_DISABLE], [ DEBUGINFO_ZFS="_without_debuginfo" ]) AC_DEFUN([ZFS_AC_DEBUGINFO], [ AC_MSG_CHECKING([whether debuginfo support will be forced]) AC_ARG_ENABLE([debuginfo], [AS_HELP_STRING([--enable-debuginfo], [Force generation of debuginfo @<:@default=no@:>@])], [], [enable_debuginfo=no]) AS_CASE(["x$enable_debuginfo"], ["xyes"], [ZFS_AC_DEBUGINFO_ENABLE], ["xno"], [ZFS_AC_DEBUGINFO_DISABLE], [AC_MSG_ERROR([Unknown option $enable_debuginfo])]) AC_SUBST(DEBUG_CFLAGS) AC_SUBST(DEBUGINFO_ZFS) AC_SUBST(KERNEL_DEBUG_CFLAGS) AC_SUBST(KERNEL_MAKE) AC_MSG_RESULT([$enable_debuginfo]) ]) dnl # dnl # Disabled by default, provides basic memory tracking. Track the total dnl # number of bytes allocated with kmem_alloc() and freed with kmem_free(). dnl # Then at module unload time if any bytes were leaked it will be reported dnl # on the console. dnl # AC_DEFUN([ZFS_AC_DEBUG_KMEM], [ AC_MSG_CHECKING([whether basic kmem accounting is enabled]) AC_ARG_ENABLE([debug-kmem], [AS_HELP_STRING([--enable-debug-kmem], [Enable basic kmem accounting @<:@default=no@:>@])], [], [enable_debug_kmem=no]) AS_IF([test "x$enable_debug_kmem" = xyes], [ KERNEL_DEBUG_CPPFLAGS+=" -DDEBUG_KMEM" DEBUG_KMEM_ZFS="_with_debug_kmem" ], [ DEBUG_KMEM_ZFS="_without_debug_kmem" ]) AC_SUBST(KERNEL_DEBUG_CPPFLAGS) AC_SUBST(DEBUG_KMEM_ZFS) AC_MSG_RESULT([$enable_debug_kmem]) ]) dnl # dnl # Disabled by default, provides detailed memory tracking. This feature dnl # also requires --enable-debug-kmem to be set. When enabled not only will dnl # total bytes be tracked but also the location of every kmem_alloc() and dnl # kmem_free(). When the module is unloaded a list of all leaked addresses dnl # and where they were allocated will be dumped to the console. Enabling dnl # this feature has a significant impact on performance but it makes finding dnl # memory leaks straight forward. dnl # AC_DEFUN([ZFS_AC_DEBUG_KMEM_TRACKING], [ AC_MSG_CHECKING([whether detailed kmem tracking is enabled]) AC_ARG_ENABLE([debug-kmem-tracking], [AS_HELP_STRING([--enable-debug-kmem-tracking], [Enable detailed kmem tracking @<:@default=no@:>@])], [], [enable_debug_kmem_tracking=no]) AS_IF([test "x$enable_debug_kmem_tracking" = xyes], [ KERNEL_DEBUG_CPPFLAGS+=" -DDEBUG_KMEM_TRACKING" DEBUG_KMEM_TRACKING_ZFS="_with_debug_kmem_tracking" ], [ DEBUG_KMEM_TRACKING_ZFS="_without_debug_kmem_tracking" ]) AC_SUBST(KERNEL_DEBUG_CPPFLAGS) AC_SUBST(DEBUG_KMEM_TRACKING_ZFS) AC_MSG_RESULT([$enable_debug_kmem_tracking]) ]) AC_DEFUN([ZFS_AC_CONFIG_ALWAYS], [ ZFS_AC_CONFIG_ALWAYS_CC_NO_UNUSED_BUT_SET_VARIABLE ZFS_AC_CONFIG_ALWAYS_CC_NO_BOOL_COMPARE ZFS_AC_CONFIG_ALWAYS_CC_FRAME_LARGER_THAN ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION ZFS_AC_CONFIG_ALWAYS_CC_NO_OMIT_FRAME_POINTER ZFS_AC_CONFIG_ALWAYS_CC_ASAN ZFS_AC_CONFIG_ALWAYS_TOOLCHAIN_SIMD + ZFS_AC_CONFIG_ALWAYS_SYSTEM ZFS_AC_CONFIG_ALWAYS_ARCH ZFS_AC_CONFIG_ALWAYS_PYTHON ZFS_AC_CONFIG_ALWAYS_PYZFS ]) AC_DEFUN([ZFS_AC_CONFIG], [ ZFS_CONFIG=all AC_ARG_WITH([config], AS_HELP_STRING([--with-config=CONFIG], [Config file 'kernel|user|all|srpm']), [ZFS_CONFIG="$withval"]) AC_ARG_ENABLE([linux-builtin], [AC_HELP_STRING([--enable-linux-builtin], [Configure for builtin in-tree kernel modules @<:@default=no@:>@])], [], [enable_linux_builtin=no]) AC_MSG_CHECKING([zfs config]) AC_MSG_RESULT([$ZFS_CONFIG]); AC_SUBST(ZFS_CONFIG) ZFS_AC_CONFIG_ALWAYS case "$ZFS_CONFIG" in kernel) ZFS_AC_CONFIG_KERNEL ;; user) ZFS_AC_CONFIG_USER ;; all) ZFS_AC_CONFIG_USER ZFS_AC_CONFIG_KERNEL ;; srpm) ;; *) AC_MSG_RESULT([Error!]) AC_MSG_ERROR([Bad value "$ZFS_CONFIG" for --with-config, user kernel|user|all|srpm]) ;; esac AM_CONDITIONAL([CONFIG_USER], [test "$ZFS_CONFIG" = user -o "$ZFS_CONFIG" = all]) AM_CONDITIONAL([CONFIG_KERNEL], [test "$ZFS_CONFIG" = kernel -o "$ZFS_CONFIG" = all] && [test "x$enable_linux_builtin" != xyes ]) AM_CONDITIONAL([CONFIG_QAT], [test "$ZFS_CONFIG" = kernel -o "$ZFS_CONFIG" = all] && [test "x$qatsrc" != x ]) AM_CONDITIONAL([WANT_DEVNAME2DEVID], [test "x$user_libudev" = xyes ]) AM_CONDITIONAL([WANT_MMAP_LIBAIO], [test "x$user_libaio" = xyes ]) ]) dnl # dnl # Check for rpm+rpmbuild to build RPM packages. If these tools dnl # are missing it is non-fatal but you will not be able to build dnl # RPM packages and will be warned if you try too. dnl # dnl # By default the generic spec file will be used because it requires dnl # minimal dependencies. Distribution specific spec files can be dnl # placed under the 'rpm/' directory and enabled using dnl # the --with-spec= configure option. dnl # AC_DEFUN([ZFS_AC_RPM], [ RPM=rpm RPMBUILD=rpmbuild AC_MSG_CHECKING([whether $RPM is available]) AS_IF([tmp=$($RPM --version 2>/dev/null)], [ RPM_VERSION=$(echo $tmp | $AWK '/RPM/ { print $[3] }') HAVE_RPM=yes AC_MSG_RESULT([$HAVE_RPM ($RPM_VERSION)]) ],[ HAVE_RPM=no AC_MSG_RESULT([$HAVE_RPM]) ]) AC_MSG_CHECKING([whether $RPMBUILD is available]) AS_IF([tmp=$($RPMBUILD --version 2>/dev/null)], [ RPMBUILD_VERSION=$(echo $tmp | $AWK '/RPM/ { print $[3] }') HAVE_RPMBUILD=yes AC_MSG_RESULT([$HAVE_RPMBUILD ($RPMBUILD_VERSION)]) ],[ HAVE_RPMBUILD=no AC_MSG_RESULT([$HAVE_RPMBUILD]) ]) RPM_DEFINE_COMMON='--define "$(DEBUG_ZFS) 1"' RPM_DEFINE_COMMON+=' --define "$(DEBUG_KMEM_ZFS) 1"' RPM_DEFINE_COMMON+=' --define "$(DEBUG_KMEM_TRACKING_ZFS) 1"' RPM_DEFINE_COMMON+=' --define "$(DEBUGINFO_ZFS) 1"' RPM_DEFINE_COMMON+=' --define "$(ASAN_ZFS) 1"' RPM_DEFINE_UTIL=' --define "_initconfdir $(DEFAULT_INITCONF_DIR)"' dnl # Make the next three RPM_DEFINE_UTIL additions conditional, since dnl # their values may not be set when running: dnl # dnl # ./configure --with-config=srpm dnl # AS_IF([test -n "$dracutdir" ], [ RPM_DEFINE_UTIL='--define "_dracutdir $(dracutdir)"' ]) AS_IF([test -n "$udevdir" ], [ RPM_DEFINE_UTIL+=' --define "_udevdir $(udevdir)"' ]) AS_IF([test -n "$udevruledir" ], [ RPM_DEFINE_UTIL+=' --define "_udevdir $(udevruledir)"' ]) RPM_DEFINE_UTIL+=' $(DEFINE_INITRAMFS)' RPM_DEFINE_UTIL+=' $(DEFINE_SYSTEMD)' RPM_DEFINE_UTIL+=' $(DEFINE_PYZFS)' RPM_DEFINE_UTIL+=' $(DEFINE_PYTHON_VERSION)' RPM_DEFINE_UTIL+=' $(DEFINE_PYTHON_PKG_VERSION)' dnl # Override default lib directory on Debian/Ubuntu systems. The dnl # provided /usr/lib/rpm/platform//macros files do not dnl # specify the correct path for multiarch systems as described dnl # by the packaging guidelines. dnl # dnl # https://wiki.ubuntu.com/MultiarchSpec dnl # https://wiki.debian.org/Multiarch/Implementation dnl # AS_IF([test "$DEFAULT_PACKAGE" = "deb"], [ MULTIARCH_LIBDIR="lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)" RPM_DEFINE_UTIL+=' --define "_lib $(MULTIARCH_LIBDIR)"' AC_SUBST(MULTIARCH_LIBDIR) ]) RPM_DEFINE_KMOD='--define "kernels $(LINUX_VERSION)"' RPM_DEFINE_KMOD+=' --define "ksrc $(LINUX)"' RPM_DEFINE_KMOD+=' --define "kobj $(LINUX_OBJ)"' RPM_DEFINE_KMOD+=' --define "_wrong_version_format_terminate_build 0"' RPM_DEFINE_DKMS='' SRPM_DEFINE_COMMON='--define "build_src_rpm 1"' SRPM_DEFINE_UTIL= SRPM_DEFINE_KMOD= SRPM_DEFINE_DKMS= RPM_SPEC_DIR="rpm/generic" AC_ARG_WITH([spec], AS_HELP_STRING([--with-spec=SPEC], [Spec files 'generic|redhat']), [RPM_SPEC_DIR="rpm/$withval"]) AC_MSG_CHECKING([whether spec files are available]) AC_MSG_RESULT([yes ($RPM_SPEC_DIR/*.spec.in)]) AC_SUBST(HAVE_RPM) AC_SUBST(RPM) AC_SUBST(RPM_VERSION) AC_SUBST(HAVE_RPMBUILD) AC_SUBST(RPMBUILD) AC_SUBST(RPMBUILD_VERSION) AC_SUBST(RPM_SPEC_DIR) AC_SUBST(RPM_DEFINE_UTIL) AC_SUBST(RPM_DEFINE_KMOD) AC_SUBST(RPM_DEFINE_DKMS) AC_SUBST(RPM_DEFINE_COMMON) AC_SUBST(SRPM_DEFINE_UTIL) AC_SUBST(SRPM_DEFINE_KMOD) AC_SUBST(SRPM_DEFINE_DKMS) AC_SUBST(SRPM_DEFINE_COMMON) ]) dnl # dnl # Check for dpkg+dpkg-buildpackage to build DEB packages. If these dnl # tools are missing it is non-fatal but you will not be able to build dnl # DEB packages and will be warned if you try too. dnl # AC_DEFUN([ZFS_AC_DPKG], [ DPKG=dpkg DPKGBUILD=dpkg-buildpackage AC_MSG_CHECKING([whether $DPKG is available]) AS_IF([tmp=$($DPKG --version 2>/dev/null)], [ DPKG_VERSION=$(echo $tmp | $AWK '/Debian/ { print $[7] }') HAVE_DPKG=yes AC_MSG_RESULT([$HAVE_DPKG ($DPKG_VERSION)]) ],[ HAVE_DPKG=no AC_MSG_RESULT([$HAVE_DPKG]) ]) AC_MSG_CHECKING([whether $DPKGBUILD is available]) AS_IF([tmp=$($DPKGBUILD --version 2>/dev/null)], [ DPKGBUILD_VERSION=$(echo $tmp | \ $AWK '/Debian/ { print $[4] }' | cut -f-4 -d'.') HAVE_DPKGBUILD=yes AC_MSG_RESULT([$HAVE_DPKGBUILD ($DPKGBUILD_VERSION)]) ],[ HAVE_DPKGBUILD=no AC_MSG_RESULT([$HAVE_DPKGBUILD]) ]) AC_SUBST(HAVE_DPKG) AC_SUBST(DPKG) AC_SUBST(DPKG_VERSION) AC_SUBST(HAVE_DPKGBUILD) AC_SUBST(DPKGBUILD) AC_SUBST(DPKGBUILD_VERSION) ]) dnl # dnl # Until native packaging for various different packing systems dnl # can be added the least we can do is attempt to use alien to dnl # convert the RPM packages to the needed package type. This is dnl # a hack but so far it has worked reasonable well. dnl # AC_DEFUN([ZFS_AC_ALIEN], [ ALIEN=alien AC_MSG_CHECKING([whether $ALIEN is available]) AS_IF([tmp=$($ALIEN --version 2>/dev/null)], [ ALIEN_VERSION=$(echo $tmp | $AWK '{ print $[3] }') HAVE_ALIEN=yes AC_MSG_RESULT([$HAVE_ALIEN ($ALIEN_VERSION)]) ],[ HAVE_ALIEN=no AC_MSG_RESULT([$HAVE_ALIEN]) ]) AC_SUBST(HAVE_ALIEN) AC_SUBST(ALIEN) AC_SUBST(ALIEN_VERSION) ]) dnl # dnl # Using the VENDOR tag from config.guess set the default dnl # package type for 'make pkg': (rpm | deb | tgz) dnl # AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ AC_MSG_CHECKING([linux distribution]) if test -f /etc/toss-release ; then VENDOR=toss ; elif test -f /etc/fedora-release ; then VENDOR=fedora ; elif test -f /etc/redhat-release ; then VENDOR=redhat ; elif test -f /etc/gentoo-release ; then VENDOR=gentoo ; elif test -f /etc/arch-release ; then VENDOR=arch ; elif test -f /etc/SuSE-release ; then VENDOR=sles ; elif test -f /etc/slackware-version ; then VENDOR=slackware ; elif test -f /etc/lunar.release ; then VENDOR=lunar ; elif test -f /etc/lsb-release ; then VENDOR=ubuntu ; elif test -f /etc/debian_version ; then VENDOR=debian ; elif test -f /etc/alpine-release ; then VENDOR=alpine ; else VENDOR= ; fi AC_MSG_RESULT([$VENDOR]) AC_SUBST(VENDOR) AC_MSG_CHECKING([default package type]) case "$VENDOR" in toss) DEFAULT_PACKAGE=rpm ;; redhat) DEFAULT_PACKAGE=rpm ;; fedora) DEFAULT_PACKAGE=rpm ;; gentoo) DEFAULT_PACKAGE=tgz ;; alpine) DEFAULT_PACKAGE=tgz ;; arch) DEFAULT_PACKAGE=tgz ;; sles) DEFAULT_PACKAGE=rpm ;; slackware) DEFAULT_PACKAGE=tgz ;; lunar) DEFAULT_PACKAGE=tgz ;; ubuntu) DEFAULT_PACKAGE=deb ;; debian) DEFAULT_PACKAGE=deb ;; *) DEFAULT_PACKAGE=rpm ;; esac AC_MSG_RESULT([$DEFAULT_PACKAGE]) AC_SUBST(DEFAULT_PACKAGE) DEFAULT_INIT_DIR=$sysconfdir/init.d AC_MSG_CHECKING([default init directory]) AC_MSG_RESULT([$DEFAULT_INIT_DIR]) AC_SUBST(DEFAULT_INIT_DIR) AC_MSG_CHECKING([default init script type]) case "$VENDOR" in toss) DEFAULT_INIT_SCRIPT=redhat ;; redhat) DEFAULT_INIT_SCRIPT=redhat ;; fedora) DEFAULT_INIT_SCRIPT=fedora ;; gentoo) DEFAULT_INIT_SCRIPT=openrc ;; alpine) DEFAULT_INIT_SCRIPT=openrc ;; arch) DEFAULT_INIT_SCRIPT=lsb ;; sles) DEFAULT_INIT_SCRIPT=lsb ;; slackware) DEFAULT_INIT_SCRIPT=lsb ;; lunar) DEFAULT_INIT_SCRIPT=lunar ;; ubuntu) DEFAULT_INIT_SCRIPT=lsb ;; debian) DEFAULT_INIT_SCRIPT=lsb ;; *) DEFAULT_INIT_SCRIPT=lsb ;; esac AC_MSG_RESULT([$DEFAULT_INIT_SCRIPT]) AC_SUBST(DEFAULT_INIT_SCRIPT) AC_MSG_CHECKING([default init config directory]) case "$VENDOR" in alpine) DEFAULT_INITCONF_DIR=/etc/conf.d ;; gentoo) DEFAULT_INITCONF_DIR=/etc/conf.d ;; toss) DEFAULT_INITCONF_DIR=/etc/sysconfig ;; redhat) DEFAULT_INITCONF_DIR=/etc/sysconfig ;; fedora) DEFAULT_INITCONF_DIR=/etc/sysconfig ;; sles) DEFAULT_INITCONF_DIR=/etc/sysconfig ;; ubuntu) DEFAULT_INITCONF_DIR=/etc/default ;; debian) DEFAULT_INITCONF_DIR=/etc/default ;; *) DEFAULT_INITCONF_DIR=/etc/default ;; esac AC_MSG_RESULT([$DEFAULT_INITCONF_DIR]) AC_SUBST(DEFAULT_INITCONF_DIR) AC_MSG_CHECKING([whether initramfs-tools is available]) if test -d /usr/share/initramfs-tools ; then DEFINE_INITRAMFS='--define "_initramfs 1"' AC_MSG_RESULT([yes]) else DEFINE_INITRAMFS='' AC_MSG_RESULT([no]) fi AC_SUBST(DEFINE_INITRAMFS) ]) dnl # dnl # Default ZFS package configuration dnl # AC_DEFUN([ZFS_AC_PACKAGE], [ ZFS_AC_DEFAULT_PACKAGE ZFS_AC_RPM ZFS_AC_DPKG ZFS_AC_ALIEN ]) diff --git a/configure.ac b/configure.ac index 973ae307c36f..eccc09c28a06 100644 --- a/configure.ac +++ b/configure.ac @@ -1,371 +1,376 @@ /* * This file is part of the ZFS Linux port. * * Copyright (c) 2009 Lawrence Livermore National Security, LLC. * Produced at Lawrence Livermore National Laboratory * Written by: * Brian Behlendorf , * Herb Wartens , * Jim Garlick * LLNL-CODE-403049 * * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License, Version 1.0 only * (the "License"). You may not use this file except in compliance * with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ AC_INIT(m4_esyscmd(grep ^Name: META | cut -d ':' -f 2 | tr -d ' \n'), m4_esyscmd(grep ^Version: META | cut -d ':' -f 2 | tr -d ' \n')) AC_LANG(C) ZFS_AC_META AC_CONFIG_AUX_DIR([config]) AC_CONFIG_MACRO_DIR([config]) AC_CANONICAL_SYSTEM AM_MAINTAINER_MODE m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])]) AM_INIT_AUTOMAKE([subdir-objects]) AC_CONFIG_HEADERS([zfs_config.h], [ (mv zfs_config.h zfs_config.h.tmp && awk -f ${ac_srcdir}/config/config.awk zfs_config.h.tmp >zfs_config.h && rm zfs_config.h.tmp) || exit 1]) AC_PROG_INSTALL AC_PROG_CC AC_PROG_LIBTOOL PKG_PROG_PKG_CONFIG AM_PROG_AS AM_PROG_CC_C_O AX_CODE_COVERAGE _AM_PROG_TAR(pax) ZFS_AC_LICENSE ZFS_AC_PACKAGE ZFS_AC_CONFIG ZFS_AC_DEBUG ZFS_AC_DEBUGINFO ZFS_AC_DEBUG_KMEM ZFS_AC_DEBUG_KMEM_TRACKING AC_CONFIG_FILES([ Makefile udev/Makefile udev/rules.d/Makefile etc/Makefile etc/init.d/Makefile etc/zfs/Makefile etc/systemd/Makefile etc/systemd/system/Makefile etc/systemd/system-generators/Makefile etc/sudoers.d/Makefile etc/modules-load.d/Makefile man/Makefile man/man1/Makefile man/man5/Makefile man/man8/Makefile lib/Makefile lib/libspl/Makefile lib/libspl/asm-generic/Makefile lib/libspl/asm-i386/Makefile lib/libspl/asm-x86_64/Makefile lib/libspl/include/Makefile lib/libspl/include/ia32/Makefile lib/libspl/include/ia32/sys/Makefile lib/libspl/include/rpc/Makefile lib/libspl/include/sys/Makefile lib/libspl/include/sys/dktp/Makefile lib/libspl/include/util/Makefile lib/libavl/Makefile lib/libefi/Makefile lib/libicp/Makefile lib/libnvpair/Makefile lib/libzutil/Makefile lib/libtpool/Makefile lib/libunicode/Makefile lib/libuutil/Makefile lib/libzpool/Makefile lib/libzfs/libzfs.pc lib/libzfs/libzfs_core.pc lib/libzfs/Makefile lib/libzfs_core/Makefile lib/libshare/Makefile cmd/Makefile cmd/zdb/Makefile cmd/zhack/Makefile cmd/zfs/Makefile cmd/zinject/Makefile cmd/zpool/Makefile cmd/zstreamdump/Makefile cmd/ztest/Makefile cmd/mount_zfs/Makefile cmd/fsck_zfs/Makefile cmd/zvol_id/Makefile cmd/vdev_id/Makefile cmd/arcstat/Makefile cmd/dbufstat/Makefile cmd/arc_summary/Makefile cmd/zed/Makefile cmd/zed/zed.d/Makefile cmd/raidz_test/Makefile cmd/zgenhostid/Makefile cmd/zvol_wait/Makefile contrib/Makefile contrib/bash_completion.d/Makefile contrib/dracut/Makefile contrib/dracut/02zfsexpandknowledge/Makefile contrib/dracut/90zfs/Makefile contrib/initramfs/Makefile contrib/initramfs/hooks/Makefile contrib/initramfs/scripts/Makefile contrib/initramfs/scripts/local-top/Makefile contrib/pyzfs/Makefile contrib/pyzfs/setup.py contrib/zcp/Makefile module/Makefile module/avl/Makefile module/nvpair/Makefile module/unicode/Makefile module/zcommon/Makefile module/zfs/Makefile module/lua/Makefile module/icp/Makefile module/spl/Makefile include/Makefile - include/linux/Makefile - include/spl/Makefile - include/spl/rpc/Makefile - include/spl/sys/Makefile + include/os/Makefile + include/os/linux/Makefile + include/os/linux/kernel/Makefile + include/os/linux/kernel/linux/Makefile + include/os/linux/spl/Makefile + include/os/linux/spl/rpc/Makefile + include/os/linux/spl/sys/Makefile + include/os/linux/zfs/Makefile + include/os/linux/zfs/sys/Makefile include/sys/Makefile include/sys/fs/Makefile include/sys/fm/Makefile include/sys/fm/fs/Makefile include/sys/crypto/Makefile include/sys/sysevent/Makefile include/sys/lua/Makefile scripts/Makefile tests/Makefile tests/test-runner/Makefile tests/test-runner/bin/Makefile tests/test-runner/include/Makefile tests/test-runner/man/Makefile tests/runfiles/Makefile tests/zfs-tests/Makefile tests/zfs-tests/callbacks/Makefile tests/zfs-tests/cmd/Makefile tests/zfs-tests/cmd/chg_usr_exec/Makefile tests/zfs-tests/cmd/user_ns_exec/Makefile tests/zfs-tests/cmd/devname2devid/Makefile tests/zfs-tests/cmd/dir_rd_update/Makefile tests/zfs-tests/cmd/file_check/Makefile tests/zfs-tests/cmd/file_trunc/Makefile tests/zfs-tests/cmd/file_write/Makefile tests/zfs-tests/cmd/get_diff/Makefile tests/zfs-tests/cmd/largest_file/Makefile tests/zfs-tests/cmd/libzfs_input_check/Makefile tests/zfs-tests/cmd/mkbusy/Makefile tests/zfs-tests/cmd/mkfile/Makefile tests/zfs-tests/cmd/mkfiles/Makefile tests/zfs-tests/cmd/mktree/Makefile tests/zfs-tests/cmd/mmap_exec/Makefile tests/zfs-tests/cmd/mmap_libaio/Makefile tests/zfs-tests/cmd/mmapwrite/Makefile tests/zfs-tests/cmd/nvlist_to_lua/Makefile tests/zfs-tests/cmd/randfree_file/Makefile tests/zfs-tests/cmd/randwritecomp/Makefile tests/zfs-tests/cmd/readmmap/Makefile tests/zfs-tests/cmd/rename_dir/Makefile tests/zfs-tests/cmd/rm_lnkcnt_zero_file/Makefile tests/zfs-tests/cmd/threadsappend/Makefile tests/zfs-tests/cmd/xattrtest/Makefile tests/zfs-tests/cmd/stride_dd/Makefile tests/zfs-tests/include/Makefile tests/zfs-tests/tests/Makefile tests/zfs-tests/tests/functional/Makefile tests/zfs-tests/tests/functional/acl/Makefile tests/zfs-tests/tests/functional/acl/posix/Makefile tests/zfs-tests/tests/functional/arc/Makefile tests/zfs-tests/tests/functional/atime/Makefile tests/zfs-tests/tests/functional/bootfs/Makefile tests/zfs-tests/tests/functional/cache/Makefile tests/zfs-tests/tests/functional/cachefile/Makefile tests/zfs-tests/tests/functional/casenorm/Makefile tests/zfs-tests/tests/functional/checksum/Makefile tests/zfs-tests/tests/functional/channel_program/Makefile tests/zfs-tests/tests/functional/channel_program/lua_core/Makefile tests/zfs-tests/tests/functional/channel_program/synctask_core/Makefile tests/zfs-tests/tests/functional/chattr/Makefile tests/zfs-tests/tests/functional/clean_mirror/Makefile tests/zfs-tests/tests/functional/cli_root/Makefile tests/zfs-tests/tests/functional/cli_root/zdb/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_bookmark/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_change-key/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_clone/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_copies/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_create/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_destroy/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_diff/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_get/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_inherit/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_load-key/Makefile tests/zfs-tests/tests/functional/cli_root/zfs/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_mount/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_program/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_promote/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_property/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_receive/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_rename/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_reservation/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_rollback/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_send/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_set/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_share/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_snapshot/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_sysfs/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_unload-key/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_unmount/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_unshare/Makefile tests/zfs-tests/tests/functional/cli_root/zfs_upgrade/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_add/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_attach/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_clear/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_create/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_destroy/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_detach/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_events/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_expand/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_export/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_get/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_history/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_import/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_import/blockfiles/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_initialize/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_labelclear/Makefile tests/zfs-tests/tests/functional/cli_root/zpool/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_offline/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_online/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_remove/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_reopen/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_resilver/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_replace/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_scrub/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_set/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_split/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_status/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_sync/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_trim/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_upgrade/Makefile tests/zfs-tests/tests/functional/cli_root/zpool_upgrade/blockfiles/Makefile tests/zfs-tests/tests/functional/cli_user/Makefile tests/zfs-tests/tests/functional/cli_user/misc/Makefile tests/zfs-tests/tests/functional/cli_user/zfs_list/Makefile tests/zfs-tests/tests/functional/cli_user/zpool_iostat/Makefile tests/zfs-tests/tests/functional/cli_user/zpool_list/Makefile tests/zfs-tests/tests/functional/cli_user/zpool_status/Makefile tests/zfs-tests/tests/functional/compression/Makefile tests/zfs-tests/tests/functional/cp_files/Makefile tests/zfs-tests/tests/functional/ctime/Makefile tests/zfs-tests/tests/functional/deadman/Makefile tests/zfs-tests/tests/functional/delegate/Makefile tests/zfs-tests/tests/functional/devices/Makefile tests/zfs-tests/tests/functional/events/Makefile tests/zfs-tests/tests/functional/exec/Makefile tests/zfs-tests/tests/functional/fault/Makefile tests/zfs-tests/tests/functional/features/async_destroy/Makefile tests/zfs-tests/tests/functional/features/large_dnode/Makefile tests/zfs-tests/tests/functional/features/Makefile tests/zfs-tests/tests/functional/grow/Makefile tests/zfs-tests/tests/functional/history/Makefile tests/zfs-tests/tests/functional/hkdf/Makefile tests/zfs-tests/tests/functional/inheritance/Makefile tests/zfs-tests/tests/functional/inuse/Makefile tests/zfs-tests/tests/functional/io/Makefile tests/zfs-tests/tests/functional/large_files/Makefile tests/zfs-tests/tests/functional/largest_pool/Makefile tests/zfs-tests/tests/functional/link_count/Makefile tests/zfs-tests/tests/functional/libzfs/Makefile tests/zfs-tests/tests/functional/limits/Makefile tests/zfs-tests/tests/functional/log_spacemap/Makefile tests/zfs-tests/tests/functional/migration/Makefile tests/zfs-tests/tests/functional/mmap/Makefile tests/zfs-tests/tests/functional/mmp/Makefile tests/zfs-tests/tests/functional/mount/Makefile tests/zfs-tests/tests/functional/mv_files/Makefile tests/zfs-tests/tests/functional/nestedfs/Makefile tests/zfs-tests/tests/functional/no_space/Makefile tests/zfs-tests/tests/functional/nopwrite/Makefile tests/zfs-tests/tests/functional/online_offline/Makefile tests/zfs-tests/tests/functional/pool_names/Makefile tests/zfs-tests/tests/functional/pool_checkpoint/Makefile tests/zfs-tests/tests/functional/poolversion/Makefile tests/zfs-tests/tests/functional/privilege/Makefile tests/zfs-tests/tests/functional/procfs/Makefile tests/zfs-tests/tests/functional/projectquota/Makefile tests/zfs-tests/tests/functional/pyzfs/Makefile tests/zfs-tests/tests/functional/quota/Makefile tests/zfs-tests/tests/functional/raidz/Makefile tests/zfs-tests/tests/functional/redacted_send/Makefile tests/zfs-tests/tests/functional/redundancy/Makefile tests/zfs-tests/tests/functional/refquota/Makefile tests/zfs-tests/tests/functional/refreserv/Makefile tests/zfs-tests/tests/functional/removal/Makefile tests/zfs-tests/tests/functional/rename_dirs/Makefile tests/zfs-tests/tests/functional/replacement/Makefile tests/zfs-tests/tests/functional/reservation/Makefile tests/zfs-tests/tests/functional/rootpool/Makefile tests/zfs-tests/tests/functional/rsend/Makefile tests/zfs-tests/tests/functional/scrub_mirror/Makefile tests/zfs-tests/tests/functional/slog/Makefile tests/zfs-tests/tests/functional/snapshot/Makefile tests/zfs-tests/tests/functional/snapused/Makefile tests/zfs-tests/tests/functional/sparse/Makefile tests/zfs-tests/tests/functional/suid/Makefile tests/zfs-tests/tests/functional/alloc_class/Makefile tests/zfs-tests/tests/functional/threadsappend/Makefile tests/zfs-tests/tests/functional/tmpfile/Makefile tests/zfs-tests/tests/functional/trim/Makefile tests/zfs-tests/tests/functional/truncate/Makefile tests/zfs-tests/tests/functional/user_namespace/Makefile tests/zfs-tests/tests/functional/userquota/Makefile tests/zfs-tests/tests/functional/upgrade/Makefile tests/zfs-tests/tests/functional/vdev_zaps/Makefile tests/zfs-tests/tests/functional/write_dirs/Makefile tests/zfs-tests/tests/functional/xattr/Makefile tests/zfs-tests/tests/functional/zvol/Makefile tests/zfs-tests/tests/functional/zvol/zvol_cli/Makefile tests/zfs-tests/tests/functional/zvol/zvol_ENOSPC/Makefile tests/zfs-tests/tests/functional/zvol/zvol_misc/Makefile tests/zfs-tests/tests/functional/zvol/zvol_swap/Makefile tests/zfs-tests/tests/perf/Makefile tests/zfs-tests/tests/perf/fio/Makefile tests/zfs-tests/tests/perf/regression/Makefile tests/zfs-tests/tests/perf/scripts/Makefile tests/zfs-tests/tests/stress/Makefile rpm/Makefile rpm/redhat/Makefile rpm/redhat/zfs.spec rpm/redhat/zfs-kmod.spec rpm/redhat/zfs-dkms.spec rpm/generic/Makefile rpm/generic/zfs.spec rpm/generic/zfs-kmod.spec rpm/generic/zfs-dkms.spec zfs.release ]) AC_OUTPUT diff --git a/copy-builtin b/copy-builtin index 1dcfcb961ee8..adb3bd544c0a 100755 --- a/copy-builtin +++ b/copy-builtin @@ -1,113 +1,115 @@ #!/bin/bash set -e usage() { echo "usage: $0 " >&2 exit 1 } [ "$#" -eq 1 ] || usage KERNEL_DIR="$(readlink --canonicalize-existing "$1")" MODULES=() MODULES+="spl" for MODULE_DIR in module/* do [ -d "$MODULE_DIR" ] || continue [ "spl" = "${MODULE_DIR##*/}" ] && continue MODULES+=("${MODULE_DIR##*/}") done if ! [ -e 'zfs_config.h' ] then echo >&2 echo " $0: you did not run configure, or you're not in the ZFS source directory." >&2 echo " $0: run configure with --with-linux=$KERNEL_DIR and --enable-linux-builtin." >&2 echo >&2 exit 1 fi make clean || true scripts/make_gitrev.sh || true rm -rf "$KERNEL_DIR/include/zfs" "$KERNEL_DIR/fs/zfs" cp --recursive include "$KERNEL_DIR/include/zfs" cp --recursive module "$KERNEL_DIR/fs/zfs" cp zfs_config.h "$KERNEL_DIR/include/zfs/" for MODULE in "${MODULES[@]}" do sed -i.bak '/obj =/d' "$KERNEL_DIR/fs/zfs/$MODULE/Makefile" sed -i.bak '/src =/d' "$KERNEL_DIR/fs/zfs/$MODULE/Makefile" done cat > "$KERNEL_DIR/fs/zfs/Kconfig" <<"EOF" config ZFS tristate "ZFS filesystem support" depends on EFI_PARTITION select ZLIB_INFLATE select ZLIB_DEFLATE help This is the ZFS filesystem from the ZFS On Linux project. See http://zfsonlinux.org/ To compile this file system support as a module, choose M here. If unsure, say N. EOF { cat <<-"EOF" ZFS_MODULE_CFLAGS = -I$(srctree)/include/zfs - ZFS_MODULE_CFLAGS += -I$(srctree)/include/zfs/spl + ZFS_MODULE_CFLAGS += -I$(srctree)/include/zfs/os/linux/spl + ZFS_MODULE_CFLAGS += -I$(srctree)/include/zfs/os/linux/zfs + ZFS_MODULE_CFLAGS += -I$(srctree)/include/zfs/os/linux/kernel ZFS_MODULE_CFLAGS += -include $(srctree)/include/zfs/zfs_config.h ZFS_MODULE_CFLAGS += -std=gnu99 -Wno-declaration-after-statement ZFS_MODULE_CPPFLAGS = -D_KERNEL ZFS_MODULE_CPPFLAGS += -UDEBUG -DNDEBUG export ZFS_MODULE_CFLAGS ZFS_MODULE_CPPFLAGS obj-$(CONFIG_ZFS) := EOF for MODULE in "${MODULES[@]}" do echo 'obj-$(CONFIG_ZFS) += ' "$MODULE/" done } > "$KERNEL_DIR/fs/zfs/Kbuild" add_after() { local FILE="$1" local MARKER="$2" local NEW="$3" local LINE while IFS='' read -r LINE do echo "$LINE" if [ -n "$MARKER" -a "$LINE" = "$MARKER" ] then echo "$NEW" MARKER='' if IFS='' read -r LINE then [ "$LINE" != "$NEW" ] && echo "$LINE" fi fi done < "$FILE" > "$FILE.new" mv "$FILE.new" "$FILE" } add_after "$KERNEL_DIR/fs/Kconfig" 'if BLOCK' 'source "fs/zfs/Kconfig"' add_after "$KERNEL_DIR/fs/Makefile" 'endif' 'obj-$(CONFIG_ZFS) += zfs/' echo >&2 echo " $0: done." >&2 echo " $0: now you can build the kernel with ZFS support." >&2 echo " $0: make sure you enable ZFS support (CONFIG_ZFS) before building." >&2 echo >&2 diff --git a/include/.gitignore b/include/.gitignore index e6eb2116fa29..416f36b9e77d 100644 --- a/include/.gitignore +++ b/include/.gitignore @@ -1 +1,2 @@ /zfs_gitrev.h +/spl diff --git a/include/Makefile.am b/include/Makefile.am index bac47d98d9de..5b37dc765d26 100644 --- a/include/Makefile.am +++ b/include/Makefile.am @@ -1,32 +1,32 @@ -SUBDIRS = linux spl sys +SUBDIRS = sys os COMMON_H = \ $(top_srcdir)/include/zfeature_common.h \ $(top_srcdir)/include/zfs_comutil.h \ $(top_srcdir)/include/zfs_deleg.h \ $(top_srcdir)/include/zfs_fletcher.h \ $(top_srcdir)/include/zfs_namecheck.h \ $(top_srcdir)/include/zfs_prop.h USER_H = \ $(top_srcdir)/include/libnvpair.h \ $(top_srcdir)/include/libuutil_common.h \ $(top_srcdir)/include/libuutil.h \ $(top_srcdir)/include/libuutil_impl.h \ $(top_srcdir)/include/libzfs.h \ $(top_srcdir)/include/libzfs_core.h \ $(top_srcdir)/include/libzfs_impl.h \ $(top_srcdir)/include/libzutil.h \ $(top_srcdir)/include/thread_pool.h EXTRA_DIST = $(COMMON_H) $(USER_H) if CONFIG_USER libzfsdir = $(includedir)/libzfs libzfs_HEADERS = $(COMMON_H) $(USER_H) endif if CONFIG_KERNEL kerneldir = @prefix@/src/zfs-$(VERSION)/include kernel_HEADERS = $(COMMON_H) endif diff --git a/include/linux/Makefile.am b/include/linux/Makefile.am deleted file mode 100644 index 2455759e8138..000000000000 --- a/include/linux/Makefile.am +++ /dev/null @@ -1,29 +0,0 @@ -COMMON_H = - -KERNEL_H = \ - $(top_srcdir)/include/linux/dcache_compat.h \ - $(top_srcdir)/include/linux/xattr_compat.h \ - $(top_srcdir)/include/linux/vfs_compat.h \ - $(top_srcdir)/include/linux/blkdev_compat.h \ - $(top_srcdir)/include/linux/utsname_compat.h \ - $(top_srcdir)/include/linux/kmap_compat.h \ - $(top_srcdir)/include/linux/simd.h \ - $(top_srcdir)/include/linux/simd_x86.h \ - $(top_srcdir)/include/linux/simd_aarch64.h \ - $(top_srcdir)/include/linux/mod_compat.h \ - $(top_srcdir)/include/linux/page_compat.h \ - $(top_srcdir)/include/linux/compiler_compat.h - -USER_H = - -EXTRA_DIST = $(COMMON_H) $(KERNEL_H) $(USER_H) - -if CONFIG_USER -libzfsdir = $(includedir)/libzfs/linux -libzfs_HEADERS = $(COMMON_H) $(USER_H) -endif - -if CONFIG_KERNEL -kerneldir = @prefix@/src/zfs-$(VERSION)/include/linux -kernel_HEADERS = $(COMMON_H) $(KERNEL_H) -endif diff --git a/include/os/Makefile.am b/include/os/Makefile.am new file mode 100644 index 000000000000..09c0beec4757 --- /dev/null +++ b/include/os/Makefile.am @@ -0,0 +1,3 @@ +if BUILD_LINUX +SUBDIRS = linux +endif diff --git a/include/os/linux/Makefile.am b/include/os/linux/Makefile.am new file mode 100644 index 000000000000..605a1fcb7506 --- /dev/null +++ b/include/os/linux/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = kernel spl zfs diff --git a/include/os/linux/kernel/Makefile.am b/include/os/linux/kernel/Makefile.am new file mode 100644 index 000000000000..08b2f5fc5c99 --- /dev/null +++ b/include/os/linux/kernel/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = linux diff --git a/include/os/linux/kernel/linux/Makefile.am b/include/os/linux/kernel/linux/Makefile.am new file mode 100644 index 000000000000..06ce7c7aaa80 --- /dev/null +++ b/include/os/linux/kernel/linux/Makefile.am @@ -0,0 +1,29 @@ +COMMON_H = + +KERNEL_H = \ + $(top_srcdir)/include/os/linux/kernel/linux/dcache_compat.h \ + $(top_srcdir)/include/os/linux/kernel/linux/xattr_compat.h \ + $(top_srcdir)/include/os/linux/kernel/linux/vfs_compat.h \ + $(top_srcdir)/include/os/linux/kernel/linux/blkdev_compat.h \ + $(top_srcdir)/include/os/linux/kernel/linux/utsname_compat.h \ + $(top_srcdir)/include/os/linux/kernel/linux/kmap_compat.h \ + $(top_srcdir)/include/os/linux/kernel/linux/simd.h \ + $(top_srcdir)/include/os/linux/kernel/linux/simd_x86.h \ + $(top_srcdir)/include/os/linux/kernel/linux/simd_aarch64.h \ + $(top_srcdir)/include/os/linux/kernel/linux/mod_compat.h \ + $(top_srcdir)/include/os/linux/kernel/linux/page_compat.h \ + $(top_srcdir)/include/os/linux/kernel/linux/compiler_compat.h + +USER_H = + +EXTRA_DIST = $(COMMON_H) $(KERNEL_H) $(USER_H) + +if CONFIG_USER +libzfsdir = $(includedir)/libzfs/linux +libzfs_HEADERS = $(COMMON_H) $(USER_H) +endif + +if CONFIG_KERNEL +kerneldir = @prefix@/src/zfs-$(VERSION)/include/linux +kernel_HEADERS = $(COMMON_H) $(KERNEL_H) +endif diff --git a/include/linux/blkdev_compat.h b/include/os/linux/kernel/linux/blkdev_compat.h similarity index 100% rename from include/linux/blkdev_compat.h rename to include/os/linux/kernel/linux/blkdev_compat.h diff --git a/include/linux/compiler_compat.h b/include/os/linux/kernel/linux/compiler_compat.h similarity index 100% rename from include/linux/compiler_compat.h rename to include/os/linux/kernel/linux/compiler_compat.h diff --git a/include/linux/dcache_compat.h b/include/os/linux/kernel/linux/dcache_compat.h similarity index 100% rename from include/linux/dcache_compat.h rename to include/os/linux/kernel/linux/dcache_compat.h diff --git a/include/linux/kmap_compat.h b/include/os/linux/kernel/linux/kmap_compat.h similarity index 100% rename from include/linux/kmap_compat.h rename to include/os/linux/kernel/linux/kmap_compat.h diff --git a/include/linux/mod_compat.h b/include/os/linux/kernel/linux/mod_compat.h similarity index 100% rename from include/linux/mod_compat.h rename to include/os/linux/kernel/linux/mod_compat.h diff --git a/include/linux/page_compat.h b/include/os/linux/kernel/linux/page_compat.h similarity index 100% rename from include/linux/page_compat.h rename to include/os/linux/kernel/linux/page_compat.h diff --git a/include/linux/simd.h b/include/os/linux/kernel/linux/simd.h similarity index 92% rename from include/linux/simd.h rename to include/os/linux/kernel/linux/simd.h index d2b60996a495..1f6574a90e49 100644 --- a/include/linux/simd.h +++ b/include/os/linux/kernel/linux/simd.h @@ -1,41 +1,41 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2019 Lawrence Livermore National Security, LLC. */ -#ifndef _SIMD_H -#define _SIMD_H +#ifndef _LINUX_SIMD_H +#define _LINUX_SIMD_H #if defined(__x86) #include #elif defined(__aarch64__) #include #else -#define kfpu_allowed() 1 +#define kfpu_allowed() 0 #define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) #endif -#endif /* _SIMD_H */ +#endif /* _LINUX_SIMD_H */ diff --git a/include/linux/simd_aarch64.h b/include/os/linux/kernel/linux/simd_aarch64.h similarity index 79% rename from include/linux/simd_aarch64.h rename to include/os/linux/kernel/linux/simd_aarch64.h index b45d31c48167..ac530d920015 100644 --- a/include/linux/simd_aarch64.h +++ b/include/os/linux/kernel/linux/simd_aarch64.h @@ -1,62 +1,52 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Romain Dolbeau . */ /* * USER API: * * Kernel fpu methods: * kfpu_allowed() * kfpu_initialize() * kfpu_begin() * kfpu_end() */ -#ifndef _SIMD_AARCH64_H -#define _SIMD_AARCH64_H +#ifndef _LINUX_SIMD_AARCH64_H +#define _LINUX_SIMD_AARCH64_H #include #if defined(__aarch64__) #include - -#if defined(_KERNEL) #include + #define kfpu_allowed() 1 #define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() kernel_neon_begin() #define kfpu_end() kernel_neon_end() -#else -/* - * fpu dummy methods for userspace - */ -#define kfpu_allowed() 1 -#define kfpu_initialize(tsk) do {} while (0) -#define kfpu_begin() do {} while (0) -#define kfpu_end() do {} while (0) -#endif /* defined(_KERNEL) */ #endif /* __aarch64__ */ -#endif /* _SIMD_AARCH64_H */ +#endif /* _LINUX_SIMD_AARCH64_H */ diff --git a/include/linux/simd_x86.h b/include/os/linux/kernel/linux/simd_x86.h similarity index 59% rename from include/linux/simd_x86.h rename to include/os/linux/kernel/linux/simd_x86.h index edd4560981a6..c59ba4174d97 100644 --- a/include/linux/simd_x86.h +++ b/include/os/linux/kernel/linux/simd_x86.h @@ -1,814 +1,523 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Gvozden Neskovic . */ /* * USER API: * * Kernel fpu methods: * kfpu_allowed() * kfpu_initialize() * kfpu_begin() * kfpu_end() * * SIMD support: * * Following functions should be called to determine whether CPU feature * is supported. All functions are usable in kernel and user space. * If a SIMD algorithm is using more than one instruction set * all relevant feature test functions should be called. * * Supported features: * zfs_sse_available() * zfs_sse2_available() * zfs_sse3_available() * zfs_ssse3_available() * zfs_sse4_1_available() * zfs_sse4_2_available() * * zfs_avx_available() * zfs_avx2_available() * * zfs_bmi1_available() * zfs_bmi2_available() * * zfs_avx512f_available() * zfs_avx512cd_available() * zfs_avx512er_available() * zfs_avx512pf_available() * zfs_avx512bw_available() * zfs_avx512dq_available() * zfs_avx512vl_available() * zfs_avx512ifma_available() * zfs_avx512vbmi_available() * * NOTE(AVX-512VL): If using AVX-512 instructions with 128Bit registers * also add zfs_avx512vl_available() to feature check. */ -#ifndef _SIMD_X86_H -#define _SIMD_X86_H - -#include +#ifndef _LINUX_SIMD_X86_H +#define _LINUX_SIMD_X86_H /* only for __x86 */ #if defined(__x86) #include - -#if defined(_KERNEL) #include -#else -#include -#endif - -#if defined(_KERNEL) /* * Disable the WARN_ON_FPU() macro to prevent additional dependencies * when providing the kfpu_* functions. Relevant warnings are included * as appropriate and are unconditionally enabled. */ #if defined(CONFIG_X86_DEBUG_FPU) && !defined(KERNEL_EXPORTS_X86_FPU) #undef CONFIG_X86_DEBUG_FPU #endif #if defined(HAVE_KERNEL_FPU_API_HEADER) #include #include #else #include #include #endif /* * The following cases are for kernels which export either the * kernel_fpu_* or __kernel_fpu_* functions. */ #if defined(KERNEL_EXPORTS_X86_FPU) #define kfpu_allowed() 1 #define kfpu_initialize(tsk) do {} while (0) #if defined(HAVE_UNDERSCORE_KERNEL_FPU) #define kfpu_begin() \ { \ preempt_disable(); \ __kernel_fpu_begin(); \ } #define kfpu_end() \ { \ __kernel_fpu_end(); \ preempt_enable(); \ } #elif defined(HAVE_KERNEL_FPU) #define kfpu_begin() kernel_fpu_begin() #define kfpu_end() kernel_fpu_end() #else /* * This case is unreachable. When KERNEL_EXPORTS_X86_FPU is defined then * either HAVE_UNDERSCORE_KERNEL_FPU or HAVE_KERNEL_FPU must be defined. */ #error "Unreachable kernel configuration" #endif #else /* defined(KERNEL_EXPORTS_X86_FPU) */ /* * When the kernel_fpu_* symbols are unavailable then provide our own * versions which allow the FPU to be safely used in kernel threads. * In practice, this is not a significant restriction for ZFS since the * vast majority of SIMD operations are performed by the IO pipeline. */ /* * Returns non-zero if FPU operations are allowed in the current context. */ #if defined(HAVE_KERNEL_TIF_NEED_FPU_LOAD) #define kfpu_allowed() ((current->flags & PF_KTHREAD) && \ test_thread_flag(TIF_NEED_FPU_LOAD)) #elif defined(HAVE_KERNEL_FPU_INITIALIZED) #define kfpu_allowed() ((current->flags & PF_KTHREAD) && \ current->thread.fpu.initialized) #else #define kfpu_allowed() 0 #endif static inline void kfpu_initialize(void) { WARN_ON_ONCE(!(current->flags & PF_KTHREAD)); #if defined(HAVE_KERNEL_TIF_NEED_FPU_LOAD) __fpu_invalidate_fpregs_state(¤t->thread.fpu); set_thread_flag(TIF_NEED_FPU_LOAD); #elif defined(HAVE_KERNEL_FPU_INITIALIZED) __fpu_invalidate_fpregs_state(¤t->thread.fpu); current->thread.fpu.initialized = 1; #endif } static inline void kfpu_begin(void) { WARN_ON_ONCE(!kfpu_allowed()); /* * Preemption and interrupts must be disabled for the critical * region where the FPU state is being modified. */ preempt_disable(); local_irq_disable(); #if defined(HAVE_KERNEL_TIF_NEED_FPU_LOAD) /* * The current FPU registers need to be preserved by kfpu_begin() * and restored by kfpu_end(). This is required because we can * not call __cpu_invalidate_fpregs_state() to invalidate the * per-cpu FPU state and force them to be restored during a * context switch. */ copy_fpregs_to_fpstate(¤t->thread.fpu); #elif defined(HAVE_KERNEL_FPU_INITIALIZED) /* * There is no need to preserve and restore the FPU registers. * They will always be restored from the task's stored FPU state * when switching contexts. */ WARN_ON_ONCE(current->thread.fpu.initialized == 0); #endif } static inline void kfpu_end(void) { #if defined(HAVE_KERNEL_TIF_NEED_FPU_LOAD) union fpregs_state *state = ¤t->thread.fpu.state; int error; if (use_xsave()) { error = copy_kernel_to_xregs_err(&state->xsave, -1); } else if (use_fxsr()) { error = copy_kernel_to_fxregs_err(&state->fxsave); } else { error = copy_kernel_to_fregs_err(&state->fsave); } WARN_ON_ONCE(error); #endif local_irq_enable(); preempt_enable(); } #endif /* defined(HAVE_KERNEL_FPU) */ -#else /* defined(_KERNEL) */ -/* - * FPU dummy methods for user space. - */ -#define kfpu_allowed() 1 -#define kfpu_initialize(tsk) do {} while (0) -#define kfpu_begin() do {} while (0) -#define kfpu_end() do {} while (0) -#endif /* defined(_KERNEL) */ - -/* - * CPUID feature tests for user-space. Linux kernel provides an interface for - * CPU feature testing. - */ -#if !defined(_KERNEL) - -/* - * x86 registers used implicitly by CPUID - */ -typedef enum cpuid_regs { - EAX = 0, - EBX, - ECX, - EDX, - CPUID_REG_CNT = 4 -} cpuid_regs_t; - -/* - * List of instruction sets identified by CPUID - */ -typedef enum cpuid_inst_sets { - SSE = 0, - SSE2, - SSE3, - SSSE3, - SSE4_1, - SSE4_2, - OSXSAVE, - AVX, - AVX2, - BMI1, - BMI2, - AVX512F, - AVX512CD, - AVX512DQ, - AVX512BW, - AVX512IFMA, - AVX512VBMI, - AVX512PF, - AVX512ER, - AVX512VL, - AES, - PCLMULQDQ -} cpuid_inst_sets_t; - -/* - * Instruction set descriptor. - */ -typedef struct cpuid_feature_desc { - uint32_t leaf; /* CPUID leaf */ - uint32_t subleaf; /* CPUID sub-leaf */ - uint32_t flag; /* bit mask of the feature */ - cpuid_regs_t reg; /* which CPUID return register to test */ -} cpuid_feature_desc_t; - -#define _AVX512F_BIT (1U << 16) -#define _AVX512CD_BIT (_AVX512F_BIT | (1U << 28)) -#define _AVX512DQ_BIT (_AVX512F_BIT | (1U << 17)) -#define _AVX512BW_BIT (_AVX512F_BIT | (1U << 30)) -#define _AVX512IFMA_BIT (_AVX512F_BIT | (1U << 21)) -#define _AVX512VBMI_BIT (1U << 1) /* AVX512F_BIT is on another leaf */ -#define _AVX512PF_BIT (_AVX512F_BIT | (1U << 26)) -#define _AVX512ER_BIT (_AVX512F_BIT | (1U << 27)) -#define _AVX512VL_BIT (1U << 31) /* if used also check other levels */ -#define _AES_BIT (1U << 25) -#define _PCLMULQDQ_BIT (1U << 1) - -/* - * Descriptions of supported instruction sets - */ -static const cpuid_feature_desc_t cpuid_features[] = { - [SSE] = {1U, 0U, 1U << 25, EDX }, - [SSE2] = {1U, 0U, 1U << 26, EDX }, - [SSE3] = {1U, 0U, 1U << 0, ECX }, - [SSSE3] = {1U, 0U, 1U << 9, ECX }, - [SSE4_1] = {1U, 0U, 1U << 19, ECX }, - [SSE4_2] = {1U, 0U, 1U << 20, ECX }, - [OSXSAVE] = {1U, 0U, 1U << 27, ECX }, - [AVX] = {1U, 0U, 1U << 28, ECX }, - [AVX2] = {7U, 0U, 1U << 5, EBX }, - [BMI1] = {7U, 0U, 1U << 3, EBX }, - [BMI2] = {7U, 0U, 1U << 8, EBX }, - [AVX512F] = {7U, 0U, _AVX512F_BIT, EBX }, - [AVX512CD] = {7U, 0U, _AVX512CD_BIT, EBX }, - [AVX512DQ] = {7U, 0U, _AVX512DQ_BIT, EBX }, - [AVX512BW] = {7U, 0U, _AVX512BW_BIT, EBX }, - [AVX512IFMA] = {7U, 0U, _AVX512IFMA_BIT, EBX }, - [AVX512VBMI] = {7U, 0U, _AVX512VBMI_BIT, ECX }, - [AVX512PF] = {7U, 0U, _AVX512PF_BIT, EBX }, - [AVX512ER] = {7U, 0U, _AVX512ER_BIT, EBX }, - [AVX512VL] = {7U, 0U, _AVX512ER_BIT, EBX }, - [AES] = {1U, 0U, _AES_BIT, ECX }, - [PCLMULQDQ] = {1U, 0U, _PCLMULQDQ_BIT, ECX }, -}; - -/* - * Check if OS supports AVX and AVX2 by checking XCR0 - * Only call this function if CPUID indicates that AVX feature is - * supported by the CPU, otherwise it might be an illegal instruction. - */ -static inline uint64_t -xgetbv(uint32_t index) -{ - uint32_t eax, edx; - /* xgetbv - instruction byte code */ - __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0" - : "=a" (eax), "=d" (edx) - : "c" (index)); - - return ((((uint64_t)edx)<<32) | (uint64_t)eax); -} - -/* - * Check if CPU supports a feature - */ -static inline boolean_t -__cpuid_check_feature(const cpuid_feature_desc_t *desc) -{ - uint32_t r[CPUID_REG_CNT]; - - if (__get_cpuid_max(0, NULL) >= desc->leaf) { - /* - * __cpuid_count is needed to properly check - * for AVX2. It is a macro, so return parameters - * are passed by value. - */ - __cpuid_count(desc->leaf, desc->subleaf, - r[EAX], r[EBX], r[ECX], r[EDX]); - return ((r[desc->reg] & desc->flag) == desc->flag); - } - return (B_FALSE); -} - -#define CPUID_FEATURE_CHECK(name, id) \ -static inline boolean_t \ -__cpuid_has_ ## name(void) \ -{ \ - return (__cpuid_check_feature(&cpuid_features[id])); \ -} - /* - * Define functions for user-space CPUID features testing + * Linux kernel provides an interface for CPU feature testing. */ -CPUID_FEATURE_CHECK(sse, SSE); -CPUID_FEATURE_CHECK(sse2, SSE2); -CPUID_FEATURE_CHECK(sse3, SSE3); -CPUID_FEATURE_CHECK(ssse3, SSSE3); -CPUID_FEATURE_CHECK(sse4_1, SSE4_1); -CPUID_FEATURE_CHECK(sse4_2, SSE4_2); -CPUID_FEATURE_CHECK(avx, AVX); -CPUID_FEATURE_CHECK(avx2, AVX2); -CPUID_FEATURE_CHECK(osxsave, OSXSAVE); -CPUID_FEATURE_CHECK(bmi1, BMI1); -CPUID_FEATURE_CHECK(bmi2, BMI2); -CPUID_FEATURE_CHECK(avx512f, AVX512F); -CPUID_FEATURE_CHECK(avx512cd, AVX512CD); -CPUID_FEATURE_CHECK(avx512dq, AVX512DQ); -CPUID_FEATURE_CHECK(avx512bw, AVX512BW); -CPUID_FEATURE_CHECK(avx512ifma, AVX512IFMA); -CPUID_FEATURE_CHECK(avx512vbmi, AVX512VBMI); -CPUID_FEATURE_CHECK(avx512pf, AVX512PF); -CPUID_FEATURE_CHECK(avx512er, AVX512ER); -CPUID_FEATURE_CHECK(avx512vl, AVX512VL); -CPUID_FEATURE_CHECK(aes, AES); -CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ); - -#endif /* !defined(_KERNEL) */ - - /* * Detect register set support */ static inline boolean_t __simd_state_enabled(const uint64_t state) { boolean_t has_osxsave; uint64_t xcr0; -#if defined(_KERNEL) #if defined(X86_FEATURE_OSXSAVE) has_osxsave = !!boot_cpu_has(X86_FEATURE_OSXSAVE); #else has_osxsave = B_FALSE; #endif -#elif !defined(_KERNEL) - has_osxsave = __cpuid_has_osxsave(); -#endif - if (!has_osxsave) return (B_FALSE); xcr0 = xgetbv(0); return ((xcr0 & state) == state); } #define _XSTATE_SSE_AVX (0x2 | 0x4) #define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX) #define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX) #define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512) - /* * Check if SSE instruction set is available */ static inline boolean_t zfs_sse_available(void) { -#if defined(_KERNEL) return (!!boot_cpu_has(X86_FEATURE_XMM)); -#elif !defined(_KERNEL) - return (__cpuid_has_sse()); -#endif } /* * Check if SSE2 instruction set is available */ static inline boolean_t zfs_sse2_available(void) { -#if defined(_KERNEL) return (!!boot_cpu_has(X86_FEATURE_XMM2)); -#elif !defined(_KERNEL) - return (__cpuid_has_sse2()); -#endif } /* * Check if SSE3 instruction set is available */ static inline boolean_t zfs_sse3_available(void) { -#if defined(_KERNEL) return (!!boot_cpu_has(X86_FEATURE_XMM3)); -#elif !defined(_KERNEL) - return (__cpuid_has_sse3()); -#endif } /* * Check if SSSE3 instruction set is available */ static inline boolean_t zfs_ssse3_available(void) { -#if defined(_KERNEL) return (!!boot_cpu_has(X86_FEATURE_SSSE3)); -#elif !defined(_KERNEL) - return (__cpuid_has_ssse3()); -#endif } /* * Check if SSE4.1 instruction set is available */ static inline boolean_t zfs_sse4_1_available(void) { -#if defined(_KERNEL) return (!!boot_cpu_has(X86_FEATURE_XMM4_1)); -#elif !defined(_KERNEL) - return (__cpuid_has_sse4_1()); -#endif } /* * Check if SSE4.2 instruction set is available */ static inline boolean_t zfs_sse4_2_available(void) { -#if defined(_KERNEL) return (!!boot_cpu_has(X86_FEATURE_XMM4_2)); -#elif !defined(_KERNEL) - return (__cpuid_has_sse4_2()); -#endif } /* * Check if AVX instruction set is available */ static inline boolean_t zfs_avx_available(void) { - boolean_t has_avx; -#if defined(_KERNEL) - has_avx = !!boot_cpu_has(X86_FEATURE_AVX); -#elif !defined(_KERNEL) - has_avx = __cpuid_has_avx(); -#endif - - return (has_avx && __ymm_enabled()); + return (boot_cpu_has(X86_FEATURE_AVX) && __ymm_enabled()); } /* * Check if AVX2 instruction set is available */ static inline boolean_t zfs_avx2_available(void) { - boolean_t has_avx2; -#if defined(_KERNEL) - has_avx2 = !!boot_cpu_has(X86_FEATURE_AVX2); -#elif !defined(_KERNEL) - has_avx2 = __cpuid_has_avx2(); -#endif - - return (has_avx2 && __ymm_enabled()); + return (boot_cpu_has(X86_FEATURE_AVX2) && __ymm_enabled()); } /* * Check if BMI1 instruction set is available */ static inline boolean_t zfs_bmi1_available(void) { -#if defined(_KERNEL) #if defined(X86_FEATURE_BMI1) return (!!boot_cpu_has(X86_FEATURE_BMI1)); #else return (B_FALSE); #endif -#elif !defined(_KERNEL) - return (__cpuid_has_bmi1()); -#endif } /* * Check if BMI2 instruction set is available */ static inline boolean_t zfs_bmi2_available(void) { -#if defined(_KERNEL) #if defined(X86_FEATURE_BMI2) return (!!boot_cpu_has(X86_FEATURE_BMI2)); #else return (B_FALSE); #endif -#elif !defined(_KERNEL) - return (__cpuid_has_bmi2()); -#endif } /* * Check if AES instruction set is available */ static inline boolean_t zfs_aes_available(void) { -#if defined(_KERNEL) #if defined(X86_FEATURE_AES) return (!!boot_cpu_has(X86_FEATURE_AES)); #else return (B_FALSE); #endif -#elif !defined(_KERNEL) - return (__cpuid_has_aes()); -#endif } /* * Check if PCLMULQDQ instruction set is available */ static inline boolean_t zfs_pclmulqdq_available(void) { -#if defined(_KERNEL) #if defined(X86_FEATURE_PCLMULQDQ) return (!!boot_cpu_has(X86_FEATURE_PCLMULQDQ)); #else return (B_FALSE); #endif -#elif !defined(_KERNEL) - return (__cpuid_has_pclmulqdq()); -#endif } /* * AVX-512 family of instruction sets: * * AVX512F Foundation * AVX512CD Conflict Detection Instructions * AVX512ER Exponential and Reciprocal Instructions * AVX512PF Prefetch Instructions * * AVX512BW Byte and Word Instructions * AVX512DQ Double-word and Quadword Instructions * AVX512VL Vector Length Extensions * * AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4) * AVX512VBMI Vector Byte Manipulation Instructions */ - -/* Check if AVX512F instruction set is available */ +/* + * Check if AVX512F instruction set is available + */ static inline boolean_t zfs_avx512f_available(void) { boolean_t has_avx512 = B_FALSE; -#if defined(_KERNEL) #if defined(X86_FEATURE_AVX512F) has_avx512 = !!boot_cpu_has(X86_FEATURE_AVX512F); -#else - has_avx512 = B_FALSE; #endif -#elif !defined(_KERNEL) - has_avx512 = __cpuid_has_avx512f(); -#endif - return (has_avx512 && __zmm_enabled()); } -/* Check if AVX512CD instruction set is available */ +/* + * Check if AVX512CD instruction set is available + */ static inline boolean_t zfs_avx512cd_available(void) { boolean_t has_avx512 = B_FALSE; -#if defined(_KERNEL) #if defined(X86_FEATURE_AVX512CD) has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512CD); -#else - has_avx512 = B_FALSE; -#endif -#elif !defined(_KERNEL) - has_avx512 = __cpuid_has_avx512cd(); #endif - return (has_avx512 && __zmm_enabled()); } -/* Check if AVX512ER instruction set is available */ +/* + * Check if AVX512ER instruction set is available + */ static inline boolean_t zfs_avx512er_available(void) { boolean_t has_avx512 = B_FALSE; -#if defined(_KERNEL) #if defined(X86_FEATURE_AVX512ER) has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512ER); -#else - has_avx512 = B_FALSE; -#endif -#elif !defined(_KERNEL) - has_avx512 = __cpuid_has_avx512er(); #endif - return (has_avx512 && __zmm_enabled()); } -/* Check if AVX512PF instruction set is available */ +/* + * Check if AVX512PF instruction set is available + */ static inline boolean_t zfs_avx512pf_available(void) { boolean_t has_avx512 = B_FALSE; -#if defined(_KERNEL) #if defined(X86_FEATURE_AVX512PF) has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512PF); -#else - has_avx512 = B_FALSE; -#endif -#elif !defined(_KERNEL) - has_avx512 = __cpuid_has_avx512pf(); #endif - return (has_avx512 && __zmm_enabled()); } -/* Check if AVX512BW instruction set is available */ +/* + * Check if AVX512BW instruction set is available + */ static inline boolean_t zfs_avx512bw_available(void) { boolean_t has_avx512 = B_FALSE; -#if defined(_KERNEL) #if defined(X86_FEATURE_AVX512BW) has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512BW); -#else - has_avx512 = B_FALSE; -#endif -#elif !defined(_KERNEL) - has_avx512 = __cpuid_has_avx512bw(); #endif return (has_avx512 && __zmm_enabled()); } -/* Check if AVX512DQ instruction set is available */ +/* + * Check if AVX512DQ instruction set is available + */ static inline boolean_t zfs_avx512dq_available(void) { boolean_t has_avx512 = B_FALSE; -#if defined(_KERNEL) #if defined(X86_FEATURE_AVX512DQ) has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512DQ); -#else - has_avx512 = B_FALSE; -#endif -#elif !defined(_KERNEL) - has_avx512 = __cpuid_has_avx512dq(); #endif - return (has_avx512 && __zmm_enabled()); } -/* Check if AVX512VL instruction set is available */ +/* + * Check if AVX512VL instruction set is available + */ static inline boolean_t zfs_avx512vl_available(void) { boolean_t has_avx512 = B_FALSE; -#if defined(_KERNEL) #if defined(X86_FEATURE_AVX512VL) has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512VL); -#else - has_avx512 = B_FALSE; -#endif -#elif !defined(_KERNEL) - has_avx512 = __cpuid_has_avx512vl(); #endif - return (has_avx512 && __zmm_enabled()); } -/* Check if AVX512IFMA instruction set is available */ +/* + * Check if AVX512IFMA instruction set is available + */ static inline boolean_t zfs_avx512ifma_available(void) { boolean_t has_avx512 = B_FALSE; -#if defined(_KERNEL) #if defined(X86_FEATURE_AVX512IFMA) has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512IFMA); -#else - has_avx512 = B_FALSE; -#endif -#elif !defined(_KERNEL) - has_avx512 = __cpuid_has_avx512ifma(); #endif - return (has_avx512 && __zmm_enabled()); } -/* Check if AVX512VBMI instruction set is available */ +/* + * Check if AVX512VBMI instruction set is available + */ static inline boolean_t zfs_avx512vbmi_available(void) { boolean_t has_avx512 = B_FALSE; -#if defined(_KERNEL) #if defined(X86_FEATURE_AVX512VBMI) has_avx512 = boot_cpu_has(X86_FEATURE_AVX512F) && boot_cpu_has(X86_FEATURE_AVX512VBMI); -#else - has_avx512 = B_FALSE; #endif -#elif !defined(_KERNEL) - has_avx512 = __cpuid_has_avx512f() && - __cpuid_has_avx512vbmi(); -#endif - return (has_avx512 && __zmm_enabled()); } #endif /* defined(__x86) */ -#endif /* _SIMD_X86_H */ +#endif /* _LINUX_SIMD_X86_H */ diff --git a/include/linux/utsname_compat.h b/include/os/linux/kernel/linux/utsname_compat.h similarity index 100% rename from include/linux/utsname_compat.h rename to include/os/linux/kernel/linux/utsname_compat.h diff --git a/include/linux/vfs_compat.h b/include/os/linux/kernel/linux/vfs_compat.h similarity index 100% rename from include/linux/vfs_compat.h rename to include/os/linux/kernel/linux/vfs_compat.h diff --git a/include/linux/xattr_compat.h b/include/os/linux/kernel/linux/xattr_compat.h similarity index 100% rename from include/linux/xattr_compat.h rename to include/os/linux/kernel/linux/xattr_compat.h diff --git a/include/spl/Makefile.am b/include/os/linux/spl/Makefile.am similarity index 100% rename from include/spl/Makefile.am rename to include/os/linux/spl/Makefile.am diff --git a/include/spl/rpc/Makefile.am b/include/os/linux/spl/rpc/Makefile.am similarity index 72% rename from include/spl/rpc/Makefile.am rename to include/os/linux/spl/rpc/Makefile.am index 5110cc0f0c58..9477dd59d82a 100644 --- a/include/spl/rpc/Makefile.am +++ b/include/os/linux/spl/rpc/Makefile.am @@ -1,7 +1,7 @@ KERNEL_H = \ - $(top_srcdir)/include/spl/rpc/xdr.h + $(top_srcdir)/include/os/linux/spl/rpc/xdr.h if CONFIG_KERNEL kerneldir = @prefix@/src/zfs-$(VERSION)/include/spl/rpc kernel_HEADERS = $(KERNEL_H) endif diff --git a/include/spl/rpc/xdr.h b/include/os/linux/spl/rpc/xdr.h similarity index 100% rename from include/spl/rpc/xdr.h rename to include/os/linux/spl/rpc/xdr.h diff --git a/include/os/linux/spl/sys/Makefile.am b/include/os/linux/spl/sys/Makefile.am new file mode 100644 index 000000000000..de2f74d8cda3 --- /dev/null +++ b/include/os/linux/spl/sys/Makefile.am @@ -0,0 +1,62 @@ +KERNEL_H = \ + $(top_srcdir)/include/os/linux/spl/sys/acl.h \ + $(top_srcdir)/include/os/linux/spl/sys/atomic.h \ + $(top_srcdir)/include/os/linux/spl/sys/byteorder.h \ + $(top_srcdir)/include/os/linux/spl/sys/callb.h \ + $(top_srcdir)/include/os/linux/spl/sys/callo.h \ + $(top_srcdir)/include/os/linux/spl/sys/cmn_err.h \ + $(top_srcdir)/include/os/linux/spl/sys/condvar.h \ + $(top_srcdir)/include/os/linux/spl/sys/console.h \ + $(top_srcdir)/include/os/linux/spl/sys/cred.h \ + $(top_srcdir)/include/os/linux/spl/sys/ctype.h \ + $(top_srcdir)/include/os/linux/spl/sys/debug.h \ + $(top_srcdir)/include/os/linux/spl/sys/disp.h \ + $(top_srcdir)/include/os/linux/spl/sys/dkio.h \ + $(top_srcdir)/include/os/linux/spl/sys/errno.h \ + $(top_srcdir)/include/os/linux/spl/sys/fcntl.h \ + $(top_srcdir)/include/os/linux/spl/sys/file.h \ + $(top_srcdir)/include/os/linux/spl/sys/inttypes.h \ + $(top_srcdir)/include/os/linux/spl/sys/isa_defs.h \ + $(top_srcdir)/include/os/linux/spl/sys/kmem_cache.h \ + $(top_srcdir)/include/os/linux/spl/sys/kmem.h \ + $(top_srcdir)/include/os/linux/spl/sys/kobj.h \ + $(top_srcdir)/include/os/linux/spl/sys/kstat.h \ + $(top_srcdir)/include/os/linux/spl/sys/list.h \ + $(top_srcdir)/include/os/linux/spl/sys/mode.h \ + $(top_srcdir)/include/os/linux/spl/sys/mutex.h \ + $(top_srcdir)/include/os/linux/spl/sys/param.h \ + $(top_srcdir)/include/os/linux/spl/sys/processor.h \ + $(top_srcdir)/include/os/linux/spl/sys/proc.h \ + $(top_srcdir)/include/os/linux/spl/sys/procfs_list.h \ + $(top_srcdir)/include/os/linux/spl/sys/random.h \ + $(top_srcdir)/include/os/linux/spl/sys/rwlock.h \ + $(top_srcdir)/include/os/linux/spl/sys/shrinker.h \ + $(top_srcdir)/include/os/linux/spl/sys/sid.h \ + $(top_srcdir)/include/os/linux/spl/sys/signal.h \ + $(top_srcdir)/include/os/linux/spl/sys/simd.h \ + $(top_srcdir)/include/os/linux/spl/sys/stat.h \ + $(top_srcdir)/include/os/linux/spl/sys/strings.h \ + $(top_srcdir)/include/os/linux/spl/sys/sunddi.h \ + $(top_srcdir)/include/os/linux/spl/sys/sysmacros.h \ + $(top_srcdir)/include/os/linux/spl/sys/systeminfo.h \ + $(top_srcdir)/include/os/linux/spl/sys/taskq.h \ + $(top_srcdir)/include/os/linux/spl/sys/thread.h \ + $(top_srcdir)/include/os/linux/spl/sys/time.h \ + $(top_srcdir)/include/os/linux/spl/sys/timer.h \ + $(top_srcdir)/include/os/linux/spl/sys/tsd.h \ + $(top_srcdir)/include/os/linux/spl/sys/types32.h \ + $(top_srcdir)/include/os/linux/spl/sys/types.h \ + $(top_srcdir)/include/os/linux/spl/sys/uio.h \ + $(top_srcdir)/include/os/linux/spl/sys/user.h \ + $(top_srcdir)/include/os/linux/spl/sys/vfs.h \ + $(top_srcdir)/include/os/linux/spl/sys/vmem.h \ + $(top_srcdir)/include/os/linux/spl/sys/vmsystm.h \ + $(top_srcdir)/include/os/linux/spl/sys/vnode.h \ + $(top_srcdir)/include/os/linux/spl/sys/wait.h \ + $(top_srcdir)/include/os/linux/spl/sys/zmod.h \ + $(top_srcdir)/include/os/linux/spl/sys/zone.h + +if CONFIG_KERNEL +kerneldir = @prefix@/src/zfs-$(VERSION)/include/spl/sys +kernel_HEADERS = $(KERNEL_H) +endif diff --git a/include/spl/sys/acl.h b/include/os/linux/spl/sys/acl.h similarity index 100% rename from include/spl/sys/acl.h rename to include/os/linux/spl/sys/acl.h diff --git a/include/spl/sys/atomic.h b/include/os/linux/spl/sys/atomic.h similarity index 100% rename from include/spl/sys/atomic.h rename to include/os/linux/spl/sys/atomic.h diff --git a/include/spl/sys/byteorder.h b/include/os/linux/spl/sys/byteorder.h similarity index 100% rename from include/spl/sys/byteorder.h rename to include/os/linux/spl/sys/byteorder.h diff --git a/include/spl/sys/callb.h b/include/os/linux/spl/sys/callb.h similarity index 100% rename from include/spl/sys/callb.h rename to include/os/linux/spl/sys/callb.h diff --git a/include/spl/sys/callo.h b/include/os/linux/spl/sys/callo.h similarity index 100% rename from include/spl/sys/callo.h rename to include/os/linux/spl/sys/callo.h diff --git a/include/spl/sys/cmn_err.h b/include/os/linux/spl/sys/cmn_err.h similarity index 100% rename from include/spl/sys/cmn_err.h rename to include/os/linux/spl/sys/cmn_err.h diff --git a/include/spl/sys/condvar.h b/include/os/linux/spl/sys/condvar.h similarity index 100% rename from include/spl/sys/condvar.h rename to include/os/linux/spl/sys/condvar.h diff --git a/include/spl/sys/console.h b/include/os/linux/spl/sys/console.h similarity index 100% rename from include/spl/sys/console.h rename to include/os/linux/spl/sys/console.h diff --git a/include/spl/sys/cred.h b/include/os/linux/spl/sys/cred.h similarity index 100% rename from include/spl/sys/cred.h rename to include/os/linux/spl/sys/cred.h diff --git a/include/spl/sys/ctype.h b/include/os/linux/spl/sys/ctype.h similarity index 100% copy from include/spl/sys/ctype.h copy to include/os/linux/spl/sys/ctype.h diff --git a/include/spl/sys/debug.h b/include/os/linux/spl/sys/debug.h similarity index 100% rename from include/spl/sys/debug.h rename to include/os/linux/spl/sys/debug.h diff --git a/include/spl/sys/disp.h b/include/os/linux/spl/sys/disp.h similarity index 100% rename from include/spl/sys/disp.h rename to include/os/linux/spl/sys/disp.h diff --git a/include/spl/sys/dkio.h b/include/os/linux/spl/sys/dkio.h similarity index 100% rename from include/spl/sys/dkio.h rename to include/os/linux/spl/sys/dkio.h diff --git a/include/spl/sys/errno.h b/include/os/linux/spl/sys/errno.h similarity index 100% rename from include/spl/sys/errno.h rename to include/os/linux/spl/sys/errno.h diff --git a/include/spl/sys/fcntl.h b/include/os/linux/spl/sys/fcntl.h similarity index 100% rename from include/spl/sys/fcntl.h rename to include/os/linux/spl/sys/fcntl.h diff --git a/include/spl/sys/file.h b/include/os/linux/spl/sys/file.h similarity index 100% rename from include/spl/sys/file.h rename to include/os/linux/spl/sys/file.h diff --git a/include/spl/sys/inttypes.h b/include/os/linux/spl/sys/inttypes.h similarity index 100% rename from include/spl/sys/inttypes.h rename to include/os/linux/spl/sys/inttypes.h diff --git a/include/spl/sys/isa_defs.h b/include/os/linux/spl/sys/isa_defs.h similarity index 100% rename from include/spl/sys/isa_defs.h rename to include/os/linux/spl/sys/isa_defs.h diff --git a/include/spl/sys/kmem.h b/include/os/linux/spl/sys/kmem.h similarity index 100% rename from include/spl/sys/kmem.h rename to include/os/linux/spl/sys/kmem.h diff --git a/include/spl/sys/kmem_cache.h b/include/os/linux/spl/sys/kmem_cache.h similarity index 100% rename from include/spl/sys/kmem_cache.h rename to include/os/linux/spl/sys/kmem_cache.h diff --git a/include/spl/sys/kobj.h b/include/os/linux/spl/sys/kobj.h similarity index 100% rename from include/spl/sys/kobj.h rename to include/os/linux/spl/sys/kobj.h diff --git a/include/spl/sys/kstat.h b/include/os/linux/spl/sys/kstat.h similarity index 100% rename from include/spl/sys/kstat.h rename to include/os/linux/spl/sys/kstat.h diff --git a/include/spl/sys/list.h b/include/os/linux/spl/sys/list.h similarity index 100% rename from include/spl/sys/list.h rename to include/os/linux/spl/sys/list.h diff --git a/include/spl/sys/mode.h b/include/os/linux/spl/sys/mode.h similarity index 100% rename from include/spl/sys/mode.h rename to include/os/linux/spl/sys/mode.h diff --git a/include/spl/sys/mutex.h b/include/os/linux/spl/sys/mutex.h similarity index 100% rename from include/spl/sys/mutex.h rename to include/os/linux/spl/sys/mutex.h diff --git a/include/spl/sys/param.h b/include/os/linux/spl/sys/param.h similarity index 100% rename from include/spl/sys/param.h rename to include/os/linux/spl/sys/param.h diff --git a/include/spl/sys/proc.h b/include/os/linux/spl/sys/proc.h similarity index 100% rename from include/spl/sys/proc.h rename to include/os/linux/spl/sys/proc.h diff --git a/include/spl/sys/processor.h b/include/os/linux/spl/sys/processor.h similarity index 100% rename from include/spl/sys/processor.h rename to include/os/linux/spl/sys/processor.h diff --git a/include/spl/sys/procfs_list.h b/include/os/linux/spl/sys/procfs_list.h similarity index 100% rename from include/spl/sys/procfs_list.h rename to include/os/linux/spl/sys/procfs_list.h diff --git a/include/spl/sys/random.h b/include/os/linux/spl/sys/random.h similarity index 100% rename from include/spl/sys/random.h rename to include/os/linux/spl/sys/random.h diff --git a/include/spl/sys/rwlock.h b/include/os/linux/spl/sys/rwlock.h similarity index 100% rename from include/spl/sys/rwlock.h rename to include/os/linux/spl/sys/rwlock.h diff --git a/include/spl/sys/shrinker.h b/include/os/linux/spl/sys/shrinker.h similarity index 100% rename from include/spl/sys/shrinker.h rename to include/os/linux/spl/sys/shrinker.h diff --git a/include/spl/sys/sid.h b/include/os/linux/spl/sys/sid.h similarity index 100% rename from include/spl/sys/sid.h rename to include/os/linux/spl/sys/sid.h diff --git a/include/spl/sys/signal.h b/include/os/linux/spl/sys/signal.h similarity index 100% rename from include/spl/sys/signal.h rename to include/os/linux/spl/sys/signal.h diff --git a/include/spl/sys/ctype.h b/include/os/linux/spl/sys/simd.h similarity index 89% rename from include/spl/sys/ctype.h rename to include/os/linux/spl/sys/simd.h index 18beb1daa5d9..f2048d9e121c 100644 --- a/include/spl/sys/ctype.h +++ b/include/os/linux/spl/sys/simd.h @@ -1,30 +1,31 @@ /* * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. * Copyright (C) 2007 The Regents of the University of California. * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). * Written by Brian Behlendorf . * UCRL-CODE-235197 * * This file is part of the SPL, Solaris Porting Layer. * For details, see . * * The SPL is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * * The SPL is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * for more details. * * You should have received a copy of the GNU General Public License along * with the SPL. If not, see . */ -#ifndef _SPL_CTYPE_H -#define _SPL_CTYPE_H +#ifndef _SPL_SYS_SIMD_H +#define _SPL_SYS_SIMD_H -#include +#include +#include -#endif /* SPL_CTYPE_H */ +#endif /* _SPL_SYS_SIMD_H */ diff --git a/include/spl/sys/stat.h b/include/os/linux/spl/sys/stat.h similarity index 100% rename from include/spl/sys/stat.h rename to include/os/linux/spl/sys/stat.h diff --git a/include/spl/sys/strings.h b/include/os/linux/spl/sys/strings.h similarity index 100% rename from include/spl/sys/strings.h rename to include/os/linux/spl/sys/strings.h diff --git a/include/spl/sys/sunddi.h b/include/os/linux/spl/sys/sunddi.h similarity index 100% rename from include/spl/sys/sunddi.h rename to include/os/linux/spl/sys/sunddi.h diff --git a/include/spl/sys/sysmacros.h b/include/os/linux/spl/sys/sysmacros.h similarity index 100% rename from include/spl/sys/sysmacros.h rename to include/os/linux/spl/sys/sysmacros.h diff --git a/include/spl/sys/systeminfo.h b/include/os/linux/spl/sys/systeminfo.h similarity index 100% rename from include/spl/sys/systeminfo.h rename to include/os/linux/spl/sys/systeminfo.h diff --git a/include/spl/sys/taskq.h b/include/os/linux/spl/sys/taskq.h similarity index 100% rename from include/spl/sys/taskq.h rename to include/os/linux/spl/sys/taskq.h diff --git a/include/spl/sys/thread.h b/include/os/linux/spl/sys/thread.h similarity index 100% rename from include/spl/sys/thread.h rename to include/os/linux/spl/sys/thread.h diff --git a/include/spl/sys/time.h b/include/os/linux/spl/sys/time.h similarity index 100% rename from include/spl/sys/time.h rename to include/os/linux/spl/sys/time.h diff --git a/include/spl/sys/timer.h b/include/os/linux/spl/sys/timer.h similarity index 100% rename from include/spl/sys/timer.h rename to include/os/linux/spl/sys/timer.h diff --git a/include/spl/sys/tsd.h b/include/os/linux/spl/sys/tsd.h similarity index 100% rename from include/spl/sys/tsd.h rename to include/os/linux/spl/sys/tsd.h diff --git a/include/spl/sys/types.h b/include/os/linux/spl/sys/types.h similarity index 100% rename from include/spl/sys/types.h rename to include/os/linux/spl/sys/types.h diff --git a/include/spl/sys/types32.h b/include/os/linux/spl/sys/types32.h similarity index 100% rename from include/spl/sys/types32.h rename to include/os/linux/spl/sys/types32.h diff --git a/include/spl/sys/uio.h b/include/os/linux/spl/sys/uio.h similarity index 100% rename from include/spl/sys/uio.h rename to include/os/linux/spl/sys/uio.h diff --git a/include/spl/sys/user.h b/include/os/linux/spl/sys/user.h similarity index 100% rename from include/spl/sys/user.h rename to include/os/linux/spl/sys/user.h diff --git a/include/spl/sys/vfs.h b/include/os/linux/spl/sys/vfs.h similarity index 100% rename from include/spl/sys/vfs.h rename to include/os/linux/spl/sys/vfs.h diff --git a/include/spl/sys/vmem.h b/include/os/linux/spl/sys/vmem.h similarity index 100% rename from include/spl/sys/vmem.h rename to include/os/linux/spl/sys/vmem.h diff --git a/include/spl/sys/vmsystm.h b/include/os/linux/spl/sys/vmsystm.h similarity index 100% rename from include/spl/sys/vmsystm.h rename to include/os/linux/spl/sys/vmsystm.h diff --git a/include/spl/sys/vnode.h b/include/os/linux/spl/sys/vnode.h similarity index 100% rename from include/spl/sys/vnode.h rename to include/os/linux/spl/sys/vnode.h diff --git a/include/spl/sys/wait.h b/include/os/linux/spl/sys/wait.h similarity index 100% rename from include/spl/sys/wait.h rename to include/os/linux/spl/sys/wait.h diff --git a/include/spl/sys/zmod.h b/include/os/linux/spl/sys/zmod.h similarity index 100% rename from include/spl/sys/zmod.h rename to include/os/linux/spl/sys/zmod.h diff --git a/include/spl/sys/zone.h b/include/os/linux/spl/sys/zone.h similarity index 100% rename from include/spl/sys/zone.h rename to include/os/linux/spl/sys/zone.h diff --git a/include/os/linux/zfs/Makefile.am b/include/os/linux/zfs/Makefile.am new file mode 100644 index 000000000000..081839c48c8f --- /dev/null +++ b/include/os/linux/zfs/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = sys diff --git a/include/os/linux/zfs/sys/Makefile.am b/include/os/linux/zfs/sys/Makefile.am new file mode 100644 index 000000000000..5aa87da37c4a --- /dev/null +++ b/include/os/linux/zfs/sys/Makefile.am @@ -0,0 +1,12 @@ +KERNEL_H = \ + $(top_srcdir)/include/os/linux/zfs/sys/policy.h \ + $(top_srcdir)/include/os/linux/zfs/sys/zfs_ctldir.h \ + $(top_srcdir)/include/os/linux/zfs/sys/zfs_dir.h \ + $(top_srcdir)/include/os/linux/zfs/sys/zfs_vfsops.h \ + $(top_srcdir)/include/os/linux/zfs/sys/zfs_vnops.h \ + $(top_srcdir)/include/os/linux/zfs/sys/zpl.h + +if CONFIG_KERNEL +kerneldir = @prefix@/src/zfs-$(VERSION)/include/sys +kernel_HEADERS = $(KERNEL_H) +endif diff --git a/include/sys/policy.h b/include/os/linux/zfs/sys/policy.h similarity index 100% rename from include/sys/policy.h rename to include/os/linux/zfs/sys/policy.h diff --git a/include/sys/zfs_ctldir.h b/include/os/linux/zfs/sys/zfs_ctldir.h similarity index 100% rename from include/sys/zfs_ctldir.h rename to include/os/linux/zfs/sys/zfs_ctldir.h diff --git a/include/sys/zfs_dir.h b/include/os/linux/zfs/sys/zfs_dir.h similarity index 100% rename from include/sys/zfs_dir.h rename to include/os/linux/zfs/sys/zfs_dir.h diff --git a/include/sys/zfs_vfsops.h b/include/os/linux/zfs/sys/zfs_vfsops.h similarity index 100% rename from include/sys/zfs_vfsops.h rename to include/os/linux/zfs/sys/zfs_vfsops.h diff --git a/include/sys/zfs_vnops.h b/include/os/linux/zfs/sys/zfs_vnops.h similarity index 100% rename from include/sys/zfs_vnops.h rename to include/os/linux/zfs/sys/zfs_vnops.h diff --git a/include/sys/zpl.h b/include/os/linux/zfs/sys/zpl.h similarity index 100% rename from include/sys/zpl.h rename to include/os/linux/zfs/sys/zpl.h diff --git a/include/spl/sys/Makefile.am b/include/spl/sys/Makefile.am deleted file mode 100644 index 3b5b2755a2f2..000000000000 --- a/include/spl/sys/Makefile.am +++ /dev/null @@ -1,61 +0,0 @@ -KERNEL_H = \ - $(top_srcdir)/include/spl/sys/acl.h \ - $(top_srcdir)/include/spl/sys/atomic.h \ - $(top_srcdir)/include/spl/sys/byteorder.h \ - $(top_srcdir)/include/spl/sys/callb.h \ - $(top_srcdir)/include/spl/sys/callo.h \ - $(top_srcdir)/include/spl/sys/cmn_err.h \ - $(top_srcdir)/include/spl/sys/condvar.h \ - $(top_srcdir)/include/spl/sys/console.h \ - $(top_srcdir)/include/spl/sys/cred.h \ - $(top_srcdir)/include/spl/sys/ctype.h \ - $(top_srcdir)/include/spl/sys/debug.h \ - $(top_srcdir)/include/spl/sys/disp.h \ - $(top_srcdir)/include/spl/sys/dkio.h \ - $(top_srcdir)/include/spl/sys/errno.h \ - $(top_srcdir)/include/spl/sys/fcntl.h \ - $(top_srcdir)/include/spl/sys/file.h \ - $(top_srcdir)/include/spl/sys/inttypes.h \ - $(top_srcdir)/include/spl/sys/isa_defs.h \ - $(top_srcdir)/include/spl/sys/kmem_cache.h \ - $(top_srcdir)/include/spl/sys/kmem.h \ - $(top_srcdir)/include/spl/sys/kobj.h \ - $(top_srcdir)/include/spl/sys/kstat.h \ - $(top_srcdir)/include/spl/sys/list.h \ - $(top_srcdir)/include/spl/sys/mode.h \ - $(top_srcdir)/include/spl/sys/mutex.h \ - $(top_srcdir)/include/spl/sys/param.h \ - $(top_srcdir)/include/spl/sys/processor.h \ - $(top_srcdir)/include/spl/sys/proc.h \ - $(top_srcdir)/include/spl/sys/procfs_list.h \ - $(top_srcdir)/include/spl/sys/random.h \ - $(top_srcdir)/include/spl/sys/rwlock.h \ - $(top_srcdir)/include/spl/sys/shrinker.h \ - $(top_srcdir)/include/spl/sys/sid.h \ - $(top_srcdir)/include/spl/sys/signal.h \ - $(top_srcdir)/include/spl/sys/stat.h \ - $(top_srcdir)/include/spl/sys/strings.h \ - $(top_srcdir)/include/spl/sys/sunddi.h \ - $(top_srcdir)/include/spl/sys/sysmacros.h \ - $(top_srcdir)/include/spl/sys/systeminfo.h \ - $(top_srcdir)/include/spl/sys/taskq.h \ - $(top_srcdir)/include/spl/sys/thread.h \ - $(top_srcdir)/include/spl/sys/time.h \ - $(top_srcdir)/include/spl/sys/timer.h \ - $(top_srcdir)/include/spl/sys/tsd.h \ - $(top_srcdir)/include/spl/sys/types32.h \ - $(top_srcdir)/include/spl/sys/types.h \ - $(top_srcdir)/include/spl/sys/uio.h \ - $(top_srcdir)/include/spl/sys/user.h \ - $(top_srcdir)/include/spl/sys/vfs.h \ - $(top_srcdir)/include/spl/sys/vmem.h \ - $(top_srcdir)/include/spl/sys/vmsystm.h \ - $(top_srcdir)/include/spl/sys/vnode.h \ - $(top_srcdir)/include/spl/sys/wait.h \ - $(top_srcdir)/include/spl/sys/zmod.h \ - $(top_srcdir)/include/spl/sys/zone.h - -if CONFIG_KERNEL -kerneldir = @prefix@/src/zfs-$(VERSION)/include/spl/sys -kernel_HEADERS = $(KERNEL_H) -endif diff --git a/include/sys/Makefile.am b/include/sys/Makefile.am index c5d64f9fd4d8..b3845cb95b46 100644 --- a/include/sys/Makefile.am +++ b/include/sys/Makefile.am @@ -1,162 +1,156 @@ SUBDIRS = fm fs crypto lua sysevent COMMON_H = \ $(top_srcdir)/include/sys/abd.h \ $(top_srcdir)/include/sys/aggsum.h \ $(top_srcdir)/include/sys/arc.h \ $(top_srcdir)/include/sys/arc_impl.h \ $(top_srcdir)/include/sys/avl.h \ $(top_srcdir)/include/sys/avl_impl.h \ $(top_srcdir)/include/sys/blkptr.h \ $(top_srcdir)/include/sys/bplist.h \ $(top_srcdir)/include/sys/bpobj.h \ $(top_srcdir)/include/sys/bptree.h \ $(top_srcdir)/include/sys/bqueue.h \ $(top_srcdir)/include/sys/cityhash.h \ $(top_srcdir)/include/sys/dataset_kstats.h \ $(top_srcdir)/include/sys/dbuf.h \ $(top_srcdir)/include/sys/ddt.h \ $(top_srcdir)/include/sys/dmu.h \ $(top_srcdir)/include/sys/dmu_impl.h \ $(top_srcdir)/include/sys/dmu_objset.h \ $(top_srcdir)/include/sys/dmu_recv.h \ $(top_srcdir)/include/sys/dmu_redact.h \ $(top_srcdir)/include/sys/dmu_send.h \ $(top_srcdir)/include/sys/dmu_traverse.h \ $(top_srcdir)/include/sys/dmu_tx.h \ $(top_srcdir)/include/sys/dmu_zfetch.h \ $(top_srcdir)/include/sys/dnode.h \ $(top_srcdir)/include/sys/dsl_bookmark.h \ $(top_srcdir)/include/sys/dsl_dataset.h \ $(top_srcdir)/include/sys/dsl_deadlist.h \ $(top_srcdir)/include/sys/dsl_deleg.h \ $(top_srcdir)/include/sys/dsl_destroy.h \ $(top_srcdir)/include/sys/dsl_dir.h \ $(top_srcdir)/include/sys/dsl_crypt.h \ $(top_srcdir)/include/sys/dsl_pool.h \ $(top_srcdir)/include/sys/dsl_prop.h \ $(top_srcdir)/include/sys/dsl_scan.h \ $(top_srcdir)/include/sys/dsl_synctask.h \ $(top_srcdir)/include/sys/dsl_userhold.h \ $(top_srcdir)/include/sys/edonr.h \ $(top_srcdir)/include/sys/efi_partition.h \ $(top_srcdir)/include/sys/frame.h \ $(top_srcdir)/include/sys/hkdf.h \ $(top_srcdir)/include/sys/metaslab.h \ $(top_srcdir)/include/sys/metaslab_impl.h \ $(top_srcdir)/include/sys/mmp.h \ $(top_srcdir)/include/sys/mntent.h \ $(top_srcdir)/include/sys/multilist.h \ $(top_srcdir)/include/sys/note.h \ $(top_srcdir)/include/sys/nvpair.h \ $(top_srcdir)/include/sys/nvpair_impl.h \ $(top_srcdir)/include/sys/objlist.h \ $(top_srcdir)/include/sys/pathname.h \ - $(top_srcdir)/include/sys/policy.h \ $(top_srcdir)/include/sys/range_tree.h \ $(top_srcdir)/include/sys/refcount.h \ $(top_srcdir)/include/sys/rrwlock.h \ $(top_srcdir)/include/sys/sa.h \ $(top_srcdir)/include/sys/sa_impl.h \ $(top_srcdir)/include/sys/sdt.h \ $(top_srcdir)/include/sys/sha2.h \ $(top_srcdir)/include/sys/skein.h \ $(top_srcdir)/include/sys/spa_boot.h \ $(top_srcdir)/include/sys/spa_checkpoint.h \ $(top_srcdir)/include/sys/spa_log_spacemap.h \ $(top_srcdir)/include/sys/space_map.h \ $(top_srcdir)/include/sys/space_reftree.h \ $(top_srcdir)/include/sys/spa.h \ $(top_srcdir)/include/sys/spa_impl.h \ $(top_srcdir)/include/sys/spa_checksum.h \ $(top_srcdir)/include/sys/sysevent.h \ $(top_srcdir)/include/sys/trace.h \ $(top_srcdir)/include/sys/trace_acl.h \ $(top_srcdir)/include/sys/trace_arc.h \ $(top_srcdir)/include/sys/trace_common.h \ $(top_srcdir)/include/sys/trace_dbgmsg.h \ $(top_srcdir)/include/sys/trace_dbuf.h \ $(top_srcdir)/include/sys/trace_dmu.h \ $(top_srcdir)/include/sys/trace_dnode.h \ $(top_srcdir)/include/sys/trace_multilist.h \ $(top_srcdir)/include/sys/trace_rrwlock.h \ $(top_srcdir)/include/sys/trace_txg.h \ $(top_srcdir)/include/sys/trace_vdev.h \ $(top_srcdir)/include/sys/trace_zil.h \ $(top_srcdir)/include/sys/trace_zio.h \ $(top_srcdir)/include/sys/trace_zrlock.h \ $(top_srcdir)/include/sys/txg.h \ $(top_srcdir)/include/sys/txg_impl.h \ $(top_srcdir)/include/sys/u8_textprep_data.h \ $(top_srcdir)/include/sys/u8_textprep.h \ $(top_srcdir)/include/sys/uberblock.h \ $(top_srcdir)/include/sys/uberblock_impl.h \ $(top_srcdir)/include/sys/uio_impl.h \ $(top_srcdir)/include/sys/unique.h \ $(top_srcdir)/include/sys/uuid.h \ $(top_srcdir)/include/sys/vdev_disk.h \ $(top_srcdir)/include/sys/vdev_file.h \ $(top_srcdir)/include/sys/vdev.h \ $(top_srcdir)/include/sys/vdev_impl.h \ $(top_srcdir)/include/sys/vdev_indirect_births.h \ $(top_srcdir)/include/sys/vdev_indirect_mapping.h \ $(top_srcdir)/include/sys/vdev_initialize.h \ $(top_srcdir)/include/sys/vdev_raidz.h \ $(top_srcdir)/include/sys/vdev_raidz_impl.h \ $(top_srcdir)/include/sys/vdev_removal.h \ $(top_srcdir)/include/sys/vdev_trim.h \ $(top_srcdir)/include/sys/xvattr.h \ $(top_srcdir)/include/sys/zap.h \ $(top_srcdir)/include/sys/zap_impl.h \ $(top_srcdir)/include/sys/zap_leaf.h \ $(top_srcdir)/include/sys/zcp.h \ $(top_srcdir)/include/sys/zcp_global.h \ $(top_srcdir)/include/sys/zcp_iter.h \ $(top_srcdir)/include/sys/zcp_prop.h \ $(top_srcdir)/include/sys/zfeature.h \ $(top_srcdir)/include/sys/zfs_acl.h \ $(top_srcdir)/include/sys/zfs_context.h \ - $(top_srcdir)/include/sys/zfs_ctldir.h \ $(top_srcdir)/include/sys/zfs_debug.h \ $(top_srcdir)/include/sys/zfs_delay.h \ - $(top_srcdir)/include/sys/zfs_dir.h \ $(top_srcdir)/include/sys/zfs_fuid.h \ $(top_srcdir)/include/sys/zfs_project.h \ $(top_srcdir)/include/sys/zfs_ratelimit.h \ $(top_srcdir)/include/sys/zfs_rlock.h \ $(top_srcdir)/include/sys/zfs_sa.h \ $(top_srcdir)/include/sys/zfs_stat.h \ $(top_srcdir)/include/sys/zfs_sysfs.h \ - $(top_srcdir)/include/sys/zfs_vfsops.h \ - $(top_srcdir)/include/sys/zfs_vnops.h \ $(top_srcdir)/include/sys/zfs_znode.h \ $(top_srcdir)/include/sys/zil.h \ $(top_srcdir)/include/sys/zil_impl.h \ $(top_srcdir)/include/sys/zio_checksum.h \ $(top_srcdir)/include/sys/zio_compress.h \ $(top_srcdir)/include/sys/zio_crypt.h \ $(top_srcdir)/include/sys/zio.h \ $(top_srcdir)/include/sys/zio_impl.h \ $(top_srcdir)/include/sys/zio_priority.h \ $(top_srcdir)/include/sys/zrlock.h \ $(top_srcdir)/include/sys/zthr.h KERNEL_H = \ $(top_srcdir)/include/sys/zfs_ioctl.h \ $(top_srcdir)/include/sys/zfs_onexit.h \ - ${top_srcdir}/include/sys/zpl.h \ $(top_srcdir)/include/sys/zvol.h USER_H = EXTRA_DIST = $(COMMON_H) $(KERNEL_H) $(USER_H) if CONFIG_USER libzfsdir = $(includedir)/libzfs/sys libzfs_HEADERS = $(COMMON_H) $(USER_H) endif if CONFIG_KERNEL kerneldir = @prefix@/src/zfs-$(VERSION)/include/sys kernel_HEADERS = $(COMMON_H) $(KERNEL_H) endif diff --git a/lib/libspl/include/sys/Makefile.am b/lib/libspl/include/sys/Makefile.am index e7af317e0c6c..5156e2912ca5 100644 --- a/lib/libspl/include/sys/Makefile.am +++ b/lib/libspl/include/sys/Makefile.am @@ -1,53 +1,54 @@ SUBDIRS = dktp libspldir = $(includedir)/libspl/sys libspl_HEADERS = \ $(top_srcdir)/lib/libspl/include/sys/acl.h \ $(top_srcdir)/lib/libspl/include/sys/acl_impl.h \ $(top_srcdir)/lib/libspl/include/sys/bitmap.h \ $(top_srcdir)/lib/libspl/include/sys/byteorder.h \ $(top_srcdir)/lib/libspl/include/sys/callb.h \ $(top_srcdir)/lib/libspl/include/sys/cmn_err.h \ $(top_srcdir)/lib/libspl/include/sys/cred.h \ $(top_srcdir)/lib/libspl/include/sys/debug.h \ $(top_srcdir)/lib/libspl/include/sys/dkio.h \ $(top_srcdir)/lib/libspl/include/sys/dklabel.h \ $(top_srcdir)/lib/libspl/include/sys/errno.h \ $(top_srcdir)/lib/libspl/include/sys/feature_tests.h \ $(top_srcdir)/lib/libspl/include/sys/file.h \ $(top_srcdir)/lib/libspl/include/sys/int_limits.h \ $(top_srcdir)/lib/libspl/include/sys/int_types.h \ $(top_srcdir)/lib/libspl/include/sys/inttypes.h \ $(top_srcdir)/lib/libspl/include/sys/isa_defs.h \ $(top_srcdir)/lib/libspl/include/sys/kmem.h \ $(top_srcdir)/lib/libspl/include/sys/kstat.h \ $(top_srcdir)/lib/libspl/include/sys/list.h \ $(top_srcdir)/lib/libspl/include/sys/list_impl.h \ $(top_srcdir)/lib/libspl/include/sys/mhd.h \ $(top_srcdir)/lib/libspl/include/sys/mkdev.h \ $(top_srcdir)/lib/libspl/include/sys/mnttab.h \ $(top_srcdir)/lib/libspl/include/sys/mount.h \ $(top_srcdir)/lib/libspl/include/sys/param.h \ $(top_srcdir)/lib/libspl/include/sys/policy.h \ $(top_srcdir)/lib/libspl/include/sys/poll.h \ $(top_srcdir)/lib/libspl/include/sys/priv.h \ $(top_srcdir)/lib/libspl/include/sys/processor.h \ $(top_srcdir)/lib/libspl/include/sys/signal.h \ + $(top_srcdir)/lib/libspl/include/sys/simd.h \ $(top_srcdir)/lib/libspl/include/sys/stack.h \ $(top_srcdir)/lib/libspl/include/sys/stat.h \ $(top_srcdir)/lib/libspl/include/sys/stdtypes.h \ $(top_srcdir)/lib/libspl/include/sys/strings.h \ $(top_srcdir)/lib/libspl/include/sys/stropts.h \ $(top_srcdir)/lib/libspl/include/sys/sunddi.h \ $(top_srcdir)/lib/libspl/include/sys/sysmacros.h \ $(top_srcdir)/lib/libspl/include/sys/systeminfo.h \ $(top_srcdir)/lib/libspl/include/sys/time.h \ $(top_srcdir)/lib/libspl/include/sys/types32.h \ $(top_srcdir)/lib/libspl/include/sys/types.h \ $(top_srcdir)/lib/libspl/include/sys/tzfile.h \ $(top_srcdir)/lib/libspl/include/sys/uio.h \ $(top_srcdir)/lib/libspl/include/sys/va_list.h \ $(top_srcdir)/lib/libspl/include/sys/varargs.h \ $(top_srcdir)/lib/libspl/include/sys/vnode.h \ $(top_srcdir)/lib/libspl/include/sys/vtoc.h \ $(top_srcdir)/lib/libspl/include/sys/zone.h diff --git a/lib/libspl/include/sys/simd.h b/lib/libspl/include/sys/simd.h new file mode 100644 index 000000000000..6a2b3a0226f6 --- /dev/null +++ b/lib/libspl/include/sys/simd.h @@ -0,0 +1,448 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License, Version 1.0 only + * (the "License"). You may not use this file except in compliance + * with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _LIBSPL_SYS_SIMD_H +#define _LIBSPL_SYS_SIMD_H + +#include +#include + +#if defined(__x86) +#include + +#define kfpu_allowed() 1 +#define kfpu_initialize(tsk) do {} while (0) +#define kfpu_begin() do {} while (0) +#define kfpu_end() do {} while (0) + +/* + * CPUID feature tests for user-space. + * + * x86 registers used implicitly by CPUID + */ +typedef enum cpuid_regs { + EAX = 0, + EBX, + ECX, + EDX, + CPUID_REG_CNT = 4 +} cpuid_regs_t; + +/* + * List of instruction sets identified by CPUID + */ +typedef enum cpuid_inst_sets { + SSE = 0, + SSE2, + SSE3, + SSSE3, + SSE4_1, + SSE4_2, + OSXSAVE, + AVX, + AVX2, + BMI1, + BMI2, + AVX512F, + AVX512CD, + AVX512DQ, + AVX512BW, + AVX512IFMA, + AVX512VBMI, + AVX512PF, + AVX512ER, + AVX512VL, + AES, + PCLMULQDQ +} cpuid_inst_sets_t; + +/* + * Instruction set descriptor. + */ +typedef struct cpuid_feature_desc { + uint32_t leaf; /* CPUID leaf */ + uint32_t subleaf; /* CPUID sub-leaf */ + uint32_t flag; /* bit mask of the feature */ + cpuid_regs_t reg; /* which CPUID return register to test */ +} cpuid_feature_desc_t; + +#define _AVX512F_BIT (1U << 16) +#define _AVX512CD_BIT (_AVX512F_BIT | (1U << 28)) +#define _AVX512DQ_BIT (_AVX512F_BIT | (1U << 17)) +#define _AVX512BW_BIT (_AVX512F_BIT | (1U << 30)) +#define _AVX512IFMA_BIT (_AVX512F_BIT | (1U << 21)) +#define _AVX512VBMI_BIT (1U << 1) /* AVX512F_BIT is on another leaf */ +#define _AVX512PF_BIT (_AVX512F_BIT | (1U << 26)) +#define _AVX512ER_BIT (_AVX512F_BIT | (1U << 27)) +#define _AVX512VL_BIT (1U << 31) /* if used also check other levels */ +#define _AES_BIT (1U << 25) +#define _PCLMULQDQ_BIT (1U << 1) + +/* + * Descriptions of supported instruction sets + */ +static const cpuid_feature_desc_t cpuid_features[] = { + [SSE] = {1U, 0U, 1U << 25, EDX }, + [SSE2] = {1U, 0U, 1U << 26, EDX }, + [SSE3] = {1U, 0U, 1U << 0, ECX }, + [SSSE3] = {1U, 0U, 1U << 9, ECX }, + [SSE4_1] = {1U, 0U, 1U << 19, ECX }, + [SSE4_2] = {1U, 0U, 1U << 20, ECX }, + [OSXSAVE] = {1U, 0U, 1U << 27, ECX }, + [AVX] = {1U, 0U, 1U << 28, ECX }, + [AVX2] = {7U, 0U, 1U << 5, EBX }, + [BMI1] = {7U, 0U, 1U << 3, EBX }, + [BMI2] = {7U, 0U, 1U << 8, EBX }, + [AVX512F] = {7U, 0U, _AVX512F_BIT, EBX }, + [AVX512CD] = {7U, 0U, _AVX512CD_BIT, EBX }, + [AVX512DQ] = {7U, 0U, _AVX512DQ_BIT, EBX }, + [AVX512BW] = {7U, 0U, _AVX512BW_BIT, EBX }, + [AVX512IFMA] = {7U, 0U, _AVX512IFMA_BIT, EBX }, + [AVX512VBMI] = {7U, 0U, _AVX512VBMI_BIT, ECX }, + [AVX512PF] = {7U, 0U, _AVX512PF_BIT, EBX }, + [AVX512ER] = {7U, 0U, _AVX512ER_BIT, EBX }, + [AVX512VL] = {7U, 0U, _AVX512ER_BIT, EBX }, + [AES] = {1U, 0U, _AES_BIT, ECX }, + [PCLMULQDQ] = {1U, 0U, _PCLMULQDQ_BIT, ECX }, +}; + +/* + * Check if OS supports AVX and AVX2 by checking XCR0 + * Only call this function if CPUID indicates that AVX feature is + * supported by the CPU, otherwise it might be an illegal instruction. + */ +static inline uint64_t +xgetbv(uint32_t index) +{ + uint32_t eax, edx; + /* xgetbv - instruction byte code */ + __asm__ __volatile__(".byte 0x0f; .byte 0x01; .byte 0xd0" + : "=a" (eax), "=d" (edx) + : "c" (index)); + + return ((((uint64_t)edx)<<32) | (uint64_t)eax); +} + +/* + * Check if CPU supports a feature + */ +static inline boolean_t +__cpuid_check_feature(const cpuid_feature_desc_t *desc) +{ + uint32_t r[CPUID_REG_CNT]; + + if (__get_cpuid_max(0, NULL) >= desc->leaf) { + /* + * __cpuid_count is needed to properly check + * for AVX2. It is a macro, so return parameters + * are passed by value. + */ + __cpuid_count(desc->leaf, desc->subleaf, + r[EAX], r[EBX], r[ECX], r[EDX]); + return ((r[desc->reg] & desc->flag) == desc->flag); + } + return (B_FALSE); +} + +#define CPUID_FEATURE_CHECK(name, id) \ +static inline boolean_t \ +__cpuid_has_ ## name(void) \ +{ \ + return (__cpuid_check_feature(&cpuid_features[id])); \ +} + +/* + * Define functions for user-space CPUID features testing + */ +CPUID_FEATURE_CHECK(sse, SSE); +CPUID_FEATURE_CHECK(sse2, SSE2); +CPUID_FEATURE_CHECK(sse3, SSE3); +CPUID_FEATURE_CHECK(ssse3, SSSE3); +CPUID_FEATURE_CHECK(sse4_1, SSE4_1); +CPUID_FEATURE_CHECK(sse4_2, SSE4_2); +CPUID_FEATURE_CHECK(avx, AVX); +CPUID_FEATURE_CHECK(avx2, AVX2); +CPUID_FEATURE_CHECK(osxsave, OSXSAVE); +CPUID_FEATURE_CHECK(bmi1, BMI1); +CPUID_FEATURE_CHECK(bmi2, BMI2); +CPUID_FEATURE_CHECK(avx512f, AVX512F); +CPUID_FEATURE_CHECK(avx512cd, AVX512CD); +CPUID_FEATURE_CHECK(avx512dq, AVX512DQ); +CPUID_FEATURE_CHECK(avx512bw, AVX512BW); +CPUID_FEATURE_CHECK(avx512ifma, AVX512IFMA); +CPUID_FEATURE_CHECK(avx512vbmi, AVX512VBMI); +CPUID_FEATURE_CHECK(avx512pf, AVX512PF); +CPUID_FEATURE_CHECK(avx512er, AVX512ER); +CPUID_FEATURE_CHECK(avx512vl, AVX512VL); +CPUID_FEATURE_CHECK(aes, AES); +CPUID_FEATURE_CHECK(pclmulqdq, PCLMULQDQ); + +/* + * Detect register set support + */ +static inline boolean_t +__simd_state_enabled(const uint64_t state) +{ + boolean_t has_osxsave; + uint64_t xcr0; + + has_osxsave = __cpuid_has_osxsave(); + if (!has_osxsave) + return (B_FALSE); + + xcr0 = xgetbv(0); + return ((xcr0 & state) == state); +} + +#define _XSTATE_SSE_AVX (0x2 | 0x4) +#define _XSTATE_AVX512 (0xE0 | _XSTATE_SSE_AVX) + +#define __ymm_enabled() __simd_state_enabled(_XSTATE_SSE_AVX) +#define __zmm_enabled() __simd_state_enabled(_XSTATE_AVX512) + +/* + * Check if SSE instruction set is available + */ +static inline boolean_t +zfs_sse_available(void) +{ + return (__cpuid_has_sse()); +} + +/* + * Check if SSE2 instruction set is available + */ +static inline boolean_t +zfs_sse2_available(void) +{ + return (__cpuid_has_sse2()); +} + +/* + * Check if SSE3 instruction set is available + */ +static inline boolean_t +zfs_sse3_available(void) +{ + return (__cpuid_has_sse3()); +} + +/* + * Check if SSSE3 instruction set is available + */ +static inline boolean_t +zfs_ssse3_available(void) +{ + return (__cpuid_has_ssse3()); +} + +/* + * Check if SSE4.1 instruction set is available + */ +static inline boolean_t +zfs_sse4_1_available(void) +{ + return (__cpuid_has_sse4_1()); +} + +/* + * Check if SSE4.2 instruction set is available + */ +static inline boolean_t +zfs_sse4_2_available(void) +{ + return (__cpuid_has_sse4_2()); +} + +/* + * Check if AVX instruction set is available + */ +static inline boolean_t +zfs_avx_available(void) +{ + return (__cpuid_has_avx() && __ymm_enabled()); +} + +/* + * Check if AVX2 instruction set is available + */ +static inline boolean_t +zfs_avx2_available(void) +{ + return (__cpuid_has_avx2() && __ymm_enabled()); +} + +/* + * Check if BMI1 instruction set is available + */ +static inline boolean_t +zfs_bmi1_available(void) +{ + return (__cpuid_has_bmi1()); +} + +/* + * Check if BMI2 instruction set is available + */ +static inline boolean_t +zfs_bmi2_available(void) +{ + return (__cpuid_has_bmi2()); +} + +/* + * Check if AES instruction set is available + */ +static inline boolean_t +zfs_aes_available(void) +{ + return (__cpuid_has_aes()); +} + +/* + * Check if PCLMULQDQ instruction set is available + */ +static inline boolean_t +zfs_pclmulqdq_available(void) +{ + return (__cpuid_has_pclmulqdq()); +} + +/* + * AVX-512 family of instruction sets: + * + * AVX512F Foundation + * AVX512CD Conflict Detection Instructions + * AVX512ER Exponential and Reciprocal Instructions + * AVX512PF Prefetch Instructions + * + * AVX512BW Byte and Word Instructions + * AVX512DQ Double-word and Quadword Instructions + * AVX512VL Vector Length Extensions + * + * AVX512IFMA Integer Fused Multiply Add (Not supported by kernel 4.4) + * AVX512VBMI Vector Byte Manipulation Instructions + */ + +/* + * Check if AVX512F instruction set is available + */ +static inline boolean_t +zfs_avx512f_available(void) +{ + return (__cpuid_has_avx512f() && __zmm_enabled()); +} + +/* + * Check if AVX512CD instruction set is available + */ +static inline boolean_t +zfs_avx512cd_available(void) +{ + return (__cpuid_has_avx512cd() && __zmm_enabled()); +} + +/* + * Check if AVX512ER instruction set is available + */ +static inline boolean_t +zfs_avx512er_available(void) +{ + return (__cpuid_has_avx512er() && __zmm_enabled()); +} + +/* + * Check if AVX512PF instruction set is available + */ +static inline boolean_t +zfs_avx512pf_available(void) +{ + return (__cpuid_has_avx512pf() && __zmm_enabled()); +} + +/* + * Check if AVX512BW instruction set is available + */ +static inline boolean_t +zfs_avx512bw_available(void) +{ + return (__cpuid_has_avx512bw() && __zmm_enabled()); +} + +/* + * Check if AVX512DQ instruction set is available + */ +static inline boolean_t +zfs_avx512dq_available(void) +{ + return (__cpuid_has_avx512dq() && __zmm_enabled()); +} + +/* + * Check if AVX512VL instruction set is available + */ +static inline boolean_t +zfs_avx512vl_available(void) +{ + return (__cpuid_has_avx512vl() && __zmm_enabled()); +} + +/* + * Check if AVX512IFMA instruction set is available + */ +static inline boolean_t +zfs_avx512ifma_available(void) +{ + return (__cpuid_has_avx512ifma() && __zmm_enabled()); +} + +/* + * Check if AVX512VBMI instruction set is available + */ +static inline boolean_t +zfs_avx512vbmi_available(void) +{ + return (__cpuid_has_avx512f() && __cpuid_has_avx512vbmi() && + __zmm_enabled()); +} + +#elif defined(__aarch64__) + +#define kfpu_allowed() 1 +#define kfpu_initialize(tsk) do {} while (0) +#define kfpu_begin() do {} while (0) +#define kfpu_end() do {} while (0) + +#else + +#define kfpu_allowed() 0 +#define kfpu_initialize(tsk) do {} while (0) +#define kfpu_begin() do {} while (0) +#define kfpu_end() do {} while (0) + +#endif + +#endif /* _LIBSPL_SYS_SIMD_H */ diff --git a/module/Makefile.in b/module/Makefile.in index 7477dbe56509..bf0eb101c06c 100644 --- a/module/Makefile.in +++ b/module/Makefile.in @@ -1,78 +1,80 @@ obj-m += avl/ obj-m += icp/ obj-m += lua/ obj-m += nvpair/ obj-m += spl/ obj-m += unicode/ obj-m += zcommon/ obj-m += zfs/ INSTALL_MOD_DIR ?= extra ZFS_MODULE_CFLAGS += -std=gnu99 -Wno-declaration-after-statement ZFS_MODULE_CFLAGS += @KERNEL_DEBUG_CFLAGS@ ZFS_MODULE_CFLAGS += -include @abs_top_builddir@/zfs_config.h -ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include/spl +ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include/os/linux/kernel +ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include/os/linux/spl +ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include/os/linux/zfs ZFS_MODULE_CFLAGS += -I@abs_top_srcdir@/include ZFS_MODULE_CPPFLAGS += -D_KERNEL ZFS_MODULE_CPPFLAGS += @KERNEL_DEBUG_CPPFLAGS@ @CONFIG_QAT_TRUE@ZFS_MODULE_CFLAGS += -I@QAT_SRC@/include @CONFIG_QAT_TRUE@KBUILD_EXTRA_SYMBOLS += @QAT_SYMBOLS@ export ZFS_MODULE_CFLAGS ZFS_MODULE_CPPFLAGS SUBDIR_TARGETS = icp lua modules: list='$(SUBDIR_TARGETS)'; for targetdir in $$list; do \ $(MAKE) -C $$targetdir; \ done $(MAKE) -C @LINUX_OBJ@ M=`pwd` @KERNEL_MAKE@ CONFIG_ZFS=m $@ clean: @# Only cleanup the kernel build directories when CONFIG_KERNEL @# is defined. This indicates that kernel modules should be built. @CONFIG_KERNEL_TRUE@ $(MAKE) -C @LINUX_OBJ@ M=`pwd` @KERNEL_MAKE@ $@ if [ -f @LINUX_SYMBOLS@ ]; then $(RM) @LINUX_SYMBOLS@; fi if [ -f Module.markers ]; then $(RM) Module.markers; fi find . -name '*.ur-safe' -type f -print | xargs $(RM) modules_install: @# Install the kernel modules $(MAKE) -C @LINUX_OBJ@ M=`pwd` $@ \ INSTALL_MOD_PATH=$(DESTDIR)$(INSTALL_MOD_PATH) \ INSTALL_MOD_DIR=$(INSTALL_MOD_DIR) \ KERNELRELEASE=@LINUX_VERSION@ @# Remove extraneous build products when packaging kmoddir=$(DESTDIR)$(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@; \ if [ -n "$(DESTDIR)" ]; then \ find $$kmoddir -name 'modules.*' | xargs $(RM); \ fi sysmap=$(DESTDIR)$(INSTALL_MOD_PATH)/boot/System.map-@LINUX_VERSION@; \ if [ -f $$sysmap ]; then \ depmod -ae -F $$sysmap @LINUX_VERSION@; \ fi modules_uninstall: @# Uninstall the kernel modules kmoddir=$(DESTDIR)$(INSTALL_MOD_PATH)/lib/modules/@LINUX_VERSION@ list='$(obj-m)'; for objdir in $$list; do \ $(RM) -R $$kmoddir/$(INSTALL_MOD_DIR)/$$objdir; \ done distdir: list='$(obj-m)'; for objdir in $$list; do \ (cd @top_srcdir@/module && find $$objdir \ -name '*.c' -o -name '*.h' -o -name '*.S' | \ xargs cp --parents -t @abs_top_builddir@/module/$$distdir); \ done distclean maintainer-clean: clean install: modules_install uninstall: modules_uninstall all: modules check: diff --git a/module/icp/algs/aes/aes_impl.c b/module/icp/algs/aes/aes_impl.c index 0f11f9999ff9..b60b16cc01d3 100644 --- a/module/icp/algs/aes/aes_impl.c +++ b/module/icp/algs/aes/aes_impl.c @@ -1,445 +1,445 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. */ #include #include #include +#include #include #include -#include /* * Initialize AES encryption and decryption key schedules. * * Parameters: * cipherKey User key * keyBits AES key size (128, 192, or 256 bits) * keysched AES key schedule to be initialized, of type aes_key_t. * Allocated by aes_alloc_keysched(). */ void aes_init_keysched(const uint8_t *cipherKey, uint_t keyBits, void *keysched) { const aes_impl_ops_t *ops = aes_impl_get_ops(); aes_key_t *newbie = keysched; uint_t keysize, i, j; union { uint64_t ka64[4]; uint32_t ka32[8]; } keyarr; switch (keyBits) { case 128: newbie->nr = 10; break; case 192: newbie->nr = 12; break; case 256: newbie->nr = 14; break; default: /* should never get here */ return; } keysize = CRYPTO_BITS2BYTES(keyBits); /* * Generic C implementation requires byteswap for little endian * machines, various accelerated implementations for various * architectures may not. */ if (!ops->needs_byteswap) { /* no byteswap needed */ if (IS_P2ALIGNED(cipherKey, sizeof (uint64_t))) { for (i = 0, j = 0; j < keysize; i++, j += 8) { /* LINTED: pointer alignment */ keyarr.ka64[i] = *((uint64_t *)&cipherKey[j]); } } else { bcopy(cipherKey, keyarr.ka32, keysize); } } else { /* byte swap */ for (i = 0, j = 0; j < keysize; i++, j += 4) { keyarr.ka32[i] = htonl(*(uint32_t *)(void *)&cipherKey[j]); } } ops->generate(newbie, keyarr.ka32, keyBits); newbie->ops = ops; /* * Note: if there are systems that need the AES_64BIT_KS type in the * future, move setting key schedule type to individual implementations */ newbie->type = AES_32BIT_KS; } /* * Encrypt one block using AES. * Align if needed and (for x86 32-bit only) byte-swap. * * Parameters: * ks Key schedule, of type aes_key_t * pt Input block (plain text) * ct Output block (crypto text). Can overlap with pt */ int aes_encrypt_block(const void *ks, const uint8_t *pt, uint8_t *ct) { aes_key_t *ksch = (aes_key_t *)ks; const aes_impl_ops_t *ops = ksch->ops; if (IS_P2ALIGNED2(pt, ct, sizeof (uint32_t)) && !ops->needs_byteswap) { /* LINTED: pointer alignment */ ops->encrypt(&ksch->encr_ks.ks32[0], ksch->nr, /* LINTED: pointer alignment */ (uint32_t *)pt, (uint32_t *)ct); } else { uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)]; /* Copy input block into buffer */ if (ops->needs_byteswap) { buffer[0] = htonl(*(uint32_t *)(void *)&pt[0]); buffer[1] = htonl(*(uint32_t *)(void *)&pt[4]); buffer[2] = htonl(*(uint32_t *)(void *)&pt[8]); buffer[3] = htonl(*(uint32_t *)(void *)&pt[12]); } else bcopy(pt, &buffer, AES_BLOCK_LEN); ops->encrypt(&ksch->encr_ks.ks32[0], ksch->nr, buffer, buffer); /* Copy result from buffer to output block */ if (ops->needs_byteswap) { *(uint32_t *)(void *)&ct[0] = htonl(buffer[0]); *(uint32_t *)(void *)&ct[4] = htonl(buffer[1]); *(uint32_t *)(void *)&ct[8] = htonl(buffer[2]); *(uint32_t *)(void *)&ct[12] = htonl(buffer[3]); } else bcopy(&buffer, ct, AES_BLOCK_LEN); } return (CRYPTO_SUCCESS); } /* * Decrypt one block using AES. * Align and byte-swap if needed. * * Parameters: * ks Key schedule, of type aes_key_t * ct Input block (crypto text) * pt Output block (plain text). Can overlap with pt */ int aes_decrypt_block(const void *ks, const uint8_t *ct, uint8_t *pt) { aes_key_t *ksch = (aes_key_t *)ks; const aes_impl_ops_t *ops = ksch->ops; if (IS_P2ALIGNED2(ct, pt, sizeof (uint32_t)) && !ops->needs_byteswap) { /* LINTED: pointer alignment */ ops->decrypt(&ksch->decr_ks.ks32[0], ksch->nr, /* LINTED: pointer alignment */ (uint32_t *)ct, (uint32_t *)pt); } else { uint32_t buffer[AES_BLOCK_LEN / sizeof (uint32_t)]; /* Copy input block into buffer */ if (ops->needs_byteswap) { buffer[0] = htonl(*(uint32_t *)(void *)&ct[0]); buffer[1] = htonl(*(uint32_t *)(void *)&ct[4]); buffer[2] = htonl(*(uint32_t *)(void *)&ct[8]); buffer[3] = htonl(*(uint32_t *)(void *)&ct[12]); } else bcopy(ct, &buffer, AES_BLOCK_LEN); ops->decrypt(&ksch->decr_ks.ks32[0], ksch->nr, buffer, buffer); /* Copy result from buffer to output block */ if (ops->needs_byteswap) { *(uint32_t *)(void *)&pt[0] = htonl(buffer[0]); *(uint32_t *)(void *)&pt[4] = htonl(buffer[1]); *(uint32_t *)(void *)&pt[8] = htonl(buffer[2]); *(uint32_t *)(void *)&pt[12] = htonl(buffer[3]); } else bcopy(&buffer, pt, AES_BLOCK_LEN); } return (CRYPTO_SUCCESS); } /* * Allocate key schedule for AES. * * Return the pointer and set size to the number of bytes allocated. * Memory allocated must be freed by the caller when done. * * Parameters: * size Size of key schedule allocated, in bytes * kmflag Flag passed to kmem_alloc(9F); ignored in userland. */ /* ARGSUSED */ void * aes_alloc_keysched(size_t *size, int kmflag) { aes_key_t *keysched; keysched = (aes_key_t *)kmem_alloc(sizeof (aes_key_t), kmflag); if (keysched != NULL) { *size = sizeof (aes_key_t); return (keysched); } return (NULL); } /* AES implementation that contains the fastest methods */ static aes_impl_ops_t aes_fastest_impl = { .name = "fastest" }; /* All compiled in implementations */ const aes_impl_ops_t *aes_all_impl[] = { &aes_generic_impl, #if defined(__x86_64) &aes_x86_64_impl, #endif #if defined(__x86_64) && defined(HAVE_AES) &aes_aesni_impl, #endif }; /* Indicate that benchmark has been completed */ static boolean_t aes_impl_initialized = B_FALSE; /* Select aes implementation */ #define IMPL_FASTEST (UINT32_MAX) #define IMPL_CYCLE (UINT32_MAX-1) #define AES_IMPL_READ(i) (*(volatile uint32_t *) &(i)) static uint32_t icp_aes_impl = IMPL_FASTEST; static uint32_t user_sel_impl = IMPL_FASTEST; /* Hold all supported implementations */ static size_t aes_supp_impl_cnt = 0; static aes_impl_ops_t *aes_supp_impl[ARRAY_SIZE(aes_all_impl)]; /* * Returns the AES operations for encrypt/decrypt/key setup. When a * SIMD implementation is not allowed in the current context, then * fallback to the fastest generic implementation. */ const aes_impl_ops_t * aes_impl_get_ops(void) { if (!kfpu_allowed()) return (&aes_generic_impl); const aes_impl_ops_t *ops = NULL; const uint32_t impl = AES_IMPL_READ(icp_aes_impl); switch (impl) { case IMPL_FASTEST: ASSERT(aes_impl_initialized); ops = &aes_fastest_impl; break; case IMPL_CYCLE: /* Cycle through supported implementations */ ASSERT(aes_impl_initialized); ASSERT3U(aes_supp_impl_cnt, >, 0); static size_t cycle_impl_idx = 0; size_t idx = (++cycle_impl_idx) % aes_supp_impl_cnt; ops = aes_supp_impl[idx]; break; default: ASSERT3U(impl, <, aes_supp_impl_cnt); ASSERT3U(aes_supp_impl_cnt, >, 0); if (impl < ARRAY_SIZE(aes_all_impl)) ops = aes_supp_impl[impl]; break; } ASSERT3P(ops, !=, NULL); return (ops); } /* * Initialize all supported implementations. */ /* ARGSUSED */ void aes_impl_init(void *arg) { aes_impl_ops_t *curr_impl; int i, c; /* Move supported implementations into aes_supp_impls */ for (i = 0, c = 0; i < ARRAY_SIZE(aes_all_impl); i++) { curr_impl = (aes_impl_ops_t *)aes_all_impl[i]; if (curr_impl->is_supported()) aes_supp_impl[c++] = (aes_impl_ops_t *)curr_impl; } aes_supp_impl_cnt = c; /* * Set the fastest implementation given the assumption that the * hardware accelerated version is the fastest. */ #if defined(__x86_64) #if defined(HAVE_AES) if (aes_aesni_impl.is_supported()) { memcpy(&aes_fastest_impl, &aes_aesni_impl, sizeof (aes_fastest_impl)); } else #endif { memcpy(&aes_fastest_impl, &aes_x86_64_impl, sizeof (aes_fastest_impl)); } #else memcpy(&aes_fastest_impl, &aes_generic_impl, sizeof (aes_fastest_impl)); #endif strcpy(aes_fastest_impl.name, "fastest"); /* Finish initialization */ atomic_swap_32(&icp_aes_impl, user_sel_impl); aes_impl_initialized = B_TRUE; } static const struct { char *name; uint32_t sel; } aes_impl_opts[] = { { "cycle", IMPL_CYCLE }, { "fastest", IMPL_FASTEST }, }; /* * Function sets desired aes implementation. * * If we are called before init(), user preference will be saved in * user_sel_impl, and applied in later init() call. This occurs when module * parameter is specified on module load. Otherwise, directly update * icp_aes_impl. * * @val Name of aes implementation to use * @param Unused. */ int aes_impl_set(const char *val) { int err = -EINVAL; char req_name[AES_IMPL_NAME_MAX]; uint32_t impl = AES_IMPL_READ(user_sel_impl); size_t i; /* sanitize input */ i = strnlen(val, AES_IMPL_NAME_MAX); if (i == 0 || i >= AES_IMPL_NAME_MAX) return (err); strlcpy(req_name, val, AES_IMPL_NAME_MAX); while (i > 0 && isspace(req_name[i-1])) i--; req_name[i] = '\0'; /* Check mandatory options */ for (i = 0; i < ARRAY_SIZE(aes_impl_opts); i++) { if (strcmp(req_name, aes_impl_opts[i].name) == 0) { impl = aes_impl_opts[i].sel; err = 0; break; } } /* check all supported impl if init() was already called */ if (err != 0 && aes_impl_initialized) { /* check all supported implementations */ for (i = 0; i < aes_supp_impl_cnt; i++) { if (strcmp(req_name, aes_supp_impl[i]->name) == 0) { impl = i; err = 0; break; } } } if (err == 0) { if (aes_impl_initialized) atomic_swap_32(&icp_aes_impl, impl); else atomic_swap_32(&user_sel_impl, impl); } return (err); } #if defined(_KERNEL) #include static int icp_aes_impl_set(const char *val, zfs_kernel_param_t *kp) { return (aes_impl_set(val)); } static int icp_aes_impl_get(char *buffer, zfs_kernel_param_t *kp) { int i, cnt = 0; char *fmt; const uint32_t impl = AES_IMPL_READ(icp_aes_impl); ASSERT(aes_impl_initialized); /* list mandatory options */ for (i = 0; i < ARRAY_SIZE(aes_impl_opts); i++) { fmt = (impl == aes_impl_opts[i].sel) ? "[%s] " : "%s "; cnt += sprintf(buffer + cnt, fmt, aes_impl_opts[i].name); } /* list all supported implementations */ for (i = 0; i < aes_supp_impl_cnt; i++) { fmt = (i == impl) ? "[%s] " : "%s "; cnt += sprintf(buffer + cnt, fmt, aes_supp_impl[i]->name); } return (cnt); } module_param_call(icp_aes_impl, icp_aes_impl_set, icp_aes_impl_get, NULL, 0644); MODULE_PARM_DESC(icp_aes_impl, "Select aes implementation."); #endif diff --git a/module/icp/algs/aes/aes_impl_aesni.c b/module/icp/algs/aes/aes_impl_aesni.c index 222c176aabab..4b5eefd71b17 100644 --- a/module/icp/algs/aes/aes_impl_aesni.c +++ b/module/icp/algs/aes/aes_impl_aesni.c @@ -1,123 +1,124 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. */ #if defined(__x86_64) && defined(HAVE_AES) -#include +#include +#include /* These functions are used to execute AES-NI instructions: */ extern int rijndael_key_setup_enc_intel(uint32_t rk[], const uint32_t cipherKey[], uint64_t keyBits); extern int rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[], uint64_t keyBits); extern void aes_encrypt_intel(const uint32_t rk[], int Nr, const uint32_t pt[4], uint32_t ct[4]); extern void aes_decrypt_intel(const uint32_t rk[], int Nr, const uint32_t ct[4], uint32_t pt[4]); #include /* * Expand the 32-bit AES cipher key array into the encryption and decryption * key schedules. * * Parameters: * key AES key schedule to be initialized * keyarr32 User key * keyBits AES key size (128, 192, or 256 bits) */ static void aes_aesni_generate(aes_key_t *key, const uint32_t *keyarr32, int keybits) { kfpu_begin(); key->nr = rijndael_key_setup_enc_intel(&(key->encr_ks.ks32[0]), keyarr32, keybits); key->nr = rijndael_key_setup_dec_intel(&(key->decr_ks.ks32[0]), keyarr32, keybits); kfpu_end(); } /* * Encrypt one block of data. The block is assumed to be an array * of four uint32_t values, so copy for alignment (and byte-order * reversal for little endian systems might be necessary on the * input and output byte streams. * The size of the key schedule depends on the number of rounds * (which can be computed from the size of the key), i.e. 4*(Nr + 1). * * Parameters: * rk Key schedule, of aes_ks_t (60 32-bit integers) * Nr Number of rounds * pt Input block (plain text) * ct Output block (crypto text). Can overlap with pt */ static void aes_aesni_encrypt(const uint32_t rk[], int Nr, const uint32_t pt[4], uint32_t ct[4]) { kfpu_begin(); aes_encrypt_intel(rk, Nr, pt, ct); kfpu_end(); } /* * Decrypt one block of data. The block is assumed to be an array * of four uint32_t values, so copy for alignment (and byte-order * reversal for little endian systems might be necessary on the * input and output byte streams. * The size of the key schedule depends on the number of rounds * (which can be computed from the size of the key), i.e. 4*(Nr + 1). * * Parameters: * rk Key schedule, of aes_ks_t (60 32-bit integers) * Nr Number of rounds * ct Input block (crypto text) * pt Output block (plain text). Can overlap with pt */ static void aes_aesni_decrypt(const uint32_t rk[], int Nr, const uint32_t ct[4], uint32_t pt[4]) { kfpu_begin(); aes_decrypt_intel(rk, Nr, ct, pt); kfpu_end(); } static boolean_t aes_aesni_will_work(void) { return (kfpu_allowed() && zfs_aes_available()); } const aes_impl_ops_t aes_aesni_impl = { .generate = &aes_aesni_generate, .encrypt = &aes_aesni_encrypt, .decrypt = &aes_aesni_decrypt, .is_supported = &aes_aesni_will_work, .needs_byteswap = B_FALSE, .name = "aesni" }; #endif /* defined(__x86_64) && defined(HAVE_AES) */ diff --git a/module/icp/algs/aes/aes_impl_x86-64.c b/module/icp/algs/aes/aes_impl_x86-64.c index b4515fa22cba..0ee7ee99c3cd 100644 --- a/module/icp/algs/aes/aes_impl_x86-64.c +++ b/module/icp/algs/aes/aes_impl_x86-64.c @@ -1,75 +1,75 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. */ #if defined(__x86_64) -#include +#include /* These functions are used to execute amd64 instructions for AMD or Intel: */ extern int rijndael_key_setup_enc_amd64(uint32_t rk[], const uint32_t cipherKey[], int keyBits); extern int rijndael_key_setup_dec_amd64(uint32_t rk[], const uint32_t cipherKey[], int keyBits); extern void aes_encrypt_amd64(const uint32_t rk[], int Nr, const uint32_t pt[4], uint32_t ct[4]); extern void aes_decrypt_amd64(const uint32_t rk[], int Nr, const uint32_t ct[4], uint32_t pt[4]); #include /* * Expand the 32-bit AES cipher key array into the encryption and decryption * key schedules. * * Parameters: * key AES key schedule to be initialized * keyarr32 User key * keyBits AES key size (128, 192, or 256 bits) */ static void aes_x86_64_generate(aes_key_t *key, const uint32_t *keyarr32, int keybits) { key->nr = rijndael_key_setup_enc_amd64(&(key->encr_ks.ks32[0]), keyarr32, keybits); key->nr = rijndael_key_setup_dec_amd64(&(key->decr_ks.ks32[0]), keyarr32, keybits); } static boolean_t aes_x86_64_will_work(void) { return (B_TRUE); } const aes_impl_ops_t aes_x86_64_impl = { .generate = &aes_x86_64_generate, .encrypt = &aes_encrypt_amd64, .decrypt = &aes_decrypt_amd64, .is_supported = &aes_x86_64_will_work, .needs_byteswap = B_FALSE, .name = "x86_64" }; #endif /* defined(__x86_64) */ diff --git a/module/icp/algs/modes/gcm.c b/module/icp/algs/modes/gcm.c index 423b70e2c86b..1fb8e256a52b 100644 --- a/module/icp/algs/modes/gcm.c +++ b/module/icp/algs/modes/gcm.c @@ -1,848 +1,848 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. */ #include #include #include #include #include #include +#include #include -#include #define GHASH(c, d, t, o) \ xor_block((uint8_t *)(d), (uint8_t *)(c)->gcm_ghash); \ (o)->mul((uint64_t *)(void *)(c)->gcm_ghash, (c)->gcm_H, \ (uint64_t *)(void *)(t)); /* * Encrypt multiple blocks of data in GCM mode. Decrypt for GCM mode * is done in another function. */ int gcm_mode_encrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length, crypto_data_t *out, size_t block_size, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*copy_block)(uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)) { const gcm_impl_ops_t *gops; size_t remainder = length; size_t need = 0; uint8_t *datap = (uint8_t *)data; uint8_t *blockp; uint8_t *lastp; void *iov_or_mp; offset_t offset; uint8_t *out_data_1; uint8_t *out_data_2; size_t out_data_1_len; uint64_t counter; uint64_t counter_mask = ntohll(0x00000000ffffffffULL); if (length + ctx->gcm_remainder_len < block_size) { /* accumulate bytes here and return */ bcopy(datap, (uint8_t *)ctx->gcm_remainder + ctx->gcm_remainder_len, length); ctx->gcm_remainder_len += length; ctx->gcm_copy_to = datap; return (CRYPTO_SUCCESS); } lastp = (uint8_t *)ctx->gcm_cb; if (out != NULL) crypto_init_ptrs(out, &iov_or_mp, &offset); gops = gcm_impl_get_ops(); do { /* Unprocessed data from last call. */ if (ctx->gcm_remainder_len > 0) { need = block_size - ctx->gcm_remainder_len; if (need > remainder) return (CRYPTO_DATA_LEN_RANGE); bcopy(datap, &((uint8_t *)ctx->gcm_remainder) [ctx->gcm_remainder_len], need); blockp = (uint8_t *)ctx->gcm_remainder; } else { blockp = datap; } /* * Increment counter. Counter bits are confined * to the bottom 32 bits of the counter block. */ counter = ntohll(ctx->gcm_cb[1] & counter_mask); counter = htonll(counter + 1); counter &= counter_mask; ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, (uint8_t *)ctx->gcm_tmp); xor_block(blockp, (uint8_t *)ctx->gcm_tmp); lastp = (uint8_t *)ctx->gcm_tmp; ctx->gcm_processed_data_len += block_size; if (out == NULL) { if (ctx->gcm_remainder_len > 0) { bcopy(blockp, ctx->gcm_copy_to, ctx->gcm_remainder_len); bcopy(blockp + ctx->gcm_remainder_len, datap, need); } } else { crypto_get_ptrs(out, &iov_or_mp, &offset, &out_data_1, &out_data_1_len, &out_data_2, block_size); /* copy block to where it belongs */ if (out_data_1_len == block_size) { copy_block(lastp, out_data_1); } else { bcopy(lastp, out_data_1, out_data_1_len); if (out_data_2 != NULL) { bcopy(lastp + out_data_1_len, out_data_2, block_size - out_data_1_len); } } /* update offset */ out->cd_offset += block_size; } /* add ciphertext to the hash */ GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash, gops); /* Update pointer to next block of data to be processed. */ if (ctx->gcm_remainder_len != 0) { datap += need; ctx->gcm_remainder_len = 0; } else { datap += block_size; } remainder = (size_t)&data[length] - (size_t)datap; /* Incomplete last block. */ if (remainder > 0 && remainder < block_size) { bcopy(datap, ctx->gcm_remainder, remainder); ctx->gcm_remainder_len = remainder; ctx->gcm_copy_to = datap; goto out; } ctx->gcm_copy_to = NULL; } while (remainder > 0); out: return (CRYPTO_SUCCESS); } /* ARGSUSED */ int gcm_encrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*copy_block)(uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)) { const gcm_impl_ops_t *gops; uint64_t counter_mask = ntohll(0x00000000ffffffffULL); uint8_t *ghash, *macp = NULL; int i, rv; if (out->cd_length < (ctx->gcm_remainder_len + ctx->gcm_tag_len)) { return (CRYPTO_DATA_LEN_RANGE); } gops = gcm_impl_get_ops(); ghash = (uint8_t *)ctx->gcm_ghash; if (ctx->gcm_remainder_len > 0) { uint64_t counter; uint8_t *tmpp = (uint8_t *)ctx->gcm_tmp; /* * Here is where we deal with data that is not a * multiple of the block size. */ /* * Increment counter. */ counter = ntohll(ctx->gcm_cb[1] & counter_mask); counter = htonll(counter + 1); counter &= counter_mask; ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, (uint8_t *)ctx->gcm_tmp); macp = (uint8_t *)ctx->gcm_remainder; bzero(macp + ctx->gcm_remainder_len, block_size - ctx->gcm_remainder_len); /* XOR with counter block */ for (i = 0; i < ctx->gcm_remainder_len; i++) { macp[i] ^= tmpp[i]; } /* add ciphertext to the hash */ GHASH(ctx, macp, ghash, gops); ctx->gcm_processed_data_len += ctx->gcm_remainder_len; } ctx->gcm_len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(ctx->gcm_processed_data_len)); GHASH(ctx, ctx->gcm_len_a_len_c, ghash, gops); encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0, (uint8_t *)ctx->gcm_J0); xor_block((uint8_t *)ctx->gcm_J0, ghash); if (ctx->gcm_remainder_len > 0) { rv = crypto_put_output_data(macp, out, ctx->gcm_remainder_len); if (rv != CRYPTO_SUCCESS) return (rv); } out->cd_offset += ctx->gcm_remainder_len; ctx->gcm_remainder_len = 0; rv = crypto_put_output_data(ghash, out, ctx->gcm_tag_len); if (rv != CRYPTO_SUCCESS) return (rv); out->cd_offset += ctx->gcm_tag_len; return (CRYPTO_SUCCESS); } /* * This will only deal with decrypting the last block of the input that * might not be a multiple of block length. */ static void gcm_decrypt_incomplete_block(gcm_ctx_t *ctx, size_t block_size, size_t index, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)) { uint8_t *datap, *outp, *counterp; uint64_t counter; uint64_t counter_mask = ntohll(0x00000000ffffffffULL); int i; /* * Increment counter. * Counter bits are confined to the bottom 32 bits */ counter = ntohll(ctx->gcm_cb[1] & counter_mask); counter = htonll(counter + 1); counter &= counter_mask; ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; datap = (uint8_t *)ctx->gcm_remainder; outp = &((ctx->gcm_pt_buf)[index]); counterp = (uint8_t *)ctx->gcm_tmp; /* authentication tag */ bzero((uint8_t *)ctx->gcm_tmp, block_size); bcopy(datap, (uint8_t *)ctx->gcm_tmp, ctx->gcm_remainder_len); /* add ciphertext to the hash */ GHASH(ctx, ctx->gcm_tmp, ctx->gcm_ghash, gcm_impl_get_ops()); /* decrypt remaining ciphertext */ encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, counterp); /* XOR with counter block */ for (i = 0; i < ctx->gcm_remainder_len; i++) { outp[i] = datap[i] ^ counterp[i]; } } /* ARGSUSED */ int gcm_mode_decrypt_contiguous_blocks(gcm_ctx_t *ctx, char *data, size_t length, crypto_data_t *out, size_t block_size, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*copy_block)(uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)) { size_t new_len; uint8_t *new; /* * Copy contiguous ciphertext input blocks to plaintext buffer. * Ciphertext will be decrypted in the final. */ if (length > 0) { new_len = ctx->gcm_pt_buf_len + length; new = vmem_alloc(new_len, ctx->gcm_kmflag); bcopy(ctx->gcm_pt_buf, new, ctx->gcm_pt_buf_len); vmem_free(ctx->gcm_pt_buf, ctx->gcm_pt_buf_len); if (new == NULL) return (CRYPTO_HOST_MEMORY); ctx->gcm_pt_buf = new; ctx->gcm_pt_buf_len = new_len; bcopy(data, &ctx->gcm_pt_buf[ctx->gcm_processed_data_len], length); ctx->gcm_processed_data_len += length; } ctx->gcm_remainder_len = 0; return (CRYPTO_SUCCESS); } int gcm_decrypt_final(gcm_ctx_t *ctx, crypto_data_t *out, size_t block_size, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)) { const gcm_impl_ops_t *gops; size_t pt_len; size_t remainder; uint8_t *ghash; uint8_t *blockp; uint8_t *cbp; uint64_t counter; uint64_t counter_mask = ntohll(0x00000000ffffffffULL); int processed = 0, rv; ASSERT(ctx->gcm_processed_data_len == ctx->gcm_pt_buf_len); gops = gcm_impl_get_ops(); pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len; ghash = (uint8_t *)ctx->gcm_ghash; blockp = ctx->gcm_pt_buf; remainder = pt_len; while (remainder > 0) { /* Incomplete last block */ if (remainder < block_size) { bcopy(blockp, ctx->gcm_remainder, remainder); ctx->gcm_remainder_len = remainder; /* * not expecting anymore ciphertext, just * compute plaintext for the remaining input */ gcm_decrypt_incomplete_block(ctx, block_size, processed, encrypt_block, xor_block); ctx->gcm_remainder_len = 0; goto out; } /* add ciphertext to the hash */ GHASH(ctx, blockp, ghash, gops); /* * Increment counter. * Counter bits are confined to the bottom 32 bits */ counter = ntohll(ctx->gcm_cb[1] & counter_mask); counter = htonll(counter + 1); counter &= counter_mask; ctx->gcm_cb[1] = (ctx->gcm_cb[1] & ~counter_mask) | counter; cbp = (uint8_t *)ctx->gcm_tmp; encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_cb, cbp); /* XOR with ciphertext */ xor_block(cbp, blockp); processed += block_size; blockp += block_size; remainder -= block_size; } out: ctx->gcm_len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(pt_len)); GHASH(ctx, ctx->gcm_len_a_len_c, ghash, gops); encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_J0, (uint8_t *)ctx->gcm_J0); xor_block((uint8_t *)ctx->gcm_J0, ghash); /* compare the input authentication tag with what we calculated */ if (bcmp(&ctx->gcm_pt_buf[pt_len], ghash, ctx->gcm_tag_len)) { /* They don't match */ return (CRYPTO_INVALID_MAC); } else { rv = crypto_put_output_data(ctx->gcm_pt_buf, out, pt_len); if (rv != CRYPTO_SUCCESS) return (rv); out->cd_offset += pt_len; } return (CRYPTO_SUCCESS); } static int gcm_validate_args(CK_AES_GCM_PARAMS *gcm_param) { size_t tag_len; /* * Check the length of the authentication tag (in bits). */ tag_len = gcm_param->ulTagBits; switch (tag_len) { case 32: case 64: case 96: case 104: case 112: case 120: case 128: break; default: return (CRYPTO_MECHANISM_PARAM_INVALID); } if (gcm_param->ulIvLen == 0) return (CRYPTO_MECHANISM_PARAM_INVALID); return (CRYPTO_SUCCESS); } static void gcm_format_initial_blocks(uchar_t *iv, ulong_t iv_len, gcm_ctx_t *ctx, size_t block_size, void (*copy_block)(uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)) { const gcm_impl_ops_t *gops; uint8_t *cb; ulong_t remainder = iv_len; ulong_t processed = 0; uint8_t *datap, *ghash; uint64_t len_a_len_c[2]; gops = gcm_impl_get_ops(); ghash = (uint8_t *)ctx->gcm_ghash; cb = (uint8_t *)ctx->gcm_cb; if (iv_len == 12) { bcopy(iv, cb, 12); cb[12] = 0; cb[13] = 0; cb[14] = 0; cb[15] = 1; /* J0 will be used again in the final */ copy_block(cb, (uint8_t *)ctx->gcm_J0); } else { /* GHASH the IV */ do { if (remainder < block_size) { bzero(cb, block_size); bcopy(&(iv[processed]), cb, remainder); datap = (uint8_t *)cb; remainder = 0; } else { datap = (uint8_t *)(&(iv[processed])); processed += block_size; remainder -= block_size; } GHASH(ctx, datap, ghash, gops); } while (remainder > 0); len_a_len_c[0] = 0; len_a_len_c[1] = htonll(CRYPTO_BYTES2BITS(iv_len)); GHASH(ctx, len_a_len_c, ctx->gcm_J0, gops); /* J0 will be used again in the final */ copy_block((uint8_t *)ctx->gcm_J0, (uint8_t *)cb); } } /* * The following function is called at encrypt or decrypt init time * for AES GCM mode. */ int gcm_init(gcm_ctx_t *ctx, unsigned char *iv, size_t iv_len, unsigned char *auth_data, size_t auth_data_len, size_t block_size, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*copy_block)(uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)) { const gcm_impl_ops_t *gops; uint8_t *ghash, *datap, *authp; size_t remainder, processed; /* encrypt zero block to get subkey H */ bzero(ctx->gcm_H, sizeof (ctx->gcm_H)); encrypt_block(ctx->gcm_keysched, (uint8_t *)ctx->gcm_H, (uint8_t *)ctx->gcm_H); gcm_format_initial_blocks(iv, iv_len, ctx, block_size, copy_block, xor_block); gops = gcm_impl_get_ops(); authp = (uint8_t *)ctx->gcm_tmp; ghash = (uint8_t *)ctx->gcm_ghash; bzero(authp, block_size); bzero(ghash, block_size); processed = 0; remainder = auth_data_len; do { if (remainder < block_size) { /* * There's not a block full of data, pad rest of * buffer with zero */ bzero(authp, block_size); bcopy(&(auth_data[processed]), authp, remainder); datap = (uint8_t *)authp; remainder = 0; } else { datap = (uint8_t *)(&(auth_data[processed])); processed += block_size; remainder -= block_size; } /* add auth data to the hash */ GHASH(ctx, datap, ghash, gops); } while (remainder > 0); return (CRYPTO_SUCCESS); } int gcm_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*copy_block)(uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)) { int rv; CK_AES_GCM_PARAMS *gcm_param; if (param != NULL) { gcm_param = (CK_AES_GCM_PARAMS *)(void *)param; if ((rv = gcm_validate_args(gcm_param)) != 0) { return (rv); } gcm_ctx->gcm_tag_len = gcm_param->ulTagBits; gcm_ctx->gcm_tag_len >>= 3; gcm_ctx->gcm_processed_data_len = 0; /* these values are in bits */ gcm_ctx->gcm_len_a_len_c[0] = htonll(CRYPTO_BYTES2BITS(gcm_param->ulAADLen)); rv = CRYPTO_SUCCESS; gcm_ctx->gcm_flags |= GCM_MODE; } else { rv = CRYPTO_MECHANISM_PARAM_INVALID; goto out; } if (gcm_init(gcm_ctx, gcm_param->pIv, gcm_param->ulIvLen, gcm_param->pAAD, gcm_param->ulAADLen, block_size, encrypt_block, copy_block, xor_block) != 0) { rv = CRYPTO_MECHANISM_PARAM_INVALID; } out: return (rv); } int gmac_init_ctx(gcm_ctx_t *gcm_ctx, char *param, size_t block_size, int (*encrypt_block)(const void *, const uint8_t *, uint8_t *), void (*copy_block)(uint8_t *, uint8_t *), void (*xor_block)(uint8_t *, uint8_t *)) { int rv; CK_AES_GMAC_PARAMS *gmac_param; if (param != NULL) { gmac_param = (CK_AES_GMAC_PARAMS *)(void *)param; gcm_ctx->gcm_tag_len = CRYPTO_BITS2BYTES(AES_GMAC_TAG_BITS); gcm_ctx->gcm_processed_data_len = 0; /* these values are in bits */ gcm_ctx->gcm_len_a_len_c[0] = htonll(CRYPTO_BYTES2BITS(gmac_param->ulAADLen)); rv = CRYPTO_SUCCESS; gcm_ctx->gcm_flags |= GMAC_MODE; } else { rv = CRYPTO_MECHANISM_PARAM_INVALID; goto out; } if (gcm_init(gcm_ctx, gmac_param->pIv, AES_GMAC_IV_LEN, gmac_param->pAAD, gmac_param->ulAADLen, block_size, encrypt_block, copy_block, xor_block) != 0) { rv = CRYPTO_MECHANISM_PARAM_INVALID; } out: return (rv); } void * gcm_alloc_ctx(int kmflag) { gcm_ctx_t *gcm_ctx; if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL) return (NULL); gcm_ctx->gcm_flags = GCM_MODE; return (gcm_ctx); } void * gmac_alloc_ctx(int kmflag) { gcm_ctx_t *gcm_ctx; if ((gcm_ctx = kmem_zalloc(sizeof (gcm_ctx_t), kmflag)) == NULL) return (NULL); gcm_ctx->gcm_flags = GMAC_MODE; return (gcm_ctx); } void gcm_set_kmflag(gcm_ctx_t *ctx, int kmflag) { ctx->gcm_kmflag = kmflag; } /* GCM implementation that contains the fastest methods */ static gcm_impl_ops_t gcm_fastest_impl = { .name = "fastest" }; /* All compiled in implementations */ const gcm_impl_ops_t *gcm_all_impl[] = { &gcm_generic_impl, #if defined(__x86_64) && defined(HAVE_PCLMULQDQ) &gcm_pclmulqdq_impl, #endif }; /* Indicate that benchmark has been completed */ static boolean_t gcm_impl_initialized = B_FALSE; /* Select GCM implementation */ #define IMPL_FASTEST (UINT32_MAX) #define IMPL_CYCLE (UINT32_MAX-1) #define GCM_IMPL_READ(i) (*(volatile uint32_t *) &(i)) static uint32_t icp_gcm_impl = IMPL_FASTEST; static uint32_t user_sel_impl = IMPL_FASTEST; /* Hold all supported implementations */ static size_t gcm_supp_impl_cnt = 0; static gcm_impl_ops_t *gcm_supp_impl[ARRAY_SIZE(gcm_all_impl)]; /* * Returns the GCM operations for encrypt/decrypt/key setup. When a * SIMD implementation is not allowed in the current context, then * fallback to the fastest generic implementation. */ const gcm_impl_ops_t * gcm_impl_get_ops() { if (!kfpu_allowed()) return (&gcm_generic_impl); const gcm_impl_ops_t *ops = NULL; const uint32_t impl = GCM_IMPL_READ(icp_gcm_impl); switch (impl) { case IMPL_FASTEST: ASSERT(gcm_impl_initialized); ops = &gcm_fastest_impl; break; case IMPL_CYCLE: /* Cycle through supported implementations */ ASSERT(gcm_impl_initialized); ASSERT3U(gcm_supp_impl_cnt, >, 0); static size_t cycle_impl_idx = 0; size_t idx = (++cycle_impl_idx) % gcm_supp_impl_cnt; ops = gcm_supp_impl[idx]; break; default: ASSERT3U(impl, <, gcm_supp_impl_cnt); ASSERT3U(gcm_supp_impl_cnt, >, 0); if (impl < ARRAY_SIZE(gcm_all_impl)) ops = gcm_supp_impl[impl]; break; } ASSERT3P(ops, !=, NULL); return (ops); } /* * Initialize all supported implementations. */ /* ARGSUSED */ void gcm_impl_init(void *arg) { gcm_impl_ops_t *curr_impl; int i, c; /* Move supported implementations into gcm_supp_impls */ for (i = 0, c = 0; i < ARRAY_SIZE(gcm_all_impl); i++) { curr_impl = (gcm_impl_ops_t *)gcm_all_impl[i]; if (curr_impl->is_supported()) gcm_supp_impl[c++] = (gcm_impl_ops_t *)curr_impl; } gcm_supp_impl_cnt = c; /* * Set the fastest implementation given the assumption that the * hardware accelerated version is the fastest. */ #if defined(__x86_64) && defined(HAVE_PCLMULQDQ) if (gcm_pclmulqdq_impl.is_supported()) { memcpy(&gcm_fastest_impl, &gcm_pclmulqdq_impl, sizeof (gcm_fastest_impl)); } else #endif { memcpy(&gcm_fastest_impl, &gcm_generic_impl, sizeof (gcm_fastest_impl)); } strcpy(gcm_fastest_impl.name, "fastest"); /* Finish initialization */ atomic_swap_32(&icp_gcm_impl, user_sel_impl); gcm_impl_initialized = B_TRUE; } static const struct { char *name; uint32_t sel; } gcm_impl_opts[] = { { "cycle", IMPL_CYCLE }, { "fastest", IMPL_FASTEST }, }; /* * Function sets desired gcm implementation. * * If we are called before init(), user preference will be saved in * user_sel_impl, and applied in later init() call. This occurs when module * parameter is specified on module load. Otherwise, directly update * icp_gcm_impl. * * @val Name of gcm implementation to use * @param Unused. */ int gcm_impl_set(const char *val) { int err = -EINVAL; char req_name[GCM_IMPL_NAME_MAX]; uint32_t impl = GCM_IMPL_READ(user_sel_impl); size_t i; /* sanitize input */ i = strnlen(val, GCM_IMPL_NAME_MAX); if (i == 0 || i >= GCM_IMPL_NAME_MAX) return (err); strlcpy(req_name, val, GCM_IMPL_NAME_MAX); while (i > 0 && isspace(req_name[i-1])) i--; req_name[i] = '\0'; /* Check mandatory options */ for (i = 0; i < ARRAY_SIZE(gcm_impl_opts); i++) { if (strcmp(req_name, gcm_impl_opts[i].name) == 0) { impl = gcm_impl_opts[i].sel; err = 0; break; } } /* check all supported impl if init() was already called */ if (err != 0 && gcm_impl_initialized) { /* check all supported implementations */ for (i = 0; i < gcm_supp_impl_cnt; i++) { if (strcmp(req_name, gcm_supp_impl[i]->name) == 0) { impl = i; err = 0; break; } } } if (err == 0) { if (gcm_impl_initialized) atomic_swap_32(&icp_gcm_impl, impl); else atomic_swap_32(&user_sel_impl, impl); } return (err); } #if defined(_KERNEL) #include static int icp_gcm_impl_set(const char *val, zfs_kernel_param_t *kp) { return (gcm_impl_set(val)); } static int icp_gcm_impl_get(char *buffer, zfs_kernel_param_t *kp) { int i, cnt = 0; char *fmt; const uint32_t impl = GCM_IMPL_READ(icp_gcm_impl); ASSERT(gcm_impl_initialized); /* list mandatory options */ for (i = 0; i < ARRAY_SIZE(gcm_impl_opts); i++) { fmt = (impl == gcm_impl_opts[i].sel) ? "[%s] " : "%s "; cnt += sprintf(buffer + cnt, fmt, gcm_impl_opts[i].name); } /* list all supported implementations */ for (i = 0; i < gcm_supp_impl_cnt; i++) { fmt = (i == impl) ? "[%s] " : "%s "; cnt += sprintf(buffer + cnt, fmt, gcm_supp_impl[i]->name); } return (cnt); } module_param_call(icp_gcm_impl, icp_gcm_impl_set, icp_gcm_impl_get, NULL, 0644); MODULE_PARM_DESC(icp_gcm_impl, "Select gcm implementation."); #endif diff --git a/module/icp/algs/modes/gcm_pclmulqdq.c b/module/icp/algs/modes/gcm_pclmulqdq.c index 8a43ba33a6e5..05920115ce86 100644 --- a/module/icp/algs/modes/gcm_pclmulqdq.c +++ b/module/icp/algs/modes/gcm_pclmulqdq.c @@ -1,64 +1,64 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. */ #if defined(__x86_64) && defined(HAVE_PCLMULQDQ) -#include +#include +#include /* These functions are used to execute pclmulqdq based assembly methods */ extern void gcm_mul_pclmulqdq(uint64_t *, uint64_t *, uint64_t *); - #include /* * Perform a carry-less multiplication (that is, use XOR instead of the * multiply operator) on *x_in and *y and place the result in *res. * * Byte swap the input (*x_in and *y) and the output (*res). * * Note: x_in, y, and res all point to 16-byte numbers (an array of two * 64-bit integers). */ static void gcm_pclmulqdq_mul(uint64_t *x_in, uint64_t *y, uint64_t *res) { kfpu_begin(); gcm_mul_pclmulqdq(x_in, y, res); kfpu_end(); } static boolean_t gcm_pclmulqdq_will_work(void) { return (kfpu_allowed() && zfs_pclmulqdq_available()); } const gcm_impl_ops_t gcm_pclmulqdq_impl = { .mul = &gcm_pclmulqdq_mul, .is_supported = &gcm_pclmulqdq_will_work, .name = "pclmulqdq" }; #endif /* defined(__x86_64) && defined(HAVE_PCLMULQDQ) */ diff --git a/module/icp/io/aes.c b/module/icp/io/aes.c index 51538bc60780..4b2dbd6e170e 100644 --- a/module/icp/io/aes.c +++ b/module/icp/io/aes.c @@ -1,1470 +1,1470 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. */ /* * AES provider for the Kernel Cryptographic Framework (KCF) */ #include #include #include #include #include #include #include #define _AES_IMPL #include #include #define CRYPTO_PROVIDER_NAME "aes" extern struct mod_ops mod_cryptoops; /* * Module linkage information for the kernel. */ static struct modlcrypto modlcrypto = { &mod_cryptoops, "AES Kernel SW Provider" }; static struct modlinkage modlinkage = { MODREV_1, { (void *)&modlcrypto, NULL } }; /* * Mechanism info structure passed to KCF during registration. */ static crypto_mech_info_t aes_mech_info_tab[] = { /* AES_ECB */ {SUN_CKM_AES_ECB, AES_ECB_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC, AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, /* AES_CBC */ {SUN_CKM_AES_CBC, AES_CBC_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC, AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, /* AES_CTR */ {SUN_CKM_AES_CTR, AES_CTR_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC, AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, /* AES_CCM */ {SUN_CKM_AES_CCM, AES_CCM_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC, AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, /* AES_GCM */ {SUN_CKM_AES_GCM, AES_GCM_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC, AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}, /* AES_GMAC */ {SUN_CKM_AES_GMAC, AES_GMAC_MECH_INFO_TYPE, CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC | CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC | CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC | CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC | CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC, AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES} }; /* operations are in-place if the output buffer is NULL */ #define AES_ARG_INPLACE(input, output) \ if ((output) == NULL) \ (output) = (input); static void aes_provider_status(crypto_provider_handle_t, uint_t *); static crypto_control_ops_t aes_control_ops = { aes_provider_status }; static int aes_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int aes_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int aes_common_init(crypto_ctx_t *, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t, boolean_t); static int aes_common_init_ctx(aes_ctx_t *, crypto_spi_ctx_template_t *, crypto_mechanism_t *, crypto_key_t *, int, boolean_t); static int aes_encrypt_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t); static int aes_decrypt_final(crypto_ctx_t *, crypto_data_t *, crypto_req_handle_t); static int aes_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int aes_encrypt_update(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int aes_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int aes_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int aes_decrypt_update(crypto_ctx_t *, crypto_data_t *, crypto_data_t *, crypto_req_handle_t); static int aes_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static crypto_cipher_ops_t aes_cipher_ops = { .encrypt_init = aes_encrypt_init, .encrypt = aes_encrypt, .encrypt_update = aes_encrypt_update, .encrypt_final = aes_encrypt_final, .encrypt_atomic = aes_encrypt_atomic, .decrypt_init = aes_decrypt_init, .decrypt = aes_decrypt, .decrypt_update = aes_decrypt_update, .decrypt_final = aes_decrypt_final, .decrypt_atomic = aes_decrypt_atomic }; static int aes_mac_atomic(crypto_provider_handle_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static int aes_mac_verify_atomic(crypto_provider_handle_t, crypto_session_id_t, crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t); static crypto_mac_ops_t aes_mac_ops = { .mac_init = NULL, .mac = NULL, .mac_update = NULL, .mac_final = NULL, .mac_atomic = aes_mac_atomic, .mac_verify_atomic = aes_mac_verify_atomic }; static int aes_create_ctx_template(crypto_provider_handle_t, crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t *, size_t *, crypto_req_handle_t); static int aes_free_context(crypto_ctx_t *); static crypto_ctx_ops_t aes_ctx_ops = { .create_ctx_template = aes_create_ctx_template, .free_context = aes_free_context }; static crypto_ops_t aes_crypto_ops = {{{{{ &aes_control_ops, NULL, &aes_cipher_ops, &aes_mac_ops, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, &aes_ctx_ops }}}}}; static crypto_provider_info_t aes_prov_info = {{{{ CRYPTO_SPI_VERSION_1, "AES Software Provider", CRYPTO_SW_PROVIDER, NULL, &aes_crypto_ops, sizeof (aes_mech_info_tab)/sizeof (crypto_mech_info_t), aes_mech_info_tab }}}}; static crypto_kcf_provider_handle_t aes_prov_handle = 0; static crypto_data_t null_crypto_data = { CRYPTO_DATA_RAW }; int aes_mod_init(void) { int ret; #if defined(_KERNEL) /* * Determine the fastest available implementation. The benchmarks * are run in dedicated kernel threads to allow Linux 5.0+ kernels * to use SIMD operations. If for some reason this isn't possible, * fallback to the generic implementations. See the comment in - * include/linux/simd_x86.h for additional details. Additionally, - * this has the benefit of allowing them to be run in parallel. + * linux/simd_x86.h for additional details. Additionally, this has + * the benefit of allowing them to be run in parallel. */ taskqid_t aes_id = taskq_dispatch(system_taskq, aes_impl_init, NULL, TQ_SLEEP); taskqid_t gcm_id = taskq_dispatch(system_taskq, gcm_impl_init, NULL, TQ_SLEEP); if (aes_id != TASKQID_INVALID) { taskq_wait_id(system_taskq, aes_id); } else { aes_impl_init(NULL); } if (gcm_id != TASKQID_INVALID) { taskq_wait_id(system_taskq, gcm_id); } else { gcm_impl_init(NULL); } #else aes_impl_init(NULL); gcm_impl_init(NULL); #endif if ((ret = mod_install(&modlinkage)) != 0) return (ret); /* Register with KCF. If the registration fails, remove the module. */ if (crypto_register_provider(&aes_prov_info, &aes_prov_handle)) { (void) mod_remove(&modlinkage); return (EACCES); } return (0); } int aes_mod_fini(void) { /* Unregister from KCF if module is registered */ if (aes_prov_handle != 0) { if (crypto_unregister_provider(aes_prov_handle)) return (EBUSY); aes_prov_handle = 0; } return (mod_remove(&modlinkage)); } static int aes_check_mech_param(crypto_mechanism_t *mechanism, aes_ctx_t **ctx, int kmflag) { void *p = NULL; boolean_t param_required = B_TRUE; size_t param_len; void *(*alloc_fun)(int); int rv = CRYPTO_SUCCESS; switch (mechanism->cm_type) { case AES_ECB_MECH_INFO_TYPE: param_required = B_FALSE; alloc_fun = ecb_alloc_ctx; break; case AES_CBC_MECH_INFO_TYPE: param_len = AES_BLOCK_LEN; alloc_fun = cbc_alloc_ctx; break; case AES_CTR_MECH_INFO_TYPE: param_len = sizeof (CK_AES_CTR_PARAMS); alloc_fun = ctr_alloc_ctx; break; case AES_CCM_MECH_INFO_TYPE: param_len = sizeof (CK_AES_CCM_PARAMS); alloc_fun = ccm_alloc_ctx; break; case AES_GCM_MECH_INFO_TYPE: param_len = sizeof (CK_AES_GCM_PARAMS); alloc_fun = gcm_alloc_ctx; break; case AES_GMAC_MECH_INFO_TYPE: param_len = sizeof (CK_AES_GMAC_PARAMS); alloc_fun = gmac_alloc_ctx; break; default: rv = CRYPTO_MECHANISM_INVALID; return (rv); } if (param_required && mechanism->cm_param != NULL && mechanism->cm_param_len != param_len) { rv = CRYPTO_MECHANISM_PARAM_INVALID; } if (ctx != NULL) { p = (alloc_fun)(kmflag); *ctx = p; } return (rv); } /* * Initialize key schedules for AES */ static int init_keysched(crypto_key_t *key, void *newbie) { /* * Only keys by value are supported by this module. */ switch (key->ck_format) { case CRYPTO_KEY_RAW: if (key->ck_length < AES_MINBITS || key->ck_length > AES_MAXBITS) { return (CRYPTO_KEY_SIZE_RANGE); } /* key length must be either 128, 192, or 256 */ if ((key->ck_length & 63) != 0) return (CRYPTO_KEY_SIZE_RANGE); break; default: return (CRYPTO_KEY_TYPE_INCONSISTENT); } aes_init_keysched(key->ck_data, key->ck_length, newbie); return (CRYPTO_SUCCESS); } /* * KCF software provider control entry points. */ /* ARGSUSED */ static void aes_provider_status(crypto_provider_handle_t provider, uint_t *status) { *status = CRYPTO_PROVIDER_READY; } static int aes_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_spi_ctx_template_t template, crypto_req_handle_t req) { return (aes_common_init(ctx, mechanism, key, template, req, B_TRUE)); } static int aes_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_spi_ctx_template_t template, crypto_req_handle_t req) { return (aes_common_init(ctx, mechanism, key, template, req, B_FALSE)); } /* * KCF software provider encrypt entry points. */ static int aes_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_spi_ctx_template_t template, crypto_req_handle_t req, boolean_t is_encrypt_init) { aes_ctx_t *aes_ctx; int rv; int kmflag; /* * Only keys by value are supported by this module. */ if (key->ck_format != CRYPTO_KEY_RAW) { return (CRYPTO_KEY_TYPE_INCONSISTENT); } kmflag = crypto_kmflag(req); if ((rv = aes_check_mech_param(mechanism, &aes_ctx, kmflag)) != CRYPTO_SUCCESS) return (rv); rv = aes_common_init_ctx(aes_ctx, template, mechanism, key, kmflag, is_encrypt_init); if (rv != CRYPTO_SUCCESS) { crypto_free_mode_ctx(aes_ctx); return (rv); } ctx->cc_provider_private = aes_ctx; return (CRYPTO_SUCCESS); } static void aes_copy_block64(uint8_t *in, uint64_t *out) { if (IS_P2ALIGNED(in, sizeof (uint64_t))) { /* LINTED: pointer alignment */ out[0] = *(uint64_t *)&in[0]; /* LINTED: pointer alignment */ out[1] = *(uint64_t *)&in[8]; } else { uint8_t *iv8 = (uint8_t *)&out[0]; AES_COPY_BLOCK(in, iv8); } } static int aes_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext, crypto_data_t *ciphertext, crypto_req_handle_t req) { int ret = CRYPTO_FAILED; aes_ctx_t *aes_ctx; size_t saved_length, saved_offset, length_needed; ASSERT(ctx->cc_provider_private != NULL); aes_ctx = ctx->cc_provider_private; /* * For block ciphers, plaintext must be a multiple of AES block size. * This test is only valid for ciphers whose blocksize is a power of 2. */ if (((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) && (plaintext->cd_length & (AES_BLOCK_LEN - 1)) != 0) return (CRYPTO_DATA_LEN_RANGE); AES_ARG_INPLACE(plaintext, ciphertext); /* * We need to just return the length needed to store the output. * We should not destroy the context for the following case. */ switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) { case CCM_MODE: length_needed = plaintext->cd_length + aes_ctx->ac_mac_len; break; case GCM_MODE: length_needed = plaintext->cd_length + aes_ctx->ac_tag_len; break; case GMAC_MODE: if (plaintext->cd_length != 0) return (CRYPTO_ARGUMENTS_BAD); length_needed = aes_ctx->ac_tag_len; break; default: length_needed = plaintext->cd_length; } if (ciphertext->cd_length < length_needed) { ciphertext->cd_length = length_needed; return (CRYPTO_BUFFER_TOO_SMALL); } saved_length = ciphertext->cd_length; saved_offset = ciphertext->cd_offset; /* * Do an update on the specified input data. */ ret = aes_encrypt_update(ctx, plaintext, ciphertext, req); if (ret != CRYPTO_SUCCESS) { return (ret); } /* * For CCM mode, aes_ccm_encrypt_final() will take care of any * left-over unprocessed data, and compute the MAC */ if (aes_ctx->ac_flags & CCM_MODE) { /* * ccm_encrypt_final() will compute the MAC and append * it to existing ciphertext. So, need to adjust the left over * length value accordingly */ /* order of following 2 lines MUST not be reversed */ ciphertext->cd_offset = ciphertext->cd_length; ciphertext->cd_length = saved_length - ciphertext->cd_length; ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext, AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); if (ret != CRYPTO_SUCCESS) { return (ret); } if (plaintext != ciphertext) { ciphertext->cd_length = ciphertext->cd_offset - saved_offset; } ciphertext->cd_offset = saved_offset; } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { /* * gcm_encrypt_final() will compute the MAC and append * it to existing ciphertext. So, need to adjust the left over * length value accordingly */ /* order of following 2 lines MUST not be reversed */ ciphertext->cd_offset = ciphertext->cd_length; ciphertext->cd_length = saved_length - ciphertext->cd_length; ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); if (ret != CRYPTO_SUCCESS) { return (ret); } if (plaintext != ciphertext) { ciphertext->cd_length = ciphertext->cd_offset - saved_offset; } ciphertext->cd_offset = saved_offset; } ASSERT(aes_ctx->ac_remainder_len == 0); (void) aes_free_context(ctx); return (ret); } static int aes_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext, crypto_data_t *plaintext, crypto_req_handle_t req) { int ret = CRYPTO_FAILED; aes_ctx_t *aes_ctx; off_t saved_offset; size_t saved_length, length_needed; ASSERT(ctx->cc_provider_private != NULL); aes_ctx = ctx->cc_provider_private; /* * For block ciphers, plaintext must be a multiple of AES block size. * This test is only valid for ciphers whose blocksize is a power of 2. */ if (((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) && (ciphertext->cd_length & (AES_BLOCK_LEN - 1)) != 0) { return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); } AES_ARG_INPLACE(ciphertext, plaintext); /* * Return length needed to store the output. * Do not destroy context when plaintext buffer is too small. * * CCM: plaintext is MAC len smaller than cipher text * GCM: plaintext is TAG len smaller than cipher text * GMAC: plaintext length must be zero */ switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) { case CCM_MODE: length_needed = aes_ctx->ac_processed_data_len; break; case GCM_MODE: length_needed = ciphertext->cd_length - aes_ctx->ac_tag_len; break; case GMAC_MODE: if (plaintext->cd_length != 0) return (CRYPTO_ARGUMENTS_BAD); length_needed = 0; break; default: length_needed = ciphertext->cd_length; } if (plaintext->cd_length < length_needed) { plaintext->cd_length = length_needed; return (CRYPTO_BUFFER_TOO_SMALL); } saved_offset = plaintext->cd_offset; saved_length = plaintext->cd_length; /* * Do an update on the specified input data. */ ret = aes_decrypt_update(ctx, ciphertext, plaintext, req); if (ret != CRYPTO_SUCCESS) { goto cleanup; } if (aes_ctx->ac_flags & CCM_MODE) { ASSERT(aes_ctx->ac_processed_data_len == aes_ctx->ac_data_len); ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len); /* order of following 2 lines MUST not be reversed */ plaintext->cd_offset = plaintext->cd_length; plaintext->cd_length = saved_length - plaintext->cd_length; ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); if (ret == CRYPTO_SUCCESS) { if (plaintext != ciphertext) { plaintext->cd_length = plaintext->cd_offset - saved_offset; } } else { plaintext->cd_length = saved_length; } plaintext->cd_offset = saved_offset; } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { /* order of following 2 lines MUST not be reversed */ plaintext->cd_offset = plaintext->cd_length; plaintext->cd_length = saved_length - plaintext->cd_length; ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext, AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); if (ret == CRYPTO_SUCCESS) { if (plaintext != ciphertext) { plaintext->cd_length = plaintext->cd_offset - saved_offset; } } else { plaintext->cd_length = saved_length; } plaintext->cd_offset = saved_offset; } ASSERT(aes_ctx->ac_remainder_len == 0); cleanup: (void) aes_free_context(ctx); return (ret); } /* ARGSUSED */ static int aes_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext, crypto_data_t *ciphertext, crypto_req_handle_t req) { off_t saved_offset; size_t saved_length, out_len; int ret = CRYPTO_SUCCESS; aes_ctx_t *aes_ctx; ASSERT(ctx->cc_provider_private != NULL); aes_ctx = ctx->cc_provider_private; AES_ARG_INPLACE(plaintext, ciphertext); /* compute number of bytes that will hold the ciphertext */ out_len = aes_ctx->ac_remainder_len; out_len += plaintext->cd_length; out_len &= ~(AES_BLOCK_LEN - 1); /* return length needed to store the output */ if (ciphertext->cd_length < out_len) { ciphertext->cd_length = out_len; return (CRYPTO_BUFFER_TOO_SMALL); } saved_offset = ciphertext->cd_offset; saved_length = ciphertext->cd_length; /* * Do the AES update on the specified input data. */ switch (plaintext->cd_format) { case CRYPTO_DATA_RAW: ret = crypto_update_iov(ctx->cc_provider_private, plaintext, ciphertext, aes_encrypt_contiguous_blocks, aes_copy_block64); break; case CRYPTO_DATA_UIO: ret = crypto_update_uio(ctx->cc_provider_private, plaintext, ciphertext, aes_encrypt_contiguous_blocks, aes_copy_block64); break; default: ret = CRYPTO_ARGUMENTS_BAD; } /* * Since AES counter mode is a stream cipher, we call * ctr_mode_final() to pick up any remaining bytes. * It is an internal function that does not destroy * the context like *normal* final routines. */ if ((aes_ctx->ac_flags & CTR_MODE) && (aes_ctx->ac_remainder_len > 0)) { ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, ciphertext, aes_encrypt_block); } if (ret == CRYPTO_SUCCESS) { if (plaintext != ciphertext) ciphertext->cd_length = ciphertext->cd_offset - saved_offset; } else { ciphertext->cd_length = saved_length; } ciphertext->cd_offset = saved_offset; return (ret); } static int aes_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext, crypto_data_t *plaintext, crypto_req_handle_t req) { off_t saved_offset; size_t saved_length, out_len; int ret = CRYPTO_SUCCESS; aes_ctx_t *aes_ctx; ASSERT(ctx->cc_provider_private != NULL); aes_ctx = ctx->cc_provider_private; AES_ARG_INPLACE(ciphertext, plaintext); /* * Compute number of bytes that will hold the plaintext. * This is not necessary for CCM, GCM, and GMAC since these * mechanisms never return plaintext for update operations. */ if ((aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) { out_len = aes_ctx->ac_remainder_len; out_len += ciphertext->cd_length; out_len &= ~(AES_BLOCK_LEN - 1); /* return length needed to store the output */ if (plaintext->cd_length < out_len) { plaintext->cd_length = out_len; return (CRYPTO_BUFFER_TOO_SMALL); } } saved_offset = plaintext->cd_offset; saved_length = plaintext->cd_length; if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) gcm_set_kmflag((gcm_ctx_t *)aes_ctx, crypto_kmflag(req)); /* * Do the AES update on the specified input data. */ switch (ciphertext->cd_format) { case CRYPTO_DATA_RAW: ret = crypto_update_iov(ctx->cc_provider_private, ciphertext, plaintext, aes_decrypt_contiguous_blocks, aes_copy_block64); break; case CRYPTO_DATA_UIO: ret = crypto_update_uio(ctx->cc_provider_private, ciphertext, plaintext, aes_decrypt_contiguous_blocks, aes_copy_block64); break; default: ret = CRYPTO_ARGUMENTS_BAD; } /* * Since AES counter mode is a stream cipher, we call * ctr_mode_final() to pick up any remaining bytes. * It is an internal function that does not destroy * the context like *normal* final routines. */ if ((aes_ctx->ac_flags & CTR_MODE) && (aes_ctx->ac_remainder_len > 0)) { ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, plaintext, aes_encrypt_block); if (ret == CRYPTO_DATA_LEN_RANGE) ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; } if (ret == CRYPTO_SUCCESS) { if (ciphertext != plaintext) plaintext->cd_length = plaintext->cd_offset - saved_offset; } else { plaintext->cd_length = saved_length; } plaintext->cd_offset = saved_offset; return (ret); } /* ARGSUSED */ static int aes_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req) { aes_ctx_t *aes_ctx; int ret; ASSERT(ctx->cc_provider_private != NULL); aes_ctx = ctx->cc_provider_private; if (data->cd_format != CRYPTO_DATA_RAW && data->cd_format != CRYPTO_DATA_UIO) { return (CRYPTO_ARGUMENTS_BAD); } if (aes_ctx->ac_flags & CTR_MODE) { if (aes_ctx->ac_remainder_len > 0) { ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, data, aes_encrypt_block); if (ret != CRYPTO_SUCCESS) return (ret); } } else if (aes_ctx->ac_flags & CCM_MODE) { ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data, AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); if (ret != CRYPTO_SUCCESS) { return (ret); } } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { size_t saved_offset = data->cd_offset; ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); if (ret != CRYPTO_SUCCESS) { return (ret); } data->cd_length = data->cd_offset - saved_offset; data->cd_offset = saved_offset; } else { /* * There must be no unprocessed plaintext. * This happens if the length of the last data is * not a multiple of the AES block length. */ if (aes_ctx->ac_remainder_len > 0) { return (CRYPTO_DATA_LEN_RANGE); } data->cd_length = 0; } (void) aes_free_context(ctx); return (CRYPTO_SUCCESS); } /* ARGSUSED */ static int aes_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *data, crypto_req_handle_t req) { aes_ctx_t *aes_ctx; int ret; off_t saved_offset; size_t saved_length; ASSERT(ctx->cc_provider_private != NULL); aes_ctx = ctx->cc_provider_private; if (data->cd_format != CRYPTO_DATA_RAW && data->cd_format != CRYPTO_DATA_UIO) { return (CRYPTO_ARGUMENTS_BAD); } /* * There must be no unprocessed ciphertext. * This happens if the length of the last ciphertext is * not a multiple of the AES block length. */ if (aes_ctx->ac_remainder_len > 0) { if ((aes_ctx->ac_flags & CTR_MODE) == 0) return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); else { ret = ctr_mode_final((ctr_ctx_t *)aes_ctx, data, aes_encrypt_block); if (ret == CRYPTO_DATA_LEN_RANGE) ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; if (ret != CRYPTO_SUCCESS) return (ret); } } if (aes_ctx->ac_flags & CCM_MODE) { /* * This is where all the plaintext is returned, make sure * the plaintext buffer is big enough */ size_t pt_len = aes_ctx->ac_data_len; if (data->cd_length < pt_len) { data->cd_length = pt_len; return (CRYPTO_BUFFER_TOO_SMALL); } ASSERT(aes_ctx->ac_processed_data_len == pt_len); ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len); saved_offset = data->cd_offset; saved_length = data->cd_length; ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); if (ret == CRYPTO_SUCCESS) { data->cd_length = data->cd_offset - saved_offset; } else { data->cd_length = saved_length; } data->cd_offset = saved_offset; if (ret != CRYPTO_SUCCESS) { return (ret); } } else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) { /* * This is where all the plaintext is returned, make sure * the plaintext buffer is big enough */ gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx; size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len; if (data->cd_length < pt_len) { data->cd_length = pt_len; return (CRYPTO_BUFFER_TOO_SMALL); } saved_offset = data->cd_offset; saved_length = data->cd_length; ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data, AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); if (ret == CRYPTO_SUCCESS) { data->cd_length = data->cd_offset - saved_offset; } else { data->cd_length = saved_length; } data->cd_offset = saved_offset; if (ret != CRYPTO_SUCCESS) { return (ret); } } if ((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) { data->cd_length = 0; } (void) aes_free_context(ctx); return (CRYPTO_SUCCESS); } /* ARGSUSED */ static int aes_encrypt_atomic(crypto_provider_handle_t provider, crypto_session_id_t session_id, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext, crypto_spi_ctx_template_t template, crypto_req_handle_t req) { aes_ctx_t aes_ctx; /* on the stack */ off_t saved_offset; size_t saved_length; size_t length_needed; int ret; AES_ARG_INPLACE(plaintext, ciphertext); /* * CTR, CCM, GCM, and GMAC modes do not require that plaintext * be a multiple of AES block size. */ switch (mechanism->cm_type) { case AES_CTR_MECH_INFO_TYPE: case AES_CCM_MECH_INFO_TYPE: case AES_GCM_MECH_INFO_TYPE: case AES_GMAC_MECH_INFO_TYPE: break; default: if ((plaintext->cd_length & (AES_BLOCK_LEN - 1)) != 0) return (CRYPTO_DATA_LEN_RANGE); } if ((ret = aes_check_mech_param(mechanism, NULL, 0)) != CRYPTO_SUCCESS) return (ret); bzero(&aes_ctx, sizeof (aes_ctx_t)); ret = aes_common_init_ctx(&aes_ctx, template, mechanism, key, crypto_kmflag(req), B_TRUE); if (ret != CRYPTO_SUCCESS) return (ret); switch (mechanism->cm_type) { case AES_CCM_MECH_INFO_TYPE: length_needed = plaintext->cd_length + aes_ctx.ac_mac_len; break; case AES_GMAC_MECH_INFO_TYPE: if (plaintext->cd_length != 0) return (CRYPTO_ARGUMENTS_BAD); /* FALLTHRU */ case AES_GCM_MECH_INFO_TYPE: length_needed = plaintext->cd_length + aes_ctx.ac_tag_len; break; default: length_needed = plaintext->cd_length; } /* return size of buffer needed to store output */ if (ciphertext->cd_length < length_needed) { ciphertext->cd_length = length_needed; ret = CRYPTO_BUFFER_TOO_SMALL; goto out; } saved_offset = ciphertext->cd_offset; saved_length = ciphertext->cd_length; /* * Do an update on the specified input data. */ switch (plaintext->cd_format) { case CRYPTO_DATA_RAW: ret = crypto_update_iov(&aes_ctx, plaintext, ciphertext, aes_encrypt_contiguous_blocks, aes_copy_block64); break; case CRYPTO_DATA_UIO: ret = crypto_update_uio(&aes_ctx, plaintext, ciphertext, aes_encrypt_contiguous_blocks, aes_copy_block64); break; default: ret = CRYPTO_ARGUMENTS_BAD; } if (ret == CRYPTO_SUCCESS) { if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) { ret = ccm_encrypt_final((ccm_ctx_t *)&aes_ctx, ciphertext, AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); if (ret != CRYPTO_SUCCESS) goto out; ASSERT(aes_ctx.ac_remainder_len == 0); } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE || mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) { ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx, ciphertext, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); if (ret != CRYPTO_SUCCESS) goto out; ASSERT(aes_ctx.ac_remainder_len == 0); } else if (mechanism->cm_type == AES_CTR_MECH_INFO_TYPE) { if (aes_ctx.ac_remainder_len > 0) { ret = ctr_mode_final((ctr_ctx_t *)&aes_ctx, ciphertext, aes_encrypt_block); if (ret != CRYPTO_SUCCESS) goto out; } } else { ASSERT(aes_ctx.ac_remainder_len == 0); } if (plaintext != ciphertext) { ciphertext->cd_length = ciphertext->cd_offset - saved_offset; } } else { ciphertext->cd_length = saved_length; } ciphertext->cd_offset = saved_offset; out: if (aes_ctx.ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) { bzero(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len); kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len); } return (ret); } /* ARGSUSED */ static int aes_decrypt_atomic(crypto_provider_handle_t provider, crypto_session_id_t session_id, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext, crypto_spi_ctx_template_t template, crypto_req_handle_t req) { aes_ctx_t aes_ctx; /* on the stack */ off_t saved_offset; size_t saved_length; size_t length_needed; int ret; AES_ARG_INPLACE(ciphertext, plaintext); /* * CCM, GCM, CTR, and GMAC modes do not require that ciphertext * be a multiple of AES block size. */ switch (mechanism->cm_type) { case AES_CTR_MECH_INFO_TYPE: case AES_CCM_MECH_INFO_TYPE: case AES_GCM_MECH_INFO_TYPE: case AES_GMAC_MECH_INFO_TYPE: break; default: if ((ciphertext->cd_length & (AES_BLOCK_LEN - 1)) != 0) return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE); } if ((ret = aes_check_mech_param(mechanism, NULL, 0)) != CRYPTO_SUCCESS) return (ret); bzero(&aes_ctx, sizeof (aes_ctx_t)); ret = aes_common_init_ctx(&aes_ctx, template, mechanism, key, crypto_kmflag(req), B_FALSE); if (ret != CRYPTO_SUCCESS) return (ret); switch (mechanism->cm_type) { case AES_CCM_MECH_INFO_TYPE: length_needed = aes_ctx.ac_data_len; break; case AES_GCM_MECH_INFO_TYPE: length_needed = ciphertext->cd_length - aes_ctx.ac_tag_len; break; case AES_GMAC_MECH_INFO_TYPE: if (plaintext->cd_length != 0) return (CRYPTO_ARGUMENTS_BAD); length_needed = 0; break; default: length_needed = ciphertext->cd_length; } /* return size of buffer needed to store output */ if (plaintext->cd_length < length_needed) { plaintext->cd_length = length_needed; ret = CRYPTO_BUFFER_TOO_SMALL; goto out; } saved_offset = plaintext->cd_offset; saved_length = plaintext->cd_length; if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE || mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) gcm_set_kmflag((gcm_ctx_t *)&aes_ctx, crypto_kmflag(req)); /* * Do an update on the specified input data. */ switch (ciphertext->cd_format) { case CRYPTO_DATA_RAW: ret = crypto_update_iov(&aes_ctx, ciphertext, plaintext, aes_decrypt_contiguous_blocks, aes_copy_block64); break; case CRYPTO_DATA_UIO: ret = crypto_update_uio(&aes_ctx, ciphertext, plaintext, aes_decrypt_contiguous_blocks, aes_copy_block64); break; default: ret = CRYPTO_ARGUMENTS_BAD; } if (ret == CRYPTO_SUCCESS) { if (mechanism->cm_type == AES_CCM_MECH_INFO_TYPE) { ASSERT(aes_ctx.ac_processed_data_len == aes_ctx.ac_data_len); ASSERT(aes_ctx.ac_processed_mac_len == aes_ctx.ac_mac_len); ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx, plaintext, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); ASSERT(aes_ctx.ac_remainder_len == 0); if ((ret == CRYPTO_SUCCESS) && (ciphertext != plaintext)) { plaintext->cd_length = plaintext->cd_offset - saved_offset; } else { plaintext->cd_length = saved_length; } } else if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE || mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) { ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx, plaintext, AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); ASSERT(aes_ctx.ac_remainder_len == 0); if ((ret == CRYPTO_SUCCESS) && (ciphertext != plaintext)) { plaintext->cd_length = plaintext->cd_offset - saved_offset; } else { plaintext->cd_length = saved_length; } } else if (mechanism->cm_type != AES_CTR_MECH_INFO_TYPE) { ASSERT(aes_ctx.ac_remainder_len == 0); if (ciphertext != plaintext) plaintext->cd_length = plaintext->cd_offset - saved_offset; } else { if (aes_ctx.ac_remainder_len > 0) { ret = ctr_mode_final((ctr_ctx_t *)&aes_ctx, plaintext, aes_encrypt_block); if (ret == CRYPTO_DATA_LEN_RANGE) ret = CRYPTO_ENCRYPTED_DATA_LEN_RANGE; if (ret != CRYPTO_SUCCESS) goto out; } if (ciphertext != plaintext) plaintext->cd_length = plaintext->cd_offset - saved_offset; } } else { plaintext->cd_length = saved_length; } plaintext->cd_offset = saved_offset; out: if (aes_ctx.ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) { bzero(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len); kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len); } if (aes_ctx.ac_flags & CCM_MODE) { if (aes_ctx.ac_pt_buf != NULL) { vmem_free(aes_ctx.ac_pt_buf, aes_ctx.ac_data_len); } } else if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) { if (((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf != NULL) { vmem_free(((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf, ((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf_len); } } return (ret); } /* * KCF software provider context template entry points. */ /* ARGSUSED */ static int aes_create_ctx_template(crypto_provider_handle_t provider, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_spi_ctx_template_t *tmpl, size_t *tmpl_size, crypto_req_handle_t req) { void *keysched; size_t size; int rv; if (mechanism->cm_type != AES_ECB_MECH_INFO_TYPE && mechanism->cm_type != AES_CBC_MECH_INFO_TYPE && mechanism->cm_type != AES_CTR_MECH_INFO_TYPE && mechanism->cm_type != AES_CCM_MECH_INFO_TYPE && mechanism->cm_type != AES_GCM_MECH_INFO_TYPE && mechanism->cm_type != AES_GMAC_MECH_INFO_TYPE) return (CRYPTO_MECHANISM_INVALID); if ((keysched = aes_alloc_keysched(&size, crypto_kmflag(req))) == NULL) { return (CRYPTO_HOST_MEMORY); } /* * Initialize key schedule. Key length information is stored * in the key. */ if ((rv = init_keysched(key, keysched)) != CRYPTO_SUCCESS) { bzero(keysched, size); kmem_free(keysched, size); return (rv); } *tmpl = keysched; *tmpl_size = size; return (CRYPTO_SUCCESS); } static int aes_free_context(crypto_ctx_t *ctx) { aes_ctx_t *aes_ctx = ctx->cc_provider_private; if (aes_ctx != NULL) { if (aes_ctx->ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) { ASSERT(aes_ctx->ac_keysched_len != 0); bzero(aes_ctx->ac_keysched, aes_ctx->ac_keysched_len); kmem_free(aes_ctx->ac_keysched, aes_ctx->ac_keysched_len); } crypto_free_mode_ctx(aes_ctx); ctx->cc_provider_private = NULL; } return (CRYPTO_SUCCESS); } static int aes_common_init_ctx(aes_ctx_t *aes_ctx, crypto_spi_ctx_template_t *template, crypto_mechanism_t *mechanism, crypto_key_t *key, int kmflag, boolean_t is_encrypt_init) { int rv = CRYPTO_SUCCESS; void *keysched; size_t size = 0; if (template == NULL) { if ((keysched = aes_alloc_keysched(&size, kmflag)) == NULL) return (CRYPTO_HOST_MEMORY); /* * Initialize key schedule. * Key length is stored in the key. */ if ((rv = init_keysched(key, keysched)) != CRYPTO_SUCCESS) { kmem_free(keysched, size); return (rv); } aes_ctx->ac_flags |= PROVIDER_OWNS_KEY_SCHEDULE; aes_ctx->ac_keysched_len = size; } else { keysched = template; } aes_ctx->ac_keysched = keysched; switch (mechanism->cm_type) { case AES_CBC_MECH_INFO_TYPE: rv = cbc_init_ctx((cbc_ctx_t *)aes_ctx, mechanism->cm_param, mechanism->cm_param_len, AES_BLOCK_LEN, aes_copy_block64); break; case AES_CTR_MECH_INFO_TYPE: { CK_AES_CTR_PARAMS *pp; if (mechanism->cm_param == NULL || mechanism->cm_param_len != sizeof (CK_AES_CTR_PARAMS)) { return (CRYPTO_MECHANISM_PARAM_INVALID); } pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param; rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits, pp->cb, aes_copy_block); break; } case AES_CCM_MECH_INFO_TYPE: if (mechanism->cm_param == NULL || mechanism->cm_param_len != sizeof (CK_AES_CCM_PARAMS)) { return (CRYPTO_MECHANISM_PARAM_INVALID); } rv = ccm_init_ctx((ccm_ctx_t *)aes_ctx, mechanism->cm_param, kmflag, is_encrypt_init, AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block); break; case AES_GCM_MECH_INFO_TYPE: if (mechanism->cm_param == NULL || mechanism->cm_param_len != sizeof (CK_AES_GCM_PARAMS)) { return (CRYPTO_MECHANISM_PARAM_INVALID); } rv = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); break; case AES_GMAC_MECH_INFO_TYPE: if (mechanism->cm_param == NULL || mechanism->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) { return (CRYPTO_MECHANISM_PARAM_INVALID); } rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param, AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block, aes_xor_block); break; case AES_ECB_MECH_INFO_TYPE: aes_ctx->ac_flags |= ECB_MODE; } if (rv != CRYPTO_SUCCESS) { if (aes_ctx->ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) { bzero(keysched, size); kmem_free(keysched, size); } } return (rv); } static int process_gmac_mech(crypto_mechanism_t *mech, crypto_data_t *data, CK_AES_GCM_PARAMS *gcm_params) { /* LINTED: pointer alignment */ CK_AES_GMAC_PARAMS *params = (CK_AES_GMAC_PARAMS *)mech->cm_param; if (mech->cm_type != AES_GMAC_MECH_INFO_TYPE) return (CRYPTO_MECHANISM_INVALID); if (mech->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) return (CRYPTO_MECHANISM_PARAM_INVALID); if (params->pIv == NULL) return (CRYPTO_MECHANISM_PARAM_INVALID); gcm_params->pIv = params->pIv; gcm_params->ulIvLen = AES_GMAC_IV_LEN; gcm_params->ulTagBits = AES_GMAC_TAG_BITS; if (data == NULL) return (CRYPTO_SUCCESS); if (data->cd_format != CRYPTO_DATA_RAW) return (CRYPTO_ARGUMENTS_BAD); gcm_params->pAAD = (uchar_t *)data->cd_raw.iov_base; gcm_params->ulAADLen = data->cd_length; return (CRYPTO_SUCCESS); } static int aes_mac_atomic(crypto_provider_handle_t provider, crypto_session_id_t session_id, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac, crypto_spi_ctx_template_t template, crypto_req_handle_t req) { CK_AES_GCM_PARAMS gcm_params; crypto_mechanism_t gcm_mech; int rv; if ((rv = process_gmac_mech(mechanism, data, &gcm_params)) != CRYPTO_SUCCESS) return (rv); gcm_mech.cm_type = AES_GCM_MECH_INFO_TYPE; gcm_mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); gcm_mech.cm_param = (char *)&gcm_params; return (aes_encrypt_atomic(provider, session_id, &gcm_mech, key, &null_crypto_data, mac, template, req)); } static int aes_mac_verify_atomic(crypto_provider_handle_t provider, crypto_session_id_t session_id, crypto_mechanism_t *mechanism, crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac, crypto_spi_ctx_template_t template, crypto_req_handle_t req) { CK_AES_GCM_PARAMS gcm_params; crypto_mechanism_t gcm_mech; int rv; if ((rv = process_gmac_mech(mechanism, data, &gcm_params)) != CRYPTO_SUCCESS) return (rv); gcm_mech.cm_type = AES_GCM_MECH_INFO_TYPE; gcm_mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS); gcm_mech.cm_param = (char *)&gcm_params; return (aes_decrypt_atomic(provider, session_id, &gcm_mech, key, mac, &null_crypto_data, template, req)); } diff --git a/module/spl/spl-taskq.c b/module/spl/spl-taskq.c index 69d591ff74af..90e1d0a4d4d0 100644 --- a/module/spl/spl-taskq.c +++ b/module/spl/spl-taskq.c @@ -1,1292 +1,1292 @@ /* * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. * Copyright (C) 2007 The Regents of the University of California. * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). * Written by Brian Behlendorf . * UCRL-CODE-235197 * * This file is part of the SPL, Solaris Porting Layer. * For details, see . * * The SPL is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * * The SPL is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * for more details. * * You should have received a copy of the GNU General Public License along * with the SPL. If not, see . * * Solaris Porting Layer (SPL) Task Queue Implementation. */ #include #include #include #include -#include +#include int spl_taskq_thread_bind = 0; module_param(spl_taskq_thread_bind, int, 0644); MODULE_PARM_DESC(spl_taskq_thread_bind, "Bind taskq thread to CPU by default"); int spl_taskq_thread_dynamic = 1; module_param(spl_taskq_thread_dynamic, int, 0644); MODULE_PARM_DESC(spl_taskq_thread_dynamic, "Allow dynamic taskq threads"); int spl_taskq_thread_priority = 1; module_param(spl_taskq_thread_priority, int, 0644); MODULE_PARM_DESC(spl_taskq_thread_priority, "Allow non-default priority for taskq threads"); int spl_taskq_thread_sequential = 4; module_param(spl_taskq_thread_sequential, int, 0644); MODULE_PARM_DESC(spl_taskq_thread_sequential, "Create new taskq threads after N sequential tasks"); /* Global system-wide dynamic task queue available for all consumers */ taskq_t *system_taskq; EXPORT_SYMBOL(system_taskq); /* Global dynamic task queue for long delay */ taskq_t *system_delay_taskq; EXPORT_SYMBOL(system_delay_taskq); /* Private dedicated taskq for creating new taskq threads on demand. */ static taskq_t *dynamic_taskq; static taskq_thread_t *taskq_thread_create(taskq_t *); /* List of all taskqs */ LIST_HEAD(tq_list); struct rw_semaphore tq_list_sem; static uint_t taskq_tsd; static int task_km_flags(uint_t flags) { if (flags & TQ_NOSLEEP) return (KM_NOSLEEP); if (flags & TQ_PUSHPAGE) return (KM_PUSHPAGE); return (KM_SLEEP); } /* * taskq_find_by_name - Find the largest instance number of a named taskq. */ static int taskq_find_by_name(const char *name) { struct list_head *tql; taskq_t *tq; list_for_each_prev(tql, &tq_list) { tq = list_entry(tql, taskq_t, tq_taskqs); if (strcmp(name, tq->tq_name) == 0) return (tq->tq_instance); } return (-1); } /* * NOTE: Must be called with tq->tq_lock held, returns a list_t which * is not attached to the free, work, or pending taskq lists. */ static taskq_ent_t * task_alloc(taskq_t *tq, uint_t flags, unsigned long *irqflags) { taskq_ent_t *t; int count = 0; ASSERT(tq); retry: /* Acquire taskq_ent_t's from free list if available */ if (!list_empty(&tq->tq_free_list) && !(flags & TQ_NEW)) { t = list_entry(tq->tq_free_list.next, taskq_ent_t, tqent_list); ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC)); ASSERT(!(t->tqent_flags & TQENT_FLAG_CANCEL)); ASSERT(!timer_pending(&t->tqent_timer)); list_del_init(&t->tqent_list); return (t); } /* Free list is empty and memory allocations are prohibited */ if (flags & TQ_NOALLOC) return (NULL); /* Hit maximum taskq_ent_t pool size */ if (tq->tq_nalloc >= tq->tq_maxalloc) { if (flags & TQ_NOSLEEP) return (NULL); /* * Sleep periodically polling the free list for an available * taskq_ent_t. Dispatching with TQ_SLEEP should always succeed * but we cannot block forever waiting for an taskq_ent_t to * show up in the free list, otherwise a deadlock can happen. * * Therefore, we need to allocate a new task even if the number * of allocated tasks is above tq->tq_maxalloc, but we still * end up delaying the task allocation by one second, thereby * throttling the task dispatch rate. */ spin_unlock_irqrestore(&tq->tq_lock, *irqflags); schedule_timeout(HZ / 100); spin_lock_irqsave_nested(&tq->tq_lock, *irqflags, tq->tq_lock_class); if (count < 100) { count++; goto retry; } } spin_unlock_irqrestore(&tq->tq_lock, *irqflags); t = kmem_alloc(sizeof (taskq_ent_t), task_km_flags(flags)); spin_lock_irqsave_nested(&tq->tq_lock, *irqflags, tq->tq_lock_class); if (t) { taskq_init_ent(t); tq->tq_nalloc++; } return (t); } /* * NOTE: Must be called with tq->tq_lock held, expects the taskq_ent_t * to already be removed from the free, work, or pending taskq lists. */ static void task_free(taskq_t *tq, taskq_ent_t *t) { ASSERT(tq); ASSERT(t); ASSERT(list_empty(&t->tqent_list)); ASSERT(!timer_pending(&t->tqent_timer)); kmem_free(t, sizeof (taskq_ent_t)); tq->tq_nalloc--; } /* * NOTE: Must be called with tq->tq_lock held, either destroys the * taskq_ent_t if too many exist or moves it to the free list for later use. */ static void task_done(taskq_t *tq, taskq_ent_t *t) { ASSERT(tq); ASSERT(t); /* Wake tasks blocked in taskq_wait_id() */ wake_up_all(&t->tqent_waitq); list_del_init(&t->tqent_list); if (tq->tq_nalloc <= tq->tq_minalloc) { t->tqent_id = TASKQID_INVALID; t->tqent_func = NULL; t->tqent_arg = NULL; t->tqent_flags = 0; list_add_tail(&t->tqent_list, &tq->tq_free_list); } else { task_free(tq, t); } } /* * When a delayed task timer expires remove it from the delay list and * add it to the priority list in order for immediate processing. */ static void task_expire_impl(taskq_ent_t *t) { taskq_ent_t *w; taskq_t *tq = t->tqent_taskq; struct list_head *l; unsigned long flags; spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); if (t->tqent_flags & TQENT_FLAG_CANCEL) { ASSERT(list_empty(&t->tqent_list)); spin_unlock_irqrestore(&tq->tq_lock, flags); return; } t->tqent_birth = jiffies; /* * The priority list must be maintained in strict task id order * from lowest to highest for lowest_id to be easily calculable. */ list_del(&t->tqent_list); list_for_each_prev(l, &tq->tq_prio_list) { w = list_entry(l, taskq_ent_t, tqent_list); if (w->tqent_id < t->tqent_id) { list_add(&t->tqent_list, l); break; } } if (l == &tq->tq_prio_list) list_add(&t->tqent_list, &tq->tq_prio_list); spin_unlock_irqrestore(&tq->tq_lock, flags); wake_up(&tq->tq_work_waitq); } static void task_expire(spl_timer_list_t tl) { struct timer_list *tmr = (struct timer_list *)tl; taskq_ent_t *t = from_timer(t, tmr, tqent_timer); task_expire_impl(t); } /* * Returns the lowest incomplete taskqid_t. The taskqid_t may * be queued on the pending list, on the priority list, on the * delay list, or on the work list currently being handled, but * it is not 100% complete yet. */ static taskqid_t taskq_lowest_id(taskq_t *tq) { taskqid_t lowest_id = tq->tq_next_id; taskq_ent_t *t; taskq_thread_t *tqt; ASSERT(tq); if (!list_empty(&tq->tq_pend_list)) { t = list_entry(tq->tq_pend_list.next, taskq_ent_t, tqent_list); lowest_id = MIN(lowest_id, t->tqent_id); } if (!list_empty(&tq->tq_prio_list)) { t = list_entry(tq->tq_prio_list.next, taskq_ent_t, tqent_list); lowest_id = MIN(lowest_id, t->tqent_id); } if (!list_empty(&tq->tq_delay_list)) { t = list_entry(tq->tq_delay_list.next, taskq_ent_t, tqent_list); lowest_id = MIN(lowest_id, t->tqent_id); } if (!list_empty(&tq->tq_active_list)) { tqt = list_entry(tq->tq_active_list.next, taskq_thread_t, tqt_active_list); ASSERT(tqt->tqt_id != TASKQID_INVALID); lowest_id = MIN(lowest_id, tqt->tqt_id); } return (lowest_id); } /* * Insert a task into a list keeping the list sorted by increasing taskqid. */ static void taskq_insert_in_order(taskq_t *tq, taskq_thread_t *tqt) { taskq_thread_t *w; struct list_head *l; ASSERT(tq); ASSERT(tqt); list_for_each_prev(l, &tq->tq_active_list) { w = list_entry(l, taskq_thread_t, tqt_active_list); if (w->tqt_id < tqt->tqt_id) { list_add(&tqt->tqt_active_list, l); break; } } if (l == &tq->tq_active_list) list_add(&tqt->tqt_active_list, &tq->tq_active_list); } /* * Find and return a task from the given list if it exists. The list * must be in lowest to highest task id order. */ static taskq_ent_t * taskq_find_list(taskq_t *tq, struct list_head *lh, taskqid_t id) { struct list_head *l; taskq_ent_t *t; list_for_each(l, lh) { t = list_entry(l, taskq_ent_t, tqent_list); if (t->tqent_id == id) return (t); if (t->tqent_id > id) break; } return (NULL); } /* * Find an already dispatched task given the task id regardless of what * state it is in. If a task is still pending it will be returned. * If a task is executing, then -EBUSY will be returned instead. * If the task has already been run then NULL is returned. */ static taskq_ent_t * taskq_find(taskq_t *tq, taskqid_t id) { taskq_thread_t *tqt; struct list_head *l; taskq_ent_t *t; t = taskq_find_list(tq, &tq->tq_delay_list, id); if (t) return (t); t = taskq_find_list(tq, &tq->tq_prio_list, id); if (t) return (t); t = taskq_find_list(tq, &tq->tq_pend_list, id); if (t) return (t); list_for_each(l, &tq->tq_active_list) { tqt = list_entry(l, taskq_thread_t, tqt_active_list); if (tqt->tqt_id == id) { /* * Instead of returning tqt_task, we just return a non * NULL value to prevent misuse, since tqt_task only * has two valid fields. */ return (ERR_PTR(-EBUSY)); } } return (NULL); } /* * Theory for the taskq_wait_id(), taskq_wait_outstanding(), and * taskq_wait() functions below. * * Taskq waiting is accomplished by tracking the lowest outstanding task * id and the next available task id. As tasks are dispatched they are * added to the tail of the pending, priority, or delay lists. As worker * threads become available the tasks are removed from the heads of these * lists and linked to the worker threads. This ensures the lists are * kept sorted by lowest to highest task id. * * Therefore the lowest outstanding task id can be quickly determined by * checking the head item from all of these lists. This value is stored * with the taskq as the lowest id. It only needs to be recalculated when * either the task with the current lowest id completes or is canceled. * * By blocking until the lowest task id exceeds the passed task id the * taskq_wait_outstanding() function can be easily implemented. Similarly, * by blocking until the lowest task id matches the next task id taskq_wait() * can be implemented. * * Callers should be aware that when there are multiple worked threads it * is possible for larger task ids to complete before smaller ones. Also * when the taskq contains delay tasks with small task ids callers may * block for a considerable length of time waiting for them to expire and * execute. */ static int taskq_wait_id_check(taskq_t *tq, taskqid_t id) { int rc; unsigned long flags; spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); rc = (taskq_find(tq, id) == NULL); spin_unlock_irqrestore(&tq->tq_lock, flags); return (rc); } /* * The taskq_wait_id() function blocks until the passed task id completes. * This does not guarantee that all lower task ids have completed. */ void taskq_wait_id(taskq_t *tq, taskqid_t id) { wait_event(tq->tq_wait_waitq, taskq_wait_id_check(tq, id)); } EXPORT_SYMBOL(taskq_wait_id); static int taskq_wait_outstanding_check(taskq_t *tq, taskqid_t id) { int rc; unsigned long flags; spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); rc = (id < tq->tq_lowest_id); spin_unlock_irqrestore(&tq->tq_lock, flags); return (rc); } /* * The taskq_wait_outstanding() function will block until all tasks with a * lower taskqid than the passed 'id' have been completed. Note that all * task id's are assigned monotonically at dispatch time. Zero may be * passed for the id to indicate all tasks dispatch up to this point, * but not after, should be waited for. */ void taskq_wait_outstanding(taskq_t *tq, taskqid_t id) { id = id ? id : tq->tq_next_id - 1; wait_event(tq->tq_wait_waitq, taskq_wait_outstanding_check(tq, id)); } EXPORT_SYMBOL(taskq_wait_outstanding); static int taskq_wait_check(taskq_t *tq) { int rc; unsigned long flags; spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); rc = (tq->tq_lowest_id == tq->tq_next_id); spin_unlock_irqrestore(&tq->tq_lock, flags); return (rc); } /* * The taskq_wait() function will block until the taskq is empty. * This means that if a taskq re-dispatches work to itself taskq_wait() * callers will block indefinitely. */ void taskq_wait(taskq_t *tq) { wait_event(tq->tq_wait_waitq, taskq_wait_check(tq)); } EXPORT_SYMBOL(taskq_wait); int taskq_member(taskq_t *tq, kthread_t *t) { return (tq == (taskq_t *)tsd_get_by_thread(taskq_tsd, t)); } EXPORT_SYMBOL(taskq_member); /* * Cancel an already dispatched task given the task id. Still pending tasks * will be immediately canceled, and if the task is active the function will * block until it completes. Preallocated tasks which are canceled must be * freed by the caller. */ int taskq_cancel_id(taskq_t *tq, taskqid_t id) { taskq_ent_t *t; int rc = ENOENT; unsigned long flags; ASSERT(tq); spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); t = taskq_find(tq, id); if (t && t != ERR_PTR(-EBUSY)) { list_del_init(&t->tqent_list); t->tqent_flags |= TQENT_FLAG_CANCEL; /* * When canceling the lowest outstanding task id we * must recalculate the new lowest outstanding id. */ if (tq->tq_lowest_id == t->tqent_id) { tq->tq_lowest_id = taskq_lowest_id(tq); ASSERT3S(tq->tq_lowest_id, >, t->tqent_id); } /* * The task_expire() function takes the tq->tq_lock so drop * drop the lock before synchronously cancelling the timer. */ if (timer_pending(&t->tqent_timer)) { spin_unlock_irqrestore(&tq->tq_lock, flags); del_timer_sync(&t->tqent_timer); spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); } if (!(t->tqent_flags & TQENT_FLAG_PREALLOC)) task_done(tq, t); rc = 0; } spin_unlock_irqrestore(&tq->tq_lock, flags); if (t == ERR_PTR(-EBUSY)) { taskq_wait_id(tq, id); rc = EBUSY; } return (rc); } EXPORT_SYMBOL(taskq_cancel_id); static int taskq_thread_spawn(taskq_t *tq); taskqid_t taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags) { taskq_ent_t *t; taskqid_t rc = TASKQID_INVALID; unsigned long irqflags; ASSERT(tq); ASSERT(func); spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class); /* Taskq being destroyed and all tasks drained */ if (!(tq->tq_flags & TASKQ_ACTIVE)) goto out; /* Do not queue the task unless there is idle thread for it */ ASSERT(tq->tq_nactive <= tq->tq_nthreads); if ((flags & TQ_NOQUEUE) && (tq->tq_nactive == tq->tq_nthreads)) { /* Dynamic taskq may be able to spawn another thread */ if (!(tq->tq_flags & TASKQ_DYNAMIC) || taskq_thread_spawn(tq) == 0) goto out; } if ((t = task_alloc(tq, flags, &irqflags)) == NULL) goto out; spin_lock(&t->tqent_lock); /* Queue to the front of the list to enforce TQ_NOQUEUE semantics */ if (flags & TQ_NOQUEUE) list_add(&t->tqent_list, &tq->tq_prio_list); /* Queue to the priority list instead of the pending list */ else if (flags & TQ_FRONT) list_add_tail(&t->tqent_list, &tq->tq_prio_list); else list_add_tail(&t->tqent_list, &tq->tq_pend_list); t->tqent_id = rc = tq->tq_next_id; tq->tq_next_id++; t->tqent_func = func; t->tqent_arg = arg; t->tqent_taskq = tq; t->tqent_timer.function = NULL; t->tqent_timer.expires = 0; t->tqent_birth = jiffies; ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC)); spin_unlock(&t->tqent_lock); wake_up(&tq->tq_work_waitq); out: /* Spawn additional taskq threads if required. */ if (!(flags & TQ_NOQUEUE) && tq->tq_nactive == tq->tq_nthreads) (void) taskq_thread_spawn(tq); spin_unlock_irqrestore(&tq->tq_lock, irqflags); return (rc); } EXPORT_SYMBOL(taskq_dispatch); taskqid_t taskq_dispatch_delay(taskq_t *tq, task_func_t func, void *arg, uint_t flags, clock_t expire_time) { taskqid_t rc = TASKQID_INVALID; taskq_ent_t *t; unsigned long irqflags; ASSERT(tq); ASSERT(func); spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class); /* Taskq being destroyed and all tasks drained */ if (!(tq->tq_flags & TASKQ_ACTIVE)) goto out; if ((t = task_alloc(tq, flags, &irqflags)) == NULL) goto out; spin_lock(&t->tqent_lock); /* Queue to the delay list for subsequent execution */ list_add_tail(&t->tqent_list, &tq->tq_delay_list); t->tqent_id = rc = tq->tq_next_id; tq->tq_next_id++; t->tqent_func = func; t->tqent_arg = arg; t->tqent_taskq = tq; t->tqent_timer.function = task_expire; t->tqent_timer.expires = (unsigned long)expire_time; add_timer(&t->tqent_timer); ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC)); spin_unlock(&t->tqent_lock); out: /* Spawn additional taskq threads if required. */ if (tq->tq_nactive == tq->tq_nthreads) (void) taskq_thread_spawn(tq); spin_unlock_irqrestore(&tq->tq_lock, irqflags); return (rc); } EXPORT_SYMBOL(taskq_dispatch_delay); void taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, uint_t flags, taskq_ent_t *t) { unsigned long irqflags; ASSERT(tq); ASSERT(func); spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class); /* Taskq being destroyed and all tasks drained */ if (!(tq->tq_flags & TASKQ_ACTIVE)) { t->tqent_id = TASKQID_INVALID; goto out; } if ((flags & TQ_NOQUEUE) && (tq->tq_nactive == tq->tq_nthreads)) { /* Dynamic taskq may be able to spawn another thread */ if (!(tq->tq_flags & TASKQ_DYNAMIC) || taskq_thread_spawn(tq) == 0) goto out2; flags |= TQ_FRONT; } spin_lock(&t->tqent_lock); /* * Make sure the entry is not on some other taskq; it is important to * ASSERT() under lock */ ASSERT(taskq_empty_ent(t)); /* * Mark it as a prealloc'd task. This is important * to ensure that we don't free it later. */ t->tqent_flags |= TQENT_FLAG_PREALLOC; /* Queue to the priority list instead of the pending list */ if (flags & TQ_FRONT) list_add_tail(&t->tqent_list, &tq->tq_prio_list); else list_add_tail(&t->tqent_list, &tq->tq_pend_list); t->tqent_id = tq->tq_next_id; tq->tq_next_id++; t->tqent_func = func; t->tqent_arg = arg; t->tqent_taskq = tq; t->tqent_birth = jiffies; spin_unlock(&t->tqent_lock); wake_up(&tq->tq_work_waitq); out: /* Spawn additional taskq threads if required. */ if (tq->tq_nactive == tq->tq_nthreads) (void) taskq_thread_spawn(tq); out2: spin_unlock_irqrestore(&tq->tq_lock, irqflags); } EXPORT_SYMBOL(taskq_dispatch_ent); int taskq_empty_ent(taskq_ent_t *t) { return (list_empty(&t->tqent_list)); } EXPORT_SYMBOL(taskq_empty_ent); void taskq_init_ent(taskq_ent_t *t) { spin_lock_init(&t->tqent_lock); init_waitqueue_head(&t->tqent_waitq); timer_setup(&t->tqent_timer, NULL, 0); INIT_LIST_HEAD(&t->tqent_list); t->tqent_id = 0; t->tqent_func = NULL; t->tqent_arg = NULL; t->tqent_flags = 0; t->tqent_taskq = NULL; } EXPORT_SYMBOL(taskq_init_ent); /* * Return the next pending task, preference is given to tasks on the * priority list which were dispatched with TQ_FRONT. */ static taskq_ent_t * taskq_next_ent(taskq_t *tq) { struct list_head *list; if (!list_empty(&tq->tq_prio_list)) list = &tq->tq_prio_list; else if (!list_empty(&tq->tq_pend_list)) list = &tq->tq_pend_list; else return (NULL); return (list_entry(list->next, taskq_ent_t, tqent_list)); } /* * Spawns a new thread for the specified taskq. */ static void taskq_thread_spawn_task(void *arg) { taskq_t *tq = (taskq_t *)arg; unsigned long flags; if (taskq_thread_create(tq) == NULL) { /* restore spawning count if failed */ spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); tq->tq_nspawn--; spin_unlock_irqrestore(&tq->tq_lock, flags); } } /* * Spawn addition threads for dynamic taskqs (TASKQ_DYNAMIC) the current * number of threads is insufficient to handle the pending tasks. These * new threads must be created by the dedicated dynamic_taskq to avoid * deadlocks between thread creation and memory reclaim. The system_taskq * which is also a dynamic taskq cannot be safely used for this. */ static int taskq_thread_spawn(taskq_t *tq) { int spawning = 0; if (!(tq->tq_flags & TASKQ_DYNAMIC)) return (0); if ((tq->tq_nthreads + tq->tq_nspawn < tq->tq_maxthreads) && (tq->tq_flags & TASKQ_ACTIVE)) { spawning = (++tq->tq_nspawn); taskq_dispatch(dynamic_taskq, taskq_thread_spawn_task, tq, TQ_NOSLEEP); } return (spawning); } /* * Threads in a dynamic taskq should only exit once it has been completely * drained and no other threads are actively servicing tasks. This prevents * threads from being created and destroyed more than is required. * * The first thread is the thread list is treated as the primary thread. * There is nothing special about the primary thread but in order to avoid * all the taskq pids from changing we opt to make it long running. */ static int taskq_thread_should_stop(taskq_t *tq, taskq_thread_t *tqt) { if (!(tq->tq_flags & TASKQ_DYNAMIC)) return (0); if (list_first_entry(&(tq->tq_thread_list), taskq_thread_t, tqt_thread_list) == tqt) return (0); return ((tq->tq_nspawn == 0) && /* No threads are being spawned */ (tq->tq_nactive == 0) && /* No threads are handling tasks */ (tq->tq_nthreads > 1) && /* More than 1 thread is running */ (!taskq_next_ent(tq)) && /* There are no pending tasks */ (spl_taskq_thread_dynamic)); /* Dynamic taskqs are allowed */ } static int taskq_thread(void *args) { DECLARE_WAITQUEUE(wait, current); sigset_t blocked; taskq_thread_t *tqt = args; taskq_t *tq; taskq_ent_t *t; int seq_tasks = 0; unsigned long flags; taskq_ent_t dup_task = {}; ASSERT(tqt); ASSERT(tqt->tqt_tq); tq = tqt->tqt_tq; current->flags |= PF_NOFREEZE; (void) spl_fstrans_mark(); sigfillset(&blocked); sigprocmask(SIG_BLOCK, &blocked, NULL); flush_signals(current); kfpu_initialize(); tsd_set(taskq_tsd, tq); spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); /* * If we are dynamically spawned, decrease spawning count. Note that * we could be created during taskq_create, in which case we shouldn't * do the decrement. But it's fine because taskq_create will reset * tq_nspawn later. */ if (tq->tq_flags & TASKQ_DYNAMIC) tq->tq_nspawn--; /* Immediately exit if more threads than allowed were created. */ if (tq->tq_nthreads >= tq->tq_maxthreads) goto error; tq->tq_nthreads++; list_add_tail(&tqt->tqt_thread_list, &tq->tq_thread_list); wake_up(&tq->tq_wait_waitq); set_current_state(TASK_INTERRUPTIBLE); while (!kthread_should_stop()) { if (list_empty(&tq->tq_pend_list) && list_empty(&tq->tq_prio_list)) { if (taskq_thread_should_stop(tq, tqt)) { wake_up_all(&tq->tq_wait_waitq); break; } add_wait_queue_exclusive(&tq->tq_work_waitq, &wait); spin_unlock_irqrestore(&tq->tq_lock, flags); schedule(); seq_tasks = 0; spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); remove_wait_queue(&tq->tq_work_waitq, &wait); } else { __set_current_state(TASK_RUNNING); } if ((t = taskq_next_ent(tq)) != NULL) { list_del_init(&t->tqent_list); /* * A TQENT_FLAG_PREALLOC task may be reused or freed * during the task function call. Store tqent_id and * tqent_flags here. * * Also use an on stack taskq_ent_t for tqt_task * assignment in this case. We only populate the two * fields used by the only user in taskq proc file. */ tqt->tqt_id = t->tqent_id; tqt->tqt_flags = t->tqent_flags; if (t->tqent_flags & TQENT_FLAG_PREALLOC) { dup_task.tqent_func = t->tqent_func; dup_task.tqent_arg = t->tqent_arg; t = &dup_task; } tqt->tqt_task = t; taskq_insert_in_order(tq, tqt); tq->tq_nactive++; spin_unlock_irqrestore(&tq->tq_lock, flags); /* Perform the requested task */ t->tqent_func(t->tqent_arg); spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); tq->tq_nactive--; list_del_init(&tqt->tqt_active_list); tqt->tqt_task = NULL; /* For prealloc'd tasks, we don't free anything. */ if (!(tqt->tqt_flags & TQENT_FLAG_PREALLOC)) task_done(tq, t); /* * When the current lowest outstanding taskqid is * done calculate the new lowest outstanding id */ if (tq->tq_lowest_id == tqt->tqt_id) { tq->tq_lowest_id = taskq_lowest_id(tq); ASSERT3S(tq->tq_lowest_id, >, tqt->tqt_id); } /* Spawn additional taskq threads if required. */ if ((++seq_tasks) > spl_taskq_thread_sequential && taskq_thread_spawn(tq)) seq_tasks = 0; tqt->tqt_id = TASKQID_INVALID; tqt->tqt_flags = 0; wake_up_all(&tq->tq_wait_waitq); } else { if (taskq_thread_should_stop(tq, tqt)) break; } set_current_state(TASK_INTERRUPTIBLE); } __set_current_state(TASK_RUNNING); tq->tq_nthreads--; list_del_init(&tqt->tqt_thread_list); error: kmem_free(tqt, sizeof (taskq_thread_t)); spin_unlock_irqrestore(&tq->tq_lock, flags); tsd_set(taskq_tsd, NULL); return (0); } static taskq_thread_t * taskq_thread_create(taskq_t *tq) { static int last_used_cpu = 0; taskq_thread_t *tqt; tqt = kmem_alloc(sizeof (*tqt), KM_PUSHPAGE); INIT_LIST_HEAD(&tqt->tqt_thread_list); INIT_LIST_HEAD(&tqt->tqt_active_list); tqt->tqt_tq = tq; tqt->tqt_id = TASKQID_INVALID; tqt->tqt_thread = spl_kthread_create(taskq_thread, tqt, "%s", tq->tq_name); if (tqt->tqt_thread == NULL) { kmem_free(tqt, sizeof (taskq_thread_t)); return (NULL); } if (spl_taskq_thread_bind) { last_used_cpu = (last_used_cpu + 1) % num_online_cpus(); kthread_bind(tqt->tqt_thread, last_used_cpu); } if (spl_taskq_thread_priority) set_user_nice(tqt->tqt_thread, PRIO_TO_NICE(tq->tq_pri)); wake_up_process(tqt->tqt_thread); return (tqt); } taskq_t * taskq_create(const char *name, int nthreads, pri_t pri, int minalloc, int maxalloc, uint_t flags) { taskq_t *tq; taskq_thread_t *tqt; int count = 0, rc = 0, i; unsigned long irqflags; ASSERT(name != NULL); ASSERT(minalloc >= 0); ASSERT(maxalloc <= INT_MAX); ASSERT(!(flags & (TASKQ_CPR_SAFE))); /* Unsupported */ /* Scale the number of threads using nthreads as a percentage */ if (flags & TASKQ_THREADS_CPU_PCT) { ASSERT(nthreads <= 100); ASSERT(nthreads >= 0); nthreads = MIN(nthreads, 100); nthreads = MAX(nthreads, 0); nthreads = MAX((num_online_cpus() * nthreads) / 100, 1); } tq = kmem_alloc(sizeof (*tq), KM_PUSHPAGE); if (tq == NULL) return (NULL); spin_lock_init(&tq->tq_lock); INIT_LIST_HEAD(&tq->tq_thread_list); INIT_LIST_HEAD(&tq->tq_active_list); tq->tq_name = strdup(name); tq->tq_nactive = 0; tq->tq_nthreads = 0; tq->tq_nspawn = 0; tq->tq_maxthreads = nthreads; tq->tq_pri = pri; tq->tq_minalloc = minalloc; tq->tq_maxalloc = maxalloc; tq->tq_nalloc = 0; tq->tq_flags = (flags | TASKQ_ACTIVE); tq->tq_next_id = TASKQID_INITIAL; tq->tq_lowest_id = TASKQID_INITIAL; INIT_LIST_HEAD(&tq->tq_free_list); INIT_LIST_HEAD(&tq->tq_pend_list); INIT_LIST_HEAD(&tq->tq_prio_list); INIT_LIST_HEAD(&tq->tq_delay_list); init_waitqueue_head(&tq->tq_work_waitq); init_waitqueue_head(&tq->tq_wait_waitq); tq->tq_lock_class = TQ_LOCK_GENERAL; INIT_LIST_HEAD(&tq->tq_taskqs); if (flags & TASKQ_PREPOPULATE) { spin_lock_irqsave_nested(&tq->tq_lock, irqflags, tq->tq_lock_class); for (i = 0; i < minalloc; i++) task_done(tq, task_alloc(tq, TQ_PUSHPAGE | TQ_NEW, &irqflags)); spin_unlock_irqrestore(&tq->tq_lock, irqflags); } if ((flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic) nthreads = 1; for (i = 0; i < nthreads; i++) { tqt = taskq_thread_create(tq); if (tqt == NULL) rc = 1; else count++; } /* Wait for all threads to be started before potential destroy */ wait_event(tq->tq_wait_waitq, tq->tq_nthreads == count); /* * taskq_thread might have touched nspawn, but we don't want them to * because they're not dynamically spawned. So we reset it to 0 */ tq->tq_nspawn = 0; if (rc) { taskq_destroy(tq); tq = NULL; } else { down_write(&tq_list_sem); tq->tq_instance = taskq_find_by_name(name) + 1; list_add_tail(&tq->tq_taskqs, &tq_list); up_write(&tq_list_sem); } return (tq); } EXPORT_SYMBOL(taskq_create); void taskq_destroy(taskq_t *tq) { struct task_struct *thread; taskq_thread_t *tqt; taskq_ent_t *t; unsigned long flags; ASSERT(tq); spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); tq->tq_flags &= ~TASKQ_ACTIVE; spin_unlock_irqrestore(&tq->tq_lock, flags); /* * When TASKQ_ACTIVE is clear new tasks may not be added nor may * new worker threads be spawned for dynamic taskq. */ if (dynamic_taskq != NULL) taskq_wait_outstanding(dynamic_taskq, 0); taskq_wait(tq); /* remove taskq from global list used by the kstats */ down_write(&tq_list_sem); list_del(&tq->tq_taskqs); up_write(&tq_list_sem); spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); /* wait for spawning threads to insert themselves to the list */ while (tq->tq_nspawn) { spin_unlock_irqrestore(&tq->tq_lock, flags); schedule_timeout_interruptible(1); spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); } /* * Signal each thread to exit and block until it does. Each thread * is responsible for removing itself from the list and freeing its * taskq_thread_t. This allows for idle threads to opt to remove * themselves from the taskq. They can be recreated as needed. */ while (!list_empty(&tq->tq_thread_list)) { tqt = list_entry(tq->tq_thread_list.next, taskq_thread_t, tqt_thread_list); thread = tqt->tqt_thread; spin_unlock_irqrestore(&tq->tq_lock, flags); kthread_stop(thread); spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); } while (!list_empty(&tq->tq_free_list)) { t = list_entry(tq->tq_free_list.next, taskq_ent_t, tqent_list); ASSERT(!(t->tqent_flags & TQENT_FLAG_PREALLOC)); list_del_init(&t->tqent_list); task_free(tq, t); } ASSERT0(tq->tq_nthreads); ASSERT0(tq->tq_nalloc); ASSERT0(tq->tq_nspawn); ASSERT(list_empty(&tq->tq_thread_list)); ASSERT(list_empty(&tq->tq_active_list)); ASSERT(list_empty(&tq->tq_free_list)); ASSERT(list_empty(&tq->tq_pend_list)); ASSERT(list_empty(&tq->tq_prio_list)); ASSERT(list_empty(&tq->tq_delay_list)); spin_unlock_irqrestore(&tq->tq_lock, flags); strfree(tq->tq_name); kmem_free(tq, sizeof (taskq_t)); } EXPORT_SYMBOL(taskq_destroy); static unsigned int spl_taskq_kick = 0; /* * 2.6.36 API Change * module_param_cb is introduced to take kernel_param_ops and * module_param_call is marked as obsolete. Also set and get operations * were changed to take a 'const struct kernel_param *'. */ static int #ifdef module_param_cb param_set_taskq_kick(const char *val, const struct kernel_param *kp) #else param_set_taskq_kick(const char *val, struct kernel_param *kp) #endif { int ret; taskq_t *tq; taskq_ent_t *t; unsigned long flags; ret = param_set_uint(val, kp); if (ret < 0 || !spl_taskq_kick) return (ret); /* reset value */ spl_taskq_kick = 0; down_read(&tq_list_sem); list_for_each_entry(tq, &tq_list, tq_taskqs) { spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class); /* Check if the first pending is older than 5 seconds */ t = taskq_next_ent(tq); if (t && time_after(jiffies, t->tqent_birth + 5*HZ)) { (void) taskq_thread_spawn(tq); printk(KERN_INFO "spl: Kicked taskq %s/%d\n", tq->tq_name, tq->tq_instance); } spin_unlock_irqrestore(&tq->tq_lock, flags); } up_read(&tq_list_sem); return (ret); } #ifdef module_param_cb static const struct kernel_param_ops param_ops_taskq_kick = { .set = param_set_taskq_kick, .get = param_get_uint, }; module_param_cb(spl_taskq_kick, ¶m_ops_taskq_kick, &spl_taskq_kick, 0644); #else module_param_call(spl_taskq_kick, param_set_taskq_kick, param_get_uint, &spl_taskq_kick, 0644); #endif MODULE_PARM_DESC(spl_taskq_kick, "Write nonzero to kick stuck taskqs to spawn more threads"); int spl_taskq_init(void) { init_rwsem(&tq_list_sem); tsd_create(&taskq_tsd, NULL); system_taskq = taskq_create("spl_system_taskq", MAX(boot_ncpus, 64), maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC); if (system_taskq == NULL) return (1); system_delay_taskq = taskq_create("spl_delay_taskq", MAX(boot_ncpus, 4), maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE|TASKQ_DYNAMIC); if (system_delay_taskq == NULL) { taskq_destroy(system_taskq); return (1); } dynamic_taskq = taskq_create("spl_dynamic_taskq", 1, maxclsyspri, boot_ncpus, INT_MAX, TASKQ_PREPOPULATE); if (dynamic_taskq == NULL) { taskq_destroy(system_taskq); taskq_destroy(system_delay_taskq); return (1); } /* * This is used to annotate tq_lock, so * taskq_dispatch -> taskq_thread_spawn -> taskq_dispatch * does not trigger a lockdep warning re: possible recursive locking */ dynamic_taskq->tq_lock_class = TQ_LOCK_DYNAMIC; return (0); } void spl_taskq_fini(void) { taskq_destroy(dynamic_taskq); dynamic_taskq = NULL; taskq_destroy(system_delay_taskq); system_delay_taskq = NULL; taskq_destroy(system_taskq); system_taskq = NULL; tsd_destroy(&taskq_tsd); } diff --git a/module/spl/spl-thread.c b/module/spl/spl-thread.c index 07e3a1bfff29..29de9252a48a 100644 --- a/module/spl/spl-thread.c +++ b/module/spl/spl-thread.c @@ -1,163 +1,163 @@ /* * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC. * Copyright (C) 2007 The Regents of the University of California. * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). * Written by Brian Behlendorf . * UCRL-CODE-235197 * * This file is part of the SPL, Solaris Porting Layer. * For details, see . * * The SPL is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the * Free Software Foundation; either version 2 of the License, or (at your * option) any later version. * * The SPL is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License * for more details. * * You should have received a copy of the GNU General Public License along * with the SPL. If not, see . * * Solaris Porting Layer (SPL) Thread Implementation. */ #include #include #include -#include +#include /* * Thread interfaces */ typedef struct thread_priv_s { unsigned long tp_magic; /* Magic */ int tp_name_size; /* Name size */ char *tp_name; /* Name (without _thread suffix) */ void (*tp_func)(void *); /* Registered function */ void *tp_args; /* Args to be passed to function */ size_t tp_len; /* Len to be passed to function */ int tp_state; /* State to start thread at */ pri_t tp_pri; /* Priority to start threat at */ } thread_priv_t; static int thread_generic_wrapper(void *arg) { thread_priv_t *tp = (thread_priv_t *)arg; void (*func)(void *); void *args; ASSERT(tp->tp_magic == TP_MAGIC); func = tp->tp_func; args = tp->tp_args; set_current_state(tp->tp_state); set_user_nice((kthread_t *)current, PRIO_TO_NICE(tp->tp_pri)); kfpu_initialize(); kmem_free(tp->tp_name, tp->tp_name_size); kmem_free(tp, sizeof (thread_priv_t)); if (func) func(args); return (0); } void __thread_exit(void) { tsd_exit(); complete_and_exit(NULL, 0); /* Unreachable */ } EXPORT_SYMBOL(__thread_exit); /* * thread_create() may block forever if it cannot create a thread or * allocate memory. This is preferable to returning a NULL which Solaris * style callers likely never check for... since it can't fail. */ kthread_t * __thread_create(caddr_t stk, size_t stksize, thread_func_t func, const char *name, void *args, size_t len, proc_t *pp, int state, pri_t pri) { thread_priv_t *tp; struct task_struct *tsk; char *p; /* Option pp is simply ignored */ /* Variable stack size unsupported */ ASSERT(stk == NULL); tp = kmem_alloc(sizeof (thread_priv_t), KM_PUSHPAGE); if (tp == NULL) return (NULL); tp->tp_magic = TP_MAGIC; tp->tp_name_size = strlen(name) + 1; tp->tp_name = kmem_alloc(tp->tp_name_size, KM_PUSHPAGE); if (tp->tp_name == NULL) { kmem_free(tp, sizeof (thread_priv_t)); return (NULL); } strncpy(tp->tp_name, name, tp->tp_name_size); /* * Strip trailing "_thread" from passed name which will be the func * name since the exposed API has no parameter for passing a name. */ p = strstr(tp->tp_name, "_thread"); if (p) p[0] = '\0'; tp->tp_func = func; tp->tp_args = args; tp->tp_len = len; tp->tp_state = state; tp->tp_pri = pri; tsk = spl_kthread_create(thread_generic_wrapper, (void *)tp, "%s", tp->tp_name); if (IS_ERR(tsk)) return (NULL); wake_up_process(tsk); return ((kthread_t *)tsk); } EXPORT_SYMBOL(__thread_create); /* * spl_kthread_create - Wrapper providing pre-3.13 semantics for * kthread_create() in which it is not killable and less likely * to return -ENOMEM. */ struct task_struct * spl_kthread_create(int (*func)(void *), void *data, const char namefmt[], ...) { struct task_struct *tsk; va_list args; char name[TASK_COMM_LEN]; va_start(args, namefmt); vsnprintf(name, sizeof (name), namefmt, args); va_end(args); do { tsk = kthread_create(func, data, "%s", name); if (IS_ERR(tsk)) { if (signal_pending(current)) { clear_thread_flag(TIF_SIGPENDING); continue; } if (PTR_ERR(tsk) == -ENOMEM) continue; return (NULL); } else { return (tsk); } } while (1); } EXPORT_SYMBOL(spl_kthread_create); diff --git a/module/zcommon/zfs_fletcher.c b/module/zcommon/zfs_fletcher.c index 9187a7c1ed5f..c14c95fa2285 100644 --- a/module/zcommon/zfs_fletcher.c +++ b/module/zcommon/zfs_fletcher.c @@ -1,950 +1,950 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * Copyright (C) 2016 Gvozden NeÅ¡ković. All rights reserved. */ /* * Copyright 2013 Saso Kiselkov. All rights reserved. */ /* * Copyright (c) 2016 by Delphix. All rights reserved. */ /* * Fletcher Checksums * ------------------ * * ZFS's 2nd and 4th order Fletcher checksums are defined by the following * recurrence relations: * * a = a + f * i i-1 i-1 * * b = b + a * i i-1 i * * c = c + b (fletcher-4 only) * i i-1 i * * d = d + c (fletcher-4 only) * i i-1 i * * Where * a_0 = b_0 = c_0 = d_0 = 0 * and * f_0 .. f_(n-1) are the input data. * * Using standard techniques, these translate into the following series: * * __n_ __n_ * \ | \ | * a = > f b = > i * f * n /___| n - i n /___| n - i * i = 1 i = 1 * * * __n_ __n_ * \ | i*(i+1) \ | i*(i+1)*(i+2) * c = > ------- f d = > ------------- f * n /___| 2 n - i n /___| 6 n - i * i = 1 i = 1 * * For fletcher-2, the f_is are 64-bit, and [ab]_i are 64-bit accumulators. * Since the additions are done mod (2^64), errors in the high bits may not * be noticed. For this reason, fletcher-2 is deprecated. * * For fletcher-4, the f_is are 32-bit, and [abcd]_i are 64-bit accumulators. * A conservative estimate of how big the buffer can get before we overflow * can be estimated using f_i = 0xffffffff for all i: * * % bc * f=2^32-1;d=0; for (i = 1; d<2^64; i++) { d += f*i*(i+1)*(i+2)/6 }; (i-1)*4 * 2264 * quit * % * * So blocks of up to 2k will not overflow. Our largest block size is * 128k, which has 32k 4-byte words, so we can compute the largest possible * accumulators, then divide by 2^64 to figure the max amount of overflow: * * % bc * a=b=c=d=0; f=2^32-1; for (i=1; i<=32*1024; i++) { a+=f; b+=a; c+=b; d+=c } * a/2^64;b/2^64;c/2^64;d/2^64 * 0 * 0 * 1365 * 11186858 * quit * % * * So a and b cannot overflow. To make sure each bit of input has some * effect on the contents of c and d, we can look at what the factors of * the coefficients in the equations for c_n and d_n are. The number of 2s * in the factors determines the lowest set bit in the multiplier. Running * through the cases for n*(n+1)/2 reveals that the highest power of 2 is * 2^14, and for n*(n+1)*(n+2)/6 it is 2^15. So while some data may overflow * the 64-bit accumulators, every bit of every f_i effects every accumulator, * even for 128k blocks. * * If we wanted to make a stronger version of fletcher4 (fletcher4c?), * we could do our calculations mod (2^32 - 1) by adding in the carries * periodically, and store the number of carries in the top 32-bits. * * -------------------- * Checksum Performance * -------------------- * * There are two interesting components to checksum performance: cached and * uncached performance. With cached data, fletcher-2 is about four times * faster than fletcher-4. With uncached data, the performance difference is * negligible, since the cost of a cache fill dominates the processing time. * Even though fletcher-4 is slower than fletcher-2, it is still a pretty * efficient pass over the data. * * In normal operation, the data which is being checksummed is in a buffer * which has been filled either by: * * 1. a compression step, which will be mostly cached, or * 2. a bcopy() or copyin(), which will be uncached (because the * copy is cache-bypassing). * * For both cached and uncached data, both fletcher checksums are much faster * than sha-256, and slower than 'off', which doesn't touch the data at all. */ #include #include #include #include +#include #include #include #include -#include #define FLETCHER_MIN_SIMD_SIZE 64 static void fletcher_4_scalar_init(fletcher_4_ctx_t *ctx); static void fletcher_4_scalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp); static void fletcher_4_scalar_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size); static void fletcher_4_scalar_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size); static boolean_t fletcher_4_scalar_valid(void); static const fletcher_4_ops_t fletcher_4_scalar_ops = { .init_native = fletcher_4_scalar_init, .fini_native = fletcher_4_scalar_fini, .compute_native = fletcher_4_scalar_native, .init_byteswap = fletcher_4_scalar_init, .fini_byteswap = fletcher_4_scalar_fini, .compute_byteswap = fletcher_4_scalar_byteswap, .valid = fletcher_4_scalar_valid, .name = "scalar" }; static fletcher_4_ops_t fletcher_4_fastest_impl = { .name = "fastest", .valid = fletcher_4_scalar_valid }; static const fletcher_4_ops_t *fletcher_4_impls[] = { &fletcher_4_scalar_ops, &fletcher_4_superscalar_ops, &fletcher_4_superscalar4_ops, #if defined(HAVE_SSE2) &fletcher_4_sse2_ops, #endif #if defined(HAVE_SSE2) && defined(HAVE_SSSE3) &fletcher_4_ssse3_ops, #endif #if defined(HAVE_AVX) && defined(HAVE_AVX2) &fletcher_4_avx2_ops, #endif #if defined(__x86_64) && defined(HAVE_AVX512F) &fletcher_4_avx512f_ops, #endif #if defined(__aarch64__) &fletcher_4_aarch64_neon_ops, #endif }; /* Hold all supported implementations */ static uint32_t fletcher_4_supp_impls_cnt = 0; static fletcher_4_ops_t *fletcher_4_supp_impls[ARRAY_SIZE(fletcher_4_impls)]; /* Select fletcher4 implementation */ #define IMPL_FASTEST (UINT32_MAX) #define IMPL_CYCLE (UINT32_MAX - 1) #define IMPL_SCALAR (0) static uint32_t fletcher_4_impl_chosen = IMPL_FASTEST; #define IMPL_READ(i) (*(volatile uint32_t *) &(i)) static struct fletcher_4_impl_selector { const char *fis_name; uint32_t fis_sel; } fletcher_4_impl_selectors[] = { { "cycle", IMPL_CYCLE }, { "fastest", IMPL_FASTEST }, { "scalar", IMPL_SCALAR } }; #if defined(_KERNEL) static kstat_t *fletcher_4_kstat; static struct fletcher_4_kstat { uint64_t native; uint64_t byteswap; } fletcher_4_stat_data[ARRAY_SIZE(fletcher_4_impls) + 1]; #endif /* Indicate that benchmark has been completed */ static boolean_t fletcher_4_initialized = B_FALSE; /*ARGSUSED*/ void fletcher_init(zio_cksum_t *zcp) { ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); } int fletcher_2_incremental_native(void *buf, size_t size, void *data) { zio_cksum_t *zcp = data; const uint64_t *ip = buf; const uint64_t *ipend = ip + (size / sizeof (uint64_t)); uint64_t a0, b0, a1, b1; a0 = zcp->zc_word[0]; a1 = zcp->zc_word[1]; b0 = zcp->zc_word[2]; b1 = zcp->zc_word[3]; for (; ip < ipend; ip += 2) { a0 += ip[0]; a1 += ip[1]; b0 += a0; b1 += a1; } ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1); return (0); } /*ARGSUSED*/ void fletcher_2_native(const void *buf, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { fletcher_init(zcp); (void) fletcher_2_incremental_native((void *) buf, size, zcp); } int fletcher_2_incremental_byteswap(void *buf, size_t size, void *data) { zio_cksum_t *zcp = data; const uint64_t *ip = buf; const uint64_t *ipend = ip + (size / sizeof (uint64_t)); uint64_t a0, b0, a1, b1; a0 = zcp->zc_word[0]; a1 = zcp->zc_word[1]; b0 = zcp->zc_word[2]; b1 = zcp->zc_word[3]; for (; ip < ipend; ip += 2) { a0 += BSWAP_64(ip[0]); a1 += BSWAP_64(ip[1]); b0 += a0; b1 += a1; } ZIO_SET_CHECKSUM(zcp, a0, a1, b0, b1); return (0); } /*ARGSUSED*/ void fletcher_2_byteswap(const void *buf, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { fletcher_init(zcp); (void) fletcher_2_incremental_byteswap((void *) buf, size, zcp); } static void fletcher_4_scalar_init(fletcher_4_ctx_t *ctx) { ZIO_SET_CHECKSUM(&ctx->scalar, 0, 0, 0, 0); } static void fletcher_4_scalar_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) { memcpy(zcp, &ctx->scalar, sizeof (zio_cksum_t)); } static void fletcher_4_scalar_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { const uint32_t *ip = buf; const uint32_t *ipend = ip + (size / sizeof (uint32_t)); uint64_t a, b, c, d; a = ctx->scalar.zc_word[0]; b = ctx->scalar.zc_word[1]; c = ctx->scalar.zc_word[2]; d = ctx->scalar.zc_word[3]; for (; ip < ipend; ip++) { a += ip[0]; b += a; c += b; d += c; } ZIO_SET_CHECKSUM(&ctx->scalar, a, b, c, d); } static void fletcher_4_scalar_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { const uint32_t *ip = buf; const uint32_t *ipend = ip + (size / sizeof (uint32_t)); uint64_t a, b, c, d; a = ctx->scalar.zc_word[0]; b = ctx->scalar.zc_word[1]; c = ctx->scalar.zc_word[2]; d = ctx->scalar.zc_word[3]; for (; ip < ipend; ip++) { a += BSWAP_32(ip[0]); b += a; c += b; d += c; } ZIO_SET_CHECKSUM(&ctx->scalar, a, b, c, d); } static boolean_t fletcher_4_scalar_valid(void) { return (B_TRUE); } int fletcher_4_impl_set(const char *val) { int err = -EINVAL; uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); size_t i, val_len; val_len = strlen(val); while ((val_len > 0) && !!isspace(val[val_len-1])) /* trim '\n' */ val_len--; /* check mandatory implementations */ for (i = 0; i < ARRAY_SIZE(fletcher_4_impl_selectors); i++) { const char *name = fletcher_4_impl_selectors[i].fis_name; if (val_len == strlen(name) && strncmp(val, name, val_len) == 0) { impl = fletcher_4_impl_selectors[i].fis_sel; err = 0; break; } } if (err != 0 && fletcher_4_initialized) { /* check all supported implementations */ for (i = 0; i < fletcher_4_supp_impls_cnt; i++) { const char *name = fletcher_4_supp_impls[i]->name; if (val_len == strlen(name) && strncmp(val, name, val_len) == 0) { impl = i; err = 0; break; } } } if (err == 0) { atomic_swap_32(&fletcher_4_impl_chosen, impl); membar_producer(); } return (err); } /* * Returns the Fletcher 4 operations for checksums. When a SIMD * implementation is not allowed in the current context, then fallback * to the fastest generic implementation. */ static inline const fletcher_4_ops_t * fletcher_4_impl_get(void) { if (!kfpu_allowed()) return (&fletcher_4_superscalar4_ops); const fletcher_4_ops_t *ops = NULL; uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); switch (impl) { case IMPL_FASTEST: ASSERT(fletcher_4_initialized); ops = &fletcher_4_fastest_impl; break; case IMPL_CYCLE: /* Cycle through supported implementations */ ASSERT(fletcher_4_initialized); ASSERT3U(fletcher_4_supp_impls_cnt, >, 0); static uint32_t cycle_count = 0; uint32_t idx = (++cycle_count) % fletcher_4_supp_impls_cnt; ops = fletcher_4_supp_impls[idx]; break; default: ASSERT3U(fletcher_4_supp_impls_cnt, >, 0); ASSERT3U(impl, <, fletcher_4_supp_impls_cnt); ops = fletcher_4_supp_impls[impl]; break; } ASSERT3P(ops, !=, NULL); return (ops); } static inline void fletcher_4_native_impl(const void *buf, uint64_t size, zio_cksum_t *zcp) { fletcher_4_ctx_t ctx; const fletcher_4_ops_t *ops = fletcher_4_impl_get(); ops->init_native(&ctx); ops->compute_native(&ctx, buf, size); ops->fini_native(&ctx, zcp); } /*ARGSUSED*/ void fletcher_4_native(const void *buf, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { const uint64_t p2size = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE); ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); if (size == 0 || p2size == 0) { ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); if (size > 0) fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp, buf, size); } else { fletcher_4_native_impl(buf, p2size, zcp); if (p2size < size) fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp, (char *)buf + p2size, size - p2size); } } void fletcher_4_native_varsize(const void *buf, uint64_t size, zio_cksum_t *zcp) { ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp, buf, size); } static inline void fletcher_4_byteswap_impl(const void *buf, uint64_t size, zio_cksum_t *zcp) { fletcher_4_ctx_t ctx; const fletcher_4_ops_t *ops = fletcher_4_impl_get(); ops->init_byteswap(&ctx); ops->compute_byteswap(&ctx, buf, size); ops->fini_byteswap(&ctx, zcp); } /*ARGSUSED*/ void fletcher_4_byteswap(const void *buf, uint64_t size, const void *ctx_template, zio_cksum_t *zcp) { const uint64_t p2size = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE); ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); if (size == 0 || p2size == 0) { ZIO_SET_CHECKSUM(zcp, 0, 0, 0, 0); if (size > 0) fletcher_4_scalar_byteswap((fletcher_4_ctx_t *)zcp, buf, size); } else { fletcher_4_byteswap_impl(buf, p2size, zcp); if (p2size < size) fletcher_4_scalar_byteswap((fletcher_4_ctx_t *)zcp, (char *)buf + p2size, size - p2size); } } /* Incremental Fletcher 4 */ #define ZFS_FLETCHER_4_INC_MAX_SIZE (8ULL << 20) static inline void fletcher_4_incremental_combine(zio_cksum_t *zcp, const uint64_t size, const zio_cksum_t *nzcp) { const uint64_t c1 = size / sizeof (uint32_t); const uint64_t c2 = c1 * (c1 + 1) / 2; const uint64_t c3 = c2 * (c1 + 2) / 3; /* * Value of 'c3' overflows on buffer sizes close to 16MiB. For that * reason we split incremental fletcher4 computation of large buffers * to steps of (ZFS_FLETCHER_4_INC_MAX_SIZE) size. */ ASSERT3U(size, <=, ZFS_FLETCHER_4_INC_MAX_SIZE); zcp->zc_word[3] += nzcp->zc_word[3] + c1 * zcp->zc_word[2] + c2 * zcp->zc_word[1] + c3 * zcp->zc_word[0]; zcp->zc_word[2] += nzcp->zc_word[2] + c1 * zcp->zc_word[1] + c2 * zcp->zc_word[0]; zcp->zc_word[1] += nzcp->zc_word[1] + c1 * zcp->zc_word[0]; zcp->zc_word[0] += nzcp->zc_word[0]; } static inline void fletcher_4_incremental_impl(boolean_t native, const void *buf, uint64_t size, zio_cksum_t *zcp) { while (size > 0) { zio_cksum_t nzc; uint64_t len = MIN(size, ZFS_FLETCHER_4_INC_MAX_SIZE); if (native) fletcher_4_native(buf, len, NULL, &nzc); else fletcher_4_byteswap(buf, len, NULL, &nzc); fletcher_4_incremental_combine(zcp, len, &nzc); size -= len; buf += len; } } int fletcher_4_incremental_native(void *buf, size_t size, void *data) { zio_cksum_t *zcp = data; /* Use scalar impl to directly update cksum of small blocks */ if (size < SPA_MINBLOCKSIZE) fletcher_4_scalar_native((fletcher_4_ctx_t *)zcp, buf, size); else fletcher_4_incremental_impl(B_TRUE, buf, size, zcp); return (0); } int fletcher_4_incremental_byteswap(void *buf, size_t size, void *data) { zio_cksum_t *zcp = data; /* Use scalar impl to directly update cksum of small blocks */ if (size < SPA_MINBLOCKSIZE) fletcher_4_scalar_byteswap((fletcher_4_ctx_t *)zcp, buf, size); else fletcher_4_incremental_impl(B_FALSE, buf, size, zcp); return (0); } #if defined(_KERNEL) /* * Fletcher 4 kstats */ static int fletcher_4_kstat_headers(char *buf, size_t size) { ssize_t off = 0; off += snprintf(buf + off, size, "%-17s", "implementation"); off += snprintf(buf + off, size - off, "%-15s", "native"); (void) snprintf(buf + off, size - off, "%-15s\n", "byteswap"); return (0); } static int fletcher_4_kstat_data(char *buf, size_t size, void *data) { struct fletcher_4_kstat *fastest_stat = &fletcher_4_stat_data[fletcher_4_supp_impls_cnt]; struct fletcher_4_kstat *curr_stat = (struct fletcher_4_kstat *)data; ssize_t off = 0; if (curr_stat == fastest_stat) { off += snprintf(buf + off, size - off, "%-17s", "fastest"); off += snprintf(buf + off, size - off, "%-15s", fletcher_4_supp_impls[fastest_stat->native]->name); off += snprintf(buf + off, size - off, "%-15s\n", fletcher_4_supp_impls[fastest_stat->byteswap]->name); } else { ptrdiff_t id = curr_stat - fletcher_4_stat_data; off += snprintf(buf + off, size - off, "%-17s", fletcher_4_supp_impls[id]->name); off += snprintf(buf + off, size - off, "%-15llu", (u_longlong_t)curr_stat->native); off += snprintf(buf + off, size - off, "%-15llu\n", (u_longlong_t)curr_stat->byteswap); } return (0); } static void * fletcher_4_kstat_addr(kstat_t *ksp, loff_t n) { if (n <= fletcher_4_supp_impls_cnt) ksp->ks_private = (void *) (fletcher_4_stat_data + n); else ksp->ks_private = NULL; return (ksp->ks_private); } #endif #define FLETCHER_4_FASTEST_FN_COPY(type, src) \ { \ fletcher_4_fastest_impl.init_ ## type = src->init_ ## type; \ fletcher_4_fastest_impl.fini_ ## type = src->fini_ ## type; \ fletcher_4_fastest_impl.compute_ ## type = src->compute_ ## type; \ } #define FLETCHER_4_BENCH_NS (MSEC2NSEC(50)) /* 50ms */ typedef void fletcher_checksum_func_t(const void *, uint64_t, const void *, zio_cksum_t *); #if defined(_KERNEL) static void fletcher_4_benchmark_impl(boolean_t native, char *data, uint64_t data_size) { struct fletcher_4_kstat *fastest_stat = &fletcher_4_stat_data[fletcher_4_supp_impls_cnt]; hrtime_t start; uint64_t run_bw, run_time_ns, best_run = 0; zio_cksum_t zc; uint32_t i, l, sel_save = IMPL_READ(fletcher_4_impl_chosen); fletcher_checksum_func_t *fletcher_4_test = native ? fletcher_4_native : fletcher_4_byteswap; for (i = 0; i < fletcher_4_supp_impls_cnt; i++) { struct fletcher_4_kstat *stat = &fletcher_4_stat_data[i]; uint64_t run_count = 0; /* temporary set an implementation */ fletcher_4_impl_chosen = i; kpreempt_disable(); start = gethrtime(); do { for (l = 0; l < 32; l++, run_count++) fletcher_4_test(data, data_size, NULL, &zc); run_time_ns = gethrtime() - start; } while (run_time_ns < FLETCHER_4_BENCH_NS); kpreempt_enable(); run_bw = data_size * run_count * NANOSEC; run_bw /= run_time_ns; /* B/s */ if (native) stat->native = run_bw; else stat->byteswap = run_bw; if (run_bw > best_run) { best_run = run_bw; if (native) { fastest_stat->native = i; FLETCHER_4_FASTEST_FN_COPY(native, fletcher_4_supp_impls[i]); } else { fastest_stat->byteswap = i; FLETCHER_4_FASTEST_FN_COPY(byteswap, fletcher_4_supp_impls[i]); } } } /* restore original selection */ atomic_swap_32(&fletcher_4_impl_chosen, sel_save); } #endif /* _KERNEL */ /* * Initialize and benchmark all supported implementations. */ static void fletcher_4_benchmark(void *arg) { fletcher_4_ops_t *curr_impl; int i, c; /* Move supported implementations into fletcher_4_supp_impls */ for (i = 0, c = 0; i < ARRAY_SIZE(fletcher_4_impls); i++) { curr_impl = (fletcher_4_ops_t *)fletcher_4_impls[i]; if (curr_impl->valid && curr_impl->valid()) fletcher_4_supp_impls[c++] = curr_impl; } membar_producer(); /* complete fletcher_4_supp_impls[] init */ fletcher_4_supp_impls_cnt = c; /* number of supported impl */ #if defined(_KERNEL) static const size_t data_size = 1 << SPA_OLD_MAXBLOCKSHIFT; /* 128kiB */ char *databuf = vmem_alloc(data_size, KM_SLEEP); for (i = 0; i < data_size / sizeof (uint64_t); i++) ((uint64_t *)databuf)[i] = (uintptr_t)(databuf+i); /* warm-up */ fletcher_4_benchmark_impl(B_FALSE, databuf, data_size); fletcher_4_benchmark_impl(B_TRUE, databuf, data_size); vmem_free(databuf, data_size); #else /* * Skip the benchmark in user space to avoid impacting libzpool * consumers (zdb, zhack, zinject, ztest). The last implementation * is assumed to be the fastest and used by default. */ memcpy(&fletcher_4_fastest_impl, fletcher_4_supp_impls[fletcher_4_supp_impls_cnt - 1], sizeof (fletcher_4_fastest_impl)); fletcher_4_fastest_impl.name = "fastest"; membar_producer(); #endif /* _KERNEL */ } void fletcher_4_init(void) { #if defined(_KERNEL) /* * For 5.0 and latter Linux kernels the fletcher 4 benchmarks are * run in a kernel threads. This is needed to take advantage of the - * SIMD functionality, see include/linux/simd_x86.h for details. + * SIMD functionality, see linux/simd_x86.h for details. */ taskqid_t id = taskq_dispatch(system_taskq, fletcher_4_benchmark, NULL, TQ_SLEEP); if (id != TASKQID_INVALID) { taskq_wait_id(system_taskq, id); } else { fletcher_4_benchmark(NULL); } /* Install kstats for all implementations */ fletcher_4_kstat = kstat_create("zfs", 0, "fletcher_4_bench", "misc", KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); if (fletcher_4_kstat != NULL) { fletcher_4_kstat->ks_data = NULL; fletcher_4_kstat->ks_ndata = UINT32_MAX; kstat_set_raw_ops(fletcher_4_kstat, fletcher_4_kstat_headers, fletcher_4_kstat_data, fletcher_4_kstat_addr); kstat_install(fletcher_4_kstat); } #else fletcher_4_benchmark(NULL); #endif /* Finish initialization */ fletcher_4_initialized = B_TRUE; } void fletcher_4_fini(void) { #if defined(_KERNEL) if (fletcher_4_kstat != NULL) { kstat_delete(fletcher_4_kstat); fletcher_4_kstat = NULL; } #endif } /* ABD adapters */ static void abd_fletcher_4_init(zio_abd_checksum_data_t *cdp) { const fletcher_4_ops_t *ops = fletcher_4_impl_get(); cdp->acd_private = (void *) ops; if (cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE) ops->init_native(cdp->acd_ctx); else ops->init_byteswap(cdp->acd_ctx); } static void abd_fletcher_4_fini(zio_abd_checksum_data_t *cdp) { fletcher_4_ops_t *ops = (fletcher_4_ops_t *)cdp->acd_private; ASSERT(ops); if (cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE) ops->fini_native(cdp->acd_ctx, cdp->acd_zcp); else ops->fini_byteswap(cdp->acd_ctx, cdp->acd_zcp); } static void abd_fletcher_4_simd2scalar(boolean_t native, void *data, size_t size, zio_abd_checksum_data_t *cdp) { zio_cksum_t *zcp = cdp->acd_zcp; ASSERT3U(size, <, FLETCHER_MIN_SIMD_SIZE); abd_fletcher_4_fini(cdp); cdp->acd_private = (void *)&fletcher_4_scalar_ops; if (native) fletcher_4_incremental_native(data, size, zcp); else fletcher_4_incremental_byteswap(data, size, zcp); } static int abd_fletcher_4_iter(void *data, size_t size, void *private) { zio_abd_checksum_data_t *cdp = (zio_abd_checksum_data_t *)private; fletcher_4_ctx_t *ctx = cdp->acd_ctx; fletcher_4_ops_t *ops = (fletcher_4_ops_t *)cdp->acd_private; boolean_t native = cdp->acd_byteorder == ZIO_CHECKSUM_NATIVE; uint64_t asize = P2ALIGN(size, FLETCHER_MIN_SIMD_SIZE); ASSERT(IS_P2ALIGNED(size, sizeof (uint32_t))); if (asize > 0) { if (native) ops->compute_native(ctx, data, asize); else ops->compute_byteswap(ctx, data, asize); size -= asize; data = (char *)data + asize; } if (size > 0) { ASSERT3U(size, <, FLETCHER_MIN_SIMD_SIZE); /* At this point we have to switch to scalar impl */ abd_fletcher_4_simd2scalar(native, data, size, cdp); } return (0); } zio_abd_checksum_func_t fletcher_4_abd_ops = { .acf_init = abd_fletcher_4_init, .acf_fini = abd_fletcher_4_fini, .acf_iter = abd_fletcher_4_iter }; #if defined(_KERNEL) #include static int fletcher_4_param_get(char *buffer, zfs_kernel_param_t *unused) { const uint32_t impl = IMPL_READ(fletcher_4_impl_chosen); char *fmt; int i, cnt = 0; /* list fastest */ fmt = (impl == IMPL_FASTEST) ? "[%s] " : "%s "; cnt += sprintf(buffer + cnt, fmt, "fastest"); /* list all supported implementations */ for (i = 0; i < fletcher_4_supp_impls_cnt; i++) { fmt = (i == impl) ? "[%s] " : "%s "; cnt += sprintf(buffer + cnt, fmt, fletcher_4_supp_impls[i]->name); } return (cnt); } static int fletcher_4_param_set(const char *val, zfs_kernel_param_t *unused) { return (fletcher_4_impl_set(val)); } /* * Choose a fletcher 4 implementation in ZFS. * Users can choose "cycle" to exercise all implementations, but this is * for testing purpose therefore it can only be set in user space. */ module_param_call(zfs_fletcher_4_impl, fletcher_4_param_set, fletcher_4_param_get, NULL, 0644); MODULE_PARM_DESC(zfs_fletcher_4_impl, "Select fletcher 4 implementation."); EXPORT_SYMBOL(fletcher_init); EXPORT_SYMBOL(fletcher_2_incremental_native); EXPORT_SYMBOL(fletcher_2_incremental_byteswap); EXPORT_SYMBOL(fletcher_4_init); EXPORT_SYMBOL(fletcher_4_fini); EXPORT_SYMBOL(fletcher_2_native); EXPORT_SYMBOL(fletcher_2_byteswap); EXPORT_SYMBOL(fletcher_4_native); EXPORT_SYMBOL(fletcher_4_native_varsize); EXPORT_SYMBOL(fletcher_4_byteswap); EXPORT_SYMBOL(fletcher_4_incremental_native); EXPORT_SYMBOL(fletcher_4_incremental_byteswap); EXPORT_SYMBOL(fletcher_4_abd_ops); #endif diff --git a/module/zcommon/zfs_fletcher_aarch64_neon.c b/module/zcommon/zfs_fletcher_aarch64_neon.c index 3b3c1b52b804..c95a71681584 100644 --- a/module/zcommon/zfs_fletcher_aarch64_neon.c +++ b/module/zcommon/zfs_fletcher_aarch64_neon.c @@ -1,215 +1,215 @@ /* * Implement fast Fletcher4 with NEON instructions. (aarch64) * * Use the 128-bit NEON SIMD instructions and registers to compute * Fletcher4 in two incremental 64-bit parallel accumulator streams, * and then combine the streams to form the final four checksum words. * This implementation is a derivative of the AVX SIMD implementation by * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c). * * Copyright (C) 2016 Romain Dolbeau. * * Authors: * Romain Dolbeau * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if defined(__aarch64__) -#include +#include #include #include #include static void fletcher_4_aarch64_neon_init(fletcher_4_ctx_t *ctx) { bzero(ctx->aarch64_neon, 4 * sizeof (zfs_fletcher_aarch64_neon_t)); } static void fletcher_4_aarch64_neon_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) { uint64_t A, B, C, D; A = ctx->aarch64_neon[0].v[0] + ctx->aarch64_neon[0].v[1]; B = 2 * ctx->aarch64_neon[1].v[0] + 2 * ctx->aarch64_neon[1].v[1] - ctx->aarch64_neon[0].v[1]; C = 4 * ctx->aarch64_neon[2].v[0] - ctx->aarch64_neon[1].v[0] + 4 * ctx->aarch64_neon[2].v[1] - 3 * ctx->aarch64_neon[1].v[1]; D = 8 * ctx->aarch64_neon[3].v[0] - 4 * ctx->aarch64_neon[2].v[0] + 8 * ctx->aarch64_neon[3].v[1] - 8 * ctx->aarch64_neon[2].v[1] + ctx->aarch64_neon[1].v[1]; ZIO_SET_CHECKSUM(zcp, A, B, C, D); } #define NEON_INIT_LOOP() \ asm("eor %[ZERO].16b,%[ZERO].16b,%[ZERO].16b\n" \ "ld1 { %[ACC0].4s }, %[CTX0]\n" \ "ld1 { %[ACC1].4s }, %[CTX1]\n" \ "ld1 { %[ACC2].4s }, %[CTX2]\n" \ "ld1 { %[ACC3].4s }, %[CTX3]\n" \ : [ZERO] "=w" (ZERO), \ [ACC0] "=w" (ACC0), [ACC1] "=w" (ACC1), \ [ACC2] "=w" (ACC2), [ACC3] "=w" (ACC3) \ : [CTX0] "Q" (ctx->aarch64_neon[0]), \ [CTX1] "Q" (ctx->aarch64_neon[1]), \ [CTX2] "Q" (ctx->aarch64_neon[2]), \ [CTX3] "Q" (ctx->aarch64_neon[3])) #define NEON_DO_REVERSE "rev32 %[SRC].16b, %[SRC].16b\n" #define NEON_DONT_REVERSE "" #define NEON_MAIN_LOOP(REVERSE) \ asm("ld1 { %[SRC].4s }, %[IP]\n" \ REVERSE \ "zip1 %[TMP1].4s, %[SRC].4s, %[ZERO].4s\n" \ "zip2 %[TMP2].4s, %[SRC].4s, %[ZERO].4s\n" \ "add %[ACC0].2d, %[ACC0].2d, %[TMP1].2d\n" \ "add %[ACC1].2d, %[ACC1].2d, %[ACC0].2d\n" \ "add %[ACC2].2d, %[ACC2].2d, %[ACC1].2d\n" \ "add %[ACC3].2d, %[ACC3].2d, %[ACC2].2d\n" \ "add %[ACC0].2d, %[ACC0].2d, %[TMP2].2d\n" \ "add %[ACC1].2d, %[ACC1].2d, %[ACC0].2d\n" \ "add %[ACC2].2d, %[ACC2].2d, %[ACC1].2d\n" \ "add %[ACC3].2d, %[ACC3].2d, %[ACC2].2d\n" \ : [SRC] "=&w" (SRC), \ [TMP1] "=&w" (TMP1), [TMP2] "=&w" (TMP2), \ [ACC0] "+w" (ACC0), [ACC1] "+w" (ACC1), \ [ACC2] "+w" (ACC2), [ACC3] "+w" (ACC3) \ : [ZERO] "w" (ZERO), [IP] "Q" (*ip)) #define NEON_FINI_LOOP() \ asm("st1 { %[ACC0].4s },%[DST0]\n" \ "st1 { %[ACC1].4s },%[DST1]\n" \ "st1 { %[ACC2].4s },%[DST2]\n" \ "st1 { %[ACC3].4s },%[DST3]\n" \ : [DST0] "=Q" (ctx->aarch64_neon[0]), \ [DST1] "=Q" (ctx->aarch64_neon[1]), \ [DST2] "=Q" (ctx->aarch64_neon[2]), \ [DST3] "=Q" (ctx->aarch64_neon[3]) \ : [ACC0] "w" (ACC0), [ACC1] "w" (ACC1), \ [ACC2] "w" (ACC2), [ACC3] "w" (ACC3)) static void fletcher_4_aarch64_neon_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); #if defined(_KERNEL) register unsigned char ZERO asm("v0") __attribute__((vector_size(16))); register unsigned char ACC0 asm("v1") __attribute__((vector_size(16))); register unsigned char ACC1 asm("v2") __attribute__((vector_size(16))); register unsigned char ACC2 asm("v3") __attribute__((vector_size(16))); register unsigned char ACC3 asm("v4") __attribute__((vector_size(16))); register unsigned char TMP1 asm("v5") __attribute__((vector_size(16))); register unsigned char TMP2 asm("v6") __attribute__((vector_size(16))); register unsigned char SRC asm("v7") __attribute__((vector_size(16))); #else unsigned char ZERO __attribute__((vector_size(16))); unsigned char ACC0 __attribute__((vector_size(16))); unsigned char ACC1 __attribute__((vector_size(16))); unsigned char ACC2 __attribute__((vector_size(16))); unsigned char ACC3 __attribute__((vector_size(16))); unsigned char TMP1 __attribute__((vector_size(16))); unsigned char TMP2 __attribute__((vector_size(16))); unsigned char SRC __attribute__((vector_size(16))); #endif kfpu_begin(); NEON_INIT_LOOP(); for (; ip < ipend; ip += 2) { NEON_MAIN_LOOP(NEON_DONT_REVERSE); } NEON_FINI_LOOP(); kfpu_end(); } static void fletcher_4_aarch64_neon_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); #if defined(_KERNEL) register unsigned char ZERO asm("v0") __attribute__((vector_size(16))); register unsigned char ACC0 asm("v1") __attribute__((vector_size(16))); register unsigned char ACC1 asm("v2") __attribute__((vector_size(16))); register unsigned char ACC2 asm("v3") __attribute__((vector_size(16))); register unsigned char ACC3 asm("v4") __attribute__((vector_size(16))); register unsigned char TMP1 asm("v5") __attribute__((vector_size(16))); register unsigned char TMP2 asm("v6") __attribute__((vector_size(16))); register unsigned char SRC asm("v7") __attribute__((vector_size(16))); #else unsigned char ZERO __attribute__((vector_size(16))); unsigned char ACC0 __attribute__((vector_size(16))); unsigned char ACC1 __attribute__((vector_size(16))); unsigned char ACC2 __attribute__((vector_size(16))); unsigned char ACC3 __attribute__((vector_size(16))); unsigned char TMP1 __attribute__((vector_size(16))); unsigned char TMP2 __attribute__((vector_size(16))); unsigned char SRC __attribute__((vector_size(16))); #endif kfpu_begin(); NEON_INIT_LOOP(); for (; ip < ipend; ip += 2) { NEON_MAIN_LOOP(NEON_DO_REVERSE); } NEON_FINI_LOOP(); kfpu_end(); } static boolean_t fletcher_4_aarch64_neon_valid(void) { return (kfpu_allowed()); } const fletcher_4_ops_t fletcher_4_aarch64_neon_ops = { .init_native = fletcher_4_aarch64_neon_init, .compute_native = fletcher_4_aarch64_neon_native, .fini_native = fletcher_4_aarch64_neon_fini, .init_byteswap = fletcher_4_aarch64_neon_init, .compute_byteswap = fletcher_4_aarch64_neon_byteswap, .fini_byteswap = fletcher_4_aarch64_neon_fini, .valid = fletcher_4_aarch64_neon_valid, .name = "aarch64_neon" }; #endif /* defined(__aarch64__) */ diff --git a/module/zcommon/zfs_fletcher_avx512.c b/module/zcommon/zfs_fletcher_avx512.c index 0d4cff21a506..43806f264e5e 100644 --- a/module/zcommon/zfs_fletcher_avx512.c +++ b/module/zcommon/zfs_fletcher_avx512.c @@ -1,174 +1,174 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Gvozden NeÅ¡ković. All rights reserved. */ #if defined(__x86_64) && defined(HAVE_AVX512F) -#include #include #include #include #include +#include #include #define __asm __asm__ __volatile__ static void fletcher_4_avx512f_init(fletcher_4_ctx_t *ctx) { bzero(ctx->avx512, 4 * sizeof (zfs_fletcher_avx512_t)); } static void fletcher_4_avx512f_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) { static const uint64_t CcA[] = { 0, 0, 1, 3, 6, 10, 15, 21 }, CcB[] = { 28, 36, 44, 52, 60, 68, 76, 84 }, DcA[] = { 0, 0, 0, 1, 4, 10, 20, 35 }, DcB[] = { 56, 84, 120, 164, 216, 276, 344, 420 }, DcC[] = { 448, 512, 576, 640, 704, 768, 832, 896 }; uint64_t A, B, C, D; uint64_t i; A = ctx->avx512[0].v[0]; B = 8 * ctx->avx512[1].v[0]; C = 64 * ctx->avx512[2].v[0] - CcB[0] * ctx->avx512[1].v[0]; D = 512 * ctx->avx512[3].v[0] - DcC[0] * ctx->avx512[2].v[0] + DcB[0] * ctx->avx512[1].v[0]; for (i = 1; i < 8; i++) { A += ctx->avx512[0].v[i]; B += 8 * ctx->avx512[1].v[i] - i * ctx->avx512[0].v[i]; C += 64 * ctx->avx512[2].v[i] - CcB[i] * ctx->avx512[1].v[i] + CcA[i] * ctx->avx512[0].v[i]; D += 512 * ctx->avx512[3].v[i] - DcC[i] * ctx->avx512[2].v[i] + DcB[i] * ctx->avx512[1].v[i] - DcA[i] * ctx->avx512[0].v[i]; } ZIO_SET_CHECKSUM(zcp, A, B, C, D); } #define FLETCHER_4_AVX512_RESTORE_CTX(ctx) \ { \ __asm("vmovdqu64 %0, %%zmm0" :: "m" ((ctx)->avx512[0])); \ __asm("vmovdqu64 %0, %%zmm1" :: "m" ((ctx)->avx512[1])); \ __asm("vmovdqu64 %0, %%zmm2" :: "m" ((ctx)->avx512[2])); \ __asm("vmovdqu64 %0, %%zmm3" :: "m" ((ctx)->avx512[3])); \ } #define FLETCHER_4_AVX512_SAVE_CTX(ctx) \ { \ __asm("vmovdqu64 %%zmm0, %0" : "=m" ((ctx)->avx512[0])); \ __asm("vmovdqu64 %%zmm1, %0" : "=m" ((ctx)->avx512[1])); \ __asm("vmovdqu64 %%zmm2, %0" : "=m" ((ctx)->avx512[2])); \ __asm("vmovdqu64 %%zmm3, %0" : "=m" ((ctx)->avx512[3])); \ } static void fletcher_4_avx512f_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { const uint32_t *ip = buf; const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); kfpu_begin(); FLETCHER_4_AVX512_RESTORE_CTX(ctx); for (; ip < ipend; ip += 8) { __asm("vpmovzxdq %0, %%zmm4"::"m" (*ip)); __asm("vpaddq %zmm4, %zmm0, %zmm0"); __asm("vpaddq %zmm0, %zmm1, %zmm1"); __asm("vpaddq %zmm1, %zmm2, %zmm2"); __asm("vpaddq %zmm2, %zmm3, %zmm3"); } FLETCHER_4_AVX512_SAVE_CTX(ctx); kfpu_end(); } STACK_FRAME_NON_STANDARD(fletcher_4_avx512f_native); static void fletcher_4_avx512f_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { static const uint64_t byteswap_mask = 0xFFULL; const uint32_t *ip = buf; const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); kfpu_begin(); FLETCHER_4_AVX512_RESTORE_CTX(ctx); __asm("vpbroadcastq %0, %%zmm8" :: "r" (byteswap_mask)); __asm("vpsllq $8, %zmm8, %zmm9"); __asm("vpsllq $16, %zmm8, %zmm10"); __asm("vpsllq $24, %zmm8, %zmm11"); for (; ip < ipend; ip += 8) { __asm("vpmovzxdq %0, %%zmm5"::"m" (*ip)); __asm("vpsrlq $24, %zmm5, %zmm6"); __asm("vpandd %zmm8, %zmm6, %zmm6"); __asm("vpsrlq $8, %zmm5, %zmm7"); __asm("vpandd %zmm9, %zmm7, %zmm7"); __asm("vpord %zmm6, %zmm7, %zmm4"); __asm("vpsllq $8, %zmm5, %zmm6"); __asm("vpandd %zmm10, %zmm6, %zmm6"); __asm("vpord %zmm6, %zmm4, %zmm4"); __asm("vpsllq $24, %zmm5, %zmm5"); __asm("vpandd %zmm11, %zmm5, %zmm5"); __asm("vpord %zmm5, %zmm4, %zmm4"); __asm("vpaddq %zmm4, %zmm0, %zmm0"); __asm("vpaddq %zmm0, %zmm1, %zmm1"); __asm("vpaddq %zmm1, %zmm2, %zmm2"); __asm("vpaddq %zmm2, %zmm3, %zmm3"); } FLETCHER_4_AVX512_SAVE_CTX(ctx) kfpu_end(); } STACK_FRAME_NON_STANDARD(fletcher_4_avx512f_byteswap); static boolean_t fletcher_4_avx512f_valid(void) { return (kfpu_allowed() && zfs_avx512f_available()); } const fletcher_4_ops_t fletcher_4_avx512f_ops = { .init_native = fletcher_4_avx512f_init, .fini_native = fletcher_4_avx512f_fini, .compute_native = fletcher_4_avx512f_native, .init_byteswap = fletcher_4_avx512f_init, .fini_byteswap = fletcher_4_avx512f_fini, .compute_byteswap = fletcher_4_avx512f_byteswap, .valid = fletcher_4_avx512f_valid, .name = "avx512f" }; #endif /* defined(__x86_64) && defined(HAVE_AVX512F) */ diff --git a/module/zcommon/zfs_fletcher_intel.c b/module/zcommon/zfs_fletcher_intel.c index 7f12efe6d8c5..5136a01eca51 100644 --- a/module/zcommon/zfs_fletcher_intel.c +++ b/module/zcommon/zfs_fletcher_intel.c @@ -1,173 +1,173 @@ /* * Implement fast Fletcher4 with AVX2 instructions. (x86_64) * * Use the 256-bit AVX2 SIMD instructions and registers to compute * Fletcher4 in four incremental 64-bit parallel accumulator streams, * and then combine the streams to form the final four checksum words. * * Copyright (C) 2015 Intel Corporation. * * Authors: * James Guilford * Jinshan Xiong * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if defined(HAVE_AVX) && defined(HAVE_AVX2) -#include #include +#include #include #include static void fletcher_4_avx2_init(fletcher_4_ctx_t *ctx) { bzero(ctx->avx, 4 * sizeof (zfs_fletcher_avx_t)); } static void fletcher_4_avx2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) { uint64_t A, B, C, D; A = ctx->avx[0].v[0] + ctx->avx[0].v[1] + ctx->avx[0].v[2] + ctx->avx[0].v[3]; B = 0 - ctx->avx[0].v[1] - 2 * ctx->avx[0].v[2] - 3 * ctx->avx[0].v[3] + 4 * ctx->avx[1].v[0] + 4 * ctx->avx[1].v[1] + 4 * ctx->avx[1].v[2] + 4 * ctx->avx[1].v[3]; C = ctx->avx[0].v[2] + 3 * ctx->avx[0].v[3] - 6 * ctx->avx[1].v[0] - 10 * ctx->avx[1].v[1] - 14 * ctx->avx[1].v[2] - 18 * ctx->avx[1].v[3] + 16 * ctx->avx[2].v[0] + 16 * ctx->avx[2].v[1] + 16 * ctx->avx[2].v[2] + 16 * ctx->avx[2].v[3]; D = 0 - ctx->avx[0].v[3] + 4 * ctx->avx[1].v[0] + 10 * ctx->avx[1].v[1] + 20 * ctx->avx[1].v[2] + 34 * ctx->avx[1].v[3] - 48 * ctx->avx[2].v[0] - 64 * ctx->avx[2].v[1] - 80 * ctx->avx[2].v[2] - 96 * ctx->avx[2].v[3] + 64 * ctx->avx[3].v[0] + 64 * ctx->avx[3].v[1] + 64 * ctx->avx[3].v[2] + 64 * ctx->avx[3].v[3]; ZIO_SET_CHECKSUM(zcp, A, B, C, D); } #define FLETCHER_4_AVX2_RESTORE_CTX(ctx) \ { \ asm volatile("vmovdqu %0, %%ymm0" :: "m" ((ctx)->avx[0])); \ asm volatile("vmovdqu %0, %%ymm1" :: "m" ((ctx)->avx[1])); \ asm volatile("vmovdqu %0, %%ymm2" :: "m" ((ctx)->avx[2])); \ asm volatile("vmovdqu %0, %%ymm3" :: "m" ((ctx)->avx[3])); \ } #define FLETCHER_4_AVX2_SAVE_CTX(ctx) \ { \ asm volatile("vmovdqu %%ymm0, %0" : "=m" ((ctx)->avx[0])); \ asm volatile("vmovdqu %%ymm1, %0" : "=m" ((ctx)->avx[1])); \ asm volatile("vmovdqu %%ymm2, %0" : "=m" ((ctx)->avx[2])); \ asm volatile("vmovdqu %%ymm3, %0" : "=m" ((ctx)->avx[3])); \ } static void fletcher_4_avx2_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); kfpu_begin(); FLETCHER_4_AVX2_RESTORE_CTX(ctx); for (; ip < ipend; ip += 2) { asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip)); asm volatile("vpaddq %ymm4, %ymm0, %ymm0"); asm volatile("vpaddq %ymm0, %ymm1, %ymm1"); asm volatile("vpaddq %ymm1, %ymm2, %ymm2"); asm volatile("vpaddq %ymm2, %ymm3, %ymm3"); } FLETCHER_4_AVX2_SAVE_CTX(ctx); asm volatile("vzeroupper"); kfpu_end(); } static void fletcher_4_avx2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { static const zfs_fletcher_avx_t mask = { .v = { 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B, 0xFFFFFFFF00010203, 0xFFFFFFFF08090A0B } }; const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); kfpu_begin(); FLETCHER_4_AVX2_RESTORE_CTX(ctx); asm volatile("vmovdqu %0, %%ymm5" :: "m" (mask)); for (; ip < ipend; ip += 2) { asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip)); asm volatile("vpshufb %ymm5, %ymm4, %ymm4"); asm volatile("vpaddq %ymm4, %ymm0, %ymm0"); asm volatile("vpaddq %ymm0, %ymm1, %ymm1"); asm volatile("vpaddq %ymm1, %ymm2, %ymm2"); asm volatile("vpaddq %ymm2, %ymm3, %ymm3"); } FLETCHER_4_AVX2_SAVE_CTX(ctx); asm volatile("vzeroupper"); kfpu_end(); } static boolean_t fletcher_4_avx2_valid(void) { return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available()); } const fletcher_4_ops_t fletcher_4_avx2_ops = { .init_native = fletcher_4_avx2_init, .fini_native = fletcher_4_avx2_fini, .compute_native = fletcher_4_avx2_native, .init_byteswap = fletcher_4_avx2_init, .fini_byteswap = fletcher_4_avx2_fini, .compute_byteswap = fletcher_4_avx2_byteswap, .valid = fletcher_4_avx2_valid, .name = "avx2" }; #endif /* defined(HAVE_AVX) && defined(HAVE_AVX2) */ diff --git a/module/zcommon/zfs_fletcher_sse.c b/module/zcommon/zfs_fletcher_sse.c index e6389d6e5db8..15ce9b07ffbe 100644 --- a/module/zcommon/zfs_fletcher_sse.c +++ b/module/zcommon/zfs_fletcher_sse.c @@ -1,232 +1,232 @@ /* * Implement fast Fletcher4 with SSE2,SSSE3 instructions. (x86) * * Use the 128-bit SSE2/SSSE3 SIMD instructions and registers to compute * Fletcher4 in two incremental 64-bit parallel accumulator streams, * and then combine the streams to form the final four checksum words. * This implementation is a derivative of the AVX SIMD implementation by * James Guilford and Jinshan Xiong from Intel (see zfs_fletcher_intel.c). * * Copyright (C) 2016 Tyler J. Stachecki. * * Authors: * Tyler J. Stachecki * * This software is available to you under a choice of one of two * licenses. You may choose to be licensed under the terms of the GNU * General Public License (GPL) Version 2, available from the file * COPYING in the main directory of this source tree, or the * OpenIB.org BSD license below: * * Redistribution and use in source and binary forms, with or * without modification, are permitted provided that the following * conditions are met: * * - Redistributions of source code must retain the above * copyright notice, this list of conditions and the following * disclaimer. * * - Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if defined(HAVE_SSE2) -#include +#include #include #include #include #include static void fletcher_4_sse2_init(fletcher_4_ctx_t *ctx) { bzero(ctx->sse, 4 * sizeof (zfs_fletcher_sse_t)); } static void fletcher_4_sse2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) { uint64_t A, B, C, D; /* * The mixing matrix for checksum calculation is: * a = a0 + a1 * b = 2b0 + 2b1 - a1 * c = 4c0 - b0 + 4c1 -3b1 * d = 8d0 - 4c0 + 8d1 - 8c1 + b1; * * c and d are multiplied by 4 and 8, respectively, * before spilling the vectors out to memory. */ A = ctx->sse[0].v[0] + ctx->sse[0].v[1]; B = 2 * ctx->sse[1].v[0] + 2 * ctx->sse[1].v[1] - ctx->sse[0].v[1]; C = 4 * ctx->sse[2].v[0] - ctx->sse[1].v[0] + 4 * ctx->sse[2].v[1] - 3 * ctx->sse[1].v[1]; D = 8 * ctx->sse[3].v[0] - 4 * ctx->sse[2].v[0] + 8 * ctx->sse[3].v[1] - 8 * ctx->sse[2].v[1] + ctx->sse[1].v[1]; ZIO_SET_CHECKSUM(zcp, A, B, C, D); } #define FLETCHER_4_SSE_RESTORE_CTX(ctx) \ { \ asm volatile("movdqu %0, %%xmm0" :: "m" ((ctx)->sse[0])); \ asm volatile("movdqu %0, %%xmm1" :: "m" ((ctx)->sse[1])); \ asm volatile("movdqu %0, %%xmm2" :: "m" ((ctx)->sse[2])); \ asm volatile("movdqu %0, %%xmm3" :: "m" ((ctx)->sse[3])); \ } #define FLETCHER_4_SSE_SAVE_CTX(ctx) \ { \ asm volatile("movdqu %%xmm0, %0" : "=m" ((ctx)->sse[0])); \ asm volatile("movdqu %%xmm1, %0" : "=m" ((ctx)->sse[1])); \ asm volatile("movdqu %%xmm2, %0" : "=m" ((ctx)->sse[2])); \ asm volatile("movdqu %%xmm3, %0" : "=m" ((ctx)->sse[3])); \ } static void fletcher_4_sse2_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); kfpu_begin(); FLETCHER_4_SSE_RESTORE_CTX(ctx); asm volatile("pxor %xmm4, %xmm4"); for (; ip < ipend; ip += 2) { asm volatile("movdqu %0, %%xmm5" :: "m"(*ip)); asm volatile("movdqa %xmm5, %xmm6"); asm volatile("punpckldq %xmm4, %xmm5"); asm volatile("punpckhdq %xmm4, %xmm6"); asm volatile("paddq %xmm5, %xmm0"); asm volatile("paddq %xmm0, %xmm1"); asm volatile("paddq %xmm1, %xmm2"); asm volatile("paddq %xmm2, %xmm3"); asm volatile("paddq %xmm6, %xmm0"); asm volatile("paddq %xmm0, %xmm1"); asm volatile("paddq %xmm1, %xmm2"); asm volatile("paddq %xmm2, %xmm3"); } FLETCHER_4_SSE_SAVE_CTX(ctx); kfpu_end(); } static void fletcher_4_sse2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { const uint32_t *ip = buf; const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); kfpu_begin(); FLETCHER_4_SSE_RESTORE_CTX(ctx); for (; ip < ipend; ip += 2) { uint32_t scratch1 = BSWAP_32(ip[0]); uint32_t scratch2 = BSWAP_32(ip[1]); asm volatile("movd %0, %%xmm5" :: "r"(scratch1)); asm volatile("movd %0, %%xmm6" :: "r"(scratch2)); asm volatile("punpcklqdq %xmm6, %xmm5"); asm volatile("paddq %xmm5, %xmm0"); asm volatile("paddq %xmm0, %xmm1"); asm volatile("paddq %xmm1, %xmm2"); asm volatile("paddq %xmm2, %xmm3"); } FLETCHER_4_SSE_SAVE_CTX(ctx); kfpu_end(); } static boolean_t fletcher_4_sse2_valid(void) { return (kfpu_allowed() && zfs_sse2_available()); } const fletcher_4_ops_t fletcher_4_sse2_ops = { .init_native = fletcher_4_sse2_init, .fini_native = fletcher_4_sse2_fini, .compute_native = fletcher_4_sse2_native, .init_byteswap = fletcher_4_sse2_init, .fini_byteswap = fletcher_4_sse2_fini, .compute_byteswap = fletcher_4_sse2_byteswap, .valid = fletcher_4_sse2_valid, .name = "sse2" }; #endif /* defined(HAVE_SSE2) */ #if defined(HAVE_SSE2) && defined(HAVE_SSSE3) static void fletcher_4_ssse3_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { static const zfs_fletcher_sse_t mask = { .v = { 0x0405060700010203, 0x0C0D0E0F08090A0B } }; const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); kfpu_begin(); FLETCHER_4_SSE_RESTORE_CTX(ctx); asm volatile("movdqu %0, %%xmm7"::"m" (mask)); asm volatile("pxor %xmm4, %xmm4"); for (; ip < ipend; ip += 2) { asm volatile("movdqu %0, %%xmm5"::"m" (*ip)); asm volatile("pshufb %xmm7, %xmm5"); asm volatile("movdqa %xmm5, %xmm6"); asm volatile("punpckldq %xmm4, %xmm5"); asm volatile("punpckhdq %xmm4, %xmm6"); asm volatile("paddq %xmm5, %xmm0"); asm volatile("paddq %xmm0, %xmm1"); asm volatile("paddq %xmm1, %xmm2"); asm volatile("paddq %xmm2, %xmm3"); asm volatile("paddq %xmm6, %xmm0"); asm volatile("paddq %xmm0, %xmm1"); asm volatile("paddq %xmm1, %xmm2"); asm volatile("paddq %xmm2, %xmm3"); } FLETCHER_4_SSE_SAVE_CTX(ctx); kfpu_end(); } static boolean_t fletcher_4_ssse3_valid(void) { return (kfpu_allowed() && zfs_sse2_available() && zfs_ssse3_available()); } const fletcher_4_ops_t fletcher_4_ssse3_ops = { .init_native = fletcher_4_sse2_init, .fini_native = fletcher_4_sse2_fini, .compute_native = fletcher_4_sse2_native, .init_byteswap = fletcher_4_sse2_init, .fini_byteswap = fletcher_4_sse2_fini, .compute_byteswap = fletcher_4_ssse3_byteswap, .valid = fletcher_4_ssse3_valid, .name = "ssse3" }; #endif /* defined(HAVE_SSE2) && defined(HAVE_SSSE3) */ diff --git a/module/zfs/vdev_raidz_math.c b/module/zfs/vdev_raidz_math.c index ef514e9e102a..86d4aabdc032 100644 --- a/module/zfs/vdev_raidz_math.c +++ b/module/zfs/vdev_raidz_math.c @@ -1,678 +1,678 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Gvozden NeÅ¡ković. All rights reserved. */ #include #include #include #include #include #include #include -#include +#include extern boolean_t raidz_will_scalar_work(void); /* Opaque implementation with NULL methods to represent original methods */ static const raidz_impl_ops_t vdev_raidz_original_impl = { .name = "original", .is_supported = raidz_will_scalar_work, }; /* RAIDZ parity op that contain the fastest methods */ static raidz_impl_ops_t vdev_raidz_fastest_impl = { .name = "fastest" }; /* All compiled in implementations */ const raidz_impl_ops_t *raidz_all_maths[] = { &vdev_raidz_original_impl, &vdev_raidz_scalar_impl, #if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */ &vdev_raidz_sse2_impl, #endif #if defined(__x86_64) && defined(HAVE_SSSE3) /* only x86_64 for now */ &vdev_raidz_ssse3_impl, #endif #if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */ &vdev_raidz_avx2_impl, #endif #if defined(__x86_64) && defined(HAVE_AVX512F) /* only x86_64 for now */ &vdev_raidz_avx512f_impl, #endif #if defined(__x86_64) && defined(HAVE_AVX512BW) /* only x86_64 for now */ &vdev_raidz_avx512bw_impl, #endif #if defined(__aarch64__) &vdev_raidz_aarch64_neon_impl, &vdev_raidz_aarch64_neonx2_impl, #endif }; /* Indicate that benchmark has been completed */ static boolean_t raidz_math_initialized = B_FALSE; /* Select raidz implementation */ #define IMPL_FASTEST (UINT32_MAX) #define IMPL_CYCLE (UINT32_MAX - 1) #define IMPL_ORIGINAL (0) #define IMPL_SCALAR (1) #define RAIDZ_IMPL_READ(i) (*(volatile uint32_t *) &(i)) static uint32_t zfs_vdev_raidz_impl = IMPL_SCALAR; static uint32_t user_sel_impl = IMPL_FASTEST; /* Hold all supported implementations */ static size_t raidz_supp_impl_cnt = 0; static raidz_impl_ops_t *raidz_supp_impl[ARRAY_SIZE(raidz_all_maths)]; #if defined(_KERNEL) /* * kstats values for supported implementations * Values represent per disk throughput of 8 disk+parity raidz vdev [B/s] */ static raidz_impl_kstat_t raidz_impl_kstats[ARRAY_SIZE(raidz_all_maths) + 1]; /* kstat for benchmarked implementations */ static kstat_t *raidz_math_kstat = NULL; #endif /* * Returns the RAIDZ operations for raidz_map() parity calculations. When * a SIMD implementation is not allowed in the current context, then fallback * to the fastest generic implementation. */ const raidz_impl_ops_t * vdev_raidz_math_get_ops(void) { if (!kfpu_allowed()) return (&vdev_raidz_scalar_impl); raidz_impl_ops_t *ops = NULL; const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl); switch (impl) { case IMPL_FASTEST: ASSERT(raidz_math_initialized); ops = &vdev_raidz_fastest_impl; break; case IMPL_CYCLE: /* Cycle through all supported implementations */ ASSERT(raidz_math_initialized); ASSERT3U(raidz_supp_impl_cnt, >, 0); static size_t cycle_impl_idx = 0; size_t idx = (++cycle_impl_idx) % raidz_supp_impl_cnt; ops = raidz_supp_impl[idx]; break; case IMPL_ORIGINAL: ops = (raidz_impl_ops_t *)&vdev_raidz_original_impl; break; case IMPL_SCALAR: ops = (raidz_impl_ops_t *)&vdev_raidz_scalar_impl; break; default: ASSERT3U(impl, <, raidz_supp_impl_cnt); ASSERT3U(raidz_supp_impl_cnt, >, 0); if (impl < ARRAY_SIZE(raidz_all_maths)) ops = raidz_supp_impl[impl]; break; } ASSERT3P(ops, !=, NULL); return (ops); } /* * Select parity generation method for raidz_map */ int vdev_raidz_math_generate(raidz_map_t *rm) { raidz_gen_f gen_parity = NULL; switch (raidz_parity(rm)) { case 1: gen_parity = rm->rm_ops->gen[RAIDZ_GEN_P]; break; case 2: gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQ]; break; case 3: gen_parity = rm->rm_ops->gen[RAIDZ_GEN_PQR]; break; default: gen_parity = NULL; cmn_err(CE_PANIC, "invalid RAID-Z configuration %d", raidz_parity(rm)); break; } /* if method is NULL execute the original implementation */ if (gen_parity == NULL) return (RAIDZ_ORIGINAL_IMPL); gen_parity(rm); return (0); } static raidz_rec_f reconstruct_fun_p_sel(raidz_map_t *rm, const int *parity_valid, const int nbaddata) { if (nbaddata == 1 && parity_valid[CODE_P]) { return (rm->rm_ops->rec[RAIDZ_REC_P]); } return ((raidz_rec_f) NULL); } static raidz_rec_f reconstruct_fun_pq_sel(raidz_map_t *rm, const int *parity_valid, const int nbaddata) { if (nbaddata == 1) { if (parity_valid[CODE_P]) { return (rm->rm_ops->rec[RAIDZ_REC_P]); } else if (parity_valid[CODE_Q]) { return (rm->rm_ops->rec[RAIDZ_REC_Q]); } } else if (nbaddata == 2 && parity_valid[CODE_P] && parity_valid[CODE_Q]) { return (rm->rm_ops->rec[RAIDZ_REC_PQ]); } return ((raidz_rec_f) NULL); } static raidz_rec_f reconstruct_fun_pqr_sel(raidz_map_t *rm, const int *parity_valid, const int nbaddata) { if (nbaddata == 1) { if (parity_valid[CODE_P]) { return (rm->rm_ops->rec[RAIDZ_REC_P]); } else if (parity_valid[CODE_Q]) { return (rm->rm_ops->rec[RAIDZ_REC_Q]); } else if (parity_valid[CODE_R]) { return (rm->rm_ops->rec[RAIDZ_REC_R]); } } else if (nbaddata == 2) { if (parity_valid[CODE_P] && parity_valid[CODE_Q]) { return (rm->rm_ops->rec[RAIDZ_REC_PQ]); } else if (parity_valid[CODE_P] && parity_valid[CODE_R]) { return (rm->rm_ops->rec[RAIDZ_REC_PR]); } else if (parity_valid[CODE_Q] && parity_valid[CODE_R]) { return (rm->rm_ops->rec[RAIDZ_REC_QR]); } } else if (nbaddata == 3 && parity_valid[CODE_P] && parity_valid[CODE_Q] && parity_valid[CODE_R]) { return (rm->rm_ops->rec[RAIDZ_REC_PQR]); } return ((raidz_rec_f) NULL); } /* * Select data reconstruction method for raidz_map * @parity_valid - Parity validity flag * @dt - Failed data index array * @nbaddata - Number of failed data columns */ int vdev_raidz_math_reconstruct(raidz_map_t *rm, const int *parity_valid, const int *dt, const int nbaddata) { raidz_rec_f rec_fn = NULL; switch (raidz_parity(rm)) { case PARITY_P: rec_fn = reconstruct_fun_p_sel(rm, parity_valid, nbaddata); break; case PARITY_PQ: rec_fn = reconstruct_fun_pq_sel(rm, parity_valid, nbaddata); break; case PARITY_PQR: rec_fn = reconstruct_fun_pqr_sel(rm, parity_valid, nbaddata); break; default: cmn_err(CE_PANIC, "invalid RAID-Z configuration %d", raidz_parity(rm)); break; } if (rec_fn == NULL) return (RAIDZ_ORIGINAL_IMPL); else return (rec_fn(rm, dt)); } const char *raidz_gen_name[] = { "gen_p", "gen_pq", "gen_pqr" }; const char *raidz_rec_name[] = { "rec_p", "rec_q", "rec_r", "rec_pq", "rec_pr", "rec_qr", "rec_pqr" }; #if defined(_KERNEL) #define RAIDZ_KSTAT_LINE_LEN (17 + 10*12 + 1) static int raidz_math_kstat_headers(char *buf, size_t size) { int i; ssize_t off; ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN); off = snprintf(buf, size, "%-17s", "implementation"); for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) off += snprintf(buf + off, size - off, "%-16s", raidz_gen_name[i]); for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) off += snprintf(buf + off, size - off, "%-16s", raidz_rec_name[i]); (void) snprintf(buf + off, size - off, "\n"); return (0); } static int raidz_math_kstat_data(char *buf, size_t size, void *data) { raidz_impl_kstat_t *fstat = &raidz_impl_kstats[raidz_supp_impl_cnt]; raidz_impl_kstat_t *cstat = (raidz_impl_kstat_t *)data; ssize_t off = 0; int i; ASSERT3U(size, >=, RAIDZ_KSTAT_LINE_LEN); if (cstat == fstat) { off += snprintf(buf + off, size - off, "%-17s", "fastest"); for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) { int id = fstat->gen[i]; off += snprintf(buf + off, size - off, "%-16s", raidz_supp_impl[id]->name); } for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) { int id = fstat->rec[i]; off += snprintf(buf + off, size - off, "%-16s", raidz_supp_impl[id]->name); } } else { ptrdiff_t id = cstat - raidz_impl_kstats; off += snprintf(buf + off, size - off, "%-17s", raidz_supp_impl[id]->name); for (i = 0; i < ARRAY_SIZE(raidz_gen_name); i++) off += snprintf(buf + off, size - off, "%-16llu", (u_longlong_t)cstat->gen[i]); for (i = 0; i < ARRAY_SIZE(raidz_rec_name); i++) off += snprintf(buf + off, size - off, "%-16llu", (u_longlong_t)cstat->rec[i]); } (void) snprintf(buf + off, size - off, "\n"); return (0); } static void * raidz_math_kstat_addr(kstat_t *ksp, loff_t n) { if (n <= raidz_supp_impl_cnt) ksp->ks_private = (void *) (raidz_impl_kstats + n); else ksp->ks_private = NULL; return (ksp->ks_private); } #define BENCH_D_COLS (8ULL) #define BENCH_COLS (BENCH_D_COLS + PARITY_PQR) #define BENCH_ZIO_SIZE (1ULL << SPA_OLD_MAXBLOCKSHIFT) /* 128 kiB */ #define BENCH_NS MSEC2NSEC(25) /* 25ms */ typedef void (*benchmark_fn)(raidz_map_t *rm, const int fn); static void benchmark_gen_impl(raidz_map_t *rm, const int fn) { (void) fn; vdev_raidz_generate_parity(rm); } static void benchmark_rec_impl(raidz_map_t *rm, const int fn) { static const int rec_tgt[7][3] = { {1, 2, 3}, /* rec_p: bad QR & D[0] */ {0, 2, 3}, /* rec_q: bad PR & D[0] */ {0, 1, 3}, /* rec_r: bad PQ & D[0] */ {2, 3, 4}, /* rec_pq: bad R & D[0][1] */ {1, 3, 4}, /* rec_pr: bad Q & D[0][1] */ {0, 3, 4}, /* rec_qr: bad P & D[0][1] */ {3, 4, 5} /* rec_pqr: bad & D[0][1][2] */ }; vdev_raidz_reconstruct(rm, rec_tgt[fn], 3); } /* * Benchmarking of all supported implementations (raidz_supp_impl_cnt) * is performed by setting the rm_ops pointer and calling the top level * generate/reconstruct methods of bench_rm. */ static void benchmark_raidz_impl(raidz_map_t *bench_rm, const int fn, benchmark_fn bench_fn) { uint64_t run_cnt, speed, best_speed = 0; hrtime_t t_start, t_diff; raidz_impl_ops_t *curr_impl; raidz_impl_kstat_t *fstat = &raidz_impl_kstats[raidz_supp_impl_cnt]; int impl, i; for (impl = 0; impl < raidz_supp_impl_cnt; impl++) { /* set an implementation to benchmark */ curr_impl = raidz_supp_impl[impl]; bench_rm->rm_ops = curr_impl; run_cnt = 0; t_start = gethrtime(); do { for (i = 0; i < 25; i++, run_cnt++) bench_fn(bench_rm, fn); t_diff = gethrtime() - t_start; } while (t_diff < BENCH_NS); speed = run_cnt * BENCH_ZIO_SIZE * NANOSEC; speed /= (t_diff * BENCH_COLS); if (bench_fn == benchmark_gen_impl) raidz_impl_kstats[impl].gen[fn] = speed; else raidz_impl_kstats[impl].rec[fn] = speed; /* Update fastest implementation method */ if (speed > best_speed) { best_speed = speed; if (bench_fn == benchmark_gen_impl) { fstat->gen[fn] = impl; vdev_raidz_fastest_impl.gen[fn] = curr_impl->gen[fn]; } else { fstat->rec[fn] = impl; vdev_raidz_fastest_impl.rec[fn] = curr_impl->rec[fn]; } } } } #endif /* * Initialize and benchmark all supported implementations. */ static void benchmark_raidz(void *arg) { raidz_impl_ops_t *curr_impl; int i, c; /* Move supported impl into raidz_supp_impl */ for (i = 0, c = 0; i < ARRAY_SIZE(raidz_all_maths); i++) { curr_impl = (raidz_impl_ops_t *)raidz_all_maths[i]; if (curr_impl->init) curr_impl->init(); if (curr_impl->is_supported()) raidz_supp_impl[c++] = (raidz_impl_ops_t *)curr_impl; } membar_producer(); /* complete raidz_supp_impl[] init */ raidz_supp_impl_cnt = c; /* number of supported impl */ #if defined(_KERNEL) zio_t *bench_zio = NULL; raidz_map_t *bench_rm = NULL; uint64_t bench_parity; /* Fake a zio and run the benchmark on a warmed up buffer */ bench_zio = kmem_zalloc(sizeof (zio_t), KM_SLEEP); bench_zio->io_offset = 0; bench_zio->io_size = BENCH_ZIO_SIZE; /* only data columns */ bench_zio->io_abd = abd_alloc_linear(BENCH_ZIO_SIZE, B_TRUE); memset(abd_to_buf(bench_zio->io_abd), 0xAA, BENCH_ZIO_SIZE); /* Benchmark parity generation methods */ for (int fn = 0; fn < RAIDZ_GEN_NUM; fn++) { bench_parity = fn + 1; /* New raidz_map is needed for each generate_p/q/r */ bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT, BENCH_D_COLS + bench_parity, bench_parity); benchmark_raidz_impl(bench_rm, fn, benchmark_gen_impl); vdev_raidz_map_free(bench_rm); } /* Benchmark data reconstruction methods */ bench_rm = vdev_raidz_map_alloc(bench_zio, SPA_MINBLOCKSHIFT, BENCH_COLS, PARITY_PQR); for (int fn = 0; fn < RAIDZ_REC_NUM; fn++) benchmark_raidz_impl(bench_rm, fn, benchmark_rec_impl); vdev_raidz_map_free(bench_rm); /* cleanup the bench zio */ abd_free(bench_zio->io_abd); kmem_free(bench_zio, sizeof (zio_t)); #else /* * Skip the benchmark in user space to avoid impacting libzpool * consumers (zdb, zhack, zinject, ztest). The last implementation * is assumed to be the fastest and used by default. */ memcpy(&vdev_raidz_fastest_impl, raidz_supp_impl[raidz_supp_impl_cnt - 1], sizeof (vdev_raidz_fastest_impl)); strcpy(vdev_raidz_fastest_impl.name, "fastest"); #endif /* _KERNEL */ } void vdev_raidz_math_init(void) { #if defined(_KERNEL) /* * For 5.0 and latter Linux kernels the fletcher 4 benchmarks are * run in a kernel threads. This is needed to take advantage of the * SIMD functionality, see include/linux/simd_x86.h for details. */ taskqid_t id = taskq_dispatch(system_taskq, benchmark_raidz, NULL, TQ_SLEEP); if (id != TASKQID_INVALID) { taskq_wait_id(system_taskq, id); } else { benchmark_raidz(NULL); } /* Install kstats for all implementations */ raidz_math_kstat = kstat_create("zfs", 0, "vdev_raidz_bench", "misc", KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); if (raidz_math_kstat != NULL) { raidz_math_kstat->ks_data = NULL; raidz_math_kstat->ks_ndata = UINT32_MAX; kstat_set_raw_ops(raidz_math_kstat, raidz_math_kstat_headers, raidz_math_kstat_data, raidz_math_kstat_addr); kstat_install(raidz_math_kstat); } #else benchmark_raidz(NULL); #endif /* Finish initialization */ atomic_swap_32(&zfs_vdev_raidz_impl, user_sel_impl); raidz_math_initialized = B_TRUE; } void vdev_raidz_math_fini(void) { raidz_impl_ops_t const *curr_impl; #if defined(_KERNEL) if (raidz_math_kstat != NULL) { kstat_delete(raidz_math_kstat); raidz_math_kstat = NULL; } #endif for (int i = 0; i < ARRAY_SIZE(raidz_all_maths); i++) { curr_impl = raidz_all_maths[i]; if (curr_impl->fini) curr_impl->fini(); } } static const struct { char *name; uint32_t sel; } math_impl_opts[] = { { "cycle", IMPL_CYCLE }, { "fastest", IMPL_FASTEST }, { "original", IMPL_ORIGINAL }, { "scalar", IMPL_SCALAR } }; /* * Function sets desired raidz implementation. * * If we are called before init(), user preference will be saved in * user_sel_impl, and applied in later init() call. This occurs when module * parameter is specified on module load. Otherwise, directly update * zfs_vdev_raidz_impl. * * @val Name of raidz implementation to use * @param Unused. */ int vdev_raidz_impl_set(const char *val) { int err = -EINVAL; char req_name[RAIDZ_IMPL_NAME_MAX]; uint32_t impl = RAIDZ_IMPL_READ(user_sel_impl); size_t i; /* sanitize input */ i = strnlen(val, RAIDZ_IMPL_NAME_MAX); if (i == 0 || i == RAIDZ_IMPL_NAME_MAX) return (err); strlcpy(req_name, val, RAIDZ_IMPL_NAME_MAX); while (i > 0 && !!isspace(req_name[i-1])) i--; req_name[i] = '\0'; /* Check mandatory options */ for (i = 0; i < ARRAY_SIZE(math_impl_opts); i++) { if (strcmp(req_name, math_impl_opts[i].name) == 0) { impl = math_impl_opts[i].sel; err = 0; break; } } /* check all supported impl if init() was already called */ if (err != 0 && raidz_math_initialized) { /* check all supported implementations */ for (i = 0; i < raidz_supp_impl_cnt; i++) { if (strcmp(req_name, raidz_supp_impl[i]->name) == 0) { impl = i; err = 0; break; } } } if (err == 0) { if (raidz_math_initialized) atomic_swap_32(&zfs_vdev_raidz_impl, impl); else atomic_swap_32(&user_sel_impl, impl); } return (err); } #if defined(_KERNEL) #include static int zfs_vdev_raidz_impl_set(const char *val, zfs_kernel_param_t *kp) { return (vdev_raidz_impl_set(val)); } static int zfs_vdev_raidz_impl_get(char *buffer, zfs_kernel_param_t *kp) { int i, cnt = 0; char *fmt; const uint32_t impl = RAIDZ_IMPL_READ(zfs_vdev_raidz_impl); ASSERT(raidz_math_initialized); /* list mandatory options */ for (i = 0; i < ARRAY_SIZE(math_impl_opts) - 2; i++) { fmt = (impl == math_impl_opts[i].sel) ? "[%s] " : "%s "; cnt += sprintf(buffer + cnt, fmt, math_impl_opts[i].name); } /* list all supported implementations */ for (i = 0; i < raidz_supp_impl_cnt; i++) { fmt = (i == impl) ? "[%s] " : "%s "; cnt += sprintf(buffer + cnt, fmt, raidz_supp_impl[i]->name); } return (cnt); } module_param_call(zfs_vdev_raidz_impl, zfs_vdev_raidz_impl_set, zfs_vdev_raidz_impl_get, NULL, 0644); MODULE_PARM_DESC(zfs_vdev_raidz_impl, "Select raidz implementation."); #endif diff --git a/module/zfs/vdev_raidz_math_aarch64_neon_common.h b/module/zfs/vdev_raidz_math_aarch64_neon_common.h index 0ea2ad611c77..5312b9094d9e 100644 --- a/module/zfs/vdev_raidz_math_aarch64_neon_common.h +++ b/module/zfs/vdev_raidz_math_aarch64_neon_common.h @@ -1,684 +1,684 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Romain Dolbeau. All rights reserved. */ #include -#include +#include #define __asm __asm__ __volatile__ #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) #define VR0_(REG, ...) "%[w"#REG"]" #define VR1_(_1, REG, ...) "%[w"#REG"]" #define VR2_(_1, _2, REG, ...) "%[w"#REG"]" #define VR3_(_1, _2, _3, REG, ...) "%[w"#REG"]" #define VR4_(_1, _2, _3, _4, REG, ...) "%[w"#REG"]" #define VR5_(_1, _2, _3, _4, _5, REG, ...) "%[w"#REG"]" #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "%[w"#REG"]" #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "%[w"#REG"]" /* * Here we need registers not used otherwise. * They will be used in unused ASM for the case * with more registers than required... but GCC * will still need to make sure the constraints * are correct, and duplicate constraints are illegal * ... and we use the "register" number as a name */ #define VR0(r...) VR0_(r) #define VR1(r...) VR1_(r) #define VR2(r...) VR2_(r, 36) #define VR3(r...) VR3_(r, 36, 35) #define VR4(r...) VR4_(r, 36, 35, 34, 33) #define VR5(r...) VR5_(r, 36, 35, 34, 33, 32) #define VR6(r...) VR6_(r, 36, 35, 34, 33, 32, 31) #define VR7(r...) VR7_(r, 36, 35, 34, 33, 32, 31, 30) #define VR(X) "%[w"#X"]" #define RVR0_(REG, ...) [w##REG] "w" (w##REG) #define RVR1_(_1, REG, ...) [w##REG] "w" (w##REG) #define RVR2_(_1, _2, REG, ...) [w##REG] "w" (w##REG) #define RVR3_(_1, _2, _3, REG, ...) [w##REG] "w" (w##REG) #define RVR4_(_1, _2, _3, _4, REG, ...) [w##REG] "w" (w##REG) #define RVR5_(_1, _2, _3, _4, _5, REG, ...) [w##REG] "w" (w##REG) #define RVR6_(_1, _2, _3, _4, _5, _6, REG, ...) [w##REG] "w" (w##REG) #define RVR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) [w##REG] "w" (w##REG) #define RVR0(r...) RVR0_(r) #define RVR1(r...) RVR1_(r) #define RVR2(r...) RVR2_(r, 36) #define RVR3(r...) RVR3_(r, 36, 35) #define RVR4(r...) RVR4_(r, 36, 35, 34, 33) #define RVR5(r...) RVR5_(r, 36, 35, 34, 33, 32) #define RVR6(r...) RVR6_(r, 36, 35, 34, 33, 32, 31) #define RVR7(r...) RVR7_(r, 36, 35, 34, 33, 32, 31, 30) #define RVR(X) [w##X] "w" (w##X) #define WVR0_(REG, ...) [w##REG] "=w" (w##REG) #define WVR1_(_1, REG, ...) [w##REG] "=w" (w##REG) #define WVR2_(_1, _2, REG, ...) [w##REG] "=w" (w##REG) #define WVR3_(_1, _2, _3, REG, ...) [w##REG] "=w" (w##REG) #define WVR4_(_1, _2, _3, _4, REG, ...) [w##REG] "=w" (w##REG) #define WVR5_(_1, _2, _3, _4, _5, REG, ...) [w##REG] "=w" (w##REG) #define WVR6_(_1, _2, _3, _4, _5, _6, REG, ...) [w##REG] "=w" (w##REG) #define WVR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) [w##REG] "=w" (w##REG) #define WVR0(r...) WVR0_(r) #define WVR1(r...) WVR1_(r) #define WVR2(r...) WVR2_(r, 36) #define WVR3(r...) WVR3_(r, 36, 35) #define WVR4(r...) WVR4_(r, 36, 35, 34, 33) #define WVR5(r...) WVR5_(r, 36, 35, 34, 33, 32) #define WVR6(r...) WVR6_(r, 36, 35, 34, 33, 32, 31) #define WVR7(r...) WVR7_(r, 36, 35, 34, 33, 32, 31, 30) #define WVR(X) [w##X] "=w" (w##X) #define UVR0_(REG, ...) [w##REG] "+&w" (w##REG) #define UVR1_(_1, REG, ...) [w##REG] "+&w" (w##REG) #define UVR2_(_1, _2, REG, ...) [w##REG] "+&w" (w##REG) #define UVR3_(_1, _2, _3, REG, ...) [w##REG] "+&w" (w##REG) #define UVR4_(_1, _2, _3, _4, REG, ...) [w##REG] "+&w" (w##REG) #define UVR5_(_1, _2, _3, _4, _5, REG, ...) [w##REG] "+&w" (w##REG) #define UVR6_(_1, _2, _3, _4, _5, _6, REG, ...) [w##REG] "+&w" (w##REG) #define UVR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) [w##REG] "+&w" (w##REG) #define UVR0(r...) UVR0_(r) #define UVR1(r...) UVR1_(r) #define UVR2(r...) UVR2_(r, 36) #define UVR3(r...) UVR3_(r, 36, 35) #define UVR4(r...) UVR4_(r, 36, 35, 34, 33) #define UVR5(r...) UVR5_(r, 36, 35, 34, 33, 32) #define UVR6(r...) UVR6_(r, 36, 35, 34, 33, 32, 31) #define UVR7(r...) UVR7_(r, 36, 35, 34, 33, 32, 31, 30) #define UVR(X) [w##X] "+&w" (w##X) #define R_01(REG1, REG2, ...) REG1, REG2 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 #define R_23(REG...) _R_23(REG, 1, 2, 3) #define ZFS_ASM_BUG() ASSERT(0) #define OFFSET(ptr, val) (((unsigned char *)(ptr))+val) extern const uint8_t gf_clmul_mod_lt[4*256][16]; #define ELEM_SIZE 16 typedef struct v { uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); } v_t; #define XOR_ACC(src, r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "ld1 { v21.4s },%[SRC0]\n" \ "ld1 { v20.4s },%[SRC1]\n" \ "ld1 { v19.4s },%[SRC2]\n" \ "ld1 { v18.4s },%[SRC3]\n" \ "eor " VR0(r) ".16b," VR0(r) ".16b,v21.16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b,v20.16b\n" \ "eor " VR2(r) ".16b," VR2(r) ".16b,v19.16b\n" \ "eor " VR3(r) ".16b," VR3(r) ".16b,v18.16b\n" \ "ld1 { v21.4s },%[SRC4]\n" \ "ld1 { v20.4s },%[SRC5]\n" \ "ld1 { v19.4s },%[SRC6]\n" \ "ld1 { v18.4s },%[SRC7]\n" \ "eor " VR4(r) ".16b," VR4(r) ".16b,v21.16b\n" \ "eor " VR5(r) ".16b," VR5(r) ".16b,v20.16b\n" \ "eor " VR6(r) ".16b," VR6(r) ".16b,v19.16b\n" \ "eor " VR7(r) ".16b," VR7(r) ".16b,v18.16b\n" \ : UVR0(r), UVR1(r), UVR2(r), UVR3(r), \ UVR4(r), UVR5(r), UVR6(r), UVR7(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16))), \ [SRC2] "Q" (*(OFFSET(src, 32))), \ [SRC3] "Q" (*(OFFSET(src, 48))), \ [SRC4] "Q" (*(OFFSET(src, 64))), \ [SRC5] "Q" (*(OFFSET(src, 80))), \ [SRC6] "Q" (*(OFFSET(src, 96))), \ [SRC7] "Q" (*(OFFSET(src, 112))) \ : "v18", "v19", "v20", "v21"); \ break; \ case 4: \ __asm( \ "ld1 { v21.4s },%[SRC0]\n" \ "ld1 { v20.4s },%[SRC1]\n" \ "ld1 { v19.4s },%[SRC2]\n" \ "ld1 { v18.4s },%[SRC3]\n" \ "eor " VR0(r) ".16b," VR0(r) ".16b,v21.16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b,v20.16b\n" \ "eor " VR2(r) ".16b," VR2(r) ".16b,v19.16b\n" \ "eor " VR3(r) ".16b," VR3(r) ".16b,v18.16b\n" \ : UVR0(r), UVR1(r), UVR2(r), UVR3(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16))), \ [SRC2] "Q" (*(OFFSET(src, 32))), \ [SRC3] "Q" (*(OFFSET(src, 48))) \ : "v18", "v19", "v20", "v21"); \ break; \ case 2: \ __asm( \ "ld1 { v21.4s },%[SRC0]\n" \ "ld1 { v20.4s },%[SRC1]\n" \ "eor " VR0(r) ".16b," VR0(r) ".16b,v21.16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b,v20.16b\n" \ : UVR0(r), UVR1(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16))) \ : "v20", "v21"); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define XOR(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "eor " VR4(r) ".16b," VR4(r) ".16b," VR0(r) ".16b\n" \ "eor " VR5(r) ".16b," VR5(r) ".16b," VR1(r) ".16b\n" \ "eor " VR6(r) ".16b," VR6(r) ".16b," VR2(r) ".16b\n" \ "eor " VR7(r) ".16b," VR7(r) ".16b," VR3(r) ".16b\n" \ : UVR4(r), UVR5(r), UVR6(r), UVR7(r) \ : RVR0(r), RVR1(r), RVR2(r), RVR3(r)); \ break; \ case 4: \ __asm( \ "eor " VR2(r) ".16b," VR2(r) ".16b," VR0(r) ".16b\n" \ "eor " VR3(r) ".16b," VR3(r) ".16b," VR1(r) ".16b\n" \ : UVR2(r), UVR3(r) \ : RVR0(r), RVR1(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define ZERO(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "eor " VR0(r) ".16b," VR0(r) ".16b," VR0(r) ".16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b," VR1(r) ".16b\n" \ "eor " VR2(r) ".16b," VR2(r) ".16b," VR2(r) ".16b\n" \ "eor " VR3(r) ".16b," VR3(r) ".16b," VR3(r) ".16b\n" \ "eor " VR4(r) ".16b," VR4(r) ".16b," VR4(r) ".16b\n" \ "eor " VR5(r) ".16b," VR5(r) ".16b," VR5(r) ".16b\n" \ "eor " VR6(r) ".16b," VR6(r) ".16b," VR6(r) ".16b\n" \ "eor " VR7(r) ".16b," VR7(r) ".16b," VR7(r) ".16b\n" \ : WVR0(r), WVR1(r), WVR2(r), WVR3(r), \ WVR4(r), WVR5(r), WVR6(r), WVR7(r)); \ break; \ case 4: \ __asm( \ "eor " VR0(r) ".16b," VR0(r) ".16b," VR0(r) ".16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b," VR1(r) ".16b\n" \ "eor " VR2(r) ".16b," VR2(r) ".16b," VR2(r) ".16b\n" \ "eor " VR3(r) ".16b," VR3(r) ".16b," VR3(r) ".16b\n" \ : WVR0(r), WVR1(r), WVR2(r), WVR3(r)); \ break; \ case 2: \ __asm( \ "eor " VR0(r) ".16b," VR0(r) ".16b," VR0(r) ".16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b," VR1(r) ".16b\n" \ : WVR0(r), WVR1(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define COPY(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "mov " VR4(r) ".16b," VR0(r) ".16b\n" \ "mov " VR5(r) ".16b," VR1(r) ".16b\n" \ "mov " VR6(r) ".16b," VR2(r) ".16b\n" \ "mov " VR7(r) ".16b," VR3(r) ".16b\n" \ : WVR4(r), WVR5(r), WVR6(r), WVR7(r) \ : RVR0(r), RVR1(r), RVR2(r), RVR3(r)); \ break; \ case 4: \ __asm( \ "mov " VR2(r) ".16b," VR0(r) ".16b\n" \ "mov " VR3(r) ".16b," VR1(r) ".16b\n" \ : WVR2(r), WVR3(r) \ : RVR0(r), RVR1(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define LOAD(src, r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "ld1 { " VR0(r) ".4s },%[SRC0]\n" \ "ld1 { " VR1(r) ".4s },%[SRC1]\n" \ "ld1 { " VR2(r) ".4s },%[SRC2]\n" \ "ld1 { " VR3(r) ".4s },%[SRC3]\n" \ "ld1 { " VR4(r) ".4s },%[SRC4]\n" \ "ld1 { " VR5(r) ".4s },%[SRC5]\n" \ "ld1 { " VR6(r) ".4s },%[SRC6]\n" \ "ld1 { " VR7(r) ".4s },%[SRC7]\n" \ : WVR0(r), WVR1(r), WVR2(r), WVR3(r), \ WVR4(r), WVR5(r), WVR6(r), WVR7(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16))), \ [SRC2] "Q" (*(OFFSET(src, 32))), \ [SRC3] "Q" (*(OFFSET(src, 48))), \ [SRC4] "Q" (*(OFFSET(src, 64))), \ [SRC5] "Q" (*(OFFSET(src, 80))), \ [SRC6] "Q" (*(OFFSET(src, 96))), \ [SRC7] "Q" (*(OFFSET(src, 112)))); \ break; \ case 4: \ __asm( \ "ld1 { " VR0(r) ".4s },%[SRC0]\n" \ "ld1 { " VR1(r) ".4s },%[SRC1]\n" \ "ld1 { " VR2(r) ".4s },%[SRC2]\n" \ "ld1 { " VR3(r) ".4s },%[SRC3]\n" \ : WVR0(r), WVR1(r), WVR2(r), WVR3(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16))), \ [SRC2] "Q" (*(OFFSET(src, 32))), \ [SRC3] "Q" (*(OFFSET(src, 48)))); \ break; \ case 2: \ __asm( \ "ld1 { " VR0(r) ".4s },%[SRC0]\n" \ "ld1 { " VR1(r) ".4s },%[SRC1]\n" \ : WVR0(r), WVR1(r) \ : [SRC0] "Q" (*(OFFSET(src, 0))), \ [SRC1] "Q" (*(OFFSET(src, 16)))); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define STORE(dst, r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "st1 { " VR0(r) ".4s },%[DST0]\n" \ "st1 { " VR1(r) ".4s },%[DST1]\n" \ "st1 { " VR2(r) ".4s },%[DST2]\n" \ "st1 { " VR3(r) ".4s },%[DST3]\n" \ "st1 { " VR4(r) ".4s },%[DST4]\n" \ "st1 { " VR5(r) ".4s },%[DST5]\n" \ "st1 { " VR6(r) ".4s },%[DST6]\n" \ "st1 { " VR7(r) ".4s },%[DST7]\n" \ : [DST0] "=Q" (*(OFFSET(dst, 0))), \ [DST1] "=Q" (*(OFFSET(dst, 16))), \ [DST2] "=Q" (*(OFFSET(dst, 32))), \ [DST3] "=Q" (*(OFFSET(dst, 48))), \ [DST4] "=Q" (*(OFFSET(dst, 64))), \ [DST5] "=Q" (*(OFFSET(dst, 80))), \ [DST6] "=Q" (*(OFFSET(dst, 96))), \ [DST7] "=Q" (*(OFFSET(dst, 112))) \ : RVR0(r), RVR1(r), RVR2(r), RVR3(r), \ RVR4(r), RVR5(r), RVR6(r), RVR7(r)); \ break; \ case 4: \ __asm( \ "st1 { " VR0(r) ".4s },%[DST0]\n" \ "st1 { " VR1(r) ".4s },%[DST1]\n" \ "st1 { " VR2(r) ".4s },%[DST2]\n" \ "st1 { " VR3(r) ".4s },%[DST3]\n" \ : [DST0] "=Q" (*(OFFSET(dst, 0))), \ [DST1] "=Q" (*(OFFSET(dst, 16))), \ [DST2] "=Q" (*(OFFSET(dst, 32))), \ [DST3] "=Q" (*(OFFSET(dst, 48))) \ : RVR0(r), RVR1(r), RVR2(r), RVR3(r)); \ break; \ case 2: \ __asm( \ "st1 { " VR0(r) ".4s },%[DST0]\n" \ "st1 { " VR1(r) ".4s },%[DST1]\n" \ : [DST0] "=Q" (*(OFFSET(dst, 0))), \ [DST1] "=Q" (*(OFFSET(dst, 16))) \ : RVR0(r), RVR1(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } /* * Unfortunately cannot use the macro, because GCC * will try to use the macro name and not value * later on... * Kept as a reference to what a numbered variable is */ #define _00 "v17" #define _1d "v16" #define _temp0 "v19" #define _temp1 "v18" #define MUL2_SETUP() \ { \ __asm( \ "eor " VR(17) ".16b," VR(17) ".16b," VR(17) ".16b\n" \ "movi " VR(16) ".16b,#0x1d\n" \ : WVR(16), WVR(17)); \ } #define MUL2(r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "cmgt v19.16b," VR(17) ".16b," VR0(r) ".16b\n" \ "cmgt v18.16b," VR(17) ".16b," VR1(r) ".16b\n" \ "cmgt v21.16b," VR(17) ".16b," VR2(r) ".16b\n" \ "cmgt v20.16b," VR(17) ".16b," VR3(r) ".16b\n" \ "and v19.16b,v19.16b," VR(16) ".16b\n" \ "and v18.16b,v18.16b," VR(16) ".16b\n" \ "and v21.16b,v21.16b," VR(16) ".16b\n" \ "and v20.16b,v20.16b," VR(16) ".16b\n" \ "shl " VR0(r) ".16b," VR0(r) ".16b,#1\n" \ "shl " VR1(r) ".16b," VR1(r) ".16b,#1\n" \ "shl " VR2(r) ".16b," VR2(r) ".16b,#1\n" \ "shl " VR3(r) ".16b," VR3(r) ".16b,#1\n" \ "eor " VR0(r) ".16b,v19.16b," VR0(r) ".16b\n" \ "eor " VR1(r) ".16b,v18.16b," VR1(r) ".16b\n" \ "eor " VR2(r) ".16b,v21.16b," VR2(r) ".16b\n" \ "eor " VR3(r) ".16b,v20.16b," VR3(r) ".16b\n" \ : UVR0(r), UVR1(r), UVR2(r), UVR3(r) \ : RVR(17), RVR(16) \ : "v18", "v19", "v20", "v21"); \ break; \ case 2: \ __asm( \ "cmgt v19.16b," VR(17) ".16b," VR0(r) ".16b\n" \ "cmgt v18.16b," VR(17) ".16b," VR1(r) ".16b\n" \ "and v19.16b,v19.16b," VR(16) ".16b\n" \ "and v18.16b,v18.16b," VR(16) ".16b\n" \ "shl " VR0(r) ".16b," VR0(r) ".16b,#1\n" \ "shl " VR1(r) ".16b," VR1(r) ".16b,#1\n" \ "eor " VR0(r) ".16b,v19.16b," VR0(r) ".16b\n" \ "eor " VR1(r) ".16b,v18.16b," VR1(r) ".16b\n" \ : UVR0(r), UVR1(r) \ : RVR(17), RVR(16) \ : "v18", "v19"); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL4(r...) \ { \ MUL2(r); \ MUL2(r); \ } /* * Unfortunately cannot use the macro, because GCC * will try to use the macro name and not value * later on... * Kept as a reference to what a register is * (here we're using actual registers for the * clobbered ones) */ #define _0f "v15" #define _a_save "v14" #define _b_save "v13" #define _lt_mod_a "v12" #define _lt_clmul_a "v11" #define _lt_mod_b "v10" #define _lt_clmul_b "v15" #define _MULx2(c, r...) \ { \ switch (REG_CNT(r)) { \ case 2: \ __asm( \ /* lts for upper part */ \ "movi v15.16b,#0x0f\n" \ "ld1 { v10.4s },%[lt0]\n" \ "ld1 { v11.4s },%[lt1]\n" \ /* upper part */ \ "and v14.16b," VR0(r) ".16b,v15.16b\n" \ "and v13.16b," VR1(r) ".16b,v15.16b\n" \ "sshr " VR0(r) ".8h," VR0(r) ".8h,#4\n" \ "sshr " VR1(r) ".8h," VR1(r) ".8h,#4\n" \ "and " VR0(r) ".16b," VR0(r) ".16b,v15.16b\n" \ "and " VR1(r) ".16b," VR1(r) ".16b,v15.16b\n" \ \ "tbl v12.16b,{v10.16b}," VR0(r) ".16b\n" \ "tbl v10.16b,{v10.16b}," VR1(r) ".16b\n" \ "tbl v15.16b,{v11.16b}," VR0(r) ".16b\n" \ "tbl v11.16b,{v11.16b}," VR1(r) ".16b\n" \ \ "eor " VR0(r) ".16b,v15.16b,v12.16b\n" \ "eor " VR1(r) ".16b,v11.16b,v10.16b\n" \ /* lts for lower part */ \ "ld1 { v10.4s },%[lt2]\n" \ "ld1 { v15.4s },%[lt3]\n" \ /* lower part */ \ "tbl v12.16b,{v10.16b},v14.16b\n" \ "tbl v10.16b,{v10.16b},v13.16b\n" \ "tbl v11.16b,{v15.16b},v14.16b\n" \ "tbl v15.16b,{v15.16b},v13.16b\n" \ \ "eor " VR0(r) ".16b," VR0(r) ".16b,v12.16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b,v10.16b\n" \ "eor " VR0(r) ".16b," VR0(r) ".16b,v11.16b\n" \ "eor " VR1(r) ".16b," VR1(r) ".16b,v15.16b\n" \ : UVR0(r), UVR1(r) \ : [lt0] "Q" ((gf_clmul_mod_lt[4*(c)+0][0])), \ [lt1] "Q" ((gf_clmul_mod_lt[4*(c)+1][0])), \ [lt2] "Q" ((gf_clmul_mod_lt[4*(c)+2][0])), \ [lt3] "Q" ((gf_clmul_mod_lt[4*(c)+3][0])) \ : "v10", "v11", "v12", "v13", "v14", "v15"); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL(c, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ _MULx2(c, R_23(r)); \ _MULx2(c, R_01(r)); \ break; \ case 2: \ _MULx2(c, R_01(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define raidz_math_begin() kfpu_begin() #define raidz_math_end() kfpu_end() /* Overkill... */ #if defined(_KERNEL) #define GEN_X_DEFINE_0_3() \ register unsigned char w0 asm("v0") __attribute__((vector_size(16))); \ register unsigned char w1 asm("v1") __attribute__((vector_size(16))); \ register unsigned char w2 asm("v2") __attribute__((vector_size(16))); \ register unsigned char w3 asm("v3") __attribute__((vector_size(16))); #define GEN_X_DEFINE_4_5() \ register unsigned char w4 asm("v4") __attribute__((vector_size(16))); \ register unsigned char w5 asm("v5") __attribute__((vector_size(16))); #define GEN_X_DEFINE_6_7() \ register unsigned char w6 asm("v6") __attribute__((vector_size(16))); \ register unsigned char w7 asm("v7") __attribute__((vector_size(16))); #define GEN_X_DEFINE_8_9() \ register unsigned char w8 asm("v8") __attribute__((vector_size(16))); \ register unsigned char w9 asm("v9") __attribute__((vector_size(16))); #define GEN_X_DEFINE_10_11() \ register unsigned char w10 asm("v10") __attribute__((vector_size(16))); \ register unsigned char w11 asm("v11") __attribute__((vector_size(16))); #define GEN_X_DEFINE_12_15() \ register unsigned char w12 asm("v12") __attribute__((vector_size(16))); \ register unsigned char w13 asm("v13") __attribute__((vector_size(16))); \ register unsigned char w14 asm("v14") __attribute__((vector_size(16))); \ register unsigned char w15 asm("v15") __attribute__((vector_size(16))); #define GEN_X_DEFINE_16() \ register unsigned char w16 asm("v16") __attribute__((vector_size(16))); #define GEN_X_DEFINE_17() \ register unsigned char w17 asm("v17") __attribute__((vector_size(16))); #define GEN_X_DEFINE_18_21() \ register unsigned char w18 asm("v18") __attribute__((vector_size(16))); \ register unsigned char w19 asm("v19") __attribute__((vector_size(16))); \ register unsigned char w20 asm("v20") __attribute__((vector_size(16))); \ register unsigned char w21 asm("v21") __attribute__((vector_size(16))); #define GEN_X_DEFINE_22_23() \ register unsigned char w22 asm("v22") __attribute__((vector_size(16))); \ register unsigned char w23 asm("v23") __attribute__((vector_size(16))); #define GEN_X_DEFINE_24_27() \ register unsigned char w24 asm("v24") __attribute__((vector_size(16))); \ register unsigned char w25 asm("v25") __attribute__((vector_size(16))); \ register unsigned char w26 asm("v26") __attribute__((vector_size(16))); \ register unsigned char w27 asm("v27") __attribute__((vector_size(16))); #define GEN_X_DEFINE_28_30() \ register unsigned char w28 asm("v28") __attribute__((vector_size(16))); \ register unsigned char w29 asm("v29") __attribute__((vector_size(16))); \ register unsigned char w30 asm("v30") __attribute__((vector_size(16))); #define GEN_X_DEFINE_31() \ register unsigned char w31 asm("v31") __attribute__((vector_size(16))); #define GEN_X_DEFINE_32() \ register unsigned char w32 asm("v31") __attribute__((vector_size(16))); #define GEN_X_DEFINE_33_36() \ register unsigned char w33 asm("v31") __attribute__((vector_size(16))); \ register unsigned char w34 asm("v31") __attribute__((vector_size(16))); \ register unsigned char w35 asm("v31") __attribute__((vector_size(16))); \ register unsigned char w36 asm("v31") __attribute__((vector_size(16))); #define GEN_X_DEFINE_37_38() \ register unsigned char w37 asm("v31") __attribute__((vector_size(16))); \ register unsigned char w38 asm("v31") __attribute__((vector_size(16))); #define GEN_X_DEFINE_ALL() \ GEN_X_DEFINE_0_3() \ GEN_X_DEFINE_4_5() \ GEN_X_DEFINE_6_7() \ GEN_X_DEFINE_8_9() \ GEN_X_DEFINE_10_11() \ GEN_X_DEFINE_12_15() \ GEN_X_DEFINE_16() \ GEN_X_DEFINE_17() \ GEN_X_DEFINE_18_21() \ GEN_X_DEFINE_22_23() \ GEN_X_DEFINE_24_27() \ GEN_X_DEFINE_28_30() \ GEN_X_DEFINE_31() \ GEN_X_DEFINE_32() \ GEN_X_DEFINE_33_36() \ GEN_X_DEFINE_37_38() #else #define GEN_X_DEFINE_0_3() \ unsigned char w0 __attribute__((vector_size(16))); \ unsigned char w1 __attribute__((vector_size(16))); \ unsigned char w2 __attribute__((vector_size(16))); \ unsigned char w3 __attribute__((vector_size(16))); #define GEN_X_DEFINE_4_5() \ unsigned char w4 __attribute__((vector_size(16))); \ unsigned char w5 __attribute__((vector_size(16))); #define GEN_X_DEFINE_6_7() \ unsigned char w6 __attribute__((vector_size(16))); \ unsigned char w7 __attribute__((vector_size(16))); #define GEN_X_DEFINE_8_9() \ unsigned char w8 __attribute__((vector_size(16))); \ unsigned char w9 __attribute__((vector_size(16))); #define GEN_X_DEFINE_10_11() \ unsigned char w10 __attribute__((vector_size(16))); \ unsigned char w11 __attribute__((vector_size(16))); #define GEN_X_DEFINE_12_15() \ unsigned char w12 __attribute__((vector_size(16))); \ unsigned char w13 __attribute__((vector_size(16))); \ unsigned char w14 __attribute__((vector_size(16))); \ unsigned char w15 __attribute__((vector_size(16))); #define GEN_X_DEFINE_16() \ unsigned char w16 __attribute__((vector_size(16))); #define GEN_X_DEFINE_17() \ unsigned char w17 __attribute__((vector_size(16))); #define GEN_X_DEFINE_18_21() \ unsigned char w18 __attribute__((vector_size(16))); \ unsigned char w19 __attribute__((vector_size(16))); \ unsigned char w20 __attribute__((vector_size(16))); \ unsigned char w21 __attribute__((vector_size(16))); #define GEN_X_DEFINE_22_23() \ unsigned char w22 __attribute__((vector_size(16))); \ unsigned char w23 __attribute__((vector_size(16))); #define GEN_X_DEFINE_24_27() \ unsigned char w24 __attribute__((vector_size(16))); \ unsigned char w25 __attribute__((vector_size(16))); \ unsigned char w26 __attribute__((vector_size(16))); \ unsigned char w27 __attribute__((vector_size(16))); #define GEN_X_DEFINE_28_30() \ unsigned char w28 __attribute__((vector_size(16))); \ unsigned char w29 __attribute__((vector_size(16))); \ unsigned char w30 __attribute__((vector_size(16))); #define GEN_X_DEFINE_31() \ unsigned char w31 __attribute__((vector_size(16))); #define GEN_X_DEFINE_32() \ unsigned char w32 __attribute__((vector_size(16))); #define GEN_X_DEFINE_33_36() \ unsigned char w33 __attribute__((vector_size(16))); \ unsigned char w34 __attribute__((vector_size(16))); \ unsigned char w35 __attribute__((vector_size(16))); \ unsigned char w36 __attribute__((vector_size(16))); #define GEN_X_DEFINE_37_38() \ unsigned char w37 __attribute__((vector_size(16))); \ unsigned char w38 __attribute__((vector_size(16))); #define GEN_X_DEFINE_ALL() \ GEN_X_DEFINE_0_3() \ GEN_X_DEFINE_4_5() \ GEN_X_DEFINE_6_7() \ GEN_X_DEFINE_8_9() \ GEN_X_DEFINE_10_11() \ GEN_X_DEFINE_12_15() \ GEN_X_DEFINE_16() \ GEN_X_DEFINE_17() \ GEN_X_DEFINE_18_21() \ GEN_X_DEFINE_22_23() \ GEN_X_DEFINE_24_27() \ GEN_X_DEFINE_28_30() \ GEN_X_DEFINE_31() \ GEN_X_DEFINE_32() \ GEN_X_DEFINE_33_36() \ GEN_X_DEFINE_37_38() #endif diff --git a/module/zfs/vdev_raidz_math_avx2.c b/module/zfs/vdev_raidz_math_avx2.c index a12eb672081f..008e848d4c5e 100644 --- a/module/zfs/vdev_raidz_math_avx2.c +++ b/module/zfs/vdev_raidz_math_avx2.c @@ -1,411 +1,411 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Gvozden NeÅ¡ković. All rights reserved. */ #include #if defined(__x86_64) && defined(HAVE_AVX2) #include -#include +#include #define __asm __asm__ __volatile__ #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) #define VR0_(REG, ...) "ymm"#REG #define VR1_(_1, REG, ...) "ymm"#REG #define VR2_(_1, _2, REG, ...) "ymm"#REG #define VR3_(_1, _2, _3, REG, ...) "ymm"#REG #define VR4_(_1, _2, _3, _4, REG, ...) "ymm"#REG #define VR5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG #define VR0(r...) VR0_(r) #define VR1(r...) VR1_(r) #define VR2(r...) VR2_(r, 1) #define VR3(r...) VR3_(r, 1, 2) #define VR4(r...) VR4_(r, 1, 2) #define VR5(r...) VR5_(r, 1, 2, 3) #define VR6(r...) VR6_(r, 1, 2, 3, 4) #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) #define R_01(REG1, REG2, ...) REG1, REG2 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 #define R_23(REG...) _R_23(REG, 1, 2, 3) #define ZFS_ASM_BUG() ASSERT(0) extern const uint8_t gf_clmul_mod_lt[4*256][16]; #define ELEM_SIZE 32 typedef struct v { uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); } v_t; #define XOR_ACC(src, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ "vpxor 0x40(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ "vpxor 0x60(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ : : [SRC] "r" (src)); \ break; \ case 2: \ __asm( \ "vpxor 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ "vpxor 0x20(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ : : [SRC] "r" (src)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define XOR(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "vpxor %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ "vpxor %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ "vpxor %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ "vpxor %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ break; \ case 4: \ __asm( \ "vpxor %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ "vpxor %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define ZERO(r...) XOR(r, r) #define COPY(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "vmovdqa %" VR0(r) ", %" VR4(r) "\n" \ "vmovdqa %" VR1(r) ", %" VR5(r) "\n" \ "vmovdqa %" VR2(r) ", %" VR6(r) "\n" \ "vmovdqa %" VR3(r) ", %" VR7(r)); \ break; \ case 4: \ __asm( \ "vmovdqa %" VR0(r) ", %" VR2(r) "\n" \ "vmovdqa %" VR1(r) ", %" VR3(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define LOAD(src, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \ "vmovdqa 0x40(%[SRC]), %%" VR2(r) "\n" \ "vmovdqa 0x60(%[SRC]), %%" VR3(r) "\n" \ : : [SRC] "r" (src)); \ break; \ case 2: \ __asm( \ "vmovdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ "vmovdqa 0x20(%[SRC]), %%" VR1(r) "\n" \ : : [SRC] "r" (src)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define STORE(dst, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \ "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \ "vmovdqa %%" VR2(r) ", 0x40(%[DST])\n" \ "vmovdqa %%" VR3(r) ", 0x60(%[DST])\n" \ : : [DST] "r" (dst)); \ break; \ case 2: \ __asm( \ "vmovdqa %%" VR0(r) ", 0x00(%[DST])\n" \ "vmovdqa %%" VR1(r) ", 0x20(%[DST])\n" \ : : [DST] "r" (dst)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define FLUSH() \ { \ __asm("vzeroupper"); \ } #define MUL2_SETUP() \ { \ __asm("vmovq %0, %%xmm14" :: "r"(0x1d1d1d1d1d1d1d1d)); \ __asm("vpbroadcastq %xmm14, %ymm14"); \ __asm("vpxor %ymm15, %ymm15 ,%ymm15"); \ } #define _MUL2(r...) \ { \ switch (REG_CNT(r)) { \ case 2: \ __asm( \ "vpcmpgtb %" VR0(r)", %ymm15, %ymm12\n" \ "vpcmpgtb %" VR1(r)", %ymm15, %ymm13\n" \ "vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \ "vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \ "vpand %ymm14, %ymm12, %ymm12\n" \ "vpand %ymm14, %ymm13, %ymm13\n" \ "vpxor %ymm12, %" VR0(r)", %" VR0(r) "\n" \ "vpxor %ymm13, %" VR1(r)", %" VR1(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL2(r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ _MUL2(R_01(r)); \ _MUL2(R_23(r)); \ break; \ case 2: \ _MUL2(r); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL4(r...) \ { \ MUL2(r); \ MUL2(r); \ } #define _0f "ymm15" #define _as "ymm14" #define _bs "ymm13" #define _ltmod "ymm12" #define _ltmul "ymm11" #define _ta "ymm10" #define _tb "ymm15" static const uint8_t __attribute__((aligned(32))) _mul_mask = 0x0F; #define _MULx2(c, r...) \ { \ switch (REG_CNT(r)) { \ case 2: \ __asm( \ "vpbroadcastb (%[mask]), %%" _0f "\n" \ /* upper bits */ \ "vbroadcasti128 0x00(%[lt]), %%" _ltmod "\n" \ "vbroadcasti128 0x10(%[lt]), %%" _ltmul "\n" \ \ "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \ "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \ "vpand %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \ "vpand %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \ "vpand %%" _0f ", %%" _as ", %%" _as "\n" \ "vpand %%" _0f ", %%" _bs ", %%" _bs "\n" \ \ "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \ "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \ "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \ "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \ /* lower bits */ \ "vbroadcasti128 0x20(%[lt]), %%" _ltmod "\n" \ "vbroadcasti128 0x30(%[lt]), %%" _ltmul "\n" \ \ "vpxor %%" _ta ", %%" _as ", %%" _as "\n" \ "vpxor %%" _tb ", %%" _bs ", %%" _bs "\n" \ \ "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \ "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \ "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\ "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\ \ "vpxor %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \ "vpxor %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \ "vpxor %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \ "vpxor %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \ : : [mask] "r" (&_mul_mask), \ [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL(c, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ _MULx2(c, R_01(r)); \ _MULx2(c, R_23(r)); \ break; \ case 2: \ _MULx2(c, R_01(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define raidz_math_begin() kfpu_begin() #define raidz_math_end() \ { \ FLUSH(); \ kfpu_end(); \ } #define SYN_STRIDE 4 #define ZERO_STRIDE 4 #define ZERO_DEFINE() {} #define ZERO_D 0, 1, 2, 3 #define COPY_STRIDE 4 #define COPY_DEFINE() {} #define COPY_D 0, 1, 2, 3 #define ADD_STRIDE 4 #define ADD_DEFINE() {} #define ADD_D 0, 1, 2, 3 #define MUL_STRIDE 4 #define MUL_DEFINE() {} #define MUL_D 0, 1, 2, 3 #define GEN_P_STRIDE 4 #define GEN_P_DEFINE() {} #define GEN_P_P 0, 1, 2, 3 #define GEN_PQ_STRIDE 4 #define GEN_PQ_DEFINE() {} #define GEN_PQ_D 0, 1, 2, 3 #define GEN_PQ_C 4, 5, 6, 7 #define GEN_PQR_STRIDE 4 #define GEN_PQR_DEFINE() {} #define GEN_PQR_D 0, 1, 2, 3 #define GEN_PQR_C 4, 5, 6, 7 #define SYN_Q_DEFINE() {} #define SYN_Q_D 0, 1, 2, 3 #define SYN_Q_X 4, 5, 6, 7 #define SYN_R_DEFINE() {} #define SYN_R_D 0, 1, 2, 3 #define SYN_R_X 4, 5, 6, 7 #define SYN_PQ_DEFINE() {} #define SYN_PQ_D 0, 1, 2, 3 #define SYN_PQ_X 4, 5, 6, 7 #define REC_PQ_STRIDE 2 #define REC_PQ_DEFINE() {} #define REC_PQ_X 0, 1 #define REC_PQ_Y 2, 3 #define REC_PQ_T 4, 5 #define SYN_PR_DEFINE() {} #define SYN_PR_D 0, 1, 2, 3 #define SYN_PR_X 4, 5, 6, 7 #define REC_PR_STRIDE 2 #define REC_PR_DEFINE() {} #define REC_PR_X 0, 1 #define REC_PR_Y 2, 3 #define REC_PR_T 4, 5 #define SYN_QR_DEFINE() {} #define SYN_QR_D 0, 1, 2, 3 #define SYN_QR_X 4, 5, 6, 7 #define REC_QR_STRIDE 2 #define REC_QR_DEFINE() {} #define REC_QR_X 0, 1 #define REC_QR_Y 2, 3 #define REC_QR_T 4, 5 #define SYN_PQR_DEFINE() {} #define SYN_PQR_D 0, 1, 2, 3 #define SYN_PQR_X 4, 5, 6, 7 #define REC_PQR_STRIDE 2 #define REC_PQR_DEFINE() {} #define REC_PQR_X 0, 1 #define REC_PQR_Y 2, 3 #define REC_PQR_Z 4, 5 #define REC_PQR_XS 6, 7 #define REC_PQR_YS 8, 9 #include #include "vdev_raidz_math_impl.h" DEFINE_GEN_METHODS(avx2); DEFINE_REC_METHODS(avx2); static boolean_t raidz_will_avx2_work(void) { return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available()); } const raidz_impl_ops_t vdev_raidz_avx2_impl = { .init = NULL, .fini = NULL, .gen = RAIDZ_GEN_METHODS(avx2), .rec = RAIDZ_REC_METHODS(avx2), .is_supported = &raidz_will_avx2_work, .name = "avx2" }; #endif /* defined(__x86_64) && defined(HAVE_AVX2) */ diff --git a/module/zfs/vdev_raidz_math_avx512bw.c b/module/zfs/vdev_raidz_math_avx512bw.c index 2f545c9ec078..38cdbedf75f3 100644 --- a/module/zfs/vdev_raidz_math_avx512bw.c +++ b/module/zfs/vdev_raidz_math_avx512bw.c @@ -1,409 +1,409 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Romain Dolbeau. All rights reserved. * Copyright (C) 2016 Gvozden NeÅ¡ković. All rights reserved. */ #include #if defined(__x86_64) && defined(HAVE_AVX512BW) #include -#include +#include #define __asm __asm__ __volatile__ #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) #define VR0_(REG, ...) "zmm"#REG #define VR1_(_1, REG, ...) "zmm"#REG #define VR2_(_1, _2, REG, ...) "zmm"#REG #define VR3_(_1, _2, _3, REG, ...) "zmm"#REG #define VR4_(_1, _2, _3, _4, REG, ...) "zmm"#REG #define VR5_(_1, _2, _3, _4, _5, REG, ...) "zmm"#REG #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "zmm"#REG #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "zmm"#REG #define VR0(r...) VR0_(r) #define VR1(r...) VR1_(r) #define VR2(r...) VR2_(r, 1) #define VR3(r...) VR3_(r, 1, 2) #define VR4(r...) VR4_(r, 1, 2) #define VR5(r...) VR5_(r, 1, 2, 3) #define VR6(r...) VR6_(r, 1, 2, 3, 4) #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) #define R_01(REG1, REG2, ...) REG1, REG2 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 #define R_23(REG...) _R_23(REG, 1, 2, 3) #define ZFS_ASM_BUG() ASSERT(0) extern const uint8_t gf_clmul_mod_lt[4*256][16]; #define ELEM_SIZE 64 typedef struct v { uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); } v_t; #define XOR_ACC(src, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ "vpxorq 0x80(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ "vpxorq 0xc0(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ : : [SRC] "r" (src)); \ break; \ case 2: \ __asm( \ "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ : : [SRC] "r" (src)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define XOR(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "vpxorq %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ "vpxorq %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ "vpxorq %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ "vpxorq %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ break; \ case 4: \ __asm( \ "vpxorq %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ "vpxorq %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define ZERO(r...) XOR(r, r) #define COPY(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "vmovdqa64 %" VR0(r) ", %" VR4(r) "\n" \ "vmovdqa64 %" VR1(r) ", %" VR5(r) "\n" \ "vmovdqa64 %" VR2(r) ", %" VR6(r) "\n" \ "vmovdqa64 %" VR3(r) ", %" VR7(r)); \ break; \ case 4: \ __asm( \ "vmovdqa64 %" VR0(r) ", %" VR2(r) "\n" \ "vmovdqa64 %" VR1(r) ", %" VR3(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define LOAD(src, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \ "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \ "vmovdqa64 0x80(%[SRC]), %%" VR2(r) "\n" \ "vmovdqa64 0xc0(%[SRC]), %%" VR3(r) "\n" \ : : [SRC] "r" (src)); \ break; \ case 2: \ __asm( \ "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \ "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \ : : [SRC] "r" (src)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define STORE(dst, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \ "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \ "vmovdqa64 %%" VR2(r) ", 0x80(%[DST])\n" \ "vmovdqa64 %%" VR3(r) ", 0xc0(%[DST])\n" \ : : [DST] "r" (dst)); \ break; \ case 2: \ __asm( \ "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \ "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \ : : [DST] "r" (dst)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL2_SETUP() \ { \ __asm("vmovq %0, %%xmm22" :: "r"(0x1d1d1d1d1d1d1d1d)); \ __asm("vpbroadcastq %xmm22, %zmm22"); \ __asm("vpxord %zmm23, %zmm23 ,%zmm23"); \ } #define _MUL2(r...) \ { \ switch (REG_CNT(r)) { \ case 2: \ __asm( \ "vpcmpb $1, %zmm23, %" VR0(r)", %k1\n" \ "vpcmpb $1, %zmm23, %" VR1(r)", %k2\n" \ "vpaddb %" VR0(r)", %" VR0(r)", %" VR0(r) "\n" \ "vpaddb %" VR1(r)", %" VR1(r)", %" VR1(r) "\n" \ "vpxord %zmm22, %" VR0(r)", %zmm12\n" \ "vpxord %zmm22, %" VR1(r)", %zmm13\n" \ "vmovdqu8 %zmm12, %" VR0(r) "{%k1}\n" \ "vmovdqu8 %zmm13, %" VR1(r) "{%k2}"); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL2(r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ _MUL2(R_01(r)); \ _MUL2(R_23(r)); \ break; \ case 2: \ _MUL2(r); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL4(r...) \ { \ MUL2(r); \ MUL2(r); \ } #define _0f "zmm15" #define _as "zmm14" #define _bs "zmm13" #define _ltmod "zmm12" #define _ltmul "zmm11" #define _ta "zmm10" #define _tb "zmm15" static const uint8_t __attribute__((aligned(64))) _mul_mask = 0x0F; #define _MULx2(c, r...) \ { \ switch (REG_CNT(r)) { \ case 2: \ __asm( \ "vpbroadcastb (%[mask]), %%" _0f "\n" \ /* upper bits */ \ "vbroadcasti32x4 0x00(%[lt]), %%" _ltmod "\n" \ "vbroadcasti32x4 0x10(%[lt]), %%" _ltmul "\n" \ \ "vpsraw $0x4, %%" VR0(r) ", %%"_as "\n" \ "vpsraw $0x4, %%" VR1(r) ", %%"_bs "\n" \ "vpandq %%" _0f ", %%" VR0(r) ", %%" VR0(r) "\n" \ "vpandq %%" _0f ", %%" VR1(r) ", %%" VR1(r) "\n" \ "vpandq %%" _0f ", %%" _as ", %%" _as "\n" \ "vpandq %%" _0f ", %%" _bs ", %%" _bs "\n" \ \ "vpshufb %%" _as ", %%" _ltmod ", %%" _ta "\n" \ "vpshufb %%" _bs ", %%" _ltmod ", %%" _tb "\n" \ "vpshufb %%" _as ", %%" _ltmul ", %%" _as "\n" \ "vpshufb %%" _bs ", %%" _ltmul ", %%" _bs "\n" \ /* lower bits */ \ "vbroadcasti32x4 0x20(%[lt]), %%" _ltmod "\n" \ "vbroadcasti32x4 0x30(%[lt]), %%" _ltmul "\n" \ \ "vpxorq %%" _ta ", %%" _as ", %%" _as "\n" \ "vpxorq %%" _tb ", %%" _bs ", %%" _bs "\n" \ \ "vpshufb %%" VR0(r) ", %%" _ltmod ", %%" _ta "\n" \ "vpshufb %%" VR1(r) ", %%" _ltmod ", %%" _tb "\n" \ "vpshufb %%" VR0(r) ", %%" _ltmul ", %%" VR0(r) "\n"\ "vpshufb %%" VR1(r) ", %%" _ltmul ", %%" VR1(r) "\n"\ \ "vpxorq %%" _ta ", %%" VR0(r) ", %%" VR0(r) "\n" \ "vpxorq %%" _as ", %%" VR0(r) ", %%" VR0(r) "\n" \ "vpxorq %%" _tb ", %%" VR1(r) ", %%" VR1(r) "\n" \ "vpxorq %%" _bs ", %%" VR1(r) ", %%" VR1(r) "\n" \ : : [mask] "r" (&_mul_mask), \ [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL(c, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ _MULx2(c, R_01(r)); \ _MULx2(c, R_23(r)); \ break; \ case 2: \ _MULx2(c, R_01(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define raidz_math_begin() kfpu_begin() #define raidz_math_end() kfpu_end() /* * ZERO, COPY, and MUL operations are already 2x unrolled, which means that * the stride of these operations for avx512 must not exceed 4. Otherwise, a * single step would exceed 512B block size. */ #define SYN_STRIDE 4 #define ZERO_STRIDE 4 #define ZERO_DEFINE() {} #define ZERO_D 0, 1, 2, 3 #define COPY_STRIDE 4 #define COPY_DEFINE() {} #define COPY_D 0, 1, 2, 3 #define ADD_STRIDE 4 #define ADD_DEFINE() {} #define ADD_D 0, 1, 2, 3 #define MUL_STRIDE 4 #define MUL_DEFINE() {} #define MUL_D 0, 1, 2, 3 #define GEN_P_STRIDE 4 #define GEN_P_DEFINE() {} #define GEN_P_P 0, 1, 2, 3 #define GEN_PQ_STRIDE 4 #define GEN_PQ_DEFINE() {} #define GEN_PQ_D 0, 1, 2, 3 #define GEN_PQ_C 4, 5, 6, 7 #define GEN_PQR_STRIDE 4 #define GEN_PQR_DEFINE() {} #define GEN_PQR_D 0, 1, 2, 3 #define GEN_PQR_C 4, 5, 6, 7 #define SYN_Q_DEFINE() {} #define SYN_Q_D 0, 1, 2, 3 #define SYN_Q_X 4, 5, 6, 7 #define SYN_R_DEFINE() {} #define SYN_R_D 0, 1, 2, 3 #define SYN_R_X 4, 5, 6, 7 #define SYN_PQ_DEFINE() {} #define SYN_PQ_D 0, 1, 2, 3 #define SYN_PQ_X 4, 5, 6, 7 #define REC_PQ_STRIDE 2 #define REC_PQ_DEFINE() {} #define REC_PQ_X 0, 1 #define REC_PQ_Y 2, 3 #define REC_PQ_T 4, 5 #define SYN_PR_DEFINE() {} #define SYN_PR_D 0, 1, 2, 3 #define SYN_PR_X 4, 5, 6, 7 #define REC_PR_STRIDE 2 #define REC_PR_DEFINE() {} #define REC_PR_X 0, 1 #define REC_PR_Y 2, 3 #define REC_PR_T 4, 5 #define SYN_QR_DEFINE() {} #define SYN_QR_D 0, 1, 2, 3 #define SYN_QR_X 4, 5, 6, 7 #define REC_QR_STRIDE 2 #define REC_QR_DEFINE() {} #define REC_QR_X 0, 1 #define REC_QR_Y 2, 3 #define REC_QR_T 4, 5 #define SYN_PQR_DEFINE() {} #define SYN_PQR_D 0, 1, 2, 3 #define SYN_PQR_X 4, 5, 6, 7 #define REC_PQR_STRIDE 2 #define REC_PQR_DEFINE() {} #define REC_PQR_X 0, 1 #define REC_PQR_Y 2, 3 #define REC_PQR_Z 4, 5 #define REC_PQR_XS 6, 7 #define REC_PQR_YS 8, 9 #include #include "vdev_raidz_math_impl.h" DEFINE_GEN_METHODS(avx512bw); DEFINE_REC_METHODS(avx512bw); static boolean_t raidz_will_avx512bw_work(void) { return (kfpu_allowed() && zfs_avx_available() && zfs_avx512f_available() && zfs_avx512bw_available()); } const raidz_impl_ops_t vdev_raidz_avx512bw_impl = { .init = NULL, .fini = NULL, .gen = RAIDZ_GEN_METHODS(avx512bw), .rec = RAIDZ_REC_METHODS(avx512bw), .is_supported = &raidz_will_avx512bw_work, .name = "avx512bw" }; #endif /* defined(__x86_64) && defined(HAVE_AVX512BW) */ diff --git a/module/zfs/vdev_raidz_math_avx512f.c b/module/zfs/vdev_raidz_math_avx512f.c index 75af7a8eea96..adbe9b0ef185 100644 --- a/module/zfs/vdev_raidz_math_avx512f.c +++ b/module/zfs/vdev_raidz_math_avx512f.c @@ -1,486 +1,486 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Romain Dolbeau. All rights reserved. * Copyright (C) 2016 Gvozden NeÅ¡ković. All rights reserved. */ #include #if defined(__x86_64) && defined(HAVE_AVX512F) #include -#include +#include #define __asm __asm__ __volatile__ #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) #define VR0_(REG, ...) "zmm"#REG #define VR1_(_1, REG, ...) "zmm"#REG #define VR2_(_1, _2, REG, ...) "zmm"#REG #define VR3_(_1, _2, _3, REG, ...) "zmm"#REG #define VR4_(_1, _2, _3, _4, REG, ...) "zmm"#REG #define VR5_(_1, _2, _3, _4, _5, REG, ...) "zmm"#REG #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "zmm"#REG #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "zmm"#REG #define VR0(r...) VR0_(r) #define VR1(r...) VR1_(r) #define VR2(r...) VR2_(r, 1) #define VR3(r...) VR3_(r, 1, 2) #define VR4(r...) VR4_(r, 1, 2) #define VR5(r...) VR5_(r, 1, 2, 3) #define VR6(r...) VR6_(r, 1, 2, 3, 4) #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) #define VRy0_(REG, ...) "ymm"#REG #define VRy1_(_1, REG, ...) "ymm"#REG #define VRy2_(_1, _2, REG, ...) "ymm"#REG #define VRy3_(_1, _2, _3, REG, ...) "ymm"#REG #define VRy4_(_1, _2, _3, _4, REG, ...) "ymm"#REG #define VRy5_(_1, _2, _3, _4, _5, REG, ...) "ymm"#REG #define VRy6_(_1, _2, _3, _4, _5, _6, REG, ...) "ymm"#REG #define VRy7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "ymm"#REG #define VRy0(r...) VRy0_(r) #define VRy1(r...) VRy1_(r) #define VRy2(r...) VRy2_(r, 1) #define VRy3(r...) VRy3_(r, 1, 2) #define VRy4(r...) VRy4_(r, 1, 2) #define VRy5(r...) VRy5_(r, 1, 2, 3) #define VRy6(r...) VRy6_(r, 1, 2, 3, 4) #define VRy7(r...) VRy7_(r, 1, 2, 3, 4, 5) #define R_01(REG1, REG2, ...) REG1, REG2 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 #define R_23(REG...) _R_23(REG, 1, 2, 3) #define ELEM_SIZE 64 typedef struct v { uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); } v_t; #define XOR_ACC(src, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "vpxorq 0x00(%[SRC]), %%" VR0(r)", %%" VR0(r) "\n" \ "vpxorq 0x40(%[SRC]), %%" VR1(r)", %%" VR1(r) "\n" \ "vpxorq 0x80(%[SRC]), %%" VR2(r)", %%" VR2(r) "\n" \ "vpxorq 0xc0(%[SRC]), %%" VR3(r)", %%" VR3(r) "\n" \ : : [SRC] "r" (src)); \ break; \ } \ } #define XOR(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "vpxorq %" VR0(r) ", %" VR4(r)", %" VR4(r) "\n" \ "vpxorq %" VR1(r) ", %" VR5(r)", %" VR5(r) "\n" \ "vpxorq %" VR2(r) ", %" VR6(r)", %" VR6(r) "\n" \ "vpxorq %" VR3(r) ", %" VR7(r)", %" VR7(r)); \ break; \ case 4: \ __asm( \ "vpxorq %" VR0(r) ", %" VR2(r)", %" VR2(r) "\n" \ "vpxorq %" VR1(r) ", %" VR3(r)", %" VR3(r)); \ break; \ } \ } #define ZERO(r...) XOR(r, r) #define COPY(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "vmovdqa64 %" VR0(r) ", %" VR4(r) "\n" \ "vmovdqa64 %" VR1(r) ", %" VR5(r) "\n" \ "vmovdqa64 %" VR2(r) ", %" VR6(r) "\n" \ "vmovdqa64 %" VR3(r) ", %" VR7(r)); \ break; \ case 4: \ __asm( \ "vmovdqa64 %" VR0(r) ", %" VR2(r) "\n" \ "vmovdqa64 %" VR1(r) ", %" VR3(r)); \ break; \ } \ } #define LOAD(src, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "vmovdqa64 0x00(%[SRC]), %%" VR0(r) "\n" \ "vmovdqa64 0x40(%[SRC]), %%" VR1(r) "\n" \ "vmovdqa64 0x80(%[SRC]), %%" VR2(r) "\n" \ "vmovdqa64 0xc0(%[SRC]), %%" VR3(r) "\n" \ : : [SRC] "r" (src)); \ break; \ } \ } #define STORE(dst, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "vmovdqa64 %%" VR0(r) ", 0x00(%[DST])\n" \ "vmovdqa64 %%" VR1(r) ", 0x40(%[DST])\n" \ "vmovdqa64 %%" VR2(r) ", 0x80(%[DST])\n" \ "vmovdqa64 %%" VR3(r) ", 0xc0(%[DST])\n" \ : : [DST] "r" (dst)); \ break; \ } \ } #define MUL2_SETUP() \ { \ __asm("vmovq %0, %%xmm31" :: "r"(0x1d1d1d1d1d1d1d1d)); \ __asm("vpbroadcastq %xmm31, %zmm31"); \ __asm("vmovq %0, %%xmm30" :: "r"(0x8080808080808080)); \ __asm("vpbroadcastq %xmm30, %zmm30"); \ __asm("vmovq %0, %%xmm29" :: "r"(0xfefefefefefefefe)); \ __asm("vpbroadcastq %xmm29, %zmm29"); \ } #define _MUL2(r...) \ { \ switch (REG_CNT(r)) { \ case 2: \ __asm( \ "vpandq %" VR0(r)", %zmm30, %zmm26\n" \ "vpandq %" VR1(r)", %zmm30, %zmm25\n" \ "vpsrlq $7, %zmm26, %zmm28\n" \ "vpsrlq $7, %zmm25, %zmm27\n" \ "vpsllq $1, %zmm26, %zmm26\n" \ "vpsllq $1, %zmm25, %zmm25\n" \ "vpsubq %zmm28, %zmm26, %zmm26\n" \ "vpsubq %zmm27, %zmm25, %zmm25\n" \ "vpsllq $1, %" VR0(r)", %" VR0(r) "\n" \ "vpsllq $1, %" VR1(r)", %" VR1(r) "\n" \ "vpandq %zmm26, %zmm31, %zmm26\n" \ "vpandq %zmm25, %zmm31, %zmm25\n" \ "vpternlogd $0x6c,%zmm29, %zmm26, %" VR0(r) "\n" \ "vpternlogd $0x6c,%zmm29, %zmm25, %" VR1(r)); \ break; \ } \ } #define MUL2(r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ _MUL2(R_01(r)); \ _MUL2(R_23(r)); \ break; \ case 2: \ _MUL2(r); \ break; \ } \ } #define MUL4(r...) \ { \ MUL2(r); \ MUL2(r); \ } /* General multiplication by adding powers of two */ #define _mul_x2_in 21, 22 #define _mul_x2_acc 23, 24 #define _MUL_PARAM(x, in, acc) \ { \ if (x & 0x01) { COPY(in, acc); } else { ZERO(acc); } \ if (x & 0xfe) { MUL2(in); } \ if (x & 0x02) { XOR(in, acc); } \ if (x & 0xfc) { MUL2(in); } \ if (x & 0x04) { XOR(in, acc); } \ if (x & 0xf8) { MUL2(in); } \ if (x & 0x08) { XOR(in, acc); } \ if (x & 0xf0) { MUL2(in); } \ if (x & 0x10) { XOR(in, acc); } \ if (x & 0xe0) { MUL2(in); } \ if (x & 0x20) { XOR(in, acc); } \ if (x & 0xc0) { MUL2(in); } \ if (x & 0x40) { XOR(in, acc); } \ if (x & 0x80) { MUL2(in); XOR(in, acc); } \ } #define MUL_x2_DEFINE(x) \ static void \ mul_x2_ ## x(void) { _MUL_PARAM(x, _mul_x2_in, _mul_x2_acc); } MUL_x2_DEFINE(0); MUL_x2_DEFINE(1); MUL_x2_DEFINE(2); MUL_x2_DEFINE(3); MUL_x2_DEFINE(4); MUL_x2_DEFINE(5); MUL_x2_DEFINE(6); MUL_x2_DEFINE(7); MUL_x2_DEFINE(8); MUL_x2_DEFINE(9); MUL_x2_DEFINE(10); MUL_x2_DEFINE(11); MUL_x2_DEFINE(12); MUL_x2_DEFINE(13); MUL_x2_DEFINE(14); MUL_x2_DEFINE(15); MUL_x2_DEFINE(16); MUL_x2_DEFINE(17); MUL_x2_DEFINE(18); MUL_x2_DEFINE(19); MUL_x2_DEFINE(20); MUL_x2_DEFINE(21); MUL_x2_DEFINE(22); MUL_x2_DEFINE(23); MUL_x2_DEFINE(24); MUL_x2_DEFINE(25); MUL_x2_DEFINE(26); MUL_x2_DEFINE(27); MUL_x2_DEFINE(28); MUL_x2_DEFINE(29); MUL_x2_DEFINE(30); MUL_x2_DEFINE(31); MUL_x2_DEFINE(32); MUL_x2_DEFINE(33); MUL_x2_DEFINE(34); MUL_x2_DEFINE(35); MUL_x2_DEFINE(36); MUL_x2_DEFINE(37); MUL_x2_DEFINE(38); MUL_x2_DEFINE(39); MUL_x2_DEFINE(40); MUL_x2_DEFINE(41); MUL_x2_DEFINE(42); MUL_x2_DEFINE(43); MUL_x2_DEFINE(44); MUL_x2_DEFINE(45); MUL_x2_DEFINE(46); MUL_x2_DEFINE(47); MUL_x2_DEFINE(48); MUL_x2_DEFINE(49); MUL_x2_DEFINE(50); MUL_x2_DEFINE(51); MUL_x2_DEFINE(52); MUL_x2_DEFINE(53); MUL_x2_DEFINE(54); MUL_x2_DEFINE(55); MUL_x2_DEFINE(56); MUL_x2_DEFINE(57); MUL_x2_DEFINE(58); MUL_x2_DEFINE(59); MUL_x2_DEFINE(60); MUL_x2_DEFINE(61); MUL_x2_DEFINE(62); MUL_x2_DEFINE(63); MUL_x2_DEFINE(64); MUL_x2_DEFINE(65); MUL_x2_DEFINE(66); MUL_x2_DEFINE(67); MUL_x2_DEFINE(68); MUL_x2_DEFINE(69); MUL_x2_DEFINE(70); MUL_x2_DEFINE(71); MUL_x2_DEFINE(72); MUL_x2_DEFINE(73); MUL_x2_DEFINE(74); MUL_x2_DEFINE(75); MUL_x2_DEFINE(76); MUL_x2_DEFINE(77); MUL_x2_DEFINE(78); MUL_x2_DEFINE(79); MUL_x2_DEFINE(80); MUL_x2_DEFINE(81); MUL_x2_DEFINE(82); MUL_x2_DEFINE(83); MUL_x2_DEFINE(84); MUL_x2_DEFINE(85); MUL_x2_DEFINE(86); MUL_x2_DEFINE(87); MUL_x2_DEFINE(88); MUL_x2_DEFINE(89); MUL_x2_DEFINE(90); MUL_x2_DEFINE(91); MUL_x2_DEFINE(92); MUL_x2_DEFINE(93); MUL_x2_DEFINE(94); MUL_x2_DEFINE(95); MUL_x2_DEFINE(96); MUL_x2_DEFINE(97); MUL_x2_DEFINE(98); MUL_x2_DEFINE(99); MUL_x2_DEFINE(100); MUL_x2_DEFINE(101); MUL_x2_DEFINE(102); MUL_x2_DEFINE(103); MUL_x2_DEFINE(104); MUL_x2_DEFINE(105); MUL_x2_DEFINE(106); MUL_x2_DEFINE(107); MUL_x2_DEFINE(108); MUL_x2_DEFINE(109); MUL_x2_DEFINE(110); MUL_x2_DEFINE(111); MUL_x2_DEFINE(112); MUL_x2_DEFINE(113); MUL_x2_DEFINE(114); MUL_x2_DEFINE(115); MUL_x2_DEFINE(116); MUL_x2_DEFINE(117); MUL_x2_DEFINE(118); MUL_x2_DEFINE(119); MUL_x2_DEFINE(120); MUL_x2_DEFINE(121); MUL_x2_DEFINE(122); MUL_x2_DEFINE(123); MUL_x2_DEFINE(124); MUL_x2_DEFINE(125); MUL_x2_DEFINE(126); MUL_x2_DEFINE(127); MUL_x2_DEFINE(128); MUL_x2_DEFINE(129); MUL_x2_DEFINE(130); MUL_x2_DEFINE(131); MUL_x2_DEFINE(132); MUL_x2_DEFINE(133); MUL_x2_DEFINE(134); MUL_x2_DEFINE(135); MUL_x2_DEFINE(136); MUL_x2_DEFINE(137); MUL_x2_DEFINE(138); MUL_x2_DEFINE(139); MUL_x2_DEFINE(140); MUL_x2_DEFINE(141); MUL_x2_DEFINE(142); MUL_x2_DEFINE(143); MUL_x2_DEFINE(144); MUL_x2_DEFINE(145); MUL_x2_DEFINE(146); MUL_x2_DEFINE(147); MUL_x2_DEFINE(148); MUL_x2_DEFINE(149); MUL_x2_DEFINE(150); MUL_x2_DEFINE(151); MUL_x2_DEFINE(152); MUL_x2_DEFINE(153); MUL_x2_DEFINE(154); MUL_x2_DEFINE(155); MUL_x2_DEFINE(156); MUL_x2_DEFINE(157); MUL_x2_DEFINE(158); MUL_x2_DEFINE(159); MUL_x2_DEFINE(160); MUL_x2_DEFINE(161); MUL_x2_DEFINE(162); MUL_x2_DEFINE(163); MUL_x2_DEFINE(164); MUL_x2_DEFINE(165); MUL_x2_DEFINE(166); MUL_x2_DEFINE(167); MUL_x2_DEFINE(168); MUL_x2_DEFINE(169); MUL_x2_DEFINE(170); MUL_x2_DEFINE(171); MUL_x2_DEFINE(172); MUL_x2_DEFINE(173); MUL_x2_DEFINE(174); MUL_x2_DEFINE(175); MUL_x2_DEFINE(176); MUL_x2_DEFINE(177); MUL_x2_DEFINE(178); MUL_x2_DEFINE(179); MUL_x2_DEFINE(180); MUL_x2_DEFINE(181); MUL_x2_DEFINE(182); MUL_x2_DEFINE(183); MUL_x2_DEFINE(184); MUL_x2_DEFINE(185); MUL_x2_DEFINE(186); MUL_x2_DEFINE(187); MUL_x2_DEFINE(188); MUL_x2_DEFINE(189); MUL_x2_DEFINE(190); MUL_x2_DEFINE(191); MUL_x2_DEFINE(192); MUL_x2_DEFINE(193); MUL_x2_DEFINE(194); MUL_x2_DEFINE(195); MUL_x2_DEFINE(196); MUL_x2_DEFINE(197); MUL_x2_DEFINE(198); MUL_x2_DEFINE(199); MUL_x2_DEFINE(200); MUL_x2_DEFINE(201); MUL_x2_DEFINE(202); MUL_x2_DEFINE(203); MUL_x2_DEFINE(204); MUL_x2_DEFINE(205); MUL_x2_DEFINE(206); MUL_x2_DEFINE(207); MUL_x2_DEFINE(208); MUL_x2_DEFINE(209); MUL_x2_DEFINE(210); MUL_x2_DEFINE(211); MUL_x2_DEFINE(212); MUL_x2_DEFINE(213); MUL_x2_DEFINE(214); MUL_x2_DEFINE(215); MUL_x2_DEFINE(216); MUL_x2_DEFINE(217); MUL_x2_DEFINE(218); MUL_x2_DEFINE(219); MUL_x2_DEFINE(220); MUL_x2_DEFINE(221); MUL_x2_DEFINE(222); MUL_x2_DEFINE(223); MUL_x2_DEFINE(224); MUL_x2_DEFINE(225); MUL_x2_DEFINE(226); MUL_x2_DEFINE(227); MUL_x2_DEFINE(228); MUL_x2_DEFINE(229); MUL_x2_DEFINE(230); MUL_x2_DEFINE(231); MUL_x2_DEFINE(232); MUL_x2_DEFINE(233); MUL_x2_DEFINE(234); MUL_x2_DEFINE(235); MUL_x2_DEFINE(236); MUL_x2_DEFINE(237); MUL_x2_DEFINE(238); MUL_x2_DEFINE(239); MUL_x2_DEFINE(240); MUL_x2_DEFINE(241); MUL_x2_DEFINE(242); MUL_x2_DEFINE(243); MUL_x2_DEFINE(244); MUL_x2_DEFINE(245); MUL_x2_DEFINE(246); MUL_x2_DEFINE(247); MUL_x2_DEFINE(248); MUL_x2_DEFINE(249); MUL_x2_DEFINE(250); MUL_x2_DEFINE(251); MUL_x2_DEFINE(252); MUL_x2_DEFINE(253); MUL_x2_DEFINE(254); MUL_x2_DEFINE(255); typedef void (*mul_fn_ptr_t)(void); static const mul_fn_ptr_t __attribute__((aligned(256))) gf_x2_mul_fns[256] = { mul_x2_0, mul_x2_1, mul_x2_2, mul_x2_3, mul_x2_4, mul_x2_5, mul_x2_6, mul_x2_7, mul_x2_8, mul_x2_9, mul_x2_10, mul_x2_11, mul_x2_12, mul_x2_13, mul_x2_14, mul_x2_15, mul_x2_16, mul_x2_17, mul_x2_18, mul_x2_19, mul_x2_20, mul_x2_21, mul_x2_22, mul_x2_23, mul_x2_24, mul_x2_25, mul_x2_26, mul_x2_27, mul_x2_28, mul_x2_29, mul_x2_30, mul_x2_31, mul_x2_32, mul_x2_33, mul_x2_34, mul_x2_35, mul_x2_36, mul_x2_37, mul_x2_38, mul_x2_39, mul_x2_40, mul_x2_41, mul_x2_42, mul_x2_43, mul_x2_44, mul_x2_45, mul_x2_46, mul_x2_47, mul_x2_48, mul_x2_49, mul_x2_50, mul_x2_51, mul_x2_52, mul_x2_53, mul_x2_54, mul_x2_55, mul_x2_56, mul_x2_57, mul_x2_58, mul_x2_59, mul_x2_60, mul_x2_61, mul_x2_62, mul_x2_63, mul_x2_64, mul_x2_65, mul_x2_66, mul_x2_67, mul_x2_68, mul_x2_69, mul_x2_70, mul_x2_71, mul_x2_72, mul_x2_73, mul_x2_74, mul_x2_75, mul_x2_76, mul_x2_77, mul_x2_78, mul_x2_79, mul_x2_80, mul_x2_81, mul_x2_82, mul_x2_83, mul_x2_84, mul_x2_85, mul_x2_86, mul_x2_87, mul_x2_88, mul_x2_89, mul_x2_90, mul_x2_91, mul_x2_92, mul_x2_93, mul_x2_94, mul_x2_95, mul_x2_96, mul_x2_97, mul_x2_98, mul_x2_99, mul_x2_100, mul_x2_101, mul_x2_102, mul_x2_103, mul_x2_104, mul_x2_105, mul_x2_106, mul_x2_107, mul_x2_108, mul_x2_109, mul_x2_110, mul_x2_111, mul_x2_112, mul_x2_113, mul_x2_114, mul_x2_115, mul_x2_116, mul_x2_117, mul_x2_118, mul_x2_119, mul_x2_120, mul_x2_121, mul_x2_122, mul_x2_123, mul_x2_124, mul_x2_125, mul_x2_126, mul_x2_127, mul_x2_128, mul_x2_129, mul_x2_130, mul_x2_131, mul_x2_132, mul_x2_133, mul_x2_134, mul_x2_135, mul_x2_136, mul_x2_137, mul_x2_138, mul_x2_139, mul_x2_140, mul_x2_141, mul_x2_142, mul_x2_143, mul_x2_144, mul_x2_145, mul_x2_146, mul_x2_147, mul_x2_148, mul_x2_149, mul_x2_150, mul_x2_151, mul_x2_152, mul_x2_153, mul_x2_154, mul_x2_155, mul_x2_156, mul_x2_157, mul_x2_158, mul_x2_159, mul_x2_160, mul_x2_161, mul_x2_162, mul_x2_163, mul_x2_164, mul_x2_165, mul_x2_166, mul_x2_167, mul_x2_168, mul_x2_169, mul_x2_170, mul_x2_171, mul_x2_172, mul_x2_173, mul_x2_174, mul_x2_175, mul_x2_176, mul_x2_177, mul_x2_178, mul_x2_179, mul_x2_180, mul_x2_181, mul_x2_182, mul_x2_183, mul_x2_184, mul_x2_185, mul_x2_186, mul_x2_187, mul_x2_188, mul_x2_189, mul_x2_190, mul_x2_191, mul_x2_192, mul_x2_193, mul_x2_194, mul_x2_195, mul_x2_196, mul_x2_197, mul_x2_198, mul_x2_199, mul_x2_200, mul_x2_201, mul_x2_202, mul_x2_203, mul_x2_204, mul_x2_205, mul_x2_206, mul_x2_207, mul_x2_208, mul_x2_209, mul_x2_210, mul_x2_211, mul_x2_212, mul_x2_213, mul_x2_214, mul_x2_215, mul_x2_216, mul_x2_217, mul_x2_218, mul_x2_219, mul_x2_220, mul_x2_221, mul_x2_222, mul_x2_223, mul_x2_224, mul_x2_225, mul_x2_226, mul_x2_227, mul_x2_228, mul_x2_229, mul_x2_230, mul_x2_231, mul_x2_232, mul_x2_233, mul_x2_234, mul_x2_235, mul_x2_236, mul_x2_237, mul_x2_238, mul_x2_239, mul_x2_240, mul_x2_241, mul_x2_242, mul_x2_243, mul_x2_244, mul_x2_245, mul_x2_246, mul_x2_247, mul_x2_248, mul_x2_249, mul_x2_250, mul_x2_251, mul_x2_252, mul_x2_253, mul_x2_254, mul_x2_255 }; #define MUL(c, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ COPY(R_01(r), _mul_x2_in); \ gf_x2_mul_fns[c](); \ COPY(_mul_x2_acc, R_01(r)); \ COPY(R_23(r), _mul_x2_in); \ gf_x2_mul_fns[c](); \ COPY(_mul_x2_acc, R_23(r)); \ } \ } #define raidz_math_begin() kfpu_begin() #define raidz_math_end() kfpu_end() #define SYN_STRIDE 4 #define ZERO_STRIDE 4 #define ZERO_DEFINE() {} #define ZERO_D 0, 1, 2, 3 #define COPY_STRIDE 4 #define COPY_DEFINE() {} #define COPY_D 0, 1, 2, 3 #define ADD_STRIDE 4 #define ADD_DEFINE() {} #define ADD_D 0, 1, 2, 3 #define MUL_STRIDE 4 #define MUL_DEFINE() MUL2_SETUP() #define MUL_D 0, 1, 2, 3 #define GEN_P_STRIDE 4 #define GEN_P_DEFINE() {} #define GEN_P_P 0, 1, 2, 3 #define GEN_PQ_STRIDE 4 #define GEN_PQ_DEFINE() {} #define GEN_PQ_D 0, 1, 2, 3 #define GEN_PQ_C 4, 5, 6, 7 #define GEN_PQR_STRIDE 4 #define GEN_PQR_DEFINE() {} #define GEN_PQR_D 0, 1, 2, 3 #define GEN_PQR_C 4, 5, 6, 7 #define SYN_Q_DEFINE() {} #define SYN_Q_D 0, 1, 2, 3 #define SYN_Q_X 4, 5, 6, 7 #define SYN_R_DEFINE() {} #define SYN_R_D 0, 1, 2, 3 #define SYN_R_X 4, 5, 6, 7 #define SYN_PQ_DEFINE() {} #define SYN_PQ_D 0, 1, 2, 3 #define SYN_PQ_X 4, 5, 6, 7 #define REC_PQ_STRIDE 4 #define REC_PQ_DEFINE() MUL2_SETUP() #define REC_PQ_X 0, 1, 2, 3 #define REC_PQ_Y 4, 5, 6, 7 #define REC_PQ_T 8, 9, 10, 11 #define SYN_PR_DEFINE() {} #define SYN_PR_D 0, 1, 2, 3 #define SYN_PR_X 4, 5, 6, 7 #define REC_PR_STRIDE 4 #define REC_PR_DEFINE() MUL2_SETUP() #define REC_PR_X 0, 1, 2, 3 #define REC_PR_Y 4, 5, 6, 7 #define REC_PR_T 8, 9, 10, 11 #define SYN_QR_DEFINE() {} #define SYN_QR_D 0, 1, 2, 3 #define SYN_QR_X 4, 5, 6, 7 #define REC_QR_STRIDE 4 #define REC_QR_DEFINE() MUL2_SETUP() #define REC_QR_X 0, 1, 2, 3 #define REC_QR_Y 4, 5, 6, 7 #define REC_QR_T 8, 9, 10, 11 #define SYN_PQR_DEFINE() {} #define SYN_PQR_D 0, 1, 2, 3 #define SYN_PQR_X 4, 5, 6, 7 #define REC_PQR_STRIDE 4 #define REC_PQR_DEFINE() MUL2_SETUP() #define REC_PQR_X 0, 1, 2, 3 #define REC_PQR_Y 4, 5, 6, 7 #define REC_PQR_Z 8, 9, 10, 11 #define REC_PQR_XS 12, 13, 14, 15 #define REC_PQR_YS 16, 17, 18, 19 #include #include "vdev_raidz_math_impl.h" DEFINE_GEN_METHODS(avx512f); DEFINE_REC_METHODS(avx512f); static boolean_t raidz_will_avx512f_work(void) { return (kfpu_allowed() && zfs_avx_available() && zfs_avx2_available() && zfs_avx512f_available()); } const raidz_impl_ops_t vdev_raidz_avx512f_impl = { .init = NULL, .fini = NULL, .gen = RAIDZ_GEN_METHODS(avx512f), .rec = RAIDZ_REC_METHODS(avx512f), .is_supported = &raidz_will_avx512f_work, .name = "avx512f" }; #endif /* defined(__x86_64) && defined(HAVE_AVX512F) */ diff --git a/module/zfs/vdev_raidz_math_sse2.c b/module/zfs/vdev_raidz_math_sse2.c index 5b3a9385c9d8..70a21c10c87e 100644 --- a/module/zfs/vdev_raidz_math_sse2.c +++ b/module/zfs/vdev_raidz_math_sse2.c @@ -1,622 +1,622 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Gvozden NeÅ¡ković. All rights reserved. */ #include #if defined(__x86_64) && defined(HAVE_SSE2) #include -#include +#include #define __asm __asm__ __volatile__ #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) #define VR0_(REG, ...) "xmm"#REG #define VR1_(_1, REG, ...) "xmm"#REG #define VR2_(_1, _2, REG, ...) "xmm"#REG #define VR3_(_1, _2, _3, REG, ...) "xmm"#REG #define VR4_(_1, _2, _3, _4, REG, ...) "xmm"#REG #define VR5_(_1, _2, _3, _4, _5, REG, ...) "xmm"#REG #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "xmm"#REG #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "xmm"#REG #define VR0(r...) VR0_(r, 1, 2, 3, 4, 5, 6) #define VR1(r...) VR1_(r, 1, 2, 3, 4, 5, 6) #define VR2(r...) VR2_(r, 1, 2, 3, 4, 5, 6) #define VR3(r...) VR3_(r, 1, 2, 3, 4, 5, 6) #define VR4(r...) VR4_(r, 1, 2, 3, 4, 5, 6) #define VR5(r...) VR5_(r, 1, 2, 3, 4, 5, 6) #define VR6(r...) VR6_(r, 1, 2, 3, 4, 5, 6) #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5, 6) #define ELEM_SIZE 16 typedef struct v { uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); } v_t; #define XOR_ACC(src, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "pxor 0x00(%[SRC]), %%" VR0(r) "\n" \ "pxor 0x10(%[SRC]), %%" VR1(r) "\n" \ "pxor 0x20(%[SRC]), %%" VR2(r) "\n" \ "pxor 0x30(%[SRC]), %%" VR3(r) "\n" \ : : [SRC] "r" (src)); \ break; \ case 2: \ __asm( \ "pxor 0x00(%[SRC]), %%" VR0(r) "\n" \ "pxor 0x10(%[SRC]), %%" VR1(r) "\n" \ : : [SRC] "r" (src)); \ break; \ case 1: \ __asm("pxor 0x00(%[SRC]), %%" VR0(r) "\n" \ : : [SRC] "r" (src)); \ break; \ } \ } #define XOR(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "pxor %" VR0(r) ", %" VR4(r) "\n" \ "pxor %" VR1(r) ", %" VR5(r) "\n" \ "pxor %" VR2(r) ", %" VR6(r) "\n" \ "pxor %" VR3(r) ", %" VR7(r)); \ break; \ case 4: \ __asm( \ "pxor %" VR0(r) ", %" VR2(r) "\n" \ "pxor %" VR1(r) ", %" VR3(r)); \ break; \ case 2: \ __asm( \ "pxor %" VR0(r) ", %" VR1(r)); \ break; \ } \ } #define ZERO(r...) XOR(r, r) #define COPY(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "movdqa %" VR0(r) ", %" VR4(r) "\n" \ "movdqa %" VR1(r) ", %" VR5(r) "\n" \ "movdqa %" VR2(r) ", %" VR6(r) "\n" \ "movdqa %" VR3(r) ", %" VR7(r)); \ break; \ case 4: \ __asm( \ "movdqa %" VR0(r) ", %" VR2(r) "\n" \ "movdqa %" VR1(r) ", %" VR3(r)); \ break; \ case 2: \ __asm( \ "movdqa %" VR0(r) ", %" VR1(r)); \ break; \ } \ } #define LOAD(src, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "movdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ "movdqa 0x10(%[SRC]), %%" VR1(r) "\n" \ "movdqa 0x20(%[SRC]), %%" VR2(r) "\n" \ "movdqa 0x30(%[SRC]), %%" VR3(r) "\n" \ : : [SRC] "r" (src)); \ break; \ case 2: \ __asm( \ "movdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ "movdqa 0x10(%[SRC]), %%" VR1(r) "\n" \ : : [SRC] "r" (src)); \ break; \ case 1: \ __asm( \ "movdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ : : [SRC] "r" (src)); \ break; \ } \ } #define STORE(dst, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "movdqa %%" VR0(r)", 0x00(%[DST])\n" \ "movdqa %%" VR1(r)", 0x10(%[DST])\n" \ "movdqa %%" VR2(r)", 0x20(%[DST])\n" \ "movdqa %%" VR3(r)", 0x30(%[DST])\n" \ : : [DST] "r" (dst)); \ break; \ case 2: \ __asm( \ "movdqa %%" VR0(r)", 0x00(%[DST])\n" \ "movdqa %%" VR1(r)", 0x10(%[DST])\n" \ : : [DST] "r" (dst)); \ break; \ case 1: \ __asm( \ "movdqa %%" VR0(r)", 0x00(%[DST])\n" \ : : [DST] "r" (dst)); \ break; \ } \ } #define MUL2_SETUP() \ { \ __asm( \ "movd %[mask], %%xmm15\n" \ "pshufd $0x0, %%xmm15, %%xmm15\n" \ : : [mask] "r" (0x1d1d1d1d)); \ } #define _MUL2_x1(a0) \ { \ __asm( \ "pxor %xmm14, %xmm14\n" \ "pcmpgtb %" a0", %xmm14\n" \ "pand %xmm15, %xmm14\n" \ "paddb %" a0", %" a0 "\n" \ "pxor %xmm14, %" a0); \ } #define _MUL2_x2(a0, a1) \ { \ __asm( \ "pxor %xmm14, %xmm14\n" \ "pxor %xmm13, %xmm13\n" \ "pcmpgtb %" a0", %xmm14\n" \ "pcmpgtb %" a1", %xmm13\n" \ "pand %xmm15, %xmm14\n" \ "pand %xmm15, %xmm13\n" \ "paddb %" a0", %" a0 "\n" \ "paddb %" a1", %" a1 "\n" \ "pxor %xmm14, %" a0 "\n" \ "pxor %xmm13, %" a1); \ } #define MUL2(r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ _MUL2_x2(VR0(r), VR1(r)); \ _MUL2_x2(VR2(r), VR3(r)); \ break; \ case 2: \ _MUL2_x2(VR0(r), VR1(r)); \ break; \ case 1: \ _MUL2_x1(VR0(r)); \ break; \ } \ } #define MUL4(r...) \ { \ MUL2(r); \ MUL2(r); \ } /* General multiplication by adding powers of two */ #define _MUL_PARAM(x, in, acc) \ { \ if (x & 0x01) { COPY(in, acc); } else { ZERO(acc); } \ if (x & 0xfe) { MUL2(in); } \ if (x & 0x02) { XOR(in, acc); } \ if (x & 0xfc) { MUL2(in); } \ if (x & 0x04) { XOR(in, acc); } \ if (x & 0xf8) { MUL2(in); } \ if (x & 0x08) { XOR(in, acc); } \ if (x & 0xf0) { MUL2(in); } \ if (x & 0x10) { XOR(in, acc); } \ if (x & 0xe0) { MUL2(in); } \ if (x & 0x20) { XOR(in, acc); } \ if (x & 0xc0) { MUL2(in); } \ if (x & 0x40) { XOR(in, acc); } \ if (x & 0x80) { MUL2(in); XOR(in, acc); } \ } #define _mul_x1_in 11 #define _mul_x1_acc 12 #define MUL_x1_DEFINE(x) \ static void \ mul_x1_ ## x(void) { _MUL_PARAM(x, _mul_x1_in, _mul_x1_acc); } #define _mul_x2_in 9, 10 #define _mul_x2_acc 11, 12 #define MUL_x2_DEFINE(x) \ static void \ mul_x2_ ## x(void) { _MUL_PARAM(x, _mul_x2_in, _mul_x2_acc); } MUL_x1_DEFINE(0); MUL_x1_DEFINE(1); MUL_x1_DEFINE(2); MUL_x1_DEFINE(3); MUL_x1_DEFINE(4); MUL_x1_DEFINE(5); MUL_x1_DEFINE(6); MUL_x1_DEFINE(7); MUL_x1_DEFINE(8); MUL_x1_DEFINE(9); MUL_x1_DEFINE(10); MUL_x1_DEFINE(11); MUL_x1_DEFINE(12); MUL_x1_DEFINE(13); MUL_x1_DEFINE(14); MUL_x1_DEFINE(15); MUL_x1_DEFINE(16); MUL_x1_DEFINE(17); MUL_x1_DEFINE(18); MUL_x1_DEFINE(19); MUL_x1_DEFINE(20); MUL_x1_DEFINE(21); MUL_x1_DEFINE(22); MUL_x1_DEFINE(23); MUL_x1_DEFINE(24); MUL_x1_DEFINE(25); MUL_x1_DEFINE(26); MUL_x1_DEFINE(27); MUL_x1_DEFINE(28); MUL_x1_DEFINE(29); MUL_x1_DEFINE(30); MUL_x1_DEFINE(31); MUL_x1_DEFINE(32); MUL_x1_DEFINE(33); MUL_x1_DEFINE(34); MUL_x1_DEFINE(35); MUL_x1_DEFINE(36); MUL_x1_DEFINE(37); MUL_x1_DEFINE(38); MUL_x1_DEFINE(39); MUL_x1_DEFINE(40); MUL_x1_DEFINE(41); MUL_x1_DEFINE(42); MUL_x1_DEFINE(43); MUL_x1_DEFINE(44); MUL_x1_DEFINE(45); MUL_x1_DEFINE(46); MUL_x1_DEFINE(47); MUL_x1_DEFINE(48); MUL_x1_DEFINE(49); MUL_x1_DEFINE(50); MUL_x1_DEFINE(51); MUL_x1_DEFINE(52); MUL_x1_DEFINE(53); MUL_x1_DEFINE(54); MUL_x1_DEFINE(55); MUL_x1_DEFINE(56); MUL_x1_DEFINE(57); MUL_x1_DEFINE(58); MUL_x1_DEFINE(59); MUL_x1_DEFINE(60); MUL_x1_DEFINE(61); MUL_x1_DEFINE(62); MUL_x1_DEFINE(63); MUL_x1_DEFINE(64); MUL_x1_DEFINE(65); MUL_x1_DEFINE(66); MUL_x1_DEFINE(67); MUL_x1_DEFINE(68); MUL_x1_DEFINE(69); MUL_x1_DEFINE(70); MUL_x1_DEFINE(71); MUL_x1_DEFINE(72); MUL_x1_DEFINE(73); MUL_x1_DEFINE(74); MUL_x1_DEFINE(75); MUL_x1_DEFINE(76); MUL_x1_DEFINE(77); MUL_x1_DEFINE(78); MUL_x1_DEFINE(79); MUL_x1_DEFINE(80); MUL_x1_DEFINE(81); MUL_x1_DEFINE(82); MUL_x1_DEFINE(83); MUL_x1_DEFINE(84); MUL_x1_DEFINE(85); MUL_x1_DEFINE(86); MUL_x1_DEFINE(87); MUL_x1_DEFINE(88); MUL_x1_DEFINE(89); MUL_x1_DEFINE(90); MUL_x1_DEFINE(91); MUL_x1_DEFINE(92); MUL_x1_DEFINE(93); MUL_x1_DEFINE(94); MUL_x1_DEFINE(95); MUL_x1_DEFINE(96); MUL_x1_DEFINE(97); MUL_x1_DEFINE(98); MUL_x1_DEFINE(99); MUL_x1_DEFINE(100); MUL_x1_DEFINE(101); MUL_x1_DEFINE(102); MUL_x1_DEFINE(103); MUL_x1_DEFINE(104); MUL_x1_DEFINE(105); MUL_x1_DEFINE(106); MUL_x1_DEFINE(107); MUL_x1_DEFINE(108); MUL_x1_DEFINE(109); MUL_x1_DEFINE(110); MUL_x1_DEFINE(111); MUL_x1_DEFINE(112); MUL_x1_DEFINE(113); MUL_x1_DEFINE(114); MUL_x1_DEFINE(115); MUL_x1_DEFINE(116); MUL_x1_DEFINE(117); MUL_x1_DEFINE(118); MUL_x1_DEFINE(119); MUL_x1_DEFINE(120); MUL_x1_DEFINE(121); MUL_x1_DEFINE(122); MUL_x1_DEFINE(123); MUL_x1_DEFINE(124); MUL_x1_DEFINE(125); MUL_x1_DEFINE(126); MUL_x1_DEFINE(127); MUL_x1_DEFINE(128); MUL_x1_DEFINE(129); MUL_x1_DEFINE(130); MUL_x1_DEFINE(131); MUL_x1_DEFINE(132); MUL_x1_DEFINE(133); MUL_x1_DEFINE(134); MUL_x1_DEFINE(135); MUL_x1_DEFINE(136); MUL_x1_DEFINE(137); MUL_x1_DEFINE(138); MUL_x1_DEFINE(139); MUL_x1_DEFINE(140); MUL_x1_DEFINE(141); MUL_x1_DEFINE(142); MUL_x1_DEFINE(143); MUL_x1_DEFINE(144); MUL_x1_DEFINE(145); MUL_x1_DEFINE(146); MUL_x1_DEFINE(147); MUL_x1_DEFINE(148); MUL_x1_DEFINE(149); MUL_x1_DEFINE(150); MUL_x1_DEFINE(151); MUL_x1_DEFINE(152); MUL_x1_DEFINE(153); MUL_x1_DEFINE(154); MUL_x1_DEFINE(155); MUL_x1_DEFINE(156); MUL_x1_DEFINE(157); MUL_x1_DEFINE(158); MUL_x1_DEFINE(159); MUL_x1_DEFINE(160); MUL_x1_DEFINE(161); MUL_x1_DEFINE(162); MUL_x1_DEFINE(163); MUL_x1_DEFINE(164); MUL_x1_DEFINE(165); MUL_x1_DEFINE(166); MUL_x1_DEFINE(167); MUL_x1_DEFINE(168); MUL_x1_DEFINE(169); MUL_x1_DEFINE(170); MUL_x1_DEFINE(171); MUL_x1_DEFINE(172); MUL_x1_DEFINE(173); MUL_x1_DEFINE(174); MUL_x1_DEFINE(175); MUL_x1_DEFINE(176); MUL_x1_DEFINE(177); MUL_x1_DEFINE(178); MUL_x1_DEFINE(179); MUL_x1_DEFINE(180); MUL_x1_DEFINE(181); MUL_x1_DEFINE(182); MUL_x1_DEFINE(183); MUL_x1_DEFINE(184); MUL_x1_DEFINE(185); MUL_x1_DEFINE(186); MUL_x1_DEFINE(187); MUL_x1_DEFINE(188); MUL_x1_DEFINE(189); MUL_x1_DEFINE(190); MUL_x1_DEFINE(191); MUL_x1_DEFINE(192); MUL_x1_DEFINE(193); MUL_x1_DEFINE(194); MUL_x1_DEFINE(195); MUL_x1_DEFINE(196); MUL_x1_DEFINE(197); MUL_x1_DEFINE(198); MUL_x1_DEFINE(199); MUL_x1_DEFINE(200); MUL_x1_DEFINE(201); MUL_x1_DEFINE(202); MUL_x1_DEFINE(203); MUL_x1_DEFINE(204); MUL_x1_DEFINE(205); MUL_x1_DEFINE(206); MUL_x1_DEFINE(207); MUL_x1_DEFINE(208); MUL_x1_DEFINE(209); MUL_x1_DEFINE(210); MUL_x1_DEFINE(211); MUL_x1_DEFINE(212); MUL_x1_DEFINE(213); MUL_x1_DEFINE(214); MUL_x1_DEFINE(215); MUL_x1_DEFINE(216); MUL_x1_DEFINE(217); MUL_x1_DEFINE(218); MUL_x1_DEFINE(219); MUL_x1_DEFINE(220); MUL_x1_DEFINE(221); MUL_x1_DEFINE(222); MUL_x1_DEFINE(223); MUL_x1_DEFINE(224); MUL_x1_DEFINE(225); MUL_x1_DEFINE(226); MUL_x1_DEFINE(227); MUL_x1_DEFINE(228); MUL_x1_DEFINE(229); MUL_x1_DEFINE(230); MUL_x1_DEFINE(231); MUL_x1_DEFINE(232); MUL_x1_DEFINE(233); MUL_x1_DEFINE(234); MUL_x1_DEFINE(235); MUL_x1_DEFINE(236); MUL_x1_DEFINE(237); MUL_x1_DEFINE(238); MUL_x1_DEFINE(239); MUL_x1_DEFINE(240); MUL_x1_DEFINE(241); MUL_x1_DEFINE(242); MUL_x1_DEFINE(243); MUL_x1_DEFINE(244); MUL_x1_DEFINE(245); MUL_x1_DEFINE(246); MUL_x1_DEFINE(247); MUL_x1_DEFINE(248); MUL_x1_DEFINE(249); MUL_x1_DEFINE(250); MUL_x1_DEFINE(251); MUL_x1_DEFINE(252); MUL_x1_DEFINE(253); MUL_x1_DEFINE(254); MUL_x1_DEFINE(255); MUL_x2_DEFINE(0); MUL_x2_DEFINE(1); MUL_x2_DEFINE(2); MUL_x2_DEFINE(3); MUL_x2_DEFINE(4); MUL_x2_DEFINE(5); MUL_x2_DEFINE(6); MUL_x2_DEFINE(7); MUL_x2_DEFINE(8); MUL_x2_DEFINE(9); MUL_x2_DEFINE(10); MUL_x2_DEFINE(11); MUL_x2_DEFINE(12); MUL_x2_DEFINE(13); MUL_x2_DEFINE(14); MUL_x2_DEFINE(15); MUL_x2_DEFINE(16); MUL_x2_DEFINE(17); MUL_x2_DEFINE(18); MUL_x2_DEFINE(19); MUL_x2_DEFINE(20); MUL_x2_DEFINE(21); MUL_x2_DEFINE(22); MUL_x2_DEFINE(23); MUL_x2_DEFINE(24); MUL_x2_DEFINE(25); MUL_x2_DEFINE(26); MUL_x2_DEFINE(27); MUL_x2_DEFINE(28); MUL_x2_DEFINE(29); MUL_x2_DEFINE(30); MUL_x2_DEFINE(31); MUL_x2_DEFINE(32); MUL_x2_DEFINE(33); MUL_x2_DEFINE(34); MUL_x2_DEFINE(35); MUL_x2_DEFINE(36); MUL_x2_DEFINE(37); MUL_x2_DEFINE(38); MUL_x2_DEFINE(39); MUL_x2_DEFINE(40); MUL_x2_DEFINE(41); MUL_x2_DEFINE(42); MUL_x2_DEFINE(43); MUL_x2_DEFINE(44); MUL_x2_DEFINE(45); MUL_x2_DEFINE(46); MUL_x2_DEFINE(47); MUL_x2_DEFINE(48); MUL_x2_DEFINE(49); MUL_x2_DEFINE(50); MUL_x2_DEFINE(51); MUL_x2_DEFINE(52); MUL_x2_DEFINE(53); MUL_x2_DEFINE(54); MUL_x2_DEFINE(55); MUL_x2_DEFINE(56); MUL_x2_DEFINE(57); MUL_x2_DEFINE(58); MUL_x2_DEFINE(59); MUL_x2_DEFINE(60); MUL_x2_DEFINE(61); MUL_x2_DEFINE(62); MUL_x2_DEFINE(63); MUL_x2_DEFINE(64); MUL_x2_DEFINE(65); MUL_x2_DEFINE(66); MUL_x2_DEFINE(67); MUL_x2_DEFINE(68); MUL_x2_DEFINE(69); MUL_x2_DEFINE(70); MUL_x2_DEFINE(71); MUL_x2_DEFINE(72); MUL_x2_DEFINE(73); MUL_x2_DEFINE(74); MUL_x2_DEFINE(75); MUL_x2_DEFINE(76); MUL_x2_DEFINE(77); MUL_x2_DEFINE(78); MUL_x2_DEFINE(79); MUL_x2_DEFINE(80); MUL_x2_DEFINE(81); MUL_x2_DEFINE(82); MUL_x2_DEFINE(83); MUL_x2_DEFINE(84); MUL_x2_DEFINE(85); MUL_x2_DEFINE(86); MUL_x2_DEFINE(87); MUL_x2_DEFINE(88); MUL_x2_DEFINE(89); MUL_x2_DEFINE(90); MUL_x2_DEFINE(91); MUL_x2_DEFINE(92); MUL_x2_DEFINE(93); MUL_x2_DEFINE(94); MUL_x2_DEFINE(95); MUL_x2_DEFINE(96); MUL_x2_DEFINE(97); MUL_x2_DEFINE(98); MUL_x2_DEFINE(99); MUL_x2_DEFINE(100); MUL_x2_DEFINE(101); MUL_x2_DEFINE(102); MUL_x2_DEFINE(103); MUL_x2_DEFINE(104); MUL_x2_DEFINE(105); MUL_x2_DEFINE(106); MUL_x2_DEFINE(107); MUL_x2_DEFINE(108); MUL_x2_DEFINE(109); MUL_x2_DEFINE(110); MUL_x2_DEFINE(111); MUL_x2_DEFINE(112); MUL_x2_DEFINE(113); MUL_x2_DEFINE(114); MUL_x2_DEFINE(115); MUL_x2_DEFINE(116); MUL_x2_DEFINE(117); MUL_x2_DEFINE(118); MUL_x2_DEFINE(119); MUL_x2_DEFINE(120); MUL_x2_DEFINE(121); MUL_x2_DEFINE(122); MUL_x2_DEFINE(123); MUL_x2_DEFINE(124); MUL_x2_DEFINE(125); MUL_x2_DEFINE(126); MUL_x2_DEFINE(127); MUL_x2_DEFINE(128); MUL_x2_DEFINE(129); MUL_x2_DEFINE(130); MUL_x2_DEFINE(131); MUL_x2_DEFINE(132); MUL_x2_DEFINE(133); MUL_x2_DEFINE(134); MUL_x2_DEFINE(135); MUL_x2_DEFINE(136); MUL_x2_DEFINE(137); MUL_x2_DEFINE(138); MUL_x2_DEFINE(139); MUL_x2_DEFINE(140); MUL_x2_DEFINE(141); MUL_x2_DEFINE(142); MUL_x2_DEFINE(143); MUL_x2_DEFINE(144); MUL_x2_DEFINE(145); MUL_x2_DEFINE(146); MUL_x2_DEFINE(147); MUL_x2_DEFINE(148); MUL_x2_DEFINE(149); MUL_x2_DEFINE(150); MUL_x2_DEFINE(151); MUL_x2_DEFINE(152); MUL_x2_DEFINE(153); MUL_x2_DEFINE(154); MUL_x2_DEFINE(155); MUL_x2_DEFINE(156); MUL_x2_DEFINE(157); MUL_x2_DEFINE(158); MUL_x2_DEFINE(159); MUL_x2_DEFINE(160); MUL_x2_DEFINE(161); MUL_x2_DEFINE(162); MUL_x2_DEFINE(163); MUL_x2_DEFINE(164); MUL_x2_DEFINE(165); MUL_x2_DEFINE(166); MUL_x2_DEFINE(167); MUL_x2_DEFINE(168); MUL_x2_DEFINE(169); MUL_x2_DEFINE(170); MUL_x2_DEFINE(171); MUL_x2_DEFINE(172); MUL_x2_DEFINE(173); MUL_x2_DEFINE(174); MUL_x2_DEFINE(175); MUL_x2_DEFINE(176); MUL_x2_DEFINE(177); MUL_x2_DEFINE(178); MUL_x2_DEFINE(179); MUL_x2_DEFINE(180); MUL_x2_DEFINE(181); MUL_x2_DEFINE(182); MUL_x2_DEFINE(183); MUL_x2_DEFINE(184); MUL_x2_DEFINE(185); MUL_x2_DEFINE(186); MUL_x2_DEFINE(187); MUL_x2_DEFINE(188); MUL_x2_DEFINE(189); MUL_x2_DEFINE(190); MUL_x2_DEFINE(191); MUL_x2_DEFINE(192); MUL_x2_DEFINE(193); MUL_x2_DEFINE(194); MUL_x2_DEFINE(195); MUL_x2_DEFINE(196); MUL_x2_DEFINE(197); MUL_x2_DEFINE(198); MUL_x2_DEFINE(199); MUL_x2_DEFINE(200); MUL_x2_DEFINE(201); MUL_x2_DEFINE(202); MUL_x2_DEFINE(203); MUL_x2_DEFINE(204); MUL_x2_DEFINE(205); MUL_x2_DEFINE(206); MUL_x2_DEFINE(207); MUL_x2_DEFINE(208); MUL_x2_DEFINE(209); MUL_x2_DEFINE(210); MUL_x2_DEFINE(211); MUL_x2_DEFINE(212); MUL_x2_DEFINE(213); MUL_x2_DEFINE(214); MUL_x2_DEFINE(215); MUL_x2_DEFINE(216); MUL_x2_DEFINE(217); MUL_x2_DEFINE(218); MUL_x2_DEFINE(219); MUL_x2_DEFINE(220); MUL_x2_DEFINE(221); MUL_x2_DEFINE(222); MUL_x2_DEFINE(223); MUL_x2_DEFINE(224); MUL_x2_DEFINE(225); MUL_x2_DEFINE(226); MUL_x2_DEFINE(227); MUL_x2_DEFINE(228); MUL_x2_DEFINE(229); MUL_x2_DEFINE(230); MUL_x2_DEFINE(231); MUL_x2_DEFINE(232); MUL_x2_DEFINE(233); MUL_x2_DEFINE(234); MUL_x2_DEFINE(235); MUL_x2_DEFINE(236); MUL_x2_DEFINE(237); MUL_x2_DEFINE(238); MUL_x2_DEFINE(239); MUL_x2_DEFINE(240); MUL_x2_DEFINE(241); MUL_x2_DEFINE(242); MUL_x2_DEFINE(243); MUL_x2_DEFINE(244); MUL_x2_DEFINE(245); MUL_x2_DEFINE(246); MUL_x2_DEFINE(247); MUL_x2_DEFINE(248); MUL_x2_DEFINE(249); MUL_x2_DEFINE(250); MUL_x2_DEFINE(251); MUL_x2_DEFINE(252); MUL_x2_DEFINE(253); MUL_x2_DEFINE(254); MUL_x2_DEFINE(255); typedef void (*mul_fn_ptr_t)(void); static const mul_fn_ptr_t __attribute__((aligned(256))) gf_x1_mul_fns[256] = { mul_x1_0, mul_x1_1, mul_x1_2, mul_x1_3, mul_x1_4, mul_x1_5, mul_x1_6, mul_x1_7, mul_x1_8, mul_x1_9, mul_x1_10, mul_x1_11, mul_x1_12, mul_x1_13, mul_x1_14, mul_x1_15, mul_x1_16, mul_x1_17, mul_x1_18, mul_x1_19, mul_x1_20, mul_x1_21, mul_x1_22, mul_x1_23, mul_x1_24, mul_x1_25, mul_x1_26, mul_x1_27, mul_x1_28, mul_x1_29, mul_x1_30, mul_x1_31, mul_x1_32, mul_x1_33, mul_x1_34, mul_x1_35, mul_x1_36, mul_x1_37, mul_x1_38, mul_x1_39, mul_x1_40, mul_x1_41, mul_x1_42, mul_x1_43, mul_x1_44, mul_x1_45, mul_x1_46, mul_x1_47, mul_x1_48, mul_x1_49, mul_x1_50, mul_x1_51, mul_x1_52, mul_x1_53, mul_x1_54, mul_x1_55, mul_x1_56, mul_x1_57, mul_x1_58, mul_x1_59, mul_x1_60, mul_x1_61, mul_x1_62, mul_x1_63, mul_x1_64, mul_x1_65, mul_x1_66, mul_x1_67, mul_x1_68, mul_x1_69, mul_x1_70, mul_x1_71, mul_x1_72, mul_x1_73, mul_x1_74, mul_x1_75, mul_x1_76, mul_x1_77, mul_x1_78, mul_x1_79, mul_x1_80, mul_x1_81, mul_x1_82, mul_x1_83, mul_x1_84, mul_x1_85, mul_x1_86, mul_x1_87, mul_x1_88, mul_x1_89, mul_x1_90, mul_x1_91, mul_x1_92, mul_x1_93, mul_x1_94, mul_x1_95, mul_x1_96, mul_x1_97, mul_x1_98, mul_x1_99, mul_x1_100, mul_x1_101, mul_x1_102, mul_x1_103, mul_x1_104, mul_x1_105, mul_x1_106, mul_x1_107, mul_x1_108, mul_x1_109, mul_x1_110, mul_x1_111, mul_x1_112, mul_x1_113, mul_x1_114, mul_x1_115, mul_x1_116, mul_x1_117, mul_x1_118, mul_x1_119, mul_x1_120, mul_x1_121, mul_x1_122, mul_x1_123, mul_x1_124, mul_x1_125, mul_x1_126, mul_x1_127, mul_x1_128, mul_x1_129, mul_x1_130, mul_x1_131, mul_x1_132, mul_x1_133, mul_x1_134, mul_x1_135, mul_x1_136, mul_x1_137, mul_x1_138, mul_x1_139, mul_x1_140, mul_x1_141, mul_x1_142, mul_x1_143, mul_x1_144, mul_x1_145, mul_x1_146, mul_x1_147, mul_x1_148, mul_x1_149, mul_x1_150, mul_x1_151, mul_x1_152, mul_x1_153, mul_x1_154, mul_x1_155, mul_x1_156, mul_x1_157, mul_x1_158, mul_x1_159, mul_x1_160, mul_x1_161, mul_x1_162, mul_x1_163, mul_x1_164, mul_x1_165, mul_x1_166, mul_x1_167, mul_x1_168, mul_x1_169, mul_x1_170, mul_x1_171, mul_x1_172, mul_x1_173, mul_x1_174, mul_x1_175, mul_x1_176, mul_x1_177, mul_x1_178, mul_x1_179, mul_x1_180, mul_x1_181, mul_x1_182, mul_x1_183, mul_x1_184, mul_x1_185, mul_x1_186, mul_x1_187, mul_x1_188, mul_x1_189, mul_x1_190, mul_x1_191, mul_x1_192, mul_x1_193, mul_x1_194, mul_x1_195, mul_x1_196, mul_x1_197, mul_x1_198, mul_x1_199, mul_x1_200, mul_x1_201, mul_x1_202, mul_x1_203, mul_x1_204, mul_x1_205, mul_x1_206, mul_x1_207, mul_x1_208, mul_x1_209, mul_x1_210, mul_x1_211, mul_x1_212, mul_x1_213, mul_x1_214, mul_x1_215, mul_x1_216, mul_x1_217, mul_x1_218, mul_x1_219, mul_x1_220, mul_x1_221, mul_x1_222, mul_x1_223, mul_x1_224, mul_x1_225, mul_x1_226, mul_x1_227, mul_x1_228, mul_x1_229, mul_x1_230, mul_x1_231, mul_x1_232, mul_x1_233, mul_x1_234, mul_x1_235, mul_x1_236, mul_x1_237, mul_x1_238, mul_x1_239, mul_x1_240, mul_x1_241, mul_x1_242, mul_x1_243, mul_x1_244, mul_x1_245, mul_x1_246, mul_x1_247, mul_x1_248, mul_x1_249, mul_x1_250, mul_x1_251, mul_x1_252, mul_x1_253, mul_x1_254, mul_x1_255 }; static const mul_fn_ptr_t __attribute__((aligned(256))) gf_x2_mul_fns[256] = { mul_x2_0, mul_x2_1, mul_x2_2, mul_x2_3, mul_x2_4, mul_x2_5, mul_x2_6, mul_x2_7, mul_x2_8, mul_x2_9, mul_x2_10, mul_x2_11, mul_x2_12, mul_x2_13, mul_x2_14, mul_x2_15, mul_x2_16, mul_x2_17, mul_x2_18, mul_x2_19, mul_x2_20, mul_x2_21, mul_x2_22, mul_x2_23, mul_x2_24, mul_x2_25, mul_x2_26, mul_x2_27, mul_x2_28, mul_x2_29, mul_x2_30, mul_x2_31, mul_x2_32, mul_x2_33, mul_x2_34, mul_x2_35, mul_x2_36, mul_x2_37, mul_x2_38, mul_x2_39, mul_x2_40, mul_x2_41, mul_x2_42, mul_x2_43, mul_x2_44, mul_x2_45, mul_x2_46, mul_x2_47, mul_x2_48, mul_x2_49, mul_x2_50, mul_x2_51, mul_x2_52, mul_x2_53, mul_x2_54, mul_x2_55, mul_x2_56, mul_x2_57, mul_x2_58, mul_x2_59, mul_x2_60, mul_x2_61, mul_x2_62, mul_x2_63, mul_x2_64, mul_x2_65, mul_x2_66, mul_x2_67, mul_x2_68, mul_x2_69, mul_x2_70, mul_x2_71, mul_x2_72, mul_x2_73, mul_x2_74, mul_x2_75, mul_x2_76, mul_x2_77, mul_x2_78, mul_x2_79, mul_x2_80, mul_x2_81, mul_x2_82, mul_x2_83, mul_x2_84, mul_x2_85, mul_x2_86, mul_x2_87, mul_x2_88, mul_x2_89, mul_x2_90, mul_x2_91, mul_x2_92, mul_x2_93, mul_x2_94, mul_x2_95, mul_x2_96, mul_x2_97, mul_x2_98, mul_x2_99, mul_x2_100, mul_x2_101, mul_x2_102, mul_x2_103, mul_x2_104, mul_x2_105, mul_x2_106, mul_x2_107, mul_x2_108, mul_x2_109, mul_x2_110, mul_x2_111, mul_x2_112, mul_x2_113, mul_x2_114, mul_x2_115, mul_x2_116, mul_x2_117, mul_x2_118, mul_x2_119, mul_x2_120, mul_x2_121, mul_x2_122, mul_x2_123, mul_x2_124, mul_x2_125, mul_x2_126, mul_x2_127, mul_x2_128, mul_x2_129, mul_x2_130, mul_x2_131, mul_x2_132, mul_x2_133, mul_x2_134, mul_x2_135, mul_x2_136, mul_x2_137, mul_x2_138, mul_x2_139, mul_x2_140, mul_x2_141, mul_x2_142, mul_x2_143, mul_x2_144, mul_x2_145, mul_x2_146, mul_x2_147, mul_x2_148, mul_x2_149, mul_x2_150, mul_x2_151, mul_x2_152, mul_x2_153, mul_x2_154, mul_x2_155, mul_x2_156, mul_x2_157, mul_x2_158, mul_x2_159, mul_x2_160, mul_x2_161, mul_x2_162, mul_x2_163, mul_x2_164, mul_x2_165, mul_x2_166, mul_x2_167, mul_x2_168, mul_x2_169, mul_x2_170, mul_x2_171, mul_x2_172, mul_x2_173, mul_x2_174, mul_x2_175, mul_x2_176, mul_x2_177, mul_x2_178, mul_x2_179, mul_x2_180, mul_x2_181, mul_x2_182, mul_x2_183, mul_x2_184, mul_x2_185, mul_x2_186, mul_x2_187, mul_x2_188, mul_x2_189, mul_x2_190, mul_x2_191, mul_x2_192, mul_x2_193, mul_x2_194, mul_x2_195, mul_x2_196, mul_x2_197, mul_x2_198, mul_x2_199, mul_x2_200, mul_x2_201, mul_x2_202, mul_x2_203, mul_x2_204, mul_x2_205, mul_x2_206, mul_x2_207, mul_x2_208, mul_x2_209, mul_x2_210, mul_x2_211, mul_x2_212, mul_x2_213, mul_x2_214, mul_x2_215, mul_x2_216, mul_x2_217, mul_x2_218, mul_x2_219, mul_x2_220, mul_x2_221, mul_x2_222, mul_x2_223, mul_x2_224, mul_x2_225, mul_x2_226, mul_x2_227, mul_x2_228, mul_x2_229, mul_x2_230, mul_x2_231, mul_x2_232, mul_x2_233, mul_x2_234, mul_x2_235, mul_x2_236, mul_x2_237, mul_x2_238, mul_x2_239, mul_x2_240, mul_x2_241, mul_x2_242, mul_x2_243, mul_x2_244, mul_x2_245, mul_x2_246, mul_x2_247, mul_x2_248, mul_x2_249, mul_x2_250, mul_x2_251, mul_x2_252, mul_x2_253, mul_x2_254, mul_x2_255 }; #define MUL(c, r...) \ { \ switch (REG_CNT(r)) { \ case 2: \ COPY(r, _mul_x2_in); \ gf_x2_mul_fns[c](); \ COPY(_mul_x2_acc, r); \ break; \ case 1: \ COPY(r, _mul_x1_in); \ gf_x1_mul_fns[c](); \ COPY(_mul_x1_acc, r); \ break; \ } \ } #define raidz_math_begin() kfpu_begin() #define raidz_math_end() kfpu_end() #define SYN_STRIDE 4 #define ZERO_STRIDE 4 #define ZERO_DEFINE() {} #define ZERO_D 0, 1, 2, 3 #define COPY_STRIDE 4 #define COPY_DEFINE() {} #define COPY_D 0, 1, 2, 3 #define ADD_STRIDE 4 #define ADD_DEFINE() {} #define ADD_D 0, 1, 2, 3 #define MUL_STRIDE 2 #define MUL_DEFINE() MUL2_SETUP() #define MUL_D 0, 1 #define GEN_P_STRIDE 4 #define GEN_P_DEFINE() {} #define GEN_P_P 0, 1, 2, 3 #define GEN_PQ_STRIDE 4 #define GEN_PQ_DEFINE() {} #define GEN_PQ_D 0, 1, 2, 3 #define GEN_PQ_C 4, 5, 6, 7 #define GEN_PQR_STRIDE 4 #define GEN_PQR_DEFINE() {} #define GEN_PQR_D 0, 1, 2, 3 #define GEN_PQR_C 4, 5, 6, 7 #define SYN_Q_DEFINE() {} #define SYN_Q_D 0, 1, 2, 3 #define SYN_Q_X 4, 5, 6, 7 #define SYN_R_DEFINE() {} #define SYN_R_D 0, 1, 2, 3 #define SYN_R_X 4, 5, 6, 7 #define SYN_PQ_DEFINE() {} #define SYN_PQ_D 0, 1, 2, 3 #define SYN_PQ_X 4, 5, 6, 7 #define REC_PQ_STRIDE 2 #define REC_PQ_DEFINE() MUL2_SETUP() #define REC_PQ_X 0, 1 #define REC_PQ_Y 2, 3 #define REC_PQ_T 4, 5 #define SYN_PR_DEFINE() {} #define SYN_PR_D 0, 1, 2, 3 #define SYN_PR_X 4, 5, 6, 7 #define REC_PR_STRIDE 2 #define REC_PR_DEFINE() MUL2_SETUP() #define REC_PR_X 0, 1 #define REC_PR_Y 2, 3 #define REC_PR_T 4, 5 #define SYN_QR_DEFINE() {} #define SYN_QR_D 0, 1, 2, 3 #define SYN_QR_X 4, 5, 6, 7 #define REC_QR_STRIDE 2 #define REC_QR_DEFINE() MUL2_SETUP() #define REC_QR_X 0, 1 #define REC_QR_Y 2, 3 #define REC_QR_T 4, 5 #define SYN_PQR_DEFINE() {} #define SYN_PQR_D 0, 1, 2, 3 #define SYN_PQR_X 4, 5, 6, 7 #define REC_PQR_STRIDE 1 #define REC_PQR_DEFINE() MUL2_SETUP() #define REC_PQR_X 0 #define REC_PQR_Y 1 #define REC_PQR_Z 2 #define REC_PQR_XS 3 #define REC_PQR_YS 4 #include #include "vdev_raidz_math_impl.h" DEFINE_GEN_METHODS(sse2); DEFINE_REC_METHODS(sse2); static boolean_t raidz_will_sse2_work(void) { return (kfpu_allowed() && zfs_sse_available() && zfs_sse2_available()); } const raidz_impl_ops_t vdev_raidz_sse2_impl = { .init = NULL, .fini = NULL, .gen = RAIDZ_GEN_METHODS(sse2), .rec = RAIDZ_REC_METHODS(sse2), .is_supported = &raidz_will_sse2_work, .name = "sse2" }; #endif /* defined(__x86_64) && defined(HAVE_SSE2) */ diff --git a/module/zfs/vdev_raidz_math_ssse3.c b/module/zfs/vdev_raidz_math_ssse3.c index 62247cf8eb8d..d5776a38a2e4 100644 --- a/module/zfs/vdev_raidz_math_ssse3.c +++ b/module/zfs/vdev_raidz_math_ssse3.c @@ -1,2475 +1,2475 @@ /* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (C) 2016 Gvozden NeÅ¡ković. All rights reserved. */ #include #if defined(__x86_64) && defined(HAVE_SSSE3) #include -#include +#include #define __asm __asm__ __volatile__ #define _REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N #define REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1) #define VR0_(REG, ...) "xmm"#REG #define VR1_(_1, REG, ...) "xmm"#REG #define VR2_(_1, _2, REG, ...) "xmm"#REG #define VR3_(_1, _2, _3, REG, ...) "xmm"#REG #define VR4_(_1, _2, _3, _4, REG, ...) "xmm"#REG #define VR5_(_1, _2, _3, _4, _5, REG, ...) "xmm"#REG #define VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "xmm"#REG #define VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "xmm"#REG #define VR0(r...) VR0_(r) #define VR1(r...) VR1_(r) #define VR2(r...) VR2_(r, 1) #define VR3(r...) VR3_(r, 1, 2) #define VR4(r...) VR4_(r, 1, 2) #define VR5(r...) VR5_(r, 1, 2, 3) #define VR6(r...) VR6_(r, 1, 2, 3, 4) #define VR7(r...) VR7_(r, 1, 2, 3, 4, 5) #define R_01(REG1, REG2, ...) REG1, REG2 #define _R_23(_0, _1, REG2, REG3, ...) REG2, REG3 #define R_23(REG...) _R_23(REG, 1, 2, 3) #define ZFS_ASM_BUG() ASSERT(0) const uint8_t gf_clmul_mod_lt[4*256][16]; #define ELEM_SIZE 16 typedef struct v { uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE))); } v_t; #define XOR_ACC(src, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "pxor 0x00(%[SRC]), %%" VR0(r) "\n" \ "pxor 0x10(%[SRC]), %%" VR1(r) "\n" \ "pxor 0x20(%[SRC]), %%" VR2(r) "\n" \ "pxor 0x30(%[SRC]), %%" VR3(r) "\n" \ : : [SRC] "r" (src)); \ break; \ case 2: \ __asm( \ "pxor 0x00(%[SRC]), %%" VR0(r) "\n" \ "pxor 0x10(%[SRC]), %%" VR1(r) "\n" \ : : [SRC] "r" (src)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define XOR(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "pxor %" VR0(r) ", %" VR4(r) "\n" \ "pxor %" VR1(r) ", %" VR5(r) "\n" \ "pxor %" VR2(r) ", %" VR6(r) "\n" \ "pxor %" VR3(r) ", %" VR7(r)); \ break; \ case 4: \ __asm( \ "pxor %" VR0(r) ", %" VR2(r) "\n" \ "pxor %" VR1(r) ", %" VR3(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define ZERO(r...) XOR(r, r) #define COPY(r...) \ { \ switch (REG_CNT(r)) { \ case 8: \ __asm( \ "movdqa %" VR0(r) ", %" VR4(r) "\n" \ "movdqa %" VR1(r) ", %" VR5(r) "\n" \ "movdqa %" VR2(r) ", %" VR6(r) "\n" \ "movdqa %" VR3(r) ", %" VR7(r)); \ break; \ case 4: \ __asm( \ "movdqa %" VR0(r) ", %" VR2(r) "\n" \ "movdqa %" VR1(r) ", %" VR3(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define LOAD(src, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "movdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ "movdqa 0x10(%[SRC]), %%" VR1(r) "\n" \ "movdqa 0x20(%[SRC]), %%" VR2(r) "\n" \ "movdqa 0x30(%[SRC]), %%" VR3(r) "\n" \ : : [SRC] "r" (src)); \ break; \ case 2: \ __asm( \ "movdqa 0x00(%[SRC]), %%" VR0(r) "\n" \ "movdqa 0x10(%[SRC]), %%" VR1(r) "\n" \ : : [SRC] "r" (src)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define STORE(dst, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ __asm( \ "movdqa %%" VR0(r)", 0x00(%[DST])\n" \ "movdqa %%" VR1(r)", 0x10(%[DST])\n" \ "movdqa %%" VR2(r)", 0x20(%[DST])\n" \ "movdqa %%" VR3(r)", 0x30(%[DST])\n" \ : : [DST] "r" (dst)); \ break; \ case 2: \ __asm( \ "movdqa %%" VR0(r)", 0x00(%[DST])\n" \ "movdqa %%" VR1(r)", 0x10(%[DST])\n" \ : : [DST] "r" (dst)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL2_SETUP() \ { \ __asm( \ "movd %[mask], %%xmm15\n" \ "pshufd $0x0, %%xmm15, %%xmm15\n" \ : : [mask] "r" (0x1d1d1d1d)); \ } #define _MUL2_x2(r...) \ { \ switch (REG_CNT(r)) { \ case 2: \ __asm( \ "pxor %xmm14, %xmm14\n" \ "pxor %xmm13, %xmm13\n" \ "pcmpgtb %" VR0(r)", %xmm14\n" \ "pcmpgtb %" VR1(r)", %xmm13\n" \ "pand %xmm15, %xmm14\n" \ "pand %xmm15, %xmm13\n" \ "paddb %" VR0(r)", %" VR0(r) "\n" \ "paddb %" VR1(r)", %" VR1(r) "\n" \ "pxor %xmm14, %" VR0(r) "\n" \ "pxor %xmm13, %" VR1(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL2(r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ _MUL2_x2(R_01(r)); \ _MUL2_x2(R_23(r)); \ break; \ case 2: \ _MUL2_x2(r); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL4(r...) \ { \ MUL2(r); \ MUL2(r); \ } #define _0f "xmm15" #define _a_save "xmm14" #define _b_save "xmm13" #define _lt_mod_a "xmm12" #define _lt_clmul_a "xmm11" #define _lt_mod_b "xmm10" #define _lt_clmul_b "xmm15" #define _MULx2(c, r...) \ { \ switch (REG_CNT(r)) { \ case 2: \ __asm( \ /* lts for upper part */ \ "movd %[mask], %%" _0f "\n" \ "pshufd $0x0, %%" _0f ", %%" _0f "\n" \ "movdqa 0x00(%[lt]), %%" _lt_mod_a "\n" \ "movdqa 0x10(%[lt]), %%" _lt_clmul_a "\n" \ /* upper part */ \ "movdqa %%" VR0(r) ", %%" _a_save "\n" \ "movdqa %%" VR1(r) ", %%" _b_save "\n" \ "psraw $0x4, %%" VR0(r) "\n" \ "psraw $0x4, %%" VR1(r) "\n" \ "pand %%" _0f ", %%" _a_save "\n" \ "pand %%" _0f ", %%" _b_save "\n" \ "pand %%" _0f ", %%" VR0(r) "\n" \ "pand %%" _0f ", %%" VR1(r) "\n" \ \ "movdqa %%" _lt_mod_a ", %%" _lt_mod_b "\n" \ "movdqa %%" _lt_clmul_a ", %%" _lt_clmul_b "\n" \ \ "pshufb %%" VR0(r) ",%%" _lt_mod_a "\n" \ "pshufb %%" VR1(r) ",%%" _lt_mod_b "\n" \ "pshufb %%" VR0(r) ",%%" _lt_clmul_a "\n" \ "pshufb %%" VR1(r) ",%%" _lt_clmul_b "\n" \ \ "pxor %%" _lt_mod_a ",%%" _lt_clmul_a "\n" \ "pxor %%" _lt_mod_b ",%%" _lt_clmul_b "\n" \ "movdqa %%" _lt_clmul_a ",%%" VR0(r) "\n" \ "movdqa %%" _lt_clmul_b ",%%" VR1(r) "\n" \ /* lts for lower part */ \ "movdqa 0x20(%[lt]), %%" _lt_mod_a "\n" \ "movdqa 0x30(%[lt]), %%" _lt_clmul_a "\n" \ "movdqa %%" _lt_mod_a ", %%" _lt_mod_b "\n" \ "movdqa %%" _lt_clmul_a ", %%" _lt_clmul_b "\n" \ /* lower part */ \ "pshufb %%" _a_save ",%%" _lt_mod_a "\n" \ "pshufb %%" _b_save ",%%" _lt_mod_b "\n" \ "pshufb %%" _a_save ",%%" _lt_clmul_a "\n" \ "pshufb %%" _b_save ",%%" _lt_clmul_b "\n" \ \ "pxor %%" _lt_mod_a ",%%" VR0(r) "\n" \ "pxor %%" _lt_mod_b ",%%" VR1(r) "\n" \ "pxor %%" _lt_clmul_a ",%%" VR0(r) "\n" \ "pxor %%" _lt_clmul_b ",%%" VR1(r) "\n" \ : : [mask] "r" (0x0f0f0f0f), \ [lt] "r" (gf_clmul_mod_lt[4*(c)])); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define MUL(c, r...) \ { \ switch (REG_CNT(r)) { \ case 4: \ _MULx2(c, R_23(r)); \ _MULx2(c, R_01(r)); \ break; \ case 2: \ _MULx2(c, R_01(r)); \ break; \ default: \ ZFS_ASM_BUG(); \ } \ } #define raidz_math_begin() kfpu_begin() #define raidz_math_end() kfpu_end() #define SYN_STRIDE 4 #define ZERO_STRIDE 4 #define ZERO_DEFINE() {} #define ZERO_D 0, 1, 2, 3 #define COPY_STRIDE 4 #define COPY_DEFINE() {} #define COPY_D 0, 1, 2, 3 #define ADD_STRIDE 4 #define ADD_DEFINE() {} #define ADD_D 0, 1, 2, 3 #define MUL_STRIDE 4 #define MUL_DEFINE() {} #define MUL_D 0, 1, 2, 3 #define GEN_P_STRIDE 4 #define GEN_P_DEFINE() {} #define GEN_P_P 0, 1, 2, 3 #define GEN_PQ_STRIDE 4 #define GEN_PQ_DEFINE() {} #define GEN_PQ_D 0, 1, 2, 3 #define GEN_PQ_C 4, 5, 6, 7 #define GEN_PQR_STRIDE 4 #define GEN_PQR_DEFINE() {} #define GEN_PQR_D 0, 1, 2, 3 #define GEN_PQR_C 4, 5, 6, 7 #define SYN_Q_DEFINE() {} #define SYN_Q_D 0, 1, 2, 3 #define SYN_Q_X 4, 5, 6, 7 #define SYN_R_DEFINE() {} #define SYN_R_D 0, 1, 2, 3 #define SYN_R_X 4, 5, 6, 7 #define SYN_PQ_DEFINE() {} #define SYN_PQ_D 0, 1, 2, 3 #define SYN_PQ_X 4, 5, 6, 7 #define REC_PQ_STRIDE 2 #define REC_PQ_DEFINE() {} #define REC_PQ_X 0, 1 #define REC_PQ_Y 2, 3 #define REC_PQ_T 4, 5 #define SYN_PR_DEFINE() {} #define SYN_PR_D 0, 1, 2, 3 #define SYN_PR_X 4, 5, 6, 7 #define REC_PR_STRIDE 2 #define REC_PR_DEFINE() {} #define REC_PR_X 0, 1 #define REC_PR_Y 2, 3 #define REC_PR_T 4, 5 #define SYN_QR_DEFINE() {} #define SYN_QR_D 0, 1, 2, 3 #define SYN_QR_X 4, 5, 6, 7 #define REC_QR_STRIDE 2 #define REC_QR_DEFINE() {} #define REC_QR_X 0, 1 #define REC_QR_Y 2, 3 #define REC_QR_T 4, 5 #define SYN_PQR_DEFINE() {} #define SYN_PQR_D 0, 1, 2, 3 #define SYN_PQR_X 4, 5, 6, 7 #define REC_PQR_STRIDE 2 #define REC_PQR_DEFINE() {} #define REC_PQR_X 0, 1 #define REC_PQR_Y 2, 3 #define REC_PQR_Z 4, 5 #define REC_PQR_XS 6, 7 #define REC_PQR_YS 8, 9 #include #include "vdev_raidz_math_impl.h" DEFINE_GEN_METHODS(ssse3); DEFINE_REC_METHODS(ssse3); static boolean_t raidz_will_ssse3_work(void) { return (kfpu_allowed() && zfs_sse_available() && zfs_sse2_available() && zfs_ssse3_available()); } const raidz_impl_ops_t vdev_raidz_ssse3_impl = { .init = NULL, .fini = NULL, .gen = RAIDZ_GEN_METHODS(ssse3), .rec = RAIDZ_REC_METHODS(ssse3), .is_supported = &raidz_will_ssse3_work, .name = "ssse3" }; #endif /* defined(__x86_64) && defined(HAVE_SSSE3) */ #if defined(__x86_64) #if defined(HAVE_SSSE3) || defined(HAVE_AVX2) || defined(HAVE_AVX512BW) /* BEGIN CSTYLED */ const uint8_t __attribute__((aligned(256))) gf_clmul_mod_lt[4*256][16] = { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c, 0x0e, 0x10, 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x03, 0x06, 0x05, 0x0c, 0x0f, 0x0a, 0x09, 0x18, 0x1b, 0x1e, 0x1d, 0x14, 0x17, 0x12, 0x11 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x04, 0x08, 0x0c, 0x10, 0x14, 0x18, 0x1c, 0x20, 0x24, 0x28, 0x2c, 0x30, 0x34, 0x38, 0x3c }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x05, 0x0a, 0x0f, 0x14, 0x11, 0x1e, 0x1b, 0x28, 0x2d, 0x22, 0x27, 0x3c, 0x39, 0x36, 0x33 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x06, 0x0c, 0x0a, 0x18, 0x1e, 0x14, 0x12, 0x30, 0x36, 0x3c, 0x3a, 0x28, 0x2e, 0x24, 0x22 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x07, 0x0e, 0x09, 0x1c, 0x1b, 0x12, 0x15, 0x38, 0x3f, 0x36, 0x31, 0x24, 0x23, 0x2a, 0x2d }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x08, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38, 0x40, 0x48, 0x50, 0x58, 0x60, 0x68, 0x70, 0x78 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x09, 0x12, 0x1b, 0x24, 0x2d, 0x36, 0x3f, 0x48, 0x41, 0x5a, 0x53, 0x6c, 0x65, 0x7e, 0x77 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x0a, 0x14, 0x1e, 0x28, 0x22, 0x3c, 0x36, 0x50, 0x5a, 0x44, 0x4e, 0x78, 0x72, 0x6c, 0x66 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x0b, 0x16, 0x1d, 0x2c, 0x27, 0x3a, 0x31, 0x58, 0x53, 0x4e, 0x45, 0x74, 0x7f, 0x62, 0x69 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x0c, 0x18, 0x14, 0x30, 0x3c, 0x28, 0x24, 0x60, 0x6c, 0x78, 0x74, 0x50, 0x5c, 0x48, 0x44 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x0d, 0x1a, 0x17, 0x34, 0x39, 0x2e, 0x23, 0x68, 0x65, 0x72, 0x7f, 0x5c, 0x51, 0x46, 0x4b }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x0e, 0x1c, 0x12, 0x38, 0x36, 0x24, 0x2a, 0x70, 0x7e, 0x6c, 0x62, 0x48, 0x46, 0x54, 0x5a }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x0f, 0x1e, 0x11, 0x3c, 0x33, 0x22, 0x2d, 0x78, 0x77, 0x66, 0x69, 0x44, 0x4b, 0x5a, 0x55 }, { 0x00, 0x1d, 0x3a, 0x27, 0x74, 0x69, 0x4e, 0x53, 0xe8, 0xf5, 0xd2, 0xcf, 0x9c, 0x81, 0xa6, 0xbb }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x1d, 0x3a, 0x27, 0x74, 0x69, 0x4e, 0x53, 0xe8, 0xf5, 0xd2, 0xcf, 0x9c, 0x81, 0xa6, 0xbb }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff }, { 0x00, 0x1d, 0x3a, 0x27, 0x74, 0x69, 0x4e, 0x53, 0xf5, 0xe8, 0xcf, 0xd2, 0x81, 0x9c, 0xbb, 0xa6 }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x12, 0x24, 0x36, 0x48, 0x5a, 0x6c, 0x7e, 0x90, 0x82, 0xb4, 0xa6, 0xd8, 0xca, 0xfc, 0xee }, { 0x00, 0x1d, 0x3a, 0x27, 0x74, 0x69, 0x4e, 0x53, 0xf5, 0xe8, 0xcf, 0xd2, 0x81, 0x9c, 0xbb, 0xa6 }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x13, 0x26, 0x35, 0x4c, 0x5f, 0x6a, 0x79, 0x98, 0x8b, 0xbe, 0xad, 0xd4, 0xc7, 0xf2, 0xe1 }, { 0x00, 0x1d, 0x3a, 0x27, 0x69, 0x74, 0x53, 0x4e, 0xd2, 0xcf, 0xe8, 0xf5, 0xbb, 0xa6, 0x81, 0x9c }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x14, 0x28, 0x3c, 0x50, 0x44, 0x78, 0x6c, 0xa0, 0xb4, 0x88, 0x9c, 0xf0, 0xe4, 0xd8, 0xcc }, { 0x00, 0x1d, 0x3a, 0x27, 0x69, 0x74, 0x53, 0x4e, 0xd2, 0xcf, 0xe8, 0xf5, 0xbb, 0xa6, 0x81, 0x9c }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x15, 0x2a, 0x3f, 0x54, 0x41, 0x7e, 0x6b, 0xa8, 0xbd, 0x82, 0x97, 0xfc, 0xe9, 0xd6, 0xc3 }, { 0x00, 0x1d, 0x3a, 0x27, 0x69, 0x74, 0x53, 0x4e, 0xcf, 0xd2, 0xf5, 0xe8, 0xa6, 0xbb, 0x9c, 0x81 }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x16, 0x2c, 0x3a, 0x58, 0x4e, 0x74, 0x62, 0xb0, 0xa6, 0x9c, 0x8a, 0xe8, 0xfe, 0xc4, 0xd2 }, { 0x00, 0x1d, 0x3a, 0x27, 0x69, 0x74, 0x53, 0x4e, 0xcf, 0xd2, 0xf5, 0xe8, 0xa6, 0xbb, 0x9c, 0x81 }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x17, 0x2e, 0x39, 0x5c, 0x4b, 0x72, 0x65, 0xb8, 0xaf, 0x96, 0x81, 0xe4, 0xf3, 0xca, 0xdd }, { 0x00, 0x1d, 0x27, 0x3a, 0x4e, 0x53, 0x69, 0x74, 0x9c, 0x81, 0xbb, 0xa6, 0xd2, 0xcf, 0xf5, 0xe8 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x18, 0x30, 0x28, 0x60, 0x78, 0x50, 0x48, 0xc0, 0xd8, 0xf0, 0xe8, 0xa0, 0xb8, 0x90, 0x88 }, { 0x00, 0x1d, 0x27, 0x3a, 0x4e, 0x53, 0x69, 0x74, 0x9c, 0x81, 0xbb, 0xa6, 0xd2, 0xcf, 0xf5, 0xe8 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x19, 0x32, 0x2b, 0x64, 0x7d, 0x56, 0x4f, 0xc8, 0xd1, 0xfa, 0xe3, 0xac, 0xb5, 0x9e, 0x87 }, { 0x00, 0x1d, 0x27, 0x3a, 0x4e, 0x53, 0x69, 0x74, 0x81, 0x9c, 0xa6, 0xbb, 0xcf, 0xd2, 0xe8, 0xf5 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x1a, 0x34, 0x2e, 0x68, 0x72, 0x5c, 0x46, 0xd0, 0xca, 0xe4, 0xfe, 0xb8, 0xa2, 0x8c, 0x96 }, { 0x00, 0x1d, 0x27, 0x3a, 0x4e, 0x53, 0x69, 0x74, 0x81, 0x9c, 0xa6, 0xbb, 0xcf, 0xd2, 0xe8, 0xf5 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x1b, 0x36, 0x2d, 0x6c, 0x77, 0x5a, 0x41, 0xd8, 0xc3, 0xee, 0xf5, 0xb4, 0xaf, 0x82, 0x99 }, { 0x00, 0x1d, 0x27, 0x3a, 0x53, 0x4e, 0x74, 0x69, 0xa6, 0xbb, 0x81, 0x9c, 0xf5, 0xe8, 0xd2, 0xcf }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x1c, 0x38, 0x24, 0x70, 0x6c, 0x48, 0x54, 0xe0, 0xfc, 0xd8, 0xc4, 0x90, 0x8c, 0xa8, 0xb4 }, { 0x00, 0x1d, 0x27, 0x3a, 0x53, 0x4e, 0x74, 0x69, 0xa6, 0xbb, 0x81, 0x9c, 0xf5, 0xe8, 0xd2, 0xcf }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x1d, 0x3a, 0x27, 0x74, 0x69, 0x4e, 0x53, 0xe8, 0xf5, 0xd2, 0xcf, 0x9c, 0x81, 0xa6, 0xbb }, { 0x00, 0x1d, 0x27, 0x3a, 0x53, 0x4e, 0x74, 0x69, 0xbb, 0xa6, 0x9c, 0x81, 0xe8, 0xf5, 0xcf, 0xd2 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x1e, 0x3c, 0x22, 0x78, 0x66, 0x44, 0x5a, 0xf0, 0xee, 0xcc, 0xd2, 0x88, 0x96, 0xb4, 0xaa }, { 0x00, 0x1d, 0x27, 0x3a, 0x53, 0x4e, 0x74, 0x69, 0xbb, 0xa6, 0x9c, 0x81, 0xe8, 0xf5, 0xcf, 0xd2 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x1f, 0x3e, 0x21, 0x7c, 0x63, 0x42, 0x5d, 0xf8, 0xe7, 0xc6, 0xd9, 0x84, 0x9b, 0xba, 0xa5 }, { 0x00, 0x3a, 0x74, 0x4e, 0xe8, 0xd2, 0x9c, 0xa6, 0xcd, 0xf7, 0xb9, 0x83, 0x25, 0x1f, 0x51, 0x6b }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x3a, 0x74, 0x4e, 0xe8, 0xd2, 0x9c, 0xa6, 0xcd, 0xf7, 0xb9, 0x83, 0x25, 0x1f, 0x51, 0x6b }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x21, 0x42, 0x63, 0x84, 0xa5, 0xc6, 0xe7, 0x08, 0x29, 0x4a, 0x6b, 0x8c, 0xad, 0xce, 0xef }, { 0x00, 0x3a, 0x74, 0x4e, 0xe8, 0xd2, 0x9c, 0xa6, 0xd0, 0xea, 0xa4, 0x9e, 0x38, 0x02, 0x4c, 0x76 }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x22, 0x44, 0x66, 0x88, 0xaa, 0xcc, 0xee, 0x10, 0x32, 0x54, 0x76, 0x98, 0xba, 0xdc, 0xfe }, { 0x00, 0x3a, 0x74, 0x4e, 0xe8, 0xd2, 0x9c, 0xa6, 0xd0, 0xea, 0xa4, 0x9e, 0x38, 0x02, 0x4c, 0x76 }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x23, 0x46, 0x65, 0x8c, 0xaf, 0xca, 0xe9, 0x18, 0x3b, 0x5e, 0x7d, 0x94, 0xb7, 0xd2, 0xf1 }, { 0x00, 0x3a, 0x74, 0x4e, 0xf5, 0xcf, 0x81, 0xbb, 0xf7, 0xcd, 0x83, 0xb9, 0x02, 0x38, 0x76, 0x4c }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x24, 0x48, 0x6c, 0x90, 0xb4, 0xd8, 0xfc, 0x20, 0x04, 0x68, 0x4c, 0xb0, 0x94, 0xf8, 0xdc }, { 0x00, 0x3a, 0x74, 0x4e, 0xf5, 0xcf, 0x81, 0xbb, 0xf7, 0xcd, 0x83, 0xb9, 0x02, 0x38, 0x76, 0x4c }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x25, 0x4a, 0x6f, 0x94, 0xb1, 0xde, 0xfb, 0x28, 0x0d, 0x62, 0x47, 0xbc, 0x99, 0xf6, 0xd3 }, { 0x00, 0x3a, 0x74, 0x4e, 0xf5, 0xcf, 0x81, 0xbb, 0xea, 0xd0, 0x9e, 0xa4, 0x1f, 0x25, 0x6b, 0x51 }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x26, 0x4c, 0x6a, 0x98, 0xbe, 0xd4, 0xf2, 0x30, 0x16, 0x7c, 0x5a, 0xa8, 0x8e, 0xe4, 0xc2 }, { 0x00, 0x3a, 0x74, 0x4e, 0xf5, 0xcf, 0x81, 0xbb, 0xea, 0xd0, 0x9e, 0xa4, 0x1f, 0x25, 0x6b, 0x51 }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x27, 0x4e, 0x69, 0x9c, 0xbb, 0xd2, 0xf5, 0x38, 0x1f, 0x76, 0x51, 0xa4, 0x83, 0xea, 0xcd }, { 0x00, 0x3a, 0x69, 0x53, 0xd2, 0xe8, 0xbb, 0x81, 0xb9, 0x83, 0xd0, 0xea, 0x6b, 0x51, 0x02, 0x38 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x28, 0x50, 0x78, 0xa0, 0x88, 0xf0, 0xd8, 0x40, 0x68, 0x10, 0x38, 0xe0, 0xc8, 0xb0, 0x98 }, { 0x00, 0x3a, 0x69, 0x53, 0xd2, 0xe8, 0xbb, 0x81, 0xb9, 0x83, 0xd0, 0xea, 0x6b, 0x51, 0x02, 0x38 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x29, 0x52, 0x7b, 0xa4, 0x8d, 0xf6, 0xdf, 0x48, 0x61, 0x1a, 0x33, 0xec, 0xc5, 0xbe, 0x97 }, { 0x00, 0x3a, 0x69, 0x53, 0xd2, 0xe8, 0xbb, 0x81, 0xa4, 0x9e, 0xcd, 0xf7, 0x76, 0x4c, 0x1f, 0x25 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x2a, 0x54, 0x7e, 0xa8, 0x82, 0xfc, 0xd6, 0x50, 0x7a, 0x04, 0x2e, 0xf8, 0xd2, 0xac, 0x86 }, { 0x00, 0x3a, 0x69, 0x53, 0xd2, 0xe8, 0xbb, 0x81, 0xa4, 0x9e, 0xcd, 0xf7, 0x76, 0x4c, 0x1f, 0x25 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x2b, 0x56, 0x7d, 0xac, 0x87, 0xfa, 0xd1, 0x58, 0x73, 0x0e, 0x25, 0xf4, 0xdf, 0xa2, 0x89 }, { 0x00, 0x3a, 0x69, 0x53, 0xcf, 0xf5, 0xa6, 0x9c, 0x83, 0xb9, 0xea, 0xd0, 0x4c, 0x76, 0x25, 0x1f }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x2c, 0x58, 0x74, 0xb0, 0x9c, 0xe8, 0xc4, 0x60, 0x4c, 0x38, 0x14, 0xd0, 0xfc, 0x88, 0xa4 }, { 0x00, 0x3a, 0x69, 0x53, 0xcf, 0xf5, 0xa6, 0x9c, 0x83, 0xb9, 0xea, 0xd0, 0x4c, 0x76, 0x25, 0x1f }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x2d, 0x5a, 0x77, 0xb4, 0x99, 0xee, 0xc3, 0x68, 0x45, 0x32, 0x1f, 0xdc, 0xf1, 0x86, 0xab }, { 0x00, 0x3a, 0x69, 0x53, 0xcf, 0xf5, 0xa6, 0x9c, 0x9e, 0xa4, 0xf7, 0xcd, 0x51, 0x6b, 0x38, 0x02 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x2e, 0x5c, 0x72, 0xb8, 0x96, 0xe4, 0xca, 0x70, 0x5e, 0x2c, 0x02, 0xc8, 0xe6, 0x94, 0xba }, { 0x00, 0x3a, 0x69, 0x53, 0xcf, 0xf5, 0xa6, 0x9c, 0x9e, 0xa4, 0xf7, 0xcd, 0x51, 0x6b, 0x38, 0x02 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x2f, 0x5e, 0x71, 0xbc, 0x93, 0xe2, 0xcd, 0x78, 0x57, 0x26, 0x09, 0xc4, 0xeb, 0x9a, 0xb5 }, { 0x00, 0x27, 0x4e, 0x69, 0x9c, 0xbb, 0xd2, 0xf5, 0x25, 0x02, 0x6b, 0x4c, 0xb9, 0x9e, 0xf7, 0xd0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x27, 0x4e, 0x69, 0x9c, 0xbb, 0xd2, 0xf5, 0x25, 0x02, 0x6b, 0x4c, 0xb9, 0x9e, 0xf7, 0xd0 }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x31, 0x62, 0x53, 0xc4, 0xf5, 0xa6, 0x97, 0x88, 0xb9, 0xea, 0xdb, 0x4c, 0x7d, 0x2e, 0x1f }, { 0x00, 0x27, 0x4e, 0x69, 0x9c, 0xbb, 0xd2, 0xf5, 0x38, 0x1f, 0x76, 0x51, 0xa4, 0x83, 0xea, 0xcd }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x32, 0x64, 0x56, 0xc8, 0xfa, 0xac, 0x9e, 0x90, 0xa2, 0xf4, 0xc6, 0x58, 0x6a, 0x3c, 0x0e }, { 0x00, 0x27, 0x4e, 0x69, 0x9c, 0xbb, 0xd2, 0xf5, 0x38, 0x1f, 0x76, 0x51, 0xa4, 0x83, 0xea, 0xcd }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x33, 0x66, 0x55, 0xcc, 0xff, 0xaa, 0x99, 0x98, 0xab, 0xfe, 0xcd, 0x54, 0x67, 0x32, 0x01 }, { 0x00, 0x27, 0x4e, 0x69, 0x81, 0xa6, 0xcf, 0xe8, 0x1f, 0x38, 0x51, 0x76, 0x9e, 0xb9, 0xd0, 0xf7 }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x34, 0x68, 0x5c, 0xd0, 0xe4, 0xb8, 0x8c, 0xa0, 0x94, 0xc8, 0xfc, 0x70, 0x44, 0x18, 0x2c }, { 0x00, 0x27, 0x4e, 0x69, 0x81, 0xa6, 0xcf, 0xe8, 0x1f, 0x38, 0x51, 0x76, 0x9e, 0xb9, 0xd0, 0xf7 }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x35, 0x6a, 0x5f, 0xd4, 0xe1, 0xbe, 0x8b, 0xa8, 0x9d, 0xc2, 0xf7, 0x7c, 0x49, 0x16, 0x23 }, { 0x00, 0x27, 0x4e, 0x69, 0x81, 0xa6, 0xcf, 0xe8, 0x02, 0x25, 0x4c, 0x6b, 0x83, 0xa4, 0xcd, 0xea }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x36, 0x6c, 0x5a, 0xd8, 0xee, 0xb4, 0x82, 0xb0, 0x86, 0xdc, 0xea, 0x68, 0x5e, 0x04, 0x32 }, { 0x00, 0x27, 0x4e, 0x69, 0x81, 0xa6, 0xcf, 0xe8, 0x02, 0x25, 0x4c, 0x6b, 0x83, 0xa4, 0xcd, 0xea }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x37, 0x6e, 0x59, 0xdc, 0xeb, 0xb2, 0x85, 0xb8, 0x8f, 0xd6, 0xe1, 0x64, 0x53, 0x0a, 0x3d }, { 0x00, 0x27, 0x53, 0x74, 0xa6, 0x81, 0xf5, 0xd2, 0x51, 0x76, 0x02, 0x25, 0xf7, 0xd0, 0xa4, 0x83 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x38, 0x70, 0x48, 0xe0, 0xd8, 0x90, 0xa8, 0xc0, 0xf8, 0xb0, 0x88, 0x20, 0x18, 0x50, 0x68 }, { 0x00, 0x27, 0x53, 0x74, 0xa6, 0x81, 0xf5, 0xd2, 0x51, 0x76, 0x02, 0x25, 0xf7, 0xd0, 0xa4, 0x83 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x39, 0x72, 0x4b, 0xe4, 0xdd, 0x96, 0xaf, 0xc8, 0xf1, 0xba, 0x83, 0x2c, 0x15, 0x5e, 0x67 }, { 0x00, 0x27, 0x53, 0x74, 0xa6, 0x81, 0xf5, 0xd2, 0x4c, 0x6b, 0x1f, 0x38, 0xea, 0xcd, 0xb9, 0x9e }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x3a, 0x74, 0x4e, 0xe8, 0xd2, 0x9c, 0xa6, 0xd0, 0xea, 0xa4, 0x9e, 0x38, 0x02, 0x4c, 0x76 }, { 0x00, 0x27, 0x53, 0x74, 0xa6, 0x81, 0xf5, 0xd2, 0x4c, 0x6b, 0x1f, 0x38, 0xea, 0xcd, 0xb9, 0x9e }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x3b, 0x76, 0x4d, 0xec, 0xd7, 0x9a, 0xa1, 0xd8, 0xe3, 0xae, 0x95, 0x34, 0x0f, 0x42, 0x79 }, { 0x00, 0x27, 0x53, 0x74, 0xbb, 0x9c, 0xe8, 0xcf, 0x6b, 0x4c, 0x38, 0x1f, 0xd0, 0xf7, 0x83, 0xa4 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x3c, 0x78, 0x44, 0xf0, 0xcc, 0x88, 0xb4, 0xe0, 0xdc, 0x98, 0xa4, 0x10, 0x2c, 0x68, 0x54 }, { 0x00, 0x27, 0x53, 0x74, 0xbb, 0x9c, 0xe8, 0xcf, 0x6b, 0x4c, 0x38, 0x1f, 0xd0, 0xf7, 0x83, 0xa4 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x3d, 0x7a, 0x47, 0xf4, 0xc9, 0x8e, 0xb3, 0xe8, 0xd5, 0x92, 0xaf, 0x1c, 0x21, 0x66, 0x5b }, { 0x00, 0x27, 0x53, 0x74, 0xbb, 0x9c, 0xe8, 0xcf, 0x76, 0x51, 0x25, 0x02, 0xcd, 0xea, 0x9e, 0xb9 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x3e, 0x7c, 0x42, 0xf8, 0xc6, 0x84, 0xba, 0xf0, 0xce, 0x8c, 0xb2, 0x08, 0x36, 0x74, 0x4a }, { 0x00, 0x27, 0x53, 0x74, 0xbb, 0x9c, 0xe8, 0xcf, 0x76, 0x51, 0x25, 0x02, 0xcd, 0xea, 0x9e, 0xb9 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x00, 0x3f, 0x7e, 0x41, 0xfc, 0xc3, 0x82, 0xbd, 0xf8, 0xc7, 0x86, 0xb9, 0x04, 0x3b, 0x7a, 0x45 }, { 0x00, 0x74, 0xe8, 0x9c, 0xcd, 0xb9, 0x25, 0x51, 0x87, 0xf3, 0x6f, 0x1b, 0x4a, 0x3e, 0xa2, 0xd6 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x74, 0xe8, 0x9c, 0xcd, 0xb9, 0x25, 0x51, 0x87, 0xf3, 0x6f, 0x1b, 0x4a, 0x3e, 0xa2, 0xd6 }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x41, 0x82, 0xc3, 0x04, 0x45, 0x86, 0xc7, 0x08, 0x49, 0x8a, 0xcb, 0x0c, 0x4d, 0x8e, 0xcf }, { 0x00, 0x74, 0xe8, 0x9c, 0xcd, 0xb9, 0x25, 0x51, 0x9a, 0xee, 0x72, 0x06, 0x57, 0x23, 0xbf, 0xcb }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x42, 0x84, 0xc6, 0x08, 0x4a, 0x8c, 0xce, 0x10, 0x52, 0x94, 0xd6, 0x18, 0x5a, 0x9c, 0xde }, { 0x00, 0x74, 0xe8, 0x9c, 0xcd, 0xb9, 0x25, 0x51, 0x9a, 0xee, 0x72, 0x06, 0x57, 0x23, 0xbf, 0xcb }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x43, 0x86, 0xc5, 0x0c, 0x4f, 0x8a, 0xc9, 0x18, 0x5b, 0x9e, 0xdd, 0x14, 0x57, 0x92, 0xd1 }, { 0x00, 0x74, 0xe8, 0x9c, 0xd0, 0xa4, 0x38, 0x4c, 0xbd, 0xc9, 0x55, 0x21, 0x6d, 0x19, 0x85, 0xf1 }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x44, 0x88, 0xcc, 0x10, 0x54, 0x98, 0xdc, 0x20, 0x64, 0xa8, 0xec, 0x30, 0x74, 0xb8, 0xfc }, { 0x00, 0x74, 0xe8, 0x9c, 0xd0, 0xa4, 0x38, 0x4c, 0xbd, 0xc9, 0x55, 0x21, 0x6d, 0x19, 0x85, 0xf1 }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x45, 0x8a, 0xcf, 0x14, 0x51, 0x9e, 0xdb, 0x28, 0x6d, 0xa2, 0xe7, 0x3c, 0x79, 0xb6, 0xf3 }, { 0x00, 0x74, 0xe8, 0x9c, 0xd0, 0xa4, 0x38, 0x4c, 0xa0, 0xd4, 0x48, 0x3c, 0x70, 0x04, 0x98, 0xec }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x46, 0x8c, 0xca, 0x18, 0x5e, 0x94, 0xd2, 0x30, 0x76, 0xbc, 0xfa, 0x28, 0x6e, 0xa4, 0xe2 }, { 0x00, 0x74, 0xe8, 0x9c, 0xd0, 0xa4, 0x38, 0x4c, 0xa0, 0xd4, 0x48, 0x3c, 0x70, 0x04, 0x98, 0xec }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x47, 0x8e, 0xc9, 0x1c, 0x5b, 0x92, 0xd5, 0x38, 0x7f, 0xb6, 0xf1, 0x24, 0x63, 0xaa, 0xed }, { 0x00, 0x74, 0xf5, 0x81, 0xf7, 0x83, 0x02, 0x76, 0xf3, 0x87, 0x06, 0x72, 0x04, 0x70, 0xf1, 0x85 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x48, 0x90, 0xd8, 0x20, 0x68, 0xb0, 0xf8, 0x40, 0x08, 0xd0, 0x98, 0x60, 0x28, 0xf0, 0xb8 }, { 0x00, 0x74, 0xf5, 0x81, 0xf7, 0x83, 0x02, 0x76, 0xf3, 0x87, 0x06, 0x72, 0x04, 0x70, 0xf1, 0x85 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x49, 0x92, 0xdb, 0x24, 0x6d, 0xb6, 0xff, 0x48, 0x01, 0xda, 0x93, 0x6c, 0x25, 0xfe, 0xb7 }, { 0x00, 0x74, 0xf5, 0x81, 0xf7, 0x83, 0x02, 0x76, 0xee, 0x9a, 0x1b, 0x6f, 0x19, 0x6d, 0xec, 0x98 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x4a, 0x94, 0xde, 0x28, 0x62, 0xbc, 0xf6, 0x50, 0x1a, 0xc4, 0x8e, 0x78, 0x32, 0xec, 0xa6 }, { 0x00, 0x74, 0xf5, 0x81, 0xf7, 0x83, 0x02, 0x76, 0xee, 0x9a, 0x1b, 0x6f, 0x19, 0x6d, 0xec, 0x98 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x4b, 0x96, 0xdd, 0x2c, 0x67, 0xba, 0xf1, 0x58, 0x13, 0xce, 0x85, 0x74, 0x3f, 0xe2, 0xa9 }, { 0x00, 0x74, 0xf5, 0x81, 0xea, 0x9e, 0x1f, 0x6b, 0xc9, 0xbd, 0x3c, 0x48, 0x23, 0x57, 0xd6, 0xa2 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x4c, 0x98, 0xd4, 0x30, 0x7c, 0xa8, 0xe4, 0x60, 0x2c, 0xf8, 0xb4, 0x50, 0x1c, 0xc8, 0x84 }, { 0x00, 0x74, 0xf5, 0x81, 0xea, 0x9e, 0x1f, 0x6b, 0xc9, 0xbd, 0x3c, 0x48, 0x23, 0x57, 0xd6, 0xa2 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x4d, 0x9a, 0xd7, 0x34, 0x79, 0xae, 0xe3, 0x68, 0x25, 0xf2, 0xbf, 0x5c, 0x11, 0xc6, 0x8b }, { 0x00, 0x74, 0xf5, 0x81, 0xea, 0x9e, 0x1f, 0x6b, 0xd4, 0xa0, 0x21, 0x55, 0x3e, 0x4a, 0xcb, 0xbf }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x4e, 0x9c, 0xd2, 0x38, 0x76, 0xa4, 0xea, 0x70, 0x3e, 0xec, 0xa2, 0x48, 0x06, 0xd4, 0x9a }, { 0x00, 0x74, 0xf5, 0x81, 0xea, 0x9e, 0x1f, 0x6b, 0xd4, 0xa0, 0x21, 0x55, 0x3e, 0x4a, 0xcb, 0xbf }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x4f, 0x9e, 0xd1, 0x3c, 0x73, 0xa2, 0xed, 0x78, 0x37, 0xe6, 0xa9, 0x44, 0x0b, 0xda, 0x95 }, { 0x00, 0x69, 0xd2, 0xbb, 0xb9, 0xd0, 0x6b, 0x02, 0x6f, 0x06, 0xbd, 0xd4, 0xd6, 0xbf, 0x04, 0x6d }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x69, 0xd2, 0xbb, 0xb9, 0xd0, 0x6b, 0x02, 0x6f, 0x06, 0xbd, 0xd4, 0xd6, 0xbf, 0x04, 0x6d }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x51, 0xa2, 0xf3, 0x44, 0x15, 0xe6, 0xb7, 0x88, 0xd9, 0x2a, 0x7b, 0xcc, 0x9d, 0x6e, 0x3f }, { 0x00, 0x69, 0xd2, 0xbb, 0xb9, 0xd0, 0x6b, 0x02, 0x72, 0x1b, 0xa0, 0xc9, 0xcb, 0xa2, 0x19, 0x70 }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x52, 0xa4, 0xf6, 0x48, 0x1a, 0xec, 0xbe, 0x90, 0xc2, 0x34, 0x66, 0xd8, 0x8a, 0x7c, 0x2e }, { 0x00, 0x69, 0xd2, 0xbb, 0xb9, 0xd0, 0x6b, 0x02, 0x72, 0x1b, 0xa0, 0xc9, 0xcb, 0xa2, 0x19, 0x70 }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x53, 0xa6, 0xf5, 0x4c, 0x1f, 0xea, 0xb9, 0x98, 0xcb, 0x3e, 0x6d, 0xd4, 0x87, 0x72, 0x21 }, { 0x00, 0x69, 0xd2, 0xbb, 0xa4, 0xcd, 0x76, 0x1f, 0x55, 0x3c, 0x87, 0xee, 0xf1, 0x98, 0x23, 0x4a }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x54, 0xa8, 0xfc, 0x50, 0x04, 0xf8, 0xac, 0xa0, 0xf4, 0x08, 0x5c, 0xf0, 0xa4, 0x58, 0x0c }, { 0x00, 0x69, 0xd2, 0xbb, 0xa4, 0xcd, 0x76, 0x1f, 0x55, 0x3c, 0x87, 0xee, 0xf1, 0x98, 0x23, 0x4a }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x55, 0xaa, 0xff, 0x54, 0x01, 0xfe, 0xab, 0xa8, 0xfd, 0x02, 0x57, 0xfc, 0xa9, 0x56, 0x03 }, { 0x00, 0x69, 0xd2, 0xbb, 0xa4, 0xcd, 0x76, 0x1f, 0x48, 0x21, 0x9a, 0xf3, 0xec, 0x85, 0x3e, 0x57 }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x56, 0xac, 0xfa, 0x58, 0x0e, 0xf4, 0xa2, 0xb0, 0xe6, 0x1c, 0x4a, 0xe8, 0xbe, 0x44, 0x12 }, { 0x00, 0x69, 0xd2, 0xbb, 0xa4, 0xcd, 0x76, 0x1f, 0x48, 0x21, 0x9a, 0xf3, 0xec, 0x85, 0x3e, 0x57 }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x57, 0xae, 0xf9, 0x5c, 0x0b, 0xf2, 0xa5, 0xb8, 0xef, 0x16, 0x41, 0xe4, 0xb3, 0x4a, 0x1d }, { 0x00, 0x69, 0xcf, 0xa6, 0x83, 0xea, 0x4c, 0x25, 0x1b, 0x72, 0xd4, 0xbd, 0x98, 0xf1, 0x57, 0x3e }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x58, 0xb0, 0xe8, 0x60, 0x38, 0xd0, 0x88, 0xc0, 0x98, 0x70, 0x28, 0xa0, 0xf8, 0x10, 0x48 }, { 0x00, 0x69, 0xcf, 0xa6, 0x83, 0xea, 0x4c, 0x25, 0x1b, 0x72, 0xd4, 0xbd, 0x98, 0xf1, 0x57, 0x3e }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x59, 0xb2, 0xeb, 0x64, 0x3d, 0xd6, 0x8f, 0xc8, 0x91, 0x7a, 0x23, 0xac, 0xf5, 0x1e, 0x47 }, { 0x00, 0x69, 0xcf, 0xa6, 0x83, 0xea, 0x4c, 0x25, 0x06, 0x6f, 0xc9, 0xa0, 0x85, 0xec, 0x4a, 0x23 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x5a, 0xb4, 0xee, 0x68, 0x32, 0xdc, 0x86, 0xd0, 0x8a, 0x64, 0x3e, 0xb8, 0xe2, 0x0c, 0x56 }, { 0x00, 0x69, 0xcf, 0xa6, 0x83, 0xea, 0x4c, 0x25, 0x06, 0x6f, 0xc9, 0xa0, 0x85, 0xec, 0x4a, 0x23 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x5b, 0xb6, 0xed, 0x6c, 0x37, 0xda, 0x81, 0xd8, 0x83, 0x6e, 0x35, 0xb4, 0xef, 0x02, 0x59 }, { 0x00, 0x69, 0xcf, 0xa6, 0x9e, 0xf7, 0x51, 0x38, 0x21, 0x48, 0xee, 0x87, 0xbf, 0xd6, 0x70, 0x19 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x5c, 0xb8, 0xe4, 0x70, 0x2c, 0xc8, 0x94, 0xe0, 0xbc, 0x58, 0x04, 0x90, 0xcc, 0x28, 0x74 }, { 0x00, 0x69, 0xcf, 0xa6, 0x9e, 0xf7, 0x51, 0x38, 0x21, 0x48, 0xee, 0x87, 0xbf, 0xd6, 0x70, 0x19 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x5d, 0xba, 0xe7, 0x74, 0x29, 0xce, 0x93, 0xe8, 0xb5, 0x52, 0x0f, 0x9c, 0xc1, 0x26, 0x7b }, { 0x00, 0x69, 0xcf, 0xa6, 0x9e, 0xf7, 0x51, 0x38, 0x3c, 0x55, 0xf3, 0x9a, 0xa2, 0xcb, 0x6d, 0x04 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x5e, 0xbc, 0xe2, 0x78, 0x26, 0xc4, 0x9a, 0xf0, 0xae, 0x4c, 0x12, 0x88, 0xd6, 0x34, 0x6a }, { 0x00, 0x69, 0xcf, 0xa6, 0x9e, 0xf7, 0x51, 0x38, 0x3c, 0x55, 0xf3, 0x9a, 0xa2, 0xcb, 0x6d, 0x04 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x3a, 0x3a, 0x3a, 0x3a, 0x27, 0x27, 0x27, 0x27 }, { 0x00, 0x5f, 0xbe, 0xe1, 0x7c, 0x23, 0xc2, 0x9d, 0xf8, 0xa7, 0x46, 0x19, 0x84, 0xdb, 0x3a, 0x65 }, { 0x00, 0x4e, 0x9c, 0xd2, 0x25, 0x6b, 0xb9, 0xf7, 0x4a, 0x04, 0xd6, 0x98, 0x6f, 0x21, 0xf3, 0xbd }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x4e, 0x9c, 0xd2, 0x25, 0x6b, 0xb9, 0xf7, 0x4a, 0x04, 0xd6, 0x98, 0x6f, 0x21, 0xf3, 0xbd }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x61, 0xc2, 0xa3, 0x84, 0xe5, 0x46, 0x27, 0x08, 0x69, 0xca, 0xab, 0x8c, 0xed, 0x4e, 0x2f }, { 0x00, 0x4e, 0x9c, 0xd2, 0x25, 0x6b, 0xb9, 0xf7, 0x57, 0x19, 0xcb, 0x85, 0x72, 0x3c, 0xee, 0xa0 }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x62, 0xc4, 0xa6, 0x88, 0xea, 0x4c, 0x2e, 0x10, 0x72, 0xd4, 0xb6, 0x98, 0xfa, 0x5c, 0x3e }, { 0x00, 0x4e, 0x9c, 0xd2, 0x25, 0x6b, 0xb9, 0xf7, 0x57, 0x19, 0xcb, 0x85, 0x72, 0x3c, 0xee, 0xa0 }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x63, 0xc6, 0xa5, 0x8c, 0xef, 0x4a, 0x29, 0x18, 0x7b, 0xde, 0xbd, 0x94, 0xf7, 0x52, 0x31 }, { 0x00, 0x4e, 0x9c, 0xd2, 0x38, 0x76, 0xa4, 0xea, 0x70, 0x3e, 0xec, 0xa2, 0x48, 0x06, 0xd4, 0x9a }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x64, 0xc8, 0xac, 0x90, 0xf4, 0x58, 0x3c, 0x20, 0x44, 0xe8, 0x8c, 0xb0, 0xd4, 0x78, 0x1c }, { 0x00, 0x4e, 0x9c, 0xd2, 0x38, 0x76, 0xa4, 0xea, 0x70, 0x3e, 0xec, 0xa2, 0x48, 0x06, 0xd4, 0x9a }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x65, 0xca, 0xaf, 0x94, 0xf1, 0x5e, 0x3b, 0x28, 0x4d, 0xe2, 0x87, 0xbc, 0xd9, 0x76, 0x13 }, { 0x00, 0x4e, 0x9c, 0xd2, 0x38, 0x76, 0xa4, 0xea, 0x6d, 0x23, 0xf1, 0xbf, 0x55, 0x1b, 0xc9, 0x87 }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x66, 0xcc, 0xaa, 0x98, 0xfe, 0x54, 0x32, 0x30, 0x56, 0xfc, 0x9a, 0xa8, 0xce, 0x64, 0x02 }, { 0x00, 0x4e, 0x9c, 0xd2, 0x38, 0x76, 0xa4, 0xea, 0x6d, 0x23, 0xf1, 0xbf, 0x55, 0x1b, 0xc9, 0x87 }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x67, 0xce, 0xa9, 0x9c, 0xfb, 0x52, 0x35, 0x38, 0x5f, 0xf6, 0x91, 0xa4, 0xc3, 0x6a, 0x0d }, { 0x00, 0x4e, 0x81, 0xcf, 0x1f, 0x51, 0x9e, 0xd0, 0x3e, 0x70, 0xbf, 0xf1, 0x21, 0x6f, 0xa0, 0xee }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x68, 0xd0, 0xb8, 0xa0, 0xc8, 0x70, 0x18, 0x40, 0x28, 0x90, 0xf8, 0xe0, 0x88, 0x30, 0x58 }, { 0x00, 0x4e, 0x81, 0xcf, 0x1f, 0x51, 0x9e, 0xd0, 0x3e, 0x70, 0xbf, 0xf1, 0x21, 0x6f, 0xa0, 0xee }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x69, 0xd2, 0xbb, 0xa4, 0xcd, 0x76, 0x1f, 0x48, 0x21, 0x9a, 0xf3, 0xec, 0x85, 0x3e, 0x57 }, { 0x00, 0x4e, 0x81, 0xcf, 0x1f, 0x51, 0x9e, 0xd0, 0x23, 0x6d, 0xa2, 0xec, 0x3c, 0x72, 0xbd, 0xf3 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x6a, 0xd4, 0xbe, 0xa8, 0xc2, 0x7c, 0x16, 0x50, 0x3a, 0x84, 0xee, 0xf8, 0x92, 0x2c, 0x46 }, { 0x00, 0x4e, 0x81, 0xcf, 0x1f, 0x51, 0x9e, 0xd0, 0x23, 0x6d, 0xa2, 0xec, 0x3c, 0x72, 0xbd, 0xf3 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x6b, 0xd6, 0xbd, 0xac, 0xc7, 0x7a, 0x11, 0x58, 0x33, 0x8e, 0xe5, 0xf4, 0x9f, 0x22, 0x49 }, { 0x00, 0x4e, 0x81, 0xcf, 0x02, 0x4c, 0x83, 0xcd, 0x04, 0x4a, 0x85, 0xcb, 0x06, 0x48, 0x87, 0xc9 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x6c, 0xd8, 0xb4, 0xb0, 0xdc, 0x68, 0x04, 0x60, 0x0c, 0xb8, 0xd4, 0xd0, 0xbc, 0x08, 0x64 }, { 0x00, 0x4e, 0x81, 0xcf, 0x02, 0x4c, 0x83, 0xcd, 0x04, 0x4a, 0x85, 0xcb, 0x06, 0x48, 0x87, 0xc9 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x6d, 0xda, 0xb7, 0xb4, 0xd9, 0x6e, 0x03, 0x68, 0x05, 0xb2, 0xdf, 0xdc, 0xb1, 0x06, 0x6b }, { 0x00, 0x4e, 0x81, 0xcf, 0x02, 0x4c, 0x83, 0xcd, 0x19, 0x57, 0x98, 0xd6, 0x1b, 0x55, 0x9a, 0xd4 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x6e, 0xdc, 0xb2, 0xb8, 0xd6, 0x64, 0x0a, 0x70, 0x1e, 0xac, 0xc2, 0xc8, 0xa6, 0x14, 0x7a }, { 0x00, 0x4e, 0x81, 0xcf, 0x02, 0x4c, 0x83, 0xcd, 0x19, 0x57, 0x98, 0xd6, 0x1b, 0x55, 0x9a, 0xd4 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x6f, 0xde, 0xb1, 0xbc, 0xd3, 0x62, 0x0d, 0x78, 0x17, 0xa6, 0xc9, 0xc4, 0xab, 0x1a, 0x75 }, { 0x00, 0x53, 0xa6, 0xf5, 0x51, 0x02, 0xf7, 0xa4, 0xa2, 0xf1, 0x04, 0x57, 0xf3, 0xa0, 0x55, 0x06 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x53, 0xa6, 0xf5, 0x51, 0x02, 0xf7, 0xa4, 0xa2, 0xf1, 0x04, 0x57, 0xf3, 0xa0, 0x55, 0x06 }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x71, 0xe2, 0x93, 0xc4, 0xb5, 0x26, 0x57, 0x88, 0xf9, 0x6a, 0x1b, 0x4c, 0x3d, 0xae, 0xdf }, { 0x00, 0x53, 0xa6, 0xf5, 0x51, 0x02, 0xf7, 0xa4, 0xbf, 0xec, 0x19, 0x4a, 0xee, 0xbd, 0x48, 0x1b }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x72, 0xe4, 0x96, 0xc8, 0xba, 0x2c, 0x5e, 0x90, 0xe2, 0x74, 0x06, 0x58, 0x2a, 0xbc, 0xce }, { 0x00, 0x53, 0xa6, 0xf5, 0x51, 0x02, 0xf7, 0xa4, 0xbf, 0xec, 0x19, 0x4a, 0xee, 0xbd, 0x48, 0x1b }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x73, 0xe6, 0x95, 0xcc, 0xbf, 0x2a, 0x59, 0x98, 0xeb, 0x7e, 0x0d, 0x54, 0x27, 0xb2, 0xc1 }, { 0x00, 0x53, 0xa6, 0xf5, 0x4c, 0x1f, 0xea, 0xb9, 0x98, 0xcb, 0x3e, 0x6d, 0xd4, 0x87, 0x72, 0x21 }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x74, 0xe8, 0x9c, 0xd0, 0xa4, 0x38, 0x4c, 0xa0, 0xd4, 0x48, 0x3c, 0x70, 0x04, 0x98, 0xec }, { 0x00, 0x53, 0xa6, 0xf5, 0x4c, 0x1f, 0xea, 0xb9, 0x98, 0xcb, 0x3e, 0x6d, 0xd4, 0x87, 0x72, 0x21 }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x75, 0xea, 0x9f, 0xd4, 0xa1, 0x3e, 0x4b, 0xa8, 0xdd, 0x42, 0x37, 0x7c, 0x09, 0x96, 0xe3 }, { 0x00, 0x53, 0xa6, 0xf5, 0x4c, 0x1f, 0xea, 0xb9, 0x85, 0xd6, 0x23, 0x70, 0xc9, 0x9a, 0x6f, 0x3c }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x76, 0xec, 0x9a, 0xd8, 0xae, 0x34, 0x42, 0xb0, 0xc6, 0x5c, 0x2a, 0x68, 0x1e, 0x84, 0xf2 }, { 0x00, 0x53, 0xa6, 0xf5, 0x4c, 0x1f, 0xea, 0xb9, 0x85, 0xd6, 0x23, 0x70, 0xc9, 0x9a, 0x6f, 0x3c }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x77, 0xee, 0x99, 0xdc, 0xab, 0x32, 0x45, 0xb8, 0xcf, 0x56, 0x21, 0x64, 0x13, 0x8a, 0xfd }, { 0x00, 0x53, 0xbb, 0xe8, 0x6b, 0x38, 0xd0, 0x83, 0xd6, 0x85, 0x6d, 0x3e, 0xbd, 0xee, 0x06, 0x55 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x78, 0xf0, 0x88, 0xe0, 0x98, 0x10, 0x68, 0xc0, 0xb8, 0x30, 0x48, 0x20, 0x58, 0xd0, 0xa8 }, { 0x00, 0x53, 0xbb, 0xe8, 0x6b, 0x38, 0xd0, 0x83, 0xd6, 0x85, 0x6d, 0x3e, 0xbd, 0xee, 0x06, 0x55 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x79, 0xf2, 0x8b, 0xe4, 0x9d, 0x16, 0x6f, 0xc8, 0xb1, 0x3a, 0x43, 0x2c, 0x55, 0xde, 0xa7 }, { 0x00, 0x53, 0xbb, 0xe8, 0x6b, 0x38, 0xd0, 0x83, 0xcb, 0x98, 0x70, 0x23, 0xa0, 0xf3, 0x1b, 0x48 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x7a, 0xf4, 0x8e, 0xe8, 0x92, 0x1c, 0x66, 0xd0, 0xaa, 0x24, 0x5e, 0x38, 0x42, 0xcc, 0xb6 }, { 0x00, 0x53, 0xbb, 0xe8, 0x6b, 0x38, 0xd0, 0x83, 0xcb, 0x98, 0x70, 0x23, 0xa0, 0xf3, 0x1b, 0x48 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x7b, 0xf6, 0x8d, 0xec, 0x97, 0x1a, 0x61, 0xd8, 0xa3, 0x2e, 0x55, 0x34, 0x4f, 0xc2, 0xb9 }, { 0x00, 0x53, 0xbb, 0xe8, 0x76, 0x25, 0xcd, 0x9e, 0xec, 0xbf, 0x57, 0x04, 0x9a, 0xc9, 0x21, 0x72 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x7c, 0xf8, 0x84, 0xf0, 0x8c, 0x08, 0x74, 0xe0, 0x9c, 0x18, 0x64, 0x10, 0x6c, 0xe8, 0x94 }, { 0x00, 0x53, 0xbb, 0xe8, 0x76, 0x25, 0xcd, 0x9e, 0xec, 0xbf, 0x57, 0x04, 0x9a, 0xc9, 0x21, 0x72 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x7d, 0xfa, 0x87, 0xf4, 0x89, 0x0e, 0x73, 0xe8, 0x95, 0x12, 0x6f, 0x1c, 0x61, 0xe6, 0x9b }, { 0x00, 0x53, 0xbb, 0xe8, 0x76, 0x25, 0xcd, 0x9e, 0xf1, 0xa2, 0x4a, 0x19, 0x87, 0xd4, 0x3c, 0x6f }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x7e, 0xfc, 0x82, 0xf8, 0x86, 0x04, 0x7a, 0xf0, 0x8e, 0x0c, 0x72, 0x08, 0x76, 0xf4, 0x8a }, { 0x00, 0x53, 0xbb, 0xe8, 0x76, 0x25, 0xcd, 0x9e, 0xf1, 0xa2, 0x4a, 0x19, 0x87, 0xd4, 0x3c, 0x6f }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x00, 0x00, 0x1d, 0x1d, 0x1d, 0x1d, 0x27, 0x27, 0x27, 0x27, 0x3a, 0x3a, 0x3a, 0x3a }, { 0x00, 0x7f, 0xfe, 0x81, 0xfc, 0x83, 0x02, 0x7d, 0xf8, 0x87, 0x06, 0x79, 0x04, 0x7b, 0xfa, 0x85 }, { 0x00, 0xe8, 0xcd, 0x25, 0x87, 0x6f, 0x4a, 0xa2, 0x13, 0xfb, 0xde, 0x36, 0x94, 0x7c, 0x59, 0xb1 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0xe8, 0xcd, 0x25, 0x87, 0x6f, 0x4a, 0xa2, 0x13, 0xfb, 0xde, 0x36, 0x94, 0x7c, 0x59, 0xb1 }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x81, 0x02, 0x83, 0x04, 0x85, 0x06, 0x87, 0x08, 0x89, 0x0a, 0x8b, 0x0c, 0x8d, 0x0e, 0x8f }, { 0x00, 0xe8, 0xcd, 0x25, 0x87, 0x6f, 0x4a, 0xa2, 0x0e, 0xe6, 0xc3, 0x2b, 0x89, 0x61, 0x44, 0xac }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x82, 0x04, 0x86, 0x08, 0x8a, 0x0c, 0x8e, 0x10, 0x92, 0x14, 0x96, 0x18, 0x9a, 0x1c, 0x9e }, { 0x00, 0xe8, 0xcd, 0x25, 0x87, 0x6f, 0x4a, 0xa2, 0x0e, 0xe6, 0xc3, 0x2b, 0x89, 0x61, 0x44, 0xac }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x83, 0x06, 0x85, 0x0c, 0x8f, 0x0a, 0x89, 0x18, 0x9b, 0x1e, 0x9d, 0x14, 0x97, 0x12, 0x91 }, { 0x00, 0xe8, 0xcd, 0x25, 0x9a, 0x72, 0x57, 0xbf, 0x29, 0xc1, 0xe4, 0x0c, 0xb3, 0x5b, 0x7e, 0x96 }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x84, 0x08, 0x8c, 0x10, 0x94, 0x18, 0x9c, 0x20, 0xa4, 0x28, 0xac, 0x30, 0xb4, 0x38, 0xbc }, { 0x00, 0xe8, 0xcd, 0x25, 0x9a, 0x72, 0x57, 0xbf, 0x29, 0xc1, 0xe4, 0x0c, 0xb3, 0x5b, 0x7e, 0x96 }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x85, 0x0a, 0x8f, 0x14, 0x91, 0x1e, 0x9b, 0x28, 0xad, 0x22, 0xa7, 0x3c, 0xb9, 0x36, 0xb3 }, { 0x00, 0xe8, 0xcd, 0x25, 0x9a, 0x72, 0x57, 0xbf, 0x34, 0xdc, 0xf9, 0x11, 0xae, 0x46, 0x63, 0x8b }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x86, 0x0c, 0x8a, 0x18, 0x9e, 0x14, 0x92, 0x30, 0xb6, 0x3c, 0xba, 0x28, 0xae, 0x24, 0xa2 }, { 0x00, 0xe8, 0xcd, 0x25, 0x9a, 0x72, 0x57, 0xbf, 0x34, 0xdc, 0xf9, 0x11, 0xae, 0x46, 0x63, 0x8b }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x87, 0x0e, 0x89, 0x1c, 0x9b, 0x12, 0x95, 0x38, 0xbf, 0x36, 0xb1, 0x24, 0xa3, 0x2a, 0xad }, { 0x00, 0xe8, 0xd0, 0x38, 0xbd, 0x55, 0x6d, 0x85, 0x67, 0x8f, 0xb7, 0x5f, 0xda, 0x32, 0x0a, 0xe2 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x88, 0x10, 0x98, 0x20, 0xa8, 0x30, 0xb8, 0x40, 0xc8, 0x50, 0xd8, 0x60, 0xe8, 0x70, 0xf8 }, { 0x00, 0xe8, 0xd0, 0x38, 0xbd, 0x55, 0x6d, 0x85, 0x67, 0x8f, 0xb7, 0x5f, 0xda, 0x32, 0x0a, 0xe2 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x89, 0x12, 0x9b, 0x24, 0xad, 0x36, 0xbf, 0x48, 0xc1, 0x5a, 0xd3, 0x6c, 0xe5, 0x7e, 0xf7 }, { 0x00, 0xe8, 0xd0, 0x38, 0xbd, 0x55, 0x6d, 0x85, 0x7a, 0x92, 0xaa, 0x42, 0xc7, 0x2f, 0x17, 0xff }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x8a, 0x14, 0x9e, 0x28, 0xa2, 0x3c, 0xb6, 0x50, 0xda, 0x44, 0xce, 0x78, 0xf2, 0x6c, 0xe6 }, { 0x00, 0xe8, 0xd0, 0x38, 0xbd, 0x55, 0x6d, 0x85, 0x7a, 0x92, 0xaa, 0x42, 0xc7, 0x2f, 0x17, 0xff }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x8b, 0x16, 0x9d, 0x2c, 0xa7, 0x3a, 0xb1, 0x58, 0xd3, 0x4e, 0xc5, 0x74, 0xff, 0x62, 0xe9 }, { 0x00, 0xe8, 0xd0, 0x38, 0xa0, 0x48, 0x70, 0x98, 0x5d, 0xb5, 0x8d, 0x65, 0xfd, 0x15, 0x2d, 0xc5 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x8c, 0x18, 0x94, 0x30, 0xbc, 0x28, 0xa4, 0x60, 0xec, 0x78, 0xf4, 0x50, 0xdc, 0x48, 0xc4 }, { 0x00, 0xe8, 0xd0, 0x38, 0xa0, 0x48, 0x70, 0x98, 0x5d, 0xb5, 0x8d, 0x65, 0xfd, 0x15, 0x2d, 0xc5 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x8d, 0x1a, 0x97, 0x34, 0xb9, 0x2e, 0xa3, 0x68, 0xe5, 0x72, 0xff, 0x5c, 0xd1, 0x46, 0xcb }, { 0x00, 0xe8, 0xd0, 0x38, 0xa0, 0x48, 0x70, 0x98, 0x40, 0xa8, 0x90, 0x78, 0xe0, 0x08, 0x30, 0xd8 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x8e, 0x1c, 0x92, 0x38, 0xb6, 0x24, 0xaa, 0x70, 0xfe, 0x6c, 0xe2, 0x48, 0xc6, 0x54, 0xda }, { 0x00, 0xe8, 0xd0, 0x38, 0xa0, 0x48, 0x70, 0x98, 0x40, 0xa8, 0x90, 0x78, 0xe0, 0x08, 0x30, 0xd8 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x8f, 0x1e, 0x91, 0x3c, 0xb3, 0x22, 0xad, 0x78, 0xf7, 0x66, 0xe9, 0x44, 0xcb, 0x5a, 0xd5 }, { 0x00, 0xf5, 0xf7, 0x02, 0xf3, 0x06, 0x04, 0xf1, 0xfb, 0x0e, 0x0c, 0xf9, 0x08, 0xfd, 0xff, 0x0a }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0xf5, 0xf7, 0x02, 0xf3, 0x06, 0x04, 0xf1, 0xfb, 0x0e, 0x0c, 0xf9, 0x08, 0xfd, 0xff, 0x0a }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x91, 0x22, 0xb3, 0x44, 0xd5, 0x66, 0xf7, 0x88, 0x19, 0xaa, 0x3b, 0xcc, 0x5d, 0xee, 0x7f }, { 0x00, 0xf5, 0xf7, 0x02, 0xf3, 0x06, 0x04, 0xf1, 0xe6, 0x13, 0x11, 0xe4, 0x15, 0xe0, 0xe2, 0x17 }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x92, 0x24, 0xb6, 0x48, 0xda, 0x6c, 0xfe, 0x90, 0x02, 0xb4, 0x26, 0xd8, 0x4a, 0xfc, 0x6e }, { 0x00, 0xf5, 0xf7, 0x02, 0xf3, 0x06, 0x04, 0xf1, 0xe6, 0x13, 0x11, 0xe4, 0x15, 0xe0, 0xe2, 0x17 }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x93, 0x26, 0xb5, 0x4c, 0xdf, 0x6a, 0xf9, 0x98, 0x0b, 0xbe, 0x2d, 0xd4, 0x47, 0xf2, 0x61 }, { 0x00, 0xf5, 0xf7, 0x02, 0xee, 0x1b, 0x19, 0xec, 0xc1, 0x34, 0x36, 0xc3, 0x2f, 0xda, 0xd8, 0x2d }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x94, 0x28, 0xbc, 0x50, 0xc4, 0x78, 0xec, 0xa0, 0x34, 0x88, 0x1c, 0xf0, 0x64, 0xd8, 0x4c }, { 0x00, 0xf5, 0xf7, 0x02, 0xee, 0x1b, 0x19, 0xec, 0xc1, 0x34, 0x36, 0xc3, 0x2f, 0xda, 0xd8, 0x2d }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x95, 0x2a, 0xbf, 0x54, 0xc1, 0x7e, 0xeb, 0xa8, 0x3d, 0x82, 0x17, 0xfc, 0x69, 0xd6, 0x43 }, { 0x00, 0xf5, 0xf7, 0x02, 0xee, 0x1b, 0x19, 0xec, 0xdc, 0x29, 0x2b, 0xde, 0x32, 0xc7, 0xc5, 0x30 }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x96, 0x2c, 0xba, 0x58, 0xce, 0x74, 0xe2, 0xb0, 0x26, 0x9c, 0x0a, 0xe8, 0x7e, 0xc4, 0x52 }, { 0x00, 0xf5, 0xf7, 0x02, 0xee, 0x1b, 0x19, 0xec, 0xdc, 0x29, 0x2b, 0xde, 0x32, 0xc7, 0xc5, 0x30 }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x97, 0x2e, 0xb9, 0x5c, 0xcb, 0x72, 0xe5, 0xb8, 0x2f, 0x96, 0x01, 0xe4, 0x73, 0xca, 0x5d }, { 0x00, 0xf5, 0xea, 0x1f, 0xc9, 0x3c, 0x23, 0xd6, 0x8f, 0x7a, 0x65, 0x90, 0x46, 0xb3, 0xac, 0x59 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x98, 0x30, 0xa8, 0x60, 0xf8, 0x50, 0xc8, 0xc0, 0x58, 0xf0, 0x68, 0xa0, 0x38, 0x90, 0x08 }, { 0x00, 0xf5, 0xea, 0x1f, 0xc9, 0x3c, 0x23, 0xd6, 0x8f, 0x7a, 0x65, 0x90, 0x46, 0xb3, 0xac, 0x59 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x99, 0x32, 0xab, 0x64, 0xfd, 0x56, 0xcf, 0xc8, 0x51, 0xfa, 0x63, 0xac, 0x35, 0x9e, 0x07 }, { 0x00, 0xf5, 0xea, 0x1f, 0xc9, 0x3c, 0x23, 0xd6, 0x92, 0x67, 0x78, 0x8d, 0x5b, 0xae, 0xb1, 0x44 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x9a, 0x34, 0xae, 0x68, 0xf2, 0x5c, 0xc6, 0xd0, 0x4a, 0xe4, 0x7e, 0xb8, 0x22, 0x8c, 0x16 }, { 0x00, 0xf5, 0xea, 0x1f, 0xc9, 0x3c, 0x23, 0xd6, 0x92, 0x67, 0x78, 0x8d, 0x5b, 0xae, 0xb1, 0x44 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x9b, 0x36, 0xad, 0x6c, 0xf7, 0x5a, 0xc1, 0xd8, 0x43, 0xee, 0x75, 0xb4, 0x2f, 0x82, 0x19 }, { 0x00, 0xf5, 0xea, 0x1f, 0xd4, 0x21, 0x3e, 0xcb, 0xb5, 0x40, 0x5f, 0xaa, 0x61, 0x94, 0x8b, 0x7e }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x9c, 0x38, 0xa4, 0x70, 0xec, 0x48, 0xd4, 0xe0, 0x7c, 0xd8, 0x44, 0x90, 0x0c, 0xa8, 0x34 }, { 0x00, 0xf5, 0xea, 0x1f, 0xd4, 0x21, 0x3e, 0xcb, 0xb5, 0x40, 0x5f, 0xaa, 0x61, 0x94, 0x8b, 0x7e }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x9d, 0x3a, 0xa7, 0x74, 0xe9, 0x4e, 0xd3, 0xe8, 0x75, 0xd2, 0x4f, 0x9c, 0x01, 0xa6, 0x3b }, { 0x00, 0xf5, 0xea, 0x1f, 0xd4, 0x21, 0x3e, 0xcb, 0xa8, 0x5d, 0x42, 0xb7, 0x7c, 0x89, 0x96, 0x63 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x9e, 0x3c, 0xa2, 0x78, 0xe6, 0x44, 0xda, 0xf0, 0x6e, 0xcc, 0x52, 0x88, 0x16, 0xb4, 0x2a }, { 0x00, 0xf5, 0xea, 0x1f, 0xd4, 0x21, 0x3e, 0xcb, 0xa8, 0x5d, 0x42, 0xb7, 0x7c, 0x89, 0x96, 0x63 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x74, 0x74, 0x69, 0x69, 0x4e, 0x4e, 0x53, 0x53 }, { 0x00, 0x9f, 0x3e, 0xa1, 0x7c, 0xe3, 0x42, 0xdd, 0xf8, 0x67, 0xc6, 0x59, 0x84, 0x1b, 0xba, 0x25 }, { 0x00, 0xd2, 0xb9, 0x6b, 0x6f, 0xbd, 0xd6, 0x04, 0xde, 0x0c, 0x67, 0xb5, 0xb1, 0x63, 0x08, 0xda }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0xd2, 0xb9, 0x6b, 0x6f, 0xbd, 0xd6, 0x04, 0xde, 0x0c, 0x67, 0xb5, 0xb1, 0x63, 0x08, 0xda }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa1, 0x42, 0xe3, 0x84, 0x25, 0xc6, 0x67, 0x08, 0xa9, 0x4a, 0xeb, 0x8c, 0x2d, 0xce, 0x6f }, { 0x00, 0xd2, 0xb9, 0x6b, 0x6f, 0xbd, 0xd6, 0x04, 0xc3, 0x11, 0x7a, 0xa8, 0xac, 0x7e, 0x15, 0xc7 }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa2, 0x44, 0xe6, 0x88, 0x2a, 0xcc, 0x6e, 0x10, 0xb2, 0x54, 0xf6, 0x98, 0x3a, 0xdc, 0x7e }, { 0x00, 0xd2, 0xb9, 0x6b, 0x6f, 0xbd, 0xd6, 0x04, 0xc3, 0x11, 0x7a, 0xa8, 0xac, 0x7e, 0x15, 0xc7 }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa3, 0x46, 0xe5, 0x8c, 0x2f, 0xca, 0x69, 0x18, 0xbb, 0x5e, 0xfd, 0x94, 0x37, 0xd2, 0x71 }, { 0x00, 0xd2, 0xb9, 0x6b, 0x72, 0xa0, 0xcb, 0x19, 0xe4, 0x36, 0x5d, 0x8f, 0x96, 0x44, 0x2f, 0xfd }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa4, 0x48, 0xec, 0x90, 0x34, 0xd8, 0x7c, 0x20, 0x84, 0x68, 0xcc, 0xb0, 0x14, 0xf8, 0x5c }, { 0x00, 0xd2, 0xb9, 0x6b, 0x72, 0xa0, 0xcb, 0x19, 0xe4, 0x36, 0x5d, 0x8f, 0x96, 0x44, 0x2f, 0xfd }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa5, 0x4a, 0xef, 0x94, 0x31, 0xde, 0x7b, 0x28, 0x8d, 0x62, 0xc7, 0xbc, 0x19, 0xf6, 0x53 }, { 0x00, 0xd2, 0xb9, 0x6b, 0x72, 0xa0, 0xcb, 0x19, 0xf9, 0x2b, 0x40, 0x92, 0x8b, 0x59, 0x32, 0xe0 }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa6, 0x4c, 0xea, 0x98, 0x3e, 0xd4, 0x72, 0x30, 0x96, 0x7c, 0xda, 0xa8, 0x0e, 0xe4, 0x42 }, { 0x00, 0xd2, 0xb9, 0x6b, 0x72, 0xa0, 0xcb, 0x19, 0xf9, 0x2b, 0x40, 0x92, 0x8b, 0x59, 0x32, 0xe0 }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa7, 0x4e, 0xe9, 0x9c, 0x3b, 0xd2, 0x75, 0x38, 0x9f, 0x76, 0xd1, 0xa4, 0x03, 0xea, 0x4d }, { 0x00, 0xd2, 0xa4, 0x76, 0x55, 0x87, 0xf1, 0x23, 0xaa, 0x78, 0x0e, 0xdc, 0xff, 0x2d, 0x5b, 0x89 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa8, 0x50, 0xf8, 0xa0, 0x08, 0xf0, 0x58, 0x40, 0xe8, 0x10, 0xb8, 0xe0, 0x48, 0xb0, 0x18 }, { 0x00, 0xd2, 0xa4, 0x76, 0x55, 0x87, 0xf1, 0x23, 0xaa, 0x78, 0x0e, 0xdc, 0xff, 0x2d, 0x5b, 0x89 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xa9, 0x52, 0xfb, 0xa4, 0x0d, 0xf6, 0x5f, 0x48, 0xe1, 0x1a, 0xb3, 0xec, 0x45, 0xbe, 0x17 }, { 0x00, 0xd2, 0xa4, 0x76, 0x55, 0x87, 0xf1, 0x23, 0xb7, 0x65, 0x13, 0xc1, 0xe2, 0x30, 0x46, 0x94 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xaa, 0x54, 0xfe, 0xa8, 0x02, 0xfc, 0x56, 0x50, 0xfa, 0x04, 0xae, 0xf8, 0x52, 0xac, 0x06 }, { 0x00, 0xd2, 0xa4, 0x76, 0x55, 0x87, 0xf1, 0x23, 0xb7, 0x65, 0x13, 0xc1, 0xe2, 0x30, 0x46, 0x94 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xab, 0x56, 0xfd, 0xac, 0x07, 0xfa, 0x51, 0x58, 0xf3, 0x0e, 0xa5, 0xf4, 0x5f, 0xa2, 0x09 }, { 0x00, 0xd2, 0xa4, 0x76, 0x48, 0x9a, 0xec, 0x3e, 0x90, 0x42, 0x34, 0xe6, 0xd8, 0x0a, 0x7c, 0xae }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xac, 0x58, 0xf4, 0xb0, 0x1c, 0xe8, 0x44, 0x60, 0xcc, 0x38, 0x94, 0xd0, 0x7c, 0x88, 0x24 }, { 0x00, 0xd2, 0xa4, 0x76, 0x48, 0x9a, 0xec, 0x3e, 0x90, 0x42, 0x34, 0xe6, 0xd8, 0x0a, 0x7c, 0xae }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xad, 0x5a, 0xf7, 0xb4, 0x19, 0xee, 0x43, 0x68, 0xc5, 0x32, 0x9f, 0xdc, 0x71, 0x86, 0x2b }, { 0x00, 0xd2, 0xa4, 0x76, 0x48, 0x9a, 0xec, 0x3e, 0x8d, 0x5f, 0x29, 0xfb, 0xc5, 0x17, 0x61, 0xb3 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xae, 0x5c, 0xf2, 0xb8, 0x16, 0xe4, 0x4a, 0x70, 0xde, 0x2c, 0x82, 0xc8, 0x66, 0x94, 0x3a }, { 0x00, 0xd2, 0xa4, 0x76, 0x48, 0x9a, 0xec, 0x3e, 0x8d, 0x5f, 0x29, 0xfb, 0xc5, 0x17, 0x61, 0xb3 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xaf, 0x5e, 0xf1, 0xbc, 0x13, 0xe2, 0x4d, 0x78, 0xd7, 0x26, 0x89, 0xc4, 0x6b, 0x9a, 0x35 }, { 0x00, 0xcf, 0x83, 0x4c, 0x1b, 0xd4, 0x98, 0x57, 0x36, 0xf9, 0xb5, 0x7a, 0x2d, 0xe2, 0xae, 0x61 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0xcf, 0x83, 0x4c, 0x1b, 0xd4, 0x98, 0x57, 0x36, 0xf9, 0xb5, 0x7a, 0x2d, 0xe2, 0xae, 0x61 }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb1, 0x62, 0xd3, 0xc4, 0x75, 0xa6, 0x17, 0x88, 0x39, 0xea, 0x5b, 0x4c, 0xfd, 0x2e, 0x9f }, { 0x00, 0xcf, 0x83, 0x4c, 0x1b, 0xd4, 0x98, 0x57, 0x2b, 0xe4, 0xa8, 0x67, 0x30, 0xff, 0xb3, 0x7c }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb2, 0x64, 0xd6, 0xc8, 0x7a, 0xac, 0x1e, 0x90, 0x22, 0xf4, 0x46, 0x58, 0xea, 0x3c, 0x8e }, { 0x00, 0xcf, 0x83, 0x4c, 0x1b, 0xd4, 0x98, 0x57, 0x2b, 0xe4, 0xa8, 0x67, 0x30, 0xff, 0xb3, 0x7c }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb3, 0x66, 0xd5, 0xcc, 0x7f, 0xaa, 0x19, 0x98, 0x2b, 0xfe, 0x4d, 0x54, 0xe7, 0x32, 0x81 }, { 0x00, 0xcf, 0x83, 0x4c, 0x06, 0xc9, 0x85, 0x4a, 0x0c, 0xc3, 0x8f, 0x40, 0x0a, 0xc5, 0x89, 0x46 }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb4, 0x68, 0xdc, 0xd0, 0x64, 0xb8, 0x0c, 0xa0, 0x14, 0xc8, 0x7c, 0x70, 0xc4, 0x18, 0xac }, { 0x00, 0xcf, 0x83, 0x4c, 0x06, 0xc9, 0x85, 0x4a, 0x0c, 0xc3, 0x8f, 0x40, 0x0a, 0xc5, 0x89, 0x46 }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb5, 0x6a, 0xdf, 0xd4, 0x61, 0xbe, 0x0b, 0xa8, 0x1d, 0xc2, 0x77, 0x7c, 0xc9, 0x16, 0xa3 }, { 0x00, 0xcf, 0x83, 0x4c, 0x06, 0xc9, 0x85, 0x4a, 0x11, 0xde, 0x92, 0x5d, 0x17, 0xd8, 0x94, 0x5b }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb6, 0x6c, 0xda, 0xd8, 0x6e, 0xb4, 0x02, 0xb0, 0x06, 0xdc, 0x6a, 0x68, 0xde, 0x04, 0xb2 }, { 0x00, 0xcf, 0x83, 0x4c, 0x06, 0xc9, 0x85, 0x4a, 0x11, 0xde, 0x92, 0x5d, 0x17, 0xd8, 0x94, 0x5b }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb7, 0x6e, 0xd9, 0xdc, 0x6b, 0xb2, 0x05, 0xb8, 0x0f, 0xd6, 0x61, 0x64, 0xd3, 0x0a, 0xbd }, { 0x00, 0xcf, 0x9e, 0x51, 0x21, 0xee, 0xbf, 0x70, 0x42, 0x8d, 0xdc, 0x13, 0x63, 0xac, 0xfd, 0x32 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb8, 0x70, 0xc8, 0xe0, 0x58, 0x90, 0x28, 0xc0, 0x78, 0xb0, 0x08, 0x20, 0x98, 0x50, 0xe8 }, { 0x00, 0xcf, 0x9e, 0x51, 0x21, 0xee, 0xbf, 0x70, 0x42, 0x8d, 0xdc, 0x13, 0x63, 0xac, 0xfd, 0x32 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xb9, 0x72, 0xcb, 0xe4, 0x5d, 0x96, 0x2f, 0xc8, 0x71, 0xba, 0x03, 0x2c, 0x95, 0x5e, 0xe7 }, { 0x00, 0xcf, 0x9e, 0x51, 0x21, 0xee, 0xbf, 0x70, 0x5f, 0x90, 0xc1, 0x0e, 0x7e, 0xb1, 0xe0, 0x2f }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xba, 0x74, 0xce, 0xe8, 0x52, 0x9c, 0x26, 0xd0, 0x6a, 0xa4, 0x1e, 0x38, 0x82, 0x4c, 0xf6 }, { 0x00, 0xcf, 0x9e, 0x51, 0x21, 0xee, 0xbf, 0x70, 0x5f, 0x90, 0xc1, 0x0e, 0x7e, 0xb1, 0xe0, 0x2f }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xbb, 0x76, 0xcd, 0xec, 0x57, 0x9a, 0x21, 0xd8, 0x63, 0xae, 0x15, 0x34, 0x8f, 0x42, 0xf9 }, { 0x00, 0xcf, 0x9e, 0x51, 0x3c, 0xf3, 0xa2, 0x6d, 0x78, 0xb7, 0xe6, 0x29, 0x44, 0x8b, 0xda, 0x15 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xbc, 0x78, 0xc4, 0xf0, 0x4c, 0x88, 0x34, 0xe0, 0x5c, 0x98, 0x24, 0x10, 0xac, 0x68, 0xd4 }, { 0x00, 0xcf, 0x9e, 0x51, 0x3c, 0xf3, 0xa2, 0x6d, 0x78, 0xb7, 0xe6, 0x29, 0x44, 0x8b, 0xda, 0x15 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xbd, 0x7a, 0xc7, 0xf4, 0x49, 0x8e, 0x33, 0xe8, 0x55, 0x92, 0x2f, 0x1c, 0xa1, 0x66, 0xdb }, { 0x00, 0xcf, 0x9e, 0x51, 0x3c, 0xf3, 0xa2, 0x6d, 0x65, 0xaa, 0xfb, 0x34, 0x59, 0x96, 0xc7, 0x08 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xbe, 0x7c, 0xc2, 0xf8, 0x46, 0x84, 0x3a, 0xf0, 0x4e, 0x8c, 0x32, 0x08, 0xb6, 0x74, 0xca }, { 0x00, 0xcf, 0x9e, 0x51, 0x3c, 0xf3, 0xa2, 0x6d, 0x65, 0xaa, 0xfb, 0x34, 0x59, 0x96, 0xc7, 0x08 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x1d, 0x1d, 0x3a, 0x3a, 0x27, 0x27, 0x69, 0x69, 0x74, 0x74, 0x53, 0x53, 0x4e, 0x4e }, { 0x00, 0xbf, 0x7e, 0xc1, 0xfc, 0x43, 0x82, 0x3d, 0xf8, 0x47, 0x86, 0x39, 0x04, 0xbb, 0x7a, 0xc5 }, { 0x00, 0x9c, 0x25, 0xb9, 0x4a, 0xd6, 0x6f, 0xf3, 0x94, 0x08, 0xb1, 0x2d, 0xde, 0x42, 0xfb, 0x67 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x9c, 0x25, 0xb9, 0x4a, 0xd6, 0x6f, 0xf3, 0x94, 0x08, 0xb1, 0x2d, 0xde, 0x42, 0xfb, 0x67 }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc1, 0x82, 0x43, 0x04, 0xc5, 0x86, 0x47, 0x08, 0xc9, 0x8a, 0x4b, 0x0c, 0xcd, 0x8e, 0x4f }, { 0x00, 0x9c, 0x25, 0xb9, 0x4a, 0xd6, 0x6f, 0xf3, 0x89, 0x15, 0xac, 0x30, 0xc3, 0x5f, 0xe6, 0x7a }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc2, 0x84, 0x46, 0x08, 0xca, 0x8c, 0x4e, 0x10, 0xd2, 0x94, 0x56, 0x18, 0xda, 0x9c, 0x5e }, { 0x00, 0x9c, 0x25, 0xb9, 0x4a, 0xd6, 0x6f, 0xf3, 0x89, 0x15, 0xac, 0x30, 0xc3, 0x5f, 0xe6, 0x7a }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc3, 0x86, 0x45, 0x0c, 0xcf, 0x8a, 0x49, 0x18, 0xdb, 0x9e, 0x5d, 0x14, 0xd7, 0x92, 0x51 }, { 0x00, 0x9c, 0x25, 0xb9, 0x57, 0xcb, 0x72, 0xee, 0xae, 0x32, 0x8b, 0x17, 0xf9, 0x65, 0xdc, 0x40 }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc4, 0x88, 0x4c, 0x10, 0xd4, 0x98, 0x5c, 0x20, 0xe4, 0xa8, 0x6c, 0x30, 0xf4, 0xb8, 0x7c }, { 0x00, 0x9c, 0x25, 0xb9, 0x57, 0xcb, 0x72, 0xee, 0xae, 0x32, 0x8b, 0x17, 0xf9, 0x65, 0xdc, 0x40 }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc5, 0x8a, 0x4f, 0x14, 0xd1, 0x9e, 0x5b, 0x28, 0xed, 0xa2, 0x67, 0x3c, 0xf9, 0xb6, 0x73 }, { 0x00, 0x9c, 0x25, 0xb9, 0x57, 0xcb, 0x72, 0xee, 0xb3, 0x2f, 0x96, 0x0a, 0xe4, 0x78, 0xc1, 0x5d }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc6, 0x8c, 0x4a, 0x18, 0xde, 0x94, 0x52, 0x30, 0xf6, 0xbc, 0x7a, 0x28, 0xee, 0xa4, 0x62 }, { 0x00, 0x9c, 0x25, 0xb9, 0x57, 0xcb, 0x72, 0xee, 0xb3, 0x2f, 0x96, 0x0a, 0xe4, 0x78, 0xc1, 0x5d }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc7, 0x8e, 0x49, 0x1c, 0xdb, 0x92, 0x55, 0x38, 0xff, 0xb6, 0x71, 0x24, 0xe3, 0xaa, 0x6d }, { 0x00, 0x9c, 0x38, 0xa4, 0x70, 0xec, 0x48, 0xd4, 0xe0, 0x7c, 0xd8, 0x44, 0x90, 0x0c, 0xa8, 0x34 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc8, 0x90, 0x58, 0x20, 0xe8, 0xb0, 0x78, 0x40, 0x88, 0xd0, 0x18, 0x60, 0xa8, 0xf0, 0x38 }, { 0x00, 0x9c, 0x38, 0xa4, 0x70, 0xec, 0x48, 0xd4, 0xe0, 0x7c, 0xd8, 0x44, 0x90, 0x0c, 0xa8, 0x34 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xc9, 0x92, 0x5b, 0x24, 0xed, 0xb6, 0x7f, 0x48, 0x81, 0xda, 0x13, 0x6c, 0xa5, 0xfe, 0x37 }, { 0x00, 0x9c, 0x38, 0xa4, 0x70, 0xec, 0x48, 0xd4, 0xfd, 0x61, 0xc5, 0x59, 0x8d, 0x11, 0xb5, 0x29 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xca, 0x94, 0x5e, 0x28, 0xe2, 0xbc, 0x76, 0x50, 0x9a, 0xc4, 0x0e, 0x78, 0xb2, 0xec, 0x26 }, { 0x00, 0x9c, 0x38, 0xa4, 0x70, 0xec, 0x48, 0xd4, 0xfd, 0x61, 0xc5, 0x59, 0x8d, 0x11, 0xb5, 0x29 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xcb, 0x96, 0x5d, 0x2c, 0xe7, 0xba, 0x71, 0x58, 0x93, 0xce, 0x05, 0x74, 0xbf, 0xe2, 0x29 }, { 0x00, 0x9c, 0x38, 0xa4, 0x6d, 0xf1, 0x55, 0xc9, 0xda, 0x46, 0xe2, 0x7e, 0xb7, 0x2b, 0x8f, 0x13 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xcc, 0x98, 0x54, 0x30, 0xfc, 0xa8, 0x64, 0x60, 0xac, 0xf8, 0x34, 0x50, 0x9c, 0xc8, 0x04 }, { 0x00, 0x9c, 0x38, 0xa4, 0x6d, 0xf1, 0x55, 0xc9, 0xda, 0x46, 0xe2, 0x7e, 0xb7, 0x2b, 0x8f, 0x13 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xcd, 0x9a, 0x57, 0x34, 0xf9, 0xae, 0x63, 0x68, 0xa5, 0xf2, 0x3f, 0x5c, 0x91, 0xc6, 0x0b }, { 0x00, 0x9c, 0x38, 0xa4, 0x6d, 0xf1, 0x55, 0xc9, 0xc7, 0x5b, 0xff, 0x63, 0xaa, 0x36, 0x92, 0x0e }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xce, 0x9c, 0x52, 0x38, 0xf6, 0xa4, 0x6a, 0x70, 0xbe, 0xec, 0x22, 0x48, 0x86, 0xd4, 0x1a }, { 0x00, 0x9c, 0x38, 0xa4, 0x6d, 0xf1, 0x55, 0xc9, 0xc7, 0x5b, 0xff, 0x63, 0xaa, 0x36, 0x92, 0x0e }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xcf, 0x9e, 0x51, 0x3c, 0xf3, 0xa2, 0x6d, 0x78, 0xb7, 0xe6, 0x29, 0x44, 0x8b, 0xda, 0x15 }, { 0x00, 0x81, 0x1f, 0x9e, 0x3e, 0xbf, 0x21, 0xa0, 0x7c, 0xfd, 0x63, 0xe2, 0x42, 0xc3, 0x5d, 0xdc }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x81, 0x1f, 0x9e, 0x3e, 0xbf, 0x21, 0xa0, 0x7c, 0xfd, 0x63, 0xe2, 0x42, 0xc3, 0x5d, 0xdc }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd1, 0xa2, 0x73, 0x44, 0x95, 0xe6, 0x37, 0x88, 0x59, 0x2a, 0xfb, 0xcc, 0x1d, 0x6e, 0xbf }, { 0x00, 0x81, 0x1f, 0x9e, 0x3e, 0xbf, 0x21, 0xa0, 0x61, 0xe0, 0x7e, 0xff, 0x5f, 0xde, 0x40, 0xc1 }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd2, 0xa4, 0x76, 0x48, 0x9a, 0xec, 0x3e, 0x90, 0x42, 0x34, 0xe6, 0xd8, 0x0a, 0x7c, 0xae }, { 0x00, 0x81, 0x1f, 0x9e, 0x3e, 0xbf, 0x21, 0xa0, 0x61, 0xe0, 0x7e, 0xff, 0x5f, 0xde, 0x40, 0xc1 }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd3, 0xa6, 0x75, 0x4c, 0x9f, 0xea, 0x39, 0x98, 0x4b, 0x3e, 0xed, 0xd4, 0x07, 0x72, 0xa1 }, { 0x00, 0x81, 0x1f, 0x9e, 0x23, 0xa2, 0x3c, 0xbd, 0x46, 0xc7, 0x59, 0xd8, 0x65, 0xe4, 0x7a, 0xfb }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd4, 0xa8, 0x7c, 0x50, 0x84, 0xf8, 0x2c, 0xa0, 0x74, 0x08, 0xdc, 0xf0, 0x24, 0x58, 0x8c }, { 0x00, 0x81, 0x1f, 0x9e, 0x23, 0xa2, 0x3c, 0xbd, 0x46, 0xc7, 0x59, 0xd8, 0x65, 0xe4, 0x7a, 0xfb }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd5, 0xaa, 0x7f, 0x54, 0x81, 0xfe, 0x2b, 0xa8, 0x7d, 0x02, 0xd7, 0xfc, 0x29, 0x56, 0x83 }, { 0x00, 0x81, 0x1f, 0x9e, 0x23, 0xa2, 0x3c, 0xbd, 0x5b, 0xda, 0x44, 0xc5, 0x78, 0xf9, 0x67, 0xe6 }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd6, 0xac, 0x7a, 0x58, 0x8e, 0xf4, 0x22, 0xb0, 0x66, 0x1c, 0xca, 0xe8, 0x3e, 0x44, 0x92 }, { 0x00, 0x81, 0x1f, 0x9e, 0x23, 0xa2, 0x3c, 0xbd, 0x5b, 0xda, 0x44, 0xc5, 0x78, 0xf9, 0x67, 0xe6 }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd7, 0xae, 0x79, 0x5c, 0x8b, 0xf2, 0x25, 0xb8, 0x6f, 0x16, 0xc1, 0xe4, 0x33, 0x4a, 0x9d }, { 0x00, 0x81, 0x02, 0x83, 0x04, 0x85, 0x06, 0x87, 0x08, 0x89, 0x0a, 0x8b, 0x0c, 0x8d, 0x0e, 0x8f }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd8, 0xb0, 0x68, 0x60, 0xb8, 0xd0, 0x08, 0xc0, 0x18, 0x70, 0xa8, 0xa0, 0x78, 0x10, 0xc8 }, { 0x00, 0x81, 0x02, 0x83, 0x04, 0x85, 0x06, 0x87, 0x08, 0x89, 0x0a, 0x8b, 0x0c, 0x8d, 0x0e, 0x8f }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xd9, 0xb2, 0x6b, 0x64, 0xbd, 0xd6, 0x0f, 0xc8, 0x11, 0x7a, 0xa3, 0xac, 0x75, 0x1e, 0xc7 }, { 0x00, 0x81, 0x02, 0x83, 0x04, 0x85, 0x06, 0x87, 0x15, 0x94, 0x17, 0x96, 0x11, 0x90, 0x13, 0x92 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xda, 0xb4, 0x6e, 0x68, 0xb2, 0xdc, 0x06, 0xd0, 0x0a, 0x64, 0xbe, 0xb8, 0x62, 0x0c, 0xd6 }, { 0x00, 0x81, 0x02, 0x83, 0x04, 0x85, 0x06, 0x87, 0x15, 0x94, 0x17, 0x96, 0x11, 0x90, 0x13, 0x92 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xdb, 0xb6, 0x6d, 0x6c, 0xb7, 0xda, 0x01, 0xd8, 0x03, 0x6e, 0xb5, 0xb4, 0x6f, 0x02, 0xd9 }, { 0x00, 0x81, 0x02, 0x83, 0x19, 0x98, 0x1b, 0x9a, 0x32, 0xb3, 0x30, 0xb1, 0x2b, 0xaa, 0x29, 0xa8 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xdc, 0xb8, 0x64, 0x70, 0xac, 0xc8, 0x14, 0xe0, 0x3c, 0x58, 0x84, 0x90, 0x4c, 0x28, 0xf4 }, { 0x00, 0x81, 0x02, 0x83, 0x19, 0x98, 0x1b, 0x9a, 0x32, 0xb3, 0x30, 0xb1, 0x2b, 0xaa, 0x29, 0xa8 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xdd, 0xba, 0x67, 0x74, 0xa9, 0xce, 0x13, 0xe8, 0x35, 0x52, 0x8f, 0x9c, 0x41, 0x26, 0xfb }, { 0x00, 0x81, 0x02, 0x83, 0x19, 0x98, 0x1b, 0x9a, 0x2f, 0xae, 0x2d, 0xac, 0x36, 0xb7, 0x34, 0xb5 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xde, 0xbc, 0x62, 0x78, 0xa6, 0xc4, 0x1a, 0xf0, 0x2e, 0x4c, 0x92, 0x88, 0x56, 0x34, 0xea }, { 0x00, 0x81, 0x02, 0x83, 0x19, 0x98, 0x1b, 0x9a, 0x2f, 0xae, 0x2d, 0xac, 0x36, 0xb7, 0x34, 0xb5 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x4e, 0x4e, 0x53, 0x53, 0x69, 0x69, 0x74, 0x74 }, { 0x00, 0xdf, 0xbe, 0x61, 0x7c, 0xa3, 0xc2, 0x1d, 0xf8, 0x27, 0x46, 0x99, 0x84, 0x5b, 0x3a, 0xe5 }, { 0x00, 0xa6, 0x51, 0xf7, 0xa2, 0x04, 0xf3, 0x55, 0x59, 0xff, 0x08, 0xae, 0xfb, 0x5d, 0xaa, 0x0c }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0xa6, 0x51, 0xf7, 0xa2, 0x04, 0xf3, 0x55, 0x59, 0xff, 0x08, 0xae, 0xfb, 0x5d, 0xaa, 0x0c }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe1, 0xc2, 0x23, 0x84, 0x65, 0x46, 0xa7, 0x08, 0xe9, 0xca, 0x2b, 0x8c, 0x6d, 0x4e, 0xaf }, { 0x00, 0xa6, 0x51, 0xf7, 0xa2, 0x04, 0xf3, 0x55, 0x44, 0xe2, 0x15, 0xb3, 0xe6, 0x40, 0xb7, 0x11 }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe2, 0xc4, 0x26, 0x88, 0x6a, 0x4c, 0xae, 0x10, 0xf2, 0xd4, 0x36, 0x98, 0x7a, 0x5c, 0xbe }, { 0x00, 0xa6, 0x51, 0xf7, 0xa2, 0x04, 0xf3, 0x55, 0x44, 0xe2, 0x15, 0xb3, 0xe6, 0x40, 0xb7, 0x11 }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe3, 0xc6, 0x25, 0x8c, 0x6f, 0x4a, 0xa9, 0x18, 0xfb, 0xde, 0x3d, 0x94, 0x77, 0x52, 0xb1 }, { 0x00, 0xa6, 0x51, 0xf7, 0xbf, 0x19, 0xee, 0x48, 0x63, 0xc5, 0x32, 0x94, 0xdc, 0x7a, 0x8d, 0x2b }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe4, 0xc8, 0x2c, 0x90, 0x74, 0x58, 0xbc, 0x20, 0xc4, 0xe8, 0x0c, 0xb0, 0x54, 0x78, 0x9c }, { 0x00, 0xa6, 0x51, 0xf7, 0xbf, 0x19, 0xee, 0x48, 0x63, 0xc5, 0x32, 0x94, 0xdc, 0x7a, 0x8d, 0x2b }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe5, 0xca, 0x2f, 0x94, 0x71, 0x5e, 0xbb, 0x28, 0xcd, 0xe2, 0x07, 0xbc, 0x59, 0x76, 0x93 }, { 0x00, 0xa6, 0x51, 0xf7, 0xbf, 0x19, 0xee, 0x48, 0x7e, 0xd8, 0x2f, 0x89, 0xc1, 0x67, 0x90, 0x36 }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe6, 0xcc, 0x2a, 0x98, 0x7e, 0x54, 0xb2, 0x30, 0xd6, 0xfc, 0x1a, 0xa8, 0x4e, 0x64, 0x82 }, { 0x00, 0xa6, 0x51, 0xf7, 0xbf, 0x19, 0xee, 0x48, 0x7e, 0xd8, 0x2f, 0x89, 0xc1, 0x67, 0x90, 0x36 }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe7, 0xce, 0x29, 0x9c, 0x7b, 0x52, 0xb5, 0x38, 0xdf, 0xf6, 0x11, 0xa4, 0x43, 0x6a, 0x8d }, { 0x00, 0xa6, 0x4c, 0xea, 0x98, 0x3e, 0xd4, 0x72, 0x2d, 0x8b, 0x61, 0xc7, 0xb5, 0x13, 0xf9, 0x5f }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe8, 0xd0, 0x38, 0xa0, 0x48, 0x70, 0x98, 0x40, 0xa8, 0x90, 0x78, 0xe0, 0x08, 0x30, 0xd8 }, { 0x00, 0xa6, 0x4c, 0xea, 0x98, 0x3e, 0xd4, 0x72, 0x2d, 0x8b, 0x61, 0xc7, 0xb5, 0x13, 0xf9, 0x5f }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xe9, 0xd2, 0x3b, 0xa4, 0x4d, 0x76, 0x9f, 0x48, 0xa1, 0x9a, 0x73, 0xec, 0x05, 0x3e, 0xd7 }, { 0x00, 0xa6, 0x4c, 0xea, 0x98, 0x3e, 0xd4, 0x72, 0x30, 0x96, 0x7c, 0xda, 0xa8, 0x0e, 0xe4, 0x42 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xea, 0xd4, 0x3e, 0xa8, 0x42, 0x7c, 0x96, 0x50, 0xba, 0x84, 0x6e, 0xf8, 0x12, 0x2c, 0xc6 }, { 0x00, 0xa6, 0x4c, 0xea, 0x98, 0x3e, 0xd4, 0x72, 0x30, 0x96, 0x7c, 0xda, 0xa8, 0x0e, 0xe4, 0x42 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xeb, 0xd6, 0x3d, 0xac, 0x47, 0x7a, 0x91, 0x58, 0xb3, 0x8e, 0x65, 0xf4, 0x1f, 0x22, 0xc9 }, { 0x00, 0xa6, 0x4c, 0xea, 0x85, 0x23, 0xc9, 0x6f, 0x17, 0xb1, 0x5b, 0xfd, 0x92, 0x34, 0xde, 0x78 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xec, 0xd8, 0x34, 0xb0, 0x5c, 0x68, 0x84, 0x60, 0x8c, 0xb8, 0x54, 0xd0, 0x3c, 0x08, 0xe4 }, { 0x00, 0xa6, 0x4c, 0xea, 0x85, 0x23, 0xc9, 0x6f, 0x17, 0xb1, 0x5b, 0xfd, 0x92, 0x34, 0xde, 0x78 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xed, 0xda, 0x37, 0xb4, 0x59, 0x6e, 0x83, 0x68, 0x85, 0xb2, 0x5f, 0xdc, 0x31, 0x06, 0xeb }, { 0x00, 0xa6, 0x4c, 0xea, 0x85, 0x23, 0xc9, 0x6f, 0x0a, 0xac, 0x46, 0xe0, 0x8f, 0x29, 0xc3, 0x65 }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xee, 0xdc, 0x32, 0xb8, 0x56, 0x64, 0x8a, 0x70, 0x9e, 0xac, 0x42, 0xc8, 0x26, 0x14, 0xfa }, { 0x00, 0xa6, 0x4c, 0xea, 0x85, 0x23, 0xc9, 0x6f, 0x0a, 0xac, 0x46, 0xe0, 0x8f, 0x29, 0xc3, 0x65 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xef, 0xde, 0x31, 0xbc, 0x53, 0x62, 0x8d, 0x78, 0x97, 0xa6, 0x49, 0xc4, 0x2b, 0x1a, 0xf5 }, { 0x00, 0xbb, 0x6b, 0xd0, 0xd6, 0x6d, 0xbd, 0x06, 0xb1, 0x0a, 0xda, 0x61, 0x67, 0xdc, 0x0c, 0xb7 }, { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0xbb, 0x6b, 0xd0, 0xd6, 0x6d, 0xbd, 0x06, 0xb1, 0x0a, 0xda, 0x61, 0x67, 0xdc, 0x0c, 0xb7 }, { 0x00, 0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80, 0x90, 0xa0, 0xb0, 0xc0, 0xd0, 0xe0, 0xf0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf1, 0xe2, 0x13, 0xc4, 0x35, 0x26, 0xd7, 0x88, 0x79, 0x6a, 0x9b, 0x4c, 0xbd, 0xae, 0x5f }, { 0x00, 0xbb, 0x6b, 0xd0, 0xd6, 0x6d, 0xbd, 0x06, 0xac, 0x17, 0xc7, 0x7c, 0x7a, 0xc1, 0x11, 0xaa }, { 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0, 0x00, 0x20, 0x40, 0x60, 0x80, 0xa0, 0xc0, 0xe0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf2, 0xe4, 0x16, 0xc8, 0x3a, 0x2c, 0xde, 0x90, 0x62, 0x74, 0x86, 0x58, 0xaa, 0xbc, 0x4e }, { 0x00, 0xbb, 0x6b, 0xd0, 0xd6, 0x6d, 0xbd, 0x06, 0xac, 0x17, 0xc7, 0x7c, 0x7a, 0xc1, 0x11, 0xaa }, { 0x00, 0x30, 0x60, 0x50, 0xc0, 0xf0, 0xa0, 0x90, 0x80, 0xb0, 0xe0, 0xd0, 0x40, 0x70, 0x20, 0x10 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf3, 0xe6, 0x15, 0xcc, 0x3f, 0x2a, 0xd9, 0x98, 0x6b, 0x7e, 0x8d, 0x54, 0xa7, 0xb2, 0x41 }, { 0x00, 0xbb, 0x6b, 0xd0, 0xcb, 0x70, 0xa0, 0x1b, 0x8b, 0x30, 0xe0, 0x5b, 0x40, 0xfb, 0x2b, 0x90 }, { 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0, 0x00, 0x40, 0x80, 0xc0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf4, 0xe8, 0x1c, 0xd0, 0x24, 0x38, 0xcc, 0xa0, 0x54, 0x48, 0xbc, 0x70, 0x84, 0x98, 0x6c }, { 0x00, 0xbb, 0x6b, 0xd0, 0xcb, 0x70, 0xa0, 0x1b, 0x8b, 0x30, 0xe0, 0x5b, 0x40, 0xfb, 0x2b, 0x90 }, { 0x00, 0x50, 0xa0, 0xf0, 0x40, 0x10, 0xe0, 0xb0, 0x80, 0xd0, 0x20, 0x70, 0xc0, 0x90, 0x60, 0x30 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf5, 0xea, 0x1f, 0xd4, 0x21, 0x3e, 0xcb, 0xa8, 0x5d, 0x42, 0xb7, 0x7c, 0x89, 0x96, 0x63 }, { 0x00, 0xbb, 0x6b, 0xd0, 0xcb, 0x70, 0xa0, 0x1b, 0x96, 0x2d, 0xfd, 0x46, 0x5d, 0xe6, 0x36, 0x8d }, { 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20, 0x00, 0x60, 0xc0, 0xa0, 0x80, 0xe0, 0x40, 0x20 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf6, 0xec, 0x1a, 0xd8, 0x2e, 0x34, 0xc2, 0xb0, 0x46, 0x5c, 0xaa, 0x68, 0x9e, 0x84, 0x72 }, { 0x00, 0xbb, 0x6b, 0xd0, 0xcb, 0x70, 0xa0, 0x1b, 0x96, 0x2d, 0xfd, 0x46, 0x5d, 0xe6, 0x36, 0x8d }, { 0x00, 0x70, 0xe0, 0x90, 0xc0, 0xb0, 0x20, 0x50, 0x80, 0xf0, 0x60, 0x10, 0x40, 0x30, 0xa0, 0xd0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf7, 0xee, 0x19, 0xdc, 0x2b, 0x32, 0xc5, 0xb8, 0x4f, 0x56, 0xa1, 0x64, 0x93, 0x8a, 0x7d }, { 0x00, 0xbb, 0x76, 0xcd, 0xec, 0x57, 0x9a, 0x21, 0xc5, 0x7e, 0xb3, 0x08, 0x29, 0x92, 0x5f, 0xe4 }, { 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80, 0x00, 0x80 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf8, 0xf0, 0x08, 0xe0, 0x18, 0x10, 0xe8, 0xc0, 0x38, 0x30, 0xc8, 0x20, 0xd8, 0xd0, 0x28 }, { 0x00, 0xbb, 0x76, 0xcd, 0xec, 0x57, 0x9a, 0x21, 0xc5, 0x7e, 0xb3, 0x08, 0x29, 0x92, 0x5f, 0xe4 }, { 0x00, 0x90, 0x20, 0xb0, 0x40, 0xd0, 0x60, 0xf0, 0x80, 0x10, 0xa0, 0x30, 0xc0, 0x50, 0xe0, 0x70 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xf9, 0xf2, 0x0b, 0xe4, 0x1d, 0x16, 0xef, 0xc8, 0x31, 0x3a, 0xc3, 0x2c, 0xd5, 0xde, 0x27 }, { 0x00, 0xbb, 0x76, 0xcd, 0xec, 0x57, 0x9a, 0x21, 0xd8, 0x63, 0xae, 0x15, 0x34, 0x8f, 0x42, 0xf9 }, { 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60, 0x00, 0xa0, 0x40, 0xe0, 0x80, 0x20, 0xc0, 0x60 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xfa, 0xf4, 0x0e, 0xe8, 0x12, 0x1c, 0xe6, 0xd0, 0x2a, 0x24, 0xde, 0x38, 0xc2, 0xcc, 0x36 }, { 0x00, 0xbb, 0x76, 0xcd, 0xec, 0x57, 0x9a, 0x21, 0xd8, 0x63, 0xae, 0x15, 0x34, 0x8f, 0x42, 0xf9 }, { 0x00, 0xb0, 0x60, 0xd0, 0xc0, 0x70, 0xa0, 0x10, 0x80, 0x30, 0xe0, 0x50, 0x40, 0xf0, 0x20, 0x90 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xfb, 0xf6, 0x0d, 0xec, 0x17, 0x1a, 0xe1, 0xd8, 0x23, 0x2e, 0xd5, 0x34, 0xcf, 0xc2, 0x39 }, { 0x00, 0xbb, 0x76, 0xcd, 0xf1, 0x4a, 0x87, 0x3c, 0xff, 0x44, 0x89, 0x32, 0x0e, 0xb5, 0x78, 0xc3 }, { 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40, 0x00, 0xc0, 0x80, 0x40 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xfc, 0xf8, 0x04, 0xf0, 0x0c, 0x08, 0xf4, 0xe0, 0x1c, 0x18, 0xe4, 0x10, 0xec, 0xe8, 0x14 }, { 0x00, 0xbb, 0x76, 0xcd, 0xf1, 0x4a, 0x87, 0x3c, 0xff, 0x44, 0x89, 0x32, 0x0e, 0xb5, 0x78, 0xc3 }, { 0x00, 0xd0, 0xa0, 0x70, 0x40, 0x90, 0xe0, 0x30, 0x80, 0x50, 0x20, 0xf0, 0xc0, 0x10, 0x60, 0xb0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xfd, 0xfa, 0x07, 0xf4, 0x09, 0x0e, 0xf3, 0xe8, 0x15, 0x12, 0xef, 0x1c, 0xe1, 0xe6, 0x1b }, { 0x00, 0xbb, 0x76, 0xcd, 0xf1, 0x4a, 0x87, 0x3c, 0xe2, 0x59, 0x94, 0x2f, 0x13, 0xa8, 0x65, 0xde }, { 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0, 0x00, 0xe0, 0xc0, 0x20, 0x80, 0x60, 0x40, 0xa0 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xfe, 0xfc, 0x02, 0xf8, 0x06, 0x04, 0xfa, 0xf0, 0x0e, 0x0c, 0xf2, 0x08, 0xf6, 0xf4, 0x0a }, { 0x00, 0xbb, 0x76, 0xcd, 0xf1, 0x4a, 0x87, 0x3c, 0xe2, 0x59, 0x94, 0x2f, 0x13, 0xa8, 0x65, 0xde }, { 0x00, 0xf0, 0xe0, 0x10, 0xc0, 0x30, 0x20, 0xd0, 0x80, 0x70, 0x60, 0x90, 0x40, 0xb0, 0xa0, 0x50 }, { 0x00, 0x00, 0x1d, 0x1d, 0x27, 0x27, 0x3a, 0x3a, 0x53, 0x53, 0x4e, 0x4e, 0x74, 0x74, 0x69, 0x69 }, { 0x00, 0xff, 0xfe, 0x01, 0xfc, 0x03, 0x02, 0xfd, 0xf8, 0x07, 0x06, 0xf9, 0x04, 0xfb, 0xfa, 0x05 } }; /* END CSTYLED */ #endif /* defined(HAVE_SSSE3) || defined(HAVE_AVX2) || defined(HAVE_AVX512BW) */ #endif /* defined(__x86_64) */