diff --git a/en_US.ISO8859-1/books/handbook/cutting-edge/chapter.xml b/en_US.ISO8859-1/books/handbook/cutting-edge/chapter.xml --- a/en_US.ISO8859-1/books/handbook/cutting-edge/chapter.xml +++ b/en_US.ISO8859-1/books/handbook/cutting-edge/chapter.xml @@ -1195,9 +1195,10 @@ &prompt.root; make -j4 buildworld &prompt.root; make -j4 kernel &prompt.root; shutdown -r now +&prompt.root; etcupdate -p &prompt.root; cd /usr/src &prompt.root; make installworld -&prompt.root; mergemaster -Ui +&prompt.root; etcupdate -B &prompt.root; shutdown -r now @@ -1214,6 +1215,12 @@ from source. + + Update and merge configuration files in + /etc/ required before + installworld. + + Go to the source directory. @@ -1241,7 +1248,7 @@ Install the world. - + Update and merge configuration files in /etc/. @@ -1536,13 +1543,84 @@ libraries are located and removed, then the system is restarted. - + + Merging Configuration Files with + &man.etcupdate.8; + + &man.etcupdate.8; is a tool for managing updates to + files that are not updated as part of an installworld such + as files in /etc/. It manages updates + by doing a three-way merge of changes made to these files + against the local versions. It is also designed to minimize + the amount of user intervention which is the largest benefit + in comparison to &man.mergemaster.8;. + + + In general, &man.etcupdate.8; does not need any + specific arguments for its job. There is however a handy in + between command for sanity checking what will be done the + first time &man.etcupdate.8; is used: + + &prompt.root; etcupdate diff + + By giving the output a quick look and making sure the diff + looks OK, the merge should work just fine. + + + If &man.etcupdate.8; is not able to merge a file + automatically, the merge conflicts can be resolved with + manual interaction by issuing: + + &prompt.root; etcupdate resolve + + + When switching from &man.mergemaster.8; to + &man.etcupdate.8; the first run might print some bogus + conflicts due to already applied changes. In general, the + following command sequence should recover from this + behavior: + + + + Bootstrapping &man.etcupdate.8; + + &prompt.root; etcupdate diff +&prompt.root; etcupdate bootstrap +&prompt.root; etcupdate diff + + + + Check the output before updating sources and + building the new world. If the output shows a lot + of diffs that are not expected, continue with the + bootstrap command. + + + + Bootstrap the program, for more information + see &man.etcupdate.8;. + + + + Check the diff again after bootstrapping. The + unexpected diffs should be gone. + + + + + + + + Merging Configuration Files with &man.mergemaster.8; - &man.mergemaster.8; provides an easy - way to merge changes that have been made to system - configuration files with new versions of those files. + &man.mergemaster.8; provides a way to merge changes that + have been made to system configuration files with new + versions of those files. &man.mergemaster.8; is an + alternative to the preferred &man.etcupdate.8; tool, + although &man.mergemaster.8; might still be needed on legacy + installs. With , &man.mergemaster.8; automatically updates files that have not been user-modified @@ -1699,3 +1777,4 @@ directory similar to DISTDIR. + diff --git a/en_US.ISO8859-1/books/handbook/zfs/chapter.xml b/en_US.ISO8859-1/books/handbook/zfs/chapter.xml --- a/en_US.ISO8859-1/books/handbook/zfs/chapter.xml +++ b/en_US.ISO8859-1/books/handbook/zfs/chapter.xml @@ -2928,6 +2928,72 @@ backups. + + Zstandard Compression + + In OpenZFS 2.0, a new compression + algorithm was added. Zstandard (Zstd) + offers higher compression ratios than the default + LZ4 while offering much greater speeds + than the alternative, gzip. + OpenZFS 2.0 is available starting with + &os; 12.1-RELEASE via + sysutils/openzfs and has been the + default in &os; 13-CURRENT since September 2020, and + will by in &os; 13.0-RELEASE. + + Zstd provides a large selection of + compression levels, providing fine-grained control over + performance versus compression ratio. One of the main + advantages of Zstd is that the + decompression speed is independent of the compression + level. For data that is written once but read many times, + Zstd allows the use of the highest + compression levels without a read performance + penalty. + + Even when data is updated frequently, there are often + performance gains that come from enabling compression. One + of the biggest advantages comes from the compressed ARC + feature. ZFS's Adaptive Replacement + Cache (ARC) caches the compressed version + of the data in RAM, decompressing it each + time it is needed. This allows the same amount of + RAM to store more data and metadata, + increasing the cache hit ratio. + + ZFS offers 19 levels of + Zstd compression, each offering + incrementally more space savings in exchange for slower + compression. The default level is + zstd-3 and offers greater compression + than LZ4 without being significantly + slower. Levels above 10 require significant amounts of + memory to compress each block, so they are discouraged on + systems with less than 16 GB of RAM. + ZFS also implements a selection of the + Zstd fast levels, + which get correspondingly faster but offer lower + compression ratios. ZFS supports + zstd-fast-1 through + zstd-fast-10, + zstd-fast-20 through + zstd-fast-100 in increments of 10, and + finally zstd-fast-500 and + zstd-fast-1000 which provide minimal + compression, but offer very high performance. + + If ZFS is not able to allocate the required memory to + compress a block with Zstd, it will fall + back to storing the block uncompressed. This is unlikely + to happen outside of the highest levels of + Zstd on systems that are memory + constrained. The sysctl + kstat.zfs.misc.zstd.compress_alloc_fail + counts how many times this has occurred since the + ZFS module was loaded. + + Deduplication diff --git a/share/tools/dobuild.sh b/share/tools/dobuild.sh new file mode 100755 --- /dev/null +++ b/share/tools/dobuild.sh @@ -0,0 +1,116 @@ +#!/bin/sh +# Copyright (c) 2001 Wolfram Schneider +# Copyright (c) 2001 Dima Dorfman +# Copyright (c) 2004 Simon L. Nielsen +# +# Build the FreeBSD docs from the Git repository. +# +# +# Major variables: +# +# PATH - The search path as interpreted by the shell. +# DOCGIT - Url to the FreeBSD doc Git repository. +# BUILDDIR - Where the checked out copies of the files are stored. +# DESTDIR - Where the rendered copies should wind up. +# BUILDARGS - Arguments to pass to make(1) when {build,install}ing. +# INSTARGS - Arguments to pass to make(1) when installing. +# WEBMAILTO - Address to send mail to if the build fails. +# +# subtrees - List of directores in $BUILDDIR which are from SCM. +# +# Variables which are in uppercase are derived from the environment +# unless they don't exist, in which case a value suitable for +# www.FreeBSD.org is used. Variables in lowercase can't be safely +# changed without editing other parts of the script; thus, their value +# in the environment is ignored. +# +# Exit codes: +# +# 0 - success +# 1 - unknown failure +# 2 - failure in Git operations +# 3 - failure in make operations +# + +# +# NOTE: This script is not automatically updated on the builder. +# + +# WARNING! This script depend on dobuild_wrap.sh setting the +# configuration variables in the enviroment. +# +if [ -z "${DOBUILDWRAP}" ]; then + echo "Error: This script should only be called from dobuild_wrap.sh!" + exit 1 +fi + +# Only install some compression types +INSTALL_COMPRESSED=${COMPTYPES}; export INSTALL_COMPRESSED + +# Abort on all errors +set -e + +umask 002 +cd $BUILDDIR || exit 1 + +# Remove the old copies. +if [ -z "${NOCLEAN}" ]; then + echo "===> Removing old stuff" + rm -Rf doc out 2>/dev/null || true + chflags -R noschg doc out 2>/dev/null || true + rm -Rf doc out 2>/dev/null + + echo "===> Check out the new doc" + git clone -b main ${DOCGIT} doc || exit 2 +fi + +cd $BUILDDIR/doc || exit 1 +mkdir -p $DOCDIR +mkdir -p $DOCDIR/packages +rm -f ${BUILDDIR}/packages # TMP +ln -s ${DOCDIR}/packages ${BUILDDIR}/doc/packages + +if [ -d ${PATCHDIR} -a -n "$(find ${PATCHDIR} -name \*.patch)" ]; then + echo "===> Patching bugs in doc tree" + for p in `echo ${PATCHDIR}/*.patch`; do + patch < ${p} + done +fi + +echo "===> Building doc" +time make ${BUILDARGS} all || exit 3 + +echo "===> Installing doc package to temp roots" +time make ${INSTARGS} install || exit 3 + +#echo "===> Building doc packages" +#time make ${BUILDARGS} package || exit 3 + +# XXX TODO, check for correct string when tex run out of resources +#echo "===> Testing for broken tex output" +#grep 'hash' $LOGFILE > /dev/null && \ +# (echo "Error: Possible TeX out of resources."; false) + +#OKFILENAME + +# build a string to use with find(1) to identify +# uncompressed files to be copied +UNCOMP_FIND_STR="" +for fmt in ${UNCOMPRESSED_FORMATS}; do + UNCOMP_FIND_STR="${UNCOMP_FIND_STR}( -name article.${fmt} -or -name book.${fmt} ) -or " +done +# add outer parens, remove the final "-or " +UNCOMP_FIND_STR="( ${UNCOMP_FIND_STR%-or } )" + +if [ -n "${UNCOMP_FIND_STR}" ]; then + echo "copying uncompressed formats: ${UNCOMPRESSED_FORMATS}" + cd ${BUILDDIR}/doc + find -X ??_* ${UNCOMP_FIND_STR} -print0 \ + | xargs -0 -I % install -v -C -o docbuild -g docbuild -m 444 % ${DOCDIR}/% +fi + +rm -rf ${OUTDIR}/* +mkdir -p ${OUTDIR} +cp -Rp ${DOCDIR}/* ${OUTDIR} + +exit 0 diff --git a/share/tools/dobuild_wrap.sh b/share/tools/dobuild_wrap.sh new file mode 100755 --- /dev/null +++ b/share/tools/dobuild_wrap.sh @@ -0,0 +1,137 @@ +#!/bin/sh +# +# Copyright (c) 2004 Simon L. Nielsen +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# + +# +# Wrapper around doc build. Simplifies main build script. +# NOTE: This script is not automatically updated on the builder. + +if [ -f "${PWD}/local.conf" ]; then + . ${PWD}/local.conf +fi + +# +# Default configuration. +# +DEFAULT_PATH=/bin:/usr/bin:/usr/local/bin +DEFAULT_BUILDROOT=/local0/docbuild +DEFAULT_DOCGIT="https://git.freebsd.org/doc.git" +DEFAULT_BUILDARGS="NO_JPMAN=yes GEN_INDEX=yes" +DEFAULT_INSTARGS="NO_JPMAN=yes GEN_INDEX=yes INSTALL_ONLY_COMPRESSED=yes" +# Must keep space between addresses. +DEFAULT_WEBMAILTO="doceng@FreeBSD.org, wblock@wonkity.com" +#DEFAULT_FORMATS="html html.tar html-split html-split.tar txt ps pdf rtf pdb" +#DEFAULT_FORMATS="html html.tar html-split html-split.tar txt ps pdf epub" +DEFAULT_FORMATS="html html.tar html-split html-split.tar txt pdf epub" +DEFAULT_UNCOMPRESSED_FORMATS="pdf epub" +DEFAULT_COMPTYPES="bz2 zip" + +# +# Variable setup. +# +BUILDROOT=${BUILDROOT:-${DEFAULT_BUILDROOT}}; export BUILDROOT +PATH="${PATH}:${DEFAULT_PATH}"; export PATH +DOCGIT=${DOCGIT:-${DEFAULT_DOCGIT}}; export DOCGIT +BUILDDIR=${BUILDDIR:-${BUILDROOT}/build}; export BUILDDIR +DOCDIR=${DOCDIR:-${BUILDROOT}/build/out}; export DOCDIR +FORMATS=${FORMATS:-${DEFAULT_FORMATS}}; export FORMATS +UNCOMPRESSED_FORMATS=${UNCOMPRESSED_FORMATS:-${DEFAULT_UNCOMPRESSED_FORMATS}}; export UNCOMPRESSED_FORMATS +COMPTYPES=${COMPTYPES:-${DEFAULT_COMPTYPES}}; export COMPTYPES +BUILDARGS=${BUILDARGS:-${DEFAULT_BUILDARGS}}; export BUILDARGS +INSTARGS=${INSTARGS:-${DEFAULT_INSTARGS}}; export INSTARGS +WEBMAILTO=${WEBMAILTO:-${DEFAULT_WEBMAILTO}}; export WEBMAILTO +PATCHDIR=${PATCHDIR:-${BUILDROOT}/patches}; export PATCHDIR +OUTDIR=${OUTDIR:-${BUILDROOT}/docout}; export OUTDIR +LOGROOT=${LOGBUILDROOT:-${BUILDROOT}/logs}; export LOGROOT +LOGDIR=${LOGROOT}/$(date +%Y%m%d%H%M%S); export LOGDIR +LOGFILE=${LOGDIR}/buildlog; export LOGFILE +LOCKFILE=${LOCKFILE:-${BUILDROOT}/lock}; export LOCKFILE +STATUSFILE=${LOGDIR}/status.xml; export STATUSFILE +BUILDSFILE=${LOGROOT}/builds.xml; export BUILDSFILE + +DOBUILDWRAP="yes"; export DOBUILDWRAP + +# Run the actual script inside a lock so only one instance can run at +# the time. +lockf -t 0 ${LOCKFILE} /bin/sh << 'E*O*F' + if [ ! -d ${LOGDIR} ]; then + mkdir -p ${LOGDIR} + fi + LOGFILE_REL=${LOGFILE##${LOGROOT}/} + echo "" > ${STATUSFILE} + echo " $(hostname)" >> ${STATUSFILE} + echo " $(date +%Y%m%d%H%M%S)" >> ${STATUSFILE} + echo " ${LOGFILE_REL}" >> ${STATUSFILE} + echo " running" >> ${STATUSFILE} + echo "" >> ${STATUSFILE} + + echo '' > ${BUILDSFILE} + find ${LOGROOT} -name status.xml | xargs cat >> ${BUILDSFILE} + echo '' >> ${BUILDSFILE} + + # Do the actual build. + starttime=$(date +%s) + sh -x ${BUILDROOT}/dobuild.sh > ${LOGFILE} 2>&1 + ret=$? + touch ${LOGFILE} + endtime=$(date +%s) + runtime=$((${endtime} - ${starttime})) + + # Remove the status and "footer" of ${STATUSFILE}. + keeplines=$(($(wc -l < ${STATUSFILE}) - 2)) + mv ${STATUSFILE} ${STATUSFILE}.tmp + head -n ${keeplines} ${STATUSFILE}.tmp > ${STATUSFILE} + rm -f ${STATUSFILE}.tmp + + echo " $(date +%Y%m%d%H%M%S)" >> ${STATUSFILE} + echo " ${runtime}" >> ${STATUSFILE} + echo " ${ret}" >> ${STATUSFILE} + if [ ${ret} -eq 0 ]; then + echo " completed" >> ${STATUSFILE} + else + echo " failed" >> ${STATUSFILE} + fi + + if [ ${ret} -ne 0 ]; then + tail -100 ${LOGFILE} | \ + mail -s "FreeBSD FTP doc build failed" ${WEBMAILTO} + fi + + if egrep '^! TeX capacity exceeded' ${LOGFILE} > /dev/null; then + egrep '^! TeX capacity exceeded' ${LOGFILE} | \ + mail -s "FreeBSD FTP doc build warning" ${WEBMAILTO} + echo " Possibly out of TeX resources." \ + >> ${STATUSFILE} + fi + echo "" >> ${STATUSFILE} + + echo '' > ${BUILDSFILE} + find ${LOGROOT} -name status.xml | xargs cat >> ${BUILDSFILE} + echo '' >> ${BUILDSFILE} +E*O*F + + +exit 0