diff --git a/fix_typos_wrong_links.diff b/fix_typos_wrong_links.diff new file mode 100644 --- /dev/null +++ b/fix_typos_wrong_links.diff @@ -0,0 +1,34472 @@ +From 99ce8bc6e36e639c1fc428fef316561cc568f216 Mon Sep 17 00:00:00 2001 +From: Vladlen Popolitov +Date: Tue, 9 Sep 2025 00:26:57 +0300 +Subject: [PATCH] fix typos, wrong links in books and articles + +--- + .../en/articles/committers-guide/_index.adoc | 20 +++++++++---------- + .../en/articles/contributing/_index.adoc | 8 ++++---- + .../en/articles/filtering-bridges/_index.adoc | 2 +- + .../en/articles/freebsd-questions/_index.adoc | 4 ++-- + .../freebsd-update-server/_index.adoc | 10 +++++----- + .../en/articles/gjournal-desktop/_index.adoc | 12 +++++------ + .../content/en/articles/hubs/_index.adoc | 2 +- + .../en/articles/ipsec-must/_index.adoc | 2 +- + .../en/articles/leap-seconds/_index.adoc | 2 +- + .../en/articles/linux-emulation/_index.adoc | 2 +- + .../en/articles/linux-users/_index.adoc | 20 +++++++++---------- + .../en/articles/rc-scripting/_index.adoc | 4 ++-- + .../content/en/articles/releng/_index.adoc | 4 ++-- + .../en/articles/remote-install/_index.adoc | 2 +- + .../en/articles/serial-uart/_index.adoc | 16 +++++++-------- + .../content/en/articles/vinum/_index.adoc | 10 +++++----- + .../en/books/arch-handbook/_index.adoc | 4 ++-- + .../content/en/books/arch-handbook/book.adoc | 4 ++-- + .../en/books/arch-handbook/boot/_index.adoc | 2 +- + .../en/books/arch-handbook/jail/_index.adoc | 2 +- + .../en/books/developers-handbook/_index.adoc | 2 +- + .../en/books/developers-handbook/book.adoc | 2 +- + .../developers-handbook/tools/_index.adoc | 4 ++-- + .../en/books/handbook/config/_index.adoc | 4 ++-- + .../books/handbook/cutting-edge/_index.adoc | 2 +- + .../en/books/handbook/desktop/_index.adoc | 2 +- + .../en/books/handbook/disks/_index.adoc | 2 +- + .../en/books/handbook/introduction.adoc | 2 +- + .../porters-handbook/makefiles/_index.adoc | 4 ++-- + .../porters-handbook/special/_index.adoc | 2 +- + .../porters-handbook/upgrading/_index.adoc | 2 +- + 31 files changed, 80 insertions(+), 80 deletions(-) + +diff --git a/documentation/content/en/articles/committers-guide/_index.adoc b/documentation/content/en/articles/committers-guide/_index.adoc +index 2650a480ba..5da77337ac 100644 +--- a/documentation/content/en/articles/committers-guide/_index.adoc ++++ b/documentation/content/en/articles/committers-guide/_index.adoc +@@ -1,3835 +1,3835 @@ + --- + title: Committer's Guide + authors: + - author: The FreeBSD Documentation Project + copyright: 1999-2022 The FreeBSD Documentation Project + description: Introductory information for FreeBSD committers + trademarks: ["freebsd", "coverity", "git", "github", "gitlab", "ibm", "intel", "general"] + weight: 25 + tags: ["FreeBSD Committer's Guide", "Guide", "Community"] + --- + + = Committer's Guide + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/committers-guide/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + This document provides information for the FreeBSD committer community. + All new committers should read this document before they start, and existing committers are strongly encouraged to review it from time to time. + + Almost all FreeBSD developers have commit rights to one or more repositories. + However, a few developers do not, and some of the information here applies to them as well. + (For instance, some people only have rights to work with the Problem Report database.) + Please see crossref:committers-guide[non-committers, Issues Specific to Developers Who Are Not Committers] for more information. + + This document may also be of interest to members of the FreeBSD community who want to learn more about how the project works. + + ''' + + toc::[] + + [[admin]] + == Administrative Details + + [.informaltable] + [cols="1,1", frame="none"] + |=== + + |_Login Methods_ + |man:ssh[1], protocol 2 only + + |_Main Shell Host_ + |`freefall.FreeBSD.org` + + |_Reference Machines_ + |`ref*.FreeBSD.org`, `universe*.freeBSD.org` (see also link:https://www.FreeBSD.org/internal/machines/[FreeBSD Project Hosts]) + + |_SMTP Host_ + |`smtp.FreeBSD.org:587` (see also crossref:committers-guide[smtp-setup, SMTP Access Setup]). + + |`_src/_` Git Repository + |`ssh://git@gitrepo.FreeBSD.org/src.git` + + |`_doc/_` Git Repository + |`ssh://git@gitrepo.FreeBSD.org/doc.git` + + |`_ports/_` Git Repository + |`ssh://git@gitrepo.FreeBSD.org/ports.git` + + |_Internal Mailing Lists_ + |developers (technically called all-developers), doc-developers, doc-committers, ports-developers, ports-committers, src-developers, src-committers. (Each project repository has its own -developers and -committers mailing lists. Archives for these lists can be found in the files [.filename]#/local/mail/repository-name-developers-archive# and [.filename]#/local/mail/repository-name-committers-archive# on `freefall.FreeBSD.org`.) + + |_Core Team monthly reports_ + |[.filename]#/home/core/public/reports# on the `FreeBSD.org` cluster. + + |_Ports Management Team monthly reports_ + |[.filename]#/home/portmgr/public/monthly-reports# on the `FreeBSD.org` cluster. + + |_Noteworthy `src/` Git Branches:_ + |`stable/n` (`n`-STABLE), `main` (-CURRENT) + |=== + + man:ssh[1] is required to connect to the project hosts. For more information, + see crossref:committers-guide[ssh.guide, SSH Quick-Start Guide]. + + Useful links: + + * link:https://www.FreeBSD.org/internal/[FreeBSD Project Internal Pages] + * link:https://www.FreeBSD.org/internal/machines/[FreeBSD Project Hosts] + * link:https://www.FreeBSD.org/administration/[FreeBSD Project Administrative Groups] + + [[pgpkeys]] + == OpenPGP Keys for FreeBSD + + Cryptographic keys conforming to the OpenPGP (__Pretty Good Privacy__) standard are used by the FreeBSD project to authenticate committers. + Messages carrying important information like public SSH keys can be signed with the OpenPGP key to prove that they are really from the committer. + See https://nostarch.com/releases/pgp_release.pdf[PGP & GPG: Email for the Practical Paranoid by Michael Lucas] and https://en.wikipedia.org/wiki/Pretty_Good_Privacy[] for more information. + + [[pgpkeys-creating]] + === Creating a Key + + Existing keys can be used, but should be checked with [.filename]#documentation/tools/checkkey.sh# first. + In this case, make sure the key has a FreeBSD user ID. + + For those who do not yet have an OpenPGP key, or need a new key to meet FreeBSD security requirements, here we show how to generate one. + + [[pgpkeys-create-steps]] + [.procedure] + ==== + . Install [.filename]#security/gnupg#. Enter these lines in [.filename]#~/.gnupg/gpg.conf# to set minimum acceptable defaults for signing and new key preferences (see the link:https://www.gnupg.org/documentation/manuals/gnupg/GPG-Options.html[GnuPG options documentation] for more details): + + + [.programlisting] + .... + # Sorted list of preferred algorithms for signing (strongest to weakest). + personal-digest-preferences SHA512 SHA384 SHA256 SHA224 + # Default preferences for new keys + default-preference-list SHA512 SHA384 SHA256 SHA224 AES256 CAMELLIA256 AES192 CAMELLIA192 AES CAMELLIA128 CAST5 BZIP2 ZLIB ZIP Uncompressed + .... + . Generate a key: + + + [source,shell] + .... + % gpg --full-gen-key + gpg (GnuPG) 2.1.8; Copyright (C) 2015 Free Software Foundation, Inc. + This is free software: you are free to change and redistribute it. + There is NO WARRANTY, to the extent permitted by law. + + Warning: using insecure memory! + Please select what kind of key you want: + (1) RSA and RSA (default) + (2) DSA and Elgamal + (3) DSA (sign only) + (4) RSA (sign only) + Your selection? 1 + RSA keys may be between 1024 and 4096 bits long. + What keysize do you want? (2048) 2048 <.> + Requested keysize is 2048 bits + Please specify how long the key should be valid. + 0 = key does not expire + = key expires in n days + w = key expires in n weeks + m = key expires in n months + y = key expires in n years + Key is valid for? (0) 3y <.> + Key expires at Wed Nov 4 17:20:20 2015 MST + Is this correct? (y/N) y + GnuPG needs to construct a user ID to identify your key. + + Real name: Chucky Daemon <.> + Email address: notreal@example.com + Comment: + You selected this USER-ID: + "Chucky Daemon " + + Change (N)ame, (C)omment, (E)mail or (O)kay/(Q)uit? o + You need a Passphrase to protect your secret key. + .... + + <.> 2048-bit keys with a three-year expiration provide adequate protection at present (2022-10). + + <.> A three year key lifespan is short enough to obsolete keys weakened by advancing computer power, but long enough to reduce key management problems. + + <.> Use your real name here, preferably matching that shown on government-issued ID to make it easier for others to verify your identity. Text that may help others identify you can be entered in the `Comment` section. + + + After the email address is entered, a passphrase is requested. + Methods of creating a secure passphrase are contentious. + Rather than suggest a single way, here are some links to sites that describe various methods: https://world.std.com/~reinhold/diceware.html[], https://www.iusmentis.com/security/passphrasefaq/[], https://xkcd.com/936/[], https://en.wikipedia.org/wiki/Passphrase[]. + ==== + + Protect the private key and passphrase. + If either the private key or passphrase may have been compromised or disclosed, immediately notify mailto:accounts@FreeBSD.org[accounts@FreeBSD.org] and revoke the key. + + Committing the new key is shown in crossref:committers-guide[commit-steps, Steps for New Committers]. + + [[kerberos-ldap]] + == Kerberos and LDAP web Password for FreeBSD Cluster + + The FreeBSD cluster requires a Kerberos password to access certain services. + The Kerberos password also serves as the LDAP web password, since LDAP is proxying to Kerberos in the cluster. + Some of the services which require this include: + + * https://bugs.freebsd.org/bugzilla[Bugzilla] + + To create a new Kerberos account in the FreeBSD cluster, or to reset a Kerberos password for an existing account using a random password generator: + + [source,shell] + .... + % ssh kpasswd.freebsd.org + .... + + [NOTE] + ==== + This must be done from a machine outside of the FreeBSD.org cluster. + ==== + + A Kerberos password can also be set manually by logging into `freefall.FreeBSD.org` and running: + + [source,shell] + .... + % kpasswd + .... + + [NOTE] + ==== + Unless the Kerberos-authenticated services of the FreeBSD.org cluster have been used previously, `Client unknown` will be shown. + This error means that the `ssh kpasswd.freebsd.org` method shown above must be used first to initialize the Kerberos account. + ==== + + [[committer.types]] + == Commit Bit Types + + The FreeBSD repository has a number of components which, when combined, support the basic operating system source, documentation, third party application ports infrastructure, and various maintained utilities. + When FreeBSD commit bits are allocated, the areas of the tree where the bit may be used are specified. + Generally, the areas associated with a bit reflect who authorized the allocation of the commit bit. + Additional areas of authority may be added at a later date: when this occurs, the committer should follow normal commit bit allocation procedures for that area of the tree, seeking approval from the appropriate entity and possibly getting a mentor for that area for some period of time. + + [.informaltable] + [cols="1,1,1", frame="none"] + |=== + + |__Committer Type__ + |__Responsible__ + |__Tree Components__ + + |src + |srcmgr@ + |src/ + + |doc + |doceng@ + |doc/, ports/, src/ documentation + + |ports + |portmgr@ + |ports/ + |=== + + Commit bits allocated prior to the development of the notion of areas of authority may be appropriate for use in many parts of the tree. + However, common sense dictates that a committer who has not previously worked in an area of the tree seek review prior to committing, seek approval from the appropriate responsible party, and/or work with a mentor. + Since the rules regarding code maintenance differ by area of the tree, this is as much for the benefit of the committer working in an area of less familiarity as it is for others working on the tree. + + Committers are encouraged to seek review for their work as part of the normal development process, regardless of the area of the tree where the work is occurring. + + === Policy for Committer Activity in Other Trees + + * All committers may modify [.filename]#src/share/misc/committers-*.dot#, [.filename]#src/usr.bin/calendar/calendars/calendar.freebsd#, and [.filename]#ports/astro/xearth/files#. + * doc committers may commit documentation changes to [.filename]#src# files, such as manual pages, READMEs, fortune databases, calendar files, and comment fixes without approval from a src committer, subject to the normal care and tending of commits. + * Any committer may make changes to any other tree with an "Approved by" from a non-mentored committer with the appropriate bit. + Mentored committers can provide a "Reviewed by" but not an "Approved by". + * Committers can acquire an additional bit by the usual process of finding a mentor who will propose them to srcmgr, doceng, or portmgr, as appropriate. When approved, they will be added to 'access' and the normal mentoring period will ensue, which will involve a continuing of "Approved by" for some period. + + [[doc-blanket-approval]] + ==== Documentation Implicit (Blanket) Approval + + Some types of fixes have "blanket approval" from the {doceng}, allowing any committer to fix those categories of problems on any part of the doc tree. + These fixes do not need approval or review from a doc committer if the author doesn't have a doc commit bit. + + Blanket approval applies to these types of fixes: + + * Typos + * Trivial fixes + + + Punctuation, URLs, dates, paths and file names with outdated or incorrect information, and other common mistakes that may confound the readers. + + Over the years, some implicit approvals were granted in the doc tree. + This list shows the most common cases: + + * Changes in [.filename]#documentation/content/en/books/porters-handbook/versions/_index.adoc# + + + extref:{porters-handbook}versions/[__FreeBSD_version Values (Porter's Handbook)], mainly used for src committers. + * Changes in [.filename]#doc/shared/contrib-additional.adoc# + + + extref:{contributors}[Additional FreeBSD Contributors, contrib-additional] maintenance. + * All link:#commit-steps[Steps for New Committers], doc related + * Security advisories; Errata Notices; Releases; + + + Used by {security-officer} and {re}. + * Changes in [.filename]#website/content/en/donations/donors.adoc# + + + Used by {donations}. + + Before any commit, a build test is necessary; see the 'Overview' and 'The FreeBSD Documentation Build Process' sections of the extref:{fdp-primer}[FreeBSD Documentation Project Primer for New Contributors] for more details. + + [[git-primer]] + == Git Primer + + [[git-basics]] + === Git basics + + When one searches for "Git Primer" a number of good ones come up. + Daniel Miessler's link:https://danielmiessler.com/study/git/[A git primer] and Willie Willus' link:https://gist.github.com/williewillus/068e9a8543de3a7ef80adb2938657b6b[Git - Quick Primer] are both good overviews. + The Git book is also complete, but much longer https://git-scm.com/book/en/v2. + There is also this website https://dangitgit.com/ for common traps and pitfalls of Git, in case you need guidance to fix things up. + Finally, an introduction link:https://eagain.net/articles/git-for-computer-scientists/[targeted at computer scientists] has proven helpful to some at explaining the Git world view. + + This document will assume that you've read through it and will try not to belabor the basics (though it will cover them briefly). + + [[git-mini-primer]] + === Git Mini Primer + + This primer is less ambitiously scoped than the old Subversion Primer, but should cover the basics. + + ==== Scope + + If you want to download FreeBSD, compile it from sources, and generally keep up to date that way, this primer is for you. + It covers getting the sources, updating the sources, bisecting and touches briefly on how to cope with a few local changes. + It covers the basics, and tries to give good pointers to more in-depth treatment for when the reader finds the basics insufficient. + Other sections of this guide cover more advanced topics related to contributing to the project. + + The goal of this section is to highlight those bits of Git needed to track sources. + They assume a basic understanding of Git. + There are many primers for Git on the web, but the https://git-scm.com/book/en/v2[Git Book] provides one of the better treatments. + + [[git-mini-primer-getting-started]] + ==== Getting Started For Developers + + This section describes the read-write access for committers to push the commits from developers or contributors. + + [[git-mini-daily-use]] + ===== Daily use + + [NOTE] + ==== + In the examples below, replace `${repo}` with the name of the desired FreeBSD repository: `doc`, `ports`, or `src`. + ==== + + * Clone the repository: + + + [source,shell] + .... + % git clone -o freebsd --config remote.freebsd.fetch='+refs/notes/*:refs/notes/*' https://git.freebsd.org/${repo}.git + .... + + + Then you should have the official mirrors as your remote: + + + [source,shell] + .... + % git remote -v + freebsd https://git.freebsd.org/${repo}.git (fetch) + freebsd https://git.freebsd.org/${repo}.git (push) + .... + + * Configure the FreeBSD committer data: + + + The commit hook in repo.freebsd.org checks the "Commit" field matches the committer's information in FreeBSD.org. + The easiest way to get the suggested config is by executing `/usr/local/bin/gen-gitconfig.sh` script on freefall: + + + [source,shell] + .... + % gen-gitconfig.sh + [...] + % git config user.name (your name in gecos) + % git config user.email (your login)@FreeBSD.org + .... + + * Set the push URL: + + + [source,shell] + .... + % git remote set-url --push freebsd git@gitrepo.freebsd.org:${repo}.git + .... + + + Then you should have separated fetch and push URLs as the most efficient setup: + + + [source,shell] + .... + % git remote -v + freebsd https://git.freebsd.org/${repo}.git (fetch) + freebsd git@gitrepo.freebsd.org:${repo}.git (push) + .... + + + Again, note that `gitrepo.freebsd.org` has been canonicalized to `repo.freebsd.org`. + + * Install commit message template hook: + + + For doc repository: + + + [source,shell] + .... + % cd .git/hooks + % ln -s ../../.hooks/prepare-commit-msg + .... + + + For ports repository: + + + [source,shell] + .... + % git config --add core.hooksPath .hooks + .... + + + For src repository: + + + [source,shell] + .... + % cd .git/hooks + % ln -s ../../tools/tools/git/hooks/prepare-commit-msg + .... + + [[admin-branch]] + ===== "admin" branch + + The `access` and `mentors` files are stored in an orphan branch, `internal/admin`, in each repository. + + Following example is how to check out the `internal/admin` branch to a local branch named `admin`: + + [source,shell] + .... + % git config --add remote.freebsd.fetch '+refs/internal/*:refs/internal/*' + % git fetch + % git checkout -b admin internal/admin + .... + Alternatively, you can add a worktree for the `admin` branch: + + [source,shell] + .... + git worktree add -b admin ../${repo}-admin internal/admin + .... + + For browsing `internal/admin` branch on web: + `https://cgit.freebsd.org/${repo}/log/?h=internal/admin` + + For pushing, specify the full refspec: + + [source,shell] + .... + git push freebsd HEAD:refs/internal/admin + .... + + ==== Keeping Current With The FreeBSD src Tree + [[keeping_current]] + First step: cloning a tree. + This downloads the entire tree. + There are two ways to download. + Most people will want to do a deep clone of the repository. + However, there are times when you may wish to do a shallow clone. + + ===== Branch Names + FreeBSD-CURRENT uses the `main` branch. + + `main` is the default branch. + + For FreeBSD-STABLE, branch names include `stable/12` and `stable/13`. + + For FreeBSD-RELEASE, release engineering branch names include `releng/12.4` and `releng/13.2`. + + https://www.freebsd.org/releng/[] shows: + + * `main` and `stable/⋯` branches open + * `releng/⋯` branches, each of which is frozen when a release is tagged. + + Examples: + + * tag https://cgit.freebsd.org/src/tag/?h=release/13.1.0[release/13.1.0] on the https://cgit.freebsd.org/src/log/?h=releng/13.1[releng/13.1] branch + * tag https://cgit.freebsd.org/src/tag/?h=release/13.2.0[release/13.2.0] on the https://cgit.freebsd.org/src/log/?h=releng/13.2[releng/13.2] branch. + + ===== Repositories + Please see the crossref:committers-guide[admin,Administrative Details] for the latest information on where to get FreeBSD sources. + $URL below can be obtained from that page. + + Note: The project doesn't use submodules as they are a poor fit for our workflows and development model. + How we track changes in third-party applications is discussed elsewhere and generally of little concern to the casual user. + + ===== Deep Clone + A deep clone pulls in the entire tree, as well as all the history and branches. + It is the easiest to do. + It also allows you to use Git's worktree feature to have all your active branches checked out into separate directories but with only one copy of the repository. + [source,shell] + .... + % git clone -o freebsd $URL -b branch [] + .... + -- will create a deep clone. + `branch` should be one of the branches listed in the previous section. + If no `branch` is given: the default (`main`) will be used. + If no `` is given: the name of the new directory will match the name of the repo ([.filename]#doc#, [.filename]#ports# or [.filename]#src#). + + You will want a deep clone if you are interested in the history, plan on making local changes, or plan on working on more than one branch. + It is the easiest to keep up to date as well. + If you are interested in the history, but are working with only one branch and are short on space, you can also use --single-branch to only download the one branch + (though some merge commits will not reference the merged-from branch which may be important for some users who are interested in detailed versions of history). + + ===== Shallow Clone + + A shallow clone copies just the most current code, but none or little of the history. + This can be useful when you need to build a specific revision of FreeBSD, or when you are just starting out and plan to track the tree more fully. + You can also use it to limit history to only so many revisions. + However, see below for a significant limitation of this approach. + + [source,shell] + .... + % git clone -o freebsd -b branch --depth 1 $URL [dir] + .... + + This clones the repository, but only has the most recent version in the repository. + The rest of the history is not downloaded. + Should you change your mind later, you can do `git fetch --unshallow` to get the old history. + + [WARNING] + ==== + When you make a shallow clone, you will lose the commit count in your uname output. + This can make it more difficult to determine if your system needs to be updated when a security advisory is issued. + ==== + + ===== Building + + Once you've downloaded, building is done as described in the handbook, + e.g.: + [source,shell] + .... + % cd src + % make buildworld + % make buildkernel + % make installkernel + % make installworld + .... + so that won't be covered in depth here. + +-If you want to build a custom kernel, extref:{handbook}[the kernel config section, kernelconfig] of the FreeBSD Handbook recommends creating a file MYKERNEL under sys/${ARCH}/conf with your changes against GENERIC. ++If you want to build a custom kernel, extref:{handbook}kernelconfig[the kernel config section, kernelconfig] of the FreeBSD Handbook recommends creating a file MYKERNEL under sys/${ARCH}/conf with your changes against GENERIC. + To have MYKERNEL disregarded by Git, it can be added to .git/info/exclude. + + ===== Updating + + To update both types of trees uses the same commands. + This pulls in all the revisions since your last update. + [source,shell] + .... + % git pull --ff-only + .... + will update the tree. + In Git, a 'fast forward' merge is one that only needs to set a new branch pointer and doesn't need to re-create the commits. + By always doing a fast forward merge/pull, you'll ensure that you have an exact copy of the FreeBSD tree. + This will be important if you want to maintain local patches. + + See below for how to manage local changes. + The simplest is to use `--autostash` on the `git pull` command, but more sophisticated options are available. + + ==== Selecting a Specific Version + + In Git, `git checkout` checks out both branches and specific versions. + Git's versions are the long hashes rather than a sequential number. + + When you checkout a specific version, just specify the hash you want on the command line (the git log command can help you decide which hash you might want): + [source,shell] + .... + % git checkout 08b8197a74 + .... + and you have that checked out. + You will be greeted with a message similar to the following: + [source,shell] + .... + Note: checking out '08b8197a742a96964d2924391bf9fdfeb788865d'. + + You are in a 'detached HEAD' state. You can look around, make experimental + changes and commit them, and you can discard any commits you make in this + state without impacting any branches by performing another checkout. + + If you want to create a new branch to retain commits you create, you may + do so (now or later) by using -b with the checkout command again. Example: + + git checkout -b + + HEAD is now at 08b8197a742a hook gpiokeys.4 to the build + .... + where the last line is generated from the hash you are checking out and the first line of the commit message from that revision. + The hash can be abbreviated to the shortest unique length. + Git itself is inconsistent about how many digits it displays. + + ==== Bisecting + Sometimes, things go wrong. + The last version worked, but the one you just updated to does not. + A developer may ask you to bisect the problem to track down which commit caused the regression. + + Git makes bisecting changes easy with a powerful `git bisect` command. + Here's a brief outline of how to use it. + For more information, you can view https://www.metaltoad.com/blog/beginners-guide-git-bisect-process-elimination or https://git-scm.com/docs/git-bisect for more details. + The man git-bisect page is good at describing what can go wrong, what to do when versions won't build, when you want to use terms other than 'good' and 'bad', etc, none of which will be covered here. + + `git bisect start --first-parent` will start the bisection process. + Next, you need to tell a range to go through. + `git bisect good XXXXXX` will tell it the working version and `git bisect bad XXXXX` will tell it the bad version. + The bad version will almost always be HEAD (a special tag for what you have checked out). + The good version will be the last one you checked out. + The `--first-parent` argument is necessary so that subsequent `git bisect` commands do not try to check out a vendor branch which lacks the full FreeBSD source tree. + + [TIP] + ==== + If you want to know the last version you checked out, you should use `git reflog`: + [source,shell] + .... + 5ef0bd68b515 (HEAD -> main, freebsd/main, freebsd/HEAD) HEAD@{0}: pull --ff-only: Fast-forward + a8163e165c5b (upstream/main) HEAD@{1}: checkout: moving from b6fb97efb682994f59b21fe4efb3fcfc0e5b9eeb to main + ... + .... + shows me moving the working tree to the `main` branch (a816...) and then updating from upstream (to 5ef0...). + In this case, bad would be HEAD (or 5ef0bd68b515) and good would be a8163e165c5b. + As you can see from the output, HEAD@{1} also often works, but isn't foolproof if you have done other things to your Git tree after updating, but before you discover the need to bisect. + ==== + + Set the 'good' version first, then set the bad (though the order doesn't matter). + When you set the bad version, it will give you some statistics on the process: + [source,shell] + .... + % git bisect start --first-parent + % git bisect good a8163e165c5b + % git bisect bad HEAD + Bisecting: 1722 revisions left to test after this (roughly 11 steps) + [c427b3158fd8225f6afc09e7e6f62326f9e4de7e] Fixup r361997 by balancing parens. Duh. + .... + + You would then build/install that version. + If it's good you'd type `git bisect good` otherwise `git bisect bad`. + If the version doesn't compile, type `git bisect skip`. + You will get a similar message to the above after each step. + When you are done, report the bad version to the developer (or fix the bug yourself and send a patch). + `git bisect reset` will end the process and return you back to where you started (usually tip of `main`). + Again, the git-bisect manual (linked above) is a good resource for when things go wrong or for unusual cases. + + [[git-gpg-signing]] + ==== Signing the commits, tags, and pushes, with GnuPG + + Git knows how to sign commits, tags, and pushes. + When you sign a Git commit or a tag, you can prove that the code you submitted came from you and wasn't altered while you were transferring it. + You also can prove that you submitted the code and not someone else. + + A more in-depth documentation on signing commits and tags can be found in the https://git-scm.com/book/en/v2/Git-Tools-Signing-Your-Work[Git Tools - Signing Your Work] chapter of the Git's book. + + The rationale behind signing pushes can be found in the https://github.com/git/git/commit/a85b377d0419a9dfaca8af2320cc33b051cbed04[commit that introduced the feature]. + + The best way is to simply tell Git you always want to sign commits, tags, and pushes. + You can do this by setting a few configuration variables: + + [source,shell] + .... + % git config --add user.signingKey LONG-KEY-ID + % git config --add commit.gpgSign true + % git config --add tag.gpgSign true + % git config --add push.gpgSign if-asked + .... + + // push.gpgSign should probably be set to `yes` once we enable it, or be set with --global, so that it is enabled for all repositories. + + [NOTE] + ====== + To avoid possible collisions, make sure you give a long key id to Git. + You can get the long id with: `gpg --list-secret-keys --keyid-format LONG`. + ====== + + [TIP] + ====== + To use specific subkeys, and not have GnuPG to resolve the subkey to a primary key, attach `!` to the key. + For example, to encrypt for the subkey `DEADBEEF`, use `DEADBEEF!`. + ====== + + ===== Verifying signatures + + Commit signatures can be verified by running either `git verify-commit `, or `git log --show-signature`. + + Tag signatures can be verified with `git verify-tag `, or `git tag -v `. + + //// + Commented out for now until we decide what to do. + + Git pushes are a bit different, they live in a special ref in the repository. + TODO: write how to verify them + + //// + + ==== Ports Considerations + The ports tree operates the same way. + The branch names are different and the repositories are in different locations. + + The cgit repository web interface for use with web browsers is at https://cgit.FreeBSD.org/ports/ . +-The production Git repository is at https://git.FreeBSD.org/ports.git and at ssh://anongit@git.FreeBSD.org/ports.git (or anongit@git.FreeBSD.org:ports.git). ++The production Git repository is at https://git.FreeBSD.org/ports.git and at ssh://anongit@git.FreeBSD.org/ports.git (or `anongit@git.FreeBSD.org:ports.git`). + +-There is also a mirror on GitHub, see extref:{handbook}/mirrors[External mirrors, mirrors] for an overview. ++There is also a mirror on GitHub, see extref:{handbook}mirrors[External mirrors, mirrors] for an overview. + The _latest_ branch is `main`. + The _quarterly_ branches are named `yyyyQn` for year 'yyyy' and quarter 'n'. + + [[port-commit-message-formats]] + ===== Commit message formats + + A hook is available in the ports repository to help you write up your commit messages in https://cgit.freebsd.org/ports/tree/.hooks/prepare-commit-msg[.hooks/prepare-commit-message]. + It can be enabled by running ``git config --add core.hooksPath .hooks``. + + The main point being that a commit message should be formatted in the following way: + + .... + category/port: Summary. + + Description of why the changes where made. + + PR: 12345 + .... + + [IMPORTANT] + ==== + The first line is the subject of the commit, it contains what port was changed, and a summary of the commit. + It should contain 50 characters or less. + + A blank line should separate it from the rest of the commit message. + + The rest of the commit message should be wrapped at the 72 characters boundary. + + Another blank line should be added if there are any metadata fields, so that they are easily distinguishable from the commit message. + ==== + + ==== Managing Local Changes + This section addresses tracking local changes. + If you have no local changes you can skip this section. + + One item that is important for all of them: all changes are local until pushed. + Unlike Subversion, Git uses a distributed model. + For users, for most things, there is very little difference. + However, if you have local changes, you can use the same tool to manage them as you use to pull in changes from FreeBSD. + All changes that you have not pushed are local and can easily be modified (git rebase, discussed below does this). + + ===== Keeping local changes + The simplest way to keep local changes (especially trivial ones) is to use `git stash`. + In its simplest form, you use `git stash` to record the changes (which pushes them onto the stash stack). + Most people use this to save changes before updating the tree as described above. + They then use `git stash apply` to re-apply them to the tree. + The stash is a stack of changes that can be examined with `git stash list`. + The git-stash man page (https://git-scm.com/docs/git-stash) has all the details. + + This method is suitable when you have tiny tweaks to the tree. + When you have anything non trivial, you'll likely be better off keeping a local branch and rebasing. + Stashing is also integrated with the `git pull` command: just add `--autostash` to the command line. + + ===== Keeping a local branch + [[keeping_a_local_branch]] + It is much easier to keep a local branch with Git than Subversion. + In Subversion you need to merge the commit, and resolve the conflicts. + This is manageable, but can lead to a convoluted history that's hard to upstream should that ever be necessary, or hard to replicate if you need to do so. + Git also allows one to merge, along with the same problems. + That's one way to manage the branch, but it's the least flexible. + + In addition to merging, Git supports the concept of 'rebasing' which avoids these issues. + The `git rebase` command replays all the commits of a branch at a newer location on the parent branch. + We will cover the most common scenarios that arise using it. + + ====== Create a branch + + Let's say you want to make a change to FreeBSD's ls command to never, ever do color. + There are many reasons to do this, but this example will use that as a baseline. + The FreeBSD ls command changes from time to time, and you'll need to cope with those changes. + Fortunately, with Git rebase it usually is automatic. + [source,shell] + .... + % cd src + % git checkout main + % git checkout -b no-color-ls + % cd bin/ls + % vi ls.c # hack the changes in + % git diff # check the changes + diff --git a/bin/ls/ls.c b/bin/ls/ls.c + index 7378268867ef..cfc3f4342531 100644 + --- a/bin/ls/ls.c + +++ b/bin/ls/ls.c + @@ -66,6 +66,7 @@ __FBSDID("$FreeBSD$"); + #include + #include + #include + +#undef COLORLS + #ifdef COLORLS + #include + #include + % # these look good, make the commit... + % git commit ls.c + .... + + The commit will pop you into an editor to describe what you've done. + Once you enter that, you have your own **local** branch in the Git repo. + Build and install it like you normally would, following the directions in the handbook. + Git differs from other version control systems in that you have to tell it explicitly which files to commit. + I have opted to do it on the commit command line, but you can also do it with `git add` which many of the more in depth tutorials cover. + + ====== Time to update + + When it is time to bring in a new version, it is almost the same as w/o the branches. + You would update like you would above, but there is one extra command before you update, and one after. + The following assumes you are starting with an unmodified tree. + It is important to start rebasing operations with a clean tree (Git requires this). + + [source,shell] + .... + % git checkout main + % git pull --ff-only + % git rebase -i main no-color-ls + .... + + This will bring up an editor that lists all the commits in it. + For this example, do not change it at all. + This is typically what you are doing while updating the baseline (though you also use the Git rebase command to curate the commits you have in the branch). + + Once you are done with the above, you have to move the commits to ls.c forward from the old version of FreeBSD to the newer one. + + Sometimes there are merge conflicts. + That is OK. + Do not panic. + Instead, handle them the same as any other merge conflicts. + To keep it simple, I will just describe a common issue that may arise. + A pointer to a complete treatment can be found at the end of this section. + + Let's say the includes changes upstream in a radical shift to terminfo as well as a name change for the option. + When you updated, you might see something like this: + [source,shell] + .... + Auto-merging bin/ls/ls.c + CONFLICT (content): Merge conflict in bin/ls/ls.c + error: could not apply 646e0f9cda11... no color ls + Resolve all conflicts manually, mark them as resolved with + "git add/rm ", then run "git rebase --continue". + You can instead skip this commit: run "git rebase --skip". + To abort and get back to the state before "git rebase", run "git rebase --abort". + Could not apply 646e0f9cda11... no color ls + .... + which looks scary. + + If you bring up an editor, you will see it is a typical 3-way merge conflict resolution that you may be familiar with from other source code systems (the rest of ls.c has been omitted): + [source,shell] + .... + <<<<<<< HEAD + #ifdef COLORLS_NEW + #include + ======= + #undef COLORLS + #ifdef COLORLS + #include + >>>>>>> 646e0f9cda11... no color ls + .... + + The new code is first, and your code is second. + + The right fix here is to just add a #undef COLORLS_NEW before #ifdef and then delete the old changes: + [source,shell] + .... + #undef COLORLS_NEW + #ifdef COLORLS_NEW + #include + .... + save the file. + + The rebase was interrupted, so you have to complete it: + [source,shell] + .... + % git add ls.c + % git rebase --continue + .... + + which tells Git that ls.c has been fixed and to continue the rebase operation. + Since there was a conflict, you will get kicked into the editor to update the commit message if necessary. + If the commit message is still accurate, just exit the editor. + + If you get stuck during the rebase, do not panic. + git rebase --abort will take you back to a clean slate. + It is important, though, to start with an unmodified tree. + An aside: The above mentioned `git reflog` comes in handy here, as it will have a list of all the (intermediate) commits that you can view or inspect or cherry-pick. + + For more on this topic, https://www.freecodecamp.org/news/the-ultimate-guide-to-git-merge-and-git-rebase/ provides a rather extensive treatment. + It is a good resource for issues that arise occasionally but are too obscure for this guide. + + ===== Switching to a Different FreeBSD Branch + If you wish to shift from stable/12 to the current branch. + If you have a deep clone, the following will suffice: + [source,shell] + .... + % git checkout main + % # build and install here... + .... + If you have a local branch, though, there are one or two caveats. + First, rebase will rewrite history, so you will likely want to do something to save it. + Second, jumping branches tends to cause more conflicts. + If we pretend the example above was relative to stable/12, then to move to `main`, I'd suggest the following: + [source,shell] + .... + % git checkout no-color-ls + % git checkout -b no-color-ls-stable-12 # create another name for this branch + % git rebase -i stable/12 no-color-ls --onto main + .... + + What the above does is checkout no-color-ls. + Then create a new name for it (no-color-ls-stable-12) in case you need to get back to it. + Then you rebase onto the `main` branch. + This will find all the commits to the current no-color-ls branch (back to where it meets up with the stable/12 branch) and then it will + replay them onto the `main` branch creating a new no-color-ls branch there (which is why I had you create a place holder name). + + [[mfc-with-git]] + === MFC (Merge From Current) Procedures + ==== Summary + + MFC workflow can be summarized as `git cherry-pick -x` plus `git commit --amend` to adjust the commit message. + For multiple commits, use `git rebase -i` to squash them together and edit the commit message. + + ==== Single commit MFC + + [source,shell] + .... + % git checkout stable/X + % git cherry-pick -x $HASH --edit + .... + + For MFC commits, for example a vendor import, you would need to specify one parent for cherry-pick purposes. + Normally, that would be the "first parent" of the branch you are cherry-picking from, so: + + [source,shell] + .... + % git checkout stable/X + % git cherry-pick -x $HASH -m 1 --edit + .... + + If things go wrong, you'll either need to abort the cherry-pick with `git cherry-pick --abort` or fix it up and do a `git cherry-pick --continue`. + + Once the cherry-pick is finished, push with `git push`. + If you get an error due to losing the commit race, use `git pull --rebase` and try to push again. + + ==== MFC to RELENG branch + + MFCs to branches that require approval require a bit more care. + The process is the same for either a typical merge or an exceptional direct commit. + + * Merge or direct commit to the appropriate `stable/X` branch first before merging to the `releng/X.Y` branch. + * Use the hash that's in the `stable/X` branch for the MFC to `releng/X.Y` branch. + * Leave both "cherry picked from" lines in the commit message. + * Be sure to add the `Approved by:` line when you are in the editor. + + [source,shell] + .... + % git checkout releng/13.0 + % git cherry-pick -x $HASH --edit + .... + + If you forget to to add the `Approved by:` line, you can do a `git commit --amend` to edit the commit message before you push the change. + + ==== Multiple commit MFC + + [source,shell] + .... + % git checkout -b tmp-branch stable/X + % for h in $HASH_LIST; do git cherry-pick -x $h; done + % git rebase -i stable/X + # mark each of the commits after the first as 'squash' + # Update the commit message to reflect all elements of commit, if necessary. + # Be sure to retain the "cherry picked from" lines. + % git push freebsd HEAD:stable/X + .... + + If the push fails due to losing the commit race, rebase and try again: + + [source,shell] + .... + % git checkout stable/X + % git pull + % git checkout tmp-branch + % git rebase stable/X + % git push freebsd HEAD:stable/X + .... + + Once the MFC is complete, you can delete the temporary branch: + + [source,shell] + .... + % git checkout stable/X + % git branch -d tmp-branch + .... + + ==== MFC a vendor import + + Vendor imports are the only thing in the tree that creates a merge commit in the `main` branch. + Cherry picking merge commits into stable/XX presents an additional difficulty because there are two parents for a merge commit. + Generally, you'll want the first parent's diff since that's the diff to `main` (though there may be some exceptions). + + [source,shell] + .... + % git cherry-pick -x -m 1 $HASH + .... + is typically what you want. + This will tell cherry-pick to apply the correct diff. + + There are some, hopefully, rare cases where it's possible that the `main` branch was merged backwards by the conversion script. + Should that be the case (and we've not found any yet), you'd change the above to `-m 2` to pickup the proper parent. + Just do: + [source,shell] + .... + % git cherry-pick --abort + % git cherry-pick -x -m 2 $HASH + .... + to do that. The `--abort` will cleanup the failed first attempt. + + ==== Redoing a MFC + + If you do a MFC, and it goes horribly wrong and you want to start over, + then the easiest way is to use `git reset --hard` like so: + [source,shell] + .... + % git reset --hard freebsd/stable/12 + .... + though if you have some revs you want to keep, and others you don't, + using `git rebase -i` is better. + + ==== Considerations when MFCing + + When committing source commits to stable and releng branches, we have the following goals: + + * Clearly mark direct commits distinct from commits that land a change from another branch. + * Avoid introducing known breakage into stable and releng branches. + * Allow developers to determine which changes have or have not been landed from one branch to another. + + With Subversion, we used the following practices to achieve these goals: + + * Using `MFC` and `MFS` tags to mark commits that merged changes from another branch. + * Squashing fixup commits into the main commit when merging a change. + * Recording mergeinfo so that `svn mergeinfo --show-revs` worked. + + With Git, we will need to use different strategies to achieve the same goals. + This document aims to define best practices when merging source commits using Git that achieve these goals. + In general, we aim to use Git's native support to achieve these goals rather than enforcing practices built on Subversion's model. + + One general note: due to technical differences with Git, we will not be using Git "merge commits" (created via `git merge`) in stable or releng branches. + Instead, when this document refers to "merge commits", it means a commit originally made to `main` that is replicated or "landed" to a stable branch, or a commit from a stable branch that is replicated to a releng branch with some variation of `git cherry-pick`. + + ==== Finding Eligible Hashes to MFC + + Git provides some built-in support for this via the `git cherry` and `git log --cherry` commands. + These commands compare the raw diffs of commits (but not other metadata such as log messages) to determine if two commits are identical. + This works well when each commit from `main` is landed as a single commit to a stable branch, but it falls over if multiple commits from `main` are squashed together as a single commit to a stable branch. + The project makes extensive use of `git cherry-pick -x` with all lines preserved to work around these difficulties and is working on automated tooling to take advantage of this. + + ==== Commit message standards + ===== Marking MFCs + + The project has adopted the following practice for marking MFCs: + + * Use the `-x` flag with `git cherry-pick`. This adds a line to the commit message that includes the hash of the original commit when merging. Since it is added by Git directly, committers do not have to manually edit the commit log when merging. + + When merging multiple commits, keep all the "cherry picked from" lines. + + ===== Trim Metadata? + + One area that was not clearly documented with Subversion (or even CVS) is how to format metadata in log messages for MFC commits. + Should it include the metadata from the original commit unchanged, or should it be altered to reflect information about the MFC commit itself? + + Historical practice has varied, though some of the variance is by field. + For example, MFCs that are relevant to a PR generally include the PR field in the MFC so that MFC commits are included in the bug tracker's audit trail. + Other fields are less clear. + For example, Phabricator shows the diff of the last commit tagged to a review, so including Phabricator URLs replaces the main commit with the landed commits. + The list of reviewers is also not clear. + If a reviewer has approved a change to `main`, does that mean they have approved the MFC commit? Is that true if it's identical code only, or with merely trivial rework? It's clearly not true for more extensive reworks. + Even for identical code what if the commit doesn't conflict but introduces an ABI change? A reviewer may have ok'd a commit for `main` due to the ABI breakage but may not approve of merging the same commit as-is. + One will have to use one's best judgment until clear guidelines can be agreed upon. + + For MFCs regulated by re@, new metadata fields are added, such as the Approved by tag for approved commits. + This new metadata will have to be added via `git commit --amend` or similar after the original commit has been reviewed and approved. + We may also want to reserve some metadata fields in MFC commits such as Phabricator URLs for use by re@ in the future. + + Preserving existing metadata provides a very simple workflow. + Developers use `git cherry-pick -x` without having to edit the log message. + + If instead we choose to adjust metadata in MFCs, developers will have to edit log messages explicitly via the use of `git cherry-pick --edit` or `git commit --amend`. + However, as compared to svn, at least the existing commit message can be pre-populated and metadata fields can be added or removed without having to re-enter the entire commit message. + + The bottom line is that developers will likely need to curate their commit message for MFCs that are non-trivial. + + [[vendor-import-git]] + === Vendor Imports with Git + + This section describes the vendor import procedure with Git in detail. + + ==== Branch naming convention + + All vendor branches and tags start with `vendor/`. These branches and tags are visible by default. + + [NOTE] + ==== + This chapter follows the convention that the `freebsd` origin is the origin name for the official FreeBSD Git repository. + If you use a different convention, replace `freebsd` with the name you use instead in the examples below. + ==== + + We will explore an example for updating NetBSD's mtree that is in our tree. + The vendor branch for this is `vendor/NetBSD/mtree`. + + ==== Updating an old vendor import + + The vendor trees usually have only the subset of the third-party software that is appropriate to FreeBSD. + These trees are usually tiny in comparison to the FreeBSD tree. + Git worktrees are thus quite small and fast and the preferred method to use. + Make sure that whatever directory you choose below (the `../mtree`) does not currently exist. + + [source,shell] + .... + % git worktree add ../mtree vendor/NetBSD/mtree + .... + + ==== Update the Sources in the Vendor Branch + + Prepare a full, clean tree of the vendor sources. Import everything but merge only what is needed. + + This example assumes the NetBSD source is checked out from their GitHub mirror in `~/git/NetBSD`. + Note that "upstream" might have added or removed files, so we want to make sure deletions are propagated as well. + package:net/rsync[] is commonly installed, so I'll use that. + + [source,shell] + .... + % cd ../mtree + % rsync -va --del --exclude=".git" ~/git/NetBSD/usr.sbin/mtree/ . + % git add -A + % git status + ... + % git diff --staged + ... + % git commit -m "Vendor import of NetBSD's mtree at 2020-12-11" + [vendor/NetBSD/mtree 8e7aa25fcf1] Vendor import of NetBSD's mtree at 2020-12-11 + 7 files changed, 114 insertions(+), 82 deletions(-) + % git tag -a vendor/NetBSD/mtree/20201211 + .... + + It is critical to verify that the source code you are importing comes from a + trustworthy source. Many open-source projects use cryptographic signatures to + sign code changes, git tags, and/or source code tarballs. Always verify these + signatures, and use isolation mechanisms like jails, chroot, in combination with + a dedicated, non-privileged user account that is different from the one you + regularly use (see the Updating the FreeBSD source tree section below for + more details), until you are confident that the source code you are importing + looks safe. Following the upstream development and occasionally reviewing the + upstream code changes can greatly help in improving code quality and benefit + everyone involved. It is also a good idea to examine the git diff results + before importing them into the vendor area. + + Always run the `git diff` and `git status` commands and examine the results + carefully. When in doubt, it is useful to do a `git annotate` on the vendor + branch or the upstream git repository to see who and why a change was made. + + In the example above we used `-m` to illustrate, but you should compose a + proper message in an editor (using a commit message template). + + It is also important to create an annotated tag using `git tag -a`, otherwise the push will be rejected. + Only annotated tags are allowed to be pushed. + The annotated tag gives you a chance to enter a commit message. + Enter the version you are importing, along with any salient new features or fixes in that version. + + ==== Updating the FreeBSD Copy + + At this point you can push the import to `vendor` into our repo. + + [source,shell] + .... + % git push --follow-tags freebsd vendor/NetBSD/mtree + .... + + `--follow-tags` tells `git push` to also push tags associated with the locally committed revision. + + ==== Updating the FreeBSD source tree + + Now you need to update the mtree in FreeBSD. + The sources live in `contrib/mtree` since it is upstream software. + + From time to time, we may have to make changes to the contributed code to + better satisfy FreeBSD's needs. Whenever possible, please try to contribute + the local changes back to the upstream projects, this helps them to better + support FreeBSD, and also saves your time for future conflict resolutions + when importing updates. + + [source,shell] + .... + % cd ../src + % git subtree merge -P contrib/mtree vendor/NetBSD/mtree + .... + + This would generate a subtree merge commit of `contrib/mtree` against the local `vendor/NetBSD/mtree` branch. + Examine the diff from the merge result and the contents of the + upstream branch. If the merge reduced our local changes to more + trivial difference like blank line or indenting changes, try amending + the local changes to reduce diff against upstream, or try to + contribute the remaining changes back to the upstream project. + If there were conflicts, you would need to fix them before committing. + Include details about the changes being merged in the merge commit message. + + Some open-source software includes a `configure` script that generates files + used to define how the code is built; usually, these generated files like + `config.h` should be updated as part of the import process. When doing + this, always keep in mind that these scripts are executable code running + under the current user's credentials. This process should always be run + in an isolated environment, ideally inside a jail that does not have + network access, and with an unprivileged account; or, at minimum, a + dedicated account that is different from the user account you normally + use for everyday purposes or for pushing to the FreeBSD source + code repository. This minimizes the risk of encountering bugs that can + cause data loss or, in worse cases, maliciously planted code. Using an + isolated jail also prevents the configure scripts from detecting locally + installed software packages, which may lead to unexpected results. + + When testing your changes, run them in a chroot or jailed environment, + or even within a virtual machine first, especially for kernel or library + modifications. This approach helps prevent adverse interactions with + your working environment. It can be particularly beneficial for + changes to libraries that many base system components use, + among others. + + ==== Rebasing your change against latest FreeBSD source tree + + Because the current policy recommends against using merges, if the upstream FreeBSD `main` moved forward before you get a chance to push, you would have to redo the merge. + + Regular `git rebase` or `git pull --rebase` doesn't know how to rebase a merge commit **as a merge commit**, + so instead of that you would have to recreate the commit. + + The following steps should be taken to easily recreate the merge commit as if `git rebase --merge-commits` worked properly: + + * cd to the top of the repo + * Create a side branch `XXX` with the **contents** of the merged tree. + * Update this side branch `XXX` to be merged and up-to-date with FreeBSD's `main` branch. + ** In the worst case scenario, you would still have to resolve merge conflicts, if there was any, but this should be really rare. + ** Resolve conflicts, and collapse multiple commits down to 1 if need be (without conflicts, there's no collapse needed) + * checkout `main` + * create a branch `YYY` (allows for easier unwinding if things go wrong) + * Re-do the subtree merge + * Instead of resolving any conflicts from the subtree merge, checkout the contents of XXX on top of it. + ** The trailing `.` is important, as is being at the top level of the repo. + ** Rather than switching branches to XXX, it splats the contents of XXX on top of the repo + * Commit the results with the prior commit message (the example assumes there's only one merge on the XXX branch). + * Make sure the branches are the same. + * Do whatever review you need, including having others check it out if you think that's needed. + * Push the commit, if you 'lost the race' again, just redo these steps again (see below for a recipe) + * Delete the branches once the commit is upstream. They are throw-a-way. + + The commands one would use, following the above example of mtree, would be like so (the `#` starts a comment to help link commands to descriptions above): + + [source,shell] + .... + % cd ../src # CD to top of tree + % git checkout -b XXX # create new throw-away XXX branch for merge + % git fetch freebsd # Get changes from upstream from upstream + % git merge freebsd/main # Merge the changes and resolve conflicts + % git checkout -b YYY freebsd/main # Create new throw-away YYY branch for redo + % git subtree merge -P contrib/mtree vendor/NetBSD/mtree # Redo subtree merge + % git checkout XXX . # XXX branch has the conflict resolution + % git commit -c XXX~1 # -c reuses the commit message from commit before rebase + % git diff XXX YYY # Should be empty + % git show YYY # Should only have changes you want, and be a merge commit from vendor branch + .... + + Note: if things go wrong with the commit, you can reset the `YYY` branch by reissuing the checkout command that created it with -B to start over: + [source,shell] + .... + % git checkout -B YYY freebsd/main # Create new throw-away YYY branch if starting over is just going to be easier + .... + + ==== Pushing the changes + + Once you think you have a set of changes that are good, you can push it to a fork off GitHub or GitLab for others to review. + One nice thing about Git is that it allows you to publish rough drafts of your work for others to review. + While Phabricator is good for content review, publishing the updated vendor branch and merge commits lets others check the details as they will eventually appear in the repository. + + After review, when you are sure it is a good change, you can push it to the FreeBSD repo: + + [source,shell] + .... + % git push freebsd YYY:main # put the commit on upstream's 'main' branch + % git branch -D XXX # Throw away the throw-a-way branches. + % git branch -D YYY + .... + + Note: I used `XXX` and `YYY` to make it obvious they are terrible names and should not leave your machine. + If you use such names for other work, then you'll need to pick different names, or risk losing the other work. + There is nothing magic about these names. + Upstream will not allow you to push them, but never the less, please pay attention to the exact commands above. + Some commands use syntax that differs only slightly from typical uses and that different behavior is critical to this recipe working. + + ==== How to redo things if need be + + If you've tried to do the push in the previous section and it fails, then you should do the following to 'redo' things. + This sequence keeps the commit with the commit message always at XXX~1 to make committing easier. + + [source,shell] + .... + % git checkout -B XXX YYY # recreate that throw-away-branch XXX and switch to it + % git merge freebsd/main # Merge the changes and resolve conflicts + % git checkout -B YYY freebsd/main # Recreate new throw-away YYY branch for redo + % git subtree merge -P contrib/mtree vendor/NetBSD/mtree # Redo subtree merge + % git checkout XXX . # XXX branch has the conflict resolution + % git commit -c XXX~1 # -c reuses the commit message from commit before rebase + .... + + Then go check it out as above and push as above when ready. + + === Creating a new vendor branch + + There are a number of ways to create a new vendor branch. + The recommended way is to create a new repository and then merge that with FreeBSD. + If one is importing `glorbnitz` into the FreeBSD tree, release 3.1415. + For the sake of simplicity, we will not trim this release. + It is a simple user command that puts the nitz device into different magical glorb states and is small enough trimming will not save much. + + ==== Create the repo + + [source,shell] + .... + % cd /some/where + % mkdir glorbnitz + % cd glorbnitz + % git init + % git checkout -b vendor/glorbnitz + .... + + At this point, you have a new repo, where all new commits will go on the `vendor/glorbnitz` branch. + + Git experts can also do this right in their FreeBSD clone, using `git checkout --orphan vendor/glorbnitz` if they are more comfortable with that. + + ==== Copy the sources in + + Since this is a new import, you can just cp the sources in, or use tar or even rsync as shown above. + And we will add everything, assuming no dot files. + + [source,shell] + .... + % cp -r ~/glorbnitz/* . + % git add * + .... + + At this point, you should have a pristine copy of glorbnitz ready to commit. + + [source,shell] + .... + % git commit -m "Import GlorbNitz frobnosticator revision 3.1415" + .... + + As above, I used `-m` for simplicity, but you should likely create a commit message that explains what a Glorb is and why you'd use a Nitz to get it. + Not everybody will know so, for your actual commit, you should follow the + crossref:committers-guide[commit-log-message,commit log message] section instead of emulating the brief style used here. + + ==== Now import it into our repository + + Now you need to import the branch into our repository. + + [source,shell] + .... + % cd /path/to/freebsd/repo/src + % git remote add glorbnitz /some/where/glorbnitz + % git fetch glorbnitz vendor/glorbnitz + .... + + Note the vendor/glorbnitz branch is in the repo. At this point the `/some/where/glorbnitz` can be deleted, if you like. + It was only a means to an end. + // perhaps the real treasure was the friends it made along the way... + + ==== Tag and push + + Steps from here on out are much the same as they are in the case of updating a vendor branch, though without the updating the vendor branch step. + + [source,shell] + .... + % git worktree add ../glorbnitz vendor/glorbnitz + % cd ../glorbnitz + % git tag --annotate vendor/glorbnitz/3.1415 + # Make sure the commit is good with "git show" + % git push --follow-tags freebsd vendor/glorbnitz + .... + + By 'good' we mean: + + . All the right files are present + . None of the wrong files are present + . The vendor branch points at something sensible + . The tag looks good, and is annotated + . The commit message for the tag has a quick summary of what's new since the last tag + + ==== Time to finally merge it into the base tree + + [source,shell] + .... + % cd ../src + % git subtree add -P contrib/glorbnitz vendor/glorbnitz + # Make sure the commit is good with "git show" + % git commit --amend # one last sanity check on commit message + % git push freebsd + .... + + Here 'good' means: + + . All the right files, and none of the wrong ones, were merged into contrib/glorbnitz. + . No other changes are in the tree. + . The commit messages look crossref:committers-guide[commit-log-message,good]. It should contain a summary of what's changed since the last merge to the FreeBSD `main` branch and any caveats. + . UPDATING should be updated if there is anything of note, such as user visible changes, important upgrade concerns, etc. + + [NOTE] + ==== + This hasn't connected `glorbnitz` to the build yet. + How so do that is specific to the software being imported and is beyond the scope of this tutorial. + ==== + + ===== Keeping current + + So, time passes. + It's time now to update the tree for the latest changes upstream. + When you checkout `main` make sure that you have no diffs. + It's a lot easier to commit those to a branch (or use `git stash`) before doing the following. + + If you are used to `git pull`, we strongly recommend using the `--ff-only` option, and further setting it as the default option. + Alternatively, `git pull --rebase` is useful if you have changes staged in the `main` branch. + + [source,shell] + .... + % git config --global pull.ff only + .... + + You may need to omit the --global if you want this setting to apply to only this repository. + + [source,shell] + .... + % cd freebsd-src + % git checkout main + % git pull (--ff-only|--rebase) + .... + + There is a common trap, that the combination command `git pull` will try to perform a merge, which would sometimes creates a merge commit that didn't exist before. + This can be harder to recover from. + + The longer form is also recommended. + + [source,shell] + .... + % cd freebsd-src + % git checkout main + % git fetch freebsd + % git merge --ff-only freebsd/main + .... + + These commands reset your tree to the `main` branch, and then update it from where you pulled the tree from originally. + It's important to switch to `main` before doing this so it moves forward. + Now, it's time to move the changes forward: + + [source,shell] + .... + % git rebase -i main working + .... + + This will bring up an interactive screen to change the defaults. + For now, just exit the editor. + Everything should just apply. + If not, then you'll need to resolve the diffs. + https://docs.github.com/en/free-pro-team@latest/github/using-git/resolving-merge-conflicts-after-a-git-rebase[This github document] can help you navigate this process. + + [[git-push-upstream]] + ===== Time to push changes upstream + + First, ensure that the push URL is properly configured for the upstream repository. + + [source,shell] + .... + % git remote set-url --push freebsd ssh://git@gitrepo.freebsd.org/src.git + .... + + Then, verify that user name and email are configured right. + We require that they exactly match the passwd entry in FreeBSD cluster. + + Use + + [source,shell] + .... + freefall% gen-gitconfig.sh + .... + + on freefall.freebsd.org to get a recipe that you can use directly, assuming /usr/local/bin is in the PATH. + + The below command merges the `working` branch into the upstream `main` branch. + It's important that you curate your changes to be just like you want them in the FreeBSD source repo before doing this. + This syntax pushes the `working` branch to `main`, moving the `main` branch forward. + You will only be able to do this if this results in a linear change to `main` (e.g. no merges). + + [source,shell] + .... + % git push freebsd working:main + .... + + If your push is rejected due to losing a commit race, rebase your branch before trying again: + + [source,shell] + .... + % git checkout working + % git fetch freebsd + % git rebase freebsd/main + % git push freebsd working:main + .... + + [[git-push-upstream-alt]] + ===== Time to push changes upstream (alternative) + + Some people find it easier to merge their changes to their local `main` before pushing to the remote repository. + Also, `git arc stage` moves changes from a branch to the local `main` when you need to do a subset of a branch. + The instructions are similar to the prior section: + [source,shell] + .... + % git checkout main + % git merge --ff-only `working` + % git push freebsd + .... + + If you lose the race, then try again with + [source,shell] + .... + % git pull --rebase + % git push freebsd + .... + These commands will fetch the most recent `freebsd/main` and then rebase the local `main` changes on top of that, which is what you want when you lose the commit race. + Note: merging vendor branch commits will not work with this technique. + + ===== Finding the Subversion Revision + + You'll need to make sure that you've fetched the notes (see the crossref:committers-guide[git-mini-daily-use, Daily use]for details). + Once you have these, notes will show up in the git log command like so: + + [source,shell] + .... + % git log + .... + + If you have a specific version in mind, you can use this construct: + + [source,shell] + .... + % git log --grep revision=XXXX + .... + + to find the specific revision. + The hex number after 'commit' is the hash you can use to refer to this commit. + + [[git-faq]] + === Git FAQ + + This section provides a number of targeted answers to questions that are likely to come up often for users and developers. + + [NOTE] + ==== + We use the common convention of having the origin for the FreeBSD repository being 'freebsd' rather than the default 'origin' to allow + people to use that for their own development and to minimize "whoops" pushes to the wrong repository. + ==== + + ==== Users + + ===== How do I track -current and -stable with only one copy of the repository? + + **Q:** Although disk space is not a huge issue, it's more efficient to use only one copy of the repository. + With SVN mirroring, I could checkout multiple trees from the same repository. + How do I do this with Git? + + **A:** You can use Git worktrees. + There's a number of ways to do this, but the simplest way is to use a clone to track -current, and a worktree to track stable releases. + While using a 'bare repository' has been put forward as a way to cope, it's more complicated and will not be documented here. + + First, you need to clone the FreeBSD repository, shown here cloning into `freebsd-current` to reduce confusion. + $URL is whatever mirror works best for you: + + [source,shell] + .... + % git clone -o freebsd --config remote.freebsd.fetch='+refs/notes/*:refs/notes/*' $URL freebsd-current + .... + + then once that's cloned, you can simply create a worktree from it: + + [source,shell] + .... + % cd freebsd-current + % git worktree add ../freebsd-stable-12 stable/12 + .... + + this will checkout `stable/12` into a directory named `freebsd-stable-12` that's a peer to the `freebsd-current` directory. + Once created, it's updated very similarly to how you might expect: + + [source,shell] + .... + % cd freebsd-current + % git checkout main + % git pull --ff-only + # changes from upstream now local and current tree updated + % cd ../freebsd-stable-12 + % git merge --ff-only freebsd/stable/12 + # now your stable/12 is up to date too + .... + + I recommend using `--ff-only` because it's safer and you avoid accidentally getting into a 'merge nightmare' where you have an extra change in your tree, forcing a complicated merge rather than a simple one. + + Here's https://adventurist.me/posts/00296[a good writeup] that goes into more detail. + + ==== Developers + + ===== Ooops! I committed to `main`, instead of another branch. + + **Q:** From time to time, I goof up and mistakenly commit to the `main` branch. What do I do? + + **A:** First, don't panic. + + Second, don't push. + In fact, you can fix almost anything if you haven't pushed. + All the answers in this section assume no push has happened. + + The following answer assumes you committed to `main` and want to create a branch called `issue`: + + [source,shell] + .... + % git checkout -b issue # Create the 'issue' branch + % git checkout -B main freebsd/main # Reset main to upstream + % git checkout issue # Back to where you were + .... + + ===== Ooops! I committed something to the wrong branch! + + **Q:** I was working on feature on the `wilma` branch, but accidentally committed a change relevant to the `fred` branch in 'wilma'. + What do I do? + + **A:** The answer is similar to the previous one, but with cherry picking. + This assumes there's only one commit on wilma, but will generalize to more complicated situations. + It also assumes that it's the last commit on wilma (hence using wilma in the `git cherry-pick` command), but that too can be generalized. + + [source,shell] + .... + # We're on branch wilma + % git checkout fred # move to fred branch + % git cherry-pick wilma # copy the misplaced commit + % git checkout wilma # go back to wilma branch + % git reset --hard HEAD^ # move what wilma refers to back 1 commit + .... + + If it is not the last commit, you can cherry-pick that one change from wilma onto fred, then use `git rebase -i` to remove the change from wilma. + + [source,shell] + .... + # We're on branch wilma + % git checkout fred # move to fred branch + % git cherry-pick HASH_OF_CHANGE # copy the misplaced commit + % git rebase -i main wilma # drop the cherry-picked change + .... + + **Q:** But what if I want to commit a few changes to `main`, but keep the rest in `wilma` for some reason? + + **A:** The same technique above also works if you are wanting to 'land' parts of the branch you are working on into `main` before the rest of the branch is ready (say you noticed an unrelated typo, or fixed an incidental bug). + You can cherry pick those changes into `main`, then push to the parent repository. + Once you've done that, cleanup couldn't be simpler: just `git rebase -i`. + Git will notice you've done this and skip the common changes automatically (even if you had to change the commit message or tweak the commit slightly). + There's no need to switch back to wilma to adjust it: just rebase! + + **Q:** I want to split off some changes from branch `wilma` into branch `fred` + + **A:** The more general answer would be the same as the previous. + You'd checkout/create the `fred` branch, cherry pick the changes you want from `wilma` one at a time, then rebase `wilma` to remove those changes you cherry picked. + `git rebase -i main wilma` will toss you into an editor, and remove the `pick` lines that correspond to the commits you copied to `fred`. + If all goes well, and there are no conflicts, you're done. + If not, you'll need to resolve the conflicts as you go. + + The other way to do this would be to checkout `wilma` and then create the branch `fred` to point to the same point in the tree. + You can then `git rebase -i` both these branches, selecting the changes you want in `fred` or `wilma` by retaining the pick likes, and deleting the rest from the editor. + Some people would create a tag/branch called `pre-split` before starting in case something goes wrong in the split. + You can undo it with the following sequence: + + [source,shell] + .... + % git checkout pre-split # Go back + % git branch -D fred # delete the fred branch + % git checkout -B wilma # reset the wilma branch + % git branch -d pre-split # Pretend it didn't happen + .... + + The last step is optional. + If you are going to try again to split, you'd omit it. + + **Q:** But I did things as I read along and didn't see your advice at the end to create a branch, and now `fred` and `wilma` are all screwed up. + How do I find what `wilma` was before I started. + I don't know how many times I moved things around. + + **A:** All is not lost. You can figure out it, so long as it hasn't been too long, or too many commits (hundreds). + + So I created a wilma branch and committed a couple of things to it, then decided I wanted to split it into fred and wilma. + Nothing weird happened when I did that, but let's say it did. + The way to look at what you've done is with the `git reflog`: + + [source,shell] + .... + % git reflog + 6ff9c25 (HEAD -> wilma) HEAD@{0}: rebase -i (finish): returning to refs/heads/wilma + 6ff9c25 (HEAD -> wilma) HEAD@{1}: rebase -i (start): checkout main + 869cbd3 HEAD@{2}: rebase -i (start): checkout wilma + a6a5094 (fred) HEAD@{3}: rebase -i (finish): returning to refs/heads/fred + a6a5094 (fred) HEAD@{4}: rebase -i (pick): Encourage contributions + 1ccd109 (freebsd/main, main) HEAD@{5}: rebase -i (start): checkout main + 869cbd3 HEAD@{6}: rebase -i (start): checkout fred + 869cbd3 HEAD@{7}: checkout: moving from wilma to fred + 869cbd3 HEAD@{8}: commit: Encourage contributions + ... + % + .... + + Here we see the changes I've made. + You can use it to figure out where things went wrong. + I'll just point out a few things here. + The first one is that HEAD@{X} is a 'commitish' thing, so you can use that as an argument to a command. + Although if that command commits anything to the repository, the X numbers change. + You can also use the hash (first column). + + Next, 'Encourage contributions' was the last commit I made to `wilma` before I decided to split things up. + You can also see the same hash is there when I created the `fred` branch to do that. + I started by rebasing `fred` and you see the 'start', each step, and the 'finish' for that process. + While we don't need it here, you can figure out exactly what happened. + Fortunately, to fix this, you can follow the prior answer's steps, but with the hash `869cbd3` instead of `pre-split`. + While that seems a bit verbose, it's easy to remember since you're doing one thing at a time. + You can also stack: + + [source,shell] + .... + % git checkout -B wilma 869cbd3 + % git branch -D fred + .... + + and you are ready to try again. + The `checkout -B` with the hash combines checking out and creating a branch for it. + The `-B` instead of `-b` forces the movement of a pre-existing branch. + Either way works, which is what's great (and awful) about Git. + One reason I tend to use `git checkout -B xxxx hash` instead of checking out the hash, and then creating / moving the branch is purely to avoid the slightly distressing message about detached heads: + + [source,shell] + .... + % git checkout 869cbd3 + M faq.md + Note: checking out '869cbd3'. + + You are in 'detached HEAD' state. You can look around, make experimental + changes and commit them, and you can discard any commits you make in this + state without impacting any branches by performing another checkout. + + If you want to create a new branch to retain commits you create, you may + do so (now or later) by using -b with the checkout command again. Example: + + git checkout -b + + HEAD is now at 869cbd3 Encourage contributions + % git checkout -B wilma + .... + + this produces the same effect, but I have to read a lot more and severed heads aren't an image I like to contemplate. + + ===== Ooops! I did a `git pull` and it created a merge commit, what do I do? + + **Q:** I was on autopilot and did a `git pull` for my development tree and that created a merge commit on `main`. + How do I recover? + + **A:** This can happen when you invoke the pull with your development branch checked out. + + Many developers use `git pull --rebase` to avoid this situation. + + Right after the pull, you will have the new merge commit checked out. + Git supports a `HEAD^#` syntax to examine the parents of a merge commit: + + [source,shell] + .... + git log --oneline HEAD^1 # Look at the first parent's commits + git log --oneline HEAD^2 # Look at the second parent's commits + .... + + From those logs, you can easily identify which commit is your development work. + Then you simply reset your branch to the corresponding `HEAD^#`: + + [source,shell] + .... + git reset --hard HEAD^1 + .... + + In addition, a `git pull --rebase` at this stage will rebase your changes to 'main' to the latest 'freebsd/main'. + + **Q:** But I also need to fix my `main` branch. How do I do that? + + **A:** Git keeps track of the remote repository branches in a `freebsd/` namespace. + To fix your `main` branch, just make it point to the remote's `main`: + + [source,shell] + .... + git branch -f main freebsd/main + .... + + There's nothing magical about branches in Git: they are just labels on a graph that are automatically moved forward by making commits. + So the above works because you're just moving a label. + There's no metadata about the branch that needs to be preserved due to this. + + ===== Mixing and matching branches + + **Q:** So I have two branches `worker` and `async` that I'd like to combine into one branch called `feature` + while maintaining the commits in both. + + **A:** This is a job for cherry pick. + + [source,shell] + .... + % git checkout worker + % git checkout -b feature # create a new branch + % git cherry-pick main..async # bring in the changes + .... + + You now have a new branch called `feature`. + This branch combines commits from both branches. + You can further curate it with `git rebase`. + + **Q:** I have a branch called `driver` and I'd like to break it up into `kernel` and `userland` so I can evolve them separately and commit each branch as it becomes ready. + + **A:** This takes a little bit of prep work, but `git rebase` will do the heavy + lifting here. + + [source,shell] + .... + % git checkout driver # Checkout the driver + % git checkout -b kernel # Create kernel branch + % git checkout -b userland # Create userland branch + .... + + Now you have two identical branches. + So, it's time to separate out the commits. + We'll assume first that all the commits in `driver` go into either the `kernel` or the `userland` branch, but not both. + + [source,shell] + .... + % git rebase -i main kernel + .... + + and just include the changes you want (with a 'p' or 'pick' line) and just delete the commits you don't (this sounds scary, but if worse comes to worse, you can throw this all away and start over with the `driver` branch since you've not yet moved it). + + [source,shell] + .... + % git rebase -i main userland + .... + + and do the same thing you did with the `kernel` branch. + + **Q:** Oh great! I followed the above and forgot a commit in the `kernel` branch. + How do I recover? + + **A:** You can use the `driver` branch to find the hash of the commit is missing and + cherry pick it. + + [source,shell] + .... + % git checkout kernel + % git log driver + % git cherry-pick $HASH + .... + + **Q:** OK. I have the same situation as the above, but my commits are all mixed up. + I need parts of one commit to go to one branch and the rest to go to the other. + In fact, I have several. + Your rebase method to select sounds tricky. + + **A:** In this situation, you'd be better off to curate the original branch to separate + out the commits, and then use the above method to split the branch. + + So let's assume that there's just one commit with a clean tree. + You can either use `git rebase` with an `edit` line, or you can use this with the commit on the tip. + The steps are the same either way. + The first thing we need to do is to back up one commit while leaving the changes uncommitted in the tree: + + [source,shell] + .... + % git reset HEAD^ + .... + + Note: Do not, repeat do not, add `--hard` here since that also removes the changes from your tree. + + Now, if you are lucky, the change needing to be split up falls entirely along file lines. + In that case you can just do the usual `git add` for the files in each group than do a `git commit`. + Note: when you do this, you'll lose the commit message when you do the reset, so if you need it for some reason, you should save a copy (though `git log $HASH` can recover it). + + If you are not lucky, you'll need to split apart files. + There's another tool to do that which you can apply one file at a time. + + [source,shell] + .... + git add -i foo/bar.c + .... + + will step through the diffs, prompting you, one at time, whether to include or exclude the hunk. + Once you're done, `git commit` and you'll have the remainder in your tree. + You can run it multiple times as well, and even over multiple files (though I find it easier to do one file at a time + and use the `git rebase -i` to fold the related commits together). + + ===== Joining the FreeBSD GitHub oranization. + + **Q:** How do I join the FreeBSD GitHub organization? + + **A:** Please see https://wiki.freebsd.org/GitHub#Joining_the_Organisation[our GitHub Wiki Info] page for details. + Briefly, all FreeBSD committers may join. + Those who are not committers who request joining will be considered on a case by case basis. + + ==== Cloning and Mirroring + + **Q:** I'd like to mirror the entire Git repository, how do I do that? + + **A:** If all you want to do is mirror, then + + [source,shell] + .... + % git clone --mirror $URL + .... + + will do the trick. + However, there are two disadvantages to this if you want to use it for anything other than a mirror you'll reclone. + + First, this is a 'bare repository' which has the repository database, but no checked out worktree. + This is great for mirroring, but terrible for day to day work. + There's a number of ways around this with `git worktree`: + + [source,shell] + .... + % git clone --mirror https://git.freebsd.org/ports.git ports.git + % cd ports.git + % git worktree add ../ports main + % git worktree add ../quarterly branches/2020Q4 + % cd ../ports + .... + + But if you aren't using your mirror for further local clones, then it's a poor match. + + The second disadvantage is that Git normally rewrites the refs (branch name, tags, etc) from upstream so that your local refs can evolve independently of upstream. + This means that you'll lose changes if you are committing to this repository on anything other than private project branches. + + **Q:** So what can I do instead? + + **A:** Well, you can stuff all of the upstream repository's refs into a private namespace in your local repository. + Git clones everything via a 'refspec' and the default refspec is: + + [source,shell] + .... + fetch = +refs/heads/*:refs/remotes/freebsd/* + .... + + which says just fetch the branch refs. + + However, the FreeBSD repository has a number of other things in it. + To see those, you can add explicit refspecs for each ref namespace, or you can fetch everything. + To setup your repository to do that: + + [source,shell] + .... + git config --add remote.freebsd.fetch '+refs/*:refs/freebsd/*' + .... + + which will put everything in the upstream repository into your local repository's `refs/freebsd/` namespace. + Please note, that this also grabs all the unconverted vendor branches and the number of refs associated with them is quite large. + + You'll need to refer to these 'refs' with their full name because they aren't in and of Git's regular namespaces. + + [source,shell] + .... + git log refs/freebsd/vendor/zlib/1.2.10 + .... + + would look at the log for the vendor branch for zlib starting at 1.2.10. + + === Collaborating with others + + One of the keys to good software development on a project as large as FreeBSD is the ability to collaborate with others before you push your changes to the tree. + The FreeBSD project's Git repositories do not, yet, allow user-created branches to be pushed to the repository, and therefore if you wish to share your changes with others you must use another mechanism, such as a hosted GitLab or GitHub, to share changes in a user-generated branch. + + The following instructions show how to set up a user-generated branch, based on the FreeBSD `main` branch, and push it to GitHub. + + Before you begin, make sure that your local Git repo is up to date and has the + correct origins set crossref:committers-guide[keeping_current,as shown above]. + + [source,shell] +-```` ++.... + % git remote -v + freebsd https://git.freebsd.org/src.git (fetch) + freebsd ssh://git@gitrepo.freebsd.org/src.git (push) +-```` ++.... + + The first step is to create a fork of https://github.com/freebsd/freebsd-src[FreeBSD] on GitHub following these https://docs.github.com/en/github/getting-started-with-github/fork-a-repo[guidelines]. + The destination of the fork should be your own, personal, GitHub account (gvnn3 in my case). + + Now add a remote on your local system that points to your fork: + [source,shell] + .... + % git remote add github git@github.com:gvnn3/freebsd-src.git + % git remote -v + github git@github.com:gvnn3/freebsd-src.git (fetch) + github git@github.com:gvnn3/freebsd-src.git (push) + freebsd https://git.freebsd.org/src.git (fetch) + freebsd ssh://git@gitrepo.freebsd.org/src.git (push) + .... + With this in place you can create a branch + crossref:committers-guide[keeping_a_local_branch,as shown above]. + + [source,shell] + .... + % git checkout -b gnn-pr2001-fix + .... + + Make whatever modifications you wish in your branch. Build, test, and once you're ready to collaborate with others it's time to push your changes into your hosted branch. + Before you can push you'll have to set the appropriate upstream, as Git will tell you the first time you try to push to your +github+ remote: + + [source,shell] + .... + % git push github + fatal: The current branch gnn-pr2001-fix has no upstream branch. + To push the current branch and set the remote as upstream, use + + git push --set-upstream github gnn-pr2001-fix + .... + + Setting the push as +git+ advises allows it to succeed: + + [source,shell] + .... + % git push --set-upstream github gnn-feature + Enumerating objects: 20486, done. + Counting objects: 100% (20486/20486), done. + Delta compression using up to 8 threads + Compressing objects: 100% (12202/12202), done. + Writing objects: 100% (20180/20180), 56.25 MiB | 13.15 MiB/s, done. + Total 20180 (delta 11316), reused 12972 (delta 7770), pack-reused 0 + remote: Resolving deltas: 100% (11316/11316), completed with 247 local objects. + remote: + remote: Create a pull request for 'gnn-feature' on GitHub by visiting: + remote: https://github.com/gvnn3/freebsd-src/pull/new/gnn-feature + remote: + To github.com:gvnn3/freebsd-src.git + * [new branch] gnn-feature -> gnn-feature + Branch 'gnn-feature' set up to track remote branch 'gnn-feature' from 'github'. + .... + + Subsequent changes to the same branch will push correctly by default: + + [source,shell] + .... + % git push + Enumerating objects: 4, done. + Counting objects: 100% (4/4), done. + Delta compression using up to 8 threads + Compressing objects: 100% (2/2), done. + Writing objects: 100% (3/3), 314 bytes | 1024 bytes/s, done. + Total 3 (delta 1), reused 1 (delta 0), pack-reused 0 + remote: Resolving deltas: 100% (1/1), completed with 1 local object. + To github.com:gvnn3/freebsd-src.git + 9e5243d7b659..cf6aeb8d7dda gnn-feature -> gnn-feature + .... + + At this point your work is now in your branch on +GitHub+ and you can + share the link with other collaborators. + + [[github-pull-land]] + === Landing a github pull request + This section documents how to land a GitHub pull request that's submitted against the FreeBSD Git mirrors at GitHub. + While this is not an official way to submit patches at this time, sometimes good fixes come in this way and it is easiest just to bring them into a committer's tree and have them pushed into the FreeBSD's tree from there. + Similar steps can be used to pull branches from other repositories and land those. + When committing pull requests from others, one should take extra care to examine all the changes to ensure they are exactly as represented. + + Before beginning, make sure that the local Git repo is up to date and has the + correct origins set crossref:committers-guide[keeping_current,as shown above]. + In addition, make sure to have the following origins: + [source,shell] + .... + % git remote -v + freebsd https://git.freebsd.org/src.git (fetch) + freebsd ssh://git@gitrepo.freebsd.org/src.git (push) + github https://github.com/freebsd/freebsd-src (fetch) + github https://github.com/freebsd/freebsd-src (fetch) + .... + Often pull requests are simple: requests that contain only a single commit. + In this case, a streamlined approach may be used, though the approach in the prior section will also work. + Here, a branch is created, the change is cherry picked, the commit message adjusted, and sanity-checked before being pushed. + The branch `staging` is used in this example but it can be any name. + This technique works for any number of commits in the pull request, especially when the changes apply cleanly to the FreeBSD tree. + However, when there's multiple commits, especially when minor adjustments are needed, `git rebase -i` works better than `git cherry-pick`. + Briefly, these commands create a branch; cherry-picks the changes from the pull request; tests it; adjusts the commit messages; and fast forward merges it back to `main`. + The PR number is `$PR` below. + When adjusting the message, add `Pull Request: https://github.com/freebsd-src/pull/$PR`. + All pull requests committed to the FreeBSD repository should be reviewed by at least one person. + This need not be the person committing it, but in that case the person committing it should trust the other reviewers competence to review the commit. + Committers that do a code review of pull requests before pushing them into the repo should add a `Reviewed by:` line to the commit, because in this case it is not implicit. + Add anybody that reviews and approves the commit on github to `Reviewed by:` as well. + As always, care should be taken to ensure the change does what it is supposed to, and that no malicious code is present. + [NOTE] + ====== + In addition, please check to make sure that the pull request author name is not anonymous. + Github's web editing interface generates names like: + [source,shell] + .... + Author: github-user <38923459+github-user@users.noreply.github.com> + .... + A polite request to the author for a better name and/or email should be made. + Extra care should be taken to ensure no style issue or malicious code is introduced. + ====== + + [source,shell] + .... + % git fetch github pull/$PR/head:staging + % git rebase -i main staging # to move the staging branch forward, adjust commit message here + + % git checkout main + % git pull --ff-only # to get the latest if time has passed + % git checkout main + % git merge --ff-only staging + + % git push freebsd --push-option=confirm-author + .... + + [.procedure] + ==== + For complicated pull requests that have multiple commits with conflicts, follow the following outline. + + . checkout the pull request `git checkout github/pull/XXX` + . create a branch to rebase `git checkout -b staging` + . rebase the `staging` branch to the latest `main` with `git rebase -i main staging` + . resolve conflicts and do whatever testing is needed + . fast forward the `staging` branch into `main` as above + . final sanity check of changes to make sure all is well + . push to FreeBSD's Git repository. + + This will also work when bringing branches developed elsewhere into the local tree for committing. + ==== + Once finished with the pull request, close it using GitHub's web interface. + It is worth noting that if your `github` origin uses `https://`, the only step you'll need a GitHub account for is closing the pull request. + + [[vcs-history]] + == Version Control History + + The project has moved to crossref:committers-guide[git-primer,git]. + + The FreeBSD source repository switched from CVS to Subversion on May 31st, 2008. + The first real SVN commit is __r179447__. + The source repository switched from Subversion to Git on December 23rd, 2020. + The last real svn commit is __r368820__. + The first real git commit hash is __5ef5f51d2bef80b0ede9b10ad5b0e9440b60518c__. + + The FreeBSD `doc/www` repository switched from CVS to Subversion on May 19th, 2012. + The first real SVN commit is __r38821__. + The documentation repository switched from Subversion to Git on December 8th, 2020. + The last SVN commit is __r54737__. + The first real git commit hash is __3be01a475855e7511ad755b2defd2e0da5d58bbe__. + + The FreeBSD `ports` repository switched from CVS to Subversion on July 14th, 2012. + The first real SVN commit is __r300894__. + The ports repository switched from Subversion to Git on April 6, 2021. + The last SVN commit is __r569609__ + The first real git commit hash is __ed8d3eda309dd863fb66e04bccaa513eee255cbf__. + + [[conventions]] + == Setup, Conventions, and Traditions + + There are a number of things to do as a new developer. + The first set of steps is specific to committers only. + These steps must be done by a mentor for those who are not committers. + + [[conventions-committers]] + === For New Committers + + Those who have been given commit rights to the FreeBSD repositories must follow these steps. + + * Get mentor approval before committing each of these changes! + * All [.filename]#src# commits go to FreeBSD-CURRENT first before being merged to FreeBSD-STABLE. The FreeBSD-STABLE branch must maintain ABI and API compatibility with earlier versions of that branch. Do not merge changes that break this compatibility. + + [[commit-steps]] + [.procedure] + ==== + *Steps for New Committers* + + . Add an Author Entity + + + [.filename]#doc/shared/authors.adoc# - Add an author entity. Later steps depend on this entity, and missing this step will cause the [.filename]#doc/# build to fail. This is a relatively easy task, but remains a good first test of version control skills. + . Update the List of Developers and Contributors + + + [.filename]#doc/shared/contrib-committers.adoc# - Add an entry, which will then appear in the "Developers" section of the extref:{contributors}[Contributors List, staff-committers]. Entries are sorted by last name. + + + [.filename]#doc/shared/contrib-additional.adoc# - _Remove_ the entry. Entries are sorted by first name. + . Add a News Item + + + [.filename]#doc/website/data/en/news/news.toml# - Add an entry. Look for the other entries that announce new committers and follow the format. Use the date from the commit bit approval email. + . Add a PGP Key + + + `{des}` has written a shell script ([.filename]#doc/documentation/tools/addkey.sh#) to make this easier. See the https://cgit.freebsd.org/doc/plain/documentation/static/pgpkeys/README[README] file for more information. + + + Use [.filename]#doc/documentation/tools/checkkey.sh# to verify that keys meet minimal best-practices standards. + + + After adding and checking a key, add both updated files to source control and then commit them. Entries in this file are sorted by last name. + + + [NOTE] + ====== + It is very important to have a current PGP/GnuPG key in the repository. The key may be required for positive identification of a committer. For example, the `{admins}` might need it for account recovery. A complete keyring of `FreeBSD.org` users is available for download from link:https://docs.FreeBSD.org/pgpkeys/pgpkeys.txt[https://docs.FreeBSD.org/pgpkeys/pgpkeys.txt]. + ====== + . Update Mentor and Mentee Information + + + [.filename]#src/share/misc/committers-.dot# - Add an entry to the current committers section, where _repository_ is `doc`, `ports`, or `src`, depending on the commit privileges granted. + + + Add an entry for each additional mentor/mentee relationship in the bottom section. + . Generate a Kerberos Password + + + See crossref:committers-guide[kerberos-ldap, Kerberos and LDAP web Password for FreeBSD Cluster] to generate or set a Kerberos account for use with other FreeBSD services like the link:https://bugs.freebsd.org/bugzilla/[bug-tracking database] (you get a bug-tracking account as part of that step). + . Optional: Enable Wiki Account + + + link:https://wiki.freebsd.org[FreeBSD Wiki] Account - A wiki account allows sharing projects and ideas. + Those who do not yet have an account can follow instructions on the link:https://wiki.freebsd.org/Wiki/About[Wiki/About page] to obtain one. + Contact mailto:wiki-admin@FreeBSD.org[wiki-admin@FreeBSD.org] if you need help with your Wiki account. + . Optional: Update Wiki Information + + + Wiki Information - After gaining access to the wiki, some people add entries to the https://wiki.freebsd.org/HowWeGotHere[How We Got Here], https://wiki.freebsd.org/IRC/Nicknames[IRC Nicks], https://wiki.freebsd.org/Community/Dogs[Dogs of FreeBSD], and or https://wiki.freebsd.org/Community/Cats[Cats of FreeBSD] pages. + . Optional: Update Ports with Personal Information + + + [.filename]#ports/astro/xearth/files/freebsd.committers.markers# and [.filename]#src/usr.bin/calendar/calendars/calendar.freebsd# - Some people add entries for themselves to these files to show where they are located or the date of their birthday. + . Optional: Prevent Duplicate Mailings + + + Subscribers to {dev-commits-doc-all}, {dev-commits-ports-all} or {dev-commits-src-all} might wish to unsubscribe to avoid receiving duplicate copies of commit messages and followups. + ==== + + [[conventions-everyone]] + === For Everyone + + [[conventions-everyone-steps]] + [.procedure] + ==== + . Introduce yourself to the other developers, otherwise no one will have any idea who you are or what you are working on. The introduction need not be a comprehensive biography, just write a paragraph or two about who you are, what you plan to be working on as a developer in FreeBSD, and who will be your mentor. Email this to the {developers-name} and you will be on your way! + . Log into `freefall.FreeBSD.org` and create a [.filename]#/var/forward/user# (where _user_ is your username) file containing the e-mail address where you want mail addressed to _yourusername_@FreeBSD.org to be forwarded. This includes all of the commit messages as well as any other mail addressed to the {committers-name} and the {developers-name}. Really large mailboxes which have taken up permanent residence on `freefall` may get truncated without warning if space needs to be freed, so forward it or save it elsewhere. + + + [NOTE] + ====== + If your e-mail system uses SPF with strict rules, you should exclude `mx2.FreeBSD.org` from SPF checks. + ====== + + + Due to the severe load dealing with SPAM places on the central mail servers that do the mailing list processing, the front-end server does do some basic checks and will drop some messages based on these checks. At the moment proper DNS information for the connecting host is the only check in place but that may change. Some people blame these checks for bouncing valid email. To have these checks turned off for your email, create a file named [.filename]#~/.spam_lover# on `freefall.FreeBSD.org`. + + + [NOTE] + ====== + Those who are developers but not committers will not be subscribed to the committers or developers mailing lists. The subscriptions are derived from the access rights. + ====== + ==== + + [[smtp-setup]] + ==== SMTP Access Setup + + For those willing to send e-mail messages through the FreeBSD.org infrastructure, follow the instructions below: + + [.procedure] + ==== + . Point your mail client at `smtp.FreeBSD.org:587`. + . Enable STARTTLS. + . Ensure your `From:` address is set to `_yourusername_@FreeBSD.org`. + . For authentication, you can use your FreeBSD Kerberos username and password + (see crossref:committers-guide[kerberos-ldap, Kerberos and LDAP web Password for FreeBSD Cluster]). The `_yourusername_/mail` principal is preferred, as it is only valid for authenticating to mail resources. + + + [NOTE] + ====== + Do not include `@FreeBSD.org` when entering in your username. + ====== + + + .Additional Notes + [NOTE] + ====== + * Will only accept mail from `_yourusername_@FreeBSD.org`. If you are authenticated as one user, you are not permitted to send mail from another. + * A header will be appended with the SASL username: (`Authenticated sender: _username_`). + * Host has various rate limits in place to cut down on brute force attempts. + ====== + ==== + + [[smtp-setup-local-mta]] + ===== Using a Local MTA to Forward Emails to the FreeBSD.org SMTP Service + + It is also possible to use a local MTA to forward locally sent emails to the FreeBSD.org SMTP servers. + + [[smtp-setup-local-postfix]] + .Using Postfix + [example] + ==== + + To tell a local Postfix instance that anything from `_yourusername_@FreeBSD.org` should be forwarded to the FreeBSD.org servers, add this to your [.filename]#main.cf#: + + [.programlisting] + .... + sender_dependent_relayhost_maps = hash:/usr/local/etc/postfix/relayhost_maps + smtp_sasl_auth_enable = yes + smtp_sasl_security_options = noanonymous + smtp_sasl_password_maps = hash:/usr/local/etc/postfix/sasl_passwd + smtp_use_tls = yes + .... + + Create [.filename]#/usr/local/etc/postfix/relayhost_maps# with the following content: + + [.programlisting] + .... + yourusername@FreeBSD.org [smtp.freebsd.org]:587 + .... + + Create [.filename]#/usr/local/etc/postfix/sasl_passwd# with the following content: + + [.programlisting] + .... + [smtp.freebsd.org]:587 yourusername:yourpassword + .... + + If the email server is used by other people, you may want to prevent them from sending e-mails from your address. To achieve this, add this to your [.filename]#main.cf#: + + [.programlisting] + .... + smtpd_sender_login_maps = hash:/usr/local/etc/postfix/sender_login_maps + smtpd_sender_restrictions = reject_known_sender_login_mismatch + .... + + Create [.filename]#/usr/local/etc/postfix/sender_login_maps# with the following content: + + [.programlisting] + .... + yourusername@FreeBSD.org yourlocalusername + .... + + Where _yourlocalusername_ is the SASL username used to connect to the local instance of Postfix. + ==== + + [[smtp-setup-local-opensmtpd]] + .Using OpenSMTPD + [example] + ==== + + To tell a local OpenSMTPD instance that anything from `_yourusername_@FreeBSD.org` should be forwarded to the FreeBSD.org servers, add this to your [.filename]#smtpd.conf#: + + [.programlisting] + .... + action "freebsd" relay host smtp+tls://freebsd@smtp.freebsd.org:587 auth + match from any auth yourlocalusername mail-from "_yourusername_@freebsd.org" for any action "freebsd" + .... + + Where _yourlocalusername_ is the SASL username used to connect to the local instance of OpenSMTPD. + + Create [.filename]#/usr/local/etc/mail/secrets# with the following content: + + [.programlisting] + .... + freebsd yourusername:yourpassword + .... + + ==== + [[smtp-setup-local-exim]] + .Using Exim + [example] + ==== + + To direct a local Exim instance to forward all mail from `_example_@FreeBSD.org` + to FreeBSD.org servers, add this to Exim [.filename]#configuration#: + + [.programlisting] + .... + Routers section: (at the top of the list): + freebsd_send: + driver = manualroute + domains = !+local_domains + transport = freebsd_smtp + route_data = ${lookup {${lc:$sender_address}} lsearch {/usr/local/etc/exim/freebsd_send}} + + Transport Section: + freebsd_smtp: + driver = smtp + tls_certificate= + tls_privatekey= + tls_require_ciphers = EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH+AESGCM:EECDH:EDH+AESGCM:EDH+aRSA:HIGH:!MEDIUM:!LOW:!aNULL:!eNULL:!LOW:!RC4:!MD5:!EXP:!PSK:!SRP:!DSS + dkim_domain = + dkim_selector = + dkim_private_key= + dnssec_request_domains = * + hosts_require_auth = smtp.freebsd.org + + Authenticators: + freebsd_plain: + driver = plaintext + public_name = PLAIN + client_send = ^example/mail^examplePassword + client_condition = ${if eq{$host}{smtp.freebsd.org}} + .... + + Create [.filename]#/usr/local/etc/exim/freebsd_send# with the following content: + + [.programlisting] + .... + example@freebsd.org:smtp.freebsd.org::587 + .... + + ==== + + [[mentors]] + === Mentors + + All new developers have a mentor assigned to them for the first few months. + A mentor is responsible for teaching the mentee the rules and conventions of the project and guiding their first steps in the developer community. + The mentor is also personally responsible for the mentee's actions during this initial period. + + For committers: do not commit anything without first getting mentor approval. + Document that approval with an `Approved by:` line in the commit message. + + When the mentor decides that a mentee has learned the ropes and is ready to commit on their own, the mentor announces it with a commit to [.filename]#mentors#. + This file is in the [.filename]#admin# orphan branch of each repository. + Detailed information on how to access these branches can be found in + crossref:committers-guide[admin-branch, "admin" branch]. + + [[pre-commit-review]] + == Pre-Commit Review + + Code review is one way to increase the quality of software. + The following guidelines apply to commits to the `main` (-CURRENT) branch of the `src` repository. + Other branches and the `ports` and `docs` trees have their own review policies, but these guidelines generally apply to commits requiring review: + + * All non-trivial changes should be reviewed before they are committed to the repository. + * Reviews may be conducted by email, in Bugzilla, in Phabricator, or by another mechanism. Where possible, reviews should be public. + * The developer responsible for a code change is also responsible for making all necessary review-related changes. + * Code review can be an iterative process, which continues until the patch is ready to be committed. Specifically, once a patch is sent out for review, it should receive an explicit "looks good" before it is committed. So long as it is explicit, this can take whatever form makes sense for the review method. + * Timeouts are not a substitute for review. + + Sometimes code reviews will take longer than you would hope for, especially for larger features. Accepted ways to speed up review times for your patches are: + + * Review other people's patches. If you help out, everybody will be more willing to do the same for you; goodwill is our currency. + * Ping the patch. If it is urgent, provide reasons why it is important to you to get this patch landed and ping it every couple of days. If it is not urgent, the common courtesy ping rate is one week. Remember that you are asking for valuable time from other professional developers. + * Ask for help on mailing lists, IRC, etc. Others may be able to either help you directly, or suggest a reviewer. + * Split your patch into multiple smaller patches that build on each other. The smaller your patch, the higher the probability that somebody will take a quick look at it. + + + When making large changes, it is helpful to keep this in mind from the beginning of the effort as breaking large changes into smaller ones is often difficult after the fact. + + Developers should participate in code reviews as both reviewers and reviewees. + If someone is kind enough to review your code, you should return the favor for someone else. + Note that while anyone is welcome to review and give feedback on a patch, only an appropriate subject-matter expert can approve a change. + This will usually be a committer who works with the code in question on a regular basis. + + In some cases, no subject-matter expert may be available. + In those cases, a review by an experienced developer is sufficient when coupled with appropriate testing. + + [[commit-log-message]] + == Commit Log Messages + + This section contains some suggestions and traditions for how commit logs are formatted. + + === Why are commit messages important? + + When you commit a change in Git, Subversion, or another version control system (VCS), you're prompted to write some text describing the commit -- a commit message. + How important is this commit message? Should you spend some significant effort writing it? Does it really matter if you write simply `fixed a bug`? + + Most projects have more than one developer and last for some length of time. + Commit messages are a very important method of communicating with other developers, in the present and for the future. + + FreeBSD has hundreds of active developers and hundreds of thousands of commits spanning decades of history. + Over that time the developer community has learned how valuable good commit messages are; sometimes these are hard-learned lessons. + + Commit messages serve at least three purposes: + + * Communicating with other developers + + + FreeBSD commits generate email to various mailing lists. + These include the commit message along with a copy of the patch itself. + Commit messages are also viewed through commands like git log. + These serve to make other developers aware of changes that are ongoing; that other developer may want to test the change, may have an interest in the topic and will want to review in more detail, or may have their own projects underway that would benefit from interaction. + + * Making Changes Discoverable + + + In a large project with a long history it may be difficult to find changes of interest when investigating an issue or change in behaviour. + Verbose, detailed commit messages allow searches for changes that might be relevant. + For example, `git log --since 1year --grep 'USB timeout'`. + + * Providing historical documentation + + + Commit messages serve to document changes for future developers, perhaps years or decades later. + This future developer may even be you, the original author. + A change that seems obvious today may be decidedly not so much later on. + + The `git blame` command annotates each line of a source file with the change (hash and subject line) that brought it in. + + Having established the importance, here are elements of a good FreeBSD commit message: + + === Start with a subject line + + Commit messages should start with a single-line subject that briefly summarizes the change. + The subject should, by itself, allow the reader to quickly determine if the change is of interest or not. + + === Keep subject lines short + + The subject line should be as short as possible while still retaining the required information. + This is to make browsing Git log more efficient, and so that git log --oneline can display the short hash and subject on a single 80-column line. + A good rule of thumb is to stay below 67 characters, and aim for about 50 or fewer if possible. + + === Prefix the subject line with a component, if applicable + + If the change relates to a specific component the subject line may be prefixed with that component name and a colon (:). + If applicable, try to use the same prefix used in previous commits to the same files. + + ✓ `foo: Add -k option to keep temporary data` + + Include the prefix in the 67-character limit suggested above, so that `git log --oneline` avoids wrapping. + + === Capitalize the first letter of the subject + + Capitalize the first letter of the subject itself. + The prefix, if any, is not capitalized unless necessary (e.g., `USB:` is capitalized). + + === Do not end the subject line with punctuation + + Do not end with a period or other punctuation. + In this regard the subject line is like a newspaper headline. + + === Separate the subject and body with a blank line + + Separate the body from the subject with a blank line. + + Some trivial commits do not require a body, and will have only a subject. + + ✓ `ls: Fix typo in usage text` + + === Limit messages to 72 columns + + `git log` and `git format-patch` indent the commit message by four spaces. + Wrapping at 72 columns provides a matching margin on the right edge. + Limiting messages to 72 characters also keeps the commit message in formatted patches below RFC 2822's suggested email line length limit of 78 characters. + This limit works well with a variety of tools that may render commit messages; line wrapping might be inconsistent with longer line length. + + === Use the present tense, imperative mood + + This facilitates short subject lines and provides consistency, including with automatically generated commit messages (e.g., as generated by git revert). + This is important when reading a list of commit subjects. + Think of the subject as finishing the sentence "when applied, this change will ...". + + ✓ `foo: Implement the -k (keep) option` + + ✗ `foo: Implemented the -k option` + + ✗ `This change implements the -k option in foo` + + ✗ `-k option added` + + === Focus on what and why, not how + + Explain what the change accomplishes and why it is being done, rather than how. + + Do not assume that the reader is familiar with the issue. + Explain the background and motivation for the change. + Include benchmark data if you have it. + + If there are limitations or incomplete aspects of the change, describe them in the commit message. + + === Consider whether parts of the commit message could be code comments instead + + Sometimes while writing a commit message you may find yourself writing a sentence or two explaining some tricky or confusing aspect of the change. When this happens consider whether it would be valuable to have that explanation as a comment in the code itself. + + === Write commit messages for your future self + + While writing the commit message for a change you have all of the context in mind - what prompted the change, alternate approaches that were considered and rejected, limitations of the change, and so on. + Imagine yourself revisiting the change a year or two in the future, and write the commit message in a way that would provide that necessary context. + + === Commit messages should stand alone + + You may include references to mailing list postings, benchmark result web sites, or code review links. + However, the commit message should contain all of the relevant information in case these references are no longer available in the future. + + Similarly, a commit may refer to a previous commit, for example in the case of a bug fix or revert. + In addition to the commit identifier (revision or hash), include the subject line from the referenced commit (or another suitable brief reference). + With each VCS migration (from CVS to Subversion to Git) revision identifiers from previous systems may become difficult to follow. + + === Include appropriate metadata in a footer + + As well as including an informative message with each commit, some additional information may be needed. + + This information consists of one or more lines containing the key word or phrase, a colon, tabs for formatting, and then the additional information. + + For key words where multiple values make sense (e.g., `PR:` with a comma-separated list of PRs), it is permitted to use the same keyword multiple times to avoid ambiguity or improve readability. + + The key words or phrases are: + + [.informaltable] + [cols="20%,80%", frame="none"] + |=== + + |`PR:` + |The problem report (if any) which is affected (typically, by being closed) by this commit. Multiple PRs may be specified on one line, separated by commas or spaces. + + |`Reported by:` + |The name and e-mail address of the person that reported the issue; for developers, just the username on the FreeBSD cluster. + Typically used when there is no PR, for example if the issue was reported on + a mailing list. + + |`Submitted by:` + + (deprecated) + |This has been deprecated with git; submitted patches should have the author set by using `git commit --author` with a full name and valid email. + + |`Reviewed by:` + a| + The name and e-mail address of the person or people that reviewed the change; for developers, just the username on the FreeBSD cluster. If a patch was submitted to a mailing list for review, and the review was favorable, then just include the list name. If the reviewer is not a member of the project, provide the name, email, and if ports an external role like maintainer: + + Reviewed by a developer: + [source,shell] + .... + Reviewed by: username + .... + + Reviewed by a ports maintainer that is not a developer: + [source,shell] + .... + Reviewed by: Full Name (maintainer) + .... + + |`Tested by:` + |The name and e-mail address of the person or people that tested the change; for developers, just the username on the FreeBSD cluster. + + |`Discussed with:` + |The name and e-mail address of the person or people that contributed to the patch by providing meaningful feedback; for developers, just the username on the FreeBSD cluster. + Typically used to credit those who did not explicitly review, test, or approve the change, but nevertheless contributed to the discussion surrounding the change, which led to improvements and a better understanding of its impact on the FreeBSD project. + + |`Approved by:` + a| + + The name and e-mail address of the person or people that approved the change; for developers, just the username on the FreeBSD cluster. + + There are several cases where approval is customary: + + * while a new committer is under mentorship + * commits to an area of the tree covered by the LOCKS file (src) + * during a release cycle + * committing to a repo where you do not hold a commit bit (e.g. src committer committing to docs) + * committing to a port maintained by someone else + + While under mentorship, get mentor approval before the commit. Enter the mentor's username in this field, and note that they are a mentor: + + [source,shell] + .... + Approved by: username-of-mentor (mentor) + .... + + If a team approved these commits then include the team name followed by the username of the approver in parentheses. For example: + + [source,shell] + .... + Approved by: re (username) + .... + + |`Obtained from:` + |The name of the project (if any) from which the code was obtained. Do not use this line for the name of an individual person. + + |`Fixes:` + |The Git short hash and the title line of a commit that is fixed by this change as returned by `git log -n 1 --pretty=format:'%h ("%s")' GIT-COMMIT-HASH`. + We include the commit title so that the referenced commit can be located even in the case that a future VCS migration invalidates hash references. + + |`MFC after:` + |To receive an e-mail reminder to MFC at a later date, specify the number of days, weeks, or months after which an MFC is planned. + + |`MFC to:` + |If the commit should be merged to a subset of stable branches, specify the branch names. + + |`MFH:` + |If the commit is to be merged into a ports quarterly branch name, specify the quarterly branch. For example `2021Q2`. + + |`Relnotes:` + |If the change is a candidate for inclusion in the release notes for the next release from the branch, set to `yes`. + + |`Security:` + |If the change is related to a security vulnerability or security exposure, include one or more references or a description of the issue. If possible, include a VuXML URL or a CVE ID. + + |`Event:` + |The description for the event where this commit was made. If this is a recurring event, add the year or even the month to it. For example, this could be `FooBSDcon 2019`. The idea behind this line is to put recognition to conferences, gatherings, and other types of meetups and to show that these are useful to have. Please do not use the `Sponsored by:` line for this as that is meant for organizations sponsoring certain features or developers working on them. + + |`Sponsored by:` + |Sponsoring organizations for this change, if any. Separate multiple organizations with commas. If only a portion of the work was sponsored, or different amounts of sponsorship were provided to different authors, please give appropriate credit in parentheses after each sponsor name. For example, `Example.com (alice, code refactoring), Wormulon (bob), Momcorp (cindy)` shows that Alice was sponsored by Example.com to do code refactoring, while Wormulon sponsored Bob's work and Momcorp sponsored Cindy's work. Other authors were either not sponsored or chose not to list sponsorship. + + |`Pull Request:` + |This change was submitted as a pull request or merge request against one of FreeBSD's public read-only Git repositories. + It should include the entire URL to the pull request, as these often act as code reviews for the code. + For example: `https://github.com/freebsd/freebsd-src/pull/745` + + |`Co-authored-by:` + |The name and email address of an additional author of the commit. + GitHub has a detailed description of the Co-authored-by trailer at https://docs.github.com/en/pull-requests/committing-changes-to-your-project/creating-and-editing-commits/creating-a-commit-with-multiple-authors. + + |`Signed-off-by:` + |ID certifies compliance with https://developercertificate.org/ + + |`Differential Revision:` + |The full URL of the Phabricator review. This line __must be the last line__. For example: `https://reviews.freebsd.org/D1708`. + + |=== + + .Commit Log for a Commit Based on a PR + [example] + ==== + + The commit is based on a patch from a PR submitted by John Smith. + The commit message "PR" field is filled. + + [.programlisting] + .... + ... + + PR: 12345 + .... + + The committer sets the author of the patch with `git commit --author "John Smith "`. + + ==== + + .Commit Log for a Commit Needing Review + [example] + ==== + + The virtual memory system is being changed. + After posting patches to the appropriate mailing list (in this case, `freebsd-arch`) and the changes have been approved. + + [.programlisting] + .... + ... + + Reviewed by: -arch + .... + + ==== + + .Commit Log for a Commit Needing Approval + [example] + ==== + + Commit a port, after working with the listed MAINTAINER, who said to go ahead and commit. + + [.programlisting] + .... + ... + + Approved by: abc (maintainer) + .... + + Where _abc_ is the account name of the person who approved. + ==== + + .Commit Log for a Commit Bringing in Code from OpenBSD + [example] + ==== + + Committing some code based on work done in the OpenBSD project. + + [.programlisting] + .... + ... + + Obtained from: OpenBSD + .... + + ==== + + .Commit Log for a Change to FreeBSD-CURRENT with a Planned Commit to FreeBSD-STABLE to Follow at a Later Date. + [example] + ==== + + Committing some code which will be merged from FreeBSD-CURRENT into the FreeBSD-STABLE branch after two weeks. + + [.programlisting] + .... + ... + + MFC after: 2 weeks + .... + + Where _2_ is the number of days, weeks, or months after which an MFC is planned. The _weeks_ option may be `day`, `days`, `week`, `weeks`, `month`, `months`. + ==== + + It is often necessary to combine these. + + Consider the situation where a user has submitted a PR containing code from the NetBSD project. + Looking at the PR, the developer sees it is not an area of the tree they normally work in, so they have the change reviewed by the `arch` mailing list. + Since the change is complex, the developer opts to MFC after one month to allow adequate testing. + + The extra information to include in the commit would look something like + + .Example Combined Commit Log + [example] + ==== + + [.programlisting] + .... + PR: 54321 + Reviewed by: -arch + Obtained from: NetBSD + MFC after: 1 month + Relnotes: yes + .... + + ==== + + [[pref-license]] + == Preferred License for New Files + + The FreeBSD Project's full license policy can be found at link:https://www.FreeBSD.org/internal/software-license/[https://www.FreeBSD.org/internal/software-license]. + The rest of this section is intended to help you get started. + As a rule, when in doubt, ask. + It is much easier to give advice than to fix the source tree. + + The FreeBSD Project suggests and uses this text as the preferred license scheme: + + [.programlisting] + .... + /* + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) [year] [your name] + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * [id for your version control system, if any] + */ + .... + + The FreeBSD project strongly discourages the so-called "advertising clause" in new code. + Due to the large number of contributors to the FreeBSD project, complying with this clause for many commercial vendors has become difficult. + If you have code in the tree with the advertising clause, please consider removing it. + In fact, please consider using the above license for your code. + + The FreeBSD project discourages completely new licenses and variations on the standard licenses. + New licenses require the approval of {core-email} to reside in the `src` repository. + The more different licenses that are used in the tree, the more problems that this causes to those wishing to utilize this code, typically from unintended consequences from a poorly worded license. + + Project policy dictates that code under some non-BSD licenses must be placed only in specific sections of the repository, and in some cases, compilation must be conditional or even disabled by default. + For example, the GENERIC kernel must be compiled under only licenses identical to or substantially similar to the BSD license. + GPL, APSL, CDDL, etc, licensed software must not be compiled into GENERIC. + + Developers are reminded that in open source, getting "open" right is just as important as getting "source" right, as improper handling of intellectual property has serious consequences. + Any questions or concerns should immediately be brought to the attention of the core team. + + [[tracking.license.grants]] + == Keeping Track of Licenses Granted to the FreeBSD Project + + Various software or data exist in the repositories where the FreeBSD project has been granted a special license to be able to use them. + A case in point are the Terminus fonts for use with man:vt[4]. + Here the author Dimitar Zhekov has allowed us to use the "Terminus BSD Console" font under a 2-clause BSD license rather than the regular Open Font License he normally uses. + + It is clearly sensible to keep a record of any such license grants. + To that end, the {core-email} has decided to keep an archive of them. + Whenever the FreeBSD project is granted a special license we require the {core-email} to be notified. + Any developers involved in arranging such a license grant, please send details to the {core-email} including: + + * Contact details for people or organizations granting the special license. + * What files, directories etc. in the repositories are covered by the license grant including the revision numbers where any specially licensed material was committed. + * The date the license comes into effect from. Unless otherwise agreed, this will be the date the license was issued by the authors of the software in question. + * The license text. + * A note of any restrictions, limitations or exceptions that apply specifically to FreeBSD's usage of the licensed material. + * Any other relevant information. + + Once the {core-email} is satisfied that all the necessary details have been gathered and are correct, the secretary will send a PGP-signed acknowledgment of receipt including the license details. + This receipt will be persistently archived and serve as our permanent record of the license grant. + + The license archive should contain only details of license grants; this is not the place for any discussions around licensing or other subjects. + Access to data within the license archive will be available on request to the {core-email}. + + [[spdx.tags]] + == SPDX Tags in the tree + + The project uses https://spdx.dev[SPDX] tags in our source base. + At present, these tags are indented to help automated tools reconstruct license requirements mechanically. + All _SPDX-License-Identifier_ tags in the tree should be considered to be informative. + All files in the FreeBSD source tree with these tags also have a copy of the license which governs use of that file. + In the event of a discrepancy, the verbatim license is controlling. + The project tries to follow the https://spdx.github.io/spdx-spec/v2.2.2/[SPDX Specification, Version 2.2]. + How to mark source files and valid algebraic expressions are found in https://spdx.github.io/spdx-spec/v2.2.2/SPDX-license-expressions/[Annex D] and https://spdx.github.io/spdx-spec/v2.2.2/using-SPDX-short-identifiers-in-source-files/[Annex E]. + The project draws identifiers from SPDX's list of valid https://spdx.org/licenses/[short license identifiers]. + The project uses only the _SPDX-License-Identifier_ tag. + + As of March 2021, approximately 25,000 out of 90,000 files in the tree have been marked. + [[developer.relations]] + == Developer Relations + + When working directly on your own code or on code which is already well established as your responsibility, then there is probably little need to check with other committers before jumping in with a commit. + When working on a bug in an area of the system which is clearly orphaned (and there are a few such areas, to our shame), the same applies. + When modifying parts of the system which are maintained, formally or informally, consider asking for a review just as a developer would have before becoming a committer. + For ports, contact the listed `MAINTAINER` in the [.filename]#Makefile#. + + To determine if an area of the tree is maintained, check the MAINTAINERS file at the root of the tree. + If nobody is listed, scan the revision history to see who has committed changes in the past. + To list the names and email addresses of all commit authors for a given file in the last 2 years and the number of commits each has authored, ordered by descending number of commits, use: + + [source,shell] + ---- + % git -C /path/to/repo shortlog -sne --since="2 years" -- relative/path/to/file + ---- + + If queries go unanswered or the committer otherwise indicates a lack of interest in the area affected, go ahead and commit it. + + [IMPORTANT] + ==== + Avoid sending private emails to maintainers. + Other people might be interested in the conversation, not just the final output. + ==== + + If there is any doubt about a commit for any reason at all, have it reviewed before committing. + Better to have it flamed then and there rather than when it is part of the repository. + If a commit does results in controversy erupting, it may be advisable to consider backing the change out again until the matter is settled. + Remember, with a version control system we can always change it back. + + Do not impugn the intentions of others. + If they see a different solution to a problem, or even a different problem, it is probably not because they are stupid, because they have questionable parentage, or because they are trying to destroy hard work, personal image, or FreeBSD, but basically because they have a different outlook on the world. + Different is good. + + Disagree honestly. + Argue your position from its merits, be honest about any shortcomings it may have, and be open to seeing their solution, or even their vision of the problem, with an open mind. + + Accept correction. + We are all fallible. + When you have made a mistake, apologize and get on with life. + Do not beat up yourself, and certainly do not beat up others for your mistake. + Do not waste time on embarrassment or recrimination, just fix the problem and move on. + + Ask for help. + Seek out (and give) peer reviews. + One of the ways open source software is supposed to excel is in the number of eyeballs applied to it; this does not apply if nobody will review code. + + [[if-in-doubt]] + == If in Doubt... + + When unsure about something, whether it be a technical issue or a project convention be sure to ask. + If you stay silent you will never make progress. + + If it relates to a technical issue ask on the public mailing lists. + Avoid the temptation to email the individual person that knows the answer. + This way everyone will be able to learn from the question and the answer. + + For project specific or administrative questions ask, in order: + + * Your mentor or former mentor. + * An experienced committer on IRC, email, etc. + * Any team with a "hat", as they can give you a definitive answer. + * If still not sure, ask on {developers-name}. + + Once your question is answered, if no one pointed you to documentation that spelled out the answer to your question, document it, as others will have the same question. + + [[bugzilla]] + == Bugzilla + + The FreeBSD Project utilizes Bugzilla for tracking bugs and change requests. + If you commit a fix or suggestion found in the PR database, be sure to close the PR. + It is also considered nice if you take time to close any other PRs associated with your commits. + + Committers with non-``FreeBSD.org`` Bugzilla accounts can have the old account merged with the `FreeBSD.org` account by following these steps: + + [.procedure] + ==== + . Log in using your old account. + . Open new bug. Choose `Services` as the Product, and `Bug Tracker` as the Component. In bug description list accounts you wish to be merged. + . Log in using `FreeBSD.org` account and post comment to newly opened bug to + confirm ownership. See crossref:committers-guide[kerberos-ldap, Kerberos and LDAP web Password for FreeBSD Cluster] for more details on how to generate or set a password for your `FreeBSD.org` account. + . If there are more than two accounts to merge, post comments from each of them. + ==== + + You can find out more about Bugzilla at: + + * extref:{pr-guidelines}[FreeBSD Problem Report Handling Guidelines] + * link:https://www.FreeBSD.org/support/[https://www.FreeBSD.org/support] + + [[phabricator]] + == Phabricator + + The FreeBSD Project utilizes https://reviews.freebsd.org[Phabricator] for code review requests. + See the https://wiki.freebsd.org/Phabricator[Phabricator wiki page] for details. + + Please use the `git arc` command provided by `devel/freebsd-git-devtools` (install the port or package, then type `git help arc` for documentation) to create and update Phabricator reviews. + This will make it easier for others to review and test your patches. + + Committers with non-``FreeBSD.org`` Phabricator accounts can have the old account renamed to the ``FreeBSD.org`` account by following these steps: + + [.procedure] + ==== + . Change your Phabricator account email to your `FreeBSD.org` email. + . Open new bug on our bug tracker using your `FreeBSD.org` account, see + crossref:committers-guide[bugzilla, Bugzilla] for more information. Choose `Services` as the Product, and `Code Review` as the Component. In bug description request that your Phabricator account be renamed, and provide a link to your Phabricator user. For example, `https://reviews.freebsd.org/p/bob_example.com/` + ==== + + [IMPORTANT] + ==== + Phabricator accounts cannot be merged, please do not open a new account. + ==== + + [[people]] + == Who's Who + + Besides the repository meisters, there are other FreeBSD project members and teams whom you will probably get to know in your role as a committer. Briefly, and by no means all-inclusively, these are: + + `{doceng}`:: + doceng is the group responsible for the documentation build infrastructure, approving new documentation committers, and ensuring that the FreeBSD website and documentation on the FTP site is up to date with respect to the Subversion tree. + It is not a conflict resolution body. + The vast majority of documentation related discussion takes place on the {freebsd-doc}. + More details regarding the doceng team can be found in its https://www.FreeBSD.org/internal/doceng/[charter]. + Committers interested in contributing to the documentation should familiarize themselves with the extref:{fdp-primer}[Documentation Project Primer]. + + `{re-members}`:: + These are the members of the `{re}`. + This team is responsible for setting release deadlines and controlling the release process. + During code freezes, the release engineers have final authority on all changes to the system for whichever branch is pending release status. + If there is something you want merged from FreeBSD-CURRENT to FreeBSD-STABLE (whatever values those may have at any given time), these are the people to talk to about it. + + `{so}`:: + `{so-name}` is the link:https://www.FreeBSD.org/security/[FreeBSD Security Officer] and oversees the `{security-officer}`. + + {committers-name}:: + {dev-src-all}, {dev-ports-all} and {dev-doc-all} are the mailing lists that the version control system uses to send commit messages to. + _Never_ send email directly to these lists. + Only send replies to this list when they are short and are directly related to a commit. + + {developers-name}:: + All committers are subscribed to -developers. + This list was created to be a forum for the committers "community" issues. + Examples are Core voting, announcements, etc. + + + The {developers-name} is for the exclusive use of FreeBSD committers. + To develop FreeBSD, committers must have the ability to openly discuss matters that will be resolved before they are publicly announced. + Frank discussions of work in progress are not suitable for open publication and may harm FreeBSD. + + + All FreeBSD committers are expected not to not publish or forward messages from the {developers-name} outside the list membership without permission of all of the authors. + Violators will be removed from the {developers-name}, resulting in a suspension of commit privileges. + Repeated or flagrant violations may result in permanent revocation of commit privileges. + + + This list is _not_ intended as a place for code reviews or for any technical discussion. + In fact using it as such hurts the FreeBSD Project as it gives a sense of a closed list where general decisions affecting all of the FreeBSD using community are made without being "open". + Last, but not least __never, never ever, email the {developers-name} and CC:/BCC: another FreeBSD list__. + Never, ever email another FreeBSD email list and CC:/BCC: the {developers-name}. + Doing so can greatly diminish the benefits of this list. + [[ssh.guide]] + == SSH Quick-Start Guide + + [.procedure] + ==== + . If you do not wish to type your password in every time you use man:ssh[1], and you use keys to authenticate, man:ssh-agent[1] is there for your convenience. If you want to use man:ssh-agent[1], make sure that you run it before running other applications. X users, for example, usually do this from their [.filename]#.xsession# or [.filename]#.xinitrc#. See man:ssh-agent[1] for details. + . Generate a key pair using man:ssh-keygen[1]. The key pair will wind up in your [.filename]#$HOME/.ssh/# directory. + + + [IMPORTANT] + ====== + Only ECDSA, Ed25519 or RSA keys are supported. + ====== + . Send your public key ([.filename]#$HOME/.ssh/id_ecdsa.pub#, [.filename]#$HOME/.ssh/id_ed25519.pub#, or [.filename]#$HOME/.ssh/id_rsa.pub#) to the person setting you up as a committer so it can be put into [.filename]#yourlogin# in [.filename]#/etc/ssh-keys/# on `freefall`. + ==== + + Now man:ssh-add[1] can be used for authentication once per session. + It prompts for the private key's pass phrase, and then stores it in the authentication agent (man:ssh-agent[1]). + Use `ssh-add -d` to remove keys stored in the agent. + + Test with a simple remote command: `ssh freefall.FreeBSD.org ls /usr`. + + For more information, see package:security/openssh-portable[], man:ssh[1], man:ssh-add[1], man:ssh-agent[1], man:ssh-keygen[1], and man:scp[1]. + + For information on adding, changing, or removing man:ssh[1] keys, see https://wiki.freebsd.org/clusteradm/ssh-keys[this article]. + + [[coverity]] + == Coverity(R) Availability for FreeBSD Committers + + All FreeBSD developers can obtain access to Coverity analysis results of all FreeBSD Project software. + All who are interested in obtaining access to the analysis results of the automated Coverity runs, can sign up at http://scan.coverity.com/[Coverity Scan]. + + The FreeBSD wiki includes a mini-guide for developers who are interested in working with the Coverity(R) analysis reports: https://wiki.freebsd.org/CoverityPrevent[https://wiki.freebsd.org/CoverityPrevent]. + Please note that this mini-guide is only readable by FreeBSD developers, so if you cannot access this page, you will have to ask someone to add you to the appropriate Wiki access list. + + Finally, all FreeBSD developers who are going to use Coverity(R) are always encouraged to ask for more details and usage information, by posting any questions to the mailing list of the FreeBSD developers. + + [[rules]] + == The FreeBSD Committers' Big List of Rules + + Everyone involved with the FreeBSD project is expected to abide by the _Code of Conduct_ available from link:https://www.FreeBSD.org/internal/code-of-conduct/[https://www.FreeBSD.org/internal/code-of-conduct]. + As committers, you form the public face of the project, and how you behave has a vital impact on the public perception of it. + This guide expands on the parts of the _Code of Conduct_ specific to committers. + + . Respect other committers. + . Respect other contributors. + . Discuss any significant change _before_ committing. + . Respect existing maintainers (if listed in the `MAINTAINER` field in [.filename]#Makefile# or in [.filename]#MAINTAINER# in the top-level directory). + . Any disputed change must be backed out pending resolution of the dispute if requested by a maintainer. Security related changes may override a maintainer's wishes at the Security Officer's discretion. + . Changes go to FreeBSD-CURRENT before FreeBSD-STABLE unless specifically permitted by the release engineer or unless they are not applicable to FreeBSD-CURRENT. Any non-trivial or non-urgent change which is applicable should also be allowed to sit in FreeBSD-CURRENT for at least 3 days before merging so that it can be given sufficient testing. The release engineer has the same authority over the FreeBSD-STABLE branch as outlined for the maintainer in rule #5. + . Do not fight in public with other committers; it looks bad. + . Respect all code freezes and read the `committers` and `developers` mailing lists in a timely manner so you know when a code freeze is in effect. + . When in doubt on any procedure, ask first! + . Test your changes before committing them. + . Do not commit to contributed software without _explicit_ approval from the respective maintainers. + + As noted, breaking some of these rules can be grounds for suspension or, upon repeated offense, permanent removal of commit privileges. + Individual members of core have the power to temporarily suspend commit privileges until core as a whole has the chance to review the issue. + In case of an "emergency" (a committer doing damage to the repository), a temporary suspension may also be done by the repository meisters. + Only a 2/3 majority of core has the authority to suspend commit privileges for longer than a week or to remove them permanently. + This rule does not exist to set core up as a bunch of cruel dictators who can dispose of committers as casually as empty soda cans, but to give the project a kind of safety fuse. + If someone is out of control, it is important to be able to deal with this immediately rather than be paralyzed by debate. + In all cases, a committer whose privileges are suspended or revoked is entitled to a "hearing" by core, the total duration of the suspension being determined at that time. + A committer whose privileges are suspended may also request a review of the decision after 30 days and every 30 days thereafter (unless the total suspension period is less than 30 days). + A committer whose privileges have been revoked entirely may request a review after a period of 6 months has elapsed. + This review policy is _strictly informal_ and, in all cases, core reserves the right to either act on or disregard requests for review if they feel their original decision to be the right one. + + In all other aspects of project operation, core is a subset of committers and is bound by the __same rules__. + Just because someone is in core this does not mean that they have special dispensation to step outside any of the lines painted here; core's "special powers" only kick in when it acts as a group, not on an individual basis. + As individuals, the core team members are all committers first and core second. + + === Details + + [[respect]] + . Respect other committers. + + + This means that you need to treat other committers as the peer-group developers that they are. + Despite our occasional attempts to prove the contrary, one does not get to be a committer by being stupid and nothing rankles more than being treated that way by one of your peers. + Whether we always feel respect for one another or not (and everyone has off days), we still have to _treat_ other committers with respect at all times, on public forums and in private email. + + + Being able to work together long term is this project's greatest asset, one far more important than any set of changes to the code, and turning arguments about code into issues that affect our long-term ability to work harmoniously together is just not worth the trade-off by any conceivable stretch of the imagination. + + + To comply with this rule, do not send email when you are angry or otherwise behave in a manner which is likely to strike others as needlessly confrontational. + First calm down, then think about how to communicate in the most effective fashion for convincing the other persons that your side of the argument is correct, do not just blow off some steam so you can feel better in the short term at the cost of a long-term flame war. + Not only is this very bad "energy economics", but repeated displays of public aggression which impair our ability to work well together will be dealt with severely by the project leadership and may result in suspension or termination of your commit privileges. + The project leadership will take into account both public and private communications brought before it. + It will not seek the disclosure of private communications, but it will take it into account if it is volunteered by the committers involved in the complaint. + + + All of this is never an option which the project's leadership enjoys in the slightest, but unity comes first. + No amount of code or good advice is worth trading that away. + . Respect other contributors. + + + You were not always a committer. + At one time you were a contributor. + Remember that at all times. + Remember what it was like trying to get help and attention. + Do not forget that your work as a contributor was very important to you. + Remember what it was like. Do not discourage, belittle, or demean contributors. + Treat them with respect. They are our committers in waiting. + They are every bit as important to the project as committers. + Their contributions are as valid and as important as your own. + After all, you made many contributions before you became a committer. + Always remember that. + + + Consider the points raised under crossref:committers-guide[respect,Respect other committers] and apply them also to contributors. + . Discuss any significant change _before_ committing. + + + The repository is not where changes are initially submitted for correctness or argued over, that happens first in the mailing lists or by use of the Phabricator service. + The commit will only happen once something resembling consensus has been reached. + This does not mean that permission is required before correcting every obvious syntax error or manual page misspelling, just that it is good to develop a feel for when a proposed change is not quite such a no-brainer and requires some feedback first. + People really do not mind sweeping changes if the result is something clearly better than what they had before, they just do not like being _surprised_ by those changes. + The very best way of making sure that things are on the right track is to have code reviewed by one or more other committers. + + + When in doubt, ask for review! + . Respect existing maintainers if listed. + + + Many parts of FreeBSD are not "owned" in the sense that any specific individual will jump up and yell if you commit a change to "their" area, but it still pays to check first. +-One convention we use is to put a maintainer line in the [.filename]#Makefile# for any package or subtree which is being actively maintained by one or more people; see extref:{developers-handbook}[Source Tree Guidelines and Policies, policies] for documentation on this. ++One convention we use is to put a maintainer line in the [.filename]#Makefile# for any package or subtree which is being actively maintained by one or more people; see extref:{developers-handbook}policies[Source Tree Guidelines and Policies, policies] for documentation on this. + Where sections of code have several maintainers, commits to affected areas by one maintainer need to be reviewed by at least one other maintainer. + In cases where the "maintainer-ship" of something is not clear, look at the repository logs for the files in question and see if someone has been working recently or predominantly in that area. + . Any disputed change must be backed out pending resolution of the dispute if requested by a maintainer. Security related changes may override a maintainer's wishes at the Security Officer's discretion. + + + This may be hard to swallow in times of conflict (when each side is convinced that they are in the right, of course) but a version control system makes it unnecessary to have an ongoing dispute raging when it is far easier to simply reverse the disputed change, get everyone calmed down again and then try to figure out what is the best way to proceed. + If the change turns out to be the best thing after all, it can be easily brought back. + If it turns out not to be, then the users did not have to live with the bogus change in the tree while everyone was busily debating its merits. + People _very_ rarely call for back-outs in the repository since discussion generally exposes bad or controversial changes before the commit even happens, but on such rare occasions the back-out should be done without argument so that we can get immediately on to the topic of figuring out whether it was bogus or not. + . Changes go to FreeBSD-CURRENT before FreeBSD-STABLE unless specifically permitted by the release engineer or unless they are not applicable to FreeBSD-CURRENT. Any non-trivial or non-urgent change which is applicable should also be allowed to sit in FreeBSD-CURRENT for at least 3 days before merging so that it can be given sufficient testing. The release engineer has the same authority over the FreeBSD-STABLE branch as outlined in rule #5. + + + This is another "do not argue about it" issue since it is the release engineer who is ultimately responsible (and gets beaten up) if a change turns out to be bad. + Please respect this and give the release engineer your full cooperation when it comes to the FreeBSD-STABLE branch. + The management of FreeBSD-STABLE may frequently seem to be overly conservative to the casual observer, but also bear in mind the fact that conservatism is supposed to be the hallmark of FreeBSD-STABLE and different rules apply there than in FreeBSD-CURRENT. + There is also really no point in having FreeBSD-CURRENT be a testing ground if changes are merged over to FreeBSD-STABLE immediately. + Changes need a chance to be tested by the FreeBSD-CURRENT developers, so allow some time to elapse before merging unless the FreeBSD-STABLE fix is critical, time sensitive or so obvious as to make further testing unnecessary (spelling fixes to manual pages, obvious bug/typo fixes, etc.) In other words, apply common sense. + + + Changes to the security branches (for example, `releng/9.3`) must be approved by a member of the `{security-officer}`, or in some cases, by a member of the `{re}`. + . Do not fight in public with other committers; it looks bad. + + + This project has a public image to uphold and that image is very important to all of us, especially if we are to continue to attract new members. + There will be occasions when, despite everyone's very best attempts at self-control, tempers are lost and angry words are exchanged. + The best thing that can be done in such cases is to minimize the effects of this until everyone has cooled back down. + Do not air angry words in public and do not forward private correspondence or other private communications to public mailing lists, mail aliases, instant messaging channels or social media sites. + What people say one-to-one is often much less sugar-coated than what they would say in public, and such communications therefore have no place there - they only serve to inflame an already bad situation. + If the person sending a flame-o-gram at least had the grace to send it privately, then have the grace to keep it private yourself. + If you feel you are being unfairly treated by another developer, and it is causing you anguish, bring the matter up with core rather than taking it public. Core will do its best to play peace makers and get things back to sanity. + In cases where the dispute involves a change to the codebase and the participants do not appear to be reaching an amicable agreement, core may appoint a mutually-agreeable third party to resolve the dispute. + All parties involved must then agree to be bound by the decision reached by this third party. + . Respect all code freezes and read the `committers` and `developers` mailing list on a timely basis so you know when a code freeze is in effect. + + + Committing unapproved changes during a code freeze is a really big mistake and committers are expected to keep up-to-date on what is going on before jumping in after a long absence and committing 10 megabytes worth of accumulated stuff. + People who abuse this on a regular basis will have their commit privileges suspended until they get back from the FreeBSD Happy Reeducation Camp we run in Greenland. + . When in doubt on any procedure, ask first! + + + Many mistakes are made because someone is in a hurry and just assumes they know the right way of doing something. + If you have not done it before, chances are good that you do not actually know the way we do things and really need to ask first or you are going to completely embarrass yourself in public. + There is no shame in asking "how in the heck do I do this?" We already know you are an intelligent person; otherwise, you would not be a committer. + . Test your changes before committing them. + + + If your changes are to the kernel, make sure you can still compile both GENERIC and LINT. + If your changes are anywhere else, make sure you can still compile userspace via `make buildworld`. + If your changes are to a branch, make sure your testing occurs with a machine which is running that code. + If you have a change which also may break another architecture, be sure and test on all supported architectures. + Please ensure your change works for + crossref:committers-guide[compilers,supported toolchains]. + Please refer to the https://www.FreeBSD.org/internal/[FreeBSD Internal Page] for a list of available resources. + As other architectures are added to the FreeBSD supported platforms list, the appropriate shared testing resources will be made available. + . Do not commit to contributed software without _explicit_ approval from the respective maintainers. + + + Contributed software is anything under the [.filename]#src/contrib#, [.filename]#src/crypto#, or [.filename]#src/sys/contrib# trees. + + + The trees mentioned above are for contributed software usually imported onto a vendor branch. + Committing something there may cause unnecessary headaches when importing newer versions of the software. + As a general consider sending patches upstream to the vendor. + Patches may be committed to FreeBSD first with permission of the maintainer. + + + Reasons for modifying upstream software range from wanting strict control over a tightly coupled dependency to lack of portability in the canonical repository's distribution of their code. + Regardless of the reason, effort to minimize the maintenance burden of fork is helpful to fellow maintainers. + Avoid committing trivial or cosmetic changes to files since it makes every merge thereafter more difficult: such patches need to be manually re-verified every import. + + + If a particular piece of software lacks a maintainer, you are encouraged to take up ownership. + If you are unsure of the current maintainership email {freebsd-arch} and ask. + + === Policy on Multiple Architectures + + In an effort to make it easier to keep FreeBSD portable across the platforms we support, core has developed this mandate: + + [.blockquote] + Major design work (including major API and ABI changes) must prove itself on at least one Tier 1 platform before it may be committed to the source tree. + + Developers should also be aware of our Tier Policy for the long term support of hardware architectures. + The rules here are intended to provide guidance during the development process, and are distinct from the requirements for features and architectures listed in that section. + The Tier rules for feature support on architectures at release-time are more strict than the rules for changes during the development process. + + [[compilers]] + === Policy on Multiple Compilers + + The FreeBSD base system builds with both Clang and GCC. + The project does this in a careful and controlled way to maximize benefits from this extra work, while keeping the extra work to a minimum. + Supporting both Clang and GCC improves the flexibility our users have. + These compilers have different strengths and weaknesses, and supporting both allows users to pick the best one for their needs. + Clang and GCC support similar dialects of C and C++, necessitating a relatively small amount of conditional code. + The project gains increased code coverage and improves the code quality by using features from both compilers. + The project is able to build in more user environments and leverage more CI environments by supporting this range, increasing convenience for users and giving them more tools to test with. + By carefully constraining the range of versions supported to modern versions of these compilers, the project avoids unduly increasing the testing matrix. + Older and obscure compilers, as well as older dialects of the languages, have extremely limited support that allow user programs to build with them, but without constraining the base system to being built with them. + The exact balance continues to evolve to ensure the benefits of extra work remain greater than the burdens it imposes. + The project used to support really old Intel compilers or old GCC versions, but we traded supporting those obsolete compilers for a carefully selected range of modern compilers. + This section documents where we use different compilers, and the expectations around that. + + The FreeBSD base system includes an in-tree Clang compiler. + Due to being in the tree, this compiler is the most supported compiler. + All changes must compile with it, prior to commit. + Complete testing, as appropriate for the change, should be done with this compiler. + + The FreeBSD base system also supports various versions of Clang and GCC as out-of-tree compilers. + For large or risky changes, committers should do a test build with a + supported version of GCC. + Out of tree compilers are available as packages. + GCC compilers are available as `${TARGET_ARCH}-gcc${VERSION}` packages, such as package:devel/freebsd-gcc14@aarch64[aarch64-gcc14]. + Clang compilers are available as `llvm${VERSION}` packages, such as + package:devel/llvm18[llvm18]. + The project runs automated CI jobs to build everything with these compilers. + Committers are expected to fix the jobs they break with their changes. + Committers may test builds of userspace or individual kernels by setting `CROSS_TOOLCHAIN` to the package name, for example `CROSS_TOOLCHAIN=aarch64-gcc14` or `CROSS_TOOLCHAIN=llvm18`. + For universe or tinderbox builds, `USE_GCC_TOOLCHAINS=gcc${VERSION}` builds all architectures using the appropriate GCC compiler packages. + For universe or tinderbox builds using an out-of-tree Clang, pass `CROSS_TOOLCHAIN=llvm${VERSION}`. + Note that while all architectures in the base system can be compiled by Clang, + only a few architectures can be fully built by GCC. + + The FreeBSD project also has some CI pipelines on github. + For pull requests on github and some branches pushed to github forks, a number of cross compilation jobs run. + These test FreeBSD building using versions of Clang that lag the in-tree compiler by one or more major versions. + + The FreeBSD project is also upgrading compilers. + Both Clang and GCC are fast moving targets. + Some work to change things in the tree, for example removing the old-style K&R function declarations and definitions, will land in the tree prior to the compiler landing. + Committers should try to be mindful about this and be receptive to looking into problems with their code or changes with these new compilers. + Also, just after a new compiler version hits the tree, people may need to compile things with the old version if there was an undetected regression suspected. + + In addition to the compiler, LLVM's LLD and GNU's binutils are used indirectly by the compiler. + Committers should be mindful of variations in assembler syntax and features of the linkers and ensure both variants work. + These components will be tested as part of FreeBSD's CI jobs for Clang or GCC. + + The FreeBSD project provides headers and libraries that allow other compilers to be used to build software not in the base system. + These headers have support for making the environment as strict as the standard, supporting prior dialects of ANSI-C back to C89, and other edge cases our large ports collection has uncovered. + This support constrains retirement of older standards in places like header files, but does not constrain updating the base system to newer dialects. + Nor does it require the base system to compile with these older standards as a whole. + Breaking this support will cause packages in the ports collection to fail, so should be avoided where possible, and promptly fixed when it is easy to do so. + + The FreeBSD build system currently accommodates these different environments. + As new warnings are added to compilers, the project tries to fix them. + However, sometimes these warnings require extensive rework, so are suppressed in some way by using make variables that evaluate to the proper thing depending on the compiler version. + Developers should be mindful of this, and ensure any compiler specific flags are properly conditionalized. + + ==== Current Compiler Versions + The versions of supported compilers for a given branch such as `main` or `stable/X` varies over time. + The authoritative source for supported compiler versions are automated CI jobs tested in GitHub's cross-build actions and Jenkins. + + For `main`, the in-tree compiler is currently Clang 19. + Currently, GCC 12, 13, and 14 are tested for amd64 via CI jobs in Jenkins. + Clang 14 and 18 are tested for aarch64 and arm64 in GitHub's cross-build actions. + + === Other Suggestions + + When committing documentation changes, use a spell checker before committing. + For all XML docs, verify that the formatting directives are correct by running `make lint` and package:textproc/igor[]. + + For manual pages, run package:sysutils/manck[] and package:textproc/igor[] over the manual page to verify all of the cross references and file references are correct and that the man page has all of the appropriate `MLINKS` installed. + + Do not mix style fixes with new functionality. + A style fix is any change which does not modify the functionality of the code. + Mixing the changes obfuscates the functionality change when asking for differences between revisions, which can hide any new bugs. + Do not include whitespace changes with content changes in commits to [.filename]#doc/#. + The extra clutter in the diffs makes the translators' job much more difficult. + Instead, make any style or whitespace changes in separate commits that are clearly labeled as such in the commit message. + + === Deprecating Features + + When it is necessary to remove functionality from software in the base system, follow these guidelines whenever possible: + + . Mention is made in the manual page and possibly the release notes that the option, utility, or interface is deprecated. Use of the deprecated feature generates a warning. + . The option, utility, or interface is preserved until the next major (point zero) release. + . The option, utility, or interface is removed and no longer documented. It is now obsolete. It is also generally a good idea to note its removal in the release notes. + + === Privacy and Confidentiality + + . Most FreeBSD business is done in public. + + + FreeBSD is an _open_ project. + Which means that not only can anyone use the source code, but that most of the development process is open to public scrutiny. + . Certain sensitive matters must remain private or held under embargo. + + + There unfortunately cannot be complete transparency. + As a FreeBSD developer you will have a certain degree of privileged access to information. + Consequently you are expected to respect certain requirements for confidentiality. + Sometimes the need for confidentiality comes from external collaborators or has a specific time limit. + Mostly though, it is a matter of not releasing private communications. + . The Security Officer has sole control over the release of security advisories. + + + Where there are security problems that affect many different operating systems, FreeBSD frequently depends on early access to be able to prepare advisories for coordinated release. + Unless FreeBSD developers can be trusted to maintain security, such early access will not be made available. + The Security Officer is responsible for controlling pre-release access to information about vulnerabilities, and for timing the release of all advisories. + He may request help under condition of confidentiality from any developer with relevant knowledge to prepare security fixes. + . Communications with Core are kept confidential for as long as necessary. + + + Communications to core will initially be treated as confidential. + Eventually however, most of Core's business will be summarized into the monthly or quarterly core reports. + Care will be taken to avoid publicising any sensitive details. + Records of some particularly sensitive subjects may not be reported on at all and will be retained only in Core's private archives. + . Non-disclosure Agreements may be required for access to certain commercially sensitive data. + + + Access to certain commercially sensitive data may only be available under a Non-Disclosure Agreement. + The FreeBSD Foundation legal staff must be consulted before any binding agreements are entered into. + . Private communications must not be made public without permission. + + + Beyond the specific requirements above there is a general expectation not to publish private communications between developers without the consent of all parties involved. + Ask permission before forwarding a message onto a public mailing list, or posting it to a forum or website that can be accessed by other than the original correspondents. + . Communications on project-only or restricted access channels must be kept private. + + + Similarly to personal communications, certain internal communications channels, including FreeBSD Committer only mailing lists and restricted access IRC channels are considered private communications. + Permission is required to publish material from these sources. + . Core may approve publication. + + + Where it is impractical to obtain permission due to the number of correspondents or where permission to publish is unreasonably withheld, Core may approve release of such private matters that merit more general publication. + + [[archs]] + == Support for Multiple Architectures + + FreeBSD is a highly portable operating system intended to function on many different types of hardware architectures. + Maintaining clean separation of Machine Dependent (MD) and Machine Independent (MI) code, as well as minimizing MD code, is an important part of our strategy to remain agile with regards to current hardware trends. + Each new hardware architecture supported by FreeBSD adds substantially to the cost of code maintenance, toolchain support, and release engineering. + It also dramatically increases the cost of effective testing of kernel changes. + As such, there is strong motivation to differentiate between classes of support for various architectures while remaining strong in a few key architectures that are seen as the FreeBSD "target audience". + + === Statement of General Intent + + The FreeBSD Project targets "production quality commercial off-the-shelf (COTS) workstation, server, and high-end embedded systems". + By retaining a focus on a narrow set of architectures of interest in these environments, the FreeBSD Project is able to maintain high levels of quality, stability, and performance, as well as minimize the load on various support teams on the project, such as the ports team, documentation team, security officer, and release engineering teams. + Diversity in hardware support broadens the options for FreeBSD consumers by offering new features and usage opportunities, but these benefits must always be carefully considered in terms of the real-world maintenance cost associated with additional platform support. + + The FreeBSD Project differentiates platform targets into four tiers. + Each tier includes a list of guarantees consumers may rely on as well as obligations by the Project and developers to fulfill those guarantees. + These lists define the minimum guarantees for each tier. + The Project and developers may provide additional levels of support beyond the minimum guarantees for a given tier, but such additional support is not guaranteed. + Each platform target is assigned to a specific tier for each stable branch. + As a result, a platform target might be assigned to different tiers on concurrent stable branches. + + === Platform Targets + + Support for a hardware platform consists of two components: kernel support and userland Application Binary Interfaces (ABIs). + Kernel platform support includes things needed to run a FreeBSD kernel on a hardware platform such as machine-dependent virtual memory management and device drivers. + A userland ABI specifies an interface for user processes to interact with a FreeBSD kernel and base system libraries. + A userland ABI includes system call interfaces, the layout and semantics of public data structures, and the layout and semantics of arguments passed to subroutines. + Some components of an ABI may be defined by specifications such as the layout of C++ exception objects or calling conventions for C functions. + + A FreeBSD kernel also uses an ABI (sometimes referred to as the Kernel Binary Interface (KBI)) which includes the semantics and layouts of public data structures and the layout and semantics of arguments to public functions within the kernel itself. + + A FreeBSD kernel may support multiple userland ABIs. + For example, FreeBSD's amd64 kernel supports FreeBSD amd64 and i386 userland ABIs as well as Linux x86_64 and i386 userland ABIs. + A FreeBSD kernel should support a "native" ABI as the default ABI. + The native "ABI" generally shares certain properties with the kernel ABI such as the C calling convention, sizes of basic types, etc. + + Tiers are defined for both kernels and userland ABIs. In the common case, a platform's kernel and FreeBSD ABIs are assigned to the same tier. + + ==== Tier 1: Fully-Supported Architectures + + Tier 1 platforms are the most mature FreeBSD platforms. + They are supported by the security officer, release engineering, and Ports Management Team. + Tier 1 architectures are expected to be Production Quality with respect to all aspects of the FreeBSD operating system, including installation and development environments. + + The FreeBSD Project provides the following guarantees to consumers of Tier 1 platforms: + + * Official FreeBSD release images will be provided by the release engineering team. + * Binary updates and source patches for Security Advisories and Errata Notices will be provided for supported releases. + * Source patches for Security Advisories will be provided for supported branches. + * Binary updates and source patches for cross-platform Security Advisories will typically be provided at the time of the announcement. + * Changes to userland ABIs will generally include compatibility shims to ensure correct operation of binaries compiled against any stable branch where the platform is Tier 1. These shims might not be enabled in the default install. If compatibility shims are not provided for an ABI change, the lack of shims will be clearly documented in the release notes. + * Changes to certain portions of the kernel ABI will include compatibility shims to ensure correct operation of kernel modules compiled against the oldest supported release on the branch. Note that not all parts of the kernel ABI are protected. + * Official binary packages for third party software will be provided by the ports team. For embedded architectures, these packages may be cross-built from a different architecture. + * Most relevant ports should either build or have the appropriate filters to prevent inappropriate ones from building. + * New features which are not inherently platform-specific will be fully functional on all Tier 1 architectures. + * Features and compatibility shims used by binaries compiled against older stable branches may be removed in newer major versions. Such removals will be clearly documented in the release notes. + * Tier 1 platforms should be fully documented. Basic operations will be documented in the FreeBSD Handbook. + * Tier 1 platforms will be included in the source tree. + * Tier 1 platforms should be self-hosting either via the in-tree toolchain or an external toolchain. If an external toolchain is required, official binary packages for an external toolchain will be provided. + + To maintain maturity of Tier 1 platforms, the FreeBSD Project will maintain the following resources to support development: + + * Build and test automation support either in the FreeBSD.org cluster or some other location easily available for all developers. Embedded platforms may substitute an emulator available in the FreeBSD.org cluster for actual hardware. + * Inclusion in the `make universe` and `make tinderbox` targets. + * Dedicated hardware in one of the FreeBSD clusters for package building (either natively or via qemu-user). + + Collectively, developers are required to provide the following to maintain the Tier 1 status of a platform: + + * Changes to the source tree should not knowingly break the build of a Tier 1 platform. + * Tier 1 architectures must have a mature, healthy ecosystem of users and active developers. + * Developers should be able to build packages on commonly available, non-embedded Tier 1 systems. This can mean either native builds if non-embedded systems are commonly available for the platform in question, or it can mean cross-builds hosted on some other Tier 1 architecture. + * Changes cannot break the userland ABI. If an ABI change is required, ABI compatibility for existing binaries should be provided via use of symbol versioning or shared library version bumps. + * Changes merged to stable branches cannot break the protected portions of the kernel ABI. If a kernel ABI change is required, the change should be modified to preserve functionality of existing kernel modules. + + ==== Tier 2: Developmental and Niche Architectures + + Tier 2 platforms are functional, but less mature FreeBSD platforms. + They are not supported by the security officer, release engineering, and Ports Management Team. + + Tier 2 platforms may be Tier 1 platform candidates that are still under active development. + Architectures reaching end of life may also be moved from Tier 1 status to Tier 2 status as the availability of resources to continue to maintain the system in a Production Quality state diminishes. + Well-supported niche architectures may also be Tier 2. + + The FreeBSD Project provides the following guarantees to consumers of Tier 2 platforms: + + * The ports infrastructure should include basic support for Tier 2 architectures sufficient to support building ports and packages. This includes support for basic packages such as ports-mgmt/pkg, but there is no guarantee that arbitrary ports will be buildable or functional. + * New features which are not inherently platform-specific should be feasible on all Tier 2 architectures if not implemented. + * Tier 2 platforms will be included in the source tree. + * Tier 2 platforms should be self-hosting either via the in-tree toolchain or an external toolchain. If an external toolchain is required, official binary packages for an external toolchain will be provided. + * Tier 2 platforms should provide functional kernels and userlands even if an official release distribution is not provided. + + To maintain maturity of Tier 2 platforms, the FreeBSD Project will maintain the following resources to support development: + + * Inclusion in the `make universe` and `make tinderbox` targets. + + Collectively, developers are required to provide the following to maintain the Tier 2 status of a platform: + + * Changes to the source tree should not knowingly break the build of a Tier 2 platform. + * Tier 2 architectures must have an active ecosystem of users and developers. + * While changes are permitted to break the userland ABI, the ABI should not be broken gratuitously. Significant userland ABI changes should be restricted to major versions. + * New features that are not yet implemented on Tier 2 architectures should provide a means of disabling them on those architectures. + + ==== Tier 3: Experimental Architectures + + Tier 3 platforms have at least partial FreeBSD support. + They are _not_ supported by the security officer, release engineering, and Ports Management Team. + + Tier 3 platforms are architectures in the early stages of development, for non-mainstream hardware platforms, or which are considered legacy systems unlikely to see broad future use. + Initial support for Tier 3 platforms may exist in a separate repository rather than the main source repository. + + The FreeBSD Project provides no guarantees to consumers of Tier 3 platforms and is not committed to maintaining resources to support development. + Tier 3 platforms may not always be buildable, nor are any kernel or userland ABIs considered stable. + + ==== Unsupported Architectures + + Other platforms are not supported in any form by the project. + The project previously described these as Tier 4 systems. + + After a platform transitions to unsupported, all support for the platform is removed from the source, ports and documentation trees. + Note that ports support should remain as long as the platform is supported in a branch supported by ports. + + === Policy on Changing the Tier of an Architecture + + Systems may only be moved from one tier to another by approval of the FreeBSD Core Team, which shall make that decision in collaboration with the Security Officer, Release Engineering, and ports management teams. + For a platform to be promoted to a higher tier, any missing support guarantees must be satisfied before the promotion is completed. + + [[ports]] + == Ports Specific FAQ + + [[ports-qa-adding]] + === Adding a New Port + + [[ports-qa-add-new]] + ==== How do I add a new port? + + Adding a port to the tree is relatively simple. Once the port is ready to be + added, as explained later crossref:committers-guide[ports-qa-add-new-extra,here], you need to add the port's directory entry in the category's [.filename]#Makefile#. + In this [.filename]#Makefile#, ports are listed in alphabetical order and added to the `SUBDIR` variable, like this: + + [.programlisting] + .... + SUBDIR += newport + .... + + Once the port and its category's Makefile are ready, the new port can be committed: + [source,shell] + .... + % git add category/Makefile category/newport + % git commit + % git push + .... + [TIP] + ==== + Don't forget to crossref:committers-guide[port-commit-message-formats,setup git hooks for the ports tree as explained here]; a specific hook has been developed to verify the category's [.filename]#Makefile#. + ==== + + [[ports-qa-add-new-extra]] + ==== Any other things I need to know when I add a new port? + + Check the port, preferably to make sure it compiles and packages correctly. + + The extref:{porters-handbook}testing[Porters Handbook's Testing Chapter] contains more detailed instructions. + See the extref:{porters-handbook}testing[Portclippy / Portfmt, testing-portclippy] and the extref:{porters-handbook}testing[poudriere, testing-poudriere] sections. + + You do not necessarily have to eliminate all warnings but make sure you have fixed the simple ones. + + If the port came from a submitter who has not contributed to the Project before, add that person's name to the extref:{contributors}[Additional Contributors, contrib-additional] section of the FreeBSD Contributors List. + + Close the PR if the port came in as a PR. + To close a PR, change the state to `Issue Resolved` and the resolution as `Fixed`. + + [NOTE] + ==== + If for some reason using extref:{porters-handbook}testing[poudriere, testing-poudriere] to test the new port is not possible, the bare minimum of testing includes this sequence: + + [source,shell] + .... + # make install + # make package + # make deinstall + # pkg add package you built above + # make deinstall + # make reinstall + # make package + .... + + Note that poudriere is the reference for package building, it the port does not build in poudriere, it will be removed. + ==== + + [[ports-qa-removing]] + === Removing an Existing Port + + [[ports-qa-remove-one]] + ==== How do I remove an existing port? + + First, please read the section about repository copies. Before you remove the port, you have to verify there are no other ports depending on it. + + * Make sure there is no dependency on the port in the ports collection: + ** The port's PKGNAME appears in exactly one line in a recent INDEX file. + ** No other ports contains any reference to the port's directory or PKGNAME in their Makefiles + + + [TIP] + ==== + When using Git, consider using man:git-grep[1], it is much faster than `grep -r`. + ==== + + + * Then, remove the port: + + + [.procedure] + ==== + * Remove the port's files and directory with `git rm`. + * Remove the `SUBDIR` listing of the port in the parent directory [.filename]#Makefile#. + * Add an entry to [.filename]#ports/MOVED#. + * Remove the port from [.filename]#ports/LEGAL# if it is there. + ==== + + Alternatively, you can use the rmport script, from [.filename]#ports/Tools/scripts#. + This script was written by {vd}. + When sending questions about this script to the {freebsd-ports}, please also CC {crees}, the current maintainer. + + [[ports-qa-move-port]] + === How do I move a port to a new location? + + [.procedure] + ==== + . Perform a thorough check of the ports collection for any dependencies on the old port location/name, and update them. Running `grep` on [.filename]#INDEX# is not enough because some ports have dependencies enabled by compile-time options. A full man:git-grep[1] of the ports collection is recommended. + . Remove the `SUBDIR` entry from the old category Makefile and add a `SUBDIR` entry to the new category Makefile. + . Add an entry to [.filename]#ports/MOVED#. + . Search for entries in xml files inside [.filename]#ports/security/vuxml# and adjust them accordingly. In particular, check for previous packages with the new name which version could include the new port. + . Move the port with `git mv`. + . Commit the changes. + ==== + + [[ports-qa-copy-port]] + === How do I copy a port to a new location? + + [.procedure] + ==== + . Copy port with `cp -R old-cat/old-port new-cat/new-port`. + . Add the new port to the [.filename]#new-cat/Makefile#. + . Change stuff in [.filename]#new-cat/new-port#. + . Commit the changes. + ==== + + [[ports-qa-freeze]] + === Ports Freeze + + [[ports-qa-freeze-what]] + ==== What is a “ports freeze”? + + A “ports freeze” was a restricted state the ports tree was put in before a release. + It was used to ensure a higher quality for the packages shipped with a release. + It usually lasted a couple of weeks. + During that time, build problems were fixed, and the release packages were built. + This practice is no longer used, as the packages for the releases are built from the current stable, quarterly branch. + + For more information on how to merge commits to the quarterly branch, see + crossref:committers-guide[ports-qa-misc-request-mfh, What is the procedure to request authorization for merging a commit to the quarterly branch?]. + + [[ports-qa-quarterly]] + === Quarterly Branches + + [[ports-qa-misc-request-mfh]] + ==== What is the procedure to request authorization for merging a commit to the quarterly branch? + + As of November 30, 2020, there is no need to seek explicit approval to commit to the quarterly branch. + + [[ports-qa-misc-commit-mfh]] + ==== What is the procedure for merging commits to the quarterly branch? + + Merging commits to the quarterly branch (a process we call MFH for a historical reason) is very similar to MFC'ing a commit in the src repository, so basically: + [source,shell] + .... + % git checkout 2021Q2 + % git cherry-pick -x $HASH + (verify everything is OK, for example by doing a build test) + % git push + .... + + where `$HASH` is the hash of the commit you want to copy over to the quarterly branch. + The `-x` parameter ensures the hash `$HASH` of the `main` branch is included in the new commit message of the quarterly branch. + + [[ports-qa-new-category]] + === Creating a New Category + + [[ports-qa-new-category-how]] + ==== What is the procedure for creating a new category? + +-Please see extref:{porters-handbook}[Proposing a New Category, proposing-categories] in the Porter's Handbook. ++Please see extref:{porters-handbook}makefiles[Proposing a New Category, proposing-categories] in the Porter's Handbook. + Once that procedure has been followed and the PR has been assigned to the {portmgr}, it is their decision whether or not to approve it. + If they do, it is their responsibility to: + + [.procedure] + ==== + . Perform any needed moves. (This only applies to physical categories.) + . Update the `VALID_CATEGORIES` definition in [.filename]#ports/Mk/bsd.port.mk#. + . Assign the PR back to you. + ==== + + [[ports-qa-new-category-physical]] + ==== What do I need to do to implement a new physical category? + + [.procedure] + ==== + . Upgrade each moved port's [.filename]#Makefile#. Do not connect the new category to the build yet. + + + To do this, you will need to: + + + [.procedure] + ====== + . Change the port's `CATEGORIES` (this was the point of the exercise, remember?) The new category is listed first. This will help to ensure that the PKGORIGIN is correct. + . Run a `make describe`. Since the top-level `make index` that you will be running in a few steps is an iteration of `make describe` over the entire ports hierarchy, catching any errors here will save you having to re-run that step later on. + . If you want to be really thorough, now might be a good time to run man:portlint[1]. + ====== + + + . Check that the ``PKGORIGIN``s are correct. The ports system uses each port's `CATEGORIES` entry to create its `PKGORIGIN`, which is used to connect installed packages to the port directory they were built from. If this entry is wrong, common port tools like man:pkg-version[8] and man:portupgrade[1] fail. + + + To do this, use the [.filename]#chkorigin.sh# tool: `env PORTSDIR=/path/to/ports sh -e /path/to/ports/Tools/scripts/chkorigin.sh`. This will check every port in the ports tree, even those not connected to the build, so you can run it directly after the move operation. Hint: do not forget to look at the ``PKGORIGIN``s of any slave ports of the ports you just moved! + . On your own local system, test the proposed changes: first, comment out the SUBDIR entries in the old ports' categories' [.filename]##Makefile##s; then enable building the new category in [.filename]#ports/Makefile#. Run make checksubdirs in the affected category directories to check the SUBDIR entries. Next, in the [.filename]#ports/# directory, run make index. This can take over 40 minutes on even modern systems; however, it is a necessary step to prevent problems for other people. + . Once this is done, you can commit the updated [.filename]#ports/Makefile# to connect the new category to the build and also commit the [.filename]#Makefile# changes for the old category or categories. + . Add appropriate entries to [.filename]#ports/MOVED#. + . Update the documentation by modifying: +-** the extref:{porters-handbook}[list of categories, PORTING-CATEGORIES] in the Porter's Handbook ++** the extref:{porters-handbook}makefiles[list of categories, porting-categories] in the Porter's Handbook + + + . Only once all the above have been done, and no one is any longer reporting problems with the new ports, should the old ports be deleted from their previous locations in the repository. + ==== + + ==== What do I need to do to implement a new virtual category? + + This is much simpler than a physical category. Only a few modifications are needed: + +-* the extref:{porters-handbook}[list of categories, PORTING-CATEGORIES] in the Porter's Handbook ++* the extref:{porters-handbook}makefiles[list of categories, porting-categories] in the Porter's Handbook + + [[ports-qa-misc-questions]] + === Miscellaneous Questions + + [[ports-qa-misc-blanket-approval]] + ==== Are there changes that can be committed without asking the maintainer for approval? + + Blanket approval for most ports applies to these types of fixes: + + * Most infrastructure changes to a port (that is, modernizing, but not changing the functionality). For example, the blanket covers converting to new `USES` macros, enabling verbose builds, and switching to new ports system syntaxes. + * Trivial and _tested_ build and runtime fixes. + * Documentations or metadata changes to ports, like [.filename]#pkg-descr# or `COMMENT`. + + [IMPORTANT] + ==== + Exceptions to this are anything maintained by the {portmgr}, or the {security-officer}. + No unauthorized commits may ever be made to ports maintained by those groups. + ==== + + [[ports-qa-misc-correctly-building]] + ==== How do I know if my port is building correctly or not? + + The packages are built multiple times each week. + If a port fails, the maintainer will receive an email from `pkg-fallout@FreeBSD.org`. + +-Reports for all the package builds (official, experimental, and non-regression) are aggregated at link:pkg-status.FreeBSD.org[pkg-status.FreeBSD.org]. ++Reports for all the package builds (official, experimental, and non-regression) are aggregated at link:https://pkg-status.FreeBSD.org[pkg-status.FreeBSD.org]. + + [[ports-qa-misc-INDEX]] + ==== I added a new port. Do I need to add it to the [.filename]#INDEX#? + + No. The file can either be generated by running `make index`, or a pre-generated version can be downloaded with `make fetchindex`. + + [[ports-qa-misc-no-touch]] + ==== Are there any other files I am not allowed to touch? + + Any file directly under [.filename]#ports/#, or any file under a subdirectory that starts with an uppercase letter ([.filename]#Mk/#, [.filename]#Tools/#, etc.). + In particular, the {portmgr} is very protective of [.filename]#ports/Mk/bsd.port*.mk# so do not commit changes to those files unless you want to face their wrath. + + [[ports-qa-misc-updated-distfile]] + ==== What is the proper procedure for updating the checksum for a port distfile when the file changes without a version change? + + When the checksum for a distribution file is updated due to the author updating the file without changing the port revision, the commit message includes a summary of the relevant diffs between the original and new distfile to ensure that the distfile has not been corrupted or maliciously altered. + If the current version of the port has been in the ports tree for a while, a copy of the old distfile will usually be available on the ftp servers; otherwise the author or maintainer should be contacted to find out why the distfile has changed. + + [[ports-exp-run]] + ==== How can an experimental test build of the ports tree (exp-run) be requested? + + An exp-run must be completed before patches with a significant ports impact are committed. + The patch can be against the ports tree or the base system. + + Full package builds will be done with the patches provided by the submitter, and the submitter is required to fix detected problems _(fallout)_ before commit. + + [.procedure] + ==== + . Go to the link:https://bugs.freebsd.org/submit[Bugzilla new PR page]. + . Select the product your patch is about. + . Fill in the bug report as normal. Remember to attach the patch. + . If at the top it says “Show Advanced Fields” click on it. It will now say “Hide Advanced Fields”. Many new fields will be available. If it already says “Hide Advanced Fields”, no need to do anything. + . In the “Flags” section, set the “exp-run” one to `?`. As for all other fields, hovering the mouse over any field shows more details. + . Submit. Wait for the build to run. + . {portmgr} will reply with a possible fallout. + . Depending on the fallout: + ** If there is no fallout, the procedure stops here, and the change can be committed, pending any other approval required. + ... If there is fallout, it _must_ be fixed, either by fixing the ports directly in the ports tree, or adding to the submitted patch. + ... When this is done, go back to step 6 saying the fallout was fixed and wait for the exp-run to be run again. Repeat as long as there are broken ports. + ==== + + [[non-committers]] + == Issues Specific to Developers Who Are Not Committers + + A few people who have access to the FreeBSD machines do not have commit bits. + Almost all of this document will apply to these developers as well (except things specific to commits and the mailing list memberships that go with them). + In particular, we recommend that you read: + + * crossref:committers-guide[admin, Administrative Details] + * crossref:committers-guide[conventions-everyone, For Everyone] + + + [NOTE] + ==== + Get your mentor to add you to the "Additional Contributors" ([.filename]#doc/shared/contrib-additional.adoc#), if you are not already listed there. + ==== + * crossref:committers-guide[developer.relations, Developer Relations] + * crossref:committers-guide[ssh.guide, SSH Quick-Start Guide] + * crossref:committers-guide[rules, The FreeBSD Committers' Big List of Rules] + + [[google-analytics]] + == Information About Google Analytics + + As of December 12, 2012, Google Analytics was enabled on the FreeBSD Project website to collect anonymized usage statistics regarding usage of the site. + + [NOTE] + ==== + As of March 3, 2022, Google Analytics was removed from the FreeBSD Project. + ==== + + [[misc]] + == Miscellaneous Questions + + === How do I access people.FreeBSD.org to put up personal or project information? + + `people.FreeBSD.org` is the same as `freefall.FreeBSD.org`. + Just create a [.filename]#public_html# directory. Anything you place in that directory will automatically be visible under https://people.FreeBSD.org/[https://people.FreeBSD.org/]. + + === Where are the mailing list archives stored? + + The mailing lists are archived under [.filename]#/local/mail# on `freefall.FreeBSD.org`. + + === I would like to mentor a new committer. What process do I need to follow? + + See the https://www.freebsd.org/internal/new-account/[New Account Creation Procedure] document on the internal pages. + + [[benefits]] + == Benefits and Perks for FreeBSD Committers + + [[benefits-recognition]] + === Recognition + + Recognition as a competent software engineer is the longest lasting value. + In addition, getting a chance to work with some of the best people that every engineer would dream of meeting is a great perk! + + [[benefits-freebsdmall]] + === FreeBSD Mall + + FreeBSD committers can get a free 4-CD or DVD set at conferences from http://www.freebsdmall.com[FreeBSD Mall, Inc.]. + + [[benefits-gandi]] + === `Gandi.net` + + https://gandi.net[Gandi] provides website hosting, cloud computing, domain registration, and X.509 certificate services. + + Gandi offers an E-rate discount to all FreeBSD developers. + To streamline the process of getting the discount first set up a Gandi account, fill in the billing information and select the currency. + Then send an mail to mailto:non-profit@gandi.net[non-profit@gandi.net] using your `@freebsd.org` mail address, and indicate your Gandi handle. + + [[benefits-rsync]] + === `rsync.net` + + https://rsync.net[rsync.net] provides cloud storage for offsite backup that is optimized for UNIX users. Their service runs entirely on FreeBSD and ZFS. + + rsync.net offers a free-forever 500 GB account to FreeBSD developers. Simply sign up at https://www.rsync.net/freebsd.html[https://www.rsync.net/freebsd.html] using your `@freebsd.org` address to receive this free account. +diff --git a/documentation/content/en/articles/contributing/_index.adoc b/documentation/content/en/articles/contributing/_index.adoc +index d98ff31de0..022dc29e2e 100644 +--- a/documentation/content/en/articles/contributing/_index.adoc ++++ b/documentation/content/en/articles/contributing/_index.adoc +@@ -1,648 +1,648 @@ + --- + title: Contributing to FreeBSD + authors: + - author: Jordan Hubbard + - author: Sam Lawrance + - author: Mark Linimon + description: How to contribute to the FreeBSD Project + trademarks: ["freebsd", "ieee", "general"] + weight: 15 + tags: ["Contributing", "FreeBSD", "Non-Programmer Tasks", "Programmer Tasks"] + --- + + = Contributing to FreeBSD + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/contributing/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + pass:[] + + [.abstract-title] + Abstract + + This article describes the different ways in which an individual or organization may contribute to the FreeBSD Project. + + ''' + + toc::[] + + So you want to contribute to FreeBSD? + That is great! + FreeBSD _relies_ on the contributions of its user base to survive. + Your contributions are not only appreciated, they are vital to FreeBSD's continued growth. + + A large and growing number of international contributors, of greatly varying ages and areas of technical expertise, develop FreeBSD. + There is always more work to be done than there are people available to do it, and more help is always appreciated. + + As a volunteer, what you do is limited only by what you want to do. + However, we do ask that you are aware of what other members of the FreeBSD community will expect of you. + You may want to take this into account before deciding to volunteer. + + The FreeBSD project is responsible for an entire operating system environment, rather than just a kernel or a few scattered utilities. + As such, our [.filename]#TODO# lists span a very wide range of tasks: from documentation, beta testing and presentation, to the system installer and highly specialized types of kernel development. + People of any skill level, in almost any area, can almost certainly help the project. + + Commercial entities engaged in FreeBSD-related enterprises are also encouraged to contact us. + Do you need a special extension to make your product work? + You will find us receptive to your requests, given that they are not too outlandish. + Are you working on a value-added product? + Please let us know! + We may be able to work cooperatively on some aspect of it. + The free software world is challenging many existing assumptions about how software is developed, sold, and maintained, and we urge you to at least give it a second look. + + [[contrib-what]] + == What Is Needed + + The following list of tasks and sub-projects represents something of an amalgam of various [.filename]#TODO# lists and user requests. + + [[non-programmer-tasks]] + === Ongoing Non-Programmer Tasks + + Many people who are involved in FreeBSD are not programmers. + The Project includes documentation writers, Web designers, and support people. + All that these people need to contribute is an investment of time and a willingness to learn. + + . Read through the FAQ and Handbook periodically. + If anything is poorly explained, ambiguous, out of date or incorrect, let us know. + Even better, send us a fix (AsciiDoc is not difficult to learn, but there is no objection to plain text submissions). + . Help translate FreeBSD documentation into your native language. + If documentation already exists for your language, you can help translate additional documents or verify that the translations are up-to-date and correct. +-First take a look at the extref:{fdp-primer}[Translations FAQ, translations] in the FreeBSD Documentation Project Primer. ++First take a look at the extref:{fdp-primer}translations[Translations FAQ, translations] in the FreeBSD Documentation Project Primer. + You are not committing yourself to translating every single FreeBSD document by doing this - as a volunteer, you can do as much or as little translation as you desire. + Once someone begins translating, others almost always join the effort. + If you only have the time or energy to translate one part of the documentation, please translate the installation instructions. + . Read the {freebsd-questions} occasionally (or even regularly). + It can be very satisfying to share your expertise and help people solve their problems; sometimes you may even learn something new yourself! + These forums can also be a source of ideas for things to improve upon. + + [[ongoing-programmer-tasks]] + === Ongoing Programmer Tasks + + Most of the tasks listed here may require a considerable investment of time, an in-depth knowledge of the FreeBSD kernel, or both. + However, there are also many useful tasks which are suitable for "weekend hackers". + + . If you run FreeBSD-CURRENT and have a good Internet connection, there is a machine `current.FreeBSD.org` which builds a full release once a day-every now and again, try to install the latest release from it and report any failures in the process. + . Read the {freebsd-bugs}. + There may be a problem you can comment constructively on or with patches you can test. + Or you could even try to fix one of the problems yourself. + . If you know of any bug fixes which have been successfully applied to -CURRENT but have not been merged into -STABLE after a decent interval (normally a couple of weeks), send the committer a polite reminder. + . Move contributed software to [.filename]#src/contrib# in the source tree. + . Make sure code in [.filename]#src/contrib# is up to date. + . Build the source tree (or just part of it) with extra warnings enabled and clean up the warnings. + A list of build warnings can also be found from our https://ci.freebsd.org[CI] by selecting a build and checking "LLVM/Clang Warnings". + . Fix warnings for ports which do deprecated things like using `gets()` or including [.filename]#malloc.h#. + . If you have contributed any ports and you had to make FreeBSD-specific changes, send your patches back to the original authors (this will make your life easier when they bring out the next version). + . Get copies of formal standards like POSIX(R). + Compare FreeBSD's behavior to that required by the standard. + If the behavior differs, particularly in subtle or obscure corners of the specification, send in a PR about it. + If you are able, figure out how to fix it and include a patch in the PR. + If you think the standard is wrong, ask the standards body to consider the question. + . Suggest further tasks for this list! + + === Work through the PR Database + + The https://bugs.FreeBSD.org/search/[FreeBSD PR list] shows all the current active problem reports and requests for enhancement that have been submitted by FreeBSD users. + The PR database includes both programmer and non-programmer tasks. + Look through the open PRs, and see if anything there takes your interest. + Some of these might be very simple tasks that just need an extra pair of eyes to look over them and confirm that the fix in the PR is a good one. + Others might be much more complex, or might not even have a fix included at all. + + Start with the PRs that have not been assigned to anyone else. + If a PR is assigned to someone else, but it looks like something you can handle, email the person it is assigned to and ask if you can work on it-they might already have a patch ready to be tested, or further ideas that you can discuss with them. + + === Ongoing Ports Tasks + + The Ports Collection is a perpetual work in progress. + We want to provide our users with an easy to use, up to date, high quality repository of third party software. + We need people to donate some of their time and effort to help us achieve this goal. + + Anyone can get involved, and there are lots of different ways to do so. + Contributing to ports is an excellent way to help "give back" something to the project. + Whether you are looking for an ongoing role, or a fun challenge for a rainy day, we would love to have your help! + + There are a number of easy ways you can contribute to keeping the ports tree up to date and in good working order: + + * Find some cool or useful software and extref:{porters-handbook}[create a port] for it. + * There are a large number of ports that have no maintainer. + Become a maintainer and crossref:contributing[adopt-port, Adopting an unmaintained port]. + * If you have created or adopted a port, be aware of crossref:contributing[maintain-port, The challenge for port maintainers]. + * When you are looking for a quick challenge you could crossref:contributing[fix-broken, Finding and fixing a broken port]. + + === Pick one of the items from the Ideas page + + The https://wiki.freebsd.org/IdeasPage[FreeBSD list of projects and ideas for volunteers] is also available for people willing to contribute to the FreeBSD project. + The list is being regularly updated and contains items for both programmers and non-programmers with information about each project. + + [[contrib-how]] + == How to Contribute + + Contributions to the system generally fall into one or more of the following 5 categories: + + [[contrib-general]] + === Bug Reports and General Commentary + + An idea or suggestion of _general_ technical interest should be mailed to the {freebsd-hackers}. + Likewise, people with an interest in such things (and a tolerance for a _high_ volume of mail!) may subscribe to the {freebsd-hackers}. +-See extref:{handbook}[The FreeBSD Handbook, eresources-mail] for more information about this and other mailing lists. ++See extref:{handbook}eresources[The FreeBSD Handbook, eresources-mail] for more information about this and other mailing lists. + + If you are submitting a simple patch to the src repo, please consider submitting it to the project's GitHub mirror as https://github.com/freebsd/freebsd-src/pulls[a pull request]. + Suitable submissions should: + + * It is ready or nearly ready to be committed. + A committer should be able to land this patch with less than 10 minutes of additional work. + * It passes all the GitHub CI jobs. + * You can respond to feedback quickly. + * It touches fewer than about 10 files and the changes are less than about 200 lines. + Changes larger than this may be OK, or you may be asked to submit multiple pull requests of a more manageable size. + * Each logical change is a separate commit within the pull request. + Commit messages for each change should follow extref:{committers-guide}#commit-log-message[commit log guide]. + * All commits have your name and valid email address as you'd like to see them in the FreeBSD repository as the author. + Fake github.com addresses cannot be used. + * The scope of the pull request should not change during review. + If the review suggests changes that expand the scope, please create an independent pull request. + * Fixup commits should be squashed with the commit they are fixing. + Each commit in your branch should be suitable for FreeBSD's repository. + * Commits should include one or more `Signed-off-by:` lines with full name and email address certifying https://developercertificate.org/[Developer Certificate of Origin]. + + When updating pull request, please rebase with a forced push rather than a merge commit. + More complex changes may be submitted as pull requests, but they may be closed if they are too large, too unwieldy, become inactive, need further discussion in the community, or need extensive revision. + Please avoid creating large, wide-ranging cleanup patches: they are too large and lack the focus needed for a good review. + Misdirected patches may be redirected to a more appropriate forum for the patch to be resolved. + + Pull requests submitted to the ports repository may or may not see action, based on the whims of developers. + For now, you will have a better experience if you follow the ports submission + process crossref:contributing[ports-contributing, Contributing to ports]. + + The docs team also accepts pull requests via GitHub, but has not established any policy for them yet. + + If you find a bug or are submitting a specific change, please report it using the https://bugs.FreeBSD.org/submit/[bug submission form]. + Try to fill-in each field of the bug report. + Unless they exceed 65KB, include any patches directly in the report. + If the patch is suitable to be applied to the source tree put `[PATCH]` in the synopsis of the report. + When including patches, _do not_ use cut-and-paste because cut-and-paste turns tabs into spaces and makes them unusable. + When patches are a lot larger than 20KB, consider compressing them (for example with man:gzip[1] or man:bzip2[1]) prior to uploading them. + + After filing a report, you should receive confirmation along with a tracking number. + Keep this tracking number so that you can update us with details about the problem. + + See also extref:{problem-reports}[this article] on how to write good problem reports. + + === Changes to the Documentation + + Changes to the documentation are overseen by the {freebsd-doc}. + Please look at the extref:{fdp-primer}[FreeBSD Documentation Project Primer] for complete instructions. + Send submissions and changes (even small ones are welcome!) using the same method as any other bug report. + + === Changes to Existing Source Code + + An addition or change to the existing source code is a somewhat trickier affair and depends a lot on how far out of date you are with the current state of FreeBSD development. + There is a special on-going release of FreeBSD known as "FreeBSD-CURRENT" which is made available in a variety of ways for the convenience of developers working actively on the system. +-See extref:{handbook}[The FreeBSD Handbook, current-stable] for more information about getting and using FreeBSD-CURRENT. ++See extref:{handbook}cutting-edge[The FreeBSD Handbook, current-stable] for more information about getting and using FreeBSD-CURRENT. + + Working from older sources unfortunately means that your changes may sometimes be too obsolete or too divergent for easy re-integration into FreeBSD. + Chances of this can be minimized somewhat by subscribing to the {freebsd-announce} and the {freebsd-current} lists, where discussions on the current state of the system take place. + + Assuming that you can manage to secure fairly up-to-date sources to base your changes on, the next step is to produce a set of diffs to send to the FreeBSD maintainers. + This is done with the man:diff[1] command. + + The preferred man:diff[1] format for submitting patches is the unified output format generated by `diff -u`. + + [source,shell] + .... + % diff -u oldfile newfile + .... + + or + + [source,shell] + .... + % diff -u -r -N olddir newdir + .... + + would generate a set of unified diffs for the given source file or directory hierarchy. + + See man:diff[1] for more information. + + Once you have a set of diffs (which you may test with the man:patch[1] command), you should submit them for inclusion with FreeBSD as a bug report. + _Do not_ just send the diffs to the {freebsd-hackers} or they will get lost! We greatly appreciate your submission (this is a volunteer project!); because we are busy, we may not be able to address it immediately, but it will remain in the PR database until we do. + Indicate your submission by including `[PATCH]` in the synopsis of the report. + + If you feel it appropriate (for example you have added, deleted, or renamed files), bundle your changes into a `tar` file. + + If your change is of a potentially sensitive nature, such as if you are unsure of copyright issues governing its further distribution then you should send it to {core-email} directly rather than submitting as a bug report. + The {core-email} reaches a much smaller group of people who do much of the day-to-day work on FreeBSD. + Note that this group is also _very busy_ and so you should only send mail to them where it is truly necessary. + + Please refer to man:intro[9] and man:style[9] for some information on coding style. + We would appreciate it if you were at least aware of this information before submitting code. + + === New Code or Major Value-Added Packages + + In the case of a significant contribution of a large body work, or the addition of an important new feature to FreeBSD, it becomes almost always necessary to either send changes as tar files or upload them to a web or FTP site for other people to access. + If you do not have access to a web or FTP site, ask on an appropriate FreeBSD mailing list for someone to host the changes for you. + + When working with large amounts of code, the touchy subject of copyrights also invariably comes up. + FreeBSD prefers free software licenses such as BSD or ISC. + Copyleft licenses such as GPLv2 are sometimes permitted. + The complete listing can be found on the link:https://www.FreeBSD.org/internal/software-license/[core team licensing policy] page. + + === Money or Hardware + + We are always very happy to accept donations to further the cause of the FreeBSD Project and, in a volunteer effort like ours, a little can go a long way! + Donations of hardware are also very important to expanding our list of supported peripherals since we generally lack the funds to buy such items ourselves. + + [[donations]] + ==== Donating Funds + + The https://www.freebsdfoundation.org[FreeBSD Foundation] is a non-profit, tax-exempt foundation established to further the goals of the FreeBSD Project. + As a 501(c)3 entity, the Foundation is generally exempt from US federal income tax as well as Colorado State income tax. + Donations to a tax-exempt entity are often deductible from taxable federal income. + + Donations may be sent in check form to: + + [.address] + **** + The FreeBSD Foundation + + 3980 Broadway Street + + STE #103-107 + + Boulder CO 80304 + + USA + **** + + The FreeBSD Foundation is also able to accept https://www.freebsdfoundation.org/donate/[online donations] through various payment options. + + More information about the FreeBSD Foundation can be found in https://people.FreeBSD.org/~jdp/foundation/announcement.html[The FreeBSD Foundation -- an Introduction]. + To contact the Foundation by email, write to mailto:info@FreeBSDFoundation.org[info@FreeBSDFoundation.org]. + + ==== Donating Hardware + + The FreeBSD Project happily accepts donations of hardware that it can find good use for. + If you are interested in donating hardware, please contact the link:https://www.FreeBSD.org/donations/[Donations Liaison Office]. + + [[ports-contributing]] + == Contributing to ports + + [[adopt-port]] + === Adopting an unmaintained port + + ==== Choosing an unmaintained port + + Taking over maintainership of ports that are unmaintained is a great way to get involved. + Unmaintained ports are only updated and fixed when somebody volunteers to work on them. + There are a large number of unmaintained ports. + It is a good idea to start with adopting a port that you use regularly. + + Unmaintained ports have their `MAINTAINER` set to `ports@FreeBSD.org`. + Many unmaintained ports can have pending updates, this can be seen at the https://portscout.freebsd.org/ports@freebsd.org.html[FreeBSD Ports distfile scanner]. + + On https://portsfallout.com/fallout?port=&maintainer=ports%40FreeBSD.org[PortsFallout] can be seen a list of unmaintained ports with errors. + + Some ports affect a large number of others due to dependencies and secondary port relationships. + Generally, we want people to have some experience before they maintain such ports. + + You can find out whether or not a port has dependencies or secondary ports by looking at a primary index of ports called [.filename]#INDEX#. + (The name of the file varies by release of FreeBSD; for instance, [.filename]#INDEX-13#.) Some ports have conditional dependencies that are not included in a default [.filename]#INDEX# build. + We expect you to be able to recognize such ports by looking through other ports' [.filename]#Makefile#'s. + + ==== How to adopt the port + + First make sure you understand your crossref:contributing[maintain-port, The challenge for port maintainers]. + Also read the extref:{porters-handbook}[Porter's Handbook]. + _Please do not commit yourself to more than you feel you can comfortably handle._ + + You may request maintainership of any unmaintained port as soon as you wish. + Simply set `MAINTAINER` to your own email address and send a PR (Problem Report) with the change. + If the port has build errors or needs updating, you may wish to include any other changes in the same PR. + This will help because many committers are less willing to assign maintainership to someone who does not have a known track record with FreeBSD. + Submitting PRs that fix build errors or update ports are the best ways to establish one. + + File your PR with Product `Ports & Packages`. + A committer will examine your PR, commit the changes, and finally close the PR. + Sometimes this process can take a little while (committers are volunteers, too :). + + [[maintain-port]] + === The challenge for port maintainers + + This section will give you an idea of why ports need to be maintained and outline the responsibilities of a port maintainer. + + [[why-maintenance]] + ==== Why ports require maintenance + + Creating a port is a once-off task. + Ensuring that a port is up to date and continues to build and run requires an ongoing maintenance effort. + Maintainers are the people who dedicate some of their time to meeting these goals. + + The foremost reason ports need maintenance is to bring the latest and greatest in third party software to the FreeBSD community. + An additional challenge is to keep individual ports working within the Ports Collection framework as it evolves. + + As a maintainer, you will need to manage the following challenges: + + * *New software versions and updates.* New versions and updates of existing ported software become available all the time, and these need to be incorporated into the Ports Collection to provide up-to-date software. + + * *Changes to dependencies.* If significant changes are made to the dependencies of your port, it may need to be updated so that it will continue to work correctly. + + * *Changes affecting dependent ports.* If other ports depend on a port that you maintain, changes to your port may require coordination with other maintainers. + + * *Interaction with other users, maintainers and developers.* Part of being a maintainer is taking on a support role. + You are not expected to provide general support (but we welcome it if you choose to do so). What you should provide is a point of coordination for FreeBSD-specific issues regarding your ports. + + * *Bug hunting.* A port may be affected by bugs which are specific to FreeBSD. + You will need to investigate, find, and fix these bugs when they are reported. + Thoroughly testing a port to identify problems before they make their way into the Ports Collection is even better. + + * *Changes to ports infrastructure and policy.* Occasionally the systems that are used to build ports and packages are updated or a new recommendation affecting the infrastructure is made. + You should be aware of these changes in case your ports are affected and require updating. + + * *Changes to the base system.* FreeBSD is under constant development. + Changes to software, libraries, the kernel or even policy changes can cause flow-on change requirements to ports. + + ==== Maintainer responsibilities + + ===== Keep your ports up to date + + This section outlines the process to follow to keep your ports up to date. + + This is an overview. + More information about upgrading a port is available in the extref:{porters-handbook}[Porter's Handbook]. + + [.procedure] + ==== + . Watch for updates + + + Monitor the upstream vendor for new versions, updates and security fixes for the software. + Announcement mailing lists or news web pages are useful for doing this. + Sometimes users will contact you and ask when your port will be updated. + If you are busy with other things or for any reason just cannot update it at the moment, ask if they will help you by submitting an update. + + + You may also receive automated email from the `FreeBSD Ports Version Check` informing you that a newer version of your port's distfile is available. + More information about that system (including how to stop future emails) will be provided in the message. + . Incorporate changes + + + When they become available, incorporate the changes into the port. + You need to be able to generate a patch between the original port and your updated port. + . Review and test + + + Thoroughly review and test your changes: + + ** Build, install and test your port on as many platforms and architectures as you can. + It is common for a port to work on one branch or platform and fail on another. + ** Make sure your port's dependencies are complete. + The recommended way of doing this is by installing your own ports tinderbox. + See crossref:contributing[resources, Resources for ports maintainers and contributors] for more information. + ** Check that the packing list is up to date. + This involves adding in any new files and directories and removing unused entries. + ** Verify your port using man:portlint[1] as a guide. + See crossref:contributing[resources, Resources for ports maintainers and contributors] for important information about using portlint. + ** Consider whether changes to your port might cause any other ports to break. + If this is the case, coordinate the changes with the maintainers of those ports. + This is especially important if your update changes the shared library version; in this case, at the very least, the dependent ports will need to get a `PORTREVISION` bump so that they will automatically be upgraded by automated tools such as package:ports-mgmt/poudriere[]. + + . Submit changes + + + Send your update by submitting a PR with an explanation of the changes and a patch containing the differences between the original port and the updated one. + Please refer to extref:{problem-reports}[Writing FreeBSD Problem Reports] for information on how to write a really good PR. + + + [NOTE] + ====== + Please do not submit a man:shar[1] archive of the entire port; instead, use man:git-format-patch[1] or man:diff[1] `-ruN`. + In this way, committers can much more easily see exactly what changes are being made. +-The Porter's Handbook section on extref:{porters-handbook}[Upgrading, port-upgrading] has more information. ++The Porter's Handbook section on extref:{porters-handbook}upgrading[Upgrading, port-upgrading] has more information. + ====== + . Wait + + + At some stage a committer will deal with your PR. + It may take minutes, or it may take one or two weeks - so please be patient. + If it takes any longer, please seek for help on mailing lists ({freebsd-ports}), IRC: #bsdports on EFNet or #freebsd-ports on Libera for example. + . Give feedback + + + If a committer finds a problem with your changes, they will most likely refer it back to you. + A prompt response will help get your PR committed faster, and is better for maintaining a thread of conversation when trying to resolve any problems. + . And Finally + + + Your changes will be committed and your port will have been updated. + The PR will then be closed by the committer. + That is it! + ==== + + ===== Ensure your ports continue to build correctly + + This section is about discovering and fixing problems that stop your ports from building correctly. + + FreeBSD only guarantees that the Ports Collection works on the `-STABLE` branches. + In theory, you should be able to get by with running the latest release of each stable branch (since the ABIs are not supposed to change) but if you can run the branch, that is even better. + + Since the majority of FreeBSD installations run on PC-compatible machines (what is termed the `i386` architecture), we expect you to keep the port working on that architecture. + We prefer that ports also work on the `amd64` architecture running native. + It is completely fair to ask for help if you do not have one of these machines. + + [NOTE] + ==== + The usual failure modes for non-`x86` machines are that the original programmers assumed that, for instance, pointers are `int`-s, or that a relatively lax older gcc compiler was being used. + More and more, application authors are reworking their code to remove these assumptions - but if the author is not actively maintaining their code, you may need to do this yourself. + ==== + + These are the tasks you need to perform to ensure your port is able to be built: + + [.procedure] + ==== + . Watch for build failures + + + Check your mail for mail from `pkg-fallout@FreeBSD.org` and the http://portscout.FreeBSD.org[distfiles scanner] to see if any of the port which are failing to build are out of date. + . Collect information + + + Once you are aware of a problem, collect information to help you fix it. + Build errors reported by `pkg-fallout` are accompanied by logs which will show you where the build failed. + If the failure was reported to you by a user, ask them to send you information which may help in diagnosing the problem, such as: + + ** Build logs + ** The commands and options used to build the port (including options set in [.filename]#/etc/make.conf#) + ** A list of packages installed on their system as shown by man:pkg-info[8] + ** The version of FreeBSD they are running as shown by man:uname[1] `-a` + ** When their ports collection was last updated + ** When their ports tree and [.filename]#INDEX# was last updated + + . Investigate and find a solution + + + Unfortunately there is no straightforward process to follow to do this. + Remember, though: if you are stuck, ask for help! + The {freebsd-ports} is a good place to start, and the upstream developers are often very helpful. + . Submit changes + + + Just as with updating a port, you should now incorporate changes, review and test, submit your changes in a PR, and provide feedback if required. + . Send patches to upstream authors + + + In some cases, you will have to make patches to the port to make it run on FreeBSD. + Some (but not all) upstream authors will accept such patches back into their code for the next release. + If so, this may even help their users on other BSD-based systems as well and perhaps save duplicated effort. + Please consider sending any applicable patches to the authors as a courtesy. + ==== + + ===== Investigate bug reports and PRs related to your port + + This section is about discovering and fixing bugs. + + FreeBSD-specific bugs are generally caused by assumptions about the build and runtime environments that do not apply to FreeBSD. + You are less likely to encounter a problem of this type, but it can be more subtle and difficult to diagnose. + + These are the tasks you need to perform to ensure your port continues to work as intended: + + [.procedure] + ==== + . Respond to bug reports + + + Bugs may be reported to you through email via the https://bugs.FreeBSD.org/search/[Problem Report database]. + Bugs may also be reported directly to you by users. + + + You should respond to PRs and other reports within 14 days, but please try not to take that long. + Try to respond as soon as possible, even if it is just to say you need some more time before you can work on the PR. + + + If you have not responded after 14 days, any committer may commit from a PR that you have not responded to via a `maintainer-timeout`. + . Collect information + + + If the person reporting the bug has not also provided a fix, you need to collect the information that will allow you to generate one. + + + If the bug is reproducible, you can collect most of the required information yourself. + If not, ask the person who reported the bug to collect the information for you, such as: + + ** A detailed description of their actions, expected program behavior and actual behavior + ** Copies of input data used to trigger the bug + ** Information about their build and execution environment - for example, a list of installed packages and the output of man:env[1] + ** Core dumps + ** Stack traces + + . Eliminate incorrect reports + + + Some bug reports may be incorrect. + For example, the user may have simply misused the program; or their installed packages may be out of date and require updating. + Sometimes a reported bug is not specific to FreeBSD. + In this case report the bug to the upstream developers. + If the bug is within your capabilities to fix, you can also patch the port so that the fix is applied before the next upstream release. + . Find a solution + + + As with build errors, you will need to sort out a fix to the problem. + Again, remember to ask if you are stuck! + . Submit or approve changes + + + Just as with updating a port, you should now incorporate changes, review and test, and submit your changes in a PR (or send a follow-up if a PR already exists for the problem). + If another user has submitted changes in the PR, you can also send a follow-up saying whether or not you approve the changes. + ==== + + ===== Providing support + + Part of being a maintainer is providing support - not for the software in general - but for the port and any FreeBSD-specific quirks and problems. + Users may contact you with questions, suggestions, problems and patches. + Most of the time their correspondence will be specific to FreeBSD. + + Occasionally you may have to invoke your skills in diplomacy, and kindly point users seeking general support to the appropriate resources. + Less frequently you will encounter a person asking why the `RPMS` are not up to date or how can they get the software to run under Foo Linux. + Take the opportunity to tell them that your port is up to date (if it is, of course!), and suggest that they try FreeBSD. + + Sometimes users and developers will decide that you are a busy person whose time is valuable and do some of the work for you. + For example, they might: + + * submit a PR or send you patches to update your port, + * investigate and perhaps provide a fix to a PR, or + * otherwise submit changes to your port. + + In these cases your main obligation is to respond in a timely manner. + Again, the timeout for non-responsive maintainers is 14 days. + After this period changes may be committed unapproved. + They have taken the trouble to do this for you; so please try to at least respond promptly. + Then review, approve, modify or discuss their changes with them as soon as possible. + + If you can make them feel that their contribution is appreciated (and it should be) you will have a better chance persuading them to do more things for you in the future :-). + + [[fix-broken]] + === Finding and fixing a broken port + + There are some really good places to find a port that needs some attention. + + You can use the https://bugs.freebsd.org/search[web interface] to the Problem Report database to search through and view unresolved PRs. + The majority of ports PRs are updates, but with a little searching and skimming over synopses you should be able to find something interesting to work on. + + https://portsfallout.com/[PortsFallout] shows port issues gathered from the FreeBSD package building. + + It is OK to send changes for a maintained port as well, but remember to ask the maintainer in case they are already working on the problem. + + Once you have found a bug or problem, collect information, investigate and fix! + If there is an existing PR, follow up to that. + Otherwise create a new PR. + Your changes will be reviewed and, if everything checks out, committed. + + [[mortal-coil]] + === When to call it quits + + As your interests and commitments change, you may find that you no longer have time to continue some (or all) of your ports contributions. + That is fine! + Please let us know if you are no longer using a port or have otherwise lost time or interest in being a maintainer. + In this way we can go ahead and allow other people to try to work on existing problems with the port without waiting for your response. + Remember, FreeBSD is a volunteer project, so if maintaining a port is no fun any more, it is probably time to let someone else do it! + + In any case, the Ports Management Team (`portmgr`) reserves the right to reset your maintainership if you have not actively maintained your port in some time. + (Currently, this is set to 3 months.) + By this, we mean that there are unresolved problems or pending updates that have not been worked on during that time. + + [[resources]] + === Resources for ports maintainers and contributors + + The extref:{porters-handbook}[Porter's Handbook] is your hitchhiker's guide to the ports system. + Keep it handy! + + extref:{problem-reports}[Writing FreeBSD Problem Reports] describes how to best formulate and submit a PR. + In 2005 more than eleven thousand ports PRs were submitted! + Following this article will greatly assist us in reducing the time needed to handle your PRs. + + The https://bugs.freebsd.org/bugzilla/query.cgi[Problem Report database]. + + The http://portscout.FreeBSD.org[FreeBSD Ports distfile scanner (portscout)] can show you ports for which the distfiles are not fetchable. + You can check on your own ports or use it to find ports that need their `MASTER_SITES` updated. + + package:ports-mgmt/poudriere[] is the most thorough way to test a port through the entire cycle of installation, packaging, and deinstallation. + Documentation is located at the https://github.com/freebsd/poudriere[poudriere GitHub repository] + + man:portlint[1] is an application which can be used to verify that your port conforms to many important stylistic and functional guidelines. + portlint is a simple heuristic application, so you should use it __only as a guide__. + If portlint suggests changes which seem unreasonable, consult the extref:{porters-handbook}[Porter's Handbook] or ask for advice. + + The {freebsd-ports} is for general ports-related discussion. + It is a good place to ask for help. + You can link:https://lists.freebsd.org/[subscribe, or read and search the list archives]. + Reading the archives of the {freebsd-ports-bugs} and the {svn-ports-head} may also be of interest. + + https://portsfallout.com/[PortsFallout] is a place to help in searching for the https://lists.freebsd.org/archives/freebsd-pkg-fallout/[FreeBSD package-fallout archive]. + + [[ideas-contributing]] + == Getting Started in Other Areas + + Looking for something interesting to get started that is not mentioned elsewhere in this article? + The FreeBSD Project has several Wiki pages containing areas within which new contributors can get ideas on how to get started. + + The https://wiki.freebsd.org/JuniorJobs[Junior Jobs] page has a list of projects that might be of interest to people just getting started in FreeBSD, and want to work on interesting things to get their feet wet. + + The https://wiki.freebsd.org/IdeasPage[Ideas Page] contains various "nice to have" or "interesting" things to work on in the Project. +diff --git a/documentation/content/en/articles/filtering-bridges/_index.adoc b/documentation/content/en/articles/filtering-bridges/_index.adoc +index 1ef6adc085..64c7bd781b 100644 +--- a/documentation/content/en/articles/filtering-bridges/_index.adoc ++++ b/documentation/content/en/articles/filtering-bridges/_index.adoc +@@ -1,294 +1,294 @@ + --- + title: Filtering Bridges + authors: + - author: Alex Dupre + email: ale@FreeBSD.org + description: Configuring firewalls and filtering on FreeBSD hosts acting as bridges rather than routers + trademarks: ["freebsd", "3com", "intel", "general"] + tags: ["network", "filtering", "bridges", "FreeBSD"] + --- + + = Filtering Bridges + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/filtering-bridges/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + Often it is useful to divide one physical network (like an Ethernet) into two separate segments without having to create subnets, and use a router to link them together. + The device that connects the two networks in this way is called a bridge. + A FreeBSD system with two network interfaces is enough to act as a bridge. + + A bridge works by scanning the addresses of MAC level (Ethernet addresses) of the devices connected to each of its network interfaces and then forwarding the traffic between the two networks only if the source and the destination are on different segments. + Under many points of view a bridge is similar to an Ethernet switch with only two ports. + + ''' + + toc::[] + + [[filtering-bridges-why]] + == Why use a filtering bridge? + + More and more frequently, thanks to the lowering costs of broad band Internet connections (xDSL) and also because of the reduction of available IPv4 addresses, many companies are connected to the Internet 24 hours on 24 and with few (sometimes not even a power of 2) IP addresses. + In these situations it is often desirable to have a firewall that filters incoming and outgoing traffic from and towards Internet, but a packet filtering solution based on router may not be applicable, either due to subnetting issues, the router is owned by the connectivity supplier (ISP), or because it does not support such functionalities. + In these scenarios the use of a filtering bridge is highly advised. + + A bridge-based firewall can be configured and inserted between the xDSL router and your Ethernet hub/switch without any IP numbering issues. + + [[filtering-bridges-how]] + == How to Install + + Adding bridge functionalities to a FreeBSD system is not difficult. + Since 4.5 release it is possible to load such functionalities as modules instead of having to rebuild the kernel, simplifying the procedure a great deal. + In the following subsections I will explain both installation ways. + + [IMPORTANT] + ==== + _Do not_ follow both instructions: a procedure _excludes_ the other one. + Select the best choice according to your needs and abilities. + ==== + + Before going on, be sure to have at least two Ethernet cards that support the promiscuous mode for both reception and transmission, since they must be able to send Ethernet packets with any address, not just their own. + Moreover, to have a good throughput, the cards should be PCI bus mastering cards. + The best choices are still the Intel EtherExpress(TM) Pro, followed by the 3Com(R) 3c9xx series. + To simplify the firewall configuration it may be useful to have two cards of different manufacturers (using different drivers) to distinguish clearly which interface is connected to the router and which to the inner network. + + [[filtering-bridges-kernel]] + === Kernel Configuration + + So you have decided to use the older but well tested installation method. + To begin, you have to add the following rows to your kernel configuration file: + + [.programlisting] + .... + options BRIDGE + options IPFIREWALL + options IPFIREWALL_VERBOSE + .... + + The first line is to compile the bridge support, the second one is the firewall and the third one is the logging functions of the firewall. + + Now it is necessary to build and install the new kernel. +-You may find detailed instructions in the extref:{handbook}[Building and Installing a Custom Kernel, kernelconfig-building] section of the FreeBSD Handbook. ++You may find detailed instructions in the extref:{handbook}kernelconfig[Building and Installing a Custom Kernel, kernelconfig-building] section of the FreeBSD Handbook. + + [[filtering-bridges-modules]] + === Modules Loading + + If you have chosen to use the new and simpler installation method, the only thing to do now is add the following row to [.filename]#/boot/loader.conf#: + + [.programlisting] + .... + bridge_load="YES" + .... + + In this way, during the system startup, the [.filename]#bridge.ko# module will be loaded together with the kernel. + It is not required to add a similar row for the [.filename]#ipfw.ko# module, since it will be loaded automatically after the execution of the steps in the following section. + + [[filtering-bridges-finalprep]] + == Final Preparation + + Before rebooting to load the new kernel or the required modules (according to the previously chosen installation method), you have to make some changes to the [.filename]#/etc/rc.conf# configuration file. + The default rule of the firewall is to reject all IP packets. + Initially we will set up an `open` firewall, to verify its operation without any issue related to packet filtering (in case you are going to execute this procedure remotely, such configuration will avoid you to remain isolated from the network). + Put these lines in [.filename]#/etc/rc.conf#: + + [.programlisting] + .... + firewall_enable="YES" + firewall_type="open" + firewall_quiet="YES" + firewall_logging="YES" + .... + + The first row will enable the firewall (and will load the module [.filename]#ipfw.ko# if it is not compiled in the kernel), the second one to set up it in `open` mode (as explained in [.filename]#/etc/rc.firewall#), the third one to not show rules loading and the fourth one to enable logging support. + + About the configuration of the network interfaces, the most used way is to assign an IP to only one of the network cards, but the bridge will work equally even if both interfaces or none has a configured IP. + In the last case (IP-less) the bridge machine will be still more hidden, as inaccessible from the network: to configure it, you have to login from console or through a third network interface separated from the bridge. + Sometimes, during the system startup, some programs require network access, say for domain resolution: in this case it is necessary to assign an IP to the external interface (the one connected to Internet, where DNS server resides), since the bridge will be activated at the end of the startup procedure. + It means that the [.filename]#fxp0# interface (in our case) must be mentioned in the ifconfig section of the [.filename]#/etc/rc.conf# file, while the [.filename]#xl0# is not. + Assigning an IP to both the network cards does not make much sense, unless, during the start procedure, applications should access to services on both Ethernet segments. + + There is another important thing to know. + When running IP over Ethernet, there are actually two Ethernet protocols in use: one is IP, the other is ARP. + ARP does the conversion of the IP address of a host into its Ethernet address (MAC layer). + To allow the communication between two hosts separated by the bridge, it is necessary that the bridge will forward ARP packets. + Such protocol is not included in the IP layer, since it exists only with IP over Ethernet. + The FreeBSD firewall filters exclusively on the IP layer and therefore all non-IP packets (ARP included) will be forwarded without being filtered, even if the firewall is configured to not permit anything. + + Now it is time to reboot the system and use it as before: there will be some new messages about the bridge and the firewall, but the bridge will not be activated and the firewall, being in `open` mode, will not avoid any operations. + + If there are any problems, you should sort them out now before proceeding. + + [[filtering-bridges-enabling]] + == Enabling the Bridge + + At this point, to enable the bridge, you have to execute the following commands (having the shrewdness to replace the names of the two network interfaces [.filename]#fxp0# and [.filename]#xl0# with your own ones): + + [source,shell] + .... + # sysctl net.link.ether.bridge.config=fxp0:0,xl0:0 + # sysctl net.link.ether.bridge.ipfw=1 + # sysctl net.link.ether.bridge.enable=1 + .... + + The first row specifies which interfaces should be activated by the bridge, the second one will enable the firewall on the bridge and finally the third one will enable the bridge. + + At this point you should be able to insert the machine between two sets of hosts without compromising any communication abilities between them. + If so, the next step is to add the `net.link.ether.bridge._[blah]_=_[blah]_` portions of these rows to the [.filename]#/etc/sysctl.conf# file, to have them execute at startup. + + [[filtering-bridges-ipfirewall]] + == Configuring The Firewall + + Now it is time to create your own file with custom firewall rules, to secure the inside network. + There will be some complication in doing this because not all of the firewall functionalities are available on bridged packets. + Furthermore, there is a difference between the packets that are in the process of being forwarded and packets that are being received by the local machine. + In general, incoming packets are run through the firewall only once, not twice as is normally the case; in fact they are filtered only upon receipt, so rules that use `out` or `xmit` will never match. + Personally, I use `in via` which is an older syntax, but one that has a sense when you read it. + Another limitation is that you are restricted to use only `pass` or `drop` commands for packets filtered by a bridge. + Sophisticated things like `divert`, `forward` or `reject` are not available. + Such options can still be used, but only on traffic to or from the bridge machine itself (if it has an IP address). + + New in FreeBSD 4.0, is the concept of stateful filtering. + This is a big improvement for UDP traffic, which typically is a request going out, followed shortly thereafter by a response with the same set of IP addresses and port numbers (but with source and destination reversed, of course). + For firewalls that have no statekeeping, there is almost no way to deal with this sort of traffic as a single session. + But with a firewall that can "remember" an outgoing UDP packet and, for the next few minutes, allow a response, handling UDP services is trivial. + The following example shows how to do it. + It is possible to do the same thing with TCP packets. + This allows you to avoid some denial of service attacks and other nasty tricks, but it also typically makes your state table grow quickly in size. + + Let's look at an example setup. + Note first that at the top of [.filename]#/etc/rc.firewall# there are already standard rules for the loopback interface [.filename]#lo0#, so we should not have to care for them anymore. + Custom rules should be put in a separate file (say [.filename]#/etc/rc.firewall.local#) and loaded at system startup, by modifying the row of [.filename]#/etc/rc.conf# where we defined the `open` firewall: + + [.programlisting] + .... + firewall_type="/etc/rc.firewall.local" + .... + + [IMPORTANT] + ==== + You have to specify the _full_ path, otherwise it will not be loaded with the risk to remain isolated from the network. + ==== + + For our example imagine to have the [.filename]#fxp0# interface connected towards the outside (Internet) and the [.filename]#xl0# towards the inside (LAN). The bridge machine has the IP `1.2.3.4` (it is not possible that your ISP can give you an address quite like this, but for our example it is good). + + [.programlisting] + .... + # Things that we have kept state on before get to go through in a hurry + add check-state + + # Throw away RFC 1918 networks + add drop all from 10.0.0.0/8 to any in via fxp0 + add drop all from 172.16.0.0/12 to any in via fxp0 + add drop all from 192.168.0.0/16 to any in via fxp0 + + # Allow the bridge machine to say anything it wants + # (if the machine is IP-less do not include these rows) + add pass tcp from 1.2.3.4 to any setup keep-state + add pass udp from 1.2.3.4 to any keep-state + add pass ip from 1.2.3.4 to any + + # Allow the inside hosts to say anything they want + add pass tcp from any to any in via xl0 setup keep-state + add pass udp from any to any in via xl0 keep-state + add pass ip from any to any in via xl0 + + # TCP section + # Allow SSH + add pass tcp from any to any 22 in via fxp0 setup keep-state + # Allow SMTP only towards the mail server + add pass tcp from any to relay 25 in via fxp0 setup keep-state + # Allow zone transfers only by the secondary name server [dns2.nic.it] + add pass tcp from 193.205.245.8 to ns 53 in via fxp0 setup keep-state + # Pass ident probes. It is better than waiting for them to timeout + add pass tcp from any to any 113 in via fxp0 setup keep-state + # Pass the "quarantine" range + add pass tcp from any to any 49152-65535 in via fxp0 setup keep-state + + # UDP section + # Allow DNS only towards the name server + add pass udp from any to ns 53 in via fxp0 keep-state + # Pass the "quarantine" range + add pass udp from any to any 49152-65535 in via fxp0 keep-state + + # ICMP section + # Pass 'ping' + add pass icmp from any to any icmptypes 8 keep-state + # Pass error messages generated by 'traceroute' + add pass icmp from any to any icmptypes 3 + add pass icmp from any to any icmptypes 11 + + # Everything else is suspect + add drop log all from any to any + .... + + Those of you who have set up firewalls before may notice some things missing. + In particular, there are no anti-spoofing rules, in fact we did _not_ add: + + [.programlisting] + .... + add deny all from 1.2.3.4/8 to any in via fxp0 + .... + + That is, drop packets that are coming in from the outside claiming to be from our network. + This is something that you would commonly do to be sure that someone does not try to evade the packet filter, by generating nefarious packets that look like they are from the inside. + The problem with that is that there is _at least_ one host on the outside interface that you do not want to ignore: the router. + But usually, the ISP anti-spoofs at their router, so we do not need to bother that much. + + The last rule seems to be an exact duplicate of the default rule, that is, do not let anything pass that is not specifically allowed. + But there is a difference: all suspected traffic will be logged. + + There are two rules for passing SMTP and DNS traffic towards the mail server and the name server, if you have them. + Obviously the whole rule set should be flavored to personal taste, this is only a specific example (rule format is described accurately in the man:ipfw[8] man page). + Note that for "relay" and "ns" to work, name service lookups must work _before_ the bridge is enabled. + This is an example of making sure that you set the IP on the correct network card. + Alternatively it is possible to specify the IP address instead of the host name (required if the machine is IP-less). + + People that are used to setting up firewalls are probably also used to either having a `reset` or a `forward` rule for ident packets (TCP port 113). + Unfortunately, this is not an applicable option with the bridge, so the best thing is to simply pass them to their destination. + As long as that destination machine is not running an ident daemon, this is relatively harmless. + The alternative is dropping connections on port 113, which creates some problems with services like IRC (the ident probe must timeout). + + The only other thing that is a little weird that you may have noticed is that there is a rule to let the bridge machine speak, and another for internal hosts. + Remember that this is because the two sets of traffic will take different paths through the kernel and into the packet filter. + The inside net will go through the bridge, while the local machine will use the normal IP stack to speak. + Thus the two rules to handle the different cases. + The `in via fxp0` rules work for both paths. + In general, if you use `in via` rules throughout the filter, you will need to make an exception for locally generated packets, because they did not come in via any of our interfaces. + + [[filtering-bridges-contributors]] + == Contributors + + Many parts of this article have been taken, updated and adapted from an old text about bridging, edited by Nick Sayer. + A pair of inspirations are due to an introduction on bridging by Steve Peterson. + + A big thanks to Luigi Rizzo for the implementation of the bridge code in FreeBSD and for the time he has dedicated to me answering all of my related questions. + + A thanks goes out also to Tom Rhodes who looked over my job of translation from Italian (the original language of this article) into English. +diff --git a/documentation/content/en/articles/freebsd-questions/_index.adoc b/documentation/content/en/articles/freebsd-questions/_index.adoc +index 2869c685ab..052c202ffd 100644 +--- a/documentation/content/en/articles/freebsd-questions/_index.adoc ++++ b/documentation/content/en/articles/freebsd-questions/_index.adoc +@@ -1,223 +1,223 @@ + --- + title: How to get Best Results from the FreeBSD-questions Mailing List + authors: + - author: Greg Lehey + email: grog@FreeBSD.org + description: How to get Best Results from the FreeBSD-questions Mailing List + trademarks: ["freebsd", "microsoft", "opengroup", "qualcomm", "general"] + tags: ["questions", "mailing", "FreeBSD"] + --- + + = How to get Best Results from the FreeBSD-questions Mailing List + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/freebsd-questions/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + This document provides useful information for people looking to prepare an e-mail to the FreeBSD-questions mailing list. + Advice and hints are given that will maximize the chance that the reader will receive useful replies. + + This document is regularly posted to the FreeBSD-questions mailing list. + + ''' + + toc::[] + + == Introduction + + `FreeBSD-questions` is a mailing list maintained by the FreeBSD project to help people who have questions about the normal use of FreeBSD. + Another group, `FreeBSD-hackers`, discusses more advanced questions such as future development work. + + [NOTE] + ==== + The term "hacker" has nothing to do with breaking into other people's computers. + The correct term for the latter activity is "cracker", but the popular press has not found out yet. + The FreeBSD hackers disapprove strongly of cracking security, and have nothing to do with it. + For a longer description of hackers, see Eric Raymond's http://www.catb.org/~esr/faqs/hacker-howto.html[How To Become A Hacker] + ==== + + This is a regular posting aimed to help both those seeking advice from FreeBSD-questions (the "newcomers"), and also those who answer the questions (the "hackers"). + + Inevitably there is some friction, which stems from the different viewpoints of the two groups. + The newcomers accuse the hackers of being arrogant, stuck-up, and unhelpful, while the hackers accuse the newcomers of being stupid, unable to read plain English, and expecting everything to be handed to them on a silver platter. + Of course, there is an element of truth in both these claims, but for the most part these viewpoints come from a sense of frustration. + + In this document, I would like to do something to relieve this frustration and help everybody get better results from FreeBSD-questions. + In the following section, I recommend how to submit a question; after that, we will look at how to answer one. + + == How to Subscribe to FreeBSD-questions + + FreeBSD-questions is a mailing list, so you need mail access. + Point your WWW browser to the {freebsd-questions}. + In the section titled "Subscribe or unsubscribe online" fill in the "Your email address" field and hit "Subscribe". + Or send an email to freebsd-questions+subscribe@freebsd.org. + + You will receive a confirmation message from mlmmj; follow the included instructions to complete your subscription. + + == How to Unsubscribe from FreeBSD-questions + + Point your WWW browser to the {freebsd-questions}. + In the section titled "Subscribe or unsubscribe online" fill in the "Your email address" field and hit "Unsubscribe". + Or send an email to freebsd-questions+unsubscribe@freebsd.org. + + A confirmation message will be sent to you from mlmmj; follow the included instructions to finish unsubscribing. + + == Should I ask `-questions` or `-hackers`? + + Two mailing lists handle general questions about FreeBSD, `FreeBSD-questions` and `FreeBSD-hackers`. + In some cases, it is not really clear which group you should ask. + The following criteria should help for 99% of all questions, however: + + . If the question is of a general nature, ask `FreeBSD-questions`. Examples might be questions about installing FreeBSD or the use of a particular UNIX(R) utility. + . If you think the question relates to a bug, but you are not sure, or you do not know how to look for it, send the message to `FreeBSD-questions`. + . If the question relates to a bug, and you are _sure_ that it is a bug (for example, you can pinpoint the place in the code where it happens, and you maybe have a fix), then send the message to `FreeBSD-hackers`. + . If the question relates to enhancements to FreeBSD, and you can make suggestions about how to implement them, then send the message to `FreeBSD-hackers`. + +-There are also a number of other extref:{handbook}[specialized mailing lists, eresources-mail], which caters to more specific interests. ++There are also a number of other extref:{handbook}eresources[specialized mailing lists, eresources-mail], which caters to more specific interests. + The criteria above still apply, and it is in your interest to stick to them, since you are more likely to get good results that way. + + == Before Submitting a Question + + You can (and should) do some things yourself before asking a question on one of the mailing lists: + + * Try solving the problem on your own. If you post a question which shows that you have tried to solve the problem, your question will generally attract more positive attention from people reading it. Trying to solve the problem yourself will also enhance your understanding of FreeBSD, and will eventually let you use your knowledge to help others by answering questions posted to the mailing lists. + * Read the manual pages, and the FreeBSD documentation (either installed in [.filename]#/usr/doc# or accessible via WWW at http://www.FreeBSD.org[http://www.FreeBSD.org]), especially the extref:{handbook}[handbook] and the extref:{faq}[FAQ]. +-* Browse and/or search the archives for the mailing list, to see if your question or a similar one has been asked (and possibly answered) on the list. You can browse and/or search the mailing list archives at https://www.FreeBSD.org/mail[https://www.FreeBSD.org/mail] and https://www.FreeBSD.org/search/#mailinglists[https://www.FreeBSD.org/search/#mailinglists] respectively. ++* Browse and/or search the archives for the mailing list, to see if your question or a similar one has been asked (and possibly answered) on the list. You can browse and/or search the mailing list archives at https://www.FreeBSD.org/mail[https://www.FreeBSD.org/mail] and https://www.FreeBSD.org/search/#mailinglists[https://www.FreeBSD.org/search/#mailinglists] respectively. + * Use a search engine such as http://www.google.com[Google] or http://www.yahoo.com[Yahoo] to find answers to your question. + + == How to Submit a Question + + When submitting a question to FreeBSD-questions, consider the following points: + + * Remember that nobody gets paid for answering a FreeBSD question. They do it of their own free will. You can influence this free will positively by submitting a well-formulated question supplying as much relevant information as possible. You can influence this free will negatively by submitting an incomplete, illegible, or rude question. It is perfectly possible to send a message to FreeBSD-questions and not get an answer even if you follow these rules. It is much more possible to not get an answer if you do not. In the rest of this document, we will look at how to get the most out of your question to FreeBSD-questions. + * Not everybody who answers FreeBSD questions reads every message: they look at the subject line and decide whether it interests them. Clearly, it is in your interest to specify a subject. "FreeBSD problem" or "Help" are not enough. If you provide no subject at all, many people will not bother reading it. If your subject is not specific enough, the people who can answer it may not read it. + * Format your message so that it is legible, and PLEASE DO NOT SHOUT!!!!!. We appreciate that a lot of people do not speak English as their first language, and we try to make allowances for that, but it is really painful to try to read a message written full of typos or without any line breaks. + + + Do not underestimate the effect that a poorly formatted mail message has, not just on the FreeBSD-questions mailing list. + Your mail message is all people see of you, and if it is poorly formatted, one line per paragraph, badly spelt, or full of errors, it will give people a poor impression of you. + + + A lot of badly formatted messages come from http://www.lemis.com/email.html[bad mailers or badly configured mailers]. + The following mailers are known to send out badly formatted messages without you finding out about them: + + ** exmh + ** Microsoft(R) Exchange + ** Microsoft(R) Outlook(R) + + + Try not to use MIME: a lot of people use mailers which do not get on very well with MIME. + * Make sure your time and time zone are set correctly. This may seem a little silly, since your message still gets there, but many of the people you are trying to reach get several hundred messages a day. They frequently sort the incoming messages by subject and by date, and if your message does not come before the first answer, they may assume they missed it and not bother to look. + * Do not include unrelated questions in the same message. Firstly, a long message tends to scare people off, and secondly, it is more difficult to get all the people who can answer all the questions to read the message. + * Specify as much information as possible. This is a difficult area, and we need to expand on what information you need to submit, but here is a start: + + ** In nearly every case, it is important to know the version of FreeBSD you are running. This is particularly the case for FreeBSD-CURRENT, where you should also specify the date of the sources, though of course you should not be sending questions about -CURRENT to FreeBSD-questions. + ** With any problem which _could_ be hardware related, tell us about your hardware. In case of doubt, assume it is possible that it is hardware. What kind of CPU are you using? How fast? What motherboard? How much memory? What peripherals? + + + There is a judgement call here, of course, but the output of the man:dmesg[8] command can frequently be very useful, since it tells not just what hardware you are running, but what version of FreeBSD as well. + ** If you get error messages, do not say "I get error messages", say (for example) "I get the error message 'No route to host'". + ** If your system panics, do not say "My system panicked", say (for example) "my system panicked with the message 'free vnode isn't'". + ** If you have difficulty installing FreeBSD, please tell us what hardware you have. In particular, it is important to know the IRQs and I/O addresses of the boards installed in your machine. + ** If you have difficulty getting PPP to run, describe the configuration. Which version of PPP do you use? What kind of authentication do you have? Do you have a static or dynamic IP address? What kind of messages do you get in the log file? + + * A lot of the information you need to supply is the output of programs, such as man:dmesg[8], or console messages, which usually appear in [.filename]#/var/log/messages#. Do not try to copy this information by typing it in again; it is a real pain, and you are bound to make a mistake. To send log file contents, either make a copy of the file and use an editor to trim the information to what is relevant, or cut and paste into your message. For the output of programs like man:dmesg[8], redirect the output to a file and include that. For example, + + + [source,shell] + .... + % dmesg > /tmp/dmesg.out + .... + + + This redirects the information to the file [.filename]#/tmp/dmesg.out#. + * If you do all this, and you still do not get an answer, there could be other reasons. For example, the problem is so complicated that nobody knows the answer, or the person who does know the answer was offline. If you do not get an answer after, say, a week, it might help to re-send the message. If you do not get an answer to your second message, though, you are probably not going to get one from this forum. Resending the same message again and again will only make you unpopular. + + To summarize, let's assume you know the answer to the following question (yes, it is the same one in each case). + You choose which of these two questions you would be more prepared to answer: + + .Message 1 + [example] + ==== + + .... + Subject: HELP!!?!?? + I just can't get hits damn silly FereBSD system to + workd, and Im really good at this tsuff, but I have never seen + anythign sho difficult to install, it jst wont work whatever I try + so why don't you guys tell me what I doing wrong. + .... + ==== + + .Message 2 + [example] + ==== + + .... + Subject: Problems installing FreeBSD + + I've just got the FreeBSD 2.1.5 CDROM from Walnut Creek, and I'm having a lot + of difficulty installing it. I have a 66 MHz 486 with 16 MB of + memory and an Adaptec 1540A SCSI board, a 1.2GB Quantum Fireball + disk and a Toshiba 3501XA CDROM drive. The installation works just + fine, but when I try to reboot the system, I get the message + Missing Operating System. + .... + ==== + + == How to Follow up to a Question + + Often you will want to send in additional information to a question you have already sent. + The best way to do this is to reply to your original message. + This has three advantages: + + . You include the original message text, so people will know what you are talking about. Do not forget to trim unnecessary text out, though. + . The text in the subject line stays the same (you did remember to put one in, did you not?). Many mailers will sort messages by subject. This helps group messages together. + . The message reference numbers in the header will refer to the previous message. Some mailers, such as http://www.mutt.org/[mutt], can _thread_ messages, showing the exact relationships between the messages. + + == How to Answer a Question + + Before you answer a question to FreeBSD-questions, consider: + + . A lot of the points on submitting questions also apply to answering questions. Read them. + . Has somebody already answered the question? The easiest way to check this is to sort your incoming mail by subject: then (hopefully) you will see the question followed by any answers, all together. + + + If somebody has already answered it, it does not automatically mean that you should not send another answer. + But it makes sense to read all the other answers first. + . Do you have something to contribute beyond what has already been said? In general, "Yeah, me too" answers do not help much, although there are exceptions, like when somebody is describing a problem they are having, and they do not know whether it is their fault or whether there is something wrong with the hardware or software. If you do send a "me too" answer, you should also include any further relevant information. + . Are you sure you understand the question? Very frequently, the person who asks the question is confused or does not express themselves very well. Even with the best understanding of the system, it is easy to send a reply which does not answer the question. This does not help: you will leave the person who submitted the question more frustrated or confused than ever. If nobody else answers, and you are not too sure either, you can always ask for more information. + . Are you sure your answer is correct? If not, wait a day or so. If nobody else comes up with a better answer, you can still reply and say, for example, "I do not know if this is correct, but since nobody else has replied, why don't you try replacing your ATAPI CDROM with a frog?". + . Unless there is a good reason to do otherwise, reply to the sender and to FreeBSD-questions. Many people on the FreeBSD-questions are "lurkers": they learn by reading messages sent and replied to by others. If you take a message which is of general interest off the list, you are depriving these people of their information. Be careful with group replies; lots of people send messages with hundreds of CCs. If this is the case, be sure to trim the Cc: lines appropriately. + . Include relevant text from the original message. Trim it to the minimum, but do not overdo it. It should still be possible for somebody who did not read the original message to understand what you are talking about. + . Use some technique to identify which text came from the original message, and which text you add. I personally find that prepending "`>`" to the original message works best. Leaving white space after the "`> ;`" and leave empty lines between your text and the original text both make the result more readable. + . Put your response in the correct place (after the text to which it replies). It is very difficult to read a thread of responses where each reply comes before the text to which it replies. + . Most mailers change the subject line on a reply by prepending a text such as "Re: ". If your mailer does not do it automatically, you should do it manually. + . If the submitter did not abide by format conventions (lines too long, inappropriate subject line) _please_ fix it. In the case of an incorrect subject line (such as "HELP!!??"), change the subject line to (say) "Re: Difficulties with sync PPP (was: HELP!!??)". That way other people trying to follow the thread will have less difficulty following it. + + + In such cases, it is appropriate to say what you did and why you did it, but try not to be rude. + If you find you can not answer without being rude, do not answer. + + + If you just want to reply to a message because of its bad format, just reply to the submitter, not to the list. + You can just send him this message in reply, if you like. +diff --git a/documentation/content/en/articles/freebsd-update-server/_index.adoc b/documentation/content/en/articles/freebsd-update-server/_index.adoc +index 78f44784c8..fd7c066605 100644 +--- a/documentation/content/en/articles/freebsd-update-server/_index.adoc ++++ b/documentation/content/en/articles/freebsd-update-server/_index.adoc +@@ -1,641 +1,641 @@ + --- + title: Build Your Own FreeBSD Update Server + authors: + - author: Jason Helfman + email: jgh@FreeBSD.org + copyright: 2009-2011, 2013 Jason Helfman + description: Building your own freebsd-update server allows a system administrator to perform fast updates for a number of machines from a local mirror + trademarks: ["freebsd", "amd", "intel", "general"] + tags: ["FreeBSD", "Update", "Server", "internal"] + --- + + = Build Your Own FreeBSD Update Server + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/freebsd-update-server/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + + [WARNING] + ==== + The instructions in this article refer to an older version of FreeBSD and may + not work properly on recent versions of the OS. With the availability of + pkgbase, the freebsd-update utility is scheduled to be removed from FreeBSD in + the future. When that happens, this article is either updated to reflect the + new procedures or removed entirely. + ==== + + [.abstract-title] + Abstract + + This article describes building an internal FreeBSD Update Server. + The https://github.com/freebsd/freebsd-update-build/[freebsd-update-server] is written by `{cperciva}`, Security Officer Emeritus of FreeBSD. + For users that think it is convenient to update their systems against an official update server, building their own FreeBSD Update Server may help to extend its functionality by supporting manually-tweaked FreeBSD releases or by providing a local mirror that will allow faster updates for a number of machines. + + ''' + + toc::[] + + [[acknowledgments]] + == Acknowledgments + + This article was subsequently printed at https://people.freebsd.org/~jgh/files/fus/BSD_03_2010_EN.pdf[BSD Magazine]. + + [[introduction]] + == Introduction + + Experienced users or administrators are often responsible for several machines or environments. + They understand the difficult demands and challenges of maintaining such an infrastructure. + Running a FreeBSD Update Server makes it easier to deploy security and software patches to selected test machines before rolling them out to production. + It also means a number of systems can be updated from the local network rather than a potentially slower Internet connection. + This article outlines the steps involved in creating an internal FreeBSD Update Server. + + [[prerequisites]] + == Prerequisites + + To build an internal FreeBSD Update Server some requirements should be met. + + * A running FreeBSD system. + + + [NOTE] + ==== + At a minimum, updates require building on a FreeBSD release greater than or equal to the target release version for distribution. + ==== + * A user account with at least 4 GB of available space. This will allow the creation of updates for 7.1 and 7.2, but the exact space requirements may change from version to version. + * An man:ssh[1] account on a remote machine to upload distributed updates. +-* A web server, like extref:{handbook}[Apache, network-apache], with over half of the space required for the build. For instance, test builds for 7.1 and 7.2 consume a total amount of 4 GB, and the webserver space needed to distribute these updates is 2.6 GB. ++* A web server, like extref:{handbook}network-servers[Apache, network-apache], with over half of the space required for the build. For instance, test builds for 7.1 and 7.2 consume a total amount of 4 GB, and the webserver space needed to distribute these updates is 2.6 GB. + * Basic knowledge of shell scripting with Bourne shell, man:sh[1]. + + [[Configuration]] + == Configuration: Installation & Setup + + Download the https://github.com/freebsd/freebsd-update-build/[freebsd-update-server] software by installing package:devel/git[] and package:security/ca_root_nss[], and execute: + + [source,shell] + .... + % git clone https://github.com/freebsd/freebsd-update-build.git freebsd-update-server + .... + + Update [.filename]#scripts/build.conf# appropriately. + It is sourced during all build operations. + + Here is the default [.filename]#build.conf#, which should be modified to suit your environment. + + [.programlisting] + .... + # Main configuration file for FreeBSD Update builds. The + # release-specific configuration data is lower down in + # the scripts tree. + + # Location from which to fetch releases + export FTP=ftp://ftp2.freebsd.org/pub/FreeBSD/releases <.> + + # Host platform + export HOSTPLATFORM=`uname -m` + + # Host name to use inside jails + export BUILDHOSTNAME=${HOSTPLATFORM}-builder.daemonology.net <.> + + # Location of SSH key + export SSHKEY=/root/.ssh/id_dsa <.> + + # SSH account into which files are uploaded + MASTERACCT=builder@wadham.daemonology.net <.> + + # Directory into which files are uploaded + MASTERDIR=update-master.freebsd.org <.> + .... + + Parameters for consideration would be: + + <.> This is the location where ISO images are downloaded from (by the `fetchiso()` subroutine of [.filename]#scripts/build.subr#). The location configured is not limited to FTP URIs. Any URI scheme supported by standard man:fetch[1] utility should work fine. + Customizations to the `fetchiso()` code can be installed by copying the default [.filename]#build.subr# script to the release and architecture-specific area at [.filename]#scripts/RELEASE/ARCHITECTURE/build.subr# and applying local changes. + + <.> The name of the build host. This information will be displayed on updated systems when issuing: + + + [source,shell] + .... + % uname -v + .... + + + <.> The SSH key for uploading files to the update server. A key pair can be created by typing `ssh-keygen -t dsa`. This parameter is optional; standard password authentication will be used as a fallback authentication method when `SSHKEY` is not defined. + The man:ssh-keygen[1] manual page has more detailed information about SSH and the appropriate steps for creating and using one. + + <.> Account for uploading files to the update server. + + <.> Directory on the update server where files are uploaded to. + + The default [.filename]#build.conf# shipped with the freebsd-update-server sources is suitable for building i386 releases of FreeBSD. As an example of building an update server for other architectures, the following steps outline the configuration changes needed for amd64: + + [.procedure] + ==== + . Create a build environment for amd64: + + + [source,shell] + .... + % mkdir -p /usr/local/freebsd-update-server/scripts/7.2-RELEASE/amd64 + .... + + . Install a [.filename]#build.conf# in the newly created build directory. The build configuration options for FreeBSD 7.2-RELEASE on amd64 should be similar to: + + + [.programlisting] + .... + # SHA256 hash of RELEASE disc1.iso image. + export RELH=1ea1f6f652d7c5f5eab7ef9f8edbed50cb664b08ed761850f95f48e86cc71ef5 <.> + # Components of the world, source, and kernels + export WORLDPARTS="base catpages dict doc games info manpages proflibs lib32" + export SOURCEPARTS="base bin contrib crypto etc games gnu include krb5 \ + lib libexec release rescue sbin secure share sys tools \ + ubin usbin cddl" + export KERNELPARTS="generic" + + # EOL date + export EOL=1275289200 <.> + .... + + + <.> The man:sha256[1] hash key for the desired release, is published within the respective link:https://www.FreeBSD.org/releases/[release announcement]. + <.> To generate the "End of Life" number for [.filename]#build.conf#, refer to the "Estimated EOL" posted on the link:https://www.FreeBSD.org/security/security/[FreeBSD Security Website]. The value of `EOL` can be derived from the date listed on the web site, using the man:date[1] utility, for example: + + + [source,shell] + .... + % date -j -f '%Y%m%d-%H%M%S' '20090401-000000' '+%s' + .... + ==== + + [[build]] + == Building Update Code + + The first step is to run [.filename]#scripts/make.sh#. + This will build some binaries, create directories, and generate an RSA signing key used for approving builds. + In this step, a passphrase will have to be supplied for the final creation of the signing key. + + [source,shell] + .... + # sh scripts/make.sh + cc -O2 -fno-strict-aliasing -pipe findstamps.c -o findstamps + findstamps.c: In function 'usage': + findstamps.c:45: warning: incompatible implicit declaration of built-in function 'exit' + cc -O2 -fno-strict-aliasing -pipe unstamp.c -o unstamp + install findstamps ../bin + install unstamp ../bin + rm -f findstamps unstamp + Generating RSA private key, 4096 bit long modulus + ................................................................................++ + ...................++ + e is 65537 (0x10001) + + Public key fingerprint: + 27ef53e48dc869eea6c3136091cc6ab8589f967559824779e855d58a2294de9e + + Encrypting signing key for root + enter aes-256-cbc encryption password: + Verifying - enter aes-256-cbc encryption password: + .... + + [NOTE] + ==== + Keep a note of the generated key fingerprint. + This value is required in [.filename]#/etc/freebsd-update.conf# for binary updates. + ==== + + At this point, we are ready to stage a build. + + [source,shell] + .... + # cd /usr/local/freebsd-update-server + # sh scripts/init.sh amd64 7.2-RELEASE + .... + + What follows is a sample of an _initial_ build run. + + [source,shell] + .... + # sh scripts/init.sh amd64 7.2-RELEASE + Mon Aug 24 16:04:36 PDT 2009 Starting fetch for FreeBSD/amd64 7.2-RELEASE + /usr/local/freebsd-update-server/work/7.2-RELE100 of 588 MB 359 kBps 00m00s + Mon Aug 24 16:32:38 PDT 2009 Verifying disc1 hash for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 16:32:44 PDT 2009 Extracting components for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 16:34:05 PDT 2009 Constructing world+src image for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 16:35:57 PDT 2009 Extracting world+src for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 23:36:24 UTC 2009 Building world for FreeBSD/amd64 7.2-RELEASE + Tue Aug 25 00:31:29 UTC 2009 Distributing world for FreeBSD/amd64 7.2-RELEASE + Tue Aug 25 00:32:36 UTC 2009 Building and distributing kernels for FreeBSD/amd64 7.2-RELEASE + Tue Aug 25 00:44:44 UTC 2009 Constructing world components for FreeBSD/amd64 7.2-RELEASE + Tue Aug 25 00:44:56 UTC 2009 Distributing source for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 17:46:18 PDT 2009 Moving components into staging area for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 17:46:33 PDT 2009 Identifying extra documentation for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 17:47:13 PDT 2009 Extracting extra docs for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 17:47:18 PDT 2009 Indexing release for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 17:50:44 PDT 2009 Indexing world0 for FreeBSD/amd64 7.2-RELEASE + + Files built but not released: + Files released but not built: + Files which differ by more than contents: + Files which differ between release and build: + kernel|generic|/GENERIC/hptrr.ko + kernel|generic|/GENERIC/kernel + src|sys|/sys/conf/newvers.sh + world|base|/boot/loader + world|base|/boot/pxeboot + world|base|/etc/mail/freebsd.cf + world|base|/etc/mail/freebsd.submit.cf + world|base|/etc/mail/sendmail.cf + world|base|/etc/mail/submit.cf + world|base|/lib/libcrypto.so.5 + world|base|/usr/bin/ntpq + world|base|/usr/lib/libalias.a + world|base|/usr/lib/libalias_cuseeme.a + world|base|/usr/lib/libalias_dummy.a + world|base|/usr/lib/libalias_ftp.a + ... + .... + + Then the build of the world is performed again, with world patches. + A more detailed explanation may be found in [.filename]#scripts/build.subr#. + + [WARNING] + ==== + During this second build cycle, the network time protocol daemon, man:ntpd[8], is turned off. + Per `{cperciva}`, Security Officer Emeritus of FreeBSD, "the https://github.com/freebsd/freebsd-update-build/[freebsd-update-server] build code needs to identify timestamps which are stored in files so that they can be ignored when comparing builds to determine which files need to be updated. + This timestamp-finding works by doing two builds 400 days apart and comparing the results." + ==== + + [source,shell] + .... + Mon Aug 24 17:54:07 PDT 2009 Extracting world+src for FreeBSD/amd64 7.2-RELEASE + Wed Sep 29 00:54:34 UTC 2010 Building world for FreeBSD/amd64 7.2-RELEASE + Wed Sep 29 01:49:42 UTC 2010 Distributing world for FreeBSD/amd64 7.2-RELEASE + Wed Sep 29 01:50:50 UTC 2010 Building and distributing kernels for FreeBSD/amd64 7.2-RELEASE + Wed Sep 29 02:02:56 UTC 2010 Constructing world components for FreeBSD/amd64 7.2-RELEASE + Wed Sep 29 02:03:08 UTC 2010 Distributing source for FreeBSD/amd64 7.2-RELEASE + Tue Sep 28 19:04:31 PDT 2010 Moving components into staging area for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 19:04:46 PDT 2009 Extracting extra docs for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 19:04:51 PDT 2009 Indexing world1 for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 19:08:04 PDT 2009 Locating build stamps for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 19:10:19 PDT 2009 Cleaning staging area for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 19:10:19 PDT 2009 Preparing to copy files into staging area for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 19:10:20 PDT 2009 Copying data files into staging area for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 12:16:57 PDT 2009 Copying metadata files into staging area for FreeBSD/amd64 7.2-RELEASE + Mon Aug 24 12:16:59 PDT 2009 Constructing metadata index and tag for FreeBSD/amd64 7.2-RELEASE + + Files found which include build stamps: + kernel|generic|/GENERIC/hptrr.ko + kernel|generic|/GENERIC/kernel + world|base|/boot/loader + world|base|/boot/pxeboot + world|base|/etc/mail/freebsd.cf + world|base|/etc/mail/freebsd.submit.cf + world|base|/etc/mail/sendmail.cf + world|base|/etc/mail/submit.cf + world|base|/lib/libcrypto.so.5 + world|base|/usr/bin/ntpq + world|base|/usr/include/osreldate.h + world|base|/usr/lib/libalias.a + world|base|/usr/lib/libalias_cuseeme.a + world|base|/usr/lib/libalias_dummy.a + world|base|/usr/lib/libalias_ftp.a + ... + .... + + Finally, the build completes. + + [source,shell] + .... + Values of build stamps, excluding library archive headers: + v1.2 (Aug 25 2009 00:40:36) + v1.2 (Aug 25 2009 00:38:22) + @()FreeBSD 7.2-RELEASE 0: Tue Aug 25 00:38:29 UTC 2009 + FreeBSD 7.2-RELEASE 0: Tue Aug 25 00:38:29 UTC 2009 + root@server.myhost.com:/usr/obj/usr/src/sys/GENERIC + 7.2-RELEASE + Mon Aug 24 23:55:25 UTC 2009 + Mon Aug 24 23:55:25 UTC 2009 + built by root@server.myhost.com on Tue Aug 25 00:16:15 UTC 2009 + built by root@server.myhost.com on Tue Aug 25 00:16:15 UTC 2009 + built by root@server.myhost.com on Tue Aug 25 00:16:15 UTC 2009 + built by root@server.myhost.com on Tue Aug 25 00:16:15 UTC 2009 + Mon Aug 24 23:46:47 UTC 2009 + ntpq 4.2.4p5-a Mon Aug 24 23:55:53 UTC 2009 (1) + * Copyright (c) 1992-2009 The FreeBSD Project. + Mon Aug 24 23:46:47 UTC 2009 + Mon Aug 24 23:55:40 UTC 2009 + Aug 25 2009 + ntpd 4.2.4p5-a Mon Aug 24 23:55:52 UTC 2009 (1) + ntpdate 4.2.4p5-a Mon Aug 24 23:55:53 UTC 2009 (1) + ntpdc 4.2.4p5-a Mon Aug 24 23:55:53 UTC 2009 (1) + Tue Aug 25 00:21:21 UTC 2009 + Tue Aug 25 00:21:21 UTC 2009 + Tue Aug 25 00:21:21 UTC 2009 + Mon Aug 24 23:46:47 UTC 2009 + + FreeBSD/amd64 7.2-RELEASE initialization build complete. Please + review the list of build stamps printed above to confirm that + they look sensible, then run + sh -e approve.sh amd64 7.2-RELEASE + to sign the release. + .... + + Approve the build if everything is correct. + More information on determining this can be found in the distributed source file named [.filename]#USAGE#. Execute [.filename]#scripts/approve.sh#, as directed. + This will sign the release, and move components into a staging area suitable for uploading. + + [source,shell] + .... + # cd /usr/local/freebsd-update-server + # sh scripts/mountkey.sh + .... + + [source,shell] + .... + # sh -e scripts/approve.sh amd64 7.2-RELEASE + Wed Aug 26 12:50:06 PDT 2009 Signing build for FreeBSD/amd64 7.2-RELEASE + Wed Aug 26 12:50:06 PDT 2009 Copying files to patch source directories for FreeBSD/amd64 7.2-RELEASE + Wed Aug 26 12:50:06 PDT 2009 Copying files to upload staging area for FreeBSD/amd64 7.2-RELEASE + Wed Aug 26 12:50:07 PDT 2009 Updating databases for FreeBSD/amd64 7.2-RELEASE + Wed Aug 26 12:50:07 PDT 2009 Cleaning staging area for FreeBSD/amd64 7.2-RELEASE + .... + + After the approval process is complete, the upload procedure may be started. + + [source,shell] + .... + # cd /usr/local/freebsd-update-server + # sh scripts/upload.sh amd64 7.2-RELEASE + .... + + [NOTE] + ==== + In the event update code needs to be re-uploaded, this may be done by changing to the public distributions directory for the target release and updating attributes of the _uploaded_ file. + + [source,shell] + .... + # cd /usr/local/freebsd-update-server/pub/7.2-RELEASE/amd64 + # touch -t 200801010101.01 uploaded + .... + + ==== + + The uploaded files will need to be in the document root of the webserver in order for updates to be distributed. + The exact configuration will vary depending on the web server used. +-For the Apache web server, please refer to the extref:{handbook}[Configuration of Apache servers, network-apache] section in the Handbook. ++For the Apache web server, please refer to the extref:{handbook}network-servers[Configuration of Apache servers, network-apache] section in the Handbook. + +-Update client's `KeyPrint` and `ServerName` in [.filename]#/etc/freebsd-update.conf#, and perform updates as instructed in the extref:{handbook}[FreeBSD Update, updating-upgrading-freebsdupdate] section of the Handbook. ++Update client's `KeyPrint` and `ServerName` in [.filename]#/etc/freebsd-update.conf#, and perform updates as instructed in the extref:{handbook}cutting-edge[FreeBSD Update, updating-upgrading-freebsdupdate] section of the Handbook. + + [IMPORTANT] + ==== + In order for FreeBSD Update Server to work properly, updates for both the _current_ release and the release _one wants to upgrade to_ need to be built. + This is necessary for determining the differences of files between releases. + For example, when upgrading a FreeBSD system from 7.1-RELEASE to 7.2-RELEASE, updates will need to be built and uploaded to your distribution server for both versions. + ==== + + For reference, the entire run of link:../../source/articles/freebsd-update-server/init.txt[init.sh] is attached. + + [[patch]] + == Building a Patch + + Every time a link:https://www.FreeBSD.org/security/advisories/[security advisory] or link:https://www.FreeBSD.org/security/notices/[security notice] is announced, a patch update can be built. + + For this example, 7.1-RELEASE will be used. + + A couple of assumptions are made for a different release build: + + * Setup the correct directory structure for the initial build. + * Perform an initial build for 7.1-RELEASE. + + Create the patch directory of the respective release under [.filename]#/usr/local/freebsd-update-server/patches/#. + + [source,shell] + .... + % mkdir -p /usr/local/freebsd-update-server/patches/7.1-RELEASE/ + % cd /usr/local/freebsd-update-server/patches/7.1-RELEASE + .... + + As an example, take the patch for man:named[8]. + Read the advisory, and grab the necessary file from link:https://www.FreeBSD.org/security/advisories/[FreeBSD Security Advisories]. +-More information on interpreting the advisory, can be found in the extref:{handbook}[FreeBSD Handbook, security-advisories]. ++More information on interpreting the advisory, can be found in the extref:{handbook}security[FreeBSD Handbook, security-advisories]. + + In the https://security.freebsd.org/advisories/FreeBSD-SA-09:12.bind.asc[security brief], this advisory is called `SA-09:12.bind`. + After downloading the file, it is required to rename the file to an appropriate patch level. + It is suggested to keep this consistent with official FreeBSD patch levels, but its name may be freely chosen. + For this build, let us follow the currently established practice of FreeBSD and call this `p7`. Rename the file: + + [source,shell] + .... + % cd /usr/local/freebsd-update-server/patches/7.1-RELEASE/; mv bind.patch 7-SA-09:12.bind + .... + + [NOTE] + ==== + When running a patch level build, it is assumed that previous patches are in place. + When a patch build is run, it will run all patches contained in the patch directory. + + There can be custom patches added to any build. Use the number zero, or any other number. + ==== + + [WARNING] + ==== + + It is up to the administrator of the FreeBSD Update Server to take appropriate measures to verify the authenticity of every patch. + ==== + + At this point, a _diff_ is ready to be built. + The software checks first to see if a [.filename]#scripts/init.sh# has been run on the respective release prior to running the diff build. + + [source,shell] + .... + # cd /usr/local/freebsd-update-server + # sh scripts/diff.sh amd64 7.1-RELEASE 7 + .... + + What follows is a sample of a _differential_ build run. + + [source,shell] + .... + # sh -e scripts/diff.sh amd64 7.1-RELEASE 7 + Wed Aug 26 10:09:59 PDT 2009 Extracting world+src for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 17:10:25 UTC 2009 Building world for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 18:05:11 UTC 2009 Distributing world for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 18:06:16 UTC 2009 Building and distributing kernels for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 18:17:50 UTC 2009 Constructing world components for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 18:18:02 UTC 2009 Distributing source for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 11:19:23 PDT 2009 Moving components into staging area for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 11:19:37 PDT 2009 Extracting extra docs for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 11:19:42 PDT 2009 Indexing world0 for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 11:23:02 PDT 2009 Extracting world+src for FreeBSD/amd64 7.1-RELEASE-p7 + Thu Sep 30 18:23:29 UTC 2010 Building world for FreeBSD/amd64 7.1-RELEASE-p7 + Thu Sep 30 19:18:15 UTC 2010 Distributing world for FreeBSD/amd64 7.1-RELEASE-p7 + Thu Sep 30 19:19:18 UTC 2010 Building and distributing kernels for FreeBSD/amd64 7.1-RELEASE-p7 + Thu Sep 30 19:30:52 UTC 2010 Constructing world components for FreeBSD/amd64 7.1-RELEASE-p7 + Thu Sep 30 19:31:03 UTC 2010 Distributing source for FreeBSD/amd64 7.1-RELEASE-p7 + Thu Sep 30 12:32:25 PDT 2010 Moving components into staging area for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 12:32:39 PDT 2009 Extracting extra docs for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 12:32:43 PDT 2009 Indexing world1 for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 12:35:54 PDT 2009 Locating build stamps for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 12:36:58 PDT 2009 Reverting changes due to build stamps for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 12:37:14 PDT 2009 Cleaning staging area for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 12:37:14 PDT 2009 Preparing to copy files into staging area for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 12:37:15 PDT 2009 Copying data files into staging area for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 12:43:23 PDT 2009 Copying metadata files into staging area for FreeBSD/amd64 7.1-RELEASE-p7 + Wed Aug 26 12:43:25 PDT 2009 Constructing metadata index and tag for FreeBSD/amd64 7.1-RELEASE-p7 + ... + Files found which include build stamps: + kernel|generic|/GENERIC/hptrr.ko + kernel|generic|/GENERIC/kernel + world|base|/boot/loader + world|base|/boot/pxeboot + world|base|/etc/mail/freebsd.cf + world|base|/etc/mail/freebsd.submit.cf + world|base|/etc/mail/sendmail.cf + world|base|/etc/mail/submit.cf + world|base|/lib/libcrypto.so.5 + world|base|/usr/bin/ntpq + world|base|/usr/include/osreldate.h + world|base|/usr/lib/libalias.a + world|base|/usr/lib/libalias_cuseeme.a + world|base|/usr/lib/libalias_dummy.a + world|base|/usr/lib/libalias_ftp.a + ... + Values of build stamps, excluding library archive headers: + v1.2 (Aug 26 2009 18:13:46) + v1.2 (Aug 26 2009 18:11:44) + @()FreeBSD 7.1-RELEASE-p7 0: Wed Aug 26 18:11:50 UTC 2009 + FreeBSD 7.1-RELEASE-p7 0: Wed Aug 26 18:11:50 UTC 2009 + root@server.myhost.com:/usr/obj/usr/src/sys/GENERIC + 7.1-RELEASE-p7 + Wed Aug 26 17:29:15 UTC 2009 + Wed Aug 26 17:29:15 UTC 2009 + built by root@server.myhost.com on Wed Aug 26 17:49:58 UTC 2009 + built by root@server.myhost.com on Wed Aug 26 17:49:58 UTC 2009 + built by root@server.myhost.com on Wed Aug 26 17:49:58 UTC 2009 + built by root@server.myhost.com on Wed Aug 26 17:49:58 UTC 2009 + Wed Aug 26 17:20:39 UTC 2009 + ntpq 4.2.4p5-a Wed Aug 26 17:29:42 UTC 2009 (1) + * Copyright (c) 1992-2009 The FreeBSD Project. + Wed Aug 26 17:20:39 UTC 2009 + Wed Aug 26 17:29:30 UTC 2009 + Aug 26 2009 + ntpd 4.2.4p5-a Wed Aug 26 17:29:41 UTC 2009 (1) + ntpdate 4.2.4p5-a Wed Aug 26 17:29:42 UTC 2009 (1) + ntpdc 4.2.4p5-a Wed Aug 26 17:29:42 UTC 2009 (1) + Wed Aug 26 17:55:02 UTC 2009 + Wed Aug 26 17:55:02 UTC 2009 + Wed Aug 26 17:55:02 UTC 2009 + Wed Aug 26 17:20:39 UTC 2009 + ... + .... + + Updates are printed, and approval is requested. + + [source,shell] + .... + New updates: + kernel|generic|/GENERIC/kernel.symbols|f|0|0|0555|0|7c8dc176763f96ced0a57fc04e7c1b8d793f27e006dd13e0b499e1474ac47e10| + kernel|generic|/GENERIC/kernel|f|0|0|0555|0|33197e8cf15bbbac263d17f39c153c9d489348c2c534f7ca1120a1183dec67b1| + kernel|generic|/|d|0|0|0755|0|| + src|base|/|d|0|0|0755|0|| + src|bin|/|d|0|0|0755|0|| + src|cddl|/|d|0|0|0755|0|| + src|contrib|/contrib/bind9/bin/named/update.c|f|0|10000|0644|0|4d434abf0983df9bc47435670d307fa882ef4b348ed8ca90928d250f42ea0757| + src|contrib|/contrib/bind9/lib/dns/openssldsa_link.c|f|0|10000|0644|0|c6805c39f3da2a06dd3f163f26c314a4692d4cd9a2d929c0acc88d736324f550| + src|contrib|/contrib/bind9/lib/dns/opensslrsa_link.c|f|0|10000|0644|0|fa0f7417ee9da42cc8d0fd96ad24e7a34125e05b5ae075bd6e3238f1c022a712| + ... + FreeBSD/amd64 7.1-RELEASE update build complete. Please review + the list of build stamps printed above and the list of updated + files to confirm that they look sensible, then run + sh -e approve.sh amd64 7.1-RELEASE + to sign the build. + .... + + Follow the same process as noted before for approving a build: + + [source,shell] + .... + # sh -e scripts/approve.sh amd64 7.1-RELEASE + Wed Aug 26 12:50:06 PDT 2009 Signing build for FreeBSD/amd64 7.1-RELEASE + Wed Aug 26 12:50:06 PDT 2009 Copying files to patch source directories for FreeBSD/amd64 7.1-RELEASE + Wed Aug 26 12:50:06 PDT 2009 Copying files to upload staging area for FreeBSD/amd64 7.1-RELEASE + Wed Aug 26 12:50:07 PDT 2009 Updating databases for FreeBSD/amd64 7.1-RELEASE + Wed Aug 26 12:50:07 PDT 2009 Cleaning staging area for FreeBSD/amd64 7.1-RELEASE + + The FreeBSD/amd64 7.1-RELEASE update build has been signed and is + ready to be uploaded. Remember to run + sh -e umountkey.sh + to unmount the decrypted key once you have finished signing all + the new builds. + .... + + After approving the build, upload the software: + + [source,shell] + .... + # cd /usr/local/freebsd-update-server + # sh scripts/upload.sh amd64 7.1-RELEASE + .... + + For reference, the entire run of link:../../source/articles/freebsd-update-server/diff.txt[diff.sh] is attached. + + [[tips]] + == Tips + + * If a custom release is built using the native `make release` extref:{releng}[procedure, release-build], freebsd-update-server code will work from your release. As an example, a release without ports or documentation can be built by clearing functionality pertaining to documentation subroutines `findextradocs ()`, `addextradocs ()` and altering the download location in `fetchiso ()`, respectively, in [.filename]#scripts/build.subr#. As a last step, change the man:sha256[1] hash in [.filename]#build.conf# under your respective release and architecture and you are ready to build off your custom release. + + + [.programlisting] + .... + # Compare ${WORKDIR}/release and ${WORKDIR}/$1, identify which parts + # of the world|doc subcomponent are missing from the latter, and + # build a tarball out of them. + findextradocs () { + } + # Add extra docs to ${WORKDIR}/$1 + addextradocs () { + } + .... + + * Adding `-j _NUMBER_` flags to `buildworld` and `obj` targets in the [.filename]#scripts/build.subr# script may speed up processing depending on the hardware used, however it is not necessary. Using these flags in other targets is not recommended, as it may cause the build to become unreliable. + + + [.programlisting] + .... + # Build the world + log "Building world" + cd /usr/src && + make -j 2 ${COMPATFLAGS} buildworld 2>&1 + # Distribute the world + log "Distributing world" + cd /usr/src/release && + make -j 2 obj && + make ${COMPATFLAGS} release.1 release.2 2>&1 + .... + +-* Create an appropriate extref:{handbook}[DNS, network-dns] SRV record for the update server, and put others behind it with variable weights. Using this facility will provide update mirrors, however this tip is not necessary unless you wish to provide a redundant service. ++* Create an appropriate extref:{handbook}network-servers[DNS, network-dns] SRV record for the update server, and put others behind it with variable weights. Using this facility will provide update mirrors, however this tip is not necessary unless you wish to provide a redundant service. + + + [.programlisting] + .... + _http._tcp.update.myserver.com. IN SRV 0 2 80 host1.myserver.com. + IN SRV 0 1 80 host2.myserver.com. + IN SRV 0 0 80 host3.myserver.com. + .... +diff --git a/documentation/content/en/articles/gjournal-desktop/_index.adoc b/documentation/content/en/articles/gjournal-desktop/_index.adoc +index 1f8c15f843..164c92985b 100644 +--- a/documentation/content/en/articles/gjournal-desktop/_index.adoc ++++ b/documentation/content/en/articles/gjournal-desktop/_index.adoc +@@ -1,510 +1,510 @@ + --- + title: Implementing UFS Journaling on a Desktop PC + authors: + - author: Manolis Kiagias + email: manolis@FreeBSD.org + description: Implementing UFS Journaling on a Desktop PC + trademarks: ["freebsd", "general"] + tags: ["UFS", "Journaling" , "Desktop", "FreeBSD"] + --- + + = Implementing UFS Journaling on a Desktop PC + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/gjournal-desktop/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + A journaling file system uses a log to record all transactions that take place in the file system, and preserves its integrity in the event of a system crash or power failure. + Although it is still possible to lose unsaved changes to files, journaling almost completely eliminates the possibility of file system corruption caused by an unclean shutdown. + It also shortens to a minimum the time required for after-failure file system checking. + Although the UFS file system employed by FreeBSD does not implement journaling itself, the new journal class of the GEOM framework in FreeBSD 7._X_ can be used to provide file system independent journaling. + This article explains how to implement UFS journaling on a typical desktop PC scenario. + + ''' + + toc::[] + + [[introduction]] + == Introduction + + While professional servers are usually well protected from unforeseen shutdowns, the typical desktop is at the mercy of power failures, accidental resets, and other user related incidents that can lead to unclean shutdowns. + Soft Updates usually protect the file system efficiently in such cases, although most of the times a lengthy background check is required. + On rare occasions, file system corruption reaches a point where user intervention is required and data may be lost. + + The new journaling capability provided by GEOM can greatly assist in such scenarios, by virtually eliminating the time required for file system checking, and ensuring that the file system is quickly restored to a consistent state. + +-This article describes a procedure for implementing UFS journaling on a typical desktop PC scenario (one hard disk used for both operating system and data). ++This article describes a procedure for implementing UFS journaling on a typical desktop PC scenario (one hard disk used for both operating system and data). + It should be followed during a fresh installation of FreeBSD. + The steps are simple enough and do not require overly complex interaction with the command line. + + After reading this article, you will know: + + * How to reserve space for journaling during a new installation of FreeBSD. + * How to load and enable the `geom_journal` module (or build support for it in your custom kernel). + * How to convert your existing file systems to utilize journaling, and what options to use in [.filename]#/etc/fstab# to mount them. + * How to implement journaling in new (empty) partitions. + * How to troubleshoot common problems associated with journaling. + + Before reading this article, you should be able to: + + * Understand basic UNIX(R) and FreeBSD concepts. + * Be familiar with the installation procedure of FreeBSD and the sysinstall utility. + + [WARNING] + ==== + The procedure described here is intended for preparing a new installation where no actual user data is stored on the disk yet. + While it is possible to modify and extend this procedure for systems already in production, you should _backup_ all important data before doing so. + Messing around with disks and partitions at a low level can lead to fatal mistakes and data loss. + ==== + + [[understanding-journaling]] + == Understanding Journaling in FreeBSD + + The journaling provided by GEOM in FreeBSD 7._X_ is not file system specific (unlike for example the ext3 file system in Linux(R)) but is functioning at the block level. + Though this means it can be applied to different file systems, for FreeBSD 7.0-RELEASE, it can only be used on UFS2. + + This functionality is provided by loading the [.filename]#geom_journal.ko# module into the kernel (or building it into a custom kernel) and using the `gjournal` command to configure the file systems. + In general, you would like to journal large file systems, like [.filename]#/usr#. + You will need however (see the following section) to reserve some free disk space. + + When a file system is journaled, some disk space is needed to keep the journal itself. + The disk space that holds the actual data is referred to as the __data provider__, while the one that holds the journal is referred to as the __journal provider__. + The data and journal providers need to be on different partitions when journaling an existing (non-empty) partition. + When journaling a new partition, you have the option to use a single provider for both data and journal. + In any case, the `gjournal` command combines both providers to create the final journaled file system. + For example: + + * You wish to journal your [.filename]#/usr# file system, stored in [.filename]#/dev/ad0s1f# (which already contains data). + * You reserved some free disk space in a partition in [.filename]#/dev/ad0s1g#. + * Using `gjournal`, a new [.filename]#/dev/ad0s1f.journal# device is created where [.filename]#/dev/ad0s1f# is the data provider, and [.filename]#/dev/ad0s1g# is the journal provider. This new device is then used for all subsequent file operations. + +-The amount of disk space you need to reserve for the journal provider depends on the usage load of the file system and not on the size of the data provider. ++The amount of disk space you need to reserve for the journal provider depends on the usage load of the file system and not on the size of the data provider. + For example on a typical office desktop, a 1 GB journal provider for the [.filename]#/usr# file system will suffice, while a machine that deals with heavy disk I/O (i.e. video editing) may need more. + A kernel panic will occur if the journal space is exhausted before it has a chance to be committed. + + [NOTE] + ==== + The journal sizes suggested here, are highly unlikely to cause problems in typical desktop use (such as web browsing, word processing and playback of media files). +-If your workload includes intense disk activity, use the following rule for maximum reliability: Your RAM size should fit in 30% of the journal provider's space. ++If your workload includes intense disk activity, use the following rule for maximum reliability: Your RAM size should fit in 30% of the journal provider's space. + For example, if your system has 1 GB RAM, create an approximately 3.3 GB journal provider. + (Multiply your RAM size with 3.3 to obtain the size of the journal). + ==== + + For more information about journaling, please read the manual page of man:gjournal[8]. + + [[reserve-space]] + == Steps During the Installation of FreeBSD + + === Reserving Space for Journaling + + A typical desktop machine usually has one hard disk that stores both the OS and user data. + Arguably, the default partitioning scheme selected by sysinstall is more or less suitable: A desktop machine does not need a large [.filename]#/var# partition, while [.filename]#/usr# is allocated the bulk of the disk space, since user data and a lot of packages are installed into its subdirectories. + + The default partitioning (the one obtained by pressing kbd:[A] at the FreeBSD partition editor, called Disklabel) does not leave any unallocated space. + Each partition that will be journaled, requires another partition for the journal. + Since the [.filename]#/usr# partition is the largest, it makes sense to shrink this partition slightly, to obtain the space required for journaling. + + In our example, an 80 GB disk is used. + The following screenshot shows the default partitions created by Disklabel during installation: + + image::disklabel1.png[] + + If this is more or less what you need, it is very easy to adjust for journaling. + Simply use the arrow keys to move the highlight to the [.filename]#/usr# partition and press kbd:[D] to delete it. + + Now, move the highlight to the disk name at the top of the screen and press kbd:[C] to create a new partition for [.filename]#/usr#. + This new partition should be smaller by 1 GB (if you intend to journal [.filename]#/usr# only), or 2 GB (if you intend to journal both [.filename]#/usr# and [.filename]#/var#). + From the pop-up that appears, opt to create a file system, and type [.filename]#/usr# as the mount point. + + [NOTE] + ==== + Should you journal the [.filename]#/var# partition? Normally, journaling makes sense on quite large partitions. + You may decide not to journal [.filename]#/var#, although doing so on a typical desktop will cause no harm. + If the file system is lightly used (quite probable for a desktop) you may wish to allocate less disk space for its journal. + + In our example, we journal both [.filename]#/usr# and [.filename]#/var#. + You may of course adjust the procedure to your own needs. + ==== + + To keep things as easy going as possible, we are going to use sysinstall to create the partitions required for journaling. + However, during installation, sysinstall insists on asking a mount point for each partition you create. + At this point, you do not have any mount points for the partitions that will hold the journals, and in reality you __do not even need them__. + These are not partitions that we are ever going to mount somewhere. + + To avoid these problems with sysinstall, we are going to create the journal partitions as swap space. + Swap is never mounted, and sysinstall has no problem creating as many swap partitions as needed. + After the first reboot, [.filename]#/etc/fstab# will have to be edited, and the extra swap space entries removed. + + To create the swap, again use the arrow keys to move the highlight to the top of Disklabel screen, so that the disk name itself is highlighted. + Then press kbd:[N], enter the desired size (_1024M_), and select "swap space" from the pop-up menu that appears. + Repeat for every journal you wish to create. + In our example, we create two partitions to provide for the journals of [.filename]#/usr# and [.filename]#/var#. + The final result is shown in the following screenshot: + + image::disklabel2.png[] + + When you have completed creating the partitions, we suggest you write down the partition names, and mount points, so you can easily refer to this information during the configuration phase. + This will help alleviate mistakes that may damage your installation. + The following table shows our notes for the sample configuration: + + .Partitions and Journals + [cols="1,1,1", options="header"] + |=== + | Partition + | Mount Point + | Journal + + |ad0s1d + |/var + |ad0s1h + + |ad0s1f + |/usr + |ad0s1g + |=== + + Continue the installation as you would normally do. + We would however suggest you postpone installation of third party software (packages) until you have completely setup journaling. + + [[first-boot]] + === Booting for the first time + +-Your system will come up normally, but you will need to edit [.filename]#/etc/fstab# and remove the extra swap partitions you created for the journals. ++Your system will come up normally, but you will need to edit [.filename]#/etc/fstab# and remove the extra swap partitions you created for the journals. + Normally, the swap partition you will actually use is the one with the "b" suffix (i.e. ad0s1b in our example). + Remove all other swap space entries and reboot so that FreeBSD will stop using them. + + When the system comes up again, we will be ready to configure journaling. + + [[configure-journal]] + == Setting Up Journaling + + [[running-gjournal]] + === Executing `gjournal` + + Having prepared all the required partitions, it is quite easy to configure journaling. + We will need to switch to single user mode, so login as `root` and type: + + [source,shell] + .... + # shutdown now + .... + + Press kbd:[Enter] to get the default shell. + We will need to unmount the partitions that will be journaled, in our example [.filename]#/usr# and [.filename]#/var#: + + [source,shell] + .... + # umount /usr /var + .... + + Load the module required for journaling: + + [source,shell] + .... + # gjournal load + .... + + Now, use your notes to determine which partition will be used for each journal. + In our example, [.filename]#/usr# is [.filename]#ad0s1f# and its journal will be [.filename]#ad0s1g#, while [.filename]#/var# is [.filename]#ad0s1d# and will be journaled to [.filename]#ad0s1h#. + The following commands are required: + + [source,shell] + .... + # gjournal label ad0s1f ad0s1g + GEOM_JOURNAL: Journal 2948326772: ad0s1f contains data. + GEOM_JOURNAL: Journal 2948326772: ad0s1g contains journal. + + # gjournal label ad0s1d ad0s1h + GEOM_JOURNAL: Journal 3193218002: ad0s1d contains data. + GEOM_JOURNAL: Journal 3193218002: ad0s1h contains journal. + .... + + [NOTE] + ==== + If the last sector of either partition is used, `gjournal` will return an error. + You will have to run the command using the `-f` flag to force an overwrite, i.e.: + + [source,shell] + .... + # gjournal label -f ad0s1d ad0s1h + .... + + Since this is a new installation, it is highly unlikely that anything will be actually overwritten. + ==== + + At this point, two new devices are created, namely [.filename]#ad0s1d.journal# and [.filename]#ad0s1f.journal#. + These represent the [.filename]#/var# and [.filename]#/usr# partitions we have to mount. + Before mounting, we must however set the journal flag on them and clear the Soft Updates flag: + + [source,shell] + .... + # tunefs -J enable -n disable ad0s1d.journal + tunefs: gjournal set + tunefs: soft updates cleared + + # tunefs -J enable -n disable ad0s1f.journal + tunefs: gjournal set + tunefs: soft updates cleared + .... + + Now, mount the new devices manually at their respective places (note that we can now use the `async` mount option): + + [source,shell] + .... + # mount -o async /dev/ad0s1d.journal /var + # mount -o async /dev/ad0s1f.journal /usr + .... + + Edit [.filename]#/etc/fstab# and update the entries for [.filename]#/usr# and [.filename]#/var#: + + [.programlisting] + .... + /dev/ad0s1f.journal /usr ufs rw,async 2 2 + /dev/ad0s1d.journal /var ufs rw,async 2 2 + .... + + [WARNING] + ==== + Make sure the above entries are correct, or you will have trouble starting up normally after you reboot! + ==== + + Finally, edit [.filename]#/boot/loader.conf# and add the following line so the man:gjournal[8] module is loaded at every boot: + + [.programlisting] + .... + geom_journal_load="YES" + .... + + Congratulations! Your system is now set for journaling. + You can either type `exit` to return to multi-user mode, or reboot to test your configuration (recommended). + During the boot you will see messages like the following: + + [source,shell] + .... + ad0: 76293MB XEC XE800JD-00HBC0 08.02D08 at ata0-master SATA150 + GEOM_JOURNAL: Journal 2948326772: ad0s1g contains journal. + GEOM_JOURNAL: Journal 3193218002: ad0s1h contains journal. + GEOM_JOURNAL: Journal 3193218002: ad0s1d contains data. + GEOM_JOURNAL: Journal ad0s1d clean. + GEOM_JOURNAL: Journal 2948326772: ad0s1f contains data. + GEOM_JOURNAL: Journal ad0s1f clean. + .... + + After an unclean shutdown, the messages will vary slightly, i.e.: + + [source,shell] + .... + GEOM_JOURNAL: Journal ad0s1d consistent. + .... + + This usually means that man:gjournal[8] used the information in the journal provider to return the file system to a consistent state. + + [[gjournal-new]] + === Journaling Newly Created Partitions + + While the above procedure is necessary for journaling partitions that already contain data, journaling an empty partition is somewhat easier, since both the data and the journal provider can be stored in the same partition. + For example, assume a new disk was installed, and a new partition [.filename]#/dev/ad1s1d# was created. + Creating the journal would be as simple as: + + [source,shell] + .... + # gjournal label ad1s1d + .... + + The journal size will be 1 GB by default. + You may adjust it by using the `-s` option. + The value can be given in bytes, or appended by `K`, `M` or `G` to denote Kilobytes, Megabytes or Gigabytes respectively. + Note that `gjournal` will not allow you to create unsuitably small journal sizes. + + For example, to create a 2 GB journal, you could use the following command: + + [source,shell] + .... + # gjournal label -s 2G ad1s1d + .... + + You can then create a file system on your new partition, and enable journaling using the `-J` option: + + [source,shell] + .... + # newfs -J /dev/ad1s1d.journal + .... + + [[configure-kernel]] + === Building Journaling into Your Custom Kernel + + If you do not wish to load `geom_journal` as a module, you can build its functions right into your kernel. + Edit your custom kernel configuration file, and make sure it includes these two lines: + + [.programlisting] + .... + options UFS_GJOURNAL # Note: This is already in GENERIC + + options GEOM_JOURNAL # You will have to add this one + .... + +-Rebuild and reinstall your kernel following the relevant extref:{handbook}[instructions in the FreeBSD Handbook., kernelconfig] ++Rebuild and reinstall your kernel following the relevant extref:{handbook}kernelconfig[instructions in the FreeBSD Handbook., kernelconfig] + + Do not forget to remove the relevant "load" entry from [.filename]#/boot/loader.conf# if you have previously used it. + + [[troubleshooting-gjournal]] + == Troubleshooting Journaling + + The following section covers frequently asked questions regarding problems related to journaling. + + === I am getting kernel panics during periods of high disk activity. How is this related to journaling? + + The journal probably fills up before it has a chance to get committed (flushed) to disk. + Keep in mind the size of the journal depends on the usage load, and not the size of the data provider. + If your disk activity is high, you need a larger partition for the journal. + See the note in the crossref:gjournal-desktop[understanding-journaling, Understanding Journaling in FreeBSD] section. + + === I made some mistake during configuration, and I cannot boot normally now. Can this be fixed some way? + + You either forgot (or misspelled) the entry in [.filename]#/boot/loader.conf#, or there are errors in your [.filename]#/etc/fstab# file. + These are usually easy to fix. + Press kbd:[Enter] to get to the default single user shell. + Then locate the root of the problem: + + [source,shell] + .... + # cat /boot/loader.conf + .... + + If the `geom_journal_load` entry is missing or misspelled, the journaled devices are never created. + Load the module manually, mount all partitions, and continue with multi-user boot: + + [source,shell] + .... + # gjournal load + + GEOM_JOURNAL: Journal 2948326772: ad0s1g contains journal. + GEOM_JOURNAL: Journal 3193218002: ad0s1h contains journal. + GEOM_JOURNAL: Journal 3193218002: ad0s1d contains data. + GEOM_JOURNAL: Journal ad0s1d clean. + GEOM_JOURNAL: Journal 2948326772: ad0s1f contains data. + GEOM_JOURNAL: Journal ad0s1f clean. + + # mount -a + # exit + (boot continues) + .... + + If, on the other hand, this entry is correct, have a look at [.filename]#/etc/fstab#. + You will probably find a misspelled or missing entry. + In this case, mount all remaining partitions by hand and continue with the multi-user boot. + + === Can I remove journaling and return to my standard file system with Soft Updates? + + Sure. + Use the following procedure, which reverses the changes. + The partitions you created for the journal providers can then be used for other purposes, if you so wish. + + Login as `root` and switch to single user mode: + + [source,shell] + .... + # shutdown now + .... + + Unmount the journaled partitions: + + [source,shell] + .... + # umount /usr /var + .... + + Synchronize the journals: + + [source,shell] + .... + # gjournal sync + .... + + Stop the journaling providers: + + [source,shell] + .... + # gjournal stop ad0s1d.journal + # gjournal stop ad0s1f.journal + .... + + Clear journaling metadata from all the devices used: + + [source,shell] + .... + # gjournal clear ad0s1d + # gjournal clear ad0s1f + # gjournal clear ad0s1g + # gjournal clear ad0s1h + .... + + Clear the file system journaling flag, and restore the Soft Updates flag: + + [source,shell] + .... + # tunefs -J disable -n enable ad0s1d + tunefs: gjournal cleared + tunefs: soft updates set + + # tunefs -J disable -n enable ad0s1f + tunefs: gjournal cleared + tunefs: soft updates set + .... + + Remount the old devices by hand: + + [source,shell] + .... + # mount -o rw /dev/ad0s1d /var + # mount -o rw /dev/ad0s1f /usr + .... + + Edit [.filename]#/etc/fstab# and restore it to its original state: + + [.programlisting] + .... + /dev/ad0s1f /usr ufs rw 2 2 + /dev/ad0s1d /var ufs rw 2 2 + .... + + Finally, edit [.filename]#/boot/loader.conf#, remove the entry that loads the `geom_journal` module and reboot. + + [[further-reading]] + == Further Reading + + Journaling is a fairly new feature of FreeBSD, and as such, it is not very well documented yet. + You may however find the following additional references useful: + +-* A extref:{handbook}[new section on journaling, geom-gjournal] is now part of the FreeBSD Handbook. ++* A extref:{handbook}geom[new section on journaling, geom-gjournal] is now part of the FreeBSD Handbook. + * https://lists.freebsd.org/pipermail/freebsd-current/2006-June/064043.html[This post] in {freebsd-current} by man:gjournal[8]'s developer, `{pjd}`. + * https://lists.freebsd.org/pipermail/freebsd-questions/2008-April/173501.html[This post] in {freebsd-questions} by `{ivoras}`. + * The manual pages of man:gjournal[8] and man:geom[8]. +diff --git a/documentation/content/en/articles/hubs/_index.adoc b/documentation/content/en/articles/hubs/_index.adoc +index 1ddbe3065a..8445423a5d 100644 +--- a/documentation/content/en/articles/hubs/_index.adoc ++++ b/documentation/content/en/articles/hubs/_index.adoc +@@ -1,420 +1,420 @@ + --- + title: Mirroring FreeBSD + authors: + - author: Jun Kuriyama + email: kuriyama@FreeBSD.org + - author: Valentino Vaschetto + email: logo@FreeBSD.org + - author: Daniel Lang + email: dl@leo.org + - author: Ken Smith + email: kensmith@FreeBSD.org + description: The all in one guide for mirroring the FreeBSD website, FTP servers, and more + trademarks: ["freebsd", "general"] + tags: ["Mirroring", "FreeBSD", "Hub"] + --- + + = Mirroring FreeBSD + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/hubs/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + An in-progress article on how to mirror FreeBSD, aimed at hub administrators. + + ''' + + toc::[] + + [NOTE] + ==== + We are not accepting new community mirrors at this time. + ==== + + [[mirror-contact]] + == Contact Information + + The Mirror System Coordinators can be reached through email at mailto:mirror-admin@FreeBSD.org[mirror-admin@FreeBSD.org]. + There is also a {freebsd-hubs}. + + [[mirror-requirements]] + == Requirements for FreeBSD Mirrors + + [[mirror-diskspace]] + === Disk Space + + Disk space is one of the most important requirements. + Depending on the set of releases, architectures, and degree of completeness you want to mirror, a huge amount of disk space may be consumed. + Also keep in mind that _official_ mirrors are probably required to be complete. + The web pages should always be mirrored completely. + Also note that the numbers stated here are reflecting the current state (at {rel120-current}-RELEASE/{rel113-current}-RELEASE). + Further development and releases will only increase the required amount. + Also make sure to keep some (ca. 10-20%) extra space around just to be sure. + Here are some approximate figures: + + * Full FTP Distribution: 1.4 TB + * CTM deltas: 10 GB + * Web pages: 1GB + + The current disk usage of FTP Distribution can be found at link:ftp://ftp.FreeBSD.org/pub/FreeBSD/dir.sizes[ftp://ftp.FreeBSD.org/pub/FreeBSD/dir.sizes]. + + [[mirror-bandwidth]] + === Network Connection/Bandwidth + + Of course, you need to be connected to the Internet. + The required bandwidth depends on your intended use of the mirror. + If you just want to mirror some parts of FreeBSD for local use at your site/intranet, the demand may be much smaller than if you want to make the files publicly available. + If you intend to become an official mirror, the bandwidth required will be even higher. + We can only give rough estimates here: + + * Local site, no public access: basically no minimum, but < 2 Mbps could make syncing too slow. + * Unofficial public site: 34 Mbps is probably a good start. + * Official site: > 100 Mbps is recommended, and your host should be connected as close as possible to your border router. + + [[mirror-system]] + === System Requirements, CPU, RAM + + One thing this depends on the expected number of clients, which is determined by the server's policy. + It is also affected by the types of services you want to offer. + Plain FTP or HTTP services may not require a huge amount of resources. + Watch out if you provide rsync. + This can have a huge impact on CPU and memory requirements as it is considered a memory hog. + The following are just examples to give you a very rough hint. + + For a moderately visited site that offers rsync, you might consider a current CPU with around 800MHz - 1 GHz, and at least 512MB RAM. + This is probably the minimum you want for an _official_ site. + + For a frequently used site you definitely need more RAM (consider 2GB as a good start) and possibly more CPU, which could also mean that you need to go for a SMP system. + + You also want to consider a fast disk subsystem. + Operations on the SVN repository require a fast disk subsystem (RAID is highly advised). + A SCSI controller that has a cache of its own can also speed up things since most of these services incur a large number of small modifications to the disk. + + [[mirror-services]] + === Services to Offer + + Every mirror site is required to have a set of core services available. + In addition to these required services, there are a number of optional services that server administrators may choose to offer. + This section explains which services you can provide and how to go about implementing them. + + [[mirror-serv-ftp]] + ==== FTP (required for FTP Fileset) + + This is one of the most basic services, and it is required for each mirror offering public FTP distributions. + FTP access must be anonymous, and no upload/download ratios are allowed (a ridiculous thing anyway). + Upload capability is not required (and _must_ never be allowed for the FreeBSD file space). + Also the FreeBSD archive should be available under the path [.filename]#/pub/FreeBSD#. + + There is a lot of software available which can be set up to allow anonymous FTP (in alphabetical order). + + * `/usr/libexec/ftpd`: FreeBSD's own ftpd can be used. Be sure to read man:ftpd[8]. + * package:ftp/ncftpd[]: A commercial package, free for educational use. + * package:ftp/oftpd[]: An ftpd designed with security as a main focus. + * package:ftp/proftpd[]: A modular and very flexible ftpd. + * package:ftp/pure-ftpd[]: Another ftpd developed with security in mind. + * package:ftp/twoftpd[]: As above. + * package:ftp/vsftpd[]: The "very secure" ftpd. + + FreeBSD's `ftpd`, `proftpd` and maybe `ncftpd` are among the most commonly used FTPds. + The others do not have a large userbase among mirror sites. + One thing to consider is that you may need flexibility in limiting how many simultaneous connections are allowed, thus limiting how much network bandwidth and system resources are consumed. + + [[mirror-serv-rsync]] + ==== Rsync (optional for FTP Fileset) + + Rsync is often offered for access to the contents of the FTP area of FreeBSD, so other mirror sites can use your system as their source. + The protocol is different from FTP in many ways. + It is much more bandwidth friendly, as only differences between files are transferred instead of whole files when they change. + Rsync does require a significant amount of memory for each instance. + The size depends on the size of the synced module in terms of the number of directories and files. + Rsync can use `rsh` and `ssh` (now default) as a transport, or use its own protocol for stand-alone access (this is the preferred method for public rsync servers). + Authentication, connection limits, and other restrictions may be applied. + There is just one software package available: + + * package:net/rsync[] + + [[mirror-serv-http]] + ==== HTTP (required for Web Pages, Optional for FTP Fileset) + + If you want to offer the FreeBSD web pages, you will need to install a web server. + You may optionally offer the FTP fileset via HTTP. + The choice of web server software is left up to the mirror administrator. + Some of the most popular choices are: + + * package:www/apache24[]: Apache is still one of the most widely deployed web servers on the Internet. It is used extensively by the FreeBSD Project. + * package:www/boa[]: Boa is a single-tasking HTTP server. Unlike traditional web servers, it does not fork for each incoming connection, nor does it fork many copies of itself to handle multiple connections. Although, it should provide considerably great performance for purely static content. + * package:www/cherokee[]: Cherokee is a very fast, flexible and easy to configure web server. It supports the widespread technologies nowadays: FastCGI, SCGI, PHP, CGI, SSL/TLS encrypted connections, vhosts, users authentication, on the fly encoding and load balancing. It also generates Apache compatible log files. + * package:www/lighttpd[]: lighttpd is a secure, fast, compliant and very flexible web server which has been optimized for high-performance environments. It has a very low memory footprint compared to other web servers and takes care of cpu-load. + * package:www/nginx[]: nginx is a high performance edge web server with a low memory footprint and key features to build a modern and efficient web infrastructure. Features include a HTTP server, HTTP and mail reverse proxy, caching, load balancing, compression, request throttling, connection multiplexing and reuse, SSL offload and HTTP media streaming. + * package:www/thttpd[]: If you are going to be serving a large amount of static content you may find that using an application such as thttpd is more efficient than others. It is also optimized for excellent performance on FreeBSD. + + [[mirror-howto]] + == How to Mirror FreeBSD + + Ok, now you know the requirements and how to offer the services, but not how to get it. + :-) This section explains how to actually mirror the various parts of FreeBSD, what tools to use, and where to mirror from. + + [[mirror-ftp-rsync]] + === Mirroring the FTP Site + + The FTP area is the largest amount of data that needs to be mirrored. + It includes the _distribution sets_ required for network installation, the _branches_ which are actually snapshots of checked-out source trees, the _ISO Images_ to write CD-ROMs with the installation distribution, a live file system, and a snapshot of the ports tree. + All of course for various FreeBSD versions, and various architectures. + + The best way to mirror the FTP area is rsync. + You can install the port package:net/rsync[] and then use rsync to sync with your upstream host. + rsync is already mentioned in crossref:hubs[mirror-serv-rsync, Rsync (optional for FTP Fileset)]. + Since rsync access is not required, your preferred upstream site may not allow it. + You may need to hunt around a little bit to find a site that allows rsync access. + + [NOTE] + ==== + Since the number of rsync clients will have a significant impact on the server machine, most admins impose limitations on their server. + For a mirror, you should ask the site maintainer you are syncing from about their policy, and maybe an exception for your host (since you are a mirror). + ==== + + A command line to mirror FreeBSD might look like: + + [source,shell] + .... + % rsync -vaHz --delete rsync://ftp4.de.FreeBSD.org/FreeBSD/ /pub/FreeBSD/ + .... + + Consult the documentation for rsync, which is also available at http://rsync.samba.org/[http://rsync.samba.org/], about the various options to be used with rsync. + If you sync the whole module (unlike subdirectories), be aware that the module-directory (here "FreeBSD") will not be created, so you cannot omit the target directory. Also you might want to set up a script framework that calls such a command via man:cron[8]. + + [[mirror-www]] + === Mirroring the WWW Pages + + [WARNING] + ==== + Since doc migration to Hugo/Asciidoctor on 2021-01-25, mirroring the website with rsync no longer works. + ==== + + There are ongoing studies to implement a website mirror with the extref:{handbook}mirrors/[official infrastructure]. + + For the former website mirrors, a way to achieve mirroring the website today is building the website locally with the corresponding address it will be hosted. + + [source,shell] + .... + % cd website && env HUGO_baseURL="https://www.XX.freebsd.org/" make + .... + + Check for more details about the build tools on extref:{fdp-primer}overview/[FreeBSD Documentation Project Primer for New Contributors, overview-quick-start] book. + + //// + [source,shell] + .... + % rsync -vaHz --delete rsync://bit0.us-west.freebsd.org/FreeBSD-www-data/ /usr/local/www/ + .... + //// + + [NOTE] + ==== + Notice the website was split into www.FreeBSD.org and docs.FreeBSD.org, and there are links between them; plus, at this moment, `HUGO_baseURL` variable won't cover all links, this way, mirroring the website is discouraged. + ==== + + [[mirror-pkgs]] + === Mirroring Packages + + Due to very high requirements of bandwidth, storage and administration the FreeBSD Project has decided not to allow public mirrors of packages. + For sites with lots of machines, it might be advantagous to run a caching HTTP proxy for the man:pkg[8] process. + Alternatively specific packages and their dependencies can be fetched by running something like the following: + + [source,shell] + .... + % pkg fetch -d -o /usr/local/mirror vim + .... + + Once those packages have been fetched, the repository metadata must be generated by running: + + [source,shell] + .... + % pkg repo /usr/local/mirror + .... + + Once the packages have been fetched and the metadata for the repository has been generated, serve the packages up to the client machines via HTTP. + For additional information see the man pages for man:pkg[8], specifically the man:pkg-repo[8] page. + + [[mirror-how-often]] + === How Often Should I Mirror? + + Every mirror should be updated at a minimum of once per day. + Certainly a script with locking to prevent multiple runs happening at the same time will be needed to run from man:cron[8]. + Since nearly every admin does this in their own way, specific instructions cannot be provided. + It could work something like this: + + [.procedure] + ==== + . Put the command to run your mirroring application in a script. Use of a plain `/bin/sh` script is recommended. + . Add some output redirections so diagnostic messages are logged to a file. + . Test if your script works. Check the logs. + . Use man:crontab[1] to add the script to the appropriate user's man:crontab[5]. This should be a different user than what your FTP daemon runs as so that if file permissions inside your FTP area are not world-readable those files cannot be accessed by anonymous FTP. This is used to "stage" releases - making sure all of the official mirror sites have all of the necessary release files on release day. + ==== + + Here are some recommended schedules: + + * FTP fileset: daily + * WWW pages: daily + + [[mirror-where]] + == Where to Mirror From + + This is an important issue. + So this section will spend some effort to explain the backgrounds. + We will say this several times: under no circumstances should you mirror from `ftp.FreeBSD.org`. + + [[mirror-where-organization]] + === A few Words About the Organization + + Mirrors are organized by country. + All official mirrors have a DNS entry of the form `ftpN.CC.FreeBSD.org`. + _CC_ (i.e., country code) is the _top level domain_ (TLD) of the country where this mirror is located. + _N_ is a number, telling that the host would be the _Nth_ mirror in that country. + (Same applies to `wwwN.CC.FreeBSD.org`, etc.) There are mirrors with no _CC_ part. + These are the mirror sites that are very well connected and allow a large number of concurrent users. + `ftp.FreeBSD.org` is actually two machines, one currently located in Denmark and the other in the United States. + It is _NOT_ a master site and should never be used to mirror from. + Lots of online documentation leads "interactive"users to `ftp.FreeBSD.org` so automated mirroring systems should find a different machine to mirror from. + + Additionally there exists a hierarchy of mirrors, which is described in terms of __tiers__. + The master sites are not referred to but can be described as __Tier-0__. + Mirrors that mirror from these sites can be considered __Tier-1__, mirrors of __Tier-1__-mirrors, are __Tier-2__, etc. + Official sites are encouraged to be of a low __tier__, but the lower the tier + the higher the requirements in terms as described in + crossref:hubs[mirror-requirements, Requirements for FreeBSD Mirrors]. + Also access to low-tier-mirrors may be restricted, and access to master sites is definitely restricted. + The __tier__-hierarchy is not reflected by DNS and generally not documented anywhere except for the master sites. + However, official mirrors with low numbers like 1-4, are usually _Tier-1_ (this is just a rough hint, and there is no rule). + + [[mirror-where-where]] + === Ok, but Where Should I get the Stuff Now? + + Under no circumstances should you mirror from `ftp.FreeBSD.org`. + The short answer is: from the site that is closest to you in Internet terms, or gives you the fastest access. + + [[mirror-where-simple]] + ==== I Just Want to Mirror from Somewhere! + + If you have no special intentions or requirements, the statement in + crossref:hubs[mirror-where-where, Ok, but Where Should I get the Stuff Now?] applies. + This means: + + [.procedure] + ==== + . Check for those which provide fastest access (number of hops, round-trip-times) and offer the services you intend to use (like rsync). + . Contact the administrators of your chosen site stating your request, and asking about their terms and policies. + . Set up your mirror as described above. + ==== + + [[mirror-where-official]] + ==== I am an Official Mirror, What is the Right Site for Me? + + In general the description in crossref:hubs[mirror-where-simple, I Just Want to Mirror from Somewhere!] still applies. + Of course you may want to put some weight on the fact that your upstream should be of a low tier. + There are some other considerations about _official_ mirrors that are described + in crossref:hubs[mirror-official, Official Mirrors]. + + [[mirror-where-master]] + ==== I Want to Access the Master Sites! + + If you have good reasons and good prerequisites, you may want and get access to one of the master sites. + Access to these sites is generally restricted, and there are special policies for access. + If you are already an _official_ mirror, this certainly helps you getting access. + In any other case make sure your country really needs another mirror. + If it already has three or more, ask the "zone administrator" (mailto:hostmaster@CC.FreeBSD.org[hostmaster@CC.FreeBSD.org]) or {freebsd-hubs} first. + + Whoever helped you become, an _official_ should have helped you gain access to an appropriate upstream host, either one of the master sites or a suitable Tier-1 site. + If not, you can send email to mailto:mirror-admin@FreeBSD.org[mirror-admin@FreeBSD.org] to request help with that. + + There is one master site for the FTP fileset. + + [[mirror-where-master-ftp]] + ===== ftp-master.FreeBSD.org + + This is the master site for the FTP fileset. + + `ftp-master.FreeBSD.org` provides rsync access, in addition to FTP. + Refer to crossref:hubs[mirror-ftp-rsync, Mirroring the FTP Site]. + + Mirrors are also encouraged to allow rsync access for the FTP contents, since they are __Tier-1__-mirrors. + + [[mirror-official]] + == Official Mirrors + + Official mirrors are mirrors that + + * a) have a `FreeBSD.org` DNS entry (usually a CNAME). + * b) are listed as an official mirror in the FreeBSD documentation (like handbook). + + So far to distinguish official mirrors. Official mirrors are not necessarily __Tier-1__-mirrors. + However you probably will not find a __Tier-1__-mirror, that is not also official. + + [[mirror-official-requirements]] + === Special Requirements for Official (tier-1) Mirrors + + It is not so easy to state requirements for all official mirrors, since the project is sort of tolerant here. + It is more easy to say, what _official tier-1 mirrors_ are required to. + All other official mirrors can consider this a big __should__. + + Tier-1 mirrors are required to: + + * carry the complete fileset + * allow access to other mirror sites + * provide FTP and rsync access + + Furthermore, admins should be subscribed to the {freebsd-hubs}. +-See extref:{handbook}[this link, eresources-mail] for details, how to subscribe. ++See extref:{handbook}eresources[this link, eresources-mail] for details, how to subscribe. + + [IMPORTANT] + ==== + It is _very_ important for a hub administrator, especially Tier-1 hub admins, to check the https://www.FreeBSD.org/releng/[release schedule] for the next FreeBSD release. + This is important because it will tell you when the next release is scheduled to come out, and thus giving you time to prepare for the big spike of traffic which follows it. + + It is also important that hub administrators try to keep their mirrors as up-to-date as possible (again, even more crucial for Tier-1 mirrors). + If Mirror1 does not update for a while, lower tier mirrors will begin to mirror old data from Mirror1 and thus begins a downward spiral... Keep your mirrors up to date! + ==== + + [[mirror-official-become]] + === How to Become Official Then? + + Please contact the Cluster Administrators as documented at https://www.FreeBSD.org/administration/#t-clusteradm. + + [[mirror-statpages]] + == Some Statistics from Mirror Sites + + Here are links to the stat pages of your favorite mirrors (aka the only ones who feel like providing stats). + + [[mirror-statpagesftp]] + === FTP Site Statistics + + * ftp.is.FreeBSD.org - mailto:hostmaster@is.FreeBSD.org[hostmaster@is.FreeBSD.org] - http://www.rhnet.is/status/draupnir/draupnir.html[ (Bandwidth)] http://www.rhnet.is/status/ftp/ftp-notendur.html[(FTP processes)] http://www.rhnet.is/status/ftp/http-notendur.html[(HTTP processes)] + * ftp2.ru.FreeBSD.org - mailto:mirror@macomnet.ru[mirror@macomnet.ru] - http://mirror.macomnet.net/mrtg/mirror.macomnet.net_195.128.64.25.html[(Bandwidth)] http://mirror.macomnet.net/mrtg/mirror.macomnet.net_proc.html[(HTTP and FTP users)] +diff --git a/documentation/content/en/articles/ipsec-must/_index.adoc b/documentation/content/en/articles/ipsec-must/_index.adoc +index dbca759c6e..262ffef368 100644 +--- a/documentation/content/en/articles/ipsec-must/_index.adoc ++++ b/documentation/content/en/articles/ipsec-must/_index.adoc +@@ -1,295 +1,295 @@ + --- + title: Independent Verification of IPsec Functionality in FreeBSD + authors: + - author: David Honig + email: honig@sprynet.com + description: Independent Verification of IPsec Functionality in FreeBSD + trademarks: ["freebsd", "opengroup", "general"] + tags: ["IPsec", "verification", "FreeBSD"] + --- + + = Independent Verification of IPsec Functionality in FreeBSD + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/ipsec-must/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + You installed IPsec and it seems to be working. + How do you know? I describe a method for experimentally verifying that IPsec is working. + + ''' + + toc::[] + + [[problem]] + == The Problem + + First, lets assume you have crossref::ipsec-must[ipsec-install, Installing IPsec]. + How do you know it is crossref::ipsec-must[caveat, Caveat]? Sure, your connection will not work if it is misconfigured, and it will work when you finally get it right. + man:netstat[1] will list it. But can you independently confirm it? + + [[solution]] + == The Solution + + First, some crypto-relevant info theory: + + . Encrypted data is uniformly distributed, i.e., has maximal entropy per symbol; + . Raw, uncompressed data is typically redundant, i.e., has sub-maximal entropy. + + Suppose you could measure the entropy of the data to- and from- your network interface. + Then you could see the difference between unencrypted data and encrypted data. + This would be true even if some of the data in "encrypted mode" was not encrypted---as the outermost IP header must be if the packet is to be routable. + + [[MUST]] + === MUST + + Ueli Maurer's "Universal Statistical Test for Random Bit Generators"(https://web.archive.org/web/20011115002319/http://www.geocities.com/SiliconValley/Code/4704/universal.pdf[MUST]) quickly measures the entropy of a sample. + It uses a compression-like algorithm. + crossref::ipsec-must[code, Maurer's Universal Statistical Test (for block size8 bits)] for a variant which measures successive (~quarter megabyte) chunks of a file. + + [[tcpdump]] + === Tcpdump + + We also need a way to capture the raw network data. + A program called man:tcpdump[1] lets you do this, if you have enabled the + _Berkeley Packet Filter_ interface in your crossref::ipsec-must[kernel,src/sys/i386/conf/KERNELNAME]. + + The command: + + [source,shell] + .... + tcpdump -c 4000 -s 10000 -w dumpfile.bin + .... + + will capture 4000 raw packets to _dumpfile.bin_. + Up to 10,000 bytes per packet will be captured in this example. + + [[experiment]] + == The Experiment + + Here is the experiment: + + [.procedure] + ==== + . Open a window to an IPsec host and another window to an insecure host. + . Now start crossref::ipsec-must[tcpdump, Tcpdump]. + . In the "secure" window, run the UNIX(R) command man:yes[1], which will stream the `y` character. After a while, stop this. Switch to the insecure window, and repeat. After a while, stop. + . Now run crossref::ipsec-must[code, Maurer's Universal Statistical Test (for block size8 bits)] on the captured packets. You should see something like the following. The important thing to note is that the secure connection has 93% (6.7) of the expected value (7.18), and the "normal" connection has 29% (2.1) of the expected value. + + + [source,shell] + .... + % tcpdump -c 4000 -s 10000 -w ipsecdemo.bin + % uliscan ipsecdemo.bin + Uliscan 21 Dec 98 + L=8 256 258560 + Measuring file ipsecdemo.bin + Init done + Expected value for L=8 is 7.1836656 + 6.9396 -------------------------------------------------------- + 6.6177 ----------------------------------------------------- + 6.4100 --------------------------------------------------- + 2.1101 ----------------- + 2.0838 ----------------- + 2.0983 ----------------- + .... + ==== + + [[caveat]] + == Caveat + + This experiment shows that IPsec _does_ seem to be distributing the payload data __uniformly__, as encryption should. + However, the experiment described here _cannot_ detect many possible flaws in a system (none of which do I have any evidence for). + These include poor key generation or exchange, data or keys being visible to others, use of weak algorithms, kernel subversion, etc. + Study the source; know the code. + + [[IPsec]] + == IPsec---Definition + + Internet Protocol security extensions to IPv4; required for IPv6. + A protocol for negotiating encryption and authentication at the IP (host-to-host) level. + SSL secures only one application socket; SSH secures only a login; PGP secures only a specified file or message. + IPsec encrypts everything between two hosts. + + [[ipsec-install]] + == Installing IPsec + + Most of the modern versions of FreeBSD have IPsec support in their base source. + So you will need to include the `IPSEC` option in your kernel config and, after kernel rebuild and reinstall, configure IPsec connections using man:setkey[8] command. + +-A comprehensive guide on running IPsec on FreeBSD is provided in extref:{handbook}[FreeBSD Handbook, ipsec]. ++A comprehensive guide on running IPsec on FreeBSD is provided in extref:{vpn-ipsec}[VPN over IPsec]. + + [[kernel]] + == src/sys/i386/conf/KERNELNAME + + This needs to be present in the kernel config file to capture network data with man:tcpdump[1]. + Be sure to run man:config[8] after adding this, and rebuild and reinstall. + + [.programlisting] + .... + device bpf + .... + + [[code]] + == Maurer's Universal Statistical Test (for block size=8 bits) + + You can find the same code at https://web.archive.org/web/20031204230654/http://www.geocities.com:80/SiliconValley/Code/4704/uliscanc.txt[this link]. + + [.programlisting] + .... + /* + ULISCAN.c ---blocksize of 8 + + 1 Oct 98 + 1 Dec 98 + 21 Dec 98 uliscan.c derived from ueli8.c + + This version has // comments removed for Sun cc + + This implements Ueli M Maurer's "Universal Statistical Test for Random + Bit Generators" using L=8 + + Accepts a filename on the command line; writes its results, with other + info, to stdout. + + Handles input file exhaustion gracefully. + + Ref: J. Cryptology v 5 no 2, 1992 pp 89-105 + also on the web somewhere, which is where I found it. + + -David Honig + honig@sprynet.com + + Usage: + ULISCAN filename + outputs to stdout + */ + + #define L 8 + #define V (1< + #include + + int main(argc, argv) + int argc; + char **argv; + { + FILE *fptr; + int i,j; + int b, c; + int table[V]; + double sum = 0.0; + int iproduct = 1; + int run; + + extern double log(/* double x */); + + printf("Uliscan 21 Dec 98 \nL=%d %d %d \n", L, V, MAXSAMP); + + if (argc < 2) { + printf("Usage: Uliscan filename\n"); + exit(-1); + } else { + printf("Measuring file %s\n", argv[1]); + } + + fptr = fopen(argv[1],"rb"); + + if (fptr == NULL) { + printf("Can't find %s\n", argv[1]); + exit(-1); + } + + for (i = 0; i < V; i++) { + table[i] = 0; + } + + for (i = 0; i < Q; i++) { + b = fgetc(fptr); + table[b] = i; + } + + printf("Init done\n"); + + printf("Expected value for L=8 is 7.1836656\n"); + + run = 1; + + while (run) { + sum = 0.0; + iproduct = 1; + + if (run) + for (i = Q; run && i < Q + K; i++) { + j = i; + b = fgetc(fptr); + + if (b < 0) + run = 0; + + if (run) { + if (table[b] > j) + j += K; + + sum += log((double)(j-table[b])); + + table[b] = i; + } + } + + if (!run) + printf("Premature end of file; read %d blocks.\n", i - Q); + + sum = (sum/((double)(i - Q))) / log(2.0); + printf("%4.4f ", sum); + + for (i = 0; i < (int)(sum*8.0 + 0.50); i++) + printf("-"); + + printf("\n"); + + /* refill initial table */ + if (0) { + for (i = 0; i < Q; i++) { + b = fgetc(fptr); + if (b < 0) { + run = 0; + } else { + table[b] = i; + } + } + } + } + } + .... +diff --git a/documentation/content/en/articles/leap-seconds/_index.adoc b/documentation/content/en/articles/leap-seconds/_index.adoc +index d28629b005..55a1dfb5fc 100644 +--- a/documentation/content/en/articles/leap-seconds/_index.adoc ++++ b/documentation/content/en/articles/leap-seconds/_index.adoc +@@ -1,104 +1,104 @@ + --- + title: FreeBSD Support for Leap Seconds + description: A short description of how leap seconds are handled on FreeBSD + tags: ["Leap Seconds", "Support", "Verification", "FreeBSD"] + --- + + = FreeBSD Support for Leap Seconds + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/leap-seconds/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + ''' + + toc::[] + + [[leapseconds-definition]] + == Introduction + + A _leap second_ is an one second adjustment made at specific times of year to UTC to synchronize atomic time scales with variations in the rotation of the Earth. + This article describes how FreeBSD interacts with leap seconds. + + As of this writing, the next leap second will occur at 2015-Jun-30 23:59:60 UTC. + This leap second will occur during a business day for North and South America and the Asia/Pacific region. + + Leap seconds are announced by https://www.iers.org/IERS/EN/Home/home_node.html[IERS] on https://datacenter.iers.org/data/latestVersion/16_BULLETIN_C16.txt[Bulletin C]. + + Standard leap second behavior is described in https://datatracker.ietf.org/doc/html/rfc7164#section-3[RFC 7164]. + Also see man:time2posix[3]. + + [[leapseconds-posix]] + == Default Leap Second Handling on FreeBSD + +-The easiest way to handle leap seconds is with the POSIX time rules FreeBSD uses by default, combined with extref:{handbook}[NTP, network-ntp]. ++The easiest way to handle leap seconds is with the POSIX time rules FreeBSD uses by default, combined with extref:{handbook}network-servers[NTP, network-ntp]. + When man:ntpd[8] is running and the time is synchronized with upstream NTP servers that handle leap seconds correctly, the leap second will cause the system time to automatically repeat the last second of the day. + No other adjustments are necessary. + + If the upstream NTP servers do not handle leap seconds correctly, man:ntpd[8] will step the time by one second after the errant upstream server has noticed and stepped itself. + + If NTP is not being used, manual adjustment of the system clock will be required after the leap second has passed. + + [[leapseconds-cautions]] + == Cautions + + Leap seconds are inserted at the same instant all over the world: UTC midnight. + In Japan that is mid-morning, in the Pacific mid-day, in the Americas late afternoon, and in Europe at night. + + We believe and expect that FreeBSD, if provided correct and stable NTP service, will work as designed during this leap second, as it did during the previous ones. + + However, we caution that practically no applications have ever asked the kernel about leap seconds. + Our experience is that, as designed, leap seconds are essentially a replay of the second before the leap second, and this is a surprise to most application programmers. + + Other operating systems and other computers may or may not handle the leap-second the same way as FreeBSD, and systems without correct and stable NTP service will not know anything about leap seconds at all. + + It is not unheard of for computers to crash because of leap seconds, and experience has shown that a large fraction of all public NTP servers might handle and announce the leap second incorrectly. + + Please try to make sure nothing horrible happens because of the leap second. + + [[leapseconds-testing]] + == Testing + + It is possible to test whether a leap second will be used. + Due to the nature of NTP, the test might work up to 24 hours before the leap second. + Some major reference clock sources only announce leap seconds one hour ahead of the event. + Query the NTP daemon: + + [source,shell] + .... + % ntpq -c 'rv 0 leap' + .... + + Output that includes `leap_add_sec` indicates proper support of the leap second. + Before the 24 hours leading up to the leap second, or after the leap second has passed, `leap_none` will be shown. + + [[leapseconds-conclusion]] + == Conclusion + + In practice, leap seconds are usually not a problem on FreeBSD. + We hope that this overview helps clarify what to expect and how to make the leap second event proceed more smoothly. +diff --git a/documentation/content/en/articles/linux-emulation/_index.adoc b/documentation/content/en/articles/linux-emulation/_index.adoc +index a252df5f54..151398ff40 100644 +--- a/documentation/content/en/articles/linux-emulation/_index.adoc ++++ b/documentation/content/en/articles/linux-emulation/_index.adoc +@@ -1,1436 +1,1436 @@ + --- + title: Linux® emulation in FreeBSD + authors: + - author: Roman Divacky + email: rdivacky@FreeBSD.org + description: A technical description about the internals of the Linux emulation layer in FreeBSD + trademarks: ["freebsd", "ibm", "adobe", "netbsd", "realnetworks", "oracle", "linux", "sun", "general"] + tags: ["Emulation", "Linuxulator", "kernel", "FreeBSD"] + --- + + = Linux(R) emulation in FreeBSD + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/linux-emulation/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + This masters thesis deals with updating the Linux(R) emulation layer (the so called _Linuxulator_). + The task was to update the layer to match the functionality of Linux(R) 2.6. + As a reference implementation, the Linux(R) 2.6.16 kernel was chosen. + The concept is loosely based on the NetBSD implementation. + Most of the work was done in the summer of 2006 as a part of the Google Summer of Code students program. + The focus was on bringing the _NPTL_ (new POSIX(R) thread library) support into the emulation layer, including _TLS_ (thread local storage), _futexes_ (fast user space mutexes), _PID mangling_, and some other minor things. + Many small problems were identified and fixed in the process. + My work was integrated into the main FreeBSD source repository and will be shipped in the upcoming 7.0R release. + We, the emulation development team, are working on making the Linux(R) 2.6 emulation the default emulation layer in FreeBSD. + + ''' + + toc::[] + + [[intro]] + == Introduction + + In the last few years the open source UNIX(R) based operating systems started to be widely deployed on server and client machines. + Among these operating systems I would like to point out two: FreeBSD, for its BSD heritage, time proven code base and many interesting features and Linux(R) for its wide user base, enthusiastic open developer community and support from large companies. + FreeBSD tends to be used on server class machines serving heavy duty networking tasks with less usage on desktop class machines for ordinary users. + While Linux(R) has the same usage on servers, but it is used much more by home based users. + This leads to a situation where there are many binary only programs available for Linux(R) that lack support for FreeBSD. + + Naturally, a need for the ability to run Linux(R) binaries on a FreeBSD system arises and this is what this thesis deals with: the emulation of the Linux(R) kernel in the FreeBSD operating system. + + During the Summer of 2006 Google Inc. sponsored a project which focused on extending the Linux(R) emulation layer (the so called Linuxulator) in FreeBSD to include Linux(R) 2.6 facilities. + This thesis is written as a part of this project. + + [[inside]] + == A look inside... + + In this section we are going to describe every operating system in question. + How they deal with syscalls, trapframes etc., all the low-level stuff. + We also describe the way they understand common UNIX(R) primitives like what a PID is, what a thread is, etc. + In the third subsection we talk about how UNIX(R) on UNIX(R) emulation could be done in general. + + [[what-is-unix]] + === What is UNIX(R) + + UNIX(R) is an operating system with a long history that has influenced almost every other operating system currently in use. + Starting in the 1960s, its development continues to this day (although in different projects). + UNIX(R) development soon forked into two main ways: the BSDs and System III/V families. + They mutually influenced themselves by growing a common UNIX(R) standard. + Among the contributions originated in BSD we can name virtual memory, TCP/IP networking, FFS, and many others. + The System V branch contributed to SysV interprocess communication primitives, copy-on-write, etc. + UNIX(R) itself does not exist any more but its ideas have been used by many other operating systems world wide thus forming the so called UNIX(R)-like operating systems. + These days the most influential ones are Linux(R), Solaris, and possibly (to some extent) FreeBSD. + There are in-company UNIX(R) derivatives (AIX, HP-UX etc.), but these have been more and more migrated to the aforementioned systems. + Let us summarize typical UNIX(R) characteristics. + + [[tech-details]] + === Technical details + + Every running program constitutes a process that represents a state of the computation. + Running process is divided between kernel-space and user-space. + Some operations can be done only from kernel space (dealing with hardware etc.), but the process should spend most of its lifetime in the user space. + The kernel is where the management of the processes, hardware, and low-level details take place. + The kernel provides a standard unified UNIX(R) API to the user space. + The most important ones are covered below. + + [[kern-proc-comm]] + ==== Communication between kernel and user space process + + Common UNIX(R) API defines a syscall as a way to issue commands from a user space process to the kernel. + The most common implementation is either by using an interrupt or specialized instruction (think of `SYSENTER`/`SYSCALL` instructions for ia32). + Syscalls are defined by a number. + For example in FreeBSD, the syscall number 85 is the man:swapon[2] syscall and the syscall number 132 is man:mkfifo[2]. + Some syscalls need parameters, which are passed from the user-space to the kernel-space in various ways (implementation dependent). + Syscalls are synchronous. + + Another possible way to communicate is by using a _trap_. + Traps occur asynchronously after some event occurs (division by zero, page fault etc.). + A trap can be transparent for a process (page fault) or can result in a reaction like sending a _signal_ (division by zero). + + [[proc-proc-comm]] + ==== Communication between processes + + There are other APIs (System V IPC, shared memory etc.) but the single most important API is signal. + Signals are sent by processes or by the kernel and received by processes. + Some signals can be ignored or handled by a user supplied routine, some result in a predefined action that cannot be altered or ignored. + + [[proc-mgmt]] + ==== Process management + + Kernel instances are processed first in the system (so called init). + Every running process can create its identical copy using the man:fork[2] syscall. + Some slightly modified versions of this syscall were introduced but the basic semantic is the same. + Every running process can morph into some other process using the man:exec[3] syscall. + Some modifications of this syscall were introduced but all serve the same basic purpose. + Processes end their lives by calling the man:exit[2] syscall. + Every process is identified by a unique number called PID. + Every process has a defined parent (identified by its PID). + + [[thread-mgmt]] + ==== Thread management + + Traditional UNIX(R) does not define any API nor implementation for threading, while POSIX(R) defines its threading API but the implementation is undefined. + Traditionally there were two ways of implementing threads. + Handling them as separate processes (1:1 threading) or envelope the whole thread group in one process and managing the threading in userspace (1:N threading). + Comparing main features of each approach: + + 1:1 threading + + * - heavyweight threads + * - the scheduling cannot be altered by the user (slightly mitigated by the POSIX(R) API) + * + no syscall wrapping necessary + * + can utilize multiple CPUs + + 1:N threading + + * + lightweight threads + * + scheduling can be easily altered by the user + * - syscalls must be wrapped + * - cannot utilize more than one CPU + + [[what-is-freebsd]] + === What is FreeBSD? + + The FreeBSD project is one of the oldest open source operating systems currently available for daily use. + It is a direct descendant of the genuine UNIX(R) so it could be claimed that it is a true UNIX(R) although licensing issues do not permit that. + The start of the project dates back to the early 1990's when a crew of fellow BSD users patched the 386BSD operating system. + Based on this patchkit a new operating system arose named FreeBSD for its liberal license. + Another group created the NetBSD operating system with different goals in mind. + We will focus on FreeBSD. + + FreeBSD is a modern UNIX(R)-based operating system with all the features of UNIX(R). + Preemptive multitasking, multiuser facilities, TCP/IP networking, memory protection, symmetric multiprocessing support, virtual memory with merged VM and buffer cache, they are all there. + One of the interesting and extremely useful features is the ability to emulate other UNIX(R)-like operating systems. + As of December 2006 and 7-CURRENT development, the following emulation functionalities are supported: + + * FreeBSD/i386 emulation on FreeBSD/amd64 + * FreeBSD/i386 emulation on FreeBSD/ia64 + * Linux(R)-emulation of Linux(R) operating system on FreeBSD + * NDIS-emulation of Windows networking drivers interface + * NetBSD-emulation of NetBSD operating system + * PECoff-support for PECoff FreeBSD executables + * SVR4-emulation of System V revision 4 UNIX(R) + + Actively developed emulations are the Linux(R) layer and various FreeBSD-on-FreeBSD layers. + Others are not supposed to work properly nor be usable these days. + + [[freebsd-tech-details]] + ==== Technical details + + FreeBSD is traditional flavor of UNIX(R) in the sense of dividing the run of processes into two halves: kernel space and user space run. + There are two types of process entry to the kernel: a syscall and a trap. + There is only one way to return. + In the subsequent sections we will describe the three gates to/from the kernel. + The whole description applies to the i386 architecture as the Linuxulator only exists there but the concept is similar on other architectures. + The information was taken from [1] and the source code. + + [[freebsd-sys-entries]] + ===== System entries + + FreeBSD has an abstraction called an execution class loader, which is a wedge into the man:execve[2] syscall. + This employs a structure `sysentvec`, which describes an executable ABI. + It contains things like errno translation table, signal translation table, various functions to serve syscall needs (stack fixup, coredumping, etc.). + Every ABI the FreeBSD kernel wants to support must define this structure, as it is used later in the syscall processing code and at some other places. + System entries are handled by trap handlers, where we can access both the kernel-space and the user-space at once. + + [[freebsd-syscalls]] + ===== Syscalls + + Syscalls on FreeBSD are issued by executing interrupt `0x80` with register `%eax` set to a desired syscall number with arguments passed on the stack. + + When a process issues an interrupt `0x80`, the `int0x80` syscall trap handler is issued (defined in [.filename]#sys/i386/i386/exception.s#), which prepares arguments (i.e. copies them on to the stack) for a call to a C function man:syscall[2] (defined in [.filename]#sys/i386/i386/trap.c#), which processes the passed in trapframe. + The processing consists of preparing the syscall (depending on the `sysvec` entry), determining if the syscall is 32-bit or 64-bit one (changes size of the parameters), then the parameters are copied, including the syscall. + Next, the actual syscall function is executed with processing of the return code (special cases for `ERESTART` and `EJUSTRETURN` errors). + Finally an `userret()` is scheduled, switching the process back to the users-pace. + The parameters to the actual syscall handler are passed in the form of `struct thread *td`, `struct syscall args *` arguments where the second parameter is a pointer to the copied in structure of parameters. + + [[freebsd-traps]] + ===== Traps + + Handling of traps in FreeBSD is similar to the handling of syscalls. + Whenever a trap occurs, an assembler handler is called. + It is chosen between alltraps, alltraps with regs pushed or calltrap depending on the type of the trap. + This handler prepares arguments for a call to a C function `trap()` (defined in [.filename]#sys/i386/i386/trap.c#), which then processes the occurred trap. + After the processing it might send a signal to the process and/or exit to userland using `userret()`. + + [[freebsd-exits]] + ===== Exits + + Exits from kernel to userspace happen using the assembler routine `doreti` regardless of whether the kernel was entered via a trap or via a syscall. + This restores the program status from the stack and returns to the userspace. + + [[freebsd-unix-primitives]] + ===== UNIX(R) primitives + + FreeBSD operating system adheres to the traditional UNIX(R) scheme, where every process has a unique identification number, the so called _PID_ (Process ID). + PID numbers are allocated either linearly or randomly ranging from `0` to `PID_MAX`. + The allocation of PID numbers is done using linear searching of PID space. + Every thread in a process receives the same PID number as result of the man:getpid[2] call. + + There are currently two ways to implement threading in FreeBSD. + The first way is M:N threading followed by the 1:1 threading model. + The default library used is M:N threading (`libpthread`) and you can switch at runtime to 1:1 threading (`libthr`). + The plan is to switch to 1:1 library by default soon. + Although those two libraries use the same kernel primitives, they are accessed through different API(es). + The M:N library uses the `kse_*` family of syscalls while the 1:1 library uses the `thr_*` family of syscalls. + Due to this, there is no general concept of thread ID shared between kernel and userspace. + Of course, both threading libraries implement the pthread thread ID API. + Every kernel thread (as described by `struct thread`) has td tid identifier but this is not directly accessible from userland and solely serves the kernel's needs. + It is also used for 1:1 threading library as pthread's thread ID but handling of this is internal to the library and cannot be relied on. + + As stated previously there are two implementations of threading in FreeBSD. + The M:N library divides the work between kernel space and userspace. + Thread is an entity that gets scheduled in the kernel but it can represent various number of userspace threads. + M userspace threads get mapped to N kernel threads thus saving resources while keeping the ability to exploit multiprocessor parallelism. + Further information about the implementation can be obtained from the man page or [1]. + The 1:1 library directly maps a userland thread to a kernel thread thus greatly simplifying the scheme. + None of these designs implement a fairness mechanism (such a mechanism was implemented but it was removed recently because it caused serious slowdown and made the code more difficult to deal with). + + [[what-is-linux]] + === What is Linux(R) + + Linux(R) is a UNIX(R)-like kernel originally developed by Linus Torvalds, and now being contributed to by a massive crowd of programmers all around the world. + From its mere beginnings to today, with wide support from companies such as IBM or Google, Linux(R) is being associated with its fast development pace, full hardware support and benevolent dictator model of organization. + + Linux(R) development started in 1991 as a hobbyist project at University of Helsinki in Finland. + Since then it has obtained all the features of a modern UNIX(R)-like OS: multiprocessing, multiuser support, virtual memory, networking, basically everything is there. + There are also highly advanced features like virtualization etc. + + As of 2006 Linux(R) seems to be the most widely used open source operating system with support from independent software vendors like Oracle, RealNetworks, Adobe, etc. + Most of the commercial software distributed for Linux(R) can only be obtained in a binary form so recompilation for other operating systems is impossible. + + Most of the Linux(R) development happens in a Git version control system. + Git is a distributed system so there is no central source of the Linux(R) code, but some branches are considered prominent and official. + The version number scheme implemented by Linux(R) consists of four numbers A.B.C.D. + Currently development happens in 2.6.C.D, where C represents major version, where new features are added or changed while D is a minor version for bugfixes only. + + More information can be obtained from [3]. + + [[linux-tech-details]] + ==== Technical details + + Linux(R) follows the traditional UNIX(R) scheme of dividing the run of a process in two halves: the kernel and user space. + The kernel can be entered in two ways: via a trap or via a syscall. + The return is handled only in one way. + The further description applies to Linux(R) 2.6 on the i386(TM) architecture. + This information was taken from [2]. + + [[linux-syscalls]] + ===== Syscalls + + Syscalls in Linux(R) are performed (in userspace) using `syscallX` macros where X substitutes a number representing the number of parameters of the given syscall. + This macro translates to a code that loads `%eax` register with a number of the syscall and executes interrupt `0x80`. + After this syscall return is called, which translates negative return values to positive `errno` values and sets `res` to `-1` in case of an error. + Whenever the interrupt `0x80` is called the process enters the kernel in system call trap handler. + This routine saves all registers on the stack and calls the selected syscall entry. + Note that the Linux(R) calling convention expects parameters to the syscall to be passed via registers as shown here: + + . parameter -> `%ebx` + . parameter -> `%ecx` + . parameter -> `%edx` + . parameter -> `%esi` + . parameter -> `%edi` + . parameter -> `%ebp` + + There are some exceptions to this, where Linux(R) uses different calling convention (most notably the `clone` syscall). + + [[linux-traps]] + ===== Traps + + The trap handlers are introduced in [.filename]#arch/i386/kernel/traps.c# and most of these handlers live in [.filename]#arch/i386/kernel/entry.S#, where handling of the traps happens. + + [[linux-exits]] + ===== Exits + + Return from the syscall is managed by syscall man:exit[3], which checks for the process having unfinished work, then checks whether we used user-supplied selectors. + If this happens stack fixing is applied and finally the registers are restored from the stack and the process returns to the userspace. + + [[linux-unix-primitives]] + ===== UNIX(R) primitives + + In the 2.6 version, the Linux(R) operating system redefined some of the traditional UNIX(R) primitives, notably PID, TID and thread. + PID is defined not to be unique for every process, so for some processes (threads) man:getppid[2] returns the same value. + Unique identification of process is provided by TID. + This is because _NPTL_ (New POSIX(R) Thread Library) defines threads to be normal processes (so called 1:1 threading). + Spawning a new process in Linux(R) 2.6 happens using the `clone` syscall (fork variants are reimplemented using it). + This clone syscall defines a set of flags that affect behavior of the cloning process regarding thread implementation. + The semantic is a bit fuzzy as there is no single flag telling the syscall to create a thread. + + Implemented clone flags are: + + * `CLONE_VM` - processes share their memory space + * `CLONE_FS` - share umask, cwd and namespace + * `CLONE_FILES` - share open files + * `CLONE_SIGHAND` - share signal handlers and blocked signals + * `CLONE_PARENT` - share parent + * `CLONE_THREAD` - be thread (further explanation below) + * `CLONE_NEWNS` - new namespace + * `CLONE_SYSVSEM` - share SysV undo structures + * `CLONE_SETTLS` - setup TLS at supplied address + * `CLONE_PARENT_SETTID` - set TID in the parent + * `CLONE_CHILD_CLEARTID` - clear TID in the child + * `CLONE_CHILD_SETTID` - set TID in the child + + `CLONE_PARENT` sets the real parent to the parent of the caller. + This is useful for threads because if thread A creates thread B we want thread B to be parented to the parent of the whole thread group. + `CLONE_THREAD` does exactly the same thing as `CLONE_PARENT`, `CLONE_VM` and `CLONE_SIGHAND`, rewrites PID to be the same as PID of the caller, sets exit signal to be none and enters the thread group. + `CLONE_SETTLS` sets up GDT entries for TLS handling. + The `CLONE_*_*TID` set of flags sets/clears user supplied address to TID or 0. + + As you can see the `CLONE_THREAD` does most of the work and does not seem to fit the scheme very well. + The original intention is unclear (even for authors, according to comments in the code) but I think originally there was one threading flag, which was then parcelled among many other flags but this separation was never fully finished. + It is also unclear what this partition is good for as glibc does not use that so only hand-written use of the clone permits a programmer to access this features. + + For non-threaded programs the PID and TID are the same. + For threaded programs the first thread PID and TID are the same and every created thread shares the same PID and gets assigned a unique TID (because `CLONE_THREAD` is passed in) also parent is shared for all processes forming this threaded program. + + The code that implements man:pthread_create[3] in NPTL defines the clone flags like this: + + [.programlisting] + .... + int clone_flags = (CLONE_VM | CLONE_FS | CLONE_FILES | CLONE_SIGNAL + + | CLONE_SETTLS | CLONE_PARENT_SETTID + + | CLONE_CHILD_CLEARTID | CLONE_SYSVSEM + #if __ASSUME_NO_CLONE_DETACHED == 0 + + | CLONE_DETACHED + #endif + + | 0); + .... + + The `CLONE_SIGNAL` is defined like + + [.programlisting] + .... + #define CLONE_SIGNAL (CLONE_SIGHAND | CLONE_THREAD) + .... + + the last 0 means no signal is sent when any of the threads exits. + + [[what-is-emu]] + === What is emulation + + According to a dictionary definition, emulation is the ability of a program or device to imitate another program or device. + This is achieved by providing the same reaction to a given stimulus as the emulated object. + In practice, the software world mostly sees three types of emulation - a program used to emulate a machine (QEMU, various game console emulators etc.), software emulation of a hardware facility (OpenGL emulators, floating point units emulation etc.) and operating system emulation (either in kernel of the operating system or as a userspace program). + + Emulation is usually used in a place, where using the original component is not feasible nor possible at all. + For example someone might want to use a program developed for a different operating system than they use. + Then emulation comes in handy. + Sometimes there is no other way but to use emulation - e.g. when the hardware device you try to use does not exist (yet/anymore) then there is no other way but emulation. + This happens often when porting an operating system to a new (non-existent) platform. + Sometimes it is just cheaper to emulate. + + Looking from an implementation point of view, there are two main approaches to the implementation of emulation. + You can either emulate the whole thing - accepting possible inputs of the original object, maintaining inner state and emitting correct output based on the state and/or input. + This kind of emulation does not require any special conditions and basically can be implemented anywhere for any device/program. + The drawback is that implementing such emulation is quite difficult, time-consuming and error-prone. + In some cases we can use a simpler approach. + Imagine you want to emulate a printer that prints from left to right on a printer that prints from right to left. + It is obvious that there is no need for a complex emulation layer but simply reversing of the printed text is sufficient. + Sometimes the emulating environment is very similar to the emulated one so just a thin layer of some translation is necessary to provide fully working emulation! As you can see this is much less demanding to implement, so less time-consuming and error-prone than the previous approach. + But the necessary condition is that the two environments must be similar enough. + The third approach combines the two previous. + Most of the time the objects do not provide the same capabilities so in a case of emulating the more powerful one on the less powerful we have to emulate the missing features with full emulation described above. + + This master thesis deals with emulation of UNIX(R) on UNIX(R), which is exactly the case, where only a thin layer of translation is sufficient to provide full emulation. + The UNIX(R) API consists of a set of syscalls, which are usually self contained and do not affect some global kernel state. + + There are a few syscalls that affect inner state but this can be dealt with by providing some structures that maintain the extra state. + + No emulation is perfect and emulations tend to lack some parts but this usually does not cause any serious drawbacks. + Imagine a game console emulator that emulates everything but music output. No doubt that the games are playable and one can use the emulator. + It might not be that comfortable as the original game console but its an acceptable compromise between price and comfort. + + The same goes with the UNIX(R) API. + Most programs can live with a very limited set of syscalls working. + Those syscalls tend to be the oldest ones (man:read[2]/man:write[2], man:fork[2] family, man:signal[3] handling, man:exit[3], man:socket[2] API) hence it is easy to emulate because their semantics is shared among all UNIX(R)es, which exist todays. + + [[freebsd-emulation]] + == Emulation + + === How emulation works in FreeBSD + + As stated earlier, FreeBSD supports running binaries from several other UNIX(R)es. + This works because FreeBSD has an abstraction called the execution class loader. + This wedges into the man:execve[2] syscall, so when man:execve[2] is about to execute a binary it examines its type. + + There are basically two types of binaries in FreeBSD. + Shell-like text scripts which are identified by `#!` as their first two characters and normal (typically _ELF_) binaries, which are a representation of a compiled executable object. + The vast majority (one could say all of them) of binaries in FreeBSD are from type ELF. + ELF files contain a header, which specifies the OS ABI for this ELF file. + By reading this information, the operating system can accurately determine what type of binary the given file is. + + Every OS ABI must be registered in the FreeBSD kernel. + This applies to the FreeBSD native OS ABI, as well. + So when man:execve[2] executes a binary it iterates through the list of registered APIs and when it finds the right one it starts to use the information contained in the OS ABI description (its syscall table, `errno` translation table, etc.). + So every time the process calls a syscall, it uses its own set of syscalls instead of some global one. + This effectively provides a very elegant and easy way of supporting execution of various binary formats. + + The nature of emulation of different OSes (and also some other subsystems) led developers to invite a handler event mechanism. + There are various places in the kernel, where a list of event handlers are called. + Every subsystem can register an event handler and they are called accordingly. + For example, when a process exits there is a handler called that possibly cleans up whatever the subsystem needs to be cleaned. + + Those simple facilities provide basically everything that is needed for the emulation infrastructure and in fact these are basically the only things necessary to implement the Linux(R) emulation layer. + + [[freebsd-common-primitives]] + === Common primitives in the FreeBSD kernel + + Emulation layers need some support from the operating system. + I am going to describe some of the supported primitives in the FreeBSD operating system. + + [[freebsd-locking-primitives]] + ==== Locking primitives + + Contributed by: `{attilio}` + + The FreeBSD synchronization primitive set is based on the idea to supply a rather huge number of different primitives in a way that the better one can be used for every particular, appropriate situation. + + To a high level point of view you can consider three kinds of synchronization primitives in the FreeBSD kernel: + + * atomic operations and memory barriers + * locks + * scheduling barriers + + Below there are descriptions for the 3 families. + For every lock, you should really check the linked manpage (where possible) for more detailed explanations. + + [[freebsd-atomic-op]] + ===== Atomic operations and memory barriers + + Atomic operations are implemented through a set of functions performing simple arithmetics on memory operands in an atomic way with respect to external events (interrupts, preemption, etc.). + Atomic operations can guarantee atomicity just on small data types (in the magnitude order of the `.long.` architecture C data type), so should be rarely used directly in the end-level code, if not only for very simple operations (like flag setting in a bitmap, for example). + In fact, it is rather simple and common to write down a wrong semantic based on just atomic operations (usually referred as lock-less). + The FreeBSD kernel offers a way to perform atomic operations in conjunction with a memory barrier. + The memory barriers will guarantee that an atomic operation will happen following some specified ordering with respect to other memory accesses. + For example, if we need that an atomic operation happen just after all other pending writes (in terms of instructions reordering buffers activities) are completed, we need to explicitly use a memory barrier in conjunction to this atomic operation. + So it is simple to understand why memory barriers play a key role for higher-level locks building (just as refcounts, mutexes, etc.). + For a detailed explanatory on atomic operations, please refer to man:atomic[9]. + It is far, however, noting that atomic operations (and memory barriers as well) should ideally only be used for building front-ending locks (as mutexes). + + [[freebsd-refcounts]] + ===== Refcounts + + Refcounts are interfaces for handling reference counters. + They are implemented through atomic operations and are intended to be used just for cases, where the reference counter is the only one thing to be protected, so even something like a spin-mutex is deprecated. + Using the refcount interface for structures, where a mutex is already used is often wrong since we should probably close the reference counter in some already protected paths. + A manpage discussing refcount does not exist currently, just check [.filename]#sys/refcount.h# for an overview of the existing API. + + [[freebsd-locks]] + ===== Locks + + FreeBSD kernel has huge classes of locks. + Every lock is defined by some peculiar properties, but probably the most important is the event linked to contesting holders (or in other terms, the behavior of threads unable to acquire the lock). + FreeBSD's locking scheme presents three different behaviors for contenders: + + . spinning + . blocking + . sleeping + + [NOTE] + ==== + numbers are not casual + ==== + + [[freebsd-spinlocks]] + ===== Spinning locks + + Spin locks let waiters to spin until they cannot acquire the lock. + An important matter do deal with is when a thread contests on a spin lock if it is not descheduled. + Since the FreeBSD kernel is preemptive, this exposes spin lock at the risk of deadlocks that can be solved just disabling interrupts while they are acquired. + For this and other reasons (like lack of priority propagation support, poorness in load balancing schemes between CPUs, etc.), spin locks are intended to protect very small paths of code, or ideally not to be used at all if not explicitly requested (explained later). + + [[freebsd-blocking]] + ===== Blocking + + Block locks let waiters to be descheduled and blocked until the lock owner does not drop it and wakes up one or more contenders. + To avoid starvation issues, blocking locks do priority propagation from the waiters to the owner. + Block locks must be implemented through the turnstile interface and are intended to be the most used kind of locks in the kernel, if no particular conditions are met. + + [[freebsd-sleeping]] + ===== Sleeping + + Sleep locks let waiters to be descheduled and fall asleep until the lock holder does not drop it and wakes up one or more waiters. + Since sleep locks are intended to protect large paths of code and to cater asynchronous events, they do not do any form of priority propagation. + They must be implemented through the man:sleepqueue[9] interface. + + The order used to acquire locks is very important, not only for the possibility to deadlock due at lock order reversals, but even because lock acquisition should follow specific rules linked to locks natures. + If you give a look at the table above, the practical rule is that if a thread holds a lock of level n (where the level is the number listed close to the kind of lock) it is not allowed to acquire a lock of superior levels, since this would break the specified semantic for a path. + For example, if a thread holds a block lock (level 2), it is allowed to acquire a spin lock (level 1) but not a sleep lock (level 3), since block locks are intended to protect smaller paths than sleep lock (these rules are not about atomic operations or scheduling barriers, however). + + This is a list of lock with their respective behaviors: + + * spin mutex - spinning - man:mutex[9] + * sleep mutex - blocking - man:mutex[9] + * pool mutex - blocking - man:mtx[pool] + * sleep family - sleeping - man:sleep[9] pause tsleep msleep msleep spin msleep rw msleep sx + * condvar - sleeping - man:condvar[9] + * rwlock - blocking - man:rwlock[9] + * sxlock - sleeping - man:sx[9] + * lockmgr - sleeping - man:lockmgr[9] + * semaphores - sleeping - man:sema[9] + + Among these locks only mutexes, sxlocks, rwlocks and lockmgrs are intended to handle recursion, but currently recursion is only supported by mutexes and lockmgrs. + + [[freebsd-scheduling]] + ===== Scheduling barriers + + Scheduling barriers are intended to be used to drive scheduling of threading. + They consist mainly of three different stubs: + + * critical sections (and preemption) + * sched_bind + * sched_pin + + Generally, these should be used only in a particular context and even if they can often replace locks, they should be avoided because they do not let the diagnose of simple eventual problems with locking debugging tools (as man:witness[4]). + + [[freebsd-critical]] + ===== Critical sections + + The FreeBSD kernel has been made preemptive basically to deal with interrupt threads. + In fact, to avoid high interrupt latency, time-sharing priority threads can be preempted by interrupt threads (in this way, they do not need to wait to be scheduled as the normal path previews). + Preemption, however, introduces new racing points that need to be handled, as well. + Often, to deal with preemption, the simplest thing to do is to completely disable it. + A critical section defines a piece of code (borderlined by the pair of functions man:critical_enter[9] and man:critical_exit[9], where preemption is guaranteed to not happen (until the protected code is fully executed). + This can often replace a lock effectively but should be used carefully to not lose the whole advantage that preemption brings. + + [[freebsd-schedpin]] + ===== sched_pin/sched_unpin + + Another way to deal with preemption is the `sched_pin()` interface. + If a piece of code is closed in the `sched_pin()` and `sched_unpin()` pair of functions it is guaranteed that the respective thread, even if it can be preempted, it will always be executed on the same CPU. + Pinning is very effective in the particular case when we have to access at per-cpu datas and we assume other threads will not change those data. + The latter condition will determine a critical section as a too strong condition for our code. + + [[freebsd-schedbind]] + ===== sched_bind/sched_unbind + + `sched_bind` is an API used to bind a thread to a particular CPU for all the time it executes the code, until a `sched_unbind` function call does not unbind it. + This feature has a key role in situations where you cannot trust the current state of CPUs (for example, at very early stages of boot), as you want to avoid your thread to migrate on inactive CPUs. + Since `sched_bind` and `sched_unbind` manipulate internal scheduler structures, they need to be enclosed in `sched_lock` acquisition/releasing when used. + + [[freebsd-proc]] + ==== Proc structure + + Various emulation layers sometimes require some additional per-process data. + It can manage separate structures (a list, a tree etc.) containing these data for every process but this tends to be slow and memory consuming. + To solve this problem the FreeBSD `proc` structure contains `p_emuldata`, which is a void pointer to some emulation layer specific data. + This `proc` entry is protected by the proc mutex. + + The FreeBSD `proc` structure contains a `p_sysent` entry that identifies, which ABI this process is running. + In fact, it is a pointer to the `sysentvec` described above. + So by comparing this pointer to the address where the `sysentvec` structure for the given ABI is stored we can effectively determine whether the process belongs to our emulation layer. + The code typically looks like: + + [.programlisting] + .... + if (__predict_true(p->p_sysent != &elf_Linux(R)_sysvec)) + return; + .... + + As you can see, we effectively use the `__predict_true` modifier to collapse the most common case (FreeBSD process) to a simple return operation thus preserving high performance. + This code should be turned into a macro because currently it is not very flexible, i.e. we do not support Linux(R)64 emulation nor A.OUT Linux(R) processes on i386. + + [[freebsd-vfs]] + ==== VFS + + The FreeBSD VFS subsystem is very complex but the Linux(R) emulation layer uses just a small subset via a well defined API. + It can either operate on vnodes or file handlers. + Vnode represents a virtual vnode, i.e. representation of a node in VFS. + Another representation is a file handler, which represents an opened file from the perspective of a process. + A file handler can represent a socket or an ordinary file. + A file handler contains a pointer to its vnode. + More then one file handler can point to the same vnode. + + [[freebsd-namei]] + ===== namei + + The man:namei[9] routine is a central entry point to pathname lookup and translation. + It traverses the path point by point from the starting point to the end point using lookup function, which is internal to VFS. + The man:namei[9] syscall can cope with symlinks, absolute and relative paths. + When a path is looked up using man:namei[9] it is inputed to the name cache. This behavior can be suppressed. + This routine is used all over the kernel and its performance is very critical. + + [[freebsd-vn]] + ===== vn_fullpath + + The man:vn_fullpath[9] function takes the best effort to traverse VFS name cache and returns a path for a given (locked) vnode. + This process is unreliable but works just fine for the most common cases. + The unreliability is because it relies on VFS cache (it does not traverse the on medium structures), it does not work with hardlinks, etc. + This routine is used in several places in the Linuxulator. + + [[freebsd-vnode]] + ===== Vnode operations + + * `fgetvp` - given a thread and a file descriptor number it returns the associated vnode + * man:vn_lock[9] - locks a vnode + * `vn_unlock` - unlocks a vnode + * man:VOP_READDIR[9] - reads a directory referenced by a vnode + * man:VOP_GETATTR[9] - gets attributes of a file or a directory referenced by a vnode + * man:VOP_LOOKUP[9] - looks up a path to a given directory + * man:VOP_OPEN[9] - opens a file referenced by a vnode + * man:VOP_CLOSE[9] - closes a file referenced by a vnode + * man:vput[9] - decrements the use count for a vnode and unlocks it + * man:vrele[9] - decrements the use count for a vnode + * man:vref[9] - increments the use count for a vnode + + [[freebsd-file-handler]] + ===== File handler operations + + * `fget` - given a thread and a file descriptor number it returns associated file handler and references it + * `fdrop` - drops a reference to a file handler + * `fhold` - references a file handler + + [[md]] + == Linux(R) emulation layer -MD part + + This section deals with implementation of Linux(R) emulation layer in FreeBSD operating system. + It first describes the machine dependent part talking about how and where interaction between userland and kernel is implemented. + It talks about syscalls, signals, ptrace, traps, stack fixup. + This part discusses i386 but it is written generally so other architectures should not differ very much. + The next part is the machine independent part of the Linuxulator. + This section only covers i386 and ELF handling. A.OUT is obsolete and untested. + + [[syscall-handling]] + === Syscall handling + + Syscall handling is mostly written in [.filename]#linux_sysvec.c#, which covers most of the routines pointed out in the `sysentvec` structure. + When a Linux(R) process running on FreeBSD issues a syscall, the general syscall routine calls linux prepsyscall routine for the Linux(R) ABI. + + [[linux-prepsyscall]] + ==== Linux(R) prepsyscall + + Linux(R) passes arguments to syscalls via registers (that is why it is limited to 6 parameters on i386) while FreeBSD uses the stack. + The Linux(R) prepsyscall routine must copy parameters from registers to the stack. + The order of the registers is: `%ebx`, `%ecx`, `%edx`, `%esi`, `%edi`, `%ebp`. + The catch is that this is true for only _most_ of the syscalls. + Some (most notably `clone`) uses a different order but it is luckily easy to fix by inserting a dummy parameter in the `linux_clone` prototype. + + [[syscall-writing]] + ==== Syscall writing + + Every syscall implemented in the Linuxulator must have its prototype with various flags in [.filename]#syscalls.master#. + The form of the file is: + + [.programlisting] + .... + ... + AUE_FORK STD { int linux_fork(void); } + ... + AUE_CLOSE NOPROTO { int close(int fd); } + ... + .... + + The first column represents the syscall number. + The second column is for auditing support. + The third column represents the syscall type. + It is either `STD`, `OBSOL`, `NOPROTO` and `UNIMPL`. + `STD` is a standard syscall with full prototype and implementation. + `OBSOL` is obsolete and defines just the prototype. + `NOPROTO` means that the syscall is implemented elsewhere so do not prepend ABI prefix, etc. + `UNIMPL` means that the syscall will be substituted with the `nosys` syscall (a syscall just printing out a message about the syscall not being implemented and returning `ENOSYS`). + + From [.filename]#syscalls.master# a script generates three files: [.filename]#linux_syscall.h#, [.filename]#linux_proto.h# and [.filename]#linux_sysent.c#. + The [.filename]#linux_syscall.h# contains definitions of syscall names and their numerical value, e.g.: + + [.programlisting] + .... + ... + #define LINUX_SYS_linux_fork 2 + ... + #define LINUX_SYS_close 6 + ... + .... + + The [.filename]#linux_proto.h# contains structure definitions of arguments to every syscall, e.g.: + + [.programlisting] + .... + struct linux_fork_args { + register_t dummy; + }; + .... + + And finally, [.filename]#linux_sysent.c# contains structure describing the system entry table, used to actually dispatch a syscall, e.g.: + + [.programlisting] + .... + { 0, (sy_call_t *)linux_fork, AUE_FORK, NULL, 0, 0 }, /* 2 = linux_fork */ + { AS(close_args), (sy_call_t *)close, AUE_CLOSE, NULL, 0, 0 }, /* 6 = close */ + .... + + As you can see `linux_fork` is implemented in Linuxulator itself so the definition is of `STD` type and has no argument, which is exhibited by the dummy argument structure. + On the other hand `close` is just an alias for real FreeBSD man:close[2] so it has no linux arguments structure associated and in the system entry table it is not prefixed with linux as it calls the real man:close[2] in the kernel. + + [[dummy-syscalls]] + ==== Dummy syscalls + + The Linux(R) emulation layer is not complete, as some syscalls are not implemented properly and some are not implemented at all. + The emulation layer employs a facility to mark unimplemented syscalls with the `DUMMY` macro. + These dummy definitions reside in [.filename]#linux_dummy.c# in a form of `DUMMY(syscall);`, which is then translated to various syscall auxiliary files and the implementation consists of printing a message saying that this syscall is not implemented. + The `UNIMPL` prototype is not used because we want to be able to identify the name of the syscall that was called to know what syscalls are more important to implement. + + [[signal-handling]] + === Signal handling + + Signal handling is done generally in the FreeBSD kernel for all binary compatibilities with a call to a compat-dependent layer. + Linux(R) compatibility layer defines `linux_sendsig` routine for this purpose. + + [[linux-sendsig]] + ==== Linux(R) sendsig + + This routine first checks whether the signal has been installed with a `SA_SIGINFO` in which case it calls `linux_rt_sendsig` routine instead. + Furthermore, it allocates (or reuses an already existing) signal handle context, then it builds a list of arguments for the signal handler. + It translates the signal number based on the signal translation table, assigns a handler, translates sigset. + Then it saves context for the `sigreturn` routine (various registers, translated trap number and signal mask). + Finally, it copies out the signal context to the userspace and prepares context for the actual signal handler to run. + + [[linux-rt-sendsig]] + ==== linux_rt_sendsig + + This routine is similar to `linux_sendsig` just the signal context preparation is different. + It adds `siginfo`, `ucontext`, and some POSIX(R) parts. + It might be worth considering whether those two functions could not be merged with a benefit of less code duplication and possibly even faster execution. + + [[linux-sigreturn]] + ==== linux_sigreturn + + This syscall is used for return from the signal handler. + It does some security checks and restores the original process context. + It also unmasks the signal in process signal mask. + + [[ptrace]] + === Ptrace + + Many UNIX(R) derivates implement the man:ptrace[2] syscall to allow various tracking and debugging features. + This facility enables the tracing process to obtain various information about the traced process, like register dumps, any memory from the process address space, etc. and also to trace the process like in stepping an instruction or between system entries (syscalls and traps). + man:ptrace[2] also lets you set various information in the traced process (registers etc.). + man:ptrace[2] is a UNIX(R)-wide standard implemented in most UNIX(R)es around the world. + + Linux(R) emulation in FreeBSD implements the man:ptrace[2] facility in [.filename]#linux_ptrace.c#. + The routines for converting registers between Linux(R) and FreeBSD and the actual man:ptrace[2] syscall emulation syscall. + The syscall is a long switch block that implements its counterpart in FreeBSD for every man:ptrace[2] command. + The man:ptrace[2] commands are mostly equal between Linux(R) and FreeBSD so usually just a small modification is needed. + For example, `PT_GETREGS` in Linux(R) operates on direct data while FreeBSD uses a pointer to the data so after performing a (native) man:ptrace[2] syscall, a copyout must be done to preserve Linux(R) semantics. + + The man:ptrace[2] implementation in Linuxulator has some known weaknesses. + There have been panics seen when using `strace` (which is a man:ptrace[2] consumer) in the Linuxulator environment. + Also `PT_SYSCALL` is not implemented. + + [[traps]] + === Traps + + Whenever a Linux(R) process running in the emulation layer traps the trap itself is handled transparently with the only exception of the trap translation. + Linux(R) and FreeBSD differs in opinion on what a trap is so this is dealt with here. + The code is actually very short: + + [.programlisting] + .... + static int + translate_traps(int signal, int trap_code) + { + + if (signal != SIGBUS) + return signal; + + switch (trap_code) { + + case T_PROTFLT: + case T_TSSFLT: + case T_DOUBLEFLT: + case T_PAGEFLT: + return SIGSEGV; + + default: + return signal; + } + } + .... + + [[stack-fixup]] + === Stack fixup + + The RTLD run-time link-editor expects so called AUX tags on stack during an `execve` so a fixup must be done to ensure this. + Of course, every RTLD system is different so the emulation layer must provide its own stack fixup routine to do this. + So does Linuxulator. + The `elf_linux_fixup` simply copies out AUX tags to the stack and adjusts the stack of the user space process to point right after those tags. + So RTLD works in a smart way. + + [[aout-support]] + === A.OUT support + + The Linux(R) emulation layer on i386 also supports Linux(R) A.OUT binaries. + Pretty much everything described in the previous sections must be implemented for A.OUT support (beside traps translation and signals sending). + The support for A.OUT binaries is no longer maintained, especially the 2.6 emulation does not work with it but this does not cause any problem, as the linux-base in ports probably do not support A.OUT binaries at all. + This support will probably be removed in future. + Most of the stuff necessary for loading Linux(R) A.OUT binaries is in [.filename]#imgact_linux.c# file. + + [[mi]] + == Linux(R) emulation layer -MI part + + This section talks about machine independent part of the Linuxulator. + It covers the emulation infrastructure needed for Linux(R) 2.6 emulation, the thread local storage (TLS) implementation (on i386) and futexes. + Then we talk briefly about some syscalls. + + [[nptl-desc]] + === Description of NPTL + + One of the major areas of progress in development of Linux(R) 2.6 was threading. + Prior to 2.6, the Linux(R) threading support was implemented in the linuxthreads library. + The library was a partial implementation of POSIX(R) threading. + The threading was implemented using separate processes for each thread using the `clone` syscall to let them share the address space (and other things). + The main weaknesses of this approach was that every thread had a different PID, signal handling was broken (from the pthreads perspective), etc. + Also the performance was not very good (use of `SIGUSR` signals for threads synchronization, kernel resource consumption, etc.) so to overcome these problems a new threading system was developed and named NPTL. + + The NPTL library focused on two things but a third thing came along so it is usually considered a part of NPTL. + Those two things were embedding of threads into a process structure and futexes. + The additional third thing was TLS, which is not directly required by NPTL but the whole NPTL userland library depends on it. + Those improvements yielded in much improved performance and standards conformance. + NPTL is a standard threading library in Linux(R) systems these days. + + The FreeBSD Linuxulator implementation approaches the NPTL in three main areas. + The TLS, futexes and PID mangling, which is meant to simulate the Linux(R) threads. + Further sections describe each of these areas. + + [[linux26-emu]] + === Linux(R) 2.6 emulation infrastructure + + These sections deal with the way Linux(R) threads are managed and how we simulate that in FreeBSD. + + [[linux26-runtime]] + ==== Runtime determining of 2.6 emulation + + The Linux(R) emulation layer in FreeBSD supports runtime setting of the emulated version. + This is done via man:sysctl[8], namely `compat.linux.osrelease`. + Setting this man:sysctl[8] affects runtime behavior of the emulation layer. + When set to 2.6.x it sets the value of `linux_use_linux26` while setting to something else keeps it unset. + This variable (plus per-prison variables of the very same kind) determines whether 2.6 infrastructure (mainly PID mangling) is used in the code or not. + The version setting is done system-wide and this affects all Linux(R) processes. + The man:sysctl[8] should not be changed when running any Linux(R) binary as it might harm things. + + [[linux-proc-thread]] + ==== Linux(R) processes and thread identifiers + + The semantics of Linux(R) threading are a little confusing and uses entirely different nomenclature to FreeBSD. + A process in Linux(R) consists of a `struct task` embedding two identifier fields - PID and TGID. + PID is _not_ a process ID but it is a thread ID. + The TGID identifies a thread group in other words a process. + For single-threaded process the PID equals the TGID. + + The thread in NPTL is just an ordinary process that happens to have TGID not equal to PID and have a group leader not equal to itself (and shared VM etc. of course). + Everything else happens in the same way as to an ordinary process. + There is no separation of a shared status to some external structure like in FreeBSD. + This creates some duplication of information and possible data inconsistency. + The Linux(R) kernel seems to use task -> group information in some places and task information elsewhere and it is really not very consistent and looks error-prone. + + Every NPTL thread is created by a call to the `clone` syscall with a specific set of flags (more in the next subsection). + The NPTL implements strict 1:1 threading. + + In FreeBSD we emulate NPTL threads with ordinary FreeBSD processes that share VM space, etc. and the PID gymnastic is just mimicked in the emulation specific structure attached to the process. The structure attached to the process looks like: + + [.programlisting] + .... + struct linux_emuldata { + pid_t pid; + + int *child_set_tid; /* in clone(): Child.s TID to set on clone */ + int *child_clear_tid;/* in clone(): Child.s TID to clear on exit */ + + struct linux_emuldata_shared *shared; + + int pdeath_signal; /* parent death signal */ + + LIST_ENTRY(linux_emuldata) threads; /* list of linux threads */ + }; + .... + + The PID is used to identify the FreeBSD process that attaches this structure. + The `child_se_tid` and `child_clear_tid` are used for TID address copyout when a process exits and is created. + The `shared` pointer points to a structure shared among threads. + The `pdeath_signal` variable identifies the parent death signal and the `threads` pointer is used to link this structure to the list of threads. + The `linux_emuldata_shared` structure looks like: + + [.programlisting] + .... + struct linux_emuldata_shared { + + int refs; + + pid_t group_pid; + + LIST_HEAD(, linux_emuldata) threads; /* head of list of linux threads */ + }; + .... + + The `refs` is a reference counter being used to determine when we can free the structure to avoid memory leaks. + The `group_pid` is to identify PID ( = TGID) of the whole process ( = thread group). + The `threads` pointer is the head of the list of threads in the process. + + The `linux_emuldata` structure can be obtained from the process using `em_find`. + The prototype of the function is: + + [.programlisting] + .... + struct linux_emuldata *em_find(struct proc *, int locked); + .... + + Here, `proc` is the process we want the emuldata structure from and the locked parameter determines whether we want to lock or not. + The accepted values are `EMUL_DOLOCK` and `EMUL_DOUNLOCK`. + More about locking later. + + [[pid-mangling]] + ==== PID mangling + + As there is a difference in view as what to the idea of a process ID and thread ID is between FreeBSD and Linux(R) we have to translate the view somehow. + We do it by PID mangling. + This means that we fake what a PID (=TGID) and TID (=PID) is between kernel and userland. + The rule of thumb is that in kernel (in Linuxulator) PID = PID and TGID = shared -> group pid and to userland we present `PID = shared -> group_pid` and `TID = proc -> p_pid`. + The PID member of `linux_emuldata structure` is a FreeBSD PID. + + The above affects mainly getpid, getppid, gettid syscalls. + Where we use PID/TGID respectively. + In copyout of TIDs in `child_clear_tid` and `child_set_tid` we copy out FreeBSD PID. + + [[clone-syscall]] + ==== Clone syscall + + The `clone` syscall is the way threads are created in Linux(R). + The syscall prototype looks like this: + + [.programlisting] + .... + int linux_clone(l_int flags, void *stack, void *parent_tidptr, int dummy, + void * child_tidptr); + .... + + The `flags` parameter tells the syscall how exactly the processes should be cloned. + As described above, Linux(R) can create processes sharing various things independently, for example two processes can share file descriptors but not VM, etc. + Last byte of the `flags` parameter is the exit signal of the newly created process. + The `stack` parameter if non-`NULL` tells, where the thread stack is and if it is `NULL` we are supposed to copy-on-write the calling process stack (i.e. do what normal man:fork[2] routine does). + The `parent_tidptr` parameter is used as an address for copying out process PID (i.e. thread id) once the process is sufficiently instantiated but is not runnable yet. + The `dummy` parameter is here because of the very strange calling convention of this syscall on i386. + It uses the registers directly and does not let the compiler do it what results in the need of a dummy syscall. + The `child_tidptr` parameter is used as an address for copying out PID once the process has finished forking and when the process exits. + + The syscall itself proceeds by setting corresponding flags depending on the flags passed in. + For example, `CLONE_VM` maps to RFMEM (sharing of VM), etc. + The only nit here is `CLONE_FS` and `CLONE_FILES` because FreeBSD does not allow setting this separately so we fake it by not setting RFFDG (copying of fd table and other fs information) if either of these is defined. + This does not cause any problems, because those flags are always set together. + After setting the flags the process is forked using the internal `fork1` routine, the process is instrumented not to be put on a run queue, i.e. not to be set runnable. + After the forking is done we possibly reparent the newly created process to emulate `CLONE_PARENT` semantics. + Next part is creating the emulation data. + Threads in Linux(R) does not signal their parents so we set exit signal to be 0 to disable this. + After that setting of `child_set_tid` and `child_clear_tid` is performed enabling the functionality later in the code. + At this point we copy out the PID to the address specified by `parent_tidptr`. + The setting of process stack is done by simply rewriting thread frame `%esp` register (`%rsp` on amd64). + Next part is setting up TLS for the newly created process. + After this man:vfork[2] semantics might be emulated and finally the newly created process is put on a run queue and copying out its PID to the parent process via `clone` return value is done. + + The `clone` syscall is able and in fact is used for emulating classic man:fork[2] and man:vfork[2] syscalls. + Newer glibc in a case of 2.6 kernel uses `clone` to implement man:fork[2] and man:vfork[2] syscalls. + + [[locking]] + ==== Locking + + The locking is implemented to be per-subsystem because we do not expect a lot of contention on these. + There are two locks: `emul_lock` used to protect manipulating of `linux_emuldata` and `emul_shared_lock` used to manipulate `linux_emuldata_shared`. + The `emul_lock` is a nonsleepable blocking mutex while `emul_shared_lock` is a sleepable blocking `sx_lock`. +-Due to of the per-subsystem locking we can coalesce some locks and that is why the em find offers the non-locking access. ++Due to of the per-subsystem locking we can coalesce some locks and that is why the em_find offers the non-locking access. + + [[tls]] + === TLS + + This section deals with TLS also known as thread local storage. + + [[trheading-intro]] + ==== Introduction to threading + + Threads in computer science are entities within a process that can be scheduled independently from each other. + The threads in the process share process wide data (file descriptors, etc.) but also have their own stack for their own data. + Sometimes there is a need for process-wide data specific to a given thread. + Imagine a name of the thread in execution or something like that. + The traditional UNIX(R) threading API, pthreads provides a way to do it via man:pthread_key_create[3], man:pthread_setspecific[3] and man:pthread_getspecific[3] where a thread can create a key to the thread local data and using man:pthread_getspecific[3] or man:pthread_getspecific[3] to manipulate those data. + You can easily see that this is not the most comfortable way this could be accomplished. + So various producers of C/C++ compilers introduced a better way. + They defined a new modifier keyword thread that specifies that a variable is thread specific. + A new method of accessing such variables was developed as well (at least on i386). + The pthreads method tends to be implemented in userspace as a trivial lookup table. + The performance of such a solution is not very good. + So the new method uses (on i386) segment registers to address a segment, where TLS area is stored so the actual accessing of a thread variable is just appending the segment register to the address thus addressing via it. + The segment registers are usually `%gs` and `%fs` acting like segment selectors. + Every thread has its own area where the thread local data are stored and the segment must be loaded on every context switch. + This method is very fast and used almost exclusively in the whole i386 UNIX(R) world. + Both FreeBSD and Linux(R) implement this approach and it yields very good results. + The only drawback is the need to reload the segment on every context switch which can slowdown context switches. + FreeBSD tries to avoid this overhead by using only 1 segment descriptor for this while Linux(R) uses 3. + Interesting thing is that almost nothing uses more than 1 descriptor (only Wine seems to use 2) so Linux(R) pays this unnecessary price for context switches. + + [[i386-segs]] + ==== Segments on i386 + + The i386 architecture implements the so called segments. + A segment is a description of an area of memory. + The base address (bottom) of the memory area, the end of it (ceiling), type, protection, etc. + The memory described by a segment can be accessed using segment selector registers (`%cs`, `%ds`, `%ss`, `%es`, `%fs`, `%gs`). + For example let us suppose we have a segment which base address is 0x1234 and length and this code: + + [.programlisting] + .... + mov %edx,%gs:0x10 + .... + + This will load the content of the `%edx` register into memory location 0x1244. + Some segment registers have a special use, for example `%cs` is used for code segment and `%ss` is used for stack segment but `%fs` and `%gs` are generally unused. + Segments are either stored in a global GDT table or in a local LDT table. + LDT is accessed via an entry in the GDT. + The LDT can store more types of segments. + LDT can be per process. + Both tables define up to 8191 entries. + + [[linux-i386]] + ==== Implementation on Linux(R) i386 + + There are two main ways of setting up TLS in Linux(R). + It can be set when cloning a process using the `clone` syscall or it can call `set_thread_area`. + When a process passes `CLONE_SETTLS` flag to `clone`, the kernel expects the memory pointed to by the `%esi` register a Linux(R) user space representation of a segment, which gets translated to the machine representation of a segment and loaded into a GDT slot. + The GDT slot can be specified with a number or -1 can be used meaning that the system itself should choose the first free slot. + In practice, the vast majority of programs use only one TLS entry and does not care about the number of the entry. + We exploit this in the emulation and in fact depend on it. + + [[tls-emu]] + ==== Emulation of Linux(R) TLS + + [[tls-i386]] + ===== i386 + + Loading of TLS for the current thread happens by calling `set_thread_area` while loading TLS for a second process in `clone` is done in the separate block in `clone`. + Those two functions are very similar. + The only difference being the actual loading of the GDT segment, which happens on the next context switch for the newly created process while `set_thread_area` must load this directly. + The code basically does this. + It copies the Linux(R) form segment descriptor from the userland. + The code checks for the number of the descriptor but because this differs between FreeBSD and Linux(R) we fake it a little. + We only support indexes of 6, 3 and -1. + The 6 is genuine Linux(R) number, 3 is genuine FreeBSD one and -1 means autoselection. + Then we set the descriptor number to constant 3 and copy out this to the userspace. + We rely on the userspace process using the number from the descriptor but this works most of the time (have never seen a case where this did not work) as the userspace process typically passes in 1. + Then we convert the descriptor from the Linux(R) form to a machine dependant form (i.e. operating system independent form) and copy this to the FreeBSD defined segment descriptor. + Finally we can load it. + We assign the descriptor to threads PCB (process control block) and load the `%gs` segment using `load_gs`. + This loading must be done in a critical section so that nothing can interrupt us. + The `CLONE_SETTLS` case works exactly like this just the loading using `load_gs` is not performed. + The segment used for this (segment number 3) is shared for this use between FreeBSD processes and Linux(R) processes so the Linux(R) emulation layer does not add any overhead over plain FreeBSD. + + [[tls-amd64]] + ===== amd64 + + The amd64 implementation is similar to the i386 one but there was initially no 32bit segment descriptor used for this purpose (hence not even native 32bit TLS users worked) so we had to add such a segment and implement its loading on every context switch (when a flag signaling use of 32bit is set). + Apart from this the TLS loading is exactly the same just the segment numbers are different and the descriptor format and the loading differs slightly. + + [[futexes]] + === Futexes + + [[sync-intro]] + ==== Introduction to synchronization + + Threads need some kind of synchronization and POSIX(R) provides some of them: mutexes for mutual exclusion, read-write locks for mutual exclusion with biased ratio of reads and writes and condition variables for signaling a status change. + It is interesting to note that POSIX(R) threading API lacks support for semaphores. + Those synchronization routines implementations are heavily dependant on the type threading support we have. + In pure 1:M (userspace) model the implementation can be solely done in userspace and thus be very fast (the condition variables will probably end up being implemented using signals, i.e. not fast) and simple. + In 1:1 model, the situation is also quite clear - the threads must be synchronized using kernel facilities (which is very slow because a syscall must be performed). + The mixed M:N scenario just combines the first and second approach or rely solely on kernel. + Threads synchronization is a vital part of thread-enabled programming and its performance can affect resulting program a lot. + Recent benchmarks on FreeBSD operating system showed that an improved sx_lock implementation yielded 40% speedup in _ZFS_ (a heavy sx user), this is in-kernel stuff but it shows clearly how important the performance of synchronization primitives is. + + Threaded programs should be written with as little contention on locks as possible. + Otherwise, instead of doing useful work the thread just waits on a lock. + As a result of this, the most well written threaded programs show little locks contention. + + [[futex-intro]] + ==== Futexes introduction + + Linux(R) implements 1:1 threading, i.e. it has to use in-kernel synchronization primitives. + As stated earlier, well written threaded programs have little lock contention. + So a typical sequence could be performed as two atomic increase/decrease mutex reference counter, which is very fast, as presented by the following example: + + [.programlisting] + .... + pthread_mutex_lock(&mutex); + ... + pthread_mutex_unlock(&mutex); + .... + + 1:1 threading forces us to perform two syscalls for those mutex calls, which is very slow. + + The solution Linux(R) 2.6 implements is called futexes. + Futexes implement the check for contention in userspace and call kernel primitives only in a case of contention. + Thus the typical case takes place without any kernel intervention. + This yields reasonably fast and flexible synchronization primitives implementation. + + [[futex-api]] + ==== Futex API + + The futex syscall looks like this: + + [.programlisting] + .... + int futex(void *uaddr, int op, int val, struct timespec *timeout, void *uaddr2, int val3); + .... + + In this example `uaddr` is an address of the mutex in userspace, `op` is an operation we are about to perform and the other parameters have per-operation meaning. + + Futexes implement the following operations: + + * `FUTEX_WAIT` + * `FUTEX_WAKE` + * `FUTEX_FD` + * `FUTEX_REQUEUE` + * `FUTEX_CMP_REQUEUE` + * `FUTEX_WAKE_OP` + + [[futex-wait]] + ===== FUTEX_WAIT + + This operation verifies that on address `uaddr` the value `val` is written. + If not, `EWOULDBLOCK` is returned, otherwise the thread is queued on the futex and gets suspended. + If the argument `timeout` is non-zero it specifies the maximum time for the sleeping, otherwise the sleeping is infinite. + + [[futex-wake]] + ===== FUTEX_WAKE + + This operation takes a futex at `uaddr` and wakes up `val` first futexes queued on this futex. + + [[futex-fd]] + ===== FUTEX_FD + + This operations associates a file descriptor with a given futex. + + [[futex-requeue]] + ===== FUTEX_REQUEUE + + This operation takes `val` threads queued on futex at `uaddr`, wakes them up, and takes `val2` next threads and requeues them on futex at `uaddr2`. + + [[futex-cmp-requeue]] + ===== FUTEX_CMP_REQUEUE + + This operation does the same as `FUTEX_REQUEUE` but it checks that `val3` equals to `val` first. + + [[futex-wake-op]] + ===== FUTEX_WAKE_OP + + This operation performs an atomic operation on `val3` (which contains coded some other value) and `uaddr`. + Then it wakes up `val` threads on futex at `uaddr` and if the atomic operation returned a positive number it wakes up `val2` threads on futex at `uaddr2`. + + The operations implemented in `FUTEX_WAKE_OP`: + + * `FUTEX_OP_SET` + * `FUTEX_OP_ADD` + * `FUTEX_OP_OR` + * `FUTEX_OP_AND` + * `FUTEX_OP_XOR` + + [NOTE] + ==== + There is no `val2` parameter in the futex prototype. + The `val2` is taken from the `struct timespec *timeout` parameter for operations `FUTEX_REQUEUE`, `FUTEX_CMP_REQUEUE` and `FUTEX_WAKE_OP`. + ==== + + [[futex-emu]] + ==== Futex emulation in FreeBSD + + The futex emulation in FreeBSD is taken from NetBSD and further extended by us. + It is placed in `linux_futex.c` and [.filename]#linux_futex.h# files. + The `futex` structure looks like: + + [.programlisting] + .... + struct futex { + void *f_uaddr; + int f_refcount; + + LIST_ENTRY(futex) f_list; + + TAILQ_HEAD(lf_waiting_paroc, waiting_proc) f_waiting_proc; + }; + .... + + And the structure `waiting_proc` is: + + [.programlisting] + .... + struct waiting_proc { + + struct thread *wp_t; + + struct futex *wp_new_futex; + + TAILQ_ENTRY(waiting_proc) wp_list; + }; + .... + + [[futex-get]] + ===== futex_get / futex_put + + A futex is obtained using the `futex_get` function, which searches a linear list of futexes and returns the found one or creates a new futex. + When releasing a futex from the use we call the `futex_put` function, which decreases a reference counter of the futex and if the refcount reaches zero it is released. + + [[futex-sleep]] + ===== futex_sleep + + When a futex queues a thread for sleeping it creates a `working_proc` structure and puts this structure to the list inside the futex structure then it just performs a man:tsleep[9] to suspend the thread. + The sleep can be timed out. + After man:tsleep[9] returns (the thread was woken up or it timed out) the `working_proc` structure is removed from the list and is destroyed. + All this is done in the `futex_sleep` function. + If we got woken up from `futex_wake` we have `wp_new_futex` set so we sleep on it. + This way the actual requeueing is done in this function. + + [[futex-wake-2]] + ===== futex_wake + + Waking up a thread sleeping on a futex is performed in the `futex_wake` function. + First in this function we mimic the strange Linux(R) behavior, where it wakes up N threads for all operations, the only exception is that the REQUEUE operations are performed on N+1 threads. + But this usually does not make any difference as we are waking up all threads. + Next in the function in the loop we wake up n threads, after this we check if there is a new futex for requeueing. + If so, we requeue up to n2 threads on the new futex. + This cooperates with `futex_sleep`. + + [[futex-wake-op-2]] + ===== futex_wake_op + + The `FUTEX_WAKE_OP` operation is quite complicated. + First we obtain two futexes at addresses `uaddr` and `uaddr2` then we perform the atomic operation using `val3` and `uaddr2`. + Then `val` waiters on the first futex is woken up and if the atomic operation condition holds we wake up `val2` (i.e. `timeout`) waiter on the second futex. + + [[futex-atomic-op]] + ===== futex atomic operation + + The atomic operation takes two parameters `encoded_op` and `uaddr`. + The encoded operation encodes the operation itself, comparing value, operation argument, and comparing argument. + The pseudocode for the operation is like this one: + + [.programlisting] + .... + oldval = *uaddr2 + *uaddr2 = oldval OP oparg + .... + + And this is done atomically. First a copying in of the number at `uaddr` is performed and the operation is done. + The code handles page faults and if no page fault occurs `oldval` is compared to `cmparg` argument with cmp comparator. + + [[futex-locking]] + ===== Futex locking + + Futex implementation uses two lock lists protecting `sx_lock` and global locks (either Giant or another `sx_lock`). + Every operation is performed locked from the start to the very end. + + [[syscall-impl]] + === Various syscalls implementation + + In this section I am going to describe some smaller syscalls that are worth mentioning because their implementation is not obvious or those syscalls are interesting from other point of view. + + [[syscall-at]] + ==== *at family of syscalls + + During development of Linux(R) 2.6.16 kernel, the *at syscalls were added. + Those syscalls (`openat` for example) work exactly like their at-less counterparts with the slight exception of the `dirfd` parameter. + This parameter changes where the given file, on which the syscall is to be performed, is. + When the `filename` parameter is absolute `dirfd` is ignored but when the path to the file is relative, it comes to the play. + The `dirfd` parameter is a directory relative to which the relative pathname is checked. + The `dirfd` parameter is a file descriptor of some directory or `AT_FDCWD`. + So for example the `openat` syscall can be like this: + + [.programlisting] + .... + file descriptor 123 = /tmp/foo/, current working directory = /tmp/ + + openat(123, /tmp/bah\, flags, mode) /* opens /tmp/bah */ + openat(123, bah\, flags, mode) /* opens /tmp/foo/bah */ + openat(AT_FDWCWD, bah\, flags, mode) /* opens /tmp/bah */ + openat(stdio, bah\, flags, mode) /* returns error because stdio is not a directory */ + .... + + This infrastructure is necessary to avoid races when opening files outside the working directory. + Imagine that a process consists of two threads, thread A and thread B. + Thread A issues `open(./tmp/foo/bah., flags, mode)` and before returning it gets preempted and thread B runs. + Thread B does not care about the needs of thread A and renames or removes [.filename]#/tmp/foo/#. + We got a race. + To avoid this we can open [.filename]#/tmp/foo# and use it as `dirfd` for `openat` syscall. + This also enables user to implement per-thread working directories. + + Linux(R) family of *at syscalls contains: `linux_openat`, `linux_mkdirat`, `linux_mknodat`, `linux_fchownat`, `linux_futimesat`, `linux_fstatat64`, `linux_unlinkat`, `linux_renameat`, `linux_linkat`, `linux_symlinkat`, `linux_readlinkat`, `linux_fchmodat` and `linux_faccessat`. + All these are implemented using the modified man:namei[9] routine and simple wrapping layer. + + [[implementation]] + ===== Implementation + + The implementation is done by altering the man:namei[9] routine (described above) to take additional parameter `dirfd` in its `nameidata` structure, which specifies the starting point of the pathname lookup instead of using the current working directory every time. + The resolution of `dirfd` from file descriptor number to a vnode is done in native *at syscalls. + When `dirfd` is `AT_FDCWD` the `dvp` entry in `nameidata` structure is `NULL` but when `dirfd` is a different number we obtain a file for this file descriptor, check whether this file is valid and if there is vnode attached to it then we get a vnode. Then we check this vnode for being a directory. + In the actual man:namei[9] routine we simply substitute the `dvp` vnode for `dp` variable in the man:namei[9] function, which determines the starting point. + The man:namei[9] is not used directly but via a trace of different functions on various levels. + For example the `openat` goes like this: + + [.programlisting] + .... + openat() --> kern_openat() --> vn_open() -> namei() + .... + + For this reason `kern_open` and `vn_open` must be altered to incorporate the additional `dirfd` parameter. + No compat layer is created for those because there are not many users of this and the users can be easily converted. + This general implementation enables FreeBSD to implement their own *at syscalls. + This is being discussed right now. + + [[ioctl]] + ==== Ioctl + + The ioctl interface is quite fragile due to its generality. + We have to bear in mind that devices differ between Linux(R) and FreeBSD so some care must be applied to do ioctl emulation work right. + The ioctl handling is implemented in [.filename]#linux_ioctl.c#, where `linux_ioctl` function is defined. + This function simply iterates over sets of ioctl handlers to find a handler that implements a given command. + The ioctl syscall has three parameters, the file descriptor, command and an argument. + The command is a 16-bit number, which in theory is divided into high 8 bits determining class of the ioctl command and low 8 bits, which are the actual command within the given set. + The emulation takes advantage of this division. + We implement handlers for each set, like `sound_handler` or `disk_handler`. + Each handler has a maximum command and a minimum command defined, which is used for determining what handler is used. + There are slight problems with this approach because Linux(R) does not use the set division consistently so sometimes ioctls for a different set are inside a set they should not belong to (SCSI generic ioctls inside cdrom set, etc.). + FreeBSD currently does not implement many Linux(R) ioctls (compared to NetBSD, for example) but the plan is to port those from NetBSD. + The trend is to use Linux(R) ioctls even in the native FreeBSD drivers because of the easy porting of applications. + + [[debugging]] + ==== Debugging + + Every syscall should be debuggable. + For this purpose we introduce a small infrastructure. + We have the ldebug facility, which tells whether a given syscall should be debugged (settable via a sysctl). + For printing we have LMSG and ARGS macros. + Those are used for altering a printable string for uniform debugging messages. + + [[conclusion]] + == Conclusion + + [[results]] + === Results + + As of April 2007 the Linux(R) emulation layer is capable of emulating the Linux(R) 2.6.16 kernel quite well. + The remaining problems concern futexes, unfinished *at family of syscalls, problematic signals delivery, missing `epoll` and `inotify` and probably some bugs we have not discovered yet. + Despite this we are capable of running basically all the Linux(R) programs included in FreeBSD Ports Collection with Fedora Core 4 at 2.6.16 and there are some rudimentary reports of success with Fedora Core 6 at 2.6.16. + The Fedora Core 6 linux_base was recently committed enabling some further testing of the emulation layer and giving us some more hints where we should put our effort in implementing missing stuff. + + We are able to run the most used applications like package:www/linux-firefox[], package:net-im/skype[] and some games from the Ports Collection. + Some of the programs exhibit bad behavior under 2.6 emulation but this is currently under investigation and hopefully will be fixed soon. + The only big application that is known not to work is the Linux(R) Java(TM) Development Kit and this is because of the requirement of `epoll` facility which is not directly related to the Linux(R) kernel 2.6. + + We hope to enable 2.6.16 emulation by default some time after FreeBSD 7.0 is released at least to expose the 2.6 emulation parts for some wider testing. + Once this is done we can switch to Fedora Core 6 linux_base, which is the ultimate plan. + + [[future-work]] + === Future work + + Future work should focus on fixing the remaining issues with futexes, implement the rest of the *at family of syscalls, fix the signal delivery and possibly implement the `epoll` and `inotify` facilities. + + We hope to be able to run the most important programs flawlessly soon, so we will be able to switch to the 2.6 emulation by default and make the Fedora Core 6 the default linux_base because our currently used Fedora Core 4 is not supported any more. + + The other possible goal is to share our code with NetBSD and DragonflyBSD. + NetBSD has some support for 2.6 emulation but its far from finished and not really tested. + DragonflyBSD has expressed some interest in porting the 2.6 improvements. + + Generally, as Linux(R) develops we would like to keep up with their development, implementing newly added syscalls. + Splice comes to mind first. + Some already implemented syscalls are also suboptimal, for example `mremap` and others. + Some performance improvements can also be made, finer grained locking and others. + + [[team]] + === Team + + I cooperated on this project with (in alphabetical order): + + * `{jhb}` + * `{kib}` + * Emmanuel Dreyfus + * Scot Hetzel + * `{jkim}` + * `{netchild}` + * `{ssouhlal}` + * Li Xiao + * `{davidxu}` + + I would like to thank all those people for their advice, code reviews and general support. + + [[literatures]] + == Literatures + + . Marshall Kirk McKusick - George V. Nevile-Neil. Design and Implementation of the FreeBSD operating system. Addison-Wesley, 2005. + . https://tldp.org[https://tldp.org] + . https://www.kernel.org[https://www.kernel.org] +diff --git a/documentation/content/en/articles/linux-users/_index.adoc b/documentation/content/en/articles/linux-users/_index.adoc +index 50a0c2d54f..343b523c6b 100644 +--- a/documentation/content/en/articles/linux-users/_index.adoc ++++ b/documentation/content/en/articles/linux-users/_index.adoc +@@ -1,378 +1,378 @@ + --- + title: FreeBSD Quickstart Guide for Linux® Users + authors: + - author: John Ferrell + copyright: 2008 The FreeBSD Documentation Project + description: This document is intended to quickly familiarize intermediate to advanced Linux® users with the basics of FreeBSD. + trademarks: ["freebsd", "intel", "redhat", "linux", "unix", "general"] + tags: ["Quickstart", "guide", "Linux", "FreeBSD"] + --- + + = FreeBSD Quickstart Guide for Linux(R) Users + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/linux-users/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + This document is intended to quickly familiarize intermediate to advanced Linux(R) users with the basics of FreeBSD. + + ''' + + toc::[] + + [[intro]] + == Introduction + + This document highlights some of the technical differences between FreeBSD and Linux(R) so that intermediate to advanced Linux(R) users can quickly familiarize themselves with the basics of FreeBSD. + + This document assumes that FreeBSD is already installed. +-Refer to the extref:{handbook}[Installing FreeBSD, bsdinstall] chapter of the FreeBSD Handbook for help with the installation process. ++Refer to the extref:{handbook}bsdinstall[Installing FreeBSD, bsdinstall] chapter of the FreeBSD Handbook for help with the installation process. + + [[shells]] + == Default Shell + + Linux(R) users are often surprised to find that Bash is not the default shell in FreeBSD. + In fact, Bash is not included in the default installation. + Instead, the Bourne shell-compatible man:sh[1] as the default user shell. + The root shell is man:tcsh[1] by default on FreeBSD 13 and earlier and man:sh[1] on FreeBSD 14 and later. + man:sh[1] is very similar to Bash but with a much smaller feature-set. + Generally shell scripts written for man:sh[1] will run in Bash, but the reverse is not always true. + +-However, Bash and other shells are available for installation using the FreeBSD extref:{handbook}[Packages and Ports Collection, ports]. ++However, Bash and other shells are available for installation using the FreeBSD extref:{handbook}ports[Packages and Ports Collection, ports]. + + After installing another shell, use man:chsh[1] to change a user's default shell. + It is recommended that the `root` user's default shell remain unchanged since shells which are not included in the base distribution are installed to [.filename]#/usr/local/bin#. + In the event of a problem, the file system where [.filename]#/usr/local/bin# is located may not be mounted. + In this case, `root` would not have access to its default shell, preventing `root` from logging in and fixing the problem. + + [[software]] + == Packages and Ports: Adding Software in FreeBSD + + FreeBSD provides two methods for installing applications: binary packages and compiled ports. + Each method has its own benefits: + + .Binary Packages + * Faster installation as compared to compiling large applications. + * Does not require an understanding of how to compile software. + * No need to install a compiler. + + .Ports + * Ability to customize installation options. + * Custom patches can be applied. + + If an application installation does not require any customization, installing the package is sufficient. + Compile the port instead whenever an application requires customization of the default options. + If needed, a custom package can be compiled from ports using `make package`. + + A complete list of all available ports and packages can be found https://ports.freebsd.org[here]. + + [[packages]] + === Packages + + Packages are pre-compiled applications, the FreeBSD equivalents of [.filename]#.deb# files on Debian/Ubuntu based systems and [.filename]#.rpm# files on Red Hat/Fedora based systems. + Packages are installed using `pkg`. + For example, the following command installs Apache 2.4: + + [source,shell] + .... + # pkg install apache24 + .... + +-For more information on packages refer to section 5.4 of the FreeBSD Handbook: extref:{handbook}[Using pkgng for Binary Package Management, pkgng-intro]. ++For more information on packages refer to section 4.4 of the FreeBSD Handbook: extref:{handbook}ports[Using pkgng for Binary Package Management, pkgng-intro]. + + [[ports]] + === Ports + + The FreeBSD Ports Collection is a framework of [.filename]#Makefiles# and patches specifically customized for installing applications from source on FreeBSD. + When installing a port, the system will fetch the source code, apply any required patches, compile the code, and install the application and any required dependencies. + + The Ports Collection, sometimes referred to as the ports tree, can be installed to [.filename]#/usr/ports# using link:{handbook}mirrors/#git[Git]. +-Detailed instructions for installing the Ports Collection can be found in extref:{handbook}[section 4.5.1, ports-using-installation-methods] of the FreeBSD Handbook. ++Detailed instructions for installing the Ports Collection can be found in extref:{handbook}ports[section 4.5.1, ports-using-installation-methods] of the FreeBSD Handbook. + + To compile a port, change to the port's directory and start the build process. The following example installs Apache 2.4 from the Ports Collection: + + [source,shell] + .... + # cd /usr/ports/www/apache24 + # make install clean + .... + + A benefit of using ports to install software is the ability to customize the installation options. + This example specifies that the mod_ldap module should also be installed: + + [source,shell] + .... + # cd /usr/ports/www/apache24 + # make WITH_LDAP="YES" install clean + .... + +-Refer to extref:{handbook}[Using the Ports Collection, ports-using] for more information. ++Refer to extref:{handbook}ports[Using the Ports Collection, ports-using] for more information. + + [[startup]] + == System Startup + + Many Linux(R) distributions use the SysV init system, whereas FreeBSD uses the traditional BSD-style man:init[8]. + Under the BSD-style man:init[8], there are no run-levels and [.filename]#/etc/inittab# does not exist. + Instead, startup is controlled by man:rc[8] scripts. + At system boot, [.filename]#/etc/rc# reads [.filename]#/etc/rc.conf# and [.filename]#/etc/defaults/rc.conf# to determine which services are to be started. + The specified services are then started by running the corresponding service initialization scripts located in [.filename]#/etc/rc.d/# and [.filename]#/usr/local/etc/rc.d/#. These scripts are similar to the scripts located in [.filename]#/etc/init.d/# on Linux(R) systems. + + The scripts found in [.filename]#/etc/rc.d/# are for applications that are part of the "base" system, such as man:cron[8], man:sshd[8], and man:syslog[3]. + The scripts in [.filename]#/usr/local/etc/rc.d/# are for user-installed applications such as Apache and Squid. + + Since FreeBSD is developed as a complete operating system, user-installed applications are not considered to be part of the "base" system. +-User-installed applications are generally installed using extref:{handbook}[Packages or Ports, ports-using]. ++User-installed applications are generally installed using extref:{handbook}ports[Packages or Ports, ports-using]. + In order to keep them separate from the base system, user-installed applications are installed under [.filename]#/usr/local/#. + Therefore, user-installed binaries reside in [.filename]#/usr/local/bin/#, configuration files are in [.filename]#/usr/local/etc/#, and so on. + + Services are enabled by adding an entry for the service in [.filename]#/etc/rc.conf#. + The system defaults are found in [.filename]#/etc/defaults/rc.conf# and these default settings are overridden by settings in [.filename]#/etc/rc.conf#. + Refer to man:rc.conf[5] for more information about the available entries. + When installing additional applications, review the application's install message to determine how to enable any associated services. + + The following entries in [.filename]#/etc/rc.conf# enable man:sshd[8], enable Apache 2.4, and specify that Apache should be started with SSL. + + [.programlisting] + .... + # enable SSHD + sshd_enable="YES" + # enable Apache with SSL + apache24_enable="YES" + apache24_flags="-DSSL" + .... + + Once a service has been enabled in [.filename]#/etc/rc.conf#, it can be started without rebooting the system: + + [source,shell] + .... + # service sshd start + # service apache24 start + .... + + If a service has not been enabled, it can be started from the command line using `onestart`: + + [source,shell] + .... + # service sshd onestart + .... + + [[network]] + == Network Configuration + + Instead of a generic _ethX_ identifier that Linux(R) uses to identify a network interface, FreeBSD uses the driver name followed by a number. + The following output from man:ifconfig[8] shows two Intel(R) Pro 1000 network interfaces ([.filename]#em0# and [.filename]#em1#): + + [source,shell] + .... + % ifconfig + em0: flags=8843 mtu 1500 + options=b + inet 10.10.10.100 netmask 0xffffff00 broadcast 10.10.10.255 + ether 00:50:56:a7:70:b2 + media: Ethernet autoselect (1000baseTX ) + status: active + em1: flags=8843 mtu 1500 + options=b + inet 192.168.10.222 netmask 0xffffff00 broadcast 192.168.10.255 + ether 00:50:56:a7:03:2b + media: Ethernet autoselect (1000baseTX ) + status: active + .... + + An IP address can be assigned to an interface using man:ifconfig[8]. + To remain persistent across reboots, the IP configuration must be included in [.filename]#/etc/rc.conf#. + The following [.filename]#/etc/rc.conf# entries specify the hostname, IP address, and default gateway: + + [.programlisting] + .... + hostname="server1.example.com" + ifconfig_em0="inet 10.10.10.100 netmask 255.255.255.0" + defaultrouter="10.10.10.1" + .... + + Use the following entries to instead configure an interface for DHCP: + + [.programlisting] + .... + hostname="server1.example.com" + ifconfig_em0="DHCP" + .... + + [[firewall]] + == Firewall + + FreeBSD does not use Linux(R) IPTABLES for its firewall. + Instead, FreeBSD offers a choice of three kernel level firewalls: + +-* extref:{handbook}[PF, firewalls-pf] +-* extref:{handbook}[IPFILTER, firewalls-ipf] +-* extref:{handbook}[IPFW, firewalls-ipfw] ++* extref:{handbook}firewalls[PF, firewalls-pf] ++* extref:{handbook}firewalls[IPFILTER, firewalls-ipf] ++* extref:{handbook}firewalls[IPFW, firewalls-ipfw] + + PF is developed by the OpenBSD project and ported to FreeBSD. + PF was created as a replacement for IPFILTER and its syntax is similar to that of IPFILTER. + PF can be paired with man:altq[4] to provide QoS features. + + This sample PF entry allows inbound SSH: + + [.programlisting] + .... + pass in on $ext_if inet proto tcp from any to ($ext_if) port 22 + .... + + IPFILTER is the firewall application developed by Darren Reed. + It is not specific to FreeBSD and has been ported to several operating systems including NetBSD, OpenBSD, SunOS, HP/UX, and Solaris. + + The IPFILTER syntax to allow inbound SSH is: + + [.programlisting] + .... + pass in on $ext_if proto tcp from any to any port = 22 + .... + + IPFW is the firewall developed and maintained by FreeBSD. + It can be paired with man:dummynet[4] to provide traffic shaping capabilities and simulate different types of network connections. + + The IPFW syntax to allow inbound SSH would be: + + [.programlisting] + .... + ipfw add allow tcp from any to me 22 in via $ext_if + .... + + [[updates]] + == Updating FreeBSD + + There are two methods for updating a FreeBSD system: from source or binary updates. + + Updating from source is the most involved update method, but offers the greatest amount of flexibility. + The process involves synchronizing a local copy of the FreeBSD source code with the FreeBSD Git repository. + Once the local source code is up-to-date, a new version of the kernel and userland can be compiled. + + Binary updates are similar to using `yum` or `apt-get` to update a Linux(R) system. + In FreeBSD, man:freebsd-update[8] can be used fetch new binary updates and install them. + These updates can be scheduled using man:cron[8]. + + [NOTE] + ==== + When using man:cron[8] to schedule updates, use `freebsd-update cron` in the man:crontab[1] to reduce the possibility of a large number of machines all pulling updates at the same time: + + [.programlisting] + .... + 0 3 * * * root /usr/sbin/freebsd-update cron + .... + + ==== + +-For more information on source and binary updates, refer to extref:{handbook}[the chapter on updating, updating-upgrading-freebsdupdate] in the FreeBSD Handbook. ++For more information on source and binary updates, refer to extref:{handbook}cutting-edge[the chapter on updating, updating-upgrading-freebsdupdate] in the FreeBSD Handbook. + + [[procfs]] + == procfs: Gone But Not Forgotten + + In some Linux(R) distributions, one could look at [.filename]#/proc/sys/net/ipv4/ip_forward# to determine if IP forwarding is enabled. + In FreeBSD, man:sysctl[8] is instead used to view this and other system settings. + + For example, use the following to determine if IP forwarding is enabled on a FreeBSD system: + + [source,shell] + .... + % sysctl net.inet.ip.forwarding + net.inet.ip.forwarding: 0 + .... + + Use `-a` to list all the system settings: + + [source,shell] + .... + % sysctl -a | more + .... + + If an application requires procfs, add the following entry to [.filename]#/etc/fstab#: + + [source,shell] + .... + proc /proc procfs rw,noauto 0 0 + .... + + Including `noauto` will prevent [.filename]#/proc# from being automatically mounted at boot. + + To mount the file system without rebooting: + + [source,shell] + .... + # mount /proc + .... + + [[commands]] + == Common Commands + + Some common command equivalents are as follows: + + [.informaltable] + [cols="1,1,1", frame="none", options="header"] + |=== + | Linux(R) command (Red Hat/Debian) + | FreeBSD equivalent + | Purpose + + |`yum install _package_` / `apt-get install _package_` + |`pkg install _package_` + |Install package from remote repository + + |`rpm -ivh _package_` / `dpkg -i _package_` + |`pkg add _package_` + |Install local package + + |`rpm -qa` / `dpkg -l` + |`pkg info` + |List installed packages + + |`lspci` + |`pciconf` + |List PCI devices + + |`lsmod` + |`kldstat` + |List loaded kernel modules + + |`modprobe` + |`kldload` / `kldunload` + |Load/Unload kernel modules + + |`strace` + |`truss` + |Trace system calls + |=== + + [[conclusion]] + == Conclusion + + This document has provided an overview of FreeBSD. + Refer to the extref:{handbook}[FreeBSD Handbook] for more in-depth coverage of these topics as well as the many topics not covered by this document. +diff --git a/documentation/content/en/articles/rc-scripting/_index.adoc b/documentation/content/en/articles/rc-scripting/_index.adoc +index 14e0ad4bb4..dbb718ebb9 100644 +--- a/documentation/content/en/articles/rc-scripting/_index.adoc ++++ b/documentation/content/en/articles/rc-scripting/_index.adoc +@@ -1,1025 +1,1025 @@ + --- + title: Practical rc.d scripting in BSD + authors: + - author: Yar Tikhiy + email: yar@FreeBSD.org + copyright: 2005-2006, 2012 The FreeBSD Project + description: A guide to writing new rc.d scripts and understanding those already written + trademarks: ["freebsd", "netbsd", "general"] + tags: ["rc.d", "scripting", "guide", "tutorial", "FreeBSD"] + --- + + = Practical rc.d scripting in BSD + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/rc-scripting/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + Beginners may find it difficult to relate the facts from the formal documentation on the BSD [.filename]#rc.d# framework with the practical tasks of [.filename]#rc.d# scripting. + In this article, we consider a few typical cases of increasing complexity, show [.filename]#rc.d# features suited for each case, and discuss how they work. + Such an examination should provide reference points for further study of the design and efficient application of [.filename]#rc.d#. + + ''' + + toc::[] + + [[rcng-intro]] + == Introduction + + The historical BSD had a monolithic startup script, [.filename]#/etc/rc#. + It was invoked by man:init[8] at system boot time and performed all userland tasks required for multi-user operation: checking and mounting file systems, setting up the network, starting daemons, and so on. + The precise list of tasks was not the same in every system; admins needed to customize it. + With few exceptions, [.filename]#/etc/rc# had to be modified, and true hackers liked it. + + The real problem with the monolithic approach was that it provided no control over the individual components started from [.filename]#/etc/rc#. + For instance, [.filename]#/etc/rc# could not restart a single daemon. + The system admin had to find the daemon process by hand, kill it, wait until it actually exited, then browse through [.filename]#/etc/rc# for the flags, and finally type the full command line to start the daemon again. + The task would become even more difficult and prone to errors if the service to restart consisted of more than one daemon or demanded additional actions. + In a few words, the single script failed to fulfil what scripts are for: to make the system admin's life easier. + + Later there was an attempt to split out some parts of [.filename]#/etc/rc# for the sake of starting the most important subsystems separately. + The notorious example was [.filename]#/etc/netstart# to bring up networking. + It did allow for accessing the network from single-user mode, but it did not integrate well into the automatic startup process because parts of its code needed to interleave with actions essentially unrelated to networking. + That was why [.filename]#/etc/netstart# mutated into [.filename]#/etc/rc.network#. + The latter was no longer an ordinary script; it comprised of large, tangled man:sh[1] functions called from [.filename]#/etc/rc# at different stages of system startup. + However, as the startup tasks grew diverse and sophisticated, the "quasi-modular" approach became even more of a drag than the monolithic [.filename]#/etc/rc# had been. + + Without a clean and well-designed framework, the startup scripts had to bend over backwards to satisfy the needs of rapidly developing BSD-based operating systems. + It became obvious at last that more steps are necessary on the way to a fine-grained and extensible [.filename]#rc# system. + Thus BSD [.filename]#rc.d# was born. + Its acknowledged fathers were Luke Mewburn and the NetBSD community. + Later it was imported into FreeBSD. + Its name refers to the location of system scripts for individual services, which is in [.filename]#/etc/rc.d#. + Soon we will learn about more components of the [.filename]#rc.d# system and see how the individual scripts are invoked. + + The basic ideas behind BSD [.filename]#rc.d# are _fine modularity_ and __code reuse__. + _Fine modularity_ means that each basic "service" such as a system daemon or primitive startup task gets its own man:sh[1] script able to start the service, stop it, reload it, check its status. + A particular action is chosen by the command-line argument to the script. + The [.filename]#/etc/rc# script still drives system startup, but now it merely invokes the smaller scripts one by one with the `start` argument. + It is easy to perform shutdown tasks as well by running the same set of scripts with the `stop` argument, which is done by [.filename]#/etc/rc.shutdown#. + Note how closely this follows the Unix way of having a set of small specialized tools, each fulfilling its task as well as possible. + _Code reuse_ means that common operations are implemented as man:sh[1] functions and collected in [.filename]#/etc/rc.subr#. + Now a typical script can be just a few lines' worth of man:sh[1] code. + Finally, an important part of the [.filename]#rc.d# framework is man:rcorder[8], which helps [.filename]#/etc/rc# to run the small scripts orderly with respect to dependencies between them. + It can help [.filename]#/etc/rc.shutdown#, too, because the proper order for the shutdown sequence is opposite to that of startup. + + The BSD [.filename]#rc.d# design is described in crossref:rc-scripting[lukem, the original article by Luke Mewburn], and the [.filename]#rc.d# components are documented in great detail in crossref:rc-scripting[manpages, the respective manual pages]. + However, it might not appear obvious to an [.filename]#rc.d# newbie how to tie the numerous bits and pieces together to create a well-styled script for a particular task. + Therefore this article will try a different approach to describe [.filename]#rc.d#. + It will show which features should be used in a number of typical cases, and why. + Note that this is not a how-to document because our aim is not at giving ready-made recipes, but at showing a few easy entrances into the [.filename]#rc.d# realm. + Neither is this article a replacement for the relevant manual pages. + Do not hesitate to refer to them for more formal and complete documentation while reading this article. + + There are prerequisites to understanding this article. + First of all, you should be familiar with the man:sh[1] scripting language to master [.filename]#rc.d#. + In addition, you should know how the system performs userland startup and shutdown tasks, which is described in man:rc[8]. + + This article focuses on the FreeBSD branch of [.filename]#rc.d#. + Nevertheless, it may be useful to NetBSD developers, too, because the two branches of BSD [.filename]#rc.d# not only share the same design but also stay similar in their aspects visible to script authors. + + [[rcng-task]] + == Outlining the task + + A little consideration before starting `$EDITOR` will not hurt. + To write a well-tempered [.filename]#rc.d# script for a system service, we should be able to answer the following questions first: + + * Is the service mandatory or optional? + * Will the script serve a single program, e.g., a daemon, or perform more complex actions? + * Which other services will our service depend on, and vice versa? + + From the examples that follow we will see why it is important to know the answers to these questions. + + [[rcng-dummy]] + == A dummy script + + The following script just emits a message each time the system boots up: + + [.programlisting] + .... + #!/bin/sh <.> + + . /etc/rc.subr <.> + + name="dummy" <.> + start_cmd="${name}_start" <.> + stop_cmd=":" <.> + + dummy_start() <.> + { + echo "Nothing started." + } + + load_rc_config $name <.> + run_rc_command "$1" <.> + .... + + Things to note are: + + ➊ An interpreted script should begin with the magic "shebang" line. + That line specifies the interpreter program for the script. + Due to the shebang line, the script can be invoked exactly like a binary program provided that it has the execute bit set. + (See man:chmod[1].) + For example, a system admin can run our script manually, from the command line: + + [source,shell] + .... + # /etc/rc.d/dummy start + .... + + [NOTE] + ==== + To be properly managed by the [.filename]#rc.d# framework, its scripts need to be written in the man:sh[1] language. + If you have a service or port that uses a binary control utility or a startup routine written in another language, install that element in [.filename]#/usr/sbin# (for the system) or [.filename]#/usr/local/sbin# (for ports) and call it from a man:sh[1] script in the appropriate [.filename]#rc.d# directory. + ==== + + [TIP] + ==== + If you would like to learn the details of why [.filename]#rc.d# scripts must be written in the man:sh[1] language, see how [.filename]#/etc/rc# invokes them by means of `run_rc_script`, then study the implementation of `run_rc_script` in [.filename]#/etc/rc.subr#. + ==== + + ➋ In [.filename]#/etc/rc.subr#, a number of man:sh[1] functions are defined for an [.filename]#rc.d# script to use. + The functions are documented in man:rc.subr[8]. + While it is theoretically possible to write an [.filename]#rc.d# script without ever using man:rc.subr[8], its functions prove extremely handy and make the job an order of magnitude easier. So it is no surprise that everybody resorts to man:rc.subr[8] in [.filename]#rc.d# scripts. + We are not going to be an exception. + + An [.filename]#rc.d# script must "source"[.filename]#/etc/rc.subr# (include it using "`.`") _before_ it calls man:rc.subr[8] functions so that man:sh[1] has an opportunity to learn the functions. + The preferred style is to source [.filename]#/etc/rc.subr# first of all. + + [NOTE] + ==== + Some useful functions related to networking are provided by another include file, [.filename]#/etc/network.subr#. + ==== + + ➌ [[name-var]]The mandatory variable `name` specifies the name of our script. + It is required by man:rc.subr[8]. + That is, each [.filename]#rc.d# script _must_ set `name` before it calls man:rc.subr[8] functions. + + Now it is the right time to choose a unique name for our script once and for all. + We will use it in a number of places while developing the script. + The content of the name variable needs to match the script name, + some parts of FreeBSD (e.g., crossref:rc-scripting[rcng-service-jails, service jails] and the cpuset feature of the rc framework) depend upon this. + As such the filename shall also not contain characters which may be troublesome in scripting (e.g., do not use a hyphen "-" and others). + + [NOTE] + ==== + The current style of [.filename]#rc.d# scripting is to enclose values assigned to variables in double quotes. + Keep in mind that it is just a style issue that may not always be applicable. + You can safely omit quotes from around simple words without man:sh[1] metacharacters in them, while in certain cases you will need single quotes to prevent any interpretation of the value by man:sh[1]. + A programmer should be able to tell the language syntax from style conventions and use both of them wisely. + ==== + + ➍ The main idea behind man:rc.subr[8] is that an [.filename]#rc.d# script provides handlers, or methods, for man:rc.subr[8] to invoke. + In particular, `start`, `stop`, and other arguments to an [.filename]#rc.d# script are handled this way. + A method is a man:sh[1] expression stored in a variable named `argument_cmd`, where _argument_ corresponds to what can be specified on the script's command line. + We will see later how man:rc.subr[8] provides default methods for the standard arguments. + + [NOTE] + ==== + To make the code in [.filename]#rc.d# more uniform, it is common to use `${name}` wherever appropriate. + Thus a number of lines can be just copied from one script to another. + ==== + + ➎ We should keep in mind that man:rc.subr[8] provides default methods for the standard arguments. + Consequently, we must override a standard method with a no-op man:sh[1] expression if we want it to do nothing. + + ➏ The body of a sophisticated method can be implemented as a function. + It is a good idea to make the function name meaningful. + + [IMPORTANT] + ==== + It is strongly recommended to add the prefix `${name}` to the names of all functions defined in our script so they never clash with the functions from man:rc.subr[8] or another common include file. + ==== + + ➐ This call to man:rc.subr[8] loads man:rc.conf[5] variables. + Our script makes no use of them yet, but it still is recommended to load man:rc.conf[5] because there can be man:rc.conf[5] variables controlling man:rc.subr[8] itself. + + ➑ Usually this is the last command in an [.filename]#rc.d# script. + It invokes the man:rc.subr[8] machinery to perform the requested action using the variables and methods our script has provided. + + [[rcng-confdummy]] + == A configurable dummy script + + Now let us add some controls to our dummy script. + As you may know, [.filename]#rc.d# scripts are controlled with man:rc.conf[5]. + Fortunately, man:rc.subr[8] hides all the complications from us. + The following script uses man:rc.conf[5] via man:rc.subr[8] to see whether it is enabled in the first place, and to fetch a message to show at boot time. + These two tasks in fact are independent. + On the one hand, an [.filename]#rc.d# script can just support enabling and disabling its service. + On the other hand, a mandatory [.filename]#rc.d# script can have configuration variables. + We will do both things in the same script though: + + [.programlisting] + .... + #!/bin/sh + + . /etc/rc.subr + + name=dummy + rcvar=dummy_enable <.> + + start_cmd="${name}_start" + stop_cmd=":" + + load_rc_config $name <.> + : ${dummy_enable:=no} <.> + : ${dummy_msg="Nothing started."} <.> + + dummy_start() + { + echo "$dummy_msg" <.> + } + + run_rc_command "$1" + .... + + What changed in this example? + + ➊ The variable `rcvar` specifies the name of the ON/OFF knob variable. + + ➋ Now `load_rc_config` is invoked earlier in the script, before any man:rc.conf[5] variables are accessed. + + [NOTE] + ==== + While examining [.filename]#rc.d# scripts, keep in mind that man:sh[1] defers the evaluation of expressions in a function until the latter is called. + Therefore it is not an error to invoke `load_rc_config` as late as just before `run_rc_command` and still access man:rc.conf[5] variables from the method functions exported to `run_rc_command`. + This is because the method functions are to be called by `run_rc_command`, which is invoked _after_ `load_rc_config`. + ==== + + ➌ A warning will be emitted by `run_rc_command` if `rcvar` itself is set, but the indicated knob variable is unset. + If your [.filename]#rc.d# script is for the base system, you should add a default setting for the knob to [.filename]#/etc/defaults/rc.conf# and document it in man:rc.conf[5]. + Otherwise it is your script that should provide a default setting for the knob. + The canonical approach to the latter case is shown in the example. + + [NOTE] + ==== + You can make man:rc.subr[8] act as though the knob is set to `ON`, irrespective of its current setting, by prefixing the argument to the script with `one` or `force`, as in `onestart` or `forcestop`. + Keep in mind though that `force` has other dangerous effects we will touch upon below, while `one` just overrides the ON/OFF knob. + E.g., assume that `dummy_enable` is `OFF`. + The following command will run the `start` method in spite of the setting: + + [source,shell] + .... + # /etc/rc.d/dummy onestart + .... + + ==== + + ➍ Now the message to be shown at boot time is no longer hard-coded in the script. + It is specified by an man:rc.conf[5] variable named `dummy_msg`. + This is a trivial example of how man:rc.conf[5] variables can control an [.filename]#rc.d# script. + + [IMPORTANT] + ==== + The names of all man:rc.conf[5] variables used exclusively by our script _must_ have the same prefix: `${name}_`. + For example: `dummy_mode`, `dummy_state_file`, and so on. + ==== + + [NOTE] + ==== + While it is possible to use a shorter name internally, e.g., just `msg`, adding the unique prefix `${name}_` to all global names introduced by our script will save us from possible collisions with the man:rc.subr[8] namespace. + + As a rule, [.filename]#rc.d# scripts of the base system need not provide defaults for their man:rc.conf[5] variables because the defaults should be set in [.filename]#/etc/defaults/rc.conf# instead. + On the other hand, [.filename]#rc.d# scripts for ports should provide the defaults as shown in the example. + ==== + + ➎ Here we use `dummy_msg` to actually control our script, i.e., to emit a variable message. + Use of a shell function is overkill here, since it only runs a single command; an equally valid alternative is: + + [.programlisting] + .... + start_cmd="echo \"$dummy_msg\"" + .... + + [[rcng-daemon]] + == Startup and shutdown of a simple daemon + + We said earlier that man:rc.subr[8] could provide default methods. + Obviously, such defaults cannot be too general. + They are suited for the common case of starting and shutting down a simple daemon program. + Let us assume now that we need to write an [.filename]#rc.d# script for such a daemon called `mumbled`. + Here it is: + + [.programlisting] + .... + #!/bin/sh + + . /etc/rc.subr + + name=mumbled + rcvar=mumbled_enable + + command="/usr/sbin/${name}" <.> + + load_rc_config $name + run_rc_command "$1" + .... + + Pleasingly simple, isn't it? Let us examine our little script. + The only new thing to note is as follows: + + ➊ The `command` variable is meaningful to man:rc.subr[8]. + If it is set, man:rc.subr[8] will act according to the scenario of serving a conventional daemon. + In particular, the default methods will be provided for such arguments: `start`, `stop`, `restart`, `poll`, and `status`. + + The daemon will be started by running `$command` with command-line flags specified by `$mumbled_flags`. + Thus all the input data for the default `start` method are available in the variables set by our script. + Unlike `start`, other methods may require additional information about the process started. + For instance, `stop` must know the PID of the process to terminate it. + In the present case, man:rc.subr[8] will scan through the list of all processes, looking for a process with its name equal to `procname`. + The latter is another variable of meaning to man:rc.subr[8], and its value defaults to that of `command`. + In other words, when we set `command`, `procname` is effectively set to the same value. + This enables our script to kill the daemon and to check if it is running in the first place. + + [NOTE] + ==== + Some programs are in fact executable scripts. + The system runs such a script by starting its interpreter and passing the name of the script to it as a command-line argument. + This is reflected in the list of processes, which can confuse man:rc.subr[8]. + You should additionally set `command_interpreter` to let man:rc.subr[8] know the actual name of the process if `$command` is a script. + + For each [.filename]#rc.d# script, there is an optional man:rc.conf[5] variable that takes precedence over `command`. + Its name is constructed as follows: `${name}_program`, where `name` is the + mandatory variable we discussed crossref:rc-scripting[name-var, earlier]. + E.g., in this case it will be `mumbled_program`. + It is man:rc.subr[8] that arranges `${name}_program` to override `command`. + + Of course, man:sh[1] will permit you to set `${name}_program` from man:rc.conf[5] or the script itself even if `command` is unset. + In that case, the special properties of `${name}_program` are lost, and it becomes an ordinary variable your script can use for its own purposes. + However, the sole use of `${name}_program` is discouraged because using it together with `command` became an idiom of [.filename]#rc.d# scripting. + ==== + + For more detailed information on default methods, refer to man:rc.subr[8]. + + [[rcng-daemon-adv]] + == Startup and shutdown of an advanced daemon + + Let us add some meat onto the bones of the previous script and make it more complex and featureful. + The default methods can do a good job for us, but we may need some of their aspects tweaked. + Now we will learn how to tune the default methods to our needs. + + [.programlisting] + .... + #!/bin/sh + + . /etc/rc.subr + + name=mumbled + rcvar=mumbled_enable + + command="/usr/sbin/${name}" + command_args="mock arguments > /dev/null 2>&1" <.> + + pidfile="/var/run/${name}.pid" <.> + + required_files="/etc/${name}.conf /usr/share/misc/${name}.rules" <.> + + sig_reload="USR1" <.> + + start_precmd="${name}_prestart" <.> + stop_postcmd="echo Bye-bye" <.> + + extra_commands="reload plugh xyzzy" <.> + + plugh_cmd="mumbled_plugh" <.> + xyzzy_cmd="echo 'Nothing happens.'" + + mumbled_prestart() + { + if checkyesno mumbled_smart; then <.> + rc_flags="-o smart ${rc_flags}" <.> + fi + case "$mumbled_mode" in + foo) + rc_flags="-frotz ${rc_flags}" + ;; + bar) + rc_flags="-baz ${rc_flags}" + ;; + *) + warn "Invalid value for mumbled_mode" <.> + return 1 <.> + ;; + esac + run_rc_command xyzzy <.> + return 0 + } + + mumbled_plugh() <.> + { + echo 'A hollow voice says "plugh".' + } + + load_rc_config $name + run_rc_command "$1" + .... + + ➊ Additional arguments to `$command` can be passed in `command_args`. + They will be added to the command line after `$mumbled_flags`. + Since the final command line is passed to `eval` for its actual execution, input and output redirections can be specified in `command_args`. + + [NOTE] + ==== + _Never_ include dashed options, like `-X` or `--foo`, in `command_args`. + The contents of `command_args` will appear at the end of the final command line, hence they are likely to follow arguments present in `${name}_flags`; but most commands will not recognize dashed options after ordinary arguments. + A better way of passing additional options to `$command` is to add them to the beginning of `${name}_flags`. + Another way is to modify `rc_flags` crossref:rc-scripting[rc-flags, as shown later]. + ==== + + ➋ A good-mannered daemon should create a _pidfile_ so that its process can be found more easily and reliably. + The variable `pidfile`, if set, tells man:rc.subr[8] where it can find the pidfile for its default methods to use. + + [NOTE] + ==== + In fact, man:rc.subr[8] will also use the pidfile to see if the daemon is already running before starting it. + This check can be skipped by using the `faststart` argument. + ==== + + ➌ If the daemon cannot run unless certain files exist, just list them in `required_files`, and man:rc.subr[8] will check that those files do exist before starting the daemon. + There also are `required_dirs` and `required_vars` for directories and environment variables, respectively. + They all are described in detail in man:rc.subr[8]. + + [NOTE] + ==== + The default method from man:rc.subr[8] can be forced to skip the prerequisite checks by using `forcestart` as the argument to the script. + ==== + + ➍ We can customize signals to send to the daemon in case they differ from the well-known ones. + In particular, `sig_reload` specifies the signal that makes the daemon reload its configuration; it is SIGHUP by default. + Another signal is sent to stop the daemon process; + the default is SIGTERM, but this can be changed by setting `sig_stop` appropriately. + + [NOTE] + ==== + The signal names should be specified to man:rc.subr[8] without the `SIG` prefix, as it is shown in the example. + The FreeBSD version of man:kill[1] can recognize the `SIG` prefix, but the versions from other OS types may not. + ==== + + ➎➏ Performing additional tasks before or after the default methods is easy. + For each command-argument supported by our script, we can define `argument_precmd` and `argument_postcmd`. + These man:sh[1] commands are invoked before and after the respective method, as it is evident from their names. + + [NOTE] + ==== + Overriding a default method with a custom `argument_cmd` still does not prevent us from making use of `argument_precmd` or `argument_postcmd` if we need to. + In particular, the former is good for checking custom, sophisticated conditions that should be met before performing the command itself. + Using `argument_precmd` along with `argument_cmd` lets us logically separate the checks from the action. + + Do not forget that you can cram any valid man:sh[1] expressions into the methods, pre-, and post-commands you define. + Just invoking a function that makes the real job is a good style in most cases, but never let style limit your understanding of what is going on behind the curtain. + ==== + + ➐ If we would like to implement custom arguments, which can also be thought of as _commands_ to our script, we need to list them in `extra_commands` and provide methods to handle them. + + [NOTE] + ==== + The `reload` command is special. On the one hand, it has a preset method in man:rc.subr[8]. + On the other hand, `reload` is not offered by default. + The reason is that not all daemons use the same reload mechanism and some have nothing to reload at all. + So we need to ask explicitly that the builtin functionality be provided. + We can do so via `extra_commands`. + + What do we get from the default method for `reload`? Quite often daemons reload their configuration upon reception of a signal - typically, SIGHUP. + Therefore man:rc.subr[8] attempts to reload the daemon by sending a signal to it. + The signal is preset to SIGHUP but can be customized via `sig_reload` if necessary. + ==== + + ➑⓮ Our script supports two non-standard commands, `plugh` and `xyzzy`. + We saw them listed in `extra_commands`, and now it is time to provide methods for them. + The method for `xyzzy` is just inlined while that for `plugh` is implemented as the `mumbled_plugh` function. + + Non-standard commands are not invoked during startup or shutdown. + Usually they are for the system admin's convenience. + They can also be used from other subsystems, e.g., man:devd[8] if specified in man:devd.conf[5]. + + The full list of available commands can be found in the usage line printed by man:rc.subr[8] when the script is invoked without arguments. + For example, here is the usage line from the script under study: + + [source,shell] + .... + # /etc/rc.d/mumbled + Usage: /etc/rc.d/mumbled [fast|force|one](start|stop|restart|rcvar|reload|plugh|xyzzy|status|poll) + .... + + ⓭ A script can invoke its own standard or non-standard commands if needed. + This may look similar to calling functions, but we know that commands and shell functions are not always the same thing. + For instance, `xyzzy` is not implemented as a function here. + In addition, there can be a pre-command and post-command, which should be invoked orderly. + So the proper way for a script to run its own command is by means of man:rc.subr[8], as shown in the example. + + ➒ A handy function named `checkyesno` is provided by man:rc.subr[8]. + It takes a variable name as its argument and returns a zero exit code if and only if the variable is set to `YES`, or `TRUE`, or `ON`, or `1`, case insensitive; + a non-zero exit code is returned otherwise. + In the latter case, the function tests the variable for being set to `NO`, `FALSE`, `OFF`, or `0`, case insensitive; + it prints a warning message if the variable contains anything else, i.e., junk. + + Keep in mind that for man:sh[1] a zero exit code means true and a non-zero exit code means false. + + [IMPORTANT] + ==== + The `checkyesno` function takes a __variable name__. + Do not pass the expanded _value_ of a variable to it; it will not work as expected. + + The following is the correct usage of `checkyesno`: + + [.programlisting] + .... + if checkyesno mumbled_enable; then + foo + fi + .... + + On the contrary, calling `checkyesno` as shown below will not work - at least not as expected: + + [.programlisting] + .... + if checkyesno "${mumbled_enable}"; then + foo + fi + .... + + ==== + + ➓ [[rc-flags]]We can affect the flags to be passed to `$command` by modifying `rc_flags` in `$start_precmd`. + + ⓫ In certain cases we may need to emit an important message that should go to `syslog` as well. + This can be done easily with the following man:rc.subr[8] functions: `debug`, `info`, `warn`, and `err`. + The latter function then exits the script with the code specified. + + ⓬ The exit codes from methods and their pre-commands are not just ignored by default. + If `argument_precmd` returns a non-zero exit code, the main method will not be performed. + In turn, `argument_postcmd` will not be invoked unless the main method returns a zero exit code. + + [NOTE] + ==== + However, man:rc.subr[8] can be instructed from the command line to ignore those exit codes and invoke all commands anyway by prefixing an argument with `force`, as in `forcestart`. + ==== + + [[rcng-hookup]] + == Connecting a script to the rc.d framework + + After a script has been written, it needs to be integrated into [.filename]#rc.d#. + The crucial step is to install the script in [.filename]#/etc/rc.d# (for the base system) or [.filename]#/usr/local/etc/rc.d# (for ports). + Both [.filename]#bsd.prog.mk# and [.filename]#bsd.port.mk# provide convenient hooks for that, and usually you do not have to worry about the proper ownership and mode. + System scripts should be installed from [.filename]#src/libexec/rc/rc.d# through the [.filename]#Makefile# found there. +-Port scripts can be installed using `USE_RC_SUBR` as described extref:{porters-handbook}[in the Porter's Handbook, rc-scripts]. ++Port scripts can be installed using `USE_RC_SUBR` as described extref:{porters-handbook}special[in the Porter's Handbook, rc-scripts]. + + However, we should consider beforehand the place of our script in the system startup sequence. + The service handled by our script is likely to depend on other services. + For instance, a network daemon cannot function without the network interfaces and routing up and running. + Even if a service seems to demand nothing, it can hardly start before the basic filesystems have been checked and mounted. + + We mentioned man:rcorder[8] already. + Now it is time to have a close look at it. + In a nutshell, man:rcorder[8] takes a set of files, examines their contents, and prints a dependency-ordered list of files from the set to `stdout`. + The point is to keep dependency information _inside_ the files so that each file can speak for itself only. + A file can specify the following information: + + * the names of the "conditions" (which means services to us) it __provides__; + * the names of the "conditions" it __requires__; + * the names of the "conditions" this file should run __before__; + * additional _keywords_ that can be used to select a subset from the whole set of files (man:rcorder[8] can be instructed via options to include or omit the files having particular keywords listed.) + + It is no surprise that man:rcorder[8] can handle only text files with a syntax close to that of man:sh[1]. + That is, special lines understood by man:rcorder[8] look like man:sh[1] comments. + The syntax of such special lines is rather rigid to simplify their processing. + See man:rcorder[8] for details. + + Besides using man:rcorder[8] special lines, a script can insist on its dependency upon another service by just starting it forcibly. + This can be needed when the other service is optional and will not start by itself because the system admin has disabled it mistakenly in man:rc.conf[5]. + + With this general knowledge in mind, let us consider the simple daemon script enhanced with dependency stuff: + + [.programlisting] + .... + #!/bin/sh + + # PROVIDE: mumbled oldmumble <.> + # REQUIRE: DAEMON cleanvar frotz <.> + # BEFORE: LOGIN <.> + # KEYWORD: nojail shutdown <.> + + . /etc/rc.subr + + name=mumbled + rcvar=mumbled_enable + + command="/usr/sbin/${name}" + start_precmd="${name}_prestart" + + mumbled_prestart() + { + if ! checkyesno frotz_enable && \ + ! /etc/rc.d/frotz forcestatus 1>/dev/null 2>&1; then + force_depend frotz || return 1 <.> + fi + return 0 + } + + load_rc_config $name + run_rc_command "$1" + .... + + As before, detailed analysis follows: + + ➊ That line declares the names of "conditions" our script provides. + Now other scripts can record a dependency on our script by those names. + + [NOTE] + ==== + Usually a script specifies a single condition provided. + However, nothing prevents us from listing several conditions there, e.g., for compatibility reasons. + + In any case, the name of the main, or the only, `PROVIDE:` condition should be the same as `${name}`. + ==== + + ➋➌ So our script indicates which "conditions" provided by other scripts it depends on. + According to the lines, our script asks man:rcorder[8] to put it after the script(s) providing [.filename]#DAEMON# and [.filename]#cleanvar#, but before that providing [.filename]#LOGIN#. + + [NOTE] + ==== + The `BEFORE:` line should not be abused to work around an incomplete dependency list in the other script. + The appropriate case for using `BEFORE:` is when the other script does not care about ours, but our script can do its task better if run before the other one. + A typical real-life example is the network interfaces vs. the firewall: While the interfaces do not depend on the firewall in doing their job, the system security will benefit from the firewall being ready before there is any network traffic. + + Besides conditions corresponding to a single service each, there are meta-conditions and their "placeholder" scripts used to ensure that certain groups of operations are performed before others. + These are denoted by [.filename]#UPPERCASE# names. + Their list and purposes can be found in man:rc[8]. + + Keep in mind that putting a service name in the `REQUIRE:` line does not guarantee that the service will actually be running by the time our script starts. + The required service may fail to start or just be disabled in man:rc.conf[5]. + Obviously, man:rcorder[8] cannot track such details, and man:rc[8] will not do that either. + Consequently, the application started by our script should be able to cope with any required services being unavailable. + In certain cases, we can help it as discussed crossref:rc-scripting[forcedep, below] + ==== + + [[keywords]]➍ As we remember from the above text, man:rcorder[8] keywords can be used to select or leave out some scripts. + Namely any man:rcorder[8] consumer can specify through `-k` and `-s` options which keywords are on the "keep list" and "skip list", respectively. + From all the files to be dependency sorted, man:rcorder[8] will pick only those having a keyword from the keep list (unless empty) and not having a keyword from the skip list. + + In FreeBSD, man:rcorder[8] is used by [.filename]#/etc/rc# and [.filename]#/etc/rc.shutdown#. + These two scripts define the standard list of FreeBSD [.filename]#rc.d# keywords and their meanings as follows: + + nojail:: The service is not for man:jail[8] environment. + The automatic startup and shutdown procedures will ignore the script if inside a jail. + + nostart:: The service is to be started manually or not started at all. + The automatic startup procedure will ignore the script. + In conjunction with the [.filename]#shutdown# keyword, this can be used to write scripts that do something only at system shutdown. + + shutdown:: This keyword is to be listed __explicitly__ if the service needs to be stopped before system shutdown. + + [NOTE] + ==== + When the system is going to shut down, [.filename]#/etc/rc.shutdown# runs. + It assumes that most [.filename]#rc.d# scripts have nothing to do at that time. + Therefore [.filename]#/etc/rc.shutdown# selectively invokes [.filename]#rc.d# scripts with the [.filename]#shutdown# keyword, effectively ignoring the rest of the scripts. + For even faster shutdown, [.filename]#/etc/rc.shutdown# passes the [.filename]#faststop# command to the scripts it runs so that they skip preliminary checks, e.g., the pidfile check. + As dependent services should be stopped before their prerequisites, [.filename]#/etc/rc.shutdown# runs the scripts in reverse dependency order. + If writing a real [.filename]#rc.d# script, you should consider whether it is relevant at system shutdown time. + E.g., if your script does its work in response to the [.filename]#start# command only, then you need not to include this keyword. + However, if your script manages a service, it is probably a good idea to stop it before the system proceeds to the final stage of its shutdown sequence described in man:halt[8]. + In particular, a service should be stopped explicitly if it needs considerable time or special actions to shut down cleanly. + A typical example of such a service is a database engine. + ==== + + [[forcedep]]➎ To begin with, `force_depend` should be used with much care. + It is generally better to revise the hierarchy of configuration variables for your [.filename]#rc.d# scripts if they are interdependent. + + If you still cannot do without `force_depend`, the example offers an idiom of how to invoke it conditionally. + In the example, our `mumbled` daemon requires that another one, `frotz`, be started in advance. + However, `frotz` is optional, too; and man:rcorder[8] knows nothing about such details. + Fortunately, our script has access to all man:rc.conf[5] variables. + If `frotz_enable` is true, we hope for the best and rely on [.filename]#rc.d# to have started `frotz`. + Otherwise we forcibly check the status of `frotz`. + Finally, we enforce our dependency on `frotz` if it is found to be not running. + A warning message will be emitted by `force_depend` because it should be invoked only if a misconfiguration has been detected. + + [[rcng-args]] + == Giving more flexibility to an rc.d script + + When invoked during startup or shutdown, an [.filename]#rc.d# script is supposed to act on the entire subsystem it is responsible for. + E.g., [.filename]#/etc/rc.d/netif# should start or stop all network interfaces described by man:rc.conf[5]. + Either task can be uniquely indicated by a single command argument such as `start` or `stop`. + Between startup and shutdown, [.filename]#rc.d# scripts help the admin to control the running system, and it is when the need for more flexibility and precision arises. + For instance, the admin may want to add the settings of a new network interface to man:rc.conf[5] and then to start it without interfering with the operation of the existing interfaces. + Next time the admin may need to shut down a single network interface. + In the spirit of the command line, the respective [.filename]#rc.d# script calls for an extra argument, the interface name. + + Fortunately, man:rc.subr[8] allows for passing any number of arguments to script's methods (within the system limits). + Due to that, the changes in the script itself can be minimal. + + How can man:rc.subr[8] gain access to the extra command-line arguments. + Should it just grab them directly? Not by any means. + Firstly, an man:sh[1] function has no access to the positional parameters of its caller, but man:rc.subr[8] is just a sack of such functions. + Secondly, the good manner of [.filename]#rc.d# dictates that it is for the main script to decide which arguments are to be passed to its methods. + + So the approach adopted by man:rc.subr[8] is as follows: `run_rc_command` passes on all its arguments but the first one to the respective method verbatim. + The first, omitted, argument is the name of the method itself: `start`, `stop`, etc. + It will be shifted out by `run_rc_command`, so what is `$2` in the original command line will be presented as `$1` to the method, and so on. + + To illustrate this opportunity, let us modify the primitive dummy script so that its messages depend on the additional arguments supplied. + Here we go: + + [.programlisting] + .... + #!/bin/sh + + . /etc/rc.subr + + name="dummy" + start_cmd="${name}_start" + stop_cmd=":" + kiss_cmd="${name}_kiss" + extra_commands="kiss" + + dummy_start() + { + if [ $# -gt 0 ]; then <.> + echo "Greeting message: $*" + else + echo "Nothing started." + fi + } + + dummy_kiss() + { + echo -n "A ghost gives you a kiss" + if [ $# -gt 0 ]; then <.> + echo -n " and whispers: $*" + fi + case "$*" in + *[.!?]) + echo + ;; + *) + echo . + ;; + esac + } + + load_rc_config $name + run_rc_command "$@" <.> + .... + + What essential changes can we notice in the script? + + ➊ All arguments you type after `start` can end up as positional parameters to the respective method. + We can use them in any way according to our task, skills, and fancy. + In the current example, we just pass all of them to man:echo[1] as one string in the next line - note `$*` within the double quotes. + Here is how the script can be invoked now: + + [source,shell] + .... + # /etc/rc.d/dummy start + Nothing started. + + # /etc/rc.d/dummy start Hello world! + Greeting message: Hello world! + .... + + ➋ The same applies to any method our script provides, not only to a standard one. + We have added a custom method named `kiss`, and it can take advantage of the extra arguments not less than `start` does. E.g.: + + [source,shell] + .... + # /etc/rc.d/dummy kiss + A ghost gives you a kiss. + + # /etc/rc.d/dummy kiss Once I was Etaoin Shrdlu... + A ghost gives you a kiss and whispers: Once I was Etaoin Shrdlu... + .... + + ➌ If we want just to pass all extra arguments to any method, we can merely substitute `"$@"` for `"$1"` in the last line of our script, where we invoke `run_rc_command`. + + [IMPORTANT] + ==== + An man:sh[1] programmer ought to understand the subtle difference between `$*` and `$@` as the ways to designate all positional parameters. + For its in-depth discussion, refer to a good handbook on man:sh[1] scripting. + _Do not_ use the expressions until you fully understand them because their misuse will result in buggy and insecure scripts. + ==== + + [NOTE] + ==== + Currently `run_rc_command` may have a bug that prevents it from keeping the original boundaries between arguments. + That is, arguments with embedded whitespace may not be processed correctly. + The bug stems from `$*` misuse. + ==== + + [[rcng-service-jails]] + == Making a script ready for Service Jails + + Scripts which start a long running service are suitable for service jails, and should come with a suitable service jail configuration. + + Some examples of scripts which are not suitable to run in a service jail: + + * any script which in the start command only changes a runtime setting for programs or the kernel, + * or tries to mount something, + * or finds and deletes files + + Scripts not suitable to run in a service jail need to prevent the use within service jails. + + A script with a long running service which needs to do something listed above before the start or after the stop, can either be split-up into two scripts with dependencies, or use the precommand and postcommand parts of the script to perform this action. + + By default, only the start and stop parts of a script are run within a service jail, the rest is run outside the jail. + As such any setting used in the start/stop parts of the script can not be set from e.g. a precommand. + +-To make a script ready for use with extref:../../books/handbook/jails/#service-jails[Service Jails], only one more config line needs to be inserted: ++To make a script ready for use with extref:{handbook}jails[Service Jails, service-jails], only one more config line needs to be inserted: + + [.programlisting] + .... + #!/bin/sh + + . /etc/rc.subr + + name="dummy" + start_cmd="${name}_start" + stop_cmd=":" + + : ${dummy_svcj_options:=""} <.> + + dummy_start() + { + echo "Nothing started." + } + + load_rc_config $name + run_rc_command "$1" + .... + + ➊ If it makes sense that the script runs in a jail, it must have an overridable service jails configuration. + If it does not need network access or access to any other resource which is restricted in jails, an empty config like displayed is enough. + + Strictly speaking an empty config is not needed, but it explicitly describes that the script is service jails ready, and that it does not need additional jail permissions. + As such it is highly recommended to add such an empty config in such a case. + The most common option to use is "net_basic", which enables the use of the hosts IPv4 and IPv6 addresses. + All possible options are explained in man:rc.conf[5]. + + If a setting for the start/stop depends on variables from the rc-framework (e.g., set inside man:rc.conf[5]), this needs to be handled by ``load_rc_config`` and ``run_rc_command`` instead of inside a precommand. + + If for some reason a script can not be run within a service jail, e.g., because it is not possible to run or it does not make sense to run it in a jail, use the following: + + [.programlisting] + .... + #!/bin/sh + + . /etc/rc.subr + + name="dummy" + start_cmd="${name}_start" + stop_cmd=":" + + dummy_start() + { + echo "Nothing started." + } + + load_rc_config $name + dummy_svcj="NO" # does not make sense to run in a svcj <.> + run_rc_command "$1" + .... + + ➊ The disabling needs to happen after the ``load_rc_config`` call, else a man:rc.conf[5] setting may override it. + + [[rcng-instancing]] + == Advanced rc-scripting: Instancing + + Sometimes it is useful to run several instances of a service. + Typically you want to be able to start/stop such instances independently, + and you want to have a separate config file for each instance. + Each instance should be started at boot, + survive updates, + and benefit from updates. + + Here is an example of a rc script which supports this: + + [.programlisting] + .... + #!/bin/sh + + # + # PROVIDE: dummy + # REQUIRE: NETWORKING SERVERS + # KEYWORD: shutdown + # + # Add these following line to /etc/rc.conf.local or /etc/rc.conf + # to enable this service: + # + # dummy_enable (bool): Set it to YES to enable dummy on startup. + # Default: NO + # dummy_user (string): User account to run with. + # Default: www + # + + . /etc/rc.subr + + case $0 in <.> + /etc/rc*) + # during boot (shutdown) $0 is /etc/rc (/etc/rc.shutdown), + # so get the name of the script from $_file + name=$_file + ;; + *) + name=$0 + ;; + esac + + name=${name##*/} <.> + rcvar="${name}_enable" <.> + desc="Short description of this service" + command="/usr/local/sbin/dummy" + + load_rc_config "$name" + + eval "${rcvar}=\${${rcvar}:-'NO'}" <.> + eval "${name}_svcj_options=\${${name}_svcj_options:-'net_basic'}" <.> + eval "_dummy_user=\${${name}_user:-'www'}" <.> + + _dummy_configname=/usr/local/etc/${name}.cfg <.> + pidfile=/var/run/dummy/${name}.pid + required_files ${_dummy_configname} + command_args="-u ${_dummy_user} -c ${_dummy_configfile} -p ${pidfile}" + + run_rc_command "$1" + .... + + ➊ and ➋ make sure to set the name variable to the man:basename[1] of the script name. + If the filename is [.filename]#/usr/local/etc/rc.d/dummy#, + name is set to [.filename]#dummy#. + This way changing the filename of the rc script changes automatically the content of the name variable. + + ➌ specifies the variable name which is used in [.filename]#rc.conf# to enable this service based upon the filename of this script. + In this example this resolves to dummy_enable. + + ➍ makes sure the default for the _enable variable is NO. + + ➎ is an example of having some defaults for service specific framework variables, + in this case the service jails options. + + ➏ and ➐ set variables internal to the script (pay attention to the underscore in front of _dummy_user to make it different from dummy_user which can be set in [.filename]#rc.conf#). + + The part in ➎ is for variables which are not used inside the script itself but in the rc framework. + All the variables which are used as parameters somewhere in the script are assigned to a generic variable like in ➐ to make it more easy to reference them (no need to eval them at each place of use). + + This script will now behave differently if the start script has a different name. + This allows to create symlinks to it: + + [source,shell] + .... + # ln -s dummy /usr/local/etc/rc.d/dummy_foo + # sysrc dummy_foo_enable=YES + # service dummy_foo start + .... + + The above creates an instance of the dummy service with the name dummy_foo. + It does not use the config file [.filename]#/usr/local/etc/dummy.cfg# but the config file [.filename]#/usr/local/etc/dummy_foo.cfg# (➐), + and it uses the PID file [.filename]#/var/run/dummy/dummy_foo.pid# instead of [.filename]#/var/run/dummy/dummy.pid#. + + The services dummy and dummy_foo can be managed independently of each other, + while having the start script update itself on package update (due to the symlink). + This does not update the REQUIRE line, + as such there is no easy way of depending on a specific instance. + To depend upon a specific instance in the startup order a copy needs to be made instead of using a symlink. + This prevents the automatic pick-up of changes to the start script when an update is installed. + + [[rcng-furthur]] + == Further reading + + [[lukem]]http://www.mewburn.net/luke/papers/rc.d.pdf[The original article by Luke Mewburn] offers a general overview of [.filename]#rc.d# and detailed rationale for its design decisions. + It provides insight on the whole [.filename]#rc.d# framework and its place in a modern BSD operating system. + + [[manpages]]The manual pages man:rc[8], man:rc.subr[8], and man:rcorder[8] document the [.filename]#rc.d# components in great detail. + You cannot fully use the [.filename]#rc.d# power without studying the manual pages and referring to them while writing your own scripts. + + The major source of working, real-life examples is [.filename]#/etc/rc.d# in a live system. + Its contents are easy and pleasant to read because most rough corners are hidden deep in man:rc.subr[8]. + Keep in mind though that the [.filename]#/etc/rc.d# scripts were not written by angels, so they might suffer from bugs and suboptimal design decisions. + Now you can improve them! +diff --git a/documentation/content/en/articles/releng/_index.adoc b/documentation/content/en/articles/releng/_index.adoc +index f19ccb2bdd..56cacf4f0f 100644 +--- a/documentation/content/en/articles/releng/_index.adoc ++++ b/documentation/content/en/articles/releng/_index.adoc +@@ -1,447 +1,447 @@ + --- + title: Legacy FreeBSD Release Engineering + authors: + - author: Murray Stokely + email: murray@FreeBSD.org + webpage: https://people.FreeBSD.org/~murray/ + description: This paper describes the approach previously used by the FreeBSD release engineering team to make production quality releases of the FreeBSD Operating System + trademarks: ["freebsd", "intel", "general"] + tags: ["Release", "Engineering", "Historical", "FreeBSD"] + --- + + = FreeBSD Release Engineering + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/releng/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + [NOTE] + ==== + This document is outdated and does not accurately describe the current release procedures of the FreeBSD Release Engineering team. + It is retained for historical purposes. + The current procedures used by the FreeBSD Release Engineering team are available in the extref:{freebsd-releng}[FreeBSD Release Engineering] article. + ==== + + This paper describes the approach used by the FreeBSD release engineering team to make production quality releases of the FreeBSD Operating System. + It details the methodology used for the official FreeBSD releases and describes the tools available for those interested in producing customized FreeBSD releases for corporate rollouts or commercial productization. + + ''' + + toc::[] + + [[introduction]] + == Introduction + + The development of FreeBSD is a very open process. + FreeBSD is comprised of contributions from thousands of people around the world. + The FreeBSD Project provides Subversion footnote:[Subversion, http://subversion.apache.org] access to the general public so that others can have access to log messages, diffs (patches) between development branches, and other productivity enhancements that formal source code management provides. + This has been a huge help in attracting more talented developers to FreeBSD. + However, I think everyone would agree that chaos would soon manifest if write access to the main repository was opened up to everyone on the Internet. + Therefore only a "select" group of nearly 300 people are given write access to the Subversion repository. + These extref:{contributors}[FreeBSD committers, staff-committers]footnote:[extref:{contributors}[FreeBSD committers, staff-committers]] are usually the people who do the bulk of FreeBSD development. + An elected link:https://www.FreeBSD.org/administration/#t-core[Core Team]footnote:[link:https://www.FreeBSD.org/administration/#t-core[FreeBSD Core Team]] of developers provide some level of direction over the project. + + The rapid pace of `FreeBSD` development makes the main development branch unsuitable for the everyday use by the general public. + In particular, stabilizing efforts are required for polishing the development system into a production quality release. + To solve this conflict, development continues on several parallel tracks. + The main development branch is the _HEAD_ or _trunk_ of our Subversion tree, known as "FreeBSD-CURRENT" or "-CURRENT" for short. + + A set of more stable branches are maintained, known as "FreeBSD-STABLE" or "-STABLE" for short. + All branches live in a master Subversion repository maintained by the FreeBSD Project. + FreeBSD-CURRENT is the "bleeding-edge" of FreeBSD development where all new changes first enter the system. + FreeBSD-STABLE is the development branch from which major releases are made. + Changes go into this branch at a different pace, and with the general assumption that they have first gone into FreeBSD-CURRENT and have been thoroughly tested by our user community. + + The term _stable_ in the name of the branch refers to the presumed Application Binary Interface stability, which is promised by the project. + This means that a user application compiled on an older version of the system from the same branch works on a newer system from the same branch. + The ABI stability has improved greatly from the compared to previous releases. + In most cases, binaries from the older _STABLE_ systems run unmodified on newer systems, including __HEAD__, assuming that the system management interfaces are not used. + + In the interim period between releases, weekly snapshots are built automatically by the FreeBSD Project build machines and made available for download from `https:/download.FreeBSD.org/snapshots/`. +-The widespread availability of binary release snapshots, and the tendency of our user community to keep up with -STABLE development with Subversion and "`make buildworld`" footnote:[extref:{handbook}[Rebuilding world, makeworld]] helps to keep FreeBSD-STABLE in a very reliable condition even before the quality assurance activities ramp up pending a major release. ++The widespread availability of binary release snapshots, and the tendency of our user community to keep up with -STABLE development with Subversion and "`make buildworld`" footnote:[extref:{handbook}cutting-edge[Rebuilding world, makeworld]] helps to keep FreeBSD-STABLE in a very reliable condition even before the quality assurance activities ramp up pending a major release. + + In addition to installation ISO snapshots, weekly virtual machine images are also provided for use with VirtualBox, qemu, or other popular emulation software. + The virtual machine images can be downloaded from `https://download.FreeBSD.org/snapshots/VM-IMAGES/`. + + The virtual machine images are approximately 150MB man:xz[1] compressed, and contain a 10GB sparse filesystem when attached to a virtual machine. + + Bug reports and feature requests are continuously submitted by users throughout the release cycle. + Problems reports are entered into our Bugzilla database through the web interface provided at https://www.freebsd.org/support/bugreports/[https://www.freebsd.org/support/bugreports/]. + + To service our most conservative users, individual release branches were introduced with FreeBSD 4.3. + These release branches are created shortly before a final release is made. + After the release goes out, only the most critical security fixes and additions are merged onto the release branch. + In addition to source updates via Subversion, binary patchkits are available to keep systems on the _releng/X.Y_ branches updated. + + === What This Article Describes + + The following sections of this article describe: + + crossref:releng[release-proc, Release Process]:: + The different phases of the release engineering process leading up to the actual system build. + + crossref:releng[release-build, Release Building]:: + The actual build process. + + crossref:releng[extensibility, Extensibility]:: + How the base release may be extended by third parties. + + crossref:releng[lessons-learned, Lessons Learned from FreeBSD 4.4]:: + Some of the lessons learned through the release of FreeBSD 4.4. + + crossref:releng[future, Future Directions]:: + Future directions of development. + + [[release-proc]] + == Release Process + + New releases of FreeBSD are released from the -STABLE branch at approximately four month intervals. + The FreeBSD release process begins to ramp up 70-80 days before the anticipated release date when the release engineer sends an email to the development mailing lists to remind developers that they only have 15 days to integrate new changes before the code freeze. + During this time, many developers perform what have become known as "MFC sweeps". + + MFC stands for "Merge From CURRENT" and it describes the process of merging a tested change from our -CURRENT development branch to our -STABLE branch. + Project policy requires any change to be first applied to trunk, and merged to the -STABLE branches after sufficient external testing was done by -CURRENT users (developers are expected to extensively test the change before committing to -CURRENT, but it is impossible for a person to exercise all usages of the general-purpose operating system). + Minimal MFC period is 3 days, which is typically used only for trivial or critical bugfixes. + + === Code Review + + Sixty days before the anticipated release, the source repository enters a "code freeze". + During this time, all commits to the -STABLE branch must be approved by `{re}`. + The approval process is technically enforced by a pre-commit hook. + The kinds of changes that are allowed during this period include: + + * Bug fixes. + * Documentation updates. + * Security-related fixes of any kind. + * Minor changes to device drivers, such as adding new Device IDs. + * Driver updates from the vendors. + * Any additional change that the release engineering team feels is justified, given the potential risk. + + Shortly after the code freeze is started, a _BETA1_ image is built and released for widespread testing. + During the code freeze, at least one beta image or release candidate is released every two weeks until the final release is ready. + During the days preceding the final release, the release engineering team is in constant communication with the security-officer team, the documentation maintainers, and the port maintainers to ensure that all of the different components required for a successful release are available. + + After the quality of the BETA images is satisfying enough, and no large and potentially risky changes are planned, the release branch is created and _Release Candidate_ (RC) images are built from the release branch, instead of the BETA images from the STABLE branch. + Also, the freeze on the STABLE branch is lifted and release branch enters a "hard code freeze" where it becomes much harder to justify new changes to the system unless a serious bug-fix or security issue is involved. + + === Final Release Checklist + + When several BETA images have been made available for widespread testing and all major issues have been resolved, the final release "polishing" can begin. + + [[rel-branch]] + ==== Creating the Release Branch + + [NOTE] + ==== + In all examples below, `$FSVN` refers to the location of the FreeBSD Subversion repository, `svn+ssh://svn.FreeBSD.org/base/`. + ==== + + The layout of FreeBSD branches in Subversion is described in the extref:{committers-guide}[Committer's Guide, subversion-primer-base-layout]. + The first step in creating a branch is to identify the revision of the `stable/_X_` sources that you want to branch _from_. + + [source,shell] + .... + # svn log -v $FSVN/stable/9 + .... + + The next step is to create the _release branch_ + + [source,shell] + .... + # svn cp $FSVN/stable/9@REVISION $FSVN/releng/9.2 + .... + + This branch can be checked out: + + [source,shell] + .... + # svn co $FSVN/releng/9.2 src + .... + + [NOTE] + ==== + Creating the `releng` branch and `release` tags is done by the link:https://www.FreeBSD.org/administration/#t-re[Release Engineering Team]. + ==== + + image::branches-head.png[FreeBSD Development Branch] + + image::branches-releng3.png[FreeBSD 3.x STABLE Branch] + + image::branches-releng4.png[FreeBSD 4.x STABLE Branch] + + image::branches-releng5.png[FreeBSD 5.x STABLE Branch] + + image::branches-releng6.png[FreeBSD 6.x STABLE Branch] + + image::branches-releng7.png[FreeBSD 7.x STABLE Branch] + + image::branches-releng8.png[FreeBSD 8.x STABLE Branch] + + image::branches-releng9.png[FreeBSD 9.x STABLE Branch] + + [[versionbump]] + ==== Bumping up the Version Number + + Before the final release can be tagged, built, and released, the following files need to be modified to reflect the correct version of FreeBSD: + + * [.filename]#doc/en_US.ISO8859-1/books/handbook/mirrors/chapter.xml# + * [.filename]#doc/en_US.ISO8859-1/books/porters-handbook/book.xml# + * [.filename]#doc/en_US.ISO8859-1/htdocs/cgi/ports.cgi# + * [.filename]#ports/Tools/scripts/release/config# + * [.filename]#doc/shared/xml/freebsd.ent# + * [.filename]#src/Makefile.inc1# + * [.filename]#src/UPDATING# + * [.filename]#src/gnu/usr.bin/groff/tmac/mdoc.local# + * [.filename]#src/release/Makefile# + * [.filename]#src/release/doc/en_US.ISO8859-1/shared/xml/release.dsl# + * [.filename]#src/release/doc/shared/examples/Makefile.relnotesng# + * [.filename]#src/release/doc/shared/xml/release.ent# + * [.filename]#src/sys/conf/newvers.sh# + * [.filename]#src/sys/sys/param.h# + * [.filename]#src/usr.sbin/pkg_install/add/main.c# + * [.filename]#doc/en_US.ISO8859-1/htdocs/search/opensearch/man.xml# + + The release notes and errata files also need to be adjusted for the new release (on the release branch) and truncated appropriately (on the stable/current branch): + + * [.filename]#src/release/doc/en_US.ISO8859-1/relnotes/common/new.xml# + * [.filename]#src/release/doc/en_US.ISO8859-1/errata/article.xml# + + Sysinstall should be updated to note the number of available ports and the amount of disk space required for the Ports Collection. + footnote:[FreeBSD Ports Collection https://ports.FreeBSD.org] + This information is currently kept in [.filename]#src/usr.sbin/bsdinstall/dist.c#. + + After the release has been built, a number of files should be updated to announce the release to the world. + These files are relative to `head/` within the `doc/` subversion tree. + + * [.filename]#share/images/articles/releng/branches-relengX.pic# + * [.filename]#head/shared/xml/release.ent# + * [.filename]#en_US.ISO8859-1/htdocs/releases/*# + * [.filename]#en_US.ISO8859-1/htdocs/releng/index.xml# + * [.filename]#share/xml/news.xml# + + Additionally, update the "BSD Family Tree" file: + + * [.filename]#src/shared/misc/bsd-family-tree# + + ==== Creating the Release Tag + + When the final release is ready, the following command will create the `release/9.2.0` tag. + + [source,shell] + .... + # svn cp $FSVN/releng/9.2 $FSVN/release/9.2.0 + .... + + The Documentation and Ports managers are responsible for tagging their respective trees with the `tags/RELEASE_9_2_0` tag. + + When the Subversion `svn cp` command is used to create a __release tag__, this identifies the source at a specific point in time. + By creating tags, we ensure that future release builders will always be able to use the same source we used to create the official FreeBSD Project releases. + + [[release-build]] + == Release Building + + FreeBSD "releases" can be built by anyone with a fast machine and access to a source repository. + (That should be everyone, since we offer Subversion access! See the extref:{handbook}[Subversion section in the Handbook, svn] for details.) + The _only_ special requirement is that the man:md[4] device must be available. + If the device is not loaded into your kernel, then the kernel module should be automatically loaded when man:mdconfig[8] is executed during the boot media creation phase. + All of the tools necessary to build a release are available from the Subversion repository in [.filename]#src/release#. + These tools aim to provide a consistent way to build FreeBSD releases. + A complete release can actually be built with only a single command, including the creation of ISO images suitable for burning to CDROM or DVD, and an FTP install directory. + man:release[7] fully documents the `src/release/generate-release.sh` script which is used to build a release. + `generate-release.sh` is a wrapper around the Makefile target: `make release`. + + === Building a Release + + man:release[7] documents the exact commands required to build a FreeBSD release. + The following sequences of commands can build an 9.2.0 release: + + [source,shell] + .... + # cd /usr/src/release + # sh generate-release.sh release/9.2.0 /local3/release + .... + + After running these commands, all prepared release files are available in [.filename]#/local3/release/R# directory. + + The release [.filename]#Makefile# can be broken down into several distinct steps. + + * Creation of a sanitized system environment in a separate directory hierarchy with "`make installworld`". + * Checkout from Subversion of a clean version of the system source, documentation, and ports into the release build hierarchy. + * Population of [.filename]#/etc# and [.filename]#/dev# in the chrooted environment. + * chroot into the release build hierarchy, to make it harder for the outside environment to taint this build. + * `make world` in the chrooted environment. + * Build of Kerberos-related binaries. + * Build [.filename]#GENERIC# kernel. + * Creation of a staging directory tree where the binary distributions will be built and packaged. + * Build and installation of the documentation toolchain needed to convert the documentation source (SGML) into HTML and text documents that will accompany the release. + * Build and installation of the actual documentation (user manuals, tutorials, release notes, hardware compatibility lists, and so on.) + * Package up distribution tarballs of the binaries and sources. + * Create FTP installation hierarchy. + * _(optionally)_ Create ISO images for CDROM/DVD media. + + For more information about the release build infrastructure, please see man:release[7]. + + [NOTE] + ==== + It is important to remove any site-specific settings from [.filename]#/etc/make.conf#. + For example, it would be unwise to distribute binaries that were built on a system with `CPUTYPE` set to a specific processor. + ==== + + === Contributed Software ("ports") + + The https://ports.FreeBSD.org[FreeBSD Ports collection] is a collection of over {numports} third-party software packages available for FreeBSD. + The `{portmgr}` is responsible for maintaining a consistent ports tree that can be used to create the binary packages that accompany official FreeBSD releases. + + === Release ISOs + + Starting with FreeBSD 4.4, the FreeBSD Project decided to release all four ISO images that were previously sold on the _BSDi/Wind River Systems/FreeBSD Mall_ "official" CDROM distributions. + Each of the four discs must contain a [.filename]#README.TXT# file that explains the contents of the disc, a [.filename]#CDROM.INF# file that provides meta-data for the disc so that man:bsdinstall[8] can validate and use the contents, and a [.filename]#filename.txt# file that provides a manifest for the disc. + This _manifest_ can be created with a simple command: + + [source,shell] + .... + /stage/cdrom# find . -type f | sed -e 's/^\.\///' | sort > filename.txt + .... + + The specific requirements of each CD are outlined below. + + ==== Disc 1 + + The first disc is almost completely created by `make release`. + The only changes that should be made to the [.filename]#disc1# directory are the addition of a [.filename]#tools# directory, and as many popular third party software packages as will fit on the disc. + The [.filename]#tools# directory contains software that allow users to create installation floppies from other operating systems. + This disc should be made bootable so that users of modern PCs do not need to create installation floppy disks. + + If a custom kernel of FreeBSD is to be included, then man:bsdinstall[8] and man:release[7] must be updated to include installation instructions. + The relevant code is contained in [.filename]#src/release# and [.filename]#src/usr.sbin/bsdinstall#. + Specifically, the file [.filename]#src/release/Makefile#, and [.filename]#dist.c#, [.filename]#dist.h#, [.filename]#menus.c#, [.filename]#install.c#, and [.filename]#Makefile# will need to be updated under [.filename]#src/usr.sbin/bsdinstall#. + Optionally, you may choose to update [.filename]#bsdinstall.8#. + + ==== Disc 2 + + The second disc is also largely created by `make release`. + This disc contains a "live filesystem" that can be used from man:bsdinstall[8] to troubleshoot a FreeBSD installation. + This disc should be bootable and should also contain a compressed copy of the CVS repository in the [.filename]#CVSROOT# directory and commercial software demos in the [.filename]#commerce# directory. + + ==== Multi-volume Support + + Sysinstall supports multiple volume package installations. + This requires that each disc have an [.filename]#INDEX# file containing all of the packages on all volumes of a set, along with an extra field that indicates which volume that particular package is on. + Each volume in the set must also have the `CD_VOLUME` variable set in the [.filename]#cdrom.inf# file so that bsdinstall can tell which volume is which. + When a user attempts to install a package that is not on the current disc, bsdinstall will prompt the user to insert the appropriate one. + + [[distribution]] + == Distribution + + [[dist-ftp]] + === FTP Sites + + When the release has been thoroughly tested and packaged for distribution, the master FTP site must be updated. + The official FreeBSD public FTP sites are all mirrors of a master server that is open only to other FTP sites. + This site is known as `ftp-master`. + When the release is ready, the following files must be modified on `ftp-master`: + + [.filename]#/pub/FreeBSD/releases/arch/X.Y-RELEASE/#:: + The installable FTP directory as output from `make release`. + + [.filename]#/pub/FreeBSD/ports/arch/packages-X.Y-release/#:: + The complete package build for this release. + + [.filename]#/pub/FreeBSD/releases/arch/X.Y-RELEASE/tools#:: + A symlink to [.filename]#../../../tools#. + + [.filename]#/pub/FreeBSD/releases/arch/X.Y-RELEASE/packages#:: + A symlink to [.filename]#../../../ports/arch/packages-X.Y-release#. + + [.filename]#/pub/FreeBSD/releases/arch/ISO-IMAGES/X.Y/X.Y-RELEASE-arch-*.iso#:: + The ISO images. The "*" is [.filename]#disc1#, [.filename]#disc2#, etc. + Only if there is a [.filename]#disc1# and there is an alternative first installation CD (for example a stripped-down install with no windowing system) there may be a [.filename]#mini# as well. + + For more information about the distribution mirror architecture of the FreeBSD FTP sites, please see the extref:{hubs}[Mirroring FreeBSD] article. + + It may take many hours to two days after updating `ftp-master` before a majority of the Tier-1 FTP sites have the new software depending on whether or not a package set got loaded at the same time. + It is imperative that the release engineers coordinate with the {mirror-announce} before announcing the general availability of new software on the FTP sites. + Ideally the release package set should be loaded at least four days prior to release day. + The release bits should be loaded between 24 and 48 hours before the planned release time with "other" file permissions turned off. + This will allow the mirror sites to download it but the general public will not be able to download it from the mirror sites. + Mail should be sent to {mirror-announce} at the time the release bits get posted saying the release has been staged and giving the time that the mirror sites should begin allowing access. + Be sure to include a time zone with the time, for example make it relative to GMT. + + [[dist-cdrom]] + === CD-ROM Replication + + Coming soon: Tips for sending FreeBSD ISOs to a replicator and quality assurance measures to be taken. + + [[extensibility]] + == Extensibility + + Although FreeBSD forms a complete operating system, there is nothing that forces you to use the system exactly as we have packaged it up for distribution. + We have tried to design the system to be as extensible as possible so that it can serve as a platform that other commercial products can be built on top of. + The only "rule" we have about this is that if you are going to distribute FreeBSD with non-trivial changes, we encourage you to document your enhancements! + The FreeBSD community can only help support users of the software we provide. + We certainly encourage innovation in the form of advanced installation and administration tools, for example, but we cannot be expected to answer questions about it. + + === Scripting `bsdinstall` + + The FreeBSD system installation and configuration tool, man:bsdinstall[8], can be scripted to provide automated installs for large sites. +-This functionality can be used in conjunction with Intel(R) PXE footnote:[extref:{handbook}[Diskless Operation with PXE, network-diskless]] to bootstrap systems from the network. ++This functionality can be used in conjunction with Intel(R) PXE footnote:[extref:{handbook}advanced-networking[Diskless Operation with PXE, network-diskless]] to bootstrap systems from the network. + + [[lessons-learned]] + == Lessons Learned from FreeBSD 4.4 + + The release engineering process for 4.4 formally began on August 1st, 2001. + After that date all commits to the `RELENG_4` branch of FreeBSD had to be explicitly approved by the `{re}`. + The first release candidate for the x86 architecture was released on August 16, followed by 4 more release candidates leading up to the final release on September 18th. + The security officer was very involved in the last week of the process as several security issues were found in the earlier release candidates. + A total of over _500_ emails were sent to the `{re}` in little over a month. + + Our user community has made it very clear that the security and stability of a FreeBSD release should not be sacrificed for any self-imposed deadlines or target release dates. + The FreeBSD Project has grown tremendously over its lifetime and the need for standardized release engineering procedures has never been more apparent. + This will become even more important as FreeBSD is ported to new platforms. + + [[future]] + == Future Directions + + It is imperative for our release engineering activities to scale with our growing userbase. + Along these lines we are working very hard to document the procedures involved in producing FreeBSD releases. + + * _Parallelism_ - Certain portions of the release build are actually "embarrassingly parallel". Most of the tasks are very I/O intensive, so having multiple high-speed disk drives is actually more important than using multiple processors in speeding up the `make release` process. If multiple disks are used for different hierarchies in the man:chroot[2] environment, then the CVS checkout of the [.filename]#ports# and [.filename]#doc# trees can be happening simultaneously as the `make world` on another disk. Using a RAID solution (hardware or software) can significantly decrease the overall build time. + * _Cross-building releases_ - Building IA-64 or Alpha release on x86 hardware? `make TARGET=ia64 release`. + * _Regression Testing_ - We need better automated correctness testing for FreeBSD. + * _Installation Tools_ - Our installation program has long since outlived its intended life span. Several projects are under development to provide a more advanced installation mechanism. The libh project was one such project that aimed to provide an intelligent new package framework and GUI installation program. + + [[ackno]] + == Acknowledgements + + I would like to thank Jordan Hubbard for giving me the opportunity to take on some of the release engineering responsibilities for FreeBSD 4.4 and also for all of his work throughout the years making FreeBSD what it is today. + Of course the release would not have been possible without all of the release-related work done by `{asami}`, `{steve}`, `{bmah}`, `{nik}`, `{obrien}`, `{kris}`, `{jhb}` and the rest of the FreeBSD development community. + I would also like to thank `{rgrimes}`, `{phk}`, and others who worked on the release engineering tools in the very early days of FreeBSD. + This article was influenced by release engineering documents from the CSRG footnote:[Marshall Kirk McKusick, Michael J. Karels, and Keith Bostic: link:http://docs.FreeBSD.org/44doc/papers/releng.html[The Release Engineering of 4.3BSD]] , the NetBSD Project, footnote:[NetBSD Developer Documentation: Release Engineering http://www.NetBSD.org/developers/releng/index.html] , and John Baldwin's proposed release engineering process notes. footnote:[John Baldwin's FreeBSD Release Engineering Proposal https://people.FreeBSD.org/~jhb/docs/releng.txt] +diff --git a/documentation/content/en/articles/remote-install/_index.adoc b/documentation/content/en/articles/remote-install/_index.adoc +index ba9bf48256..2575637b94 100644 +--- a/documentation/content/en/articles/remote-install/_index.adoc ++++ b/documentation/content/en/articles/remote-install/_index.adoc +@@ -1,395 +1,395 @@ + --- + title: Remote Installation of the FreeBSD Operating System Without a Remote Console + authors: + - author: Daniel Gerzo + email: danger@FreeBSD.org + copyright: 2008-2021 The FreeBSD Documentation Project + description: Describes the remote installation of the FreeBSD operating system when the console of the remote system is unavailable + trademarks: ["freebsd", "general"] + tags: ["Remote", "Installation", "FreeBSD"] + --- + + = Remote Installation of the FreeBSD Operating System Without a Remote Console + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/remote-install/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + This article documents the remote installation of the FreeBSD operating system when the console of the remote system is unavailable. + The main idea behind this article is the result of a collaboration with `{mm}` with valuable input provided by `{pjd}`. + + ''' + + toc::[] + + [[background]] + == Background + + There are many server hosting providers in the world, but very few of them are officially supporting FreeBSD. + They usually provide support for a Linux(R) distribution to be installed on the servers they offer. + + In some cases, these companies will install your preferred Linux(R) distribution if you request it. + Using this option, we will attempt to install FreeBSD. In other cases, they may offer a rescue system which would be used in an emergency. + It is possible to use this for our purposes as well. + + This article covers the basic installation and configuration steps required to bootstrap a remote installation of FreeBSD with RAID-1 and ZFS capabilities. + + [[intro]] + == Introduction + + This section will summarize the purpose of this article and better explain what is covered herein. + The instructions included in this article will benefit those using services provided by colocation facilities not supporting FreeBSD. + + [.procedure] + ==== + . As we have mentioned in the crossref:remote-install[background, Background] section, many of the reputable server hosting companies provide some kind of rescue system, which is booted from their LAN and accessible over SSH. They usually provide this support to help their customers fix broken operating systems. As this article will explain, it is possible to install FreeBSD with the help of these rescue systems. + + + . The next section of this article will describe how to configure, and build minimalistic FreeBSD on the local machine. That version will eventually be running on the remote machine from a ramdisk, which will allow us to install a complete FreeBSD operating system from an FTP mirror using the sysinstall utility. + . The rest of this article will describe the installation procedure itself, as well as the configuration of the ZFS file system. + ==== + + [[requirements]] + === Requirements + + To continue successfully, you must: + + * Have a network accessible operating system with SSH access + * Understand the FreeBSD installation process + * Be familiar with the man:sysinstall[8] utility + * Have the FreeBSD installation SO image or CD handy + + [[preparation]] + == Preparation - mfsBSD + + Before FreeBSD may be installed on the target system, it is necessary to build the minimal FreeBSD operating system image which will boot from the hard drive. + This way the new system can be accessed from the network, and the rest of the installation can be done without remote access to the system console. + + The mfsBSD tool-set can be used to build a tiny FreeBSD image. + As the name of mfsBSD suggests ("mfs" means "memory file system"), the resulting image runs entirely from a ramdisk. + Thanks to this feature, the manipulation of hard drives will not be limited, therefore it will be possible to install a complete FreeBSD operating system. + The mfsBSD http://mfsbsd.vx.sk/[home page] includes pointers to the latest release of the toolset. + + Please note that the internals of mfsBSD and how it all fits together is beyond the scope of this article. + The interested reader should consult the original documentation of mfsBSD for more details. + + Download and extract the latest mfsBSD release and change your working directory to the directory where the mfsBSD scripts will reside: + + [source,shell] + .... + # fetch http://mfsbsd.vx.sk/release/mfsbsd-2.1.tar.gz + # tar xvzf mfsbsd-2.1.tar.gz + # cd mfsbsd-2.1/ + .... + + [[mfsbsd-config]] + === Configuration of mfsBSD + + Before booting mfsBSD, a few important configuration options have to be set. + The most important that we have to get right is, naturally, the network setup. + The most suitable method to configure networking options depends on whether we know beforehand the type of the network interface we will use, and the network interface driver to be loaded for our hardware. + We will see how mfsBSD can be configured in either case. + + Another important thing to set is the `root` password. + This can be done by editing [.filename]#conf/loader.conf#. + Please see the included comments. + + ==== The [.filename]#conf/interfaces.conf# method + + When the installed network interface card is unknown, it is possible to use the auto-detection features of mfsBSD. + The startup scripts of mfsBSD can detect the correct driver to use, based on the MAC address of the interface, if we set the following options in [.filename]#conf/interfaces.conf#: + + [.programlisting] + .... + mac_interfaces="ext1" + ifconfig_ext1_mac="00:00:00:00:00:00" + ifconfig_ext1="inet 192.168.0.2/24" + .... + + Do not forget to add the `defaultrouter` information to [.filename]#conf/rc.conf#: + + [.programlisting] + .... + defaultrouter="192.168.0.1" + .... + + ==== The [.filename]#conf/rc.conf# Method + + When the network interface driver is known, it is more convenient to use [.filename]#conf/rc.conf# for networking options. + The syntax of this file is the same as the one used in the standard man:rc.conf[5] file of FreeBSD. + + For example, if you know that a man:re[4] network interface is going to be available, you can set the following options in [.filename]#conf/rc.conf#: + + [.programlisting] + .... + defaultrouter="192.168.0.1" + ifconfig_re0="inet 192.168.0.2/24" + .... + + [[mfsbsd-build]] + === Building an mfsBSD Image + + The process of building an mfsBSD image is pretty straightforward. + + The first step is to mount the FreeBSD installation CD, or the installation ISO image to [.filename]#/cdrom#. + For the sake of example, in this article we will assume that you have downloaded the FreeBSD 10.1-RELEASE ISO. + Mounting this ISO image to the [.filename]#/cdrom# directory is easy with the man:mdconfig[8] utility: + + [source,shell] + .... + # mdconfig -a -t vnode -u 10 -f FreeBSD-10.1-RELEASE-amd64-disc1.iso + # mount_cd9660 /dev/md10 /cdrom + .... + + Since the recent FreeBSD releases do not contain regular distribution sets, it is required to extract the FreeBSD distribution files from the distribution archives located on the ISO image: + + [source,shell] + .... + # mkdir DIST + # tar -xvf /cdrom/usr/freebsd-dist/base.txz -C DIST + # tar -xvf /cdrom/usr/freebsd-dist/kernel.txz -C DIST + .... + + Next, build the bootable mfsBSD image: + + [source,shell] + .... + # make BASE=DIST + .... + + [NOTE] + ==== + The above `make` has to be run from the top level of the mfsBSD directory tree, for example [.filename]#~/mfsbsd-2.1/#. + ==== + + === Booting mfsBSD + + Now that the mfsBSD image is ready, it must be uploaded to the remote system running a live rescue system or pre-installed Linux(R) distribution. + The most suitable tool for this task is scp: + + [source,shell] + .... + # scp disk.img root@192.168.0.2:. + .... + + To boot mfsBSD image properly, it must be placed on the first (bootable) device of the given machine. + This may be accomplished using this example providing that [.filename]#sda# is the first bootable disk device: + + [source,shell] + .... + # dd if=/root/disk.img of=/dev/sda bs=1m + .... + + If all went well, the image should now be in the MBR of the first device and the machine can be rebooted. + Watch for the machine to boot up properly with the man:ping[8] tool. + Once it has came back on-line, it should be possible to access it over man:ssh[1] as user `root` with the configured password. + + [[installation]] + == Installation of the FreeBSD Operating System + + The mfsBSD has been successfully booted and it should be possible to log in through man:ssh[1]. + This section will describe how to create and label slices, set up `gmirror` for RAID-1, and how to use `sysinstall` to install a minimal distribution of the FreeBSD operating system. + + === Preparation of Hard Drives + + The first task is to allocate disk space for FreeBSD, i.e.: to create slices and partitions. + Obviously, the currently running system is fully loaded in system memory and therefore there will be no problems with manipulating hard drives. + To complete this task, it is possible to use either `sysinstall` or man:fdisk[8] in conjunction to man:bsdlabel[8]. + + At the start, mark all system disks as empty. + Repeat the following command for each hard drive: + + [source,shell] + .... + # dd if=/dev/zero of=/dev/ad0 count=2 + .... + + Next, create slices and label them with your preferred tool. + While it is considered easier to use `sysinstall`, a powerful and also probably less buggy method will be to use standard text-based UNIX(R) tools, such as man:fdisk[8] and man:bsdlabel[8], which will also be covered in this section. +-The former option is well documented in the extref:{handbook}[Installing FreeBSD, install-steps] chapter of the FreeBSD Handbook. ++The former option is well documented in the extref:{handbook}install[Installing FreeBSD, install-steps] chapter of the FreeBSD Handbook. + As it was mentioned in the introduction, this article will present how to set up a system with RAID-1 and ZFS capabilities. + Our set up will consist of a small man:gmirror[8] mirrored [.filename]#/# (root), [.filename]#/usr# and [.filename]#/var# dataset, and the rest of the disk space will be allocated for a man:zpool[8] mirrored ZFS file system. + Please note, that the ZFS file system will be configured after the FreeBSD operating system is successfully installed and booted. + + The following example will describe how to create slices and labels, initialize man:gmirror[8] on each partition and how to create a UFS2 file system in each mirrored partition: + + [source,shell] + .... + # fdisk -BI /dev/ad0 <.> + # fdisk -BI /dev/ad1 + # bsdlabel -wB /dev/ad0s1 <.> + # bsdlabel -wB /dev/ad1s1 + # bsdlabel -e /dev/ad0s1 <.> + # bsdlabel /dev/ad0s1 > /tmp/bsdlabel.txt && bsdlabel -R /dev/ad1s1 /tmp/bsdlabel.txt <.> + # gmirror label root /dev/ad[01]s1a <.> + # gmirror label var /dev/ad[01]s1d + # gmirror label usr /dev/ad[01]s1e + # gmirror label -F swap /dev/ad[01]s1b <.> + # newfs /dev/mirror/root <.> + # newfs /dev/mirror/var + # newfs /dev/mirror/usr + .... + + <.> Create a slice covering the entire disk and initialize the boot code contained in sector 0 of the given disk. Repeat this command for all hard drives in the system. + + <.> Write a standard label for each disk including the bootstrap code. + + <.> Now, manually edit the label of the given disk. Refer to the man:bsdlabel[8] manual page to find out how to create partitions. Create partitions `a` for [.filename]#/# (root) file system, `b` for swap, `d` for [.filename]#/var#, `e` for [.filename]#/usr# and finally `f` which will later be used for ZFS. + + <.> Import the recently created label for the second hard drive, so both hard drives will be labeled in the same way. + + <.> Initialize man:gmirror[8] on each partition. + + <.> Note that `-F` is used for the swap partition. This instructs man:gmirror[8] to assume that the device is in the consistent state after the power/system failure. + + <.> Create a UFS2 file system on each mirrored partition. + + === System Installation + + This is the most important part. + This section will describe how to actually install the minimal distribution of FreeBSD on the hard drives that we have prepared in the previous section. + To accomplish this goal, all file systems need to be mounted so `sysinstall` may write the contents of FreeBSD to the hard drives: + + [source,shell] + .... + # mount /dev/mirror/root /mnt + # mkdir /mnt/var /mnt/usr + # mount /dev/mirror/var /mnt/var + # mount /dev/mirror/usr /mnt/usr + .... + + When you are done, start man:sysinstall[8]. + Select the [.guimenuitem]#Custom# installation from the main menu. + Select [.guimenuitem]#Options# and press kbd:[Enter]. + With the help of arrow keys, move the cursor on the `Install Root` item, press kbd:[Space] and change it to [.filename]#/mnt#. + Press kbd:[Enter] to submit your changes and exit the [.guimenuitem]#Options# menu by pressing kbd:[q]. + + [WARNING] + ==== + Note that this step is very important and if skipped, `sysinstall` will be unable to install FreeBSD. + ==== + + Go to the [.guimenuitem]#Distributions# menu, move the cursor with the arrow keys to `Minimal`, and check it by pressing kbd:[Space]. + This article uses the Minimal distribution to save network traffic, because the system itself will be installed over ftp. + Exit this menu by choosing `Exit`. + + [NOTE] + ==== + The [.guimenuitem]#Partition# and [.guimenuitem]#Label# menus will be skipped, as these are useless now. + ==== + + In the [.guimenuitem]#Media# menu, select `FTP`. + Select the nearest mirror and let `sysinstall` assume that the network is already configured. + You will be returned back to the [.guimenuitem]#Custom# menu. + + Finally, perform the system installation by selecting the last option, [.guimenuitem]#Commit#. + Exit `sysinstall` when it finishes the installation. + + === Post Installation Steps + + The FreeBSD operating system should be installed now; however, the process is not finished yet. + It is necessary to perform some post installation steps to allow FreeBSD to boot in the future and to be able to log in to the system. + + You must now man:chroot[8] into the freshly installed system to finish the installation. + Use the following command: + + [source,shell] + .... + # chroot /mnt + .... + + To complete our goal, perform these steps: + + * Copy the `GENERIC` kernel to the [.filename]#/boot/kernel# directory: + + + [source,shell] + .... + # cp -Rp /boot/GENERIC/* /boot/kernel + .... + + * Create the [.filename]#/etc/rc.conf#, [.filename]#/etc/resolv.conf# and [.filename]#/etc/fstab# files. Do not forget to properly set the network information and to enable sshd in [.filename]#/etc/rc.conf#. The contents of [.filename]#/etc/fstab# will be similar to the following: + + + [.programlisting] + .... + # Device Mountpoint FStype Options Dump Pass# + /dev/mirror/swap none swap sw 0 0 + /dev/mirror/root / ufs rw 1 1 + /dev/mirror/usr /usr ufs rw 2 2 + /dev/mirror/var /var ufs rw 2 2 + /dev/cd0 /cdrom cd9660 ro,noauto 0 0 + .... + * Create [.filename]#/boot/loader.conf# with the following contents: + + + [.programlisting] + .... + geom_mirror_load="YES" + zfs_load="YES" + .... + * Perform the following command, which will make ZFS available on the next boot: + + + [source,shell] + .... + # sysrc zfs_enable="YES" + .... + + * Add additional users to the system using the man:adduser[8] tool. Do not forget to add a user to the `wheel` group so you may obtain root access after the reboot. + * Double-check all your settings. + + The system should now be ready for the next boot. + Use the man:reboot[8] command to reboot your system. + + [[zfs]] + == ZFS + + If your system survived the reboot, it should now be possible to log in. + Welcome to the fresh FreeBSD installation, performed remotely without the use of a remote console! + + The only remaining step is to configure man:zpool[8] and create some man:zfs[8] file systems. + Creating and administering ZFS is very straightforward. First, create a mirrored pool: + + [source,shell] + .... + # zpool create tank mirror /dev/ad[01]s1f + .... + + Next, create some file systems: + + [source,shell] + .... + # zfs create tank/ports + # zfs create tank/src + # zfs set compression=gzip tank/ports + # zfs set compression=on tank/src + # zfs set mountpoint=/usr/ports tank/ports + # zfs set mountpoint=/usr/src tank/src + .... + + That is all. + If you are interested in more details about ZFS on FreeBSD, please refer to the https://wiki.freebsd.org/ZFS[ZFS] section of the FreeBSD Wiki. +diff --git a/documentation/content/en/articles/serial-uart/_index.adoc b/documentation/content/en/articles/serial-uart/_index.adoc +index 3852251b85..e953644b93 100644 +--- a/documentation/content/en/articles/serial-uart/_index.adoc ++++ b/documentation/content/en/articles/serial-uart/_index.adoc +@@ -1,1182 +1,1182 @@ + --- + title: Serial and UART Tutorial + authors: + - author: Frank Durda + email: uhclem@FreeBSD.org + description: Detailed information about the use of serial ports and UART with FreeBSD + trademarks: ["freebsd", "microsoft", "general"] + tags: ["Serial", "hardware", "UART", "Tutorial", "FreeBSD"] + --- + + = Serial and UART Tutorial + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/serial-uart/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + This article talks about using serial hardware with FreeBSD. + + ''' + + toc::[] + + [[uart]] + == The UART: What it is and how it works + + _Copyright (R) 1996 `{uhclem}`, All Rights Reserved. 13 January 1996._ + + The Universal Asynchronous Receiver/Transmitter (UART) controller is the key component of the serial communications subsystem of a computer. + The UART takes bytes of data and transmits the individual bits in a sequential fashion. + At the destination, a second UART re-assembles the bits into complete bytes. + + Serial transmission is commonly used with modems and for non-networked communication between computers, terminals and other devices. + + There are two primary forms of serial transmission: Synchronous and Asynchronous. + Depending on the modes that are supported by the hardware, the name of the communication sub-system will usually include a `A` if it supports Asynchronous communications, and a `S` if it supports Synchronous communications. + Both forms are described below. + + Some common acronyms are: + + [.blockquote] + UART Universal Asynchronous Receiver/Transmitter + + [.blockquote] + USART Universal Synchronous-Asynchronous Receiver/Transmitter + + === Synchronous Serial Transmission + + Synchronous serial transmission requires that the sender and receiver share a clock with one another, or that the sender provide a strobe or other timing signal so that the receiver knows when to "read" the next bit of the data. + In most forms of serial Synchronous communication, if there is no data available at a given instant to transmit, a fill character must be sent instead so that data is always being transmitted. + Synchronous communication is usually more efficient because only data bits are transmitted between sender and receiver, and synchronous communication can be more costly if extra wiring and circuits are required to share a clock signal between the sender and receiver. + + A form of Synchronous transmission is used with printers and fixed disk devices in that the data is sent on one set of wires while a clock or strobe is sent on a different wire. + Printers and fixed disk devices are not normally serial devices because most fixed disk interface standards send an entire word of data for each clock or strobe signal by using a separate wire for each bit of the word. + In the PC industry, these are known as Parallel devices. + + The standard serial communications hardware in the PC does not support Synchronous operations. + This mode is described here for comparison purposes only. + + === Asynchronous Serial Transmission + + Asynchronous transmission allows data to be transmitted without the sender having to send a clock signal to the receiver. + Instead, the sender and receiver must agree on timing parameters in advance and special bits are added to each word which are used to synchronize the sending and receiving units. + + When a word is given to the UART for Asynchronous transmissions, a bit called the "Start Bit" is added to the beginning of each word that is to be transmitted. + The Start Bit is used to alert the receiver that a word of data is about to be sent, and to force the clock in the receiver into synchronization with the clock in the transmitter. + These two clocks must be accurate enough to not have the frequency drift by more than 10% during the transmission of the remaining bits in the word. + (This requirement was set in the days of mechanical teleprinters and is easily met by modern electronic equipment.) + + After the Start Bit, the individual bits of the word of data are sent, with the Least Significant Bit (LSB) being sent first. + Each bit in the transmission is transmitted for exactly the same amount of time as all of the other bits, and the receiver "looks" at the wire at approximately halfway through the period assigned to each bit to determine if the bit is a `1` or a `0`. + For example, if it takes two seconds to send each bit, the receiver will examine the signal to determine if it is a `1` or a `0` after one second has passed, then it will wait two seconds and then examine the value of the next bit, and so on. + + The sender does not know when the receiver has "looked" at the value of the bit. + The sender only knows when the clock says to begin transmitting the next bit of the word. + + When the entire data word has been sent, the transmitter may add a Parity Bit that the transmitter generates. + The Parity Bit may be used by the receiver to perform simple error checking. + Then at least one Stop Bit is sent by the transmitter. + + When the receiver has received all of the bits in the data word, it may check for the Parity Bits (both sender and receiver must agree on whether a Parity Bit is to be used), and then the receiver looks for a Stop Bit. + If the Stop Bit does not appear when it is supposed to, the UART considers the entire word to be garbled and will report a Framing Error to the host processor when the data word is read. + The usual cause of a Framing Error is that the sender and receiver clocks were not running at the same speed, or that the signal was interrupted. + + Regardless of whether the data was received correctly or not, the UART automatically discards the Start, Parity and Stop bits. + If the sender and receiver are configured identically, these bits are not passed to the host. + + If another word is ready for transmission, the Start Bit for the new word can be sent as soon as the Stop Bit for the previous word has been sent. + + As asynchronous data is "self synchronizing", if there is no data to transmit, the transmission line can be idle. + + === Other UART Functions + + In addition to the basic job of converting data from parallel to serial for transmission and from serial to parallel on reception, a UART will usually provide additional circuits for signals that can be used to indicate the state of the transmission media, and to regulate the flow of data in the event that the remote device is not prepared to accept more data. + For example, when the device connected to the UART is a modem, the modem may report the presence of a carrier on the phone line while the computer may be able to instruct the modem to reset itself or to not take calls by raising or lowering one more of these extra signals. + The function of each of these additional signals is defined in the EIA RS232-C standard. + + === The RS232-C and V.24 Standards + + In most computer systems, the UART is connected to circuitry that generates signals that comply with the EIA RS232-C specification. + There is also a CCITT standard named V.24 that mirrors the specifications included in RS232-C. + + ==== RS232-C Bit Assignments (Marks and Spaces) + + In RS232-C, a value of `1` is called a `Mark` and a value of `0` is called a `Space`. + When a communication line is idle, the line is said to be "Marking", or transmitting continuous `1` values. + + The Start bit always has a value of `0` (a Space). + The Stop Bit always has a value of `1` (a Mark). + This means that there will always be a Mark (1) to Space (0) transition on the line at the start of every word, even when multiple word are transmitted back to back. + This guarantees that sender and receiver can resynchronize their clocks regardless of the content of the data bits that are being transmitted. + + The idle time between Stop and Start bits does not have to be an exact multiple (including zero) of the bit rate of the communication link, but most UARTs are designed this way for simplicity. + + In RS232-C, the "Marking" signal (a `1`) is represented by a voltage between -2 VDC and -12 VDC, and a "Spacing" signal (a `0`) is represented by a voltage between 0 and +12 VDC. + The transmitter is supposed to send +12 VDC or -12 VDC, and the receiver is supposed to allow for some voltage loss in long cables. + Some transmitters in low power devices (like portable computers) sometimes use only +5 VDC and -5 VDC, but these values are still acceptable to a RS232-C receiver, provided that the cable lengths are short. + + ==== RS232-C Break Signal + + RS232-C also specifies a signal called a `Break`, which is caused by sending continuous Spacing values (no Start or Stop bits). + When there is no electricity present on the data circuit, the line is considered to be sending `Break`. + + The `Break` signal must be of a duration longer than the time it takes to send a complete byte plus Start, Stop and Parity bits. + Most UARTs can distinguish between a Framing Error and a Break, but if the UART cannot do this, the Framing Error detection can be used to identify Breaks. + + In the days of teleprinters, when numerous printers around the country were wired in series (such as news services), any unit could cause a `Break` by temporarily opening the entire circuit so that no current flowed. + This was used to allow a location with urgent news to interrupt some other location that was currently sending information. + + In modern systems there are two types of Break signals. + If the Break is longer than 1.6 seconds, it is considered a "Modem Break", and some modems can be programmed to terminate the conversation and go on-hook or enter the modems' command mode when the modem detects this signal. + If the Break is smaller than 1.6 seconds, it signifies a Data Break and it is up to the remote computer to respond to this signal. + Sometimes this form of Break is used as an Attention or Interrupt signal and sometimes is accepted as a substitute for the ASCII CONTROL-C character. + + Marks and Spaces are also equivalent to "Holes" and "No Holes" in paper tape systems. + + [NOTE] + ==== + Breaks cannot be generated from paper tape or from any other byte value, since bytes are always sent with Start and Stop bit. + The UART is usually capable of generating the continuous Spacing signal in response to a special command from the host processor. + ==== + + ==== RS232-C DTE and DCE Devices + + The RS232-C specification defines two types of equipment: the Data Terminal Equipment (DTE) and the Data Carrier Equipment (DCE). + Usually, the DTE device is the terminal (or computer), and the DCE is a modem. + Across the phone line at the other end of a conversation, the receiving modem is also a DCE device and the computer that is connected to that modem is a DTE device. + The DCE device receives signals on the pins that the DTE device transmits on, and vice versa. + + When two devices that are both DTE or both DCE must be connected together without a modem or a similar media translator between them, a NULL modem must be used. + The NULL modem electrically re-arranges the cabling so that the transmitter output is connected to the receiver input on the other device, and vice versa. + Similar translations are performed on all of the control signals so that each device will see what it thinks are DCE (or DTE) signals from the other device. + + The number of signals generated by the DTE and DCE devices are not symmetrical. + The DTE device generates fewer signals for the DCE device than the DTE device receives from the DCE. + + ==== RS232-C Pin Assignments + + The EIA RS232-C specification (and the ITU equivalent, V.24) calls for a twenty-five pin connector (usually a DB25) and defines the purpose of most of the pins in that connector. + + In the IBM Personal Computer and similar systems, a subset of RS232-C signals are provided via nine pin connectors (DB9). + The signals that are not included on the PC connector deal mainly with synchronous operation, and this transmission mode is not supported by the UART that IBM selected for use in the IBM PC. + + Depending on the computer manufacturer, a DB25, a DB9, or both types of connector may be used for RS232-C communications. + (The IBM PC also uses a DB25 connector for the parallel printer interface which causes some confusion.) + + Below is a table of the RS232-C signal assignments in the DB25 and DB9 connectors. + + [.informaltable] + [cols="1,1,1,1,1,1,1", frame="none", options="header"] + |=== + | DB25 RS232-C Pin + | DB9 IBM PC Pin + | EIA Circuit Symbol + | CCITT Circuit Symbol + | Common Name + | Signal Source + | Description + + |1 + |- + |AA + |101 + |PG/FG + |- + |Frame/Protective Ground + + |2 + |3 + |BA + |103 + |TD + |DTE + |Transmit Data + + |3 + |2 + |BB + |104 + |RD + |DCE + |Receive Data + + |4 + |7 + |CA + |105 + |RTS + |DTE + |Request to Send + + |5 + |8 + |CB + |106 + |CTS + |DCE + |Clear to Send + + |6 + |6 + |CC + |107 + |DSR + |DCE + |Data Set Ready + + |7 + |5 + |AV + |102 + |SG/GND + |- + |Signal Ground + + |8 + |1 + |CF + |109 + |DCD/CD + |DCE + |Data Carrier Detect + + |9 + |- + |- + |- + |- + |- + |Reserved for Test + + |10 + |- + |- + |- + |- + |- + |Reserved for Test + + |11 + |- + |- + |- + |- + |- + |Reserved for Test + + |12 + |- + |CI + |122 + |SRLSD + |DCE + |Sec. Recv. Line Signal Detector + + |13 + |- + |SCB + |121 + |SCTS + |DCE + |Secondary Clear to Send + + |14 + |- + |SBA + |118 + |STD + |DTE + |Secondary Transmit Data + + |15 + |- + |DB + |114 + |TSET + |DCE + |Trans. Sig. Element Timing + + |16 + |- + |SBB + |119 + |SRD + |DCE + |Secondary Received Data + + |17 + |- + |DD + |115 + |RSET + |DCE + |Receiver Signal Element Timing + + |18 + |- + |- + |141 + |LOOP + |DTE + |Local Loopback + + |19 + |- + |SCA + |120 + |SRS + |DTE + |Secondary Request to Send + + |20 + |4 + |CD + |108.2 + |DTR + |DTE + |Data Terminal Ready + + |21 + |- + |- + |- + |RDL + |DTE + |Remote Digital Loopback + + |22 + |9 + |CE + |125 + |RI + |DCE + |Ring Indicator + + |23 + |- + |CH + |111 + |DSRS + |DTE + |Data Signal Rate Selector + + |24 + |- + |DA + |113 + |TSET + |DTE + |Trans. Sig. Element Timing + + |25 + |- + |- + |142 + |- + |DCE + |Test Mode + |=== + + === Bits, Baud and Symbols + + Baud is a measurement of transmission speed in asynchronous communication. + Due to advances in modem communication technology, this term is frequently misused when describing the data rates in newer devices. + + Traditionally, a Baud Rate represents the number of bits that are actually being sent over the media, not the amount of data that is actually moved from one DTE device to the other. + The Baud count includes the overhead bits Start, Stop and Parity that are generated by the sending UART and removed by the receiving UART. + This means that seven-bit words of data actually take 10 bits to be completely transmitted. + Therefore, a modem capable of moving 300 bits per second from one place to another can normally only move 30 7-bit words if Parity is used and one Start and Stop bit are present. + + If 8-bit data words are used and Parity bits are also used, the data rate falls to 27.27 words per second, because it now takes 11 bits to send the eight-bit words, and the modem still only sends 300 bits per second. + + The formula for converting bytes per second into a baud rate and vice versa was simple until error-correcting modems came along. + These modems receive the serial stream of bits from the UART in the host computer (even when internal modems are used the data is still frequently serialized) and converts the bits back into bytes. + These bytes are then combined into packets and sent over the phone line using a Synchronous transmission method. + This means that the Stop, Start, and Parity bits added by the UART in the DTE (the computer) were removed by the modem before transmission by the sending modem. + When these bytes are received by the remote modem, the remote modem adds Start, Stop and Parity bits to the words, converts them to a serial format and then sends them to the receiving UART in the remote computer, who then strips the Start, Stop and Parity bits. + + The reason all these extra conversions are done is so that the two modems can perform error correction, which means that the receiving modem is able to ask the sending modem to resend a block of data that was not received with the correct checksum. + This checking is handled by the modems, and the DTE devices are usually unaware that the process is occurring. + + By striping the Start, Stop and Parity bits, the additional bits of data that the two modems must share between themselves to perform error-correction are mostly concealed from the effective transmission rate seen by the sending and receiving DTE equipment. + For example, if a modem sends ten 7-bit words to another modem without including the Start, Stop and Parity bits, the sending modem will be able to add 30 bits of its own information that the receiving modem can use to do error-correction without impacting the transmission speed of the real data. + + The use of the term Baud is further confused by modems that perform compression. + A single 8-bit word passed over the telephone line might represent a dozen words that were transmitted to the sending modem. + The receiving modem will expand the data back to its original content and pass that data to the receiving DTE. + + Modern modems also include buffers that allow the rate that bits move across the phone line (DCE to DCE) to be a different speed than the speed that the bits move between the DTE and DCE on both ends of the conversation. + Normally the speed between the DTE and DCE is higher than the DCE to DCE speed because of the use of compression by the modems. + + As the number of bits needed to describe a byte varied during the trip between the two machines plus the differing bits-per-seconds speeds that are used present on the DTE-DCE and DCE-DCE links, the usage of the term Baud to describe the overall communication speed causes problems and can misrepresent the true transmission speed. + So Bits Per Second (bps) is the correct term to use to describe the transmission rate seen at the DCE to DCE interface and Baud or Bits Per Second are acceptable terms to use when a connection is made between two systems with a wired connection, or if a modem is in use that is not performing error-correction or compression. + + Modern high speed modems (2400, 9600, 14,400, and 19,200bps) in reality still operate at or below 2400 baud, or more accurately, 2400 Symbols per second. + High speed modem are able to encode more bits of data into each Symbol using a technique called Constellation Stuffing, which is why the effective bits per second rate of the modem is higher, but the modem continues to operate within the limited audio bandwidth that the telephone system provides. + Modems operating at 28,800 and higher speeds have variable Symbol rates, but the technique is the same. + + === The IBM Personal Computer UART + +-Starting with the original IBM Personal Computer, IBM selected the National Semiconductor INS8250 UART for use in the IBM PC Parallel/Serial Adapter. ++Starting with the original IBM Personal Computer, IBM selected the National Semiconductor INS8250 UART for use in the IBM PC Parallel/Serial Adapter. + Subsequent generations of compatible computers from IBM and other vendors continued to use the INS8250 or improved versions of the National Semiconductor UART family. + + ==== National Semiconductor UART Family Tree + + There have been several versions and subsequent generations of the INS8250 UART. Each major version is described below. + + [.programlisting] + .... + INS8250 -> INS8250B + \ + \ + \-> INS8250A -> INS82C50A + \ + \ + \-> NS16450 -> NS16C450 + \ + \ + \-> NS16550 -> NS16550A -> PC16550D + .... + + INS8250:: + This part was used in the original IBM PC and IBM PC/XT. + The original name for this part was the INS8250 ACE (Asynchronous Communications Element) and it is made from NMOS technology. + + + The 8250 uses eight I/O ports and has a one-byte send and a one-byte receive buffer. + This original UART has several race conditions and other flaws. + The original IBM BIOS includes code to work around these flaws, but this made the BIOS dependent on the flaws being present, so subsequent parts like the 8250A, 16450 or 16550 could not be used in the original IBM PC or IBM PC/XT. + INS8250-B:: + This is the slower speed of the INS8250 made from NMOS technology. + It contains the same problems as the original INS8250. + + INS8250A:: + An improved version of the INS8250 using XMOS technology with various functional flaws corrected. + The INS8250A was used initially in PC clone computers by vendors who used "clean" BIOS designs. + Due to the corrections in the chip, this part could not be used with a BIOS compatible with the INS8250 or INS8250B. + + INS82C50A:: + This is a CMOS version (low power consumption) of the INS8250A and has similar functional characteristics. + + NS16450:: + Same as NS8250A with improvements so it can be used with faster CPU bus designs. + IBM used this part in the IBM AT and updated the IBM BIOS to no longer rely on the bugs in the INS8250. + + NS16C450:: + This is a CMOS version (low power consumption) of the NS16450. + + NS16550:: + Same as NS16450 with a 16-byte send and receive buffer but the buffer design was flawed and could not be reliably be used. + + NS16550A:: + Same as NS16550 with the buffer flaws corrected. + The 16550A and its successors have become the most popular UART design in the PC industry, mainly due to its ability to reliably handle higher data rates on operating systems with sluggish interrupt response times. + + NS16C552:: + This component consists of two NS16C550A CMOS UARTs in a single package. + + PC16550D:: + Same as NS16550A with subtle flaws corrected. + This is revision D of the 16550 family and is the latest design available from National Semiconductor. + + ==== The NS16550AF and the PC16550D are the same thing + + National reorganized their part numbering system a few years ago, and the NS16550AFN no longer exists by that name. + (If you have a NS16550AFN, look at the date code on the part, which is a four digit number that usually starts with a nine. + The first two digits of the number are the year, and the last two digits are the week in that year when the part was packaged. + If you have a NS16550AFN, it is probably a few years old.) + + The new numbers are like PC16550DV, with minor differences in the suffix letters depending on the package material and its shape. + (A description of the numbering system can be found below.) + + It is important to understand that in some stores, you may pay $15(US) for a NS16550AFN made in 1990 and in the next bin are the new PC16550DN parts with minor fixes that National has made since the AFN part was in production, the PC16550DN was probably made in the past six months and it costs half (as low as $5(US) in volume) as much as the NS16550AFN because they are readily available. + + As the supply of NS16550AFN chips continues to shrink, the price will probably continue to increase until more people discover and accept that the PC16550DN really has the same function as the old part number. + + ==== National Semiconductor Part Numbering System + + The older NS``__nnnnnrqp__`` part numbers are now of the format PC``__nnnnnrgp__``. + + The `_r_` is the revision field. The current revision of the 16550 from National Semiconductor is `D`. + + The `_p_` is the package-type field. The types are: + + [.informaltable] + [cols="1,1,1", frame="none"] + |=== + + |"F" + |QFP + |(quad flat pack) L lead type + + |"N" + |DIP + |(dual inline package) through hole straight lead type + + |"V" + |LPCC + |(lead plastic chip carrier) J lead type + |=== + + The _g_ is the product grade field. + If an `I` precedes the package-type letter, it indicates an "industrial" grade part, which has higher specs than a standard part but not as high as Military Specification (Milspec) component. + This is an optional field. + + So what we used to call a NS16550AFN (DIP Package) is now called a PC16550DN or PC16550DIN. + + === Other Vendors and Similar UARTs + + Over the years, the 8250, 8250A, 16450 and 16550 have been licensed or copied by other chip vendors. + In the case of the 8250, 8250A and 16450, the exact circuit (the "megacell") was licensed to many vendors, including Western Digital and Intel. + Other vendors reverse-engineered the part or produced emulations that had similar behavior. + + In internal modems, the modem designer will frequently emulate the 8250A/16450 with the modem microprocessor, and the emulated UART will frequently have a hidden buffer consisting of several hundred bytes. + Due to the size of the buffer, these emulations can be as reliable as a 16550A in their ability to handle high speed data. + However, most operating systems will still report that the UART is only a 8250A or 16450, and may not make effective use of the extra buffering present in the emulated UART unless special drivers are used. + + Some modem makers are driven by market forces to abandon a design that has hundreds of bytes of buffer and instead use a 16550A UART so that the product will compare favorably in market comparisons even though the effective performance may be lowered by this action. + + A common misconception is that all parts with "16550A" written on them are identical in performance. + There are differences, and in some cases, outright flaws in most of these 16550A clones. + + When the NS16550 was developed, the National Semiconductor obtained several patents on the design and they also limited licensing, making it harder for other vendors to provide a chip with similar features. + As a result of the patents, reverse-engineered designs and emulations had to avoid infringing the claims covered by the patents. + Subsequently, these copies almost never perform exactly the same as the NS16550A or PC16550D, which are the parts most computer and modem makers want to buy but are sometimes unwilling to pay the price required to get the genuine part. + + Some of the differences in the clone 16550A parts are unimportant, while others can prevent the device from being used at all with a given operating system or driver. + These differences may show up when using other drivers, or when particular combinations of events occur that were not well tested or considered in the Windows(R) driver. + This is because most modem vendors and 16550-clone makers use the Microsoft drivers from Windows(R) for Workgroups 3.11 and the Microsoft(R) MS-DOS(R) utility as the primary tests for compatibility with the NS16550A. + This over-simplistic criteria means that if a different operating system is used, problems could appear due to subtle differences between the clones and genuine components. + + National Semiconductor has made available a program named COMTEST that performs compatibility tests independent of any OS drivers. + It should be remembered that the purpose of this type of program is to demonstrate the flaws in the products of the competition, so the program will report major as well as extremely subtle differences in behavior in the part being tested. + + In a series of tests performed by the author of this document in 1994, components made by National Semiconductor, TI, StarTech, and CMD as well as megacells and emulations embedded in internal modems were tested with COMTEST. A difference count for some of these components is listed below. + Since these tests were performed in 1994, they may not reflect the current performance of the given product from a vendor. + + It should be noted that COMTEST normally aborts when an excessive number or certain types of problems have been detected. + As part of this testing, COMTEST was modified so that it would not abort no matter how many differences were encountered. + + [.informaltable] + [cols="1,1,1", frame="none", options="header"] + |=== + | Vendor + | Part Number + | Errors (aka "differences" reported) + + |National + |(PC16550DV) + |0 + + |National + |(NS16550AFN) + |0 + + |National + |(NS16C552V) + |0 + + |TI + |(TL16550AFN) + |3 + + |CMD + |(16C550PE) + |19 + + |StarTech + |(ST16C550J) + |23 + + |Rockwell + |Reference modem with internal 16550 or an emulation (RC144DPi/C3000-25) + |117 + + |Sierra + |Modem with an internal 16550 (SC11951/SC11351) + |91 + |=== + + [NOTE] + ==== + To date, the author of this document has not found any non-National parts that report zero differences using the COMTEST program. + It should also be noted that National has had five versions of the 16550 over the years and the newest parts behave a bit differently than the classic NS16550AFN that is considered the benchmark for functionality. + COMTEST appears to turn a blind eye to the differences within the National product line and reports no errors on the National parts (except for the original 16550) even when there are official erratas that describe bugs in the A, B and C revisions of the parts, so this bias in COMTEST must be taken into account. + ==== + + It is important to understand that a simple count of differences from COMTEST does not reveal a lot about what differences are important and which are not. + For example, about half of the differences reported in the two modems listed above that have internal UARTs were caused by the clone UARTs not supporting five- and six-bit character modes. + The real 16550, 16450, and 8250 UARTs all support these modes and COMTEST checks the functionality of these modes so over fifty differences are reported. + However, almost no modern modem supports five- or six-bit characters, particularly those with error-correction and compression capabilities. + This means that the differences related to five- and six-bit character modes can be discounted. + + Many of the differences COMTEST reports have to do with timing. + In many of the clone designs, when the host reads from one port, the status bits in some other port may not update in the same amount of time (some faster, some slower) as a _real_ NS16550AFN and COMTEST looks for these differences. + This means that the number of differences can be misleading in that one device may only have one or two differences but they are extremely serious, and some other device that updates the status registers faster or slower than the reference part (that would probably never affect the operation of a properly written driver) could have dozens of differences reported. + + COMTEST can be used as a screening tool to alert the administrator to the presence of potentially incompatible components that might cause problems or have to be handled as a special case. + + If you run COMTEST on a 16550 that is in a modem or a modem is attached to the serial port, you need to first issue a ATE0&W command to the modem so that the modem will not echo any of the test characters. + If you forget to do this, COMTEST will report at least this one difference: + + [source,shell] + .... + Error (6)...Timeout interrupt failed: IIR = c1 LSR = 61 + .... + + === 8250/16450/16550 Registers + + The 8250/16450/16550 UART occupies eight contiguous I/O port addresses. + In the IBM PC, there are two defined locations for these eight ports and they are known collectively as [.filename]#COM1# and [.filename]#COM2#. + The makers of PC-clones and add-on cards have created two additional areas known as [.filename]#COM3# and [.filename]#COM4#, but these extra COM ports conflict with other hardware on some systems. + The most common conflict is with video adapters that provide IBM 8514 emulation. + + [.filename]#COM1# is located from 0x3f8 to 0x3ff and normally uses IRQ 4. + [.filename]#COM2# is located from 0x2f8 to 0x2ff and normally uses IRQ 3. + [.filename]#COM3# is located from 0x3e8 to 0x3ef and has no standardized IRQ. + [.filename]#COM4# is located from 0x2e8 to 0x2ef and has no standardized IRQ. + + A description of the I/O ports of the 8250/16450/16550 UART is provided below. + + [.informaltable] + [cols="10%,10%,80%", frame="none", options="header"] + |=== + | I/O Port + | Access Allowed + | Description + + |+0x00 + |write (DLAB==0) + | + + Transmit Holding Register (THR). + + Information written to this port are treated as data words and will be transmitted by the UART. + + |+0x00 + |read (DLAB==0) + | + + Receive Buffer Register (RBR). + + Any data words received by the UART form the serial link are accessed by the host by reading this port. + + |+0x00 + |write/read (DLAB==1) + | + + Divisor Latch LSB (DLL) + + This value will be divided from the master input clock (in the IBM PC, the master clock is 1.8432MHz) and the resulting clock will determine the baud rate of the UART. This register holds bits 0 thru 7 of the divisor. + + |+0x01 + |write/read (DLAB==1) + | + + Divisor Latch MSB (DLH) + + This value will be divided from the master input clock (in the IBM PC, the master clock is 1.8432MHz) and the resulting clock will determine the baud rate of the UART. This register holds bits 8 thru 15 of the divisor. + + |+0x01 + |write/read (DLAB==0) + |Interrupt Enable Register (IER) + + + The 8250/16450/16550 UART classifies events into one of four categories. Each category can be configured to generate an interrupt when any of the events occurs. The 8250/16450/16550 UART generates a single external interrupt signal regardless of how many events in the enabled categories have occurred. It is up to the host processor to respond to the interrupt and then poll the enabled interrupt categories (usually all categories have interrupts enabled) to determine the true cause(s) of the interrupt. + + Bit 7 -> Reserved, always 0. + + Bit 6 -> Reserved, always 0. + + Bit 5 -> Reserved, always 0. + + Bit 4 -> Reserved, always 0. + + Bit 3 -> Enable Modem Status Interrupt (EDSSI). Setting this bit to "1" allows the UART to generate an interrupt when a change occurs on one or more of the status lines. + + Bit 2 -> Enable Receiver Line Status Interrupt (ELSI) Setting this bit to "1" causes the UART to generate an interrupt when the an error (or a BREAK signal) has been detected in the incoming data. + + Bit 1 -> Enable Transmitter Holding Register Empty Interrupt (ETBEI) Setting this bit to "1" causes the UART to generate an interrupt when the UART has room for one or more additional characters that are to be transmitted. + + Bit 0 -> Enable Received Data Available Interrupt (ERBFI) Setting this bit to "1" causes the UART to generate an interrupt when the UART has received enough characters to exceed the trigger level of the FIFO, or the FIFO timer has expired (stale data), or a single character has been received when the FIFO is disabled. + + |+0x02 + |write + |FIFO Control Register (FCR) (This port does not exist on the 8250 and 16450 UART.) + + Bit 7 -> Receiver Trigger Bit #1 + + Bit 6 -> Receiver Trigger Bit #0 + + + These two bits control at what point the receiver is to generate an interrupt when the FIFO is active. + + 7 6 How many words are received before an interrupt is generated + + 0 0 1 + + 0 1 4 + + 1 0 8 + + 1 1 14 + + Bit 5 -> Reserved, always 0. + + Bit 4 -> Reserved, always 0. + + Bit 3 -> DMA Mode Select. If Bit 0 is set to "1" (FIFOs enabled), setting this bit changes the operation of the -RXRDY and -TXRDY signals from Mode 0 to Mode 1. + + Bit 2 -> Transmit FIFO Reset. When a "1" is written to this bit, the contents of the FIFO are discarded. Any word currently being transmitted will be sent intact. This function is useful in aborting transfers. + + Bit 1 -> Receiver FIFO Reset. When a "1" is written to this bit, the contents of the FIFO are discarded. Any word currently being assembled in the shift register will be received intact. + + Bit 0 -> 16550 FIFO Enable. When set, both the transmit and receive FIFOs are enabled. Any contents in the holding register, shift registers or FIFOs are lost when FIFOs are enabled or disabled. + + + |+0x02 + |read + |Interrupt Identification Register + + Bit 7 -> FIFOs enabled. On the 8250/16450 UART, this bit is zero. + + Bit 6 -> FIFOs enabled. On the 8250/16450 UART, this bit is zero. + + Bit 5 -> Reserved, always 0. + + Bit 4 -> Reserved, always 0. + + Bit 3 -> Interrupt ID Bit #2. On the 8250/16450 UART, this bit is zero. + + Bit 2 -> Interrupt ID Bit #1 + + Bit 1 -> Interrupt ID Bit #0.These three bits combine to report the category of event that caused the interrupt that is in progress. These categories have priorities, so if multiple categories of events occur at the same time, the UART will report the more important events first and the host must resolve the events in the order they are reported. All events that caused the current interrupt must be resolved before any new interrupts will be generated. (This is a limitation of the PC architecture.) + + 2 1 0 Priority Description + + 0 1 1 First Received Error (OE, PE, BI, or FE) + + 0 1 0 Second Received Data Available + + 1 1 0 Second Trigger level identification (Stale data in receive buffer) + + 0 0 1 Third Transmitter has room for more words (THRE) + + 0 0 0 Fourth Modem Status Change (-CTS, -DSR, -RI, or -DCD) + + Bit 0 -> Interrupt Pending Bit. If this bit is set to "0", then at least one interrupt is pending. + + |+0x03 + |write/read + |Line Control Register (LCR) + + Bit 7 -> Divisor Latch Access Bit (DLAB). When set, access to the data transmit/receive register (THR/RBR) and the Interrupt Enable Register (IER) is disabled. Any access to these ports is now redirected to the Divisor Latch Registers. Setting this bit, loading the Divisor Registers, and clearing DLAB should be done with interrupts disabled. + + Bit 6 -> Set Break. When set to "1", the transmitter begins to transmit continuous Spacing until this bit is set to "0". This overrides any bits of characters that are being transmitted. + + Bit 5 -> Stick Parity. When parity is enabled, setting this bit causes parity to always be "1" or "0", based on the value of Bit 4. + Bit 4 -> Even Parity Select (EPS). When parity is enabled and Bit 5 is "0", setting this bit causes even parity to be transmitted and expected. Otherwise, odd parity is used. + + Bit 3 -> Parity Enable (PEN). When set to "1", a parity bit is inserted between the last bit of the data and the Stop Bit. The UART will also expect parity to be present in the received data. + + Bit 2 -> Number of Stop Bits (STB). If set to "1" and using 5-bit data words, 1.5 Stop Bits are transmitted and expected in each data word. For 6, 7 and 8-bit data words, 2 Stop Bits are transmitted and expected. When this bit is set to "0", one Stop Bit is used on each data word. + + Bit 1 -> Word Length Select Bit #1 (WLSB1) + + Bit 0 -> Word Length Select Bit #0 (WLSB0) + + Together these bits specify the number of bits in each data word. + + 1 0 Word Length + + 0 0 5 Data Bits + + 0 1 6 Data Bits + + 1 0 7 Data Bits + + 1 1 8 Data Bits + + + |+0x04 + |write/read + |Modem Control Register (MCR) + + Bit 7 -> Reserved, always 0. + + Bit 6 -> Reserved, always 0. + + Bit 5 -> Reserved, always 0. + + Bit 4 -> Loop-Back Enable. When set to "1", the UART transmitter and receiver are internally connected together to allow diagnostic operations. In addition, the UART modem control outputs are connected to the UART modem control inputs. CTS is connected to RTS, DTR is connected to DSR, OUT1 is connected to RI, and OUT 2 is connected to DCD. + + Bit 3 -> OUT 2. An auxiliary output that the host processor may set high or low. In the IBM PC serial adapter (and most clones), OUT 2 is used to tri-state (disable) the interrupt signal from the 8250/16450/16550 UART. + + Bit 2 -> OUT 1. An auxiliary output that the host processor may set high or low. This output is not used on the IBM PC serial adapter. + + Bit 1 -> Request to Send (RTS). When set to "1", the output of the UART -RTS line is Low (Active). + + Bit 0 -> Data Terminal Ready (DTR). When set to "1", the output of the UART -DTR line is Low (Active). + + + |+0x05 + |write/read + |Line Status Register (LSR) + + Bit 7 -> Error in Receiver FIFO. On the 8250/16450 UART, this bit is zero. This bit is set to "1" when any of the bytes in the FIFO have one or more of the following error conditions: PE, FE, or BI. + + Bit 6 -> Transmitter Empty (TEMT). When set to "1", there are no words remaining in the transmit FIFO or the transmit shift register. The transmitter is completely idle. + + Bit 5 -> Transmitter Holding Register Empty (THRE). When set to "1", the FIFO (or holding register) now has room for at least one additional word to transmit. The transmitter may still be transmitting when this bit is set to "1". + + Bit 4 -> Break Interrupt (BI). The receiver has detected a Break signal. + + Bit 3 -> Framing Error (FE). A Start Bit was detected but the Stop Bit did not appear at the expected time. The received word is probably garbled. + + Bit 2 -> Parity Error (PE). The parity bit was incorrect for the word received. + + Bit 1 -> Overrun Error (OE). A new word was received and there was no room in the receive buffer. The newly-arrived word in the shift register is discarded. On 8250/16450 UARTs, the word in the holding register is discarded and the newly- arrived word is put in the holding register. + + Bit 0 -> Data Ready (DR) One or more words are in the receive FIFO that the host may read. A word must be completely received and moved from the shift register into the FIFO (or holding register for 8250/16450 designs) before this bit is set. + + |+0x06 + |write/read + |Modem Status Register (MSR) + + Bit 7 -> Data Carrier Detect (DCD). Reflects the state of the DCD line on the UART. + + Bit 6 -> Ring Indicator (RI). Reflects the state of the RI line on the UART. + + Bit 5 -> Data Set Ready (DSR). Reflects the state of the DSR line on the UART. + + Bit 4 -> Clear To Send (CTS). Reflects the state of the CTS line on the UART. + + Bit 3 -> Delta Data Carrier Detect (DDCD). Set to "1" if the -DCD line has changed state one more time since the last time the MSR was read by the host. + + Bit 2 -> Trailing Edge Ring Indicator (TERI). Set to "1" if the -RI line has had a low to high transition since the last time the MSR was read by the host. + + Bit 1 -> Delta Data Set Ready (DDSR). Set to "1" if the -DSR line has changed state one more time since the last time the MSR was read by the host. + + Bit 0 -> Delta Clear To Send (DCTS). Set to "1" if the -CTS line has changed state one more time since the last time the MSR was read by the host. + + |+0x07 + |write/read + |Scratch Register (SCR). This register performs no function in the UART. Any value can be written by the host to this location and read by the host later on. + |=== + + === Beyond the 16550A UART + +-Although National Semiconductor has not offered any components compatible with the 16550 that provide additional features, various other vendors have. ++Although National Semiconductor has not offered any components compatible with the 16550 that provide additional features, various other vendors have. + Some of these components are described below. + It should be understood that to effectively utilize these improvements, drivers may have to be provided by the chip vendor since most of the popular operating systems do not support features beyond those provided by the 16550. + + ST16650:: + By default this part is similar to the NS16550A, but an extended 32-byte send and receive buffer can be optionally enabled. + Made by StarTech. + + TIL16660:: + By default this part behaves similar to the NS16550A, but an extended 64-byte send and receive buffer can be optionally enabled. + Made by Texas Instruments. + + Hayes ESP:: + This proprietary plug-in card contains a 2048-byte send and receive buffer, and supports data rates to 230.4Kbit/sec. + Made by Hayes. + + In addition to these "dumb" UARTs, many vendors produce intelligent serial communication boards. + This type of design usually provides a microprocessor that interfaces with several UARTs, processes and buffers the data, and then alerts the main PC processor when necessary. + As the UARTs are not directly accessed by the PC processor in this type of communication system, it is not necessary for the vendor to use UARTs that are compatible with the 8250, 16450, or the 16550 UART. + This leaves the designer free to components that may have better performance characteristics. + + [[sio]] + == Configuring the [.filename]#sio# driver + +-The [.filename]#sio# driver provides support for NS8250-, NS16450-, NS16550 and NS16550A-based EIA RS-232C (CCITT V.24) communications interfaces. ++The [.filename]#sio# driver provides support for NS8250-, NS16450-, NS16550 and NS16550A-based EIA RS-232C (CCITT V.24) communications interfaces. + Several multiport cards are supported as well. + See the man:sio[4] manual page for detailed technical documentation. + + === Digi International (DigiBoard) PC/8 + + _Contributed by `{awebster}`. 26 August 1995._ + + Here is a config snippet from a machine with a Digi International PC/8 with 16550. + It has 8 modems connected to these 8 lines, and they work just great. + Do not forget to add `options COM_MULTIPORT` or it will not work very well! + + [.programlisting] + .... + device sio4 at isa? port 0x100 flags 0xb05 + device sio5 at isa? port 0x108 flags 0xb05 + device sio6 at isa? port 0x110 flags 0xb05 + device sio7 at isa? port 0x118 flags 0xb05 + device sio8 at isa? port 0x120 flags 0xb05 + device sio9 at isa? port 0x128 flags 0xb05 + device sio10 at isa? port 0x130 flags 0xb05 + device sio11 at isa? port 0x138 flags 0xb05 irq 9 + .... + + The trick in setting this up is that the MSB of the flags represent the last SIO port, in this case 11 so flags are 0xb05. + + === Boca 16 + + _Contributed by `{whiteside}`. 26 August 1995._ + + The procedures to make a Boca 16 port board with FreeBSD are pretty straightforward, but you will need a couple things to make it work: + + . You either need the kernel sources installed so you can recompile the necessary options or you will need someone else to compile it for you. The 2.0.5 default kernel does _not_ come with multiport support enabled and you will need to add a device entry for each port anyways. + . Two, you will need to know the interrupt and IO setting for your Boca Board so you can set these options properly in the kernel. + + One important note - the actual UART chips for the Boca 16 are in the connector box, not on the internal board itself. + So if you have it unplugged, probes of those ports will fail. + I have never tested booting with the box unplugged and plugging it back in, and I suggest you do not either. + +-If you do not already have a custom kernel configuration file set up, refer to extref:{handbook}[Kernel Configuration, kernelconfig] chapter of the FreeBSD Handbook for general procedures. ++If you do not already have a custom kernel configuration file set up, refer to extref:{handbook}kernelconfig[Kernel Configuration, kernelconfig] chapter of the FreeBSD Handbook for general procedures. + The following are the specifics for the Boca 16 board and assume you are using the kernel name MYKERNEL and editing with vi. + + [.procedure] + ==== +-. Add the line ++. Add the line + + + [.programlisting] + .... + options COM_MULTIPORT + .... + to the config file. + . Where the current `device sio__n__` lines are, you will need to add 16 more devices. The following example is for a Boca Board with an interrupt of 3, and a base IO address 100h. The IO address for Each port is +8 hexadecimal from the previous port, thus the 100h, 108h, 110h... addresses. + + + [.programlisting] + .... + device sio1 at isa? port 0x100 flags 0x1005 + device sio2 at isa? port 0x108 flags 0x1005 + device sio3 at isa? port 0x110 flags 0x1005 + device sio4 at isa? port 0x118 flags 0x1005 + ... + device sio15 at isa? port 0x170 flags 0x1005 + device sio16 at isa? port 0x178 flags 0x1005 irq 3 + .... + + + The flags entry _must_ be changed from this example unless you are using the exact same sio assignments. + Flags are set according to 0x``__MYY__`` where _M_ indicates the minor number of the master port (the last port on a Boca 16) and _YY_ indicates if FIFO is enabled or disabled(enabled), IRQ sharing is used(yes) and if there is an AST/4 compatible IRQ control register(no). +-In this example, ++In this example, + + + [.programlisting] + .... + flags + 0x1005 + .... + indicates that the master port is sio16. + If I added another board and assigned sio17 through sio28, the flags for all 16 ports on _that_ board would be 0x1C05, where 1C indicates the minor number of the master port. + Do not change the 05 setting. + . Save and complete the kernel configuration, recompile, install and reboot. Presuming you have successfully installed the recompiled kernel and have it set to the correct address and IRQ, your boot message should indicate the successful probe of the Boca ports as follows: (obviously the sio numbers, IO and IRQ could be different) + + + [source,shell] + .... + sio1 at 0x100-0x107 flags 0x1005 on isa + sio1: type 16550A (multiport) + sio2 at 0x108-0x10f flags 0x1005 on isa + sio2: type 16550A (multiport) + sio3 at 0x110-0x117 flags 0x1005 on isa + sio3: type 16550A (multiport) + sio4 at 0x118-0x11f flags 0x1005 on isa + sio4: type 16550A (multiport) + sio5 at 0x120-0x127 flags 0x1005 on isa + sio5: type 16550A (multiport) + sio6 at 0x128-0x12f flags 0x1005 on isa + sio6: type 16550A (multiport) + sio7 at 0x130-0x137 flags 0x1005 on isa + sio7: type 16550A (multiport) + sio8 at 0x138-0x13f flags 0x1005 on isa + sio8: type 16550A (multiport) + sio9 at 0x140-0x147 flags 0x1005 on isa + sio9: type 16550A (multiport) + sio10 at 0x148-0x14f flags 0x1005 on isa + sio10: type 16550A (multiport) + sio11 at 0x150-0x157 flags 0x1005 on isa + sio11: type 16550A (multiport) + sio12 at 0x158-0x15f flags 0x1005 on isa + sio12: type 16550A (multiport) + sio13 at 0x160-0x167 flags 0x1005 on isa + sio13: type 16550A (multiport) + sio14 at 0x168-0x16f flags 0x1005 on isa + sio14: type 16550A (multiport) + sio15 at 0x170-0x177 flags 0x1005 on isa + sio15: type 16550A (multiport) + sio16 at 0x178-0x17f irq 3 flags 0x1005 on isa + sio16: type 16550A (multiport master) + .... + + +-If the messages go by too fast to see, ++If the messages go by too fast to see, + + + [source,shell] + .... + # dmesg | more + .... + will show you the boot messages. + . Next, appropriate entries in [.filename]#/dev# for the devices must be made using the [.filename]#/dev/MAKEDEV# script. This step can be omitted if you are running FreeBSD 5.X with a kernel that has man:devfs[5] support compiled in. + + + If you do need to create the [.filename]#/dev# entries, run the following as `root`: + + + [source,shell] + .... + # cd /dev + # ./MAKEDEV tty1 + # ./MAKEDEV cua1 + + (everything in between) + # ./MAKEDEV ttyg + # ./MAKEDEV cuag + .... + + + If you do not want or need call-out devices for some reason, you can dispense with making the [.filename]#cua*# devices. +-. If you want a quick and sloppy way to make sure the devices are working, you can simply plug a modem into each port and (as root) ++. If you want a quick and sloppy way to make sure the devices are working, you can simply plug a modem into each port and (as root) + + + [source,shell] + .... + # echo at > ttyd* + .... + for each device you have made. You _should_ see the RX lights flash for each working port. + ==== + + === Support for Cheap Multi-UART Cards + + _Contributed by Helge Oldach_ mailto:hmo@sep.hamburg.com[hmo@sep.hamburg.com], September 1999 + + Ever wondered about FreeBSD support for your 20$ multi-I/O card with two (or more) COM ports, sharing IRQs? Here is how: + + Usually the only option to support these kind of boards is to use a distinct IRQ for each port. + For example, if your CPU board has an on-board [.filename]#COM1# port (aka [.filename]#sio0#-I/O address 0x3F8 and IRQ 4) and you have an extension board with two UARTs, you will commonly need to configure them as [.filename]#COM2# (aka [.filename]#sio1#-I/O address 0x2F8 and IRQ 3), and the third port (aka [.filename]#sio2#) as I/O 0x3E8 and IRQ 5. + Obviously this is a waste of IRQ resources, as it should be basically possible to run both extension board ports using a single IRQ with the `COM_MULTIPORT` configuration described in the previous sections. + + Such cheap I/O boards commonly have a 4 by 3 jumper matrix for the COM ports, similar to the following: + + [.programlisting] + .... + o o o * + Port A | + o * o * + Port B | + o * o o + IRQ 2 3 4 5 + .... + + Shown here is port A wired for IRQ 5 and port B wired for IRQ 3. + The IRQ columns on your specific board may vary-other boards may supply jumpers for IRQs 3, 4, 5, and 7 instead. + + One could conclude that wiring both ports for IRQ 3 using a handcrafted wire-made jumper covering all three connection points in the IRQ 3 column would solve the issue, but no. + You cannot duplicate IRQ 3 because the output drivers of each UART are wired in a "totem pole" fashion, so if one of the UARTs drives IRQ 3, the output signal will not be what you would expect. + Depending on the implementation of the extension board or your motherboard, the IRQ 3 line will continuously stay up, or always stay low. + + You need to decouple the IRQ drivers for the two UARTs, so that the IRQ line of the board only goes up if (and only if) one of the UARTs asserts a IRQ, and stays low otherwise. + The solution was proposed by Joerg Wunsch mailto:j@ida.interface-business.de[j@ida.interface-business.de]: To solder up a wired-or consisting of two diodes (Germanium or Schottky-types strongly preferred) and a 1 kOhm resistor. + Here is the schematic, starting from the 4 by 3 jumper field above: + + [.programlisting] + .... + Diode + +---------->|-------+ + / | + o * o o | 1 kOhm + Port A +----|######|-------+ + o * o o | | + Port B `-------------------+ ==+== + o * o o | Ground + \ | + +--------->|-------+ + IRQ 2 3 4 5 Diode + .... + + The cathodes of the diodes are connected to a common point, together with a 1 kOhm pull-down resistor. + It is essential to connect the resistor to ground to avoid floating of the IRQ line on the bus. + + Now we are ready to configure a kernel. + Staying with this example, we would configure: + + [.programlisting] + .... + # standard on-board COM1 port + device sio0 at isa? port "IO_COM1" flags 0x10 + # patched-up multi-I/O extension board + options COM_MULTIPORT + device sio1 at isa? port "IO_COM2" flags 0x205 + device sio2 at isa? port "IO_COM3" flags 0x205 irq 3 + .... + + Note that the `flags` setting for [.filename]#sio1# and [.filename]#sio2# is truly essential; refer to man:sio[4] for details. + (Generally, the `2` in the "flags" attribute refers to [.filename]#sio#`2` which holds the IRQ, and you surely want a `5` low nibble.) + With kernel verbose mode turned on this should yield something similar to this: + + [source,shell] + .... + sio0: irq maps: 0x1 0x11 0x1 0x1 + sio0 at 0x3f8-0x3ff irq 4 flags 0x10 on isa + sio0: type 16550A + sio1: irq maps: 0x1 0x9 0x1 0x1 + sio1 at 0x2f8-0x2ff flags 0x205 on isa + sio1: type 16550A (multiport) + sio2: irq maps: 0x1 0x9 0x1 0x1 + sio2 at 0x3e8-0x3ef irq 3 flags 0x205 on isa + sio2: type 16550A (multiport master) + .... + + Though [.filename]#/sys/i386/isa/sio.c# is somewhat cryptic with its use of the "irq maps" array above, the basic idea is that you observe `0x1` in the first, third, and fourth place. + This means that the corresponding IRQ was set upon output and cleared after, which is just what we would expect. + If your kernel does not display this behavior, most likely there is something wrong with your wiring. + + [[cy]] + == Configuring the [.filename]#cy# driver + + _Contributed by Alex Nash. 6 June 1996._ + + The Cyclades multiport cards are based on the [.filename]#cy# driver instead of the usual [.filename]#sio# driver used by other multiport cards. + Configuration is a simple matter of: + + [.procedure] + ==== + . Add the [.filename]#cy# device to your kernel configuration (note that your irq and iomem settings may differ). + + + [.programlisting] + .... + device cy0 at isa? irq 10 iomem 0xd4000 iosiz 0x2000 + .... + . Rebuild and install the new kernel. + . Make the device nodes by typing (the following example assumes an 8-port board): + + + [source,shell] + .... + # cd /dev + # for i in 0 1 2 3 4 5 6 7;do ./MAKEDEV cuac$i ttyc$i;done + .... + + . If appropriate, add dialup entries to [.filename]#/etc/ttys# by duplicating serial device (`ttyd`) entries and using `ttyc` in place of `ttyd`. For example: + + + [.programlisting] + .... + ttyc0 "/usr/libexec/getty std.38400" unknown on insecure + ttyc1 "/usr/libexec/getty std.38400" unknown on insecure + ttyc2 "/usr/libexec/getty std.38400" unknown on insecure + ... + ttyc7 "/usr/libexec/getty std.38400" unknown on insecure + .... + . Reboot with the new kernel. + ==== + + == Configuring the [.filename]#si# driver + + _Contributed by `{nsayer}`. 25 March 1998._ + + The Specialix SI/XIO and SX multiport cards use the [.filename]#si# driver. + A single machine can have up to 4 host cards. + The following host cards are supported: + + * ISA SI/XIO host card (2 versions) + * EISA SI/XIO host card + * PCI SI/XIO host card + * ISA SX host card + * PCI SX host card + + Although the SX and SI/XIO host cards look markedly different, their functionality are basically the same. + The host cards do not use I/O locations, but instead require a 32K chunk of memory. + The factory configuration for ISA cards places this at `0xd0000-0xd7fff`. + They also require an IRQ. + PCI cards will, of course, auto-configure themselves. + + You can attach up to 4 external modules to each host card. + The external modules contain either 4 or 8 serial ports. + They come in the following varieties: + + * SI 4 or 8 port modules. Up to 57600 bps on each port supported. + * XIO 8 port modules. Up to 115200 bps on each port supported. One type of XIO module has 7 serial and 1 parallel port. + * SXDC 8 port modules. Up to 921600 bps on each port supported. Like XIO, a module is available with one parallel port as well. + + To configure an ISA host card, add the following line to your kernel configuration file, changing the numbers as appropriate: + + [.programlisting] + .... + device si0 at isa? iomem 0xd0000 irq 11 + .... + + Valid IRQ numbers are 9, 10, 11, 12 and 15 for SX ISA host cards and 11, 12 and 15 for SI/XIO ISA host cards. + + To configure an EISA or PCI host card, use this line: + + [.programlisting] + .... + device si0 + .... + + After adding the configuration entry, rebuild and install your new kernel. + + [NOTE] + ==== + The following step, is not necessary if you are using man:devfs[5] in FreeBSD 5._X_. + ==== + + After rebooting with the new kernel, you need to make the device nodes in [.filename]#/dev#. + The [.filename]#MAKEDEV# script will take care of this for you. + Count how many total ports you have and type: + + [source,shell] + .... + # cd /dev + # ./MAKEDEV ttyAnn cuaAnn + .... + + (where _nn_ is the number of ports) + + If you want login prompts to appear on these ports, you will need to add lines like this to [.filename]#/etc/ttys#: + + [.programlisting] + .... + ttyA01 "/usr/libexec/getty std.9600" vt100 on insecure + .... + + Change the terminal type as appropriate. + For modems, `dialup` or `unknown` is fine. +diff --git a/documentation/content/en/articles/vinum/_index.adoc b/documentation/content/en/articles/vinum/_index.adoc +index e1c87acf48..6c078fa1c6 100644 +--- a/documentation/content/en/articles/vinum/_index.adoc ++++ b/documentation/content/en/articles/vinum/_index.adoc +@@ -1,723 +1,723 @@ + --- + title: The vinum Volume Manager + authors: + - author: Greg Lehey + description: The vinum Volume Manager in FreeBSD + tags: ["vinum", "Volume Manager", "FreeBSD"] + --- + + //// + The Vinum Volume Manager + By Greg Lehey (grog at lemis dot com) + + Added to the Handbook by Hiten Pandya + and Tom Rhodes + + For the FreeBSD Documentation Project + //// + + = The vinum Volume Manager + :doctype: article + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :source-highlighter: rouge + :experimental: + :images-path: articles/vinum/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :imagesdir: ../../../images/{images-path} + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + ''' + + toc::[] + + [[vinum-synopsis]] + == Synopsis + + No matter the type of disks, there are always potential problems. + The disks can be too small, too slow, or too unreliable to meet the system's requirements. + While disks are getting bigger, so are data storage requirements. + Often a file system is needed that is bigger than a disk's capacity. + Various solutions to these problems have been proposed and implemented. + + One method is through the use of multiple, and sometimes redundant, disks. + In addition to supporting various cards and controllers for hardware Redundant Array of Independent Disks RAID systems, the base FreeBSD system includes the [.filename]#vinum# volume manager, a block device driver that implements virtual disk drives and addresses these three problems. + [.filename]#vinum# provides more flexibility, performance, and reliability than traditional disk storage and implements `RAID`-0, `RAID`-1, and `RAID`-5 models, both individually and in combination. + + This chapter provides an overview of potential problems with traditional disk storage, and an introduction to the [.filename]#vinum# volume manager. + + [WARNING] + ==== + vinum is deprecated and is not present in FreeBSD 15.0 and later. + Users are advised to migrate to man:gconcat[8], man:gmirror[8], man:gstripe[8], man:graid[8], or man:zfs[8]. + ==== + + [NOTE] + ==== +-Starting with FreeBSD 5, [.filename]#vinum# has been rewritten to fit into the extref:{handbook}[GEOM architecture, geom], while retaining the original ideas, terminology, and on-disk metadata. ++Starting with FreeBSD 5, [.filename]#vinum# has been rewritten to fit into the extref:{handbook}geom[GEOM architecture, geom], while retaining the original ideas, terminology, and on-disk metadata. + This rewrite is called _gvinum_ (for _GEOM vinum_). + While this chapter uses the term [.filename]#vinum#, any command invocations should be performed with `gvinum`. + The name of the kernel module has changed from the original [.filename]#vinum.ko# to [.filename]#geom_vinum.ko#, and all device nodes reside under [.filename]#/dev/gvinum# instead of [.filename]#/dev/vinum#. + As of FreeBSD 6, the original [.filename]#vinum# implementation is no longer available in the code base. + ==== + + [[vinum-access-bottlenecks]] + == Access Bottlenecks + + Modern systems frequently need to access data in a highly concurrent manner. + For example, large FTP or HTTP servers can maintain thousands of concurrent sessions and have multiple 100 Mbit/s connections to the outside world, well beyond the sustained transfer rate of most disks. + + Current disk drives can transfer data sequentially at up to 70 MB/s, but this value is of little importance in an environment where many independent processes access a drive, and where they may achieve only a fraction of these values. + In such cases, it is more interesting to view the problem from the viewpoint of the disk subsystem. + The important parameter is the load that a transfer places on the subsystem, or the time for which a transfer occupies the drives involved in the transfer. + + In any disk transfer, the drive must first position the heads, wait for the first sector to pass under the read head, and then perform the transfer. + These actions can be considered to be atomic as it does not make any sense to interrupt them. + + [[vinum-latency]] Consider a typical transfer of about 10 kB: the current generation of high-performance disks can position the heads in an average of 3.5 ms. + The fastest drives spin at 15,000 rpm, so the average rotational latency (half a revolution) is 2 ms. + At 70 MB/s, the transfer itself takes about 150 μs, almost nothing compared to the positioning time. + In such a case, the effective transfer rate drops to a little over 1 MB/s and is clearly highly dependent on the transfer size. + + The traditional and obvious solution to this bottleneck is "more spindles": rather than using one large disk, use several smaller disks with the same aggregate storage space. + Each disk is capable of positioning and transferring independently, so the effective throughput increases by a factor close to the number of disks used. + + The actual throughput improvement is smaller than the number of disks involved. + Although each drive is capable of transferring in parallel, there is no way to ensure that the requests are evenly distributed across the drives. + Inevitably the load on one drive will be higher than on another. + + The evenness of the load on the disks is strongly dependent on the way the data is shared across the drives. + In the following discussion, it is convenient to think of the disk storage as a large number of data sectors which are addressable by number, rather like the pages in a book. + The most obvious method is to divide the virtual disk into groups of consecutive sectors the size of the individual physical disks and store them in this manner, rather like taking a large book and tearing it into smaller sections. + This method is called _concatenation_ and has the advantage that the disks are not required to have any specific size relationships. + It works well when the access to the virtual disk is spread evenly about its address space. + When access is concentrated on a smaller area, the improvement is less marked. + crossref:vinum[vinum-concat, Concatenated Organization] illustrates the sequence in which storage units are allocated in a concatenated organization. + + [[vinum-concat]] + .Concatenated Organization + image::vinum-concat.png[] + + An alternative mapping is to divide the address space into smaller, equal-sized components and store them sequentially on different devices. + For example, the first 256 sectors may be stored on the first disk, the next 256 sectors on the next disk and so on. + After filling the last disk, the process repeats until the disks are full. + This mapping is called _striping_ or RAID-0. + + `RAID` offers various forms of fault tolerance, though RAID-0 is somewhat misleading as it provides no redundancy. + Striping requires somewhat more effort to locate the data, and it can cause additional I/O load where a transfer is spread over multiple disks, but it can also provide a more constant load across the disks. + crossref:vinum[vinum-striped, Striped Organization] illustrates the sequence in which storage units are allocated in a striped organization. + + [[vinum-striped]] + .Striped Organization + image::vinum-striped.png[] + + [[vinum-data-integrity]] + == Data Integrity + + The final problem with disks is that they are unreliable. + Although reliability has increased tremendously over the last few years, disk drives are still the most likely core component of a server to fail. + When they do, the results can be catastrophic and replacing a failed disk drive and restoring data can result in server downtime. + + One approach to this problem is _mirroring_, or `RAID-1`, which keeps two copies of the data on different physical hardware. + Any write to the volume writes to both disks; a read can be satisfied from either, so if one drive fails, the data is still available on the other drive. + + Mirroring has two problems: + + * It requires twice as much disk storage as a non-redundant solution. + * Writes must be performed to both drives, so they take up twice the bandwidth of a non-mirrored volume. Reads do not suffer from a performance penalty and can even be faster. + + An alternative solution is _parity_, implemented in `RAID` levels 2, 3, 4 and 5. + Of these, `RAID-5` is the most interesting. + As implemented in [.filename]#vinum#, it is a variant on a striped organization which dedicates one block of each stripe to parity one of the other blocks. + As implemented by [.filename]#vinum#, a `RAID-5` plex is similar to a striped plex, except that it implements `RAID-5` by including a parity block in each stripe. + As required by `RAID-5`, the location of this parity block changes from one stripe to the next. + The numbers in the data blocks indicate the relative block numbers. + + [[vinum-raid5-org]] + .`RAID`-5 Organization + image::vinum-raid5-org.png[] + + Compared to mirroring, `RAID-5` has the advantage of requiring significantly less storage space. + Read access is similar to that of striped organizations, but write access is significantly slower, approximately 25% of the read performance. + If one drive fails, the array can continue to operate in degraded mode where a read from one of the remaining accessible drives continues normally, but a read from the failed drive is recalculated from the corresponding block from all the remaining drives. + + [[vinum-objects]] + == [.filename]#vinum# Objects + + To address these problems, [.filename]#vinum# implements a four-level hierarchy of objects: + + * The most visible object is the virtual disk, called a _volume_. Volumes have essentially the same properties as a UNIX(R) disk drive, though there are some minor differences. For one, they have no size limitations. + * Volumes are composed of _plexes_, each of which represent the total address space of a volume. This level in the hierarchy provides redundancy. Think of plexes as individual disks in a mirrored array, each containing the same data. + * Since [.filename]#vinum# exists within the UNIX(R) disk storage framework, it would be possible to use UNIX(R) partitions as the building block for multi-disk plexes. In fact, this turns out to be too inflexible as UNIX(R) disks can have only a limited number of partitions. Instead, [.filename]#vinum# subdivides a single UNIX(R) partition, the _drive_, into contiguous areas called _subdisks_, which are used as building blocks for plexes. + * Subdisks reside on [.filename]#vinum#_drives_, currently UNIX(R) partitions. [.filename]#vinum# drives can contain any number of subdisks. With the exception of a small area at the beginning of the drive, which is used for storing configuration and state information, the entire drive is available for data storage. + + The following sections describe the way these objects provide the functionality required of [.filename]#vinum#. + + === Volume Size Considerations + + Plexes can include multiple subdisks spread over all drives in the [.filename]#vinum# configuration. + As a result, the size of an individual drive does not limit the size of a plex or a volume. + + === Redundant Data Storage + + [.filename]#vinum# implements mirroring by attaching multiple plexes to a volume. + Each plex is a representation of the data in a volume. + A volume may contain between one and eight plexes. + + Although a plex represents the complete data of a volume, it is possible for parts of the representation to be physically missing, either by design (by not defining a subdisk for parts of the plex) or by accident (as a result of the failure of a drive). + As long as at least one plex can provide the data for the complete address range of the volume, the volume is fully functional. + + === Which Plex Organization? + + [.filename]#vinum# implements both concatenation and striping at the plex level: + + * A _concatenated plex_ uses the address space of each subdisk in turn. Concatenated plexes are the most flexible as they can contain any number of subdisks, and the subdisks may be of different length. The plex may be extended by adding additional subdisks. They require less CPU time than striped plexes, though the difference in CPU overhead is not measurable. On the other hand, they are most susceptible to hot spots, where one disk is very active and others are idle. + * A _striped plex_ stripes the data across each subdisk. The subdisks must all be the same size and there must be at least two subdisks to distinguish it from a concatenated plex. The greatest advantage of striped plexes is that they reduce hot spots. By choosing an optimum sized stripe, about 256 kB, the load can be evened out on the component drives. Extending a plex by adding new subdisks is so complicated that [.filename]#vinum# does not implement it. + + crossref:vinum[vinum-comparison, [.filename]#vinum# Plex Organizations] summarizes the advantages and disadvantages of each plex organization. + + [[vinum-comparison]] + .[.filename]#vinum# Plex Organizations + [cols="1,1,1,1,1", frame="none", options="header"] + |=== + | Plex type + | Minimum subdisks + | Can add subdisks + | Must be equal size + | Application + + |concatenated + |1 + |yes + |no + |Large data storage with maximum placement flexibility and moderate performance + + |striped + |2 + |no + |yes + |High performance in combination with highly concurrent access + |=== + + [[vinum-examples]] + == Some Examples + + [.filename]#vinum# maintains a _configuration database_ which describes the objects known to an individual system. + Initially, the user creates the configuration database from one or more configuration files using man:gvinum[8]. + [.filename]#vinum# stores a copy of its configuration database on each disk _device_ under its control. + This database is updated on each state change, so that a restart accurately restores the state of each [.filename]#vinum# object. + + === The Configuration File + + The configuration file describes individual [.filename]#vinum# objects. + The definition of a simple volume might be: + + [.programlisting] + .... + drive a device /dev/da3h + volume myvol + plex org concat + sd length 512m drive a + .... + + This file describes four [.filename]#vinum# objects: + + * The _drive_ line describes a disk partition (_drive_) and its location relative to the underlying hardware. It is given the symbolic name _a_. This separation of symbolic names from device names allows disks to be moved from one location to another without confusion. + * The _volume_ line describes a volume. The only required attribute is the name, in this case _myvol_. + * The _plex_ line defines a plex. The only required parameter is the organization, in this case _concat_. No name is necessary as the system automatically generates a name from the volume name by adding the suffix _.px_, where _x_ is the number of the plex in the volume. Thus this plex will be called _myvol.p0_. + * The _sd_ line describes a subdisk. The minimum specifications are the name of a drive on which to store it, and the length of the subdisk. No name is necessary as the system automatically assigns names derived from the plex name by adding the suffix _.sx_, where _x_ is the number of the subdisk in the plex. Thus [.filename]#vinum# gives this subdisk the name _myvol.p0.s0_. + + After processing this file, man:gvinum[8] produces the following output: + + [.programlisting] + .... + + # gvinum -> create config1 + Configuration summary + Drives: 1 (4 configured) + Volumes: 1 (4 configured) + Plexes: 1 (8 configured) + Subdisks: 1 (16 configured) + + D a State: up Device /dev/da3h Avail: 2061/2573 MB (80%) + + V myvol State: up Plexes: 1 Size: 512 MB + + P myvol.p0 C State: up Subdisks: 1 Size: 512 MB + + S myvol.p0.s0 State: up PO: 0 B Size: 512 MB + .... + + This output shows the brief listing format of man:gvinum[8]. + It is represented graphically in crossref:vinum[vinum-simple-vol, A Simple [.filename]#vinum# Volume]. + + [[vinum-simple-vol]] + .A Simple [.filename]#vinum# Volume + image::vinum-simple-vol.png[] + + This figure, and the ones which follow, represent a volume, which contains the plexes, which in turn contains the subdisks. + In this example, the volume contains one plex, and the plex contains one subdisk. + + This particular volume has no specific advantage over a conventional disk partition. + It contains a single plex, so it is not redundant. + The plex contains a single subdisk, so there is no difference in storage allocation from a conventional disk partition. + The following sections illustrate various more interesting configuration methods. + + === Increased Resilience: Mirroring + + The resilience of a volume can be increased by mirroring. + When laying out a mirrored volume, it is important to ensure that the subdisks of each plex are on different drives, so that a drive failure will not take down both plexes. + The following configuration mirrors a volume: + + [.programlisting] + .... + drive b device /dev/da4h + volume mirror + plex org concat + sd length 512m drive a + plex org concat + sd length 512m drive b + .... + + In this example, it was not necessary to specify a definition of drive _a_ again, since [.filename]#vinum# keeps track of all objects in its configuration database. + After processing this definition, the configuration looks like: + + [.programlisting] + .... + + Drives: 2 (4 configured) + Volumes: 2 (4 configured) + Plexes: 3 (8 configured) + Subdisks: 3 (16 configured) + + D a State: up Device /dev/da3h Avail: 1549/2573 MB (60%) + D b State: up Device /dev/da4h Avail: 2061/2573 MB (80%) + + V myvol State: up Plexes: 1 Size: 512 MB + V mirror State: up Plexes: 2 Size: 512 MB + + P myvol.p0 C State: up Subdisks: 1 Size: 512 MB + P mirror.p0 C State: up Subdisks: 1 Size: 512 MB + P mirror.p1 C State: initializing Subdisks: 1 Size: 512 MB + + S myvol.p0.s0 State: up PO: 0 B Size: 512 MB + S mirror.p0.s0 State: up PO: 0 B Size: 512 MB + S mirror.p1.s0 State: empty PO: 0 B Size: 512 MB + .... + + crossref:vinum[vinum-mirrored-vol, A Mirrored [.filename]#vinum# Volume] shows the structure graphically. + + [[vinum-mirrored-vol]] + .A Mirrored [.filename]#vinum# Volume + image::vinum-mirrored-vol.png[] + + In this example, each plex contains the full 512 MB of address space. + As in the previous example, each plex contains only a single subdisk. + + === Optimizing Performance + + The mirrored volume in the previous example is more resistant to failure than an unmirrored volume, but its performance is less as each write to the volume requires a write to both drives, using up a greater proportion of the total disk bandwidth. + Performance considerations demand a different approach: instead of mirroring, the data is striped across as many disk drives as possible. + The following configuration shows a volume with a plex striped across four disk drives: + + [.programlisting] + .... + drive c device /dev/da5h + drive d device /dev/da6h + volume stripe + plex org striped 512k + sd length 128m drive a + sd length 128m drive b + sd length 128m drive c + sd length 128m drive d + .... + + As before, it is not necessary to define the drives which are already known to [.filename]#vinum#. + After processing this definition, the configuration looks like: + + [.programlisting] + .... + + Drives: 4 (4 configured) + Volumes: 3 (4 configured) + Plexes: 4 (8 configured) + Subdisks: 7 (16 configured) + + D a State: up Device /dev/da3h Avail: 1421/2573 MB (55%) + D b State: up Device /dev/da4h Avail: 1933/2573 MB (75%) + D c State: up Device /dev/da5h Avail: 2445/2573 MB (95%) + D d State: up Device /dev/da6h Avail: 2445/2573 MB (95%) + + V myvol State: up Plexes: 1 Size: 512 MB + V mirror State: up Plexes: 2 Size: 512 MB + V striped State: up Plexes: 1 Size: 512 MB + + P myvol.p0 C State: up Subdisks: 1 Size: 512 MB + P mirror.p0 C State: up Subdisks: 1 Size: 512 MB + P mirror.p1 C State: initializing Subdisks: 1 Size: 512 MB + P striped.p1 State: up Subdisks: 1 Size: 512 MB + + S myvol.p0.s0 State: up PO: 0 B Size: 512 MB + S mirror.p0.s0 State: up PO: 0 B Size: 512 MB + S mirror.p1.s0 State: empty PO: 0 B Size: 512 MB + S striped.p0.s0 State: up PO: 0 B Size: 128 MB + S striped.p0.s1 State: up PO: 512 kB Size: 128 MB + S striped.p0.s2 State: up PO: 1024 kB Size: 128 MB + S striped.p0.s3 State: up PO: 1536 kB Size: 128 MB + .... + + [[vinum-striped-vol]] + .A Striped [.filename]#vinum# Volume + image::vinum-striped-vol.png[] + + This volume is represented in crossref:vinum[vinum-striped-vol, A Striped [.filename]#vinum# Volume]. + The darkness of the stripes indicates the position within the plex address space, where the lightest stripes come first and the darkest last. + + === Resilience and Performance + + [[vinum-resilience]]With sufficient hardware, it is possible to build volumes which show both increased resilience and increased performance compared to standard UNIX(R) partitions. + A typical configuration file might be: + + [.programlisting] + .... + volume raid10 + plex org striped 512k + sd length 102480k drive a + sd length 102480k drive b + sd length 102480k drive c + sd length 102480k drive d + sd length 102480k drive e + plex org striped 512k + sd length 102480k drive c + sd length 102480k drive d + sd length 102480k drive e + sd length 102480k drive a + sd length 102480k drive b + .... + + The subdisks of the second plex are offset by two drives from those of the first plex. + This helps to ensure that writes do not go to the same subdisks even if a transfer goes over two drives. + + crossref:vinum[vinum-raid10-vol, A Mirrored, Striped [.filename]#vinum# Volume] represents the structure of this volume. + + [[vinum-raid10-vol]] + .A Mirrored, Striped [.filename]#vinum# Volume + image::vinum-raid10-vol.png[] + + [[vinum-object-naming]] + == Object Naming + + [.filename]#vinum# assigns default names to plexes and subdisks, although they may be overridden. + Overriding the default names is not recommended as it does not bring a significant advantage and it can cause confusion. + + Names may contain any non-blank character, but it is recommended to restrict them to letters, digits and the underscore characters. + The names of volumes, plexes, and subdisks may be up to 64 characters long, and the names of drives may be up to 32 characters long. + + [.filename]#vinum# objects are assigned device nodes in the hierarchy [.filename]#/dev/gvinum#. + The configuration shown above would cause [.filename]#vinum# to create the following device nodes: + + * Device entries for each volume. These are the main devices used by [.filename]#vinum#. The configuration above would include the devices [.filename]#/dev/gvinum/myvol#, [.filename]#/dev/gvinum/mirror#, [.filename]#/dev/gvinum/striped#, [.filename]#/dev/gvinum/raid5# and [.filename]#/dev/gvinum/raid10#. + * All volumes get direct entries under [.filename]#/dev/gvinum/#. + * The directories [.filename]#/dev/gvinum/plex#, and [.filename]#/dev/gvinum/sd#, which contain device nodes for each plex and for each subdisk, respectively. + + For example, consider the following configuration file: + + [.programlisting] + .... + drive drive1 device /dev/sd1h + drive drive2 device /dev/sd2h + drive drive3 device /dev/sd3h + drive drive4 device /dev/sd4h + volume s64 setupstate + plex org striped 64k + sd length 100m drive drive1 + sd length 100m drive drive2 + sd length 100m drive drive3 + sd length 100m drive drive4 + .... + + After processing this file, man:gvinum[8] creates the following structure in [.filename]#/dev/gvinum#: + + [.programlisting] + .... + drwxr-xr-x 2 root wheel 512 Apr 13 + 16:46 plex + crwxr-xr-- 1 root wheel 91, 2 Apr 13 16:46 s64 + drwxr-xr-x 2 root wheel 512 Apr 13 16:46 sd + + /dev/vinum/plex: + total 0 + crwxr-xr-- 1 root wheel 25, 0x10000002 Apr 13 16:46 s64.p0 + + /dev/vinum/sd: + total 0 + crwxr-xr-- 1 root wheel 91, 0x20000002 Apr 13 16:46 s64.p0.s0 + crwxr-xr-- 1 root wheel 91, 0x20100002 Apr 13 16:46 s64.p0.s1 + crwxr-xr-- 1 root wheel 91, 0x20200002 Apr 13 16:46 s64.p0.s2 + crwxr-xr-- 1 root wheel 91, 0x20300002 Apr 13 16:46 s64.p0.s3 + .... + + Although it is recommended that plexes and subdisks should not be allocated specific names, [.filename]#vinum# drives must be named. + This makes it possible to move a drive to a different location and still recognize it automatically. + Drive names may be up to 32 characters long. + + === Creating File Systems + + Volumes appear to the system to be identical to disks, with one exception. + Unlike UNIX(R) drives, [.filename]#vinum# does not partition volumes, which thus do not contain a partition table. + This has required modification to some disk utilities, notably man:newfs[8], so that it does not try to interpret the last letter of a [.filename]#vinum# volume name as a partition identifier. + For example, a disk drive may have a name like [.filename]#/dev/ad0a# or [.filename]#/dev/da2h#. + These names represent the first partition ([.filename]#a#) on the first (0) IDE disk ([.filename]#ad#) and the eighth partition ([.filename]#h#) on the third (2) SCSI disk ([.filename]#da#) respectively. + By contrast, a [.filename]#vinum# volume might be called [.filename]#/dev/gvinum/concat#, which has no relationship with a partition name. + + To create a file system on this volume, use man:newfs[8]: + + [source,shell] + .... + # newfs /dev/gvinum/concat + .... + + [[vinum-config]] + == Configuring [.filename]#vinum# + + The [.filename]#GENERIC# kernel does not contain [.filename]#vinum#. + It is possible to build a custom kernel which includes [.filename]#vinum#, but this is not recommended. + The standard way to start [.filename]#vinum# is as a kernel module. + man:kldload[8] is not needed because when man:gvinum[8] starts, it checks whether the module has been loaded, and if it is not, it loads it automatically. + + === Startup + + [.filename]#vinum# stores configuration information on the disk slices in essentially the same form as in the configuration files. + When reading from the configuration database, [.filename]#vinum# recognizes a number of keywords which are not allowed in the configuration files. + For example, a disk configuration might contain the following text: + + [.programlisting] + .... + volume myvol state up + volume bigraid state down + plex name myvol.p0 state up org concat vol myvol + plex name myvol.p1 state up org concat vol myvol + plex name myvol.p2 state init org striped 512b vol myvol + plex name bigraid.p0 state initializing org raid5 512b vol bigraid + sd name myvol.p0.s0 drive a plex myvol.p0 state up len 1048576b driveoffset 265b plexoffset 0b + sd name myvol.p0.s1 drive b plex myvol.p0 state up len 1048576b driveoffset 265b plexoffset 1048576b + sd name myvol.p1.s0 drive c plex myvol.p1 state up len 1048576b driveoffset 265b plexoffset 0b + sd name myvol.p1.s1 drive d plex myvol.p1 state up len 1048576b driveoffset 265b plexoffset 1048576b + sd name myvol.p2.s0 drive a plex myvol.p2 state init len 524288b driveoffset 1048841b plexoffset 0b + sd name myvol.p2.s1 drive b plex myvol.p2 state init len 524288b driveoffset 1048841b plexoffset 524288b + sd name myvol.p2.s2 drive c plex myvol.p2 state init len 524288b driveoffset 1048841b plexoffset 1048576b + sd name myvol.p2.s3 drive d plex myvol.p2 state init len 524288b driveoffset 1048841b plexoffset 1572864b + sd name bigraid.p0.s0 drive a plex bigraid.p0 state initializing len 4194304b driveoff set 1573129b plexoffset 0b + sd name bigraid.p0.s1 drive b plex bigraid.p0 state initializing len 4194304b driveoff set 1573129b plexoffset 4194304b + sd name bigraid.p0.s2 drive c plex bigraid.p0 state initializing len 4194304b driveoff set 1573129b plexoffset 8388608b + sd name bigraid.p0.s3 drive d plex bigraid.p0 state initializing len 4194304b driveoff set 1573129b plexoffset 12582912b + sd name bigraid.p0.s4 drive e plex bigraid.p0 state initializing len 4194304b driveoff set 1573129b plexoffset 16777216b + .... + + The obvious differences here are the presence of explicit location information and naming, both of which are allowed but discouraged, and the information on the states. + [.filename]#vinum# does not store information about drives in the configuration information. + It finds the drives by scanning the configured disk drives for partitions with a [.filename]#vinum# label. + This enables [.filename]#vinum# to identify drives correctly even if they have been assigned different UNIX(R) drive IDs. + + [[vinum-rc-startup]] + ==== Automatic Startup + + _Gvinum_ always features an automatic startup once the kernel module is loaded, via man:loader.conf[5]. + To load the _Gvinum_ module at boot time, add `geom_vinum_load="YES"` to [.filename]#/boot/loader.conf#. + +-When [.filename]#vinum# is started with `gvinum start`, [.filename]#vinum# reads the configuration database from one of the [.filename]#vinum# drives. ++When [.filename]#vinum# is started with `gvinum start`, [.filename]#vinum# reads the configuration database from one of the [.filename]#vinum# drives. + Under normal circumstances, each drive contains an identical copy of the configuration database, so it does not matter which drive is read. + After a crash, however, [.filename]#vinum# must determine which drive was updated most recently and read the configuration from this drive. + It then updates the configuration, if necessary, from progressively older drives. + + [[vinum-root]] + == Using [.filename]#vinum# for the Root File System + + For a machine that has fully-mirrored file systems using [.filename]#vinum#, it is desirable to also mirror the root file system. + Setting up such a configuration is less trivial than mirroring an arbitrary file system because: + + * The root file system must be available very early during the boot process, so the [.filename]#vinum# infrastructure must already be available at this time. + * The volume containing the root file system also contains the system bootstrap and the kernel. These must be read using the host system's native utilities, such as the BIOS, which often cannot be taught about the details of [.filename]#vinum#. + + In the following sections, the term "root volume" is generally used to describe the [.filename]#vinum# volume that contains the root file system. + + === Starting up [.filename]#vinum# Early Enough for the Root File System + +-[.filename]#vinum# must be available early in the system boot as man:loader[8] must be able to load the vinum kernel module before starting the kernel. ++[.filename]#vinum# must be available early in the system boot as man:loader[8] must be able to load the vinum kernel module before starting the kernel. + This can be accomplished by putting this line in [.filename]#/boot/loader.conf#: + + [.programlisting] + .... + geom_vinum_load="YES" + .... + + === Making a [.filename]#vinum#-based Root Volume Accessible to the Bootstrap + + The current FreeBSD bootstrap is only 7.5 KB of code and does not understand the internal [.filename]#vinum# structures. + This means that it cannot parse the [.filename]#vinum# configuration data or figure out the elements of a boot volume. + Thus, some workarounds are necessary to provide the bootstrap code with the illusion of a standard `a` partition that contains the root file system. + + For this to be possible, the following requirements must be met for the root volume: + + * The root volume must not be a stripe or `RAID`-5. + * The root volume must not contain more than one concatenated subdisk per plex. + + Note that it is desirable and possible to use multiple plexes, each containing one replica of the root file system. + The bootstrap process will only use one replica for finding the bootstrap and all boot files, until the kernel mounts the root file system. + Each single subdisk within these plexes needs its own `a` partition illusion, for the respective device to be bootable. + It is not strictly needed that each of these faked `a` partitions is located at the same offset within its device, compared with other devices containing plexes of the root volume. + However, it is probably a good idea to create the [.filename]#vinum# volumes that way so the resulting mirrored devices are symmetric, to avoid confusion. + + To set up these `a` partitions for each device containing part of the root volume, the following is required: + + [.procedure] + ==== + . The location, offset from the beginning of the device, and size of this device's subdisk that is part of the root volume needs to be examined, using the command: + + + [source,shell] + .... + # gvinum l -rv root + .... + + + [.filename]#vinum# offsets and sizes are measured in bytes. + They must be divided by 512 to obtain the block numbers that are to be used by `bsdlabel`. + . Run this command for each device that participates in the root volume: + + + [source,shell] + .... + # bsdlabel -e devname + .... + + + _devname_ must be either the name of the disk, like [.filename]#da0# for disks without a slice table, or the name of the slice, like [.filename]#ad0s1#. + + + If there is already an `a` partition on the device from a pre-[.filename]#vinum# root file system, it should be renamed to something else so that it remains accessible (just in case), but will no longer be used by default to bootstrap the system. + A currently mounted root file system cannot be renamed, so this must be executed either when being booted from a "Fixit" media, or in a two-step process where, in a mirror, the disk that is not been currently booted is manipulated first. + + + The offset of the [.filename]#vinum# partition on this device (if any) must be added to the offset of the respective root volume subdisk on this device. + The resulting value will become the `offset` value for the new `a` partition. + The `size` value for this partition can be taken verbatim from the calculation above. + The `fstype` should be `4.2BSD`. + The `fsize`, `bsize`, and `cpg` values should be chosen to match the actual file system, though they are fairly unimportant within this context. + + + That way, a new `a` partition will be established that overlaps the [.filename]#vinum# partition on this device. + `bsdlabel` will only allow for this overlap if the [.filename]#vinum# partition has properly been marked using the `vinum` fstype. + . A faked `a` partition now exists on each device that has one replica of the root volume. It is highly recommendable to verify the result using a command like: + + + [source,shell] + .... + # fsck -n /dev/devnamea + .... + ==== + + It should be remembered that all files containing control information must be relative to the root file system in the [.filename]#vinum# volume which, when setting up a new [.filename]#vinum# root volume, might not match the root file system that is currently active. + So in particular, [.filename]#/etc/fstab# and [.filename]#/boot/loader.conf# need to be taken care of. + + At next reboot, the bootstrap should figure out the appropriate control information from the new [.filename]#vinum#-based root file system, and act accordingly. + At the end of the kernel initialization process, after all devices have been announced, the prominent notice that shows the success of this setup is a message like: + + [source,shell] + .... + Mounting root from ufs:/dev/gvinum/root + .... + + === Example of a [.filename]#vinum#-based Root Setup + + After the [.filename]#vinum# root volume has been set up, the output of `gvinum l -rv root` could look like: + + [source,shell] + .... + ... + Subdisk root.p0.s0: + Size: 125829120 bytes (120 MB) + State: up + Plex root.p0 at offset 0 (0 B) + Drive disk0 (/dev/da0h) at offset 135680 (132 kB) + + Subdisk root.p1.s0: + Size: 125829120 bytes (120 MB) + State: up + Plex root.p1 at offset 0 (0 B) + Drive disk1 (/dev/da1h) at offset 135680 (132 kB) + .... + + The values to note are `135680` for the offset, relative to partition [.filename]#/dev/da0h#. + This translates to 265 512-byte disk blocks in `bsdlabel`'s terms. + Likewise, the size of this root volume is 245760 512-byte blocks. + [.filename]#/dev/da1h#, containing the second replica of this root volume, has a symmetric setup. + + The bsdlabel for these devices might look like: + + [source,shell] + .... + ... + 8 partitions: + # size offset fstype [fsize bsize bps/cpg] + a: 245760 281 4.2BSD 2048 16384 0 # (Cyl. 0*- 15*) + c: 71771688 0 unused 0 0 # (Cyl. 0 - 4467*) + h: 71771672 16 vinum # (Cyl. 0*- 4467*) + .... + + It can be observed that the `size` parameter for the faked `a` partition matches the value outlined above, while the `offset` parameter is the sum of the offset within the [.filename]#vinum# partition `h`, and the offset of this partition within the device or slice. + This is a typical setup that is necessary to avoid the problem described in + crossref:vinum[vinum-root-panic, Nothing Boots, the Bootstrap Panics]. + The entire `a` partition is completely within the `h` partition containing all the [.filename]#vinum# data for this device. + + In the above example, the entire device is dedicated to [.filename]#vinum# and there is no leftover pre-[.filename]#vinum# root partition. + + === Troubleshooting + + The following list contains a few known pitfalls and solutions. + + ==== System Bootstrap Loads, but System Does Not Boot + + If for any reason the system does not continue to boot, the bootstrap can be interrupted by pressing kbd:[space] at the 10-seconds warning. + The loader variable `vinum.autostart` can be examined by typing `show` and manipulated using `set` or `unset`. + + If the [.filename]#vinum# kernel module was not yet in the list of modules to load automatically, type `load geom_vinum`. + + When ready, the boot process can be continued by typing `boot -as` which `-as` requests the kernel to ask for the root file system to mount (`-a`) and make the boot process stop in single-user mode (`-s`), where the root file system is mounted read-only. + That way, even if only one plex of a multi-plex volume has been mounted, no data inconsistency between plexes is being risked. + + At the prompt asking for a root file system to mount, any device that contains a valid root file system can be entered. + If [.filename]#/etc/fstab# is set up correctly, the default should be something like `ufs:/dev/gvinum/root`. + A typical alternate choice would be something like `ufs:da0d` which could be a hypothetical partition containing the pre-[.filename]#vinum# root file system. + Care should be taken if one of the alias `a` partitions is entered here, that it actually references the subdisks of the [.filename]#vinum# root device, because in a mirrored setup, this would only mount one piece of a mirrored root device. + If this file system is to be mounted read-write later on, it is necessary to remove the other plex(es) of the [.filename]#vinum# root volume since these plexes would otherwise carry inconsistent data. + + ==== Only Primary Bootstrap Loads + + If [.filename]#/boot/loader# fails to load, but the primary bootstrap still loads (visible by a single dash in the left column of the screen right after the boot process starts), an attempt can be made to interrupt the primary bootstrap by pressing kbd:[space]. +-This will make the bootstrap stop in extref:{handbook}[stage two, boot-boot1]. ++This will make the bootstrap stop in extref:{handbook}boot[stage two, boot-boot1]. + An attempt can be made here to boot off an alternate partition, like the partition containing the previous root file system that has been moved away from `a`. + + [[vinum-root-panic]] + ==== Nothing Boots, the Bootstrap Panics + + This situation will happen if the bootstrap had been destroyed by the [.filename]#vinum# installation. + Unfortunately, [.filename]#vinum# accidentally leaves only 4 KB at the beginning of its partition free before starting to write its [.filename]#vinum# header information. + However, the stage one and two bootstraps plus the bsdlabel require 8 KB. + So if a [.filename]#vinum# partition was started at offset 0 within a slice or disk that was meant to be bootable, the [.filename]#vinum# setup will trash the bootstrap. + +-Similarly, if the above situation has been recovered, by booting from a "Fixit" media, and the bootstrap has been re-installed using `bsdlabel -B` as described in extref:{handbook}[stage two, boot-boot1], the bootstrap will trash the [.filename]#vinum# header, and [.filename]#vinum# will no longer find its disk(s). ++Similarly, if the above situation has been recovered, by booting from a "Fixit" media, and the bootstrap has been re-installed using `bsdlabel -B` as described in extref:{handbook}boot[stage two, boot-boot1], the bootstrap will trash the [.filename]#vinum# header, and [.filename]#vinum# will no longer find its disk(s). + Though no actual [.filename]#vinum# configuration data or data in [.filename]#vinum# volumes will be trashed, and it would be possible to recover all the data by entering exactly the same [.filename]#vinum# configuration data again, the situation is hard to fix. + It is necessary to move the entire [.filename]#vinum# partition by at least 4 KB, to have the [.filename]#vinum# header and the system bootstrap no longer collide. +diff --git a/documentation/content/en/books/arch-handbook/_index.adoc b/documentation/content/en/books/arch-handbook/_index.adoc +index 7124ef877e..85f45ef41c 100644 +--- a/documentation/content/en/books/arch-handbook/_index.adoc ++++ b/documentation/content/en/books/arch-handbook/_index.adoc +@@ -1,56 +1,56 @@ + --- + title: FreeBSD Architecture Handbook +-authors: ++authors: + - author: The FreeBSD Documentation Project + copyright: 2000-2006, 2012-2023 The FreeBSD Documentation Project + description: For FreeBSD system developers. This book covers the architectural details of many important FreeBSD kernel subsystems + trademarks: ["freebsd", "apple", "microsoft", "unix", "general"] + tags: ["Arch Handbook", "FreeBSD"] + next: books/arch-handbook/parti + add_single_page_link: true + showBookMenu: true + weight: 0 + params: + path: "/books/arch-handbook/" + bookOrder: 50 + --- + + = FreeBSD Architecture Handbook + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :partnums: + :source-highlighter: rouge + :experimental: + :images-path: books/arch-handbook/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + Welcome to the FreeBSD Architecture Handbook. This manual is a _work in progress_ and is the work of many individuals. Many sections do not yet exist and some of those that do exist need to be updated. If you are interested in helping with this project, send email to the {freebsd-doc}. + +-The latest version of this document is always available from the link:https://www.FreeBSD.org/[FreeBSD World Wide Web server]. It may also be downloaded in a variety of formats and compression options from the https://download.freebsd.org/doc/[FreeBSD download server] or one of the numerous extref:{handbook}[mirror sites, mirrors]. ++The latest version of this document is always available from the link:https://www.FreeBSD.org/[FreeBSD World Wide Web server]. It may also be downloaded in a variety of formats and compression options from the https://download.freebsd.org/doc/[FreeBSD download server] or one of the numerous extref:{handbook}mirrors[mirror sites, mirrors]. + + ''' +diff --git a/documentation/content/en/books/arch-handbook/book.adoc b/documentation/content/en/books/arch-handbook/book.adoc +index 3273926895..14d618f830 100644 +--- a/documentation/content/en/books/arch-handbook/book.adoc ++++ b/documentation/content/en/books/arch-handbook/book.adoc +@@ -1,82 +1,82 @@ + --- + title: FreeBSD Architecture Handbook +-authors: ++authors: + - author: The FreeBSD Documentation Project + copyright: 2000-2006, 2012-2023 The FreeBSD Documentation Project + description: For FreeBSD system developers. This book covers the architectural details of many important FreeBSD kernel subsystems + trademarks: ["freebsd", "apple", "microsoft", "unix", "general"] + tags: ["Arch Handbook", "FreeBSD"] + add_split_page_link: true + --- + + = FreeBSD Architecture Handbook + :doctype: book + :toc: macro + :toclevels: 2 + :icons: font + :sectnums: + :sectnumlevels: 6 + :partnums: + :source-highlighter: rouge + :experimental: + :book: true + :pdf: false + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :chapters-path: content/{{% lang %}}/books/arch-handbook/ + endif::[] + ifdef::backend-pdf,backend-epub3[] + :chapters-path: + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + :chapters-path: + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + Welcome to the FreeBSD Architecture Handbook. This manual is a _work in progress_ and is the work of many individuals. Many sections do not yet exist and some of those that do exist need to be updated. If you are interested in helping with this project, send email to the {freebsd-doc}. + +-The latest version of this document is always available from the link:https://www.FreeBSD.org/[FreeBSD World Wide Web server]. It may also be downloaded in a variety of formats and compression options from the https://download.freebsd.org/doc/[FreeBSD download server] or one of the numerous extref:{handbook}[mirror sites, mirrors]. ++The latest version of this document is always available from the link:https://www.FreeBSD.org/[FreeBSD World Wide Web server]. It may also be downloaded in a variety of formats and compression options from the https://download.freebsd.org/doc/[FreeBSD download server] or one of the numerous extref:{handbook}mirrors[mirror sites, mirrors]. + + ''' + + toc::[] + + // Section one + include::{chapters-path}parti.adoc[] + include::{chapters-path}boot/_index.adoc[leveloffset=+1] + include::{chapters-path}locking/_index.adoc[leveloffset=+1] + include::{chapters-path}kobj/_index.adoc[leveloffset=+1] + include::{chapters-path}jail/_index.adoc[leveloffset=+1] + include::{chapters-path}sysinit/_index.adoc[leveloffset=+1]] + include::{chapters-path}mac/_index.adoc[leveloffset=+1] + include::{chapters-path}vm/_index.adoc[leveloffset=+1] + include::{chapters-path}smp/_index.adoc[leveloffset=+1] + + // Section two + include::{chapters-path}partii.adoc[] + include::{chapters-path}driverbasics/_index.adoc[leveloffset=+1] + include::{chapters-path}isa/_index.adoc[leveloffset=+1] + include::{chapters-path}pci/_index.adoc[leveloffset=+1] + include::{chapters-path}scsi/_index.adoc[leveloffset=+1] + include::{chapters-path}usb/_index.adoc[leveloffset=+1] + include::{chapters-path}newbus/_index.adoc[leveloffset=+1] + include::{chapters-path}sound/_index.adoc[leveloffset=+1] + include::{chapters-path}pccard/_index.adoc[leveloffset=+1] + + // Section three + include::{chapters-path}partiii.adoc[] + include::{chapters-path}bibliography/_index.adoc[leveloffset=+1] +diff --git a/documentation/content/en/books/arch-handbook/boot/_index.adoc b/documentation/content/en/books/arch-handbook/boot/_index.adoc +index 9065a9a183..ef1d8932d1 100644 +--- a/documentation/content/en/books/arch-handbook/boot/_index.adoc ++++ b/documentation/content/en/books/arch-handbook/boot/_index.adoc +@@ -1,1702 +1,1702 @@ + --- + title: Chapter 1. Bootstrapping and Kernel Initialization + prev: books/arch-handbook/parti + next: books/arch-handbook/locking + description: Bootstrapping and Kernel Initialization + tags: ["boot", "BIOS", "kernel", "MBR", "FreeBSD"] + showBookMenu: true + weight: 2 + params: + path: "/books/arch-handbook/boot/" + --- + + [[boot]] + = Bootstrapping and Kernel Initialization + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :sectnumoffset: 1 + :partnums: + :source-highlighter: rouge + :experimental: + :images-path: books/arch-handbook/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + :imagesdir: ../../../../images/{images-path} + endif::[] + ifndef::book[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + toc::[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + toc::[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [[boot-synopsis]] + == Synopsis + + This chapter is an overview of the boot and system initialization processes, starting from the BIOS (firmware) POST, to the first user process creation. + Since the initial steps of system startup are very architecture dependent, the IA-32 architecture is used as an example. + But the AMD64 and ARM64 architectures are much more important and compelling examples and should be explained in the near future according to the topic of this document. + + The FreeBSD boot process can be surprisingly complex. + After control is passed from the BIOS, a considerable amount of low-level configuration must be done before the kernel can be loaded and executed. + This setup must be done in a simple and flexible manner, allowing the user a great deal of customization possibilities. + + [[boot-overview]] + == Overview + + The boot process is an extremely machine-dependent activity. + Not only must code be written for every computer architecture, but there may also be multiple types of booting on the same architecture. + For example, a directory listing of [.filename]#stand# reveals a great amount of architecture-dependent code. + There is a directory for each of the various supported architectures. + FreeBSD supports the CSM boot standard (Compatibility Support Module). + So CSM is supported (with both GPT and MBR partitioning support) and UEFI booting (GPT is totally supported, MBR is mostly supported). + It also supports loading files from ext2fs, MSDOS, UFS and ZFS. + FreeBSD also supports the boot environment feature of ZFS which allows the HOST OS to communicate details about what to boot that go beyond a simple partition as was possible in the past. + But UEFI is more relevant than the CSM these days. + The example that follows shows booting an x86 computer from an MBR-partitioned hard drive with the FreeBSD [.filename]#boot0# multi-boot loader stored in the very first sector. + That boot code starts the FreeBSD three-stage boot process. + + The key to understanding this process is that it is a series of stages of increasing complexity. + These stages are [.filename]#boot1#, [.filename]#boot2#, and [.filename]#loader# (see man:boot[8] for more detail). + The boot system executes each stage in sequence. + The last stage, [.filename]#loader#, is responsible for loading the FreeBSD kernel. + Each stage is examined in the following sections. + + Here is an example of the output generated by the different boot stages. + Actual output may differ from machine to machine: + + [.informaltable] + [cols="20%,80%", frame="none"] + |=== + + |*FreeBSD Component* + |*Output (may vary)* + + |`boot0` + a| + + [source,bash] + .... + F1 FreeBSD + F2 BSD + F5 Disk 2 + .... + + |`boot2` footnote:[This prompt will appear if the user presses a key just after selecting an OS to boot at the boot0 stage.] + a| + + [source,bash] + .... + >>FreeBSD/x86 BOOT + Default: 0:ad(0p4)/boot/loader + boot: + .... + + |[.filename]#loader# + a| + + [source,bash] + .... + BTX loader 1.00 BTX version is 1.02 + Consoles: internal video/keyboard + BIOS drive C: is disk0 + BIOS 639kB/2096064kB available memory + + FreeBSD/x86 bootstrap loader, Revision 1.1 + Console internal video/keyboard + (root@releng1.nyi.freebsd.org, Fri Apr 9 04:04:45 UTC 2021) + Loading /boot/defaults/loader.conf + /boot/kernel/kernel text=0xed9008 data=0x117d28+0x176650 syms=[0x8+0x137988+0x8+0x1515f8] + .... + + |kernel + a| + + [source,bash] + .... + Copyright (c) 1992-2021 The FreeBSD Project. + Copyright (c) 1979, 1980, 1983, 1986, 1988, 1989, 1991, 1992, 1993, 1994 + The Regents of the University of California. All rights reserved. + FreeBSD is a registered trademark of The FreeBSD Foundation. + FreeBSD 13.0-RELEASE 0 releng/13.0-n244733-ea31abc261f: Fri Apr 9 04:04:45 UTC 2021 + root@releng1.nyi.freebsd.org:/usr/obj/usr/src/i386.i386/sys/GENERIC i386 + FreeBSD clang version 11.0.1 (git@github.com:llvm/llvm-project.git llvmorg-11.0.1-0-g43ff75f2c3fe) + .... + + |=== + + [[boot-bios]] + == The BIOS + + When the computer powers on, the processor's registers are set to some predefined values. + One of the registers is the _instruction pointer_ register, and its value after a power on is well defined: it is a 32-bit value of `0xfffffff0`. + The instruction pointer register (also known as the Program Counter) points to code to be executed by the processor. + Another important register is the `cr0` 32-bit control register, and its value just after a reboot is `0`. + One of ``cr0``'s bits, the PE (Protection Enabled) bit, indicates whether the processor is running in 32-bit protected mode or 16-bit real mode. + Since this bit is cleared at boot time, the processor boots in 16-bit real mode. + Real mode means, among other things, that linear and physical addresses are identical. + The reason for the processor not to start immediately in 32-bit protected mode is backwards compatibility. + In particular, the boot process relies on the services provided by the BIOS, and the BIOS itself works in legacy, 16-bit code. + + The value of `0xfffffff0` is slightly less than 4 GB, so unless the machine has 4 GB of physical memory, it cannot point to a valid memory address. + The computer's hardware translates this address so that it points to a BIOS memory block. + + The BIOS (Basic Input Output System) is a chip on the motherboard that has a relatively small amount of read-only memory (ROM). + This memory contains various low-level routines that are specific to the hardware supplied with the motherboard. + The processor will first jump to the address 0xfffffff0, which really resides in the BIOS's memory. + Usually this address contains a jump instruction to the BIOS's POST routines. + + The POST (Power On Self Test) is a set of routines including the memory check, system bus check, and other low-level initialization so the CPU can set up the computer properly. + The important step of this stage is determining the boot device. + Modern BIOS implementations permit the selection of a boot device, allowing booting from a floppy, CD-ROM, hard disk, or other devices. + + The very last thing in the POST is the `INT 0x19` instruction. + The `INT 0x19` handler reads 512 bytes from the first sector of boot device into the memory at address `0x7c00`. + The term _first sector_ originates from hard drive architecture, where the magnetic plate is divided into a number of cylindrical tracks. + Tracks are numbered, and every track is divided into a number (usually 64) of sectors. + Track numbers start at 0, but sector numbers start from 1. + Track 0 is the outermost on the magnetic plate, and sector 1, the first sector, has a special purpose. + It is also called the MBR, or Master Boot Record. + The remaining sectors on the first track are never used. + + This sector is our boot-sequence starting point. + As we will see, this sector contains a copy of our [.filename]#boot0# program. + A jump is made by the BIOS to address `0x7c00` so it starts executing. + + [[boot-boot0]] + == The Master Boot Record (`boot0`) + + After control is received from the BIOS at memory address `0x7c00`, [.filename]#boot0# starts executing. + It is the first piece of code under FreeBSD control. + The task of [.filename]#boot0# is quite simple: scan the partition table and let the user choose which partition to boot from. + The Partition Table is a special, standard data structure embedded in the MBR (hence embedded in [.filename]#boot0#) describing the four standard PC "partitions". + [.filename]#boot0# resides in the filesystem as [.filename]#/boot/boot0#. + It is a small 512-byte file, and it is exactly what FreeBSD's installation procedure wrote to the hard disk's MBR if you chose the "bootmanager" option at installation time. + Indeed, [.filename]#boot0# _is_ the MBR. + + As mentioned previously, we're calling the BIOS `INT 0x19` to load the MBR ([.filename]#boot0#) into memory at address `0x7c00`. + The source file for [.filename]#boot0# can be found in [.filename]#stand/i386/boot0/boot0.S# - which is an awesome piece of code written by Robert Nordier. + + A special structure starting from offset `0x1be` in the MBR is called the _partition table_. + It has four records of 16 bytes each, called _partition records_, which represent how the hard disk is partitioned, or, in FreeBSD's terminology, sliced. + One byte of those 16 says whether a partition (slice) is bootable or not. + Exactly one record must have that flag set, otherwise [.filename]#boot0#'s code will refuse to proceed. + + A partition record has the following fields: + + * the 1-byte filesystem type + * the 1-byte bootable flag + * the 6 byte descriptor in CHS format + * the 8 byte descriptor in LBA format + + A partition record descriptor contains information about where exactly the partition resides on the drive. + Both descriptors, LBA and CHS, describe the same information, but in different ways: LBA (Logical Block Addressing) has the starting sector for the partition and the partition's length, while CHS (Cylinder Head Sector) has coordinates for the first and last sectors of the partition. + The partition table ends with the special signature `0xaa55`. + + The MBR must fit into 512 bytes, a single disk sector. + This program uses low-level "tricks" like taking advantage of the side effects of certain instructions and reusing register values from previous operations to make the most out of the fewest possible instructions. + Care must also be taken when handling the partition table, which is embedded in the MBR itself. + For these reasons, be very careful when modifying [.filename]#boot0.S#. + + Note that the [.filename]#boot0.S# source file is assembled "as is": instructions are translated one by one to binary, with no additional information (no ELF file format, for example). + This kind of low-level control is achieved at link time through special control flags passed to the linker. + For example, the text section of the program is set to be located at address `0x600`. + In practice this means that [.filename]#boot0# must be loaded to memory address `0x600` in order to function properly. + + It is worth looking at the [.filename]#Makefile# for [.filename]#boot0# ([.filename]#stand/i386/boot0/Makefile#), as it defines some of the run-time behavior of [.filename]#boot0#. + For instance, if a terminal connected to the serial port (COM1) is used for I/O, the macro `SIO` must be defined (`-DSIO`). + `-DPXE` enables boot through PXE by pressing kbd:[F6]. + Additionally, the program defines a set of _flags_ that allow further modification of its behavior. + All of this is illustrated in the [.filename]#Makefile#. + For example, look at the linker directives which command the linker to start the text section at address `0x600`, and to build the output file "as is" (strip out any file formatting): + + [.programlisting] + .... + BOOT_BOOT0_ORG?=0x600 + ORG=${BOOT_BOOT0_ORG} + .... + + .[.filename]#stand/i386/boot0/Makefile# [[boot-boot0-makefile-as-is]] + Let us now start our study of the MBR, or [.filename]#boot0#, starting where execution begins. + + [NOTE] + ==== + Some modifications have been made to some instructions in favor of better exposition. + For example, some macros are expanded, and some macro tests are omitted when the result of the test is known. + This applies to all of the code examples shown. + ==== + + [.programlisting] + .... + start: + cld # String ops inc + xorw %ax,%ax # Zero + movw %ax,%es # Address + movw %ax,%ds # data + movw %ax,%ss # Set up + movw $LOAD,%sp # stack + .... + + .[.filename]#stand/i386/boot0/boot0.S# [[boot-boot0-entrypoint]] + This first block of code is the entry point of the program. + It is where the BIOS transfers control. + First, it makes sure that the string operations autoincrement its pointer operands (the `cld` instruction) footnote:[When in doubt, we refer the reader to the official Intel manuals, which describe the exact semantics for each instruction: .]. + Then, as it makes no assumption about the state of the segment registers, it initializes them. + Finally, it sets the stack pointer register (`%sp`) to ($LOAD = address `0x7c00`), so we have a working stack. + + The next block is responsible for the relocation and subsequent jump to the relocated code. + + [.programlisting] + .... + movw %sp,%si # Source + movw $start,%di # Destination + movw $0x100,%cx # Word count + rep # Relocate + movsw # code + movw %di,%bp # Address variables + movb $0x8,%cl # Words to clear + rep # Zero + stosw # them + incb -0xe(%di) # Set the S field to 1 + jmp main-LOAD+ORIGIN # Jump to relocated code + .... + + .[.filename]#stand/i386/boot0/boot0.S# [[boot-boot0-relocation]] + As [.filename]#boot0# is loaded by the BIOS to address `0x7C00`, it copies itself to address `0x600` and then transfers control there (recall that it was linked to execute at address `0x600`). + The source address, `0x7c00`, is copied to register `%si`. + The destination address, `0x600`, to register `%di`. + The number of words to copy, `256` (the program's size = 512 bytes), is copied to register `%cx`. + Next, the `rep` instruction repeats the instruction that follows, that is, `movsw`, the number of times dictated by the `%cx` register. + The `movsw` instruction copies the word pointed to by `%si` to the address pointed to by `%di`. + This is repeated another 255 times. + On each repetition, both the source and destination registers, `%si` and `%di`, are incremented by one. + Thus, upon completion of the 256-word (512-byte) copy, `%di` has the value `0x600`+`512`= `0x800`, and `%si` has the value `0x7c00`+`512`= `0x7e00`; we have thus completed the code _relocation_. + Since the last update of this document, the copy instructions have changed in the code, so instead of the movsb and stosb, movsw and stosw have been introduced, which copy 2 bytes(1 word) in one iteration. + + Next, the destination register `%di` is copied to `%bp`. + `%bp` gets the value `0x800`. + The value `8` is copied to `%cl` in preparation for a new string operation (like our previous `movsw`). + Now, `stosw` is executed 8 times. + This instruction copies a `0` value to the address pointed to by the destination register (`%di`, which is `0x800`), and increments it. + This is repeated another 7 times, so `%di` ends up with value `0x810`. + Effectively, this clears the address range `0x800`-`0x80f`. + This range is used as a (fake) partition table for writing the MBR back to disk. + Finally, the sector field for the CHS addressing of this fake partition is given the value 1 and a jump is made to the main function from the relocated code. + Note that until this jump to the relocated code, any reference to an absolute address was avoided. + + The following code block tests whether the drive number provided by the BIOS should be used, or the one stored in [.filename]#boot0#. + + [.programlisting] + .... + main: + testb $SETDRV,_FLAGS(%bp) # Set drive number? + #ifndef CHECK_DRIVE /* disable drive checks */ + jz save_curdrive # no, use the default + #else + jnz disable_update # Yes + testb %dl,%dl # Drive number valid? + js save_curdrive # Possibly (0x80 set) + #endif + .... + + .[.filename]#stand/i386/boot0/boot0.S# [[boot-boot0-drivenumber]] + This code tests the `SETDRV` bit (`0x20`) in the _flags_ variable. + Recall that register `%bp` points to address location `0x800`, so the test is done to the _flags_ variable at address `0x800`-`69`= `0x7bb`. + This is an example of the type of modifications that can be done to [.filename]#boot0#. + The `SETDRV` flag is not set by default, but it can be set in the [.filename]#Makefile#. + When set, the drive number stored in the MBR is used instead of the one provided by the BIOS. + We assume the defaults, and that the BIOS provided a valid drive number, so we jump to `save_curdrive`. + + The next block saves the drive number provided by the BIOS, and calls `putn` to print a new line on the screen. + + [.programlisting] + .... + save_curdrive: + movb %dl, (%bp) # Save drive number + pushw %dx # Also in the stack + #ifdef TEST /* test code, print internal bios drive */ + rolb $1, %dl + movw $drive, %si + call putkey + #endif + callw putn # Print a newline + .... + + .[.filename]#stand/i386/boot0/boot0.S# [[boot-boot0-savedrivenumber]] + Note that we assume `TEST` is not defined, so the conditional code in it is not assembled and will not appear in our executable [.filename]#boot0#. + + Our next block implements the actual scanning of the partition table. + It prints to the screen the partition type for each of the four entries in the partition table. + It compares each type with a list of well-known operating system file systems. + Examples of recognized partition types are NTFS (Windows(R), ID 0x7), `ext2fs` (Linux(R), ID 0x83), and, of course, `ffs`/`ufs2` (FreeBSD, ID 0xa5). + The implementation is fairly simple. + + [.programlisting] + .... + movw $(partbl+0x4),%bx # Partition table (+4) + xorw %dx,%dx # Item number + + read_entry: + movb %ch,-0x4(%bx) # Zero active flag (ch == 0) + btw %dx,_FLAGS(%bp) # Entry enabled? + jnc next_entry # No + movb (%bx),%al # Load type + test %al, %al # skip empty partition + jz next_entry + movw $bootable_ids,%di # Lookup tables + movb $(TLEN+1),%cl # Number of entries + repne # Locate + scasb # type + addw $(TLEN-1), %di # Adjust + movb (%di),%cl # Partition + addw %cx,%di # description + callw putx # Display it + + next_entry: + incw %dx # Next item + addb $0x10,%bl # Next entry + jnc read_entry # Till done + .... + + .[.filename]#stand/i386/boot0/boot0.S# [[boot-boot0-partition-scan]] + It is important to note that the active flag for each entry is cleared, so after the scanning, _no_ partition entry is active in our memory copy of [.filename]#boot0#. + Later, the active flag will be set for the selected partition. + This ensures that only one active partition exists if the user chooses to write the changes back to disk. + + The next block tests for other drives. + At startup, the BIOS writes the number of drives present in the computer to address `0x475`. + If there are any other drives present, [.filename]#boot0# prints the current drive to screen. + The user may command [.filename]#boot0# to scan partitions on another drive later. + + [.programlisting] + .... + popw %ax # Drive number + subb $0x80-0x1,%al # Does next + cmpb NHRDRV,%al # drive exist? (from BIOS?) + jb print_drive # Yes + decw %ax # Already drive 0? + jz print_prompt # Yes + .... + + .[.filename]#stand/i386/boot0/boot0.S# [[boot-boot0-test-drives]] + We make the assumption that a single drive is present, so the jump to `print_drive` is not performed. + We also assume nothing strange happened, so we jump to `print_prompt`. + + This next block just prints out a prompt followed by the default option: + + [.programlisting] + .... + print_prompt: + movw $prompt,%si # Display + callw putstr # prompt + movb _OPT(%bp),%dl # Display + decw %si # default + callw putkey # key + jmp start_input # Skip beep + .... + + .[.filename]#stand/i386/boot0/boot0.S# [[boot-boot0-prompt]] + Finally, a jump is performed to `start_input`, where the BIOS services are used to start a timer and for reading user input from the keyboard; if the timer expires, the default option will be selected: + + [.programlisting] + .... + start_input: + xorb %ah,%ah # BIOS: Get + int $0x1a # system time + movw %dx,%di # Ticks when + addw _TICKS(%bp),%di # timeout + read_key: + movb $0x1,%ah # BIOS: Check + int $0x16 # for keypress + jnz got_key # Have input + xorb %ah,%ah # BIOS: int 0x1a, 00 + int $0x1a # get system time + cmpw %di,%dx # Timeout? + jb read_key # No + .... + + .[.filename]#stand/i386/boot0/boot0.S# [[boot-boot0-start-input]] + An interrupt is requested with number `0x1a` and argument `0` in register `%ah`. + The BIOS has a predefined set of services, requested by applications as software-generated interrupts through the `int` instruction and receiving arguments in registers (in this case, `%ah`). + Here, particularly, we are requesting the number of clock ticks since last midnight; this value is computed by the BIOS through the RTC (Real Time Clock). + This clock can be programmed to work at frequencies ranging from 2 Hz to 8192 Hz. + The BIOS sets it to 18.2 Hz at startup. + When the request is satisfied, a 32-bit result is returned by the BIOS in registers `%cx` and `%dx` (lower bytes in `%dx`). + This result (the `%dx` part) is copied to register `%di`, and the value of the `TICKS` variable is added to `%di`. + This variable resides in [.filename]#boot0# at offset `_TICKS` (a negative value) from register `%bp` (which, recall, points to `0x800`). + The default value of this variable is `0xb6` (182 in decimal). + Now, the idea is that [.filename]#boot0# constantly requests the time from the BIOS, and when the value returned in register `%dx` is greater than the value stored in `%di`, the time is up and the default selection will be made. + Since the RTC ticks 18.2 times per second, this condition will be met after 10 seconds (this default behavior can be changed in the [.filename]#Makefile#). + Until this time has passed, [.filename]#boot0# continually asks the BIOS for any user input; this is done through `int 0x16`, argument `1` in `%ah`. + + Whether a key was pressed or the time expired, subsequent code validates the selection. + Based on the selection, the register `%si` is set to point to the appropriate partition entry in the partition table. + This new selection overrides the previous default one. + Indeed, it becomes the new default. + Finally, the ACTIVE flag of the selected partition is set. + If it was enabled at compile time, the in-memory version of [.filename]#boot0# with these modified values is written back to the MBR on disk. + We leave the details of this implementation to the reader. + + We now end our study with the last code block from the [.filename]#boot0# program: + + [.programlisting] + .... + movw $LOAD,%bx # Address for read + movb $0x2,%ah # Read sector + callw intx13 # from disk + jc beep # If error + cmpw $MAGIC,0x1fe(%bx) # Bootable? + jne beep # No + pushw %si # Save ptr to selected part. + callw putn # Leave some space + popw %si # Restore, next stage uses it + jmp *%bx # Invoke bootstrap + .... + + .[.filename]#stand/i386/boot0/boot0.S# [[boot-boot0-check-bootable]] + Recall that `%si` points to the selected partition entry. + This entry tells us where the partition begins on disk. + We assume, of course, that the partition selected is actually a FreeBSD slice. + + [NOTE] + ==== + From now on, we will favor the use of the technically more accurate term "slice" rather than "partition". + ==== + + The transfer buffer is set to `0x7c00` (register `%bx`), and a read for the first sector of the FreeBSD slice is requested by calling `intx13`. + We assume that everything went okay, so a jump to `beep` is not performed. + In particular, the new sector read must end with the magic sequence `0xaa55`. + Finally, the value at `%si` (the pointer to the selected partition table) is preserved for use by the next stage, and a jump is performed to address `0x7c00`, where execution of our next stage (the just-read block) is started. + + [[boot-boot1]] + == `boot1` Stage + + So far we have gone through the following sequence: + + * The BIOS did some early hardware initialization, including the POST. + The MBR ([.filename]#boot0#) was loaded from absolute disk sector one to address `0x7c00`. + Execution control was passed to that location. + * [.filename]#boot0# relocated itself to the location it was linked to execute (`0x600`), followed by a jump to continue execution at the appropriate place. + Finally, [.filename]#boot0# loaded the first disk sector from the FreeBSD slice to address `0x7c00`. + Execution control was passed to that location. + + [.filename]#boot1# is the next step in the boot-loading sequence. + It is the first of three boot stages. + Note that we have been dealing exclusively with disk sectors. + Indeed, the BIOS loads the absolute first sector, while [.filename]#boot0# loads the first sector of the FreeBSD slice. + Both loads are to address `0x7c00`. + We can conceptually think of these disk sectors as containing the files [.filename]#boot0# and [.filename]#boot1#, respectively, but in reality this is not entirely true for [.filename]#boot1#. + Strictly speaking, unlike [.filename]#boot0#, [.filename]#boot1# is not part of the boot blocks footnote:[There is a file /boot/boot1, but it is not the written to the beginning of the FreeBSD slice. + Instead, it is concatenated with boot2 to form boot, which is written to the beginning of the FreeBSD slice and read at boot time.]. + Instead, a single, full-blown file, [.filename]#boot# ([.filename]#/boot/boot#), is what ultimately is written to disk. + This file is a combination of [.filename]#boot1#, [.filename]#boot2# and the `Boot Extender` (or BTX). + This single file is greater in size than a single sector (greater than 512 bytes). + Fortunately, [.filename]#boot1# occupies _exactly_ the first 512 bytes of this single file, so when [.filename]#boot0# loads the first sector of the FreeBSD slice (512 bytes), it is actually loading [.filename]#boot1# and transferring control to it. + + The main task of [.filename]#boot1# is to load the next boot stage. + This next stage is somewhat more complex. + It is composed of a server called the "Boot Extender", or BTX, and a client, called [.filename]#boot2#. + As we will see, the last boot stage, [.filename]#loader#, is also a client of the BTX server. + + Let us now look in detail at what exactly is done by [.filename]#boot1#, starting like we did for [.filename]#boot0#, at its entry point: + + [.programlisting] + .... + start: + jmp main + .... + + .[.filename]#stand/i386/boot2/boot1.S# [[boot-boot1-entry]] + The entry point at `start` simply jumps past a special data area to the label `main`, which in turn looks like this: + + [.programlisting] + .... + main: + cld # String ops inc + xor %cx,%cx # Zero + mov %cx,%es # Address + mov %cx,%ds # data + mov %cx,%ss # Set up + mov $start,%sp # stack + mov %sp,%si # Source + mov $MEM_REL,%di # Destination + incb %ch # Word count + rep # Copy + movsw # code + .... + + .[.filename]#stand/i386/boot2/boot1.S# [[boot-boot1-main]] + Just like [.filename]#boot0#, this code relocates [.filename]#boot1#, this time to memory address `0x700`. + However, unlike [.filename]#boot0#, it does not jump there. + [.filename]#boot1# is linked to execute at address `0x7c00`, effectively where it was loaded in the first place. + The reason for this relocation will be discussed shortly. + + Next comes a loop that looks for the FreeBSD slice. + Although [.filename]#boot0# loaded [.filename]#boot1# from the FreeBSD slice, no information was passed to it about this footnote:[Actually we did pass a pointer to the slice entry in register %si. + However, boot1 does not assume that it was loaded by boot0 (perhaps some other MBR loaded it, and did not pass this information), so it assumes nothing.], so [.filename]#boot1# must rescan the partition table to find where the FreeBSD slice starts. + Therefore it rereads the MBR: + + [.programlisting] + .... + mov $part4,%si # Partition + cmpb $0x80,%dl # Hard drive? + jb main.4 # No + movb $0x1,%dh # Block count + callw nread # Read MBR + .... + + .[.filename]#stand/i386/boot2/boot1.S# [[boot-boot1-find-freebsd]] + In the code above, register `%dl` maintains information about the boot device. + This is passed on by the BIOS and preserved by the MBR. + Numbers `0x80` and greater tells us that we are dealing with a hard drive, so a call is made to `nread`, where the MBR is read. + Arguments to `nread` are passed through `%si` and `%dh`. + The memory address at label `part4` is copied to `%si`. + This memory address holds a "fake partition" to be used by `nread`. + The following is the data in the fake partition: + + [.programlisting] + .... + part4: + .byte 0x80, 0x00, 0x01, 0x00 + .byte 0xa5, 0xfe, 0xff, 0xff + .byte 0x00, 0x00, 0x00, 0x00 + .byte 0x50, 0xc3, 0x00, 0x00 + .... + + .[.filename]#stand/i386/boot2/boot1.S# [[boot-boot2-make-fake-partition]] + In particular, the LBA for this fake partition is hardcoded to zero. + This is used as an argument to the BIOS for reading absolute sector one from the hard drive. + Alternatively, CHS addressing could be used. + In this case, the fake partition holds cylinder 0, head 0 and sector 1, which is equivalent to absolute sector one. + + Let us now proceed to take a look at `nread`: + + [.programlisting] + .... + nread: + mov $MEM_BUF,%bx # Transfer buffer + mov 0x8(%si),%ax # Get + mov 0xa(%si),%cx # LBA + push %cs # Read from + callw xread.1 # disk + jnc return # If success, return + .... + + .[.filename]#stand/i386/boot2/boot1.S# [[boot-boot1-nread]] + Recall that `%si` points to the fake partition. + The word footnote:[In the context of 16-bit real mode, a word is 2 bytes.] at offset `0x8` is copied to register `%ax` and word at offset `0xa` to `%cx`. + They are interpreted by the BIOS as the lower 4-byte value denoting the LBA to be read (the upper four bytes are assumed to be zero). + Register `%bx` holds the memory address where the MBR will be loaded. + The instruction pushing `%cs` onto the stack is very interesting. + In this context, it accomplishes nothing. + However, as we will see shortly, [.filename]#boot2#, in conjunction with the BTX server, also uses `xread.1`. + This mechanism will be discussed in the next section. + + The code at `xread.1` further calls the `read` function, which actually calls the BIOS asking for the disk sector: + + [.programlisting] + .... + xread.1: + pushl $0x0 # absolute + push %cx # block + push %ax # number + push %es # Address of + push %bx # transfer buffer + xor %ax,%ax # Number of + movb %dh,%al # blocks to + push %ax # transfer + push $0x10 # Size of packet + mov %sp,%bp # Packet pointer + callw read # Read from disk + lea 0x10(%bp),%sp # Clear stack + lret # To far caller + .... + + .[.filename]#stand/i386/boot2/boot1.S# [[boot-boot1-xread1]] + Note the long return instruction at the end of this block. + This instruction pops out the `%cs` register pushed by `nread`, and returns. + Finally, `nread` also returns. + + With the MBR loaded to memory, the actual loop for searching the FreeBSD slice begins: + + [.programlisting] + .... + mov $0x1,%cx # Two passes + main.1: + mov $MEM_BUF+PRT_OFF,%si # Partition table + movb $0x1,%dh # Partition + main.2: + cmpb $PRT_BSD,0x4(%si) # Our partition type? + jne main.3 # No + jcxz main.5 # If second pass + testb $0x80,(%si) # Active? + jnz main.5 # Yes + main.3: + add $0x10,%si # Next entry + incb %dh # Partition + cmpb $0x1+PRT_NUM,%dh # In table? + jb main.2 # Yes + dec %cx # Do two + jcxz main.1 # passes + .... + + .[.filename]#stand/i386/boot2/boot1.S# [[boot-boot1-find-part]] + If a FreeBSD slice is identified, execution continues at `main.5`. + Note that when a FreeBSD slice is found `%si` points to the appropriate entry in the partition table, and `%dh` holds the partition number. + We assume that a FreeBSD slice is found, so we continue execution at `main.5`: + + [.programlisting] + .... + main.5: + mov %dx,MEM_ARG # Save args + movb $NSECT,%dh # Sector count + callw nread # Read disk + mov $MEM_BTX,%bx # BTX + mov 0xa(%bx),%si # Get BTX length and set + add %bx,%si # %si to start of boot2.bin + mov $MEM_USR+SIZ_PAG*2,%di # Client page 2 + mov $MEM_BTX+(NSECT-1)*SIZ_SEC,%cx # Byte + sub %si,%cx # count + rep # Relocate + movsb # client + .... + + .[.filename]#stand/i386/boot2/boot1.S# [[boot-boot1-main5]] + Recall that at this point, register `%si` points to the FreeBSD slice entry in the MBR partition table, so a call to `nread` will effectively read sectors at the beginning of this partition. + The argument passed on register `%dh` tells `nread` to read 16 disk sectors. + Recall that the first 512 bytes, or the first sector of the FreeBSD slice, coincides with the [.filename]#boot1# program. + Also recall that the file written to the beginning of the FreeBSD slice is not [.filename]#/boot/boot1#, but [.filename]#/boot/boot#. + Let us look at the size of these files in the filesystem: + + [source,bash] + .... + -r--r--r-- 1 root wheel 512B Jan 8 00:15 /boot/boot0 + -r--r--r-- 1 root wheel 512B Jan 8 00:15 /boot/boot1 + -r--r--r-- 1 root wheel 7.5K Jan 8 00:15 /boot/boot2 + -r--r--r-- 1 root wheel 8.0K Jan 8 00:15 /boot/boot + .... + + Both [.filename]#boot0# and [.filename]#boot1# are 512 bytes each, so they fit _exactly_ in one disk sector. + [.filename]#boot2# is much bigger, holding both the BTX server and the [.filename]#boot2# client. + Finally, a file called simply [.filename]#boot# is 512 bytes larger than [.filename]#boot2#. + This file is a concatenation of [.filename]#boot1# and [.filename]#boot2#. + As already noted, [.filename]#boot0# is the file written to the absolute first disk sector (the MBR), and [.filename]#boot# is the file written to the first sector of the FreeBSD slice; [.filename]#boot1# and [.filename]#boot2# are _not_ written to disk. + The command used to concatenate [.filename]#boot1# and [.filename]#boot2# into a single [.filename]#boot# is merely `cat boot1 boot2 > boot`. + + So [.filename]#boot1# occupies exactly the first 512 bytes of [.filename]#boot# and, because [.filename]#boot# is written to the first sector of the FreeBSD slice, [.filename]#boot1# fits exactly in this first sector. + When `nread` reads the first 16 sectors of the FreeBSD slice, it effectively reads the entire [.filename]#boot# file footnote:[512*16=8192 bytes, exactly the size of boot]. + We will see more details about how [.filename]#boot# is formed from [.filename]#boot1# and [.filename]#boot2# in the next section. + + Recall that `nread` uses memory address `0x8c00` as the transfer buffer to hold the sectors read. + This address is conveniently chosen. + Indeed, because [.filename]#boot1# belongs to the first 512 bytes, it ends up in the address range `0x8c00`-`0x8dff`. + The 512 bytes that follows (range `0x8e00`-`0x8fff`) is used to store the _bsdlabel_ footnote:[Historically known as disklabel. + If you ever wondered where FreeBSD stored this information, it is in this region - see man:bsdlabel[8]]. + + Starting at address `0x9000` is the beginning of the BTX server, and immediately following is the [.filename]#boot2# client. + The BTX server acts as a kernel, and executes in protected mode in the most privileged level. + In contrast, the BTX clients ([.filename]#boot2#, for example), execute in user mode. + We will see how this is accomplished in the next section. + The code after the call to `nread` locates the beginning of [.filename]#boot2# in the memory buffer, and copies it to memory address `0xc000`. + This is because the BTX server arranges [.filename]#boot2# to execute in a segment starting at `0xa000`. + We explore this in detail in the following section. + + The last code block of [.filename]#boot1# enables access to memory above 1MB footnote:[This is necessary for legacy reasons. + Interested readers should see .] and concludes with a jump to the starting point of the BTX server: + + [.programlisting] + .... + seta20: + cli # Disable interrupts + seta20.1: + dec %cx # Timeout? + jz seta20.3 # Yes + + inb $0x64,%al # Get status + testb $0x2,%al # Busy? + jnz seta20.1 # Yes + movb $0xd1,%al # Command: Write + outb %al,$0x64 # output port + seta20.2: + inb $0x64,%al # Get status + testb $0x2,%al # Busy? + jnz seta20.2 # Yes + movb $0xdf,%al # Enable + outb %al,$0x60 # A20 + seta20.3: + sti # Enable interrupts + jmp 0x9010 # Start BTX + .... + + .[.filename]#stand/i386/boot2/boot1.S# [[boot-boot1-seta20]] + Note that right before the jump, interrupts are enabled. + + [[btx-server]] + == The BTX Server + + Next in our boot sequence is the BTX Server. + Let us quickly remember how we got here: + + * The BIOS loads the absolute sector one (the MBR, or [.filename]#boot0#), to address `0x7c00` and jumps there. + * [.filename]#boot0# relocates itself to `0x600`, the address it was linked to execute, and jumps over there. + It then reads the first sector of the FreeBSD slice (which consists of [.filename]#boot1#) into address `0x7c00` and jumps over there. + * [.filename]#boot1# loads the first 16 sectors of the FreeBSD slice into address `0x8c00`. + This 16 sectors, or 8192 bytes, is the whole file [.filename]#boot#. + The file is a concatenation of [.filename]#boot1# and [.filename]#boot2#. + [.filename]#boot2#, in turn, contains the BTX server and the [.filename]#boot2# client. + Finally, a jump is made to address `0x9010`, the entry point of the BTX server. + + Before studying the BTX Server in detail, let us further review how the single, all-in-one [.filename]#boot# file is created. + The way [.filename]#boot# is built is defined in its [.filename]#Makefile# ([.filename]#stand/i386/boot2/Makefile#). + Let us look at the rule that creates the [.filename]#boot# file: + + [.programlisting] + .... + boot: boot1 boot2 + cat boot1 boot2 > boot + .... + + .[.filename]#stand/i386/boot2/Makefile# [[boot-boot1-make-boot]] + This tells us that [.filename]#boot1# and [.filename]#boot2# are needed, and the rule simply concatenates them to produce a single file called [.filename]#boot#. + The rules for creating [.filename]#boot1# are also quite simple: + + [.programlisting] + .... + boot1: boot1.out + ${OBJCOPY} -S -O binary boot1.out ${.TARGET} + + boot1.out: boot1.o + ${LD} ${LD_FLAGS} -e start --defsym ORG=${ORG1} -T ${LDSCRIPT} -o ${.TARGET} boot1.o + .... + + .[.filename]#stand/i386/boot2/Makefile# [[boot-boot1-make-boot1]] + To apply the rule for creating [.filename]#boot1#, [.filename]#boot1.out# must be resolved. + This, in turn, depends on the existence of [.filename]#boot1.o#. + This last file is simply the result of assembling our familiar [.filename]#boot1.S#, without linking. + Now, the rule for creating [.filename]#boot1.out# is applied. + This tells us that [.filename]#boot1.o# should be linked with `start` as its entry point, and starting at address `0x7c00`. + Finally, [.filename]#boot1# is created from [.filename]#boot1.out# applying the appropriate rule. + This rule is the [.filename]#objcopy# command applied to [.filename]#boot1.out#. + Note the flags passed to [.filename]#objcopy#: `-S` tells it to strip all relocation and symbolic information; `-O binary` indicates the output format, that is, a simple, unformatted binary file. + + Having [.filename]#boot1#, let us take a look at how [.filename]#boot2# is constructed: + + [.programlisting] + .... + boot2: boot2.ld + @set -- `ls -l ${.ALLSRC}`; x=$$((${BOOT2SIZE}-$$5)); \ + echo "$$x bytes available"; test $$x -ge 0 + ${DD} if=${.ALLSRC} of=${.TARGET} bs=${BOOT2SIZE} conv=sync + + boot2.ld: boot2.ldr boot2.bin ${BTXKERN} + btxld -v -E ${ORG2} -f bin -b ${BTXKERN} -l boot2.ldr \ + -o ${.TARGET} -P 1 boot2.bin + + boot2.ldr: + ${DD} if=/dev/zero of=${.TARGET} bs=512 count=1 + + boot2.bin: boot2.out + ${OBJCOPY} -S -O binary boot2.out ${.TARGET} + + boot2.out: ${BTXCRT} boot2.o sio.o ashldi3.o + ${LD} ${LD_FLAGS} --defsym ORG=${ORG2} -T ${LDSCRIPT} -o ${.TARGET} ${.ALLSRC} + + boot2.h: boot1.out + ${NM} -t d ${.ALLSRC} | awk '/([0-9])+ T xread/ \ + { x = $$1 - ORG1; \ + printf("#define XREADORG %#x\n", REL1 + x) }' \ + ORG1=`printf "%d" ${ORG1}` \ + REL1=`printf "%d" ${REL1}` > ${.TARGET} + .... + + .[.filename]#stand/i386/boot2/Makefile# [[boot-boot1-make-boot2]] + The mechanism for building [.filename]#boot2# is far more elaborate. + Let us point out the most relevant facts. + The dependency list is as follows: + + [.programlisting] + .... + boot2: boot2.ld + boot2.ld: boot2.ldr boot2.bin ${BTXDIR} + boot2.bin: boot2.out + boot2.out: ${BTXDIR} boot2.o sio.o ashldi3.o + boot2.h: boot1.out + .... + + .[.filename]#stand/i386/boot2/Makefile# [[boot-boot1-make-boot2-more]] + Note that initially there is no header file [.filename]#boot2.h#, but its creation depends on [.filename]#boot1.out#, which we already have. + The rule for its creation is a bit terse, but the important thing is that the output, [.filename]#boot2.h#, is something like this: + + [.programlisting] + .... + #define XREADORG 0x725 + .... + + .[.filename]#stand/i386/boot2/boot2.h# [[boot-boot1-make-boot2h]] + Recall that [.filename]#boot1# was relocated (i.e., copied from `0x7c00` to `0x700`). + This relocation will now make sense, because as we will see, the BTX server reclaims some memory, including the space where [.filename]#boot1# was originally loaded. + However, the BTX server needs access to [.filename]#boot1#'s `xread` function; this function, according to the output of [.filename]#boot2.h#, is at location `0x725`. + Indeed, the BTX server uses the `xread` function from [.filename]#boot1#'s relocated code. + This function is now accessible from within the [.filename]#boot2# client. + + The next rule directs the linker to link various files ([.filename]#ashldi3.o#, [.filename]#boot2.o# and [.filename]#sio.o#). + Note that the output file, [.filename]#boot2.out#, is linked to execute at address `0x2000` (${ORG2}). + Recall that [.filename]#boot2# will be executed in user mode, within a special user segment set up by the BTX server. + This segment starts at `0xa000`. + Also, remember that the [.filename]#boot2# portion of [.filename]#boot# was copied to address `0xc000`, that is, offset `0x2000` from the start of the user segment, so [.filename]#boot2# will work properly when we transfer control to it. + Next, [.filename]#boot2.bin# is created from [.filename]#boot2.out# by stripping its symbols and format information; boot2.bin is a _raw_ binary. + Now, note that a file [.filename]#boot2.ldr# is created as a 512-byte file full of zeros. + This space is reserved for the bsdlabel. + + Now that we have files [.filename]#boot1#, [.filename]#boot2.bin# and [.filename]#boot2.ldr#, only the BTX server is missing before creating the all-in-one [.filename]#boot# file. + The BTX server is located in [.filename]#stand/i386/btx/btx#; it has its own [.filename]#Makefile# with its own set of rules for building. + The important thing to notice is that it is also compiled as a _raw_ binary, and that it is linked to execute at address `0x9000`. + The details can be found in [.filename]#stand/i386/btx/btx/Makefile#. + + Having the files that comprise the [.filename]#boot# program, the final step is to _merge_ them. + This is done by a special program called [.filename]#btxld# (source located in [.filename]#/usr/src/usr.sbin/btxld#). + Some arguments to this program include the name of the output file ([.filename]#boot#), its entry point (`0x2000`) and its file format (raw binary). + The various files are finally merged by this utility into the file [.filename]#boot#, which consists of [.filename]#boot1#, [.filename]#boot2#, the `bsdlabel` and the BTX server. + This file, which takes exactly 16 sectors, or 8192 bytes, is what is actually written to the beginning of the FreeBSD slice during installation. + Let us now proceed to study the BTX server program. + + The BTX server prepares a simple environment and switches from 16-bit real mode to 32-bit protected mode, right before passing control to the client. + This includes initializing and updating the following data structures: + + * Modifies the `Interrupt Vector Table (IVT)`. + The IVT provides exception and interrupt handlers for Real-Mode code. + * The `Interrupt Descriptor Table (IDT)` is created. + Entries are provided for processor exceptions, hardware interrupts, two system calls and V86 interface. + The IDT provides exception and interrupt handlers for Protected-Mode code. + * A `Task-State Segment (TSS)` is created. + This is necessary because the processor works in the _least_ privileged level when executing the client ([.filename]#boot2#), but in the _most_ privileged level when executing the BTX server. + * The GDT (Global Descriptor Table) is set up. + Entries (descriptors) are provided for supervisor code and data, user code and data, and real-mode code and data. + footnote:[Real-mode code and data are necessary when switching back to real mode from protected mode, as suggested by the Intel manuals.] + + Let us now start studying the actual implementation. + Recall that [.filename]#boot1# made a jump to address `0x9010`, the BTX server's entry point. + Before studying program execution there, note that the BTX server has a special header at address range `0x9000-0x900f`, right before its entry point. + This header is defined as follows: + + [.programlisting] + .... + start: # Start of code + /* + * BTX header. + */ + btx_hdr: .byte 0xeb # Machine ID + .byte 0xe # Header size + .ascii "BTX" # Magic + .byte 0x1 # Major version + .byte 0x2 # Minor version + .byte BTX_FLAGS # Flags + .word PAG_CNT-MEM_ORG>>0xc # Paging control + .word break-start # Text size + .long 0x0 # Entry address + .... + + .[.filename]#stand/i386/btx/btx/btx.S# [[btx-header]] + Note the first two bytes are `0xeb` and `0xe`. + In the IA-32 architecture, these two bytes are interpreted as a relative jump past the header into the entry point, so in theory, [.filename]#boot1# could jump here (address `0x9000`) instead of address `0x9010`. + Note that the last field in the BTX header is a pointer to the client's ([.filename]#boot2#) entry pointb2. + This field is patched at link time. + + Immediately following the header is the BTX server's entry point: + + [.programlisting] + .... + /* + * Initialization routine. + */ + init: cli # Disable interrupts + xor %ax,%ax # Zero/segment + mov %ax,%ss # Set up + mov $MEM_ESP0,%sp # stack + mov %ax,%es # Address + mov %ax,%ds # data + pushl $0x2 # Clear + popfl # flags + .... + + .[.filename]#stand/i386/btx/btx/btx.S# [[btx-init]] + This code disables interrupts, sets up a working stack (starting at address `0x1800`) and clears the flags in the EFLAGS register. + Note that the `popfl` instruction pops out a doubleword (4 bytes) from the stack and places it in the EFLAGS register. + As the value actually popped is `2`, the EFLAGS register is effectively cleared (IA-32 requires that bit 2 of the EFLAGS register always be 1). + + Our next code block clears (sets to `0`) the memory range `0x5e00-0x8fff`. + This range is where the various data structures will be created: + + [.programlisting] + .... + /* + * Initialize memory. + */ + mov $MEM_IDT,%di # Memory to initialize + mov $(MEM_ORG-MEM_IDT)/2,%cx # Words to zero + rep # Zero-fill + stosw # memory + .... + + .[.filename]#stand/i386/btx/btx/btx.S# [[btx-clear-mem]] + Recall that [.filename]#boot1# was originally loaded to address `0x7c00`, so, with this memory initialization, that copy effectively disappeared. + However, also recall that [.filename]#boot1# was relocated to `0x700`, so _that_ copy is still in memory, and the BTX server will make use of it. + + Next, the real-mode IVT (Interrupt Vector Table is updated. + The IVT is an array of segment/offset pairs for exception and interrupt handlers. + The BIOS normally maps hardware interrupts to interrupt vectors `0x8` to `0xf` and `0x70` to `0x77` but, as will be seen, the 8259A Programmable Interrupt Controller, the chip controlling the actual mapping of hardware interrupts to interrupt vectors, is programmed to remap these interrupt vectors from `0x8-0xf` to `0x20-0x27` and from `0x70-0x77` to `0x28-0x2f`. + Thus, interrupt handlers are provided for interrupt vectors `0x20-0x2f`. + The reason the BIOS-provided handlers are not used directly is because they work in 16-bit real mode, but not 32-bit protected mode. + Processor mode will be switched to 32-bit protected mode shortly. + However, the BTX server sets up a mechanism to effectively use the handlers provided by the BIOS: + + [.programlisting] + .... + /* + * Update real mode IDT for reflecting hardware interrupts. + */ + mov $intr20,%bx # Address first handler + mov $0x10,%cx # Number of handlers + mov $0x20*4,%di # First real mode IDT entry + init.0: mov %bx,(%di) # Store IP + inc %di # Address next + inc %di # entry + stosw # Store CS + add $4,%bx # Next handler + loop init.0 # Next IRQ + .... + + .[.filename]#stand/i386/btx/btx/btx.S# [[btx-ivt]] + The next block creates the IDT (Interrupt Descriptor Table). + The IDT is analogous, in protected mode, to the IVT in real mode. + That is, the IDT describes the various exception and interrupt handlers used when the processor is executing in protected mode. + In essence, it also consists of an array of segment/offset pairs, although the structure is somewhat more complex, because segments in protected mode are different than in real mode, and various protection mechanisms apply: + + [.programlisting] + .... + /* + * Create IDT. + */ + mov $MEM_IDT,%di # IDT's address + mov $idtctl,%si # Control string + init.1: lodsb # Get entry + cbw # count + xchg %ax,%cx # as word + jcxz init.4 # If done + lodsb # Get segment + xchg %ax,%dx # P:DPL:type + lodsw # Get control + xchg %ax,%bx # set + lodsw # Get handler offset + mov $SEL_SCODE,%dh # Segment selector + init.2: shr %bx # Handle this int? + jnc init.3 # No + mov %ax,(%di) # Set handler offset + mov %dh,0x2(%di) # and selector + mov %dl,0x5(%di) # Set P:DPL:type + add $0x4,%ax # Next handler + init.3: lea 0x8(%di),%di # Next entry + loop init.2 # Till set done + jmp init.1 # Continue + .... + + .[.filename]#stand/i386/btx/btx/btx.S# [[btx-idt]] + Each entry in the `IDT` is 8 bytes long. + Besides the segment/offset information, they also describe the segment type, privilege level, and whether the segment is present in memory or not. + The construction is such that interrupt vectors from `0` to `0xf` (exceptions) are handled by function `intx00`; vector `0x10` (also an exception) is handled by `intx10`; hardware interrupts, which are later configured to start at interrupt vector `0x20` all the way to interrupt vector `0x2f`, are handled by function `intx20`. + Lastly, interrupt vector `0x30`, which is used for system calls, is handled by `intx30`, and vectors `0x31` and `0x32` are handled by `intx31`. + It must be noted that only descriptors for interrupt vectors `0x30`, `0x31` and `0x32` are given privilege level 3, the same privilege level as the [.filename]#boot2# client, which means the client can execute a software-generated interrupt to this vectors through the `int` instruction without failing (this is the way [.filename]#boot2# use the services provided by the BTX server). + Also, note that _only_ software-generated interrupts are protected from code executing in lesser privilege levels. + Hardware-generated interrupts and processor-generated exceptions are _always_ handled adequately, regardless of the actual privileges involved. + + The next step is to initialize the TSS (Task-State Segment). + The TSS is a hardware feature that helps the operating system or executive software implement multitasking functionality through process abstraction. + The IA-32 architecture demands the creation and use of _at least_ one TSS if multitasking facilities are used or different privilege levels are defined. + Since the [.filename]#boot2# client is executed in privilege level 3, but the BTX server runs in privilege level 0, a TSS must be defined: + + [.programlisting] + .... + /* + * Initialize TSS. + */ + init.4: movb $_ESP0H,TSS_ESP0+1(%di) # Set ESP0 + movb $SEL_SDATA,TSS_SS0(%di) # Set SS0 + movb $_TSSIO,TSS_MAP(%di) # Set I/O bit map base + .... + + .[.filename]#stand/i386/btx/btx/btx.S# [[btx-tss]] + Note that a value is given for the Privilege Level 0 stack pointer and stack segment in the TSS. + This is needed because, if an interrupt or exception is received while executing [.filename]#boot2# in Privilege Level 3, a change to Privilege Level 0 is automatically performed by the processor, so a new working stack is needed. + Finally, the I/O Map Base Address field of the TSS is given a value, which is a 16-bit offset from the beginning of the TSS to the I/O Permission Bitmap and the Interrupt Redirection Bitmap. + + After the IDT and TSS are created, the processor is ready to switch to protected mode. + This is done in the next block: + + [.programlisting] + .... + /* + * Bring up the system. + */ + mov $0x2820,%bx # Set protected mode + callw setpic # IRQ offsets + lidt idtdesc # Set IDT + lgdt gdtdesc # Set GDT + mov %cr0,%eax # Switch to protected + inc %ax # mode + mov %eax,%cr0 # + ljmp $SEL_SCODE,$init.8 # To 32-bit code + .code32 + init.8: xorl %ecx,%ecx # Zero + movb $SEL_SDATA,%cl # To 32-bit + movw %cx,%ss # stack + .... + + .[.filename]#stand/i386/btx/btx/btx.S# [[btx-prot]] + First, a call is made to `setpic` to program the 8259A PIC (Programmable Interrupt Controller). + This chip is connected to multiple hardware interrupt sources. + Upon receiving an interrupt from a device, it signals the processor with the appropriate interrupt vector. + This can be customized so that specific interrupts are associated with specific interrupt vectors, as explained before. + Next, the IDTR (Interrupt Descriptor Table Register) and GDTR (Global Descriptor Table Register) are loaded with the instructions `lidt` and `lgdt`, respectively. + These registers are loaded with the base address and limit address for the IDT and GDT. + The following three instructions set the Protection Enable (PE) bit of the `%cr0` register. + This effectively switches the processor to 32-bit protected mode. + Next, a long jump is made to `init.8` using segment selector SEL_SCODE, which selects the Supervisor Code Segment. + The processor is effectively executing in CPL 0, the most privileged level, after this jump. + Finally, the Supervisor Data Segment is selected for the stack by assigning the segment selector SEL_SDATA to the `%ss` register. + This data segment also has a privilege level of `0`. + + Our last code block is responsible for loading the TR (Task Register) with the segment selector for the TSS we created earlier, and setting the User Mode environment before passing execution control to the [.filename]#boot2# client. + + [.programlisting] + .... + /* + * Launch user task. + */ + movb $SEL_TSS,%cl # Set task + ltr %cx # register + movl $MEM_USR,%edx # User base address + movzwl %ss:BDA_MEM,%eax # Get free memory + shll $0xa,%eax # To bytes + subl $ARGSPACE,%eax # Less arg space + subl %edx,%eax # Less base + movb $SEL_UDATA,%cl # User data selector + pushl %ecx # Set SS + pushl %eax # Set ESP + push $0x202 # Set flags (IF set) + push $SEL_UCODE # Set CS + pushl btx_hdr+0xc # Set EIP + pushl %ecx # Set GS + pushl %ecx # Set FS + pushl %ecx # Set DS + pushl %ecx # Set ES + pushl %edx # Set EAX + movb $0x7,%cl # Set remaining + init.9: push $0x0 # general + loop init.9 # registers + #ifdef BTX_SERIAL + call sio_init # setup the serial console + #endif + popa # and initialize + popl %es # Initialize + popl %ds # user + popl %fs # segment + popl %gs # registers + iret # To user mode + .... + + .[.filename]#stand/i386/btx/btx/btx.S# [[btx-end]] + Note that the client's environment include a stack segment selector and stack pointer (registers `%ss` and `%esp`). + Indeed, once the TR is loaded with the appropriate stack segment selector (instruction `ltr`), the stack pointer is calculated and pushed onto the stack along with the stack's segment selector. + Next, the value `0x202` is pushed onto the stack; it is the value that the EFLAGS will get when control is passed to the client. + Also, the User Mode code segment selector and the client's entry point are pushed. + Recall that this entry point is patched in the BTX header at link time. + Finally, segment selectors (stored in register `%ecx`) for the segment registers `%gs, %fs, %ds and %es` are pushed onto the stack, along with the value at `%edx` (`0xa000`). + Keep in mind the various values that have been pushed onto the stack (they will be popped out shortly). + Next, values for the remaining general purpose registers are also pushed onto the stack (note the `loop` that pushes the value `0` seven times). + Now, values will be started to be popped out of the stack. + First, the `popa` instruction pops out of the stack the latest seven values pushed. + They are stored in the general purpose registers in order `%edi, %esi, %ebp, %ebx, %edx, %ecx, %eax`. + Then, the various segment selectors pushed are popped into the various segment registers. + Five values still remain on the stack. + They are popped when the `iret` instruction is executed. + This instruction first pops the value that was pushed from the BTX header. + This value is a pointer to [.filename]#boot2#'s entry point. + It is placed in the register `%eip`, the instruction pointer register. + Next, the segment selector for the User Code Segment is popped and copied to register `%cs`. + Remember that this segment's privilege level is 3, the least privileged level. + This means that we must provide values for the stack of this privilege level. + This is why the processor, besides further popping the value for the EFLAGS register, does two more pops out of the stack. + These values go to the stack pointer (`%esp`) and the stack segment (`%ss`). + Now, execution continues at ``boot0``'s entry point. + + It is important to note how the User Code Segment is defined. + This segment's _base address_ is set to `0xa000`. + This means that code memory addresses are _relative_ to address 0xa000; if code being executed is fetched from address `0x2000`, the _actual_ memory addressed is `0xa000+0x2000=0xc000`. + + [[boot2]] + == boot2 Stage + + `boot2` defines an important structure, `struct bootinfo`. + This structure is initialized by `boot2` and passed to the loader, and then further to the kernel. + Some nodes of this structures are set by `boot2`, the rest by the loader. + This structure, among other information, contains the kernel filename, BIOS harddisk geometry, BIOS drive number for boot device, physical memory available, `envp` pointer etc. + The definition for it is: + + [.programlisting] + .... + /usr/include/machine/bootinfo.h: + struct bootinfo { + u_int32_t bi_version; + u_int32_t bi_kernelname; /* represents a char * */ + u_int32_t bi_nfs_diskless; /* struct nfs_diskless * */ + /* End of fields that are always present. */ + #define bi_endcommon bi_n_bios_used + u_int32_t bi_n_bios_used; + u_int32_t bi_bios_geom[N_BIOS_GEOM]; + u_int32_t bi_size; + u_int8_t bi_memsizes_valid; + u_int8_t bi_bios_dev; /* bootdev BIOS unit number */ + u_int8_t bi_pad[2]; + u_int32_t bi_basemem; + u_int32_t bi_extmem; + u_int32_t bi_symtab; /* struct symtab * */ + u_int32_t bi_esymtab; /* struct symtab * */ + /* Items below only from advanced bootloader */ + u_int32_t bi_kernend; /* end of kernel space */ + u_int32_t bi_envp; /* environment */ + u_int32_t bi_modulep; /* preloaded modules */ + }; + .... + + `boot2` enters into an infinite loop waiting for user input, then calls `load()`. + If the user does not press anything, the loop breaks by a timeout, so `load()` will load the default file ([.filename]#/boot/loader#). + Functions `ino_t lookup(char *filename)` and `int xfsread(ino_t inode, void *buf, size_t nbyte)` are used to read the content of a file into memory. + [.filename]#/boot/loader# is an ELF binary, but where the ELF header is prepended with [.filename]#a.out#'s `struct exec` structure. + `load()` scans the loader's ELF header, loading the content of [.filename]#/boot/loader# into memory, and passing the execution to the loader's entry: + + [.programlisting] + .... + stand/i386/boot2/boot2.c: + __exec((caddr_t)addr, RB_BOOTINFO | (opts & RBX_MASK), + MAKEBOOTDEV(dev_maj[dsk.type], dsk.slice, dsk.unit, dsk.part), + 0, 0, 0, VTOP(&bootinfo)); + .... + + [[boot-loader]] + == loader Stage + + loader is a BTX client as well. + I will not describe it here in detail, there is a comprehensive man page written by Mike Smith, man:loader[8]. + The underlying mechanisms and BTX were discussed above. + + The main task for the loader is to boot the kernel. + When the kernel is loaded into memory, it is being called by the loader: + + [.programlisting] + .... + stand/common/boot.c: + /* Call the exec handler from the loader matching the kernel */ + file_formats[fp->f_loader]->l_exec(fp); + .... + + [[boot-kernel]] + == Kernel Initialization + + Let us take a look at the command that links the kernel. + This will help identify the exact location where the loader passes execution to the kernel. + This location is the kernel's actual entry point. + This command is now excluded from [.filename]#sys/conf/Makefile.i386#. + The content that interests us can be found in [.filename]#/usr/obj/usr/src/i386.i386/sys/GENERIC/#. + + [.programlisting] + .... + /usr/obj/usr/src/i386.i386/sys/GENERIC/kernel.meta: + ld -m elf_i386_fbsd -Bdynamic -T /usr/src/sys/conf/ldscript.i386 --build-id=sha1 --no-warn-mismatch \ + --warn-common --export-dynamic --dynamic-linker /red/herring -X -o kernel locore.o + + .... + + A few interesting things can be seen here. + First, the kernel is an ELF dynamically linked binary, but the dynamic linker for kernel is [.filename]#/red/herring#, which is definitely a bogus file. + Second, taking a look at the file [.filename]#sys/conf/ldscript.i386# gives an idea about what ld options are used when compiling a kernel. + Reading through the first few lines, the string + + [.programlisting] + .... + sys/conf/ldscript.i386: + ENTRY(btext) + .... + + says that a kernel's entry point is the symbol `btext`. + This symbol is defined in [.filename]#locore.s#: + + [.programlisting] + .... + sys/i386/i386/locore.s: + .text + /********************************************************************** + * + * This is where the bootblocks start us, set the ball rolling... + * + */ + NON_GPROF_ENTRY(btext) + .... + + First, the register EFLAGS is set to a predefined value of 0x00000002. + Then all the segment registers are initialized: + + [.programlisting] + .... + sys/i386/i386/locore.s: + /* Don't trust what the BIOS gives for eflags. */ + pushl $PSL_KERNEL + popfl + + /* + * Don't trust what the BIOS gives for %fs and %gs. Trust the bootstrap + * to set %cs, %ds, %es and %ss. + */ + mov %ds, %ax + mov %ax, %fs + mov %ax, %gs + .... + + btext calls the routines `recover_bootinfo()`, `identify_cpu()`, which are also defined in [.filename]#locore.s#. + Here is a description of what they do: + + [.informaltable] + [cols="1,1", frame="none"] + |=== + + |`recover_bootinfo` + |This routine parses the parameters to the kernel passed from the bootstrap. + The kernel may have been booted in 3 ways: by the loader, described above, by the old disk boot blocks, or by the old diskless boot procedure. + This function determines the booting method, and stores the `struct bootinfo` structure into the kernel memory. + + |`identify_cpu` + |This function tries to find out what CPU it is running on, storing the value found in a variable `_cpu`. + |=== + + The next steps are enabling VME, if the CPU supports it: + + [.programlisting] + .... + sys/i386/i386/mpboot.s: + testl $CPUID_VME,%edx + jz 3f + orl $CR4_VME,%eax + 3: movl %eax,%cr4 + .... + + Then, enabling paging: + + [.programlisting] + .... + sys/i386/i386/mpboot.s: + /* Now enable paging */ + movl IdlePTD_nopae, %eax + movl %eax,%cr3 /* load ptd addr into mmu */ + movl %cr0,%eax /* get control word */ + orl $CR0_PE|CR0_PG,%eax /* enable paging */ + movl %eax,%cr0 /* and let's page NOW! */ + .... + + The next three lines of code are because the paging was set, so the jump is needed to continue the execution in virtualized address space: + + [.programlisting] + .... + sys/i386/i386/mpboot.s: + pushl $mp_begin /* jump to high mem */ + ret + + /* now running relocated at KERNBASE where the system is linked to run */ + mp_begin: /* now running relocated at KERNBASE */ + .... + + The function `init386()` is called with a pointer to the first free physical page, after that `mi_startup()`. + `init386` is an architecture dependent initialization function, and `mi_startup()` is an architecture independent one (the 'mi_' prefix stands for Machine Independent). + The kernel never returns from `mi_startup()`, and by calling it, the kernel finishes booting: + + [.programlisting] + .... + sys/i386/i386/locore.s: + pushl physfree /* value of first for init386(first) */ + call init386 /* wire 386 chip for unix operation */ + addl $4,%esp + movl %eax,%esp /* Switch to true top of stack. */ + call mi_startup /* autoconfiguration, mountroot etc */ + /* NOTREACHED */ + .... + + === `init386()` + + `init386()` is defined in [.filename]#sys/i386/i386/machdep.c# and performs low-level initialization specific to the i386 chip. The switch to protected mode was performed by the loader. + The loader has created the very first task, in which the kernel continues to operate. + Before looking at the code, consider the tasks the processor must complete to initialize protected mode execution: + + * Initialize the kernel tunable parameters, passed from the bootstrapping program. + * Prepare the GDT. + * Prepare the IDT. + * Initialize the system console. + * Initialize the DDB, if it is compiled into kernel. + * Initialize the TSS. + * Prepare the LDT. + * Set up thread0's pcb. + + `init386()` initializes the tunable parameters passed from bootstrap by setting the environment pointer (envp) and calling `init_param1()`. + The envp pointer has been passed from loader in the `bootinfo` structure: + + [.programlisting] + .... + sys/i386/i386/machdep.c: + /* Init basic tunables, hz etc */ + init_param1(); + .... + + `init_param1()` is defined in [.filename]#sys/kern/subr_param.c#. + That file has a number of sysctls, and two functions, `init_param1()` and `init_param2()`, that are called from `init386()`: + + [.programlisting] + .... + sys/kern/subr_param.c: + hz = -1; + TUNABLE_INT_FETCH("kern.hz", &hz); + if (hz == -1) + hz = vm_guest > VM_GUEST_NO ? HZ_VM : HZ; + .... + + TUNABLE__FETCH is used to fetch the value from the environment: + + [.programlisting] + .... + /usr/src/sys/sys/kernel.h: + #define TUNABLE_INT_FETCH(path, var) getenv_int((path), (var)) + .... + + Sysctl `kern.hz` is the system clock tick. + Additionally, these sysctls are set by `init_param1()`: `kern.maxswzone, kern.maxbcache, kern.maxtsiz, kern.dfldsiz, kern.maxdsiz, kern.dflssiz, kern.maxssiz, kern.sgrowsiz`. + + Then `init386()` prepares the Global Descriptors Table (GDT). + Every task on an x86 is running in its own virtual address space, and this space is addressed by a segment:offset pair. + Say, for instance, the current instruction to be executed by the processor lies at CS:EIP, then the linear virtual address for that instruction would be "the virtual address of code segment CS" + EIP. + For convenience, segments begin at virtual address 0 and end at a 4GB boundary. + Therefore, the instruction's linear virtual address for this example would just be the value of EIP. + Segment registers such as CS, DS etc are the selectors, i.e., indexes, into GDT (to be more precise, an index is not a selector itself, but the INDEX field of a selector). + FreeBSD's GDT holds descriptors for 15 selectors per CPU: + + [.programlisting] + .... + sys/i386/i386/machdep.c: + union descriptor gdt0[NGDT]; /* initial global descriptor table */ + union descriptor *gdt = gdt0; /* global descriptor table */ + + sys/x86/include/segments.h: + /* + * Entries in the Global Descriptor Table (GDT) + */ + #define GNULL_SEL 0 /* Null Descriptor */ + #define GPRIV_SEL 1 /* SMP Per-Processor Private Data */ + #define GUFS_SEL 2 /* User %fs Descriptor (order critical: 1) */ + #define GUGS_SEL 3 /* User %gs Descriptor (order critical: 2) */ + #define GCODE_SEL 4 /* Kernel Code Descriptor (order critical: 1) */ + #define GDATA_SEL 5 /* Kernel Data Descriptor (order critical: 2) */ + #define GUCODE_SEL 6 /* User Code Descriptor (order critical: 3) */ + #define GUDATA_SEL 7 /* User Data Descriptor (order critical: 4) */ + #define GBIOSLOWMEM_SEL 8 /* BIOS low memory access (must be entry 8) */ + #define GPROC0_SEL 9 /* Task state process slot zero and up */ + #define GLDT_SEL 10 /* Default User LDT */ + #define GUSERLDT_SEL 11 /* User LDT */ + #define GPANIC_SEL 12 /* Task state to consider panic from */ + #define GBIOSCODE32_SEL 13 /* BIOS interface (32bit Code) */ + #define GBIOSCODE16_SEL 14 /* BIOS interface (16bit Code) */ + #define GBIOSDATA_SEL 15 /* BIOS interface (Data) */ + #define GBIOSUTIL_SEL 16 /* BIOS interface (Utility) */ + #define GBIOSARGS_SEL 17 /* BIOS interface (Arguments) */ + #define GNDIS_SEL 18 /* For the NDIS layer */ + #define NGDT 19 + .... + + Note that those #defines are not selectors themselves, but just a field INDEX of a selector, so they are exactly the indices of the GDT. + for example, an actual selector for the kernel code (GCODE_SEL) has the value 0x20. + + The next step is to initialize the Interrupt Descriptor Table (IDT). + This table is referenced by the processor when a software or hardware interrupt occurs. + For example, to make a system call, user application issues the `INT 0x80` instruction. + This is a software interrupt, so the processor's hardware looks up a record with index 0x80 in the IDT. + This record points to the routine that handles this interrupt, in this particular case, this will be the kernel's syscall gate. + The IDT may have a maximum of 256 (0x100) records. + The kernel allocates NIDT records for the IDT, where NIDT is the maximum (256): + + [.programlisting] + .... + sys/i386/i386/machdep.c: + static struct gate_descriptor idt0[NIDT]; + struct gate_descriptor *idt = &idt0[0]; /* interrupt descriptor table */ + .... + + For each interrupt, an appropriate handler is set. + The syscall gate for `INT 0x80` is set as well: + + [.programlisting] + .... + sys/i386/i386/machdep.c: + setidt(IDT_SYSCALL, &IDTVEC(int0x80_syscall), + SDT_SYS386IGT, SEL_UPL, GSEL(GCODE_SEL, SEL_KPL)); + .... + + So when a userland application issues the `INT 0x80` instruction, control will transfer to the function `_Xint0x80_syscall`, which is in the kernel code segment and will be executed with supervisor privileges. + + Console and DDB are then initialized: + + [.programlisting] + .... + sys/i386/i386/machdep.c: + cninit(); + /* skipped */ + kdb_init(); + #ifdef KDB + if (boothowto & RB_KDB) + kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger"); + #endif + .... + + The Task State Segment is another x86 protected mode structure, the TSS is used by the hardware to store task information when a task switch occurs. + + The Local Descriptors Table is used to reference userland code and data. + Several selectors are defined to point to the LDT, they are the system call gates and the user code and data selectors: + + [.programlisting] + .... + sys/x86/include/segments.h: + #define LSYS5CALLS_SEL 0 /* forced by intel BCS */ + #define LSYS5SIGR_SEL 1 + #define LUCODE_SEL 3 + #define LUDATA_SEL 5 + #define NLDT (LUDATA_SEL + 1) + .... + + Next, proc0's Process Control Block (`struct pcb`) structure is initialized. + proc0 is a `struct proc` structure that describes a kernel process. + It is always present while the kernel is running, therefore it is linked with thread0: + + [.programlisting] + .... + sys/i386/i386/machdep.c: + register_t + init386(int first) + { + /* ... skipped ... */ + + proc_linkup0(&proc0, &thread0); + /* ... skipped ... */ + } + .... + + The structure `struct pcb` is a part of a proc structure. + It is defined in [.filename]#/usr/include/machine/pcb.h# and has a process's information specific to the i386 architecture, such as registers values. + + === `mi_startup()` + + This function performs a bubble sort of all the system initialization objects and then calls the entry of each object one by one: + + [.programlisting] + .... + sys/kern/init_main.c: + for (sipp = sysinit; sipp < sysinit_end; sipp++) { + + /* ... skipped ... */ + + /* Call function */ + (*((*sipp)->func))((*sipp)->udata); + /* ... skipped ... */ + } + .... + +-Although the sysinit framework is described in the link:/books/developers-handbook[Developers' Handbook], I will discuss the internals of it. ++Although the sysinit framework is described in the extref:{developers-handbook}[Developers' Handbook], I will discuss the internals of it. + + Every system initialization object (sysinit object) is created by calling a SYSINIT() macro. + Let us take as example an `announce` sysinit object. + This object prints the copyright message: + + [.programlisting] + .... + sys/kern/init_main.c: + static void + print_caddr_t(void *data __unused) + { + printf("%s", (char *)data); + } + /* ... skipped ... */ + SYSINIT(announce, SI_SUB_COPYRIGHT, SI_ORDER_FIRST, print_caddr_t, copyright); + .... + + The subsystem ID for this object is SI_SUB_COPYRIGHT (0x0800001). + So, the copyright message will be printed out first, just after the console initialization. + + Let us take a look at what exactly the macro `SYSINIT()` does. + It expands to a `C_SYSINIT()` macro. + The `C_SYSINIT()` macro then expands to a static `struct sysinit` structure declaration with another `DATA_SET` macro call: + + [.programlisting] + .... + /usr/include/sys/kernel.h: + #define C_SYSINIT(uniquifier, subsystem, order, func, ident) \ + static struct sysinit uniquifier ## _sys_init = { \ subsystem, \ + order, \ func, \ (ident) \ }; \ DATA_WSET(sysinit_set,uniquifier ## + _sys_init); + + #define SYSINIT(uniquifier, subsystem, order, func, ident) \ + C_SYSINIT(uniquifier, subsystem, order, \ + (sysinit_cfunc_t)(sysinit_nfunc_t)func, (void *)(ident)) + .... + + The `DATA_SET()` macro expands to a `_MAKE_SET()`, and that macro is the point where all the sysinit magic is hidden: + + [.programlisting] + .... + /usr/include/linker_set.h: + #define TEXT_SET(set, sym) _MAKE_SET(set, sym) + #define DATA_SET(set, sym) _MAKE_SET(set, sym) + .... + + After executing these macros, various sections were made in the kernel, including`set.sysinit_set`. + Running objdump on a kernel binary, you may notice the presence of such small sections: + + [source,bash] + .... + % llvm-objdump -h /kernel + Sections: + Idx Name Size VMA Type + 10 set_sysctl_set 000021d4 01827078 DATA + 16 set_kbddriver_set 00000010 0182a4d0 DATA + 20 set_scterm_set 0000000c 0182c75c DATA + 21 set_cons_set 00000014 0182c768 DATA + 33 set_scrndr_set 00000024 0182c828 DATA + 41 set_sysinit_set 000014d8 018fabb0 DATA + .... + + This screen dump shows that the size of set.sysinit_set section is 0x14d8 bytes, so `0x14d8/sizeof(void *)` sysinit objects are compiled into the kernel. + The other sections such as `set.sysctl_set` represent other linker sets. + + By defining a variable of type `struct sysinit` the content of `set.sysinit_set` section will be "collected" into that variable: + + [.programlisting] + .... + sys/kern/init_main.c: + SET_DECLARE(sysinit_set, struct sysinit); + .... + + The `struct sysinit` is defined as follows: + + [.programlisting] + .... + sys/sys/kernel.h: + struct sysinit { + enum sysinit_sub_id subsystem; /* subsystem identifier*/ + enum sysinit_elem_order order; /* init order within subsystem*/ + sysinit_cfunc_t func; /* function */ + const void *udata; /* multiplexer/argument */ + }; + .... + + Returning to the `mi_startup()` discussion, it is must be clear now, how the sysinit objects are being organized. + The `mi_startup()` function sorts them and calls each. + The very last object is the system scheduler: + + [.programlisting] + .... + /usr/include/sys/kernel.h: + enum sysinit_sub_id { + SI_SUB_DUMMY = 0x0000000, /* not executed; for linker*/ + SI_SUB_DONE = 0x0000001, /* processed*/ + SI_SUB_TUNABLES = 0x0700000, /* establish tunable values */ + SI_SUB_COPYRIGHT = 0x0800001, /* first use of console*/ + ... + SI_SUB_LAST = 0xfffffff /* final initialization */ + }; + .... + + The system scheduler sysinit object is defined in the file [.filename]#sys/vm/vm_glue.c#, and the entry point for that object is `scheduler()`. + That function is actually an infinite loop, and it represents a process with PID 0, the swapper process. + The thread0 structure, mentioned before, is used to describe it. + + The first user process, called _init_, is created by the sysinit object `init`: + + [.programlisting] + .... + sys/kern/init_main.c: + static void + create_init(const void *udata __unused) + { + struct fork_req fr; + struct ucred *newcred, *oldcred; + struct thread *td; + int error; + + bzero(&fr, sizeof(fr)); + fr.fr_flags = RFFDG | RFPROC | RFSTOPPED; + fr.fr_procp = &initproc; + error = fork1(&thread0, &fr); + if (error) + panic("cannot fork init: %d\n", error); + KASSERT(initproc->p_pid == 1, ("create_init: initproc->p_pid != 1")); + /* divorce init's credentials from the kernel's */ + newcred = crget(); + sx_xlock(&proctree_lock); + PROC_LOCK(initproc); + initproc->p_flag |= P_SYSTEM | P_INMEM; + initproc->p_treeflag |= P_TREE_REAPER; + oldcred = initproc->p_ucred; + crcopy(newcred, oldcred); + #ifdef MAC + mac_cred_create_init(newcred); + #endif + #ifdef AUDIT + audit_cred_proc1(newcred); + #endif + proc_set_cred(initproc, newcred); + td = FIRST_THREAD_IN_PROC(initproc); + crcowfree(td); + td->td_realucred = crcowget(initproc->p_ucred); + td->td_ucred = td->td_realucred; + PROC_UNLOCK(initproc); + sx_xunlock(&proctree_lock); + crfree(oldcred); + cpu_fork_kthread_handler(FIRST_THREAD_IN_PROC(initproc), start_init, NULL); + } + SYSINIT(init, SI_SUB_CREATE_INIT, SI_ORDER_FIRST, create_init, NULL); + .... + + The function `create_init()` allocates a new process by calling `fork1()`, but does not mark it runnable. + When this new process is scheduled for execution by the scheduler, the `start_init()` will be called. + That function is defined in [.filename]#init_main.c#. + It tries to load and exec the [.filename]#init# binary, probing [.filename]#/sbin/init# first, then [.filename]#/sbin/oinit#, [.filename]#/sbin/init.bak#, and finally [.filename]#/rescue/init#: + + [.programlisting] + .... + sys/kern/init_main.c: + static char init_path[MAXPATHLEN] = + #ifdef INIT_PATH + __XSTRING(INIT_PATH); + #else + "/sbin/init:/sbin/oinit:/sbin/init.bak:/rescue/init"; + #endif + .... +diff --git a/documentation/content/en/books/arch-handbook/jail/_index.adoc b/documentation/content/en/books/arch-handbook/jail/_index.adoc +index a77ce2ae10..734b224060 100644 +--- a/documentation/content/en/books/arch-handbook/jail/_index.adoc ++++ b/documentation/content/en/books/arch-handbook/jail/_index.adoc +@@ -1,529 +1,529 @@ + --- + title: Chapter 4. The Jail Subsystem + prev: books/arch-handbook/kobj + next: books/arch-handbook/sysinit + description: The Jail Subsystem + tags: ["jail", "architecture", "networking", "kernel"] + showBookMenu: true + weight: 5 + params: + path: "/books/arch-handbook/jail/" + --- + + [[jail]] + = The Jail Subsystem + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :sectnumoffset: 4 + :partnums: + :source-highlighter: rouge + :experimental: + :images-path: books/arch-handbook/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + :imagesdir: ../../../../images/{images-path} + endif::[] + ifndef::book[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + toc::[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + toc::[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + On most UNIX(R) systems, `root` has omnipotent power. This promotes insecurity. If an attacker gained `root` on a system, he would have every function at his fingertips. In FreeBSD there are sysctls which dilute the power of `root`, in order to minimize the damage caused by an attacker. Specifically, one of these functions is called `secure levels`. Similarly, another function which is present from FreeBSD 4.0 and onward, is a utility called man:jail[8]. Jail chroots an environment and sets certain restrictions on processes which are forked within the jail. For example, a jailed process cannot affect processes outside the jail, utilize certain system calls, or inflict any damage on the host environment. + +-Jail is becoming the new security model. People are running potentially vulnerable servers such as Apache, BIND, and sendmail within jails, so that if an attacker gains `root` within the jail, it is only an annoyance, and not a devastation. This article mainly focuses on the internals (source code) of jail. For information on how to set up a jail see the extref:{handbook}[handbook entry on jails, jails]. ++Jail is becoming the new security model. People are running potentially vulnerable servers such as Apache, BIND, and sendmail within jails, so that if an attacker gains `root` within the jail, it is only an annoyance, and not a devastation. This article mainly focuses on the internals (source code) of jail. For information on how to set up a jail see the extref:{handbook}jails[handbook entry on jails, jails]. + + [[jail-arch]] + == Architecture + + Jail consists of two realms: the userland program, man:jail[8], and the code implemented within the kernel: the man:jail[2] system call and associated restrictions. I will be discussing the userland program and then how jail is implemented within the kernel. + + === Userland Code + + The source for the userland jail is located in [.filename]#/usr/src/usr.sbin/jail#, consisting of one file, [.filename]#jail.c#. The program takes these arguments: the path of the jail, hostname, IP address, and the command to be executed. + + ==== Data Structures + + In [.filename]#jail.c#, the first thing I would note is the declaration of an important structure `struct jail j;` which was included from [.filename]#/usr/include/sys/jail.h#. + + The definition of the `jail` structure is: + + [.programlisting] + .... + /usr/include/sys/jail.h: + + struct jail { + u_int32_t version; + char *path; + char *hostname; + u_int32_t ip_number; + }; + .... + + As you can see, there is an entry for each of the arguments passed to the man:jail[8] program, and indeed, they are set during its execution. + + [.programlisting] + .... + /usr/src/usr.sbin/jail/jail.c + char path[PATH_MAX]; + ... + if (realpath(argv[0], path) == NULL) + err(1, "realpath: %s", argv[0]); + if (chdir(path) != 0) + err(1, "chdir: %s", path); + memset(&j, 0, sizeof(j)); + j.version = 0; + j.path = path; + j.hostname = argv[1]; + .... + + ==== Networking + + One of the arguments passed to the man:jail[8] program is an IP address with which the jail can be accessed over the network. man:jail[8] translates the IP address given into host byte order and then stores it in `j` (the `jail` structure). + + [.programlisting] + .... + /usr/src/usr.sbin/jail/jail.c: + struct in_addr in; + ... + if (inet_aton(argv[2], &in) == 0) + errx(1, "Could not make sense of ip-number: %s", argv[2]); + j.ip_number = ntohl(in.s_addr); + .... + + The man:inet_aton[3] function "interprets the specified character string as an Internet address, placing the address into the structure provided." The `ip_number` member in the `jail` structure is set only when the IP address placed onto the `in` structure by man:inet_aton[3] is translated into host byte order by man:ntohl[3]. + + ==== Jailing the Process + + Finally, the userland program jails the process. Jail now becomes an imprisoned process itself and then executes the command given using man:execv[3]. + + [.programlisting] + .... + /usr/src/usr.sbin/jail/jail.c + i = jail(&j); + ... + if (execv(argv[3], argv + 3) != 0) + err(1, "execv: %s", argv[3]); + .... + + As you can see, the `jail()` function is called, and its argument is the `jail` structure which has been filled with the arguments given to the program. Finally, the program you specify is executed. I will now discuss how jail is implemented within the kernel. + + === Kernel Space + + We will now be looking at the file [.filename]#/usr/src/sys/kern/kern_jail.c#. This is the file where the man:jail[2] system call, appropriate sysctls, and networking functions are defined. + + ==== Sysctls + + In [.filename]#kern_jail.c#, the following sysctls are defined: + + [.programlisting] + .... + /usr/src/sys/kern/kern_jail.c: + int jail_set_hostname_allowed = 1; + SYSCTL_INT(_security_jail, OID_AUTO, set_hostname_allowed, CTLFLAG_RW, + &jail_set_hostname_allowed, 0, + "Processes in jail can set their hostnames"); + + int jail_socket_unixiproute_only = 1; + SYSCTL_INT(_security_jail, OID_AUTO, socket_unixiproute_only, CTLFLAG_RW, + &jail_socket_unixiproute_only, 0, + "Processes in jail are limited to creating UNIX/IPv4/route sockets only"); + + int jail_sysvipc_allowed = 0; + SYSCTL_INT(_security_jail, OID_AUTO, sysvipc_allowed, CTLFLAG_RW, + &jail_sysvipc_allowed, 0, + "Processes in jail can use System V IPC primitives"); + + static int jail_enforce_statfs = 2; + SYSCTL_INT(_security_jail, OID_AUTO, enforce_statfs, CTLFLAG_RW, + &jail_enforce_statfs, 0, + "Processes in jail cannot see all mounted file systems"); + + int jail_allow_raw_sockets = 0; + SYSCTL_INT(_security_jail, OID_AUTO, allow_raw_sockets, CTLFLAG_RW, + &jail_allow_raw_sockets, 0, + "Prison root can create raw sockets"); + + int jail_chflags_allowed = 0; + SYSCTL_INT(_security_jail, OID_AUTO, chflags_allowed, CTLFLAG_RW, + &jail_chflags_allowed, 0, + "Processes in jail can alter system file flags"); + + int jail_mount_allowed = 0; + SYSCTL_INT(_security_jail, OID_AUTO, mount_allowed, CTLFLAG_RW, + &jail_mount_allowed, 0, + "Processes in jail can mount/unmount jail-friendly file systems"); + .... + + Each of these sysctls can be accessed by the user through the man:sysctl[8] program. Throughout the kernel, these specific sysctls are recognized by their name. For example, the name of the first sysctl is `security.jail.set_hostname_allowed`. + + ==== man:jail[2] System Call + + Like all system calls, the man:jail[2] system call takes two arguments, `struct thread *td` and `struct jail_args *uap`. `td` is a pointer to the `thread` structure which describes the calling thread. In this context, `uap` is a pointer to the structure in which a pointer to the `jail` structure passed by the userland [.filename]#jail.c# is contained. When I described the userland program before, you saw that the man:jail[2] system call was given a `jail` structure as its own argument. + + [.programlisting] + .... + /usr/src/sys/kern/kern_jail.c: + /* + * struct jail_args { + * struct jail *jail; + * }; + */ + int + jail(struct thread *td, struct jail_args *uap) + .... + + Therefore, `uap->jail` can be used to access the `jail` structure which was passed to the system call. Next, the system call copies the `jail` structure into kernel space using the man:copyin[9] function. man:copyin[9] takes three arguments: the address of the data which is to be copied into kernel space, `uap->jail`, where to store it, `j` and the size of the storage. The `jail` structure pointed by `uap->jail` is copied into kernel space and is stored in another `jail` structure, `j`. + + [.programlisting] + .... + /usr/src/sys/kern/kern_jail.c: + error = copyin(uap->jail, &j, sizeof(j)); + .... + + There is another important structure defined in [.filename]#jail.h#. It is the `prison` structure. The `prison` structure is used exclusively within kernel space. Here is the definition of the `prison` structure. + + [.programlisting] + .... + /usr/include/sys/jail.h: + struct prison { + LIST_ENTRY(prison) pr_list; /* (a) all prisons */ + int pr_id; /* (c) prison id */ + int pr_ref; /* (p) refcount */ + char pr_path[MAXPATHLEN]; /* (c) chroot path */ + struct vnode *pr_root; /* (c) vnode to rdir */ + char pr_host[MAXHOSTNAMELEN]; /* (p) jail hostname */ + u_int32_t pr_ip; /* (c) ip addr host */ + void *pr_linux; /* (p) linux abi */ + int pr_securelevel; /* (p) securelevel */ + struct task pr_task; /* (d) destroy task */ + struct mtx pr_mtx; + void **pr_slots; /* (p) additional data */ + }; + .... + + The man:jail[2] system call then allocates memory for a `prison` structure and copies data between the `jail` and `prison` structure. + + [.programlisting] + .... + /usr/src/sys/kern/kern_jail.c: + MALLOC(pr, struct prison *, sizeof(*pr), M_PRISON, M_WAITOK | M_ZERO); + ... + error = copyinstr(j.path, &pr->pr_path, sizeof(pr->pr_path), 0); + if (error) + goto e_killmtx; + ... + error = copyinstr(j.hostname, &pr->pr_host, sizeof(pr->pr_host), 0); + if (error) + goto e_dropvnref; + pr->pr_ip = j.ip_number; + .... + + Next, we will discuss another important system call man:jail_attach[2], which implements the function to put a process into the jail. + + [.programlisting] + .... + /usr/src/sys/kern/kern_jail.c: + /* + * struct jail_attach_args { + * int jid; + * }; + */ + int + jail_attach(struct thread *td, struct jail_attach_args *uap) + .... + + This system call makes the changes that can distinguish a jailed process from those unjailed ones. To understand what man:jail_attach[2] does for us, certain background information is needed. + + On FreeBSD, each kernel visible thread is identified by its `thread` structure, while the processes are described by their `proc` structures. You can find the definitions of the `thread` and `proc` structure in [.filename]#/usr/include/sys/proc.h#. For example, the `td` argument in any system call is actually a pointer to the calling thread's `thread` structure, as stated before. The `td_proc` member in the `thread` structure pointed by `td` is a pointer to the `proc` structure which represents the process that contains the thread represented by `td`. The `proc` structure contains members which can describe the owner's identity(`p_ucred`), the process resource limits(`p_limit`), and so on. In the `ucred` structure pointed by `p_ucred` member in the `proc` structure, there is a pointer to the `prison` structure(`cr_prison`). + + [.programlisting] + .... + /usr/include/sys/proc.h: + struct thread { + ... + struct proc *td_proc; + ... + }; + struct proc { + ... + struct ucred *p_ucred; + ... + }; + /usr/include/sys/ucred.h + struct ucred { + ... + struct prison *cr_prison; + ... + }; + .... + + In [.filename]#kern_jail.c#, the function `jail()` then calls function `jail_attach()` with a given `jid`. And `jail_attach()` calls function `change_root()` to change the root directory of the calling process. The `jail_attach()` then creates a new `ucred` structure, and attaches the newly created `ucred` structure to the calling process after it has successfully attached the `prison` structure to the `ucred` structure. From then on, the calling process is recognized as jailed. When the kernel routine `jailed()` is called in the kernel with the newly created `ucred` structure as its argument, it returns 1 to tell that the credential is connected with a jail. The public ancestor process of all the process forked within the jail, is the process which runs man:jail[8], as it calls the man:jail[2] system call. When a program is executed through man:execve[2], it inherits the jailed property of its parent's `ucred` structure, therefore it has a jailed `ucred` structure. + + [.programlisting] + .... + /usr/src/sys/kern/kern_jail.c + int + jail(struct thread *td, struct jail_args *uap) + { + ... + struct jail_attach_args jaa; + ... + error = jail_attach(td, &jaa); + if (error) + goto e_dropprref; + ... + } + + int + jail_attach(struct thread *td, struct jail_attach_args *uap) + { + struct proc *p; + struct ucred *newcred, *oldcred; + struct prison *pr; + ... + p = td->td_proc; + ... + pr = prison_find(uap->jid); + ... + change_root(pr->pr_root, td); + ... + newcred->cr_prison = pr; + p->p_ucred = newcred; + ... + } + .... + + When a process is forked from its parent process, the man:fork[2] system call uses `crhold()` to maintain the credential for the newly forked process. It inherently keep the newly forked child's credential consistent with its parent, so the child process is also jailed. + + [.programlisting] + .... + /usr/src/sys/kern/kern_fork.c: + p2->p_ucred = crhold(td->td_ucred); + ... + td2->td_ucred = crhold(p2->p_ucred); + .... + + [[jail-restrictions]] + == Restrictions + + Throughout the kernel there are access restrictions relating to jailed processes. Usually, these restrictions only check whether the process is jailed, and if so, returns an error. For example: + + [.programlisting] + .... + if (jailed(td->td_ucred)) + return (EPERM); + .... + + === SysV IPC + + System V IPC is based on messages. Processes can send each other these messages which tell them how to act. The functions which deal with messages are: man:msgctl[3], man:msgget[3], man:msgsnd[3] and man:msgrcv[3]. Earlier, I mentioned that there were certain sysctls you could turn on or off in order to affect the behavior of jail. One of these sysctls was `security.jail.sysvipc_allowed`. By default, this sysctl is set to 0. If it were set to 1, it would defeat the whole purpose of having a jail; privileged users from the jail would be able to affect processes outside the jailed environment. The difference between a message and a signal is that the message only consists of the signal number. + + [.filename]#/usr/src/sys/kern/sysv_msg.c#: + + * `msgget(key, msgflg)`: `msgget` returns (and possibly creates) a message descriptor that designates a message queue for use in other functions. + * `msgctl(msgid, cmd, buf)`: Using this function, a process can query the status of a message descriptor. + * `msgsnd(msgid, msgp, msgsz, msgflg)`: `msgsnd` sends a message to a process. + * `msgrcv(msgid, msgp, msgsz, msgtyp, msgflg)`: a process receives messages using this function + + In each of the system calls corresponding to these functions, there is this conditional: + + [.programlisting] + .... + /usr/src/sys/kern/sysv_msg.c: + if (!jail_sysvipc_allowed && jailed(td->td_ucred)) + return (ENOSYS); + .... + + Semaphore system calls allow processes to synchronize execution by doing a set of operations atomically on a set of semaphores. Basically semaphores provide another way for processes lock resources. However, process waiting on a semaphore, that is being used, will sleep until the resources are relinquished. The following semaphore system calls are blocked inside a jail: man:semget[2], man:semctl[2] and man:semop[2]. + + [.filename]#/usr/src/sys/kern/sysv_sem.c#: + + * `semctl(semid, semnum, cmd, ...)`: `semctl` does the specified `cmd` on the semaphore queue indicated by `semid`. + * `semget(key, nsems, flag)`: `semget` creates an array of semaphores, corresponding to `key`. + + + `key and flag take on the same meaning as they do in msgget.` + * `semop(semid, array, nops)`: `semop` performs a group of operations indicated by `array`, to the set of semaphores identified by `semid`. + + System V IPC allows for processes to share memory. Processes can communicate directly with each other by sharing parts of their virtual address space and then reading and writing data stored in the shared memory. These system calls are blocked within a jailed environment: man:shmdt[2], man:shmat[2], man:shmctl[2] and man:shmget[2]. + + [.filename]#/usr/src/sys/kern/sysv_shm.c#: + + * `shmctl(shmid, cmd, buf)`: `shmctl` does various control operations on the shared memory region identified by `shmid`. + * `shmget(key, size, flag)`: `shmget` accesses or creates a shared memory region of `size` bytes. + * `shmat(shmid, addr, flag)`: `shmat` attaches a shared memory region identified by `shmid` to the address space of a process. + * `shmdt(addr)`: `shmdt` detaches the shared memory region previously attached at `addr`. + + === Sockets + + Jail treats the man:socket[2] system call and related lower-level socket functions in a special manner. In order to determine whether a certain socket is allowed to be created, it first checks to see if the sysctl `security.jail.socket_unixiproute_only` is set. If set, sockets are only allowed to be created if the family specified is either `PF_LOCAL`, `PF_INET` or `PF_ROUTE`. Otherwise, it returns an error. + + [.programlisting] + .... + /usr/src/sys/kern/uipc_socket.c: + int + socreate(int dom, struct socket **aso, int type, int proto, + struct ucred *cred, struct thread *td) + { + struct protosw *prp; + ... + if (jailed(cred) && jail_socket_unixiproute_only && + prp->pr_domain->dom_family != PF_LOCAL && + prp->pr_domain->dom_family != PF_INET && + prp->pr_domain->dom_family != PF_ROUTE) { + return (EPROTONOSUPPORT); + } + ... + } + .... + + === Berkeley Packet Filter + + The Berkeley Packet Filter provides a raw interface to data link layers in a protocol independent fashion. BPF is now controlled by the man:devfs[8] whether it can be used in a jailed environment. + + === Protocols + + There are certain protocols which are very common, such as TCP, UDP, IP and ICMP. IP and ICMP are on the same level: the network layer 2. There are certain precautions which are taken in order to prevent a jailed process from binding a protocol to a certain address only if the `nam` parameter is set. `nam` is a pointer to a `sockaddr` structure, which describes the address on which to bind the service. A more exact definition is that `sockaddr` "may be used as a template for referring to the identifying tag and length of each address". In the function `in_pcbbind_setup()`, `sin` is a pointer to a `sockaddr_in` structure, which contains the port, address, length and domain family of the socket which is to be bound. Basically, this disallows any processes from jail to be able to specify the address that does not belong to the jail in which the calling process exists. + + [.programlisting] + .... + /usr/src/sys/netinet/in_pcb.c: + int + in_pcbbind_setup(struct inpcb *inp, struct sockaddr *nam, in_addr_t *laddrp, + u_short *lportp, struct ucred *cred) + { + ... + struct sockaddr_in *sin; + ... + if (nam) { + sin = (struct sockaddr_in *)nam; + ... + if (sin->sin_addr.s_addr != INADDR_ANY) + if (prison_ip(cred, 0, &sin->sin_addr.s_addr)) + return(EINVAL); + ... + if (lport) { + ... + if (prison && prison_ip(cred, 0, &sin->sin_addr.s_addr)) + return (EADDRNOTAVAIL); + ... + } + } + if (lport == 0) { + ... + if (laddr.s_addr != INADDR_ANY) + if (prison_ip(cred, 0, &laddr.s_addr)) + return (EINVAL); + ... + } + ... + if (prison_ip(cred, 0, &laddr.s_addr)) + return (EINVAL); + ... + } + .... + + You might be wondering what function `prison_ip()` does. `prison_ip()` is given three arguments, a pointer to the credential(represented by `cred`), any flags, and an IP address. It returns 1 if the IP address does NOT belong to the jail or 0 otherwise. As you can see from the code, if it is indeed an IP address not belonging to the jail, the protocol is not allowed to bind to that address. + + [.programlisting] + .... + /usr/src/sys/kern/kern_jail.c: + int + prison_ip(struct ucred *cred, int flag, u_int32_t *ip) + { + u_int32_t tmp; + + if (!jailed(cred)) + return (0); + if (flag) + tmp = *ip; + else + tmp = ntohl(*ip); + if (tmp == INADDR_ANY) { + if (flag) + *ip = cred->cr_prison->pr_ip; + else + *ip = htonl(cred->cr_prison->pr_ip); + return (0); + } + if (tmp == INADDR_LOOPBACK) { + if (flag) + *ip = cred->cr_prison->pr_ip; + else + *ip = htonl(cred->cr_prison->pr_ip); + return (0); + } + if (cred->cr_prison->pr_ip != tmp) + return (1); + return (0); + } + .... + + === Filesystem + + Even `root` users within the jail are not allowed to unset or modify any file flags, such as immutable, append-only, and undeleteable flags, if the securelevel is greater than 0. + + [.programlisting] + .... + /usr/src/sys/ufs/ufs/ufs_vnops.c: + static int + ufs_setattr(ap) + ... + { + ... + if (!priv_check_cred(cred, PRIV_VFS_SYSFLAGS, 0)) { + if (ip->i_flags + & (SF_NOUNLINK | SF_IMMUTABLE | SF_APPEND)) { + error = securelevel_gt(cred, 0); + if (error) + return (error); + } + ... + } + } + /usr/src/sys/kern/kern_priv.c + int + priv_check_cred(struct ucred *cred, int priv, int flags) + { + ... + error = prison_priv_check(cred, priv); + if (error) + return (error); + ... + } + /usr/src/sys/kern/kern_jail.c + int + prison_priv_check(struct ucred *cred, int priv) + { + ... + switch (priv) { + ... + case PRIV_VFS_SYSFLAGS: + if (jail_chflags_allowed) + return (0); + else + return (EPERM); + ... + } + ... + } + .... +diff --git a/documentation/content/en/books/developers-handbook/_index.adoc b/documentation/content/en/books/developers-handbook/_index.adoc +index b25ae41a62..b4979f3a2f 100644 +--- a/documentation/content/en/books/developers-handbook/_index.adoc ++++ b/documentation/content/en/books/developers-handbook/_index.adoc +@@ -1,60 +1,60 @@ + --- + title: FreeBSD Developers' Handbook + authors: + - author: The FreeBSD Documentation Project + copyright: 1995-2023 The FreeBSD Documentation Project + description: For people who want to develop software for FreeBSD (and not just people who are developing FreeBSD itself) + trademarks: ["freebsd", "apple", "ibm", "ieee", "intel", "linux", "microsoft", "opengroup", "sun", "general"] + next: books/developers-handbook/parti + bookOrder: 25 + tags: ["FreeBSD Developers' Handbook"] + add_single_page_link: true + showBookMenu: true + weight: 0 + params: + path: "/books/developers-handbook/" + --- + + = FreeBSD Developers' Handbook + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :partnums: + :source-highlighter: rouge + :experimental: + :images-path: books/developers-handbook/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + Welcome to the Developers' Handbook. + This manual is a _work in progress_ and is the work of many individuals. + Many sections do not yet exist and some of those that do exist need to be updated. + If you are interested in helping with this project, send email to the {freebsd-doc}. + + The latest version of this document is always available from the link:https://www.FreeBSD.org[FreeBSD World Wide Web server]. +-It may also be downloaded in a variety of formats and compression options from the link:https://download.freebsd.org/doc/[FreeBSD download server] or one of the numerous extref:{handbook}[mirror sites, mirrors]. ++It may also be downloaded in a variety of formats and compression options from the link:https://download.freebsd.org/doc/[FreeBSD download server] or one of the numerous extref:{handbook}mirrors[mirror sites, mirrors]. + + ''' +diff --git a/documentation/content/en/books/developers-handbook/book.adoc b/documentation/content/en/books/developers-handbook/book.adoc +index 63a89edddc..510467b290 100644 +--- a/documentation/content/en/books/developers-handbook/book.adoc ++++ b/documentation/content/en/books/developers-handbook/book.adoc +@@ -1,87 +1,87 @@ + --- + title: FreeBSD Developers' Handbook + authors: + - author: The FreeBSD Documentation Project + copyright: 1995-2023 The FreeBSD Documentation Project + description: For people who want to develop software for FreeBSD (and not just people who are developing FreeBSD itself) + trademarks: ["freebsd", "apple", "ibm", "ieee", "intel", "linux", "microsoft", "opengroup", "sun", "general"] + tags: ["FreeBSD Developers' Handbook"] + add_split_page_link: true + --- + + = FreeBSD Developers' Handbook + :doctype: book + :toc: macro + :toclevels: 2 + :icons: font + :sectnums: + :sectnumlevels: 6 + :partnums: + :source-highlighter: rouge + :experimental: + :book: true + :pdf: false + + ifdef::env-beastie[] + ifdef::backend-html5[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + :chapters-path: content/{{% lang %}}/books/developers-handbook/ + endif::[] + ifdef::backend-pdf,backend-epub3[] + :chapters-path: + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + :chapters-path: + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [.abstract-title] + Abstract + + Welcome to the Developers' Handbook. + This manual is a _work in progress_ and is the work of many individuals. + Many sections do not yet exist and some of those that do exist need to be updated. + If you are interested in helping with this project, send email to the {freebsd-doc}. + + The latest version of this document is always available from the link:https://www.FreeBSD.org[FreeBSD World Wide Web server]. +-It may also be downloaded in a variety of formats and compression options from the link:https://download.freebsd.org/doc/[FreeBSD download server] or one of the numerous extref:{handbook}[mirror sites, mirrors]. ++It may also be downloaded in a variety of formats and compression options from the link:https://download.freebsd.org/doc/[FreeBSD download server] or one of the numerous extref:{handbook}mirrors[mirror sites, mirrors]. + + ''' + + toc::[] + + // Section one + include::{chapters-path}parti.adoc[] + include::{chapters-path}introduction/_index.adoc[leveloffset=+1] + include::{chapters-path}tools/_index.adoc[leveloffset=+1] + include::{chapters-path}secure/_index.adoc[leveloffset=+1] + include::{chapters-path}l10n/_index.adoc[leveloffset=+1] + include::{chapters-path}policies/_index.adoc[leveloffset=+1] + include::{chapters-path}testing/_index.adoc[leveloffset=+1] + + // Section two + include::{chapters-path}partii.adoc[] + include::{chapters-path}sockets/_index.adoc[leveloffset=+1] + include::{chapters-path}ipv6/_index.adoc[leveloffset=+1] + + // Section three + include::{chapters-path}partiii.adoc[] + include::{chapters-path}kernelbuild/_index.adoc[leveloffset=+1] + include::{chapters-path}kerneldebug/_index.adoc[leveloffset=+1] + + // Section four + include::{chapters-path}partiv.adoc[] + include::{chapters-path}x86/_index.adoc[leveloffset=+1] + + // Appendices + include::{chapters-path}partv.adoc[] + include::{chapters-path}bibliography/_index.adoc[leveloffset=+1] +diff --git a/documentation/content/en/books/developers-handbook/tools/_index.adoc b/documentation/content/en/books/developers-handbook/tools/_index.adoc +index bcae63ad03..14a6250329 100644 +--- a/documentation/content/en/books/developers-handbook/tools/_index.adoc ++++ b/documentation/content/en/books/developers-handbook/tools/_index.adoc +@@ -1,1638 +1,1638 @@ + --- + title: Chapter 2. Programming Tools + authors: + - author: James Raynard + - author: Murray Stokely + prev: books/developers-handbook/introduction + next: books/developers-handbook/secure + description: Programming Tools + tags: ["tools", "Interpreters", "Compilers", "cc", "make", "Debugging", "lldb", "gdb", "clang", "Emacs"] + showBookMenu: true + weight: 3 + params: + path: "/books/developers-handbook/tools/" + --- + + [[tools]] + = Programming Tools + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :sectnumoffset: 2 + :partnums: + :source-highlighter: rouge + :experimental: + :c-plus-plus-command: c++ + :clang-plus-plus-command: clang++ + :images-path: books/developers-handbook/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + :imagesdir: ../../../../images/{images-path} + endif::[] + ifndef::book[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + toc::[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + toc::[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [[tools-synopsis]] + == Synopsis + + This chapter is an introduction to using some of the programming tools supplied with FreeBSD, + although much of it will be applicable to many other versions of UNIX(R). + It does _not_ attempt to describe coding in any detail. + Most of the chapter assumes little or no previous programming knowledge, + although it is hoped that most programmers will find something of value in it. + + [[tools-intro]] + == Introduction + + FreeBSD offers an excellent development environment. + Compilers for C and C++ and an assembler come with the basic system, not to mention classic UNIX(R) tools such as `sed` and `awk`. + If that is not enough, there are many more compilers and interpreters in the Ports collection. + The following section, crossref:tools[tools-programming,Introduction to Programming], lists some of the available options. + FreeBSD is very compatible with standards such as POSIX(R) and ANSI C, as well with its own BSD heritage, so it is possible to write applications that will compile and run with little or no modification on a wide range of platforms. + + However, all this power can be rather overwhelming at first if you have never written programs on a UNIX(R) platform before. + This document aims to help you get up and running, without getting too deeply into more advanced topics. + The intention is that this document should give you enough of the basics to be able to make some sense of the documentation. + + Most of the document requires little or no knowledge of programming, although it does assume a basic competence with using UNIX(R) and a willingness to learn! + + [[tools-programming]] + == Introduction to Programming + + A program is a set of instructions that tell the computer to do various things; sometimes the instruction it has to perform depends on what happened when it performed a previous instruction. + This section gives an overview of the two main ways in which you can give these instructions, or "commands" as they are usually called. + One way uses an _interpreter_, the other a _compiler_. + As human languages are too difficult for a computer to understand in an unambiguous way, commands are usually written in one or other languages specially designed for the purpose. + + === Interpreters + + With an interpreter, the language comes as an environment, where you type in commands at a prompt and the environment executes them for you. + For more complicated programs, you can type the commands into a file and get the interpreter to load the file and execute the commands in it. + If anything goes wrong, many interpreters will drop you into a debugger to help you track down the problem. + + The advantage of this is that you can see the results of your commands immediately, and mistakes can be corrected readily. + The biggest disadvantage comes when you want to share your programs with someone. + They must have the same interpreter, or you must have some way of giving it to them, and they need to understand how to use it. + Also users may not appreciate being thrown into a debugger if they press the wrong key! From a performance point of view, interpreters can use up a lot of memory, and generally do not generate code as efficiently as compilers. + + In my opinion, interpreted languages are the best way to start if you have not done any programming before. + This kind of environment is typically found with languages like Lisp, Smalltalk, Perl and Basic. + It could also be argued that the UNIX(R) shell (`sh`, `csh`) is itself an interpreter, and many people do in fact write shell "scripts" to help with various "housekeeping" tasks on their machine. + Indeed, part of the original UNIX(R) philosophy was to provide lots of small utility programs that could be linked together in shell scripts to perform useful tasks. + + === Interpreters Available with FreeBSD + + Here is a list of interpreters that are available from the FreeBSD Ports Collection, with a brief discussion of some of the more popular interpreted languages. + +-Instructions on how to get and install applications from the Ports Collection can be found in the extref:{handbook}[Ports section, ports-using] of the handbook. ++Instructions on how to get and install applications from the Ports Collection can be found in the extref:{handbook}ports[Ports section, ports-using] of the handbook. + + BASIC:: + Short for Beginner's All-purpose Symbolic Instruction Code. + Developed in the 1950s for teaching University students to program and provided with every self-respecting personal computer in the 1980s, BASIC has been the first programming language for many programmers. + It is also the foundation for Visual Basic. + + + The Bywater Basic Interpreter can be found in the Ports Collection as package:lang/bwbasic[] and the Phil Cockroft's Basic Interpreter (formerly Rabbit Basic) is available as package:lang/pbasic[]. + + Lisp:: + A language that was developed in the late 1950s as an alternative to the "number-crunching" languages that were popular at the time. + Instead of being based on numbers, Lisp is based on lists; in fact, the name is short for "List Processing". + It is very popular in AI (Artificial Intelligence) circles. + + + Lisp is an extremely powerful and sophisticated language, but can be rather large and unwieldy. + + + Various implementations of Lisp that can run on UNIX(R) systems are available in the Ports Collection for FreeBSD. + CLISP by Bruno Haible and Michael Stoll is available as package:lang/clisp[]. + SLisp, a simpler Lisp implementations, is available as package:lang/slisp[]. + + Perl:: + Very popular with system administrators for writing scripts; also often used on World Wide Web servers for writing CGI scripts. + + + Perl is available in the Ports Collection as package:lang/perl5.36[] for all FreeBSD releases. + + Scheme:: + A dialect of Lisp that is rather more compact and cleaner than Common Lisp. + Popular in Universities as it is simple enough to teach to undergraduates as a first language, + while it has a high enough level of abstraction to be used in research work. + + + Scheme is available from the Ports Collection as package:lang/elk[] for the Elk Scheme Interpreter. + The MIT Scheme Interpreter can be found in package:lang/mit-scheme[] and the SCM Scheme Interpreter in package:lang/scm[]. + + Lua:: + Lua is a lightweight embeddable scripting language. + It is widely portable and relatively simple. + Lua is available in the Ports Collection in package:lang/lua54[]. + It is also included in the base system as [.filename]#/usr/libexec/flua# for use by base system components. + Third party software should not depend on [.filename]#flua#. + + Python:: + Python is an Object-Oriented, interpreted language. + Its advocates argue that it is one of the best languages to start programming with, since it is relatively easy to start with, but is not limited in comparison to other popular interpreted languages that are used for the development of large, complex applications (Perl and Tcl are two other languages that are popular for such tasks). + + + The latest version of Python is available from the Ports Collection in package:lang/python[]. + + Ruby:: + Ruby is an interpreter, pure object-oriented programming language. + It has become widely popular because of its easy to understand syntax, flexibility when writing code, and the ability to easily develop and maintain large, complex programs. + + + Ruby is available from the Ports Collection as package:lang/ruby32[]. + + Tcl and Tk:: + Tcl is an embeddable, interpreted language, that has become widely used and became popular mostly because of its portability to many platforms. + It can be used both for quickly writing small, prototype applications, or (when combined with Tk, a GUI toolkit) fully-fledged, featureful programs. + + + Various versions of Tcl are available as ports for FreeBSD. + The latest version, Tcl 8.7, can be found in package:lang/tcl87[]. + + === Compilers + + Compilers are rather different. + First of all, you write your code in a file (or files) using an editor. + You then run the compiler and see if it accepts your program. + If it did not compile, grit your teeth and go back to the editor; + if it did compile and gave you a program, you can run it either at a shell command prompt or in a debugger to see if it works properly.footnote:[If you run it in the shell, you may get a core dump.] + + Obviously, this is not quite as direct as using an interpreter. + However it allows you to do a lot of things which are very difficult or even impossible with an interpreter, + such as writing code which interacts closely with the operating system-or even writing your own operating system! + It is also useful if you need to write very efficient code, as the compiler can take its time and optimize the code, + which would not be acceptable in an interpreter. + Moreover, distributing a program written for a compiler is usually more straightforward than one written for an interpreter-you can just give them a copy of the executable, assuming they have the same operating system as you. + + As the edit-compile-run-debug cycle is rather tedious when using separate programs, many commercial compiler makers have produced Integrated Development Environments (IDEs for short). + FreeBSD does not include an IDE in the base system, but package:devel/kdevelop[] is available in the Ports Collection and many use Emacs for this purpose. + Using Emacs as an IDE is discussed in crossref:tools[emacs, Using Emacs as a Development Environment]. + + [[tools-compiling]] + == Compiling with `cc` + + This section deals with the clang compiler for C and C++, as it's installed with the FreeBSD base system. + Clang is installed as `cc`; the GNU compiler package:lang/gcc[gcc] is available in the Ports Collection. + The details of producing a program with an interpreter vary considerably between interpreters, and are usually well covered in the documentation and on-line help for the interpreter. + + Once you have written your masterpiece, the next step is to convert it into something that will (hopefully!) run on FreeBSD. + This usually involves several steps, each of which is done by a separate program. + + [.procedure] + . Pre-process your source code to remove comments and do other tricks like expanding macros in C. + . Check the syntax of your code to see if you have obeyed the rules of the language. If you have not, it will complain! + . Convert the source code into assembly language-this is very close to machine code, but still understandable by humans. Allegedly. + . Convert the assembly language into machine code-yep, we are talking bits and bytes, ones and zeros here. + . Check that you have used things like functions and global variables in a consistent way. For example, if you have called a non-existent function, it will complain. + . If you are trying to produce an executable from several source code files, work out how to fit them all together. + . Work out how to produce something that the system's run-time loader will be able to load into memory and run. + . Finally, write the executable on the filesystem. + + The word _compiling_ is often used to refer to just steps 1 to 4-the others are referred to as _linking_. + Sometimes step 1 is referred to as _pre-processing_ and steps 3-4 as _assembling_. + + Fortunately, almost all this detail is hidden from you, as `cc` is a front end that manages calling all these programs with the right arguments for you; simply typing + + [source,bash] + .... + % cc foobar.c + .... + + will cause [.filename]#foobar.c# to be compiled by all the steps above. + If you have more than one file to compile, just do something like + + [source,bash] + .... + % cc foo.c bar.c + .... + + Note that the syntax checking is just that - checking the syntax. + It will not check for any logical mistakes you may have made, like putting the program into an infinite loop, + or using a bubble sort when you meant to use a binary sort.footnote:[In case you did not know, a binary sort is an efficient way of sorting things into order and a bubble sort is not.] + + There are lots and lots of options for `cc`, which are all in the manual page. + Here are a few of the most important ones, with examples of how to use them. + + `-o _filename_`:: + The output name of the file. If you do not use this option, `cc` will produce an executable called [.filename]#a.out#.footnote:[The reasons for this are buried in the mists of history.] + + + [source,bash] + .... + % cc foobar.c executable is a.out + % cc -o foobar foobar.c executable is foobar + .... + + `-c`:: + Just compile the file, do not link it. + Useful for toy programs where you just want to check the syntax, or if you are using a [.filename]#Makefile#. + + + [source,bash] + .... + % cc -c foobar.c + .... + + + This will produce an _object file_ (not an executable) called [.filename]#foobar.o#. + This can be linked together with other object files into an executable. + + `-g`:: + Create a debug version of the executable. + This makes the compiler put information into the executable about which line of which source file corresponds to which function call. + A debugger can use this information to show the source code as you step through the program, which is _very_ useful; + the disadvantage is that all this extra information makes the program much bigger. + Normally, you compile with `-g` while you are developing a program and then compile a "release version" without `-g` when you are satisfied it works properly. + + + + [source,bash] + .... + % cc -g foobar.c + .... + + + This will produce a debug version of the program. footnote:[Note, we did not use the -o flag to specify the executable name, so we will get an executable called a.out. Producing a debug version called foobar is left as an exercise for the reader!] + + `-O`:: + Create an optimized version of the executable. + The compiler performs various clever tricks to try to produce an executable that runs faster than normal. + You can add a number after the `-O` to specify a higher level of optimization, but this often exposes bugs in the compiler's optimizer. + + + [source,bash] + .... + % cc -O -o foobar foobar.c + .... + + + This will produce an optimized version of [.filename]#foobar#. + + The following three flags will force `cc` to check that your code complies to the relevant international standard, + often referred to as the ANSI standard, though strictly speaking it is an ISO standard. + + `-Wall`:: + Enable all the warnings which the authors of `cc` believe are worthwhile. + Despite the name, it will not enable all the warnings `cc` is capable of. + + `-ansi`:: + Turn off most, but not all, of the non-ANSI C features provided by `cc`. + Despite the name, it does not guarantee strictly that your code will comply to the standard. + + `-pedantic`:: + Turn off _all_ ``cc``'s non-ANSI C features. + + Without these flags, `cc` will allow you to use some of its non-standard extensions to the standard. + Some of these are very useful, but will not work with other compilers - in fact, + one of the main aims of the standard is to allow people to write code that will work with any compiler on any system. + This is known as _portable code_. + + Generally, you should try to make your code as portable as possible, + as otherwise you may have to completely rewrite the program later to get it to work somewhere else - and who knows what you may be using in a few years time? + + [source,bash] + .... + % cc -Wall -ansi -pedantic -o foobar foobar.c + .... + + This will produce an executable [.filename]#foobar# after checking [.filename]#foobar.c# for standard compliance. + + `-l__library__`:: + Specify a function library to be used at link time. + + + The most common example of this is when compiling a program that uses some of the mathematical functions in C. + Unlike most other platforms, these are in a separate library from the standard C one and you have to tell the compiler to add it. + + + The rule is that if the library is called [.filename]#libsomething.a#, you give `cc` the argument `-l__something__`. + For example, the math library is [.filename]#libm.a#, so you give `cc` the argument `-lm`. + A common "gotcha" with the math library is that it has to be the last library on the command line. + + + [source,bash] + .... + % cc -o foobar foobar.c -lm + .... + + + This will link the math library functions into [.filename]#foobar#. + + + If you are compiling C++ code, use {c-plus-plus-command}. + {c-plus-plus-command} can also be invoked as {clang-plus-plus-command} on FreeBSD. + + + [source,bash] + .... + % c++ -o foobar foobar.cc + .... + + + This will both produce an executable [.filename]#foobar# from the C++ source file [.filename]#foobar.cc#. + + === Common `cc` Queries and Problems + + ==== I compiled a file called foobar.c and I cannot find an executable called foobar. Where has it gone? + + Remember, `cc` will call the executable [.filename]#a.out# unless you tell it differently. + Use the `-o _filename_` option: + + [source,bash] + .... + % cc -o foobar foobar.c + .... + + ==== OK, I have an executable called foobar, I can see it when I run ls, but when I type in foobar at the command prompt it tells me there is no such file. Why can it not find it? + + Unlike MS-DOS(R), UNIX(R) does not look in the current directory when it is trying to find out which executable you want it to run, unless you tell it to. + Type `./foobar`, which means "run the file called [.filename]#foobar# in the current directory." + + === I called my executable test, but nothing happens when I run it. What is going on? + + Most UNIX(R) systems have a program called `test` in [.filename]#/usr/bin# and the shell is picking that one up before it gets to checking the current directory. + Either type: + + [source,bash] + .... + % ./test + .... + + or choose a better name for your program! + + ==== I compiled my program and it seemed to run all right at first, then there was an error and it said something about core dumped. What does that mean? + + The name _core dump_ dates back to the very early days of UNIX(R), when the machines used core memory for storing data. + Basically, if the program failed under certain conditions, the system would write the contents of core memory to disk in a file called [.filename]#core#, which the programmer could then pore over to find out what went wrong. + + ==== Fascinating stuff, but what I am supposed to do now? + + Use a debugger to analyze the core (see crossref:tools[debugging, Debugging]). + + ==== When my program dumped core, it said something about a segmentation fault. What is that? + + This basically means that your program tried to perform some sort of illegal operation on memory; + UNIX(R) is designed to protect the operating system and other programs from rogue programs. + + Common causes for this are: + + * Trying to write to a NULL pointer, eg + + + [.programlisting] + .... + char *foo = NULL; + strcpy(foo, "bang!"); + .... + + * Using a pointer that has not been initialized, eg + + + [.programlisting] + .... + char *foo; + strcpy(foo, "bang!"); + .... + + + The pointer will have some random value that, with luck, will point into an area of memory that is not available to your program and the kernel will kill your program before it can do any damage. + If you are unlucky, it will point somewhere inside your own program and corrupt one of your data structures, causing the program to fail mysteriously. + * Trying to access past the end of an array, eg + + + [.programlisting] + .... + int bar[20]; + bar[27] = 6; + .... + + * Trying to store something in read-only memory, eg + + + [.programlisting] + .... + char *foo = "My string"; + strcpy(foo, "bang!"); + .... + + + UNIX(R) compilers often put string literals like `"My string"` into read-only areas of memory. + * Doing naughty things with `malloc()` and `free()`, eg + + + [.programlisting] + .... + char bar[80]; + free(bar); + .... + + + or + + + [.programlisting] + .... + char *foo = malloc(27); + free(foo); + free(foo); + .... + + Making one of these mistakes will not always lead to an error, but they are always bad practice. + Some systems and compilers are more tolerant than others, + which is why programs that run well on one system can crash when you try them on another. + + ==== Sometimes when I get a core dump it says bus error. It says in my UNIX(R) book that this means a hardware problem, but the computer still seems to be working. Is this true? + + No, fortunately not (unless of course you really do have a hardware problem...). + This is usually another way of saying that you accessed memory in a way you should not have. + + ==== This dumping core business sounds as though it could be quite useful, if I can make it happen when I want to. Can I do this, or do I have to wait until there is an error? + + Yes, just go to another console or xterm, do + + [source,bash] + .... + % ps + .... + + to find out the process ID of your program, and do + + [source,bash] + .... + % kill -ABRT pid + .... + + where `_pid_` is the process ID you looked up. + + This is useful if your program has got stuck in an infinite loop, for instance. + If your program happens to trap SIGABRT, there are several other signals which have a similar effect. + + Alternatively, you can create a core dump from inside your program, by calling the `abort()` function. + See the manual page of man:abort[3] to learn more. + + If you want to create a core dump from outside your program, but do not want the process to terminate, you can use the `gcore` program. + See the manual page of man:gcore[1] for more information. + + [[tools-make]] + == Make + + === What is `make`? + + When you are working on a simple program with only one or two source files, typing in + + [source,bash] + .... + % cc file1.c file2.c + .... + + is not too bad, but it quickly becomes very tedious when there are several files-and it can take a while to compile, too. + + One way to get around this is to use object files and only recompile the source file if the source code has changed. + So we could have something like: + + [source,bash] + .... + % cc file1.o file2.o … file37.c … + .... + + if we had changed [.filename]#file37.c#, but not any of the others, since the last time we compiled. + This may speed up the compilation quite a bit, but does not solve the typing problem. + + Or we could write a shell script to solve the typing problem, but it would have to re-compile everything, making it very inefficient on a large project. + + What happens if we have hundreds of source files lying about? What if we are working in a team with other people who forget to tell us when they have changed one of their source files that we use? + + Perhaps we could put the two solutions together and write something like a shell script that would contain some kind of magic rule saying when a source file needs compiling. + Now all we need now is a program that can understand these rules, as it is a bit too complicated for the shell. + + This program is called `make`. + It reads in a file, called a _makefile_, that tells it how different files depend on each other, and works out which files need to be re-compiled and which ones do not. + For example, a rule could say something like "if [.filename]#fromboz.o# is older than [.filename]#fromboz.c#, that means someone must have changed [.filename]#fromboz.c#, so it needs to be re-compiled." + The makefile also has rules telling make _how_ to re-compile the source file, making it a much more powerful tool. + + Makefiles are typically kept in the same directory as the source they apply to, and can be called [.filename]#makefile#, [.filename]#Makefile# or [.filename]#MAKEFILE#. + Most programmers use the name [.filename]#Makefile#, as this puts it near the top of a directory listing, where it can easily be seen.footnote:[They do not use the MAKEFILE form as block capitals are often used for documentation files like README.] + + === Example of Using `make` + + Here is a very simple make file: + + [.programlisting] + .... + foo: foo.c + cc -o foo foo.c + .... + + It consists of two lines, a dependency line and a creation line. + + The dependency line here consists of the name of the program (known as the _target_), + followed by a colon, then whitespace, then the name of the source file. + When `make` reads this line, it looks to see if [.filename]#foo# exists; + if it exists, it compares the time [.filename]#foo# was last modified to the time [.filename]#foo.c# was last modified. + If [.filename]#foo# does not exist, or is older than [.filename]#foo.c#, it then looks at the creation line to find out what to do. + In other words, this is the rule for working out when [.filename]#foo.c# needs to be re-compiled. + + The creation line starts with a tab (press kbd:[tab]) and then the command you would type to create [.filename]#foo# if you were doing it at a command prompt. + If [.filename]#foo# is out of date, or does not exist, `make` then executes this command to create it. + In other words, this is the rule which tells make how to re-compile [.filename]#foo.c#. + + So, when you type `make`, it will make sure that [.filename]#foo# is up to date with respect to your latest changes to [.filename]#foo.c#. + This principle can be extended to [.filename]#Makefile#'s with hundreds of targets-in fact, on FreeBSD, + it is possible to compile the entire operating system just by typing `make buildworld buildkernel` at the top level directory in the src tree. + + Another useful property of makefiles is that the targets do not have to be programs. + For instance, we could have a make file that looks like this: + + [.programlisting] + .... + foo: foo.c + cc -o foo foo.c + + install: + cp foo /home/me + .... + + We can tell make which target we want to make by typing: + + [source,bash] + .... + % make target + .... + + `make` will then only look at that target and ignore any others. + For example, if we type `make foo` with the makefile above, make will ignore the `install` target. + + If we just type `make` on its own, make will always look at the first target and then stop without looking at any others. + So if we typed `make` here, it will just go to the `foo` target, re-compile [.filename]#foo# if necessary, and then stop without going on to the `install` target. + + Notice that the `install` target does not actually depend on anything! This means that the command on the following line is always executed when we try to make that target by typing `make install`. + In this case, it will copy [.filename]#foo# into the user's home directory. + This is often used by application makefiles, so that the application can be installed in the correct directory when it has been correctly compiled. + + This is a slightly confusing subject to try to explain. + If you do not quite understand how `make` works, the best thing to do is to write a simple program like "hello world" and a make file like the one above and experiment. + Then progress to using more than one source file, or having the source file include a header file. + `touch` is very useful here-it changes the date on a file without you having to edit it. + + === Make and include-files + + C code often starts with a list of files to include, for example stdio.h. + Some of these files are system-include files, some of them are from the project you are now working on: + + [.programlisting] + .... + #include + #include "foo.h" + + int main(.... + .... + + To make sure that this file is recompiled the moment [.filename]#foo.h# is changed, you have to add it in your [.filename]#Makefile#: + + [.programlisting] + .... + foo: foo.c foo.h + .... + + The moment your project is getting bigger and you have more and more own include-files to maintain, + it will be a pain to keep track of all include files and the files which are depending on it. + If you change an include-file but forget to recompile all the files which are depending on it, + the results will be devastating. + `clang` has an option to analyze your files and to produce a list of include-files and their dependencies: `-MM`. + + If you add this to your Makefile: + + [.programlisting] + .... + depend: + cc -E -MM *.c > .depend + .... + + and run `make depend`, the file [.filename]#.depend# will appear with a list of object-files, C-files and the include-files: + + [.programlisting] + .... + foo.o: foo.c foo.h + .... + + If you change [.filename]#foo.h#, next time you run `make` all files depending on [.filename]#foo.h# will be recompiled. + + Do not forget to run `make depend` each time you add an include-file to one of your files. + + === FreeBSD Makefiles + + Makefiles can be rather complicated to write. + Fortunately, BSD-based systems like FreeBSD come with some very powerful ones as part of the system. + One very good example of this is the FreeBSD ports system. + Here is the essential part of a typical ports [.filename]#Makefile#: + + [.programlisting] + .... + MASTER_SITES= ftp://freefall.cdrom.com/pub/FreeBSD/LOCAL_PORTS/ + DISTFILES= scheme-microcode+dist-7.3-freebsd.tgz + + .include + .... + + Now, if we go to the directory for this port and type `make`, the following happens: + + [.procedure] + . A check is made to see if the source code for this port is already on the system. + . If it is not, an FTP connection to the URL in MASTER_SITES is set up to download the source. + . The checksum for the source is calculated and compared it with one for a known, good, copy of the source. This is to make sure that the source was not corrupted while in transit. + . Any changes required to make the source work on FreeBSD are applied-this is known as _patching_. + . Any special configuration needed for the source is done. (Many UNIX(R) program distributions try to work out which version of UNIX(R) they are being compiled on and which optional UNIX(R) features are present-this is where they are given the information in the FreeBSD ports scenario). + . The source code for the program is compiled. In effect, we change to the directory where the source was unpacked and do `make`-the program's own make file has the necessary information to build the program. + . We now have a compiled version of the program. If we wish, we can test it now; when we feel confident about the program, we can type `make install`. This will cause the program and any supporting files it needs to be copied into the correct location; an entry is also made into a `package database`, so that the port can easily be uninstalled later if we change our mind about it. + + Now I think you will agree that is rather impressive for a four line script! + + The secret lies in the last line, which tells `make` to look in the system makefile called [.filename]#bsd.port.mk#. + It is easy to overlook this line, but this is where all the clever stuff comes from-someone has written a makefile that tells `make` to do all the things above (plus a couple of other things I did not mention, + including handling any errors that may occur) and anyone can get access to that just by putting a single line in their own make file! + + If you want to have a look at these system makefiles, they are in [.filename]#/usr/share/mk#, + but it is probably best to wait until you have had a bit of practice with makefiles, + as they are very complicated (and if you do look at them, make sure you have a flask of strong coffee handy!) + + === More Advanced Uses of `make` + + `Make` is a very powerful tool, and can do much more than the simple example above shows. + Unfortunately, there are several different versions of `make`, and they all differ considerably. + The best way to learn what they can do is probably to read the documentation-hopefully this introduction will have given you a base from which you can do this. + The man:make[1] manual page offers a comprehensive discussion of variables, arguments, and how to use make. + + Many applications in the ports use GNU make, which has a very good set of "info" pages. + If you have installed any of these ports, GNU make will automatically have been installed as `gmake`. + It is also available as a port and package in its own right. + + To view the info pages for GNU make, you will have to edit [.filename]#dir# in the [.filename]#/usr/local/info# directory to add an entry for it. + This involves adding a line like + + [.programlisting] + .... + * Make: (make). The GNU Make utility. + .... + + to the file. Once you have done this, you can type `info` and then select [.guimenuitem]#make# from the menu (or in Emacs, do `C-h i`). + + [[debugging]] + == Debugging + + === Introduction to Available Debuggers + + Using a debugger allows running the program under more controlled circumstances. + Typically, it is possible to step through the program a line at a time, inspect the value of variables, change them, tell the debugger to run up to a certain point and then stop, and so on. + It is also possible to attach to a program that is already running, or load a core file to investigate why the program crashed. + + This section is intended to be a quick introduction to using debuggers and does not cover specialized topics such as debugging the kernel. + For more information about that, refer to crossref:kerneldebug[kerneldebug,Kernel Debugging]. + + The standard debugger supplied with FreeBSD is called `lldb` (LLVM debugger). + As it is part of the standard installation for that release, there is no need to do anything special to use it. + It has good command help, accessible via the `help` command, as well as https://lldb.llvm.org/[a web tutorial and documentation]. + + [NOTE] + ==== + The `lldb` command is also available extref:{handbook}ports/[from ports or packages, ports-using] as package:devel/llvm[]. + ==== + + The other debugger available with FreeBSD is called `gdb` (GNU debugger). + Unlike lldb, it is not installed by default on FreeBSD; + to use it, extref:{handbook}#ports-using/[install] package:devel/gdb[] from ports or packages. + It has excellent on-line help, as well as a set of info pages. + + The two debuggers have a similar feature set, so which one to use is largely a matter of taste. + If familiar with one only, use that one. + People familiar with neither or both but wanting to use one from inside Emacs will need to use `gdb` as `lldb` is unsupported by Emacs. + Otherwise, try both and see which one you prefer. + + === Using lldb + + ==== Starting lldb + + Start up lldb by typing + + [source,bash] + .... + % lldb -- progname + .... + + ==== Running a Program with lldb + + Compile the program with `-g` to get the most out of using `lldb`. + It will work without, but will only display the name of the function currently running, instead of the source code. + If it displays a line like: + + [source,bash] + .... + Breakpoint 1: where = temp`main, address = … + .... + + (without an indication of source code filename and line number) when setting a breakpoint, this means that the program was not compiled with `-g`. + + [TIP] + ==== + Most `lldb` commands have shorter forms that can be used instead. + The longer forms are used here for clarity. + ==== + + At the `lldb` prompt, type `breakpoint set -n main`. + This will tell the debugger not to display the preliminary set-up code in the program being run and to stop execution at the beginning of the program's code. + Now type `process launch` to actually start the program- it will start at the beginning of the set-up code and then get stopped by the debugger when it calls `main()`. + + To step through the program a line at a time, type `thread step-over`. + When the program gets to a function call, step into it by typing `thread step-in`. + Once in a function call, return from it by typing `thread step-out` or use `up` and `down` to take a quick look at the caller. + + Here is a simple example of how to spot a mistake in a program with `lldb`. + This is our program (with a deliberate mistake): + + [.programlisting] + .... + #include + + int bazz(int anint); + + main() { + int i; + + printf("This is my program\n"); + bazz(i); + return 0; + } + + int bazz(int anint) { + printf("You gave me %d\n", anint); + return anint; + } + .... + + This program sets i to be `5` and passes it to a function `bazz()` which prints out the number we gave it. + + Compiling and running the program displays + + [source,bash] + .... + % cc -g -o temp temp.c + % ./temp + This is my program + anint = -5360 + .... + + That is not what was expected! Time to see what is going on! + + [source,bash] + .... + % lldb -- temp + (lldb) target create "temp" + Current executable set to 'temp' (x86_64). + (lldb) breakpoint set -n main Skip the set-up code + Breakpoint 1: where = temp`main + 15 at temp.c:8:2, address = 0x00000000002012ef lldb puts breakpoint at main() + (lldb) process launch Run as far as main() + Process 9992 launching + Process 9992 launched: '/home/pauamma/tmp/temp' (x86_64) Program starts running + + Process 9992 stopped + * thread #1, name = 'temp', stop reason = breakpoint 1.1 lldb stops at main() + frame #0: 0x00000000002012ef temp`main at temp.c:8:2 + 5 main() { + 6 int i; + 7 + -> 8 printf("This is my program\n"); Indicates the line where it stopped + 9 bazz(i); + 10 return 0; + 11 } + (lldb) thread step-over Go to next line + This is my program Program prints out + Process 9992 stopped + * thread #1, name = 'temp', stop reason = step over + frame #0: 0x0000000000201300 temp`main at temp.c:9:7 + 6 int i; + 7 + 8 printf("This is my program\n"); + -> 9 bazz(i); + 10 return 0; + 11 } + 12 + (lldb) thread step-in step into bazz() + Process 9992 stopped + * thread #1, name = 'temp', stop reason = step in + frame #0: 0x000000000020132b temp`bazz(anint=-5360) at temp.c:14:29 lldb displays stack frame + 11 } + 12 + 13 int bazz(int anint) { + -> 14 printf("You gave me %d\n", anint); + 15 return anint; + 16 } + (lldb) + .... + + Hang on a minute! How did anint get to be `-5360`? Was it not set to `5` in `main()`? Let us move up to `main()` and have a look. + + [source,bash] + .... + (lldb) up Move up call stack + frame #1: 0x000000000020130b temp`main at temp.c:9:2 lldb displays stack frame + 6 int i; + 7 + 8 printf("This is my program\n"); + -> 9 bazz(i); + 10 return 0; + 11 } + 12 + (lldb) frame variable i Show us the value of i + (int) i = -5360 lldb displays -5360 + .... + + Oh dear! Looking at the code, we forgot to initialize i. + We meant to put + + [.programlisting] + .... + ... + main() { + int i; + + i = 5; + printf("This is my program\n"); + ... + .... + + but we left the `i=5;` line out. + As we did not initialize i, it had whatever number happened to be in that area of memory when the program ran, + which in this case happened to be `-5360`. + + [NOTE] + ==== + The `lldb` command displays the stack frame every time we go into or out of a function, even if we are using `up` and `down` to move around the call stack. + This shows the name of the function and the values of its arguments, which helps us keep track of where we are and what is going on. + (The stack is a storage area where the program stores information about the arguments passed to functions and where to go when it returns from a function call.) + ==== + + ==== Examining a Core File with lldb + + A core file is basically a file which contains the complete state of the process when it crashed. + In "the good old days", programmers had to print out hex listings of core files and sweat over machine code manuals, but now life is a bit easier. + Incidentally, under FreeBSD and other 4.4BSD systems, a core file is called [.filename]#progname.core# instead of just [.filename]#core#, to make it clearer which program a core file belongs to. + + To examine a core file, specify the name of the core file in addition to the program itself. + Instead of starting up `lldb` in the usual way, type `lldb -c _progname_.core \-- _progname_`. + + The debugger will display something like this: + + [source,bash,subs="verbatim,quotes"] + .... + % lldb -c [.filename]#progname.core# -- [.filename]#progname# + (lldb) target create "[.filename]#progname#" --core "[.filename]#progname#.core" + Core file '/home/pauamma/tmp/[.filename]#progname.core#' (x86_64) was loaded. + (lldb) + .... + + In this case, the program was called [.filename]#progname#, so the core file is called [.filename]#progname.core#. + The debugger does not display why the program crashed or where. + For this, use `thread backtrace all`. + This will also show how the function where the program dumped core was called. + + [source,bash,subs="verbatim,quotes"] + .... + (lldb) thread backtrace all + * thread #1, name = 'progname', stop reason = signal SIGSEGV + * frame #0: 0x0000000000201347 progname`bazz(anint=5) at temp2.c:17:10 + frame #1: 0x0000000000201312 progname`main at temp2.c:10:2 + frame #2: 0x000000000020110f progname`_start(ap=, cleanup=) at crt1.c:76:7 + (lldb) + .... + + `SIGSEGV` indicates that the program tried to access memory (run code or read/write data usually) at a location that does not belong to it, but does not give any specifics. + For that, look at the source code at line 10 of file temp2.c, in `bazz()`. + The backtrace also says that in this case, `bazz()` was called from `main()`. + + ==== Attaching to a Running Program with lldb + + One of the neatest features about `lldb` is that it can attach to a program that is already running. + Of course, that requires sufficient permissions to do so. + A common problem is stepping through a program that forks and wanting to trace the child, but the debugger will only trace the parent. + + To do that, start up another `lldb`, use `ps` to find the process ID for the child, and do + + [source,bash] + .... + (lldb) process attach -p pid + .... + + in `lldb`, and then debug as usual. + + For that to work well, the code that calls `fork` to create the child needs to do something like the following (courtesy of the `gdb` info pages): + + [.programlisting] + .... + ... + if ((pid = fork()) < 0) /* _Always_ check this */ + error(); + else if (pid == 0) { /* child */ + int PauseMode = 1; + + while (PauseMode) + sleep(10); /* Wait until someone attaches to us */ + ... + } else { /* parent */ + ... + .... + + Now all that is needed is to attach to the child, set PauseMode to `0` with `expr PauseMode = 0` and wait for the `sleep()` call to return. + + === Remote Debugging Using LLDB + + [NOTE] + ==== + The described functionality is available starting with LLDB version 12.0.0. +-Users of FreeBSD releases containing an earlier LLDB version may wish to use the snapshot available in extref:{handbook}[ports or packages, ports-using], as package:devel/llvm-devel[]. ++Users of FreeBSD releases containing an earlier LLDB version may wish to use the snapshot available in extref:{handbook}ports[ports or packages, ports-using], as package:devel/llvm-devel[]. + ==== + + Starting with LLDB 12.0.0, remote debugging is supported on FreeBSD. + This means that `lldb-server` can be started to debug a program on one host, while the interactive `lldb` client connects to it from another one. + + To launch a new process to be debugged remotely, run `lldb-server` on the remote server by typing + + [source,bash] + .... + % lldb-server g host:port -- progname + .... + + The process will be stopped immediately after launching, and `lldb-server` will wait for the client to connect. + + Start `lldb` locally and type the following command to connect to the remote server: + + [source,bash] + .... + (lldb) gdb-remote host:port + .... + + `lldb-server` can also attach to a running process. + To do that, type the following on the remote server: + + [source,bash] + .... + % lldb-server g host:port --attach pid-or-name + .... + + === Using gdb + + ==== Starting gdb + + Start up gdb by typing + + [source,bash] + .... + % gdb progname + .... + + although many people prefer to run it inside Emacs. + To do this, type: + + [source,bash] + .... + M-x gdb RET progname RET + .... + + Finally, for those finding its text-based command-prompt style off-putting, there is a graphical front-end for it (package:devel/xxgdb[]) in the Ports Collection. + + ==== Running a Program with gdb + + Compile the program with `-g` to get the most out of using `gdb`. + It will work without, but will only display the name of the function currently running, instead of the source code. + A line like: + + [source,bash] + .... + ... (no debugging symbols found) ... + .... + + when `gdb` starts up means that the program was not compiled with `-g`. + + At the `gdb` prompt, type `break main`. + This will tell the debugger to skip the preliminary set-up code in the program being run and to stop execution at the beginning of the program's code. + Now type `run` to start the program- it will start at the beginning of the set-up code and then get stopped by the debugger when it calls `main()`. + + To step through the program a line at a time, press `n`. + When at a function call, step into it by pressing `s`. + Once in a function call, return from it by pressing `f`, or use `up` and `down` to take a quick look at the caller. + + Here is a simple example of how to spot a mistake in a program with `gdb`. + This is our program (with a deliberate mistake): + + [.programlisting] + .... + #include + + int bazz(int anint); + + main() { + int i; + + printf("This is my program\n"); + bazz(i); + return 0; + } + + int bazz(int anint) { + printf("You gave me %d\n", anint); + return anint; + } + .... + + This program sets i to be `5` and passes it to a function `bazz()` which prints out the number we gave it. + + Compiling and running the program displays + + [source,bash] + .... + % cc -g -o temp temp.c + % ./temp + This is my program + anint = 4231 + .... + + That was not what we expected! Time to see what is going on! + + [source,bash] + .... + % gdb temp + GDB is free software and you are welcome to distribute copies of it + under certain conditions; type "show copying" to see the conditions. + There is absolutely no warranty for GDB; type "show warranty" for details. + GDB 4.13 (i386-unknown-freebsd), Copyright 1994 Free Software Foundation, Inc. + (gdb) break main Skip the set-up code + Breakpoint 1 at 0x160f: file temp.c, line 9. gdb puts breakpoint at main() + (gdb) run Run as far as main() + Starting program: /home/james/tmp/temp Program starts running + + Breakpoint 1, main () at temp.c:9 gdb stops at main() + (gdb) n Go to next line + This is my program Program prints out + (gdb) s step into bazz() + bazz (anint=4231) at temp.c:17 gdb displays stack frame + (gdb) + .... + + Hang on a minute! How did anint get to be `4231`? Was it not set to `5` in `main()`? Let us move up to `main()` and have a look. + + [source,bash] + .... + (gdb) up Move up call stack + #1 0x1625 in main () at temp.c:11 gdb displays stack frame + (gdb) p i Show us the value of i + $1 = 4231 gdb displays 4231 + .... + + Oh dear! Looking at the code, we forgot to initialize i. + We meant to put + + [.programlisting] + .... + ... + main() { + int i; + + i = 5; + printf("This is my program\n"); + ... + .... + + but we left the `i=5;` line out. + As we did not initialize i, it had whatever number happened to be in that area of memory when the program ran, + which in this case happened to be `4231`. + + [NOTE] + ==== + The `gdb` command displays the stack frame every time we go into or out of a function, even if we are using `up` and `down` to move around the call stack. + This shows the name of the function and the values of its arguments, which helps us keep track of where we are and what is going on. + (The stack is a storage area where the program stores information about the arguments passed to functions and where to go when it returns from a function call.) + ==== + + ==== Examining a Core File with gdb + + A core file is basically a file which contains the complete state of the process when it crashed. + In "the good old days", programmers had to print out hex listings of core files and sweat over machine code manuals, but now life is a bit easier. + Incidentally, under FreeBSD and other 4.4BSD systems, a core file is called [.filename]#progname.core# instead of just [.filename]#core#, to make it clearer which program a core file belongs to. + + To examine a core file, start up `gdb` in the usual way. + Instead of typing `break` or `run`, type + + [source,bash] + .... + (gdb) core progname.core + .... + + If the core file is not in the current directory, type `dir /path/to/core/file` first. + + The debugger should display something like this: + + [source,bash,subs="verbatim,quotes"] + .... + % gdb [.filename]#progname# + GDB is free software and you are welcome to distribute copies of it + under certain conditions; type "show copying" to see the conditions. + There is absolutely no warranty for GDB; type "show warranty" for details. + GDB 4.13 (i386-unknown-freebsd), Copyright 1994 Free Software Foundation, Inc. + (gdb) core [.filename]#progname.core# + Core was generated by `[.filename]#progname#'. + Program terminated with signal 11, Segmentation fault. + Cannot access memory at address 0x7020796d. + #0 0x164a in bazz (anint=0x5) at temp.c:17 + (gdb) + .... + + In this case, the program was called [.filename]#progname#, so the core file is called [.filename]#progname.core#. + We can see that the program crashed due to trying to access an area in memory that was not available to it in a function called `bazz`. + + Sometimes it is useful to be able to see how a function was called, + as the problem could have occurred a long way up the call stack in a complex program. + `bt` causes `gdb` to print out a back-trace of the call stack: + + [source,bash] + .... + (gdb) bt + #0 0x164a in bazz (anint=0x5) at temp.c:17 + #1 0xefbfd888 in end () + #2 0x162c in main () at temp.c:11 + (gdb) + .... + + The `end()` function is called when a program crashes; + in this case, the `bazz()` function was called from `main()`. + + ==== Attaching to a Running Program with gdb + + One of the neatest features about `gdb` is that it can attach to a program that is already running. + Of course, that requires sufficient permissions to do so. + A common problem is stepping through a program that forks and wanting to trace the child, but the debugger will only trace the parent. + + To do that, start up another `gdb`, use `ps` to find the process ID for the child, and do + + [source,bash] + .... + (gdb) attach pid + .... + + in `gdb`, and then debug as usual. + + For that to work well, the code that calls `fork` to create the child needs to do something like the following (courtesy of the `gdb` info pages): + + [.programlisting] + .... + ... + if ((pid = fork()) < 0) /* _Always_ check this */ + error(); + else if (pid == 0) { /* child */ + int PauseMode = 1; + + while (PauseMode) + sleep(10); /* Wait until someone attaches to us */ + ... + } else { /* parent */ + ... + .... + + Now all that is needed is to attach to the child, set PauseMode to `0`, and wait for the `sleep()` call to return! + + [[emacs]] + == Using Emacs as a Development Environment + + === Emacs + + Emacs is a highly customizable editor-indeed, it has been customized to the point where it is more like an operating system than an editor! + Many developers and sysadmins do in fact spend practically all their time working inside Emacs, leaving it only to log out. + + It is impossible even to summarize everything Emacs can do here, but here are some of the features of interest to developers: + + * Very powerful editor, allowing search-and-replace on both strings and regular expressions (patterns), jumping to start/end of block expression, etc, etc. + * Pull-down menus and online help. + * Language-dependent syntax highlighting and indentation. + * Completely customizable. + * You can compile and debug programs within Emacs. + * On a compilation error, you can jump to the offending line of source code. + * Friendly-ish front-end to the `info` program used for reading GNU hypertext documentation, including the documentation on Emacs itself. + * Friendly front-end to `gdb`, allowing you to look at the source code as you step through your program. + + And doubtless many more that have been overlooked. + + Emacs can be installed on FreeBSD using the package:editors/emacs[] port. + + Once it is installed, start it up and do `C-h t` to read an Emacs tutorial-that means hold down kbd:[control], press kbd:[h], let go of kbd:[control], and then press kbd:[t]. + (Alternatively, you can use the mouse to select [.guimenuitem]#Emacs Tutorial# from the menu:Help[] menu.) + + Although Emacs does have menus, it is well worth learning the key bindings, + as it is much quicker when you are editing something to press a couple of keys than to try to find the mouse and then click on the right place. + And, when you are talking to seasoned Emacs users, you will find they often casually throw around expressions like "`M-x replace-s RET foo RET bar RET`" so it is useful to know what they mean. + And in any case, Emacs has far too many useful functions for them to all fit on the menu bars. + + Fortunately, it is quite easy to pick up the key-bindings, as they are displayed next to the menu item. + My advice is to use the menu item for, say, opening a file until you understand how it works and feel confident with it, then try doing C-x C-f. + When you are happy with that, move on to another menu command. + + If you cannot remember what a particular combination of keys does, select [.guimenuitem]#Describe Key# from the menu:Help[] menu and type it in-Emacs will tell you what it does. + You can also use the [.guimenuitem]#Command Apropos# menu item to find out all the commands which contain a particular word in them, with the key binding next to it. + + By the way, the expression above means hold down the kbd:[Meta] key, press kbd:[x], release the kbd:[Meta] key, type `replace-s` (short for `replace-string`-another feature of Emacs is that you can abbreviate commands), press the kbd:[return] key, type `foo` (the string you want replaced), press the kbd:[return] key, type bar (the string you want to replace `foo` with) and press kbd:[return] again. + Emacs will then do the search-and-replace operation you have just requested. + + If you are wondering what on earth kbd:[Meta] is, it is a special key that many UNIX(R) workstations have. + Unfortunately, PC's do not have one, so it is usually kbd:[alt] (or if you are unlucky, the kbd:[escape] key). + + Oh, and to get out of Emacs, do `C-x C-c` (that means hold down the kbd:[control] key, press kbd:[x], press kbd:[c] and release the kbd:[control] key). + If you have any unsaved files open, Emacs will ask you if you want to save them. + (Ignore the bit in the documentation where it says `C-z` is the usual way to leave Emacs-that leaves Emacs hanging around in the background, and is only really useful if you are on a system which does not have virtual terminals). + + === Configuring Emacs + + Emacs does many wonderful things; some of them are built in, some of them need to be configured. + + Instead of using a proprietary macro language for configuration, Emacs uses a version of Lisp specially adapted for editors, known as Emacs Lisp. + Working with Emacs Lisp can be quite helpful if you want to go on and learn something like Common Lisp. + Emacs Lisp has many features of Common Lisp, although it is considerably smaller (and thus easier to master). + + The best way to learn Emacs Lisp is to read the online link:https://www.gnu.org/software/emacs/manual/elisp.html[Emacs Reference] manual. + + However, there is no need to actually know any Lisp to get started with configuring Emacs, + as I have included a sample [.filename]#.emacs#, which should be enough to get you started. + Just copy it into your home directory and restart Emacs if it is already running; + it will read the commands from the file and (hopefully) give you a useful basic setup. + + === A Sample [.filename]#.emacs# + + Unfortunately, there is far too much here to explain it in detail; however there are one or two points worth mentioning. + + * Everything beginning with a `;` is a comment and is ignored by Emacs. + * In the first line, the `-*- Emacs-Lisp -*-` is so that we can edit [.filename]#.emacs# itself within Emacs and get all the fancy features for editing Emacs Lisp. Emacs usually tries to guess this based on the filename, and may not get it right for [.filename]#.emacs#. + * The kbd:[tab] key is bound to an indentation function in some modes, so when you press the tab key, it will indent the current line of code. If you want to put a tab character in whatever you are writing, hold the kbd:[control] key down while you are pressing the kbd:[tab] key. + * This file supports syntax highlighting for C, C++, Perl, Lisp and Scheme, by guessing the language from the filename. + * Emacs already has a pre-defined function called `next-error`. In a compilation output window, this allows you to move from one compilation error to the next by doing `M-n`; we define a complementary function, `previous-error`, that allows you to go to a previous error by doing `M-p`. The nicest feature of all is that `C-c C-c` will open up the source file in which the error occurred and jump to the appropriate line. + * We enable Emacs's ability to act as a server, so that if you are doing something outside Emacs and you want to edit a file, you can just type in + + + [source,bash] + .... + % emacsclient filename + .... + + + and then you can edit the file in your Emacs!footnote:[Many Emacs users set their EDITOR environment to emacsclient so this happens every time they need to edit a file.] + + .A Sample [.filename]#.emacs# + ==== + [.programlisting] + .... + ;; -*-Emacs-Lisp-*- + + ;; This file is designed to be re-evaled; use the variable first-time + ;; to avoid any problems with this. + (defvar first-time t + "Flag signifying this is the first time that .emacs has been evaled") + + ;; Meta + (global-set-key "\M- " 'set-mark-command) + (global-set-key "\M-\C-h" 'backward-kill-word) + (global-set-key "\M-\C-r" 'query-replace) + (global-set-key "\M-r" 'replace-string) + (global-set-key "\M-g" 'goto-line) + (global-set-key "\M-h" 'help-command) + + ;; Function keys + (global-set-key [f1] 'manual-entry) + (global-set-key [f2] 'info) + (global-set-key [f3] 'repeat-complex-command) + (global-set-key [f4] 'advertised-undo) + (global-set-key [f5] 'eval-current-buffer) + (global-set-key [f6] 'buffer-menu) + (global-set-key [f7] 'other-window) + (global-set-key [f8] 'find-file) + (global-set-key [f9] 'save-buffer) + (global-set-key [f10] 'next-error) + (global-set-key [f11] 'compile) + (global-set-key [f12] 'grep) + (global-set-key [C-f1] 'compile) + (global-set-key [C-f2] 'grep) + (global-set-key [C-f3] 'next-error) + (global-set-key [C-f4] 'previous-error) + (global-set-key [C-f5] 'display-faces) + (global-set-key [C-f8] 'dired) + (global-set-key [C-f10] 'kill-compilation) + + ;; Keypad bindings + (global-set-key [up] "\C-p") + (global-set-key [down] "\C-n") + (global-set-key [left] "\C-b") + (global-set-key [right] "\C-f") + (global-set-key [home] "\C-a") + (global-set-key [end] "\C-e") + (global-set-key [prior] "\M-v") + (global-set-key [next] "\C-v") + (global-set-key [C-up] "\M-\C-b") + (global-set-key [C-down] "\M-\C-f") + (global-set-key [C-left] "\M-b") + (global-set-key [C-right] "\M-f") + (global-set-key [C-home] "\M-<") + (global-set-key [C-end] "\M->") + (global-set-key [C-prior] "\M-<") + (global-set-key [C-next] "\M->") + + ;; Mouse + (global-set-key [mouse-3] 'imenu) + + ;; Misc + (global-set-key [C-tab] "\C-q\t") ; Control tab quotes a tab. + (setq backup-by-copying-when-mismatch t) + + ;; Treat 'y' or as yes, 'n' as no. + (fset 'yes-or-no-p 'y-or-n-p) + (define-key query-replace-map [return] 'act) + (define-key query-replace-map [?\C-m] 'act) + + ;; Load packages + (require 'desktop) + (require 'tar-mode) + + ;; Pretty diff mode + (autoload 'ediff-buffers "ediff" "Intelligent Emacs interface to diff" t) + (autoload 'ediff-files "ediff" "Intelligent Emacs interface to diff" t) + (autoload 'ediff-files-remote "ediff" + "Intelligent Emacs interface to diff") + + (if first-time + (setq auto-mode-alist + (append '(("\\.cpp$" . c++-mode) + ("\\.hpp$" . c++-mode) + ("\\.lsp$" . lisp-mode) + ("\\.scm$" . scheme-mode) + ("\\.pl$" . perl-mode) + ) auto-mode-alist))) + + ;; Auto font lock mode + (defvar font-lock-auto-mode-list + (list 'c-mode 'c++-mode 'c++-c-mode 'emacs-lisp-mode 'lisp-mode 'perl-mode 'scheme-mode) + "List of modes to always start in font-lock-mode") + + (defvar font-lock-mode-keyword-alist + '((c++-c-mode . c-font-lock-keywords) + (perl-mode . perl-font-lock-keywords)) + "Associations between modes and keywords") + + (defun font-lock-auto-mode-select () + "Automatically select font-lock-mode if the current major mode is in font-lock-auto-mode-list" + (if (memq major-mode font-lock-auto-mode-list) + (progn + (font-lock-mode t)) + ) + ) + + (global-set-key [M-f1] 'font-lock-fontify-buffer) + + ;; New dabbrev stuff + ;(require 'new-dabbrev) + (setq dabbrev-always-check-other-buffers t) + (setq dabbrev-abbrev-char-regexp "\\sw\\|\\s_") + (add-hook 'emacs-lisp-mode-hook + '(lambda () + (set (make-local-variable 'dabbrev-case-fold-search) nil) + (set (make-local-variable 'dabbrev-case-replace) nil))) + (add-hook 'c-mode-hook + '(lambda () + (set (make-local-variable 'dabbrev-case-fold-search) nil) + (set (make-local-variable 'dabbrev-case-replace) nil))) + (add-hook 'text-mode-hook + '(lambda () + (set (make-local-variable 'dabbrev-case-fold-search) t) + (set (make-local-variable 'dabbrev-case-replace) t))) + + ;; C++ and C mode... + (defun my-c++-mode-hook () + (setq tab-width 4) + (define-key c++-mode-map "\C-m" 'reindent-then-newline-and-indent) + (define-key c++-mode-map "\C-ce" 'c-comment-edit) + (setq c++-auto-hungry-initial-state 'none) + (setq c++-delete-function 'backward-delete-char) + (setq c++-tab-always-indent t) + (setq c-indent-level 4) + (setq c-continued-statement-offset 4) + (setq c++-empty-arglist-indent 4)) + + (defun my-c-mode-hook () + (setq tab-width 4) + (define-key c-mode-map "\C-m" 'reindent-then-newline-and-indent) + (define-key c-mode-map "\C-ce" 'c-comment-edit) + (setq c-auto-hungry-initial-state 'none) + (setq c-delete-function 'backward-delete-char) + (setq c-tab-always-indent t) + ;; BSD-ish indentation style + (setq c-indent-level 4) + (setq c-continued-statement-offset 4) + (setq c-brace-offset -4) + (setq c-argdecl-indent 0) + (setq c-label-offset -4)) + + ;; Perl mode + (defun my-perl-mode-hook () + (setq tab-width 4) + (define-key c++-mode-map "\C-m" 'reindent-then-newline-and-indent) + (setq perl-indent-level 4) + (setq perl-continued-statement-offset 4)) + + ;; Scheme mode... + (defun my-scheme-mode-hook () + (define-key scheme-mode-map "\C-m" 'reindent-then-newline-and-indent)) + + ;; Emacs-Lisp mode... + (defun my-lisp-mode-hook () + (define-key lisp-mode-map "\C-m" 'reindent-then-newline-and-indent) + (define-key lisp-mode-map "\C-i" 'lisp-indent-line) + (define-key lisp-mode-map "\C-j" 'eval-print-last-sexp)) + + ;; Add all of the hooks... + (add-hook 'c++-mode-hook 'my-c++-mode-hook) + (add-hook 'c-mode-hook 'my-c-mode-hook) + (add-hook 'scheme-mode-hook 'my-scheme-mode-hook) + (add-hook 'emacs-lisp-mode-hook 'my-lisp-mode-hook) + (add-hook 'lisp-mode-hook 'my-lisp-mode-hook) + (add-hook 'perl-mode-hook 'my-perl-mode-hook) + + ;; Complement to next-error + (defun previous-error (n) + "Visit previous compilation error message and corresponding source code." + (interactive "p") + (next-error (- n))) + + ;; Misc... + (transient-mark-mode 1) + (setq mark-even-if-inactive t) + (setq visible-bell nil) + (setq next-line-add-newlines nil) + (setq compile-command "make") + (setq suggest-key-bindings nil) + (put 'eval-expression 'disabled nil) + (put 'narrow-to-region 'disabled nil) + (put 'set-goal-column 'disabled nil) + (if (>= emacs-major-version 21) + (setq show-trailing-whitespace t)) + + ;; Elisp archive searching + (autoload 'format-lisp-code-directory "lispdir" nil t) + (autoload 'lisp-dir-apropos "lispdir" nil t) + (autoload 'lisp-dir-retrieve "lispdir" nil t) + (autoload 'lisp-dir-verify "lispdir" nil t) + + ;; Font lock mode + (defun my-make-face (face color &optional bold) + "Create a face from a color and optionally make it bold" + (make-face face) + (copy-face 'default face) + (set-face-foreground face color) + (if bold (make-face-bold face)) + ) + + (if (eq window-system 'x) + (progn + (my-make-face 'blue "blue") + (my-make-face 'red "red") + (my-make-face 'green "dark green") + (setq font-lock-comment-face 'blue) + (setq font-lock-string-face 'bold) + (setq font-lock-type-face 'bold) + (setq font-lock-keyword-face 'bold) + (setq font-lock-function-name-face 'red) + (setq font-lock-doc-string-face 'green) + (add-hook 'find-file-hooks 'font-lock-auto-mode-select) + + (setq baud-rate 1000000) + (global-set-key "\C-cmm" 'menu-bar-mode) + (global-set-key "\C-cms" 'scroll-bar-mode) + (global-set-key [backspace] 'backward-delete-char) + ; (global-set-key [delete] 'delete-char) + (standard-display-european t) + (load-library "iso-transl"))) + + ;; X11 or PC using direct screen writes + (if window-system + (progn + ;; (global-set-key [M-f1] 'hilit-repaint-command) + ;; (global-set-key [M-f2] [?\C-u M-f1]) + (setq hilit-mode-enable-list + '(not text-mode c-mode c++-mode emacs-lisp-mode lisp-mode + scheme-mode) + hilit-auto-highlight nil + hilit-auto-rehighlight 'visible + hilit-inhibit-hooks nil + hilit-inhibit-rebinding t) + (require 'hilit19) + (require 'paren)) + (setq baud-rate 2400) ; For slow serial connections + ) + + ;; TTY type terminal + (if (and (not window-system) + (not (equal system-type 'ms-dos))) + (progn + (if first-time + (progn + (keyboard-translate ?\C-h ?\C-?) + (keyboard-translate ?\C-? ?\C-h))))) + + ;; Under UNIX + (if (not (equal system-type 'ms-dos)) + (progn + (if first-time + (server-start)))) + + ;; Add any face changes here + (add-hook 'term-setup-hook 'my-term-setup-hook) + (defun my-term-setup-hook () + (if (eq window-system 'pc) + (progn + ;; (set-face-background 'default "red") + ))) + + ;; Restore the "desktop" - do this as late as possible + (if first-time + (progn + (desktop-load-default) + (desktop-read))) + + ;; Indicate that this file has been read at least once + (setq first-time nil) + + ;; No need to debug anything now + + (setq debug-on-error nil) + + ;; All done + (message "All done, %s%s" (user-login-name) ".") + .... + ==== + + === Extending the Range of Languages Emacs Understands + + Now, this is all very well if you only want to program in the languages already catered for in [.filename]#.emacs# (C, C++, Perl, Lisp and Scheme), but what happens if a new language called "whizbang" comes out, full of exciting features? + + The first thing to do is find out if whizbang comes with any files that tell Emacs about the language. + These usually end in [.filename]#.el#, short for "Emacs Lisp". + For example, if whizbang is a FreeBSD port, we can locate these files by doing + + [source,bash] + .... + % find /usr/ports/lang/whizbang -name "*.el" -print + .... + + and install them by copying them into the Emacs site Lisp directory. + On FreeBSD, this is [.filename]#/usr/local/share/emacs/site-lisp#. + + So for example, if the output from the find command was + + [source,bash] + .... + /usr/ports/lang/whizbang/work/misc/whizbang.el + .... + + we would do + + [source,bash] + .... + # cp /usr/ports/lang/whizbang/work/misc/whizbang.el /usr/local/share/emacs/site-lisp + .... + + Next, we need to decide what extension whizbang source files have. + Let us say for the sake of argument that they all end in [.filename]#.wiz#. + We need to add an entry to our [.filename]#.emacs# to make sure Emacs will be able to use the information in [.filename]#whizbang.el#. + + Find the auto-mode-alist entry in [.filename]#.emacs# and add a line for whizbang, such as: + + [.programlisting] + .... + ... + ("\\.lsp$" . lisp-mode) + ("\\.wiz$" . whizbang-mode) + ("\\.scm$" . scheme-mode) + ... + .... + + This means that Emacs will automatically go into `whizbang-mode` when you edit a file ending in [.filename]#.wiz#. + + Just below this, you will find the font-lock-auto-mode-list entry. + Add `whizbang-mode` to it like so: + + [.programlisting] + .... + ;; Auto font lock mode + (defvar font-lock-auto-mode-list + (list 'c-mode 'c++-mode 'c++-c-mode 'emacs-lisp-mode 'whizbang-mode 'lisp-mode 'perl-mode 'scheme-mode) + "List of modes to always start in font-lock-mode") + .... + + This means that Emacs will always enable `font-lock-mode` (ie syntax highlighting) when editing a [.filename]#.wiz# file. + + And that is all that is needed. If there is anything else you want done automatically when you open up [.filename]#.wiz#, + you can add a `whizbang-mode hook` (see `my-scheme-mode-hook` for a simple example that adds `auto-indent`). + + [[tools-reading]] + == Further Reading + + For information about setting up a development environment for contributing fixes to FreeBSD itself, please see man:development[7]. + + * Brian Harvey and Matthew Wright _Simply Scheme_ MIT 1994. ISBN 0-262-08226-8 + * Randall Schwartz _Learning Perl_ O'Reilly 1993 ISBN 1-56592-042-2 + * Patrick Henry Winston and Berthold Klaus Paul Horn _Lisp (3rd Edition)_ Addison-Wesley 1989 ISBN 0-201-08319-1 + * Brian W. Kernighan and Rob Pike _The Unix Programming Environment_ Prentice-Hall 1984 ISBN 0-13-937681-X + * Brian W. Kernighan and Dennis M. Ritchie _The C Programming Language (2nd Edition)_ Prentice-Hall 1988 ISBN 0-13-110362-8 + * Bjarne Stroustrup _The C++ Programming Language_ Addison-Wesley 1991 ISBN 0-201-53992-6 + * W. Richard Stevens _Advanced Programming in the Unix Environment_ Addison-Wesley 1992 ISBN 0-201-56317-7 + * W. Richard Stevens _Unix Network Programming_ Prentice-Hall 1990 ISBN 0-13-949876-1 +diff --git a/documentation/content/en/books/handbook/config/_index.adoc b/documentation/content/en/books/handbook/config/_index.adoc +index 7675b13762..3d5bf5ff59 100644 +--- a/documentation/content/en/books/handbook/config/_index.adoc ++++ b/documentation/content/en/books/handbook/config/_index.adoc +@@ -1,1593 +1,1593 @@ + --- + title: Chapter 14. Configuration, Services, Logging and Power Management + part: Part III. System Administration + prev: books/handbook/partiii + next: books/handbook/boot + description: This chapter explains much of the FreeBSD configuration files, how to enable or disable a service, how to configure the logging system and the power management area. + tags: ["configuration", "services", "cron", "periodic", "logging", "configuration files", "sysctl", "swap", "power management"] + showBookMenu: true + weight: 18 + params: + path: "/books/handbook/config/" + --- + + [[config-tuning]] + = Configuration, Services, Logging and Power Management + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :sectnumoffset: 14 + :partnums: + :source-highlighter: rouge + :experimental: + :images-path: books/handbook/config/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + :imagesdir: ../../../../images/{images-path} + endif::[] + ifndef::book[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + toc::[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + toc::[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [[config-synopsis]] + == Synopsis + + One of the important aspects of FreeBSD is proper system configuration. + This chapter explains much of the FreeBSD configuration process, including some of the parameters which can be set to tune a FreeBSD system. + + Before reading this chapter, you should: + + * Understand UNIX(R) and FreeBSD basics (crossref:basics[basics,FreeBSD Basics]). + + After reading this chapter, you will know: + + * How to use the various configuration files in [.filename]#/etc#. + * The basics of [.filename]#rc.conf# configuration and [.filename]#/usr/local/etc/rc.d# startup scripts. + * How to tune FreeBSD using man:sysctl[8] variables. + * How to configure the power management in FreeBSD. + + [[configtuning-configfiles]] + == Configuration Files + + FreeBSD maintains a clear separation between the base system and third party applications and therefore this affects where the configuration files of these applications are located. + + FreeBSD base system configuration is located at the [.filename]#/etc# directory, + and the [.filename]#/usr/local/etc# directory contains all the configuration files of the applications installed on the system through the ports collection and packages. + + The kernel state configuration is located in [.filename]#/etc/sysctl.conf#. + In the section crossref:config[configtuning-sysctl, The sysctl utility], the operation of man:sysctl[8] will be explained in more detail. + + For more information about the FreeBSD file system structure refer to man:hier[7]. + + As a general rule, configuration files do not use a standard on what syntax they must follow. + Although it is true that the `#` character is normally used to comment a line and that each line has a configuration variable. + + [NOTE] + ==== + Some applications like man:pkg[8] are starting to use the link:https://github.com/vstakhov/libucl[Universal Configuration Language (UCL)]. + ==== + + === The [.filename]#/etc# directory + + The [.filename]#/etc# directory contains all of the FreeBSD base system configuration files that are responsible for configuring FreeBSD. + + [CAUTION] + ==== + *Extreme* caution must be taken when modifying files in the [.filename]#/etc# directory; misconfiguration could make FreeBSD unbootable or malfunction. + ==== + + [.informaltable] + [cols="1,1", frame="none"] + |=== + + |[.filename]#/etc# + |System configuration files and scripts. + + |[.filename]#/etc/defaults# + |Default system configuration files, see man:rc[8] for more information. + + |[.filename]#/etc/fstab# + |man:fstab[5] contains descriptive information about the various file systems. + + |[.filename]#/etc/mail# + |Extra man:sendmail[8] configuration and other MTA configuration files. + + |[.filename]#/etc/mtree# + |mtree configuration files, see man: mtree[8] for more information. + + |[.filename]#/etc/pam.d# + |Configuration files for the Pluggable Authentication Modules (PAM) library. + + |[.filename]#/etc/periodic# + |Scripts that are run daily, weekly, and monthly, via man:cron[8], see man:periodic[8] for more information. + + |[.filename]#/etc/rc.d# + |System and daemon startup/control scripts, see man:rc[8] for more information. + + |[.filename]#/etc/rc.conf# + |Contains descriptive information about the local host name, configuration + details for any potential network interfaces and which services should be + started up at system initial boot time. More information in + crossref:bsdinstall[configtuning-core-configuration, Managing System-Specific Configuration] + + |[.filename]#/etc/security# + |OpenBSM audit configuration files, see man:audit[8] for more information. + + |[.filename]#/etc/ppp# + |ppp configuration files, see man:ppp[8] for more information. + + |[.filename]#/etc/ssh# + |OpenSSH configuration files, see man:ssh[1] for more information. + + |[.filename]#/etc/ssl# + |OpenSSL configuration files. + + |[.filename]#/etc/sysctl.conf# + |Contains settings for the kernel. More information in + crossref:bsdinstall[configtuning-sysctl, The sysctl utility] + + |=== + + [[configtuning-sysctl]] + === The sysctl utility + + The man:sysctl[8] utility is used to make changes to a running FreeBSD system. + + The man:sysctl[8] utility retrieves kernel state and allows processes with appropriate privilege to set kernel state. + The state to be retrieved or set is described using a "Management Information Base" ("MIB") style name, described as a dotted set of components. + + .Management Information Base + [.informaltable] + [cols="1,1", frame="none"] + |=== + + |sysctl + |"Magic" numbers + + |kern + |Kernel functions and features + + |vm +-|virtual memory ++|Virtual memory + + |vfs + |Filesystem + + |net + |Network + + |debug + |Debugging parameters + + |hw + |Hardware + + |machdep + |Machine dependent + + |user + |Userland + + |p1003_1b + |POSIX 1003.1B + + |=== + + At its core, man:sysctl[8] serves two functions: to read and to modify system settings. + + To view all readable variables: + + [source,shell] + .... + % sysctl -a + .... + + The output should be similar to the following: + + [.programlisting] + .... + kern.ostype: FreeBSD + ... + vm.swap_enabled: 1 + vm.overcommit: 0 + vm.domain.0.pidctrl.kdd: 8 + vm.domain.0.pidctrl.kid: 4 + vm.domain.0.pidctrl.kpd: 3 + ... + vfs.zfs.sync_pass_rewrite: 2 + vfs.zfs.sync_pass_dont_compress: 8 + vfs.zfs.sync_pass_deferred_free: 2 + .... + + To read a particular variable, specify its name: + + [source,shell] + .... + % sysctl kern.maxproc + .... + + The output should be similar to the following: + + [.programlisting] + .... + kern.maxproc: 1044 + .... + + The Management Information Base (MIB) is hierarchical and hence, specifying a prefix prints all the nodes hanging from it: + + [source,shell] + .... + % sysctl net + .... + + The output should be similar to the following: + + [.programlisting] + .... + net.local.stream.recvspace: 8192 + net.local.stream.sendspace: 8192 + net.local.dgram.recvspace: 16384 + net.local.dgram.maxdgram: 2048 + net.local.seqpacket.recvspace: 8192 + net.local.seqpacket.maxseqpacket: 8192 + net.local.sockcount: 60 + net.local.taskcount: 25 + net.local.recycled: 0 + net.local.deferred: 0 + net.local.inflight: 0 + net.inet.ip.portrange.randomtime: 1 + net.inet.ip.portrange.randomcps: 9999 + [...] + .... + + To set a particular variable, use the _variable_=_value_ syntax: + + [source,shell] + .... + # sysctl kern.maxfiles=5000 + .... + + The output should be similar to the following: + + [.programlisting] + .... + kern.maxfiles: 2088 -> 5000 + .... + + [NOTE] + ==== + To keep the configuration after a reboot it is necessary to add these variables to the [.filename]#/etc/sysctl.conf# file as explained below. + ==== + + [[configtuning-sysctlconf]] + === The [.filename]#/etc/sysctl.conf# file + + The configuration file for man:sysctl[8], [.filename]#/etc/sysctl.conf#, looks much like [.filename]#/etc/rc.conf#. + + Values are set using a `variable=value` syntax. + + [NOTE] + ==== + The specified values are set after the system goes into multi-user mode. + Not all variables are settable in this mode. + ==== + + For example, to turn off logging of fatal signal exits and prevent users from seeing processes started by other users, the following tunables can be set in [.filename]#/etc/sysctl.conf#: + + [.programlisting] + .... + # Do not log fatal signal exits (e.g., sig 11) + kern.logsigexit=0 + + # Prevent users from seeing information about processes that + # are being run under another UID. + security.bsd.see_other_uids=0 + .... + + To obtain more information about what function a particular sysctl has, the following command can be executed: + + [source,shell] + .... + % sysctl -d kern.dfldsiz + .... + + The output should be similar to the following: + + [.programlisting] + .... + kern.dfldsiz: Initial data size limit + .... + + [[configtuning-core-configuration]] + === Managing System-Specific Configuration + + The principal location for system configuration information is [.filename]#/etc/rc.conf#. + + This file contains a wide range of configuration information and it is read at system startup to configure the system. + It provides the configuration information for the [.filename]#rc*# files. + + The entries in [.filename]#/etc/rc.conf# override the default settings in [.filename]#/etc/defaults/rc.conf#. + + [TIP] + ==== + The file [.filename]#/etc/defaults/rc.conf# containing the default settings should not be edited. + Instead, all system-specific changes should be made to [.filename]#/etc/rc.conf#. + ==== + + A number of strategies may be applied in clustered applications to separate site-wide configuration from system-specific configuration in order to reduce administration overhead. + + The recommended approach is to place system-specific configuration into [.filename]#/etc/rc.conf.local#. + + For example, these entries in [.filename]#/etc/rc.conf# apply to all systems: + + [.programlisting] + .... + sshd_enable="YES" + keyrate="fast" + defaultrouter="10.1.1.254" + .... + + Whereas these entries in [.filename]#/etc/rc.conf.local# apply to this system only: + + [.programlisting] + .... + hostname="node1.example.org" + ifconfig_fxp0="inet 10.1.1.1/8" + .... + + Distribute [.filename]#/etc/rc.conf# to every system using an application such as rsync or puppet, while [.filename]#/etc/rc.conf.local# remains unique. + + Upgrading the system will not overwrite [.filename]#/etc/rc.conf#, so system configuration information will not be lost. + + [TIP] + ==== + Both [.filename]#/etc/rc.conf# and [.filename]#/etc/rc.conf.local# are parsed by man:sh[1]. + This allows system operators to create complex configuration scenarios. + Refer to man:rc.conf[5] for further information on this topic. + ==== + + [[configtuning-rcd]] + == Managing Services in FreeBSD + + FreeBSD uses the man:rc[8] system of startup scripts during system initialization and for managing services. + + The scripts listed in [.filename]#/etc/rc.d# provide basic services which can be controlled with the `start`, `stop`, and `restart` options to man:service[8]. + + A basic script may look similar to the following: + + [.programlisting] + .... + #!/bin/sh + # + # PROVIDE: utility + # REQUIRE: DAEMON + # KEYWORD: shutdown + + . /etc/rc.subr + + name=utility + rcvar=utility_enable + + command="/usr/local/sbin/utility" + + load_rc_config $name + + # + # DO NOT CHANGE THESE DEFAULT VALUES HERE + # SET THEM IN THE /etc/rc.conf FILE + # + utility_enable=${utility_enable-"NO"} + pidfile=${utility_pidfile-"/var/run/utility.pid"} + + run_rc_command "$1" + .... + + Refer to extref:{rc-scripting}[this article] for instructions on how to create custom man:rc[8] scripts. + + [[configtuning-starting-services]] + === Starting Services + + Many users install third party software on FreeBSD from the Ports Collection and require the installed services to be started upon system initialization. + + Services, such as package:security/openssh-portable[] or package:www/nginx[] are just two of the many software packages which may be started during system initialization. + This section explains the procedures available for starting services. + + Since the man:rc[8] system is primarily intended to start and stop services at system startup and shutdown time, the `start`, `stop` and `restart` options will only perform their action if the appropriate [.filename]#/etc/rc.conf# variable is set. + + So the first step to start a service, like for example package:www/nginx[] is to add it to [.filename]#/etc/rc.conf# by executing the following command: + + [source,shell] + .... + # sysrc nginx_enable="YES" + .... + + Then nginx can be started executing the following command: + + [source,shell] + .... + # service nginx start + .... + + [TIP] + ==== + To `start`, `stop` or `restart` a service regardless of the settings in [.filename]#/etc/rc.conf#, these commands should be prefixed with "one". + For instance, to start package:www/nginx[] regardless of the current [.filename]#/etc/rc.conf# setting, execute the following command: + + [source,shell] + .... + # service nginx onestart + .... + ==== + + It is also possible to put a service automatically into a jail, see the corresponding crossref:jails[service-jails,Service Jails] explanation. + + [[configtuning-status-services]] + === Status of a Service + + To determine if a service is running, use the `status` subcommand. + + For example, to verify that package:www/nginx[] is running: + + [source,shell] + .... + # service nginx status + .... + + The output should be similar to the following: + + [.programlisting] + .... + nginx is running as pid 27871. + .... + + [[configtuning-reload-services]] + === Reload a Service + + In some cases, it is also possible to `reload` a service. + This attempts to send a signal to an individual service, forcing the service to reload its configuration files. + + In most cases, this means sending the service a `SIGHUP` signal. + + *Not all services support this feature.* + + The man:rc[8] system is used for network services and it also contributes to most of the system initialization. + For instance, when the [.filename]#/etc/rc.d/bgfsck# script is executed, it prints out the following message: + + [source,shell] + .... + Starting background file system checks in 60 seconds. + .... + + This script is used for background file system checks, which occur only during system initialization. + + Many system services depend on other services to function properly. + For example, man:yp[8] and other RPC-based services may fail to start until after the man:rpcbind[8] service has started. + + Additional information can be found in man:rc[8] and man:rc.subr[8]. + + === Using Services to Start Services + + Other services can be started using man:inetd[8]. + Working with man:inetd[8] and its configuration is described in depth in crossref:network-servers[network-inetd,“The inetd Super-Server”]. + + In some cases, it may make more sense to use man:cron[8] to start system services. + This approach has a number of advantages as man:cron[8] runs these processes as the owner of the man:crontab[5]. + This allows regular users to start and maintain their own applications. + + The `@reboot` feature of man:cron[8], may be used in place of the time specification. + This causes the job to run when man:cron[8] is started, normally during system initialization. + + [[cron-periodic]] + == Cron and Periodic + + Scheduling tasks to run at a certain day or time is a very common task on FreeBSD. + The tool in charge of performing this task is man:cron[8]. + + In addition to tasks that can be scheduled by the user via man:cron[8], FreeBSD performs routine background tasks managed by man:periodic[8]. + + [[configtuning-cron]] + === Cron + + The man:cron[8] utility runs in the background and regularly checks [.filename]#/etc/crontab# for tasks to execute and searches [.filename]#/var/cron/tabs# for custom crontab files. + + These files are used to schedule tasks which cron runs at the specified times. + + Each entry in a crontab defines a task to run and is known as a _cron job_. + + Two different types of configuration files are used: the system crontab, which should not be modified, and user crontabs, which can be created and edited as needed. + The format used by these files is documented in man:crontab[5]. + The format of the system crontab, [.filename]#/etc/crontab# includes a `who` column which does not exist in user crontabs. + In the system crontab, cron runs the command as the user specified in this column. + In a user crontab, all commands run as the user who created the crontab. + + User crontabs allow individual users to schedule their own tasks. + The `root` user can also have a user [.filename]#crontab# which can be used to schedule tasks that do not exist in the system [.filename]#crontab#. + + Here is a sample entry from the system crontab, [.filename]#/etc/crontab#: + + [.programlisting] + .... + # /etc/crontab - root's crontab for FreeBSD + # + # <.> + # + SHELL=/bin/sh + PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin <.> + # + #minute hour mday month wday who command <.> + # + # Save some entropy so that /dev/random can re-seed on boot. + */11 * * * * operator /usr/libexec/save-entropy <.> + # + # Rotate log files every hour, if necessary. + 0 * * * * root newsyslog + # + # Perform daily/weekly/monthly maintenance. + 1 3 * * * root periodic daily + 15 4 * * 6 root periodic weekly + 30 5 1 * * root periodic monthly + # + # Adjust the time zone if the CMOS clock keeps local time, as opposed to + # UTC time. See adjkerntz(8) for details. + 1,31 0-5 * * * root adjkerntz -a + + .... + + <.> Lines that begin with the `+#+` character are comments. A comment can be placed in the file as a reminder of what and why a desired action is performed. Comments cannot be on the same line as a command or else they will be interpreted as part of the command; they must be on a new line. Blank lines are ignored. + + <.> The equals (`=`) character is used to define any environment settings. In this example, it is used to define the `SHELL` and `PATH`. If the `SHELL` is omitted, cron will use the default Bourne shell. If the `PATH` is omitted, the full path must be given to the command or script to run. + + <.> This line defines the seven fields used in a system crontab: `minute`, `hour`, `mday`, `month`, `wday`, `who`, and `command`. The `minute` field is the time in minutes when the specified command will be run, the `hour` is the hour when the specified command will be run, the `mday` is the day of the month, `month` is the month, and `wday` is the day of the week. These fields must be numeric values, representing the twenty-four hour clock, or a `*`, representing all values for that field. The `who` field only exists in the system crontab and specifies which user the command should be run as. The last field is the command to be executed. + + <.> This entry defines the values for this cron job. The `\*/11`, followed by several more `*` characters, specifies that `/usr/libexec/save-entropy` is invoked by `operator` every eleven minutes of every hour, of every day and day of the week, of every month. Commands can include any number of switches. However, commands which extend to multiple lines need to be broken with the backslash "\" continuation character. + + [[configtuning-installcrontab]] + === Creating a User Crontab + + To create a user crontab, invoke `crontab` in editor mode: + + [source,shell] + .... + % crontab -e + .... + + This will open the user's crontab using the default text editor. + The first time a user runs this command, it will open an empty file. + Once a user creates a crontab, this command will open that file for editing. + + It is useful to add these lines to the top of the crontab file in order to set the environment variables and to remember the meanings of the fields in the crontab: + + [.programlisting] + .... + SHELL=/bin/sh + PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin + # Order of crontab fields + # minute hour mday month wday command + .... + + Then add a line for each command or script to run, specifying the time to run the command. + This example runs the specified custom Bourne shell script every day at two in the afternoon. + Since the path to the script is not specified in `PATH`, the full path to the script is given: + + [.programlisting] + .... + 0 14 * * * /home/user/bin/mycustomscript.sh + .... + + [TIP] + ==== + Before using a custom script, make sure it is executable and test it with the limited set of environment variables set by cron. + To replicate the environment that would be used to run the above cron entry, use: + + [.programlisting] + .... + env -i SHELL=/bin/sh PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin HOME=/home/user LOGNAME=user /home/user/bin/mycustomscript.sh + .... + + The environment set by cron is discussed in man:crontab[5]. + Checking that scripts operate correctly in a cron environment is especially important if they include any commands that delete files using wildcards. + ==== + + When finished editing the crontab, save the file. + It will automatically be installed, and cron will read the crontab and run its cron jobs at their specified times. + To list the cron jobs in a crontab, use this command: + + [source,shell] + .... + % crontab -l + .... + + The output should be similar to the following: + + [.programlisting] + .... + 0 14 * * * /home/user/bin/mycustomscript.sh + .... + + To remove all of the cron jobs in a user crontab: + + [source,shell] + .... + % crontab -r + .... + + The output should be similar to the following: + + [.programlisting] + .... + remove crontab for user? y + .... + + [[configtuning-periodic]] + === Periodic + + FreeBSD provides a set of system management scripts to check status of various subsystems, perform security-related checks, rotate log files, etc. + These scripts are run on a periodic basis: daily. weekly, or monthly. + The management of these tasks is performed by man:periodic[8] and its configuration resides in man:periodic.conf[5]. + The periodic tasks are initiated by entries in the system crontab, shown above. + + Scripts executed by man:periodic[8] are located in [.filename]#/etc/periodic/# for base utilities and in [.filename]#/usr/local/etc/periodic/# for third-party software. + + They are organized in 4 subdirectories, daily, weekly, monthly and security. + + [[enable-disable-periodic]] + === Enable or Disable Periodic Tasks + + FreeBSD has some scripts enabled by default to run periodically. + + To enable or disable a task, the first step is to edit [.filename]#/etc/periodic.conf# executing the following command: + + [source,shell] + .... + # ee /etc/periodic.conf + .... + + And then to enable, for example, `daily_status_zfs_enable` put the following content in the file: + + [.programlisting] + .... + daily_status_zfs_enable="YES" + .... + + To disable a task that is active by default, all that needs to be done is to change `YES` to `NO`. + + [[configuring-output-periodic-tasks]] + === Configuring the Output of Periodic Tasks + + In [.filename]#/etc/periodic.conf# the variables `daily_output`, `weekly_output` and `monthly_output` specifies where to send the results of the script execution. + + By default the output of the periodic scripts are emailed to root, and therefore it is best to read root's mail or alias root to a mailbox that is monitored. + + To send the results to another email or to other emails, add the email addresses separated by spaces to [.filename]#/etc/periodic.conf#: + + [.programlisting] + .... + daily_output="email1@example.com email2@example.com" + weekly_output="email1@example.com email2@example.com" + monthly_output="email1@example.com email2@example.com" + .... + + To log periodic output instead of receiving it as email, add the following lines to [.filename]#/etc/periodic.conf#. man:newsyslog[8] will rotate these files at the appropriate times: + + [.programlisting] + .... + daily_output=/var/log/daily.log + weekly_output=/var/log/weekly.log + monthly_output=/var/log/monthly.log + .... + + [[configtuning-syslog]] + == Configuring System Logging + + Generating and reading system logs is an important aspect of system administration. + The information in system logs can be used to detect hardware and software issues as well as application and system configuration errors. + This information also plays an important role in security auditing and incident response. + Most system daemons and applications will generate log entries. + + FreeBSD provides a system logger, man:syslogd[8], to manage logging. + By default, syslogd is enabled and started when the system boots. + + This section describes how to configure the FreeBSD system logger for both local and remote logging and how to perform log rotation and log management. + + === Configuring Local Logging + + The configuration file, [.filename]#/etc/syslog.conf#, controls what syslogd does with log entries as they are received. + There are several parameters to control the handling of incoming events. + The _facility_ describes which subsystem generated the message, such as the kernel or a daemon, and the _level_ describes the severity of the event that occurred. + This makes it possible to configure if and where a log message is logged, depending on the facility and level. + It is also possible to take action depending on the application that sent the message, and in the case of remote logging, the hostname of the machine generating the logging event. + + This configuration file contains one line per action, where the syntax for each line is a selector field followed by an action field. + The syntax of the selector field is _facility.level_ which will match log messages from _facility_ at level _level_ or higher. + It is also possible to add an optional comparison flag before the level to specify more precisely what is logged. + Multiple selector fields can be used for the same action, and are separated with a semicolon (`;`). + Using `*` will match everything. + The action field denotes where to send the log message, such as to a file or remote log host. + + As an example, here is the default [.filename]#/etc/syslog.conf# from FreeBSD: + + [.programlisting] + .... + # Spaces ARE valid field separators in this file. However, + # other *nix-like systems still insist on using tabs as field + # separators. If you are sharing this file between systems, you + # may want to use only tabs as field separators here. + # Consult the syslog.conf(5) manpage. + *.err;kern.warning;auth.notice;mail.crit /dev/console <.> + *.notice;authpriv.none;kern.debug;lpr.info;mail.crit;news.err /var/log/messages + security.* /var/log/security + auth.info;authpriv.info /var/log/auth.log + mail.info /var/log/maillog <.> + cron.* /var/log/cron + !-devd + *.=debug /var/log/debug.log <.> + *.emerg * + daemon.info /var/log/daemon.log + # uncomment this to log all writes to /dev/console to /var/log/console.log + # touch /var/log/console.log and chmod it to mode 600 before it will work + #console.info /var/log/console.log + # uncomment this to enable logging of all log messages to /var/log/all.log + # touch /var/log/all.log and chmod it to mode 600 before it will work + #*.* /var/log/all.log + # uncomment this to enable logging to a remote loghost named loghost + #*.* @loghost + # uncomment these if you're running inn + # news.crit /var/log/news/news.crit + # news.err /var/log/news/news.err + # news.notice /var/log/news/news.notice + # Uncomment this if you wish to see messages produced by devd + # !devd + # *.>=notice /var/log/devd.log <.> + !* + include /etc/syslog.d + include /usr/local/etc/syslog.d + .... + + <.> Matches all messages with a level of `err` or higher, as well as `kern.warning`, `auth.notice` and `mail.crit`, and sends these log messages to the console ([.filename]#/dev/console#). + <.> Matches all messages from the `mail` facility at level `info` or above and logs the messages to [.filename]#/var/log/maillog#. + <.> Uses a comparison flag (`=`) to only match messages at level `debug` and logs them to [.filename]#/var/log/debug.log#. + <.> Is an example usage of a program specification. This makes the rules following it only valid for the specified program. In this case, only the messages generated by man:devd[8] are logged to [.filename]#/var/log/devd.log#. + + For more information about [.filename]#/etc/syslog.conf#, its syntax, and more advanced usage examples, see man:syslog.conf[5]. + + [[logging-facilities]] + === Logging Facilities + + A facility describes the part of the system generating the message. + Facilities are a way of separating the different messages so that it is easier for the user to consult the logs. + + .syslog facilities + [options="header", cols="1,1"] + |=== + | Name | Description + + | auth + | The authorization system: man:login[1], man:su[1], man:getty[8], etc. + + | authpriv + | The same as auth, but logged to a file readable only by root. + + | console + | Messages written to [.filename]#/dev/console# by the kernel console output driver. + + | cron + | Messages written by the man:cron[8] daemon. + + | daemon + | System daemons, such as man:routed[8], that are not provided for explicitly by other facilities. + + | ftp + | The file transfer protocol daemons: man:ftpd[8], man:tftpd[8]. + + | kern + | Messages generated by the kernel. These cannot be generated by any user processes. + + | lpr + | The line printer spooling system: man:lpr[1], man:lpc[8], man:lpd[8], etc. + + | mail + | The mail system. + + | mark + | This facility adds a record every 20 minutes. + + | news + | The network news system. + + | ntp + | The network time protocol system. + + | security + | Security subsystems, such as man:ipfw[4]. + + | syslog + | Messages generated internally by syslogd(8). + + | user + | Messages generated by random user processes. *This is the default facility identifier if none is specified*. + + | uucp + | The Unix-to-Unix Copy system. An ancient protocol. Really weird to see messages from this facility. + + | local0 through local7 + | Reserved for local use. + + |=== + + [[logging-levels]] + === Logging Levels + + The level describes the severity of the message, and is a keyword from the following ordered list (higher to lower): + + .syslog levels + [options="header", cols="1,1"] + |=== + | Name | Description + + | emerg + | A panic condition. This is normally broadcast to all users. + + | alert + | A condition that should be corrected immediately, such as a corrupted system database. + + | crit + | Critical conditions, e.g., hard device errors. + + | err + | Errors. + + | warning + | Warning messages. + + | notice + | Conditions that are not error conditions, but should possibly be handled specially. + + | info + | Informational messages. + + | debug + | Messages that contain information normally of use only when debugging a program. + + | none + | This special level disables a particular facility. + + |=== + + [[read-log-messages]] + === Read Log Messages + + By default FreeBSD log files use the format link:https://datatracker.ietf.org/doc/html/rfc3164[rfc3164], also known as The BSD syslog Protocol. + Learn more about other formats and how to use them at man:syslog[8]. + + Typically the logs have the following syntax: + + [.programlisting] + .... + date time hostname program[pid]: the message + .... + + The output of the [.filename]#/var/log/cron# file will be used as an example: + + [.programlisting] + .... + [...] + Jul 16 12:40:00 FreeBSD /usr/sbin/cron[81519]: (root) CMD (/usr/libexec/atrun) + Jul 16 12:44:00 FreeBSD /usr/sbin/cron[83072]: (operator) CMD (/usr/libexec/save-entropy) + [...] + .... + + Verbose logging, so the facility and the level on each message will be added, can be enabled in man:syslog[8] by running the following command: + + [source,shell] + .... + # sysrc syslogd_flags="-vv" + .... + + Once the function is activated, the facility and the level will be displayed in the log as shown in the following example: + + [.programlisting] + .... + [...] + Jul 16 17:40:00 FreeBSD /usr/sbin/cron[1016]: (root) CMD (/usr/libexec/atrun) + Jul 16 17:44:00 FreeBSD /usr/sbin/cron[1030]: (operator) CMD (/usr/libexec/save-entropy) + [...] + .... + + === Log Management and Rotation + + Log files can grow quickly, taking up disk space and making it more difficult to locate useful information. + + In FreeBSD, man:newsyslog[8] is used to manage log files and attempt to mitigate this. + + This built-in program periodically rotates and compresses log files, and optionally creates missing log files and signals programs when log files are moved. + + [NOTE] + ==== + Since newsyslog is run from man:cron[8], it cannot rotate files more often than it is scheduled to run from man:cron[8]. + In the default configuration, it runs every hour. + ==== + + Here is the default configuration in FreeBSD, more information in man:newsyslog.conf[5]: + + [.programlisting] + .... + # configuration file for newsyslog + # + # Entries which do not specify the '/pid_file' field will cause the + # syslogd process to be signalled when that log file is rotated. This + # action is only appropriate for log files which are written to by the + # syslogd process (ie, files listed in /etc/syslog.conf). If there + # is no process which needs to be signalled when a given log file is + # rotated, then the entry for that file should include the 'N' flag. + # + # Note: some sites will want to select more restrictive protections than the + # defaults. In particular, it may be desirable to switch many of the 644 + # entries to 640 or 600. For example, some sites will consider the + # contents of maillog, messages, and lpd-errs to be confidential. In the + # future, these defaults may change to more conservative ones. + # + # logfilename [owner:group] mode count size when flags [/pid_file] [sig_num] + /var/log/all.log 600 7 * @T00 J + /var/log/auth.log 600 7 1000 @0101T JC + /var/log/console.log 600 5 1000 * J + /var/log/cron 600 3 1000 * JC + /var/log/daily.log 640 7 * @T00 JN + /var/log/debug.log 600 7 1000 * JC + /var/log/init.log 644 3 1000 * J + /var/log/kerberos.log 600 7 1000 * J + /var/log/maillog 640 7 * @T00 JC + /var/log/messages 644 5 1000 @0101T JC + /var/log/monthly.log 640 12 * $M1D0 JN + /var/log/devd.log 644 3 1000 * JC + /var/log/security 600 10 1000 * JC + /var/log/utx.log 644 3 * @01T05 B + /var/log/weekly.log 640 5 * $W6D0 JN + /var/log/daemon.log 644 5 1000 @0101T JC + + /etc/newsyslog.conf.d/[!.]*.conf + /usr/local/etc/newsyslog.conf.d/[!.]*.conf + .... + + . `logfilename` - Name of the system log file to be archived. + . `[owner:group]` - This optional field specifies the owner and group for the archive file. + . `mode` - Specify the file mode of the log file and archives. Valid mode bits are 0666. (That is, read and write permissions for the rotated log may be specified for the owner, group, and others.) + . `count` - Specify the maximum number of archive files which may exist. + . `size` - When the size of the log file reaches size in kilobytes, the log file will be trimmed as described above. If this field contains an asterisk ('*'), the log file will not be trimmed based on size. + . `when` - Consist of an interval, a specific time, or both. Supported options in man:newsyslog.conf[5]. + . `flags` - Indicates the flags that newsyslog accepts, supported options in man:newsyslog.conf[5]. + . `[/pid_file]` - This optional field specifies the file name containing a daemon's process ID or to find a group process ID. + . `[sig_num]` - This optional field specifies the signal that will be sent to the daemon process. + + [NOTE] + ==== + The last two fields are optional and specify the name of the Process ID (PID) file of a process and a signal number to send to that process when the file is rotated. + ==== + + [[network-syslogd]] + === Configuring Remote Logging + + Monitoring the log files of multiple hosts can become unwieldy as the number of systems increases. + Configuring centralized logging can reduce some of the administrative burden of log file administration. + + In FreeBSD, centralized log file aggregation, merging, and rotation can be configured using syslogd and newsyslog. + +-This section demonstrates an example configuration, where host `A`, named `logserv.example.com`, will collect logging information for the local network. ++This section demonstrates an example configuration, where host `A`, named `logserv.example.com`, will collect logging information for the local network. + + Host `B`, named `logclient.example.com`, will be configured to pass logging information to the logging server. + + ==== Log Server Configuration + + A log server is a system that has been configured to accept logging information from other hosts. + + Before configuring a log server, check the following: + + * If there is a firewall between the logging server and any logging clients, ensure that the firewall ruleset allows UDP port 514 for both the clients and the server. + * The logging server and all client machines must have forward and reverse entries in the local DNS. If the network does not have a DNS server, create entries in each system's [.filename]#/etc/hosts#. Proper name resolution is required so that log entries are not rejected by the logging server. + + On the log server, edit [.filename]#/etc/syslog.conf# to specify the name of the client to receive log entries from, the logging facility to be used, and the name of the log to store the host's log entries. + This example adds the hostname of `B`, logs all facilities, and stores the log entries in [.filename]#/var/log/logclient.log#. + + .Sample Log Server Configuration + [example] + ==== + + [.programlisting] + .... + +logclient.example.com + *.* /var/log/logclient.log + .... + + ==== + + When adding multiple log clients, add a similar two-line entry for each client. + More information about the available facilities may be found in man:syslog.conf[5]. + + Next, execute the following commands: + + [source,shell] + .... + # sysrc syslogd_enable="YES" + # sysrc syslogd_flags="-a logclient.example.com -v -v" + .... + + The first entry starts syslogd at system boot. + The second entry allows log entries from the specified client. + The `-v -v` increases the verbosity of logged messages. + This is useful for tweaking facilities as administrators are able to see what type of messages are being logged under each facility. + + Multiple `-a` options may be specified to allow logging from multiple clients. + IP addresses and whole netblocks may also be specified. + Refer to man:syslogd[8] for a full list of possible options. + + Finally, create the log file: + + [source,shell] + .... + # touch /var/log/logclient.log + .... + + At this point, syslogd should be restarted and verified: + + [source,shell] + .... + # service syslogd restart + # pgrep syslog + .... + + If a PID is returned, the server restarted successfully, and client configuration can begin. + If the server did not restart, consult [.filename]#/var/log/messages# for the error. + + ==== Log Client Configuration + + A logging client sends log entries to a logging server on the network. + The client also keeps a local copy of its own logs. + + Once a logging server has been configured, execute the following commands on the logging client: + + [source,shell] + .... + # sysrc syslogd_enable="YES" + # sysrc syslogd_flags="-s -v -v" + .... + + The first entry enables syslogd on boot up. + The second entry prevents logs from being accepted by this client from other hosts (`-s`) and increases the verbosity of logged messages. + + Next, define the logging server in the client's [.filename]#/etc/syslog.conf#. + In this example, all logged facilities are sent to a remote system, denoted by the `@` symbol, with the specified hostname: + + [.programlisting] + .... + *.* @logserv.example.com + .... + + After saving the edit, restart syslogd for the changes to take effect: + + [source,shell] + .... + # service syslogd restart + .... + + To test that log messages are being sent across the network, use man:logger[1] on the client to send a message to syslogd: + + [source,shell] + .... + # logger "Test message from logclient" + .... + + This message should now exist both in [.filename]#/var/log/messages# on the client and [.filename]#/var/log/logclient.log# on the log server. + + ==== Debugging Log Servers + + If no messages are being received on the log server, the cause is most likely a network connectivity issue, a hostname resolution issue, or a typo in a configuration file. + To isolate the cause, ensure that both the logging server and the logging client are able to `ping` each other using the hostname specified in their [.filename]#/etc/rc.conf#. + If this fails, check the network cabling, the firewall ruleset, and the hostname entries in the DNS server or [.filename]#/etc/hosts# on both the logging server and clients. + Repeat until the `ping` is successful from both hosts. + + If the `ping` succeeds on both hosts but log messages are still not being received, temporarily increase logging verbosity to narrow down the configuration issue. + In the following example, [.filename]#/var/log/logclient.log# on the logging server is empty and [.filename]#/var/log/messages# on the logging client does not indicate a reason for the failure. + + To increase debugging output, edit the `syslogd_flags` entry on the logging server and issue a restart: + + [source,shell] + .... + sysrc syslogd_flags="-d -a logclient.example.com -v -v" + .... + + [source,shell] + .... + # service syslogd restart + .... + + Debugging data similar to the following will flash on the console immediately after the restart: + + [.programlisting] + .... + logmsg: pri 56, flags 4, from logserv.example.com, msg syslogd: restart + syslogd: restarted + logmsg: pri 6, flags 4, from logserv.example.com, msg syslogd: kernel boot file is /boot/kernel/kernel + Logging to FILE /var/log/messages + syslogd: kernel boot file is /boot/kernel/kernel + cvthname(192.168.1.10) + validate: dgram from IP 192.168.1.10, port 514, name logclient.example.com; + rejected in rule 0 due to name mismatch. + .... + + In this example, the log messages are being rejected due to a typo which results in a hostname mismatch. + The client's hostname should be `logclient`, not `logclien`. + Fix the typo, issue a restart, and verify the results: + + [source,shell] + .... + # service syslogd restart + .... + + The output should be similar to the following: + + [.programlisting] + .... + logmsg: pri 56, flags 4, from logserv.example.com, msg syslogd: restart + syslogd: restarted + logmsg: pri 6, flags 4, from logserv.example.com, msg syslogd: kernel boot file is /boot/kernel/kernel + syslogd: kernel boot file is /boot/kernel/kernel + logmsg: pri 166, flags 17, from logserv.example.com, + msg Dec 10 20:55:02 logserv.example.com syslogd: exiting on signal 2 + cvthname(192.168.1.10) + validate: dgram from IP 192.168.1.10, port 514, name logclient.example.com; + accepted in rule 0. + logmsg: pri 15, flags 0, from logclient.example.com, msg Dec 11 02:01:28 trhodes: Test message 2 + Logging to FILE /var/log/logclient.log + Logging to FILE /var/log/messages + .... + + At this point, the messages are being properly received and placed in the correct file. + + ==== Security Considerations + + As with any network service, security requirements should be considered before implementing a logging server. + Log files may contain sensitive data about services enabled on the local host, user accounts, and configuration data. + Network data sent from the client to the server will not be encrypted or password protected. + If a need for encryption exists, consider using package:security/stunnel[], which will transmit the logging data over an encrypted tunnel. + + Local security is also an issue. + Log files are not encrypted during use or after log rotation. + Local users may access log files to gain additional insight into system configuration. + Setting proper permissions on log files is critical. + The built-in log rotator, newsyslog, supports setting permissions on newly created and rotated log files. + Setting log files to mode `600` should prevent unwanted access by local users. + Refer to man:newsyslog.conf[5] for additional information. + + [[acpi-overview]] + == Power and Resource Management + + It is important to utilize hardware resources in an efficient manner. + Power and resource management allows the operating system to monitor system limits and to possibly run some actions triggered by events related to those limits. + + [[acpi-config]] + === ACPI configuration + + On FreeBSD the management of these resources is managed by the man:acpi[4] kernel device. + + [NOTE] + ==== + In FreeBSD the man:acpi[4] driver is loaded by default at system boot. + + This driver *cannot be unloaded after boot* because the system bus uses it for various hardware interactions. + ==== + + In addition to man:acpi[4], FreeBSD has several dedicated kernel modules for various ACPI vendor subsystems. + These modules will add some extra functionality like fan speed, keyboard backlit or screen brightness. + + The list can be obtained by running the following command: + + [source,shell] + .... + % ls /boot/kernel | grep acpi + .... + + The output should be similar to the following: + + [.programlisting] + .... + acpi_asus.ko + acpi_asus_wmi.ko + acpi_dock.ko + acpi_fujitsu.ko + acpi_hp.ko + acpi_ibm.ko + acpi_panasonic.ko + acpi_sony.ko + acpi_toshiba.ko + acpi_video.ko + acpi_wmi.ko + sdhci_acpi.ko + uacpi.ko + .... + + In the event that, for example, an IBM/Lenovo laptop is used, it will be necessary to load the module man:acpi_ibm[4] by executing the following command: + + [source,shell] + .... + # kldload acpi_ibm + .... + + And add this line to [.filename]#/boot/loader.conf# to load it at boot: + + [.programlisting] + .... + acpi_ibm_load="YES" + .... + + An alternative to the man:acpi_video[4] module is the man:backlight[9] driver. + It provides a generic way for handling a panel backlight. + The default GENERIC kernel includes this driver. + The man:backlight[8] utility can be used to query and adjust the brightness of + the panel backlight. + In this example the brightness is decreased by 10%: + + [source,shell] + .... + % backlight decr 10 + .... + + [[cpu-power-management]] + === CPU Power Management + + CPU is the most consuming part of the system. + Knowing how to improve CPU efficiency is a fundamental part of our system in order to save energy. + + In order to make proper use of the machine's resources in a correct way, FreeBSD supports technologies such as Intel Turbo Boost, AMD Turbo Core, Intel Speed Shift among others through the use of man:powerd[8] and man:cpufreq[4]. + + The first step will be to obtain the CPU information by executing the following command: + + [source,shell] + .... + % sysctl dev.cpu.0 <.> + .... + + <.> In this case the `0` digit represents the first core of the CPU. + + The output should be similar to the following: + + [.programlisting] + .... + dev.cpu.0.cx_method: C1/mwait/hwc C2/mwait/hwc C3/mwait/hwc/bma + dev.cpu.0.cx_usage_counters: 3507294 0 0 + dev.cpu.0.cx_usage: 100.00% 0.00% 0.00% last 3804us + dev.cpu.0.cx_lowest: C3 <1> + dev.cpu.0.cx_supported: C1/1/1 C2/2/1 C3/3/57 <2> + dev.cpu.0.freq_levels: 2267/35000 2266/35000 1600/15000 800/12000 <3> + dev.cpu.0.freq: 1600 <4> + dev.cpu.0.temperature: 40.0C <5> + dev.cpu.0.coretemp.throttle_log: 0 + dev.cpu.0.coretemp.tjmax: 105.0C + dev.cpu.0.coretemp.resolution: 1 + dev.cpu.0.coretemp.delta: 65 + dev.cpu.0.%parent: acpi0 + dev.cpu.0.%pnpinfo: _HID=none _UID=0 _CID=none + dev.cpu.0.%location: handle=\_PR_.CPU0 + dev.cpu.0.%driver: cpu + dev.cpu.0.%desc: ACPI CPU + .... + + <1> Lowest Cx state to use for idling the CPU. + <2> CPU supported Cx states. + <3> Currently available levels for the CPU (frequency/power usage). + <4> Current active CPU frequency in MHz. + <5> Current temperature of the CPU. + + [NOTE] + ==== + If the temperature information is not displayed, load the man:coretemp[4] module. + In case of using an AMD CPU, load the man:amdtemp[4] module. + ==== + + Once the CPU information is available the easiest way to configure power saving is to let man:powerd[8] take over. + + Enable man:powerd[8] service in [.filename]#/etc/rc.conf# to start at system boot: + + [source,shell] + .... + # sysrc powerd_enable=YES + .... + + It will also be necessary to indicate certain parameters to man:powerd[8] to tell it how to manage the state of the CPU executing the following command: + + [source,shell] + .... + # sysrc powerd_flags="-a hiadaptive -i 25 -r 85 -N" + .... + + . `-a`: Selects the mode to use while on AC power. + . `hiadaptive`: Operation mode. More info at man:powerd[8]. + . `-i`: Specifies the CPU load percent level when adaptive mode should begin to degrade performance to save power. + . `-r`: Specifies the CPU load percent level where adaptive mode should consider the CPU running and increase performance. + . `-N`: Treat "nice" time as idle for the purpose of load calculation; i.e., do not increase the CPU frequency if the CPU is only busy with "nice" processes. + + And then enable the service executing the following command: + + [source,shell] + .... + # service powerd start + .... + + [[cpufreq]] + === CPU Frequency Control + + FreeBSD includes a generic man:cpufreq[4] driver to allow the administrator, or software such as man:powerd[8] and package:sysutils/powerdxx[], to manage the frequency of the CPU to achieve the desired balance between performance and economy. + A lower setting will save power while reducing the heat generated by the CPU. + A higher setting will increase performance at the cost of using additional power and generating more heat. + + [[est]] + === Intel(R) Enhanced Speed Step(TM) + + The Intel(R) Enhanced Speed Step(TM) driver, man:est[4], replaces the generic man:cpufreq[4] driver for CPUs that provide this feature. + The CPU frequency can be statically adjusted using man:sysctl[8], or with the `/etc/rc.d/power_profile` startup script. + Additional software, such as man:powerd[8] or package:sysutils/powerdxx[], can be used to automatically adjust the CPU frequency based on processor utilization. + + Each supported frequency, along with its expected power consumption, can be listed by examining the man:sysctl[3] tree: + + [source,shell] + .... + # sysctl dev.cpufreq.0.freq_driver dev.cpu.0.freq_levels dev.cpu.0.freq + .... + + The output should be similar to the following: + + [.programlisting] + .... + dev.cpufreq.0.freq_driver: est0 + dev.cpu.0.freq_levels: 3001/53000 3000/53000 2900/50301 2700/46082 2600/43525 2400/39557 2300/37137 2100/33398 2000/31112 1800/27610 1700/25455 1500/22171 1400/20144 1200/17084 1100/15181 900/12329 800/10550 + dev.cpu.0.freq: 800 + .... + + A frequency 1 MHz higher than the maximum frequency of the CPU indicates the Intel(R) Turbo Boost(TM) feature. + + [[hwpstate_intel]] + === Intel Speed Shift(TM) + + Users running newer Intel(R) CPUs may find some differences in dynamic frequency control when upgrading to FreeBSD 13. + A new driver for the Intel(R) Speed Shift(TM) feature set, available on certain SKUs, exposes the ability for the hardware to dynamically vary the core frequencies, including on a per core basis. + FreeBSD 13 comes with the man:hwpstate_intel[4] driver to automatically enable Speed Shift(TM) control on equipped CPUs, replacing the older Enhanced Speed Step(TM) man:est[4] driver. + The man:sysctl[8] `dev.cpufreq.%d.freq_driver` will indicate if the system is using Speed Shift. + + To determine which frequency control driver is being used, examining the `dev.cpufreq.0.freq_driver` oid. + + [source,shell] + .... + # sysctl dev.cpufreq.0.freq_driver + .... + + The output should be similar to the following: + + [.programlisting] + .... + dev.cpufreq.0.freq_driver: hwpstate_intel0 + .... + + This indicates that the new man:hwpstate_intel[4] driver is in use. + On such systems, the oid `dev.cpu.%d.freq_levels` will show only the maximum CPU frequency, and will indicate a power consumption level of `-1`. + + The current CPU frequency can be determined by examining the `dev.cpu.%d.freq` oid. + + [source,shell] + .... + # sysctl dev.cpu.0.freq_levels dev.cpu.0.freq + .... + + The output should be similar to the following: + + [.programlisting] + .... + dev.cpu.0.freq_levels: 3696/-1 + dev.cpu.0.freq: 898 + .... + + For more information, including on how to balance performance and energy use, and on how to disable this driver, refer to the man page man:hwpstate_intel[4]. + + [NOTE] + ==== + Users accustomed to using man:powerd[8] or package:sysutils/powerdxx[] will find these utilities have been superseded by the man:hwpstate_intel[4] driver and no longer work as expected. + ==== + + [[graphics-card-power-management]] + === Graphics Card Power Management + + Graphics cards have become a fundamental part of computing in recent years. + Some graphics cards may have excessive power consumption. + FreeBSD allows certain configurations to improve power consumption. + + In case of using a Intel(R) graphics card with the package:graphics/drm-kmod[] driver these options can be added to [.filename]#/boot/loader.conf#: + + [.programlisting] + .... + compat.linuxkpi.fastboot=1 <.> + compat.linuxkpi.enable_dc=2 <.> + compat.linuxkpi.enable_fbc=1 <.> + .... + + <.> Try to skip unnecessary mode sets at boot time. + <.> Enable power-saving display C-states. + <.> Enable frame buffer compression for power savings + + === Suspend/Resume + + The suspend/resume function allows the machine to be kept in a state in which there is no a big energy consumption and allows the system to be resumed without having to lose the state of the running programs. + + [NOTE] + ==== + In order for the suspend/resume functionality to work correctly the graphics drivers must be loaded on the system. + In non-KMS-supported graphics cards man:sc[4] must be used not to break the suspend/resume functionality. + + More information about which driver to use and how to configure it can be found at the crossref:x11[x11, The X Window System chapter]. + ==== + + man:acpi[4] supports the next list of sleep states: + + .Supported Sleep States + [options="header", cols="1,1"] + |=== + + |S1 + |Quick suspend to RAM. The CPU enters a lower power state, but most peripherals are left running. + + |S2 + |Lower power state than S1, but with the same basic characteristics. Not supported by many systems. + + |S3 (Sleep mode) + |Suspend to RAM. Most devices are powered off, and the system stops running except for memory refresh. + + |S4 (Hibernation) + |Suspend to disk. All devices are powered off, and the system stops running. When resuming, the system starts as if from a cold power on. *Not yet supported by FreeBSD*. + + |S5 + |System shuts down cleanly and powers off. + + |=== + + [[configure-suspend-resume]] + ==== Configuring Suspend/Resume + + The first step will be to know which type of sleep states supports the hardware we are using executing the following command: + + [source,shell] + .... + % sysctl hw.acpi.supported_sleep_state + .... + + The output should be similar to the following: + + [.programlisting] + .... + hw.acpi.supported_sleep_state: S3 S4 S5 + .... + + [WARNING] + ==== + As stated above FreeBSD does *not* yet support the `S4` state. + ==== + + man:acpiconf[8] can be used to check if the `S3` state works correctly by running the following command, if it succeeds, the screen should go black and the machine will turn off: + + [source,shell] + .... + # acpiconf -s 3 + .... + + In the vast majority of cases the Suspend/Resume functionality wants to be used on a laptop. + + FreeBSD can be configured to enter the `S3` state when closing the lid by adding the following line to the [.filename]#/etc/sysctl.conf# file. + + [.programlisting] + .... + hw.acpi.lid_switch_state=S3 + .... + + [[troubleshooting-suspend-resume]] + ==== Troubleshooting in Suspend/Resume + + A lot of effort has been made to make the Suspend and Resume functions work properly and in the best way on FreeBSD. + But currently the Suspend and Resume functions only work properly on some specific laptops. + + Some checks can be done in case it doesn't work properly. + + In some cases it is enough to turn off the bluetooth. + In others it is enough loading the correct driver for the graphics card, etc. + + In case it doesn't work correctly, some tips can be found on the FreeBSD Wiki in the section link:https://wiki.freebsd.org/SuspendResume[Suspend/Resume]. + + [[adding-swap-space]] + == Adding Swap Space + + Sometimes a FreeBSD system requires more swap space. + This section describes two methods to increase swap space: adding swap to an existing partition or new hard drive, and creating a swap file on an existing file system. + + For information on how to encrypt swap space, which options exist, and why it should be done, refer to crossref:disks[swap-encrypting,“Encrypting Swap”]. + + [[new-drive-swap]] + === Swap on a New Hard Drive or Existing Partition + + Adding a new drive for swap gives better performance than using a partition on an existing drive. + Setting up partitions and drives is explained in crossref:disks[disks-adding,"Adding Disks"] while crossref:bsdinstall[configtuning-initial,"Designing the Partition Layout"] discusses partition layouts and swap partition size considerations. + + [WARNING] + ==== + It is possible to use any partition not currently mounted, even if it already contains data. + Using `swapon` on a partition that contains data will overwrite and destroy that data. + Make sure that the partition to be added as swap is really the intended partition before running `swapon`. + ==== + + man:swapon[8] can be used to add a swap partition to the system executing the following command: + + [source,shell] + .... + # swapon /dev/ada1p2 + .... + + To automatically add this swap partition on boot, add an entry to [.filename]#/etc/fstab#: + + [.programlisting] + .... + /dev/ada1p2 none swap sw 0 0 + .... + + See man:fstab[5] for an explanation of the entries in [.filename]#/etc/fstab#. + + [[create-swapfile]] + === Creating a Swap File + + [[swapfile-10-and-later]] + These examples create a 512M swap file called [.filename]#/usr/swap0#. + + [WARNING] + ==== + Swap files on ZFS file systems are strongly discouraged, as swapping can lead to system hangs. + ==== + + The first step is to create the swap file: + + [source,shell] + .... + # dd if=/dev/zero of=/usr/swap0 bs=1m count=512 + .... + + The second step is to put the proper permissions on the new file: + + [source,shell] + .... + # chmod 0600 /usr/swap0 + .... + + The third step is to inform the system about the swap file by adding a line to [.filename]#/etc/fstab#: + + [.programlisting] + .... + md none swap sw,file=/usr/swap0,late 0 0 + .... + + Swap space will be added on system startup. To add swap space immediately, use man:swapon[8]: + + [source,shell] + .... + # swapon -aL + .... +diff --git a/documentation/content/en/books/handbook/cutting-edge/_index.adoc b/documentation/content/en/books/handbook/cutting-edge/_index.adoc +index 59e38e2bc9..a27b6500d5 100644 +--- a/documentation/content/en/books/handbook/cutting-edge/_index.adoc ++++ b/documentation/content/en/books/handbook/cutting-edge/_index.adoc +@@ -1,1453 +1,1453 @@ + --- + title: Chapter 26. Updating and Upgrading FreeBSD + part: Part III. System Administration + prev: books/handbook/l10n + next: books/handbook/dtrace + description: Information about how to keep a FreeBSD system up-to-date with freebsd-update or Git, how to rebuild and reinstall the entire base system, etc + tags: ["updating", "upgrading", "documentation", "FreeBSD-STABLE", "FreeBSD-CURRENT", "Security Patches"] + showBookMenu: true + weight: 30 + params: + path: "/books/handbook/cutting-edge/" + --- + + [[updating-upgrading]] + = Updating and Upgrading FreeBSD + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :sectnumoffset: 26 + :partnums: + :source-highlighter: rouge + :experimental: + :images-path: books/handbook/cutting-edge/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + :imagesdir: ../../../../images/{images-path} + endif::[] + ifndef::book[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + toc::[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + toc::[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [[updating-upgrading-synopsis]] + == Synopsis + + FreeBSD is under constant development between releases. + Some people prefer to use the officially released versions, while others prefer to keep in sync with the latest developments. + However, even official releases are often updated with security and other critical fixes. + Regardless of the version used, FreeBSD provides all the necessary tools to keep the system updated, and allows for easy upgrades between versions. + This chapter describes how to track the development system and the basic tools for keeping a FreeBSD system up-to-date. + + After reading this chapter, you will know: + + * How to keep a FreeBSD system up-to-date with freebsd-update or Git. + * How to compare the state of an installed system against a known pristine copy. + * How to keep the installed documentation up-to-date with Git or documentation ports. + * The difference between the two development branches: FreeBSD-STABLE and FreeBSD-CURRENT. + * How to rebuild and reinstall the entire base system. + + Before reading this chapter, you should: + + * Properly set up the network connection (crossref:advanced-networking[advanced-networking,Advanced Networking]). + * Know how to install additional third-party software (crossref:ports[ports,Installing Applications: Packages and Ports]). + + [NOTE] + ==== + Throughout this chapter, `git` is used to obtain and update FreeBSD sources. + Optionally, the package:devel/git[] port or package may be used. + ==== + + [[updating-upgrading-freebsdupdate]] + == FreeBSD Update + + Applying security patches in a timely manner and upgrading to a newer release of an operating system are important aspects of ongoing system administration. + FreeBSD includes a utility called `freebsd-update` which can be used to perform both these tasks. + + This utility supports binary security and errata updates to FreeBSD, without the need to manually compile and install the patch or a new kernel. + Binary updates are available for all architectures and releases currently supported by the security team. + The list of supported releases and their estimated end-of-life dates are listed at https://www.FreeBSD.org/security/[https://www.FreeBSD.org/security/]. + + This utility also supports operating system upgrades to minor point releases as well as upgrades to another release branch. + Before upgrading to a new release, review its release announcement as it contains important information pertinent to the release. + Release announcements are available from https://www.FreeBSD.org/releases/[https://www.FreeBSD.org/releases/]. + + [NOTE] + ==== + If a man:crontab[5] utilizing the features of man:freebsd-update[8] exists, it must be disabled before upgrading the operating system. + ==== + + This section describes the configuration file used by `freebsd-update`, demonstrates how to apply a security patch and how to upgrade to a minor or major operating system release, and discusses some of the considerations when upgrading the operating system. + + [[freebsdupdate-config-file]] + === The Configuration File + + The default configuration file for `freebsd-update` works as-is. + Some users may wish to tweak the default configuration in [.filename]#/etc/freebsd-update.conf#, allowing better control of the process. + The comments in this file explain the available options, but the following may require a bit more explanation: + + [.programlisting] + .... + # Components of the base system which should be kept updated. + Components world kernel + .... + + This parameter controls which parts of FreeBSD will be kept up-to-date. + The default is to update the entire base system and the kernel. + Individual components can instead be specified, such as `src/base` or `src/sys`. + However, the best option is to leave this at the default as changing it to include specific items requires every needed item to be listed. + Over time, this could have disastrous consequences as source code and binaries may become out of sync. + + [.programlisting] + .... + # Paths which start with anything matching an entry in an IgnorePaths + # statement will be ignored. + IgnorePaths /boot/kernel/linker.hints + .... + + To leave specified directories, such as [.filename]#/bin# or [.filename]#/sbin#, untouched during the update process, add their paths to this statement. + This option may be used to prevent `freebsd-update` from overwriting local modifications. + + [.programlisting] + .... + # Paths which start with anything matching an entry in an UpdateIfUnmodified + # statement will only be updated if the contents of the file have not been + # modified by the user (unless changes are merged; see below). + UpdateIfUnmodified /etc/ /var/ /root/ /.cshrc /.profile + .... + + This option will only update unmodified configuration files in the specified directories. + Any changes made by the user will prevent the automatic updating of these files. + There is another option, `KeepModifiedMetadata`, which will instruct `freebsd-update` to save the changes during the merge. + + [.programlisting] + .... + # When upgrading to a new FreeBSD release, files which match MergeChanges + # will have any local changes merged into the version from the new release. + MergeChanges /etc/ /var/named/etc/ /boot/device.hints + .... + + List of directories with configuration files that `freebsd-update` should attempt to merge. + The file merge process is a series of man:diff[1] patches. + Merges are either accepted, open an editor, or cause `freebsd-update` to abort. + When in doubt, backup [.filename]#/etc# and just accept the merges. + + [.programlisting] + .... + # Directory in which to store downloaded updates and temporary + # files used by FreeBSD Update. + # WorkDir /var/db/freebsd-update + .... + + This directory is where all patches and temporary files are placed. + In cases where the user is doing a version upgrade, this location should have at least a gigabyte of disk space available. + + [.programlisting] + .... + # When upgrading between releases, should the list of Components be + # read strictly (StrictComponents yes) or merely as a list of components + # which *might* be installed of which FreeBSD Update should figure out + # which actually are installed and upgrade those (StrictComponents no)? + # StrictComponents no + .... + + When this option is set to `yes`, `freebsd-update` will assume that the `Components` list is complete and will not attempt to make changes outside of the list. + Effectively, `freebsd-update` will attempt to update every file which belongs to the `Components` list. + + Refer to man:freebsd-update.conf[5] for more details. + + [[freebsdupdate-security-patches]] + === Applying Security Patches + + The process of applying FreeBSD security patches has been simplified, allowing an administrator to keep a system fully patched using `freebsd-update`. + More information about FreeBSD security advisories can be found in crossref:security[security-advisories,"FreeBSD Security Advisories"]. + + FreeBSD security patches may be downloaded and installed using the following commands. + The first command will determine if any outstanding patches are available, and if so, will list the files that will be modified if the patches are applied. + The second command will apply the patches. + + [source,shell] + .... + # freebsd-update fetch + # freebsd-update install + .... + + If the update applies any kernel patches, the system will need a reboot in order to boot into the patched kernel. + If the patch was applied to any running binaries, the affected applications should be restarted so that the patched version of the binary is used. + + [NOTE] + ==== + Usually, the user needs to be prepared to reboot the system. + To know if the system requires a reboot due to a kernel update, execute the commands `freebsd-version -k` and `uname -r`. + Reboot the system if the outputs differ. + ==== + + The system can be configured to automatically check for updates once every day by adding this entry to [.filename]#/etc/crontab#: + + [.programlisting] + .... + @daily root freebsd-update cron + .... + + If patches exist, they will automatically be downloaded but will not be applied. + The `root` user will be sent an email so that the patches may be reviewed and manually installed with `freebsd-update install`. + + If anything goes wrong, `freebsd-update` has the ability to roll back the last set of changes with the following command: + + [source,shell] + .... + # freebsd-update rollback + Uninstalling updates... done. + .... + + Again, the system should be restarted if the kernel or any kernel modules were modified and any affected binaries should be restarted. + + Only the [.filename]#GENERIC# kernel can be automatically updated by `freebsd-update`. + If a custom kernel is installed, it will have to be rebuilt and reinstalled after `freebsd-update` finishes installing the updates. + The default kernel name is _GENERIC_. + The man:uname[1] command may be used to verify its installation. + + [NOTE] + ==== + Always keep a copy of the [.filename]#GENERIC# kernel in [.filename]#/boot/GENERIC#. + It will be helpful in diagnosing a variety of problems and in performing version upgrades. + Refer to crossref:cutting-edge[freebsd-update-custom-kernel-9x, Custom Kernels with FreeBSD 9.X and Later] for instructions on how to get a copy of the [.filename]#GENERIC# kernel. + ==== + + Unless the default configuration in [.filename]#/etc/freebsd-update.conf# has been changed, + `freebsd-update` will install the updated kernel sources along with the rest of the updates. + Rebuilding and reinstalling a new custom kernel can then be performed in the usual way. + + The updates distributed by `freebsd-update` do not always involve the kernel. + It is not necessary to rebuild a custom kernel if the kernel sources have not been modified by `freebsd-update install`. + However, `freebsd-update` will always update [.filename]#/usr/src/sys/conf/newvers.sh#. + The current patch level, as indicated by the `-p` number reported by `uname -r`, is obtained from this file. + Rebuilding a custom kernel, even if nothing else changed, allows `uname` to accurately report the current patch level of the system. + This is particularly helpful when maintaining multiple systems, as it allows for a quick assessment of the updates installed in each one. + + [[freebsdupdate-upgrade]] + === Performing Minor and Major Version Upgrades + + Upgrades from one minor version of FreeBSD to another are called _minor version_ upgrades. An example: + + - FreeBSD 13.1 to 13.2. + + _Major version_ upgrades increase the major version number. An example: + + - FreeBSD 13.2 to 14.0. + + Both types of upgrade can be performed by providing `freebsd-update` with a release version target. + + [WARNING] + ==== + After each new `RELEASE`, FreeBSD package build servers will, for a limited period, *not* use the newer version of the operating system. + This provides continuity for the many users who do not upgrade immediately after a release announcement. + For example: + + * packages for users of 13.1 and 13.2 will be built on a server running 13.1, until 13.1 reaches end of life + + -- and, critically: + + * a kernel module that is built on 13.1 might *not* be suitable for 13.2. + + So, with any minor or major OS upgrade, if your package requirements include any kernel module: + + * *be prepared to build the module from source*. + + ==== + + [NOTE] + ==== + If the system is running a custom kernel, make sure that a copy of the [.filename]#GENERIC# kernel exists in [.filename]#/boot/GENERIC# before starting the upgrade. + Refer to crossref:cutting-edge[freebsd-update-custom-kernel-9x, Custom Kernels with FreeBSD 9.X and Later] for instructions on how to get a copy of the [.filename]#GENERIC# kernel. + ==== + + Before upgrading to a new version, ensure the existing FreeBSD installation is up to date with respect to security and errata patches: + + [source,shell] + .... + # freebsd-update fetch + # freebsd-update install + .... + + The following command, when run on a FreeBSD 13.1 system, will upgrade it to FreeBSD 13.2: + + [source,shell] + .... + # freebsd-update -r 13.2-RELEASE upgrade + .... + + After the command has been received, `freebsd-update` will evaluate the configuration file and current system in an attempt to gather the information necessary to perform the upgrade. + A screen listing will display which components have and have not been detected. + For example: + + [source,shell] + .... + Looking up update.FreeBSD.org mirrors... 1 mirrors found. + Fetching metadata signature for 13.1-RELEASE from update1.FreeBSD.org... done. + Fetching metadata index... done. + Inspecting system... done. + + The following components of FreeBSD seem to be installed: + kernel/smp src/base src/bin src/contrib src/crypto src/etc src/games + src/gnu src/include src/krb5 src/lib src/libexec src/release src/rescue + src/sbin src/secure src/share src/sys src/tools src/ubin src/usbin + world/base world/info world/lib32 world/manpages + + The following components of FreeBSD do not seem to be installed: + kernel/generic world/catpages world/dict world/doc world/games + world/proflibs + + Does this look reasonable (y/n)? y + .... + + At this point, `freebsd-update` will attempt to download all files required for the upgrade. + In some cases, the user may be prompted with questions regarding what to install or how to proceed. + + When using a custom kernel, the above step will produce a warning similar to the following: + + [source,shell] + .... + WARNING: This system is running a "MYKERNEL" kernel, which is not a + kernel configuration distributed as part of FreeBSD 13.1-RELEASE. + This kernel will not be updated: you MUST update the kernel manually + before running "/usr/sbin/freebsd-update install" + .... + + This warning may be safely ignored at this point. + The updated [.filename]#GENERIC# kernel will be used as an intermediate step in the upgrade process. + + Once all the patches have been downloaded to the local system, they will be applied. + This process may take a while, depending on the speed and workload of the machine. + Configuration files will then be merged. + The merging process requires some user intervention as a file may be merged or an editor may appear on screen for a manual merge. + The results of every successful merge will be shown to the user as the process continues. + A failed or ignored merge will cause the process to abort. + Users may wish to make a backup of [.filename]#/etc# and manually merge important files, + such as [.filename]#master.passwd# or [.filename]#group# at a later time. + + [NOTE] + ==== + The system is not being altered yet as all patching and merging is happening in another directory. + Once all patches have been applied successfully, + all configuration files have been merged and it seems the process will go smoothly, + the changes can be committed to disk by the user using the following command: + + [source,shell] + .... + # freebsd-update install + .... + + ==== + + The kernel and kernel modules will be patched first. + If the system is running with a custom kernel, + use man:nextboot[8] to set the kernel for the next boot to the updated [.filename]#/boot/GENERIC#: + + [source,shell] + .... + # nextboot -k GENERIC + .... + + [WARNING] + ==== + Before rebooting with the [.filename]#GENERIC# kernel, + make sure it contains all the drivers required for the system to boot properly and connect to the network, + if the machine being updated is accessed remotely. + In particular, if the running custom kernel contains built-in functionality usually provided by kernel modules, make sure to temporarily load these modules into the [.filename]#GENERIC# kernel using the [.filename]#/boot/loader.conf# facility. + It is recommended to disable non-essential services as well as any disk and network mounts until the upgrade process is complete. + ==== + + The machine should now be restarted with the updated kernel: + + [source,shell] + .... + # shutdown -r now + .... + + Once the system has come back online, restart `freebsd-update` using the following command. + Since the state of the process has been saved, `freebsd-update` will not start from the beginning, + but will instead move on to the next phase and remove all old shared libraries and object files. + + [source,shell] + .... + # freebsd-update install + .... + + [NOTE] + ==== + Depending upon whether any library version numbers were bumped, there may only be two install phases instead of three. + ==== + + The upgrade is now complete. + If this was a major version upgrade, reinstall all ports and packages as + described in crossref:cutting-edge[freebsdupdate-portsrebuild, Upgrading Packages After a Major Version Upgrade]. + + [[freebsd-update-custom-kernel-9x]] + ==== Custom Kernels with FreeBSD 9.X and Later + + Before using `freebsd-update`, ensure that a copy of the [.filename]#GENERIC# kernel exists in [.filename]#/boot/GENERIC#. + If a custom kernel has only been built once, the kernel in [.filename]#/boot/kernel.old# is the `GENERIC` kernel. + Simply rename this directory to [.filename]#/boot/GENERIC#. + + If a custom kernel has been built more than once or if it is unknown how many times the custom kernel has been built, + obtain a copy of the `GENERIC` kernel that matches the current version of the operating system. + If physical access to the system is available, a copy of the `GENERIC` kernel can be installed from the installation media: + + [source,shell] + .... + # mount /media + # cd /media/usr/freebsd-dist + # tar -C/ -xvf kernel.txz boot/kernel/kernel + .... + + Alternately, the `GENERIC` kernel may be rebuilt and installed from source: + + [source,shell] + .... + # cd /usr/src + # make kernel __MAKE_CONF=/dev/null SRCCONF=/dev/null + .... + + For this kernel to be identified as the `GENERIC` kernel by `freebsd-update`, + the [.filename]#GENERIC# configuration file must not have been modified in any way. + It is also suggested that the kernel is built without any other special options. + + Rebooting into the [.filename]#GENERIC# kernel is not required as `freebsd-update` only needs [.filename]#/boot/GENERIC# to exist. + + [[freebsdupdate-portsrebuild]] + ==== Upgrading Packages After a Major Version Upgrade + + Generally, installed applications will continue to work without problems after minor version upgrades. + Major versions use different Application Binary Interfaces (ABIs), which will break most third-party applications. + After a major version upgrade, all installed packages and ports need to be upgraded. + Packages can be upgraded using `pkg upgrade`. + To upgrade installed ports, use a utility such as package:ports-mgmt/portmaster[]. + + A forced upgrade of all installed packages will replace the packages with fresh versions from the repository even if the version number has not increased. + This is required because of the ABI version change when upgrading between major versions of FreeBSD. + The forced upgrade can be accomplished by performing: + + [source,shell] + .... + # pkg-static upgrade -f + .... + + A rebuild of all applications installed from the ports collection can be accomplished with this command: + + [source,shell] + .... + # portmaster -af + .... + + This command will display the configuration screens for each application that has configurable options and wait for the user to interact with those screens. + To prevent this behavior, and use only the default options, include `-G` in the above command. + + Once the software upgrades are complete, + finish the upgrade process with a final call to `freebsd-update` in order to tie up all the loose ends in the upgrade process: + + [source,shell] + .... + # freebsd-update install + .... + + If the [.filename]#GENERIC# kernel was temporarily used, + this is the time to build and install a new custom kernel using the instructions in crossref:kernelconfig[kernelconfig,Configuring the FreeBSD Kernel]. + + Reboot the machine into the new FreeBSD version. + The upgrade process is now complete. + + [[freebsdupdate-system-comparison]] + === System State Comparison + + The state of the installed FreeBSD version against a known good copy can be tested using `freebsd-update IDS`. + This command evaluates the current version of system utilities, libraries, and configuration files and can be used as a built-in Intrusion Detection System (IDS). + + [WARNING] + ==== + This command is not a replacement for a real IDS such as package:security/snort[]. + As `freebsd-update` stores data on disk, the possibility of tampering is evident. + While this possibility may be reduced using `kern.securelevel` and by storing the `freebsd-update` data on a read-only file system when not in use, + a better solution would be to compare the system against a secure disk, such as a DVD or securely stored external USB disk device. + An alternative method for providing IDS functionality using a built-in utility is described in crossref:security[security-ids,"Binary Verification"] + ==== + + To begin the comparison, specify the output file to save the results to: + + [source,shell] + .... + # freebsd-update IDS >> outfile.ids + .... + + The system will now be inspected and a lengthy listing of files, along with the SHA256 hash values for both the known value in the release and the current installation, will be sent to the specified output file. + + The entries in the listing are extremely long, but the output format may be easily parsed. + For instance, to obtain a list of all files which differ from those in the release, issue the following command: + + [source,shell] + .... + # cat outfile.ids | awk '{ print $1 }' | more + /etc/master.passwd + /etc/motd + /etc/passwd + /etc/pf.conf + .... + + This sample output has been truncated as many more files exist. + Some files have natural modifications. + For example, [.filename]#/etc/passwd# will be modified if users have been added to the system. + Kernel modules may differ as `freebsd-update` may have updated them. + To exclude specific files or directories, add them to the `IDSIgnorePaths` option in [.filename]#/etc/freebsd-update.conf#. + + [[updating-bootcode]] + == Updating Bootcode + + The following manuals describe the upgrade process of bootcode and boot loaders: man:gpart[8], man:gptboot[8], man:gptzfsboot[8], and man:loader.efi[8]. + + [[updating-upgrading-documentation]] + == Updating the Documentation Set + + Documentation is an integral part of the FreeBSD operating system. + While an up-to-date version of the FreeBSD documentation is always available on the FreeBSD web site (link:https://docs.FreeBSD.org[Documentation Portal]), it can be handy to have an up-to-date, local copy of the FreeBSD website, handbooks, FAQ, and articles. + + This section describes how to use either source or the FreeBSD Ports Collection to keep a local copy of the FreeBSD documentation up-to-date. + + For information on editing and submitting corrections to the documentation, + refer to the FreeBSD Documentation Project Primer for New Contributors (extref:{fdp-primer}[FreeBSD Documentation Project Primer for New Contributors]). + + [[updating-installed-documentation]] + === Updating Documentation from Source + + Rebuilding the FreeBSD documentation from source requires a collection of tools which are not part of the FreeBSD base system. +-The required tools can be installed following extref:{fdp-primer}[these steps, overview-quick-start] from the FreeBSD Documentation Project Primer. ++The required tools can be installed following extref:{fdp-primer}overview[these steps, overview-quick-start] from the FreeBSD Documentation Project Primer. + + Once installed, use `git` to fetch a clean copy of the documentation source: + + [source,shell] + .... + # git clone https://git.FreeBSD.org/doc.git /usr/doc + .... + + The initial download of the documentation sources may take a while. + Let it run until it completes. + + Future updates of the documentation sources may be fetched by running: + + [source,shell] + .... + # git pull + .... + + Once an up-to-date snapshot of the documentation sources has been fetched to [.filename]#/usr/doc#, + everything is ready for an update of the installed documentation. + + A full update may be performed by typing: + + [source,shell] + .... + # cd /usr/doc + # make + .... + + [[current-stable]] + == Tracking a Development Branch + + FreeBSD has two development branches: FreeBSD-CURRENT and FreeBSD-STABLE. + + This section provides an explanation of each branch and its intended audience, as well as how to keep a system up-to-date with each respective branch. + + [[current]] + === Using FreeBSD-CURRENT + + FreeBSD-CURRENT is the "bleeding edge" of FreeBSD development and FreeBSD-CURRENT users are expected to have a high degree of technical skill. + Less technical users who wish to track a development branch should track FreeBSD-STABLE instead. + + FreeBSD-CURRENT is the very latest source code for FreeBSD and includes works in progress, experimental changes, and transitional mechanisms that might or might not be present in the next official release. + While many FreeBSD developers compile the FreeBSD-CURRENT source code daily, there are short periods of time when the source may not be buildable. + These problems are resolved as quickly as possible, but whether or not FreeBSD-CURRENT brings disaster or new functionality can be a matter of when the source code was synced. + + FreeBSD-CURRENT is made available for three primary interest groups: + + . Members of the FreeBSD community who are actively working on some part of the source tree. + . Members of the FreeBSD community who are active testers. They are willing to spend time solving problems, making topical suggestions on changes and the general direction of FreeBSD, and submitting patches. + . Users who wish to keep an eye on things, use the current source for reference purposes, or make the occasional comment or code contribution. + + FreeBSD-CURRENT should _not_ be considered a fast-track to getting new features before the next release as pre-release features are not yet fully tested and most likely contain bugs. + It is not a quick way of getting bug fixes as any given commit is just as likely to introduce new bugs as to fix existing ones. + FreeBSD-CURRENT is not in any way "officially supported". + + To track FreeBSD-CURRENT: + + . Join the {freebsd-current} and the {dev-commits-src-main} lists. This is _essential_ in order to see the comments that people are making about the current state of the system and to receive important bulletins about the current state of FreeBSD-CURRENT. + + + The {dev-commits-src-main} list records the commit log entry for each change as it is made, along with any pertinent information on possible side effects. + + + To join these lists, go to {mailing-lists}, click on the list to subscribe to, and follow the instructions. + In order to track changes to the whole source tree, not just the changes to FreeBSD-CURRENT, subscribe to the {dev-commits-src-all}. + . Synchronize with the FreeBSD-CURRENT sources. Typically, `git` is used to check out the -CURRENT code from the `main` branch of the FreeBSD Git repository (see crossref:mirrors[git,“Using Git”] for details). + . Due to the size of the repository, some users choose to only synchronize the sections of source that interest them or which they are contributing patches to. However, users that plan to compile the operating system from source must download _all_ of FreeBSD-CURRENT, not just selected portions. + + + Before compiling FreeBSD-CURRENT, read [.filename]#/usr/src/Makefile# very + carefully and follow the instructions in crossref:cutting-edge[makeworld, Updating FreeBSD from Source]. + Read the {freebsd-current} and [.filename]#/usr/src/UPDATING# to stay up-to-date on other bootstrapping procedures that sometimes become necessary on the road to the next release. + . Be active! FreeBSD-CURRENT users are encouraged to submit their suggestions for enhancements or bug fixes. Suggestions with accompanying code are always welcome. + + [[stable]] + === Using FreeBSD-STABLE + + FreeBSD-STABLE is the development branch from which major releases are made. + Changes go into this branch at a slower pace and with the general assumption that they have first been tested in FreeBSD-CURRENT. + This is _still_ a development branch and, at any given time, the sources for FreeBSD-STABLE may or may not be suitable for general use. + It is simply another engineering development track, not a resource for end-users. + Users who do not have the resources to perform testing should instead run the most recent release of FreeBSD. + + Those interested in tracking or contributing to the FreeBSD development process, especially as it relates to the next release of FreeBSD, should consider following FreeBSD-STABLE. + + While the FreeBSD-STABLE branch should compile and run at all times, this cannot be guaranteed. + Since more people run FreeBSD-STABLE than FreeBSD-CURRENT, it is inevitable that bugs and corner cases will sometimes be found in FreeBSD-STABLE that were not apparent in FreeBSD-CURRENT. + For this reason, one should not blindly track FreeBSD-STABLE. + It is particularly important _not_ to update any production servers to FreeBSD-STABLE without thoroughly testing the code in a development or testing environment. + + To track FreeBSD-STABLE: + + . Join the {freebsd-stable} in order to stay informed of build dependencies that may appear in FreeBSD-STABLE or any other issues requiring special attention. Developers will also make announcements in this mailing list when they are contemplating some controversial fix or update, giving the users a chance to respond if they have any issues to raise concerning the proposed change. + + + Join the relevant git list for the branch being tracked. + For example, users tracking the {betarel-current-major}-STABLE branch should join the {dev-commits-src-branches}. + This list records the commit log entry for each change as it is made, along with any pertinent information on possible side effects. + + + To join these lists, go to {mailing-lists}, click on the list to subscribe to, and follow the instructions. + In order to track changes for the whole source tree, subscribe to {dev-commits-src-all}. + . To install a new FreeBSD-STABLE system, install the most recent FreeBSD-STABLE release from the crossref:mirrors[mirrors,FreeBSD mirror sites] or use a monthly snapshot built from FreeBSD-STABLE. Refer to link:https://www.FreeBSD.org/snapshots/[www.freebsd.org/snapshots] for more information about snapshots. + + + To compile or upgrade an existing FreeBSD system to FreeBSD-STABLE, use `git` to check out the source for the desired branch. + Branch names, such as `stable/13`, are listed at link:https://www.FreeBSD.org/releng/[www.freebsd.org/releng]. + . Before compiling or upgrading to FreeBSD-STABLE , read + [.filename]#/usr/src/Makefile# carefully and follow the instructions in + crossref:cutting-edge[makeworld, Updating FreeBSD from Source]. Read the {freebsd-stable} and [.filename]#/usr/src/UPDATING# to keep up-to-date on other bootstrapping procedures that sometimes become necessary on the road to the next release. + + [[translate-n-number]] + === The N-number + When tracking down bugs it is important to know which versions of the source code have been used to create the system exhibiting an issue. + FreeBSD provides version information compiled into the kernel. + man:uname[1] retrieves this information, for example: + [source,shell] + .... + % uname -v + FreeBSD 14.0-CURRENT #112 main-n247514-031260d64c18: Tue Jun 22 20:43:19 MDT 2021 fred@machine:/usr/home/fred/obj/usr/home/fred/git/head/amd64.amd64/sys/FRED + .... + The final field gives information regarding the kernel name, the person that built it, and the location that it was compiled in. + Looking at the 4th field, it is made up of several parts: + [source,shell] + .... + main-n247514-031260d64c18 + + main <.> + n247514 <.> + 031260d64c18 <.> + <.> + .... + <.> Git branch name. + Note: comparisons of n-numbers are only valid on branches published by the project (`main`, `stable/XX` and `releng/XX`). + Local branches will have n-numbers that will overlap commits of their parent branch. + <.> The n-number is a linear count of commits back to the start of the Git repository starting with the Git hash included in the line. + <.> Git hash of the checked out tree + <.> Sometimes a suffix of `-dirty` is present when the kernel was built in a tree with uncommitted changes. + It is absent in this example because the FRED kernel was built from a pristine checkout. + + The `git rev-list` command is used to find the n-number corresponding to a Git hash. + For example: + [source,shell] + .... + % git rev-list --first-parent --count 031260d64c18 <.> + 247514 <.> + .... + <.> git hash to translate (the hash from the above example is reused) + <.> The n-number. + + Usually this number is not all that important. + However, when bug fixes are committed, this number makes it easy to quickly determine whether the fix is present in the currently running system. + Developers will often refer to the hash of the commit (or provide a URL which has that hash), but not the n-number since the hash is the easily visible identifier for a change while the n-number is not. + Security advisories and errata notices will also note an n-number, which can be directly compared against your system. + When you need to use shallow Git clones, you cannot compare n-numbers reliably as the `git rev-list` command counts all the revisions in the repository which a shallow clone omits. + + [[makeworld]] + == Updating FreeBSD from Source + + Updating FreeBSD by compiling from source offers several advantages over binary updates. + Code can be built with options to take advantage of specific hardware. + Parts of the base system can be built with non-default settings, or left out entirely where they are not needed or desired. + The build process takes longer to update a system than just installing binary updates, but allows complete customization to produce a tailored version of FreeBSD. + + [[updating-src-quick-start]] + === Quick Start + + This is a quick reference for the typical steps used to update FreeBSD by building from source. + Later sections describe the process in more detail. + + [WARNING] + ==== + When switching from man:mergemaster[8] to man:etcupdate[8], the first run might merge changes incorrectly generating spurious conflicts. + To prevent this, perform the following steps *before* updating sources and building the new world: + + [source,shell] + .... + # etcupdate extract <.> + # etcupdate diff <.> + .... + + <.> Bootstrap the database of stock [.filename]#/etc# files; for more information see man:etcupdate[8]. + + <.> Check the diff after bootstrapping. Trim any local changes that are no longer needed to reduce the chance of conflicts in future updates. + ==== + + [.procedure] + ==== + * Update and Build + + + [source,shell] + .... + # git -C /usr/src pull <.> + check /usr/src/UPDATING <.> + # cd /usr/src <.> + # make -j4 buildworld <.> + # make -j4 kernel <.> + # shutdown -r now <.> + # etcupdate -p <.> + # cd /usr/src <.> + # make installworld <.> + # etcupdate -B <.> + # shutdown -r now <.> + .... + + <.> Get the latest version of the source. See + crossref:cutting-edge[updating-src-obtaining-src, Updating the Source] for more information on obtaining and updating source. + + <.> Check [.filename]#/usr/src/UPDATING# for any manual steps required before or after building from source. + + <.> Go to the source directory. + + <.> Compile the world, everything except the kernel. + + <.> Compile and install the kernel. This is equivalent to `make buildkernel installkernel`. + + <.> Reboot the system to the new kernel. + + <.> Update and merge configuration files in [.filename]#/etc/# required before installworld. + + <.> Go to the source directory. + + <.> Install the world. + + <.> Update and merge configuration files in [.filename]#/etc/#. + + <.> Restart the system to use the newly-built world and kernel. + ==== + + [[updating-src-preparing]] + === Preparing for a Source Update + + Read [.filename]#/usr/src/UPDATING#. Any manual steps that must be performed before or after an update are described in this file. + + [[updating-src-obtaining-src]] + === Updating the Source + + FreeBSD source code is located in [.filename]#/usr/src/#. + The preferred method of updating this source is through the Git version control system. + Verify that the source code is under version control: + + [source,shell] + .... + # cd /usr/src + # git remote --v + origin https://git.freebsd.org/src.git (fetch) + origin https://git.freebsd.org/src.git (push) + .... + + This indicates that [.filename]#/usr/src/# is under version control and can be updated with man:git[1]: + + [[synching]] + [source,shell] + .... + # git -C /usr/src pull + .... + + The update process can take some time if the directory has not been updated recently. + After it finishes, the source code is up to date and the build process described in the next section can begin. + + [NOTE] + ==== + Obtaining the source: + + If the output says `fatal: not a git repository`, the files there are missing or were installed with a different method. + A new checkout of the source is required. + ==== + + [[updating-src-obtaining-src-repopath]] + .FreeBSD Versions and Repository Branches + [cols="10%,10%,80%", options="header"] + |=== + | uname ‑r Output + | Repository Path + | Description + + |`_X.Y_-RELEASE` + |`releng/_X.Y_` + |The Release version plus only critical security and bug fix patches. This branch is recommended for most users. + + |`_X.Y_-STABLE` + |`stable/_X_` + | + + The Release version plus all additional development on that branch. _STABLE_ refers to the Applications Binary Interface (ABI) not changing, so software compiled for earlier versions still runs. For example, software compiled to run on FreeBSD 10.1 will still run on FreeBSD 10-STABLE compiled later. + + STABLE branches occasionally have bugs or incompatibilities which might affect users, although these are typically fixed quickly. + + |`_X_-CURRENT` + |`main` + |The latest unreleased development version of FreeBSD. The CURRENT branch can have major bugs or incompatibilities and is recommended only for advanced users. + |=== + + Determine which version of FreeBSD is being used with man:uname[1]: + + [source,shell] + .... + # uname -r + 13.2-RELEASE + .... + + Based on crossref:cutting-edge[updating-src-obtaining-src-repopath,FreeBSD Versions and Repository Branches], the source used to update `13.2-RELEASE` has a repository path of `releng/13.2`. + That path is used when checking out the source: + + [source,shell] + .... + # mv /usr/src /usr/src.bak <.> + # git clone --branch releng/13.2 https://git.FreeBSD.org/src.git /usr/src <.> + .... + + <.> Move the old directory out of the way. If there are no local modifications in this directory, it can be deleted. + + <.> The path from crossref:cutting-edge[updating-src-obtaining-src-repopath,FreeBSD Versions and Repository Branches] is added to the repository URL. The third parameter is the destination directory for the source code on the local system. + + [[updating-src-building]] + === Building from Source + + The _world_, or all of the operating system except the kernel, is compiled. + This is done first to provide up-to-date tools to build the kernel. + Then the kernel itself is built: + + [source,shell] + .... + # cd /usr/src + # make buildworld + # make buildkernel + .... + + The compiled code is written to [.filename]#/usr/obj#. + + These are the basic steps. + Additional options to control the build are described below. + + [[updating-src-building-clean-build]] + ==== Performing a Clean Build + + Some versions of the FreeBSD build system leave previously-compiled code in the temporary object directory, [.filename]#/usr/obj#. + This can speed up later builds by avoiding recompiling code that has not changed. + To force a clean rebuild of everything, use `cleanworld` before starting a build: + + [source,shell] + .... + # make cleanworld + .... + + [[updating-src-building-jobs]] + ==== Setting the Number of Jobs + + Increasing the number of build jobs on multi-core processors can improve build speed. + Determine the number of cores with `sysctl hw.ncpu`. + Processors vary, as do the build systems used with different versions of FreeBSD, so testing is the only sure method to tell how a different number of jobs affects the build speed. + For a starting point, consider values between half and double the number of cores. + The number of jobs is specified with `-j`. + + [[updating-src-building-jobs-example]] + .Increasing the Number of Build Jobs + [example] + ==== + Building the world and kernel with four jobs: + + [source,shell] + .... + # make -j4 buildworld buildkernel + .... + + ==== + + [[updating-src-building-only-kernel]] + ==== Building Only the Kernel + + A `buildworld` must be completed if the source code has changed. + After that, a `buildkernel` to build a kernel can be run at any time. + To build just the kernel: + + [source,shell] + .... + # cd /usr/src + # make buildkernel + .... + + [[updating-src-building-custom-kernel]] + ==== Building a Custom Kernel + + The standard FreeBSD kernel is based on a _kernel config file_ called [.filename]#GENERIC#. + The [.filename]#GENERIC# kernel includes the most commonly-needed device drivers and options. + Sometimes it is useful or necessary to build a custom kernel, adding or removing device drivers or options to fit a specific need. + + For example, someone developing a small embedded computer with severely limited RAM could remove unneeded device drivers or options to make the kernel slightly smaller. + + Kernel config files are located in [.filename]#/usr/src/sys/arch/conf/#, where _arch_ is the output from `uname -m`. + On most computers, that is `amd64`, giving a config file directory of [.filename]#/usr/src/sys/amd64/conf/#. + + [TIP] + ==== + [.filename]#/usr/src# can be deleted or recreated, so it is preferable to keep custom kernel config files in a separate directory, like [.filename]#/root#. + Link the kernel config file into the [.filename]#conf# directory. + If that directory is deleted or overwritten, the kernel config can be re-linked into the new one. + ==== + + A custom config file can be created by copying the [.filename]#GENERIC# config file. + In this example, the new custom kernel is for a storage server, so is named [.filename]#STORAGESERVER#: + + [source,shell] + .... + # cp /usr/src/sys/amd64/conf/GENERIC /root/STORAGESERVER + # cd /usr/src/sys/amd64/conf + # ln -s /root/STORAGESERVER . + .... + + [.filename]#/root/STORAGESERVER# is then edited, adding or removing devices or options as shown in man:config[5]. + + The custom kernel is built by setting `KERNCONF` to the kernel config file on the command line: + + [source,shell] + .... + # make buildkernel KERNCONF=STORAGESERVER + .... + + [[updating-src-installing]] + === Installing the Compiled Code + + After the `buildworld` and `buildkernel` steps have been completed, the new kernel and world are installed: + + [source,shell] + .... + # cd /usr/src + # make installkernel + # shutdown -r now + # cd /usr/src + # make installworld + # shutdown -r now + .... + + If a custom kernel was built, `KERNCONF` must also be set to use the new custom kernel: + + [source,shell] + .... + # cd /usr/src + # make installkernel KERNCONF=STORAGESERVER + # shutdown -r now + # cd /usr/src + # make installworld + # shutdown -r now + .... + + [[updating-src-completing]] + === Completing the Update + + A few final tasks complete the update. + Any modified configuration files are merged with the new versions, outdated libraries are located and removed, then the system is restarted. + + [[updating-src-completing-merge-etcupdate]] + ==== Merging Configuration Files with man:etcupdate[8] + + man:etcupdate[8] is a tool for managing updates to files that are not updated as part of an installworld such as files located in [.filename]#/etc/#. + It manages updates by doing a three-way merge of changes made to these files against the local versions. + man:etcupdate[8] is designed to minimize the amount of user intervention. + + [NOTE] + ==== + In general, man:etcupdate[8] does not need any specific arguments for its job. + There is however a handy in between command for sanity checking what will be done the first time man:etcupdate[8] is used: + + [source,shell] + .... + # etcupdate diff + .... + + This command allows the user to audit configuration changes. + ==== + + If man:etcupdate[8] is not able to merge a file automatically, the merge conflicts can be resolved with manual interaction by issuing: + + [source,shell] + .... + # etcupdate resolve + .... + + [WARNING] + ==== + When switching from man:mergemaster[8] to man:etcupdate[8], the first run might merge changes incorrectly generating spurious conflicts. + To prevent this, perform the following steps *before* updating sources and building the new world: + + [source,shell] + .... + # etcupdate extract <.> + # etcupdate diff <.> + .... + + <.> Bootstrap the database of stock [.filename]#/etc# files; for more information see man:etcupdate[8]. + + <.> Check the diff after bootstrapping. Trim any local changes that are no longer needed to reduce the chance of conflicts in future updates. + ==== + + [[updating-src-completing-check-old]] + ==== Checking for Outdated Files and Libraries + + Some obsolete files or directories can remain after an update. + These files can be located: + + [source,shell] + .... + # make check-old + .... + + and deleted: + + [source,shell] + .... + # make delete-old + .... + + Some obsolete libraries can also remain. + These can be detected with: + + [source,shell] + .... + # make check-old-libs + .... + + and deleted with + + [source,shell] + .... + # make delete-old-libs + .... + + Programs which were still using those old libraries will stop working when the library has been deleted. + These programs must be rebuilt or replaced after deleting the old libraries. + + [TIP] + ==== + When all the old files or directories are known to be safe to delete, + pressing kbd:[y] and kbd:[Enter] to delete each file can be avoided by setting `BATCH_DELETE_OLD_FILES` in the command. + For example: + + [source,shell] + .... + # make BATCH_DELETE_OLD_FILES=yes delete-old-libs + .... + + ==== + + [[updating-src-completing-restart]] + ==== Restarting After the Update + + The last step after updating is to restart the computer so all the changes take effect: + + [source,shell] + .... + # shutdown -r now + .... + + [[pkgbase]] + == Updating FreeBSD with Base System Packages + + Starting from 14.0-RELEASE, the FreeBSD project has published a set of packages of the kernel and base system, using man:pkg[8]. + These can be used in the same convenient way that users are used to, for adding and upgrading ported software, but for the kernel and userland itself. + The packages, and usage thereof, are often referred to as pkgbase. + + Packages have been available since link:https://lists.freebsd.org/archives/freebsd-pkgbase/2023-October/000221.html[October 2023], considered experimental for FreeBSD's 14 Release. + + Starting from 15.0-RELEASE, Base System packages will be the default and officially supported way to both install new FreeBSD instances, and also to update and upgrade between minor and major releases. + + [NOTE] + ==== + From 15.0-RELEASE onwards, the long-running man:freebsd-update[8] tool will only be supported on the earlier 13 and 14 release branches. + ==== + + Base System Packages replace: + + * tarball distribution sets, such as `base.txz` or `kernel.txz`, which are historically used for installation of the OS with man:bsdinstall[8] + * man:freebsd-update[8] for updates to the OS. + + Base System packages complement crossref:cutting-edge[makeworld,"building and installing from source"], which is still available for those who wish to build their own custom kernels or userland. + It is also possible to build custom base system packages from local sources, as well as just relying on officially provided packages. + + === Converting a Host to use pkgbase + + Systems installed with FreeBSD 14.0-RELEASE or later can be converted to use Base System packages. + For earlier versions, it is recommended to first upgrade to a recent Version, and then convert. + + The FreeBSD Foundation has sponsored development of a tool called link:https://github.com/FreeBSDFoundation/pkgbasify[pkgbasify], which for most users, will be the easiest and safest way to convert systems to use Base System packages. + + [WARNING] + ==== + Note that this migration requires up to 5GiB additional free space, to download, unpack, and relocate any conflicting files. + The pkgbasify tool does not check for this and it is the responsibility of the user to ensure that enough space is available before running the migration. + ==== + + man:pkgbasify[8] (or whatever outcome link:https://reviews.freebsd.org/D51594[D51594] or link:https://wiki.freebsd.org/WantedPorts#O-P[port request] will have) performs 6 main tasks: + + * Creates a backup boot environment (ZFS only) with man:bectl[8] + * Creates the new package repository config files + * Upgrades existing system components such as base, kernel, lib32, debug + * Merges existing and new config files + * Updates passwd and capabilities databases + * Restarts sshd immediately + + [source,shell] + .... + # cd /tmp + # fetch https://github.com/FreeBSDFoundation/pkgbasify/raw/refs/heads/main/pkgbasify.lua + # chmod +x pkgbasify.lua + # ./pkgbasify.lua + .... + + === Upgrading a Host using pkgbase + + [WARNING] + ==== + This is still in development, so please be careful especially when converting an existing system to use pkgbase. + ==== + + Create a folder for custom pkg repository config files, if there is none present already. + + [source,shell] + .... + # mkdir -p /usr/local/etc/pkg/repos/ + .... + + For using the pkgbase repository, create a pkg repository configuration file called `FreeBSD-base.conf`: + + [[pgk-base-repo-configuration]] + [.programlisting] + .... + FreeBSD-base { + url = "pkg+https://pkg.freebsd.org/${ABI}/base_release_${VERSION_MINOR}"; + mirror_type = "srv"; + signature_type = "fingerprints"; + fingerprints = "/usr/share/keys/pkg"; + enabled = yes; + } + .... + + for more information on specific configuration options see man:pkg.conf[5]. + + There are different branches to choose from (by changing the url accordingly): + + [[table-of-packagebase-branches]] + .Base system package Branches + [cols="10%,20%,70%, options="header"] + |=== + | Branch + | Frequency + | URL + + | main + | twice daily - 12:00 and 00:00 UTC + | `https://pkg.freebsd.org/${ABI}/base_latest` + + | main + | weekly – Sunday at 12:00 UTC + | `https://pkg.freebsd.org/${ABI}/base_weekly` + + | stable/14 + | twice daily – 12:00 and 00:00 UTC + | `https://pkg.freebsd.org/${ABI}/base_latest` + + | stable/14 + | weekly – Sunday at 12:00 UTC + | `https://pkg.freebsd.org/${ABI}/base_weekly` + + | releng/14.2 + | twice daily – 12:00 and 00:00 UTC + | `https://pkg.freebsd.org/${ABI}/base_release_2` + + | releng/14.3 + | twice daily – 12:00 and 00:00 UTC + | `https://pkg.freebsd.org/${ABI}/base_release_3` + |=== + + To upgrade the system, change the configuration file according to the desired release, and run: + + [source,shell] + .... + # pkg update -r FreeBSD-base + # pkg upgrade -r FreeBSD-base + .... + + check, if these packages are correct and accept the changes. + + Reboot the OS executing the following command: + + [source,shell] + .... + # shutdown -r now + .... + + ==== Performing Major version upgrades + + When running ZFS, consider creating a boot environment before upgrading to a newer version. + To create a new boot environment using the man:bectl[8] tool run: + + [source,shell] + .... + # bectl create 14.2-RELEASE-p4 + .... + + Use this boot environment to start the system as it was before the update if something goes wrong. + + Save a list of your non-base packages in case you need that later: + + [source,shell] + .... + pkg prime-origins | sort -u > /var/tmp/pkg-prime-origins.txt + .... + + Change `/usr/local/etc/pkg/repos/FreeBSD-base.conf` to target the correct major release like `base_latest`, so it looks like: + + [.programlisting] + .... + FreeBSD-base { + url = "pkg+https://pkg.freebsd.org/${ABI}/base_latest"; + mirror_type = "srv"; + signature_type = "fingerprints"; + fingerprints = "/usr/share/keys/pkg"; + enabled = yes; + } + .... + + The next step will upgrade your system to the specified version. + + [WARNING] + ==== + This step might remove non-base packages, which could include your running desktop environment. + Be careful. + ==== + + Set the environment variable ABI to upgrade the major version (replace amd64 with your architecture and 15 with your targeted version). + + [source,shell] + .... + # env ABI=FreeBSD:15:amd64 pkg-static upgrade -r FreeBSD-base + .... + + There will be a prompt to ask if you want to ignore the version mismatch looking like this: + + [source,shell] + .... + Newer FreeBSD version for package FreeBSD-zoneinfo: + To ignore this error set IGNORE_OSVERSION=yes + - package: 1500058 + - running userland: 1500000 + Ignore the mismatch and continue? [y/N]: + .... + + Check and confirm that. + + To check if that was successful, run `freebsd-version -kru`. + + Then reboot. + + After upgrading to a new major version, updates and upgrades of installed packages to match the ABI version may be necessary. + + [source,shell] + .... + # pkg update + # pkg upgrade + .... + + If something broke, go back and activate the backup boot environment created before. + + [source,shell] + .... + # bectl activate 14.2-RELEASE-p4 + .... + Reboot, and the system will be back to the state before upgrading. + + If you haven't created a boot environment, you might want to consider getting help from link:https://www.freebsd.org/support/[FreeBSD Support]. + + [[build-pkgbase-packages-locally]] + === Manually building pkgbase and publishing it to the local network + + If you want to start building your own pkgbase packages clone the FreeBSD source tree: + + [source,shell] + .... + # cd /usr/src + # git clone https://github.com/freebsd/freebsd-src.git /usr/src + .... + + Check out the branch for the release to build packages for: + + [source,shell] + .... + # git checkout releng/14.3 + .... + + Start the building process, depending on the resources available this could take some while. + Set the parallel processes according to the CPU core count. + + This example is written for an 8 core CPU: + + [source,shell] + .... + # make -j8 buildworld && make -j8 buildkernel && make -j8 packages + .... + + When building frequently, consider using package:devel/ccache[] to speed up subsequent builds from the cache. + + After building, the packages will get saved into `/usr/obj/usr/src/repo/FreeBSD:14:amd64/14.3p2` or something like that, depending on the version build. + + To publish these packages to the network set up a nginx service and use this location in the http server configuration: + + [.programlisting] + .... + location /FreeBSD:14:amd64 { + alias /usr/obj/usr/src/repo/FreeBSD:14:amd64/; + autoindex on; + } + .... + + And reload the nginx service. + + When not using https, use a small configuration file on the clients to target the pkgbase version just built by editing `/usr/local/etc/pkg/repos/upgrade.conf`: + + [.programlisting] + .... + upgrade: { + url = http://ip.of.the.server/FreeBSD:14:amd64/14.3p2 + enabled = yes + } + .... + + and use it as written above (but use -r upgrade instead of FreeBSD-base). + + [[small-lan]] + == Tracking for Multiple Machines + + When multiple machines need to track the same source tree, it is a waste of disk space, network bandwidth, and CPU cycles to have each system download the sources and rebuild everything. + The solution is to have one machine do most of the work, while the rest of the machines mount that work via NFS. + This section outlines a method of doing so. + For more information about using NFS, refer to crossref:network-servers[network-nfs,"Network File System (NFS)"]. + + First, identify a set of machines which will run the same set of binaries, known as a _build set_. + Each machine can have a custom kernel, but will run the same userland binaries. + From that set, choose a machine to be the _build machine_ that the world and kernel are built on. + Ideally, this is a fast machine that has sufficient spare CPU to run `make buildworld` and `make buildkernel`. + + Select a machine to be the _test machine_, which will test software updates before they are put into production. + This _must_ be a machine that can afford to be down for an extended period of time. + It can be the build machine, but need not be. + + All the machines in this build set need to mount [.filename]#/usr/obj# and [.filename]#/usr/src# from the build machine via NFS. + For multiple build sets, [.filename]#/usr/src# should be on one build machine, and NFS mounted on the rest. + + Ensure that [.filename]#/etc/make.conf# and [.filename]#/etc/src.conf# on all the machines in the build set agree with the build machine. + That means that the build machine must build all the parts of the base system that any machine in the build set is going to install. + Also, each build machine should have its kernel name set with `KERNCONF` in [.filename]#/etc/make.conf#, + and the build machine should list them all in its `KERNCONF`, listing its own kernel first. + The build machine must have the kernel configuration files for each machine in its [.filename]#/usr/src/sys/arch/conf#. + + On the build machine, build the kernel and world as described in + crossref:cutting-edge[makeworld, Updating FreeBSD from Source], + but do not install anything on the build machine. + Instead, install the built kernel on the test machine. + On the test machine, mount [.filename]#/usr/src# and [.filename]#/usr/obj# via NFS. + Then, run `shutdown now` to go to single-user mode in order to install the new kernel and world and run `etcupdate` as usual. + When done, reboot to return to normal multi-user operations. + + After verifying that everything on the test machine is working properly, + use the same procedure to install the new software on each of the other machines in the build set. + + The same methodology can be used for the ports tree. + The first step is to share [.filename]#/usr/ports# via NFS to all the machines in the build set. + To configure [.filename]#/etc/make.conf# to share distfiles, + set `DISTDIR` to a common shared directory that is writable by whichever user `root` is mapped to by the NFS mount. + Each machine should set `WRKDIRPREFIX` to a local build directory, if ports are to be built locally. + Alternately, if the build system is to build and distribute packages to the machines in the build set, + set `PACKAGES` on the build system to a directory similar to `DISTDIR`. + + [[building-on-non-freebsd-hosts]] + == Building on non-FreeBSD Hosts + + Historically, building required a FreeBSD host. + Nowadays, FreeBSD can be built on Linux and macOS. + + To build on a non-FreeBSD host, the `tools/build/make.py` script is recommended. + This script acts as a wrapper around `bmake`, which is the make implementation used by FreeBSD. + It ensures that the necessary tooling, including the actual FreeBSD's man:make[1], is bootstrapped and that the build environment is properly configured. + In particular, it sets the external toolchain variables, such as `XCC`, `XLD`, and others. + Additionally, the script can pass any additional command arguments, such as `-j 4` for parallel builds or specific make targets, to `bmake`. + + [NOTE] + ==== + A recent version of `bmake` can be used instead of the `tools/build/make.py` script as well. + In that case, however, required environment variables need to be set manually (the easiest way to obtain a list of them is by running `tools/build/make.py --debug`). + ==== + + Otherwise, the list of prerequisites for building FreeBSD is rather short. + In fact, it boils down to installing a couple of dependencies. + + On macOS, the only dependency is LLVM. + The necessary dependencies can be installed with package manager (e.g., link:https://brew.sh/[Homebrew]): + + [source,shell] + .... + brew install llvm + .... + + On a Linux distributions, install link:https://clang.llvm.org/[Clang] version 10.0 or newer and the headers for libarchive and libbz2 (often packaged as libarchive-dev and libbz2-dev). + + Once the dependencies are installed, the host should be able to build FreeBSD. + + For example, the following `tools/build/make.py` invocation builds the world: + + [source,shell] + .... + MAKEOBJDIRPREFIX=/tmp/obj tools/build/make.py -j 8 TARGET=arm64 TARGET_ARCH=aarch64 buildworld + .... + + It builds the world for target `aarch64:arm64` on 8 CPUs and uses [.filename]#/tmp/obj# for object files. + Note that the variables `MAKEOBJDIRPREFIX`, `TARGET`, and `TARGET_ARCH` are mandatory when building on non-FreeBSD hosts. + Also, make sure to create the object directory pointed to by the `MAKEOBJDIRPREFIX` environment variable. + + Refer to man:arch[7] and man:build[7] for more details. +diff --git a/documentation/content/en/books/handbook/desktop/_index.adoc b/documentation/content/en/books/handbook/desktop/_index.adoc +index 829a63d77c..ae999cf717 100644 +--- a/documentation/content/en/books/handbook/desktop/_index.adoc ++++ b/documentation/content/en/books/handbook/desktop/_index.adoc +@@ -1,1148 +1,1148 @@ + --- + title: Chapter 8. Desktop Environments + part: Part II. Common Tasks + prev: books/handbook/partii + next: books/handbook/multimedia + description: This chapter demonstrates how to install numerous desktop environments, including web browsers, productivity software, document viewers, and financial software + tags: ["desktop", "KDE Plasma", "GNOME", "XFCE", "MATE", "Cinnamon", "LXQT", "browsers", "Firefox", "Chromium", "Iridium", "Falkon", "Konqueror", "Epiphany", "qutebrowser", "Dillo", "Links", "w3m", "Development tools", "Visual Studio Code", "Qt Creator", "Kdevelop", "Eclipse IDE", "Vim", "Neovim", "GNU Emacs", "Productivity", "LibreOffice", "Calligra", "AbiWord", "Viewers", "Okular", "Evince", "ePDFView", "Xpdf", "Finance", "KMyMoney", "GnuCash"] + showBookMenu: true + weight: 11 + params: + path: "/books/handbook/desktop/" + --- + + [[desktop]] + = Desktop Environments + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :sectnumoffset: 8 + :partnums: + :source-highlighter: rouge + :experimental: + :images-path: books/handbook/desktop/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + :imagesdir: ../../../../images/{images-path} + endif::[] + ifndef::book[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + toc::[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + toc::[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [[desktop-synopsis]] + == Synopsis + + While FreeBSD is popular as a server for its performance and stability, it is also well suited for day-to-day use as a desktop. + With over {numports} applications available in the FreeBSD ports tree, it is straightforward to build a customized desktop that can run a wide variety of desktop applications. + This chapter demonstrates how to install popular desktop environments as well as desktop applications such as web browsers, productivity software, document viewers, and financial software. + + Prerequisites: + + * Readers of this chapter should already understand how to install either the crossref:x11[x11,X Window System] or crossref:wayland[wayland,Wayland] on FreeBSD. + * Readers are instructed throughout this chapter to install official packages. + Refer to the section on crossref:ports[ports-using,using the ports collection] to build customized packages from ports. + + [[desktop-environments]] + == Desktop Environments + + This section describes how to install and configure some popular desktop environments on a FreeBSD system. + A desktop environment can range from a simple window manager to a complete suite of desktop applications. + + .Supported desktop environments + [options="header", cols="1,1,1"] + |=== + | Name | License | Package + + | KDE Plasma + | GPL 2.0 or later + | x11/kde + + | GNOME + | GPL 2.0 or later + | x11/gnome + + | XFCE + | GPL, LGPL, BSD + | x11-wm/xfce4 + + | MATE + | GPL 2.0, LGPL 2.0 + | x11/mate + + | Cinnamon + | GPL 2.0 or later + | x11/cinnamon + + | LXQT + | GPL, LGPL + | x11-wm/lxqt + + |=== + + [[kde-environment]] + === KDE Plasma + + KDE Plasma is an easy-to-use desktop environment. + This desktop provides a suite of applications with a consistent look and feel, a standardized menu and toolbars, keybindings, color-schemes, internationalization, and a centralized, dialog-driven desktop configuration. + More information on KDE can be found at the link:https://kde.org/[KDE homepage]. + For FreeBSD-specific information, consult the link:https://freebsd.kde.org/[FreeBSD homepage at KDE]. + + [[kde-meta-install]] + ==== Install KDE Plasma meta package + + To install the KDE Plasma meta package with KDE Frameworks, Plasma Desktop and Applications execute: + + [source,shell] + .... + # pkg install kde + .... + + [[kde-minimal-install]] + ==== Minimal KDE Plasma installation + + To install a minimal KDE Plasma execute: + + [source,shell] + .... + # pkg install plasma6-plasma + .... + + [TIP] + ==== + This installation is *really* minimal. + Konsole must be installed separately executing: + + [source,shell] + .... + # pkg install konsole + .... + ==== + + [[kde-configuration]] + ==== Configure KDE Plasma + + KDE Plasma uses man:dbus-daemon[1] for a message bus and hardware abstraction. + This application is automatically installed as a dependency of KDE Plasma. + + Enable D-BUS service in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc dbus_enable="YES" + .... + + KDE Plasma requires larger message sizes for optimal performance. + + Add the following lines to man:sysctl.conf[5]: + + [.programlisting] + .... + sysctl net.local.stream.recvspace=65536 + sysctl net.local.stream.sendspace=65536 + .... + + To apply the change, either run the following command as root or simply reboot the system: + + [source,shell] + .... + # sysctl -f /etc/sysctl.conf + .... + + [[kde-start]] + ==== Start KDE Plasma + + The preferred KDE Plasma display manager is package:x11/sddm[]. + To install package:x11/sddm[], execute: + + [source,shell] + .... + # pkg install sddm + .... + + Enable SDDM service in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc sddm_enable="YES" + .... + + The keyboard language can be set in SDDM by running the following command (for Spanish, for example): + + [source,shell] + .... + # sysrc sddm_lang="es_ES" + .... + + A second method to start KDE Plasma is by manually invoking man:startx[1]. + For this to work, the following line is needed in ~/.xinitrc: + + [source,shell] + .... + % echo "exec dbus-launch --exit-with-x11 ck-launch-session startplasma-x11" > ~/.xinitrc + .... + + [[gnome-environment]] + === GNOME + + GNOME is a user-friendly desktop environment. + It includes a panel for starting applications and displaying status, a desktop, a set of tools and applications, and a set of conventions that make it easy for applications to cooperate and be consistent with each other. + + [[gnome-meta-install]] + ==== Install GNOME meta package + + To install the GNOME meta package with GNOME Desktop and Applications, execute: + + [source,shell] + .... + # pkg install gnome + .... + + [[gnome-minimal-install]] + ==== Minimal GNOME installation + + To install the GNOME-lite meta package with a GNOME desktop slimmed down for only the basics, execute: + + [source,shell] + .... + # pkg install gnome-lite + .... + + [[gnome-configuration]] + ==== Configure GNOME + + GNOME requires `/proc` to be mounted. + Add this line to `/etc/fstab` to mount this file system automatically during system startup: + + [.programlisting] + .... + # Device Mountpoint FStype Options Dump Pass# + proc /proc procfs rw 0 0 + .... + + GNOME uses man:dbus-daemon[1] for a message bus and hardware abstraction. + This application is automatically installed as a dependency of GNOME. + + Enable D-BUS service in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc dbus_enable="YES" + .... + + [[gnome-start]] + ==== Start GNOME + + GNOME Display Manager is the preferred display manager for GNOME. + GDM is installed as part of the GNOME package. + + Enable GDM in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc gdm_enable="YES" + .... + + A second method to start GNOME is by manually invoking man:startx[1]. + For this to work, the following line is needed in `~/.xinitrc`: + + [source,shell] + .... + % echo "exec gnome-session" > ~/.xinitrc + .... + + [[xfce-environment]] + === XFCE + + XFCE is a desktop environment based on the GTK+, lightweight and provides a simple, efficient, easy-to-use desktop. + It is fully configurable, has a main panel with menus, applets, and application launchers, provides a file manager and sound manager, and is themeable. + Since it is fast, light, and efficient, it is ideal for older or slower machines with memory limitations. + + [[xfce-install]] + ==== Install XFCE + + To install the XFCE meta package, execute: + + [source,shell] + .... + # pkg install xfce + .... + + [[xfce-configuration]] + ==== Configure XFCE + + XFCE uses man:dbus-daemon[1] for a message bus and hardware abstraction. + This application is automatically installed as a dependency of XFCE. + + Enable D-BUS in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc dbus_enable="YES" + .... + + [[xfce-start]] + ==== Start XFCE + + package:x11/lightdm[] is a display manager that supports different display technologies and is a good choice as it is very lightweight, requires little memory usage, and has fast performance. + + To install it, execute: + + [source,shell] + .... + # pkg install lightdm lightdm-gtk-greeter + .... + + Enable lightdm in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc lightdm_enable="YES" + .... + + A second method to start XFCE is by manually invoking man:startx[1]. + For this to work, the following line is needed in `~/.xinitrc`: + + [source,shell] + .... + % echo '. /usr/local/etc/xdg/xfce4/xinitrc' > ~/.xinitrc + .... + + [[mate-environment]] + === MATE + + The MATE Desktop Environment is the continuation of GNOME 2. + It provides an intuitive and attractive desktop environment using traditional metaphors. + + [[mate-meta-install]] + ==== Install MATE meta package + + To install the MATE meta package that includes the MATE Desktop with some extra applications such as text editor, archiver manager, etc., execute: + + [source,shell] + .... + # pkg install mate + .... + + [[mate-minimal-install]] + ==== Minimal MATE installation + + To install the MATE lite meta package with MATE desktop slimmed down for only the basics, execute: + + [source,shell] + .... + # pkg install mate-base + .... + + [[mate-configuration]] + ==== Configure MATE + + MATE requires `/proc` to be mounted. + Add this line to `/etc/fstab` to mount this file system automatically during system startup: + + [.programlisting] + .... + # Device Mountpoint FStype Options Dump Pass# + proc /proc procfs rw 0 0 + .... + + MATE uses man:dbus-daemon[1] for a message bus and hardware abstraction. + This application is automatically installed as a dependency of MATE. + Enable D-BUS in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc dbus_enable="YES" + .... + + [[mate-start]] + ==== Start MATE + + package:x11/lightdm[] is a display manager that supports different display technologies and is a good choice as it is very lightweight, requires little memory usage, and has fast performance. + + To install it, execute: + + [source,shell] + .... + # pkg install lightdm lightdm-gtk-greeter + .... + + Enable lightdm in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc lightdm_enable="YES" + .... + + A second method to start MATE is by manually invoking man:startx[1]. + For this to work, the following line is needed in `~/.xinitrc`: + + [source,shell] + .... + % echo "exec dbus-launch --exit-with-x11 ck-launch-session mate-session" > ~/.xinitrc + .... + + [[cinnamon-environment]] + === Cinnamon + + Cinnamon is a UNIX(R) desktop which provides advanced innovative features and a traditional user experience. + The desktop layout is similar to Gnome 2. + The underlying technology is forked from Gnome Shell. + The emphasis is put on making users feel at home and providing them with an easy to use and comfortable desktop experience. + + [[cinnamon-install]] + ==== Install Cinnamon + + To install the Cinnamon package, execute: + + [source,shell] + .... + # pkg install cinnamon + .... + + [[cinnamon-configuration]] + ==== Configure Cinnamon + + Cinnamon requires `/proc` to be mounted. + Add this line to `/etc/fstab` to mount this file system automatically during system startup: + + [.programlisting] + .... + # Device Mountpoint FStype Options Dump Pass# + proc /proc procfs rw 0 0 + .... + + Cinnamon uses man:dbus-daemon[1] for a message bus and hardware abstraction. + This application is automatically installed as a dependency of Cinnamon. + Enable D-BUS in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc dbus_enable="YES" + .... + + [[cinnamon-start]] + ==== Start Cinnamon + + package:x11/lightdm[] is a display manager that supports different display technologies and is a good choice as it is very lightweight, requires little memory usage, and has fast performance. + + To install it execute: + + [source,shell] + .... + # pkg install lightdm lightdm-gtk-greeter + .... + + Enable lightdm in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc lightdm_enable="YES" + .... + + A second method to start Cinnamon is by manually invoking man:startx[1]. + For this to work, the following line is needed in `~/.xinitrc`: + + [source,shell] + .... + % echo "exec dbus-launch --exit-with-x11 ck-launch-session cinnamon-session" > ~/.xinitrc + .... + + [[lxqt-environment]] + === LXQT + + LXQt is an advanced, easy-to-use, and fast desktop environment based on Qt technologies. + It has been tailored for users who value simplicity, speed, and an intuitive interface. + Unlike most desktop environments, LXQt also works fine with less powerful machines. + + [[lxqt-install]] + ==== Install LXQT + + To install the LXQT meta package, execute: + + [source,shell] + .... + # pkg install lxqt + .... + + [[lxqt-configuration]] + ==== Configure LXQT + + LXQT requires `/proc` to be mounted. + Add this line to `/etc/fstab` to mount this file system automatically during system startup: + + [.programlisting] + .... + # Device Mountpoint FStype Options Dump Pass# + proc /proc procfs rw 0 0 + .... + + LXQT uses man:dbus-daemon[1] for a message bus and hardware abstraction. + This application is automatically installed as a dependency of LXQT. + + Enable D-BUS in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc dbus_enable="YES" + .... + + [[lxqt-start]] + ==== Start LXQT + + The preferred LXQT display manager is package:x11/sddm[]. + To install package:x11/sddm[], execute: + + [source,shell] + .... + # pkg install sddm + .... + + Enable SDDM service in `/etc/rc.conf` to start at system boot: + + [source,shell] + .... + # sysrc sddm_enable="YES" + .... + + The keyboard language can be set in SDDM by running the following command (for example, for Spanish): + + [source,shell] + .... + # sysrc sddm_lang="es_ES" + .... + + A second method to start LXQT is by manually invoking man:startx[1]. + For this to work, the following line is needed in `~/.xinitrc`: + + [source,shell] + .... + % echo "exec dbus-launch --exit-with-x11 ck-launch-session startlxqt" > ~/.xinitrc + .... + + [[desktop-browsers]] + == Browsers + + This section describes how to install and configure some popular web browsers on a FreeBSD system, from full web browsers with high resource consumption to command line web browsers with reduced resource usage. + + .Supported browsers + [options="header", cols="1,1,1,1"] + |=== + | Name | License | Package | Resources Needed + + | Firefox + | MPL 2.0 + | package:www/firefox[] + | Heavy + + | Chromium + | BSD-3 and others + | package:www/chromium[] + | Heavy + + | Iridium browser + | BSD-3 and others + | package:www/iridium-browser[] + | Heavy + + | Falkon + | MPL 2.0 + | package:www/falkon-qtonly[] + | Heavy + + | Konqueror + | GPL 2.0 or later + | package:x11-fm/konqueror[] + | Medium + + | Gnome Web (Epiphany) + | GPL 3.0 or later + | package:www/epiphany[] + | Medium + + | qutebrowser + | GPL 3.0 or later + | package:www/qutebrowser[] + | Medium + + | Dillo + | GPL 3.0 or later + | package:www/dillo2[] + | Light + + | Links + | GPL 2.0 or later + | package:www/links[] + | Light + + | w3m + | MIT + | package:www/w3m[] + | Light + + |=== + + [[firefox]] + === Firefox + + Firefox is an open source browser that features a standards-compliant HTML display engine, tabbed browsing, popup blocking, extensions, improved security, and more. + Firefox is based on the Mozilla codebase. + + To install the package of the latest release version of Firefox, execute: + + [source,shell] + .... + # pkg install firefox + .... + + To instead install Firefox Extended Support Release (ESR) version, execute: + + [source,shell] + .... + # pkg install firefox-esr + .... + + [[chromium]] + === Chromium + + Chromium is an open source browser project that aims to build a safer, faster, and more stable web browsing experience. + Chromium features tabbed browsing, popup blocking, extensions, and much more. + Chromium is the open source project upon which the Google Chrome web browser is based. + + To install Chromium, execute: + + [source,shell] + .... + # pkg install chromium + .... + + [NOTE] + ==== + The executable for Chromium is [.filename]#/usr/local/bin/chrome#, not [.filename]#/usr/local/bin/chromium#. + ==== + + [[iridium]] + === Iridium browser + + Iridium is a free, open, and libre browser modification of the Chromium code base, with privacy being enhanced in several key areas. + Automatic transmission of partial queries, keywords, metrics to central services is inhibited and only occurs with consent. + + To install Iridium, execute: + + [source,shell] + .... + # pkg install iridium-browser + .... + + [[falkon]] + === Falkon + + Falkon is a new-ish and very fast QtWebEngine browser. + It aims to be a lightweight web browser available on all major platforms. + Falkon has all standard functions someone can expect from a web browser. + It includes bookmarks, history (both also in sidebar) and tabs. + Beyond that, AdBlock plugin can block ads, Click2Flash can block Flash content and SSL Manager can edit the local CA Certificates database. + + To install Falkon, execute: + + [source,shell] + .... + # pkg install falkon + .... + + [[konqueror]] + === Konqueror + + Konqueror is more than a web browser as it is also a file manager and a multimedia viewer. + It supports WebKit, a rendering engine used by many modern browsers including Chromium, as well as its own KHTML engine. + + To install Konqueror, execute: + + [source,shell] + .... + # pkg install konqueror + .... + + [[gnome-web-epiphany]] + === Gnome Web (Epiphany) + + Gnome Web (Epiphany) is a web browser designed to be as lightweight and fast as possible, at the expense of many of the features found in other browsers. + + To install Gnome Web (Epiphany), execute: + + [source,shell] + .... + # pkg install epiphany + .... + + [[qutebrowser]] + === qutebrowser + + Qutebrowser is a keyboard-focused browser with a minimal GUI. + It is based on Python and PyQt5 and free software, licensed under the GPL. + + To install qutebrowser, execute: + + [source,shell] + .... + # pkg install qutebrowser + .... + + [[dillo]] + === Dillo + + Dillo aims to be a multiplatform alternative browser that is small, stable, developer-friendly, usable, fast, and extensible. + This new, experimental version of Dillo is based upon FLTK toolkit, rather than GTK1, and has been substantially rewritten. + + To install Dillo, execute: + + [source,shell] + .... + # pkg install dillo2 + .... + + [[links]] + === Links + + A lynx-like web browser with text and graphics modes with many features like displaying tables, menus, etc. + + To install Links, execute: + + [source,shell] + .... + # pkg install links + .... + + [[w3m]] + === w3m + + w3m is a pager/text-based web browser. + It is a similar application to Lynx, but it has several features Lynx does not have like rendering tables and rendering frames. + + To install w3m, execute: + + [source,shell] + .... + # pkg install w3m + .... + + [[desktop-development]] + == Development tools + + This section describes how to install and configure some popular development tools on a FreeBSD system. + + .Supported development tools + [options="header", cols="1,1,1,1"] + |=== + | Name | License | Package | Resources Needed + + | Visual Studio Code + | MIT + | package:editors/vscode[] + | Heavy + + | Qt Creator + | QtGPL + | package:devel/qtcreator[] + | Heavy + + | Kdevelop + | GPL 2.0 or later and LGPL 2.0 or later + | package:devel/kdevelop[] + | Heavy + + | Eclipse IDE + | EPL + | package:java/eclipse[] + | Heavy + + | Vim + | VIM + | package:editors/vim[] + | Light + + | Neovim + | Apache 2.0 + | package:editors/neovim[] + | Light + + | GNU Emacs + | GPL 3.0 or later + | package:editors/emacs[] + | Light + + |=== + + [[vs-code]] + === Visual Studio Code + + Visual Studio Code is a type of tool that combines the simplicity of a code editor with what developers need for their core edit-build-debug cycle. + It provides comprehensive editing and debugging support, an extensibility model, and lightweight integration with existing tools. + + To install Visual Studio Code, execute: + + [source,shell] + .... + # pkg install vscode + .... + + [[qt-creator]] + === Qt Creator + + Qt Creator is a cross-platform IDE (integrated development environment) tailored to the needs of Qt developers. + Functionalities included with Qt Creator are: + + * code editor with C++, QML and ECMAscript support; + * rapid code navigation tools; + * static code checking and style hints as you type; + * context sensitive help; + * visual debugger; + * integrated GUI layout and forms designer. + + To install Qt Creator, execute: + + [source,shell] + .... + # pkg install qtcreator + .... + + [[kdevelop]] + === kdevelop + + Open source, feature-full, plugin extensible IDE for C/C++ and other programming languages. + It is based on KDevPlatform and the KDE and Qt libraries, and it has been under development since 1998. + + To install kdevelop, execute: + + [source,shell] + .... + # pkg install kdevelop + .... + + [[eclipse]] + === Eclipse IDE + + The Eclipse Platform is an open extensible IDE for anything and yet nothing in particular. + The Eclipse Platform provides building blocks and a foundation for constructing and running integrated software-development tools. + The Eclipse Platform allows tool builders to independently develop tools that integrate with other people's tools. + + To install Eclipse IDE, execute: + + [source,shell] + .... + # pkg install eclipse + .... + + [[vim]] + === Vim + + Vim is a highly configurable text editor built to enable efficient text editing. + It is an improved version of the vi editor distributed with most UNIX systems. + + Vim is often called a "programmer's editor," and so useful for programming that many consider it an entire IDE. + It is not just for programmers, though. + Vim is perfect for all kinds of text editing, from composing email to editing configuration files. + + To install Vim, execute: + + [source,shell] + .... + # pkg install vim + .... + + [[neovim]] + === Neovim + + Neovim is an aggressive refactor of package:editors/vim[]. + It represents a complete overhaul of the codebase with many sanity improvements, including sensible defaults, a built-in terminal emulator, asynchronous plugin architecture, and powerful APIs designed for speed and extensibility. + It retains full compatibility with almost all Vim plugins and scripts. + + To install Neovim, execute: + + [source,shell] + .... + # pkg install neovim + .... + + [[gnu-emacs]] + === GNU Emacs + + GNU Emacs is an extensible, customizable, free/libre text editor. + At its core is an interpreter for Emacs Lisp, a dialect of the Lisp programming language with extensions to support text editing. + + To install GNU Emacs, execute: + + [source,shell] + .... + # pkg install emacs + .... + + [[desktop-productivity]] + == Desktop office productivity + + When it comes to productivity, users often look for an office suite or an easy-to-use word processor. + While some desktop environments like crossref:desktop[kde-environment, KDE Plasma] provide an office suite, there is no default productivity package. + Several office suites and graphical word processors are available for FreeBSD, regardless of the installed desktop environments. + + This section demonstrates how to install the following popular productivity software and indicates if the application is resource-heavy, takes time to compile from ports, or has any major dependencies. + + .Supported Desktop office productivity suites + [options="header", cols="1,1,1,1"] + |=== + | Name | License | Package | Resources Needed + + | LibreOffice + | MPL 2.0 + | package:editors/libreoffice[] + | Heavy + + | Calligra Suite + | LGPL and GPL + | package:editors/calligra[] + | Medium + + | AbiWord + | GPL 2.0 or later + | package:editors/abiword[] + | Medium + + |=== + + [[libreoffice]] + === LibreOffice + + LibreOffice is a free software office suite developed by http://www.documentfoundation.org/[The Document Foundation]. + It is compatible with other major office suites and available on a variety of platforms. + It is a rebranded fork of Apache OpenOffice and includes applications found in a complete office productivity suite: a word processor, spreadsheet, presentation manager, drawing program, database management program, and a tool for creating and editing mathematical formulæ. + It is available in a number of different languages and internationalization has been extended to interfaces, spell checkers, and dictionaries. + More information about LibreOffice can be found at http://www.libreoffice.org/[libreoffice.org]. + + To install LibreOffice, execute: + + [source,shell] + .... + # pkg install libreoffice + .... + + The LibreOffice package comes by default only in English. + To have a localized version of LibreOffice it is necessary to install a language pack. + For example, for the version localized in Spanish, it is necessary to install the package package:editors/libreoffice-es[] with the command: + + [source,shell] + .... +-# pkg install libreoffice-es ++# pkg install es-libreoffice + .... + + [[calligra]] + === Calligra + + The KDE Plasma desktop environment includes an office suite which can be installed separately from KDE Plasma. + Calligra includes standard components that can be found in other office suites. + Words is the word processor, Sheets is the spreadsheet program, Stage manages slide presentations, and Karbon is used to draw graphical documents. + + To install Calligra, execute: + + [source,shell] + .... + # pkg install calligra + .... + + [[abiword]] + === AbiWord + + AbiWord is a free word processing program similar in look and feel to Microsoft(R) Word. + It is fast, contains many features, and is user-friendly. + + AbiWord can import or export many file formats, including some proprietary ones like Microsoft(R) [.filename]#.rtf#. + + To install AbiWord, execute: + + [source,shell] + .... + # pkg install abiword + .... + + [[desktop-viewers]] + == Document Viewers + + Some new document formats have gained popularity since the advent of UNIX(R) and the viewers they require may not be available in the base system. + This section demonstrates how to install the following document viewers: + + .Supported Document Viewers + [options="header", cols="1,1,1,1"] + |=== + | Name | License | Package | Resources Needed + + | Okular + | GPL 2.0 + | package:graphics/okular[] + | Heavy + + | Evince + | GPL 2.0 + | package:graphics/evince[] + | Medium + + | ePDFView + | GPL 2.0 + | package:graphics/epdfview[] + | Medium + + | Xpdf + | GPL 2.0 + | package:graphics/xpdf[] + | light + + | Zathura + | Zlib + | package:graphics/zathura[] + | light + + |=== + + [[okular]] + === Okular + + Okular is a universal document viewer, part of the KDE Plasma project. + + Okular combines excellent functionality with the versatility of supporting different kind of documents, like PDF, Postscript, DjVu, CHM, XPS, ePub and others. + + To install Okular, execute: + + [source,shell] + .... + # pkg install okular + .... + + [[evince]] + === Evince + + Evince is a document viewer for multiple document formats including PDF and Postscript. + Part of the GNOME project. + The goal of evince is to replace document viewers such as ggv and gpdf with a single, simple application. + + To install Evince, execute: + + [source,shell] + .... + # pkg install evince + .... + + [[epdfview]] + === ePDFView + + ePDFView is a lightweight PDF document viewer that only uses the Gtk+ and Poppler libraries. + The aim of ePDFView is to make a simple PDF document viewer, similar to Evince but without using the GNOME libraries. + + To install ePDFView, execute: + + [source,shell] + .... + # pkg install epdfview + .... + + [[xpdf]] + === Xpdf + + For users that prefer a small FreeBSD PDF viewer, Xpdf provides a light-weight and efficient viewer which requires few resources. + It uses the standard X fonts and does not require any additional toolkit. + + To install Xpdf, execute: + + [source,shell] + .... + # pkg install xpdf + .... + + [[zathura]] + === Zathura + + Zathura is a highly customizable and functional document viewer. It provides a minimalistic and space saving interface as well + as an easy usage that mainly focuses on keyboard interaction. + + To install zathura, with PDF support, execute: + + [source,shell] + .... + # pkg install zathura zathura-pdf-mupdf + .... + + Additionally, one can install package:graphics/zathura-pdf-poppler[] for alternative PDF support, package:graphics/zathura-ps[] for PostScript support, package:graphics/zathura-djvu[] for DjVu support, and package:graphics/zathura-cb[] for comic book support. + + [[desktop-finance]] + == Finance + + For managing personal finances on a FreeBSD desktop, some powerful and easy-to-use applications can be installed. + Some are compatible with widespread file formats, such as the formats used by Quicken and Excel. + + This section covers these programs: + + .Supported Finance programs + [options="header", cols="1,1,1,1"] + |=== + | Name | License | Package | Resources Needed + + | KMyMoney + | GPL 2.0 + | package:finance/kmymoney[] + | Heavy + + | GnuCash + | GPL 2.0 and GPL 3.0 + | package:finance/gnucash[] + | Heavy + + |=== + + [[kmymoney]] + === KMyMoney + + KMyMoney is a personal finance application created by the KDE community. + KMyMoney aims to provide the important features found in commercial personal finance manager applications. + It also highlights ease-of-use and proper double-entry accounting among its features. + KMyMoney imports from standard Quicken QIF files, tracks investments, handles multiple currencies, and provides a wealth of reports. + + To install KMyMoney, execute: + + [source,shell] + .... + # pkg install kmymoney + .... + + [[gnucash]] + === GnuCash + + GnuCash is part of the GNOME effort to provide user-friendly, yet powerful, applications to end-users. + GnuCash can be used to keep track of income and expenses, bank accounts, and stocks. + It features an intuitive interface while remaining professional. + + GnuCash provides a smart register, a hierarchical system of accounts, and many keyboard accelerators and auto-completion methods. + It can split a single transaction into several more detailed pieces. + GnuCash can import and merge Quicken QIF files. + It also handles most international date and currency formats. + + To install GnuCash, execute: + + [source,shell] + .... + # pkg install gnucash + .... +diff --git a/documentation/content/en/books/handbook/disks/_index.adoc b/documentation/content/en/books/handbook/disks/_index.adoc +index b86395f9e4..5b792ccb47 100644 +--- a/documentation/content/en/books/handbook/disks/_index.adoc ++++ b/documentation/content/en/books/handbook/disks/_index.adoc +@@ -1,2492 +1,2492 @@ + --- + title: Chapter 20. Storage + part: Part III. System Administration + prev: books/handbook/audit + next: books/handbook/geom + description: This chapter covers the use of disks and storage media in FreeBSD. This includes SCSI and IDE disks, CD and DVD media, memory-backed disks, and USB storage devices. + tags: ["storage", "disks", "gpart", "mount", "quotas", "encrypt", "GPT", "cdrecord", "quotas", "swap", "HAST", "CD", "DVD", "resizing", "growing"] + showBookMenu: true + weight: 24 + params: + path: "/books/handbook/disks/" + --- + + [[disks]] + = Storage + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :sectnumoffset: 20 + :partnums: + :source-highlighter: rouge + :experimental: + :images-path: books/handbook/disks/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + :imagesdir: ../../../../images/{images-path} + endif::[] + ifndef::book[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + toc::[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + toc::[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + [[disks-synopsis]] + == Synopsis + + This chapter covers the use of disks and storage media in FreeBSD. + This includes SCSI and IDE disks, CD and DVD media, memory-backed disks, and USB storage devices. + + Read this chapter to learn: + + * How to add additional hard disks to a FreeBSD system. + * How to grow the size of a disk's partition on FreeBSD. + * How to configure FreeBSD to use USB storage devices. + * How to use CD and DVD media on a FreeBSD system. + * How to use the backup programs available under FreeBSD. + * How to set up memory disks. + * What file system snapshots are and how to use them efficiently. + * How to use quotas to limit disk space usage. + * How to encrypt disks and swap to secure them against attackers. + * How to configure a highly available storage network. + + Before reading this chapter: + + * Know how to crossref:kernelconfig[kernelconfig,configure and install a new FreeBSD kernel]. + + [[disks-adding]] + == Adding Disks + + This section describes how to add a new SATA disk to a machine that currently only has a single drive. + First, turn off the computer and install the drive in the computer following the instructions of the computer, controller, and drive manufacturers. + Reboot the system and become `root`. + + Inspect [.filename]#/var/run/dmesg.boot# to ensure the new disk was found. + In this example, the newly added SATA drive will appear as [.filename]#ada1#. + + For this example, a single large partition will be created on the new disk. + The https://en.wikipedia.org/wiki/GUID_Partition_Table[GPT] partitioning scheme will be used in preference to the older and less versatile MBR scheme. + + [NOTE] + ==== + If the disk to be added is not blank, old partition information can be removed with `gpart delete`. + See man:gpart[8] for details. + ==== + + The partition scheme is created, and then a single partition is added. + To improve performance on newer disks with larger hardware block sizes, the partition is aligned to one megabyte boundaries: + + [source,shell] + .... + # gpart create -s GPT ada1 + # gpart add -t freebsd-ufs -a 1M ada1 + .... + + Depending on use, several smaller partitions may be desired. + See man:gpart[8] for options to create partitions smaller than a whole disk. + + The disk partition information can be viewed with `gpart show`: + + [source,shell] + .... + % gpart show ada1 + => 34 1465146988 ada1 GPT (699G) + 34 2014 - free - (1.0M) + 2048 1465143296 1 freebsd-ufs (699G) + 1465145344 1678 - free - (839K) + .... + + A file system is created in the new partition on the new disk: + + [source,shell] + .... + # newfs -U /dev/ada1p1 + .... + + An empty directory is created as a _mountpoint_, a location for mounting the new disk in the original disk's file system: + + [source,shell] + .... + # mkdir /newdisk + .... + + Finally, an entry is added to [.filename]#/etc/fstab# so the new disk will be mounted automatically at startup: + + [.programlisting] + .... + /dev/ada1p1 /newdisk ufs rw 2 2 + .... + + The new disk can be mounted manually, without restarting the system: + + [source,shell] + .... + # mount /newdisk + .... + + [[disks-growing]] + == Resizing and Growing Disks + + A disk's capacity can increase without any changes to the data already present. + This happens commonly with virtual machines, when the virtual disk turns out to be too small and is enlarged. + Sometimes a disk image is written to a USB memory stick, but does not use the full capacity. + Here we describe how to resize or _grow_ disk contents to take advantage of increased capacity. + + Determine the device name of the disk to be resized by inspecting [.filename]#/var/run/dmesg.boot#. + In this example, there is only one SATA disk in the system, so the drive will appear as [.filename]#ada0#. + + List the partitions on the disk to see the current configuration: + + [source,shell] + .... + # gpart show ada0 + => 34 83886013 ada0 GPT (48G) [CORRUPT] + 34 128 1 freebsd-boot (64k) + 162 79691648 2 freebsd-ufs (38G) + 79691810 4194236 3 freebsd-swap (2G) + 83886046 1 - free - (512B) + .... + + [NOTE] + ==== + If the disk was formatted with the https://en.wikipedia.org/wiki/GUID_Partition_Table[GPT] partitioning scheme, it may show as "corrupted" because the GPT backup partition table is no longer at the end of the drive. + Fix the backup partition table with `gpart`: + + [source,shell] + .... + # gpart recover ada0 + ada0 recovered + .... + + ==== + + Now the additional space on the disk is available for use by a new partition, or an existing partition can be expanded: + + [source,shell] + .... + # gpart show ada0 + => 34 102399933 ada0 GPT (48G) + 34 128 1 freebsd-boot (64k) + 162 79691648 2 freebsd-ufs (38G) + 79691810 4194236 3 freebsd-swap (2G) + 83886046 18513921 - free - (8.8G) + .... + + Partitions can only be resized into contiguous free space. + Here, the last partition on the disk is the swap partition, but the second partition is the one that needs to be resized. + Swap partitions only contain temporary data, so it can safely be unmounted, deleted, and then recreate the third partition after resizing the second partition. + + Disable the swap partition: + + [source,shell] + .... + # swapoff /dev/ada0p3 + .... + + Delete the third partition, specified by the `-i` flag, from the disk _ada0_. + + [source,shell] + .... + # gpart delete -i 3 ada0 + ada0p3 deleted + # gpart show ada0 + => 34 102399933 ada0 GPT (48G) + 34 128 1 freebsd-boot (64k) + 162 79691648 2 freebsd-ufs (38G) + 79691810 22708157 - free - (10G) + .... + + [WARNING] + ==== + + There is risk of data loss when modifying the partition table of a mounted file system. + It is best to perform the following steps on an unmounted file system while running from a live CD-ROM or USB device. + However, if absolutely necessary, a mounted file system can be resized after disabling GEOM safety features: + + [source,shell] + .... + # sysctl kern.geom.debugflags=16 + .... + + ==== + + Resize the partition, leaving room to recreate a swap partition of the desired size. + The partition to resize is specified with `-i`, and the new desired size with `-s`. + Optionally, alignment of the partition is controlled with `-a`. + This only modifies the size of the partition. + The file system in the partition will be expanded in a separate step. + + [source,shell] + .... + # gpart resize -i 2 -s 47G -a 4k ada0 + ada0p2 resized + # gpart show ada0 + => 34 102399933 ada0 GPT (48G) + 34 128 1 freebsd-boot (64k) + 162 98566144 2 freebsd-ufs (47G) + 98566306 3833661 - free - (1.8G) + .... + + Recreate the swap partition and activate it. + If no size is specified with `-s`, all remaining space is used: + + [source,shell] + .... + # gpart add -t freebsd-swap -a 4k ada0 + ada0p3 added + # gpart show ada0 + => 34 102399933 ada0 GPT (48G) + 34 128 1 freebsd-boot (64k) + 162 98566144 2 freebsd-ufs (47G) + 98566306 3833661 3 freebsd-swap (1.8G) + # swapon /dev/ada0p3 + .... + + Grow the UFS file system to use the new capacity of the resized partition: + + [source,shell] + .... + # growfs /dev/ada0p2 + Device is mounted read-write; resizing will result in temporary write suspension for /. + It's strongly recommended to make a backup before growing the file system. + OK to grow file system on /dev/ada0p2, mounted on /, from 38GB to 47GB? [Yes/No] Yes + super-block backups (for fsck -b #) at: + 80781312, 82063552, 83345792, 84628032, 85910272, 87192512, 88474752, + 89756992, 91039232, 92321472, 93603712, 94885952, 96168192, 97450432 + .... + + If the file system is ZFS, the resize is triggered by running the `online` subcommand with `-e`: + + [source,shell] + .... + # zpool online -e zroot /dev/ada0p2 + .... + + Both the partition and the file system on it have now been resized to use the newly-available disk space. + + [[usb-disks]] + == USB Storage Devices + + Many external storage solutions, such as hard drives, USB thumbdrives, and CD and DVD burners, use the Universal Serial Bus (USB). + FreeBSD provides support for USB 1.x, 2.0, and 3.0 devices. + + [NOTE] + ==== + USB 3.0 support is not compatible with some hardware, including Haswell (Lynx point) chipsets. + If FreeBSD boots with a `failed with error 19` message, disable xHCI/USB3 in the system BIOS. + ==== + + Support for USB storage devices is built into the [.filename]#GENERIC# kernel. + For a custom kernel, be sure that the following lines are present in the kernel configuration file: + + [.programlisting] + .... + device scbus # SCSI bus (required for ATA/SCSI) + device da # Direct Access (disks) + device pass # Passthrough device (direct ATA/SCSI access) + device uhci # provides USB 1.x support + device ohci # provides USB 1.x support + device ehci # provides USB 2.0 support + device xhci # provides USB 3.0 support + device usb # USB Bus (required) + device umass # Disks/Mass storage - Requires scbus and da + device cd # needed for CD and DVD burners + .... + + FreeBSD uses the man:umass[4] driver which uses the SCSI subsystem to access USB storage devices. + Since any USB device will be seen as a SCSI device by the system, if the USB device is a CD or DVD burner, do _not_ include `device atapicam` in a custom kernel configuration file. + + The rest of this section demonstrates how to verify that a USB storage device is recognized by FreeBSD and how to configure the device so that it can be used. + + === Device Configuration + + To test the USB configuration, plug in the USB device. + Use `dmesg` to confirm that the drive appears in the system message buffer. + It should look something like this: + + [source,shell] + .... + umass0: on usbus0 + umass0: SCSI over Bulk-Only; quirks = 0x0100 + umass0:4:0:-1: Attached to scbus4 + da0 at umass-sim0 bus 0 scbus4 target 0 lun 0 + da0: Fixed Direct Access SCSI-4 device + da0: Serial Number WD-WXE508CAN263 + da0: 40.000MB/s transfers + da0: 152627MB (312581808 512 byte sectors: 255H 63S/T 19457C) + da0: quirks=0x2 + .... + + The brand, device node ([.filename]#da0#), speed, and size will differ according to the device. + + Since the USB device is seen as a SCSI one, `camcontrol` can be used to list the USB storage devices attached to the system: + + [source,shell] + .... + # camcontrol devlist + at scbus4 target 0 lun 0 (pass3,da0) + .... + + Alternately, `usbconfig` can be used to list the device. + Refer to man:usbconfig[8] for more information about this command. + + [source,shell] + .... + # usbconfig + ugen0.3: at usbus0, cfg=0 md=HOST spd=HIGH (480Mbps) pwr=ON (2mA) + .... + + If the device has not been formatted, refer to crossref:disks[disks-adding, Adding Disks] for instructions on how to format and create partitions on the USB drive. + If the drive comes with a file system, it can be mounted by `root` using the instructions in crossref:basics[mount-unmount,“Mounting and Unmounting File Systems”]. + + [WARNING] + ==== + Allowing untrusted users to mount arbitrary media, by enabling `vfs.usermount` as described below, should not be considered safe from a security point of view. + Most file systems were not built to safeguard against malicious devices. + ==== + + To make the device mountable as a normal user, one solution is to make all users of the device a member of the `operator` group using man:pw[8]. + Next, ensure that `operator` is able to read and write the device by adding these lines to [.filename]#/etc/devfs.rules#: + + [.programlisting] + .... + [localrules=5] + add path 'da*' mode 0660 group operator + .... + + [NOTE] + ==== + If internal SCSI disks are also installed in the system, change the second line as follows: + + [.programlisting] + .... + add path 'da[3-9]*' mode 0660 group operator + .... + + This will exclude the first three SCSI disks ([.filename]#da0# to [.filename]#da2#) from belonging to the `operator` group. + Replace _3_ with the number of internal SCSI disks. + Refer to man:devfs.rules[5] for more information about this file. + ==== + + Next, enable the ruleset in [.filename]#/etc/rc.conf#: + + [.programlisting] + .... + devfs_system_ruleset="localrules" + .... + + Then, instruct the system to allow regular users to mount file systems by adding the following line to [.filename]#/etc/sysctl.conf#: + + [.programlisting] + .... + vfs.usermount=1 + .... + + Since this only takes effect after the next reboot, use `sysctl` to set this variable now: + + [source,shell] + .... + # sysctl vfs.usermount=1 + vfs.usermount: 0 -> 1 + .... + + The final step is to create a directory where the file system is to be mounted. + This directory needs to be owned by the user that is to mount the file system. + One way to do that is for `root` to create a subdirectory owned by that user as [.filename]#/mnt/username#. + In the following example, replace _username_ with the login name of the user and _usergroup_ with the user's primary group: + + [source,shell] + .... + # mkdir /mnt/username + # chown username:usergroup /mnt/username + .... + + Suppose a USB thumbdrive is plugged in, and a device [.filename]#/dev/da0s1# appears. + If the device is formatted with a FAT file system, the user can mount it using: + + [source,shell] + .... + % mount -t msdosfs -o -m=644,-M=755 /dev/da0s1 /mnt/username + .... + + Before the device can be unplugged, it _must_ be unmounted first: + + [source,shell] + .... + % umount /mnt/username + .... + + After device removal, the system message buffer will show messages similar to the following: + + [source,shell] + .... + umass0: at uhub3, port 2, addr 3 (disconnected) + da0 at umass-sim0 bus 0 scbus4 target 0 lun 0 + da0: s/n WD-WXE508CAN263 detached + (da0:umass-sim0:0:0:0): Periph destroyed + .... + + === Automounting Removable Media + + USB devices can be automatically mounted by uncommenting this line in [.filename]#/etc/auto_master#: + + [source,shell] + .... + /media -media -nosuid + .... + + Then add these lines to [.filename]#/etc/devd.conf#: + + [source,shell] + .... + notify 100 { + match "system" "GEOM"; + match "subsystem" "DEV"; + action "/usr/sbin/automount -c"; + }; + .... + + Reload the configuration if man:autofs[5] and man:devd[8] are already running: + + [source,shell] + .... + # service automount restart + # service devd restart + .... + + man:autofs[5] can be set to start at boot by adding this line to [.filename]#/etc/rc.conf#: + + [.programlisting] + .... + autofs_enable="YES" + .... + + man:autofs[5] requires man:devd[8] to be enabled, as it is by default. + + Start the services immediately with: + + [source,shell] + .... + # service automount start + # service automountd start + # service autounmountd start + # service devd start + .... + + Each file system that can be automatically mounted appears as a directory in [.filename]#/media/#. + The directory is named after the file system label. + If the label is missing, the directory is named after the device node. + + The file system is transparently mounted on the first access, and unmounted after a period of inactivity. + Automounted drives can also be unmounted manually: + + [source,shell] + .... + # automount -fu + .... + + This mechanism is typically used for memory cards and USB memory sticks. + It can be used with any block device, including optical drives or iSCSILUNs. + + [[creating-cds]] + == Creating and Using CD Media + + Compact Disc (CD) media provide a number of features that differentiate them from conventional disks. + They are designed so that they can be read continuously without delays to move the head between tracks. + While CD media do have tracks, these refer to a section of data to be read continuously, and not a physical property of the disk. + The ISO 9660 file system was designed to deal with these differences. + + The FreeBSD Ports Collection provides several utilities for burning and duplicating audio and data CDs. + This chapter demonstrates the use of several command line utilities. + For CD burning software with a graphical utility, consider installing the package:sysutils/xcdroast[] or package:sysutils/k3b[] packages or ports. + + [[atapicam]] + === Supported Devices + + The [.filename]#GENERIC# kernel provides support for SCSI, USB, and ATAPICD readers and burners. + If a custom kernel is used, the options that need to be present in the kernel configuration file vary by the type of device. + + For a SCSI burner, make sure these options are present: + + [.programlisting] + .... + device scbus # SCSI bus (required for ATA/SCSI) + device da # Direct Access (disks) + device pass # Passthrough device (direct ATA/SCSI access) + device cd # needed for CD and DVD burners + .... + + For a USB burner, make sure these options are present: + + [.programlisting] + .... + device scbus # SCSI bus (required for ATA/SCSI) + device da # Direct Access (disks) + device pass # Passthrough device (direct ATA/SCSI access) + device cd # needed for CD and DVD burners + device uhci # provides USB 1.x support + device ohci # provides USB 1.x support + device ehci # provides USB 2.0 support + device xhci # provides USB 3.0 support + device usb # USB Bus (required) + device umass # Disks/Mass storage - Requires scbus and da + .... + + For an ATAPI burner, make sure these options are present: + + [.programlisting] + .... + device ata # Legacy ATA/SATA controllers + device scbus # SCSI bus (required for ATA/SCSI) + device pass # Passthrough device (direct ATA/SCSI access) + device cd # needed for CD and DVD burners + .... + + [NOTE] + ==== + On FreeBSD versions prior to 10.x, this line is also needed in the kernel configuration file if the burner is an ATAPI device: + + [.programlisting] + .... + device atapicam + .... + + Alternately, this driver can be loaded at boot time by adding the following line to [.filename]#/boot/loader.conf#: + + [.programlisting] + .... + atapicam_load="YES" + .... + + This will require a reboot of the system as this driver can only be loaded at boot time. + ==== + + To verify that FreeBSD recognizes the device, run `dmesg` and look for an entry for the device. + On systems prior to 10.x, the device name in the first line of the output will be [.filename]#acd0# instead of [.filename]#cd0#. + + [source,shell] + .... + % dmesg | grep cd + cd0 at ahcich1 bus 0 scbus1 target 0 lun 0 + cd0: Removable CD-ROM SCSI-0 device + cd0: Serial Number M3OD3S34152 + cd0: 150.000MB/s transfers (SATA 1.x, UDMA6, ATAPI 12bytes, PIO 8192bytes) + cd0: Attempt to query device size failed: NOT READY, Medium not present - tray closed + .... + + [[cdrecord]] + === Burning a CD + + In FreeBSD, `cdrecord` can be used to burn CDs. + This command is installed with the package:sysutils/cdrtools[] package or port. + + While `cdrecord` has many options, basic usage is simple. + Specify the name of the ISO file to burn and, if the system has multiple burner devices, specify the name of the device to use: + + [source,shell] + .... + # cdrecord dev=device imagefile.iso + .... + + To determine the device name of the burner, use `-scanbus` which might produce results like this: + + [source,shell] + .... + # cdrecord -scanbus + ProDVD-ProBD-Clone 3.00 (amd64-unknown-freebsd10.0) Copyright (C) 1995-2010 Jörg Schilling + Using libscg version 'schily-0.9' + scsibus0: + 0,0,0 0) 'SEAGATE ' 'ST39236LW ' '0004' Disk + 0,1,0 1) 'SEAGATE ' 'ST39173W ' '5958' Disk + 0,2,0 2) * + 0,3,0 3) 'iomega ' 'jaz 1GB ' 'J.86' Removable Disk + 0,4,0 4) 'NEC ' 'CD-ROM DRIVE:466' '1.26' Removable CD-ROM + 0,5,0 5) * + 0,6,0 6) * + 0,7,0 7) * + scsibus1: + 1,0,0 100) * + 1,1,0 101) * + 1,2,0 102) * + 1,3,0 103) * + 1,4,0 104) * + 1,5,0 105) 'YAMAHA ' 'CRW4260 ' '1.0q' Removable CD-ROM + 1,6,0 106) 'ARTEC ' 'AM12S ' '1.06' Scanner + 1,7,0 107) * + .... + + Locate the entry for the CD burner and use the three numbers separated by commas as the value for `dev`. + In this case, the Yamaha burner device is `1,5,0`, so the appropriate input to specify that device is `dev=1,5,0`. + Refer to the manual page for `cdrecord` for other ways to specify this value and for information on writing audio tracks and controlling the write speed. + + Alternately, run the following command to get the device address of the burner: + + [source,shell] + .... + # camcontrol devlist + at scbus1 target 0 lun 0 (cd0,pass0) + .... + + Use the numeric values for `scbus`, `target`, and `lun`. + For this example, `1,0,0` is the device name to use. + + [[mkisofs]] + === Writing Data to an ISO File System + + In order to produce a data CD, the data files that are going to make up the tracks on the CD must be prepared before they can be burned to the CD. + In FreeBSD, package:sysutils/cdrtools[] installs `mkisofs`, which can be used to produce an ISO 9660 file system that is an image of a directory tree within a UNIX(R) file system. + The simplest usage is to specify the name of the ISO file to create and the path to the files to place into the ISO 9660 file system: + + [source,shell] + .... + # mkisofs -o imagefile.iso /path/to/tree + .... + + This command maps the file names in the specified path to names that fit the limitations of the standard ISO 9660 file system, and will exclude files that do not meet the standard for ISO file systems. + + A number of options are available to overcome the restrictions imposed by the standard. + In particular, `-R` enables the Rock Ridge extensions common to UNIX(R) systems and `-J` enables Joliet extensions used by Microsoft(R) systems. + + For CDs that are going to be used only on FreeBSD systems, `-U` can be used to disable all filename restrictions. + When used with `-R`, it produces a file system image that is identical to the specified FreeBSD tree, even if it violates the ISO 9660 standard. + + The last option of general use is `-b`. + This is used to specify the location of a boot image for use in producing an "El Torito" bootable CD. + This option takes an argument which is the path to a boot image from the top of the tree being written to the CD. + By default, `mkisofs` creates an ISO image in "floppy disk emulation" mode, and thus expects the boot image to be exactly 1200, 1440 or 2880 KB in size. + Some boot loaders, like the one used by the FreeBSD distribution media, do not use emulation mode. + In this case, `-no-emul-boot` should be used. + So, if [.filename]#/tmp/myboot# holds a bootable FreeBSD system with the boot image in [.filename]#/tmp/myboot/boot/cdboot#, this command would produce [.filename]#/tmp/bootable.iso#: + + [source,shell] + .... + # mkisofs -R -no-emul-boot -b boot/cdboot -o /tmp/bootable.iso /tmp/myboot + .... + + The resulting ISO image can be mounted as a memory disk with: + + [source,shell] + .... + # mdconfig -a -t vnode -f /tmp/bootable.iso -u 0 + # mount -t cd9660 /dev/md0 /mnt + .... + + One can then verify that [.filename]#/mnt# and [.filename]#/tmp/myboot# are identical. + + There are many other options available for `mkisofs` to fine-tune its behavior. + Refer to man:mkisofs[8] for details. + + [NOTE] + ==== + It is possible to copy a data CD to an image file that is functionally equivalent to the image file created with `mkisofs`. + To do so, use [.filename]#dd# with the device name as the input file and the name of the ISO to create as the output file: + + [source,shell] + .... + # dd if=/dev/cd0 of=file.iso bs=2048 + .... + + The resulting image file can be burned to CD as described in crossref:disks[cdrecord, Burning a CD]. + ==== + + [[mounting-cd]] + === Using Data CDs + + Once an ISO has been burned to a CD, it can be mounted by specifying the file system type, the name of the device containing the CD, and an existing mount point: + + [source,shell] + .... + # mount -t cd9660 /dev/cd0 /mnt + .... + + Since `mount` assumes that a file system is of type `ufs`, an `Incorrect super block` error will occur if `-t cd9660` is not included when mounting a data CD. + + While any data CD can be mounted this way, disks with certain ISO 9660 extensions might behave oddly. + For example, Joliet disks store all filenames in two-byte Unicode characters. + If some non-English characters show up as question marks, specify the local charset with `-C`. + For more information, refer to man:mount_cd9660[8]. + + [NOTE] + ==== + In order to do this character conversion with the help of `-C`, the kernel requires the [.filename]#cd9660_iconv.ko# module to be loaded. + This can be done either by adding this line to [.filename]#loader.conf#: + + [.programlisting] + .... + cd9660_iconv_load="YES" + .... + + and then rebooting the machine, or by directly loading the module with `kldload`. + ==== + + Occasionally, `Device not configured` will be displayed when trying to mount a data CD. + This usually means that the CD drive has not detected a disk in the tray, or that the drive is not visible on the bus. + It can take a couple of seconds for a CD drive to detect media, so be patient. + + Sometimes, a SCSICD drive may be missed because it did not have enough time to answer the bus reset. + To resolve this, a custom kernel can be created which increases the default SCSI delay. + Add the following option to the custom kernel configuration file and rebuild the kernel using the instructions in crossref:kernelconfig[kernelconfig-building,“Building and Installing a Custom Kernel”]: + + [.programlisting] + .... + options SCSI_DELAY=15000 + .... + + This tells the SCSI bus to pause 15 seconds during boot, to give the CD drive every possible chance to answer the bus reset. + + [NOTE] + ==== + It is possible to burn a file directly to CD, without creating an ISO 9660 file system. + This is known as burning a raw data CD and some people do this for backup purposes. + + This type of disk can not be mounted as a normal data CD. + In order to retrieve the data burned to such a CD, the data must be read from the raw device node. + For example, this command will extract a compressed tar file located on the second CD device into the current working directory: + + [source,shell] + .... + # tar xzvf /dev/cd1 + .... + + In order to mount a data CD, the data must be written using `mkisofs`. + ==== + + [[duplicating-audiocds]] + === Duplicating Audio CDs + + To duplicate an audio CD, extract the audio data from the CD to a series of files, then write these files to a blank CD. + + crossref:disks[using-cdrecord, Duplicating an Audio CD] describes how to duplicate and burn an audio CD. + If the FreeBSD version is less than 10.0 and the device is ATAPI, the `atapicam` + module must be first loaded using the instructions in crossref:disks[atapicam, Supported Devices]. + + [[using-cdrecord]] + [.procedure] + .Procedure: Duplicating an Audio CD + . The package:sysutils/cdrtools[] package or port installs `cdda2wav`. This command can be used to extract all of the audio tracks, with each track written to a separate WAV file in the current working directory: + + + [source,shell] + .... + % cdda2wav -vall -B -Owav + .... + + + A device name does not need to be specified if there is only one CD device on the system. + Refer to the `cdda2wav` manual page for instructions on how to specify a device and to learn more about the other options available for this command. + . Use `cdrecord` to write the [.filename]#.wav# files: + + + [source,shell] + .... + % cdrecord -v dev=2,0 -dao -useinfo *.wav + .... + + + Make sure that _2,0_ is set appropriately, as described in crossref:disks[cdrecord, Burning a CD]. + + [[creating-dvds]] + == Creating and Using DVD Media + + Compared to the CD, the DVD is the next generation of optical media storage technology. + The DVD can hold more data than any CD and is the standard for video publishing. + + Five physical recordable formats can be defined for a recordable DVD: + + * DVD-R: This was the first DVD recordable format available. The DVD-R standard is defined by the http://www.dvdforum.org/forum.shtml[DVD Forum]. This format is write once. + * DVD-RW: This is the rewritable version of the DVD-R standard. A DVD-RW can be rewritten about 1000 times. + * DVD-RAM: This is a rewritable format which can be seen as a removable hard + drive. However, this media is not compatible with most DVD-ROM drives and + DVD-Video players as only a few DVD writers support the DVD-RAM format. Refer + to crossref:disks[creating-dvd-ram, Using a DVD-RAM] for more information on DVD-RAM use. + * DVD+RW: This is a rewritable format defined by the https://en.wikipedia.org/wiki/DVD%2BRW_Alliance[DVD+RW Alliance]. A DVD+RW can be rewritten about 1000 times. + * DVD+R: This format is the write once variation of the DVD+RW format. + + A single layer recordable DVD can hold up to 4,700,000,000 bytes which is actually 4.38 GB or 4485 MB as 1 kilobyte is 1024 bytes. + + [NOTE] + ==== + A distinction must be made between the physical media and the application. + For example, a DVD-Video is a specific file layout that can be written on any recordable DVD physical media such as DVD-R, DVD+R, or DVD-RW. + Before choosing the type of media, ensure that both the burner and the DVD-Video player are compatible with the media under consideration. + ==== + + === Configuration + + To perform DVD recording, use man:growisofs[1]. + This command is part of the package:sysutils/dvd+rw-tools[] utilities which support all DVD media types. + + These tools use the SCSI subsystem to access the devices, therefore + crossref:disks[atapicam,ATAPI/CAM support] must be loaded or statically compiled into the kernel. + This support is not needed if the burner uses the USB interface. + Refer to crossref:disks[usb-disks, USB Storage Devices] for more details on USB device configuration. + + DMA access must also be enabled for ATAPI devices, by adding the following line to [.filename]#/boot/loader.conf#: + + [.programlisting] + .... + hw.ata.atapi_dma="1" + .... + + Before attempting to use dvd+rw-tools, consult the http://fy.chalmers.se/~appro/linux/DVD+RW/hcn.html[Hardware Compatibility Notes]. + + [NOTE] + ==== + For a graphical user interface, consider using package:sysutils/k3b[] which provides a user friendly interface to man:growisofs[1] and many other burning tools. + ==== + + === Burning Data DVDs + + Since man:growisofs[1] is a front-end to crossref:disks[mkisofs,mkisofs], it will invoke man:mkisofs[8] to create the file system layout and perform the write on the DVD. + This means that an image of the data does not need to be created before the burning process. + + To burn to a DVD+R or a DVD-R the data in [.filename]#/path/to/data#, use the following command: + + [source,shell] + .... + # growisofs -dvd-compat -Z /dev/cd0 -J -R /path/to/data + .... + + In this example, `-J -R` is passed to man:mkisofs[8] to create an ISO 9660 file system with Joliet and Rock Ridge extensions. + Refer to man:mkisofs[8] for more details. + + For the initial session recording, `-Z` is used for both single and multiple sessions. + Replace _/dev/cd0_, with the name of the DVD device. + Using `-dvd-compat` indicates that the disk will be closed and that the recording will be unappendable. + This should also provide better media compatibility with DVD-ROM drives. + + To burn a pre-mastered image, such as _imagefile.iso_, use: + + [source,shell] + .... + # growisofs -dvd-compat -Z /dev/cd0=imagefile.iso + .... + + The write speed should be detected and automatically set according to the media and the drive being used. + To force the write speed, use `-speed=`. Refer to man:growisofs[1] for example usage. + + [NOTE] + ==== + In order to support working files larger than 4.38GB, an UDF/ISO-9660 hybrid file system must be created by passing `-udf -iso-level 3` to man:mkisofs[8] and all related programs, such as man:growisofs[1]. + This is required only when creating an ISO image file or when writing files directly to a disk. + Since a disk created this way must be mounted as an UDF file system with man:mount_udf[8], it will be usable only on an UDF aware operating system. + Otherwise it will look as if it contains corrupted files. + + To create this type of ISO file: + + [source,shell] + .... + % mkisofs -R -J -udf -iso-level 3 -o imagefile.iso /path/to/data + .... + + To burn files directly to a disk: + + [source,shell] + .... + # growisofs -dvd-compat -udf -iso-level 3 -Z /dev/cd0 -J -R /path/to/data + .... + + When an ISO image already contains large files, no additional options are required for man:growisofs[1] to burn that image on a disk. + + Be sure to use an up-to-date version of package:sysutils/cdrtools[], which contains man:mkisofs[8], as an older version may not contain large files support. + If the latest version does not work, install package:sysutils/cdrtools-devel[] and read its man:mkisofs[8]. + ==== + + === Burning a DVD-Video + + A DVD-Video is a specific file layout based on the ISO 9660 and micro-UDF (M-UDF) specifications. + Since DVD-Video presents a specific data structure hierarchy, a particular program such as package:multimedia/dvdauthor[] is needed to author the DVD. + + If an image of the DVD-Video file system already exists, it can be burned in the same way as any other image. + If `dvdauthor` was used to make the DVD and the result is in [.filename]#/path/to/video#, the following command should be used to burn the DVD-Video: + + [source,shell] + .... + # growisofs -Z /dev/cd0 -dvd-video /path/to/video + .... + + `-dvd-video` is passed to man:mkisofs[8] to instruct it to create a DVD-Video file system layout. + This option implies the `-dvd-compat` man:growisofs[1] option. + + === Using a DVD+RW + + Unlike CD-RW, a virgin DVD+RW needs to be formatted before first use. + It is _recommended_ to let man:growisofs[1] take care of this automatically whenever appropriate. + However, it is possible to use `dvd+rw-format` to format the DVD+RW: + + [source,shell] + .... + # dvd+rw-format /dev/cd0 + .... + + Only perform this operation once and keep in mind that only virgin DVD+RW medias need to be formatted. + Once formatted, the DVD+RW can be burned as usual. + + To burn a totally new file system and not just append some data onto a DVD+RW, the media does not need to be blanked first. + Instead, write over the previous recording like this: + + [source,shell] + .... + # growisofs -Z /dev/cd0 -J -R /path/to/newdata + .... + + The DVD+RW format supports appending data to a previous recording. + This operation consists of merging a new session to the existing one as it is not considered to be multi-session writing. + man:growisofs[1] will _grow_ the ISO 9660 file system present on the media. + + For example, to append data to a DVD+RW, use the following: + + [source,shell] + .... + # growisofs -M /dev/cd0 -J -R /path/to/nextdata + .... + + The same man:mkisofs[8] options used to burn the initial session should be used during next writes. + + [NOTE] + ==== + Use `-dvd-compat` for better media compatibility with DVD-ROM drives. + When using DVD+RW, this option will not prevent the addition of data. + ==== + + To blank the media, use: + + [source,shell] + .... + # growisofs -Z /dev/cd0=/dev/zero + .... + + === Using a DVD-RW + + A DVD-RW accepts two disc formats: incremental sequential and restricted overwrite. + By default, DVD-RW discs are in sequential format. + + A virgin DVD-RW can be directly written without being formatted. + However, a non-virgin DVD-RW in sequential format needs to be blanked before writing a new initial session. + + To blank a DVD-RW in sequential mode: + + [source,shell] + .... + # dvd+rw-format -blank=full /dev/cd0 + .... + + [NOTE] + ==== + A full blanking using `-blank=full` will take about one hour on a 1x media. + A fast blanking can be performed using `-blank`, if the DVD-RW will be recorded in Disk-At-Once (DAO) mode. + To burn the DVD-RW in DAO mode, use the command: + + [source,shell] + .... + # growisofs -use-the-force-luke=dao -Z /dev/cd0=imagefile.iso + .... + + Since man:growisofs[1] automatically attempts to detect fast blanked media and engage DAO write, `-use-the-force-luke=dao` should not be required. + + One should instead use restricted overwrite mode with any DVD-RW as this format is more flexible than the default of incremental sequential. + ==== + + To write data on a sequential DVD-RW, use the same instructions as for the other DVD formats: + + [source,shell] + .... + # growisofs -Z /dev/cd0 -J -R /path/to/data + .... + + To append some data to a previous recording, use `-M` with man:growisofs[1]. + However, if data is appended on a DVD-RW in incremental sequential mode, a new session will be created on the disc and the result will be a multi-session disc. + + A DVD-RW in restricted overwrite format does not need to be blanked before a new initial session. + Instead, overwrite the disc with `-Z`. + It is also possible to grow an existing ISO 9660 file system written on the disc with `-M`. + The result will be a one-session DVD. + + To put a DVD-RW in restricted overwrite format, the following command must be used: + + [source,shell] + .... + # dvd+rw-format /dev/cd0 + .... + + To change back to sequential format, use: + + [source,shell] + .... + # dvd+rw-format -blank=full /dev/cd0 + .... + + === Multi-Session + + Few DVD-ROM drives support multi-session DVDs and most of the time only read the first session. + DVD+R, DVD-R and DVD-RW in sequential format can accept multiple sessions. + The notion of multiple sessions does not exist for the DVD+RW and the DVD-RW restricted overwrite formats. + + Using the following command after an initial non-closed session on a DVD+R, DVD-R, or DVD-RW in sequential format, will add a new session to the disc: + + [source,shell] + .... + # growisofs -M /dev/cd0 -J -R /path/to/nextdata + .... + + Using this command with a DVD+RW or a DVD-RW in restricted overwrite mode will append data while merging the new session to the existing one. + The result will be a single-session disc. + Use this method to add data after an initial write on these types of media. + + [NOTE] + ==== + Since some space on the media is used between each session to mark the end and start of sessions, one should add sessions with a large amount of data to optimize media space. + The number of sessions is limited to 154 for a DVD+R, about 2000 for a DVD-R, and 127 for a DVD+R Double Layer. + ==== + + === For More Information + + To obtain more information about a DVD, use `dvd+rw-mediainfo _/dev/cd0_` while the disc in the specified drive. + + More information about dvd+rw-tools can be found in man:growisofs[1], on the http://fy.chalmers.se/~appro/linux/DVD+RW/[dvd+rw-tools web site], and in the http://lists.debian.org/cdwrite/[cdwrite mailing list] archives. + + [NOTE] + ==== + When creating a problem report related to the use of dvd+rw-tools, always include the output of `dvd+rw-mediainfo`. + ==== + + [[creating-dvd-ram]] + === Using a DVD-RAM + + DVD-RAM writers can use either a SCSI or ATAPI interface. + For ATAPI devices, DMA access has to be enabled by adding the following line to [.filename]#/boot/loader.conf#: + + [.programlisting] + .... + hw.ata.atapi_dma="1" + .... + + A DVD-RAM can be seen as a removable hard drive. + Like any other hard drive, the DVD-RAM must be formatted before it can be used. + In this example, the whole disk space will be formatted with a standard UFS2 file system: + + [source,shell] + .... + # dd if=/dev/zero of=/dev/acd0 bs=2k count=1 + # bsdlabel -Bw acd0 + # newfs /dev/acd0 + .... + + The DVD device, [.filename]#acd0#, must be changed according to the configuration. + + Once the DVD-RAM has been formatted, it can be mounted as a normal hard drive: + + [source,shell] + .... + # mount /dev/acd0 /mnt + .... + + Once mounted, the DVD-RAM will be both readable and writeable. + + [[floppies]] + == Creating and Using Floppy Disks + + This section explains how to format a 3.5 inch floppy disk in FreeBSD. + + [.procedure] + ==== + *Procedure: Steps to Format a Floppy* + + A floppy disk needs to be low-level formatted before it can be used. + This is usually done by the vendor, but formatting is a good way to check media integrity. + To low-level format the floppy disk on FreeBSD, use man:fdformat[1]. + When using this utility, make note of any error messages, as these can help determine if the disk is good or bad. + + . To format the floppy, insert a new 3.5 inch floppy disk into the first floppy drive and issue: + + + [source,shell] + .... + # /usr/sbin/fdformat -f 1440 /dev/fd0 + .... + + + . After low-level formatting the disk, create a disk label as it is needed by the system to determine the size of the disk and its geometry. The supported geometry values are listed in [.filename]#/etc/disktab#. + + + To write the disk label, use man:bsdlabel[8]: + + + [source,shell] + .... + # /sbin/bsdlabel -B -w /dev/fd0 fd1440 + .... + + + . The floppy is now ready to be high-level formatted with a file system. The floppy's file system can be either UFS or FAT, where FAT is generally a better choice for floppies. + + + To format the floppy with FAT, issue: + + + [source,shell] + .... + # /sbin/newfs_msdos /dev/fd0 + .... + ==== + + The disk is now ready for use. + To use the floppy, mount it with man:mount_msdosfs[8]. + One can also install and use package:emulators/mtools[] from the Ports Collection. + + [[backup-basics]] + == Backup Basics + + Implementing a backup plan is essential in order to have the ability to recover from disk failure, accidental file deletion, random file corruption, or complete machine destruction, including destruction of on-site backups. + + The backup type and schedule will vary, depending upon the importance of the data, the granularity needed for file restores, and the amount of acceptable downtime. + Some possible backup techniques include: + + * Archives of the whole system, backed up onto permanent, off-site media. This provides protection against all of the problems listed above, but is slow and inconvenient to restore from, especially for non-privileged users. + * File system snapshots, which are useful for restoring deleted files or previous versions of files. + * Copies of whole file systems or disks which are synchronized with another system on the network using a scheduled package:net/rsync[]. + * Hardware or software RAID, which minimizes or avoids downtime when a disk fails. + + Typically, a mix of backup techniques is used. + For example, one could create a schedule to automate a weekly, full system backup that is stored off-site and to supplement this backup with hourly ZFS snapshots. + In addition, one could make a manual backup of individual directories or files before making file edits or deletions. + + This section describes some of the utilities which can be used to create and manage backups on a FreeBSD system. + + === File System Backups + + The traditional UNIX(R) programs for backing up a file system are man:dump[8], which creates the backup, and man:restore[8], which restores the backup. + These utilities work at the disk block level, below the abstractions of the files, links, and directories that are created by file systems. + Unlike other backup software, `dump` backs up an entire file system and is unable to backup only part of a file system or a directory tree that spans multiple file systems. + Instead of writing files and directories, `dump` writes the raw data blocks that comprise files and directories. + + [NOTE] + ==== + If `dump` is used on the root directory, it will not back up [.filename]#/home#, [.filename]#/usr#, or many other directories since these are typically mount points for other file systems or symbolic links into those file systems. + ==== + + When used to restore data, `restore` stores temporary files in [.filename]#/tmp/# by default. + When using a recovery disk with a small [.filename]#/tmp#, set `TMPDIR` to a directory with more free space for the restore to succeed. + + When using `dump`, be aware that some quirks remain from its early days in Version 6 of AT&T UNIX(R),circa 1975. + The default parameters assume a backup to a 9-track tape, rather than to another type of media or to the high-density tapes available today. + These defaults must be overridden on the command line. + + It is possible to backup a file system across the network to another system or a tape drive attached to another computer. + While the man:rdump[8] and man:rrestore[8] utilities can be used for this purpose, they are not considered to be secure. + + Instead, one can use `dump` and `restore` more securely over an SSH connection. + This example creates a full, compressed backup of [.filename]#/usr# and sends the backup file to the specified host over an SSH connection. + + .Using `dump` over ssh + [example] + ==== + [source,shell] + .... + # /sbin/dump -0uan -f - /usr | gzip -2 | ssh -c blowfish \ + targetuser@targetmachine.example.com dd of=/mybigfiles/dump-usr-l0.gz + .... + ==== + + This example sets `RSH` in order to write the backup to a tape drive on a remote system over an SSH connection: + + .Using `dump` over ssh with `RSH` Set + [example] + ==== + [source,shell] + .... + # env RSH=/usr/bin/ssh /sbin/dump -0uan -f targetuser@targetmachine.example.com:/dev/sa0 /usr + .... + ==== + + [TIP] + ==== + Systems using the crossref:zfs[,Z file system (ZFS)] can make use of man:zfs[8] for creating snapshots, as well as crossref:zfs[zfs-zfs-send,sending and receiving] them to/from remote systems. + ==== + + === Directory Backups + + Several built-in utilities are available for backing up and restoring specified files and directories as needed. + + A good choice for making a backup of all of the files in a directory is man:tar[1]. + This utility dates back to Version 6 of AT&T UNIX(R) and by default assumes a recursive backup to a local tape device. + Switches can be used to instead specify the name of a backup file. + + This example creates a compressed backup of the current directory and saves it to [.filename]#/tmp/mybackup.tgz#. + When creating a backup file, make sure that the backup is not saved to the same directory that is being backed up. + + .Backing Up the Current Directory with `tar` + [example] + ==== + [source,shell] + .... + # tar czvf /tmp/mybackup.tgz . + .... + ==== + + To restore the entire backup, `cd` into the directory to restore into and specify the name of the backup. + Note that this will overwrite any newer versions of files in the restore directory. + When in doubt, restore to a temporary directory or specify the name of the file within the backup to restore. + + .Restoring Up the Current Directory with `tar` + [example] + ==== + [source,shell] + .... + # tar xzvf /tmp/mybackup.tgz + .... + ==== + + There are dozens of available switches which are described in man:tar[1]. + This utility also supports the use of exclude patterns to specify which files should not be included when backing up the specified directory or restoring files from a backup. + + To create a backup using a specified list of files and directories, man:cpio[1] is a good choice. + Unlike `tar`, `cpio` does not know how to walk the directory tree and it must be provided the list of files to backup. + + For example, a list of files can be created using `ls` or `find`. + This example creates a recursive listing of the current directory which is then piped to `cpio` in order to create an output backup file named [.filename]#/tmp/mybackup.cpio#. + + .Using `ls` and `cpio` to Make a Recursive Backup of the Current Directory + [example] + ==== + [source,shell] + .... + # ls -R | cpio -ovF /tmp/mybackup.cpio + .... + ==== + + A backup utility which tries to bridge the features provided by `tar` and `cpio` is man:pax[1]. + Over the years, the various versions of `tar` and `cpio` became slightly incompatible. + POSIX(R) created `pax` which attempts to read and write many of the various `cpio` and `tar` formats, plus new formats of its own. + + The `pax` equivalent to the previous examples would be: + + .Backing Up the Current Directory with `pax` + [example] + ==== + [source,shell] + .... + # pax -wf /tmp/mybackup.pax . + .... + ==== + + [[backups-tapebackups]] + === Using Data Tapes for Backups + + While tape technology has continued to evolve, modern backup systems tend to combine off-site backups with local removable media. + FreeBSD supports any tape drive that uses SCSI, such as LTO or DAT. + There is limited support for SATA and USB tape drives. + + For SCSI tape devices, FreeBSD uses the man:sa[4] driver and the [.filename]#/dev/sa0#, [.filename]#/dev/nsa0#, and [.filename]#/dev/esa0# devices. + The physical device name is [.filename]#/dev/sa0#. When [.filename]#/dev/nsa0# is used, the backup application will not rewind the tape after writing a file, which allows writing more than one file to a tape. + Using [.filename]#/dev/esa0# ejects the tape after the device is closed. + + In FreeBSD, `mt` is used to control operations of the tape drive, such as seeking through files on a tape or writing tape control marks to the tape. + For example, the first three files on a tape can be preserved by skipping past them before writing a new file: + + [source,shell] + .... + # mt -f /dev/nsa0 fsf 3 + .... + + This utility supports many operations. Refer to man:mt[1] for details. + + To write a single file to tape using `tar`, specify the name of the tape device and the file to backup: + + [source,shell] + .... + # tar cvf /dev/sa0 file + .... + + To recover files from a `tar` archive on tape into the current directory: + + [source,shell] + .... + # tar xvf /dev/sa0 + .... + + To backup a UFS file system, use `dump`. + This examples backs up [.filename]#/usr# without rewinding the tape when finished: + + [source,shell] + .... + # dump -0aL -b64 -f /dev/nsa0 /usr + .... + + To interactively restore files from a `dump` file on tape into the current directory: + + [source,shell] + .... + # restore -i -f /dev/nsa0 + .... + + [[backups-programs-amanda]] + === Third-Party Backup Utilities + + The FreeBSD Ports Collection provides many third-party utilities which can be used to schedule the creation of backups, simplify tape backup, and make backups easier and more convenient. + Many of these applications are client/server based and can be used to automate the backups of a single system or all of the computers in a network. + + Popular utilities include: + + * Amanda (package:misc/amanda-server[] and package:misc/amanda-client[]), + * Bacula (package:sysutils/bacula13-server[] and package:sysutils/bacula13-client[]), + * Bareos (package:sysutils/bareos-server[] and package:sysutils/bareos-client[]), + * package:net/rsync[], + * package:sysutils/duply[], and + * package:sysutils/duplicity[]. + + === Emergency Recovery + + In addition to regular backups, it is recommended to perform the following steps as part of an emergency preparedness plan. + + Create a print copy of the output of the following commands: + + * `gpart show` + * `more /etc/fstab` + * `pkg prime-list` + * `dmesg` + + Store this printout and a copy of the installation media in a secure location. + Should an emergency restore be needed, boot into the installation media and select `Live CD` to access a rescue shell. + This rescue mode can be used to view the current state of the system, and if needed, to reformat disks and restore data from backups. + + Next, test the rescue shell and the backups. + Make notes of the procedure. + Store these notes with the media, the printouts, and the backups. + These notes may prevent the inadvertent destruction of the backups while under the stress of performing an emergency recovery. + + For an added measure of security, store the latest backup at a remote location which is physically separated from the computers and disk drives by a significant distance. + + [[disks-virtual]] + == Memory Disks + + In addition to physical disks, FreeBSD also supports the creation and use of memory disks. + One possible use for a memory disk is to access the contents of an ISO file system without the overhead of first burning it to a CD or DVD, then mounting the CD/DVD media. + + In FreeBSD, the man:md[4] driver is used to provide support for memory disks. + The [.filename]#GENERIC# kernel includes this driver. + When using a custom kernel configuration file, ensure it includes this line: + + [.programlisting] + .... + device md + .... + + [[disks-mdconfig]] + === Attaching and Detaching Existing Images + + To mount an existing file system image, use `mdconfig` to specify the name of the ISO file and a free unit number. + Then, refer to that unit number to mount it on an existing mount point. + Once mounted, the files in the ISO will appear in the mount point. + This example attaches _diskimage.iso_ to the memory device [.filename]#/dev/md0# then mounts that memory device on [.filename]#/mnt#: + + [source,shell] + .... + # mdconfig -f diskimage.iso -u 0 + # mount -t cd9660 /dev/md0 /mnt + .... + + Notice that `-t cd9660` was used to mount an ISO format. + If a unit number is not specified with `-u`, `mdconfig` will automatically allocate an unused memory device and output the name of the allocated unit, such as [.filename]#md4#. + Refer to man:mdconfig[8] for more details about this command and its options. + + When a memory disk is no longer in use, its resources should be released back to the system. + First, unmount the file system, then use `mdconfig` to detach the disk from the system and release its resources. + To continue this example: + + [source,shell] + .... + # umount /mnt + # mdconfig -d -u 0 + .... + + To determine if any memory disks are still attached to the system, type `mdconfig -l`. + + [[disks-md-freebsd5]] + === Creating a File- or Memory-Backed Memory Disk + + FreeBSD also supports memory disks where the storage to use is allocated from either a hard disk or an area of memory. + The first method is commonly referred to as a file-backed file system and the second method as a memory-backed file system. + Both types can be created using `mdconfig`. + + To create a new memory-backed file system, specify a type of `swap` and the size of the memory disk to create. + Then, format the memory disk with a file system and mount as usual. + This example creates a 5M memory disk on unit `1`. + That memory disk is then formatted with the UFS file system before it is mounted: + + [source,shell] + .... + # mdconfig -a -t swap -s 5m -u 1 + # newfs -U md1 + /dev/md1: 5.0MB (10240 sectors) block size 16384, fragment size 2048 + using 4 cylinder groups of 1.27MB, 81 blks, 192 inodes. + with soft updates + super-block backups (for fsck -b #) at: + 160, 2752, 5344, 7936 + # mount /dev/md1 /mnt + # df /mnt + Filesystem 1K-blocks Used Avail Capacity Mounted on + /dev/md1 4718 4 4338 0% /mnt + .... + + To create a new file-backed memory disk, first allocate an area of disk to use. + This example creates an empty 5MB file named [.filename]#newimage#: + + [source,shell] + .... + # dd if=/dev/zero of=newimage bs=1k count=5k + 5120+0 records in + 5120+0 records out + .... + + Next, attach that file to a memory disk, label the memory disk and format it with the UFS file system, mount the memory disk, and verify the size of the file-backed disk: + + [source,shell] + .... + # mdconfig -f newimage -u 0 + # bsdlabel -w md0 auto + # newfs -U md0a + /dev/md0a: 5.0MB (10224 sectors) block size 16384, fragment size 2048 + using 4 cylinder groups of 1.25MB, 80 blks, 192 inodes. + super-block backups (for fsck -b #) at: + 160, 2720, 5280, 7840 + # mount /dev/md0a /mnt + # df /mnt + Filesystem 1K-blocks Used Avail Capacity Mounted on + /dev/md0a 4710 4 4330 0% /mnt + .... + + It takes several commands to create a file- or memory-backed file system using `mdconfig`. + FreeBSD also comes with `mdmfs` which automatically configures a memory disk, formats it with the UFS file system, and mounts it. + For example, after creating _newimage_ with `dd`, this one command is equivalent to running the `bsdlabel`, `newfs`, and `mount` commands shown above: + + [source,shell] + .... + # mdmfs -F newimage -s 5m md0 /mnt + .... + + To instead create a new memory-based memory disk with `mdmfs`, use this one command: + + [source,shell] + .... + # mdmfs -s 5m md1 /mnt + .... + + If the unit number is not specified, `mdmfs` will automatically select an unused memory device. + For more details about `mdmfs`, refer to man:mdmfs[8]. + + [[snapshots]] + == File System Snapshots + +-FreeBSD offers a feature in conjunction with crossref:config[soft-updates,Soft Updates]: file system snapshots. ++FreeBSD offers a feature in conjunction with soft updates: file system snapshots. + + UFS snapshots allow a user to create images of specified file systems, and treat them as a file. + When using the crossref:zfs[,Z file system (ZFS)], refer to crossref:zfs[zfs-zfs-snapshot,"Managing Snapshots"] on how to use snapshots. + + Snapshot files must be created in the file system that the action is performed on, and a user may create no more than 20 snapshots per file system. + Active snapshots are recorded in the superblock so they are persistent across unmount and remount operations along with system reboots. + When a snapshot is no longer required, it can be removed using man:rm[1]. + While snapshots may be removed in any order, all the used space may not be acquired because another snapshot will possibly claim some of the released blocks. + + The un-alterable `snapshot` file flag is set by man:mksnap_ffs[8] after initial creation of a snapshot file. + man:unlink[1] makes an exception for snapshot files since it allows them to be removed. + + Snapshots are created using man:mount[8]. + To place a snapshot of [.filename]#/var# in the file [.filename]#/var/snapshot/snap#, use the following command: + + [source,shell] + .... + # mount -u -o snapshot /var/snapshot/snap /var + .... + + Alternatively, use man:mksnap_ffs[8] to create the snapshot: + + [source,shell] + .... + # mksnap_ffs /var /var/snapshot/snap + .... + + One can find snapshot files on a file system, such as [.filename]#/var#, using man:find[1]: + + [source,shell] + .... + # find /var -flags snapshot + .... + + Once a snapshot has been created, it has several uses: + + * Some administrators will use a snapshot file for backup purposes, because the snapshot can be transferred to CDs or tape. + * The file system integrity checker, man:fsck[8], may be run on the snapshot. Assuming that the file system was clean when it was mounted, this should always provide a clean and unchanging result. + * Running man:dump[8] on the snapshot will produce a dump file that is consistent with the file system and the timestamp of the snapshot. man:dump[8] can also take a snapshot, create a dump image, and then remove the snapshot in one command by using `-L`. + * The snapshot can be mounted as a frozen image of the file system. To man:mount[8] the snapshot [.filename]#/var/snapshot/snap# run: + + + [source,shell] + .... + # mdconfig -a -t vnode -o readonly -f /var/snapshot/snap -u 4 + # mount -r /dev/md4 /mnt + .... + + The frozen [.filename]#/var# is now available through [.filename]#/mnt#. + Everything will initially be in the same state it was during the snapshot creation time. + The only exception is that any earlier snapshots will appear as zero length files. + To unmount the snapshot, use: + + [source,shell] + .... + # umount /mnt + # mdconfig -d -u 4 + .... + + For more information about `softupdates` and file system snapshots, including technical papers, visit Marshall Kirk McKusick's website at http://www.mckusick.com/[http://www.mckusick.com/]. + + [[quotas]] + == Disk Quotas + + Disk quotas can be used to limit the amount of disk space or the number of files a user or members of a group may allocate on a per-file system basis. + This prevents one user or group of users from consuming all of the available disk space. + + This section describes how to configure disk quotas for the UFS file system. + To configure quotas on the ZFS file system, refer to crossref:zfs[zfs-zfs-quota,"Dataset, User, and Group Quotas"] + + === Enabling Disk Quotas + + To determine if the FreeBSD kernel provides support for disk quotas: + + [source,shell] + .... + % sysctl kern.features.ufs_quota + kern.features.ufs_quota: 1 + .... + + In this example, the `1` indicates quota support. + If the value is instead `0`, add the following line to a custom kernel configuration file and rebuild the kernel using the instructions in crossref:kernelconfig[kernelconfig,Configuring the FreeBSD Kernel]: + + [.programlisting] + .... + options QUOTA + .... + + Next, enable disk quotas in [.filename]#/etc/rc.conf#: + + [.programlisting] + .... + quota_enable="YES" + .... + + Normally on bootup, the quota integrity of each file system is checked by man:quotacheck[8]. + This program insures that the data in the quota database properly reflects the data on the file system. + This is a time consuming process that will significantly affect the time the system takes to boot. + To skip this step, add this variable to [.filename]#/etc/rc.conf#: + + [.programlisting] + .... + check_quotas="NO" + .... + + Finally, edit [.filename]#/etc/fstab# to enable disk quotas on a per-file system basis. + To enable per-user quotas on a file system, add `userquota` to the options field in the [.filename]#/etc/fstab# entry for the file system to enable quotas on. + For example: + + [.programlisting] + .... + /dev/da1s2g /home ufs rw,userquota 1 2 + .... + + To enable group quotas, use `groupquota` instead. + To enable both user and group quotas, separate the options with a comma: + + [.programlisting] + .... + /dev/da1s2g /home ufs rw,userquota,groupquota 1 2 + .... + + By default, quota files are stored in the root directory of the file system as [.filename]#quota.user# and [.filename]#quota.group#. + Refer to man:fstab[5] for more information. + Specifying an alternate location for the quota files is not recommended. + + Once the configuration is complete, reboot the system and [.filename]#/etc/rc# will automatically run the appropriate commands to create the initial quota files for all of the quotas enabled in [.filename]#/etc/fstab#. + + In the normal course of operations, there should be no need to manually run man:quotacheck[8], man:quotaon[8], or man:quotaoff[8]. + However, one should read these manual pages to be familiar with their operation. + + === Setting Quota Limits + + To verify that quotas are enabled, run: + + [source,shell] + .... + # quota -v + .... + + There should be a one line summary of disk usage and current quota limits for each file system that quotas are enabled on. + + The system is now ready to be assigned quota limits with `edquota`. + + Several options are available to enforce limits on the amount of disk space a user or group may allocate, and how many files they may create. + Allocations can be limited based on disk space (block quotas), number of files (inode quotas), or a combination of both. + Each limit is further broken down into two categories: hard and soft limits. + + A hard limit may not be exceeded. + Once a user reaches a hard limit, no further allocations can be made on that file system by that user. + For example, if the user has a hard limit of 500 kbytes on a file system and is currently using 490 kbytes, the user can only allocate an additional 10 kbytes. + Attempting to allocate an additional 11 kbytes will fail. + + Soft limits can be exceeded for a limited amount of time, known as the grace period, which is one week by default. + If a user stays over their limit longer than the grace period, the soft limit turns into a hard limit and no further allocations are allowed. + When the user drops back below the soft limit, the grace period is reset. + + In the following example, the quota for the `test` account is being edited. + When `edquota` is invoked, the editor specified by `EDITOR` is opened in order to edit the quota limits. + The default editor is set to vi. + + [source,shell] + .... + # edquota -u test + Quotas for user test: + /usr: kbytes in use: 65, limits (soft = 50, hard = 75) + inodes in use: 7, limits (soft = 50, hard = 60) + /usr/var: kbytes in use: 0, limits (soft = 50, hard = 75) + inodes in use: 0, limits (soft = 50, hard = 60) + .... + + There are normally two lines for each file system that has quotas enabled. + One line represents the block limits and the other represents the inode limits. + Change the value to modify the quota limit. + For example, to raise the block limit on [.filename]#/usr# to a soft limit of `500` and a hard limit of `600`, change the values in that line as follows: + + [.programlisting] + .... + /usr: kbytes in use: 65, limits (soft = 500, hard = 600) + .... + + The new quota limits take effect upon exiting the editor. + + Sometimes it is desirable to set quota limits on a range of users. + This can be done by first assigning the desired quota limit to a user. + Then, use `-p` to duplicate that quota to a specified range of user IDs (UIDs). + The following command will duplicate those quota limits for UIDs `10,000` through `19,999`: + + [source,shell] + .... + # edquota -p test 10000-19999 + .... + + For more information, refer to man:edquota[8]. + + === Checking Quota Limits and Disk Usage + + To check individual user or group quotas and disk usage, use man:quota[1]. + A user may only examine their own quota and the quota of a group they are a member of. + Only the superuser may view all user and group quotas. + To get a summary of all quotas and disk usage for file systems with quotas enabled, use man:repquota[8]. + + Normally, file systems that the user is not using any disk space on will not show in the output of `quota`, even if the user has a quota limit assigned for that file system. + Use `-v` to display those file systems. + The following is sample output from `quota -v` for a user that has quota limits on two file systems. + + [.programlisting] + .... + Disk quotas for user test (uid 1002): + Filesystem usage quota limit grace files quota limit grace + /usr 65* 50 75 5days 7 50 60 + /usr/var 0 50 75 0 50 60 + .... + + In this example, the user is currently 15 kbytes over the soft limit of 50 kbytes on [.filename]#/usr# and has 5 days of grace period left. + The asterisk `*` indicates that the user is currently over the quota limit. + + === Quotas over NFS + + Quotas are enforced by the quota subsystem on the NFS server. + The man:rpc.rquotad[8] daemon makes quota information available to `quota` on NFS clients, allowing users on those machines to see their quota statistics. + + On the NFS server, enable `rpc.rquotad` by removing the `+#+` from this line in [.filename]*/etc/inetd.conf*: + + [.programlisting] + .... + rquotad/1 dgram rpc/udp wait root /usr/libexec/rpc.rquotad rpc.rquotad + .... + + Then, restart `inetd`: + + [source,shell] + .... + # service inetd restart + .... + + [[disks-encrypting]] + == Encrypting Disk Partitions + + FreeBSD offers excellent online protections against unauthorized data access. + File permissions and crossref:mac[mac,Mandatory Access Control] (MAC) help prevent unauthorized users from accessing data while the operating system is active and the computer is powered up. + However, the permissions enforced by the operating system are irrelevant if an attacker has physical access to a computer and can move the computer's hard drive to another system to copy and analyze the data. + + Regardless of how an attacker may have come into possession of a hard drive or powered-down computer, the GEOM-based cryptographic subsystems built into FreeBSD are able to protect the data on the computer's file systems against even highly-motivated attackers with significant resources. + Unlike encryption methods that encrypt individual files, the built-in `gbde` and `geli` utilities can be used to transparently encrypt entire file systems. + No cleartext ever touches the hard drive's platter. + + This chapter demonstrates how to create an encrypted file system on FreeBSD. + It first demonstrates the process using `gbde` and then demonstrates the same example using `geli`. + + === Disk Encryption with gbde + + The objective of the man:gbde[4] facility is to provide a formidable challenge for an attacker to gain access to the contents of a _cold_ storage device. + However, if the computer is compromised while up and running and the storage device is actively attached, or the attacker has access to a valid passphrase, it offers no protection to the contents of the storage device. + Thus, it is important to provide physical security while the system is running and to protect the passphrase used by the encryption mechanism. + + This facility provides several barriers to protect the data stored in each disk sector. + It encrypts the contents of a disk sector using 128-bit AES in CBC mode. + Each sector on the disk is encrypted with a different AES key. + For more information on the cryptographic design, including how the sector keys are derived from the user-supplied passphrase, refer to man:gbde[4]. + + FreeBSD provides a kernel module for gbde which can be loaded with this command: + + [source,shell] + .... + # kldload geom_bde + .... + + If using a custom kernel configuration file, ensure it contains this line: + + `options GEOM_BDE` + + The following example demonstrates adding a new hard drive to a system that will hold a single encrypted partition that will be mounted as [.filename]#/private#. + + [.procedure] + .Procedure: Encrypting a Partition with gbde + . Add the New Hard Drive + + + Install the new drive to the system as explained in crossref:disks[disks-adding, Adding Disks]. + For the purposes of this example, a new hard drive partition has been added as [.filename]#/dev/ad4s1c# and [.filename]#/dev/ad0s1*# represents the existing standard FreeBSD partitions. + + + [source,shell] + .... + # ls /dev/ad* + /dev/ad0 /dev/ad0s1b /dev/ad0s1e /dev/ad4s1 + /dev/ad0s1 /dev/ad0s1c /dev/ad0s1f /dev/ad4s1c + /dev/ad0s1a /dev/ad0s1d /dev/ad4 + .... + + . Create a Directory to Hold `gbde` Lock Files + + + [source,shell] + .... + # mkdir /etc/gbde + .... + + + The gbde lock file contains information that gbde requires to access encrypted partitions. + Without access to the lock file, gbde will not be able to decrypt the data contained in the encrypted partition without significant manual intervention which is not supported by the software. + Each encrypted partition uses a separate lock file. + . Initialize the `gbde` Partition + + + A gbde partition must be initialized before it can be used. + This initialization needs to be performed only once. + This command will open the default editor, in order to set various configuration options in a template. + For use with the UFS file system, set the sector_size to 2048: + + + [source,shell] + .... + # gbde init /dev/ad4s1c -i -L /etc/gbde/ad4s1c.lock + # + # Sector size is the smallest unit of data which can be read or written. + # Making it too small decreases performance and decreases available space. + # Making it too large may prevent filesystems from working. 512 is the + # minimum and always safe. For UFS, use the fragment size + # + sector_size = 2048 + [...] + .... + + + Once the edit is saved, the user will be asked twice to type the passphrase used to secure the data. + The passphrase must be the same both times. + The ability of gbde to protect data depends entirely on the quality of the passphrase. + For tips on how to select a secure passphrase that is easy to remember, see http://world.std.com/\~reinhold/diceware.html[http://world.std.com/~reinhold/diceware.htm]. + + + This initialization creates a lock file for the gbde partition. + In this example, it is stored as [.filename]#/etc/gbde/ad4s1c.lock#. + Lock files must end in ".lock" in order to be correctly detected by the [.filename]#/etc/rc.d/gbde# start up script. + + + [CAUTION] + ==== + Lock files _must_ be backed up together with the contents of any encrypted partitions. + Without the lock file, the legitimate owner will be unable to access the data on the encrypted partition. + ==== + + . Attach the Encrypted Partition to the Kernel + + + [source,shell] + .... + # gbde attach /dev/ad4s1c -l /etc/gbde/ad4s1c.lock + .... + + + This command will prompt to input the passphrase that was selected during the initialization of the encrypted partition. + The new encrypted device will appear in [.filename]#/dev# as [.filename]#/dev/device_name.bde#: + + + [source,shell] + .... + # ls /dev/ad* + /dev/ad0 /dev/ad0s1b /dev/ad0s1e /dev/ad4s1 + /dev/ad0s1 /dev/ad0s1c /dev/ad0s1f /dev/ad4s1c + /dev/ad0s1a /dev/ad0s1d /dev/ad4 /dev/ad4s1c.bde + .... + + . Create a File System on the Encrypted Device + + + Once the encrypted device has been attached to the kernel, a file system can be created on the device. + This example creates a UFS file system with soft updates enabled. + Be sure to specify the partition which has a [.filename]#*.bde# extension: + + + [source,shell] + .... + # newfs -U /dev/ad4s1c.bde + .... + + . Mount the Encrypted Partition + + + Create a mount point and mount the encrypted file system: + + + [source,shell] + .... + # mkdir /private + # mount /dev/ad4s1c.bde /private + .... + + . Verify That the Encrypted File System is Available + + + The encrypted file system should now be visible and available for use: + + + [source,shell] + .... + % df -H + Filesystem Size Used Avail Capacity Mounted on + /dev/ad0s1a 1037M 72M 883M 8% / + /devfs 1.0K 1.0K 0B 100% /dev + /dev/ad0s1f 8.1G 55K 7.5G 0% /home + /dev/ad0s1e 1037M 1.1M 953M 0% /tmp + /dev/ad0s1d 6.1G 1.9G 3.7G 35% /usr + /dev/ad4s1c.bde 150G 4.1K 138G 0% /private + .... + + After each boot, any encrypted file systems must be manually re-attached to the kernel, checked for errors, and mounted, before the file systems can be used. + To configure these steps, add the following lines to [.filename]#/etc/rc.conf#: + + [.programlisting] + .... + gbde_autoattach_all="YES" + gbde_devices="ad4s1c" + gbde_lockdir="/etc/gbde" + .... + + This requires that the passphrase be entered at the console at boot time. + After typing the correct passphrase, the encrypted partition will be mounted automatically. + Additional gbde boot options are available and listed in man:rc.conf[5]. + + [NOTE] + ==== + sysinstall is incompatible with gbde-encrypted devices. + All [.filename]#*.bde# devices must be detached from the kernel before starting sysinstall or it will crash during its initial probing for devices. + To detach the encrypted device used in the example, use the following command: + + [source,shell] + .... + # gbde detach /dev/ad4s1c + .... + ==== + + [[disks-encrypting-geli]] + === Disk Encryption with `geli` + + An alternative cryptographic GEOM class is available using `geli`. + This control utility adds some features and uses a different scheme for doing cryptographic work. + It provides the following features: + + * Utilizes the man:crypto[9] framework and automatically uses cryptographic hardware when it is available. + * Supports multiple cryptographic algorithms such as AES-XTS, AES-CBC, and Camellia-CBCAES. + * Allows the root partition to be encrypted. The passphrase used to access the encrypted root partition will be requested during system boot. + * Allows the use of two independent keys. + * It is fast as it performs simple sector-to-sector encryption. + * Allows backup and restore of master keys. If a user destroys their keys, it is still possible to get access to the data by restoring keys from the backup. + * Allows a disk to attach with a random, one-time key which is useful for swap partitions and temporary file systems. + + More features and usage examples can be found in man:geli[8]. + + The following example describes how to generate a key file which will be used as part of the master key for the encrypted provider mounted under [.filename]#/private#. + The key file will provide some random data used to encrypt the master key. + The master key will also be protected by a passphrase. + The provider's sector size will be 4kB. + The example describes how to attach to the `geli` provider, create a file system on it, mount it, work with it, and finally, how to detach it. + + [.procedure] + .Procedure: Encrypting a Partition with `geli` + . Load `geli` Support + + + Support for `geli` is available as a loadable kernel module. To configure the system to automatically load the module at boot time, add the following line to [.filename]#/boot/loader.conf#: + + + [.programlisting] + .... + geom_eli_load="YES" + .... + + + To load the kernel module now: + + + [source,shell] + .... + # kldload geom_eli + .... + + + For a custom kernel, ensure the kernel configuration file contains these lines: + + + [.programlisting] + .... + options GEOM_ELI + device crypto + .... + + . Generate the Master Key + + + The following commands generate a master key that all data will be encrypted with. + This key can never be changed. + Rather than using it directly, it is encrypted with one or more user keys. + The user keys are made up of an optional combination of random bytes from a file, [.filename]#/root/da2.key#, and/or a passphrase. + In this case, the data source for the key file is [.filename]#/dev/random#. + This command also configures the sector size of the provider ([.filename]#/dev/da2.eli#) as 4kB, for better performance: + + + [source,shell] + .... + # dd if=/dev/random of=/root/da2.key bs=64 count=1 + # geli init -K /root/da2.key -s 4096 /dev/da2 + Enter new passphrase: + Reenter new passphrase: + .... + + + It is not mandatory to use both a passphrase and a key file as either method of securing the master key can be used in isolation. + + + If the key file is given as "-", standard input will be used. + For example, this command generates three key files: + + + [source,shell] + .... + # cat keyfile1 keyfile2 keyfile3 | geli init -K - /dev/da2 + .... + + . Attach the Provider with the Generated Key + + + To attach the provider, specify the key file, the name of the disk, and the passphrase: + + + [source,shell] + .... + # geli attach -k /root/da2.key /dev/da2 + Enter passphrase: + .... + + + This creates a new device with an [.filename]#.eli# extension: + + + [source,shell] + .... + # ls /dev/da2* + /dev/da2 /dev/da2.eli + .... + + . Create the New File System + + + Next, format the device with the UFS file system and mount it on an existing mount point: + + + [source,shell] + .... + # dd if=/dev/random of=/dev/da2.eli bs=1m + # newfs /dev/da2.eli + # mount /dev/da2.eli /private + .... + + + The encrypted file system should now be available for use: + + + [source,shell] + .... + # df -H + Filesystem Size Used Avail Capacity Mounted on + /dev/ad0s1a 248M 89M 139M 38% / + /devfs 1.0K 1.0K 0B 100% /dev + /dev/ad0s1f 7.7G 2.3G 4.9G 32% /usr + /dev/ad0s1d 989M 1.5M 909M 0% /tmp + /dev/ad0s1e 3.9G 1.3G 2.3G 35% /var + /dev/da2.eli 150G 4.1K 138G 0% /private + .... + + Once the work on the encrypted partition is done, and the [.filename]#/private# partition is no longer needed, it is prudent to put the device into cold storage by unmounting and detaching the `geli` encrypted partition from the kernel: + + [source,shell] + .... + # umount /private + # geli detach da2.eli + .... + + An [.filename]#rc.d# script is provided to simplify the mounting of `geli`-encrypted devices at boot time. For this example, add these lines to [.filename]#/etc/rc.conf#: + + [.programlisting] + .... + geli_devices="da2" + geli_da2_flags="-k /root/da2.key" + .... + + This configures [.filename]#/dev/da2# as a `geli` provider with a master key of [.filename]#/root/da2.key#. + The system will automatically detach the provider from the kernel before the system shuts down. + During the startup process, the script will prompt for the passphrase before attaching the provider. + Other kernel messages might be shown before and after the password prompt. + If the boot process seems to stall, look carefully for the password prompt among the other messages. + Once the correct passphrase is entered, the provider is attached. + The file system is then mounted, typically by an entry in [.filename]#/etc/fstab#. + Refer to crossref:basics[mount-unmount,“Mounting and Unmounting File Systems”] for instructions on how to configure a file system to mount at boot time. + + [[swap-encrypting]] + == Encrypting Swap + + Like the encryption of disk partitions, encryption of swap space is used to protect sensitive information. + Consider an application that deals with passwords. + As long as these passwords stay in physical memory, they are not written to disk and will be cleared after a reboot. + However, if FreeBSD starts swapping out memory pages to free space, the passwords may be written to the disk unencrypted. + Encrypting swap space can be a solution for this scenario. + + This section demonstrates how to configure an encrypted swap partition using man:gbde[8] or man:geli[8] encryption. + It assumes that [.filename]#/dev/ada0s1b# is the swap partition. + + === Configuring Encrypted Swap + + Swap partitions are not encrypted by default and should be cleared of any sensitive data before continuing. + To overwrite the current swap partition with random garbage, execute the following command: + + [source,shell] + .... + # dd if=/dev/random of=/dev/ada0s1b bs=1m + .... + + To encrypt the swap partition using man:gbde[8], add the `.bde` suffix to the swap line in [.filename]#/etc/fstab#: + + [.programlisting] + .... + # Device Mountpoint FStype Options Dump Pass# + /dev/ada0s1b.bde none swap sw 0 0 + .... + + To instead encrypt the swap partition using man:geli[8], use the `.eli` suffix: + + [.programlisting] + .... + # Device Mountpoint FStype Options Dump Pass# + /dev/ada0s1b.eli none swap sw 0 0 + .... + + By default, man:geli[8] uses the AES algorithm with a key length of 128 bits. + Normally the default settings will suffice. + If desired, these defaults can be altered in the options field in [.filename]#/etc/fstab#. The possible flags are: + + aalgo:: + Data integrity verification algorithm used to ensure that the encrypted data has not been tampered with. + See man:geli[8] for a list of supported algorithms. + + ealgo:: + Encryption algorithm used to protect the data. + See man:geli[8] for a list of supported algorithms. + + keylen:: + The length of the key used for the encryption algorithm. + See man:geli[8] for the key lengths that are supported by each encryption algorithm. + + sectorsize:: + The size of the blocks data is broken into before it is encrypted. + Larger sector sizes increase performance at the cost of higher storage overhead. + The recommended size is 4096 bytes. + + This example configures an encrypted swap partition using the AES-XTS algorithm with a key length of 128 bits and a sectorsize of 4 kilobytes: + + [.programlisting] + .... + # Device Mountpoint FStype Options Dump Pass# + /dev/ada0s1b.eli none swap sw,ealgo=AES-XTS,keylen=128,sectorsize=4096 0 0 + .... + + === Encrypted Swap Verification + + Once the system has rebooted, proper operation of the encrypted swap can be verified using `swapinfo`. + + If man:gbde[8] is being used: + + [source,shell] + .... + % swapinfo + Device 1K-blocks Used Avail Capacity + /dev/ada0s1b.bde 542720 0 542720 0 + .... + + If man:geli[8] is being used: + + [source,shell] + .... + % swapinfo + Device 1K-blocks Used Avail Capacity + /dev/ada0s1b.eli 542720 0 542720 0 + .... + + [[disks-hast]] + == Highly Available Storage (HAST) + + High availability is one of the main requirements in serious business applications and highly-available storage is a key component in such environments. + In FreeBSD, the Highly Available STorage (HAST) framework allows transparent storage of the same data across several physically separated machines connected by a TCP/IP network. + HAST can be understood as a network-based RAID1 (mirror), and is similar to the DRBD(R) storage system used in the GNU/Linux(R) platform. + In combination with other high-availability features of FreeBSD like CARP, HAST makes it possible to build a highly-available storage cluster that is resistant to hardware failures. + + The following are the main features of HAST: + + * Can be used to mask I/O errors on local hard drives. + * File system agnostic as it works with any file system supported by FreeBSD. + * Efficient and quick resynchronization as only the blocks that were modified during the downtime of a node are synchronized. + * Can be used in an already deployed environment to add additional redundancy. + * Together with CARP, Heartbeat, or other tools, it can be used to build a robust and durable storage system. + + Read this chapter to learn: + + * What HAST is, how it works, and which features it provides. + * How to set up and use HAST on FreeBSD. + * How to integrate CARP and man:devd[8] to build a robust storage system. + + Before reading this section: + + * Understand UNIX(R) and FreeBSD basics (crossref:basics[basics,FreeBSD Basics]). + * Know how to configure network interfaces and other core FreeBSD subsystems (crossref:config[config-tuning,Configuration and Tuning]). + * Have a good understanding of FreeBSD networking (crossref:partiv[network-communication,"Network Communication"]). + + The HAST project was sponsored by The FreeBSD Foundation with support from http://www.omc.net/[http://www.omc.net/] and http://www.transip.nl/[http://www.transip.nl/]. + + === HAST Operation + + HAST provides synchronous block-level replication between two physical machines: the _primary_ node and the _secondary_ node. + These two machines together are referred to as a cluster. + + Since HAST works in a primary-secondary configuration, it allows only one of the cluster nodes to be active at any given time. + The primary node, also called _active_, is the one which will handle all the I/O requests to HAST-managed devices. + The secondary node is automatically synchronized from the primary node. + + The physical components of the HAST system are the local disk on primary node, and the disk on the remote, secondary node. + + HAST operates synchronously on a block level, making it transparent to file systems and applications. + HAST provides regular GEOM providers in [.filename]#/dev/hast/# for use by other tools or applications. + There is no difference between using HAST-provided devices and raw disks or partitions. + + Each write, delete, or flush operation is sent to both the local disk and to the remote disk over TCP/IP. + Each read operation is served from the local disk, unless the local disk is not up-to-date or an I/O error occurs. + In such cases, the read operation is sent to the secondary node. + + HAST tries to provide fast failure recovery. + For this reason, it is important to reduce synchronization time after a node's outage. + To provide fast synchronization, HAST manages an on-disk bitmap of dirty extents and only synchronizes those during a regular synchronization, with an exception of the initial sync. + + There are many ways to handle synchronization. + HAST implements several replication modes to handle different synchronization methods: + + * _memsync_: This mode reports a write operation as completed when the local write operation is finished and when the remote node acknowledges data arrival, but before actually storing the data. The data on the remote node will be stored directly after sending the acknowledgement. This mode is intended to reduce latency, but still provides good reliability. This mode is the default. + * _fullsync_: This mode reports a write operation as completed when both the local write and the remote write complete. This is the safest and the slowest replication mode. + * _async_: This mode reports a write operation as completed when the local write completes. This is the fastest and the most dangerous replication mode. It should only be used when replicating to a distant node where latency is too high for other modes. + + === HAST Configuration + + The HAST framework consists of several components: + + * The man:hastd[8] daemon which provides data synchronization. When this daemon is started, it will automatically load `geom_gate.ko`. + * The userland management utility, man:hastctl[8]. + * The man:hast.conf[5] configuration file. This file must exist before starting hastd. + + Users who prefer to statically build `GEOM_GATE` support into the kernel should add this line to the custom kernel configuration file, then rebuild the kernel using the instructions in crossref:kernelconfig[kernelconfig,Configuring the FreeBSD Kernel]: + + [.programlisting] + .... + options GEOM_GATE + .... + + The following example describes how to configure two nodes in primary-secondary operation using HAST to replicate the data between the two. + The nodes will be called `hasta`, with an IP address of `172.16.0.1`, and `hastb`, with an IP address of `172.16.0.2`. + Both nodes will have a dedicated hard drive [.filename]#/dev/ad6# of the same size for HAST operation. + The HAST pool, sometimes referred to as a resource or the GEOM provider in [.filename]#/dev/hast/#, will be called `test`. + + Configuration of HAST is done using [.filename]#/etc/hast.conf#. + This file should be identical on both nodes. + The simplest configuration is: + + [.programlisting] + .... + resource test { + on hasta { + local /dev/ad6 + remote 172.16.0.2 + } + on hastb { + local /dev/ad6 + remote 172.16.0.1 + } + } + .... + + For more advanced configuration, refer to man:hast.conf[5]. + + [TIP] + ==== + It is also possible to use host names in the `remote` statements if the hosts are resolvable and defined either in [.filename]#/etc/hosts# or in the local DNS. + ==== + + Once the configuration exists on both nodes, the HAST pool can be created. + Run these commands on both nodes to place the initial metadata onto the local disk and to start man:hastd[8]: + + [source,shell] + .... + # hastctl create test + # service hastd onestart + .... + + [NOTE] + ==== + It is _not_ possible to use GEOM providers with an existing file system or to convert an existing storage to a HAST-managed pool. + This procedure needs to store some metadata on the provider and there will not be enough required space available on an existing provider. + ==== + + A HAST node's `primary` or `secondary` role is selected by an administrator, or software like Heartbeat, using man:hastctl[8]. + On the primary node, `hasta`, issue this command: + + [source,shell] + .... + # hastctl role primary test + .... + + Run this command on the secondary node, `hastb`: + + [source,shell] + .... + # hastctl role secondary test + .... + + Verify the result by running `hastctl` on each node: + + [source,shell] + .... + # hastctl status test + .... + + Check the `status` line in the output. + If it says `degraded`, something is wrong with the configuration file. + It should say `complete` on each node, meaning that the synchronization between the nodes has started. + The synchronization completes when `hastctl status` reports 0 bytes of `dirty` extents. + + The next step is to create a file system on the GEOM provider and mount it. + This must be done on the `primary` node. + Creating the file system can take a few minutes, depending on the size of the hard drive. + This example creates a UFS file system on [.filename]#/dev/hast/test#: + + [source,shell] + .... + # newfs -U /dev/hast/test + # mkdir /hast/test + # mount /dev/hast/test /hast/test + .... + + Once the HAST framework is configured properly, the final step is to make sure that HAST is started automatically during system boot. + Add this line to [.filename]#/etc/rc.conf#: + + [.programlisting] + .... + hastd_enable="YES" + .... + + ==== Failover Configuration + + The goal of this example is to build a robust storage system which is resistant to the failure of any given node. + If the primary node fails, the secondary node is there to take over seamlessly, check and mount the file system, and continue to work without missing a single bit of data. + + To accomplish this task, the Common Address Redundancy Protocol (CARP) is used to provide for automatic failover at the IP layer. + CARP allows multiple hosts on the same network segment to share an IP address. + Set up CARP on both nodes of the cluster according to the documentation available in crossref:advanced-networking[carp,“Common Address Redundancy Protocol (CARP)”]. + In this example, each node will have its own management IP address and a shared IP address of _172.16.0.254_. + The primary HAST node of the cluster must be the primary CARP node. + + The HAST pool created in the previous section is now ready to be exported to the other hosts on the network. + This can be accomplished by exporting it through NFS or Samba, using the shared IP address _172.16.0.254_. + The only problem which remains unresolved is an automatic failover should the primary node fail. + + In the event of CARP interfaces going up or down, the FreeBSD operating system generates a man:devd[8] event, making it possible to watch for state changes on the CARP interfaces. + A state change on the CARP interface is an indication that one of the nodes failed or came back online. + These state change events make it possible to run a script which will automatically handle the HAST failover. + + To catch state changes on the CARP interfaces, add this configuration to [.filename]#/etc/devd.conf# on each node, while replacing `` with the virtual host id and `` with the associated interface name: + + [.programlisting] + .... + notify 30 { + match "system" "CARP"; + match "subsystem" "@"; + match "type" "MASTER"; + action "/usr/local/sbin/carp-hast-switch primary"; + }; + + notify 30 { + match "system" "CARP"; + match "subsystem" "@"; + match "type" "BACKUP"; + action "/usr/local/sbin/carp-hast-switch secondary"; + }; + .... + + Restart man:devd[8] on both nodes to put the new configuration into effect: + + [source,shell] + .... + # service devd restart + .... + + When the specified interface state changes by going up or down , the system generates a notification, allowing the man:devd[8] subsystem to run the specified automatic failover script, [.filename]#/usr/local/sbin/carp-hast-switch#. + For further clarification about this configuration, refer to man:devd.conf[5]. + + Here is an example of an automated failover script: + + [.programlisting] + .... + #!/bin/sh + + # Original script by Freddie Cash + # Modified by Michael W. Lucas + # and Viktor Petersson + + # The names of the HAST resources, as listed in /etc/hast.conf + resources="test" + + # delay in mounting HAST resource after becoming primary + # make your best guess + delay=3 + + # logging + log="local0.debug" + name="carp-hast" + + # end of user configurable stuff + + case "$1" in + primary) + logger -p $log -t $name "Switching to primary provider for ${resources}." + sleep ${delay} + + # Wait for any "hastd secondary" processes to stop + for disk in ${resources}; do + while $( pgrep -lf "hastd: ${disk} \(secondary\)" > /dev/null 2>&1 ); do + sleep 1 + done + + # Switch role for each disk + hastctl role primary ${disk} + if [ $? -ne 0 ]; then + logger -p $log -t $name "Unable to change role to primary for resource ${disk}." + exit 1 + fi + done + + # Wait for the /dev/hast/* devices to appear + for disk in ${resources}; do + for I in $( jot 60 ); do + [ -c "/dev/hast/${disk}" ] && break + sleep 0.5 + done + + if [ ! -c "/dev/hast/${disk}" ]; then + logger -p $log -t $name "GEOM provider /dev/hast/${disk} did not appear." + exit 1 + fi + done + + logger -p $log -t $name "Role for HAST resources ${resources} switched to primary." + + logger -p $log -t $name "Mounting disks." + for disk in ${resources}; do + mkdir -p /hast/${disk} + fsck -p -y -t ufs /dev/hast/${disk} + mount /dev/hast/${disk} /hast/${disk} + done + + ;; + + secondary) + logger -p $log -t $name "Switching to secondary provider for ${resources}." + + # Switch roles for the HAST resources + for disk in ${resources}; do + if ! mount | grep -q "^/dev/hast/${disk} on " + then + else + umount -f /hast/${disk} + fi + sleep $delay + hastctl role secondary ${disk} 2>&1 + if [ $? -ne 0 ]; then + logger -p $log -t $name "Unable to switch role to secondary for resource ${disk}." + exit 1 + fi + logger -p $log -t $name "Role switched to secondary for resource ${disk}." + done + ;; + esac + .... + + In a nutshell, the script takes these actions when a node becomes primary: + + * Promotes the HAST pool to primary on the other node. + * Checks the file system under the HAST pool. + * Mounts the pool. + + When a node becomes secondary: + + * Unmounts the HAST pool. + * Degrades the HAST pool to secondary. + + [CAUTION] + ==== + This is just an example script which serves as a proof of concept. + It does not handle all the possible scenarios and can be extended or altered in any way, for example, to start or stop required services. + ==== + + [TIP] + ==== + For this example, a standard UFS file system was used. + To reduce the time needed for recovery, a journal-enabled UFS or ZFS file system can be used instead. + ==== + + Instead of using the highly available storage locally, it can also be shared to other computers on a network via crossref:network-servers[network-nfs,NFS], crossref:network-servers[network-iscsi,iSCSI], man:sshfs[1], or programs in ports (i.e. package:net/samba419[]). + + More detailed information with additional examples can be found at http://wiki.FreeBSD.org/HAST[http://wiki.FreeBSD.org/HAST]. + + === Troubleshooting + + HAST should generally work without issues. + However, as with any other software product, there may be times when it does not work as supposed. + The sources of the problems may be different, but the rule of thumb is to ensure that the time is synchronized between the nodes of the cluster. + + When troubleshooting HAST, the debugging level of man:hastd[8] should be increased by starting `hastd` with `-d`. + This argument may be specified multiple times to further increase the debugging level. + Consider also using `-F`, which starts `hastd` in the foreground. + + [[disks-hast-sb]] + ==== Recovering from the Split-brain Condition + + _Split-brain_ occurs when the nodes of the cluster are unable to communicate with each other, and both are configured as primary. + This is a dangerous condition because it allows both nodes to make incompatible changes to the data. + This problem must be corrected manually by the system administrator. + + The administrator must either decide which node has more important changes, or perform the merge manually. + Then, let HAST perform full synchronization of the node which has the broken data. + To do this, issue these commands on the node which needs to be resynchronized: + + [source,shell] + .... + # hastctl role init test + # hastctl create test + # hastctl role secondary test + .... +diff --git a/documentation/content/en/books/handbook/introduction.adoc b/documentation/content/en/books/handbook/introduction.adoc +index e5cb8134a9..5cd3572d97 100644 +--- a/documentation/content/en/books/handbook/introduction.adoc ++++ b/documentation/content/en/books/handbook/introduction.adoc +@@ -1,12 +1,12 @@ + [.abstract-title] + Abstract + + Welcome to FreeBSD! This handbook covers the installation and day to day use of _FreeBSD {rel143-current}-RELEASE_ and _{rel135-current}-RELEASE_. + This book is the result of ongoing work by many individuals. + Some sections might be outdated. + Those interested in helping to update and expand this document should send email to the {freebsd-doc}. + + The latest version of this book is available from the https://www.FreeBSD.org/[FreeBSD web site]. + Previous versions can be obtained from https://docs.FreeBSD.org/doc/[https://docs.FreeBSD.org/doc/]. +-The book can be downloaded in a variety of formats and compression options from the https://download.freebsd.org/doc/[FreeBSD download server] or one of the numerous link:./mirrors#mirrors[mirror sites]. ++The book can be downloaded in a variety of formats and compression options from the https://download.freebsd.org/doc/[FreeBSD download server] or one of the numerous link:#mirrors[mirror sites]. + Searches can be performed on the handbook and other documents on the link:https://www.FreeBSD.org/search/[search page]. +diff --git a/documentation/content/en/books/porters-handbook/makefiles/_index.adoc b/documentation/content/en/books/porters-handbook/makefiles/_index.adoc +index e98d2ff336..47b95afc43 100644 +--- a/documentation/content/en/books/porters-handbook/makefiles/_index.adoc ++++ b/documentation/content/en/books/porters-handbook/makefiles/_index.adoc +@@ -1,5448 +1,5448 @@ + --- + title: Chapter 5. Configuring the Makefile + prev: books/porters-handbook/slow-porting + next: books/porters-handbook/special + description: Configuring the Makefile for FreeBSD Ports + tags: ["makefiles", "configuring", "naming", "versions"] + showBookMenu: true + weight: 5 + params: + path: "/books/porters-handbook/makefiles/" + --- + + [[makefiles]] + = Configuring the Makefile + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :sectnumoffset: 5 + :partnums: + :source-highlighter: rouge + :experimental: + :g-plus-plus: g++ + :images-path: books/porters-handbook/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + :imagesdir: ../../../../images/{images-path} + endif::[] + ifndef::book[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + toc::[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + toc::[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + Configuring the [.filename]#Makefile# is pretty simple, and again we suggest looking at existing examples before starting. + Also, there is a crossref:porting-samplem[porting-samplem,sample Makefile] in this handbook, + so take a look and please follow the ordering of variables and sections in that template to make the port easier for others to read. + + Consider these problems in sequence during the design of the new [.filename]#Makefile#: + + [[makefile-source]] + == The Original Source + + Does it live in `DISTDIR` as a standard ``gzip``ped tarball named something like [.filename]#foozolix-1.2.tar.gz#? If so, go on to the next step. + If not, the distribution file format might require overriding one or more of `DISTVERSION`, `DISTNAME`, `EXTRACT_CMD`, `EXTRACT_BEFORE_ARGS`, `EXTRACT_AFTER_ARGS`, `EXTRACT_SUFX`, or `DISTFILES`. + + In the worst case, create a custom `do-extract` target to override the default. + This is rarely, if ever, necessary. + + [[makefile-naming]] + == Naming + + The first part of the port's [.filename]#Makefile# names the port, describes its version number, and lists it in the correct category. + + [[makefile-portname]] + === `PORTNAME` + + Set `PORTNAME` to the base name of the software. + It is used as the base for the FreeBSD package, and for crossref:makefiles[makefile-distname,`DISTNAME`]. + + [IMPORTANT] + ==== + The package name must be unique across the entire ports tree. + Make sure that the `PORTNAME` is not already in use by an existing port, and that no other port already has the same `PKGBASE`. + If the name has already been used, add either + crossref:makefiles[porting-pkgnameprefix-suffix,`PKGNAMEPREFIX` or `PKGNAMESUFFIX`]. + ==== + + [[makefile-versions]] + === Versions, `DISTVERSION` _or_ `PORTVERSION` + + Set `DISTVERSION` to the version number of the software. + + `PORTVERSION` is the version used for the FreeBSD package. + It will be automatically derived from `DISTVERSION` to be compatible with FreeBSD's package versioning scheme. + If the version contains _letters_, it might be needed to set `PORTVERSION` and not `DISTVERSION`. + + [IMPORTANT] + ==== + Only one of `PORTVERSION` and `DISTVERSION` can be set at a time. + ==== + + From time to time, some software will use a version scheme that is not compatible with how `DISTVERSION` translates in `PORTVERSION`. + + [TIP] + ==== + When updating a port, it is possible to use the `-t` argument of man:pkg-version[8] to check if the new version is greater or lesser than before. + See below on how to use man:pkg-version[8] to compare versions. + ==== + + [[makefile-versions-ex-pkg-version]] + .Using man:pkg-version[8] to Compare Versions + [example] + ==== + `pkg version -t` takes two versions as arguments, it will respond with `<`, `=` or `>` if the first version is less, equal, or more than the second version, respectively. + + [source,shell] + .... + % pkg version -t 1.2 1.3 + < <.> + % pkg version -t 1.2 1.2 + = <.> + % pkg version -t 1.2 1.2.0 + = <.> + % pkg version -t 1.2 1.2.p1 + > <.> + % pkg version -t 1.2.a1 1.2.b1 + < <.> + % pkg version -t 1.2 1.2p1 + < <.> + .... + + <.> `1.2` is before `1.3`. + <.> `1.2` and `1.2` are equal as they have the same version. + <.> `1.2` and `1.2.0` are equal as nothing equals zero. + <.> `1.2` is after `1.2.p1` as `.p1`, think "pre-release 1". + <.> `1.2.a1` is before `1.2.b1`, think "alpha" and "beta", and `a` is before `b`. + <.> `1.2` is before `1.2p1` as `2p1`, think "2, patch level 1" which is a version after any `2.X` but before `3`. + + [NOTE] + **** + In here, the `a`, `b`, and `p` are used as if meaning "alpha", "beta" or "pre-release" and "patch level", + but they are only letters and are sorted alphabetically, so any letter can be used, and they will be sorted appropriately. + **** + + ==== + + .Examples of `DISTVERSION` and the Derived `PORTVERSION` + [cols="10%,90%", frame="none", options="header"] + |=== + | DISTVERSION + | PORTVERSION + + |0.7.1d + |0.7.1.d + + |10Alpha3 + |10.a3 + + |3Beta7-pre2 + |3.b7.p2 + + |8:f_17 + |8f.17 + |=== + + [[makefile-versions-ex1]] + .Using `DISTVERSION` + [example] + ==== + When the version only contains numbers separated by dots, dashes or underscores, use `DISTVERSION`. + + [.programlisting] + .... + PORTNAME= nekoto + DISTVERSION= 1.2-4 + .... + + It will generate a `PORTVERSION` of `1.2.4`. + ==== + + [[makefile-versions-ex2]] + .Using `DISTVERSION` When the Version Starts with a Letter or a Prefix + [example] + ==== + When the version starts or ends with a letter, or a prefix or a suffix that is not part of the version, use `DISTVERSIONPREFIX`, `DISTVERSION`, and `DISTVERSIONSUFFIX`. + + If the version is `v1.2-4`: + + [.programlisting] + .... + PORTNAME= nekoto + DISTVERSIONPREFIX= v + DISTVERSION= 1_2_4 + .... + + Some of the time, projects using GitHub will use their name in their versions. + For example, the version could be `nekoto-1.2-4`: + + [.programlisting] + .... + PORTNAME= nekoto + DISTVERSIONPREFIX= nekoto- + DISTVERSION= 1.2_4 + .... + + Those projects also sometimes use some string at the end of the version, for example, `1.2-4_RELEASE`: + + [.programlisting] + .... + PORTNAME= nekoto + DISTVERSION= 1.2-4 + DISTVERSIONSUFFIX= _RELEASE + .... + + Or they do both, for example, `nekoto-1.2-4_RELEASE`: + + [.programlisting] + .... + PORTNAME= nekoto + DISTVERSIONPREFIX= nekoto- + DISTVERSION= 1.2-4 + DISTVERSIONSUFFIX= _RELEASE + .... + + `DISTVERSIONPREFIX` and `DISTVERSIONSUFFIX` will not be used while constructing `PORTVERSION`, but only used in `DISTNAME`. + + All will generate a `PORTVERSION` of `1.2.4`. + ==== + + [[makefile-versions-ex3]] + .Using `DISTVERSION` When the Version Contains Letters Meaning "alpha", "beta", or "pre-release" + [example] + ==== + When the version contains numbers separated by dots, dashes or underscores, and letters are used to mean "alpha", "beta" or "pre-release", which is, before the version without the letters, use `DISTVERSION`. + + [.programlisting] + .... + PORTNAME= nekoto + DISTVERSION= 1.2-pre4 + .... + + [.programlisting] + .... + PORTNAME= nekoto + DISTVERSION= 1.2p4 + .... + + Both will generate a `PORTVERSION` of `1.2.p4` which is before than 1.2. man:pkg-version[8] can be used to check that fact: + + [source,shell] + .... + % pkg version -t 1.2.p4 1.2 + < + .... + + ==== + + [[makefile-versions-ex4]] + .Not Using `DISTVERSION` When the Version Contains Letters Meaning "Patch Level" + [example] + ==== + When the version contains letters that are not meant as "alpha", "beta", or "pre", but more in a "patch level", and meaning after the version without the letters, use `PORTVERSION`. + + [.programlisting] + .... + PORTNAME= nekoto + PORTVERSION= 1.2p4 + .... + + In this case, using `DISTVERSION` is not possible because it would generate a version of `1.2.p4` which would be before `1.2` and not after. + man:pkg-version[8] will verify this: + + [source,shell] + .... + % pkg version -t 1.2 1.2.p4 + > <.> + % pkg version -t 1.2 1.2p4 + < <.> + .... + + <.> `1.2` is after `1.2.p4`, which is _wrong_ in this case. + <.> `1.2` is before `1.2p4`, which is what was needed. + ==== + + For some more advanced examples of setting `PORTVERSION`, when the software's + versioning is really not compatible with FreeBSD's, or `DISTNAME` when the + distribution file does not contain the version itself, see + crossref:makefiles[makefile-distname, `DISTNAME`]. + + [[makefile-naming-revepoch]] + === `PORTREVISION` and `PORTEPOCH` + + [[makefile-portrevision]] + ==== `PORTREVISION` + + `PORTREVISION` is a monotonically increasing value which is reset to 0 with every increase of `DISTVERSION`, typically every time there is a new official vendor release. If `PORTREVISION` is non-zero, the value is appended to the package name. + Changes to `PORTREVISION` are used by automated tools like man:pkg-version[8] to determine that a new package is available. + + `PORTREVISION` must be increased each time a change is made to the port that changes the generated package in any way. + That includes changes that only affect a package built with non-default + crossref:makefiles[makefile-options,options]. + + Examples of when `PORTREVISION` must be bumped: + + * Addition of patches to correct security vulnerabilities, bugs, or to add new functionality to the port. + * Changes to the port [.filename]#Makefile# to enable or disable compile-time options in the package. + * Changes in the packing list or the install-time behavior of the package. For example, a change to a script which generates initial data for the package, like man:ssh[1] host keys. + * Version bump of a port's shared library dependency (in this case, someone trying to install the old package after installing a newer version of the dependency will fail since it will look for the old libfoo.x instead of libfoo.(x+1)). + * Silent changes to the port distfile which have significant functional differences. For example, changes to the distfile requiring a correction to [.filename]#distinfo# with no corresponding change to `DISTVERSION`, where a `diff -ru` of the old and new versions shows non-trivial changes to the code. + * Changes to `MAINTAINER`. + + Examples of changes which do not require a `PORTREVISION` bump: + + * Style changes to the port skeleton with no functional change to what appears in the resulting package. + * Changes to `MASTER_SITES` or other functional changes to the port which do not affect the resulting package. + * Trivial patches to the distfile such as correction of typos, which are not important enough that users of the package have to go to the trouble of upgrading. + * Build fixes which cause a package to become compilable where it was previously failing. As long as the changes do not introduce any functional change on any other platforms on which the port did previously build. Since `PORTREVISION` reflects the content of the package, if the package was not previously buildable then there is no need to increase `PORTREVISION` to mark a change. + + A rule of thumb is to decide whether a change committed to a port is something which _some_ people would benefit from having. + Either because of an enhancement, fix, or by virtue that the new package will actually work at all. + Then weigh that against that fact that it will cause everyone who regularly updates their ports tree to be compelled to update. + If yes, `PORTREVISION` must be bumped. + + [NOTE] + ==== + People using binary packages will _never_ see the update if `PORTREVISION` is not bumped. + Without increasing `PORTREVISION`, the package builders have no way to detect the change and thus, will not rebuild the package. + ==== + + [[makefile-portepoch]] + ==== `PORTEPOCH` + + From time to time a software vendor or FreeBSD porter will do something silly and release a version of their software which is actually numerically less than the previous version. + An example of this is a port which goes from foo-20000801 to foo-1.0 (the former will be incorrectly treated as a newer version since 20000801 is a numerically greater value than 1). + + [TIP] + ==== + The results of version number comparisons are not always obvious. + `pkg version` (see man:pkg-version[8]) can be used to test the comparison of two version number strings. + For example: + + [source,shell] + .... + % pkg version -t 0.031 0.29 + > + .... + + The `>` output indicates that version 0.031 is considered greater than version 0.29, which may not have been obvious to the porter. + ==== + + In situations such as this, `PORTEPOCH` must be increased. + If `PORTEPOCH` is nonzero it is appended to the package name as described in section 0 above. + `PORTEPOCH` must never be decreased or reset to zero, because that would cause comparison to a package from an earlier epoch to fail. + For example, the package would not be detected as out of date. + The new version number, `1.0,1` in the above example, is still numerically less than the previous version, 20000801, but the `,1` suffix is treated specially by automated tools and found to be greater than the implied suffix `,0` on the earlier package. + + Dropping or resetting `PORTEPOCH` incorrectly leads to no end of grief. + If the discussion above was not clear enough, please consult the {freebsd-ports}. + + It is expected that `PORTEPOCH` will not be used for the majority of ports, and that sensible use of `DISTVERSION`, or that use `PORTVERSION` carefully, can often preempt it becoming necessary if a future release of the software changes the version structure. + However, care is needed by FreeBSD porters when a vendor release is made without an official version number - such as a code "snapshot" release. + The temptation is to label the release with the release date, which will cause problems as in the example above when a new "official" release is made. + + For example, if a snapshot release is made on the date `20000917`, and the previous version of the software was version `1.2`, do not use `20000917` for `DISTVERSION`. + The correct way is a `DISTVERSION` of `1.2.20000917`, or similar, so that the succeeding release, say `1.3`, is still a numerically greater value. + + [[makefile-portrevision-example]] + ==== Example of `PORTREVISION` and `PORTEPOCH` Usage + + The `gtkmumble` port, version `0.10`, is committed to the ports collection: + + [.programlisting] + .... + PORTNAME= gtkmumble + DISTVERSION= 0.10 + .... + + `PKGNAME` becomes `gtkmumble-0.10`. + + A security hole is discovered which requires a local FreeBSD patch. + `PORTREVISION` is bumped accordingly. + + [.programlisting] + .... + PORTNAME= gtkmumble + DISTVERSION= 0.10 + PORTREVISION= 1 + .... + + `PKGNAME` becomes `gtkmumble-0.10_1` + + A new version is released by the vendor, numbered `0.2` (it turns out the author actually intended `0.10` to actually mean `0.1.0`, not "what comes after 0.9" - oops, too late now). + Since the new minor version `2` is numerically less than the previous version `10`, `PORTEPOCH` must be bumped to manually force the new package to be detected as "newer". + Since it is a new vendor release of the code, `PORTREVISION` is reset to 0 (or removed from the [.filename]#Makefile#). + + [.programlisting] + .... + PORTNAME= gtkmumble + DISTVERSION= 0.2 + PORTEPOCH= 1 + .... + + `PKGNAME` becomes `gtkmumble-0.2,1` + + The next release is 0.3. + Since `PORTEPOCH` never decreases, the version variables are now: + + [.programlisting] + .... + PORTNAME= gtkmumble + DISTVERSION= 0.3 + PORTEPOCH= 1 + .... + + `PKGNAME` becomes `gtkmumble-0.3,1` + + [NOTE] + ==== + If `PORTEPOCH` were reset to `0` with this upgrade, someone who had installed the `gtkmumble-0.10_1` package would not detect the `gtkmumble-0.3` package as newer, since `3` is still numerically less than `10`. + Remember, this is the whole point of `PORTEPOCH` in the first place. + ==== + + [[porting-pkgnameprefix-suffix]] + === `PKGNAMEPREFIX` and `PKGNAMESUFFIX` + + Two optional variables, `PKGNAMEPREFIX` and `PKGNAMESUFFIX`, are combined with `PORTNAME` and `PORTVERSION` to form `PKGNAME` as `${PKGNAMEPREFIX}${PORTNAME}${PKGNAMESUFFIX}-${PORTVERSION}`. + Make sure this conforms to our crossref:makefiles[porting-pkgname,guidelines for a good package name]. + In particular, the use of a hyphen (`-`) in `PORTVERSION` is _not_ allowed. + Also, if the package name has the _language-_ or the _-compiled.specifics_ part (see below), use `PKGNAMEPREFIX` and `PKGNAMESUFFIX`, respectively. + Do not make them part of `PORTNAME`. + + [[porting-pkgname]] + === Package Naming Conventions + + These are the conventions to follow when naming packages. + This is to make the package directory easy to scan, as there are already thousands of packages and users are going to turn away if they hurt their eyes! + + Package names take the form of [.filename]#language_region-name-compiled.specifics-version.numbers#. + + The package name is defined as `${PKGNAMEPREFIX}${PORTNAME}${PKGNAMESUFFIX}-${PORTVERSION}`. + Make sure to set the variables to conform to that format. + + [[porting-pkgname-language]] + [.filename]#language_region-#:: + FreeBSD strives to support the native language of its users. + The _language-_ part is a two letter abbreviation of the natural language defined by ISO-639 when the port is specific to a certain language. + Examples are `ja` for Japanese, `ru` for Russian, `vi` for Vietnamese, `zh` for Chinese, `ko` for Korean and `de` for German. + + + If the port is specific to a certain region within the language area, add the two letter country code as well. + Examples are `en_US` for US English and `fr_CH` for Swiss French. + + + The _language-_ part is set in `PKGNAMEPREFIX`. + + [[porting-pkgname-name]] + [.filename]#name#:: + Make sure that the port's name and version are clearly separated and placed into `PORTNAME` and `DISTVERSION`. + The only reason for `PORTNAME` to contain a version part is if the upstream distribution is really named that way, as in the package:textproc/libxml2[] or package:japanese/kinput2-freewnn[] ports. + Otherwise, `PORTNAME` cannot contain any version-specific information. + It is quite normal for several ports to have the same `PORTNAME`, as the package:www/apache*[] ports do; in that case, different versions (and different index entries) are distinguished by `PKGNAMEPREFIX` and `PKGNAMESUFFIX` values. + + + There is a tradition of naming `Perl 5` modules by prepending `p5-` and converting the double-colon separator to a hyphen. + For example, the `Data::Dumper` module becomes `p5-Data-Dumper`. + [[porting-pkgname-compiled-specifics]] + [.filename]#-compiled.specifics#:: + If the port can be built with different + crossref:makefiles[makefile-masterdir,hardcoded defaults] (usually part of the directory name in a family of ports), the _-compiled.specifics_ part states the compiled-in defaults. + The hyphen is optional. + Examples are paper size and font units. + + + The _-compiled.specifics_ part is set in `PKGNAMESUFFIX`. + + [[porting-pkgname-version-numbers]] + [.filename]#-version.numbers#:: + The version string follows a dash (`-`) and is a period-separated list of integers and single lowercase alphabetics. + In particular, it is not permissible to have another dash inside the version string. + The only exception is the string `pl` (meaning "patchlevel"), which can be used _only_ when there are no major and minor version numbers in the software. + If the software version has strings like "alpha", "beta", "rc", or "pre", take the first letter and put it immediately after a period. + If the version string continues after those names, the numbers follow the single alphabet without an extra period between them (for example, `1.0b2`). + + + The idea is to make it easier to sort ports by looking at the version string. + In particular, make sure version number components are always delimited by a period, and if the date is part of the string, use the `d__yyyy.mm.dd__` format, not `_dd.mm.yyyy_` or the non-Y2K compliant `_yy.mm.dd_` format. + It is important to prefix the version with a letter, here `d` (for date), in case a release with an actual version number is made, which would be numerically less than `_yyyy_`. + + [IMPORTANT] + ==== + Package name must be unique among all of the ports tree, check that there is not + already a port with the same `PORTNAME` and if there is add one of + crossref:makefiles[porting-pkgnameprefix-suffix,`PKGNAMEPREFIX` or `PKGNAMESUFFIX`]. + ==== + + Here are some (real) examples on how to convert the name as called by the software authors to a suitable package name, for each line, only one of `DISTVERSION` or `PORTVERSION` is set in, depending on which would be used in the port's [.filename]#Makefile#: + + .Package Naming Examples + [cols="1,1,1,1,1,1,1", frame="none", options="header"] + |=== + | Distribution Name + | PKGNAMEPREFIX + | PORTNAME + | PKGNAMESUFFIX + | DISTVERSION + | PORTVERSION + | Reason or comment + + |mule-2.2.2 + |(empty) + |mule + |(empty) + |2.2.2 + | + |No changes required + + |mule-1.0.1 + |(empty) + |mule + |1 + |1.0.1 + | + |This is version 1 of mule, and version 2 already exists + + |EmiClock-1.0.2 + |(empty) + |emiclock + |(empty) + |1.0.2 + | + |No uppercase names for single programs + + |rdist-1.3alpha + |(empty) + |rdist + |(empty) + |1.3alpha + | + |Version will be `1.3.a` + + |es-0.9-beta1 + |(empty) + |es + |(empty) + |0.9-beta1 + | + |Version will be `0.9.b1` + + |mailman-2.0rc3 + |(empty) + |mailman + |(empty) + |2.0rc3 + | + |Version will be `2.0.r3` + + |v3.3beta021.src + |(empty) + |tiff + |(empty) + | + |3.3 + |What the heck was that anyway? + + |tvtwm + |(empty) + |tvtwm + |(empty) + | + |p11 + |No version in the filename, use what upstream says it is + + |piewm + |(empty) + |piewm + |(empty) + |1.0 + | + |No version in the filename, use what upstream says it is + + |xvgr-2.10pl1 + |(empty) + |xvgr + |(empty) + | + |2.10.pl1 + |In that case, `pl1` means patch level, so using DISTVERSION is not possible. + + |gawk-2.15.6 + |ja- + |gawk + |(empty) + |2.15.6 + | + |Japanese language version + + |psutils-1.13 + |(empty) + |psutils + |-letter + |1.13 + | + |Paper size hardcoded at package build time + + |pkfonts + |(empty) + |pkfonts + |300 + |1.0 + | + |Package for 300dpi fonts + |=== + + If there is absolutely no trace of version information in the original source and it is unlikely that the original author will ever release another version, just set the version string to `1.0` (like the `piewm` example above). + Otherwise, ask the original author or use the date string the source file was released on (`d__yyyy.mm.dd__`, or `d__yyyymmdd__`) as the version. + + [TIP] + ==== + Use any letter. + Here, `d` here stands for date, if the source is a Git repository, `g` followed by the commit date is commonly used, using `s` for snapshot is also common. + ==== + + [[makefile-categories]] + == Categorization + + [[makefile-categories-definition]] + === `CATEGORIES` + + When a package is created, it is put under [.filename]#/usr/ports/packages/All# and links are made from one or more subdirectories of [.filename]#/usr/ports/packages#. + The names of these subdirectories are specified by the variable `CATEGORIES`. + It is intended to make life easier for the user when he is wading through the pile of packages on the FTP site or the CDROM. + Please take a look at the crossref:makefiles[porting-categories,current list of categories] and pick the ones that are suitable for the port. + + This list also determines where in the ports tree the port is imported. + If there is more than one category here, the port files must be put in the subdirectory with the name of the first category. + See crossref:makefiles[choosing-categories,below] for more discussion about how to pick the right categories. + + [[porting-categories]] + === Current List of Categories + + Here is the current list of port categories. + Those marked with an asterisk (`*`) are _virtual_ categories-those that do not have a corresponding subdirectory in the ports tree. + They are only used as secondary categories, and only for search purposes. + + [NOTE] + ==== + For non-virtual categories, there is a one-line description in `COMMENT` in that subdirectory's [.filename]#Makefile#. + ==== + + [.informaltable] + [cols="1,1,1", frame="none", options="header"] + |=== + | Category + | Description + | Notes + + |[.filename]#accessibility# + |Ports to help disabled users. + | + + |[.filename]#afterstep#`*` + |Ports to support the http://www.afterstep.org/[AfterStep] window manager. + | + + |[.filename]#arabic# + |Arabic language support. + | + + |[.filename]#archivers# + |Archiving tools. + | + + |[.filename]#astro# + |Astronomical ports. + | + + |[.filename]#audio# + |Sound support. + | + + |[.filename]#benchmarks# + |Benchmarking utilities. + | + + |[.filename]#biology# + |Biology-related software. + | + + |[.filename]#cad# + |Computer aided design tools. + | + + |[.filename]#chinese# + |Chinese language support. + | + + |[.filename]#comms# + |Communication software. + |Mostly software to talk to the serial port. + + |[.filename]#converters# + |Character code converters. + | + + |[.filename]#databases# + |Databases. + | + + |[.filename]#deskutils# + |Things that used to be on the desktop before computers were invented. + | + + |[.filename]#devel# + |Development utilities. + |Do not put libraries here just because they are libraries. They should _not_ be in this category unless they truly do not belong anywhere else. + + |[.filename]#dns# + |DNS-related software. + | + + |[.filename]#docs#`*` + |Meta-ports for FreeBSD documentation. + | + + |[.filename]#editors# + |General editors. + |Specialized editors go in the section for those tools. For example, a mathematical-formula editor will go in [.filename]#math#, and have [.filename]#editors# as a second category. + + |[.filename]#education#`*` + |Education-related software. + |This includes applications, utilities, or games primarily or substantially designed to help the user learn a specific topic or study in general. It also includes course-writing applications, course-delivery applications, and classroom or school management applications + + |[.filename]#elisp#`*` + |Emacs-lisp ports. + | + + |[.filename]#emulators# + |Emulators for other operating systems. + |Terminal emulators do _not_ belong here. X-based ones go to [.filename]#x11# and text-based ones to either [.filename]#comms# or [.filename]#misc#, depending on the exact functionality. + + |[.filename]#enlightenment#`*` + |Ports related to the Enlightenment window manager. + | + + |[.filename]#filesystems# + |File systems and related utilities. + | + + |[.filename]#finance# + |Monetary, financial and related applications. + | + + |[.filename]#french# + |French language support. + | + + |[.filename]#ftp# + |FTP client and server utilities. + |If the port speaks both FTP and HTTP, put it in [.filename]#ftp# with a secondary category of [.filename]#www#. + + |[.filename]#games# + |Games. + | + + |[.filename]#geography#`*` + |Geography-related software. + | + + |[.filename]#german# + |German language support. + | + + |[.filename]#gnome#`*` + |Ports from the https://www.gnome.org/[GNOME] Project. + | + + |[.filename]#gnustep#`*` + |Software related to the GNUstep desktop environment. + | + + |[.filename]#graphics# + |Graphics utilities. + | + + |[.filename]#hamradio#`*` + |Software for amateur radio. + | + + |[.filename]#haskell#`*` + |Software related to the Haskell language. + | + + |[.filename]#hebrew# + |Hebrew language support. + | + + |[.filename]#hungarian# + |Hungarian language support. + | + + |[.filename]#irc# + |Internet Relay Chat utilities. + | + + |[.filename]#japanese# + |Japanese language support. + | + + |[.filename]#java# + |Software related to the Java(TM) language. + |The [.filename]#java# category must not be the only one for a port. Save for ports directly related to the Java language, porters are also encouraged not to use [.filename]#java# as the main category of a port. + + |[.filename]#kde#`*` + |Ports from the https://www.kde.org/[KDE] Project (generic). + | + + |[.filename]#kde-applications#`*` + |Applications from the https://www.kde.org/[KDE] Project. + | + + |[.filename]#kde-frameworks#`*` + |Add-on libraries from the https://www.kde.org/[KDE] Project for programming with Qt. + | + + |[.filename]#kde-plasma#`*` + |Desktop from the https://www.kde.org/[KDE] Project. + | + + |[.filename]#kld#`*` + |Kernel loadable modules. + | + + |[.filename]#korean# + |Korean language support. + | + + |[.filename]#lang# + |Programming languages. + | + + |[.filename]#linux#`*` + |Linux applications and support utilities. + | + + |[.filename]#lisp#`*` + |Software related to the Lisp language. + | + + |[.filename]#mail# + |Mail software. + | + + |[.filename]#mate#`*` + |Ports related to the MATE desktop environment, a fork of GNOME 2. + | + + |[.filename]#math# + |Numerical computation software and other utilities for mathematics. + | + + |[.filename]#mbone#`*` + |MBone applications. + | + + |[.filename]#misc# + |Miscellaneous utilities + |Things that do not belong anywhere else. If at all possible, try to find a better category for the port than `misc`, as ports tend to be overlooked in here. + + |[.filename]#multimedia# + |Multimedia software. + | + + |[.filename]#net# + |Miscellaneous networking software. + | + + |[.filename]#net-im# + |Instant messaging software. + | + + |[.filename]#net-mgmt# + |Networking management software. + | + + |[.filename]#net-p2p# + |Peer to peer network applications. + | + + |[.filename]#net-vpn#`*` + |Virtual Private Network applications. + | + + |[.filename]#news# + |USENET news software. + | + + |[.filename]#parallel#`*` + |Applications dealing with parallelism in computing. + | + + |[.filename]#pear#`*` + |Ports related to the Pear PHP framework. + | + + |[.filename]#perl5#`*` + |Ports that require Perl version 5 to run. + | + + |[.filename]#plan9#`*` + |Various programs from https://9p.io/wiki/plan9/Download/index.html[Plan9]. + | + + |[.filename]#polish# + |Polish language support. + | + + |[.filename]#ports-mgmt# + |Ports for managing, installing and developing FreeBSD ports and packages. + | + + |[.filename]#portuguese# + |Portuguese language support. + | + + |[.filename]#print# + |Printing software. + |Desktop publishing tools (previewers, etc.) belong here too. + + |[.filename]#python#`*` + |Software related to the https://www.python.org/[Python] language. + | + + |[.filename]#ruby#`*` + |Software related to the https://www.ruby-lang.org/[Ruby] language. + | + + |[.filename]#rubygems#`*` + |Ports of https://www.rubygems.org/[RubyGems] packages. + | + + |[.filename]#russian# + |Russian language support. + | + + |[.filename]#scheme#`*` + |Software related to the Scheme language. + | + + |[.filename]#science# + |Scientific ports that do not fit into other categories such as [.filename]#astro#, [.filename]#biology# and [.filename]#math#. + | + + |[.filename]#security# + |Security utilities. + | + + |[.filename]#shells# + |Command line shells. + | + + |[.filename]#spanish#`*` + |Spanish language support. + | + + |[.filename]#sysutils# + |System utilities. + | + + |[.filename]#tcl#`*` + |Ports that use Tcl to run. + | + + |[.filename]#textproc# + |Text processing utilities. + |It does not include desktop publishing tools, which go to [.filename]#print#. + + |[.filename]#tk#`*` + |Ports that use Tk to run. + | + + |[.filename]#ukrainian# + |Ukrainian language support. + | + + |[.filename]#vietnamese# + |Vietnamese language support. + | + + |[.filename]#wayland#`*` + |Ports to support the Wayland display server. + | + + |[.filename]#windowmaker#`*` + |Ports to support the Window Maker window manager. + | + + |[.filename]#www# + |Software related to the World Wide Web. + |HTML language support belongs here too. + + |[.filename]#x11# + |The X Window System and friends. + |This category is only for software that directly supports the window system. Do not put regular X applications here. Most of them go into other [.filename]#x11-*# categories (see below). + + |[.filename]#x11-clocks# + |X11 clocks. + | + + |[.filename]#x11-drivers# + |X11 drivers. + | + + |[.filename]#x11-fm# + |X11 file managers. + | + + |[.filename]#x11-fonts# + |X11 fonts and font utilities. + | + + |[.filename]#x11-servers# + |X11 servers. + | + + |[.filename]#x11-themes# + |X11 themes. + | + + |[.filename]#x11-toolkits# + |X11 toolkits. + | + + |[.filename]#x11-wm# + |X11 window managers. + | + + |[.filename]#xfce#`*` + |Ports related to the https://www.xfce.org/[Xfce] desktop environment. + | + + |[.filename]#zope#`*` + |https://www.zope.org/[Zope] support. + | + |=== + + [[choosing-categories]] + === Choosing the Right Category + + As many of the categories overlap, choosing which of the categories will be the primary category of the port can be tedious. + There are several rules that govern this issue. + Here is the list of priorities, in decreasing order of precedence: + + * The first category must be a physical category (see + crossref:makefiles[porting-categories,above]). This is necessary to make the packaging work. Virtual categories and physical categories may be intermixed after that. + * Language specific categories always come first. For example, if the port installs Japanese X11 fonts, then the `CATEGORIES` line would read [.filename]#japanese x11-fonts#. + * Specific categories are listed before less-specific ones. For instance, an HTML editor is listed as [.filename]#www editors#, not the other way around. Also, do not list [.filename]#net# when the port belongs to any of [.filename]#irc#, [.filename]#mail#, [.filename]#news#, [.filename]#security#, or [.filename]#www#, as [.filename]#net# is included implicitly. + * [.filename]#x11# is used as a secondary category only when the primary category is a natural language. In particular, do not put [.filename]#x11# in the category line for X applications. + * Emacs modes are placed in the same ports category as the application supported by the mode, not in [.filename]#editors#. For example, an Emacs mode to edit source files of some programming language goes into [.filename]#lang#. + * Ports installing loadable kernel modules also have the virtual category [.filename]#kld# in their `CATEGORIES` line. This is one of the things handled automatically by adding `USES=kmod`. + * [.filename]#misc# does not appear with any other non-virtual category. If there is `misc` with something else in `CATEGORIES`, that means `misc` can safely be deleted and the port placed only in the other subdirectory. + * If the port truly does not belong anywhere else, put it in [.filename]#misc#. + + If the category is not clearly defined, please put a comment to that effect in the https://bugs.freebsd.org/submit/[port submission] in the bug database so we can discuss it before we import it. + As a committer, send a note to the {freebsd-ports} so we can discuss it first. + Too often, new ports are imported to the wrong category only to be moved right away. + + [[proposing-categories]] + === Proposing a New Category + + As the Ports Collection has grown over time, various new categories have been introduced. + New categories can either be _virtual_ categories-those that do not have a corresponding subdirectory in the ports tree- or _physical_ categories-those that do. This section discusses the issues involved in creating a new physical category. + Read it thoroughly before proposing a new one. + + Our existing practice has been to avoid creating a new physical category unless either a large number of ports would logically belong to it, or the ports that would belong to it are a logically distinct group that is of limited general interest (for instance, categories related to spoken human languages), or preferably both. + + The rationale for this is that such a change creates a extref:{committers-guide}[fair amount of work, ports] for both the committers and also for all users who track changes to the Ports Collection. + In addition, proposed category changes just naturally seem to attract controversy. + (Perhaps this is because there is no clear consensus on when a category is "too big", nor whether categories should lend themselves to browsing (and thus what number of categories would be an ideal number), and so forth.) + + Here is the procedure: + + [.procedure] + . Propose the new category on {freebsd-ports}. Include a detailed rationale for the new category, including why the existing categories are not sufficient, and the list of existing ports proposed to move. (If there are new ports pending in Bugzilla that would fit this category, list them too.) If you are the maintainer and/or submitter, respectively, mention that as it may help the case. + . Participate in the discussion. + . If it seems that there is support for the idea, file a PR which includes both the rationale and the list of existing ports that need to be moved. Ideally, this PR would also include these patches: + + ** [.filename]##Makefile##s for the new ports once they are repocopied + ** [.filename]#Makefile# for the new category + ** [.filename]#Makefile# for the old ports' categories + ** [.filename]##Makefile##s for ports that depend on the old ports + ** (for extra credit, include the other files that have to change, as per the procedure in the Committer's Guide.) + + . Since it affects the ports infrastructure and involves moving and patching many ports but also possibly running regression tests on the build cluster, assign the PR to the {portmgr}. + . If that PR is approved, a committer will need to follow the rest of the procedure that is extref:{committers-guide}[outlined in the Committer's Guide, ports]. + + Proposing a new virtual category is similar to the above but much less involved, since no ports will actually have to move. + In this case, the only patches to include in the PR would be those to add the new category to `CATEGORIES` of the affected ports. + + [[proposing-reorg]] + === Proposing Reorganizing All the Categories + + Occasionally someone proposes reorganizing the categories with either a 2-level structure, or some other kind of keyword structure. + To date, nothing has come of any of these proposals because, while they are very easy to make, the effort involved to retrofit the entire existing ports collection with any kind of reorganization is daunting to say the very least. + Please read the history of these proposals in the mailing list archives before posting this idea. + Furthermore, be prepared to be challenged to offer a working prototype. + + [[makefile-distfiles]] + == The Distribution Files + + The second part of the [.filename]#Makefile# describes the files that must be downloaded to build the port, and where they can be downloaded. + + [[makefile-distname]] + === `DISTNAME` + + `DISTNAME` is the name of the port as called by the authors of the software. + `DISTNAME` defaults to `${PORTNAME}-${DISTVERSIONPREFIX}${DISTVERSION}${DISTVERSIONSUFFIX}`, and if not set, `DISTVERSION` defaults to `${PORTVERSION}` so override `DISTNAME` only if necessary. + `DISTNAME` is only used in two places. + First, the distribution file list (`DISTFILES`) defaults to `${DISTNAME}${EXTRACT_SUFX}`. + Second, the distribution file is expected to extract into a subdirectory named `WRKSRC`, which defaults to [.filename]#work/${DISTNAME}#. + + Some vendor's distribution names which do not fit into the `${PORTNAME}-${PORTVERSION}`-scheme can be handled automatically by setting `DISTVERSIONPREFIX`, `DISTVERSION`, and `DISTVERSIONSUFFIX`. + `PORTVERSION` will be derived from `DISTVERSION` automatically. + + [IMPORTANT] + ==== + Only one of `PORTVERSION` and `DISTVERSION` can be set at a time. + If `DISTVERSION` does not derive a correct `PORTVERSION`, do not use `DISTVERSION`. + ==== + + If the upstream version scheme can be derived into a ports-compatible version scheme, set some variable to the upstream version, _do not_ use `DISTVERSION` as the variable name. + Set `PORTVERSION` to the computed version based on the variable you created, and set `DISTNAME` accordingly. + + If the upstream version scheme cannot easily be coerced into a ports-compatible value, set `PORTVERSION` to a sensible value, and set `DISTNAME` with `PORTNAME` with the verbatim upstream version. + + [[makefile-distname-ex1]] + .Deriving `PORTVERSION` Manually + [example] + ==== + BIND9 uses a version scheme that is not compatible with the ports versions (it has `-` in its versions) and cannot be derived using `DISTVERSION` because after the 9.9.9 release, it will release a "patchlevels" in the form of `9.9.9-P1`. + DISTVERSION would translate that into `9.9.9.p1`, which, in the ports versioning scheme means 9.9.9 pre-release 1, which is before 9.9.9 and not after. + So `PORTVERSION` is manually derived from an `ISCVERSION` variable to output `9.9.9p1`. + + The order into which the ports framework, and pkg, will sort versions is checked using the `-t` argument of man:pkg-version[8]: + + [source,shell] + .... + % pkg version -t 9.9.9 9.9.9.p1 + > <.> + % pkg version -t 9.9.9 9.9.9p1 + < <.> + .... + + <.> The `>` sign means that the first argument passed to `-t` is greater than the second argument. `9.9.9` is after `9.9.9.p1`. + <.> The `<` sign means that the first argument passed to `-t` is less than the second argument. `9.9.9` is before `9.9.9p1`. + + In the port [.filename]#Makefile#, for example package:dns/bind99[], it is achieved by: + + [.programlisting] + .... + PORTNAME= bind + PORTVERSION= ${ISCVERSION:S/-P/P/:S/b/.b/:S/a/.a/:S/rc/.rc/} + CATEGORIES= dns net + MASTER_SITES= ISC/bind9/${ISCVERSION} + PKGNAMESUFFIX= 99 + DISTNAME= ${PORTNAME}-${ISCVERSION} + + MAINTAINER= mat@FreeBSD.org + COMMENT= BIND DNS suite with updated DNSSEC and DNS64 + WWW= https://www.isc.org/bind/ + + LICENSE= ISCL + + # ISC releases things like 9.8.0-P1 or 9.8.1rc1, which our versioning does not like + ISCVERSION= 9.9.9-P6 + .... + + Define upstream version in `ISCVERSION`, with a comment saying _why_ it is needed. + Use `ISCVERSION` to get a ports-compatible `PORTVERSION`. + Use `ISCVERSION` directly to get the correct URL for fetching the distribution file. + Use `ISCVERSION` directly to name the distribution file. + ==== + + [[makefile-distname-ex2]] + .Derive `DISTNAME` from `PORTVERSION` + [example] + ==== + From time to time, the distribution file name has little or no relation to the version of the software. + + In package:comms/kermit[], only the last element of the version is present in the distribution file: + + [.programlisting] + .... + PORTNAME= kermit + PORTVERSION= 9.0.304 + CATEGORIES= comms ftp net + MASTER_SITES= ftp://ftp.kermitproject.org/kermit/test/tar/ + DISTNAME= cku${PORTVERSION:E}-dev20 + .... + + The `:E` man:make[1] modifier returns the suffix of the variable, in this case, `304`. + The distribution file is correctly generated as `cku304-dev20.tar.gz`. + ==== + + [[makefile-distname-ex3]] + .Exotic Case 1 + [example] + ==== + Sometimes, there is no relation between the software name, its version, and the distribution file it is distributed in. + + From package:audio/libworkman[]: + + [.programlisting] + .... + PORTNAME= libworkman + PORTVERSION= 1.4 + CATEGORIES= audio + MASTER_SITES= LOCAL/jim + DISTNAME= ${PORTNAME}-1999-06-20 + .... + + ==== + + [[makefile-distname-ex4]] + .Exotic Case 2 + [example] + ==== + In package:comms/librs232[], the distribution file is not versioned, so using + crossref:makefiles[makefile-dist_subdir,`DIST_SUBDIR`] is needed: + + [.programlisting] + .... + PORTNAME= librs232 + PORTVERSION= 20160710 + CATEGORIES= comms + MASTER_SITES= http://www.teuniz.net/RS-232/ + DISTNAME= RS-232 + DIST_SUBDIR= ${PORTNAME}-${PORTVERSION} + .... + + ==== + + [NOTE] + ==== + `PKGNAMEPREFIX` and `PKGNAMESUFFIX` do not affect `DISTNAME`. + Also note that if `WRKSRC` is equal to [.filename]#${WRKDIR}/${DISTNAME}# while the original source archive is named something other than `${PORTNAME}-${PORTVERSION}${EXTRACT_SUFX}`, leave `DISTNAME` alone- defining only `DISTFILES` is easier than both `DISTNAME` and `WRKSRC` (and possibly `EXTRACT_SUFX`). + ==== + + [[makefile-master_sites]] + === `MASTER_SITES` + + Record the directory part of the FTP/HTTP-URL pointing at the original tarball in `MASTER_SITES`. + Do not forget the trailing slash ([.filename]#/#)! + + The `make` macros will try to use this specification for grabbing the distribution file with `FETCH` if they cannot find it already on the system. + + It is recommended that multiple sites are included on this list, preferably from different continents. + This will safeguard against wide-area network problems. + + [IMPORTANT] + ==== + `MASTER_SITES` must not be blank. + It must point to the actual site hosting the distribution files. + It cannot point to web archives, or the FreeBSD distribution files cache sites. + The only exception to this rule is ports that do not have any distribution files. + For example, meta-ports do not have any distribution files, so `MASTER_SITES` does not need to be set. + ==== + + [[makefile-master_sites-shorthand]] + ==== Using `MASTER_SITE_*` Variables + + Shortcut abbreviations are available for popular archives like SourceForge (`SOURCEFORGE`), GNU (`GNU`), or Perl CPAN (`PERL_CPAN`). `MASTER_SITES` can use them directly: + + [.programlisting] + .... + MASTER_SITES= GNU/make + .... + + The older expanded format still works, but all ports have been converted to the compact format. + The expanded format looks like this: + + [.programlisting] + .... + MASTER_SITES= ${MASTER_SITE_GNU} + MASTER_SITE_SUBDIR= make + .... + + These values and variables are defined in https://cgit.freebsd.org/ports/tree/Mk/bsd.sites.mk[Mk/bsd.sites.mk]. + New entries are added often, so make sure to check the latest version of this file before submitting a port. + + [TIP] + ==== + For any `MASTER_SITE_FOO` variable, the shorthand `_FOO_` can be used. + For example, use: + + [.programlisting] + .... + MASTER_SITES= FOO + .... + + If `MASTER_SITE_SUBDIR` is needed, use this: + + [.programlisting] + .... + MASTER_SITES= FOO/bar + .... + + ==== + + [NOTE] + ==== + Some `MASTER_SITE_*` names are quite long, and for ease of use, shortcuts have been defined: + + [[makefile-master_sites-shortcut]] + .Shortcuts for `MASTER_SITE_*` Macros + [cols="1,1", frame="none", options="header"] + |=== + | Macro + | Shortcut + + |`PERL_CPAN` + |`CPAN` + + |`GITHUB` + |`GH` + + |`GITHUB_CLOUD` + |`GHC` + + |`LIBREOFFICE_DEV` + |`LODEV` + + |`NETLIB` + |`NL` + + |`RUBYGEMS` + |`RG` + + |`SOURCEFORGE` + |`SF` + |=== + ==== + + [[makefile-master_sites-magic]] + ==== Magic MASTER_SITES Macros + + Several "magic" macros exist for popular sites with a predictable directory structure. + For these, just use the abbreviation and the system will choose a subdirectory automatically. + For a port named `Stardict`, of version `1.2.3`, and hosted on SourceForge, adding this line: + + [.programlisting] + .... + MASTER_SITES= SF + .... + + infers a subdirectory named `/project/stardict/stardict/1.2.3`. + If the inferred directory is incorrect, it can be overridden: + + [.programlisting] + .... + MASTER_SITES= SF/stardict/WyabdcRealPeopleTTS/${PORTVERSION} + .... + + This can also be written as + + [.programlisting] + .... + MASTER_SITES= SF + MASTER_SITE_SUBDIR= stardict/WyabdcRealPeopleTTS/${PORTVERSION} + .... + + [[makefile-master_sites-popular]] + .Magic `MASTER_SITES` Macros + [cols="1,1", frame="none", options="header"] + |=== + | Macro + | Assumed subdirectory + + |`APACHE_COMMONS_BINARIES` + |`${PORTNAME:S,commons-,,}` + + |`APACHE_COMMONS_SOURCE` + |`${PORTNAME:S,commons-,,}` + + |`APACHE_JAKARTA` + |`${PORTNAME:S,-,/,}/source` + + |`BERLIOS` + |`${PORTNAME:tl}.berlios` + + |`CHEESESHOP` + |`source/${DISTNAME:C/(.).\*/\1/}/${DISTNAME:C/(.*)-[0-9].*/\1/}` + + |`CPAN` + |`${PORTNAME:C/-.*//}` + + |`DEBIAN` + |`pool/main/${PORTNAME:C/^((lib)?.).*$/\1/}/${PORTNAME}` + + |`FARSIGHT` + |`${PORTNAME}` + + |`FESTIVAL` + |`${PORTREVISION}` + + |`GCC` + |`releases/${DISTNAME}` + + |`GENTOO` + |`distfiles` + + |`GIMP` + |`${PORTNAME}/${PORTVERSION:R}/` + + |`GH` + |`${GH_ACCOUNT}/${GH_PROJECT}/tar.gz/${GH_TAGNAME}?dummy=/` + + |`GHC` + |`${GH_ACCOUNT}/${GH_PROJECT}/` + + |`GNOME` + |`sources/${PORTNAME}/${PORTVERSION:C/^([0-9]+\.[0-9]+).*/\1/}` + + |`GNU` + |`${PORTNAME}` + + |`GNUPG` + |`${PORTNAME}` + + |`GNU_ALPHA` + |`${PORTNAME}` + + |`HORDE` + |`${PORTNAME}` + + |`LODEV` + |`${PORTNAME}` + + |`MATE` + |`${PORTVERSION:C/^([0-9]+\.[0-9]+).*/\1/}` + + |`MOZDEV` + |`${PORTNAME:tl}` + + |`NL` + |`${PORTNAME}` + + |`QT` + |`archive/qt/${PORTVERSION:R}` + + |`SAMBA` + |`${PORTNAME}` + + |`SAVANNAH` + |`${PORTNAME:tl}` + + |`SF` + |`${PORTNAME:tl}/${PORTNAME:tl}/${PORTVERSION}` + |=== + + [[makefile-master_sites-github]] + === `USE_GITHUB` + + If the distribution file comes from a specific commit or tag on https://github.com/[GitHub] for which there is no officially released file, + there is an easy way to set the right `DISTNAME` and `MASTER_SITES` automatically. + + [WARNING] + ==== + As of 2023-02-21 link:https://github.blog/2023-02-21-update-on-the-future-stability-of-source-code-archives-and-hashes/[GitHub] have announced that source downloads will be stable for a year. + Please switch to release assets and if not available ask upstream to generate ones. + ==== + + These variables are available: + + [[makefile-master_sites-github-description]] + .`USE_GITHUB` Description + [cols="1,1,1", options="header"] + |=== + | Variable + | Description + | Default + + |`GH_ACCOUNT` + |Account name of the GitHub user hosting the project + |`${PORTNAME}` + + |`GH_PROJECT` + |Name of the project on GitHub + |`${PORTNAME}` + + |`GH_TAGNAME` + |Name of the tag to download (2.0.1, hash, ...) Using the name of a branch here is incorrect. It is also possible to use the hash of a commit id to do a snapshot. + |`${DISTVERSIONPREFIX}${DISTVERSION}${DISTVERSIONSUFFIX}` + + |`GH_SUBDIR` + |When the software needs an additional distribution file to be extracted within + `${WRKSRC}`, this variable can be used. See the examples in + crossref:makefiles[makefile-master_sites-github-multiple, Fetching Multiple Files from GitHub] for more information. + |(none) + + |`GH_TUPLE` + |`GH_TUPLE` allows putting `GH_ACCOUNT`, `GH_PROJECT`, `GH_TAGNAME`, and `GH_SUBDIR` into a single variable. The format is _account_`:`_project_`:`_tagname_`:`_group_`/`_subdir_. The `/`_subdir_ part is optional. It is helpful when there is more than one GitHub project from which to fetch. + | + |=== + + [IMPORTANT] + ==== + Do not use `GH_TUPLE` for the default distribution file, as it has no default. + ==== + + [[makefile-master_sites-github-ex1]] + .Simple Use of `USE_GITHUB` + [example] + ==== + + While trying to make a port for version `1.2.7` of pkg from the FreeBSD user on github, at https://github.com/freebsd/pkg/[], The [.filename]#Makefile# would end up looking like this (slightly stripped for the example): + + [.programlisting] + .... + PORTNAME= pkg + DISTVERSION= 1.2.7 + + USE_GITHUB= yes + GH_ACCOUNT= freebsd + .... + + It will automatically have `MASTER_SITES` set to `GH` and `WRKSRC` to `${WRKDIR}/pkg-1.2.7`. + ==== + + [[makefile-master_sites-github-ex2]] + .More Complete Use of `USE_GITHUB` + [example] + ==== + While trying to make a port for the bleeding edge version of pkg from the FreeBSD user on github, at https://github.com/freebsd/pkg/[], the [.filename]#Makefile# ends up looking like this (slightly stripped for the example): + + [.programlisting] + .... + PORTNAME= pkg-devel + DISTVERSION= 1.3.0.a.20140411 + + USE_GITHUB= yes + GH_ACCOUNT= freebsd + GH_PROJECT= pkg + GH_TAGNAME= 6dbb17b + .... + + It will automatically have `MASTER_SITES` set to `GH` and `WRKSRC` to `${WRKDIR}/pkg-6dbb17b`. + + [TIP] + **** + `20140411` is the date of the commit referenced in `GH_TAGNAME`, not the date the [.filename]#Makefile# is edited, or the date the commit is made. + **** + + ==== + + [[makefile-master_sites-github-ex3]] + .Use of `USE_GITHUB` with `DISTVERSIONPREFIX` + [example] + ==== + From time to time, `GH_TAGNAME` is a slight variation from `DISTVERSION`. + For example, if the version is `1.0.2`, the tag is `v1.0.2`. + In those cases, it is possible to use `DISTVERSIONPREFIX` or `DISTVERSIONSUFFIX`: + + [.programlisting] + .... + PORTNAME= foo + DISTVERSIONPREFIX= v + DISTVERSION= 1.0.2 + + USE_GITHUB= yes + .... + + It will automatically set `GH_TAGNAME` to `v1.0.2`, while `WRKSRC` will be kept to `${WRKDIR}/foo-1.0.2`. + ==== + + [[makefile-master_sites-github-ex4]] + .Using `USE_GITHUB` When Upstream Does Not Use Versions + [example] + ==== + If there never was a version upstream, do not invent one like `0.1` or `1.0`. + Create the port with a `DISTVERSION` of `g__YYYYMMDD__`, where `g` is for Git, and `_YYYYMMDD_` represents the date the commit referenced in `GH_TAGNAME`. + + [.programlisting] + .... + PORTNAME= bar + DISTVERSION= g20140411 + + USE_GITHUB= yes + GH_TAGNAME= c472d66b + .... + + This creates a versioning scheme that increases over time, and that is still before version `0`. +-See crossref:makefiles[makefile-versions-ex-pkg-version, this secion on how to compare versions] using man:pkg-version[8]): ++See crossref:makefiles[makefile-versions-ex-pkg-version, this secion on how to compare versions] using man:pkg-version[8]: + + [source,shell] + .... + % pkg version -t g20140411 0 + < + .... + + Which means using `PORTEPOCH` will not be needed in case upstream decides to cut versions in the future. + ==== + + [[makefile-master_sites-github-ex5]] + .Using `USE_GITHUB` to Access a Commit Between Two Versions + [example] + ==== + If the current version of the software uses a Git tag, and the port needs to be updated to a newer, intermediate version, without a tag, use man:git-describe[1] to find out the version to use: + + [source,shell] + .... + % git describe --tags f0038b1 + v0.7.3-14-gf0038b1 + .... + + `v0.7.3-14-gf0038b1` can be split into three parts: + + `v0.7.3`:: + This is the last Git tag that appears in the commit history before the requested commit. + + `-14`:: + This means that the requested commit, `f0038b1`, is the 14th commit after the `v0.7.3` tag. + + `-gf0038b1`:: + The `-g` means "Git", and the `f0038b1` is the commit hash that this reference points to. + + [.programlisting] + .... + PORTNAME= bar + DISTVERSIONPREFIX= v + DISTVERSION= 0.7.3-14 + DISTVERSIONSUFFIX= -gf0038b1 + + USE_GITHUB= yes + .... + + This creates a versioning scheme that increases over time (well, over commits), and does not conflict with the creation of a `0.7.4` version. +-See crossref:makefiles[makefile-versions-ex-pkg-version, this section for how to compare versions] using man:pkg-version[8]): ++See crossref:makefiles[makefile-versions-ex-pkg-version, this section for how to compare versions] using man:pkg-version[8]: + + [source,shell] + .... + % pkg version -t 0.7.3 0.7.3.14 + < + % pkg version -t 0.7.3.14 0.7.4 + < + .... + + [NOTE] + **** + If the requested commit is the same as a tag, a shorter description is shown by default. + The longer version is equivalent: + + [source,shell] + .... + % git describe --tags c66c71d + v0.7.3 + + % git describe --tags --long c66c71d + v0.7.3-0-gc66c71d + .... + + **** + + ==== + + [[makefile-master_sites-github-multiple]] + ==== Fetching Multiple Files from GitHub + + The `USE_GITHUB` framework also supports fetching multiple distribution files from different places in GitHub. + It works in a way very similar to crossref:makefiles[porting-master-sites-n, Multiple Distribution or Patches Files from Multiple Locations]. + + Multiple values are added to `GH_ACCOUNT`, `GH_PROJECT`, and `GH_TAGNAME`. + Each different value is assigned a group. + The main value can either have no group, or the `:DEFAULT` group. + A value can be omitted if it is the same as the default as listed in + crossref:makefiles[makefile-master_sites-github-description,`USE_GITHUB` Description]. + + `GH_TUPLE` can also be used when there are a lot of distribution files. + It helps keep the account, project, tagname, and group information at the same place. + + For each group, a `${WRKSRC_group}` helper variable is created, containing the directory into which the file has been extracted. + The `${WRKSRC_group}` variables can be used to move directories around during `post-extract`, or add to `CONFIGURE_ARGS`, or whatever is needed so that the software builds correctly. + + [CAUTION] + ==== + The `:__group__` part _must_ be used for _only one_ distribution file. + It is used as a unique key and using it more than once will overwrite the previous values. + ==== + + [NOTE] + ==== + As this is only syntactic sugar above `DISTFILES` and `MASTER_SITES`, the group + names must adhere to the restrictions on group names outlined in + crossref:makefiles[porting-master-sites-n, Multiple Distribution or Patches Files from Multiple Locations] + ==== + + When fetching multiple files from GitHub, sometimes the default distribution file is not fetched from GitHub. + To disable fetching the default distribution, set: + + [.programlisting] + .... + USE_GITHUB= nodefault + .... + + [IMPORTANT] + ==== + When using `USE_GITHUB=nodefault`, the [.filename]#Makefile# must set `DISTFILES` in its crossref:porting-order[porting-order-portname,top block]. The definition should be: + + [.programlisting] + .... + DISTFILES= ${DISTNAME}${EXTRACT_SUFX} + .... + + ==== + + [[makefile-master_sites-github-multi]] + .Use of `USE_GITHUB` with Multiple Distribution Files + [example] + ==== + From time to time, there is a need to fetch more than one distribution file. + For example, when the upstream git repository uses submodules. + This can be done easily using groups in the `GH_*` variables: + + [.programlisting] + .... + PORTNAME= foo + DISTVERSION= 1.0.2 + + USE_GITHUB= yes + GH_ACCOUNT= bar:icons,contrib + GH_PROJECT= foo-icons:icons foo-contrib:contrib + GH_TAGNAME= 1.0:icons fa579bc:contrib + GH_SUBDIR= ext/icons:icons + + CONFIGURE_ARGS= --with-contrib=${WRKSRC_contrib} + .... + + This will fetch three distribution files from github. + The default one comes from [.filename]#foo/foo# and is version `1.0.2`. + The second one, with the `icons` group, comes from [.filename]#bar/foo-icons# and is in version `1.0`. + The third one comes from [.filename]#bar/foo-contrib# and uses the Git commit `fa579bc`. + The distribution files are named [.filename]#foo-foo-1.0.2_GH0.tar.gz#, [.filename]#bar-foo-icons-1.0_GH0.tar.gz#, and [.filename]#bar-foo-contrib-fa579bc_GH0.tar.gz#. + + All the distribution files are extracted in `${WRKDIR}` in their respective subdirectories. + The default file is still extracted in `${WRKSRC}`, in this case, [.filename]#${WRKDIR}/foo-1.0.2#. + Each additional distribution file is extracted in `${WRKSRC_group}`. + Here, for the `icons` group, it is called `${WRKSRC_icons}` and it contains [.filename]#${WRKDIR}/foo-icons-1.0#. + The file with the `contrib` group is called `${WRKSRC_contrib}` and contains `${WRKDIR}/foo-contrib-fa579bc`. + + The software's build system expects to find the icons in a [.filename]#ext/icons# subdirectory in its sources, so `GH_SUBDIR` is used. + `GH_SUBDIR` makes sure that [.filename]#ext# exists, but that [.filename]#ext/icons# does not already exist. + Then it does this: + + [.programlisting] + .... + post-extract: + @${MV} ${WRKSRC_icons} ${WRKSRC}/ext/icons + .... + + ==== + + [[makefile-master_sites-github-multi2]] + .Use of `USE_GITHUB` with Multiple Distribution Files Using `GH_TUPLE` + [example] + ==== + + This is functionally equivalent to + crossref:makefiles[makefile-master_sites-github-multi,Use of `USE_GITHUB` with Multiple Distribution Files], but using `GH_TUPLE`: + + [.programlisting] + .... + PORTNAME= foo + DISTVERSION= 1.0.2 + + USE_GITHUB= yes + GH_TUPLE= bar:foo-icons:1.0:icons/ext/icons \ + bar:foo-contrib:fa579bc:contrib + + CONFIGURE_ARGS= --with-contrib=${WRKSRC_contrib} + .... + + Grouping was used in the previous example with `bar:icons,contrib`. + Some redundant information is present with `GH_TUPLE` because grouping is not possible. + ==== + + [[makefile-master_sites-github-submodules]] + .How to Use `USE_GITHUB` with Git Submodules? + [example] + ==== + Ports with GitHub as an upstream repository sometimes use submodules. + See man:git-submodule[1] for more information. + + The problem with submodules is that each is a separate repository. + As such, they each must be fetched separately. + + Using package:finance/moneymanagerex[] as an example, its GitHub repository is https://github.com/moneymanagerex/moneymanagerex/[]. + It has a https://github.com/moneymanagerex/moneymanagerex/blob/master/.gitmodules[.gitmodules] file at the root. + This file describes all the submodules used in this repository, and lists additional repositories needed. + This file will tell what additional repositories are needed: + + [.programlisting] + .... + [submodule "lib/wxsqlite3"] + path = lib/wxsqlite3 + url = https://github.com/utelle/wxsqlite3.git + [submodule "3rd/mongoose"] + path = 3rd/mongoose + url = https://github.com/cesanta/mongoose.git + [submodule "3rd/LuaGlue"] + path = 3rd/LuaGlue + url = https://github.com/moneymanagerex/LuaGlue.git + [submodule "3rd/cgitemplate"] + path = 3rd/cgitemplate + url = https://github.com/moneymanagerex/html-template.git + [...] + .... + + The only information missing from that file is the commit hash or tag to use as a version. + This information is found after cloning the repository: + + [source,shell] + .... + % git clone --recurse-submodules https://github.com/moneymanagerex/moneymanagerex.git + Cloning into 'moneymanagerex'... + remote: Counting objects: 32387, done. + [...] + Submodule '3rd/LuaGlue' (https://github.com/moneymanagerex/LuaGlue.git) registered for path '3rd/LuaGlue' + Submodule '3rd/cgitemplate' (https://github.com/moneymanagerex/html-template.git) registered for path '3rd/cgitemplate' + Submodule '3rd/mongoose' (https://github.com/cesanta/mongoose.git) registered for path '3rd/mongoose' + Submodule 'lib/wxsqlite3' (https://github.com/utelle/wxsqlite3.git) registered for path 'lib/wxsqlite3' + [...] + Cloning into '/home/mat/work/freebsd/ports/finance/moneymanagerex/moneymanagerex/3rd/LuaGlue'... + Cloning into '/home/mat/work/freebsd/ports/finance/moneymanagerex/moneymanagerex/3rd/cgitemplate'... + Cloning into '/home/mat/work/freebsd/ports/finance/moneymanagerex/moneymanagerex/3rd/mongoose'... + Cloning into '/home/mat/work/freebsd/ports/finance/moneymanagerex/moneymanagerex/lib/wxsqlite3'... + [...] + Submodule path '3rd/LuaGlue': checked out 'c51d11a247ee4d1e9817dfa2a8da8d9e2f97ae3b' + Submodule path '3rd/cgitemplate': checked out 'cd434eeeb35904ebcd3d718ba29c281a649b192c' + Submodule path '3rd/mongoose': checked out '2140e5992ab9a3a9a34ce9a281abf57f00f95cda' + Submodule path 'lib/wxsqlite3': checked out 'fb66eb230d8aed21dec273b38c7c054dcb7d6b51' + [...] + % cd moneymanagerex + % git submodule status + c51d11a247ee4d1e9817dfa2a8da8d9e2f97ae3b 3rd/LuaGlue (heads/master) + cd434eeeb35904ebcd3d718ba29c281a649b192c 3rd/cgitemplate (cd434ee) + 2140e5992ab9a3a9a34ce9a281abf57f00f95cda 3rd/mongoose (6.2-138-g2140e59) + fb66eb230d8aed21dec273b38c7c054dcb7d6b51 lib/wxsqlite3 (v3.4.0) + [...] + .... + + It can also be found on GitHub. + Each subdirectory that is a submodule is shown as `_directory @ hash_`, for example, `mongoose @ 2140e59`. + + [NOTE] + **** + While getting the information from GitHub seems more straightforward, the information found using `git submodule status` will provide more meaningful information. + For example, here, ``lib/wxsqlite3``'s commit hash `fb66eb2` correspond to `v3.4.0`. + Both can be used interchangeably, but when a tag is available, use it. + **** + + Now that all the required information has been gathered, the [.filename]#Makefile# can be written (only GitHub-related lines are shown): + + [.programlisting] + .... + PORTNAME= moneymanagerex + DISTVERSIONPREFIX= v + DISTVERSION= 1.3.0 + + USE_GITHUB= yes + GH_TUPLE= utelle:wxsqlite3:v3.4.0:wxsqlite3/lib/wxsqlite3 \ + moneymanagerex:LuaGlue:c51d11a:lua_glue/3rd/LuaGlue \ + moneymanagerex:html-template:cd434ee:html_template/3rd/cgitemplate \ + cesanta:mongoose:2140e59:mongoose/3rd/mongoose \ + [...] + .... + + ==== + + [[makefile-master_sites-gitlab]] + === `USE_GITLAB` + + Similar to GitHub, if the distribution file comes from https://gitlab.com/[gitlab.com] or is hosting the GitLab software, these variables are available for use and might need to be set. + + [[makefile-master_sites-gitlab-description]] + .`USE_GITLAB` Description + [cols="1,1,1", options="header"] + |=== + | Variable + | Description + | Default + + |`GL_SITE` + |Site name hosting the GitLab project + |https://gitlab.com/ + + |`GL_ACCOUNT` + |Account name of the GitLab user hosting the project + |`${PORTNAME}` + + |`GL_PROJECT` + |Name of the project on GitLab + |`${PORTNAME}` + + |`GL_COMMIT` + |The commit hash to download. Must be the full 160 bit, 40 character hex sha1 hash. This is a required variable for GitLab. + |`(none)` + + |`GL_SUBDIR` + |When the software needs an additional distribution file to be extracted within + `${WRKSRC}`, this variable can be used. See the examples in + crossref:makefiles[makefile-master_sites-gitlab-multiple, Fetching Multiple Files from GitLab] for more information. + |(none) + + |`GL_TUPLE` + |`GL_TUPLE` allows putting `GL_SITE`, `GL_ACCOUNT`, `GL_PROJECT`, `GL_COMMIT`, and `GL_SUBDIR` into a single variable. The format is _site_`:`_account_`:`_project_`:`_commit_`:`_group_`/`_subdir_. The _site_`:` and `/`_subdir_ part is optional. It is helpful when there are more than one GitLab project from which to fetch. + | + |=== + + [[makefile-master_sites-gitlab-ex1]] + .Simple Use of `USE_GITLAB` + [example] + ==== + While trying to make a port for version `1.14` of libsignon-glib from the accounts-sso user on gitlab.com, at https://gitlab.com/accounts-sso/libsignon-glib/[], The [.filename]#Makefile# would end up looking like this for fetching the distribution files: + + [.programlisting] + .... + PORTNAME= libsignon-glib + DISTVERSION= 1.14 + + USE_GITLAB= yes + GL_ACCOUNT= accounts-sso + GL_COMMIT= e90302e342bfd27bc8c9132ab9d0ea3d8723fd03 + .... + + It will automatically have `MASTER_SITES` set to https://gitlab.com/[gitlab.com] and `WRKSRC` to `${WRKDIR}/libsignon-glib-e90302e342bfd27bc8c9132ab9d0ea3d8723fd03-e90302e342bfd27bc8c9132ab9d0ea3d8723fd03`. + ==== + + [[makefile-master_sites-gitlab-ex2]] + .More Complete Use of `USE_GITLAB` + [example] + ==== + + A more complete use of the above if port had no versioning and foobar from the foo user on project bar on a self hosted GitLab site `https://gitlab.example.com/`, the [.filename]#Makefile# ends up looking like this for fetching distribution files: + + [.programlisting] + .... + PORTNAME= foobar + DISTVERSION= g20170906 + + USE_GITLAB= yes + GL_SITE= https://gitlab.example.com + GL_ACCOUNT= foo + GL_PROJECT= bar + GL_COMMIT= 9c1669ce60c3f4f5eb43df874d7314483fb3f8a6 + .... + + It will have `MASTER_SITES` set to `"https://gitlab.example.com"` and `WRKSRC` to `${WRKDIR}/bar-9c1669ce60c3f4f5eb43df874d7314483fb3f8a6-9c1669ce60c3f4f5eb43df874d7314483fb3f8a6`. + + [TIP] + ====== + `20170906` is the date of the commit referenced in `GL_COMMIT`, not the date the [.filename]#Makefile# is edited, or the date the commit to the FreeBSD ports tree is made. + ====== + + [NOTE] + ====== + ``GL_SITE``'s protocol, port and webroot can all be modified in the same variable. + ====== + + ==== + + [[makefile-master_sites-gitlab-multiple]] + ==== Fetching Multiple Files from GitLab + + The `USE_GITLAB` framework also supports fetching multiple distribution files from different places from GitLab and GitLab hosted sites. + It works in a way very similar to crossref:makefiles[porting-master-sites-n, Multiple Distribution or Patches Files from Multiple Locations] and + crossref:makefiles[makefile-master_sites-gitlab-multiple, Fetching Multiple Files from GitLab]. + + Multiple values are added to `GL_SITE`, `GL_ACCOUNT`, `GL_PROJECT` and `GL_COMMIT`. + Each different value is assigned a group. + crossref:makefiles[makefile-master_sites-gitlab-description,`USE_GITLAB` Description]. + + `GL_TUPLE` can also be used when there are a lot of distribution files. + It helps keep the site, account, project, commit, and group information at the same place. + + For each group, a `${WRKSRC_group}` helper variable is created, containing the directory into which the file has been extracted. + The `${WRKSRC_group}` variables can be used to move directories around during `post-extract`, or add to `CONFIGURE_ARGS`, or whatever is needed so that the software builds correctly. + + [CAUTION] + ==== + The `:__group__` part _must_ be used for _only one_ distribution file. + It is used as a unique key and using it more than once will overwrite the previous values. + ==== + + [NOTE] + ==== + As this is only syntactic sugar above `DISTFILES` and `MASTER_SITES`, the group + names must adhere to the restrictions on group names outlined in + crossref:makefiles[porting-master-sites-n, Multiple Distribution or Patches Files from Multiple Locations] + ==== + + When fetching multiple files using GitLab, sometimes the default distribution file is not fetched from a GitLab site. + To disable fetching the default distribution, set: + + [.programlisting] + .... + USE_GITLAB= nodefault + .... + + [IMPORTANT] + ==== + When using `USE_GITLAB=nodefault`, the [.filename]#Makefile# must set + `DISTFILES` in its crossref:makefiles[porting-order-portname,top block]. + The definition should be: + + [.programlisting] + .... + DISTFILES= ${DISTNAME}${EXTRACT_SUFX} + .... + + ==== + + [[makefile-master_sites-gitlab-multi]] + .Use of `USE_GITLAB` with Multiple Distribution Files + [example] + ==== + From time to time, there is a need to fetch more than one distribution file. + For example, when the upstream git repository uses submodules. + This can be done easily using groups in the `GL_*` variables: + + [.programlisting] + .... + PORTNAME= foo + DISTVERSION= 1.0.2 + + USE_GITLAB= yes + GL_SITE= https://gitlab.example.com:9434/gitlab:icons + GL_ACCOUNT= bar:icons,contrib + GL_PROJECT= foo-icons:icons foo-contrib:contrib + GL_COMMIT= c189207a55da45305c884fe2b50e086fcad4724b ae7368cab1ca7ca754b38d49da064df87968ffe4:icons 9e4dd76ad9b38f33fdb417a4c01935958d5acd2a:contrib + GL_SUBDIR= ext/icons:icons + + CONFIGURE_ARGS= --with-contrib=${WRKSRC_contrib} + .... + + This will fetch two distribution files from gitlab.com and one from `gitlab.example.com` hosting GitLab. + The default one comes from [.filename]#https://gitlab.com/foo/foo# and commit is `c189207a55da45305c884fe2b50e086fcad4724b`. + The second one, with the `icons` group, comes from [.filename]#https://gitlab.example.com:9434/gitlab/bar/foo-icons# and commit is `ae7368cab1ca7ca754b38d49da064df87968ffe4`. + The third one comes from [.filename]#https://gitlab.com/bar/foo-contrib# and is commit `9e4dd76ad9b38f33fdb417a4c01935958d5acd2a`. + The distribution files are named [.filename]#foo-foo-c189207a55da45305c884fe2b50e086fcad4724b_GL0.tar.gz#, [.filename]#bar-foo-icons-ae7368cab1ca7ca754b38d49da064df87968ffe4_GL0.tar.gz#, and [.filename]#bar-foo-contrib-9e4dd76ad9b38f33fdb417a4c01935958d5acd2a_GL0.tar.gz#. + + All the distribution files are extracted in `${WRKDIR}` in their respective subdirectories. + The default file is still extracted in `${WRKSRC}`, in this case, [.filename]#${WRKDIR}/foo-c189207a55da45305c884fe2b50e086fcad4724b-c189207a55da45305c884fe2b50e086fcad4724b#. + Each additional distribution file is extracted in `${WRKSRC_group}`. + Here, for the `icons` group, it is called `${WRKSRC_icons}` and it contains [.filename]#${WRKDIR}/foo-icons-ae7368cab1ca7ca754b38d49da064df87968ffe4-ae7368cab1ca7ca754b38d49da064df87968ffe4#. + The file with the `contrib` group is called `${WRKSRC_contrib}` and contains `${WRKDIR}/foo-contrib-9e4dd76ad9b38f33fdb417a4c01935958d5acd2a-9e4dd76ad9b38f33fdb417a4c01935958d5acd2a`. + + The software's build system expects to find the icons in a [.filename]#ext/icons# subdirectory in its sources, so `GL_SUBDIR` is used. + `GL_SUBDIR` makes sure that [.filename]#ext# exists, but that [.filename]#ext/icons# does not already exist. + Then it does this: + + [.programlisting] + .... + post-extract: + @${MV} ${WRKSRC_icons} ${WRKSRC}/ext/icons + .... + + ==== + + [[makefile-master_sites-gitlab-multi2]] + .Use of `USE_GITLAB` with Multiple Distribution Files Using `GL_TUPLE` + [example] + ==== + This is functionally equivalent to + crossref:makefiles[makefile-master_sites-gitlab-multi,Use of `USE_GITLAB` with Multiple Distribution Files], but using `GL_TUPLE`: + + [.programlisting] + .... + PORTNAME= foo + DISTVERSION= 1.0.2 + + USE_GITLAB= yes + GL_COMMIT= c189207a55da45305c884fe2b50e086fcad4724b + GL_TUPLE= https://gitlab.example.com:9434/gitlab:bar:foo-icons:ae7368cab1ca7ca754b38d49da064df87968ffe4:icons/ext/icons \ + bar:foo-contrib:9e4dd76ad9b38f33fdb417a4c01935958d5acd2a:contrib + + CONFIGURE_ARGS= --with-contrib=${WRKSRC_contrib} + .... + + Grouping was used in the previous example with `bar:icons,contrib`. + Some redundant information is present with `GL_TUPLE` because grouping is not possible. + ==== + + [[makefile-extract_sufx]] + === `EXTRACT_SUFX` + + If there is one distribution file, and it uses an odd suffix to indicate the compression mechanism, set `EXTRACT_SUFX`. + + For example, if the distribution file was named [.filename]#foo.tar.gzip# instead of the more normal [.filename]#foo.tar.gz#, write: + + [.programlisting] + .... + DISTNAME= foo + EXTRACT_SUFX= .tar.gzip + .... + + The `USES=tar[:__xxx__]`, `USES=lha` or `USES=zip` automatically set `EXTRACT_SUFX` to the most common archives extensions as necessary, see crossref:uses[uses,Using `USES` Macros] for more details. + If neither of these are set then `EXTRACT_SUFX` defaults to `.tar.gz`. + + [NOTE] + ==== + As `EXTRACT_SUFX` is only used in `DISTFILES`, only set one of them.. + ==== + + [[makefile-distfiles-definition]] + === `DISTFILES` + + Sometimes the names of the files to be downloaded have no resemblance to the name of the port. + For example, it might be called [.filename]#source.tar.gz# or similar. + In other cases the application's source code might be in several different archives, all of which must be downloaded. + + If this is the case, set `DISTFILES` to be a space separated list of all the files that must be downloaded. + + [.programlisting] + .... + DISTFILES= source1.tar.gz source2.tar.gz + .... + + If not explicitly set, `DISTFILES` defaults to `${DISTNAME}${EXTRACT_SUFX}`. + + [[makefile-extract_only]] + === `EXTRACT_ONLY` + + If only some of the `DISTFILES` must be extracted-for example, one of them is the source code, while another is an uncompressed document-list the filenames that must be extracted in `EXTRACT_ONLY`. + + [.programlisting] + .... + DISTFILES= source.tar.gz manual.html + EXTRACT_ONLY= source.tar.gz + .... + + When none of the `DISTFILES` need to be uncompressed, set `EXTRACT_ONLY` to the empty string. + + [.programlisting] + .... + EXTRACT_ONLY= + .... + + [[porting-patchfiles]] + === `PATCHFILES` + + If the port requires some additional patches that are available by FTP or HTTP, set `PATCHFILES` to the names of the files and `PATCH_SITES` to the URL of the directory that contains them (the format is the same as `MASTER_SITES`). + + If the patch is not relative to the top of the source tree (that is, `WRKSRC`) because it contains some extra pathnames, set `PATCH_DIST_STRIP` accordingly. + For instance, if all the pathnames in the patch have an extra `foozolix-1.0/` in front of the filenames, then set `PATCH_DIST_STRIP=-p1`. + + Do not worry if the patches are compressed; they will be decompressed automatically if the filenames end with [.filename]#.Z#, [.filename]#.gz#, [.filename]#.bz2# or [.filename]#.xz#. + + If the patch is distributed with some other files, such as documentation, in a compressed tarball, using `PATCHFILES` is not possible. + If that is the case, add the name and the location of the patch tarball to `DISTFILES` and `MASTER_SITES`. + Then, use `EXTRA_PATCHES` to point to those files and [.filename]#bsd.port.mk# will automatically apply them. + In particular, do _not_ copy patch files into [.filename]#${PATCHDIR}#. + That directory may not be writable. + + [TIP] + ==== + If there are multiple patches and they need mixed values for the strip parameter, it can be added alongside the patch name in `PATCHFILES`, e.g: + + [.programlisting] + .... + PATCHFILES= patch1 patch2:-p1 + .... + + This does not conflict with crossref:makefiles[porting-master-sites-n,the master site grouping feature], adding a group also works: + + [.programlisting] + .... + PATCHFILES= patch2:-p1:source2 + .... + + ==== + + [NOTE] + ==== + The tarball will have been extracted alongside the regular source by then, so there is no need to explicitly extract it if it is a regular compressed tarball. Take extra care not to overwrite something that already exists in that directory if extracting it manually. + Also, do not forget to add a command to remove the copied patch in the `pre-clean` target. + ==== + + [[porting-master-sites-n]] + === Multiple Distribution or Patches Files from Multiple Locations + + (Consider this to be a somewhat "advanced topic"; those new to this document may wish to skip this section at first). + + This section has information on the fetching mechanism known as both `MASTER_SITES:n` and `MASTER_SITES_NN`. + We will refer to this mechanism as `MASTER_SITES:n`. + + A little background first. + OpenBSD has a neat feature inside `DISTFILES` and `PATCHFILES` which allows files and patches to be postfixed with `:n` identifiers. + Here, `n` can be any word containing `[0-9a-zA-Z_]` and denote a group designation. + For example: + + [.programlisting] + .... + DISTFILES= alpha:0 beta:1 + .... + + In OpenBSD, distribution file [.filename]#alpha# will be associated with variable `MASTER_SITES0` instead of our common `MASTER_SITES` and [.filename]#beta# with `MASTER_SITES1`. + + This is a very interesting feature which can decrease that endless search for the correct download site. + + Just picture 2 files in `DISTFILES` and 20 sites in `MASTER_SITES`, the sites slow as hell where [.filename]#beta# is carried by all sites in `MASTER_SITES`, and [.filename]#alpha# can only be found in the 20th site. + It would be such a waste to check all of them if the maintainer knew this beforehand, would it not? Not a good start for that lovely weekend! + + Now that you have the idea, just imagine more `DISTFILES` and more `MASTER_SITES`. + Surely our "distfiles survey meister" would appreciate the relief to network strain that this would bring. + + In the next sections, information will follow on the FreeBSD implementation of this idea. + We improved a bit on OpenBSD's concept. + + [IMPORTANT] + ==== + The group names cannot have dashes in them (`-`), in fact, they cannot have any characters out of the `[a-zA-Z0-9_]` range. + This is because, while man:make[1] is ok with variable names containing dashes, man:sh[1] is not. + ==== + + [[porting-master-sites-n-simplified]] + ==== Simplified Information + + This section explains how to quickly prepare fine grained fetching of multiple distribution files and patches from different sites and subdirectories. + We describe here a case of simplified `MASTER_SITES:n` usage. + This will be sufficient for most scenarios. + More detailed information are available in + crossref:makefiles[ports-master-sites-n-detailed, Detailed Information]. + + Some applications consist of multiple distribution files that must be downloaded from a number of different sites. + For example, Ghostscript consists of the core of the program, and then a large number of driver files that are used depending on the user's printer. + Some of these driver files are supplied with the core, but many others must be downloaded from a variety of different sites. + + To support this, each entry in `DISTFILES` may be followed by a colon and a "group name". + Each site listed in `MASTER_SITES` is then followed by a colon, and the group that indicates which distribution files are downloaded from this site. + + For example, consider an application with the source split in two parts, [.filename]#source1.tar.gz# and [.filename]#source2.tar.gz#, which must be downloaded from two different sites. + The port's [.filename]#Makefile# would include lines like + crossref:makefiles[ports-master-sites-n-example-simple-use-one-file-per-site,Simplified Use of `MASTER_SITES:n` with One File Per Site]. + + [[ports-master-sites-n-example-simple-use-one-file-per-site]] + .Simplified Use of `MASTER_SITES:n` with One File Per Site + [example] + ==== + + [.programlisting] + .... + MASTER_SITES= ftp://ftp1.example.com/:source1 \ + http://www.example.com/:source2 + DISTFILES= source1.tar.gz:source1 \ + source2.tar.gz:source2 + .... + + ==== + + Multiple distribution files can have the same group. + Continuing the previous example, suppose that there was a third distfile, [.filename]#source3.tar.gz#, that is downloaded from `ftp.example2.com`. + The [.filename]#Makefile# would then be written like + crossref:makefiles[ports-master-sites-n-example-simple-use-more-than-one-file-per-site,Simplified Use of `MASTER_SITES:n` with More Than One File Per Site]. + + [[ports-master-sites-n-example-simple-use-more-than-one-file-per-site]] + .Simplified Use of `MASTER_SITES:n` with More Than One File Per Site + [example] + ==== + + [.programlisting] + .... + MASTER_SITES= ftp://ftp.example.com/:source1 \ + http://www.example.com/:source2 + DISTFILES= source1.tar.gz:source1 \ + source2.tar.gz:source2 \ + source3.tar.gz:source2 + .... + + ==== + + [[ports-master-sites-n-detailed]] + ==== Detailed Information + + Okay, so the previous example did not reflect the new port's needs? In this section we will explain in detail how the fine grained fetching mechanism `MASTER_SITES:n` works and how it can be used. + + . Elements can be postfixed with `:__n__` where _n_ is `[^:,]\+`, that is, _n_ could conceptually be any alphanumeric string but we will limit it to `[a-zA-Z_][0-9a-zA-Z_]+` for now. + + + Moreover, string matching is case sensitive; that is, `n` is different from `N`. + + + However, these words cannot be used for postfixing purposes since they yield + special meaning: `default`, `all` and `ALL` (they are used internally in item + crossref:makefiles[porting-master-sites-n-what-changes-in-port-targets, ii]). + Furthermore, `DEFAULT` is a special purpose word (check item + crossref:makefiles[porting-master-sites-n-DEFAULT-group,3]). + . Elements postfixed with `:n` belong to the group `n`, `:m` belong to group `m` and so forth. + + + [[porting-master-sites-n-DEFAULT-group]] + . Elements without a postfix are groupless, they all belong to the special group + `DEFAULT`. Any elements postfixed with `DEFAULT`, is just being redundant + unless an element belongs to both `DEFAULT` and other groups at the same time + (check item crossref:makefiles[porting-master-sites-n-comma-operator,5]). + + + These examples are equivalent but the first one is preferred: + + + [.programlisting] + .... + MASTER_SITES= alpha + .... + + + [.programlisting] + .... + MASTER_SITES= alpha:DEFAULT + .... + + . Groups are not exclusive, an element may belong to several different groups at the same time and a group can either have either several different elements or none at all. + + + [[porting-master-sites-n-comma-operator]] + . When an element belongs to several groups at the same time, use the comma operator (`,`). + + + Instead of repeating it several times, each time with a different postfix, we can list several groups at once in a single postfix. + For instance, `:m,n,o` marks an element that belongs to group `m`, `n` and `o`. + + + All these examples are equivalent but the last one is preferred: + + + [.programlisting] + .... + MASTER_SITES= alpha alpha:SOME_SITE + .... + + + [.programlisting] + .... + MASTER_SITES= alpha:DEFAULT alpha:SOME_SITE + .... + + + [.programlisting] + .... + MASTER_SITES= alpha:SOME_SITE,DEFAULT + .... + + + [.programlisting] + .... + MASTER_SITES= alpha:DEFAULT,SOME_SITE + .... + + . All sites within a given group are sorted according to `MASTER_SORT_AWK`. All groups within `MASTER_SITES` and `PATCH_SITES` are sorted as well. + + + [[porting-master-sites-n-group-semantics]] + . Group semantics can be used in any of the variables `MASTER_SITES`, `PATCH_SITES`, `MASTER_SITE_SUBDIR`, `PATCH_SITE_SUBDIR`, `DISTFILES`, and `PATCHFILES` according to this syntax: + .. All `MASTER_SITES`, `PATCH_SITES`, `MASTER_SITE_SUBDIR` and + `PATCH_SITE_SUBDIR` elements must be terminated with the forward slash `/` + character. If any elements belong to any groups, the group postfix `:__n__` + must come right after the terminator `/`. The `MASTER_SITES:n` mechanism + relies on the existence of the terminator `/` to avoid confusing elements + where a `:n` is a valid part of the element with occurrences where `:n` + denotes group `n`. For compatibility purposes, since the `/` terminator was + not required before in both `MASTER_SITE_SUBDIR` and `PATCH_SITE_SUBDIR` + elements, if the postfix immediate preceding character is not a `/` then `:n` + will be considered a valid part of the element instead of a group postfix + even if an element is postfixed with `:n`. See both + crossref:makefiles[ports-master-sites-n-example-detailed-use-master-site-subdir,Detailed Use of `MASTER_SITES:n` in `MASTER_SITE_SUBDIR`] + and + crossref:makefiles[ports-master-sites-n-example-detailed-use-complete-example-master-sites,Detailed Use of `MASTER_SITES:n` with Comma Operator, Multiple Files, Multiple Sites and Multiple Subdirectories]. + + + [[ports-master-sites-n-example-detailed-use-master-site-subdir]] + .Detailed Use of `MASTER_SITES:n` in `MASTER_SITE_SUBDIR` + [example] + ==== + + [.programlisting] + .... + MASTER_SITE_SUBDIR= old:n new/:NEW + .... + + *** Directories within group `DEFAULT` -> old:n + *** Directories within group `NEW` -> new + + ==== + + + [[ports-master-sites-n-example-detailed-use-complete-example-master-sites]] + .Detailed Use of `MASTER_SITES:n` with Comma Operator, Multiple Files, Multiple Sites and Multiple Subdirectories + [example] + ==== + + [.programlisting] + .... + MASTER_SITES= http://site1/%SUBDIR%/ http://site2/:DEFAULT \ + http://site3/:group3 http://site4/:group4 \ + http://site5/:group5 http://site6/:group6 \ + http://site7/:DEFAULT,group6 \ + http://site8/%SUBDIR%/:group6,group7 \ + http://site9/:group8 + DISTFILES= file1 file2:DEFAULT file3:group3 \ + file4:group4,group5,group6 file5:grouping \ + file6:group7 + MASTER_SITE_SUBDIR= directory-trial:1 directory-n/:groupn \ + directory-one/:group6,DEFAULT \ + directory + .... + + The previous example results in this fine grained fetching. + Sites are listed in the exact order they will be used. + + *** [.filename]#file1# will be fetched from + + **** `MASTER_SITE_OVERRIDE` + **** http://site1/directory-trial:1/ + **** http://site1/directory-one/ + **** http://site1/directory/ + **** http://site2/ + **** http://site7/ + **** `MASTER_SITE_BACKUP` + + *** [.filename]#file2# will be fetched exactly as [.filename]#file1# since they both belong to the same group + + **** `MASTER_SITE_OVERRIDE` + **** http://site1/directory-trial:1/ + **** http://site1/directory-one/ + **** http://site1/directory/ + **** http://site2/ + **** http://site7/ + **** `MASTER_SITE_BACKUP` + + *** [.filename]#file3# will be fetched from + + **** `MASTER_SITE_OVERRIDE` + **** http://site3/ + **** `MASTER_SITE_BACKUP` + + *** [.filename]#file4# will be fetched from + + **** `MASTER_SITE_OVERRIDE` + **** http://site4/ + **** http://site5/ + **** http://site6/ + **** http://site7/ + **** http://site8/directory-one/ + **** `MASTER_SITE_BACKUP` + + *** [.filename]#file5# will be fetched from + + **** `MASTER_SITE_OVERRIDE` + **** `MASTER_SITE_BACKUP` + + *** [.filename]#file6# will be fetched from + + **** `MASTER_SITE_OVERRIDE` + **** http://site8/ + **** `MASTER_SITE_BACKUP` + + ==== + + . How do I group one of the special macros from [.filename]#bsd.sites.mk#, for example, SourceForge (`SF`)? + + + This has been simplified as much as possible. + See + crossref:makefiles[ports-master-sites-n-example-detailed-use-master-site-sourceforge,Detailed Use of `MASTER_SITES:n` with SourceForge (`SF`)]. + + + [[ports-master-sites-n-example-detailed-use-master-site-sourceforge]] + .Detailed Use of `MASTER_SITES:n` with SourceForge (`SF`) + [example] + ==== + + [.programlisting] + .... + MASTER_SITES= http://site1/ SF/something/1.0:sourceforge,TEST + DISTFILES= something.tar.gz:sourceforge + .... + + [.filename]#something.tar.gz# will be fetched from all sites within SourceForge. + ==== + . How do I use this with `PATCH*`? + + + All examples were done with `MASTER*` but they work exactly the same for + `PATCH*` ones as can be seen in + crossref:makefiles[ports-master-sites-n-example-detailed-use-patch-sites,Simplified Use of `MASTER_SITES:n` with `PATCH_SITES`]. + + + [[ports-master-sites-n-example-detailed-use-patch-sites]] + .Simplified Use of `MASTER_SITES:n` with `PATCH_SITES` + [example] + ==== + + [.programlisting] + .... + PATCH_SITES= http://site1/ http://site2/:test + PATCHFILES= patch1:test + .... + + ==== + + [[port-master-sites-n-what-changed]] + ==== What Does Change for Ports? What Does Not? + + [lowerroman] + . All current ports remain the same. The `MASTER_SITES:n` feature code is only + activated if there are elements postfixed with `:__n__` like elements + according to the aforementioned syntax rules, especially as shown in item + crossref:makefiles[porting-master-sites-n-group-semantics, 7]. + + + [[porting-master-sites-n-what-changes-in-port-targets]] + . The port targets remain the same: `checksum`, `makesum`, `patch`, `configure`, `build`, etc. With the obvious exceptions of `do-fetch`, `fetch-list`, `master-sites` and `patch-sites`. + + ** `do-fetch`: deploys the new grouping postfixed `DISTFILES` and `PATCHFILES` + with their matching group elements within both `MASTER_SITES` and + `PATCH_SITES` which use matching group elements within both + `MASTER_SITE_SUBDIR` and `PATCH_SITE_SUBDIR`. Check + crossref:makefiles[ports-master-sites-n-example-detailed-use-complete-example-master-sites,Detailed Use of `MASTER_SITES:n` with Comma Operator, Multiple Files, Multiple Sites and Multiple Subdirectories]. + ** `fetch-list`: works like old `fetch-list` with the exception that it groups just like `do-fetch`. + ** `master-sites` and `patch-sites`: (incompatible with older versions) only return the elements of group `DEFAULT`; in fact, they execute targets `master-sites-default` and `patch-sites-default` respectively. + + + Furthermore, using target either `master-sites-all` or `patch-sites-all` is preferred to directly checking either `MASTER_SITES` or `PATCH_SITES`. + Also, directly checking is not guaranteed to work in any future versions. + Check item + crossref:makefiles[porting-master-sites-n-new-port-targets-master-sites-all, B] for more information on these new port targets. + + . New port targets + .. There are `master-sites-_n_` and `patch-sites-_n_` targets which will list the elements of the respective group _n_ within `MASTER_SITES` and `PATCH_SITES` respectively. For instance, both `master-sites-DEFAULT` and `patch-sites-DEFAULT` will return the elements of group `DEFAULT`, `master-sites-test` and `patch-sites-test` of group `test`, and thereon. + + + [[porting-master-sites-n-new-port-targets-master-sites-all]] + .. There are new targets `master-sites-all` and `patch-sites-all` which do the work of the old `master-sites` and `patch-sites` ones. They return the elements of all groups as if they all belonged to the same group with the caveat that it lists as many `MASTER_SITE_BACKUP` and `MASTER_SITE_OVERRIDE` as there are groups defined within either `DISTFILES` or `PATCHFILES`; respectively for `master-sites-all` and `patch-sites-all`. + + [[makefile-dist_subdir]] + === `DIST_SUBDIR` + + Do not let the port clutter [.filename]#/usr/ports/distfiles#. + If the port requires a lot of files to be fetched, or contains a file that has a name that might conflict with other ports (for example, [.filename]#Makefile#), set `DIST_SUBDIR` to the name of the port (`${PORTNAME}` or `${PKGNAMEPREFIX}${PORTNAME}` are fine). + This will change `DISTDIR` from the default [.filename]#/usr/ports/distfiles# to [.filename]#/usr/ports/distfiles/${DIST_SUBDIR}#, and in effect puts everything that is required for the port into that subdirectory. + + It will also look at the subdirectory with the same name on the backup master site at http://distcache.FreeBSD.org[http://distcache.FreeBSD.org] (Setting `DISTDIR` explicitly in [.filename]#Makefile# will not accomplish this, so please use `DIST_SUBDIR`.) + + [NOTE] + ==== + This does not affect `MASTER_SITES` defined in the [.filename]#Makefile#. + ==== + + [[makefile-maintainer]] + == `MAINTAINER` + + Set your mail-address here. Please. _:-)_ + + Only a single address without the comment part is allowed as a `MAINTAINER` value. + The format used is `user@hostname.domain`. + Please do not include any descriptive text such as a real name in this entry. + That merely confuses the Ports infrastructure and most tools using it. + + The maintainer is responsible for keeping the port up to date and making sure that it works correctly. + For a detailed description of the responsibilities of a port maintainer, refer to extref:{contributing}[The challenge for port maintainers, maintain-port]. + + [NOTE] + ==== + A maintainer volunteers to keep a port in good working order. + Maintainers have the primary responsibility for their ports, but not exclusive ownership. + Ports exist for the benefit of the community and, in reality, belong to the community. + What this means is that people other than the maintainer can make changes to a port. + Large changes to the Ports Collection might require changes to many ports. + The FreeBSD Ports Management Team or members of other teams might modify ports to fix dependency issues or other problems, like a version bump for a shared library update. + + Some types of fixes have "blanket approval" from the {portmgr}, allowing any committer to fix those categories of problems on any port. + These fixes do not need approval from the maintainer. + + Blanket approval for most ports applies to fixes like infrastructure changes, or trivial and _tested_ build and runtime fixes. + The current list is available in extref:{committers-guide}[Ports section of the Committer's Guide, ports-qa-misc-blanket-approval]. + ==== + + Other changes to the port will be sent to the maintainer for review and approval before being committed. + If the maintainer does not respond to an update request after two weeks (excluding major public holidays), then that is considered a maintainer timeout, and the update can be made without explicit maintainer approval. + If the maintainer does not respond within three months, or if there have been three consecutive timeouts, then that maintainer is considered absent without leave, and all of their ports can be assigned back to the pool. + Exceptions to this are anything maintained by the {portmgr}, or the {security-officer}. + No unauthorized commits may ever be made to ports maintained by those groups. + + We reserve the right to modify the maintainer's submission to better match existing policies and style of the Ports Collection without explicit blessing from the submitter or the maintainer. + Also, large infrastructural changes can result in a port being modified without the maintainer's consent. + These kinds of changes will never affect the port's functionality. + + The {portmgr} reserves the right to revoke or override anyone's maintainership for any reason, and the {security-officer} reserves the right to revoke or override maintainership for security reasons. + + [[makefile-comment]] + == `COMMENT` + + The comment is a one-line description of a port shown by `pkg info`. + Please follow these rules when composing it: + + . The COMMENT string should be 70 characters or less. + . Do _not_ include the package name or version number of software. + . The comment must begin with a capital and end without a period. + . Do not start with an indefinite article (that is, A or An). + . Capitalize names such as Apache, JavaScript, or Perl. + . Use a serial comma for lists of words: "green, red, and blue." + . Check for spelling errors. + + Here is an example: + + [.programlisting] + .... + COMMENT= Cat chasing a mouse all over the screen + .... + + The COMMENT variable immediately follows the MAINTAINER variable in the [.filename]#Makefile#. + + [[makefile-www]] + == Project website + + Each port should point to a website that provides more information about the software. + + Whenever possible, this should be the official project website maintained by the developers of the software. + + [.programlisting] + .... + WWW= https://ffmpeg.org/ + .... + + But it can also be a directory or resource in the source code repository: + + [.programlisting] + .... + WWW= https://sourceforge.net/projects/mpd/ + .... + + The WWW variable immediately follows the COMMENT variable in the [.filename]#Makefile#. + + If the same content can be accessed via HTTP and HTTPS, the URL starting with `https://` shall be used. + If the URI is the root of the website or directory, it must be terminated with a slash. + + This information used to be placed into the last line of the [.filename]#pkg-descr# file. + It has been moved into the Makefile for easier maintenance and processing. + Having a `WWW:` line at the end of the [.filename]#pkg-descr# file is deprecated. + + [[licenses]] + == Licenses + + Each port must document the license under which it is available. + If it is not an OSI approved license it must also document any restrictions on redistribution. + + [[licenses-license]] + === `LICENSE` + + A short name for the license or licenses if more than one license apply. + + If it is one of the licenses listed in crossref:makefiles[licenses-license-list,Predefined License List], only `LICENSE_FILE` and `LICENSE_DISTFILES` variables can be set. + + If this is a license that has not been defined in the ports framework (see + crossref:makefiles[licenses-license-list,Predefined License List]), the `LICENSE_PERMS` and `LICENSE_NAME` must be set, along with either `LICENSE_FILE` or `LICENSE_TEXT`. + `LICENSE_DISTFILES` and `LICENSE_GROUPS` can also be set, but are not required. + + The predefined licenses are shown in crossref:makefiles[licenses-license-list,Predefined License List]. + The current list is always available in [.filename]#Mk/bsd.licenses.db.mk#. + + [[licenses-license-ex1]] + .Simplest Usage, Predefined Licenses + [example] + ==== + + When the [.filename]#README# of some software says "This software is under the terms of the GNU Lesser General Public License as published by the Free Software Foundation; either version 2.1 of the License, or (at your option) any later version." but does not provide the license file, use this: + + [.programlisting] + .... + LICENSE= LGPL21+ + .... + + When the software provides the license file, use this: + + [.programlisting] + .... + LICENSE= LGPL21+ + LICENSE_FILE= ${WRKSRC}/COPYING + .... + + ==== + + For the predefined licenses, the default permissions are `dist-mirror dist-sell pkg-mirror pkg-sell auto-accept`. + + [[licenses-license-list]] + .Predefined License List + [cols="1,1,1,1", frame="none", options="header"] + |=== + | Short Name + | Name + | Group + | Permissions + + |`AGPLv3` + |GNU Affero General Public License version 3 + |`FSF GPL OSI` + |(default) + + |`AGPLv3+` + |GNU Affero General Public License version 3 (or later) + |`FSF GPL OSI` + |(default) + + |`APACHE10` + |Apache License 1.0 + |`FSF` + |(default) + + |`APACHE11` + |Apache License 1.1 + |`FSF OSI` + |(default) + + |`APACHE20` + |Apache License 2.0 + |`FSF OSI` + |(default) + + |`ART10` + |Artistic License version 1.0 + |`OSI` + |(default) + + |`ART20` + |Artistic License version 2.0 + |`FSF GPL OSI` + |(default) + + |`ARTPERL10` + |Artistic License (perl) version 1.0 + |`OSI` + |(default) + + |`BSD` + |BSD license Generic Version (deprecated) + |`FSF OSI COPYFREE` + |(default) + + |`BSD2CLAUSE` + |BSD 2-clause "Simplified" License + |`FSF OSI COPYFREE` + |(default) + + |`BSD3CLAUSE` + |BSD 3-clause "New" or "Revised" License + |`FSF OSI COPYFREE` + |(default) + + |`BSD4CLAUSE` + |BSD 4-clause "Original" or "Old" License + |`FSF` + |(default) + + |`BSL` + |Boost Software License + |`FSF OSI COPYFREE` + |(default) + + |`CC-BY-1.0` + |Creative Commons Attribution 1.0 + | + |(default) + + |`CC-BY-2.0` + |Creative Commons Attribution 2.0 + | + |(default) + + |`CC-BY-2.5` + |Creative Commons Attribution 2.5 + | + |(default) + + |`CC-BY-3.0` + |Creative Commons Attribution 3.0 + | + |(default) + + |`CC-BY-4.0` + |Creative Commons Attribution 4.0 + | + |(default) + + |`CC-BY-NC-1.0` + |Creative Commons Attribution Non Commercial 1.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-2.0` + |Creative Commons Attribution Non Commercial 2.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-2.5` + |Creative Commons Attribution Non Commercial 2.5 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-3.0` + |Creative Commons Attribution Non Commercial 3.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-4.0` + |Creative Commons Attribution Non Commercial 4.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-ND-1.0` + |Creative Commons Attribution Non Commercial No Derivatives 1.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-ND-2.0` + |Creative Commons Attribution Non Commercial No Derivatives 2.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-ND-2.5` + |Creative Commons Attribution Non Commercial No Derivatives 2.5 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-ND-3.0` + |Creative Commons Attribution Non Commercial No Derivatives 3.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-ND-4.0` + |Creative Commons Attribution Non Commercial No Derivatives 4.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-SA-1.0` + |Creative Commons Attribution Non Commercial Share Alike 1.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-SA-2.0` + |Creative Commons Attribution Non Commercial Share Alike 2.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-SA-2.5` + |Creative Commons Attribution Non Commercial Share Alike 2.5 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-SA-3.0` + |Creative Commons Attribution Non Commercial Share Alike 3.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-NC-SA-4.0` + |Creative Commons Attribution Non Commercial Share Alike 4.0 + | + |`dist-mirror``pkg-mirror``auto-accept` + + |`CC-BY-ND-1.0` + |Creative Commons Attribution No Derivatives 1.0 + | + |(default) + + |`CC-BY-ND-2.0` + |Creative Commons Attribution No Derivatives 2.0 + | + |(default) + + |`CC-BY-ND-2.5` + |Creative Commons Attribution No Derivatives 2.5 + | + |(default) + + |`CC-BY-ND-3.0` + |Creative Commons Attribution No Derivatives 3.0 + | + |(default) + + |`CC-BY-ND-4.0` + |Creative Commons Attribution No Derivatives 4.0 + | + |(default) + + |`CC-BY-SA-1.0` + |Creative Commons Attribution Share Alike 1.0 + | + |(default) + + |`CC-BY-SA-2.0` + |Creative Commons Attribution Share Alike 2.0 + | + |(default) + + |`CC-BY-SA-2.5` + |Creative Commons Attribution Share Alike 2.5 + | + |(default) + + |`CC-BY-SA-3.0` + |Creative Commons Attribution Share Alike 3.0 + | + |(default) + + |`CC-BY-SA-4.0` + |Creative Commons Attribution Share Alike 4.0 + | + |(default) + + |`CC0-1.0` + |Creative Commons Zero v1.0 Universal + |`FSF GPL COPYFREE` + |(default) + + |`CDDL` + |Common Development and Distribution License + |`FSF OSI` + |(default) + + |`CPAL-1.0` + |Common Public Attribution License + |`FSF OSI` + |(default) + + |`ClArtistic` + |Clarified Artistic License + |`FSF GPL OSI` + |(default) + + |`EPL` + |Eclipse Public License + |`FSF OSI` + |(default) + + |`GFDL` + |GNU Free Documentation License + |`FSF` + |(default) + + |`GMGPL` + |GNAT Modified General Public License + |`FSF GPL OSI` + |(default) + + |`GPLv1` + |GNU General Public License version 1 + |`FSF GPL OSI` + |(default) + + |`GPLv1+` + |GNU General Public License version 1 (or later) + |`FSF GPL OSI` + |(default) + + |`GPLv2` + |GNU General Public License version 2 + |`FSF GPL OSI` + |(default) + + |`GPLv2+` + |GNU General Public License version 2 (or later) + |`FSF GPL OSI` + |(default) + + |`GPLv3` + |GNU General Public License version 3 + |`FSF GPL OSI` + |(default) + + |`GPLv3+` + |GNU General Public License version 3 (or later) + |`FSF GPL OSI` + |(default) + + |`GPLv3RLE` + |GNU GPL version 3 Runtime Library Exception + |`FSF GPL OSI` + |(default) + + |`GPLv3RLE+` + |GNU GPL version 3 Runtime Library Exception (or later) + |`FSF GPL OSI` + |(default) + + |`ISCL` + |Internet Systems Consortium License + |`FSF GPL OSI COPYFREE` + |(default) + + |`LGPL20` + |GNU Library General Public License version 2.0 + |`FSF GPL OSI` + |(default) + + |`LGPL20+` + |GNU Library General Public License version 2.0 (or later) + |`FSF GPL OSI` + |(default) + + |`LGPL21` + |GNU Lesser General Public License version 2.1 + |`FSF GPL OSI` + |(default) + + |`LGPL21+` + |GNU Lesser General Public License version 2.1 (or later) + |`FSF GPL OSI` + |(default) + + |`LGPL3` + |GNU Lesser General Public License version 3 + |`FSF GPL OSI` + |(default) + + |`LGPL3+` + |GNU Lesser General Public License version 3 (or later) + |`FSF GPL OSI` + |(default) + + |`LPPL10` + |LaTeX Project Public License version 1.0 + |`FSF OSI` + |`dist-mirror dist-sell` + + |`LPPL11` + |LaTeX Project Public License version 1.1 + |`FSF OSI` + |`dist-mirror dist-sell` + + |`LPPL12` + |LaTeX Project Public License version 1.2 + |`FSF OSI` + |`dist-mirror dist-sell` + + |`LPPL13` + |LaTeX Project Public License version 1.3 + |`FSF OSI` + |`dist-mirror dist-sell` + + |`LPPL13a` + |LaTeX Project Public License version 1.3a + |`FSF OSI` + |`dist-mirror dist-sell` + + |`LPPL13b` + |LaTeX Project Public License version 1.3b + |`FSF OSI` + |`dist-mirror dist-sell` + + |`LPPL13c` + |LaTeX Project Public License version 1.3c + |`FSF OSI` + |`dist-mirror dist-sell` + + |`MIT` + |MIT license / X11 license + |`COPYFREE FSF GPL OSI` + |(default) + + |`MPL10` + |Mozilla Public License version 1.0 + |`FSF OSI` + |(default) + + |`MPL11` + |Mozilla Public License version 1.1 + |`FSF OSI` + |(default) + + |`MPL20` + |Mozilla Public License version 2.0 + |`FSF OSI` + |(default) + + |`NCSA` + |University of Illinois/NCSA Open Source License + |`COPYFREE FSF GPL OSI` + |(default) + + |`NONE` + |No license specified + | + |`none` + + |`OFL10` + |SIL Open Font License version 1.0 (https://scripts.sil.org/OFL/) + |`FONTS` + |(default) + + |`OFL11` + |SIL Open Font License version 1.1 (https://scripts.sil.org/OFL/) + |`FONTS` + |(default) + + |`OWL` + |Open Works License (owl.apotheon.org) + |`COPYFREE` + |(default) + + |`OpenSSL` + |OpenSSL License + |`FSF` + |(default) + + |`PD` + |Public Domain + |`GPL COPYFREE` + |(default) + + |`PHP202` + |PHP License version 2.02 + |`FSF OSI` + |(default) + + |`PHP30` + |PHP License version 3.0 + |`FSF OSI` + |(default) + + |`PHP301` + |PHP License version 3.01 + |`FSF OSI` + |(default) + + |`PSFL` + |Python Software Foundation License + |`FSF GPL OSI` + |(default) + + |`PostgreSQL` + |PostgreSQL License + |`FSF GPL OSI COPYFREE` + |(default) + + |`RUBY` + |Ruby License + |`FSF` + |(default) + + |`UNLICENSE` + |The Unlicense + |`COPYFREE FSF GPL` + |(default) + + |`WTFPL` + |Do What the Fuck You Want To Public License version 2 + |`GPL FSF COPYFREE` + |(default) + + |`WTFPL1` + |Do What the Fuck You Want To Public License version 1 + |`GPL FSF COPYFREE` + |(default) + + |`ZLIB` + |zlib License + |`GPL FSF OSI` + |(default) + + |`ZPL21` + |Zope Public License version 2.1 + |`GPL OSI` + |(default) + |=== + + [[licenses-license_perms]] + === `LICENSE_PERMS` and `LICENSE_PERMS_NAME_` + + Permissions. use `none` if empty. + + .License Permissions List + [[licenses-license_perms-dist-mirror]] + `dist-mirror`:: + Redistribution of the distribution files is permitted. + The distribution files will be added to the FreeBSD `MASTER_SITE_BACKUP` CDN. + + [[licenses-license_perms-no-dist-mirror]] + `no-dist-mirror`:: + Redistribution of the distribution files is prohibited. + This is equivalent to setting crossref:special[porting-restrictions-restricted,`RESTRICTED`]. + The distribution files will _not_ be added to the FreeBSD `MASTER_SITE_BACKUP` CDN. + + [[licenses-license_perms-dist-sell]] + `dist-sell`:: + Selling of distribution files is permitted. + The distribution files will be present on the installer images. + + [[licenses-license_perms-no-dist-sell]] + `no-dist-sell`:: + Selling of distribution files is prohibited. + This is equivalent to setting crossref:special[porting-restrictions-no_cdrom,`NO_CDROM`]. + + [[licenses-license_perms-pkg-mirror]] + `pkg-mirror`:: + Free redistribution of package is permitted. + The package will be distributed on the FreeBSD package CDN https://pkg.freebsd.org/[https://pkg.freebsd.org/]. + + [[licenses-license_perms-no-pkg-mirror]] + `no-pkg-mirror`:: + Free redistribution of package is prohibited. + Equivalent to setting crossref:special[porting-restrictions-no_package,`NO_PACKAGE`]. + The package will _not_ be distributed from the FreeBSD package CDN https://pkg.freebsd.org/[https://pkg.freebsd.org/]. + + [[licenses-license_perms-pkg-sell]] + `pkg-sell`:: + Selling of package is permitted. + The package will be present on the installer images. + + [[licenses-license_perms-no-pkg-sell]] + `no-pkg-sell`:: + Selling of package is prohibited. + This is equivalent to setting crossref:special[porting-restrictions-no_cdrom,`NO_CDROM`]. + The package will _not_ be present on the installer images. + + [[licenses-license_perms-auto-accept]] + `auto-accept`:: + License is accepted by default. + Prompts to accept a license are not displayed unless the user has defined `LICENSES_ASK`. + Use this unless the license states the user must accept the terms of the license. + + [[licenses-license_perms-no-auto-accept]] + `no-auto-accept`:: + License is not accepted by default. + The user will always be asked to confirm the acceptance of this license. + This must be used if the license states that the user must accept its terms. + + When both `_permission_` and `no-_permission_` is present the `no-_permission_` will cancel `_permission_`. + + When `_permission_` is not present, it is considered to be a `no-_permission_`. + + [WARNING] + ==== + Some missing permissions will prevent a port (and all ports depending on it) from being usable by package users: + + A port without the `auto-accept` permission will never be be built and all the ports depending on it will be ignored. + + A port without the `pkg-mirror` permission, and any ports that depend on it, will be removed after the build, thus ensuring they are not distributed. + ==== + + [[licenses-license_perms-ex1]] + .Nonstandard License + [example] + ==== + Read the terms of the license and translate those using the available permissions. + + [.programlisting] + .... + LICENSE= UNKNOWN + LICENSE_NAME= unknown + LICENSE_TEXT= This program is NOT in public domain.\ + It can be freely distributed for non-commercial purposes only. + LICENSE_PERMS= dist-mirror no-dist-sell pkg-mirror no-pkg-sell auto-accept + .... + + ==== + + [[licenses-license_perms-ex2]] + .Standard and Nonstandard Licenses + [example] + ==== + + Read the terms of the license and express those using the available permissions. + In case of doubt, please ask for guidance on the {freebsd-ports}. + + [.programlisting] + .... + LICENSE= WARSOW GPLv2 + LICENSE_COMB= multi + LICENSE_NAME_WARSOW= Warsow Content License + LICENSE_FILE_WARSOW= ${WRKSRC}/docs/license.txt + LICENSE_PERMS_WARSOW= dist-mirror pkg-mirror auto-accept + .... + + When the permissions of the GPLv2 and the UNKNOWN licenses are mixed, the port ends up with `dist-mirror dist-sell pkg-mirror pkg-sell auto-accept dist-mirror no-dist-sell pkg-mirror no-pkg-sell auto-accept`. + The `no-_permissions_` cancel the _permissions_. + The resulting list of permissions are _dist-mirror pkg-mirror auto-accept_. + The distribution files and the packages will not be available on the installer images. + ==== + + [[licenses-license_groups]] + === `LICENSE_GROUPS` and `LICENSE_GROUPS_NAME` + + Groups the license belongs. + + .Predefined License Groups List + [[licenses-license_groups-FSF]] + `FSF`:: + Free Software Foundation Approved, see the https://www.fsf.org/licensing/[FSF Licensing & Compliance Team]. + + [[licenses-license_groups-GPL]] + `GPL`:: + GPL Compatible + + [[licenses-license_groups-OSI]] + `OSI`:: + OSI Approved, see the Open Source Initiative https://opensource.org/licenses/[Open Source Licenses] page. + + [[licenses-license_groups-COPYFREE]] + `COPYFREE`:: + Comply with Copyfree Standard Definition, see the https://copyfree.org/standard/licenses/[Copyfree Licenses] page. + + [[licenses-license_groups-FONTS]] + `FONTS`:: + Font licenses + + [[licenses-license_name]] + === `LICENSE_NAME` and `LICENSE_NAME_NAME` + + Full name of the license. + + [[licenses-license_name-ex1]] + .`LICENSE_NAME` + [example] + ==== + + [.programlisting] + .... + LICENSE= UNRAR + LICENSE_NAME= UnRAR License + LICENSE_FILE= ${WRKSRC}/license.txt + LICENSE_PERMS= dist-mirror dist-sell pkg-mirror pkg-sell auto-accept + .... + + ==== + + [[licenses-license_file]] + === `LICENSE_FILE` and `LICENSE_FILE_NAME` + + Full path to the file containing the license text, usually [.filename]#${WRKSRC}/some/file#. + If the file is not in the distfile, and its content is too long to be put in + crossref:makefiles[licenses-license_text,`LICENSE_TEXT`], put it in a new file in [.filename]#${FILESDIR}#. + + [[licenses-license_file-ex1]] + .`LICENSE_FILE` + [example] + ==== + + [.programlisting] + .... + LICENSE= GPLv3+ + LICENSE_FILE= ${WRKSRC}/COPYING + .... + + ==== + + [[licenses-license_text]] + === `LICENSE_TEXT` and `LICENSE_TEXT_NAME` + + Text to use as a license. + Useful when the license is not in the distribution files and its text is short. + + [[licenses-license_text-ex1]] + .`LICENSE_TEXT` + [example] + ==== + [.programlisting] + .... + LICENSE= UNKNOWN + LICENSE_NAME= unknown + LICENSE_TEXT= This program is NOT in public domain.\ + It can be freely distributed for non-commercial purposes only,\ + and THERE IS NO WARRANTY FOR THIS PROGRAM. + LICENSE_PERMS= dist-mirror no-dist-sell pkg-mirror no-pkg-sell auto-accept + .... + + ==== + + [[licenses-license_distfiles]] + === `LICENSE_DISTFILES` and `LICENSE_DISTFILES_NAME` + + The distribution files to which the licenses apply. + Defaults to all the distribution files. + + [[licenses-license_distfiles-ex1]] + .`LICENSE_DISTFILES` + [example] + ==== + Used when the distribution files do not all have the same license. + For example, one has a code license, and another has some artwork that cannot be redistributed: + + [.programlisting] + .... + MASTER_SITES= SF/some-game + DISTFILES= ${DISTNAME}${EXTRACT_SUFX} artwork.zip + + LICENSE= BSD3CLAUSE ARTWORK + LICENSE_COMB= dual + LICENSE_NAME_ARTWORK= The game artwork license + LICENSE_TEXT_ARTWORK= The README says that the files cannot be redistributed + LICENSE_PERMS_ARTWORK= pkg-mirror pkg-sell auto-accept + LICENSE_DISTFILES_BSD3CLAUSE= ${DISTNAME}${EXTRACT_SUFX} + LICENSE_DISTFILES_ARTWORK= artwork.zip + .... + + ==== + + [[licenses-license_comb]] + === `LICENSE_COMB` + + Set to `multi` if all licenses apply. + Set to `dual` if any license applies. + Defaults to `single`. + + [[licenses-license_comb-ex1]] + .Dual Licenses + [example] + ==== + + When a port says "This software may be distributed under the GNU General Public License or the Artistic License", + it means that either license can be used. + Use this: + + [.programlisting] + .... + LICENSE= ART10 GPLv1 + LICENSE_COMB= dual + .... + + If license files are provided, use this: + + [.programlisting] + .... + LICENSE= ART10 GPLv1 + LICENSE_COMB= dual + LICENSE_FILE_ART10= ${WRKSRC}/Artistic + LICENSE_FILE_GPLv1= ${WRKSRC}/Copying + .... + + ==== + + [[licenses-license_comb-ex2]] + .Multiple Licenses + [example] + ==== + + When part of a port has one license, and another part has a different license, use `multi`: + + [.programlisting] + .... + LICENSE= GPLv2 LGPL21+ + LICENSE_COMB= multi + .... + + ==== + + [[makefile-portscout]] + == `PORTSCOUT` + + Portscout is an automated distfile check utility for the FreeBSD Ports Collection, described in detail in crossref:keeping-up[distfile-survey,Portscout: the FreeBSD Ports Distfile Scanner]. + + `PORTSCOUT` defines special conditions within which the Portscout distfile scanner is restricted. + + Situations where `PORTSCOUT` is set include: + + * When distfiles have to be ignored for specific versions. For example, to exclude version _8.2_ and version _8.3_ from distfile version checks because they are known to be broken, add: + + + [.programlisting] + .... + PORTSCOUT= skipv:8.2,8.3 + .... + + * When distfile version checks have to be disabled completely. For example, if a port is not going to be updated ever again, add: + + + [.programlisting] + .... + PORTSCOUT= ignore:1 + .... + + * When specific versions or specific major and minor revisions of a distfile must be checked. For example, if only version _0.6.4_ must be monitored because newer versions have compatibility issues with FreeBSD, add: + + + [.programlisting] + .... + PORTSCOUT= limit:^0\.6\.4 + .... + + * When URLs listing the available versions differ from the download URLs. For example, to limit distfile version checks to the download page for the package:databases/pgtune[] port, add: + + + [.programlisting] + .... + PORTSCOUT= site:http://www.renpy.org/dl/release/ + .... + + [[makefile-depend]] + == Dependencies + + Many ports depend on other ports. + This is a very convenient feature of most Unix-like operating systems, including FreeBSD. + Multiple ports can share a common dependency, rather than bundling that dependency with every port or package that needs it. + There are seven variables that can be used to ensure that all the required bits will be on the user's machine. + There are also some pre-supported dependency variables for common cases, plus a few more to control the behavior of dependencies. + + [IMPORTANT] + ==== + When software has extra dependencies that provide extra features, the base dependencies listed in `*_DEPENDS` should include the extra dependencies that would benefit most users. + The base dependencies should never be a "minimal" dependency set. + The goal is not to include every dependency possible. Only include those that will benefit most people. + ==== + + [[makefile-lib_depends]] + === `LIB_DEPENDS` + + This variable specifies the shared libraries this port depends on. + It is a list of `_lib:dir_` tuples where `_lib_` is the name of the shared library, `_dir_` is the directory in which to find it in case it is not available. + For example, + + [.programlisting] + .... + LIB_DEPENDS= libjpeg.so:graphics/jpeg + .... + + will check for a shared jpeg library with any version, and descend into the [.filename]#graphics/jpeg# subdirectory of the ports tree to build and install it if it is not found. + + The dependency is checked twice, once from within the `build` target and then from within the `install` target. + Also, the name of the dependency is put into the package so that `pkg install` (see man:pkg-install[8]) will automatically install it if it is not on the user's system. + + [[makefile-run_depends]] + === `RUN_DEPENDS` + + This variable specifies executables or files this port depends on during run-time. + It is a list of ``_path:dir_``[:``_target_``] tuples where `_path_` is the name of the executable or file, _dir_ is the directory in which to find it in case it is not available, and _target_ is the target to call in that directory. + If _path_ starts with a slash (`/`), it is treated as a file and its existence is tested with `test -e`; otherwise, it is assumed to be an executable, and `which -s` is used to determine if the program exists in the search path. + + For example, + + [.programlisting] + .... + RUN_DEPENDS= ${LOCALBASE}/news/bin/innd:news/inn \ + xmlcatmgr:textproc/xmlcatmgr + .... + + will check if the file or directory [.filename]#/usr/local/news/bin/innd# exists, and build and install it from the [.filename]#news/inn# subdirectory of the ports tree if it is not found. + It will also see if an executable called `xmlcatmgr` is in the search path, and descend into [.filename]#textproc/xmlcatmgr# to build and install it if it is not found. + + [NOTE] + ==== + In this case, `innd` is actually an executable; + if an executable is in a place that is not expected to be in the search path, use the full pathname. + ==== + + [NOTE] + ==== + The official search `PATH` used on the ports build cluster is + + [.programlisting] + .... + /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin + .... + + ==== + + The dependency is checked from within the `install` target. + Also, the name of the dependency is put into the package so that `pkg install` (see man:pkg-install[8]) will automatically install it if it is not on the user's system. + The _target_ part can be omitted if it is the same as `DEPENDS_TARGET`. + + A quite common situation is when `RUN_DEPENDS` is literally the same as `BUILD_DEPENDS`, especially if ported software is written in a scripted language or if it requires the same build and run-time environment. + In this case, it is both tempting and intuitive to directly assign one to the other: + + [.programlisting] + .... + RUN_DEPENDS= ${BUILD_DEPENDS} + .... + + However, such assignment can pollute run-time dependencies with entries not defined in the port's original `BUILD_DEPENDS`. + This happens because of man:make[1]'s lazy evaluation of variable assignment. + Consider a [.filename]#Makefile# with `USE_*`, which are processed by [.filename]#ports/Mk/bsd.*.mk# to augment initial build dependencies. + For example, `USES= gmake` adds package:devel/gmake[] to `BUILD_DEPENDS`. + To prevent such additional dependencies from polluting `RUN_DEPENDS`, create another variable with the current content of `BUILD_DEPENDS` and assign it to both `BUILD_DEPENDS` and `RUN_DEPENDS`: + + [.programlisting] + .... + MY_DEPENDS= some:devel/some \ + other:lang/other + BUILD_DEPENDS= ${MY_DEPENDS} + RUN_DEPENDS= ${MY_DEPENDS} + .... + + [IMPORTANT] + ==== + _Do not_ use `:=` to assign `BUILD_DEPENDS` to `RUN_DEPENDS` or vice-versa. + All variables are expanded immediately, which is exactly the wrong thing to do and almost always a failure. + ==== + + [[makefile-build_depends]] + === `BUILD_DEPENDS` + + This variable specifies executables or files this port requires to build. + Like `RUN_DEPENDS`, it is a list of ``_path:dir_``[:``_target_``] tuples. + For example, + + [.programlisting] + .... + BUILD_DEPENDS= unzip:archivers/unzip + .... + + will check for an executable called `unzip`, and descend into the [.filename]#archivers/unzip# subdirectory of the ports tree to build and install it if it is not found. + + [NOTE] + ==== + "build" here means everything from extraction to compilation. + The dependency is checked from within the `extract` target. + The _target_ part can be omitted if it is the same as `DEPENDS_TARGET` + ==== + + [[makefile-fetch_depends]] + === `FETCH_DEPENDS` + + This variable specifies executables or files this port requires to fetch. + Like the previous two, it is a list of ``_path:dir_``[:``_target_``] tuples. + For example, + + [.programlisting] + .... + FETCH_DEPENDS= ncftp2:net/ncftp2 + .... + + will check for an executable called `ncftp2`, and descend into the [.filename]#net/ncftp2# subdirectory of the ports tree to build and install it if it is not found. + + The dependency is checked from within the `fetch` target. + The _target_ part can be omitted if it is the same as `DEPENDS_TARGET`. + + [[makefile-extract_depends]] + === `EXTRACT_DEPENDS` + + This variable specifies executables or files this port requires for extraction. + Like the previous, it is a list of ``_path:dir_``[:``_target_``] tuples. + For example, + + [.programlisting] + .... + EXTRACT_DEPENDS= unzip:archivers/unzip + .... + + will check for an executable called `unzip`, and descend into the [.filename]#archivers/unzip# subdirectory of the ports tree to build and install it if it is not found. + + The dependency is checked from within the `extract` target. + The _target_ part can be omitted if it is the same as `DEPENDS_TARGET`. + + [NOTE] + ==== + Use this variable only if the extraction does not already work (the default assumes `tar`) and cannot be made to work using `USES=tar`, `USES=lha` or `USES=zip` described in crossref:uses[uses,Using `USES` Macros]. + ==== + + [[makefile-patch_depends]] + === `PATCH_DEPENDS` + + This variable specifies executables or files this port requires to patch. + Like the previous, it is a list of ``_path:dir_``[:``_target_``] tuples. For example, + + [.programlisting] + .... + PATCH_DEPENDS= ${NONEXISTENT}:java/jfc:extract + .... + + will descend into the [.filename]#java/jfc# subdirectory of the ports tree to extract it. + + The dependency is checked from within the `patch` target. + The _target_ part can be omitted if it is the same as `DEPENDS_TARGET`. + + [[makefile-uses]] + === `USES` + + Parameters can be added to define different features and dependencies used by the port. + They are specified by adding this line to the [.filename]#Makefile#: + + [.programlisting] + .... + USES= feature[:arguments] + .... + + For the complete list of values, please see crossref:uses[uses,Using `USES` Macros]. + + [WARNING] + ==== + `USES` cannot be assigned after inclusion of [.filename]#bsd.port.pre.mk#. + ==== + + [[makefile-use-vars]] + === `USE_*` + + Several variables exist to define common dependencies shared by many ports. + Their use is optional, but helps to reduce the verbosity of the port [.filename]##Makefile##s. + Each of them is styled as `USE_*`. + These variables may be used only in the port [.filename]##Makefile##s and [.filename]#ports/Mk/bsd.*.mk#. + They are not meant for user-settable options - use `PORT_OPTIONS` for that purpose. + + [NOTE] + ==== + It is _always_ incorrect to set any `USE_*` in [.filename]#/etc/make.conf#. + For instance, setting + + [.programlisting] + .... + USE_GCC=X.Y + .... + + (where X.Y is version number) would add a dependency on gccXY for every port, including `lang/gccXY` itself! + ==== + + [[makefile-use-vars-table]] + .`USE_*` + [cols="1,1", frame="none", options="header"] + |=== + | Variable + | Means + + |`USE_GCC` + a| + + The port requires GCC (`gcc` or `{g-plus-plus}`) to build. + Some ports need a specific, old GCC version, some require modern, recent versions. + It is typically set to `yes` (means always use stable, modern GCC from ports per `GCC_DEFAULT` in [.filename]#Mk/bsd.default-versions.mk#). + This is also the default value. + The exact version can also be specified, with a value such as `10`. + GCC from the base system is used when it satisfies the requested version, otherwise an appropriate compiler is built from ports, and `CC` and `CXX` are adjusted accordingly. + The `:build` argument following the version specifier adds only a build time dependency to the port. + + For example: + [example] + ==== + [.programlisting] + .... + USE_GCC=yes # port requires a current version of GCC + USE_GCC=11:build # port requires GCC 11 at build time only + .... + ==== + + [NOTE] + ==== + `USE_GCC=any` is deprecated and should not be used in new ports + ==== + + |=== + + Variables related to gmake and [.filename]#configure# are described in crossref:special[building,Building Mechanisms], while autoconf, automake and libtool are described in crossref:special[using-autotools,Using GNU Autotools]. + Perl related variables are described in crossref:special[using-perl,Using Perl]. + X11 variables are listed in crossref:special[using-x11,Using X11]. + crossref:special[using-gnome,Using Gnome] deals with GNOME and crossref:special[using-kde,Using KDE] with KDE related variables. + crossref:special[using-java,Using Java] documents Java variables, while crossref:special[using-php,Web Applications, Apache and PHP] contains information on Apache, PHP and PEAR modules. + Python is discussed in crossref:special[using-python,Using Python], while Ruby in crossref:special[using-ruby,Using Ruby]. + crossref:special[using-sdl,Using SDL] provides variables used for SDL applications and finally, crossref:special[using-xfce,Using Xfce] contains information on Xfce. + + [[makefile-version-dependency]] + === Minimal Version of a Dependency + + A minimal version of a dependency can be specified in any `*_DEPENDS` except `LIB_DEPENDS` using this syntax: + + [.programlisting] + .... + p5-Spiffy>=0.26:devel/p5-Spiffy + .... + + The first field contains a dependent package name, which must match the entry in the package database, a comparison sign, and a package version. + The dependency is satisfied if p5-Spiffy-0.26 or newer is installed on the machine. + + [[makefile-note-on-dependencies]] + === Notes on Dependencies + + As mentioned above, the default target to call when a dependency is required is `DEPENDS_TARGET`. + It defaults to `install`. + This is a user variable; it is never defined in a port's [.filename]#Makefile#. + If the port needs a special way to handle a dependency, use the `:target` part of `*_DEPENDS` instead of redefining `DEPENDS_TARGET`. + + When running `make clean`, the port dependencies are automatically cleaned too. + If this is not desirable, define `NOCLEANDEPENDS` in the environment. + This may be particularly desirable if the port has something that takes a long time to rebuild in its dependency list, such as KDE, GNOME or Mozilla. + + To depend on another port unconditionally, use the variable `${NONEXISTENT}` as the first field of `BUILD_DEPENDS` or `RUN_DEPENDS`. + Use this only when the source of the other port is needed. + Compilation time can be saved by specifying the target too. + For instance + + [.programlisting] + .... + BUILD_DEPENDS= ${NONEXISTENT}:graphics/jpeg:extract + .... + + will always descend to the `jpeg` port and extract it. + + [[makefile-circular-dependencies]] + === Circular Dependencies Are Fatal + + [IMPORTANT] + ==== + Do not introduce any circular dependencies into the ports tree! + ==== + + The ports building technology does not tolerate circular dependencies. + If one is introduced, someone, somewhere in the world, will have their FreeBSD installation broken almost immediately, with many others quickly to follow. + These can really be hard to detect. + If in doubt, before making that change, make sure to run: `cd /usr/ports; make index`. + That process can be quite slow on older machines, but it may be able to save a large number of people, including yourself, a lot of grief in the process. + + [[makefile-automatic-dependencies]] + === Problems Caused by Automatic Dependencies + + Dependencies must be declared either explicitly or by using the + crossref:makefiles[makefile-options,OPTIONS framework]. + Using other methods like automatic detection complicates indexing, which causes problems for port and package management. + + [[makefile-automatic-dependencies-bad]] + .Wrong Declaration of an Optional Dependency + [example] + ==== + + [.programlisting] + .... + .include + + .if exists(${LOCALBASE}/bin/foo) + LIB_DEPENDS= libbar.so:foo/bar + .endif + .... + + ==== + + The problem with trying to automatically add dependencies is that files and settings outside an individual port can change at any time. + For example: an index is built, then a batch of ports are installed. + But one of the ports installs the tested file. + The index is now incorrect, because an installed port unexpectedly has a new dependency. + The index may still be wrong even after rebuilding if other ports also determine their need for dependencies based on the existence of other files. + + [[makefile-automatic-dependencies-good]] + .Correct Declaration of an Optional Dependency + [example] + ==== + + [.programlisting] + .... + OPTIONS_DEFINE= BAR + BAR_DESC= Calling cellphones via bar + + BAR_LIB_DEPENDS= libbar.so:foo/bar + .... + + ==== + + Testing option variables is the correct method. + It will not cause inconsistencies in the index of a batch of ports, provided the options were defined prior to the index build. + Simple scripts can then be used to automate the building, installation, and updating of these ports and their packages. + + [[makefile-masterdir]] + == Slave Ports and `MASTERDIR` + + If the port needs to build slightly different versions of packages by having a variable (for instance, resolution, or paper size) take different values, create one subdirectory per package to make it easier for users to see what to do, but try to share as many files as possible between ports. + Typically, by using variables cleverly, only a very short [.filename]#Makefile# is needed in all but one of the directories. + In the sole [.filename]#Makefile#, use `MASTERDIR` to specify the directory where the rest of the files are. + Also, use a variable as part of + crossref:makefiles[porting-pkgname,`PKGNAMESUFFIX`] so the packages will have different names. + + This will be best demonstrated by an example. + This is part of [.filename]#print/pkfonts300/Makefile#; + + [.programlisting] + .... + PORTNAME= pkfonts${RESOLUTION} + PORTVERSION= 1.0 + DISTFILES= pk${RESOLUTION}.tar.gz + + PLIST= ${PKGDIR}/pkg-plist.${RESOLUTION} + + .if !defined(RESOLUTION) + RESOLUTION= 300 + .else + .if ${RESOLUTION} != 118 && ${RESOLUTION} != 240 && \ + ${RESOLUTION} != 300 && ${RESOLUTION} != 360 && \ + ${RESOLUTION} != 400 && ${RESOLUTION} != 600 + .BEGIN: + @${ECHO_MSG} "Error: invalid value for RESOLUTION: \"${RESOLUTION}\"" + @${ECHO_MSG} "Possible values are: 118, 240, 300, 360, 400 and 600." + @${FALSE} + .endif + .endif + .... + + package:print/pkfonts300[] also has all the regular patches, package files, etc. + Running `make` there, it will take the default value for the resolution (300) and build the port normally. + + As for other resolutions, this is the _entire_ [.filename]#print/pkfonts360/Makefile#: + + [.programlisting] + .... + RESOLUTION= 360 + MASTERDIR= ${.CURDIR}/../pkfonts300 + + .include "${MASTERDIR}/Makefile" + .... + + ([.filename]#print/pkfonts118/Makefile#, [.filename]#print/pkfonts600/Makefile#, and all the other are similar). + `MASTERDIR` definition tells [.filename]#bsd.port.mk# that the regular set of subdirectories like `FILESDIR` and `SCRIPTDIR` are to be found under [.filename]#pkfonts300#. + The `RESOLUTION=360` line will override the `RESOLUTION=300` line in [.filename]#pkfonts300/Makefile# and the port will be built with resolution set to 360. + + [[makefile-manpages]] + == Man Pages + + If the port anchors its man tree somewhere other than `PREFIX`, use `MANDIRS` to specify those directories. + Note that the files corresponding to manual pages must be placed in [.filename]#pkg-plist# along with the rest of the files. + The purpose of `MANDIRS` is to enable automatic compression of manual pages, therefore the file names are suffixed with [.filename]#.gz#. + + [[makefile-info]] + == Info Files + + If the package needs to install GNU info files, list them in `INFO` (without the trailing `.info`), one entry per document. + These files are assumed to be installed to [.filename]#PREFIX/INFO_PATH#. + Change `INFO_PATH` if the package uses a different location. + However, this is not recommended. These entries contain just the path relative to [.filename]#PREFIX/INFO_PATH#. + For example, package:lang/gcc34[] installs info files to [.filename]#PREFIX/INFO_PATH/gcc34#, and `INFO` will be something like this: + + [.programlisting] + .... + INFO= gcc34/cpp gcc34/cppinternals gcc34/g77 ... + .... + + Appropriate installation/de-installation code will be automatically added to the temporary [.filename]#pkg-plist# before package registration. + + [[makefile-options]] + == Makefile Options + + Many applications can be built with optional or differing configurations. + Examples include choice of natural (human) language, GUI versus command-line, or type of database to support. + Users may need a different configuration than the default, so the ports system provides hooks the port author can use to control which variant will be built. Supporting these options properly will make users happy, and effectively provide two or more ports for the price of one. + + [[makefile-options-options]] + === `OPTIONS` + + [[makefile-options-background]] + ==== Background + + `OPTIONS_*` give the user installing the port a dialog showing the available options, and then saves those options to [.filename]#${PORT_DBDIR}/${OPTIONS_NAME}/options#. + The next time the port is built, the options are reused. + `PORT_DBDIR` defaults to [.filename]#/var/db/ports#. + `OPTIONS_NAME` is to the port origin with an underscore as the space separator, for example, for package:dns/bind99[] it will be `dns_bind99`. + + When the user runs `make config` (or runs `make build` for the first time), the framework checks for [.filename]#${PORT_DBDIR}/${OPTIONS_NAME}/options#. + If that file does not exist, the values of `OPTIONS_*` are used, and a dialog box is displayed where the options can be enabled or disabled. + Then [.filename]#options# is saved and the configured variables are used when building the port. + + If a new version of the port adds new `OPTIONS`, the dialog will be presented to the user with the saved values of old `OPTIONS` prefilled. + + `make showconfig` shows the saved configuration. + Use `make rmconfig` to remove the saved configuration. + + [[makefile-options-syntax]] + ==== Syntax + + `OPTIONS_DEFINE` contains a list of `OPTIONS` to be used. + These are independent of each other and are not grouped: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 OPT2 + .... + + Once defined, `OPTIONS` are described (optional, but strongly recommended): + + [.programlisting] + .... + OPT1_DESC= Describe OPT1 + OPT2_DESC= Describe OPT2 + OPT3_DESC= Describe OPT3 + OPT4_DESC= Describe OPT4 + OPT5_DESC= Describe OPT5 + OPT6_DESC= Describe OPT6 + .... + + [.filename]#ports/Mk/bsd.options.desc.mk# has descriptions for many common `OPTIONS`. + While often useful, override them if the description is insufficient for the port. + + [TIP] + ==== + When describing options, view it from the perspective of the user: "What functionality does it change?" and "Why would I want to enable this?" Do not just repeat the name. + For example, describing the `NLS` option as "include NLS support" does not help the user, who can already see the option name but may not know what it means. + Describing it as "Native Language Support via gettext utilities" is much more helpful. + ==== + + [IMPORTANT] + ==== + Option names are always in all uppercase. + They cannot use mixed case or lowercase. + ==== + + `OPTIONS` can be grouped as radio choices, where only one choice from each group is allowed: + + [.programlisting] + .... + OPTIONS_SINGLE= SG1 + OPTIONS_SINGLE_SG1= OPT3 OPT4 + .... + + [WARNING] + ==== + There _must_ be one of each `OPTIONS_SINGLE` group selected at all times for the options to be valid. + One option of each group _must_ be added to `OPTIONS_DEFAULT`. + ==== + + `OPTIONS` can be grouped as radio choices, where none or only one choice from each group is allowed: + + [.programlisting] + .... + OPTIONS_RADIO= RG1 + OPTIONS_RADIO_RG1= OPT7 OPT8 + .... + + `OPTIONS` can also be grouped as "multiple-choice" lists, where _at least one_ option must be enabled: + + [.programlisting] + .... + OPTIONS_MULTI= MG1 + OPTIONS_MULTI_MG1= OPT5 OPT6 + .... + + `OPTIONS` can also be grouped as "multiple-choice" lists, where none or any option can be enabled: + + [.programlisting] + .... + OPTIONS_GROUP= GG1 + OPTIONS_GROUP_GG1= OPT9 OPT10 + .... + + `OPTIONS` are unset by default, unless they are listed in `OPTIONS_DEFAULT`: + + [.programlisting] + .... + OPTIONS_DEFAULT= OPT1 OPT3 OPT6 + .... + + `OPTIONS` definitions must appear before the inclusion of [.filename]#bsd.port.options.mk#. + `PORT_OPTIONS` values can only be tested after the inclusion of [.filename]#bsd.port.options.mk#. + Inclusion of [.filename]#bsd.port.pre.mk# can be used instead, too, and is still widely used in ports written before the introduction of [.filename]#bsd.port.options.mk#. + But be aware that some variables will not work as expected after the inclusion of [.filename]#bsd.port.pre.mk#, typically some `USE_*` flags. + + [[ports-options-simple-use]] + .Simple Use of `OPTIONS` + [example] + ==== + + [.programlisting] + .... + OPTIONS_DEFINE= FOO BAR + OPTIONS_DEFAULT=FOO + + FOO_DESC= Option foo support + BAR_DESC= Feature bar support + + # Will add --with-foo / --without-foo + FOO_CONFIGURE_WITH= foo + BAR_RUN_DEPENDS= bar:bar/bar + + .include + .... + + ==== + + [[ports-options-check-unset]] + .Check for Unset Port `OPTIONS` + [example] + ==== + + [.programlisting] + .... + .if ! ${PORT_OPTIONS:MEXAMPLES} + CONFIGURE_ARGS+=--without-examples + .endif + .... + + The form shown above is discouraged. + The preferred method is using a configure knob to really enable and disable the feature to match the option: + + [.programlisting] + .... + # Will add --with-examples / --without-examples + EXAMPLES_CONFIGURE_WITH= examples + .... + + ==== + + [[ports-options-practical-use]] + .Practical Use of `OPTIONS` + [example] + ==== + + [.programlisting] + .... + OPTIONS_DEFINE= EXAMPLES + OPTIONS_DEFAULT= PGSQL LDAP SSL + + OPTIONS_SINGLE= BACKEND + OPTIONS_SINGLE_BACKEND= MYSQL PGSQL BDB + + OPTIONS_MULTI= AUTH + OPTIONS_MULTI_AUTH= LDAP PAM SSL + + EXAMPLES_DESC= Install extra examples + MYSQL_DESC= Use MySQL as backend + PGSQL_DESC= Use PostgreSQL as backend + BDB_DESC= Use Berkeley DB as backend + LDAP_DESC= Build with LDAP authentication support + PAM_DESC= Build with PAM support + SSL_DESC= Build with OpenSSL support + + # Will add USE_PGSQL=yes + PGSQL_USE= pgsql=yes + # Will add --enable-postgres / --disable-postgres + PGSQL_CONFIGURE_ENABLE= postgres + + ICU_LIB_DEPENDS= libicuuc.so:devel/icu + + # Will add --with-examples / --without-examples + EXAMPLES_CONFIGURE_WITH= examples + + # Check other OPTIONS + + .include + .... + + ==== + + [[makefile-options-default]] + ==== Default Options + + These options are always on by default. + + * `DOCS` - build and install documentation. + * `NLS` - Native Language Support. + * `EXAMPLES` - build and install examples. + * `IPV6` - IPv6 protocol support. + + [NOTE] + ==== + There is no need to add these to `OPTIONS_DEFAULT`. + To have them active, and show up in the options selection dialog, however, they must be added to `OPTIONS_DEFINE`. + ==== + + [[makefile-options-auto-activation]] + === Feature Auto-Activation + + When using a GNU configure script, keep an eye on which optional features are activated by auto-detection. + Explicitly disable optional features that are not needed by adding `--without-xxx` or `--disable-xxx` in `CONFIGURE_ARGS`. + + [[makefile-options-auto-activation-bad]] + .Wrong Handling of an Option + [example] + ==== + + [.programlisting] + .... + .if ${PORT_OPTIONS:MFOO} + LIB_DEPENDS+= libfoo.so:devel/foo + CONFIGURE_ARGS+= --enable-foo + .endif + .... + + ==== + + In the example above, imagine a library libfoo is installed on the system. + The user does not want this application to use libfoo, so he toggled the option off in the `make config` dialog. + But the application's configure script detects the library present in the system and includes its support in the resulting executable. + Now when the user decides to remove libfoo from the system, the ports system does not protest (no dependency on libfoo was recorded) but the application breaks. + + [[makefile-options-auto-activation-good]] + .Correct Handling of an Option + [example] + ==== + + [.programlisting] + .... + FOO_LIB_DEPENDS= libfoo.so:devel/foo + # Will add --enable-foo / --disable-foo + FOO_CONFIGURE_ENABLE= foo + .... + + ==== + + [NOTE] + ==== + Under some circumstances, the shorthand conditional syntax can cause problems with complex constructs. + The errors are usually `Malformed conditional`, an alternative syntax can be used. + + [.programlisting] + .... + .if !empty(VARIABLE:MVALUE) + .... + + as an alternative to + + [.programlisting] + .... + .if ${VARIABLE:MVALUE} + .... + + ==== + + [[options-helpers]] + === Options Helpers + + There are some macros to help simplify conditional values which differ based on the options set. + For easier access, a comprehensive list is provided: + + `PLIST_SUB`, `SUB_LIST`:: + For automatic `%%_OPT_%%` and `%%NO__OPT__%%` generation, see crossref:makefiles[options_sub, `OPTIONS_SUB`]. + + + For more complex usage, see crossref:makefiles[options-variables, Generic Variables Replacement, `OPT_VARIABLE` and `OPT_VARIABLE_OFF`]. + + `CONFIGURE_ARGS`:: + For `--enable-_x_` and `--disable-_x_`, see crossref:makefiles[options-configure_enable, `OPT_CONFIGURE_ENABLE`]. + + + For `--with-_x_` and `--without-_x_`, see crossref:makefiles[options-configure_with, `OPT_CONFIGURE_WITH`]. + + + For all other cases, see crossref:makefiles[options-configure_on, `OPT_CONFIGURE_ON` and `OPT_CONFIGURE_OFF`]. + + `CMAKE_ARGS`:: + For arguments that are booleans (`on`, `off`, `true`, `false`, `0`, `1`) see + crossref:makefiles[options-cmake_bool, `OPT_CMAKE_BOOL` and `OPT_CMAKE_BOOL_OFF`]. + + + For all other cases, see crossref:makefiles[options-cmake_on, `OPT_CMAKE_ON` and `OPT_CMAKE_OFF`]. + + `MESON_ARGS`:: + For arguments that take `true` or `false`, see crossref:makefiles[options-meson_true, `OPT_MESON_TRUE` and `OPT_MESON_FALSE`]. + + + For arguments that take `yes` or `no`, use crossref:makefiles[options-meson_yes, `OPT_MESON_YES` and `OPT_MESON_NO`]. + + + For arguments that take `enabled` or `disabled`, see crossref:makefiles[options-meson_enabled, `OPT_MESON_ENABLED` and `OPT_MESON_DISABLED`]. + + + For all other cases, use crossref:makefiles[options-meson_on, `OPT_MESON_ON` and `OPT_MESON_OFF`]. + + `QMAKE_ARGS`:: + See crossref:makefiles[options-qmake_on, `OPT_QMAKE_ON` and `OPT_QMAKE_OFF`]. + + `USE_*`:: + See crossref:makefiles[options-use, `OPT_USE` and `OPT_USE_OFF`]. + + `*_DEPENDS`:: + See crossref:makefiles[options-dependencies, Dependencies, `OPT_DEPTYPE` and `OPT_DEPTYPE_OFF`]. + + `*` (Any variable):: + The most used variables have direct helpers, see + crossref:makefiles[options-variables, Generic Variables Replacement, `OPT_VARIABLE` and `OPT_VARIABLE_OFF`]. + + + For any variable without a specific helper, see crossref:makefiles[options-vars, `OPT_VARS` and `OPT_VARS_OFF`]. + + Options dependencies:: + When an option need another option to work, see + crossref:makefiles[options-implies, `OPT_IMPLIES`]. + + Options conflicts:: + When an option cannot work if another is also enabled, see + crossref:makefiles[options-prevents, `OPT_PREVENTS` and `OPT_PREVENTS_MSG`]. + + Build targets:: + When an option need some extra processing, see + crossref:makefiles[options-targets, Additional Build Targets, `_target_-_OPT_-on` and `_target_-_OPT_-off`]. + + [[options_sub]] + ==== `OPTIONS_SUB` + + If `OPTIONS_SUB` is set to `yes` then each of the options added to `OPTIONS_DEFINE` will be added to `PLIST_SUB` and `SUB_LIST`, for example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPTIONS_SUB= yes + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + PLIST_SUB+= OPT1="" NO_OPT1="@comment " + SUB_LIST+= OPT1="" NO_OPT1="@comment " + .else + PLIST_SUB+= OPT1="@comment " NO_OPT1="" + SUB_LIST+= OPT1="@comment " NO_OPT1="" + .endif + .... + + [NOTE] + ==== + The value of `OPTIONS_SUB` is ignored. + Setting it to any value will add `PLIST_SUB` and `SUB_LIST` entries for _all_ options. + ==== + + [[options-use]] + ==== `OPT_USE` and `OPT_USE_OFF` + + When option _OPT_ is selected, for each `_key=value_` pair in ``OPT_USE``, _value_ is appended to the corresponding `USE_KEY`. + If _value_ has spaces in it, replace them with commas and they will be changed back to spaces during processing. + `OPT_USE_OFF` works the same way, but when `OPT` is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_USES= xorg + OPT1_USE= mysql=yes xorg=x11,xextproto,xext,xrandr + OPT1_USE_OFF= openssl=yes + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + USE_MYSQL= yes + USES+= xorg + USE_XORG= x11 xextproto xext xrandr + .else + USE_OPENSSL= yes + .endif + .... + + [[options-configure-helpers]] + ==== `CONFIGURE_ARGS` Helpers + + [[options-configure_enable]] + ===== `OPT_CONFIGURE_ENABLE` + + When option _OPT_ is selected, for each _entry_ in `OPT_CONFIGURE_ENABLE` then `--enable-_entry_` is appended to `CONFIGURE_ARGS`. + When option _OPT_ is _not_ selected, `--disable-_entry_` is appended to `CONFIGURE_ARGS`. + An optional argument can be specified with an `=` symbol. + This argument is only appended to the `--enable-_entry_` configure option. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 OPT2 + OPT1_CONFIGURE_ENABLE= test1 test2 + OPT2_CONFIGURE_ENABLE= test2=exhaustive + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + CONFIGURE_ARGS+= --enable-test1 --enable-test2 + .else + CONFIGURE_ARGS+= --disable-test1 --disable-test2 + .endif + + .if ${PORT_OPTIONS:MOPT2} + CONFIGURE_ARGS+= --enable-test2=exhaustive + .else + CONFIGURE_ARGS+= --disable-test2 + .endif + .... + + [[options-configure_with]] + ===== `OPT_CONFIGURE_WITH` + + When option _OPT_ is selected, for each _entry_ in `_OPT_CONFIGURE_WITH` then `--with-_entry_` is appended to `CONFIGURE_ARGS`. + When option _OPT_ is _not_ selected, `--without-_entry_` is appended to `CONFIGURE_ARGS`. + An optional argument can be specified with an `=` symbol. + This argument is only appended to the `--with-_entry_` configure option. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 OPT2 + OPT1_CONFIGURE_WITH= test1 + OPT2_CONFIGURE_WITH= test2=exhaustive + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 OPT2 + + .include + + .if ${PORT_OPTIONS:MOPT1} + CONFIGURE_ARGS+= --with-test1 + .else + CONFIGURE_ARGS+= --without-test1 + .endif + + .if ${PORT_OPTIONS:MOPT2} + CONFIGURE_ARGS+= --with-test2=exhaustive + .else + CONFIGURE_ARGS+= --without-test2 + .endif + .... + + [[options-configure_on]] + ===== `OPT_CONFIGURE_ON` and `OPT_CONFIGURE_OFF` + + When option _OPT_ is selected, the value of `OPT_CONFIGURE_ON`, if defined, is appended to `CONFIGURE_ARGS`. + `OPT_CONFIGURE_OFF` works the same way, but when `OPT` is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_CONFIGURE_ON= --add-test + OPT1_CONFIGURE_OFF= --no-test + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + CONFIGURE_ARGS+= --add-test + .else + CONFIGURE_ARGS+= --no-test + .endif + .... + + [TIP] + ==== + Most of the time, the helpers in crossref:makefiles[options-configure_enable, `OPT_CONFIGURE_ENABLE`] + and crossref:makefiles[options-configure_with, `OPT_CONFIGURE_WITH`] provide a shorter and more comprehensive functionality. + ==== + + [[options-cmake-helpers]] + ==== `CMAKE_ARGS` Helpers + + [[options-cmake_on]] + ===== `OPT_CMAKE_ON` and `OPT_CMAKE_OFF` + + When option _OPT_ is selected, the value of `OPT_CMAKE_ON`, if defined, is appended to `CMAKE_ARGS`. `OPT_CMAKE_OFF` works the same way, but when `OPT` is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_CMAKE_ON= -DTEST:BOOL=true -DDEBUG:BOOL=true + OPT1_CMAKE_OFF= -DOPTIMIZE:BOOL=true + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + CMAKE_ARGS+= -DTEST:BOOL=true -DDEBUG:BOOL=true + .else + CMAKE_ARGS+= -DOPTIMIZE:BOOL=true + .endif + .... + + [TIP] + ==== + + See crossref:makefiles[options-cmake_bool, `OPT_CMAKE_BOOL` and `OPT_CMAKE_BOOL_OFF`] for a shorter helper when the value is boolean. + ==== + + [[options-cmake_bool]] + ===== `OPT_CMAKE_BOOL` and `OPT_CMAKE_BOOL_OFF` + + When option _OPT_ is selected, for each _entry_ in `OPT_CMAKE_BOOL` then `-D_entry_:BOOL=true` is appended to `CMAKE_ARGS`. + When option _OPT_ is _not_ selected, `-D_entry_:BOOL=false` is appended to `CONFIGURE_ARGS`. + `OPT_CMAKE_BOOL_OFF` is the opposite, `-D_entry_:BOOL=false` is appended to `CMAKE_ARGS` when the option is selected, and `-D_entry_:BOOL=true` when the option is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_CMAKE_BOOL= TEST DEBUG + OPT1_CMAKE_BOOL_OFF= OPTIMIZE + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + CMAKE_ARGS+= -DTEST:BOOL=true -DDEBUG:BOOL=true \ + -DOPTIMIZE:BOOL=false + .else + CMAKE_ARGS+= -DTEST:BOOL=false -DDEBUG:BOOL=false \ + -DOPTIMIZE:BOOL=true + .endif + .... + + [[options-meson-helpers]] + ==== `MESON_ARGS` Helpers + + [[options-meson_on]] + ===== `OPT_MESON_ON` and `OPT_MESON_OFF` + + When option _OPT_ is selected, the value of `OPT_MESON_ON`, if defined, is appended to `MESON_ARGS`. + `OPT_MESON_OFF` works the same way, but when `OPT` is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_MESON_ON= -Dopt=1 + OPT1_MESON_OFF= -Dopt=2 + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + MESON_ARGS+= -Dopt=1 + .else + MESON_ARGS+= -Dopt=2 + .endif + .... + + [[options-meson_true]] + ===== `OPT_MESON_TRUE` and `OPT_MESON_FALSE` + + When option _OPT_ is selected, for each _entry_ in `OPT_MESON_TRUE` then `-D_entry_=true` is appended to `MESON_ARGS`. + When option _OPT_ is _not_ selected, `-D_entry_=false` is appended to `MESON_ARGS`. + `OPT_MESON_FALSE` is the opposite, `-D_entry_=false` is appended to `MESON_ARGS` when the option is selected, and `-D_entry_=true` when the option is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_MESON_TRUE= test debug + OPT1_MESON_FALSE= optimize + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + MESON_ARGS+= -Dtest=true -Ddebug=true \ + -Doptimize=false + .else + MESON_ARGS+= -Dtest=false -Ddebug=false \ + -Doptimize=true + .endif + .... + + [[options-meson_yes]] + ===== `OPT_MESON_YES` and `OPT_MESON_NO` + + When option _OPT_ is selected, for each _entry_ in `OPT_MESON_YES` then `-D_entry_=yes` is appended to `MESON_ARGS`. + When option _OPT_ is _not_ selected, `-D_entry_=no` is appended to `MESON_ARGS`. + `OPT_MESON_NO` is the opposite, `-D_entry_=no` is appended to `MESON_ARGS` when the option is selected, and `-D_entry_=yes` when the option is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_MESON_YES= test debug + OPT1_MESON_NO= optimize + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + MESON_ARGS+= -Dtest=yes -Ddebug=yes \ + -Doptimize=no + .else + MESON_ARGS+= -Dtest=no -Ddebug=no \ + -Doptimize=yes + .endif + .... + + [[options-meson_enabled]] + ===== `OPT_MESON_ENABLED` and `OPT_MESON_DISABLED` + + When option _OPT_ is selected, for each _entry_ in `OPT_MESON_ENABLED` then `-D_entry_=enabled` is appended to `MESON_ARGS`. + When option _OPT_ is _not_ selected, `-D_entry_=disabled` is appended to `MESON_ARGS`. + `OPT_MESON_DISABLED` is the opposite, `-D_entry_=disabled` is appended to `MESON_ARGS` when the option is selected, and `-D_entry_=enabled` when the option is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_MESON_ENABLED= test + OPT1_MESON_DISABLED= debug + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + MESON_ARGS+= -Dtest=enabled -Ddebug=disabled + .else + MESON_ARGS+= -Dtest=disabled -Ddebug=enabled + .endif + .... + + [[options-qmake_on]] + ==== `OPT_QMAKE_ON` and `OPT_QMAKE_OFF` + + When option _OPT_ is selected, the value of `OPT_QMAKE_ON`, if defined, is appended to `QMAKE_ARGS`. + `OPT_QMAKE_OFF` works the same way, but when `OPT` is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_QMAKE_ON= -DTEST:BOOL=true + OPT1_QMAKE_OFF= -DPRODUCTION:BOOL=true + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + QMAKE_ARGS+= -DTEST:BOOL=true + .else + QMAKE_ARGS+= -DPRODUCTION:BOOL=true + .endif + .... + + [[options-implies]] + ==== `OPT_IMPLIES` + + Provides a way to add dependencies between options. + + When _OPT_ is selected, all the options listed in this variable will be selected too. + Using the crossref:makefiles[options-configure_enable,`OPT_CONFIGURE_ENABLE`] described earlier to illustrate: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 OPT2 + OPT1_IMPLIES= OPT2 + + OPT1_CONFIGURE_ENABLE= opt1 + OPT2_CONFIGURE_ENABLE= opt2 + .... + + Is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 OPT2 + + .include + + .if ${PORT_OPTIONS:MOPT1} + CONFIGURE_ARGS+= --enable-opt1 + .else + CONFIGURE_ARGS+= --disable-opt1 + .endif + + .if ${PORT_OPTIONS:MOPT2} || ${PORT_OPTIONS:MOPT1} + CONFIGURE_ARGS+= --enable-opt2 + .else + CONFIGURE_ARGS+= --disable-opt2 + .endif + .... + + [[options-implies-ex1]] + .Simple Use of `OPT_IMPLIES` + [example] + ==== + + This port has a `X11` option, and a `GNOME` option that needs the `X11` option to be selected to build. + + [.programlisting] + .... + OPTIONS_DEFINE= X11 GNOME + OPTIONS_DEFAULT= X11 + + X11_USES= xorg + X11_USE= xorg=xi,xextproto + GNOME_USE= gnome=gtk30 + GNOME_IMPLIES= X11 + .... + + ==== + + [[options-prevents]] + ==== `OPT_PREVENTS` and `OPT_PREVENTS_MSG` + + Provides a way to add conflicts between options. + + When _OPT_ is selected, all the options listed in `OPT_PREVENTS` must be un-selected. + If `OPT_PREVENTS_MSG` is set and a conflict is triggered, its content will be shown explaining why they conflict. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 OPT2 + OPT1_PREVENTS= OPT2 + OPT1_PREVENTS_MSG= OPT1 and OPT2 enable conflicting options + .... + + Is roughly equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 OPT2 + + .include + + .if ${PORT_OPTIONS:MOPT2} && ${PORT_OPTIONS:MOPT1} + BROKEN= Option OPT1 conflicts with OPT2 (select only one) + .endif + .... + + The only difference is that the first one will write an error after running `make config`, suggesting changing the selected options. + + [[options-prevents-ex1]] + .Simple Use of `OPT_PREVENTS` + [example] + ==== + + This port has `X509` and `SCTP` options. + Both options add patches, but the patches conflict with each other, so they cannot be selected at the same time. + + [.programlisting] + .... + OPTIONS_DEFINE= X509 SCTP + + SCTP_PATCHFILES= ${PORTNAME}-6.8p1-sctp-2573.patch.gz:-p1 + SCTP_CONFIGURE_WITH= sctp + + X509_PATCH_SITES= http://www.roumenpetrov.info/openssh/x509/:x509 + X509_PATCHFILES= ${PORTNAME}-7.0p1+x509-8.5.diff.gz:-p1:x509 + X509_PREVENTS= SCTP + X509_PREVENTS_MSG= X509 and SCTP patches conflict + .... + + ==== + + [[options-vars]] + ==== `OPT_VARS` and `OPT_VARS_OFF` + + Provides a generic way to set and append to variables. + + [WARNING] + ==== + Before using `OPT_VARS` and `OPT_VARS_OFF`, see if there is already a more + specific helper available in crossref:makefiles[options-variables, Generic Variables Replacement, `OPT_VARIABLE` and `OPT_VARIABLE_OFF`]. + ==== + + When option _OPT_ is selected, and `OPT_VARS` defined, `_key_=_value_` and `_key_+=_value_` pairs are evaluated from `OPT_VARS`. + An `=` cause the existing value of `KEY` to be overwritten, an `+=` appends to the value. + `OPT_VARS_OFF` works the same way, but when `OPT` is _not_ selected. + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 OPT2 OPT3 + OPT1_VARS= also_build+=bin1 + OPT2_VARS= also_build+=bin2 + OPT3_VARS= bin3_build=yes + OPT3_VARS_OFF= bin3_build=no + + MAKE_ARGS= ALSO_BUILD="${ALSO_BUILD}" BIN3_BUILD="${BIN3_BUILD}" + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 OPT2 + + MAKE_ARGS= ALSO_BUILD="${ALSO_BUILD}" BIN3_BUILD="${BIN3_BUILD}" + + .include + + .if ${PORT_OPTIONS:MOPT1} + ALSO_BUILD+= bin1 + .endif + + .if ${PORT_OPTIONS:MOPT2} + ALSO_BUILD+= bin2 + .endif + + .if ${PORT_OPTIONS:MOPT2} + BIN3_BUILD= yes + .else + BIN3_BUILD= no + .endif + .... + + [IMPORTANT] + ==== + Values containing whitespace must be enclosed in quotes: + + [.programlisting] + .... + OPT_VARS= foo="bar baz" + .... + + This is due to the way man:make[1] variable expansion deals with whitespace. + When `OPT_VARS= foo=bar baz` is expanded, the variable ends up containing two strings, `foo=bar` and `baz`. + But the submitter probably intended there to be only one string, `foo=bar baz`. + Quoting the value prevents whitespace from being used as a delimiter. + + Also, _do not_ add extra spaces after the `_var_=` sign and before the value, it would also be split into two strings. + _This will not work_: + + [.programlisting] + .... + OPT_VARS= foo= bar + .... + + ==== + + [[options-dependencies]] + ==== Dependencies, `OPT_DEPTYPE` and `OPT_DEPTYPE_OFF` + + For any of these dependency types: + + * `PKG_DEPENDS` + * `EXTRACT_DEPENDS` + * `PATCH_DEPENDS` + * `FETCH_DEPENDS` + * `BUILD_DEPENDS` + * `LIB_DEPENDS` + * `RUN_DEPENDS` + + When option _OPT_ is selected, the value of `OPT_DEPTYPE`, if defined, is appended to `DEPTYPE`. + `OPT_DEPTYPE_OFF` works the same, but when `OPT` is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_LIB_DEPENDS= liba.so:devel/a + OPT1_LIB_DEPENDS_OFF= libb.so:devel/b + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + LIB_DEPENDS+= liba.so:devel/a + .else + LIB_DEPENDS+= libb.so:devel/b + .endif + .... + + [[options-variables]] + ==== Generic Variables Replacement, `OPT_VARIABLE` and `OPT_VARIABLE_OFF` + + For any of these variables: + + * `ALL_TARGET` + * `BINARY_ALIAS` + * `BROKEN` + * `CATEGORIES` + * `CFLAGS` + * `CONFIGURE_ENV` + * `CONFLICTS` + * `CONFLICTS_BUILD` + * `CONFLICTS_INSTALL` + * `CPPFLAGS` + * `CXXFLAGS` + * `DESKTOP_ENTRIES` + * `DISTFILES` + * `EXTRACT_ONLY` + * `EXTRA_PATCHES` + * `GH_ACCOUNT` + * `GH_PROJECT` + * `GH_SUBDIR` + * `GH_TAGNAME` + * `GH_TUPLE` + * `GL_ACCOUNT` + * `GL_COMMIT` + * `GL_PROJECT` + * `GL_SITE` + * `GL_SUBDIR` + * `GL_TUPLE` + * `IGNORE` + * `INFO` + * `INSTALL_TARGET` + * `LDFLAGS` + * `LIBS` + * `MAKE_ARGS` + * `MAKE_ENV` + * `MASTER_SITES` + * `PATCHFILES` + * `PATCH_SITES` + * `PLIST_DIRS` + * `PLIST_FILES` + * `PLIST_SUB` + * `PORTDOCS` + * `PORTEXAMPLES` + * `SUB_FILES` + * `SUB_LIST` + * `TEST_TARGET` + * `USES` + + When option _OPT_ is selected, the value of `OPT_ABOVEVARIABLE`, if defined, is appended to `_ABOVEVARIABLE_`. + `OPT_ABOVEVARIABLE_OFF` works the same way, but when `OPT` is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + OPT1_USES= gmake + OPT1_CFLAGS_OFF= -DTEST + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + .if ${PORT_OPTIONS:MOPT1} + USES+= gmake + .else + CFLAGS+= -DTEST + .endif + .... + + [NOTE] + ==== + Some variables are not in this list, in particular `PKGNAMEPREFIX` and `PKGNAMESUFFIX`. + This is intentional. + A port _must not_ change its name when its option set changes. + ==== + + [WARNING] + ==== + Some of these variables, at least `ALL_TARGET`, `DISTFILES` and `INSTALL_TARGET`, have their default values set _after_ the options are processed. + + With these lines in the [.filename]#Makefile#: + + [.programlisting] + .... + ALL_TARGET= all + + DOCS_ALL_TARGET= doc + .... + + If the `DOCS` option is enabled, `ALL_TARGET` will have a final value of `all doc`; if the option is disabled, it would have a value of `all`. + + With only the options helper line in the [.filename]#Makefile#: + + [.programlisting] + .... + DOCS_ALL_TARGET= doc + .... + + If the `DOCS` option is enabled, `ALL_TARGET` will have a final value of `doc`; if the option is disabled, it would have a value of `all`. + ==== + + [[options-targets]] + ==== Additional Build Targets, `_target_-_OPT_-on` and `_target_-_OPT_-off` + + These [.filename]#Makefile# targets can accept optional extra build targets: + + * `pre-fetch` + * `do-fetch` + * `post-fetch` + * `pre-extract` + * `do-extract` + * `post-extract` + * `pre-patch` + * `do-patch` + * `post-patch` + * `pre-configure` + * `do-configure` + * `post-configure` + * `pre-build` + * `do-build` + * `post-build` + * `pre-install` + * `do-install` + * `post-install` + * `post-stage` + * `pre-package` + * `do-package` + * `post-package` + + When option _OPT_ is selected, the target `_TARGET_-_OPT_-on`, if defined, is executed after `_TARGET_`. + `_TARGET_-_OPT_-off` works the same way, but when `OPT` is _not_ selected. + For example: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + post-patch-OPT1-on: + @${REINPLACE_CMD} -e '/opt1/s|/usr/bin/|${EXAMPLESDIR}/|' ${WRKSRC}/Makefile + + post-patch-OPT1-off: + @${REINPLACE_CMD} -e '/opt1/s|/usr/bin/|${PREFIX}/bin/|' ${WRKSRC}/Makefile + .... + + is equivalent to: + + [.programlisting] + .... + OPTIONS_DEFINE= OPT1 + + .include + + post-patch: + .if ${PORT_OPTIONS:MOPT1} + @${REINPLACE_CMD} -e '/opt1/s|/usr/bin/|${EXAMPLESDIR}/|' ${WRKSRC}/Makefile + .else + @${REINPLACE_CMD} -e '/opt1/s|/usr/bin/|${PREFIX}/bin/|' ${WRKSRC}/Makefile + .endif + .... + + [[makefile-wrkdir]] + == Specifying the Working Directory + + Each port is extracted into a working directory, which must be writable. + The ports system defaults to having `DISTFILES` unpack in to a directory called `${DISTNAME}`. + In other words, if the [.filename]#Makefile# has: + + [.programlisting] + .... + PORTNAME= foo + DISTVERSION= 1.0 + .... + + then the port's distribution files contain a top-level directory, [.filename]#foo-1.0#, and the rest of the files are located under that directory. + + A number of variables can be overridden if that is not the case. + + [[makefile-wrksrc]] + === `WRKSRC` + + The variable lists the name of the directory that is created when the application's distfiles are extracted. + If our previous example extracted into a directory called [.filename]#foo# (and not [.filename]#foo-1.0#) write: + + [.programlisting] + .... + WRKSRC= ${WRKDIR}/foo + .... + + or possibly + + [.programlisting] + .... + WRKSRC= ${WRKDIR}/${PORTNAME} + .... + + [[makefile-wrksrc_subdir]] + === `WRKSRC_SUBDIR` + + If the source files needed for the port are in a subdirectory of the extracted distribution file, set `WRKSRC_SUBDIR` to that directory. + + [.programlisting] + .... + WRKSRC_SUBDIR= src + .... + + [[makefile-no_wrksubdir]] + === `NO_WRKSUBDIR` + + If the port does not extract in to a subdirectory at all, then set `NO_WRKSUBDIR` to indicate that. + + [.programlisting] + .... + NO_WRKSUBDIR= yes + .... + + [NOTE] + ==== + Because `WRKDIR` is the only directory that is supposed to be writable during the build, and is used to store many files recording the status of the build, the port's extraction will be forced into a subdirectory. + ==== + + [[conflicts]] + == Conflict Handling + + There are three different variables to register a conflict between packages and ports: `CONFLICTS`, `CONFLICTS_INSTALL` and `CONFLICTS_BUILD`. + + [NOTE] + ==== + The conflict variables automatically set the variable `IGNORE`, which is more fully documented in crossref:porting-dads[dads-noinstall,Marking a Port Not Installable with `BROKEN`, `FORBIDDEN`, or `IGNORE`]. + ==== + + When removing one of several conflicting ports, it is advisable to retain `CONFLICTS` in those other ports for a few months to cater for users who only update once in a while. + + [[conclicts-conflicts_install]] + `CONFLICTS_INSTALL`:: + If the package cannot coexist with other packages (because of file conflicts, runtime incompatibilities, etc.). + `CONFLICTS_INSTALL` check is done after the build stage and prior to the install stage. + + [[conclicts-conflicts_build]] + `CONFLICTS_BUILD`:: + If the port cannot be built when other specific ports are already installed. + Build conflicts are not recorded in the resulting package. + + [[conclicts-conflicts]] + `CONFLICTS`:: + If the port cannot be built if a certain port is already installed and the resulting package cannot coexist with the other package. + `CONFLICTS` check is done prior to the build stage and prior to the install stage. + + Each space-separated item in the `CONFLICTS*` variable values is matched against packages except the one being built, using shell globbing rules. + This allows listing all flavors of a port in a conflict list instead of having to take pains to exclude the flavor being built from that list. + For example, if git-lite is installed, `CONFLICTS_INSTALL=git git-lite` would allow to perform: + [source,shell] + .... + % make -C devel/git FLAVOR=lite all deinstall install + .... + + But the following command would report a conflict, since the package base name installed is `git-lite`, while `git` would be built, but cannot be installed in addition to `git-lite`: + [source,shell] + .... + % make -C devel/git FLAVOR=default all deinstall install + .... + + Without that feature, the Makefile would need one `_flavor__CONFLICTS_INSTALL` for each flavor, listing every other flavor. + + The most common content of one of these variable is the package base of another port. + The package base is the package name without the appended version, it can be obtained by running `make -V PKGBASE`. + + [[conflicts-ex1]] + .Basic usage of `CONFLICTS*` + [example] + ==== + + package:dns/bind99[] cannot be installed if package:dns/bind910[] is present because they install same files. + First gather the package base to use: + + [source,shell] + .... + % make -C dns/bind99 -V PKGBASE + bind99 + % make -C dns/bind910 -V PKGBASE + bind910 + .... + + Then add to the [.filename]#Makefile# of package:dns/bind99[]: + + [.programlisting] + .... + CONFLICTS_INSTALL= bind910 + .... + + And add to the [.filename]#Makefile# of package:dns/bind910[]: + + [.programlisting] + .... + CONFLICTS_INSTALL= bind99 + .... + + ==== + + Sometimes, only certain versions of another port are incompatible. + When this is the case, use the full package name including the version. + If necessary, use shell globs like `*` and `?` so that all necessary versions are matched. + + [[conflicts-ex2]] + .Using `CONFLICTS*` With Globs. + [example] + ==== + + From versions from 2.0 and up-to 2.4.1_2, package:deskutils/gnotime[] used to install a bundled version of package:databases/qof[]. + + To reflect this past, the [.filename]#Makefile# of package:databases/qof[] contains: + + [.programlisting] + .... + CONFLICTS_INSTALL= gnotime-2.[0-3]* \ + gnotime-2.4.0* gnotime-2.4.1 \ + gnotime-2.4.1_[12] + .... + + The first entry match versions `2.0` through `2.3`, the second all the revisions of `2.4.0`, the third the exact `2.4.1` version, and the last the first and second revisions of the `2.4.1` version. + + package:deskutils/gnotime[] does not have any conflicts line because its current version does not conflict with anything else. + ==== + + The variable `DISABLE_CONFLICTS` may be temporarily set when making targets that are not affected by conflicts. + The variable is not to be set in port Makefiles. + + [source,shell] + .... + % make -DDISABLE_CONFLICTS patch + .... + + [[install]] + == Installing Files + + [IMPORTANT] + ==== + The `install` phase is very important to the end user because it adds files to their system. + All the additional commands run in the port [.filename]#Makefile#'s `*-install` targets should be echoed to the screen. + _Do not_ silence these commands with `@` or `.SILENT`. + ==== + + [[install-macros]] + === `INSTALL_*` Macros + + Use the macros provided in [.filename]#bsd.port.mk# to ensure correct modes of files in the port's `*-install` targets. + Set ownership directly in [.filename]#pkg-plist# with the corresponding entries, such as `@(_owner_,_group_,)`, `@owner _owner_`, and `@group _group_`. + These operators work until overridden, or until the end of [.filename]#pkg-plist#, so remember to reset them after they are no longer needed. + The default ownership is `root:wheel`. + See crossref:plist[plist-keywords-base,Base Keywords] for more information. + + * `INSTALL_PROGRAM` is a command to install binary executables. + * `INSTALL_SCRIPT` is a command to install executable scripts. + * `INSTALL_LIB` is a command to install shared libraries (but not static libraries). + * `INSTALL_KLD` is a command to install kernel loadable modules. Some architectures do not like having the modules stripped, so use this command instead of `INSTALL_PROGRAM`. + * `INSTALL_DATA` is a command to install sharable data, including static libraries. + * `INSTALL_MAN` is a command to install manpages and other documentation (it does not compress anything). + + These variables are set to the man:install[1] command with the appropriate flags for each situation. + + [IMPORTANT] + ==== + Do not use `INSTALL_LIB` to install static libraries, because stripping them renders them useless. Use `INSTALL_DATA` instead. + ==== + + [[install-strip]] + === Stripping Binaries and Shared Libraries + + Installed binaries should be stripped. Do not strip binaries manually unless absolutely required. + The `INSTALL_PROGRAM` macro installs and strips a binary at the same time. + The `INSTALL_LIB` macro does the same thing to shared libraries. + + When a file must be stripped, but neither `INSTALL_PROGRAM` nor `INSTALL_LIB` macros are desirable, `${STRIP_CMD}` strips the program or shared library. + This is typically done within the `post-install` target. For example: + + [.programlisting] + .... + post-install: + ${STRIP_CMD} ${STAGEDIR}${PREFIX}/bin/xdl + .... + + When multiple files need to be stripped: + + [.programlisting] + .... + post-install: + .for l in geometry media body track world + ${STRIP_CMD} ${STAGEDIR}${PREFIX}/lib/lib${PORTNAME}-${l}.so.0 + .endfor + .... + + Use man:file[1] on a file to determine if it has been stripped. + Binaries are reported by man:file[1] as `stripped`, or `not stripped`. + Additionally, man:strip[1] will detect programs that have already been stripped and exit cleanly. + + [IMPORTANT] + ==== + When `WITH_DEBUG` is defined, elf files _must not_ be stripped. + + The variables (`STRIP_CMD`, `INSTALL_PROGRAM`, `INSTALL_LIB`, ...) and crossref:uses[uses,`USES`] provided by the framework handle this automatically. + + Some software, add `-s` to their `LDFLAGS`, in this case, either remove `-s` if `WITH_DEBUG` is set, or remove it unconditionally and use `STRIP_CMD` in `post-install`. + ==== + + [[install-copytree]] + === Installing a Whole Tree of Files + + Sometimes, a large number of files must be installed while preserving their hierarchical organization. + For example, copying over a whole directory tree from `WRKSRC` to a target directory under `PREFIX`. + Note that `PREFIX`, `EXAMPLESDIR`, `DATADIR`, and other path variables must always be prepended with `STAGEDIR` to respect staging (see crossref:special[staging,Staging]). + + Two macros exist for this situation. + The advantage of using these macros instead of `cp` is that they guarantee proper file ownership and permissions on target files. + The first macro, `COPYTREE_BIN`, will set all the installed files to be executable, thus being suitable for installing into [.filename]#PREFIX/bin#. + The second macro, `COPYTREE_SHARE`, does not set executable permissions on files, and is therefore suitable for installing files under [.filename]#PREFIX/share# target. + + [.programlisting] + .... + post-install: + ${MKDIR} ${STAGEDIR}${EXAMPLESDIR} + (cd ${WRKSRC}/examples && ${COPYTREE_SHARE} . ${STAGEDIR}${EXAMPLESDIR}) + .... + + This example will install the contents of the [.filename]#examples# directory in the vendor distfile to the proper examples location of the port. + + [.programlisting] + .... + post-install: + ${MKDIR} ${STAGEDIR}${DATADIR}/summer + (cd ${WRKSRC}/temperatures && ${COPYTREE_SHARE} "June July August" ${STAGEDIR}${DATADIR}/summer) + .... + + And this example will install the data of summer months to the [.filename]#summer# subdirectory of a [.filename]#DATADIR#. + + Additional `find` arguments can be passed via the third argument to `COPYTREE_*` macros. + For example, to install all files from the first example except Makefiles, one can use these commands. + + [.programlisting] + .... + post-install: + ${MKDIR} ${STAGEDIR}${EXAMPLESDIR} + (cd ${WRKSRC}/examples && \ + ${COPYTREE_SHARE} . ${STAGEDIR}${EXAMPLESDIR} "! -name Makefile") + .... + + These macros do not add the installed files to [.filename]#pkg-plist#. + They must be added manually. + For optional documentation (`PORTDOCS`, see + crossref:makefiles[install-documentation, Install Additional Documentation]) and examples (`PORTEXAMPLES`), the `%%PORTDOCS%%` or `%%PORTEXAMPLES%%` prefixes must be prepended in [.filename]#pkg-plist#. + + [[install-documentation]] + === Install Additional Documentation + + If the software has some documentation other than the standard man and info pages that is useful for the user, install it under `DOCSDIR`. + This can be done, like the previous item, in the `post-install` target. + + Create a new directory for the port. + The directory name is `DOCSDIR`. + This usually equals `PORTNAME`. + However, if the user might want different versions of the port to be installed at the same time, the whole `PKGNAME` can be used. + + Since only the files listed in [.filename]#pkg-plist# are installed, it is safe to always install documentation to `STAGEDIR` (see crossref:special[staging,Staging]). + Hence `.if` blocks are only needed when the installed files are large enough to cause significant I/O overhead. + + [.programlisting] + .... + post-install: + ${MKDIR} ${STAGEDIR}${DOCSDIR} + ${INSTALL_DATA} ${WRKSRC}/docs/xvdocs.ps ${STAGEDIR}${DOCSDIR} + .... + + On the other hand, if there is a DOCS option in the port, install the documentation in a `post-install-DOCS-on` target. + These targets are described in crossref:makefiles[options-targets, Additional Build Targets, `_target_-_OPT_-on` and `_target_-_OPT_-off`]. + + Here are some handy variables and how they are expanded by default when used in the [.filename]#Makefile#: + + * `DATADIR` gets expanded to [.filename]#PREFIX/share/PORTNAME#. + * `DATADIR_REL` gets expanded to [.filename]#share/PORTNAME#. + * `DOCSDIR` gets expanded to [.filename]#PREFIX/share/doc/PORTNAME#. + * `DOCSDIR_REL` gets expanded to [.filename]#share/doc/PORTNAME#. + * `EXAMPLESDIR` gets expanded to [.filename]#PREFIX/share/examples/PORTNAME#. + * `EXAMPLESDIR_REL` gets expanded to [.filename]#share/examples/PORTNAME#. + + [NOTE] + ==== + The `DOCS` option only controls additional documentation installed in `DOCSDIR`. + It does not apply to standard man pages and info pages. + Things installed in `EXAMPLESDIR` are controlled by the `EXAMPLES` option. + ==== + + These variables are exported to `PLIST_SUB`. + Their values will appear there as pathnames relative to [.filename]#PREFIX# if possible. + That is, [.filename]#share/doc/PORTNAME# will be substituted for `%%DOCSDIR%%` in the packing list by default, and so on. + (See more on [.filename]#pkg-plist# substitution crossref:plist[plist-sub,here].) + + All conditionally installed documentation files and directories are included in [.filename]#pkg-plist# with the `%%PORTDOCS%%` prefix, for example: + + [.programlisting] + .... + %%PORTDOCS%%%%DOCSDIR%%/AUTHORS + %%PORTDOCS%%%%DOCSDIR%%/CONTACT + .... + + As an alternative to enumerating the documentation files in [.filename]#pkg-plist#, a port can set the variable `PORTDOCS` to a list of file names and shell glob patterns to add to the final packing list. + The names will be relative to `DOCSDIR`. + Therefore, a port that utilizes `PORTDOCS`, and uses a non-default location for its documentation, must set `DOCSDIR` accordingly. + If a directory is listed in `PORTDOCS` or matched by a glob pattern from this variable, the entire subtree of contained files and directories will be registered in the final packing list. + If the `DOCS` option has been unset then files and directories listed in `PORTDOCS` would not be installed or added to port packing list. + Installing the documentation at `PORTDOCS` as shown above remains up to the port itself. + A typical example of utilizing `PORTDOCS`: + + [.programlisting] + .... + PORTDOCS= README.* ChangeLog docs/* + .... + + [NOTE] + ==== + The equivalents of `PORTDOCS` for files installed under `DATADIR` and `EXAMPLESDIR` are `PORTDATA` and `PORTEXAMPLES`, respectively. + + The contents of [.filename]#pkg-message# are displayed upon installation. + See crossref:pkg-files[porting-message,the section on using [.filename]#pkg-message#] for details. + [.filename]#pkg-message# does not need to be added to [.filename]#pkg-plist#. + ==== + + [[install-subdirs]] + === Subdirectories Under `PREFIX` + + Try to let the port put things in the right subdirectories of `PREFIX`. + Some ports lump everything and put it in the subdirectory with the port's name, which is incorrect. + Also, many ports put everything except binaries, header files and manual pages in a subdirectory of [.filename]#lib#, which does not work well with the BSD paradigm. + Many of the files must be moved to one of these directories: [.filename]#etc# (setup/configuration files), [.filename]#libexec# (executables started internally), [.filename]#sbin# (executables for superusers/managers), [.filename]#info# (documentation for info browser) or [.filename]#share# (architecture independent files). + See man:hier[7] for details; the rules governing [.filename]#/usr# pretty much apply to [.filename]#/usr/local# too. + The exception are ports dealing with USENET "news". + They may use [.filename]#PREFIX/news# as a destination for their files. + + [[binary-alias]] + == Use `BINARY_ALIAS` to Rename Commands Instead of Patching the Build + + When `BINARY_ALIAS` is defined it will create symlinks of the given commands in a directory which will be prepended to `PATH`. + + Use it to substitute hardcoded commands the build phase relies on without having to patch any build files. + + [[binary-alias-ex1]] + .Using `BINARY_ALIAS` to Make `gsed` Available as `sed` + [example] + ==== + Some ports expect `sed` to behave like GNU sed and use features that man:sed[1] does not provide. + GNU sed is available from package:textproc/gsed[] on FreeBSD. + + Use `BINARY_ALIAS` to substitute `sed` with `gsed` for the duration of the build: + + [.programlisting] + .... + BUILD_DEPENDS= gsed:textproc/gsed + ... + BINARY_ALIAS= sed=gsed + .... + + ==== + + [[binary-alias-ex2]] + .Using `BINARY_ALIAS` to Provide Aliases for Hardcoded `python3` Commands + [example] + ==== + A port that has a hardcoded reference to `python3` in its build scripts will need to have it available in `PATH` at build time. + Use `BINARY_ALIAS` to create an alias that points to the right Python 3 binary: + + [.programlisting] + .... + USES= python:3.4+,build + ... + BINARY_ALIAS= python3=${PYTHON_CMD} + .... + + See crossref:special[using-python,Using Python] for more information about `USES=python`. + ==== + + [NOTE] + ==== + Binary aliases are created after the dependencies provided via `BUILD_DEPENDS` and `LIB_DEPENDS` are processed and before the `configure` target. + This leads to various limitations. + For example, programs installed via `TEST_DEPENDS` cannot be used to create a binary alias as test dependencies specified this way are processed after binary aliases are created. + ==== +diff --git a/documentation/content/en/books/porters-handbook/special/_index.adoc b/documentation/content/en/books/porters-handbook/special/_index.adoc +index 15a559a033..87604ac0cb 100644 +--- a/documentation/content/en/books/porters-handbook/special/_index.adoc ++++ b/documentation/content/en/books/porters-handbook/special/_index.adoc +@@ -1,5046 +1,5046 @@ + --- + title: Chapter 6. Special Considerations + prev: books/porters-handbook/makefiles + next: books/porters-handbook/flavors + description: Special considerations when creating a new FreeBSD Port + tags: ["special considerations", "Handling Symbolic Links", "Bundled Libraries"] + showBookMenu: true + weight: 6 + params: + path: "/books/porters-handbook/special/" + --- + + [[special]] + = Special Considerations + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :sectnumoffset: 6 + :partnums: + :source-highlighter: rouge + :experimental: + :images-path: books/porters-handbook/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + :imagesdir: ../../../../images/{images-path} + endif::[] + ifndef::book[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + toc::[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + toc::[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + This section explains the most common things to consider when creating a port. + + [[splitting-long-files]] + == Splitting long files + + Sometimes, port [.filename]#Makefiles# can be really long. + For example, rust ports can have a very long `CARGO_CRATES` list. + In other cases, the [.filename]#Makefile# might have code that varies depending on the architecture. + In such cases, it can be convenient to split the original [.filename]#Makefile# into several files. + [.filename]#bsd.port.mk# automatically includes some types of [.filename]#Makefiles# into the main port [.filename]#Makefile#. + + These are the files that the framework handles automatically if they are found: + + * [.filename]#Makefile.crates#. An example can be found in package:audio/ebur128[]. + * [.filename]#Makefile.inc#. An example can be found in package:net/ntp[]. + * [.filename]#Makefile.${ARCH}-${OPSYS}# + * [.filename]#Makefile.${OPSYS}#. An example can be found in package:net/cvsup-static[]. + * [.filename]#Makefile.${ARCH}# + * [.filename]#Makefile.local# + + It is also usual practice to split the packaging list of the port into several files if the list varies a lot from one architecture to another or depends on the selected flavor. + In this case, the [.filename]#pkg-plist# file for each architecture is named following the pattern [.filename]#pkg-plist.${ARCH}# or [.filename]#pkg-plist.${FLAVOR}#. + The framework does not create the packaging list automatically if multiple [.filename]#pkg-plist# files exist. + It is the responsibility of the porter to select the proper [.filename]#pkg-plist# and assign it to the `PLIST` variable. + Examples on how to deal with this can be found in package:audio/logitechmediaserver[] and package:deskutils/libportal[]. + + [[staging]] + == Staging + + [.filename]#bsd.port.mk# expects ports to work with a "stage directory". + This means that a port must not install files directly to the regular destination directories (that is, under `PREFIX`, for example) but instead into a separate directory from which the package is then built. + In many cases, this does not require root privileges, making it possible to build packages as an unprivileged user. + With staging, the port is built and installed into the stage directory, `STAGEDIR`. + A package is created from the stage directory and then installed on the system. Automake tools refer to this concept as `DESTDIR`, but in FreeBSD, `DESTDIR` has a different meaning (see crossref:testing[porting-prefix,`PREFIX` and `DESTDIR`]). + + [NOTE] + ==== + No port _really_ needs to be root. + It can mostly be avoided by using crossref:uses[uses-uidfix,`USES=uidfix`]. + If the port still runs commands like man:chown[8], man:chgrp[1], or forces owner or group with man:install[1] then use crossref:uses[uses-fakeroot,`USES=fakeroot`] to fake those calls. + Some patching of the port's [.filename]#Makefiles# will be needed. + ==== + + Meta ports, or ports that do not install files themselves but only depend on other ports, must avoid needlessly extracting the man:mtree[8] to the stage directory. + This is the basic directory layout of the package, and these empty directories will be seen as orphans. + To prevent man:mtree[8] extraction, add this line: + + [.programlisting] + .... + NO_MTREE= yes + .... + + [TIP] + ==== + Metaports should use crossref:special[uses-metaport,`USES=metaport`]. + It sets up defaults for ports that do not fetch, build, or install anything. + ==== + + Staging is enabled by prepending `STAGEDIR` to paths used in the `pre-install`, `do-install`, and `post-install` targets (see the examples through the book). + Typically, this includes `PREFIX`, `ETCDIR`, `DATADIR`, `EXAMPLESDIR`, `DOCSDIR`, and so on. + Directories should be created as part of the `post-install` target. + Avoid using absolute paths whenever possible. + + [TIP] + ==== + Ports that install kernel modules must prepend `STAGEDIR` to their destination, by default [.filename]#/boot/modules#. + ==== + + [[staging-symlink]] + === Handling Symbolic Links + + When creating a symbolic link, relative ones are strongly recommended. + Use `${RLN}` to create relative symbolic links. + It uses man:install[1] under the hood to automatically figure out the relative link to create. + + [[staging-ex1]] + .Create Relative Symbolic Links Automatically + [example] + ==== + `${RLN}` uses man:install[1]'s relative symbolic feature which frees the porter of computing the relative path. + + [.programlisting] + .... + ${RLN} ${STAGEDIR}${PREFIX}/lib/libfoo.so.42 ${STAGEDIR}${PREFIX}/lib/libfoo.so + ${RLN} ${STAGEDIR}${PREFIX}/libexec/foo/bar ${STAGEDIR}${PREFIX}/bin/bar + ${RLN} ${STAGEDIR}/var/cache/foo ${STAGEDIR}${PREFIX}/share/foo + .... + + Will generate: + + [source,shell] + .... + % ls -lF ${STAGEDIR}${PREFIX}/lib + lrwxr-xr-x 1 nobody nobody 181 Aug 3 11:27 libfoo.so@ -> libfoo.so.42 + -rwxr-xr-x 1 nobody nobody 15 Aug 3 11:24 libfoo.so.42* + % ls -lF ${STAGEDIR}${PREFIX}/bin + lrwxr-xr-x 1 nobody nobody 181 Aug 3 11:27 bar@ -> ../libexec/foo/bar + % ls -lF ${STAGEDIRDIR}${PREFIX}/share + lrwxr-xr-x 1 nobody nobody 181 Aug 3 11:27 foo@ -> ../../../var/cache/foo + .... + + ==== + + [[bundled-libs]] + == Bundled Libraries + + This section explains why bundled dependencies are considered bad and what to do about them. + + [[bundled-libs-why-bad]] + === Why Bundled Libraries Are Bad + + Some software requires the porter to locate third-party libraries and add the required dependencies to the port. + Other software bundles all necessary libraries into the distribution file. + The second approach seems easier at first, but there are some serious drawbacks: + + This list is loosely based on the https://fedoraproject.org/wiki/Packaging:No_Bundled_Libraries[Fedora] and https://wiki.gentoo.org/wiki/Why_not_bundle_dependencies[Gentoo] wikis, both licensed under the https://creativecommons.org/licenses/by-sa/3.0/[CC-BY-SA 3.0] license. + + Security:: + If vulnerabilities are found in the upstream library and fixed there, they might not be fixed in the library bundled with the port. + One reason could be that the author is not aware of the problem. + This means that the porter must fix them, or upgrade to a non-vulnerable version, and send a patch to the author. + This all takes time, which results in software being vulnerable longer than necessary. + This in turn makes it harder to coordinate a fix without unnecessarily leaking information about the vulnerability. + + Bugs:: + This problem is similar to the problem with security in the last paragraph, but generally less severe. + + Forking:: + It is easier for the author to fork the upstream library once it is bundled. + While convenient on first sight, it means that the code diverges from upstream making it harder to address security or other problems with the software. + A reason for this is that patching becomes harder. + + + Another problem of forking is that because code diverges from upstream, bugs get solved over and over again instead of just once at a central location. + This defeats the idea of open source software in the first place. + + Symbol collision:: + When a library is installed on the system, it might collide with the bundled version. + This can cause immediate errors at compile or link time. + It can also cause errors when running the program which might be harder to track down. + The latter problem could be caused because the versions of the two libraries are incompatible. + + Licensing:: + When bundling projects from different sources, license issues can arise more easily, especially when licenses are incompatible. + + Waste of resources:: + Bundled libraries waste resources on several levels. + It takes longer to build the actual application, especially if these libraries are already present on the system. + At run-time, they can take up unnecessary memory when the system-wide library is already loaded by one program and the bundled library is loaded by another program. + + Waste of effort:: + When a library needs patches for FreeBSD, these patches have to be duplicated again in the bundled library. + This wastes developer time because the patches might not apply cleanly. + It can also be hard to notice that these patches are required in the first place. + + [[bundled-libs-practices]] + === What to do About Bundled Libraries + + Whenever possible, use the unbundled version of the library by adding a `LIB_DEPENDS` to the port. + If such a port does not exist yet, consider creating it. + + Only use bundled libraries if the upstream has a good track record on security and using unbundled versions leads to overly complex patches. + + [NOTE] + ==== + In some very special cases, for example emulators, like Wine, a port has to bundle libraries, because they are in a different architecture, or they have been modified to fit the software's use. + In that case, those libraries should not be exposed to other ports for linking. + Add `BUNDLE_LIBS=yes` to the port's [.filename]#Makefile#. + This will tell man:pkg[8] to not compute provided libraries. + Always ask the {portmgr} before adding this to a port. + ==== + + [[porting-shlibs]] + == Shared Libraries + + If the port installs one or more shared libraries, define a `USE_LDCONFIG` make variable, which will instruct a [.filename]#bsd.port.mk# to run `${LDCONFIG} -m` on the directory where the new library is installed (usually [.filename]#PREFIX/lib#) during `post-install` target to register it into the shared library cache. + This variable, when defined, will also facilitate addition of an appropriate `@exec /sbin/ldconfig -m` and `@unexec /sbin/ldconfig -R` pair into [.filename]#pkg-plist#, so that a user who installed the package can start using the shared library immediately and de-installation will not cause the system to still believe the library is there. + + [.programlisting] + .... + USE_LDCONFIG= yes + .... + + The default directory can be overridden by setting `USE_LDCONFIG` to a list of directories into which shared libraries are to be installed. + For example, if the port installs shared libraries into [.filename]#PREFIX/lib/foo# and [.filename]#PREFIX/lib/bar# use this in [.filename]#Makefile#: + + [.programlisting] + .... + USE_LDCONFIG= ${PREFIX}/lib/foo ${PREFIX}/lib/bar + .... + + Please double-check, often this is not necessary at all or can be avoided through `-rpath` or setting `LD_RUN_PATH` during linking (see package:lang/mosml[] for an example), or through a shell-wrapper which sets `LD_LIBRARY_PATH` before invoking the binary, like package:www/seamonkey[] does. + + When installing 32-bit libraries on a 64-bit system, use `USE_LDCONFIG32` instead. + + If the software uses crossref:special[using-autotools,autotools], and specifically `libtool`, add crossref:uses[uses-libtool,`USES=libtool`]. + + When the major library version number increments in the update to the new port version, all other ports that link to the affected library must have their `PORTREVISION` incremented, to force recompilation with the new library version. + + [[porting-restrictions]] + == Ports with Distribution Restrictions or Legal Concerns + + Licenses vary, and some of them place restrictions on how the application can be packaged, whether it can be sold for profit, and so on. + + [IMPORTANT] + ==== + It is the responsibility of a porter to read the licensing terms of the software and make sure that the FreeBSD project will not be held accountable for violating them by redistributing the source or compiled binaries either via FTP/HTTP or CD-ROM. + If in doubt, please contact the {freebsd-ports}. + ==== + + In situations like this, the variables described in the next sections can be set. + + [[porting-restrictions-no_package]] + === `NO_PACKAGE` + + This variable indicates that we may not generate a binary package of the application. + For instance, the license may disallow binary redistribution, or it may prohibit distribution of packages created from patched sources. + + However, the port's `DISTFILES` may be freely mirrored on FTP/HTTP. + They may also be distributed on a CD-ROM (or similar media) unless `NO_CDROM` is set as well. + + If the binary package is not generally useful, and the application must always be compiled from the source code, use `NO_PACKAGE`. + For example, if the application has configuration information that is site specific hard coded into it at compile time, set `NO_PACKAGE`. + + Set `NO_PACKAGE` to a string describing the reason why the package cannot be generated. + + [[porting-restrictions-no_cdrom]] + === `NO_CDROM` + + This variable alone indicates that, although we are allowed to generate binary packages, we may put neither those packages nor the port's `DISTFILES` onto a CD-ROM (or similar media) for resale. + However, the binary packages and the port's `DISTFILES` will still be available via FTP/HTTP. + + If this variable is set along with `NO_PACKAGE`, then only the port's `DISTFILES` will be available, and only via FTP/HTTP. + + Set `NO_CDROM` to a string describing the reason why the port cannot be redistributed on CD-ROM. + For instance, use this if the port's license is for "non-commercial" use only. + + [[porting-restrictions-nofetchfiles]] + === `NOFETCHFILES` + + Files defined in `NOFETCHFILES` are not fetchable from any of `MASTER_SITES`. + An example of such a file is when the file is supplied on CD-ROM by the vendor. + + Tools which check for the availability of these files on `MASTER_SITES` have to ignore these files and not report about them. + + [[porting-restrictions-restricted]] + === `RESTRICTED` + + Set this variable alone if the application's license permits neither mirroring the application's `DISTFILES` nor distributing the binary package in any way. + + Do not set `NO_CDROM` or `NO_PACKAGE` along with `RESTRICTED`, since the latter variable implies the former ones. + + Set `RESTRICTED` to a string describing the reason why the port cannot be redistributed. + Typically, this indicates that the port contains proprietary software and that the user will need to manually download the `DISTFILES`, + possibly after registering for the software or agreeing to accept the terms of an EULA. + + [[porting-restrictions-restricted_files]] + === `RESTRICTED_FILES` + + When `RESTRICTED` or `NO_CDROM` is set, this variable defaults to `${DISTFILES} ${PATCHFILES}`, otherwise it is empty. + If only some of the distribution files are restricted, then set this variable to list them. + + [[porting-restrictions-legal_text]] + === `LEGAL_TEXT` + + If the port has legal concerns not addressed by the above variables, set `LEGAL_TEXT` to a string explaining the concern. + For example, if special permission was obtained for FreeBSD to redistribute the binary, this variable must indicate so. + + [[porting-restrictions-legal]] + === [.filename]#/usr/ports/LEGAL# and `LEGAL` + + A port which sets any of the above variables must also be added to [.filename]#/usr/ports/LEGAL#. + The first column is a glob which matches the restricted distfiles. + The second column is the port's origin. + The third column is the output of `make -VLEGAL`. + + [[porting-restrictions-examples]] + === Examples + + The preferred way to state "the distfiles for this port must be fetched manually" is as follows: + + [.programlisting] + .... + .if !exists(${DISTDIR}/${DISTNAME}${EXTRACT_SUFX}) + IGNORE= may not be redistributed because of licensing reasons. Please visit some-website to accept their license and download ${DISTFILES} into ${DISTDIR} + .endif + .... + + This both informs the user, and sets the proper metadata on the user's machine for use by automated programs. + + Note that this stanza must be preceded by an inclusion of [.filename]#bsd.port.pre.mk#. + + [[building]] + == Building Mechanisms + + [[parallel-builds]] + === Building Ports in Parallel + + The FreeBSD ports framework supports parallel building using multiple `make` sub-processes, + which allows SMP systems to utilize all of their available CPU power, allowing port builds to be faster and more effective. + + This is achieved by passing `-jX` flag to man:make[1] running on vendor code. + This is the default build behavior of ports. + Unfortunately, not all ports handle parallel building well and it may be required to explicitly disable this feature by adding the `MAKE_JOBS_UNSAFE=yes` variable. + It is used when a port is known to be broken with `-jX` due to race conditions causing intermittent build failures. + + [IMPORTANT] + ==== + When setting `MAKE_JOBS_UNSAFE`, it is very important to explain either with a comment in the [.filename]#Makefile#, or at least in the commit message, _why_ the port does not build when enabling. + Otherwise, it is almost impossible to either fix the problem, or test if it has been fixed when committing an update at a later date. + ==== + + [[using-make]] + === `make`, `gmake`, and `imake` + + Several differing `make` implementations exist. Ported software often requires a particular implementation, like GNU `make`, known in FreeBSD as `gmake`. + + If the port uses GNU make, add `gmake` to `USES`. + + `MAKE_CMD` can be used to reference the specific command configured by the `USES` setting in the port's [.filename]#Makefile#. + Only use `MAKE_CMD` within the application [.filename]##Makefile##s in `WRKSRC` to call the `make` implementation expected by the ported software. + + If the port is an X application that uses imake to create [.filename]##Makefile##s from [.filename]##Imakefile##s, set `USES= imake`. + See the crossref:uses[uses-imake,`USES=imake`] section of crossref:uses[uses,Using `USES` Macros] for more details. + + If the port's source [.filename]#Makefile# has something other than `all` as the main build target, set `ALL_TARGET` accordingly. + The same goes for `install` and `INSTALL_TARGET`. + + [[using-configure]] + === `configure` Script + + If the port uses the `configure` script to generate [.filename]#Makefile# from [.filename]#Makefile.in#, set `GNU_CONFIGURE=yes`. + To give extra arguments to the `configure` script (the default argument is `--prefix=${PREFIX} --infodir=${PREFIX}/${INFO_PATH} --mandir=${PREFIX}/man --build=${CONFIGURE_TARGET}`), set those extra arguments in `CONFIGURE_ARGS`. + Extra environment variables can be passed using `CONFIGURE_ENV`. + + [[using-configure-variables]] + .Variables for Ports That Use `configure` + [cols="1,1", frame="none", options="header"] + |=== + | Variable + | Means + + |`GNU_CONFIGURE` + |The port uses `configure` script to prepare build. + + |`HAS_CONFIGURE` + |Same as `GNU_CONFIGURE`, except default configure target is not added to `CONFIGURE_ARGS`. + + |`CONFIGURE_ARGS` + |Additional arguments passed to `configure` script. + + |`CONFIGURE_ENV` + |Additional environment variables to be set for `configure` script run. + + |`CONFIGURE_TARGET` + |Override default configure target. Default value is `${MACHINE_ARCH}-portbld-freebsd${OSREL}`. + |=== + + [[using-cmake]] + === Using `cmake` + + For ports that use CMake, define `USES= cmake`. + + [[using-cmake-variables]] + .Variables for Ports That Use `cmake` + [cols="1,1", frame="none", options="header"] + |=== + | Variable + | Means + + |`CMAKE_ARGS` + |Port specific CMake flags to be passed to the `cmake` binary. + + |`CMAKE_ON` + |For each entry in `CMAKE_ON`, an enabled boolean value is added to + `CMAKE_ARGS`. See crossref:special[using-cmake-example2,`CMAKE_ON` and `CMAKE_OFF`]. + + |`CMAKE_OFF` + |For each entry in `CMAKE_OFF`, a disabled boolean value is added to + `CMAKE_ARGS`. See crossref:special[using-cmake-example2,`CMAKE_ON` and `CMAKE_OFF`]. + + |`CMAKE_BUILD_TYPE` + |Type of build (CMake predefined build profiles). Default is `Release`, or `Debug` if `WITH_DEBUG` is set. + + |`CMAKE_SOURCE_PATH` + |Path to the source directory. Default is `${WRKSRC}`. + + |`CONFIGURE_ENV` + |Additional environment variables to be set for the `cmake` binary. + |=== + + [[using-cmake-user-variables]] + .Variables the Users Can Define for `cmake` Builds + [cols="1,1", frame="none", options="header"] + |=== + | Variable + | Means + + |`CMAKE_NOCOLOR` + |Disables color build output. Default not set, unless `BATCH` or `PACKAGE_BUILDING` are set. + |=== + + CMake supports these build profiles: `Debug`, `Release`, `RelWithDebInfo` and `MinSizeRel`. + `Debug` and `Release` profiles respect system `\*FLAGS`, `RelWithDebInfo` and `MinSizeRel` will set `CFLAGS` to `-O2 -g` and `-Os -DNDEBUG` correspondingly. + The lower-cased value of `CMAKE_BUILD_TYPE` is exported to `PLIST_SUB` and must be used if the port installs [.filename]#*.cmake# depending on the build type (see package:devel/kf5-kcrash[] for an example). + Please note that some projects may define their own build profiles and/or force particular build type by setting `CMAKE_BUILD_TYPE` in [.filename]#CMakeLists.txt#. + To make a port for such a project respect `CFLAGS` and `WITH_DEBUG`, the `CMAKE_BUILD_TYPE` definitions must be removed from those files. + + Most CMake-based projects support an out-of-source method of building. + The out-of-source build for a port is the default setting. + An in-source build can be requested by using the `:insource` suffix. + With out-of-source builds, `CONFIGURE_WRKSRC`, `BUILD_WRKSRC` and `INSTALL_WRKSRC` will be set to `${WRKDIR}/.build` and this directory will be used to keep all files generated during configuration and build stages, leaving the source directory intact. + + [[using-cmake-example]] + .`USES= cmake` Example + [example] + ==== + This snippet demonstrates the use of CMake for a port. + `CMAKE_SOURCE_PATH` is not usually required, but can be set when the sources are not located in the top directory, or if only a subset of the project is intended to be built by the port. + + [.programlisting] + .... + USES= cmake + CMAKE_SOURCE_PATH= ${WRKSRC}/subproject + .... + + ==== + + [[using-cmake-example2]] + .`CMAKE_ON` and `CMAKE_OFF` + [example] + ==== + When adding boolean values to `CMAKE_ARGS`, it is easier to use the `CMAKE_ON` and `CMAKE_OFF` variables instead. This: + + [.programlisting] + .... + CMAKE_ON= VAR1 VAR2 + CMAKE_OFF= VAR3 + .... + + Is equivalent to: + + [.programlisting] + .... + CMAKE_ARGS= -DVAR1:BOOL=TRUE -DVAR2:BOOL=TRUE -DVAR3:BOOL=FALSE + .... + + [IMPORTANT] + ====== + This is only for the default values off `CMAKE_ARGS`. + The helpers described in crossref:makefiles[options-cmake_bool,`OPT_CMAKE_BOOL` and `OPT_CMAKE_BOOL_OFF`] use the same semantics, but for optional values. + ====== + + ==== + + [[using-scons]] + === Using `scons` + + If the port uses SCons, define `USES=scons`. + + To make third party [.filename]#SConstruct# respect everything that is passed to SCons in the environment (that is, most importantly, `CC/CXX/CFLAGS/CXXFLAGS`), patch [.filename]#SConstruct# so build `Environment` is constructed like this: + + [.programlisting] + .... + env = Environment(**ARGUMENTS) + .... + + It may be then modified with `env.Append` and `env.Replace`. + + [[using-cargo]] + === Building Rust Applications with `cargo` + + For ports that use Cargo, define `USES=cargo`. + + [[using-cargo-user-variables]] + .Variables the Users Can Define for `cargo` Builds + [cols="1,1,1", frame="none", options="header"] + |=== + | Variable + | Default + | Description + + |`CARGO_CRATES` + | + |List of crates the port depends on. Each entry needs to have a format like `cratename-semver` for example, `libc-0.2.40`. Port maintainers can generate this list from [.filename]#Cargo.lock# using `make cargo-crates`. Manually bumping crate versions is possible but be mindful of transitive dependencies. + If the list generated by `make cargo-crates` is big, it might be convenient to place it inside a `Makefile.crates` file in the top-level port directory. + If present, the ports framework sources that file automatically. + This help keep the main port Makefile within a manageable size. + + |`CARGO_FEATURES` + | + |List of application features to build (space separated list). To deactivate all default features add the special token `--no-default-features` to `CARGO_FEATURES`. Manually passing it to `CARGO_BUILD_ARGS`, `CARGO_INSTALL_ARGS`, and `CARGO_TEST_ARGS` is not needed. + + |`CARGO_CARGOTOML` + |`${WRKSRC}/Cargo.toml` + |The path to the [.filename]#Cargo.toml# to use. + + |`CARGO_CARGOLOCK` + |`${WRKSRC}/Cargo.lock` + |The path to the [.filename]#Cargo.lock# to use for `make cargo-crates`. It is possible to specify more than one lock file when necessary. + + |`CARGO_ENV` + | + |A list of environment variables to pass to Cargo similar to `MAKE_ENV`. + + |`RUSTFLAGS` + | + |Flags to pass to the Rust compiler. + + |`CARGO_CONFIGURE` + |`yes` + |Use the default `do-configure`. + + |`CARGO_UPDATE_ARGS` + | + |Extra arguments to pass to Cargo during the configure phase. Valid arguments can be looked up with `cargo update --help`. + + |`CARGO_BUILDDEP` + |`yes` + |Add a build dependency on package:lang/rust[]. + + |`CARGO_CARGO_BIN` + |`${LOCALBASE}/bin/cargo` + |Location of the `cargo` binary. + + |`CARGO_BUILD` + |`yes` + |Use the default `do-build`. + + |`CARGO_BUILD_ARGS` + | + |Extra arguments to pass to Cargo during the build phase. Valid arguments can be looked up with `cargo build --help`. + + |`CARGO_INSTALL` + |`yes` + |Use the default `do-install`. + + |`CARGO_INSTALL_ARGS` + | + |Extra arguments to pass to Cargo during the install phase. Valid arguments can be looked up with `cargo install --help`. + + |`CARGO_INSTALL_PATH` + |`.` + |Path to the crate to install. This is passed to `cargo install` via its `--path` argument. When multiple paths are specified `cargo install` is run multiple times. + + |`CARGO_TEST` + |`yes` + |Use the default `do-test`. + + |`CARGO_TEST_ARGS` + | + |Extra arguments to pass to Cargo during the test phase. Valid arguments can be looked up with `cargo test --help`. + + |`CARGO_TARGET_DIR` + |`${WRKDIR}/target` + |Location of the cargo output directory. + + |`CARGO_DIST_SUBDIR` + |[.filename]#rust/crates# + |Directory relative to `DISTDIR` where the crate distribution files will be stored. + + |`CARGO_VENDOR_DIR` + |`${WRKSRC}/cargo-crates` + |Location of the vendor directory where all crates will be extracted to. Try to keep this under `PATCH_WRKSRC`, so that patches can be applied easily. + + |`CARGO_USE_GITHUB` + |`no` + |Enable fetching of crates locked to specific Git commits on GitHub via `GH_TUPLE`. This will try to patch all [.filename]#Cargo.toml# under `WRKDIR` to point to the offline sources instead of fetching them from a Git repository during the build. + + |`CARGO_USE_GITLAB` + |`no` + |Same as `CARGO_USE_GITHUB` but for GitLab instances and `GL_TUPLE`. + |=== + + [[cargo-ex1]] + .Creating a Port for a Simple Rust Application + [example] + ==== + Creating a Cargo based port is a three stage process. + First we need to provide a ports template that fetches the application distribution file: + + [.programlisting] + .... + PORTNAME= tokei + DISTVERSIONPREFIX= v + DISTVERSION= 7.0.2 + CATEGORIES= devel + + MAINTAINER= tobik@FreeBSD.org + COMMENT= Display statistics about your code + WWW= https://github.com/XAMPPRocky/tokei/ + + USES= cargo + USE_GITHUB= yes + GH_ACCOUNT= Aaronepower + + .include + .... + + Generate an initial [.filename]#distinfo#: + + [source,shell] + .... + % make makesum + => Aaronepower-tokei-v7.0.2_GH0.tar.gz doesn't seem to exist in /usr/ports/distfiles/. + => Attempting to fetch https://codeload.github.com/Aaronepower/tokei/tar.gz/v7.0.2?dummy=/Aaronepower-tokei-v7.0.2_GH0.tar.gz + fetch: https://codeload.github.com/Aaronepower/tokei/tar.gz/v7.0.2?dummy=/Aaronepower-tokei-v7.0.2_GH0.tar.gz: size of remote file is not known + Aaronepower-tokei-v7.0.2_GH0.tar.gz 45 kB 239 kBps 00m00s + .... + + Now the distribution file is ready to use and we can go ahead and extract crate dependencies from the bundled [.filename]#Cargo.lock#: + + [source,shell] + .... + % make cargo-crates + CARGO_CRATES= aho-corasick-0.6.4 \ + ansi_term-0.11.0 \ + arrayvec-0.4.7 \ + atty-0.2.9 \ + bitflags-1.0.1 \ + byteorder-1.2.2 \ + [...] + .... + + The output of this command needs to be pasted directly into the Makefile: + + [.programlisting] + .... + PORTNAME= tokei + DISTVERSIONPREFIX= v + DISTVERSION= 7.0.2 + CATEGORIES= devel + + MAINTAINER= tobik@FreeBSD.org + COMMENT= Display statistics about your code + WWW= https://github.com/XAMPPRocky/tokei/ + + USES= cargo + USE_GITHUB= yes + GH_ACCOUNT= Aaronepower + + CARGO_CRATES= aho-corasick-0.6.4 \ + ansi_term-0.11.0 \ + arrayvec-0.4.7 \ + atty-0.2.9 \ + bitflags-1.0.1 \ + byteorder-1.2.2 \ + [...] + + .include + .... + + [.filename]#distinfo# needs to be regenerated to contain all the crate distribution files: + + [source,shell] + .... + % make makesum + => rust/crates/aho-corasick-0.6.4.tar.gz doesn't seem to exist in /usr/ports/distfiles/. + => Attempting to fetch https://crates.io/api/v1/crates/aho-corasick/0.6.4/download?dummy=/rust/crates/aho-corasick-0.6.4.tar.gz + rust/crates/aho-corasick-0.6.4.tar.gz 100% of 24 kB 6139 kBps 00m00s + => rust/crates/ansi_term-0.11.0.tar.gz doesn't seem to exist in /usr/ports/distfiles/. + => Attempting to fetch https://crates.io/api/v1/crates/ansi_term/0.11.0/download?dummy=/rust/crates/ansi_term-0.11.0.tar.gz + rust/crates/ansi_term-0.11.0.tar.gz 100% of 16 kB 21 MBps 00m00s + => rust/crates/arrayvec-0.4.7.tar.gz doesn't seem to exist in /usr/ports/distfiles/. + => Attempting to fetch https://crates.io/api/v1/crates/arrayvec/0.4.7/download?dummy=/rust/crates/arrayvec-0.4.7.tar.gz + rust/crates/arrayvec-0.4.7.tar.gz 100% of 22 kB 3237 kBps 00m00s + => rust/crates/atty-0.2.9.tar.gz doesn't seem to exist in /usr/ports/distfiles/. + => Attempting to fetch https://crates.io/api/v1/crates/atty/0.2.9/download?dummy=/rust/crates/atty-0.2.9.tar.gz + rust/crates/atty-0.2.9.tar.gz 100% of 5898 B 81 MBps 00m00s + => rust/crates/bitflags-1.0.1.tar.gz doesn't seem to exist in /usr/ports/distfiles/. + [...] + .... + + The port is now ready for a test build and further adjustments like creating a plist, writing a description, adding license information, options, etc. as normal. + + If you are not testing your port in a clean environment like with poudriere, remember to run `make clean` before any testing. + ==== + + [[cargo-ex2]] + .Enabling Additional Application Features + [example] + ==== + Some applications define additional features in their [.filename]#Cargo.toml#. + They can be compiled in by setting `CARGO_FEATURES` in the port. + + Here we enable Tokei's `json` and `yaml` features: + + [.programlisting] + .... + CARGO_FEATURES= json yaml + .... + + ==== + + [[cargo-ex4]] + .Encoding Application Features As Port Options + [example] + ==== + An example `[features]` section in [.filename]#Cargo.toml# could look like this: + + [.programlisting] + .... + [features] + pulseaudio_backend = ["librespot-playback/pulseaudio-backend"] + portaudio_backend = ["librespot-playback/portaudio-backend"] + default = ["pulseaudio_backend"] + .... + + `pulseaudio_backend` is a default feature. + It is always enabled unless we explicitly turn off default features by adding `--no-default-features` to `CARGO_FEATURES`. + Here we turn the `portaudio_backend` and `pulseaudio_backend` features into port options: + + [.programlisting] + .... + CARGO_FEATURES= --no-default-features + + OPTIONS_DEFINE= PORTAUDIO PULSEAUDIO + + PORTAUDIO_VARS= CARGO_FEATURES+=portaudio_backend + PULSEAUDIO_VARS= CARGO_FEATURES+=pulseaudio_backend + .... + + ==== + + [[cargo-ex3]] + .Listing Crate Licenses + [example] + ==== + Crates have their own licenses. + It is important to know what they are when adding a `LICENSE` block to the port (see crossref:makefiles[licenses,Licenses]). + The helper target `cargo-crates-licenses` will try to list all the licenses of all crates defined in `CARGO_CRATES`. + + [source,shell] + .... + % make cargo-crates-licenses + aho-corasick-0.6.4 Unlicense/MIT + ansi_term-0.11.0 MIT + arrayvec-0.4.7 MIT/Apache-2.0 + atty-0.2.9 MIT + bitflags-1.0.1 MIT/Apache-2.0 + byteorder-1.2.2 Unlicense/MIT + [...] + .... + + [NOTE] + ====== + The license names `make cargo-crates-licenses` outputs are SPDX 2.1 licenses expression which do not match the license names defined in the ports framework. + They need to be translated to the names from crossref:makefiles[licenses-license-list,Predefined License List]. + ====== + + ==== + + [[using-meson]] + === Using `meson` + + For ports that use Meson, define `USES=meson`. + + [[using-meson-variables]] + .Variables for Ports That Use `meson` + [cols="1,1", frame="none", options="header"] + |=== + | Variable + | Description + + |`MESON_ARGS` + |Port specific Meson flags to be passed to the `meson` binary. + + |`MESON_BUILD_DIR` + |Path to the build directory relative to `WRKSRC`. Default is `_build`. + |=== + + [[using-meson-example]] + .`USES=meson` Example + [example] + ==== + This snippet demonstrates the use of Meson for a port. + + [.programlisting] + .... + USES= meson + MESON_ARGS= -Dfoo=enabled + .... + + ==== + + [[using-go]] + === Building Go Applications + + For ports that use Go, define `USES=go`. + Refer to crossref:uses[uses-go,`go`] for a list of variables that can be set to control the build process. + + [[go-ex1]] + .Creating a Port for a Go Modules Based Application + [example] + ==== + In most cases, it is sufficient to set the `GO_MODULE` variable to the value specified by the `module` directive in `go.mod`: + + [.programlisting] + .... + PORTNAME= hey + DISTVERSIONPREFIX= v + DISTVERSION= 0.1.4 + CATEGORIES= benchmarks + + MAINTAINER= dmgk@FreeBSD.org + COMMENT= Tiny program that sends some load to a web application + WWW= https://github.com/rakyll/hey/ + + LICENSE= APACHE20 + LICENSE_FILE= ${WRKSRC}/LICENSE + + USES= go:modules + GO_MODULE= github.com/rakyll/hey + + PLIST_FILES= bin/hey + + .include + .... + + If the "easy" way is not adequate or more control over dependencies is needed, the full porting process is described below. + + Creating a Go-based port is a five-stage process. + First we need to provide a ports template that fetches the application distribution file: + + [.programlisting] + .... + PORTNAME= ghq + DISTVERSIONPREFIX= v + DISTVERSION= 0.12.5 + CATEGORIES= devel + + MAINTAINER= tobik@FreeBSD.org + COMMENT= Remote repository management made easy + WWW= https://github.com/x-motemen/ghq/ + + USES= go:modules + USE_GITHUB= yes + GH_ACCOUNT= motemen + + .include + .... + + Generate an initial [.filename]#distinfo#: + + [source,shell] + .... + % make makesum + ===> License MIT accepted by the user + => motemen-ghq-v0.12.5_GH0.tar.gz doesn't seem to exist in /usr/ports/distfiles/. + => Attempting to fetch https://codeload.github.com/motemen/ghq/tar.gz/v0.12.5?dummy=/motemen-ghq-v0.12.5_GH0.tar.gz + fetch: https://codeload.github.com/motemen/ghq/tar.gz/v0.12.5?dummy=/motemen-ghq-v0.12.5_GH0.tar.gz: size of remote file is not known + motemen-ghq-v0.12.5_GH0.tar.gz 32 kB 177 kBps 00s + .... + + Now the distribution file is ready to use and we can extract the required Go module dependencies. + This step requires having package:ports-mgmt/modules2tuple[] installed: + + [source,shell] + .... + % make gomod-vendor + [...] + GH_TUPLE= \ + Songmu:gitconfig:v0.0.2:songmu_gitconfig/vendor/github.com/Songmu/gitconfig \ + daviddengcn:go-colortext:186a3d44e920:daviddengcn_go_colortext/vendor/github.com/daviddengcn/go-colortext \ + go-yaml:yaml:v2.2.2:go_yaml_yaml/vendor/gopkg.in/yaml.v2 \ + golang:net:3ec191127204:golang_net/vendor/golang.org/x/net \ + golang:sync:112230192c58:golang_sync/vendor/golang.org/x/sync \ + golang:xerrors:3ee3066db522:golang_xerrors/vendor/golang.org/x/xerrors \ + motemen:go-colorine:45d19169413a:motemen_go_colorine/vendor/github.com/motemen/go-colorine \ + urfave:cli:v1.20.0:urfave_cli/vendor/github.com/urfave/cli + .... + + The output of this command needs to be pasted directly into the Makefile: + + [.programlisting] + .... + PORTNAME= ghq + DISTVERSIONPREFIX= v + DISTVERSION= 0.12.5 + CATEGORIES= devel + + MAINTAINER= tobik@FreeBSD.org + COMMENT= Remote repository management made easy + WWW= https://github.com/x-motemen/ghq/ + + USES= go:modules + USE_GITHUB= yes + GH_ACCOUNT= motemen + GH_TUPLE= Songmu:gitconfig:v0.0.2:songmu_gitconfig/vendor/github.com/Songmu/gitconfig \ + daviddengcn:go-colortext:186a3d44e920:daviddengcn_go_colortext/vendor/github.com/daviddengcn/go-colortext \ + go-yaml:yaml:v2.2.2:go_yaml_yaml/vendor/gopkg.in/yaml.v2 \ + golang:net:3ec191127204:golang_net/vendor/golang.org/x/net \ + golang:sync:112230192c58:golang_sync/vendor/golang.org/x/sync \ + golang:xerrors:3ee3066db522:golang_xerrors/vendor/golang.org/x/xerrors \ + motemen:go-colorine:45d19169413a:motemen_go_colorine/vendor/github.com/motemen/go-colorine \ + urfave:cli:v1.20.0:urfave_cli/vendor/github.com/urfave/cli + + .include + .... + + [.filename]#distinfo# needs to be regenerated to contain all the distribution files: + + [source,shell] + .... + % make makesum + => Songmu-gitconfig-v0.0.2_GH0.tar.gz doesn't seem to exist in /usr/ports/distfiles/. + => Attempting to fetch https://codeload.github.com/Songmu/gitconfig/tar.gz/v0.0.2?dummy=/Songmu-gitconfig-v0.0.2_GH0.tar.gz + fetch: https://codeload.github.com/Songmu/gitconfig/tar.gz/v0.0.2?dummy=/Songmu-gitconfig-v0.0.2_GH0.tar.gz: size of remote file is not known + Songmu-gitconfig-v0.0.2_GH0.tar.gz 5662 B 936 kBps 00s + => daviddengcn-go-colortext-186a3d44e920_GH0.tar.gz doesn't seem to exist in /usr/ports/distfiles/. + => Attempting to fetch https://codeload.github.com/daviddengcn/go-colortext/tar.gz/186a3d44e920?dummy=/daviddengcn-go-colortext-186a3d44e920_GH0.tar.gz + fetch: https://codeload.github.com/daviddengcn/go-colortext/tar.gz/186a3d44e920?dummy=/daviddengcn-go-colortext-186a3d44e920_GH0.tar.gz: size of remote file is not known + daviddengcn-go-colortext-186a3d44e920_GH0.tar. 4534 B 1098 kBps 00s + [...] + .... + + The port is now ready for a test build and further adjustments like creating a plist, writing a description, adding license information, options, etc. as normal. + + If you are not testing your port in a clean environment like with poudriere, remember to run `make clean` before any testing. + ==== + + [[go-ex2]] + .Setting Output Binary Name or Installation Path + [example] + ==== + Some ports need to install the resulting binary under a different name or to a path other than the default `${PREFIX}/bin`. + This can be done by using `GO_TARGET` tuple syntax, for example: + + [.programlisting] + .... + GO_TARGET= ./cmd/ipfs:ipfs-go + .... + + will install `ipfs` binary as `${PREFIX}/bin/ipfs-go` and + + [.programlisting] + .... + GO_TARGET= ./dnscrypt-proxy:${PREFIX}/sbin/dnscrypt-proxy + .... + + will install `dnscrypt-proxy` to `${PREFIX}/sbin`. + ==== + + [[using-cabal]] + === Building Haskell Applications with `cabal` + + For ports that use Cabal, build system defines `USES=cabal`. + Refer to crossref:uses[uses-cabal,`cabal`] for a list of variables that can be set to control the build process. + + [[cabal-ex1]] + .Creating a Port for a Hackage-hosted Haskell Application + [example] + ==== + When preparing a Haskell Cabal port, package:devel/hs-cabal-install[] and package:ports-mgmt/hs-cabal2tuple[] programs are required, so make sure they are installed beforehand. + First we need to define common ports variables that allow cabal-install to fetch the package distribution file: + + [.programlisting] + .... + PORTNAME= ShellCheck + DISTVERSION= 0.6.0 + CATEGORIES= devel + + MAINTAINER= haskell@FreeBSD.org + COMMENT= Shell script analysis tool + WWW= https://www.shellcheck.net/ + + USES= cabal + + .include + .... + + This minimal Makefile fetches the distribution file with the `cabal-extract` helper target: + + [source,shell] + .... + % make cabal-extract + [...] + Downloading the latest package list from hackage.haskell.org + cabal get ShellCheck-0.6.0 + Downloading ShellCheck-0.6.0 + Downloaded ShellCheck-0.6.0 + Unpacking to ShellCheck-0.6.0/ + .... + + Now that we have ShellCheck.cabal package description file under `${WRKSRC}`, we can use `cabal-configure` to generate the build plan: + + [source,shell] + .... + % make cabal-configure + [...] + Resolving dependencies... + Build profile: -w ghc-8.10.7 -O1 + In order, the following would be built (use -v for more details): + - Diff-0.4.1 (lib) (requires download & build) + - OneTuple-0.3.1 (lib) (requires download & build) + [...] + .... + + Once done, a list of required dependencies can generated: + + [source,shell] + .... + % make make-use-cabal + USE_CABAL= QuickCheck-2.12.6.1 \ + hashable-1.3.0.0 \ + integer-logarithms-1.0.3 \ + [...] + .... + + Haskell packages may contain revisions, just like FreeBSD ports. + Revisions can affect [.filename]#.cabal# files only. + Note additional version numbers after the `_` symbol. + Put newly generated `USE_CABAL` list instead of an old one. + + Finally, [.filename]#distinfo# needs to be regenerated to contain all the distribution files: + + [source,shell] + .... + % make makesum + => ShellCheck-0.6.0.tar.gz doesn't seem to exist in /usr/local/poudriere/ports/git/distfiles/cabal. + => Attempting to fetch https://hackage.haskell.org/package/ShellCheck-0.6.0/ShellCheck-0.6.0.tar.gz + ShellCheck-0.6.0.tar.gz 136 kB 642 kBps 00s + => QuickCheck-2.12.6.1/QuickCheck-2.12.6.1.tar.gz doesn't seem to exist in /usr/local/poudriere/ports/git/distfiles/cabal. + => Attempting to fetch https://hackage.haskell.org/package/QuickCheck-2.12.6.1/QuickCheck-2.12.6.1.tar.gz + QuickCheck-2.12.6.1/QuickCheck-2.12.6.1.tar.gz 65 kB 361 kBps 00s + [...] + .... + + The port is now ready for a test build and further adjustments like creating a plist, writing a description, adding license information, options, etc. as normal. + + If you are not testing your port in a clean environment like with poudriere, remember to run `make clean` before any testing. + ==== + + Some Haskell ports install various data files under `share/${PORTNAME}`. For such cases special handling is required on the port side. + The port should define the `CABAL_WRAPPER_SCRIPTS` knob listing each executable that is going to use data files. Moreover, in rare cases the program + being ported uses data files of other Haskell packages, in which case the `FOO_DATADIR_VARS` comes to the rescue. + + [[cabal-ex2]] + .Handling Data Files in a Haskell Port + [example] + ==== + `devel/hs-profiteur` is a Haskell application that generates a single-page HTML with some content. + + [.programlisting] + .... + PORTNAME= profiteur + + [...] + + USES= cabal + + USE_CABAL= OneTuple-0.3.1_2 \ + QuickCheck-2.14.2 \ + [...] + + .include + .... + + It installs HTML templates under `share/profiteur`, so we need to add `CABAL_WRAPPER_SCRIPTS` knob: + + [.programlisting] + .... + [...] + + USE_CABAL= OneTuple-0.3.1_2 \ + QuickCheck-2.14.2 \ + [...] + + + CABAL_WRAPPER_SCRIPTS= ${CABAL_EXECUTABLES} + + .include + .... + + The program also tries to access the `jquery.js` file, which is a part of `js-jquery-3.3.1` Haskell package. + For that file to be found, we need to make the wrapper script to look for `js-jquery` data files in `share/profiteur` too. + We use `profiteur_DATADIR_VARS` for this: + + [.programlisting] + .... + [...] + + CABAL_WRAPPER_SCRIPTS= ${CABAL_EXECUTABLES} + profiteur_DATADIR_VARS= js-jquery + + .include + .... + + Now the port will install the actual binary into `libexec/cabal/profiteur` and the script into `bin/profiteur`. + + ==== + + There is no easy way to find out a proper value for the `FOO_DATADIR_VARS` knob apart from running the program and checking that everything works. + Luckily, the need to use `FOO_DATADIR_VARS` is very rare. + + Another corner case when porting complex Haskell programs is the presence of VCS dependencies in the `cabal.project` file. + + [[cabal-ex3]] + .Porting Haskell Applications with VCS Dependencies + [example] + ==== + + `net-p2p/cardano-node` is an extremely complex piece of software. In its `cabal.project` there are a lot of blocks like this: + + [.programlisting] + .... + [...] + source-repository-package + type: git + location: https://github.com/input-output-hk/cardano-crypto + tag: f73079303f663e028288f9f4a9e08bcca39a923e + [...] + .... + + Dependencies of type `source-repository-package` are automatically pulled in by `cabal` during the build process. + Unfortunately, this makes use of the network after the `fetch` stage. This is disallowed by the ports framework. + These sources need to be listed in the port's Makefile. The `make-use-cabal` helper target can make it easy for packages hosted on GitHub. + Running this target after the usual `cabal-extract` and `cabal-configure` will produce not only the `USE_CABAL` knob, but also `GH_TUPLE`: + + [source,shell] + .... + % make make-use-cabal + USE_CABAL= Diff-0.4.1 \ + Glob-0.10.2_3 \ + HUnit-1.6.2.0 \ + [...] + + GH_TUPLE= input-output-hk:cardano-base:0f3a867493059e650cda69e20a5cbf1ace289a57:cardano_base/dist-newstyle/src/cardano-b_-c8db9876882556ed \ + input-output-hk:cardano-crypto:f73079303f663e028288f9f4a9e08bcca39a923e:cardano_crypto/dist-newstyle/src/cardano-c_-253fd88117badd8f \ + [...] + .... + + It might be useful to separate the `GH_TUPLE` items coming from `make-use-cabal` from the other ones to make it easy to update the port: + + [.programlisting] + .... + GH_TUPLE= input-output-hk:cardano-base:0f3a867493059e650cda69e20a5cbf1ace289a57:cardano_base/dist-newstyle/src/cardano-b_-c8db9876882556ed \ + input-output-hk:cardano-crypto:f73079303f663e028288f9f4a9e08bcca39a923e:cardano_crypto/dist-newstyle/src/cardano-c_-253fd88117badd8f \ + [...] + + GH_TUPLE+= bitcoin-core:secp256k1:ac83be33d0956faf6b7f61a60ab524ef7d6a473a:secp + .... + + Haskell ports with VCS dependencies also require the following hack for the time being: + + [.programlisting] + .... + BINARY_ALIAS= git=true + .... + + ==== + + [[using-autotools]] + == Using GNU Autotools + + If a port needs any of the GNU Autotools software, add `USES=autoreconf`. + See crossref:uses[uses-autoreconf,`autoreconf`] for more information. + + [[using-gettext]] + == Using GNU `gettext` + + [[using-gettext-basic]] + === Basic Usage + + If the port requires `gettext`, set `USES= gettext`, and the port will inherit a dependency on [.filename]#libintl.so# from package:devel/gettext[]. + Other values for `gettext` usage are listed in crossref:uses[uses-gettext,`USES=gettext`]. + + A rather common case is a port using `gettext` and `configure`. + Generally, GNU `configure` should be able to locate `gettext` automatically. + + [.programlisting] + .... + USES= gettext + GNU_CONFIGURE= yes + .... + + If it ever fails to, hints at the location of `gettext` can be passed in `CPPFLAGS` and `LDFLAGS` using `localbase` as follows: + + [.programlisting] + .... + USES= gettext localbase:ldflags + GNU_CONFIGURE= yes + .... + + [[using-gettext-optional]] + === Optional Usage + + Some software products allow for disabling NLS. + For example, through passing `--disable-nls` to `configure`. + In that case, the port must use `gettext` conditionally, depending on the status of the `NLS` option. + For ports of low to medium complexity, use this idiom: + + [.programlisting] + .... + GNU_CONFIGURE= yes + + OPTIONS_DEFINE= NLS + OPTIONS_SUB= yes + + NLS_USES= gettext + NLS_CONFIGURE_ENABLE= nls + + .include + .... + + Or using the older way of using options: + + [.programlisting] + .... + GNU_CONFIGURE= yes + + OPTIONS_DEFINE= NLS + + .include + + .if ${PORT_OPTIONS:MNLS} + USES+= gettext + PLIST_SUB+= NLS="" + .else + CONFIGURE_ARGS+= --disable-nls + PLIST_SUB+= NLS="@comment " + .endif + + .include + .... + + The next item on the to-do list is to arrange so that the message catalog files are included in the packing list conditionally. + The [.filename]#Makefile# part of this task is already provided by the idiom. + It is explained in the section on crossref:plist[plist-sub,advanced [.filename]#pkg-plist# practices]. + In a nutshell, each occurrence of `%%NLS%%` in [.filename]#pkg-plist# will be replaced by "`@comment `" if NLS is disabled, or by a null string if NLS is enabled. + Consequently, the lines prefixed by `%%NLS%%` will become mere comments in the final packing list if NLS is off; + otherwise the prefix will be just left out. Then insert `%%NLS%%` before each path to a message catalog file in [.filename]#pkg-plist#. + For example: + + [.programlisting] + .... + %%NLS%%share/locale/fr/LC_MESSAGES/foobar.mo + %%NLS%%share/locale/no/LC_MESSAGES/foobar.mo + .... + + In high complexity cases, more advanced techniques may be needed, such as crossref:plist[plist-dynamic,dynamic packing list generation]. + + [[using-gettext-catalog-directories]] + === Handling Message Catalog Directories + + There is a point to note about installing message catalog files. + The target directories for them, which reside under [.filename]#LOCALBASE/share/locale#, must not be created and removed by a port. + The most popular languages have their respective directories listed in [.filename]#PORTSDIR/Templates/BSD.local.dist#. + The directories for many other languages are governed by the package:devel/gettext[] port. + Consult its [.filename]#pkg-plist# and see whether the port is going to install a message catalog file for a unique language. + + [[using-perl]] + == Using Perl + + If `MASTER_SITES` is set to `CPAN`, the correct subdirectory is usually selected automatically. + If the default subdirectory is wrong, `CPAN/Module` can be used to change it. + `MASTER_SITES` can also be set to the old `MASTER_SITE_PERL_CPAN`, then the preferred value of `MASTER_SITE_SUBDIR` is the top-level hierarchy name. + For example, the recommended value for `p5-Module-Name` is `Module`. + The top-level hierarchy can be examined at https://cpan.org/modules/by-module/[cpan.org]. + This keeps the port working when the author of the module changes. + + The exception to this rule is when the relevant directory does not exist or the distfile does not exist in that directory. + In such case, using author's id as `MASTER_SITE_SUBDIR` is allowed. + The `CPAN:AUTHOR` macro can be used, which will be translated to the hashed author directory. + For example, `CPAN:AUTHOR` will be converted to `authors/id/A/AU/AUTHOR`. + + When a port needs Perl support, it must set `USES=perl5` with the optional `USE_PERL5` described in crossref:uses[uses-perl5,the perl5 USES description]. + + [[using-perl-variables]] + .Read-Only Variables for Ports That Use Perl + [cols="1,1", frame="none", options="header"] + |=== + | Read only variables + | Means + + |`PERL` + |The full path of the Perl 5 interpreter, either in the system or installed from a port, but without the version number. Use this when the software needs the path to the Perl interpreter. To replace "``#!``"lines in scripts, use crossref:uses[uses-shebangfix,`shebangfix`]. + + |`PERL_VERSION` + |The full version of Perl installed (for example, `5.8.9`). + + |`PERL_LEVEL` + |The installed Perl version as an integer of the form `MNNNPP` (for example, `500809`). + + |`PERL_ARCH` + |Where Perl stores architecture dependent libraries. Defaults to `${ARCH}-freebsd`. + + |`PERL_PORT` + |Name of the Perl port that is installed (for example, `perl5`). + + |`SITE_PERL` + |Directory name where site specific Perl packages go. This value is added to `PLIST_SUB`. + |=== + + [NOTE] + ==== + Ports of Perl modules which do not have an official website must link to `cpan.org` in the WWW line of [.filename]#Makefile#. + The preferred URL form is `https://search.cpan.org/dist/Module-Name/` (including the trailing slash). + ==== + + [NOTE] + ==== + Do not use `${SITE_PERL}` in dependency declarations. + Doing so assumes that [.filename]#perl5.mk# has been included, which is not always true. + Ports depending on this port will have incorrect dependencies if this port's files move later in an upgrade. + The right way to declare Perl module dependencies is shown in the example below. + ==== + + [[use-perl-dependency-example]] + .Perl Dependency Example + [example] + ==== + [.programlisting] + .... + p5-IO-Tee>=0.64:devel/p5-IO-Tee + .... + + ==== + + For Perl ports that install manual pages, the macro `PERL5_MAN3` and `PERL5_MAN1` can be used inside [.filename]#pkg-plist#. For example, + + [.programlisting] + .... + lib/perl5/5.14/man/man1/event.1.gz + lib/perl5/5.14/man/man3/AnyEvent::I3.3.gz + .... + + can be replaced with + + [.programlisting] + .... + %%PERL5_MAN1%%/event.1.gz + %%PERL5_MAN3%%/AnyEvent::I3.3.gz + .... + + [NOTE] + ==== + There are no `PERL5_MAN_x_` macros for the other sections (_x_ in `2` and `4` to `9`) because those get installed in the regular directories. + ==== + + [[use-perl-ex-build]] + .A Port Which Only Requires Perl to Build + [example] + ==== + As the default USE_PERL5 value is build and run, set it to: + + [.programlisting] + .... + USES= perl5 + USE_PERL5= build + .... + + ==== + + [[use-perl-ex-patch]] + .A Port Which Also Requires Perl to Patch + [example] + ==== + From time to time, using man:sed[1] for patching is not enough. + When using man:perl[1] is easier, use: + + [.programlisting] + .... + USES= perl5 + USE_PERL5= patch build run + .... + + ==== + + [[use-perl-ex-configure]] + .A Perl Module Which Needs `ExtUtils::MakeMaker` to Build + [example] + ==== + Most Perl modules come with a [.filename]#Makefile.PL# configure script. + In this case, set: + + [.programlisting] + .... + USES= perl5 + USE_PERL5= configure + .... + + ==== + + [[use-perl-ex-modbuild]] + .A Perl Module Which Needs `Module::Build` to Build + [example] + ==== + When a Perl module comes with a [.filename]#Build.PL# configure script, it can require Module::Build, in which case, set + + [.programlisting] + .... + USES= perl5 + USE_PERL5= modbuild + .... + + If it instead requires Module::Build::Tiny, set + + [.programlisting] + .... + USES= perl5 + USE_PERL5= modbuildtiny + .... + + ==== + + [[using-x11]] + == Using X11 + + [[x11-variables]] + === X.Org Components + + The X11 implementation available in The Ports Collection is X.Org. + If the application depends on X components, add `USES= xorg` and set `USE_XORG` to the list of required components. + A full list can be found in crossref:uses[uses-xorg,`xorg`]. + + The Mesa Project is an effort to provide free OpenGL implementation. + To specify a dependency on various components of this project, use `USES= gl` and `USE_GL`. + See crossref:uses[uses-gl,`gl`] for a full list of available components. + For backwards compatibility, the value of `yes` maps to `glu`. + + [[use-xorg-example]] + .`USE_XORG` Example + [example] + ==== + [.programlisting] + .... + USES= gl xorg + USE_GL= glu + USE_XORG= xrender xft xkbfile xt xaw + .... + + ==== + + [[using-xorg-variables]] + .Variables for Ports That Use X + [cols="1,1", frame="none"] + |=== + |`USES= imake` + |The port uses `imake`. + + |`XMKMF` + |Set to the path of `xmkmf` if not in the `PATH`. Defaults to `xmkmf -a`. + |=== + + [[using-x11-vars]] + .Using X11-Related Variables + [example] + ==== + [.programlisting] + .... + # Use some X11 libraries + USES= xorg + USE_XORG= x11 xpm + .... + + ==== + + [[x11-motif]] + === Ports That Require Motif + + If the port requires a Motif library, define `USES= motif` in the [.filename]#Makefile#. + Default Motif implementation is package:x11-toolkits/open-motif[]. + Users can choose package:x11-toolkits/lesstif[] instead by setting `WANT_LESSTIF` in their [.filename]#make.conf#. + Similarly package:x11-toolkits/open-motif-devel[] can be chosen by setting `WANT_OPEN_MOTIF_DEVEL` in [.filename]#make.conf#. + + `MOTIFLIB` will be set by [.filename]#motif.mk# to reference the appropriate Motif library. + Please patch the source of the port to use `${MOTIFLIB}` wherever the Motif library is referenced in the original [.filename]#Makefile# or [.filename]#Imakefile#. + + There are two common cases: + + * If the port refers to the Motif library as `-lXm` in its [.filename]#Makefile# or [.filename]#Imakefile#, substitute `${MOTIFLIB}` for it. + * If the port uses `XmClientLibs` in its [.filename]#Imakefile#, change it to `${MOTIFLIB} ${XTOOLLIB} ${XLIB}`. + + Note that `MOTIFLIB` (usually) expands to `-L/usr/local/lib -lXm -lXp` or `/usr/local/lib/libXm.a`, so there is no need to add `-L` or `-l` in front. + + [[x11-fonts]] + === X11 Fonts + + If the port installs fonts for the X Window System, put them in [.filename]#LOCALBASE/lib/X11/fonts/local#. + + [[x11-fake-display]] + === Getting a Fake `DISPLAY` with Xvfb + + Some applications require a working X11 display for compilation to succeed. + This poses a problem for machines that operate headless. + When this variable is used, the build infrastructure will start the virtual framebuffer X server. + The working `DISPLAY` is then passed to the build. + See crossref:uses[uses-display,`USES=display`] for the possible arguments. + + [.programlisting] + .... + USES= display + .... + + + [[desktop-entries]] + === Desktop Entries + + Desktop entries (https://standards.freedesktop.org/desktop-entry-spec/latest/[a Freedesktop standard]) provide a way to automatically adjust desktop features when a new program is installed, without requiring user intervention. + For example, newly-installed programs automatically appear in the application menus of compatible desktop environments. + Desktop entries originated in the GNOME desktop environment, but are now a standard and also work with KDE and Xfce. + This bit of automation provides a real benefit to the user, and desktop entries are encouraged for applications which can be used in a desktop environment. + + [[desktop-entries-predefined]] + ==== Using Predefined [.filename]#.desktop# Files + + Ports that include predefined [.filename]#*.desktop# must include those files in [.filename]#pkg-plist# and install them in the [.filename]#$LOCALBASE/share/applications# directory. + The crossref:makefiles[install-macros,`INSTALL_DATA` macro] is useful for installing these files. + + [[updating-desktop-database]] + ==== Updating Desktop Database + + If a port has a MimeType entry in its [.filename]#portname.desktop#, the desktop database must be updated after install and deinstall. + To do this, define `USES`= desktop-file-utils. + + [[desktop-entries-macro]] + ==== Creating Desktop Entries with `DESKTOP_ENTRIES` + + Desktop entries can be easily created for applications by using `DESKTOP_ENTRIES`. + A file named [.filename]#name.desktop# will be created, installed, and added to [.filename]#pkg-plist# automatically. + Syntax is: + + [.programlisting] + .... + DESKTOP_ENTRIES= "NAME" "COMMENT" "ICON" "COMMAND" "CATEGORY" StartupNotify + .... + + The list of possible categories is available on the https://standards.freedesktop.org/menu-spec/latest/apa.html[Freedesktop website]. + `StartupNotify` indicates whether the application is compatible with _startup notifications_. + These are typically a graphic indicator like a clock that appear at the mouse pointer, menu, or panel to give the user an indication when a program is starting. + A program that is compatible with startup notifications clears the indicator after it has started. + Programs that are not compatible with startup notifications would never clear the indicator (potentially confusing and infuriating the user), and must have `StartupNotify` set to `false` so the indicator is not shown at all. + + Example: + + [.programlisting] + .... + DESKTOP_ENTRIES= "ToME" "Roguelike game based on JRR Tolkien's work" \ + "${DATADIR}/xtra/graf/tome-128.png" \ + "tome -v -g" "Application;Game;RolePlaying;" \ + false + .... + + `DESKTOP_ENTRIES` are installed in the directory pointed to by the `DESKTOPDIR` + variable. + `DESKTOPDIR` defaults to [.filename]#${PREFIX}/share/applications# + + + [[using-gnome]] + == Using GNOME + + [[using-gnome-introduction]] + === Introduction + + This chapter explains the GNOME framework as used by ports. + The framework can be loosely divided into the base components, GNOME desktop components, and a few special macros that simplify the work of port maintainers. + + [[use-gnome]] + === Using `USE_GNOME` + + Adding this variable to the port allows the use of the macros and components defined in [.filename]#bsd.gnome.mk#. + The code in [.filename]#bsd.gnome.mk# adds the needed build-time, run-time or library dependencies or the handling of special files. + GNOME applications under FreeBSD use the `USE_GNOME` infrastructure. + Include all the needed components as a space-separated list. + The `USE_GNOME` components are divided into these virtual lists: basic components, GNOME 3 components and legacy components. + If the port needs only GTK3 libraries, this is the shortest way to define it: + + [.programlisting] + .... + USE_GNOME= gtk30 + .... + + `USE_GNOME` components automatically add the dependencies they need. + Please see crossref:special[gnome-components, GNOME Components] for an exhaustive list of all `USE_GNOME` components and which other components they imply and their dependencies. + + Here is an example [.filename]#Makefile# for a GNOME port that uses many of the techniques outlined in this document. + Please use it as a guide for creating new ports. + + [.programlisting] + .... + PORTNAME= regexxer + DISTVERSION= 0.10 + CATEGORIES= devel textproc gnome + MASTER_SITES= GNOME + + MAINTAINER= kwm@FreeBSD.org + COMMENT= Interactive tool for performing search and replace operations + WWW= http://regexxer.sourceforge.net/ + + USES= gettext gmake localbase:ldflags pathfix pkgconfig tar:xz + GNU_CONFIGURE= yes + USE_GNOME= gnomeprefix intlhack gtksourceviewmm3 + + GLIB_SCHEMAS= org.regexxer.gschema.xml + + .include + .... + + [NOTE] + ==== + The `USE_GNOME` macro without any arguments does not add any dependencies to the port. + `USE_GNOME` cannot be set after [.filename]#bsd.port.pre.mk#. + ==== + + [[using-gnome-variables]] + === Variables + + This section explains which macros are available and how they are used. + Like they are used in the above example. + The crossref:special[gnome-components, GNOME Components] has a more in-depth explanation. + `USE_GNOME` has to be set for these macros to be of use. + + `GLIB_SCHEMAS`:: + List of all the glib schema files the port installs. + The macro will add the files to the port plist and handle the registration of these files on install and deinstall. + + + The glib schema files are written in XML and end with the [.filename]#gschema.xml# extension. + They are installed in the [.filename]#share/glib-2.0/schemas/# directory. + These schema files contain all application config values with their default settings. + The actual database used by the applications is built by glib-compile-schema, which is run by the `GLIB_SCHEMAS` macro. + + + [.programlisting] + .... + GLIB_SCHEMAS=foo.gschema.xml + .... + + + [NOTE] + ==== + Do not add glib schemas to the [.filename]#pkg-plist#. + If they are listed in [.filename]#pkg-plist#, they will not be registered and the applications might not work properly. + ==== + + `GCONF_SCHEMAS`:: + List all the gconf schema files. + The macro will add the schema files to the port plist and will handle their registration on install and deinstall. + + + GConf is the XML-based database that virtually all GNOME applications use for storing their settings. + These files are installed into the [.filename]#etc/gconf/schemas# directory. + This database is defined by installed schema files that are used to generate [.filename]#%gconf.xml# key files. + For each schema file installed by the port, there must be an entry in the [.filename]#Makefile#: + + + [.programlisting] + .... + GCONF_SCHEMAS=my_app.schemas my_app2.schemas my_app3.schemas + .... + + + [NOTE] + ==== + Gconf schemas are listed in the `GCONF_SCHEMAS` macro rather than [.filename]#pkg-plist#. + If they are listed in [.filename]#pkg-plist#, they will not be registered and the applications might not work properly. + ==== + + `INSTALLS_OMF`:: + Open Source Metadata Framework (OMF) files are commonly used by GNOME 2 applications. + These files contain the application help file information, and require special processing by ScrollKeeper/rarian. + To properly register OMF files when installing GNOME applications from packages, make sure that `omf` files are listed in `pkg-plist` and that the port [.filename]#Makefile# has `INSTALLS_OMF` defined: + + + [.programlisting] + .... + INSTALLS_OMF=yes + .... + + + When set, [.filename]#bsd.gnome.mk# automatically scans [.filename]#pkg-plist# and adds appropriate `@exec` and `@unexec` directives for each [.filename]#.omf# to track in the OMF registration database. + + [[gnome-components]] + == GNOME Components + + For further help with a GNOME port, look at some of the link:https://ports.FreeBSD.org[existing ports] for examples. + The link:https://www.FreeBSD.org/gnome/[FreeBSD GNOME page] has contact information if more help is needed. + The components are divided into GNOME components that are currently in use and legacy components. + If the component supports argument, they are listed between parenthesis in the description. + The first is the default. + "Both" is shown if the component defaults to adding to both build and run dependencies. + + [[gnome-components-list]] + .GNOME Components + [cols="1,1,1", options="header"] + |=== + | Component + | Associated program + | Description + + |`atk` + |accessibility/atk + |Accessibility toolkit (ATK) + + |`atkmm` + |accessibility/atkmm + |c++ bindings for atk + + |`cairo` + |graphics/cairo + |Vector graphics library with cross-device output support + + |`cairomm` + |graphics/cairomm + |c++ bindings for cairo + + |`dconf` + |devel/dconf + |Configuration database system (both, build, run) + + |`evolutiondataserver3` + |databases/evolution-data-server + |Data backends for the Evolution integrated mail/PIM suite + + |`gdkpixbuf2` + |graphics/gdk-pixbuf2 + |Graphics library for GTK+ + + |`glib20` + |devel/glib20 + |GNOME core library `glib20` + + |`glibmm` + |devel/glibmm + |c++ bindings for glib20 + + |`gnomecontrolcenter3` + |sysutils/gnome-control-center + |GNOME 3 Control Center + + |`gnomedesktop3` + |x11/gnome-desktop + |GNOME 3 desktop UI library + + |`gsound` + |audio/gsound + |GObject library for playing system sounds (both, build, run) + + |`gtk-update-icon-cache` + |graphics/gtk-update-icon-cache + |Gtk-update-icon-cache utility from the Gtk+ toolkit + + |`gtk20` + |x11-toolkits/gtk20 + |Gtk+ 2 toolkit + + |`gtk30` + |x11-toolkits/gtk30 + |Gtk+ 3 toolkit + + |`gtkmm20` + |x11-toolkits/gtkmm20 + |c++ bindings 2.0 for the gtk20 toolkit + + |`gtkmm24` + |x11-toolkits/gtkmm24 + |c++ bindings 2.4 for the gtk20 toolkit + + |`gtkmm30` + |x11-toolkits/gtkmm30 + |c++ bindings 3.0 for the gtk30 toolkit + + |`gtksourceview2` + |x11-toolkits/gtksourceview2 + |Widget that adds syntax highlighting to GtkTextView + + |`gtksourceview3` + |x11-toolkits/gtksourceview3 + |Text widget that adds syntax highlighting to the GtkTextView widget + + |`gtksourceviewmm3` + |x11-toolkits/gtksourceviewmm3 + |c++ bindings for the gtksourceview3 library + + |`gvfs` + |devel/gvfs + |GNOME virtual file system + + |`intltool` + |textproc/intltool + |Tool for internationalization (also see intlhack) + + |`introspection` + |devel/gobject-introspection + |Basic introspection bindings and tools to generate introspection bindings. Most of the time :build is enough, :both/:run is only need for applications that use introspection bindings. (both, build, run) + + |`libgda5` + |databases/libgda5 + |Provides uniform access to different kinds of data sources + + |`libgda5-ui` + |databases/libgda5-ui + |UI library from the libgda5 library + + |`libgdamm5` + |databases/libgdamm5 + |c++ bindings for the libgda5 library + + |`libgsf` + |devel/libgsf + |Extensible I/O abstraction for dealing with structured file formats + + |`librsvg2` + |graphics/librsvg2 + |Library for parsing and rendering SVG vector-graphic files + + |`libsigc++20` + |devel/libsigc++20 + |Callback Framework for C++ + + |`libxml++26` + |textproc/libxml++26 + |c++ bindings for the libxml2 library + + |`libxml2` + |textproc/libxml2 + |XML parser library (both, build, run) + + |`libxslt` + |textproc/libxslt + |XSLT C library (both, build, run) + + |`metacity` + |x11-wm/metacity + |Window manager from GNOME + + |`nautilus3` + |x11-fm/nautilus + |GNOME file manager + + |`pango` + |x11-toolkits/pango + |Open-source framework for the layout and rendering of i18n text + + |`pangomm` + |x11-toolkits/pangomm + |c++ bindings for the pango library + + |`py3gobject3` + |devel/py3-gobject3 + |Python 3, GObject 3.0 bindings + + |`pygobject3` + |devel/py-gobject3 + |Python 2, GObject 3.0 bindings + + |`vte3` + |x11-toolkits/vte3 + |Terminal widget with improved accessibility and I18N support + |=== + + [[gnome-components-macro]] + .GNOME Macro Components + [cols="1,1", options="header"] + |=== + | Component + | Description + + |`gnomeprefix` + |Supply `configure` with some default locations. + + |`intlhack` + |Same as intltool, but patches to make sure [.filename]#share/locale/# is used. Please only use when `intltool` alone is not enough. + + |`referencehack` + |This macro is there to help splitting of the API or reference documentation into its own port. + |=== + + [[gnome-components-legacy]] + .GNOME Legacy Components + [cols="1,1,1", options="header"] + |=== + | Component + | Associated program + | Description + + |`atspi` + |accessibility/at-spi + |Assistive Technology Service Provider Interface + + |`esound` + |audio/esound + |Enlightenment sound package + + |`gal2` + |x11-toolkits/gal2 + |Collection of widgets taken from GNOME 2 gnumeric + + |`gconf2` + |devel/gconf2 + |Configuration database system for GNOME 2 + + |`gconfmm26` + |devel/gconfmm26 + |c++ bindings for gconf2 + + |`gdkpixbuf` + |graphics/gdk-pixbuf + |Graphics library for GTK+ + + |`glib12` + |devel/glib12 + |glib 1.2 core library + + |`gnomedocutils` + |textproc/gnome-doc-utils + |GNOME doc utils + + |`gnomemimedata` + |misc/gnome-mime-data + |MIME and Application database for GNOME 2 + + |`gnomesharp20` + |x11-toolkits/gnome-sharp20 + |GNOME 2 interfaces for the .NET runtime + + |`gnomespeech` + |accessibility/gnome-speech + |GNOME 2 text-to-speech API + + |`gnomevfs2` + |devel/gnome-vfs + |GNOME 2 Virtual File System + + |`gtk12` + |x11-toolkits/gtk12 + |Gtk+ 1.2 toolkit + + |`gtkhtml3` + |www/gtkhtml3 + |Lightweight HTML rendering/printing/editing engine + + |`gtkhtml4` + |www/gtkhtml4 + |Lightweight HTML rendering/printing/editing engine + + |`gtksharp20` + |x11-toolkits/gtk-sharp20 + |GTK+ and GNOME 2 interfaces for the .NET runtime + + |`gtksourceview` + |x11-toolkits/gtksourceview + |Widget that adds syntax highlighting to GtkTextView + + |`libartgpl2` + |graphics/libart_lgpl + |Library for high-performance 2D graphics + + |`libbonobo` + |devel/libbonobo + |Component and compound document system for GNOME 2 + + |`libbonoboui` + |x11-toolkits/libbonoboui + |GUI frontend to the libbonobo component of GNOME 2 + + |`libgda4` + |databases/libgda4 + |Provides uniform access to different kinds of data sources + + |`libglade2` + |devel/libglade2 + |GNOME 2 glade library + + |`libgnome` + |x11/libgnome + |Libraries for GNOME 2, a GNU desktop environment + + |`libgnomecanvas` + |graphics/libgnomecanvas + |Graphics library for GNOME 2 + + |`libgnomekbd` + |x11/libgnomekbd + |GNOME 2 keyboard shared library + + |`libgnomeprint` + |print/libgnomeprint + |Gnome 2 print support library + + |`libgnomeprintui` + |x11-toolkits/libgnomeprintui + |Gnome 2 print support library + + |`libgnomeui` + |x11-toolkits/libgnomeui + |Libraries for the GNOME 2 GUI, a GNU desktop environment + + |`libgtkhtml` + |www/libgtkhtml + |Lightweight HTML rendering/printing/editing engine + + |`libgtksourceviewmm` + |x11-toolkits/libgtksourceviewmm + |c++ binding of GtkSourceView + + |`libidl` + |devel/libIDL + |Library for creating trees of CORBA IDL file + + |`libsigc++12` + |devel/libsigc++12 + |Callback Framework for C++ + + |`libwnck` + |x11-toolkits/libwnck + |Library used for writing pagers and taskslists + + |`libwnck3` + |x11-toolkits/libwnck3 + |Library used for writing pagers and taskslists + + |`orbit2` + |devel/ORBit2 + |High-performance CORBA ORB with support for the C language + + |`pygnome2` + |x11-toolkits/py-gnome2 + |Python bindings for GNOME 2 + + |`pygobject` + |devel/py-gobject + |Python 2, GObject 2.0 bindings + + |`pygtk2` + |x11-toolkits/py-gtk2 + |Set of Python bindings for GTK+ + + |`pygtksourceview` + |x11-toolkits/py-gtksourceview + |Python bindings for GtkSourceView 2 + + |`vte` + |x11-toolkits/vte + |Terminal widget with improved accessibility and I18N support + |=== + + [[gnome-components-deprecated]] + .Deprecated Components: Do Not Use + [cols="1,1", options="header"] + |=== + | Component + | Description + + |`pangox-compat` + |pangox-compat has been deprecated and split off from the pango package. + |=== + + [[using-qt]] + == Using Qt + + [NOTE] + ==== + For ports that are part of Qt itself, see crossref:uses[uses-qt-dist,`qt-dist`]. + ==== + + [[qt-common]] + === Ports That Require Qt + + The Ports Collection provides support for Qt 5 and Qt 6 with `USES+=qt:5` and + `USES+=qt:6` respectively. + Set `USE_QT` to the list of required Qt components (libraries, tools, plugins). + + The Qt framework exports a number of variables which can be used by ports, some of them listed below: + + [[using-qt-variables]] + .Variables Provided to Ports That Use Qt + [cols="1,1", frame="none"] + |=== + |`QMAKE` + |Full path to `qmake` binary. + + |`LRELEASE` + |Full path to `lrelease` utility. + + |`MOC` + |Full path to `moc`. + + |`RCC` + |Full path to `rcc`. + + |`UIC` + |Full path to `uic`. + + |`QT_INCDIR` + |Qt include directory. + + |`QT_LIBDIR` + |Qt libraries path. + + |`QT_PLUGINDIR` + |Qt plugins path. + |=== + + [[qt-components]] + === Component Selection + + Individual Qt tool and library dependencies must be specified in `USE_QT`. + Every component can be suffixed with `_build` or `_run`, the suffix indicating whether the dependency on the component is at buildtime or runtime. + If unsuffixed, the component will be depended on at both build- and runtime. + Usually, library components are specified unsuffixed, tool components are mostly specified with the `_build` suffix and plugin components are specified with the `_run` suffix. + The most commonly used components are listed below (all available components are listed in `_USE_QT_ALL`, which is generated from `_USE_QT_COMMON` and `_USE_QT[56]_ONLY` in [.filename]#/usr/ports/Mk/Uses/qt.mk#): + + [[using-qt-library-list]] + .Available Qt Library Components + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Description + + |`3d` + |Qt3D module + + |`5compat` + |Qt 5 compatibility module for Qt 6 + + |`assistant` + |Qt 5 documentation browser + + |`base` + |Qt 6 base module + + |`canvas3d` + |Qt canvas3d module + + |`charts` + |Qt 5 charts module + + |`concurrent` + |Qt multi-threading module + + |`connectivity` + |Qt connectivity (Bluetooth/NFC) module + + |`core` + |Qt core non-graphical module + + |`datavis3d` + |Qt 5 3D data visualization module + + |`dbus` + |Qt D-Bus inter-process communication module + + |`declarative` + |Qt declarative framework for dynamic user interfaces + + |`designer` + |Qt 5 graphical user interface designer + + |`diag` + |Tool for reporting diagnostic information about Qt and its environment + + |`doc` + |Qt 5 documentation + + |`examples` + |Qt 5 examples sourcecode + + |`gamepad` + |Qt 5 Gamepad Module + + |`graphicaleffects` + |Qt Quick graphical effects + + |`gui` + |Qt graphical user interface module + + |`help` + |Qt online help integration module + + |`l10n` + |Qt localized messages + + |`languageserver` + |Qt 6 Language Server Protocol implementation + + |`linguist` + |Qt 5 translation tool + + |`location` + |Qt location module + + |`lottie` + |Qt 6 QML API for rendering graphics and animations + + |`multimedia` + |Qt audio, video, radio and camera support module + + |`network` + |Qt network module + + |`networkauth` + |Qt network auth module + + |`opengl` + |Qt 5-compatible OpenGL support module + + |`paths` + |Command line client to QStandardPaths + + |`phonon4` + |KDE multimedia framework + + |`pixeltool` + |Qt 5 screen magnifier + + |`plugininfo` + |Qt 5 plugin metadata dumper + + |`positioning` + |Qt 6 positioning API from sources such as satellite, wifi or text files. + + |`printsupport` + |Qt print support module + + |`qdbus` + |Qt command-line interface to D-Bus + + |`qdbusviewer` + |Qt 5 graphical interface to D-Bus + + |`qdoc` + |Qt documentation generator + + |`qdoc-data` + |QDoc configuration files + + |`qev` + |Qt QWidget events introspection tool + + |`qmake` + |Qt Makefile generator + + |`quickcontrols` + |Set of controls for building complete interfaces in Qt Quick + + |`quickcontrols2` + |Set of controls for building complete interfaces in Qt Quick + + |`remoteobjects` + |Qt 5 SXCML module + + |`script` + |Qt 4-compatible scripting module + + |`scripttools` + |Qt Script additional components + + |`scxml` + |Qt 5 SXCML module + + |`sensors` + |Qt sensors module + + |`serialbus` + |Qt functions to access industrial bus systems + + |`serialport` + |Qt functions to access serial ports + + |`shadertools` + |Qt 6 tools for the cross-platform Qt shader pipeline + + |`speech` + |Accessibility features for Qt5 + + |`sql` + |Qt SQL database integration module + + |`sql-ibase` + |Qt InterBase/Firebird database plugin + + |`sql-mysql` + |Qt MySQL database plugin + + |`sql-odbc` + |Qt Open Database Connectivity plugin + + |`sql-pgsql` + |Qt PostgreSQL database plugin + + |`sql-sqlite2` + |Qt SQLite 2 database plugin + + |`sql-sqlite3` + |Qt SQLite 3 database plugin + + |`sql-tds` + |Qt TDS Database Connectivity database plugin + + |`svg` + |Qt SVG support module + + |`testlib` + |Qt unit testing module + + |`tools` + |Qt 6 assorted tools + + |`translations` + |Qt 6 translation module + + |`uiplugin` + |Custom Qt widget plugin interface for Qt Designer + + |`uitools` + |Qt Designer UI forms support module + + |`virtualkeyboard` + |Qt 5 Virtual Keyboard Module + + |`wayland` + |Qt 5 wrapper for Wayland + + |`webchannel` + |Qt 5 library for integration of C++/QML with HTML/js clients + + |`webengine` + |Qt 5 library to render web content + + |`webkit` + |QtWebKit with a more modern WebKit code base + + |`websockets` + |Qt implementation of WebSocket protocol + + |`websockets-qml` + |Qt implementation of WebSocket protocol (QML bindings) + + |`webview` + |Qt component for displaying web content + + |`widgets` + |Qt C++ widgets module + + |`x11extras` + |Qt platform-specific features for X11-based systems + + |`xml` + |Qt SAX and DOM implementations + + |`xmlpatterns` + |Qt support for XPath, XQuery, XSLT and XML Schema + |=== + + To determine the libraries an application depends on, run `ldd` on the main executable after a successful compilation. + + [[using-qt-tools-list]] + .Available Qt Tool Components + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Description + + |`buildtools` + |build tools (`moc`, `rcc`), needed for almost every Qt application. + + |`linguisttools` + |localization tools: `lrelease`, `lupdate` + + |`qmake` + |Makefile generator/build utility + |=== + + [[using-qt-plugins-list]] + .Available Qt Plugin Components + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Description + + |`imageformats` + |plugins for TGA, TIFF, and MNG image formats + |=== + + [[qt5-components-example]] + .Selecting Qt 5 Components + [example] + ==== + In this example, the ported application uses the Qt 5 graphical user interface library, the Qt 5 core library, all of the Qt 5 code generation tools and Qt 5's Makefile generator. + Since the `gui` library implies a dependency on the core library, `core` does not need to be specified. + The Qt 5 code generation tools `moc`, `uic` and `rcc`, as well as the Makefile generator `qmake` are only needed at buildtime, thus they are specified with the `_build` suffix: + + [.programlisting] + .... + USES= qt:5 + USE_QT= gui buildtools_build qmake_build + .... + + ==== + + [[using-qmake]] + === Using `qmake` + + If the application provides a qmake project file ([.filename]#*.pro#), define `USES= qmake` along with `USE_QT`. + `USES= qmake` already implies a build dependency on qmake, therefore the qmake component can be omitted from `USE_QT`. + Similar to crossref:special[using-cmake,CMake], qmake supports out-of-source + builds, which can be enabled by specifying the `outsource` argument (see + crossref:special[using-qmake-example,`USES= qmake` example]). + Also see crossref:special[using-qmake-arguments,Possible Arguments for `USES qmake`]. + + [[using-qmake-arguments]] + .Possible Arguments for `USES= qmake` + [cols="1,1", frame="none", options="header"] + |=== + | Variable + | Description + + |`no_configure` + |Do not add the configure target. This is implied by `HAS_CONFIGURE=yes` and `GNU_CONFIGURE=yes`. It is required when the build only needs the environment setup from `USES= qmake`, but otherwise runs `qmake` on its own. + + |`no_env` + |Suppress modification of the configure and make environments. It is only required when `qmake` is used to configure the software and the build fails to understand the environment setup by `USES= qmake`. + + |`norecursive` + |Do not pass the `-recursive` argument to `qmake`. + + |`outsource` + |Perform an out-of-source build. + |=== + + [[using-qmake-variables]] + .Variables for Ports That Use `qmake` + [cols="1,1", frame="none", options="header"] + |=== + | Variable + | Description + + |`QMAKE_ARGS` + |Port specific qmake flags to be passed to the `qmake` binary. + + |`QMAKE_ENV` + |Environment variables to be set for the `qmake` binary. The default is `${CONFIGURE_ENV}`. + + |`QMAKE_SOURCE_PATH` + |Path to qmake project files ([.filename]#.pro#). The default is `${WRKSRC}` if an out-of-source build is requested, empty otherwise. + |=== + + When using `USES= qmake`, these settings are deployed: + + [.programlisting] + .... + CONFIGURE_ARGS+= --with-qt-includes=${QT_INCDIR} \ + --with-qt-libraries=${QT_LIBDIR} \ + --with-extra-libs=${LOCALBASE}/lib \ + --with-extra-includes=${LOCALBASE}/include + + CONFIGURE_ENV+= QTDIR="${QT_PREFIX}" QMAKE="${QMAKE}" \ + MOC="${MOC}" RCC="${RCC}" UIC="${UIC}" \ + QMAKESPEC="${QMAKESPEC}" + + PLIST_SUB+= QT_INCDIR=${QT_INCDIR_REL} \ + QT_LIBDIR=${QT_LIBDIR_REL} \ + QT_PLUGINDIR=${QT_PLUGINDIR_REL} + .... + + Some configure scripts do not support the arguments above. + To suppress modification of `CONFIGURE_ENV` and `CONFIGURE_ARGS`, set `USES= qmake:no_env`. + + [[using-qmake-example]] + .`USES= qmake` Example + [example] + ==== + This snippet demonstrates the use of qmake for a Qt 5 port: + + [.programlisting] + .... + USES= qmake:outsource qt:5 + USE_QT= buildtools_build + .... + + ==== + + Qt applications are often written to be cross-platform and often X11/Unix is not the platform they are developed on, which in turn leads to certain loose ends, like: + + * _Missing additional include paths._ Many applications come with system tray icon support, but neglect to look for includes and/or libraries in the X11 directories. To add directories to `qmake`'s include and library search paths via the command line, use: + + + [.programlisting] + .... + QMAKE_ARGS+= INCLUDEPATH+=${LOCALBASE}/include \ + LIBS+=-L${LOCALBASE}/lib + .... + + * _Bogus installation paths._ Sometimes data such as icons or .desktop files are by default installed into directories which are not scanned by XDG-compatible applications. package:editors/texmaker[] is an example for this - look at [.filename]#patch-texmaker.pro# in the [.filename]#files# directory of that port for a template on how to remedy this directly in the `qmake` project file. + + [[using-kde]] + == Using KDE + + [[kde5-variables]] + === KDE Variable Definitions + + If the application depends on KDE, set `USES+=kde:5` and `USE_KDE` to the list of required components. + `_build` and `_run` suffixes can be used to force components dependency type (for example, `baseapps_run`). + If no suffix is set, a default dependency type will be used. + To force both types, add the component twice with both suffixes (for example, `ecm_build ecm_run`). + Available components are listed below (up-to-date components are also listed in [.filename]#/usr/ports/Mk/Uses/kde.mk#): + + [[using-kde-components]] + .Available KDE Components + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Description + + |`activities` + |KF5 runtime and library to organize work in separate activities + + |`activities-stats` + |KF5 statistics for activities + + |`activitymanagerd` + |System service to manage user's activities, track the usage patterns + + |`akonadi` + |Storage server for KDE-Pim + + |`akonadicalendar` + |Akonadi Calendar Integration + + |`akonadiconsole` + |Akonadi management and debugging console + + |`akonadicontacts` + |Libraries and daemons to implement Contact Management in Akonadi + + |`akonadiimportwizard` + |Import data from other mail clients to KMail + + |`akonadimime` + |Libraries and daemons to implement basic email handling + + |`akonadinotes` + |KDE library for accessing mail storages in MBox format + + |`akonadisearch` + |Libraries and daemons to implement searching in Akonadi + + |`akregator` + |A Feed Reader by KDE + + |`alarmcalendar` + |KDE API for KAlarm alarms + + |`apidox` + |KF5 API Documentation Tools + + |`archive` + |KF5 library that provides classes for handling archive formats + + |`attica` + |Open Collaboration Services API library KDE5 version + + |`attica5` + |Open Collaboration Services API library KDE5 version + + |`auth` + |KF5 abstraction to system policy and authentication features + + |`baloo` + |KF5 Framework for searching and managing user metadata + + |`baloo-widgets` + |BalooWidgets library + + |`baloo5` + |KF5 Framework for searching and managing user metadata + + |`blog` + |KDE API for weblogging access + + |`bookmarks` + |KF5 library for bookmarks and the XBEL format + + |`breeze` + |Plasma5 artwork, styles and assets for the Breeze visual style + + |`breeze-gtk` + |Plasma5 Breeze visual style for Gtk + + |`breeze-icons` + |Breeze icon theme for KDE + + |`calendarcore` + |KDE calendar access library + + |`calendarsupport` + |Calendar support libraries for KDEPim + + |`calendarutils` + |KDE utility and user interface functions for accessing calendar + + |`codecs` + |KF5 library for string manipulation + + |`completion` + |KF5 text completion helpers and widgets + + |`config` + |KF5 widgets for configuration dialogs + + |`configwidgets` + |KF5 widgets for configuration dialogs + + |`contacts` + |KDE api to manage contact information + + |`coreaddons` + |KF5 addons to QtCore + + |`crash` + |KF5 library to handle crash analysis and bug report from apps + + |`dbusaddons` + |KF5 addons to QtDBus + + |`decoration` + |Plasma5 library to create window decorations + + |`designerplugin` + |KF5 integration of Frameworks widgets in Qt Designer/Creator + + |`discover` + |Plasma5 package management tools + + |`dnssd` + |KF5 abstraction to system DNSSD features + + |`doctools` + |KF5 documentation generation from docbook + + |`drkonqi` + |Plasma5 crash handler + + |`ecm` + |Extra modules and scripts for CMake + + |`emoticons` + |KF5 library to convert emoticons + + |`eventviews` + |Event view libriares for KDEPim + + |`filemetadata` + |KF5 library for extracting file metadata + + |`frameworkintegration` + |KF5 workspace and cross-framework integration plugins + + |`gapi` + |KDE based library to access google services + + |`globalaccel` + |KF5 library to add support for global workspace shortcuts + + |`grantlee-editor` + |Editor for Grantlee themes + + |`grantleetheme` + |KDE PIM grantleetheme + + |`gravatar` + |Library for gravatar support + + |`guiaddons` + |KF5 addons to QtGui + + |`holidays` + |KDE library for calendar holidays + + |`hotkeys` + |Plasma5 library for hotkeys + + |`i18n` + |KF5 advanced internationalization framework + + |`iconthemes` + |KF5 library for handling icons in applications + + |`identitymanagement` + |KDE pim identities + + |`idletime` + |KF5 library for monitoring user activity + + |`imap` + |KDE API for IMAP support + + |`incidenceeditor` + |Incidence editor libriares for KDEPim + + |`infocenter` + |Plasma5 utility providing system information + + |`init` + |KF5 process launcher to speed up launching KDE applications + + |`itemmodels` + |KF5 models for Qt Model/View system + + |`itemviews` + |KF5 widget addons for Qt Model/View + + |`jobwidgets` + |KF5 widgets for tracking KJob instance + + |`js` + |KF5 library providing an ECMAScript interpreter + + |`jsembed` + |KF5 library for binding JavaScript objects to QObjects + + |`kaddressbook` + |KDE contact manager + + |`kalarm` + |Personal alarm scheduler + + |`kalarm` + |Personal alarm scheduler + + |`kate` + |Basic editor framework for the KDE system + + |`kcmutils` + |KF5 utilities for working with KCModules + + |`kde-cli-tools` + |Plasma5 non-interactive system tools + + |`kde-gtk-config` + |Plasma5 GTK2 and GTK3 configurator + + |`kdeclarative` + |KF5 library providing integration of QML and KDE Frameworks + + |`kded` + |KF5 extensible daemon for providing system level services + + |`kdelibs4support` + |KF5 porting aid from KDELibs4 + + |`kdepim-addons` + |KDE PIM addons + + |`kdepim-apps-libs` + |KDE PIM mail related libraries + + |`kdepim-runtime5` + |KDE PIM tools and services + + |`kdeplasma-addons` + |Plasma5 addons to improve the Plasma experience + + |`kdesu` + |KF5 integration with su for elevated privileges + + |`kdewebkit` + |KF5 library providing integration of QtWebKit + + |`kgamma5` + |Plasma5 monitor's gamma settings + + |`khtml` + |KF5 KTHML rendering engine + + |`kimageformats` + |KF5 library providing support for additional image formats + + |`kio` + |KF5 resource and network access abstraction + + |`kirigami2` + |QtQuick based components set + + |`kitinerary` + |Data Model and Extraction System for Travel Reservation information + + |`kmail` + |KDE mail client + + |`kmail` + |KDE mail client + + |`kmail-account-wizard` + |KDE mail account wizard + + |`kmenuedit` + |Plasma5 menu editor + + |`knotes` + |Popup notes + + |`kontact` + |KDE Personal Information Manager + + |`kontact` + |KDE Personal Information Manager + + |`kontactinterface` + |KDE glue for embedding KParts into Kontact + + |`korganizer` + |Calendar and scheduling Program + + |`kpimdav` + |A DAV protocol implementation with KJobs + + |`kpkpass` + |Library to deal with Apple Wallet pass files + + |`kross` + |KF5 multi-language application scripting + + |`kscreen` + |Plasma5 screen management library + + |`kscreenlocker` + |Plasma5 secure lock screen architecture + + |`ksmtp` + |Job-based library to send email through an SMTP server + + |`ksshaskpass` + |Plasma5 ssh-add frontend + + |`ksysguard` + |Plasma5 utility to track and control the running processes + + |`kwallet-pam` + |Plasma5 KWallet PAM Integration + + |`kwayland-integration` + |Integration plugins for a Wayland-based desktop + + |`kwin` + |Plasma5 window manager + + |`kwrited` + |Plasma5 daemon listening for wall and write messages + + |`ldap` + |LDAP access API for KDE + + |`libkcddb` + |KDE CDDB library + + |`libkcompactdisc` + |KDE library for interfacing with audio CDs + + |`libkdcraw` + |LibRaw interface for KDE + + |`libkdegames` + |Libraries used by KDE games + + |`libkdepim` + |KDE PIM Libraries + + |`libkeduvocdocument` + |Library for reading and writing vocabulary files + + |`libkexiv2` + |Exiv2 library interface for KDE + + |`libkipi` + |KDE Image Plugin Interface + + |`libkleo` + |Certificate manager for KDE + + |`libksane` + |SANE library interface for KDE + + |`libkscreen` + |Plasma5 screen management library + + |`libksieve` + |Sieve libriares for KDEPim + + |`libksysguard` + |Plasma5 library to track and control running processes + + |`mailcommon` + |Common libriares for KDEPim + + |`mailimporter` + |Import mbox files to KMail + + |`mailtransport` + |KDE library to managing mail transport + + |`marble` + |Virtual globe and world atlas for KDE + + |`mbox` + |KDE library for accessing mail storages in MBox format + + |`mbox-importer` + |Import mbox files to KMail + + |`mediaplayer` + |KF5 plugin interface for media player features + + |`messagelib` + |Library for handling messages + + |`milou` + |Plasma5 Plasmoid for search + + |`mime` + |Library for handling MIME data + + |`newstuff` + |KF5 library for downloading application assets from the network + + |`notifications` + |KF5 abstraction for system notifications + + |`notifyconfig` + |KF5 configuration system for KNotify + + |`okular` + |KDE universal document viewer + + |`oxygen` + |Plasma5 Oxygen style + + |`oxygen-icons5` + |The Oxygen icon theme for KDE + + |`package` + |KF5 library to load and install packages + + |`parts` + |KF5 document centric plugin system + + |`people` + |KF5 library providing access to contacts + + |`pim-data-exporter` + |Import and export KDE PIM settings + + |`pimcommon` + |Common libriares for KDEPim + + |`pimtextedit` + |KDE library for PIM-specific text editing utilities + + |`plasma-browser-integration` + |Plasma5 components to integrate browsers into the desktop + + |`plasma-desktop` + |Plasma5 plasma desktop + + |`plasma-framework` + |KF5 plugin based UI runtime used to write user interfaces + + |`plasma-integration` + |Qt Platform Theme integration plugins for the Plasma workspaces + + |`plasma-pa` + |Plasma5 Plasma pulse audio mixer + + |`plasma-sdk` + |Plasma5 applications useful for Plasma development + + |`plasma-workspace` + |Plasma5 Plasma workspace + + |`plasma-workspace-wallpapers` + |Plasma5 wallpapers + + |`plotting` + |KF5 lightweight plotting framework + + |`polkit-kde-agent-1` + |Plasma5 daemon providing a polkit authentication UI + + |`powerdevil` + |Plasma5 tool to manage the power consumption settings + + |`prison` + |API to produce barcodes + + |`pty` + |KF5 pty abstraction + + |`purpose` + |Offers available actions for a specific purpose + + |`qqc2-desktop-style` + |Qt QuickControl2 style for KDE + + |`runner` + |KF5 parallelized query system + + |`service` + |KF5 advanced plugin and service introspection + + |`solid` + |KF5 hardware integration and detection + + |`sonnet` + |KF5 plugin-based spell checking library + + |`syndication` + |KDE RSS feed handling library + + |`syntaxhighlighting` + |KF5 syntax highlighting engine for structured text and code + + |`systemsettings` + |Plasma5 system settings + + |`texteditor` + |KF5 advanced embeddable text editor + + |`textwidgets` + |KF5 advanced text editing widgets + + |`threadweaver` + |KF5 addons to QtDBus + + |`tnef` + |KDE API for the handling of TNEF data + + |`unitconversion` + |KF5 library for unit conversion + + |`user-manager` + |Plasma5 user manager + + |`wallet` + |KF5 secure and unified container for user passwords + + |`wayland` + |KF5 Client and Server library wrapper for the Wayland libraries + + |`widgetsaddons` + |KF5 addons to QtWidgets + + |`windowsystem` + |KF5 library for access to the windowing system + + |`xmlgui` + |KF5 user configurable main windows + + |`xmlrpcclient` + |KF5 interaction with XMLRPC services + |=== + + [[kde5-components-example]] + .`USE_KDE` Example + [example] + ==== + This is a simple example for a KDE port. + `USES= cmake` instructs the port to utilize CMake, a configuration tool widely + used by KDE projects (see crossref:special[using-cmake, Using `cmake`] for detailed usage). + `USE_KDE` brings dependency on KDE libraries. + Required KDE components and other dependencies can be determined through the configure log. + `USE_KDE` does not imply `USE_QT`. + If a port requires some Qt components, specify them in `USE_QT`. + + [.programlisting] + .... + USES= cmake kde:5 qt:5 + USE_KDE= ecm + USE_QT= core buildtools_build qmake_build + .... + + ==== + + [[using-lxqt]] + == Using LXQt + + Applications depending on LXQt should set `USES+= lxqt` and set `USE_LXQT` to the list of required components from the table below + + [[using-lxqt-components]] + .Available LXQt Components + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Description + + |`buildtools` + |Helpers for additional CMake modules + + |`libfmqt` + |Libfm Qt bindings + + |`lxqt` + |LXQt core library + + |`qtxdg` + |Qt implementation of freedesktop.org XDG specifications + |=== + + [[lxqt-components-example]] + .`USE_LXQT` Example + [example] + ==== + This is a simple example, `USE_LXQT` adds a dependency on LXQt libraries. + Required LXQt components and other dependencies can be determined from the configure log. + + [.programlisting] + .... + USES= cmake lxqt qt:5 tar:xz + USE_QT= core dbus widgets buildtools_build qmake_build + USE_LXQT= buildtools libfmqt + .... + + ==== + + [[using-java]] + == Using Java + + [[java-variables]] + === Variable Definitions + + If the port needs a Java(TM) Development Kit (JDK(TM)) to either build, run or even extract the distfile, then define `USE_JAVA`. + + There are several JDKs in the ports collection, from various vendors, and in several versions. + If the port must use a particular version, specify it using the `JAVA_VERSION` variable. + The most current version is package:java/openjdk18[], with package:java/openjdk17[], package:java/openjdk16[], package:java/openjdk15[], package:java/openjdk14[], package:java/openjdk13[], package:java/openjdk12[], package:java/openjdk11[], package:java/openjdk8[], and package:java/openjdk7[] also available. + + [[using-java-variables]] + .Variables Which May be Set by Ports That Use Java + [cols="1,1", frame="none", options="header"] + |=== + | Variable + | Means + + |`USE_JAVA` + |Define for the remaining variables to have any effect. + + |`JAVA_VERSION` + |List of space-separated suitable Java versions for the port. + An optional `\+` allows specifying a range of versions (allowed values: `8[+] 11[\+] 17[+] 18[\+] 19[+] 20[\+] 21[+]`). + + |`JAVA_OS` + |List of space-separated suitable JDK port operating systems for the port (allowed values: `native linux`). + + |`JAVA_VENDOR` + |List of space-separated suitable JDK port vendors for the port (allowed values: `openjdk oracle`). + + |`JAVA_BUILD` + |When set, add the selected JDK port to the build dependencies. + + |`JAVA_RUN` + |When set, add the selected JDK port to the run dependencies. + + |`JAVA_EXTRACT` + |When set, add the selected JDK port to the extract dependencies. + |=== + + Below is the list of all settings a port will receive after setting `USE_JAVA`: + + [[using-java-variables2]] + .Variables Provided to Ports That Use Java + [cols="1,1", frame="none", options="header"] + |=== + | Variable + | Value + + |`JAVA_PORT` + |The name of the JDK port (for example, `java/openjdk6`). + + |`JAVA_PORT_VERSION` + |The full version of the JDK port (for example, `1.6.0`). Only the first two digits of this version number are needed, use `${JAVA_PORT_VERSION:C/^([0-9])\.([0-9])(.*)$/\1.\2/}`. + + |`JAVA_PORT_OS` + |The operating system used by the JDK port (for example, `'native'`). + + |`JAVA_PORT_VENDOR` + |The vendor of the JDK port (for example, `'openjdk'`). + + |`JAVA_PORT_OS_DESCRIPTION` + |Description of the operating system used by the JDK port (for example, `'Native'`). + + |`JAVA_PORT_VENDOR_DESCRIPTION` + |Description of the vendor of the JDK port (for example, `'OpenJDK BSD Porting Team'`). + + |`JAVA_HOME` + |Path to the installation directory of the JDK (for example, [.filename]#'/usr/local/openjdk6'#). + + |`JAVAC` + |Path to the Java compiler to use (for example, [.filename]#'/usr/local/openjdk6/bin/javac'#). + + |`JAR` + |Path to the `jar` tool to use (for example, [.filename]#'/usr/local/openjdk6/bin/jar'# or [.filename]#'/usr/local/bin/fastjar'#). + + |`APPLETVIEWER` + |Path to the `appletviewer` utility (for example, [.filename]#'/usr/local/openjdk6/bin/appletviewer'#). + + |`JAVA` + |Path to the `java` executable. Use this for executing Java programs (for example, [.filename]#'/usr/local/openjdk6/bin/java'#). + + |`JAVADOC` + |Path to the `javadoc` utility program. + + |`JAVAH` + |Path to the `javah` program. + + |`JAVAP` + |Path to the `javap` program. + + |`JAVA_KEYTOOL` + |Path to the `keytool` utility program. + + |`JAVA_N2A` + |Path to the `native2ascii` tool. + + |`JAVA_POLICYTOOL` + |Path to the `policytool` program. + + |`JAVA_SERIALVER` + |Path to the `serialver` utility program. + + |`RMIC` + |Path to the RMI stub/skeleton generator, `rmic`. + + |`RMIREGISTRY` + |Path to the RMI registry program, `rmiregistry`. + + |`RMID` + |Path to the RMI daemon program `rmid`. + + |`JAVA_CLASSES` + |Path to the archive that contains the JDK class files, [.filename]#${JAVA_HOME}/jre/lib/rt.jar#. + |=== + + Use the `java-debug` make target to get information for debugging the port. + It will display the value of many of the previously listed variables. + + Additionally, these constants are defined so all Java ports may be installed in a consistent way: + + [[using-java-constants]] + .Constants Defined for Ports That Use Java + [cols="1,1", frame="none", options="header"] + |=== + | Constant + | Value + + |`JAVASHAREDIR` + |The base directory for everything related to Java. Default: [.filename]#${PREFIX}/share/java#. + + |`JAVAJARDIR` + |The directory where JAR files is installed. Default: [.filename]#${JAVASHAREDIR}/classes#. + + |`JAVALIBDIR` + |The directory where JAR files installed by other ports are located. Default: [.filename]#${LOCALBASE}/share/java/classes#. + |=== + + The related entries are defined in both `PLIST_SUB` (documented in crossref:plist[plist-sub,Changing pkg-plist Based on Make Variables]) and `SUB_LIST`. + + [[java-building-with-ant]] + === Building with Ant + + When the port is to be built using Apache Ant, it has to define `USE_ANT`. + Ant is thus considered to be the sub-make command. + When no `do-build` target is defined by the port, a default one will be set that runs Ant according to `MAKE_ENV`, `MAKE_ARGS` and `ALL_TARGET`. + This is similar to the `USES= gmake` mechanism, which is documented in crossref:special[building, Building Mechanisms]. + + [[java-best-practices]] + === Best Practices + + When porting a Java library, the port has to install the JAR file(s) in [.filename]#${JAVAJARDIR}#, and everything else under [.filename]#${JAVASHAREDIR}/${PORTNAME}# (except for the documentation, see below). + To reduce the packing file size, reference the JAR file(s) directly in the [.filename]#Makefile#. + Use this statement (where [.filename]#myport.jar# is the name of the JAR file installed as part of the port): + + [.programlisting] + .... + PLIST_FILES+= ${JAVAJARDIR}/myport.jar + .... + + When porting a Java application, the port usually installs everything under a single directory (including its JAR dependencies). + The use of [.filename]#${JAVASHAREDIR}/${PORTNAME}# is strongly encouraged in this regard. + It is up the porter to decide whether the port installs the additional JAR dependencies under this directory or uses the already installed ones (from [.filename]#${JAVAJARDIR}#). + + When porting a Java(TM) application that requires an application server such as package:www/tomcat7[] to run the service, it is quite common for a vendor to distribute a [.filename]#.war#. + A [.filename]#.war# is a Web application ARchive and is extracted when called by the application. + Avoid adding a [.filename]#.war# to [.filename]#pkg-plist#. + It is not considered best practice. + An application server will expand war archive, but not clean it up properly if the port is removed. + A more desirable way of working with this file is to extract the archive, then install the files, and lastly add these files to [.filename]#pkg-plist#. + + [.programlisting] + .... + TOMCATDIR= ${LOCALBASE}/apache-tomcat-7.0 + WEBAPPDIR= myapplication + + post-extract: + @${MKDIR} ${WRKDIR}/${PORTDIRNAME} + @${TAR} xf ${WRKDIR}/myapplication.war -C ${WRKDIR}/${PORTDIRNAME} + + do-install: + cd ${WRKDIR} && \ + ${INSTALL} -d -o ${WWWOWN} -g ${WWWGRP} ${TOMCATDIR}/webapps/${PORTDIRNAME} + cd ${WRKDIR}/${PORTDIRNAME} && ${COPYTREE_SHARE} \* ${WEBAPPDIR}/${PORTDIRNAME} + .... + + Regardless of the type of port (library or application), the additional documentation is installed in the crossref:makefiles[install-documentation,same location] as for any other port. + The Javadoc tool is known to produce a different set of files depending on the version of the JDK that is used. + For ports that do not enforce the use of a particular JDK, it is therefore a complex task to specify the packing list ([.filename]#pkg-plist#). + This is one reason why porters are strongly encouraged to use `PORTDOCS`. + Moreover, even if the set of files that will be generated by `javadoc` can be predicted, the size of the resulting [.filename]#pkg-plist# advocates for the use of `PORTDOCS`. + + The default value for `DATADIR` is [.filename]#${PREFIX}/share/${PORTNAME}#. + It is a good idea to override `DATADIR` to [.filename]#${JAVASHAREDIR}/${PORTNAME}# for Java ports. + Indeed, `DATADIR` is automatically added to `PLIST_SUB` (documented in crossref:plist[plist-sub,Changing pkg-plist Based on Make Variables]) so use `%%DATADIR%%` directly in [.filename]#pkg-plist#. + + As for the choice of building Java ports from source or directly installing them from a binary distribution, there is no defined policy at the time of writing. + However, people from the https://www.freebsd.org/java/[FreeBSD Java Project] encourage porters to have their ports built from source whenever it is a trivial task. + + All the features that have been presented in this section are implemented in [.filename]#bsd.java.mk#. + If the port needs more sophisticated Java support, please first have a look at the https://cgit.FreeBSD.org/ports/tree/Mk/bsd.java.mk[bsd.java.mk Git log] as it usually takes some time to document the latest features. + Then, if the needed support that is lacking would be beneficial to many other Java ports, feel free to discuss it on the freebsd-java. + + Although there is a `java` category for PRs, it refers to the JDK porting effort from the FreeBSD Java project. + Therefore, submit the Java port in the `ports` category as for any other port, unless the issue is related to either a JDK implementation or [.filename]#bsd.java.mk#. + + Similarly, there is a defined policy regarding the `CATEGORIES` of a Java port, which is detailed in crossref:makefiles[makefile-categories,Categorization]. + + [[using-php]] + == Web Applications, Apache and PHP + + [[using-apache]] + === Apache + + [[using-apache-variables]] + .Variables for Ports That Use Apache + [cols="1,1", frame="none"] + |=== + |`USE_APACHE` + |The port requires Apache. Possible values: `yes` (gets any version), `22`, `24`, `22-24`, `22+`, etc. The default APACHE version is `22`. More details are available in [.filename]#ports/Mk/bsd.apache.mk# and at https://wiki.freebsd.org/Apache/[wiki.freebsd.org/Apache/]. + + |`APXS` + |Full path to the `apxs` binary. Can be overridden in the port. + + |`HTTPD` + |Full path to the `httpd` binary. Can be overridden in the port. + + |`APACHE_VERSION` + |The version of present Apache installation (read-only variable). This variable is only available after inclusion of [.filename]#bsd.port.pre.mk#. Possible values: `22`, `24`. + + |`APACHEMODDIR` + |Directory for Apache modules. This variable is automatically expanded in [.filename]#pkg-plist#. + + |`APACHEINCLUDEDIR` + |Directory for Apache headers. This variable is automatically expanded in [.filename]#pkg-plist#. + + |`APACHEETCDIR` + |Directory for Apache configuration files. This variable is automatically expanded in [.filename]#pkg-plist#. + |=== + + [[using-apache-modules]] + .Useful Variables for Porting Apache Modules + [cols="1,1", frame="none"] + |=== + |`MODULENAME` + |Name of the module. Default value is `PORTNAME`. Example: `mod_hello` + + |`SHORTMODNAME` + |Short name of the module. Automatically derived from `MODULENAME`, but can be overridden. Example: `hello` + + |`AP_FAST_BUILD` + |Use `apxs` to compile and install the module. + + |`AP_GENPLIST` + |Also automatically creates a [.filename]#pkg-plist#. + + |`AP_INC` + |Adds a directory to a header search path during compilation. + + |`AP_LIB` + |Adds a directory to a library search path during compilation. + + |`AP_EXTRAS` + |Additional flags to pass to `apxs`. + |=== + + [[web-apps]] + === Web Applications + + Web applications must be installed into [.filename]#PREFIX/www/appname#. + This path is available both in [.filename]#Makefile# and in [.filename]#pkg-plist# as `WWWDIR`, and the path relative to `PREFIX` is available in [.filename]#Makefile# as `WWWDIR_REL`. + + The user and group of web server process are available as `WWWOWN` and `WWWGRP`, in case the ownership of some files needs to be changed. + The default values of both are `www`. + Use `WWWOWN?= myuser` and `WWWGRP?= mygroup` if the port needs different values. + This allows the user to override them easily. + + [IMPORTANT] + ==== + Use `WWWOWN` and `WWWGRP` sparingly. + Remember that every file the web server can write to is a security risk waiting to happen. + ==== + + Do not depend on Apache unless the web app explicitly needs Apache. + Respect that users may wish to run a web application on a web server other than Apache. + + [[php-variables]] + === PHP + + PHP web applications declare their dependency on it with `USES=php`. + See crossref:uses[uses-php,`php`] for more information. + + [[php-pear]] + === PEAR Modules + + Porting PEAR modules is a very simple process. + + Add `USES=pear` to the port's [.filename]#Makefile#. + The framework will install the relevant files in the right places and automatically generate the plist at install time. + + [[pear-makefile]] + .Example Makefile for PEAR Class + [example] + ==== + [.programlisting] + .... + PORTNAME= Date + DISTVERSION= 1.4.3 + CATEGORIES= devel www pear + + MAINTAINER= someone@example.org + COMMENT= PEAR Date and Time Zone Classes + WWW= https://pear.php.net/package/Date/ + + USES= pear + + .include + .... + + ==== + + [TIP] + ==== + PEAR modules will automatically be flavorized using crossref:flavors[flavors-auto-php,PHP flavors]. + ==== + + [NOTE] + ==== + If a non default `PEAR_CHANNEL` is used, the build and run-time dependencies will automatically be added. + ==== + + [IMPORTANT] + ==== + PEAR modules do not need to defined `PKGNAMESUFFIX` it is automatically filled in using `PEAR_PKGNAMEPREFIX`. + If a port needs to add to `PKGNAMEPREFIX`, it must also use `PEAR_PKGNAMEPREFIX` to differentiate between different flavors. + ==== + + [[php-horde]] + ==== Horde Modules + + In the same way, porting Horde modules is a simple process. + + Add `USES=horde` to the port's [.filename]#Makefile#. + The framework will install the relevant files in the right places and automatically generate the plist at install time. + + The `USE_HORDE_BUILD` and `USE_HORDE_RUN` variables can be used to add buildtime and runtime dependencies on other Horde modules. + See [.filename]#Mk/Uses/horde.mk# for a complete list of available modules. + + [[horde-Makefile]] + .Example Makefile for Horde Module + [example] + ==== + [.programlisting] + .... + PORTNAME= Horde_Core + DISTVERSION= 2.14.0 + CATEGORIES= devel www pear + + MAINTAINER= horde@FreeBSD.org + COMMENT= Horde Core Framework libraries + WWW= https://pear.horde.org/ + + OPTIONS_DEFINE= KOLAB SOCKETS + KOLAB_DESC= Enable Kolab server support + SOCKETS_DESC= Depend on sockets PHP extension + + USES= horde + USE_PHP= session + + USE_HORDE_BUILD= Horde_Role + USE_HORDE_RUN= Horde_Role Horde_History Horde_Pack \ + Horde_Text_Filter Horde_View + + KOLAB_USE= HORDE_RUN=Horde_Kolab_Server,Horde_Kolab_Session + SOCKETS_USE= PHP=sockets + + .include + .... + + ==== + + [TIP] + ==== + As Horde modules are also PEAR modules they will also automatically be flavorized using crossref:flavors[flavors-auto-php,PHP flavors]. + ==== + + [[using-python]] + == Using Python + + The Ports Collection supports parallel installation of multiple Python versions. + Ports must use a correct `python` interpreter, according to the user-settable `PYTHON_VERSION`. + Most prominently, this means replacing the path to `python` executable in scripts with the value of `PYTHON_CMD`. + + Ports that install files under `PYTHON_SITELIBDIR` must use the `pyXY-` package name prefix, so their package name embeds the version of Python they are installed into. + + [.programlisting] + .... + PKGNAMEPREFIX= ${PYTHON_PKGNAMEPREFIX} + .... + + + [[using-python-variables]] + .Most Useful Variables for Ports That Use Python + [cols="1,1", frame="none"] + |=== + |`USES=python` + |The port needs Python. The minimal required version can be specified with values such as `3.10+`. Version ranges can also be specified by separating two version numbers with a dash: `USES=python:3.8-3.9`. Note that `USES=python` does _not_ cover Python 2.7, it needs to be requested explicitly with `USES=python:2.7+`. + + |`USE_PYTHON=distutils` + |Use Python distutils for configuring, compiling, and installing. This is required when the port comes with [.filename]#setup.py#. This overrides the `do-build` and `do-install` targets and may also override `do-configure` if `GNU_CONFIGURE` is not defined. Additionally, it implies `USE_PYTHON=flavors`. + + |`USE_PYTHON=autoplist` + |Create the packaging list automatically. This also requires `USE_PYTHON=distutils` to be set. + + |`USE_PYTHON=concurrent` + |The port will use an unique prefix, typically `PYTHON_PKGNAMEPREFIX` for certain directories, such as `EXAMPLESDIR` and `DOCSDIR` and also will append a suffix, the python version from `PYTHON_VER`, to binaries and scripts to be installed. This allows ports to be installed for different Python versions at the same time, which otherwise would install conflicting files. + + |`USE_PYTHON=flavors` + |The port does not use distutils but still supports multiple Python versions. `FLAVORS` will be set to the supported Python versions. See crossref:flavors[flavors-auto-python,`USES`=python and Flavors] for more information. + + |`USE_PYTHON=optsuffix` + |If the current Python version is not the default version, the port will gain `PKGNAMESUFFIX=${PYTHON_PKGNAMESUFFIX}`. Only useful with flavors. + + |`PYTHON_PKGNAMEPREFIX` + |Used as a `PKGNAMEPREFIX` to distinguish packages for different Python versions. Example: `py27-` + + |`PYTHON_SITELIBDIR` + |Location of the site-packages tree, that contains installation path of Python (usually `LOCALBASE`). `PYTHON_SITELIBDIR` can be very useful when installing Python modules. + + |`PYTHONPREFIX_SITELIBDIR` + |The PREFIX-clean variant of PYTHON_SITELIBDIR. Always use `%%PYTHON_SITELIBDIR%%` in [.filename]#pkg-plist# when possible. The default value of `%%PYTHON_SITELIBDIR%%` is `lib/python%%PYTHON_VERSION%%/site-packages` + + |`PYTHON_CMD` + |Python interpreter command line, including version number. + |=== + + [[using-python-variables-helpers]] + .Python Module Dependency Helpers + [cols="1,1", frame="none"] + |=== + |`PYNUMERIC` + |Dependency line for numeric extension. + + |`PYNUMPY` + |Dependency line for the new numeric extension, numpy. (PYNUMERIC is deprecated by upstream vendor). + + |`PYXML` + |Dependency line for XML extension (not needed for Python 2.0 and higher as it is also in base distribution). + + |`PY_ENUM34` + |Conditional dependency on package:devel/py-enum34[] depending on the Python version. + + |`PY_ENUM_COMPAT` + |Conditional dependency on package:devel/py-enum-compat[] depending on the Python version. + + |`PY_PATHLIB` + |Conditional dependency on package:devel/py-pathlib[] depending on the Python version. + + |`PY_IPADDRESS` + |Conditional dependency on package:net/py-ipaddress[] depending on the Python version. + + |`PY_FUTURES` + |Conditional dependency on package:devel/py-futures[] depending on the Python version. + |=== + + A complete list of available variables can be found in [.filename]#/usr/ports/Mk/Uses/python.mk#. + + [IMPORTANT] + ==== + All dependencies to Python ports using crossref:flavors[flavors-auto-python,Python flavors] (either with `USE_PYTHON=distutils` or `USE_PYTHON=flavors`) must have the Python flavor appended to their origin using `@${PY_FLAVOR}`. + See crossref:special[python-Makefile,Makefile for a Simple Python Module]. + ==== + + [[python-Makefile]] + .Makefile for a Simple Python Module + [example] + ==== + [.programlisting] + .... + PORTNAME= sample + DISTVERSION= 1.2.3 + CATEGORIES= devel + + MAINTAINER= fred.bloggs@example.com + COMMENT= Python sample module + WWW= https://example.com/project/sample/ + + RUN_DEPENDS= ${PYTHON_PKGNAMEPREFIX}six>0:devel/py-six@${PY_FLAVOR} + + USES= python + USE_PYTHON= autoplist distutils + + .include + .... + + ==== + + Some Python applications claim to have `DESTDIR` support (which would be required for staging) but it is broken (Mailman up to 2.1.16, for instance). + This can be worked around by recompiling the scripts. + This can be done, for example, in the `post-build` target. + Assuming the Python scripts are supposed to reside in `PYTHONPREFIX_SITELIBDIR` after installation, this solution can be applied: + + [.programlisting] + .... + (cd ${STAGEDIR}${PREFIX} \ + && ${PYTHON_CMD} ${PYTHON_LIBDIR}/compileall.py \ + -d ${PREFIX} -f ${PYTHONPREFIX_SITELIBDIR:S;${PREFIX}/;;}) + .... + + This recompiles the sources with a path relative to the stage directory, and prepends the value of `PREFIX` to the file name recorded in the byte-compiled output file by `-d`. `-f` is required to force recompilation, and the `:S;${PREFIX}/;;` strips prefixes from the value of `PYTHONPREFIX_SITELIBDIR` to make it relative to `PREFIX`. + + [[using-tcl]] + == Using Tcl/Tk + + The Ports Collection supports parallel installation of multiple Tcl/Tk versions. + Ports should try to support at least the default Tcl/Tk version and higher with `USES=tcl`. + It is possible to specify the desired version of `tcl` by appending `:_xx_`, for example, `USES=tcl:85`. + + [[using-tcl-variables]] + .The Most Useful Read-Only Variables for Ports That Use Tcl/Tk + [cols="1,1", frame="none"] + |=== + |`TCL_VER` + | chosen major.minor version of Tcl + + |`TCLSH` + | full path of the Tcl interpreter + + |`TCL_LIBDIR` + | path of the Tcl libraries + + |`TCL_INCLUDEDIR` + | path of the Tcl C header files + + |`TCL_PKG_LIB_PREFIX` + | Library prefix, as per TIP595 + + |`TCL_PKG_STUB_POSTFIX` + |Stub library postfix + + |`TK_VER` + | chosen major.minor version of Tk + + |`WISH` + | full path of the Tk interpreter + + |`TK_LIBDIR` + | path of the Tk libraries + + |`TK_INCLUDEDIR` + | path of the Tk C header files + |=== + + See the crossref:uses[uses-tcl,`USES=tcl`] and crossref:uses[uses-tk,`USES=tk`] of crossref:uses[uses,Using `USES` Macros] for a full description of those variables. + A complete list of those variables is available in [.filename]#/usr/ports/Mk/Uses/tcl.mk#. + + [[using-sdl]] + == Using SDL + + `USE_SDL` is used to autoconfigure the dependencies for ports which use an SDL based library like package:devel/sdl12[] and package:graphics/sdl_image[]. + + These SDL libraries for version 1.2 are recognized: + + * sdl: package:devel/sdl12[] + * console: package:devel/sdl_console[] + * gfx: package:graphics/sdl_gfx[] + * image: package:graphics/sdl_image[] + * mixer: package:audio/sdl_mixer[] + * mm: package:devel/sdlmm[] + * net: package:net/sdl_net[] + * pango: package:x11-toolkits/sdl_pango[] + * sound: package:audio/sdl_sound[] + * ttf: package:graphics/sdl_ttf[] + + These SDL libraries for version 2.0 are recognized: + + * sdl: package:devel/sdl20[] + * gfx: package:graphics/sdl2_gfx[] + * image: package:graphics/sdl2_image[] + * mixer: package:audio/sdl2_mixer[] + * net: package:net/sdl2_net[] + * ttf: package:graphics/sdl2_ttf[] + + Therefore, if a port has a dependency on package:net/sdl_net[] and package:audio/sdl_mixer[], the syntax will be: + + [.programlisting] + .... + USE_SDL= net mixer + .... + + The dependency package:devel/sdl12[], which is required by package:net/sdl_net[] and package:audio/sdl_mixer[], is automatically added as well. + + Using `USE_SDL` with entries for SDL 1.2, it will automatically: + + * Add a dependency on sdl12-config to `BUILD_DEPENDS` + * Add the variable `SDL_CONFIG` to `CONFIGURE_ENV` + * Add the dependencies of the selected libraries to `LIB_DEPENDS` + + Using `USE_SDL` with entries for SDL 2.0, it will automatically: + + * Add a dependency on sdl2-config to `BUILD_DEPENDS` + * Add the variable `SDL2_CONFIG` to `CONFIGURE_ENV` + * Add the dependencies of the selected libraries to `LIB_DEPENDS` + + + [[using-wx]] + == Using wxWidgets + + This section describes the status of the wxWidgets libraries in the ports tree and its integration with the ports system. + + [[wx-introduction]] + === Introduction + + There are many versions of the wxWidgets libraries which conflict between them (install files under the same name). + In the ports tree this problem has been solved by installing each version under a different name using version number suffixes. + + The obvious disadvantage of this is that each application has to be modified to find the expected version. + Fortunately, most of the applications call the `wx-config` script to determine the necessary compiler and linker flags. + The script is named differently for every available version. + Majority of applications respect an environment variable, or accept a configure argument, to specify which `wx-config` script to call. + Otherwise they have to be patched. + + [[wx-version]] + === Version Selection + + To make the port use a specific version of wxWidgets there are two variables available for defining (if only one is defined the other will be set to a default value): + + [[wx-ver-sel-table]] + .Variables to Select wxWidgets Versions + [cols="1,1,1", frame="none", options="header"] + |=== + | Variable + | Description + | Default value + + |`USE_WX` + |List of versions the port can use + |All available versions + + |`USE_WX_NOT` + |List of versions the port cannot use + |None + |=== + + The available wxWidgets versions and the corresponding ports in the tree are: + + [[wx-widgets-versions-table]] + .Available wxWidgets Versions + [cols="1,1", frame="none", options="header"] + |=== + | Version + | Port + + |`2.8` + |package:x11-toolkits/wxgtk28[] + + |`3.0` + |package:x11-toolkits/wxgtk30[] + |=== + + The variables in crossref:special[wx-ver-sel-table,Variables to Select wxWidgets Versions] can be set to one or more of these combinations separated by spaces: + + [[wx-widgets-versions-specification]] + .wxWidgets Version Specifications + [cols="1,1", frame="none", options="header"] + |=== + | Description + | Example + + |Single version + |`2.8` + + |Ascending range + |`2.8+` + + |Descending range + |`3.0-` + + |Full range (must be ascending) + |`2.8-3.0` + |=== + + There are also some variables to select the preferred versions from the available ones. + They can be set to a list of versions, the first ones will have higher priority. + [[wx-widgets-preferred-version]] + .Variables to Select Preferred wxWidgets Versions + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Designed for + + |`WANT_WX_VER` + |the port + + |`WITH_WX_VER` + |the user + |=== + + [[wx-components]] + === Component Selection + + There are other applications that, while not being wxWidgets libraries, are related to them. + These applications can be specified in `WX_COMPS`. These components are available: + + [[wx-widgets-components-table]] + .Available wxWidgets Components + [cols="1,1,1", frame="none", options="header"] + |=== + | Name + | Description + | Version restriction + + |`wx` + |main library + |none + + |`contrib` + |contributed libraries + |`none` + + |`python` + |wxPython (Python bindings) + |`2.8-3.0` + |=== + + The dependency type can be selected for each component by adding a suffix separated by a semicolon. + If not present then a default type will be used (see crossref:special[wx-def-dep-types,Default wxWidgets Dependency Types]). + These types are available: + + [[wx-widgets-dependency-table]] + .Available wxWidgets Dependency Types + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Description + + |`build` + |Component is required for building, equivalent to `BUILD_DEPENDS` + + |`run` + |Component is required for running, equivalent to `RUN_DEPENDS` + + |`lib` + |Component is required for building and running, equivalent to `LIB_DEPENDS` + |=== + + The default values for the components are detailed in this table: + + [[wx-def-dep-types]] + .Default wxWidgets Dependency Types + [cols="1,1", frame="none", options="header"] + |=== + | Component + | Dependency type + + |`wx` + |`lib` + + |`contrib` + |`lib` + + |`python` + |`run` + + |`mozilla` + |`lib` + + |`svg` + |`lib` + |=== + + [[wx-components-example]] + .Selecting wxWidgets Components + [example] + ==== + This fragment corresponds to a port which uses wxWidgets version `2.4` and its contributed libraries. + + [.programlisting] + .... + USE_WX= 2.8 + WX_COMPS= wx contrib + .... + + ==== + + [[wx-version-detection]] + === Detecting Installed Versions + + To detect an installed version, define `WANT_WX`. + If it is not set to a specific version then the components will have a version suffix. + `HAVE_WX` will be filled after detection. + + [[wx-ver-det-example]] + .Detecting Installed wxWidgets Versions and Components + [example] + ==== + This fragment can be used in a port that uses wxWidgets if it is installed, or an option is selected. + + [.programlisting] + .... + WANT_WX= yes + + .include + + .if defined(WITH_WX) || !empty(PORT_OPTIONS:MWX) || !empty(HAVE_WX:Mwx-2.8) + USE_WX= 2.8 + CONFIGURE_ARGS+= --enable-wx + .endif + .... + + This fragment can be used in a port that enables wxPython support if it is installed or if an option is selected, in addition to wxWidgets, both version `2.8`. + + [.programlisting] + .... + USE_WX= 2.8 + WX_COMPS= wx + WANT_WX= 2.8 + + .include + + .if defined(WITH_WXPYTHON) || !empty(PORT_OPTIONS:MWXPYTHON) || !empty(HAVE_WX:Mpython) + WX_COMPS+= python + CONFIGURE_ARGS+= --enable-wxpython + .endif + .... + + ==== + + [[wx-defined-variables]] + === Defined Variables + + These variables are available in the port (after defining one from crossref:special[wx-ver-sel-table,Variables to Select wxWidgets Versions]). + + [[wx-widgets-variables]] + .Variables Defined for Ports That Use wxWidgets + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Description + + |`WX_CONFIG` + |The path to the wxWidgets`wx-config` script (with different name) + + |`WXRC_CMD` + |The path to the wxWidgets`wxrc` program (with different name) + + |`WX_VERSION` + |The wxWidgets version that is going to be used (for example, `2.6`) + |=== + + [[wx-premk]] + === Processing in [.filename]#bsd.port.pre.mk# + + Define `WX_PREMK` to be able to use the variables right after including [.filename]#bsd.port.pre.mk#. + + [IMPORTANT] + ==== + When defining `WX_PREMK`, then the version, dependencies, components and defined variables will not change if modifying the wxWidgets port variables _after_ including [.filename]#bsd.port.pre.mk#. + ==== + + [[wx-premk-example]] + .Using wxWidgets Variables in Commands + [example] + ==== + This fragment illustrates the use of `WX_PREMK` by running the `wx-config` script to obtain the full version string, assign it to a variable and pass it to the program. + + [.programlisting] + .... + USE_WX= 2.8 + WX_PREMK= yes + + .include + + .if exists(${WX_CONFIG}) + VER_STR!= ${WX_CONFIG} --release + + PLIST_SUB+= VERSION="${VER_STR}" + .endif + .... + + ==== + + [NOTE] + ==== + The wxWidgets variables can be safely used in commands when they are inside targets without the need of `WX_PREMK`. + ==== + + [[wx-additional-config-args]] + === Additional `configure` Arguments + + Some GNU `configure` scripts cannot find wxWidgets with just the `WX_CONFIG` environment variable set, requiring additional arguments. `WX_CONF_ARGS` can be used for provide them. + + [[wx-conf-args-values]] + .Legal Values for `WX_CONF_ARGS` + [cols="1,1", frame="none", options="header"] + |=== + | Possible value + | Resulting argument + + |`absolute` + |`--with-wx-config=${WX_CONFIG}` + + |`relative` + |`--with-wx=${LOCALBASE} --with-wx-config=${WX_CONFIG:T}` + |=== + + [[using-lua]] + == Using Lua + + This section describes the status of the Lua libraries in the ports tree and its integration with the ports system. + + [[lua-introduction]] + === Introduction + + There are many versions of the Lua libraries and corresponding interpreters, which conflict between them (install files under the same name). + In the ports tree this problem has been solved by installing each version under a different name using version number suffixes. + + The obvious disadvantage of this is that each application has to be modified to find the expected version. + But it can be solved by adding some additional flags to the compiler and linker. + + Applications that use Lua should normally build for just one version. + However, loadable modules for Lua are built in a separate flavor for each Lua version that they support, and dependencies on such modules should specify the flavor using the `@${LUA_FLAVOR}` suffix on the port origin. + + [[lua-version]] + === Version Selection + + A port using Lua should have a line of this form: + + [.programlisting] + .... + USES= lua + .... + + If a specific version of Lua, or range of versions, is needed, it can be specified as a parameter in the form `XY` (which may be used multiple times), `XY+`, `-XY`, or `XY-ZA`. + The default version of Lua as set via `DEFAULT_VERSIONS` will be used if it falls in the requested range, otherwise the closest requested version to the default will be used. + For example: + + [.programlisting] + .... + USES= lua:52-53 + .... + + Note that no attempt is made to adjust the version selection based on the presence of any already-installed Lua version. + + [NOTE] + ==== + The `XY+` form of version specification should not be used without careful consideration; + the Lua API changes to some extent in every version, and configuration tools like CMake or Autoconf will often fail to work on future versions of Lua until updated to do so. + ==== + + [[lua-version-config]] + === Configuration and Compiler flags + + Software that uses Lua may have been written to auto-detect the Lua version in use. + In general ports should override this assumption, and force the use of the specific Lua version selected as described above. + Depending on the software being ported, this might require any or all of: + + * Using `LUA_VER` as part of a parameter to the software's configuration script via `CONFIGURE_ARGS` or `CONFIGURE_ENV` (or equivalent for other build systems); + * Adding `-I${LUA_INCDIR}`, `-L${LUA_LIBDIR}`, and `-llua-${LUA_VER}` to `CFLAGS`, `LDFLAGS`, `LIBS` respectively as appropriate; + * Patch the software's configuration or build files to select the correct version. + + + [[lua-version-flavors]] + === Version Flavors + + A port which installs a Lua module (rather than an application that simply makes use of Lua) should build a separate flavor for each supported Lua version. + This is done by adding the `module` parameter: + + [.programlisting] + .... + USES= lua:module + .... + + A version number or range of versions can be specified as well; use a comma to separate parameters. + + Since each flavor must have a different package name, the variable `LUA_PKGNAMEPREFIX` is provided which will be set to an appropriate value; the intended usage is: + + [.programlisting] + .... + PKGNAMEPREFIX= ${LUA_PKGNAMEPREFIX} + .... + + Module ports should normally install files only to `LUA_MODLIBDIR`, `LUA_MODSHAREDIR`, `LUA_DOCSDIR`, and `LUA_EXAMPLESDIR`, all of which are set up to refer to version-specific subdirectories. + Installing any other files must be done with care to avoid conflicts between versions. + + A port (other than a Lua module) which wishes to build a separate package for each Lua version should use the `flavors` parameter: + + [.programlisting] + .... + USES= lua:flavors + .... + + This operates the same way as the `module` parameter described above, but without the assumption that the package should be documented as a Lua module (so `LUA_DOCSDIR` and `LUA_EXAMPLESDIR` are not defined by default). + However, the port may choose to define `LUA_DOCSUBDIR` as a suitable subdirectory name (usually the port's `PORTNAME` as long as this does not conflict with the `PORTNAME` of any module), in which case the framework will define both `LUA_DOCSDIR` and `LUA_EXAMPLESDIR`. + + As with module ports, a flavored port should avoid installing files that would conflict between versions. + Typically this is done by adding `LUA_VER_STR` as a suffix to program names (e.g. using crossref:uses[uses-uniquefiles,`uniquefiles`]), and otherwise using either `LUA_VER` or `LUA_VER_STR` as part of any other files or subdirectories used outside of `LUA_MODLIBDIR` and `LUA_MODSHAREDIR`. + + [[lua-defined-variables]] + === Defined Variables + + These variables are available in the port. + + [[using-lua-variables-ports]] + .Variables Defined for Ports That Use Lua + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Description + + |`LUA_VER` + |The Lua version that is going to be used (for example, `5.4`) + + |`LUA_VER_STR` + |The Lua version without the dots (for example, `54`) + + |`LUA_FLAVOR` + |The flavor name corresponding to the selected Lua version, to be used for specifying dependencies + + |`LUA_BASE` + |The prefix that should be used to locate Lua (and components) that are already installed + + |`LUA_PREFIX` + |The prefix where Lua (and components) are to be installed by this port + + |`LUA_INCDIR` + |The directory where Lua header files are installed + + |`LUA_LIBDIR` + |The directory where Lua libraries are installed + + |`LUA_REFMODLIBDIR` + |The directory where Lua module libraries ([.filename]#.so#) that are already installed are to be found + + |`LUA_REFMODSHAREDIR` + |The directory where Lua modules ([.filename]#.lua#) that are already installed are to be found + + |`LUA_MODLIBDIR` + |The directory where Lua module libraries ([.filename]#.so#) are to be installed by this port + + |`LUA_MODSHAREDIR` + |The directory where Lua modules ([.filename]#.lua#) are to be installed by this port + + |`LUA_PKGNAMEPREFIX` + |The package name prefix used by Lua modules + + |`LUA_CMD` + |The name of the Lua interpreter (e.g. `lua54`) + + |`LUAC_CMD` + |The name of the Lua compiler (e.g. `luac54`) + |=== + + These additional variables are available for ports that specified the `module` parameter: + + [[using-lua-variables-modules]] + .Variables Defined for Lua Module Ports + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Description + + |`LUA_DOCSDIR` + |the directory to which the module's documentation should be installed. + + |`LUA_EXAMPLESDIR` + |the directory to which the module's example files should be installed. + |=== + + [[lua-examples]] + === Examples + + [[lua-app-Makefile]] + .Makefile for an application using Lua + [example] + ==== + This example shows how to reference a Lua module required at run time. + Notice that the reference must specify a flavor. + + [.programlisting] + .... + PORTNAME= sample + DISTVERSION= 1.2.3 + CATEGORIES= whatever + + MAINTAINER= fred.bloggs@example.com + COMMENT= Sample + WWW= https://example.com/lua_sample/sample/ + + RUN_DEPENDS= ${LUA_REFMODLIBDIR}/lpeg.so:devel/lua-lpeg@${LUA_FLAVOR} + + USES= lua + + .include + .... + + ==== + + [[lua-mod-Makefile]] + .Makefile for a simple Lua module + [example] + ==== + [.programlisting] + .... + PORTNAME= sample + DISTVERSION= 1.2.3 + CATEGORIES= whatever + PKGNAMEPREFIX= ${LUA_PKGNAMEPREFIX} + + MAINTAINER= fred.bloggs@example.com + COMMENT= Sample + WWW= https://example.com/lua_sample/sample/ + + USES= lua:module + + DOCSDIR= ${LUA_DOCSDIR} + + .include + .... + + ==== + + [[using-guile]] + == Using Guile + + This section describes the status of Guile in the ports tree and its integration with the ports system. + + [[guile-introduction]] + === Introduction + + There are multiple versions of the Guile libraries and corresponding interpreters, which conflict between them (install files under the same name). + In the ports tree this problem has been solved by installing each version under a different name using version number suffixes. + In most cases, applications should detect the correct version from the configuration variables provided and use `pkg-config` to determine the name and associated paths. + However, some applications (especially those using their own configuration rules for `cmake` or `meson`) will always try to use the latest available version. + In this case, either patch the port or declare a build conflict (see the `conflicts` option below) to ensure that the correct dependency is generated when building outside of poudriere. + + Applications that use Guile should normally build for just one version, + preferably the one specified in `DEFAULT_VERSIONS`, + or failing that the latest version that they support. + However, Guile or Scheme libraries, or extension modules for Guile are built in a separate flavor for each Guile version that they support, + and dependencies on such ports should specify the flavor using the `@${GUILE_FLAVOR}` suffix on the port origin. + + [[guile-version]] + === Version Selection + + A port using Guile should define `USES=guile:__arg,arg...__` with appropriate arguments as follows: + + [[guile-defined-uses-args]] + .Arguments Defined for Ports That Use Guile + [cols="1m,4", frame="none", options="header"] + |=== + | Name + | Description + + |_X.Y_ + |Declare compatibility with Guile version `X.Y`. + Currently available versions are `1.8` (obsolete), `2.2` and `3.0`. + Multiple versions may be specified. + + |flavors + |Create a flavor for every Guile version specified. + The version specified by `DEFAULT_VERSIONS` will become the default flavor. + Flavor names are of the form `guileXY`. + + |build + |Add the Guile interpreter as a build dependency only, rather than a library dependency. + `build` and `run` may both be specified. + + |run + |Add the Guile interpreter as a runtime dependency only, rather than a library dependency. + `build` and `run` may both be specified. + + |alias + |Add `BINARY_ALIAS` values for the interpreter and tools. + + |conflicts + |Declare `CONFLICTS_BUILD` for Guile versions newer than the one selected. + Use this when the port cannot be configured to use a specific Guile version. + |=== + + Some additional arguments are available for handling unusual cases; see `Mk/Uses/guile.mk` for details. + + Unless `build` or `run` is specified, then `LIB_DEPENDS` receives both the `libguile` library dependency and also any additional dependencies required by the guile version, e.g. `libgc`. + Normally the port should not need any additional dependencies related to its use of Guile. + + [[guile-version-config]] + === Configuration flags + + Software that uses Guile should be using the `pkg-config` mechanism to obtain compiler and linker flags. + Some older or esoteric ports may be using `guile-config` or obtaining values directly from `guile` instead, + which should also work (the `alias` argument may be useful in some of these cases). + + The framework tries to inform the port of the desired Guile version using the following methods: + + * `GUILE_EFFECTIVE_VERSION` is added to `CONFIGURE_ENV`; + * The full path to the Guile binary is specified in the `GUILE` variable in `CONFIGURE_ENV` and `MAKE_ENV`; + * If the `alias` option is used, the desired Guile version's binaries are the ones aliased; + * If the `alias` option is not used, paths to the desired Guile version's tools (`guild`, `guile-config`, etc.) are added to `CONFIGURE_ENV` and `MAKE_ENV` as variables `GUILD`, `GUILE_CONFIG`, etc. + + For some ports, it may be necessary to specify the version in additional ways, such as via `CONFIGURE_ARGS` or `MESON_ARGS`, + depending on the port. + + If none of these methods cause the port to select the specified Guile version when other versions are present, + then preferably patch it to do so. + If that is not feasible, specify the `conflicts` option to prevent building the port under conditions where it will detect the wrong version. + + [[guile-version-flavors]] + === Version Flavors + + A port which installs a Guile extension or library, or a Scheme library that precompiles for Guile, + should build a separate flavor for each supported Guile version. + This is done by adding the `flavors` option. + + Since each flavor must have a different package name, such ports must set `PKGNAMESUFFIX`, typically: + + [.programlisting] + .... + PKGNAMESUFFIX= -${FLAVOR} + .... + + Such ports must install Scheme files to `GUILE_SITE_DIR` rather than to `GUILE_GLOBAL_SITE_DIR` even when the files are not version-specific. + This often requires patching the port. + + Additionally, if such a port installs a `.pc` file, it must be placed in `GUILE_PKGCONFIG_PATH` rather than in the global `pkgconfig` directory. + This allows dependent ports to find a correct configuration for the specific Guile version in use. + + If a Guile extension port installs a `.so` file, then it must usually be placed in the Guile-version-specific `extensions` directory. + `USE_LDCONFIG` should usually not be used. + + Any other files installed by a flavored port must likewise be in version-specific directories or use version-specific filenames. + For documentation and examples, `GUILE_DOCS_DIR` and `GUILE_EXAMPLES_DIR` specify suitable locations in which the port should create a subdirectory, + see below. + + [[guile-defined-variables]] + === Defined Variables + + These variables are available in the port. + + [[using-guile-variables-ports]] + .Variables Defined for Ports That Use Guile + [cols="1,3m,6", frame="none", options="header"] + |=== + | Name + | Sample Value + | Description + + |`GUILE_VER` + |3.0 + |Guile version in use. + + |`GUILE_SFX` + |3 + |Short suffix used on some names. + Use only with care; may be non-unique or may change in the future. + + |`GUILE_FLAVOR` + |guile30 + |Flavor name corresponding to the selected version. + + |`GUILE_PORT` + |lang/guile3 + |Port origin of the specified Guile version. + + |`GUILE_PREFIX` + |${PREFIX} + |Directory prefix to be used for installation. + + |`GUILE_CMD` + |guile-3.0 + |Name of the Guile interpreter, with version suffix. + + |`GUILE_CMDPATH` + |${LOCALBASE}/bin/guile-3.0 + |Full path to the Guile interpreter. + + |`GUILD_CMD` + |guild-3.0 + |Name of the Guild tool, with version suffix. + + |`GUILD_CMDPATH` + |${LOCALBASE}/bin/guild-3.0 + |Full path to the Guild tool. + + |`++GUILE_*_CMD++` + + `++GUILE_*_CMDPATH++` + | + |Like `GUILE_CMD` and `GUILE_CMDPATH`, but for other tool binaries. + + |`GUILE_PKGCONFIG_PATH` + |${LOCALBASE}/libdata/pkgconfig/guile/3.0 + |Where packages using `flavors` should install `.pc` files. + + |`GUILE_INFO_PATH` + |share/info/guile3 + |A suitable value for `INFO_PATH` for ports using the `flavors` option. + |=== + + The following are defined as variables and as `PLIST_SUB` entries. + The variable form is suffixed with `_DIR` and is a full path (prefixed with `GUILE_PREFIX`). + + [[using-guile-path-variables-ports]] + .Path Substitutions Defined for Ports That Use Guile + [cols="1m,3m,6", frame="none", options="header"] + |=== + | Name + | Sample Value + | Description + + |GUILE_GLOBAL_SITE + |share/guile/site + |Site directory shared by all guile versions; this should not usually be used. + + |GUILE_SITE + |share/guile/3.0/site + |Site directory for the selected Guile version. + + |GUILE_SITE_CCACHE + |lib/guile/3.0/site-ccache + |Directory for compiled bytecode files. + + |GUILE_DOCS + |share/doc/guile30 + |Parent directory for version-specific documentation. + + |GUILE_EXAMPLES + |share/examples/guile30 + |Parent directory for version-specific examples. + |=== + + [[guile-examples]] + === Examples + + [[guile-app-Makefile]] + .Makefile for an application using Guile + [example] + ==== + This example shows how to reference a Guile library required at build and run time. + Notice that the reference must specify a flavor. + This example assumes that the application is using `pkg-config` to locate dependencies. + + [.programlisting] + .... + PORTNAME= sample + DISTVERSION= 1.2.3 + CATEGORIES= whatever + + MAINTAINER= fred.bloggs@example.com + COMMENT= Sample + WWW= https://example.com/guile_sample/sample/ + + BUILD_DEPENDS= guile-lib-${GUILE_FLAVOR}>=0.2.5:devel/guile-lib@${GUILE_FLAVOR} + RUN_DEPENDS= guile-lib-${GUILE_FLAVOR}>=0.2.5:devel/guile-lib@${GUILE_FLAVOR} + + USES= guile:2.2,3.0 pkgconfig + + .include + .... + + ==== + + [[using-iconv]] + == Using `iconv` + + FreeBSD has a native `iconv` in the operating system. + + For software that needs `iconv`, define `USES=iconv`. + + When a port defines `USES=iconv`, these variables will be available: + + [.informaltable] + [cols="1,1,1,1", frame="none", options="header"] + |=== + | Variable name + | Purpose + | Port iconv (when using WCHAR_T or //TRANSLIT extensions) + | Base iconv + + + |`ICONV_CMD` + |Directory where the `iconv` binary resides + |`${LOCALBASE}/bin/iconv` + |[.filename]#/usr/bin/iconv# + + |`ICONV_LIB` + |`ld` argument to link to [.filename]#libiconv# (if needed) + |`-liconv` + |(empty) + + |`ICONV_PREFIX` + |Directory where the `iconv` implementation resides (useful for configure scripts) + |`${LOCALBASE}` + |[.filename]#/usr# + + |`ICONV_CONFIGURE_ARG` + |Preconstructed configure argument for configure scripts + |`--with-libiconv-prefix=${LOCALBASE}` + |(empty) + + |`ICONV_CONFIGURE_BASE` + |Preconstructed configure argument for configure scripts + |`--with-libiconv=${LOCALBASE}` + |(empty) + |=== + + These two examples automatically populate the variables with the correct value for systems using package:converters/libiconv[] or the native `iconv` respectively: + + [[iconv-simple-use]] + .Simple `iconv` Usage + [example] + ==== + [.programlisting] + .... + USES= iconv + LDFLAGS+= -L${LOCALBASE}/lib ${ICONV_LIB} + .... + + ==== + + [[iconv-configure-use]] + .`iconv` Usage with `configure` + [example] + ==== + [.programlisting] + .... + USES= iconv + CONFIGURE_ARGS+=${ICONV_CONFIGURE_ARG} + .... + + ==== + + As shown above, `ICONV_LIB` is empty when a native `iconv` is present. + This can be used to detect the native `iconv` and respond appropriately. + + Sometimes a program has an `ld` argument or search path hardcoded in a [.filename]#Makefile# or configure script. + This approach can be used to solve that problem: + + [[iconv-reinplace]] + .Fixing Hardcoded `-liconv` + [example] + ==== + [.programlisting] + .... + USES= iconv + + post-patch: + @${REINPLACE_CMD} -e 's/-liconv/${ICONV_LIB}/' ${WRKSRC}/Makefile + .... + + ==== + + In some cases it is necessary to set alternate values or perform operations depending on whether there is a native `iconv`. + [.filename]#bsd.port.pre.mk# must be included before testing the value of `ICONV_LIB`: + + [[iconv-conditional]] + .Checking for Native `iconv` Availability + [example] + ==== + [.programlisting] + .... + USES= iconv + + .include + + post-patch: + .if empty(ICONV_LIB) + # native iconv detected + @${REINPLACE_CMD} -e 's|iconv||' ${WRKSRC}/Config.sh + .endif + + .include + .... + + ==== + + [[using-xfce]] + == Using Xfce + + Ports that need Xfce libraries or applications set `USES=xfce`. + + Specific Xfce library and application dependencies are set with values assigned to `USE_XFCE`. + They are defined in [.filename]#/usr/ports/Mk/Uses/xfce.mk#. + The possible values are: + + .Values of `USE_XFCE` + garcon:: + package:sysutils/garcon[] + + libexo:: + package:x11/libexo[] + + libgui:: + package:x11-toolkits/libxfce4gui[] + + libmenu:: + package:x11/libxfce4menu[] + + libutil:: + package:x11/libxfce4util[] + + panel:: + package:x11-wm/xfce4-panel[] + + thunar:: + package:x11-fm/thunar[] + + xfconf:: + package:x11/xfce4-conf[] + + [[use-xfce]] + .`USES=xfce` Example + [example] + ==== + [.programlisting] + .... + USES= xfce + USE_XFCE= libmenu + .... + + ==== + + [[use-xfce-gtk2]] + .Using Xfce's Own GTK2 Widgets + [example] + ==== + In this example, the ported application uses the GTK2-specific widgets package:x11/libxfce4menu[] and package:x11/xfce4-conf[]. + + [.programlisting] + .... + USES= xfce:gtk2 + USE_XFCE= libmenu xfconf + .... + + ==== + + [TIP] + ==== + Xfce components included this way will automatically include any dependencies they need. + It is no longer necessary to specify the entire list. + If the port only needs package:x11-wm/xfce4-panel[], use: + + [.programlisting] + .... + USES= xfce + USE_XFCE= panel + .... + + There is no need to list the components package:x11-wm/xfce4-panel[] needs itself like this: + + [.programlisting] + .... + USES= xfce + USE_XFCE= libexo libmenu libutil panel + .... + + However, Xfce components and non-Xfce dependencies of the port must be included explicitly. + Do not count on an Xfce component to provide a sub-dependency other than itself for the main port. + ==== + + [[using-budgie]] + == Using Budgie + + Applications or libraries depending on the Budgie desktop should set `USES= budgie` and set `USE_BUDGIE` to the list of required components. + + [cols="1,1", frame="none", options="header"] + |=== + | Name + | Description + + | `libbudgie` + | Desktop core (library) + + | `libmagpie` + | Budgie's X11 window manager and compositor library + + | `raven` + | All-in-one center in panel for accessing different applications widgets + + | `screensaver` + | Desktop-specific screensaver + + |=== + + [NOTE] + ==== + All application widgets communicate through the *org.budgie_desktop.Raven* service. + + The default dependency is lib- and run-time, it can be changed with `:build` or `:run`, for example: + + [.programlisting] + .... + USES= budgie + USE_BUDGIE= screensaver:build + .... + ==== + + [[budgie-components-example]] + .`USE_BUDGIE` Example + [example] + ==== + [.programlisting] + .... + USES= budgie gettext gnome meson pkgconfig + USE_BUDGIE= libbudgie + .... + ==== + + [[using-databases]] + == Using Databases + + Use one of the `USES` macros from crossref:special[using-databases-uses,Database `USES` Macros] to add a dependency on a database. + + [[using-databases-uses]] + .Database `USES` Macros + [cols="1,1", frame="none", options="header"] + |=== + | Database + | USES Macro + + |Berkeley DB + |crossref:uses[uses-bdb,`bdb`] + + |MariaDB, MySQL, Percona + |crossref:uses[uses-mysql,`mysql`] + + |PostgreSQL + |crossref:uses[uses-pgsql,`pgsql`] + + |SQLite + |crossref:uses[uses-sqlite,`sqlite`] + |=== + + [[using-databases-bdb-ex1]] + .Using Berkeley DB 6 + [example] + ==== + [.programlisting] + .... + USES= bdb:6 + .... + + See crossref:uses[uses-bdb,`bdb`] for more information. + ==== + + [[using-databases-mysql-ex1]] + .Using MySQL + [example] + ==== + When a port needs the MySQL client library add + + [.programlisting] + .... + USES= mysql + .... + + See crossref:uses[uses-mysql,`mysql`] for more information. + ==== + + [[using-databases-pgsql-ex1]] + .Using PostgreSQL + [example] + ==== + When a port needs the PostgreSQL server version 9.6 or later add + + [.programlisting] + .... + USES= pgsql:9.6+ + WANT_PGSQL= server + .... + + See crossref:uses[uses-pgsql,`pgsql`] for more information. + ==== + + [[using-databases-sqlite-ex1]] + .Using SQLite 3 + [example] + ==== + [.programlisting] + .... + USES= sqlite:3 + .... + + See crossref:uses[uses-sqlite,`sqlite`] for more information. + ==== + + [[rc-scripts]] + == Starting and Stopping Services (`rc` Scripts) + + [.filename]#rc.d# scripts are used to start services on system startup, and to give administrators a standard way of stopping, starting and restarting the service. + Ports integrate into the system [.filename]#rc.d# framework. +-Details on its usage can be found in extref:{handbook}[the rc.d Handbook chapter, configtuning-rcd]. ++Details on its usage can be found in extref:{handbook}config[the rc.d Handbook chapter, configtuning-rcd]. + Detailed explanation of the available commands is provided in man:rc[8] and man:rc.subr[8]. + Finally, there is extref:{rc-scripting}[an article] on practical aspects of [.filename]#rc.d# scripting. + + With a mythical port called _doorman_, which needs to start a _doormand_ daemon. + Add the following to the [.filename]#Makefile#: + + [.programlisting] + .... + USE_RC_SUBR= doormand + .... + + Multiple scripts may be listed and will be installed. + Scripts must be placed in the [.filename]#files# subdirectory and a `.in` suffix must be added to their filename. + Standard `SUB_LIST` expansions will be ran against this file. + Use of the `%%PREFIX%%` and `%%LOCALBASE%%` expansions is strongly encouraged as well. + More on `SUB_LIST` in crossref:pkg-files[using-sub-files,the relevant section]. + + As of FreeBSD 6.1-RELEASE, local [.filename]#rc.d# scripts (including those installed by ports) are included in the overall man:rcorder[8] of the base system. + + An example simple [.filename]#rc.d# script to start the doormand daemon: + + [.programlisting] + .... + #!/bin/sh + + # PROVIDE: doormand + # REQUIRE: LOGIN + # KEYWORD: shutdown + # + # Add these lines to /etc/rc.conf.local or /etc/rc.conf + # to enable this service: + # + # doormand_enable (bool): Set to NO by default. + # Set it to YES to enable doormand. + # doormand_config (path): Set to %%PREFIX%%/etc/doormand/doormand.cf + # by default. + + . /etc/rc.subr + + name=doormand + rcvar=doormand_enable + + load_rc_config $name + + : ${doormand_enable:="NO"} + : ${doormand_config="%%PREFIX%%/etc/doormand/doormand.cf"} + + command=%%PREFIX%%/sbin/${name} + pidfile=/var/run/${name}.pid + + command_args="-p $pidfile -f $doormand_config" + + run_rc_command "$1" + .... + + Unless there is a very good reason to start the service earlier, or it runs as a particular user (other than root), all ports scripts must use: + + [.programlisting] + .... + REQUIRE: LOGIN + .... + + If the startup script launches a daemon that must be shutdown, the following will trigger a stop of the service on system shutdown: + + [.programlisting] + .... + KEYWORD: shutdown + .... + + If the script is not starting a persistent service this is not necessary. + + For optional configuration elements the "=" style of default variable assignment is preferable to the ":=" style here, since the former sets a default value only if the variable is unset, and the latter sets one if the variable is unset _or_ null. + A user might very well include something like: + + [.programlisting] + .... + doormand_flags="" + .... + + in their [.filename]#rc.conf.local#, and a variable substitution using ":=" would inappropriately override the user's intention. + The `_enable` variable is not optional, and must use the ":" for the default. + + [IMPORTANT] + ==== + Ports _must not_ start and stop their services when installing and deinstalling. + Do not abuse the [.filename]#plist# keywords described in crossref:plist[plist-keywords-base-exec, "the @preexec command,@postexec command,@preunexec command,@postunexec command section"] by running commands that modify the currently running system, including starting or stopping services. + ==== + + [[rc-scripts-checklist]] + === Pre-Commit Checklist + + Before contributing a port with an [.filename]#rc.d# script, and more importantly, before committing one, please consult this checklist to be sure that it is ready. + + The package:devel/rclint[] port can check for most of these, but it is not a substitute for proper review. + + [.procedure] + . If this is a new file, does it have a [.filename]#.sh# extension? If so, that must be changed to just [.filename]#file.in# since [.filename]#rc.d# files may not end with that extension. + . Do the name of the file (minus [.filename]#.in#), the `PROVIDE` line, and `$` _name_ all match? The file name matching `PROVIDE` makes debugging easier, especially for man:rcorder[8] issues. Matching the file name and `$`_name_ makes it easier to figure out which variables are relevant in [.filename]#rc.conf[.local]#. It is also a policy for all new scripts, including those in the base system. + . Is the `REQUIRE` line set to `LOGIN`? This is mandatory for scripts that run as a non-root user. If it runs as root, is there a good reason for it to run prior to `LOGIN`? If not, it must run after so that local scrips can be loosely grouped to a point in man:rcorder[8] after most everything in the base is already running. + . Does the script start a persistent service? If so, it must have `KEYWORD: shutdown`. + . Make sure there is no `KEYWORD: FreeBSD` present. This has not been necessary nor desirable for years. It is also an indication that the new script was copy/pasted from an old script, so extra caution must be given to the review. + . If the script uses an interpreted language like `perl`, `python`, or `ruby`, make certain that `command_interpreter` is set appropriately, for example, for Perl, by adding `PERL=${PERL}` to `SUB_LIST` and using `%%PERL%%`. Otherwise, + + + [source,shell] + .... + # service name stop + .... + + + will probably not work properly. See man:service[8] for more information. + . Have all occurrences of [.filename]#/usr/local# been replaced with `%%PREFIX%%`? + . Do the default variable assignments come after `load_rc_config`? + . Are there default assignments to empty strings? They should be removed, but double-check that the option is documented in the comments at the top of the file. + . Are things that are set in variables actually used in the script? + . Are options listed in the default _name_`_flags` things that are actually mandatory? If so, they must be in `command_args`. `-d` is a red flag (pardon the pun) here, since it is usually the option to "daemonize" the process, and therefore is actually mandatory. + . `_name__flags` must never be included in `command_args` (and vice versa, although that error is less common). + . Does the script execute any code unconditionally? This is frowned on. Usually these things must be dealt with through a `start_precmd`. + . All boolean tests must use the `checkyesno` function. No hand-rolled tests for `[Yy][Ee][Ss]`, etc. + . If there is a loop (for example, waiting for something to start) does it have a counter to terminate the loop? We do not want the boot to be stuck forever if there is an error. + . Does the script create files or directories that need specific permissions, for example, a [.filename]#pid# that needs to be owned by the user that runs the process? Rather than the traditional man:touch[1]/man:chown[8]/man:chmod[1] routine, consider using man:install[1] with the proper command line arguments to do the whole procedure with one step. + + [[users-and-groups]] + == Adding Users and Groups + + Some ports require a particular user account to be present, usually for daemons that run as that user. + For these ports, choose a _unique_ UID from 50 to 999 and register it in [.filename]#ports/UIDs# (for users) and [.filename]#ports/GIDs# (for groups). + The unique identification should be the same for users and groups. + + Please include a patch against these two files when requiring a new user or group to be created for the port. + + Then use `USERS` and `GROUPS` in [.filename]#Makefile#, and the user will be automatically created when installing the port. + + [.programlisting] + .... + USERS= pulse + GROUPS= pulse pulse-access pulse-rt + .... + + The current list of reserved UIDs and GIDs can be found in [.filename]#ports/UIDs# and [.filename]#ports/GIDs#. + + [[requiring-kernel-sources]] + == Ports That Rely on Kernel Sources + + Some ports (such as kernel loadable modules) need the kernel source files so that the port can compile. + Here is the correct way to determine if the user has them installed: + + [.programlisting] + .... + USES= kmod + .... + + Apart from this check, the `kmod` feature takes care of most items that these ports need to take into account. + + [[go-libs]] + == Go Libraries + + Ports must not package or install Go libs or source code. + Go ports must fetch the required deps at the normal fetch time and should only install the programs and things users need, not the things Go developers would need. + + Ports should (in order of preference): + + * Use vendored dependencies included with the package source. + * Fetch the versions of deps specified by upstream (in the case of go.mod, vendor.json or similar). + * As a last resort (deps are not included nor versions specified exactly) fetch versions of dependencies available at the time of upstream development/release. + + [[haskell-libs]] + == Haskell Libraries + + Just like in case of Go language, Ports must not package or install Haskell libraries. + Haskell ports must link statically to their dependencies and fetch all distribution files on fetch stage. + + [[shell-completion]] + == Shell Completion Files + + Many modern shells (including bash, fish, tcsh and zsh) support parameter and/or option tab-completion. + This support usually comes from completion files, which contain the definitions for how tab completion will work for a certain command. + Ports sometimes ship with their own completion files, or porters may have created them themselves. + + When available, completion files should always be installed. + It is not necessary to make an option for it. + If an option is used, though, always enable it in `OPTIONS_DEFAULT`. + + [[shell-completion-paths]] + .Full shell completion file names + [cols="1,1,1", frame="none"] + |=== + |`bash` + |[.filename]#${PREFIX}/etc/bash_completion.d# or [.filename]#${PREFIX}/share/bash-completion/completions# + |(any unique file names in one of these folders) + + |`fish` + |[.filename]#${PREFIX}/share/fish/completions/${PORTNAME}.fish# + | + + |`zsh` + |[.filename]#${PREFIX}/share/zsh/site-functions/_${PORTNAME}# + | + |=== + + Do not register any dependencies on the shells themselves. +diff --git a/documentation/content/en/books/porters-handbook/upgrading/_index.adoc b/documentation/content/en/books/porters-handbook/upgrading/_index.adoc +index e61a88dbaf..d55608d8d4 100644 +--- a/documentation/content/en/books/porters-handbook/upgrading/_index.adoc ++++ b/documentation/content/en/books/porters-handbook/upgrading/_index.adoc +@@ -1,293 +1,293 @@ + --- + title: Chapter 11. Upgrading a Port + prev: books/porters-handbook/testing + next: books/porters-handbook/security + description: Upgrading a FreeBSD Port + tags: ["upgrading", "port", "git"] + showBookMenu: true + weight: 11 + params: + path: "/books/porters-handbook/upgrading/" + --- + + [[port-upgrading]] + = Upgrading a Port + :doctype: book + :toc: macro + :toclevels: 1 + :icons: font + :sectnums: + :sectnumlevels: 6 + :sectnumoffset: 11 + :partnums: + :source-highlighter: rouge + :experimental: + :images-path: books/porters-handbook/ + + ifdef::env-beastie[] + ifdef::backend-html5[] + :imagesdir: ../../../../images/{images-path} + endif::[] + ifndef::book[] + include::shared/authors.adoc[] + include::shared/mirrors.adoc[] + include::shared/releases.adoc[] + include::shared/attributes/attributes-{{% lang %}}.adoc[] + include::shared/{{% lang %}}/teams.adoc[] + include::shared/{{% lang %}}/mailing-lists.adoc[] + include::shared/{{% lang %}}/urls.adoc[] + toc::[] + endif::[] + ifdef::backend-pdf,backend-epub3[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + endif::[] + + ifndef::env-beastie[] + toc::[] + include::../../../../../shared/asciidoctor.adoc[] + endif::[] + + When a port is not the most recent version available from the authors, update the local working copy of [.filename]#/usr/ports#. + The port might have already been updated to the new version. + + When working with more than a few ports, it will probably be easier to use Git to keep the whole ports collection up-to-date, + as described in extref:{handbook}ports[Using the Ports Collection, ports-using]. + This will have the added benefit of tracking all the port's dependencies. + + The next step is to see if there is an update already pending. + To do this, there are two options. + There is a searchable interface to the https://bugs.freebsd.org/search/[FreeBSD Problem Report (PR) or bug database]. + Select `Ports & Packages` in the `Product` multiple select menu, and enter the name of the port in the `Summary` field. + + If there is no pending PR, the next step is to send an email to the port's maintainer, as shown by `make maintainer`. + That person may already be working on an upgrade, or have a reason to not upgrade the port right now (because of, for example, stability problems of the new version), and there is no need to duplicate their work. + Note that unmaintained ports are listed with a maintainer of `ports@FreeBSD.org`, which is just the general ports mailing list, so sending mail there probably will not help in this case. + + If the maintainer asks you to do the upgrade or there is no maintainer, then help out FreeBSD by preparing the update! Please do this by using the man:diff[1] command in the base system. + + To create a suitable `diff` for a single patch, copy the file that needs patching to [.filename]#something.orig#, save the changes to [.filename]#something# and then create the patch: + + [source,shell] + .... + % diff -u something.orig something > something.diff + .... + + Otherwise, either use the `git diff` method (crossref:upgrading[git-diff, Using Git to Make Patches]) or copy the contents of the port to an entire different directory and use the result of the recursive man:diff[1] output of the new and old ports directories (for example, if the modified port directory is called [.filename]#superedit# and the original is in our tree as [.filename]#superedit.bak#, then save the result of `diff -ruN superedit.bak superedit`). + Either unified or context diff is fine, but port committers generally prefer unified diffs. + Note the use of the `-N` option-this is the accepted way to force diff to properly deal with the case of new files being added or old files being deleted. + Before sending us the diff, please examine the output to make sure all the changes make sense. + (In particular, make sure to first clean out the work directories with `make clean`). + + [NOTE] + ==== + If some files have been added, copied, moved, or removed, add this information to the problem report so that the committer picking up the patch will know what man:git[1] commands to run. + ==== + + To simplify common operations with patch files, use `make makepatch` as described in crossref:slow-porting[slow-patch,Patching]. + Other tools exists, like [.filename]#/usr/ports/Tools/scripts/patchtool.py#. + Before using it, please read [.filename]#/usr/ports/Tools/scripts/README.patchtool#. + + If the port is unmaintained, and you are actively using it, please consider volunteering to become its maintainer. + FreeBSD has over 4000 ports without maintainers, and this is an area where more volunteers are always needed. +-(For a detailed description of the responsibilities of maintainers, refer to the section in the extref:{developers-handbook}[Developer's Handbook, POLICIES-MAINTAINER].) ++(For a detailed description of the responsibilities of maintainers, refer to the section in the extref:{developers-handbook}policies[Developer's Handbook, policies-maintainer].) + + To submit the diff, use the https://bugs.freebsd.org/submit/[bug submit form] (product `Ports & Packages`, component `Individual Port(s)`). + Always include the category with the port name, followed by colon, and brief description of the issue. + Examples: `_category/portname_: _add FOO option_`; `_category/portname_: _Update to X.Y_`. + Please mention any added or deleted files in the message, as they have to be explicitly specified to man:git[1] when doing a commit. + Do not compress or encode the diff. + + Before submitting the bug, review the extref:{problem-reports}[Writing the problem report, pr-writing] section in the Problem Reports article. + It contains far more information about how to write useful problem reports. + + [IMPORTANT] + ==== + If the upgrade is motivated by security concerns or a serious fault in the currently committed port, + please notify the {portmgr} to request immediate rebuilding and redistribution of the port's package. + Unsuspecting users of `pkg` will otherwise continue to install the old version via `pkg install` for several weeks. + ==== + + [NOTE] + ==== + Please use man:diff[1] or `git diff` to create updates to existing ports. + Other formats include the whole file and make it impossible to see just what has changed. + When diffs are not included, the entire update might be ignored. + ==== + + Now that all of that is done, read about how to keep up-to-date in crossref:keeping-up[keeping-up,Keeping Up]. + + [[git-diff]] + == Using Git to Make Patches + + When possible, please submit a man:git[1] patch or diff. + They are easier to handle than diffs between "new and old" directories. + It is easier to see what has changed, and to update the diff if something was modified in the Ports Collection since the work on it began, + or if the committer asks for something to be fixed. + Also, a patch generated with man:git-format-patch[1] or man:git-diff[1] can be easily applied with man:git-am[1] or man:git-apply[1] and will save some time for the committer. + Finally, the git patch generated by man:git-format-patch[1] includes your author information and commit messages. + These will be recorded in the log of the repository and this is the recommended way to submit your changes. + + [source,shell] + .... + % git clone https://git.FreeBSD.org/ports.git ~/my_wrkdir <.> <.> + % cd ~/my_wrkdir + .... + + <.> This can be anywhere, of course. Building ports is not limited to within [.filename]#/usr/ports/#. + + <.> https://git.FreeBSD.org/[git.FreeBSD.org] is the FreeBSD public Git server. See extref:{handbook}mirrors[FreeBSD Git Repository URL Table, git-url-table] for more information. + + While in the port directory, make any changes that are needed. + If adding, moving, or removing a file, use `git` to track these changes: + + [source,shell] + .... + % git add new_file + % git mv old_name new_name + % git rm deleted_file + .... + + Make sure to check the port using the checklist in crossref:quick-porting[porting-testing,Testing the Port] and crossref:quick-porting[porting-portlint,Checking the Port with `portlint`]. + + Also, update the checksum reference in distinfo with `make makesum`. + + Before making the patch, fetch the latest repository and rebase the changes on top of it. + Watch and follow the output carefully. + If any of the files failed to rebase, it means that the upstream files changed while you were editing the same file, and the conflicts need to be resolved manually. + + [source,shell] + .... + % git fetch origin main + % git rebase origin/main + .... + + Check the changes staged for the patch: + + [source,shell] + .... + % git status + % git diff --staged + .... + + The last step is to make an unified diff or patch of the changes: + + To generate a patch with man:git-format-patch[1]: + [source,shell] + .... + % git checkout -b my_branch + % git commit + % git format-patch main + .... + + This will generate a patch named like `0001-foo.patch`. + This is the preferred way as it would include author identity, + and it is also easier when you are making a series of changes that + are not meant to be squashed together. + + Alternatively, to generate an unified diff with man:git-diff[1]: + [source,shell] + .... + % git diff --staged > ../`make -VPKGNAME`.diff + .... + This will generate a diff named like `foo-1.2.3.diff`. + Where `foo` is replaced with the first line of the commit message, i.e., the subject of the commit message. + + After patch has been created, you can switch to the main branch for starting other developments. + [source,shell] + .... + % git checkout main + .... + + Once the patch is accepted and merged, you can delete the local development branch if you want: + [source,shell] + .... + % git branch -D my_branch + .... + + [NOTE] + ==== + If files have been added, moved, or removed, include the man:git[1] `add`, `mv`, and `rm` commands that were used. + `git mv` must be run before the patch can be applied. + `git add` or `git rm` must be run after the patch is applied. + ==== + + Send the patch following the extref:{problem-reports}[problem report submission guidelines, pr-writing]. + + [[moved-and-updating-files]] + == UPDATING and MOVED + + [[moved-and-updating-updating]] + === /usr/ports/UPDATING + + If upgrading the port requires special steps like changing configuration files or running a specific program, it must be documented in this file. + The format of an entry in this file is: + + [.programlisting] + .... + YYYYMMDD: + AFFECTS: users of portcategory/portname + AUTHOR: Your name + + Special instructions + .... + + [TIP] + ==== + + When including exact portmaster, portupgrade, and/or pkg instructions, please make sure to get the shell escaping right. + For example, do _not_ use: + + [source,shell] + .... + # pkg delete -g -f docbook-xml* docbook-sk* docbook[2345]??-* docbook-4* + .... + + As shown, the command will only work with bourne shells. + Instead, use the form shown below, which will work with both bourne shell and c-shell: + + [source,shell] + .... + # pkg delete -g -f docbook-xml\* docbook-sk\* docbook\[2345\]\?\?-\* docbook-4\* + .... + + ==== + + [NOTE] + ==== + It is recommended that the AFFECTS line contains a glob matching all the ports affected by the entry so that automated tools can parse it as easily as possible. + If an update concerns all the existing BIND 9 versions the `AFFECTS` content must be `users of dns/bind9*`, it must _not_ be `users of BIND 9` + ==== + + [[moved-and-updating-moved]] + === /usr/ports/MOVED + + This file is used to list moved or removed ports. + Each line in the file is made up of the name of the port, where the port was moved, when, and why. + If the port was removed, the section detailing where it was moved can be left blank. + Each section must be separated by the `|` (pipe) character, like so: + + [.programlisting] + .... + old name|new name (blank for deleted)|date of move|reason + .... + + The date must be entered in the form `YYYY-MM-DD`. + New entries are added to the end of the list to keep it in chronological order, with the oldest entry at the top of the list. + + If a port was removed but has since been restored, delete the line in this file that states that it was removed. + + If a port was renamed and then renamed back to its original name, add a new one with the intermediate name to the old name, and remove the old entry as to not create a loop. + + [NOTE] + ==== + Any changes must be validated with `Tools/scripts/MOVEDlint.awk`. + + If using a ports directory other than [.filename]#/usr/ports#, use: + + [source,shell] + .... + % cd /home/user/ports + % env PORTSDIR=$PWD Tools/scripts/MOVEDlint.awk + .... + ==== +-- +2.47.1 + diff --git a/tools/immitate_translation.pl b/tools/immitate_translation.pl new file mode 100755 --- /dev/null +++ b/tools/immitate_translation.pl @@ -0,0 +1,88 @@ +#!/usr/bin/env perl +# +# Copyright (c) 2025 Vladlen Popolitov +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +use strict; + +my ($poen,$polang,$preptext); + +$poen = $ARGV[0]; +$polang = $ARGV[1]; +$preptext = $ARGV[2]; + +if (defined($poen) && defined($polang)) +{ + process_po($poen,$polang,$preptext); +} + + +sub process_po { + my ($pofilein,$pofileout, $text) = @_; + my ($state) = (0); + my (@polines,$inmsgid); + my ($msgid,$line); + open(FILEIN,"<$pofilein"); + @polines = ; + close(FILEIN); + open(FILEOUT,">$pofileout"); + print "Immitate translation of $pofileout (prepend text \"$text\" to all strings)\n"; + $inmsgid = 0; + $msgid=""; + for (my $i=0;$i<=$#polines;++$i) + { + $line = $polines[$i]; + chomp($line); + if ($line =~ /^msgid "(.*)"$/) + { + $msgid = $1; + $inmsgid = 1; + } + else + { + if ($inmsgid) + { + if ($line =~ /^[ \t]*"(.*)"[ ]*$/) + { + $msgid .= $1; + } + else + { + print FILEOUT "msgid \"${msgid}\"\nmsgstr \"${text}\"\n\"${msgid}\"\n"; + $msgid = ""; + $inmsgid = 0; + } + + } + else + { + print FILEOUT "$line\n"; + } + } + } + close(FILEOUT); + +} + + +=pod diff --git a/tools/immitate_translation.sh b/tools/immitate_translation.sh new file mode 100755 --- /dev/null +++ b/tools/immitate_translation.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env sh +# +# Copyright (c) 2025 Vladlen Popolitov +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +LANGUAGE="${1:-ru}" +TEXT="${2:-$LANGUAGE}> " + +cd website/content/en + +find . -name '_indexdir.po' -exec cp -v '{}' ../$LANGUAGE/'{}' ';' + +cd ../../.. + +find website/content/$LANGUAGE -name '_indexdir.po' -exec perl tools/immitate_translation.pl '{}' '{}' $TEXT ';' diff --git a/tools/translate_website.sh b/tools/translate_website.sh new file mode 100755 --- /dev/null +++ b/tools/translate_website.sh @@ -0,0 +1,171 @@ +#!/bin/sh +# +# Copyright (c) 2021 Danilo G. Baio +# Copyright (c) 2021 Fernando Apesteguia +# Copyright (c) 2025 Vladlen Popolitov +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +######################################################### +# This is a temporary fix for po4a-translate command # +# po4a: Fix YAML Front Matter / tags and trademarks # +# https://wiki.freebsd.org/Doc/IdeaList#Translation # +# $1: File to fix # +######################################################### +fixup_lists() +{ + sed -i '' -E -e "s/(tags|trademarks).*'\[(.*)]'/\1: [\2]/g" "${1}" +} + +######################################################### +# Fix includes. In a few cases we want to include the # +# master (aka English) version of the includes # +# $1: file to fix # +# $2: language # +######################################################### +fixup_includes() +{ + # Replace ...shared/en/... with shared/$LANGUAGE + # content/en with content/$LANGUAGE in includes + sed -i '' -E -e "s,include::(.*)shared/en/,include::\1shared/${2}/," \ + -e "s,\{include-path\}(contrib*),content/en/articles/contributors/\1," \ + -e "s,include-path: content/en/,include-path: content/${2}/," \ + -e "s,(include::.*)contrib-develinmemoriam(.*),include::{include-path}contrib-develinmemoriam\2," \ + -e "s,(:chapters-path: |include::)content/en/books,\1content/${2}/books," \ + "${1}" +} + +if [ "$1" = "" ] || [ "$2" = "" ]; then + echo "Need to inform component and language:" + echo " $0 documentation es" + echo "A third (optional) argument can be informed to translate only a specific document:" + echo " $0 documentation pt-br articles/bsdl-gpl" + exit 1 +fi + +COMPONENT="website" +LANGUAGE="$1" +SEARCH_RESTRICT="$2" + +# po4a-translate option: -k, --keep +# Minimal threshold for translation percentage to keep (i.e. write) +# the resulting file (default: 80). I.e. by default, files have to be +# translated at least at 80% to get written. +# # KEEP_ENV=10 ./tools/translate.sh documentation pt_BR +KEEP="${KEEP_ENV:-80}" + +if [ "$LANGUAGE" = "en" ]; then + echo "Language 'en' can't be translated." + exit 1 +fi + +if [ ! -d "$COMPONENT/content/$LANGUAGE" ]; then + echo "$COMPONENT/content/$LANGUAGE does not exist." + exit 1 +fi + +for documentdir in $(find "$COMPONENT/content/en/$SEARCH_RESTRICT" -name "*" -type d); do + dirnamesrc=$(basename "$documentdir") + echo "Dirnamesrc $dirnamesrc $documentdir" + + # Ignore some dirßs + if echo $documentdir | grep "^website/content/en/releases"; then + continue + fi + + # Ignore some dirs + if echo $documentdir | grep "^website/content/en/status"; then + continue + fi + pofile=$(echo "$documentdir/_indexdir.po" | sed s,$COMPONENT/content/en,$COMPONENT/content/$LANGUAGE,) + echo "pofile $pofile" + if [ ! -f "$pofile" ]; then + echo "PO file '$pofile' not found." + continue + fi + + for adocfile in $(find "$documentdir" -maxdepth 1 -name "*.adoc"); do + name=$(basename -s .adoc "$adocfile") + #if [ "$name" = "chapters-order" ]; then + # continue + #fi + + dirbase=$(dirname "$pofile") + + adoc_orig="$adocfile" + adoc_lang=$(echo "$adoc_orig" | sed s,$COMPONENT/content/en,$COMPONENT/content/$LANGUAGE,) + + echo "....." + echo "lang $adoc_lang orig $adoc_orig" + + echo po4a-updatepo \ + --format asciidoc \ + --option compat=asciidoctor \ + --option tablecells=1 \ + --option yfm_keys=title,part,description \ + --master "$adoc_orig" \ + --master-charset "UTF-8" \ + --copyright-holder "The FreeBSD Project" \ + --package-name "FreeBSD Documentation" \ + --po "$pofile" + po4a-updatepo \ + --format asciidoc \ + --option compat=asciidoctor \ + --option tablecells=1 \ + --option yfm_keys=title,part,description \ + --master "$adoc_orig" \ + --master-charset "UTF-8" \ + --copyright-holder "The FreeBSD Project" \ + --package-name "FreeBSD Documentation" \ + --po "$pofile" + if [ -f "${pofile}~" ]; then + rm -f "${pofile}~" + fi + echo po4a-translate \ + --format asciidoc \ + --option compat=asciidoctor \ + --option tablecells=1 \ + --option yfm_keys=title,part,description \ + --master "$adoc_orig" \ + --master-charset "UTF-8" \ + --po "$pofile" \ + --localized "$adoc_lang" \ + --localized-charset "UTF-8" \ + --keep "$KEEP" + + po4a-translate \ + --format asciidoc \ + --option compat=asciidoctor \ + --option tablecells=1 \ + --option yfm_keys=title,part,description \ + --master "$adoc_orig" \ + --master-charset "UTF-8" \ + --po "$pofile" \ + --localized "$adoc_lang" \ + --localized-charset "UTF-8" \ + --keep "$KEEP" + + fixup_lists "${adoc_lang}" + fixup_includes "${adoc_lang}" "${LANGUAGE}" + done +done + diff --git a/tools/update_translate_template_toml.sh b/tools/update_translate_template_toml.sh new file mode 100755 --- /dev/null +++ b/tools/update_translate_template_toml.sh @@ -0,0 +1,92 @@ +#!/bin/sh +# +# Copyright (c) 2021 Danilo G. Baio +# Copyright (c) 2025 Vladlen Popolitov +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +ALL_COMPONENTS="website/data +website/themes" + +COMPONENTS="${1:-$ALL_COMPONENTS}" + +GIT_IGNORE_FILES="toc-examples.adoc +toc-figures.adoc +toc-tables.adoc +toc.adoc +toc-1.adoc +toc-2.adoc +toc-3.adoc +toc-4.adoc +toc-5.adoc" + +IGNORE_FILES="contrib-386bsd +contrib-additional +contrib-committers +contrib-corealumni +contrib-develalumni +contrib-portmgralumni" + +for remove_file in $GIT_IGNORE_FILES; do + find documentation/content/en/ -name "$remove_file" -delete -print || exit 1 +done + +for component in $COMPONENTS; do + + if [ ! -d "$component/en" ]; then + echo "Directory '$component/content/en' not found." + exit 1 + fi + + for document in $(find "$component/en/" -name "*.toml" ); do + name=$(basename -s .toml "$document") + + # Ignore some files + if [ "$name" = "chapters-order" ]; then + continue + fi + + if [ "$document" = "documentation/content/en/books/books.adoc" ]; then + continue + fi + + if echo "$IGNORE_FILES" | grep -q -w "$name"; then + continue + fi + + dirbase=$(dirname "$document") + echo "$document" + yq -oyaml $document > "$dirbase/$name.yaml" + po4a-updatepo \ + --format yaml \ + --master "$dirbase/$name.yaml" \ + --master-charset "UTF-8" \ + --copyright-holder "The FreeBSD Project" \ + --package-name "FreeBSD Documentation" \ + --po "$dirbase/$name.po" + if [ -f "$dirbase/$name.po~" ]; then + rm -f "$dirbase/$name.po~" + fi + done +done + diff --git a/tools/update_translate_template_website.sh b/tools/update_translate_template_website.sh new file mode 100755 --- /dev/null +++ b/tools/update_translate_template_website.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# +# Copyright (c) 2021 Danilo G. Baio +# Copyright (c) 2025 Vladlen Popolitov +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +ALL_COMPONENTS="website" + +COMPONENTS="${1:-$ALL_COMPONENTS}" + +for component in $COMPONENTS; do + + if [ ! -d "$component/content/en" ]; then + echo "Directory '$component/content/en' not found." + exit 1 + fi + + # instead of loop by all adoc file + #for document in $(find "$component/content/en/" -name "*.adoc" ); do + # we search all directories + for document in $(find "$component/content/en" -name "*" -type d ); do + dirname=$(basename "$document") + #echo "Dirname $dirname $document" + + # Ignore some dirs + if echo $document | grep "^website/content/en/releases"; then + continue + fi + + # Ignore some dirs + if echo $document | grep "^website/content/en/status"; then + continue + fi + + # find all adoc in the directory and append names to --master option" + documents=`find $document -maxdepth 1 -name "*.adoc" -print | sort | tr '\n' '|' | sed 's/\|website\// --master website\//g' | tr '|' ' ' ` + + #echo "List of all adocs in directory $document --master $documents" + po4a-updatepo \ + --format asciidoc \ + --option compat=asciidoctor \ + --option tablecells=1 \ + --option yfm_keys=title,part,description \ + --no-deprecation \ + --master $documents \ + --master-charset "UTF-8" \ + --copyright-holder "The FreeBSD Project" \ + --package-name "FreeBSD Documentation" \ + --po "$document/_indexdir.po" + if [ -f "$document/_indexdir.po~" ]; then + rm -f "$document/_indexdir.po~" + fi + done +done +