diff --git a/release/Makefile.vm b/release/Makefile.vm --- a/release/Makefile.vm +++ b/release/Makefile.vm @@ -42,8 +42,9 @@ BASIC-CLOUDINIT_DESC?= Images for VM with cloudinit disk config support EC2_FORMAT= raw EC2_FSLIST?= ufs zfs -EC2_FLAVOURS?= BASE CLOUD-INIT SMALL +EC2_FLAVOURS?= BASE BUILDER CLOUD-INIT SMALL EC2-BASE_DESC= Amazon EC2 image +EC2-BUILDER_DESC= Amazon EC2 AMI Builder image EC2-CLOUD-INIT_DESC= Amazon EC2 Cloud-Init image EC2-SMALL_DESC= Amazon EC2 small image GCE_FORMAT= raw @@ -124,14 +125,22 @@ ${_CW:tl}.${_FS}.${_FMT}.raw ${_CW:tu}${_FS:tu}${_FMT:tu}IMAGE= ${_CW:tl}.${_FS}.${_FMT} +# Special handling: EC2 "AMI Builder" images need a copy of the "base" disk +# built first. +.if ${_CW} == EC2-BUILDER +cw-${_CW:tl}-${_FS}-${_FMT}: cw-ec2-base-${_FS}-${_FMT} +.endif + cw-${_CW:tl}-${_FS}-${_FMT}: ${QEMUTGT} mkdir -p ${.OBJDIR}/${.TARGET} env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} SWAPSIZE=${SWAPSIZE} \ QEMUSTATIC=${QEMUSTATIC} \ + EC2BASEIMG=${.OBJDIR}/${EC2-BASE${_FS:tu}${_FMT:tu}IMAGE} \ ${WITHOUT_QEMU:DWITHOUT_QEMU=true} \ ${NO_ROOT:DNO_ROOT=true} \ ${.CURDIR}/scripts/mk-vmimage.sh \ - -C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} -F ${_FS} \ + -C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \ + -F ${"${_CW:MEC2-BUILDER}" != "":?ufs:${_FS}} \ -i ${.OBJDIR}/${_CW:tl}.${_FS}.${_FMT}.img -s ${VMSIZE} -f ${_FMT} \ -S ${WORLDDIR} -o ${.OBJDIR}/${${_CW:tu}${_FS:tu}${_FMT:tu}IMAGE} -c ${${_CW:tu}CONF} touch ${.TARGET} diff --git a/release/tools/ec2-builder.conf b/release/tools/ec2-builder.conf new file mode 100644 --- /dev/null +++ b/release/tools/ec2-builder.conf @@ -0,0 +1,57 @@ +#!/bin/sh + +. ${WORLDDIR}/release/tools/ec2.conf + +# Build with a 7.9 GB partition; this is enough for our stripped-down +# base system plus the compressed ec2-base image. +export VMSIZE=8000m + +# Flags to installworld/kernel: We don't want debug symbols (kernel or +# userland), 32-bit libraries, tests, or the debugger. +export INSTALLOPTS="WITHOUT_DEBUG_FILES=YES WITHOUT_KERNEL_SYMBOLS=YES \ + WITHOUT_LIB32=YES WITHOUT_TESTS=YES WITHOUT_LLDB=YES" + +# Packages to install into the image we're creating. In addition to packages +# present on all EC2 AMIs, we install: +# * ec2-scripts, which provides a range of EC2ification startup scripts, +# * isc-dhcp44-client, used for IPv6 network setup, and +# * py-awscli, to make it easier for users to create AMIs. +export VM_EXTRA_PACKAGES="${VM_EXTRA_PACKAGES} ec2-scripts \ + isc-dhcp44-client devel/py-awscli" + +# Services to enable in rc.conf(5). +export VM_RC_LIST="${VM_RC_LIST} ec2_configinit ec2_ephemeral_swap \ + ec2_fetchkey ec2_loghostkey sshd" + +vm_extra_pre_umount() { + # Any EC2 ephemeral disks seen when the system first boots will + # be "new" disks; there is no "previous boot" when they might have + # been seen and used already. + touch ${DESTDIR}/var/db/ec2_ephemeral_diskseen + + # Configuration common to all EC2 AMIs + ec2_common + + # Standard FreeBSD network configuration + ec2_base_networking + + # Grab a copy of the ec2-base disk image, and compress it + zstd < ${EC2BASEIMG} > ${DESTDIR}/image.zst + + # Disable fortune so we don't have extra noise at login + chmod a-x ${DESTDIR}/usr/bin/fortune + + # Install the AMI-building script + install -m 755 ${WORLDDIR}/release/tools/mkami.sh ${DESTDIR}/bin/mkami + + # Install an /etc/rc which juggles disks around for us + install -m 755 ${WORLDDIR}/release/tools/rc.amibuilder ${DESTDIR}/etc + + # We want to mount from the UFS disk and juggle disks first + cat >> ${DESTDIR}/boot/loader.conf <<-EOF + vfs.root.mountfrom="ufs:/dev/gpt/rootfs" + init_script="/etc/rc.amibuilder" + EOF + + return 0 +} diff --git a/release/tools/mkami.sh b/release/tools/mkami.sh new file mode 100644 --- /dev/null +++ b/release/tools/mkami.sh @@ -0,0 +1,85 @@ +#!/bin/sh -e +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2015 Colin Percival +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# mkami.sh: Create an AMI from the currently running EC2 instance. +# + +export PATH=$PATH:/usr/local/bin + +NAME=$1 +if [ -z "$NAME" ]; then + echo "usage: mkami []" + exit 1 +fi +DESC=$2 +if ! [ -z "$DESC" ]; then + DESCOPT="--description '$DESC'" +fi + +# Get the instance ID and region from the EC2 Instance Metadata Service: +# https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html +TMPFILE=`mktemp` +fetch -qo $TMPFILE http://169.254.169.254/latest/dynamic/instance-identity/document +INST=`awk -F \" '/"instanceId"/ { print $4 }' $TMPFILE` +REGION=`awk -F \" '/"region"/ { print $4 }' $TMPFILE` +rm $TMPFILE +CMD="aws --region $REGION ec2 create-image --instance-id $INST --output text --no-reboot --name '$NAME' $DESCOPT" + +# Unmount the new system image +if mount -p | grep -q '/mnt.*ufs'; then + echo -n "Unmounting new system image..." + sync + umount /mnt + sync + sleep 5 + sync + echo " done." +elif mount -p | grep -q '/mnt.*zfs'; then + echo -n "Unmounting new system image..." + sync + zfs umount -a + zfs umount zroot/ROOT/default + sync + sleep 5 + sync + echo " done." +fi + +if eval "$CMD" --dry-run 2>&1 | + grep -qE 'UnauthorizedOperation|Unable to locate credentials'; then + echo "This EC2 instance does not have permission to create AMIs." + echo "Launch an AMI-builder instance with an appropriate IAM Role," + echo "create an AMI from this instance via the AWS Console, or run" + echo "the following command from a system with the necessary keys:" + echo + echo "$CMD" + exit +fi + +echo -n "Creating AMI..." +AMINAME=`eval "$CMD"` +echo " done." +echo "AMI created in $REGION: $AMINAME" diff --git a/release/tools/rc.amibuilder b/release/tools/rc.amibuilder new file mode 100644 --- /dev/null +++ b/release/tools/rc.amibuilder @@ -0,0 +1,94 @@ +#!/bin/sh +# SPDX-License-Identifier: BSD-2-Clause +# +# Copyright (c) 2025 Colin Percival +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# rc.amibuilder: Juggle disks to reroot into a memory disk and install a clean +# copy of FreeBSD onto the root disk. +# + +# Do nothing if init(8) is in the middle of rerooting +if ps -p 1 -o command | grep -q reroot; then + exit 0 +fi + +# Figure out which partition we boot from +BOOTPART=$(sysctl -n kern.geom.conftxt | + grep -E 'PART|gpt/rootfs' | + grep -B 1 gpt/rootfs | + awk '{ print $3 }' | + head -1) +BOOTDISK=${BOOTPART%%p*} +BOOTPARTNUM=${BOOTPART##*p} + +# First pass: Copy ourselves into a memory disk and reroot into it +if ! [ -c /dev/md0 ]; then + # Create a memory disk of appropriate size and copy the disk + echo "Copying FreeBSD into memory disk..." + DISKBYTES=$(diskinfo ${BOOTDISK} | awk '{print $3}') + mdconfig -a -t swap -s ${DISKBYTES}b -u 0 + dd if=/dev/${BOOTDISK} of=/dev/md0 bs=1M + + # Reboot into the memory disk we just created + echo "Rebooting into memory disk..." + kenv vfs.root.mountfrom="ufs:/dev/md0p${BOOTPARTNUM}" + reboot -r + + # Lose a race against init + sleep 10 + exit 1 +fi + +# Second pass: Extract a clean copy of FreeBSD onto the physical disk +echo "Installing base FreeBSD image..." +sysctl kern.geom.debugflags=16 +zstdcat < /image.zst | dd bs=1M of=/dev/${BOOTDISK} + +# Mount the clean image +if gpart show ${BOOTDISK} | grep -q freebsd-ufs; then + mount /dev/${BOOTPART} /mnt +else + zpool import -aNR /mnt + zfs mount zroot/ROOT/default + zfs mount -a +fi + +# Provide instructions for when the user logs in +mount -w / +cat >/etc/motd.template < [] + +to create the AMI. Don't forget to shut down this instance when +you're done! + +EOF +mount -o ro / + +# After we exit, the boot proceeds with init spawning /etc/rc normally +exit 0