diff --git a/release/scripts/make-oci-image.sh b/release/scripts/make-oci-image.sh --- a/release/scripts/make-oci-image.sh +++ b/release/scripts/make-oci-image.sh @@ -39,22 +39,26 @@ local abi=$1; shift local workdir=$1; shift local rootdir=${workdir}/rootfs + local installflags="-U -M ${workdir}/METALOG -o root -g wheel -D ${rootdir}" # Make sure we have the keys needed for verifying package integrity if # not already added by a parent image. if [ ! -d ${rootdir}/usr/share/keys/pkg/trusted ]; then - mkdir -p ${rootdir}/usr/share/keys/pkg/trusted + install ${installflags} -m 755 -d ${rootdir}/usr/share/keys/pkg/trusted fi for i in ${curdir}/../share/keys/pkg/trusted/pkg.*; do if [ ! -f ${rootdir}/usr/share/keys/pkg/trusted/$(basename $i) ]; then - cp $i ${rootdir}/usr/share/keys/pkg/trusted + install ${installflags} -m 644 $i ${rootdir}/usr/share/keys/pkg/trusted fi done - # We install the packages and then remove repository metadata (keeping the - # metadata for what was installed). This trims more than 40Mb from the - # resulting image. - env IGNORE_OSVERSION=yes ABI=${abi} pkg --rootdir ${rootdir} --repo-conf-dir ${workdir}/repos \ + # Keep the metadata files that track which packages are installed. + echo "./var/db/pkg/local.sqlite type=file mode=0644 uname=root gname=wheel" >> ${workdir}/METALOG + echo "./var/db/pkg/local.sqlite-journal type=file mode=0644 uname=root gname=wheel" >> ${workdir}/METALOG + # After installing packages remove the rest of the metadata. This trims + # more than 40Mb from the resulting image. + pkg -o IGNORE_OSVERSION=yes -o ABI=${abi} -o METALOG=${workdir}/METALOG \ + --rootdir ${rootdir} --repo-conf-dir ${workdir}/repos \ install -yq -g "$@" || exit $? rm -rf ${rootdir}/var/db/pkg/repos } @@ -93,11 +97,61 @@ rm -rf ${workdir}/rootfs fi mkdir -p ${workdir}/rootfs + echo '#mtree 2.0' > ${workdir}/METALOG + mkdir -p ${workdir}/oci/blobs/sha256 if [ "${base_workdir}" != "" ]; then - tar -C ${workdir}/rootfs -xf ${base_workdir}/rootfs.tar.gz + cp ${base_workdir}/diff_ids ${workdir}/diff_ids + cp ${base_workdir}/layer_hashes ${workdir}/layer_hashes + cp ${base_workdir}/history ${workdir}/history + for i in $(cat ${base_workdir}/layer_hashes); do + # We suppress file flags to avoid confusing (but + # harmless) error message when extracting directories + # marked as immutable. + # + # Any new immutable files or directories will be + # recorded correctly in the new layer via the METALOG + tar -C ${workdir}/rootfs --no-fflags -xf ${base_workdir}/oci/blobs/sha256/$i + ln ${base_workdir}/oci/blobs/sha256/$i ${workdir}/oci/blobs/sha256/$i + done fi } +# Generate a json-formatted list. The first argument is the name of a function +# to format each item in the list and the remainder are the items. +format_list() { + local format_item=$1; shift + local result="" + for i in "$@"; do + if [ -z "${result}" ]; then + result="[" + else + result="${result}," + fi + result="${result}$(eval ${format_item} $i)" + done + result="${result}]" + echo ${result} +} + +# Format a digest as a json string with the hash type and hash value. We only +# use sha256 digests here. +format_digest() { + echo "\"sha256:$1\"" +} + +# Format a history item as a json object +format_history() { + local time=${1%,*} + local img=${1#*,} + echo "{\"created\":\"${time}\",\"created_by\":\"make-oci-image.sh ${img}\"}" +} + +# Format a layer reference as a json object +format_layer() { + local size=$(stat -f %z ${workdir}/oci/blobs/sha256/$1) + echo "{\"mediaType\":\"application/vnd.oci.image.layer.v1.tar+gzip\",\"digest\":$(format_digest $1),\"size\":${size}}" +} + commit_container() { local workdir=$1; shift local image=$1; shift @@ -109,31 +163,36 @@ # For compatibility with Podman, we must disable sparse-file # handling. See https://github.com/containers/podman/issues/25270 for # more details. - tar -C ${workdir}/rootfs --strip-components 1 --no-read-sparse -cf ${workdir}/rootfs.tar . - local diff_id=$(sha256 -q < ${workdir}/rootfs.tar) - gzip -f ${workdir}/rootfs.tar + tar -C ${workdir}/rootfs --strip-components 1 --no-read-sparse -cf ${workdir}/layer.tar @${workdir}/METALOG + echo $(sha256 -q < ${workdir}/layer.tar) >> ${workdir}/diff_ids + gzip -f ${workdir}/layer.tar local create_time=$(date -u +%Y-%m-%dT%TZ) - local root_hash=$(sha256 -q < ${workdir}/rootfs.tar.gz) - local root_size=$(stat -f %z ${workdir}/rootfs.tar.gz) + local layer_hash=$(sha256 -q < ${workdir}/layer.tar.gz) + ln ${workdir}/layer.tar.gz ${workdir}/oci/blobs/sha256/${layer_hash} + echo ${layer_hash} >> ${workdir}/layer_hashes + echo "${create_time},${image}" >> ${workdir}/history oci_arch=$(normalize_arch ${arch}) - config= + local config= if [ -n "${oci_cmd}" ]; then config=",\"config\":{\"cmd\":[\"${oci_cmd}\"]}" fi - echo "{\"created\":\"${create_time}\",\"architecture\":\"${oci_arch}\",\"os\":\"freebsd\"${config},\"rootfs\":{\"type\":\"layers\",\"diff_ids\":[\"sha256:${diff_id}\"]},\"history\":[{\"created\":\"${create_time}\",\"created_by\":\"make-oci-image.sh\"}]}" > ${workdir}/config.json + + local diff_ids=$(format_list format_digest $(cat ${workdir}/diff_ids)) + local history=$(format_list format_history $(cat ${workdir}/history)) + echo "{\"created\":\"${create_time}\",\"architecture\":\"${oci_arch}\",\"os\":\"freebsd\"${config},\"rootfs\":{\"type\":\"layers\",\"diff_ids\":${diff_ids}},\"history\":${history}}" > ${workdir}/config.json local config_hash=$(sha256 -q < ${workdir}/config.json) local config_size=$(stat -f %z ${workdir}/config.json) - echo "{\"schemaVersion\":2,\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"config\":{\"mediaType\":\"application/vnd.oci.image.config.v1+json\",\"digest\":\"sha256:${config_hash}\",\"size\":${config_size}},\"layers\":[{\"mediaType\":\"application/vnd.oci.image.layer.v1.tar+gzip\",\"digest\":\"sha256:${root_hash}\",\"size\":${root_size}}],\"annotations\":{}}" > ${workdir}/manifest.json + local layers=$(format_list format_layer $(cat ${workdir}/layer_hashes)) + echo "{\"schemaVersion\":2,\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"config\":{\"mediaType\":\"application/vnd.oci.image.config.v1+json\",\"digest\":\"sha256:${config_hash}\",\"size\":${config_size}},\"layers\":${layers},\"annotations\":{}}" > ${workdir}/manifest.json local manifest_hash=$(sha256 -q < ${workdir}/manifest.json) local manifest_size=$(stat -f %z ${workdir}/manifest.json) mkdir -p ${workdir}/oci/blobs/sha256 echo "{\"imageLayoutVersion\": \"1.0.0\"}" > ${workdir}/oci/oci-layout echo "{\"schemaVersion\":2,\"manifests\":[{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"digest\":\"sha256:${manifest_hash}\",\"size\":${manifest_size},\"annotations\":{\"org.opencontainers.image.ref.name\":\"freebsd-${image}:${ver}\"}}]}" > ${workdir}/oci/index.json - ln ${workdir}/rootfs.tar.gz ${workdir}/oci/blobs/sha256/${root_hash} ln ${workdir}/config.json ${workdir}/oci/blobs/sha256/${config_hash} ln ${workdir}/manifest.json ${workdir}/oci/blobs/sha256/${manifest_hash} diff --git a/release/tools/oci-image-notoolchain.conf b/release/tools/oci-image-notoolchain.conf --- a/release/tools/oci-image-notoolchain.conf +++ b/release/tools/oci-image-notoolchain.conf @@ -64,4 +64,6 @@ FreeBSD-yp \ FreeBSD-zfs \ FreeBSD-zoneinfo + # FreeBSD-utilities installs login.conf and generates login.conf.db using cap_mkdb + echo "./etc/login.conf.db type=file mode=0644 uname=root gname=wheel" >> ${workdir}/METALOG } diff --git a/release/tools/oci-image-static.conf b/release/tools/oci-image-static.conf --- a/release/tools/oci-image-static.conf +++ b/release/tools/oci-image-static.conf @@ -5,27 +5,33 @@ # a few other config files. OCI_BASE_IMAGE= +MTREES="root: var:/var usr:/usr include:/usr/include debug:/usr/lib" oci_image_build() { local srcdir=${curdir}/.. local m=${workdir}/rootfs - mtree -deU -p $m/ -f ${srcdir}/etc/mtree/BSD.root.dist > /dev/null - mtree -deU -p $m/var -f ${srcdir}/etc/mtree/BSD.var.dist > /dev/null - mtree -deU -p $m/usr -f ${srcdir}/etc/mtree/BSD.usr.dist > /dev/null - mtree -deU -p $m/usr/include -f ${srcdir}/etc/mtree/BSD.include.dist > /dev/null - mtree -deU -p $m/usr/lib -f ${srcdir}/etc/mtree/BSD.debug.dist > /dev/null + local installflags="-U -M ${workdir}/METALOG -o root -g wheel -D $m" + for e in ${MTREES}; do + mtree=${srcdir}/etc/mtree/BSD.${e%:*}.dist + path=${e#*:} + mtree -deU -W -p $m$path -f ${mtree} > /dev/null + mtree -C -f ${mtree} -K all | sed s#^\.#.$path# >> ${workdir}/METALOG + done install_packages ${abi} ${workdir} FreeBSD-zoneinfo - cp ${srcdir}/etc/master.passwd $m/etc + install ${installflags} -m 600 ${srcdir}/etc/master.passwd $m/etc || return $? pwd_mkdb -p -d $m/etc $m/etc/master.passwd || return $? - cp ${srcdir}/etc/group $m/etc || return $? + echo "./etc/pwd.db type=file mode=0644 uname=root gname=wheel" >> ${workdir}/METALOG + echo "./etc/spwd.db type=file mode=0600 uname=root gname=wheel" >> ${workdir}/METALOG + echo "./etc/passwd type=file mode=0644 uname=root gname=wheel" >> ${workdir}/METALOG + install ${installflags} -m 600 ${srcdir}/etc/group $m/etc || return $? # termcap.small is generated so we get it from OBJDIR - make sets our # working directory to OBJDIR/release - cp ../etc/termcap/termcap.small $m/etc/termcap.small || return $? - cp ../etc/termcap/termcap.small $m/usr/share/misc/termcap || return $? + install ${installflags} -m 600 ../etc/termcap/termcap.small $m/etc || return $? + install ${installflags} -m 600 ../etc/termcap/termcap.small $m/usr/share/misc/termcap || return $? env DESTDIR=$m \ TRUSTPATH=${srcdir}/secure/caroot/trusted \ UNTRUSTPATH=${srcdir}/secure/caroot/untrusted \ - certctl -c rehash + certctl -c -U -M ${workdir}/METALOG rehash # Generate a suitable repo config for pkgbase case ${branch} in CURRENT|STABLE|BETA*) @@ -35,8 +41,8 @@ repo=base_release_${minor} ;; esac - mkdir -p $m/usr/local/etc/pkg/repos - cat > $m/usr/local/etc/pkg/repos/base.conf < ${workdir}/base.conf <