diff --git a/release/Makefile.oracle b/release/Makefile.oracle new file mode 100644 --- /dev/null +++ b/release/Makefile.oracle @@ -0,0 +1,101 @@ +# +# Makefile for preparing & uploading Oracle Cloud images from existing +# .raw files created by cloudware-release. +# +# Overview: +# +# The base image is already created by cloudware-release. +# +# Construct the custom OCI metadata, derived from exported official OCI images. +# It is architecture-specific but appears mostly stable over time. +# Compress the raw image and place it in the same directory as the metadata. +# Make a GNU format tarball of these files. +# Upload the tarball to Oracle Cloud via a pre-approved curl URI, into +# the FreeBSD Foundation's Oracle Cloud account. +# +# These images go into the "re" bucket in us-ashburn-1 region, which +# is mounted into the FreeBSD Foundation Oracle Marketplace account. +# Once uploaded, a manual step is needed to import the images as local +# custom images. These can then be tested within the us-ashburn-1 region. +# Once tested, follow the manual Oracle Marketplace import process to +# create a new FreeBSD version, attach the images, and initiate validation +# by Oracle. This can take up to 5 working days. Once complete, a final +# manual step is needed to mark the currently private images, public. +# Syncing to all sites should take 2-3 hours after this final step. + +ORACLE_BASENAME= ${OSRELEASE}-${BUILDDATE}${GITREV:C/^(.+)/-\1/} +ORACLE_PORTS_LIST= ftp/curl emulators/qemu@tools +CLEANFILES+= cw-oracle-portinstall + +cw-oracle-portinstall: .PHONY +.if !exists(/usr/local/bin/curl) || !exists(/usr/local/bin/qemu-img) +. if !exists(${PORTSDIR}/Makefile) +. if !exists(/usr/local/sbin/pkg-static) + env ASSUME_ALWAYS_YES=yes pkg bootstrap -yf +. endif + env ASSUME_ALWAYS_YES=yes pkg install -y ${ORACLE_PORTS_LIST} +. else + env UNAME_r=${UNAME_r} make -C \ + ${PORTSDIR}/ftp/curl \ + BATCH=1 WRKDIRPREFIX=/tmp/ports DISTDIR=/tmp/distfiles \ + all install clean + env UNAME_r=${UNAME_r} FLAVOR=tools make -C \ + ${PORTSDIR}/emulators/qemu \ + BATCH=1 WRKDIRPREFIX=/tmp/ports DISTDIR=/tmp/distfiles \ + all install clean +. endif +.endif + +.for _FS in ${ORACLE_FSLIST} +ORACLE_OCI_LIST+= cw-oracle-${_FS}.oci +ORACLE_UPLOAD_LIST+= cw-oracle-upload-${_FS} +CLEANFILES+= cw-oracle-${_FS}.oci +ORACLE_TMP= cw-oracle-${_FS}.oci.tmpdir +CLEANDIRS+= ${ORACLE_TMP} +ORACLE_METADATA= ${.CURDIR}/scripts/oracle +ORACLE_CAPABILITY= ${.CURDIR}/scripts/oracle/image_capability_data.json +ORACLE_TEMPLATE= ${.CURDIR}/scripts/oracle/image_metadata.json +ORACLE_OUTPUT= ${ORACLE_TMP}/image_metadata.json +.if ${TARGET} == "arm64" +ORACLE_SHAPES= ${ORACLE_METADATA}/arm64_shape_compatibilities.json +.else +ORACLE_SHAPES= ${ORACLE_METADATA}/default_shape_compatibilities.json +.endif + +cw-oracle-${_FS}.oci: cw-oracle-${_FS}-raw + mkdir -p ${ORACLE_TMP} + # create architecture-specific JSON metadata + env TYPE="${TYPE}" \ + OSRELEASE="${OSRELEASE}" \ + ORACLE_CAPABILITY="${ORACLE_CAPABILITY}" \ + ORACLE_SHAPES="${ORACLE_SHAPES}" \ + ORACLE_TEMPLATE="${ORACLE_TEMPLATE}" \ + ORACLE_OUTPUT="${ORACLE_OUTPUT}" \ + ${ORACLE_METADATA}/generate_metadata.lua + + # convert raw to native qcow2 for zstd compression, saves ~ 8GiB + qemu-img convert -S 512b -p -O qcow2 -c -o compression_type=zstd \ + ${.OBJDIR}/${ORACLE${_FS:tu}RAWIMAGE} \ + ${ORACLE_TMP}/output.QCOW2 + + # Create GNU-compatible tarball using BSD tar + tar --format=gnutar -cf ${.TARGET} -C ${ORACLE_TMP} \ + image_metadata.json output.QCOW2 + + echo "Oracle image ${.TARGET} is ready for upload." + +cw-oracle-upload-${_FS}: cw-oracle-${_FS}.oci +.if !defined(ORACLE_PAR_URL) || empty(ORACLE_PAR_URL) + @echo "--------------------------------------------------------------" + @echo ">>> ORACLE_PAR_URL must be set for Oracle image upload" + @echo "--------------------------------------------------------------" + @false +.endif + echo "Please wait ... uploading cw-oracle-${_FS}.oci to ${ORACLE_BASENAME}-${_FS}.oci" + curl -s ${ORACLE_PAR_URL}/${ORACLE_BASENAME}-${_FS}.oci --upload-file cw-oracle-${_FS}.oci + echo "Uploaded cw-oracle-${_FS}.oci as ${ORACLE_BASENAME}-${_FS}.oci" + touch ${.TARGET} +.endfor + +cw-oracle-prepare: ${ORACLE_OCI_LIST} +cw-oracle-upload: cw-oracle-portinstall ${ORACLE_UPLOAD_LIST} diff --git a/release/Makefile.vm b/release/Makefile.vm --- a/release/Makefile.vm +++ b/release/Makefile.vm @@ -274,5 +274,6 @@ .include "${.CURDIR}/Makefile.ec2" .include "${.CURDIR}/Makefile.firecracker" .include "${.CURDIR}/Makefile.gce" +.include "${.CURDIR}/Makefile.oracle" .include "${.CURDIR}/Makefile.vagrant" .include "${.CURDIR}/Makefile.inc1" diff --git a/release/scripts/oracle/arm64_shape_compatibilities.json b/release/scripts/oracle/arm64_shape_compatibilities.json new file mode 100644 --- /dev/null +++ b/release/scripts/oracle/arm64_shape_compatibilities.json @@ -0,0 +1,24 @@ +[ + { + "internalShapeName": "VM.Standard.A1.Flex", + "ocpuConstraints": { + "min": 1, + "max": 80 + }, + "memoryConstraints": { + "minInGBs": 1, + "maxInGBs": 512 + } + }, + { + "internalShapeName": "VM.Standard.A2.Flex", + "ocpuConstraints": { + "min": 1, + "max": 78 + }, + "memoryConstraints": { + "minInGBs": 1, + "maxInGBs": 946 + } + } +] diff --git a/release/scripts/oracle/default_shape_compatibilities.json b/release/scripts/oracle/default_shape_compatibilities.json new file mode 100644 --- /dev/null +++ b/release/scripts/oracle/default_shape_compatibilities.json @@ -0,0 +1 @@ +[] diff --git a/release/scripts/oracle/generate_metadata.lua b/release/scripts/oracle/generate_metadata.lua new file mode 100755 --- /dev/null +++ b/release/scripts/oracle/generate_metadata.lua @@ -0,0 +1,72 @@ +#!/usr/local/bin/flua + +local ucl = require("ucl") + +-- read from environment variables +local os_type = os.getenv("TYPE") +local os_version = os.getenv("OSRELEASE") +-- the raw file +local capability_file = os.getenv("ORACLE_CAPABILITY") +-- the platform-specific file +local shapes_file = os.getenv("ORACLE_SHAPES") +-- base template +local template_file = os.getenv("ORACLE_TEMPLATE") +local output_file = os.getenv("ORACLE_OUTPUT") + +if not os_type or not os_version or not capability_file or + not shapes_file or not template_file or not output_file then + io.stderr:write("Error: Oracle script is missing required environment variables:\n") + io.stderr:write("TYPE, OSRELEASE, ORACLE_CAPABILITY, ORACLE_SHAPES, ORACLE_TEMPLATE, ORACLE_OUTPUT\n") + os.exit(1) +end + +-- read files +local function read_file(path) + local f = io.open(path, "r") + if not f then + io.stderr:write("Error: Oracle script cannot open file: " .. path .. "\n") + os.exit(1) + end + local content = f:read("*a") + f:close() + return content +end + +-- parse the template +local template = read_file(template_file) +local metadata = ucl.parser() +metadata:parse_string(template) +local data = metadata:get_object() + +-- update the simple fields +data.operatingSystem = os_type +data.operatingSystemVersion = os_version + +-- capability data is actually JSON, but needs to be inserted as a raw blob +local caps = read_file(capability_file) +-- remove all newlines and preceding spaces to match Oracle's format +caps = caps:gsub("\n", "") +caps = caps:gsub("%s+", "") +-- is it still valid JSON? +local caps_parser = ucl.parser() +if not caps_parser:parse_string(caps) then + io.stderr:write("Error: Oracle script found invalid JSON in capability file\n") + os.exit(1) +end +-- insert as a raw blob +data.imageCapabilityData = caps + +-- parse and insert architecture-dependent shape compatibilities data +local shapes_data = read_file(shapes_file) +local shapes = ucl.parser() +shapes:parse_string(shapes_data) +data.additionalMetadata.shapeCompatibilities = shapes:get_object() + +-- save the metadata +local out = io.open(output_file, "w") +if not out then + io.stderr:write("Error: Oracle script cannot create output file: " .. output_file .. "\n") + os.exit(1) +end +out:write(ucl.to_format(data, "json", {pretty = true})) +out:close() diff --git a/release/scripts/oracle/image_capability_data.json b/release/scripts/oracle/image_capability_data.json new file mode 100644 --- /dev/null +++ b/release/scripts/oracle/image_capability_data.json @@ -0,0 +1,96 @@ +{ + "capabilities": { + "Compute.AMD_SecureEncryptedVirtualization": { + "descriptorType": "boolean", + "defaultValue": false + }, + "Storage.BootVolumeType": { + "descriptorType": "enumstring", + "values": [ + "ISCSI", + "PARAVIRTUALIZED", + "SCSI", + "IDE", + "NVME" + ], + "defaultValue": "PARAVIRTUALIZED" + }, + "Storage.Iscsi.MultipathDeviceSupported": { + "descriptorType": "boolean", + "defaultValue": false + }, + "Storage.ParaVirtualization.EncryptionInTransit": { + "descriptorType": "boolean", + "defaultValue": true + }, + "Storage.ConsistentVolumeNaming": { + "descriptorType": "boolean", + "defaultValue": true + }, + "Compute.SecureBoot": { + "descriptorType": "boolean", + "defaultValue": false + }, + "Storage.ParaVirtualization.AttachmentVersion": { + "descriptorType": "enuminteger", + "values": [ + 1, + 2 + ], + "defaultValue": 2 + }, + "Storage.LocalDataVolumeType": { + "descriptorType": "enumstring", + "values": [ + "ISCSI", + "PARAVIRTUALIZED", + "SCSI", + "IDE", + "NVME" + ], + "defaultValue": "PARAVIRTUALIZED" + }, + "Network.AttachmentType": { + "descriptorType": "enumstring", + "values": [ + "PARAVIRTUALIZED", + "VDPA" + ], + "defaultValue": "PARAVIRTUALIZED" + }, + "Storage.RemoteDataVolumeType": { + "descriptorType": "enumstring", + "values": [ + "ISCSI", + "PARAVIRTUALIZED", + "SCSI", + "IDE", + "NVME" + ], + "defaultValue": "PARAVIRTUALIZED" + }, + "Compute.LaunchMode": { + "descriptorType": "enumstring", + "values": [ + "NATIVE", + "EMULATED", + "VDPA", + "PARAVIRTUALIZED", + "CUSTOM" + ], + "defaultValue": "PARAVIRTUALIZED" + }, + "Network.IPv6Only": { + "descriptorType": "boolean", + "defaultValue": false + }, + "Compute.Firmware": { + "descriptorType": "enumstring", + "values": [ + "BIOS", + "UEFI_64" + ], + "defaultValue": "UEFI_64" + } + } +} diff --git a/release/scripts/oracle/image_metadata.json b/release/scripts/oracle/image_metadata.json new file mode 100644 --- /dev/null +++ b/release/scripts/oracle/image_metadata.json @@ -0,0 +1,21 @@ +{ + "version": 2, + "externalLaunchOptions": { + "firmware": "UEFI_64", + "networkType": "PARAVIRTUALIZED", + "bootVolumeType": "PARAVIRTUALIZED", + "remoteDataVolumeType": "PARAVIRTUALIZED", + "localDataVolumeType": "PARAVIRTUALIZED", + "launchOptionsSource": "PARAVIRTUALIZED", + "pvAttachmentVersion": 2, + "pvEncryptionInTransitEnabled": false, + "consistentVolumeNamingEnabled": false + }, + "imageCapabilityData": "REPLACE", + "imageCapsFormatVersion": "23cfd738-ad9c-4f56-9281-67be6c8cd14c", + "operatingSystem": "REPLACE", + "operatingSystemVersion": "REPLACE", + "additionalMetadata": { + "shapeCompatibilities": "REPLACE" + } +}