osbuild: support more platforms; reduce memory usage for mkfs.erofs

build.sh

@@ -171,6 +171,8 @@ patch_osbuild() {
     # Now all the software is under the /usr/lib/osbuild dir and we can patch
     cat /usr/lib/coreos-assembler/0001-stages-coreos.live-artifacts-add-erofs-support.patch       \
         /usr/lib/coreos-assembler/0002-stages-coreos.live-artifacts-update-comments-names-t.patch \
+        /usr/lib/coreos-assembler/0001-stages-add-more-options-to-qemu-vmdk-disk-type.patch       \
+        /usr/lib/coreos-assembler/0001-stages-coreos.live-artifacts-use-var-tmp-for-mkfs.er.patch \
             | patch -d /usr/lib/osbuild -p1
 
     # And then move the files back; supermin appliance creation will need it back

src/0001-stages-add-more-options-to-qemu-vmdk-disk-type.patch

@@ -0,0 +1,74 @@
+From 60f54f468bbdb6f245bf16bd5faf95b21e5b25d9 Mon Sep 17 00:00:00 2001
+From: Dusty Mabe <dusty@dustymabe.com>
+Date: Fri, 14 Feb 2025 08:36:25 -0500
+Subject: [PATCH] stages: add more options to qemu vmdk disk type
+
+The CoreOS team uses the compat6 and adapter_type options when creating
+a VMDK for AWS.
+
+https://github.com/coreos/coreos-assembler/blob/e1943d6adb8f0cb257630a76d02d0aa333261c70/src/cosalib/qemuvariants.py#L48
+---
+ stages/org.osbuild.qemu           | 14 ++++++++++++--
+ stages/org.osbuild.qemu.meta.json | 15 +++++++++++++++
+ 2 files changed, 27 insertions(+), 2 deletions(-)
+
+diff --git a/stages/org.osbuild.qemu b/stages/org.osbuild.qemu
+index a6df9166..caad5e88 100755
+--- a/stages/org.osbuild.qemu
++++ b/stages/org.osbuild.qemu
+@@ -21,14 +21,24 @@ def qcow2_arguments(options):
+
+ def vmdk_arguments(options):
+     argv = []
++    adapter_type = options.get("adapter_type")
++    compat6 = options.get("compat6", False)
+     compression = options.get("compression", True)
+     subformat = options.get("subformat")
+
+     if compression:
+         argv += ["-c"]
+
+-    if subformat:
+-        argv += ["-o", f"subformat={subformat}"]
++    if compat6 or subformat or adapter_type:
++        opts = []
++        if adapter_type:
++            opts += [f"adapter_type={adapter_type}"]
++        if compat6:
++            opts += ["compat6"]
++        if subformat:
++            opts += [f"subformat={subformat}"]
++        argv += ["-o", ",".join(opts)]
++
+     return argv
+
+
+diff --git a/stages/org.osbuild.qemu.meta.json b/stages/org.osbuild.qemu.meta.json
+index 5542d27f..5fb38e04 100644
+--- a/stages/org.osbuild.qemu.meta.json
++++ b/stages/org.osbuild.qemu.meta.json
+@@ -68,6 +68,21 @@
+               "vmdk"
+             ]
+           },
++          "adapter_type": {
++            "description": "Virtual adapter type",
++            "type": "string",
++            "enum": [
++              "ide",
++              "lsilogic",
++              "buslogic",
++              "legacyESX"
++            ]
++          },
++          "compat6": {
++            "description": "VMDK version 6 image",
++            "type": "boolean",
++            "default": false
++          },
+           "compression": {
+             "description": "Enable/disable compression of the vmdk image",
+             "type": "boolean",
+-- 
+2.48.1
+

src/0001-stages-coreos.live-artifacts-use-var-tmp-for-mkfs.er.patch

@@ -0,0 +1,35 @@
+From 31faba33b71feb160f436aaf4c9ac9932cb68830 Mon Sep 17 00:00:00 2001
+From: Dusty Mabe <dusty@dustymabe.com>
+Date: Fri, 14 Feb 2025 12:23:48 -0500
+Subject: [PATCH] stages/coreos.live-artifacts: use /var/tmp for mkfs.erofs
+
+This helps reduce the memory requirements of mkfs.erofs since
+it won't be writing to tmpfs (memory backed) storage, which is
+what is mounted on /tmp/.
+---
+ stages/org.osbuild.coreos.live-artifacts.mono | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/stages/org.osbuild.coreos.live-artifacts.mono b/stages/org.osbuild.coreos.live-artifacts.mono
+index 302b24d3..a5837deb 100755
+--- a/stages/org.osbuild.coreos.live-artifacts.mono
++++ b/stages/org.osbuild.coreos.live-artifacts.mono
+@@ -606,10 +606,14 @@ def mkrootfs_metal(paths, workdir, img_metal, fstype, fsoptions, loop_client):
+             # Note the filename must be exactly "root.[squash|ero]fs"
+             # because the 35coreos-live dracut module requires it.
+             if fstype == "erofs":
++                # Set TMPDIR='/var/tmp' to write temporary files into non-tmpfs
++                # (memory backed) storage. This helps reduce memory requirements.
++                # https://github.com/erofs/erofs-utils/issues/13
+                 subprocess.check_call(['mkfs.erofs',
+                                        *fsoptions.split(' '),
+                                        paths["initrd-rootfs/root.erofs"],
+-                                       tmp_rootfs_dir])
++                                       tmp_rootfs_dir],
++                                      env=dict(os.environ, TMPDIR='/var/tmp'))
+             else:
+                 subprocess.check_call(['mksquashfs', tmp_rootfs_dir,
+                                        paths["initrd-rootfs/root.squashfs"],
+-- 
+2.48.1
+

src/cmd-compress

@@ -274,7 +274,7 @@ if args.mode == "compress":
     # Find what compressor we should use, either picking it up from
     # CLI args or from image.json
     image_json = get_image_json()
-    gzip_level = 6
+    gzip_level = 9
     if args.fast:
         args.compressor = 'gzip'
         gzip_level = 1

src/cmd-osbuild

@@ -8,14 +8,24 @@ dn=$(dirname "$0")
 # A list of supported platforms and the filename suffix of the main
 # artifact that platform produces.
 declare -A SUPPORTED_PLATFORMS=(
+    ['aliyun']='qcow2'
     ['applehv']='raw'
+    ['aws']='vmdk'
+    ['azure']='vhd'
+    ['azurestack']='vhd'
+    ['digitalocean']='qcow2'
+    ['exoscale']='qcow2'
     ['gcp']='tar.gz'
     ['hetzner']='raw'
     ['hyperv']='vhdx'
+    ['ibmcloud']='qcow2'
     ['metal4k']='raw'
     ['metal']='raw'
+    ['nutanix']='qcow2'
+    ['openstack']='qcow2'
     ['qemu']='qcow2'
     ['qemu-secex']='qcow2'
+    ['vultr']='raw'
     ['live']='iso'
 )
 
@@ -385,9 +395,10 @@ main() {
 
         # Perform postprocessing
         case "$platform" in
-            gcp)
+            gcp|nutanix)
                 # Update the meta.json and builddir with the generated artifact.
-                # Skip Compression on these platforms as they are already compressed.
+                # Skip Compression on these platforms as they are either already
+                # compressed or the artifact itself has internal compression enabled.
                 postprocess_artifact "${platform}" "${imgpath}" "${imgname}" 'True'
                 ;;
             live)

src/cmdlib.sh

@@ -579,7 +579,7 @@ runcompose_tree() {
 
 # Run with cache disk.
 runvm_with_cache() {
-    local cache_size=${RUNVM_CACHE_SIZE:-30G}
+    local cache_size=${RUNVM_CACHE_SIZE:-40G}
     # "cache2" has an explicit label so we can find it in qemu easily
     if [ ! -f "${workdir}"/cache/cache2.qcow2 ]; then
         qemu-img create -f qcow2 cache2.qcow2.tmp "$cache_size"
@@ -767,7 +767,7 @@ EOF
 
     # There seems to be some false positives in shellcheck
     # https://github.com/koalaman/shellcheck/issues/2217
-    memory_default=2048
+    memory_default=3072
     # shellcheck disable=2031
     case $arch in
     # Power 8 page faults with 2G of memory in rpm-ostree

src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml

@@ -772,6 +772,10 @@ pipelines:
                 default: true
   - mpp-import-pipelines:
       path: platform.applehv.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.aws.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.azure.ipp.yaml
   - mpp-import-pipelines:
       path: platform.gcp.ipp.yaml
   - mpp-import-pipelines:
@@ -780,6 +784,8 @@ pipelines:
       path: platform.hyperv.ipp.yaml
   - mpp-import-pipelines:
       path: platform.metal.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.openstack.ipp.yaml
   - mpp-import-pipelines:
       path: platform.qemu.ipp.yaml
   - mpp-import-pipelines:

src/osbuild-manifests/coreos.osbuild.ppc64le.mpp.yaml

@@ -720,6 +720,8 @@ pipelines:
                 default: true
   - mpp-import-pipelines:
       path: platform.metal.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.openstack.ipp.yaml
   - mpp-import-pipelines:
       path: platform.qemu.ipp.yaml
   - mpp-import-pipelines:

src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml

@@ -660,8 +660,12 @@ pipelines:
               source: mount
               deployment:
                 default: true
+  - mpp-import-pipelines:
+      path: platform.ibmcloud.ipp.yaml
   - mpp-import-pipelines:
       path: platform.metal.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.openstack.ipp.yaml
   - mpp-import-pipelines:
       path: platform.qemu.ipp.yaml
   - mpp-import-pipelines:

src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml

@@ -774,17 +774,37 @@ pipelines:
               source: mount
               deployment:
                 default: true
+  - mpp-import-pipelines:
+      path: platform.aliyun.ipp.yaml
   - mpp-import-pipelines:
       path: platform.applehv.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.aws.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.azure.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.azurestack.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.digitalocean.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.exoscale.ipp.yaml
   - mpp-import-pipelines:
       path: platform.gcp.ipp.yaml
   - mpp-import-pipelines:
       path: platform.hetzner.ipp.yaml
   - mpp-import-pipelines:
       path: platform.hyperv.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.ibmcloud.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.openstack.ipp.yaml
   - mpp-import-pipelines:
       path: platform.metal.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.nutanix.ipp.yaml
   - mpp-import-pipelines:
       path: platform.qemu.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.vultr.ipp.yaml
   - mpp-import-pipelines:
       path: platform.live.ipp.yaml

src/osbuild-manifests/platform.aliyun.ipp.yaml

@@ -0,0 +1,71 @@
+# This file defines the artifact to be used for the aliyun platform.
+version: '2'
+pipelines:
+  - name: raw-aliyun-image
+    build:
+      mpp-format-string: '{buildroot}'
+    stages:
+      - type: org.osbuild.copy
+        inputs:
+          tree:
+            type: org.osbuild.tree
+            origin: org.osbuild.pipeline
+            references:
+              - name:raw-image
+        options:
+          paths:
+            - from: input://tree/disk.img
+              to: tree:///disk.img
+      # Increase the size to the cloud image size
+      - type: org.osbuild.truncate
+        options:
+          filename: disk.img
+          size:
+            mpp-format-string: "{cloud_image_size_mb * 1024 * 1024}"
+      - type: org.osbuild.coreos.platform
+        options:
+          platform: aliyun
+        devices:
+          disk:
+            type: org.osbuild.loopback
+            options:
+              filename: disk.img
+              partscan: true
+        mounts:
+           - name: root
+             type: org.osbuild.xfs
+             source: disk
+             partition:
+               mpp-format-int: '{image.layout[''root''].partnum}'
+             target: /
+           - name: ostree.deployment
+             type: org.osbuild.ostree.deployment
+             options:
+               source: mount
+               deployment:
+                 default: true
+           - name: boot
+             type: org.osbuild.ext4
+             source: disk
+             partition:
+               mpp-format-int: '{image.layout[''boot''].partnum}'
+             target: /boot
+  - name: aliyun
+    build:
+      mpp-format-string: '{host_as_buildroot}'
+    stages:
+      - type: org.osbuild.qemu
+        inputs:
+          image:
+            type: org.osbuild.files
+            origin: org.osbuild.pipeline
+            references:
+              name:raw-aliyun-image:
+                file: disk.img
+        options:
+          filename:
+            mpp-format-string: '{artifact_name_prefix}-aliyun.{arch}.qcow2'
+          format:
+            type: qcow2
+            compression: false
+            compat: '1.1'