diff --git a/.github/actions/test/integration/build/action.yml b/.github/actions/test/integration/build/action.yml
index 447de18c..a6e955db 100644
--- a/.github/actions/test/integration/build/action.yml
+++ b/.github/actions/test/integration/build/action.yml
@@ -2,13 +2,30 @@ name: build
description: Build Bootable Virtual Machine Image
inputs:
image_tag:
- description: "Image tag to use for the build"
- required: true
+ description: "Image tag to use for the build (legacy mode)"
+ required: false
+ default: ""
+ mode:
+ description: "legacy: build the Garden Linux HV image; sysext: only prepare the Ubuntu guest"
+ required: false
+ default: "legacy"
+ sysext_key:
+ description: "Private test key prepared by prepare-sysext-hv.sh (sysext mode); its .pub is reused for the Ubuntu guest"
+ required: false
+ default: ""
runs:
using: "composite"
steps:
+ - name: Validate the test mode
+ if: ${{ inputs.mode != 'legacy' && inputs.mode != 'sysext' }}
+ shell: bash
+ run: |
+ echo "unsupported mode '${{ inputs.mode }}' (expected legacy or sysext)" >&2
+ exit 1
+
# TODO https://github.com/actions/cache
- name: Set ENV
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
env:
REPO: ghcr.io/gardenlinux/gardenlinux-sci
@@ -22,6 +39,7 @@ runs:
echo "DISK_SIZE=$DISK_SIZE" >> $GITHUB_ENV
- name: Pull ORAS artifacts
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
run: |
LAYERS=$(oras manifest fetch "$REPO:$TAG" | jq -r '.layers[]')
@@ -32,11 +50,13 @@ runs:
oras blob fetch "$REPO@$TAR_SHA" -o "$RUNNER_TEMP/$TAG.tar" # wasteful, we only systemd-boot
- name: Extract systemd-boot
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
run: |
tar -xf "$RUNNER_TEMP/$TAG.tar" --no-same-owner --strip-components=5 -C "$RUNNER_TEMP" usr/lib/systemd/boot/efi/systemd-bootx64.efi
- name: Set up a raw disk image
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
run: |
ESP_END="${ESP_SIZE}GiB"
@@ -62,6 +82,7 @@ runs:
sudo mkfs.ext4 "$VAR_PART" -L VAR
- name: Populate the ESP partition
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
run: |
mkdir -p "$RUNNER_TEMP/esp"
@@ -88,25 +109,39 @@ runs:
sudo losetup -d "$LOOPDEV"
- name: Convert to QCOW2
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
run: |
qemu-img convert -O qcow2 "$DISK_IMG" "/opt/${TAG}.qcow2"
- - name: Generate ignition
+ - name: Generate test SSH key and password
shell: bash
+ env:
+ SYSEXT_KEY: ${{ inputs.sysext_key }}
run: |
- ssh-keygen -t ed25519 -f "/opt/ssh_host_ed25519_key" -N "" -C "user" -q
- chmod 600 /opt/ssh_host_ed25519_key
+ if [ -n "$SYSEXT_KEY" ]; then
+ # One key for the HVs and the Ubuntu guests.
+ install -m 600 "$SYSEXT_KEY" /opt/ssh_host_ed25519_key
+ install -m 644 "$SYSEXT_KEY.pub" /opt/ssh_host_ed25519_key.pub
+ else
+ ssh-keygen -t ed25519 -f "/opt/ssh_host_ed25519_key" -N "" -C "user" -q
+ chmod 600 /opt/ssh_host_ed25519_key
+ fi
SSH_KEY=$(cat /opt/ssh_host_ed25519_key.pub)
+ PASSWORD=$(openssl passwd "password")
echo "SSH_KEY=$SSH_KEY" >> $GITHUB_ENV
+ echo "PASSWORD=$PASSWORD" >> $GITHUB_ENV
+
+ - name: Generate ignition
+ if: ${{ inputs.mode != 'sysext' }}
+ shell: bash
+ run: |
cp ./.github/actions/test/integration/build/dev-user-butane.yaml ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
yq '.storage.files += load("./.github/actions/test/integration/build/root-hints.yaml")' ./.github/actions/test/integration/build/dev-user-butane.yaml > ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml
sed -i "s|SSH_KEY_GOES_HERE|$SSH_KEY|g" ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
sed -i "s|SSH_KEY_GOES_HERE|$SSH_KEY|g" ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml
- PASSWORD=$(openssl passwd "password")
- echo "PASSWORD=$PASSWORD" >> $GITHUB_ENV
sed -i "s|PASSWORD_GOES_HERE|$PASSWORD|g" ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
sed -i "s|PASSWORD_GOES_HERE|$PASSWORD|g" ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml
sed -i "s|TAG_GOES_HERE|${{ inputs.image_tag }}|g" ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
diff --git a/.github/actions/test/integration/collect-logs/action.yml b/.github/actions/test/integration/collect-logs/action.yml
index 5bb85850..2d9ad441 100644
--- a/.github/actions/test/integration/collect-logs/action.yml
+++ b/.github/actions/test/integration/collect-logs/action.yml
@@ -13,6 +13,11 @@ runs:
sudo cp /var/log/libvirt/qemu/HV1.log "libvirt-HV1.log" || true
sudo cp /var/log/libvirt/qemu/HV2.log "libvirt-HV2.log" || true
+ # The HV serial consoles: the only record of a frozen guest's
+ # final kernel output (the journal goes down with the guest).
+ sudo cp /var/log/HV1.log "serial-HV1.log" || true
+ sudo cp /var/log/HV2.log "serial-HV2.log" || true
+
- name: Obtain logs from hypervisors
shell: bash
run: |
diff --git a/.github/actions/test/integration/dependencies/action.yml b/.github/actions/test/integration/dependencies/action.yml
index efb047f6..3003cb61 100644
--- a/.github/actions/test/integration/dependencies/action.yml
+++ b/.github/actions/test/integration/dependencies/action.yml
@@ -10,7 +10,7 @@ runs:
sudo apt-get install -y \
parted \
squashfs-tools \
- qemu-kvm \
+ qemu-system-x86 \
genisoimage \
libvirt-daemon-system \
libvirt-clients \
@@ -26,6 +26,12 @@ runs:
echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf
sudo systemctl restart libvirtd
+ # The libvirt qemu user must reach /dev/kvm for the hypervisor VMs.
+ - name: Make /dev/kvm accessible
+ shell: bash
+ run: |
+ if [ -e /dev/kvm ]; then sudo chmod 666 /dev/kvm; fi
+
- name: oras
uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2
with:
diff --git a/.github/actions/test/integration/setup/action.yml b/.github/actions/test/integration/setup/action.yml
index 1b8b1bea..3a27848c 100644
--- a/.github/actions/test/integration/setup/action.yml
+++ b/.github/actions/test/integration/setup/action.yml
@@ -1,9 +1,27 @@
name: setup
description: Set up Hypervisor and VM Environment for Integration Tests
+inputs:
+ mode:
+ description: "legacy: Garden Linux image + ignition; sysext: published base + delivery container"
+ required: false
+ default: "legacy"
+ hv_dir:
+ description: "Output of prepare-sysext-hv.sh --prepare (sysext mode)"
+ required: false
+ default: ""
runs:
using: "composite"
steps:
+ - name: Validate the test mode
+ if: ${{ inputs.mode != 'legacy' && inputs.mode != 'sysext' }}
+ shell: bash
+ run: |
+ echo "unsupported mode '${{ inputs.mode }}' (expected legacy or sysext)" >&2
+ exit 1
+
+ # ---- legacy path ------------------------------------------------------
- name: Configure the default libvirt network
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
run: |
sudo virsh net-destroy default
@@ -13,12 +31,14 @@ runs:
sudo virsh net-autostart default
- name: Copy ignition file
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
run: |
sudo cp "/opt/${TAG}-hv1.ign" /var/lib/libvirt/images/HV1.ign
sudo cp "/opt/${TAG}-hv2.ign" /var/lib/libvirt/images/HV2.ign
- name: Create HyperVisor 1
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
run: |
cp ./.github/actions/test/integration/setup/hv.xml ./.github/actions/test/integration/setup/hv1.xml
@@ -30,6 +50,7 @@ runs:
sudo virsh start HV1
- name: Create HyperVisor 2
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
run: |
cp ./.github/actions/test/integration/setup/hv.xml ./.github/actions/test/integration/setup/hv2.xml
@@ -41,6 +62,7 @@ runs:
sudo virsh start HV2
- name: Configure SSH and libvirt on HyperVisors
+ if: ${{ inputs.mode != 'sysext' }}
shell: bash
run: |
KEY=/opt/ssh_host_ed25519_key
@@ -110,3 +132,72 @@ runs:
wait_for_ssh HV1
wait_for_ssh HV2
+
+ # ---- sysext path ------------------------------------------------------
+ - name: Boot the sysext HyperVisors and run the delivery container
+ if: ${{ inputs.mode == 'sysext' }}
+ shell: bash
+ env:
+ HV_DIR: ${{ inputs.hv_dir }}
+ run: |
+ set -Eeuo pipefail
+ [ -n "$HV_DIR" ] && [ -f "$HV_DIR/metadata.env" ] || { echo "hv_dir with metadata.env is required in sysext mode" >&2; exit 1; }
+ source "$HV_DIR/metadata.env"
+ KEY=$HV_DIR/hv-test-key
+ SSH="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+ for f in "$KEY" "$HV_KERNEL" "$HV_ARTIFACT_TAR" "$HV_DIR/sci-delivery.ref" "$HV_DIR/HV1.xml" "$HV_DIR/HV2.xml" "$HV_DIR/disks/HV1.qcow2" "$HV_DIR/disks/HV2.qcow2"; do
+ [ -s "$f" ] || { echo "missing prepared artifact: $f" >&2; exit 1; }
+ done
+ [ -n "$HV_ARTIFACT_REF" ] || { echo "empty delivery image reference" >&2; exit 1; }
+ sudo virsh net-info default >/dev/null || { echo "the libvirt default network is not active; sysext mode does not create it" >&2; exit 1; }
+ # The runner home ships as 750; qemu must traverse it to reach the disks.
+ sudo chmod o+x "$HOME"
+ add_host() { sudo virsh net-dumpxml default | grep -q "mac='$1'" || sudo virsh net-update default add ip-dhcp-host "" --live --config; }
+ add_host 52:54:00:12:34:56 192.168.122.2 hv1
+ add_host 52:54:00:65:43:21 192.168.122.3 hv2
+ for spec in "HV1 192.168.122.2" "HV2 192.168.122.3"; do
+ set -- $spec
+ sudo virsh dominfo "$1" >/dev/null 2>&1 && { echo "refusing to replace a pre-existing domain: $1" >&2; exit 1; }
+ sudo virsh define "$HV_DIR/$1.xml"
+ sudo virsh start "$1" || { namei -l "$HV_DIR/disks/$1.qcow2" >&2 || true; exit 1; }
+ for ((t=1; t<=60; t++)); do
+ ssh $SSH "root@$2" true 2>/dev/null && break
+ [ "$t" -lt 60 ] || { echo "timeout waiting for $1 to respond to SSH" >&2; sudo cat "/var/log/$1.log" || true; exit 1; }
+ sleep 5
+ done
+ scp -q $SSH "$KEY" "root@$2:/opt/ssh_host_ed25519_key"
+ scp -q $SSH "$HV_ARTIFACT_TAR" "root@$2:/opt/sci-delivery.tar"
+ scp -q $SSH "$HV_DIR/sci-delivery.ref" "root@$2:/opt/sci-delivery.ref"
+ scp -q $SSH ./.github/actions/test/integration/setup/virbr-vm.xml "root@$2:/opt/virbr-vm.xml"
+ ssh $SSH "root@$2" bash -s <<'EOF'
+ set -Eeuo pipefail
+ chmod 700 /root/.ssh
+ chmod 600 /opt/ssh_host_ed25519_key
+ systemctl is-active containerd.service >/dev/null || systemctl start containerd.service
+ REF=$(head -n 1 /opt/sci-delivery.ref)
+ test -n "$REF"
+ ctr -n k8s.io images import /opt/sci-delivery.tar
+ rm -f /opt/sci-delivery.tar
+ ctr -n k8s.io container rm -f sci-delivery >/dev/null 2>&1 || true
+ # hostIPC is required: systemd-sysext waits on a dm-verity udev cookie
+ # posted by the host udevd, so join PID 1's IPC namespace.
+ ctr -n k8s.io run --rm --privileged --with-ns pid:/proc/1/ns/pid \
+ --with-ns ipc:/proc/1/ns/ipc \
+ --env HOST_ROOT=/host --env INTERVAL=0 --env NSENTER=1 \
+ --mount type=bind,src=/,dst=/host,options=rbind:rw \
+ "$REF" sci-delivery < /dev/null
+ systemd-sysext status
+ systemctl is-active libvirtd.service
+ mkdir -p /var/log/libvirt/ch
+ printf 'Host 192.168.122.*\n StrictHostKeyChecking no\n UserKnownHostsFile /dev/null\n IdentityFile /opt/ssh_host_ed25519_key\n IdentitiesOnly yes\n User root\n' > /root/.ssh/config
+ chmod 600 /root/.ssh/config /opt/ssh_host_ed25519_key
+ virsh pool-destroy default || true
+ virsh pool-undefine default || true
+ virsh pool-define-as default dir --target /var/lib/libvirt/images
+ virsh pool-build default; virsh pool-start default; virsh pool-autostart default
+ virsh net-destroy default || true
+ virsh net-undefine default || true
+ virsh net-define /opt/virbr-vm.xml
+ virsh net-start default; virsh net-autostart default
+ EOF
+ done
diff --git a/.github/actions/test/integration/setup/hv-sysext.xml b/.github/actions/test/integration/setup/hv-sysext.xml
new file mode 100644
index 00000000..6c45562a
--- /dev/null
+++ b/.github/actions/test/integration/setup/hv-sysext.xml
@@ -0,0 +1,42 @@
+
+ HV_NAME_GOES_HERE
+ 6
+ 4
+
+ hvm
+ KERNEL_GOES_HERE
+ root=PARTUUID=ROOT_UUID_GOES_HERE console=ttyS0 systemd.gpt_auto=0 audit=0 rw
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/.github/actions/test/integration/test/cloudhypervisor/action.yml b/.github/actions/test/integration/test/cloudhypervisor/action.yml
index 078de40e..27fb4c6c 100644
--- a/.github/actions/test/integration/test/cloudhypervisor/action.yml
+++ b/.github/actions/test/integration/test/cloudhypervisor/action.yml
@@ -12,6 +12,11 @@ runs:
IP_HV1="192.168.122.2"
IP_HV2="192.168.122.3"
+ # The qemu scenario's migrated VM still runs on HV2. Its nested I/O
+ # starves the staging copy: the destination HV froze ~2h into the
+ # ubuntu.raw transfer while idle HV1 takes the same copy in seconds.
+ ssh $SSH_OPTS "$USER@$IP_HV2" 'virsh destroy VM-QEMU >/dev/null 2>&1 || true; virsh undefine VM-QEMU >/dev/null 2>&1 || true' || true
+
scp $SSH_OPTS ./.github/actions/test/integration/test/cloudhypervisor/vm.xml "$USER@$IP_HV1:/opt/vm-chv.xml"
scp $SSH_OPTS /opt/ubuntu.raw "$USER@$IP_HV1:/var/lib/libvirt/images/ubuntu.raw"
scp $SSH_OPTS /opt/ubuntu-cloud-init-ds.iso "$USER@$IP_HV1:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
@@ -82,7 +87,7 @@ runs:
exit 1
fi
echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
- sleep 10
+ sleep 60
done
EOF
@@ -154,6 +159,6 @@ runs:
exit 1
fi
echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
- sleep 10
+ sleep 60
done
EOF
diff --git a/.github/actions/test/integration/test/cloudhypervisor/vm.xml b/.github/actions/test/integration/test/cloudhypervisor/vm.xml
index 68108ff0..a51e6593 100644
--- a/.github/actions/test/integration/test/cloudhypervisor/vm.xml
+++ b/.github/actions/test/integration/test/cloudhypervisor/vm.xml
@@ -1,7 +1,7 @@
VM-CHV
2
- 1
+ 2
hvm
diff --git a/.github/actions/test/integration/test/qemu/action.yml b/.github/actions/test/integration/test/qemu/action.yml
index e6887e51..0a264344 100644
--- a/.github/actions/test/integration/test/qemu/action.yml
+++ b/.github/actions/test/integration/test/qemu/action.yml
@@ -17,7 +17,9 @@ runs:
scp $SSH_OPTS /opt/ubuntu-cloud-init-ds.iso "$USER@$IP_HV1:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
# the cloud-init data is not migrated from source to destination, but needed for the VM to run at the destination
+ # the disk itself must already exist at the destination, because --copy-storage-all does not create it
scp $SSH_OPTS /opt/ubuntu-cloud-init-ds.iso "$USER@$IP_HV2:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
+ scp $SSH_OPTS /opt/ubuntu.qcow2 "$USER@$IP_HV2:/var/lib/libvirt/images/ubuntu.qcow2"
- name: Start VM on HyperVisor 1
shell: bash
@@ -80,7 +82,7 @@ runs:
exit 1
fi
echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
- sleep 10
+ sleep 60
done
EOF
@@ -152,6 +154,6 @@ runs:
exit 1
fi
echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
- sleep 10
+ sleep 60
done
EOF
diff --git a/.github/scripts/prepare-sysext-hv.sh b/.github/scripts/prepare-sysext-hv.sh
new file mode 100755
index 00000000..42b751e4
--- /dev/null
+++ b/.github/scripts/prepare-sysext-hv.sh
@@ -0,0 +1,315 @@
+#!/usr/bin/env bash
+# CI adapter for the sysext integration test: --render, --prepare, --check.
+# Only test adapter files and .build/ are touched.
+# Contract: docs/sysext-virt.md §11.
+set -Eeuo pipefail
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
+REPO_ROOT=$(cd -- "$SCRIPT_DIR/../.." && pwd -P)
+MKOSI_DIR=$REPO_ROOT/.build/mkosi
+MKOSI_VENV=$MKOSI_DIR/venv/bin/mkosi
+SANDBOX_DIR=$MKOSI_DIR/sandbox
+SANDBOX_SOURCES=$SANDBOX_DIR/etc/apt/sources.list.d/mkosi.sources
+TOOLS_DIR=$MKOSI_DIR/tools
+TOOLS_TREE=$TOOLS_DIR/mkosi.tools
+TOOLS_PIN_FILE=$TOOLS_DIR/pin
+TOOLS_PIN=ubuntu/noble
+TOOLS_MANIFEST=$TOOLS_DIR/mkosi.tools.manifest
+TOOLS_PIN_SOURCE=$REPO_ROOT/mkosi/tools.pin
+BASE_TREE=$MKOSI_DIR/base-image/current
+BASE_KERNEL=$MKOSI_DIR/base-image/current-kernel
+TEMPLATE=$REPO_ROOT/.github/actions/test/integration/setup/hv-sysext.xml
+ROOT_GUID=4f68bce3-e8cd-4db1-96e7-fbcaf984b709
+HV_DISK_SIZE=${HV_DISK_SIZE:-8G}
+HV_ROOT_SIZE=${HV_ROOT_SIZE:-7G}
+
+die() { printf 'prepare-sysext-hv: error: %s\n' "$*" >&2; exit 2; }
+
+usage() {
+ cat <<'EOF'
+usage: prepare-sysext-hv.sh --render
+ prepare-sysext-hv.sh --prepare
+ prepare-sysext-hv.sh --check (preflight: tools tree, sandbox, base cache)
+Env: HV_DISK_SIZE=4G HV_ROOT_SIZE=3G SCI_HV_NETWORK=default
+EOF
+}
+
+render() {
+ [ "$#" -eq 4 ] || { usage >&2; exit 2; }
+ [ -f "$TEMPLATE" ] || die "missing XML template: $TEMPLATE"
+ python3 - "$TEMPLATE" "$1" "$2" "$3" "$4" <<'PY'
+import json, os, re, sys, xml.sax.saxutils as sax
+TEMPLATE, TABLE, OUT, KERNEL, PREFIX = sys.argv[1:6]
+GUID = "4f68bce3-e8cd-4db1-96e7-fbcaf984b709"
+UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
+MACS = {"HV1": "52:54:00:12:34:56", "HV2": "52:54:00:65:43:21"}
+def fail(msg): sys.exit(f"prepare-sysext-hv: error: {msg}")
+def esc(value): return sax.escape(str(value), {'"': """, "'": "'"})
+try:
+ parts = json.load(open(TABLE, encoding="utf-8"))["partitiontable"]["partitions"]
+except (OSError, ValueError, KeyError, TypeError) as exc:
+ fail(f"invalid JSON partition table {TABLE}: {exc}")
+roots = [p for p in parts if isinstance(p, dict)
+ and str(p.get("type", "")).lower() == GUID]
+if len(roots) != 1:
+ fail(f"expected exactly one root partition (GPT type {GUID}), found {len(roots)}")
+root = roots[0]
+uuid = str(root.get("uuid", "")).strip()
+if not UUID_RE.fullmatch(uuid):
+ fail(f"root partition uuid is not a valid UUID: {uuid!r}")
+network = os.environ.get("SCI_HV_NETWORK") or "default"
+if not network.strip() or "\n" in network:
+ fail(f"invalid network name: {network!r}")
+template = open(TEMPLATE, encoding="utf-8").read()
+os.makedirs(OUT, exist_ok=True)
+for hv, mac in MACS.items():
+ text = template
+ for key, value in {"HV_NAME_GOES_HERE": hv, "MAC_GOES_HERE": mac,
+ "KERNEL_GOES_HERE": esc(KERNEL), "ROOT_UUID_GOES_HERE": esc(uuid),
+ "DISK_PREFIX_GOES_HERE": esc(PREFIX), "NETWORK_GOES_HERE": esc(network)}.items():
+ text = text.replace(key, value)
+ open(os.path.join(OUT, f"{hv}.xml"), "w", encoding="utf-8").write(text)
+print(f"prepare-sysext-hv: rendered HV1, HV2 in {OUT}")
+PY
+}
+
+preflight() {
+ [ "$#" -eq 0 ] || { usage >&2; exit 2; }
+ local -a bad=()
+ need() { [ -e "$1" ] || bad+=("$1 ($2)"); }
+ need_x() { [ -x "$1" ] || bad+=("$1 ($2; must be executable)"); }
+ need_x "$REPO_ROOT/bin/garden-version" "gardenlinux submodule; run: git submodule update --init gardenlinux"
+ need_x "$MKOSI_VENV" "mkosi 27.1 venv; create it and install the pinned mkosi (see the bootstrap step of .github/workflows/test.yml)"
+ need "$SANDBOX_DIR" "assembled sandbox; run: bash mkosi/build.sh --bootstrap"
+ need "$SANDBOX_SOURCES" "assembled sandbox apt source; run: bash mkosi/build.sh --bootstrap"
+ need "$TOOLS_TREE" "pinned tools tree; run: bash mkosi/build.sh --bootstrap"
+ need_x "$TOOLS_TREE/usr/bin/systemd-repart" "pinned tools tree; run: bash mkosi/build.sh --bootstrap"
+ need "$TOOLS_MANIFEST" "tools-tree manifest; run: bash mkosi/build.sh --bootstrap"
+ need "$BASE_TREE" "published base tree; run: bash mkosi/base-image.sh ensure"
+ need "$BASE_KERNEL" "published base kernel; run: bash mkosi/base-image.sh ensure"
+ if [ -x "$MKOSI_VENV" ] \
+ && ! "$MKOSI_VENV" --version 2>/dev/null | grep -qx 'mkosi 27.1'; then
+ bad+=("$MKOSI_VENV (must report 'mkosi 27.1')")
+ fi
+ if [ -f "$TOOLS_MANIFEST" ]; then
+ local pin
+ pin=$(grep -vE '^[[:space:]]*(#|$)' -- "$TOOLS_PIN_SOURCE" | head -n 1 || true)
+ pin=${pin//[[:space:]]/}
+ if [[ ! $pin =~ ^[0-9a-f]{64}$ ]]; then
+ bad+=("$TOOLS_PIN_SOURCE (expected exactly one sha256)")
+ elif [ "$(sha256sum -- "$TOOLS_MANIFEST" | awk '{print $1}')" != "$pin" ]; then
+ bad+=("$TOOLS_MANIFEST (sha256 does not match $TOOLS_PIN_SOURCE)")
+ fi
+ fi
+ if [ -f "$SANDBOX_SOURCES" ]; then
+ local suite sources_suite
+ suite=$(sed -n 's/^Release=//p' -- "$REPO_ROOT/mkosi/mkosi.conf" | head -n 1)
+ sources_suite=$(sed -n 's/^Suites:[[:space:]]*//p' -- "$SANDBOX_SOURCES" | head -n 1)
+ [ "$sources_suite" = "$suite" ] \
+ || bad+=("$SANDBOX_SOURCES (Suites: ${sources_suite:-none} != Release=$suite)")
+ fi
+ [ "$(cat "$TOOLS_PIN_FILE" 2>/dev/null || true)" = "$TOOLS_PIN" ] \
+ || bad+=("$TOOLS_PIN_FILE (must contain '$TOOLS_PIN')")
+ if [ "${#bad[@]}" -gt 0 ]; then
+ printf 'prepare-sysext-hv: preflight failed; missing or invalid inputs:\n' >&2
+ printf ' %s\n' "${bad[@]}" >&2
+ cat >&2 <<'EOF'
+Bootstrap the toolchain on this checkout (no images are built):
+
+ bash mkosi/build.sh --bootstrap
+
+It assembles the sandbox, bootstraps the pinned tools tree (verified against
+mkosi/tools.pin) and ensures the published base tree. The mkosi venv is a
+precondition (create it with python3 -m venv and install the pinned mkosi;
+see the bootstrap step of .github/workflows/test.yml). It must provide:
+ .build/mkosi/venv mkosi 27.1
+ .build/mkosi/sandbox assembled by mkosi/build.sh
+ .build/mkosi/tools/mkosi.tools pinned tools tree
+ .build/mkosi/tools/pin ubuntu/noble
+ .build/mkosi/tools/mkosi.tools.manifest sha256 must match mkosi/tools.pin
+ .build/mkosi/base-image/current published base tree
+ .build/mkosi/base-image/current-kernel published base kernel
+EOF
+ exit 2
+ fi
+ printf 'prepare-sysext-hv: preflight ok (mkosi 27.1, tools pin %s)\n' "$TOOLS_PIN"
+}
+
+find_one() { #
+ local count
+ count=$(find "$1" -maxdepth 1 -type f -name "*.$2" | wc -l)
+ [ "$count" -eq 1 ] || die "artifact directory $1 must contain exactly one *.$2 (found $count)"
+ find "$1" -maxdepth 1 -type f -name "*.$2" | head -n 1
+}
+
+prepare() {
+ [ "$#" -eq 2 ] || { usage >&2; exit 2; }
+ local artifact_dir=$1 out=$2
+ [ -d "$artifact_dir" ] || die "artifact directory not found: $artifact_dir"
+ [ -n "$out" ] && [ "$out" != / ] || die "refusing to use '$out' as the output directory"
+ preflight
+
+ local cmd
+ for cmd in qemu-img sfdisk e2fsck resize2fs ssh-keygen; do command -v "$cmd" >/dev/null 2>&1 || die "$cmd is required (dependencies action)"; done
+
+ local artifact_tar artifact_ref
+ artifact_tar=$(find_one "$artifact_dir" tar)
+ artifact_ref=$(find_one "$artifact_dir" ref)
+ [ -s "$artifact_tar" ] || die "empty delivery container archive: $artifact_tar"
+ [ -s "$artifact_ref" ] || die "empty delivery container reference: $artifact_ref"
+ artifact_ref=$(head -n 1 "$artifact_ref")
+ [ -n "$artifact_ref" ] || die "delivery container reference is empty"
+ tar -tf "$artifact_tar" >/dev/null 2>&1 || die "not a readable tar: $artifact_tar"
+
+ local suite epoch
+ suite=$(sed -n 's/^Release=//p' -- "$REPO_ROOT/mkosi/mkosi.conf" | head -n 1)
+ [ -n "$suite" ] || die "no Release= in mkosi/mkosi.conf"
+ epoch=$("$REPO_ROOT/bin/garden-version" --epoch "$suite") \
+ || die "cannot derive the epoch for suite $suite"
+
+ local out_parent
+ out_parent=$(dirname -- "$out")
+ mkdir -p -- "$out_parent"
+ out=$(cd -- "$out_parent" && pwd -P)/$(basename -- "$out")
+ mkdir -p -- "$out/overlay/etc/ssh/sshd_config.d" \
+ "$out/overlay/etc/systemd/system/multi-user.target.wants" \
+ "$out/overlay/etc/systemd/system-preset" \
+ "$out/overlay/etc/cloud" "$out/overlay/usr/lib/verity.d" \
+ "$out/mkosi" "$out/repart.d" "$out/disks"
+ printf '%s\n' "$artifact_ref" >"$out/sci-delivery.ref"
+
+ local key=$out/hv-test-key
+ [ -f "$key" ] || ssh-keygen -q -t ed25519 -N '' -C 'sci-sysext-ci' -f "$key" \
+ || die "cannot generate the test SSH key"
+ chmod 600 "$key"
+ local pubkey
+ pubkey=$(cat "$key.pub")
+
+ # Test-environment overlay only; the published base stays untouched.
+ printf '%s\n' "$pubkey" >"$out/overlay/etc/ssh/authorized_keys"
+ chmod 600 "$out/overlay/etc/ssh/authorized_keys"
+ printf 'PasswordAuthentication no\nUsePAM no\n' \
+ >"$out/overlay/etc/ssh/sshd_config.d/10-sci-ci.conf"
+ # sshd uses the first value; patch the main config, not a drop-in.
+ sed 's/^PermitRootLogin .*/PermitRootLogin yes/' \
+ "$BASE_TREE/etc/ssh/sshd_config" >"$out/overlay/etc/ssh/sshd_config"
+ grep -q '^AuthorizedKeysFile' "$out/overlay/etc/ssh/sshd_config" \
+ || printf '\nAuthorizedKeysFile /etc/ssh/authorized_keys\n' \
+ >>"$out/overlay/etc/ssh/sshd_config"
+ # Presets: first match wins, so 00-* must sort before the base 00-*-disable.
+ printf 'enable ssh.service\ndisable ssh.socket\nenable containerd.service\n' \
+ >"$out/overlay/etc/systemd/system-preset/00-00-sci-ci-enable.preset"
+ ln -sfn /usr/lib/systemd/system/ssh.service \
+ "$out/overlay/etc/systemd/system/multi-user.target.wants/ssh.service"
+ ln -sfn /usr/lib/systemd/system/containerd.service \
+ "$out/overlay/etc/systemd/system/multi-user.target.wants/containerd.service"
+ : >"$out/overlay/etc/cloud/cloud-init.disabled"
+ ln -sfn /dev/null "$out/overlay/etc/systemd/system/tgt.service"
+ cp -- "$REPO_ROOT/mkosi/verity.crt" "$out/overlay/usr/lib/verity.d/verity.crt"
+ # libvirt derives the p2p migration address from the destination
+ # hostname; the guests have no upstream DNS, so pin the names.
+ cp -- "$BASE_TREE/etc/hosts" "$out/overlay/etc/hosts"
+ printf '192.168.122.2 hv1\n192.168.122.3 hv2\n' >>"$out/overlay/etc/hosts"
+
+ cat >"$out/mkosi/mkosi.conf" <"$out/build.log" 2>&1 || {
+ tail -n 60 -- "$out/build.log" >&2 || true
+ die "mkosi build failed; see $out/build.log"
+ }
+ local raw=$out/image/sysext-hv-base.raw
+ [ -f "$raw" ] || die "mkosi build produced no $raw"
+
+ # Grow GPT and ext4 offline; no loop device or root privilege needed.
+ printf '[Partition]\nType=root-x86-64\nSizeMinBytes=%s\nSizeMaxBytes=%s\n' \
+ "$HV_ROOT_SIZE" "$HV_ROOT_SIZE" >"$out/repart.d/80-root.conf"
+ # The tools-tree repart finds its private libsystemd-shared only through
+ # an absolute host RUNPATH; point the loader back into the tools tree.
+ LD_LIBRARY_PATH="$TOOLS_TREE/usr/lib/x86_64-linux-gnu/systemd" \
+ "$TOOLS_TREE/usr/bin/systemd-repart" --offline=yes --size="$HV_DISK_SIZE" \
+ --dry-run=no --definitions="$out/repart.d" "$raw" >>"$out/build.log" 2>&1 || {
+ tail -n 60 -- "$out/build.log" >&2 || true
+ die "cannot grow the HV root image to $HV_DISK_SIZE"
+ }
+
+ local table start size uuid name number
+ table=$(python3 - "$raw" "$ROOT_GUID" <<'PY'
+import json, subprocess, sys
+raw, guid = sys.argv[1], sys.argv[2].lower()
+parts = json.loads(subprocess.run(["sfdisk", "--json", raw], check=True,
+ capture_output=True, text=True).stdout)["partitiontable"]["partitions"]
+roots = [p for p in parts if p["type"].lower() == guid]
+if len(roots) != 1:
+ raise SystemExit(f"expected one root partition, found {len(roots)}")
+p = roots[0]
+print(p["start"] * 512, p["size"] * 512, p["uuid"].lower(),
+ p.get("name", "ROOT"), p.get("number", 1))
+PY
+ ) || die "cannot read the grown root partition"
+ read -r start size uuid name number <<<"$table"
+
+ local part=$out/rootfs.part
+ dd if="$raw" of="$part" bs=1M iflag=skip_bytes,count_bytes skip="$start" \
+ count="$size" conv=sparse status=none || die "cannot extract the root filesystem"
+ e2fsck -f -y "$part" >/dev/null 2>&1 || true
+ resize2fs "$part" >>"$out/build.log" 2>&1 || {
+ tail -n 60 -- "$out/build.log" >&2 || true
+ die "cannot grow the root filesystem"
+ }
+ dd if="$part" of="$raw" bs=1M oflag=seek_bytes conv=sparse,notrunc \
+ seek="$start" status=none || die "cannot write the root filesystem back"
+ rm -f -- "$part"
+
+ cp -- "$BASE_KERNEL" "$out/vmlinuz"
+ printf '{"partitiontable":{"label":"gpt","device":"/dev/nvme0n1","partitions":[{"number":%s,"type":"%s","name":"%s","uuid":"%s"}]}}\n' \
+ "$number" "$ROOT_GUID" "$name" "$uuid" >"$out/partition-table.json"
+ "$SCRIPT_DIR/prepare-sysext-hv.sh" --render "$out/partition-table.json" \
+ "$out" "$out/vmlinuz" "$out/disks"
+ rm -f -- "$out/partition-table.json"
+
+ local hv
+ for hv in HV1 HV2; do
+ qemu-img convert -O qcow2 "$raw" "$out/disks/$hv.qcow2" \
+ || die "cannot convert the HV root image to $hv.qcow2"
+ # The integration libvirt runs qemu unprivileged with
+ # security_driver=none: no chowning, so the disk carries raw access.
+ chmod 666 "$out/disks/$hv.qcow2"
+ done
+
+ cat >"$out/metadata.env" <&2; exit 2 ;;
+esac
diff --git a/.github/workflows/sysext.yml b/.github/workflows/sysext.yml
new file mode 100644
index 00000000..a4119e15
--- /dev/null
+++ b/.github/workflows/sysext.yml
@@ -0,0 +1,207 @@
+name: sysext
+on:
+ push:
+ branches:
+ - sysext-virt
+ - sysext-*
+ pull_request:
+ paths-ignore:
+ - "**/README.md"
+ - "docs/**"
+ workflow_dispatch:
+permissions:
+ contents: read
+jobs:
+ quick-suite:
+ name: Quick extension suite
+ runs-on: ubuntu-24.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ submodules: recursive
+ - name: Run the quick extension suite
+ shell: bash
+ run: |
+ python3 -m venv .build/venv-test
+ .build/venv-test/bin/pip install --quiet pytest==8.4.2 PyYAML==6.0.3
+ .build/venv-test/bin/python -m pytest extensions/tests/ -q -m 'not vm and not build'
+ producer:
+ name: Build the sysext delivery artifacts
+ needs: quick-suite
+ if: github.event_name != 'pull_request'
+ runs-on: ubuntu-24.04
+ steps:
+ - name: Checkout
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ submodules: recursive
+ - name: Provide the signing key
+ shell: bash
+ env:
+ SYSEXT_UNHARDENED_SIGNING_KEY: ${{ secrets.SYSEXT_UNHARDENED_SIGNING_KEY }}
+ run: |
+ install -d -m 0700 .build/keys
+ if [ -n "${SYSEXT_UNHARDENED_SIGNING_KEY:-}" ]; then
+ printf '%s\n' "$SYSEXT_UNHARDENED_SIGNING_KEY" > .build/keys/sysext-verity.key
+ chmod 600 .build/keys/sysext-verity.key
+ openssl req -x509 -key .build/keys/sysext-verity.key -days 3650 \
+ -subj "/CN=sci-sysext-unhardened" \
+ -out mkosi/verity.crt
+ echo "using the repository signing key"
+ else
+ openssl req -x509 -newkey rsa:3072 -nodes -days 1 \
+ -subj "/CN=sci-sysext-run" \
+ -keyout .build/keys/sysext-verity.key \
+ -out mkosi/verity.crt
+ chmod 600 .build/keys/sysext-verity.key
+ echo "no repository signing key; using a run-local key"
+ fi
+ - name: Create the mkosi venv
+ shell: bash
+ run: |
+ python3 -m venv .build/mkosi/venv
+ .build/mkosi/venv/bin/pip install --quiet "mkosi @ git+https://github.com/systemd/mkosi.git@v27.1"
+ - name: Ensure the published base tree
+ shell: bash
+ run: bash mkosi/base-image.sh ensure
+ # mkosi's sandbox builds unprivileged through user namespaces; the
+ # runner's AppArmor blocks them by default.
+ - name: Relax the runner's user-namespace restriction
+ shell: bash
+ run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+ - name: Build the extensions
+ shell: bash
+ run: bash mkosi/build.sh
+ - name: Pull the pinned builder image
+ shell: bash
+ run: docker pull ghcr.io/gardenlinux/builder:f7cf91087f7580c6c3aefb26af6cee7df89a9938
+ - name: Build the guest runtime archive
+ shell: bash
+ run: |
+ install -d .build/sysext-runtime/build \
+ .build/sysext-runtime/work/tmp \
+ .build/sysext-runtime/work/home \
+ .build/sysext-runtime/work/pip-cache
+ docker build -q -t sci-sysext-runtime-builder -f - .build/sysext-runtime/build <<'EOF'
+ FROM debian:stable
+ RUN apt-get update \
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates curl
+ EOF
+ docker run --rm \
+ -v "$PWD/gardenlinux/tests/util/build_runtime.sh:/init:ro" \
+ -v "$PWD/gardenlinux/tests/util:/src:ro" \
+ -v "$PWD/.build/sysext-runtime:/work" \
+ -w /work --user "$(id -u):$(id -g)" \
+ -e HOME=/work/work/home -e TMPDIR=/work/work/tmp \
+ -e PIP_CACHE_DIR=/work/work/pip-cache \
+ sci-sysext-runtime-builder \
+ /init /src/requirements.txt /src/python.env.sh /work/runtime.tar.gz
+ - name: Create the test venv
+ shell: bash
+ run: |
+ python3 -m venv .build/venv-test
+ .build/venv-test/bin/pip install --quiet pytest==8.4.2 PyYAML==6.0.3
+ - name: Install the VM boot prerequisites
+ shell: bash
+ run: |
+ if command -v sudo >/dev/null 2>&1; then SUDO=sudo; else SUDO=; fi
+ $SUDO apt-get update
+ $SUDO apt-get install -y qemu-system-x86 fdisk
+ # The VM boot test needs /dev/kvm as the runner user.
+ if [ -e /dev/kvm ]; then $SUDO chmod 666 /dev/kvm; fi
+ - name: Run the full extension suite
+ shell: bash
+ run: .build/venv-test/bin/python -m pytest extensions/tests/ -q
+ # The raws artifact: the staging as plain files, no container runtime
+ # needed in the integration test.
+ - name: Stage the raws for download
+ shell: bash
+ run: |
+ STAGING_OUT=.build/delivery/staging-raws bash delivery/build-image.sh --stage-only
+ cp mkosi/verity.crt .build/delivery/staging-raws/verity-run.crt
+ - name: Upload the raws artifact
+ uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+ with:
+ name: sysext-raws
+ path: .build/delivery/staging-raws/
+ retention-days: 7
+ if-no-files-found: error
+ integration:
+ name: Hypervisor integration tests
+ needs: producer
+ if: github.event_name != 'pull_request'
+ permissions:
+ contents: read
+ actions: read
+ uses: ./.github/workflows/test.yml
+ with:
+ sysext_artifact: sysext-raws
+ publish:
+ name: Publish the signed release
+ needs: producer
+ if: github.event_name != 'pull_request'
+ runs-on: ubuntu-24.04
+ permissions:
+ contents: write
+ steps:
+ - name: Checkout
+ uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ with:
+ submodules: recursive
+ - name: Download the sysext raws artifact
+ uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+ with:
+ name: sysext-raws
+ path: .build/ci-sysext-hv/publish
+ - name: Check the signing secret
+ id: key
+ shell: bash
+ env:
+ SYSEXT_UNHARDENED_SIGNING_KEY: ${{ secrets.SYSEXT_UNHARDENED_SIGNING_KEY }}
+ run: |
+ if [ -n "${SYSEXT_UNHARDENED_SIGNING_KEY:-}" ]; then
+ echo "present=true" >> "$GITHUB_OUTPUT"
+ else
+ echo "present=false" >> "$GITHUB_OUTPUT"
+ echo "::notice::no SYSEXT_UNHARDENED_SIGNING_KEY configured; skipping the release"
+ fi
+ - name: Regenerate the published inventory
+ if: steps.key.outputs.present == 'true'
+ shell: bash
+ working-directory: .build/ci-sysext-hv/publish
+ run: |
+ # The release carries data, not the boot-test vehicle.
+ rm -f sci-delivery.sh
+ test ! -e sci-delivery.sh || { echo "error: the published set still carries sci-delivery.sh" >&2; exit 1; }
+ find . -maxdepth 1 -type f ! -name SHA256SUMS ! -name 'SHA256SUMS.sig' -printf '%f\n' \
+ | LC_ALL=C sort | xargs sha256sum > SHA256SUMS
+ sha256sum -c -- SHA256SUMS
+ - name: Sign the published inventory
+ if: steps.key.outputs.present == 'true'
+ shell: bash
+ env:
+ SYSEXT_UNHARDENED_SIGNING_KEY: ${{ secrets.SYSEXT_UNHARDENED_SIGNING_KEY }}
+ run: |
+ signing_key="$RUNNER_TEMP/sysext-signing.key"
+ install -m 0600 /dev/null "$signing_key"
+ trap 'rm -f "$signing_key"' EXIT
+ printf '%s\n' "$SYSEXT_UNHARDENED_SIGNING_KEY" > "$signing_key"
+ openssl dgst -sha256 -sign "$signing_key" \
+ -out .build/ci-sysext-hv/publish/SHA256SUMS.sig \
+ .build/ci-sysext-hv/publish/SHA256SUMS
+ rm -f "$signing_key"
+ - name: Publish the release
+ if: steps.key.outputs.present == 'true'
+ shell: bash
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ suite=$(sed -n 's/^Release=//p' mkosi/mkosi.conf | head -n 1)
+ tag="${suite}-g${GITHUB_SHA:0:7}"
+ gh release view "$tag" >/dev/null 2>&1 \
+ || gh release create "$tag" --target "$GITHUB_SHA" \
+ --title "$tag" --notes "SCI sysext artifacts for $tag"
+ gh release upload "$tag" .build/ci-sysext-hv/publish/* --clobber
+ gh release view "$tag" --json assets --jq '.assets[].name' | grep -qx 'SHA256SUMS.sig'
+ echo "published $tag: $(gh release view "$tag" --json url --jq .url)"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 372046d5..2072ae00 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -3,15 +3,35 @@ on:
workflow_dispatch:
inputs:
image_tag:
- description: "Image tag to test (must be usi-sci)"
+ description: "Image tag to test (must be usi-sci); mutually exclusive with sysext_artifact"
type: string
- required: true
+ required: false
+ sysext_artifact:
+ description: "Raws actions artifact (signed extensions, manifest, SHA256SUMS, run cert); see artifact_run_id"
+ type: string
+ required: false
+ artifact_run_id:
+ description: "Run ID in this repository that produced sysext_artifact (default: this run)"
+ type: string
+ required: false
workflow_call:
inputs:
image_tag:
- description: "Image tag to test (must be usi-sci)"
+ description: "Image tag to test (must be usi-sci); mutually exclusive with sysext_artifact"
+ type: string
+ required: false
+ sysext_artifact:
+ description: "Raws actions artifact (signed extensions, manifest, SHA256SUMS, run cert); see artifact_run_id"
+ type: string
+ required: false
+ artifact_run_id:
+ description: "Run ID in this repository that produced sysext_artifact (default: this run)"
type: string
- required: true
+ required: false
+
+permissions:
+ contents: read
+ actions: read
jobs:
test:
@@ -19,11 +39,41 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+ - name: Validate exactly one test input
+ shell: bash
+ env:
+ IMAGE_TAG: ${{ inputs.image_tag }}
+ SYSEXT_ARTIFACT: ${{ inputs.sysext_artifact }}
+ ARTIFACT_RUN_ID: ${{ inputs.artifact_run_id }}
+ run: |
+ if [ -n "$IMAGE_TAG" ] && [ -n "$SYSEXT_ARTIFACT" ]; then
+ echo "Error: image_tag and sysext_artifact are mutually exclusive" >&2
+ exit 1
+ fi
+ if [ -z "$IMAGE_TAG" ] && [ -z "$SYSEXT_ARTIFACT" ]; then
+ echo "Error: exactly one of image_tag or sysext_artifact is required" >&2
+ exit 1
+ fi
+ if [ -n "$ARTIFACT_RUN_ID" ] && ! [[ "$ARTIFACT_RUN_ID" =~ ^[1-9][0-9]*$ ]]; then
+ echo "Error: artifact_run_id must be a positive integer" >&2
+ exit 1
+ fi
+
+ - name: Check out the gardenlinux submodule
+ if: ${{ inputs.sysext_artifact != '' }}
+ shell: bash
+ run: git submodule update --init gardenlinux
+
+ # ---- legacy image_tag branch (unchanged default) --------------------
- name: Install oras
+ if: ${{ inputs.image_tag != '' }}
uses: oras-project/setup-oras@5e2f2482b30b0cb3e14e67f89a80c88c659a7c63 # v2
with:
version: 1.2.2
+
- name: Validate image tag
+ if: ${{ inputs.image_tag != '' }}
run: |
if [ -z "${{ inputs.image_tag }}" ]; then
echo "Error: image_tag is required"
@@ -39,12 +89,76 @@ jobs:
echo "Image tag present in GHCR (digest: ${digest:0:12}...)"
- name: Install Dependencies
uses: ./.github/actions/test/integration/dependencies
+
+ # ---- sysext branch --------------------------------------------------
+ - name: Download the sysext raws artifact
+ if: ${{ inputs.sysext_artifact != '' }}
+ uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
+ with:
+ name: ${{ inputs.sysext_artifact }}
+ path: .build/ci-sysext-hv/raws
+ run-id: ${{ inputs.artifact_run_id || github.run_id }}
+ github-token: ${{ github.token }}
+
+ - name: Restore the run signing certificate
+ if: ${{ inputs.sysext_artifact != '' }}
+ shell: bash
+ run: |
+ if [ -f .build/ci-sysext-hv/raws/verity-run.crt ]; then
+ cp .build/ci-sysext-hv/raws/verity-run.crt mkosi/verity.crt
+ echo "restored the producer's run-local signing certificate"
+ else
+ echo "no run certificate in the raws artifact; keeping the committed certificate"
+ fi
+ # mkosi's sandbox builds the tools tree and the HV root image
+ # unprivileged through user namespaces; the runner's AppArmor blocks
+ # them by default.
+ - name: Relax the runner's user-namespace restriction
+ if: ${{ inputs.sysext_artifact != '' }}
+ shell: bash
+ run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+
+ - name: Bootstrap the toolchain
+ if: ${{ inputs.sysext_artifact != '' }}
+ shell: bash
+ run: |
+ python3 -m venv .build/mkosi/venv
+ .build/mkosi/venv/bin/pip install --quiet "mkosi @ git+https://github.com/systemd/mkosi.git@v27.1"
+ bash mkosi/build.sh --bootstrap
+
+ - name: Preflight the accepted toolchain
+ if: ${{ inputs.sysext_artifact != '' }}
+ shell: bash
+ run: bash ./.github/scripts/prepare-sysext-hv.sh --check
+
+ - name: Rebuild the delivery image from the raws
+ if: ${{ inputs.sysext_artifact != '' }}
+ shell: bash
+ run: |
+ bash delivery/build-image.sh --from-staging .build/ci-sysext-hv/raws
+ mkdir -p .build/ci-sysext-hv/artifact
+ docker save sci-delivery:latest -o .build/ci-sysext-hv/artifact/sci-delivery.tar
+ printf 'docker.io/library/sci-delivery:latest\n' > .build/ci-sysext-hv/artifact/sci-delivery.ref
+
+ - name: Prepare the sysext HyperVisor base
+ if: ${{ inputs.sysext_artifact != '' }}
+ shell: bash
+ run: |
+ bash ./.github/scripts/prepare-sysext-hv.sh \
+ --prepare .build/ci-sysext-hv/artifact .build/ci-sysext-hv/out
+ cat .build/ci-sysext-hv/out/metadata.env >> "$GITHUB_ENV"
+
- name: Build
uses: ./.github/actions/test/integration/build
with:
+ mode: ${{ inputs.sysext_artifact != '' && 'sysext' || 'legacy' }}
image_tag: ${{ inputs.image_tag }}
+ sysext_key: ${{ env.HV_SSH_KEY }}
- name: Setup
uses: ./.github/actions/test/integration/setup
+ with:
+ mode: ${{ inputs.sysext_artifact != '' && 'sysext' || 'legacy' }}
+ hv_dir: ${{ env.HV_DIR }}
- name: Test QEMU
uses: ./.github/actions/test/integration/test/qemu
- name: Test CloudHypervisor
diff --git a/.gitignore b/.gitignore
index 19fd6f02..0254c18d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,7 @@
.envrc
.venv
shell.nix
+__pycache__/
+.pytest_cache/
+docs/sysext-handover.md
+.actrc
diff --git a/delivery/Containerfile b/delivery/Containerfile
new file mode 100644
index 00000000..66539e53
--- /dev/null
+++ b/delivery/Containerfile
@@ -0,0 +1,19 @@
+# SCI delivery image; the staging directory is the build context.
+# FROM is pinned by digest: resolve with `docker buildx imagetools inspect`.
+FROM docker.io/library/debian@sha256:62bd423aeee912639ca8ee681c2b762cb3e25137e0a92b20be455898a9f034ad
+
+ENV HOST_ROOT=/host \
+ STAGING_DIR=/opt/sci \
+ INTERVAL=60 \
+ NSENTER=1
+
+# Fetch mode downloads and verifies a signed build from the host network.
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends curl ca-certificates openssl \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY . /opt/sci/
+COPY sci-delivery.sh /usr/local/bin/sci-delivery.sh
+RUN chmod 0755 /usr/local/bin/sci-delivery.sh
+
+ENTRYPOINT ["/usr/local/bin/sci-delivery.sh"]
diff --git a/delivery/README.md b/delivery/README.md
new file mode 100644
index 00000000..59a3da4a
--- /dev/null
+++ b/delivery/README.md
@@ -0,0 +1,304 @@
+# SCI delivery step
+
+**Scope.** `sci-delivery.sh` is this repository's boot-test vehicle: the
+boot test and the integration job run it inside a Garden Linux VM to
+prove that the built set merges and activates. It is not a supported
+consumer agent. The supported output is the signed release (see "What
+is published"); how a consumer selects, stages
+and merges those bytes, provisions the verity anchor in the base image,
+and keeps a staging directory across restarts is the consumer's
+decision. Image mode and fetch mode are two ways to run the test
+vehicle, not two delivery products.
+
+## What the delivery step does
+
+The following is the pass as the boot test runs it; fetch mode is a
+variant of the same pass.
+
+`sci-delivery.sh` runs against a host root mounted at `HOST_ROOT` (default
+`/host`):
+
+* a manifest entry is ` ...`; the unit list may be empty,
+ `#` comments and empty lines are ignored, and the manifest is validated
+ completely before anything is changed. The manifest also carries the
+ `TARGET_BASE=` directive, and the pass is gated on the host's base
+ line (see "Base-version gate" below);
+* the staged raws are verified against `SHA256SUMS` right after manifest
+ validation and before the base-version gate or any deployment (see
+ "Staged-raw integrity" below);
+* a missing or content-different `.raw` is copied to
+ `/var/lib/extensions/` (write `.tmp`, then `mv`);
+* a missing or content-different `-activation.service` shim is copied
+ to `/etc/systemd/system/` the same way;
+* a `REMOVE=` directive makes the delivery step delete only what it
+ names: `/var/lib/extensions/.raw`,
+ `/etc/systemd/system/-activation.service` and its
+ `multi-user.target.wants` enable symlink, before the entries reconcile,
+ and never restarts that extension's units;
+* when anything changed, exactly one sequence runs: `systemctl
+ daemon-reload` → `systemd-sysext refresh` → `systemctl reload
+ dbus.service` → `systemctl restart ` for each extension whose raw
+ changed → `systemctl enable -activation.service` for every shim,
+ plus `systemctl restart -activation.service` for changed or new
+ entries;
+* a failing per-extension unit restart is tolerated: it is logged on
+ stderr with the extension name and the units, and the flow continues —
+ the remaining restarts and the whole shim loop still run, so one broken
+ unit (for example a service whose artifact and base version are skewed)
+ does not block the activation of healthy extensions. The pass then
+ exits non-zero, so the failure stays visible to the operator and in the
+ container status. Only the unit restarts are tolerant; the refresh trio and
+ the shim enable/restart calls stay fatal, because they are the
+ activation machinery itself. Service retries belong to systemd; the
+ delivery step owns deployment and activation;
+* when nothing changed, no host-mutating command runs at all;
+* crash-safe activation reconciliation:
+ `/var/lib/extensions/.sci-delivery-pending` is written
+ immediately before the activation flow
+ and removed only after the flow completed. A pass that starts with the
+ marker present replays the full activation flow for every entry even when
+ no file changed, so a pass killed after deploying files (SIGKILL, OOM,
+ eviction, a fatal activation call) cannot strand raw+shim files without
+ refresh, restarts and enablement. The marker is cleared whenever the flow
+ completes, even if an individual unit restart failed: a service-level
+ failure is not an activation stranding. The dot-file is not an
+ extension: it is not in the manifest and is never deployed or
+ md5-compared;
+* as container PID 1 the delivery step traps `TERM`/`INT` and exits immediately.
+ The interval wait uses a background `sleep` plus `wait`, so the trap is
+ not deferred until the sleep returns.
+
+Host-mutating commands go through `nsenter -t 1 -m --` (`NSENTER=1`, the
+default) so they run in PID 1's mount namespace; file operations use the
+`HOST_ROOT` paths directly. `INTERVAL` (default 60) is the seconds between
+passes; `INTERVAL=0` runs one pass and exits.
+
+## Staged-raw integrity
+
+The staging directory carries `SHA256SUMS` in standard `sha256sum` output:
+
+```
+<64-hex-digest> .raw
+```
+
+one line per manifest entry (extra lines are ignored; a staged raw without a
+manifest entry is never deployed). Right after manifest validation and
+**before** the base-version gate and any deployment, the delivery step
+
+* requires the file (a missing `SHA256SUMS` is fatal and names it),
+* requires a line for every manifest entry (a missing line is fatal and names
+ the entry), and
+* recomputes every staged raw's sha256 and compares it with its line (a
+ mismatch is fatal and names the raw).
+
+All three are pre-deployment failures: no raw, no shim, no pending marker and
+no `systemctl`/`systemd-sysext` call. Integrity runs before the gate on
+purpose — a corrupt staging set must be reported as such even on a host whose
+base line would also be refused (the raw-corrupt (b) variant pins that order).
+
+This check guards the content the delivery image ships. The pass's md5
+comparison is a different thing: it detects drift between an already-deployed
+host file and the staged copy, and triggers re-deployment. md5 is fine for
+that; the sha256 check is the integrity statement about the staged bytes.
+
+## Base-version gate
+
+The manifest grammar is v2: ` ...` is an entry, `#`
+comments and empty lines are ignored, and a line matching
+`^[A-Z][A-Z0-9_]*=$` is a **directive**, not an entry. Unknown
+directives are a validation failure (fail closed, so a typo cannot leave the
+gate unset). Two directives exist: `TARGET_BASE`, which is required,
+and `REMOVE`, the tombstone (see "The delivery manifest"). The gate
+reads:
+
+```
+TARGET_BASE=2150
+```
+
+`TARGET_BASE` is the major line of the Garden Linux release the extensions
+were built for — the major of `Release=` in `mkosi/mkosi.conf` (today
+`2150.9.0`). It sits in the header block of the manifest
+(`delivery/extensions.manifest`).
+Zero, duplicate, or non-digit values fail validation before anything is
+deployed, and `test_artifact_contracts::test_static_input_contract` keeps the
+committed manifest in sync with `Release=`.
+
+Why it exists: systemd-sysext binds an extension to the host by the `ID=` in
+its extension-release only, so a 2150 build merges on *any* Garden Linux
+line and silently mixes ABIs. The upstream `SYSEXT_LEVEL` proposal
+(nova-issues#909) would make systemd enforce this natively; until it lands
+the delivery step does. After manifest validation and the staged-raw integrity
+check, and before any file deployment, it
+
+* reads `/etc/os-release`,
+* takes the first dot-separated component of `IMAGE_VERSION` (fallback
+ `GARDENLINUX_VERSION`) as the host's major line, and
+* compares it with `TARGET_BASE`.
+
+On a match the pass proceeds exactly as described above. On a mismatch, a
+missing version field, or an unreadable `os-release`, the pass deploys
+nothing — no raw, no shim, no pending marker — makes no `systemctl`/
+`systemd-sysext` call, and exits non-zero with one `[sci-delivery] error:`
+line naming the host line and `TARGET_BASE` (plus `os-release` when that file
+is the problem).
+
+Failure semantics: a mismatched host is a **loud no-op**. The gate never
+deletes anything and runs on every pass, so a host that moved to another
+base line stays undeployed while the delivery step's log repeats the error on
+every pass — the signal to rebuild (or re-select) the delivery image for that
+line, not to widen the gate.
+
+## Build the test image
+
+Build the extension raws first with mkosi: `bash mkosi/build.sh` (no
+argument) builds every image in one invocation, which is what a delivery
+image needs; `bash mkosi/build.sh ` builds one image and its
+dependency closure and stays the iteration mode.
+`build-image.sh` stages every manifest entry from `.build/mkosi/output/`; a
+missing raw fails the build before docker runs. It also stages
+`SHA256SUMS` (one standard `sha256sum` line per manifest entry, manifest
+order) and verifies the staged file with `sha256sum -c` before the docker
+build. Source choice: mkosi's own checksum output is preferred when
+`/SHA256SUMS` covers every manifest raw **and** its hashes still
+match the copied raws (builder provenance); otherwise the script generates
+the file with `sha256sum` over the staged raws. mkosi's `Checksum=` is
+deliberately not enabled here: the setting has `local` scope in mkosi 27.1,
+so a root `mkosi.conf` value never reaches the sysext subimages, and each
+subimage would write a separate `.SHA256SUMS` in binary-mode format.
+
+The image is a test artifact, tagged only `$IMAGE_TAG` (default
+`sci-delivery:latest`). It is not published; the published product is
+the release below.
+
+```sh
+bash delivery/build-image.sh # validates, stages into .build/delivery/staging, builds the image
+docker images | grep sci-delivery # sci-delivery:latest
+```
+
+The Containerfile pins its base image by digest
+(`docker.io/library/debian@sha256:…`, resolved from `debian:trixie-slim`). To bump it:
+
+```sh
+docker buildx imagetools inspect docker.io/library/debian:trixie-slim
+```
+
+take the index `Digest:`, and update the `FROM` line in the same commit that
+records why. A tag would let the delivery step's base image move without
+review.
+
+`build-image.sh` fails loudly (before invoking docker) when a manifest entry
+has no raw, or when a staged shim is not listed in the manifest.
+
+## What is published
+
+The `publish` job creates one GitHub release per build, tagged
+`-g` (for example `2150.9.0-g1a2b3c4`). Without
+`SYSEXT_UNHARDENED_SIGNING_KEY` the release steps are skipped, so no
+release exists. The release carries:
+
+* every `.raw` named in `extensions.manifest`;
+* every activation shim the extensions ship;
+* `extensions.manifest`;
+* `verity-run.crt`, the public certificate that signed the raws;
+* `SHA256SUMS` and `SHA256SUMS.sig`.
+
+The release carries no delivery program or scripts: `sci-delivery.sh`
+stays in the `sysext-raws` test artifact, which the integration job uses
+to rebuild the image.
+
+## The delivery manifest
+
+`delivery/extensions.manifest` lists every extension the repo ships, in
+merge order. The boot test uses the full set; fetch mode deploys the
+release's manifest. The step never deploys a raw the manifest does not
+list.
+
+A tombstone is a directive, not an entry:
+
+```
+REMOVE=sci-51-virt-qemu
+```
+
+For each `REMOVE` name the delivery step deletes
+`/var/lib/extensions/.raw`,
+`/etc/systemd/system/-activation.service` and its
+`multi-user.target.wants` enable symlink, then runs the normal activation
+flow. Removal is idempotent: a name that is
+already absent is not a change. A name that is also an entry, a
+duplicate `REMOVE` name, or a name that does not match
+`[A-Za-z0-9][A-Za-z0-9._-]*` fails validation, so a tombstone cannot
+unmerge content the same manifest deploys, and it cannot reach outside
+the two deployment trees. Dropping an entry deletes nothing on the host;
+only a tombstone does.
+
+The `sci-51-virt-qemu` entry carries no restart units: the qemu driver
+plugs into the already-running `libvirtd`, and that service is restarted
+by `sci-40-libvirt`'s entry — this extension ships no units of its own.
+
+Build every raw first (`bash mkosi/build.sh`), then stage:
+
+```sh
+bash delivery/build-image.sh
+```
+
+`build-image.sh` stages the entries of the manifest and fails before
+docker runs when one of them has no raw.
+
+## Remote delivery (fetch mode, boot test)
+
+`FETCH_URL` switches the delivery step from image mode to fetch mode: it
+then downloads one build from that base URL, proves it, and reconciles
+it. This mode proves that a published release is consumable; it is not a
+supported deployment mechanism. A fetch pass does not trust the staging
+directory it finds.
+
+| Variable | Meaning |
+|---|---|
+| `FETCH_URL` | base URL of one build (unset selects image mode) |
+| `FETCH_PUBKEY` | PEM public key for `SHA256SUMS.sig` (required) |
+| `FETCH_SUMS_DIGEST` | pinned sha256 of `SHA256SUMS` (optional) |
+| `FETCH_DIR` | writable fetch root (default `/var/lib/sci-sysext`) |
+
+`STAGING_DIR` defaults to `$FETCH_DIR/staging` in fetch mode when the
+caller does not set it. Image mode keeps `/opt/sci`.
+
+A pass verifies `SHA256SUMS` and `SHA256SUMS.sig`, downloads only the
+files whose staged hash differs, proves the complete set with
+`sha256sum -c`, and prunes files the inventory no longer lists. Every
+failure exits non-zero and writes nothing to the host.
+
+A converged pass downloads the two manifest files only. That needs
+`STAGING_DIR` and `FETCH_DIR` on a directory that outlives the
+container; the shipped image keeps them in the container layer.
+
+## Safety properties
+
+* **Deletes only what a `REMOVE` directive names**: unlisted host
+ extensions in `/var/lib/extensions` are left alone, and the delivery step
+ only writes manifest entries. A `REMOVE` name loses its raw and its shim,
+ and nothing else. The only other file it removes is its own
+ `.sci-delivery-pending` marker.
+* **Idempotent**: unchanged files are not rewritten (no mtime change) and
+ an unchanged pass makes zero systemctl/systemd-sysext calls.
+* **All-or-nothing validation**: a missing manifest, a missing raw, an
+ unlisted shim, a missing `HOST_ROOT`, a non-numeric `INTERVAL`, a
+ missing/duplicate/non-digit `TARGET_BASE`, a missing `SHA256SUMS`, a
+ missing entry line, a staged raw that does not match its line, or a host
+ base line that does not match `TARGET_BASE` aborts before any host
+ mutation.
+* **Staged-content integrity**: every raw the delivery step deploys was
+ verified against the image's `SHA256SUMS` in that same pass, before any file
+ deployment; the check runs before the base-version gate so a corrupt
+ staging set is never masked by a host-line mismatch.
+* **Per-unit restart tolerance**: a failing `systemctl restart ` for
+ one extension is logged on stderr and the flow continues (all shims are
+ still enabled); the pass exits non-zero. The failure is visible instead
+ of silently ignored, but it does not strand other extensions' activation.
+* **Atomic writes**: each file is copied to `.tmp` and renamed.
+
+## Known limitation
+
+`EXTENSION_RESTART_UNITS=` in the extension-release files is inert on
+Garden Linux 2150.x (systemd 259 needs >= 262). The restart lists therefore
+live in `delivery/extensions.manifest` and must be kept in sync with the
+extensions' `EXTENSION_RESTART_UNITS=` metadata.
diff --git a/delivery/build-image.sh b/delivery/build-image.sh
new file mode 100755
index 00000000..19148547
--- /dev/null
+++ b/delivery/build-image.sh
@@ -0,0 +1,284 @@
+#!/usr/bin/env bash
+# Assemble the SCI delivery image from the built raws, shims, and manifest.
+# The image is a test artifact that carries only $IMAGE_TAG.
+# Usage: bash delivery/build-image.sh [--stage-only | --from-staging ] [--help]
+# Contract: delivery/README.md.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
+REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd -P)
+
+RAW_DIR=${RAW_DIR:-$REPO_ROOT/.build/mkosi/output}
+SHIM_GLOB=${SHIM_GLOB:-$REPO_ROOT/mkosi/mkosi.images/*/mkosi.extra/usr/share/sci/activation/*.service}
+STAGING_OUT=${STAGING_OUT:-$REPO_ROOT/.build/delivery/staging}
+CONTAINERFILE=${CONTAINERFILE:-$SCRIPT_DIR/Containerfile}
+DELIVERY_SCRIPT=${DELIVERY_SCRIPT:-$SCRIPT_DIR/sci-delivery.sh}
+MANIFEST=${MANIFEST:-$SCRIPT_DIR/extensions.manifest}
+IMAGE_TAG=${IMAGE_TAG:-sci-delivery:latest}
+
+die() {
+ printf 'build-image: error: %s\n' "$*" >&2
+ exit 1
+}
+
+log() {
+ printf 'build-image: %s\n' "$*"
+}
+
+usage() {
+ cat <<'EOF'
+usage: build-image.sh [--stage-only | --from-staging ] [--help]
+
+Assembles /.build/delivery/staging from the built extension images,
+the activation shim templates and delivery/extensions.manifest, validates it,
+stages SHA256SUMS over the staged raws, and runs `docker build` with the
+staging directory as the context. The image carries only $IMAGE_TAG. It is a
+test artifact, not a published product.
+
+--stage-only stops after the validated staging directory: no docker build and
+no image tags, for callers that take the staged files as plain files.
+
+--from-staging skips the assembly and builds the image from an
+already-staged directory (for example an unpacked raws artifact): the staged
+SHA256SUMS must verify the staged raws as shipped, and the image carries only
+$IMAGE_TAG.
+
+Environment overrides: RAW_DIR, SHIM_GLOB, STAGING_OUT, CONTAINERFILE,
+DELIVERY_SCRIPT, MANIFEST, IMAGE_TAG.
+EOF
+}
+
+declare -a NAMES=()
+
+load_manifest() {
+ [ -f "$MANIFEST" ] || die "missing manifest: $MANIFEST"
+
+ NAMES=()
+ local line name rest lineno=0
+ while IFS= read -r line || [ -n "$line" ]; do
+ lineno=$((lineno + 1))
+ line=${line%$'\r'}
+ if [ -z "$line" ]; then
+ continue
+ fi
+ if [[ $line =~ ^[[:space:]]*$ ]]; then
+ die "malformed manifest line $lineno (whitespace-only): $MANIFEST"
+ fi
+ if [[ $line =~ ^[[:space:]]*# ]]; then
+ continue
+ fi
+ if [[ $line =~ ^[A-Z][A-Z0-9_]*=[^[:space:]]*$ ]]; then
+ continue
+ fi
+ read -r name rest <<<"$line"
+ if [[ ! $name =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
+ die "malformed manifest line $lineno (invalid name '$name'): $MANIFEST"
+ fi
+ NAMES+=("$name")
+ done <"$MANIFEST"
+
+ [ "${#NAMES[@]}" -gt 0 ] || die "manifest has no entries: $MANIFEST"
+}
+
+shim_templates() {
+ local shim
+ while IFS= read -r shim; do
+ [ -n "$shim" ] || continue
+ printf '%s\n' "$shim"
+ done < <(compgen -G "$SHIM_GLOB" || true)
+}
+
+validate_inputs() {
+ [ -d "$RAW_DIR" ] \
+ || die "raw output directory not found: $RAW_DIR (build the extensions first)"
+ [ -f "$CONTAINERFILE" ] || die "missing Containerfile: $CONTAINERFILE"
+ [ -f "$DELIVERY_SCRIPT" ] || die "missing delivery step script: $DELIVERY_SCRIPT"
+
+ local i name
+ for i in "${!NAMES[@]}"; do
+ name=${NAMES[$i]}
+ [ -f "$RAW_DIR/$name.raw" ] \
+ || die "missing raw for manifest entry '$name': $RAW_DIR/$name.raw"
+ done
+
+ local shim base shim_name j found
+ while IFS= read -r shim; do
+ [ -n "$shim" ] || continue
+ [ -f "$shim" ] || continue
+ base=${shim##*/}
+ shim_name=${base%-activation.service}
+ found=0
+ for j in "${!NAMES[@]}"; do
+ if [ "${NAMES[$j]}" = "$shim_name" ]; then
+ found=1
+ break
+ fi
+ done
+ if [ "$found" -eq 0 ]; then
+ die "staging shim '$base' has no manifest entry (name '$shim_name')"
+ fi
+ done < <(shim_templates)
+}
+
+assemble_staging() {
+ case $STAGING_OUT in
+ '' | /) die "refusing to use '$STAGING_OUT' as the staging directory" ;;
+ esac
+
+ rm -rf -- "$STAGING_OUT"
+ mkdir -p -- "$STAGING_OUT"
+
+ local i name shim
+ for i in "${!NAMES[@]}"; do
+ name=${NAMES[$i]}
+ cp -- "$RAW_DIR/$name.raw" "$STAGING_OUT/$name.raw"
+ done
+ while IFS= read -r shim; do
+ [ -n "$shim" ] || continue
+ cp -- "$shim" "$STAGING_OUT/${shim##*/}"
+ done < <(shim_templates)
+
+ cp -- "$MANIFEST" "$STAGING_OUT/extensions.manifest"
+ cp -- "$DELIVERY_SCRIPT" "$STAGING_OUT/sci-delivery.sh"
+ chmod 0755 "$STAGING_OUT/sci-delivery.sh"
+
+ log "staged ${#NAMES[@]} extensions in $STAGING_OUT"
+}
+
+stage_sha256sums() {
+ local -a raws=()
+ local i name raw
+ for i in "${!NAMES[@]}"; do
+ raws+=("${NAMES[$i]}.raw")
+ done
+
+ local mkosi_sums=$RAW_DIR/SHA256SUMS
+ local -A mkosi_hash=()
+ local sums_line_re='^([0-9a-fA-F]{64}) [ *]([^[:space:]]+)$'
+ local line hash lineno=0 usable=0
+
+ if [ -f "$mkosi_sums" ]; then
+ usable=1
+ while IFS= read -r line || [ -n "$line" ]; do
+ lineno=$((lineno + 1))
+ line=${line%$'\r'}
+ [ -n "$line" ] || continue
+ if [[ ! $line =~ $sums_line_re ]]; then
+ log "$mkosi_sums line $lineno is not sha256sum output; not using the mkosi checksums"
+ usable=0
+ break
+ fi
+ hash=${BASH_REMATCH[1],,}
+ name=${BASH_REMATCH[2]}
+ mkosi_hash[$name]=$hash
+ done <"$mkosi_sums"
+ fi
+
+ local actual
+ if [ "$usable" -eq 1 ]; then
+ for raw in "${raws[@]}"; do
+ if [ -z "${mkosi_hash[$raw]+set}" ]; then
+ usable=0
+ break
+ fi
+ actual=$(sha256sum -- "$STAGING_OUT/$raw")
+ actual=${actual%% *}
+ if [ "$actual" != "${mkosi_hash[$raw]}" ]; then
+ usable=0
+ break
+ fi
+ done
+ fi
+
+ local source
+ if [ "$usable" -eq 1 ]; then
+ : >"$STAGING_OUT/SHA256SUMS"
+ for raw in "${raws[@]}"; do
+ printf '%s %s\n' "${mkosi_hash[$raw]}" "$raw" >>"$STAGING_OUT/SHA256SUMS"
+ done
+ source="mkosi checksum output $mkosi_sums"
+ else
+ (
+ cd -- "$STAGING_OUT"
+ sha256sum -- "${raws[@]}" >SHA256SUMS
+ )
+ source="generated with sha256sum over the staged raws"
+ fi
+
+ (
+ cd -- "$STAGING_OUT"
+ sha256sum -c -- SHA256SUMS >/dev/null
+ ) || die "staged SHA256SUMS does not verify the staged raws: $STAGING_OUT/SHA256SUMS"
+
+ log "staged SHA256SUMS for ${#raws[@]} raws ($source)"
+}
+
+build_image() {
+ log "docker build -t $IMAGE_TAG -f $CONTAINERFILE $STAGING_OUT"
+ docker build -t "$IMAGE_TAG" -f "$CONTAINERFILE" "$STAGING_OUT"
+}
+
+build_from_staging() {
+ local dir=$1
+ case $dir in
+ '' | /) die "refusing to use '$dir' as the staging directory" ;;
+ esac
+ [ -d "$dir" ] || die "--from-staging: no such directory: $dir"
+ [ -f "$dir/SHA256SUMS" ] \
+ || die "--from-staging: $dir carries no SHA256SUMS: a staging without checksums cannot be verified"
+ [ -f "$dir/sci-delivery.sh" ] \
+ || die "--from-staging: $dir carries no sci-delivery.sh: the build context must contain the delivery step script"
+ [ -f "$dir/extensions.manifest" ] \
+ || die "--from-staging: $dir carries no extensions.manifest"
+ # The shipped checksums are verified, never regenerated: this is the
+ # artifact contract, not a fresh assembly.
+ (
+ cd -- "$dir"
+ sha256sum -c -- SHA256SUMS >/dev/null
+ ) || die "--from-staging: SHA256SUMS does not verify the staged raws: $dir"
+ log "verified the staged SHA256SUMS in $dir"
+ log "docker build -t $IMAGE_TAG -f $CONTAINERFILE $dir"
+ docker build -t "$IMAGE_TAG" -f "$CONTAINERFILE" "$dir"
+}
+
+main() {
+ local stage_only=0 from_staging=""
+ while [ "$#" -gt 0 ]; do
+ case $1 in
+ -h | --help)
+ usage
+ exit 0
+ ;;
+ --stage-only)
+ stage_only=1
+ ;;
+ --from-staging)
+ [ $# -ge 2 ] || die "--from-staging requires a directory argument (try --help)"
+ from_staging=$2
+ shift
+ ;;
+ *)
+ die "unknown argument: $1 (try --help)"
+ ;;
+ esac
+ shift
+ done
+
+ if [ -n "$from_staging" ]; then
+ [ "$stage_only" -eq 0 ] \
+ || die "--from-staging and --stage-only are mutually exclusive"
+ build_from_staging "$from_staging"
+ return 0
+ fi
+
+ load_manifest
+ validate_inputs
+ assemble_staging
+ stage_sha256sums
+ if [ "$stage_only" -eq 0 ]; then
+ build_image
+ fi
+}
+
+main "$@"
diff --git a/delivery/extensions.manifest b/delivery/extensions.manifest
new file mode 100644
index 00000000..1e631b32
--- /dev/null
+++ b/delivery/extensions.manifest
@@ -0,0 +1,11 @@
+# SCI extension delivery manifest:
+# TARGET_BASE is the Garden Linux major line; the delivery step
+# refuses other lines.
+TARGET_BASE=2150
+sci-10-base
+sci-20-compute
+sci-21-network openvswitch-switch.service ovsdb-server.service ovs-vswitchd.service
+sci-22-storage multipathd.service iscsid.service
+sci-40-libvirt libvirtd.service
+sci-50-virt-ch
+sci-51-virt-qemu
diff --git a/delivery/sci-delivery.sh b/delivery/sci-delivery.sh
new file mode 100755
index 00000000..43d078a6
--- /dev/null
+++ b/delivery/sci-delivery.sh
@@ -0,0 +1,626 @@
+#!/usr/bin/env bash
+# Idempotent SCI systemd-sysext delivery step; contract in delivery/README.md.
+# Usage: sci-delivery.sh [--help]
+
+set -Eeuo pipefail
+
+HOST_ROOT=${HOST_ROOT:-/host}
+# Remember whether the caller chose STAGING_DIR; fetch mode may supply its own
+# default only when they did not.
+STAGING_DIR_EXPLICIT=${STAGING_DIR:+1}
+STAGING_DIR=${STAGING_DIR:-/opt/sci}
+INTERVAL=${INTERVAL:-60}
+NSENTER=${NSENTER:-1}
+
+# Fetch mode: download one signed build instead of reading a pre-staged tree.
+FETCH_URL=${FETCH_URL:-}
+FETCH_PUBKEY=${FETCH_PUBKEY:-}
+FETCH_SUMS_DIGEST=${FETCH_SUMS_DIGEST:-}
+FETCH_DIR=${FETCH_DIR:-/var/lib/sci-sysext}
+FETCH_TMP=
+
+# Dot-file replay marker; not an extension.
+PENDING_MARKER=.sci-delivery-pending
+
+log() {
+ printf '[sci-delivery] %s\n' "$*"
+}
+
+warn() {
+ printf '[sci-delivery] warning: %s\n' "$*" >&2
+}
+
+die() {
+ printf '[sci-delivery] error: %s\n' "$*" >&2
+ exit 1
+}
+
+fetch_cleanup() {
+ if [ -n "$FETCH_TMP" ] && [ -d "$FETCH_TMP" ]; then
+ rm -rf -- "$FETCH_TMP"
+ fi
+ return 0
+}
+
+trap fetch_cleanup EXIT
+trap 'exit 0' TERM INT
+
+usage() {
+ cat <<'EOF'
+usage: sci-delivery.sh [--help]
+
+Reconciles the SCI systemd-sysext staging directory into the host root.
+Intended to run in a privileged pod with hostPID and the host / mounted at
+HOST_ROOT.
+
+Staging layout:
+ .raw extension image
+ -activation.service optional activation shim template
+ extensions.manifest directives plus " ..."
+ per line; TARGET_BASE= (the
+ Garden Linux base line) is required
+ SHA256SUMS plain sha256sum output, one
+ " .raw" line per entry
+
+A staging set whose raws do not match SHA256SUMS, or a host os-release that
+does not name the manifest's TARGET_BASE line, deploys nothing and exits
+non-zero.
+
+Environment:
+ HOST_ROOT host root mount point (default: /host)
+ STAGING_DIR extension staging directory (default: /opt/sci, or
+ $FETCH_DIR/staging in fetch mode when unset)
+ INTERVAL seconds between passes; 0 runs one pass (default: 60)
+ NSENTER 1 runs systemctl/systemd-sysext via nsenter -t 1 -m,
+ 0 runs them directly (default: 1)
+
+Fetch mode is active when FETCH_URL is set. It downloads one build from that
+base URL instead of trusting whatever is already staged:
+ FETCH_URL base URL of the build (unset selects image mode)
+ FETCH_PUBKEY PEM public key that verifies SHA256SUMS.sig (required)
+ FETCH_SUMS_DIGEST optional pinned sha256 of SHA256SUMS (64 lowercase hex)
+ FETCH_DIR writable fetch root (default: /var/lib/sci-sysext)
+
+Manifest directives:
+ TARGET_BASE= required Garden Linux base line
+ REMOVE= tombstone: delete a previously deployed extension
+EOF
+}
+
+# Run a host-mutating tool through PID 1's mount namespace when NSENTER=1.
+run_host() {
+ if [ "$NSENTER" = "1" ]; then
+ nsenter -t 1 -m -- "$@"
+ else
+ "$@"
+ fi
+}
+
+files_equal() { #
+ [ -f "$1" ] && [ -f "$2" ] || return 1
+ local a b
+ a=$(md5sum -- "$1") || return 1
+ b=$(md5sum -- "$2") || return 1
+ [ "${a%% *}" = "${b%% *}" ]
+}
+
+# Restore the parent mtime so only the file content changes.
+write_atomic() {
+ local src=$1 dest=$2 dir mtime
+ dir=$(dirname -- "$dest")
+ mtime=$(LC_ALL=C stat -c '%.9Y' -- "$dir")
+ cp -- "$src" "$dest.tmp"
+ mv -- "$dest.tmp" "$dest"
+ LC_ALL=C touch -m -d "@$mtime" -- "$dir"
+}
+
+declare -a NAMES=()
+declare -a UNIT_LISTS=()
+declare -a REMOVE_NAMES=()
+TARGET_BASE=
+
+validate_env() {
+ [ -d "$HOST_ROOT" ] || die "HOST_ROOT is not an existing directory: $HOST_ROOT"
+ case $INTERVAL in
+ '' | *[!0-9]*) die "INTERVAL must be a non-negative integer: $INTERVAL" ;;
+ esac
+}
+
+fetch_get() { #
+ curl -fsS --max-time 30 --retry 2 --retry-delay 1 \
+ -o "$2" "$FETCH_URL/$1" \
+ || die "fetch: cannot download $1 from $FETCH_URL"
+}
+
+fetch_get_file() { #
+ curl -fsS --connect-timeout 15 --max-time 600 \
+ --retry 2 --retry-delay 1 \
+ -o "$2" "$FETCH_URL/$1" \
+ || die "fetch: cannot download $1 from $FETCH_URL"
+}
+
+# Populate STAGING_DIR from FETCH_URL and prove the downloaded set before any
+# host mutation. Returns immediately in image mode.
+fetch_staging() {
+ [ -n "$FETCH_URL" ] || return 0
+
+ if [ -z "$STAGING_DIR_EXPLICIT" ]; then
+ STAGING_DIR=$FETCH_DIR/staging
+ fi
+
+ [ -n "$FETCH_PUBKEY" ] \
+ || die "fetch: FETCH_PUBKEY is required when FETCH_URL is set"
+ [ -f "$FETCH_PUBKEY" ] && [ -r "$FETCH_PUBKEY" ] \
+ || die "fetch: FETCH_PUBKEY is not a readable file: $FETCH_PUBKEY"
+ if [ -n "$FETCH_SUMS_DIGEST" ] \
+ && [[ ! $FETCH_SUMS_DIGEST =~ ^[0-9a-f]{64}$ ]]; then
+ die "fetch: FETCH_SUMS_DIGEST must be 64 lowercase hex characters: $FETCH_SUMS_DIGEST"
+ fi
+
+ mkdir -p -- "$FETCH_DIR/staging" \
+ || die "fetch: cannot create $FETCH_DIR/staging"
+ mkdir -p -- "$STAGING_DIR" \
+ || die "fetch: cannot create the staging directory: $STAGING_DIR"
+
+ FETCH_TMP=$(mktemp -d "$FETCH_DIR/.fetch.XXXXXX") \
+ || die "fetch: cannot create a temporary directory under $FETCH_DIR"
+
+ fetch_get SHA256SUMS "$FETCH_TMP/SHA256SUMS"
+ fetch_get SHA256SUMS.sig "$FETCH_TMP/SHA256SUMS.sig"
+
+ local actual
+ if [ -n "$FETCH_SUMS_DIGEST" ]; then
+ actual=$(sha256sum -- "$FETCH_TMP/SHA256SUMS") \
+ || die "fetch: cannot hash the downloaded SHA256SUMS"
+ actual=${actual%% *}
+ [ "$actual" = "$FETCH_SUMS_DIGEST" ] \
+ || die "fetch: SHA256SUMS digest mismatch: expected $FETCH_SUMS_DIGEST"
+ fi
+
+ if ! openssl dgst -sha256 -verify "$FETCH_PUBKEY" \
+ -signature "$FETCH_TMP/SHA256SUMS.sig" "$FETCH_TMP/SHA256SUMS" \
+ >/dev/null 2>&1; then
+ die "fetch: SHA256SUMS signature verification failed"
+ fi
+ log "fetch: verified SHA256SUMS signature"
+
+ local -a fetch_names=()
+ local -A fetch_sums=()
+ local -A fetch_listed=()
+ local line fname sum lineno=0
+ local sums_line_re='^([0-9a-f]{64}) [ *]([A-Za-z0-9][A-Za-z0-9._-]*)$'
+ while IFS= read -r line || [ -n "$line" ]; do
+ lineno=$((lineno + 1))
+ line=${line%$'\r'}
+ [ -n "$line" ] || continue
+ if [[ ! $line =~ $sums_line_re ]]; then
+ die "fetch: invalid SHA256SUMS line $lineno: $line"
+ fi
+ sum=${BASH_REMATCH[1]}
+ fname=${BASH_REMATCH[2]}
+ if [ -n "${fetch_sums[$fname]+set}" ]; then
+ die "fetch: duplicate SHA256SUMS line for '$fname' at line $lineno"
+ fi
+ fetch_sums[$fname]=$sum
+ fetch_listed[$fname]=1
+ fetch_names+=("$fname")
+ done <"$FETCH_TMP/SHA256SUMS"
+
+ cp -- "$FETCH_TMP/SHA256SUMS" "$STAGING_DIR/SHA256SUMS" \
+ || die "fetch: cannot copy SHA256SUMS into $STAGING_DIR"
+
+ local dest part
+ for fname in "${fetch_names[@]}"; do
+ dest=$STAGING_DIR/$fname
+ if [ -f "$dest" ]; then
+ actual=$(sha256sum -- "$dest") \
+ || die "fetch: cannot hash $dest"
+ actual=${actual%% *}
+ if [ "$actual" = "${fetch_sums[$fname]}" ]; then
+ log "fetch: up to date $fname"
+ continue
+ fi
+ fi
+ part=$dest.part
+ fetch_get_file "$fname" "$part"
+ actual=$(sha256sum -- "$part") \
+ || die "fetch: cannot hash $part"
+ actual=${actual%% *}
+ [ "$actual" = "${fetch_sums[$fname]}" ] \
+ || die "fetch: hash mismatch for $fname"
+ mv -- "$part" "$dest" \
+ || die "fetch: cannot move $fname into $STAGING_DIR"
+ log "fetch: downloaded $fname"
+ done
+
+ ( cd -- "$STAGING_DIR" && sha256sum -c -- SHA256SUMS >/dev/null ) \
+ || die "fetch: staged set is incomplete or corrupt"
+ log "fetch: staging verified (${#fetch_names[@]} files)"
+
+ # Prune what the verified inventory no longer lists, so a dropped
+ # extension does not leave a stale raw or shim behind for load_manifest.
+ local -a stale=()
+ local stale_path stale_base
+ local prune_nullglob=0
+ if shopt -q nullglob; then
+ prune_nullglob=1
+ else
+ shopt -s nullglob
+ fi
+ for stale_path in "$STAGING_DIR"/*.raw \
+ "$STAGING_DIR"/*-activation.service "$STAGING_DIR"/*.part; do
+ [ -f "$stale_path" ] || continue
+ stale_base=${stale_path##*/}
+ case $stale_base in
+ *.part)
+ stale+=("$stale_base")
+ ;;
+ *)
+ if [ -z "${fetch_listed[$stale_base]+set}" ]; then
+ stale+=("$stale_base")
+ fi
+ ;;
+ esac
+ done
+ if [ "$prune_nullglob" -eq 0 ]; then
+ shopt -u nullglob
+ fi
+ for stale_base in "${stale[@]}"; do
+ rm -f -- "$STAGING_DIR/$stale_base"
+ log "fetch: pruned $stale_base"
+ done
+ if [ "${#stale[@]}" -gt 0 ]; then
+ log "fetch: pruned ${#stale[@]} stale file(s)"
+ fi
+
+ rm -rf -- "$FETCH_TMP"
+ FETCH_TMP=
+}
+
+load_manifest() {
+ local manifest=$STAGING_DIR/extensions.manifest
+ [ -f "$manifest" ] || die "missing manifest: $manifest"
+
+ NAMES=()
+ UNIT_LISTS=()
+ REMOVE_NAMES=()
+ TARGET_BASE=
+ local -A remove_seen=()
+ local line name rest lineno=0
+ while IFS= read -r line || [ -n "$line" ]; do
+ lineno=$((lineno + 1))
+ line=${line%$'\r'}
+ if [ -z "$line" ]; then
+ continue
+ fi
+ if [[ $line =~ ^[[:space:]]*$ ]]; then
+ die "malformed manifest line $lineno (whitespace-only): $manifest"
+ fi
+ if [[ $line =~ ^[[:space:]]*# ]]; then
+ continue
+ fi
+ if [[ $line =~ ^([A-Z][A-Z0-9_]*)=([^[:space:]]*)$ ]]; then
+ case ${BASH_REMATCH[1]} in
+ TARGET_BASE)
+ if [ -n "$TARGET_BASE" ]; then
+ die "duplicate TARGET_BASE directive at line $lineno (exactly one is required): $manifest"
+ fi
+ TARGET_BASE=${BASH_REMATCH[2]}
+ if [[ ! $TARGET_BASE =~ ^[0-9]+$ ]]; then
+ die "TARGET_BASE='$TARGET_BASE' at line $lineno is not digits-only (expected the Garden Linux major line, e.g. 2150): $manifest"
+ fi
+ ;;
+ REMOVE)
+ name=${BASH_REMATCH[2]}
+ if [[ ! $name =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
+ die "fetch: REMOVE='$name' at line $lineno is not a valid extension name: $manifest"
+ fi
+ if [ -n "${remove_seen[$name]+set}" ]; then
+ die "fetch: duplicate REMOVE directive for '$name' at line $lineno: $manifest"
+ fi
+ remove_seen[$name]=1
+ REMOVE_NAMES+=("$name")
+ ;;
+ *)
+ die "unknown manifest directive '${BASH_REMATCH[1]}' at line $lineno (TARGET_BASE and REMOVE are the only directives): $manifest"
+ ;;
+ esac
+ continue
+ fi
+ read -r name rest <<<"$line"
+ if [[ ! $name =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then
+ die "malformed manifest line $lineno (invalid name '$name'): $manifest"
+ fi
+ NAMES+=("$name")
+ UNIT_LISTS+=("$rest")
+ done <"$manifest"
+
+ [ -n "$TARGET_BASE" ] \
+ || die "manifest $manifest carries no TARGET_BASE= directive (expected exactly one; the base-version gate refuses to run without it)"
+
+ local remove_name j
+ for remove_name in "${REMOVE_NAMES[@]}"; do
+ for j in "${!NAMES[@]}"; do
+ if [ "${NAMES[$j]}" = "$remove_name" ]; then
+ die "fetch: REMOVE='$remove_name' is also a manifest entry: $manifest"
+ fi
+ done
+ done
+
+ local i name_entry
+ for i in "${!NAMES[@]}"; do
+ name_entry=${NAMES[$i]}
+ [ -f "$STAGING_DIR/$name_entry.raw" ] \
+ || die "manifest entry '$name_entry' has no staging raw: $STAGING_DIR/$name_entry.raw"
+ done
+
+ local -a shims=()
+ local shopt_was_set=0
+ if shopt -q nullglob; then
+ shopt_was_set=1
+ else
+ shopt -s nullglob
+ fi
+ shims=("$STAGING_DIR"/*-activation.service)
+ if [ "$shopt_was_set" -eq 0 ]; then
+ shopt -u nullglob
+ fi
+
+ local shim base shim_name j found
+ for shim in "${shims[@]}"; do
+ [ -f "$shim" ] || continue
+ base=${shim##*/}
+ shim_name=${base%-activation.service}
+ found=0
+ for j in "${!NAMES[@]}"; do
+ if [ "${NAMES[$j]}" = "$shim_name" ]; then
+ found=1
+ break
+ fi
+ done
+ if [ "$found" -eq 0 ]; then
+ die "staging shim '$base' has no manifest entry (name '$shim_name')"
+ fi
+ done
+}
+
+verify_staging_integrity() {
+ local sums=$STAGING_DIR/SHA256SUMS
+ [ -f "$sums" ] \
+ || die "staged-raw integrity: missing SHA256SUMS: $sums (the staged raws cannot be verified; refusing to deploy)"
+
+ declare -A sums_by_name=()
+ local line sum raw lineno=0
+ local sums_line_re='^([0-9a-fA-F]{64}) [ *]([^[:space:]]+)$'
+ while IFS= read -r line || [ -n "$line" ]; do
+ lineno=$((lineno + 1))
+ line=${line%$'\r'}
+ [ -n "$line" ] || continue
+ if [[ ! $line =~ $sums_line_re ]]; then
+ die "staged-raw integrity: malformed SHA256SUMS line $lineno (expected '<64-hex>.raw'): $sums"
+ fi
+ sum=${BASH_REMATCH[1],,}
+ raw=${BASH_REMATCH[2]}
+ if [ -n "${sums_by_name[$raw]+set}" ]; then
+ die "staged-raw integrity: duplicate SHA256SUMS line for '$raw' at line $lineno: $sums"
+ fi
+ sums_by_name[$raw]=$sum
+ done <"$sums"
+
+ local i name expected actual
+ for i in "${!NAMES[@]}"; do
+ name=${NAMES[$i]}.raw
+ [ -n "${sums_by_name[$name]+set}" ] \
+ || die "staged-raw integrity: SHA256SUMS has no line for manifest entry '${NAMES[$i]}' ($name): $sums"
+ expected=${sums_by_name[$name]}
+ actual=$(sha256sum -- "$STAGING_DIR/$name") \
+ || die "staged-raw integrity: cannot hash $STAGING_DIR/$name"
+ actual=${actual%% *}
+ [ "$actual" = "$expected" ] \
+ || die "staged-raw integrity: $name does not match its SHA256SUMS line (recorded $expected, actual $actual): $sums"
+ done
+
+ log "staged-raw integrity: SHA256SUMS verifies ${#NAMES[@]} staged raw(s)"
+}
+
+check_base_gate() {
+ local os_release=$HOST_ROOT/etc/os-release version host_line
+ [ -r "$os_release" ] \
+ || die "base-version gate: cannot read $os_release; refusing to deploy the TARGET_BASE=$TARGET_BASE extensions onto an unknown Garden Linux line"
+ version=$(sed -n 's/^IMAGE_VERSION=//p' -- "$os_release" | head -n 1)
+ if [ -z "$version" ]; then
+ version=$(sed -n 's/^GARDENLINUX_VERSION=//p' -- "$os_release" | head -n 1)
+ fi
+ # os-release values may be quoted.
+ version=${version#\"}
+ version=${version%\"}
+ version=${version#\'}
+ version=${version%\'}
+ [ -n "$version" ] \
+ || die "base-version gate: $os_release carries neither IMAGE_VERSION nor GARDENLINUX_VERSION; cannot compare the Garden Linux line with TARGET_BASE=$TARGET_BASE"
+ host_line=${version%%.*}
+ if [ "$host_line" != "$TARGET_BASE" ]; then
+ die "base-version gate: host Garden Linux line $host_line (version '$version' in $os_release) does not match TARGET_BASE=$TARGET_BASE; the extensions were built for another base line"
+ fi
+ log "base-version gate: host Garden Linux line $host_line matches TARGET_BASE=$TARGET_BASE"
+}
+
+run_pass() {
+ local -a raw_deployed=()
+ local -a shim_deployed=()
+ local -a has_shim=()
+ local -a new_entry=()
+ local -a units=()
+ local i name src dest shim_src shim_dest any_change=0 pending=0
+ local restart_failed=0
+ local marker=$HOST_ROOT/var/lib/extensions/$PENDING_MARKER
+
+ if [ -e "$marker" ]; then
+ pending=1
+ fi
+
+ # Tombstones run before the entries and never activate anything: a removed
+ # extension must not have its units restarted.
+ local remove_name remove_raw remove_shim remove_wants removed_any
+ local removals_present=0
+ for remove_name in "${REMOVE_NAMES[@]}"; do
+ if [ -e "$HOST_ROOT/var/lib/extensions/$remove_name.raw" ] \
+ || [ -e "$HOST_ROOT/etc/systemd/system/$remove_name-activation.service" ] \
+ || [ -e "$HOST_ROOT/etc/systemd/system/multi-user.target.wants/$remove_name-activation.service" ] \
+ || [ -L "$HOST_ROOT/etc/systemd/system/multi-user.target.wants/$remove_name-activation.service" ]; then
+ removals_present=1
+ break
+ fi
+ done
+ if [ "$removals_present" -eq 1 ]; then
+ # Bracket the deletions too: a kill between a delete and the activation
+ # flow must replay on the next pass.
+ mkdir -p -- "$HOST_ROOT/var/lib/extensions"
+ : >"$marker"
+ fi
+ for remove_name in "${REMOVE_NAMES[@]}"; do
+ remove_raw=$HOST_ROOT/var/lib/extensions/$remove_name.raw
+ remove_shim=$HOST_ROOT/etc/systemd/system/$remove_name-activation.service
+ remove_wants=$HOST_ROOT/etc/systemd/system/multi-user.target.wants/$remove_name-activation.service
+ removed_any=0
+ if [ -e "$remove_raw" ]; then
+ rm -f -- "$remove_raw"
+ removed_any=1
+ fi
+ if [ -e "$remove_shim" ]; then
+ rm -f -- "$remove_shim"
+ removed_any=1
+ fi
+ if [ -e "$remove_wants" ] || [ -L "$remove_wants" ]; then
+ rm -f -- "$remove_wants"
+ removed_any=1
+ fi
+ if [ "$removed_any" -eq 1 ]; then
+ any_change=1
+ log "removed $remove_name"
+ fi
+ done
+
+ for i in "${!NAMES[@]}"; do
+ raw_deployed[i]=0
+ shim_deployed[i]=0
+ has_shim[i]=0
+ new_entry[i]=0
+ done
+
+ for i in "${!NAMES[@]}"; do
+ name=${NAMES[$i]}
+
+ src=$STAGING_DIR/$name.raw
+ dest=$HOST_ROOT/var/lib/extensions/$name.raw
+ if [ ! -f "$dest" ]; then
+ new_entry[i]=1
+ fi
+ if [ "${new_entry[i]}" -eq 1 ] || ! files_equal "$src" "$dest"; then
+ mkdir -p -- "$HOST_ROOT/var/lib/extensions"
+ write_atomic "$src" "$dest"
+ raw_deployed[i]=1
+ any_change=1
+ log "deployed $name.raw"
+ fi
+
+ shim_src=$STAGING_DIR/$name-activation.service
+ if [ -f "$shim_src" ]; then
+ has_shim[i]=1
+ shim_dest=$HOST_ROOT/etc/systemd/system/$name-activation.service
+ if [ ! -f "$shim_dest" ] || ! files_equal "$shim_src" "$shim_dest"; then
+ mkdir -p -- "$HOST_ROOT/etc/systemd/system"
+ write_atomic "$shim_src" "$shim_dest"
+ shim_deployed[i]=1
+ any_change=1
+ log "deployed $name-activation.service"
+ fi
+ fi
+ done
+
+ if [ "$any_change" -eq 0 ] && [ "$pending" -eq 0 ]; then
+ log "no changes; nothing to do"
+ return 0
+ fi
+
+ if [ "$pending" -eq 1 ]; then
+ log "pending activation found; replaying the activation flow"
+ for i in "${!NAMES[@]}"; do
+ raw_deployed[i]=1
+ if [ "${has_shim[i]}" -eq 1 ]; then
+ shim_deployed[i]=1
+ fi
+ done
+ fi
+
+ mkdir -p -- "$HOST_ROOT/var/lib/extensions"
+ : >"$marker"
+
+ run_host systemctl daemon-reload
+ run_host systemd-sysext refresh
+ run_host systemctl reload dbus.service
+
+ for i in "${!NAMES[@]}"; do
+ [ "${raw_deployed[i]}" -eq 1 ] || continue
+ units=()
+ read -r -a units <<<"${UNIT_LISTS[$i]}"
+ if [ "${#units[@]}" -gt 0 ]; then
+ if ! run_host systemctl restart "${units[@]}"; then
+ restart_failed=1
+ warn "extension ${NAMES[$i]}: restart failed for ${units[*]} (continuing)"
+ fi
+ fi
+ done
+
+ # Enable every activation shim; restart the changed/new ones.
+ for i in "${!NAMES[@]}"; do
+ [ "${has_shim[i]}" -eq 1 ] || continue
+ name=${NAMES[$i]}
+ run_host systemctl enable "$name-activation.service"
+ if [ "${shim_deployed[i]}" -eq 1 ] \
+ || [ "${raw_deployed[i]}" -eq 1 ] \
+ || [ "${new_entry[i]}" -eq 1 ]; then
+ run_host systemctl restart "$name-activation.service"
+ fi
+ done
+
+ rm -f -- "$marker"
+
+ if [ "$restart_failed" -ne 0 ]; then
+ warn "activation flow completed, but at least one extension unit restart failed"
+ return 1
+ fi
+
+ return 0
+}
+
+main() {
+ local arg
+ for arg in "$@"; do
+ case $arg in
+ -h | --help)
+ usage
+ exit 0
+ ;;
+ *)
+ die "unknown argument: $arg (try --help)"
+ ;;
+ esac
+ done
+
+ while :; do
+ validate_env
+ fetch_staging
+ load_manifest
+ verify_staging_integrity
+ check_base_gate
+ run_pass
+ if [ "$INTERVAL" -eq 0 ]; then
+ break
+ fi
+ sleep "$INTERVAL" &
+ wait $! || true
+ done
+}
+
+main "$@"
diff --git a/docs/sysext-virt.md b/docs/sysext-virt.md
new file mode 100644
index 00000000..11d20e52
--- /dev/null
+++ b/docs/sysext-virt.md
@@ -0,0 +1,900 @@
+# SCI system extensions — design
+
+How the SCI layer is built, shaped, signed, activated and delivered as
+systemd-sysext images. The delivery step's operational contract is in
+`delivery/README.md`. The pytest modules under `extensions/tests/` are
+the specification. This document gives the reasons behind them.
+
+## 1. Purpose
+
+The SCI layer is SAP's virtualization stack on Garden Linux. Inside an
+image feature it couples every libvirt, Cloud Hypervisor or Open
+vSwitch change to a base-image build and release, and a host must
+re-image to run the stack.
+
+This project ships the stack as systemd-sysext extensions. An
+extension is an additive, read-only `/usr` overlay delivered at runtime
+to a *standard* Garden Linux base. A stock `openstack-gardener_prod`
+worker plus the extensions becomes a KVM hypervisor host: Open vSwitch,
+multipath/iSCSI/NFS tuning, libvirt, a hypervisor driver, and the
+chart-consumed agent identities.
+
+Four properties of the mechanism shape every decision below:
+
+- **The base is not touched.** No base image is rebuilt or patched. A
+ base A/B update leaves extensions, users and shims in place. They
+ re-merge on the new base through ID-only binding.
+- **Extensions are purely additive.** They carry `/usr` content and the
+ `usr/lib/...` fragments systemd and kmod read. Everything else stays
+ with the base.
+- **There is no package manager on the host** (`_nopkg` variant). An
+ extension is a file set with declarative fragments, never an install
+ transaction.
+- **Immutable systems are first-class.** Nothing assumes a writable
+ `/usr` or a `/var` on the root disk (section 12).
+
+## 2. Target environment
+
+| Item | Value |
+|---|---|
+| Base | Garden Linux 2150.9.0, `openstack-gardener_prod` amd64, features include `_nopkg` |
+| Kernel | 6.18.44-cloud-amd64 (kvm, openvswitch, nf_conntrack in-tree) |
+| Userland | Debian trixie, systemd 259.7 (`259.7-3gl1+bp2150`) |
+| Consumers | Gardener shoot workers. The consumer does not pin the machine image, so the base drifts across 2150.x minor versions |
+| Variants | mutable Gardener worker, and the immutable Garden Linux USI variants (UKI with embedded EROFS root, `/var` split off, `/etc` as a `/var`-backed overlay) |
+
+The suite version is an input, not a detail. It selects the apt suite,
+it derives the build timestamp, and it seeds every image's GPT
+identifiers (section 5). `Release=` in `mkosi/mkosi.conf` is its single
+source of truth.
+
+## 3. Deployables and compositions
+
+Seven images are deployable. Each is a mkosi `Format=sysext` +
+`Overlay=yes` delta over the published base tree, so it contains only
+its own files.
+
+| Extension | Role | Delta over |
+|---|---|---|
+| `sci-10-base` | the shared base extension in every composition | `base` |
+| `sci-20-compute` | the chart-consumed identities, with memberships and state-dir tmpfiles | `base` |
+| `sci-21-network` | Open vSwitch, its units and the OVS identities | `base` |
+| `sci-22-storage` | multipath, iSCSI and NFS client tuning | `base` |
+| `sci-40-libvirt` | the hypervisor-agnostic libvirt daemon and clients, with vTPM | `base` |
+| `sci-50-virt-ch` | the Cloud Hypervisor driver and its firmware | `base` |
+| `sci-51-virt-qemu` | the qemu flavour: driver and userland, plus the VNC port policy for `etc/libvirt/qemu.conf` | `base` |
+
+**The prefixes encode the merge order.** systemd-sysext merges the
+deployed raws (`/var/lib/extensions/.raw`) in lexical filename
+order. The names put `sci-10-base` and the service extensions
+(`sci-20-compute`, `sci-21-network`, `sci-22-storage`) ahead of
+`sci-40-libvirt`. `sci-40-libvirt` precedes the drivers that need the
+libvirt files (`sci-50-virt-ch`, `sci-51-virt-qemu`). The order is a
+safety net on top of the single-owner rule (section 8): with every path
+owned by exactly one extension, the order cannot shadow content. The
+gaps in the numbering (10, 20–22, 40, 50–51) leave room to insert an
+extension without renumbering.
+
+The base is not a mkosi subimage. It is the verified extraction of the
+published Garden Linux release named by `mkosi/base-image.pin`
+(section 4).
+
+The composition is chosen by the delivery manifest, not at build time.
+The manifest lists every extension in merge order: `sci-10-base`,
+`sci-20-compute`, `sci-21-network`, `sci-22-storage`, `sci-40-libvirt`,
+`sci-50-virt-ch`, `sci-51-virt-qemu`. The repo curates the full set.
+The boot test uses the full set — fetch mode deploys the release's manifest.
+
+**Every extension builds standalone.** Each has exactly one
+`BaseTrees=` entry, the published base tree. None carries
+`Dependencies=`. The virt driver packages hard-depend on the libvirt
+package set, so apt installs that set into each virt build anyway.
+`RemoveFiles=` then drops the paths that belong to `sci-40-libvirt`.
+The path lists live in the image configs. At runtime the merged
+`sci-40-libvirt` supplies them. A libvirt package change therefore
+rebuilds only `sci-40-libvirt`. A virt package change rebuilds only its
+own image.
+
+Two alternatives do not work. `RemovePackages=` cannot purge the
+libvirt set: the driver payload hard-depends on it, and an apt
+simulation deletes the driver with it. Double-shipping the paths is
+worse. Overlay shadowing is silent on version skew, and the package
+ships `usr/lib/sysusers.d/libvirt.conf` as `g libvirt -`, while
+`sci-40-libvirt` pins `g libvirt 42437` (section 8).
+
+A build-tier test backstops the rule: the extracted file sets of
+`sci-50-virt-ch` and `sci-51-virt-qemu` must be disjoint from
+`sci-40-libvirt`'s. A package update that adds a libvirt-owned path
+outside the globs fails the suite. The drift is asymmetric: only
+under-removal is caught. Over-removal — a future payload file landing
+under a generic glob such as `usr/share/man/*` — is silent, and shows
+at best as a missing REQUIRED pin or a guest boot-contract failure.
+
+**The qemu stack is its own extension.**
+`libvirt-daemon-system` is never installed, because it depends on the
+qemu driver. The signed-delta check asserts the absence of the qemu
+binaries and drivers in `sci-40-libvirt` and `sci-50-virt-ch`. A
+deployment that merges no qemu extension runs no qemu binaries. One
+that wants them merges `sci-51-virt-qemu`.
+
+## 4. Build pipeline
+
+mkosi 27.1 builds everything. It is not on PyPI and the host has no
+`ensurepip`, so the venv under `.build/mkosi/venv` is a precondition,
+not a build product:
+
+```sh
+python3 -m venv .build/mkosi/venv
+.build/mkosi/venv/bin/pip install 'mkosi @ git+https://github.com/systemd/mkosi.git@v27.1'
+```
+
+- **Base tree.** The extensions are deltas over the **published** Garden
+ Linux image, the pinned release artifact. The overlay baseline is
+ byte-for-byte the `/usr` the delivery target runs. `mkosi/base-image.pin`
+ records the accepted artifact and its digests. `mkosi/base-image.sh
+ ensure` verifies both, extracts the tree, and marks it ready under
+ `.build/mkosi/base-image//`. The image configs name the stable
+ `current` / `current-kernel` symlinks in that cache, so no config text
+ embeds the pin-derived key. One exception: mkosi dereferences the
+ symlink when it writes `summary --json`, so the resolved-config pin
+ embeds the key path. A base-image pin bump therefore moves
+ `mkosi/summary.pin` too. `mkosi/build.sh` runs `ensure` before every
+ build, and `VERSION` must equal `Release=` in `mkosi/mkosi.conf`. A
+ skew, a digest mismatch or a missing inner member fails closed. A
+ rebuilt base tree would make each delta relative to an approximation.
+ An extension could then re-ship content the real base owns and shadow
+ it after the merge.
+- **Deltas.** `Format=sysext`, `Overlay=yes`, and a single `BaseTrees=`
+ entry naming the published base tree. mkosi drops every path that
+ exists in the base and keeps `usr/` and `opt/`. The emitted image is
+ exactly the set of paths the base does not own.
+- **Two modes.** `bash mkosi/build.sh` with no argument builds every
+ configured subimage in one invocation. The delivery image consumes
+ that output. `bash mkosi/build.sh ` builds one subimage alone.
+ That is the cheap iteration mode. The artifact contract reads the
+ build-all output.
+- **Tooling.** The tools tree (`mkfs.erofs`, `systemd-repart`, …) is
+ bootstrapped once under `.build/mkosi/tools/mkosi.tools` from a
+ generated config, and reused through `ToolsTree=`. Runtime state stays
+ under `.build/`, so the tracked `mkosi/` directory never changes
+ during a build. Builds are unprivileged.
+- **Recommendations stay off.** `WithRecommends` must not be set
+ anywhere. The apt backend's `Install-Recommends=false` default keeps
+ `libvirt-daemon`'s Recommends (the qemu driver) out of the
+ `sci-40-libvirt` delta, and `passt`/`qemu-block-extra` out of the qemu
+ flavour. mkosi
+ copies `mkosi.extra/` files after package installation, so an extra
+ tree deliberately overwrites package content. That is how the
+ libvirt gid pin and the resident dropin win.
+
+## 5. Reproducibility
+
+Two builds of the same tree must produce the same bytes. A delivered raw
+can then be traced to its inputs and re-verified. Two sources of
+nondeterminism exist. The build eliminates both by construction:
+
+- **Timestamp.** File times and image contents derive from the suite
+ version, never from build time or commit time. `bin/garden-version
+ --epoch "$SUITE"` is passed as `--source-date-epoch=` on the mkosi
+ command line. That is builder parity with Garden Linux, which derives
+ its timestamps from the version too. The setting is CLI-only. mkosi
+ 27.1 treats `SourceDateEpoch` as multiversal: it cannot be set in a
+ subimage, and the CLI value passes down to all of them.
+- **GPT identifiers.** Each deployable carries a committed `Seed=` in
+ `[Output]`, derived as `sha256("sci::")` formatted as a
+ UUID (the Garden Linux builder's `uuid_hash` scheme). It fixes the
+ repart disk id, the partition UUIDs, and — under signing — the verity
+ salt. The identities do not depend on whether an image was built alone
+ or in a build-all invocation. `` is the `mkosi.images/`
+ directory name, so a rename recomputes the seed. A Garden Linux suite
+ bump invalidates every seed. The static input contract re-derives them
+ all.
+
+Three committed pins make the rest of the input space conscious:
+
+| Pin | What it pins |
+|---|---|
+| `mkosi/suite.pin` | `SUITE=` and the sha256 of the apt suite's `InRelease` |
+| `mkosi/tools.pin` | sha256 of the bootstrapped tools tree's `mkosi.tools.manifest` |
+| `mkosi/summary.pin` | sha256 of the *normalized* `mkosi summary --json` over the whole config tree |
+
+The `InRelease` digest has no automatic check today.
+
+Why three pins: suite.pin covers the apt suite and tools.pin the
+toolchain. Neither sees the *resolved* configuration inside the build.
+A mkosi default change on a version bump, or an unremarked config edit,
+moves the output bytes while every input file stays unchanged. The
+summary pin closes that gap. The normalizer
+(`extensions/tests/summary.py`, also the regeneration CLI) maps
+repo-root-relative paths to `@REPO/` and other absolute host
+paths to `@ABS`, nulls per-image `Seed`, `DelegateRanges` and
+`ProxyExclude`, and sorts keys. `DelegateRanges` is uid-dependent in
+mkosi 27.1 (root 0, non-root 3). mkosi 27.1 reads `ProxyExclude` from
+the invoking shell's no_proxy/NO_PROXY. Without those rules the pin
+depends on the host. A *new* volatile field fails the pin instead
+of making it flaky. Comment-only edits do not move it.
+
+The suite version has one further guard inside the build. It aborts
+when the assembled sandbox apt source `Suites:` differs from `Release=`.
+Packages can then never come from a suite other than the one the
+version and the epoch name.
+
+**A conscious bump** is the same mechanical sequence for every pin.
+Accept the change, then update in the same commit: `Release=` in
+`mkosi/mkosi.conf`, the sandbox `Suites:`, `mkosi/suite.pin`, all seven
+`Seed=` values, `mkosi/summary.pin`, `mkosi/base-image.pin`, and each
+manifest's `TARGET_BASE` if the major line moved.
+
+## 6. Artifact shape
+
+Every deployable is a GPT disk image with a single EROFS root and a
+signed dm-verity pair:
+
+| # | Partition name | Type GUID | Content |
+|---|---|---|---|
+| 1 | `root-x86-64` | `4F68BCE3-E8CD-4DB1-96E7-FBCAF984B709` | the delta as EROFS |
+| 2 | `root-x86-64-verity` | `2C7357ED-EBD2-46D9-AEC1-23D437EC2BF5` | the dm-verity hash tree |
+| 3 | `root-x86-64-verity-sig` | `41092B05-9FC8-4523-994F-2DEF0408B176` | the PKCS#7 signature over the root hash |
+
+The type GUIDs are the UAPI discoverable-partition values. The verity
+pair's values are not guessable from the naming, so a shape assertion
+must use these exact GUIDs.
+
+Inside the root partition, each image carries exactly one
+`usr/lib/extension-release.d/extension-release.`. The file name
+must match the image name (`extension-release.sci-20-compute`, …). A
+bare filename is rejected. The file carries:
+
+- `ID=gardenlinux` — the only binding systemd applies.
+- `EXTENSION_RELOAD_MANAGER=1` on every image except `sci-10-base`, so
+ the merge reloads the manager and makes the extension's units known.
+ `sci-20-compute` ships no units. It carries the field for pattern
+ consistency.
+- `EXTENSION_RESTART_UNITS=` where a restart list exists. It is forward
+ compatibility (section 9).
+- `ARCHITECTURE=x86-64` (systemd naming, not Debian's `amd64`) and
+ `SYSEXT_SCOPE=system`. The value comes from
+ `Environment=SYSEXT_SCOPE=system` in each image's `[Build]`, because
+ mkosi does not read the process environment.
+
+`VERSION_ID` is deliberately absent (section 13). A wrong
+`ARCHITECTURE` or a bare filename does not fail loudly. systemd skips
+the extension silently. The artifact contract pins `ID` and
+`SYSEXT_SCOPE` on every deployable.
+
+**The `RemoveFiles=usr/lib/verity.d` guard.** The consumer's trust
+anchor is `/usr/lib/verity.d/verity.crt`. mkosi does not install it
+into extension images, and the published base tree carries no such
+directory, so the deployables never ship the anchor. Every deployable
+nevertheless carries `RemoveFiles=usr/lib/verity.d` as an explicit
+guard. A future base change, or a different mkosi path, must not leak a
+copy of the anchor into an extension. The anchor belongs to the host,
+and a copy would go stale when it is rotated. Today the line is a
+no-op: the directory is absent from all seven raws. `RemoveFiles=`
+deletes what a glob matches, so a missing path is not an error.
+
+## 7. Verity signing and the trust anchor
+
+Integrity is enforced at activation, by the consumer. A `SHA256SUMS`
+list is enforced by software a booting target has no reason to trust.
+An embedded signature plus dm-verity lets `systemd-sysext` (through
+`systemd-dissect`) refuse a tampered or substituted image, with no
+trusted intermediary at deployment time. The signature travels inside
+the image. There is no sidecar to lose or spoof.
+
+Wiring, all in `mkosi/mkosi.conf` under `[Validation]`:
+
+- `Verity=signed` configures the *top-level* image and is
+ `SettingScope.local` in mkosi 27.1, so it is not inherited. The
+ subimages sign because `Verity=auto` (the default) plus the
+ **universal** `VerityKey=` and `VerityCertificate=` selects the
+ signed repart definitions (`root` + `root-verity` + `root-verity-sig`).
+- The committed anchor is `mkosi/verity.crt`, RSA-4096. A CI run
+ replaces the working-tree copy with its run-local certificate and
+ never commits it.
+- The private key is machine-local, gitignored state at
+ `.build/keys/sysext-verity.key` (mode 0600). A machine without it
+ fails closed at build time, in the subimage build. The `Format=none`
+ main config early-continues before mkosi's input checks, so the
+ failure is not at config-parse time.
+- `systemd-repart` does the signing, driven by mkosi. `mkosi genkey` is
+ not usable here: it hardcodes RSA-2048, writes fixed `mkosi.key` /
+ `mkosi.crt` names, and refuses to overwrite. The certificate came
+ from an `openssl` invocation with `rsa:4096` and explicit output
+ paths.
+
+**This is a development key.** It is an experiment trust root, not a
+production one. Key custody — generation ceremony, storage, rotation,
+distribution of the public certificate — is an open decision. The key
+custody is not hardened, and an operator is expected to rotate the key
+into a hardened custodian before the `unhardened` qualifier disappears.
+Every build machine and the landing tree must provision the key, or the
+build stops.
+
+**The verification path on GL 259.** The kernel dm-verity signature
+check runs first and fails `-ENOKEY`. There is no key in the kernel key
+ring, and the boot check uses `Firmware=linux`, so there is no UEFI db
+either. systemd then falls back to userspace PKCS#7 verification against
+`/usr/lib/verity.d/verity.crt` and succeeds. It activates by root hash
+and reports the verity volume as verified. The signature is detached
+PKCS#7 over the ASCII-hex root hash. The anchor is never shipped by an
+extension. The boot runner copies the committed `mkosi/verity.crt` to
+`usr/lib/verity.d/verity.crt` in the generated overlay. On a real host
+the anchor must arrive through the base pipeline or configuration
+management. A future UKI or Secure Boot deployment should load the
+certificate into the key ring instead of relying on the userspace path.
+
+**All-or-nothing merge semantics.** One refused or broken extension
+aborts the whole merge. The host then has *no* extension merged,
+not the healthy subset. That is upstream design in systemd v259
+and current main, not a defect. Per-image skips exist only for
+`-ENOMEDIUM`, forbidden content and an `extension-release` mismatch.
+It is an availability consideration: there is no per-image containment,
+and it is not something to patch locally.
+
+**dm-verity is lazy.** dm-verity verifies only the blocks a consumer
+reads. A tamper test must hit such a block. A one-byte flip at
+offset 8192 is *not* such a block for a two-block EROFS: the byte sits
+in zero padding that nothing reads, and the tampered image merges. A
+flip in the superblock region (offset 1024) lands in a read block. The
+refresh refuses it and aborts the merge. Anyone writing a tamper proof must aim at real
+content.
+
+**The consumer policy is the host's decision.** The default policy
+still accepts `unprotected` images, so the downgrade hole stays open on
+a host until it carries a tightened `sysext.conf`
+(`[SysExt] ImagePolicy=root=signed`). Nothing in `delivery/` writes
+`sysext.conf`, deliberately. That is a host-side fleet decision. The guest
+verity scenario pins the refusal under a tightened policy, and the
+all-or-nothing abort that comes with it.
+
+**What is proven where.** The local boot runner demonstrates three
+proofs against the prepared published base tree (section 10): a signed
+set merging at boot, a read-block tamper refused, and an unsigned probe
+refused under a tightened policy. No real host has exercised it yet.
+Delivery to a host still rests on the delivery step's `SHA256SUMS` check.
+It keeps resting on it until the anchor arrives through an independent
+channel and the host policy is tightened.
+
+## 8. Content and identity rules
+
+**Single owner per path.** Every path in a deployed composition has
+exactly one owning extension. The base delta covers the base: a path
+the published tree owns never appears in an extension. Overlap between
+extensions is *not* covered by construction and needs an explicit
+assignment plus a build-time assertion. The canonical case is
+`usr/lib/sysusers.d/libvirt-qemu.conf`. It lives in `sci-40-libvirt`
+only, and `sci-51-virt-qemu` deletes the qemu driver package's
+divergent copy with `RemoveFiles=`. The whole libvirt-owned path set
+follows the same rule in both virt deltas (section 3). The build-tier
+set-difference test is the assertion. Two extensions shipping the same
+path do not merge. The overlay shadows one file silently,
+`systemd-sysusers` processes only the visible one, and divergence
+breaks NFS instance-directory and `/dev/kvm` access on exactly the
+hosts that carry the other copy.
+
+**Fixed identities**, shipped as `usr/lib/sysusers.d/` fragments in the
+owning extension:
+
+| Identity | ID | Owner | Reason |
+|---|---|---|---|
+| `openvswitch` | 476 | `sci-21-network` | group owner of the OVS runtime sockets |
+| `neutron` | 42435 | `sci-21-network` | OVS agent → `/run/openvswitch/db.sock` |
+| `openstack` | 42424 | `sci-20-compute` | legacy identity carried over from the feature |
+| `nova` | 42436 | `sci-20-compute` | `libvirt` + `openvswitch`. `/var/lib/nova` owned `nova:libvirt-qemu`. Shell `/bin/bash` for live-migration SSH. The fixed uid is load-bearing for the shared instances NFS |
+| `kvm-node-agent` | 42438 | `sci-20-compute` | `libvirt` group, owns `/etc/pki/{CA,libvirt,qemu}` |
+| `libvirt` | 42437 | `sci-40-libvirt` | group on `/run/libvirt/libvirt-sock`. `libvirt-daemon-common` declares it dynamically, so the pin is required. Both fragments must agree |
+| `libvirt-qemu` | 64055 | `sci-40-libvirt` | the VMM identity. `kvm` member (`/dev/kvm` is root:kvm 0660) |
+
+`g kvm` belongs to the base image and stays dynamic. Fragments
+reference it in membership lines only. No fragment pins a gid for it.
+
+**Cross-fragment rule.** sysusers `m` lines fail when the referenced
+group does not exist in the same pass. Any fragment referencing a
+foreign group also declares it with the same fixed gid.
+`sci-20-compute` declares `g openvswitch 476`, `g libvirt 42437` and
+`g libvirt-qemu 64055` before its `m` lines. That is idempotent, and it
+makes `sci-20-compute` self-sufficient in a composition without
+`sci-21-network` or `sci-40-libvirt`. Sysusers never renumbers an
+existing group, so a base that pre-creates `libvirt` at another gid
+makes the pin unreachable. A future plain-base variant must not
+pre-create it.
+
+**State directories** that a sysext cannot carry ship as `tmpfiles.d`
+fragments in the owning extension: `/var/lib/nova*`,
+`/etc/pki/{CA,libvirt,qemu}`, `/var/lib/libvirt`, `/var/log/libvirt`,
+`/var/lib/libvirt/{ch,qemu}`, `/var/log/libvirt/{ch,qemu}`,
+`/etc/openvswitch` with
+`L /etc/openvswitch/conf.db … /var/lib/openvswitch/conf.db`,
+`/etc/iscsi`, `/var/lib/iscsi`. The conf.db rule uses type `L`, never
+`L+`, so an existing real database is never replaced. The regular
+boot-time tmpfiles pass reads the merged fragments. Fragments naming
+extension users only succeed after those users exist. That is why the
+activation shim applies them itself (section 9).
+
+**Config placement.** `modprobe.d`, `modules-load.d`, systemd presets,
+tmpfiles and sysusers ship under `usr/lib/...`, whose fallbacks systemd
+and kmod already read. That keeps `/etc` writable and avoids confext.
+Confext is not used: it mounts `/etc` read-only while merged, the
+riskiest semantic, and oneshot units cover the need.
+
+The one `/etc` file with no `/usr` fallback is libvirt's
+`qemu.conf`. The qemu extension ships a template under
+`usr/share/sci/etc/` and its tmpfiles fragment carries
+`C /etc/libvirt/qemu.conf`, which the libvirt activation shim applies
+after the merge and before `libvirtd` starts. Type `C` is copy-once:
+the first activation creates the file, later activations leave
+operator edits alone, and a policy change needs the file deleted or
+the rule moved to `C+`.
+
+**Payload fragment filenames are not identities.** The sci-prefixed
+fragments keep their pre-rename stems on purpose:
+`usr/lib/sysusers.d/sci-compute.conf`,
+`usr/lib/sysusers.d/sci-network.conf`,
+`usr/lib/tmpfiles.d/sci-compute.conf`,
+`usr/lib/tmpfiles.d/sci-network.conf`,
+`usr/lib/tmpfiles.d/sci-storage.conf`,
+`usr/lib/tmpfiles.d/sci-libvirt.conf`,
+`usr/lib/tmpfiles.d/sci-virt-ch.conf`,
+`usr/lib/tmpfiles.d/sci-virt-qemu.conf` and
+`usr/lib/modules-load.d/sci-network.conf`. They are content paths
+inside the images, not extension identities, so they are intentionally
+unchanged. Do not "complete" the rename there. An extension identity
+lives in its `mkosi.images//` directory, its raw, its
+`extension-release.` file and its activation shim. The other
+`sci-` names are not identities either and keep their spelling: the
+`sci-delivery` delivery step and image, `sci-resident.conf`, the
+`sci-boot-*` invocation prefixes, the `sysext-*` branch names, and the
+base image's pre-existing `00-sci-libvirt.preset`.
+
+**Kernel modules.** Extensions ship only `modprobe.d` /
+`modules-load.d` fragments, never module binaries. The kvm modules are
+in-tree and proven on the target kernel, and out-of-tree modules cannot
+be shipped in a sysext. `sci-40-libvirt` ships the `kvm-intel` /
+`kvm-amd` install rules and a `modules-load.d` list. `sci-21-network`
+ships the OVS one. `sci-22-storage` ships the NFS client tuning.
+`systemd-modules-load` finishes before the merge at boot, so the
+shim's `modprobe` is the reliable path.
+
+**CH processes run as root** under libvirtd. libvirt 11.10 has no
+`ch.conf` and no `user`/`group` knob for the CH driver, so there is no
+non-root path, and `nova` needs no `kvm` membership. If upstream ever
+adds the knob, adopting it is a one-line `m nova kvm` in
+`sci-20-compute`.
+
+## 9. Activation model
+
+Extension-shipped units cannot be boot-enabled. PID 1 computes the
+whole boot transaction (the `default.target` chain, `Wants=`
+enumeration) several seconds *before* the sysext merge.
+`EXTENSION_RELOAD_MANAGER=1` makes the units known, but a
+daemon-reload cannot inject `Wants=` into an already-queued
+transaction. `/etc` symlinks to extension units fail too, because the
+unit files do not exist before the merge. `systemctl preset-all` runs
+at image build time only.
+
+So activation is **delivery-time activation plus a delivery-installed
+`/etc` shim**, and that is the permanent mechanism, not a stopgap. The
+initrd-merge path is structurally unavailable for the immutable
+target. `systemd-sysext(8)` states that initrd services cannot merge
+extensions from `/sysroot/var/lib/extensions` when `/var` is a separate
+partition, and a split `/var` is a defining property of the USI
+variants.
+
+The delivery step places the raws in `/var/lib/extensions`, runs
+`systemd-sysext refresh`, and enables the shim. Each extension ships a
+template at `usr/share/sci/activation/-activation.service`, named
+for the extension (`sci-20-compute-activation.service`, …). The
+basename minus `-activation.service` must equal the manifest entry.
+It copies the template to `/etc/systemd/system/` and enables it
+once. The shim works on stock GL because:
+
+- it exists before the merge, so the boot transaction includes it.
+- it runs `After=systemd-sysext.service` and carries
+ `Wants=systemd-sysext.service`, so a failed merge still lets it fail
+ loudly instead of being skipped.
+- it uses only base-image binaries: `systemd-sysusers`,
+ `systemd-tmpfiles`, `modprobe`, `systemctl`.
+
+It runs sysusers, then `systemd-tmpfiles --create`, then starts
+services **directly**. It never starts socket units, because systemd
+259 refuses "Socket service already active". The services' own
+`Wants=` co-start their sockets. Four extensions have shims
+(`sci-20-compute`, `sci-21-network`, `sci-22-storage`,
+`sci-40-libvirt`). `sci-10-base`, `sci-50-virt-ch` and
+`sci-51-virt-qemu` need none: the flavour drivers plug into `libvirtd`.
+
+`libvirtd` stays resident. `sci-resident.conf` clears the stock
+`--timeout 120`, so the service does not read `inactive` while its
+sockets keep serving. kvm-node-agent's `is-active` check would
+otherwise report a stopped hypervisor.
+
+**The update flow** (all bases):
+
+1. Replace the raws in `/var/lib/extensions` with the new versions.
+2. Run `systemctl daemon-reload` and `systemd-sysext refresh`.
+3. Run `systemctl reload dbus.service`.
+4. Run `systemctl restart` on the changed extensions' units (the lists
+ live in the manifest), or reboot.
+
+Step 3 exists because a merged extension can introduce D-Bus policy
+and activation files under `usr/share/dbus-1/`, and the daemon running
+since boot has not read them. `sci-50-virt-ch` is the real case: it
+carries `systemd-container` (a hard dependency of the CH driver
+package, with its `machine1` policy), so `systemd-machined.service`
+cannot acquire `org.freedesktop.machine1` before the reload. Boot is
+unaffected. dbus starts fresh after the merge and reads the merged
+policy anyway.
+
+`EXTENSION_RESTART_UNITS=` first appears in systemd 262 and restarts
+only units that are already running. GL 2150.x ships 259, where the
+field is silently ignored. It stays in the extension-release for
+forward compatibility. The manifest's restart lists are what the delivery
+step actually uses. A shim's `systemctl start` is a no-op for an
+already-active unit, so it does not cover step 4 either.
+
+Units that matter: `openvswitch-switch.service` (+
+`ovs-record-hostname`), `libvirtd.service` (+ its three sockets),
+`multipathd.service` and `iscsid.socket`. `multipathd.socket` is
+deliberately not started. The daemon creates its own sockets, and on a
+base that boot-enables `multipathd.service` the socket start fails
+deterministically. That would put the shim in a failed state on every
+boot.
+
+## 10. Verification
+
+### Host test strategy
+
+The host suite is `extensions/tests/`, run from `.build/venv-test`
+(`pytest==8.4.2`, PyYAML). The markers select three tiers: the quick
+tier (`not vm and not build`), the expensive `build` tier (image
+builds and the resolved-config/tools pins), and one `vm` item that
+boots three VMs. The tests that rewrite the shared `.build` state also
+carry `build_state_mutating`. Run the suite sequentially: one process
+owns the `.build/` scratch and KVM, so two runs must not overlap. There
+is no xdist, no collection hook and no `--selftest`. There is no
+automatic skip either: a missing dependency fails the check that needs
+it.
+
+Each module's docstring states the contract it pins. Fixtures live under
+pytest `tmp_path` or the case scratch, so the suite never mutates
+tracked files.
+
+**The acceptance pin.** The suite recomputes the resolved mkosi
+configuration, normalizes it with `extensions/tests/summary.py`, and
+compares the digest with `mkosi/summary.pin`. A conscious configuration
+or normalizer change moves the pin in the same commit. A comment-only
+edit does not move it.
+
+**Negative proofs are part of the contract.** A negative proof feeds a
+broken input and requires a refusal, because a fail-closed guard is a
+contract too. An acceptance-only suite would stay green while that
+guard silently opened.
+
+**The signed-delta guard.** The base owns the kernel, so an extension
+must carry no kernel and no verity anchor. The guard requires exactly
+three signed partitions, `ID=gardenlinux`, `SYSEXT_SCOPE=system`, no
+`usr/lib/verity.d` and no `usr/lib/modules/*/vmlinuz`. A kernel copy
+inside an extension would shadow the base and bind the extension to an
+unrelated ABI.
+
+### Boot verification
+
+Merge-at-boot is verifiable locally, without a shoot.
+`mkosi/boot-test.sh` calls `extensions/tests/boot_runner.py`, which
+generates a per-scenario config under
+`.build/test-scratch-vmboot/scenarios//`.
+`mkosi/vmtest/mkosi.extra/` is the overlay asset source (boot unit,
+cloud-init/tgt masks, anchor). Keeping the check out of
+`mkosi/mkosi.images/` avoids build-all and summary-pin coupling.
+
+Each generated image is `Format=disk` with the prepared published base
+tree as its root filesystem (`BaseTrees=`). The base tree's kernel is
+booted directly (`Linux=`, `Firmware=linux`, no initrd, no bootloader).
+`DiskType=nvme` because the GL kernel has `CONFIG_BLK_DEV_NVME=y` but
+`CONFIG_VIRTIO_BLK=m`: a virtio-blk root without an initrd panics.
+`KVM=yes` makes a missing `/dev/kvm` fail loudly instead of silently
+falling back to TCG. The booted identity is the published base's own
+`/etc/os-release` (`ID=gardenlinux`, `IMAGE_VERSION`). The runner
+stages no os-release.
+
+The overlay shapes the environment for emulation. An empty
+`etc/cloud/cloud-init.disabled` disables cloud-init through its own
+gate. Otherwise `cloud-init-network` retries the EC2 metadata endpoint
+for 240 s. `etc/systemd/system/tgt.service -> /dev/null` masks the
+flavor's iSCSI target daemon. It has no targets under QEMU, and its
+`ExecStop` stalls shutdown. The runner adds the runtime distribution at
+`/opt/sci-tests/dist`, a generated `run_guest.sh`, the
+expected-scenario JSON and the committed verity certificate. The guest
+unit runs after the merge attempt (`Wants=`, not `Requires=`), so a
+refusal is still a valid negative result. On this base the verity
+verification and refusal lines are journal-only, not hvc0. The guest
+verity test takes a journal cursor before `systemd-sysext refresh` and
+looks for the victim and the reason after it.
+
+The runtime archive is a build input, not a build product. The
+upstream `gardenlinux/tests/util` builder produces it and it is not
+committed, so it must be rebuilt when that upstream revision moves.
+
+The three scenarios are separate builds and boots:
+
+1. **signed set** — the boot runner stages the delivery manifest's
+ signed raws and the real `delivery/sci-delivery.sh` under `/opt/sci`.
+ The guest runs the delivery step with `HOST_ROOT=/`, expects all seven
+ extensions merged, asserts the seven numeric names, checks the fixed
+ identities, the tmpfiles/OVS link rule and active services, and runs
+ a second pass for idempotence. The staged payload is exercised
+ directly, not through the delivery image or Kubernetes.
+2. **read tamper** — the scenario places the signed raws in
+ `/var/lib/extensions` and flips one byte in the victim
+ `sci-20-compute`, at the root partition's superblock offset
+ (`start + 1024`). That is a block the consumer reads. The refresh
+ must fail, and no extension may merge.
+3. **policy refusal** — the scenario builds an unsigned probe, places
+ it in `/var/lib/extensions`, and tightens the policy to `[SysExt]
+ ImagePolicy=root=signed`. The refresh must refuse the probe.
+
+Each guest wrapper emits exactly one
+`SCI-RESULT:::` line to the
+console. The payload is JSON with the guest pytest return code and
+base64 JUnit XML. `runtime_adapter.parse_guest_result` rejects a
+missing or duplicate record, a mismatched invocation or scenario, a
+non-zero rc, unnamed cases, or any failure/error/skip. A clean
+poweroff alone is never trusted. `boot-test.sh` prints one final JSON
+object keyed by scenario with the counts and guest test names. The `vm`
+host item runs `--scenario all` and checks that JSON. The manifest
+defaults to `delivery/extensions.manifest` (`MANIFEST=` or
+`--manifest`). All state stays under `.build/test-scratch-vmboot/`, so
+the shared build output is never written.
+
+Not covered, by design:
+
+- SAN/iSCSI hardware discovery. The guest checks the published base's
+ binaries and services, not a fabric.
+- The flavor's cloud-init and iSCSI target services. The overlay
+ disables them, and the boot assertion is the sysext merge, not the
+ flavor's cloud plumbing.
+- aarch64 execution. The runtime archive contains both architectures,
+ but only x86-64 boots here.
+
+## 11. Delivery path
+
+`delivery/` carries the delivery step, which reconciles a staged set
+on an interval. The supported output is the signed release: every
+manifest raw, every activation shim, `extensions.manifest`,
+`verity-run.crt`, `SHA256SUMS` and `SHA256SUMS.sig`. The `publish` job
+creates it as release `-g` when the signing key is
+configured. Without the key the release steps are skipped and no
+release exists. The release carries no delivery program or scripts:
+`sci-delivery.sh` stays in the test artifact. The boot test and
+integration job prove the merge and activation. Fetch mode only runs
+the same test vehicle. A consumer that reuses it owns selection,
+staging and restart windows.
+
+**Remote delivery.** `FETCH_URL` switches the step to fetch mode: it
+downloads one build under `FETCH_DIR` (default `/var/lib/sci-sysext`),
+staging at `$FETCH_DIR/staging`. It requires `FETCH_PUBKEY`, the PEM
+public key that verifies `SHA256SUMS.sig`, and compares the optional
+`FETCH_SUMS_DIGEST` with the sums. It reads the sums as its inventory,
+downloads only the files whose staged hash differs, proves the complete
+staged set with `sha256sum -c`, and prunes files the inventory no longer
+lists. Each raw's dm-verity signature checks against the base image's
+`/usr/lib/verity.d/verity.crt`, which no delivery path can install. Any
+failure exits non-zero and writes nothing to the host. A converged pass
+needs a staging directory that outlives the container. The shipped
+image keeps `STAGING_DIR` and `FETCH_DIR` in the container layer.
+
+**Tombstones.** A manifest may carry `REMOVE=` lines. For each
+name the delivery step deletes `$HOST_ROOT/var/lib/extensions/.raw`,
+`$HOST_ROOT/etc/systemd/system/-activation.service` and its
+`multi-user.target.wants` enable symlink, then runs the normal
+activation flow. Removal is idempotent: a name that is
+already absent is not a change. A `REMOVE` name that is also an entry,
+a duplicate `REMOVE` name or a traversing name fails validation, so a
+tombstone cannot unmerge what the same manifest also deploys, and it
+cannot reach outside the two deployment trees. This is the only
+deletion of a host extension the delivery step performs. Version retention
+stays out of `/var/lib/extensions` on purpose (section 13).
+
+**Base-version gate.** systemd-sysext binds an extension to the host by
+`ID=` alone, so a 2150 build merges on any Garden Linux line and
+silently mixes ABIs. The manifest therefore carries a required
+`TARGET_BASE=` directive, the Garden Linux major line the
+extensions target (the committed manifest carries
+`TARGET_BASE=2150`). Directives fail closed: an unknown directive is a
+validation error, so a typo cannot leave the gate unset. After
+manifest validation and before any file deployment, the delivery step reads
+`$HOST_ROOT/etc/os-release`, takes the first dot-separated component of
+`IMAGE_VERSION` (fallback `GARDENLINUX_VERSION`), and compares it with
+`TARGET_BASE`. A mismatch, a missing field or an unreadable os-release
+is a loud no-op: non-zero exit naming the host line and `TARGET_BASE`,
+no raw, no shim, no marker, no tool call, and never a delete. The gate
+runs on every pass, before a pass can decide there is nothing to do.
+
+**Staged-raw integrity.** Right after manifest validation and *before*
+the gate, every manifest entry must have a `SHA256SUMS` line, and every
+staged raw must hash to it. A missing file, a missing line or a
+mismatch is fatal and deploys nothing. Integrity runs first, so the
+delivery step reports a corrupt staging set as such even on a mismatched
+host. The md5 comparison inside the pass is a different check: it detects
+deployed-state drift against the staged copy.
+
+`delivery/build-image.sh` stages the raws and shims for a manifest,
+stages `SHA256SUMS` (preferring mkosi's own if it covers every manifest
+raw and still matches), verifies it with `sha256sum -c`, and tags the
+image `sci-delivery:latest` only. The image is a test artifact, not a
+published product. It carries no tag that names a producing commit.
+`--stage-only` stops after staging and the `SHA256SUMS` verification:
+no docker build, no tag. It exists for the `sysext-raws` artifact.
+`delivery/Containerfile` pins its Debian base by index digest, not by
+tag.
+
+**CI integration.** The delivery pipeline is
+`.github/workflows/sysext.yml`, which builds, signs and publishes.
+`.github/workflows/test.yml` consumes its output. The workflow files are
+the source of truth for their own wiring, and `test_workflows` pins that
+structure. `test_prepare_sysext_hv` pins the helper script's behaviour.
+In short: the producer's quick tier runs on every push to `sysext-*` and
+on PRs (`paths-ignore` for docs and READMEs). Its `producer` job signs
+with `secrets.SYSEXT_UNHARDENED_SIGNING_KEY` when the repository secret
+exists and with a run-local CA otherwise — the key at
+`.build/keys/sysext-verity.key` mode
+600, the certificate overwriting the working-tree `mkosi/verity.crt`,
+never committed — then builds the venv, ensures the base image, lifts
+the runner's AppArmor restriction on unprivileged user namespaces
+(mkosi's sandbox builds unprivileged), builds the seven raws and runs
+the full suite inside the job before any artifact upload. The test
+artifact is `sysext-raws` (the staging as plain files: every raw,
+`SHA256SUMS` over all of them, the shims, the manifest, the run cert
+and the delivery step), `actions/upload-artifact` pinned to v7.0.2. The
+image and the toolchain are test-side derivations, so uploading them
+would publish test scaffolding as if it were the product. The boot test
+uses the full set, and fetch mode deploys the release's manifest.
+The private key never travels in an artifact. The release carries only
+its certificate. The third job, `integration` (needs
+`producer`, skipped on PRs), calls `test.yml` with the raws artifact name.
+It declares its own permissions, `contents: read` plus `actions: read`,
+because a called workflow can only downgrade the token the caller
+passes down, never elevate it. A fourth job, `publish` (`needs:
+producer`, `contents: write`, skipped on PRs), derives the signed
+release from the same artifact: it drops the delivery step, regenerates
+the sums over the remaining files, signs them, and uploads that set as
+release `-g`. Without the signing secret the release
+steps are skipped and no release exists.
+
+`test.yml` accepts exactly one of `image_tag` (legacy) or
+`sysext_artifact` (the raws artifact), and optionally reads
+`artifact_run_id` (default: the current run). The `sysext-raws` artifact
+is the integration test's input. It carries the signed extensions, the
+manifest, the shims, `SHA256SUMS` and the run certificate, and the
+release is derived from it. It restores the run certificate
+(`.build/ci-sysext-hv/raws/verity-run.crt` over `mkosi/verity.crt`)
+when the artifact carries one, bootstraps the toolchain with
+`mkosi/build.sh --bootstrap` (sandbox assembly, pinned tools tree
+verified against `mkosi/tools.pin`, published base), preflights it
+with `.github/scripts/prepare-sysext-hv.sh --check`, and rebuilds
+the delivery image with `delivery/build-image.sh --from-staging`:
+the staged `SHA256SUMS` must check the shipped raws, and the image
+carries only the unversioned tag because the test checkout need not
+be the producing commit. The cert restore is
+what makes the run self-consistent: `--prepare` copies the checkout's
+`mkosi/verity.crt` into the HV overlay as the dm-verity trust anchor,
+and a foreign anchor refuses every merge. A tolerant `else` keeps the
+committed certificate when the artifact has no run cert. The manual
+local flow signs with the machine-local dev key. Only its certificate
+is committed. `--prepare` also grows the HV root disk with the tools
+tree's `systemd-repart`. That binary loads its private
+`libsystemd-shared` through an absolute host `RUNPATH`, so the script
+points `LD_LIBRARY_PATH` back into the tools tree. A host with a newer
+systemd does not carry the pinned library. The prepared
+disks carry mode 666: the integration libvirt runs qemu as an
+unprivileged user and sets `security_driver=none`, so no ownership
+transfer happens at domain start and the disk itself must grant the
+write access. The setup step also grants `o+x` on the runner home: the
+runner images ship it as mode 750, and the qemu user must traverse it
+to reach the workspace paths. The delivery `.ref` carries the
+fully-qualified image name: the consumer's `ctr` resolves image names
+literally, with no docker shortname normalization, so a short ref
+cannot address the imported image. The HV root overlay pins `hv1` and
+`hv2` in `/etc/hosts`: libvirt derives the p2p migration address from
+the destination hostname, and the guests run no upstream DNS. The HV
+root defaults to 7G on an 8G disk: the Cloud-Hypervisor stage writes
+the full 2.2G Ubuntu raw into the guest, next to the staged qcow2 and
+the extensions.
+
+What is proven, and what is not: a pushed run proves the whole chain
+on runner hardware — `workflow_call` execution, the same-run artifact
+download, the single `sysext-raws` upload, the test-side bootstrap
+and rebuild, and both live migrations. Local `act` runs cannot prove
+the upload (act's artifact server rejects `upload-artifact` v7.0.2) or
+`test_reproducible` (the nested sandbox bakes differing
+`user.overlay.origin` xattrs). Run-local signing makes CI artifacts
+test-only by construction.
+
+## 12. Immutable systems
+
+The mechanism lands on the immutable target without special-casing:
+
+| Concern | Immutable (USI) mechanics |
+|---|---|
+| Extension storage | `/var/lib/extensions` — writable by requirement |
+| Activation shim | `/etc/systemd/system/` — the `/var`-backed overlay, persists across reboots |
+| Users/groups | sysusers writes `/etc/passwd` and `/etc/group` in the same overlay |
+| State dirs | `/var/...` via tmpfiles — writable |
+| Merging | a read-only EROFS `/usr` is sysext's native case |
+| Base updates | a UKI A/B update leaves `/var` untouched: extensions, shim and users survive, ID-only binding re-merges, the shim re-runs after `systemd-sysext.service` |
+| Kernel modules | the kernel lives in the UKI. kvm and openvswitch are in-tree |
+
+The remaining gap in the USI line is a base-flavour question, not a
+mechanism question. The transitional `sci_usi` image bakes the full SCI
+stack (a libvirt backport plus pre-created identities), which is the
+very problem this migration eliminates. The clean target is a plain
+ignition-capable USI base.
+
+## 13. Decisions and rationale
+
+- **ID-only version binding, gated at delivery time.**
+ `extension-release` carries `ID=gardenlinux` and no `VERSION_ID` /
+ `SYSEXT_LEVEL`, because GL os-release sets neither, and the
+ CloudProfile minor-update strategy drifts the base across 2150.x
+ within a stable ABI. systemd cannot express the base line, so the
+ manifest's `TARGET_BASE` does it in the delivery step. The upstream
+ `SYSEXT_LEVEL` proposal (nova-issues#909) would make systemd do it
+ natively. The gate is explicitly a bridge.
+- **One build line per Garden Linux line.** Only 2150 needs care now.
+ If another line ever needs these extensions, the answer is a separate
+ build line — the mechanical release-suite re-pin of section 5 — not a
+ conditional build. Until then a 1877 host is a loud, documented no-op
+ at the gate.
+- **The delivery step is the boot-test vehicle.** It exists to prove, in the
+ boot test and the integration job, that a built set merges and
+ activates. The supported output is the signed release. A consumer
+ that reuses the step owns selection, staging persistence, key and
+ anchor provisioning, and restart windows.
+- **Reproducibility is an input contract, not a hope.** Version-keyed
+ timestamps and per-image seeds, plus the three pins, make a rebuild
+ byte-identical and a changed input a reviewable commit. A delivered
+ raw can therefore be traced to its inputs and re-verified.
+- **Integrity at activation, not only at delivery.** The embedded
+ verity signature is what a booting target can check by itself. The
+ delivery integrity check protects the payload before it lands.
+ Neither replaces the other.
+- **Test-first, per behaviour.** The pytest modules are the
+ specification, and section 10 states the rules they enforce.
+- **The test run is sequential.** One pytest process owns `.build/`
+ scratch and KVM. There is no xdist and no collection hook. The
+ build-marked reproducibility item rewrites the shared output, and the
+ vm item owns the boot scratch, so two runs must not overlap.
+- **Activation is delivery-time, permanently.** The initrd path cannot
+ serve split-`/var` systems, so the `/etc` shim is the design, not a
+ stopgap.
+- **Deletion is explicit, never implicit.** The delivery step deletes a host
+ extension only for a `REMOVE=` line that the signed manifest
+ carries. A merge directory cannot hold two versions of one extension:
+ systemd-sysext merges every `*.raw`, so a leftover copy would merge
+ beside the new one instead of staying dormant. Trimming an entry
+ therefore unmerges nothing, and a removal is a decision an operator
+ writes down and signs.
+- **Version retention stays out of the merge directory.**
+ `systemd-sysupdate` names files with the version and keeps two
+ instances. Both copies would merge, and files that the newer version
+ dropped would survive the update. sysupdate may fetch, but it must
+ not own `/var/lib/extensions`.
+- **Single owner per path.** Overlay shadowing is silent. One owner
+ plus the artifact and guest contracts is the only way to keep
+ identities and configs from diverging across hosts.
+- **`/etc` content has narrow carriers.** A sysext merges `/usr` only.
+ Most configuration has a `/usr/lib` fallback, and the one file that
+ does not — libvirt's `qemu.conf` — is materialized at activation by a
+ tmpfiles `C` rule from a template in the qemu extension. That carrier
+ is copy-once and fits single files. Broader `/etc` content still needs
+ a decision: confexts can carry it, but a merged confext mounts `/etc`
+ read-only, the riskiest semantic for a hypervisor host. The 2150 base
+ ships `systemd-confext` (259.7), so the runtime support exists.
+ Whether confexts arrive as a separate artifact set or as manifest
+ entries remains open.
+- **Sequential composition over one big image.** Separating identities,
+ networking, storage and libvirt means a change to one does not churn
+ the others. `sci-20-compute` keeps the chart-consumed UID contract
+ stable across virt flavours, and a composition can drop what it does
+ not need.
diff --git a/extensions/tests/boot_runner.py b/extensions/tests/boot_runner.py
new file mode 100644
index 00000000..af606f8a
--- /dev/null
+++ b/extensions/tests/boot_runner.py
@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+"""Host helper for mkosi/boot-test.sh: stage the runtime, build one isolated
+image per scenario, boot it, and return the runtime adapter's counts."""
+import argparse, hashlib, json, os, re, shutil, signal, subprocess, sys, uuid
+from pathlib import Path
+HERE = Path(__file__).resolve().parent
+sys.path.insert(0, str(HERE))
+import runtime_adapter # noqa: E402
+TYPES = ("4f68bce3-e8cd-4db1-96e7-fbcaf984b709", "2c7357ed-ebd2-46d9-aec1-23d437ec2bf5",
+ "41092b05-9fc8-4523-994f-2def0408b176")
+MODULES = {"signed": "test_sysext.py", "tamper": "test_verity.py", "policy": "test_verity.py"}
+VICTIMS = {"tamper": "sci-20-compute", "policy": "unsignedprobe"}
+WRAPPER = r'''#!/bin/sh
+set -u
+dist=/opt/sci-tests/dist; arch=$(uname -m); log=/run/sci-pytest.log; xml=/run/sci-junit.xml
+scenario=${SCI_SCENARIO:-}; invocation=${SCI_INVOCATION:-}
+[ -d /run/systemd/system ] && [ -x "$dist/run_tests" ] && [ -f /opt/sci/expected.json ] || { printf 'SCI-GUEST: runtime contract missing\n' >/dev/console; exit 1; }
+case "$scenario" in signed) module=test_sysext.py;; tamper|policy) module=test_verity.py;; *) printf 'SCI-GUEST: invalid scenario %s\n' "$scenario" >/dev/console; exit 1;; esac
+rc=0
+SCI_GUEST=1 SCI_SCENARIO="$scenario" SCI_EXPECTED_JSON=/opt/sci/expected.json TEST_NG_EXIT_0=0 PYTHONDONTWRITEBYTECODE=1 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 SYSTEMD_LOG_TARGET=journal \
+"$dist/run_tests" --system-booted --allow-system-modifications -p pytest_metadata.plugin --show-capture=no -o junit_logging=no --junitxml="$xml" "integration/sci/$module" >"$log" 2>&1 || rc=$?
+"$dist/runtime/$arch/bin/python" - "$rc" "$invocation" "$scenario" "$xml" <<'PY'
+import base64,json,os,sys,xml.etree.ElementTree as ET
+rc,inv,scen=int(sys.argv[1]),sys.argv[2],sys.argv[3]
+try: root=ET.parse(sys.argv[4]).getroot()
+except Exception: junit=b""
+else:
+ [p.remove(c) for p in root.iter() for c in list(p) if c.tag in ("system-out","system-err")]
+ for e in root.iter():
+ if e.tag in ("failure","error"):
+ if e.get("message"): e.set("message", e.get("message")[:4096])
+ if e.text: e.text=e.text[:4096]
+ junit=ET.tostring(root,encoding="utf-8")
+payload={"rc":rc,"junit":base64.b64encode(junit).decode()}
+line="\nSCI-RESULT:%s:%s:%s\n"%(inv,scen,base64.b64encode(json.dumps(payload).encode()).decode())
+fd=os.open("/dev/console",os.O_WRONLY); os.write(fd,line.encode()); os.close(fd)
+PY
+if [ "$rc" -ne 0 ]; then printf 'SCI-GUEST-DIAG: begin\n' >/dev/console; tail -n 100 "$log" >/dev/console 2>/dev/null || true; printf 'SCI-GUEST-DIAG: end\n' >/dev/console; fi
+exit 0
+'''
+def die(message): raise SystemExit(f"boot-runner: error: {message}")
+def log(message): print(f"boot-runner: {message}", file=sys.stderr, flush=True)
+def rm(path):
+ if path.is_symlink() or path.is_file(): path.unlink()
+ elif path.exists(): shutil.rmtree(path)
+def copy_file(src, dst, mode=None):
+ dst.parent.mkdir(parents=True, exist_ok=True); shutil.copy2(src, dst)
+ if mode is not None: dst.chmod(mode)
+def names(path):
+ out, seen = [], set()
+ for lineno, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
+ line = raw.strip()
+ if not line or line.startswith("#"): continue
+ directive = re.fullmatch(r"([A-Z][A-Z0-9_]*)=\S*", line)
+ if directive:
+ if directive.group(1) != "TARGET_BASE":
+ raise ValueError(f"invalid manifest: unknown directive {directive.group(1)!r} at line {lineno}: {path}")
+ continue
+ name = line.split()[0]
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]*", name):
+ raise ValueError(f"invalid manifest: invalid name {name!r} at line {lineno}: {path}")
+ if name in seen:
+ raise ValueError(f"invalid manifest: duplicate entry {name!r} at line {lineno}: {path}")
+ seen.add(name); out.append(name)
+ if not out: raise ValueError(f"invalid manifest: no entries: {path}")
+ return out
+def identity(base):
+ values = {key: value.strip('"') for key, sep, value
+ in (line.partition("=") for line in (base / "etc/os-release").read_text().splitlines())
+ if sep and not key.startswith("#")}
+ return {"id": values["ID"], "image_version": values["IMAGE_VERSION"]}
+def signed_shape(raw):
+ parts = json.loads(subprocess.run(["sfdisk", "--json", str(raw)], check=True,
+ capture_output=True, text=True).stdout)["partitiontable"]["partitions"]
+ if [p["type"].lower() for p in parts] != list(TYPES):
+ die(f"{raw} is not a 3-partition signed sysext: {[p['type'] for p in parts]}")
+ return parts
+def write_conf(directory, release, base, tools, sandbox, cache, output, extras, kernel=None, probe=None):
+ text = (f"[Distribution]\nDistribution=debian\nRelease={release}\n\n[Build]\nToolsTree={tools}\n"
+ f"SandboxTrees={sandbox}\nCacheDirectory={cache}\nWithNetwork=no\n\n[Output]\n")
+ if probe:
+ text += (f"Format=sysext\nOverlay=yes\nOutput=unsignedprobe\nOutputDirectory={output}\n\n"
+ f"[Content]\nBaseTrees={base}\nPackages=\nExtraTrees={probe}:/\n")
+ else:
+ text += f"Format=disk\nOutput=sysext-vmtest\nOutputDirectory={output}\n\n[Content]\nBaseTrees={base}\nPackages=\n"
+ text += "".join(f"ExtraTrees={src}:{target}\n" for src, target in extras)
+ text += f"Bootable=no\n\n[Runtime]\nLinux={kernel}\nFirmware=linux\nDiskType=nvme\nKVM=yes\n"
+ (directory / "mkosi.conf").write_text(text, encoding="utf-8")
+def build_probe(mkosi, directory, release, base, tools, sandbox, cache, output):
+ rm(directory)
+ release_file = directory / "extra/usr/lib/extension-release.d/extension-release.unsignedprobe"
+ release_file.parent.mkdir(parents=True)
+ release_file.write_text("ID=gardenlinux\nSYSEXT_SCOPE=system\n", encoding="utf-8")
+ write_conf(directory, release, base, tools, sandbox, cache, output, [], probe=directory / "extra")
+ spawn([str(mkosi), "-f", "build"], directory, directory / "build.log", 900)
+ raw = output / "unsignedprobe.raw"
+ if not raw.is_file(): die(f"unsignedprobe build produced no {raw}")
+ return raw
+def stage_delivery(target, raw_names, model):
+ target.mkdir(parents=True, exist_ok=True)
+ sums = []
+ for name in raw_names:
+ source = model["raws"] / f"{name}.raw"
+ signed_shape(source)
+ copy_file(source, target / f"{name}.raw")
+ sums.append(f"{hashlib.sha256(source.read_bytes()).hexdigest()} {name}.raw")
+ (target / "SHA256SUMS").write_text("\n".join(sums) + "\n", encoding="utf-8")
+ copy_file(model["manifest"], target / "extensions.manifest")
+ for name in raw_names:
+ shim = model["shims"] / name / f"mkosi.extra/usr/share/sci/activation/{name}-activation.service"
+ if shim.is_file(): copy_file(shim, target / shim.name)
+ copy_file(model["agent"], target / "sci-delivery.sh", 0o755)
+def prepare(directory, scenario, raw_names, base, model, tools_extra, cert):
+ rm(directory)
+ overlay = directory / "overlay"
+ for rel in ("etc/cloud/cloud-init.disabled", "etc/systemd/system/vm-boot-test.service"):
+ copy_file(tools_extra / rel, overlay / rel)
+ wants = overlay / "etc/systemd/system/multi-user.target.wants"
+ wants.mkdir(parents=True); os.symlink("../vm-boot-test.service", wants / "vm-boot-test.service")
+ os.symlink("/dev/null", overlay / "etc/systemd/system/tgt.service")
+ copy_file(cert, overlay / "usr/lib/verity.d/verity.crt")
+ invocation = f"sci-boot-{scenario}-{uuid.uuid4().hex}"
+ dropin = overlay / "etc/systemd/system/vm-boot-test.service.d/scenario.conf"
+ dropin.parent.mkdir(parents=True)
+ dropin.write_text(f"[Service]\nEnvironment=SCI_SCENARIO={scenario}\nEnvironment=SCI_INVOCATION={invocation}\n", encoding="utf-8")
+ wrapper = overlay / "opt/sci-tests/run_guest.sh"
+ wrapper.parent.mkdir(parents=True); wrapper.write_text(WRAPPER, encoding="utf-8"); wrapper.chmod(0o755)
+ expected = {"scenario": scenario, "extensions": raw_names, "base": base}
+ if scenario in VICTIMS: expected["victim"] = VICTIMS[scenario]
+ expected_path = overlay / "opt/sci/expected.json"
+ expected_path.parent.mkdir(parents=True, exist_ok=True)
+ expected_path.write_text(json.dumps(expected) + "\n", encoding="utf-8")
+ (overlay / "var/lib/extensions").mkdir(parents=True)
+ if scenario == "signed":
+ stage_delivery(overlay / "opt/sci", raw_names, model)
+ elif scenario == "tamper":
+ for name in raw_names:
+ signed_shape(model["raws"] / f"{name}.raw")
+ copy_file(model["raws"] / f"{name}.raw", overlay / "var/lib/extensions" / f"{name}.raw")
+ target = overlay / "var/lib/extensions/sci-20-compute.raw"
+ parts = signed_shape(target)
+ with target.open("r+b") as handle:
+ handle.seek(parts[0]["start"] * 512 + 1024)
+ byte = handle.read(1); handle.seek(-1, os.SEEK_CUR); handle.write(bytes([byte[0] ^ 0xFF]))
+ elif scenario == "policy":
+ copy_file(model["probe"], overlay / "var/lib/extensions/unsignedprobe.raw")
+ (overlay / "etc/systemd/sysext.conf").write_text("[SysExt]\nImagePolicy=root=signed\n", encoding="utf-8")
+ return overlay, invocation
+def spawn(command, cwd, path, timeout):
+ with path.open("wb") as output:
+ process = subprocess.Popen(command, cwd=str(cwd), stdout=output, stderr=subprocess.STDOUT, start_new_session=True)
+ try: process.wait(timeout=timeout)
+ except subprocess.TimeoutExpired:
+ try: os.killpg(process.pid, signal.SIGKILL)
+ except ProcessLookupError: pass
+ process.wait()
+ die(f"{' '.join(command)} timed out after {timeout}s; see {path}")
+ return subprocess.CompletedProcess(command, process.returncode)
+def main(argv):
+ if argv[:1] == ["--preflight"]:
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--manifest", required=True, type=Path)
+ args = parser.parse_args(argv[1:])
+ try: names(args.manifest)
+ except ValueError as exc:
+ print(f"boot-runner: error: {exc}", file=sys.stderr)
+ return 2
+ return 0
+ parser = argparse.ArgumentParser()
+ for flag, kind in (("--scenario", str), ("--manifest", Path), ("--runtime", Path),
+ ("--repo-root", Path), ("--scratch", Path), ("--dist", Path),
+ ("--base", Path), ("--kernel", Path), ("--tools", Path),
+ ("--sandbox", Path), ("--cert", Path), ("--cache", Path), ("--mkosi", Path)):
+ parser.add_argument(flag, required=True, type=kind)
+ args = parser.parse_args(argv)
+ if args.scenario not in (*MODULES, "all"): die(f"unknown scenario {args.scenario!r}")
+ if not args.runtime.is_file(): die(f"missing runtime archive: {args.runtime}")
+ repo, manifest = args.repo_root, args.manifest.resolve()
+ try: raw_names = names(manifest)
+ except ValueError as exc: die(str(exc))
+ base = identity(args.base)
+ release = re.search(r"^Release=(.*)$", (repo / "mkosi/mkosi.conf").read_text(encoding="utf-8"), re.M).group(1).strip()
+ rm(args.dist)
+ runtime_adapter.stage_distribution(args.runtime, args.dist, repo, repo / "extensions/tests/guest")
+ model = {"manifest": manifest, "raws": repo / ".build/mkosi/output",
+ "shims": repo / "mkosi/mkosi.images", "agent": repo / "delivery/sci-delivery.sh", "probe": None}
+ results = {}
+ for scenario in (MODULES if args.scenario == "all" else (args.scenario,)):
+ directory = args.scratch / "scenarios" / scenario
+ if scenario == "policy":
+ probe_dir = args.scratch / "unsignedprobe"
+ model["probe"] = build_probe(args.mkosi, probe_dir, release, args.base, args.tools,
+ args.sandbox, args.cache, probe_dir / "image")
+ overlay, invocation = prepare(directory, scenario, raw_names, base, model,
+ repo / "mkosi/vmtest/mkosi.extra", args.cert)
+ write_conf(directory, release, args.base, args.tools, args.sandbox, args.cache,
+ directory / "image", [(overlay, "/"), (args.dist, "/opt/sci-tests/dist")], args.kernel)
+ log(f"{scenario}: building {directory / 'mkosi.conf'}")
+ if spawn([str(args.mkosi), "-f", "build"], directory, directory / "build.log", 1800).returncode:
+ die(f"{scenario}: image build failed; see {directory / 'build.log'}")
+ log(f"{scenario}: booting")
+ started = spawn([str(args.mkosi), "qemu"], directory, directory / "console.log", 900)
+ text = (directory / "console.log").read_text(encoding="utf-8", errors="replace")
+ try:
+ counts = runtime_adapter.parse_guest_result(text, invocation, scenario)
+ except ValueError as exc:
+ log(f"{scenario}: FAIL: {exc}; qemu rc={started.returncode}; see {directory / 'console.log'}")
+ results[scenario] = {"rc": 1, "tests": 0, "failures": 0, "errors": 1, "skipped": 0,
+ "names": [], "reason": str(exc)}
+ continue
+ if started.returncode:
+ die(f"{scenario}: mkosi qemu exited {started.returncode}; see {directory / 'console.log'}")
+ (directory / "result.json").write_text(json.dumps(counts) + "\n", encoding="utf-8")
+ results[scenario] = counts
+ log(f"{scenario}: {counts['tests']} tests ok: {', '.join(counts['names'])}")
+ print(json.dumps(results, separators=(",", ":")))
+ return 0 if all(entry["rc"] == 0 for entry in results.values()) else 1
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
diff --git a/extensions/tests/conftest.py b/extensions/tests/conftest.py
new file mode 100644
index 00000000..0807ff63
--- /dev/null
+++ b/extensions/tests/conftest.py
@@ -0,0 +1,23 @@
+"""Minimal pytest configuration for the extension boundary tests.
+
+The guest suite under ``guest/`` runs inside the VM image, not on the host.
+"""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import pytest
+
+collect_ignore = ["guest"]
+
+_HERE = Path(__file__).resolve().parent
+if str(_HERE) not in sys.path:
+ sys.path.insert(0, str(_HERE))
+
+
+@pytest.fixture(scope="session")
+def repo_root() -> Path:
+ """The resolved repository root."""
+ return Path(__file__).resolve().parents[2]
diff --git a/extensions/tests/data/summary-sample.json b/extensions/tests/data/summary-sample.json
new file mode 100644
index 00000000..de862dbd
--- /dev/null
+++ b/extensions/tests/data/summary-sample.json
@@ -0,0 +1,43 @@
+{
+ "Images": [
+ {
+ "Architecture": "x86-64",
+ "BaseTrees": [
+ "/repo/.build/mkosi/base-image/current",
+ "/repo/mkosi/mkosi.images/sci-libvirt"
+ ],
+ "Distribution": "gardenlinux",
+ "Format": "sysext",
+ "Image": "sci-base",
+ "Note": "h\u00e9llo w\u00f6rld \u2713",
+ "Output": "/repo/.build/mkosi/output/sci-base.raw",
+ "Packages": [
+ "conntrack",
+ "parted"
+ ],
+ "Release": "2150.9.0",
+ "Seed": "8b7f2b0e-9c4a-4a1e-9a55-0c3d2e1f6a77",
+ "Scale": 1.5,
+ "SourceDateEpoch": 1771372800,
+ "Tags": []
+ },
+ {
+ "Architecture": "x86-64",
+ "Distribution": "gardenlinux",
+ "Flag": true,
+ "Format": "disk",
+ "Image": "main",
+ "Output": "/var/tmp/machine-local/main.raw",
+ "RepoRoot": "/repo",
+ "Release": "2150.9.0",
+ "Settings": {
+ "CaBundle": "/etc/ssl/certs/ca-certificates.crt",
+ "Empty": {},
+ "Relative": "mkosi.conf",
+ "Value": null
+ },
+ "SourceDateEpoch": 1771372800
+ }
+ ],
+ "Version": "27.1"
+}
diff --git a/extensions/tests/data/summary-sample.normalized.json b/extensions/tests/data/summary-sample.normalized.json
new file mode 100644
index 00000000..0ff0fb1e
--- /dev/null
+++ b/extensions/tests/data/summary-sample.normalized.json
@@ -0,0 +1,48 @@
+{
+ "Images": [
+ {
+ "Architecture": "x86-64",
+ "BaseTrees": [
+ "@REPO/.build/mkosi/base-image/current",
+ "@REPO/mkosi/mkosi.images/sci-libvirt"
+ ],
+ "DelegateRanges": null,
+ "Distribution": "gardenlinux",
+ "Format": "sysext",
+ "Image": "sci-base",
+ "Note": "héllo wörld ✓",
+ "Output": "@REPO/.build/mkosi/output/sci-base.raw",
+ "Packages": [
+ "conntrack",
+ "parted"
+ ],
+ "ProxyExclude": null,
+ "Release": "2150.9.0",
+ "Scale": 1.5,
+ "Seed": null,
+ "SourceDateEpoch": 1771372800,
+ "Tags": []
+ },
+ {
+ "Architecture": "x86-64",
+ "DelegateRanges": null,
+ "Distribution": "gardenlinux",
+ "Flag": true,
+ "Format": "disk",
+ "Image": "main",
+ "Output": "@ABS",
+ "ProxyExclude": null,
+ "Release": "2150.9.0",
+ "RepoRoot": "@ABS",
+ "Seed": null,
+ "Settings": {
+ "CaBundle": "@ABS",
+ "Empty": {},
+ "Relative": "mkosi.conf",
+ "Value": null
+ },
+ "SourceDateEpoch": 1771372800
+ }
+ ],
+ "Version": "27.1"
+}
diff --git a/extensions/tests/guest/test_sysext.py b/extensions/tests/guest/test_sysext.py
new file mode 100644
index 00000000..1f627a21
--- /dev/null
+++ b/extensions/tests/guest/test_sysext.py
@@ -0,0 +1,174 @@
+"""Run through the staged Garden Linux runtime, never on the build host."""
+import grp
+import json
+import os
+from pathlib import Path
+import pwd
+import subprocess
+
+EXTENSIONS = {"sci-10-base", "sci-20-compute", "sci-21-network", "sci-22-storage",
+ "sci-40-libvirt", "sci-50-virt-ch", "sci-51-virt-qemu"}
+
+
+def run(*args, **kwargs):
+ return subprocess.run(args, text=True, capture_output=True, timeout=120, **kwargs)
+
+
+def guest_expected(scenarios):
+ # Check all preconditions before any command can change the guest.
+ guest = os.environ.get("SCI_GUEST")
+ assert guest == "1", "SCI_GUEST=1 is required"
+ path = os.environ.get("SCI_EXPECTED_JSON")
+ assert path == "/opt/sci/expected.json", "SCI_EXPECTED_JSON must name the staged contract"
+ expected = json.loads(Path(path).read_text())
+ assert expected["scenario"] in scenarios
+ assert len(expected["extensions"]) == 7
+ assert set(expected["extensions"]) == EXTENSIONS
+ assert expected["base"] == {"id": "gardenlinux", "image_version": "2150.9.0"}
+ actual = dict(line.split("=", 1) for line in Path("/etc/os-release").read_text().splitlines()
+ if "=" in line and not line.startswith("#"))
+ assert actual["ID"].strip('"') == expected["base"]["id"]
+ assert actual["IMAGE_VERSION"].strip('"') == expected["base"]["image_version"]
+ return expected
+
+
+def merged_extensions():
+ status = run("systemd-sysext", "status", "--json=short")
+ assert status.returncode == 0, status.stderr
+ rows = json.loads(status.stdout)
+ assert isinstance(rows, list) and rows, status.stdout
+ merged = set()
+ for row in rows:
+ assert "hierarchy" in row and "extensions" in row, row
+ value = row["extensions"]
+ names = value if isinstance(value, list) else value.split()
+ merged.update(name for name in names if name != "none")
+ return merged
+
+
+def test_delivery_and_host_contract():
+ guest_expected({"signed"})
+ destination = Path("/var/lib/extensions")
+ assert not destination.exists() or not list(destination.iterdir())
+ base = {p: Path(p).read_bytes() for p in ("/etc/os-release", "/usr/lib/os-release")}
+ staging = Path("/opt/sci")
+ for name in EXTENSIONS:
+ assert (staging / f"{name}.raw").is_file()
+ for name in ("sci-20-compute", "sci-21-network", "sci-22-storage", "sci-40-libvirt"):
+ assert (staging / f"{name}-activation.service").is_file()
+ for name in ("sci-delivery.sh", "extensions.manifest", "SHA256SUMS"):
+ assert (staging / name).is_file()
+ env = dict(os.environ, HOST_ROOT="/", STAGING_DIR=str(staging), INTERVAL="0", NSENTER="0")
+ first = run("bash", str(staging / "sci-delivery.sh"), env=env)
+ assert first.returncode == 0, first.stdout + first.stderr
+ assert merged_extensions() == EXTENSIONS
+ qemu_conf = Path("/etc/libvirt/qemu.conf")
+ assert qemu_conf.is_file() and not qemu_conf.is_symlink(), qemu_conf
+ qemu_text = qemu_conf.read_text()
+ assert "remote_display_port_min = 32200" in qemu_text
+ assert "remote_display_port_max = 32299" in qemu_text
+ assert {p: Path(p).read_bytes() for p in base} == base
+ # (uid, home, shell) as the sysusers fragments declare; libvirt-qemu's '-' is the default.
+ identities = {
+ "nova": (42436, "/var/lib/nova", "/bin/bash"),
+ "openstack": (42424, "/var/lib/openstack", "/usr/sbin/nologin"),
+ "neutron": (42435, "/var/lib/neutron", "/usr/sbin/nologin"),
+ "kvm-node-agent": (42438, "/var/lib/kvm-node-agent", "/usr/sbin/nologin"),
+ "openvswitch": (476, "/var/lib/openvswitch", "/usr/sbin/nologin"),
+ "libvirt-qemu": (64055, "/var/lib/libvirt", "/usr/sbin/nologin"),
+ }
+ for name, (uid, home, shell) in identities.items():
+ account = pwd.getpwnam(name)
+ assert (account.pw_uid, account.pw_dir, account.pw_shell) == (uid, home, shell), name
+ assert grp.getgrnam("libvirt").gr_gid == 42437
+ assert grp.getgrnam("libvirt-qemu").gr_gid == 64055
+ for user, groups in {"nova": {"libvirt", "openvswitch"}, "neutron": {"openvswitch"},
+ "kvm-node-agent": {"libvirt"}, "libvirt-qemu": {"kvm"},
+ "root": {"openvswitch"}}.items():
+ account = pwd.getpwnam(user)
+ actual = {grp.getgrgid(g).gr_name for g in os.getgrouplist(user, account.pw_gid)}
+ assert groups <= actual, (user, actual)
+ dirs = [(f"/var/lib/nova{suffix}", mode, 42436, 64055)
+ for suffix, mode in (("", 0o750), ("/instances", 0o750), ("/.ssh", 0o600), ("/mnt", 0o750))]
+ dirs += [(f"/etc/pki/{name}", 0o755, 42438, pwd.getpwnam("kvm-node-agent").pw_gid)
+ for name in ("CA", "libvirt", "qemu")]
+ dirs += [(p, 0o755, 0, 0) for p in ("/var/lib/libvirt", "/var/log/libvirt",
+ "/var/lib/libvirt/qemu", "/var/log/libvirt/qemu",
+ "/etc/openvswitch", "/var/lib/openvswitch", "/var/log/openvswitch")]
+ for path, mode, uid, gid in dirs:
+ stat = Path(path).stat()
+ assert Path(path).is_dir()
+ assert (stat.st_mode & 0o7777, stat.st_uid, stat.st_gid) == (mode, uid, gid), path
+ for name in ("sci-20-compute", "sci-21-network", "sci-22-storage", "sci-40-libvirt"):
+ unit = f"{name}-activation.service"
+ enabled = run("systemctl", "is-enabled", unit)
+ assert enabled.returncode == 0 and enabled.stdout.strip() == "enabled"
+ state = run("systemctl", "show", unit, "--property=ActiveState,SubState,Result")
+ assert state.returncode == 0, state.stderr
+ assert set(state.stdout.splitlines()) == {"ActiveState=active", "SubState=exited", "Result=success"}
+ for unit in ("ovsdb-server.service", "ovs-vswitchd.service", "libvirtd.service"):
+ state = run("systemctl", "show", unit, "--property=ActiveState,SubState,MainPID")
+ assert state.returncode == 0, state.stderr
+ fields = dict(line.split("=", 1) for line in state.stdout.splitlines())
+ assert fields["ActiveState"] == "active" and fields["SubState"] == "running", fields
+ assert int(fields["MainPID"]) > 0, fields
+ for command, fragment in (("qemu-system-x86_64", "QEMU"), ("conntrack", "conntrack")):
+ version = run(command, "--version")
+ assert version.returncode == 0 and fragment in version.stdout + version.stderr
+ # The base owns the storage stack; the delta only activates it.
+ for path in ("/usr/sbin/multipathd", "/usr/sbin/iscsid"):
+ assert Path(path).is_file(), path
+ for unit in ("multipathd.service", "iscsid.socket"):
+ state = run("systemctl", "show", unit, "--property=ActiveState")
+ assert state.returncode == 0, state.stderr
+ assert state.stdout.strip() == "ActiveState=active", (unit, state.stdout)
+ # Exercise the delivered L rule on isolated roots with and without a real DB.
+ conf = Path("/usr/lib/tmpfiles.d/sci-network.conf")
+ assert conf.is_file()
+ scratch = Path("/opt/sci/tmpfiles-contract")
+ assert not scratch.exists(), "runtime must provide a fresh scratch root"
+ for existing in (True, False):
+ root = scratch / str(existing)
+ db = root / "etc/openvswitch/conf.db"
+ db.parent.mkdir(parents=True)
+ if existing:
+ db.write_bytes(b"existing OVS database\n")
+ result = run("systemd-tmpfiles", f"--root={root}", "--create", str(conf))
+ assert result.returncode == 0, result.stderr
+ if existing:
+ assert not db.is_symlink() and db.read_bytes() == b"existing OVS database\n"
+ else:
+ assert db.is_symlink() and os.readlink(db) == "/var/lib/openvswitch/conf.db"
+ # The qemu C rule copies once: absent destinations are materialized, edits survive.
+ qemu_fragment = Path("/usr/lib/tmpfiles.d/sci-virt-qemu.conf")
+ qemu_payload = Path("/usr/share/sci/etc/libvirt/qemu.conf")
+ assert qemu_fragment.is_file(), qemu_fragment
+ assert qemu_payload.is_file(), qemu_payload
+ for existing in (True, False):
+ root = scratch / f"qemu-{existing}"
+ source = root / "usr/share/sci/etc/libvirt/qemu.conf"
+ source.parent.mkdir(parents=True)
+ source.write_bytes(qemu_payload.read_bytes())
+ target = root / "etc/libvirt/qemu.conf"
+ if existing:
+ target.parent.mkdir(parents=True)
+ target.write_bytes(b"operator override\n")
+ result = run("systemd-tmpfiles", f"--root={root}", "--create", str(qemu_fragment))
+ assert result.returncode == 0, result.stderr
+ if existing:
+ assert not target.is_symlink() and target.read_bytes() == b"operator override\n"
+ else:
+ assert target.is_file() and not target.is_symlink()
+ copied = target.read_text()
+ assert "remote_display_port_min = 32200" in copied
+ assert "remote_display_port_max = 32299" in copied
+ files = list(destination.glob("*.raw")) + [Path("/etc/passwd"), Path("/etc/group"),
+ Path("/etc/libvirt/qemu.conf")]
+ files += [Path(f"/etc/systemd/system/{name}-activation.service")
+ for name in ("sci-20-compute", "sci-21-network", "sci-22-storage", "sci-40-libvirt")]
+ before = {p: (p.read_bytes(), p.stat().st_mtime_ns) for p in files}
+ second = run("bash", str(staging / "sci-delivery.sh"), env=env)
+ assert second.returncode == 0, second.stdout + second.stderr
+ assert {p: (p.read_bytes(), p.stat().st_mtime_ns) for p in files} == before
+ assert merged_extensions() == EXTENSIONS
+ assert {p: Path(p).read_bytes() for p in base} == base
diff --git a/extensions/tests/guest/test_verity.py b/extensions/tests/guest/test_verity.py
new file mode 100644
index 00000000..6528d4a3
--- /dev/null
+++ b/extensions/tests/guest/test_verity.py
@@ -0,0 +1,32 @@
+"""A refusal is success only when the guest proves the victim caused it."""
+from pathlib import Path
+
+from test_sysext import guest_expected, merged_extensions, run
+
+
+def test_verity_refusal():
+ expected = guest_expected({"tamper", "policy"})
+ victim = expected["victim"]
+ assert isinstance(victim, str) and victim, "the scenario must name its victim"
+ if expected["scenario"] == "tamper":
+ # The tamper fixture must name one of the raws it staged.
+ assert victim in expected["extensions"], (victim, expected["extensions"])
+ victim_raw = Path(f"/var/lib/extensions/{victim}.raw")
+ assert victim_raw.is_file(), f"victim raw not staged: {victim_raw}"
+ assert not merged_extensions(), "refusal fixture must start unmerged"
+ cursor = run("journalctl", "--show-cursor", "-n", "0", "--no-pager")
+ assert cursor.returncode == 0, cursor.stderr
+ cursors = [line.removeprefix("-- cursor: ") for line in cursor.stdout.splitlines()
+ if line.startswith("-- cursor: ")]
+ assert len(cursors) == 1, cursor.stdout
+ refresh = run("systemd-sysext", "refresh")
+ assert refresh.returncode != 0, "the invalid extension was accepted"
+ assert not merged_extensions(), "the refused scenario merged an extension"
+ journal = run("journalctl", f"--after-cursor={cursors[0]}", "--no-pager", "-o", "cat")
+ assert journal.returncode == 0, journal.stderr
+ reasons = ("does not match image policy",) if expected["scenario"] == "policy" else (
+ "failed to read metadata", "failed to mount", "failed to read data")
+ # The diagnostic may be on stderr instead of the journal; victim and reason must match.
+ output = journal.stdout + refresh.stderr
+ assert any(victim in line and any(reason in line.lower() for reason in reasons)
+ for line in output.splitlines()), output
diff --git a/extensions/tests/pytest.ini b/extensions/tests/pytest.ini
new file mode 100644
index 00000000..32312c26
--- /dev/null
+++ b/extensions/tests/pytest.ini
@@ -0,0 +1,6 @@
+[pytest]
+addopts = -ra
+markers =
+ vm: boots a VM or needs the packaged guest runtime
+ build: expensive image builds or resolved-config/tools pins
+ build_state_mutating: mutates the shared .build state
diff --git a/extensions/tests/runtime_adapter.py b/extensions/tests/runtime_adapter.py
new file mode 100644
index 00000000..448f327c
--- /dev/null
+++ b/extensions/tests/runtime_adapter.py
@@ -0,0 +1,185 @@
+"""Stage the upstream test distribution and parse its framed result records.
+
+The tree mirrors tests/util/build_dist.sh: runtime/, the upstream run_tests
+launcher and conftest.py, the closed plugin set, empty handlers/, and the SCI
+guest tests under tests/integration/sci/.
+"""
+
+import base64
+import binascii
+import json
+import shutil
+import sys
+import tarfile
+import xml.etree.ElementTree as ET
+from pathlib import Path, PurePosixPath
+
+MAX_RECORD_LINE = 65536
+CLOSED_PLUGINS = ("booted", "docstring", "file", "modify", "parse", "shell", "systemd")
+LAUNCHER_MARKER = 'cat >"$tmpdir/dist/run_tests" <<\'EOF\''
+RESULT_PREFIX = "SCI-RESULT:"
+
+
+def _fail(message: str) -> ValueError:
+ return ValueError(f"invalid evidence: {message}")
+
+
+def _decode_record(line: str, invocation: str, scenario: str):
+ try:
+ record_invocation, record_scenario, payload = line[len(RESULT_PREFIX) :].rsplit(":", 2)
+ except ValueError:
+ raise _fail("malformed result record") from None
+ if record_invocation != invocation or record_scenario != scenario:
+ raise _fail(f"result record is for {record_invocation!r}/{record_scenario!r}, "
+ f"expected {invocation!r}/{scenario!r}")
+ try:
+ data = json.loads(base64.b64decode(payload, validate=True))
+ except (binascii.Error, ValueError, UnicodeDecodeError) as exc:
+ raise _fail("malformed result payload") from exc
+ if not isinstance(data, dict) or type(data.get("rc")) is not int or not isinstance(data.get("junit"), str):
+ raise _fail("result payload must contain integer rc and base64 junit")
+ if data["rc"] != 0:
+ raise _fail(f"guest tests exited with rc {data['rc']}")
+ try:
+ junit = base64.b64decode(data["junit"], validate=True)
+ except (binascii.Error, ValueError) as exc:
+ raise _fail("malformed JUnit base64") from exc
+ return _junit_counts(junit)
+
+
+def _junit_counts(payload: bytes):
+ try:
+ root = ET.fromstring(payload)
+ except ET.ParseError as exc:
+ raise _fail("malformed JUnit XML") from exc
+ if root.tag not in ("testsuite", "testsuites"):
+ raise _fail(f"unexpected JUnit root {root.tag!r}")
+ suites = [root] if root.tag == "testsuite" else root.findall("testsuite")
+ if not suites:
+ raise _fail("JUnit has no testsuite")
+ cases = [case for suite in suites for case in suite.findall("testcase")]
+ if not cases:
+ raise _fail("JUnit has no testcases")
+ names = [case.get("name") for case in cases]
+ if any(not name or not name.strip() for name in names):
+ raise _fail("JUnit testcase has a missing or empty name")
+ counts = {tag: sum(case.find(tag) is not None for case in cases)
+ for tag in ("failure", "error", "skipped")}
+ if any(counts.values()):
+ raise _fail(f"JUnit reports {counts}")
+ return 0, len(cases), counts["failure"], counts["error"], counts["skipped"], names
+
+
+def parse_guest_result(console: str, invocation: str, scenario: str) -> dict[str, object]:
+ """Parse exactly one bounded, matching SCI-RESULT record from a console."""
+ if not isinstance(console, str):
+ raise _fail("console must be text")
+ record_lines = [line for line in console.splitlines() if line.startswith(RESULT_PREFIX)]
+ if any(len(line) > MAX_RECORD_LINE for line in record_lines):
+ raise _fail(f"result record exceeds {MAX_RECORD_LINE} characters")
+ records = [_decode_record(line, invocation, scenario) for line in record_lines]
+ if len(records) != 1:
+ raise _fail(f"expected exactly one result record, found {len(records)}")
+ return dict(zip(("rc", "tests", "failures", "errors", "skipped", "names"), records[0]))
+
+
+def _read(path: Path) -> bytes:
+ try:
+ return path.read_bytes()
+ except OSError as exc:
+ raise ValueError(f"invalid input: cannot read {path}: {exc}") from exc
+
+
+def _launcher(repo_root: Path) -> bytes:
+ script = repo_root / "tests" / "util" / "build_dist.sh"
+ try:
+ lines = script.read_text(encoding="utf-8").splitlines(keepends=True)
+ except OSError as exc:
+ raise ValueError(f"invalid input: cannot read {script}: {exc}") from exc
+ try:
+ start = next(i for i, line in enumerate(lines) if LAUNCHER_MARKER in line)
+ end = next(i for i in range(start + 1, len(lines)) if lines[i].rstrip("\n") == "EOF")
+ except StopIteration:
+ raise ValueError(f"invalid input: no run_tests heredoc in {script}") from None
+ return "".join(lines[start + 1 : end]).encode()
+
+
+def _check_destination(destination: Path, repo_root: Path) -> None:
+ submodule, resolved = (repo_root / "gardenlinux").resolve(), destination.resolve()
+ if resolved == submodule or submodule in resolved.parents:
+ raise ValueError(f"invalid input: destination {destination} is inside the submodule")
+ if destination.exists() or destination.is_symlink():
+ raise ValueError(f"invalid input: destination {destination} already exists")
+
+
+def _check_members(members: list[tarfile.TarInfo], runtime_root: Path) -> None:
+ if not members:
+ raise ValueError("invalid input: runtime archive is empty")
+ for member in members:
+ name = PurePosixPath(member.name)
+ if not member.name or name.is_absolute() or ".." in name.parts:
+ raise ValueError(f"invalid input: unsafe archive member {member.name!r}")
+ try:
+ tarfile.data_filter(member, runtime_root)
+ except (tarfile.FilterError, OSError, ValueError) as exc:
+ raise ValueError(f"invalid input: unsafe archive member {member.name!r}: {exc}") from exc
+
+
+def stage_distribution(runtime_archive: Path, destination: Path, repo_root: Path, guest_tests: Path) -> Path:
+ """Build the upstream-shaped test tree; return its run_tests launcher."""
+ runtime_archive, destination, repo_root, guest_tests = map(
+ Path, (runtime_archive, destination, repo_root, guest_tests))
+ _check_destination(destination, repo_root)
+ runtime_root = destination / "runtime"
+ if not guest_tests.is_dir():
+ raise ValueError(f"invalid input: guest tests {guest_tests} is not a directory")
+ launcher = _launcher(repo_root)
+ conftest = _read(repo_root / "tests" / "conftest.py")
+ plugins = {name: _read(repo_root / "tests" / "plugins" / name)
+ for name in ("__init__.py", *(f"{plugin}.py" for plugin in CLOSED_PLUGINS))}
+ if not runtime_archive.is_file():
+ raise ValueError(f"invalid input: runtime archive {runtime_archive} is missing")
+ try:
+ archive = tarfile.open(runtime_archive, "r:*")
+ except (tarfile.TarError, OSError, EOFError) as exc:
+ raise ValueError(f"invalid input: cannot read {runtime_archive}: {exc}") from exc
+ with archive:
+ try:
+ members = archive.getmembers()
+ _check_members(members, runtime_root)
+ except (tarfile.TarError, OSError, EOFError) as exc:
+ raise ValueError(f"invalid input: cannot read {runtime_archive}: {exc}") from exc
+ try:
+ destination.mkdir(parents=True)
+ tests = destination / "tests"
+ (tests / "plugins").mkdir(parents=True)
+ (tests / "handlers").mkdir()
+ (destination / "run_tests").write_bytes(launcher)
+ (destination / "run_tests").chmod(0o755)
+ (tests / "conftest.py").write_bytes(conftest)
+ for name, data in plugins.items():
+ (tests / "plugins" / name).write_bytes(data)
+ shutil.copytree(guest_tests, tests / "integration" / "sci")
+ runtime_root.mkdir()
+ archive.extractall(runtime_root, members=members, filter="data")
+ except (OSError, shutil.Error, tarfile.TarError, EOFError) as exc:
+ shutil.rmtree(destination, ignore_errors=True)
+ raise ValueError(f"invalid input: cannot stage distribution: {exc}") from exc
+ return destination / "run_tests"
+
+
+def _main(argv: list[str]) -> int:
+ if len(argv) != 4 or argv[0] != "parse":
+ print("invalid evidence: usage: runtime_adapter.py parse ", file=sys.stderr)
+ return 2
+ try:
+ result = parse_guest_result(argv[1], argv[2], argv[3])
+ except ValueError as exc:
+ print(str(exc), file=sys.stderr)
+ return 1
+ print(json.dumps(result))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(_main(sys.argv[1:]))
diff --git a/extensions/tests/summary.py b/extensions/tests/summary.py
new file mode 100644
index 00000000..6840bcce
--- /dev/null
+++ b/extensions/tests/summary.py
@@ -0,0 +1,236 @@
+"""The resolved-config pin machinery: `mkosi summary --json` normalization."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+from typing import Any
+
+
+class SummaryError(RuntimeError):
+ """A summary invocation or pin precondition failed (the build would too)."""
+
+
+class SummaryShapeError(SummaryError):
+ """The JSON shape is not what `normalize_summary` expects (jq parity)."""
+
+
+def _repo_prefix(repo_root: Path | str) -> str:
+ """`$repo` with one trailing "/" stripped, plus the "/" suffix."""
+ repo = str(repo_root)
+ if repo.endswith("/"):
+ repo = repo[:-1]
+ return repo + "/"
+
+
+def _rewrite(value: Any, prefix: str) -> Any:
+ """Recursive `walk(norm($prefix))`: values only, never dict keys."""
+ if isinstance(value, dict):
+ return {key: _rewrite(item, prefix) for key, item in value.items()}
+ if isinstance(value, list):
+ return [_rewrite(item, prefix) for item in value]
+ if isinstance(value, str):
+ if value.startswith(prefix):
+ return "@REPO/" + value[len(prefix) :]
+ if value.startswith("/"):
+ return "@ABS"
+ return value
+ return value
+
+
+def _null_seeds(data: Any) -> Any:
+ """`.Images |= map(.Seed = null)` with the shape errors made explicit."""
+ if not isinstance(data, dict):
+ raise SummaryShapeError(
+ "the summary top level is not a JSON object; .Images cannot be indexed"
+ )
+ if "Images" not in data:
+ # An absent `Images` key is a no-op, unlike jq 1.7 (documented contract).
+ return data
+ images = data["Images"]
+ if not isinstance(images, list):
+ raise SummaryShapeError(f"Images is {type(images).__name__}, not a list")
+ for index, image in enumerate(images):
+ if not isinstance(image, dict):
+ raise SummaryShapeError(f"Images[{index}] is not an object")
+ image["Seed"] = None
+ # uid-dependent mkosi default (root=0, non-root=3): null it like Seed
+ image["DelegateRanges"] = None
+ # environment-derived via no_proxy/NO_PROXY: null it like Seed/DelegateRanges
+ image["ProxyExclude"] = None
+ return data
+
+
+def normalize_summary_data(data: Any, repo_root: Path | str) -> bytes:
+ """Normalize an already-parsed summary object into the pinned bytes."""
+ prefix = _repo_prefix(repo_root)
+ normalized = _null_seeds(_rewrite(data, prefix))
+ text = json.dumps(normalized, sort_keys=True, indent=2, ensure_ascii=False) + "\n"
+ return text.encode("utf-8")
+
+
+def normalize_summary(summary_json: Path | str, repo_root: Path | str) -> bytes:
+ """Normalize `` against `` (byte-exact)."""
+ data = json.loads(Path(summary_json).read_text(encoding="utf-8"))
+ return normalize_summary_data(data, repo_root)
+
+
+def normalize_summary_text(text: str, repo_root: Path | str) -> bytes:
+ """Normalize raw `mkosi summary --json` stdout."""
+ return normalize_summary_data(json.loads(text), repo_root)
+
+
+def normalized_digest(summary_json: Path | str, repo_root: Path | str) -> str:
+ """sha256 hex of the normalized bytes."""
+ return hashlib.sha256(normalize_summary(summary_json, repo_root)).hexdigest()
+
+
+def summary_command(mkosi_bin: Path | str, epoch: int) -> list[str]:
+ """build.sh's invocation shape; pure, therefore unit-testable."""
+ return [str(mkosi_bin), f"--source-date-epoch={epoch}", "summary", "--json"]
+
+
+def _abspath(path: Path | str) -> Path:
+ """Absolute paths pass through; a relative path resolves against the cwd."""
+ p = Path(path)
+ return p if p.is_absolute() else (Path.cwd() / p)
+
+
+def _release_suite(config_dir: Path) -> str:
+ """The one `^Release=` value of `/mkosi.conf`."""
+ conf = config_dir / "mkosi.conf"
+ if not conf.is_file():
+ raise SummaryError(f"{conf} missing")
+ values = [
+ line[len("Release=") :].strip()
+ for line in conf.read_text(encoding="utf-8").splitlines()
+ if line.startswith("Release=")
+ ]
+ if len(values) != 1:
+ raise SummaryError(f"{conf} carries {len(values)} Release= lines (expected one)")
+ if not values[0]:
+ raise SummaryError(f"{conf} has a Release= line but no suite value")
+ return values[0]
+
+
+def _tail(text: str, n: int = 20) -> str:
+ return "\n".join(text.splitlines()[-n:])
+
+
+def run_summary(
+ mkosi_bin: Path | str,
+ config_dir: Path | str,
+ garden_version: Path | str,
+ *,
+ repo_root: Path | str,
+ timeout: int | None = 600,
+) -> str:
+ """Return the raw `--json` summary on stdout."""
+ mkosi_path = _abspath(mkosi_bin)
+ config_path = _abspath(config_dir)
+ version_path = _abspath(garden_version)
+ _ = str(repo_root) # kept for API symmetry; the invocation does not use it
+
+ suite = _release_suite(config_path)
+ if not (version_path.is_file() and os.access(version_path, os.X_OK)):
+ raise SummaryError(
+ f"{version_path} missing or not executable (git submodule update "
+ "--init gardenlinux)"
+ )
+
+ try:
+ epoch_res = subprocess.run(
+ [str(version_path), "--epoch", suite],
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ )
+ except (OSError, subprocess.TimeoutExpired) as exc:
+ raise SummaryError(f"cannot derive the epoch from Release={suite}: {exc}") from exc
+ epoch = epoch_res.stdout.strip()
+ if epoch_res.returncode != 0:
+ raise SummaryError(
+ f"cannot derive the epoch from Release={suite}:\n"
+ f"{_tail(epoch_res.stderr, 5)}"
+ )
+ if not re.fullmatch(r"[0-9]+", epoch):
+ raise SummaryError(f"'{version_path} --epoch {suite}' is no timestamp: '{epoch}'")
+
+ try:
+ res = subprocess.run(
+ summary_command(mkosi_path, int(epoch)),
+ cwd=config_path,
+ capture_output=True,
+ text=True,
+ timeout=timeout,
+ )
+ except subprocess.TimeoutExpired as exc:
+ tail = _tail(exc.stderr) if isinstance(exc.stderr, str) else ""
+ raise SummaryError(f"timeout: mkosi summary took over {timeout}s\n{tail}") from exc
+ except OSError as exc:
+ raise SummaryError(f"cannot execute {mkosi_path}: {exc}") from exc
+ if res.returncode != 0:
+ raise SummaryError(
+ f"mkosi summary exited {res.returncode}\n{_tail(res.stderr)}"
+ )
+ return res.stdout
+
+
+def resolved_config_digest(
+ mkosi_bin: Path | str,
+ config_dir: Path | str,
+ garden_version: Path | str,
+ repo_root: Path | str,
+ *,
+ runs: int = 2,
+) -> str:
+ """Digest of the normalized resolved config, stable across repeated runs."""
+ digests: list[str] = []
+ for index in range(1, max(runs, 1) + 1):
+ raw = run_summary(
+ mkosi_bin, config_dir, garden_version, repo_root=repo_root
+ )
+ try:
+ normalized = normalize_summary_text(raw, repo_root)
+ except ValueError as exc: # json.JSONDecodeError: the shape moved
+ raise SummaryShapeError(
+ f"normalization of the mkosi summary failed: {exc}"
+ ) from exc
+ digests.append(hashlib.sha256(normalized).hexdigest())
+ if len(set(digests)) > 1:
+ raise SummaryError(
+ "mkosi summary is NOT stable run-to-run after normalization "
+ f"(digests: {', '.join(digests)}): a field is volatile, not skipped"
+ )
+ return digests[0]
+
+
+USAGE = "usage: python3 extensions/tests/summary.py "
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = list(sys.argv[1:] if argv is None else argv)
+ if len(args) != 2 or not Path(args[0]).is_file():
+ print(USAGE, file=sys.stderr)
+ return 2
+ try:
+ normalized = normalize_summary(args[0], args[1])
+ except SummaryError as exc:
+ print(f"summary-normalize: {exc}", file=sys.stderr)
+ return 1
+ except (OSError, ValueError) as exc:
+ print(f"summary-normalize: cannot parse {args[0]}: {exc}", file=sys.stderr)
+ return 1
+ # bytes, not text: stdout must be byte-identical to jq's output
+ sys.stdout.buffer.write(normalized)
+ sys.stdout.buffer.flush()
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/extensions/tests/test_artifact_contracts.py b/extensions/tests/test_artifact_contracts.py
new file mode 100644
index 00000000..b4accf6b
--- /dev/null
+++ b/extensions/tests/test_artifact_contracts.py
@@ -0,0 +1,283 @@
+"""Public input and raw-image contracts. Build checks require explicit consent."""
+import hashlib
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import uuid
+
+import pytest
+
+import summary
+
+ROOT = Path(__file__).resolve().parents[2]
+MKOSI = ROOT / "mkosi"
+STATE = ROOT / ".build/mkosi"
+IMAGES = ("sci-10-base", "sci-20-compute", "sci-21-network", "sci-22-storage", "sci-40-libvirt", "sci-50-virt-ch", "sci-51-virt-qemu")
+TYPES = ("4f68bce3-e8cd-4db1-96e7-fbcaf984b709", "2c7357ed-ebd2-46d9-aec1-23d437ec2bf5",
+ "41092b05-9fc8-4523-994f-2def0408b176")
+REQUIRED = {
+ "sci-10-base": ("usr/sbin/conntrack", "usr/sbin/parted"),
+ "sci-20-compute": ("usr/lib/sysusers.d/sci-compute.conf", "usr/lib/tmpfiles.d/sci-compute.conf"),
+ "sci-21-network": ("usr/bin/ovs-vsctl", "usr/sbin/ovsdb-server", "usr/sbin/ovs-vswitchd"),
+ # Base ships the multipath/iSCSI stack; the delta carries only its activation contract.
+ "sci-22-storage": ("usr/lib/modprobe.d/nfsclient.conf", "usr/lib/tmpfiles.d/sci-storage.conf",
+ "usr/share/sci/activation/sci-22-storage-activation.service"),
+ "sci-40-libvirt": ("usr/sbin/libvirtd", "usr/bin/virsh", "usr/lib/tmpfiles.d/sci-libvirt.conf",
+ "usr/bin/swtpm"),
+ "sci-50-virt-ch": ("usr/bin/cloud-hypervisor", "usr/bin/ch-remote", "usr/sbin/virtchd",
+ "usr/lib/x86_64-linux-gnu/libvirt/connection-driver/libvirt_driver_ch.so",
+ "usr/share/cloud-hypervisor/CLOUDHV_EFI.fd"),
+ "sci-51-virt-qemu": ("usr/bin/qemu-system-x86_64", "usr/bin/qemu-img", "usr/bin/qemu-nbd",
+ "usr/share/OVMF/OVMF_CODE_4M.fd",
+ "usr/lib/x86_64-linux-gnu/libvirt/connection-driver/libvirt_driver_qemu.so",
+ "usr/share/sci/etc/libvirt/qemu.conf"),
+}
+# Base-owned content in the merged host view; a delta must not re-ship it.
+BASE_OWNED = {
+ "sci-22-storage": ("usr/sbin/multipathd", "usr/sbin/multipath", "usr/sbin/iscsid",
+ "usr/sbin/iscsiadm", "usr/bin/iscsiadm",
+ "usr/lib/systemd/system/multipathd.service",
+ "usr/lib/systemd/system/multipathd.socket",
+ "usr/lib/systemd/system/iscsid.service",
+ "usr/lib/systemd/system/iscsid.socket",
+ "usr/lib/udev/rules.d/56-dm-mpath.rules"),
+}
+PARENT_ONLY = ("usr/sbin/libvirtd", "usr/bin/virsh", "usr/lib/sysusers.d/libvirt-qemu.conf",
+ "usr/lib/sysusers.d/libvirt.conf", "usr/lib/tmpfiles.d/sci-libvirt.conf",
+ "usr/lib/systemd/system/libvirtd.service.d/sci-resident.conf",
+ # swtpm is hypervisor-agnostic: a qemu-free trim keeps vTPM.
+ "usr/bin/swtpm", "usr/bin/swtpm_setup", "usr/sbin/swtpm_setup")
+QEMU_ONLY = ("usr/bin/qemu-system-x86_64", "usr/bin/qemu-img", "usr/bin/qemu-nbd",
+ "usr/share/doc/libvirt-daemon-system",
+ "usr/lib/x86_64-linux-gnu/libvirt/connection-driver/libvirt_driver_qemu.so")
+
+
+def run(*args, cwd=ROOT):
+ result = subprocess.run(args, cwd=cwd, capture_output=True, text=True, timeout=1800)
+ assert result.returncode == 0, f"{args} in {cwd}: {result.stdout}\n{result.stderr}"
+ return result.stdout
+
+
+def values(path, key):
+ return re.findall(rf"^{re.escape(key)}=(.*)$", path.read_text(), re.M)
+
+
+def pin(path):
+ lines = [line.strip() for line in path.read_text().splitlines()
+ if line.strip() and not line.lstrip().startswith("#")]
+ assert len(lines) == 1 and re.fullmatch(r"[0-9a-f]{64}", lines[0]), path
+ return lines[0]
+
+
+def epoch():
+ release, = values(MKOSI / "mkosi.conf", "Release")
+ return run(str(ROOT / "bin/garden-version"), "--epoch", release).strip()
+
+
+@pytest.fixture(scope="session")
+def evidence_root() -> Path:
+ """A fresh per-run extraction root under the repo's persistent .build."""
+ base = ROOT / ".build" / "sysext-tests"
+ base.mkdir(parents=True, exist_ok=True)
+ root = base / uuid.uuid4().hex
+ root.mkdir()
+ return root
+
+
+def extract_root(image, dest):
+ """EROFS root of .raw extracted under dest/root, idempotent: an
+ extraction left by an earlier test in this session is reused as-is."""
+ root = dest / "root"
+ if (root / "usr").exists():
+ return root
+ raw = STATE / "output" / f"{image}.raw"
+ assert raw.is_file(), f"build artifact missing: {raw}"
+ table = json.loads(run("sfdisk", "--json", str(raw)))["partitiontable"]
+ assert table["label"] == "gpt" and table["sectorsize"] == 512
+ partitions = table["partitions"]
+ assert [p["type"].lower() for p in partitions] == list(TYPES), partitions
+ assert all(p["size"] > 0 for p in partitions)
+ dest.mkdir(parents=True, exist_ok=True)
+ part = dest / "root.erofs"
+ run("dd", f"if={raw}", f"of={part}", "bs=512", f"skip={partitions[0]['start']}",
+ f"count={partitions[0]['size']}", "status=none")
+ builder = os.environ.get("MKOSI_BUILDER_IMAGE")
+ if not builder:
+ builder, = values(ROOT / "build", "container_image")
+ assert re.search(r"(?:@sha256:[0-9a-f]{64}|:[0-9a-f]{40})$", builder), builder
+ root.mkdir(exist_ok=True)
+ run(os.environ.get("MKOSI_CONTAINER_ENGINE", "docker"), "run", "--rm", "--pull=never",
+ "--network=none", "--security-opt", "seccomp=unconfined", "--security-opt", "apparmor=unconfined",
+ "--user", f"{os.getuid()}:{os.getgid()}", "--entrypoint", "fsck.erofs",
+ "-v", f"{ROOT}:{ROOT}", builder, f"--extract={root}", "--force", "--preserve-perms", str(part))
+ return root
+
+
+def root_files(root):
+ """Every file and symlink under an extracted root, as posix relative paths."""
+ return {p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file() or p.is_symlink()}
+
+
+def overlapping_paths(a, b):
+ """Paths present in both path sets: non-empty means one layer re-ships the other."""
+ return a & b
+
+
+def test_static_input_contract():
+ release, = values(MKOSI / "mkosi.conf", "Release")
+ assert release == "2150.9.0"
+ assert values(MKOSI / "suite.pin", "SUITE") == [release]
+ assert not (MKOSI / "mkosi.images/base").exists()
+ assert not (MKOSI / "generate-base-packages.sh").exists()
+ deployables = {p.parent.name for p in (MKOSI / "mkosi.images").glob("*/mkosi.conf")
+ if values(p, "Format") == ["sysext"]}
+ assert deployables == set(IMAGES)
+ assert not (MKOSI / "mkosi.images/sci-40-libvirt-tree").exists(), (
+ "the directory-tree twin of sci-40-libvirt existed only as an Overlay dedup parent for the "
+ "virt images; the decoupling removed that role")
+ for image in IMAGES:
+ conf = MKOSI / "mkosi.images" / image / "mkosi.conf"
+ # The seed derives from the directory name, so a rename recomputes it.
+ digest = hashlib.sha256(f"sci:{release}:{image}".encode()).hexdigest()[:32]
+ seed = "-".join(digest[a:b] for a, b in ((0, 8), (8, 12), (12, 16), (16, 20), (20, 32)))
+ assert values(conf, "Seed") == [seed], image
+ assert values(conf, "Overlay") == ["yes"]
+ base, = values(conf, "BaseTrees")
+ parents = base.split()
+ assert (conf.parent / parents[0]).resolve() == (STATE / "base-image/current").resolve()
+ assert len(parents) == 1, (
+ image, "every extension must build standalone on the published base: a second "
+ "base-tree entry re-introduces a build-time parent that couples rebuilds")
+ assert values(conf, "Dependencies") == [], (
+ image, "inter-extension build dependencies couple rebuilds: a package change must "
+ "rebuild exactly one extension; runtime merge order is handled by the deployment "
+ "naming, not by build topology")
+ assert "usr/lib/verity.d" in values(conf, "RemoveFiles")
+ assert "usr/lib/modules/*/vmlinuz" in values(conf, "RemoveFiles")
+ if image in {"sci-50-virt-ch", "sci-51-virt-qemu"}:
+ # apt installs the full libvirt set; explicit globs keep it out of the delta.
+ globs = values(conf, "RemoveFiles")
+ for path in ("usr/share/libvirt/cpu_map", "usr/share/libvirt/schemas",
+ "usr/lib/sysusers.d/libvirt.conf"):
+ assert any(path in g for g in globs), (
+ image, path, "the driver packages' Depends pulls the libvirt set into the "
+ "virt builds; without explicit removal the delta double-ships the "
+ "libvirt-owned paths (420 in ch, 265 in qemu) and the lexical overlay merge "
+ "silently shadows one copy")
+ manifest, = (ROOT / "delivery").glob("extensions.manifest*")
+ lines = [line for line in manifest.read_text().splitlines() if line and not line.startswith("#")]
+ assert lines.count("TARGET_BASE=2150") == 1
+ names = [line.split()[0] for line in lines if "=" not in line]
+ assert len(names) == len(set(names)) and set(names) == deployables, (
+ "the delivery manifest must list every deployable exactly once and name "
+ "no other extension: the repo curates the full set")
+ for name in ("tools.pin", "summary.pin"):
+ pin(MKOSI / name)
+ assert values(MKOSI / "base-image.pin", "VERSION") == [release]
+ for field in ("TARBALL_SHA256", "ROOTFS_TAR_SHA256"):
+ digest, = values(MKOSI / "base-image.pin", field)
+ assert re.fullmatch(r"[0-9a-f]{64}", digest)
+ key, = values(MKOSI / "mkosi.conf", "VerityKey")
+ key_path = (MKOSI / key).resolve().relative_to(ROOT)
+ assert key_path.is_relative_to(".build")
+ assert not run("git", "ls-files", "--", str(key_path)).strip()
+ run("git", "check-ignore", "-q", "--", str(key_path))
+
+
+def test_qemu_vnc_range_carrier():
+ extra = ROOT / "mkosi/mkosi.images/sci-51-virt-qemu/mkosi.extra"
+ rules = [line.strip() for line in
+ (extra / "usr/lib/tmpfiles.d/sci-virt-qemu.conf").read_text().splitlines()]
+ payload = extra / "usr/share/sci/etc/libvirt/qemu.conf"
+ assert payload.is_file(), f"the qemu VNC range payload must exist at {payload}"
+ lines = [line.strip() for line in payload.read_text().splitlines()]
+ assert any(re.fullmatch(r"d\s+/etc/libvirt\s+0755\s+root\s+root\s+-", line) for line in rules), (
+ "the qemu fragment must create /etc/libvirt before the C rule copies into it")
+ assert any(re.fullmatch(
+ r"C\s+/etc/libvirt/qemu\.conf\s+0644\s+root\s+root\s+-\s+/usr/share/sci/etc/libvirt/qemu\.conf",
+ line) for line in rules), (
+ "the qemu fragment must materialize /etc/libvirt/qemu.conf from the /usr payload with type C")
+ assert sum("remote_display_port" in line for line in lines) == 2, (
+ "the payload must declare exactly the two VNC port settings")
+ assert sum(bool(re.fullmatch(r"remote_display_port_min\s*=\s*32200", line)) for line in lines) == 1, (
+ "the payload must set remote_display_port_min = 32200 for the NodePort range")
+ assert sum(bool(re.fullmatch(r"remote_display_port_max\s*=\s*32299", line)) for line in lines) == 1, (
+ "the payload must set remote_display_port_max = 32299 for the NodePort range")
+ assert not [line for line in lines if line.startswith("#") and "remote_display_port" in line], (
+ "the payload must be a partial conffile with no commented-out port settings")
+
+
+@pytest.mark.build
+def test_tools_manifest_pin():
+ manifest = STATE / "tools/mkosi.tools.manifest"
+ assert manifest.is_file(), f"pinned tools manifest missing: {manifest}"
+ assert hashlib.sha256(manifest.read_bytes()).hexdigest() == pin(MKOSI / "tools.pin")
+
+
+@pytest.mark.build
+def test_resolved_summary_pin():
+ binary = STATE / "venv/bin/mkosi"
+ assert binary.is_file(), f"mkosi venv missing: {binary}"
+ digest = summary.resolved_config_digest(binary, MKOSI, ROOT / "bin/garden-version", ROOT)
+ assert digest == pin(MKOSI / "summary.pin")
+
+
+@pytest.mark.build
+@pytest.mark.parametrize("image", IMAGES)
+def test_signed_delta(image, evidence_root):
+ dest = extract_root(image, evidence_root / "extracted" / image)
+ for path in REQUIRED[image]:
+ assert (dest / path).is_file() or (dest / path).is_symlink(), (image, path)
+ base = STATE / "base-image/current"
+ for path in BASE_OWNED.get(image, ()):
+ assert (base / path).is_file() or (base / path).is_symlink(), (image, path)
+ assert not ((dest / path).is_file() or (dest / path).is_symlink()), (image, path)
+ release = dest / f"usr/lib/extension-release.d/extension-release.{image}"
+ assert values(release, "ID") == ["gardenlinux"]
+ assert values(release, "SYSEXT_SCOPE") == ["system"]
+ assert not (dest / "usr/lib/verity.d").exists()
+ for path in ("etc/os-release", "usr/lib/os-release", "usr/bin/systemctl", "usr/lib/systemd/systemd",
+ "usr/bin/dpkg", "usr/lib/x86_64-linux-gnu/libc.so.6"):
+ assert not (dest / path).exists() and not (dest / path).is_symlink(), (image, path)
+ assert not list((dest / "usr/lib/modules").glob("*/vmlinuz"))
+ if image in {"sci-40-libvirt", "sci-50-virt-ch"}:
+ for path in QEMU_ONLY:
+ assert not (dest / path).exists() and not (dest / path).is_symlink(), (image, path)
+ assert not list((dest / "usr/bin").glob("qemu*"))
+ assert not (dest / "usr/share/qemu").exists()
+ if image in {"sci-50-virt-ch", "sci-51-virt-qemu"}:
+ for path in PARENT_ONLY:
+ assert not (dest / path).exists() and not (dest / path).is_symlink(), (image, path)
+ if image != "sci-40-libvirt":
+ assert not (dest / "usr/lib/sysusers.d/sci-libvirt.conf").exists()
+
+
+@pytest.mark.build
+def test_virt_deltas_do_not_reship_libvirt(evidence_root):
+ libvirt = root_files(extract_root("sci-40-libvirt", evidence_root / "extracted" / "sci-40-libvirt"))
+ for image in ("sci-50-virt-ch", "sci-51-virt-qemu"):
+ virt = root_files(extract_root(image, evidence_root / "extracted" / image))
+ overlap = overlapping_paths(virt, libvirt)
+ assert not overlap, (
+ image, "the virt deltas must not re-ship any sci-40-libvirt-owned path: with the "
+ "parent-tree dedup gone, RemoveFiles= is the only guard, and a package update that "
+ "adds a file outside the globs must fail here rather than silently shadow in the "
+ "overlay merge; first offenders: " + ", ".join(sorted(overlap)[:10]))
+
+
+@pytest.mark.build
+@pytest.mark.build_state_mutating
+def test_reproducible():
+ run("bash", str(MKOSI / "build.sh"))
+ def hashes():
+ return {image: hashlib.sha256((STATE / "output" / f"{image}.raw").read_bytes()).hexdigest()
+ for image in IMAGES}
+ first = hashes()
+ run(str(STATE / "venv/bin/mkosi"), f"--source-date-epoch={epoch()}", "-f", "build", cwd=MKOSI)
+ second = hashes()
+ assert second == first, f"rebuild changed the output hashes: {first} != {second}"
+ for image in IMAGES:
+ print(f"build1 {image} {first[image]}")
+ print(f"build2 {image} {second[image]}")
diff --git a/extensions/tests/test_boot_contract.py b/extensions/tests/test_boot_contract.py
new file mode 100644
index 00000000..74c28f92
--- /dev/null
+++ b/extensions/tests/test_boot_contract.py
@@ -0,0 +1,133 @@
+"""Contract for the mkosi/boot-test.sh scenario CLI and the three boot proofs.
+
+The runner keeps its no-arg and --manifest behavior and gains
+--scenario signed|tamper|policy|all (default all) and
+--runtime (default .build/sysext-runtime/runtime.tar.gz).
+An unknown scenario or a missing runtime archive must be refused before
+any VM boot, image build, or shared .build state mutation. With
+--scenario all the final stdout line is JSON with one entry per scenario:
+the runtime-adapter counts and the guest test names.
+"""
+import json
+import subprocess
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[2]
+RUNNER = "mkosi/boot-test.sh"
+RUNTIME = ROOT / ".build/sysext-runtime/runtime.tar.gz"
+SCRATCH = ROOT / ".build/test-scratch-vmboot"
+REQUIRED = {"signed": "test_delivery_and_host_contract",
+ "tamper": "test_verity_refusal", "policy": "test_verity_refusal"}
+
+
+def run_cli(*args, timeout=5):
+ return subprocess.run(["bash", RUNNER, *args], cwd=ROOT, capture_output=True, text=True, timeout=timeout)
+
+
+def snapshot(path: Path) -> dict:
+ if not path.exists():
+ return {}
+ return {str(p.relative_to(path)): (p.stat().st_mtime_ns, p.stat().st_size) for p in path.rglob("*") if p.is_file()}
+
+
+def refused_clean(args: tuple) -> subprocess.CompletedProcess:
+ before = snapshot(SCRATCH)
+ result = run_cli(*args)
+ assert result.returncode != 0, f"{args} exited 0: {result.stdout}"
+ assert snapshot(SCRATCH) == before, "the refusal mutated the shared boot scratch"
+ return result
+
+
+def test_cli_rejects_unknown_scenario():
+ result = refused_clean(("--scenario", "bogus"))
+ text = result.stdout + result.stderr
+ assert "bogus" in text and "scenario" in text.lower(), f"no diagnostic naming the invalid scenario: {text}"
+
+
+def test_cli_missing_runtime_refused(tmp_path: Path):
+ missing = tmp_path / "no-such-runtime.tar.gz"
+ result = refused_clean(("--scenario", "signed", "--runtime", str(missing)))
+ text = result.stdout + result.stderr
+ assert str(missing) in text, f"the refusal does not name the missing archive: {text}"
+ assert "missing mkosi venv" not in text, f"the runtime check must precede the mkosi-venv check: {text}"
+
+
+def _counts(entry):
+ assert isinstance(entry, dict), f"scenario entry is not an object: {entry!r}"
+ counts = entry.get("counts", entry)
+ assert isinstance(counts, dict) and "tests" in counts, f"scenario entry lacks the counts: {entry!r}"
+ return counts
+
+
+@pytest.mark.vm
+@pytest.mark.build_state_mutating
+def test_three_boot_contract():
+ assert RUNTIME.is_file(), f"runtime archive missing: {RUNTIME}"
+ result = run_cli("--scenario", "all", "--runtime", str(RUNTIME), timeout=1800)
+ assert result.returncode == 0, result.stdout + result.stderr
+ lines = [line for line in result.stdout.splitlines() if line.strip()]
+ assert lines, "the runner printed no stdout"
+ data = json.loads(lines[-1])
+ assert set(data) == set(REQUIRED), f"the final JSON line carries the wrong scenarios: {sorted(data)}"
+ for scenario, name in REQUIRED.items():
+ assert name in data[scenario]["names"], f"{scenario}: guest test {name} is absent: {data[scenario]}"
+ counts = _counts(data[scenario])
+ assert int(counts["tests"]) > 0, f"{scenario}: no guest tests ran: {counts}"
+ assert int(counts.get("failures", 0)) + int(counts.get("errors", 0)) == 0, f"{scenario}: guest failures: {counts}"
+ assert int(counts.get("skipped", 0)) == 0, f"{scenario}: guest tests skipped: {counts}"
+
+
+# boot_runner.names must refuse hostile entries before any scratch mutation.
+def _manifest(tmp_path: Path, text: str):
+ import boot_runner
+ manifest = tmp_path / "extensions.manifest"
+ manifest.write_text(text, encoding="utf-8")
+ return boot_runner, manifest
+
+
+@pytest.mark.parametrize("text", [
+ pytest.param("../outside", id="traversal"),
+ pytest.param("/abs/outside", id="absolute"),
+ pytest.param("sci-20-compute\nsci-20-compute", id="duplicate"),
+ pytest.param("# only a comment", id="empty"),
+])
+def test_names_rejects_hostile_entries(tmp_path: Path, text):
+ boot_runner, manifest = _manifest(tmp_path, text + "\n")
+ before = snapshot(SCRATCH)
+ with pytest.raises(ValueError):
+ boot_runner.names(manifest)
+ assert snapshot(SCRATCH) == before, "names() touched the shared boot scratch"
+
+
+def test_names_accepts_plain_entries(tmp_path: Path):
+ boot_runner, manifest = _manifest(tmp_path, "sci-20-compute\nunsignedprobe\n")
+ assert boot_runner.names(manifest) == ["sci-20-compute", "unsignedprobe"]
+
+
+# The open getty prompt has no trailing newline; the record must start its own line.
+def test_wrapper_result_record_starts_its_own_line(tmp_path: Path):
+ import ast, sys
+ import boot_runner, runtime_adapter
+ code = boot_runner.WRAPPER.split("<<'PY'\n", 1)[1].rsplit("\nPY\n", 1)[0]
+ ast.parse(code)
+ assert "SCI-RESULT:" in code and "/dev/console" in code
+ (tmp_path / "junit.xml").write_text(""
+ "")
+ (tmp_path / "emitter.py").write_text(code + "\n")
+ console = tmp_path / "console"
+ console.write_bytes(b"localhost login: ")
+ driver = tmp_path / "driver.py"
+ driver.write_text("import os,sys\nconsole,xml,emitter=sys.argv[1:4]\n"
+ "fd=os.open(console,os.O_WRONLY|os.O_APPEND);real=os.open\n"
+ "os.open=lambda p,*a,**k: fd if p=='/dev/console' else real(p,*a,**k)\n"
+ "sys.argv=[emitter,'0','sci-boot-tamper-f1','tamper',xml]\n"
+ "exec(compile(open(emitter).read(),'emitter','exec'),{'__name__':'__main__'})\n")
+ result = subprocess.run([sys.executable, str(driver), str(console), str(tmp_path / "junit.xml"),
+ str(tmp_path / "emitter.py")], capture_output=True, text=True)
+ assert result.returncode == 0, result.stdout + result.stderr
+ text = console.read_bytes().decode()
+ assert text[: text.index("SCI-RESULT:")].endswith("\n"), f"record glued to the prompt: {text[-60:]!r}"
+ counts = runtime_adapter.parse_guest_result(text, "sci-boot-tamper-f1", "tamper")
+ assert counts["tests"] == 1 and counts["names"] == ["test_verity_refusal"]
diff --git a/extensions/tests/test_fetch_contracts.py b/extensions/tests/test_fetch_contracts.py
new file mode 100644
index 00000000..311819cd
--- /dev/null
+++ b/extensions/tests/test_fetch_contracts.py
@@ -0,0 +1,489 @@
+"""Fetch-mode and REMOVE-directive contract tests for delivery/sci-delivery.sh.
+
+The tests pin fetch mode (a signed SHA256SUMS inventory served from FETCH_URL, verified against FETCH_PUBKEY, optionally pinned by FETCH_SUMS_DIGEST, staged under FETCH_DIR) and the manifest's REMOVE= tombstone directive.
+
+Harness: each fixture build is served over HTTP from a localhost
+ThreadingHTTPServer that records every request path; an RSA keypair signs
+the inventory; ``systemctl`` and ``systemd-sysext`` are stubs that log their
+argv; ``curl``, ``openssl`` and ``sha256sum`` are the real tools. No lib
+imports, builds, VMs, docker or non-loopback network.
+"""
+from __future__ import annotations
+import contextlib
+import hashlib
+import http.server
+import os
+import re
+import shutil
+import subprocess
+import threading
+from pathlib import Path
+import pytest
+REPO = Path(__file__).resolve().parents[2]
+DELIVERY_SH = REPO / "delivery/sci-delivery.sh"
+NAMES = ("sci-50-virt-ch", "sci-51-virt-qemu")
+MANIFEST = "TARGET_BASE=2150\nsci-50-virt-ch foo.service\nsci-51-virt-qemu\n"
+MANIFEST_REMOVE = "TARGET_BASE=2150\nsci-50-virt-ch foo.service\nREMOVE=sci-51-virt-qemu\n"
+MANIFEST_REMOVE_LISTED = "TARGET_BASE=2150\nsci-50-virt-ch foo.service\nREMOVE=sci-50-virt-ch\n"
+MANIFEST_REMOVE_TRAVERSE = "TARGET_BASE=2150\nsci-50-virt-ch foo.service\nREMOVE=../../etc/passwd\n"
+# Two-build drop sequence: build A deploys sci-21-network and sci-51-virt-qemu;
+# build B drops sci-51-virt-qemu behind a tombstone and its signed inventory no
+# longer lists the sci-51 files.
+MANIFEST_DROP_A = "TARGET_BASE=2150\nsci-21-network\nsci-51-virt-qemu\n"
+MANIFEST_DROP_B = "TARGET_BASE=2150\nsci-21-network\nREMOVE=sci-51-virt-qemu\n"
+INVENTORY = ("extensions.manifest", "sci-50-virt-ch.raw", "sci-51-virt-qemu.raw",
+ "sci-50-virt-ch-activation.service", "sci-51-virt-qemu-activation.service")
+SUMS_PATHS = {"/SHA256SUMS", "/SHA256SUMS.sig"}
+SHIM_TEMPLATE = ("[Unit]\nDescription=SCI {name} activation (fetch-contracts fixture)\nAfter=systemd-sysext.service\n"
+ "[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/bin/true\n"
+ "[Install]\nWantedBy=multi-user.target\n")
+LOG_STUB = "#!/bin/sh\nprintf 'TOOL %s\\n' \"$*\" >> \"${STUB_LOG:-/dev/null}\"\nexit 0\n"
+SLEEP_STUB = "#!/bin/sh\nprintf '%s\\n' \"$$\" >\"${STUB_SLEEP_STARTED:-/dev/null}\"\nexec /bin/sleep \"$@\"\n"
+# Like LOG_STUB, but also reproduces the filesystem effect of enable/disable:
+# real systemctl enable creates the wants symlink the activation shim's
+# [Install] section asks for (WantedBy=multi-user.target), and disable
+# removes it again. $2 is the unit name on the systemctl command line.
+SYSTEMCTL_ENABLE_STUB = (
+ "#!/bin/sh\n"
+ "printf 'systemctl %s\\n' \"$*\" >> \"${STUB_LOG:-/dev/null}\"\n"
+ "case \"$1\" in\n"
+ " enable)\n"
+ " d=\"${HOST_ROOT}/etc/systemd/system/multi-user.target.wants\"\n"
+ " mkdir -p \"$d\" && ln -sf \"../$2\" \"$d/$2\"\n"
+ " ;;\n"
+ " disable)\n"
+ " rm -f \"${HOST_ROOT}/etc/systemd/system/multi-user.target.wants/$2\"\n"
+ " ;;\n"
+ "esac\n"
+ "exit 0\n"
+)
+TRIO = ("systemctl daemon-reload", "systemd-sysext refresh", "systemctl reload dbus.service")
+# README activation flow for a fresh set: trio, restart units of changed raws
+# (manifest order), then enable every shim and restart the changed/new ones.
+SEQ_FRESH = TRIO + (
+ "systemctl restart foo.service",
+ "systemctl enable sci-50-virt-ch-activation.service",
+ "systemctl restart sci-50-virt-ch-activation.service",
+ "systemctl enable sci-51-virt-qemu-activation.service",
+ "systemctl restart sci-51-virt-qemu-activation.service",
+)
+# Only sci-50-virt-ch's raw changed: its units and shim are re-activated;
+# sci-51-virt-qemu's shim is enabled but not restarted.
+SEQ_CHANGED_RAW = TRIO + (
+ "systemctl restart foo.service",
+ "systemctl enable sci-50-virt-ch-activation.service",
+ "systemctl restart sci-50-virt-ch-activation.service",
+ "systemctl enable sci-51-virt-qemu-activation.service",
+)
+# A removal-only change: trio plus the unconditional enable of every remaining
+# shim, no restarts (no raw or shim was deployed).
+SEQ_REMOVE_ONLY = TRIO + ("systemctl enable sci-50-virt-ch-activation.service",)
+def sh(argv, *, cwd=None, env=None, timeout=180):
+ return subprocess.run([str(a) for a in argv], cwd=str(cwd or REPO), env=env, timeout=timeout, capture_output=True, text=True)
+def sha256_file(path: Path) -> str:
+ return hashlib.sha256(Path(path).read_bytes()).hexdigest()
+def write_stub(directory: Path, name: str, body: str) -> None:
+ directory.mkdir(parents=True, exist_ok=True)
+ path = directory / name
+ path.write_text(body, encoding="utf-8")
+ path.chmod(0o755)
+def entry_names(manifest_text: str) -> list[str]:
+ names = []
+ for line in manifest_text.splitlines():
+ line = line.strip()
+ if not line or line.startswith("#") or re.match(r"^[A-Z][A-Z0-9_]*=", line):
+ continue
+ names.append(line.split()[0])
+ return names
+class _Serve:
+ """A ThreadingHTTPServer on 127.0.0.1 that records every request path."""
+ def __init__(self, directory: Path):
+ self.directory = Path(directory)
+ self.paths: list[str] = []
+ self._lock = threading.Lock()
+ self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), _make_handler(self))
+ self.port = self.httpd.server_address[1]
+ self._thread = threading.Thread(target=self.httpd.serve_forever, daemon=True)
+ @property
+ def url(self) -> str:
+ return f"http://127.0.0.1:{self.port}"
+ def start(self) -> None:
+ self._thread.start()
+ def reset(self) -> None:
+ with self._lock:
+ self.paths.clear()
+ def stop(self) -> None:
+ self.httpd.shutdown()
+ self.httpd.server_close()
+ self._thread.join(timeout=5)
+def _make_handler(server: _Serve):
+ class Handler(http.server.SimpleHTTPRequestHandler):
+ def __init__(self, *args, **kwargs):
+ super().__init__(*args, directory=str(server.directory), **kwargs)
+ def do_GET(self):
+ with server._lock:
+ server.paths.append(self.path)
+ super().do_GET()
+ def log_message(self, format, *args):
+ pass
+ return Handler
+@contextlib.contextmanager
+def serve(directory: Path):
+ server = _Serve(directory)
+ server.start()
+ try:
+ yield server
+ finally:
+ server.stop()
+@pytest.fixture(scope="module")
+def keys(tmp_path_factory) -> dict[str, Path]:
+ root = tmp_path_factory.mktemp("fetch-keys")
+ def pair(name: str) -> tuple[Path, Path]:
+ d = root / name
+ d.mkdir()
+ key, pub = d / "key.pem", d / "pub.pem"
+ res = sh(["openssl", "genpkey", "-algorithm", "RSA", "-pkeyopt", "rsa_keygen_bits:2048", "-out", str(key)])
+ assert res.returncode == 0, res.stderr
+ res = sh(["openssl", "pkey", "-in", str(key), "-pubout", "-out", str(pub)])
+ assert res.returncode == 0, res.stderr
+ return key, pub
+ key, pub = pair("primary")
+ other_key, other_pub = pair("other")
+ return {"key": key, "pub": pub, "other_key": other_key, "other_pub": other_pub}
+def sign_sums(served: Path, key: Path) -> None:
+ res = sh(["openssl", "dgst", "-sha256", "-sign", str(key), "-out", "SHA256SUMS.sig", "SHA256SUMS"], cwd=served)
+ assert res.returncode == 0, res.stderr
+def make_build(served: Path, manifest_text: str, key: Path, *, raw_content: dict[str, str] | None = None) -> None:
+ """Write a fixture build (manifest, raws, shims) and its signed SHA256SUMS inventory."""
+ if served.exists():
+ shutil.rmtree(served)
+ served.mkdir(parents=True)
+ (served / "extensions.manifest").write_text(manifest_text, encoding="utf-8")
+ entries = entry_names(manifest_text)
+ raw_content = raw_content or {}
+ for name in entries:
+ (served / f"{name}.raw").write_text(raw_content.get(name, f"{name} raw content v1\n"), encoding="utf-8")
+ (served / f"{name}-activation.service").write_text(SHIM_TEMPLATE.format(name=name), encoding="utf-8")
+ files = ["extensions.manifest"]
+ files += [f"{name}.raw" for name in entries]
+ files += [f"{name}-activation.service" for name in entries]
+ (served / "SHA256SUMS").write_text("".join(f"{sha256_file(served / fname)} {fname}\n" for fname in files), encoding="utf-8")
+ sign_sums(served, key)
+def rehash_and_resign(served: Path, fname: str, key: Path) -> None:
+ sums = served / "SHA256SUMS"
+ lines = sums.read_text(encoding="utf-8").splitlines()
+ lines = [f"{sha256_file(served / fname)} {fname}" if line.rsplit(" ", 1)[-1] == fname else line for line in lines]
+ sums.write_text("\n".join(lines) + "\n", encoding="utf-8")
+ sign_sums(served, key)
+class FetchEnv:
+ """One delivery-step invocation environment: host root, fetch root, stubs, calls log."""
+ def __init__(self, root: Path):
+ self.root = Path(root)
+ self.host = self.root / "host"
+ self.stubs = self.root / "stubs"
+ self.calls_log = self.root / "calls.log"
+ self.fetch_dir = self.root / "fetch"
+ self.staging = self.fetch_dir / "staging"
+ for tool in ("systemctl", "systemd-sysext", "nsenter"):
+ write_stub(self.stubs, tool, LOG_STUB.replace("TOOL", tool))
+ write_stub(self.stubs, "sleep", SLEEP_STUB)
+ self.host.mkdir(parents=True)
+ (self.host / "etc").mkdir()
+ (self.host / "etc/os-release").write_text("ID=gardenlinux\nIMAGE_VERSION=2150.9.0\n", encoding="utf-8")
+ def delivery_env(self, *, url: str, pub: Path, extra: dict | None = None) -> dict:
+ # STAGING_DIR is deliberately not set: fetch mode must default it to
+ # $FETCH_DIR/staging.
+ env = {k: v for k, v in os.environ.items()
+ if k not in ("HOST_ROOT", "STAGING_DIR", "INTERVAL", "NSENTER", "STUB_LOG")
+ and not k.startswith("FETCH_")}
+ env.update(HOST_ROOT=str(self.host), INTERVAL="0", NSENTER="0",
+ STUB_LOG=str(self.calls_log),
+ FETCH_URL=url, FETCH_PUBKEY=str(pub), FETCH_DIR=str(self.fetch_dir))
+ env["PATH"] = os.pathsep.join([str(self.stubs), os.environ.get("PATH", "")])
+ env.update({k: str(v) for k, v in (extra or {}).items()})
+ return env
+ def run(self, *, url: str, pub: Path, extra: dict | None = None) -> subprocess.CompletedProcess:
+ self.calls_log.write_text("", encoding="utf-8")
+ return sh(["bash", DELIVERY_SH], env=self.delivery_env(url=url, pub=pub, extra=extra))
+ def calls(self) -> list[str]:
+ return self.calls_log.read_text(encoding="utf-8").splitlines() if self.calls_log.is_file() else []
+ def deployed(self) -> dict[str, bytes]:
+ """Every host file under the two deployment trees, keyed by relative path."""
+ found = {}
+ for sub in ("var/lib/extensions", "etc/systemd/system"):
+ base = self.host / sub
+ if base.is_dir():
+ for path in base.rglob("*"):
+ if path.is_file():
+ found[str(path.relative_to(self.host))] = path.read_bytes()
+ return found
+ def host_tree(self) -> dict[str, bytes]:
+ return {str(p.relative_to(self.host)): p.read_bytes() for p in self.host.rglob("*") if p.is_file()}
+def preplace(env: FetchEnv, names: tuple[str, ...]) -> dict[str, bytes]:
+ """Pre-place raw+shim pairs on the host as a previous build would have left them."""
+ placed = {}
+ for name in names:
+ raw = env.host / "var/lib/extensions" / f"{name}.raw"
+ raw.parent.mkdir(parents=True, exist_ok=True)
+ raw.write_text(f"{name} raw content v1\n", encoding="utf-8")
+ shim = env.host / "etc/systemd/system" / f"{name}-activation.service"
+ shim.parent.mkdir(parents=True, exist_ok=True)
+ shim.write_text(SHIM_TEMPLATE.format(name=name), encoding="utf-8")
+ placed[str(raw.relative_to(env.host))] = raw.read_bytes()
+ placed[str(shim.relative_to(env.host))] = shim.read_bytes()
+ return placed
+def assert_placed(env: FetchEnv, placed: dict[str, bytes]) -> None:
+ for rel, data in placed.items():
+ path = env.host / rel
+ assert path.is_file() and path.read_bytes() == data, f"{rel} was deleted or modified"
+def assert_no_passwd_strays(env: FetchEnv) -> None:
+ strays = [str(p) for p in env.root.rglob("*") if p.name == "passwd"]
+ assert not strays, f"traversal wrote files outside staging: {strays}"
+ # A relative write against the delivery step's working directory (the repo) would
+ # land two levels up.
+ assert not (REPO.resolve().parent.parent / "etc" / "passwd").exists(), "a traversing name escaped to the filesystem"
+def expect_refused(env: FetchEnv, res: subprocess.CompletedProcess, *, placed: dict[str, bytes] | None = None) -> None:
+ assert res.returncode != 0, f"the pass must fail, rc=0: {res.stdout}"
+ assert "error: fetch:" in res.stderr, f"expected a '[sci-delivery] error: fetch: ...' message: {res.stderr}"
+ if placed is None:
+ assert env.deployed() == {}, "the refused pass performed host writes"
+ else:
+ assert env.deployed() == placed, "the refused pass changed the pre-placed host state (deletions or writes)"
+ assert env.calls() == [], "the refused pass called host tools"
+def test_fetch_installs_a_signed_set(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST, keys["key"])
+ with serve(served) as http:
+ res = env.run(url=http.url, pub=keys["pub"])
+ assert res.returncode == 0, res.stderr or res.stdout
+ # sums, sig, then every inventory file (manifest, raws, shims).
+ assert len(http.paths) == 7, f"GETs: {http.paths}"
+ assert set(http.paths[:2]) == SUMS_PATHS, f"the pass must download SHA256SUMS and SHA256SUMS.sig first: {http.paths}"
+ assert set(http.paths[2:]) == {f"/{fname}" for fname in INVENTORY}, f"GETs: {http.paths}"
+ for fname in INVENTORY:
+ assert f"fetch: downloaded {fname}" in res.stdout, res.stdout
+ assert any("fetch:" in line and "signature" in line.lower() for line in res.stdout.splitlines()), \
+ f"no signature-verified fetch log line in: {res.stdout}"
+ assert env.deployed() == {
+ "var/lib/extensions/sci-50-virt-ch.raw": (served / "sci-50-virt-ch.raw").read_bytes(),
+ "var/lib/extensions/sci-51-virt-qemu.raw": (served / "sci-51-virt-qemu.raw").read_bytes(),
+ "etc/systemd/system/sci-50-virt-ch-activation.service": (served / "sci-50-virt-ch-activation.service").read_bytes(),
+ "etc/systemd/system/sci-51-virt-qemu-activation.service": (served / "sci-51-virt-qemu-activation.service").read_bytes(),
+ }, env.deployed()
+ assert env.calls() == list(SEQ_FRESH), env.calls()
+ # In fetch mode the staging directory defaults to $FETCH_DIR/staging.
+ assert (env.staging / "SHA256SUMS").is_file(), "fetch staging is not under $FETCH_DIR/staging"
+ for fname in INVENTORY:
+ assert (env.staging / fname).read_bytes() == (served / fname).read_bytes(), fname
+def test_fetch_is_converged_on_second_pass(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST, keys["key"])
+ with serve(served) as http:
+ first = env.run(url=http.url, pub=keys["pub"])
+ assert first.returncode == 0, first.stderr or first.stdout
+ http.reset()
+ second = env.run(url=http.url, pub=keys["pub"])
+ assert second.returncode == 0, second.stderr or second.stdout
+ assert len(http.paths) == 2 and set(http.paths) == SUMS_PATHS, \
+ f"a converged pass downloads exactly SHA256SUMS and SHA256SUMS.sig, got: {http.paths}"
+ assert "no changes; nothing to do" in second.stdout, second.stdout
+ assert "fetch: downloaded" not in second.stdout, second.stdout
+ assert env.calls() == [], env.calls()
+def test_fetch_downloads_only_the_changed_raw(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST, keys["key"])
+ with serve(served) as http:
+ first = env.run(url=http.url, pub=keys["pub"])
+ assert first.returncode == 0, first.stderr or first.stdout
+ (served / "sci-50-virt-ch.raw").write_text("sci-50-virt-ch raw content v2\n", encoding="utf-8")
+ rehash_and_resign(served, "sci-50-virt-ch.raw", keys["key"])
+ http.reset()
+ second = env.run(url=http.url, pub=keys["pub"])
+ assert second.returncode == 0, second.stderr or second.stdout
+ assert len(http.paths) == 3 and set(http.paths) == SUMS_PATHS | {"/sci-50-virt-ch.raw"}, \
+ f"the second pass must fetch sums, sig and only the changed raw, got: {http.paths}"
+ assert "fetch: downloaded sci-50-virt-ch.raw" in second.stdout, second.stdout
+ for fname in INVENTORY:
+ if fname != "sci-50-virt-ch.raw":
+ assert f"fetch: up to date {fname}" in second.stdout, second.stdout
+ assert (env.host / "var/lib/extensions/sci-50-virt-ch.raw").read_bytes() == b"sci-50-virt-ch raw content v2\n"
+ assert (env.host / "var/lib/extensions/sci-51-virt-qemu.raw").read_bytes() == (served / "sci-51-virt-qemu.raw").read_bytes()
+ assert env.calls() == list(SEQ_CHANGED_RAW), env.calls()
+def test_fetch_refuses_a_bad_signature(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST, keys["other_key"]) # signature from a different key
+ with serve(served) as http:
+ res = env.run(url=http.url, pub=keys["pub"])
+ expect_refused(env, res)
+ assert "signature" in res.stderr.lower(), f"the error must mention the signature: {res.stderr}"
+def test_fetch_refuses_a_wrong_sums_digest(tmp_path: Path, keys: dict[str, Path]):
+ served = tmp_path / "served"
+ make_build(served, MANIFEST, keys["key"])
+ good_digest = sha256_file(served / "SHA256SUMS")
+ env_ok = FetchEnv(tmp_path / "env-good")
+ with serve(served) as http:
+ good = env_ok.run(url=http.url, pub=keys["pub"], extra={"FETCH_SUMS_DIGEST": good_digest})
+ assert good.returncode == 0, f"the correct digest must be accepted: {good.stderr or good.stdout}"
+ env = FetchEnv(tmp_path / "env-bad")
+ with serve(served) as http:
+ res = env.run(url=http.url, pub=keys["pub"], extra={"FETCH_SUMS_DIGEST": "0" * 64})
+ expect_refused(env, res)
+def test_fetch_refuses_a_traversing_name(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST, keys["key"])
+ sums = served / "SHA256SUMS"
+ lines = sums.read_text(encoding="utf-8").splitlines()
+ lines[2] = f"{hashlib.sha256(b'traversal fixture\n').hexdigest()} ../../etc/passwd"
+ sums.write_text("\n".join(lines) + "\n", encoding="utf-8")
+ sign_sums(served, keys["key"]) # the signature stays valid: only the name is illegal
+ with serve(served) as http:
+ res = env.run(url=http.url, pub=keys["pub"])
+ expect_refused(env, res)
+ assert_no_passwd_strays(env)
+def test_fetch_refuses_a_file_that_does_not_match_its_hash(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST, keys["key"])
+ (served / "sci-50-virt-ch.raw").write_text("served bytes differ from the summed content\n", encoding="utf-8")
+ with serve(served) as http:
+ res = env.run(url=http.url, pub=keys["pub"])
+ expect_refused(env, res)
+def test_fetch_offline_keeps_the_deployed_set(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST, keys["key"])
+ url = None
+ with serve(served) as http:
+ url = http.url
+ first = env.run(url=http.url, pub=keys["pub"])
+ assert first.returncode == 0, first.stderr or first.stdout
+ assert env.calls() == list(SEQ_FRESH), env.calls()
+ snapshot = env.host_tree()
+ offline = env.run(url=url, pub=keys["pub"])
+ assert offline.returncode != 0, f"an offline fetch pass must fail, rc=0: {offline.stdout}"
+ assert "error: fetch:" in offline.stderr, offline.stderr
+ assert env.host_tree() == snapshot, "the offline pass mutated the deployed set"
+ assert env.calls() == [], f"the offline pass made host calls (incl. refresh): {env.calls()}"
+def test_fetch_prunes_a_dropped_extension(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST_DROP_A, keys["key"])
+ with serve(served) as http:
+ first = env.run(url=http.url, pub=keys["pub"])
+ assert first.returncode == 0, first.stderr or first.stdout
+ # Build B drops sci-51-virt-qemu: the manifest tombstones it and the
+ # signed inventory no longer lists its raw or shim.
+ make_build(served, MANIFEST_DROP_B, keys["key"])
+ second = env.run(url=http.url, pub=keys["pub"])
+ assert second.returncode == 0, (
+ f"the pass that drops an extension must succeed, rc={second.returncode}: "
+ f"{second.stderr or second.stdout}")
+ assert "has no manifest entry" not in second.stderr, (
+ f"the pass died on a staging shim left over from build A: {second.stderr}")
+ assert "removed sci-51-virt-qemu" in second.stdout, second.stdout
+ calls_second = env.calls()
+ assert not any("sci-51-virt-qemu" in call for call in calls_second), (
+ f"units of a removed entry must never be touched: {calls_second}")
+ http.reset()
+ third = env.run(url=http.url, pub=keys["pub"])
+ assert third.returncode == 0, third.stderr or third.stdout
+ assert len(http.paths) == 2 and set(http.paths) == SUMS_PATHS, (
+ f"the third pass must be a converged no-op that downloads only the sums: {http.paths}")
+ assert "no changes; nothing to do" in third.stdout, third.stdout
+ assert "fetch: downloaded" not in third.stdout, third.stdout
+ assert not (env.host / "var/lib/extensions/sci-51-virt-qemu.raw").exists(), \
+ "the dropped extension's raw was not removed from the host"
+ assert not (env.host / "etc/systemd/system/sci-51-virt-qemu-activation.service").exists(), \
+ "the dropped extension's activation shim was not removed from the host"
+ stale = sorted(p.name for p in env.staging.glob("sci-51-virt-qemu*"))
+ assert not stale, (
+ f"$FETCH_DIR/staging still holds build-A files of the dropped extension: {stale}")
+ assert env.deployed() == {
+ "var/lib/extensions/sci-21-network.raw": (served / "sci-21-network.raw").read_bytes(),
+ "etc/systemd/system/sci-21-network-activation.service": (served / "sci-21-network-activation.service").read_bytes(),
+ }, env.deployed()
+ assert env.calls() == [], f"the third pass made host calls: {env.calls()}"
+def test_remove_directive_unmerges_an_extension(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST_REMOVE, keys["key"])
+ preplace(env, NAMES) # a previous build deployed both extensions
+ with serve(served) as http:
+ res = env.run(url=http.url, pub=keys["pub"])
+ assert res.returncode == 0, res.stderr or res.stdout
+ assert not (env.host / "var/lib/extensions/sci-51-virt-qemu.raw").exists(), "the removed raw was not deleted"
+ assert not (env.host / "etc/systemd/system/sci-51-virt-qemu-activation.service").exists(), "the removed shim was not deleted"
+ assert "removed sci-51-virt-qemu" in res.stdout, res.stdout
+ assert env.deployed() == {
+ "var/lib/extensions/sci-50-virt-ch.raw": (served / "sci-50-virt-ch.raw").read_bytes(),
+ "etc/systemd/system/sci-50-virt-ch-activation.service": (served / "sci-50-virt-ch-activation.service").read_bytes(),
+ }, env.deployed()
+ assert env.calls() == list(SEQ_REMOVE_ONLY), env.calls()
+ assert not any("sci-51-virt-qemu" in call for call in env.calls()), \
+ f"units of a removed entry must never be restarted: {env.calls()}"
+def test_remove_is_idempotent(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST_REMOVE, keys["key"])
+ preplace(env, NAMES)
+ with serve(served) as http:
+ first = env.run(url=http.url, pub=keys["pub"])
+ assert first.returncode == 0, first.stderr or first.stdout
+ http.reset()
+ second = env.run(url=http.url, pub=keys["pub"])
+ assert second.returncode == 0, second.stderr or second.stdout
+ assert len(http.paths) == 2 and set(http.paths) == SUMS_PATHS, \
+ f"removing an absent extension is not a change; the second pass must be converged: {http.paths}"
+ assert "no changes; nothing to do" in second.stdout, second.stdout
+ assert "removed" not in second.stdout, second.stdout
+ assert env.calls() == [], f"the second pass made host calls: {env.calls()}"
+def test_remove_drops_the_enable_symlink(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ # Real systemctl enable creates the multi-user.target.wants symlink the
+ # activation shim's [Install] section asks for; the plain logging stub
+ # would not, so this test swaps in a stub that reproduces both effects
+ # (enable creates the symlink, disable removes it).
+ write_stub(env.stubs, "systemctl", SYSTEMCTL_ENABLE_STUB)
+ served = tmp_path / "served"
+ make_build(served, MANIFEST_REMOVE, keys["key"])
+ preplace(env, NAMES) # a previous build deployed both extensions
+ wants = env.host / "etc/systemd/system/multi-user.target.wants" / "sci-51-virt-qemu-activation.service"
+ wants.parent.mkdir(parents=True, exist_ok=True)
+ wants.symlink_to("../sci-51-virt-qemu-activation.service") # the enable symlink a prior build left
+ with serve(served) as http:
+ res = env.run(url=http.url, pub=keys["pub"])
+ assert res.returncode == 0, res.stderr or res.stdout
+ assert "removed sci-51-virt-qemu" in res.stdout, res.stdout
+ assert not wants.exists() and not wants.is_symlink(), \
+ f"the enable symlink of the removed extension survived the REMOVE pass: {wants}"
+ assert (env.host / "etc/systemd/system/multi-user.target.wants/sci-50-virt-ch-activation.service").is_symlink(), \
+ "the stub created no enable symlink on 'systemctl enable': the test cannot exercise the defect"
+def test_remove_refuses_a_listed_entry(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST_REMOVE_LISTED, keys["key"])
+ placed = preplace(env, NAMES)
+ with serve(served) as http:
+ res = env.run(url=http.url, pub=keys["pub"])
+ expect_refused(env, res, placed=placed)
+ assert_placed(env, placed)
+def test_remove_refuses_a_traversing_name(tmp_path: Path, keys: dict[str, Path]):
+ env = FetchEnv(tmp_path / "env")
+ served = tmp_path / "served"
+ make_build(served, MANIFEST_REMOVE_TRAVERSE, keys["key"])
+ placed = preplace(env, NAMES)
+ with serve(served) as http:
+ res = env.run(url=http.url, pub=keys["pub"])
+ expect_refused(env, res, placed=placed)
+ assert_placed(env, placed)
+ assert_no_passwd_strays(env)
+def test_help_documents_fetch_and_remove():
+ res = sh(["bash", DELIVERY_SH, "--help"], env=dict(os.environ))
+ assert res.returncode == 0, res.stderr
+ for token in ("FETCH_URL", "FETCH_PUBKEY", "FETCH_SUMS_DIGEST", "FETCH_DIR", "REMOVE="):
+ assert token in res.stdout, f"--help does not document {token!r}"
diff --git a/extensions/tests/test_host_contracts.py b/extensions/tests/test_host_contracts.py
new file mode 100644
index 00000000..a3dd830b
--- /dev/null
+++ b/extensions/tests/test_host_contracts.py
@@ -0,0 +1,536 @@
+"""Host-safe contract tests: the real sysext-virt scripts run against tiny
+fixtures with host-mutating tools stubbed. No lib imports, builds, VMs, docker
+or network; negative cases pin fail-closed behavior.
+"""
+from __future__ import annotations
+import hashlib
+import os
+import re
+import shutil
+import signal
+import subprocess
+import sys
+import time
+from pathlib import Path
+import pytest
+REPO = Path(__file__).resolve().parents[2]
+BASE_IMAGE_SH, MKOSI_CONF = REPO / "mkosi/base-image.sh", REPO / "mkosi/mkosi.conf"
+DELIVERY_SH = REPO / "delivery/sci-delivery.sh"
+MANIFEST = REPO / "delivery/extensions.manifest"
+FLAVOR, ARCH, COMMIT = "sysext-fixture", "amd64", "a" * 40
+TRIO = ("systemctl daemon-reload", "systemd-sysext refresh", "systemctl reload dbus.service")
+SHIM_TEMPLATE = ("[Unit]\nDescription=SCI {name} activation (host-contracts fixture)\nAfter=systemd-sysext.service\n"
+ "[Service]\nType=oneshot\nRemainAfterExit=yes\nExecStart=/usr/bin/true\n"
+ "[Install]\nWantedBy=multi-user.target\n")
+FIXTURE_MANIFEST = "TARGET_BASE=2150\nsci-alpha foo-one.service foo-two.service\nsci-beta\n"
+LOG_STUB = "#!/bin/sh\nprintf 'TOOL %s\\n' \"$*\" >> \"${STUB_LOG:-/dev/null}\"\nexit 0\n"
+def seq(*rest, enabled=("sci-alpha",)):
+ return TRIO + rest + tuple(f"systemctl {verb} {name}-activation.service" for name in enabled for verb in ("enable", "restart"))
+SEQ_FRESH = seq("systemctl restart foo-one.service foo-two.service")
+SEQ_SHIM = seq()
+SEQ_NEW = TRIO + ("systemctl restart gamma-one.service", "systemctl enable sci-alpha-activation.service")
+SEQ_TOLERATED = seq("systemctl restart foo-one.service foo-two.service", "systemctl restart multipathd.service iscsid.service", enabled=("sci-alpha", "sci-beta"))
+def sha256_file(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+def sh(argv, *, cwd=None, env=None, timeout=180):
+ return subprocess.run([str(a) for a in argv], cwd=str(cwd or REPO), env=env, timeout=timeout, capture_output=True, text=True)
+def last_line(text: str) -> str:
+ return text.rstrip("\n").splitlines()[-1].strip()
+def tree_digest(tree: Path) -> str:
+ proc = subprocess.run(["tar", "-C", str(tree), "-cf", "-", "--sort=name", "--owner=0", "--group=0", "--numeric-owner", "--mtime=@0", "."], check=True, capture_output=True)
+ return hashlib.sha256(proc.stdout).hexdigest()
+def sleeping(started: Path) -> bool:
+ try:
+ return Path(f"/proc/{int(started.read_text(encoding='utf-8').strip())}/comm").read_text(encoding="utf-8").strip() == "sleep"
+ except (OSError, ValueError):
+ return False
+def release() -> str:
+ found = re.findall(r"^Release=(.*)$", MKOSI_CONF.read_text(encoding="utf-8"), re.M)
+ assert len(found) == 1, f"mkosi/mkosi.conf must carry exactly one Release= line: {found}"
+ return found[0]
+def write_stub(directory: Path, name: str, body: str) -> None:
+ directory.mkdir(parents=True, exist_ok=True)
+ path = directory / name
+ path.write_text(body, encoding="utf-8")
+ path.chmod(0o755)
+def manifest_names(path: Path = MANIFEST) -> list[str]:
+ names = [l.split()[0] for l in path.read_text(encoding="utf-8").splitlines() if l.strip() and not l.lstrip().startswith("#") and not re.match(r"^[A-Z][A-Z0-9_]*=\S*$", l)]
+ assert names, f"{path} carries no entries"
+ return names
+
+@pytest.fixture
+def base_artifact(tmp_path: Path) -> dict:
+ version = release()
+ rootfs = tmp_path / "rootfs"
+ (rootfs / "etc").mkdir(parents=True)
+ (rootfs / "boot").mkdir()
+ (rootfs / "etc/os-release").write_text(f"ID=gardenlinux\nID_LIKE=debian\nIMAGE_VERSION={version}\nGARDENLINUX_VERSION={version}\nGARDENLINUX_COMMIT_ID_LONG={COMMIT}\nVARIANT_ID={FLAVOR}-{ARCH}\n", encoding="utf-8")
+ (rootfs / "etc/machine-id").write_text("uninitialized\n", encoding="utf-8")
+ (rootfs / "boot/vmlinuz-1.0.0").write_text("fixture kernel\n", encoding="utf-8")
+ (rootfs / "boot/initrd.img-1.0.0").write_text("fixture initrd\n", encoding="utf-8")
+ inner = f"{FLAVOR}-{ARCH}-{version}-{COMMIT[:8]}.tar"
+ sh(["tar", "-cf", tmp_path / inner, "."], cwd=rootfs)
+ outer = tmp_path / "outer.tar.xz"
+ sh(["tar", "-C", tmp_path, "-cJf", outer, "--", inner])
+ pin = tmp_path / "pin"
+ pin.write_text(f"FLAVOR={FLAVOR}\nARCH={ARCH}\nVERSION={version}\nCOMMIT={COMMIT}\nTARBALL_SHA256={sha256_file(outer)}\nROOTFS_TAR_SHA256={sha256_file(tmp_path / inner)}\nURL=https://github.com/gardenlinux/gardenlinux/releases/download/{version}/{inner}.xz\n", encoding="utf-8")
+ return {"rootfs": rootfs, "inner": inner, "outer": outer, "pin": pin}
+def base_tool(pin: Path, args, cache: Path, tarball: Path | None = None):
+ env = {k: v for k, v in os.environ.items() if not k.startswith("BASE_IMAGE_")}
+ env.update(BASE_IMAGE_PIN=str(pin), BASE_IMAGE_DIR=str(cache), BASE_IMAGE_OFFLINE="1")
+ if tarball is not None:
+ env["BASE_IMAGE_TARBALL"] = str(tarball)
+ return sh(["bash", BASE_IMAGE_SH, *args], env=env)
+def variant_pin(fx: dict, tmp_path: Path, tree: Path) -> tuple[Path, Path]:
+ inner = tmp_path / fx["inner"]
+ inner.unlink()
+ sh(["tar", "-cf", inner, "."], cwd=tree)
+ outer = tmp_path / "variant.tar.xz"
+ sh(["tar", "-C", tmp_path, "-cJf", outer, "--", fx["inner"]])
+ text = fx["pin"].read_text(encoding="utf-8")
+ text = re.sub(r"^TARBALL_SHA256=.*$", f"TARBALL_SHA256={sha256_file(outer)}", text, flags=re.M)
+ text = re.sub(r"^ROOTFS_TAR_SHA256=.*$", f"ROOTFS_TAR_SHA256={sha256_file(inner)}", text, flags=re.M)
+ (tmp_path / "variant.pin").write_text(text, encoding="utf-8")
+ return tmp_path / "variant.pin", outer
+def assert_cache_links(cache: Path, tree: Path) -> None:
+ assert (cache / "current").is_symlink() and os.readlink(cache / "current") == tree.name and (cache / "current").resolve() == tree, f"cache/current does not resolve to {tree}"
+ for link, boot in (("current-kernel", "vmlinuz-1.0.0"), ("current-initrd", "initrd.img-1.0.0")):
+ assert (cache / link).is_symlink() and os.readlink(cache / link) == f"{tree.name}/boot/{boot}" and (cache / link).resolve() == tree / "boot" / boot, f"{link} does not resolve to {tree}/boot/{boot}"
+def test_base_clean_extraction(base_artifact, tmp_path: Path):
+ fx = base_artifact
+ cache = tmp_path / "cache"
+ res = base_tool(fx["pin"], ("ensure",), cache, tarball=fx["outer"])
+ tree = Path(last_line(res.stdout))
+ assert res.returncode == 0 and tree.is_dir() and tree_digest(tree) == tree_digest(fx["rootfs"]), res.stderr
+ assert_cache_links(cache, tree)
+ dest = tmp_path / "extract"
+ res = base_tool(fx["pin"], ("extract", str(dest)), tmp_path / "cache2", tarball=fx["outer"])
+ assert res.returncode == 0 and last_line(res.stdout) == tree_digest(tree) == tree_digest(dest), res.stderr
+ again = base_tool(fx["pin"], ("ensure",), cache) # offline cache hit, no tarball
+ assert again.returncode == 0 and last_line(again.stdout) == str(tree), again.stderr
+ assert_cache_links(cache, tree)
+
+PIN_DEFECTS = [
+ ("missing-flavor", lambda t: t.replace(f"FLAVOR={FLAVOR}\n", "", 1), "exactly one FLAVOR"),
+ ("duplicate-url", lambda t: t + "URL=stale\n", "exactly one URL"),
+ ("bad-commit", lambda t: re.sub(r"^COMMIT=.*$", "COMMIT=abc123", t, flags=re.M), "40-hex-digit commit"),
+ ("short-outer-sha", lambda t: re.sub(r"^TARBALL_SHA256=.*$", "TARBALL_SHA256=" + "0" * 63, t, flags=re.M), "64-hex-digit sha256"),
+ ("url-skew", lambda t: re.sub(r"^URL=.*$", "URL=https://evil.invalid/x.tar.xz", t, flags=re.M), "URL derived"),
+ ("version-skew", lambda t: re.sub(r"^VERSION=.*$", "VERSION=9999.9.9", t, flags=re.M), "does not match Release="),
+]
+@pytest.mark.parametrize("defect,mutate,fragment", PIN_DEFECTS, ids=[d for d, *_ in PIN_DEFECTS])
+def test_base_pin_refused(base_artifact, tmp_path: Path, defect, mutate, fragment):
+ fx = base_artifact
+ pin = tmp_path / "pin2"
+ pin.write_text(mutate(fx["pin"].read_text(encoding="utf-8")), encoding="utf-8")
+ res = base_tool(pin, ("ensure",), tmp_path / "cache") # offline, no tarball: only validation can pass
+ assert res.returncode != 0 and fragment in res.stderr, res.stderr
+ assert not (tmp_path / "cache").exists(), "the cache was created before the refusal (not fail-closed)"
+@pytest.mark.parametrize("kind", ["outer", "inner"])
+def test_base_corrupt_refused(base_artifact, tmp_path: Path, kind):
+ fx = base_artifact
+ if kind == "outer":
+ bad, data = tmp_path / "bad.tar.xz", bytearray(fx["outer"].read_bytes())
+ data[0] ^= 0xFF
+ bad.write_bytes(data)
+ pin, expected = fx["pin"], sha256_file(fx["outer"])
+ else:
+ rogue = tmp_path / "rogue"
+ shutil.copytree(fx["rootfs"], rogue)
+ (rogue / "etc/rogue-file").write_text("rogue\n", encoding="utf-8")
+ sh(["tar", "-cf", tmp_path / fx["inner"], "."], cwd=rogue)
+ sh(["tar", "-C", tmp_path, "-cJf", tmp_path / "outer2.tar.xz", "--", fx["inner"]])
+ bad, pin = tmp_path / "outer2.tar.xz", tmp_path / "pin2"
+ pin.write_text(re.sub(r"^TARBALL_SHA256=.*$", f"TARBALL_SHA256={sha256_file(bad)}", fx["pin"].read_text(encoding="utf-8"), flags=re.M), encoding="utf-8")
+ expected = sha256_file(tmp_path / fx["inner"])
+ res = base_tool(pin, ("ensure",), tmp_path / "cache", tarball=bad)
+ assert res.returncode != 0 and "sha256 mismatch" in res.stderr and expected in res.stderr, res.stderr
+@pytest.mark.parametrize("kind", ["offline", "nonempty"])
+def test_base_refused_without_side_effects(base_artifact, tmp_path: Path, kind):
+ fx = base_artifact
+ if kind == "offline":
+ res = base_tool(fx["pin"], ("ensure",), tmp_path / "cache")
+ needle = "offline"
+ else:
+ dest = tmp_path / "occupied"
+ dest.mkdir()
+ (dest / "stray.txt").write_text("stray\n", encoding="utf-8")
+ res = base_tool(fx["pin"], ("extract", str(dest)), tmp_path / "cache", tarball=fx["outer"])
+ needle = "not empty"
+ assert res.returncode != 0 and needle in res.stderr, res.stderr
+def test_base_wrong_identity_refused(base_artifact, tmp_path: Path):
+ fx = base_artifact
+ rel = fx["rootfs"] / "etc/os-release"
+ rel.write_text(rel.read_text(encoding="utf-8") + "VERSION_ID=99\n", encoding="utf-8")
+ pin, outer = variant_pin(fx, tmp_path, fx["rootfs"])
+ res = base_tool(pin, ("ensure",), tmp_path / "cache", tarball=outer)
+ assert res.returncode != 0 and "carries VERSION_ID" in res.stderr, f"the VERSION_ID tree was not refused (rc={res.returncode}): {res.stderr}"
+ assert not list((tmp_path / "cache").glob("*.ready")), "a rejected tree was marked ready"
+def test_base_cache_marker_reextracts(base_artifact, tmp_path: Path):
+ fx = base_artifact
+ cache = tmp_path / "cache"
+ first = base_tool(fx["pin"], ("ensure",), cache, tarball=fx["outer"])
+ assert first.returncode == 0, first.stderr
+ tree = Path(last_line(first.stdout))
+ (fx["rootfs"] / "etc/rogue-file").write_text("rogue\n", encoding="utf-8")
+ pin, outer = variant_pin(fx, tmp_path, fx["rootfs"])
+ second = base_tool(pin, ("ensure",), cache, tarball=outer)
+ assert second.returncode == 0 and Path(last_line(second.stdout)) == tree, second.stderr
+ rogue = tree / "etc/rogue-file"
+ assert rogue.is_file() and rogue.read_text(encoding="utf-8") == "rogue\n", "the stale same-key tree was reused"
+ assert (cache / f"{tree.name}.ready").read_text(encoding="utf-8").strip() == sha256_file(tmp_path / fx["inner"])
+
+# delivery/sci-delivery.sh (the reconciling delivery step)
+class DeliveryEnv:
+ def __init__(self, root: Path):
+ self.root = Path(root)
+ self.host, self.staging, self.stubs = self.root / "host", self.root / "staging", self.root / "stubs"
+ self.calls_log = self.root / "calls.log"
+ for tool in ("systemctl", "systemd-sysext", "nsenter", "docker"):
+ write_stub(self.stubs, tool, LOG_STUB.replace("TOOL", tool))
+ write_stub(self.stubs, "sleep", "#!/bin/sh\nprintf '%s\\n' \"$$\" >\"${STUB_SLEEP_STARTED:-/dev/null}\"\nexec /bin/sleep \"$@\"\n")
+ self.host.mkdir(parents=True)
+ self.staging.mkdir()
+ for name in ("sci-alpha", "sci-beta"):
+ (self.staging / f"{name}.raw").write_text(f"{name} raw content v1\n", encoding="utf-8")
+ (self.staging / "sci-alpha-activation.service").write_text(SHIM_TEMPLATE.format(name="alpha"), encoding="utf-8")
+ self.set_manifest(FIXTURE_MANIFEST)
+ def set_manifest(self, text: str) -> None:
+ (self.staging / "extensions.manifest").write_text(text, encoding="utf-8")
+ names = manifest_names(self.staging / "extensions.manifest")
+ (self.staging / "SHA256SUMS").write_text("".join(f"{sha256_file(self.staging / (n + '.raw'))} {n}.raw\n" for n in names), encoding="utf-8")
+ def failing(self, guard: str) -> Path:
+ directory = self.root / "failing"
+ write_stub(directory, "systemctl", LOG_STUB.replace("TOOL", "systemctl").replace("exit 0\n", f"{guard}\nexit 0\n"))
+ return directory
+ def os_release(self, version="2150.10.3", *, quoted=False, fallback_only=False) -> None:
+ value = f'"{version}"' if quoted else version
+ (self.host / "etc").mkdir(parents=True, exist_ok=True)
+ image = "" if fallback_only else f"IMAGE_VERSION={value}\n"
+ (self.host / "etc/os-release").write_text(f"ID=gardenlinux\n{image}GARDENLINUX_VERSION={value}\n", encoding="utf-8")
+ def env(self, *, interval=0, nsenter=0, extra=None, path_prefix=None) -> dict:
+ env = dict(os.environ, HOST_ROOT=str(self.host), STAGING_DIR=str(self.staging), INTERVAL=str(interval), NSENTER=str(nsenter), STUB_LOG=str(self.calls_log))
+ env["PATH"] = os.pathsep.join(([str(path_prefix)] if path_prefix else []) + [str(self.stubs), os.environ.get("PATH", "")])
+ return env | {k: str(v) for k, v in (extra or {}).items()}
+ def run(self, **kwargs):
+ self.calls_log.write_text("", encoding="utf-8")
+ res = sh(["bash", DELIVERY_SH], env=self.env(**kwargs))
+ return res.returncode, res.stdout, res.stderr, self.calls()
+ def calls(self) -> list[str]:
+ return self.calls_log.read_text(encoding="utf-8").splitlines() if self.calls_log.is_file() else []
+ def host_state(self) -> dict:
+ return {str(p.relative_to(self.host)): (p.stat().st_mtime_ns, p.stat().st_size) for p in self.host.rglob("*") if p.is_file()}
+def check(env, *, rc=0, calls=None, before=None, errors=(), changed=None, files=(), **kwargs):
+ before = env.host_state() if before is None else before
+ code, out, err, seen = env.run(**kwargs)
+ assert code == rc, f"rc={code}, expected {rc}: {err or out}"
+ assert calls is None or seen == list(calls), f"calls differ:\n {seen}"
+ assert all(needle in err for needle in errors), err
+ if changed is not None:
+ after = env.host_state()
+ assert {p for p in set(before) | set(after) if before.get(p) != after.get(p)} == set(changed)
+ for rel in files:
+ assert (env.host / rel).read_bytes() == (env.staging / Path(rel).name).read_bytes(), rel
+ return out, err, seen, before
+
+def deploy(env) -> None:
+ env.os_release()
+ assert env.run()[0] == 0
+
+@pytest.fixture
+def delivery(tmp_path: Path) -> DeliveryEnv:
+ return DeliveryEnv(tmp_path / "delivery")
+@pytest.mark.parametrize("nsenter", [0, 1], ids=["direct", "nsenter"])
+def test_delivery_fresh(delivery, nsenter):
+ delivery.os_release()
+ expected = SEQ_FRESH if not nsenter else tuple(f"nsenter -t 1 -m -- {c}" for c in SEQ_FRESH)
+ check(delivery, calls=expected, nsenter=nsenter, files=("var/lib/extensions/sci-alpha.raw", "var/lib/extensions/sci-beta.raw", "etc/systemd/system/sci-alpha-activation.service"))
+@pytest.mark.parametrize("kind", ["noop", "foreign", "raw", "shim", "new"])
+def test_delivery_reconcile(delivery, kind):
+ deploy(delivery)
+ legacy = delivery.host / "var/lib/extensions/sci-legacy.raw"
+ shim_dir = delivery.host / "etc/systemd/system"
+ shim_mtime = shim_dir.stat().st_mtime_ns if kind == "shim" else None
+ if kind == "foreign":
+ legacy.write_text("legacy raw\n", encoding="utf-8")
+ if kind == "new":
+ (delivery.staging / "sci-gamma.raw").write_text("sci-gamma raw content v1\n", encoding="utf-8")
+ delivery.set_manifest(FIXTURE_MANIFEST + "sci-gamma gamma-one.service\n")
+ target, calls = "var/lib/extensions/sci-gamma.raw", SEQ_NEW
+ elif kind in ("raw", "shim"):
+ target = "var/lib/extensions/sci-alpha.raw" if kind == "raw" else "etc/systemd/system/sci-alpha-activation.service"
+ (delivery.host / target).write_text("drifted\n", encoding="utf-8")
+ calls = SEQ_FRESH if kind == "raw" else SEQ_SHIM
+ else:
+ target, calls = None, []
+ check(delivery, calls=calls, before=delivery.host_state(), changed=(target,) if target else (), files=(target,) if target else ())
+ assert kind != "foreign" or legacy.read_text(encoding="utf-8") == "legacy raw\n"
+ assert shim_mtime is None or shim_dir.stat().st_mtime_ns == shim_mtime, "the shim directory mtime changed"
+def stage(*, version="2150.10.3", manifest=None, edit=None, host_root=False, gate_absent=False):
+ def apply(env):
+ env.os_release(version)
+ if manifest is not None:
+ (env.staging / "extensions.manifest").write_text(manifest, encoding="utf-8")
+ if edit:
+ edit(env)
+ return {"HOST_ROOT": str(env.root / "no-such-root")} if host_root else None
+ apply.gate_absent = gate_absent
+ return apply
+def drop_beta_sum(env) -> None:
+ sums = env.staging / "SHA256SUMS"
+ sums.write_text(next(l for l in sums.read_text(encoding="utf-8").splitlines() if "sci-alpha" in l) + "\n", encoding="utf-8")
+
+REFUSED = [
+ ("whitespace", stage(manifest=" \t\n"), "whitespace-only"),
+ ("unknown-directive", stage(manifest="FOO=1\n"), "unknown manifest directive"),
+ ("no-target-base", stage(manifest="sci-alpha\n"), "no TARGET_BASE"),
+ ("duplicate-target-base", stage(version="1877.26.1", manifest="TARGET_BASE=2150\nTARGET_BASE=1877\nsci-alpha foo-one.service foo-two.service\n"), "duplicate TARGET_BASE"),
+ ("missing-raw", stage(manifest="TARGET_BASE=2150\nsci-missing\n"), "has no staging raw"),
+ ("unlisted-shim", stage(edit=lambda e: (e.staging / "sci-rogue-activation.service").write_text("[Unit]\n")), "has no manifest entry"),
+ ("missing-manifest", stage(edit=lambda e: (e.staging / "extensions.manifest").unlink()), "missing manifest"),
+ ("bad-name", stage(manifest="TARGET_BASE=2150\nsci/alpha\n"), "invalid name"),
+ ("host-root", stage(host_root=True), "not an existing directory"),
+ ("missing-sums", stage(edit=lambda e: (e.staging / "SHA256SUMS").unlink()), "missing SHA256SUMS"),
+ ("missing-line", stage(edit=drop_beta_sum), "no line for manifest entry"),
+ ("corrupt-raw", stage(edit=lambda e: (e.staging / "sci-alpha.raw").write_text("CORRUPTED\n")), "does not match its SHA256SUMS line"),
+ ("before-gate", stage(version="1877.26.1", gate_absent=True, edit=lambda e: (e.staging / "sci-alpha.raw").write_text("CORRUPTED\n")), "does not match its SHA256SUMS line"),
+]
+@pytest.mark.parametrize("defect,setup,needle", REFUSED, ids=[d for d, *_ in REFUSED])
+def test_delivery_refused(delivery, defect, setup, needle):
+ extra = setup(delivery) or {}
+ _out, err, _seen, _before = check(delivery, rc=1, calls=[], before=delivery.host_state(), errors=(needle,), changed=(), extra=extra)
+ assert not setup.gate_absent or "base-version gate" not in err, "the gate ran before the integrity check"
+ assert not extra or not (delivery.root / "no-such-root").exists(), "the delivery step created HOST_ROOT"
+@pytest.mark.parametrize("scenario", ["accept-quoted", "accept-fallback", "reject-1877", "reject-fallback-1877", "reject-missing-os-release", "reject-already-deployed"])
+def test_delivery_gate(delivery, scenario):
+ if scenario.startswith("accept-"):
+ delivery.os_release(quoted=scenario == "accept-quoted", fallback_only=scenario == "accept-fallback")
+ out = check(delivery, calls=SEQ_FRESH)[0]
+ assert scenario != "accept-fallback" or "host Garden Linux line 2150 matches TARGET_BASE=2150" in out
+ return
+ if scenario == "reject-already-deployed":
+ deploy(delivery)
+ delivery.os_release("1877.26.1")
+ elif scenario != "reject-missing-os-release":
+ delivery.os_release("1877.26.1", fallback_only=scenario == "reject-fallback-1877")
+ needles = ("2150",) if scenario == "reject-missing-os-release" else ("1877", "2150")
+ err = check(delivery, rc=1, calls=[], before=delivery.host_state(), errors=needles, changed=())[1]
+ assert scenario != "reject-missing-os-release" or "os-release" in err, err
+
+@pytest.mark.parametrize("failure", ["trio", "restart"])
+def test_delivery_activation_failure(delivery, failure):
+ delivery.os_release()
+ marker = delivery.host / "var/lib/extensions/.sci-delivery-pending"
+ flag = delivery.root / "fail-daemon-reload"
+ if failure == "restart":
+ (delivery.staging / "sci-beta-activation.service").write_text(SHIM_TEMPLATE.format(name="beta"), encoding="utf-8")
+ delivery.set_manifest("TARGET_BASE=2150\nsci-alpha foo-one.service foo-two.service\nsci-beta multipathd.service iscsid.service\n")
+ guard = 'if [ "$*" = "restart multipathd.service iscsid.service" ]; then exit 1; fi'
+ else:
+ flag.write_text("", encoding="utf-8")
+ guard = f'if [ -e "{flag}" ] && [ "$1" = daemon-reload ]; then exit 1; fi'
+ expected = ["systemctl daemon-reload"] if failure == "trio" else SEQ_TOLERATED
+ errors = ("sci-beta", "multipathd.service", "restart failed") if failure == "restart" else ()
+ files = ("var/lib/extensions/sci-beta.raw", "etc/systemd/system/sci-beta-activation.service") if failure == "restart" else ()
+ check(delivery, rc=1, calls=expected, path_prefix=delivery.failing(guard), errors=errors, files=files)
+ if failure == "restart":
+ assert not marker.exists(), "a completed flow left the pending-activation marker"
+ else:
+ assert marker.is_file(), "the interrupted pass left no pending-activation marker"
+ flag.unlink()
+ check(delivery, calls=SEQ_FRESH)
+ assert not marker.exists(), "the reconciliation pass left the marker behind"
+def test_delivery_interval_term(delivery):
+ delivery.os_release()
+ started = delivery.root / "sleep.started"
+ check(delivery, calls=SEQ_FRESH, extra={"STUB_SLEEP_STARTED": str(started)})
+ assert not started.exists(), "INTERVAL=0 still invoked sleep"
+ loop = DeliveryEnv(delivery.root / "loop")
+ loop.os_release()
+ started = loop.root / "sleep.started"
+ interval = 8 # real seconds: a foreground sleep defers the trap for ~this long
+ proc = subprocess.Popen(["bash", str(DELIVERY_SH)], cwd=str(REPO), env=loop.env(interval=interval, extra={"STUB_SLEEP_STARTED": str(started)}), start_new_session=True)
+ try:
+ deadline = time.monotonic() + 15
+ while time.monotonic() < deadline and not sleeping(started):
+ assert proc.poll() is None, f"the delivery step exited before the interval sleep (rc={proc.returncode})"
+ time.sleep(0.05)
+ assert sleeping(started), "the delivery step never entered the real interval sleep"
+ term_at = time.monotonic()
+ proc.send_signal(signal.SIGTERM)
+ try:
+ rc = proc.wait(timeout=2.0)
+ except subprocess.TimeoutExpired:
+ assert False, "the delivery step did not exit promptly after SIGTERM (a foreground sleep defers the trap)"
+ elapsed = time.monotonic() - term_at
+ assert rc == 0 and elapsed < 2.0 < interval, f"TERM->exit {elapsed:.2f}s (rc={rc}, interval={interval})"
+ assert loop.calls() == list(SEQ_FRESH)
+ finally:
+ try:
+ os.killpg(proc.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ if proc.poll() is None:
+ proc.wait(timeout=5)
+
+# delivery/build-image.sh (packaging)
+
+FIXTURE_HEAD = "abc1234" # the stubbed HEAD keeps a reintroduced versioned tag observable in the tag assertions
+
+def _mini_repo(root: Path) -> Path:
+ shutil.copytree(REPO / "delivery", root / "delivery")
+ (root / "mkosi").mkdir()
+ (root / "mkosi/mkosi.conf").write_text(f"Release={release()}\n", encoding="utf-8")
+ output = root / ".build" / "mkosi" / "output"
+ output.mkdir(parents=True)
+ for name in manifest_names():
+ (output / f"{name}.raw").write_text(f"fake raw {name}\n", encoding="utf-8")
+ activation = root / "mkosi/mkosi.images" / name / "mkosi.extra/usr/share/sci/activation"
+ activation.mkdir(parents=True)
+ (activation / f"{name}-activation.service").write_text(SHIM_TEMPLATE.format(name=name), encoding="utf-8")
+ return root
+
+def _run_build_image(repo: Path, stubs: Path, log: Path,
+ args: list[str] | None = None, env_extra: dict[str, str] | None = None):
+ log.write_text("", encoding="utf-8")
+ write_stub(stubs, "docker", LOG_STUB.replace("TOOL", "docker"))
+ # A stubbed HEAD and the fixture Release= make a versioned tag reachable, so
+ # the tag assertions below pin that the image carries only $IMAGE_TAG.
+ write_stub(stubs, "git", f"#!/bin/sh\ncase \"$*\" in *rev-parse*) echo {FIXTURE_HEAD};; esac\nexit 0\n")
+ env = dict(os.environ, STUB_LOG=str(log), **(env_extra or {}))
+ env["PATH"] = os.pathsep.join([str(stubs), os.environ.get("PATH", "")])
+ return sh(["bash", "delivery/build-image.sh", *(args or [])], cwd=repo, env=env, timeout=120)
+
+def assert_staging(context: Path, repo: Path) -> None:
+ """Assert the staged delivery set: delivery step, raws, shims, manifest and checksums.
+
+ Takes the fixture repository that produced the raws the staging is compared to.
+ """
+ names = manifest_names()
+ delivery_step = context / "sci-delivery.sh"
+ assert delivery_step.is_file() and delivery_step.read_bytes() == DELIVERY_SH.read_bytes() and delivery_step.stat().st_mode & 0o111, \
+ "the staging must carry sci-delivery.sh byte-identical to delivery/sci-delivery.sh and executable"
+ for name in names:
+ built = repo / ".build/mkosi/output" / f"{name}.raw"
+ assert (context / f"{name}.raw").read_bytes() == built.read_bytes(), \
+ f"the staging must carry {name}.raw byte-identical to .build/mkosi/output/{name}.raw"
+ assert (context / f"{name}-activation.service").is_file(), \
+ f"the staging must carry the {name}-activation.service shim"
+ assert (context / "extensions.manifest").read_bytes() == MANIFEST.read_bytes(), \
+ "the staging must carry extensions.manifest byte-identical to delivery/extensions.manifest"
+ sums = context / "SHA256SUMS"
+ assert sums.is_file(), "the staging must carry SHA256SUMS"
+ recorded = {line.split()[1] for line in sums.read_text(encoding="utf-8").splitlines() if line.split()}
+ missing = [f"{name}.raw" for name in names if f"{name}.raw" not in recorded]
+ assert not missing, f"SHA256SUMS must record a line for every manifest raw, missing: {missing}"
+ check = sh(["sha256sum", "-c", "--", "SHA256SUMS"], cwd=context)
+ assert check.returncode == 0, f"sha256sum -c SHA256SUMS must verify every staged raw: {check.stdout}{check.stderr}"
+
+@pytest.mark.parametrize("variant", ["complete", "missing-raw"])
+def test_build_image(tmp_path: Path, variant):
+ names = manifest_names()
+ repo = _mini_repo(tmp_path / "repo")
+ if variant == "missing-raw":
+ (repo / ".build/mkosi/output" / f"{names[0]}.raw").unlink()
+ log = tmp_path / "docker.log"
+ res = _run_build_image(repo, tmp_path / "stubs", log)
+ lines = [l for l in log.read_text(encoding="utf-8").splitlines() if l.startswith("docker ")]
+ if variant == "missing-raw":
+ assert res.returncode != 0 and lines == [] and names[0] in res.stderr, res.stderr
+ return
+ assert res.returncode == 0 and len(lines) == 1, res.stderr or lines
+ tokens = lines[0].split()
+ tags = [tokens[i + 1] for i, token in enumerate(tokens) if token == "-t"]
+ assert tags == ["sci-delivery:latest"], f"the default build must tag exactly sci-delivery:latest, found: {tags}"
+ context = Path(tokens[-1])
+ context = context if context.is_absolute() else repo / context
+ assert_staging(context, repo)
+
+
+def test_build_image_stage_only(tmp_path: Path):
+ """--stage-only assembles the validated staging set without any image build.
+
+ The raws artifact publishes the staging directory as plain files, so the
+ staging must be reachable without docker: same manifest validation, same
+ staged SHA256SUMS contract, no image build, no tags.
+ """
+ repo = _mini_repo(tmp_path / "repo")
+ log = tmp_path / "docker.log"
+ res = _run_build_image(repo, tmp_path / "stubs", log, args=["--stage-only"])
+ assert res.returncode == 0, res.stderr
+ docker_calls = [l for l in log.read_text(encoding="utf-8").splitlines()
+ if l.startswith("docker ")]
+ assert docker_calls == [], "the --stage-only run must not invoke docker, expected no docker calls"
+ assert_staging(repo / ".build" / "delivery" / "staging", repo)
+
+
+def test_build_image_from_staging(tmp_path: Path):
+ """--from-staging builds the image from an already-staged directory.
+
+ The integration job rebuilds the delivery image from the raws artifact
+ instead of downloading a produced image. The shipped staging must verify
+ as-is (SHA256SUMS is checked, never regenerated) and the build carries only
+ `sci-delivery:latest`: the image is a test artifact, not the product.
+ """
+ repo = _mini_repo(tmp_path / "repo")
+ staging = repo / ".build" / "delivery" / "raws"
+ staged = _run_build_image(repo, tmp_path / "stubs", tmp_path / "stage.log",
+ args=["--stage-only"], env_extra={"STAGING_OUT": str(staging)})
+ assert staged.returncode == 0, staged.stderr
+ assert_staging(staging, repo)
+
+ log = tmp_path / "docker.log"
+ res = _run_build_image(repo, tmp_path / "stubs", log,
+ args=["--from-staging", str(staging)])
+ assert res.returncode == 0, res.stderr
+ calls = [l for l in log.read_text(encoding="utf-8").splitlines() if l.startswith("docker ")]
+ assert len(calls) == 1, f"--from-staging must run exactly one docker build, found: {calls}"
+ tokens = calls[0].split()
+ tags = [tokens[i + 1] for i, token in enumerate(tokens) if token == "-t"]
+ assert tags == ["sci-delivery:latest"], \
+ f"the from-staging build must tag exactly sci-delivery:latest, found: {tags}"
+ assert tokens[-1] == str(staging), \
+ f"the from-staging build must use the given staging directory as the context, expected {staging} (found: {tokens[-1]})"
+
+ tampered_raw = repo / ".build/mkosi/output" / f"{manifest_names()[0]}.raw"
+ tampered_raw.write_text("tampered\n", encoding="utf-8")
+ # A tampered staging must fail closed: the checksums are verified, never
+ # regenerated from whatever files are present.
+ shutil.copy(tampered_raw, staging / f"{manifest_names()[0]}.raw")
+ res = _run_build_image(repo, tmp_path / "stubs", tmp_path / "docker3.log",
+ args=["--from-staging", str(staging)])
+ assert res.returncode != 0 and "SHA256SUMS" in res.stderr, res.stderr
+ assert [l for l in tmp_path.joinpath("docker3.log").read_text(encoding="utf-8").splitlines()
+ if l.startswith("docker ")] == [], \
+ "a staging whose checksums do not verify must not reach docker build, expected no docker calls"
+
+ (staging / "SHA256SUMS").unlink()
+ res = _run_build_image(repo, tmp_path / "stubs", tmp_path / "docker4.log",
+ args=["--from-staging", str(staging)])
+ assert res.returncode != 0 and "SHA256SUMS" in res.stderr, res.stderr
+
+# delivery/Containerfile (the delivery image itself)
+
+CONTAINERFILE = REPO / "delivery" / "Containerfile"
+
+def _apt_install_lines(text: str) -> list[str]:
+ """Every 'apt-get install' line, extended over its backslash continuation lines."""
+ lines = text.splitlines()
+ out, i = [], 0
+ while i < len(lines):
+ if "apt-get install" in lines[i]:
+ out.append(lines[i])
+ while lines[i].rstrip().endswith("\\") and i + 1 < len(lines):
+ i += 1
+ out.append(lines[i])
+ i += 1
+ return out
+
+def test_containerfile_ships_the_fetch_tools():
+ text = CONTAINERFILE.read_text(encoding="utf-8")
+ blob = " ".join(_apt_install_lines(text))
+ assert blob, "the Containerfile has no 'apt-get install' line: curl, ca-certificates and openssl are not installed"
+ missing = [p for p in ("curl", "ca-certificates", "openssl") if not re.search(rf"(?
+ python3.14); symlinks and hardlinks escaping the runtime,
+ path-traversing members, and destinations resolving into the gardenlinux
+ submodule are refused.
+- parse_guest_result() accepts exactly one SCI-RESULT:::
+ line per console (id == invocation, scenario == scenario); payload is
+ base64 JSON {"rc": int, "junit": b64 JUnit XML}; rc == 0, JUnit has named
+ testcases, none skipped/failed/errored; record lines over MAX_RECORD_LINE
+ are rejected; returns rc, tests, failures, errors, skipped, and names
+ (the JUnit testcase names in order, for the caller's explicit-name check).
+- Rejections raise ValueError ("invalid evidence:" parse, "invalid input:"
+ staging); CLI `parse` prints the dict as JSON, exits 0, else exits nonzero
+ with the error text on stderr.
+"""
+
+import base64
+import importlib
+import io
+import json
+import os
+import subprocess
+import sys
+import tarfile
+from pathlib import Path
+
+import pytest
+
+MAX_RECORD_LINE = 65536
+CLOSED_PLUGINS = ("booted", "docstring", "file", "modify", "parse", "shell", "systemd")
+ADAPTER_FILE = Path(__file__).resolve().parents[0] / "runtime_adapter.py"
+INVOCATION, SCENARIO = "./run_tests -m sysext", "sysext-boot"
+TC_NAMES = [f"tc-{i}" for i in range(3)]
+NAMELESS_JUNIT = ''
+
+
+def _b64(data):
+ return base64.b64encode(data.encode() if isinstance(data, str) else data).decode()
+
+
+def _junit_xml(n, child=""):
+ cases = [f'{child if i == 0 else ""}' for i in range(n)]
+ f, e, s = (c in child for c in ("{"".join(cases)}'
+
+
+def _record(invocation, scenario, payload):
+ return f"SCI-RESULT:{invocation}:{scenario}:{payload}"
+
+
+def _console(*lines):
+ return "boot noise line\n" + "\n".join(lines) + "\ntrailing noise\n"
+
+
+def _launcher_bytes(repo_root):
+ lines = (repo_root / "tests" / "util" / "build_dist.sh").read_text(encoding="utf-8").splitlines(keepends=True)
+ marker = 'cat >"$tmpdir/dist/run_tests" <<\'EOF\''
+ start = next(i for i, l in enumerate(lines) if marker in l)
+ end = next(i for i in range(start + 1, len(lines)) if lines[i].rstrip("\n") == "EOF")
+ return "".join(lines[start + 1 : end]).encode()
+
+
+def _adapter():
+ return importlib.import_module("runtime_adapter") # lazy: no import at collection
+
+
+def _runtime_archive(path):
+ """Synthetic runtime.tar.gz with the real runtime's safe link shapes."""
+ root = path / "rt"
+ py = root / "x86_64" / "bin" / "python3.14"
+ py.parent.mkdir(parents=True)
+ py.write_bytes(b"#!/bin/sh\n# synthetic python\n")
+ py.chmod(0o755)
+ (root / "x86_64" / "bin" / "python").symlink_to("python3.14")
+ (root / "site-packages").mkdir()
+ (root / "site-packages" / "marker.txt").write_text("synthetic runtime\n")
+ (root / "x86_64" / "lib" / "python3.14").mkdir(parents=True)
+ (root / "x86_64" / "lib" / "python3.14" / "site-packages").symlink_to("../../../site-packages")
+ with tarfile.open(path / "runtime.tar.gz", "w:gz") as tf:
+ tf.add(root, arcname=".")
+ return path / "runtime.tar.gz"
+
+
+def _guest_tests(path):
+ g = path / "guest-tests"
+ g.mkdir()
+ (g / "test_boot.py").write_text("def test_boot():\n assert True\n")
+ (g / "test_policy.py").write_text("def test_policy():\n assert True\n")
+ return g
+
+
+def _evil_archive(path, member):
+ with tarfile.open(path / "evil.tar.gz", "w:gz") as tf:
+ info = tarfile.TarInfo(member)
+ info.size = 5
+ tf.addfile(info, io.BytesIO(b"evil\n"))
+ return path / "evil.tar.gz"
+
+
+def _link_archive(path, typ, target):
+ with tarfile.open(path / "esc.tar.gz", "w:gz") as tf:
+ info = tarfile.TarInfo("bin/link")
+ info.type, info.linkname = tarfile.SYMTYPE if typ == "sym" else tarfile.LNKTYPE, target
+ tf.addfile(info)
+ return path / "esc.tar.gz"
+
+
+BIG_RECORD = _record(INVOCATION, SCENARIO, _b64(_junit_xml(2500)))
+GOOD_RECORD = _record(INVOCATION, SCENARIO, _b64(json.dumps({"rc": 0, "junit": _b64(_junit_xml(3))})))
+
+REJECTIONS = [
+ ("missing-record", "boot noise without any result line\n", INVOCATION, SCENARIO),
+ ("duplicate-record", _console(GOOD_RECORD, GOOD_RECORD), INVOCATION, SCENARIO),
+ ("three-fields", f"SCI-RESULT:{INVOCATION}:{SCENARIO}\n", INVOCATION, SCENARIO),
+ ("bad-base64", f"SCI-RESULT:{INVOCATION}:{SCENARIO}:####\n", INVOCATION, SCENARIO),
+ ("bad-json", _record(INVOCATION, SCENARIO, _b64("not json")), INVOCATION, SCENARIO),
+ ("nonzero-rc", _record(INVOCATION, SCENARIO, _b64(json.dumps({"rc": 1, "junit": _b64(_junit_xml(3))}))), INVOCATION, SCENARIO),
+ ("empty-junit", _record(INVOCATION, SCENARIO, _b64(_junit_xml(0))), INVOCATION, SCENARIO),
+ *[(f"{t}-case", _record(INVOCATION, SCENARIO, _b64(json.dumps({"rc": 0, "junit": _b64(_junit_xml(3, f"<{t}/>"))}))), INVOCATION, SCENARIO) for t in ("skipped", "failure", "error")],
+ ("bad-junit-base64", _record(INVOCATION, SCENARIO, _b64(json.dumps({"rc": 0, "junit": "####"}))), INVOCATION, SCENARIO),
+ ("oversized-record", BIG_RECORD + "\n", INVOCATION, SCENARIO),
+ ("nameless-case", _record(INVOCATION, SCENARIO, _b64(json.dumps({"rc": 0, "junit": _b64(NAMELESS_JUNIT)}))), INVOCATION, SCENARIO),
+]
+
+ESCAPES = [("escape-symlink", "sym", "../../../outside.txt"), ("absolute-symlink", "sym", "/nonexistent-r4-abs"),
+ ("escape-hardlink", "lnk", "../../../outside.txt"), ("absolute-hardlink", "lnk", "/nonexistent-r4-abs")]
+
+
+@pytest.mark.parametrize("kind,console,inv,scen", REJECTIONS, ids=[k for k, *_ in REJECTIONS])
+def test_parse_guest_result_rejects(kind, console, inv, scen):
+ if kind == "oversized-record":
+ assert len(console) > MAX_RECORD_LINE
+ with pytest.raises(ValueError, match="invalid evidence"):
+ _adapter().parse_guest_result(console, inv, scen)
+
+
+@pytest.mark.parametrize("inv,scen", [(INVOCATION + "-other", SCENARIO), (INVOCATION, SCENARIO + "-other")], ids=["invocation", "scenario"])
+def test_parse_guest_result_mismatches(inv, scen):
+ with pytest.raises(ValueError, match="invalid evidence"):
+ _adapter().parse_guest_result(_console(GOOD_RECORD), inv, scen)
+
+
+def test_parse_guest_result_accepts():
+ result = _adapter().parse_guest_result(_console("noise", GOOD_RECORD, "noise"), INVOCATION, SCENARIO)
+ assert (result["rc"], result["tests"], result["failures"], result["errors"], result["skipped"], result["names"]) == (0, 3, 0, 0, 0, TC_NAMES)
+
+
+def test_stage_distribution_happy(repo_root, tmp_path):
+ guest = _guest_tests(tmp_path)
+ launcher = _adapter().stage_distribution(_runtime_archive(tmp_path), tmp_path / "dist", repo_root, guest)
+ assert launcher == tmp_path / "dist" / "run_tests" and launcher.is_file() and os.access(launcher, os.X_OK)
+ assert launcher.read_bytes() == _launcher_bytes(repo_root)
+ tests = launcher.parent / "tests"
+ assert (tests / "conftest.py").read_bytes() == (repo_root / "tests" / "conftest.py").read_bytes()
+ assert sorted(p.name for p in (tests / "plugins").glob("*.py")) == sorted(["__init__.py", *[f"{n}.py" for n in CLOSED_PLUGINS]])
+ assert all((tests / "plugins" / f"{n}.py").read_bytes() == (repo_root / "tests" / "plugins" / f"{n}.py").read_bytes() for n in CLOSED_PLUGINS)
+ assert not list((tests / "handlers").glob("*.py"))
+ assert all((tests / "integration" / "sci" / f.name).read_bytes() == f.read_bytes() for f in guest.iterdir())
+ py = launcher.parent / "runtime" / "x86_64" / "bin" / "python"
+ assert py.is_symlink() and os.readlink(py) == "python3.14" and py.read_bytes() == b"#!/bin/sh\n# synthetic python\n" and os.access(py, os.X_OK)
+ sp = launcher.parent / "runtime" / "x86_64" / "lib" / "python3.14" / "site-packages"
+ assert sp.is_symlink() and os.readlink(sp) == "../../../site-packages" and (sp / "marker.txt").read_text() == "synthetic runtime\n"
+
+
+@pytest.mark.parametrize("kind,typ,target", ESCAPES, ids=[k for k, *_ in ESCAPES])
+def test_stage_distribution_rejects_escaping_links(repo_root, tmp_path, kind, typ, target):
+ with pytest.raises(ValueError, match="invalid input"):
+ _adapter().stage_distribution(_link_archive(tmp_path, typ, target), tmp_path / "dist", repo_root, _guest_tests(tmp_path))
+ assert not (tmp_path / "outside.txt" if target.startswith("..") else Path(target)).exists() and not (tmp_path / "dist").exists()
+
+
+@pytest.mark.parametrize("kind", ["traversal", "dest-in-submodule", "dest-symlink-into-submodule"])
+def test_stage_distribution_rejects(repo_root, tmp_path, kind):
+ dest = {"traversal": tmp_path / "dist", "dest-in-submodule": repo_root / "gardenlinux" / "staged",
+ "dest-symlink-into-submodule": repo_root / "tests" / "staged"}[kind]
+ arc = _evil_archive(tmp_path, "../../evil.txt") if kind == "traversal" else _runtime_archive(tmp_path)
+ with pytest.raises(ValueError, match="invalid input"):
+ _adapter().stage_distribution(arc, dest, repo_root, _guest_tests(tmp_path))
+ assert kind not in {"traversal", "dest-in-submodule"} or not {"traversal": tmp_path / "evil.txt",
+ "dest-in-submodule": repo_root / "gardenlinux" / "staged"}[kind].exists()
+
+
+@pytest.mark.parametrize("console,ok", [(_console(GOOD_RECORD), True), ("no result in this console", False)], ids=["valid", "invalid"])
+def test_cli_parse(console, ok):
+ proc = subprocess.run([sys.executable, str(ADAPTER_FILE), "parse", console, INVOCATION, SCENARIO],
+ capture_output=True, text=True)
+ assert (proc.returncode == 0) == ok, proc.stderr
+ if ok:
+ out = json.loads(proc.stdout)
+ assert (out["rc"], out["tests"], out["failures"], out["errors"], out["skipped"], out["names"]) == (0, 3, 0, 0, 0, TC_NAMES)
+ else:
+ assert "invalid evidence" in proc.stderr
diff --git a/extensions/tests/test_summary.py b/extensions/tests/test_summary.py
new file mode 100644
index 00000000..95ca1e73
--- /dev/null
+++ b/extensions/tests/test_summary.py
@@ -0,0 +1,72 @@
+"""Golden and CLI contract for the retained summary normalizer."""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+import summary
+
+HERE = Path(__file__).resolve().parent
+SAMPLE = HERE / "data/summary-sample.json"
+GOLDEN = HERE / "data/summary-sample.normalized.json"
+REPO = "/repo"
+GOLDEN_SHA = "db139583aba7cb540a3f371d458a45cc9267f5449b352d69de38b8e0a50178de"
+
+
+def test_golden_normalize_and_digest():
+ assert summary.normalize_summary(SAMPLE, REPO) == GOLDEN.read_bytes()
+ assert summary.normalized_digest(SAMPLE, REPO) == GOLDEN_SHA
+
+
+def test_cli_parity():
+ out = subprocess.run([sys.executable, str(HERE / "summary.py"), str(SAMPLE), REPO],
+ capture_output=True, check=True)
+ assert out.stdout == GOLDEN.read_bytes()
+
+
+def test_changed_release_changes_normalization(tmp_path):
+ data = json.loads(SAMPLE.read_text(encoding="utf-8"))
+ data["Images"][0]["Release"] = "2150.9.1"
+ changed = tmp_path / "changed.json"
+ changed.write_text(json.dumps(data), encoding="utf-8")
+ assert summary.normalize_summary(changed, REPO) != GOLDEN.read_bytes()
+
+
+def test_delegate_ranges_uid_default_does_not_move_digest(tmp_path):
+ # mkosi resolves DelegateRanges from uid (root=0, non-root=3); the pin must not move.
+ root_run = json.loads(SAMPLE.read_text(encoding="utf-8"))
+ for image in root_run["Images"]:
+ image["DelegateRanges"] = 0
+ nonroot_run = json.loads(SAMPLE.read_text(encoding="utf-8"))
+ for image in nonroot_run["Images"]:
+ image["DelegateRanges"] = 3
+ root_file = tmp_path / "root-run.json"
+ root_file.write_text(json.dumps(root_run), encoding="utf-8")
+ nonroot_file = tmp_path / "nonroot-run.json"
+ nonroot_file.write_text(json.dumps(nonroot_run), encoding="utf-8")
+ assert summary.normalized_digest(root_file, REPO) == \
+ summary.normalized_digest(nonroot_file, REPO), (
+ "DelegateRanges is a uid-dependent mkosi default (root=0, non-root=3); "
+ "the pin must not depend on who ran mkosi"
+ )
+
+
+def test_proxy_exclude_env_default_does_not_move_digest(tmp_path):
+ # mkosi emits no_proxy/NO_PROXY as ProxyExclude; the pin must not depend on the shell.
+ no_proxy_run = json.loads(SAMPLE.read_text(encoding="utf-8"))
+ for image in no_proxy_run["Images"]:
+ image["ProxyExclude"] = ["127.0.0.1", "localhost", "::1"]
+ clean_run = json.loads(SAMPLE.read_text(encoding="utf-8"))
+ no_proxy_file = tmp_path / "no-proxy-run.json"
+ no_proxy_file.write_text(json.dumps(no_proxy_run), encoding="utf-8")
+ clean_file = tmp_path / "clean-run.json"
+ clean_file.write_text(json.dumps(clean_run), encoding="utf-8")
+ assert summary.normalized_digest(clean_file, REPO) == \
+ summary.normalized_digest(no_proxy_file, REPO), (
+ "ProxyExclude is an environment-derived mkosi default (read from "
+ "no_proxy/NO_PROXY); the pin must not depend on the invoking "
+ "shell's proxy variables, or contributors with no_proxy set (and "
+ "the development harness here) fail the pin test for reasons unrelated "
+ "to the tree"
+ )
diff --git a/extensions/tests/test_workflows.py b/extensions/tests/test_workflows.py
new file mode 100644
index 00000000..d0ade0ea
--- /dev/null
+++ b/extensions/tests/test_workflows.py
@@ -0,0 +1,636 @@
+"""CI wiring pins: the composite test actions and the sysext/test workflows."""
+import os
+import re
+import subprocess
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
+
+def load(path):
+ """Parsed workflow or composite-action YAML; the only YAML accessor."""
+ assert path.is_file(), f"{path.name} is missing: the file under test does not exist"
+ return yaml.safe_load(path.read_text(encoding="utf-8"))
+
+
+def triggers(data):
+ """The 'on:' section; PyYAML resolves the bare top-level key 'on' to True."""
+ return data["on"] if "on" in data else data[True]
+
+
+def job(data, name):
+ """The named job mapping of a parsed workflow."""
+ jobs = data.get("jobs") or {}
+ assert name in jobs, f"the workflow has no '{name}' job"
+ assert isinstance(jobs[name], dict), f"the '{name}' job is not a mapping"
+ return jobs[name]
+
+
+def steps_of(section):
+ """The 'steps:' list of a job or of a parsed composite action's 'runs' block."""
+ steps = section.get("steps") or []
+ assert isinstance(steps, list), "the 'steps:' section is not a list"
+ return steps
+
+
+def step_named(section, name, *, contains=False):
+ """The one step named exactly `name`, or the one step containing it when `contains` is true."""
+ def matches(s):
+ actual = str(s.get("name", ""))
+ return name in actual if contains else actual == name
+ found = [s for s in steps_of(section) if matches(s)]
+ assert len(found) == 1, f"exactly one step named '{name}' expected, found {len(found)}"
+ return found[0]
+
+
+def run_text(step):
+ """The step's 'run:' script as written, without YAML indentation."""
+ return step.get("run") or ""
+
+
+def run_lines(step):
+ """Stripped, non-empty, non-comment lines of the step's 'run:' script."""
+ return [l.strip() for l in run_text(step).splitlines()
+ if l.strip() and not l.lstrip().startswith("#")]
+
+
+def with_values(step):
+ """The step's 'with:' mapping, or an empty mapping."""
+ return step.get("with") or {}
+
+
+def step_index(steps, pred):
+ """Index of the first step whose run block has a line for which `pred` is true."""
+ return next((i for i, s in enumerate(steps) if any(pred(l) for l in run_lines(s))), -1)
+
+
+ACTION = REPO_ROOT / ".github" / "actions" / "test" / "integration" / "setup" / "action.yml"
+STEP = "Boot the sysext HyperVisors and run the delivery container"
+
+
+def test_setup_grants_qemu_traversal_through_the_runner_home():
+ lines = run_text(step_named(load(ACTION)["runs"], STEP)).splitlines()
+ starts = [i for i, l in enumerate(lines) if "virsh start" in l]
+ assert starts, "the sysext setup step no longer starts the HV domains"
+ chmods = [i for i, l in enumerate(lines) if "chmod o+x" in l and "$HOME" in l]
+ assert chmods and chmods[0] < starts[0], (
+ "the sysext setup step must 'sudo chmod o+x $HOME' before the first 'virsh start'")
+
+
+def guest_script():
+ """Text of the guest script the sysext step pipes into 'bash -s' via ssh."""
+ lines = run_text(step_named(load(ACTION)["runs"], STEP)).splitlines()
+ a = next((i for i, l in enumerate(lines) if l.endswith("bash -s <<'EOF'")), None)
+ assert a is not None, "the sysext step no longer pipes the guest script via \"bash -s <<'EOF'\""
+ b = next((i for i in range(a + 1, len(lines)) if lines[i] == "EOF"), None)
+ assert b is not None, "the closing 'EOF' of the sysext heredoc is missing"
+ return "\n".join(lines[a + 1:b])
+
+
+def test_sysext_setup_heredoc_runs_post_delivery_commands(tmp_path):
+ script = guest_script()
+ run_lines_ = [l for l in script.splitlines() if re.match(r"\s*ctr .* run\b", l)]
+ assert len(run_lines_) == 1, (
+ f"the sysext step must start exactly one 'ctr ... run' container, found {len(run_lines_)}")
+ sb = tmp_path / "sb"
+ (sb / "root" / ".ssh").mkdir(parents=True)
+ # Only the absolute paths the script writes to are relocated; stdin semantics untouched.
+ for p in ("/root/", "/opt/", "/var/log/libvirt/", "/var/lib/libvirt/"):
+ script = script.replace(p, str(sb) + p)
+ stubs = tmp_path / "stubs"
+ stubs.mkdir()
+ # 'ctr run' (foreground) reads stdin; 'ctr images import ' does not. 'systemd-sysext' is stubbed.
+ stub = {
+ "head": "#!/bin/sh\necho registry.local/sci/delivery:ci-test\n",
+ "systemctl": "#!/bin/sh\nexit 0\n",
+ "systemd-sysext": "#!/bin/sh\nexit 0\n",
+ "chmod": "#!/bin/sh\nexit 0\n",
+ "virsh": f"#!/bin/sh\nprintf '%s\\n' \"$*\" >> {sb / 'virsh.log'}\nexit 0\n",
+ "ctr": (f"#!/bin/sh\nprintf '%s\\n' \"$3\" >> {sb / 'ctr.log'}\n"
+ f"[ \"$3\" = run ] && cat > /dev/null\nexit 0\n"),
+ }
+ for name, text in stub.items():
+ (stubs / name).write_text(text, encoding="utf-8")
+ (stubs / name).chmod(0o755)
+ env = {"PATH": f"{stubs}:{os.environ.get('PATH', '/usr/bin:/bin')}"}
+ res = subprocess.run(["bash", "-s"], input=script, text=True, capture_output=True,
+ cwd=str(sb), env=env, timeout=60)
+ assert res.returncode == 0, f"the guest script exited {res.returncode}: {res.stderr.strip()[:120]}"
+ ctr = (sb / "ctr.log").read_text(encoding="utf-8").split()
+ assert ctr, "the 'ctr' stub was never invoked: the test cannot exercise the defect"
+ assert "run" in ctr, "the 'ctr ... run' delivery invocation never executed"
+ log = (sb / "virsh.log").read_text(encoding="utf-8") if (sb / "virsh.log").is_file() else ""
+ missing = [c for c in ("pool-define-as default", "pool-start default", "net-define",
+ "net-start default", "net-autostart default") if c not in log]
+ assert not missing, f"post-delivery commands were skipped; missing virsh calls: {missing}"
+ assert (sb / "root" / ".ssh" / "config").is_file(), (
+ "/root/.ssh/config was never written: the lines after 'ctr ... run' were consumed as container stdin")
+
+
+QEMU_ACTION = REPO_ROOT / ".github" / "actions" / "test" / "integration" / "test" / "qemu" / "action.yml"
+QEMU_STEP = "Copy VM defintions and artifacts to the HyperVisors"
+QEMU_DISK_DESTS = {
+ "ubuntu.qcow2": "/var/lib/libvirt/images/ubuntu.qcow2",
+ "ubuntu-cloud-init-ds.iso": "/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso",
+}
+
+
+def action_run_block(action, step):
+ """The step's 'run:' script from the composite action `action`."""
+ block = run_text(step_named(load(action)["runs"], step))
+ assert block.strip(), f"the '{step}' run block is empty in {action.name}"
+ return block
+
+
+def qemu_copy_block():
+ """Text of the 'Copy VM defintions...' run block of the QEMU test action."""
+ return action_run_block(QEMU_ACTION, QEMU_STEP)
+
+
+def scp_stages(run, image, hv, dest):
+ """scp lines of the run block that copy `image` to the `dest` path on `hv`."""
+ return [l for l in run.splitlines()
+ if l.strip().startswith("scp") and image in l and hv in l and dest in l]
+
+
+def test_qemu_migration_stages_disk_on_both_hvs():
+ section = load(QEMU_ACTION)["runs"]
+ steps = steps_of(section)
+ copy_i = steps.index(step_named(section, QEMU_STEP))
+ # Substring on purpose: the step name itself carries the upstream typo "Mirgrate".
+ migrate_i = steps.index(step_named(section, "Mirgrate", contains=True))
+ assert copy_i < migrate_i, (
+ "the 'Copy VM defintions' step must occur before the migration step: the disk must exist at the destination")
+ run = qemu_copy_block()
+ for image, dest in QEMU_DISK_DESTS.items():
+ missing = [hv for hv in ("$IP_HV1", "$IP_HV2") if not scp_stages(run, image, hv, dest)]
+ assert not missing, (
+ f"the QEMU copy step must stage {image} at {dest} on both HVs; missing on: {', '.join(missing)}")
+
+
+CHV_ACTION = REPO_ROOT / ".github" / "actions" / "test" / "integration" / "test" / "cloudhypervisor" / "action.yml"
+CHV_VM_XML = REPO_ROOT / ".github" / "actions" / "test" / "integration" / "test" / "cloudhypervisor" / "vm.xml"
+MIGRATION_ACTIONS = [(QEMU_ACTION, "QEMU"), (CHV_ACTION, "cloudhypervisor")]
+WAIT_STEP = "Wait for VM to be ready"
+VERIFY_STEP = "Verify VM status on HyperVisor 2"
+
+
+def loop_window(run, step, action):
+ """MAX_ITER and sleep seconds of the SSH retry loop in the step run block."""
+ # Last match: a decoy earlier in the block must not shadow the loop's values.
+ m_iter = re.findall(r"MAX_ITER=(\d+)", run)
+ m_sleep = re.findall(r"sleep (\d+)", run)
+ assert m_iter and m_sleep, (
+ f"cannot find 'MAX_ITER=' and 'sleep ' in the '{step}' run block: no plain retry loop")
+ return int(m_iter[-1]), int(m_sleep[-1])
+
+
+def test_chv_vm_defines_at_least_two_vcpus():
+ doc = ET.parse(CHV_VM_XML).getroot()
+ vcpu = doc.find("vcpu")
+ assert vcpu is not None, f"{CHV_VM_XML.name}: no element: cannot check the vCPU count"
+ count = int(vcpu.text.strip())
+ assert count >= 2, (
+ f"{CHV_VM_XML.name}: is {count}: the guest needs at least 2 vCPUs to boot to SSH")
+
+
+def test_wait_for_vm_step_allows_at_least_60_minutes():
+ for action, label in MIGRATION_ACTIONS:
+ run = action_run_block(action, WAIT_STEP)
+ iters, sleep_s = loop_window(run, WAIT_STEP, action)
+ assert iters * sleep_s >= 3600, (
+ f"{label}: '{WAIT_STEP}' waits {iters * sleep_s} s (MAX_ITER={iters} x sleep {sleep_s}), "
+ "below the 3600 s window")
+
+
+def test_verify_vm_step_allows_at_least_15_minutes():
+ for action, label in MIGRATION_ACTIONS:
+ run = action_run_block(action, VERIFY_STEP)
+ iters, sleep_s = loop_window(run, VERIFY_STEP, action)
+ assert iters * sleep_s >= 900, (
+ f"{label}: '{VERIFY_STEP}' waits {iters * sleep_s} s (MAX_ITER={iters} x sleep {sleep_s}), "
+ "below the 900 s window")
+
+
+SYSEXT_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "sysext.yml"
+ACTRC = REPO_ROOT / ".actrc"
+CHECKOUT_SHA = "3d3c42e5aac5ba805825da76410c181273ba90b1"
+QUICK_MARKERS = "-m 'not vm and not build'"
+QUICK_COMMAND = ".build/venv-test/bin/python -m pytest extensions/tests/ -q " + QUICK_MARKERS
+
+
+def assert_quick_suite_contract(data):
+ """Quick-tier contract of the sysext workflow, on the parsed YAML."""
+ on = triggers(data)
+ assert isinstance(on, dict), (
+ "the sysext workflow has no 'on:' trigger section: push and pull_request must self-trigger")
+ push = on.get("push")
+ assert isinstance(push, dict), (
+ "the push trigger is missing: the branch must self-trigger before it lands on the default branch")
+ branches = push.get("branches")
+ assert set(branches or []) == {"sysext-virt", "sysext-*"}, (
+ f"the push trigger must fire only on branches sysext-virt and sysext-* (found: {branches})")
+ assert "pull_request" in on, (
+ "the pull_request trigger is missing: PRs must also run the quick tier")
+
+ steps = steps_of(job(data, "quick-suite"))
+ code = [l for s in steps for l in run_lines(s)]
+ assert any("python3 -m venv .build/venv-test" in l for l in code), (
+ "the quick-suite job must create its test venv: 'python3 -m venv .build/venv-test'")
+ assert any(l == QUICK_COMMAND for l in code), (
+ f"the quick-suite job must run exactly {QUICK_COMMAND!r} with the quick markers")
+ assert any(".build/venv-test/bin/pip" in l and "pytest==8.4.2" in l for l in code), (
+ "the quick-suite job must pin pytest==8.4.2 in its pip line")
+
+
+def test_sysext_workflow_runs_quick_suite_on_github():
+ assert_quick_suite_contract(load(SYSEXT_WORKFLOW))
+
+
+def test_actrc_stays_machine_local():
+ res = subprocess.run(["git", "ls-files", ".actrc"], capture_output=True, text=True,
+ cwd=str(REPO_ROOT))
+ assert res.returncode == 0, (
+ f"'git ls-files .actrc' failed with exit {res.returncode}: {res.stderr.strip()[:100]}")
+ assert not res.stdout.strip(), (
+ f"{ACTRC} is tracked: .actrc is workstation-local config and must not be committed")
+ res = subprocess.run(
+ ["git", "-c", "core.excludesFile=/dev/null", "check-ignore", ".actrc"],
+ capture_output=True, text=True, cwd=str(REPO_ROOT))
+ assert res.returncode == 0, (
+ "the repo .gitignore must list .actrc so the local file cannot be committed accidentally")
+
+
+UPLOAD_SHA = "actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9"
+BUILDER_IMAGE = "ghcr.io/gardenlinux/builder:f7cf91087f7580c6c3aefb26af6cee7df89a9938"
+FULL_SUITE_INVOCATION = ".build/venv-test/bin/python -m pytest extensions/tests/ -q"
+MKOSI_SPEC = "mkosi @ git+https://github.com/systemd/mkosi.git@v27.1"
+
+
+def test_producer_job_shape():
+ producer = job(load(SYSEXT_WORKFLOW), "producer")
+ assert producer.get("runs-on") == "ubuntu-24.04", (
+ f"the producer job must keep 'runs-on: ubuntu-24.04' (found: {producer.get('runs-on')!r})")
+ needs = producer.get("needs")
+ assert needs in ("quick-suite", ["quick-suite"]), (
+ f"the producer job must declare exactly 'needs: quick-suite' (found: {needs!r})")
+ assert producer.get("if") == "github.event_name != 'pull_request'", (
+ f"the producer job must set \"if: github.event_name != 'pull_request'\" (found: {producer.get('if')!r})")
+ assert steps_of(producer), "the producer job has no 'steps:' section: nothing would build"
+
+
+def test_producer_pins_its_checkout_and_toolchain():
+ steps = steps_of(job(load(SYSEXT_WORKFLOW), "producer"))
+ checkout = [s for s in steps if s.get("uses") == f"actions/checkout@{CHECKOUT_SHA}"]
+ assert len(checkout) == 1, (
+ f"exactly one producer step must pin 'uses: actions/checkout@{CHECKOUT_SHA}'; found {len(checkout)}")
+ submodules = with_values(checkout[0]).get("submodules")
+ assert submodules in ("recursive", True), (
+ f"the producer's checkout step must enable submodules (found: {submodules!r})")
+ assert any(MKOSI_SPEC in l for s in steps for l in run_lines(s)), (
+ f"no producer step installs the exact spec '{MKOSI_SPEC}'")
+
+
+def test_producer_steps_run_in_a_safe_order():
+ steps = steps_of(job(load(SYSEXT_WORKFLOW), "producer"))
+ ensure_i = step_index(steps, lambda l: "bash mkosi/base-image.sh ensure" in l)
+ build_i = step_index(steps, lambda l: "bash mkosi/build.sh" in l)
+ delivery_i = step_index(steps, lambda l: "bash delivery/build-image.sh" in l)
+ userns_i = step_index(steps, lambda l: "apparmor_restrict_unprivileged_userns" in l)
+ assert ensure_i >= 0, (
+ "no producer step runs 'bash mkosi/base-image.sh ensure': build.sh demands the base tree")
+ assert build_i >= 0, "no producer step runs 'bash mkosi/build.sh'"
+ assert userns_i >= 0, "no producer step sets kernel.apparmor_restrict_unprivileged_userns=0"
+ assert userns_i < build_i, "the user-namespace relaxation must run before the mkosi build"
+ assert any("/dev/kvm" in l and "chmod" in l
+ for s in steps for l in run_lines(s)), (
+ "no producer step makes /dev/kvm readable and writable by the runner user")
+ assert delivery_i >= 0, (
+ "no producer step runs 'bash delivery/build-image.sh': nothing would stage the raws artifact")
+ assert ensure_i < build_i, "the base-image ensure step must occur before the mkosi build"
+ assert build_i < delivery_i, "the mkosi build step must occur before the raws staging"
+ assert not any("MANIFEST=" in l for l in run_lines(steps[delivery_i])), (
+ "the delivery build step must not override MANIFEST: the repo manifest lists every deployable")
+ assert any(f"docker pull {BUILDER_IMAGE}" in l
+ for s in steps for l in run_lines(s)), (
+ f"no producer step runs 'docker pull {BUILDER_IMAGE}': the build tiers pin that image")
+
+
+def test_producer_signs_with_a_run_local_ca():
+ steps = steps_of(job(load(SYSEXT_WORKFLOW), "producer"))
+ ca_i = step_index(steps, lambda l: "openssl req -x509" in l)
+ assert ca_i >= 0, (
+ "no producer step runs 'openssl req -x509': the producer must generate its own signing CA per run")
+ ca_run = run_lines(steps[ca_i])
+ assert (any(".build/keys/sysext-verity.key" in l for l in ca_run)
+ and any("mkosi/verity.crt" in l for l in ca_run)
+ and any("chmod 600 .build/keys/sysext-verity.key" in l for l in ca_run)
+ and any("install -d -m 0700 .build/keys" in l for l in ca_run)), (
+ "the run-local CA step must write '.build/keys/sysext-verity.key' and 'mkosi/verity.crt', "
+ "with modes 0700/600")
+ build_i = step_index(steps, lambda l: "bash mkosi/build.sh" in l)
+ assert build_i >= 0, "no producer step runs 'bash mkosi/build.sh'"
+ assert ca_i < build_i, (
+ "the run-local CA step must occur before the mkosi build: build.sh signs the raws with the run key")
+
+
+def test_producer_runs_the_full_suite_before_uploading():
+ steps = steps_of(job(load(SYSEXT_WORKFLOW), "producer"))
+ suite_i = step_index(steps, lambda l: l == FULL_SUITE_INVOCATION)
+ assert suite_i >= 0, (
+ f"no producer step runs exactly '{FULL_SUITE_INVOCATION}': the full suite must run in this job")
+ for l in run_lines(steps[suite_i]):
+ rest = l.replace(FULL_SUITE_INVOCATION, " ")
+ assert not re.search(r"(\A|\s)-m(\s|$)", rest), (
+ f"the full-suite step must not select markers with '-m' (found {l!r})")
+ uploads = [i for i, s in enumerate(steps) if s.get("uses") == UPLOAD_SHA]
+ assert len(uploads) == 1, (
+ f"the producer job must have exactly one upload step pinned at 'uses: {UPLOAD_SHA}'; found {len(uploads)}")
+ assert suite_i < uploads[0], (
+ "the full-suite step must run before the first upload: nothing is uploaded before it proves the build")
+ withs = [with_values(steps[i]) for i in uploads]
+ names = {x.get("name") for x in withs}
+ assert names == {"sysext-raws"}, (
+ f"the upload step must be named exactly 'sysext-raws'; found: {sorted(str(n) for n in names)}")
+ for x in withs:
+ assert x.get("if-no-files-found") == "error", (
+ f"every upload step must set 'if-no-files-found: error' (found: {x.get('if-no-files-found')!r})")
+
+
+def test_integration_invokes_the_consumer():
+ data = load(SYSEXT_WORKFLOW)
+ uploads = [with_values(s) for s in steps_of(job(data, "producer"))
+ if s.get("uses") == UPLOAD_SHA]
+ assert len(uploads) == 1, (
+ f"the producer job must have exactly one upload step pinned at 'uses: {UPLOAD_SHA}'; found {len(uploads)}")
+ delivery_name = uploads[0].get("name")
+ integ = job(data, "integration")
+ needs = integ.get("needs")
+ assert needs in ("producer", ["producer"]), (
+ f"the integration job must declare 'needs: producer' (found: {needs!r})")
+ assert integ.get("uses") == "./.github/workflows/test.yml", (
+ f"the integration job must call 'uses: ./.github/workflows/test.yml' (found: {integ.get('uses')!r})")
+ with_ = with_values(integ)
+ assert with_.get("sysext_artifact") == delivery_name, (
+ f"the integration job must pass 'sysext_artifact: {delivery_name}'; found: {with_.get('sysext_artifact')!r}")
+ assert "tools_artifact" not in with_, (
+ "the integration job must not pass 'tools_artifact': the toolchain is bootstrapped on the consumer")
+ assert "!= 'pull_request'" in str(integ.get("if", "")), (
+ "the integration job must set an 'if:' that excludes pull_request: the producer does not build on PRs")
+ perms = integ.get("permissions")
+ assert perms == {"contents": "read", "actions": "read"}, (
+ f"the integration job must grant exactly 'contents: read' and 'actions: read' (found: {perms!r})")
+
+
+RAWS_STAGING = ".build/delivery/staging-raws"
+RAWS_ARTIFACT = "sysext-raws"
+
+
+def test_sysext_workflow_publishes_raws_artifact():
+ steps = steps_of(job(load(SYSEXT_WORKFLOW), "producer"))
+ stage_i = step_index(steps, lambda l: "bash delivery/build-image.sh --stage-only" in l)
+ assert stage_i >= 0, (
+ "no producer step runs 'bash delivery/build-image.sh --stage-only': the raws artifact needs the staging")
+ run = run_lines(steps[stage_i])
+ assert any(f"STAGING_OUT={RAWS_STAGING}" in l for l in run), (
+ f"the raws staging step must set 'STAGING_OUT={RAWS_STAGING}': the upload publishes that directory")
+ assert any(l.startswith("cp ")
+ and "mkosi/verity.crt" in l and "verity-run.crt" in l
+ for l in run), (
+ "the raws staging step must copy the run cert into the staging directory as 'verity-run.crt'")
+ suite_i = step_index(steps, lambda l: l == FULL_SUITE_INVOCATION)
+ assert suite_i >= 0 and suite_i < stage_i, (
+ "the raws staging step must run after the full suite: nothing ships before the suite proves it")
+ raws_uploads = [s for s in steps if s.get("uses") == UPLOAD_SHA
+ and with_values(s).get("name") == RAWS_ARTIFACT]
+ assert len(raws_uploads) == 1, (
+ f"exactly one upload step must publish the '{RAWS_ARTIFACT}' artifact (found {len(raws_uploads)})")
+ with_ = with_values(raws_uploads[0])
+ assert str(with_.get("path", "")).rstrip("/") == RAWS_STAGING, (
+ f"the {RAWS_ARTIFACT} upload must point at '{RAWS_STAGING}/' (found: {with_.get('path')!r})")
+ assert with_.get("if-no-files-found") == "error", (
+ "the raws upload must set 'if-no-files-found: error': a missing file must fail the job")
+ assert str(with_.get("retention-days")) == "7", (
+ "the raws upload must keep the 7-day retention of the sibling artifacts")
+
+
+SIGNING_SECRET = "secrets.SYSEXT_UNHARDENED_SIGNING_KEY"
+
+
+def test_producer_uses_the_stable_signing_key_when_present():
+ steps = steps_of(job(load(SYSEXT_WORKFLOW), "producer"))
+
+ def step_text(step):
+ env = step.get("env") or {}
+ return run_text(step) + "\n" + "\n".join(str(v) for v in env.values())
+
+ assert any(SIGNING_SECRET in step_text(s) for s in steps), (
+ f"no producer step's env or run references {SIGNING_SECRET}: the stable signing key is never used")
+ ca = [s for s in steps if "openssl req -x509" in run_text(s)]
+ assert ca and any("-newkey" in run_text(s) for s in ca), (
+ "the run-local fallback ('openssl req -x509' with '-newkey') is gone: the run would break when the secret is absent")
+
+
+def test_publish_job_publishes_a_signed_release():
+ data = load(SYSEXT_WORKFLOW)
+ publish = job(data, "publish")
+ needs = publish.get("needs")
+ assert needs in ("producer", ["producer"]), (
+ f"the publish job must declare exactly 'needs: producer' (found: {needs!r})")
+ assert "pull_request" in str(publish.get("if", "")), (
+ f"the publish job must not run on pull requests: its 'if:' must mention pull_request (found: {publish.get('if')!r})")
+ perms = publish.get("permissions")
+ assert perms == {"contents": "write"}, (
+ f"the publish job must grant exactly 'contents: write' (found: {perms!r})")
+ steps = steps_of(publish)
+ downloads = [s for s in steps if with_values(s).get("name") == "sysext-raws"]
+ assert len(downloads) == 1, (
+ f"exactly one publish step must download the 'sysext-raws' artifact with with.name (found {len(downloads)})")
+ sign_i = step_index(steps, lambda l: "openssl dgst -sha256 -sign" in l)
+ assert sign_i >= 0, (
+ "no publish step signs with 'openssl dgst -sha256 -sign': the release would ship an unverified SHA256SUMS")
+ assert any("SHA256SUMS.sig" in l for l in run_lines(steps[sign_i])), (
+ "the signing step must produce 'SHA256SUMS.sig'")
+ release_i = step_index(steps, lambda l: "gh release" in l)
+ assert release_i >= 0, "no publish step runs 'gh release': nothing creates or updates the GitHub release"
+ release_run = "\n".join(run_lines(steps[release_i]))
+ assert "GITHUB_SHA" in release_run, (
+ "the release step must build the tag from the release version and the short commit and reference GITHUB_SHA")
+ uploads = [s for s in steps if any("gh release upload" in l for l in run_lines(s))]
+ assert len(uploads) == 1, (
+ f"exactly one publish step must run 'gh release upload' (found {len(uploads)})")
+ up = "\n".join(run_lines(uploads[0]))
+ assert "SHA256SUMS.sig" in up, (
+ "the upload command must include 'SHA256SUMS.sig': the consumer verifies the download against it")
+ assert "*" in up or ".raw" in up, (
+ "the upload command must include the staged files: a wildcard or the staged file names")
+ vehicle = ".build/ci-sysext-hv/publish/sci-delivery.sh"
+ step = step_named(publish, "Regenerate the published inventory")
+ run = run_lines(step)
+ workdir = str(step.get("working-directory") or ".")
+
+ def repo_target(arg):
+ """Repo-relative target of an rm path argument."""
+ if "$GITHUB_WORKSPACE" in arg or "${GITHUB_WORKSPACE}" in arg:
+ tail = arg.replace("${GITHUB_WORKSPACE}", "").replace("$GITHUB_WORKSPACE", "")
+ return os.path.normpath(tail.lstrip("/"))
+ return os.path.normpath(os.path.join(workdir, arg))
+
+ removes = [i for i, l in enumerate(run) if re.match(r"rm\s+", l) and "sci-delivery.sh" in l]
+ assert removes, (
+ f"the inventory step must 'rm' the delivery step: the release carries data, not the vehicle")
+ paths = [a for a in run[removes[0]].split()[1:] if not a.startswith("-")]
+ assert os.path.normpath(vehicle) in {repo_target(a) for a in paths}, (
+ f"the rm path must resolve to {vehicle} from working-directory {workdir}; found {paths}")
+ assert any(re.search(r"test\s+!\s+-e\s+sci-delivery\.sh", l) for l in run), (
+ "the inventory step must guard 'test ! -e sci-delivery.sh': a surviving vehicle must fail the job")
+ find_i = next((i for i, l in enumerate(run) if l.startswith("find ") and "-printf" in l), -1)
+ sums_i = next((i for i, l in enumerate(run)
+ if "xargs sha256sum" in l and "SHA256SUMS" in l), -1)
+ assert find_i >= 0 and sums_i > find_i, (
+ "the inventory step no longer regenerates 'SHA256SUMS' with a 'find ... | xargs sha256sum' pipeline")
+ assert removes[0] < find_i, (
+ "the delivery step must be removed before the inventory regenerates: SHA256SUMS must not list it")
+
+
+def test_publish_targets_the_built_commit():
+ steps = steps_of(job(load(SYSEXT_WORKFLOW), "publish"))
+ release_i = step_index(steps, lambda l: "gh release" in l)
+ assert release_i >= 0, "no publish step runs 'gh release': nothing creates the GitHub release"
+ creates = [l for l in run_lines(steps[release_i]) if "gh release create" in l]
+ assert creates, "no publish step runs 'gh release create': the release tag is never created"
+ create = "\n".join(creates)
+ assert "--target" in create, (
+ "the 'gh release create' call must pass '--target': without it the git tag lands on the "
+ f"default branch instead of the built commit: {creates}")
+ assert re.search(r"--target\s+\"?\$\{?GITHUB_SHA", create), (
+ "the '--target' of 'gh release create' must reference GITHUB_SHA (the built commit): "
+ f"{creates}")
+
+
+def test_publish_is_skipped_without_the_signing_secret():
+ publish = job(load(SYSEXT_WORKFLOW), "publish")
+ steps = steps_of(publish)
+ job_text = run_text(publish) + "\n" + str(publish.get("if", ""))
+ step_text = "\n".join(
+ run_text(s) + "\n" + str(s.get("if", "")) + "\n" + "\n".join(str(v) for v in (s.get("env") or {}).values())
+ for s in steps)
+ assert "SYSEXT_UNHARDENED_SIGNING_KEY" in job_text or "SYSEXT_UNHARDENED_SIGNING_KEY" in step_text, (
+ "the publish job's run and if text never mention SYSEXT_UNHARDENED_SIGNING_KEY: a missing secret cannot be detected")
+ if_text = "\n".join([str(publish.get("if", ""))] + [str(s.get("if", "")) for s in steps])
+ assert re.search(r"==\s*['\"]?true", if_text), (
+ "no publish job or step 'if' compares an output to the string 'true': a missing secret would fail the job instead of skipping it")
+
+
+TEST_WORKFLOW = REPO_ROOT / ".github" / "workflows" / "test.yml"
+RUN_CERT = ".build/ci-sysext-hv/raws/verity-run.crt"
+CHECKOUT_CERT = "mkosi/verity.crt"
+
+
+def test_sysext_workflow_invokes_the_consumer():
+ data = load(SYSEXT_WORKFLOW)
+ job_ids = list(data.get("jobs") or {})
+ assert job_ids == ["quick-suite", "producer", "integration", "publish"], (
+ f"the job id list must be exactly ['quick-suite', 'producer', 'integration', 'publish'] in that order; found: {job_ids}")
+ integ = job(data, "integration")
+ assert integ.get("uses") == "./.github/workflows/test.yml", (
+ "the integration job must set 'uses: ./.github/workflows/test.yml'")
+ needs = integ.get("needs")
+ assert needs in ("producer", ["producer"]), (
+ f"the integration job must declare 'needs: producer' (found: {needs!r})")
+ assert "!= 'pull_request'" in str(integ.get("if", "")), (
+ "the integration job must set an 'if:' that excludes pull_request")
+ vals = with_values(integ)
+ assert vals.get("sysext_artifact") == "sysext-raws", (
+ f"the integration job must pass 'sysext_artifact: sysext-raws' (found: {vals.get('sysext_artifact')!r})")
+ assert "tools_artifact" not in vals, (
+ "the integration job must not pass 'tools_artifact': the toolchain is bootstrapped on the consumer")
+ perms = integ.get("permissions") or {}
+ grants = sorted(f"{k}: {v}" for k, v in perms.items())
+ assert perms == {"contents": "read", "actions": "read"}, (
+ f"the integration job must grant exactly 'actions: read' and 'contents: read' (found: {grants})")
+
+
+def test_consumer_restores_the_run_certificate():
+ section = job(load(TEST_WORKFLOW), "test")
+ steps = steps_of(section)
+ restorers = [i for i, s in enumerate(steps)
+ if RUN_CERT in run_text(s) and CHECKOUT_CERT in run_text(s)]
+ assert len(restorers) == 1, (
+ f"exactly one step must copy '{RUN_CERT}' over '{CHECKOUT_CERT}'; found {len(restorers)}")
+ at = restorers[0]
+ step = steps[at]
+ assert "inputs.sysext_artifact" in str(step.get("if", "")), (
+ "the cert-restore step must be gated on inputs.sysext_artifact")
+ run = run_lines(step)
+ copies = [l for l in run
+ if re.match(r"cp(\s|$)", l) and RUN_CERT in l and CHECKOUT_CERT in l
+ and l.index(RUN_CERT) < l.index(CHECKOUT_CERT)]
+ assert copies, (
+ f"the restore step must copy '{RUN_CERT}' onto '{CHECKOUT_CERT}' in that direction")
+ else_at = next((i for i, l in enumerate(run) if l == "else"), -1)
+ echo = [l for l in run[else_at + 1:] if l.startswith("echo")]
+ assert else_at >= 0 and echo, "the restore step must keep a tolerant 'else' branch with an echo"
+ download_i = steps.index(step_named(section, "Download the sysext raws artifact"))
+ preflight_i = steps.index(step_named(section, "Preflight the accepted toolchain"))
+ assert download_i < at, (
+ "the cert-restore step must appear after 'Download the sysext raws artifact'")
+ assert at < preflight_i, (
+ "the cert-restore step must appear before 'Preflight the accepted toolchain'")
+
+
+def test_consumer_relaxes_the_userns_restriction():
+ section = job(load(TEST_WORKFLOW), "test")
+ steps = steps_of(section)
+ step = step_named(section, "Relax the runner's user-namespace restriction")
+ at = steps.index(step)
+ assert any("apparmor_restrict_unprivileged_userns=0" in l for l in run_lines(step)), (
+ "the consumer's relaxation must set kernel.apparmor_restrict_unprivileged_userns=0 for mkosi")
+ assert "inputs.sysext_artifact" in str(step.get("if", "")), (
+ "the relaxation step must be gated on inputs.sysext_artifact")
+ prepare_i = steps.index(step_named(section, "Prepare the sysext HyperVisor base"))
+ bootstrap_i = steps.index(step_named(section, "Bootstrap the toolchain"))
+ assert at < bootstrap_i, "the relaxation must run before 'Bootstrap the toolchain'"
+ assert at < prepare_i, "the relaxation must run before 'Prepare the sysext HyperVisor base'"
+
+
+def test_consumer_bootstraps_the_toolchain():
+ section = job(load(TEST_WORKFLOW), "test")
+ steps = steps_of(section)
+ step = step_named(section, "Bootstrap the toolchain")
+ at = steps.index(step)
+ run = run_lines(step)
+ assert any(l == "bash mkosi/build.sh --bootstrap" for l in run), (
+ "the bootstrap step must run exactly 'bash mkosi/build.sh --bootstrap'")
+ assert any("python3 -m venv .build/mkosi/venv" in l for l in run), (
+ "the bootstrap step must create the mkosi venv: 'python3 -m venv .build/mkosi/venv'")
+ assert any("mkosi.git@v27.1" in l for l in run), (
+ "the bootstrap step must install the pinned mkosi (v27.1)")
+ assert "inputs.sysext_artifact" in str(step.get("if", "")), (
+ "the bootstrap step must be gated on inputs.sysext_artifact")
+ preflight_i = steps.index(step_named(section, "Preflight the accepted toolchain"))
+ assert at < preflight_i, "the bootstrap step must run before 'Preflight the accepted toolchain'"
+
+
+def test_consumer_rebuilds_the_delivery_image_from_the_raws():
+ section = job(load(TEST_WORKFLOW), "test")
+ steps = steps_of(section)
+ step = step_named(section, "Rebuild the delivery image from the raws")
+ at = steps.index(step)
+ run = run_lines(step)
+ assert any("delivery/build-image.sh --from-staging" in l
+ and ".build/ci-sysext-hv/raws" in l for l in run), (
+ "the rebuild step must run 'delivery/build-image.sh --from-staging .build/ci-sysext-hv/raws'")
+ assert any(l.startswith("docker save") and "sci-delivery.tar" in l for l in run), (
+ "the rebuild step must 'docker save' the image to '.build/ci-sysext-hv/artifact/sci-delivery.tar'")
+ refs = [l for l in run if "sci-delivery.ref" in l]
+ assert refs and any("docker.io/library/sci-delivery:latest" in l for l in refs), (
+ "the rebuild step must write the fully-qualified 'docker.io/library/sci-delivery:latest' "
+ "to sci-delivery.ref")
+ assert "inputs.sysext_artifact" in str(step.get("if", "")), (
+ "the rebuild step must be gated on inputs.sysext_artifact")
+ prepare_i = steps.index(step_named(section, "Prepare the sysext HyperVisor base"))
+ assert at < prepare_i, "the rebuild step must run before 'Prepare the sysext HyperVisor base'"
diff --git a/mkosi/base-image.pin b/mkosi/base-image.pin
new file mode 100644
index 00000000..1fbd56dd
--- /dev/null
+++ b/mkosi/base-image.pin
@@ -0,0 +1,11 @@
+# Published Garden Linux base image the extensions diff against; prepared by mkosi/base-image.sh.
+# Fields: FLAVOR, ARCH, VERSION (= Release=), COMMIT, TARBALL_SHA256, ROOTFS_TAR_SHA256, URL (derived).
+# Check: extensions/tests/test_host_contracts.py.
+# Update VERSION/COMMIT/hashes together with Release=, mkosi/suite.pin, and mkosi/summary.pin.
+FLAVOR=openstack-gardener_prod
+ARCH=amd64
+VERSION=2150.9.0
+COMMIT=a6e35cf0c955dde584c67163a3a8dff6d60e9d56
+TARBALL_SHA256=3b5e705fa9d683048bd8657893f31d8a2f4a639ddda0249ca31b2e1c7b679bed
+ROOTFS_TAR_SHA256=3baa1c461d87bb3a25ee86a73aca6b0f060a2b2a918c84c9fe4ad4bd7bbd4cb8
+URL=https://github.com/gardenlinux/gardenlinux/releases/download/2150.9.0/openstack-gardener_prod-amd64-2150.9.0-a6e35cf0.tar.xz
diff --git a/mkosi/base-image.sh b/mkosi/base-image.sh
new file mode 100755
index 00000000..fa4a3a01
--- /dev/null
+++ b/mkosi/base-image.sh
@@ -0,0 +1,333 @@
+#!/usr/bin/env bash
+# Prepare and verify the published Garden Linux base tree used as the diff baseline.
+# Usage: bash mkosi/base-image.sh [ensure|extract |digest ]
+# Contract: docs/sysext-virt.md; behaviour pinned by extensions/tests/test_host_contracts.py.
+
+set -Eeuo pipefail
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
+REPO_ROOT=$(cd -- "$SCRIPT_DIR/.." && pwd -P)
+
+PIN=${BASE_IMAGE_PIN:-$SCRIPT_DIR/base-image.pin}
+CACHE_ROOT=${BASE_IMAGE_DIR:-$REPO_ROOT/.build/mkosi/base-image}
+MKOSI_CONF=$SCRIPT_DIR/mkosi.conf
+
+OFFLINE=0
+case ${BASE_IMAGE_OFFLINE:-0} in
+ ''|0|no|false) ;;
+ *) OFFLINE=1 ;;
+esac
+
+TMP_DIRS=()
+cleanup() {
+ local d
+ for d in "${TMP_DIRS[@]}"; do
+ rm -rf -- "$d"
+ done
+}
+trap cleanup EXIT
+
+die() {
+ printf 'base-image: error: %s\n' "$*" >&2
+ exit 1
+}
+
+log() {
+ printf 'base-image: %s\n' "$*" >&2
+}
+
+usage() {
+ cat <<'EOF'
+usage: base-image.sh [ensure]
+ base-image.sh extract
+ base-image.sh digest
+
+Environment overrides: BASE_IMAGE_PIN, BASE_IMAGE_DIR, BASE_IMAGE_TARBALL,
+BASE_IMAGE_URL, BASE_IMAGE_OFFLINE=1
+EOF
+}
+
+# Last KEY=VALUE field of a pin file (empty when absent).
+pin_field() { #
+ sed -n "s/^$2[[:space:]]*=[[:space:]]*//p" -- "$1" | tail -n 1
+}
+
+validate_pin() {
+ [ -f "$PIN" ] || die "missing pin $PIN (the committed pin of the published base image)"
+ local field count line
+ for field in FLAVOR ARCH VERSION COMMIT TARBALL_SHA256 ROOTFS_TAR_SHA256 URL; do
+ count=$(grep -cE "^${field}[[:space:]]*=" -- "$PIN") || count=0
+ [ "$count" -eq 1 ] \
+ || die "$PIN: expected exactly one $field= line, found $count"
+ done
+ while IFS= read -r line; do
+ if [[ $line =~ ^[[:space:]]*(#|$) ]]; then
+ continue
+ fi
+ if [[ ! $line =~ ^(FLAVOR|ARCH|VERSION|COMMIT|TARBALL_SHA256|ROOTFS_TAR_SHA256|URL)[[:space:]]*= ]]; then
+ die "$PIN: unrecognized line '$line'"
+ fi
+ done <"$PIN"
+
+ PIN_FLAVOR=$(pin_field "$PIN" FLAVOR)
+ PIN_ARCH=$(pin_field "$PIN" ARCH)
+ PIN_VERSION=$(pin_field "$PIN" VERSION)
+ PIN_COMMIT=$(pin_field "$PIN" COMMIT)
+ PIN_OUTER_SHA=$(pin_field "$PIN" TARBALL_SHA256)
+ PIN_INNER_SHA=$(pin_field "$PIN" ROOTFS_TAR_SHA256)
+ PIN_URL=$(pin_field "$PIN" URL)
+
+ [[ $PIN_FLAVOR =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]] \
+ || die "$PIN: FLAVOR='$PIN_FLAVOR' is malformed"
+ [[ $PIN_ARCH =~ ^[A-Za-z0-9][A-Za-z0-9-]*$ ]] \
+ || die "$PIN: ARCH='$PIN_ARCH' is malformed"
+ [[ $PIN_VERSION =~ ^[0-9]+(\.[0-9]+)+$ ]] \
+ || die "$PIN: VERSION='$PIN_VERSION' is malformed (expected e.g. 2150.9.0)"
+ [[ $PIN_COMMIT =~ ^[0-9a-f]{40}$ ]] \
+ || die "$PIN: COMMIT='$PIN_COMMIT' is not a full 40-hex-digit commit"
+ [[ $PIN_OUTER_SHA =~ ^[0-9a-f]{64}$ ]] \
+ || die "$PIN: TARBALL_SHA256='$PIN_OUTER_SHA' is not a 64-hex-digit sha256"
+ [[ $PIN_INNER_SHA =~ ^[0-9a-f]{64}$ ]] \
+ || die "$PIN: ROOTFS_TAR_SHA256='$PIN_INNER_SHA' is not a 64-hex-digit sha256"
+
+ [ -f "$MKOSI_CONF" ] || die "missing $MKOSI_CONF (the Release= source of truth)"
+ local release_count release
+ release_count=$(grep -c '^Release=' -- "$MKOSI_CONF") || release_count=0
+ [ "$release_count" -eq 1 ] \
+ || die "$MKOSI_CONF carries $release_count Release= lines; expected exactly one"
+ release=$(sed -n 's/^Release=//p' -- "$MKOSI_CONF" | head -n 1)
+ [ "$PIN_VERSION" = "$release" ] \
+ || die "$PIN: VERSION=$PIN_VERSION does not match Release=$release in $MKOSI_CONF (the pin must name the suite the extensions are built for)"
+
+ local derived
+ derived="https://github.com/gardenlinux/gardenlinux/releases/download/$PIN_VERSION/$PIN_FLAVOR-$PIN_ARCH-$PIN_VERSION-${PIN_COMMIT:0:8}.tar.xz"
+ [ "$PIN_URL" = "$derived" ] \
+ || die "$PIN: URL='$PIN_URL' does not match the URL derived from the pin's fields ('$derived')"
+}
+
+cache_key() {
+ printf '%s-%s-%s-%s' "$PIN_FLAVOR" "$PIN_ARCH" "$PIN_VERSION" "${PIN_COMMIT:0:8}"
+}
+
+inner_member() {
+ printf '%s.tar' "$(cache_key)"
+}
+
+tree_digest() { #
+ tar -C "$1" -cf - --sort=name --owner=0 --group=0 --numeric-owner --mtime='@0' . \
+ | sha256sum | awk '{print $1}'
+}
+
+verify_sha256() { #