sci: ship the SCI layer as verity-signed systemd-sysext extensions

.github/actions/test/integration/build/action.yml

@@ -2,13 +2,30 @@ name: build
 description: Build Bootable Virtual Machine Image
 inputs:
   image_tag:
-    description: "Image tag to use for the build"
-    required: true
+    description: "Image tag to use for the build (legacy mode)"
+    required: false
+    default: ""
+  mode:
+    description: "legacy: build the Garden Linux HV image; sysext: only prepare the Ubuntu guest"
+    required: false
+    default: "legacy"
+  sysext_key:
+    description: "Private test key prepared by prepare-sysext-hv.sh (sysext mode); its .pub is reused for the Ubuntu guest"
+    required: false
+    default: ""
 runs:
   using: "composite"
   steps:
+    - name: Validate the test mode
+      if: ${{ inputs.mode != 'legacy' && inputs.mode != 'sysext' }}
+      shell: bash
+      run: |
+        echo "unsupported mode '${{ inputs.mode }}' (expected legacy or sysext)" >&2
+        exit 1
+
     # TODO https://github.com/actions/cache
     - name: Set ENV
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       env:
         REPO: ghcr.io/gardenlinux/gardenlinux-sci
@@ -22,6 +39,7 @@ runs:
         echo "DISK_SIZE=$DISK_SIZE" >> $GITHUB_ENV
 
     - name: Pull ORAS artifacts
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       run: |
         LAYERS=$(oras manifest fetch "$REPO:$TAG" | jq -r '.layers[]')
@@ -32,11 +50,13 @@ runs:
         oras blob fetch "$REPO@$TAR_SHA" -o "$RUNNER_TEMP/$TAG.tar" # wasteful, we only systemd-boot
 
     - name: Extract systemd-boot
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       run: |
         tar -xf "$RUNNER_TEMP/$TAG.tar" --no-same-owner --strip-components=5 -C "$RUNNER_TEMP" usr/lib/systemd/boot/efi/systemd-bootx64.efi
 
     - name: Set up a raw disk image
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       run: |
         ESP_END="${ESP_SIZE}GiB"
@@ -62,6 +82,7 @@ runs:
         sudo mkfs.ext4 "$VAR_PART" -L VAR
 
     - name: Populate the ESP partition
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       run: |
         mkdir -p "$RUNNER_TEMP/esp"
@@ -88,25 +109,39 @@ runs:
         sudo losetup -d "$LOOPDEV"
 
     - name: Convert to QCOW2
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       run: |
         qemu-img convert -O qcow2 "$DISK_IMG" "/opt/${TAG}.qcow2"
 
-    - name: Generate ignition
+    - name: Generate test SSH key and password
       shell: bash
+      env:
+        SYSEXT_KEY: ${{ inputs.sysext_key }}
       run: |
-        ssh-keygen -t ed25519 -f "/opt/ssh_host_ed25519_key" -N "" -C "user" -q
-        chmod 600 /opt/ssh_host_ed25519_key
+        if [ -n "$SYSEXT_KEY" ]; then
+          # One key for the HVs and the Ubuntu guests.
+          install -m 600 "$SYSEXT_KEY" /opt/ssh_host_ed25519_key
+          install -m 644 "$SYSEXT_KEY.pub" /opt/ssh_host_ed25519_key.pub
+        else
+          ssh-keygen -t ed25519 -f "/opt/ssh_host_ed25519_key" -N "" -C "user" -q
+          chmod 600 /opt/ssh_host_ed25519_key
+        fi
         SSH_KEY=$(cat /opt/ssh_host_ed25519_key.pub)
+        PASSWORD=$(openssl passwd "password")
         echo "SSH_KEY=$SSH_KEY" >> $GITHUB_ENV
+        echo "PASSWORD=$PASSWORD" >> $GITHUB_ENV
+
+    - name: Generate ignition
+      if: ${{ inputs.mode != 'sysext' }}
+      shell: bash
+      run: |
         cp ./.github/actions/test/integration/build/dev-user-butane.yaml ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
         yq '.storage.files += load("./.github/actions/test/integration/build/root-hints.yaml")' ./.github/actions/test/integration/build/dev-user-butane.yaml > ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml
 
         sed -i "s|SSH_KEY_GOES_HERE|$SSH_KEY|g" ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
         sed -i "s|SSH_KEY_GOES_HERE|$SSH_KEY|g" ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml
 
-        PASSWORD=$(openssl passwd "password")
-        echo "PASSWORD=$PASSWORD" >> $GITHUB_ENV
         sed -i "s|PASSWORD_GOES_HERE|$PASSWORD|g" ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml
         sed -i "s|PASSWORD_GOES_HERE|$PASSWORD|g" ./.github/actions/test/integration/build/dev-user-butane-hv2.yaml
         sed -i "s|TAG_GOES_HERE|${{ inputs.image_tag }}|g" ./.github/actions/test/integration/build/dev-user-butane-hv1.yaml

.github/actions/test/integration/collect-logs/action.yml

@@ -13,6 +13,11 @@ runs:
         sudo cp /var/log/libvirt/qemu/HV1.log "libvirt-HV1.log" || true
         sudo cp /var/log/libvirt/qemu/HV2.log "libvirt-HV2.log" || true
 
+        # The HV serial consoles: the only record of a frozen guest's
+        # final kernel output (the journal goes down with the guest).
+        sudo cp /var/log/HV1.log "serial-HV1.log" || true
+        sudo cp /var/log/HV2.log "serial-HV2.log" || true
+
     - name: Obtain logs from hypervisors
       shell: bash
       run: |

.github/actions/test/integration/dependencies/action.yml

@@ -10,7 +10,7 @@ runs:
         sudo apt-get install -y \
           parted \
           squashfs-tools \
-          qemu-kvm \
+          qemu-system-x86 \
           genisoimage \
           libvirt-daemon-system \
           libvirt-clients \
@@ -26,6 +26,12 @@ runs:
         echo 'security_driver = "none"' | sudo tee -a /etc/libvirt/qemu.conf
         sudo systemctl restart libvirtd
 
+    # The libvirt qemu user must reach /dev/kvm for the hypervisor VMs.
+    - name: Make /dev/kvm accessible
+      shell: bash
+      run: |
+        if [ -e /dev/kvm ]; then sudo chmod 666 /dev/kvm; fi
+
     - name: oras
       uses: oras-project/setup-oras@1d808f7d7f6995cc68b7bf507bfe5c5446e1dc9d # v2
       with:

.github/actions/test/integration/setup/action.yml

@@ -1,9 +1,27 @@
 name: setup
 description: Set up Hypervisor and VM Environment for Integration Tests
+inputs:
+  mode:
+    description: "legacy: Garden Linux image + ignition; sysext: published base + delivery container"
+    required: false
+    default: "legacy"
+  hv_dir:
+    description: "Output of prepare-sysext-hv.sh --prepare (sysext mode)"
+    required: false
+    default: ""
 runs:
   using: "composite"
   steps:
+    - name: Validate the test mode
+      if: ${{ inputs.mode != 'legacy' && inputs.mode != 'sysext' }}
+      shell: bash
+      run: |
+        echo "unsupported mode '${{ inputs.mode }}' (expected legacy or sysext)" >&2
+        exit 1
+
+    # ---- legacy path ------------------------------------------------------
     - name: Configure the default libvirt network
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       run: |
         sudo virsh net-destroy default
@@ -13,12 +31,14 @@ runs:
         sudo virsh net-autostart default
 
     - name: Copy ignition file
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       run: |
         sudo cp "/opt/${TAG}-hv1.ign" /var/lib/libvirt/images/HV1.ign
         sudo cp "/opt/${TAG}-hv2.ign" /var/lib/libvirt/images/HV2.ign
 
     - name: Create HyperVisor 1
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       run: |
         cp ./.github/actions/test/integration/setup/hv.xml ./.github/actions/test/integration/setup/hv1.xml
@@ -30,6 +50,7 @@ runs:
         sudo virsh start HV1
 
     - name: Create HyperVisor 2
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       run: |
         cp ./.github/actions/test/integration/setup/hv.xml ./.github/actions/test/integration/setup/hv2.xml
@@ -41,6 +62,7 @@ runs:
         sudo virsh start HV2
 
     - name: Configure SSH and libvirt on HyperVisors
+      if: ${{ inputs.mode != 'sysext' }}
       shell: bash
       run: |
         KEY=/opt/ssh_host_ed25519_key
@@ -110,3 +132,72 @@ runs:
 
         wait_for_ssh HV1
         wait_for_ssh HV2
+
+    # ---- sysext path ------------------------------------------------------
+    - name: Boot the sysext HyperVisors and run the delivery container
+      if: ${{ inputs.mode == 'sysext' }}
+      shell: bash
+      env:
+        HV_DIR: ${{ inputs.hv_dir }}
+      run: |
+        set -Eeuo pipefail
+        [ -n "$HV_DIR" ] && [ -f "$HV_DIR/metadata.env" ] || { echo "hv_dir with metadata.env is required in sysext mode" >&2; exit 1; }
+        source "$HV_DIR/metadata.env"
+        KEY=$HV_DIR/hv-test-key
+        SSH="-i $KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
+        for f in "$KEY" "$HV_KERNEL" "$HV_ARTIFACT_TAR" "$HV_DIR/sci-delivery.ref" "$HV_DIR/HV1.xml" "$HV_DIR/HV2.xml" "$HV_DIR/disks/HV1.qcow2" "$HV_DIR/disks/HV2.qcow2"; do
+          [ -s "$f" ] || { echo "missing prepared artifact: $f" >&2; exit 1; }
+        done
+        [ -n "$HV_ARTIFACT_REF" ] || { echo "empty delivery image reference" >&2; exit 1; }
+        sudo virsh net-info default >/dev/null || { echo "the libvirt default network is not active; sysext mode does not create it" >&2; exit 1; }
+        # The runner home ships as 750; qemu must traverse it to reach the disks.
+        sudo chmod o+x "$HOME"
+        add_host() { sudo virsh net-dumpxml default | grep -q "mac='$1'" || sudo virsh net-update default add ip-dhcp-host "<host mac='$1' ip='$2' name='$3'/>" --live --config; }
+        add_host 52:54:00:12:34:56 192.168.122.2 hv1
+        add_host 52:54:00:65:43:21 192.168.122.3 hv2
+        for spec in "HV1 192.168.122.2" "HV2 192.168.122.3"; do
+          set -- $spec
+          sudo virsh dominfo "$1" >/dev/null 2>&1 && { echo "refusing to replace a pre-existing domain: $1" >&2; exit 1; }
+          sudo virsh define "$HV_DIR/$1.xml"
+          sudo virsh start "$1" || { namei -l "$HV_DIR/disks/$1.qcow2" >&2 || true; exit 1; }
+          for ((t=1; t<=60; t++)); do
+            ssh $SSH "root@$2" true 2>/dev/null && break
+            [ "$t" -lt 60 ] || { echo "timeout waiting for $1 to respond to SSH" >&2; sudo cat "/var/log/$1.log" || true; exit 1; }
+            sleep 5
+          done
+          scp -q $SSH "$KEY" "root@$2:/opt/ssh_host_ed25519_key"
+          scp -q $SSH "$HV_ARTIFACT_TAR" "root@$2:/opt/sci-delivery.tar"
+          scp -q $SSH "$HV_DIR/sci-delivery.ref" "root@$2:/opt/sci-delivery.ref"
+          scp -q $SSH ./.github/actions/test/integration/setup/virbr-vm.xml "root@$2:/opt/virbr-vm.xml"
+          ssh $SSH "root@$2" bash -s <<'EOF'
+        set -Eeuo pipefail
+        chmod 700 /root/.ssh
+        chmod 600 /opt/ssh_host_ed25519_key
+        systemctl is-active containerd.service >/dev/null || systemctl start containerd.service
+        REF=$(head -n 1 /opt/sci-delivery.ref)
+        test -n "$REF"
+        ctr -n k8s.io images import /opt/sci-delivery.tar
+        rm -f /opt/sci-delivery.tar
+        ctr -n k8s.io container rm -f sci-delivery >/dev/null 2>&1 || true
+        # hostIPC is required: systemd-sysext waits on a dm-verity udev cookie
+        # posted by the host udevd, so join PID 1's IPC namespace.
+        ctr -n k8s.io run --rm --privileged --with-ns pid:/proc/1/ns/pid \
+          --with-ns ipc:/proc/1/ns/ipc \
+          --env HOST_ROOT=/host --env INTERVAL=0 --env NSENTER=1 \
+          --mount type=bind,src=/,dst=/host,options=rbind:rw \
+          "$REF" sci-delivery < /dev/null
+        systemd-sysext status
+        systemctl is-active libvirtd.service
+        mkdir -p /var/log/libvirt/ch
+        printf 'Host 192.168.122.*\n    StrictHostKeyChecking no\n    UserKnownHostsFile /dev/null\n    IdentityFile /opt/ssh_host_ed25519_key\n    IdentitiesOnly yes\n    User root\n' > /root/.ssh/config
+        chmod 600 /root/.ssh/config /opt/ssh_host_ed25519_key
+        virsh pool-destroy default || true
+        virsh pool-undefine default || true
+        virsh pool-define-as default dir --target /var/lib/libvirt/images
+        virsh pool-build default; virsh pool-start default; virsh pool-autostart default
+        virsh net-destroy default || true
+        virsh net-undefine default || true
+        virsh net-define /opt/virbr-vm.xml
+        virsh net-start default; virsh net-autostart default
+        EOF
+        done

.github/actions/test/integration/setup/hv-sysext.xml

@@ -0,0 +1,42 @@
+<domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
+    <name>HV_NAME_GOES_HERE</name>
+    <memory unit='GiB'>6</memory>
+    <vcpu>4</vcpu>
+    <os>
+        <type arch='x86_64'>hvm</type>
+        <kernel>KERNEL_GOES_HERE</kernel>
+        <cmdline>root=PARTUUID=ROOT_UUID_GOES_HERE console=ttyS0 systemd.gpt_auto=0 audit=0 rw</cmdline>
+        <boot dev='hd' />
+    </os>
+    <features>
+        <acpi />
+    </features>
+    <cpu mode='host-passthrough' check='none' migratable='on' />
+    <devices>
+        <interface type='network'>
+            <mac address='MAC_GOES_HERE' />
+            <source network='NETWORK_GOES_HERE' />
+            <model type='virtio' />
+            <!-- Keep the libvirt-assigned NIC out of slot 0x02: the NVMe
+                 device arrives through qemu:commandline and QEMU claims the
+                 first free PCI slot (0x02) for it. -->
+            <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0' />
+        </interface>
+        <memballoon model='virtio'>
+            <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0' />
+        </memballoon>
+        <console type='pty'>
+            <target type='serial' port='0' />
+        </console>
+        <serial type='file'>
+            <source path='/var/log/HV_NAME_GOES_HERE.log' />
+            <target port='0' />
+        </serial>
+    </devices>
+    <qemu:commandline>
+        <qemu:arg value='-blockdev' />
+        <qemu:arg value='driver=qcow2,node-name=sciroot,file.driver=file,file.filename=DISK_PREFIX_GOES_HERE/HV_NAME_GOES_HERE.qcow2' />
+        <qemu:arg value='-device' />
+        <qemu:arg value='nvme,serial=HV_NAME_GOES_HERE,drive=sciroot,bootindex=1' />
+    </qemu:commandline>
+</domain>

.github/actions/test/integration/test/cloudhypervisor/action.yml

@@ -12,6 +12,11 @@ runs:
         IP_HV1="192.168.122.2"
         IP_HV2="192.168.122.3"
 
+        # The qemu scenario's migrated VM still runs on HV2. Its nested I/O
+        # starves the staging copy: the destination HV froze ~2h into the
+        # ubuntu.raw transfer while idle HV1 takes the same copy in seconds.
+        ssh $SSH_OPTS "$USER@$IP_HV2" 'virsh destroy VM-QEMU >/dev/null 2>&1 || true; virsh undefine VM-QEMU >/dev/null 2>&1 || true' || true
+
         scp $SSH_OPTS ./.github/actions/test/integration/test/cloudhypervisor/vm.xml "$USER@$IP_HV1:/opt/vm-chv.xml"
         scp $SSH_OPTS /opt/ubuntu.raw "$USER@$IP_HV1:/var/lib/libvirt/images/ubuntu.raw"
         scp $SSH_OPTS /opt/ubuntu-cloud-init-ds.iso "$USER@$IP_HV1:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
@@ -82,7 +87,7 @@ runs:
               exit 1
             fi
             echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
-            sleep 10
+            sleep 60
           done
         EOF
 
@@ -154,6 +159,6 @@ runs:
               exit 1
             fi
             echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
-            sleep 10
+            sleep 60
           done
         EOF

.github/actions/test/integration/test/cloudhypervisor/vm.xml

@@ -1,7 +1,7 @@
 <domain type='kvm'>
     <name>VM-CHV</name>
     <memory unit='GiB'>2</memory>
-    <vcpu placement='static'>1</vcpu>
+    <vcpu placement='static'>2</vcpu>
     <os>
         <type arch='x86_64' machine='pc'>hvm</type>
         <!-- see https://github.com/cloud-hypervisor/rust-hypervisor-firmware/issues/382 -->

.github/actions/test/integration/test/qemu/action.yml

@@ -17,7 +17,9 @@ runs:
         scp $SSH_OPTS /opt/ubuntu-cloud-init-ds.iso "$USER@$IP_HV1:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
 
         # the cloud-init data is not migrated from source to destination, but needed for the VM to run at the destination
+        # the disk itself must already exist at the destination, because --copy-storage-all does not create it
         scp $SSH_OPTS /opt/ubuntu-cloud-init-ds.iso "$USER@$IP_HV2:/var/lib/libvirt/images/ubuntu-cloud-init-ds.iso"
+        scp $SSH_OPTS /opt/ubuntu.qcow2 "$USER@$IP_HV2:/var/lib/libvirt/images/ubuntu.qcow2"
 
     - name: Start VM on HyperVisor 1
       shell: bash
@@ -80,7 +82,7 @@ runs:
               exit 1
             fi
             echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
-            sleep 10
+            sleep 60
           done
         EOF
 
@@ -152,6 +154,6 @@ runs:
               exit 1
             fi
             echo "Waiting for VM to respond to SSH... (\$i/\$MAX_ITER)"
-            sleep 10
+            sleep 60
           done
         EOF