diff --git a/build/build-image.sh b/build/build-image.sh index 5bac94c..3d7961f 100755 --- a/build/build-image.sh +++ b/build/build-image.sh @@ -132,11 +132,22 @@ echo "==> variant=${VARIANT} release=${FEDORA_RELEASE} arch=${TARGET_ARCH} ref=$ echo "==> build container: ${CONTAINER_IMAGE}" echo "==> work=${WORK_DIR} cache=${CACHE_DIR}" +# Compressing an 8 GiB image is architecture-independent work. Doing it inside +# the aarch64 container means doing it under qemu-user, which measured at ~20 +# minutes against ~2 natively. Do it here when the host can. +if command -v zstd >/dev/null; then + COMPRESS_IN_CONTAINER=0 +else + echo "note: no zstd on this host, compressing inside the container instead" + echo " (install zstd to save roughly 20 minutes per build)" + COMPRESS_IN_CONTAINER=1 +fi + # --privileged is what lets stage2 bind-mount /proc and /sys into the staged # rootfs so dracut can run in a chroot. Rootless podman grants only the caps # the invoking user already has inside their user namespace, so this is not # the escalation it looks like. -exec podman run --rm \ +podman run --rm \ --arch arm64 \ --privileged \ --security-opt label=disable \ @@ -151,5 +162,21 @@ exec podman run --rm \ -e BUILD_DATE="$BUILD_DATE" \ -e KEEP_ROOTFS="$KEEP_ROOTFS" \ -e FRESH="$FRESH" \ + -e COMPRESS_IN_CONTAINER="$COMPRESS_IN_CONTAINER" \ "$CONTAINER_IMAGE" \ /bin/bash /src/build/stage2.sh + +if [ "$COMPRESS_IN_CONTAINER" = 0 ]; then + name="$(cat "$OUTPUT_DIR/.build-result")" + img="$OUTPUT_DIR/${name}.img" + [ -f "$img" ] || { echo "error: ${img} is missing" >&2; exit 1; } + + echo + echo "==> Compressing $(du -h "$img" | cut -f1) natively" + zstd -12 -T0 --rm -f -o "${img}.zst" "$img" + ( cd "$OUTPUT_DIR" && sha256sum "${name}.img.zst" > "${name}.img.zst.sha256" ) +fi + +rm -f "$OUTPUT_DIR/.build-result" +echo +ls -lh "$OUTPUT_DIR"/*.img.zst "$OUTPUT_DIR"/*.sha256 2>/dev/null diff --git a/build/stage2.sh b/build/stage2.sh index 38b9a10..6f95d20 100755 --- a/build/stage2.sh +++ b/build/stage2.sh @@ -26,6 +26,18 @@ source "$SRC/config/device.env" : "${BUILD_DATE:=unknown}" : "${KEEP_ROOTFS:=0}" : "${FRESH:=0}" +: "${COMPRESS_IN_CONTAINER:=1}" + +# Bump when the *procedure* for staging the base changes in a way that alters +# its contents. The stamp otherwise hashes only the package list, so a change +# like excluding a weak dependency would silently reuse a stale base. +BASE_RECIPE=2 + +# dracut-config-rescue arrives as a weak dependency of dracut and only costs us: +# it makes kernel-install build a second, rescue initramfs — another emulated +# dracut run — and leaves a rescue entry in the boot menu we never use. On a +# machine this slow it would also double the cost of every future kernel update. +DNF_EXCLUDE=(--exclude=dracut-config-rescue) IMAGE_NAME="fedora-${FEDORA_RELEASE}-${VARIANT}-${DEVICE_NAME}-${BUILD_DATE}-${BUILD_REF}" IMAGE_PATH="$OUT/${IMAGE_NAME}.img" @@ -86,7 +98,8 @@ echo "${#PACKAGES[@]} package specs" # copy instead of a reinstall. # --------------------------------------------------------------------------- WANT_STAMP="$(printf '%s\n' "$FEDORA_RELEASE" "$TARGET_ARCH" "$VARIANT" \ - "${PACKAGES[@]}" | sha256sum | cut -d' ' -f1)" + "recipe=$BASE_RECIPE" "${DNF_EXCLUDE[@]}" "${PACKAGES[@]}" \ + | sha256sum | cut -d' ' -f1)" if [ "$FRESH" = 1 ]; then log "Discarding the staged base (--fresh)" @@ -108,6 +121,19 @@ else cp -a /etc/pki/rpm-gpg/. "$BASE/etc/pki/rpm-gpg/" if [ -d /etc/dnf/vars ]; then cp -a /etc/dnf/vars "$BASE/etc/dnf/"; fi + # The kernel's %posttrans runs kernel-install, which runs dracut — before + # this build has written /etc/dracut.conf.d/10-c630.conf and before /proc is + # bind-mounted. The result is an initramfs that is both wrong and expensive: + # two emulated dracut runs (normal + rescue) costing roughly 40 minutes, + # immediately superseded by the one stage2 builds later with the right + # config. Both 50-dracut.install and 51-dracut-rescue.install bail out when + # KERNEL_INSTALL_INITRD_GENERATOR is anything other than "dracut". + # + # This file must not survive into the image — see the removal after the + # working copy is made. + mkdir -p "$BASE/etc/kernel" + printf 'initrd_generator=none\n' > "$BASE/etc/kernel/install.conf" + log "Installing Fedora ${FEDORA_RELEASE} (${TARGET_ARCH}) — this is the slow part" # keepcache=1 with a cachedir outside the install root: the downloaded rpms # outlive both the transaction and the staged tree, so a --fresh rebuild @@ -118,6 +144,7 @@ else --setopt=cachedir="$DNF_CACHE" \ --setopt=keepcache=1 \ --setopt=install_weak_deps=True \ + "${DNF_EXCLUDE[@]}" \ install "${PACKAGES[@]}" printf '%s\n' "$WANT_STAMP" > "$STAMP" @@ -133,6 +160,13 @@ rm -rf "$ROOTFS" cp -a --reflink=auto "$BASE" "$ROOTFS" mkdir -p "$WORK/esp" +# Undo the build-time suppression of initramfs generation. Shipping this would +# mean the laptop generates no initramfs on its next kernel update and does not +# come back up — the worst kind of bug, because it appears weeks later and looks +# nothing like an image problem. Nothing in Fedora owns this path, so removing +# it restores stock behaviour exactly. +rm -f "$ROOTFS/etc/kernel/install.conf" + KVER="$(rpm --root "$ROOTFS" -q kernel-core --qf '%{VERSION}-%{RELEASE}.%{ARCH}\n' \ | sort -V | tail -1)" [ -n "$KVER" ] || { echo "could not determine installed kernel version" >&2; exit 1; } @@ -358,6 +392,16 @@ mcopy -i "$WORK/esp.img" -s "$WORK/esp/EFI" :: mke2fs -q -t ext4 -b 4096 -O "$EXT4_OPTS" -L boot -U "$BOOT_UUID" \ -d "$WORK/boot" "$WORK/boot.img" $(( BOOT_SECTORS / 8 )) +# Last chance to catch the build-time initramfs suppression leaking into the +# image. If it shipped, the laptop would boot fine and then fail to come back +# after its next kernel update — far from here, and looking nothing like an +# image bug. Cheap to assert, so assert it. +if [ -e "$ROOTFS/etc/kernel/install.conf" ]; then + echo "error: /etc/kernel/install.conf is about to ship — it disables" >&2 + echo " initramfs generation and would brick the next kernel update" >&2 + exit 1 +fi + mke2fs -q -t ext4 -b 4096 -O "$EXT4_OPTS" -L fedora -U "$ROOT_UUID" \ -d "$ROOTFS" "$WORK/root.img" $(( ROOT_SECTORS / 8 )) @@ -369,10 +413,17 @@ dd if="$WORK/boot.img" of="$IMAGE_PATH" bs=512 seek="$BOOT_START" conv=notrunc,s dd if="$WORK/root.img" of="$IMAGE_PATH" bs=512 seek="$ROOT_START" conv=notrunc,sparse status=none # --------------------------------------------------------------------------- -log "Compressing" +# Compression is architecture-independent, so running it here runs it under +# emulation — roughly twenty minutes for work the host does in two. When the +# host has zstd, build-image.sh handles it and this just hands over the name. # --------------------------------------------------------------------------- -zstd -12 -T0 --rm -f -o "${IMAGE_PATH}.zst" "$IMAGE_PATH" -( cd "$OUT" && sha256sum "${IMAGE_NAME}.img.zst" > "${IMAGE_NAME}.img.zst.sha256" ) +printf '%s\n' "$IMAGE_NAME" > "$OUT/.build-result" + +if [ "$COMPRESS_IN_CONTAINER" = 1 ]; then + log "Compressing (in-container: the host had no zstd)" + zstd -12 -T0 --rm -f -o "${IMAGE_PATH}.zst" "$IMAGE_PATH" + ( cd "$OUT" && sha256sum "${IMAGE_NAME}.img.zst" > "${IMAGE_NAME}.img.zst.sha256" ) +fi # The intermediate filesystem images are multi-gigabyte and worthless once # they are inside the disk image. The staged base is the opposite: expensive to diff --git a/docs/runner-setup.md b/docs/runner-setup.md index 78926fb..3f8f002 100644 --- a/docs/runner-setup.md +++ b/docs/runner-setup.md @@ -54,8 +54,19 @@ harmless. Delete `/var/tmp/c630-build` to reclaim the space. ## Runtime Every aarch64 binary runs under qemu-user emulation, and rpm scriptlets are the -worst case. A cold build is roughly 45–90 minutes for `minimal` and several -hours for `workstation`; `timeout-minutes` is set to 600 accordingly. +worst case. A cold `minimal` build is roughly 40 minutes; `workstation` is +several hours. `timeout-minutes` is set to 600 accordingly. + +The first measured run took 100 minutes, and about 60 of those were avoidable: + +| Cost | Cause | Fix | +|---|---|---| +| ~40 min | The kernel's `%posttrans` ran dracut twice (normal + rescue) before the C630 dracut config existed, producing an initramfs immediately thrown away | `initrd_generator=none` in the install root during the transaction; `dracut-config-rescue` excluded | +| ~20 min | `zstd` running under emulation | Compression moved to the host | + +Both are worth remembering if either ever regresses: the symptom is a build +that appears to hang with no output, because rpm scriptlet output is buffered +until the transaction ends. A warm build — one where the package set has not changed since that runner last built — skips the dnf transaction entirely and finishes in minutes. Because the