From 5c957d08ec512d49bfb5adfa9fa8cdf23934629c Mon Sep 17 00:00:00 2001 From: rob thijssen Date: Mon, 18 May 2026 17:01:35 +0300 Subject: [PATCH] ci: add build-prerelease workflow for CUDA RPMs on rpm.lair.cafe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a manually-triggered workflow that builds CUDA-flavoured neuron binaries and a CPU cortex binary, packages them as Fedora RPMs, signs them, and rsyncs to the unstable channel at https://rpm.lair.cafe/fedora/43/x86_64/unstable/. Mirrors the build pipeline used by grenade/mistralrs-package. Pipeline: - prepare: derive {version,short_sha,commit_date} from the checkout; the prerelease Release stamp "0.1.YYYYMMDDgitSHORTSHA" sorts below the eventual "1" stable release. - build-cortex: cargo build --release -p cortex-cli on a rust runner. - build-neuron: matrix over ada (sm_89) and blackwell (sm_120) on cuda-13.0 runners; cargo build with features "cuda cudnn flash-attn" and CUDA_COMPUTE_CAP set per flavour. - package-{cortex,neuron}: rpmbuild on the rpm runner against the new prebuilt-binary specs in rpm/. - publish: import signing key, sign RPMs, rsync to oolon, createrepo_c --update, then regenerate packages.json for the UI. New specs are prebuilt-binary variants — they consume the artifact from the build job rather than running cargo at rpmbuild time. Each helexa-neuron-{flavour} package Conflicts with the other flavours and with helexa-neuron (the future source-build stable package) so one flavour is installed at a time on a given host. neuron crate gains cudnn and flash-attn feature flags forwarding to the corresponding candle features, so the CI build command compiles those kernels into the binary. sccache is intentionally NOT used in the prerelease jobs — CUDA compute cap isn't in its cache key, so flavours would mis-hit each other. Each prerelease build is a clean cargo build. Required Gitea secrets (already in place for cortex.spec / COPR workflow): - RPM_SIGNING_KEY, RPM_SIGNING_KEY_ID - RSYNC_SSH_KEY Co-Authored-By: Claude Opus 4.7 (1M context) --- .gitea/workflows/build-prerelease.yml | 319 ++++++++++++++++++++++++++ Cargo.lock | 13 ++ crates/neuron/Cargo.toml | 12 + rpm/cortex-prerelease.spec | 102 ++++++++ rpm/helexa-neuron-prerelease.spec | 122 ++++++++++ rpm/rpmmacros | 1 + script/generate-packages-json.py | 154 +++++++++++++ 7 files changed, 723 insertions(+) create mode 100644 .gitea/workflows/build-prerelease.yml create mode 100644 rpm/cortex-prerelease.spec create mode 100644 rpm/helexa-neuron-prerelease.spec create mode 100644 rpm/rpmmacros create mode 100755 script/generate-packages-json.py diff --git a/.gitea/workflows/build-prerelease.yml b/.gitea/workflows/build-prerelease.yml new file mode 100644 index 0000000..dda4110 --- /dev/null +++ b/.gitea/workflows/build-prerelease.yml @@ -0,0 +1,319 @@ +name: build-prerelease + +# Manually-dispatched workflow that builds CUDA-flavoured neuron binaries +# (and a single cortex binary), packages each as a Fedora RPM, signs +# them, and publishes to the `unstable` channel at rpm.lair.cafe. +# +# Trigger from the Gitea UI: Actions → build-prerelease → Run workflow. +# Optionally provide a `ref` to build from a non-default branch. +# +# The published packages are versioned as e.g. +# helexa-neuron-blackwell-0.1.16-0.1.20260518gitabcdef0.fc43.x86_64 +# so they sort BELOW the eventual 0.1.16-1 stable release. + +on: + workflow_dispatch: + inputs: + ref: + description: "Git ref to build (branch / tag / commit). Defaults to the workflow's branch." + required: false + default: "" + +concurrency: + group: prerelease-build + cancel-in-progress: true + +env: + CARGO_INCREMENTAL: "0" + +jobs: + prepare: + name: Resolve version stamps + runs-on: rust + outputs: + version: ${{ steps.info.outputs.version }} + release: ${{ steps.info.outputs.release }} + short_sha: ${{ steps.info.outputs.short_sha }} + commit_date: ${{ steps.info.outputs.commit_date }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - id: info + run: | + set -eux + VERSION=$(awk -F\" '/^version[[:space:]]*=/ { print $2; exit }' Cargo.toml) + SHORT_SHA=$(git rev-parse --short=7 HEAD) + COMMIT_DATE=$(git log -1 --format=%cd --date=format:%Y%m%d HEAD) + # Prerelease release stamp sorts before "1" (the stable release). + RELEASE="0.1.${COMMIT_DATE}git${SHORT_SHA}" + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "release=${RELEASE}" >> "$GITHUB_OUTPUT" + echo "short_sha=${SHORT_SHA}" >> "$GITHUB_OUTPUT" + echo "commit_date=${COMMIT_DATE}" >> "$GITHUB_OUTPUT" + + build-cortex: + name: Build cortex binary + needs: prepare + runs-on: rust + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Install/update Rust toolchain + run: | + if command -v rustup &> /dev/null; then + rustup update stable + else + curl --proto '=https' --tlsv1.2 --silent --show-error --fail https://sh.rustup.rs | sh -s -- -y + fi + echo "${HOME}/.cargo/bin" >> "$GITHUB_PATH" + + - name: Build cortex (release) + run: cargo build --release -p cortex-cli + + - name: Stage binary + run: | + mkdir --parents artifacts + cp target/release/cortex artifacts/cortex + ./artifacts/cortex --version || true + + - uses: actions/upload-artifact@v3 + with: + name: cortex-fc43 + path: artifacts/cortex + retention-days: 1 + + build-neuron: + name: Build neuron-${{ matrix.flavour }} + needs: prepare + strategy: + fail-fast: false + matrix: + include: + - flavour: ada + compute_cap: "89" + runner: cuda-13.0 + cuda_home: /usr/local/cuda-13.0 + build_jobs: 8 + nvcc_threads: 4 + cargo_features: "cuda cudnn flash-attn" + - flavour: blackwell + compute_cap: "120" + runner: cuda-13.0 + cuda_home: /usr/local/cuda-13.0 + build_jobs: 8 + nvcc_threads: 4 + cargo_features: "cuda cudnn flash-attn" + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Install/update Rust toolchain + run: | + if command -v rustup &> /dev/null; then + rustup update stable + else + curl --proto '=https' --tlsv1.2 --silent --show-error --fail https://sh.rustup.rs | sh -s -- -y + fi + echo "${HOME}/.cargo/bin" >> "$GITHUB_PATH" + + - name: Build neuron with CUDA (${{ matrix.flavour }}) + run: | + set -eux + export PATH="${{ matrix.cuda_home }}/bin:${PATH}" + export LD_LIBRARY_PATH="${{ matrix.cuda_home }}/targets/x86_64-linux/lib:${{ matrix.cuda_home }}/lib64:${LD_LIBRARY_PATH:-}" + export LIBRARY_PATH="${{ matrix.cuda_home }}/targets/x86_64-linux/lib:${{ matrix.cuda_home }}/lib64:${LIBRARY_PATH:-}" + cargo build --release -p neuron --features "${{ matrix.cargo_features }}" + env: + CUDA_COMPUTE_CAP: ${{ matrix.compute_cap }} + CARGO_BUILD_JOBS: ${{ matrix.build_jobs }} + NVCC_THREADS: ${{ matrix.nvcc_threads }} + + - name: Stage binary + run: | + mkdir --parents artifacts + cp target/release/neuron artifacts/neuron-${{ matrix.flavour }} + file "artifacts/neuron-${{ matrix.flavour }}" + + - uses: actions/upload-artifact@v3 + with: + name: neuron-${{ matrix.flavour }}-fc43 + path: artifacts/neuron-${{ matrix.flavour }} + retention-days: 1 + + package-cortex: + name: Package cortex RPM + needs: [prepare, build-cortex] + runs-on: rpm + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - uses: actions/download-artifact@v3 + with: + name: cortex-fc43 + path: artifacts/ + + - name: Build RPM + run: | + set -eux + rm -f ~/.rpmmacros + rpmdev-setuptree + cp artifacts/cortex ~/rpmbuild/SOURCES/ + cp data/cortex.service ~/rpmbuild/SOURCES/ + cp data/cortex-sysusers.conf ~/rpmbuild/SOURCES/ + cp data/cortex-firewalld.xml ~/rpmbuild/SOURCES/ + cp cortex.example.toml ~/rpmbuild/SOURCES/ + cp models.example.toml ~/rpmbuild/SOURCES/ + cp LICENSE ~/rpmbuild/SOURCES/ + rpmbuild -bb rpm/cortex-prerelease.spec \ + --define "cortex_version ${{ needs.prepare.outputs.version }}" \ + --define "cortex_prerelease ${{ needs.prepare.outputs.release }}" \ + --undefine dist \ + --define "dist .fc43" + + - uses: actions/upload-artifact@v3 + with: + name: rpm-cortex-fc43 + path: ~/rpmbuild/RPMS/x86_64/*.rpm + retention-days: 7 + + package-neuron: + name: Package helexa-neuron-${{ matrix.flavour }} RPM + needs: [prepare, build-neuron] + runs-on: rpm + strategy: + fail-fast: false + matrix: + include: + - flavour: ada + - flavour: blackwell + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - uses: actions/download-artifact@v3 + with: + name: neuron-${{ matrix.flavour }}-fc43 + path: artifacts/ + + - name: Build RPM + run: | + set -eux + rm -f ~/.rpmmacros + rpmdev-setuptree + cp artifacts/neuron-${{ matrix.flavour }} ~/rpmbuild/SOURCES/ + cp data/neuron.service ~/rpmbuild/SOURCES/ + cp data/neuron-sysusers.conf ~/rpmbuild/SOURCES/ + cp data/neuron-firewalld.xml ~/rpmbuild/SOURCES/ + cp neuron.example.toml ~/rpmbuild/SOURCES/ + cp LICENSE ~/rpmbuild/SOURCES/ + rpmbuild -bb rpm/helexa-neuron-prerelease.spec \ + --define "neuron_version ${{ needs.prepare.outputs.version }}" \ + --define "neuron_flavour ${{ matrix.flavour }}" \ + --define "neuron_prerelease ${{ needs.prepare.outputs.release }}" \ + --undefine dist \ + --define "dist .fc43" + + - uses: actions/upload-artifact@v3 + with: + name: rpm-neuron-${{ matrix.flavour }}-fc43 + path: ~/rpmbuild/RPMS/x86_64/*.rpm + retention-days: 7 + + publish: + name: Publish to rpm.lair.cafe (unstable) + needs: [package-cortex, package-neuron] + runs-on: rpm + concurrency: + group: rpm-publish + cancel-in-progress: false + env: + RPM_REPO_HOST: oolon.kosherinata.internal + FEDORA_VERSION: "43" + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + + - name: Download all built RPMs + uses: actions/download-artifact@v3 + with: + path: rpms/ + pattern: rpm-*-fc43 + + - name: Flatten RPM artifacts + run: | + set -eux + find rpms/ -name '*.rpm' -exec mv --target-directory=rpms/ {} + + find rpms/ -mindepth 1 -type d -empty -delete + ls -la rpms/ + + - name: Check for sequoia-sq + run: | + if ! command -v sq &> /dev/null; then + echo "ERROR: sequoia-sq is not installed. Install with: sudo dnf install sequoia-sq" + exit 1 + fi + + - name: Import signing key + run: | + echo "${{ secrets.RPM_SIGNING_KEY }}" | gpg --batch --import + fpr=$(gpg --batch --with-colons --list-keys "${{ secrets.RPM_SIGNING_KEY_ID }}" | awk -F: '/^fpr:/ { print $10; exit }') + echo "${fpr}:6:" | gpg --batch --import-ownertrust + sed "s/@GPG_NAME@/${{ secrets.RPM_SIGNING_KEY_ID }}/" rpm/rpmmacros > ~/.rpmmacros + + - name: Sign RPMs + run: | + set -eux + for rpm in rpms/*.rpm; do + echo "signing ${rpm}..." + rpm --addsign "${rpm}" + done + + - name: Set up SSH for rsync + run: | + install --directory --mode 700 ~/.ssh + echo "${RSYNC_SSH_KEY}" | install --mode 600 /dev/stdin ~/.ssh/id_ed25519 + env: + RSYNC_SSH_KEY: ${{ secrets.RSYNC_SSH_KEY }} + + - name: Test SSH connectivity + run: | + ssh -o StrictHostKeyChecking=accept-new "gitea_ci@${RPM_REPO_HOST}" exit + + - name: Ensure unstable repo directory exists + run: | + ssh "gitea_ci@${RPM_REPO_HOST}" \ + "mkdir --parents /var/www/rpm/fedora/${FEDORA_VERSION}/x86_64/unstable" + + - name: Sync RPMs to unstable repo + run: | + rsync \ + --archive \ + --verbose \ + --chmod D755,F644 \ + rpms/*.rpm \ + "gitea_ci@${RPM_REPO_HOST}:/var/www/rpm/fedora/${FEDORA_VERSION}/x86_64/unstable/" + + - name: Update unstable repo metadata + run: | + ssh "gitea_ci@${RPM_REPO_HOST}" \ + "cd /var/www/rpm/fedora/${FEDORA_VERSION}/x86_64/unstable && createrepo_c --update ." + + - name: Generate packages.json manifest + run: | + scp script/generate-packages-json.py "gitea_ci@${RPM_REPO_HOST}:/tmp/" + ssh "gitea_ci@${RPM_REPO_HOST}" \ + "python3 /tmp/generate-packages-json.py \ + --repodata-dir /var/www/rpm/fedora/${FEDORA_VERSION}/x86_64/unstable/repodata \ + --output /var/www/rpm/fedora/${FEDORA_VERSION}/x86_64/unstable/packages.json \ + --base-url https://rpm.lair.cafe/fedora/${FEDORA_VERSION}/x86_64/unstable" diff --git a/Cargo.lock b/Cargo.lock index 15b503b..2c34803 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -327,6 +327,18 @@ dependencies = [ "zip", ] +[[package]] +name = "candle-flash-attn" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12512cf8e706744642e9a8579305a6ed1e44a0c636ce20c416cd5c519de19b7d" +dependencies = [ + "anyhow", + "candle-core", + "cudaforge", + "half", +] + [[package]] name = "candle-kernels" version = "0.10.2" @@ -360,6 +372,7 @@ checksum = "f59d08c89e9f4af9c464e2f3a8e16199e7cc601e6f34538c2cfbb42b623b1783" dependencies = [ "byteorder", "candle-core", + "candle-flash-attn", "candle-nn", "fancy-regex", "num-traits", diff --git a/crates/neuron/Cargo.toml b/crates/neuron/Cargo.toml index 2fc0c1b..862139c 100644 --- a/crates/neuron/Cargo.toml +++ b/crates/neuron/Cargo.toml @@ -21,6 +21,18 @@ cuda = [ "candle-nn/cuda", "candle-transformers/cuda", ] +# Use cuDNN for convolution / attention kernels. Requires CUDA. +cudnn = [ + "cuda", + "candle-core/cudnn", + "candle-nn/cudnn", + "candle-transformers/cudnn", +] +# FlashAttention kernels. Requires CUDA. +flash-attn = [ + "cuda", + "candle-transformers/flash-attn", +] # Reserved for GPU-only integration tests in later stages. cuda-integration = ["cuda"] diff --git a/rpm/cortex-prerelease.spec b/rpm/cortex-prerelease.spec new file mode 100644 index 0000000..9a49c52 --- /dev/null +++ b/rpm/cortex-prerelease.spec @@ -0,0 +1,102 @@ +# Prebuilt-binary spec for cortex. +# +# Unlike cortex.spec (which builds from source via cargo), this spec +# wraps a pre-built `cortex` binary produced by an upstream CI job and +# packages it for rpm.lair.cafe. The %build phase is a no-op. +# +# Required defines at rpmbuild time: +# cortex_version e.g. "0.1.16" +# cortex_prerelease e.g. "0.1.20260518gitabcdef0" (used as Release) + +%global _build_id_links none +%global debug_package %{nil} +%global __strip /usr/bin/true + +%{!?cortex_version: %global cortex_version 0.0.0} +%if 0%{?cortex_prerelease:1} +%global cortex_release %{cortex_prerelease} +%else +%global cortex_release 1 +%endif + +Name: cortex +Version: %{cortex_version} +Release: %{cortex_release}%{?dist} +Summary: Inference gateway for multi-node GPU clusters (prebuilt) + +License: GPL-3.0-or-later +URL: https://git.lair.cafe/helexa/cortex + +Source0: cortex +Source1: cortex.service +Source2: cortex-sysusers.conf +Source3: cortex-firewalld.xml +Source4: cortex.example.toml +Source5: models.example.toml +Source6: LICENSE + +ExclusiveArch: x86_64 + +Requires(pre): shadow-utils +Requires: systemd +Requires: firewalld-filesystem + +Provides: user(cortex) + +%description +Cortex is a Rust reverse-proxy that sits in front of multiple neuron +inference daemons and presents a unified OpenAI and Anthropic +compatible API surface. + +This package wraps a binary built upstream in CI; the source-build +spec (cortex.spec) remains available for stable releases. + +%prep +cp %{SOURCE0} ./cortex +cp %{SOURCE1} . +cp %{SOURCE2} . +cp %{SOURCE3} . +cp %{SOURCE4} . +cp %{SOURCE5} . +cp %{SOURCE6} . + +%build +# Already built in the upstream CI build job. + +%install +install -Dm755 cortex %{buildroot}%{_bindir}/cortex +install -Dm644 cortex.service %{buildroot}%{_unitdir}/cortex.service +install -Dm644 cortex-sysusers.conf %{buildroot}%{_sysusersdir}/cortex.conf +install -Dm644 cortex-firewalld.xml %{buildroot}%{_prefix}/lib/firewalld/services/cortex.xml +install -dm755 %{buildroot}%{_sysconfdir}/cortex +install -Dm644 cortex.example.toml %{buildroot}%{_sysconfdir}/cortex/cortex.toml +install -Dm644 models.example.toml %{buildroot}%{_sysconfdir}/cortex/models.toml + +%pre +getent group cortex >/dev/null || groupadd -r cortex +getent passwd cortex >/dev/null || \ + useradd -r -g cortex -d /var/lib/cortex -s /sbin/nologin \ + -c "Cortex inference gateway" cortex + +%post +%systemd_post cortex.service + +%preun +%systemd_preun cortex.service + +%postun +%systemd_postun_with_restart cortex.service + +%files +%license LICENSE +%{_bindir}/cortex +%{_unitdir}/cortex.service +%{_sysusersdir}/cortex.conf +%{_prefix}/lib/firewalld/services/cortex.xml +%dir %{_sysconfdir}/cortex +%config(noreplace) %{_sysconfdir}/cortex/cortex.toml +%config(noreplace) %{_sysconfdir}/cortex/models.toml + +%changelog +* Mon May 18 2026 Gitea Actions - %{cortex_version}-%{cortex_release} +- Prerelease build from upstream CI binary. diff --git a/rpm/helexa-neuron-prerelease.spec b/rpm/helexa-neuron-prerelease.spec new file mode 100644 index 0000000..8e874d6 --- /dev/null +++ b/rpm/helexa-neuron-prerelease.spec @@ -0,0 +1,122 @@ +# Prebuilt-binary spec for helexa-neuron flavoured by CUDA compute capability. +# +# Unlike helexa-neuron.spec (which builds from source via cargo), this +# spec wraps a pre-built `neuron-{flavour}` binary produced by an +# upstream CI job and packages it for rpm.lair.cafe. The %build phase +# is a no-op. +# +# Required defines at rpmbuild time: +# neuron_version e.g. "0.1.16" +# neuron_flavour e.g. "ada", "blackwell" — matches the CI build +# matrix's compute_cap label. +# neuron_prerelease e.g. "0.1.20260518gitabcdef0" (used as Release) +# +# One flavour can be installed at a time on a given host; flavour +# packages Conflict with each other. + +%global _build_id_links none +%global debug_package %{nil} +%global __strip /usr/bin/true + +%{!?neuron_version: %global neuron_version 0.0.0} +%{!?neuron_flavour: %global neuron_flavour blackwell} +%if 0%{?neuron_prerelease:1} +%global neuron_release %{neuron_prerelease} +%else +%global neuron_release 1 +%endif + +Name: helexa-neuron-%{neuron_flavour} +Version: %{neuron_version} +Release: %{neuron_release}%{?dist} +Summary: Per-node GPU inference daemon (candle, %{neuron_flavour} flavour) + +License: GPL-3.0-or-later +URL: https://git.lair.cafe/helexa/cortex + +Source0: neuron-%{neuron_flavour} +Source1: neuron.service +Source2: neuron-sysusers.conf +Source3: neuron-firewalld.xml +Source4: neuron.example.toml +Source5: LICENSE + +ExclusiveArch: x86_64 + +# Binary links against the CUDA runtime, cuDNN, NCCL, etc. Suppress +# auto-detected exact soname deps — users may have CUDA from various +# sources (rpmfusion, nvidia-direct) at different compatible versions; +# a runtime dlopen failure surfaces a clearer error than rpm dep +# resolution would. +%global __requires_exclude ^lib(cuda|cudart|cudnn|cublas|cublasLt|curand|nvrtc|nccl) + +Requires(pre): shadow-utils +Requires: systemd +Requires: firewalld-filesystem + +Provides: helexa-neuron = %{neuron_version}-%{neuron_release} +Provides: user(neuron) + +# Mutual exclusion across flavours and the source-build variant. +Conflicts: helexa-neuron +Conflicts: helexa-neuron-ada +Conflicts: helexa-neuron-ampere +Conflicts: helexa-neuron-blackwell +# (The Conflicts: with self is filtered by rpm at install time.) + +%description +Neuron is the per-node daemon for cortex inference clusters. It +discovers local GPU hardware via nvidia-smi, runs in-process +inference via huggingface/candle, and exposes an HTTP API for model +lifecycle management (load, unload, list, inference endpoint). + +This is the %{neuron_flavour} flavour, built for that CUDA compute +capability. Install the flavour matching the GPUs on this host. + +%prep +cp %{SOURCE0} ./neuron +cp %{SOURCE1} . +cp %{SOURCE2} . +cp %{SOURCE3} . +cp %{SOURCE4} . +cp %{SOURCE5} . + +%build +# Already built in the upstream CI build job (with --features cuda). + +%install +install -Dm755 neuron %{buildroot}%{_bindir}/neuron +install -Dm644 neuron.service %{buildroot}%{_unitdir}/neuron.service +install -Dm644 neuron-sysusers.conf %{buildroot}%{_sysusersdir}/neuron.conf +install -Dm644 neuron-firewalld.xml %{buildroot}%{_prefix}/lib/firewalld/services/helexa-neuron.xml +install -dm755 %{buildroot}%{_sysconfdir}/neuron +install -Dm644 neuron.example.toml %{buildroot}%{_sysconfdir}/neuron/neuron.toml + +%pre +getent group neuron >/dev/null || groupadd -r neuron +getent passwd neuron >/dev/null || \ + useradd -r -g neuron -d /var/lib/neuron -s /sbin/nologin \ + -G video,render \ + -c "Neuron GPU node daemon" neuron + +%post +%systemd_post neuron.service + +%preun +%systemd_preun neuron.service + +%postun +%systemd_postun_with_restart neuron.service + +%files +%license LICENSE +%{_bindir}/neuron +%{_unitdir}/neuron.service +%{_sysusersdir}/neuron.conf +%{_prefix}/lib/firewalld/services/helexa-neuron.xml +%dir %{_sysconfdir}/neuron +%config(noreplace) %{_sysconfdir}/neuron/neuron.toml + +%changelog +* Mon May 18 2026 Gitea Actions - %{neuron_version}-%{neuron_release} +- Prerelease build from upstream CI binary (%{neuron_flavour} flavour). diff --git a/rpm/rpmmacros b/rpm/rpmmacros new file mode 100644 index 0000000..03ce44d --- /dev/null +++ b/rpm/rpmmacros @@ -0,0 +1 @@ +%_openpgp_sign_id @GPG_NAME@ diff --git a/script/generate-packages-json.py b/script/generate-packages-json.py new file mode 100755 index 0000000..ea663df --- /dev/null +++ b/script/generate-packages-json.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Parse RPM repodata and emit a packages.json manifest for the UI.""" + +import argparse +import gzip +import json +import os +import subprocess +import sys +import xml.etree.ElementTree as ET +from datetime import datetime, timezone + +RPM_NS = "http://linux.duke.edu/metadata/common" +OTHER_NS = "http://linux.duke.edu/metadata/other" +REPO_NS = "http://linux.duke.edu/metadata/repo" + + +def find_repodata_file(repodata_dir, data_type): + """Read repomd.xml and return the path to a specific data type's file.""" + repomd_path = os.path.join(repodata_dir, "repomd.xml") + tree = ET.parse(repomd_path) + root = tree.getroot() + + for data in root.findall(f"{{{REPO_NS}}}data"): + if data.get("type") == data_type: + location = data.find(f"{{{REPO_NS}}}location") + if location is not None: + href = location.get("href", "") + return os.path.join(os.path.dirname(repodata_dir), href) + + return None + + +def open_compressed(path): + """Open a gzip or zstd compressed file for reading.""" + if path.endswith(".zst"): + result = subprocess.run( + ["zstdcat", path], capture_output=True, check=True + ) + import io + return io.BytesIO(result.stdout) + else: + return gzip.open(path, "rb") + + +def parse_primary(repodata_dir): + """Parse primary.xml.{gz,zst} and return package metadata.""" + path = find_repodata_file(repodata_dir, "primary") + if not path: + print("error: primary metadata not found in repomd.xml", file=sys.stderr) + sys.exit(1) + + packages = {} + with open_compressed(path) as f: + tree = ET.parse(f) + + for pkg in tree.getroot().findall(f"{{{RPM_NS}}}package"): + if pkg.get("type") != "rpm": + continue + + name = pkg.findtext(f"{{{RPM_NS}}}name", "") + version_el = pkg.find(f"{{{RPM_NS}}}version") + ver = version_el.get("ver", "") if version_el is not None else "" + rel = version_el.get("rel", "") if version_el is not None else "" + arch = pkg.findtext(f"{{{RPM_NS}}}arch", "") + + size_el = pkg.find(f"{{{RPM_NS}}}size") + size = int(size_el.get("package", "0")) if size_el is not None else 0 + + time_el = pkg.find(f"{{{RPM_NS}}}time") + build_time = int(time_el.get("build", "0")) if time_el is not None else 0 + + location_el = pkg.find(f"{{{RPM_NS}}}location") + filename = os.path.basename(location_el.get("href", "")) if location_el is not None else "" + + key = f"{name}-{ver}-{rel}" + packages[key] = { + "name": name, + "version": ver, + "release": rel, + "arch": arch, + "summary": pkg.findtext(f"{{{RPM_NS}}}summary", ""), + "size": size, + "buildTime": build_time, + "rpmFilename": filename, + "changelog": [], + } + + return packages + + +def parse_other(repodata_dir, packages): + """Parse other.xml.gz and attach changelog entries to packages.""" + path = find_repodata_file(repodata_dir, "other") + if not path: + return + + with open_compressed(path) as f: + tree = ET.parse(f) + + for pkg in tree.getroot().findall(f"{{{OTHER_NS}}}package"): + name = pkg.get("name", "") + version_el = pkg.find(f"{{{OTHER_NS}}}version") + ver = version_el.get("ver", "") if version_el is not None else "" + rel = version_el.get("rel", "") if version_el is not None else "" + key = f"{name}-{ver}-{rel}" + + if key not in packages: + continue + + for entry in pkg.findall(f"{{{OTHER_NS}}}changelog"): + packages[key]["changelog"].append({ + "author": entry.get("author", ""), + "date": int(entry.get("date", "0")), + "text": (entry.text or "").strip(), + }) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--repodata-dir", + required=True, + help="path to the repodata/ directory", + ) + parser.add_argument( + "--output", + required=True, + help="path to write packages.json", + ) + parser.add_argument( + "--base-url", + required=True, + help="public base URL for the repo (e.g. https://rpm.lair.cafe/fedora/43/x86_64)", + ) + args = parser.parse_args() + + packages = parse_primary(args.repodata_dir) + parse_other(args.repodata_dir, packages) + + manifest = { + "generated": datetime.now(timezone.utc).isoformat(), + "baseUrl": args.base_url, + "packages": list(packages.values()), + } + + with open(args.output, "w") as f: + json.dump(manifest, f, indent=2) + + print(f"wrote {len(packages)} packages to {args.output}") + + +if __name__ == "__main__": + main()