chore: init

This commit is contained in:
2026-04-24 09:10:36 +03:00
commit 3b1c6843d6
13 changed files with 562 additions and 0 deletions

View File

@@ -0,0 +1,126 @@
name: build-release
on:
workflow_dispatch:
inputs:
tag:
description: "mistral.rs upstream tag"
required: true
type: string
jobs:
plan:
runs-on: fedora
outputs:
flavours: ${{ steps.plan.outputs.flavours }}
version: ${{ steps.plan.outputs.version }}
steps:
- uses: actions/checkout@v4
- id: plan
run: |
version="${TAG#v}"
echo "version=${version}" >> "$GITHUB_OUTPUT"
# Emit flavours as a JSON array for matrix consumption
flavours=$(yq -o=json -I=0 '.flavours' flavours.yml)
echo "flavours=${flavours}" >> "$GITHUB_OUTPUT"
env:
TAG: ${{ inputs.tag }}
build:
needs: plan
runs-on: cuda-13.0
strategy:
fail-fast: false
matrix:
flavour: ${{ fromJSON(needs.plan.outputs.flavours) }}
steps:
- uses: actions/checkout@v4
- name: Clone mistral.rs at tag
run: |
git clone --depth 1 --branch "${{ inputs.tag }}" \
https://github.com/EricLBuehler/mistral.rs.git src/
- name: Build
run: ./script/build-binary.sh
env:
FLAVOUR_NAME: ${{ matrix.flavour.name }}
CUDA_HOME: ${{ matrix.flavour.cuda_home }}
CARGO_FEATURES: ${{ matrix.flavour.cargo_features }}
CUDA_COMPUTE_CAP: ${{ matrix.flavour.compute_caps }}
SRC_DIR: src
- name: Upload binary artifact
uses: actions/upload-artifact@v3
with:
name: mistralrs-server-${{ matrix.flavour.name }}
path: artifacts/mistralrs-server-${{ matrix.flavour.name }}
retention-days: 1
package:
needs: [plan, build]
runs-on: fedora
strategy:
fail-fast: false
matrix:
flavour: ${{ fromJSON(needs.plan.outputs.flavours) }}
steps:
- uses: actions/checkout@v4
#- name: Install build tools
# run: sudo dnf install -y rpm-build rpmdevtools systemd-rpm-macros
- name: Download binary
uses: actions/download-artifact@v3
with:
name: mistralrs-server-${{ matrix.flavour.name }}
path: artifacts/
- name: Build RPM
run: |
rpmdev-setuptree
cp artifacts/mistralrs-server-${{ matrix.flavour.name }} ~/rpmbuild/SOURCES/
cp rpm/systemd/mistralrs@.service ~/rpmbuild/SOURCES/
cp rpm/systemd/mistralrs@.conf.example ~/rpmbuild/SOURCES/
rpmbuild -bb rpm/mistralrs.spec \
--define "mistralrs_version ${{ needs.plan.outputs.version }}" \
--define "mistralrs_flavour ${{ matrix.flavour.name }}"
- name: Upload RPM
uses: actions/upload-artifact@v3
with:
name: rpm-${{ matrix.flavour.name }}
path: ~/rpmbuild/RPMS/x86_64/*.rpm
retention-days: 7
publish:
needs: [plan, package]
runs-on: fedora
# concurrency ensures only one publish runs at a time — repo metadata
# corruption is a nightmare if two createrepo_c processes race.
concurrency:
group: rpm-publish
cancel-in-progress: false
steps:
- uses: actions/checkout@v4
#- name: Install tools
# run: sudo dnf install -y createrepo_c rpm-sign rsync
- name: Download all RPMs
uses: actions/download-artifact@v3
with:
path: rpms/
pattern: rpm-*
merge-multiple: true
- name: Import signing key
run: |
echo "${{ secrets.RPM_SIGNING_KEY }}" | gpg --batch --import
echo "%_gpg_name ${{ secrets.RPM_SIGNING_KEY_ID }}" > ~/.rpmmacros
- name: Sign and publish
run: ./script/publish-repo.sh rpms/
env:
RSYNC_TARGET: ${{ secrets.RSYNC_TARGET }}
RSYNC_SSH_KEY: ${{ secrets.RSYNC_SSH_KEY }}

View File

@@ -0,0 +1,43 @@
name: poll-upstream
on:
schedule:
- cron: "*/15 * * * *"
workflow_dispatch: {}
jobs:
check:
runs-on: fedora
steps:
- name: Get upstream latest tag
id: upstream
run: |
tag=$(curl -sSfL \
-H 'Accept: application/vnd.github+json' \
https://api.github.com/repos/EricLBuehler/mistral.rs/releases/latest \
| jq -r .tag_name)
echo "tag=${tag}" >> "$GITHUB_OUTPUT"
echo "Upstream latest: ${tag}"
- name: Get published version from our repo
id: published
run: |
# Query our own dnf repo. If the version is there, we've already built it.
# Strip leading 'v' because RPM versions don't use it.
version="${UPSTREAM_TAG#v}"
if curl -sSfI "https://rpm.lair.cafe/mistralrs/fedora-43/x86_64/mistralrs-server-cuda13-fa-${version}-1.fc43.x86_64.rpm" | grep -q '^HTTP.*200'; then
echo "already_built=true" >> "$GITHUB_OUTPUT"
else
echo "already_built=false" >> "$GITHUB_OUTPUT"
fi
env:
UPSTREAM_TAG: ${{ steps.upstream.outputs.tag }}
- name: Trigger build workflow
if: steps.published.outputs.already_built == 'false'
run: |
curl -sSfL -X POST \
-H "Authorization: token ${{ secrets.GITEA_TOKEN }}" \
-H 'Accept: application/json' \
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/actions/workflows/build-release.yml/dispatches" \
-d "{\"ref\":\"main\",\"inputs\":{\"tag\":\"${{ steps.upstream.outputs.tag }}\"}}"

55
CLAUDE.md Normal file
View File

@@ -0,0 +1,55 @@
# CLAUDE.md
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
## Purpose
This repo packages [mistral.rs](https://github.com/EricLBuehler/mistral.rs) (a Rust LLM inference server) into RPMs for Fedora 43 / x86_64. It does **not** contain the mistral.rs source — it clones upstream at a given tag, cross-compiles with CUDA, and produces signed RPMs published to a self-hosted dnf repo at `rpm.lair.cafe`.
## Architecture
### Pipeline flow
1. **poll-upstream** (`.gitea/workflows/poll-upstream.yml`) — cron every 15 min, checks GitHub for latest mistral.rs release tag. If the corresponding RPM doesn't exist on `rpm.lair.cafe`, triggers `build-release`.
2. **build-release** (`.gitea/workflows/build-release.yml`) — three-stage pipeline:
- **plan** — reads `flavours.yml`, emits a JSON matrix of flavours + stripped version.
- **build** — runs on a `cuda-13.0` runner. Clones upstream at tag, calls `script/build-binary.sh` to `cargo build --release --locked` with flavour-specific CUDA features.
- **package** — runs `rpmbuild -bb rpm/mistralrs.spec` with `--define` for version and flavour.
- **publish** — GPG-signs RPMs, rsyncs to `rpm.lair.cafe`, runs `createrepo_c --update`. Uses concurrency group `rpm-publish` to prevent metadata races.
### Flavours
Defined in `flavours.yml`. Each flavour specifies a name, `cuda_home`, `cargo_features`, and `compute_caps`. The RPM spec uses `update-alternatives` so multiple flavours can coexist, with priority: base=10, fa=20, nccl=30.
### Key files
- `flavours.yml` — flavour matrix definition (drives CI matrix)
- `rpm/mistralrs.spec` — RPM spec (binary-only package, no rebuild)
- `rpm/systemd/mistralrs@.service` — templated systemd unit (`@BINARY@` and `@FLAVOUR@` are sed-replaced during rpmbuild)
- `rpm/systemd/mistralrs@.conf.example` — example env file for instances
- `script/build-binary.sh` — compiles mistralrs-server with cargo (requires `FLAVOUR_NAME`, `CUDA_HOME`, `CARGO_FEATURES`, `CUDA_COMPUTE_CAP`, `SRC_DIR` env vars)
- `script/publish-repo.sh` — signs RPMs and rsyncs to the repo server
- `script/setup/` — one-time infra setup scripts (DNS, TLS cert, nginx) for `rpm.lair.cafe` on host `oolon`
## Commands
Build a binary locally (requires CUDA toolkit):
```bash
FLAVOUR_NAME=cuda13 CUDA_HOME=/usr/local/cuda-13.0 CARGO_FEATURES="cuda cudnn flash-attn nccl" CUDA_COMPUTE_CAP=120 SRC_DIR=./src ./script/build-binary.sh
```
Build an RPM from a pre-built binary:
```bash
rpmdev-setuptree
cp artifacts/mistralrs-server-cuda13 ~/rpmbuild/SOURCES/
cp rpm/systemd/mistralrs@.service ~/rpmbuild/SOURCES/
cp rpm/systemd/mistralrs@.conf.example ~/rpmbuild/SOURCES/
rpmbuild -bb rpm/mistralrs.spec --define "mistralrs_version 0.7.0" --define "mistralrs_flavour cuda13"
```
## Infrastructure
- CI runs on Gitea Actions (self-hosted), not GitHub Actions
- RPM repo hosted at `rpm.lair.cafe` on host `oolon.kosherinata.internal`
- TLS via Let's Encrypt with Cloudflare DNS challenge
- Publish uses rsync over SSH as `gitea_ci` user

View File

@@ -0,0 +1,36 @@
server {
server_name rpm.lair.cafe;
listen 443 ssl;
http2 on;
ssl_certificate /etc/letsencrypt/live/rpm.lair.cafe/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/rpm.lair.cafe/privkey.pem;
ssl_protocols TLSv1.2 TLSv1.3;
ssl_ecdh_curve X25519:secp256r1:secp384r1;
root /var/www/rpm;
autoindex on;
autoindex_exact_size off;
autoindex_localtime on;
types {
application/x-rpm rpm;
application/xml xml;
}
default_type application/octet-stream;
location ~ \.rpm$ {
expires 30d;
add_header Cache-Control "public, immutable";
}
location ~ /repodata/ {
expires -1;
add_header Cache-Control "no-cache, must-revalidate";
}
location = /RPM-GPG-KEY-mistralrs {
default_type text/plain;
}
}

5
flavours.yml Normal file
View File

@@ -0,0 +1,5 @@
flavours:
- name: cuda13
cuda_home: /usr/local/cuda-13.0
cargo_features: "cuda cudnn flash-attn nccl"
compute_caps: "120"

100
rpm/mistralrs.spec Normal file
View File

@@ -0,0 +1,100 @@
%global _build_id_links none
%global debug_package %{nil}
%global __strip /usr/bin/true
# Passed in via --define at rpmbuild time
%{!?mistralrs_version: %global mistralrs_version 0.7.0}
%{!?mistralrs_flavour: %global mistralrs_flavour cuda13}
Name: mistralrs-server-%{mistralrs_flavour}
Version: %{mistralrs_version}
Release: 1%{?dist}
Summary: Fast, flexible LLM inference server (mistral.rs, %{mistralrs_flavour} flavour)
License: MIT
URL: https://github.com/EricLBuehler/mistral.rs
# Pre-built binary (produced in the build job, not rebuilt here)
Source0: mistralrs-server-%{mistralrs_flavour}
Source1: mistralrs@.service
Source2: mistralrs@.conf.example
ExclusiveArch: x86_64
# Runtime requirements. We link against the CUDA runtime; consumers must have
# a matching CUDA installation or the rpmfusion nvidia driver's cuda-libs.
# We don't hard-require it at the RPM level because consumers may have CUDA
# from multiple sources (nvidia direct, rpmfusion, etc.) — failing to load
# libcuda.so at runtime gives a clearer error than RPM dep resolution would.
Requires: systemd
# Flavours are mutually exclusive with other flavours of themselves at the
# same install path, but you can install cuda13, cuda13-fa, cuda13-fa-nccl
# side by side — they all get separate /opt paths.
Provides: mistralrs-server = %{version}-%{release}
%description
mistral.rs is a blazingly fast LLM inference engine written in Rust.
This package provides the %{mistralrs_flavour} flavour, built with features:
cuda, cudnn, and optionally flash-attn and nccl depending on flavour name.
Binary installs to /opt/mistralrs/%{mistralrs_flavour}/bin/ and can coexist
with other flavours. Use `update-alternatives --config mistralrs-server` to
select the default /usr/bin/mistralrs-server symlink target.
%prep
# Nothing to unpack; Source0 is the binary itself
cp %{SOURCE0} .
cp %{SOURCE1} .
cp %{SOURCE2} .
%build
# Already built
%install
install -D -m 0755 mistralrs-server-%{mistralrs_flavour} \
%{buildroot}/opt/mistralrs/%{mistralrs_flavour}/bin/mistralrs-server
install -D -m 0644 mistralrs@.service \
%{buildroot}%{_unitdir}/mistralrs-%{mistralrs_flavour}@.service
install -D -m 0644 mistralrs@.conf.example \
%{buildroot}%{_sysconfdir}/mistralrs/%{mistralrs_flavour}.conf.example
# Patch the unit to point at this flavour's binary
sed -i "s|@BINARY@|/opt/mistralrs/%{mistralrs_flavour}/bin/mistralrs-server|g" \
%{buildroot}%{_unitdir}/mistralrs-%{mistralrs_flavour}@.service
sed -i "s|@FLAVOUR@|%{mistralrs_flavour}|g" \
%{buildroot}%{_unitdir}/mistralrs-%{mistralrs_flavour}@.service
%post
# Register this flavour as an alternative for /usr/bin/mistralrs-server.
# Priority = 10 for cuda13, 20 for cuda13-fa, 30 for cuda13-fa-nccl so that
# "more featureful" wins by default. Consumers can override with
# `update-alternatives --config mistralrs-server`.
priority=10
case "%{mistralrs_flavour}" in
*nccl*) priority=30 ;;
*fa*) priority=20 ;;
esac
update-alternatives --install /usr/bin/mistralrs-server mistralrs-server \
/opt/mistralrs/%{mistralrs_flavour}/bin/mistralrs-server "${priority}"
%systemd_post mistralrs-%{mistralrs_flavour}@.service
%preun
%systemd_preun mistralrs-%{mistralrs_flavour}@.service
%postun
if [ $1 -eq 0 ]; then
update-alternatives --remove mistralrs-server \
/opt/mistralrs/%{mistralrs_flavour}/bin/mistralrs-server
fi
%systemd_postun_with_restart mistralrs-%{mistralrs_flavour}@.service
%files
/opt/mistralrs/%{mistralrs_flavour}/
%{_unitdir}/mistralrs-%{mistralrs_flavour}@.service
%{_sysconfdir}/mistralrs/%{mistralrs_flavour}.conf.example
%changelog
* Thu Apr 23 2026 Robin Thijssen <grenade@lair.cafe> - %{mistralrs_version}-1
- Automated build for %{mistralrs_flavour} flavour

View File

@@ -0,0 +1,10 @@
# Configuration for a mistralrs instance.
# Copy to /etc/mistralrs/<instance>.conf and edit.
MISTRALRS_ARGS="--port 1234 plain -m openai/gpt-oss-20b --isq Q4K"
# HuggingFace token for gated models
# HF_TOKEN=hf_xxxx
# Where model weights are cached
HF_HOME=/var/cache/mistralrs/hf

View File

@@ -0,0 +1,29 @@
[Unit]
Description=mistral.rs inference server (@FLAVOUR@, instance %i)
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
User=mistralrs
Group=mistralrs
SupplementaryGroups=video render
EnvironmentFile=/etc/mistralrs/%i.conf
ExecStart=@BINARY@ $MISTRALRS_ARGS
Restart=on-failure
RestartSec=10s
NoNewPrivileges=yes
ProtectSystem=strict
ProtectHome=yes
ReadWritePaths=/var/lib/mistralrs /var/cache/mistralrs
PrivateTmp=yes
ProtectKernelTunables=yes
ProtectKernelModules=yes
ProtectControlGroups=yes
StateDirectory=mistralrs
CacheDirectory=mistralrs
[Install]
WantedBy=multi-user.target

25
script/build-binary.sh Executable file
View File

@@ -0,0 +1,25 @@
#!/usr/bin/env bash
set -euo pipefail
: "${FLAVOUR_NAME:?}"
: "${CUDA_HOME:?}"
: "${CARGO_FEATURES:?}"
: "${CUDA_COMPUTE_CAP:?}"
: "${SRC_DIR:?}"
export PATH="${CUDA_HOME}/bin:${PATH}"
export LD_LIBRARY_PATH="${CUDA_HOME}/targets/x86_64-linux/lib:${CUDA_HOME}/lib64:${LD_LIBRARY_PATH:-}"
cd "${SRC_DIR}"
# --locked ensures Cargo.lock is respected; fails loud if it's out of sync
# rather than silently resolving to different versions.
cargo build --release --locked --features "${CARGO_FEATURES}"
mkdir -p ../artifacts
cp target/release/mistralrs-server "../artifacts/mistralrs-server-${FLAVOUR_NAME}"
# Also grab the other binaries if you want them
cp target/release/mistralrs "../artifacts/mistralrs-${FLAVOUR_NAME}" 2>/dev/null || true
echo "Built $(../artifacts/mistralrs-server-${FLAVOUR_NAME} --version 2>&1 | head -1)"

24
script/publish-repo.sh Executable file
View File

@@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -euo pipefail
RPM_DIR="${1:?usage: $0 <rpm-directory>}"
REMOTE_DIR="/var/www/rpm/mistralrs/fedora-43/x86_64"
# sign each rpm with the imported gpg key
for rpm in "${RPM_DIR}"/*.rpm; do
rpm --addsign "${rpm}"
done
install --directory --mode 700 ~/.ssh
echo "${RSYNC_SSH_KEY}" | install --mode 600 /dev/stdin ~/.ssh/id_ed25519
ssh-keyscan -H oolon.kosherinata.internal > ~/.ssh/known_hosts 2>/dev/null
rsync \
--archive \
--verbose \
--chmod D755,F644 \
"${RPM_DIR}/"*.rpm \
"${RSYNC_TARGET}:${REMOTE_DIR}/"
ssh "${RSYNC_TARGET}" "cd ${REMOTE_DIR} && createrepo_c --update ."
echo "Published $(ls ${RPM_DIR}/*.rpm | wc -l) RPMs"

17
script/setup/cert.sh Executable file
View File

@@ -0,0 +1,17 @@
#!/usr/bin/env bash
tld=lair.cafe
fqdn=rpm.${tld}
sudo certbot certonly \
-m ops@${tld} \
--agree-tos \
--no-eff-email \
--noninteractive \
--cert-name ${fqdn} \
--expand \
--allow-subset-of-names \
--key-type ecdsa \
--dns-cloudflare \
--dns-cloudflare-credentials /root/.cloudflare/${tld} \
--dns-cloudflare-propagation-seconds 60 \
-d ${fqdn}

44
script/setup/dns.sh Executable file
View File

@@ -0,0 +1,44 @@
#!/usr/bin/env bash
cloudflare_api_token=$(cat ~/.cloudflare/lair.cafe | cut -d ' ' -f 3)
cloudflare_dns_zone_name=lair.cafe
cloudflare_dns_record_name=rpm.${cloudflare_dns_zone_name}
cloudflare_dns_record_type=CNAME
cloudflare_dns_record_content=bl.thgttg.com
cloudflare_dns_zone_id=$(curl \
--silent \
--request GET \
--header "Content-Type: application/json" \
--header "Authorization: Bearer ${cloudflare_api_token}" \
--url "https://api.cloudflare.com/client/v4/zones?name=${cloudflare_dns_zone_name}&status=active" \
| jq -r '.result[0].id//empty')
if [ -z ${cloudflare_dns_zone_id} ]; then
echo "cloudflare dns zone not found"
exit 1
else
echo "cloudflare dns zone found: ${cloudflare_dns_zone_name} (${cloudflare_dns_zone_id})"
fi
cloudflare_dns_record_id=$(curl \
--silent \
--request GET \
--header "Content-Type: application/json" \
--header "Authorization: Bearer ${cloudflare_api_token}" \
--url "https://api.cloudflare.com/client/v4/zones/${cloudflare_dns_zone_id}/dns_records?type=${cloudflare_dns_record_type}&name=${cloudflare_dns_record_name}" \
| jq -r '.result[0].id//empty')
if [ -z ${cloudflare_dns_record_id} ] && curl \
--silent \
--request POST \
--header "Content-Type: application/json" \
--header "Authorization: Bearer ${cloudflare_api_token}" \
--data "{\"type\":\"${cloudflare_dns_record_type}\",\"name\":\"${cloudflare_dns_record_name}\",\"content\":\"${cloudflare_dns_record_content}\",\"ttl\":1,\"proxied\":false}" \
--url "https://api.cloudflare.com/client/v4/zones/${cloudflare_dns_zone_id}/dns_records"; then
echo "${cloudflare_dns_record_name} ${cloudflare_dns_record_type} record created with content: ${cloudflare_dns_record_content} in zone: ${cloudflare_dns_zone_name} (${cloudflare_dns_zone_id}), record: ${cloudflare_dns_record_name} (${cloudflare_dns_record_id})"
elif curl \
--silent \
--request PUT \
--header "Content-Type: application/json" \
--header "Authorization: Bearer ${cloudflare_api_token}" \
--data "{\"type\":\"${cloudflare_dns_record_type}\",\"name\":\"${cloudflare_dns_record_name}\",\"content\":\"${cloudflare_dns_record_content}\",\"ttl\":1,\"proxied\":false}" \
--url "https://api.cloudflare.com/client/v4/zones/${cloudflare_dns_zone_id}/dns_records/${cloudflare_dns_record_id}"; then
echo "${cloudflare_dns_record_name} ${cloudflare_dns_record_type} record updated with content: ${cloudflare_dns_record_content} in zone: ${cloudflare_dns_zone_name} (${cloudflare_dns_zone_id}), record: ${cloudflare_dns_record_name} (${cloudflare_dns_record_id})"
fi

48
script/setup/nginx.sh Executable file
View File

@@ -0,0 +1,48 @@
#!/usr/bin/env bash
script_dir="$(dirname "$0")"
nginx_conf_local_path="${script_dir}/../../asset/nginx/rpm.lair.cafe.conf"
nginx_conf_remote_path="/etc/nginx/sites-available/rpm.lair.cafe.conf"
nginx_host=oolon
if [ ! -s ~/.ssh/id_gitea_ci.pub ]; then
echo "gitea_ci ssh key not found in ~/.ssh/id_gitea_ci.pub"
exit 1
fi
gitea_ssh_key=$(cat ~/.ssh/id_gitea_ci.pub)
if rsync \
--archive \
--compress \
--verbose \
${nginx_conf_local_path} \
${nginx_host}:${nginx_conf_remote_path}; then
echo "sync'd ${nginx_conf_local_path} to ${nginx_host}:${nginx_conf_remote_path}"
else
echo "failed to sync ${nginx_conf_local_path} to ${nginx_host}:${nginx_conf_remote_path}"
exit 1
fi
if ssh ${nginx_host} "id gitea_ci &> /dev/null || sudo useradd --system --create-home --home-dir /var/lib/gitea_ci gitea_ci"; then
echo "gitea_ci user created or observed on ${nginx_host}"
if ssh ${nginx_host} "sudo --user gitea_ci install --directory --mode 0700 /var/lib/gitea_ci/.ssh && echo '${gitea_ssh_key}' | sudo --user gitea_ci install --mode 0600 /dev/stdin /var/lib/gitea_ci/.ssh/authorized_keys"; then
echo "gitea_ci ssh key installed on ${nginx_host}"
else
echo "failed to install gitea_ci ssh key on ${nginx_host}"
exit 1
fi
else
echo "failed to create or observe gitea_ci user on ${nginx_host}"
exit 1
fi
if ssh ${nginx_host} "sudo install --directory /var/www/rpm && sudo setfacl -R -m u:gitea_ci:rwx /var/www/rpm/ && sudo chcon -Rt httpd_sys_content_t /var/www/rpm/"; then
echo "rpm repo directory created and permissions set on ${nginx_host}"
else
echo "failed to create rpm repo directory on ${nginx_host}"
exit 1
fi
if ssh ${nginx_host} "sudo ln -sf ${nginx_conf_remote_path} ${nginx_conf_remote_path/available/enabled} && sudo nginx -t ${nginx_conf_remote_path} && sudo systemctl reload nginx"; then
echo "nginx config reload on ${nginx_host} successful"
else
echo "nginx config reload on ${nginx_host} failed"
exit 1
fi