test(neuron): C2 — guard Responses→chat image translation contract

The Responses request translator already emits the chat `image_url`
Parts array Stage B5's vision path consumes, and the non-streaming
(`chat_completion`) and streaming (`responses_stream` → `inference_stream`,
Stage C1) Responses paths both route image content to the vision-aware
prefill — so vision works end-to-end through `/v1/responses` with no
translator change required.

Add a multi-image test asserting order preservation and that the
`detail` hint is tolerated (and dropped, since chat image_url has no
analogue), locking the translator's output to the exact
`image_url.url` shape `extract_images_from_request` walks.

Closes part of #16 (Stage C2).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.gitea/workflows/deploy.yml

@@ -70,6 +70,16 @@ jobs:
             sudo /usr/bin/systemctl daemon-reload
             sudo /usr/bin/systemctl start cortex.service'
 
+      # Wait for the service to either come up or wedge, then capture
+      # the latest-invocation journal. Runs even on prior failure so a
+      # failed start step still leaves a usable record in the deploy log.
+      - name: Capture cortex.service startup journal
+        if: always()
+        run: |
+          sleep 10
+          ssh gitea_ci@hanzalova.internal \
+              'journalctl --unit cortex.service -I --no-pager'
+
   deploy-neurons:
     needs: [deploy-cortex]
     runs-on: fedora-43
@@ -124,3 +134,13 @@ jobs:
           ssh gitea_ci@${{ matrix.host }} '
             sudo /usr/bin/systemctl daemon-reload
             sudo /usr/bin/systemctl start neuron.service'
+
+      # Wait for the service to either come up or wedge, then capture
+      # the latest-invocation journal. Runs even on prior failure so a
+      # failed start step still leaves a usable record in the deploy log.
+      - name: Capture neuron.service startup journal
+        if: always()
+        run: |
+          sleep 10
+          ssh gitea_ci@${{ matrix.host }} \
+              'journalctl --unit neuron.service -I --no-pager'

asset/manifest.yml

@@ -1,30 +0,0 @@
-# Helexa fleet manifest.
-#
-# Drives rolling deploys via script/deploy.sh and serves as the source
-# of truth for which hosts run cortex vs neuron, and which CUDA
-# compute-capability flavour each neuron host needs.
-#
-# Flavour ↔ NVIDIA generation ↔ compute cap:
-#   ampere    sm_86   (RTX 30 series — e.g. 3060)
-#   ada       sm_89   (RTX 40 series — e.g. 4090)
-#   blackwell sm_120  (RTX 50 series — e.g. 5090)
-#
-# The flavour determines which RPM is installed on a given neuron host:
-# helexa-neuron-<flavour>. Only one flavour may be installed at a time
-# (the packages Conflict: with each other).
-
-cortex:
-  host: hanzalova.internal
-
-neurons:
-  - host: beast.hanzalova.internal
-    flavour: blackwell
-    gpu: "2x RTX 5090"
-
-  - host: benjy.hanzalova.internal
-    flavour: ada
-    gpu: "RTX 4090"
-
-  - host: quadbrat.hanzalova.internal
-    flavour: ampere
-    gpu: "RTX 3060"

asset/neuron/beast.toml

@@ -5,9 +5,9 @@
 # invocation: `validate-neuron.sh beast.hanzalova.internal
 # Qwen/Qwen3.6-27B q5k 2`.
 #
-# Synced by script/deploy.sh from asset/neuron/<short-host>.toml. Edits
-# take effect on the next deploy.sh run (which stops + restarts the
-# service so default_models is re-read at activation).
+# Synced to /etc/neuron/neuron.toml by script/infra-setup.sh. Edits
+# take effect after the next deploy workflow run restarts the service
+# (default_models is read at activation).
 
 port = 13131
 

asset/neuron/benjy.toml

@@ -4,7 +4,7 @@
 # Qwen3-8B (bf16, ~18 GB), leaving ~6 GB for KV cache + activations on
 # moderate-length contexts.
 #
-# Synced by script/deploy.sh from asset/neuron/<short-host>.toml.
+# Synced to /etc/neuron/neuron.toml by script/infra-setup.sh.
 
 port = 13131
 

asset/neuron/quadbrat.toml

@@ -4,7 +4,7 @@
 # (bf16, ~4 GB), leaving ~7 GB for KV cache so long contexts on a small
 # model still have plenty of room.
 #
-# Synced by script/deploy.sh from asset/neuron/<short-host>.toml.
+# Synced to /etc/neuron/neuron.toml by script/infra-setup.sh.
 
 port = 13131
 

crates/cortex-core/src/node.rs

@@ -37,6 +37,12 @@ pub struct ModelEntry {
     pub last_accessed: Option<DateTime<Utc>>,
     /// Estimated VRAM usage in MB when loaded.
     pub vram_estimate_mb: Option<u64>,
+    /// Modalities the loaded model advertises (e.g. `["text", "vision"]`),
+    /// copied verbatim from the neuron's `ModelInfo.capabilities` at poll
+    /// time. Empty when the neuron reports none. `#[serde(default)]` keeps
+    /// older persisted/serialised entries deserialisable.
+    #[serde(default)]
+    pub capabilities: Vec<String>,
 }
 
 /// Model lifecycle status.
@@ -85,6 +91,12 @@ pub struct CortexModelEntry {
     /// disjoint from) `feasible_on` depending on whether the catalogue
     /// covers this model.
     pub locations: Vec<ModelLocation>,
+    /// Union of the modalities advertised by every neuron that has this
+    /// model loaded (e.g. `["text", "vision"]`). Empty for catalogue-only
+    /// entries with no loaded location — the catalogue profile doesn't
+    /// declare capabilities yet (tracked separately from C3).
+    #[serde(default)]
+    pub capabilities: Vec<String>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

crates/cortex-gateway/src/handlers.rs

@@ -414,6 +414,9 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
                 loaded: false,
                 feasible_on,
                 locations: Vec::new(),
+                // Catalogue profiles don't declare capabilities yet;
+                // the union is filled in Pass 2 from loaded locations.
+                capabilities: Vec::new(),
             },
         );
     }
@@ -438,6 +441,14 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
                     if was_loaded {
                         e.loaded = true;
                     }
+                    // Union the per-node capabilities so a model loaded
+                    // on several neurons reports every modality any of
+                    // them advertises.
+                    for cap in &entry.capabilities {
+                        if !e.capabilities.contains(cap) {
+                            e.capabilities.push(cap.clone());
+                        }
+                    }
                 })
                 .or_insert_with(|| CortexModelEntry {
                     id: model_id.clone(),
@@ -449,6 +460,7 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
                     // feasibility; leave empty.
                     feasible_on: Vec::new(),
                     locations: vec![location],
+                    capabilities: entry.capabilities.clone(),
                 });
         }
     }
@@ -498,6 +510,9 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
                     loaded: false,
                     feasible_on: Vec::new(),
                     locations: vec![location],
+                    // A model that's only mid-prewarm has no loaded
+                    // location to read capabilities from yet.
+                    capabilities: Vec::new(),
                 });
         }
     }
@@ -527,6 +542,7 @@ async fn list_models(State(fleet): State<Arc<CortexState>>) -> Json<Value> {
                 loaded: target_entry.loaded,
                 feasible_on: target_entry.feasible_on,
                 locations: target_entry.locations,
+                capabilities: target_entry.capabilities,
             },
         );
     }

crates/cortex-gateway/src/poller.rs

@@ -107,12 +107,14 @@ async fn poll_neuron(fleet: &CortexState, name: &str, endpoint: &str) {
                             .and_modify(|e| {
                                 e.status = status;
                                 e.vram_estimate_mb = upstream.vram_used_mb;
+                                e.capabilities = upstream.capabilities.clone();
                             })
                             .or_insert_with(|| ModelEntry {
                                 id: upstream.id.clone(),
                                 status,
                                 last_accessed: None,
                                 vram_estimate_mb: upstream.vram_used_mb,
+                                capabilities: upstream.capabilities.clone(),
                             });
                     }
 

crates/cortex-gateway/src/router.rs

@@ -244,6 +244,7 @@ async fn cold_load(
                     status: ModelStatus::Loaded,
                     last_accessed: Some(chrono::Utc::now()),
                     vram_estimate_mb: profile.vram_mb,
+                    capabilities: Vec::new(),
                 },
             );
         }

crates/cortex-gateway/tests/aliases.rs

@@ -74,6 +74,7 @@ async fn test_alias_resolves_in_chat_completions() {
                 status: ModelStatus::Loaded,
                 last_accessed: None,
                 vram_estimate_mb: None,
+                capabilities: Vec::new(),
             },
         );
     }
@@ -154,6 +155,7 @@ async fn test_aliases_surface_in_v1_models() {
                 status: ModelStatus::Loaded,
                 last_accessed: None,
                 vram_estimate_mb: Some(2000),
+                capabilities: Vec::new(),
             },
         );
     }
@@ -235,6 +237,7 @@ async fn test_alias_falls_through_for_unmapped_model() {
                 status: ModelStatus::Loaded,
                 last_accessed: None,
                 vram_estimate_mb: None,
+                capabilities: Vec::new(),
             },
         );
     }

crates/cortex-gateway/tests/common/mod.rs

@@ -305,6 +305,7 @@ pub async fn spawn_gateway_with_state(mock_url: &str) -> (Arc<CortexState>, Stri
                 status: ModelStatus::Loaded,
                 last_accessed: None,
                 vram_estimate_mb: Some(8000),
+                capabilities: Vec::new(),
             },
         );
     }

crates/cortex-gateway/tests/eviction.rs

@@ -91,6 +91,7 @@ async fn test_evict_lru_model() {
                 status: ModelStatus::Loaded,
                 last_accessed: Some(Utc::now() - chrono::Duration::hours(2)),
                 vram_estimate_mb: Some(8000),
+                capabilities: Vec::new(),
             },
         );
         node.models.insert(
@@ -100,6 +101,7 @@ async fn test_evict_lru_model() {
                 status: ModelStatus::Loaded,
                 last_accessed: Some(Utc::now()),
                 vram_estimate_mb: Some(8000),
+                capabilities: Vec::new(),
             },
         );
     }
@@ -163,6 +165,7 @@ async fn test_eviction_increments_lifecycle_cycles() {
                 status: ModelStatus::Loaded,
                 last_accessed: None,
                 vram_estimate_mb: None,
+                capabilities: Vec::new(),
             },
         );
     }

crates/cortex-gateway/tests/poller.rs

@@ -118,6 +118,87 @@ async fn test_poller_updates_gateway_models_endpoint() {
     }
 }
 
+#[tokio::test]
+async fn test_models_endpoint_unions_capabilities_across_nodes() {
+    // C3: two neurons each have the same model loaded but advertise
+    // different capability sets. The gateway's /v1/models must report
+    // the union — a model loaded text-only on one node and
+    // text+vision on another is vision-capable to the fleet.
+    let node_a = common::spawn_mock_neuron_with_models(json!([
+        {"id": "shared-model", "harness": "candle", "status": "loaded", "devices": [0], "vram_used_mb": null, "capabilities": ["text"]}
+    ]))
+    .await;
+    let node_b = common::spawn_mock_neuron_with_models(json!([
+        {"id": "shared-model", "harness": "candle", "status": "loaded", "devices": [1], "vram_used_mb": null, "capabilities": ["text", "vision"]}
+    ]))
+    .await;
+
+    let config = GatewayConfig {
+        gateway: GatewaySettings {
+            listen: "127.0.0.1:0".into(),
+            metrics_listen: "127.0.0.1:0".into(),
+        },
+        eviction: EvictionSettings {
+            strategy: EvictionStrategy::Lru,
+            defrag_after_cycles: 0,
+        },
+        neurons: vec![
+            NeuronEndpoint {
+                name: "node-a".into(),
+                endpoint: node_a,
+            },
+            NeuronEndpoint {
+                name: "node-b".into(),
+                endpoint: node_b,
+            },
+        ],
+        models_config: "/dev/null".into(),
+    };
+
+    let fleet = Arc::new(CortexState::from_config(&config));
+    cortex_gateway::poller::poll_once(&fleet).await;
+
+    let app = cortex_gateway::build_app(Arc::clone(&fleet));
+    let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+    let addr = listener.local_addr().unwrap();
+    tokio::spawn(async move {
+        axum::serve(listener, app).await.unwrap();
+    });
+
+    let client = reqwest::Client::new();
+    let body: serde_json::Value = client
+        .get(format!("http://{addr}/v1/models"))
+        .send()
+        .await
+        .expect("request should succeed")
+        .json()
+        .await
+        .unwrap();
+
+    let model = body["data"]
+        .as_array()
+        .expect("data array")
+        .iter()
+        .find(|m| m["id"] == "shared-model")
+        .expect("shared-model should be present");
+
+    let caps: Vec<&str> = model["capabilities"]
+        .as_array()
+        .expect("capabilities array")
+        .iter()
+        .filter_map(|c| c.as_str())
+        .collect();
+    assert!(caps.contains(&"text"), "union must include text: {caps:?}");
+    assert!(
+        caps.contains(&"vision"),
+        "union must include vision: {caps:?}"
+    );
+    assert_eq!(caps.len(), 2, "union must not duplicate text: {caps:?}");
+
+    // Both nodes hold the model, so two locations regardless of caps.
+    assert_eq!(model["locations"].as_array().unwrap().len(), 2);
+}
+
 #[tokio::test]
 async fn test_poller_marks_unreachable_node_unhealthy() {
     let config = GatewayConfig {
@@ -216,6 +297,7 @@ async fn test_poller_removes_stale_models() {
                 status: ModelStatus::Loaded,
                 last_accessed: None,
                 vram_estimate_mb: None,
+                capabilities: Vec::new(),
             },
         );
         node.models.insert(
@@ -225,6 +307,7 @@ async fn test_poller_removes_stale_models() {
                 status: ModelStatus::Loaded,
                 last_accessed: None,
                 vram_estimate_mb: None,
+                capabilities: Vec::new(),
             },
         );
     }

crates/neuron/src/harness/candle.rs

@@ -1981,7 +1981,50 @@ impl CandleHarness {
             .tokenizer
             .encode(prompt.as_str(), true)
             .map_err(|e| InferenceError::Other(anyhow::anyhow!("tokenize: {e}")))?;
-        let prompt_tokens: Vec<u32> = encoding.get_ids().to_vec();
+        let mut prompt_tokens: Vec<u32> = encoding.get_ids().to_vec();
+
+        // Stage C1: vision routing for the streaming path. Mirrors the
+        // non-streaming `chat_completion` block — detect image content,
+        // reject it against text-only models, preprocess each image and
+        // expand its `<|image_pad|>` sentinel to the per-image patch
+        // count, then carry the payload through to a single-shot
+        // image-spliced prefill. Non-image requests skip all of this.
+        // Returning early here (before the `Start` event below) keeps a
+        // rejected vision request from opening a half-formed SSE stream.
+        let vision_route: Option<(Vec<super::device_worker::jobs::ImageInput>, u32)> =
+            if request_has_images(&request) {
+                if !loaded.has_vision {
+                    return Err(InferenceError::VisionUnsupported {
+                        model_id: request.model.clone(),
+                    });
+                }
+                let image_token_id =
+                    loaded
+                        .image_token_id
+                        .ok_or_else(|| InferenceError::VisionUnsupported {
+                            model_id: request.model.clone(),
+                        })?;
+                let patches_per_image = loaded.lm_tokens_per_image.ok_or_else(|| {
+                    InferenceError::VisionUnsupported {
+                        model_id: request.model.clone(),
+                    }
+                })?;
+                let profile = super::preprocess::PreprocessProfile::qwen3_6();
+                let images = extract_images_from_request(&request, &profile)
+                    .map_err(|e| InferenceError::Other(anyhow::anyhow!("extract_images: {e}")))?;
+                if images.is_empty() {
+                    return Err(InferenceError::Other(anyhow::anyhow!(
+                        "request has image content but extractor produced zero images"
+                    )));
+                }
+                let per_image_counts: Vec<usize> = vec![patches_per_image; images.len()];
+                prompt_tokens =
+                    expand_image_pad_tokens(&prompt_tokens, image_token_id, &per_image_counts)
+                        .map_err(InferenceError::Other)?;
+                Some((images, image_token_id))
+            } else {
+                None
+            };
 
         let temperature = request.temperature.unwrap_or(0.7);
         let top_p = request.top_p;
@@ -2048,6 +2091,7 @@ impl CandleHarness {
                 ?eos_id,
                 vram_free_mb,
                 vram_total_mb,
+                vision = vision_route.is_some(),
                 "chat_completion (stream): starting"
             );
         }
@@ -2078,6 +2122,7 @@ impl CandleHarness {
                             handle,
                             tokenizer,
                             prompt_tokens,
+                            vision_route,
                             max_new,
                             temperature,
                             top_p,
@@ -4046,6 +4091,17 @@ async fn run_inference_via_worker(
 /// forward step through `worker.forward_logits()`. Same per-step
 /// CPU-side sampling discipline — no device tensor escapes the
 /// worker thread.
+///
+/// `images` carries the Stage C vision payload. When `Some`, prefill
+/// is a single-shot `forward_logits_with_images` that splices image
+/// embeddings at `image_token_id` positions (same contract as the
+/// non-streaming [`run_inference_with_images_via_worker`]); image
+/// embeddings are prefill-only, so every decode step below takes the
+/// plain `forward_logits` path regardless. When `None`, prefill is
+/// chunked (`chunked_prefill_via_worker`) to bound activation memory
+/// — the original text-only behaviour, unchanged. The decode loop and
+/// the `route_token!` reasoning/tool-call state machine are shared
+/// across both prefill shapes, so there's exactly one copy to maintain.
 #[cfg(feature = "cuda")]
 #[allow(clippy::too_many_arguments)]
 async fn stream_inference_via_worker(
@@ -4053,6 +4109,7 @@ async fn stream_inference_via_worker(
     handle: super::device_worker::ArchHandle,
     tokenizer: Tokenizer,
     prompt_tokens: Vec<u32>,
+    images: Option<(Vec<super::device_worker::jobs::ImageInput>, u32)>,
     max_new: usize,
     temperature: f64,
     top_p: Option<f64>,
@@ -4098,11 +4155,19 @@ async fn stream_inference_via_worker(
         .await
         .map_err(|e| anyhow::anyhow!("clear_kv_cache: {e}"))?;
 
-    // Chunked prefill (see `chunked_prefill_via_worker`). The owning
-    // `prompt_tokens: Vec<u32>` is borrowed for the loop's duration;
-    // we still need `prompt_len` (already extracted above) for the
-    // decode-step offset arithmetic.
-    let logits_vec = chunked_prefill_via_worker(&*worker, handle, &prompt_tokens).await?;
+    // Prefill. Vision-bearing requests (`images = Some`) do a
+    // single-shot prefill that splices the image embeddings; text-only
+    // requests use chunked prefill (see `chunked_prefill_via_worker`)
+    // to bound activation memory. Either way the owning
+    // `prompt_tokens: Vec<u32>` outlives this step; we use `prompt_len`
+    // (already extracted above) for the decode-step offset arithmetic.
+    let logits_vec = match images {
+        Some((imgs, image_token_id)) => worker
+            .forward_logits_with_images(handle, prompt_tokens.clone(), 0, imgs, image_token_id)
+            .await
+            .map_err(|e| anyhow::anyhow!("forward_logits_with_images: {e}"))?,
+        None => chunked_prefill_via_worker(&*worker, handle, &prompt_tokens).await?,
+    };
     let logits = Tensor::new(logits_vec.as_slice(), &Device::Cpu)?;
     let mut next_token = match sample_with_penalty(&logits, &all_tokens, &mut logits_processor) {
         Ok(t) => t,
@@ -4699,4 +4764,39 @@ mod tests {
         let out = expand_image_pad_tokens(&input, pad, &[]).unwrap();
         assert_eq!(out, input);
     }
+
+    /// `request_has_images` is the gate that routes both the
+    /// non-streaming (`chat_completion`) and streaming
+    /// (`inference_stream`, Stage C1) paths to the vision-aware
+    /// prefill. Exercise the three shapes it must distinguish: plain
+    /// text, a text-only content-parts array, and a parts array
+    /// carrying an `image_url`.
+    #[test]
+    fn request_has_images_detects_image_url_parts() {
+        let text_only: ChatCompletionRequest = serde_json::from_value(serde_json::json!({
+            "model": "m",
+            "messages": [{"role": "user", "content": "hello"}],
+        }))
+        .unwrap();
+        assert!(!request_has_images(&text_only));
+
+        let parts_text_only: ChatCompletionRequest = serde_json::from_value(serde_json::json!({
+            "model": "m",
+            "messages": [{"role": "user", "content": [
+                {"type": "text", "text": "hello"}
+            ]}],
+        }))
+        .unwrap();
+        assert!(!request_has_images(&parts_text_only));
+
+        let with_image: ChatCompletionRequest = serde_json::from_value(serde_json::json!({
+            "model": "m",
+            "messages": [{"role": "user", "content": [
+                {"type": "text", "text": "what is this?"},
+                {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAA="}}
+            ]}],
+        }))
+        .unwrap();
+        assert!(request_has_images(&with_image));
+    }
 }

crates/neuron/src/wire/openai_responses.rs

@@ -646,6 +646,54 @@ mod tests {
         assert_eq!(parts[1]["image_url"]["url"], "data:image/png;base64,AAA=");
     }
 
+    #[test]
+    fn multiple_images_translate_in_order_and_tolerate_detail() {
+        // C2: a Responses request carrying several InputImage parts
+        // (with `detail` set) must translate to a chat Parts array that
+        // preserves image order and the `image_url.url` shape the chat
+        // vision path (`extract_images_from_request`) walks. The
+        // `detail` hint has no chat-completions analogue we forward, so
+        // it's dropped — but it must not break translation.
+        let req = ResponsesRequest {
+            model: "m".into(),
+            input: ResponsesInput::Items(vec![ResponsesInputItem::Message {
+                role: "user".into(),
+                content: ResponsesMessageContent::Parts(vec![
+                    ResponsesContentPart::InputText {
+                        text: "compare these".into(),
+                    },
+                    ResponsesContentPart::InputImage {
+                        image_url: "data:image/png;base64,FIRST".into(),
+                        detail: Some("high".into()),
+                    },
+                    ResponsesContentPart::InputImage {
+                        image_url: "data:image/png;base64,SECOND".into(),
+                        detail: None,
+                    },
+                ]),
+            }]),
+            instructions: None,
+            stream: false,
+            max_output_tokens: None,
+            temperature: None,
+            top_p: None,
+            previous_response_id: None,
+            extra: Value::Object(Default::default()),
+        };
+        let chat = request_to_chat(req).unwrap();
+        let parts = match &chat.messages[0].content {
+            MessageContent::Parts(p) => p.clone(),
+            other => panic!("expected Parts, got {other:?}"),
+        };
+        // text + two images, in input order.
+        assert_eq!(parts.len(), 3);
+        assert_eq!(parts[0]["type"], "text");
+        assert_eq!(parts[1]["image_url"]["url"], "data:image/png;base64,FIRST");
+        assert_eq!(parts[2]["image_url"]["url"], "data:image/png;base64,SECOND");
+        // `detail` is not forwarded into the chat image_url object.
+        assert!(parts[1]["image_url"].get("detail").is_none());
+    }
+
     #[test]
     fn text_only_parts_collapse_to_string() {
         let req = ResponsesRequest {

script/deploy.sh

@@ -1,303 +0,0 @@
-#!/bin/env bash
-#
-# Rolling deploy across the helexa fleet, driven by asset/manifest.yml.
-# Installs / upgrades cortex on the gateway host and the appropriate
-# helexa-neuron-<flavour> package on each neuron host, then restarts
-# their services.
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-REPO_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
-MANIFEST="${REPO_DIR}/asset/manifest.yml"
-
-if [[ ! -f "${MANIFEST}" ]]; then
-    echo "fatal: manifest not found at ${MANIFEST}" >&2
-    exit 1
-fi
-
-# Parse the manifest with yq. NOTE: this expects the pip-installed yq
-# (a jq wrapper using jq syntax) — `pip install yq`. The Fedora rpm
-# `yq` is mikefarah/yq and uses different (yaml-native) syntax; if a
-# host has that one instead these queries will fail.
-cortex_host=$(yq -r '.cortex.host' "${MANIFEST}")
-
-# Emit one TAB-separated 'host\tflavour' line per neuron.
-mapfile -t neuron_entries < <(
-    yq -r '.neurons[] | .host + "\t" + .flavour' "${MANIFEST}"
-)
-
-# Return the installed package's "version-release" string, or
-# "(not installed)" when rpm reports the package as absent. Capture
-# rpm's output into a variable so its "package X is not installed"
-# stdout message (rpm writes that to stdout, not stderr, when -q fails)
-# doesn't leak into the result.
-installed_nvr() {
-    local host="$1" pkg="$2"
-    local nvr
-    if nvr=$(ssh "${host}" "rpm -q --qf '%{version}-%{release}' ${pkg} 2>/dev/null"); then
-        echo "${nvr}"
-    else
-        echo "(not installed)"
-    fi
-}
-
-# Ensure the rpm.lair.cafe unstable repo is configured AND enabled on
-# the remote host.
-#
-# The upstream .repo file at https://rpm.lair.cafe/lair-cafe-unstable.repo
-# ships with `enabled=0` so a host that just fetched it won't start
-# pulling unstable packages by accident. We have to explicitly flip
-# enabled=1 via `dnf config-manager setopt`. Both addrepo and setopt
-# are idempotent.
-#
-# Non-fatal — if either step fails the subsequent `dnf install` will
-# surface a clearer diagnostic on its own.
-ensure_lair_repo() {
-    local host="$1"
-    if ! ssh "${host}" "test -f /etc/yum.repos.d/lair-cafe-unstable.repo" 2>/dev/null; then
-        echo "[${host}] adding rpm.lair.cafe unstable repo"
-        if ! ssh "${host}" sudo dnf config-manager addrepo \
-            --from-repofile=https://rpm.lair.cafe/lair-cafe-unstable.repo \
-            >/dev/null 2>&1; then
-            echo "[${host}] WARNING: failed to add lair.cafe repo file (proceeding anyway)"
-            return 0
-        fi
-    fi
-    # The .repo file ships enabled=0; flip it on. Cheap, idempotent.
-    if ! ssh "${host}" sudo dnf config-manager setopt \
-        lair-cafe-unstable.enabled=1 >/dev/null 2>&1; then
-        echo "[${host}] WARNING: failed to enable lair-cafe-unstable (proceeding anyway)"
-    fi
-}
-
-# Ensure libcudnn.so.9 is resolvable on the remote host so the
-# neuron binary (built with --features cudnn) doesn't fail at startup
-# with "cannot open shared object file: No such file or directory".
-#
-# Probes ldconfig first — if cuDNN was installed manually (.tar/.run
-# install), it'll be cached by ldconfig and we don't touch it.
-# Otherwise adds NVIDIA's RHEL9 CUDA repo (the Fedora 43 CUDA repo
-# doesn't ship cuDNN packages — only the RHEL9 one does) and installs
-# libcudnn9-cuda-13.
-ensure_cudnn_runtime() {
-    local host="$1"
-    if ssh "${host}" "ldconfig -p | grep -q libcudnn.so.9" 2>/dev/null; then
-        return 0
-    fi
-    echo "[${host}] installing cuDNN runtime"
-    if ! ssh "${host}" "test -f /etc/yum.repos.d/cuda-rhel9-x86_64.repo" 2>/dev/null; then
-        if ! ssh "${host}" sudo dnf config-manager addrepo \
-            --from-repofile=https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo \
-            >/dev/null 2>&1; then
-            echo "[${host}] WARNING: failed to add rhel9 CUDA repo (proceeding anyway)"
-        fi
-    fi
-    if ! ssh "${host}" sudo dnf install -y libcudnn9-cuda-13 >/dev/null 2>&1; then
-        echo "[${host}] WARNING: failed to install libcudnn9-cuda-13"
-        echo "[${host}]   neuron may fail to start; install cuDNN manually if so"
-    fi
-}
-
-# True when the named package needs to be installed or upgraded on the
-# remote host — either it's not present, or a newer version exists in
-# the repo. False only when the installed version is current.
-#
-# `dnf check-update <pkg>` returns 0 when the package isn't installed
-# at all (there's nothing to update), so we have to probe with rpm -q
-# first to distinguish "absent" from "current". Other dnf failures
-# collapse into "needs update" so the subsequent install step surfaces
-# the real diagnostic rather than this check swallowing it.
-needs_update() {
-    local host="$1" pkg="$2"
-    # Not installed → needs work.
-    if ! ssh "${host}" "rpm -q ${pkg}" >/dev/null 2>&1; then
-        return 0
-    fi
-    # Installed; ask dnf whether the repo has something newer.
-    if ssh "${host}" sudo dnf check-update --refresh -q "${pkg}" >/dev/null 2>&1; then
-        return 1
-    else
-        return 0
-    fi
-}
-
-# True if the named package is currently installed on the remote host.
-# Used to decide between `dnf install` (fresh) and `dnf upgrade` (stale):
-# dnf5's `install` is a no-op when the package is already present at
-# any version — it does NOT auto-upgrade to the latest available — so
-# the wrong command silently leaves the host on an old build.
-is_installed() {
-    local host="$1" pkg="$2"
-    ssh "${host}" "rpm -q ${pkg}" >/dev/null 2>&1
-}
-
-# Install or upgrade the named package on the remote, picking the
-# right dnf verb based on the installed-or-not state. Returns 0 with
-# dnf's combined stdout/stderr captured in __DNF_OUTPUT__ on success,
-# and 1 with the same captured output on failure.
-__DNF_OUTPUT__=""
-install_or_upgrade() {
-    local host="$1" pkg="$2"
-    local cmd
-    if is_installed "${host}" "${pkg}"; then
-        cmd="upgrade"
-    else
-        cmd="install"
-    fi
-    if __DNF_OUTPUT__=$(
-        ssh "${host}" sudo dnf "${cmd}" --refresh --allowerasing -y "${pkg}" 2>&1
-    ); then
-        return 0
-    else
-        return 1
-    fi
-}
-
-# ---------------------------------------------------------------------------
-# cortex (gateway)
-# ---------------------------------------------------------------------------
-
-ensure_lair_repo "${cortex_host}"
-cortex_nvr=$(installed_nvr "${cortex_host}" cortex)
-if needs_update "${cortex_host}" cortex; then
-    echo "[${cortex_host}] cortex update available (current: ${cortex_nvr})"
-    # Stop the service only if the unit file exists — fresh installs
-    # don't have it, and `systemctl stop` on a missing unit returns
-    # non-zero, which would otherwise short-circuit the install branch
-    # under set -e.
-    if ssh "${cortex_host}" "[ ! -f /usr/lib/systemd/system/cortex.service ] || sudo systemctl stop cortex.service"; then
-        echo "[${cortex_host}] stopped cortex service"
-        if install_or_upgrade "${cortex_host}" cortex; then
-            cortex_nvr=$(installed_nvr "${cortex_host}" cortex)
-            echo "[${cortex_host}] installed/upgraded cortex to ${cortex_nvr}"
-        else
-            echo "[${cortex_host}] failed to install/upgrade cortex:"
-            echo "${__DNF_OUTPUT__}" | sed "s/^/[${cortex_host}]   /"
-        fi
-    else
-        echo "[${cortex_host}] failed to stop cortex service"
-    fi
-else
-    echo "[${cortex_host}] cortex is up to date (${cortex_nvr})"
-    ssh "${cortex_host}" sudo systemctl stop cortex.service || true
-fi
-
-# Sync cortex.toml whether the package was upgraded or not — the config
-# can change without a package bump.
-if rsync \
-    --archive \
-    --compress \
-    --rsync-path 'sudo rsync' \
-    --chown root:root \
-    --chmod 644 \
-    "${REPO_DIR}/cortex.toml" \
-    "${cortex_host}:/etc/cortex/cortex.toml"; then
-    echo "[${cortex_host}] sync'd cortex.toml"
-else
-    echo "[${cortex_host}] failed to sync cortex.toml"
-fi
-
-# Sync models.toml on the same lifecycle as cortex.toml — operator-owned,
-# gitignored, drives /v1/models catalogue × topology resolution.
-if [[ -f "${REPO_DIR}/models.toml" ]]; then
-    if rsync \
-        --archive \
-        --compress \
-        --rsync-path 'sudo rsync' \
-        --chown root:root \
-        --chmod 644 \
-        "${REPO_DIR}/models.toml" \
-        "${cortex_host}:/etc/cortex/models.toml"; then
-        echo "[${cortex_host}] sync'd models.toml"
-    else
-        echo "[${cortex_host}] failed to sync models.toml"
-    fi
-else
-    echo "[${cortex_host}] no local models.toml — leaving /etc/cortex/models.toml untouched"
-fi
-
-ssh "${cortex_host}" sudo systemctl daemon-reload
-if ssh "${cortex_host}" systemctl is-active --quiet cortex.service; then
-    echo "[${cortex_host}] cortex service is active"
-elif ssh "${cortex_host}" sudo systemctl start cortex.service; then
-    echo "[${cortex_host}] started cortex service"
-else
-    echo "[${cortex_host}] failed to start cortex service"
-fi
-
-# ---------------------------------------------------------------------------
-# neuron (per-host, flavour from manifest)
-# ---------------------------------------------------------------------------
-
-for entry in "${neuron_entries[@]}"; do
-    IFS=$'\t' read -r neuron_host neuron_flavour <<< "${entry}"
-    package="helexa-neuron-${neuron_flavour}"
-    # First dot-component of the host keys the per-host config file
-    # under asset/neuron/<short>.toml. A host listed in the manifest
-    # without a corresponding config still deploys (the package's
-    # default /etc/neuron/neuron.toml stays in place; no pre-warm).
-    short_host="${neuron_host%%.*}"
-    host_config="${REPO_DIR}/asset/neuron/${short_host}.toml"
-
-    ensure_lair_repo "${neuron_host}"
-    ensure_cudnn_runtime "${neuron_host}"
-    neuron_nvr=$(installed_nvr "${neuron_host}" "${package}")
-
-    # Stop the service unconditionally before any reconfig step.
-    # `default_models` is read at activation, so a config change without
-    # a bounce silently leaves the host on the previous pre-warm set.
-    # Same shape as the cortex flow above. The `[ ! -f … ]` guard skips
-    # the stop on a fresh install where the unit file isn't there yet.
-    if ssh "${neuron_host}" "[ ! -f /usr/lib/systemd/system/neuron.service ] || sudo systemctl stop neuron.service"; then
-        echo "[${neuron_host}] stopped neuron service"
-    else
-        echo "[${neuron_host}] failed to stop neuron service (continuing)"
-    fi
-
-    if needs_update "${neuron_host}" "${package}"; then
-        echo "[${neuron_host}] ${package} update available (current: ${neuron_nvr})"
-        # --allowerasing lets dnf swap out a previously-installed
-        # bare helexa-neuron or a different flavour without manual
-        # intervention. The Conflicts: clauses in the spec ensure
-        # only one flavour is ever resident.
-        if install_or_upgrade "${neuron_host}" "${package}"; then
-            neuron_nvr=$(installed_nvr "${neuron_host}" "${package}")
-            echo "[${neuron_host}] installed/upgraded ${package} to ${neuron_nvr}"
-            # Ensure firewalld allows neuron port
-            ssh "${neuron_host}" "sudo firewall-cmd --query-service=helexa-neuron --quiet 2>/dev/null || sudo firewall-cmd --add-service=helexa-neuron --permanent && sudo firewall-cmd --reload" 2>/dev/null || true
-        else
-            echo "[${neuron_host}] failed to install ${package}:"
-            echo "${__DNF_OUTPUT__}" | sed "s/^/[${neuron_host}]   /"
-        fi
-    else
-        echo "[${neuron_host}] ${package} is up to date (${neuron_nvr})"
-    fi
-
-    # Sync per-host neuron.toml — drives default_models pre-warm so
-    # `/v1/models` on the gateway exposes the host's headline model
-    # immediately after the service comes back up. Missing per-host
-    # config leaves the package's installed neuron.toml untouched.
-    if [[ -f "${host_config}" ]]; then
-        if rsync \
-            --archive \
-            --compress \
-            --rsync-path 'sudo rsync' \
-            --chown root:root \
-            --chmod 644 \
-            "${host_config}" \
-            "${neuron_host}:/etc/neuron/neuron.toml"; then
-            echo "[${neuron_host}] sync'd asset/neuron/${short_host}.toml"
-        else
-            echo "[${neuron_host}] failed to sync neuron.toml"
-        fi
-    else
-        echo "[${neuron_host}] no asset/neuron/${short_host}.toml — leaving /etc/neuron/neuron.toml untouched"
-    fi
-
-    if ssh "${neuron_host}" "sudo systemctl daemon-reload && sudo systemctl start neuron.service"; then
-        echo "[${neuron_host}] started neuron service"
-    else
-        echo "[${neuron_host}] failed to start neuron service"
-    fi
-done

script/infra-setup.sh

@@ -53,6 +53,11 @@ for host in "${cortex_host}" "${neuron_hosts[@]}"; do
         # own fresh lookup.
         sudo install -d -o gitea_ci -g gitea_ci -m 0700 \
             /var/lib/gitea_ci/.ssh
+        # Grant journal read access so the deploy workflow can capture
+        # `journalctl -u <unit> -I` after a service start without
+        # needing a sudoers entry. Idempotent — usermod -aG on an
+        # already-member is a no-op.
+        sudo usermod -aG systemd-journal gitea_ci
     '; then
         echo "  failed to provision gitea_ci — skipping ${host}"
         continue