rob thijssen
729317d1ef
Stage 3 of the candle-native pivot. neuron now serves POST /v1/chat/completions backed by candle's quantized_qwen3 forward pass on a per-model serialised generation loop, returning the standard OpenAI ChatCompletionResponse envelope. Pipeline per request: - Look up the LoadedModel by request.model (404 if absent). - Apply the Qwen3 chat template across all messages. - Tokenize, then spawn_blocking onto tokio's blocking pool to acquire the per-model arch lock and run prefill + greedy/temperature/top-p sampling via LogitsProcessor. - Stop on <|im_end|>/<|endoftext|> EOS or max_tokens (finish_reason "stop" vs "length"). - Decode with skip_special_tokens=true, build OpenAI response with prompt/completion/total usage counts. Supporting changes: - HarnessRegistry now stores Arc<dyn Harness> and caches a typed Arc<CandleHarness> so inference routes bypass dyn-Trait dispatch. - LoadedModel.arch becomes Arc<Mutex<ModelArch>> so the lock guard can be moved into spawn_blocking. - NeuronState gains an Option<Arc<CandleHarness>> field for the new inference route. - Typed InferenceError lets the handler map ModelNotLoaded → 404 and other failures → 500 without string-matching anyhow messages. - stream=true returns 501 until Stage 4 wires up SSE. - Two leftover mistral.rs string references in proxy.rs and cortex-cli (missed during the Stage 1 sweep) are corrected here. Three new default-feature tests cover the no-candle 503, model-not- loaded 404, and stream=true 501 paths. The cuda-integration test from Stage 2 still covers real load/unload; a streaming-feature gated test exercising actual generation will arrive with Stage 4. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
55 lines
1.3 KiB
TOML
55 lines
1.3 KiB
TOML
[package]
|
|
name = "neuron"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[lib]
|
|
name = "neuron"
|
|
path = "src/lib.rs"
|
|
|
|
[[bin]]
|
|
name = "neuron"
|
|
path = "src/main.rs"
|
|
|
|
[features]
|
|
default = []
|
|
# Enables CUDA acceleration in candle. Without this feature, candle
|
|
# compiles for CPU only and Device::new_cuda calls fall back to CPU.
|
|
cuda = [
|
|
"candle-core/cuda",
|
|
"candle-nn/cuda",
|
|
"candle-transformers/cuda",
|
|
]
|
|
# Reserved for GPU-only integration tests in later stages.
|
|
cuda-integration = ["cuda"]
|
|
|
|
[dependencies]
|
|
cortex-core.workspace = true
|
|
tokio.workspace = true
|
|
axum.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
reqwest.workspace = true
|
|
tracing.workspace = true
|
|
tracing-subscriber.workspace = true
|
|
anyhow.workspace = true
|
|
async-trait.workspace = true
|
|
clap.workspace = true
|
|
thiserror.workspace = true
|
|
figment.workspace = true
|
|
toml.workspace = true
|
|
|
|
# candle for in-process inference. CUDA support is gated behind the
|
|
# crate's `cuda` feature (default off) so the workspace builds on
|
|
# non-CUDA hosts and CI runners.
|
|
candle-core = "0.10.2"
|
|
candle-nn = "0.10.2"
|
|
candle-transformers = "0.10.2"
|
|
tokenizers = { version = "0.22", default-features = false, features = ["onig"] }
|
|
hf-hub = { version = "0.4", features = ["tokio"] }
|
|
|
|
[dev-dependencies]
|
|
tokio = { workspace = true, features = ["test-util"] }
|
|
reqwest.workspace = true
|