rob thijssen
90e971dcf5
All checks were successful
CI / Format (push) Successful in 32s
CI / Format (pull_request) Successful in 35s
CI / CUDA type-check (push) Successful in 1m50s
CI / CUDA type-check (pull_request) Successful in 2m7s
CI / Clippy (pull_request) Successful in 2m18s
CI / Clippy (push) Successful in 2m46s
CI / Test (push) Successful in 5m33s
CI / Build cortex SRPM (push) Has been skipped
CI / Build neuron SRPM (push) Has been skipped
CI / Publish cortex to COPR (push) Has been skipped
CI / Publish neuron to COPR (push) Has been skipped
CI / Bump version in source (push) Has been skipped
CI / Test (pull_request) Successful in 5m33s
CI / Build cortex SRPM (pull_request) Has been skipped
CI / Publish cortex to COPR (pull_request) Has been skipped
CI / Build neuron SRPM (pull_request) Has been skipped
CI / Publish neuron to COPR (pull_request) Has been skipped
CI / Bump version in source (pull_request) Has been skipped
QTensor::quantize runs its per-block math strictly sequentially on one core (CUDA storage round-trips through the same CPU path), which made Q6K ISQ the dominant phase of the 27B TP cold load. Blocks are independent, so quantize_parallel re-implements the same encoding through candle's public per-block API (k_quants::GgmlType::from_float) with rayon fanning blocks across the CPU pool — byte-identical output, pinned by parity tests against QTensor::quantize for Q6K/Q5K/Q4K/Q8_0. Threading discipline holds: the device-to-host read and the QStorage::from_data upload stay on the calling thread (device worker / subprocess main); rayon workers touch host memory only. Also adds the per-phase timing the issue asked for first: per-layer debug + layer-loop total + lm_head info lines, so the next cold load shows where the time actually goes. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
124 lines
4.6 KiB
TOML
124 lines
4.6 KiB
TOML
[package]
|
|
name = "neuron"
|
|
version.workspace = true
|
|
edition.workspace = true
|
|
license.workspace = true
|
|
|
|
[lib]
|
|
name = "neuron"
|
|
path = "src/lib.rs"
|
|
|
|
[[bin]]
|
|
name = "neuron"
|
|
path = "src/main.rs"
|
|
|
|
[features]
|
|
default = []
|
|
# Enables CUDA acceleration in candle and the cudarc/nccl bindings the
|
|
# TP worker pool uses. Without this feature, candle compiles for CPU
|
|
# only, Device::new_cuda calls fall back to CPU, and TP Init/sanity
|
|
# requests return Error{kind="cuda_feature_not_enabled"}.
|
|
cuda = [
|
|
"candle-core/cuda",
|
|
"candle-core/nccl",
|
|
"candle-nn/cuda",
|
|
"candle-transformers/cuda",
|
|
"dep:cudarc",
|
|
"dep:half",
|
|
"dep:cudaforge",
|
|
]
|
|
# Use cuDNN for convolution / attention kernels. Requires CUDA.
|
|
cudnn = [
|
|
"cuda",
|
|
"candle-core/cudnn",
|
|
"candle-nn/cudnn",
|
|
"candle-transformers/cudnn",
|
|
]
|
|
# FlashAttention kernels. Requires CUDA.
|
|
flash-attn = [
|
|
"cuda",
|
|
"candle-transformers/flash-attn",
|
|
]
|
|
# Reserved for GPU-only integration tests in later stages.
|
|
cuda-integration = ["cuda"]
|
|
|
|
[dependencies]
|
|
cortex-core.workspace = true
|
|
tokio.workspace = true
|
|
axum.workspace = true
|
|
serde.workspace = true
|
|
serde_json.workspace = true
|
|
reqwest.workspace = true
|
|
tracing.workspace = true
|
|
tracing-subscriber.workspace = true
|
|
anyhow.workspace = true
|
|
async-trait.workspace = true
|
|
clap.workspace = true
|
|
thiserror.workspace = true
|
|
futures.workspace = true
|
|
tokio-stream.workspace = true
|
|
figment.workspace = true
|
|
toml.workspace = true
|
|
|
|
# Parallel in-situ quantization (#1): fans candle's per-block k-quant
|
|
# math across the CPU pool at model-load time. Already in the tree
|
|
# transitively via candle-core.
|
|
rayon = "1"
|
|
|
|
# candle for in-process inference. CUDA support is gated behind the
|
|
# crate's `cuda` feature (default off) so the workspace builds on
|
|
# non-CUDA hosts and CI runners.
|
|
candle-core = "0.10.2"
|
|
candle-nn = "0.10.2"
|
|
candle-transformers = "0.10.2"
|
|
# Direct dep on cudarc (matching candle's transitive version) so the
|
|
# TP worker pool can call cudarc::nccl::{Comm, Id} directly. Gated on
|
|
# the `cuda` feature; same toolchain requirement as candle's CUDA path.
|
|
cudarc = { version = "0.19", optional = true, default-features = false, features = ["nccl", "cuda-version-from-build-system"] }
|
|
# Used by the AllReduce CustomOp1 to type-dispatch on bf16/f16 candle
|
|
# storages. Matches candle-core's pinned major version to avoid double-
|
|
# compiling the `half` crate at conflicting versions.
|
|
half = { version = "2.5", optional = true }
|
|
tokenizers = { version = "0.22", default-features = false, features = ["onig"] }
|
|
hf-hub = { version = "0.4", features = ["tokio"] }
|
|
# Jinja-compatible template renderer for the model's chat template
|
|
# (standalone `chat_template.jinja` or `tokenizer_config.json::chat_template`).
|
|
# Hugging Face's chat templates lean on Python string semantics; we
|
|
# bridge them with `minijinja-contrib`'s `pycompat` callback (str
|
|
# methods like `startswith`/`split`/`strip`) plus a `raise_exception`
|
|
# global. Features: `builtins` for `is defined` / `default`; `json`
|
|
# for `tojson`; `serde` so we can hand it a serde_json::Value context.
|
|
minijinja = { version = "2", features = ["builtins", "json", "serde"] }
|
|
# Python-compatibility shim: the Qwen3-VL / Qwen3.6 template uses
|
|
# `content.startswith(...)`, `.endswith(...)`, `.split(...)`,
|
|
# `.rstrip(...)`, `.lstrip(...)` — Python str methods minijinja doesn't
|
|
# implement natively. `pycompat::unknown_method_callback` supplies them.
|
|
minijinja-contrib = { version = "2", features = ["pycompat"] }
|
|
# Direct dep on `safetensors` (re-exported by candle but its `TensorView`
|
|
# / `slice::IndexOp` types are public-but-not-re-exported). Used by the
|
|
# tp `fused_load` module to read per-rank slices of fused QKV tensors
|
|
# without materialising the full tensor on device.
|
|
safetensors = "0.7"
|
|
# Vision capability for Qwen3.6 (Stage A of the vision plan in
|
|
# doc/vision-qwen3_6-spec.md). `image` decodes PNG/JPEG/etc from
|
|
# the bytes embedded in `data:image/...;base64,...` content parts;
|
|
# `base64` does the URI decode. Default-features off on `image` to
|
|
# avoid pulling in audio/video formats we don't need.
|
|
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp", "bmp", "gif"] }
|
|
base64 = "0.22"
|
|
|
|
[dev-dependencies]
|
|
tokio = { workspace = true, features = ["test-util"] }
|
|
reqwest.workspace = true
|
|
tempfile = "3"
|
|
|
|
[build-dependencies]
|
|
# Used by `build.rs` to compile `src/cuda/*.cu` into `libneuroncuda.a`
|
|
# under the `cuda` feature. Matches mistralrs's upstream build setup
|
|
# (their `mistralrs-core/build.rs` uses the same constructor).
|
|
cudaforge = { version = "0.1", optional = true }
|
|
|
|
[package.metadata.docs.rs]
|
|
# Skip the CUDA path on docs.rs (it lacks nvcc).
|
|
no-default-features = true
|