diff --git a/crates/engine-gpu-cuda/readme.md b/crates/engine-gpu-cuda/readme.md index 8ac8276..e5902e6 100644 --- a/crates/engine-gpu-cuda/readme.md +++ b/crates/engine-gpu-cuda/readme.md @@ -43,7 +43,7 @@ Notes: ## Runtime selection and embeds -At startup, the engine prefers the embedded CUBIN; if absent it falls back to the embedded PTX. You can override with `MINER_CUDA_IMAGE=cubin|ptx`. You’ll see logs like: +At startup, the engine prefers the embedded CUBIN; if absent it falls back to the embedded PTX. You can override with `MINER_CUDA_IMAGE=cubin|ptx`. To attempt the G2 path (device SHA3 + early-exit), set `MINER_CUDA_MODE=g2`; if the G2 kernel isn’t embedded/available for the current device, the engine will fall back to G1 automatically. You’ll see logs like: - `CUDA: using CUBIN (embedded)` - `CUDA: using PTX source = embedded` - (If neither exists, the engine logs the absence and delegates to CPU fast engine.) @@ -70,6 +70,10 @@ These knobs affect GPU launch shape and how much work is returned to the host (a - Overrides the embedded image choice (debugging/testing). Default is to prefer CUBIN. - `MINER_CUDA_HASH_THREADS` (optional) - Number of host SHA3 worker threads to use to consume GPU output. Defaults to available parallelism. +- `MINER_CUDA_PINNED` = `1|true` (optional) + - Use pinned (page-locked) host buffers and asynchronous device-to-host copies for G1 copy-back to reduce PCIe latency. +- `MINER_CUDA_MODE` = `g2` (optional) + - Attempt G2 kernel (device SHA3-512 + threshold compare + early-exit). Falls back to G1 if the G2 kernel is not available for the current device image. How much data per launch? - y_out bytes = `threads × iters × 64`. @@ -182,6 +186,8 @@ Runtime: - `MINER_CUDA_ITERS` — iterations per thread (controls y_out size). - `MINER_CUDA_IMAGE` = `cubin|ptx` — force embedded image selection (optional). - `MINER_CUDA_HASH_THREADS` — parallel host SHA3 workers (optional). +- `MINER_CUDA_PINNED` = `1|true` — use pinned host buffers + async D2H copy (G1 optimization). +- `MINER_CUDA_MODE` = `g2` — try device SHA3 + early-exit; falls back to G1 if G2 kernel isn’t available. Build-time: - `CUDA_ARCH` = `sm_86|sm_89|sm_120|…` — SM target for device images (normalized internally). diff --git a/crates/engine-gpu-cuda/src/kernels/qpow_kernel.cu b/crates/engine-gpu-cuda/src/kernels/qpow_kernel.cu index 97e755c..138427e 100644 --- a/crates/engine-gpu-cuda/src/kernels/qpow_kernel.cu +++ b/crates/engine-gpu-cuda/src/kernels/qpow_kernel.cu @@ -291,5 +291,283 @@ extern "C" __global__ void qpow_montgomery_g1_kernel( // Note: For G1, the host will compute SHA3-512(y) and distances on the CPU, // validating correctness against cpu-fast/cpu-montgomery on small ranges. // - + +// ------------------------------------------------------------------------------------------------- +// G2 additions: device SHA3-512, threshold compare, and early-exit +// ------------------------------------------------------------------------------------------------- + +// 64-bit rotate-left +__device__ __forceinline__ uint64_t rotl64(uint64_t x, int n) { + return (x << n) | (x >> (64 - n)); +} + +// Keccak-f[1600] round constants +__device__ __constant__ uint64_t KECCAK_RC[24] = { + 0x0000000000000001ULL, 0x0000000000008082ULL, + 0x800000000000808aULL, 0x8000000080008000ULL, + 0x000000000000808bULL, 0x0000000080000001ULL, + 0x8000000080008081ULL, 0x8000000000008009ULL, + 0x000000000000008aULL, 0x0000000000000088ULL, + 0x0000000080008009ULL, 0x000000008000000aULL, + 0x000000008000808bULL, 0x800000000000008bULL, + 0x8000000000008089ULL, 0x8000000000008003ULL, + 0x8000000000008002ULL, 0x8000000000000080ULL, + 0x000000000000800aULL, 0x800000008000000aULL, + 0x8000000080008081ULL, 0x8000000000008080ULL, + 0x0000000080000001ULL, 0x8000000080008008ULL +}; + +// Load/store helpers (little-endian) +__device__ __forceinline__ uint64_t load64_le(const uint8_t* p) { + return ((uint64_t)p[0]) | + ((uint64_t)p[1] << 8) | + ((uint64_t)p[2] << 16) | + ((uint64_t)p[3] << 24) | + ((uint64_t)p[4] << 32) | + ((uint64_t)p[5] << 40) | + ((uint64_t)p[6] << 48) | + ((uint64_t)p[7] << 56); +} +__device__ __forceinline__ void store64_le(uint8_t* p, uint64_t v) { + p[0] = (uint8_t)(v); + p[1] = (uint8_t)(v >> 8); + p[2] = (uint8_t)(v >> 16); + p[3] = (uint8_t)(v >> 24); + p[4] = (uint8_t)(v >> 32); + p[5] = (uint8_t)(v >> 40); + p[6] = (uint8_t)(v >> 48); + p[7] = (uint8_t)(v >> 56); +} + +// Keccak-f[1600] permutation +__device__ __forceinline__ void keccak_f1600(uint64_t s[25]) { +#pragma unroll + for (int round = 0; round < 24; ++round) { + // Theta + uint64_t C[5], D[5]; +#pragma unroll + for (int x = 0; x < 5; ++x) { + C[x] = s[x] ^ s[x + 5] ^ s[x + 10] ^ s[x + 15] ^ s[x + 20]; + } +#pragma unroll + for (int x = 0; x < 5; ++x) { + D[x] = C[(x + 4) % 5] ^ rotl64(C[(x + 1) % 5], 1); + } +#pragma unroll + for (int x = 0; x < 5; ++x) { +#pragma unroll + for (int y = 0; y < 5; ++y) { + s[x + 5 * y] ^= D[x]; + } + } + + // Rho and Pi + uint64_t B[25]; + const int r[25] = { + 0, 1, 62, 28, 27, + 36, 44, 6, 55, 20, + 3, 10, 43, 25, 39, + 41, 45, 15, 21, 8, + 18, 2, 61, 56, 14 + }; +#pragma unroll + for (int x = 0; x < 5; ++x) { +#pragma unroll + for (int y = 0; y < 5; ++y) { + int idx = x + 5 * y; + int X = y; + int Y = (2 * x + 3 * y) % 5; + B[X + 5 * Y] = rotl64(s[idx], r[idx]); + } + } + + // Chi +#pragma unroll + for (int x = 0; x < 5; ++x) { +#pragma unroll + for (int y = 0; y < 5; ++y) { + s[x + 5 * y] = B[x + 5 * y] ^ ((~B[((x + 1) % 5) + 5 * y]) & B[((x + 2) % 5) + 5 * y]); + } + } + + // Iota + s[0] ^= KECCAK_RC[round]; + } +} + +// Device SHA3-512 for a single 64-byte message +__device__ __forceinline__ void sha3_512_64bytes(const uint8_t in_be64[64], uint8_t out_be64[64]) { + // Initialize state to zero + uint64_t s[25]; +#pragma unroll + for (int i = 0; i < 25; ++i) s[i] = 0ull; + + // Absorb (rate = 72 bytes). Message is 64 bytes: append 0x06 then pad with zeros and set last of rate |= 0x80 + uint8_t block[72]; +#pragma unroll + for (int i = 0; i < 72; ++i) block[i] = 0; +#pragma unroll + for (int i = 0; i < 64; ++i) block[i] = in_be64[i]; + block[64] = 0x06; + block[71] ^= 0x80; + + // XOR into state lanes as little-endian 64-bit words +#pragma unroll + for (int i = 0; i < 9; ++i) { + s[i] ^= load64_le(&block[i * 8]); + } + + // Permute + keccak_f1600(s); + + // Squeeze 64 bytes (8 lanes) + uint8_t out_le[64]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + store64_le(&out_le[i * 8], s[i]); + } + + // The standard digest byte order expected by host libraries is the emitted byte stream. + // We keep it as-is; host target/threshold bytes should be in the same orientation to XOR/compare. +#pragma unroll + for (int i = 0; i < 64; ++i) { + out_be64[i] = out_le[i]; + } +} + +// Convert 8 LE limbs into 64 BE bytes (big-endian numeric representation) +__device__ __forceinline__ void le8_to_be64_bytes(const uint64_t le[8], uint8_t out[64]) { +#pragma unroll + for (int i = 0; i < 8; ++i) { + uint64_t limb = le[7 - i]; // most significant limb first +#pragma unroll + for (int b = 0; b < 8; ++b) { + out[i * 8 + (7 - b)] = (uint8_t)((limb >> (b * 8)) & 0xFF); + } + } +} + +// Compare two 64-byte big-endian numbers: return true if a <= b +__device__ __forceinline__ bool be64_leq(const uint8_t a[64], const uint8_t b[64]) { +#pragma unroll + for (int i = 0; i < 64; ++i) { + if (a[i] != b[i]) { + return a[i] < b[i]; + } + } + return true; // equal +} + +// Kernel: G2 — device SHA3-512 + threshold compare + early-exit +// +// Notes: +// - Signature includes additional G2 parameters; host launcher must be updated to pass them. +// - Early-exit: a single global int flag claimed via atomicCAS; earliest winning thread writes result. +// +extern "C" __global__ void qpow_montgomery_g2_kernel( + // Per-job constants (each 8 limbs, LE) + const uint64_t* __restrict__ m, + const uint64_t* __restrict__ n, + const uint64_t n0_inv, + const uint64_t* __restrict__ r2, + const uint64_t* __restrict__ m_hat, + + // Per-thread starting state (normal domain) + const uint64_t* __restrict__ y0, // length: num_threads * 8 limbs + + // G2-specific inputs/outputs + const uint8_t* __restrict__ target_be, // 64 bytes + const uint8_t* __restrict__ threshold_be, // 64 bytes + int* __restrict__ found_flag, // 0 -> not found, 1 -> found + uint32_t* __restrict__ out_index, // linear index (t * iters + j) + uint8_t* __restrict__ out_distance_be, // 64 bytes + + // Threading parameters + const uint32_t num_threads, + const uint32_t iters_per_thread +) { + const uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= num_threads) { + return; + } + + // Quick early-exit check + if (atomicAdd(found_flag, 0) != 0) { + return; + } + + // Local copies of constants + uint64_t n_loc[8], r2_loc[8], mhat_loc[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + n_loc[i] = n[i]; + r2_loc[i] = r2[i]; + mhat_loc[i] = m_hat[i]; + } + + // Load this thread's y0 (normal domain) and move to Montgomery domain + uint64_t y0_loc[8]; +#pragma unroll + for (int i = 0; i < 8; ++i) { + y0_loc[i] = y0[tid * 8u + i]; + } + uint64_t yhat[8]; + to_mont_512(y0_loc, r2_loc, n_loc, n0_inv, yhat); + + // Iterate and check threshold + const uint32_t iters = iters_per_thread; + for (uint32_t j = 0; j < iters; ++j) { + // Respect early-exit + if (atomicAdd(found_flag, 0) != 0) { + return; + } + + // y_hat = y_hat * m_hat + uint64_t yhat_next[8]; + mont_mul_512(yhat, mhat_loc, n_loc, n0_inv, yhat_next); +#pragma unroll + for (int i = 0; i < 8; ++i) { + yhat[i] = yhat_next[i]; + } + + // y = from_mont(y_hat) + uint64_t y_norm[8]; + from_mont_512(yhat, n_loc, n0_inv, y_norm); + + // y_be64 (64 bytes) from LE limbs + uint8_t y_be[64]; + le8_to_be64_bytes(y_norm, y_be); + + // H = SHA3-512(y_be) + uint8_t h_be[64]; + sha3_512_64bytes(y_be, h_be); + + // distance = target XOR H (byte-wise) + uint8_t dist_be[64]; +#pragma unroll + for (int i = 0; i < 64; ++i) { + dist_be[i] = target_be[i] ^ h_be[i]; + } + + // Compare distance <= threshold + if (be64_leq(dist_be, threshold_be)) { + // Try to claim the flag + if (atomicCAS(found_flag, 0, 1) == 0) { + // Write linear index for host to reconstruct nonce + if (out_index) { + *out_index = tid * iters + j; + } + // Write distance + if (out_distance_be) { +#pragma unroll + for (int i = 0; i < 64; ++i) { + out_distance_be[i] = dist_be[i]; + } + } + } + return; // early-exit after claiming + } + } +} + } // extern "C" \ No newline at end of file diff --git a/crates/engine-gpu-cuda/src/lib.rs b/crates/engine-gpu-cuda/src/lib.rs index 6cddced..b94b9ed 100644 --- a/crates/engine-gpu-cuda/src/lib.rs +++ b/crates/engine-gpu-cuda/src/lib.rs @@ -344,7 +344,103 @@ impl CudaEngine { let kernel_ms = t_kernel_start.elapsed().as_millis(); log::info!(target: "miner", "CUDA kernel and sync OK (kernel_ms={kernel_ms})"); - // Copy back results (optional pinned host buffer + async copy) + // Branch: if G2 kernel is selected, perform device SHA3 + early-exit handling + if func_name == "qpow_montgomery_g2_kernel" { + // Compute bounds + let rem_be = end.saturating_sub(current).to_big_endian(); + let mut last8 = [0u8; 8]; + last8.copy_from_slice(&rem_be[56..64]); + let remaining_inclusive: u64 = u64::from_be_bytes(last8); + let total_elems_u64 = (num_threads as u64) * (iters_per_thread as u64); + let covered = std::cmp::min(total_elems_u64, remaining_inclusive); + + // Prepare target/threshold (64-byte big-endian) for device + let target_be = ctx.target.to_big_endian(); + let threshold_be = ctx.threshold.to_big_endian(); + let d_target = cuda::memory::DeviceBuffer::::from_slice(&target_be) + .with_context(|| "alloc/copy d_target")?; + let d_threshold = cuda::memory::DeviceBuffer::::from_slice(&threshold_be) + .with_context(|| "alloc/copy d_threshold")?; + + // Early-exit outputs + let d_found = cuda::memory::DeviceBuffer::::from_slice(&[0i32]) + .with_context(|| "alloc/copy d_found")?; + let d_index = cuda::memory::DeviceBuffer::::from_slice(&[0u32]) + .with_context(|| "alloc/copy d_index")?; + let mut d_distance = cuda::memory::DeviceBuffer::::zeroed(64) + .with_context(|| "alloc d_distance")?; + + // Launch G2 kernel: includes on-device SHA3 and threshold compare with early-exit + log::info!(target: "miner", "CUDA G2 launch: grid_dim={grid_dim}, block_dim={block_dim}, threads={num_threads}, iters={iters_per_thread}"); + let launch_result = unsafe { + launch!(func<<>>( + d_m.as_device_ptr(), + d_n.as_device_ptr(), + n0_inv as u64, + d_r2.as_device_ptr(), + d_mhat.as_device_ptr(), + d_y0.as_device_ptr(), + d_target.as_device_ptr(), + d_threshold.as_device_ptr(), + d_found.as_device_ptr(), + d_index.as_device_ptr(), + d_distance.as_device_ptr(), + num_threads as u32, + iters_per_thread as u32 + )) + }; + launch_result.with_context(|| "launch G2 kernel")?; + stream + .synchronize() + .with_context(|| "stream synchronize (G2)")?; + + // Copy back early-exit flag (and details if found) + let mut h_found = [0i32; 1]; + d_found + .copy_to(&mut h_found) + .with_context(|| "copy found flag")?; + if h_found[0] != 0 { + let mut h_idx = [0u32; 1]; + d_index.copy_to(&mut h_idx).with_context(|| "copy index")?; + let k = h_idx[0] as u64; + // Account: include the winning step + hash_count = hash_count.saturating_add(k + 1); + + // Compute nonce from linear index (t, j) + let t = (k as usize) / (iters_per_thread as usize); + let j = (k as usize) % (iters_per_thread as usize); + let nonce = current.saturating_add(U512::from( + 1u64 + (t as u64) * (iters_per_thread as u64) + (j as u64), + )); + + // Copy back distance (optional for completeness; threshold was already satisfied) + let mut h_dist = [0u8; 64]; + d_distance + .copy_to(&mut h_dist) + .with_context(|| "copy distance")?; + let distance = U512::from_big_endian(&h_dist); + let work = nonce.to_big_endian(); + log::info!(target: "miner", "CUDA G2: early-exit found at idx={k}"); + return Ok(EngineStatus::Found { + candidate: engine_cpu::EngineCandidate { + nonce, + work, + distance, + }, + hash_count, + }); + } + + // Not found in this window + hash_count = hash_count.saturating_add(covered); + if covered < total_elems_u64 { + return Ok(EngineStatus::Exhausted { hash_count }); + } + current = current.saturating_add(U512::from(1u64 + total_elems_u64)); + continue; + } + + // G1 path (host SHA3) — Copy back results (optional pinned host buffer + async copy) let use_pinned = std::env::var("MINER_CUDA_PINNED") .ok() .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) @@ -352,12 +448,10 @@ impl CudaEngine { let total_words = (num_threads as usize) * (iters_per_thread as usize) * 8; let t_copy_start = std::time::Instant::now(); let y_host_arc: std::sync::Arc + Send + Sync> = if use_pinned { - // Page-locked host buffer with async copy on the same stream let mut pinned = unsafe { cust::memory::LockedBuffer::::uninitialized(total_words) } .with_context(|| "alloc pinned host buffer")?; unsafe { - // Use fully-qualified trait path to avoid needing a 'use' cust::memory::AsyncCopyDestination::async_copy_to( &*d_y_out, pinned.as_mut_slice(), @@ -365,14 +459,12 @@ impl CudaEngine { ) .with_context(|| "async copy d_y_out -> pinned")?; } - // Ensure async copy completion before host access stream .synchronize() .with_context(|| "stream synchronize (post async D2H)")?; log::info!(target: "miner", "CUDA copy-back OK (pinned, async): elems={}, copy_ms={}", total_words, t_copy_start.elapsed().as_millis()); std::sync::Arc::new(pinned) } else { - // Pageable host buffer with synchronous copy let mut y_out_host = vec![0u64; total_words]; d_y_out .copy_to(&mut y_out_host) @@ -435,14 +527,12 @@ impl CudaEngine { let distance = pow_core::distance_from_y(&ctx_local, y_norm); if pow_core::is_valid_distance(&ctx_local, distance) { - // Compute nonce for this index: nonce = current + 1 + (t * iters) + j let t = k / iters_usize; let j = k % iters_usize; let nonce = current.saturating_add(U512::from( 1u64 + (t as u64) * (iters_per_thread as u64) + (j as u64), )); - // Update global minimum index and store candidate if earliest let prev = min_idx.load(std::sync::atomic::Ordering::Relaxed); if k < prev { min_idx.store(k, std::sync::atomic::Ordering::Relaxed); @@ -485,7 +575,6 @@ impl CudaEngine { hash_count = hash_count.saturating_add(gpu_coverage); if gpu_coverage < total_elems_u64 { - // We consumed to the end of the range in this chunk return Ok(EngineStatus::Exhausted { hash_count }); }