Add G2 kernel with device SHA3-512, early-exit, and threshold compare
- Implemented qpow_montgomery_g2_kernel for on-device SHA3-512 hashing - Added early-exit flag and linear index tracking in G2 kernel - Integrated G2 kernel launch and result handling in Rust CUDA engine - Introduced MINER_CUDA_MODE=g2 config option to select G2 kernel - Added MINER_CUDA_PINNED option for pinned host buffers and async copies - Updated README with new runtime config knobs and G2 usage instructions
This commit is contained in:
@@ -43,7 +43,7 @@ Notes:
|
||||
|
||||
## Runtime selection and embeds
|
||||
|
||||
At startup, the engine prefers the embedded CUBIN; if absent it falls back to the embedded PTX. You can override with `MINER_CUDA_IMAGE=cubin|ptx`. You’ll see logs like:
|
||||
At startup, the engine prefers the embedded CUBIN; if absent it falls back to the embedded PTX. You can override with `MINER_CUDA_IMAGE=cubin|ptx`. To attempt the G2 path (device SHA3 + early-exit), set `MINER_CUDA_MODE=g2`; if the G2 kernel isn’t embedded/available for the current device, the engine will fall back to G1 automatically. You’ll see logs like:
|
||||
- `CUDA: using CUBIN (embedded)`
|
||||
- `CUDA: using PTX source = embedded`
|
||||
- (If neither exists, the engine logs the absence and delegates to CPU fast engine.)
|
||||
@@ -70,6 +70,10 @@ These knobs affect GPU launch shape and how much work is returned to the host (a
|
||||
- Overrides the embedded image choice (debugging/testing). Default is to prefer CUBIN.
|
||||
- `MINER_CUDA_HASH_THREADS` (optional)
|
||||
- Number of host SHA3 worker threads to use to consume GPU output. Defaults to available parallelism.
|
||||
- `MINER_CUDA_PINNED` = `1|true` (optional)
|
||||
- Use pinned (page-locked) host buffers and asynchronous device-to-host copies for G1 copy-back to reduce PCIe latency.
|
||||
- `MINER_CUDA_MODE` = `g2` (optional)
|
||||
- Attempt G2 kernel (device SHA3-512 + threshold compare + early-exit). Falls back to G1 if the G2 kernel is not available for the current device image.
|
||||
|
||||
How much data per launch?
|
||||
- y_out bytes = `threads × iters × 64`.
|
||||
@@ -182,6 +186,8 @@ Runtime:
|
||||
- `MINER_CUDA_ITERS` — iterations per thread (controls y_out size).
|
||||
- `MINER_CUDA_IMAGE` = `cubin|ptx` — force embedded image selection (optional).
|
||||
- `MINER_CUDA_HASH_THREADS` — parallel host SHA3 workers (optional).
|
||||
- `MINER_CUDA_PINNED` = `1|true` — use pinned host buffers + async D2H copy (G1 optimization).
|
||||
- `MINER_CUDA_MODE` = `g2` — try device SHA3 + early-exit; falls back to G1 if G2 kernel isn’t available.
|
||||
|
||||
Build-time:
|
||||
- `CUDA_ARCH` = `sm_86|sm_89|sm_120|…` — SM target for device images (normalized internally).
|
||||
|
||||
@@ -291,5 +291,283 @@ extern "C" __global__ void qpow_montgomery_g1_kernel(
|
||||
// Note: For G1, the host will compute SHA3-512(y) and distances on the CPU,
|
||||
// validating correctness against cpu-fast/cpu-montgomery on small ranges.
|
||||
//
|
||||
|
||||
|
||||
// -------------------------------------------------------------------------------------------------
|
||||
// G2 additions: device SHA3-512, threshold compare, and early-exit
|
||||
// -------------------------------------------------------------------------------------------------
|
||||
|
||||
// 64-bit rotate-left
|
||||
__device__ __forceinline__ uint64_t rotl64(uint64_t x, int n) {
|
||||
return (x << n) | (x >> (64 - n));
|
||||
}
|
||||
|
||||
// Keccak-f[1600] round constants
|
||||
__device__ __constant__ uint64_t KECCAK_RC[24] = {
|
||||
0x0000000000000001ULL, 0x0000000000008082ULL,
|
||||
0x800000000000808aULL, 0x8000000080008000ULL,
|
||||
0x000000000000808bULL, 0x0000000080000001ULL,
|
||||
0x8000000080008081ULL, 0x8000000000008009ULL,
|
||||
0x000000000000008aULL, 0x0000000000000088ULL,
|
||||
0x0000000080008009ULL, 0x000000008000000aULL,
|
||||
0x000000008000808bULL, 0x800000000000008bULL,
|
||||
0x8000000000008089ULL, 0x8000000000008003ULL,
|
||||
0x8000000000008002ULL, 0x8000000000000080ULL,
|
||||
0x000000000000800aULL, 0x800000008000000aULL,
|
||||
0x8000000080008081ULL, 0x8000000000008080ULL,
|
||||
0x0000000080000001ULL, 0x8000000080008008ULL
|
||||
};
|
||||
|
||||
// Load/store helpers (little-endian)
|
||||
__device__ __forceinline__ uint64_t load64_le(const uint8_t* p) {
|
||||
return ((uint64_t)p[0]) |
|
||||
((uint64_t)p[1] << 8) |
|
||||
((uint64_t)p[2] << 16) |
|
||||
((uint64_t)p[3] << 24) |
|
||||
((uint64_t)p[4] << 32) |
|
||||
((uint64_t)p[5] << 40) |
|
||||
((uint64_t)p[6] << 48) |
|
||||
((uint64_t)p[7] << 56);
|
||||
}
|
||||
__device__ __forceinline__ void store64_le(uint8_t* p, uint64_t v) {
|
||||
p[0] = (uint8_t)(v);
|
||||
p[1] = (uint8_t)(v >> 8);
|
||||
p[2] = (uint8_t)(v >> 16);
|
||||
p[3] = (uint8_t)(v >> 24);
|
||||
p[4] = (uint8_t)(v >> 32);
|
||||
p[5] = (uint8_t)(v >> 40);
|
||||
p[6] = (uint8_t)(v >> 48);
|
||||
p[7] = (uint8_t)(v >> 56);
|
||||
}
|
||||
|
||||
// Keccak-f[1600] permutation
|
||||
__device__ __forceinline__ void keccak_f1600(uint64_t s[25]) {
|
||||
#pragma unroll
|
||||
for (int round = 0; round < 24; ++round) {
|
||||
// Theta
|
||||
uint64_t C[5], D[5];
|
||||
#pragma unroll
|
||||
for (int x = 0; x < 5; ++x) {
|
||||
C[x] = s[x] ^ s[x + 5] ^ s[x + 10] ^ s[x + 15] ^ s[x + 20];
|
||||
}
|
||||
#pragma unroll
|
||||
for (int x = 0; x < 5; ++x) {
|
||||
D[x] = C[(x + 4) % 5] ^ rotl64(C[(x + 1) % 5], 1);
|
||||
}
|
||||
#pragma unroll
|
||||
for (int x = 0; x < 5; ++x) {
|
||||
#pragma unroll
|
||||
for (int y = 0; y < 5; ++y) {
|
||||
s[x + 5 * y] ^= D[x];
|
||||
}
|
||||
}
|
||||
|
||||
// Rho and Pi
|
||||
uint64_t B[25];
|
||||
const int r[25] = {
|
||||
0, 1, 62, 28, 27,
|
||||
36, 44, 6, 55, 20,
|
||||
3, 10, 43, 25, 39,
|
||||
41, 45, 15, 21, 8,
|
||||
18, 2, 61, 56, 14
|
||||
};
|
||||
#pragma unroll
|
||||
for (int x = 0; x < 5; ++x) {
|
||||
#pragma unroll
|
||||
for (int y = 0; y < 5; ++y) {
|
||||
int idx = x + 5 * y;
|
||||
int X = y;
|
||||
int Y = (2 * x + 3 * y) % 5;
|
||||
B[X + 5 * Y] = rotl64(s[idx], r[idx]);
|
||||
}
|
||||
}
|
||||
|
||||
// Chi
|
||||
#pragma unroll
|
||||
for (int x = 0; x < 5; ++x) {
|
||||
#pragma unroll
|
||||
for (int y = 0; y < 5; ++y) {
|
||||
s[x + 5 * y] = B[x + 5 * y] ^ ((~B[((x + 1) % 5) + 5 * y]) & B[((x + 2) % 5) + 5 * y]);
|
||||
}
|
||||
}
|
||||
|
||||
// Iota
|
||||
s[0] ^= KECCAK_RC[round];
|
||||
}
|
||||
}
|
||||
|
||||
// Device SHA3-512 for a single 64-byte message
|
||||
__device__ __forceinline__ void sha3_512_64bytes(const uint8_t in_be64[64], uint8_t out_be64[64]) {
|
||||
// Initialize state to zero
|
||||
uint64_t s[25];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 25; ++i) s[i] = 0ull;
|
||||
|
||||
// Absorb (rate = 72 bytes). Message is 64 bytes: append 0x06 then pad with zeros and set last of rate |= 0x80
|
||||
uint8_t block[72];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 72; ++i) block[i] = 0;
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 64; ++i) block[i] = in_be64[i];
|
||||
block[64] = 0x06;
|
||||
block[71] ^= 0x80;
|
||||
|
||||
// XOR into state lanes as little-endian 64-bit words
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 9; ++i) {
|
||||
s[i] ^= load64_le(&block[i * 8]);
|
||||
}
|
||||
|
||||
// Permute
|
||||
keccak_f1600(s);
|
||||
|
||||
// Squeeze 64 bytes (8 lanes)
|
||||
uint8_t out_le[64];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
store64_le(&out_le[i * 8], s[i]);
|
||||
}
|
||||
|
||||
// The standard digest byte order expected by host libraries is the emitted byte stream.
|
||||
// We keep it as-is; host target/threshold bytes should be in the same orientation to XOR/compare.
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
out_be64[i] = out_le[i];
|
||||
}
|
||||
}
|
||||
|
||||
// Convert 8 LE limbs into 64 BE bytes (big-endian numeric representation)
|
||||
__device__ __forceinline__ void le8_to_be64_bytes(const uint64_t le[8], uint8_t out[64]) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
uint64_t limb = le[7 - i]; // most significant limb first
|
||||
#pragma unroll
|
||||
for (int b = 0; b < 8; ++b) {
|
||||
out[i * 8 + (7 - b)] = (uint8_t)((limb >> (b * 8)) & 0xFF);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compare two 64-byte big-endian numbers: return true if a <= b
|
||||
__device__ __forceinline__ bool be64_leq(const uint8_t a[64], const uint8_t b[64]) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
if (a[i] != b[i]) {
|
||||
return a[i] < b[i];
|
||||
}
|
||||
}
|
||||
return true; // equal
|
||||
}
|
||||
|
||||
// Kernel: G2 — device SHA3-512 + threshold compare + early-exit
|
||||
//
|
||||
// Notes:
|
||||
// - Signature includes additional G2 parameters; host launcher must be updated to pass them.
|
||||
// - Early-exit: a single global int flag claimed via atomicCAS; earliest winning thread writes result.
|
||||
//
|
||||
extern "C" __global__ void qpow_montgomery_g2_kernel(
|
||||
// Per-job constants (each 8 limbs, LE)
|
||||
const uint64_t* __restrict__ m,
|
||||
const uint64_t* __restrict__ n,
|
||||
const uint64_t n0_inv,
|
||||
const uint64_t* __restrict__ r2,
|
||||
const uint64_t* __restrict__ m_hat,
|
||||
|
||||
// Per-thread starting state (normal domain)
|
||||
const uint64_t* __restrict__ y0, // length: num_threads * 8 limbs
|
||||
|
||||
// G2-specific inputs/outputs
|
||||
const uint8_t* __restrict__ target_be, // 64 bytes
|
||||
const uint8_t* __restrict__ threshold_be, // 64 bytes
|
||||
int* __restrict__ found_flag, // 0 -> not found, 1 -> found
|
||||
uint32_t* __restrict__ out_index, // linear index (t * iters + j)
|
||||
uint8_t* __restrict__ out_distance_be, // 64 bytes
|
||||
|
||||
// Threading parameters
|
||||
const uint32_t num_threads,
|
||||
const uint32_t iters_per_thread
|
||||
) {
|
||||
const uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (tid >= num_threads) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Quick early-exit check
|
||||
if (atomicAdd(found_flag, 0) != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Local copies of constants
|
||||
uint64_t n_loc[8], r2_loc[8], mhat_loc[8];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
n_loc[i] = n[i];
|
||||
r2_loc[i] = r2[i];
|
||||
mhat_loc[i] = m_hat[i];
|
||||
}
|
||||
|
||||
// Load this thread's y0 (normal domain) and move to Montgomery domain
|
||||
uint64_t y0_loc[8];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
y0_loc[i] = y0[tid * 8u + i];
|
||||
}
|
||||
uint64_t yhat[8];
|
||||
to_mont_512(y0_loc, r2_loc, n_loc, n0_inv, yhat);
|
||||
|
||||
// Iterate and check threshold
|
||||
const uint32_t iters = iters_per_thread;
|
||||
for (uint32_t j = 0; j < iters; ++j) {
|
||||
// Respect early-exit
|
||||
if (atomicAdd(found_flag, 0) != 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// y_hat = y_hat * m_hat
|
||||
uint64_t yhat_next[8];
|
||||
mont_mul_512(yhat, mhat_loc, n_loc, n0_inv, yhat_next);
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
yhat[i] = yhat_next[i];
|
||||
}
|
||||
|
||||
// y = from_mont(y_hat)
|
||||
uint64_t y_norm[8];
|
||||
from_mont_512(yhat, n_loc, n0_inv, y_norm);
|
||||
|
||||
// y_be64 (64 bytes) from LE limbs
|
||||
uint8_t y_be[64];
|
||||
le8_to_be64_bytes(y_norm, y_be);
|
||||
|
||||
// H = SHA3-512(y_be)
|
||||
uint8_t h_be[64];
|
||||
sha3_512_64bytes(y_be, h_be);
|
||||
|
||||
// distance = target XOR H (byte-wise)
|
||||
uint8_t dist_be[64];
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
dist_be[i] = target_be[i] ^ h_be[i];
|
||||
}
|
||||
|
||||
// Compare distance <= threshold
|
||||
if (be64_leq(dist_be, threshold_be)) {
|
||||
// Try to claim the flag
|
||||
if (atomicCAS(found_flag, 0, 1) == 0) {
|
||||
// Write linear index for host to reconstruct nonce
|
||||
if (out_index) {
|
||||
*out_index = tid * iters + j;
|
||||
}
|
||||
// Write distance
|
||||
if (out_distance_be) {
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 64; ++i) {
|
||||
out_distance_be[i] = dist_be[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
return; // early-exit after claiming
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // extern "C"
|
||||
@@ -344,7 +344,103 @@ impl CudaEngine {
|
||||
let kernel_ms = t_kernel_start.elapsed().as_millis();
|
||||
log::info!(target: "miner", "CUDA kernel and sync OK (kernel_ms={kernel_ms})");
|
||||
|
||||
// Copy back results (optional pinned host buffer + async copy)
|
||||
// Branch: if G2 kernel is selected, perform device SHA3 + early-exit handling
|
||||
if func_name == "qpow_montgomery_g2_kernel" {
|
||||
// Compute bounds
|
||||
let rem_be = end.saturating_sub(current).to_big_endian();
|
||||
let mut last8 = [0u8; 8];
|
||||
last8.copy_from_slice(&rem_be[56..64]);
|
||||
let remaining_inclusive: u64 = u64::from_be_bytes(last8);
|
||||
let total_elems_u64 = (num_threads as u64) * (iters_per_thread as u64);
|
||||
let covered = std::cmp::min(total_elems_u64, remaining_inclusive);
|
||||
|
||||
// Prepare target/threshold (64-byte big-endian) for device
|
||||
let target_be = ctx.target.to_big_endian();
|
||||
let threshold_be = ctx.threshold.to_big_endian();
|
||||
let d_target = cuda::memory::DeviceBuffer::<u8>::from_slice(&target_be)
|
||||
.with_context(|| "alloc/copy d_target")?;
|
||||
let d_threshold = cuda::memory::DeviceBuffer::<u8>::from_slice(&threshold_be)
|
||||
.with_context(|| "alloc/copy d_threshold")?;
|
||||
|
||||
// Early-exit outputs
|
||||
let d_found = cuda::memory::DeviceBuffer::<i32>::from_slice(&[0i32])
|
||||
.with_context(|| "alloc/copy d_found")?;
|
||||
let d_index = cuda::memory::DeviceBuffer::<u32>::from_slice(&[0u32])
|
||||
.with_context(|| "alloc/copy d_index")?;
|
||||
let mut d_distance = cuda::memory::DeviceBuffer::<u8>::zeroed(64)
|
||||
.with_context(|| "alloc d_distance")?;
|
||||
|
||||
// Launch G2 kernel: includes on-device SHA3 and threshold compare with early-exit
|
||||
log::info!(target: "miner", "CUDA G2 launch: grid_dim={grid_dim}, block_dim={block_dim}, threads={num_threads}, iters={iters_per_thread}");
|
||||
let launch_result = unsafe {
|
||||
launch!(func<<<grid_dim, block_dim, 0, stream>>>(
|
||||
d_m.as_device_ptr(),
|
||||
d_n.as_device_ptr(),
|
||||
n0_inv as u64,
|
||||
d_r2.as_device_ptr(),
|
||||
d_mhat.as_device_ptr(),
|
||||
d_y0.as_device_ptr(),
|
||||
d_target.as_device_ptr(),
|
||||
d_threshold.as_device_ptr(),
|
||||
d_found.as_device_ptr(),
|
||||
d_index.as_device_ptr(),
|
||||
d_distance.as_device_ptr(),
|
||||
num_threads as u32,
|
||||
iters_per_thread as u32
|
||||
))
|
||||
};
|
||||
launch_result.with_context(|| "launch G2 kernel")?;
|
||||
stream
|
||||
.synchronize()
|
||||
.with_context(|| "stream synchronize (G2)")?;
|
||||
|
||||
// Copy back early-exit flag (and details if found)
|
||||
let mut h_found = [0i32; 1];
|
||||
d_found
|
||||
.copy_to(&mut h_found)
|
||||
.with_context(|| "copy found flag")?;
|
||||
if h_found[0] != 0 {
|
||||
let mut h_idx = [0u32; 1];
|
||||
d_index.copy_to(&mut h_idx).with_context(|| "copy index")?;
|
||||
let k = h_idx[0] as u64;
|
||||
// Account: include the winning step
|
||||
hash_count = hash_count.saturating_add(k + 1);
|
||||
|
||||
// Compute nonce from linear index (t, j)
|
||||
let t = (k as usize) / (iters_per_thread as usize);
|
||||
let j = (k as usize) % (iters_per_thread as usize);
|
||||
let nonce = current.saturating_add(U512::from(
|
||||
1u64 + (t as u64) * (iters_per_thread as u64) + (j as u64),
|
||||
));
|
||||
|
||||
// Copy back distance (optional for completeness; threshold was already satisfied)
|
||||
let mut h_dist = [0u8; 64];
|
||||
d_distance
|
||||
.copy_to(&mut h_dist)
|
||||
.with_context(|| "copy distance")?;
|
||||
let distance = U512::from_big_endian(&h_dist);
|
||||
let work = nonce.to_big_endian();
|
||||
log::info!(target: "miner", "CUDA G2: early-exit found at idx={k}");
|
||||
return Ok(EngineStatus::Found {
|
||||
candidate: engine_cpu::EngineCandidate {
|
||||
nonce,
|
||||
work,
|
||||
distance,
|
||||
},
|
||||
hash_count,
|
||||
});
|
||||
}
|
||||
|
||||
// Not found in this window
|
||||
hash_count = hash_count.saturating_add(covered);
|
||||
if covered < total_elems_u64 {
|
||||
return Ok(EngineStatus::Exhausted { hash_count });
|
||||
}
|
||||
current = current.saturating_add(U512::from(1u64 + total_elems_u64));
|
||||
continue;
|
||||
}
|
||||
|
||||
// G1 path (host SHA3) — Copy back results (optional pinned host buffer + async copy)
|
||||
let use_pinned = std::env::var("MINER_CUDA_PINNED")
|
||||
.ok()
|
||||
.map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
|
||||
@@ -352,12 +448,10 @@ impl CudaEngine {
|
||||
let total_words = (num_threads as usize) * (iters_per_thread as usize) * 8;
|
||||
let t_copy_start = std::time::Instant::now();
|
||||
let y_host_arc: std::sync::Arc<dyn AsRef<[u64]> + Send + Sync> = if use_pinned {
|
||||
// Page-locked host buffer with async copy on the same stream
|
||||
let mut pinned =
|
||||
unsafe { cust::memory::LockedBuffer::<u64>::uninitialized(total_words) }
|
||||
.with_context(|| "alloc pinned host buffer")?;
|
||||
unsafe {
|
||||
// Use fully-qualified trait path to avoid needing a 'use'
|
||||
cust::memory::AsyncCopyDestination::async_copy_to(
|
||||
&*d_y_out,
|
||||
pinned.as_mut_slice(),
|
||||
@@ -365,14 +459,12 @@ impl CudaEngine {
|
||||
)
|
||||
.with_context(|| "async copy d_y_out -> pinned")?;
|
||||
}
|
||||
// Ensure async copy completion before host access
|
||||
stream
|
||||
.synchronize()
|
||||
.with_context(|| "stream synchronize (post async D2H)")?;
|
||||
log::info!(target: "miner", "CUDA copy-back OK (pinned, async): elems={}, copy_ms={}", total_words, t_copy_start.elapsed().as_millis());
|
||||
std::sync::Arc::new(pinned)
|
||||
} else {
|
||||
// Pageable host buffer with synchronous copy
|
||||
let mut y_out_host = vec![0u64; total_words];
|
||||
d_y_out
|
||||
.copy_to(&mut y_out_host)
|
||||
@@ -435,14 +527,12 @@ impl CudaEngine {
|
||||
|
||||
let distance = pow_core::distance_from_y(&ctx_local, y_norm);
|
||||
if pow_core::is_valid_distance(&ctx_local, distance) {
|
||||
// Compute nonce for this index: nonce = current + 1 + (t * iters) + j
|
||||
let t = k / iters_usize;
|
||||
let j = k % iters_usize;
|
||||
let nonce = current.saturating_add(U512::from(
|
||||
1u64 + (t as u64) * (iters_per_thread as u64) + (j as u64),
|
||||
));
|
||||
|
||||
// Update global minimum index and store candidate if earliest
|
||||
let prev = min_idx.load(std::sync::atomic::Ordering::Relaxed);
|
||||
if k < prev {
|
||||
min_idx.store(k, std::sync::atomic::Ordering::Relaxed);
|
||||
@@ -485,7 +575,6 @@ impl CudaEngine {
|
||||
hash_count = hash_count.saturating_add(gpu_coverage);
|
||||
|
||||
if gpu_coverage < total_elems_u64 {
|
||||
// We consumed to the end of the range in this chunk
|
||||
return Ok(EngineStatus::Exhausted { hash_count });
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user