src/cpu/rms_norm.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // ─── CPU RMSNorm ops (CHUNK 2) ───────────────────────────────────────────── | ||
| 2 | // | ||
| 3 | // FP32 scalar host implementations. Ports src/cuda/rms_norm.cu — FP32 path | ||
| 4 | // only. Per-row (one block per row on the GPU). | ||
| 5 | // | ||
| 6 | // rms[b] = sqrt(mean_j x[b, j]^2 + eps) | ||
| 7 | // y[b, j] = x[b, j] * gamma[j] / rms[b] | ||
| 8 | // | ||
| 9 | // Backward (rrms = 1/rms): | ||
| 10 | // sum_xdy = sum_j x_j * dY_j * gamma_j | ||
| 11 | // coeff = (1/D) * rrms^2 * sum_xdy | ||
| 12 | // dX[b,j] = rrms * (gamma_j * dY_j - x_j * coeff) | ||
| 13 | // dGamma_j += sum_b dY[b,j] * x[b,j] * rrms | ||
| 14 | // | ||
| 15 | // ACCUMULATION: dX is OVERWRITTEN (the GPU writes dxrow[j] directly). dGamma | ||
| 16 | // ACCUMULATES (+=) — the GPU atomicAdds across the batch into dGamma, and the | ||
| 17 | // caller is responsible for zeroing dGamma beforehand. | ||
| 18 | |||
| 19 | #include <brotensor/tensor.h> | ||
| 20 | #include <brotensor/detail/cpu/thread_pool.h> | ||
| 21 | #include "cpu_jit.h" | ||
| 22 | |||
| 23 | #include <cmath> | ||
| 24 | #include <cstddef> | ||
| 25 | #include <stdexcept> | ||
| 26 | #include <vector> | ||
| 27 | |||
| 28 | namespace brotensor::detail::cpu { | ||
| 29 | |||
| 30 | 254 | void rms_norm_forward(const ::brotensor::Tensor& X, | |
| 31 | const ::brotensor::Tensor& gamma, | ||
| 32 | float eps, ::brotensor::Tensor& Y) { | ||
| 33 |
1/2✓ Branch 0 taken 254 times.
✗ Branch 1 not taken.
|
254 | if (gamma.dtype != X.dtype) { |
| 34 | ✗ | throw std::runtime_error("rms_norm_forward: gamma.dtype must match X.dtype"); | |
| 35 | } | ||
| 36 | 254 | const int B = X.rows; | |
| 37 | 254 | const int D = X.cols; | |
| 38 |
1/2✓ Branch 0 taken 254 times.
✗ Branch 1 not taken.
|
254 | if (gamma.size() != D) { |
| 39 | ✗ | throw std::runtime_error("rms_norm_forward: gamma must have D elements"); | |
| 40 | } | ||
| 41 |
3/4✓ Branch 0 taken 240 times.
✓ Branch 1 taken 14 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 240 times.
|
254 | if (Y.rows != B || Y.cols != D) Y.resize(B, D); |
| 42 |
2/4✓ Branch 0 taken 254 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 254 times.
|
254 | if (B == 0 || D == 0) return; |
| 43 | 254 | const float* Xp = X.host_f32(); | |
| 44 | 254 | const float* gp = gamma.host_f32(); | |
| 45 | 254 | float* Yp = Y.host_f32_mut(); | |
| 46 | |||
| 47 |
1/2✓ Branch 0 taken 254 times.
✗ Branch 1 not taken.
|
254 | if (jit::is_jit_available()) { |
| 48 | 254 | jit::rms_norm_forward(Xp, gp, eps, Yp, B, D); | |
| 49 | 254 | return; | |
| 50 | } | ||
| 51 | // Each row b owns Y's row b exclusively (X/gamma are read-only shared | ||
| 52 | // inputs), so this parallelizes across b with no cross-thread writes. | ||
| 53 | ✗ | parallel_for(static_cast<std::size_t>(B), [&](std::size_t bi) { | |
| 54 | ✗ | const int b = static_cast<int>(bi); | |
| 55 | ✗ | const float* xrow = Xp + static_cast<size_t>(b) * D; | |
| 56 | ✗ | float* yrow = Yp + static_cast<size_t>(b) * D; | |
| 57 | ✗ | float sum = 0.0f; | |
| 58 | ✗ | for (int j = 0; j < D; ++j) { | |
| 59 | ✗ | const float v = xrow[j]; | |
| 60 | ✗ | sum += v * v; | |
| 61 | ✗ | } | |
| 62 | ✗ | const float rrms = 1.0f / std::sqrt(sum / static_cast<float>(D) + eps); | |
| 63 | ✗ | for (int j = 0; j < D; ++j) { | |
| 64 | ✗ | yrow[j] = xrow[j] * gp[j] * rrms; | |
| 65 | ✗ | } | |
| 66 | ✗ | }); | |
| 67 | 254 | } | |
| 68 | |||
| 69 | 7 | void rms_norm_backward(const ::brotensor::Tensor& X, | |
| 70 | const ::brotensor::Tensor& gamma, | ||
| 71 | const ::brotensor::Tensor& dY, float eps, | ||
| 72 | ::brotensor::Tensor& dX, ::brotensor::Tensor& dGamma) { | ||
| 73 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (gamma.dtype != X.dtype || dY.dtype != X.dtype || |
| 74 | 7 | dGamma.dtype != X.dtype) { | |
| 75 | ✗ | throw std::runtime_error("rms_norm_backward: dtypes must match"); | |
| 76 | } | ||
| 77 | 7 | const int B = X.rows; | |
| 78 | 7 | const int D = X.cols; | |
| 79 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (dY.rows != B || dY.cols != D) { |
| 80 | ✗ | throw std::runtime_error("rms_norm_backward: dY shape mismatch"); | |
| 81 | } | ||
| 82 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (gamma.size() != D || dGamma.size() != D) { |
| 83 | ✗ | throw std::runtime_error("rms_norm_backward: gamma/dGamma size mismatch"); | |
| 84 | } | ||
| 85 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
7 | if (dX.rows != B || dX.cols != D) dX.resize(B, D); |
| 86 |
2/4✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
|
7 | if (B == 0 || D == 0) return; |
| 87 | 7 | const float* Xp = X.host_f32(); | |
| 88 | 7 | const float* gp = gamma.host_f32(); | |
| 89 | 7 | const float* dYp = dY.host_f32(); | |
| 90 | 7 | float* dXp = dX.host_f32_mut(); | |
| 91 | 7 | float* dGp = dGamma.host_f32_mut(); | |
| 92 | 7 | const float inv_D = 1.0f / static_cast<float>(D); | |
| 93 | |||
| 94 | // dGamma accumulates dGp[j] += ... summed over every row b — a shared | ||
| 95 | // reduction across the batch axis, not disjoint per b. So the dX pass | ||
| 96 | // (fully owned by row b) is parallelized, while rrms per row is cached | ||
| 97 | // (one float per b, disjoint write) so the dGamma pass below can reuse | ||
| 98 | // it without re-deriving it or touching any shared state during the | ||
| 99 | // parallel section. dGamma itself is accumulated in a separate, | ||
| 100 | // single-threaded pass afterwards. | ||
| 101 | 7 | std::vector<float> rrms_cache(static_cast<std::size_t>(B)); | |
| 102 | |||
| 103 |
2/4✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
|
43 | parallel_for(static_cast<std::size_t>(B), [&](std::size_t bi) { |
| 104 | 36 | const int b = static_cast<int>(bi); | |
| 105 | 36 | const float* xrow = Xp + static_cast<size_t>(b) * D; | |
| 106 | 36 | const float* dyrow = dYp + static_cast<size_t>(b) * D; | |
| 107 | 36 | float* dxrow = dXp + static_cast<size_t>(b) * D; | |
| 108 | |||
| 109 | 36 | float sum_xx = 0.0f; | |
| 110 |
2/2✓ Branch 0 taken 1420 times.
✓ Branch 1 taken 36 times.
|
1456 | for (int j = 0; j < D; ++j) { |
| 111 | 1420 | const float v = xrow[j]; | |
| 112 | 1420 | sum_xx += v * v; | |
| 113 | 1420 | } | |
| 114 | 36 | const float rrms = 1.0f / std::sqrt(sum_xx * inv_D + eps); | |
| 115 | 36 | rrms_cache[b] = rrms; // disjoint per-b write | |
| 116 | |||
| 117 | 36 | float sum_xdy = 0.0f; | |
| 118 |
2/2✓ Branch 0 taken 1420 times.
✓ Branch 1 taken 36 times.
|
1456 | for (int j = 0; j < D; ++j) { |
| 119 | 1420 | sum_xdy += xrow[j] * dyrow[j] * gp[j]; | |
| 120 | 1420 | } | |
| 121 | 36 | const float coeff = inv_D * rrms * rrms * sum_xdy; | |
| 122 | |||
| 123 |
2/2✓ Branch 0 taken 1420 times.
✓ Branch 1 taken 36 times.
|
1456 | for (int j = 0; j < D; ++j) { |
| 124 | 1420 | const float g = gp[j]; | |
| 125 | 1420 | const float dy = dyrow[j]; | |
| 126 | 1420 | const float x = xrow[j]; | |
| 127 | 1420 | dxrow[j] = rrms * (g * dy - x * coeff); // overwrite dX | |
| 128 | 1420 | } | |
| 129 | 36 | }); | |
| 130 | |||
| 131 | // Single-threaded: dGamma reduces across b, so it cannot be split across | ||
| 132 | // threads without a race on dGp[j]. | ||
| 133 |
2/2✓ Branch 0 taken 36 times.
✓ Branch 1 taken 7 times.
|
43 | for (int b = 0; b < B; ++b) { |
| 134 | 36 | const float* xrow = Xp + static_cast<size_t>(b) * D; | |
| 135 | 36 | const float* dyrow = dYp + static_cast<size_t>(b) * D; | |
| 136 | 36 | const float rrms = rrms_cache[b]; | |
| 137 |
2/2✓ Branch 0 taken 1420 times.
✓ Branch 1 taken 36 times.
|
1456 | for (int j = 0; j < D; ++j) { |
| 138 | 1420 | dGp[j] += dyrow[j] * xrow[j] * rrms; // accumulate dGamma | |
| 139 | 1420 | } | |
| 140 | 36 | } | |
| 141 | 7 | } | |
| 142 | |||
| 143 | } // namespace brotensor::detail::cpu | ||
| 144 |