src/cpu/rms_norm.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // ─── CPU RMSNorm ops (CHUNK 2) ───────────────────────────────────────────── | ||
| 2 | // | ||
| 3 | // FP32 scalar host implementations. Ports src/cuda/rms_norm.cu — FP32 path | ||
| 4 | // only. Per-row (one block per row on the GPU). | ||
| 5 | // | ||
| 6 | // rms[b] = sqrt(mean_j x[b, j]^2 + eps) | ||
| 7 | // y[b, j] = x[b, j] * gamma[j] / rms[b] | ||
| 8 | // | ||
| 9 | // Backward (rrms = 1/rms): | ||
| 10 | // sum_xdy = sum_j x_j * dY_j * gamma_j | ||
| 11 | // coeff = (1/D) * rrms^2 * sum_xdy | ||
| 12 | // dX[b,j] = rrms * (gamma_j * dY_j - x_j * coeff) | ||
| 13 | // dGamma_j += sum_b dY[b,j] * x[b,j] * rrms | ||
| 14 | // | ||
| 15 | // ACCUMULATION: dX is OVERWRITTEN (the GPU writes dxrow[j] directly). dGamma | ||
| 16 | // ACCUMULATES (+=) — the GPU atomicAdds across the batch into dGamma, and the | ||
| 17 | // caller is responsible for zeroing dGamma beforehand. | ||
| 18 | |||
| 19 | #include <brotensor/tensor.h> | ||
| 20 | #include <brotensor/detail/cpu/thread_pool.h> | ||
| 21 | |||
| 22 | #include <cmath> | ||
| 23 | #include <cstddef> | ||
| 24 | #include <stdexcept> | ||
| 25 | #include <vector> | ||
| 26 | |||
| 27 | namespace brotensor::detail::cpu { | ||
| 28 | |||
| 29 | 9 | void rms_norm_forward(const ::brotensor::Tensor& X, | |
| 30 | const ::brotensor::Tensor& gamma, | ||
| 31 | float eps, ::brotensor::Tensor& Y) { | ||
| 32 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | if (gamma.dtype != X.dtype) { |
| 33 | ✗ | throw std::runtime_error("rms_norm_forward: gamma.dtype must match X.dtype"); | |
| 34 | } | ||
| 35 | 9 | const int B = X.rows; | |
| 36 | 9 | const int D = X.cols; | |
| 37 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | if (gamma.size() != D) { |
| 38 | ✗ | throw std::runtime_error("rms_norm_forward: gamma must have D elements"); | |
| 39 | } | ||
| 40 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
9 | if (Y.rows != B || Y.cols != D) Y.resize(B, D); |
| 41 |
2/4✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 9 times.
|
9 | if (B == 0 || D == 0) return; |
| 42 | 9 | const float* Xp = X.host_f32(); | |
| 43 | 9 | const float* gp = gamma.host_f32(); | |
| 44 | 9 | float* Yp = Y.host_f32_mut(); | |
| 45 | // Each row b owns Y's row b exclusively (X/gamma are read-only shared | ||
| 46 | // inputs), so this parallelizes across b with no cross-thread writes. | ||
| 47 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
53 | parallel_for(static_cast<std::size_t>(B), [&](std::size_t bi) { |
| 48 | 44 | const int b = static_cast<int>(bi); | |
| 49 | 44 | const float* xrow = Xp + static_cast<size_t>(b) * D; | |
| 50 | 44 | float* yrow = Yp + static_cast<size_t>(b) * D; | |
| 51 | 44 | float sum = 0.0f; | |
| 52 |
2/2✓ Branch 0 taken 3152 times.
✓ Branch 1 taken 44 times.
|
3196 | for (int j = 0; j < D; ++j) { |
| 53 | 3152 | const float v = xrow[j]; | |
| 54 | 3152 | sum += v * v; | |
| 55 | 3152 | } | |
| 56 | 44 | const float rrms = 1.0f / std::sqrt(sum / static_cast<float>(D) + eps); | |
| 57 |
2/2✓ Branch 0 taken 3152 times.
✓ Branch 1 taken 44 times.
|
3196 | for (int j = 0; j < D; ++j) { |
| 58 | 3152 | yrow[j] = xrow[j] * gp[j] * rrms; | |
| 59 | 3152 | } | |
| 60 | 44 | }); | |
| 61 | 9 | } | |
| 62 | |||
| 63 | 7 | void rms_norm_backward(const ::brotensor::Tensor& X, | |
| 64 | const ::brotensor::Tensor& gamma, | ||
| 65 | const ::brotensor::Tensor& dY, float eps, | ||
| 66 | ::brotensor::Tensor& dX, ::brotensor::Tensor& dGamma) { | ||
| 67 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (gamma.dtype != X.dtype || dY.dtype != X.dtype || |
| 68 | 7 | dGamma.dtype != X.dtype) { | |
| 69 | ✗ | throw std::runtime_error("rms_norm_backward: dtypes must match"); | |
| 70 | } | ||
| 71 | 7 | const int B = X.rows; | |
| 72 | 7 | const int D = X.cols; | |
| 73 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (dY.rows != B || dY.cols != D) { |
| 74 | ✗ | throw std::runtime_error("rms_norm_backward: dY shape mismatch"); | |
| 75 | } | ||
| 76 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (gamma.size() != D || dGamma.size() != D) { |
| 77 | ✗ | throw std::runtime_error("rms_norm_backward: gamma/dGamma size mismatch"); | |
| 78 | } | ||
| 79 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
7 | if (dX.rows != B || dX.cols != D) dX.resize(B, D); |
| 80 |
2/4✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
|
7 | if (B == 0 || D == 0) return; |
| 81 | 7 | const float* Xp = X.host_f32(); | |
| 82 | 7 | const float* gp = gamma.host_f32(); | |
| 83 | 7 | const float* dYp = dY.host_f32(); | |
| 84 | 7 | float* dXp = dX.host_f32_mut(); | |
| 85 | 7 | float* dGp = dGamma.host_f32_mut(); | |
| 86 | 7 | const float inv_D = 1.0f / static_cast<float>(D); | |
| 87 | |||
| 88 | // dGamma accumulates dGp[j] += ... summed over every row b — a shared | ||
| 89 | // reduction across the batch axis, not disjoint per b. So the dX pass | ||
| 90 | // (fully owned by row b) is parallelized, while rrms per row is cached | ||
| 91 | // (one float per b, disjoint write) so the dGamma pass below can reuse | ||
| 92 | // it without re-deriving it or touching any shared state during the | ||
| 93 | // parallel section. dGamma itself is accumulated in a separate, | ||
| 94 | // single-threaded pass afterwards. | ||
| 95 | 7 | std::vector<float> rrms_cache(static_cast<std::size_t>(B)); | |
| 96 | |||
| 97 |
2/4✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
|
43 | parallel_for(static_cast<std::size_t>(B), [&](std::size_t bi) { |
| 98 | 36 | const int b = static_cast<int>(bi); | |
| 99 | 36 | const float* xrow = Xp + static_cast<size_t>(b) * D; | |
| 100 | 36 | const float* dyrow = dYp + static_cast<size_t>(b) * D; | |
| 101 | 36 | float* dxrow = dXp + static_cast<size_t>(b) * D; | |
| 102 | |||
| 103 | 36 | float sum_xx = 0.0f; | |
| 104 |
2/2✓ Branch 0 taken 1420 times.
✓ Branch 1 taken 36 times.
|
1456 | for (int j = 0; j < D; ++j) { |
| 105 | 1420 | const float v = xrow[j]; | |
| 106 | 1420 | sum_xx += v * v; | |
| 107 | 1420 | } | |
| 108 | 36 | const float rrms = 1.0f / std::sqrt(sum_xx * inv_D + eps); | |
| 109 | 36 | rrms_cache[b] = rrms; // disjoint per-b write | |
| 110 | |||
| 111 | 36 | float sum_xdy = 0.0f; | |
| 112 |
2/2✓ Branch 0 taken 1420 times.
✓ Branch 1 taken 36 times.
|
1456 | for (int j = 0; j < D; ++j) { |
| 113 | 1420 | sum_xdy += xrow[j] * dyrow[j] * gp[j]; | |
| 114 | 1420 | } | |
| 115 | 36 | const float coeff = inv_D * rrms * rrms * sum_xdy; | |
| 116 | |||
| 117 |
2/2✓ Branch 0 taken 1420 times.
✓ Branch 1 taken 36 times.
|
1456 | for (int j = 0; j < D; ++j) { |
| 118 | 1420 | const float g = gp[j]; | |
| 119 | 1420 | const float dy = dyrow[j]; | |
| 120 | 1420 | const float x = xrow[j]; | |
| 121 | 1420 | dxrow[j] = rrms * (g * dy - x * coeff); // overwrite dX | |
| 122 | 1420 | } | |
| 123 | 36 | }); | |
| 124 | |||
| 125 | // Single-threaded: dGamma reduces across b, so it cannot be split across | ||
| 126 | // threads without a race on dGp[j]. | ||
| 127 |
2/2✓ Branch 0 taken 36 times.
✓ Branch 1 taken 7 times.
|
43 | for (int b = 0; b < B; ++b) { |
| 128 | 36 | const float* xrow = Xp + static_cast<size_t>(b) * D; | |
| 129 | 36 | const float* dyrow = dYp + static_cast<size_t>(b) * D; | |
| 130 | 36 | const float rrms = rrms_cache[b]; | |
| 131 |
2/2✓ Branch 0 taken 1420 times.
✓ Branch 1 taken 36 times.
|
1456 | for (int j = 0; j < D; ++j) { |
| 132 | 1420 | dGp[j] += dyrow[j] * xrow[j] * rrms; // accumulate dGamma | |
| 133 | 1420 | } | |
| 134 | 36 | } | |
| 135 | 7 | } | |
| 136 | |||
| 137 | } // namespace brotensor::detail::cpu | ||
| 138 |