src/cpu/elementwise.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // ─── CPU elementwise ops (CHUNK 1) ───────────────────────────────────────── | ||
| 2 | // | ||
| 3 | // FP32 scalar host implementations. Ports the GPU elementwise kernels in | ||
| 4 | // src/cuda/elementwise.cu — kernel math reproduced verbatim, FP32 path only. | ||
| 5 | // | ||
| 6 | // clamp — in-place per-element clamp to [lo, hi]. | ||
| 7 | // mul_inplace — in-place per-element multiply y *= x. | ||
| 8 | // div_inplace — in-place per-element divide y /= x. | ||
| 9 | // axpby_inplace — y = a*y + b*x, FP32 arithmetic for all storage dtypes. | ||
| 10 | // threshold_u8 — Y = X > t ? 1 : 0 as an INT8 byte mask (FP32 or FP16 X). | ||
| 11 | |||
| 12 | #include <brotensor/tensor.h> | ||
| 13 | |||
| 14 | #include <cstdint> | ||
| 15 | #include <cstring> | ||
| 16 | #include <stdexcept> | ||
| 17 | |||
| 18 | namespace brotensor::detail::cpu { | ||
| 19 | |||
| 20 | 6 | void clamp(::brotensor::Tensor& y, float lo, float hi) { | |
| 21 | using ::brotensor::Dtype; | ||
| 22 | 6 | const int n = y.size(); | |
| 23 |
1/2✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
|
6 | if (n == 0) return; |
| 24 |
1/2✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
|
6 | if (y.dtype == Dtype::FP32) { |
| 25 | 6 | float* yp = y.host_f32_mut(); | |
| 26 |
2/2✓ Branch 0 taken 1116 times.
✓ Branch 1 taken 6 times.
|
1122 | for (int i = 0; i < n; ++i) { |
| 27 | 1116 | float v = yp[i]; | |
| 28 |
2/2✓ Branch 0 taken 661 times.
✓ Branch 1 taken 455 times.
|
1116 | if (v < lo) v = lo; |
| 29 |
2/2✓ Branch 0 taken 741 times.
✓ Branch 1 taken 375 times.
|
1116 | if (v > hi) v = hi; |
| 30 | 1116 | yp[i] = v; | |
| 31 | 1116 | } | |
| 32 |
0/2✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
6 | } else if (y.dtype == Dtype::FP16) { |
| 33 | ✗ | std::uint16_t* yp = y.host_fp16_mut(); | |
| 34 | ✗ | for (int i = 0; i < n; ++i) { | |
| 35 | ✗ | float v = ::brotensor::fp16_bits_to_fp32(yp[i]); | |
| 36 | ✗ | if (v < lo) v = lo; | |
| 37 | ✗ | if (v > hi) v = hi; | |
| 38 | ✗ | yp[i] = ::brotensor::fp32_to_fp16_bits(v); | |
| 39 | ✗ | } | |
| 40 | ✗ | } else if (y.dtype == Dtype::BF16) { | |
| 41 | ✗ | std::uint16_t* yp = y.host_bf16_mut(); | |
| 42 | ✗ | for (int i = 0; i < n; ++i) { | |
| 43 | ✗ | float v = ::brotensor::bf16_bits_to_fp32(yp[i]); | |
| 44 | ✗ | if (v < lo) v = lo; | |
| 45 | ✗ | if (v > hi) v = hi; | |
| 46 | ✗ | yp[i] = ::brotensor::fp32_to_bf16_bits(v); | |
| 47 | ✗ | } | |
| 48 | ✗ | } else { | |
| 49 | ✗ | throw std::runtime_error("clamp: unsupported dtype"); | |
| 50 | } | ||
| 51 | 6 | } | |
| 52 | |||
| 53 | 1250 | void mul_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& x) { | |
| 54 | // Matches the GPU contract: shape/dtype must match exactly. | ||
| 55 |
1/2✓ Branch 0 taken 1250 times.
✗ Branch 1 not taken.
|
1250 | if (y.dtype != x.dtype || y.rows != x.rows || y.cols != x.cols) { |
| 56 | ✗ | throw std::runtime_error("mul_inplace: shape/dtype mismatch"); | |
| 57 | } | ||
| 58 | 1250 | const int n = y.size(); | |
| 59 |
1/2✓ Branch 0 taken 1250 times.
✗ Branch 1 not taken.
|
1250 | if (n == 0) return; |
| 60 | 1250 | float* yp = y.host_f32_mut(); | |
| 61 | 1250 | const float* xp = x.host_f32(); | |
| 62 |
2/2✓ Branch 0 taken 1250 times.
✓ Branch 1 taken 36635833 times.
|
36637083 | for (int i = 0; i < n; ++i) yp[i] *= xp[i]; |
| 63 | 1250 | } | |
| 64 | |||
| 65 | ✗ | void div_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& x) { | |
| 66 | ✗ | if (y.dtype != x.dtype || y.rows != x.rows || y.cols != x.cols) { | |
| 67 | ✗ | throw std::runtime_error("div_inplace: shape/dtype mismatch"); | |
| 68 | } | ||
| 69 | ✗ | const int n = y.size(); | |
| 70 | ✗ | if (n == 0) return; | |
| 71 | ✗ | float* yp = y.host_f32_mut(); | |
| 72 | ✗ | const float* xp = x.host_f32(); | |
| 73 | ✗ | for (int i = 0; i < n; ++i) yp[i] /= xp[i]; | |
| 74 | ✗ | } | |
| 75 | |||
| 76 | // y = a*y + b*x in FP32 regardless of storage dtype; only the final store | ||
| 77 | // rounds back. Matches the GPU kernel exactly (same FP32 expression). | ||
| 78 | 1 | void axpby_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& x, | |
| 79 | float a, float b) { | ||
| 80 | using ::brotensor::Dtype; | ||
| 81 |
1/2✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
|
1 | if (y.dtype != x.dtype || y.rows != x.rows || y.cols != x.cols) { |
| 82 | ✗ | throw std::runtime_error("axpby_inplace: shape/dtype mismatch"); | |
| 83 | } | ||
| 84 | 1 | const int n = y.size(); | |
| 85 |
1/2✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
|
1 | if (n == 0) return; |
| 86 |
1/2✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
|
1 | if (y.dtype == Dtype::FP32) { |
| 87 | 1 | float* yp = y.host_f32_mut(); | |
| 88 | 1 | const float* xp = x.host_f32(); | |
| 89 |
2/2✓ Branch 0 taken 263 times.
✓ Branch 1 taken 1 time.
|
264 | for (int i = 0; i < n; ++i) yp[i] = a * yp[i] + b * xp[i]; |
| 90 |
0/2✗ Branch 0 not taken.
✗ Branch 1 not taken.
|
1 | } else if (y.dtype == Dtype::FP16) { |
| 91 | ✗ | std::uint16_t* yp = y.host_fp16_mut(); | |
| 92 | ✗ | const std::uint16_t* xp = x.host_fp16(); | |
| 93 | ✗ | for (int i = 0; i < n; ++i) { | |
| 94 | ✗ | const float v = a * ::brotensor::fp16_bits_to_fp32(yp[i]) + | |
| 95 | ✗ | b * ::brotensor::fp16_bits_to_fp32(xp[i]); | |
| 96 | ✗ | yp[i] = ::brotensor::fp32_to_fp16_bits(v); | |
| 97 | ✗ | } | |
| 98 | ✗ | } else if (y.dtype == Dtype::BF16) { | |
| 99 | ✗ | std::uint16_t* yp = y.host_bf16_mut(); | |
| 100 | ✗ | const std::uint16_t* xp = x.host_bf16(); | |
| 101 | ✗ | for (int i = 0; i < n; ++i) { | |
| 102 | ✗ | const float v = a * ::brotensor::bf16_bits_to_fp32(yp[i]) + | |
| 103 | ✗ | b * ::brotensor::bf16_bits_to_fp32(xp[i]); | |
| 104 | ✗ | yp[i] = ::brotensor::fp32_to_bf16_bits(v); | |
| 105 | ✗ | } | |
| 106 | ✗ | } else { | |
| 107 | ✗ | throw std::runtime_error("axpby_inplace: unsupported dtype"); | |
| 108 | } | ||
| 109 | 1 | } | |
| 110 | |||
| 111 | // Strict >: a value exactly at t maps to 0. | ||
| 112 | 4 | void threshold_u8(const ::brotensor::Tensor& X, float t, | |
| 113 | ::brotensor::Tensor& Y) { | ||
| 114 | using ::brotensor::Dtype; | ||
| 115 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
4 | if (X.dtype != Dtype::FP32 && X.dtype != Dtype::FP16) { |
| 116 | ✗ | throw std::runtime_error("threshold_u8: X must be FP32 or FP16"); | |
| 117 | } | ||
| 118 |
1/6✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
4 | if (Y.rows != X.rows || Y.cols != X.cols || Y.dtype != Dtype::INT8) { |
| 119 | 4 | Y.resize(X.rows, X.cols, Dtype::INT8); | |
| 120 | 4 | } | |
| 121 | 4 | const int n = X.size(); | |
| 122 |
1/2✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
|
4 | if (n == 0) return; |
| 123 | 4 | int8_t* yp = static_cast<int8_t*>(Y.host_raw_mut()); | |
| 124 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
|
4 | if (X.dtype == Dtype::FP32) { |
| 125 | 4 | const float* xp = X.host_f32(); | |
| 126 |
2/2✓ Branch 0 taken 4408 times.
✓ Branch 1 taken 4 times.
|
4412 | for (int i = 0; i < n; ++i) yp[i] = (xp[i] > t) ? 1 : 0; |
| 127 | 4 | } else { | |
| 128 | ✗ | const std::uint16_t* xp = X.host_fp16(); | |
| 129 | ✗ | for (int i = 0; i < n; ++i) { | |
| 130 | ✗ | yp[i] = (::brotensor::fp16_bits_to_fp32(xp[i]) > t) ? 1 : 0; | |
| 131 | ✗ | } | |
| 132 | } | ||
| 133 | 4 | } | |
| 134 | |||
| 135 | 94 | void cast(const ::brotensor::Tensor& src, ::brotensor::Tensor& dst, | |
| 136 | ::brotensor::Dtype out_dtype) { | ||
| 137 | using ::brotensor::Dtype; | ||
| 138 |
4/6✓ Branch 0 taken 78 times.
✓ Branch 1 taken 16 times.
✓ Branch 2 taken 78 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 78 times.
|
94 | if (dst.rows != src.rows || dst.cols != src.cols || |
| 139 | 78 | dst.dtype != out_dtype) { | |
| 140 | 16 | dst.resize(src.rows, src.cols, out_dtype); | |
| 141 | 16 | } | |
| 142 | 94 | const int n = src.size(); | |
| 143 |
1/2✓ Branch 0 taken 94 times.
✗ Branch 1 not taken.
|
94 | if (n == 0) return; |
| 144 |
2/2✓ Branch 0 taken 2 times.
✓ Branch 1 taken 92 times.
|
94 | if (src.dtype == out_dtype) { |
| 145 | 2 | std::memcpy(dst.host_raw_mut(), src.host_raw(), src.bytes()); | |
| 146 | 2 | return; | |
| 147 | } | ||
| 148 |
4/4✓ Branch 0 taken 9 times.
✓ Branch 1 taken 83 times.
✓ Branch 2 taken 5 times.
✓ Branch 3 taken 4 times.
|
92 | if (src.dtype == Dtype::FP32 && out_dtype == Dtype::FP16) { |
| 149 | 4 | const float* s = src.host_f32(); | |
| 150 | 4 | std::uint16_t* d = dst.host_fp16_mut(); | |
| 151 |
2/2✓ Branch 0 taken 2112 times.
✓ Branch 1 taken 4 times.
|
2116 | for (int i = 0; i < n; ++i) d[i] = ::brotensor::fp32_to_fp16_bits(s[i]); |
| 152 |
3/4✓ Branch 0 taken 80 times.
✓ Branch 1 taken 8 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 80 times.
|
92 | } else if (src.dtype == Dtype::FP16 && out_dtype == Dtype::FP32) { |
| 153 | 80 | const std::uint16_t* s = src.host_fp16(); | |
| 154 | 80 | float* d = dst.host_f32_mut(); | |
| 155 |
2/2✓ Branch 0 taken 1518 times.
✓ Branch 1 taken 80 times.
|
1598 | for (int i = 0; i < n; ++i) d[i] = ::brotensor::fp16_bits_to_fp32(s[i]); |
| 156 |
3/4✓ Branch 0 taken 5 times.
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 5 times.
|
88 | } else if (src.dtype == Dtype::FP32 && out_dtype == Dtype::BF16) { |
| 157 | 5 | const float* s = src.host_f32(); | |
| 158 | 5 | std::uint16_t* d = dst.host_bf16_mut(); | |
| 159 |
2/2✓ Branch 0 taken 10304 times.
✓ Branch 1 taken 5 times.
|
10309 | for (int i = 0; i < n; ++i) d[i] = ::brotensor::fp32_to_bf16_bits(s[i]); |
| 160 |
2/4✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3 times.
|
8 | } else if (src.dtype == Dtype::BF16 && out_dtype == Dtype::FP32) { |
| 161 | 3 | const std::uint16_t* s = src.host_bf16(); | |
| 162 | 3 | float* d = dst.host_f32_mut(); | |
| 163 |
2/2✓ Branch 0 taken 9248 times.
✓ Branch 1 taken 3 times.
|
9251 | for (int i = 0; i < n; ++i) d[i] = ::brotensor::bf16_bits_to_fp32(s[i]); |
| 164 |
0/4✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
3 | } else if (src.dtype == Dtype::FP16 && out_dtype == Dtype::BF16) { |
| 165 | ✗ | const std::uint16_t* s = src.host_fp16(); | |
| 166 | ✗ | std::uint16_t* d = dst.host_bf16_mut(); | |
| 167 | ✗ | for (int i = 0; i < n; ++i) { | |
| 168 | ✗ | d[i] = ::brotensor::fp32_to_bf16_bits(::brotensor::fp16_bits_to_fp32(s[i])); | |
| 169 | ✗ | } | |
| 170 | ✗ | } else if (src.dtype == Dtype::BF16 && out_dtype == Dtype::FP16) { | |
| 171 | ✗ | const std::uint16_t* s = src.host_bf16(); | |
| 172 | ✗ | std::uint16_t* d = dst.host_fp16_mut(); | |
| 173 | ✗ | for (int i = 0; i < n; ++i) { | |
| 174 | ✗ | d[i] = ::brotensor::fp32_to_fp16_bits(::brotensor::bf16_bits_to_fp32(s[i])); | |
| 175 | ✗ | } | |
| 176 | ✗ | } else { | |
| 177 | ✗ | throw std::runtime_error( | |
| 178 | "brotensor: cast: unsupported dtype pair " | ||
| 179 | "(CPU supports FP32, FP16, and BF16 conversions)"); | ||
| 180 | } | ||
| 181 | 94 | } | |
| 182 | |||
| 183 | 9 | void add_row_bias_inplace(::brotensor::Tensor& Y, const ::brotensor::Tensor& bias) { | |
| 184 | using ::brotensor::Dtype; | ||
| 185 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | if (Y.dtype != bias.dtype) { |
| 186 | ✗ | throw std::runtime_error("add_row_bias_inplace: dtype mismatch"); | |
| 187 | } | ||
| 188 |
2/2✓ Branch 0 taken 8 times.
✓ Branch 1 taken 1 time.
|
9 | if (bias.size() != Y.cols) { |
| 189 |
1/2✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
|
1 | throw std::runtime_error("add_row_bias_inplace: bias size != Y.cols"); |
| 190 | } | ||
| 191 | 8 | const int R = Y.rows; | |
| 192 | 8 | const int D = Y.cols; | |
| 193 | 8 | const int n = R * D; | |
| 194 |
1/2✓ Branch 0 taken 8 times.
✗ Branch 1 not taken.
|
8 | if (n == 0) return; |
| 195 | |||
| 196 |
2/2✓ Branch 0 taken 4 times.
✓ Branch 1 taken 4 times.
|
8 | if (Y.dtype == Dtype::FP32) { |
| 197 | 4 | float* yp = Y.host_f32_mut(); | |
| 198 | 4 | const float* bp = bias.host_f32(); | |
| 199 |
2/2✓ Branch 0 taken 23 times.
✓ Branch 1 taken 4 times.
|
27 | for (int r = 0; r < R; ++r) { |
| 200 | 23 | float* yrow = yp + static_cast<std::size_t>(r) * D; | |
| 201 |
2/2✓ Branch 0 taken 1223 times.
✓ Branch 1 taken 23 times.
|
1246 | for (int d = 0; d < D; ++d) { |
| 202 | 1223 | yrow[d] += bp[d]; | |
| 203 | 1223 | } | |
| 204 | 23 | } | |
| 205 |
2/2✓ Branch 0 taken 2 times.
✓ Branch 1 taken 2 times.
|
8 | } else if (Y.dtype == Dtype::FP16) { |
| 206 | 2 | std::uint16_t* yp = Y.host_fp16_mut(); | |
| 207 | 2 | const std::uint16_t* bp = bias.host_fp16(); | |
| 208 |
2/2✓ Branch 0 taken 12 times.
✓ Branch 1 taken 2 times.
|
14 | for (int r = 0; r < R; ++r) { |
| 209 | 12 | std::uint16_t* yrow = yp + static_cast<std::size_t>(r) * D; | |
| 210 |
2/2✓ Branch 0 taken 640 times.
✓ Branch 1 taken 12 times.
|
652 | for (int d = 0; d < D; ++d) { |
| 211 | 1280 | float v = ::brotensor::fp16_bits_to_fp32(yrow[d]) + | |
| 212 | 640 | ::brotensor::fp16_bits_to_fp32(bp[d]); | |
| 213 | 640 | yrow[d] = ::brotensor::fp32_to_fp16_bits(v); | |
| 214 | 640 | } | |
| 215 | 12 | } | |
| 216 |
1/2✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
|
4 | } else if (Y.dtype == Dtype::BF16) { |
| 217 | 2 | std::uint16_t* yp = Y.host_bf16_mut(); | |
| 218 | 2 | const std::uint16_t* bp = bias.host_bf16(); | |
| 219 |
2/2✓ Branch 0 taken 12 times.
✓ Branch 1 taken 2 times.
|
14 | for (int r = 0; r < R; ++r) { |
| 220 | 12 | std::uint16_t* yrow = yp + static_cast<std::size_t>(r) * D; | |
| 221 |
2/2✓ Branch 0 taken 640 times.
✓ Branch 1 taken 12 times.
|
652 | for (int d = 0; d < D; ++d) { |
| 222 | 1280 | float v = ::brotensor::bf16_bits_to_fp32(yrow[d]) + | |
| 223 | 640 | ::brotensor::bf16_bits_to_fp32(bp[d]); | |
| 224 | 640 | yrow[d] = ::brotensor::fp32_to_bf16_bits(v); | |
| 225 | 640 | } | |
| 226 | 12 | } | |
| 227 | 2 | } else { | |
| 228 | ✗ | throw std::runtime_error("add_row_bias_inplace: unsupported dtype"); | |
| 229 | } | ||
| 230 | 8 | } | |
| 231 | |||
| 232 | } // namespace brotensor::detail::cpu | ||
| 233 |