src/cpu/conv1d.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // ─── CPU 1D-convolution family (brosoundml CHUNK 3) ──────────────────────── | ||
| 2 | // | ||
| 3 | // FP32 scalar host implementations of the genuinely-new 1D-conv ops: | ||
| 4 | // conv_transpose1d_forward / _backward_input / _backward_weight / _backward_bias | ||
| 5 | // causal_conv1d_update | ||
| 6 | // pad1d_forward / pad1d_backward | ||
| 7 | // | ||
| 8 | // Plain conv1d, its three backward halves, conv1d_int8w_fp16, and causal_conv1d | ||
| 9 | // are header-only inline wrappers in <brotensor/ops.h> that forward to the | ||
| 10 | // conv2d ops — they do not appear here. | ||
| 11 | // | ||
| 12 | // ── Layout (NCL) ──────────────────────────────────────────────────────────── | ||
| 13 | // X / Y : NCL — ((n*C + c) * L + l). N batched signals folded into rows. | ||
| 14 | // conv_transpose1d weights: OIL, input-channel-major (transposed-conv | ||
| 15 | // convention): Wt[(c_in*Cg_out + c_out_local) * kL + kl], Cg_out = C_out/groups. | ||
| 16 | // causal_conv1d_update weights: depthwise, one row per channel: Wt[c*kL + kl]. | ||
| 17 | // | ||
| 18 | // ── Accumulation (matches the conv2d contract) ────────────────────────────── | ||
| 19 | // *_forward / *_backward_input / pad1d_* — output OVERWRITTEN. | ||
| 20 | // conv_transpose1d_backward_weight / _bias — dWt / dB ACCUMULATE (+=); | ||
| 21 | // caller zeros them first. | ||
| 22 | |||
| 23 | #include <brotensor/tensor.h> | ||
| 24 | #include <brotensor/detail/cpu/thread_pool.h> | ||
| 25 | |||
| 26 | #include <cstddef> | ||
| 27 | #include <stdexcept> | ||
| 28 | #include <string> | ||
| 29 | |||
| 30 | namespace brotensor::detail::cpu { | ||
| 31 | |||
| 32 | namespace { | ||
| 33 | |||
| 34 | ✗ | [[noreturn]] void fail(const char* op, const std::string& reason) { | |
| 35 | ✗ | throw std::runtime_error(std::string("brotensor: ") + op + ": " + reason); | |
| 36 | ✗ | } | |
| 37 | |||
| 38 | 151 | void check_groups(const char* op, int C_in, int C_out, int groups) { | |
| 39 |
1/2✓ Branch 0 taken 151 times.
✗ Branch 1 not taken.
|
151 | if (groups < 1 || C_in % groups != 0 || C_out % groups != 0) { |
| 40 | ✗ | fail(op, "groups must be >=1 and divide both C_in and C_out"); | |
| 41 | } | ||
| 42 | 151 | } | |
| 43 | |||
| 44 | 701 | void require_fp32(const char* op, const ::brotensor::Tensor& t, | |
| 45 | const char* name) { | ||
| 46 |
1/2✓ Branch 0 taken 701 times.
✗ Branch 1 not taken.
|
701 | if (t.dtype != ::brotensor::Dtype::FP32) { |
| 47 | ✗ | fail(op, std::string(name) + " must be FP32 (CPU backend is FP32-only)"); | |
| 48 | } | ||
| 49 | 701 | } | |
| 50 | |||
| 51 | // L_out of a 1D transposed convolution (torch ConvTranspose1d formula). | ||
| 52 | 151 | int convt1d_out_len(int L, int stride, int padding, int output_padding, | |
| 53 | int dilation, int kL) { | ||
| 54 | 302 | return (L - 1) * stride - 2 * padding + dilation * (kL - 1) | |
| 55 | 151 | + output_padding + 1; | |
| 56 | } | ||
| 57 | |||
| 58 | } // namespace | ||
| 59 | |||
| 60 | // ─── conv_transpose1d_forward ────────────────────────────────────────────── | ||
| 61 | 138 | void conv_transpose1d_forward(const ::brotensor::Tensor& X, | |
| 62 | const ::brotensor::Tensor& Wt, | ||
| 63 | const ::brotensor::Tensor* bias, | ||
| 64 | int N, int C_in, int L, int C_out, int kL, | ||
| 65 | int stride, int padding, int output_padding, | ||
| 66 | int dilation, int groups, | ||
| 67 | ::brotensor::Tensor& Y) { | ||
| 68 | 138 | const char* op = "conv_transpose1d_forward"; | |
| 69 | 138 | require_fp32(op, X, "X"); | |
| 70 | 138 | require_fp32(op, Wt, "Wt"); | |
| 71 |
2/2✓ Branch 0 taken 2 times.
✓ Branch 1 taken 136 times.
|
138 | if (bias) require_fp32(op, *bias, "bias"); |
| 72 | 138 | check_groups(op, C_in, C_out, groups); | |
| 73 |
1/2✓ Branch 0 taken 138 times.
✗ Branch 1 not taken.
|
138 | if (kL < 1 || stride < 1 || dilation < 1 || padding < 0 |
| 74 | 138 | || output_padding < 0) { | |
| 75 | ✗ | fail(op, "kL/stride/dilation must be >=1 and padding/output_padding >=0"); | |
| 76 | } | ||
| 77 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
138 | if (output_padding >= stride && output_padding >= dilation) { |
| 78 | ✗ | fail(op, "output_padding must be < stride or < dilation"); | |
| 79 | } | ||
| 80 | 138 | const int Cg_in = C_in / groups; | |
| 81 | 138 | const int Cg_out = C_out / groups; | |
| 82 | 276 | const int L_out = convt1d_out_len(L, stride, padding, output_padding, | |
| 83 | 138 | dilation, kL); | |
| 84 |
1/4✓ Branch 0 taken 138 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
138 | if (L_out <= 0) fail(op, "non-positive output length"); |
| 85 |
1/2✓ Branch 0 taken 138 times.
✗ Branch 1 not taken.
|
138 | if (Wt.rows != C_in || Wt.cols != Cg_out * kL) { |
| 86 | ✗ | fail(op, "Wt shape must be (C_in, (C_out/groups)*kL)"); | |
| 87 | } | ||
| 88 |
3/4✓ Branch 0 taken 2 times.
✓ Branch 1 taken 136 times.
✓ Branch 2 taken 136 times.
✗ Branch 3 not taken.
|
138 | if (bias && (bias->rows != C_out || bias->cols != 1)) { |
| 89 | ✗ | fail(op, "bias shape must be (C_out, 1)"); | |
| 90 | } | ||
| 91 | 138 | const int out_cols = C_out * L_out; | |
| 92 |
1/6✗ Branch 0 not taken.
✓ Branch 1 taken 138 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
|
138 | if (Y.rows != N || Y.cols != out_cols || Y.dtype != ::brotensor::Dtype::FP32) { |
| 93 | 138 | Y.resize(N, out_cols, ::brotensor::Dtype::FP32); | |
| 94 | 138 | } | |
| 95 |
2/4✓ Branch 0 taken 138 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 138 times.
|
138 | if (N == 0 || out_cols == 0) return; |
| 96 | |||
| 97 | 138 | const float* Xp = X.host_f32(); | |
| 98 | 138 | const float* Wp = Wt.host_f32(); | |
| 99 |
2/2✓ Branch 0 taken 136 times.
✓ Branch 1 taken 2 times.
|
138 | const float* Bp = bias ? bias->host_f32() : nullptr; |
| 100 | 138 | float* Yp = Y.host_f32_mut(); | |
| 101 | |||
| 102 | // Seed every output sample with its channel bias, then scatter-add. Each | ||
| 103 | // n exclusively owns Y's batch slice n, so this parallelizes across n | ||
| 104 | // with no cross-thread writes; it fully completes (parallel_for blocks) | ||
| 105 | // before the scatter-add pass below starts touching the same Y buffer. | ||
| 106 |
1/2✓ Branch 0 taken 138 times.
✗ Branch 1 not taken.
|
410 | parallel_for(static_cast<std::size_t>(N), [&](std::size_t ni) { |
| 107 | 272 | const int n = static_cast<int>(ni); | |
| 108 |
2/2✓ Branch 0 taken 1440 times.
✓ Branch 1 taken 272 times.
|
1712 | for (int oc = 0; oc < C_out; ++oc) { |
| 109 |
2/2✓ Branch 0 taken 1420 times.
✓ Branch 1 taken 20 times.
|
1440 | const float bv = Bp ? Bp[oc] : 0.0f; |
| 110 | 1440 | float* y_row = Yp + (static_cast<long>(n) * C_out + oc) * L_out; | |
| 111 |
2/2✓ Branch 0 taken 19327 times.
✓ Branch 1 taken 1440 times.
|
20767 | for (int lo = 0; lo < L_out; ++lo) y_row[lo] = bv; |
| 112 | 1440 | } | |
| 113 | 272 | }); | |
| 114 | // Interior region: input samples for which every kernel tap scatters | ||
| 115 | // into a valid output position (1D mirror of conv_transpose2d.cpp's | ||
| 116 | // split), computed once — independent of n/c_in. Only the thin border | ||
| 117 | // of input samples needs the per-tap bounds check; the interior runs a | ||
| 118 | // branch-free kl/oc_local loop. | ||
| 119 | 138 | int l_lo = (padding + stride - 1) / stride; | |
| 120 | 138 | int l_hi = L_out - 1 + padding - (kL - 1) * dilation; | |
| 121 |
1/2✓ Branch 0 taken 138 times.
✗ Branch 1 not taken.
|
138 | l_hi = (l_hi >= 0) ? (l_hi / stride) : -1; |
| 122 |
1/2✓ Branch 0 taken 138 times.
✗ Branch 1 not taken.
|
138 | if (l_lo < 0) l_lo = 0; |
| 123 |
1/2✓ Branch 0 taken 138 times.
✗ Branch 1 not taken.
|
138 | if (l_hi >= L) l_hi = L - 1; |
| 124 | 138 | const bool has_interior = l_lo <= l_hi; | |
| 125 | |||
| 126 | // Scatter: input sample (n, c_in, l) reaches l_out = l*stride - padding + | ||
| 127 | // kl*dilation in each output channel of c_in's group. Each n only ever | ||
| 128 | // scatters into Y's own batch slice n, so this parallelizes across n with | ||
| 129 | // no cross-thread writes — the c_in loop stays sequential per-n, so the | ||
| 130 | // += accumulation across c_in into a shared oc is untouched by another | ||
| 131 | // thread. | ||
| 132 |
1/2✓ Branch 0 taken 138 times.
✗ Branch 1 not taken.
|
410 | parallel_for(static_cast<std::size_t>(N), [&](std::size_t ni) { |
| 133 | 272 | const int n = static_cast<int>(ni); | |
| 134 |
2/2✓ Branch 0 taken 1065 times.
✓ Branch 1 taken 272 times.
|
1337 | for (int c_in = 0; c_in < C_in; ++c_in) { |
| 135 | 1065 | const int g = c_in / Cg_in; | |
| 136 | 1065 | const int oc_base = g * Cg_out; | |
| 137 | 1065 | const float* x_row = | |
| 138 | 1065 | Xp + (static_cast<long>(n) * C_in + c_in) * L; | |
| 139 | |||
| 140 | // Border sample: same bounds-checked scatter as before. | ||
| 141 | 2165 | auto scatter_bordered = [&](int l) { | |
| 142 | 1100 | const float xv = x_row[l]; | |
| 143 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1100 times.
|
1100 | if (xv == 0.0f) return; |
| 144 | 1100 | const int lo_origin = l * stride - padding; | |
| 145 |
2/2✓ Branch 0 taken 1100 times.
✓ Branch 1 taken 3253 times.
|
4353 | for (int kl = 0; kl < kL; ++kl) { |
| 146 | 3253 | const int lo = lo_origin + kl * dilation; | |
| 147 |
4/4✓ Branch 0 taken 2204 times.
✓ Branch 1 taken 1049 times.
✓ Branch 2 taken 49 times.
✓ Branch 3 taken 2155 times.
|
3253 | if (lo < 0 || lo >= L_out) continue; |
| 148 |
2/2✓ Branch 0 taken 10204 times.
✓ Branch 1 taken 2155 times.
|
12359 | for (int oc_local = 0; oc_local < Cg_out; ++oc_local) { |
| 149 | 10204 | const int oc = oc_base + oc_local; | |
| 150 | 10204 | const int w_idx = | |
| 151 | 10204 | (c_in * Cg_out + oc_local) * kL + kl; | |
| 152 | 10204 | Yp[(static_cast<long>(n) * C_out + oc) * L_out + lo] | |
| 153 | 20408 | += xv * Wp[w_idx]; | |
| 154 | 10204 | } | |
| 155 | 2155 | } | |
| 156 | 1100 | }; | |
| 157 | |||
| 158 | // Interior sample: every tap guaranteed in-bounds — no checks. | ||
| 159 | 6702 | auto scatter_interior = [&](int l) { | |
| 160 | 5637 | const float xv = x_row[l]; | |
| 161 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 5637 times.
|
5637 | if (xv == 0.0f) return; |
| 162 | 5637 | const int lo_origin = l * stride - padding; | |
| 163 |
2/2✓ Branch 0 taken 14326 times.
✓ Branch 1 taken 5637 times.
|
19963 | for (int kl = 0; kl < kL; ++kl) { |
| 164 | 14326 | const int lo = lo_origin + kl * dilation; | |
| 165 |
2/2✓ Branch 0 taken 58567 times.
✓ Branch 1 taken 14326 times.
|
72893 | for (int oc_local = 0; oc_local < Cg_out; ++oc_local) { |
| 166 | 58567 | const int oc = oc_base + oc_local; | |
| 167 | 58567 | const int w_idx = | |
| 168 | 58567 | (c_in * Cg_out + oc_local) * kL + kl; | |
| 169 | 58567 | Yp[(static_cast<long>(n) * C_out + oc) * L_out + lo] | |
| 170 | 117134 | += xv * Wp[w_idx]; | |
| 171 | 58567 | } | |
| 172 | 14326 | } | |
| 173 | 5637 | }; | |
| 174 | |||
| 175 |
1/2✓ Branch 0 taken 1065 times.
✗ Branch 1 not taken.
|
1065 | if (has_interior) { |
| 176 |
2/2✓ Branch 0 taken 1069 times.
✓ Branch 1 taken 1065 times.
|
2134 | for (int l = 0; l < l_lo; ++l) scatter_bordered(l); |
| 177 |
2/2✓ Branch 0 taken 5994 times.
✓ Branch 1 taken 1065 times.
|
7059 | for (int l = l_lo; l <= l_hi; ++l) scatter_interior(l); |
| 178 |
2/2✓ Branch 0 taken 36 times.
✓ Branch 1 taken 1065 times.
|
1101 | for (int l = l_hi + 1; l < L; ++l) scatter_bordered(l); |
| 179 | 1065 | } else { | |
| 180 | ✗ | for (int l = 0; l < L; ++l) scatter_bordered(l); | |
| 181 | } | ||
| 182 | 1065 | } | |
| 183 | 272 | }); | |
| 184 | 138 | } | |
| 185 | |||
| 186 | // ─── conv_transpose1d_backward_input ─────────────────────────────────────── | ||
| 187 | 7 | void conv_transpose1d_backward_input(const ::brotensor::Tensor& Wt, | |
| 188 | const ::brotensor::Tensor& dY, | ||
| 189 | int N, int C_in, int L, int C_out, int kL, | ||
| 190 | int stride, int padding, | ||
| 191 | int output_padding, int dilation, | ||
| 192 | int groups, ::brotensor::Tensor& dX) { | ||
| 193 | 7 | const char* op = "conv_transpose1d_backward_input"; | |
| 194 | 7 | require_fp32(op, Wt, "Wt"); | |
| 195 | 7 | require_fp32(op, dY, "dY"); | |
| 196 | 7 | check_groups(op, C_in, C_out, groups); | |
| 197 | 7 | const int Cg_in = C_in / groups; | |
| 198 | 7 | const int Cg_out = C_out / groups; | |
| 199 | 14 | const int L_out = convt1d_out_len(L, stride, padding, output_padding, | |
| 200 | 7 | dilation, kL); | |
| 201 |
1/4✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
7 | if (L_out <= 0) fail(op, "non-positive output length"); |
| 202 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (Wt.rows != C_in || Wt.cols != Cg_out * kL) { |
| 203 | ✗ | fail(op, "Wt shape must be (C_in, (C_out/groups)*kL)"); | |
| 204 | } | ||
| 205 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | if (dY.rows != N || dY.cols != C_out * L_out) { |
| 206 | ✗ | fail(op, "dY shape must be (N, C_out*L_out)"); | |
| 207 | } | ||
| 208 | 7 | const int in_cols = C_in * L; | |
| 209 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
7 | if (dX.rows != N || dX.cols != in_cols |
| 210 | ✗ | || dX.dtype != ::brotensor::Dtype::FP32) { | |
| 211 | 7 | dX.resize(N, in_cols, ::brotensor::Dtype::FP32); | |
| 212 | 7 | } | |
| 213 |
2/4✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
|
7 | if (N == 0 || in_cols == 0) return; |
| 214 | |||
| 215 | 7 | const float* Wp = Wt.host_f32(); | |
| 216 | 7 | const float* dYp = dY.host_f32(); | |
| 217 | 7 | float* dXp = dX.host_f32_mut(); | |
| 218 | |||
| 219 | // Adjoint of the transposed-conv scatter is a plain gather conv. Each n | ||
| 220 | // exclusively owns dX's batch slice n (dY/Wt are read-only), so this | ||
| 221 | // parallelizes across n with no cross-thread writes. | ||
| 222 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
19 | parallel_for(static_cast<std::size_t>(N), [&](std::size_t ni) { |
| 223 | 12 | const int n = static_cast<int>(ni); | |
| 224 |
2/2✓ Branch 0 taken 48 times.
✓ Branch 1 taken 12 times.
|
60 | for (int c_in = 0; c_in < C_in; ++c_in) { |
| 225 | 48 | const int g = c_in / Cg_in; | |
| 226 | 48 | const int oc_base = g * Cg_out; | |
| 227 |
2/2✓ Branch 0 taken 298 times.
✓ Branch 1 taken 48 times.
|
346 | for (int l = 0; l < L; ++l) { |
| 228 | 298 | const int lo_origin = l * stride - padding; | |
| 229 | 298 | float acc = 0.0f; | |
| 230 |
2/2✓ Branch 0 taken 892 times.
✓ Branch 1 taken 298 times.
|
1190 | for (int kl = 0; kl < kL; ++kl) { |
| 231 | 892 | const int lo = lo_origin + kl * dilation; | |
| 232 |
4/4✓ Branch 0 taken 844 times.
✓ Branch 1 taken 48 times.
✓ Branch 2 taken 833 times.
✓ Branch 3 taken 11 times.
|
892 | if (lo < 0 || lo >= L_out) continue; |
| 233 |
2/2✓ Branch 0 taken 3690 times.
✓ Branch 1 taken 833 times.
|
4523 | for (int oc_local = 0; oc_local < Cg_out; ++oc_local) { |
| 234 | 3690 | const int oc = oc_base + oc_local; | |
| 235 | 3690 | const int w_idx = | |
| 236 | 3690 | (c_in * Cg_out + oc_local) * kL + kl; | |
| 237 | 3690 | const int dy_idx = | |
| 238 | 3690 | (static_cast<long>(n) * C_out + oc) * L_out + lo; | |
| 239 | 3690 | acc += dYp[dy_idx] * Wp[w_idx]; | |
| 240 | 3690 | } | |
| 241 | 833 | } | |
| 242 | 298 | dXp[(static_cast<long>(n) * C_in + c_in) * L + l] = acc; | |
| 243 | 298 | } | |
| 244 | 48 | } | |
| 245 | 12 | }); | |
| 246 | 7 | } | |
| 247 | |||
| 248 | // ─── conv_transpose1d_backward_weight ────────────────────────────────────── | ||
| 249 | 6 | void conv_transpose1d_backward_weight(const ::brotensor::Tensor& X, | |
| 250 | const ::brotensor::Tensor& dY, | ||
| 251 | int N, int C_in, int L, int C_out, int kL, | ||
| 252 | int stride, int padding, | ||
| 253 | int output_padding, int dilation, | ||
| 254 | int groups, ::brotensor::Tensor& dWt) { | ||
| 255 | 6 | const char* op = "conv_transpose1d_backward_weight"; | |
| 256 | 6 | require_fp32(op, X, "X"); | |
| 257 | 6 | require_fp32(op, dY, "dY"); | |
| 258 | 6 | require_fp32(op, dWt, "dWt"); | |
| 259 | 6 | check_groups(op, C_in, C_out, groups); | |
| 260 | 6 | const int Cg_in = C_in / groups; | |
| 261 | 6 | const int Cg_out = C_out / groups; | |
| 262 | 12 | const int L_out = convt1d_out_len(L, stride, padding, output_padding, | |
| 263 | 6 | dilation, kL); | |
| 264 |
1/4✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
6 | if (L_out <= 0) fail(op, "non-positive output length"); |
| 265 |
1/2✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
|
6 | if (dWt.rows != C_in || dWt.cols != Cg_out * kL) { |
| 266 | ✗ | fail(op, "dWt shape must be (C_in, (C_out/groups)*kL)"); | |
| 267 | } | ||
| 268 |
1/2✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
|
6 | if (X.rows != N || X.cols != C_in * L) { |
| 269 | ✗ | fail(op, "X shape must be (N, C_in*L)"); | |
| 270 | } | ||
| 271 |
1/2✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
|
6 | if (dY.rows != N || dY.cols != C_out * L_out) { |
| 272 | ✗ | fail(op, "dY shape must be (N, C_out*L_out)"); | |
| 273 | } | ||
| 274 |
3/6✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 6 times.
|
6 | if (C_in == 0 || Cg_out == 0 || kL == 0) return; |
| 275 | |||
| 276 | 6 | const float* Xp = X.host_f32(); | |
| 277 | 6 | const float* dYp = dY.host_f32(); | |
| 278 | 6 | float* dWp = dWt.host_f32_mut(); | |
| 279 | |||
| 280 | // One accumulation per weight element; += into dWt (caller zeroed it). | ||
| 281 | // | ||
| 282 | // NOT parallelized over n: n is the innermost reduction axis (every | ||
| 283 | // batch item's contribution sums into the same dWp element), not an | ||
| 284 | // outer axis — parallelizing it would race every thread on the same | ||
| 285 | // dWt element. Left single-threaded per this task's scope. | ||
| 286 |
2/2✓ Branch 0 taken 6 times.
✓ Branch 1 taken 23 times.
|
29 | for (int c_in = 0; c_in < C_in; ++c_in) { |
| 287 | 23 | const int g = c_in / Cg_in; | |
| 288 | 23 | const int oc_base = g * Cg_out; | |
| 289 |
2/2✓ Branch 0 taken 99 times.
✓ Branch 1 taken 23 times.
|
122 | for (int oc_local = 0; oc_local < Cg_out; ++oc_local) { |
| 290 | 99 | const int oc = oc_base + oc_local; | |
| 291 |
2/2✓ Branch 0 taken 312 times.
✓ Branch 1 taken 99 times.
|
411 | for (int kl = 0; kl < kL; ++kl) { |
| 292 | 312 | float acc = 0.0f; | |
| 293 |
2/2✓ Branch 0 taken 528 times.
✓ Branch 1 taken 312 times.
|
840 | for (int n = 0; n < N; ++n) { |
| 294 | 528 | const float* x_row = | |
| 295 | 528 | Xp + (static_cast<long>(n) * C_in + c_in) * L; | |
| 296 | 528 | const float* dy_row = | |
| 297 | 528 | dYp + (static_cast<long>(n) * C_out + oc) * L_out; | |
| 298 |
2/2✓ Branch 0 taken 3396 times.
✓ Branch 1 taken 528 times.
|
3924 | for (int l = 0; l < L; ++l) { |
| 299 | 3396 | const int lo = l * stride - padding + kl * dilation; | |
| 300 |
4/4✓ Branch 0 taken 3261 times.
✓ Branch 1 taken 135 times.
✓ Branch 2 taken 48 times.
✓ Branch 3 taken 3213 times.
|
3396 | if (lo < 0 || lo >= L_out) continue; |
| 301 | 3213 | acc += x_row[l] * dy_row[lo]; | |
| 302 | 3213 | } | |
| 303 | 528 | } | |
| 304 | 312 | dWp[(c_in * Cg_out + oc_local) * kL + kl] += acc; | |
| 305 | 312 | } | |
| 306 | 99 | } | |
| 307 | 23 | } | |
| 308 | 6 | } | |
| 309 | |||
| 310 | // ─── conv_transpose1d_backward_bias ──────────────────────────────────────── | ||
| 311 | 5 | void conv_transpose1d_backward_bias(const ::brotensor::Tensor& dY, | |
| 312 | int N, int C_out, int L_out, | ||
| 313 | ::brotensor::Tensor& dB) { | ||
| 314 | 5 | const char* op = "conv_transpose1d_backward_bias"; | |
| 315 | 5 | require_fp32(op, dY, "dY"); | |
| 316 | 5 | require_fp32(op, dB, "dB"); | |
| 317 |
1/2✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
|
5 | if (dB.rows != C_out || dB.cols != 1) { |
| 318 | ✗ | fail(op, "dB shape must be (C_out, 1)"); | |
| 319 | } | ||
| 320 |
1/2✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
|
5 | if (dY.rows != N || dY.cols != C_out * L_out) { |
| 321 | ✗ | fail(op, "dY shape must be (N, C_out*L_out)"); | |
| 322 | } | ||
| 323 |
3/6✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 5 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 5 times.
|
5 | if (C_out == 0 || N == 0 || L_out == 0) return; |
| 324 | |||
| 325 | 5 | const float* dYp = dY.host_f32(); | |
| 326 | 5 | float* dBp = dB.host_f32_mut(); | |
| 327 | |||
| 328 | // Per-channel sum over (N, L_out); += into dB (caller zeroed it). | ||
| 329 | // | ||
| 330 | // NOT parallelized over n: same reason as backward_weight above — n is | ||
| 331 | // the reduction axis here, not the outer axis. | ||
| 332 |
2/2✓ Branch 0 taken 5 times.
✓ Branch 1 taken 29 times.
|
34 | for (int oc = 0; oc < C_out; ++oc) { |
| 333 | 29 | float acc = 0.0f; | |
| 334 |
2/2✓ Branch 0 taken 53 times.
✓ Branch 1 taken 29 times.
|
82 | for (int n = 0; n < N; ++n) { |
| 335 | 53 | const float* dy_row = | |
| 336 | 53 | dYp + (static_cast<long>(n) * C_out + oc) * L_out; | |
| 337 |
2/2✓ Branch 0 taken 631 times.
✓ Branch 1 taken 53 times.
|
684 | for (int lo = 0; lo < L_out; ++lo) acc += dy_row[lo]; |
| 338 | 53 | } | |
| 339 | 29 | dBp[oc] += acc; | |
| 340 | 29 | } | |
| 341 | 5 | } | |
| 342 | |||
| 343 | // ─── causal_conv1d_update ────────────────────────────────────────────────── | ||
| 344 | 22 | void causal_conv1d_update(const ::brotensor::Tensor& X, | |
| 345 | const ::brotensor::Tensor& Wt, | ||
| 346 | const ::brotensor::Tensor* bias, | ||
| 347 | int N, int C, int L_step, int kL, int dilation, | ||
| 348 | ::brotensor::Tensor& state, ::brotensor::Tensor& Y) { | ||
| 349 | 22 | const char* op = "causal_conv1d_update"; | |
| 350 | 22 | require_fp32(op, X, "X"); | |
| 351 | 22 | require_fp32(op, Wt, "Wt"); | |
| 352 |
2/2✓ Branch 0 taken 1 time.
✓ Branch 1 taken 21 times.
|
22 | if (bias) require_fp32(op, *bias, "bias"); |
| 353 | 22 | require_fp32(op, state, "state"); | |
| 354 |
1/2✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
|
22 | if (kL < 1 || dilation < 1 || L_step < 1 || N < 0 || C < 1) { |
| 355 | ✗ | fail(op, "kL/dilation/L_step/C must be >=1 and N >=0"); | |
| 356 | } | ||
| 357 |
1/2✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
|
22 | if (Wt.rows != C || Wt.cols != kL) { |
| 358 | ✗ | fail(op, "Wt shape must be (C, kL) — one depthwise filter per channel"); | |
| 359 | } | ||
| 360 |
3/4✓ Branch 0 taken 1 time.
✓ Branch 1 taken 21 times.
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
|
22 | if (bias && (bias->rows != C || bias->cols != 1)) { |
| 361 | ✗ | fail(op, "bias shape must be (C, 1)"); | |
| 362 | } | ||
| 363 | 22 | const int hist = (kL - 1) * dilation; // state samples per channel | |
| 364 |
1/2✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
|
22 | if (state.rows != N || state.cols != C * hist) { |
| 365 | ✗ | fail(op, "state shape must be (N, C*(kL-1)*dilation)"); | |
| 366 | } | ||
| 367 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
22 | if (Y.rows != N || Y.cols != C * L_step |
| 368 | ✗ | || Y.dtype != ::brotensor::Dtype::FP32) { | |
| 369 | 22 | Y.resize(N, C * L_step, ::brotensor::Dtype::FP32); | |
| 370 | 22 | } | |
| 371 |
3/6✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 22 times.
|
22 | if (N == 0 || C == 0 || L_step == 0) return; |
| 372 | |||
| 373 | 22 | const float* Xp = X.host_f32(); | |
| 374 | 22 | const float* Wp = Wt.host_f32(); | |
| 375 |
2/2✓ Branch 0 taken 1 time.
✓ Branch 1 taken 21 times.
|
22 | const float* Bp = bias ? bias->host_f32() : nullptr; |
| 376 | 22 | float* Sp = state.host_f32_mut(); | |
| 377 | 22 | float* Yp = Y.host_f32_mut(); | |
| 378 | |||
| 379 | 22 | const int buf_len = hist + L_step; // [state ++ new] window per channel | |
| 380 | |||
| 381 |
2/2✓ Branch 0 taken 44 times.
✓ Branch 1 taken 22 times.
|
66 | for (int n = 0; n < N; ++n) { |
| 382 |
2/2✓ Branch 0 taken 223 times.
✓ Branch 1 taken 44 times.
|
267 | for (int c = 0; c < C; ++c) { |
| 383 | 223 | const float* s_row = | |
| 384 | 223 | Sp + (static_cast<long>(n) * C + c) * hist; | |
| 385 | 223 | const float* x_row = | |
| 386 | 223 | Xp + (static_cast<long>(n) * C + c) * L_step; | |
| 387 | 223 | const float* w_row = Wp + static_cast<long>(c) * kL; | |
| 388 |
2/2✓ Branch 0 taken 215 times.
✓ Branch 1 taken 8 times.
|
223 | const float bv = Bp ? Bp[c] : 0.0f; |
| 389 | // buf index helper: [0, hist) -> state, [hist, buf_len) -> new. | ||
| 390 | 2665 | auto buf_at = [&](int idx) -> float { | |
| 391 |
2/2✓ Branch 0 taken 1270 times.
✓ Branch 1 taken 1172 times.
|
2442 | return idx < hist ? s_row[idx] : x_row[idx - hist]; |
| 392 | }; | ||
| 393 | // Output sample t convolves buf[t .. t + hist] (causal). | ||
| 394 |
2/2✓ Branch 0 taken 642 times.
✓ Branch 1 taken 223 times.
|
865 | for (int t = 0; t < L_step; ++t) { |
| 395 | 642 | float acc = bv; | |
| 396 |
2/2✓ Branch 0 taken 2442 times.
✓ Branch 1 taken 642 times.
|
3084 | for (int kl = 0; kl < kL; ++kl) { |
| 397 | 2442 | acc += w_row[kl] * buf_at(t + kl * dilation); | |
| 398 | 2442 | } | |
| 399 | 642 | Yp[(static_cast<long>(n) * C + c) * L_step + t] = acc; | |
| 400 | 642 | } | |
| 401 | 223 | } | |
| 402 | 44 | } | |
| 403 | // Roll the state forward: new state = last `hist` samples of the window. | ||
| 404 | // Done after all reads above so a tiny scratch suffices per (n,c). | ||
| 405 |
2/2✓ Branch 0 taken 22 times.
✓ Branch 1 taken 44 times.
|
66 | for (int n = 0; n < N; ++n) { |
| 406 |
2/2✓ Branch 0 taken 223 times.
✓ Branch 1 taken 44 times.
|
267 | for (int c = 0; c < C; ++c) { |
| 407 | 223 | float* s_row = Sp + (static_cast<long>(n) * C + c) * hist; | |
| 408 | 223 | const float* x_row = | |
| 409 | 223 | Xp + (static_cast<long>(n) * C + c) * L_step; | |
| 410 |
2/2✓ Branch 0 taken 836 times.
✓ Branch 1 taken 223 times.
|
1059 | for (int i = 0; i < hist; ++i) { |
| 411 | 836 | const int idx = buf_len - hist + i; // tail of the window | |
| 412 |
2/2✓ Branch 0 taken 292 times.
✓ Branch 1 taken 544 times.
|
836 | s_row[i] = idx < hist |
| 413 | 292 | ? s_row[idx] // still inside old state | |
| 414 | 544 | : x_row[idx - hist]; // inside new input | |
| 415 | 836 | } | |
| 416 | 223 | } | |
| 417 | 44 | } | |
| 418 | 22 | } | |
| 419 | |||
| 420 | // ─── pad1d_forward ───────────────────────────────────────────────────────── | ||
| 421 | namespace { | ||
| 422 | |||
| 423 | // Map an output position p in [0, L_pad) to a source index in [0, L) for the | ||
| 424 | // given mode, or return -1 for a zero-padded position. p covers | ||
| 425 | // [pad_left, pad_left + L) as the copied interior. | ||
| 426 | 7236 | int pad1d_src(const char* op, int p, int L, int pad_left, int mode) { | |
| 427 | 7236 | const int rel = p - pad_left; // index into the original [0, L) | |
| 428 |
4/4✓ Branch 0 taken 5704 times.
✓ Branch 1 taken 1532 times.
✓ Branch 2 taken 1304 times.
✓ Branch 3 taken 4400 times.
|
7236 | if (rel >= 0 && rel < L) return rel; // interior — straight copy |
| 429 |
2/2✓ Branch 0 taken 1752 times.
✓ Branch 1 taken 1084 times.
|
2836 | if (mode == 0) return -1; // zero |
| 430 |
2/2✓ Branch 0 taken 878 times.
✓ Branch 1 taken 874 times.
|
1752 | if (mode == 2) { // replicate (clamp to edge) |
| 431 |
2/2✓ Branch 0 taken 436 times.
✓ Branch 1 taken 442 times.
|
878 | return rel < 0 ? 0 : L - 1; |
| 432 | } | ||
| 433 | // mode == 1: reflect without repeating the edge sample (numpy 'reflect'). | ||
| 434 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 874 times.
|
874 | if (L == 1) return 0; |
| 435 | 874 | int q = rel; | |
| 436 | 874 | const int period = 2 * (L - 1); | |
| 437 | 874 | q %= period; | |
| 438 |
2/2✓ Branch 0 taken 428 times.
✓ Branch 1 taken 446 times.
|
874 | if (q < 0) q += period; |
| 439 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 874 times.
|
874 | return q < L ? q : period - q; |
| 440 | 7236 | } | |
| 441 | |||
| 442 | } // namespace | ||
| 443 | |||
| 444 | 154 | void pad1d_forward(const ::brotensor::Tensor& X, int N, int C, int L, | |
| 445 | int pad_left, int pad_right, int mode, | ||
| 446 | ::brotensor::Tensor& Y) { | ||
| 447 | 154 | const char* op = "pad1d_forward"; | |
| 448 | 154 | require_fp32(op, X, "X"); | |
| 449 |
1/4✓ Branch 0 taken 154 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
154 | if (N < 0 || C < 1 || L < 1) fail(op, "C/L must be >=1 and N >=0"); |
| 450 |
1/4✓ Branch 0 taken 154 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
154 | if (pad_left < 0 || pad_right < 0) fail(op, "pad counts must be >=0"); |
| 451 |
1/4✓ Branch 0 taken 154 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
154 | if (mode < 0 || mode > 2) fail(op, "mode must be 0 (zero), 1 (reflect) or 2 (replicate)"); |
| 452 |
3/4✓ Branch 0 taken 104 times.
✓ Branch 1 taken 50 times.
✓ Branch 2 taken 50 times.
✗ Branch 3 not taken.
|
154 | if (mode == 1 && (pad_left >= L || pad_right >= L)) { |
| 453 | ✗ | fail(op, "reflect padding requires pad_left and pad_right < L"); | |
| 454 | } | ||
| 455 |
1/2✓ Branch 0 taken 154 times.
✗ Branch 1 not taken.
|
154 | if (X.rows != N || X.cols != C * L) { |
| 456 | ✗ | fail(op, "X shape must be (N, C*L)"); | |
| 457 | } | ||
| 458 | 154 | const int L_pad = L + pad_left + pad_right; | |
| 459 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 154 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
154 | if (Y.rows != N || Y.cols != C * L_pad |
| 460 | ✗ | || Y.dtype != ::brotensor::Dtype::FP32) { | |
| 461 | 154 | Y.resize(N, C * L_pad, ::brotensor::Dtype::FP32); | |
| 462 | 154 | } | |
| 463 |
2/4✓ Branch 0 taken 154 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 154 times.
|
154 | if (N == 0 || C == 0) return; |
| 464 | |||
| 465 | 154 | const float* Xp = X.host_f32(); | |
| 466 | 154 | float* Yp = Y.host_f32_mut(); | |
| 467 | |||
| 468 |
2/2✓ Branch 0 taken 154 times.
✓ Branch 1 taken 307 times.
|
461 | for (int n = 0; n < N; ++n) { |
| 469 |
2/2✓ Branch 0 taken 650 times.
✓ Branch 1 taken 307 times.
|
957 | for (int c = 0; c < C; ++c) { |
| 470 | 650 | const float* x_row = Xp + (static_cast<long>(n) * C + c) * L; | |
| 471 | 650 | float* y_row = Yp + (static_cast<long>(n) * C + c) * L_pad; | |
| 472 |
2/2✓ Branch 0 taken 6920 times.
✓ Branch 1 taken 650 times.
|
7570 | for (int p = 0; p < L_pad; ++p) { |
| 473 | 6920 | const int src = pad1d_src(op, p, L, pad_left, mode); | |
| 474 |
2/2✓ Branch 0 taken 5882 times.
✓ Branch 1 taken 1038 times.
|
6920 | y_row[p] = src < 0 ? 0.0f : x_row[src]; |
| 475 | 6920 | } | |
| 476 | 650 | } | |
| 477 | 307 | } | |
| 478 | 154 | } | |
| 479 | |||
| 480 | // ─── pad1d_backward ──────────────────────────────────────────────────────── | ||
| 481 | 6 | void pad1d_backward(const ::brotensor::Tensor& dY, int N, int C, int L, | |
| 482 | int pad_left, int pad_right, int mode, | ||
| 483 | ::brotensor::Tensor& dX) { | ||
| 484 | 6 | const char* op = "pad1d_backward"; | |
| 485 | 6 | require_fp32(op, dY, "dY"); | |
| 486 |
1/4✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
6 | if (N < 0 || C < 1 || L < 1) fail(op, "C/L must be >=1 and N >=0"); |
| 487 |
1/4✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
6 | if (pad_left < 0 || pad_right < 0) fail(op, "pad counts must be >=0"); |
| 488 |
1/4✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
6 | if (mode < 0 || mode > 2) fail(op, "mode must be 0 (zero), 1 (reflect) or 2 (replicate)"); |
| 489 |
3/4✓ Branch 0 taken 4 times.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
|
6 | if (mode == 1 && (pad_left >= L || pad_right >= L)) { |
| 490 | ✗ | fail(op, "reflect padding requires pad_left and pad_right < L"); | |
| 491 | } | ||
| 492 | 6 | const int L_pad = L + pad_left + pad_right; | |
| 493 |
1/2✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
|
6 | if (dY.rows != N || dY.cols != C * L_pad) { |
| 494 | ✗ | fail(op, "dY shape must be (N, C*(L+pad_left+pad_right))"); | |
| 495 | } | ||
| 496 |
1/4✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
|
6 | if (dX.rows != N || dX.cols != C * L |
| 497 | ✗ | || dX.dtype != ::brotensor::Dtype::FP32) { | |
| 498 | 6 | dX.resize(N, C * L, ::brotensor::Dtype::FP32); | |
| 499 | 6 | } | |
| 500 |
2/4✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 6 times.
|
6 | if (N == 0 || C == 0) return; |
| 501 | |||
| 502 | 6 | const float* dYp = dY.host_f32(); | |
| 503 | 6 | float* dXp = dX.host_f32_mut(); | |
| 504 | |||
| 505 | // Adjoint: scatter each output gradient onto the input sample it read. | ||
| 506 |
2/2✓ Branch 0 taken 6 times.
✓ Branch 1 taken 11 times.
|
17 | for (int n = 0; n < N; ++n) { |
| 507 |
2/2✓ Branch 0 taken 28 times.
✓ Branch 1 taken 11 times.
|
39 | for (int c = 0; c < C; ++c) { |
| 508 | 28 | const float* dy_row = dYp + (static_cast<long>(n) * C + c) * L_pad; | |
| 509 | 28 | float* dx_row = dXp + (static_cast<long>(n) * C + c) * L; | |
| 510 |
2/2✓ Branch 0 taken 176 times.
✓ Branch 1 taken 28 times.
|
204 | for (int l = 0; l < L; ++l) dx_row[l] = 0.0f; |
| 511 |
2/2✓ Branch 0 taken 316 times.
✓ Branch 1 taken 28 times.
|
344 | for (int p = 0; p < L_pad; ++p) { |
| 512 | 316 | const int src = pad1d_src(op, p, L, pad_left, mode); | |
| 513 |
2/2✓ Branch 0 taken 46 times.
✓ Branch 1 taken 270 times.
|
316 | if (src >= 0) dx_row[src] += dy_row[p]; |
| 514 | 316 | } | |
| 515 | 28 | } | |
| 516 | 11 | } | |
| 517 | 6 | } | |
| 518 | |||
| 519 | } // namespace brotensor::detail::cpu | ||
| 520 |