GCC Code Coverage Report

Directory: brotensor/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 92.2% 588 / 0 / 638
Functions: 100.0% 85 / 0 / 85
Branches: 53.5% 106 / 0 / 198
src/ops_quant_audio.cpp
Line Branch Exec Source
1 // brotensor public op wrappers — diffusion, quantisation, spectral/FFT, 1D audio, sampling, noise.
2
3 #include <brotensor/ops.h>
4 #include <brotensor/tensor.h>
5 #include <brotensor/detail/dispatch.h>
6 #include <brotensor/detail/cpu/thread_pool.h>
7
8 #include <cmath>
9 #include <cstdint>
10 #include <stdexcept>
11
12 namespace brotensor {
13
14 namespace {
15
16 163 inline void check_quant_block_cols(const char* op, const Tensor& t,
17 const char* name, Dtype expected_dt) {
18 163 detail::check_quant_cols(op, t, name);
19 163 const int bs = dtype_block_size(expected_dt);
20
2/2
✓ Branch 0 taken 145 times.
✓ Branch 1 taken 18 times.
 163 if (t.rows <= 0 || t.cols <= 0 || (t.cols % bs) != 0) {
21 18 detail::throw_quant_cols_error(op, name, expected_dt, t.rows, t.cols);
22 }
23 145 }
24
25 } // namespace
26
27 // ─── Diffusion sampler steps + timestep embedding ──────────────────────────
28
29 20 void ddim_step(const Tensor& x_t, const Tensor& eps_pred,
30 float alpha_t, float alpha_prev, float sigma_t, Tensor& x_prev) {
31 20 const auto& v = detail::dispatch(x_t, eps_pred, x_prev);
32
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (!v.ddim_step) detail::throw_not_implemented("ddim_step", x_t.device);
33 20 detail::adopt_output(x_prev, x_t.device);
34 20 v.ddim_step(x_t, eps_pred, alpha_t, alpha_prev, sigma_t, x_prev);
35 20 }
36
37 18 void euler_step(const Tensor& x_t, const Tensor& eps_pred,
38 float sigma_t, float sigma_prev, Tensor& x_prev) {
39 18 const auto& v = detail::dispatch(x_t, eps_pred, x_prev);
40
1/2
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
 18 if (!v.euler_step) detail::throw_not_implemented("euler_step", x_t.device);
41 18 detail::adopt_output(x_prev, x_t.device);
42 18 v.euler_step(x_t, eps_pred, sigma_t, sigma_prev, x_prev);
43 18 }
44
45 18 void dpmpp_2m_step(const Tensor& x_t, const Tensor& eps_pred,
46 const Tensor& x0_prev, float sigma_t,
47 float c_xt, float c_x0t, float c_x0prev,
48 Tensor& x_prev, Tensor& x0_out) {
49 18 const auto& v = detail::dispatch(x_t, eps_pred, x0_prev, x_prev, x0_out);
50
1/2
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
 18 if (!v.dpmpp_2m_step) detail::throw_not_implemented("dpmpp_2m_step", x_t.device);
51 18 detail::adopt_output(x_prev, x_t.device);
52 18 detail::adopt_output(x0_out, x_t.device);
53 36 v.dpmpp_2m_step(x_t, eps_pred, x0_prev, sigma_t, c_xt, c_x0t, c_x0prev,
54 18 x_prev, x0_out);
55 18 }
56
57 22 void timestep_embedding(const Tensor& timesteps, int dim, float max_period,
58 Tensor& Y) {
59 22 const auto& v = detail::dispatch(timesteps, Y);
60
1/2
✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
 22 if (!v.timestep_embedding) detail::throw_not_implemented("timestep_embedding", timesteps.device);
61 22 detail::adopt_output(Y, timesteps.device);
62 22 v.timestep_embedding(timesteps, dim, max_period, Y);
63 22 }
64
65 // ─── INT8 / block-quant weight-only quantisation ───────────────────────────
66
67 82 void quantize_int8_per_row_host(const uint16_t* W_fp16,
68 int out, int in,
69 int8_t* W_int8_out,
70 float* scales_out) {
71
2/4
✓ Branch 0 taken 82 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 82 times.
 82 if (out <= 0 || in <= 0) {
72 ✗ for (int r = 0; r < out; ++r) scales_out[r] = 0.0f;
73 ✗ return;
74 }
75
1/2
✓ Branch 0 taken 82 times.
✗ Branch 1 not taken.
 22933 detail::cpu::parallel_for(static_cast<std::size_t>(out), [&](std::size_t r) {
76 22851 const uint16_t* row = W_fp16 + r * static_cast<std::size_t>(in);
77 22851 float amax = 0.0f;
78
2/2
✓ Branch 0 taken 34310946 times.
✓ Branch 1 taken 22851 times.
 34333797 for (int c = 0; c < in; ++c) {
79 34310946 const float v = fp16_bits_to_fp32(row[c]);
80 34310946 const float a = std::fabs(v);
81
2/2
✓ Branch 0 taken 34148674 times.
✓ Branch 1 taken 162272 times.
 34310946 if (a > amax) amax = a;
82 34310946 }
83
2/2
✓ Branch 0 taken 22849 times.
✓ Branch 1 taken 2 times.
 22851 const float scale = (amax > 0.0f) ? (amax / 127.0f) : 0.0f;
84
2/2
✓ Branch 0 taken 22849 times.
✓ Branch 1 taken 2 times.
 22851 const float inv = (scale > 0.0f) ? (1.0f / scale) : 0.0f;
85 22851 scales_out[r] = scale;
86 22851 int8_t* dst = W_int8_out + r * static_cast<std::size_t>(in);
87
2/2
✓ Branch 0 taken 35825804 times.
✓ Branch 1 taken 22851 times.
 35848655 for (int c = 0; c < in; ++c) {
88 35825804 const float v = fp16_bits_to_fp32(row[c]);
89 35825804 int q = static_cast<int>(std::lrint(v * inv));
90
1/2
✓ Branch 0 taken 35825804 times.
✗ Branch 1 not taken.
 35825804 if (q < -127) q = -127;
91
1/2
✓ Branch 0 taken 35825804 times.
✗ Branch 1 not taken.
 35825804 if (q > 127) q = 127;
92 35825804 dst[c] = static_cast<int8_t>(q);
93 35825804 }
94 22851 });
95 82 }
96
97 3 void matmul_int8w_fp16(const Tensor& W_int8, const Tensor& scales,
98 const Tensor& X, Tensor& Y) {
99 3 const auto& v = detail::dispatch(W_int8, scales, X, Y);
100
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 1 time.
 3 if (!v.matmul_int8w_fp16) detail::throw_not_implemented("matmul_int8w_fp16", W_int8.device);
101 2 detail::adopt_output(Y, W_int8.device);
102 2 v.matmul_int8w_fp16(W_int8, scales, X, Y);
103 2 }
104
105 18 void linear_forward_batched_int8w_fp16(const Tensor& W_int8,
106 const Tensor& scales,
107 const Tensor* bias,
108 const Tensor& X_BD, Tensor& Y_BD) {
109 18 const auto& v = detail::dispatch_with_opts(W_int8, scales, {bias, &X_BD, &Y_BD});
110
1/2
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
 18 if (!v.linear_forward_batched_int8w_fp16)
111 ✗ detail::throw_not_implemented("linear_forward_batched_int8w_fp16", W_int8.device);
112 18 detail::adopt_output(Y_BD, W_int8.device);
113 18 v.linear_forward_batched_int8w_fp16(W_int8, scales, bias, X_BD, Y_BD);
114 18 }
115
116 74 void quantize_rows(const Tensor& W, Dtype fmt, Tensor& Wq, Tensor& scales) {
117 74 const auto& v = detail::dispatch_with_opts(W, {&Wq, &scales});
118
1/2
✓ Branch 0 taken 74 times.
✗ Branch 1 not taken.
 74 if (!v.quantize_rows) detail::throw_not_implemented("quantize_rows", W.device);
119 74 detail::adopt_output(Wq, W.device);
120 74 detail::adopt_output(scales, W.device);
121 74 v.quantize_rows(W, fmt, Wq, scales);
122 74 }
123
124 8 void dequantize_rows(const Tensor& Wq, const Tensor& scales, Dtype out, Tensor& W) {
125 8 BROTENSOR_CHECK_QUANT_COLS("dequantize_rows", Wq);
126 8 const auto& v = detail::dispatch_with_opts(Wq, {&W});
127
1/2
✓ Branch 0 taken 8 times.
✗ Branch 1 not taken.
 8 if (!v.dequantize_rows) detail::throw_not_implemented("dequantize_rows", Wq.device);
128 8 detail::adopt_output(W, Wq.device);
129 8 v.dequantize_rows(Wq, scales, out, W);
130 8 }
131
132 68 void linear_forward_batched_wq_ex(const Tensor& Wq, const Tensor& scales, const Tensor* bias,
133 const Tensor& X_BD, int act, int epilogue, Tensor& Y_BD) {
134 68 BROTENSOR_CHECK_QUANT_COLS("linear_forward_batched_wq_ex", Wq);
135 68 const auto& v = detail::dispatch_with_opts(Wq, scales, {bias, &X_BD, &Y_BD});
136
1/2
✓ Branch 0 taken 68 times.
✗ Branch 1 not taken.
 68 if (!v.linear_forward_batched_wq_ex)
137 ✗ detail::throw_not_implemented("linear_forward_batched_wq_ex", Wq.device);
138 68 detail::adopt_output(Y_BD, Wq.device);
139 68 v.linear_forward_batched_wq_ex(Wq, scales, bias, X_BD, act, epilogue, Y_BD);
140 68 }
141
142 5 void dequant_q4k_to_fp16(const Tensor& W_q4k, Tensor& W_fp16) {
143 5 check_quant_block_cols("dequant_q4k_to_fp16", W_q4k, "W_q4k", Dtype::Q4_K);
144 5 const auto& v = detail::dispatch(W_q4k, W_fp16);
145
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (!v.dequant_q4k_to_fp16)
146 ✗ detail::throw_not_implemented("dequant_q4k_to_fp16", W_q4k.device);
147 5 detail::adopt_output(W_fp16, W_q4k.device);
148 5 v.dequant_q4k_to_fp16(W_q4k, W_fp16);
149 5 }
150
151 40 void linear_forward_q4k_fp16(const Tensor& W_q4k, const Tensor* bias,
152 const Tensor& x, Tensor& y) {
153 40 check_quant_block_cols("linear_forward_q4k_fp16", W_q4k, "W_q4k", Dtype::Q4_K);
154 40 const auto& v = detail::dispatch_with_opts(W_q4k, {bias, &x, &y});
155
1/2
✓ Branch 0 taken 40 times.
✗ Branch 1 not taken.
 40 if (!v.linear_forward_q4k_fp16)
156 ✗ detail::throw_not_implemented("linear_forward_q4k_fp16", W_q4k.device);
157 40 detail::adopt_output(y, W_q4k.device);
158 40 v.linear_forward_q4k_fp16(W_q4k, bias, x, y);
159 40 }
160
161 6 void linear_forward_batched_q4k_fp16(const Tensor& W_q4k, const Tensor* bias,
162 const Tensor& X_BD, Tensor& Y_BD) {
163 6 check_quant_block_cols("linear_forward_batched_q4k_fp16", W_q4k, "W_q4k", Dtype::Q4_K);
164 6 const auto& v = detail::dispatch_with_opts(W_q4k, {bias, &X_BD, &Y_BD});
165
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 if (!v.linear_forward_batched_q4k_fp16)
166 ✗ detail::throw_not_implemented("linear_forward_batched_q4k_fp16", W_q4k.device);
167 6 detail::adopt_output(Y_BD, W_q4k.device);
168 6 v.linear_forward_batched_q4k_fp16(W_q4k, bias, X_BD, Y_BD);
169 6 }
170
171 7 void dequant_q8_0_to_fp16(const Tensor& W_q8, Tensor& W_fp16) {
172 7 check_quant_block_cols("dequant_q8_0_to_fp16", W_q8, "W_q8", Dtype::Q8_0);
173 7 const auto& v = detail::dispatch(W_q8, W_fp16);
174
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 if (!v.dequant_q8_0_to_fp16)
175 ✗ detail::throw_not_implemented("dequant_q8_0_to_fp16", W_q8.device);
176 7 detail::adopt_output(W_fp16, W_q8.device);
177 7 v.dequant_q8_0_to_fp16(W_q8, W_fp16);
178 7 }
179
180 40 void linear_forward_q8_0_fp16(const Tensor& W_q8, const Tensor* bias,
181 const Tensor& x, Tensor& y) {
182 40 check_quant_block_cols("linear_forward_q8_0_fp16", W_q8, "W_q8", Dtype::Q8_0);
183 40 const auto& v = detail::dispatch_with_opts(W_q8, {bias, &x, &y});
184
1/2
✓ Branch 0 taken 40 times.
✗ Branch 1 not taken.
 40 if (!v.linear_forward_q8_0_fp16)
185 ✗ detail::throw_not_implemented("linear_forward_q8_0_fp16", W_q8.device);
186 40 detail::adopt_output(y, W_q8.device);
187 40 v.linear_forward_q8_0_fp16(W_q8, bias, x, y);
188 40 }
189
190 6 void linear_forward_batched_q8_0_fp16(const Tensor& W_q8, const Tensor* bias,
191 const Tensor& X_BD, Tensor& Y_BD) {
192 6 check_quant_block_cols("linear_forward_batched_q8_0_fp16", W_q8, "W_q8", Dtype::Q8_0);
193 6 const auto& v = detail::dispatch_with_opts(W_q8, {bias, &X_BD, &Y_BD});
194
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 if (!v.linear_forward_batched_q8_0_fp16)
195 ✗ detail::throw_not_implemented("linear_forward_batched_q8_0_fp16", W_q8.device);
196 6 detail::adopt_output(Y_BD, W_q8.device);
197 6 v.linear_forward_batched_q8_0_fp16(W_q8, bias, X_BD, Y_BD);
198 6 }
199
200 3 void dequant_q6k_to_fp16(const Tensor& W_q6k, Tensor& W_fp16) {
201 3 check_quant_block_cols("dequant_q6k_to_fp16", W_q6k, "W_q6k", Dtype::Q6_K);
202 3 const auto& v = detail::dispatch(W_q6k, W_fp16);
203
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 if (!v.dequant_q6k_to_fp16)
204 ✗ detail::throw_not_implemented("dequant_q6k_to_fp16", W_q6k.device);
205 3 detail::adopt_output(W_fp16, W_q6k.device);
206 3 v.dequant_q6k_to_fp16(W_q6k, W_fp16);
207 3 }
208
209 36 void linear_forward_q6k_fp16(const Tensor& W_q6k, const Tensor* bias,
210 const Tensor& x, Tensor& y) {
211 36 check_quant_block_cols("linear_forward_q6k_fp16", W_q6k, "W_q6k", Dtype::Q6_K);
212 36 const auto& v = detail::dispatch_with_opts(W_q6k, {bias, &x, &y});
213
1/2
✓ Branch 0 taken 36 times.
✗ Branch 1 not taken.
 36 if (!v.linear_forward_q6k_fp16)
214 ✗ detail::throw_not_implemented("linear_forward_q6k_fp16", W_q6k.device);
215 36 detail::adopt_output(y, W_q6k.device);
216 36 v.linear_forward_q6k_fp16(W_q6k, bias, x, y);
217 36 }
218
219 5 void linear_forward_batched_q6k_fp16(const Tensor& W_q6k, const Tensor* bias,
220 const Tensor& X_BD, Tensor& Y_BD) {
221 5 check_quant_block_cols("linear_forward_batched_q6k_fp16", W_q6k, "W_q6k", Dtype::Q6_K);
222 5 const auto& v = detail::dispatch_with_opts(W_q6k, {bias, &X_BD, &Y_BD});
223
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (!v.linear_forward_batched_q6k_fp16)
224 ✗ detail::throw_not_implemented("linear_forward_batched_q6k_fp16", W_q6k.device);
225 5 detail::adopt_output(Y_BD, W_q6k.device);
226 5 v.linear_forward_batched_q6k_fp16(W_q6k, bias, X_BD, Y_BD);
227 5 }
228
229 3 void flash_attention_project_kv_int8w_fp16(const Tensor& ctx,
230 const Tensor& Wk_int8,
231 const Tensor& sk,
232 const Tensor* bk,
233 const Tensor& Wv_int8,
234 const Tensor& sv,
235 const Tensor* bv,
236 Tensor& K_out, Tensor& V_out) {
237 3 const auto& v = detail::dispatch_with_opts(
238 3 ctx, Wk_int8, {&sk, bk, &Wv_int8, &sv, bv, &K_out, &V_out});
239
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 if (!v.flash_attention_project_kv_int8w_fp16)
240 ✗ detail::throw_not_implemented("flash_attention_project_kv_int8w_fp16", ctx.device);
241 3 detail::adopt_output(K_out, ctx.device);
242 3 detail::adopt_output(V_out, ctx.device);
243 6 v.flash_attention_project_kv_int8w_fp16(ctx, Wk_int8, sk, bk,
244 3 Wv_int8, sv, bv, K_out, V_out);
245 3 }
246
247 3 void flash_attention_q_with_kv_cached_int8w_fp16(const Tensor& X,
248 const Tensor& K, const Tensor& V,
249 const Tensor& Wq_int8,
250 const Tensor& sq,
251 const Tensor* bq,
252 const Tensor& Wo_int8,
253 const Tensor& so,
254 const Tensor* bo,
255 const float* d_mask,
256 int num_heads, bool causal,
257 Tensor& O) {
258 3 const auto& v = detail::dispatch_with_opts(
259 3 X, K, {&V, &Wq_int8, &sq, bq, &Wo_int8, &so, bo, &O});
260
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 if (!v.flash_attention_q_with_kv_cached_int8w_fp16)
261 ✗ detail::throw_not_implemented("flash_attention_q_with_kv_cached_int8w_fp16", X.device);
262 3 detail::adopt_output(O, X.device);
263 6 v.flash_attention_q_with_kv_cached_int8w_fp16(X, K, V, Wq_int8, sq, bq,
264 3 Wo_int8, so, bo,
265 3 d_mask, num_heads, causal, O);
266 3 }
267
268 3 void flash_attention_qkvo_int8w_fp16(const Tensor& X, const Tensor* Ctx,
269 const Tensor& Wq_int8, const Tensor& sq, const Tensor* bq,
270 const Tensor& Wk_int8, const Tensor& sk, const Tensor* bk,
271 const Tensor& Wv_int8, const Tensor& sv, const Tensor* bv,
272 const Tensor& Wo_int8, const Tensor& so, const Tensor* bo,
273 const float* d_mask, int num_heads, bool causal,
274 Tensor& O) {
275 3 const auto& v = detail::dispatch_with_opts(
276 15 X, Wq_int8, {Ctx, &sq, bq, &Wk_int8, &sk, bk, &Wv_int8, &sv, bv,
277 12 &Wo_int8, &so, bo, &O});
278
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 if (!v.flash_attention_qkvo_int8w_fp16)
279 ✗ detail::throw_not_implemented("flash_attention_qkvo_int8w_fp16", X.device);
280 3 detail::adopt_output(O, X.device);
281 6 v.flash_attention_qkvo_int8w_fp16(X, Ctx, Wq_int8, sq, bq,
282 3 Wk_int8, sk, bk, Wv_int8, sv, bv,
283 3 Wo_int8, so, bo,
284 3 d_mask, num_heads, causal, O);
285 3 }
286
287 // ─── Spectral / FFT core (brosoundml) ──────────────────────────────────────
288
289 56 void complex_mul(const Tensor& a, const Tensor& b, Tensor& y) {
290 56 const auto& v = detail::dispatch(a, b, y);
291
1/2
✓ Branch 0 taken 56 times.
✗ Branch 1 not taken.
 56 if (!v.complex_mul) detail::throw_not_implemented("complex_mul", a.device);
292 56 detail::adopt_output(y, a.device);
293 56 v.complex_mul(a, b, y);
294 56 }
295 7 void complex_mul_backward(const Tensor& a, const Tensor& b, const Tensor& dY,
296 Tensor& dA, Tensor& dB) {
297 7 const auto& v = detail::dispatch(a, b, dY, dA, dB);
298
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 if (!v.complex_mul_backward)
299 ✗ detail::throw_not_implemented("complex_mul_backward", a.device);
300 7 detail::adopt_output(dA, a.device);
301 7 detail::adopt_output(dB, a.device);
302 7 v.complex_mul_backward(a, b, dY, dA, dB);
303 7 }
304 38 void complex_abs(const Tensor& z, Tensor& y) {
305 38 const auto& v = detail::dispatch(z, y);
306
1/2
✓ Branch 0 taken 38 times.
✗ Branch 1 not taken.
 38 if (!v.complex_abs) detail::throw_not_implemented("complex_abs", z.device);
307 38 detail::adopt_output(y, z.device);
308 38 v.complex_abs(z, y);
309 38 }
310 5 void complex_abs_backward(const Tensor& z, const Tensor& dY, Tensor& dZ) {
311 5 const auto& v = detail::dispatch(z, dY, dZ);
312
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (!v.complex_abs_backward)
313 ✗ detail::throw_not_implemented("complex_abs_backward", z.device);
314 5 detail::adopt_output(dZ, z.device);
315 5 v.complex_abs_backward(z, dY, dZ);
316 5 }
317 5 void complex_angle(const Tensor& z, Tensor& y) {
318 5 const auto& v = detail::dispatch(z, y);
319
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (!v.complex_angle) detail::throw_not_implemented("complex_angle", z.device);
320 5 detail::adopt_output(y, z.device);
321 5 v.complex_angle(z, y);
322 5 }
323 5 void complex_from_polar(const Tensor& mag, const Tensor& phase, Tensor& y) {
324 5 const auto& v = detail::dispatch(mag, phase, y);
325
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (!v.complex_from_polar)
326 ✗ detail::throw_not_implemented("complex_from_polar", mag.device);
327 5 detail::adopt_output(y, mag.device);
328 5 v.complex_from_polar(mag, phase, y);
329 5 }
330 341 void fft(const Tensor& x, Tensor& y) {
331 341 const auto& v = detail::dispatch(x, y);
332
1/2
✓ Branch 0 taken 341 times.
✗ Branch 1 not taken.
 341 if (!v.fft) detail::throw_not_implemented("fft", x.device);
333 341 detail::adopt_output(y, x.device);
334 341 v.fft(x, y);
335 341 }
336 25 void ifft(const Tensor& x, Tensor& y) {
337 25 const auto& v = detail::dispatch(x, y);
338
1/2
✓ Branch 0 taken 25 times.
✗ Branch 1 not taken.
 25 if (!v.ifft) detail::throw_not_implemented("ifft", x.device);
339 25 detail::adopt_output(y, x.device);
340 25 v.ifft(x, y);
341 25 }
342 337 void rfft(const Tensor& x, Tensor& y) {
343 337 const auto& v = detail::dispatch(x, y);
344
1/2
✓ Branch 0 taken 337 times.
✗ Branch 1 not taken.
 337 if (!v.rfft) detail::throw_not_implemented("rfft", x.device);
345 337 detail::adopt_output(y, x.device);
346 337 v.rfft(x, y);
347 337 }
348 369 void irfft(const Tensor& x, int L, Tensor& y) {
349 369 const auto& v = detail::dispatch(x, y);
350
1/2
✓ Branch 0 taken 369 times.
✗ Branch 1 not taken.
 369 if (!v.irfft) detail::throw_not_implemented("irfft", x.device);
351 369 detail::adopt_output(y, x.device);
352 369 v.irfft(x, L, y);
353 369 }
354 11 void rfft_backward(const Tensor& dY, int L, Tensor& dX) {
355 11 const auto& v = detail::dispatch(dY, dX);
356
1/2
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
 11 if (!v.rfft_backward) detail::throw_not_implemented("rfft_backward", dY.device);
357 11 detail::adopt_output(dX, dY.device);
358 11 v.rfft_backward(dY, L, dX);
359 11 }
360 11 void irfft_backward(const Tensor& dY, Tensor& dX) {
361 11 const auto& v = detail::dispatch(dY, dX);
362
1/2
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
 11 if (!v.irfft_backward)
363 ✗ detail::throw_not_implemented("irfft_backward", dY.device);
364 11 detail::adopt_output(dX, dY.device);
365 11 v.irfft_backward(dY, dX);
366 11 }
367
368 1302 void stft(const Tensor& signal, const Tensor& window,
369 int N, int n_fft, int hop_length, int win_length,
370 bool center, bool normalized, Tensor& spec) {
371 1302 const auto& v = detail::dispatch(signal, window, spec);
372
1/2
✓ Branch 0 taken 1302 times.
✗ Branch 1 not taken.
 1302 if (!v.stft) detail::throw_not_implemented("stft", signal.device);
373 1302 detail::adopt_output(spec, signal.device);
374 2604 v.stft(signal, window, N, n_fft, hop_length, win_length,
375 1302 center, normalized, spec);
376 1302 }
377 16 void stft_backward(const Tensor& dSpec, const Tensor& window,
378 int N, int signal_len, int n_fft, int hop_length,
379 int win_length, bool center, bool normalized,
380 Tensor& dSignal) {
381 16 const auto& v = detail::dispatch(dSpec, window, dSignal);
382
1/2
✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
 16 if (!v.stft_backward)
383 ✗ detail::throw_not_implemented("stft_backward", dSpec.device);
384 16 detail::adopt_output(dSignal, dSpec.device);
385 32 v.stft_backward(dSpec, window, N, signal_len, n_fft, hop_length,
386 16 win_length, center, normalized, dSignal);
387 16 }
388 5126 void istft(const Tensor& spec, const Tensor& window,
389 int N, int signal_len, int n_fft, int hop_length, int win_length,
390 bool center, bool normalized, Tensor& signal) {
391 5126 const auto& v = detail::dispatch(spec, window, signal);
392
1/2
✓ Branch 0 taken 5126 times.
✗ Branch 1 not taken.
 5126 if (!v.istft) detail::throw_not_implemented("istft", spec.device);
393 5126 detail::adopt_output(signal, spec.device);
394 10252 v.istft(spec, window, N, signal_len, n_fft, hop_length, win_length,
395 5126 center, normalized, signal);
396 5126 }
397 16 void istft_backward(const Tensor& dSignal, const Tensor& window,
398 int N, int signal_len, int n_fft, int hop_length,
399 int win_length, bool center, bool normalized,
400 Tensor& dSpec) {
401 16 const auto& v = detail::dispatch(dSignal, window, dSpec);
402
1/2
✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
 16 if (!v.istft_backward)
403 ✗ detail::throw_not_implemented("istft_backward", dSignal.device);
404 16 detail::adopt_output(dSpec, dSignal.device);
405 32 v.istft_backward(dSignal, window, N, signal_len, n_fft, hop_length,
406 16 win_length, center, normalized, dSpec);
407 16 }
408
409 // ─── 1D convolution family (brosoundml) ────────────────────────────────────
410
411 147 void conv_transpose1d_forward(const Tensor& X, const Tensor& Wt,
412 const Tensor* bias,
413 int N, int C_in, int L, int C_out, int kL,
414 int stride, int padding, int output_padding,
415 int dilation, int groups, Tensor& Y) {
416 147 BROTENSOR_CHECK_SPATIAL_1D("conv_transpose1d_forward", X, N, C_in, L);
417 147 const auto& v = detail::dispatch_with_opts(X, Wt, {bias, &Y});
418
1/2
✓ Branch 0 taken 147 times.
✗ Branch 1 not taken.
 147 if (!v.conv_transpose1d_forward)
419 ✗ detail::throw_not_implemented("conv_transpose1d_forward", X.device);
420 147 detail::adopt_output(Y, X.device);
421 294 v.conv_transpose1d_forward(X, Wt, bias, N, C_in, L, C_out, kL, stride,
422 147 padding, output_padding, dilation, groups, Y);
423 147 }
424
425 14 void conv_transpose1d_backward_input(const Tensor& Wt, const Tensor& dY,
426 int N, int C_in, int L, int C_out, int kL,
427 int stride, int padding,
428 int output_padding, int dilation,
429 int groups, Tensor& dX) {
430
1/2
✓ Branch 0 taken 14 times.
✗ Branch 1 not taken.
 14 if (N <= 0 || C_in <= 0 || L <= 0 || C_out <= 0) {
431 ✗ detail::throw_spatial_1d_error("conv_transpose1d_backward_input", "dY",
432 ✗ N, C_in, L, dY.rows, dY.cols);
433 }
434 14 const auto& v = detail::dispatch(Wt, dY, dX);
435
1/2
✓ Branch 0 taken 14 times.
✗ Branch 1 not taken.
 14 if (!v.conv_transpose1d_backward_input)
436 ✗ detail::throw_not_implemented("conv_transpose1d_backward_input", Wt.device);
437 14 detail::adopt_output(dX, Wt.device);
438 28 v.conv_transpose1d_backward_input(Wt, dY, N, C_in, L, C_out, kL, stride,
439 14 padding, output_padding, dilation,
440 14 groups, dX);
441 14 }
442
443 12 void conv_transpose1d_backward_weight(const Tensor& X, const Tensor& dY,
444 int N, int C_in, int L, int C_out, int kL,
445 int stride, int padding,
446 int output_padding, int dilation,
447 int groups, Tensor& dWt) {
448 12 BROTENSOR_CHECK_SPATIAL_1D("conv_transpose1d_backward_weight", X, N, C_in, L);
449 12 const auto& v = detail::dispatch(X, dY, dWt);
450
1/2
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
 12 if (!v.conv_transpose1d_backward_weight)
451 ✗ detail::throw_not_implemented("conv_transpose1d_backward_weight", X.device);
452 12 detail::adopt_output(dWt, X.device);
453 24 v.conv_transpose1d_backward_weight(X, dY, N, C_in, L, C_out, kL, stride,
454 12 padding, output_padding, dilation,
455 12 groups, dWt);
456 12 }
457
458 10 void conv_transpose1d_backward_bias(const Tensor& dY, int N, int C_out,
459 int L_out, Tensor& dB) {
460 10 BROTENSOR_CHECK_SPATIAL_1D("conv_transpose1d_backward_bias", dY, N, C_out, L_out);
461 10 const auto& v = detail::dispatch(dY, dB);
462
1/2
✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
 10 if (!v.conv_transpose1d_backward_bias)
463 ✗ detail::throw_not_implemented("conv_transpose1d_backward_bias", dY.device);
464 10 detail::adopt_output(dB, dY.device);
465 10 v.conv_transpose1d_backward_bias(dY, N, C_out, L_out, dB);
466 10 }
467
468 30 void causal_conv1d_update(const Tensor& X, const Tensor& Wt, const Tensor* bias,
469 int N, int C, int L_step, int kL, int dilation,
470 Tensor& state, Tensor& Y) {
471 30 BROTENSOR_CHECK_SPATIAL_1D("causal_conv1d_update", X, N, C, L_step);
472 30 const auto& v = detail::dispatch_with_opts(X, Wt, {bias, &state, &Y});
473
1/2
✓ Branch 0 taken 30 times.
✗ Branch 1 not taken.
 30 if (!v.causal_conv1d_update)
474 ✗ detail::throw_not_implemented("causal_conv1d_update", X.device);
475 30 detail::adopt_output(state, X.device);
476 30 detail::adopt_output(Y, X.device);
477 30 v.causal_conv1d_update(X, Wt, bias, N, C, L_step, kL, dilation, state, Y);
478 30 }
479
480 166 void pad1d_forward(const Tensor& X, int N, int C, int L,
481 int pad_left, int pad_right, int mode, Tensor& Y) {
482 166 BROTENSOR_CHECK_SPATIAL_1D("pad1d_forward", X, N, C, L);
483 166 const auto& v = detail::dispatch(X, Y);
484
1/2
✓ Branch 0 taken 166 times.
✗ Branch 1 not taken.
 166 if (!v.pad1d_forward)
485 ✗ detail::throw_not_implemented("pad1d_forward", X.device);
486 166 detail::adopt_output(Y, X.device);
487 166 v.pad1d_forward(X, N, C, L, pad_left, pad_right, mode, Y);
488 166 }
489
490 11 void pad1d_backward(const Tensor& dY, int N, int C, int L,
491 int pad_left, int pad_right, int mode, Tensor& dX) {
492
1/2
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
 11 if (N <= 0 || C <= 0 || L <= 0) {
493 ✗ detail::throw_spatial_1d_error("pad1d_backward", "dY",
494 ✗ N, C, L, dY.rows, dY.cols);
495 }
496 11 const int L_out = L + pad_left + pad_right;
497
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (L_out > 0) {
498 11 BROTENSOR_CHECK_SPATIAL_1D("pad1d_backward", dY, N, C, L_out);
499 11 }
500 11 const auto& v = detail::dispatch(dY, dX);
501
1/2
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
 11 if (!v.pad1d_backward)
502 ✗ detail::throw_not_implemented("pad1d_backward", dY.device);
503 11 detail::adopt_output(dX, dY.device);
504 11 v.pad1d_backward(dY, N, C, L, pad_left, pad_right, mode, dX);
505 11 }
506
507 // ─── Vocoder / codec activations (brosoundml) ──────────────────────────────
508
509 129 void snake_forward(const Tensor& X, const Tensor& alpha, const Tensor* beta,
510 int N, int C, int L, Tensor& Y) {
511 129 BROTENSOR_CHECK_SPATIAL_1D("snake_forward", X, N, C, L);
512 129 const auto& v = detail::dispatch_with_opts(X, alpha, {beta, &Y});
513
1/2
✓ Branch 0 taken 129 times.
✗ Branch 1 not taken.
 129 if (!v.snake_forward)
514 ✗ detail::throw_not_implemented("snake_forward", X.device);
515 129 detail::adopt_output(Y, X.device);
516 129 v.snake_forward(X, alpha, beta, N, C, L, Y);
517 129 }
518
519 15 void snake_backward(const Tensor& X, const Tensor& alpha, const Tensor* beta,
520 const Tensor& dY, int N, int C, int L,
521 Tensor& dX, Tensor& dAlpha, Tensor* dBeta) {
522 15 BROTENSOR_CHECK_SPATIAL_1D("snake_backward", X, N, C, L);
523 15 BROTENSOR_CHECK_SPATIAL_1D("snake_backward", dY, N, C, L);
524 15 const auto& v = detail::dispatch_with_opts(
525 15 X, alpha, {beta, &dY, &dX, &dAlpha, dBeta});
526
1/2
✓ Branch 0 taken 15 times.
✗ Branch 1 not taken.
 15 if (!v.snake_backward)
527 ✗ detail::throw_not_implemented("snake_backward", X.device);
528 15 detail::adopt_output(dX, X.device);
529 15 detail::adopt_output(dAlpha, X.device);
530
2/2
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 10 times.
 15 if (dBeta) detail::adopt_output(*dBeta, X.device);
531 15 v.snake_backward(X, alpha, beta, dY, N, C, L, dX, dAlpha, dBeta);
532 15 }
533
534 141 void elu_forward(const Tensor& x, float alpha, Tensor& y) {
535 141 const auto& v = detail::dispatch(x, y);
536
1/2
✓ Branch 0 taken 141 times.
✗ Branch 1 not taken.
 141 if (!v.elu_forward) detail::throw_not_implemented("elu_forward", x.device);
537 141 detail::adopt_output(y, x.device);
538 141 v.elu_forward(x, alpha, y);
539 141 }
540
541 11 void elu_backward(const Tensor& x, const Tensor& dY, float alpha, Tensor& dX) {
542 11 const auto& v = detail::dispatch(x, dY, dX);
543
1/2
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
 11 if (!v.elu_backward) detail::throw_not_implemented("elu_backward", x.device);
544 11 detail::adopt_output(dX, x.device);
545 11 v.elu_backward(x, dY, alpha, dX);
546 11 }
547
548 96 void leaky_relu_forward(const Tensor& x, float negative_slope, Tensor& y) {
549 96 const auto& v = detail::dispatch(x, y);
550
1/2
✓ Branch 0 taken 96 times.
✗ Branch 1 not taken.
 96 if (!v.leaky_relu_forward)
551 ✗ detail::throw_not_implemented("leaky_relu_forward", x.device);
552 96 detail::adopt_output(y, x.device);
553 96 v.leaky_relu_forward(x, negative_slope, y);
554 96 }
555
556 8 void leaky_relu_backward(const Tensor& x, const Tensor& dY,
557 float negative_slope, Tensor& dX) {
558 8 const auto& v = detail::dispatch(x, dY, dX);
559
1/2
✓ Branch 0 taken 8 times.
✗ Branch 1 not taken.
 8 if (!v.leaky_relu_backward)
560 ✗ detail::throw_not_implemented("leaky_relu_backward", x.device);
561 8 detail::adopt_output(dX, x.device);
562 8 v.leaky_relu_backward(x, dY, negative_slope, dX);
563 8 }
564
565 // ─── Codec quantization (brosoundml) ───────────────────────────────────────
566
567 11 void vq_encode_forward(const Tensor& x, const Tensor& codebook,
568 Tensor& indices, Tensor& quantized) {
569 11 const auto& v = detail::dispatch(x, codebook, indices, quantized);
570
1/2
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
 11 if (!v.vq_encode_forward)
571 ✗ detail::throw_not_implemented("vq_encode_forward", x.device);
572 11 detail::adopt_output(indices, x.device);
573 11 detail::adopt_output(quantized, x.device);
574 11 v.vq_encode_forward(x, codebook, indices, quantized);
575 11 }
576
577 8 void vq_encode_backward(const Tensor& dQuantized, Tensor& dX) {
578 8 const auto& v = detail::dispatch(dQuantized, dX);
579
1/2
✓ Branch 0 taken 8 times.
✗ Branch 1 not taken.
 8 if (!v.vq_encode_backward)
580 ✗ detail::throw_not_implemented("vq_encode_backward", dQuantized.device);
581 8 detail::adopt_output(dX, dQuantized.device);
582 8 v.vq_encode_backward(dQuantized, dX);
583 8 }
584
585 10 void fsq_quantize_forward(const Tensor& x, const Tensor& levels,
586 Tensor& quantized, Tensor& packed_indices) {
587 10 const auto& v = detail::dispatch(x, levels, quantized, packed_indices);
588
1/2
✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
 10 if (!v.fsq_quantize_forward)
589 ✗ detail::throw_not_implemented("fsq_quantize_forward", x.device);
590 10 detail::adopt_output(quantized, x.device);
591 10 detail::adopt_output(packed_indices, x.device);
592 10 v.fsq_quantize_forward(x, levels, quantized, packed_indices);
593 10 }
594
595 8 void fsq_quantize_backward(const Tensor& dQuantized, Tensor& dX) {
596 8 const auto& v = detail::dispatch(dQuantized, dX);
597
1/2
✓ Branch 0 taken 8 times.
✗ Branch 1 not taken.
 8 if (!v.fsq_quantize_backward)
598 ✗ detail::throw_not_implemented("fsq_quantize_backward", dQuantized.device);
599 8 detail::adopt_output(dX, dQuantized.device);
600 8 v.fsq_quantize_backward(dQuantized, dX);
601 8 }
602
603 // ─── 1D resampling (brosoundml) ────────────────────────────────────────────
604
605 156 void resample1d_forward(const Tensor& X, int N, int C, int L_in, int L_out,
606 int mode, Tensor& Y) {
607 156 BROTENSOR_CHECK_SPATIAL_1D("resample1d_forward", X, N, C, L_in);
608 156 const auto& v = detail::dispatch(X, Y);
609
1/2
✓ Branch 0 taken 156 times.
✗ Branch 1 not taken.
 156 if (!v.resample1d_forward)
610 ✗ detail::throw_not_implemented("resample1d_forward", X.device);
611 156 detail::adopt_output(Y, X.device);
612 156 v.resample1d_forward(X, N, C, L_in, L_out, mode, Y);
613 156 }
614
615 20 void resample1d_backward(const Tensor& dY, int N, int C, int L_in, int L_out,
616 int mode, Tensor& dX) {
617
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (N <= 0 || C <= 0 || L_in <= 0) {
618 ✗ detail::throw_spatial_1d_error("resample1d_backward", "dY",
619 ✗ N, C, L_in, dY.rows, dY.cols);
620 }
621 20 BROTENSOR_CHECK_SPATIAL_1D("resample1d_backward", dY, N, C, L_out);
622 20 const auto& v = detail::dispatch(dY, dX);
623
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (!v.resample1d_backward)
624 ✗ detail::throw_not_implemented("resample1d_backward", dY.device);
625 20 detail::adopt_output(dX, dY.device);
626 20 v.resample1d_backward(dY, N, C, L_in, L_out, mode, dX);
627 20 }
628
629 // ─── log / exp / round elementwise (brosoundml) ────────────────────────────
630
631 54 void log_forward(const Tensor& x, Tensor& y) {
632 54 const auto& v = detail::dispatch(x, y);
633
1/2
✓ Branch 0 taken 54 times.
✗ Branch 1 not taken.
 54 if (!v.log_forward) detail::throw_not_implemented("log_forward", x.device);
634 54 detail::adopt_output(y, x.device);
635 54 v.log_forward(x, y);
636 54 }
637
638 7 void log_backward(const Tensor& x, const Tensor& dY, Tensor& dX) {
639 7 const auto& v = detail::dispatch(x, dY, dX);
640
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 if (!v.log_backward) detail::throw_not_implemented("log_backward", x.device);
641 7 detail::adopt_output(dX, x.device);
642 7 v.log_backward(x, dY, dX);
643 7 }
644
645 50 void exp_forward(const Tensor& x, Tensor& y) {
646 50 const auto& v = detail::dispatch(x, y);
647
1/2
✓ Branch 0 taken 50 times.
✗ Branch 1 not taken.
 50 if (!v.exp_forward) detail::throw_not_implemented("exp_forward", x.device);
648 50 detail::adopt_output(y, x.device);
649 50 v.exp_forward(x, y);
650 50 }
651
652 9 void exp_backward(const Tensor& x, const Tensor& dY, Tensor& dX) {
653 9 const auto& v = detail::dispatch(x, dY, dX);
654
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 if (!v.exp_backward) detail::throw_not_implemented("exp_backward", x.device);
655 9 detail::adopt_output(dX, x.device);
656 9 v.exp_backward(x, dY, dX);
657 9 }
658
659 9 void round_forward(const Tensor& x, Tensor& y) {
660 9 const auto& v = detail::dispatch(x, y);
661
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 if (!v.round_forward) detail::throw_not_implemented("round_forward", x.device);
662 9 detail::adopt_output(y, x.device);
663 9 v.round_forward(x, y);
664 9 }
665
666 5 void round_backward(const Tensor& dY, Tensor& dX) {
667 5 const auto& v = detail::dispatch(dY, dX);
668
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (!v.round_backward) detail::throw_not_implemented("round_backward", dY.device);
669 5 detail::adopt_output(dX, dY.device);
670 5 v.round_backward(dY, dX);
671 5 }
672
673 // ─── Autoregressive logit sampling (brosoundml) ────────────────────────────
674
675 50270 void sample_logits(const Tensor& logits, float temperature, int top_k,
676 float top_p, uint64_t key, uint64_t counter,
677 Tensor& indices) {
678 50270 const auto& v = detail::dispatch(logits, indices);
679
1/2
✓ Branch 0 taken 50270 times.
✗ Branch 1 not taken.
 50270 if (!v.sample_logits) detail::throw_not_implemented("sample_logits", logits.device);
680 50270 detail::adopt_output(indices, logits.device);
681 50270 v.sample_logits(logits, temperature, top_k, top_p, key, counter, indices);
682 50270 }
683
684 17 void sample_logits_into(const Tensor& logits, float temperature, int top_k,
685 float top_p, uint64_t key, Tensor& counter,
686 Tensor& scratch, Tensor& indices) {
687 17 const auto& v = detail::dispatch(logits, indices);
688
1/2
✓ Branch 0 taken 17 times.
✗ Branch 1 not taken.
 17 if (!v.sample_logits_into)
689 ✗ detail::throw_not_implemented("sample_logits_into", logits.device);
690 17 detail::adopt_output(indices, logits.device);
691 34 v.sample_logits_into(logits, temperature, top_k, top_p, key, counter,
692 17 scratch, indices);
693 17 }
694
695 // ─── Masked-diffusion token selection (OmniVoice codebook grids) ────────────
696
697 20 void masked_diffusion_scores(const Tensor& logits, const Tensor& tokens,
698 int T, int C, int V, int mask_id,
699 float guidance_scale, float layer_penalty,
700 float position_temperature, float class_temperature,
701 float class_top_frac, std::uint64_t seed,
702 Tensor& pred, Tensor& scores, Tensor& confidence) {
703 20 const auto& v = detail::dispatch(logits, tokens, pred, scores, confidence);
704
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (!v.masked_diffusion_scores)
705 ✗ detail::throw_not_implemented("masked_diffusion_scores", logits.device);
706 20 detail::adopt_output(pred, logits.device);
707 20 detail::adopt_output(scores, logits.device);
708 20 detail::adopt_output(confidence, logits.device);
709 40 v.masked_diffusion_scores(logits, tokens, T, C, V, mask_id,
710 20 guidance_scale, layer_penalty,
711 20 position_temperature, class_temperature,
712 20 class_top_frac, seed, pred, scores, confidence);
713 20 }
714
715 12 void masked_diffusion_commit(const Tensor& pred, const Tensor& idx, int k, int step,
716 Tensor& tokens, Tensor& unmask_step) {
717 12 const auto& v = detail::dispatch(pred, idx, tokens, unmask_step);
718
1/2
✓ Branch 0 taken 12 times.
✗ Branch 1 not taken.
 12 if (!v.masked_diffusion_commit)
719 ✗ detail::throw_not_implemented("masked_diffusion_commit", pred.device);
720 12 v.masked_diffusion_commit(pred, idx, k, step, tokens, unmask_step);
721 12 }
722
723 // ─── L2 norm + Gated Delta Rule (linear-attention text path) ───────────────
724
725 75 void l2_norm_forward(const Tensor& X, int head_dim, int num_heads,
726 float eps, Tensor& Y) {
727 75 const auto& v = detail::dispatch(X, Y);
728
1/2
✓ Branch 0 taken 75 times.
✗ Branch 1 not taken.
 75 if (!v.l2_norm_forward) detail::throw_not_implemented("l2_norm_forward", X.device);
729 75 detail::adopt_output(Y, X.device);
730 75 v.l2_norm_forward(X, head_dim, num_heads, eps, Y);
731 75 }
732
733 13 void l2_norm_backward(const Tensor& X, int head_dim, int num_heads,
734 float eps, const Tensor& dY, Tensor& dX) {
735 13 const auto& v = detail::dispatch(X, dY, dX);
736
1/2
✓ Branch 0 taken 13 times.
✗ Branch 1 not taken.
 13 if (!v.l2_norm_backward) detail::throw_not_implemented("l2_norm_backward", X.device);
737 13 detail::adopt_output(dX, X.device);
738 13 v.l2_norm_backward(X, head_dim, num_heads, eps, dY, dX);
739 13 }
740
741 15 void gated_delta_rule_chunked(const Tensor& Q, const Tensor& K, const Tensor& V,
742 const Tensor& a_raw, const Tensor& beta,
743 const Tensor& log_A,
744 int num_heads, int d_k, int d_v,
745 Tensor& state, Tensor& O) {
746 15 const auto& v = detail::dispatch(Q, K, V, a_raw, beta, log_A, state, O);
747
1/2
✓ Branch 0 taken 15 times.
✗ Branch 1 not taken.
 15 if (!v.gated_delta_rule_chunked)
748 ✗ detail::throw_not_implemented("gated_delta_rule_chunked", Q.device);
749 15 detail::adopt_output(state, Q.device);
750 15 detail::adopt_output(O, Q.device);
751 30 v.gated_delta_rule_chunked(Q, K, V, a_raw, beta, log_A,
752 15 num_heads, d_k, d_v, state, O);
753 15 }
754
755 18 void gated_delta_rule_step(const Tensor& Q, const Tensor& K, const Tensor& V,
756 const Tensor& a_raw, const Tensor& beta,
757 const Tensor& log_A,
758 int num_heads, int d_k, int d_v,
759 Tensor& state, Tensor& O) {
760 18 const auto& v = detail::dispatch(Q, K, V, a_raw, beta, log_A, state, O);
761
1/2
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
 18 if (!v.gated_delta_rule_step)
762 ✗ detail::throw_not_implemented("gated_delta_rule_step", Q.device);
763 18 detail::adopt_output(state, Q.device);
764 18 detail::adopt_output(O, Q.device);
765 36 v.gated_delta_rule_step(Q, K, V, a_raw, beta, log_A,
766 18 num_heads, d_k, d_v, state, O);
767 18 }
768
769 // ─── Counter-based noise generation (Philox 4x32-10) ───────────────────────
770
771 18 void randn(uint64_t key, uint64_t counter, Tensor& Y) {
772 18 const auto& v = detail::dispatch(Y);
773
1/2
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
 18 if (!v.randn) detail::throw_not_implemented("randn", Y.device);
774 18 detail::adopt_output(Y, Y.device);
775 18 v.randn(key, counter, Y);
776 18 }
777
778 6 void rand_uniform(uint64_t key, uint64_t counter, Tensor& Y) {
779 6 const auto& v = detail::dispatch(Y);
780
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 if (!v.rand_uniform) detail::throw_not_implemented("rand_uniform", Y.device);
781 6 detail::adopt_output(Y, Y.device);
782 6 v.rand_uniform(key, counter, Y);
783 6 }
784
785 8 void rand_bernoulli(float p, uint64_t key, uint64_t counter, Tensor& Y) {
786 8 const auto& v = detail::dispatch(Y);
787
1/2
✓ Branch 0 taken 8 times.
✗ Branch 1 not taken.
 8 if (!v.rand_bernoulli) detail::throw_not_implemented("rand_bernoulli", Y.device);
788 8 detail::adopt_output(Y, Y.device);
789 8 v.rand_bernoulli(p, key, counter, Y);
790 8 }
791
792 4 void randn_truncated(float lo, float hi, uint64_t key, uint64_t counter,
793 Tensor& Y) {
794 4 const auto& v = detail::dispatch(Y);
795
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (!v.randn_truncated) detail::throw_not_implemented("randn_truncated", Y.device);
796 4 detail::adopt_output(Y, Y.device);
797 4 v.randn_truncated(lo, hi, key, counter, Y);
798 4 }
799
800 // ─── StyleGAN3 synthesis-input primitives — sin/cos/rsqrt + pixel_norm ──────
801
802 34 void sin_forward(const Tensor& x, Tensor& y) {
803 34 const auto& v = detail::dispatch(x, y);
804
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 if (!v.sin_forward) detail::throw_not_implemented("sin_forward", x.device);
805 34 detail::adopt_output(y, x.device);
806 34 v.sin_forward(x, y);
807 34 }
808
809 5 void sin_backward(const Tensor& x, const Tensor& dY, Tensor& dX) {
810 5 const auto& v = detail::dispatch(x, dY, dX);
811
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (!v.sin_backward) detail::throw_not_implemented("sin_backward", x.device);
812 5 detail::adopt_output(dX, x.device);
813 5 v.sin_backward(x, dY, dX);
814 5 }
815
816 34 void cos_forward(const Tensor& x, Tensor& y) {
817 34 const auto& v = detail::dispatch(x, y);
818
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 if (!v.cos_forward) detail::throw_not_implemented("cos_forward", x.device);
819 34 detail::adopt_output(y, x.device);
820 34 v.cos_forward(x, y);
821 34 }
822
823 5 void cos_backward(const Tensor& x, const Tensor& dY, Tensor& dX) {
824 5 const auto& v = detail::dispatch(x, dY, dX);
825
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (!v.cos_backward) detail::throw_not_implemented("cos_backward", x.device);
826 5 detail::adopt_output(dX, x.device);
827 5 v.cos_backward(x, dY, dX);
828 5 }
829
830 34 void rsqrt_forward(const Tensor& x, Tensor& y) {
831 34 const auto& v = detail::dispatch(x, y);
832
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 if (!v.rsqrt_forward) detail::throw_not_implemented("rsqrt_forward", x.device);
833 34 detail::adopt_output(y, x.device);
834 34 v.rsqrt_forward(x, y);
835 34 }
836
837 9 void rsqrt_backward(const Tensor& y, const Tensor& dY, Tensor& dX) {
838 9 const auto& v = detail::dispatch(y, dY, dX);
839
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 if (!v.rsqrt_backward) detail::throw_not_implemented("rsqrt_backward", y.device);
840 9 detail::adopt_output(dX, y.device);
841 9 v.rsqrt_backward(y, dY, dX);
842 9 }
843
844 57 void pixel_norm_forward(const Tensor& X, float eps, Tensor& Y) {
845 57 const auto& v = detail::dispatch(X, Y);
846
1/2
✓ Branch 0 taken 57 times.
✗ Branch 1 not taken.
 57 if (!v.pixel_norm_forward) detail::throw_not_implemented("pixel_norm_forward", X.device);
847 57 detail::adopt_output(Y, X.device);
848 57 v.pixel_norm_forward(X, eps, Y);
849 57 }
850
851 9 void pixel_norm_backward(const Tensor& X, const Tensor& dY, float eps, Tensor& dX) {
852 9 const auto& v = detail::dispatch(X, dY, dX);
853
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 if (!v.pixel_norm_backward) detail::throw_not_implemented("pixel_norm_backward", X.device);
854 9 detail::adopt_output(dX, X.device);
855 9 v.pixel_norm_backward(X, dY, eps, dX);
856 9 }
857
858 } // namespace brotensor
859