GCC Code Coverage Report

Directory: brotensor/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 100.0% 286 / 0 / 286
Functions: 100.0% 1 / 0 / 1
Branches: -% 0 / 0 / 0
src/cpu/register.cpp
Line Branch Exec Source
1 // ─── CPU backend registration ──────────────────────────────────────────────
2 //
3 // Builds the CPU OpsVTable (only the slots we implement; everything else
4 // stays nullptr — the dispatcher throws "not implemented on this backend"
5 // on null lookups), pairs it with the CPU AllocVTable from alloc.cpp, and
6 // hands them to the registry at static-init time. CPU is therefore always
7 // available without a prior brotensor::init() call.
8
9 #include <brotensor/detail/dispatch.h>
10 #include <brotensor/tensor.h>
11
12 #include <cstdint>
13 #include <vector>
14
15 namespace brotensor::detail::cpu {
16
17 // ── alloc.cpp ──
18 const AllocVTable& cpu_alloc_table();
19
20 // ── ops_impl.cpp — forward decls of the 16 implemented ops ──
21 void linear_forward(const ::brotensor::Tensor& W, const ::brotensor::Tensor& b,
22 const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
23 void linear_backward(const ::brotensor::Tensor& W, const ::brotensor::Tensor& x,
24 const ::brotensor::Tensor& dY,
25 ::brotensor::Tensor& dX, ::brotensor::Tensor& dW,
26 ::brotensor::Tensor& dB);
27 void relu_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
28 void relu_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY,
29 ::brotensor::Tensor& dX);
30 void tanh_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
31 void tanh_backward(const ::brotensor::Tensor& y, const ::brotensor::Tensor& dY,
32 ::brotensor::Tensor& dX);
33 void sigmoid_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
34 void sigmoid_backward(const ::brotensor::Tensor& y, const ::brotensor::Tensor& dY,
35 ::brotensor::Tensor& dX);
36 void softmax_forward(const ::brotensor::Tensor& logits, ::brotensor::Tensor& probs,
37 const float* mask);
38 void softmax_rows_forward(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y,
39 int rows, int cols);
40 void softmax_backward(const ::brotensor::Tensor& probs,
41 const ::brotensor::Tensor& dProbs,
42 ::brotensor::Tensor& dLogits);
43 float softmax_xent_segment(const float* lp, const float* tp,
44 float* pp, float* dz,
45 int n, const float* mask);
46 float softmax_xent(const ::brotensor::Tensor& logits,
47 const ::brotensor::Tensor& target,
48 ::brotensor::Tensor& probs, ::brotensor::Tensor& dLogits,
49 const float* mask);
50 float mse_scalar(float pred, float target, float& dPred);
51 void add_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& x);
52 void add_scalar_inplace(::brotensor::Tensor& y, float s);
53 void add_channel_bias_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& bias, int C, int L);
54 void add_row_bias_inplace(::brotensor::Tensor& Y, const ::brotensor::Tensor& bias);
55 void xavier_init(::brotensor::Tensor& W, uint64_t& rng_state);
56
57 // ── ops_impl.cpp — forward decls of the 20 newly implemented ops ──
58 void sgd_step(::brotensor::Tensor& param, ::brotensor::Tensor& grad,
59 ::brotensor::Tensor& velocity, float lr, float momentum);
60 void adam_step(::brotensor::Tensor& param, const ::brotensor::Tensor& grad,
61 ::brotensor::Tensor& m, ::brotensor::Tensor& v,
62 float lr, float beta1, float beta2, float eps, int step);
63 void scale_inplace(::brotensor::Tensor& y, float s);
64 void layernorm_forward(const ::brotensor::Tensor& x,
65 const ::brotensor::Tensor& gamma,
66 const ::brotensor::Tensor& beta,
67 ::brotensor::Tensor& y, ::brotensor::Tensor& xhat,
68 float& mean_out, float& rstd_out, float eps);
69 void layernorm_backward(const ::brotensor::Tensor& dY,
70 const ::brotensor::Tensor& xhat,
71 const ::brotensor::Tensor& gamma, float rstd,
72 ::brotensor::Tensor& dX,
73 ::brotensor::Tensor& dGamma, ::brotensor::Tensor& dBeta);
74 void attention_forward(const ::brotensor::Tensor& X,
75 const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& Wk,
76 const ::brotensor::Tensor& Wv, const ::brotensor::Tensor& Wo,
77 const float* d_mask,
78 ::brotensor::Tensor& Q, ::brotensor::Tensor& K,
79 ::brotensor::Tensor& V, ::brotensor::Tensor& Attn,
80 ::brotensor::Tensor& Y_pre_Wo, ::brotensor::Tensor& O);
81 void attention_backward(const ::brotensor::Tensor& dO,
82 const ::brotensor::Tensor& X,
83 const ::brotensor::Tensor& Q, const ::brotensor::Tensor& K,
84 const ::brotensor::Tensor& V, const ::brotensor::Tensor& Attn,
85 const ::brotensor::Tensor& Y_pre_Wo,
86 const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& Wk,
87 const ::brotensor::Tensor& Wv, const ::brotensor::Tensor& Wo,
88 const float* d_mask,
89 ::brotensor::Tensor& dX,
90 ::brotensor::Tensor& dWq, ::brotensor::Tensor& dWk,
91 ::brotensor::Tensor& dWv, ::brotensor::Tensor& dWo);
92 void mha_forward(const ::brotensor::Tensor& X,
93 const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& Wk,
94 const ::brotensor::Tensor& Wv, const ::brotensor::Tensor& Wo,
95 const ::brotensor::Tensor* bq, const ::brotensor::Tensor* bk,
96 const ::brotensor::Tensor* bv, const ::brotensor::Tensor* bo,
97 const float* d_mask, int num_heads,
98 ::brotensor::Tensor& Qh, ::brotensor::Tensor& Kh,
99 ::brotensor::Tensor& Vh, ::brotensor::Tensor& Attnh,
100 ::brotensor::Tensor& Yconcat, ::brotensor::Tensor& O);
101 void mha_backward(const ::brotensor::Tensor& dO,
102 const ::brotensor::Tensor& X,
103 const ::brotensor::Tensor& Qh, const ::brotensor::Tensor& Kh,
104 const ::brotensor::Tensor& Vh, const ::brotensor::Tensor& Attnh,
105 const ::brotensor::Tensor& Yconcat,
106 const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& Wk,
107 const ::brotensor::Tensor& Wv, const ::brotensor::Tensor& Wo,
108 const float* d_mask, int num_heads,
109 ::brotensor::Tensor& dX,
110 ::brotensor::Tensor& dWq, ::brotensor::Tensor& dWk,
111 ::brotensor::Tensor& dWv, ::brotensor::Tensor& dWo,
112 ::brotensor::Tensor* dbq, ::brotensor::Tensor* dbk,
113 ::brotensor::Tensor* dbv, ::brotensor::Tensor* dbo);
114 void concat_rows(const std::vector<const ::brotensor::Tensor*>& parts,
115 ::brotensor::Tensor& out);
116 void split_rows(const ::brotensor::Tensor& in,
117 const std::vector<::brotensor::Tensor*>& parts);
118 void masked_mean_pool_forward(const ::brotensor::Tensor& X, const float* d_mask,
119 ::brotensor::Tensor& y);
120 void masked_mean_pool_backward(const ::brotensor::Tensor& dY, const float* d_mask,
121 int K, ::brotensor::Tensor& dX);
122 void build_slot_mask(const ::brotensor::Tensor& x, int offset, int K, int stride,
123 ::brotensor::Tensor& mask);
124 void copy_d2d(const ::brotensor::Tensor& src, int src_off,
125 ::brotensor::Tensor& dst, int dst_off, int n);
126 void copy_d2d_strided(const ::brotensor::Tensor& src, int src_off, int src_pitch,
127 ::brotensor::Tensor& dst, int dst_off, int dst_pitch,
128 int width, int height);
129 void add_inplace_batched(::brotensor::Tensor& Y_BD,
130 const ::brotensor::Tensor& X_BD);
131 void linear_forward_batched(const ::brotensor::Tensor& W,
132 const ::brotensor::Tensor& bias,
133 const ::brotensor::Tensor& X_BD,
134 ::brotensor::Tensor& Y_BD);
135 void linear_backward_batched(const ::brotensor::Tensor& W,
136 const ::brotensor::Tensor& X_BD,
137 const ::brotensor::Tensor& dY_BD,
138 ::brotensor::Tensor& dX_BD,
139 ::brotensor::Tensor& dW,
140 ::brotensor::Tensor& dB);
141 void relu_forward_batched(const ::brotensor::Tensor& X_BD,
142 ::brotensor::Tensor& Y_BD);
143 void relu_backward_batched(const ::brotensor::Tensor& X_BD,
144 const ::brotensor::Tensor& dY_BD,
145 ::brotensor::Tensor& dX_BD);
146 void tanh_forward_batched(const ::brotensor::Tensor& X_BD,
147 ::brotensor::Tensor& Y_BD);
148 void tanh_backward_batched(const ::brotensor::Tensor& Y_BD,
149 const ::brotensor::Tensor& dY_BD,
150 ::brotensor::Tensor& dX_BD);
151 void mse_vec_per_sample(const ::brotensor::Tensor& pred,
152 const ::brotensor::Tensor& target,
153 ::brotensor::Tensor& dPred,
154 ::brotensor::Tensor& loss_per_sample);
155 void softmax_xent_fused_batched(const ::brotensor::Tensor& logits_BL,
156 const ::brotensor::Tensor& target_BL,
157 const float* d_mask_BL,
158 const int* d_head_offsets,
159 int n_heads,
160 ::brotensor::Tensor& probs_BL,
161 ::brotensor::Tensor& dLogits_BL,
162 ::brotensor::Tensor& loss_per_sample);
163 void bce_with_logits_fused_batched(const ::brotensor::Tensor& logits_BL,
164 const ::brotensor::Tensor& target_BL,
165 const float* d_mask_BL,
166 float pos_weight,
167 ::brotensor::Tensor& probs_BL,
168 ::brotensor::Tensor& dLogits_BL,
169 ::brotensor::Tensor& loss_per_sample);
170
171 // ── CHUNK 1 — elementwise.cpp / loss.cpp / embedding.cpp / concat.cpp /
172 // public_reductions.cpp / layernorm_inference.cpp ──
173 void clamp(::brotensor::Tensor& y, float lo, float hi);
174 void mul_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& x);
175 void div_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& x);
176 void axpby_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& x,
177 float a, float b);
178 void threshold_u8(const ::brotensor::Tensor& X, float t,
179 ::brotensor::Tensor& Y);
180 void cast(const ::brotensor::Tensor& src, ::brotensor::Tensor& dst,
181 ::brotensor::Dtype out_dtype);
182 float mse_vec_forward(const ::brotensor::Tensor& pred,
183 const ::brotensor::Tensor& target);
184 void mse_vec_backward(const ::brotensor::Tensor& pred,
185 const ::brotensor::Tensor& target,
186 ::brotensor::Tensor& dPred);
187 float softmax_xent_fused(const ::brotensor::Tensor& logits,
188 const ::brotensor::Tensor& target,
189 const float* d_mask,
190 ::brotensor::Tensor& probs,
191 ::brotensor::Tensor& dLogits);
192 void embedding_lookup_forward(const ::brotensor::Tensor& table,
193 const int32_t* d_idx, int B,
194 ::brotensor::Tensor& out);
195 void embedding_lookup_backward(const ::brotensor::Tensor& dOut,
196 const int32_t* d_idx, int B,
197 ::brotensor::Tensor& dTable);
198 void concat_batched_rows(const std::vector<const ::brotensor::Tensor*>& parts,
199 ::brotensor::Tensor& out);
200 void concat_nchw_channels(const std::vector<const ::brotensor::Tensor*>& parts,
201 int N, int H, int W,
202 const std::vector<int>& C_per_part,
203 ::brotensor::Tensor& out);
204 void concat_nchw_channels_backward(const ::brotensor::Tensor& dY,
205 int N, int H, int W,
206 const std::vector<int>& C_per_part,
207 const std::vector<::brotensor::Tensor*>& parts);
208 void sum_rows(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y);
209 void sum_cols(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y);
210 void argmax_rows(const ::brotensor::Tensor& X, ::brotensor::Tensor& Idx);
211 void rows_count_above(const ::brotensor::Tensor& X, float t_lo, float t_hi,
212 ::brotensor::Tensor& counts);
213 // ── packed_encoder.cpp ──
214 void flash_attention_packed_qkv_forward(const ::brotensor::Tensor& QKV,
215 const ::brotensor::Tensor& seq_bounds,
216 int num_heads, int window,
217 ::brotensor::Tensor& O);
218 void flash_attention_packed_qkv_backward(const ::brotensor::Tensor& QKV, const ::brotensor::Tensor& dO,
219 const ::brotensor::Tensor& seq_bounds, int num_heads, int window,
220 ::brotensor::Tensor& dQKV);
221 void rope_qkv_packed_inplace(::brotensor::Tensor& QKV, const ::brotensor::Tensor& cos_tbl,
222 const ::brotensor::Tensor& sin_tbl, const ::brotensor::Tensor& pos,
223 int num_heads, int head_dim);
224 void segment_softmax_stats(const ::brotensor::Tensor& logits,
225 const ::brotensor::Tensor& seg_offsets,
226 ::brotensor::Tensor& out);
227 void linear_forward_batched_ex(const ::brotensor::Tensor& W, const ::brotensor::Tensor* bias,
228 const ::brotensor::Tensor& X, int act, int epilogue,
229 ::brotensor::Tensor* workspace, ::brotensor::Tensor& Y);
230 void quantize_rows(const ::brotensor::Tensor& W, ::brotensor::Dtype fmt, ::brotensor::Tensor& Wq,
231 ::brotensor::Tensor& scales);
232 void linear_forward_batched_wq_ex(const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& scales,
233 const ::brotensor::Tensor* bias, const ::brotensor::Tensor& X, int act,
234 int epilogue, ::brotensor::Tensor& Y);
235 void dequantize_rows(const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& scales, ::brotensor::Dtype out,
236 ::brotensor::Tensor& W);
237 void layernorm_forward_inference_batched(const ::brotensor::Tensor& X_RD,
238 const ::brotensor::Tensor& gamma,
239 const ::brotensor::Tensor* beta,
240 ::brotensor::Tensor& Y_RD, float eps);
241 void layernorm_forward_inference_batched_fp16(const ::brotensor::Tensor& X_RD,
242 const ::brotensor::Tensor& gamma,
243 const ::brotensor::Tensor* beta,
244 ::brotensor::Tensor& Y_RD, float eps);
245 void layernorm_forward_batched_with_caches(const ::brotensor::Tensor& X_RD,
246 const ::brotensor::Tensor& gamma,
247 const ::brotensor::Tensor& beta,
248 ::brotensor::Tensor& Y_RD,
249 ::brotensor::Tensor& Xhat_RD,
250 ::brotensor::Tensor& Mean_R,
251 ::brotensor::Tensor& Rstd_R,
252 float eps);
253 void layernorm_backward_batched_with_caches(const ::brotensor::Tensor& dY_RD,
254 const ::brotensor::Tensor& Xhat_RD,
255 const ::brotensor::Tensor& gamma,
256 const ::brotensor::Tensor& Rstd_R,
257 ::brotensor::Tensor& dX_RD,
258 ::brotensor::Tensor& dGamma,
259 ::brotensor::Tensor& dBeta);
260 void build_causal_mask_row(int L, int q, ::brotensor::Tensor& mask);
261
262 // ── CHUNK 2 — activations.cpp / geglu.cpp / swiglu.cpp / matmul.cpp /
263 // rope.cpp / rms_norm.cpp ──
264 void silu_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
265 void silu_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY,
266 ::brotensor::Tensor& dX);
267 void gelu_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
268 void gelu_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY,
269 ::brotensor::Tensor& dX);
270 void gelu_exact_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
271 void gelu_exact_backward(const ::brotensor::Tensor& x,
272 const ::brotensor::Tensor& dY,
273 ::brotensor::Tensor& dX);
274 void quick_gelu_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
275 void quick_gelu_backward(const ::brotensor::Tensor& x,
276 const ::brotensor::Tensor& dY,
277 ::brotensor::Tensor& dX);
278 void geglu_forward(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y);
279 void geglu_backward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& dY,
280 ::brotensor::Tensor& dX);
281 void geglu_exact_forward(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y);
282 void geglu_exact_backward(const ::brotensor::Tensor& X,
283 const ::brotensor::Tensor& dY,
284 ::brotensor::Tensor& dX);
285 void swiglu_forward(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y);
286 void swiglu_backward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& dY,
287 ::brotensor::Tensor& dX);
288 void matmul(const ::brotensor::Tensor& A, const ::brotensor::Tensor& B,
289 ::brotensor::Tensor& C);
290 void matmul_abt(const ::brotensor::Tensor& A, const ::brotensor::Tensor& B,
291 ::brotensor::Tensor& C,
292 int batch, int M, int N, int K,
293 long long strideA, long long strideB, long long strideC,
294 const ::brotensor::Tensor* bias, int act);
295 void matmul_backward(const ::brotensor::Tensor& A, const ::brotensor::Tensor& B,
296 const ::brotensor::Tensor& dC,
297 ::brotensor::Tensor& dA, ::brotensor::Tensor& dB);
298 void rope_forward(const ::brotensor::Tensor& X, int head_dim, int num_heads,
299 int seq_offset, float theta_base, ::brotensor::Tensor& Y);
300 void rope_backward(const ::brotensor::Tensor& dY, int head_dim, int num_heads,
301 int seq_offset, float theta_base, ::brotensor::Tensor& dX);
302 void rms_norm_forward(const ::brotensor::Tensor& X,
303 const ::brotensor::Tensor& gamma,
304 float eps, ::brotensor::Tensor& Y);
305 void rms_norm_backward(const ::brotensor::Tensor& X,
306 const ::brotensor::Tensor& gamma,
307 const ::brotensor::Tensor& dY, float eps,
308 ::brotensor::Tensor& dX, ::brotensor::Tensor& dGamma);
309
310 // ── CHUNK 3 — conv2d.cpp / group_norm.cpp ──
311 void conv2d_forward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& Wt,
312 const ::brotensor::Tensor* bias,
313 int N, int C_in, int H, int W, int C_out, int kH, int kW,
314 int stride_h, int stride_w, int pad_h, int pad_w,
315 int dil_h, int dil_w, int groups, ::brotensor::Tensor& Y);
316 void conv2d_backward_input(const ::brotensor::Tensor& Wt,
317 const ::brotensor::Tensor& dY,
318 int N, int C_in, int H, int W,
319 int C_out, int kH, int kW,
320 int stride_h, int stride_w, int pad_h, int pad_w,
321 int dil_h, int dil_w, int groups,
322 ::brotensor::Tensor& dX);
323 void conv2d_backward_weight(const ::brotensor::Tensor& X,
324 const ::brotensor::Tensor& dY,
325 int N, int C_in, int H, int W,
326 int C_out, int kH, int kW,
327 int stride_h, int stride_w, int pad_h, int pad_w,
328 int dil_h, int dil_w, int groups,
329 ::brotensor::Tensor& dWt);
330 void conv2d_backward_bias(const ::brotensor::Tensor& dY,
331 int N, int C_out, int H_out, int W_out,
332 ::brotensor::Tensor& dB);
333 void deform_conv2d_forward(const ::brotensor::Tensor& X,
334 const ::brotensor::Tensor& offset,
335 const ::brotensor::Tensor* mask,
336 const ::brotensor::Tensor& Wt,
337 const ::brotensor::Tensor* bias,
338 int N, int C_in, int H, int W, int C_out, int kH, int kW,
339 int stride_h, int stride_w, int pad_h, int pad_w,
340 int dil_h, int dil_w, int groups, int deform_groups,
341 ::brotensor::Tensor& Y);
342 void conv3d_forward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& Wt,
343 const ::brotensor::Tensor* bias,
344 int N, int C_in, int T, int H, int W,
345 int C_out, int kT, int kH, int kW,
346 int stride_t, int stride_h, int stride_w,
347 int pad_t, int pad_h, int pad_w,
348 int dil_t, int dil_h, int dil_w, int groups,
349 ::brotensor::Tensor& Y);
350 void group_norm_forward(const ::brotensor::Tensor& X,
351 const ::brotensor::Tensor& gamma,
352 const ::brotensor::Tensor& beta,
353 int N, int C, int H, int W, int num_groups,
354 float eps, ::brotensor::Tensor& Y);
355 void group_norm_backward(const ::brotensor::Tensor& X,
356 const ::brotensor::Tensor& gamma,
357 const ::brotensor::Tensor& dY,
358 int N, int C, int H, int W, int num_groups, float eps,
359 ::brotensor::Tensor& dX, ::brotensor::Tensor& dGamma,
360 ::brotensor::Tensor& dBeta);
361
362 // ── CHUNK 4 — resample.cpp / transpose.cpp / diffusion_samplers.cpp /
363 // kv_cache.cpp ──
364 void upsample_nearest_2x(const ::brotensor::Tensor& X,
365 int N, int C, int H, int W, ::brotensor::Tensor& Y);
366 void upsample_bilinear_2x(const ::brotensor::Tensor& X,
367 int N, int C, int H, int W, ::brotensor::Tensor& Y);
368 void downsample_avg_2x(const ::brotensor::Tensor& X,
369 int N, int C, int H, int W, ::brotensor::Tensor& Y);
370 void upsample_nearest_2x_backward(const ::brotensor::Tensor& dY,
371 int N, int C, int H, int W,
372 ::brotensor::Tensor& dX);
373 void upsample_bilinear_2x_backward(const ::brotensor::Tensor& dY,
374 int N, int C, int H, int W,
375 ::brotensor::Tensor& dX);
376 void downsample_avg_2x_backward(const ::brotensor::Tensor& dY,
377 int N, int C, int H, int W,
378 ::brotensor::Tensor& dX);
379 void interp2d_forward(const ::brotensor::Tensor& X,
380 int N, int C, int H_in, int W_in,
381 int H_out, int W_out, int mode,
382 ::brotensor::Tensor& Y);
383 void interp2d_backward(const ::brotensor::Tensor& dY,
384 int N, int C, int H_in, int W_in,
385 int H_out, int W_out, int mode,
386 ::brotensor::Tensor& dX);
387 void interp2d_align_corners_forward(const ::brotensor::Tensor& X,
388 int N, int C, int H_in, int W_in,
389 int H_out, int W_out, int mode,
390 ::brotensor::Tensor& Y);
391 void pad2d_forward(const ::brotensor::Tensor& X,
392 int N, int C, int H, int W,
393 int pad_top, int pad_bottom, int pad_left, int pad_right,
394 int mode, ::brotensor::Tensor& Y);
395 void pad2d_backward(const ::brotensor::Tensor& dY,
396 int N, int C, int H, int W,
397 int pad_top, int pad_bottom, int pad_left, int pad_right,
398 int mode, ::brotensor::Tensor& dX);
399 void unfold2d_forward(const ::brotensor::Tensor& X,
400 int N, int C, int H, int W, int kH, int kW,
401 int stride_h, int stride_w,
402 int pad_top, int pad_bottom, int pad_left, int pad_right,
403 int mode, ::brotensor::Tensor& Y);
404 void l2_normalize_nchw_forward(const ::brotensor::Tensor& X,
405 int N, int C, int H, int W, float eps,
406 ::brotensor::Tensor& Y);
407 void convex_upsample_forward(const ::brotensor::Tensor& X,
408 const ::brotensor::Tensor& Mask,
409 int N, int C, int H, int W, int scale,
410 ::brotensor::Tensor& Y);
411 void slice2d_forward(const ::brotensor::Tensor& X,
412 int N, int C, int H, int W,
413 int h0, int w0, int H_out, int W_out,
414 ::brotensor::Tensor& Y);
415 void slice2d_backward(const ::brotensor::Tensor& dY,
416 int N, int C, int H, int W,
417 int h0, int w0, int H_out, int W_out,
418 ::brotensor::Tensor& dX);
419 void top_k_rows(const ::brotensor::Tensor& X, int k,
420 ::brotensor::Tensor& Vals, ::brotensor::Tensor& Idx);
421 void adaptive_avg_pool2d_forward(const ::brotensor::Tensor& X,
422 int N, int C, int H, int W,
423 int H_out, int W_out,
424 ::brotensor::Tensor& Y);
425 void adaptive_avg_pool2d_backward(const ::brotensor::Tensor& dY,
426 int N, int C, int H, int W,
427 int H_out, int W_out,
428 ::brotensor::Tensor& dX);
429 void max_pool2d_forward(const ::brotensor::Tensor& X,
430 int N, int C, int H, int W,
431 int kH, int kW, int stride_h, int stride_w,
432 int pad_h, int pad_w,
433 ::brotensor::Tensor& Y, ::brotensor::Tensor& Idx);
434 void max_pool2d_backward(const ::brotensor::Tensor& dY,
435 const ::brotensor::Tensor& Idx,
436 int N, int C, int H, int W, int H_out, int W_out,
437 ::brotensor::Tensor& dX);
438 void gather_rows(const ::brotensor::Tensor& X,
439 const ::brotensor::Tensor& Idx,
440 ::brotensor::Tensor& Y);
441 void scatter_rows_add(const ::brotensor::Tensor& dY,
442 const ::brotensor::Tensor& Idx, int R,
443 ::brotensor::Tensor& dX);
444 void scatter_rows(const ::brotensor::Tensor& Y,
445 const ::brotensor::Tensor& Idx,
446 ::brotensor::Tensor& X);
447 void conv_transpose2d_forward(const ::brotensor::Tensor& X,
448 const ::brotensor::Tensor& Wt,
449 const ::brotensor::Tensor* bias,
450 int N, int C_in, int H, int W,
451 int C_out, int kH, int kW,
452 int stride_h, int stride_w,
453 int pad_h, int pad_w,
454 int output_padding_h, int output_padding_w,
455 int dil_h, int dil_w, int groups,
456 ::brotensor::Tensor& Y);
457 void conv_transpose2d_backward_input(const ::brotensor::Tensor& Wt,
458 const ::brotensor::Tensor& dY,
459 int N, int C_in, int H, int W,
460 int C_out, int kH, int kW,
461 int stride_h, int stride_w,
462 int pad_h, int pad_w,
463 int output_padding_h, int output_padding_w,
464 int dil_h, int dil_w, int groups,
465 ::brotensor::Tensor& dX);
466 void conv_transpose2d_backward_weight(const ::brotensor::Tensor& X,
467 const ::brotensor::Tensor& dY,
468 int N, int C_in, int H, int W,
469 int C_out, int kH, int kW,
470 int stride_h, int stride_w,
471 int pad_h, int pad_w,
472 int output_padding_h, int output_padding_w,
473 int dil_h, int dil_w, int groups,
474 ::brotensor::Tensor& dWt);
475 void conv_transpose2d_backward_bias(const ::brotensor::Tensor& dY,
476 int N, int C_out, int H_out, int W_out,
477 ::brotensor::Tensor& dB);
478 void window_partition_forward(const ::brotensor::Tensor& X,
479 int N, int C, int H, int W, int window,
480 ::brotensor::Tensor& Y);
481 void window_reverse_forward(const ::brotensor::Tensor& X,
482 int N, int C, int H, int W, int window,
483 ::brotensor::Tensor& Y);
484 void nchw_to_sequence(const ::brotensor::Tensor& X,
485 int N, int C, int H, int W, ::brotensor::Tensor& Y);
486 void sequence_to_nchw(const ::brotensor::Tensor& X,
487 int N, int C, int H, int W, ::brotensor::Tensor& Y);
488 void ddim_step(const ::brotensor::Tensor& x_t,
489 const ::brotensor::Tensor& eps_pred,
490 float alpha_t, float alpha_prev, float sigma_t,
491 ::brotensor::Tensor& x_prev);
492 void euler_step(const ::brotensor::Tensor& x_t,
493 const ::brotensor::Tensor& eps_pred,
494 float sigma_t, float sigma_prev,
495 ::brotensor::Tensor& x_prev);
496 void dpmpp_2m_step(const ::brotensor::Tensor& x_t,
497 const ::brotensor::Tensor& eps_pred,
498 const ::brotensor::Tensor& x0_prev,
499 float sigma_t,
500 float c_xt, float c_x0t, float c_x0prev,
501 ::brotensor::Tensor& x_prev, ::brotensor::Tensor& x0_out);
502 void timestep_embedding(const ::brotensor::Tensor& timesteps,
503 int dim, float max_period, ::brotensor::Tensor& Y);
504 void kv_cache_append(const ::brotensor::Tensor& K_new,
505 const ::brotensor::Tensor& V_new, int cur_len,
506 ::brotensor::Tensor& K_cache, ::brotensor::Tensor& V_cache);
507 void flash_attention_decode(const ::brotensor::Tensor& Q,
508 const ::brotensor::Tensor& K_cache,
509 const ::brotensor::Tensor& V_cache,
510 int valid_len,
511 int num_q_heads, int num_kv_heads,
512 ::brotensor::Tensor& O,
513 float attn_softcap, int window);
514 void flash_attention_decode_masked(const ::brotensor::Tensor& Q,
515 const ::brotensor::Tensor& K_cache,
516 const ::brotensor::Tensor& V_cache,
517 const float* d_mask,
518 int num_q_heads, int num_kv_heads,
519 ::brotensor::Tensor& O,
520 float attn_softcap, int window);
521 void flash_attention_decode_len(const ::brotensor::Tensor& Q, const ::brotensor::Tensor& K_cache,
522 const ::brotensor::Tensor& V_cache, const int32_t* d_len, int num_q_heads,
523 int num_kv_heads, ::brotensor::Tensor& O);
524
525 // ── CHUNK 5 — cross_attention.cpp / self_attention.cpp /
526 // attention_moments.cpp ──
527 void cross_attention_forward(const ::brotensor::Tensor& X,
528 const ::brotensor::Tensor& Ctx,
529 const ::brotensor::Tensor& Wq,
530 const ::brotensor::Tensor& Wk,
531 const ::brotensor::Tensor& Wv,
532 const ::brotensor::Tensor& Wo,
533 const float* d_mask, int num_heads,
534 ::brotensor::Tensor& O);
535 void cross_attention_forward_with_attn(const ::brotensor::Tensor& X,
536 const ::brotensor::Tensor& Ctx,
537 const ::brotensor::Tensor& Wq,
538 const ::brotensor::Tensor& Wk,
539 const ::brotensor::Tensor& Wv,
540 const ::brotensor::Tensor& Wo,
541 const float* d_mask,
542 const ::brotensor::Tensor* attn_logit_bias,
543 int num_heads,
544 ::brotensor::Tensor& O,
545 ::brotensor::Tensor& AttnAvg);
546 void self_attention_forward_train(const ::brotensor::Tensor& X,
547 const ::brotensor::Tensor& Wq,
548 const ::brotensor::Tensor& Wk,
549 const ::brotensor::Tensor& Wv,
550 const ::brotensor::Tensor& Wo,
551 const float* d_mask, int num_heads,
552 ::brotensor::Tensor& Qh,
553 ::brotensor::Tensor& Kh,
554 ::brotensor::Tensor& Vh,
555 ::brotensor::Tensor& Attnh,
556 ::brotensor::Tensor& Yconcat,
557 ::brotensor::Tensor& O);
558 void self_attention_backward(const ::brotensor::Tensor& dO,
559 const ::brotensor::Tensor& X,
560 const ::brotensor::Tensor& Qh,
561 const ::brotensor::Tensor& Kh,
562 const ::brotensor::Tensor& Vh,
563 const ::brotensor::Tensor& Attnh,
564 const ::brotensor::Tensor& Yconcat,
565 const ::brotensor::Tensor& Wq,
566 const ::brotensor::Tensor& Wk,
567 const ::brotensor::Tensor& Wv,
568 const ::brotensor::Tensor& Wo,
569 const float* d_mask, int num_heads,
570 ::brotensor::Tensor& dX,
571 ::brotensor::Tensor& dWq,
572 ::brotensor::Tensor& dWk,
573 ::brotensor::Tensor& dWv,
574 ::brotensor::Tensor& dWo);
575 void attention_token_moments(const ::brotensor::Tensor& Attn,
576 int h_lat, int w_lat,
577 ::brotensor::Tensor& mass,
578 ::brotensor::Tensor& centroid);
579 void cross_attention_forward_train(const ::brotensor::Tensor& X,
580 const ::brotensor::Tensor& Ctx,
581 const ::brotensor::Tensor& Wq,
582 const ::brotensor::Tensor& Wk,
583 const ::brotensor::Tensor& Wv,
584 const ::brotensor::Tensor& Wo,
585 const float* d_mask, int num_heads,
586 ::brotensor::Tensor& Qh,
587 ::brotensor::Tensor& Kh,
588 ::brotensor::Tensor& Vh,
589 ::brotensor::Tensor& Attnh,
590 ::brotensor::Tensor& Yconcat,
591 ::brotensor::Tensor& O);
592 void cross_attention_backward(const ::brotensor::Tensor& dO,
593 const ::brotensor::Tensor& X,
594 const ::brotensor::Tensor& Ctx,
595 const ::brotensor::Tensor& Qh,
596 const ::brotensor::Tensor& Kh,
597 const ::brotensor::Tensor& Vh,
598 const ::brotensor::Tensor& Attnh,
599 const ::brotensor::Tensor& Yconcat,
600 const ::brotensor::Tensor& Wq,
601 const ::brotensor::Tensor& Wk,
602 const ::brotensor::Tensor& Wv,
603 const ::brotensor::Tensor& Wo,
604 const float* d_mask, int num_heads,
605 ::brotensor::Tensor& dX,
606 ::brotensor::Tensor& dCtx,
607 ::brotensor::Tensor& dWq,
608 ::brotensor::Tensor& dWk,
609 ::brotensor::Tensor& dWv,
610 ::brotensor::Tensor& dWo);
611
612 // ── CHUNK 6 — flash_attention.cpp / self_attention_inference.cpp /
613 // resblock.cpp ──
614 void flash_attention_forward(const ::brotensor::Tensor& Q,
615 const ::brotensor::Tensor& K,
616 const ::brotensor::Tensor& V,
617 const float* d_mask, int num_heads, bool causal,
618 ::brotensor::Tensor& O);
619 void flash_attention_windowed_forward(const ::brotensor::Tensor& Q,
620 const ::brotensor::Tensor& K,
621 const ::brotensor::Tensor& V,
622 const float* d_mask, int num_heads, int window,
623 ::brotensor::Tensor& O, bool causal);
624 void flash_attention_gqa_forward(const ::brotensor::Tensor& Q,
625 const ::brotensor::Tensor& K,
626 const ::brotensor::Tensor& V,
627 const float* d_mask, int num_q_heads, int num_kv_heads,
628 bool causal, ::brotensor::Tensor& O);
629 void flash_attention_varlen_forward(const ::brotensor::Tensor& Q,
630 const ::brotensor::Tensor& K,
631 const ::brotensor::Tensor& V,
632 const int32_t* cu_seqlens_q,
633 const int32_t* cu_seqlens_k,
634 int batch_size, int max_seqlen_q, int max_seqlen_k,
635 int num_heads, int head_dim, bool causal,
636 ::brotensor::Tensor& O);
637 void flash_attention_varlen_backward(const ::brotensor::Tensor& Q,
638 const ::brotensor::Tensor& K,
639 const ::brotensor::Tensor& V,
640 const ::brotensor::Tensor& O,
641 const ::brotensor::Tensor& dO,
642 const int32_t* cu_seqlens_q,
643 const int32_t* cu_seqlens_k,
644 int batch_size, int max_seqlen_q, int max_seqlen_k,
645 int num_heads, int head_dim, bool causal,
646 ::brotensor::Tensor& dQ,
647 ::brotensor::Tensor& dK,
648 ::brotensor::Tensor& dV);
649 void flash_attention_qkvo_forward(const ::brotensor::Tensor& X,
650 const ::brotensor::Tensor* Ctx,
651 const ::brotensor::Tensor& Wq,
652 const ::brotensor::Tensor* bq,
653 const ::brotensor::Tensor& Wk,
654 const ::brotensor::Tensor* bk,
655 const ::brotensor::Tensor& Wv,
656 const ::brotensor::Tensor* bv,
657 const ::brotensor::Tensor& Wo,
658 const ::brotensor::Tensor* bo,
659 const float* d_mask, int num_heads,
660 bool causal, ::brotensor::Tensor& O);
661 void flash_attention_qkvo_backward(const ::brotensor::Tensor& X,
662 const ::brotensor::Tensor* Ctx,
663 const ::brotensor::Tensor& Wq,
664 const ::brotensor::Tensor* bq,
665 const ::brotensor::Tensor& Wk,
666 const ::brotensor::Tensor* bk,
667 const ::brotensor::Tensor& Wv,
668 const ::brotensor::Tensor* bv,
669 const ::brotensor::Tensor& Wo,
670 const ::brotensor::Tensor* bo,
671 const float* d_mask, int num_heads,
672 bool causal,
673 const ::brotensor::Tensor& dO,
674 ::brotensor::Tensor& dX,
675 ::brotensor::Tensor* dCtx,
676 ::brotensor::Tensor& dWq,
677 ::brotensor::Tensor* dbq,
678 ::brotensor::Tensor& dWk,
679 ::brotensor::Tensor* dbk,
680 ::brotensor::Tensor& dWv,
681 ::brotensor::Tensor* dbv,
682 ::brotensor::Tensor& dWo,
683 ::brotensor::Tensor* dbo);
684 void flash_attention_backward(const ::brotensor::Tensor& Q,
685 const ::brotensor::Tensor& K,
686 const ::brotensor::Tensor& V,
687 const ::brotensor::Tensor& O,
688 const ::brotensor::Tensor& dO,
689 const float* d_mask, int num_heads, bool causal,
690 ::brotensor::Tensor& dQ,
691 ::brotensor::Tensor& dK,
692 ::brotensor::Tensor& dV);
693 void flash_attention_project_kv(const ::brotensor::Tensor& ctx,
694 const ::brotensor::Tensor& Wk,
695 const ::brotensor::Tensor* bk,
696 const ::brotensor::Tensor& Wv,
697 const ::brotensor::Tensor* bv,
698 ::brotensor::Tensor& K_out,
699 ::brotensor::Tensor& V_out);
700 void flash_attention_q_with_kv_cached_forward(const ::brotensor::Tensor& X,
701 const ::brotensor::Tensor& K,
702 const ::brotensor::Tensor& V,
703 const ::brotensor::Tensor& Wq,
704 const ::brotensor::Tensor* bq,
705 const ::brotensor::Tensor& Wo,
706 const ::brotensor::Tensor* bo,
707 const float* d_mask,
708 int num_heads, bool causal,
709 ::brotensor::Tensor& O);
710 void self_attention_forward(const ::brotensor::Tensor& X,
711 const ::brotensor::Tensor& Wq,
712 const ::brotensor::Tensor& Wk,
713 const ::brotensor::Tensor& Wv,
714 const ::brotensor::Tensor& Wo,
715 const float* d_mask, int num_heads,
716 ::brotensor::Tensor& O);
717 void resblock_forward(const ::brotensor::Tensor& X,
718 const ::brotensor::Tensor& gamma1,
719 const ::brotensor::Tensor& beta1,
720 const ::brotensor::Tensor& W1,
721 const ::brotensor::Tensor* b1,
722 const ::brotensor::Tensor* t_emb_shift,
723 const ::brotensor::Tensor& gamma2,
724 const ::brotensor::Tensor& beta2,
725 const ::brotensor::Tensor& W2,
726 const ::brotensor::Tensor* b2,
727 const ::brotensor::Tensor* Wskip,
728 const ::brotensor::Tensor* bskip,
729 int N, int C_in, int C_out, int H, int W,
730 int num_groups, float eps, ::brotensor::Tensor& Y);
731 void resblock_backward(const ::brotensor::Tensor& X,
732 const ::brotensor::Tensor& gamma1,
733 const ::brotensor::Tensor& beta1,
734 const ::brotensor::Tensor& W1,
735 const ::brotensor::Tensor* b1,
736 const ::brotensor::Tensor* t_emb_shift,
737 const ::brotensor::Tensor& gamma2,
738 const ::brotensor::Tensor& beta2,
739 const ::brotensor::Tensor& W2,
740 const ::brotensor::Tensor* b2,
741 const ::brotensor::Tensor* Wskip,
742 const ::brotensor::Tensor* bskip,
743 int N, int C_in, int C_out, int H, int W,
744 int num_groups, float eps,
745 const ::brotensor::Tensor& dY,
746 ::brotensor::Tensor& dX,
747 ::brotensor::Tensor& dGamma1,
748 ::brotensor::Tensor& dBeta1,
749 ::brotensor::Tensor& dW1,
750 ::brotensor::Tensor* db1,
751 ::brotensor::Tensor* dt_emb_shift,
752 ::brotensor::Tensor& dGamma2,
753 ::brotensor::Tensor& dBeta2,
754 ::brotensor::Tensor& dW2,
755 ::brotensor::Tensor* db2,
756 ::brotensor::Tensor* dWskip,
757 ::brotensor::Tensor* dbskip);
758
759 // ── DiT / diffusion extras — modulate.cpp ──
760 void modulate(const ::brotensor::Tensor& X, const ::brotensor::Tensor& scale,
761 const ::brotensor::Tensor& shift, ::brotensor::Tensor& Y);
762 void broadcast_mul(const ::brotensor::Tensor& X, const ::brotensor::Tensor& v,
763 ::brotensor::Tensor& Y);
764
765 // ── RoPE with explicit cos/sin tables — rope.cpp ──
766 void rope_apply(const ::brotensor::Tensor& X, const ::brotensor::Tensor& cos_tbl,
767 const ::brotensor::Tensor& sin_tbl, int head_dim, int num_heads,
768 ::brotensor::Tensor& Y);
769 void rope_apply_perhead(const ::brotensor::Tensor& X, const ::brotensor::Tensor& cos_tbl,
770 const ::brotensor::Tensor& sin_tbl, int head_dim, int num_heads,
771 ::brotensor::Tensor& Y);
772 void rope_apply_backward(const ::brotensor::Tensor& dY,
773 const ::brotensor::Tensor& cos_tbl,
774 const ::brotensor::Tensor& sin_tbl,
775 int head_dim, int num_heads, ::brotensor::Tensor& dX);
776
777 // ── Self-attention with additive bias — self_attention_bias.cpp ──
778 void self_attention_bias_forward(const ::brotensor::Tensor& X,
779 const ::brotensor::Tensor& Wq,
780 const ::brotensor::Tensor& Wk,
781 const ::brotensor::Tensor& Wv,
782 const ::brotensor::Tensor& Wo,
783 const ::brotensor::Tensor* bq,
784 const ::brotensor::Tensor* bk,
785 const ::brotensor::Tensor* bv,
786 const ::brotensor::Tensor* bo,
787 const float* d_mask,
788 const ::brotensor::Tensor* attn_bias,
789 int num_heads, float scale,
790 ::brotensor::Tensor& O);
791 void rel_pos_bias_xl_forward(const ::brotensor::Tensor& Qv,
792 const ::brotensor::Tensor& Pk,
793 int num_heads, int head_dim,
794 ::brotensor::Tensor& Bias);
795
796 // ── Self-attention with decomposed 2D rel-pos bias (SAM/ViTDet) —
797 // self_attention_decomposed_rel_pos.cpp ──
798 void self_attention_decomposed_rel_pos_forward(
799 const ::brotensor::Tensor& X,
800 const ::brotensor::Tensor& Wq, const ::brotensor::Tensor* bq,
801 const ::brotensor::Tensor& Wk, const ::brotensor::Tensor* bk,
802 const ::brotensor::Tensor& Wv, const ::brotensor::Tensor* bv,
803 const ::brotensor::Tensor& Wo, const ::brotensor::Tensor* bo,
804 const ::brotensor::Tensor& rel_pos_h, const ::brotensor::Tensor& rel_pos_w,
805 int num_heads, int grid_h, int grid_w, float scale,
806 ::brotensor::Tensor& O);
807 void self_attention_decomposed_rel_pos_windowed_forward(
808 const ::brotensor::Tensor& X,
809 const ::brotensor::Tensor& Wq, const ::brotensor::Tensor* bq,
810 const ::brotensor::Tensor& Wk, const ::brotensor::Tensor* bk,
811 const ::brotensor::Tensor& Wv, const ::brotensor::Tensor* bv,
812 const ::brotensor::Tensor& Wo, const ::brotensor::Tensor* bo,
813 const ::brotensor::Tensor& rel_pos_h, const ::brotensor::Tensor& rel_pos_w,
814 int num_heads, int grid_h, int grid_w, int window, float scale,
815 ::brotensor::Tensor& O);
816
817 // ── Spectral / FFT core (brosoundml) — fft.cpp ──
818 void complex_mul(const ::brotensor::Tensor& a, const ::brotensor::Tensor& b,
819 ::brotensor::Tensor& y);
820 void complex_mul_backward(const ::brotensor::Tensor& a,
821 const ::brotensor::Tensor& b,
822 const ::brotensor::Tensor& dY,
823 ::brotensor::Tensor& dA, ::brotensor::Tensor& dB);
824 void complex_abs(const ::brotensor::Tensor& z, ::brotensor::Tensor& y);
825 void complex_abs_backward(const ::brotensor::Tensor& z,
826 const ::brotensor::Tensor& dY,
827 ::brotensor::Tensor& dZ);
828 void complex_angle(const ::brotensor::Tensor& z, ::brotensor::Tensor& y);
829 void complex_from_polar(const ::brotensor::Tensor& mag,
830 const ::brotensor::Tensor& phase,
831 ::brotensor::Tensor& y);
832 void fft(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
833 void ifft(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
834 void rfft(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
835 void irfft(const ::brotensor::Tensor& x, int L, ::brotensor::Tensor& y);
836 void rfft_backward(const ::brotensor::Tensor& dY, int L,
837 ::brotensor::Tensor& dX);
838 void irfft_backward(const ::brotensor::Tensor& dY, ::brotensor::Tensor& dX);
839
840 // ── STFT / iSTFT (brosoundml) — stft.cpp ──
841 void stft(const ::brotensor::Tensor& signal, const ::brotensor::Tensor& window,
842 int N, int n_fft, int hop_length, int win_length,
843 bool center, bool normalized, ::brotensor::Tensor& spec);
844 void stft_backward(const ::brotensor::Tensor& dSpec,
845 const ::brotensor::Tensor& window,
846 int N, int signal_len, int n_fft, int hop_length,
847 int win_length, bool center, bool normalized,
848 ::brotensor::Tensor& dSignal);
849 void istft(const ::brotensor::Tensor& spec, const ::brotensor::Tensor& window,
850 int N, int signal_len, int n_fft, int hop_length, int win_length,
851 bool center, bool normalized, ::brotensor::Tensor& signal);
852 void istft_backward(const ::brotensor::Tensor& dSignal,
853 const ::brotensor::Tensor& window,
854 int N, int signal_len, int n_fft, int hop_length,
855 int win_length, bool center, bool normalized,
856 ::brotensor::Tensor& dSpec);
857
858 // ── 1D convolution family (brosoundml) — conv1d.cpp ──
859 void conv_transpose1d_forward(const ::brotensor::Tensor& X,
860 const ::brotensor::Tensor& Wt,
861 const ::brotensor::Tensor* bias,
862 int N, int C_in, int L, int C_out, int kL,
863 int stride, int padding, int output_padding,
864 int dilation, int groups, ::brotensor::Tensor& Y);
865 void conv_transpose1d_backward_input(const ::brotensor::Tensor& Wt,
866 const ::brotensor::Tensor& dY,
867 int N, int C_in, int L, int C_out, int kL,
868 int stride, int padding,
869 int output_padding, int dilation,
870 int groups, ::brotensor::Tensor& dX);
871 void conv_transpose1d_backward_weight(const ::brotensor::Tensor& X,
872 const ::brotensor::Tensor& dY,
873 int N, int C_in, int L, int C_out, int kL,
874 int stride, int padding,
875 int output_padding, int dilation,
876 int groups, ::brotensor::Tensor& dWt);
877 void conv_transpose1d_backward_bias(const ::brotensor::Tensor& dY,
878 int N, int C_out, int L_out,
879 ::brotensor::Tensor& dB);
880 void causal_conv1d_update(const ::brotensor::Tensor& X,
881 const ::brotensor::Tensor& Wt,
882 const ::brotensor::Tensor* bias,
883 int N, int C, int L_step, int kL, int dilation,
884 ::brotensor::Tensor& state, ::brotensor::Tensor& Y);
885 void pad1d_forward(const ::brotensor::Tensor& X, int N, int C, int L,
886 int pad_left, int pad_right, int mode,
887 ::brotensor::Tensor& Y);
888 void pad1d_backward(const ::brotensor::Tensor& dY, int N, int C, int L,
889 int pad_left, int pad_right, int mode,
890 ::brotensor::Tensor& dX);
891
892 // ── Vocoder / codec activations (brosoundml) — vocoder_activations.cpp ──
893 void snake_forward(const ::brotensor::Tensor& X,
894 const ::brotensor::Tensor& alpha,
895 const ::brotensor::Tensor* beta,
896 int N, int C, int L, ::brotensor::Tensor& Y);
897 void snake_backward(const ::brotensor::Tensor& X,
898 const ::brotensor::Tensor& alpha,
899 const ::brotensor::Tensor* beta,
900 const ::brotensor::Tensor& dY,
901 int N, int C, int L,
902 ::brotensor::Tensor& dX, ::brotensor::Tensor& dAlpha,
903 ::brotensor::Tensor* dBeta);
904 void elu_forward(const ::brotensor::Tensor& x, float alpha,
905 ::brotensor::Tensor& y);
906 void elu_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY,
907 float alpha, ::brotensor::Tensor& dX);
908 void leaky_relu_forward(const ::brotensor::Tensor& x, float negative_slope,
909 ::brotensor::Tensor& y);
910 void leaky_relu_backward(const ::brotensor::Tensor& x,
911 const ::brotensor::Tensor& dY,
912 float negative_slope, ::brotensor::Tensor& dX);
913
914 // ── Codec quantization (brosoundml CHUNK 5, family D) — codec_quant.cpp ──
915 void vq_encode_forward(const ::brotensor::Tensor& x,
916 const ::brotensor::Tensor& codebook,
917 ::brotensor::Tensor& indices,
918 ::brotensor::Tensor& quantized);
919 void vq_encode_backward(const ::brotensor::Tensor& dQuantized,
920 ::brotensor::Tensor& dX);
921 void fsq_quantize_forward(const ::brotensor::Tensor& x,
922 const ::brotensor::Tensor& levels,
923 ::brotensor::Tensor& quantized,
924 ::brotensor::Tensor& packed_indices);
925 void fsq_quantize_backward(const ::brotensor::Tensor& dQuantized,
926 ::brotensor::Tensor& dX);
927
928 // ── 1D resampling (brosoundml CHUNK 6, family E) — resample1d.cpp ──
929 void resample1d_forward(const ::brotensor::Tensor& X,
930 int N, int C, int L_in, int L_out, int mode,
931 ::brotensor::Tensor& Y);
932 void resample1d_backward(const ::brotensor::Tensor& dY,
933 int N, int C, int L_in, int L_out, int mode,
934 ::brotensor::Tensor& dX);
935
936 // ── log / exp / round elementwise (brosoundml CHUNK 6, family G)
937 // — log_exp_round.cpp ──
938 void log_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
939 void log_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY,
940 ::brotensor::Tensor& dX);
941 void exp_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
942 void exp_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY,
943 ::brotensor::Tensor& dX);
944 void round_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
945 void round_backward(const ::brotensor::Tensor& dY, ::brotensor::Tensor& dX);
946
947 // ── Autoregressive logit sampling (brosoundml CHUNK 7, family F)
948 // — sample_logits.cpp ──
949 void sample_logits(const ::brotensor::Tensor& logits, float temperature,
950 int top_k, float top_p, uint64_t key, uint64_t counter,
951 ::brotensor::Tensor& indices);
952 void sample_logits_into(const ::brotensor::Tensor& logits, float temperature,
953 int top_k, float top_p, uint64_t key,
954 ::brotensor::Tensor& counter,
955 ::brotensor::Tensor& scratch,
956 ::brotensor::Tensor& indices);
957
958 // ── Masked-diffusion token selection — masked_diffusion.cpp ──
959 void masked_diffusion_scores(const ::brotensor::Tensor& logits,
960 const ::brotensor::Tensor& tokens,
961 int T, int C, int V, int mask_id,
962 float guidance_scale, float layer_penalty,
963 float position_temperature, float class_temperature,
964 float class_top_frac, uint64_t seed,
965 ::brotensor::Tensor& pred, ::brotensor::Tensor& scores,
966 ::brotensor::Tensor& confidence);
967 void masked_diffusion_commit(const ::brotensor::Tensor& pred,
968 const ::brotensor::Tensor& idx, int k, int step,
969 ::brotensor::Tensor& tokens,
970 ::brotensor::Tensor& unmask_step);
971
972 // ── Counter-based noise generation (Philox 4x32-10) — noise.cpp ──
973 void randn(uint64_t key, uint64_t counter, ::brotensor::Tensor& Y);
974 void rand_uniform(uint64_t key, uint64_t counter, ::brotensor::Tensor& Y);
975 void rand_bernoulli(float p, uint64_t key, uint64_t counter,
976 ::brotensor::Tensor& Y);
977 void randn_truncated(float lo, float hi, uint64_t key, uint64_t counter,
978 ::brotensor::Tensor& Y);
979
980 // ── L2 norm + Gated Delta Rule (linear-attention text path) ──
981 // l2_norm.cpp, gated_delta_rule.cpp
982 void l2_norm_forward(const ::brotensor::Tensor& X,
983 int head_dim, int num_heads, float eps,
984 ::brotensor::Tensor& Y);
985 void l2_norm_backward(const ::brotensor::Tensor& X,
986 int head_dim, int num_heads, float eps,
987 const ::brotensor::Tensor& dY,
988 ::brotensor::Tensor& dX);
989 void gated_delta_rule_chunked(const ::brotensor::Tensor& Q,
990 const ::brotensor::Tensor& K,
991 const ::brotensor::Tensor& V,
992 const ::brotensor::Tensor& a_raw,
993 const ::brotensor::Tensor& beta,
994 const ::brotensor::Tensor& log_A,
995 int num_heads, int d_k, int d_v,
996 ::brotensor::Tensor& state,
997 ::brotensor::Tensor& O);
998 void gated_delta_rule_step(const ::brotensor::Tensor& Q,
999 const ::brotensor::Tensor& K,
1000 const ::brotensor::Tensor& V,
1001 const ::brotensor::Tensor& a_raw,
1002 const ::brotensor::Tensor& beta,
1003 const ::brotensor::Tensor& log_A,
1004 int num_heads, int d_k, int d_v,
1005 ::brotensor::Tensor& state,
1006 ::brotensor::Tensor& O);
1007
1008 // ── Qwen3-VL polish: spatial_merge_2x2 + M-RoPE — spatial_merge.cpp /
1009 // rope_mrope.cpp ──
1010 void spatial_merge_2x2_forward(const ::brotensor::Tensor& X,
1011 int N, int C, int H, int W,
1012 bool channel_major,
1013 ::brotensor::Tensor& Y);
1014 void pixel_shuffle_upsample_2x_forward(const ::brotensor::Tensor& X,
1015 int N, int C_in, int H, int W,
1016 int C_out, ::brotensor::Tensor& Y);
1017 void patch_unpack_forward(const ::brotensor::Tensor& tokens,
1018 int hp, int wp, int P, int C_total, int C_keep,
1019 bool channel_major, ::brotensor::Tensor& Y);
1020 void rope_apply_mrope(const ::brotensor::Tensor& X,
1021 const ::brotensor::Tensor& cos_t,
1022 const ::brotensor::Tensor& sin_t,
1023 const ::brotensor::Tensor& cos_h,
1024 const ::brotensor::Tensor& sin_h,
1025 const ::brotensor::Tensor& cos_w,
1026 const ::brotensor::Tensor& sin_w,
1027 const int32_t* pos_t, const int32_t* pos_h,
1028 const int32_t* pos_w,
1029 int head_dim, int num_heads,
1030 int d_t, int d_h, int d_w,
1031 ::brotensor::Tensor& Y);
1032
1033 // ── BatchNorm — batch_norm.cpp ──
1034 void batch_norm_forward(const ::brotensor::Tensor& X,
1035 const ::brotensor::Tensor& gamma,
1036 const ::brotensor::Tensor& beta,
1037 ::brotensor::Tensor& running_mean,
1038 ::brotensor::Tensor& running_var,
1039 int N, int C, int H, int W,
1040 float eps, float momentum,
1041 ::brotensor::Tensor& Y,
1042 ::brotensor::Tensor& saved_mean,
1043 ::brotensor::Tensor& saved_rstd);
1044 void batch_norm_inference(const ::brotensor::Tensor& X,
1045 const ::brotensor::Tensor& gamma,
1046 const ::brotensor::Tensor& beta,
1047 const ::brotensor::Tensor& running_mean,
1048 const ::brotensor::Tensor& running_var,
1049 int N, int C, int H, int W,
1050 float eps,
1051 ::brotensor::Tensor& Y);
1052 void batch_norm_backward(const ::brotensor::Tensor& X,
1053 const ::brotensor::Tensor& gamma,
1054 const ::brotensor::Tensor& saved_mean,
1055 const ::brotensor::Tensor& saved_rstd,
1056 const ::brotensor::Tensor& dY,
1057 int N, int C, int H, int W,
1058 ::brotensor::Tensor& dX,
1059 ::brotensor::Tensor& dGamma,
1060 ::brotensor::Tensor& dBeta);
1061
1062 // ── Image preprocessing helpers — image_preproc.cpp ──
1063 void image_normalize(const ::brotensor::Tensor& X,
1064 const ::brotensor::Tensor& mean,
1065 const ::brotensor::Tensor& std_,
1066 int N, int C, int H, int W,
1067 ::brotensor::Tensor& Y);
1068 void image_u8_to_f32_nhwc_to_nchw(const uint8_t* src,
1069 int N, int H, int W, int C,
1070 float scale, float bias,
1071 ::brotensor::Tensor& Y);
1072 void lstm_forward_train(const ::brotensor::Tensor& X, const ::brotensor::Tensor& W_ih,
1073 const ::brotensor::Tensor& W_hh,
1074 const ::brotensor::Tensor* b_ih, const ::brotensor::Tensor* b_hh,
1075 const ::brotensor::Tensor* h0, const ::brotensor::Tensor* c0,
1076 int T, int B,
1077 ::brotensor::Tensor& Y, ::brotensor::Tensor& gates,
1078 ::brotensor::Tensor& C,
1079 ::brotensor::Tensor* hT, ::brotensor::Tensor* cT);
1080 void lstm_backward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& W_ih,
1081 const ::brotensor::Tensor& W_hh,
1082 const ::brotensor::Tensor* h0, const ::brotensor::Tensor* c0,
1083 const ::brotensor::Tensor& Y, const ::brotensor::Tensor& gates,
1084 const ::brotensor::Tensor& C,
1085 const ::brotensor::Tensor& dY, int T, int B,
1086 ::brotensor::Tensor& dX, ::brotensor::Tensor& dW_ih,
1087 ::brotensor::Tensor& dW_hh,
1088 ::brotensor::Tensor* db_ih, ::brotensor::Tensor* db_hh,
1089 ::brotensor::Tensor* dh0, ::brotensor::Tensor* dc0);
1090
1091 // ── StyleGAN3-R primitives — stylegan_elementwise.cpp / bias_act.cpp /
1092 // upfirdn2d.cpp / modulated_conv2d.cpp ──
1093 void sin_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
1094 void sin_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY,
1095 ::brotensor::Tensor& dX);
1096 void cos_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
1097 void cos_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY,
1098 ::brotensor::Tensor& dX);
1099 void rsqrt_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
1100 void rsqrt_backward(const ::brotensor::Tensor& y, const ::brotensor::Tensor& dY,
1101 ::brotensor::Tensor& dX);
1102 void pixel_norm_forward(const ::brotensor::Tensor& X, float eps,
1103 ::brotensor::Tensor& Y);
1104 void pixel_norm_backward(const ::brotensor::Tensor& X,
1105 const ::brotensor::Tensor& dY, float eps,
1106 ::brotensor::Tensor& dX);
1107 void bias_act_forward(const ::brotensor::Tensor& X, const ::brotensor::Tensor* b,
1108 int N, int C, int HW, int act, float alpha,
1109 float gain, float clamp, ::brotensor::Tensor& Y);
1110 void bias_act_backward(const ::brotensor::Tensor& dY, const ::brotensor::Tensor& X,
1111 const ::brotensor::Tensor* b,
1112 int N, int C, int HW, int act, float alpha,
1113 float gain, float clamp,
1114 ::brotensor::Tensor& dX, ::brotensor::Tensor* dB);
1115 void upfirdn2d_forward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& f,
1116 int N, int C, int H, int Wd, int fH, int fW,
1117 int up_x, int up_y, int down_x, int down_y,
1118 int pad_x0, int pad_x1, int pad_y0, int pad_y1,
1119 bool flip_filter, float gain, ::brotensor::Tensor& Y);
1120 void upfirdn2d_backward(const ::brotensor::Tensor& dY, const ::brotensor::Tensor& f,
1121 int N, int C, int H, int Wd, int fH, int fW,
1122 int up_x, int up_y, int down_x, int down_y,
1123 int pad_x0, int pad_x1, int pad_y0, int pad_y1,
1124 bool flip_filter, float gain, ::brotensor::Tensor& dX);
1125 void modulated_conv2d_forward(const ::brotensor::Tensor& X,
1126 const ::brotensor::Tensor& W,
1127 const ::brotensor::Tensor& s,
1128 int N, int C_in, int H, int Wd,
1129 int C_out, int kH, int kW, int pad_h, int pad_w,
1130 bool demodulate, float eps,
1131 ::brotensor::Tensor& dcoef, ::brotensor::Tensor& Y);
1132 void modulated_conv2d_backward(const ::brotensor::Tensor& X,
1133 const ::brotensor::Tensor& W,
1134 const ::brotensor::Tensor& s,
1135 const ::brotensor::Tensor& dcoef,
1136 const ::brotensor::Tensor& dY,
1137 int N, int C_in, int H, int Wd,
1138 int C_out, int kH, int kW, int pad_h, int pad_w,
1139 bool demodulate, float eps,
1140 ::brotensor::Tensor& dX, ::brotensor::Tensor& dW,
1141 ::brotensor::Tensor& ds);
1142
1143 // ── Quantized linear forward & dequant (Q8_0 & Q4_K) via Brass AVX2/FMA JIT ──
1144 void dequant_q8_0_to_fp16(const ::brotensor::Tensor& W_q8, ::brotensor::Tensor& W_fp16);
1145 void linear_forward_q8_0_fp16(const ::brotensor::Tensor& W_q8, const ::brotensor::Tensor* bias,
1146 const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
1147 void linear_forward_batched_q8_0_fp16(const ::brotensor::Tensor& W_q8, const ::brotensor::Tensor* bias,
1148 const ::brotensor::Tensor& X_BD, ::brotensor::Tensor& Y_BD);
1149 void dequant_q4k_to_fp16(const ::brotensor::Tensor& W_q4k, ::brotensor::Tensor& W_fp16);
1150 void linear_forward_q4k_fp16(const ::brotensor::Tensor& W_q4k, const ::brotensor::Tensor* bias,
1151 const ::brotensor::Tensor& x, ::brotensor::Tensor& y);
1152 void linear_forward_batched_q4k_fp16(const ::brotensor::Tensor& W_q4k, const ::brotensor::Tensor* bias,
1153 const ::brotensor::Tensor& X_BD, ::brotensor::Tensor& Y_BD);
1154
1155 } // namespace brotensor::detail::cpu
1156
1157 namespace {
1158
1159 struct CpuStaticRegistrar {
1160 360 CpuStaticRegistrar() {
1161 using namespace ::brotensor;
1162 using namespace ::brotensor::detail;
1163
1164 180 OpsVTable ops{}; // zero-init — every slot nullptr by default
1165
1166 180 ops.linear_forward = &detail::cpu::linear_forward;
1167 180 ops.linear_backward = &detail::cpu::linear_backward;
1168 180 ops.relu_forward = &detail::cpu::relu_forward;
1169 180 ops.relu_backward = &detail::cpu::relu_backward;
1170 180 ops.tanh_forward = &detail::cpu::tanh_forward;
1171 180 ops.tanh_backward = &detail::cpu::tanh_backward;
1172 180 ops.sigmoid_forward = &detail::cpu::sigmoid_forward;
1173 180 ops.sigmoid_backward = &detail::cpu::sigmoid_backward;
1174 180 ops.softmax_forward = &detail::cpu::softmax_forward;
1175 180 ops.softmax_rows_forward = &detail::cpu::softmax_rows_forward;
1176 180 ops.softmax_backward = &detail::cpu::softmax_backward;
1177 180 ops.softmax_xent = &detail::cpu::softmax_xent;
1178 180 ops.softmax_xent_segment = &detail::cpu::softmax_xent_segment;
1179 180 ops.mse_scalar = &detail::cpu::mse_scalar;
1180 180 ops.add_inplace = &detail::cpu::add_inplace;
1181 180 ops.add_scalar_inplace = &detail::cpu::add_scalar_inplace;
1182 180 ops.add_channel_bias_inplace = &detail::cpu::add_channel_bias_inplace;
1183 180 ops.add_row_bias_inplace = &detail::cpu::add_row_bias_inplace;
1184 180 ops.xavier_init = &detail::cpu::xavier_init;
1185
1186 180 ops.sgd_step = &detail::cpu::sgd_step;
1187 180 ops.adam_step = &detail::cpu::adam_step;
1188 180 ops.scale_inplace = &detail::cpu::scale_inplace;
1189 180 ops.layernorm_forward = &detail::cpu::layernorm_forward;
1190 180 ops.layernorm_backward = &detail::cpu::layernorm_backward;
1191 180 ops.attention_forward = &detail::cpu::attention_forward;
1192 180 ops.attention_backward = &detail::cpu::attention_backward;
1193 180 ops.mha_forward = &detail::cpu::mha_forward;
1194 180 ops.mha_backward = &detail::cpu::mha_backward;
1195 180 ops.concat_rows = &detail::cpu::concat_rows;
1196 180 ops.split_rows = &detail::cpu::split_rows;
1197 180 ops.add_inplace_batched = &detail::cpu::add_inplace_batched;
1198 180 ops.linear_forward_batched = &detail::cpu::linear_forward_batched;
1199 180 ops.linear_backward_batched = &detail::cpu::linear_backward_batched;
1200 180 ops.relu_forward_batched = &detail::cpu::relu_forward_batched;
1201 180 ops.relu_backward_batched = &detail::cpu::relu_backward_batched;
1202 180 ops.tanh_forward_batched = &detail::cpu::tanh_forward_batched;
1203 180 ops.tanh_backward_batched = &detail::cpu::tanh_backward_batched;
1204 180 ops.mse_vec_per_sample = &detail::cpu::mse_vec_per_sample;
1205 180 ops.softmax_xent_fused_batched = &detail::cpu::softmax_xent_fused_batched;
1206 180 ops.bce_with_logits_fused_batched = &detail::cpu::bce_with_logits_fused_batched;
1207 180 ops.masked_mean_pool_forward = &detail::cpu::masked_mean_pool_forward;
1208 180 ops.masked_mean_pool_backward = &detail::cpu::masked_mean_pool_backward;
1209 180 ops.build_slot_mask = &detail::cpu::build_slot_mask;
1210 180 ops.copy_d2d = &detail::cpu::copy_d2d;
1211 180 ops.copy_d2d_strided = &detail::cpu::copy_d2d_strided;
1212
1213 // ── CHUNK 1 ──
1214 180 ops.clamp = &detail::cpu::clamp;
1215 180 ops.mul_inplace = &detail::cpu::mul_inplace;
1216 180 ops.div_inplace = &detail::cpu::div_inplace;
1217 180 ops.axpby_inplace = &detail::cpu::axpby_inplace;
1218 180 ops.threshold_u8 = &detail::cpu::threshold_u8;
1219 180 ops.cast = &detail::cpu::cast;
1220 180 ops.mse_vec_forward = &detail::cpu::mse_vec_forward;
1221 180 ops.mse_vec_backward = &detail::cpu::mse_vec_backward;
1222 180 ops.softmax_xent_fused = &detail::cpu::softmax_xent_fused;
1223 180 ops.embedding_lookup_forward = &detail::cpu::embedding_lookup_forward;
1224 180 ops.embedding_lookup_backward = &detail::cpu::embedding_lookup_backward;
1225 180 ops.concat_batched_rows = &detail::cpu::concat_batched_rows;
1226 180 ops.concat_nchw_channels = &detail::cpu::concat_nchw_channels;
1227 180 ops.concat_nchw_channels_backward
1228 180 = &detail::cpu::concat_nchw_channels_backward;
1229 180 ops.sum_rows = &detail::cpu::sum_rows;
1230 180 ops.sum_cols = &detail::cpu::sum_cols;
1231 180 ops.argmax_rows = &detail::cpu::argmax_rows;
1232 180 ops.rows_count_above = &detail::cpu::rows_count_above;
1233 180 ops.layernorm_forward_inference_batched
1234 180 = &detail::cpu::layernorm_forward_inference_batched;
1235 180 ops.layernorm_forward_inference_batched_fp16
1236 180 = &detail::cpu::layernorm_forward_inference_batched_fp16;
1237 180 ops.layernorm_forward_batched_with_caches
1238 180 = &detail::cpu::layernorm_forward_batched_with_caches;
1239 180 ops.layernorm_backward_batched_with_caches
1240 180 = &detail::cpu::layernorm_backward_batched_with_caches;
1241 180 ops.build_causal_mask_row = &detail::cpu::build_causal_mask_row;
1242
1243 // ── CHUNK 2 ──
1244 180 ops.silu_forward = &detail::cpu::silu_forward;
1245 180 ops.silu_backward = &detail::cpu::silu_backward;
1246 180 ops.gelu_forward = &detail::cpu::gelu_forward;
1247 180 ops.gelu_backward = &detail::cpu::gelu_backward;
1248 180 ops.gelu_exact_forward = &detail::cpu::gelu_exact_forward;
1249 180 ops.gelu_exact_backward = &detail::cpu::gelu_exact_backward;
1250 180 ops.quick_gelu_forward = &detail::cpu::quick_gelu_forward;
1251 180 ops.quick_gelu_backward = &detail::cpu::quick_gelu_backward;
1252 180 ops.geglu_forward = &detail::cpu::geglu_forward;
1253 180 ops.geglu_backward = &detail::cpu::geglu_backward;
1254 180 ops.geglu_exact_forward = &detail::cpu::geglu_exact_forward;
1255 180 ops.geglu_exact_backward = &detail::cpu::geglu_exact_backward;
1256 180 ops.swiglu_forward = &detail::cpu::swiglu_forward;
1257 180 ops.swiglu_backward = &detail::cpu::swiglu_backward;
1258 180 ops.matmul = &detail::cpu::matmul;
1259 180 ops.matmul_abt = &detail::cpu::matmul_abt;
1260 180 ops.matmul_backward = &detail::cpu::matmul_backward;
1261 180 ops.lstm_forward_train = &detail::cpu::lstm_forward_train;
1262 180 ops.lstm_backward = &detail::cpu::lstm_backward;
1263 180 ops.rope_forward = &detail::cpu::rope_forward;
1264 180 ops.rope_backward = &detail::cpu::rope_backward;
1265 180 ops.rms_norm_forward = &detail::cpu::rms_norm_forward;
1266 180 ops.rms_norm_backward = &detail::cpu::rms_norm_backward;
1267
1268 // ── CHUNK 3 ──
1269 180 ops.conv2d_forward = &detail::cpu::conv2d_forward;
1270 180 ops.conv2d_backward_input = &detail::cpu::conv2d_backward_input;
1271 180 ops.conv2d_backward_weight = &detail::cpu::conv2d_backward_weight;
1272 180 ops.conv2d_backward_bias = &detail::cpu::conv2d_backward_bias;
1273 180 ops.deform_conv2d_forward = &detail::cpu::deform_conv2d_forward;
1274 180 ops.conv3d_forward = &detail::cpu::conv3d_forward;
1275 // conv3d_int8w_fp16_forward is GPU-only; CPU slot left null.
1276 180 ops.group_norm_forward = &detail::cpu::group_norm_forward;
1277 180 ops.group_norm_backward = &detail::cpu::group_norm_backward;
1278
1279 // ── CHUNK 4 ──
1280 180 ops.upsample_nearest_2x = &detail::cpu::upsample_nearest_2x;
1281 180 ops.upsample_bilinear_2x = &detail::cpu::upsample_bilinear_2x;
1282 180 ops.downsample_avg_2x = &detail::cpu::downsample_avg_2x;
1283 180 ops.upsample_nearest_2x_backward = &detail::cpu::upsample_nearest_2x_backward;
1284 180 ops.upsample_bilinear_2x_backward
1285 180 = &detail::cpu::upsample_bilinear_2x_backward;
1286 180 ops.downsample_avg_2x_backward = &detail::cpu::downsample_avg_2x_backward;
1287 180 ops.interp2d_forward = &detail::cpu::interp2d_forward;
1288 180 ops.interp2d_backward = &detail::cpu::interp2d_backward;
1289 180 ops.interp2d_align_corners_forward = &detail::cpu::interp2d_align_corners_forward;
1290 180 ops.pad2d_forward = &detail::cpu::pad2d_forward;
1291 180 ops.pad2d_backward = &detail::cpu::pad2d_backward;
1292 180 ops.slice2d_forward = &detail::cpu::slice2d_forward;
1293 180 ops.slice2d_backward = &detail::cpu::slice2d_backward;
1294 180 ops.unfold2d_forward = &detail::cpu::unfold2d_forward;
1295 180 ops.l2_normalize_nchw_forward = &detail::cpu::l2_normalize_nchw_forward;
1296 180 ops.convex_upsample_forward = &detail::cpu::convex_upsample_forward;
1297 180 ops.top_k_rows = &detail::cpu::top_k_rows;
1298 180 ops.adaptive_avg_pool2d_forward = &detail::cpu::adaptive_avg_pool2d_forward;
1299 180 ops.adaptive_avg_pool2d_backward = &detail::cpu::adaptive_avg_pool2d_backward;
1300 180 ops.max_pool2d_forward = &detail::cpu::max_pool2d_forward;
1301 180 ops.max_pool2d_backward = &detail::cpu::max_pool2d_backward;
1302 180 ops.gather_rows = &detail::cpu::gather_rows;
1303 180 ops.scatter_rows_add = &detail::cpu::scatter_rows_add;
1304 180 ops.scatter_rows = &detail::cpu::scatter_rows;
1305 180 ops.conv_transpose2d_forward = &detail::cpu::conv_transpose2d_forward;
1306 180 ops.conv_transpose2d_backward_input
1307 180 = &detail::cpu::conv_transpose2d_backward_input;
1308 180 ops.conv_transpose2d_backward_weight
1309 180 = &detail::cpu::conv_transpose2d_backward_weight;
1310 180 ops.conv_transpose2d_backward_bias
1311 180 = &detail::cpu::conv_transpose2d_backward_bias;
1312 180 ops.window_partition_forward = &detail::cpu::window_partition_forward;
1313 180 ops.window_reverse_forward = &detail::cpu::window_reverse_forward;
1314 180 ops.nchw_to_sequence = &detail::cpu::nchw_to_sequence;
1315 180 ops.sequence_to_nchw = &detail::cpu::sequence_to_nchw;
1316 180 ops.ddim_step = &detail::cpu::ddim_step;
1317 180 ops.euler_step = &detail::cpu::euler_step;
1318 180 ops.dpmpp_2m_step = &detail::cpu::dpmpp_2m_step;
1319 180 ops.timestep_embedding = &detail::cpu::timestep_embedding;
1320 180 ops.kv_cache_append = &detail::cpu::kv_cache_append;
1321 180 ops.flash_attention_decode = &detail::cpu::flash_attention_decode;
1322 180 ops.flash_attention_decode_masked
1323 180 = &detail::cpu::flash_attention_decode_masked;
1324 180 ops.flash_attention_decode_len = &detail::cpu::flash_attention_decode_len;
1325
1326 // ── CHUNK 5 ──
1327 180 ops.cross_attention_forward = &detail::cpu::cross_attention_forward;
1328 180 ops.cross_attention_forward_with_attn
1329 180 = &detail::cpu::cross_attention_forward_with_attn;
1330 180 ops.self_attention_forward_train = &detail::cpu::self_attention_forward_train;
1331 180 ops.self_attention_backward = &detail::cpu::self_attention_backward;
1332 180 ops.attention_token_moments = &detail::cpu::attention_token_moments;
1333 180 ops.cross_attention_forward_train
1334 180 = &detail::cpu::cross_attention_forward_train;
1335 180 ops.cross_attention_backward = &detail::cpu::cross_attention_backward;
1336
1337 // ── CHUNK 6 ──
1338 180 ops.flash_attention_forward = &detail::cpu::flash_attention_forward;
1339 180 ops.flash_attention_gqa_forward = &detail::cpu::flash_attention_gqa_forward;
1340 180 ops.flash_attention_windowed_forward
1341 180 = &detail::cpu::flash_attention_windowed_forward;
1342 180 ops.flash_attention_varlen_forward
1343 180 = &detail::cpu::flash_attention_varlen_forward;
1344 180 ops.flash_attention_varlen_backward
1345 180 = &detail::cpu::flash_attention_varlen_backward;
1346 180 ops.flash_attention_packed_qkv_forward
1347 180 = &detail::cpu::flash_attention_packed_qkv_forward;
1348 180 ops.flash_attention_packed_qkv_backward
1349 180 = &detail::cpu::flash_attention_packed_qkv_backward;
1350 180 ops.rope_qkv_packed_inplace = &detail::cpu::rope_qkv_packed_inplace;
1351 180 ops.segment_softmax_stats = &detail::cpu::segment_softmax_stats;
1352 180 ops.linear_forward_batched_ex = &detail::cpu::linear_forward_batched_ex;
1353 180 ops.quantize_rows = &detail::cpu::quantize_rows;
1354 180 ops.dequantize_rows = &detail::cpu::dequantize_rows;
1355 180 ops.linear_forward_batched_wq_ex = &detail::cpu::linear_forward_batched_wq_ex;
1356 180 ops.flash_attention_qkvo_forward = &detail::cpu::flash_attention_qkvo_forward;
1357 180 ops.flash_attention_qkvo_backward
1358 180 = &detail::cpu::flash_attention_qkvo_backward;
1359 180 ops.flash_attention_backward = &detail::cpu::flash_attention_backward;
1360 180 ops.flash_attention_project_kv = &detail::cpu::flash_attention_project_kv;
1361 180 ops.flash_attention_q_with_kv_cached_forward
1362 180 = &detail::cpu::flash_attention_q_with_kv_cached_forward;
1363 180 ops.self_attention_forward = &detail::cpu::self_attention_forward;
1364 180 ops.resblock_forward = &detail::cpu::resblock_forward;
1365 180 ops.resblock_backward = &detail::cpu::resblock_backward;
1366
1367 // ── DiT / diffusion extras ──
1368 180 ops.modulate = &detail::cpu::modulate;
1369 180 ops.broadcast_mul = &detail::cpu::broadcast_mul;
1370 180 ops.rope_apply = &detail::cpu::rope_apply;
1371 180 ops.rope_apply_perhead = &detail::cpu::rope_apply_perhead;
1372 180 ops.rope_apply_backward = &detail::cpu::rope_apply_backward;
1373 180 ops.self_attention_bias_forward = &detail::cpu::self_attention_bias_forward;
1374 180 ops.rel_pos_bias_xl_forward = &detail::cpu::rel_pos_bias_xl_forward;
1375 180 ops.self_attention_decomposed_rel_pos_forward =
1376 &detail::cpu::self_attention_decomposed_rel_pos_forward;
1377 180 ops.self_attention_decomposed_rel_pos_windowed_forward =
1378 &detail::cpu::self_attention_decomposed_rel_pos_windowed_forward;
1379
1380 // ── Spectral / FFT core (brosoundml) ──
1381 180 ops.complex_mul = &detail::cpu::complex_mul;
1382 180 ops.complex_mul_backward = &detail::cpu::complex_mul_backward;
1383 180 ops.complex_abs = &detail::cpu::complex_abs;
1384 180 ops.complex_abs_backward = &detail::cpu::complex_abs_backward;
1385 180 ops.complex_angle = &detail::cpu::complex_angle;
1386 180 ops.complex_from_polar = &detail::cpu::complex_from_polar;
1387 180 ops.fft = &detail::cpu::fft;
1388 180 ops.ifft = &detail::cpu::ifft;
1389 180 ops.rfft = &detail::cpu::rfft;
1390 180 ops.irfft = &detail::cpu::irfft;
1391 180 ops.rfft_backward = &detail::cpu::rfft_backward;
1392 180 ops.irfft_backward = &detail::cpu::irfft_backward;
1393
1394 // ── STFT / iSTFT (brosoundml) ──
1395 180 ops.stft = &detail::cpu::stft;
1396 180 ops.stft_backward = &detail::cpu::stft_backward;
1397 180 ops.istft = &detail::cpu::istft;
1398 180 ops.istft_backward = &detail::cpu::istft_backward;
1399
1400 // ── 1D convolution family (brosoundml) ──
1401 180 ops.conv_transpose1d_forward = &detail::cpu::conv_transpose1d_forward;
1402 180 ops.conv_transpose1d_backward_input
1403 180 = &detail::cpu::conv_transpose1d_backward_input;
1404 180 ops.conv_transpose1d_backward_weight
1405 180 = &detail::cpu::conv_transpose1d_backward_weight;
1406 180 ops.conv_transpose1d_backward_bias
1407 180 = &detail::cpu::conv_transpose1d_backward_bias;
1408 180 ops.causal_conv1d_update = &detail::cpu::causal_conv1d_update;
1409 180 ops.pad1d_forward = &detail::cpu::pad1d_forward;
1410 180 ops.pad1d_backward = &detail::cpu::pad1d_backward;
1411
1412 // ── Vocoder / codec activations (brosoundml) ──
1413 180 ops.snake_forward = &detail::cpu::snake_forward;
1414 180 ops.snake_backward = &detail::cpu::snake_backward;
1415 180 ops.elu_forward = &detail::cpu::elu_forward;
1416 180 ops.elu_backward = &detail::cpu::elu_backward;
1417 180 ops.leaky_relu_forward = &detail::cpu::leaky_relu_forward;
1418 180 ops.leaky_relu_backward = &detail::cpu::leaky_relu_backward;
1419
1420 // ── Codec quantization (brosoundml) ──
1421 180 ops.vq_encode_forward = &detail::cpu::vq_encode_forward;
1422 180 ops.vq_encode_backward = &detail::cpu::vq_encode_backward;
1423 180 ops.fsq_quantize_forward = &detail::cpu::fsq_quantize_forward;
1424 180 ops.fsq_quantize_backward = &detail::cpu::fsq_quantize_backward;
1425
1426 // ── 1D resampling (brosoundml CHUNK 6, family E) ──
1427 180 ops.resample1d_forward = &detail::cpu::resample1d_forward;
1428 180 ops.resample1d_backward = &detail::cpu::resample1d_backward;
1429
1430 // ── log / exp / round elementwise (brosoundml CHUNK 6, family G) ──
1431 180 ops.log_forward = &detail::cpu::log_forward;
1432 180 ops.log_backward = &detail::cpu::log_backward;
1433 180 ops.exp_forward = &detail::cpu::exp_forward;
1434 180 ops.exp_backward = &detail::cpu::exp_backward;
1435 180 ops.round_forward = &detail::cpu::round_forward;
1436 180 ops.round_backward = &detail::cpu::round_backward;
1437
1438 // ── Autoregressive logit sampling (brosoundml CHUNK 7, family F) ──
1439 180 ops.sample_logits = &detail::cpu::sample_logits;
1440 180 ops.sample_logits_into = &detail::cpu::sample_logits_into;
1441
1442 // ── Masked-diffusion token selection ──
1443 180 ops.masked_diffusion_scores = &detail::cpu::masked_diffusion_scores;
1444 180 ops.masked_diffusion_commit = &detail::cpu::masked_diffusion_commit;
1445
1446 // ── Counter-based noise generation (Philox 4x32-10) ──
1447 180 ops.randn = &detail::cpu::randn;
1448 180 ops.rand_uniform = &detail::cpu::rand_uniform;
1449 180 ops.rand_bernoulli = &detail::cpu::rand_bernoulli;
1450 180 ops.randn_truncated = &detail::cpu::randn_truncated;
1451
1452 // ── L2 norm + Gated Delta Rule (linear-attention text path) ──
1453 180 ops.l2_norm_forward = &detail::cpu::l2_norm_forward;
1454 180 ops.l2_norm_backward = &detail::cpu::l2_norm_backward;
1455 180 ops.gated_delta_rule_chunked = &detail::cpu::gated_delta_rule_chunked;
1456 180 ops.gated_delta_rule_step = &detail::cpu::gated_delta_rule_step;
1457
1458 // ── Qwen3-VL polish: spatial_merge_2x2 + M-RoPE ──
1459 180 ops.spatial_merge_2x2_forward = &detail::cpu::spatial_merge_2x2_forward;
1460 180 ops.pixel_shuffle_upsample_2x_forward = &detail::cpu::pixel_shuffle_upsample_2x_forward;
1461 180 ops.patch_unpack_forward = &detail::cpu::patch_unpack_forward;
1462 180 ops.rope_apply_mrope = &detail::cpu::rope_apply_mrope;
1463
1464 // ── BatchNorm (vision backbones) ──
1465 180 ops.batch_norm_forward = &detail::cpu::batch_norm_forward;
1466 180 ops.batch_norm_inference = &detail::cpu::batch_norm_inference;
1467 180 ops.batch_norm_backward = &detail::cpu::batch_norm_backward;
1468
1469 // ── Image preprocessing helpers ──
1470 180 ops.image_normalize = &detail::cpu::image_normalize;
1471 180 ops.image_u8_to_f32_nhwc_to_nchw = &detail::cpu::image_u8_to_f32_nhwc_to_nchw;
1472
1473 // ── StyleGAN3-R synthesis-input primitives ──
1474 180 ops.sin_forward = &detail::cpu::sin_forward;
1475 180 ops.sin_backward = &detail::cpu::sin_backward;
1476 180 ops.cos_forward = &detail::cpu::cos_forward;
1477 180 ops.cos_backward = &detail::cpu::cos_backward;
1478 180 ops.rsqrt_forward = &detail::cpu::rsqrt_forward;
1479 180 ops.rsqrt_backward = &detail::cpu::rsqrt_backward;
1480 180 ops.pixel_norm_forward = &detail::cpu::pixel_norm_forward;
1481 180 ops.pixel_norm_backward = &detail::cpu::pixel_norm_backward;
1482 180 ops.bias_act_forward = &detail::cpu::bias_act_forward;
1483 180 ops.bias_act_backward = &detail::cpu::bias_act_backward;
1484 180 ops.upfirdn2d_forward = &detail::cpu::upfirdn2d_forward;
1485 180 ops.upfirdn2d_backward = &detail::cpu::upfirdn2d_backward;
1486 180 ops.modulated_conv2d_forward = &detail::cpu::modulated_conv2d_forward;
1487 180 ops.modulated_conv2d_backward = &detail::cpu::modulated_conv2d_backward;
1488
1489 // ── Quantized linear forward & dequant (Q8_0 & Q4_K) via Brass AVX2/FMA JIT ──
1490 180 ops.dequant_q8_0_to_fp16 = &detail::cpu::dequant_q8_0_to_fp16;
1491 180 ops.linear_forward_q8_0_fp16 = &detail::cpu::linear_forward_q8_0_fp16;
1492 180 ops.linear_forward_batched_q8_0_fp16 = &detail::cpu::linear_forward_batched_q8_0_fp16;
1493 180 ops.dequant_q4k_to_fp16 = &detail::cpu::dequant_q4k_to_fp16;
1494 180 ops.linear_forward_q4k_fp16 = &detail::cpu::linear_forward_q4k_fp16;
1495 180 ops.linear_forward_batched_q4k_fp16 = &detail::cpu::linear_forward_batched_q4k_fp16;
1496
1497 360 detail::register_backend(Device::CPU, ops,
1498 180 detail::cpu::cpu_alloc_table());
1499 360 }
1500 };
1501
1502 180 static CpuStaticRegistrar g_cpu_registrar{};
1503
1504 } // anonymous namespace
1505