src/cpu/register.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // ─── CPU backend registration ────────────────────────────────────────────── | ||
| 2 | // | ||
| 3 | // Builds the CPU OpsVTable (only the slots we implement; everything else | ||
| 4 | // stays nullptr — the dispatcher throws "not implemented on this backend" | ||
| 5 | // on null lookups), pairs it with the CPU AllocVTable from alloc.cpp, and | ||
| 6 | // hands them to the registry at static-init time. CPU is therefore always | ||
| 7 | // available without a prior brotensor::init() call. | ||
| 8 | |||
| 9 | #include <brotensor/detail/dispatch.h> | ||
| 10 | #include <brotensor/tensor.h> | ||
| 11 | |||
| 12 | #include <cstdint> | ||
| 13 | #include <vector> | ||
| 14 | |||
| 15 | namespace brotensor::detail::cpu { | ||
| 16 | |||
| 17 | // ── alloc.cpp ── | ||
| 18 | const AllocVTable& cpu_alloc_table(); | ||
| 19 | |||
| 20 | // ── ops_impl.cpp — forward decls of the 16 implemented ops ── | ||
| 21 | void linear_forward(const ::brotensor::Tensor& W, const ::brotensor::Tensor& b, | ||
| 22 | const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 23 | void linear_backward(const ::brotensor::Tensor& W, const ::brotensor::Tensor& x, | ||
| 24 | const ::brotensor::Tensor& dY, | ||
| 25 | ::brotensor::Tensor& dX, ::brotensor::Tensor& dW, | ||
| 26 | ::brotensor::Tensor& dB); | ||
| 27 | void relu_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 28 | void relu_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY, | ||
| 29 | ::brotensor::Tensor& dX); | ||
| 30 | void tanh_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 31 | void tanh_backward(const ::brotensor::Tensor& y, const ::brotensor::Tensor& dY, | ||
| 32 | ::brotensor::Tensor& dX); | ||
| 33 | void sigmoid_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 34 | void sigmoid_backward(const ::brotensor::Tensor& y, const ::brotensor::Tensor& dY, | ||
| 35 | ::brotensor::Tensor& dX); | ||
| 36 | void softmax_forward(const ::brotensor::Tensor& logits, ::brotensor::Tensor& probs, | ||
| 37 | const float* mask); | ||
| 38 | void softmax_rows_forward(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y, | ||
| 39 | int rows, int cols); | ||
| 40 | void softmax_backward(const ::brotensor::Tensor& probs, | ||
| 41 | const ::brotensor::Tensor& dProbs, | ||
| 42 | ::brotensor::Tensor& dLogits); | ||
| 43 | float softmax_xent_segment(const float* lp, const float* tp, | ||
| 44 | float* pp, float* dz, | ||
| 45 | int n, const float* mask); | ||
| 46 | float softmax_xent(const ::brotensor::Tensor& logits, | ||
| 47 | const ::brotensor::Tensor& target, | ||
| 48 | ::brotensor::Tensor& probs, ::brotensor::Tensor& dLogits, | ||
| 49 | const float* mask); | ||
| 50 | float mse_scalar(float pred, float target, float& dPred); | ||
| 51 | void add_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& x); | ||
| 52 | void add_scalar_inplace(::brotensor::Tensor& y, float s); | ||
| 53 | void add_channel_bias_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& bias, int C, int L); | ||
| 54 | void xavier_init(::brotensor::Tensor& W, uint64_t& rng_state); | ||
| 55 | |||
| 56 | // ── ops_impl.cpp — forward decls of the 20 newly implemented ops ── | ||
| 57 | void sgd_step(::brotensor::Tensor& param, ::brotensor::Tensor& grad, | ||
| 58 | ::brotensor::Tensor& velocity, float lr, float momentum); | ||
| 59 | void adam_step(::brotensor::Tensor& param, const ::brotensor::Tensor& grad, | ||
| 60 | ::brotensor::Tensor& m, ::brotensor::Tensor& v, | ||
| 61 | float lr, float beta1, float beta2, float eps, int step); | ||
| 62 | void scale_inplace(::brotensor::Tensor& y, float s); | ||
| 63 | void layernorm_forward(const ::brotensor::Tensor& x, | ||
| 64 | const ::brotensor::Tensor& gamma, | ||
| 65 | const ::brotensor::Tensor& beta, | ||
| 66 | ::brotensor::Tensor& y, ::brotensor::Tensor& xhat, | ||
| 67 | float& mean_out, float& rstd_out, float eps); | ||
| 68 | void layernorm_backward(const ::brotensor::Tensor& dY, | ||
| 69 | const ::brotensor::Tensor& xhat, | ||
| 70 | const ::brotensor::Tensor& gamma, float rstd, | ||
| 71 | ::brotensor::Tensor& dX, | ||
| 72 | ::brotensor::Tensor& dGamma, ::brotensor::Tensor& dBeta); | ||
| 73 | void attention_forward(const ::brotensor::Tensor& X, | ||
| 74 | const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& Wk, | ||
| 75 | const ::brotensor::Tensor& Wv, const ::brotensor::Tensor& Wo, | ||
| 76 | const float* d_mask, | ||
| 77 | ::brotensor::Tensor& Q, ::brotensor::Tensor& K, | ||
| 78 | ::brotensor::Tensor& V, ::brotensor::Tensor& Attn, | ||
| 79 | ::brotensor::Tensor& Y_pre_Wo, ::brotensor::Tensor& O); | ||
| 80 | void attention_backward(const ::brotensor::Tensor& dO, | ||
| 81 | const ::brotensor::Tensor& X, | ||
| 82 | const ::brotensor::Tensor& Q, const ::brotensor::Tensor& K, | ||
| 83 | const ::brotensor::Tensor& V, const ::brotensor::Tensor& Attn, | ||
| 84 | const ::brotensor::Tensor& Y_pre_Wo, | ||
| 85 | const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& Wk, | ||
| 86 | const ::brotensor::Tensor& Wv, const ::brotensor::Tensor& Wo, | ||
| 87 | const float* d_mask, | ||
| 88 | ::brotensor::Tensor& dX, | ||
| 89 | ::brotensor::Tensor& dWq, ::brotensor::Tensor& dWk, | ||
| 90 | ::brotensor::Tensor& dWv, ::brotensor::Tensor& dWo); | ||
| 91 | void mha_forward(const ::brotensor::Tensor& X, | ||
| 92 | const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& Wk, | ||
| 93 | const ::brotensor::Tensor& Wv, const ::brotensor::Tensor& Wo, | ||
| 94 | const ::brotensor::Tensor* bq, const ::brotensor::Tensor* bk, | ||
| 95 | const ::brotensor::Tensor* bv, const ::brotensor::Tensor* bo, | ||
| 96 | const float* d_mask, int num_heads, | ||
| 97 | ::brotensor::Tensor& Qh, ::brotensor::Tensor& Kh, | ||
| 98 | ::brotensor::Tensor& Vh, ::brotensor::Tensor& Attnh, | ||
| 99 | ::brotensor::Tensor& Yconcat, ::brotensor::Tensor& O); | ||
| 100 | void mha_backward(const ::brotensor::Tensor& dO, | ||
| 101 | const ::brotensor::Tensor& X, | ||
| 102 | const ::brotensor::Tensor& Qh, const ::brotensor::Tensor& Kh, | ||
| 103 | const ::brotensor::Tensor& Vh, const ::brotensor::Tensor& Attnh, | ||
| 104 | const ::brotensor::Tensor& Yconcat, | ||
| 105 | const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& Wk, | ||
| 106 | const ::brotensor::Tensor& Wv, const ::brotensor::Tensor& Wo, | ||
| 107 | const float* d_mask, int num_heads, | ||
| 108 | ::brotensor::Tensor& dX, | ||
| 109 | ::brotensor::Tensor& dWq, ::brotensor::Tensor& dWk, | ||
| 110 | ::brotensor::Tensor& dWv, ::brotensor::Tensor& dWo, | ||
| 111 | ::brotensor::Tensor* dbq, ::brotensor::Tensor* dbk, | ||
| 112 | ::brotensor::Tensor* dbv, ::brotensor::Tensor* dbo); | ||
| 113 | void concat_rows(const std::vector<const ::brotensor::Tensor*>& parts, | ||
| 114 | ::brotensor::Tensor& out); | ||
| 115 | void split_rows(const ::brotensor::Tensor& in, | ||
| 116 | const std::vector<::brotensor::Tensor*>& parts); | ||
| 117 | void masked_mean_pool_forward(const ::brotensor::Tensor& X, const float* d_mask, | ||
| 118 | ::brotensor::Tensor& y); | ||
| 119 | void masked_mean_pool_backward(const ::brotensor::Tensor& dY, const float* d_mask, | ||
| 120 | int K, ::brotensor::Tensor& dX); | ||
| 121 | void build_slot_mask(const ::brotensor::Tensor& x, int offset, int K, int stride, | ||
| 122 | ::brotensor::Tensor& mask); | ||
| 123 | void copy_d2d(const ::brotensor::Tensor& src, int src_off, | ||
| 124 | ::brotensor::Tensor& dst, int dst_off, int n); | ||
| 125 | void copy_d2d_strided(const ::brotensor::Tensor& src, int src_off, int src_pitch, | ||
| 126 | ::brotensor::Tensor& dst, int dst_off, int dst_pitch, | ||
| 127 | int width, int height); | ||
| 128 | void add_inplace_batched(::brotensor::Tensor& Y_BD, | ||
| 129 | const ::brotensor::Tensor& X_BD); | ||
| 130 | void linear_forward_batched(const ::brotensor::Tensor& W, | ||
| 131 | const ::brotensor::Tensor& bias, | ||
| 132 | const ::brotensor::Tensor& X_BD, | ||
| 133 | ::brotensor::Tensor& Y_BD); | ||
| 134 | void linear_backward_batched(const ::brotensor::Tensor& W, | ||
| 135 | const ::brotensor::Tensor& X_BD, | ||
| 136 | const ::brotensor::Tensor& dY_BD, | ||
| 137 | ::brotensor::Tensor& dX_BD, | ||
| 138 | ::brotensor::Tensor& dW, | ||
| 139 | ::brotensor::Tensor& dB); | ||
| 140 | void relu_forward_batched(const ::brotensor::Tensor& X_BD, | ||
| 141 | ::brotensor::Tensor& Y_BD); | ||
| 142 | void relu_backward_batched(const ::brotensor::Tensor& X_BD, | ||
| 143 | const ::brotensor::Tensor& dY_BD, | ||
| 144 | ::brotensor::Tensor& dX_BD); | ||
| 145 | void tanh_forward_batched(const ::brotensor::Tensor& X_BD, | ||
| 146 | ::brotensor::Tensor& Y_BD); | ||
| 147 | void tanh_backward_batched(const ::brotensor::Tensor& Y_BD, | ||
| 148 | const ::brotensor::Tensor& dY_BD, | ||
| 149 | ::brotensor::Tensor& dX_BD); | ||
| 150 | void mse_vec_per_sample(const ::brotensor::Tensor& pred, | ||
| 151 | const ::brotensor::Tensor& target, | ||
| 152 | ::brotensor::Tensor& dPred, | ||
| 153 | ::brotensor::Tensor& loss_per_sample); | ||
| 154 | void softmax_xent_fused_batched(const ::brotensor::Tensor& logits_BL, | ||
| 155 | const ::brotensor::Tensor& target_BL, | ||
| 156 | const float* d_mask_BL, | ||
| 157 | const int* d_head_offsets, | ||
| 158 | int n_heads, | ||
| 159 | ::brotensor::Tensor& probs_BL, | ||
| 160 | ::brotensor::Tensor& dLogits_BL, | ||
| 161 | ::brotensor::Tensor& loss_per_sample); | ||
| 162 | void bce_with_logits_fused_batched(const ::brotensor::Tensor& logits_BL, | ||
| 163 | const ::brotensor::Tensor& target_BL, | ||
| 164 | const float* d_mask_BL, | ||
| 165 | float pos_weight, | ||
| 166 | ::brotensor::Tensor& probs_BL, | ||
| 167 | ::brotensor::Tensor& dLogits_BL, | ||
| 168 | ::brotensor::Tensor& loss_per_sample); | ||
| 169 | |||
| 170 | // ── CHUNK 1 — elementwise.cpp / loss.cpp / embedding.cpp / concat.cpp / | ||
| 171 | // public_reductions.cpp / layernorm_inference.cpp ── | ||
| 172 | void clamp(::brotensor::Tensor& y, float lo, float hi); | ||
| 173 | void mul_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& x); | ||
| 174 | void axpby_inplace(::brotensor::Tensor& y, const ::brotensor::Tensor& x, | ||
| 175 | float a, float b); | ||
| 176 | void threshold_u8(const ::brotensor::Tensor& X, float t, | ||
| 177 | ::brotensor::Tensor& Y); | ||
| 178 | void cast(const ::brotensor::Tensor& src, ::brotensor::Tensor& dst, | ||
| 179 | ::brotensor::Dtype out_dtype); | ||
| 180 | float mse_vec_forward(const ::brotensor::Tensor& pred, | ||
| 181 | const ::brotensor::Tensor& target); | ||
| 182 | void mse_vec_backward(const ::brotensor::Tensor& pred, | ||
| 183 | const ::brotensor::Tensor& target, | ||
| 184 | ::brotensor::Tensor& dPred); | ||
| 185 | float softmax_xent_fused(const ::brotensor::Tensor& logits, | ||
| 186 | const ::brotensor::Tensor& target, | ||
| 187 | const float* d_mask, | ||
| 188 | ::brotensor::Tensor& probs, | ||
| 189 | ::brotensor::Tensor& dLogits); | ||
| 190 | void embedding_lookup_forward(const ::brotensor::Tensor& table, | ||
| 191 | const int32_t* d_idx, int B, | ||
| 192 | ::brotensor::Tensor& out); | ||
| 193 | void embedding_lookup_backward(const ::brotensor::Tensor& dOut, | ||
| 194 | const int32_t* d_idx, int B, | ||
| 195 | ::brotensor::Tensor& dTable); | ||
| 196 | void concat_batched_rows(const std::vector<const ::brotensor::Tensor*>& parts, | ||
| 197 | ::brotensor::Tensor& out); | ||
| 198 | void concat_nchw_channels(const std::vector<const ::brotensor::Tensor*>& parts, | ||
| 199 | int N, int H, int W, | ||
| 200 | const std::vector<int>& C_per_part, | ||
| 201 | ::brotensor::Tensor& out); | ||
| 202 | void concat_nchw_channels_backward(const ::brotensor::Tensor& dY, | ||
| 203 | int N, int H, int W, | ||
| 204 | const std::vector<int>& C_per_part, | ||
| 205 | const std::vector<::brotensor::Tensor*>& parts); | ||
| 206 | void sum_rows(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y); | ||
| 207 | void sum_cols(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y); | ||
| 208 | void argmax_rows(const ::brotensor::Tensor& X, ::brotensor::Tensor& Idx); | ||
| 209 | void rows_count_above(const ::brotensor::Tensor& X, float t_lo, float t_hi, | ||
| 210 | ::brotensor::Tensor& counts); | ||
| 211 | void layernorm_forward_inference_batched(const ::brotensor::Tensor& X_RD, | ||
| 212 | const ::brotensor::Tensor& gamma, | ||
| 213 | const ::brotensor::Tensor& beta, | ||
| 214 | ::brotensor::Tensor& Y_RD, float eps); | ||
| 215 | void layernorm_forward_batched_with_caches(const ::brotensor::Tensor& X_RD, | ||
| 216 | const ::brotensor::Tensor& gamma, | ||
| 217 | const ::brotensor::Tensor& beta, | ||
| 218 | ::brotensor::Tensor& Y_RD, | ||
| 219 | ::brotensor::Tensor& Xhat_RD, | ||
| 220 | ::brotensor::Tensor& Mean_R, | ||
| 221 | ::brotensor::Tensor& Rstd_R, | ||
| 222 | float eps); | ||
| 223 | void layernorm_backward_batched_with_caches(const ::brotensor::Tensor& dY_RD, | ||
| 224 | const ::brotensor::Tensor& Xhat_RD, | ||
| 225 | const ::brotensor::Tensor& gamma, | ||
| 226 | const ::brotensor::Tensor& Rstd_R, | ||
| 227 | ::brotensor::Tensor& dX_RD, | ||
| 228 | ::brotensor::Tensor& dGamma, | ||
| 229 | ::brotensor::Tensor& dBeta); | ||
| 230 | void build_causal_mask_row(int L, int q, ::brotensor::Tensor& mask); | ||
| 231 | |||
| 232 | // ── CHUNK 2 — activations.cpp / geglu.cpp / swiglu.cpp / matmul.cpp / | ||
| 233 | // rope.cpp / rms_norm.cpp ── | ||
| 234 | void silu_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 235 | void silu_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY, | ||
| 236 | ::brotensor::Tensor& dX); | ||
| 237 | void gelu_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 238 | void gelu_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY, | ||
| 239 | ::brotensor::Tensor& dX); | ||
| 240 | void gelu_exact_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 241 | void gelu_exact_backward(const ::brotensor::Tensor& x, | ||
| 242 | const ::brotensor::Tensor& dY, | ||
| 243 | ::brotensor::Tensor& dX); | ||
| 244 | void quick_gelu_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 245 | void quick_gelu_backward(const ::brotensor::Tensor& x, | ||
| 246 | const ::brotensor::Tensor& dY, | ||
| 247 | ::brotensor::Tensor& dX); | ||
| 248 | void geglu_forward(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y); | ||
| 249 | void geglu_backward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& dY, | ||
| 250 | ::brotensor::Tensor& dX); | ||
| 251 | void geglu_exact_forward(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y); | ||
| 252 | void geglu_exact_backward(const ::brotensor::Tensor& X, | ||
| 253 | const ::brotensor::Tensor& dY, | ||
| 254 | ::brotensor::Tensor& dX); | ||
| 255 | void swiglu_forward(const ::brotensor::Tensor& X, ::brotensor::Tensor& Y); | ||
| 256 | void swiglu_backward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& dY, | ||
| 257 | ::brotensor::Tensor& dX); | ||
| 258 | void matmul(const ::brotensor::Tensor& A, const ::brotensor::Tensor& B, | ||
| 259 | ::brotensor::Tensor& C); | ||
| 260 | void matmul_abt(const ::brotensor::Tensor& A, const ::brotensor::Tensor& B, | ||
| 261 | ::brotensor::Tensor& C, | ||
| 262 | int batch, int M, int N, int K, | ||
| 263 | long long strideA, long long strideB, long long strideC, | ||
| 264 | const ::brotensor::Tensor* bias, int act); | ||
| 265 | void matmul_backward(const ::brotensor::Tensor& A, const ::brotensor::Tensor& B, | ||
| 266 | const ::brotensor::Tensor& dC, | ||
| 267 | ::brotensor::Tensor& dA, ::brotensor::Tensor& dB); | ||
| 268 | void rope_forward(const ::brotensor::Tensor& X, int head_dim, int num_heads, | ||
| 269 | int seq_offset, float theta_base, ::brotensor::Tensor& Y); | ||
| 270 | void rope_backward(const ::brotensor::Tensor& dY, int head_dim, int num_heads, | ||
| 271 | int seq_offset, float theta_base, ::brotensor::Tensor& dX); | ||
| 272 | void rms_norm_forward(const ::brotensor::Tensor& X, | ||
| 273 | const ::brotensor::Tensor& gamma, | ||
| 274 | float eps, ::brotensor::Tensor& Y); | ||
| 275 | void rms_norm_backward(const ::brotensor::Tensor& X, | ||
| 276 | const ::brotensor::Tensor& gamma, | ||
| 277 | const ::brotensor::Tensor& dY, float eps, | ||
| 278 | ::brotensor::Tensor& dX, ::brotensor::Tensor& dGamma); | ||
| 279 | |||
| 280 | // ── CHUNK 3 — conv2d.cpp / group_norm.cpp ── | ||
| 281 | void conv2d_forward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& Wt, | ||
| 282 | const ::brotensor::Tensor* bias, | ||
| 283 | int N, int C_in, int H, int W, int C_out, int kH, int kW, | ||
| 284 | int stride_h, int stride_w, int pad_h, int pad_w, | ||
| 285 | int dil_h, int dil_w, int groups, ::brotensor::Tensor& Y); | ||
| 286 | void conv2d_backward_input(const ::brotensor::Tensor& Wt, | ||
| 287 | const ::brotensor::Tensor& dY, | ||
| 288 | int N, int C_in, int H, int W, | ||
| 289 | int C_out, int kH, int kW, | ||
| 290 | int stride_h, int stride_w, int pad_h, int pad_w, | ||
| 291 | int dil_h, int dil_w, int groups, | ||
| 292 | ::brotensor::Tensor& dX); | ||
| 293 | void conv2d_backward_weight(const ::brotensor::Tensor& X, | ||
| 294 | const ::brotensor::Tensor& dY, | ||
| 295 | int N, int C_in, int H, int W, | ||
| 296 | int C_out, int kH, int kW, | ||
| 297 | int stride_h, int stride_w, int pad_h, int pad_w, | ||
| 298 | int dil_h, int dil_w, int groups, | ||
| 299 | ::brotensor::Tensor& dWt); | ||
| 300 | void conv2d_backward_bias(const ::brotensor::Tensor& dY, | ||
| 301 | int N, int C_out, int H_out, int W_out, | ||
| 302 | ::brotensor::Tensor& dB); | ||
| 303 | void deform_conv2d_forward(const ::brotensor::Tensor& X, | ||
| 304 | const ::brotensor::Tensor& offset, | ||
| 305 | const ::brotensor::Tensor* mask, | ||
| 306 | const ::brotensor::Tensor& Wt, | ||
| 307 | const ::brotensor::Tensor* bias, | ||
| 308 | int N, int C_in, int H, int W, int C_out, int kH, int kW, | ||
| 309 | int stride_h, int stride_w, int pad_h, int pad_w, | ||
| 310 | int dil_h, int dil_w, int groups, int deform_groups, | ||
| 311 | ::brotensor::Tensor& Y); | ||
| 312 | void conv3d_forward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& Wt, | ||
| 313 | const ::brotensor::Tensor* bias, | ||
| 314 | int N, int C_in, int T, int H, int W, | ||
| 315 | int C_out, int kT, int kH, int kW, | ||
| 316 | int stride_t, int stride_h, int stride_w, | ||
| 317 | int pad_t, int pad_h, int pad_w, | ||
| 318 | int dil_t, int dil_h, int dil_w, int groups, | ||
| 319 | ::brotensor::Tensor& Y); | ||
| 320 | void group_norm_forward(const ::brotensor::Tensor& X, | ||
| 321 | const ::brotensor::Tensor& gamma, | ||
| 322 | const ::brotensor::Tensor& beta, | ||
| 323 | int N, int C, int H, int W, int num_groups, | ||
| 324 | float eps, ::brotensor::Tensor& Y); | ||
| 325 | void group_norm_backward(const ::brotensor::Tensor& X, | ||
| 326 | const ::brotensor::Tensor& gamma, | ||
| 327 | const ::brotensor::Tensor& dY, | ||
| 328 | int N, int C, int H, int W, int num_groups, float eps, | ||
| 329 | ::brotensor::Tensor& dX, ::brotensor::Tensor& dGamma, | ||
| 330 | ::brotensor::Tensor& dBeta); | ||
| 331 | |||
| 332 | // ── CHUNK 4 — resample.cpp / transpose.cpp / diffusion_samplers.cpp / | ||
| 333 | // kv_cache.cpp ── | ||
| 334 | void upsample_nearest_2x(const ::brotensor::Tensor& X, | ||
| 335 | int N, int C, int H, int W, ::brotensor::Tensor& Y); | ||
| 336 | void upsample_bilinear_2x(const ::brotensor::Tensor& X, | ||
| 337 | int N, int C, int H, int W, ::brotensor::Tensor& Y); | ||
| 338 | void downsample_avg_2x(const ::brotensor::Tensor& X, | ||
| 339 | int N, int C, int H, int W, ::brotensor::Tensor& Y); | ||
| 340 | void upsample_nearest_2x_backward(const ::brotensor::Tensor& dY, | ||
| 341 | int N, int C, int H, int W, | ||
| 342 | ::brotensor::Tensor& dX); | ||
| 343 | void upsample_bilinear_2x_backward(const ::brotensor::Tensor& dY, | ||
| 344 | int N, int C, int H, int W, | ||
| 345 | ::brotensor::Tensor& dX); | ||
| 346 | void downsample_avg_2x_backward(const ::brotensor::Tensor& dY, | ||
| 347 | int N, int C, int H, int W, | ||
| 348 | ::brotensor::Tensor& dX); | ||
| 349 | void interp2d_forward(const ::brotensor::Tensor& X, | ||
| 350 | int N, int C, int H_in, int W_in, | ||
| 351 | int H_out, int W_out, int mode, | ||
| 352 | ::brotensor::Tensor& Y); | ||
| 353 | void interp2d_backward(const ::brotensor::Tensor& dY, | ||
| 354 | int N, int C, int H_in, int W_in, | ||
| 355 | int H_out, int W_out, int mode, | ||
| 356 | ::brotensor::Tensor& dX); | ||
| 357 | void interp2d_align_corners_forward(const ::brotensor::Tensor& X, | ||
| 358 | int N, int C, int H_in, int W_in, | ||
| 359 | int H_out, int W_out, int mode, | ||
| 360 | ::brotensor::Tensor& Y); | ||
| 361 | void pad2d_forward(const ::brotensor::Tensor& X, | ||
| 362 | int N, int C, int H, int W, | ||
| 363 | int pad_top, int pad_bottom, int pad_left, int pad_right, | ||
| 364 | int mode, ::brotensor::Tensor& Y); | ||
| 365 | void pad2d_backward(const ::brotensor::Tensor& dY, | ||
| 366 | int N, int C, int H, int W, | ||
| 367 | int pad_top, int pad_bottom, int pad_left, int pad_right, | ||
| 368 | int mode, ::brotensor::Tensor& dX); | ||
| 369 | void unfold2d_forward(const ::brotensor::Tensor& X, | ||
| 370 | int N, int C, int H, int W, int kH, int kW, | ||
| 371 | int stride_h, int stride_w, | ||
| 372 | int pad_top, int pad_bottom, int pad_left, int pad_right, | ||
| 373 | int mode, ::brotensor::Tensor& Y); | ||
| 374 | void l2_normalize_nchw_forward(const ::brotensor::Tensor& X, | ||
| 375 | int N, int C, int H, int W, float eps, | ||
| 376 | ::brotensor::Tensor& Y); | ||
| 377 | void convex_upsample_forward(const ::brotensor::Tensor& X, | ||
| 378 | const ::brotensor::Tensor& Mask, | ||
| 379 | int N, int C, int H, int W, int scale, | ||
| 380 | ::brotensor::Tensor& Y); | ||
| 381 | void slice2d_forward(const ::brotensor::Tensor& X, | ||
| 382 | int N, int C, int H, int W, | ||
| 383 | int h0, int w0, int H_out, int W_out, | ||
| 384 | ::brotensor::Tensor& Y); | ||
| 385 | void slice2d_backward(const ::brotensor::Tensor& dY, | ||
| 386 | int N, int C, int H, int W, | ||
| 387 | int h0, int w0, int H_out, int W_out, | ||
| 388 | ::brotensor::Tensor& dX); | ||
| 389 | void top_k_rows(const ::brotensor::Tensor& X, int k, | ||
| 390 | ::brotensor::Tensor& Vals, ::brotensor::Tensor& Idx); | ||
| 391 | void adaptive_avg_pool2d_forward(const ::brotensor::Tensor& X, | ||
| 392 | int N, int C, int H, int W, | ||
| 393 | int H_out, int W_out, | ||
| 394 | ::brotensor::Tensor& Y); | ||
| 395 | void adaptive_avg_pool2d_backward(const ::brotensor::Tensor& dY, | ||
| 396 | int N, int C, int H, int W, | ||
| 397 | int H_out, int W_out, | ||
| 398 | ::brotensor::Tensor& dX); | ||
| 399 | void max_pool2d_forward(const ::brotensor::Tensor& X, | ||
| 400 | int N, int C, int H, int W, | ||
| 401 | int kH, int kW, int stride_h, int stride_w, | ||
| 402 | int pad_h, int pad_w, | ||
| 403 | ::brotensor::Tensor& Y, ::brotensor::Tensor& Idx); | ||
| 404 | void max_pool2d_backward(const ::brotensor::Tensor& dY, | ||
| 405 | const ::brotensor::Tensor& Idx, | ||
| 406 | int N, int C, int H, int W, int H_out, int W_out, | ||
| 407 | ::brotensor::Tensor& dX); | ||
| 408 | void gather_rows(const ::brotensor::Tensor& X, | ||
| 409 | const ::brotensor::Tensor& Idx, | ||
| 410 | ::brotensor::Tensor& Y); | ||
| 411 | void scatter_rows_add(const ::brotensor::Tensor& dY, | ||
| 412 | const ::brotensor::Tensor& Idx, int R, | ||
| 413 | ::brotensor::Tensor& dX); | ||
| 414 | void scatter_rows(const ::brotensor::Tensor& Y, | ||
| 415 | const ::brotensor::Tensor& Idx, | ||
| 416 | ::brotensor::Tensor& X); | ||
| 417 | void conv_transpose2d_forward(const ::brotensor::Tensor& X, | ||
| 418 | const ::brotensor::Tensor& Wt, | ||
| 419 | const ::brotensor::Tensor* bias, | ||
| 420 | int N, int C_in, int H, int W, | ||
| 421 | int C_out, int kH, int kW, | ||
| 422 | int stride_h, int stride_w, | ||
| 423 | int pad_h, int pad_w, | ||
| 424 | int output_padding_h, int output_padding_w, | ||
| 425 | int dil_h, int dil_w, int groups, | ||
| 426 | ::brotensor::Tensor& Y); | ||
| 427 | void conv_transpose2d_backward_input(const ::brotensor::Tensor& Wt, | ||
| 428 | const ::brotensor::Tensor& dY, | ||
| 429 | int N, int C_in, int H, int W, | ||
| 430 | int C_out, int kH, int kW, | ||
| 431 | int stride_h, int stride_w, | ||
| 432 | int pad_h, int pad_w, | ||
| 433 | int output_padding_h, int output_padding_w, | ||
| 434 | int dil_h, int dil_w, int groups, | ||
| 435 | ::brotensor::Tensor& dX); | ||
| 436 | void conv_transpose2d_backward_weight(const ::brotensor::Tensor& X, | ||
| 437 | const ::brotensor::Tensor& dY, | ||
| 438 | int N, int C_in, int H, int W, | ||
| 439 | int C_out, int kH, int kW, | ||
| 440 | int stride_h, int stride_w, | ||
| 441 | int pad_h, int pad_w, | ||
| 442 | int output_padding_h, int output_padding_w, | ||
| 443 | int dil_h, int dil_w, int groups, | ||
| 444 | ::brotensor::Tensor& dWt); | ||
| 445 | void conv_transpose2d_backward_bias(const ::brotensor::Tensor& dY, | ||
| 446 | int N, int C_out, int H_out, int W_out, | ||
| 447 | ::brotensor::Tensor& dB); | ||
| 448 | void window_partition_forward(const ::brotensor::Tensor& X, | ||
| 449 | int N, int C, int H, int W, int window, | ||
| 450 | ::brotensor::Tensor& Y); | ||
| 451 | void window_reverse_forward(const ::brotensor::Tensor& X, | ||
| 452 | int N, int C, int H, int W, int window, | ||
| 453 | ::brotensor::Tensor& Y); | ||
| 454 | void nchw_to_sequence(const ::brotensor::Tensor& X, | ||
| 455 | int N, int C, int H, int W, ::brotensor::Tensor& Y); | ||
| 456 | void sequence_to_nchw(const ::brotensor::Tensor& X, | ||
| 457 | int N, int C, int H, int W, ::brotensor::Tensor& Y); | ||
| 458 | void ddim_step(const ::brotensor::Tensor& x_t, | ||
| 459 | const ::brotensor::Tensor& eps_pred, | ||
| 460 | float alpha_t, float alpha_prev, float sigma_t, | ||
| 461 | ::brotensor::Tensor& x_prev); | ||
| 462 | void euler_step(const ::brotensor::Tensor& x_t, | ||
| 463 | const ::brotensor::Tensor& eps_pred, | ||
| 464 | float sigma_t, float sigma_prev, | ||
| 465 | ::brotensor::Tensor& x_prev); | ||
| 466 | void dpmpp_2m_step(const ::brotensor::Tensor& x_t, | ||
| 467 | const ::brotensor::Tensor& eps_pred, | ||
| 468 | const ::brotensor::Tensor& x0_prev, | ||
| 469 | float sigma_t, | ||
| 470 | float c_xt, float c_x0t, float c_x0prev, | ||
| 471 | ::brotensor::Tensor& x_prev, ::brotensor::Tensor& x0_out); | ||
| 472 | void timestep_embedding(const ::brotensor::Tensor& timesteps, | ||
| 473 | int dim, float max_period, ::brotensor::Tensor& Y); | ||
| 474 | void kv_cache_append(const ::brotensor::Tensor& K_new, | ||
| 475 | const ::brotensor::Tensor& V_new, int cur_len, | ||
| 476 | ::brotensor::Tensor& K_cache, ::brotensor::Tensor& V_cache); | ||
| 477 | void flash_attention_decode(const ::brotensor::Tensor& Q, | ||
| 478 | const ::brotensor::Tensor& K_cache, | ||
| 479 | const ::brotensor::Tensor& V_cache, | ||
| 480 | int valid_len, | ||
| 481 | int num_q_heads, int num_kv_heads, | ||
| 482 | ::brotensor::Tensor& O, | ||
| 483 | float attn_softcap, int window); | ||
| 484 | void flash_attention_decode_masked(const ::brotensor::Tensor& Q, | ||
| 485 | const ::brotensor::Tensor& K_cache, | ||
| 486 | const ::brotensor::Tensor& V_cache, | ||
| 487 | const float* d_mask, | ||
| 488 | int num_q_heads, int num_kv_heads, | ||
| 489 | ::brotensor::Tensor& O, | ||
| 490 | float attn_softcap, int window); | ||
| 491 | |||
| 492 | // ── CHUNK 5 — cross_attention.cpp / self_attention.cpp / | ||
| 493 | // attention_moments.cpp ── | ||
| 494 | void cross_attention_forward(const ::brotensor::Tensor& X, | ||
| 495 | const ::brotensor::Tensor& Ctx, | ||
| 496 | const ::brotensor::Tensor& Wq, | ||
| 497 | const ::brotensor::Tensor& Wk, | ||
| 498 | const ::brotensor::Tensor& Wv, | ||
| 499 | const ::brotensor::Tensor& Wo, | ||
| 500 | const float* d_mask, int num_heads, | ||
| 501 | ::brotensor::Tensor& O); | ||
| 502 | void cross_attention_forward_with_attn(const ::brotensor::Tensor& X, | ||
| 503 | const ::brotensor::Tensor& Ctx, | ||
| 504 | const ::brotensor::Tensor& Wq, | ||
| 505 | const ::brotensor::Tensor& Wk, | ||
| 506 | const ::brotensor::Tensor& Wv, | ||
| 507 | const ::brotensor::Tensor& Wo, | ||
| 508 | const float* d_mask, | ||
| 509 | const ::brotensor::Tensor* attn_logit_bias, | ||
| 510 | int num_heads, | ||
| 511 | ::brotensor::Tensor& O, | ||
| 512 | ::brotensor::Tensor& AttnAvg); | ||
| 513 | void self_attention_forward_train(const ::brotensor::Tensor& X, | ||
| 514 | const ::brotensor::Tensor& Wq, | ||
| 515 | const ::brotensor::Tensor& Wk, | ||
| 516 | const ::brotensor::Tensor& Wv, | ||
| 517 | const ::brotensor::Tensor& Wo, | ||
| 518 | const float* d_mask, int num_heads, | ||
| 519 | ::brotensor::Tensor& Qh, | ||
| 520 | ::brotensor::Tensor& Kh, | ||
| 521 | ::brotensor::Tensor& Vh, | ||
| 522 | ::brotensor::Tensor& Attnh, | ||
| 523 | ::brotensor::Tensor& Yconcat, | ||
| 524 | ::brotensor::Tensor& O); | ||
| 525 | void self_attention_backward(const ::brotensor::Tensor& dO, | ||
| 526 | const ::brotensor::Tensor& X, | ||
| 527 | const ::brotensor::Tensor& Qh, | ||
| 528 | const ::brotensor::Tensor& Kh, | ||
| 529 | const ::brotensor::Tensor& Vh, | ||
| 530 | const ::brotensor::Tensor& Attnh, | ||
| 531 | const ::brotensor::Tensor& Yconcat, | ||
| 532 | const ::brotensor::Tensor& Wq, | ||
| 533 | const ::brotensor::Tensor& Wk, | ||
| 534 | const ::brotensor::Tensor& Wv, | ||
| 535 | const ::brotensor::Tensor& Wo, | ||
| 536 | const float* d_mask, int num_heads, | ||
| 537 | ::brotensor::Tensor& dX, | ||
| 538 | ::brotensor::Tensor& dWq, | ||
| 539 | ::brotensor::Tensor& dWk, | ||
| 540 | ::brotensor::Tensor& dWv, | ||
| 541 | ::brotensor::Tensor& dWo); | ||
| 542 | void attention_token_moments(const ::brotensor::Tensor& Attn, | ||
| 543 | int h_lat, int w_lat, | ||
| 544 | ::brotensor::Tensor& mass, | ||
| 545 | ::brotensor::Tensor& centroid); | ||
| 546 | void cross_attention_forward_train(const ::brotensor::Tensor& X, | ||
| 547 | const ::brotensor::Tensor& Ctx, | ||
| 548 | const ::brotensor::Tensor& Wq, | ||
| 549 | const ::brotensor::Tensor& Wk, | ||
| 550 | const ::brotensor::Tensor& Wv, | ||
| 551 | const ::brotensor::Tensor& Wo, | ||
| 552 | const float* d_mask, int num_heads, | ||
| 553 | ::brotensor::Tensor& Qh, | ||
| 554 | ::brotensor::Tensor& Kh, | ||
| 555 | ::brotensor::Tensor& Vh, | ||
| 556 | ::brotensor::Tensor& Attnh, | ||
| 557 | ::brotensor::Tensor& Yconcat, | ||
| 558 | ::brotensor::Tensor& O); | ||
| 559 | void cross_attention_backward(const ::brotensor::Tensor& dO, | ||
| 560 | const ::brotensor::Tensor& X, | ||
| 561 | const ::brotensor::Tensor& Ctx, | ||
| 562 | const ::brotensor::Tensor& Qh, | ||
| 563 | const ::brotensor::Tensor& Kh, | ||
| 564 | const ::brotensor::Tensor& Vh, | ||
| 565 | const ::brotensor::Tensor& Attnh, | ||
| 566 | const ::brotensor::Tensor& Yconcat, | ||
| 567 | const ::brotensor::Tensor& Wq, | ||
| 568 | const ::brotensor::Tensor& Wk, | ||
| 569 | const ::brotensor::Tensor& Wv, | ||
| 570 | const ::brotensor::Tensor& Wo, | ||
| 571 | const float* d_mask, int num_heads, | ||
| 572 | ::brotensor::Tensor& dX, | ||
| 573 | ::brotensor::Tensor& dCtx, | ||
| 574 | ::brotensor::Tensor& dWq, | ||
| 575 | ::brotensor::Tensor& dWk, | ||
| 576 | ::brotensor::Tensor& dWv, | ||
| 577 | ::brotensor::Tensor& dWo); | ||
| 578 | |||
| 579 | // ── CHUNK 6 — flash_attention.cpp / self_attention_inference.cpp / | ||
| 580 | // resblock.cpp ── | ||
| 581 | void flash_attention_forward(const ::brotensor::Tensor& Q, | ||
| 582 | const ::brotensor::Tensor& K, | ||
| 583 | const ::brotensor::Tensor& V, | ||
| 584 | const float* d_mask, int num_heads, bool causal, | ||
| 585 | ::brotensor::Tensor& O); | ||
| 586 | void flash_attention_windowed_forward(const ::brotensor::Tensor& Q, | ||
| 587 | const ::brotensor::Tensor& K, | ||
| 588 | const ::brotensor::Tensor& V, | ||
| 589 | const float* d_mask, int num_heads, int window, | ||
| 590 | ::brotensor::Tensor& O); | ||
| 591 | void flash_attention_gqa_forward(const ::brotensor::Tensor& Q, | ||
| 592 | const ::brotensor::Tensor& K, | ||
| 593 | const ::brotensor::Tensor& V, | ||
| 594 | const float* d_mask, int num_q_heads, int num_kv_heads, | ||
| 595 | bool causal, ::brotensor::Tensor& O); | ||
| 596 | void flash_attention_varlen_forward(const ::brotensor::Tensor& Q, | ||
| 597 | const ::brotensor::Tensor& K, | ||
| 598 | const ::brotensor::Tensor& V, | ||
| 599 | const int32_t* cu_seqlens_q, | ||
| 600 | const int32_t* cu_seqlens_k, | ||
| 601 | int batch_size, int max_seqlen_q, int max_seqlen_k, | ||
| 602 | int num_heads, int head_dim, bool causal, | ||
| 603 | ::brotensor::Tensor& O); | ||
| 604 | void flash_attention_varlen_backward(const ::brotensor::Tensor& Q, | ||
| 605 | const ::brotensor::Tensor& K, | ||
| 606 | const ::brotensor::Tensor& V, | ||
| 607 | const ::brotensor::Tensor& O, | ||
| 608 | const ::brotensor::Tensor& dO, | ||
| 609 | const int32_t* cu_seqlens_q, | ||
| 610 | const int32_t* cu_seqlens_k, | ||
| 611 | int batch_size, int max_seqlen_q, int max_seqlen_k, | ||
| 612 | int num_heads, int head_dim, bool causal, | ||
| 613 | ::brotensor::Tensor& dQ, | ||
| 614 | ::brotensor::Tensor& dK, | ||
| 615 | ::brotensor::Tensor& dV); | ||
| 616 | void flash_attention_qkvo_forward(const ::brotensor::Tensor& X, | ||
| 617 | const ::brotensor::Tensor* Ctx, | ||
| 618 | const ::brotensor::Tensor& Wq, | ||
| 619 | const ::brotensor::Tensor* bq, | ||
| 620 | const ::brotensor::Tensor& Wk, | ||
| 621 | const ::brotensor::Tensor* bk, | ||
| 622 | const ::brotensor::Tensor& Wv, | ||
| 623 | const ::brotensor::Tensor* bv, | ||
| 624 | const ::brotensor::Tensor& Wo, | ||
| 625 | const ::brotensor::Tensor* bo, | ||
| 626 | const float* d_mask, int num_heads, | ||
| 627 | bool causal, ::brotensor::Tensor& O); | ||
| 628 | void flash_attention_qkvo_backward(const ::brotensor::Tensor& X, | ||
| 629 | const ::brotensor::Tensor* Ctx, | ||
| 630 | const ::brotensor::Tensor& Wq, | ||
| 631 | const ::brotensor::Tensor* bq, | ||
| 632 | const ::brotensor::Tensor& Wk, | ||
| 633 | const ::brotensor::Tensor* bk, | ||
| 634 | const ::brotensor::Tensor& Wv, | ||
| 635 | const ::brotensor::Tensor* bv, | ||
| 636 | const ::brotensor::Tensor& Wo, | ||
| 637 | const ::brotensor::Tensor* bo, | ||
| 638 | const float* d_mask, int num_heads, | ||
| 639 | bool causal, | ||
| 640 | const ::brotensor::Tensor& dO, | ||
| 641 | ::brotensor::Tensor& dX, | ||
| 642 | ::brotensor::Tensor* dCtx, | ||
| 643 | ::brotensor::Tensor& dWq, | ||
| 644 | ::brotensor::Tensor* dbq, | ||
| 645 | ::brotensor::Tensor& dWk, | ||
| 646 | ::brotensor::Tensor* dbk, | ||
| 647 | ::brotensor::Tensor& dWv, | ||
| 648 | ::brotensor::Tensor* dbv, | ||
| 649 | ::brotensor::Tensor& dWo, | ||
| 650 | ::brotensor::Tensor* dbo); | ||
| 651 | void flash_attention_backward(const ::brotensor::Tensor& Q, | ||
| 652 | const ::brotensor::Tensor& K, | ||
| 653 | const ::brotensor::Tensor& V, | ||
| 654 | const ::brotensor::Tensor& O, | ||
| 655 | const ::brotensor::Tensor& dO, | ||
| 656 | const float* d_mask, int num_heads, bool causal, | ||
| 657 | ::brotensor::Tensor& dQ, | ||
| 658 | ::brotensor::Tensor& dK, | ||
| 659 | ::brotensor::Tensor& dV); | ||
| 660 | void flash_attention_project_kv(const ::brotensor::Tensor& ctx, | ||
| 661 | const ::brotensor::Tensor& Wk, | ||
| 662 | const ::brotensor::Tensor* bk, | ||
| 663 | const ::brotensor::Tensor& Wv, | ||
| 664 | const ::brotensor::Tensor* bv, | ||
| 665 | ::brotensor::Tensor& K_out, | ||
| 666 | ::brotensor::Tensor& V_out); | ||
| 667 | void flash_attention_q_with_kv_cached_forward(const ::brotensor::Tensor& X, | ||
| 668 | const ::brotensor::Tensor& K, | ||
| 669 | const ::brotensor::Tensor& V, | ||
| 670 | const ::brotensor::Tensor& Wq, | ||
| 671 | const ::brotensor::Tensor* bq, | ||
| 672 | const ::brotensor::Tensor& Wo, | ||
| 673 | const ::brotensor::Tensor* bo, | ||
| 674 | const float* d_mask, | ||
| 675 | int num_heads, bool causal, | ||
| 676 | ::brotensor::Tensor& O); | ||
| 677 | void self_attention_forward(const ::brotensor::Tensor& X, | ||
| 678 | const ::brotensor::Tensor& Wq, | ||
| 679 | const ::brotensor::Tensor& Wk, | ||
| 680 | const ::brotensor::Tensor& Wv, | ||
| 681 | const ::brotensor::Tensor& Wo, | ||
| 682 | const float* d_mask, int num_heads, | ||
| 683 | ::brotensor::Tensor& O); | ||
| 684 | void resblock_forward(const ::brotensor::Tensor& X, | ||
| 685 | const ::brotensor::Tensor& gamma1, | ||
| 686 | const ::brotensor::Tensor& beta1, | ||
| 687 | const ::brotensor::Tensor& W1, | ||
| 688 | const ::brotensor::Tensor* b1, | ||
| 689 | const ::brotensor::Tensor* t_emb_shift, | ||
| 690 | const ::brotensor::Tensor& gamma2, | ||
| 691 | const ::brotensor::Tensor& beta2, | ||
| 692 | const ::brotensor::Tensor& W2, | ||
| 693 | const ::brotensor::Tensor* b2, | ||
| 694 | const ::brotensor::Tensor* Wskip, | ||
| 695 | const ::brotensor::Tensor* bskip, | ||
| 696 | int N, int C_in, int C_out, int H, int W, | ||
| 697 | int num_groups, float eps, ::brotensor::Tensor& Y); | ||
| 698 | void resblock_backward(const ::brotensor::Tensor& X, | ||
| 699 | const ::brotensor::Tensor& gamma1, | ||
| 700 | const ::brotensor::Tensor& beta1, | ||
| 701 | const ::brotensor::Tensor& W1, | ||
| 702 | const ::brotensor::Tensor* b1, | ||
| 703 | const ::brotensor::Tensor* t_emb_shift, | ||
| 704 | const ::brotensor::Tensor& gamma2, | ||
| 705 | const ::brotensor::Tensor& beta2, | ||
| 706 | const ::brotensor::Tensor& W2, | ||
| 707 | const ::brotensor::Tensor* b2, | ||
| 708 | const ::brotensor::Tensor* Wskip, | ||
| 709 | const ::brotensor::Tensor* bskip, | ||
| 710 | int N, int C_in, int C_out, int H, int W, | ||
| 711 | int num_groups, float eps, | ||
| 712 | const ::brotensor::Tensor& dY, | ||
| 713 | ::brotensor::Tensor& dX, | ||
| 714 | ::brotensor::Tensor& dGamma1, | ||
| 715 | ::brotensor::Tensor& dBeta1, | ||
| 716 | ::brotensor::Tensor& dW1, | ||
| 717 | ::brotensor::Tensor* db1, | ||
| 718 | ::brotensor::Tensor* dt_emb_shift, | ||
| 719 | ::brotensor::Tensor& dGamma2, | ||
| 720 | ::brotensor::Tensor& dBeta2, | ||
| 721 | ::brotensor::Tensor& dW2, | ||
| 722 | ::brotensor::Tensor* db2, | ||
| 723 | ::brotensor::Tensor* dWskip, | ||
| 724 | ::brotensor::Tensor* dbskip); | ||
| 725 | |||
| 726 | // ── DiT / diffusion extras — modulate.cpp ── | ||
| 727 | void modulate(const ::brotensor::Tensor& X, const ::brotensor::Tensor& scale, | ||
| 728 | const ::brotensor::Tensor& shift, ::brotensor::Tensor& Y); | ||
| 729 | void broadcast_mul(const ::brotensor::Tensor& X, const ::brotensor::Tensor& v, | ||
| 730 | ::brotensor::Tensor& Y); | ||
| 731 | |||
| 732 | // ── RoPE with explicit cos/sin tables — rope.cpp ── | ||
| 733 | void rope_apply(const ::brotensor::Tensor& X, const ::brotensor::Tensor& cos_tbl, | ||
| 734 | const ::brotensor::Tensor& sin_tbl, int head_dim, int num_heads, | ||
| 735 | ::brotensor::Tensor& Y); | ||
| 736 | void rope_apply_perhead(const ::brotensor::Tensor& X, const ::brotensor::Tensor& cos_tbl, | ||
| 737 | const ::brotensor::Tensor& sin_tbl, int head_dim, int num_heads, | ||
| 738 | ::brotensor::Tensor& Y); | ||
| 739 | void rope_apply_backward(const ::brotensor::Tensor& dY, | ||
| 740 | const ::brotensor::Tensor& cos_tbl, | ||
| 741 | const ::brotensor::Tensor& sin_tbl, | ||
| 742 | int head_dim, int num_heads, ::brotensor::Tensor& dX); | ||
| 743 | |||
| 744 | // ── Self-attention with additive bias — self_attention_bias.cpp ── | ||
| 745 | void self_attention_bias_forward(const ::brotensor::Tensor& X, | ||
| 746 | const ::brotensor::Tensor& Wq, | ||
| 747 | const ::brotensor::Tensor& Wk, | ||
| 748 | const ::brotensor::Tensor& Wv, | ||
| 749 | const ::brotensor::Tensor& Wo, | ||
| 750 | const ::brotensor::Tensor* bq, | ||
| 751 | const ::brotensor::Tensor* bk, | ||
| 752 | const ::brotensor::Tensor* bv, | ||
| 753 | const ::brotensor::Tensor* bo, | ||
| 754 | const float* d_mask, | ||
| 755 | const ::brotensor::Tensor* attn_bias, | ||
| 756 | int num_heads, float scale, | ||
| 757 | ::brotensor::Tensor& O); | ||
| 758 | void rel_pos_bias_xl_forward(const ::brotensor::Tensor& Qv, | ||
| 759 | const ::brotensor::Tensor& Pk, | ||
| 760 | int num_heads, int head_dim, | ||
| 761 | ::brotensor::Tensor& Bias); | ||
| 762 | |||
| 763 | // ── Self-attention with decomposed 2D rel-pos bias (SAM/ViTDet) — | ||
| 764 | // self_attention_decomposed_rel_pos.cpp ── | ||
| 765 | void self_attention_decomposed_rel_pos_forward( | ||
| 766 | const ::brotensor::Tensor& X, | ||
| 767 | const ::brotensor::Tensor& Wq, const ::brotensor::Tensor* bq, | ||
| 768 | const ::brotensor::Tensor& Wk, const ::brotensor::Tensor* bk, | ||
| 769 | const ::brotensor::Tensor& Wv, const ::brotensor::Tensor* bv, | ||
| 770 | const ::brotensor::Tensor& Wo, const ::brotensor::Tensor* bo, | ||
| 771 | const ::brotensor::Tensor& rel_pos_h, const ::brotensor::Tensor& rel_pos_w, | ||
| 772 | int num_heads, int grid_h, int grid_w, float scale, | ||
| 773 | ::brotensor::Tensor& O); | ||
| 774 | void self_attention_decomposed_rel_pos_windowed_forward( | ||
| 775 | const ::brotensor::Tensor& X, | ||
| 776 | const ::brotensor::Tensor& Wq, const ::brotensor::Tensor* bq, | ||
| 777 | const ::brotensor::Tensor& Wk, const ::brotensor::Tensor* bk, | ||
| 778 | const ::brotensor::Tensor& Wv, const ::brotensor::Tensor* bv, | ||
| 779 | const ::brotensor::Tensor& Wo, const ::brotensor::Tensor* bo, | ||
| 780 | const ::brotensor::Tensor& rel_pos_h, const ::brotensor::Tensor& rel_pos_w, | ||
| 781 | int num_heads, int grid_h, int grid_w, int window, float scale, | ||
| 782 | ::brotensor::Tensor& O); | ||
| 783 | |||
| 784 | // ── Spectral / FFT core (brosoundml) — fft.cpp ── | ||
| 785 | void complex_mul(const ::brotensor::Tensor& a, const ::brotensor::Tensor& b, | ||
| 786 | ::brotensor::Tensor& y); | ||
| 787 | void complex_mul_backward(const ::brotensor::Tensor& a, | ||
| 788 | const ::brotensor::Tensor& b, | ||
| 789 | const ::brotensor::Tensor& dY, | ||
| 790 | ::brotensor::Tensor& dA, ::brotensor::Tensor& dB); | ||
| 791 | void complex_abs(const ::brotensor::Tensor& z, ::brotensor::Tensor& y); | ||
| 792 | void complex_abs_backward(const ::brotensor::Tensor& z, | ||
| 793 | const ::brotensor::Tensor& dY, | ||
| 794 | ::brotensor::Tensor& dZ); | ||
| 795 | void complex_angle(const ::brotensor::Tensor& z, ::brotensor::Tensor& y); | ||
| 796 | void complex_from_polar(const ::brotensor::Tensor& mag, | ||
| 797 | const ::brotensor::Tensor& phase, | ||
| 798 | ::brotensor::Tensor& y); | ||
| 799 | void fft(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 800 | void ifft(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 801 | void rfft(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 802 | void irfft(const ::brotensor::Tensor& x, int L, ::brotensor::Tensor& y); | ||
| 803 | void rfft_backward(const ::brotensor::Tensor& dY, int L, | ||
| 804 | ::brotensor::Tensor& dX); | ||
| 805 | void irfft_backward(const ::brotensor::Tensor& dY, ::brotensor::Tensor& dX); | ||
| 806 | |||
| 807 | // ── STFT / iSTFT (brosoundml) — stft.cpp ── | ||
| 808 | void stft(const ::brotensor::Tensor& signal, const ::brotensor::Tensor& window, | ||
| 809 | int N, int n_fft, int hop_length, int win_length, | ||
| 810 | bool center, bool normalized, ::brotensor::Tensor& spec); | ||
| 811 | void stft_backward(const ::brotensor::Tensor& dSpec, | ||
| 812 | const ::brotensor::Tensor& window, | ||
| 813 | int N, int signal_len, int n_fft, int hop_length, | ||
| 814 | int win_length, bool center, bool normalized, | ||
| 815 | ::brotensor::Tensor& dSignal); | ||
| 816 | void istft(const ::brotensor::Tensor& spec, const ::brotensor::Tensor& window, | ||
| 817 | int N, int signal_len, int n_fft, int hop_length, int win_length, | ||
| 818 | bool center, bool normalized, ::brotensor::Tensor& signal); | ||
| 819 | void istft_backward(const ::brotensor::Tensor& dSignal, | ||
| 820 | const ::brotensor::Tensor& window, | ||
| 821 | int N, int signal_len, int n_fft, int hop_length, | ||
| 822 | int win_length, bool center, bool normalized, | ||
| 823 | ::brotensor::Tensor& dSpec); | ||
| 824 | |||
| 825 | // ── 1D convolution family (brosoundml) — conv1d.cpp ── | ||
| 826 | void conv_transpose1d_forward(const ::brotensor::Tensor& X, | ||
| 827 | const ::brotensor::Tensor& Wt, | ||
| 828 | const ::brotensor::Tensor* bias, | ||
| 829 | int N, int C_in, int L, int C_out, int kL, | ||
| 830 | int stride, int padding, int output_padding, | ||
| 831 | int dilation, int groups, ::brotensor::Tensor& Y); | ||
| 832 | void conv_transpose1d_backward_input(const ::brotensor::Tensor& Wt, | ||
| 833 | const ::brotensor::Tensor& dY, | ||
| 834 | int N, int C_in, int L, int C_out, int kL, | ||
| 835 | int stride, int padding, | ||
| 836 | int output_padding, int dilation, | ||
| 837 | int groups, ::brotensor::Tensor& dX); | ||
| 838 | void conv_transpose1d_backward_weight(const ::brotensor::Tensor& X, | ||
| 839 | const ::brotensor::Tensor& dY, | ||
| 840 | int N, int C_in, int L, int C_out, int kL, | ||
| 841 | int stride, int padding, | ||
| 842 | int output_padding, int dilation, | ||
| 843 | int groups, ::brotensor::Tensor& dWt); | ||
| 844 | void conv_transpose1d_backward_bias(const ::brotensor::Tensor& dY, | ||
| 845 | int N, int C_out, int L_out, | ||
| 846 | ::brotensor::Tensor& dB); | ||
| 847 | void causal_conv1d_update(const ::brotensor::Tensor& X, | ||
| 848 | const ::brotensor::Tensor& Wt, | ||
| 849 | const ::brotensor::Tensor* bias, | ||
| 850 | int N, int C, int L_step, int kL, int dilation, | ||
| 851 | ::brotensor::Tensor& state, ::brotensor::Tensor& Y); | ||
| 852 | void pad1d_forward(const ::brotensor::Tensor& X, int N, int C, int L, | ||
| 853 | int pad_left, int pad_right, int mode, | ||
| 854 | ::brotensor::Tensor& Y); | ||
| 855 | void pad1d_backward(const ::brotensor::Tensor& dY, int N, int C, int L, | ||
| 856 | int pad_left, int pad_right, int mode, | ||
| 857 | ::brotensor::Tensor& dX); | ||
| 858 | |||
| 859 | // ── Vocoder / codec activations (brosoundml) — vocoder_activations.cpp ── | ||
| 860 | void snake_forward(const ::brotensor::Tensor& X, | ||
| 861 | const ::brotensor::Tensor& alpha, | ||
| 862 | const ::brotensor::Tensor* beta, | ||
| 863 | int N, int C, int L, ::brotensor::Tensor& Y); | ||
| 864 | void snake_backward(const ::brotensor::Tensor& X, | ||
| 865 | const ::brotensor::Tensor& alpha, | ||
| 866 | const ::brotensor::Tensor* beta, | ||
| 867 | const ::brotensor::Tensor& dY, | ||
| 868 | int N, int C, int L, | ||
| 869 | ::brotensor::Tensor& dX, ::brotensor::Tensor& dAlpha, | ||
| 870 | ::brotensor::Tensor* dBeta); | ||
| 871 | void elu_forward(const ::brotensor::Tensor& x, float alpha, | ||
| 872 | ::brotensor::Tensor& y); | ||
| 873 | void elu_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY, | ||
| 874 | float alpha, ::brotensor::Tensor& dX); | ||
| 875 | void leaky_relu_forward(const ::brotensor::Tensor& x, float negative_slope, | ||
| 876 | ::brotensor::Tensor& y); | ||
| 877 | void leaky_relu_backward(const ::brotensor::Tensor& x, | ||
| 878 | const ::brotensor::Tensor& dY, | ||
| 879 | float negative_slope, ::brotensor::Tensor& dX); | ||
| 880 | |||
| 881 | // ── Codec quantization (brosoundml CHUNK 5, family D) — codec_quant.cpp ── | ||
| 882 | void vq_encode_forward(const ::brotensor::Tensor& x, | ||
| 883 | const ::brotensor::Tensor& codebook, | ||
| 884 | ::brotensor::Tensor& indices, | ||
| 885 | ::brotensor::Tensor& quantized); | ||
| 886 | void vq_encode_backward(const ::brotensor::Tensor& dQuantized, | ||
| 887 | ::brotensor::Tensor& dX); | ||
| 888 | void fsq_quantize_forward(const ::brotensor::Tensor& x, | ||
| 889 | const ::brotensor::Tensor& levels, | ||
| 890 | ::brotensor::Tensor& quantized, | ||
| 891 | ::brotensor::Tensor& packed_indices); | ||
| 892 | void fsq_quantize_backward(const ::brotensor::Tensor& dQuantized, | ||
| 893 | ::brotensor::Tensor& dX); | ||
| 894 | |||
| 895 | // ── 1D resampling (brosoundml CHUNK 6, family E) — resample1d.cpp ── | ||
| 896 | void resample1d_forward(const ::brotensor::Tensor& X, | ||
| 897 | int N, int C, int L_in, int L_out, int mode, | ||
| 898 | ::brotensor::Tensor& Y); | ||
| 899 | void resample1d_backward(const ::brotensor::Tensor& dY, | ||
| 900 | int N, int C, int L_in, int L_out, int mode, | ||
| 901 | ::brotensor::Tensor& dX); | ||
| 902 | |||
| 903 | // ── log / exp / round elementwise (brosoundml CHUNK 6, family G) | ||
| 904 | // — log_exp_round.cpp ── | ||
| 905 | void log_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 906 | void log_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY, | ||
| 907 | ::brotensor::Tensor& dX); | ||
| 908 | void exp_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 909 | void exp_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY, | ||
| 910 | ::brotensor::Tensor& dX); | ||
| 911 | void round_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 912 | void round_backward(const ::brotensor::Tensor& dY, ::brotensor::Tensor& dX); | ||
| 913 | |||
| 914 | // ── Autoregressive logit sampling (brosoundml CHUNK 7, family F) | ||
| 915 | // — sample_logits.cpp ── | ||
| 916 | void sample_logits(const ::brotensor::Tensor& logits, float temperature, | ||
| 917 | int top_k, float top_p, uint64_t key, uint64_t counter, | ||
| 918 | ::brotensor::Tensor& indices); | ||
| 919 | void sample_logits_into(const ::brotensor::Tensor& logits, float temperature, | ||
| 920 | int top_k, float top_p, uint64_t key, | ||
| 921 | ::brotensor::Tensor& counter, | ||
| 922 | ::brotensor::Tensor& scratch, | ||
| 923 | ::brotensor::Tensor& indices); | ||
| 924 | |||
| 925 | // ── Counter-based noise generation (Philox 4x32-10) — noise.cpp ── | ||
| 926 | void randn(uint64_t key, uint64_t counter, ::brotensor::Tensor& Y); | ||
| 927 | void rand_uniform(uint64_t key, uint64_t counter, ::brotensor::Tensor& Y); | ||
| 928 | void rand_bernoulli(float p, uint64_t key, uint64_t counter, | ||
| 929 | ::brotensor::Tensor& Y); | ||
| 930 | void randn_truncated(float lo, float hi, uint64_t key, uint64_t counter, | ||
| 931 | ::brotensor::Tensor& Y); | ||
| 932 | |||
| 933 | // ── L2 norm + Gated Delta Rule (linear-attention text path) ── | ||
| 934 | // l2_norm.cpp, gated_delta_rule.cpp | ||
| 935 | void l2_norm_forward(const ::brotensor::Tensor& X, | ||
| 936 | int head_dim, int num_heads, float eps, | ||
| 937 | ::brotensor::Tensor& Y); | ||
| 938 | void l2_norm_backward(const ::brotensor::Tensor& X, | ||
| 939 | int head_dim, int num_heads, float eps, | ||
| 940 | const ::brotensor::Tensor& dY, | ||
| 941 | ::brotensor::Tensor& dX); | ||
| 942 | void gated_delta_rule_chunked(const ::brotensor::Tensor& Q, | ||
| 943 | const ::brotensor::Tensor& K, | ||
| 944 | const ::brotensor::Tensor& V, | ||
| 945 | const ::brotensor::Tensor& a_raw, | ||
| 946 | const ::brotensor::Tensor& beta, | ||
| 947 | const ::brotensor::Tensor& log_A, | ||
| 948 | int num_heads, int d_k, int d_v, | ||
| 949 | ::brotensor::Tensor& state, | ||
| 950 | ::brotensor::Tensor& O); | ||
| 951 | void gated_delta_rule_step(const ::brotensor::Tensor& Q, | ||
| 952 | const ::brotensor::Tensor& K, | ||
| 953 | const ::brotensor::Tensor& V, | ||
| 954 | const ::brotensor::Tensor& a_raw, | ||
| 955 | const ::brotensor::Tensor& beta, | ||
| 956 | const ::brotensor::Tensor& log_A, | ||
| 957 | int num_heads, int d_k, int d_v, | ||
| 958 | ::brotensor::Tensor& state, | ||
| 959 | ::brotensor::Tensor& O); | ||
| 960 | |||
| 961 | // ── Qwen3-VL polish: spatial_merge_2x2 + M-RoPE — spatial_merge.cpp / | ||
| 962 | // rope_mrope.cpp ── | ||
| 963 | void spatial_merge_2x2_forward(const ::brotensor::Tensor& X, | ||
| 964 | int N, int C, int H, int W, | ||
| 965 | bool channel_major, | ||
| 966 | ::brotensor::Tensor& Y); | ||
| 967 | void pixel_shuffle_upsample_2x_forward(const ::brotensor::Tensor& X, | ||
| 968 | int N, int C_in, int H, int W, | ||
| 969 | int C_out, ::brotensor::Tensor& Y); | ||
| 970 | void patch_unpack_forward(const ::brotensor::Tensor& tokens, | ||
| 971 | int hp, int wp, int P, int C_total, int C_keep, | ||
| 972 | bool channel_major, ::brotensor::Tensor& Y); | ||
| 973 | void rope_apply_mrope(const ::brotensor::Tensor& X, | ||
| 974 | const ::brotensor::Tensor& cos_t, | ||
| 975 | const ::brotensor::Tensor& sin_t, | ||
| 976 | const ::brotensor::Tensor& cos_h, | ||
| 977 | const ::brotensor::Tensor& sin_h, | ||
| 978 | const ::brotensor::Tensor& cos_w, | ||
| 979 | const ::brotensor::Tensor& sin_w, | ||
| 980 | const int32_t* pos_t, const int32_t* pos_h, | ||
| 981 | const int32_t* pos_w, | ||
| 982 | int head_dim, int num_heads, | ||
| 983 | int d_t, int d_h, int d_w, | ||
| 984 | ::brotensor::Tensor& Y); | ||
| 985 | |||
| 986 | // ── BatchNorm — batch_norm.cpp ── | ||
| 987 | void batch_norm_forward(const ::brotensor::Tensor& X, | ||
| 988 | const ::brotensor::Tensor& gamma, | ||
| 989 | const ::brotensor::Tensor& beta, | ||
| 990 | ::brotensor::Tensor& running_mean, | ||
| 991 | ::brotensor::Tensor& running_var, | ||
| 992 | int N, int C, int H, int W, | ||
| 993 | float eps, float momentum, | ||
| 994 | ::brotensor::Tensor& Y, | ||
| 995 | ::brotensor::Tensor& saved_mean, | ||
| 996 | ::brotensor::Tensor& saved_rstd); | ||
| 997 | void batch_norm_inference(const ::brotensor::Tensor& X, | ||
| 998 | const ::brotensor::Tensor& gamma, | ||
| 999 | const ::brotensor::Tensor& beta, | ||
| 1000 | const ::brotensor::Tensor& running_mean, | ||
| 1001 | const ::brotensor::Tensor& running_var, | ||
| 1002 | int N, int C, int H, int W, | ||
| 1003 | float eps, | ||
| 1004 | ::brotensor::Tensor& Y); | ||
| 1005 | void batch_norm_backward(const ::brotensor::Tensor& X, | ||
| 1006 | const ::brotensor::Tensor& gamma, | ||
| 1007 | const ::brotensor::Tensor& saved_mean, | ||
| 1008 | const ::brotensor::Tensor& saved_rstd, | ||
| 1009 | const ::brotensor::Tensor& dY, | ||
| 1010 | int N, int C, int H, int W, | ||
| 1011 | ::brotensor::Tensor& dX, | ||
| 1012 | ::brotensor::Tensor& dGamma, | ||
| 1013 | ::brotensor::Tensor& dBeta); | ||
| 1014 | |||
| 1015 | // ── Image preprocessing helpers — image_preproc.cpp ── | ||
| 1016 | void image_normalize(const ::brotensor::Tensor& X, | ||
| 1017 | const ::brotensor::Tensor& mean, | ||
| 1018 | const ::brotensor::Tensor& std_, | ||
| 1019 | int N, int C, int H, int W, | ||
| 1020 | ::brotensor::Tensor& Y); | ||
| 1021 | void image_u8_to_f32_nhwc_to_nchw(const uint8_t* src, | ||
| 1022 | int N, int H, int W, int C, | ||
| 1023 | float scale, float bias, | ||
| 1024 | ::brotensor::Tensor& Y); | ||
| 1025 | void lstm_forward_train(const ::brotensor::Tensor& X, const ::brotensor::Tensor& W_ih, | ||
| 1026 | const ::brotensor::Tensor& W_hh, | ||
| 1027 | const ::brotensor::Tensor* b_ih, const ::brotensor::Tensor* b_hh, | ||
| 1028 | const ::brotensor::Tensor* h0, const ::brotensor::Tensor* c0, | ||
| 1029 | int T, int B, | ||
| 1030 | ::brotensor::Tensor& Y, ::brotensor::Tensor& gates, | ||
| 1031 | ::brotensor::Tensor& C, | ||
| 1032 | ::brotensor::Tensor* hT, ::brotensor::Tensor* cT); | ||
| 1033 | void lstm_backward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& W_ih, | ||
| 1034 | const ::brotensor::Tensor& W_hh, | ||
| 1035 | const ::brotensor::Tensor* h0, const ::brotensor::Tensor* c0, | ||
| 1036 | const ::brotensor::Tensor& Y, const ::brotensor::Tensor& gates, | ||
| 1037 | const ::brotensor::Tensor& C, | ||
| 1038 | const ::brotensor::Tensor& dY, int T, int B, | ||
| 1039 | ::brotensor::Tensor& dX, ::brotensor::Tensor& dW_ih, | ||
| 1040 | ::brotensor::Tensor& dW_hh, | ||
| 1041 | ::brotensor::Tensor* db_ih, ::brotensor::Tensor* db_hh, | ||
| 1042 | ::brotensor::Tensor* dh0, ::brotensor::Tensor* dc0); | ||
| 1043 | |||
| 1044 | // ── StyleGAN3-R primitives — stylegan_elementwise.cpp / bias_act.cpp / | ||
| 1045 | // upfirdn2d.cpp / modulated_conv2d.cpp ── | ||
| 1046 | void sin_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 1047 | void sin_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY, | ||
| 1048 | ::brotensor::Tensor& dX); | ||
| 1049 | void cos_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 1050 | void cos_backward(const ::brotensor::Tensor& x, const ::brotensor::Tensor& dY, | ||
| 1051 | ::brotensor::Tensor& dX); | ||
| 1052 | void rsqrt_forward(const ::brotensor::Tensor& x, ::brotensor::Tensor& y); | ||
| 1053 | void rsqrt_backward(const ::brotensor::Tensor& y, const ::brotensor::Tensor& dY, | ||
| 1054 | ::brotensor::Tensor& dX); | ||
| 1055 | void pixel_norm_forward(const ::brotensor::Tensor& X, float eps, | ||
| 1056 | ::brotensor::Tensor& Y); | ||
| 1057 | void pixel_norm_backward(const ::brotensor::Tensor& X, | ||
| 1058 | const ::brotensor::Tensor& dY, float eps, | ||
| 1059 | ::brotensor::Tensor& dX); | ||
| 1060 | void bias_act_forward(const ::brotensor::Tensor& X, const ::brotensor::Tensor* b, | ||
| 1061 | int N, int C, int HW, int act, float alpha, | ||
| 1062 | float gain, float clamp, ::brotensor::Tensor& Y); | ||
| 1063 | void bias_act_backward(const ::brotensor::Tensor& dY, const ::brotensor::Tensor& X, | ||
| 1064 | const ::brotensor::Tensor* b, | ||
| 1065 | int N, int C, int HW, int act, float alpha, | ||
| 1066 | float gain, float clamp, | ||
| 1067 | ::brotensor::Tensor& dX, ::brotensor::Tensor* dB); | ||
| 1068 | void upfirdn2d_forward(const ::brotensor::Tensor& X, const ::brotensor::Tensor& f, | ||
| 1069 | int N, int C, int H, int Wd, int fH, int fW, | ||
| 1070 | int up_x, int up_y, int down_x, int down_y, | ||
| 1071 | int pad_x0, int pad_x1, int pad_y0, int pad_y1, | ||
| 1072 | bool flip_filter, float gain, ::brotensor::Tensor& Y); | ||
| 1073 | void upfirdn2d_backward(const ::brotensor::Tensor& dY, const ::brotensor::Tensor& f, | ||
| 1074 | int N, int C, int H, int Wd, int fH, int fW, | ||
| 1075 | int up_x, int up_y, int down_x, int down_y, | ||
| 1076 | int pad_x0, int pad_x1, int pad_y0, int pad_y1, | ||
| 1077 | bool flip_filter, float gain, ::brotensor::Tensor& dX); | ||
| 1078 | void modulated_conv2d_forward(const ::brotensor::Tensor& X, | ||
| 1079 | const ::brotensor::Tensor& W, | ||
| 1080 | const ::brotensor::Tensor& s, | ||
| 1081 | int N, int C_in, int H, int Wd, | ||
| 1082 | int C_out, int kH, int kW, int pad_h, int pad_w, | ||
| 1083 | bool demodulate, float eps, | ||
| 1084 | ::brotensor::Tensor& dcoef, ::brotensor::Tensor& Y); | ||
| 1085 | void modulated_conv2d_backward(const ::brotensor::Tensor& X, | ||
| 1086 | const ::brotensor::Tensor& W, | ||
| 1087 | const ::brotensor::Tensor& s, | ||
| 1088 | const ::brotensor::Tensor& dcoef, | ||
| 1089 | const ::brotensor::Tensor& dY, | ||
| 1090 | int N, int C_in, int H, int Wd, | ||
| 1091 | int C_out, int kH, int kW, int pad_h, int pad_w, | ||
| 1092 | bool demodulate, float eps, | ||
| 1093 | ::brotensor::Tensor& dX, ::brotensor::Tensor& dW, | ||
| 1094 | ::brotensor::Tensor& ds); | ||
| 1095 | |||
| 1096 | } // namespace brotensor::detail::cpu | ||
| 1097 | |||
| 1098 | namespace { | ||
| 1099 | |||
| 1100 | struct CpuStaticRegistrar { | ||
| 1101 | 330 | CpuStaticRegistrar() { | |
| 1102 | using namespace ::brotensor; | ||
| 1103 | using namespace ::brotensor::detail; | ||
| 1104 | |||
| 1105 | 165 | OpsVTable ops{}; // zero-init — every slot nullptr by default | |
| 1106 | |||
| 1107 | 165 | ops.linear_forward = &detail::cpu::linear_forward; | |
| 1108 | 165 | ops.linear_backward = &detail::cpu::linear_backward; | |
| 1109 | 165 | ops.relu_forward = &detail::cpu::relu_forward; | |
| 1110 | 165 | ops.relu_backward = &detail::cpu::relu_backward; | |
| 1111 | 165 | ops.tanh_forward = &detail::cpu::tanh_forward; | |
| 1112 | 165 | ops.tanh_backward = &detail::cpu::tanh_backward; | |
| 1113 | 165 | ops.sigmoid_forward = &detail::cpu::sigmoid_forward; | |
| 1114 | 165 | ops.sigmoid_backward = &detail::cpu::sigmoid_backward; | |
| 1115 | 165 | ops.softmax_forward = &detail::cpu::softmax_forward; | |
| 1116 | 165 | ops.softmax_rows_forward = &detail::cpu::softmax_rows_forward; | |
| 1117 | 165 | ops.softmax_backward = &detail::cpu::softmax_backward; | |
| 1118 | 165 | ops.softmax_xent = &detail::cpu::softmax_xent; | |
| 1119 | 165 | ops.softmax_xent_segment = &detail::cpu::softmax_xent_segment; | |
| 1120 | 165 | ops.mse_scalar = &detail::cpu::mse_scalar; | |
| 1121 | 165 | ops.add_inplace = &detail::cpu::add_inplace; | |
| 1122 | 165 | ops.add_scalar_inplace = &detail::cpu::add_scalar_inplace; | |
| 1123 | 165 | ops.add_channel_bias_inplace = &detail::cpu::add_channel_bias_inplace; | |
| 1124 | 165 | ops.xavier_init = &detail::cpu::xavier_init; | |
| 1125 | |||
| 1126 | 165 | ops.sgd_step = &detail::cpu::sgd_step; | |
| 1127 | 165 | ops.adam_step = &detail::cpu::adam_step; | |
| 1128 | 165 | ops.scale_inplace = &detail::cpu::scale_inplace; | |
| 1129 | 165 | ops.layernorm_forward = &detail::cpu::layernorm_forward; | |
| 1130 | 165 | ops.layernorm_backward = &detail::cpu::layernorm_backward; | |
| 1131 | 165 | ops.attention_forward = &detail::cpu::attention_forward; | |
| 1132 | 165 | ops.attention_backward = &detail::cpu::attention_backward; | |
| 1133 | 165 | ops.mha_forward = &detail::cpu::mha_forward; | |
| 1134 | 165 | ops.mha_backward = &detail::cpu::mha_backward; | |
| 1135 | 165 | ops.concat_rows = &detail::cpu::concat_rows; | |
| 1136 | 165 | ops.split_rows = &detail::cpu::split_rows; | |
| 1137 | 165 | ops.add_inplace_batched = &detail::cpu::add_inplace_batched; | |
| 1138 | 165 | ops.linear_forward_batched = &detail::cpu::linear_forward_batched; | |
| 1139 | 165 | ops.linear_backward_batched = &detail::cpu::linear_backward_batched; | |
| 1140 | 165 | ops.relu_forward_batched = &detail::cpu::relu_forward_batched; | |
| 1141 | 165 | ops.relu_backward_batched = &detail::cpu::relu_backward_batched; | |
| 1142 | 165 | ops.tanh_forward_batched = &detail::cpu::tanh_forward_batched; | |
| 1143 | 165 | ops.tanh_backward_batched = &detail::cpu::tanh_backward_batched; | |
| 1144 | 165 | ops.mse_vec_per_sample = &detail::cpu::mse_vec_per_sample; | |
| 1145 | 165 | ops.softmax_xent_fused_batched = &detail::cpu::softmax_xent_fused_batched; | |
| 1146 | 165 | ops.bce_with_logits_fused_batched = &detail::cpu::bce_with_logits_fused_batched; | |
| 1147 | 165 | ops.masked_mean_pool_forward = &detail::cpu::masked_mean_pool_forward; | |
| 1148 | 165 | ops.masked_mean_pool_backward = &detail::cpu::masked_mean_pool_backward; | |
| 1149 | 165 | ops.build_slot_mask = &detail::cpu::build_slot_mask; | |
| 1150 | 165 | ops.copy_d2d = &detail::cpu::copy_d2d; | |
| 1151 | 165 | ops.copy_d2d_strided = &detail::cpu::copy_d2d_strided; | |
| 1152 | |||
| 1153 | // ── CHUNK 1 ── | ||
| 1154 | 165 | ops.clamp = &detail::cpu::clamp; | |
| 1155 | 165 | ops.mul_inplace = &detail::cpu::mul_inplace; | |
| 1156 | 165 | ops.axpby_inplace = &detail::cpu::axpby_inplace; | |
| 1157 | 165 | ops.threshold_u8 = &detail::cpu::threshold_u8; | |
| 1158 | 165 | ops.cast = &detail::cpu::cast; | |
| 1159 | 165 | ops.mse_vec_forward = &detail::cpu::mse_vec_forward; | |
| 1160 | 165 | ops.mse_vec_backward = &detail::cpu::mse_vec_backward; | |
| 1161 | 165 | ops.softmax_xent_fused = &detail::cpu::softmax_xent_fused; | |
| 1162 | 165 | ops.embedding_lookup_forward = &detail::cpu::embedding_lookup_forward; | |
| 1163 | 165 | ops.embedding_lookup_backward = &detail::cpu::embedding_lookup_backward; | |
| 1164 | 165 | ops.concat_batched_rows = &detail::cpu::concat_batched_rows; | |
| 1165 | 165 | ops.concat_nchw_channels = &detail::cpu::concat_nchw_channels; | |
| 1166 | 165 | ops.concat_nchw_channels_backward | |
| 1167 | 165 | = &detail::cpu::concat_nchw_channels_backward; | |
| 1168 | 165 | ops.sum_rows = &detail::cpu::sum_rows; | |
| 1169 | 165 | ops.sum_cols = &detail::cpu::sum_cols; | |
| 1170 | 165 | ops.argmax_rows = &detail::cpu::argmax_rows; | |
| 1171 | 165 | ops.rows_count_above = &detail::cpu::rows_count_above; | |
| 1172 | 165 | ops.layernorm_forward_inference_batched | |
| 1173 | 165 | = &detail::cpu::layernorm_forward_inference_batched; | |
| 1174 | 165 | ops.layernorm_forward_batched_with_caches | |
| 1175 | 165 | = &detail::cpu::layernorm_forward_batched_with_caches; | |
| 1176 | 165 | ops.layernorm_backward_batched_with_caches | |
| 1177 | 165 | = &detail::cpu::layernorm_backward_batched_with_caches; | |
| 1178 | 165 | ops.build_causal_mask_row = &detail::cpu::build_causal_mask_row; | |
| 1179 | |||
| 1180 | // ── CHUNK 2 ── | ||
| 1181 | 165 | ops.silu_forward = &detail::cpu::silu_forward; | |
| 1182 | 165 | ops.silu_backward = &detail::cpu::silu_backward; | |
| 1183 | 165 | ops.gelu_forward = &detail::cpu::gelu_forward; | |
| 1184 | 165 | ops.gelu_backward = &detail::cpu::gelu_backward; | |
| 1185 | 165 | ops.gelu_exact_forward = &detail::cpu::gelu_exact_forward; | |
| 1186 | 165 | ops.gelu_exact_backward = &detail::cpu::gelu_exact_backward; | |
| 1187 | 165 | ops.quick_gelu_forward = &detail::cpu::quick_gelu_forward; | |
| 1188 | 165 | ops.quick_gelu_backward = &detail::cpu::quick_gelu_backward; | |
| 1189 | 165 | ops.geglu_forward = &detail::cpu::geglu_forward; | |
| 1190 | 165 | ops.geglu_backward = &detail::cpu::geglu_backward; | |
| 1191 | 165 | ops.geglu_exact_forward = &detail::cpu::geglu_exact_forward; | |
| 1192 | 165 | ops.geglu_exact_backward = &detail::cpu::geglu_exact_backward; | |
| 1193 | 165 | ops.swiglu_forward = &detail::cpu::swiglu_forward; | |
| 1194 | 165 | ops.swiglu_backward = &detail::cpu::swiglu_backward; | |
| 1195 | 165 | ops.matmul = &detail::cpu::matmul; | |
| 1196 | 165 | ops.matmul_abt = &detail::cpu::matmul_abt; | |
| 1197 | 165 | ops.matmul_backward = &detail::cpu::matmul_backward; | |
| 1198 | 165 | ops.lstm_forward_train = &detail::cpu::lstm_forward_train; | |
| 1199 | 165 | ops.lstm_backward = &detail::cpu::lstm_backward; | |
| 1200 | 165 | ops.rope_forward = &detail::cpu::rope_forward; | |
| 1201 | 165 | ops.rope_backward = &detail::cpu::rope_backward; | |
| 1202 | 165 | ops.rms_norm_forward = &detail::cpu::rms_norm_forward; | |
| 1203 | 165 | ops.rms_norm_backward = &detail::cpu::rms_norm_backward; | |
| 1204 | |||
| 1205 | // ── CHUNK 3 ── | ||
| 1206 | 165 | ops.conv2d_forward = &detail::cpu::conv2d_forward; | |
| 1207 | 165 | ops.conv2d_backward_input = &detail::cpu::conv2d_backward_input; | |
| 1208 | 165 | ops.conv2d_backward_weight = &detail::cpu::conv2d_backward_weight; | |
| 1209 | 165 | ops.conv2d_backward_bias = &detail::cpu::conv2d_backward_bias; | |
| 1210 | 165 | ops.deform_conv2d_forward = &detail::cpu::deform_conv2d_forward; | |
| 1211 | 165 | ops.conv3d_forward = &detail::cpu::conv3d_forward; | |
| 1212 | // conv3d_int8w_fp16_forward is GPU-only; CPU slot left null. | ||
| 1213 | 165 | ops.group_norm_forward = &detail::cpu::group_norm_forward; | |
| 1214 | 165 | ops.group_norm_backward = &detail::cpu::group_norm_backward; | |
| 1215 | |||
| 1216 | // ── CHUNK 4 ── | ||
| 1217 | 165 | ops.upsample_nearest_2x = &detail::cpu::upsample_nearest_2x; | |
| 1218 | 165 | ops.upsample_bilinear_2x = &detail::cpu::upsample_bilinear_2x; | |
| 1219 | 165 | ops.downsample_avg_2x = &detail::cpu::downsample_avg_2x; | |
| 1220 | 165 | ops.upsample_nearest_2x_backward = &detail::cpu::upsample_nearest_2x_backward; | |
| 1221 | 165 | ops.upsample_bilinear_2x_backward | |
| 1222 | 165 | = &detail::cpu::upsample_bilinear_2x_backward; | |
| 1223 | 165 | ops.downsample_avg_2x_backward = &detail::cpu::downsample_avg_2x_backward; | |
| 1224 | 165 | ops.interp2d_forward = &detail::cpu::interp2d_forward; | |
| 1225 | 165 | ops.interp2d_backward = &detail::cpu::interp2d_backward; | |
| 1226 | 165 | ops.interp2d_align_corners_forward = &detail::cpu::interp2d_align_corners_forward; | |
| 1227 | 165 | ops.pad2d_forward = &detail::cpu::pad2d_forward; | |
| 1228 | 165 | ops.pad2d_backward = &detail::cpu::pad2d_backward; | |
| 1229 | 165 | ops.slice2d_forward = &detail::cpu::slice2d_forward; | |
| 1230 | 165 | ops.slice2d_backward = &detail::cpu::slice2d_backward; | |
| 1231 | 165 | ops.unfold2d_forward = &detail::cpu::unfold2d_forward; | |
| 1232 | 165 | ops.l2_normalize_nchw_forward = &detail::cpu::l2_normalize_nchw_forward; | |
| 1233 | 165 | ops.convex_upsample_forward = &detail::cpu::convex_upsample_forward; | |
| 1234 | 165 | ops.top_k_rows = &detail::cpu::top_k_rows; | |
| 1235 | 165 | ops.adaptive_avg_pool2d_forward = &detail::cpu::adaptive_avg_pool2d_forward; | |
| 1236 | 165 | ops.adaptive_avg_pool2d_backward = &detail::cpu::adaptive_avg_pool2d_backward; | |
| 1237 | 165 | ops.max_pool2d_forward = &detail::cpu::max_pool2d_forward; | |
| 1238 | 165 | ops.max_pool2d_backward = &detail::cpu::max_pool2d_backward; | |
| 1239 | 165 | ops.gather_rows = &detail::cpu::gather_rows; | |
| 1240 | 165 | ops.scatter_rows_add = &detail::cpu::scatter_rows_add; | |
| 1241 | 165 | ops.scatter_rows = &detail::cpu::scatter_rows; | |
| 1242 | 165 | ops.conv_transpose2d_forward = &detail::cpu::conv_transpose2d_forward; | |
| 1243 | 165 | ops.conv_transpose2d_backward_input | |
| 1244 | 165 | = &detail::cpu::conv_transpose2d_backward_input; | |
| 1245 | 165 | ops.conv_transpose2d_backward_weight | |
| 1246 | 165 | = &detail::cpu::conv_transpose2d_backward_weight; | |
| 1247 | 165 | ops.conv_transpose2d_backward_bias | |
| 1248 | 165 | = &detail::cpu::conv_transpose2d_backward_bias; | |
| 1249 | 165 | ops.window_partition_forward = &detail::cpu::window_partition_forward; | |
| 1250 | 165 | ops.window_reverse_forward = &detail::cpu::window_reverse_forward; | |
| 1251 | 165 | ops.nchw_to_sequence = &detail::cpu::nchw_to_sequence; | |
| 1252 | 165 | ops.sequence_to_nchw = &detail::cpu::sequence_to_nchw; | |
| 1253 | 165 | ops.ddim_step = &detail::cpu::ddim_step; | |
| 1254 | 165 | ops.euler_step = &detail::cpu::euler_step; | |
| 1255 | 165 | ops.dpmpp_2m_step = &detail::cpu::dpmpp_2m_step; | |
| 1256 | 165 | ops.timestep_embedding = &detail::cpu::timestep_embedding; | |
| 1257 | 165 | ops.kv_cache_append = &detail::cpu::kv_cache_append; | |
| 1258 | 165 | ops.flash_attention_decode = &detail::cpu::flash_attention_decode; | |
| 1259 | 165 | ops.flash_attention_decode_masked | |
| 1260 | 165 | = &detail::cpu::flash_attention_decode_masked; | |
| 1261 | |||
| 1262 | // ── CHUNK 5 ── | ||
| 1263 | 165 | ops.cross_attention_forward = &detail::cpu::cross_attention_forward; | |
| 1264 | 165 | ops.cross_attention_forward_with_attn | |
| 1265 | 165 | = &detail::cpu::cross_attention_forward_with_attn; | |
| 1266 | 165 | ops.self_attention_forward_train = &detail::cpu::self_attention_forward_train; | |
| 1267 | 165 | ops.self_attention_backward = &detail::cpu::self_attention_backward; | |
| 1268 | 165 | ops.attention_token_moments = &detail::cpu::attention_token_moments; | |
| 1269 | 165 | ops.cross_attention_forward_train | |
| 1270 | 165 | = &detail::cpu::cross_attention_forward_train; | |
| 1271 | 165 | ops.cross_attention_backward = &detail::cpu::cross_attention_backward; | |
| 1272 | |||
| 1273 | // ── CHUNK 6 ── | ||
| 1274 | 165 | ops.flash_attention_forward = &detail::cpu::flash_attention_forward; | |
| 1275 | 165 | ops.flash_attention_gqa_forward = &detail::cpu::flash_attention_gqa_forward; | |
| 1276 | 165 | ops.flash_attention_windowed_forward | |
| 1277 | 165 | = &detail::cpu::flash_attention_windowed_forward; | |
| 1278 | 165 | ops.flash_attention_varlen_forward | |
| 1279 | 165 | = &detail::cpu::flash_attention_varlen_forward; | |
| 1280 | 165 | ops.flash_attention_varlen_backward | |
| 1281 | 165 | = &detail::cpu::flash_attention_varlen_backward; | |
| 1282 | 165 | ops.flash_attention_qkvo_forward = &detail::cpu::flash_attention_qkvo_forward; | |
| 1283 | 165 | ops.flash_attention_qkvo_backward | |
| 1284 | 165 | = &detail::cpu::flash_attention_qkvo_backward; | |
| 1285 | 165 | ops.flash_attention_backward = &detail::cpu::flash_attention_backward; | |
| 1286 | 165 | ops.flash_attention_project_kv = &detail::cpu::flash_attention_project_kv; | |
| 1287 | 165 | ops.flash_attention_q_with_kv_cached_forward | |
| 1288 | 165 | = &detail::cpu::flash_attention_q_with_kv_cached_forward; | |
| 1289 | 165 | ops.self_attention_forward = &detail::cpu::self_attention_forward; | |
| 1290 | 165 | ops.resblock_forward = &detail::cpu::resblock_forward; | |
| 1291 | 165 | ops.resblock_backward = &detail::cpu::resblock_backward; | |
| 1292 | |||
| 1293 | // ── DiT / diffusion extras ── | ||
| 1294 | 165 | ops.modulate = &detail::cpu::modulate; | |
| 1295 | 165 | ops.broadcast_mul = &detail::cpu::broadcast_mul; | |
| 1296 | 165 | ops.rope_apply = &detail::cpu::rope_apply; | |
| 1297 | 165 | ops.rope_apply_perhead = &detail::cpu::rope_apply_perhead; | |
| 1298 | 165 | ops.rope_apply_backward = &detail::cpu::rope_apply_backward; | |
| 1299 | 165 | ops.self_attention_bias_forward = &detail::cpu::self_attention_bias_forward; | |
| 1300 | 165 | ops.rel_pos_bias_xl_forward = &detail::cpu::rel_pos_bias_xl_forward; | |
| 1301 | 165 | ops.self_attention_decomposed_rel_pos_forward = | |
| 1302 | &detail::cpu::self_attention_decomposed_rel_pos_forward; | ||
| 1303 | 165 | ops.self_attention_decomposed_rel_pos_windowed_forward = | |
| 1304 | &detail::cpu::self_attention_decomposed_rel_pos_windowed_forward; | ||
| 1305 | |||
| 1306 | // ── Spectral / FFT core (brosoundml) ── | ||
| 1307 | 165 | ops.complex_mul = &detail::cpu::complex_mul; | |
| 1308 | 165 | ops.complex_mul_backward = &detail::cpu::complex_mul_backward; | |
| 1309 | 165 | ops.complex_abs = &detail::cpu::complex_abs; | |
| 1310 | 165 | ops.complex_abs_backward = &detail::cpu::complex_abs_backward; | |
| 1311 | 165 | ops.complex_angle = &detail::cpu::complex_angle; | |
| 1312 | 165 | ops.complex_from_polar = &detail::cpu::complex_from_polar; | |
| 1313 | 165 | ops.fft = &detail::cpu::fft; | |
| 1314 | 165 | ops.ifft = &detail::cpu::ifft; | |
| 1315 | 165 | ops.rfft = &detail::cpu::rfft; | |
| 1316 | 165 | ops.irfft = &detail::cpu::irfft; | |
| 1317 | 165 | ops.rfft_backward = &detail::cpu::rfft_backward; | |
| 1318 | 165 | ops.irfft_backward = &detail::cpu::irfft_backward; | |
| 1319 | |||
| 1320 | // ── STFT / iSTFT (brosoundml) ── | ||
| 1321 | 165 | ops.stft = &detail::cpu::stft; | |
| 1322 | 165 | ops.stft_backward = &detail::cpu::stft_backward; | |
| 1323 | 165 | ops.istft = &detail::cpu::istft; | |
| 1324 | 165 | ops.istft_backward = &detail::cpu::istft_backward; | |
| 1325 | |||
| 1326 | // ── 1D convolution family (brosoundml) ── | ||
| 1327 | 165 | ops.conv_transpose1d_forward = &detail::cpu::conv_transpose1d_forward; | |
| 1328 | 165 | ops.conv_transpose1d_backward_input | |
| 1329 | 165 | = &detail::cpu::conv_transpose1d_backward_input; | |
| 1330 | 165 | ops.conv_transpose1d_backward_weight | |
| 1331 | 165 | = &detail::cpu::conv_transpose1d_backward_weight; | |
| 1332 | 165 | ops.conv_transpose1d_backward_bias | |
| 1333 | 165 | = &detail::cpu::conv_transpose1d_backward_bias; | |
| 1334 | 165 | ops.causal_conv1d_update = &detail::cpu::causal_conv1d_update; | |
| 1335 | 165 | ops.pad1d_forward = &detail::cpu::pad1d_forward; | |
| 1336 | 165 | ops.pad1d_backward = &detail::cpu::pad1d_backward; | |
| 1337 | |||
| 1338 | // ── Vocoder / codec activations (brosoundml) ── | ||
| 1339 | 165 | ops.snake_forward = &detail::cpu::snake_forward; | |
| 1340 | 165 | ops.snake_backward = &detail::cpu::snake_backward; | |
| 1341 | 165 | ops.elu_forward = &detail::cpu::elu_forward; | |
| 1342 | 165 | ops.elu_backward = &detail::cpu::elu_backward; | |
| 1343 | 165 | ops.leaky_relu_forward = &detail::cpu::leaky_relu_forward; | |
| 1344 | 165 | ops.leaky_relu_backward = &detail::cpu::leaky_relu_backward; | |
| 1345 | |||
| 1346 | // ── Codec quantization (brosoundml) ── | ||
| 1347 | 165 | ops.vq_encode_forward = &detail::cpu::vq_encode_forward; | |
| 1348 | 165 | ops.vq_encode_backward = &detail::cpu::vq_encode_backward; | |
| 1349 | 165 | ops.fsq_quantize_forward = &detail::cpu::fsq_quantize_forward; | |
| 1350 | 165 | ops.fsq_quantize_backward = &detail::cpu::fsq_quantize_backward; | |
| 1351 | |||
| 1352 | // ── 1D resampling (brosoundml CHUNK 6, family E) ── | ||
| 1353 | 165 | ops.resample1d_forward = &detail::cpu::resample1d_forward; | |
| 1354 | 165 | ops.resample1d_backward = &detail::cpu::resample1d_backward; | |
| 1355 | |||
| 1356 | // ── log / exp / round elementwise (brosoundml CHUNK 6, family G) ── | ||
| 1357 | 165 | ops.log_forward = &detail::cpu::log_forward; | |
| 1358 | 165 | ops.log_backward = &detail::cpu::log_backward; | |
| 1359 | 165 | ops.exp_forward = &detail::cpu::exp_forward; | |
| 1360 | 165 | ops.exp_backward = &detail::cpu::exp_backward; | |
| 1361 | 165 | ops.round_forward = &detail::cpu::round_forward; | |
| 1362 | 165 | ops.round_backward = &detail::cpu::round_backward; | |
| 1363 | |||
| 1364 | // ── Autoregressive logit sampling (brosoundml CHUNK 7, family F) ── | ||
| 1365 | 165 | ops.sample_logits = &detail::cpu::sample_logits; | |
| 1366 | 165 | ops.sample_logits_into = &detail::cpu::sample_logits_into; | |
| 1367 | |||
| 1368 | // ── Counter-based noise generation (Philox 4x32-10) ── | ||
| 1369 | 165 | ops.randn = &detail::cpu::randn; | |
| 1370 | 165 | ops.rand_uniform = &detail::cpu::rand_uniform; | |
| 1371 | 165 | ops.rand_bernoulli = &detail::cpu::rand_bernoulli; | |
| 1372 | 165 | ops.randn_truncated = &detail::cpu::randn_truncated; | |
| 1373 | |||
| 1374 | // ── L2 norm + Gated Delta Rule (linear-attention text path) ── | ||
| 1375 | 165 | ops.l2_norm_forward = &detail::cpu::l2_norm_forward; | |
| 1376 | 165 | ops.l2_norm_backward = &detail::cpu::l2_norm_backward; | |
| 1377 | 165 | ops.gated_delta_rule_chunked = &detail::cpu::gated_delta_rule_chunked; | |
| 1378 | 165 | ops.gated_delta_rule_step = &detail::cpu::gated_delta_rule_step; | |
| 1379 | |||
| 1380 | // ── Qwen3-VL polish: spatial_merge_2x2 + M-RoPE ── | ||
| 1381 | 165 | ops.spatial_merge_2x2_forward = &detail::cpu::spatial_merge_2x2_forward; | |
| 1382 | 165 | ops.pixel_shuffle_upsample_2x_forward = &detail::cpu::pixel_shuffle_upsample_2x_forward; | |
| 1383 | 165 | ops.patch_unpack_forward = &detail::cpu::patch_unpack_forward; | |
| 1384 | 165 | ops.rope_apply_mrope = &detail::cpu::rope_apply_mrope; | |
| 1385 | |||
| 1386 | // ── BatchNorm (vision backbones) ── | ||
| 1387 | 165 | ops.batch_norm_forward = &detail::cpu::batch_norm_forward; | |
| 1388 | 165 | ops.batch_norm_inference = &detail::cpu::batch_norm_inference; | |
| 1389 | 165 | ops.batch_norm_backward = &detail::cpu::batch_norm_backward; | |
| 1390 | |||
| 1391 | // ── Image preprocessing helpers ── | ||
| 1392 | 165 | ops.image_normalize = &detail::cpu::image_normalize; | |
| 1393 | 165 | ops.image_u8_to_f32_nhwc_to_nchw = &detail::cpu::image_u8_to_f32_nhwc_to_nchw; | |
| 1394 | |||
| 1395 | // ── StyleGAN3-R synthesis-input primitives ── | ||
| 1396 | 165 | ops.sin_forward = &detail::cpu::sin_forward; | |
| 1397 | 165 | ops.sin_backward = &detail::cpu::sin_backward; | |
| 1398 | 165 | ops.cos_forward = &detail::cpu::cos_forward; | |
| 1399 | 165 | ops.cos_backward = &detail::cpu::cos_backward; | |
| 1400 | 165 | ops.rsqrt_forward = &detail::cpu::rsqrt_forward; | |
| 1401 | 165 | ops.rsqrt_backward = &detail::cpu::rsqrt_backward; | |
| 1402 | 165 | ops.pixel_norm_forward = &detail::cpu::pixel_norm_forward; | |
| 1403 | 165 | ops.pixel_norm_backward = &detail::cpu::pixel_norm_backward; | |
| 1404 | 165 | ops.bias_act_forward = &detail::cpu::bias_act_forward; | |
| 1405 | 165 | ops.bias_act_backward = &detail::cpu::bias_act_backward; | |
| 1406 | 165 | ops.upfirdn2d_forward = &detail::cpu::upfirdn2d_forward; | |
| 1407 | 165 | ops.upfirdn2d_backward = &detail::cpu::upfirdn2d_backward; | |
| 1408 | 165 | ops.modulated_conv2d_forward = &detail::cpu::modulated_conv2d_forward; | |
| 1409 | 165 | ops.modulated_conv2d_backward = &detail::cpu::modulated_conv2d_backward; | |
| 1410 | |||
| 1411 | 330 | detail::register_backend(Device::CPU, ops, | |
| 1412 | 165 | detail::cpu::cpu_alloc_table()); | |
| 1413 | 330 | } | |
| 1414 | }; | ||
| 1415 | |||
| 1416 | 165 | static CpuStaticRegistrar g_cpu_registrar{}; | |
| 1417 | |||
| 1418 | } // anonymous namespace | ||
| 1419 |