GCC Code Coverage Report

Directory: brotensor/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 49.9% 207 / 0 / 415
Functions: 76.9% 20 / 0 / 26
Branches: 22.9% 99 / 0 / 432
src/cpu/cpu_jit_quant.cpp
Line Branch Exec Source
1 #include "cpu_jit.h"
2 #include <brotensor/detail/cpu/thread_pool.h>
3 #include <brotensor/tensor.h>
4 #include <brotensor/ops.h>
5
6 #include <algorithm>
7 #include <cmath>
8 #include <cstdint>
9 #include <cstring>
10 #include <iostream>
11 #include <memory>
12 #include <mutex>
13 #include <stdexcept>
14 #include <vector>
15
16 namespace brotensor::detail::cpu::jit {
17 namespace {
18
19 ✗ static inline void ref_dequant_q8_0_block(const void* blk_ptr, float* out32) {
20 uint16_t d_raw;
21 ✗ std::memcpy(&d_raw, blk_ptr, 2);
22 ✗ const float d = fp16_bits_to_fp32(d_raw);
23 ✗ const int8_t* qs = reinterpret_cast<const int8_t*>(static_cast<const char*>(blk_ptr) + 2);
24 ✗ for (int i = 0; i < 32; ++i) {
25 ✗ out32[i] = static_cast<float>(qs[i]) * d;
26 ✗ }
27 ✗ }
28
29 ✗ static inline void ref_dequant_q4k_block(const void* blk_ptr, float* out256) {
30 ✗ const uint8_t* p = static_cast<const uint8_t*>(blk_ptr);
31 uint16_t d_raw, dmin_raw;
32 ✗ std::memcpy(&d_raw, p, 2);
33 ✗ std::memcpy(&dmin_raw, p + 2, 2);
34 ✗ const float d = fp16_bits_to_fp32(d_raw);
35 ✗ const float dmin = fp16_bits_to_fp32(dmin_raw);
36
37 ✗ const uint8_t* scales = p + 4;
38 ✗ const uint8_t* qs = p + 16;
39
40 uint8_t sc[8], m[8];
41 ✗ for (int j = 0; j < 8; ++j) {
42 ✗ if (j < 4) {
43 ✗ sc[j] = scales[j] & 0x3F;
44 ✗ m[j] = scales[j + 4] & 0x3F;
45 ✗ } else {
46 ✗ sc[j] = static_cast<uint8_t>((scales[j + 4] & 0x0F) | ((scales[j - 4] >> 6) << 4));
47 ✗ m[j] = static_cast<uint8_t>((scales[j + 4] >> 4) | ((scales[j - 0] >> 6) << 4));
48 }
49 ✗ }
50
51 ✗ for (int p_idx = 0; p_idx < 4; ++p_idx) {
52 ✗ const int is_lo = 2 * p_idx;
53 ✗ const int is_hi = 2 * p_idx + 1;
54 ✗ const float w_lo = static_cast<float>(sc[is_lo]) * d;
55 ✗ const float w_hi = static_cast<float>(sc[is_hi]) * d;
56 ✗ const float b_lo = static_cast<float>(m [is_lo]) * dmin;
57 ✗ const float b_hi = static_cast<float>(m [is_hi]) * dmin;
58
59 ✗ for (int l = 0; l < 32; ++l) {
60 ✗ const uint8_t byte = qs[p_idx * 32 + l];
61 ✗ const int n_lo = byte & 0x0F;
62 ✗ const int n_hi = (byte >> 4) & 0x0F;
63 ✗ out256[is_lo * 32 + l] = w_lo * static_cast<float>(n_lo) - b_lo;
64 ✗ out256[is_hi * 32 + l] = w_hi * static_cast<float>(n_hi) - b_hi;
65 ✗ }
66 ✗ }
67 ✗ }
68
69 ✗ static void ref_gemv_q8_0(const void* W, const float* X, float* Y, int N, int K) {
70 ✗ const int bpr = K / 32;
71 ✗ const char* W_bytes = static_cast<const char*>(W);
72 float blk_buf[32];
73 ✗ for (int r = 0; r < N; ++r) {
74 ✗ double acc = 0.0;
75 ✗ const char* row_ptr = W_bytes + static_cast<size_t>(r) * bpr * 34;
76 ✗ for (int b = 0; b < bpr; ++b) {
77 ✗ ref_dequant_q8_0_block(row_ptr + b * 34, blk_buf);
78 ✗ const float* x_sub = X + b * 32;
79 ✗ for (int i = 0; i < 32; ++i) {
80 ✗ acc += static_cast<double>(blk_buf[i]) * static_cast<double>(x_sub[i]);
81 ✗ }
82 ✗ }
83 ✗ Y[r] = static_cast<float>(acc);
84 ✗ }
85 ✗ }
86
87 ✗ static void ref_gemv_q4k(const void* W, const float* X, float* Y, int N, int K) {
88 ✗ const int bpr = K / 256;
89 ✗ const char* W_bytes = static_cast<const char*>(W);
90 float blk_buf[256];
91 ✗ for (int r = 0; r < N; ++r) {
92 ✗ double acc = 0.0;
93 ✗ const char* row_ptr = W_bytes + static_cast<size_t>(r) * bpr * 144;
94 ✗ for (int b = 0; b < bpr; ++b) {
95 ✗ ref_dequant_q4k_block(row_ptr + b * 144, blk_buf);
96 ✗ const float* x_sub = X + b * 256;
97 ✗ for (int i = 0; i < 256; ++i) {
98 ✗ acc += static_cast<double>(blk_buf[i]) * static_cast<double>(x_sub[i]);
99 ✗ }
100 ✗ }
101 ✗ Y[r] = static_cast<float>(acc);
102 ✗ }
103 ✗ }
104
105 } // namespace
106 } // namespace brotensor::detail::cpu::jit
107
108 #if BROTENSOR_HAS_BRASS_JIT
109
110 #include <brass/codegen/ml_fusion.hpp>
111 #include <brass/codegen/kernel_jit.hpp>
112
113 namespace brotensor::detail::cpu::jit {
114
115 namespace {
116
117 using namespace brass::codegen;
118
119 1 struct QuantJitKernels {
120 1 bool available = false;
121 1 MlFusionCompiler::GemvQ8_0Fn gemv_q8_0_fn = nullptr;
122 1 MlFusionCompiler::GemvQ4_KFn gemv_q4_k_fn = nullptr;
123 std::unique_ptr<MlFusionCompiler> compiler;
124 KernelFunction kfn_q8_0;
125 KernelFunction kfn_q4_k;
126 };
127
128 256 static const QuantJitKernels& get_quant_kernels() {
129
4/6
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 255 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 time.
✓ Branch 4 taken 1 time.
✗ Branch 5 not taken.
 257 static QuantJitKernels kernels = []() {
130 1 QuantJitKernels k;
131 try {
132
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 k.compiler = std::make_unique<MlFusionCompiler>();
133
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 k.kfn_q8_0 = k.compiler->compile_gemv_q8_0();
134
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 k.kfn_q4_k = k.compiler->compile_gemv_q4_k();
135
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 if (k.kfn_q8_0.is_valid()) {
136 1 k.gemv_q8_0_fn = k.kfn_q8_0.as<MlFusionCompiler::GemvQ8_0Fn>();
137 1 }
138
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 if (k.kfn_q4_k.is_valid()) {
139 1 k.gemv_q4_k_fn = k.kfn_q4_k.as<MlFusionCompiler::GemvQ4_KFn>();
140 1 }
141
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 k.available = (k.gemv_q8_0_fn != nullptr && k.gemv_q4_k_fn != nullptr);
142
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 1 } catch (const std::exception& e) {
143 ✗ std::cerr << "brotensor: Brass JIT quant kernels initialization failed: " << e.what() << "\n";
144 ✗ k.available = false;
145 ✗ }
146 1 return k;
147
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 }();
148 256 return kernels;
149 ✗ }
150
151 } // namespace
152
153 128 void gemv_q8_0(const void* W, const float* X, float* Y, int N, int K) {
154
2/4
✓ Branch 0 taken 128 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 128 times.
 128 if (N <= 0 || K <= 0) return;
155 128 const auto& k = get_quant_kernels();
156
2/4
✓ Branch 0 taken 128 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 128 times.
 128 if (k.available && k.gemv_q8_0_fn != nullptr) {
157
2/2
✓ Branch 0 taken 96 times.
✓ Branch 1 taken 32 times.
 128 if (N >= 8) {
158 96 const size_t row_bytes = static_cast<size_t>(K / 32) * 34;
159 96 int n_threads = std::min<int>(detail::cpu::ThreadPool::instance().num_threads(), N);
160
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 96 times.
 96 if (n_threads <= 1) {
161 ✗ k.gemv_q8_0_fn(W, X, Y, static_cast<uint64_t>(N), static_cast<uint64_t>(K));
162 ✗ } else {
163 96 int chunk_size = (N + n_threads - 1) / n_threads;
164 96 int num_chunks = (N + chunk_size - 1) / chunk_size;
165
1/2
✓ Branch 0 taken 96 times.
✗ Branch 1 not taken.
 379 detail::cpu::parallel_for(static_cast<std::size_t>(num_chunks), [&](std::size_t c) {
166 283 int r_start = static_cast<int>(c) * chunk_size;
167 283 int r_end = std::min(r_start + chunk_size, N);
168
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 283 times.
 283 if (r_start >= r_end) return;
169 283 const char* W_bytes = static_cast<const char*>(W);
170 283 const void* W_slice = W_bytes + static_cast<size_t>(r_start) * row_bytes;
171 283 float* Y_slice = Y + r_start;
172 283 k.gemv_q8_0_fn(W_slice, X, Y_slice, static_cast<uint64_t>(r_end - r_start), static_cast<uint64_t>(K));
173 283 });
174 }
175 96 } else {
176 32 k.gemv_q8_0_fn(W, X, Y, static_cast<uint64_t>(N), static_cast<uint64_t>(K));
177 }
178 128 return;
179 }
180 ✗ ref_gemv_q8_0(W, X, Y, N, K);
181 128 }
182
183 128 void gemv_q4_k(const void* W, const float* X, float* Y, int N, int K) {
184
2/4
✓ Branch 0 taken 128 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 128 times.
 128 if (N <= 0 || K <= 0) return;
185 128 const auto& k = get_quant_kernels();
186
2/4
✓ Branch 0 taken 128 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 128 times.
 128 if (k.available && k.gemv_q4_k_fn != nullptr) {
187
2/2
✓ Branch 0 taken 96 times.
✓ Branch 1 taken 32 times.
 128 if (N >= 8) {
188 96 const size_t row_bytes = static_cast<size_t>(K / 256) * 144;
189 96 int n_threads = std::min<int>(detail::cpu::ThreadPool::instance().num_threads(), N);
190
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 96 times.
 96 if (n_threads <= 1) {
191 ✗ k.gemv_q4_k_fn(W, X, Y, static_cast<uint64_t>(N), static_cast<uint64_t>(K));
192 ✗ } else {
193 96 int chunk_size = (N + n_threads - 1) / n_threads;
194 96 int num_chunks = (N + chunk_size - 1) / chunk_size;
195
1/2
✓ Branch 0 taken 96 times.
✗ Branch 1 not taken.
 362 detail::cpu::parallel_for(static_cast<std::size_t>(num_chunks), [&](std::size_t c) {
196 266 int r_start = static_cast<int>(c) * chunk_size;
197 266 int r_end = std::min(r_start + chunk_size, N);
198
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 266 times.
 266 if (r_start >= r_end) return;
199 266 const char* W_bytes = static_cast<const char*>(W);
200 266 const void* W_slice = W_bytes + static_cast<size_t>(r_start) * row_bytes;
201 266 float* Y_slice = Y + r_start;
202 266 k.gemv_q4_k_fn(W_slice, X, Y_slice, static_cast<uint64_t>(r_end - r_start), static_cast<uint64_t>(K));
203 266 });
204 }
205 96 } else {
206 32 k.gemv_q4_k_fn(W, X, Y, static_cast<uint64_t>(N), static_cast<uint64_t>(K));
207 }
208 128 return;
209 }
210 ✗ ref_gemv_q4k(W, X, Y, N, K);
211 128 }
212
213 } // namespace brotensor::detail::cpu::jit
214
215 #else // !BROTENSOR_HAS_BRASS_JIT
216
217 namespace brotensor::detail::cpu::jit {
218
219 void gemv_q8_0(const void* W, const float* X, float* Y, int N, int K) {
220 if (N <= 0 || K <= 0) return;
221 ref_gemv_q8_0(W, X, Y, N, K);
222 }
223
224 void gemv_q4_k(const void* W, const float* X, float* Y, int N, int K) {
225 if (N <= 0 || K <= 0) return;
226 ref_gemv_q4k(W, X, Y, N, K);
227 }
228
229 } // namespace brotensor::detail::cpu::jit
230
231 #endif // BROTENSOR_HAS_BRASS_JIT
232
233 // ─── OpsVTable Registrations for CPU Quantization Ops ───────────────────────
234
235 namespace brotensor::detail::cpu {
236
237 namespace {
238
239 5 static void validate_w_q8_0(const Tensor& W, const char* op) {
240
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (W.dtype != Dtype::Q8_0) {
241 ✗ throw std::runtime_error(std::string("brotensor: ") + op + ": W must be Dtype::Q8_0");
242 }
243
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (W.cols % 32 != 0) {
244 ✗ throw std::runtime_error(std::string("brotensor: ") + op + ": W cols must be a multiple of 32");
245 }
246 5 }
247
248 5 static void validate_w_q4k(const Tensor& W, const char* op) {
249
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (W.dtype != Dtype::Q4_K) {
250 ✗ throw std::runtime_error(std::string("brotensor: ") + op + ": W must be Dtype::Q4_K");
251 }
252
1/2
✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
 5 if (W.cols % 256 != 0) {
253 ✗ throw std::runtime_error(std::string("brotensor: ") + op + ": W cols must be a multiple of 256");
254 }
255 5 }
256
257 128 static inline void local_dequant_q8_0_block(const void* blk_ptr, float* out32) {
258 uint16_t d_raw;
259 128 std::memcpy(&d_raw, blk_ptr, 2);
260 128 const float d = fp16_bits_to_fp32(d_raw);
261 128 const int8_t* qs = reinterpret_cast<const int8_t*>(static_cast<const char*>(blk_ptr) + 2);
262
2/2
✓ Branch 0 taken 3885 times.
✓ Branch 1 taken 128 times.
 4013 for (int i = 0; i < 32; ++i) {
263 3885 out32[i] = static_cast<float>(qs[i]) * d;
264 3885 }
265 128 }
266
267 16 static inline void local_dequant_q4k_block(const void* blk_ptr, float* out256) {
268 16 const uint8_t* p = static_cast<const uint8_t*>(blk_ptr);
269 uint16_t d_raw, dmin_raw;
270 16 std::memcpy(&d_raw, p, 2);
271 16 std::memcpy(&dmin_raw, p + 2, 2);
272 16 const float d = fp16_bits_to_fp32(d_raw);
273 16 const float dmin = fp16_bits_to_fp32(dmin_raw);
274
275 16 const uint8_t* scales = p + 4;
276 16 const uint8_t* qs = p + 16;
277
278 uint8_t sc[8], m[8];
279
2/2
✓ Branch 0 taken 128 times.
✓ Branch 1 taken 16 times.
 144 for (int j = 0; j < 8; ++j) {
280
2/2
✓ Branch 0 taken 64 times.
✓ Branch 1 taken 64 times.
 128 if (j < 4) {
281 64 sc[j] = scales[j] & 0x3F;
282 64 m[j] = scales[j + 4] & 0x3F;
283 64 } else {
284 64 sc[j] = static_cast<uint8_t>((scales[j + 4] & 0x0F) | ((scales[j - 4] >> 6) << 4));
285 64 m[j] = static_cast<uint8_t>((scales[j + 4] >> 4) | ((scales[j - 0] >> 6) << 4));
286 }
287 128 }
288
289
2/2
✓ Branch 0 taken 60 times.
✓ Branch 1 taken 16 times.
 76 for (int p_idx = 0; p_idx < 4; ++p_idx) {
290 60 const int is_lo = 2 * p_idx;
291 60 const int is_hi = 2 * p_idx + 1;
292 60 const float w_lo = static_cast<float>(sc[is_lo]) * d;
293 60 const float w_hi = static_cast<float>(sc[is_hi]) * d;
294 60 const float b_lo = static_cast<float>(m [is_lo]) * dmin;
295 60 const float b_hi = static_cast<float>(m [is_hi]) * dmin;
296
297
2/2
✓ Branch 0 taken 1834 times.
✓ Branch 1 taken 60 times.
 1894 for (int l = 0; l < 32; ++l) {
298 1834 const uint8_t byte = qs[p_idx * 32 + l];
299 1834 const int n_lo = byte & 0x0F;
300 1834 const int n_hi = (byte >> 4) & 0x0F;
301 1834 out256[is_lo * 32 + l] = w_lo * static_cast<float>(n_lo) - b_lo;
302 1834 out256[is_hi * 32 + l] = w_hi * static_cast<float>(n_hi) - b_hi;
303 1834 }
304 60 }
305 16 }
306
307 } // namespace
308
309 1 void dequant_q8_0_to_fp16(const Tensor& W_q8, Tensor& W_fp16) {
310 1 validate_w_q8_0(W_q8, "dequant_q8_0_to_fp16");
311 1 const int out = W_q8.rows;
312 1 const int K = W_q8.cols;
313
3/6
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 1 time.
 1 if (W_fp16.rows != out || W_fp16.cols != K || W_fp16.dtype != Dtype::FP16) {
314 ✗ W_fp16.resize(out, K, Dtype::FP16);
315 ✗ }
316
2/4
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 time.
 1 if (out == 0 || K == 0) return;
317 1 const int bpr = K / 32;
318 1 const char* src = static_cast<const char*>(W_q8.data);
319 1 uint16_t* dst = static_cast<uint16_t*>(W_fp16.data);
320
321 9 auto dequant_row = [&](int r) {
322 float buf[32];
323 8 const char* rsrc = src + static_cast<size_t>(r) * bpr * 34;
324 8 uint16_t* rdst = dst + static_cast<size_t>(r) * K;
325
2/2
✓ Branch 0 taken 126 times.
✓ Branch 1 taken 8 times.
 134 for (int b = 0; b < bpr; ++b) {
326 126 local_dequant_q8_0_block(rsrc + b * 34, buf);
327
2/2
✓ Branch 0 taken 3032 times.
✓ Branch 1 taken 126 times.
 3158 for (int i = 0; i < 32; ++i) {
328 3032 rdst[b * 32 + i] = fp32_to_fp16_bits(buf[i]);
329 3032 }
330 126 }
331 8 };
332
333
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 if (out >= 8) {
334
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 9 detail::cpu::parallel_for(static_cast<std::size_t>(out), [&](std::size_t r) {
335 8 dequant_row(static_cast<int>(r));
336 8 });
337 1 } else {
338 ✗ for (int r = 0; r < out; ++r) dequant_row(r);
339 }
340 1 }
341
342 1 void dequant_q4k_to_fp16(const Tensor& W_q4k, Tensor& W_fp16) {
343 1 validate_w_q4k(W_q4k, "dequant_q4k_to_fp16");
344 1 const int out = W_q4k.rows;
345 1 const int K = W_q4k.cols;
346
3/6
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 1 time.
 1 if (W_fp16.rows != out || W_fp16.cols != K || W_fp16.dtype != Dtype::FP16) {
347 ✗ W_fp16.resize(out, K, Dtype::FP16);
348 ✗ }
349
2/4
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 time.
 1 if (out == 0 || K == 0) return;
350 1 const int bpr = K / 256;
351 1 const char* src = static_cast<const char*>(W_q4k.data);
352 1 uint16_t* dst = static_cast<uint16_t*>(W_fp16.data);
353
354 9 auto dequant_row = [&](int r) {
355 float buf[256];
356 8 const char* rsrc = src + static_cast<size_t>(r) * bpr * 144;
357 8 uint16_t* rdst = dst + static_cast<size_t>(r) * K;
358
2/2
✓ Branch 0 taken 16 times.
✓ Branch 1 taken 8 times.
 24 for (int b = 0; b < bpr; ++b) {
359 16 local_dequant_q4k_block(rsrc + b * 144, buf);
360
2/2
✓ Branch 0 taken 2730 times.
✓ Branch 1 taken 16 times.
 2746 for (int i = 0; i < 256; ++i) {
361 2730 rdst[b * 256 + i] = fp32_to_fp16_bits(buf[i]);
362 2730 }
363 16 }
364 8 };
365
366
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 if (out >= 8) {
367
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 9 detail::cpu::parallel_for(static_cast<std::size_t>(out), [&](std::size_t r) {
368 8 dequant_row(static_cast<int>(r));
369 8 });
370 1 } else {
371 ✗ for (int r = 0; r < out; ++r) dequant_row(r);
372 }
373 1 }
374
375 4 void linear_forward_q8_0_fp16(const Tensor& W_q8, const Tensor* bias,
376 const Tensor& x, Tensor& y) {
377 4 validate_w_q8_0(W_q8, "linear_forward_q8_0_fp16");
378 4 const int out = W_q8.rows;
379 4 const int K = W_q8.cols;
380
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (x.size() != K) {
381 ✗ throw std::runtime_error("brotensor: linear_forward_q8_0_fp16: x size mismatch");
382 }
383
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 4 if (bias && bias->size() > 0 && bias->size() != out) {
384 ✗ throw std::runtime_error("brotensor: linear_forward_q8_0_fp16: bias size mismatch");
385 }
386
3/8
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 4 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 4 if (y.rows != out || y.cols != 1 || (y.dtype != Dtype::FP16 && y.dtype != Dtype::FP32)) {
387 ✗ y.resize(out, 1, Dtype::FP16);
388 ✗ }
389
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (out == 0) return;
390
391 4 std::vector<float> x_f32(K);
392
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (x.dtype == Dtype::FP16) {
393 4 const uint16_t* xp = static_cast<const uint16_t*>(x.data);
394
3/4
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 16384 times.
✓ Branch 2 taken 16384 times.
✗ Branch 3 not taken.
 16388 for (int i = 0; i < K; ++i) x_f32[i] = fp16_bits_to_fp32(xp[i]);
395
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 4 } else if (x.dtype == Dtype::FP32) {
396 ✗ std::memcpy(x_f32.data(), x.data, K * sizeof(float));
397 ✗ } else {
398 ✗ throw std::runtime_error("brotensor: linear_forward_q8_0_fp16: x must be FP16 or FP32");
399 }
400
401
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 std::vector<float> y_f32(out);
402
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 jit::gemv_q8_0(W_q8.data, x_f32.data(), y_f32.data(), out, K);
403
404
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 4 if (bias && bias->size() > 0) {
405 ✗ if (bias->dtype == Dtype::FP16) {
406 ✗ const uint16_t* bp = static_cast<const uint16_t*>(bias->data);
407 ✗ for (int i = 0; i < out; ++i) y_f32[i] += fp16_bits_to_fp32(bp[i]);
408 ✗ } else if (bias->dtype == Dtype::FP32) {
409 ✗ const float* bp = static_cast<const float*>(bias->data);
410 ✗ for (int i = 0; i < out; ++i) y_f32[i] += bp[i];
411 ✗ }
412 ✗ }
413
414
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (y.dtype == Dtype::FP16) {
415 4 uint16_t* yp = static_cast<uint16_t*>(y.data);
416
3/4
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 169 times.
✓ Branch 2 taken 169 times.
✗ Branch 3 not taken.
 173 for (int i = 0; i < out; ++i) yp[i] = fp32_to_fp16_bits(y_f32[i]);
417 4 } else {
418 ✗ std::memcpy(y.data, y_f32.data(), out * sizeof(float));
419 }
420 4 }
421
422 ✗ void linear_forward_batched_q8_0_fp16(const Tensor& W_q8, const Tensor* bias,
423 const Tensor& X_BD, Tensor& Y_BD) {
424 ✗ validate_w_q8_0(W_q8, "linear_forward_batched_q8_0_fp16");
425 ✗ const int B = X_BD.rows;
426 ✗ const int K = X_BD.cols;
427 ✗ const int out = W_q8.rows;
428 ✗ if (W_q8.cols != K) {
429 ✗ throw std::runtime_error("brotensor: linear_forward_batched_q8_0_fp16: shape mismatch");
430 }
431 ✗ if (bias && bias->size() > 0 && bias->size() != out) {
432 ✗ throw std::runtime_error("brotensor: linear_forward_batched_q8_0_fp16: bias size mismatch");
433 }
434 ✗ if (Y_BD.rows != B || Y_BD.cols != out || (Y_BD.dtype != Dtype::FP16 && Y_BD.dtype != Dtype::FP32)) {
435 ✗ Y_BD.resize(B, out, Dtype::FP16);
436 ✗ }
437 ✗ if (B == 0 || out == 0) return;
438
439 ✗ for (int b = 0; b < B; ++b) {
440 ✗ std::vector<float> x_f32(K);
441 ✗ if (X_BD.dtype == Dtype::FP16) {
442 ✗ const uint16_t* xp = static_cast<const uint16_t*>(X_BD.data) + static_cast<size_t>(b) * K;
443 ✗ for (int i = 0; i < K; ++i) x_f32[i] = fp16_bits_to_fp32(xp[i]);
444 ✗ } else if (X_BD.dtype == Dtype::FP32) {
445 ✗ const float* xp = static_cast<const float*>(X_BD.data) + static_cast<size_t>(b) * K;
446 ✗ std::memcpy(x_f32.data(), xp, K * sizeof(float));
447 ✗ } else {
448 ✗ throw std::runtime_error("brotensor: linear_forward_batched_q8_0_fp16: X must be FP16 or FP32");
449 }
450
451 ✗ std::vector<float> y_f32(out);
452 ✗ jit::gemv_q8_0(W_q8.data, x_f32.data(), y_f32.data(), out, K);
453
454 ✗ if (bias && bias->size() > 0) {
455 ✗ if (bias->dtype == Dtype::FP16) {
456 ✗ const uint16_t* bp = static_cast<const uint16_t*>(bias->data);
457 ✗ for (int i = 0; i < out; ++i) y_f32[i] += fp16_bits_to_fp32(bp[i]);
458 ✗ } else if (bias->dtype == Dtype::FP32) {
459 ✗ const float* bp = static_cast<const float*>(bias->data);
460 ✗ for (int i = 0; i < out; ++i) y_f32[i] += bp[i];
461 ✗ }
462 ✗ }
463
464 ✗ if (Y_BD.dtype == Dtype::FP16) {
465 ✗ uint16_t* yp = static_cast<uint16_t*>(Y_BD.data) + static_cast<size_t>(b) * out;
466 ✗ for (int i = 0; i < out; ++i) yp[i] = fp32_to_fp16_bits(y_f32[i]);
467 ✗ } else {
468 ✗ float* yp = static_cast<float*>(Y_BD.data) + static_cast<size_t>(b) * out;
469 ✗ std::memcpy(yp, y_f32.data(), out * sizeof(float));
470 }
471 ✗ }
472 ✗ }
473
474 4 void linear_forward_q4k_fp16(const Tensor& W_q4k, const Tensor* bias,
475 const Tensor& x, Tensor& y) {
476 4 validate_w_q4k(W_q4k, "linear_forward_q4k_fp16");
477 4 const int out = W_q4k.rows;
478 4 const int K = W_q4k.cols;
479
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (x.size() != K) {
480 ✗ throw std::runtime_error("brotensor: linear_forward_q4k_fp16: x size mismatch");
481 }
482
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 4 if (bias && bias->size() > 0 && bias->size() != out) {
483 ✗ throw std::runtime_error("brotensor: linear_forward_q4k_fp16: bias size mismatch");
484 }
485
3/8
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 4 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 4 if (y.rows != out || y.cols != 1 || (y.dtype != Dtype::FP16 && y.dtype != Dtype::FP32)) {
486 ✗ y.resize(out, 1, Dtype::FP16);
487 ✗ }
488
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (out == 0) return;
489
490 4 std::vector<float> x_f32(K);
491
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (x.dtype == Dtype::FP16) {
492 4 const uint16_t* xp = static_cast<const uint16_t*>(x.data);
493
3/4
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 16384 times.
✓ Branch 2 taken 16384 times.
✗ Branch 3 not taken.
 16388 for (int i = 0; i < K; ++i) x_f32[i] = fp16_bits_to_fp32(xp[i]);
494
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 4 } else if (x.dtype == Dtype::FP32) {
495 ✗ std::memcpy(x_f32.data(), x.data, K * sizeof(float));
496 ✗ } else {
497 ✗ throw std::runtime_error("brotensor: linear_forward_q4k_fp16: x must be FP16 or FP32");
498 }
499
500
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 std::vector<float> y_f32(out);
501
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 jit::gemv_q4_k(W_q4k.data, x_f32.data(), y_f32.data(), out, K);
502
503
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 4 if (bias && bias->size() > 0) {
504 ✗ if (bias->dtype == Dtype::FP16) {
505 ✗ const uint16_t* bp = static_cast<const uint16_t*>(bias->data);
506 ✗ for (int i = 0; i < out; ++i) y_f32[i] += fp16_bits_to_fp32(bp[i]);
507 ✗ } else if (bias->dtype == Dtype::FP32) {
508 ✗ const float* bp = static_cast<const float*>(bias->data);
509 ✗ for (int i = 0; i < out; ++i) y_f32[i] += bp[i];
510 ✗ }
511 ✗ }
512
513
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 if (y.dtype == Dtype::FP16) {
514 4 uint16_t* yp = static_cast<uint16_t*>(y.data);
515
3/4
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 169 times.
✓ Branch 2 taken 169 times.
✗ Branch 3 not taken.
 173 for (int i = 0; i < out; ++i) yp[i] = fp32_to_fp16_bits(y_f32[i]);
516 4 } else {
517 ✗ std::memcpy(y.data, y_f32.data(), out * sizeof(float));
518 }
519 4 }
520
521 ✗ void linear_forward_batched_q4k_fp16(const Tensor& W_q4k, const Tensor* bias,
522 const Tensor& X_BD, Tensor& Y_BD) {
523 ✗ validate_w_q4k(W_q4k, "linear_forward_batched_q4k_fp16");
524 ✗ const int B = X_BD.rows;
525 ✗ const int K = X_BD.cols;
526 ✗ const int out = W_q4k.rows;
527 ✗ if (W_q4k.cols != K) {
528 ✗ throw std::runtime_error("brotensor: linear_forward_batched_q4k_fp16: shape mismatch");
529 }
530 ✗ if (bias && bias->size() > 0 && bias->size() != out) {
531 ✗ throw std::runtime_error("brotensor: linear_forward_batched_q4k_fp16: bias size mismatch");
532 }
533 ✗ if (Y_BD.rows != B || Y_BD.cols != out || (Y_BD.dtype != Dtype::FP16 && Y_BD.dtype != Dtype::FP32)) {
534 ✗ Y_BD.resize(B, out, Dtype::FP16);
535 ✗ }
536 ✗ if (B == 0 || out == 0) return;
537
538 ✗ for (int b = 0; b < B; ++b) {
539 ✗ std::vector<float> x_f32(K);
540 ✗ if (X_BD.dtype == Dtype::FP16) {
541 ✗ const uint16_t* xp = static_cast<const uint16_t*>(X_BD.data) + static_cast<size_t>(b) * K;
542 ✗ for (int i = 0; i < K; ++i) x_f32[i] = fp16_bits_to_fp32(xp[i]);
543 ✗ } else if (X_BD.dtype == Dtype::FP32) {
544 ✗ const float* xp = static_cast<const float*>(X_BD.data) + static_cast<size_t>(b) * K;
545 ✗ std::memcpy(x_f32.data(), xp, K * sizeof(float));
546 ✗ } else {
547 ✗ throw std::runtime_error("brotensor: linear_forward_batched_q4k_fp16: X must be FP16 or FP32");
548 }
549
550 ✗ std::vector<float> y_f32(out);
551 ✗ jit::gemv_q4_k(W_q4k.data, x_f32.data(), y_f32.data(), out, K);
552
553 ✗ if (bias && bias->size() > 0) {
554 ✗ if (bias->dtype == Dtype::FP16) {
555 ✗ const uint16_t* bp = static_cast<const uint16_t*>(bias->data);
556 ✗ for (int i = 0; i < out; ++i) y_f32[i] += fp16_bits_to_fp32(bp[i]);
557 ✗ } else if (bias->dtype == Dtype::FP32) {
558 ✗ const float* bp = static_cast<const float*>(bias->data);
559 ✗ for (int i = 0; i < out; ++i) y_f32[i] += bp[i];
560 ✗ }
561 ✗ }
562
563 ✗ if (Y_BD.dtype == Dtype::FP16) {
564 ✗ uint16_t* yp = static_cast<uint16_t*>(Y_BD.data) + static_cast<size_t>(b) * out;
565 ✗ for (int i = 0; i < out; ++i) yp[i] = fp32_to_fp16_bits(y_f32[i]);
566 ✗ } else {
567 ✗ float* yp = static_cast<float*>(Y_BD.data) + static_cast<size_t>(b) * out;
568 ✗ std::memcpy(yp, y_f32.data(), out * sizeof(float));
569 }
570 ✗ }
571 ✗ }
572
573 } // namespace brotensor::detail::cpu
574