GCC Code Coverage Report

Directory: brotensor/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 92.3% 60 / 0 / 65
Functions: 80.0% 4 / 0 / 5
Branches: 41.5% 39 / 0 / 94
src/cpu/linear_wq.cpp
Line Branch Exec Source
1 // CPU twin of src/cuda/linear_wq.cu: quantize_rows and
2 // linear_forward_batched_wq_ex in FP32 (the parity oracle).
3
4 #include <brotensor/ops/linear.h>
5 #include <brotensor/tensor.h>
6
7 #include <algorithm>
8 #include <cmath>
9 #include <cstdint>
10 #include <stdexcept>
11 #include <string>
12 #include <vector>
13
14 namespace brotensor::detail::cpu {
15
16 void linear_forward_batched_ex(const ::brotensor::Tensor& W, const ::brotensor::Tensor* bias,
17 const ::brotensor::Tensor& X, int act, int epilogue,
18 ::brotensor::Tensor* workspace, ::brotensor::Tensor& Y);
19
20 namespace {
21
22 ✗ [[noreturn]] void fail(const char* op, const std::string& reason) {
23 ✗ throw std::runtime_error(std::string("brotensor: ") + op + ": " + reason);
24 ✗ }
25
26 44387734 float decode(Dtype fmt, uint8_t b) {
27
2/2
✓ Branch 0 taken 22285003 times.
✓ Branch 1 taken 22102731 times.
 44387734 return fmt == Dtype::INT8 ? static_cast<float>(static_cast<int8_t>(b)) : fp8_e4m3_bits_to_fp32(b);
28 }
29
30 } // namespace
31
32 38 void quantize_rows(const ::brotensor::Tensor& W, Dtype fmt, ::brotensor::Tensor& Wq, ::brotensor::Tensor& scales) {
33 38 constexpr const char* op = "quantize_rows";
34
3/6
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 19 times.
✓ Branch 2 taken 19 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 38 if (fmt != Dtype::INT8 && fmt != Dtype::FP8_E4M3) fail(op, "fmt must be INT8 or FP8_E4M3");
35
1/4
✓ Branch 0 taken 38 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 38 if (W.dtype != Dtype::FP32) fail(op, "W must be FP32 on the CPU");
36 38 const int N = W.rows, K = W.cols;
37
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 38 if (Wq.rows != N || Wq.cols != K || Wq.dtype != fmt) Wq.resize(N, K, fmt);
38
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 38 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 38 if (scales.rows != N || scales.cols != 1 || scales.dtype != Dtype::FP32) scales.resize(N, 1, Dtype::FP32);
39 38 const float* w = W.host_f32();
40 38 auto* q = static_cast<uint8_t*>(Wq.data);
41 38 float* s = scales.host_f32_mut();
42 38 const float qmax = fmt == Dtype::INT8 ? 127.0f : 448.0f;
43
2/2
✓ Branch 0 taken 20060 times.
✓ Branch 1 taken 38 times.
 20098 for (int r = 0; r < N; ++r) {
44 20060 const float* wr = w + static_cast<std::size_t>(r) * K;
45 20060 float mx = 0.0f;
46
2/2
✓ Branch 0 taken 45436310 times.
✓ Branch 1 taken 20060 times.
 45456370 for (int k = 0; k < K; ++k) mx = std::max(mx, std::fabs(wr[k]));
47 20060 const float scale = mx / qmax;
48 20060 s[r] = scale;
49 20060 uint8_t* qr = q + static_cast<std::size_t>(r) * K;
50
2/2
✓ Branch 0 taken 45436310 times.
✓ Branch 1 taken 20060 times.
 45456370 for (int k = 0; k < K; ++k) {
51
1/2
✓ Branch 0 taken 45436310 times.
✗ Branch 1 not taken.
 45436310 const float v = scale > 0.0f ? wr[k] / scale : 0.0f;
52
2/2
✓ Branch 0 taken 22809291 times.
✓ Branch 1 taken 22627019 times.
 45436310 if (fmt == Dtype::INT8)
53 22809291 qr[k] = static_cast<uint8_t>(static_cast<int8_t>(std::clamp(std::nearbyint(v), -127.0f, 127.0f)));
54 else
55 22627019 qr[k] = fp32_to_fp8_e4m3_bits(v);
56 45436310 }
57 20060 }
58 38 }
59
60 2 void dequantize_rows(const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& scales, Dtype out,
61 ::brotensor::Tensor& W) {
62 2 constexpr const char* op = "dequantize_rows";
63
3/6
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 2 if (Wq.dtype != Dtype::INT8 && Wq.dtype != Dtype::FP8_E4M3) fail(op, "Wq must be INT8 or FP8_E4M3");
64
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 if (scales.dtype != Dtype::FP32 || static_cast<long long>(scales.rows) * scales.cols != Wq.rows)
65 ✗ fail(op, "scales must be (out,1) FP32");
66
1/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 2 if (out != Dtype::FP32) fail(op, "out must be FP32 on the CPU");
67 2 const int N = Wq.rows, K = Wq.cols;
68
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 2 if (W.rows != N || W.cols != K || W.dtype != Dtype::FP32) W.resize(N, K, Dtype::FP32);
69 2 const auto* q = static_cast<const uint8_t*>(Wq.data);
70 2 const float* s = scales.host_f32();
71 2 float* w = W.host_f32_mut();
72
2/2
✓ Branch 0 taken 194 times.
✓ Branch 1 taken 2 times.
 196 for (int r = 0; r < N; ++r)
73
2/2
✓ Branch 0 taken 194582 times.
✓ Branch 1 taken 194 times.
 194776 for (int k = 0; k < K; ++k) {
74 194582 const std::size_t i = static_cast<std::size_t>(r) * K + k;
75 194582 w[i] = decode(Wq.dtype, q[i]) * s[r];
76 194776 }
77 2 }
78
79 34 void linear_forward_batched_wq_ex(const ::brotensor::Tensor& Wq, const ::brotensor::Tensor& scales,
80 const ::brotensor::Tensor* bias, const ::brotensor::Tensor& X, int act,
81 int epilogue, ::brotensor::Tensor& Y) {
82 34 constexpr const char* op = "linear_forward_batched_wq_ex";
83
3/6
✓ Branch 0 taken 17 times.
✓ Branch 1 taken 17 times.
✓ Branch 2 taken 17 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 34 if (Wq.dtype != Dtype::INT8 && Wq.dtype != Dtype::FP8_E4M3) fail(op, "Wq must be INT8 or FP8_E4M3");
84
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 if (scales.dtype != Dtype::FP32 || static_cast<long long>(scales.rows) * scales.cols != Wq.rows)
85 ✗ fail(op, "scales must be (out,1) FP32");
86 34 const int N = Wq.rows, K = Wq.cols;
87 34 ::brotensor::Tensor W = ::brotensor::Tensor::mat(N, K);
88 34 const auto* q = static_cast<const uint8_t*>(Wq.data);
89
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 const float* s = scales.host_f32();
90
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 float* w = W.host_f32_mut();
91
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 19354 times.
 19388 for (int r = 0; r < N; ++r)
92
2/2
✓ Branch 0 taken 19354 times.
✓ Branch 1 taken 44193152 times.
 44212506 for (int k = 0; k < K; ++k) {
93 44193152 const std::size_t i = static_cast<std::size_t>(r) * K + k;
94
1/2
✓ Branch 0 taken 44193152 times.
✗ Branch 1 not taken.
 44193152 w[i] = decode(Wq.dtype, q[i]) * s[r];
95 44212506 }
96
1/2
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
 34 ::brotensor::detail::cpu::linear_forward_batched_ex(W, bias, X, act, epilogue, nullptr, Y);
97 34 }
98
99 } // namespace brotensor::detail::cpu
100