GCC Code Coverage Report

Directory: brotensor/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 88.0% 117 / 0 / 133
Functions: 100.0% 21 / 0 / 21
Branches: 51.3% 121 / 0 / 236
src/api/api_internal.h
Line Branch Exec Source
1 #pragma once
2
3 #include "brotensor/tensor.h"
4 #include "brotensor/ops.h"
5 #include "brotensor/runtime.h"
6 #include "embed/embed.h"
7
8 #include <cstdint>
9 #include <cstring>
10 #include <exception>
11 #include <initializer_list>
12 #include <string>
13 #include <vector>
14
15 namespace brotensor::api {
16
17 struct GpuTensorHandle {
18 brotensor::Tensor tensor;
19 std::vector<int64_t> shape;
20 };
21
22 10 inline GpuTensorHandle* toHandle(void* ptr) {
23
1/2
✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
 10 if (!ptr) return nullptr;
24 10 return static_cast<GpuTensorHandle*>(ptr);
25 10 }
26
27 inline const GpuTensorHandle* constHandle(const void* ptr) {
28 if (!ptr) return nullptr;
29 return static_cast<const GpuTensorHandle*>(ptr);
30 }
31
32 6050 inline brotensor::Tensor* toTensor(void* ptr) {
33
1/2
✓ Branch 0 taken 6050 times.
✗ Branch 1 not taken.
 6050 if (!ptr) return nullptr;
34 6050 return &static_cast<GpuTensorHandle*>(ptr)->tensor;
35 6050 }
36
37 inline const brotensor::Tensor* constTensor(const void* ptr) {
38 if (!ptr) return nullptr;
39 return &static_cast<const GpuTensorHandle*>(ptr)->tensor;
40 }
41
42 // The old QuickJS binding accepted "fp32"/"f32", "fp16"/"f16", "bf16",
43 // "int8"/"i8" (and the numeric Dtype enum, which js/tensor.js maps to the
44 // string before the call).
45 1838 inline brotensor::Dtype parseDtype(const char* dt) {
46
1/2
✓ Branch 0 taken 1838 times.
✗ Branch 1 not taken.
 1838 if (!dt) return brotensor::Dtype::FP32;
47
3/4
✓ Branch 0 taken 1666 times.
✓ Branch 1 taken 172 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1666 times.
 1838 if (std::strcmp(dt, "fp16") == 0 || std::strcmp(dt, "f16") == 0) return brotensor::Dtype::FP16;
48
1/2
✓ Branch 0 taken 1666 times.
✗ Branch 1 not taken.
 1666 if (std::strcmp(dt, "bf16") == 0) return brotensor::Dtype::BF16;
49
3/4
✓ Branch 0 taken 1648 times.
✓ Branch 1 taken 18 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1648 times.
 1666 if (std::strcmp(dt, "int8") == 0 || std::strcmp(dt, "i8") == 0) return brotensor::Dtype::INT8;
50
3/4
✓ Branch 0 taken 1594 times.
✓ Branch 1 taken 54 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1594 times.
 1648 if (std::strcmp(dt, "int32") == 0 || std::strcmp(dt, "i32") == 0) return brotensor::Dtype::INT32;
51
1/2
✓ Branch 0 taken 1594 times.
✗ Branch 1 not taken.
 1594 if (std::strcmp(dt, "f64") == 0) return brotensor::Dtype::F64;
52 1594 return brotensor::Dtype::FP32;
53 1838 }
54
55 54 inline const char* dtypeToString(brotensor::Dtype dt) {
56
4/8
✓ Branch 0 taken 18 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 14 times.
✓ Branch 5 taken 20 times.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 54 switch (dt) {
57 14 case brotensor::Dtype::FP32: return "fp32";
58 18 case brotensor::Dtype::FP16: return "fp16";
59 ✗ case brotensor::Dtype::BF16: return "bf16";
60 2 case brotensor::Dtype::INT8: return "int8";
61 20 case brotensor::Dtype::INT32: return "int32";
62 ✗ case brotensor::Dtype::F64: return "f64";
63 ✗ case brotensor::Dtype::FP8_E4M3: return "fp8_e4m3";
64 ✗ default: return "unknown";
65 }
66 54 }
67
68 // ---- error channel ---------------------------------------------------------
69 //
70 // A native is called directly from generated code, so a C++ exception must
71 // not escape it (bronze's fatal.h: those frames carry no unwind metadata).
72 // A native that fails records the message here instead; the JS wrapper reads
73 // it back through `__bro_native.tensor.takeError()` right after the call and
74 // throws. Per thread, like the realm the natives serve.
75 std::string& lastErrorSlot();
76 262 inline void setError(const std::string& msg) { lastErrorSlot() = msg; }
77
78 // ---- `dynamic` arguments ---------------------------------------------------
79 //
80 // Nullable tensor slots (mask|null, bias|null, ...) cross as `dynamic`: the
81 // raw Value bits. null / undefined give nullptr. js/tensor*.js checks
82 // `instanceof GpuTensor` before the call, so the handle data is ours.
83 678 inline brotensor::Tensor* tensorFromValue(uint64_t bits) {
84 678 void* d = bronze::embed::handleData(bronze::Value{bits});
85
2/2
✓ Branch 0 taken 176 times.
✓ Branch 1 taken 502 times.
 678 return d ? toTensor(d) : nullptr;
86 }
87
88 // The `const float* d_mask` convention of the ops: the device pointer of an
89 // optional FP32 GpuTensor, or nullptr.
90 74 inline const float* maskPtr(uint64_t bits) {
91 74 auto* t = tensorFromValue(bits);
92
2/2
✓ Branch 0 taken 14 times.
✓ Branch 1 taken 60 times.
 74 return t ? static_cast<const float*>(t->data) : nullptr;
93 }
94
95 // A JS array of GpuTensors (concatRows parts, ...) as tensor pointers. False
96 // with the error set when an element is not a GpuTensor.
97 bool readTensorArray(uint64_t bits, std::vector<brotensor::Tensor*>& out, const char* label);
98
99 // True when every pointer is non-null; otherwise records "<label>: GpuTensor
100 // argument missing" and answers false, so a body can `if (!need(...)) return;`.
101 910 inline bool need(const char* label, std::initializer_list<const void*> ptrs) {
102
2/2
✓ Branch 0 taken 3136 times.
✓ Branch 1 taken 910 times.
 4046 for (const void* p : ptrs) {
103
1/2
✓ Branch 0 taken 3136 times.
✗ Branch 1 not taken.
 3136 if (!p) {
104 ✗ setError(std::string(label) + ": GpuTensor argument missing");
105 ✗ return false;
106 }
107 }
108 910 return true;
109 910 }
110
111 // ---- input-size contract ---------------------------------------------------
112 //
113 // brotensor ops trust their caller's explicit dims: conv2d_forward reads
114 // N*C_in*H*W elements of X because the caller said so, whatever X holds, and
115 // an elementwise backward reads dY.size() == x.size() elements without
116 // checking. From C++ that is the library's contract; from JS it is an
117 // out-of-bounds read (or write) on the host or the device. So every native
118 // that hands brotensor a dim or a second operand checks the operand first.
119 //
120 // Outputs are exempt: the ops resize them.
121
122 // `t` holds at least `n` elements. `n` is computed in 64 bits by the caller
123 // (dims multiply past INT_MAX easily); a negative `n` means a negative dim.
124 902 inline bool needElems(const char* label, const char* name,
125 const brotensor::Tensor* t, int64_t n) {
126
1/2
✓ Branch 0 taken 902 times.
✗ Branch 1 not taken.
 902 if (!t) {
127 ✗ setError(std::string(label) + ": " + name + " must be a GpuTensor");
128 ✗ return false;
129 }
130
1/2
✓ Branch 0 taken 902 times.
✗ Branch 1 not taken.
 902 if (n < 0) {
131 ✗ setError(std::string(label) + ": negative dimension for " + name);
132 ✗ return false;
133 }
134
5/6
✓ Branch 0 taken 798 times.
✓ Branch 1 taken 104 times.
✓ Branch 2 taken 786 times.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 786 times.
✗ Branch 5 not taken.
 902 if (static_cast<int64_t>(t->size()) < n || (n > 0 && t->data == nullptr)) {
135
8/16
✓ Branch 0 taken 104 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 104 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 104 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 104 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 104 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 104 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 104 times.
✗ Branch 13 not taken.
✓ Branch 14 taken 104 times.
✗ Branch 15 not taken.
 208 setError(std::string(label) + ": " + name + " holds " + std::to_string(t->size()) +
136
2/4
✓ Branch 0 taken 104 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 104 times.
✗ Branch 3 not taken.
 104 " elements, the dims need " + std::to_string(n));
137 104 return false;
138 }
139 798 return true;
140 902 }
141
142 // `t` has exactly `ref`'s element count (the elementwise-pair case).
143 110 inline bool needSameSize(const char* label, const char* name,
144 const brotensor::Tensor* t, const brotensor::Tensor* ref) {
145
2/4
✓ Branch 0 taken 110 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 110 times.
 110 if (!t || !ref) {
146 ✗ setError(std::string(label) + ": " + name + " must be a GpuTensor");
147 ✗ return false;
148 }
149
2/2
✓ Branch 0 taken 34 times.
✓ Branch 1 taken 76 times.
 110 if (t->size() != ref->size()) {
150
8/16
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 34 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 34 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 34 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 34 times.
✗ Branch 13 not taken.
✓ Branch 14 taken 34 times.
✗ Branch 15 not taken.
 68 setError(std::string(label) + ": " + name + " has " + std::to_string(t->size()) +
151
3/6
✓ Branch 0 taken 34 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 34 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 34 times.
✗ Branch 5 not taken.
 34 " elements, expected " + std::to_string(ref->size()));
152 34 return false;
153 }
154 76 return true;
155 110 }
156
157 // `t` carries `ref`'s dtype (a kernel dispatches on one operand's dtype and
158 // reads the others with that element size).
159 738 inline bool needSameDtype(const char* label, const char* name,
160 const brotensor::Tensor* t, const brotensor::Tensor* ref) {
161
2/4
✓ Branch 0 taken 738 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 738 times.
 738 if (!t || !ref) {
162 ✗ setError(std::string(label) + ": " + name + " must be a GpuTensor");
163 ✗ return false;
164 }
165
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 732 times.
 738 if (t->dtype != ref->dtype) {
166
7/14
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 6 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 6 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 6 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 6 times.
✗ Branch 11 not taken.
✓ Branch 12 taken 6 times.
✗ Branch 13 not taken.
 12 setError(std::string(label) + ": " + name + " is " + dtypeToString(t->dtype) +
167
2/4
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
 6 ", expected " + dtypeToString(ref->dtype));
168 6 return false;
169 }
170 732 return true;
171 738 }
172
173 // Product of dims in 64 bits; negative when any dim is negative.
174 278 inline int64_t elems(std::initializer_list<int64_t> dims) {
175 278 int64_t n = 1;
176
2/2
✓ Branch 0 taken 852 times.
✓ Branch 1 taken 278 times.
 1130 for (int64_t d : dims) {
177
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 852 times.
 852 if (d < 0) return -1;
178 852 n *= d;
179 }
180 278 return n;
181 278 }
182
183 // An elementwise partner: same element count and dtype as `ref`.
184 74 inline bool needPair(const char* label, const char* name,
185 const brotensor::Tensor* t, const brotensor::Tensor* ref) {
186
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 52 times.
 74 return needSameSize(label, name, t, ref) && needSameDtype(label, name, t, ref);
187 }
188
189 // Each value is > 0 (kernel sizes, strides, dilations, groups, heads: a zero
190 // is a division by zero inside the op, which on the CPU backend is a crash).
191 136 inline bool needPositive(const char* label, std::initializer_list<int64_t> vals) {
192
2/2
✓ Branch 0 taken 446 times.
✓ Branch 1 taken 132 times.
 578 for (int64_t v : vals) {
193
2/2
✓ Branch 0 taken 442 times.
✓ Branch 1 taken 4 times.
 446 if (v <= 0) {
194
2/4
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
✗ Branch 3 not taken.
 4 setError(std::string(label) + ": kernel/stride/dilation/group/head counts must be positive");
195 4 return false;
196 }
197 }
198 132 return true;
199 136 }
200
201 // Each value is >= 0 (dims, paddings, offsets).
202 158 inline bool needNonNegative(const char* label, std::initializer_list<int64_t> vals) {
203
2/2
✓ Branch 0 taken 574 times.
✓ Branch 1 taken 156 times.
 730 for (int64_t v : vals) {
204
2/2
✓ Branch 0 taken 572 times.
✓ Branch 1 taken 2 times.
 574 if (v < 0) {
205
2/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 setError(std::string(label) + ": dims, paddings and offsets must be non-negative");
206 2 return false;
207 }
208 }
209 156 return true;
210 158 }
211
212 // 2-D convolution geometry: positive kernel/stride/dilation/groups, groups
213 // dividing both channel counts, non-negative dims and paddings. Answers the
214 // output extent the op will produce (0 when the window does not fit).
215 34 inline bool convGeom2d(const char* label, int64_t C_in, int64_t C_out, int64_t H, int64_t W,
216 int64_t kH, int64_t kW, int64_t sH, int64_t sW,
217 int64_t pH, int64_t pW, int64_t dH, int64_t dW, int64_t groups,
218 int64_t& H_out, int64_t& W_out) {
219
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 32 times.
 34 if (!needPositive(label, {kH, kW, sH, sW, dH, dW, groups})) return false;
220
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 32 times.
 32 if (!needNonNegative(label, {C_in, C_out, H, W, pH, pW})) return false;
221
3/4
✓ Branch 0 taken 30 times.
✓ Branch 1 taken 2 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 30 times.
 32 if (C_in % groups != 0 || C_out % groups != 0) {
222
2/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 setError(std::string(label) + ": groups must divide C_in and C_out");
223 2 return false;
224 }
225 30 const int64_t spanH = H + 2 * pH - dH * (kH - 1) - 1;
226 30 const int64_t spanW = W + 2 * pW - dW * (kW - 1) - 1;
227
1/2
✓ Branch 0 taken 30 times.
✗ Branch 1 not taken.
 30 H_out = spanH < 0 ? 0 : spanH / sH + 1;
228
1/2
✓ Branch 0 taken 30 times.
✗ Branch 1 not taken.
 30 W_out = spanW < 0 ? 0 : spanW / sW + 1;
229 30 return true;
230 34 }
231
232 // One operand of a multi-tensor op: `t` must hold `n` elements. A null `t`
233 // is an absent optional operand and passes (need() has already rejected a
234 // missing required one).
235 struct Operand {
236 const char* name;
237 const brotensor::Tensor* t;
238 int64_t n;
239 };
240
241 // Every operand holds its element count and, when `ref` is given, carries
242 // ref's dtype.
243 126 inline bool needOperands(const char* label, const brotensor::Tensor* ref,
244 std::initializer_list<Operand> ops) {
245
2/2
✓ Branch 0 taken 118 times.
✓ Branch 1 taken 528 times.
 646 for (const Operand& o : ops) {
246
2/2
✓ Branch 0 taken 444 times.
✓ Branch 1 taken 84 times.
 528 if (!o.t) continue;
247
2/2
✓ Branch 0 taken 436 times.
✓ Branch 1 taken 8 times.
 444 if (!needElems(label, o.name, o.t, o.n)) return false;
248
2/4
✓ Branch 0 taken 436 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 436 times.
✗ Branch 3 not taken.
 436 if (ref && !needSameDtype(label, o.name, o.t, ref)) return false;
249 }
250 118 return true;
251 126 }
252
253 // The ops' `const float* d_mask` operands (and the FP32 bias tables): an
254 // optional FP32 tensor of at least `n` elements.
255 92 inline bool needMask(const char* label, const brotensor::Tensor* mt, int64_t n, const char* name = "mask") {
256
2/2
✓ Branch 0 taken 32 times.
✓ Branch 1 taken 60 times.
 92 if (!mt) return true;
257
2/2
✓ Branch 0 taken 28 times.
✓ Branch 1 taken 4 times.
 32 if (!needElems(label, name, mt, n)) return false;
258
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 if (mt->dtype != brotensor::Dtype::FP32) {
259 ✗ setError(std::string(label) + ": " + name + " must be FP32");
260 ✗ return false;
261 }
262 28 return true;
263 92 }
264
265 // numHeads is positive and divides D.
266 40 inline bool needHeads(const char* label, int64_t D, int64_t numHeads) {
267
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 40 times.
 40 if (!needPositive(label, {numHeads})) return false;
268
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 38 times.
 40 if (D % numHeads != 0) {
269
6/12
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 2 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 2 times.
✗ Branch 9 not taken.
✓ Branch 10 taken 2 times.
✗ Branch 11 not taken.
 6 setError(std::string(label) + ": numHeads (" + std::to_string(numHeads) + ") must divide D (" +
270
2/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 4 std::to_string(D) + ")");
271 2 return false;
272 }
273 38 return true;
274 40 }
275
276 // Path resolution for the file loaders (api.h setPathResolver).
277 std::string resolvePath(const std::string& path);
278
279 } // namespace brotensor::api
280
281 // Guard a native body that calls into brotensor: the ops throw
282 // std::runtime_error on a shape or dtype contract violation, and that has to
283 // become a recorded error, never an unwind through generated code.
284 #define BROTENSOR_API_TRY try {
285 #define BROTENSOR_API_CATCH(label) \
286 } catch (const std::exception& e) { \
287 ::brotensor::api::setError(std::string(label) + ": " + e.what()); \
288 }
289