GCC Code Coverage Report

Directory: brotensor/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 89.8% 264 / 0 / 294
Functions: 91.4% 32 / 0 / 35
Branches: 51.2% 255 / 0 / 498
src/dispatch.cpp
Line Branch Exec Source
1 // Backend registration + per-operand dispatch: the dispatcher core.
2 //
3 // Storage is two parallel std::array<>s indexed by static_cast<int>(Device).
4 // Registration happens during static init (CPU) and at brotensor::init()
5 // time (CUDA/Metal probe success). Both are single-threaded entry points
6 // before any op call, so no mutex is required.
7
8 #include <brotensor/detail/dispatch.h>
9 #include <brotensor/tensor.h>
10 #include <brotensor/runtime.h>
11
12 #include <array>
13 #include <cstddef>
14 #include <cstring>
15 #include <initializer_list>
16 #include <stdexcept>
17 #include <string>
18
19 #if defined(BROTENSOR_HAS_CUDA)
20 #include <cuda_runtime.h>
21 #endif
22
23 namespace brotensor::detail {
24
25 namespace {
26
27 constexpr int kNumDevices = 5; // CPU, CUDA, Metal, (3 unused), VULKAN
28
29 // Upper bound on operands a single dispatch call inspects. The widest op is
30 // resblock_backward (25 operands); 32 leaves headroom. dispatch_with_opts
31 // throws if a future op ever exceeds this rather than silently truncating.
32 constexpr std::size_t kMaxOperands = 32;
33
34 struct Slot {
35 OpsVTable ops{};
36 AllocVTable alloc{};
37 bool registered = false;
38 };
39
40 338972 std::array<Slot, kNumDevices>& slots() {
41 static std::array<Slot, kNumDevices> s{};
42 338972 return s;
43 }
44
45 13 const char* dev_name(Device d) {
46 13 return device_name(d);
47 }
48
49 2 [[noreturn]] void throw_unregistered(Device d) {
50 2 std::string msg = "brotensor: backend ";
51
2/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += dev_name(d);
52
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 msg += " not registered";
53
2/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 2 times.
 2 throw std::runtime_error(msg);
54 2 }
55
56 // A tensor is "committed" to a device once it owns real storage. A freshly
57 // default-constructed (or resized-to-empty) tensor has data == nullptr and
58 // carries no device affinity yet — it is a wildcard that adopts the op's
59 // device. This is what lets callers pass an unsized output tensor to a GPU op
60 // without first pinning it to the right backend by hand.
61 250818 inline bool committed(const Tensor& t) { return t.data != nullptr; }
62
63 4 [[noreturn]] void throw_device_mismatch(Device resolved, int idx, Device got) {
64 4 std::string msg = "brotensor: dispatch: operand ";
65
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
✓ Branch 2 taken 4 times.
✗ Branch 3 not taken.
 4 msg += std::to_string(idx);
66
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 msg += " is on ";
67
2/4
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
✗ Branch 3 not taken.
 4 msg += dev_name(got);
68
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 msg += " but the op resolved to ";
69
2/4
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 4 times.
 4 msg += dev_name(resolved);
70
2/4
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 4 times.
 4 throw std::runtime_error(msg);
71 4 }
72
73 75374 inline void activate_device_context(Device dev) {
74 75374 dev = resolve_device_alias(dev);
75 #if defined(BROTENSOR_HAS_CUDA)
76 if (dev.is_cuda()) {
77 cudaSetDevice(dev.index);
78 }
79 #endif
80 (void)dev;
81 75374 }
82
83 // Resolve the op's device from the first committed operand; verify every other
84 // committed operand agrees (uncommitted operands are skipped). Returns the
85 // vtable for the resolved device.
86 73584 const OpsVTable& dispatch_v(std::initializer_list<const Tensor*> ts) {
87 73584 Device dev = Device::CPU;
88 73584 bool found = false;
89
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 73588 times.
 73606 for (const Tensor* t : ts) {
90
3/4
✓ Branch 0 taken 73588 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 22 times.
✓ Branch 3 taken 73566 times.
 73588 if (t && committed(*t)) { dev = t->device; found = true; break; }
91 }
92
4/6
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 73566 times.
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 18 times.
 73584 if (!found && ts.size() != 0 && *ts.begin() != nullptr) {
93 // All operands empty — degenerate but not an error (e.g. a fresh
94 // output passed before any input). Keep operand 0's tag.
95 18 dev = (*ts.begin())->device;
96 18 }
97 73584 int idx = 0;
98
2/2
✓ Branch 0 taken 166273 times.
✓ Branch 1 taken 73582 times.
 239855 for (const Tensor* t : ts) {
99
5/6
✓ Branch 0 taken 166273 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 109792 times.
✓ Branch 3 taken 56481 times.
✓ Branch 4 taken 2 times.
✓ Branch 5 taken 109790 times.
 166273 if (t && committed(*t) && t->device != dev) {
100 2 throw_device_mismatch(dev, idx, t->device);
101 }
102 166271 ++idx;
103 }
104 73582 activate_device_context(dev);
105 73582 return ops_for(dev);
106 }
107
108 // Same resolution rule as dispatch_v, over a plain operand array — used by the
109 // optional-operand dispatch path so it needs no heap allocation. Null entries
110 // (skipped optional operands) are tolerated.
111 1794 const OpsVTable& resolve_over(const Tensor* const* all, std::size_t count) {
112 1794 Device dev = Device::CPU;
113 1794 bool found = false;
114
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 1798 times.
 1800 for (std::size_t i = 0; i < count; ++i) {
115
4/4
✓ Branch 0 taken 1796 times.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 4 times.
✓ Branch 3 taken 1792 times.
 1798 if (all[i] && committed(*all[i])) {
116 1792 dev = all[i]->device;
117 1792 found = true;
118 1792 break;
119 }
120 6 }
121
2/2
✓ Branch 0 taken 1792 times.
✓ Branch 1 taken 2 times.
 1794 if (!found) {
122 // No committed operand — adopt the first non-null operand's tag.
123
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 for (std::size_t i = 0; i < count; ++i) {
124
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 if (all[i]) { dev = all[i]->device; break; }
125 ✗ }
126 2 }
127
2/2
✓ Branch 0 taken 10830 times.
✓ Branch 1 taken 1792 times.
 12622 for (std::size_t i = 0; i < count; ++i) {
128
6/6
✓ Branch 0 taken 9161 times.
✓ Branch 1 taken 1669 times.
✓ Branch 2 taken 7820 times.
✓ Branch 3 taken 1341 times.
✓ Branch 4 taken 2 times.
✓ Branch 5 taken 7818 times.
 10830 if (all[i] && committed(*all[i]) && all[i]->device != dev) {
129 2 throw_device_mismatch(dev, static_cast<int>(i), all[i]->device);
130 }
131 10828 }
132 1792 activate_device_context(dev);
133 1792 return ops_for(dev);
134 }
135
136 1 [[noreturn]] void throw_too_many_operands(std::size_t n) {
137
3/6
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 time.
✓ Branch 4 taken 1 time.
✗ Branch 5 not taken.
 2 throw std::runtime_error(
138
3/6
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 time.
✗ Branch 5 not taken.
 1 "brotensor: dispatch: operand count " + std::to_string(n) +
139
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 " exceeds the fixed dispatch buffer (" +
140
2/4
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 2 std::to_string(kMaxOperands) + ")");
141 1 }
142
143 } // namespace
144
145 static int g_cuda_device_count = 0;
146 static int g_vulkan_device_count = 0;
147
148 ✗ void set_cuda_device_count(int count) {
149 ✗ g_cuda_device_count = count;
150 ✗ }
151
152 ✗ void set_vulkan_device_count(int count) {
153 ✗ g_vulkan_device_count = count;
154 ✗ }
155
156 347 void register_backend(DeviceType dt, const OpsVTable& ops, const AllocVTable& alloc) {
157 347 auto& s = slots()[static_cast<int>(dt)];
158 347 std::memcpy(&s.ops, &ops, sizeof(OpsVTable));
159 347 std::memcpy(&s.alloc, &alloc, sizeof(AllocVTable));
160 347 s.registered = true;
161 347 }
162
163 347 void register_backend(Device d, const OpsVTable& ops, const AllocVTable& alloc) {
164 347 register_backend(d.type, ops, alloc);
165 347 }
166
167 14586 bool is_registered(Device d) {
168
2/2
✓ Branch 0 taken 4639 times.
✓ Branch 1 taken 9947 times.
 14586 if (d.is_cpu()) return slots()[static_cast<int>(DeviceType::CPU)].registered;
169
2/2
✓ Branch 0 taken 4936 times.
✓ Branch 1 taken 5011 times.
 9947 if (d.is_cuda()) {
170
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4936 times.
 4936 if (!slots()[static_cast<int>(DeviceType::CUDA)].registered) return false;
171 ✗ int count = ::brotensor::cuda_device_count();
172 ✗ if (count <= 0) count = 1;
173 ✗ return d.index >= 0 && d.index < count;
174 }
175
2/2
✓ Branch 0 taken 4994 times.
✓ Branch 1 taken 17 times.
 5011 if (d.is_metal()) {
176
2/2
✓ Branch 0 taken 15 times.
✓ Branch 1 taken 4979 times.
 4994 return slots()[static_cast<int>(DeviceType::Metal)].registered && d.index == 0;
177 }
178
1/2
✓ Branch 0 taken 17 times.
✗ Branch 1 not taken.
 17 if (d.is_vulkan()) {
179
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 17 times.
 17 if (!slots()[static_cast<int>(DeviceType::VULKAN)].registered) return false;
180 ✗ return d.index >= 0 && d.index < ::brotensor::vulkan_device_count();
181 }
182 ✗ return false;
183 14586 }
184
185 553538 Device resolve_device_alias(Device d) {
186
3/4
✓ Branch 0 taken 2601 times.
✓ Branch 1 taken 550937 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 2601 times.
 553538 if (d.is_cuda() && !slots()[static_cast<int>(DeviceType::CUDA)].registered) {
187 2601 const bool metal = slots()[static_cast<int>(DeviceType::Metal)].registered;
188 2601 const bool vulkan = slots()[static_cast<int>(DeviceType::VULKAN)].registered;
189 // With both, the default device decides (a Metal + Vulkan build run
190 // with BROTENSOR_DEFAULT_DEVICE=vulkan means Vulkan); otherwise Metal
191 // first, the order pick_default_from_available() uses.
192
3/6
✓ Branch 0 taken 2601 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2601 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✓ Branch 5 taken 2601 times.
 2601 if (metal && vulkan && ::brotensor::default_device().is_vulkan()) return Device::vulkan(d.index);
193
1/2
✓ Branch 0 taken 2601 times.
✗ Branch 1 not taken.
 2601 if (metal) return Device::Metal;
194 ✗ if (vulkan) return Device::vulkan(d.index);
195 ✗ }
196 550937 return d;
197 553538 }
198
199 75500 const OpsVTable& ops_for(Device d) {
200 75500 d = resolve_device_alias(d);
201 75500 auto& s = slots()[static_cast<int>(d.type)];
202
2/2
✓ Branch 0 taken 75499 times.
✓ Branch 1 taken 1 time.
 75500 if (!s.registered) throw_unregistered(d);
203 75499 return s.ops;
204 }
205
206 240736 const AllocVTable& alloc_for(Device d) {
207 240736 d = resolve_device_alias(d);
208 240736 auto& s = slots()[static_cast<int>(d.type)];
209
2/2
✓ Branch 0 taken 240735 times.
✓ Branch 1 taken 1 time.
 240736 if (!s.registered) throw_unregistered(d);
210 240735 return s.alloc;
211 }
212
213 // ─── dispatch overloads ────────────────────────────────────────────────────
214 //
215 // Each forwards to dispatch_v, which resolves the device from the first
216 // committed operand and skips uncommitted (data == nullptr) wildcards.
217
218 1776 const OpsVTable& dispatch(const Tensor& a) {
219 1776 return dispatch_v({&a});
220 }
221
222 57485 const OpsVTable& dispatch(const Tensor& a, const Tensor& b) {
223 57485 return dispatch_v({&a, &b});
224 }
225
226 9935 const OpsVTable& dispatch(const Tensor& a, const Tensor& b, const Tensor& c) {
227 9935 return dispatch_v({&a, &b, &c});
228 }
229
230 3565 const OpsVTable& dispatch(const Tensor& a, const Tensor& b, const Tensor& c,
231 const Tensor& d) {
232 3565 return dispatch_v({&a, &b, &c, &d});
233 }
234
235 193 const OpsVTable& dispatch(const Tensor& a, const Tensor& b, const Tensor& c,
236 const Tensor& d, const Tensor& e) {
237 193 return dispatch_v({&a, &b, &c, &d, &e});
238 }
239
240 222 const OpsVTable& dispatch(const Tensor& a, const Tensor& b, const Tensor& c,
241 const Tensor& d, const Tensor& e, const Tensor& f) {
242 222 return dispatch_v({&a, &b, &c, &d, &e, &f});
243 }
244
245 99 const OpsVTable& dispatch(const Tensor& a, const Tensor& b, const Tensor& c,
246 const Tensor& d, const Tensor& e, const Tensor& f,
247 const Tensor& g) {
248 99 return dispatch_v({&a, &b, &c, &d, &e, &f, &g});
249 }
250
251 309 const OpsVTable& dispatch(const Tensor& a, const Tensor& b, const Tensor& c,
252 const Tensor& d, const Tensor& e, const Tensor& f,
253 const Tensor& g, const Tensor& h) {
254 309 return dispatch_v({&a, &b, &c, &d, &e, &f, &g, &h});
255 }
256
257 271 const OpsVTable& dispatch_with_opts(const Tensor& a,
258 std::initializer_list<const Tensor*> opts) {
259 271 const std::size_t count = 1 + opts.size();
260
2/2
✓ Branch 0 taken 270 times.
✓ Branch 1 taken 1 time.
 271 if (count > kMaxOperands) throw_too_many_operands(count);
261 const Tensor* all[kMaxOperands];
262 270 std::size_t n = 0;
263 270 all[n++] = &a;
264
2/2
✓ Branch 0 taken 649 times.
✓ Branch 1 taken 270 times.
 919 for (const Tensor* p : opts) all[n++] = p;
265 270 return resolve_over(all, n);
266 }
267
268 1524 const OpsVTable& dispatch_with_opts(const Tensor& a, const Tensor& b,
269 std::initializer_list<const Tensor*> opts) {
270 1524 const std::size_t count = 2 + opts.size();
271
1/2
✓ Branch 0 taken 1524 times.
✗ Branch 1 not taken.
 1524 if (count > kMaxOperands) throw_too_many_operands(count);
272 const Tensor* all[kMaxOperands];
273 1524 std::size_t n = 0;
274 1524 all[n++] = &a;
275 1524 all[n++] = &b;
276
2/2
✓ Branch 0 taken 6863 times.
✓ Branch 1 taken 1524 times.
 8387 for (const Tensor* p : opts) all[n++] = p;
277 1524 return resolve_over(all, n);
278 }
279
280 // ─── output adoption ───────────────────────────────────────────────────────
281 //
282 // An uncommitted output tensor (data == nullptr) has no device affinity yet.
283 // The wrapper calls this after dispatch so the tensor is pinned to the op's
284 // device before the backend impl resizes/allocates it.
285 81980 void adopt_output(Tensor& t, Device d) {
286
2/2
✓ Branch 0 taken 20827 times.
✓ Branch 1 taken 61153 times.
 81980 if (t.data == nullptr) t.device = resolve_device_alias(d);
287 81980 }
288
289 3 [[noreturn]] void throw_not_implemented(const char* op_name, Device d) {
290 3 std::string msg = "brotensor: ";
291
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 msg += op_name;
292
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 msg += ": not implemented on ";
293
2/4
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3 times.
 3 msg += dev_name(d);
294
2/4
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3 times.
 3 throw std::runtime_error(msg);
295 3 }
296
297 2 [[noreturn]] void throw_spatial_1d_error(const char* op, const char* tensor_name,
298 int N, int C, int L, int rows, int cols) {
299 2 std::string msg = "brotensor: ";
300
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += op;
301
3/6
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
✗ Branch 5 not taken.
 2 if (N <= 0 || C <= 0 || L <= 0) {
302 ✗ msg += ": non-positive spatial dimensions (N=";
303 ✗ msg += std::to_string(N);
304 ✗ msg += ", C=";
305 ✗ msg += std::to_string(C);
306 ✗ msg += ", L=";
307 ✗ msg += std::to_string(L);
308 ✗ msg += ") for ";
309 ✗ msg += tensor_name;
310 ✗ msg += " (rows=";
311 ✗ msg += std::to_string(rows);
312 ✗ msg += ", cols=";
313 ✗ msg += std::to_string(cols);
314 ✗ msg += ")";
315 ✗ } else {
316 2 const std::int64_t expected_cols = static_cast<std::int64_t>(C) * L;
317
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ": shape mismatch for ";
318
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += tensor_name;
319
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ": expected (N=";
320
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(N);
321
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ", C*L=";
322
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(C);
323
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += "*";
324
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(L);
325
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += "=";
326
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(expected_cols);
327
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += "), got (rows=";
328
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(rows);
329
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ", cols=";
330
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(cols);
331
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 msg += ")";
332 }
333
2/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 2 times.
 2 throw std::runtime_error(msg);
334 2 }
335
336 9 [[noreturn]] void throw_spatial_2d_error(const char* op, const char* tensor_name,
337 int N, int C, int H, int W, int rows, int cols) {
338 9 std::string msg = "brotensor: ";
339
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 msg += op;
340
6/8
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 8 times.
✓ Branch 3 taken 1 time.
✓ Branch 4 taken 7 times.
✓ Branch 5 taken 1 time.
✓ Branch 6 taken 7 times.
✗ Branch 7 not taken.
 9 if (N <= 0 || C <= 0 || H <= 0 || W <= 0) {
341
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ": non-positive spatial dimensions (N=";
342
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(N);
343
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ", C=";
344
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(C);
345
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ", H=";
346
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(H);
347
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ", W=";
348
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(W);
349
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ") for ";
350
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += tensor_name;
351
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += " (rows=";
352
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(rows);
353
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ", cols=";
354
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(cols);
355
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 msg += ")";
356 2 } else {
357 7 const std::int64_t expected_cols = static_cast<std::int64_t>(C) * H * W;
358
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 msg += ": shape mismatch for ";
359
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 msg += tensor_name;
360
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 msg += ": expected (N=";
361
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 msg += std::to_string(N);
362
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 msg += ", C*H*W=";
363
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 msg += std::to_string(C);
364
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 msg += "*";
365
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 msg += std::to_string(H);
366
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 msg += "*";
367
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 msg += std::to_string(W);
368
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 msg += "=";
369
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 msg += std::to_string(expected_cols);
370
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 msg += "), got (rows=";
371
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 msg += std::to_string(rows);
372
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 msg += ", cols=";
373
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 7 times.
✗ Branch 3 not taken.
 7 msg += std::to_string(cols);
374
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 msg += ")";
375 }
376
2/4
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 throw std::runtime_error(msg);
377 9 }
378
379 3 [[noreturn]] void throw_spatial_3d_error(const char* op, const char* tensor_name,
380 int N, int C, int T, int H, int W, int rows, int cols) {
381 3 std::string msg = "brotensor: ";
382
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 msg += op;
383
6/10
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
✓ Branch 5 taken 1 time.
✓ Branch 6 taken 2 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 2 times.
✗ Branch 9 not taken.
 3 if (N <= 0 || C <= 0 || T <= 0 || H <= 0 || W <= 0) {
384
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 msg += ": non-positive spatial dimensions (N=";
385
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 msg += std::to_string(N);
386
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 msg += ", C=";
387
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 msg += std::to_string(C);
388
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 msg += ", T=";
389
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 msg += std::to_string(T);
390
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 msg += ", H=";
391
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 msg += std::to_string(H);
392
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 msg += ", W=";
393
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 msg += std::to_string(W);
394
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 msg += ") for ";
395
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 msg += tensor_name;
396
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 msg += " (rows=";
397
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 msg += std::to_string(rows);
398
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 msg += ", cols=";
399
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 msg += std::to_string(cols);
400
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 msg += ")";
401 1 } else {
402 2 const std::int64_t expected_cols = static_cast<std::int64_t>(C) * T * H * W;
403
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ": shape mismatch for ";
404
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += tensor_name;
405
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ": expected (N=";
406
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(N);
407
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ", C*T*H*W=";
408
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(C);
409
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += "*";
410
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(T);
411
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += "*";
412
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(H);
413
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += "*";
414
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(W);
415
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += "=";
416
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(expected_cols);
417
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += "), got (rows=";
418
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(rows);
419
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 msg += ", cols=";
420
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 msg += std::to_string(cols);
421
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 msg += ")";
422 }
423
2/4
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3 times.
✗ Branch 3 not taken.
 3 throw std::runtime_error(msg);
424 3 }
425
426 21 [[noreturn]] void throw_quant_cols_error(const char* op, const char* tensor_name,
427 Dtype dt, int rows, int cols) {
428 21 std::string msg = "brotensor: ";
429
1/2
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
 21 msg += op;
430
1/2
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
 21 msg += ": block-quant tensor ";
431
1/2
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
 21 msg += tensor_name;
432
1/2
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
 21 msg += " (";
433
2/4
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
 21 msg += dtype_name(dt);
434
1/2
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
 21 msg += ", rows=";
435
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 21 times.
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
 21 msg += std::to_string(rows);
436
1/2
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
 21 msg += ", cols=";
437
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 21 times.
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
 21 msg += std::to_string(cols);
438
1/2
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
 21 msg += ") requires positive rows and cols with cols a multiple of block size (";
439
3/6
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 21 times.
✓ Branch 4 taken 21 times.
✗ Branch 5 not taken.
 21 msg += std::to_string(dtype_block_size(dt));
440
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 21 times.
 21 msg += ")";
441
2/4
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 21 times.
 21 throw std::runtime_error(msg);
442 21 }
443
444 } // namespace brotensor::detail
445
446 namespace brotensor {
447 ✗ int cuda_device_count() {
448 ✗ return detail::g_cuda_device_count;
449 }
450 4608 int vulkan_device_count() {
451 4608 return detail::g_vulkan_device_count;
452 }
453 }
454