GCC Code Coverage Report

Directory: brotensor/
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 86.9% 535 / 0 / 616
Functions: 94.5% 69 / 1 / 74
Branches: 59.6% 295 / 0 / 495
src/tensor.cpp
Line Branch Exec Source
1 // brotensor::Tensor implementation.
2 //
3 // All allocation, freeing, and inter-device memory traffic routes through the
4 // backend AllocVTable obtained from detail::alloc_for(device). The Tensor type
5 // itself is backend-agnostic; CUDA/Metal/CPU specifics live entirely behind
6 // the vtable indirection.
7
8 #include <brotensor/tensor.h>
9 #include <brotensor/runtime.h>
10 #include <brotensor/detail/dispatch.h>
11
12 #include <atomic>
13 #include <cmath>
14 #include <cstdint>
15 #include <limits>
16 #include <cstring>
17 #include <memory>
18 #include <stdexcept>
19 #include <string>
20 #include <vector>
21
22 #include <chrono>
23 #include <cstdio>
24 #include <cstdlib>
25
26 namespace brotensor {
27
28 // ─── env-gated weight-load profiler (BROTENSOR_TIME_LOAD=1) ──────────────────
29 // Attributes GPU weight-upload wall time to allocation vs H2D copy, accumulated
30 // across every from_host_*_on call and dumped once at process exit. Purely a
31 // diagnostic; the fast path (flag unset) is a single getenv-cached bool check.
32 namespace {
33 struct LoadProf {
34 180 bool on = false;
35 180 double alloc = 0.0, copy = 0.0, bytes = 0.0;
36 180 long calls = 0;
37 540 LoadProf() { on = std::getenv("BROTENSOR_TIME_LOAD") != nullptr; }
38 ✗ ~LoadProf() {
39 ✗ if (!on || calls == 0) return;
40 ✗ std::fprintf(stderr,
41 "[brotensor load] %ld uploads, %.2f GB: alloc %.2f s, h2d %.2f s "
42 "(%.2f GB/s copy)\n",
43 ✗ calls, bytes / 1e9, alloc, copy,
44 ✗ copy > 0 ? (bytes / 1e9) / copy : 0.0);
45 ✗ }
46 };
47 180 LoadProf g_loadprof;
48 3772 inline bool loadprof_on() { return g_loadprof.on; }
49 } // namespace
50
51 // ─── dtype + device helpers ────────────────────────────────────────────────
52
53 175802 int dtype_size_bytes(Dtype dt) {
54
8/9
✓ Branch 0 taken 93544 times.
✓ Branch 1 taken 17307 times.
✓ Branch 2 taken 11539 times.
✓ Branch 3 taken 573 times.
✓ Branch 4 taken 88 times.
✓ Branch 5 taken 52734 times.
✓ Branch 6 taken 5 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 12 times.
 175802 switch (dt) {
55 93544 case Dtype::FP32: return 4;
56 17307 case Dtype::FP16: return 2;
57 11539 case Dtype::BF16: return 2;
58 573 case Dtype::INT8: return 1;
59 88 case Dtype::FP8_E4M3: return 1;
60 52734 case Dtype::INT32: return 4;
61 5 case Dtype::F64: return 8;
62 // Quant dtypes are block-addressed; no per-element size.
63 case Dtype::Q4_0: case Dtype::Q4_1:
64 case Dtype::Q5_0: case Dtype::Q5_1:
65 case Dtype::Q8_0: case Dtype::Q8_1:
66 case Dtype::Q2_K: case Dtype::Q3_K:
67 case Dtype::Q4_K: case Dtype::Q5_K:
68 case Dtype::Q6_K: case Dtype::Q8_K:
69 12 return 0;
70 }
71 ✗ return 0;
72 175802 }
73
74 388923 bool dtype_is_quant(Dtype dt) {
75
2/2
✓ Branch 0 taken 977 times.
✓ Branch 1 taken 387946 times.
 388923 switch (dt) {
76 case Dtype::Q4_0: case Dtype::Q4_1:
77 case Dtype::Q5_0: case Dtype::Q5_1:
78 case Dtype::Q8_0: case Dtype::Q8_1:
79 case Dtype::Q2_K: case Dtype::Q3_K:
80 case Dtype::Q4_K: case Dtype::Q5_K:
81 case Dtype::Q6_K: case Dtype::Q8_K:
82 977 return true;
83 default:
84 387946 return false;
85 }
86 388923 }
87
88 27 const char* dtype_name(Dtype dt) {
89
3/20
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 8 times.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✓ Branch 15 taken 13 times.
✗ Branch 16 not taken.
✓ Branch 17 taken 6 times.
✗ Branch 18 not taken.
✗ Branch 19 not taken.
 27 switch (dt) {
90 ✗ case Dtype::FP32: return "FP32";
91 ✗ case Dtype::FP16: return "FP16";
92 ✗ case Dtype::BF16: return "BF16";
93 ✗ case Dtype::INT8: return "INT8";
94 ✗ case Dtype::INT32: return "INT32";
95 ✗ case Dtype::F64: return "F64";
96 ✗ case Dtype::FP8_E4M3: return "FP8_E4M3";
97 ✗ case Dtype::Q4_0: return "Q4_0";
98 ✗ case Dtype::Q4_1: return "Q4_1";
99 ✗ case Dtype::Q5_0: return "Q5_0";
100 ✗ case Dtype::Q5_1: return "Q5_1";
101 8 case Dtype::Q8_0: return "Q8_0";
102 ✗ case Dtype::Q8_1: return "Q8_1";
103 ✗ case Dtype::Q2_K: return "Q2_K";
104 ✗ case Dtype::Q3_K: return "Q3_K";
105 13 case Dtype::Q4_K: return "Q4_K";
106 ✗ case Dtype::Q5_K: return "Q5_K";
107 6 case Dtype::Q6_K: return "Q6_K";
108 ✗ case Dtype::Q8_K: return "Q8_K";
109 }
110 ✗ return "?";
111 27 }
112
113 1150 int dtype_block_size(Dtype dt) {
114
3/3
✓ Branch 0 taken 786 times.
✓ Branch 1 taken 358 times.
✓ Branch 2 taken 6 times.
 1150 switch (dt) {
115 case Dtype::Q4_0: case Dtype::Q4_1:
116 case Dtype::Q5_0: case Dtype::Q5_1:
117 case Dtype::Q8_0: case Dtype::Q8_1:
118 786 return 32;
119 case Dtype::Q2_K: case Dtype::Q3_K:
120 case Dtype::Q4_K: case Dtype::Q5_K:
121 case Dtype::Q6_K: case Dtype::Q8_K:
122 358 return 256;
123 default:
124 6 return 1;
125 }
126 1150 }
127
128 440 int dtype_block_bytes(Dtype dt) {
129
13/13
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 18 times.
✓ Branch 2 taken 6 times.
✓ Branch 3 taken 10 times.
✓ Branch 4 taken 5 times.
✓ Branch 5 taken 5 times.
✓ Branch 6 taken 316 times.
✓ Branch 7 taken 5 times.
✓ Branch 8 taken 5 times.
✓ Branch 9 taken 5 times.
✓ Branch 10 taken 50 times.
✓ Branch 11 taken 5 times.
✓ Branch 12 taken 5 times.
 440 switch (dt) {
130 10 case Dtype::Q4_0: return 18;
131 5 case Dtype::Q4_1: return 20;
132 5 case Dtype::Q5_0: return 22;
133 5 case Dtype::Q5_1: return 24;
134 316 case Dtype::Q8_0: return 34;
135 5 case Dtype::Q8_1: return 36;
136 5 case Dtype::Q2_K: return 82;
137 5 case Dtype::Q3_K: return 110;
138 50 case Dtype::Q4_K: return 144;
139 5 case Dtype::Q5_K: return 176;
140 18 case Dtype::Q6_K: return 210;
141 5 case Dtype::Q8_K: return 292;
142 6 default: return dtype_size_bytes(dt);
143 }
144 440 }
145
146 175626 std::size_t dtype_storage_bytes(Dtype d, std::int64_t numel) {
147
2/2
✓ Branch 0 taken 175624 times.
✓ Branch 1 taken 2 times.
 175626 if (numel < 0) {
148
1/2
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
 2 throw std::runtime_error("brotensor: dtype_storage_bytes: negative numel");
149 }
150
2/2
✓ Branch 0 taken 420 times.
✓ Branch 1 taken 175204 times.
 175624 if (!dtype_is_quant(d)) {
151 350408 return static_cast<std::size_t>(numel) *
152 175204 static_cast<std::size_t>(dtype_size_bytes(d));
153 }
154 420 const int bs = dtype_block_size(d);
155 420 const int bb = dtype_block_bytes(d);
156
2/2
✓ Branch 0 taken 407 times.
✓ Branch 1 taken 13 times.
 420 if (numel % bs != 0) {
157
1/2
✓ Branch 0 taken 13 times.
✗ Branch 1 not taken.
 13 throw std::runtime_error(
158 "brotensor: dtype_storage_bytes: numel not a multiple of block size");
159 }
160 814 return (static_cast<std::size_t>(numel) / static_cast<std::size_t>(bs)) *
161 407 static_cast<std::size_t>(bb);
162 175611 }
163
164 const Device Device::CPU{DeviceType::CPU, 0};
165 const Device Device::CUDA{DeviceType::CUDA, 0};
166 const Device Device::Metal{DeviceType::Metal, 0};
167 const Device Device::VULKAN{DeviceType::VULKAN, 0};
168
169 67 const char* device_name(Device d) {
170
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 59 times.
 67 if (d.is_cpu()) return "CPU";
171
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 58 times.
 59 if (d.is_cuda()) {
172
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 if (d.index == 0) return "CUDA";
173 static thread_local char buf[32];
174 ✗ std::snprintf(buf, sizeof(buf), "CUDA:%d", d.index);
175 ✗ return buf;
176 }
177
2/2
✓ Branch 0 taken 55 times.
✓ Branch 1 taken 3 times.
 58 if (d.is_metal()) {
178
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 55 times.
 55 if (d.index == 0) return "Metal";
179 static thread_local char buf[32];
180 ✗ std::snprintf(buf, sizeof(buf), "Metal:%d", d.index);
181 ✗ return buf;
182 }
183
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 if (d.is_vulkan()) {
184
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3 times.
 3 if (d.index == 0) return "vulkan";
185 static thread_local char buf[32];
186 ✗ std::snprintf(buf, sizeof(buf), "vulkan:%d", d.index);
187 ✗ return buf;
188 }
189 ✗ return "?";
190 67 }
191
192 ✗ std::string to_string(Device d) {
193 ✗ return device_name(d);
194 }
195
196 // ─── FP16 ↔ FP32 conversion (host-side IEEE 754 binary16) ──────────────────
197
198 614281788 uint16_t fp32_to_fp16_bits(float v) {
199 uint32_t x;
200 614281788 std::memcpy(&x, &v, 4);
201 614281788 const uint32_t sign = (x >> 16) & 0x8000u;
202 614281788 int32_t exp = static_cast<int32_t>((x >> 23) & 0xFFu) - 127 + 15;
203 614281788 uint32_t mant = x & 0x7FFFFFu;
204
205
2/2
✓ Branch 0 taken 22702240 times.
✓ Branch 1 taken 591579548 times.
 614281788 if (((x >> 23) & 0xFFu) == 0xFFu) {
206 // Inf or NaN.
207 22702240 uint16_t out = static_cast<uint16_t>(sign | 0x7C00u);
208
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22702240 times.
 22702240 if (mant) out |= 0x0200u; // quiet NaN
209 22702240 return out;
210 }
211
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 591579548 times.
 591579548 if (exp >= 0x1F) {
212 ✗ return static_cast<uint16_t>(sign | 0x7C00u);
213 }
214
2/2
✓ Branch 0 taken 641333 times.
✓ Branch 1 taken 590938215 times.
 591579548 if (exp <= 0) {
215
2/2
✓ Branch 0 taken 400000 times.
✓ Branch 1 taken 241333 times.
 641333 if (exp < -10) return static_cast<uint16_t>(sign);
216 241333 mant |= 0x800000u;
217 241333 const int shift = 14 - exp;
218 // Round to nearest, ties to even, as the normal path and the device's
219 // __float2half do (half up made a tie one subnormal ulp off).
220 241333 const uint32_t round = (1u << (shift - 1)) - 1u + ((mant >> shift) & 1u);
221 241333 uint32_t r = (mant + round) >> shift;
222 241333 return static_cast<uint16_t>(sign | r);
223 }
224 590938215 const uint32_t lsb = (mant >> 13) & 1u;
225 590938215 const uint32_t round = 0x00001000u + lsb - 1u;
226 590938215 uint32_t r_mant = (mant + round) >> 13;
227 590938215 uint32_t r_exp = static_cast<uint32_t>(exp);
228
2/2
✓ Branch 0 taken 590630836 times.
✓ Branch 1 taken 307379 times.
 590938215 if (r_mant & 0x400u) {
229 307379 r_mant = 0;
230 307379 r_exp += 1;
231
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 307379 times.
 307379 if (r_exp >= 0x1F) return static_cast<uint16_t>(sign | 0x7C00u);
232 307379 }
233 590938215 return static_cast<uint16_t>(sign | (r_exp << 10) | r_mant);
234 614281788 }
235
236 378716328 float fp16_bits_to_fp32(uint16_t bits) {
237 378716328 const uint32_t sign = (static_cast<uint32_t>(bits) & 0x8000u) << 16;
238 378716328 uint32_t exp = (bits >> 10) & 0x1Fu;
239 378716328 uint32_t mant = bits & 0x3FFu;
240 uint32_t out;
241
2/2
✓ Branch 0 taken 377982303 times.
✓ Branch 1 taken 734025 times.
 378716328 if (exp == 0) {
242
2/2
✓ Branch 0 taken 490489 times.
✓ Branch 1 taken 243536 times.
 734025 if (mant == 0) {
243 243536 out = sign;
244 243536 } else {
245
2/2
✓ Branch 0 taken 1311064 times.
✓ Branch 1 taken 490489 times.
 1801553 while ((mant & 0x400u) == 0) { mant <<= 1; exp -= 1; }
246 490489 mant &= 0x3FFu;
247 490489 uint32_t e = (exp + (127 - 15) + 1) & 0xFFu;
248 490489 out = sign | (e << 23) | (mant << 13);
249 }
250
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 377982303 times.
 378716328 } else if (exp == 0x1F) {
251 ✗ out = sign | 0x7F800000u | (mant << 13);
252 ✗ } else {
253 377982303 uint32_t e = exp + (127 - 15);
254 377982303 out = sign | (e << 23) | (mant << 13);
255 }
256 float f;
257 378716328 std::memcpy(&f, &out, 4);
258 378716328 return f;
259 }
260
261 // ─── BF16 ↔ FP32 conversion (host-side bfloat16) ───────────────────────────
262
263 289371384 uint16_t fp32_to_bf16_bits(float v) {
264 uint32_t x;
265 289371384 std::memcpy(&x, &v, 4);
266 // NaN: keep it a NaN (the rounding add below could otherwise carry a
267 // NaN's mantissa into the exponent and produce an infinity).
268
4/4
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 289371380 times.
✓ Branch 2 taken 2 times.
✓ Branch 3 taken 2 times.
 289371384 if (((x >> 23) & 0xFFu) == 0xFFu && (x & 0x7FFFFFu) != 0) {
269 2 return static_cast<uint16_t>((x >> 16) | 0x0040u);
270 }
271 // Round to nearest, ties to even: add 0x7FFF + LSB-of-result.
272 289371382 const uint32_t rounding_bias = 0x7FFFu + ((x >> 16) & 1u);
273 289371382 x += rounding_bias;
274 289371382 return static_cast<uint16_t>(x >> 16);
275 289371384 }
276
277 159841288 float bf16_bits_to_fp32(uint16_t bits) {
278 159841288 const uint32_t x = static_cast<uint32_t>(bits) << 16;
279 float f;
280 159841288 std::memcpy(&f, &x, 4);
281 159841288 return f;
282 }
283
284 225668027 float fp8_e4m3_bits_to_fp32(uint8_t bits) {
285 225668027 const int s = bits >> 7, e = (bits >> 3) & 15, m = bits & 7;
286
3/4
✓ Branch 0 taken 41381926 times.
✓ Branch 1 taken 184286101 times.
✓ Branch 2 taken 41381926 times.
✗ Branch 3 not taken.
 225668027 if (e == 15 && m == 7) return std::numeric_limits<float>::quiet_NaN();
287
2/2
✓ Branch 0 taken 225655998 times.
✓ Branch 1 taken 12029 times.
 225668027 const float mag = e == 0 ? std::ldexp(static_cast<float>(m), -9) // m/8 * 2^-6
288 225655998 : std::ldexp(1.0f + m / 8.0f, e - 7);
289
2/2
✓ Branch 0 taken 17078696 times.
✓ Branch 1 taken 208589331 times.
 225668027 return s ? -mag : mag;
290 225668027 }
291
292 22627276 uint8_t fp32_to_fp8_e4m3_bits(float v) {
293
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22627276 times.
 22627276 if (std::isnan(v)) return 0x7f;
294 22627276 const uint8_t sign = std::signbit(v) ? 0x80 : 0x00;
295 22627276 const float a = std::fabs(v);
296
2/2
✓ Branch 0 taken 65701 times.
✓ Branch 1 taken 22561575 times.
 22627276 if (a >= 448.0f) return sign | 0x7e;
297 // Magnitude codes 0..126 ascend with their value: binary search the
298 // bracket, then round to the nearer, ties to the even code.
299 22561575 int lo = 0, hi = 126;
300
2/2
✓ Branch 0 taken 157917603 times.
✓ Branch 1 taken 22561575 times.
 180479178 while (hi - lo > 1) {
301 157917603 const int mid = (lo + hi) / 2;
302
2/2
✓ Branch 0 taken 106727568 times.
✓ Branch 1 taken 51190035 times.
 157917603 if (fp8_e4m3_bits_to_fp32(static_cast<uint8_t>(mid)) <= a) lo = mid;
303 51190035 else hi = mid;
304 }
305 22561575 const float dl = a - fp8_e4m3_bits_to_fp32(static_cast<uint8_t>(lo));
306 22561575 const float dh = fp8_e4m3_bits_to_fp32(static_cast<uint8_t>(hi)) - a;
307
6/6
✓ Branch 0 taken 11528642 times.
✓ Branch 1 taken 11032933 times.
✓ Branch 2 taken 10827776 times.
✓ Branch 3 taken 205157 times.
✓ Branch 4 taken 3659 times.
✓ Branch 5 taken 201498 times.
 22561575 const int code = dl < dh ? lo : dh < dl ? hi : ((lo & 1) ? hi : lo);
308 22561575 return sign | static_cast<uint8_t>(code);
309 22627276 }
310
311 // ─── internal helpers ──────────────────────────────────────────────────────
312
313 namespace {
314
315 45 [[noreturn]] void throw_msg(const std::string& m) {
316
1/2
✓ Branch 0 taken 45 times.
✗ Branch 1 not taken.
 45 throw std::runtime_error(m);
317 ✗ }
318
319 32636481923 void check_host(const Tensor& t, const char* who) {
320
2/2
✓ Branch 0 taken 32636481913 times.
✓ Branch 1 taken 10 times.
 32636481923 if (t.device != Device::CPU) {
321 10 std::string m = "brotensor: ";
322
1/2
✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
 10 m += who;
323
1/2
✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
 10 m += ": tensor is on ";
324
2/4
✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 10 times.
✗ Branch 3 not taken.
 10 m += device_name(t.device);
325
1/2
✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
 10 m += ", not CPU";
326
1/2
✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
 10 throw_msg(m);
327 10 }
328 32636481923 }
329
330 // Guard against negative dimensions before they are cast to std::size_t in
331 // bytes() — an unchecked (size_t)(-1) underflows to an astronomical count
332 // that sails past the bytes==0 short-circuits and reaches backend_alloc.
333 213807 void check_dims(int r, int c, const char* who) {
334
2/2
✓ Branch 0 taken 213800 times.
✓ Branch 1 taken 7 times.
 213807 if (r < 0 || c < 0) {
335 7 std::string m = "brotensor: ";
336
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 m += who;
337
1/2
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
 7 m += ": negative dimension";
338
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 throw_msg(m);
339 7 }
340 213807 }
341
342 213045 void check_quant_cols(Dtype dt, int c, const char* who) {
343
3/4
✓ Branch 0 taken 408 times.
✓ Branch 1 taken 212637 times.
✓ Branch 2 taken 408 times.
✗ Branch 3 not taken.
 213045 if (!dtype_is_quant(dt) || c <= 0) return;
344 408 const int bs = dtype_block_size(dt);
345
2/2
✓ Branch 0 taken 402 times.
✓ Branch 1 taken 6 times.
 408 if (c % bs != 0) {
346 6 std::string m = "brotensor: ";
347
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 m += who;
348
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 m += ": ";
349
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 m += dtype_name(dt);
350
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 m += " cols (";
351
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
 6 m += std::to_string(c);
352
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 m += ") is not a multiple of block size (";
353
2/4
✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
✓ Branch 2 taken 6 times.
✗ Branch 3 not taken.
 6 m += std::to_string(bs);
354
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 m += ")";
355
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 throw_msg(m);
356 6 }
357 213045 }
358
359 32636379548 void check_dtype(const Tensor& t, Dtype expected, const char* who) {
360
2/2
✓ Branch 0 taken 32636379539 times.
✓ Branch 1 taken 9 times.
 32636379548 if (t.dtype != expected) {
361 9 std::string m = "brotensor: ";
362
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 m += who;
363
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 m += ": dtype mismatch";
364
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
 9 throw_msg(m);
365 9 }
366 32636379548 }
367
368 // Allocation accounting. Two relaxed atomics on a path that is already an
369 // allocator call, so the cost is nil; see AllocStats in tensor.h for why the
370 // counters exist at all.
371 std::atomic<std::uint64_t> g_alloc_count{0};
372 std::atomic<std::uint64_t> g_alloc_bytes{0};
373
374 100285 void* backend_alloc(Device d, std::size_t bytes) {
375
2/2
✓ Branch 0 taken 99491 times.
✓ Branch 1 taken 794 times.
 100285 if (bytes == 0) return nullptr;
376 99491 g_alloc_count.fetch_add(1, std::memory_order_relaxed);
377 99491 g_alloc_bytes.fetch_add(bytes, std::memory_order_relaxed);
378 99491 return detail::alloc_for(d).alloc(bytes, d.index);
379 100285 }
380
381 98615 void backend_free(Device d, void* p) {
382
1/2
✓ Branch 0 taken 98615 times.
✗ Branch 1 not taken.
 98615 if (!p) return;
383 98615 detail::alloc_for(d).free(p, d.index);
384 98615 }
385
386 19067 void backend_zero(Device d, void* p, std::size_t bytes) {
387
3/4
✓ Branch 0 taken 19062 times.
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 19062 times.
 19067 if (bytes == 0 || !p) return;
388 19062 detail::alloc_for(d).memset_zero(p, bytes, d.index);
389 19067 }
390
391 } // namespace
392
393 // ─── jit symbolic-tensor tracking ──────────────────────────────────────────
394
395 namespace detail {
396 // Null until brotensor::jit starts a trace. See tensor.h.
397 JitSlotRetargetHook jit_slot_retarget = nullptr;
398 JitClampInplaceHook jit_clamp_inplace_hook = nullptr;
399 JitCastOutHook jit_cast_out_hook = nullptr;
400 } // namespace detail
401
402 namespace {
403
404 // Tensor is 48 bytes on a 64-bit build. The only hole in the 40-byte layout
405 // it had before `jit_slot` was the three bytes after `owns_`, which a
406 // four-byte field cannot use, so the tag cost one alignment word. Pinned so a
407 // future field lands by decision rather than by accident.
408 static_assert(sizeof(void*) != 8 || sizeof(Tensor) == 48,
409 "brotensor::Tensor layout changed unexpectedly");
410
411 1 [[noreturn]] void throw_symbolic(const char* who) {
412 1 std::string m = "brotensor: ";
413
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 m += who;
414
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 m += ": this Tensor is a traced intermediate, not a tensor. It stands for "
415 "a node of the trace being recorded between begin_trace() and "
416 "end_trace(), owns no memory, and has no contents to read or copy — "
417 "two Tensors cannot both become the one buffer end_trace() allocates "
418 "for it. Use it in the expression, move it, or store() it into a "
419 "buffer you already own.";
420
2/4
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 time.
 1 throw std::runtime_error(m);
421 1 }
422
423 76205 inline void check_not_symbolic(const Tensor& t, const char* who) {
424
2/2
✓ Branch 0 taken 76204 times.
✓ Branch 1 taken 1 time.
 76205 if (t.jit_slot >= 0) throw_symbolic(who);
425 76204 }
426
427 } // namespace
428
429 // ─── Tensor lifetime ───────────────────────────────────────────────────────
430
431 209876 Tensor::~Tensor() {
432
3/4
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 104916 times.
✓ Branch 2 taken 22 times.
✗ Branch 3 not taken.
 104938 if (jit_slot >= 0 && detail::jit_slot_retarget) {
433
1/2
✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
 22 detail::jit_slot_retarget(jit_slot, this, nullptr);
434 22 }
435
1/2
✓ Branch 0 taken 104938 times.
✗ Branch 1 not taken.
 104938 release_();
436 209876 }
437
438 173526 void Tensor::release_() {
439
3/4
✓ Branch 0 taken 98615 times.
✓ Branch 1 taken 74911 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 98615 times.
 173526 if (owns_ && data) {
440 // alloc_for can throw if the backend was somehow un-registered between
441 // alloc and free; let it propagate — better than leaking a different
442 // backend's pointer.
443 98615 backend_free(device, data);
444 98615 }
445 173526 data = nullptr;
446 173526 owns_ = false;
447 173526 cap_bytes_ = 0;
448 173526 }
449
450 // Move: a symbolic traced intermediate moves like any other tensor, and the
451 // jit's slot -> Tensor* table is retargeted onto the new object so end_trace()
452 // still knows where the caller is holding that value.
453 592 Tensor::Tensor(Tensor&& o) noexcept
454 296 : data(o.data), rows(o.rows), cols(o.cols),
455 296 dtype(o.dtype), device(o.device), jit_slot(o.jit_slot), owns_(o.owns_),
456 592 cap_bytes_(o.cap_bytes_) {
457 296 o.data = nullptr;
458 296 o.rows = 0;
459 296 o.cols = 0;
460 296 o.owns_ = false;
461 296 o.cap_bytes_ = 0;
462
1/2
✓ Branch 0 taken 296 times.
✗ Branch 1 not taken.
 296 if (jit_slot >= 0) {
463 ✗ o.jit_slot = -1;
464 ✗ if (detail::jit_slot_retarget) detail::jit_slot_retarget(jit_slot, &o, this);
465 ✗ }
466 592 }
467
468 6134 Tensor& Tensor::operator=(Tensor&& o) noexcept {
469
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 6133 times.
 6134 if (this != &o) {
470
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 6133 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 6133 if (jit_slot >= 0 && detail::jit_slot_retarget) {
471 ✗ detail::jit_slot_retarget(jit_slot, this, nullptr);
472 ✗ }
473 6133 jit_slot = -1;
474
1/2
✓ Branch 0 taken 6133 times.
✗ Branch 1 not taken.
 6133 release_();
475 6133 data = o.data;
476 6133 rows = o.rows;
477 6133 cols = o.cols;
478 6133 dtype = o.dtype;
479 6133 device = o.device;
480 6133 owns_ = o.owns_;
481 6133 cap_bytes_ = o.cap_bytes_;
482 6133 o.data = nullptr;
483 6133 o.rows = 0;
484 6133 o.cols = 0;
485 6133 o.owns_ = false;
486 6133 o.cap_bytes_ = 0;
487
2/2
✓ Branch 0 taken 6132 times.
✓ Branch 1 taken 1 time.
 6133 if (o.jit_slot >= 0) {
488 1 jit_slot = o.jit_slot;
489 1 o.jit_slot = -1;
490
2/4
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 if (detail::jit_slot_retarget) detail::jit_slot_retarget(jit_slot, &o, this);
491 1 }
492 6133 }
493 6134 return *this;
494 }
495
496 // Copy ctor / assignment: device-aware deep copy via clone(). Implemented in
497 // terms of the move assignment so all the ownership bookkeeping lives in one
498 // place. A symbolic traced intermediate has no contents to copy and cannot be
499 // owned twice, so copying one is rejected rather than silently producing an
500 // empty tensor.
501 1716 Tensor::Tensor(const Tensor& o) {
502 858 check_not_symbolic(o, "copy");
503 858 *this = o.clone();
504 858 }
505
506 47 Tensor& Tensor::operator=(const Tensor& o) {
507
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 46 times.
 47 if (this != &o) {
508 46 check_not_symbolic(o, "copy");
509 46 *this = o.clone();
510 46 }
511 47 return *this;
512 }
513
514 112273 std::size_t Tensor::bytes() const {
515 112273 check_dims(rows, cols, "bytes");
516
3/4
✓ Branch 0 taken 111521 times.
✓ Branch 1 taken 752 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 111521 times.
 112273 if (rows > 0 && cols > 0) {
517 111521 check_quant_cols(dtype, cols, "Tensor::bytes");
518 111521 }
519 112273 return dtype_storage_bytes(
520 112273 dtype,
521 112273 static_cast<std::int64_t>(rows) * static_cast<std::int64_t>(cols));
522 }
523
524 3 std::size_t Tensor::row_bytes() const {
525 3 check_dims(rows, cols, "row_bytes");
526 3 check_quant_cols(dtype, cols, "Tensor::row_bytes");
527 3 return dtype_storage_bytes(dtype, static_cast<std::int64_t>(cols));
528 }
529
530 // ─── Factories ─────────────────────────────────────────────────────────────
531
532 37830 Tensor Tensor::empty_on(Device d, int r, int c, Dtype dt) {
533 37830 check_dims(r, c, "empty_on");
534 37830 check_quant_cols(dt, c, "Tensor::empty_on");
535 37830 d = detail::resolve_device_alias(d);
536 37830 Tensor t;
537 37830 t.device = d;
538 37830 t.dtype = dt;
539 37830 t.rows = r;
540 37830 t.cols = c;
541
2/4
✓ Branch 0 taken 37830 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 37830 times.
✗ Branch 3 not taken.
 37830 t.data = backend_alloc(d, t.bytes());
542 37830 t.owns_ = (t.data != nullptr);
543
3/4
✓ Branch 0 taken 37097 times.
✓ Branch 1 taken 733 times.
✓ Branch 2 taken 37097 times.
✗ Branch 3 not taken.
 37830 t.cap_bytes_ = t.owns_ ? t.bytes() : 0;
544 37830 return t;
545 37830 }
546
547 18703 Tensor Tensor::zeros_on(Device d, int r, int c, Dtype dt) {
548 18703 Tensor t = empty_on(d, r, c, dt);
549
2/4
✓ Branch 0 taken 18703 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 18703 times.
✗ Branch 3 not taken.
 18703 backend_zero(d, t.data, t.bytes());
550 18703 return t;
551 18703 }
552
553 1 Tensor Tensor::empty(int r, int c, Dtype dt) {
554 1 return empty_on(default_device(), r, c, dt);
555 }
556
557 5 Tensor Tensor::zeros(int r, int c, Dtype dt) {
558 5 return zeros_on(default_device(), r, c, dt);
559 }
560
561 namespace {
562 // Shared upload core for from_host_*_on. When BROTENSOR_TIME_LOAD is set it
563 // attributes host wall time to the empty_on (alloc enqueue) and the H2D copy.
564 3772 Tensor from_host_typed(Device d, const void* src, int r, int c, Dtype dt) {
565 using clock = std::chrono::steady_clock;
566
1/2
✓ Branch 0 taken 3772 times.
✗ Branch 1 not taken.
 3772 const bool prof = loadprof_on() && d != Device::CPU;
567 3772 auto t0 = clock::now();
568 3772 Tensor t = Tensor::empty_on(d, r, c, dt);
569
1/2
✓ Branch 0 taken 3772 times.
✗ Branch 1 not taken.
 3772 const std::size_t n = t.bytes();
570
1/2
✓ Branch 0 taken 3772 times.
✗ Branch 1 not taken.
 3772 if (n == 0) return t;
571 3772 auto t1 = clock::now();
572
3/4
✓ Branch 0 taken 3772 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3131 times.
✓ Branch 3 taken 641 times.
 3772 if (d == Device::CPU) {
573 641 std::memcpy(t.data, src, n);
574 641 } else {
575
2/4
✓ Branch 0 taken 3131 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3131 times.
✗ Branch 3 not taken.
 3131 detail::alloc_for(d).memcpy_h2d(t.data, src, n, d.index);
576 }
577
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3772 times.
 3772 if (prof) {
578 ✗ auto t2 = clock::now();
579 ✗ g_loadprof.alloc += std::chrono::duration<double>(t1 - t0).count();
580 ✗ g_loadprof.copy += std::chrono::duration<double>(t2 - t1).count();
581 ✗ g_loadprof.bytes += static_cast<double>(n);
582 ✗ ++g_loadprof.calls;
583 ✗ }
584 3772 return t;
585 3772 }
586 } // namespace
587
588 1894 Tensor Tensor::from_host_on(Device d, const float* src, int r, int c) {
589 1894 return from_host_typed(d, src, r, c, Dtype::FP32);
590 }
591
592 1503 Tensor Tensor::from_host_fp16_on(Device d, const uint16_t* src, int r, int c) {
593 1503 return from_host_typed(d, src, r, c, Dtype::FP16);
594 }
595
596 375 Tensor Tensor::from_host_bf16_on(Device d, const uint16_t* src, int r, int c) {
597 375 return from_host_typed(d, src, r, c, Dtype::BF16);
598 }
599
600 23 Tensor Tensor::from_host_int8_on(Device d, const int8_t* src, int r, int c) {
601 23 Tensor t = empty_on(d, r, c, Dtype::INT8);
602
1/2
✓ Branch 0 taken 23 times.
✗ Branch 1 not taken.
 23 const std::size_t n = t.bytes();
603
1/2
✓ Branch 0 taken 23 times.
✗ Branch 1 not taken.
 23 if (n == 0) return t;
604
2/4
✓ Branch 0 taken 23 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 23 times.
✗ Branch 3 not taken.
 23 if (d == Device::CPU) {
605 ✗ std::memcpy(t.data, src, n);
606 ✗ } else {
607
2/4
✓ Branch 0 taken 23 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 23 times.
✗ Branch 3 not taken.
 23 detail::alloc_for(d).memcpy_h2d(t.data, src, n, d.index);
608 }
609 23 return t;
610 23 }
611
612 101 Tensor Tensor::from_raw_bytes_on(Device target, const void* src, int r, int c,
613 Dtype dt, std::size_t nbytes) {
614 101 Tensor t = empty_on(target, r, c, dt);
615
3/4
✓ Branch 0 taken 101 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 100 times.
✓ Branch 3 taken 1 time.
 101 if (t.bytes() != nbytes) {
616
2/4
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 time.
 1 throw std::runtime_error(
617 "brotensor: from_raw_bytes_on: nbytes does not match tensor size");
618 }
619
2/2
✓ Branch 0 taken 99 times.
✓ Branch 1 taken 1 time.
 100 if (nbytes == 0) return t;
620
3/4
✓ Branch 0 taken 99 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 90 times.
✓ Branch 3 taken 9 times.
 99 if (target == Device::CPU) {
621 9 std::memcpy(t.data, src, nbytes);
622 9 } else {
623
2/4
✓ Branch 0 taken 90 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 90 times.
✗ Branch 3 not taken.
 90 detail::alloc_for(target).memcpy_h2d(t.data, src, nbytes, target.index);
624 }
625 99 return t;
626 101 }
627
628 21 Tensor Tensor::from_host(const float* src, int r, int c) {
629 21 return from_host_on(default_device(), src, r, c);
630 }
631
632 14 Tensor Tensor::from_host_fp16(const uint16_t* src, int r, int c) {
633 14 return from_host_fp16_on(default_device(), src, r, c);
634 }
635
636 5 Tensor Tensor::from_host_bf16(const uint16_t* src, int r, int c) {
637 5 return from_host_bf16_on(default_device(), src, r, c);
638 }
639
640 ✗ Tensor Tensor::from_host_int8(const int8_t* src, int r, int c) {
641 ✗ return from_host_int8_on(default_device(), src, r, c);
642 }
643
644 392 Tensor Tensor::view(Device d, void* data, int r, int c, Dtype dt) {
645 392 check_dims(r, c, "view");
646 392 check_quant_cols(dt, c, "Tensor::view");
647 392 d = detail::resolve_device_alias(d);
648 392 Tensor t;
649 392 t.device = d;
650 392 t.dtype = dt;
651 392 t.rows = r;
652 392 t.cols = c;
653 392 t.data = data;
654 392 t.owns_ = false;
655 392 return t;
656 392 }
657
658 // ─── Migration ─────────────────────────────────────────────────────────────
659
660 2857 Tensor Tensor::clone() const {
661 2857 check_not_symbolic(*this, "clone");
662 2857 Tensor t = empty_on(device, rows, cols, dtype);
663
1/2
✓ Branch 0 taken 2857 times.
✗ Branch 1 not taken.
 2857 const std::size_t n = bytes();
664
3/4
✓ Branch 0 taken 2854 times.
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 2854 times.
✗ Branch 3 not taken.
 2857 if (n == 0 || !data) return t;
665
3/4
✓ Branch 0 taken 2854 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 20 times.
✓ Branch 3 taken 2834 times.
 2854 if (device == Device::CPU) {
666 2834 std::memcpy(t.data, data, n);
667 2834 } else {
668
2/4
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 20 times.
✗ Branch 3 not taken.
 20 detail::alloc_for(device).memcpy_d2d(t.data, data, n, device.index);
669 }
670 2854 return t;
671 2857 }
672
673 9146 Tensor Tensor::to(Device target) const {
674 9146 check_not_symbolic(*this, "to");
675
2/2
✓ Branch 0 taken 49 times.
✓ Branch 1 taken 9097 times.
 9146 if (target == device) return clone();
676 9097 Tensor t = empty_on(target, rows, cols, dtype);
677
1/2
✓ Branch 0 taken 9097 times.
✗ Branch 1 not taken.
 9097 const std::size_t n = bytes();
678
2/4
✓ Branch 0 taken 9097 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9097 times.
✗ Branch 3 not taken.
 9097 if (n == 0 || !data) return t;
679
680
3/4
✓ Branch 0 taken 9097 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 5646 times.
✓ Branch 3 taken 3451 times.
 9097 if (device.is_cpu()) {
681 // CPU → GPU.
682
2/4
✓ Branch 0 taken 5646 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 5646 times.
✗ Branch 3 not taken.
 5646 detail::alloc_for(target).memcpy_h2d(t.data, data, n, target.index);
683
2/4
✓ Branch 0 taken 3451 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3451 times.
✗ Branch 3 not taken.
 9097 } else if (target.is_cpu()) {
684 // GPU → CPU.
685
2/4
✓ Branch 0 taken 3451 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3451 times.
✗ Branch 3 not taken.
 3451 detail::alloc_for(device).memcpy_d2h(t.data, data, n, device.index);
686
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 3451 } else if (device.type == target.type) {
687 // Direct peer copy between two GPUs of one backend (CUDA or Vulkan;
688 // Metal has a single device and never gets here).
689 ✗ auto peer_fn = detail::alloc_for(target).memcpy_peer;
690 ✗ if (peer_fn) {
691 ✗ peer_fn(t.data, target.index, data, device.index, n);
692 ✗ } else {
693 ✗ std::unique_ptr<unsigned char[]> staging(new unsigned char[n]);
694 ✗ detail::alloc_for(device).memcpy_d2h(staging.get(), data, n, device.index);
695 ✗ detail::alloc_for(target).memcpy_h2d(t.data, staging.get(), n, target.index);
696 ✗ }
697 ✗ } else {
698 // GPU → different GPU backend. Bounce through host.
699 ✗ std::unique_ptr<unsigned char[]> staging(new unsigned char[n]);
700 ✗ detail::alloc_for(device).memcpy_d2h(staging.get(), data, n, device.index);
701 ✗ detail::alloc_for(target).memcpy_h2d(t.data, staging.get(), n, target.index);
702 ✗ }
703 9097 return t;
704 18243 }
705
706 // ─── Mutators ──────────────────────────────────────────────────────────────
707
708 364 void Tensor::zero() {
709 364 backend_zero(device, data, bytes());
710 364 }
711
712 63295 void Tensor::resize(int r, int c, Dtype dt) {
713 63295 check_not_symbolic(*this, "resize");
714 63295 check_dims(r, c, "resize");
715 63295 check_quant_cols(dt, c, "Tensor::resize");
716
8/8
✓ Branch 0 taken 432 times.
✓ Branch 1 taken 62863 times.
✓ Branch 2 taken 86 times.
✓ Branch 3 taken 346 times.
✓ Branch 4 taken 70 times.
✓ Branch 5 taken 16 times.
✓ Branch 6 taken 55 times.
✓ Branch 7 taken 15 times.
 63295 if (r == rows && c == cols && dt == dtype && data != nullptr) return;
717 63280 const std::size_t new_bytes = dtype_storage_bytes(
718 63280 dt,
719 63280 static_cast<std::int64_t>(r) * static_cast<std::int64_t>(c));
720 // A non-owning view over real storage cannot be reshaped: reallocating
721 // would silently allocate fresh owned memory and sever the view, leaving
722 // callers with a tensor that no longer aliases what they passed to
723 // view(). Reject it explicitly rather than converting it in place.
724 // (A default-constructed / released tensor — owns_ == false but
725 // data == nullptr — is not a view and resizes normally.)
726
4/4
✓ Branch 0 taken 62040 times.
✓ Branch 1 taken 1240 times.
✓ Branch 2 taken 62038 times.
✓ Branch 3 taken 2 times.
 63280 if (!owns_ && data != nullptr) {
727
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 throw_msg("brotensor: resize: cannot reshape a non-owning view; "
728 "allocate a fresh tensor or re-view() with the new shape");
729 }
730 // Keep the existing storage whenever the new shape fits — the buffer
731 // stabilises at its high-water mark and its device pointer stays put
732 // (required when the op sequence using it is CUDA-graph captured).
733
4/4
✓ Branch 0 taken 884 times.
✓ Branch 1 taken 62394 times.
✓ Branch 2 taken 823 times.
✓ Branch 3 taken 61 times.
 63278 if (new_bytes > cap_bytes_ || !owns_) {
734 62455 release_();
735 62455 device = detail::resolve_device_alias(device);
736 62455 data = backend_alloc(device, new_bytes);
737 62455 owns_ = (data != nullptr);
738
2/2
✓ Branch 0 taken 62394 times.
✓ Branch 1 taken 61 times.
 62455 cap_bytes_ = owns_ ? new_bytes : 0;
739 62455 }
740 63278 rows = r;
741 63278 cols = c;
742 63278 dtype = dt;
743 63295 }
744
745 // ─── Host accessors ────────────────────────────────────────────────────────
746
747 349142282 float* Tensor::host_f32_mut() {
748 349142282 check_host(*this, "host_f32_mut");
749 349142282 check_dtype(*this, Dtype::FP32, "host_f32_mut");
750 349142282 return static_cast<float*>(data);
751 }
752 32287129922 const float* Tensor::host_f32() const {
753 32287129922 check_host(*this, "host_f32");
754 32287129922 check_dtype(*this, Dtype::FP32, "host_f32");
755 32287129922 return static_cast<const float*>(data);
756 }
757
758 1431 uint16_t* Tensor::host_fp16_mut() {
759 1431 check_host(*this, "host_fp16_mut");
760 1431 check_dtype(*this, Dtype::FP16, "host_fp16_mut");
761 1431 return static_cast<uint16_t*>(data);
762 }
763 4282 const uint16_t* Tensor::host_fp16() const {
764 4282 check_host(*this, "host_fp16");
765 4282 check_dtype(*this, Dtype::FP16, "host_fp16");
766 4282 return static_cast<const uint16_t*>(data);
767 }
768
769 1310 uint16_t* Tensor::host_bf16_mut() {
770 1310 check_host(*this, "host_bf16_mut");
771 1310 check_dtype(*this, Dtype::BF16, "host_bf16_mut");
772 1310 return static_cast<uint16_t*>(data);
773 }
774 1883 const uint16_t* Tensor::host_bf16() const {
775 1883 check_host(*this, "host_bf16");
776 1883 check_dtype(*this, Dtype::BF16, "host_bf16");
777 1883 return static_cast<const uint16_t*>(data);
778 }
779
780 50534 void* Tensor::host_raw_mut() {
781 50534 check_host(*this, "host_raw_mut");
782 50534 return data;
783 }
784 51833 const void* Tensor::host_raw() const {
785 51833 check_host(*this, "host_raw");
786 51833 return data;
787 }
788
789 53537 float& Tensor::at(int r, int c) {
790 53537 check_host(*this, "at");
791 53537 check_dtype(*this, Dtype::FP32, "at");
792
2/2
✓ Branch 0 taken 53533 times.
✓ Branch 1 taken 4 times.
 53537 if (r < 0 || r >= rows || c < 0 || c >= cols) {
793
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
 4 throw_msg("brotensor: at: index out of range");
794 }
795 53533 return static_cast<float*>(data)[static_cast<std::size_t>(r) * cols + c];
796 4 }
797
798 44909 float Tensor::at(int r, int c) const {
799 44909 check_host(*this, "at");
800 44909 check_dtype(*this, Dtype::FP32, "at");
801
2/2
✓ Branch 0 taken 44908 times.
✓ Branch 1 taken 1 time.
 44909 if (r < 0 || r >= rows || c < 0 || c >= cols) {
802
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 throw_msg("brotensor: at: index out of range");
803 }
804 44908 return static_cast<const float*>(data)[static_cast<std::size_t>(r) * cols + c];
805 1 }
806
807 // ─── Host roundtrip ────────────────────────────────────────────────────────
808
809 181 std::vector<float> Tensor::to_host_vector() const {
810
2/2
✓ Branch 0 taken 180 times.
✓ Branch 1 taken 1 time.
 181 if (dtype != Dtype::FP32) {
811
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 throw_msg("brotensor: to_host_vector: dtype not FP32");
812 }
813 180 std::vector<float> out(static_cast<std::size_t>(rows) * cols);
814
1/2
✓ Branch 0 taken 180 times.
✗ Branch 1 not taken.
 180 const std::size_t n = bytes();
815
2/2
✓ Branch 0 taken 179 times.
✓ Branch 1 taken 1 time.
 180 if (n == 0) return out;
816
3/4
✓ Branch 0 taken 179 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 107 times.
✓ Branch 3 taken 72 times.
 179 if (device.is_cpu()) {
817 72 std::memcpy(out.data(), data, n);
818 72 } else {
819
2/4
✓ Branch 0 taken 107 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 107 times.
✗ Branch 3 not taken.
 107 detail::alloc_for(device).memcpy_d2h(out.data(), data, n, device.index);
820 }
821 179 return out;
822 181 }
823
824 312 std::vector<uint16_t> Tensor::to_host_vector_fp16() const {
825
2/2
✓ Branch 0 taken 311 times.
✓ Branch 1 taken 1 time.
 312 if (dtype != Dtype::FP16) {
826
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 throw_msg("brotensor: to_host_vector_fp16: dtype not FP16");
827 }
828 311 std::vector<uint16_t> out(static_cast<std::size_t>(rows) * cols);
829
1/2
✓ Branch 0 taken 311 times.
✗ Branch 1 not taken.
 311 const std::size_t n = bytes();
830
2/2
✓ Branch 0 taken 310 times.
✓ Branch 1 taken 1 time.
 311 if (n == 0) return out;
831
3/4
✓ Branch 0 taken 310 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 305 times.
✓ Branch 3 taken 5 times.
 310 if (device.is_cpu()) {
832 5 std::memcpy(out.data(), data, n);
833 5 } else {
834
2/4
✓ Branch 0 taken 305 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 305 times.
✗ Branch 3 not taken.
 305 detail::alloc_for(device).memcpy_d2h(out.data(), data, n, device.index);
835 }
836 310 return out;
837 312 }
838
839 106 std::vector<uint16_t> Tensor::to_host_vector_bf16() const {
840
2/2
✓ Branch 0 taken 105 times.
✓ Branch 1 taken 1 time.
 106 if (dtype != Dtype::BF16) {
841
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 throw_msg("brotensor: to_host_vector_bf16: dtype not BF16");
842 }
843 105 std::vector<uint16_t> out(static_cast<std::size_t>(rows) * cols);
844
1/2
✓ Branch 0 taken 105 times.
✗ Branch 1 not taken.
 105 const std::size_t n = bytes();
845
2/2
✓ Branch 0 taken 104 times.
✓ Branch 1 taken 1 time.
 105 if (n == 0) return out;
846
3/4
✓ Branch 0 taken 104 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 101 times.
✓ Branch 3 taken 3 times.
 104 if (device.is_cpu()) {
847 3 std::memcpy(out.data(), data, n);
848 3 } else {
849
2/4
✓ Branch 0 taken 101 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 101 times.
✗ Branch 3 not taken.
 101 detail::alloc_for(device).memcpy_d2h(out.data(), data, n, device.index);
850 }
851 104 return out;
852 106 }
853
854 891 void Tensor::copy_to_host(float* dst) const {
855
2/2
✓ Branch 0 taken 890 times.
✓ Branch 1 taken 1 time.
 891 if (dtype != Dtype::FP32) {
856
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 throw_msg("brotensor: copy_to_host: dtype not FP32");
857 }
858 890 const std::size_t n = bytes();
859
2/2
✓ Branch 0 taken 889 times.
✓ Branch 1 taken 1 time.
 890 if (n == 0) return;
860
2/2
✓ Branch 0 taken 107 times.
✓ Branch 1 taken 782 times.
 889 if (device.is_cpu()) {
861 107 std::memcpy(dst, data, n);
862 107 } else {
863 782 detail::alloc_for(device).memcpy_d2h(dst, data, n, device.index);
864 }
865 891 }
866
867 514 void Tensor::copy_to_host_fp16(uint16_t* dst) const {
868
2/2
✓ Branch 0 taken 513 times.
✓ Branch 1 taken 1 time.
 514 if (dtype != Dtype::FP16) {
869
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 throw_msg("brotensor: copy_to_host_fp16: dtype not FP16");
870 }
871 513 const std::size_t n = bytes();
872
2/2
✓ Branch 0 taken 512 times.
✓ Branch 1 taken 1 time.
 513 if (n == 0) return;
873
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 512 times.
 512 if (device.is_cpu()) {
874 ✗ std::memcpy(dst, data, n);
875 ✗ } else {
876 512 detail::alloc_for(device).memcpy_d2h(dst, data, n, device.index);
877 }
878 514 }
879
880 58 void Tensor::copy_to_host_bf16(uint16_t* dst) const {
881
2/2
✓ Branch 0 taken 57 times.
✓ Branch 1 taken 1 time.
 58 if (dtype != Dtype::BF16) {
882
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 throw_msg("brotensor: copy_to_host_bf16: dtype not BF16");
883 }
884 57 const std::size_t n = bytes();
885
2/2
✓ Branch 0 taken 56 times.
✓ Branch 1 taken 1 time.
 57 if (n == 0) return;
886
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 56 times.
 56 if (device.is_cpu()) {
887 ✗ std::memcpy(dst, data, n);
888 ✗ } else {
889 56 detail::alloc_for(device).memcpy_d2h(dst, data, n, device.index);
890 }
891 58 }
892
893 42 void Tensor::copy_from_host_raw(const void* src, std::size_t nbytes) {
894
1/2
✓ Branch 0 taken 42 times.
✗ Branch 1 not taken.
 42 if (nbytes > bytes()) {
895 ✗ throw_msg("brotensor: copy_from_host_raw: nbytes exceeds the tensor's size");
896 }
897
1/2
✓ Branch 0 taken 42 times.
✗ Branch 1 not taken.
 42 if (nbytes == 0) return;
898
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 42 times.
 42 if (device.is_cpu()) {
899 ✗ std::memcpy(data, src, nbytes);
900 ✗ } else {
901 42 detail::alloc_for(device).memcpy_h2d(data, src, nbytes, device.index);
902 }
903 42 }
904
905 ✗ void Tensor::copy_to_host_raw(void* dst, std::size_t nbytes) const {
906 ✗ if (nbytes > bytes()) {
907 ✗ throw_msg("brotensor: copy_to_host_raw: nbytes exceeds the tensor's size");
908 }
909 ✗ if (nbytes == 0) return;
910 ✗ if (device.is_cpu()) {
911 ✗ std::memcpy(dst, data, nbytes);
912 ✗ } else {
913 ✗ detail::alloc_for(device).memcpy_d2h(dst, data, nbytes, device.index);
914 }
915 ✗ }
916
917 // ─── Allocation accounting ─────────────────────────────────────────────────
918
919 4 AllocStats alloc_stats() {
920 4 AllocStats s;
921 4 s.count = g_alloc_count.load(std::memory_order_relaxed);
922 4 s.bytes = g_alloc_bytes.load(std::memory_order_relaxed);
923 4 return s;
924 }
925
926 } // namespace brotensor
927