src/tensor.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // brotensor::Tensor implementation. | ||
| 2 | // | ||
| 3 | // All allocation, freeing, and inter-device memory traffic routes through the | ||
| 4 | // backend AllocVTable obtained from detail::alloc_for(device). The Tensor type | ||
| 5 | // itself is backend-agnostic; CUDA/Metal/CPU specifics live entirely behind | ||
| 6 | // the vtable indirection. | ||
| 7 | |||
| 8 | #include <brotensor/tensor.h> | ||
| 9 | #include <brotensor/runtime.h> | ||
| 10 | #include <brotensor/detail/dispatch.h> | ||
| 11 | |||
| 12 | #include <cstdint> | ||
| 13 | #include <cstring> | ||
| 14 | #include <memory> | ||
| 15 | #include <stdexcept> | ||
| 16 | #include <string> | ||
| 17 | #include <vector> | ||
| 18 | |||
| 19 | #include <chrono> | ||
| 20 | #include <cstdio> | ||
| 21 | #include <cstdlib> | ||
| 22 | |||
| 23 | namespace brotensor { | ||
| 24 | |||
| 25 | // ─── env-gated weight-load profiler (BROTENSOR_TIME_LOAD=1) ────────────────── | ||
| 26 | // Attributes GPU weight-upload wall time to allocation vs H2D copy, accumulated | ||
| 27 | // across every from_host_*_on call and dumped once at process exit. Purely a | ||
| 28 | // diagnostic; the fast path (flag unset) is a single getenv-cached bool check. | ||
| 29 | namespace { | ||
| 30 | struct LoadProf { | ||
| 31 | 165 | bool on = false; | |
| 32 | 165 | double alloc = 0.0, copy = 0.0, bytes = 0.0; | |
| 33 | 165 | long calls = 0; | |
| 34 | 495 | LoadProf() { on = std::getenv("BROTENSOR_TIME_LOAD") != nullptr; } | |
| 35 | ✗ | ~LoadProf() { | |
| 36 | ✗ | if (!on || calls == 0) return; | |
| 37 | ✗ | std::fprintf(stderr, | |
| 38 | "[brotensor load] %ld uploads, %.2f GB: alloc %.2f s, h2d %.2f s " | ||
| 39 | "(%.2f GB/s copy)\n", | ||
| 40 | ✗ | calls, bytes / 1e9, alloc, copy, | |
| 41 | ✗ | copy > 0 ? (bytes / 1e9) / copy : 0.0); | |
| 42 | ✗ | } | |
| 43 | }; | ||
| 44 | 165 | LoadProf g_loadprof; | |
| 45 | 1602 | inline bool loadprof_on() { return g_loadprof.on; } | |
| 46 | } // namespace | ||
| 47 | |||
| 48 | // ─── dtype + device helpers ──────────────────────────────────────────────── | ||
| 49 | |||
| 50 | 123650 | int dtype_size_bytes(Dtype dt) { | |
| 51 |
7/8✓ Branch 0 taken 60727 times.
✓ Branch 1 taken 5319 times.
✓ Branch 2 taken 6373 times.
✓ Branch 3 taken 191 times.
✓ Branch 4 taken 51023 times.
✓ Branch 5 taken 5 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 12 times.
|
123650 | switch (dt) { |
| 52 | 60727 | case Dtype::FP32: return 4; | |
| 53 | 5319 | case Dtype::FP16: return 2; | |
| 54 | 6373 | case Dtype::BF16: return 2; | |
| 55 | 191 | case Dtype::INT8: return 1; | |
| 56 | 51023 | case Dtype::INT32: return 4; | |
| 57 | 5 | case Dtype::F64: return 8; | |
| 58 | // Quant dtypes are block-addressed; no per-element size. | ||
| 59 | case Dtype::Q4_0: case Dtype::Q4_1: | ||
| 60 | case Dtype::Q5_0: case Dtype::Q5_1: | ||
| 61 | case Dtype::Q8_0: case Dtype::Q8_1: | ||
| 62 | case Dtype::Q2_K: case Dtype::Q3_K: | ||
| 63 | case Dtype::Q4_K: case Dtype::Q5_K: | ||
| 64 | case Dtype::Q6_K: case Dtype::Q8_K: | ||
| 65 | 12 | return 0; | |
| 66 | } | ||
| 67 | ✗ | return 0; | |
| 68 | 123650 | } | |
| 69 | |||
| 70 | 123920 | bool dtype_is_quant(Dtype dt) { | |
| 71 |
2/2✓ Branch 0 taken 406 times.
✓ Branch 1 taken 123514 times.
|
123920 | switch (dt) { |
| 72 | case Dtype::Q4_0: case Dtype::Q4_1: | ||
| 73 | case Dtype::Q5_0: case Dtype::Q5_1: | ||
| 74 | case Dtype::Q8_0: case Dtype::Q8_1: | ||
| 75 | case Dtype::Q2_K: case Dtype::Q3_K: | ||
| 76 | case Dtype::Q4_K: case Dtype::Q5_K: | ||
| 77 | case Dtype::Q6_K: case Dtype::Q8_K: | ||
| 78 | 406 | return true; | |
| 79 | default: | ||
| 80 | 123514 | return false; | |
| 81 | } | ||
| 82 | 123920 | } | |
| 83 | |||
| 84 | 413 | int dtype_block_size(Dtype dt) { | |
| 85 |
3/3✓ Branch 0 taken 332 times.
✓ Branch 1 taken 75 times.
✓ Branch 2 taken 6 times.
|
413 | switch (dt) { |
| 86 | case Dtype::Q4_0: case Dtype::Q4_1: | ||
| 87 | case Dtype::Q5_0: case Dtype::Q5_1: | ||
| 88 | case Dtype::Q8_0: case Dtype::Q8_1: | ||
| 89 | 332 | return 32; | |
| 90 | case Dtype::Q2_K: case Dtype::Q3_K: | ||
| 91 | case Dtype::Q4_K: case Dtype::Q5_K: | ||
| 92 | case Dtype::Q6_K: case Dtype::Q8_K: | ||
| 93 | 75 | return 256; | |
| 94 | default: | ||
| 95 | 6 | return 1; | |
| 96 | } | ||
| 97 | 413 | } | |
| 98 | |||
| 99 | 407 | int dtype_block_bytes(Dtype dt) { | |
| 100 |
13/13✓ Branch 0 taken 5 times.
✓ Branch 1 taken 15 times.
✓ Branch 2 taken 6 times.
✓ Branch 3 taken 10 times.
✓ Branch 4 taken 5 times.
✓ Branch 5 taken 5 times.
✓ Branch 6 taken 300 times.
✓ Branch 7 taken 5 times.
✓ Branch 8 taken 5 times.
✓ Branch 9 taken 5 times.
✓ Branch 10 taken 36 times.
✓ Branch 11 taken 5 times.
✓ Branch 12 taken 5 times.
|
407 | switch (dt) { |
| 101 | 10 | case Dtype::Q4_0: return 18; | |
| 102 | 5 | case Dtype::Q4_1: return 20; | |
| 103 | 5 | case Dtype::Q5_0: return 22; | |
| 104 | 5 | case Dtype::Q5_1: return 24; | |
| 105 | 300 | case Dtype::Q8_0: return 34; | |
| 106 | 5 | case Dtype::Q8_1: return 36; | |
| 107 | 5 | case Dtype::Q2_K: return 82; | |
| 108 | 5 | case Dtype::Q3_K: return 110; | |
| 109 | 36 | case Dtype::Q4_K: return 144; | |
| 110 | 5 | case Dtype::Q5_K: return 176; | |
| 111 | 15 | case Dtype::Q6_K: return 210; | |
| 112 | 5 | case Dtype::Q8_K: return 292; | |
| 113 | 6 | default: return dtype_size_bytes(dt); | |
| 114 | } | ||
| 115 | 407 | } | |
| 116 | |||
| 117 | 123892 | std::size_t dtype_storage_bytes(Dtype d, std::int64_t numel) { | |
| 118 |
2/2✓ Branch 0 taken 123890 times.
✓ Branch 1 taken 2 times.
|
123892 | if (numel < 0) { |
| 119 |
1/2✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
|
2 | throw std::runtime_error("brotensor: dtype_storage_bytes: negative numel"); |
| 120 | } | ||
| 121 |
2/2✓ Branch 0 taken 387 times.
✓ Branch 1 taken 123503 times.
|
123890 | if (!dtype_is_quant(d)) { |
| 122 | 247006 | return static_cast<std::size_t>(numel) * | |
| 123 | 123503 | static_cast<std::size_t>(dtype_size_bytes(d)); | |
| 124 | } | ||
| 125 | 387 | const int bs = dtype_block_size(d); | |
| 126 | 387 | const int bb = dtype_block_bytes(d); | |
| 127 |
2/2✓ Branch 0 taken 373 times.
✓ Branch 1 taken 14 times.
|
387 | if (numel % bs != 0) { |
| 128 |
1/2✓ Branch 0 taken 14 times.
✗ Branch 1 not taken.
|
14 | throw std::runtime_error( |
| 129 | "brotensor: dtype_storage_bytes: numel not a multiple of block size"); | ||
| 130 | } | ||
| 131 | 746 | return (static_cast<std::size_t>(numel) / static_cast<std::size_t>(bs)) * | |
| 132 | 373 | static_cast<std::size_t>(bb); | |
| 133 | 123876 | } | |
| 134 | |||
| 135 | const Device Device::CPU{DeviceType::CPU, 0}; | ||
| 136 | const Device Device::CUDA{DeviceType::CUDA, 0}; | ||
| 137 | const Device Device::Metal{DeviceType::Metal, 0}; | ||
| 138 | |||
| 139 | 27 | const char* device_name(Device d) { | |
| 140 |
2/2✓ Branch 0 taken 9 times.
✓ Branch 1 taken 18 times.
|
27 | if (d.is_cpu()) return "CPU"; |
| 141 |
2/2✓ Branch 0 taken 17 times.
✓ Branch 1 taken 1 time.
|
18 | if (d.is_cuda()) { |
| 142 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 17 times.
|
17 | if (d.index == 0) return "CUDA"; |
| 143 | static thread_local char buf[32]; | ||
| 144 | ✗ | std::snprintf(buf, sizeof(buf), "CUDA:%d", d.index); | |
| 145 | ✗ | return buf; | |
| 146 | } | ||
| 147 |
1/2✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
|
1 | if (d.is_metal()) { |
| 148 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | if (d.index == 0) return "Metal"; |
| 149 | static thread_local char buf[32]; | ||
| 150 | ✗ | std::snprintf(buf, sizeof(buf), "Metal:%d", d.index); | |
| 151 | ✗ | return buf; | |
| 152 | } | ||
| 153 | ✗ | return "?"; | |
| 154 | 27 | } | |
| 155 | |||
| 156 | ✗ | std::string to_string(Device d) { | |
| 157 | ✗ | return device_name(d); | |
| 158 | } | ||
| 159 | |||
| 160 | // ─── FP16 ↔ FP32 conversion (host-side IEEE 754 binary16) ────────────────── | ||
| 161 | |||
| 162 | 6170996 | uint16_t fp32_to_fp16_bits(float v) { | |
| 163 | uint32_t x; | ||
| 164 | 6170996 | std::memcpy(&x, &v, 4); | |
| 165 | 6170996 | const uint32_t sign = (x >> 16) & 0x8000u; | |
| 166 | 6170996 | int32_t exp = static_cast<int32_t>((x >> 23) & 0xFFu) - 127 + 15; | |
| 167 | 6170996 | uint32_t mant = x & 0x7FFFFFu; | |
| 168 | |||
| 169 |
2/2✓ Branch 0 taken 68768 times.
✓ Branch 1 taken 6102228 times.
|
6170996 | if (((x >> 23) & 0xFFu) == 0xFFu) { |
| 170 | // Inf or NaN. | ||
| 171 | 68768 | uint16_t out = static_cast<uint16_t>(sign | 0x7C00u); | |
| 172 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 68768 times.
|
68768 | if (mant) out |= 0x0200u; // quiet NaN |
| 173 | 68768 | return out; | |
| 174 | } | ||
| 175 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 6102228 times.
|
6102228 | if (exp >= 0x1F) { |
| 176 | ✗ | return static_cast<uint16_t>(sign | 0x7C00u); | |
| 177 | } | ||
| 178 |
2/2✓ Branch 0 taken 1664 times.
✓ Branch 1 taken 6100564 times.
|
6102228 | if (exp <= 0) { |
| 179 |
2/2✓ Branch 0 taken 499 times.
✓ Branch 1 taken 1165 times.
|
1664 | if (exp < -10) return static_cast<uint16_t>(sign); |
| 180 | 1165 | mant |= 0x800000u; | |
| 181 | 1165 | const int shift = 14 - exp; | |
| 182 | 1165 | const uint32_t round = 1u << (shift - 1); | |
| 183 | 1165 | uint32_t r = (mant + round) >> shift; | |
| 184 | 1165 | return static_cast<uint16_t>(sign | r); | |
| 185 | } | ||
| 186 | 6100564 | const uint32_t lsb = (mant >> 13) & 1u; | |
| 187 | 6100564 | const uint32_t round = 0x00001000u + lsb - 1u; | |
| 188 | 6100564 | uint32_t r_mant = (mant + round) >> 13; | |
| 189 | 6100564 | uint32_t r_exp = static_cast<uint32_t>(exp); | |
| 190 |
2/2✓ Branch 0 taken 6098430 times.
✓ Branch 1 taken 2134 times.
|
6100564 | if (r_mant & 0x400u) { |
| 191 | 2134 | r_mant = 0; | |
| 192 | 2134 | r_exp += 1; | |
| 193 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 2134 times.
|
2134 | if (r_exp >= 0x1F) return static_cast<uint16_t>(sign | 0x7C00u); |
| 194 | 2134 | } | |
| 195 | 6100564 | return static_cast<uint16_t>(sign | (r_exp << 10) | r_mant); | |
| 196 | 6170996 | } | |
| 197 | |||
| 198 | 5231094 | float fp16_bits_to_fp32(uint16_t bits) { | |
| 199 | 5231094 | const uint32_t sign = (static_cast<uint32_t>(bits) & 0x8000u) << 16; | |
| 200 | 5231094 | uint32_t exp = (bits >> 10) & 0x1Fu; | |
| 201 | 5231094 | uint32_t mant = bits & 0x3FFu; | |
| 202 | uint32_t out; | ||
| 203 |
2/2✓ Branch 0 taken 5216846 times.
✓ Branch 1 taken 14248 times.
|
5231094 | if (exp == 0) { |
| 204 |
2/2✓ Branch 0 taken 3838 times.
✓ Branch 1 taken 10410 times.
|
14248 | if (mant == 0) { |
| 205 | 10410 | out = sign; | |
| 206 | 10410 | } else { | |
| 207 |
2/2✓ Branch 0 taken 7158 times.
✓ Branch 1 taken 3838 times.
|
10996 | while ((mant & 0x400u) == 0) { mant <<= 1; exp -= 1; } |
| 208 | 3838 | mant &= 0x3FFu; | |
| 209 | 3838 | uint32_t e = (exp + (127 - 15) + 1) & 0xFFu; | |
| 210 | 3838 | out = sign | (e << 23) | (mant << 13); | |
| 211 | } | ||
| 212 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 5216846 times.
|
5231094 | } else if (exp == 0x1F) { |
| 213 | ✗ | out = sign | 0x7F800000u | (mant << 13); | |
| 214 | ✗ | } else { | |
| 215 | 5216846 | uint32_t e = exp + (127 - 15); | |
| 216 | 5216846 | out = sign | (e << 23) | (mant << 13); | |
| 217 | } | ||
| 218 | float f; | ||
| 219 | 5231094 | std::memcpy(&f, &out, 4); | |
| 220 | 5231094 | return f; | |
| 221 | } | ||
| 222 | |||
| 223 | // ─── BF16 ↔ FP32 conversion (host-side bfloat16) ─────────────────────────── | ||
| 224 | |||
| 225 | 3445496 | uint16_t fp32_to_bf16_bits(float v) { | |
| 226 | uint32_t x; | ||
| 227 | 3445496 | std::memcpy(&x, &v, 4); | |
| 228 | // NaN: keep it a NaN (the rounding add below could otherwise carry a | ||
| 229 | // NaN's mantissa into the exponent and produce an infinity). | ||
| 230 |
4/4✓ Branch 0 taken 4 times.
✓ Branch 1 taken 3445492 times.
✓ Branch 2 taken 2 times.
✓ Branch 3 taken 2 times.
|
3445496 | if (((x >> 23) & 0xFFu) == 0xFFu && (x & 0x7FFFFFu) != 0) { |
| 231 | 2 | return static_cast<uint16_t>((x >> 16) | 0x0040u); | |
| 232 | } | ||
| 233 | // Round to nearest, ties to even: add 0x7FFF + LSB-of-result. | ||
| 234 | 3445494 | const uint32_t rounding_bias = 0x7FFFu + ((x >> 16) & 1u); | |
| 235 | 3445494 | x += rounding_bias; | |
| 236 | 3445494 | return static_cast<uint16_t>(x >> 16); | |
| 237 | 3445496 | } | |
| 238 | |||
| 239 | 2859433 | float bf16_bits_to_fp32(uint16_t bits) { | |
| 240 | 2859433 | const uint32_t x = static_cast<uint32_t>(bits) << 16; | |
| 241 | float f; | ||
| 242 | 2859433 | std::memcpy(&f, &x, 4); | |
| 243 | 2859433 | return f; | |
| 244 | } | ||
| 245 | |||
| 246 | // ─── internal helpers ────────────────────────────────────────────────────── | ||
| 247 | |||
| 248 | namespace { | ||
| 249 | |||
| 250 | 39 | [[noreturn]] void throw_msg(const std::string& m) { | |
| 251 |
1/2✓ Branch 0 taken 39 times.
✗ Branch 1 not taken.
|
39 | throw std::runtime_error(m); |
| 252 | ✗ | } | |
| 253 | |||
| 254 | 65805288 | void check_host(const Tensor& t, const char* who) { | |
| 255 |
2/2✓ Branch 0 taken 65805278 times.
✓ Branch 1 taken 10 times.
|
65805288 | if (t.device != Device::CPU) { |
| 256 | 10 | std::string m = "brotensor: "; | |
| 257 |
1/2✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
|
10 | m += who; |
| 258 |
1/2✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
|
10 | m += ": tensor is on "; |
| 259 |
2/4✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 10 times.
✗ Branch 3 not taken.
|
10 | m += device_name(t.device); |
| 260 |
1/2✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
|
10 | m += ", not CPU"; |
| 261 |
1/2✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
|
10 | throw_msg(m); |
| 262 | 10 | } | |
| 263 | 65805288 | } | |
| 264 | |||
| 265 | // Guard against negative dimensions before they are cast to std::size_t in | ||
| 266 | // bytes() — an unchecked (size_t)(-1) underflows to an astronomical count | ||
| 267 | // that sails past the bytes==0 short-circuits and reaches backend_alloc. | ||
| 268 | 80613 | void check_dims(int r, int c, const char* who) { | |
| 269 |
2/2✓ Branch 0 taken 80606 times.
✓ Branch 1 taken 7 times.
|
80613 | if (r < 0 || c < 0) { |
| 270 | 7 | std::string m = "brotensor: "; | |
| 271 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | m += who; |
| 272 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | m += ": negative dimension"; |
| 273 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
|
7 | throw_msg(m); |
| 274 | 7 | } | |
| 275 | 80613 | } | |
| 276 | |||
| 277 | 65704493 | void check_dtype(const Tensor& t, Dtype expected, const char* who) { | |
| 278 |
2/2✓ Branch 0 taken 65704484 times.
✓ Branch 1 taken 9 times.
|
65704493 | if (t.dtype != expected) { |
| 279 | 9 | std::string m = "brotensor: "; | |
| 280 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | m += who; |
| 281 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | m += ": dtype mismatch"; |
| 282 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
|
9 | throw_msg(m); |
| 283 | 9 | } | |
| 284 | 65704493 | } | |
| 285 | |||
| 286 | 80369 | void* backend_alloc(Device d, std::size_t bytes) { | |
| 287 |
2/2✓ Branch 0 taken 80170 times.
✓ Branch 1 taken 199 times.
|
80369 | if (bytes == 0) return nullptr; |
| 288 | 80170 | return detail::alloc_for(d).alloc(bytes, d.index); | |
| 289 | 80369 | } | |
| 290 | |||
| 291 | 80170 | void backend_free(Device d, void* p) { | |
| 292 |
1/2✓ Branch 0 taken 80170 times.
✗ Branch 1 not taken.
|
80170 | if (!p) return; |
| 293 | 80170 | detail::alloc_for(d).free(p, d.index); | |
| 294 | 80170 | } | |
| 295 | |||
| 296 | 11687 | void backend_zero(Device d, void* p, std::size_t bytes) { | |
| 297 |
3/4✓ Branch 0 taken 11682 times.
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 11682 times.
|
11687 | if (bytes == 0 || !p) return; |
| 298 | 11682 | detail::alloc_for(d).memset_zero(p, bytes, d.index); | |
| 299 | 11687 | } | |
| 300 | |||
| 301 | } // namespace | ||
| 302 | |||
| 303 | // ─── Tensor lifetime ─────────────────────────────────────────────────────── | ||
| 304 | |||
| 305 | 165166 | Tensor::~Tensor() { | |
| 306 |
1/2✓ Branch 0 taken 82583 times.
✗ Branch 1 not taken.
|
82583 | release_(); |
| 307 | 165166 | } | |
| 308 | |||
| 309 | 142981 | void Tensor::release_() { | |
| 310 |
3/4✓ Branch 0 taken 80170 times.
✓ Branch 1 taken 62811 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 80170 times.
|
142981 | if (owns_ && data) { |
| 311 | // alloc_for can throw if the backend was somehow un-registered between | ||
| 312 | // alloc and free; let it propagate — better than leaking a different | ||
| 313 | // backend's pointer. | ||
| 314 | 80170 | backend_free(device, data); | |
| 315 | 80170 | } | |
| 316 | 142981 | data = nullptr; | |
| 317 | 142981 | owns_ = false; | |
| 318 | 142981 | cap_bytes_ = 0; | |
| 319 | 142981 | } | |
| 320 | |||
| 321 | 542 | Tensor::Tensor(Tensor&& o) noexcept | |
| 322 | 271 | : data(o.data), rows(o.rows), cols(o.cols), | |
| 323 | 271 | dtype(o.dtype), device(o.device), owns_(o.owns_), | |
| 324 | 542 | cap_bytes_(o.cap_bytes_) { | |
| 325 | 271 | o.data = nullptr; | |
| 326 | 271 | o.rows = 0; | |
| 327 | 271 | o.cols = 0; | |
| 328 | 271 | o.owns_ = false; | |
| 329 | 271 | o.cap_bytes_ = 0; | |
| 330 | 542 | } | |
| 331 | |||
| 332 | 1672 | Tensor& Tensor::operator=(Tensor&& o) noexcept { | |
| 333 |
2/2✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1671 times.
|
1672 | if (this != &o) { |
| 334 |
1/2✓ Branch 0 taken 1671 times.
✗ Branch 1 not taken.
|
1671 | release_(); |
| 335 | 1671 | data = o.data; | |
| 336 | 1671 | rows = o.rows; | |
| 337 | 1671 | cols = o.cols; | |
| 338 | 1671 | dtype = o.dtype; | |
| 339 | 1671 | device = o.device; | |
| 340 | 1671 | owns_ = o.owns_; | |
| 341 | 1671 | cap_bytes_ = o.cap_bytes_; | |
| 342 | 1671 | o.data = nullptr; | |
| 343 | 1671 | o.rows = 0; | |
| 344 | 1671 | o.cols = 0; | |
| 345 | 1671 | o.owns_ = false; | |
| 346 | 1671 | o.cap_bytes_ = 0; | |
| 347 | 1671 | } | |
| 348 | 1672 | return *this; | |
| 349 | } | ||
| 350 | |||
| 351 | // Copy ctor / assignment: device-aware deep copy via clone(). Implemented in | ||
| 352 | // terms of the move assignment so all the ownership bookkeeping lives in one | ||
| 353 | // place. | ||
| 354 | 874 | Tensor::Tensor(const Tensor& o) { | |
| 355 | 437 | *this = o.clone(); | |
| 356 | 437 | } | |
| 357 | |||
| 358 | 47 | Tensor& Tensor::operator=(const Tensor& o) { | |
| 359 |
2/2✓ Branch 0 taken 1 time.
✓ Branch 1 taken 46 times.
|
47 | if (this != &o) *this = o.clone(); |
| 360 | 47 | return *this; | |
| 361 | } | ||
| 362 | |||
| 363 | 64864 | std::size_t Tensor::bytes() const { | |
| 364 | 64864 | return dtype_storage_bytes( | |
| 365 | 64864 | dtype, | |
| 366 | 64864 | static_cast<std::int64_t>(rows) * static_cast<std::int64_t>(cols)); | |
| 367 | } | ||
| 368 | |||
| 369 | // ─── Factories ───────────────────────────────────────────────────────────── | ||
| 370 | |||
| 371 | 21643 | Tensor Tensor::empty_on(Device d, int r, int c, Dtype dt) { | |
| 372 | 21643 | check_dims(r, c, "empty_on"); | |
| 373 | 21643 | Tensor t; | |
| 374 | 21643 | t.device = d; | |
| 375 | 21643 | t.dtype = dt; | |
| 376 | 21643 | t.rows = r; | |
| 377 | 21643 | t.cols = c; | |
| 378 |
3/4✓ Branch 0 taken 21642 times.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 21642 times.
✗ Branch 3 not taken.
|
21643 | t.data = backend_alloc(d, t.bytes()); |
| 379 | 21642 | t.owns_ = (t.data != nullptr); | |
| 380 |
3/4✓ Branch 0 taken 21504 times.
✓ Branch 1 taken 138 times.
✓ Branch 2 taken 21504 times.
✗ Branch 3 not taken.
|
21642 | t.cap_bytes_ = t.owns_ ? t.bytes() : 0; |
| 381 | 21642 | return t; | |
| 382 | 21643 | } | |
| 383 | |||
| 384 | 11438 | Tensor Tensor::zeros_on(Device d, int r, int c, Dtype dt) { | |
| 385 | 11438 | Tensor t = empty_on(d, r, c, dt); | |
| 386 |
2/4✓ Branch 0 taken 11438 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 11438 times.
✗ Branch 3 not taken.
|
11438 | backend_zero(d, t.data, t.bytes()); |
| 387 | 11438 | return t; | |
| 388 | 11438 | } | |
| 389 | |||
| 390 | 1 | Tensor Tensor::empty(int r, int c, Dtype dt) { | |
| 391 | 1 | return empty_on(default_device(), r, c, dt); | |
| 392 | } | ||
| 393 | |||
| 394 | 3 | Tensor Tensor::zeros(int r, int c, Dtype dt) { | |
| 395 | 3 | return zeros_on(default_device(), r, c, dt); | |
| 396 | } | ||
| 397 | |||
| 398 | namespace { | ||
| 399 | // Shared upload core for from_host_*_on. When BROTENSOR_TIME_LOAD is set it | ||
| 400 | // attributes host wall time to the empty_on (alloc enqueue) and the H2D copy. | ||
| 401 | 1602 | Tensor from_host_typed(Device d, const void* src, int r, int c, Dtype dt) { | |
| 402 | using clock = std::chrono::steady_clock; | ||
| 403 |
1/2✓ Branch 0 taken 1602 times.
✗ Branch 1 not taken.
|
1602 | const bool prof = loadprof_on() && d != Device::CPU; |
| 404 | 1602 | auto t0 = clock::now(); | |
| 405 | 1602 | Tensor t = Tensor::empty_on(d, r, c, dt); | |
| 406 |
1/2✓ Branch 0 taken 1602 times.
✗ Branch 1 not taken.
|
1602 | const std::size_t n = t.bytes(); |
| 407 |
1/2✓ Branch 0 taken 1602 times.
✗ Branch 1 not taken.
|
1602 | if (n == 0) return t; |
| 408 | 1602 | auto t1 = clock::now(); | |
| 409 |
3/4✓ Branch 0 taken 1602 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1117 times.
✓ Branch 3 taken 485 times.
|
1602 | if (d == Device::CPU) { |
| 410 | 485 | std::memcpy(t.data, src, n); | |
| 411 | 485 | } else { | |
| 412 |
2/4✓ Branch 0 taken 1117 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1117 times.
✗ Branch 3 not taken.
|
1117 | detail::alloc_for(d).memcpy_h2d(t.data, src, n, d.index); |
| 413 | } | ||
| 414 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1602 times.
|
1602 | if (prof) { |
| 415 | ✗ | auto t2 = clock::now(); | |
| 416 | ✗ | g_loadprof.alloc += std::chrono::duration<double>(t1 - t0).count(); | |
| 417 | ✗ | g_loadprof.copy += std::chrono::duration<double>(t2 - t1).count(); | |
| 418 | ✗ | g_loadprof.bytes += static_cast<double>(n); | |
| 419 | ✗ | ++g_loadprof.calls; | |
| 420 | ✗ | } | |
| 421 | 1602 | return t; | |
| 422 | 1602 | } | |
| 423 | } // namespace | ||
| 424 | |||
| 425 | 577 | Tensor Tensor::from_host_on(Device d, const float* src, int r, int c) { | |
| 426 | 577 | return from_host_typed(d, src, r, c, Dtype::FP32); | |
| 427 | } | ||
| 428 | |||
| 429 | 730 | Tensor Tensor::from_host_fp16_on(Device d, const uint16_t* src, int r, int c) { | |
| 430 | 730 | return from_host_typed(d, src, r, c, Dtype::FP16); | |
| 431 | } | ||
| 432 | |||
| 433 | 295 | Tensor Tensor::from_host_bf16_on(Device d, const uint16_t* src, int r, int c) { | |
| 434 | 295 | return from_host_typed(d, src, r, c, Dtype::BF16); | |
| 435 | } | ||
| 436 | |||
| 437 | 5 | Tensor Tensor::from_host_int8_on(Device d, const int8_t* src, int r, int c) { | |
| 438 | 5 | Tensor t = empty_on(d, r, c, Dtype::INT8); | |
| 439 |
1/2✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
|
5 | const std::size_t n = t.bytes(); |
| 440 |
1/2✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
|
5 | if (n == 0) return t; |
| 441 |
2/4✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 5 times.
✗ Branch 3 not taken.
|
5 | if (d == Device::CPU) { |
| 442 | ✗ | std::memcpy(t.data, src, n); | |
| 443 | ✗ | } else { | |
| 444 |
2/4✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 5 times.
✗ Branch 3 not taken.
|
5 | detail::alloc_for(d).memcpy_h2d(t.data, src, n, d.index); |
| 445 | } | ||
| 446 | 5 | return t; | |
| 447 | 5 | } | |
| 448 | |||
| 449 | 11 | Tensor Tensor::from_raw_bytes_on(Device target, const void* src, int r, int c, | |
| 450 | Dtype dt, std::size_t nbytes) { | ||
| 451 | 11 | Tensor t = empty_on(target, r, c, dt); | |
| 452 |
3/4✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 10 times.
✓ Branch 3 taken 1 time.
|
11 | if (t.bytes() != nbytes) { |
| 453 |
2/4✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 time.
|
1 | throw std::runtime_error( |
| 454 | "brotensor: from_raw_bytes_on: nbytes does not match tensor size"); | ||
| 455 | } | ||
| 456 |
2/2✓ Branch 0 taken 9 times.
✓ Branch 1 taken 1 time.
|
10 | if (nbytes == 0) return t; |
| 457 |
2/4✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 9 times.
|
9 | if (target == Device::CPU) { |
| 458 | 9 | std::memcpy(t.data, src, nbytes); | |
| 459 | 9 | } else { | |
| 460 | ✗ | detail::alloc_for(target).memcpy_h2d(t.data, src, nbytes, target.index); | |
| 461 | } | ||
| 462 | 9 | return t; | |
| 463 | 11 | } | |
| 464 | |||
| 465 | 13 | Tensor Tensor::from_host(const float* src, int r, int c) { | |
| 466 | 13 | return from_host_on(default_device(), src, r, c); | |
| 467 | } | ||
| 468 | |||
| 469 | 6 | Tensor Tensor::from_host_fp16(const uint16_t* src, int r, int c) { | |
| 470 | 6 | return from_host_fp16_on(default_device(), src, r, c); | |
| 471 | } | ||
| 472 | |||
| 473 | 5 | Tensor Tensor::from_host_bf16(const uint16_t* src, int r, int c) { | |
| 474 | 5 | return from_host_bf16_on(default_device(), src, r, c); | |
| 475 | } | ||
| 476 | |||
| 477 | ✗ | Tensor Tensor::from_host_int8(const int8_t* src, int r, int c) { | |
| 478 | ✗ | return from_host_int8_on(default_device(), src, r, c); | |
| 479 | } | ||
| 480 | |||
| 481 | 172 | Tensor Tensor::view(Device d, void* data, int r, int c, Dtype dt) { | |
| 482 | 172 | Tensor t; | |
| 483 | 172 | t.device = d; | |
| 484 | 172 | t.dtype = dt; | |
| 485 | 172 | t.rows = r; | |
| 486 | 172 | t.cols = c; | |
| 487 | 172 | t.data = data; | |
| 488 | 172 | t.owns_ = false; | |
| 489 | 172 | return t; | |
| 490 | 172 | } | |
| 491 | |||
| 492 | // ─── Migration ───────────────────────────────────────────────────────────── | ||
| 493 | |||
| 494 | 1663 | Tensor Tensor::clone() const { | |
| 495 | 1663 | Tensor t = empty_on(device, rows, cols, dtype); | |
| 496 |
1/2✓ Branch 0 taken 1663 times.
✗ Branch 1 not taken.
|
1663 | const std::size_t n = bytes(); |
| 497 |
3/4✓ Branch 0 taken 1660 times.
✓ Branch 1 taken 3 times.
✓ Branch 2 taken 1660 times.
✗ Branch 3 not taken.
|
1663 | if (n == 0 || !data) return t; |
| 498 |
3/4✓ Branch 0 taken 1660 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 13 times.
✓ Branch 3 taken 1647 times.
|
1660 | if (device == Device::CPU) { |
| 499 | 1647 | std::memcpy(t.data, data, n); | |
| 500 | 1647 | } else { | |
| 501 |
2/4✓ Branch 0 taken 13 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 13 times.
✗ Branch 3 not taken.
|
13 | detail::alloc_for(device).memcpy_d2d(t.data, data, n, device.index); |
| 502 | } | ||
| 503 | 1660 | return t; | |
| 504 | 1663 | } | |
| 505 | |||
| 506 | 5819 | Tensor Tensor::to(Device target) const { | |
| 507 |
2/2✓ Branch 0 taken 1 time.
✓ Branch 1 taken 5818 times.
|
5819 | if (target == device) return clone(); |
| 508 | 5818 | Tensor t = empty_on(target, rows, cols, dtype); | |
| 509 |
1/2✓ Branch 0 taken 5818 times.
✗ Branch 1 not taken.
|
5818 | const std::size_t n = bytes(); |
| 510 |
2/4✓ Branch 0 taken 5818 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 5818 times.
✗ Branch 3 not taken.
|
5818 | if (n == 0 || !data) return t; |
| 511 | |||
| 512 |
3/4✓ Branch 0 taken 5818 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3595 times.
✓ Branch 3 taken 2223 times.
|
5818 | if (device.is_cpu()) { |
| 513 | // CPU → GPU. | ||
| 514 |
2/4✓ Branch 0 taken 3595 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3595 times.
✗ Branch 3 not taken.
|
3595 | detail::alloc_for(target).memcpy_h2d(t.data, data, n, target.index); |
| 515 |
2/4✓ Branch 0 taken 2223 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2223 times.
✗ Branch 3 not taken.
|
5818 | } else if (target.is_cpu()) { |
| 516 | // GPU → CPU. | ||
| 517 |
2/4✓ Branch 0 taken 2223 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2223 times.
✗ Branch 3 not taken.
|
2223 | detail::alloc_for(device).memcpy_d2h(t.data, data, n, device.index); |
| 518 |
0/8✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
|
2223 | } else if (device.is_cuda() && target.is_cuda()) { |
| 519 | // Direct peer copy between CUDA GPUs. | ||
| 520 | ✗ | auto peer_fn = detail::alloc_for(target).memcpy_peer; | |
| 521 | ✗ | if (peer_fn) { | |
| 522 | ✗ | peer_fn(t.data, target.index, data, device.index, n); | |
| 523 | ✗ | } else { | |
| 524 | ✗ | std::unique_ptr<unsigned char[]> staging(new unsigned char[n]); | |
| 525 | ✗ | detail::alloc_for(device).memcpy_d2h(staging.get(), data, n, device.index); | |
| 526 | ✗ | detail::alloc_for(target).memcpy_h2d(t.data, staging.get(), n, target.index); | |
| 527 | ✗ | } | |
| 528 | ✗ | } else { | |
| 529 | // GPU → different GPU backend. Bounce through host. | ||
| 530 | ✗ | std::unique_ptr<unsigned char[]> staging(new unsigned char[n]); | |
| 531 | ✗ | detail::alloc_for(device).memcpy_d2h(staging.get(), data, n, device.index); | |
| 532 | ✗ | detail::alloc_for(target).memcpy_h2d(t.data, staging.get(), n, target.index); | |
| 533 | ✗ | } | |
| 534 | 5818 | return t; | |
| 535 | 11637 | } | |
| 536 | |||
| 537 | // ─── Mutators ────────────────────────────────────────────────────────────── | ||
| 538 | |||
| 539 | 249 | void Tensor::zero() { | |
| 540 | 249 | backend_zero(device, data, bytes()); | |
| 541 | 249 | } | |
| 542 | |||
| 543 | 58960 | void Tensor::resize(int r, int c, Dtype dt) { | |
| 544 | 58960 | check_dims(r, c, "resize"); | |
| 545 |
8/8✓ Branch 0 taken 121 times.
✓ Branch 1 taken 58839 times.
✓ Branch 2 taken 63 times.
✓ Branch 3 taken 58 times.
✓ Branch 4 taken 58 times.
✓ Branch 5 taken 5 times.
✓ Branch 6 taken 55 times.
✓ Branch 7 taken 3 times.
|
58960 | if (r == rows && c == cols && dt == dtype && data != nullptr) return; |
| 546 | 58957 | const std::size_t new_bytes = dtype_storage_bytes( | |
| 547 | 58957 | dt, | |
| 548 | 58957 | static_cast<std::int64_t>(r) * static_cast<std::int64_t>(c)); | |
| 549 | // A non-owning view over real storage cannot be reshaped: reallocating | ||
| 550 | // would silently allocate fresh owned memory and sever the view, leaving | ||
| 551 | // callers with a tensor that no longer aliases what they passed to | ||
| 552 | // view(). Reject it explicitly rather than converting it in place. | ||
| 553 | // (A default-constructed / released tensor — owns_ == false but | ||
| 554 | // data == nullptr — is not a view and resizes normally.) | ||
| 555 |
4/4✓ Branch 0 taken 58618 times.
✓ Branch 1 taken 339 times.
✓ Branch 2 taken 58616 times.
✓ Branch 3 taken 2 times.
|
58957 | if (!owns_ && data != nullptr) { |
| 556 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
|
2 | throw_msg("brotensor: resize: cannot reshape a non-owning view; " |
| 557 | "allocate a fresh tensor or re-view() with the new shape"); | ||
| 558 | } | ||
| 559 | // Keep the existing storage whenever the new shape fits — the buffer | ||
| 560 | // stabilises at its high-water mark and its device pointer stays put | ||
| 561 | // (required when the op sequence using it is CUDA-graph captured). | ||
| 562 |
4/4✓ Branch 0 taken 289 times.
✓ Branch 1 taken 58666 times.
✓ Branch 2 taken 228 times.
✓ Branch 3 taken 61 times.
|
58955 | if (new_bytes > cap_bytes_ || !owns_) { |
| 563 | 58727 | release_(); | |
| 564 | 58727 | data = backend_alloc(device, new_bytes); | |
| 565 | 58727 | owns_ = (data != nullptr); | |
| 566 |
2/2✓ Branch 0 taken 58666 times.
✓ Branch 1 taken 61 times.
|
58727 | cap_bytes_ = owns_ ? new_bytes : 0; |
| 567 | 58727 | } | |
| 568 | 58955 | rows = r; | |
| 569 | 58955 | cols = c; | |
| 570 | 58955 | dtype = dt; | |
| 571 | 58960 | } | |
| 572 | |||
| 573 | // ─── Host accessors ──────────────────────────────────────────────────────── | ||
| 574 | |||
| 575 | 51006659 | float* Tensor::host_f32_mut() { | |
| 576 | 51006659 | check_host(*this, "host_f32_mut"); | |
| 577 | 51006659 | check_dtype(*this, Dtype::FP32, "host_f32_mut"); | |
| 578 | 51006659 | return static_cast<float*>(data); | |
| 579 | } | ||
| 580 | 14684410 | const float* Tensor::host_f32() const { | |
| 581 | 14684410 | check_host(*this, "host_f32"); | |
| 582 | 14684410 | check_dtype(*this, Dtype::FP32, "host_f32"); | |
| 583 | 14684410 | return static_cast<const float*>(data); | |
| 584 | } | ||
| 585 | |||
| 586 | 332 | uint16_t* Tensor::host_fp16_mut() { | |
| 587 | 332 | check_host(*this, "host_fp16_mut"); | |
| 588 | 332 | check_dtype(*this, Dtype::FP16, "host_fp16_mut"); | |
| 589 | 332 | return static_cast<uint16_t*>(data); | |
| 590 | } | ||
| 591 | 282 | const uint16_t* Tensor::host_fp16() const { | |
| 592 | 282 | check_host(*this, "host_fp16"); | |
| 593 | 282 | check_dtype(*this, Dtype::FP16, "host_fp16"); | |
| 594 | 282 | return static_cast<const uint16_t*>(data); | |
| 595 | } | ||
| 596 | |||
| 597 | 668 | uint16_t* Tensor::host_bf16_mut() { | |
| 598 | 668 | check_host(*this, "host_bf16_mut"); | |
| 599 | 668 | check_dtype(*this, Dtype::BF16, "host_bf16_mut"); | |
| 600 | 668 | return static_cast<uint16_t*>(data); | |
| 601 | } | ||
| 602 | 427 | const uint16_t* Tensor::host_bf16() const { | |
| 603 | 427 | check_host(*this, "host_bf16"); | |
| 604 | 427 | check_dtype(*this, Dtype::BF16, "host_bf16"); | |
| 605 | 427 | return static_cast<const uint16_t*>(data); | |
| 606 | } | ||
| 607 | |||
| 608 | 50404 | void* Tensor::host_raw_mut() { | |
| 609 | 50404 | check_host(*this, "host_raw_mut"); | |
| 610 | 50404 | return data; | |
| 611 | } | ||
| 612 | 50383 | const void* Tensor::host_raw() const { | |
| 613 | 50383 | check_host(*this, "host_raw"); | |
| 614 | 50383 | return data; | |
| 615 | } | ||
| 616 | |||
| 617 | 8022 | float& Tensor::at(int r, int c) { | |
| 618 | 8022 | check_host(*this, "at"); | |
| 619 | 8022 | check_dtype(*this, Dtype::FP32, "at"); | |
| 620 |
2/2✓ Branch 0 taken 8018 times.
✓ Branch 1 taken 4 times.
|
8022 | if (r < 0 || r >= rows || c < 0 || c >= cols) { |
| 621 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
|
4 | throw_msg("brotensor: at: index out of range"); |
| 622 | } | ||
| 623 | 8018 | return static_cast<float*>(data)[static_cast<std::size_t>(r) * cols + c]; | |
| 624 | 4 | } | |
| 625 | |||
| 626 | 3701 | float Tensor::at(int r, int c) const { | |
| 627 | 3701 | check_host(*this, "at"); | |
| 628 | 3701 | check_dtype(*this, Dtype::FP32, "at"); | |
| 629 |
2/2✓ Branch 0 taken 3700 times.
✓ Branch 1 taken 1 time.
|
3701 | if (r < 0 || r >= rows || c < 0 || c >= cols) { |
| 630 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: at: index out of range"); |
| 631 | } | ||
| 632 | 3700 | return static_cast<const float*>(data)[static_cast<std::size_t>(r) * cols + c]; | |
| 633 | 1 | } | |
| 634 | |||
| 635 | // ─── Host roundtrip ──────────────────────────────────────────────────────── | ||
| 636 | |||
| 637 | 60 | std::vector<float> Tensor::to_host_vector() const { | |
| 638 |
2/2✓ Branch 0 taken 59 times.
✓ Branch 1 taken 1 time.
|
60 | if (dtype != Dtype::FP32) { |
| 639 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: to_host_vector: dtype not FP32"); |
| 640 | } | ||
| 641 | 59 | std::vector<float> out(static_cast<std::size_t>(rows) * cols); | |
| 642 |
1/2✓ Branch 0 taken 59 times.
✗ Branch 1 not taken.
|
59 | const std::size_t n = bytes(); |
| 643 |
2/2✓ Branch 0 taken 58 times.
✓ Branch 1 taken 1 time.
|
59 | if (n == 0) return out; |
| 644 |
3/4✓ Branch 0 taken 58 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 16 times.
✓ Branch 3 taken 42 times.
|
58 | if (device.is_cpu()) { |
| 645 | 42 | std::memcpy(out.data(), data, n); | |
| 646 | 42 | } else { | |
| 647 |
2/4✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 16 times.
✗ Branch 3 not taken.
|
16 | detail::alloc_for(device).memcpy_d2h(out.data(), data, n, device.index); |
| 648 | } | ||
| 649 | 58 | return out; | |
| 650 | 60 | } | |
| 651 | |||
| 652 | 223 | std::vector<uint16_t> Tensor::to_host_vector_fp16() const { | |
| 653 |
2/2✓ Branch 0 taken 222 times.
✓ Branch 1 taken 1 time.
|
223 | if (dtype != Dtype::FP16) { |
| 654 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: to_host_vector_fp16: dtype not FP16"); |
| 655 | } | ||
| 656 | 222 | std::vector<uint16_t> out(static_cast<std::size_t>(rows) * cols); | |
| 657 |
1/2✓ Branch 0 taken 222 times.
✗ Branch 1 not taken.
|
222 | const std::size_t n = bytes(); |
| 658 |
2/2✓ Branch 0 taken 221 times.
✓ Branch 1 taken 1 time.
|
222 | if (n == 0) return out; |
| 659 |
3/4✓ Branch 0 taken 221 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 218 times.
✓ Branch 3 taken 3 times.
|
221 | if (device.is_cpu()) { |
| 660 | 3 | std::memcpy(out.data(), data, n); | |
| 661 | 3 | } else { | |
| 662 |
2/4✓ Branch 0 taken 218 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 218 times.
✗ Branch 3 not taken.
|
218 | detail::alloc_for(device).memcpy_d2h(out.data(), data, n, device.index); |
| 663 | } | ||
| 664 | 221 | return out; | |
| 665 | 223 | } | |
| 666 | |||
| 667 | 101 | std::vector<uint16_t> Tensor::to_host_vector_bf16() const { | |
| 668 |
2/2✓ Branch 0 taken 100 times.
✓ Branch 1 taken 1 time.
|
101 | if (dtype != Dtype::BF16) { |
| 669 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: to_host_vector_bf16: dtype not BF16"); |
| 670 | } | ||
| 671 | 100 | std::vector<uint16_t> out(static_cast<std::size_t>(rows) * cols); | |
| 672 |
1/2✓ Branch 0 taken 100 times.
✗ Branch 1 not taken.
|
100 | const std::size_t n = bytes(); |
| 673 |
2/2✓ Branch 0 taken 99 times.
✓ Branch 1 taken 1 time.
|
100 | if (n == 0) return out; |
| 674 |
3/4✓ Branch 0 taken 99 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 96 times.
✓ Branch 3 taken 3 times.
|
99 | if (device.is_cpu()) { |
| 675 | 3 | std::memcpy(out.data(), data, n); | |
| 676 | 3 | } else { | |
| 677 |
2/4✓ Branch 0 taken 96 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 96 times.
✗ Branch 3 not taken.
|
96 | detail::alloc_for(device).memcpy_d2h(out.data(), data, n, device.index); |
| 678 | } | ||
| 679 | 99 | return out; | |
| 680 | 101 | } | |
| 681 | |||
| 682 | 62 | void Tensor::copy_to_host(float* dst) const { | |
| 683 |
2/2✓ Branch 0 taken 61 times.
✓ Branch 1 taken 1 time.
|
62 | if (dtype != Dtype::FP32) { |
| 684 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: copy_to_host: dtype not FP32"); |
| 685 | } | ||
| 686 | 61 | const std::size_t n = bytes(); | |
| 687 |
2/2✓ Branch 0 taken 60 times.
✓ Branch 1 taken 1 time.
|
61 | if (n == 0) return; |
| 688 |
2/2✓ Branch 0 taken 12 times.
✓ Branch 1 taken 48 times.
|
60 | if (device.is_cpu()) { |
| 689 | 12 | std::memcpy(dst, data, n); | |
| 690 | 12 | } else { | |
| 691 | 48 | detail::alloc_for(device).memcpy_d2h(dst, data, n, device.index); | |
| 692 | } | ||
| 693 | 62 | } | |
| 694 | |||
| 695 | 155 | void Tensor::copy_to_host_fp16(uint16_t* dst) const { | |
| 696 |
2/2✓ Branch 0 taken 154 times.
✓ Branch 1 taken 1 time.
|
155 | if (dtype != Dtype::FP16) { |
| 697 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: copy_to_host_fp16: dtype not FP16"); |
| 698 | } | ||
| 699 | 154 | const std::size_t n = bytes(); | |
| 700 |
2/2✓ Branch 0 taken 153 times.
✓ Branch 1 taken 1 time.
|
154 | if (n == 0) return; |
| 701 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 153 times.
|
153 | if (device.is_cpu()) { |
| 702 | ✗ | std::memcpy(dst, data, n); | |
| 703 | ✗ | } else { | |
| 704 | 153 | detail::alloc_for(device).memcpy_d2h(dst, data, n, device.index); | |
| 705 | } | ||
| 706 | 155 | } | |
| 707 | |||
| 708 | 18 | void Tensor::copy_to_host_bf16(uint16_t* dst) const { | |
| 709 |
2/2✓ Branch 0 taken 17 times.
✓ Branch 1 taken 1 time.
|
18 | if (dtype != Dtype::BF16) { |
| 710 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: copy_to_host_bf16: dtype not BF16"); |
| 711 | } | ||
| 712 | 17 | const std::size_t n = bytes(); | |
| 713 |
2/2✓ Branch 0 taken 16 times.
✓ Branch 1 taken 1 time.
|
17 | if (n == 0) return; |
| 714 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 16 times.
|
16 | if (device.is_cpu()) { |
| 715 | ✗ | std::memcpy(dst, data, n); | |
| 716 | ✗ | } else { | |
| 717 | 16 | detail::alloc_for(device).memcpy_d2h(dst, data, n, device.index); | |
| 718 | } | ||
| 719 | 18 | } | |
| 720 | |||
| 721 | } // namespace brotensor | ||
| 722 |