src/tensor.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // brotensor::Tensor implementation. | ||
| 2 | // | ||
| 3 | // All allocation, freeing, and inter-device memory traffic routes through the | ||
| 4 | // backend AllocVTable obtained from detail::alloc_for(device). The Tensor type | ||
| 5 | // itself is backend-agnostic; CUDA/Metal/CPU specifics live entirely behind | ||
| 6 | // the vtable indirection. | ||
| 7 | |||
| 8 | #include <brotensor/tensor.h> | ||
| 9 | #include <brotensor/runtime.h> | ||
| 10 | #include <brotensor/detail/dispatch.h> | ||
| 11 | |||
| 12 | #include <cstdint> | ||
| 13 | #include <cstring> | ||
| 14 | #include <memory> | ||
| 15 | #include <stdexcept> | ||
| 16 | #include <string> | ||
| 17 | #include <vector> | ||
| 18 | |||
| 19 | #include <chrono> | ||
| 20 | #include <cstdio> | ||
| 21 | #include <cstdlib> | ||
| 22 | |||
| 23 | namespace brotensor { | ||
| 24 | |||
| 25 | // ─── env-gated weight-load profiler (BROTENSOR_TIME_LOAD=1) ────────────────── | ||
| 26 | // Attributes GPU weight-upload wall time to allocation vs H2D copy, accumulated | ||
| 27 | // across every from_host_*_on call and dumped once at process exit. Purely a | ||
| 28 | // diagnostic; the fast path (flag unset) is a single getenv-cached bool check. | ||
| 29 | namespace { | ||
| 30 | struct LoadProf { | ||
| 31 | 163 | bool on = false; | |
| 32 | 163 | double alloc = 0.0, copy = 0.0, bytes = 0.0; | |
| 33 | 163 | long calls = 0; | |
| 34 | 489 | LoadProf() { on = std::getenv("BROTENSOR_TIME_LOAD") != nullptr; } | |
| 35 | ✗ | ~LoadProf() { | |
| 36 | ✗ | if (!on || calls == 0) return; | |
| 37 | ✗ | std::fprintf(stderr, | |
| 38 | "[brotensor load] %ld uploads, %.2f GB: alloc %.2f s, h2d %.2f s " | ||
| 39 | "(%.2f GB/s copy)\n", | ||
| 40 | ✗ | calls, bytes / 1e9, alloc, copy, | |
| 41 | ✗ | copy > 0 ? (bytes / 1e9) / copy : 0.0); | |
| 42 | ✗ | } | |
| 43 | }; | ||
| 44 | 163 | LoadProf g_loadprof; | |
| 45 | 1440 | inline bool loadprof_on() { return g_loadprof.on; } | |
| 46 | } // namespace | ||
| 47 | |||
| 48 | // ─── dtype + device helpers ──────────────────────────────────────────────── | ||
| 49 | |||
| 50 | 117762 | int dtype_size_bytes(Dtype dt) { | |
| 51 |
7/8✓ Branch 0 taken 56227 times.
✓ Branch 1 taken 4070 times.
✓ Branch 2 taken 6234 times.
✓ Branch 3 taken 191 times.
✓ Branch 4 taken 51023 times.
✓ Branch 5 taken 5 times.
✗ Branch 6 not taken.
✓ Branch 7 taken 12 times.
|
117762 | switch (dt) { |
| 52 | 56227 | case Dtype::FP32: return 4; | |
| 53 | 4070 | case Dtype::FP16: return 2; | |
| 54 | 6234 | case Dtype::BF16: return 2; | |
| 55 | 191 | case Dtype::INT8: return 1; | |
| 56 | 51023 | case Dtype::INT32: return 4; | |
| 57 | 5 | case Dtype::F64: return 8; | |
| 58 | // Quant dtypes are block-addressed; no per-element size. | ||
| 59 | case Dtype::Q4_0: case Dtype::Q4_1: | ||
| 60 | case Dtype::Q5_0: case Dtype::Q5_1: | ||
| 61 | case Dtype::Q8_0: case Dtype::Q8_1: | ||
| 62 | case Dtype::Q2_K: case Dtype::Q3_K: | ||
| 63 | case Dtype::Q4_K: case Dtype::Q5_K: | ||
| 64 | case Dtype::Q6_K: case Dtype::Q8_K: | ||
| 65 | 12 | return 0; | |
| 66 | } | ||
| 67 | ✗ | return 0; | |
| 68 | 117762 | } | |
| 69 | |||
| 70 | 118020 | bool dtype_is_quant(Dtype dt) { | |
| 71 |
2/2✓ Branch 0 taken 394 times.
✓ Branch 1 taken 117626 times.
|
118020 | switch (dt) { |
| 72 | case Dtype::Q4_0: case Dtype::Q4_1: | ||
| 73 | case Dtype::Q5_0: case Dtype::Q5_1: | ||
| 74 | case Dtype::Q8_0: case Dtype::Q8_1: | ||
| 75 | case Dtype::Q2_K: case Dtype::Q3_K: | ||
| 76 | case Dtype::Q4_K: case Dtype::Q5_K: | ||
| 77 | case Dtype::Q6_K: case Dtype::Q8_K: | ||
| 78 | 394 | return true; | |
| 79 | default: | ||
| 80 | 117626 | return false; | |
| 81 | } | ||
| 82 | 118020 | } | |
| 83 | |||
| 84 | 401 | int dtype_block_size(Dtype dt) { | |
| 85 |
3/3✓ Branch 0 taken 328 times.
✓ Branch 1 taken 67 times.
✓ Branch 2 taken 6 times.
|
401 | switch (dt) { |
| 86 | case Dtype::Q4_0: case Dtype::Q4_1: | ||
| 87 | case Dtype::Q5_0: case Dtype::Q5_1: | ||
| 88 | case Dtype::Q8_0: case Dtype::Q8_1: | ||
| 89 | 328 | return 32; | |
| 90 | case Dtype::Q2_K: case Dtype::Q3_K: | ||
| 91 | case Dtype::Q4_K: case Dtype::Q5_K: | ||
| 92 | case Dtype::Q6_K: case Dtype::Q8_K: | ||
| 93 | 67 | return 256; | |
| 94 | default: | ||
| 95 | 6 | return 1; | |
| 96 | } | ||
| 97 | 401 | } | |
| 98 | |||
| 99 | 395 | int dtype_block_bytes(Dtype dt) { | |
| 100 |
13/13✓ Branch 0 taken 5 times.
✓ Branch 1 taken 11 times.
✓ Branch 2 taken 6 times.
✓ Branch 3 taken 10 times.
✓ Branch 4 taken 5 times.
✓ Branch 5 taken 5 times.
✓ Branch 6 taken 296 times.
✓ Branch 7 taken 5 times.
✓ Branch 8 taken 5 times.
✓ Branch 9 taken 5 times.
✓ Branch 10 taken 32 times.
✓ Branch 11 taken 5 times.
✓ Branch 12 taken 5 times.
|
395 | switch (dt) { |
| 101 | 10 | case Dtype::Q4_0: return 18; | |
| 102 | 5 | case Dtype::Q4_1: return 20; | |
| 103 | 5 | case Dtype::Q5_0: return 22; | |
| 104 | 5 | case Dtype::Q5_1: return 24; | |
| 105 | 296 | case Dtype::Q8_0: return 34; | |
| 106 | 5 | case Dtype::Q8_1: return 36; | |
| 107 | 5 | case Dtype::Q2_K: return 82; | |
| 108 | 5 | case Dtype::Q3_K: return 110; | |
| 109 | 32 | case Dtype::Q4_K: return 144; | |
| 110 | 5 | case Dtype::Q5_K: return 176; | |
| 111 | 11 | case Dtype::Q6_K: return 210; | |
| 112 | 5 | case Dtype::Q8_K: return 292; | |
| 113 | 6 | default: return dtype_size_bytes(dt); | |
| 114 | } | ||
| 115 | 395 | } | |
| 116 | |||
| 117 | 117992 | std::size_t dtype_storage_bytes(Dtype d, std::int64_t numel) { | |
| 118 |
2/2✓ Branch 0 taken 117990 times.
✓ Branch 1 taken 2 times.
|
117992 | if (numel < 0) { |
| 119 |
1/2✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
|
2 | throw std::runtime_error("brotensor: dtype_storage_bytes: negative numel"); |
| 120 | } | ||
| 121 |
2/2✓ Branch 0 taken 375 times.
✓ Branch 1 taken 117615 times.
|
117990 | if (!dtype_is_quant(d)) { |
| 122 | 235230 | return static_cast<std::size_t>(numel) * | |
| 123 | 117615 | static_cast<std::size_t>(dtype_size_bytes(d)); | |
| 124 | } | ||
| 125 | 375 | const int bs = dtype_block_size(d); | |
| 126 | 375 | const int bb = dtype_block_bytes(d); | |
| 127 |
2/2✓ Branch 0 taken 361 times.
✓ Branch 1 taken 14 times.
|
375 | if (numel % bs != 0) { |
| 128 |
1/2✓ Branch 0 taken 14 times.
✗ Branch 1 not taken.
|
14 | throw std::runtime_error( |
| 129 | "brotensor: dtype_storage_bytes: numel not a multiple of block size"); | ||
| 130 | } | ||
| 131 | 722 | return (static_cast<std::size_t>(numel) / static_cast<std::size_t>(bs)) * | |
| 132 | 361 | static_cast<std::size_t>(bb); | |
| 133 | 117976 | } | |
| 134 | |||
| 135 | 13 | const char* device_name(Device d) { | |
| 136 |
3/4✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✓ Branch 3 taken 1 time.
|
13 | switch (d) { |
| 137 | 1 | case Device::CPU: return "CPU"; | |
| 138 | 11 | case Device::CUDA: return "CUDA"; | |
| 139 | 1 | case Device::Metal: return "Metal"; | |
| 140 | } | ||
| 141 | ✗ | return "?"; | |
| 142 | 13 | } | |
| 143 | |||
| 144 | // ─── FP16 ↔ FP32 conversion (host-side IEEE 754 binary16) ────────────────── | ||
| 145 | |||
| 146 | 2124183 | uint16_t fp32_to_fp16_bits(float v) { | |
| 147 | uint32_t x; | ||
| 148 | 2124183 | std::memcpy(&x, &v, 4); | |
| 149 | 2124183 | const uint32_t sign = (x >> 16) & 0x8000u; | |
| 150 | 2124183 | int32_t exp = static_cast<int32_t>((x >> 23) & 0xFFu) - 127 + 15; | |
| 151 | 2124183 | uint32_t mant = x & 0x7FFFFFu; | |
| 152 | |||
| 153 |
2/2✓ Branch 0 taken 68768 times.
✓ Branch 1 taken 2055415 times.
|
2124183 | if (((x >> 23) & 0xFFu) == 0xFFu) { |
| 154 | // Inf or NaN. | ||
| 155 | 68768 | uint16_t out = static_cast<uint16_t>(sign | 0x7C00u); | |
| 156 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 68768 times.
|
68768 | if (mant) out |= 0x0200u; // quiet NaN |
| 157 | 68768 | return out; | |
| 158 | } | ||
| 159 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 2055415 times.
|
2055415 | if (exp >= 0x1F) { |
| 160 | ✗ | return static_cast<uint16_t>(sign | 0x7C00u); | |
| 161 | } | ||
| 162 |
2/2✓ Branch 0 taken 997 times.
✓ Branch 1 taken 2054418 times.
|
2055415 | if (exp <= 0) { |
| 163 |
2/2✓ Branch 0 taken 499 times.
✓ Branch 1 taken 498 times.
|
997 | if (exp < -10) return static_cast<uint16_t>(sign); |
| 164 | 498 | mant |= 0x800000u; | |
| 165 | 498 | const int shift = 14 - exp; | |
| 166 | 498 | const uint32_t round = 1u << (shift - 1); | |
| 167 | 498 | uint32_t r = (mant + round) >> shift; | |
| 168 | 498 | return static_cast<uint16_t>(sign | r); | |
| 169 | } | ||
| 170 | 2054418 | const uint32_t lsb = (mant >> 13) & 1u; | |
| 171 | 2054418 | const uint32_t round = 0x00001000u + lsb - 1u; | |
| 172 | 2054418 | uint32_t r_mant = (mant + round) >> 13; | |
| 173 | 2054418 | uint32_t r_exp = static_cast<uint32_t>(exp); | |
| 174 |
2/2✓ Branch 0 taken 2053272 times.
✓ Branch 1 taken 1146 times.
|
2054418 | if (r_mant & 0x400u) { |
| 175 | 1146 | r_mant = 0; | |
| 176 | 1146 | r_exp += 1; | |
| 177 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1146 times.
|
1146 | if (r_exp >= 0x1F) return static_cast<uint16_t>(sign | 0x7C00u); |
| 178 | 1146 | } | |
| 179 | 2054418 | return static_cast<uint16_t>(sign | (r_exp << 10) | r_mant); | |
| 180 | 2124183 | } | |
| 181 | |||
| 182 | 2040335 | float fp16_bits_to_fp32(uint16_t bits) { | |
| 183 | 2040335 | const uint32_t sign = (static_cast<uint32_t>(bits) & 0x8000u) << 16; | |
| 184 | 2040335 | uint32_t exp = (bits >> 10) & 0x1Fu; | |
| 185 | 2040335 | uint32_t mant = bits & 0x3FFu; | |
| 186 | uint32_t out; | ||
| 187 |
2/2✓ Branch 0 taken 2032383 times.
✓ Branch 1 taken 7952 times.
|
2040335 | if (exp == 0) { |
| 188 |
2/2✓ Branch 0 taken 1415 times.
✓ Branch 1 taken 6537 times.
|
7952 | if (mant == 0) { |
| 189 | 6537 | out = sign; | |
| 190 | 6537 | } else { | |
| 191 |
2/2✓ Branch 0 taken 2340 times.
✓ Branch 1 taken 1415 times.
|
3755 | while ((mant & 0x400u) == 0) { mant <<= 1; exp -= 1; } |
| 192 | 1415 | mant &= 0x3FFu; | |
| 193 | 1415 | uint32_t e = (exp + (127 - 15) + 1) & 0xFFu; | |
| 194 | 1415 | out = sign | (e << 23) | (mant << 13); | |
| 195 | } | ||
| 196 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 2032383 times.
|
2040335 | } else if (exp == 0x1F) { |
| 197 | ✗ | out = sign | 0x7F800000u | (mant << 13); | |
| 198 | ✗ | } else { | |
| 199 | 2032383 | uint32_t e = exp + (127 - 15); | |
| 200 | 2032383 | out = sign | (e << 23) | (mant << 13); | |
| 201 | } | ||
| 202 | float f; | ||
| 203 | 2040335 | std::memcpy(&f, &out, 4); | |
| 204 | 2040335 | return f; | |
| 205 | } | ||
| 206 | |||
| 207 | // ─── BF16 ↔ FP32 conversion (host-side bfloat16) ─────────────────────────── | ||
| 208 | |||
| 209 | 2542136 | uint16_t fp32_to_bf16_bits(float v) { | |
| 210 | uint32_t x; | ||
| 211 | 2542136 | std::memcpy(&x, &v, 4); | |
| 212 | // NaN: keep it a NaN (the rounding add below could otherwise carry a | ||
| 213 | // NaN's mantissa into the exponent and produce an infinity). | ||
| 214 |
4/4✓ Branch 0 taken 4 times.
✓ Branch 1 taken 2542132 times.
✓ Branch 2 taken 2 times.
✓ Branch 3 taken 2 times.
|
2542136 | if (((x >> 23) & 0xFFu) == 0xFFu && (x & 0x7FFFFFu) != 0) { |
| 215 | 2 | return static_cast<uint16_t>((x >> 16) | 0x0040u); | |
| 216 | } | ||
| 217 | // Round to nearest, ties to even: add 0x7FFF + LSB-of-result. | ||
| 218 | 2542134 | const uint32_t rounding_bias = 0x7FFFu + ((x >> 16) & 1u); | |
| 219 | 2542134 | x += rounding_bias; | |
| 220 | 2542134 | return static_cast<uint16_t>(x >> 16); | |
| 221 | 2542136 | } | |
| 222 | |||
| 223 | 1997239 | float bf16_bits_to_fp32(uint16_t bits) { | |
| 224 | 1997239 | const uint32_t x = static_cast<uint32_t>(bits) << 16; | |
| 225 | float f; | ||
| 226 | 1997239 | std::memcpy(&f, &x, 4); | |
| 227 | 1997239 | return f; | |
| 228 | } | ||
| 229 | |||
| 230 | // ─── internal helpers ────────────────────────────────────────────────────── | ||
| 231 | |||
| 232 | namespace { | ||
| 233 | |||
| 234 | 39 | [[noreturn]] void throw_msg(const std::string& m) { | |
| 235 |
1/2✓ Branch 0 taken 39 times.
✗ Branch 1 not taken.
|
39 | throw std::runtime_error(m); |
| 236 | ✗ | } | |
| 237 | |||
| 238 | 56037455 | void check_host(const Tensor& t, const char* who) { | |
| 239 |
2/2✓ Branch 0 taken 56037445 times.
✓ Branch 1 taken 10 times.
|
56037455 | if (t.device != Device::CPU) { |
| 240 | 10 | std::string m = "brotensor: "; | |
| 241 |
1/2✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
|
10 | m += who; |
| 242 |
1/2✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
|
10 | m += ": tensor is on "; |
| 243 |
1/2✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
|
10 | m += device_name(t.device); |
| 244 |
1/2✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
|
10 | m += ", not CPU"; |
| 245 |
1/2✓ Branch 0 taken 10 times.
✗ Branch 1 not taken.
|
10 | throw_msg(m); |
| 246 | 10 | } | |
| 247 | 56037455 | } | |
| 248 | |||
| 249 | // Guard against negative dimensions before they are cast to std::size_t in | ||
| 250 | // bytes() — an unchecked (size_t)(-1) underflows to an astronomical count | ||
| 251 | // that sails past the bytes==0 short-circuits and reaches backend_alloc. | ||
| 252 | 78575 | void check_dims(int r, int c, const char* who) { | |
| 253 |
2/2✓ Branch 0 taken 78568 times.
✓ Branch 1 taken 7 times.
|
78575 | if (r < 0 || c < 0) { |
| 254 | 7 | std::string m = "brotensor: "; | |
| 255 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | m += who; |
| 256 |
1/2✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
|
7 | m += ": negative dimension"; |
| 257 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
|
7 | throw_msg(m); |
| 258 | 7 | } | |
| 259 | 78575 | } | |
| 260 | |||
| 261 | 55936660 | void check_dtype(const Tensor& t, Dtype expected, const char* who) { | |
| 262 |
2/2✓ Branch 0 taken 55936651 times.
✓ Branch 1 taken 9 times.
|
55936660 | if (t.dtype != expected) { |
| 263 | 9 | std::string m = "brotensor: "; | |
| 264 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | m += who; |
| 265 |
1/2✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
|
9 | m += ": dtype mismatch"; |
| 266 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
|
9 | throw_msg(m); |
| 267 | 9 | } | |
| 268 | 55936660 | } | |
| 269 | |||
| 270 | 78331 | void* backend_alloc(Device d, std::size_t bytes) { | |
| 271 |
2/2✓ Branch 0 taken 78132 times.
✓ Branch 1 taken 199 times.
|
78331 | if (bytes == 0) return nullptr; |
| 272 | 78132 | return detail::alloc_for(d).alloc(bytes); | |
| 273 | 78331 | } | |
| 274 | |||
| 275 | 78132 | void backend_free(Device d, void* p) { | |
| 276 |
1/2✓ Branch 0 taken 78132 times.
✗ Branch 1 not taken.
|
78132 | if (!p) return; |
| 277 | 78132 | detail::alloc_for(d).free(p); | |
| 278 | 78132 | } | |
| 279 | |||
| 280 | 10679 | void backend_zero(Device d, void* p, std::size_t bytes) { | |
| 281 |
3/4✓ Branch 0 taken 10674 times.
✓ Branch 1 taken 5 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 10674 times.
|
10679 | if (bytes == 0 || !p) return; |
| 282 | 10674 | detail::alloc_for(d).memset_zero(p, bytes); | |
| 283 | 10679 | } | |
| 284 | |||
| 285 | } // namespace | ||
| 286 | |||
| 287 | // ─── Tensor lifetime ─────────────────────────────────────────────────────── | ||
| 288 | |||
| 289 | 161038 | Tensor::~Tensor() { | |
| 290 |
1/2✓ Branch 0 taken 80519 times.
✗ Branch 1 not taken.
|
80519 | release_(); |
| 291 | 161038 | } | |
| 292 | |||
| 293 | 140729 | void Tensor::release_() { | |
| 294 |
3/4✓ Branch 0 taken 78132 times.
✓ Branch 1 taken 62597 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 78132 times.
|
140729 | if (owns_ && data) { |
| 295 | // alloc_for can throw if the backend was somehow un-registered between | ||
| 296 | // alloc and free; let it propagate — better than leaking a different | ||
| 297 | // backend's pointer. | ||
| 298 | 78132 | backend_free(device, data); | |
| 299 | 78132 | } | |
| 300 | 140729 | data = nullptr; | |
| 301 | 140729 | owns_ = false; | |
| 302 | 140729 | cap_bytes_ = 0; | |
| 303 | 140729 | } | |
| 304 | |||
| 305 | 542 | Tensor::Tensor(Tensor&& o) noexcept | |
| 306 | 271 | : data(o.data), rows(o.rows), cols(o.cols), | |
| 307 | 271 | dtype(o.dtype), device(o.device), owns_(o.owns_), | |
| 308 | 542 | cap_bytes_(o.cap_bytes_) { | |
| 309 | 271 | o.data = nullptr; | |
| 310 | 271 | o.rows = 0; | |
| 311 | 271 | o.cols = 0; | |
| 312 | 271 | o.owns_ = false; | |
| 313 | 271 | o.cap_bytes_ = 0; | |
| 314 | 542 | } | |
| 315 | |||
| 316 | 1645 | Tensor& Tensor::operator=(Tensor&& o) noexcept { | |
| 317 |
2/2✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1644 times.
|
1645 | if (this != &o) { |
| 318 |
1/2✓ Branch 0 taken 1644 times.
✗ Branch 1 not taken.
|
1644 | release_(); |
| 319 | 1644 | data = o.data; | |
| 320 | 1644 | rows = o.rows; | |
| 321 | 1644 | cols = o.cols; | |
| 322 | 1644 | dtype = o.dtype; | |
| 323 | 1644 | device = o.device; | |
| 324 | 1644 | owns_ = o.owns_; | |
| 325 | 1644 | cap_bytes_ = o.cap_bytes_; | |
| 326 | 1644 | o.data = nullptr; | |
| 327 | 1644 | o.rows = 0; | |
| 328 | 1644 | o.cols = 0; | |
| 329 | 1644 | o.owns_ = false; | |
| 330 | 1644 | o.cap_bytes_ = 0; | |
| 331 | 1644 | } | |
| 332 | 1645 | return *this; | |
| 333 | } | ||
| 334 | |||
| 335 | // Copy ctor / assignment: device-aware deep copy via clone(). Implemented in | ||
| 336 | // terms of the move assignment so all the ownership bookkeeping lives in one | ||
| 337 | // place. | ||
| 338 | 874 | Tensor::Tensor(const Tensor& o) { | |
| 339 | 437 | *this = o.clone(); | |
| 340 | 437 | } | |
| 341 | |||
| 342 | 47 | Tensor& Tensor::operator=(const Tensor& o) { | |
| 343 |
2/2✓ Branch 0 taken 1 time.
✓ Branch 1 taken 46 times.
|
47 | if (this != &o) *this = o.clone(); |
| 344 | 47 | return *this; | |
| 345 | } | ||
| 346 | |||
| 347 | 59125 | std::size_t Tensor::bytes() const { | |
| 348 | 59125 | return dtype_storage_bytes( | |
| 349 | 59125 | dtype, | |
| 350 | 59125 | static_cast<std::int64_t>(rows) * static_cast<std::int64_t>(cols)); | |
| 351 | } | ||
| 352 | |||
| 353 | // ─── Factories ───────────────────────────────────────────────────────────── | ||
| 354 | |||
| 355 | 19766 | Tensor Tensor::empty_on(Device d, int r, int c, Dtype dt) { | |
| 356 | 19766 | check_dims(r, c, "empty_on"); | |
| 357 | 19766 | Tensor t; | |
| 358 | 19766 | t.device = d; | |
| 359 | 19766 | t.dtype = dt; | |
| 360 | 19766 | t.rows = r; | |
| 361 | 19766 | t.cols = c; | |
| 362 |
3/4✓ Branch 0 taken 19765 times.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 19765 times.
✗ Branch 3 not taken.
|
19766 | t.data = backend_alloc(d, t.bytes()); |
| 363 | 19765 | t.owns_ = (t.data != nullptr); | |
| 364 |
3/4✓ Branch 0 taken 19627 times.
✓ Branch 1 taken 138 times.
✓ Branch 2 taken 19627 times.
✗ Branch 3 not taken.
|
19765 | t.cap_bytes_ = t.owns_ ? t.bytes() : 0; |
| 365 | 19765 | return t; | |
| 366 | 19766 | } | |
| 367 | |||
| 368 | 10430 | Tensor Tensor::zeros_on(Device d, int r, int c, Dtype dt) { | |
| 369 | 10430 | Tensor t = empty_on(d, r, c, dt); | |
| 370 |
2/4✓ Branch 0 taken 10430 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 10430 times.
✗ Branch 3 not taken.
|
10430 | backend_zero(d, t.data, t.bytes()); |
| 371 | 10430 | return t; | |
| 372 | 10430 | } | |
| 373 | |||
| 374 | 1 | Tensor Tensor::empty(int r, int c, Dtype dt) { | |
| 375 | 1 | return empty_on(default_device(), r, c, dt); | |
| 376 | } | ||
| 377 | |||
| 378 | 3 | Tensor Tensor::zeros(int r, int c, Dtype dt) { | |
| 379 | 3 | return zeros_on(default_device(), r, c, dt); | |
| 380 | } | ||
| 381 | |||
| 382 | namespace { | ||
| 383 | // Shared upload core for from_host_*_on. When BROTENSOR_TIME_LOAD is set it | ||
| 384 | // attributes host wall time to the empty_on (alloc enqueue) and the H2D copy. | ||
| 385 | 1440 | Tensor from_host_typed(Device d, const void* src, int r, int c, Dtype dt) { | |
| 386 | using clock = std::chrono::steady_clock; | ||
| 387 |
1/2✓ Branch 0 taken 1440 times.
✗ Branch 1 not taken.
|
1440 | const bool prof = loadprof_on() && d != Device::CPU; |
| 388 | 1440 | auto t0 = clock::now(); | |
| 389 | 1440 | Tensor t = Tensor::empty_on(d, r, c, dt); | |
| 390 |
1/2✓ Branch 0 taken 1440 times.
✗ Branch 1 not taken.
|
1440 | const std::size_t n = t.bytes(); |
| 391 |
1/2✓ Branch 0 taken 1440 times.
✗ Branch 1 not taken.
|
1440 | if (n == 0) return t; |
| 392 | 1440 | auto t1 = clock::now(); | |
| 393 |
2/2✓ Branch 0 taken 955 times.
✓ Branch 1 taken 485 times.
|
1440 | if (d == Device::CPU) { |
| 394 | 485 | std::memcpy(t.data, src, n); | |
| 395 | 485 | } else { | |
| 396 |
2/4✓ Branch 0 taken 955 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 955 times.
✗ Branch 3 not taken.
|
955 | detail::alloc_for(d).memcpy_h2d(t.data, src, n); |
| 397 | } | ||
| 398 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1440 times.
|
1440 | if (prof) { |
| 399 | ✗ | auto t2 = clock::now(); | |
| 400 | ✗ | g_loadprof.alloc += std::chrono::duration<double>(t1 - t0).count(); | |
| 401 | ✗ | g_loadprof.copy += std::chrono::duration<double>(t2 - t1).count(); | |
| 402 | ✗ | g_loadprof.bytes += static_cast<double>(n); | |
| 403 | ✗ | ++g_loadprof.calls; | |
| 404 | ✗ | } | |
| 405 | 1440 | return t; | |
| 406 | 1440 | } | |
| 407 | } // namespace | ||
| 408 | |||
| 409 | 574 | Tensor Tensor::from_host_on(Device d, const float* src, int r, int c) { | |
| 410 | 574 | return from_host_typed(d, src, r, c, Dtype::FP32); | |
| 411 | } | ||
| 412 | |||
| 413 | 586 | Tensor Tensor::from_host_fp16_on(Device d, const uint16_t* src, int r, int c) { | |
| 414 | 586 | return from_host_typed(d, src, r, c, Dtype::FP16); | |
| 415 | } | ||
| 416 | |||
| 417 | 280 | Tensor Tensor::from_host_bf16_on(Device d, const uint16_t* src, int r, int c) { | |
| 418 | 280 | return from_host_typed(d, src, r, c, Dtype::BF16); | |
| 419 | } | ||
| 420 | |||
| 421 | 5 | Tensor Tensor::from_host_int8_on(Device d, const int8_t* src, int r, int c) { | |
| 422 | 5 | Tensor t = empty_on(d, r, c, Dtype::INT8); | |
| 423 |
1/2✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
|
5 | const std::size_t n = t.bytes(); |
| 424 |
1/2✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
|
5 | if (n == 0) return t; |
| 425 |
1/2✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
|
5 | if (d == Device::CPU) { |
| 426 | ✗ | std::memcpy(t.data, src, n); | |
| 427 | ✗ | } else { | |
| 428 |
2/4✓ Branch 0 taken 5 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 5 times.
✗ Branch 3 not taken.
|
5 | detail::alloc_for(d).memcpy_h2d(t.data, src, n); |
| 429 | } | ||
| 430 | 5 | return t; | |
| 431 | 5 | } | |
| 432 | |||
| 433 | 11 | Tensor Tensor::from_raw_bytes_on(Device target, const void* src, int r, int c, | |
| 434 | Dtype dt, std::size_t nbytes) { | ||
| 435 | 11 | Tensor t = empty_on(target, r, c, dt); | |
| 436 |
3/4✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 10 times.
✓ Branch 3 taken 1 time.
|
11 | if (t.bytes() != nbytes) { |
| 437 |
2/4✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 1 time.
|
1 | throw std::runtime_error( |
| 438 | "brotensor: from_raw_bytes_on: nbytes does not match tensor size"); | ||
| 439 | } | ||
| 440 |
2/2✓ Branch 0 taken 9 times.
✓ Branch 1 taken 1 time.
|
10 | if (nbytes == 0) return t; |
| 441 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 9 times.
|
9 | if (target == Device::CPU) { |
| 442 | 9 | std::memcpy(t.data, src, nbytes); | |
| 443 | 9 | } else { | |
| 444 | ✗ | detail::alloc_for(target).memcpy_h2d(t.data, src, nbytes); | |
| 445 | } | ||
| 446 | 9 | return t; | |
| 447 | 11 | } | |
| 448 | |||
| 449 | 13 | Tensor Tensor::from_host(const float* src, int r, int c) { | |
| 450 | 13 | return from_host_on(default_device(), src, r, c); | |
| 451 | } | ||
| 452 | |||
| 453 | 6 | Tensor Tensor::from_host_fp16(const uint16_t* src, int r, int c) { | |
| 454 | 6 | return from_host_fp16_on(default_device(), src, r, c); | |
| 455 | } | ||
| 456 | |||
| 457 | 5 | Tensor Tensor::from_host_bf16(const uint16_t* src, int r, int c) { | |
| 458 | 5 | return from_host_bf16_on(default_device(), src, r, c); | |
| 459 | } | ||
| 460 | |||
| 461 | ✗ | Tensor Tensor::from_host_int8(const int8_t* src, int r, int c) { | |
| 462 | ✗ | return from_host_int8_on(default_device(), src, r, c); | |
| 463 | } | ||
| 464 | |||
| 465 | 153 | Tensor Tensor::view(Device d, void* data, int r, int c, Dtype dt) { | |
| 466 | 153 | Tensor t; | |
| 467 | 153 | t.device = d; | |
| 468 | 153 | t.dtype = dt; | |
| 469 | 153 | t.rows = r; | |
| 470 | 153 | t.cols = c; | |
| 471 | 153 | t.data = data; | |
| 472 | 153 | t.owns_ = false; | |
| 473 | 153 | return t; | |
| 474 | 153 | } | |
| 475 | |||
| 476 | // ─── Migration ───────────────────────────────────────────────────────────── | ||
| 477 | |||
| 478 | 1663 | Tensor Tensor::clone() const { | |
| 479 | 1663 | Tensor t = empty_on(device, rows, cols, dtype); | |
| 480 |
1/2✓ Branch 0 taken 1663 times.
✗ Branch 1 not taken.
|
1663 | const std::size_t n = bytes(); |
| 481 |
3/4✓ Branch 0 taken 1660 times.
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1660 times.
|
1663 | if (n == 0 || !data) return t; |
| 482 |
2/2✓ Branch 0 taken 13 times.
✓ Branch 1 taken 1647 times.
|
1660 | if (device == Device::CPU) { |
| 483 | 1647 | std::memcpy(t.data, data, n); | |
| 484 | 1647 | } else { | |
| 485 |
2/4✓ Branch 0 taken 13 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 13 times.
✗ Branch 3 not taken.
|
13 | detail::alloc_for(device).memcpy_d2d(t.data, data, n); |
| 486 | } | ||
| 487 | 1660 | return t; | |
| 488 | 1663 | } | |
| 489 | |||
| 490 | 5124 | Tensor Tensor::to(Device target) const { | |
| 491 |
2/2✓ Branch 0 taken 1 time.
✓ Branch 1 taken 5123 times.
|
5124 | if (target == device) return clone(); |
| 492 | 5123 | Tensor t = empty_on(target, rows, cols, dtype); | |
| 493 |
1/2✓ Branch 0 taken 5123 times.
✗ Branch 1 not taken.
|
5123 | const std::size_t n = bytes(); |
| 494 |
2/4✓ Branch 0 taken 5123 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 5123 times.
|
5123 | if (n == 0 || !data) return t; |
| 495 | |||
| 496 |
2/2✓ Branch 0 taken 3229 times.
✓ Branch 1 taken 1894 times.
|
5123 | if (device == Device::CPU) { |
| 497 | // CPU → GPU. | ||
| 498 |
2/4✓ Branch 0 taken 3229 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 3229 times.
✗ Branch 3 not taken.
|
3229 | detail::alloc_for(target).memcpy_h2d(t.data, data, n); |
| 499 |
1/2✓ Branch 0 taken 1894 times.
✗ Branch 1 not taken.
|
5123 | } else if (target == Device::CPU) { |
| 500 | // GPU → CPU. | ||
| 501 |
2/4✓ Branch 0 taken 1894 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1894 times.
✗ Branch 3 not taken.
|
1894 | detail::alloc_for(device).memcpy_d2h(t.data, data, n); |
| 502 | 1894 | } else { | |
| 503 | // GPU → different GPU backend. Bounce through host. The staging | ||
| 504 | // buffer is fully overwritten by the D2H copy below, so allocate it | ||
| 505 | // uninitialized rather than paying for a zero-fill of a multi-GB | ||
| 506 | // tensor just to immediately clobber it. | ||
| 507 | ✗ | std::unique_ptr<unsigned char[]> staging(new unsigned char[n]); | |
| 508 | ✗ | detail::alloc_for(device).memcpy_d2h(staging.get(), data, n); | |
| 509 | ✗ | detail::alloc_for(target).memcpy_h2d(t.data, staging.get(), n); | |
| 510 | ✗ | } | |
| 511 | 5123 | return t; | |
| 512 | 10247 | } | |
| 513 | |||
| 514 | // ─── Mutators ────────────────────────────────────────────────────────────── | ||
| 515 | |||
| 516 | 249 | void Tensor::zero() { | |
| 517 | 249 | backend_zero(device, data, bytes()); | |
| 518 | 249 | } | |
| 519 | |||
| 520 | 58799 | void Tensor::resize(int r, int c, Dtype dt) { | |
| 521 | 58799 | check_dims(r, c, "resize"); | |
| 522 |
8/8✓ Branch 0 taken 111 times.
✓ Branch 1 taken 58688 times.
✓ Branch 2 taken 63 times.
✓ Branch 3 taken 48 times.
✓ Branch 4 taken 58 times.
✓ Branch 5 taken 5 times.
✓ Branch 6 taken 55 times.
✓ Branch 7 taken 3 times.
|
58799 | if (r == rows && c == cols && dt == dtype && data != nullptr) return; |
| 523 | 58796 | const std::size_t new_bytes = dtype_storage_bytes( | |
| 524 | 58796 | dt, | |
| 525 | 58796 | static_cast<std::int64_t>(r) * static_cast<std::int64_t>(c)); | |
| 526 | // A non-owning view over real storage cannot be reshaped: reallocating | ||
| 527 | // would silently allocate fresh owned memory and sever the view, leaving | ||
| 528 | // callers with a tensor that no longer aliases what they passed to | ||
| 529 | // view(). Reject it explicitly rather than converting it in place. | ||
| 530 | // (A default-constructed / released tensor — owns_ == false but | ||
| 531 | // data == nullptr — is not a view and resizes normally.) | ||
| 532 |
4/4✓ Branch 0 taken 58474 times.
✓ Branch 1 taken 322 times.
✓ Branch 2 taken 58472 times.
✓ Branch 3 taken 2 times.
|
58796 | if (!owns_ && data != nullptr) { |
| 533 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
|
2 | throw_msg("brotensor: resize: cannot reshape a non-owning view; " |
| 534 | "allocate a fresh tensor or re-view() with the new shape"); | ||
| 535 | } | ||
| 536 | // Keep the existing storage whenever the new shape fits — the buffer | ||
| 537 | // stabilises at its high-water mark and its device pointer stays put | ||
| 538 | // (required when the op sequence using it is CUDA-graph captured). | ||
| 539 |
4/4✓ Branch 0 taken 289 times.
✓ Branch 1 taken 58505 times.
✓ Branch 2 taken 228 times.
✓ Branch 3 taken 61 times.
|
58794 | if (new_bytes > cap_bytes_ || !owns_) { |
| 540 | 58566 | release_(); | |
| 541 | 58566 | data = backend_alloc(device, new_bytes); | |
| 542 | 58566 | owns_ = (data != nullptr); | |
| 543 |
2/2✓ Branch 0 taken 58505 times.
✓ Branch 1 taken 61 times.
|
58566 | cap_bytes_ = owns_ ? new_bytes : 0; |
| 544 | 58566 | } | |
| 545 | 58794 | rows = r; | |
| 546 | 58794 | cols = c; | |
| 547 | 58794 | dtype = dt; | |
| 548 | 58799 | } | |
| 549 | |||
| 550 | // ─── Host accessors ──────────────────────────────────────────────────────── | ||
| 551 | |||
| 552 | 45753244 | float* Tensor::host_f32_mut() { | |
| 553 | 45753244 | check_host(*this, "host_f32_mut"); | |
| 554 | 45753244 | check_dtype(*this, Dtype::FP32, "host_f32_mut"); | |
| 555 | 45753244 | return static_cast<float*>(data); | |
| 556 | } | ||
| 557 | 10170206 | const float* Tensor::host_f32() const { | |
| 558 | 10170206 | check_host(*this, "host_f32"); | |
| 559 | 10170206 | check_dtype(*this, Dtype::FP32, "host_f32"); | |
| 560 | 10170206 | return static_cast<const float*>(data); | |
| 561 | } | ||
| 562 | |||
| 563 | 228 | uint16_t* Tensor::host_fp16_mut() { | |
| 564 | 228 | check_host(*this, "host_fp16_mut"); | |
| 565 | 228 | check_dtype(*this, Dtype::FP16, "host_fp16_mut"); | |
| 566 | 228 | return static_cast<uint16_t*>(data); | |
| 567 | } | ||
| 568 | 203 | const uint16_t* Tensor::host_fp16() const { | |
| 569 | 203 | check_host(*this, "host_fp16"); | |
| 570 | 203 | check_dtype(*this, Dtype::FP16, "host_fp16"); | |
| 571 | 203 | return static_cast<const uint16_t*>(data); | |
| 572 | } | ||
| 573 | |||
| 574 | 651 | uint16_t* Tensor::host_bf16_mut() { | |
| 575 | 651 | check_host(*this, "host_bf16_mut"); | |
| 576 | 651 | check_dtype(*this, Dtype::BF16, "host_bf16_mut"); | |
| 577 | 651 | return static_cast<uint16_t*>(data); | |
| 578 | } | ||
| 579 | 413 | const uint16_t* Tensor::host_bf16() const { | |
| 580 | 413 | check_host(*this, "host_bf16"); | |
| 581 | 413 | check_dtype(*this, Dtype::BF16, "host_bf16"); | |
| 582 | 413 | return static_cast<const uint16_t*>(data); | |
| 583 | } | ||
| 584 | |||
| 585 | 50404 | void* Tensor::host_raw_mut() { | |
| 586 | 50404 | check_host(*this, "host_raw_mut"); | |
| 587 | 50404 | return data; | |
| 588 | } | ||
| 589 | 50383 | const void* Tensor::host_raw() const { | |
| 590 | 50383 | check_host(*this, "host_raw"); | |
| 591 | 50383 | return data; | |
| 592 | } | ||
| 593 | |||
| 594 | 8022 | float& Tensor::at(int r, int c) { | |
| 595 | 8022 | check_host(*this, "at"); | |
| 596 | 8022 | check_dtype(*this, Dtype::FP32, "at"); | |
| 597 |
2/2✓ Branch 0 taken 8018 times.
✓ Branch 1 taken 4 times.
|
8022 | if (r < 0 || r >= rows || c < 0 || c >= cols) { |
| 598 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
|
4 | throw_msg("brotensor: at: index out of range"); |
| 599 | } | ||
| 600 | 8018 | return static_cast<float*>(data)[static_cast<std::size_t>(r) * cols + c]; | |
| 601 | 4 | } | |
| 602 | |||
| 603 | 3701 | float Tensor::at(int r, int c) const { | |
| 604 | 3701 | check_host(*this, "at"); | |
| 605 | 3701 | check_dtype(*this, Dtype::FP32, "at"); | |
| 606 |
2/2✓ Branch 0 taken 3700 times.
✓ Branch 1 taken 1 time.
|
3701 | if (r < 0 || r >= rows || c < 0 || c >= cols) { |
| 607 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: at: index out of range"); |
| 608 | } | ||
| 609 | 3700 | return static_cast<const float*>(data)[static_cast<std::size_t>(r) * cols + c]; | |
| 610 | 1 | } | |
| 611 | |||
| 612 | // ─── Host roundtrip ──────────────────────────────────────────────────────── | ||
| 613 | |||
| 614 | 60 | std::vector<float> Tensor::to_host_vector() const { | |
| 615 |
2/2✓ Branch 0 taken 59 times.
✓ Branch 1 taken 1 time.
|
60 | if (dtype != Dtype::FP32) { |
| 616 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: to_host_vector: dtype not FP32"); |
| 617 | } | ||
| 618 | 59 | std::vector<float> out(static_cast<std::size_t>(rows) * cols); | |
| 619 |
1/2✓ Branch 0 taken 59 times.
✗ Branch 1 not taken.
|
59 | const std::size_t n = bytes(); |
| 620 |
2/2✓ Branch 0 taken 58 times.
✓ Branch 1 taken 1 time.
|
59 | if (n == 0) return out; |
| 621 |
2/2✓ Branch 0 taken 16 times.
✓ Branch 1 taken 42 times.
|
58 | if (device == Device::CPU) { |
| 622 | 42 | std::memcpy(out.data(), data, n); | |
| 623 | 42 | } else { | |
| 624 |
2/4✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 16 times.
✗ Branch 3 not taken.
|
16 | detail::alloc_for(device).memcpy_d2h(out.data(), data, n); |
| 625 | } | ||
| 626 | 58 | return out; | |
| 627 | 60 | } | |
| 628 | |||
| 629 | 210 | std::vector<uint16_t> Tensor::to_host_vector_fp16() const { | |
| 630 |
2/2✓ Branch 0 taken 209 times.
✓ Branch 1 taken 1 time.
|
210 | if (dtype != Dtype::FP16) { |
| 631 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: to_host_vector_fp16: dtype not FP16"); |
| 632 | } | ||
| 633 | 209 | std::vector<uint16_t> out(static_cast<std::size_t>(rows) * cols); | |
| 634 |
1/2✓ Branch 0 taken 209 times.
✗ Branch 1 not taken.
|
209 | const std::size_t n = bytes(); |
| 635 |
2/2✓ Branch 0 taken 208 times.
✓ Branch 1 taken 1 time.
|
209 | if (n == 0) return out; |
| 636 |
2/2✓ Branch 0 taken 205 times.
✓ Branch 1 taken 3 times.
|
208 | if (device == Device::CPU) { |
| 637 | 3 | std::memcpy(out.data(), data, n); | |
| 638 | 3 | } else { | |
| 639 |
2/4✓ Branch 0 taken 205 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 205 times.
✗ Branch 3 not taken.
|
205 | detail::alloc_for(device).memcpy_d2h(out.data(), data, n); |
| 640 | } | ||
| 641 | 208 | return out; | |
| 642 | 210 | } | |
| 643 | |||
| 644 | 96 | std::vector<uint16_t> Tensor::to_host_vector_bf16() const { | |
| 645 |
2/2✓ Branch 0 taken 95 times.
✓ Branch 1 taken 1 time.
|
96 | if (dtype != Dtype::BF16) { |
| 646 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: to_host_vector_bf16: dtype not BF16"); |
| 647 | } | ||
| 648 | 95 | std::vector<uint16_t> out(static_cast<std::size_t>(rows) * cols); | |
| 649 |
1/2✓ Branch 0 taken 95 times.
✗ Branch 1 not taken.
|
95 | const std::size_t n = bytes(); |
| 650 |
2/2✓ Branch 0 taken 94 times.
✓ Branch 1 taken 1 time.
|
95 | if (n == 0) return out; |
| 651 |
2/2✓ Branch 0 taken 91 times.
✓ Branch 1 taken 3 times.
|
94 | if (device == Device::CPU) { |
| 652 | 3 | std::memcpy(out.data(), data, n); | |
| 653 | 3 | } else { | |
| 654 |
2/4✓ Branch 0 taken 91 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 91 times.
✗ Branch 3 not taken.
|
91 | detail::alloc_for(device).memcpy_d2h(out.data(), data, n); |
| 655 | } | ||
| 656 | 94 | return out; | |
| 657 | 96 | } | |
| 658 | |||
| 659 | 62 | void Tensor::copy_to_host(float* dst) const { | |
| 660 |
2/2✓ Branch 0 taken 61 times.
✓ Branch 1 taken 1 time.
|
62 | if (dtype != Dtype::FP32) { |
| 661 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: copy_to_host: dtype not FP32"); |
| 662 | } | ||
| 663 | 61 | const std::size_t n = bytes(); | |
| 664 |
2/2✓ Branch 0 taken 60 times.
✓ Branch 1 taken 1 time.
|
61 | if (n == 0) return; |
| 665 |
2/2✓ Branch 0 taken 48 times.
✓ Branch 1 taken 12 times.
|
60 | if (device == Device::CPU) { |
| 666 | 12 | std::memcpy(dst, data, n); | |
| 667 | 12 | } else { | |
| 668 | 48 | detail::alloc_for(device).memcpy_d2h(dst, data, n); | |
| 669 | } | ||
| 670 | 62 | } | |
| 671 | |||
| 672 | 53 | void Tensor::copy_to_host_fp16(uint16_t* dst) const { | |
| 673 |
2/2✓ Branch 0 taken 52 times.
✓ Branch 1 taken 1 time.
|
53 | if (dtype != Dtype::FP16) { |
| 674 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: copy_to_host_fp16: dtype not FP16"); |
| 675 | } | ||
| 676 | 52 | const std::size_t n = bytes(); | |
| 677 |
2/2✓ Branch 0 taken 51 times.
✓ Branch 1 taken 1 time.
|
52 | if (n == 0) return; |
| 678 |
1/2✓ Branch 0 taken 51 times.
✗ Branch 1 not taken.
|
51 | if (device == Device::CPU) { |
| 679 | ✗ | std::memcpy(dst, data, n); | |
| 680 | ✗ | } else { | |
| 681 | 51 | detail::alloc_for(device).memcpy_d2h(dst, data, n); | |
| 682 | } | ||
| 683 | 53 | } | |
| 684 | |||
| 685 | 18 | void Tensor::copy_to_host_bf16(uint16_t* dst) const { | |
| 686 |
2/2✓ Branch 0 taken 17 times.
✓ Branch 1 taken 1 time.
|
18 | if (dtype != Dtype::BF16) { |
| 687 |
1/2✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
|
1 | throw_msg("brotensor: copy_to_host_bf16: dtype not BF16"); |
| 688 | } | ||
| 689 | 17 | const std::size_t n = bytes(); | |
| 690 |
2/2✓ Branch 0 taken 16 times.
✓ Branch 1 taken 1 time.
|
17 | if (n == 0) return; |
| 691 |
1/2✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
|
16 | if (device == Device::CPU) { |
| 692 | ✗ | std::memcpy(dst, data, n); | |
| 693 | ✗ | } else { | |
| 694 | 16 | detail::alloc_for(device).memcpy_d2h(dst, data, n); | |
| 695 | } | ||
| 696 | 18 | } | |
| 697 | |||
| 698 | } // namespace brotensor | ||
| 699 |