GCC Code Coverage Report

Directory: ./
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 97.1% 135 / 0 / 139
Functions: 100.0% 6 / 0 / 6
Branches: 68.3% 82 / 0 / 120
src/cpu/conv3d.cpp
Line Branch Exec Source
1 // ─── CPU conv3d ops (Qwen3-VL patch embed) ────────────────────────────────
2 //
3 // FP32 scalar host implementation of conv3d_forward. Mirrors src/cpu/conv2d.cpp
4 // — direct convolution loop, FP32 accumulator. CPU is FP32-only per the
5 // brotensor convention (CLAUDE.md); conv3d_int8w_fp16_forward is GPU-only and
6 // has no CPU registration (its vtable slot stays null and the dispatcher
7 // throws "not implemented on CPU").
8 //
9 // Memory layout (matches the GPU exactly):
10 // X : NCTHW — (((n*C_in + c_in) * T + t) * H + h) * W + w
11 // Y : NCTHW — (((n*C_out + c_out) * T_out + ot) * H_out + oh) * W_out + ow
12 // Wt : OICTHW (grouped) — (((c_out*Cg_in + c_in_local) * kT + kt) * kH + kh) * kW + kw
13 // where Cg_in = C_in/groups is the per-group input-channel count.
14 // bias: (C_out, 1), optional (may be null)
15 //
16 // Groups convention: output channel c_out belongs to group g = c_out/Cg_out;
17 // that group's absolute input channels start at g*Cg_in (Cg_in channels wide).
18 //
19 // Output-size formula (identical to conv2d, applied per-axis):
20 // T_out = (T + 2*pad_t - dil_t*(kT-1) - 1) / stride_t + 1
21 // H_out = (H + 2*pad_h - dil_h*(kH-1) - 1) / stride_h + 1
22 // W_out = (W + 2*pad_w - dil_w*(kW-1) - 1) / stride_w + 1
23 //
24 // ACCUMULATION: conv3d_forward — Y OVERWRITTEN (kernel stores acc directly).
25
26 #include <brotensor/tensor.h>
27 #include <brotensor/detail/cpu/thread_pool.h>
28
29 #include <cstddef>
30 #include <stdexcept>
31 #include <string>
32
33 namespace brotensor::detail::cpu {
34
35 namespace {
36
37 25 inline void check_groups(const char* op, int C_in, int C_out, int groups) {
38
1/2
✓ Branch 0 taken 25 times.
✗ Branch 1 not taken.
 25 if (groups < 1 || C_in % groups != 0 || C_out % groups != 0) {
39 ✗ throw std::runtime_error(
40 ✗ std::string(op) +
41 ": groups must be >=1 and divide both C_in and C_out");
42 }
43 25 }
44
45 75 inline int out_dim(int in, int pad, int dil, int k, int stride) {
46 75 return (in + 2 * pad - dil * (k - 1) - 1) / stride + 1;
47 }
48
49 } // namespace
50
51 25 void conv3d_forward(const ::brotensor::Tensor& X,
52 const ::brotensor::Tensor& Wt,
53 const ::brotensor::Tensor* bias,
54 int N, int C_in, int T, int H, int W,
55 int C_out, int kT, int kH, int kW,
56 int stride_t, int stride_h, int stride_w,
57 int pad_t, int pad_h, int pad_w,
58 int dil_t, int dil_h, int dil_w,
59 int groups,
60 ::brotensor::Tensor& Y) {
61
1/2
✓ Branch 0 taken 25 times.
✗ Branch 1 not taken.
 25 if (Wt.dtype != X.dtype) {
62 ✗ throw std::runtime_error("conv3d_forward: Wt dtype must match X");
63 }
64
3/4
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 25 if (bias && bias->dtype != X.dtype) {
65 ✗ throw std::runtime_error("conv3d_forward: bias dtype must match X");
66 }
67 25 check_groups("conv3d_forward", C_in, C_out, groups);
68 25 const int Cg_in = C_in / groups;
69 25 const int Cg_out = C_out / groups;
70 25 const int T_out = out_dim(T, pad_t, dil_t, kT, stride_t);
71 25 const int H_out = out_dim(H, pad_h, dil_h, kH, stride_h);
72 25 const int W_out = out_dim(W, pad_w, dil_w, kW, stride_w);
73
2/2
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 1 time.
 25 if (T_out <= 0 || H_out <= 0 || W_out <= 0) {
74
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 throw std::runtime_error("conv3d_forward: non-positive output shape");
75 }
76 24 const int out_cols = C_out * T_out * H_out * W_out;
77
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 24 if (Y.rows != N || Y.cols != out_cols || Y.dtype != X.dtype) {
78 24 Y.resize(N, out_cols, X.dtype);
79 24 }
80
2/4
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 24 times.
 24 if (N == 0 || out_cols == 0) return;
81
82 24 const float* Xp = X.host_f32();
83 24 const float* Wp = Wt.host_f32();
84
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 6 times.
 24 const float* Bp = bias ? bias->host_f32() : nullptr;
85 24 float* Yp = Y.host_f32_mut();
86
87 // Interior region: the (ot, oh, ow) box for which every kernel tap is
88 // guaranteed in-bounds on every axis (mirrors conv2d.cpp's split,
89 // extended to 3 axes), computed once — independent of n/oc. Only the
90 // thin border shell outside that box needs the per-tap bounds check;
91 // the interior runs a branch-free kt/kh/kw loop.
92 24 int ot_lo = (pad_t + stride_t - 1) / stride_t;
93 24 int ot_hi = T - 1 + pad_t - (kT - 1) * dil_t;
94
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 ot_hi = (ot_hi >= 0) ? (ot_hi / stride_t) : -1;
95
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (ot_lo < 0) ot_lo = 0;
96
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (ot_hi >= T_out) ot_hi = T_out - 1;
97
98 24 int oh_lo = (pad_h + stride_h - 1) / stride_h;
99 24 int oh_hi = H - 1 + pad_h - (kH - 1) * dil_h;
100
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 oh_hi = (oh_hi >= 0) ? (oh_hi / stride_h) : -1;
101
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (oh_lo < 0) oh_lo = 0;
102
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (oh_hi >= H_out) oh_hi = H_out - 1;
103
104 24 int ow_lo = (pad_w + stride_w - 1) / stride_w;
105 24 int ow_hi = W - 1 + pad_w - (kW - 1) * dil_w;
106
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 ow_hi = (ow_hi >= 0) ? (ow_hi / stride_w) : -1;
107
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (ow_lo < 0) ow_lo = 0;
108
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (ow_hi >= W_out) ow_hi = W_out - 1;
109
110 24 const bool has_interior =
111
2/4
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 24 times.
 24 (ot_lo <= ot_hi) && (oh_lo <= oh_hi) && (ow_lo <= ow_hi);
112
113 // Each n exclusively owns Y's batch slice n (X/Wt/bias are read-only), so
114 // this parallelizes across n with no cross-thread writes.
115
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 59 parallel_for(static_cast<std::size_t>(N), [&](std::size_t ni) {
116 35 const int n = static_cast<int>(ni);
117
2/2
✓ Branch 0 taken 177 times.
✓ Branch 1 taken 35 times.
 212 for (int oc = 0; oc < C_out; ++oc) {
118 177 const int g = oc / Cg_out;
119 177 const int ic_base = g * Cg_in;
120 177 const int w_oc_base = oc * Cg_in * kT * kH * kW;
121
2/2
✓ Branch 0 taken 146 times.
✓ Branch 1 taken 31 times.
 177 const float bias_v = Bp ? Bp[oc] : 0.0f;
122
123 // Border voxel: same bounds-checked accumulation as before.
124 6183 auto compute_bordered = [&](int ot, int oh, int ow) -> float {
125 6006 const int in_t_origin = ot * stride_t - pad_t;
126 6006 const int in_h_origin = oh * stride_h - pad_h;
127 6006 const int in_w_origin = ow * stride_w - pad_w;
128 6006 float acc = 0.0f;
129
2/2
✓ Branch 0 taken 13976 times.
✓ Branch 1 taken 6006 times.
 19982 for (int ic_local = 0; ic_local < Cg_in; ++ic_local) {
130 13976 const int ic = ic_base + ic_local;
131 13976 const int w_ic_base = w_oc_base + ic_local * kT * kH * kW;
132 13976 const int x_ic_base = (n * C_in + ic) * T * H * W;
133
2/2
✓ Branch 0 taken 41168 times.
✓ Branch 1 taken 13976 times.
 55144 for (int kt = 0; kt < kT; ++kt) {
134 41168 const int in_t = in_t_origin + kt * dil_t;
135
4/4
✓ Branch 0 taken 36602 times.
✓ Branch 1 taken 4566 times.
✓ Branch 2 taken 4947 times.
✓ Branch 3 taken 31655 times.
 41168 if (in_t < 0 || in_t >= T) continue;
136
2/2
✓ Branch 0 taken 92708 times.
✓ Branch 1 taken 31655 times.
 124363 for (int kh = 0; kh < kH; ++kh) {
137 92708 const int in_h = in_h_origin + kh * dil_h;
138
4/4
✓ Branch 0 taken 82693 times.
✓ Branch 1 taken 10015 times.
✓ Branch 2 taken 74291 times.
✓ Branch 3 taken 8402 times.
 92708 if (in_h < 0 || in_h >= H) continue;
139
2/2
✓ Branch 0 taken 219913 times.
✓ Branch 1 taken 74291 times.
 294204 for (int kw = 0; kw < kW; ++kw) {
140 219913 const int in_w = in_w_origin + kw * dil_w;
141
4/4
✓ Branch 0 taken 195634 times.
✓ Branch 1 taken 24279 times.
✓ Branch 2 taken 20515 times.
✓ Branch 3 taken 175119 times.
 219913 if (in_w < 0 || in_w >= W) continue;
142 175119 acc +=
143 525357 Xp[x_ic_base +
144 350238 (in_t * H + in_h) * W + in_w] *
145 525357 Wp[w_ic_base +
146 350238 (kt * kH + kh) * kW + kw];
147 175119 }
148 74291 }
149 31655 }
150 13976 }
151 6006 return acc;
152 };
153
154 // Interior voxel: every tap is guaranteed in-bounds — no checks.
155 4848 auto compute_interior = [&](int ot, int oh, int ow) -> float {
156 4671 const int in_t_origin = ot * stride_t - pad_t;
157 4671 const int in_h_origin = oh * stride_h - pad_h;
158 4671 const int in_w_origin = ow * stride_w - pad_w;
159 4671 float acc = 0.0f;
160
2/2
✓ Branch 0 taken 16810 times.
✓ Branch 1 taken 4671 times.
 21481 for (int ic_local = 0; ic_local < Cg_in; ++ic_local) {
161 16810 const int ic = ic_base + ic_local;
162 16810 const int w_ic_base = w_oc_base + ic_local * kT * kH * kW;
163 16810 const int x_ic_base = (n * C_in + ic) * T * H * W;
164
2/2
✓ Branch 0 taken 21943 times.
✓ Branch 1 taken 16810 times.
 38753 for (int kt = 0; kt < kT; ++kt) {
165 21943 const int in_t = in_t_origin + kt * dil_t;
166 21943 const int x_t_base = x_ic_base + in_t * H * W;
167 21943 const int w_t_base = w_ic_base + kt * kH * kW;
168
2/2
✓ Branch 0 taken 38284 times.
✓ Branch 1 taken 21943 times.
 60227 for (int kh = 0; kh < kH; ++kh) {
169 38284 const int in_h = in_h_origin + kh * dil_h;
170 38284 const int x_row_base = x_t_base + in_h * W;
171 38284 const int w_row_base = w_t_base + kh * kW;
172
2/2
✓ Branch 0 taken 106516 times.
✓ Branch 1 taken 38284 times.
 144800 for (int kw = 0; kw < kW; ++kw) {
173 106516 const int in_w = in_w_origin + kw * dil_w;
174 106516 acc += Xp[x_row_base + in_w] * Wp[w_row_base + kw];
175 106516 }
176 38284 }
177 21943 }
178 16810 }
179 4671 return acc;
180 };
181
182
2/2
✓ Branch 0 taken 511 times.
✓ Branch 1 taken 177 times.
 688 for (int ot = 0; ot < T_out; ++ot) {
183
3/4
✓ Branch 0 taken 511 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 94 times.
✓ Branch 3 taken 417 times.
 511 const bool ot_interior = has_interior && ot >= ot_lo && ot <= ot_hi;
184 511 const int t_base = ((n * C_out + oc) * T_out + ot) * H_out;
185
2/2
✓ Branch 0 taken 323 times.
✓ Branch 1 taken 188 times.
 511 if (!ot_interior) {
186
2/2
✓ Branch 0 taken 871 times.
✓ Branch 1 taken 188 times.
 1059 for (int oh = 0; oh < H_out; ++oh) {
187 871 const int y_row_base = (t_base + oh) * W_out;
188
2/2
✓ Branch 0 taken 4192 times.
✓ Branch 1 taken 871 times.
 5063 for (int ow = 0; ow < W_out; ++ow) {
189 4192 Yp[y_row_base + ow] = compute_bordered(ot, oh, ow) + bias_v;
190 4192 }
191 871 }
192 188 continue;
193 }
194
2/2
✓ Branch 0 taken 1410 times.
✓ Branch 1 taken 323 times.
 1733 for (int oh = 0; oh < H_out; ++oh) {
195 1410 const int y_row_base = (t_base + oh) * W_out;
196
2/2
✓ Branch 0 taken 128 times.
✓ Branch 1 taken 1282 times.
 1410 const bool oh_interior = oh >= oh_lo && oh <= oh_hi;
197
2/2
✓ Branch 0 taken 1162 times.
✓ Branch 1 taken 248 times.
 1410 if (!oh_interior) {
198
2/2
✓ Branch 0 taken 1178 times.
✓ Branch 1 taken 248 times.
 1426 for (int ow = 0; ow < W_out; ++ow) {
199 1178 Yp[y_row_base + ow] = compute_bordered(ot, oh, ow) + bias_v;
200 1178 }
201 248 continue;
202 }
203
2/2
✓ Branch 0 taken 332 times.
✓ Branch 1 taken 1162 times.
 1494 for (int ow = 0; ow < ow_lo; ++ow) {
204 332 Yp[y_row_base + ow] = compute_bordered(ot, oh, ow) + bias_v;
205 332 }
206
2/2
✓ Branch 0 taken 4726 times.
✓ Branch 1 taken 1162 times.
 5888 for (int ow = ow_lo; ow <= ow_hi; ++ow) {
207 4726 Yp[y_row_base + ow] = compute_interior(ot, oh, ow) + bias_v;
208 4726 }
209
2/2
✓ Branch 0 taken 324 times.
✓ Branch 1 taken 1162 times.
 1486 for (int ow = ow_hi + 1; ow < W_out; ++ow) {
210 324 Yp[y_row_base + ow] = compute_bordered(ot, oh, ow) + bias_v;
211 324 }
212 1162 }
213 323 }
214 177 }
215 35 });
216 24 }
217
218 } // namespace brotensor::detail::cpu
219