GCC Code Coverage Report

Directory: ./
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 97.1% 135 / 0 / 139
Functions: 100.0% 6 / 0 / 6
Branches: 68.3% 82 / 0 / 120
src/cpu/conv3d.cpp
Line Branch Exec Source
1 // ─── CPU conv3d ops (Qwen3-VL patch embed) ────────────────────────────────
2 //
3 // FP32 scalar host implementation of conv3d_forward. Mirrors src/cpu/conv2d.cpp
4 // — direct convolution loop, FP32 accumulator. CPU is FP32-only per the
5 // brotensor convention (CLAUDE.md); conv3d_int8w_fp16_forward is GPU-only and
6 // has no CPU registration (its vtable slot stays null and the dispatcher
7 // throws "not implemented on CPU").
8 //
9 // Memory layout (matches the GPU exactly):
10 // X : NCTHW — (((n*C_in + c_in) * T + t) * H + h) * W + w
11 // Y : NCTHW — (((n*C_out + c_out) * T_out + ot) * H_out + oh) * W_out + ow
12 // Wt : OICTHW (grouped) — (((c_out*Cg_in + c_in_local) * kT + kt) * kH + kh) * kW + kw
13 // where Cg_in = C_in/groups is the per-group input-channel count.
14 // bias: (C_out, 1), optional (may be null)
15 //
16 // Groups convention: output channel c_out belongs to group g = c_out/Cg_out;
17 // that group's absolute input channels start at g*Cg_in (Cg_in channels wide).
18 //
19 // Output-size formula (identical to conv2d, applied per-axis):
20 // T_out = (T + 2*pad_t - dil_t*(kT-1) - 1) / stride_t + 1
21 // H_out = (H + 2*pad_h - dil_h*(kH-1) - 1) / stride_h + 1
22 // W_out = (W + 2*pad_w - dil_w*(kW-1) - 1) / stride_w + 1
23 //
24 // ACCUMULATION: conv3d_forward — Y OVERWRITTEN (kernel stores acc directly).
25
26 #include <brotensor/tensor.h>
27 #include <brotensor/detail/cpu/thread_pool.h>
28
29 #include <cstddef>
30 #include <stdexcept>
31 #include <string>
32
33 namespace brotensor::detail::cpu {
34
35 namespace {
36
37 25 inline void check_groups(const char* op, int C_in, int C_out, int groups) {
38
1/2
✓ Branch 0 taken 25 times.
✗ Branch 1 not taken.
 25 if (groups < 1 || C_in % groups != 0 || C_out % groups != 0) {
39 ✗ throw std::runtime_error(
40 ✗ std::string(op) +
41 ": groups must be >=1 and divide both C_in and C_out");
42 }
43 25 }
44
45 75 inline int out_dim(int in, int pad, int dil, int k, int stride) {
46 75 return (in + 2 * pad - dil * (k - 1) - 1) / stride + 1;
47 }
48
49 } // namespace
50
51 25 void conv3d_forward(const ::brotensor::Tensor& X,
52 const ::brotensor::Tensor& Wt,
53 const ::brotensor::Tensor* bias,
54 int N, int C_in, int T, int H, int W,
55 int C_out, int kT, int kH, int kW,
56 int stride_t, int stride_h, int stride_w,
57 int pad_t, int pad_h, int pad_w,
58 int dil_t, int dil_h, int dil_w,
59 int groups,
60 ::brotensor::Tensor& Y) {
61
1/2
✓ Branch 0 taken 25 times.
✗ Branch 1 not taken.
 25 if (Wt.dtype != X.dtype) {
62 ✗ throw std::runtime_error("conv3d_forward: Wt dtype must match X");
63 }
64
3/4
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 7 times.
✓ Branch 2 taken 18 times.
✗ Branch 3 not taken.
 25 if (bias && bias->dtype != X.dtype) {
65 ✗ throw std::runtime_error("conv3d_forward: bias dtype must match X");
66 }
67 25 check_groups("conv3d_forward", C_in, C_out, groups);
68 25 const int Cg_in = C_in / groups;
69 25 const int Cg_out = C_out / groups;
70 25 const int T_out = out_dim(T, pad_t, dil_t, kT, stride_t);
71 25 const int H_out = out_dim(H, pad_h, dil_h, kH, stride_h);
72 25 const int W_out = out_dim(W, pad_w, dil_w, kW, stride_w);
73
2/2
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 1 time.
 25 if (T_out <= 0 || H_out <= 0 || W_out <= 0) {
74
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 throw std::runtime_error("conv3d_forward: non-positive output shape");
75 }
76 24 const int out_cols = C_out * T_out * H_out * W_out;
77
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 24 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 24 if (Y.rows != N || Y.cols != out_cols || Y.dtype != X.dtype) {
78 24 Y.resize(N, out_cols, X.dtype);
79 24 }
80
2/4
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 24 times.
 24 if (N == 0 || out_cols == 0) return;
81
82 24 const float* Xp = X.host_f32();
83 24 const float* Wp = Wt.host_f32();
84
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 6 times.
 24 const float* Bp = bias ? bias->host_f32() : nullptr;
85 24 float* Yp = Y.host_f32_mut();
86
87 // Interior region: the (ot, oh, ow) box for which every kernel tap is
88 // guaranteed in-bounds on every axis (mirrors conv2d.cpp's split,
89 // extended to 3 axes), computed once — independent of n/oc. Only the
90 // thin border shell outside that box needs the per-tap bounds check;
91 // the interior runs a branch-free kt/kh/kw loop.
92 24 int ot_lo = (pad_t + stride_t - 1) / stride_t;
93 24 int ot_hi = T - 1 + pad_t - (kT - 1) * dil_t;
94
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 ot_hi = (ot_hi >= 0) ? (ot_hi / stride_t) : -1;
95
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (ot_lo < 0) ot_lo = 0;
96
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (ot_hi >= T_out) ot_hi = T_out - 1;
97
98 24 int oh_lo = (pad_h + stride_h - 1) / stride_h;
99 24 int oh_hi = H - 1 + pad_h - (kH - 1) * dil_h;
100
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 oh_hi = (oh_hi >= 0) ? (oh_hi / stride_h) : -1;
101
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (oh_lo < 0) oh_lo = 0;
102
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (oh_hi >= H_out) oh_hi = H_out - 1;
103
104 24 int ow_lo = (pad_w + stride_w - 1) / stride_w;
105 24 int ow_hi = W - 1 + pad_w - (kW - 1) * dil_w;
106
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 ow_hi = (ow_hi >= 0) ? (ow_hi / stride_w) : -1;
107
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (ow_lo < 0) ow_lo = 0;
108
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 24 if (ow_hi >= W_out) ow_hi = W_out - 1;
109
110 24 const bool has_interior =
111
2/4
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 24 times.
 24 (ot_lo <= ot_hi) && (oh_lo <= oh_hi) && (ow_lo <= ow_hi);
112
113 // Each n exclusively owns Y's batch slice n (X/Wt/bias are read-only), so
114 // this parallelizes across n with no cross-thread writes.
115
1/2
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
 59 parallel_for(static_cast<std::size_t>(N), [&](std::size_t ni) {
116 35 const int n = static_cast<int>(ni);
117
2/2
✓ Branch 0 taken 177 times.
✓ Branch 1 taken 35 times.
 212 for (int oc = 0; oc < C_out; ++oc) {
118 177 const int g = oc / Cg_out;
119 177 const int ic_base = g * Cg_in;
120 177 const int w_oc_base = oc * Cg_in * kT * kH * kW;
121
2/2
✓ Branch 0 taken 146 times.
✓ Branch 1 taken 31 times.
 177 const float bias_v = Bp ? Bp[oc] : 0.0f;
122
123 // Border voxel: same bounds-checked accumulation as before.
124 6107 auto compute_bordered = [&](int ot, int oh, int ow) -> float {
125 5930 const int in_t_origin = ot * stride_t - pad_t;
126 5930 const int in_h_origin = oh * stride_h - pad_h;
127 5930 const int in_w_origin = ow * stride_w - pad_w;
128 5930 float acc = 0.0f;
129
2/2
✓ Branch 0 taken 13415 times.
✓ Branch 1 taken 5930 times.
 19345 for (int ic_local = 0; ic_local < Cg_in; ++ic_local) {
130 13415 const int ic = ic_base + ic_local;
131 13415 const int w_ic_base = w_oc_base + ic_local * kT * kH * kW;
132 13415 const int x_ic_base = (n * C_in + ic) * T * H * W;
133
2/2
✓ Branch 0 taken 37546 times.
✓ Branch 1 taken 13415 times.
 50961 for (int kt = 0; kt < kT; ++kt) {
134 37546 const int in_t = in_t_origin + kt * dil_t;
135
4/4
✓ Branch 0 taken 32885 times.
✓ Branch 1 taken 4661 times.
✓ Branch 2 taken 4592 times.
✓ Branch 3 taken 28293 times.
 37546 if (in_t < 0 || in_t >= T) continue;
136
2/2
✓ Branch 0 taken 83460 times.
✓ Branch 1 taken 28293 times.
 111753 for (int kh = 0; kh < kH; ++kh) {
137 83460 const int in_h = in_h_origin + kh * dil_h;
138
4/4
✓ Branch 0 taken 72349 times.
✓ Branch 1 taken 11111 times.
✓ Branch 2 taken 65779 times.
✓ Branch 3 taken 6570 times.
 83460 if (in_h < 0 || in_h >= H) continue;
139
2/2
✓ Branch 0 taken 189765 times.
✓ Branch 1 taken 65779 times.
 255544 for (int kw = 0; kw < kW; ++kw) {
140 189765 const int in_w = in_w_origin + kw * dil_w;
141
4/4
✓ Branch 0 taken 165339 times.
✓ Branch 1 taken 24426 times.
✓ Branch 2 taken 17197 times.
✓ Branch 3 taken 148142 times.
 189765 if (in_w < 0 || in_w >= W) continue;
142 148142 acc +=
143 444426 Xp[x_ic_base +
144 296284 (in_t * H + in_h) * W + in_w] *
145 444426 Wp[w_ic_base +
146 296284 (kt * kH + kh) * kW + kw];
147 148142 }
148 65779 }
149 28293 }
150 13415 }
151 5930 return acc;
152 };
153
154 // Interior voxel: every tap is guaranteed in-bounds — no checks.
155 4513 auto compute_interior = [&](int ot, int oh, int ow) -> float {
156 4336 const int in_t_origin = ot * stride_t - pad_t;
157 4336 const int in_h_origin = oh * stride_h - pad_h;
158 4336 const int in_w_origin = ow * stride_w - pad_w;
159 4336 float acc = 0.0f;
160
2/2
✓ Branch 0 taken 14181 times.
✓ Branch 1 taken 4336 times.
 18517 for (int ic_local = 0; ic_local < Cg_in; ++ic_local) {
161 14181 const int ic = ic_base + ic_local;
162 14181 const int w_ic_base = w_oc_base + ic_local * kT * kH * kW;
163 14181 const int x_ic_base = (n * C_in + ic) * T * H * W;
164
2/2
✓ Branch 0 taken 19343 times.
✓ Branch 1 taken 14181 times.
 33524 for (int kt = 0; kt < kT; ++kt) {
165 19343 const int in_t = in_t_origin + kt * dil_t;
166 19343 const int x_t_base = x_ic_base + in_t * H * W;
167 19343 const int w_t_base = w_ic_base + kt * kH * kW;
168
2/2
✓ Branch 0 taken 35113 times.
✓ Branch 1 taken 19343 times.
 54456 for (int kh = 0; kh < kH; ++kh) {
169 35113 const int in_h = in_h_origin + kh * dil_h;
170 35113 const int x_row_base = x_t_base + in_h * W;
171 35113 const int w_row_base = w_t_base + kh * kW;
172
2/2
✓ Branch 0 taken 101766 times.
✓ Branch 1 taken 35113 times.
 136879 for (int kw = 0; kw < kW; ++kw) {
173 101766 const int in_w = in_w_origin + kw * dil_w;
174 101766 acc += Xp[x_row_base + in_w] * Wp[w_row_base + kw];
175 101766 }
176 35113 }
177 19343 }
178 14181 }
179 4336 return acc;
180 };
181
182
2/2
✓ Branch 0 taken 509 times.
✓ Branch 1 taken 177 times.
 686 for (int ot = 0; ot < T_out; ++ot) {
183
3/4
✓ Branch 0 taken 509 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 94 times.
✓ Branch 3 taken 415 times.
 509 const bool ot_interior = has_interior && ot >= ot_lo && ot <= ot_hi;
184 509 const int t_base = ((n * C_out + oc) * T_out + ot) * H_out;
185
2/2
✓ Branch 0 taken 321 times.
✓ Branch 1 taken 188 times.
 509 if (!ot_interior) {
186
2/2
✓ Branch 0 taken 871 times.
✓ Branch 1 taken 188 times.
 1059 for (int oh = 0; oh < H_out; ++oh) {
187 871 const int y_row_base = (t_base + oh) * W_out;
188
2/2
✓ Branch 0 taken 4173 times.
✓ Branch 1 taken 871 times.
 5044 for (int ow = 0; ow < W_out; ++ow) {
189 4173 Yp[y_row_base + ow] = compute_bordered(ot, oh, ow) + bias_v;
190 4173 }
191 871 }
192 188 continue;
193 }
194
2/2
✓ Branch 0 taken 1407 times.
✓ Branch 1 taken 321 times.
 1728 for (int oh = 0; oh < H_out; ++oh) {
195 1407 const int y_row_base = (t_base + oh) * W_out;
196
2/2
✓ Branch 0 taken 129 times.
✓ Branch 1 taken 1278 times.
 1407 const bool oh_interior = oh >= oh_lo && oh <= oh_hi;
197
2/2
✓ Branch 0 taken 1159 times.
✓ Branch 1 taken 248 times.
 1407 if (!oh_interior) {
198
2/2
✓ Branch 0 taken 1178 times.
✓ Branch 1 taken 248 times.
 1426 for (int ow = 0; ow < W_out; ++ow) {
199 1178 Yp[y_row_base + ow] = compute_bordered(ot, oh, ow) + bias_v;
200 1178 }
201 248 continue;
202 }
203
2/2
✓ Branch 0 taken 331 times.
✓ Branch 1 taken 1159 times.
 1490 for (int ow = 0; ow < ow_lo; ++ow) {
204 331 Yp[y_row_base + ow] = compute_bordered(ot, oh, ow) + bias_v;
205 331 }
206
2/2
✓ Branch 0 taken 4529 times.
✓ Branch 1 taken 1159 times.
 5688 for (int ow = ow_lo; ow <= ow_hi; ++ow) {
207 4529 Yp[y_row_base + ow] = compute_interior(ot, oh, ow) + bias_v;
208 4529 }
209
2/2
✓ Branch 0 taken 324 times.
✓ Branch 1 taken 1159 times.
 1483 for (int ow = ow_hi + 1; ow < W_out; ++ow) {
210 324 Yp[y_row_base + ow] = compute_bordered(ot, oh, ow) + bias_v;
211 324 }
212 1159 }
213 321 }
214 177 }
215 35 });
216 24 }
217
218 } // namespace brotensor::detail::cpu
219