GCC Code Coverage Report

Directory: ./
Coverage: low: ≥ 0% medium: ≥ 75.0% high: ≥ 90.0%
Coverage Exec / Excl / Total
Lines: 96.0% 95 / 0 / 99
Functions: 100.0% 4 / 0 / 4
Branches: 75.6% 65 / 0 / 86
src/cpu/deform_conv2d.cpp
Line Branch Exec Source
1 // ─── CPU modulated deformable conv2d (torchvision deform_conv2d v2) ─────────
2 //
3 // FP32 scalar host implementation, forward/inference only. Mirrors
4 // torchvision's deformable_im2col + GEMM, fused into a single direct loop:
5 // for each output pixel and conv group, each kH×kW tap is bilinearly sampled
6 // from X at a per-tap, per-pixel-shifted location (offset field) with ZERO
7 // padding outside the input, optionally reweighted by the mask modulator, then
8 // reduced against the OIHW weight.
9 //
10 // Layouts match conv2d_forward (X/Y/Wt NCHW/OIHW). offset/mask are dense NCHW
11 // fields produced by the model's offset_conv / modulator_conv:
12 // offset: (N, deform_groups*2*kH*kW, H_out, W_out), channel
13 // grp*(2*kH*kW) + 2*(kh*kW+kw) [+1 for the col axis].
14 // mask: (N, deform_groups*kH*kW, H_out, W_out), channel
15 // grp*(kH*kW) + (kh*kW+kw); null == all modulators 1.
16 //
17 // bilinear_interpolate matches torchvision exactly (the (h<=-1 || h>=H ...)
18 // early-out plus per-corner in-bounds guards), so taps that fall on or past the
19 // border contribute zero — NOT clamped.
20
21 #include <brotensor/tensor.h>
22 #include <brotensor/detail/cpu/thread_pool.h>
23
24 #include <cmath>
25 #include <cstddef>
26 #include <stdexcept>
27 #include <string>
28 #include <vector>
29
30 namespace brotensor::detail::cpu {
31
32 namespace {
33
34 12 inline int out_dim(int in, int pad, int dil, int k, int stride) {
35 12 return (in + 2 * pad - dil * (k - 1) - 1) / stride + 1;
36 }
37
38 // torchvision bilinear_interpolate: zero outside [0,H)×[0,W), per-corner guard.
39 137564 inline float bilinear(const float* in, int H, int W, float h, float w) {
40
6/6
✓ Branch 0 taken 129996 times.
✓ Branch 1 taken 7568 times.
✓ Branch 2 taken 122306 times.
✓ Branch 3 taken 7690 times.
✓ Branch 4 taken 6036 times.
✓ Branch 5 taken 110592 times.
 254192 if (h <= -1.0f || static_cast<float>(H) <= h ||
41
2/2
✓ Branch 0 taken 116628 times.
✓ Branch 1 taken 5678 times.
 122306 w <= -1.0f || static_cast<float>(W) <= w) {
42 26972 return 0.0f;
43 }
44 110592 int h_low = static_cast<int>(std::floor(h));
45 110592 int w_low = static_cast<int>(std::floor(w));
46 110592 int h_high = h_low + 1;
47 110592 int w_high = w_low + 1;
48 110592 float lh = h - h_low, lw = w - w_low;
49 110592 float hh = 1.0f - lh, hw = 1.0f - lw;
50
4/4
✓ Branch 0 taken 104360 times.
✓ Branch 1 taken 6232 times.
✓ Branch 2 taken 4494 times.
✓ Branch 3 taken 99866 times.
 110592 float v1 = (h_low >= 0 && w_low >= 0) ? in[h_low * W + w_low] : 0.0f;
51
4/4
✓ Branch 0 taken 104360 times.
✓ Branch 1 taken 6232 times.
✓ Branch 2 taken 4224 times.
✓ Branch 3 taken 100136 times.
 110592 float v2 = (h_low >= 0 && w_high <= W - 1) ? in[h_low * W + w_high] : 0.0f;
52
4/4
✓ Branch 0 taken 104794 times.
✓ Branch 1 taken 5798 times.
✓ Branch 2 taken 4412 times.
✓ Branch 3 taken 100382 times.
 110592 float v3 = (h_high <= H - 1 && w_low >= 0) ? in[h_high * W + w_low] : 0.0f;
53
4/4
✓ Branch 0 taken 104794 times.
✓ Branch 1 taken 5798 times.
✓ Branch 2 taken 4276 times.
✓ Branch 3 taken 100518 times.
 110592 float v4 = (h_high <= H - 1 && w_high <= W - 1) ? in[h_high * W + w_high] : 0.0f;
54 110592 float w1 = hh * hw, w2 = hh * lw, w3 = lh * hw, w4 = lh * lw;
55 110592 return w1 * v1 + w2 * v2 + w3 * v3 + w4 * v4;
56 137564 }
57
58 } // namespace
59
60 6 void deform_conv2d_forward(const ::brotensor::Tensor& X,
61 const ::brotensor::Tensor& offset,
62 const ::brotensor::Tensor* mask,
63 const ::brotensor::Tensor& Wt,
64 const ::brotensor::Tensor* bias,
65 int N, int C_in, int H, int W,
66 int C_out, int kH, int kW,
67 int stride_h, int stride_w,
68 int pad_h, int pad_w,
69 int dil_h, int dil_w,
70 int groups, int deform_groups,
71 ::brotensor::Tensor& Y) {
72
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 11 if (Wt.dtype != X.dtype || offset.dtype != X.dtype ||
73
4/4
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 5 times.
✓ Branch 3 taken 1 time.
 6 (mask && mask->dtype != X.dtype) || (bias && bias->dtype != X.dtype)) {
74 ✗ throw std::runtime_error(
75 "deform_conv2d_forward: X, offset, mask, Wt, bias dtype must match");
76 }
77
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 if (groups < 1 || C_in % groups != 0 || C_out % groups != 0) {
78 ✗ throw std::runtime_error(
79 "deform_conv2d_forward: groups must divide C_in and C_out");
80 }
81
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 if (deform_groups < 1 || C_in % deform_groups != 0) {
82 ✗ throw std::runtime_error(
83 "deform_conv2d_forward: deform_groups must divide C_in");
84 }
85 6 const int Cg_in = C_in / groups;
86 6 const int Cg_out = C_out / groups;
87 6 const int c_per_off_grp = C_in / deform_groups;
88 6 const int H_out = out_dim(H, pad_h, dil_h, kH, stride_h);
89 6 const int W_out = out_dim(W, pad_w, dil_w, kW, stride_w);
90
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 6 if (H_out <= 0 || W_out <= 0) {
91 ✗ throw std::runtime_error("deform_conv2d_forward: non-positive output shape");
92 }
93 6 const int out_cols = C_out * H_out * W_out;
94
1/6
✗ Branch 0 not taken.
✓ Branch 1 taken 6 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 6 if (Y.rows != N || Y.cols != out_cols || Y.dtype != X.dtype) {
95 6 Y.resize(N, out_cols, X.dtype);
96 6 }
97
2/4
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 6 times.
 6 if (N == 0 || out_cols == 0) return;
98
99 6 const float* Xp = X.host_f32();
100 6 const float* Op = offset.host_f32();
101
2/2
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 1 time.
 6 const float* Mp = mask ? mask->host_f32() : nullptr;
102 6 const float* Wp = Wt.host_f32();
103
2/2
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 1 time.
 6 const float* Bp = bias ? bias->host_f32() : nullptr;
104 6 float* Yp = Y.host_f32_mut();
105
106 6 const int ksz = kH * kW;
107 6 const int off_row_stride = deform_groups * 2 * ksz * H_out * W_out; // per-n
108 6 const int mask_row_stride = deform_groups * ksz * H_out * W_out; // per-n
109
110 // The modulated bilinear sample depends only on (n, oh, ow, ic, tap) — it
111 // never depends on oc — but the original oc -> ic_local -> kh,kw loop
112 // order resampled it once per output channel (O(C_out) redundant work).
113 // Reorder to n -> g -> oh -> ow: sample each group's (Cg_in * ksz) taps
114 // once into a scratch column (im2col-style), then dot that column against
115 // every oc's weight row within the group. The inner accumulation order
116 // (ic_local-major, tap-minor) is preserved exactly, so results match the
117 // original bit-for-bit.
118 // Each n exclusively owns Y's batch slice n (X/offset/mask/Wt/bias are
119 // read-only), so this parallelizes across n with no cross-thread writes.
120 // The im2col scratch `col` MUST be declared inside this lambda (fresh
121 // per invocation, even across threads) rather than hoisted above the
122 // loop — a single shared vector reused across n/g would race under
123 // parallel_for (every thread scribbling into the same buffer).
124
1/2
✓ Branch 0 taken 6 times.
✗ Branch 1 not taken.
 13 parallel_for(static_cast<std::size_t>(N), [&](std::size_t ni) {
125 7 const int n = static_cast<int>(ni);
126 7 std::vector<float> col(static_cast<size_t>(Cg_in) * ksz);
127 7 const float* off_n = Op + static_cast<size_t>(n) * off_row_stride;
128
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 1 time.
 7 const float* mask_n = Mp ? Mp + static_cast<size_t>(n) * mask_row_stride : nullptr;
129
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 7 times.
 15 for (int g = 0; g < groups; ++g) {
130 8 const int ic_base = g * Cg_in;
131
2/2
✓ Branch 0 taken 8 times.
✓ Branch 1 taken 71 times.
 79 for (int oh = 0; oh < H_out; ++oh) {
132 71 const int in_h_origin = oh * stride_h - pad_h;
133
2/2
✓ Branch 0 taken 709 times.
✓ Branch 1 taken 71 times.
 780 for (int ow = 0; ow < W_out; ++ow) {
134 709 const int in_w_origin = ow * stride_w - pad_w;
135 709 const int sp = oh * W_out + ow;
136
137
2/2
✓ Branch 0 taken 5980 times.
✓ Branch 1 taken 709 times.
 6689 for (int ic_local = 0; ic_local < Cg_in; ++ic_local) {
138 5980 const int ic = ic_base + ic_local;
139 5980 const int off_grp = ic / c_per_off_grp;
140 5980 const float* in_ch = Xp + (static_cast<size_t>(n) * C_in + ic) * H * W;
141 5980 const float* off_grp_base =
142 5980 off_n + static_cast<size_t>(off_grp) * 2 * ksz * H_out * W_out;
143 5980 const float* mask_grp_base =
144
2/2
✓ Branch 0 taken 5020 times.
✓ Branch 1 taken 960 times.
 5980 mask_n ? mask_n + static_cast<size_t>(off_grp) * ksz * H_out * W_out
145 : nullptr;
146 5980 float* col_ic = col.data() + ic_local * ksz;
147
2/2
✓ Branch 0 taken 25148 times.
✓ Branch 1 taken 5980 times.
 31128 for (int kh = 0; kh < kH; ++kh) {
148
2/2
✓ Branch 0 taken 25148 times.
✓ Branch 1 taken 137564 times.
 162712 for (int kw = 0; kw < kW; ++kw) {
149 137564 const int tap = kh * kW + kw;
150 137564 const float off_y = off_grp_base[((2 * tap) * H_out + oh) * W_out + ow];
151 137564 const float off_x = off_grp_base[((2 * tap + 1) * H_out + oh) * W_out + ow];
152
2/2
✓ Branch 0 taken 128924 times.
✓ Branch 1 taken 8640 times.
 137564 const float m = mask_grp_base
153 128924 ? mask_grp_base[(tap * H_out + oh) * W_out + ow] : 1.0f;
154 137564 const float y = in_h_origin + kh * dil_h + off_y;
155 137564 const float x = in_w_origin + kw * dil_w + off_x;
156
1/2
✓ Branch 0 taken 137564 times.
✗ Branch 1 not taken.
 137564 const float val = bilinear(in_ch, H, W, y, x);
157 137564 col_ic[tap] = m * val;
158 137564 }
159 25148 }
160 5980 }
161
162 709 const int col_len = Cg_in * ksz;
163
2/2
✓ Branch 0 taken 3960 times.
✓ Branch 1 taken 709 times.
 4669 for (int oc_local = 0; oc_local < Cg_out; ++oc_local) {
164 3960 const int oc = g * Cg_out + oc_local;
165 3960 const int w_oc_base = oc * Cg_in * ksz;
166
2/2
✓ Branch 0 taken 3312 times.
✓ Branch 1 taken 648 times.
 3960 const float bias_v = Bp ? Bp[oc] : 0.0f;
167 3960 float acc = bias_v;
168
2/2
✓ Branch 0 taken 1038448 times.
✓ Branch 1 taken 3960 times.
 1042408 for (int k = 0; k < col_len; ++k) {
169 1038448 acc += Wp[w_oc_base + k] * col[k];
170 1038448 }
171 3960 Yp[(static_cast<size_t>(n) * C_out + oc) * H_out * W_out + sp] = acc;
172 3960 }
173 709 }
174 71 }
175 8 }
176 7 });
177 6 }
178
179 } // namespace brotensor::detail::cpu
180