From 4dcef8d6c5725c49df082f54a2938ea86d11cd8a Mon Sep 17 00:00:00 2001 From: Duncan Tourolle Date: Thu, 6 Aug 2026 20:34:16 +0200 Subject: [PATCH] fix(AR-004): the TransNetV2 window stores the model's input, not the frame MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rolling window held frames as decoded — `images_.push_back(f.image)` — and left the downscale to the backend. TransNetV2's input is 48x27, so the buffer held roughly 590 MB at 1080p to feed a model that needs about 380 KB. The config note for `dense_scale` says as much outright: "TransNetV2 downsamples to 48x27 regardless". This is not a channel capacity, so no amount of tuning channel depths would ever have found it. It is a `std::deque` member, and it is the single largest allocation in the scene branch. It is also redundant work. Windows overlap by `kWindow - stride`, so a frame appears in several of them and was re-downscaled once per window it appeared in; now it is downscaled once, on arrival. **The risk here is the invariant, not the memory.** Every model gets the input it was trained for — a model run off-distribution returns confident, plausible, wrong output, and for a boundary detector that means fabricated cuts, which are indistinguishable from real ones in the output. So this reproduces the backends' preprocessing exactly rather than doing its own: both ort_backend.cpp and trt_backend.cpp guard mis-sized input with `convertTo(CV_8UC3)` and then `cv::resize(..., {kFrameW, kFrameH}, 0, 0, cv::INTER_AREA)`, in that order, and `to_model_input` performs the same two operations. The backend guard then sees a correctly-sized frame and does nothing, so the tensor the model receives is unchanged. The interface has always specified this as the caller's job — "Each frame must already be kFrameW x kFrameH, BGR, CV_8UC3" — so the node now meets a contract it was already given. The tests assert equivalence, not size. They perform the backend's own two operations independently and compare byte for byte, on a gradient rather than a flat fill, since INTER_AREA averages and a constant image would compare equal under almost any resize. Order is pinned too: converting a 4-channel frame after downscaling averages alpha into the colour channels and gives different pixels. Verified in both directions. With INTER_LINEAR substituted for INTER_AREA — the most plausible way to get this subtly wrong — three assertions fail. With the backend's own operations, byte-identical at 1920x1080, 640x360 and 720x480. 149/149. Still unmeasured on real content, as with the previous commit: the equivalence argument says the model sees the same tensor, but a run comparing scenes.json before and after on a real clip is what would settle it, and I could not launch one here. TRACES: AR-004, AR-010 | SR-002 --- src/nodes/scene_detector_node.hpp | 46 ++++++++++++- tests/test_scene_detector_node.cpp | 105 +++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) diff --git a/src/nodes/scene_detector_node.hpp b/src/nodes/scene_detector_node.hpp index e9369a2..de044bf 100644 --- a/src/nodes/scene_detector_node.hpp +++ b/src/nodes/scene_detector_node.hpp @@ -6,6 +6,8 @@ #include #include "inference/scene_detector.hpp" +#include // cv::resize, for to_model_input + #include #include #include @@ -76,7 +78,7 @@ struct SceneDetectorFunc { } prev_ts_ = f.timestamp_sec; - images_.push_back(f.image); + images_.push_back(to_model_input(f.image)); times_.push_back(f.timestamp_sec); // Once we have a full window, score it and slide forward by `stride`. @@ -90,6 +92,48 @@ struct SceneDetectorFunc { } } + /// TRACES: AR-004, AR-010 | SR-002 + /// Reduce a decoded frame to exactly what TransNetV2 consumes, once. + /// + /// The window used to hold the frames as decoded — full resolution — and + /// leave the downscale to the backend. But the model's input is 48x27 + /// (`ISceneDetector::kFrameW/H`; the config note for `dense_scale` says so + /// outright: "TransNetV2 downsamples to 48x27 regardless"), so the buffer + /// held ~590 MB at 1080p to feed something that needs ~380 KB. That is not + /// a channel capacity, so no amount of tuning channel depths would ever + /// have found it. + /// + /// It is also redundant work. Windows overlap by `kWindow - stride`, so a + /// frame appears in several of them and was re-downscaled once per window; + /// now it is downscaled once, when it arrives. + /// + /// **This must reproduce the backends' preprocessing exactly**, because the + /// project invariant is that every model gets the input it was trained for + /// — a model run off-distribution returns confident, plausible, wrong + /// output, and here that means fabricated shot boundaries. Both + /// ort_backend.cpp and trt_backend.cpp guard mis-sized input with, in this + /// order, `convertTo(CV_8UC3)` then + /// `cv::resize(..., {kFrameW, kFrameH}, 0, 0, cv::INTER_AREA)`. The same + /// two operations are done here, so the tensor the model receives is + /// unchanged; the backend guard then sees a correctly-sized frame and does + /// nothing. The interface has always specified this shape as the caller's + /// job ("Each frame must already be kFrameW x kFrameH, BGR, CV_8UC3"), so + /// this makes the node meet a contract it was already given. + static cv::Mat to_model_input(const cv::Mat& src) { + cv::Mat typed; + if (src.type() != CV_8UC3) src.convertTo(typed, CV_8UC3); + else typed = src; + + if (typed.cols == ISceneDetector::kFrameW && + typed.rows == ISceneDetector::kFrameH) + return typed; + + cv::Mat small; + cv::resize(typed, small, {ISceneDetector::kFrameW, ISceneDetector::kFrameH}, + 0, 0, cv::INTER_AREA); + return small; + } + /// TRACES: AR-011 | SR-002 // How close two boundaries have to be before they are the same boundary, // derived from the cadence the detector was actually fed. diff --git a/tests/test_scene_detector_node.cpp b/tests/test_scene_detector_node.cpp index 0ed590d..eddf2cc 100644 --- a/tests/test_scene_detector_node.cpp +++ b/tests/test_scene_detector_node.cpp @@ -14,6 +14,9 @@ #include "nodes/scene_detector_node.hpp" +#include +#include + #include namespace { @@ -84,3 +87,105 @@ TEST_CASE("too few frames to have a cadence yields an inert window", TEST_CASE("a single observed interval is enough", "[scene][AR-011]") { CHECK(SceneDetectorFunc::dedup_window_sec({1.0 / 24.0}) == 0.5 / 24.0); } + +// ── AR-004 — the window stores the model's input, not the decoded frame ─────── +// +// TRACES: AR-004, AR-010 | SR-002 | UT-003 +// +// The rolling window held frames as decoded, at full resolution, and left the +// downscale to the backend — ~590 MB at 1080p to feed a model whose input is +// 48x27, about 380 KB. Not a channel capacity, so no amount of tuning channel +// depths would have found it. +// +// The risk in fixing it is the project invariant: every model gets the input it +// was trained for. A model run off-distribution returns confident, plausible, +// wrong output, and here that means fabricated shot boundaries — which would be +// indistinguishable from a real cut in the output. +// +// So these cases do not check that the frames got smaller. They check that the +// pixels are *identical* to what the backend would have produced from the full +// frame, by performing the backend's own two operations independently and +// comparing byte for byte. Both ort_backend.cpp and trt_backend.cpp guard +// mis-sized input with convertTo(CV_8UC3) then +// cv::resize(..., {kFrameW, kFrameH}, 0, 0, cv::INTER_AREA), in that order. +namespace { + +cv::Mat gradient(int w, int h) { + // Structured content, not a flat fill: INTER_AREA averages, so a constant + // image would compare equal under almost any resize and prove nothing. + cv::Mat m(h, w, CV_8UC3); + for (int y = 0; y < h; ++y) + for (int x = 0; x < w; ++x) + m.at(y, x) = cv::Vec3b( + static_cast((x * 7 + y * 3) % 256), + static_cast((x * 13 + y * 5) % 256), + static_cast((x * 3 + y * 11) % 256)); + return m; +} + +bool identical(const cv::Mat& a, const cv::Mat& b) { + if (a.size() != b.size() || a.type() != b.type()) return false; + cv::Mat diff; + cv::absdiff(a, b, diff); + return cv::countNonZero(diff.reshape(1)) == 0; +} + +} // namespace + +TEST_CASE("the window frame is what the backend would have produced", + "[scene][AR-004]") { + for (auto [w, h] : {std::pair{1920, 1080}, std::pair{640, 360}, std::pair{720, 480}}) { + INFO("source " << w << "x" << h); + const cv::Mat full = gradient(w, h); + + // The backend's own guard, performed here independently. + cv::Mat expected; + cv::resize(full, expected, {ISceneDetector::kFrameW, ISceneDetector::kFrameH}, + 0, 0, cv::INTER_AREA); + + const cv::Mat got = SceneDetectorFunc::to_model_input(full); + + REQUIRE(got.cols == ISceneDetector::kFrameW); + REQUIRE(got.rows == ISceneDetector::kFrameH); + REQUIRE(got.type() == CV_8UC3); + CHECK(identical(got, expected)); + } +} + +TEST_CASE("a frame already at model size is passed through untouched", + "[scene][AR-004]") { + // The backend skips its guard for a correctly-sized frame, so this path must + // not resize either — resampling an already-48x27 image would change it. + const cv::Mat exact = gradient(ISceneDetector::kFrameW, ISceneDetector::kFrameH); + CHECK(identical(SceneDetectorFunc::to_model_input(exact), exact)); +} + +TEST_CASE("conversion happens before the resize, as the backend does it", + "[scene][AR-004]") { + // Order matters: converting a 4-channel frame after downscaling averages + // alpha into the colour channels and gives different pixels. The backends + // convert first, so this must too. + cv::Mat four(360, 640, CV_8UC4, cv::Scalar(10, 20, 30, 255)); + cv::Mat typed; + four.convertTo(typed, CV_8UC3); + cv::Mat expected; + cv::resize(typed, expected, {ISceneDetector::kFrameW, ISceneDetector::kFrameH}, + 0, 0, cv::INTER_AREA); + + CHECK(identical(SceneDetectorFunc::to_model_input(four), expected)); +} + +TEST_CASE("the window's memory is bounded by the model input, not the source", + "[scene][AR-004]") { + // The point of the change, stated as a number: a full window of 1080p + // frames is ~590 MB as decoded and ~380 KB as model input. + const cv::Mat full = gradient(1920, 1080); + const cv::Mat small = SceneDetectorFunc::to_model_input(full); + + const std::size_t decoded = full.total() * full.elemSize(); + const std::size_t stored = small.total() * small.elemSize(); + + INFO("decoded " << decoded << " B, stored " << stored << " B"); + CHECK(stored * 1000 < decoded); // three orders of magnitude + CHECK(stored == ISceneDetector::kFrameW * ISceneDetector::kFrameH * 3u); +}