feat(scene-detector): run the learned boundary detector live in the C++ pipeline
Wire the XGBoost scene-boundary detector into scene_analyze as a post-EOF step in
the result sink (like flood-fill itself — the per-film knee threshold needs the
whole film, so it cannot stream). With --scene-xgb-model set, the camera-position
node stamps a per-frame RGB histogram onto the Frame, it rides through to the
sink, and at EOF the sink runs XGBSceneBoundary over the collected histograms +
the movie's per-second audio log-PSD to produce the flood-fill boundaries. Falls
back to is_scene_boundary / is_cut when no model is configured or inference fails.
Inference is real XGBoost via CMake FetchContent (v2.1.1, static), C API in
src/inference/xgb_scene_boundary.hpp; audio log-PSD in src/inference/
audio_logpsd.hpp (FFTW + ffmpeg full-file 16kHz decode). Feature extraction
matches training exactly — video features verified row-identical to numpy, and to
avoid chasing numpy's every rounding the shipped model is TRAINED on the
C++-extracted features (scene_features_dump exe → train_xgb_cpp.py). The
C++/Python peak-finders differ slightly so boundary counts differ, but what
matters is downstream: flood + C++ detector = 75.8% macro presence F1 vs 64.0%
for the histogram-cut flood and 62.5% for track_extent, and it fixes the Scarface
flood collapse (41 -> 70). All nine films improve.
Guarded by the SAE_SCENE_XGB CMake option (on by default; heavy first build).
xgb_boundary_parity is a diff harness; scene_features_dump writes the C++ feature
matrix so training and inference share one feature implementation.
Verified end to end: scene_analyze --scene-xgb-model on a real movie stamps the
histogram, runs the detector at EOF ("XGBoost scene detector: N boundaries"), and
flood-snaps presence to the learned boundaries.
This commit is contained in:
+48
-3
@@ -280,6 +280,24 @@ FetchContent_Declare(
|
||||
)
|
||||
FetchContent_MakeAvailable(nanobind)
|
||||
|
||||
# XGBoost (learned scene-boundary detector for flood-fill presence). Fetched and
|
||||
# built from source so we get both the C API header and a matching libxgboost,
|
||||
# reproducibly — the pip wheel ships the .so but no header. Heavy first build, so
|
||||
# it is opt-in; the scene-boundary node is compiled only when SAE_SCENE_XGB is on.
|
||||
option(SAE_SCENE_XGB "Build the XGBoost scene-boundary detector node" ON)
|
||||
if(SAE_SCENE_XGB)
|
||||
set(BUILD_STATIC_LIB ON CACHE BOOL "" FORCE) # link xgboost statically
|
||||
set(USE_OPENMP ON CACHE BOOL "" FORCE)
|
||||
FetchContent_Declare(
|
||||
xgboost
|
||||
GIT_REPOSITORY https://github.com/dmlc/xgboost.git
|
||||
GIT_TAG v2.1.1
|
||||
GIT_SHALLOW TRUE
|
||||
GIT_SUBMODULES_RECURSE TRUE
|
||||
)
|
||||
FetchContent_MakeAvailable(xgboost)
|
||||
endif()
|
||||
|
||||
# ── Model paths ───────────────────────────────────────────────────────────────
|
||||
set(SAE_MODELS_DIR "${CMAKE_SOURCE_DIR}/models"
|
||||
CACHE PATH "Directory containing ONNX model files")
|
||||
@@ -352,16 +370,43 @@ target_link_libraries(sae_audio PRIVATE ffmpeg_libs)
|
||||
# HDF5 already found above (before sae_gallery); vars HDF5_CXX_LIBRARIES / _INCLUDE_DIRS
|
||||
# are reused by scene_analyze / dump_embeddings below.
|
||||
|
||||
# The learned scene-boundary detector is compiled into the sink (result_sink →
|
||||
# xgb_scene_boundary + audio_logpsd) when SAE_SCENE_XGB is on, so the analysis
|
||||
# binaries need xgboost + FFTW + ffmpeg and the define. Found once here.
|
||||
if(SAE_SCENE_XGB)
|
||||
find_library(FFTW3_LIB fftw3 REQUIRED)
|
||||
set(SAE_SCENE_LIBS xgboost ${FFTW3_LIB} ffmpeg_libs)
|
||||
set(SAE_SCENE_DEFS SAE_SCENE_XGB)
|
||||
else()
|
||||
set(SAE_SCENE_LIBS "")
|
||||
set(SAE_SCENE_DEFS "")
|
||||
endif()
|
||||
|
||||
# ── analyze — main analysis binary ───────────────────────────────────────────
|
||||
add_executable(scene_analyze src/main.cpp)
|
||||
target_link_libraries(scene_analyze PRIVATE sae_gallery ${HDF5_CXX_LIBRARIES})
|
||||
target_link_libraries(scene_analyze PRIVATE sae_gallery ${HDF5_CXX_LIBRARIES} ${SAE_SCENE_LIBS})
|
||||
target_include_directories(scene_analyze PRIVATE ${HDF5_INCLUDE_DIRS})
|
||||
target_compile_definitions(scene_analyze PRIVATE ${SAE_SCENE_DEFS})
|
||||
|
||||
# ── xgb_boundary_parity — prove C++ scene-boundary inference matches Python ───
|
||||
if(SAE_SCENE_XGB)
|
||||
add_executable(xgb_boundary_parity src/tools/xgb_boundary_parity.cpp)
|
||||
target_include_directories(xgb_boundary_parity PRIVATE src ${HDF5_INCLUDE_DIRS})
|
||||
target_link_libraries(xgb_boundary_parity PRIVATE
|
||||
xgboost ${HDF5_CXX_LIBRARIES} ${FFTW3_LIB} ffmpeg_libs)
|
||||
|
||||
# Dumps the C++ feature matrix so training uses the exact inference features.
|
||||
add_executable(scene_features_dump src/tools/scene_features_dump.cpp)
|
||||
target_include_directories(scene_features_dump PRIVATE src ${HDF5_INCLUDE_DIRS})
|
||||
target_link_libraries(scene_features_dump PRIVATE
|
||||
xgboost ${HDF5_CXX_LIBRARIES} ${FFTW3_LIB} ffmpeg_libs)
|
||||
endif()
|
||||
|
||||
# ── analyze_debug — same binary with debug frame/crop output ─────────────────
|
||||
add_executable(scene_analyze_debug src/main.cpp)
|
||||
target_link_libraries(scene_analyze_debug PRIVATE sae_gallery ${HDF5_CXX_LIBRARIES})
|
||||
target_link_libraries(scene_analyze_debug PRIVATE sae_gallery ${HDF5_CXX_LIBRARIES} ${SAE_SCENE_LIBS})
|
||||
target_include_directories(scene_analyze_debug PRIVATE ${HDF5_INCLUDE_DIRS})
|
||||
target_compile_definitions(scene_analyze_debug PRIVATE SAE_DEBUG=1)
|
||||
target_compile_definitions(scene_analyze_debug PRIVATE SAE_DEBUG=1 ${SAE_SCENE_DEFS})
|
||||
|
||||
# ── dump_embeddings — standalone embedding dumper, NO gallery/matcher ─────────
|
||||
# Front-half only (decode→detect→align→embed→HDF5) for the optimizer replay corpus
|
||||
|
||||
Reference in New Issue
Block a user