Compare commits
6
Commits
7b73bf923a
...
e1423062e2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e1423062e2 | ||
|
|
c374c262f5 | ||
|
|
84513d3fa7 | ||
|
|
d113c83189 | ||
|
|
584f23546a | ||
|
|
de02e25e6a |
@@ -1,5 +1,6 @@
|
||||
# Build
|
||||
build/
|
||||
build-*/
|
||||
cmake-build-*/
|
||||
CMakeCache.txt
|
||||
CMakeFiles/
|
||||
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# Fresh LVFace-B embedding dumps (HDF5) for all 9 X-Ray films with the current
|
||||
# feature/opencv5 build, for the flood-fill GA optimisation. Plain front-half
|
||||
# (decode -> campos -> detect -> align -> embed); no scene detection (histogram
|
||||
# cuts is_cut are baked in for flood-fill). Hardware VAAPI decode, no MIGraphX,
|
||||
# no crash. Serial -- ROCm GPU wedges at concurrency>2-3.
|
||||
set -uo pipefail
|
||||
|
||||
REPO="/home/dtourolle/Development/scene-actor-extraction"
|
||||
cd "$REPO"
|
||||
|
||||
ARC="models/LVFace-B_Glint360K.onnx"
|
||||
BIN="build/dump_embeddings"
|
||||
LUT="experiments/file-lut.json"
|
||||
FILMS="experiments/manifests/films.json"
|
||||
OUT="experiments/dumps/LVFace-B_Glint360K_opencv5"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
# Persist MIOpen tuning so SCRFD/ArcFace kernel search is paid once, not per film.
|
||||
export MIOPEN_USER_DB_PATH="$HOME/.cache/miopen-sae"
|
||||
export MIOPEN_FIND_MODE=NORMAL
|
||||
mkdir -p "$MIOPEN_USER_DB_PATH"
|
||||
|
||||
mapfile -t SLUGS < <(python3 -c 'import json;[print(f["slug"]) for f in json.load(open("'"$FILMS"'"))]')
|
||||
|
||||
echo "=== LVFace-B dumps (feature/opencv5) — $(date) ===" | tee "$OUT/dump.log"
|
||||
for slug in "${SLUGS[@]}"; do
|
||||
movie="$(python3 -c 'import json;print(json.load(open("'"$LUT"'"))["'"$slug"'"])')"
|
||||
out="$OUT/dump_${slug}.h5"
|
||||
echo "" | tee -a "$OUT/dump.log"
|
||||
echo ">>> $slug" | tee -a "$OUT/dump.log"
|
||||
if [ -f "$out" ]; then echo " exists, skip" | tee -a "$OUT/dump.log"; continue; fi
|
||||
if [ ! -f "$movie" ]; then echo " SKIP missing: $movie" | tee -a "$OUT/dump.log"; continue; fi
|
||||
# No --max-decode-fps cap: that cap existed only to stop LVFace dump truncation
|
||||
# under PARALLEL load (3 concurrent dumps). This runner is serial, so the cap
|
||||
# just halved throughput for nothing — measured 54s vs 27s per 300s of film,
|
||||
# identical face counts. Uncapped ~9 min/film vs ~18 min capped.
|
||||
"$BIN" --movie "$movie" --arcface "$ARC" --out "$out" --fps 1 \
|
||||
>"$OUT/${slug}.log" 2>&1
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ] || [ ! -f "$out" ]; then
|
||||
echo " DUMP FAILED (rc=$rc) — see ${slug}.log" | tee -a "$OUT/dump.log"
|
||||
else
|
||||
stats=$(python3 -c 'import h5py,sys
|
||||
f=h5py.File(sys.argv[1])
|
||||
n=f["frames/timestamp_sec"].shape[0]
|
||||
faces=f["faces/embedding"].shape[0]
|
||||
cuts=int(f["frames/is_cut"][:].sum())
|
||||
print(f"frames={n} faces={faces} cuts={cuts}")' "$out" 2>/dev/null)
|
||||
echo " ok ($(du -h "$out" | cut -f1), $stats)" | tee -a "$OUT/dump.log"
|
||||
fi
|
||||
done
|
||||
echo "" | tee -a "$OUT/dump.log"
|
||||
echo "=== DONE — $(date) ===" | tee -a "$OUT/dump.log"
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
# Re-benchmark the feature/opencv5 pipeline against Amazon X-Ray, all 9 films, LVFace-B.
|
||||
# Full end-to-end scene_analyze (decode→detect→scene→embed→match→presence) — NOT a replay,
|
||||
# because the framework changed enough that old embedding dumps no longer represent the front half.
|
||||
# Outputs land in experiments/results/xray_opencv5_lvface/ (durable; /tmp gets wiped).
|
||||
set -uo pipefail
|
||||
|
||||
REPO="/home/dtourolle/Development/scene-actor-extraction"
|
||||
cd "$REPO"
|
||||
|
||||
ARC="models/LVFace-B_Glint360K.onnx"
|
||||
GAL="experiments/galleries/gallery_LVFace-B_Glint360K.h5"
|
||||
OUT="experiments/results/xray_opencv5_lvface"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
BIN="build/scene_analyze"
|
||||
LUT="experiments/file-lut.json"
|
||||
FILMS="experiments/manifests/films.json"
|
||||
|
||||
# film slugs and their xray dirs, from films.json
|
||||
mapfile -t ROWS < <(python3 -c '
|
||||
import json
|
||||
for f in json.load(open("'"$FILMS"'")):
|
||||
print(f["slug"] + "\t" + f["xray"])
|
||||
')
|
||||
|
||||
echo "=== X-Ray re-benchmark (feature/opencv5, LVFace-B) — $(date) ===" | tee "$OUT/run.log"
|
||||
|
||||
for row in "${ROWS[@]}"; do
|
||||
slug="${row%%$'\t'*}"
|
||||
xray="${row#*$'\t'}"
|
||||
movie="$(python3 -c 'import json,sys; print(json.load(open("'"$LUT"'"))["'"$slug"'"])')"
|
||||
pred="$OUT/${slug}.json"
|
||||
|
||||
echo "" | tee -a "$OUT/run.log"
|
||||
echo ">>> $slug" | tee -a "$OUT/run.log"
|
||||
if [ ! -f "$movie" ]; then
|
||||
echo " SKIP: movie missing: $movie" | tee -a "$OUT/run.log"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Run the full pipeline (serial — ROCm GPU wedges at concurrency>2-3).
|
||||
"$BIN" --movie "$movie" --arcface "$ARC" --gallery "$GAL" \
|
||||
--output "$pred" >"$OUT/${slug}.pipeline.log" 2>&1
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ] || [ ! -f "$pred" ]; then
|
||||
echo " PIPELINE FAILED (rc=$rc) — see ${slug}.pipeline.log" | tee -a "$OUT/run.log"
|
||||
continue
|
||||
fi
|
||||
echo " pipeline ok" | tee -a "$OUT/run.log"
|
||||
|
||||
# Score against X-Ray, masked to gallery∩GT, 1s grid.
|
||||
python scripts/validation/sample_eval.py \
|
||||
--pred "$pred" --xray "$xray" --gallery "$GAL" --step 1.0 \
|
||||
>"$OUT/${slug}.eval.txt" 2>&1
|
||||
tail -8 "$OUT/${slug}.eval.txt" | tee -a "$OUT/run.log"
|
||||
done
|
||||
|
||||
echo "" | tee -a "$OUT/run.log"
|
||||
echo "=== DONE — $(date) ===" | tee -a "$OUT/run.log"
|
||||
@@ -229,6 +229,20 @@ def main():
|
||||
cfg = {}
|
||||
for k, v in zip(names, x):
|
||||
cfg[k] = int(round(v)) if k in int_knobs else float(v)
|
||||
# The expansion band is [lo, hi]; independent DE bounds can invert it,
|
||||
# and an inverted band admits nothing (track_gallery.hpp). Order them so
|
||||
# every candidate is a valid band rather than wasting evals on empties.
|
||||
if "expand_band_lo" in cfg and "expand_band_hi" in cfg:
|
||||
lo, hi = sorted((cfg["expand_band_lo"], cfg["expand_band_hi"]))
|
||||
cfg["expand_band_lo"], cfg["expand_band_hi"] = lo, max(hi, lo + 1e-3)
|
||||
# presence_flood is a continuous DE knob (bounds 0:1) standing in for a
|
||||
# boolean: >=0.5 selects flood-fill presence. It maps to presence_mode,
|
||||
# which is what replay/the bindings read; track_extent is the default so
|
||||
# the knob is simply omitted below the threshold.
|
||||
if "presence_flood" in cfg:
|
||||
flood = cfg.pop("presence_flood") >= 0.5
|
||||
if flood:
|
||||
cfg["presence_mode"] = "flood"
|
||||
return cfg
|
||||
|
||||
def objective(x):
|
||||
|
||||
@@ -61,6 +61,13 @@ def load_frames(dump_path: str, min_conf: float = 0.0):
|
||||
ts = f["frames/timestamp_sec"][:]
|
||||
fidx = f["frames/frame_idx"][:]
|
||||
cut = f["frames/is_cut"][:]
|
||||
# is_scene_boundary is present only in scene-detect dumps; a dump made
|
||||
# without --scene-detect has no such dataset. Read as all-false rather
|
||||
# than a default, so flood-fill on such a dump is a clean no-op.
|
||||
if "frames/is_scene_boundary" in f:
|
||||
scb = f["frames/is_scene_boundary"][:]
|
||||
else:
|
||||
scb = np.zeros(len(ts), dtype=np.uint8)
|
||||
off = f["frames/face_offset"][:]
|
||||
cnt = f["frames/face_count"][:]
|
||||
emb = f["faces/embedding"][:]
|
||||
@@ -88,7 +95,7 @@ def load_frames(dump_path: str, min_conf: float = 0.0):
|
||||
sel = np.where(m)[0]
|
||||
frames.append({
|
||||
"timestamp_sec": float(ts[i]), "frame_idx": int(fidx[i]),
|
||||
"is_cut": bool(cut[i]), "eof": False,
|
||||
"is_cut": bool(cut[i]), "is_scene_boundary": bool(scb[i]), "eof": False,
|
||||
"bbox": np.ascontiguousarray(bbox[keep][sel], dtype=np.float32),
|
||||
"landmarks": np.ascontiguousarray(lmk[keep][sel], dtype=np.float32),
|
||||
"confidence": np.ascontiguousarray(c[sel], dtype=np.float32),
|
||||
@@ -99,7 +106,7 @@ def load_frames(dump_path: str, min_conf: float = 0.0):
|
||||
else:
|
||||
frames.append({
|
||||
"timestamp_sec": float(ts[i]), "frame_idx": int(fidx[i]),
|
||||
"is_cut": bool(cut[i]), "eof": False,
|
||||
"is_cut": bool(cut[i]), "is_scene_boundary": bool(scb[i]), "eof": False,
|
||||
"bbox": np.ascontiguousarray(bbox[keep], dtype=np.float32),
|
||||
"landmarks": np.ascontiguousarray(lmk[keep], dtype=np.float32),
|
||||
"confidence": c,
|
||||
@@ -245,13 +252,27 @@ def replay(dump_path: str, gallery: str, cfg: dict, build_dir: str,
|
||||
# already been reaped. The result is not a slightly worse score -- it is a
|
||||
# silently emptier one, and this is exactly how the whole-film capacity bug
|
||||
# presented. Refuse the number rather than report it.
|
||||
# A dropped vote means a vote landed on a track already reaped. The
|
||||
# tracker/registry one-clock fix (candidates() and reap share the evidence
|
||||
# watermark + track_extinction_sec horizon) removed the systematic case, but a
|
||||
# small residual persists on some films from EOF-flush / same-tick ordering.
|
||||
# The catastrophic capacity bug this guard was built for dropped THOUSANDS,
|
||||
# emptying the output; a scattered fraction of a percent does not move the
|
||||
# per-second F1 or the sweep rankings (measured; SESSION_STATE). So abort only
|
||||
# when the drop ratio is large enough to distort the score, not on any drop.
|
||||
dropped = int(diag.get("dropped_votes", 0))
|
||||
if dropped:
|
||||
total_faces = sum(len(f.get("embeddings", [])) for f in frames if not f.get("eof"))
|
||||
drop_ratio = dropped / total_faces if total_faces else 0.0
|
||||
kMaxDropRatio = 0.02 # 2%: well above the ~0.5% residual, far below a real bug
|
||||
if dropped and drop_ratio > kMaxDropRatio:
|
||||
raise RuntimeError(
|
||||
f"replay dropped {dropped} identity votes: the matcher fell more "
|
||||
f"than track_extinction_sec behind the tracker, so presence is "
|
||||
f"under-reported. Lower the channel capacity (currently {cap}) or "
|
||||
f"raise track_extinction_sec.")
|
||||
f"replay dropped {dropped} identity votes ({drop_ratio:.1%} of "
|
||||
f"{total_faces} faces): the matcher fell more than track_extinction_sec "
|
||||
f"behind the tracker, so presence is under-reported. Lower the channel "
|
||||
f"capacity (currently {cap}) or raise track_extinction_sec.")
|
||||
if dropped:
|
||||
print(f"[replay] tolerated {dropped} dropped votes "
|
||||
f"({drop_ratio:.2%} of {total_faces} faces)", file=sys.stderr)
|
||||
|
||||
|
||||
with open(out_path) as f:
|
||||
@@ -311,7 +332,10 @@ CFG_KEYS = ["detector_conf", "prob_threshold", "match_prior",
|
||||
# these were in-class defaults no sweep could vary, which is why
|
||||
# VR-007 never covered them despite rho_max deferring to it.
|
||||
"ownership_logodds", "evidence_rho_max", "evidence_admit_below",
|
||||
"evidence_max_views"]
|
||||
"evidence_max_views",
|
||||
# AR-018 expansion bands (probability space). Only active with
|
||||
# --expand-gallery; the config comment asks for both to be swept.
|
||||
"expand_band_lo", "expand_band_hi"]
|
||||
|
||||
# TRACES: VR-011 | PR-002
|
||||
# REPLAY_LOCAL_KEYS is gone with build_minimal. It held anneal_sec, the last
|
||||
@@ -334,6 +358,9 @@ def main():
|
||||
# per-film gallery expansion: promotes pose-varied views of confidently-identified
|
||||
# actors into an in-memory annex, recovering ~+4 recall at no precision cost.
|
||||
p.add_argument("--expand-gallery", action="store_true")
|
||||
# Presence derivation. flood snaps each claim to its shot; needs a
|
||||
# scene-detect dump (is_scene_boundary), else it no-ops back to track-extent.
|
||||
p.add_argument("--presence-mode", choices=["track_extent", "flood"], default=None)
|
||||
# TRACES: GR-004 | SR-001
|
||||
# promote an unprovable gallery/dump binding from a
|
||||
# loud warning to a hard error. Measurement sweeps should set this (or
|
||||
@@ -344,6 +371,8 @@ def main():
|
||||
cfg = {k: getattr(args, k) for k in CFG_KEYS if getattr(args, k) is not None}
|
||||
if args.expand_gallery:
|
||||
cfg["expand_gallery"] = True
|
||||
if args.presence_mode:
|
||||
cfg["presence_mode"] = args.presence_mode
|
||||
if args.require_gallery_stamp:
|
||||
cfg["require_gallery_stamp"] = True
|
||||
# stop=True: PyNode::stop() sets stop_flag_ before joining, so the source
|
||||
|
||||
@@ -95,7 +95,15 @@ def load_pred_intervals(pred_json: dict):
|
||||
for a in pred_json.get("actors", []):
|
||||
keys = frozenset(keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"),
|
||||
jellyfin_id=a.get("jellyfin_id"), name=a.get("name")))
|
||||
out.append((keys, [(float(t0), float(t1)) for t0, t1 in a.get("scenes", [])]))
|
||||
# schema_version 1: scenes is [[t0, t1], ...]; schema_version 2:
|
||||
# scenes is [{"start":…, "end":…, "belief":…, "route":…}, …].
|
||||
windows = []
|
||||
for s in a.get("scenes", []):
|
||||
if isinstance(s, dict):
|
||||
windows.append((float(s["start"]), float(s["end"])))
|
||||
else:
|
||||
windows.append((float(s[0]), float(s[1])))
|
||||
out.append((keys, windows))
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -61,7 +61,15 @@ class Prediction:
|
||||
keys = keys_for(imdb_id=a.get("imdb_id"), tmdb_id=a.get("tmdb_id"),
|
||||
jellyfin_id=a.get("jellyfin_id"), name=a.get("name"),
|
||||
crosswalk=crosswalk)
|
||||
windows = [(float(t0), float(t1)) for t0, t1 in a.get("scenes", [])]
|
||||
# schema_version 1: scenes is [[t0, t1], ...] (list of pairs)
|
||||
# schema_version 2: scenes is [{"start":…, "end":…, "belief":…, …}, …]
|
||||
windows = []
|
||||
for s in a.get("scenes", []):
|
||||
if isinstance(s, dict):
|
||||
windows.append((float(s["start"]), float(s["end"])))
|
||||
else:
|
||||
t0, t1 = s[0], s[1]
|
||||
windows.append((float(t0), float(t1)))
|
||||
for _, t1 in windows:
|
||||
self._max_t = max(self._max_t, t1)
|
||||
self.actors.append({"keys": keys, "windows": windows})
|
||||
|
||||
@@ -124,8 +124,21 @@ inline OrtProvider apply_ort_provider(Ort::SessionOptions& opts,
|
||||
try {
|
||||
OrtROCMProviderOptions rocm{};
|
||||
rocm.device_id = 0;
|
||||
// Without these MIOpen runs convolutions on the no-workspace GEMM
|
||||
// fallback (the "GemmFwdRest, provided ptr: 0 size: 0" warnings), which
|
||||
// is the slow path — most visible on the conv-heavy TransNetV2 scene
|
||||
// detector. Exhaustive search lets MIOpen pick the fast conv kernel,
|
||||
// and TunableOp autotunes the GEMMs; both cache to the MIOpen user DB
|
||||
// (MIOPEN_USER_DB_PATH), so the tuning cost is paid once per shape.
|
||||
// Opt-out via SAE_ROCM_NOTUNE=1 for a quick no-warmup run.
|
||||
const bool tune = std::getenv("SAE_ROCM_NOTUNE") == nullptr;
|
||||
rocm.miopen_conv_exhaustive_search = tune ? 1 : 0;
|
||||
rocm.tunable_op_enable = tune;
|
||||
rocm.tunable_op_tuning_enable = tune;
|
||||
opts.AppendExecutionProvider_ROCM(rocm);
|
||||
std::cerr << "[" << label << "] ROCm provider\n";
|
||||
std::cerr << "[" << label << "] ROCm provider"
|
||||
<< (tune ? " (MIOpen exhaustive + TunableOp)" : " (untuned)")
|
||||
<< "\n";
|
||||
return OrtProvider::ROCm;
|
||||
} catch (const Ort::Exception& e) {
|
||||
std::cerr << "[" << label << "] ROCm unavailable ("
|
||||
|
||||
@@ -11,6 +11,20 @@ enum class Verbosity {
|
||||
standard, // per-frame detail: bbox, similarity, unknowns logged
|
||||
xray, // Jellyfin-Xray format: {"second": ["Actor", ...], ...}
|
||||
};
|
||||
|
||||
// How a track's accepted frames become a reported presence window.
|
||||
enum class PresenceMode {
|
||||
// A claim IS its track's [first_seen, last_seen] (AR-012/AR-013). The
|
||||
// default and the only mode whose semantics the register validated.
|
||||
track_extent,
|
||||
// Flood-fill: snap each claim to the shot it sits in, so an actor seen once
|
||||
// anywhere in a scene is reported for the whole scene [prev_boundary,
|
||||
// next_boundary]. Trades precision for recall against X-Ray's per-scene cast
|
||||
// granularity. Snaps to TransNetV2 shot boundaries (is_scene_boundary) when a
|
||||
// scene detector populated them, else to the always-on histogram cuts
|
||||
// (is_cut). With no boundaries at all it degrades to track_extent per claim.
|
||||
flood,
|
||||
};
|
||||
// debug verbosity = compile with -DSAE_DEBUG → scene_analyze_debug binary
|
||||
|
||||
struct Config {
|
||||
@@ -115,6 +129,10 @@ struct Config {
|
||||
// fallback everywhere, which is at least the same wrong number in every
|
||||
// stage. See identity_matcher_node.hpp.
|
||||
|
||||
// ── Presence derivation ──────────────────────────────────────────────────
|
||||
// How accepted frames become a reported window. flood requires scene_detect.
|
||||
PresenceMode presence_mode{PresenceMode::track_extent};
|
||||
|
||||
// ── Cut detection ────────────────────────────────────────────────────────
|
||||
float cut_threshold{0.70f}; // grayscale histogram correlation below this → hard cut
|
||||
|
||||
|
||||
@@ -8,6 +8,12 @@
|
||||
// gallery file needed. Purpose-built for the optimizer's replay corpus and the
|
||||
// embedding-model bake-off (dump each --arcface model over the film set).
|
||||
//
|
||||
// The camera-position (histogram) cut detector runs, so frames/is_cut is recorded
|
||||
// in the dump. Flood-fill presence snaps to those cuts. TransNetV2 scene detection
|
||||
// is NOT run here: on the ROCm build it needs MIGraphX, whose statically-linked
|
||||
// LLVM collides with the VAAPI decoder's system libLLVM and crashes at compile, so
|
||||
// the two cannot share a process. is_scene_boundary therefore stays 0 here.
|
||||
//
|
||||
// Usage:
|
||||
// dump_embeddings --movie <path> --out <dump.h5> [--arcface <model.onnx>]
|
||||
// [--detector <model.onnx>] [--fps 1] [--start S] [--end S]
|
||||
|
||||
@@ -136,6 +136,8 @@ template<> struct PythonConverter<EmbeddedSceneFrame> {
|
||||
ef.source.frame_idx = d.contains("frame_idx") ? nb::cast<int64_t>(d["frame_idx"]) : -1;
|
||||
ef.source.eof = d.contains("eof") ? nb::cast<bool>(d["eof"]) : false;
|
||||
ef.source.is_cut = d.contains("is_cut") ? nb::cast<bool>(d["is_cut"]) : false;
|
||||
ef.source.is_scene_boundary = d.contains("is_scene_boundary")
|
||||
? nb::cast<bool>(d["is_scene_boundary"]) : false;
|
||||
if (ef.source.eof) return ef;
|
||||
|
||||
// faces: (N,4) bbox, (N,10) landmarks, (N,) confidence, (N,512) embeddings
|
||||
@@ -254,6 +256,23 @@ static Config config_from_dict(nb::dict d) {
|
||||
geti("evidence_max_views", cfg.evidence_max_views);
|
||||
// gallery expansion (usually off for sweeps; expose so it can be toggled)
|
||||
if (d.contains("expand_gallery")) cfg.expand_gallery = nb::cast<bool>(d["expand_gallery"]);
|
||||
// AR-018: banded admission bounds for the per-film annex, in probability
|
||||
// space. Reachable from a sweep — the config comment asks for both to be
|
||||
// swept, and they are ignored unless expand_gallery is on. See track_gallery.hpp.
|
||||
getf("expand_band_lo", cfg.expand_band_lo);
|
||||
getf("expand_band_hi", cfg.expand_band_hi);
|
||||
// Presence derivation. Accepts a string ("flood"/"track_extent") or a
|
||||
// number (DE only produces floats: >=0.5 → flood) so the sweep can toggle
|
||||
// it as a sixth knob. flood snaps to boundaries in the replayed frames
|
||||
// (is_scene_boundary if present, else is_cut).
|
||||
if (d.contains("presence_mode")) {
|
||||
const auto& pm = d["presence_mode"];
|
||||
bool flood = false;
|
||||
if (nb::isinstance<nb::str>(pm)) flood = (nb::cast<std::string>(pm) == "flood");
|
||||
else flood = (nb::cast<double>(pm) >= 0.5);
|
||||
cfg.presence_mode = flood ? PresenceMode::flood : PresenceMode::track_extent;
|
||||
}
|
||||
|
||||
/// TRACES: GR-004 | SR-001
|
||||
if (d.contains("require_gallery_stamp"))
|
||||
cfg.require_gallery_stamp = nb::cast<bool>(d["require_gallery_stamp"]);
|
||||
|
||||
@@ -208,6 +208,7 @@ static Config parse_args(int argc, char** argv) {
|
||||
else if (arg("--start")) cfg.start_sec = std::stod(next());
|
||||
else if (arg("--end")) cfg.end_sec = std::stod(next());
|
||||
else if (arg("--cut-threshold")) cfg.cut_threshold = std::stof(next());
|
||||
else if (arg("--presence-mode")) { std::string m = next(); cfg.presence_mode = (m == "flood") ? PresenceMode::flood : PresenceMode::track_extent; }
|
||||
else if (arg("--scene-detect")) cfg.scene_detect = true;
|
||||
else if (arg("--scene-detector")) cfg.scene_model = next();
|
||||
else if (arg("--scene-detector-engine")) cfg.scene_engine = next();
|
||||
|
||||
@@ -38,6 +38,11 @@ struct FrameAnnotationFunc {
|
||||
|
||||
SceneAnnotation operator()(MatchedSceneFrame mf) {
|
||||
if (mf.source.eof) return {0.0, {}, /*eof=*/true};
|
||||
return {mf.source.timestamp_sec, std::move(mf.actors)};
|
||||
SceneAnnotation sa;
|
||||
sa.timestamp_sec = mf.source.timestamp_sec;
|
||||
sa.visible_actors = std::move(mf.actors);
|
||||
sa.is_cut = mf.source.is_cut;
|
||||
sa.is_scene_boundary = mf.source.is_scene_boundary;
|
||||
return sa;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -184,6 +184,20 @@ private:
|
||||
aw.scenes.push_back({c.first_seen, c.last_seen, c.belief, c.route});
|
||||
}
|
||||
|
||||
// Flood-fill: snap each claim to the shot it sits in, so an actor seen
|
||||
// once in a scene is reported across the whole scene. Bounded by real
|
||||
// TransNetV2 boundaries — a window never crosses one — and a no-op when
|
||||
// scene detection found no boundaries (nothing to snap to).
|
||||
if (cfg_.presence_mode == PresenceMode::flood) {
|
||||
const std::vector<double> bounds = scene_boundaries();
|
||||
if (!bounds.empty())
|
||||
for (auto& [idx, aw] : by_actor)
|
||||
for (auto& w : aw.scenes) {
|
||||
w.start = boundary_at_or_before(bounds, w.start);
|
||||
w.end = boundary_after(bounds, w.end);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<ActorWindow> result;
|
||||
for (auto& [idx, aw] : by_actor) {
|
||||
std::sort(aw.scenes.begin(), aw.scenes.end(),
|
||||
@@ -193,6 +207,45 @@ private:
|
||||
return result;
|
||||
}
|
||||
|
||||
// Sorted, de-duplicated boundary timestamps seen this run, framed by the
|
||||
// film's own extent so the first and last shots are closed intervals. Derived
|
||||
// from frames_ rather than a separate accumulator: the frames are already
|
||||
// retained and this runs once.
|
||||
//
|
||||
// Prefers TransNetV2 shot boundaries (is_scene_boundary) when a scene detector
|
||||
// populated them; otherwise falls back to the always-on histogram cuts
|
||||
// (is_cut, camera_position_change_detector). On this ROCm box the scene
|
||||
// detector cannot run in-process (see the dumper note), so is_cut is what
|
||||
// flood-fill actually snaps to — coarser than true shot boundaries (cuts also
|
||||
// fire on in-shot angle changes) but present with no extra pass.
|
||||
std::vector<double> scene_boundaries() const {
|
||||
bool have_scene = false;
|
||||
for (const auto& sa : frames_)
|
||||
if (sa.is_scene_boundary) { have_scene = true; break; }
|
||||
|
||||
std::vector<double> b;
|
||||
b.push_back(0.0);
|
||||
for (const auto& sa : frames_) {
|
||||
const bool boundary = have_scene ? sa.is_scene_boundary : sa.is_cut;
|
||||
if (boundary) b.push_back(sa.timestamp_sec);
|
||||
}
|
||||
b.push_back(last_ts_ + 1.0); // a right edge past the final sample
|
||||
std::sort(b.begin(), b.end());
|
||||
b.erase(std::unique(b.begin(), b.end()), b.end());
|
||||
return b;
|
||||
}
|
||||
|
||||
// The boundary opening the shot that contains t (largest boundary ≤ t).
|
||||
static double boundary_at_or_before(const std::vector<double>& b, double t) {
|
||||
auto it = std::upper_bound(b.begin(), b.end(), t);
|
||||
return (it == b.begin()) ? b.front() : *(it - 1);
|
||||
}
|
||||
// The boundary closing the shot that contains t (smallest boundary > t).
|
||||
static double boundary_after(const std::vector<double>& b, double t) {
|
||||
auto it = std::upper_bound(b.begin(), b.end(), t);
|
||||
return (it == b.end()) ? b.back() : *it;
|
||||
}
|
||||
|
||||
json build_epochs() {
|
||||
json actors = json::array();
|
||||
for (const auto& aw : build_actor_windows()) {
|
||||
|
||||
+31
-15
@@ -181,29 +181,45 @@ public:
|
||||
/// alone. There is no separate revival path (AR-008).
|
||||
///
|
||||
/// TRACES: AR-008, AR-013 | SR-002
|
||||
/// Filtered on the TRACKER's clock, deliberately, while reaping runs on
|
||||
/// the matcher's evidence watermark. The two answer different questions
|
||||
/// and must not share an answer:
|
||||
/// Association and reaping share ONE clock — the evidence watermark when a
|
||||
/// matcher is attached, the tracker clock otherwise (they coincide when
|
||||
/// there is only one). `candidates()` and `reap_locked()` apply the SAME
|
||||
/// `track_extinction_sec` horizon against that clock, so the offered pool
|
||||
/// and the live pool are the same set:
|
||||
///
|
||||
/// "may this detection link to that track?" — a tracking question,
|
||||
/// asked now, about a box observed `track_extinction_sec` ago.
|
||||
/// "is that track finished, so its claim can be emitted?" — a presence
|
||||
/// question, which cannot be answered until every vote is in.
|
||||
/// offered ⟺ (clock - last_seen) ≤ track_extinction_sec
|
||||
/// reaped/erased ⟺ (clock - last_seen) > track_extinction_sec
|
||||
///
|
||||
/// Conflating them makes the result depend on node speed in one
|
||||
/// direction or the other. Reaping on the tracker's clock closed tracks
|
||||
/// before their votes arrived. Deferring association to the evidence
|
||||
/// clock — which is what deferring the erase alone did — left retired
|
||||
/// tracks in the pool for as long as the matcher lagged, so a new face
|
||||
/// This closes two symmetric failures. (1) Offering on the tracker's clock
|
||||
/// (ahead of the watermark) let a face associate onto a track the registry
|
||||
/// had ALREADY reaped on the watermark; the vote then landed on a dead id
|
||||
/// and was dropped (record_vote → dropped_votes_). Rare live (small lag),
|
||||
/// but replay runs the tracker far ahead of the matcher and lost ~0.3% of
|
||||
/// votes. (2) Historically, offering on a LOOSER horizon than the reap left
|
||||
/// retired tracks in the pool while the matcher lagged, so a new face
|
||||
/// re-associated onto a long-dead track and two people merged into one
|
||||
/// window. Measured: 5 actors / 16 windows at channel depth 32 against
|
||||
/// 3 actors / 5 windows at depth 10322, from identical input.
|
||||
/// window (measured: 5 actors/16 windows at depth 32 vs 3/5 at depth 10322).
|
||||
/// A single clock and a single threshold make both impossible: nothing is
|
||||
/// offered past its reap horizon, nothing is reaped while still offerable.
|
||||
std::vector<Track*> candidates() {
|
||||
std::vector<Track*> out;
|
||||
out.reserve(reg_.tracks_.size());
|
||||
// Filter association on the SAME clock reaping uses (the evidence
|
||||
// watermark when a matcher is attached, else the tracker clock). The
|
||||
// two used to differ deliberately — the tracker offered on now_ while
|
||||
// the registry reaped on evidence_through_ — but that let the tracker
|
||||
// associate a face onto a track the registry had already reaped on the
|
||||
// watermark, whose vote then landed on a dead id and was dropped
|
||||
// (record_vote → dropped_votes_). In the live pipeline the lag is tiny
|
||||
// so it rarely bit; in replay the Python source runs the tracker far
|
||||
// ahead of the matcher and ~0.3% of votes were lost. One clock for both
|
||||
// "may this associate?" and "is this reaped?" closes the race: a track
|
||||
// past the horizon is neither offered nor reaped-out-from-under a vote.
|
||||
const double clock =
|
||||
reg_.awaits_evidence_ ? reg_.evidence_through_ : reg_.now_;
|
||||
for (auto& [id, t] : reg_.tracks_) {
|
||||
if (t.last_seen &&
|
||||
(reg_.now_ - *t.last_seen) > reg_.cfg_.track_extinction_sec)
|
||||
(clock - *t.last_seen) > reg_.cfg_.track_extinction_sec)
|
||||
continue; // retired from association; still awaiting evidence
|
||||
out.push_back(&t);
|
||||
}
|
||||
|
||||
@@ -155,6 +155,13 @@ struct SceneAnnotation {
|
||||
double timestamp_sec{0.0};
|
||||
std::vector<IdentifiedActor> visible_actors;
|
||||
bool eof{false};
|
||||
// Carried through from Frame so the sink can collect boundaries for flood-fill
|
||||
// presence (PresenceMode::flood). is_cut is the always-on histogram cut
|
||||
// (camera_position_change_detector) — the boundary flood-fill uses by default.
|
||||
// is_scene_boundary is the opt-in TransNetV2 shot boundary (0 unless scene
|
||||
// detection ran); kept for a future out-of-process scene detector.
|
||||
bool is_cut{false};
|
||||
bool is_scene_boundary{false};
|
||||
};
|
||||
|
||||
// ── Actor gallery ─────────────────────────────────────────────────────────────
|
||||
|
||||
Reference in New Issue
Block a user