dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: no fixed cap on faces per frame

Browse Source 
AR-003 — max_faces defaults to 0, meaning no cap. A fixed cap discards the
SMALLEST faces first, which are exactly the background cast X-Ray still credits
with scene membership, so the pipeline was systematically losing the people it
is supposed to find in crowded scenes.

This is only safe now that AR-004 landed. Previously an uncapped frame would
have pushed more work into channels that dropped on overflow, trading a visible
cap for silent loss. With backpressure the producer slows instead, so per-frame
cost is contained rather than discarded.

The matcher's kMaxFaces used to throw above 32, which made it an accidental
second cap. It sizes the similarity engine's preallocated buffer, so it bounds
memory rather than face count — the frame is now scored in batches of that size.
Memory stays bounded; faces do not.

Largest-first ordering is kept even without the cap, and the comment now says
why: the Hungarian solver tie-breaks on index order, so that ordering is
load-bearing for the replay determinism test rather than a leftover of the cap.

Verified end to end on a real clip: identical output to the capped run (385
frames, 693 faces), which is expected since that footage peaks at 4 faces per
frame — the point is the absence of a regression. The committed fixtures remain
byte-identical and valid.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

TRACES: AR-003 | SR-002
This commit is contained in:
dtourolle
2026-07-31 14:22:20 +02:00
co-authored by Claude Opus 5
parent 036f44fbdd
commit dfb8f5801e
8 changed files with 420 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/embed_faces.cpp
+154 -17
View File
@@ -25,11 +25,25 @@
// expected to contain exactly one subject). A warning is printed to stderr
// when more than one face is found.
//
// --all-faces emits every detection instead, which is what a caller analysing
// a frame rather than a gallery portrait needs:
// [ { "image": "frame.png",
// "faces": [ {"bbox": [...], "landmarks": [[x,y] x5],
// "confidence": 0.89, "embedding": [...]} , ... ] } ]
//
// --calibration <gallery> additionally emits the gallery's fitted Platt
// sigmoid, so a non-Python client can turn a similarity into P(match) with the
// same parameters the C++ matcher uses. Output becomes
// {"calibration": {...}, "images": [...]}. Clients must score through it:
// AR-024 requires the calibrated probability, never a bare cosine — a raw
// threshold means something different for every model, gallery and face size.
//
// This binary is intentionally a thin wrapper around the same ONNX models
// used by scene_analyze, so embeddings are guaranteed compatible.
#include "config.hpp"
#include "face_utils.hpp"
#include "gallery/gallery_store.hpp"
#include "inference/face_detector.hpp"
#include "inference/face_embedder.hpp"
@@ -100,6 +114,77 @@ static void save_debug(const std::string& dir,
cv::imwrite(dir + "/" + stem + "_aligned.jpg", aligned);
}
// ── Process one image, keeping every detection ────────────────────────────────
// The --all-faces path. Same detect → align → embed chain as process() below,
// but without the highest-confidence reduction: a frame legitimately contains
// several people, and dropping all but one is a gallery-portrait assumption.
// Faces that fail alignment are reported with a null embedding rather than
// silently dropped, so a caller can count what the detector found against what
// survived the ArcFace warp.
struct MultiFaceResult {
std::string image_path;
std::string error; // set only when the image itself failed
std::vector<FaceResult> faces;
};
static MultiFaceResult process_all(
const std::string& path,
const std::function<std::vector<DetectedFace>(const cv::Mat&)>& detect,
const std::function<Embedding(const cv::Mat&)>& embed_one,
int max_side,
const std::string& debug_dir = "") {
MultiFaceResult out;
out.image_path = path;
cv::Mat img = cv::imread(path);
if (img.empty()) {
out.error = "cannot read image";
return out;
}
if (max_side > 0) {
const int big = std::max(img.cols, img.rows);
if (big > max_side) {
const double s = static_cast<double>(max_side) / big;
cv::resize(img, img, {}, s, s, cv::INTER_AREA);
}
}
std::vector<DetectedFace> faces = detect(img);
if (faces.empty()) {
cv::Mat enhanced = enhance_for_retry(img);
faces = detect(enhanced);
if (!faces.empty())
img = enhanced;
}
if (faces.empty()) {
out.error = "no face detected";
return out;
}
for (const auto& face : faces) {
FaceResult r;
r.image_path = path;
r.confidence = face.confidence;
r.bbox[0] = face.bbox.x; r.bbox[1] = face.bbox.y;
r.bbox[2] = face.bbox.width; r.bbox[3] = face.bbox.height;
r.landmarks = face.landmarks;
cv::Mat crop = align_face(img, face.landmarks);
if (crop.empty()) {
r.error = "alignment failed";
} else {
r.ok = true;
r.embedding = embed_one(crop);
if (!debug_dir.empty())
save_debug(debug_dir, path, img, face, crop);
}
out.faces.push_back(std::move(r));
}
return out;
}
// ── Process one image ─────────────────────────────────────────────────────────
static FaceResult process(const std::string& path,
@@ -177,6 +262,8 @@ int main(int argc, char** argv) {
std::string arcface_model = kDefaultArcfaceModel;
std::string arcface_engine;
std::string debug_dir;
std::string calibration_gallery;
bool all_faces = false;
float conf = 0.5f, nms = 0.4f;
int max_side = 500;
std::vector<std::string> images;
@@ -190,13 +277,16 @@ int main(int argc, char** argv) {
else if (std::strcmp(argv[i], "--nms") == 0 && i+1 < argc) { nms = std::stof(argv[++i]); }
else if (std::strcmp(argv[i], "--save-debug") == 0 && i+1 < argc) { debug_dir = argv[++i]; }
else if (std::strcmp(argv[i], "--max-side") == 0 && i+1 < argc) { max_side = std::stoi(argv[++i]); }
else if (std::strcmp(argv[i], "--calibration")== 0 && i+1 < argc) { calibration_gallery = argv[++i]; }
else if (std::strcmp(argv[i], "--all-faces") == 0) { all_faces = true; }
else if (argv[i][0] != '-') { images.push_back(argv[i]); }
else { std::cerr << "[warn] unknown flag: " << argv[i] << "\n"; }
}
if (images.empty()) {
std::cerr << "Usage: embed_faces [--detector <path>] [--arcface <path>] "
"[--save-debug <dir>] [--max-side <N>] image1.jpg ...\n";
"[--save-debug <dir>] [--max-side <N>] [--all-faces] "
"[--calibration <gallery>] image1.jpg ...\n";
return 1;
}
@@ -217,31 +307,78 @@ int main(int argc, char** argv) {
std::function<Embedding(const cv::Mat&)> embed_one =
[&](const cv::Mat& c) { return embedder->embed_one(c); };
// One face's fields, shared by both output shapes.
auto face_json = [](const FaceResult& r) {
json f;
f["confidence"] = r.confidence;
f["bbox"] = {r.bbox[0], r.bbox[1], r.bbox[2], r.bbox[3]};
json lms = json::array();
for (const auto& pt : r.landmarks) lms.push_back({pt.x, pt.y});
f["landmarks"] = std::move(lms);
if (r.ok) f["embedding"] = std::vector<float>(r.embedding.begin(),
r.embedding.end());
else { f["embedding"] = nullptr; f["error"] = r.error; }
return f;
};
// Process images and build JSON output
json output = json::array();
json images_out = json::array();
for (const auto& path : images) {
std::cerr << "[embed_faces] " << path << "\n";
FaceResult res = process(path, detect, embed_one, max_side, debug_dir);
json entry;
entry["image"] = res.image_path;
if (res.ok) {
entry["embedding"] = std::vector<float>(res.embedding.begin(),
res.embedding.end());
entry["confidence"] = res.confidence;
entry["bbox"] = {res.bbox[0], res.bbox[1], res.bbox[2], res.bbox[3]};
json lms = json::array();
for (const auto& pt : res.landmarks) lms.push_back({pt.x, pt.y});
entry["landmarks"] = std::move(lms);
entry["image"] = path;
if (all_faces) {
MultiFaceResult res = process_all(path, detect, embed_one, max_side, debug_dir);
if (!res.error.empty()) {
entry["faces"] = json::array();
entry["error"] = res.error;
std::cerr << " [skip] " << res.error << "\n";
} else {
json faces = json::array();
for (const auto& f : res.faces) faces.push_back(face_json(f));
entry["faces"] = std::move(faces);
}
} else {
entry["embedding"] = nullptr;
entry["error"] = res.error;
std::cerr << " [skip] " << res.error << "\n";
FaceResult res = process(path, detect, embed_one, max_side, debug_dir);
if (res.ok) {
entry.merge_patch(face_json(res));
} else {
entry["embedding"] = nullptr;
entry["error"] = res.error;
std::cerr << " [skip] " << res.error << "\n";
}
}
output.push_back(std::move(entry));
images_out.push_back(std::move(entry));
}
std::cout << output.dump() << "\n";
// Without --calibration the output stays a bare array, unchanged, so
// existing callers (build_gallery, fetch_missing_actors) are unaffected.
if (calibration_gallery.empty()) {
std::cout << images_out.dump() << "\n";
return 0;
}
ActorGallery gallery = load_gallery(calibration_gallery);
if (!gallery.calib_valid)
std::cerr << "[warn] " << calibration_gallery
<< " carries no valid calibration; a client cannot convert a "
"similarity to a probability from it (AR-024)\n";
json out;
out["calibration"] = {
{"a", gallery.calib_a},
{"b", gallery.calib_b},
{"valid", gallery.calib_valid},
{"form", "P(match) = 1/(1+exp(-(a*similarity + b + log_prior_odds)))"},
{"note", "Score through this. AR-024: a bare cosine threshold means "
"something different for every model, gallery and face size. "
"log_prior_odds = log(p0/(1-p0)) for a base-rate prior p0; use "
"0 for association (are these two faces one person)."},
};
out["images"] = std::move(images_out);
std::cout << out.dump() << "\n";
return 0;
}