dtourolle/scene-actor-extraction
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
34b1f2c58ff0cdf92411a8035de80d3f92ebf18a
scene-actor-extraction/lvface-deep-dive/index.html
T
dtourolle 34b1f2c58f docs: deploy from 0bd2747
2026-07-21 08:56:42 +02:00

1479 lines
45 KiB
HTML
Raw Blame History

<!doctype html>
<html lang="en" class="no-js">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<meta name="description" content="Face-recognition pipeline for finding on-screen actor presence in film/TV, built on KPN++">
<link rel="canonical" href="https://pages.tourolle.paris/dtourolle/scene-actor-extraction/lvface-deep-dive/">
<link rel="prev" href="../pose-expansion/">
<link rel="next" href="../model-bakeoff/">
<link rel="icon" href="../assets/images/favicon.png">
<meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.7">
<title>LVFace Deep Dive - scene-actor-extraction</title>
<link rel="stylesheet" href="../assets/stylesheets/main.ec1eaa64.min.css">
<link rel="stylesheet" href="../assets/stylesheets/palette.ab4e12ef.min.css">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
<style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>
<link rel="stylesheet" href="../stylesheets/extra.css">
<script>__md_scope=new URL("..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>
</head>
<body dir="ltr" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="amber">
<input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
<input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
<label class="md-overlay" for="__drawer"></label>
<div data-md-component="skip">
<a href="#deep-dive-lvface-b-glint360k" class="md-skip">
Skip to content
</a>
</div>
<div data-md-component="announce">
</div>
<header class="md-header" data-md-component="header">
<nav class="md-header__inner md-grid" aria-label="Header">
<a href=".." title="scene-actor-extraction" class="md-header__button md-logo" aria-label="scene-actor-extraction" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
<label class="md-header__button md-icon" for="__drawer">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
</label>
<div class="md-header__title" data-md-component="header-title">
<div class="md-header__ellipsis">
<div class="md-header__topic">
<span class="md-ellipsis">
scene-actor-extraction
</span>
</div>
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
LVFace Deep Dive
</span>
</div>
</div>
</div>
<form class="md-header__option" data-md-component="palette">
<input class="md-option" data-md-color-media="(prefers-color-scheme: dark)" data-md-color-scheme="slate" data-md-color-primary="black" data-md-color-accent="amber" aria-label="Switch to light mode" type="radio" name="__palette" id="__palette_0">
<label class="md-header__button md-icon" title="Switch to light mode" for="__palette_1" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 7a5 5 0 0 1 5 5 5 5 0 0 1-5 5 5 5 0 0 1-5-5 5 5 0 0 1 5-5m0 2a3 3 0 0 0-3 3 3 3 0 0 0 3 3 3 3 0 0 0 3-3 3 3 0 0 0-3-3m0-7 2.39 3.42C13.65 5.15 12.84 5 12 5s-1.65.15-2.39.42zM3.34 7l4.16-.35A7.2 7.2 0 0 0 5.94 8.5c-.44.74-.69 1.5-.83 2.29zm.02 10 1.76-3.77a7.131 7.131 0 0 0 2.38 4.14zM20.65 7l-1.77 3.79a7.02 7.02 0 0 0-2.38-4.15zm-.01 10-4.14.36c.59-.51 1.12-1.14 1.54-1.86.42-.73.69-1.5.83-2.29zM12 22l-2.41-3.44c.74.27 1.55.44 2.41.44.82 0 1.63-.17 2.37-.44z"/></svg>
</label>
<input class="md-option" data-md-color-media="(prefers-color-scheme: light)" data-md-color-scheme="default" data-md-color-primary="black" data-md-color-accent="indigo" aria-label="Switch to dark mode" type="radio" name="__palette" id="__palette_1">
<label class="md-header__button md-icon" title="Switch to dark mode" for="__palette_0" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="m17.75 4.09-2.53 1.94.91 3.06-2.63-1.81-2.63 1.81.91-3.06-2.53-1.94L12.44 4l1.06-3 1.06 3zm3.5 6.91-1.64 1.25.59 1.98-1.7-1.17-1.7 1.17.59-1.98L15.75 11l2.06-.05L18.5 9l.69 1.95zm-2.28 4.95c.83-.08 1.72 1.1 1.19 1.85-.32.45-.66.87-1.08 1.27C15.17 23 8.84 23 4.94 19.07c-3.91-3.9-3.91-10.24 0-14.14.4-.4.82-.76 1.27-1.08.75-.53 1.93.36 1.85 1.19-.27 2.86.69 5.83 2.89 8.02a9.96 9.96 0 0 0 8.02 2.89m-1.64 2.02a12.08 12.08 0 0 1-7.8-3.47c-2.17-2.19-3.33-5-3.49-7.82-2.81 3.14-2.7 7.96.31 10.98 3.02 3.01 7.84 3.12 10.98.31"/></svg>
</label>
</form>
<script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>
<label class="md-header__button md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
</label>
<div class="md-search" data-md-component="search" role="dialog">
<label class="md-search__overlay" for="__search"></label>
<div class="md-search__inner" role="search">
<form class="md-search__form" name="search">
<input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
<label class="md-search__icon md-icon" for="__search">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</label>
<nav class="md-search__options" aria-label="Search">
<button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
</button>
</nav>
</form>
<div class="md-search__output">
<div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
<div class="md-search-result" data-md-component="search-result">
<div class="md-search-result__meta">
Initializing search
</div>
<ol class="md-search-result__list" role="presentation"></ol>
</div>
</div>
</div>
</div>
</div>
<div class="md-header__source">
<a href="https://gitea.tourolle.paris/dtourolle/scene-actor-extraction" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
dtourolle/scene-actor-extraction
</div>
</a>
</div>
</nav>
</header>
<div class="md-container" data-md-component="container">
<nav class="md-tabs" aria-label="Tabs" data-md-component="tabs">
<div class="md-grid">
<ul class="md-tabs__list">
<li class="md-tabs__item">
<a href=".." class="md-tabs__link">
Home
</a>
</li>
<li class="md-tabs__item">
<a href="../methodology/" class="md-tabs__link">
How We Score Against X-Ray
</a>
</li>
<li class="md-tabs__item md-tabs__item--active">
<a href="../best-model/" class="md-tabs__link">
Findings
</a>
</li>
<li class="md-tabs__item">
<a href="../model-bakeoff/" class="md-tabs__link">
Full Experiment Log
</a>
</li>
<li class="md-tabs__item">
<a href="../service-conversion/" class="md-tabs__link">
Service Conversion (proposal)
</a>
</li>
</ul>
</div>
</nav>
<main class="md-main" data-md-component="main">
<div class="md-main__inner md-grid">
<div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--primary md-nav--lifted" aria-label="Navigation" data-md-level="0">
<label class="md-nav__title" for="__drawer">
<a href=".." title="scene-actor-extraction" class="md-nav__button md-logo" aria-label="scene-actor-extraction" data-md-component="logo">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>
</a>
scene-actor-extraction
</label>
<div class="md-nav__source">
<a href="https://gitea.tourolle.paris/dtourolle/scene-actor-extraction" title="Go to repository" class="md-source" data-md-component="source">
<div class="md-source__icon md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><!--! Font Awesome Free 7.1.0 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2025 Fonticons, Inc.--><path d="M439.6 236.1 244 40.5c-5.4-5.5-12.8-8.5-20.4-8.5s-15 3-20.4 8.4L162.5 81l51.5 51.5c27.1-9.1 52.7 16.8 43.4 43.7l49.7 49.7c34.2-11.8 61.2 31 35.5 56.7-26.5 26.5-70.2-2.9-56-37.3L240.3 199v121.9c25.3 12.5 22.3 41.8 9.1 55-6.4 6.4-15.2 10.1-24.3 10.1s-17.8-3.6-24.3-10.1c-17.6-17.6-11.1-46.9 11.2-56v-123c-20.8-8.5-24.6-30.7-18.6-45L142.6 101 8.5 235.1C3 240.6 0 247.9 0 255.5s3 15 8.5 20.4l195.6 195.7c5.4 5.4 12.7 8.4 20.4 8.4s15-3 20.4-8.4l194.7-194.7c5.4-5.4 8.4-12.8 8.4-20.4s-3-15-8.4-20.4"/></svg>
</div>
<div class="md-source__repository">
dtourolle/scene-actor-extraction
</div>
</a>
</div>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href=".." class="md-nav__link">
<span class="md-ellipsis">
Home
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../methodology/" class="md-nav__link">
<span class="md-ellipsis">
How We Score Against X-Ray
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active md-nav__item--section md-nav__item--nested">
<input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" checked>
<label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="">
<span class="md-ellipsis">
Findings
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="true">
<label class="md-nav__title" for="__nav_3">
<span class="md-nav__icon md-icon"></span>
Findings
</label>
<ul class="md-nav__list" data-md-scrollfix>
<li class="md-nav__item">
<a href="../best-model/" class="md-nav__link">
<span class="md-ellipsis">
Best Model
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../gallery-scope/" class="md-nav__link">
<span class="md-ellipsis">
Gallery Scope (Full vs. Limited)
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../pose-expansion/" class="md-nav__link">
<span class="md-ellipsis">
Pose Expansion
</span>
</a>
</li>
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">
<label class="md-nav__link md-nav__link--active" for="__toc">
<span class="md-ellipsis">
LVFace Deep Dive
</span>
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
<span class="md-ellipsis">
LVFace Deep Dive
</span>
</a>
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#baseline-correctly-scored-seconds" class="md-nav__link">
<span class="md-ellipsis">
Baseline: correctly scored seconds
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#training-vs-held-out-the-generalization-gap" class="md-nav__link">
<span class="md-ellipsis">
Training vs. held-out: the generalization gap
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#mechanism-1-extinction-bridging" class="md-nav__link">
<span class="md-ellipsis">
Mechanism 1: extinction bridging
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#mechanism-2-the-face-vs-presence-ceiling" class="md-nav__link">
<span class="md-ellipsis">
Mechanism 2: the face-vs-presence ceiling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#every-distinct-out-of-cast-name" class="md-nav__link">
<span class="md-ellipsis">
Every distinct out-of-cast name
</span>
</a>
<nav class="md-nav" aria-label="Every distinct out-of-cast name">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#the-many-saints-of-newark-4-names" class="md-nav__link">
<span class="md-ellipsis">
The Many Saints of Newark: 4 names
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#lord-of-war-3-names" class="md-nav__link">
<span class="md-ellipsis">
Lord of War: 3 names
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#lovelace-1-name" class="md-nav__link">
<span class="md-ellipsis">
Lovelace: 1 name
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#scarface-1-name" class="md-nav__link">
<span class="md-ellipsis">
Scarface: 1 name
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#summary-of-the-nine" class="md-nav__link">
<span class="md-ellipsis">
Summary of the nine
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#where-lvface-beat-x-ray" class="md-nav__link">
<span class="md-ellipsis">
Where LVFace beat X-Ray
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#summary" class="md-nav__link">
<span class="md-ellipsis">
Summary
</span>
</a>
</li>
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="../model-bakeoff/" class="md-nav__link">
<span class="md-ellipsis">
Full Experiment Log
</span>
</a>
</li>
<li class="md-nav__item">
<a href="../service-conversion/" class="md-nav__link">
<span class="md-ellipsis">
Service Conversion (proposal)
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
<div class="md-sidebar__scrollwrap">
<div class="md-sidebar__inner">
<nav class="md-nav md-nav--secondary" aria-label="Table of contents">
<label class="md-nav__title" for="__toc">
<span class="md-nav__icon md-icon"></span>
Table of contents
</label>
<ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>
<li class="md-nav__item">
<a href="#baseline-correctly-scored-seconds" class="md-nav__link">
<span class="md-ellipsis">
Baseline: correctly scored seconds
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#training-vs-held-out-the-generalization-gap" class="md-nav__link">
<span class="md-ellipsis">
Training vs. held-out: the generalization gap
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#mechanism-1-extinction-bridging" class="md-nav__link">
<span class="md-ellipsis">
Mechanism 1: extinction bridging
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#mechanism-2-the-face-vs-presence-ceiling" class="md-nav__link">
<span class="md-ellipsis">
Mechanism 2: the face-vs-presence ceiling
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#every-distinct-out-of-cast-name" class="md-nav__link">
<span class="md-ellipsis">
Every distinct out-of-cast name
</span>
</a>
<nav class="md-nav" aria-label="Every distinct out-of-cast name">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#the-many-saints-of-newark-4-names" class="md-nav__link">
<span class="md-ellipsis">
The Many Saints of Newark: 4 names
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#lord-of-war-3-names" class="md-nav__link">
<span class="md-ellipsis">
Lord of War: 3 names
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#lovelace-1-name" class="md-nav__link">
<span class="md-ellipsis">
Lovelace: 1 name
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#scarface-1-name" class="md-nav__link">
<span class="md-ellipsis">
Scarface: 1 name
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#summary-of-the-nine" class="md-nav__link">
<span class="md-ellipsis">
Summary of the nine
</span>
</a>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#where-lvface-beat-x-ray" class="md-nav__link">
<span class="md-ellipsis">
Where LVFace beat X-Ray
</span>
</a>
</li>
<li class="md-nav__item">
<a href="#summary" class="md-nav__link">
<span class="md-ellipsis">
Summary
</span>
</a>
</li>
</ul>
</nav>
</div>
</div>
</div>
<div class="md-content" data-md-component="content">
<article class="md-content__inner md-typeset">
<h1 id="deep-dive-lvface-b-glint360k">Deep dive: LVFace-B Glint360K<a class="headerlink" href="#deep-dive-lvface-b-glint360k" title="Permanent link">&para;</a></h1>
<p>LVFace won the model comparison (see <a href="../best-model/">Which model is best?</a>)
and is the shipped default embedder. This page reports how it performs in
detail: a baseline of correct output, the two mechanisms behind its errors,
and every distinct case where it names someone who is not in the film's
credited cast.</p>
<p>Read <a href="../methodology/">How we score against X-Ray</a> first. X-Ray's ground truth
is scene-level, not per-frame. A name marked correct in the Offscreen column
below is the pipeline correctly reporting scene membership, not a workaround.</p>
<div class="admonition note">
<p class="admonition-title">How to read the frames on this page</p>
<p>The top of each image is the film frame, with a box and name on every
face the pipeline matched to a real detection. The panels below are the
per-second result against X-Ray. <strong>Onscreen</strong> lists names attached to a
visible face this second. <strong>Offscreen</strong> lists names the pipeline reports
present without a currently visible face. Colors mark the verdict:
<span style="color:#0ca30c"><strong>green</strong></span> correct (TPI),
<span style="color:#eb6834"><strong>orange</strong></span> wrong (FPI),
<span style="color:#3987e5"><strong>blue</strong></span> missed (FN).</p>
</div>
<h2 id="baseline-correctly-scored-seconds">Baseline: correctly scored seconds<a class="headerlink" href="#baseline-correctly-scored-seconds" title="Permanent link">&para;</a></h2>
<p><img alt="Wedding couple correctly identified, Downton Abbey: A New Era" src="../assets/images/downton_wedding_couple.jpg" /></p>
<p>Six faces on screen, all six named correctly, including Penelope Wilton at
the edge of the pews and a partly occluded Michelle Dockery. Thirteen more
cast members X-Ray lists as present in the scene are correctly reported
Offscreen. One miss: Maggie Smith (blue). Score for this second: 0.86.</p>
<p><img alt="19 of 20 correct in the funeral crowd" src="../assets/images/downton_funeral_19of20.jpg" /></p>
<p>The same film's funeral scene: dark clothing, hats, half the faces turned
away. Nineteen of the twenty cast members X-Ray lists for this scene score
correct: seven named on screen at up to 100% confidence, twelve more reported
correctly as present but not visible.</p>
<p><img alt="Herbie Hancock identified on an in-fiction video call" src="../assets/images/valerian_screen_call.jpg" /></p>
<p>The pipeline does not require a live face. This is Herbie Hancock at 98%
confidence, identified from a face displayed on a screen inside the film, on
a video call under a science-fiction HUD overlay.</p>
<h2 id="training-vs-held-out-the-generalization-gap">Training vs. held-out: the generalization gap<a class="headerlink" href="#training-vs-held-out-the-generalization-gap" title="Permanent link">&para;</a></h2>
<p>The shipped config (<code>prob_threshold=0.754</code>, <code>anneal_sec=35.54</code>,
<code>extinction_sec=57.43</code>, <code>expand_gallery=true</code>) was tuned on 4 films. Scored
on the 5 films the optimizer never saw:</p>
<p><img alt="Held-out per-film F1 vs. the training-set fit" src="../assets/images/holdout_f1_by_film.png" /></p>
<table>
<thead>
<tr>
<th>film</th>
<th>F1</th>
<th>P</th>
<th>R</th>
<th>TPI</th>
<th>FPI</th>
<th>misid</th>
<th>FN</th>
</tr>
</thead>
<tbody>
<tr>
<td>Benny &amp; Joon</td>
<td>83.0%</td>
<td>89.1%</td>
<td>77.7%</td>
<td>15125</td>
<td>1846</td>
<td>0</td>
<td>4337</td>
</tr>
<tr>
<td>Lovelace</td>
<td>77.5%</td>
<td>90.3%</td>
<td>67.9%</td>
<td>14990</td>
<td>1085</td>
<td>58</td>
<td>7085</td>
</tr>
<tr>
<td>Valerian and the City of a Thousand Planets</td>
<td>74.1%</td>
<td>97.1%</td>
<td>60.0%</td>
<td>18663</td>
<td>548</td>
<td>0</td>
<td>12467</td>
</tr>
<tr>
<td>Downton Abbey: A New Era</td>
<td>56.2%</td>
<td>97.8%</td>
<td>39.4%</td>
<td>52027</td>
<td>1173</td>
<td>0</td>
<td>80084</td>
</tr>
<tr>
<td>The Many Saints of Newark</td>
<td>46.3%</td>
<td>54.7%</td>
<td>40.1%</td>
<td>15922</td>
<td>4394</td>
<td>974</td>
<td>23791</td>
</tr>
<tr>
<td>macro average</td>
<td>67.4%</td>
<td>85.8%</td>
<td>57.0%</td>
<td></td>
<td></td>
<td></td>
<td></td>
</tr>
</tbody>
</table>
<p>The <code>P</code> column is misID-weighted (each out-of-film name counts 10x in the
denominator; see <a href="../methodology/#precision-recall-and-the-misid-weighting">methodology</a>).
That weighting is why Many Saints reads 54.7% here despite naming mostly real,
present faces: its raw (unweighted) precision is <strong>78.4%</strong>, and the gap is
entirely its 974 misIDs paying the 10x penalty. The three zero-misID films
(Benny &amp; Joon, Downton, Valerian) have identical weighted and raw precision;
Lovelace, with 58 misIDs, sits 3pp below its raw 93.3%.</p>
<p>Held-out F1 is 67.4%, against 75.3% on training, an 8pp drop. The spread
between the best and worst held-out film is 37pp. This is not unique to
LVFace: <a href="../model-bakeoff/#held-out-validation-all-3-models">the full experiment log</a>
shows mbf and r18 with the same shape of spread on the same films, at a
uniformly lower level. Two mechanisms explain the spread. Both are shown
below with frame-level evidence.</p>
<h2 id="mechanism-1-extinction-bridging">Mechanism 1: extinction bridging<a class="headerlink" href="#mechanism-1-extinction-bridging" title="Permanent link">&para;</a></h2>
<p>The extinction window keeps a name reported as present for up to
<code>extinction_sec</code> after its last real detection. This is deliberate: most
gaps in face visibility are short (a turned head, an occlusion, a cut to a
reaction shot), and the window bridges them.</p>
<p><img alt="Two faces on screen, six more correctly bridged" src="../assets/images/lovelace_polygraph_bridged.jpg" /></p>
<p>Lovelace's polygraph scene: only Eric Roberts and Amanda Seyfried have
visible faces. X-Ray lists eight cast members present. All eight score
correct; the other six are reported Offscreen through a stretch where the
camera never shows them. The extinction window is why.</p>
<p>The same mechanism fails at a hard cut into a long stretch with no faces at
all. Downton Abbey's recall (39.4%, the worst of the five held-out films) is
dominated by this failure. It is verified directly against the raw
per-frame stream and the dump's own detection counts, not inferred from the
score. Plotting the dump's per-second <code>face_count</code> (detector output,
independent of the tracker) against what the tracker reports, through
Downton Abbey's hard cut into its closing credits:</p>
<p><img alt="Detector vs. tracker through Downton Abbey's cut to credits" src="../assets/images/downton_ghost_timeline.png" /></p>
<p>From the cut onward the detector reports zero faces for close to a minute.
The tracker continues reporting the previous shot's 15 identities for the
same span (verified for Hugh Bonneville: bbox <code>(1743.2, 0.0, 171.3, 317.8)</code>,
unchanged to the pixel, at every sampled second for 57 seconds). The
staircase at the right edge is the extinction window expiring, actor by
actor. This is <code>SceneTrackerFunc::active_[actor_idx].last_bbox</code>
(<a href="https://gitea.tourolle.paris/dtourolle/scene-actor-extraction/raw/commit/0bd27470698c45cab21935d636a04512360e1008/src/nodes/scene_tracker_node.hpp"><code>src/nodes/scene_tracker_node.hpp</code></a>)
re-emitted as designed. <code>extinction_sec=57.4</code> was tuned long because
bridging is correct on most footage, as in the polygraph scene above. The
training films did not contain a faceless stretch long enough to expose the
cost side; the held-out set did.</p>
<p>The extinction window is a scoring concept, not something drawn on screen.
The shipped output is presence windows with no bounding boxes. Even the
debug overlay used for this report never draws a box for a bridged name: a
name inside its extinction window with no current detection appears only as
a name in the Offscreen column, the same as every correctly bridged name
above.</p>
<p>A related, smaller effect shows up at rapid cuts:</p>
<p><img alt="Two labels on one face after a shot/reverse-shot cut" src="../assets/images/cafe_society_rapid_cut.jpg" /></p>
<p>Café Society (a training film), a shot/reverse-shot dialog. The box on Steve
Carell's face carries two labels: his own, and Jesse Eisenberg's, left over
from the counter-shot a moment earlier. Both names score correct, because
both actors are present in this scene per X-Ray. The box position is
briefly wrong; the presence claim, which is what the pipeline ships, is
right.</p>
<h2 id="mechanism-2-the-face-vs-presence-ceiling">Mechanism 2: the face-vs-presence ceiling<a class="headerlink" href="#mechanism-2-the-face-vs-presence-ceiling" title="Permanent link">&para;</a></h2>
<p>Downton Abbey's recall did not collapse because faces were misread. It
collapsed because for most of its 80084 false-negative seconds there was no
face to read.</p>
<p><img alt="22 cast credited, nobody facing the camera" src="../assets/images/downton_crew_fn.jpg" /></p>
<p>A newsreel crew moves equipment through the hall. X-Ray credits 22 cast
members as present in this scene. None face the camera. Eight still score
correct, carried by presence windows from adjacent shots. The other fourteen
are missed, and no face-recognition system can recover them, because there
is no face in the frame. X-Ray records scene membership; the pipeline
measures visible faces. In ensemble scenes these two quantities diverge, and
that gap accounts for most of the false-negative count.</p>
<h2 id="every-distinct-out-of-cast-name">Every distinct out-of-cast name<a class="headerlink" href="#every-distinct-out-of-cast-name" title="Permanent link">&para;</a></h2>
<p>Many Saints of Newark has the largest misID count of any held-out film: 974
seconds, weighted. Rather than characterize this from a single frame, the
raw replay stream was searched directly for every name the pipeline reports
that is not in the film's credited cast. The same search was run on all 9
films in the benchmark, one rule applied uniformly: <strong>find the first second
each distinct out-of-cast name appears, and render that exact second.</strong></p>
<p>Five films produce no such name anywhere in their runtime: Benny &amp; Joon,
Café Society, Downton Abbey, Sound of Metal, Valerian. Zero out-of-cast
names across their entire length. Four films produce nine distinct names
between them, shown below in full, not a sample.</p>
<h3 id="the-many-saints-of-newark-4-names">The Many Saints of Newark: 4 names<a class="headerlink" href="#the-many-saints-of-newark-4-names" title="Permanent link">&para;</a></h3>
<p><img alt="Germar Terrell Gardner, first out-of-cast name in Many Saints" src="../assets/images/many_saints_fpi_gardner.jpg" /></p>
<p>Germar Terrell Gardner, t=848s, 78% confidence. A real, clearly visible
background actor. He is not in X-Ray's cast list for this film, but he is
credited in Jellyfin's independent cast metadata (see
<a href="#where-lvface-beat-x-ray">Where LVFace beat X-Ray</a> below). This is a
ground-truth gap, not a model error.</p>
<p><img alt="Archie Yates, second out-of-cast name in Many Saints" src="../assets/images/many_saints_fpi_yates.jpg" /></p>
<p>Archie Yates, t=2521s, 78% confidence. A real detected face, a genuine
lookalike confusion.</p>
<p><img alt="Zooey Deschanel, third out-of-cast name in Many Saints" src="../assets/images/many_saints_fpi_deschanel.jpg" /></p>
<p>Zooey Deschanel, t=2819s, 99% confidence. A real detected face at a dinner
table, high-confidence lookalike confusion.</p>
<p><img alt="Talia Balsam, fourth out-of-cast name in Many Saints" src="../assets/images/many_saints_fpi_balsam.jpg" /></p>
<p>Talia Balsam, t=4551s, 93% confidence. A real detected face. Talia Balsam
plays Mrs. Jarecki, a guidance counselor, in this film; she is confirmed
on screen by direct inspection of the frame. She does not appear in X-Ray's
<code>people.csv</code> for this title. This is a second ground-truth gap in the same
film, not a model error.</p>
<p>Two of these four names are ground-truth gaps (Gardner, Balsam), not
misidentifications. The other two (Yates, Deschanel) are genuine embedding
errors on real faces.</p>
<h3 id="lord-of-war-3-names">Lord of War: 3 names<a class="headerlink" href="#lord-of-war-3-names" title="Permanent link">&para;</a></h3>
<p><img alt="David Shumbris, first out-of-cast name in Lord of War" src="../assets/images/lord_of_war_fpi_shumbris.jpg" /></p>
<p>David Shumbris, t=418s, 81% confidence. A real face in a dim, low-detail
shot under a train track. A genuine lookalike confusion in poor lighting.</p>
<p><img alt="Ronald Reagan, second out-of-cast name in Lord of War" src="../assets/images/lord_of_war_fpi_reagan_photo.jpg" /></p>
<p>Ronald Reagan, t=1003s, 100% confidence. This is not a lookalike confusion.
The detected face is a photograph of Reagan appearing within the shot, not a
living actor. The detector and matcher both did their job correctly on the
image content in front of them; the error is that a photograph inside the
scene is not the same thing as an actor present in the scene, and the
pipeline has no way to draw that distinction from a face crop alone.</p>
<p><img alt="Lance Reddick, third out-of-cast name in Lord of War" src="../assets/images/lord_of_war_fpi_reddick.jpg" /></p>
<p>Lance Reddick, t=6424s, 78% confidence. A small, distant, low-detail face at
the edge of frame. A marginal, low-confidence lookalike confusion.</p>
<h3 id="lovelace-1-name">Lovelace: 1 name<a class="headerlink" href="#lovelace-1-name" title="Permanent link">&para;</a></h3>
<p><img alt="Chloë Sevigny, out-of-cast name in Lovelace" src="../assets/images/lovelace_fpi_sevigny.jpg" /></p>
<p>Chloë Sevigny, t=2451s, 100% confidence. Two boxes are drawn on the same
face: one correctly labeled Amanda Seyfried, one incorrectly labeled Chloë
Sevigny, both at 100%. A single detection producing two competing high-
confidence identities on the same crop.</p>
<h3 id="scarface-1-name">Scarface: 1 name<a class="headerlink" href="#scarface-1-name" title="Permanent link">&para;</a></h3>
<p><img alt="Kirstie Alley, out-of-cast name in Scarface" src="../assets/images/scarface_fpi_alley.jpg" /></p>
<p>Kirstie Alley, t=2451s, 89% confidence. Al Pacino is correctly identified in
the foreground at 100%; a background face in the same shot is wrongly
labeled Kirstie Alley. (The t=2451s here and the Lovelace Chloë Sevigny case
above landing on the identical second is a genuine coincidence, verified from
each film's raw stream by <a href="https://gitea.tourolle.paris/dtourolle/scene-actor-extraction/raw/commit/0bd27470698c45cab21935d636a04512360e1008/scripts/docs/first_fpi_frames.py"><code>first_fpi_frames.py</code></a>,
not a transcription slip, two unrelated films whose <em>first</em> out-of-cast name
happens to fall at the same timestamp.)</p>
<h3 id="summary-of-the-nine">Summary of the nine<a class="headerlink" href="#summary-of-the-nine" title="Permanent link">&para;</a></h3>
<table>
<thead>
<tr>
<th>film</th>
<th>name</th>
<th>t (s)</th>
<th>confidence</th>
<th>classification</th>
</tr>
</thead>
<tbody>
<tr>
<td>Many Saints of Newark</td>
<td>Germar Terrell Gardner</td>
<td>848</td>
<td>78%</td>
<td>ground-truth gap</td>
</tr>
<tr>
<td>Many Saints of Newark</td>
<td>Archie Yates</td>
<td>2521</td>
<td>78%</td>
<td>lookalike confusion</td>
</tr>
<tr>
<td>Many Saints of Newark</td>
<td>Zooey Deschanel</td>
<td>2819</td>
<td>99%</td>
<td>lookalike confusion</td>
</tr>
<tr>
<td>Many Saints of Newark</td>
<td>Talia Balsam</td>
<td>4551</td>
<td>93%</td>
<td>ground-truth gap</td>
</tr>
<tr>
<td>Lord of War</td>
<td>David Shumbris</td>
<td>418</td>
<td>81%</td>
<td>lookalike confusion</td>
</tr>
<tr>
<td>Lord of War</td>
<td>Ronald Reagan</td>
<td>1003</td>
<td>100%</td>
<td>photo-in-frame</td>
</tr>
<tr>
<td>Lord of War</td>
<td>Lance Reddick</td>
<td>6424</td>
<td>78%</td>
<td>lookalike confusion, marginal</td>
</tr>
<tr>
<td>Lovelace</td>
<td>Chloë Sevigny</td>
<td>2451</td>
<td>100%</td>
<td>lookalike confusion</td>
</tr>
<tr>
<td>Scarface</td>
<td>Kirstie Alley</td>
<td>2451</td>
<td>89%</td>
<td>lookalike confusion</td>
</tr>
</tbody>
</table>
<p>Of nine distinct out-of-cast names across four films, two are ground-truth
gaps, one is a photograph misread as a person, and six are genuine
embedding-space confusions on real detected faces. None trace to extinction
bridging: every one of these nine is a fresh detection on a real face crop
at the second it first appears.</p>
<h2 id="where-lvface-beat-x-ray">Where LVFace beat X-Ray<a class="headerlink" href="#where-lvface-beat-x-ray" title="Permanent link">&para;</a></h2>
<p>Not every name marked wrong is actually wrong.
<a href="https://gitea.tourolle.paris/dtourolle/scene-actor-extraction/raw/commit/0bd27470698c45cab21935d636a04512360e1008/scripts/optimizer/second_score.py"><code>scripts/optimizer/second_score.py</code></a>
scores strictly against X-Ray, and X-Ray has gaps of its own.</p>
<p><img alt="LVFace correctly identifies Germar Terrell Gardner, uncredited by X-Ray" src="../assets/images/germar_beats_xray.jpg" /></p>
<p>Germar Terrell Gardner, the same name from the table above, does not appear
in X-Ray's <code>people.csv</code> for The Many Saints of Newark. Jellyfin's
independent cast metadata does credit him for this film (cross-checked
against <code>experiments/manifests/jellyfin_casts.json</code> from the
<code>experiment-data</code> artifact package, a data source entirely separate from
X-Ray). Talia Balsam is the same case: confirmed on screen, absent from
X-Ray's cast list for this title.</p>
<p><img alt="Robert Patrick, clearly on screen, scored wrong by a ground-truth gap" src="../assets/images/lovelace_robert_patrick_fpi.jpg" /></p>
<p>This extends past uncredited background actors. This is Robert Patrick,
top-billed in Lovelace, clearly on screen reading a newspaper, identified at
100%. The frame is scored wrong because X-Ray's people-in-scene list for
this specific scene omits him, despite crediting him elsewhere in the film.
The identification is correct; the ground truth is missing an entry.</p>
<p>X-Ray is a large, convenient ground truth. It is not a complete one. The
misID and FPI counts reported throughout this document include some fixed
amount of noise from gaps in X-Ray itself, in both directions.</p>
<h2 id="summary">Summary<a class="headerlink" href="#summary" title="Permanent link">&para;</a></h2>
<p>LVFace wins the model comparison on every held-out film. It correctly names
19 of 20 people in a crowded funeral scene and correctly identifies a face
displayed on a screen inside the film. Its errors resolve into two
mechanisms: extinction bridging, which is correct on most footage and fails
specifically at hard cuts into long faceless stretches, and the
face-versus-presence ceiling, where X-Ray credits scene membership for
people whose faces never appear on screen. Of the nine distinct
out-of-cast identifications found across the benchmark, two trace to gaps in
X-Ray's own cast data, one is a photograph misread as a person, and six are
genuine lookalike confusions on real faces. The held-out generalization gap,
75.3% training to 67.4% held-out, is real and should be treated as the
expected operating point, not the training-set figure.</p>
</article>
</div>
<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
</div>
<button type="button" class="md-top md-icon" data-md-component="top" hidden>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8z"/></svg>
Back to top
</button>
</main>
<footer class="md-footer">
<nav class="md-footer__inner md-grid" aria-label="Footer" >
<a href="../pose-expansion/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Pose Expansion">
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
</div>
<div class="md-footer__title">
<span class="md-footer__direction">
Previous
</span>
<div class="md-ellipsis">
Pose Expansion
</div>
</div>
</a>
<a href="../model-bakeoff/" class="md-footer__link md-footer__link--next" aria-label="Next: Full Experiment Log">
<div class="md-footer__title">
<span class="md-footer__direction">
Next
</span>
<div class="md-ellipsis">
Full Experiment Log
</div>
</div>
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11z"/></svg>
</div>
</a>
</nav>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">
Made with
<a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
Material for MkDocs
</a>
</div>
</div>
</div>
</footer>
</div>
<div class="md-dialog" data-md-component="dialog">
<div class="md-dialog__inner md-typeset"></div>
</div>
<script id="__config" type="application/json">{"annotate": null, "base": "..", "features": ["navigation.tabs", "navigation.sections", "navigation.top", "navigation.footer", "content.code.copy", "content.code.annotate"], "search": "../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>
<script src="../assets/javascripts/bundle.d7400e89.min.js"></script>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink