From add7e22053227f6f99360b79e559020371df6253 Mon Sep 17 00:00:00 2001
From: Duncan Tourolle <duncan@tourolle.paris>
Date: Sun, 9 Aug 2026 19:19:54 +0200
Subject: [PATCH] fix(optimizer): discard replay stderr and make the timeout a
 wedge-backstop
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Two failures surfaced scaling the DE sweep up. The replay sink prints a
per-second progress line with an explicit flush; under
subprocess.run(capture_output=True) those thousands of writes fill a fixed
OS pipe buffer that nothing drains until exit, so the long films blocked on
write to stderr and looked like hangs. Discard the child's stdout/stderr
(DEVNULL) — it was captured and thrown away anyway; the long films then
finish in seconds.

Separately, the per-film timeout is only a backstop for the rare,
intermittent ROCm GEMM wedge (a wedged replay hangs forever and must be
killed so the sweep continues), not a performance bound. It had been set
huge, which let a single flake stall the whole sweep; set it to a sane 180s
(overridable via REPLAY_TIMEOUT) — well above a healthy replay, short
enough to reap a wedge quickly.
---
 scripts/optimizer/optimize.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/scripts/optimizer/optimize.py b/scripts/optimizer/optimize.py
index 4f44272..6537866 100644
--- a/scripts/optimizer/optimize.py
+++ b/scripts/optimizer/optimize.py
@@ -61,7 +61,15 @@ from replay import dump_embedder_stamp  # noqa: E402
 from sae_stamp import EmbedderMismatch, verify_gallery_stamp  # noqa: E402
 
 _GAL_KEYS: dict = {}  # gallery path → key set (fair-recall FN mask), loaded once
-_REPLAY_TIMEOUT = 45  # seconds per film; a wedged replay is killed, not left to hang
+# Seconds per film before a replay is killed. Its ONLY job is to escape the rare,
+# intermittent ROCm GEMM wedge (github ROCT-Thunk #56): a wedged replay hangs
+# forever and would otherwise stall the whole sweep, so it must be killed and that
+# film dropped (the eval is then scored as incomplete → F1=0, and DE moves on). It
+# is NOT a performance bound. A healthy replay finishes in ~15-30s even for the
+# long films with stderr discarded, so 180s is comfortably above any real run yet
+# short enough that a wedge is reaped quickly rather than after half an hour.
+# Raise via REPLAY_TIMEOUT if a legitimately slow config is being killed.
+_REPLAY_TIMEOUT = int(os.environ.get("REPLAY_TIMEOUT", "180"))
 
 REPLAY_CLI = str(Path(__file__).resolve().parent / "replay.py")
 
@@ -91,7 +99,17 @@ def _replay_subprocess(dump, gallery, cfg, build_dir):
         else:
             argv += [f"--{k.replace('_', '-')}", str(v)]
     try:
-        subprocess.run(argv, timeout=_REPLAY_TIMEOUT, capture_output=True, check=True)
+        # Discard the child's stdout/stderr rather than capture it. replay's sink
+        # prints a per-second "[result_sink] t=Ns" progress line with an explicit
+        # flush; on a long film that is thousands of writes, and under
+        # subprocess.run(capture_output=True) they accumulate in a fixed OS pipe
+        # buffer that nothing drains until the process exits. On the long films
+        # (Valerian, Sound of Metal) under DE concurrency the buffer fills and the
+        # C++ process BLOCKS on write to stderr — indistinguishable from a hang, so
+        # it hit the timeout and scored F1=0. DEVNULL never fills, so the process
+        # runs to completion. (Any real error is still surfaced by check=True.)
+        subprocess.run(argv, timeout=_REPLAY_TIMEOUT, check=True,
+                       stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
         return _json.loads(Path(out).read_text())
     except (subprocess.TimeoutExpired, subprocess.CalledProcessError,
             FileNotFoundError, ValueError) as e: