From df8cf21310dd9f534f75afb88b428e7a3f708693 Mon Sep 17 00:00:00 2001 From: 0xDevNinja Date: Thu, 9 Jul 2026 18:15:05 +0530 Subject: [PATCH 1/2] fix(green_screen): scale chromakey background to frame size, not 1x1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _process_chromakey built the composite background from a 1x1 lavfi color source and tried to size it with `[0:v]scale=iw:ih`. That scale is a no-op — iw/ih are the 1x1 source's own dimensions, and there is no cross-reference to the frame. FFmpeg's overlay then takes the size of its first input (the 1x1 background), so every processed frame is clipped to a single pixel. The exception fallback never runs because the primary command exits 0 (a valid 1x1 PNG), and _reconstruct_video upscales those 1x1 frames — producing a solid-color video with the keyed subject entirely gone. Total data loss for method="chromakey" (and method="auto" when it selects chromakey). Use scale2ref to resize the background to the actual frame dimensions before overlaying, so the keyed subject is composited at full resolution. Verified with ffmpeg: a 320x240 green frame with a red subject now produces a 320x240 output with the subject preserved and green replaced by the background, instead of a 1x1 (then upscaled solid-color) frame. --- tests/tools/test_green_screen_chromakey.py | 110 +++++++++++++++++++++ tools/video/green_screen_processor.py | 11 ++- 2 files changed, 119 insertions(+), 2 deletions(-) create mode 100644 tests/tools/test_green_screen_chromakey.py diff --git a/tests/tools/test_green_screen_chromakey.py b/tests/tools/test_green_screen_chromakey.py new file mode 100644 index 00000000..1a0cb05e --- /dev/null +++ b/tests/tools/test_green_screen_chromakey.py @@ -0,0 +1,110 @@ +"""Regression tests: chromakey compositing must not collapse frames to 1x1. + +`_process_chromakey` built the background from a 1x1 lavfi color source and +tried to size it with `[0:v]scale=iw:ih` — a no-op, since iw/ih are the 1x1 +source's own dimensions. FFmpeg's `overlay` takes the size of its first input +(the 1x1 background), so every processed frame was clipped to a single pixel. +`_reconstruct_video` then upscaled those 1x1 frames, yielding a solid-color +video with the keyed subject entirely gone. The fix uses `scale2ref` to resize +the background to the actual frame dimensions. +""" + +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +sys.path.insert(0, str(PROJECT_ROOT)) + +from tools.video.green_screen_processor import GreenScreenProcessor # noqa: E402 + + +def test_chromakey_filter_scales_background_to_frame(): + """Offline: the built filtergraph must not leave a 1x1 background.""" + captured = [] + + def fake_run(self, cmd, **kwargs): + captured.append(list(cmd)) + + class _R: + returncode = 0 + stdout = "" + stderr = "" + + return _R() + + tool = GreenScreenProcessor() + orig = GreenScreenProcessor.run_command + GreenScreenProcessor.run_command = fake_run + try: + frames_dir = Path(PROJECT_ROOT) / "tests" / "tools" # any dir; glob may be empty + # Drive the filter build directly with a temp frame present. + import tempfile + + with tempfile.TemporaryDirectory() as td: + fd = Path(td) / "frames" + pd = Path(td) / "processed" + fd.mkdir() + pd.mkdir() + (fd / "frame_0000.png").write_bytes(b"stub") + tool._process_chromakey(fd, pd, "#0E172A", 1) + finally: + GreenScreenProcessor.run_command = orig + + ffmpeg_cmds = [c for c in captured if c and c[0] == "ffmpeg"] + assert ffmpeg_cmds, "no ffmpeg command built" + fc_idx = ffmpeg_cmds[0].index("-filter_complex") + fc = ffmpeg_cmds[0][fc_idx + 1] + assert "scale2ref" in fc, f"background not scaled to frame: {fc}" + assert "[0:v]scale=iw:ih[bg]" not in fc, "still using the 1x1 no-op scale" + + +@pytest.mark.skipif(shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None, + reason="ffmpeg/ffprobe required") +def test_chromakey_preserves_frame_size_and_keys(tmp_path): + """End-to-end: output keeps the source size and keys green -> background.""" + frames_dir = tmp_path / "frames" + processed_dir = tmp_path / "processed" + frames_dir.mkdir() + processed_dir.mkdir() + + frame = frames_dir / "frame_0000.png" + subprocess.run( + ["ffmpeg", "-y", "-f", "lavfi", "-i", + "color=c=0x00FF00:size=320x240,drawbox=x=100:y=80:w=120:h=80:color=red:t=fill", + "-frames:v", "1", str(frame)], + capture_output=True, check=True, timeout=60, + ) + + ok = GreenScreenProcessor()._process_chromakey(frames_dir, processed_dir, "#0E172A", 1) + assert ok + + out = processed_dir / "frame_0000.png" + assert out.exists() + + def _size(p): + r = subprocess.run( + ["ffprobe", "-v", "quiet", "-show_entries", "stream=width,height", + "-of", "csv=p=0", str(p)], + capture_output=True, text=True, timeout=30, + ) + w, h = r.stdout.strip().split(",") + return int(w), int(h) + + assert _size(out) == (320, 240), "frame collapsed instead of keeping source size" + + def _pixel(p, x, y): + r = subprocess.run( + ["ffmpeg", "-v", "quiet", "-i", str(p), "-vf", f"crop=1:1:{x}:{y}", + "-f", "rawvideo", "-pix_fmt", "rgb24", "-"], + capture_output=True, timeout=30, + ) + return r.stdout[:3] + + center = _pixel(out, 160, 120) # red box -> stays red + corner = _pixel(out, 10, 10) # was green -> keyed to dark background + assert center[0] > 150 and center[1] < 80, f"subject lost, center={center!r}" + assert corner[0] < 60 and corner[1] < 60, f"green not keyed, corner={corner!r}" diff --git a/tools/video/green_screen_processor.py b/tools/video/green_screen_processor.py index de074589..ab5e3ce6 100644 --- a/tools/video/green_screen_processor.py +++ b/tools/video/green_screen_processor.py @@ -465,9 +465,16 @@ class GreenScreenProcessor(BaseTool): "-i", str(frame), "-filter_complex", ( - f"[0:v]scale=iw:ih[bg];" + # The background is a 1x1 color source; it MUST be scaled up + # to the frame's dimensions. `scale=iw:ih` on [0:v] alone is a + # no-op (iw/ih are the 1x1 source's own size), and overlay + # takes the size of its FIRST input — so that leaves a 1x1 + # canvas and the keyed frame gets clipped to a single pixel, + # producing a solid-color video with the subject gone. + # scale2ref resizes [0:v] to match the frame [1:v] first. f"[1:v]chromakey=color=0x00FF00:similarity=0.3:blend=0.08[fg];" - f"[bg][fg]overlay=0:0" + f"[0:v][fg]scale2ref[bg][fg2];" + f"[bg][fg2]overlay=0:0" ), "-frames:v", "1", str(out_path), From fa756fbec5c645ee7ab54459d96fbc7e0b8890d7 Mon Sep 17 00:00:00 2001 From: 0xDevNinja Date: Tue, 14 Jul 2026 14:27:40 +0530 Subject: [PATCH 2/2] fix(green_screen): make chromakey compositing portable across FFmpeg builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CI Linux FFmpeg build carried the keyed frame forward without an alpha plane, so overlay drew opaque green over the background (corner stayed green) instead of compositing — the E2E test failed there even though it passed on macOS/Windows. Force `format=yuva420p` immediately after chromakey so the keyed transparency always has an explicit alpha plane, and size the background to the frame up front (color=...:size=WxH, passing the probed width/height into _process_chromakey) instead of scaling a 1x1 source with scale2ref — dropping scale2ref also removes the format negotiation that discarded the alpha on some builds. Output is flattened to yuv420p after the overlay. --- tests/tools/test_green_screen_chromakey.py | 16 ++++++++---- tools/video/green_screen_processor.py | 29 +++++++++++++--------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/tests/tools/test_green_screen_chromakey.py b/tests/tools/test_green_screen_chromakey.py index 1a0cb05e..ab4bcf3a 100644 --- a/tests/tools/test_green_screen_chromakey.py +++ b/tests/tools/test_green_screen_chromakey.py @@ -50,16 +50,20 @@ def test_chromakey_filter_scales_background_to_frame(): fd.mkdir() pd.mkdir() (fd / "frame_0000.png").write_bytes(b"stub") - tool._process_chromakey(fd, pd, "#0E172A", 1) + tool._process_chromakey(fd, pd, "#0E172A", 1, 320, 240) finally: GreenScreenProcessor.run_command = orig ffmpeg_cmds = [c for c in captured if c and c[0] == "ffmpeg"] assert ffmpeg_cmds, "no ffmpeg command built" - fc_idx = ffmpeg_cmds[0].index("-filter_complex") - fc = ffmpeg_cmds[0][fc_idx + 1] - assert "scale2ref" in fc, f"background not scaled to frame: {fc}" + cmd = ffmpeg_cmds[0] + fc = cmd[cmd.index("-filter_complex") + 1] + # Background must be sized to the frame, not the old 1x1 no-op. + assert "size=320x240" in " ".join(cmd), "background not sized to frame" + assert "size=1x1" not in " ".join(cmd), "still using the 1x1 background" assert "[0:v]scale=iw:ih[bg]" not in fc, "still using the 1x1 no-op scale" + # Alpha must be forced so keyed transparency survives on every FFmpeg build. + assert "format=yuva420p" in fc, f"keyed alpha not forced: {fc}" @pytest.mark.skipif(shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None, @@ -79,7 +83,9 @@ def test_chromakey_preserves_frame_size_and_keys(tmp_path): capture_output=True, check=True, timeout=60, ) - ok = GreenScreenProcessor()._process_chromakey(frames_dir, processed_dir, "#0E172A", 1) + ok = GreenScreenProcessor()._process_chromakey( + frames_dir, processed_dir, "#0E172A", 1, 320, 240 + ) assert ok out = processed_dir / "frame_0000.png" diff --git a/tools/video/green_screen_processor.py b/tools/video/green_screen_processor.py index ab5e3ce6..fdc9b7bf 100644 --- a/tools/video/green_screen_processor.py +++ b/tools/video/green_screen_processor.py @@ -161,7 +161,7 @@ class GreenScreenProcessor(BaseTool): if method == "chromakey": ok = self._process_chromakey( - frames_dir, processed_dir, bg_color, frame_count + frames_dir, processed_dir, bg_color, frame_count, width, height ) else: ok = self._process_rembg( @@ -445,6 +445,8 @@ class GreenScreenProcessor(BaseTool): processed_dir: Path, bg_color: str, frame_count: int, + width: int, + height: int, ) -> bool: """Process frames using FFmpeg chromakey filter. @@ -461,20 +463,23 @@ class GreenScreenProcessor(BaseTool): out_path = processed_dir / frame.name cmd = [ "ffmpeg", "-y", - "-f", "lavfi", "-i", f"color=c={ffmpeg_bg}:size=1x1", + # Background sized to the frame up front. The old code used a 1x1 + # color source and `[0:v]scale=iw:ih` — a no-op (iw/ih were the + # 1x1 source's own size) — and overlay takes the size of its + # FIRST input, so every frame was clipped to a single pixel and + # the whole video came out a solid color with the subject gone. + "-f", "lavfi", "-i", f"color=c={ffmpeg_bg}:size={width}x{height}", "-i", str(frame), "-filter_complex", ( - # The background is a 1x1 color source; it MUST be scaled up - # to the frame's dimensions. `scale=iw:ih` on [0:v] alone is a - # no-op (iw/ih are the 1x1 source's own size), and overlay - # takes the size of its FIRST input — so that leaves a 1x1 - # canvas and the keyed frame gets clipped to a single pixel, - # producing a solid-color video with the subject gone. - # scale2ref resizes [0:v] to match the frame [1:v] first. - f"[1:v]chromakey=color=0x00FF00:similarity=0.3:blend=0.08[fg];" - f"[0:v][fg]scale2ref[bg][fg2];" - f"[bg][fg2]overlay=0:0" + # Force an explicit alpha format after chromakey so the keyed + # transparency survives filter negotiation on every FFmpeg + # build — without it, some Linux builds carry the keyed frame + # forward without an alpha plane and overlay draws opaque + # green over the background instead of compositing. + f"[1:v]chromakey=color=0x00FF00:similarity=0.3:blend=0.08," + f"format=yuva420p[fg];" + f"[0:v][fg]overlay=0:0:format=auto,format=yuv420p" ), "-frames:v", "1", str(out_path),