Merge pull request #288 from ketpatil77/codex/openmontage-scoring-tokenizer-punctuation

fix(scoring): strip trailing punctuation from tokens

lib/scoring.py

@@ -147,7 +147,7 @@ _SYNONYM_CLUSTERS: list[set[str]] = [
     {"music", "soundtrack", "background-music", "score", "ambient"},
 ]
 
-_TOKEN_RE = re.compile(r"[a-z0-9][a-z0-9+._-]*")
+_TOKEN_RE = re.compile(r"[a-z0-9](?:[a-z0-9+._-]*[a-z0-9])?")
 _GENERATED_VISUAL_TERMS = {
     "animated",
     "animation",
@@ -498,7 +498,7 @@ def score_provider(tool, task_context: dict[str, Any]) -> ProviderScore:
     # lip-sync from quoted dialogue. This is what makes Seedance 2.0 (and
     # peer premium APIs) meaningfully better than generic clip providers.
     if asset_type == "video":
-        intent_words = _expand_synonyms(set(intent.lower().split())) | set(style_keywords)
+        intent_words = _expand_synonyms(set(_tokenize_text(intent))) | set(style_keywords)
         cinematic_signal = bool(
             intent_words & {"cinematic", "film", "movie", "trailer", "teaser", "dramatic", "epic", "premium"}
         )

tests/tools/test_scoring.py

@@ -0,0 +1,47 @@
+"""Regression tests for provider scoring tokenization."""
+
+from __future__ import annotations
+
+from lib.scoring import _tokenize_text, score_provider
+from tools.base_tool import ToolStatus
+
+
+class _FakeVideoTool:
+    name = "fake-video"
+
+    def get_info(self) -> dict[str, object]:
+        return {
+            "name": "fake-video",
+            "provider": "fake",
+            "best_for": ["cinematic video"],
+            "supports": {
+                "native_audio": True,
+                "multi_shot": True,
+                "camera_direction": True,
+                "lip_sync": True,
+                "cinematic_quality": True,
+            },
+            "stability": "production",
+            "runtime": "api",
+        }
+
+    def get_status(self) -> ToolStatus:
+        return ToolStatus.AVAILABLE
+
+    def estimate_cost(self, inputs: dict[str, object]) -> float:
+        return 0.0
+
+
+def test_tokenize_text_strips_trailing_punctuation() -> None:
+    assert _tokenize_text("cinematic.") == ["cinematic"]
+    assert _tokenize_text("v1.5.") == ["v1.5"]
+    assert _tokenize_text("gpt-4.1") == ["gpt-4.1"]
+
+
+def test_cinematic_bonus_ignores_adjacent_punctuation() -> None:
+    tool = _FakeVideoTool()
+    plain = score_provider(tool, {"asset_type": "video", "intent": "make it cinematic and fast"})
+    punctuated = score_provider(tool, {"asset_type": "video", "intent": "make it cinematic, and fast"})
+
+    assert punctuated.task_fit == plain.task_fit
+    assert punctuated.output_quality == plain.output_quality