diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index 7d3f8bdba..165357b28 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -352,12 +352,16 @@ class RecallRequest(BaseModel): ) min_scores: MinScores | None = Field( default=None, - description="Optional per-stage score floors (all inclusive, AND-ed). `semantic` and `keyword` are " - "retrieval-level cutoffs pushed into the SQL arms (overriding the global similarity/BM25 minimums for " - "this request); `reranker` and `final` are post-ranking filters on the scored results. Any field left " - "unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use " - "with care — the reranker's absolute scores are not calibrated across queries (a clearly-relevant match " - "may score ~0.001 even though it is ranked first).", + description="Optional per-stage score floors, each inclusive (`>=`). `semantic` and `keyword` are " + "retrieval-level cutoffs pushed into the SQL arm they name (overriding the global similarity/BM25 " + "minimums for this request), and constrain only that arm: recall fuses four arms (semantic, keyword, " + "graph, temporal) and returns a result surfaced by any of them, so a returned result reports null for a " + "stage that did not surface it (a non-null score always clears its floor). Setting both therefore " + "does not restrict the response to results clearing both. `reranker` and `final` are post-ranking " + "filters applied to every scored result, so those floors *are* guaranteed by each result returned — " + "use them for query abstention. Any field left unset imposes no floor; omitting `min_scores` entirely " + "(the default) applies no score filtering. Use with care — the reranker's absolute scores are not " + "calibrated across queries (a clearly-relevant match may score ~0.001 even though it is ranked first).", ) temporal_window: TemporalWindow | None = Field( default=None, diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 0fbd369de..31e77b2ec 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -1212,7 +1212,12 @@ DEFAULT_ANN_MAX_SCAN_TUPLES = 4000 # Text search extension (native PostgreSQL, vchord BM25, Timescale pg_textsearch, # pgroonga, or ParadeDB pg_search). Unused by banks with enable_text_search off. -DEFAULT_TEXT_SEARCH_EXTENSION = "native" # Options: "native", "vchord", "pg_textsearch", "pgroonga", "pg_search" +# Every PostgreSQL full-text backend `build_bm25_arm` dispatches on. Exported so +# tests can enumerate the family rather than hardcoding a copy that drifts — a +# backend added here but forgotten in one of the arm builders is exactly how +# `min_scores.keyword` became a no-op on four of them (#3882). +VALID_TEXT_SEARCH_EXTENSIONS = ("native", "vchord", "pg_textsearch", "pgroonga", "pg_search") +DEFAULT_TEXT_SEARCH_EXTENSION = "native" # PostgreSQL text search dictionary used by the native tsvector backend. Only # affects text_search_extension == "native"; other backends use their own @@ -3234,10 +3239,10 @@ class HindsightConfig: ) # Validate text_search_extension - valid_text_search = ("native", "vchord", "pg_textsearch", "pgroonga", "pg_search") - if self.text_search_extension not in valid_text_search: + if self.text_search_extension not in VALID_TEXT_SEARCH_EXTENSIONS: raise ValueError( - f"Invalid text_search_extension: {self.text_search_extension}. Must be one of: {', '.join(valid_text_search)}" + f"Invalid text_search_extension: {self.text_search_extension}. " + f"Must be one of: {', '.join(VALID_TEXT_SEARCH_EXTENSIONS)}" ) # Validate text_search_extension_native_language as a PG identifier. diff --git a/hindsight-api-slim/hindsight_api/engine/response_models.py b/hindsight-api-slim/hindsight_api/engine/response_models.py index b64527f38..06b2b031a 100644 --- a/hindsight-api-slim/hindsight_api/engine/response_models.py +++ b/hindsight-api-slim/hindsight_api/engine/response_models.py @@ -200,20 +200,52 @@ class RecallScores(BaseModel): class MinScores(BaseModel): - """Optional per-stage score floors for recall (all inclusive, AND-ed). + """Optional per-stage score floors for recall. Every floor is inclusive (``>=``). - ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL - arms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` - config for this request), so they prune weak matches before fusion. ``reranker`` - and ``final`` are **post-query** filters applied to the scored results after - reranking. Any field left None imposes no floor; all-None (the default) means - no score filtering. + The four floors act at two different levels, and the distinction decides what a + returned result is guaranteed to satisfy. + + ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into their own + SQL arm (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` + config for this request), so they prune weak matches before fusion. Each one + constrains **only the arm it names**. Recall fuses four arms — semantic, keyword, + graph and temporal — and a result reaches the response if *any* arm surfaced it, + so a returned result may legitimately carry ``null`` for a stage it was not + surfaced by, and results reached through the graph or temporal arm carry neither + ``semantic`` nor ``keyword``. A *non-null* score always clears its floor — the + gap is only ever a ``null``. Setting both does **not** restrict the response to + results that clear both: they are not a predicate over each fused result. This is + deliberate — an intersection would discard the strong single-arm matches that + hybrid retrieval exists to find (a paraphrase with no lexical overlap, an exact + identifier the embedding scores poorly). + + ``reranker`` and ``final`` are **post-query** filters applied to every scored + result after fusion and reranking, so these *are* per-result predicates: a + returned result always clears them. Use them, not the retrieval floors, to make + recall abstain on low-confidence queries. + + Any field left None imposes no floor; all-None (the default) means no score + filtering. """ - semantic: float | None = Field(default=None, description="Retrieval-level: minimum vector similarity (0-1).") - keyword: float | None = Field(default=None, description="Retrieval-level: minimum keyword/full-text (BM25) score.") - reranker: float | None = Field(default=None, description="Post-query: minimum normalized reranker score (0-1).") - final: float | None = Field(default=None, description="Post-query: minimum final ranking score.") + semantic: float | None = Field( + default=None, + description="Retrieval-level, semantic arm only: minimum vector similarity (0-1). A result the semantic " + "arm did not surface reports `semantic: null` and is unaffected by this floor.", + ) + keyword: float | None = Field( + default=None, + description="Retrieval-level, keyword arm only: minimum keyword/full-text (BM25) score. A result the " + "keyword arm did not surface reports `keyword: null` and is unaffected by this floor.", + ) + reranker: float | None = Field( + default=None, + description="Post-query: minimum normalized reranker score (0-1). Applied to every returned result.", + ) + final: float | None = Field( + default=None, + description="Post-query: minimum final ranking score. Applied to every returned result.", + ) class TemporalWindow(BaseModel): diff --git a/hindsight-api-slim/hindsight_api/engine/sql/base.py b/hindsight-api-slim/hindsight_api/engine/sql/base.py index b98d83e18..95e70ded6 100644 --- a/hindsight-api-slim/hindsight_api/engine/sql/base.py +++ b/hindsight-api-slim/hindsight_api/engine/sql/base.py @@ -7,6 +7,25 @@ Business logic calls these methods instead of embedding raw SQL fragments. from abc import ABC, abstractmethod +def bm25_score_gate(bm25_min_score: float) -> str: + """Return the comparison a BM25 arm applies to its relevance score. + + Two different jobs share one number. With no caller floor (the default, + ``0.0``) the gate is structural — ``> 0`` keeps only genuine term matches on + backends whose operator ranks every document instead of pre-filtering. With a + caller floor (recall's ``min_scores.keyword``) the gate becomes that floor, + **inclusive**, matching the documented contract and the semantic arm's + ``>= min_similarity``. A positive floor subsumes the structural gate, so the + two never need to be applied together. + """ + if bm25_min_score <= 0: + return "> 0" + # `!r` (shortest round-tripping repr), not `:g` — `:g` truncates to 6 + # significant digits, so a caller echoing a `scores.keyword` value back as a + # floor could get a literal that rounds up past its own row and drops it. + return f">= {bm25_min_score!r}" + + class SQLDialect(ABC): """SQL dialect interface for portable query construction. @@ -422,11 +441,16 @@ class SQLDialect(ABC): "pg_textsearch", "pgroonga", "pg_search"). Only relevant for PostgreSQL. bm25_language: PostgreSQL text search dictionary used by the native backend (e.g. "english", "french"). Ignored by other backends. - bm25_min_score: Minimum BM25 relevance score a row must exceed to be - returned. Gates out non-matching rows on backends whose - operator (e.g. VectorChord) ranks every document instead - of pre-filtering to query-term matches. Backends that - already apply a boolean match gate ignore this. + bm25_min_score: Inclusive minimum BM25 relevance score a row must + reach to be returned (recall's ``min_scores.keyword``, + or the ``bm25_min_score`` config default). Every + backend must honour it. At the ``0.0`` default it + degrades to a structural ``> 0`` match gate, which + matters for backends whose operator (e.g. VectorChord) + ranks every document instead of pre-filtering to + query-term matches; backends with their own boolean + match gate (`@@`, `&@~`, `@@@`) need no extra + predicate at that default. pg_search_function_schema: Schema containing pg_search functions (e.g. "paradedb", "pgsearch"). Only used by the pg_search backend. extra_where: Optional additional WHERE clause fragment (e.g. time range filter). diff --git a/hindsight-api-slim/hindsight_api/engine/sql/oracle.py b/hindsight-api-slim/hindsight_api/engine/sql/oracle.py index 8c45bd897..6efc09e7d 100644 --- a/hindsight-api-slim/hindsight_api/engine/sql/oracle.py +++ b/hindsight-api-slim/hindsight_api/engine/sql/oracle.py @@ -5,7 +5,7 @@ vector distance (VECTOR_DISTANCE), full-text search (Oracle Text), and other non-portable patterns. """ -from .base import SQLDialect +from .base import SQLDialect, bm25_score_gate class OracleDialect(SQLDialect): @@ -284,9 +284,10 @@ class OracleDialect(SQLDialect): f" FROM {table}" f" WHERE bank_id = {bank_id_param}" f" AND fact_type = '{fact_type}'" - # CONTAINS already gates to genuine matches; the configurable floor - # (default 0) keeps the threshold semantics uniform across backends. - f" AND CONTAINS(text, {text_param}, {label}) > {bm25_min_score:g}" + # CONTAINS already gates to genuine matches, so at the 0.0 default the + # gate is the structural `> 0`; a caller's `min_scores.keyword` floor + # replaces it with an inclusive `>=`, uniform across backends. + f" AND CONTAINS(text, {text_param}, {label}) {bm25_score_gate(bm25_min_score)}" f" {tags_clause}" f" {groups_clause}" f" {extra_where}" diff --git a/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py b/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py index ae563f30e..deae5da99 100644 --- a/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py +++ b/hindsight-api-slim/hindsight_api/engine/sql/postgresql.py @@ -8,7 +8,7 @@ and other non-portable patterns. from dataclasses import dataclass from ..._text_search import mental_models_text_document -from .base import SQLDialect +from .base import SQLDialect, bm25_score_gate @dataclass(frozen=True) @@ -306,6 +306,12 @@ class PostgreSQLDialect(SQLDialect): pg_search_function_schema: str = "paradedb", extra_where: str = "", ) -> str: + # Whether the branch's own WHERE enforces ``bm25_min_score``. Branches that + # cannot (their score is only valid in the target list, or re-evaluating it + # in WHERE would cost a second computation) get the floor applied by the + # outer wrapper below instead. + floor_in_where = False + if text_search_extension == "vchord": # <&> returns the NEGATIVE BM25 score (lower = more relevant), negate # for a positive score where higher = more relevant. @@ -315,7 +321,11 @@ class PostgreSQLDialect(SQLDialect): # VectorChord operator ranks *every* document, so a bare ORDER BY ... # LIMIT pads the result with zero-score, non-matching rows. Gate on the # score so only genuine term matches survive into fusion/reranking. - bm25_where_filter = f"AND -(search_vector <&> to_bm25query('idx_memory_units_text_search', tokenize({text_param}, 'llmlingua2'))) > {bm25_min_score:g}" + # With no caller floor that gate is `> 0` (structural: keep genuine + # matches only); with one it becomes the caller's inclusive floor, + # which subsumes it. + bm25_where_filter = f"AND {bm25_score_expr} {bm25_score_gate(bm25_min_score)}" + floor_in_where = True elif text_search_extension == "pg_textsearch": bm25_score_expr = f"-(text <@> to_bm25query({text_param}, 'idx_memory_units_text_search'))" bm25_order_by = f"text <@> to_bm25query({text_param}, 'idx_memory_units_text_search') ASC" @@ -359,7 +369,7 @@ class PostgreSQLDialect(SQLDialect): bm25_order_by = f"{bm25_score_expr} DESC" bm25_where_filter = f"AND search_vector @@ to_tsquery('{bm25_language}', {text_param})" - return ( + arm = ( f"(SELECT {cols}," f" NULL::float AS similarity," f" {bm25_score_expr} AS bm25_score," @@ -375,6 +385,22 @@ class PostgreSQLDialect(SQLDialect): f" LIMIT {limit_param})" ) + if floor_in_where or bm25_min_score <= 0: + # Either the branch already gated on the floor, or there is no caller + # floor to apply and the branch's own boolean match gate (`@@`, `&@~`, + # `@@@`) is the only filter — the default, and unchanged by this path. + return arm + + # Apply the caller's floor from the outside. pgroonga's `pgroonga_score()` + # and pg_search's `.score()` are only valid in the target list, and + # re-evaluating native's `ts_rank_cd` or pg_textsearch's `<@>` in WHERE + # would compute the score twice per row (and, for `<@>`, forfeit the index + # scan the ORDER BY relies on). Every arm orders by score DESC, so filtering + # the ordered LIMIT slice keeps exactly the rows an inner predicate would: + # when at least `limit` rows clear the floor the slice is entirely above it, + # and otherwise both forms return precisely the rows that clear it. + return f"(SELECT * FROM {arm} AS bm25_arm_{arm_index} WHERE bm25_score {bm25_score_gate(bm25_min_score)})" + def prepare_bm25_text( self, tokens: list[str], diff --git a/hindsight-api-slim/hindsight_api/mcp_tools.py b/hindsight-api-slim/hindsight_api/mcp_tools.py index f71ddb505..88c27de84 100644 --- a/hindsight-api-slim/hindsight_api/mcp_tools.py +++ b/hindsight-api-slim/hindsight_api/mcp_tools.py @@ -1111,9 +1111,13 @@ def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) Anchors relative temporal expressions and recency scoring. min_scores: Optional per-stage score floors as an object with any of: "semantic", "keyword" (retrieval-level cutoffs), "reranker", "final" (post-ranking). E.g. {"reranker": 0.5}. - All inclusive and AND-ed; omit for no score filtering. The reranker's absolute scores are - not calibrated across queries, so only threshold against scores you've calibrated for your - own data. + Each floor is inclusive; omit for no score filtering. "semantic" and "keyword" prune only + the retrieval arm they name — recall fuses four arms (semantic, keyword, graph, temporal) + and returns what any of them surfaced, so a result may report null or a lower score for an + arm that did not surface it, and setting both does not restrict results to those clearing + both. Use "reranker"/"final" — applied to every scored result — to make recall abstain. + The reranker's absolute scores are not calibrated across queries, so only threshold + against scores you've calibrated for your own data. temporal_window: Window for the temporal arm as {"start": ISO, "end": ISO}, used instead of extracting dates from the query text — pass it when you already know the range you mean. It ranks memories dated inside the window higher; it does NOT drop memories dated outside @@ -1203,9 +1207,13 @@ def _register_recall(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) Anchors relative temporal expressions and recency scoring. min_scores: Optional per-stage score floors as an object with any of: "semantic", "keyword" (retrieval-level cutoffs), "reranker", "final" (post-ranking). E.g. {"reranker": 0.5}. - All inclusive and AND-ed; omit for no score filtering. The reranker's absolute scores are - not calibrated across queries, so only threshold against scores you've calibrated for your - own data. + Each floor is inclusive; omit for no score filtering. "semantic" and "keyword" prune only + the retrieval arm they name — recall fuses four arms (semantic, keyword, graph, temporal) + and returns what any of them surfaced, so a result may report null or a lower score for an + arm that did not surface it, and setting both does not restrict results to those clearing + both. Use "reranker"/"final" — applied to every scored result — to make recall abstain. + The reranker's absolute scores are not calibrated across queries, so only threshold + against scores you've calibrated for your own data. temporal_window: Window for the temporal arm as {"start": ISO, "end": ISO}, used instead of extracting dates from the query text — pass it when you already know the range you mean. It ranks memories dated inside the window higher; it does NOT drop memories dated outside diff --git a/hindsight-api-slim/tests/test_bm25_min_score_pushdown.py b/hindsight-api-slim/tests/test_bm25_min_score_pushdown.py new file mode 100644 index 000000000..e85f0ab99 --- /dev/null +++ b/hindsight-api-slim/tests/test_bm25_min_score_pushdown.py @@ -0,0 +1,92 @@ +"""Every text-search backend must push `min_scores.keyword` into its BM25 arm. + +`bm25_min_score` started life in #1947 as a VectorChord-only gate (vchord's `<&>` +ranks every document, so it needed the analogue of native tsvector's boolean `@@` +match gate). #2422 then built the public `min_scores.keyword` floor on top of that +same parameter without revisiting the backends, so four of the six branches — +including `native`, the default — accepted the floor and silently ignored it: a +caller asking for `keyword >= 0.30` got rows scoring 0.2 back. + +These are SQL-shape assertions rather than end-to-end queries so that a new +backend branch cannot repeat the omission without a red test, on machines with no +VectorChord/pgroonga/pg_search/Oracle available. +""" + +import pytest + +from hindsight_api.config import VALID_TEXT_SEARCH_EXTENSIONS +from hindsight_api.engine.sql.oracle import OracleDialect +from hindsight_api.engine.sql.postgresql import PostgreSQLDialect + +# Enumerate the family from config rather than restating it, so a sixth backend +# is covered by these assertions the moment it becomes selectable. +PG_EXTENSIONS = VALID_TEXT_SEARCH_EXTENSIONS + +# The backends that gate on the score even with no caller floor, because their +# operator ranks every document instead of pre-filtering to query-term matches. +RANKS_EVERY_DOC = ("vchord",) + +ARM_KWARGS = dict( + table="memory_units", + cols="id, text", + fact_type="world", + bank_id_param="$2", + limit_param="$3", + text_param="$4", +) + + +def _pg_arm(extension: str, bm25_min_score: float) -> str: + return PostgreSQLDialect().build_bm25_arm( + **ARM_KWARGS, + text_search_extension=extension, + bm25_min_score=bm25_min_score, + ) + + +@pytest.mark.parametrize("extension", PG_EXTENSIONS) +def test_pg_backend_applies_the_caller_floor(extension): + """A positive floor reaches the SQL on every backend, not just vchord.""" + assert ">= 0.3" in _pg_arm(extension, 0.3) + + +def test_oracle_applies_the_caller_floor(): + arm = OracleDialect().build_bm25_arm(**ARM_KWARGS, bm25_min_score=0.3) + assert ">= 0.3" in arm + + +@pytest.mark.parametrize("extension", PG_EXTENSIONS) +def test_pg_floor_is_inclusive_not_exclusive(extension): + """`min_scores` floors are documented as inclusive, and the semantic arm uses + `>= min_similarity`. The keyword arm must agree: a row scoring exactly the + floor is kept.""" + arm = _pg_arm(extension, 0.3) + assert "> 0.3" not in arm.replace(">= 0.3", "") + + +@pytest.mark.parametrize("extension", PG_EXTENSIONS) +def test_pg_default_floor_keeps_the_structural_match_gate(extension): + """At the 0.0 default there is no caller floor, so behaviour is unchanged: + backends whose operator ranks every row keep their `> 0` gate, and backends + with a boolean match gate get no score predicate at all.""" + arm = _pg_arm(extension, 0.0) + if extension in RANKS_EVERY_DOC: + assert "> 0" in arm + else: + assert "bm25_score >" not in arm + assert "bm25_score >=" not in arm + + +def test_oracle_default_floor_keeps_the_structural_match_gate(): + arm = OracleDialect().build_bm25_arm(**ARM_KWARGS, bm25_min_score=0.0) + assert "> 0" in arm + + +@pytest.mark.parametrize("extension", PG_EXTENSIONS) +def test_pg_arm_is_a_single_union_all_ready_subquery(extension): + """The arms are joined with UNION ALL, so wrapping one to apply the floor from + the outside must keep it a single parenthesised subquery.""" + arm = _pg_arm(extension, 0.3) + assert arm.startswith("(") and arm.endswith(")") + assert "bm25_score" in arm + assert "'bm25' AS source" in arm diff --git a/hindsight-api-slim/tests/test_db_abstraction.py b/hindsight-api-slim/tests/test_db_abstraction.py index fdf634f76..ee574077b 100644 --- a/hindsight-api-slim/tests/test_db_abstraction.py +++ b/hindsight-api-slim/tests/test_db_abstraction.py @@ -285,7 +285,11 @@ class TestPostgreSQLDialect: text_search_extension="vchord", bm25_min_score=2.5, ) - assert "> 2.5" in arm + # Inclusive (`>=`), matching the documented `min_scores` contract and the + # semantic arm's `>= min_similarity`. This was `>` until #3882: the same + # parameter served as both vchord's structural match gate and the caller's + # floor, and the gate's `>` leaked into the public contract. + assert ">= 2.5" in arm def test_build_bm25_arm_pg_textsearch_scores_each_row(self, d): arm = d.build_bm25_arm( diff --git a/hindsight-api-slim/tests/test_recall_min_score.py b/hindsight-api-slim/tests/test_recall_min_score.py index 364518778..f857ba0d4 100644 --- a/hindsight-api-slim/tests/test_recall_min_score.py +++ b/hindsight-api-slim/tests/test_recall_min_score.py @@ -4,7 +4,8 @@ Inserts memory_units with known content + real embeddings directly via SQL, then verifies that recall_async: - returns a `scores` object (final/reranker/semantic/keyword) on every result, - applies the post-query floors (`reranker`, `final`) to the scored results, - - applies the retrieval-level floors (`semantic`, `keyword`) inside the SQL arms, + - applies the retrieval-level `semantic` floor inside the SQL arm, + - keeps the retrieval floors per-arm rather than per-result, - is unchanged by the default (`min_scores=None`). Filtering is deterministic post/pre-processing, so these assertions are direct — @@ -24,6 +25,7 @@ from hindsight_api.engine.retain import embedding_utils # onto one group to avoid pk conflicts, same as test_recall_time_range.py. pytestmark = pytest.mark.xdist_group("recall_min_score") + ID_A = "00000000-0000-0000-0000-0000000000a1" ID_B = "00000000-0000-0000-0000-0000000000a2" ID_C = "00000000-0000-0000-0000-0000000000a3" @@ -176,6 +178,36 @@ class TestRetrievalLevelFilters: filtered = await _recall(engine, bank_id, min_scores=MinScores(semantic=1.1)) assert filtered.results == [] + @pytest.mark.memory_backend_incompatible + async def test_retrieval_floors_are_per_arm_not_per_result(self, seeded_memory): + """The retrieval floors constrain the arm they name, not the fused result. + + Recall returns a memory that *any* arm surfaced, so with both floors set a + result may still report `null` for the stage that did not surface it. This + is the documented contract (see MinScores) — an intersection would discard + the strong single-arm matches hybrid retrieval exists to find. Callers who + want abstention use the post-query `reranker`/`final` floors instead. + """ + engine, bank_id = seeded_memory + both = await _recall( + engine, + bank_id, + query="animals", + min_scores=MinScores(semantic=0.1, keyword=0.0001), + ) + assert both.results, "floors this low should not empty the response" + # Whatever survives, each arm's own floor held for the scores it produced. + for r in both.results: + assert r.scores.semantic is None or r.scores.semantic >= 0.1 + assert r.scores.keyword is None or r.scores.keyword >= 0.0001 + # The union behaviour itself: setting both floors does not narrow the + # response to results that cleared both arms. Every result here is + # surfaced by the semantic arm alone and reports `keyword: null`, which a + # strict post-fusion intersection would have dropped to zero results. + assert not all(r.scores.semantic is not None and r.scores.keyword is not None for r in both.results), ( + "expected at least one result missing a score for a floored arm" + ) + class TestRecallRequestDefault: """min_scores is opt-in: the HTTP recall defaults to None (no filtering).""" diff --git a/hindsight-clients/go/api/openapi.yaml b/hindsight-clients/go/api/openapi.yaml index b9bfd0697..641577a51 100644 --- a/hindsight-clients/go/api/openapi.yaml +++ b/hindsight-clients/go/api/openapi.yaml @@ -8949,14 +8949,32 @@ components: title: MentalModelTrigger MinScores: description: |- - Optional per-stage score floors for recall (all inclusive, AND-ed). + Optional per-stage score floors for recall. Every floor is inclusive (``>=``). - ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL - arms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` - config for this request), so they prune weak matches before fusion. ``reranker`` - and ``final`` are **post-query** filters applied to the scored results after - reranking. Any field left None imposes no floor; all-None (the default) means - no score filtering. + The four floors act at two different levels, and the distinction decides what a + returned result is guaranteed to satisfy. + + ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into their own + SQL arm (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` + config for this request), so they prune weak matches before fusion. Each one + constrains **only the arm it names**. Recall fuses four arms — semantic, keyword, + graph and temporal — and a result reaches the response if *any* arm surfaced it, + so a returned result may legitimately carry ``null`` for a stage it was not + surfaced by, and results reached through the graph or temporal arm carry neither + ``semantic`` nor ``keyword``. A *non-null* score always clears its floor — the + gap is only ever a ``null``. Setting both does **not** restrict the response to + results that clear both: they are not a predicate over each fused result. This is + deliberate — an intersection would discard the strong single-arm matches that + hybrid retrieval exists to find (a paraphrase with no lexical overlap, an exact + identifier the embedding scores poorly). + + ``reranker`` and ``final`` are **post-query** filters applied to every scored + result after fusion and reranking, so these *are* per-result predicates: a + returned result always clears them. Use them, not the retrieval floors, to make + recall abstain on low-confidence queries. + + Any field left None imposes no floor; all-None (the default) means no score + filtering. properties: semantic: nullable: true diff --git a/hindsight-clients/go/model_min_scores.go b/hindsight-clients/go/model_min_scores.go index 97dd16965..29f69671a 100644 --- a/hindsight-clients/go/model_min_scores.go +++ b/hindsight-clients/go/model_min_scores.go @@ -17,7 +17,7 @@ import ( // checks if the MinScores type satisfies the MappedNullable interface at compile time var _ MappedNullable = &MinScores{} -// MinScores Optional per-stage score floors for recall (all inclusive, AND-ed). ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL arms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` config for this request), so they prune weak matches before fusion. ``reranker`` and ``final`` are **post-query** filters applied to the scored results after reranking. Any field left None imposes no floor; all-None (the default) means no score filtering. +// MinScores Optional per-stage score floors for recall. Every floor is inclusive (``>=``). The four floors act at two different levels, and the distinction decides what a returned result is guaranteed to satisfy. ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into their own SQL arm (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` config for this request), so they prune weak matches before fusion. Each one constrains **only the arm it names**. Recall fuses four arms — semantic, keyword, graph and temporal — and a result reaches the response if *any* arm surfaced it, so a returned result may legitimately carry ``null`` for a stage it was not surfaced by, and results reached through the graph or temporal arm carry neither ``semantic`` nor ``keyword``. A *non-null* score always clears its floor — the gap is only ever a ``null``. Setting both does **not** restrict the response to results that clear both: they are not a predicate over each fused result. This is deliberate — an intersection would discard the strong single-arm matches that hybrid retrieval exists to find (a paraphrase with no lexical overlap, an exact identifier the embedding scores poorly). ``reranker`` and ``final`` are **post-query** filters applied to every scored result after fusion and reranking, so these *are* per-result predicates: a returned result always clears them. Use them, not the retrieval floors, to make recall abstain on low-confidence queries. Any field left None imposes no floor; all-None (the default) means no score filtering. type MinScores struct { Semantic NullableFloat32 `json:"semantic,omitempty"` Keyword NullableFloat32 `json:"keyword,omitempty"` diff --git a/hindsight-clients/python/hindsight_client_api/models/min_scores.py b/hindsight-clients/python/hindsight_client_api/models/min_scores.py index 7fcaf4c83..e4c8bd688 100644 --- a/hindsight-clients/python/hindsight_client_api/models/min_scores.py +++ b/hindsight-clients/python/hindsight_client_api/models/min_scores.py @@ -24,7 +24,7 @@ from typing_extensions import Self class MinScores(BaseModel): """ - Optional per-stage score floors for recall (all inclusive, AND-ed). ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL arms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` config for this request), so they prune weak matches before fusion. ``reranker`` and ``final`` are **post-query** filters applied to the scored results after reranking. Any field left None imposes no floor; all-None (the default) means no score filtering. + Optional per-stage score floors for recall. Every floor is inclusive (``>=``). The four floors act at two different levels, and the distinction decides what a returned result is guaranteed to satisfy. ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into their own SQL arm (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` config for this request), so they prune weak matches before fusion. Each one constrains **only the arm it names**. Recall fuses four arms — semantic, keyword, graph and temporal — and a result reaches the response if *any* arm surfaced it, so a returned result may legitimately carry ``null`` for a stage it was not surfaced by, and results reached through the graph or temporal arm carry neither ``semantic`` nor ``keyword``. A *non-null* score always clears its floor — the gap is only ever a ``null``. Setting both does **not** restrict the response to results that clear both: they are not a predicate over each fused result. This is deliberate — an intersection would discard the strong single-arm matches that hybrid retrieval exists to find (a paraphrase with no lexical overlap, an exact identifier the embedding scores poorly). ``reranker`` and ``final`` are **post-query** filters applied to every scored result after fusion and reranking, so these *are* per-result predicates: a returned result always clears them. Use them, not the retrieval floors, to make recall abstain on low-confidence queries. Any field left None imposes no floor; all-None (the default) means no score filtering. """ # noqa: E501 semantic: Optional[Union[StrictFloat, StrictInt]] = None keyword: Optional[Union[StrictFloat, StrictInt]] = None diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index 745edef27..7ffb2bdbe 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -3849,38 +3849,56 @@ export type MentalModelTriggerOutput = { /** * MinScores * - * Optional per-stage score floors for recall (all inclusive, AND-ed). + * Optional per-stage score floors for recall. Every floor is inclusive (``>=``). * - * ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL - * arms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` - * config for this request), so they prune weak matches before fusion. ``reranker`` - * and ``final`` are **post-query** filters applied to the scored results after - * reranking. Any field left None imposes no floor; all-None (the default) means - * no score filtering. + * The four floors act at two different levels, and the distinction decides what a + * returned result is guaranteed to satisfy. + * + * ``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into their own + * SQL arm (overriding the global ``semantic_min_similarity`` / ``bm25_min_score`` + * config for this request), so they prune weak matches before fusion. Each one + * constrains **only the arm it names**. Recall fuses four arms — semantic, keyword, + * graph and temporal — and a result reaches the response if *any* arm surfaced it, + * so a returned result may legitimately carry ``null`` for a stage it was not + * surfaced by, and results reached through the graph or temporal arm carry neither + * ``semantic`` nor ``keyword``. A *non-null* score always clears its floor — the + * gap is only ever a ``null``. Setting both does **not** restrict the response to + * results that clear both: they are not a predicate over each fused result. This is + * deliberate — an intersection would discard the strong single-arm matches that + * hybrid retrieval exists to find (a paraphrase with no lexical overlap, an exact + * identifier the embedding scores poorly). + * + * ``reranker`` and ``final`` are **post-query** filters applied to every scored + * result after fusion and reranking, so these *are* per-result predicates: a + * returned result always clears them. Use them, not the retrieval floors, to make + * recall abstain on low-confidence queries. + * + * Any field left None imposes no floor; all-None (the default) means no score + * filtering. */ export type MinScores = { /** * Semantic * - * Retrieval-level: minimum vector similarity (0-1). + * Retrieval-level, semantic arm only: minimum vector similarity (0-1). A result the semantic arm did not surface reports `semantic: null` and is unaffected by this floor. */ semantic?: number | null; /** * Keyword * - * Retrieval-level: minimum keyword/full-text (BM25) score. + * Retrieval-level, keyword arm only: minimum keyword/full-text (BM25) score. A result the keyword arm did not surface reports `keyword: null` and is unaffected by this floor. */ keyword?: number | null; /** * Reranker * - * Post-query: minimum normalized reranker score (0-1). + * Post-query: minimum normalized reranker score (0-1). Applied to every returned result. */ reranker?: number | null; /** * Final * - * Post-query: minimum final ranking score. + * Post-query: minimum final ranking score. Applied to every returned result. */ final?: number | null; }; @@ -4205,7 +4223,7 @@ export type RecallRequest = { */ tag_groups?: Array | null; /** - * Optional per-stage score floors (all inclusive, AND-ed). `semantic` and `keyword` are retrieval-level cutoffs pushed into the SQL arms (overriding the global similarity/BM25 minimums for this request); `reranker` and `final` are post-ranking filters on the scored results. Any field left unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use with care — the reranker's absolute scores are not calibrated across queries (a clearly-relevant match may score ~0.001 even though it is ranked first). + * Optional per-stage score floors, each inclusive (`>=`). `semantic` and `keyword` are retrieval-level cutoffs pushed into the SQL arm they name (overriding the global similarity/BM25 minimums for this request), and constrain only that arm: recall fuses four arms (semantic, keyword, graph, temporal) and returns a result surfaced by any of them, so a returned result reports null for a stage that did not surface it (a non-null score always clears its floor). Setting both therefore does not restrict the response to results clearing both. `reranker` and `final` are post-ranking filters applied to every scored result, so those floors *are* guaranteed by each result returned — use them for query abstention. Any field left unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use with care — the reranker's absolute scores are not calibrated across queries (a clearly-relevant match may score ~0.001 even though it is ranked first). */ min_scores?: MinScores | null; /** diff --git a/hindsight-docs/docs/developer/api/recall.mdx b/hindsight-docs/docs/developer/api/recall.mdx index a43bc585f..a5765cace 100644 --- a/hindsight-docs/docs/developer/api/recall.mdx +++ b/hindsight-docs/docs/developer/api/recall.mdx @@ -404,24 +404,36 @@ When set to `true`, the response includes a detailed debug trace covering the qu ### min_scores -An optional object of per-stage score floors, each compared **inclusively** (`>=`) against the matching field of a result's [`scores`](#scores) and AND-ed together. Any field you leave unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering at all. The four fields operate at **two different levels of the pipeline**: +An optional object of per-stage score floors, each compared **inclusively** (`>=`). Any field you leave unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering at all. The four fields operate at **two different levels of the pipeline**, and the level decides what a returned result is guaranteed to satisfy: -| field | level | effect | -|---|---|---| -| `semantic` | retrieval | minimum vector similarity, pushed into the SQL — prunes weak vector matches **before** fusion (overrides the global similarity minimum for this request) | -| `keyword` | retrieval | minimum keyword/full-text (BM25) score, pushed into the SQL — prunes weak keyword matches before fusion | -| `reranker` | post-query | minimum normalized cross-encoder score, applied to the ranked results | -| `final` | post-query | minimum final ranking score, applied to the ranked results | +| field | level | effect | guaranteed by every result? | +|---|---|---|---| +| `semantic` | retrieval | minimum vector similarity, pushed into the **semantic arm's** SQL — prunes weak vector matches **before** fusion (overrides the global similarity minimum for this request) | no | +| `keyword` | retrieval | minimum keyword/full-text (BM25) score, pushed into the **keyword arm's** SQL — prunes weak keyword matches before fusion | no | +| `reranker` | post-query | minimum normalized cross-encoder score, applied to the ranked results | yes | +| `final` | post-query | minimum final ranking score, applied to the ranked results | yes | ```json { "query": "...", "min_scores": { "reranker": 0.5 } } ``` -The retrieval-level floors (`semantic`/`keyword`) change *which candidates are considered*, so they can also change the final ordering; the post-query floors (`reranker`/`final`) only drop already-ranked results. Because freed slots are **not** backfilled, any floor can return fewer results than the budget allows. +#### Retrieval floors constrain one arm, not the result -**Use floors with care.** The reranker's scores are reliable for *ordering* but not as *absolute* values — a clearly-relevant memory can score `~0.001` on one query and `~1.0` on another, so a fixed cutoff risks silently dropping good results. Calibrate any threshold against the scores you actually observe (recall with no `min_scores` first and inspect the [`scores`](#scores) object). +Recall runs [four retrieval arms](#results) — semantic, keyword, graph and temporal — and a memory reaches the response if **any** of them surfaced it. `semantic` and `keyword` prune inside the arm they name, so they change *which candidates are considered*, and with them the final ordering. They are **not predicates over each returned result**: -Each threshold is compared against the matching field in the response [`scores`](#scores) object. See the note under [`scores`](#scores) on why the scale is relative, not absolute, before relying on a fixed threshold. +- a result surfaced only semantically reports `"keyword": null`, whatever `min_scores.keyword` you set; +- a result surfaced only by keyword reports `"semantic": null`, whatever `min_scores.semantic` you set; +- a result reached through the graph or temporal arm reports **neither**, and is unaffected by both floors. + +Setting `semantic` and `keyword` together therefore does not restrict the response to results that clear both. That is deliberate: an intersection would discard exactly the strong single-arm matches hybrid retrieval exists to find — a paraphrase with no lexical overlap in common with the query, or an exact identifier like `amber-17` that the embedding scores poorly. + +#### For abstention, use `reranker` or `final` + +The post-query floors are applied to every scored result after fusion and reranking, so a returned result always clears them — and a query where nothing clears them returns no results. That is the floor to reach for when you want recall to abstain on a low-confidence or nonsense query. Note they gate a *combined* signal: `final` blends RRF rank, cross-encoder relevance, recency/temporal and strategy boosts, and `reranker` depends on the cross-encoder's calibration, so neither is a drop-in equivalent of a retrieval-stage cutoff. + +Because freed slots are **not** backfilled, any floor can return fewer results than the budget allows. + +**Use floors with care.** The reranker's scores are reliable for *ordering* but not as *absolute* values — a clearly-relevant memory can score `~0.001` on one query and `~1.0` on another, so a fixed cutoff risks silently dropping good results. Calibrate any threshold against the scores you actually observe (recall with no `min_scores` first and inspect the [`scores`](#scores) object). See the note under [`scores`](#scores) on why the scale is relative, not absolute, before relying on a fixed threshold. --- @@ -492,7 +504,7 @@ An object of the per-stage scores for this result. `null` for `source_facts` ent - **`semantic`** — the raw vector cosine similarity (`0`–`1`). `null` if this result was not surfaced by semantic search. - **`keyword`** — the raw keyword/full-text (BM25) score (`≥ 0`, unbounded). `null` if this result was not surfaced by keyword search. -Each field is also a valid [`min_scores`](#min_scores) floor. +Each field is also a valid [`min_scores`](#min_scores) floor — but `semantic` and `keyword` gate their own retrieval arm rather than the returned result, so a `null` here is expected even when you set that floor. A non-null value always clears it. See [`min_scores`](#min_scores). --- diff --git a/hindsight-docs/static/openapi.json b/hindsight-docs/static/openapi.json index 2ebcf0228..2b3df71ed 100644 --- a/hindsight-docs/static/openapi.json +++ b/hindsight-docs/static/openapi.json @@ -13104,7 +13104,7 @@ } ], "title": "Semantic", - "description": "Retrieval-level: minimum vector similarity (0-1)." + "description": "Retrieval-level, semantic arm only: minimum vector similarity (0-1). A result the semantic arm did not surface reports `semantic: null` and is unaffected by this floor." }, "keyword": { "anyOf": [ @@ -13116,7 +13116,7 @@ } ], "title": "Keyword", - "description": "Retrieval-level: minimum keyword/full-text (BM25) score." + "description": "Retrieval-level, keyword arm only: minimum keyword/full-text (BM25) score. A result the keyword arm did not surface reports `keyword: null` and is unaffected by this floor." }, "reranker": { "anyOf": [ @@ -13128,7 +13128,7 @@ } ], "title": "Reranker", - "description": "Post-query: minimum normalized reranker score (0-1)." + "description": "Post-query: minimum normalized reranker score (0-1). Applied to every returned result." }, "final": { "anyOf": [ @@ -13140,12 +13140,12 @@ } ], "title": "Final", - "description": "Post-query: minimum final ranking score." + "description": "Post-query: minimum final ranking score. Applied to every returned result." } }, "type": "object", "title": "MinScores", - "description": "Optional per-stage score floors for recall (all inclusive, AND-ed).\n\n``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL\narms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score``\nconfig for this request), so they prune weak matches before fusion. ``reranker``\nand ``final`` are **post-query** filters applied to the scored results after\nreranking. Any field left None imposes no floor; all-None (the default) means\nno score filtering." + "description": "Optional per-stage score floors for recall. Every floor is inclusive (``>=``).\n\nThe four floors act at two different levels, and the distinction decides what a\nreturned result is guaranteed to satisfy.\n\n``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into their own\nSQL arm (overriding the global ``semantic_min_similarity`` / ``bm25_min_score``\nconfig for this request), so they prune weak matches before fusion. Each one\nconstrains **only the arm it names**. Recall fuses four arms \u2014 semantic, keyword,\ngraph and temporal \u2014 and a result reaches the response if *any* arm surfaced it,\nso a returned result may legitimately carry ``null`` for a stage it was not\nsurfaced by, and results reached through the graph or temporal arm carry neither\n``semantic`` nor ``keyword``. A *non-null* score always clears its floor \u2014 the\ngap is only ever a ``null``. Setting both does **not** restrict the response to\nresults that clear both: they are not a predicate over each fused result. This is\ndeliberate \u2014 an intersection would discard the strong single-arm matches that\nhybrid retrieval exists to find (a paraphrase with no lexical overlap, an exact\nidentifier the embedding scores poorly).\n\n``reranker`` and ``final`` are **post-query** filters applied to every scored\nresult after fusion and reranking, so these *are* per-result predicates: a\nreturned result always clears them. Use them, not the retrieval floors, to make\nrecall abstain on low-confidence queries.\n\nAny field left None imposes no floor; all-None (the default) means no score\nfiltering." }, "ObservationScope": { "properties": { @@ -13769,7 +13769,7 @@ "type": "null" } ], - "description": "Optional per-stage score floors (all inclusive, AND-ed). `semantic` and `keyword` are retrieval-level cutoffs pushed into the SQL arms (overriding the global similarity/BM25 minimums for this request); `reranker` and `final` are post-ranking filters on the scored results. Any field left unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use with care \u2014 the reranker's absolute scores are not calibrated across queries (a clearly-relevant match may score ~0.001 even though it is ranked first)." + "description": "Optional per-stage score floors, each inclusive (`>=`). `semantic` and `keyword` are retrieval-level cutoffs pushed into the SQL arm they name (overriding the global similarity/BM25 minimums for this request), and constrain only that arm: recall fuses four arms (semantic, keyword, graph, temporal) and returns a result surfaced by any of them, so a returned result reports null for a stage that did not surface it (a non-null score always clears its floor). Setting both therefore does not restrict the response to results clearing both. `reranker` and `final` are post-ranking filters applied to every scored result, so those floors *are* guaranteed by each result returned \u2014 use them for query abstention. Any field left unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use with care \u2014 the reranker's absolute scores are not calibrated across queries (a clearly-relevant match may score ~0.001 even though it is ranked first)." }, "temporal_window": { "anyOf": [ diff --git a/skills/hindsight-docs/references/developer/api/recall.md b/skills/hindsight-docs/references/developer/api/recall.md index ff62de295..55511d153 100644 --- a/skills/hindsight-docs/references/developer/api/recall.md +++ b/skills/hindsight-docs/references/developer/api/recall.md @@ -646,24 +646,36 @@ When set to `true`, the response includes a detailed debug trace covering the qu ### min_scores -An optional object of per-stage score floors, each compared **inclusively** (`>=`) against the matching field of a result's [`scores`](#scores) and AND-ed together. Any field you leave unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering at all. The four fields operate at **two different levels of the pipeline**: +An optional object of per-stage score floors, each compared **inclusively** (`>=`). Any field you leave unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering at all. The four fields operate at **two different levels of the pipeline**, and the level decides what a returned result is guaranteed to satisfy: -| field | level | effect | -|---|---|---| -| `semantic` | retrieval | minimum vector similarity, pushed into the SQL — prunes weak vector matches **before** fusion (overrides the global similarity minimum for this request) | -| `keyword` | retrieval | minimum keyword/full-text (BM25) score, pushed into the SQL — prunes weak keyword matches before fusion | -| `reranker` | post-query | minimum normalized cross-encoder score, applied to the ranked results | -| `final` | post-query | minimum final ranking score, applied to the ranked results | +| field | level | effect | guaranteed by every result? | +|---|---|---|---| +| `semantic` | retrieval | minimum vector similarity, pushed into the **semantic arm's** SQL — prunes weak vector matches **before** fusion (overrides the global similarity minimum for this request) | no | +| `keyword` | retrieval | minimum keyword/full-text (BM25) score, pushed into the **keyword arm's** SQL — prunes weak keyword matches before fusion | no | +| `reranker` | post-query | minimum normalized cross-encoder score, applied to the ranked results | yes | +| `final` | post-query | minimum final ranking score, applied to the ranked results | yes | ```json { "query": "...", "min_scores": { "reranker": 0.5 } } ``` -The retrieval-level floors (`semantic`/`keyword`) change *which candidates are considered*, so they can also change the final ordering; the post-query floors (`reranker`/`final`) only drop already-ranked results. Because freed slots are **not** backfilled, any floor can return fewer results than the budget allows. +#### Retrieval floors constrain one arm, not the result -**Use floors with care.** The reranker's scores are reliable for *ordering* but not as *absolute* values — a clearly-relevant memory can score `~0.001` on one query and `~1.0` on another, so a fixed cutoff risks silently dropping good results. Calibrate any threshold against the scores you actually observe (recall with no `min_scores` first and inspect the [`scores`](#scores) object). +Recall runs [four retrieval arms](#results) — semantic, keyword, graph and temporal — and a memory reaches the response if **any** of them surfaced it. `semantic` and `keyword` prune inside the arm they name, so they change *which candidates are considered*, and with them the final ordering. They are **not predicates over each returned result**: -Each threshold is compared against the matching field in the response [`scores`](#scores) object. See the note under [`scores`](#scores) on why the scale is relative, not absolute, before relying on a fixed threshold. +- a result surfaced only semantically reports `"keyword": null`, whatever `min_scores.keyword` you set; +- a result surfaced only by keyword reports `"semantic": null`, whatever `min_scores.semantic` you set; +- a result reached through the graph or temporal arm reports **neither**, and is unaffected by both floors. + +Setting `semantic` and `keyword` together therefore does not restrict the response to results that clear both. That is deliberate: an intersection would discard exactly the strong single-arm matches hybrid retrieval exists to find — a paraphrase with no lexical overlap in common with the query, or an exact identifier like `amber-17` that the embedding scores poorly. + +#### For abstention, use `reranker` or `final` + +The post-query floors are applied to every scored result after fusion and reranking, so a returned result always clears them — and a query where nothing clears them returns no results. That is the floor to reach for when you want recall to abstain on a low-confidence or nonsense query. Note they gate a *combined* signal: `final` blends RRF rank, cross-encoder relevance, recency/temporal and strategy boosts, and `reranker` depends on the cross-encoder's calibration, so neither is a drop-in equivalent of a retrieval-stage cutoff. + +Because freed slots are **not** backfilled, any floor can return fewer results than the budget allows. + +**Use floors with care.** The reranker's scores are reliable for *ordering* but not as *absolute* values — a clearly-relevant memory can score `~0.001` on one query and `~1.0` on another, so a fixed cutoff risks silently dropping good results. Calibrate any threshold against the scores you actually observe (recall with no `min_scores` first and inspect the [`scores`](#scores) object). See the note under [`scores`](#scores) on why the scale is relative, not absolute, before relying on a fixed threshold. --- @@ -734,7 +746,7 @@ An object of the per-stage scores for this result. `null` for `source_facts` ent - **`semantic`** — the raw vector cosine similarity (`0`–`1`). `null` if this result was not surfaced by semantic search. - **`keyword`** — the raw keyword/full-text (BM25) score (`≥ 0`, unbounded). `null` if this result was not surfaced by keyword search. -Each field is also a valid [`min_scores`](#min_scores) floor. +Each field is also a valid [`min_scores`](#min_scores) floor — but `semantic` and `keyword` gate their own retrieval arm rather than the returned result, so a `null` here is expected even when you set that floor. A non-null value always clears it. See [`min_scores`](#min_scores). --- diff --git a/skills/hindsight-docs/references/openapi.json b/skills/hindsight-docs/references/openapi.json index 2ebcf0228..2b3df71ed 100644 --- a/skills/hindsight-docs/references/openapi.json +++ b/skills/hindsight-docs/references/openapi.json @@ -13104,7 +13104,7 @@ } ], "title": "Semantic", - "description": "Retrieval-level: minimum vector similarity (0-1)." + "description": "Retrieval-level, semantic arm only: minimum vector similarity (0-1). A result the semantic arm did not surface reports `semantic: null` and is unaffected by this floor." }, "keyword": { "anyOf": [ @@ -13116,7 +13116,7 @@ } ], "title": "Keyword", - "description": "Retrieval-level: minimum keyword/full-text (BM25) score." + "description": "Retrieval-level, keyword arm only: minimum keyword/full-text (BM25) score. A result the keyword arm did not surface reports `keyword: null` and is unaffected by this floor." }, "reranker": { "anyOf": [ @@ -13128,7 +13128,7 @@ } ], "title": "Reranker", - "description": "Post-query: minimum normalized reranker score (0-1)." + "description": "Post-query: minimum normalized reranker score (0-1). Applied to every returned result." }, "final": { "anyOf": [ @@ -13140,12 +13140,12 @@ } ], "title": "Final", - "description": "Post-query: minimum final ranking score." + "description": "Post-query: minimum final ranking score. Applied to every returned result." } }, "type": "object", "title": "MinScores", - "description": "Optional per-stage score floors for recall (all inclusive, AND-ed).\n\n``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into the SQL\narms (overriding the global ``semantic_min_similarity`` / ``bm25_min_score``\nconfig for this request), so they prune weak matches before fusion. ``reranker``\nand ``final`` are **post-query** filters applied to the scored results after\nreranking. Any field left None imposes no floor; all-None (the default) means\nno score filtering." + "description": "Optional per-stage score floors for recall. Every floor is inclusive (``>=``).\n\nThe four floors act at two different levels, and the distinction decides what a\nreturned result is guaranteed to satisfy.\n\n``semantic`` and ``keyword`` are **retrieval-level** cutoffs pushed into their own\nSQL arm (overriding the global ``semantic_min_similarity`` / ``bm25_min_score``\nconfig for this request), so they prune weak matches before fusion. Each one\nconstrains **only the arm it names**. Recall fuses four arms \u2014 semantic, keyword,\ngraph and temporal \u2014 and a result reaches the response if *any* arm surfaced it,\nso a returned result may legitimately carry ``null`` for a stage it was not\nsurfaced by, and results reached through the graph or temporal arm carry neither\n``semantic`` nor ``keyword``. A *non-null* score always clears its floor \u2014 the\ngap is only ever a ``null``. Setting both does **not** restrict the response to\nresults that clear both: they are not a predicate over each fused result. This is\ndeliberate \u2014 an intersection would discard the strong single-arm matches that\nhybrid retrieval exists to find (a paraphrase with no lexical overlap, an exact\nidentifier the embedding scores poorly).\n\n``reranker`` and ``final`` are **post-query** filters applied to every scored\nresult after fusion and reranking, so these *are* per-result predicates: a\nreturned result always clears them. Use them, not the retrieval floors, to make\nrecall abstain on low-confidence queries.\n\nAny field left None imposes no floor; all-None (the default) means no score\nfiltering." }, "ObservationScope": { "properties": { @@ -13769,7 +13769,7 @@ "type": "null" } ], - "description": "Optional per-stage score floors (all inclusive, AND-ed). `semantic` and `keyword` are retrieval-level cutoffs pushed into the SQL arms (overriding the global similarity/BM25 minimums for this request); `reranker` and `final` are post-ranking filters on the scored results. Any field left unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use with care \u2014 the reranker's absolute scores are not calibrated across queries (a clearly-relevant match may score ~0.001 even though it is ranked first)." + "description": "Optional per-stage score floors, each inclusive (`>=`). `semantic` and `keyword` are retrieval-level cutoffs pushed into the SQL arm they name (overriding the global similarity/BM25 minimums for this request), and constrain only that arm: recall fuses four arms (semantic, keyword, graph, temporal) and returns a result surfaced by any of them, so a returned result reports null for a stage that did not surface it (a non-null score always clears its floor). Setting both therefore does not restrict the response to results clearing both. `reranker` and `final` are post-ranking filters applied to every scored result, so those floors *are* guaranteed by each result returned \u2014 use them for query abstention. Any field left unset imposes no floor; omitting `min_scores` entirely (the default) applies no score filtering. Use with care \u2014 the reranker's absolute scores are not calibrated across queries (a clearly-relevant match may score ~0.001 even though it is ranked first)." }, "temporal_window": { "anyOf": [