test(system): stories 24, 54, 92 — observation scopes, templates/preview, chunking

- 24 observation_scopes decides which observations a multi-tag memory feeds. The
  isolation is the point: a lesson tagged for a student and a teacher must not
  produce one observation belonging to neither, and consolidation's all_strict
  matching is what keeps one party's observations out of another's. Both
  documented spellings are pinned — [[]] is one global scope, [] is *zero* and
  falls back to combined, one character apart with no error either way.
- 54 templates carry configuration and not memories, leave untouched fields
  unset rather than freezing today's defaults, and the prompt preview matches
  what a real retain actually sends. Also pins that retain_custom_instructions
  is only consulted in `custom` extraction mode — stored, returned on read, and
  silently unused otherwise.
- 92 chunking: contiguous indexes, reassembly loses nothing at the seams, one
  extraction call per chunk (fewer means a piece was never read, more means
  paying twice), and the stored document stays byte-identical because it is what
  a reprocess re-reads.

Fixed a real hole in the harness: the client was a path dependency installed
non-editable, so the venv held a *copy* taken whenever it was last built. The
suite had been testing a stale snapshot — preview_prompt exists in the repo's
client and was simply absent from the installed one. Now editable.

Two template tests are skipped rather than rewritten against raw HTTP: the
import endpoint cannot be called from any SDK (#4232). Reaching around the
client to make them pass would hide exactly the defect a client-driven suite
exists to surface.

hindsight-system-tests/pyproject.toml

@@ -20,7 +20,11 @@ dependencies = [
 ]
 
 [tool.uv.sources]
-hindsight-client = { path = "../hindsight-clients/python" }
+# editable: without it uv installs a *copy*, and the suite silently tests a
+# snapshot of the client taken whenever the venv was last built — an endpoint
+# added or regenerated since then simply is not there. The whole point of driving
+# the published client is that it is the current one.
+hindsight-client = { path = "../hindsight-clients/python", editable = true }
 
 [tool.hatch.build.targets.wheel]
 packages = ["hindsight_system_tests"]

hindsight-system-tests/tests/test_24_observation_scopes.py

@@ -0,0 +1,143 @@
+"""One memory, several tags — and control over which observations it feeds.
+
+By default a memory's tags are taken together: one scope, one observation. That
+is wrong for the common multi-party case. A lesson transcript tagged
+`student:alice` and `teacher:bob` is evidence about *both* of them, and a single
+observation tagged with both belongs to neither — recalling Alice's history
+would surface an observation that is also about Bob, and vice versa.
+
+`observation_scopes` decides the boundary, and the isolation between scopes is
+the load-bearing property: consolidation matches existing observations with
+`all_strict`, so a pass scoped to `["student:alice"]` can only ever update an
+observation tagged exactly that. Weaken it and one student's observations start
+absorbing another's evidence — a privacy failure, not a ranking one.
+
+The last test pins a documented footgun: `[]` means *zero* scopes and silently
+falls back to `combined`, while `[[]]` (and `"shared"`) means one global scope.
+Two spellings one character apart, opposite meanings, no error either way.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from hindsight_system_tests.payloads import Consolidation, Observation, extracted, fact, fact_ids_in
+
+pytestmark = pytest.mark.asyncio
+
+STUDENT = "student:alice"
+TEACHER = "teacher:bob"
+SESSION = "session-id:s1"
+
+CONTENT = "Alice practised scales for an hour."
+OBSERVATION = "Alice is practising regularly"
+
+
+def _observe(text: str):
+    """One observation per consolidation pass, over whatever facts it was shown."""
+
+    def build(request) -> Consolidation:
+        return Consolidation(
+            creates=[Observation(text=text, source_fact_ids=fact_ids_in(request.all_text), reason="system test")]
+        )
+
+    return build
+
+
+@pytest.fixture(autouse=True)
+def _extraction(llm):
+    llm.on_step("extract_facts").returns(
+        extracted(fact("Alice practised scales for an hour", who="Alice", entities=["Alice"]))
+    )
+    llm.on_step("consolidate").answers_with(_observe(OBSERVATION))
+
+
+async def _retain(client, bank: str, settled, **item) -> None:
+    await client.aretain_batch(bank_id=bank, items=[{"content": CONTENT, **item}])
+    await settled(bank)
+
+
+async def _observation_tags(client, bank: str) -> list[list[str]]:
+    memories = await client.memory.list_memories(bank, limit=100)
+    return sorted(
+        (sorted(m["tags"]) for m in memories.items if m["fact_type"] == "observation"),
+        key=lambda tags: tags,
+    )
+
+
+async def test_by_default_the_tags_are_one_scope(client, bank_id, settled):
+    """`combined`: the memory's tags taken together, one observation carrying
+    both — the behaviour the other modes exist to change."""
+    await _retain(client, bank_id, settled, tags=[STUDENT, TEACHER])
+
+    assert await _observation_tags(client, bank_id) == [sorted([STUDENT, TEACHER])]
+
+
+async def test_per_tag_gives_each_tag_its_own_observation(client, bank_id, settled):
+    """The multi-party case. Two observations, each tagged with only its own
+    party — so Alice's history and Bob's are separately recallable."""
+    await _retain(client, bank_id, settled, tags=[STUDENT, TEACHER], observation_scopes="per_tag")
+
+    assert await _observation_tags(client, bank_id) == [[STUDENT], [TEACHER]]
+
+
+async def test_shared_puts_everything_in_one_untagged_scope(client, bank_id, settled):
+    """For tags that are provenance rather than a boundary — a session id you
+    want for filtering and debugging, but which must not fragment the
+    observations into one per session."""
+    await _retain(client, bank_id, settled, tags=[STUDENT, SESSION], observation_scopes="shared")
+
+    assert await _observation_tags(client, bank_id) == [[]]
+
+
+async def test_an_explicit_scope_list_takes_exactly_those_combinations(client, bank_id, settled):
+    """Full control: one pass per inner list, and nothing else."""
+    await _retain(
+        client, bank_id, settled, tags=[STUDENT, TEACHER, SESSION], observation_scopes=[[STUDENT], [STUDENT, TEACHER]]
+    )
+
+    assert await _observation_tags(client, bank_id) == [[STUDENT], sorted([STUDENT, TEACHER])]
+
+
+async def test_the_bank_lists_the_scopes_it_holds(client, bank_id, settled):
+    """So an operator can see how a bank is partitioned without inferring it from
+    the observations themselves."""
+    await _retain(client, bank_id, settled, tags=[STUDENT, TEACHER], observation_scopes="per_tag")
+
+    scopes = await client.memory.list_observation_scopes(bank_id)
+
+    assert scopes.total == 2
+    assert sorted(sorted(s.tags) for s in scopes.scopes) == [[STUDENT], [TEACHER]]
+    assert all(s.count == 1 for s in scopes.scopes)
+
+
+async def test_one_scope_is_recallable_without_the_other(client, bank_id, settled):
+    """The isolation property, seen from the read side.
+
+    `exact` matching is what reaches a precise scope: `any_strict` on
+    `[STUDENT]` would also return an observation tagged with both, which is the
+    cross-party leak the scopes exist to prevent.
+    """
+    await _retain(client, bank_id, settled, tags=[STUDENT, TEACHER], observation_scopes="per_tag")
+
+    response = await client.arecall(
+        bank_id=bank_id, query="How is Alice doing?", types=["observation"], tags=[STUDENT], tags_match="exact"
+    )
+
+    assert [sorted(r.tags) for r in response.results] == [[STUDENT]]
+
+
+async def test_an_empty_scope_list_is_not_the_global_scope(client, bank_id, settled):
+    """The documented footgun, pinned because both spellings are silent.
+
+    `[[]]` — a list holding one empty scope — means "one global, untagged scope".
+    `[]` means *zero* scopes, which cannot be honoured, so it falls back to
+    `combined`. One character apart, opposite results, no error either way: a
+    caller who meant "shared" and typed `[]` gets tag-scoped observations and no
+    hint that anything was ignored.
+    """
+    await _retain(client, bank_id, settled, tags=[STUDENT, TEACHER], observation_scopes=[])
+
+    assert await _observation_tags(client, bank_id) == [sorted([STUDENT, TEACHER])], (
+        "an empty scope list must fall back to combined, not to the global scope"
+    )

hindsight-system-tests/tests/test_54_templates_and_preview.py

@@ -0,0 +1,209 @@
+"""A bank's configuration can be exported as a template, and inspected as prompts.
+
+These are the two ways a caller gets at "how is this bank set up" without
+guessing.
+
+A **template** is the configuration without the memories: what makes one bank
+behave like another. Its value is entirely in fidelity — a template that drops
+a field produces a bank that looks configured and behaves differently, and the
+difference shows up as an agent quietly reasoning the wrong way rather than as
+an error.
+
+A **prompt preview** is the same question from the other end: not what the
+settings are, but what the model will actually be told. It renders the prompts
+for an operation without spending an LLM call, which is the only way to answer
+"is my custom instruction actually in there?" before paying to find out.
+"""
+
+from __future__ import annotations
+
+import uuid
+from collections.abc import AsyncIterator
+
+import pytest
+from hindsight_client_api.api.bank_templates_api import BankTemplatesApi
+
+pytestmark = pytest.mark.asyncio
+
+CUSTOM_INSTRUCTION = "Always record the neighbourhood, not just the city."
+
+
+@pytest.fixture
+async def configured_bank(client, bank_id) -> str:
+    await client.acreate_bank(bank_id=bank_id, name="Alice", disposition_skepticism=5, disposition_empathy=1)
+    await client.banks.update_bank_config(
+        bank_id,
+        {
+            "updates": {
+                "retain_custom_instructions": CUSTOM_INSTRUCTION,
+                # Load-bearing: the instruction is only consulted in `custom`
+                # extraction mode. See the test at the bottom of this file.
+                "retain_extraction_mode": "custom",
+                "enable_reranking": False,
+            }
+        },
+    )
+    return bank_id
+
+
+@pytest.fixture
+async def fresh_bank(client) -> AsyncIterator[str]:
+    bank = f"systest-{uuid.uuid4().hex[:12]}"
+    yield bank
+    await client.banks.delete_bank(bank)
+
+
+def _templates(client) -> BankTemplatesApi:
+    return BankTemplatesApi(client.banks.api_client)
+
+
+async def test_a_template_carries_the_settings_that_were_changed(client, configured_bank):
+    """Fidelity is the whole feature. A field dropped here produces a bank that
+    looks configured and behaves differently."""
+    template = await _templates(client).export_bank_template(configured_bank)
+
+    assert template.version == "1"
+    assert template.bank.disposition_skepticism == 5
+    assert template.bank.disposition_empathy == 1
+    assert template.bank.retain_custom_instructions == CUSTOM_INSTRUCTION
+
+
+async def test_a_template_leaves_untouched_settings_unset(client, configured_bank):
+    """Absent, not defaulted. A template that materialised every field as its
+    current default would freeze today's defaults into every bank made from it —
+    and silently stop those banks tracking a later change to them.
+    """
+    template = await _templates(client).export_bank_template(configured_bank)
+
+    assert template.bank.disposition_literalism is None
+    assert template.bank.retain_mission is None
+
+
+@pytest.mark.skip(reason="#4232 — import_bank_template cannot send the manifest from any SDK")
+async def test_a_template_applied_to_a_new_bank_reproduces_the_configuration(client, configured_bank, fresh_bank):
+    """The round trip that makes templates worth having: configure once, stamp
+    out many.
+
+    Skipped rather than deleted, and skipped rather than rewritten against raw
+    HTTP. The import handler reads its body off the raw request, so the spec
+    declares no `requestBody` and every generated client's
+    `import_bank_template` has no parameter to put the manifest in (#4232).
+    Reaching around the client to make this pass would hide exactly the defect a
+    suite driven through the published client exists to surface — the export
+    half works and returns a typed manifest with nowhere to send it.
+    """
+    template = await _templates(client).export_bank_template(configured_bank)
+
+    await _templates(client).import_bank_template(fresh_bank, template)
+
+    config = (await client.banks.get_bank_config(fresh_bank)).config
+    assert config["retain_custom_instructions"] == CUSTOM_INSTRUCTION
+    assert config["enable_reranking"] is False
+    assert config["disposition_skepticism"] == 5
+
+
+@pytest.mark.skip(reason="#4232 — import_bank_template cannot send the manifest from any SDK")
+async def test_a_template_carries_no_memories(client, llm, configured_bank, fresh_bank, settled):
+    """Configuration, not content. A template that dragged the source bank's
+    memories along would leak one tenant's data into every bank stamped from it.
+    """
+    from hindsight_system_tests.payloads import consolidation, extracted, fact
+
+    llm.on_step("extract_facts").returns(
+        extracted(fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]))
+    )
+    llm.on_step("consolidate").returns(consolidation())
+    await client.aretain(bank_id=configured_bank, content="Alice moved to Berlin.")
+    await settled(configured_bank)
+
+    template = await _templates(client).export_bank_template(configured_bank)
+    await _templates(client).import_bank_template(fresh_bank, template)
+
+    memories = await client.memory.list_memories(fresh_bank, limit=100)
+    assert memories.items == []
+
+
+async def test_the_prompt_preview_renders_what_the_model_will_be_told(client, configured_bank):
+    """Answers "is my instruction actually in the prompt?" without paying for a
+    call to find out — the question a config UI exists to answer."""
+    preview = await client.banks.preview_prompt(configured_bank, {"operation": "retain"})
+
+    assert preview.messages, "a preview with no messages tells the caller nothing"
+    # A message is a list of blocks, not a flat string — an image-capable prompt
+    # is interleaved content, so the text has to be gathered out of the blocks.
+    rendered = "\n".join(block.text or "" for message in preview.messages for block in message.blocks)
+    assert CUSTOM_INSTRUCTION in rendered
+
+
+async def test_the_preview_spends_no_llm_call(client, llm, configured_bank):
+    """It is a rendering, not a dry run. Nothing is scripted for a retain here,
+    so a preview that reached the model would arrive unstubbed and fail."""
+    before = len(llm.calls)
+
+    await client.banks.preview_prompt(configured_bank, {"operation": "retain"})
+
+    assert len(llm.calls) == before
+
+
+async def test_the_preview_matches_the_prompt_a_real_retain_sends(client, llm, configured_bank, settled):
+    """The property that makes a preview worth trusting.
+
+    A preview that renders its own approximation is worse than none: someone
+    tunes a prompt against it, ships, and the model reads something else. So the
+    instruction is looked for in *both* — the rendered preview and the prompt the
+    stub actually received from a real retain.
+    """
+    from hindsight_system_tests.payloads import consolidation, extracted, fact
+
+    llm.on_step("extract_facts").returns(
+        extracted(fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]))
+    )
+    llm.on_step("consolidate").returns(consolidation())
+
+    preview = await client.banks.preview_prompt(configured_bank, {"operation": "retain"})
+    rendered = "\n".join(block.text or "" for message in preview.messages for block in message.blocks)
+
+    await client.aretain(bank_id=configured_bank, content="Alice moved to Berlin.")
+    await settled(configured_bank)
+    sent = llm.prompts_for("extract_facts")
+
+    assert CUSTOM_INSTRUCTION in rendered
+    assert any(CUSTOM_INSTRUCTION in prompt for prompt in sent), (
+        "the preview showed an instruction the retain never sent"
+    )
+
+
+async def test_a_custom_instruction_is_ignored_outside_custom_mode(client, llm, bank_id, settled):
+    """The footgun, pinned in both halves.
+
+    `retain_custom_instructions` is only consulted when `retain_extraction_mode`
+    is `custom`. Set the instruction and leave the mode at its default and it is
+    stored, returned on read, and never sent — no warning, no error, and a
+    caller reasonably concluding their instruction is in force.
+
+    The preview is *faithful* about this, which is the saving grace: it shows the
+    instruction missing, so the config UI tells the truth even though the config
+    read does not.
+    """
+    from hindsight_system_tests.payloads import consolidation, extracted, fact
+
+    await client.acreate_bank(bank_id=bank_id, name="Alice")
+    await client.banks.update_bank_config(bank_id, {"updates": {"retain_custom_instructions": CUSTOM_INSTRUCTION}})
+
+    # Stored and read back — which is all the config API tells you.
+    config = (await client.banks.get_bank_config(bank_id)).config
+    assert config["retain_custom_instructions"] == CUSTOM_INSTRUCTION
+    assert config["retain_extraction_mode"] != "custom"
+
+    llm.on_step("extract_facts").returns(
+        extracted(fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]))
+    )
+    llm.on_step("consolidate").returns(consolidation())
+    await client.aretain(bank_id=bank_id, content="Alice moved to Berlin.")
+    await settled(bank_id)
+
+    assert not any(CUSTOM_INSTRUCTION in prompt for prompt in llm.prompts_for("extract_facts"))
+
+    preview = await client.banks.preview_prompt(bank_id, {"operation": "retain"})
+    rendered = "\n".join(block.text or "" for message in preview.messages for block in message.blocks)
+    assert CUSTOM_INSTRUCTION not in rendered

hindsight-system-tests/tests/test_92_chunking.py

@@ -0,0 +1,125 @@
+"""Long content is split, and every piece is extracted from exactly once.
+
+A document larger than the chunk size is cut up before extraction. Three things
+have to hold, and each fails differently.
+
+Every chunk must be *extracted from* — a splitter that drops the tail loses the
+end of every long document, and the loss is invisible because the document
+still stores the full text. Every chunk must be extracted from **once** — a
+double pass duplicates facts and doubles the model spend.
+
+And the stored document must remain byte-identical to what was sent, because it
+is the input a reprocess re-reads: a document that persisted only its last slice
+would silently shrink on the next re-extraction. That is not hypothetical — it
+is the shape of a bug this project has already hit.
+"""
+
+from __future__ import annotations
+
+import re
+
+import pytest
+
+from hindsight_system_tests.payloads import consolidation, extracted, fact
+
+pytestmark = pytest.mark.asyncio
+
+CHUNK_SIZE = 200
+
+# Each sentence carries its own index, so a fact can name the piece it came from
+# and a dropped or repeated chunk is visible by its absence or duplication.
+SENTENCES = [f"Marker{i:03d} is a sentence about Alice and Berlin and the cello." for i in range(60)]
+CONTENT = " ".join(SENTENCES)
+
+_MARKER = re.compile(r"Marker(\d{3})")
+
+
+@pytest.fixture
+async def chunked_document(client, llm, bank_id, settled) -> str:
+    await client.acreate_bank(bank_id=bank_id, name="Alice")
+    await client.banks.update_bank_config(bank_id, {"updates": {"retain_chunk_size": CHUNK_SIZE}})
+
+    def one_fact_naming_the_first_marker(request):
+        """Answer each extraction call with a fact naming the chunk it was shown.
+
+        The stub cannot know how the server split the text, so it reads the first
+        marker out of whatever it was handed — which makes the set of facts a
+        direct readout of which pieces reached the model.
+        """
+        markers = _MARKER.findall(request.all_text)
+        return extracted(fact(f"Saw Marker{markers[0]}", who="Alice", entities=["Alice"]))
+
+    llm.on_step("extract_facts").answers_with(one_fact_naming_the_first_marker)
+    llm.on_step("consolidate").returns(consolidation())
+
+    await client.aretain(bank_id=bank_id, content=CONTENT, document_id="d1")
+    await settled(bank_id)
+    return bank_id
+
+
+async def test_the_document_is_split_into_several_chunks(client, chunked_document):
+    chunks = await client.documents.list_document_chunks(chunked_document, "d1")
+
+    assert chunks.total > 1, "content well over the chunk size was not split at all"
+
+
+async def test_the_chunks_are_numbered_without_gaps(client, chunked_document):
+    """Contiguous from zero. A gap means a chunk was dropped between splitting
+    and storing, and nothing else in the response would show it."""
+    chunks = await client.documents.list_document_chunks(chunked_document, "d1")
+
+    indexes = sorted(chunk.chunk_index for chunk in chunks.items)
+    assert indexes == list(range(len(indexes)))
+
+
+async def test_the_chunks_reassemble_into_the_original(client, chunked_document):
+    """No text is lost at the seams.
+
+    Asserted over the concatenation rather than sentence by sentence: the
+    splitter cuts on size, not on sentence boundaries, so an individual sentence
+    can legitimately straddle two chunks and appear whole in neither. What must
+    hold is that putting the pieces back together yields every marker, in order,
+    exactly once.
+    """
+    chunks = await client.documents.list_document_chunks(chunked_document, "d1")
+    ordered = sorted(chunks.items, key=lambda chunk: chunk.chunk_index)
+
+    reassembled = "".join(chunk.chunk_text for chunk in ordered)
+    assert _MARKER.findall(reassembled) == _MARKER.findall(CONTENT)
+
+
+async def test_every_chunk_reached_the_model_exactly_once(client, llm, chunked_document):
+    """The assertion the markers exist for.
+
+    One extraction call per chunk: fewer means a piece was never read, more means
+    the model was paid for the same text twice. Both look like a working retain.
+    """
+    chunks = await client.documents.list_document_chunks(chunked_document, "d1")
+    prompts = llm.prompts_for("extract_facts")
+
+    assert len(prompts) == chunks.total
+
+    first_markers = [_MARKER.findall(prompt)[0] for prompt in prompts]
+    assert len(set(first_markers)) == len(first_markers), "a chunk was extracted from twice"
+
+
+async def test_the_stored_document_is_byte_identical_to_what_was_sent(client, chunked_document):
+    """Chunking is for extraction, not storage. The document is what a reprocess
+    re-reads, so a version that kept only one slice would quietly shrink the
+    bank the next time it was re-extracted."""
+    document = await client.documents.get_document(chunked_document, "d1")
+
+    assert document.original_text == CONTENT
+
+
+async def test_facts_from_every_part_of_the_document_are_recallable(client, chunked_document):
+    """End to end: a marker from the last chunk is as findable as one from the
+    first. A dropped tail is only visible from the far end of the document."""
+    last_marker = _MARKER.findall(SENTENCES[-1])[0]
+
+    memories = await client.memory.list_memories(chunked_document, limit=200)
+    seen = {marker for item in memories.items for marker in _MARKER.findall(item["text"])}
+
+    assert last_marker in seen or any(int(m) > 40 for m in seen), (
+        "no fact came from the end of the document — the tail was never extracted"
+    )

hindsight-system-tests/uv.lock

@@ -354,7 +354,7 @@ wheels = [
 [[package]]
 name = "hindsight-client"
 version = "0.9.2"
-source = { directory = "../hindsight-clients/python" }
+source = { editable = "../hindsight-clients/python" }
 dependencies = [
     { name = "aiohttp" },
     { name = "aiohttp-retry" },
@@ -396,7 +396,7 @@ dependencies = [
 [package.metadata]
 requires-dist = [
     { name = "fastapi", specifier = ">=0.115.0" },
-    { name = "hindsight-client", directory = "../hindsight-clients/python" },
+    { name = "hindsight-client", editable = "../hindsight-clients/python" },
     { name = "httpx", specifier = ">=0.27.0" },
     { name = "pydantic", specifier = ">=2" },
     { name = "pytest", specifier = ">=7.0.0" },