test(system): stories 31, 32 and 64 — staleness, refresh safety, webhooks

- 31 the watermark that makes 'always current' work. Both failure directions
  matter: a watermark that never advances refreshes forever at full cost, and a
  staleness check that never fires leaves a model quietly frozen while still
  answering confidently. Also pins that going stale does not blank the answer.
- 32 the two wipe guards. A refresh whose scope matches nothing must keep the
  answer it has — writing 'nothing retrieved' through as 'no content' destroys
  work the bank cannot re-derive. And a dry run touches neither content,
  watermarks, nor history, or it is not a dry run.
- 64 webhooks, asserted against a real receiver rather than the server's own
  delivery log, which only proves it tried. The signature is recomputed the way
  a receiver would — a signature over the wrong bytes is a header that looks
  right and verifies nowhere. Both SSRF cases from GHSA-ggrr-69wp-fj54 are
  pinned as security properties, refused at registration.

The stub gains a webhook receiver: it is the only endpoint a hermetic test can
offer. The test server allowlists exactly that host, so a webhook aimed at any
other private address still fails and the guard stays genuinely under test.

hindsight-system-tests/hindsight_system_tests/rulebook.py

@@ -281,6 +281,14 @@ class RerankStub:
         return lexical_relevance(query, document)
 
 
+@dataclass(frozen=True)
+class ReceivedWebhook:
+    """One delivery the stub accepted, kept so a test can assert on it."""
+
+    headers: dict[str, str]
+    body: dict[str, Any]
+
+
 @dataclass
 class Stubs:
     """The three backends a test configures, handed to it as one object."""
@@ -291,11 +299,21 @@ class Stubs:
     rejected_requests: list[str] = field(default_factory=list)
     """Requests the stub refused as malformed — see ``validation.py``."""
 
+    webhooks: list[ReceivedWebhook] = field(default_factory=list)
+    """Webhook deliveries the stub received, in arrival order.
+
+    The stub is the only receiver a hermetic test can offer, so it doubles as the
+    customer endpoint: without somewhere for a delivery to land, "the webhook
+    fired" can only be read off the server's own delivery log, which proves it
+    tried rather than that anything arrived.
+    """
+
     def reset(self) -> None:
         """Between tests. The embedding and rerank stubs are pure, so only the
         rulebook and the rejection log carry state worth clearing."""
         self.llm.reset()
         self.rejected_requests.clear()
+        self.webhooks.clear()
 
 
 def _as_tuple(value: str | list[str] | None) -> tuple[str, ...]:

hindsight-system-tests/hindsight_system_tests/server.py

@@ -153,6 +153,12 @@ def stub_environment(stub_url: str) -> dict[str, str]:
         # to stampede, and the jitter is otherwise a minute of a story waiting for
         # work it already asked for.
         "HINDSIGHT_API_MAINTENANCE_START_JITTER_SECONDS": "0",
+        # Webhook destinations are SSRF-checked, and loopback is refused — the
+        # right default, and story 64 asserts it. But the only receiver a
+        # hermetic test can offer *is* on loopback, so the stub's host is
+        # allowlisted explicitly. Nothing else is: a webhook aimed anywhere else
+        # private still fails, which is what keeps the guard under test.
+        "HINDSIGHT_API_WEBHOOK_ALLOWED_HOSTS": "127.0.0.1",
         "HINDSIGHT_API_CONSOLIDATION_RECONCILE_INTERVAL_SECONDS": "5",
     }
 

hindsight-system-tests/hindsight_system_tests/stub_server.py

@@ -24,7 +24,7 @@ from fastapi import FastAPI, Request
 from fastapi.responses import JSONResponse
 
 from .lexical import EMBEDDING_DIMENSION
-from .rulebook import ChatRequest, Stubs
+from .rulebook import ChatRequest, ReceivedWebhook, Stubs
 from .validation import RequestRejected, validate_chat, validate_embeddings, validate_rerank
 
 logger = logging.getLogger(__name__)
@@ -105,6 +105,19 @@ def create_stub_app(stubs: Stubs) -> FastAPI:
             }
         )
 
+    @app.post("/webhook")
+    async def webhook(request: Request) -> JSONResponse:
+        """Stand in for a customer's webhook endpoint.
+
+        Records the headers as well as the body: the signature a receiver is
+        expected to verify travels in a header, and a delivery that arrives
+        unsigned is indistinguishable from one anybody could forge.
+        """
+        stubs.webhooks.append(
+            ReceivedWebhook(headers={k.lower(): v for k, v in request.headers.items()}, body=await request.json())
+        )
+        return JSONResponse({"received": True})
+
     @app.post("/rerank")
     async def rerank(request: Request) -> JSONResponse:
         body = await request.json()

hindsight-system-tests/tests/test_31_mental_model_staleness.py

@@ -0,0 +1,125 @@
+"""A mental model knows when the bank has moved on without it.
+
+The promise is "always current, without asking", and the machinery behind it is
+a watermark: `last_memory_seen_at` records how far through the bank the last
+refresh read. A write later than that watermark makes the model stale; a refresh
+advances it.
+
+Both halves have to work or the feature inverts. A watermark that never advances
+leaves a model permanently stale — every scheduled sweep refreshes it again, at
+full cost, forever. A staleness check that never fires leaves a model
+permanently *fresh* — it stops updating, keeps answering confidently, and gets
+further from the truth with every retain. The second is the dangerous one,
+because nothing about it looks wrong.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from hindsight_system_tests import reflect_loop
+from hindsight_system_tests.payloads import consolidation, extracted, fact
+
+pytestmark = pytest.mark.asyncio
+
+FIRST_ANSWER = "Alice lives in Berlin."
+SECOND_ANSWER = "Alice lives in Berlin and plays the cello."
+
+
+@pytest.fixture
+async def model(client, llm, bank_id, settled):
+    llm.on_step("extract_facts", contains="Berlin").returns(
+        extracted(fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]))
+    )
+    llm.on_step("extract_facts", contains="cello").returns(
+        extracted(fact("Alice plays cello", who="Alice", entities=["Alice", "cello"]))
+    )
+    llm.on_step("consolidate").returns(consolidation())
+    reflect_loop(llm, answer=FIRST_ANSWER)
+
+    await client.aretain(bank_id=bank_id, content="Alice moved to Berlin.")
+    await settled(bank_id)
+
+    created = await client.mental_models.create_mental_model(
+        bank_id, {"name": "Housing", "source_query": "Where does Alice live?"}
+    )
+    await settled(bank_id)
+    return created.mental_model_id
+
+
+async def _read(client, bank: str, model_id: str):
+    return await client.mental_models.get_mental_model(bank, model_id, detail="full")
+
+
+async def test_a_freshly_refreshed_model_is_not_stale(client, bank_id, model):
+    current = await _read(client, bank_id, model)
+
+    assert current.content.strip() == FIRST_ANSWER
+    assert current.is_stale is False
+    assert current.last_memory_seen_at is not None, "a refresh that read the bank must record how far it got"
+
+
+async def test_a_later_write_makes_it_stale(client, bank_id, model, settled):
+    """The signal that drives every scheduled refresh. If it never fires, the
+    model quietly stops updating while still answering with confidence."""
+    await client.aretain(bank_id=bank_id, content="Alice plays cello.")
+    await settled(bank_id)
+
+    current = await _read(client, bank_id, model)
+    assert current.is_stale is True
+
+
+async def test_going_stale_does_not_change_the_answer(client, bank_id, model, settled):
+    """Stale means "worth refreshing", not "wrong". The old answer keeps serving
+    until a refresh replaces it — blanking it on staleness would leave a gap
+    every time anything was retained."""
+    await client.aretain(bank_id=bank_id, content="Alice plays cello.")
+    await settled(bank_id)
+
+    current = await _read(client, bank_id, model)
+    assert current.content.strip() == FIRST_ANSWER
+
+
+async def test_the_watermark_does_not_move_until_a_refresh_reads_the_bank(client, bank_id, model, settled):
+    """A write is not a read. The watermark records what the *model* has seen, so
+    it must not drift forward just because the bank grew."""
+    before = (await _read(client, bank_id, model)).last_memory_seen_at
+
+    await client.aretain(bank_id=bank_id, content="Alice plays cello.")
+    await settled(bank_id)
+
+    assert (await _read(client, bank_id, model)).last_memory_seen_at == before
+
+
+async def test_refreshing_takes_in_the_new_facts_and_clears_the_flag(client, llm, bank_id, model, settled):
+    """The loop closing: refresh, new answer, watermark advanced, no longer stale.
+
+    A watermark that failed to advance here would leave the model stale forever
+    and make every scheduled sweep redo the same work at full cost.
+    """
+    await client.aretain(bank_id=bank_id, content="Alice plays cello.")
+    await settled(bank_id)
+    watermark_before = (await _read(client, bank_id, model)).last_memory_seen_at
+
+    llm.reset()
+    reflect_loop(llm, answer=SECOND_ANSWER)
+    await client.mental_models.refresh_mental_model(bank_id, model)
+    await settled(bank_id)
+
+    current = await _read(client, bank_id, model)
+    assert current.content.strip() == SECOND_ANSWER
+    assert current.is_stale is False
+    assert current.last_memory_seen_at > watermark_before
+
+
+async def test_the_previous_answer_is_kept_in_history(client, llm, bank_id, model, settled):
+    """A model that rewrites itself in place is unauditable — "it used to say
+    something else" needs to be answerable."""
+    llm.reset()
+    reflect_loop(llm, answer=SECOND_ANSWER)
+    await client.mental_models.refresh_mental_model(bank_id, model)
+    await settled(bank_id)
+
+    history = await client.mental_models.get_mental_model_history(bank_id, model)
+    # History entries are untyped dicts — see #4218.
+    assert any(entry["previous_content"].strip() == FIRST_ANSWER for entry in history)

hindsight-system-tests/tests/test_32_refresh_safety.py

@@ -0,0 +1,127 @@
+"""A refresh that finds nothing leaves the answer alone, and a dry run changes
+nothing at all.
+
+Both are guards against the same class of accident: a mental model losing a good
+answer to a refresh that had nothing to say.
+
+The empty-scope case is the sharp one. A model scoped to tags that currently
+match no memories — a project not started yet, a tag renamed, a document deleted
+— will run its refresh and retrieve nothing. If "no evidence" is written through
+as "no content", a working answer is replaced by an empty one, and the only way
+back is to notice and re-derive it. Silence has to mean "keep what you have".
+
+The dry run is the other half: a way to see what a refresh *would* do before
+letting it. That is only useful if it is genuinely read-only.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from hindsight_system_tests import reflect_loop
+from hindsight_system_tests.payloads import consolidation, extracted, fact
+
+pytestmark = pytest.mark.asyncio
+
+ANSWER = "Alice lives in Berlin."
+
+
+@pytest.fixture
+async def model(client, llm, bank_id, settled):
+    llm.on_step("extract_facts").returns(
+        extracted(fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]))
+    )
+    llm.on_step("consolidate").returns(consolidation())
+    reflect_loop(llm, answer=ANSWER)
+
+    await client.aretain(bank_id=bank_id, content="Alice moved to Berlin.", tags=["housing"])
+    await settled(bank_id)
+
+    created = await client.mental_models.create_mental_model(
+        bank_id, {"name": "Housing", "source_query": "Where does Alice live?"}
+    )
+    await settled(bank_id)
+    return created.mental_model_id
+
+
+async def _read(client, bank: str, model_id: str):
+    return await client.mental_models.get_mental_model(bank, model_id, detail="full")
+
+
+async def test_a_refresh_over_a_scope_that_matches_nothing_keeps_the_answer(client, llm, bank_id, settled):
+    """The wipe guard.
+
+    This model is scoped to a tag no memory carries, so its refresh retrieves
+    nothing — the state of any model whose subject has not happened yet. The
+    answer it already has must survive: writing "nothing retrieved" through as
+    "no content" destroys work that cannot be recovered from the bank.
+    """
+    llm.on_step("extract_facts").returns(
+        extracted(fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]))
+    )
+    llm.on_step("consolidate").returns(consolidation())
+    reflect_loop(llm, answer=ANSWER)
+
+    await client.aretain(bank_id=bank_id, content="Alice moved to Berlin.", tags=["housing"])
+    await settled(bank_id)
+
+    created = await client.mental_models.create_mental_model(
+        bank_id, {"name": "Scoped", "source_query": "Where does Alice live?", "tags": ["housing"]}
+    )
+    await settled(bank_id)
+    assert (await _read(client, bank_id, created.mental_model_id)).content.strip() == ANSWER
+
+    # Retag the scope onto something the bank has never seen, then refresh.
+    await client.mental_models.update_mental_model(
+        bank_id, created.mental_model_id, {"tags": ["a-tag-nothing-carries"]}
+    )
+    llm.reset()
+    reflect_loop(llm, answer="")
+    await client.mental_models.refresh_mental_model(bank_id, created.mental_model_id)
+    await settled(bank_id)
+
+    current = await _read(client, bank_id, created.mental_model_id)
+    assert current.content.strip() == ANSWER, "an empty refresh wiped an answer it could not replace"
+
+
+async def test_a_dry_run_reports_what_would_happen(client, llm, bank_id, model, settled):
+    """The point of a dry run is the report, so it has to say something useful:
+    which mode it would run in, what scope it would read, and whether it would
+    write anything."""
+    llm.reset()
+    reflect_loop(llm, answer="Alice lives in Berlin and is settled there.")
+
+    result = await client.mental_models.dry_run_refresh_mental_model(bank_id, model)
+
+    assert result.mental_model_id == model
+    assert result.effective_mode
+    assert result.outcome
+    assert result.scope is not None
+
+
+async def test_a_dry_run_does_not_touch_the_stored_answer(client, llm, bank_id, model, settled):
+    """Read-only, or it is not a dry run. Someone checking whether a refresh is
+    safe must not perform the thing they were checking."""
+    before = await _read(client, bank_id, model)
+
+    llm.reset()
+    reflect_loop(llm, answer="A completely different answer.")
+    await client.mental_models.dry_run_refresh_mental_model(bank_id, model)
+
+    after = await _read(client, bank_id, model)
+    assert after.content == before.content
+    assert after.last_refreshed_at == before.last_refreshed_at
+    assert after.last_memory_seen_at == before.last_memory_seen_at
+
+
+async def test_a_dry_run_leaves_no_history_entry(client, llm, bank_id, model, settled):
+    """History records what actually changed. A dry run that logged itself would
+    make the audit trail lie about what the model has said."""
+    before = await client.mental_models.get_mental_model_history(bank_id, model)
+
+    llm.reset()
+    reflect_loop(llm, answer="A completely different answer.")
+    await client.mental_models.dry_run_refresh_mental_model(bank_id, model)
+
+    after = await client.mental_models.get_mental_model_history(bank_id, model)
+    assert len(after) == len(before)

hindsight-system-tests/tests/test_64_webhooks.py

@@ -0,0 +1,138 @@
+"""A webhook is delivered to somewhere allowed, signed so the receiver can trust it.
+
+Webhooks are the one place Hindsight makes an outbound request to a URL a caller
+chose, which makes them two features at once.
+
+The first is delivery: something happened, and a payload describing it reaches
+the endpoint. Asserted here against a real receiver, because the server's own
+delivery log only proves it *tried*.
+
+The second is that the caller's URL is untrusted input. A server that fetches
+whatever it is handed is an SSRF primitive — point a webhook at `169.254.169.254`
+or a service on the internal network and Hindsight becomes the attacker's HTTP
+client, from inside the perimeter. That is GHSA-ggrr-69wp-fj54, and the guard
+that fixed it is asserted here as a security property rather than a config
+detail.
+
+The signature is what makes the delivery worth acting on: an unsigned webhook is
+indistinguishable from one anybody could forge at the receiver.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import hmac
+
+import pytest
+from hindsight_client_api.exceptions import BadRequestException
+
+from hindsight_system_tests.payloads import consolidation, extracted, fact
+
+pytestmark = pytest.mark.asyncio
+
+SECRET = "s3cret"
+
+
+@pytest.fixture(autouse=True)
+def _extraction(llm):
+    llm.on_step("extract_facts").returns(
+        extracted(fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]))
+    )
+    llm.on_step("consolidate").returns(consolidation())
+
+
+async def _deliver(client, stubs, bank: str, stub_url: str, settled) -> None:
+    """Register a webhook, cause an event, wait for it to land at the receiver."""
+    await client.webhooks.create_webhook(bank, {"url": f"{stub_url}/webhook", "secret": SECRET})
+    await client.aretain(bank_id=bank, content="Alice moved to Berlin.")
+    await settled(bank)
+
+    for _ in range(30):
+        if stubs.webhooks:
+            return
+        await asyncio.sleep(1)
+    raise AssertionError("no webhook delivery arrived at the receiver")
+
+
+async def test_a_delivery_reaches_the_endpoint(client, stubs, bank_id, stub_server, settled):
+    """At the receiver, not merely in the server's own log — that would only show
+    an attempt."""
+    await _deliver(client, stubs, bank_id, stub_server.url, settled)
+
+    assert len(stubs.webhooks) == 1
+
+
+async def test_the_payload_says_what_happened(client, stubs, bank_id, stub_server, settled):
+    """A receiver has to be able to act without calling back for context: which
+    bank, which event, which operation, and when."""
+    await _deliver(client, stubs, bank_id, stub_server.url, settled)
+    body = stubs.webhooks[0].body
+
+    assert body["bank_id"] == bank_id
+    assert body["event"] == "consolidation.completed"
+    assert body["operation_id"]
+    assert body["timestamp"]
+    assert "data" in body
+
+
+async def test_the_delivery_is_signed_with_the_shared_secret(client, stubs, bank_id, stub_server, settled):
+    """Recomputed here the way a receiver would, rather than merely asserting a
+    header exists. A signature over the wrong bytes, or with the wrong secret, is
+    a header that looks right and verifies nowhere.
+    """
+    await _deliver(client, stubs, bank_id, stub_server.url, settled)
+    received = stubs.webhooks[0]
+
+    import json as _json
+
+    raw = _json.dumps(received.body, separators=(",", ":")).encode()
+    expected = hmac.new(SECRET.encode(), raw, hashlib.sha256).hexdigest()
+
+    signature = received.headers["x-hindsight-signature"]
+    assert signature.startswith("sha256=")
+    assert hmac.compare_digest(signature.removeprefix("sha256="), expected)
+
+
+async def test_the_event_type_travels_in_a_header_too(client, stubs, bank_id, stub_server, settled):
+    """So a receiver can route without parsing the body first."""
+    await _deliver(client, stubs, bank_id, stub_server.url, settled)
+
+    assert stubs.webhooks[0].headers["x-hindsight-event"] == "consolidation.completed"
+
+
+async def test_the_server_records_the_delivery_as_completed(client, stubs, bank_id, stub_server, settled):
+    """The other side of the same event: whoever is debugging a receiver that
+    saw nothing needs to know whether Hindsight sent it and what came back."""
+    await _deliver(client, stubs, bank_id, stub_server.url, settled)
+
+    webhooks = await client.webhooks.list_webhooks(bank_id)
+    deliveries = await client.webhooks.list_webhook_deliveries(bank_id, webhooks.items[0].id)
+
+    assert len(deliveries.items) == 1
+    delivery = deliveries.items[0]
+    assert delivery.status == "completed"
+    assert delivery.last_response_status == 200
+    assert delivery.last_error is None
+
+
+async def test_a_loopback_destination_is_refused(client, bank_id):
+    """The SSRF guard (GHSA-ggrr-69wp-fj54).
+
+    The caller chooses this URL, so an unguarded webhook turns Hindsight into an
+    HTTP client the attacker aims — at link-local metadata endpoints, at services
+    reachable only from inside the network. Refused at registration, where the
+    operator finds out, rather than at delivery time where it is a log line.
+
+    Note the test server allowlists exactly the stub's host so the tests above can
+    receive anything at all; this address is private and *not* on that list, so
+    the guard is genuinely still under test.
+    """
+    with pytest.raises(BadRequestException):
+        await client.webhooks.create_webhook(bank_id, {"url": "http://10.0.0.1/webhook"})
+
+
+async def test_a_link_local_metadata_address_is_refused(client, bank_id):
+    """The specific address that makes SSRF profitable on every major cloud."""
+    with pytest.raises(BadRequestException):
+        await client.webhooks.create_webhook(bank_id, {"url": "http://169.254.169.254/latest/meta-data/"})