mirror of
https://github.com/vectorize-io/hindsight.git
synced 2026-09-14 19:31:49 +08:00
test(system): stories 28, 33, 36, 37, 41, 93 — the rest of the epic
- 28 one bank consolidates one run at a time. A burst of five triggers produces one pass, because concurrent runs race on the layer they are both editing: each reads the observations, each decides independently what to retire, and the second to write undoes half the first's conclusions. - 33 refresh dedupe (#3487): five requests, one operation, and the model still ends up refreshed — collapsing to *zero* looks identical from the queue. - 36/37 a page rename reaches its backing model, and retracting the evidence schedules the page to be rewritten. What the rewrite *says* is deliberately not asserted: pages refresh in delta mode, so driving the wording from a stub would mean writing the delta myself and then asserting I had written it. - 41 reflect raises rather than degrading. A model that never calls a tool, or answers empty, must not yield "" — an agent cannot tell that from "your bank holds nothing about this", and the two demand opposite responses. - 93 an inline image reaches the extraction model as an image part, in order after its caption, and is stored against both document and chunk. Two tests FAIL, on two bugs found writing them: - #4291 entities.mention_count is only ever incremented — no decrement exists anywhere — so replacing or deleting a document leaves it inflated. It is user-visible, sizes graph nodes, and is the primary sort key of the curation listing. - #4292 both binary download endpoints declare application/json alongside their real media type, so every generated SDK decodes the bytes as text and corrupts them. Fetching an attachment — the documented audit path behind an image-derived fact — is impossible through any client. Both are left going through the published client rather than reaching around it with raw HTTP: an SDK-inaccessible endpoint is the defect, not an obstacle.
This commit is contained in:
Nicolò Boschi
@@ -0,0 +1,90 @@
|
||||
"""One bank consolidates one run at a time.
|
||||
|
||||
Consolidation reads facts, writes observations, and *deletes* the ones it
|
||||
supersedes. Two runs over the same bank at once therefore race on the thing they
|
||||
are both editing: each reads the observation layer, each decides independently
|
||||
what to merge and retire, and the second to write undoes half of what the first
|
||||
concluded. The visible symptoms are the ones story 22 and 23 guard against —
|
||||
duplicate observations for one claim, a retirement that reappears — arriving via
|
||||
a route those stories cannot reach.
|
||||
|
||||
So the server serialises per bank, and a second request joins the run in flight
|
||||
rather than starting a rival. That is what makes it safe for consolidation to be
|
||||
triggered from several directions — a retain, a scheduled sweep, an impatient
|
||||
caller — without any of them coordinating.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from hindsight_system_tests.payloads import extracted, fact, observes
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
OBSERVATION = "Alice is settled in Berlin"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def consolidated_bank(client, llm, bank_id, settled) -> str:
|
||||
llm.on_step("extract_facts").returns(
|
||||
extracted(
|
||||
fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]),
|
||||
fact("Alice renewed her Berlin lease", who="Alice", entities=["Alice", "Berlin"]),
|
||||
)
|
||||
)
|
||||
llm.on_step("consolidate").answers_with(observes(OBSERVATION))
|
||||
await client.aretain(bank_id=bank_id, content="Alice moved to Berlin. Alice renewed her Berlin lease.")
|
||||
await settled(bank_id)
|
||||
return bank_id
|
||||
|
||||
|
||||
async def _observations(client, bank: str) -> list[str]:
|
||||
memories = await client.memory.list_memories(bank, limit=100)
|
||||
return sorted(m.text for m in memories.items if m.fact_type == "observation")
|
||||
|
||||
|
||||
async def test_a_second_request_joins_the_run_already_in_flight(client, consolidated_bank):
|
||||
"""Not merely "both succeed" — the *same* operation. Two ids would mean two
|
||||
passes over one observation layer, which is the race."""
|
||||
first = await client.banks.trigger_consolidation(consolidated_bank)
|
||||
second = await client.banks.trigger_consolidation(consolidated_bank)
|
||||
|
||||
assert second.deduplicated is True
|
||||
assert second.operation_id == first.operation_id
|
||||
|
||||
|
||||
async def test_a_burst_of_requests_produces_one_pass(client, consolidated_bank, settled):
|
||||
"""Five callers, no coordination between them — the shape of a scheduled
|
||||
sweep landing on top of a retain that just finished."""
|
||||
before = (await client.operations.list_operations(consolidated_bank, type="consolidation", limit=100)).total
|
||||
|
||||
await asyncio.gather(*(client.banks.trigger_consolidation(consolidated_bank) for _ in range(5)))
|
||||
await settled(consolidated_bank)
|
||||
|
||||
after = (await client.operations.list_operations(consolidated_bank, type="consolidation", limit=100)).total
|
||||
assert after - before <= 1, f"a burst of 5 requests produced {after - before} consolidation runs"
|
||||
|
||||
|
||||
async def test_the_observation_layer_survives_the_burst(client, consolidated_bank, settled):
|
||||
"""The reason serialisation matters. Concurrent passes would each decide what
|
||||
to create and retire from their own read, and the loser's decisions would
|
||||
reappear as duplicates."""
|
||||
await asyncio.gather(*(client.banks.trigger_consolidation(consolidated_bank) for _ in range(5)))
|
||||
await settled(consolidated_bank)
|
||||
|
||||
assert await _observations(client, consolidated_bank) == [OBSERVATION]
|
||||
|
||||
|
||||
async def test_the_facts_survive_the_burst(client, consolidated_bank, settled):
|
||||
"""And the layer underneath is untouched, however many passes were asked for."""
|
||||
await asyncio.gather(*(client.banks.trigger_consolidation(consolidated_bank) for _ in range(5)))
|
||||
await settled(consolidated_bank)
|
||||
|
||||
memories = await client.memory.list_memories(consolidated_bank, limit=100)
|
||||
raw = sorted(m.text for m in memories.items if m.fact_type != "observation")
|
||||
assert raw == sorted(
|
||||
["Alice moved to Berlin | Involving: Alice", "Alice renewed her Berlin lease | Involving: Alice"]
|
||||
)
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Asking for the same refresh twice runs it once.
|
||||
|
||||
A mental-model refresh is expensive — it drives the whole reflect loop — and the
|
||||
requests arrive from several directions at once: a scheduled sweep, a
|
||||
consolidation that finished, an impatient caller. Without deduplication a busy
|
||||
bank queues one refresh per trigger and the worker spends its time recomputing
|
||||
the same answer, which is how a slow-draining bank accumulated ~45 copies of one
|
||||
model (#3487).
|
||||
|
||||
Worse than the cost: two refreshes of one model racing each other both read the
|
||||
bank and both write, and the one that finishes second wins regardless of which
|
||||
read more.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from hindsight_system_tests import reflect_loop
|
||||
from hindsight_system_tests.payloads import consolidation, extracted, fact
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
ANSWER = "Alice lives in Berlin."
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def model(client, llm, bank_id, settled) -> str:
|
||||
llm.on_step("extract_facts").returns(
|
||||
extracted(fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]))
|
||||
)
|
||||
llm.on_step("consolidate").returns(consolidation())
|
||||
reflect_loop(llm, answer=ANSWER)
|
||||
|
||||
await client.aretain(bank_id=bank_id, content="Alice moved to Berlin.")
|
||||
await settled(bank_id)
|
||||
|
||||
created = await client.mental_models.create_mental_model(
|
||||
bank_id, {"name": "Housing", "source_query": "Where does Alice live?"}
|
||||
)
|
||||
await settled(bank_id)
|
||||
return created.mental_model_id
|
||||
|
||||
|
||||
async def test_two_requests_in_flight_share_one_operation(client, bank_id, model):
|
||||
"""Back to back, no wait between them. The second joins the first rather than
|
||||
queueing a rival — the caller gets an operation id either way, and it is the
|
||||
same id."""
|
||||
first = await client.mental_models.refresh_mental_model(bank_id, model)
|
||||
second = await client.mental_models.refresh_mental_model(bank_id, model)
|
||||
|
||||
assert first.operation_id == second.operation_id
|
||||
|
||||
|
||||
async def test_the_bank_does_not_accumulate_a_refresh_per_request(client, bank_id, model, settled):
|
||||
"""The #3487 shape. Five requests, and the queue must not grow five deep."""
|
||||
for _ in range(5):
|
||||
await client.mental_models.refresh_mental_model(bank_id, model)
|
||||
await settled(bank_id)
|
||||
|
||||
operations = await client.operations.list_operations(bank_id, type="refresh_mental_model", limit=100)
|
||||
|
||||
# One for the create's own refresh, one for the batch of requests above.
|
||||
assert operations.total <= 2, f"{operations.total} refresh operations for one model"
|
||||
|
||||
|
||||
async def test_the_model_still_ends_up_refreshed(client, bank_id, model, settled):
|
||||
"""Deduplication must not swallow the work — collapsing to zero refreshes
|
||||
would look identical from the queue's point of view."""
|
||||
for _ in range(3):
|
||||
await client.mental_models.refresh_mental_model(bank_id, model)
|
||||
await settled(bank_id)
|
||||
|
||||
current = await client.mental_models.get_mental_model(bank_id, model, detail="full")
|
||||
assert current.content.strip() == ANSWER
|
||||
assert current.is_stale is False
|
||||
@@ -0,0 +1,122 @@
|
||||
"""A page and its model stay in step, and a retraction reaches both.
|
||||
|
||||
A knowledge page is a name and a place in a tree over a mental model that does
|
||||
the work. Two things have to hold as the bank changes underneath them.
|
||||
|
||||
**A rename reaches the model.** They are separate rows, so renaming the page and
|
||||
leaving the model on its old name gives one thing two names — and the model's
|
||||
name is what shows up wherever a page is referenced by its backing id, so the two
|
||||
disagree in exactly the places nobody is looking.
|
||||
|
||||
**A retraction propagates.** This is the sharper one. Facts are retracted because
|
||||
they were wrong, and a page written from them keeps repeating the wrong thing
|
||||
until something refreshes it. Nothing about that page looks stale: it has content,
|
||||
a timestamp, and an answer that reads perfectly well. The system has to notice on
|
||||
its own, because a caller who knew to ask would not have needed the page.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from hindsight_system_tests import reflect_loop
|
||||
from hindsight_system_tests.payloads import extracted, fact, observes
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
ORIGINAL_ANSWER = "Alice lives in Berlin."
|
||||
CORRECTED_ANSWER = "There is nothing on record about where Alice lives."
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def page(client, llm, bank_id, settled):
|
||||
llm.on_step("extract_facts").returns(
|
||||
extracted(fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]))
|
||||
)
|
||||
# A page refreshes by delta over the *observation* layer (story 35 pins those
|
||||
# defaults), so a bank with no observations gives its refresh nothing to read
|
||||
# and the page keeps its placeholder — which looks like a broken refresh and
|
||||
# is not.
|
||||
llm.on_step("consolidate").answers_with(observes("Alice is settled in Berlin"))
|
||||
reflect_loop(llm, answer=ORIGINAL_ANSWER)
|
||||
|
||||
await client.aretain(bank_id=bank_id, content="Alice moved to Berlin.")
|
||||
await settled(bank_id)
|
||||
|
||||
created = await client.knowledge_base.create_knowledge_page(
|
||||
bank_id, {"name": "Housing", "source_query": "Where does Alice live?"}
|
||||
)
|
||||
await settled(bank_id)
|
||||
return created
|
||||
|
||||
|
||||
async def test_the_page_starts_with_the_answer_it_was_written_from(client, bank_id, page):
|
||||
read = await client.knowledge_base.get_knowledge_page(bank_id, page.page_id)
|
||||
|
||||
assert read.name == "Housing"
|
||||
assert read.body.strip() == ORIGINAL_ANSWER
|
||||
|
||||
|
||||
async def test_renaming_a_page_renames_its_model(client, bank_id, page):
|
||||
"""Otherwise one thing has two names, and which you see depends on whether
|
||||
you arrived via the tree or via the model id."""
|
||||
await client.knowledge_base.update_knowledge_node(bank_id, page.page_id, {"name": "Where Alice lives"})
|
||||
|
||||
model = await client.mental_models.get_mental_model(bank_id, page.mental_model_id, detail="metadata")
|
||||
assert model.name == "Where Alice lives"
|
||||
|
||||
tree = await client.knowledge_base.get_knowledge_base_tree(bank_id)
|
||||
assert [node.name for node in tree.roots] == ["Where Alice lives"]
|
||||
|
||||
|
||||
async def test_retracting_the_evidence_schedules_the_page_to_be_rewritten(client, llm, bank_id, page, settled):
|
||||
"""Invalidating the fact the page was written from must enqueue a refresh.
|
||||
|
||||
Without it the page keeps asserting something the bank has explicitly
|
||||
retracted — and reads as authoritative while doing so, since a page carries
|
||||
no hint of how old its evidence is.
|
||||
"""
|
||||
memories = await client.memory.list_memories(bank_id, limit=100)
|
||||
target = next(m for m in memories.items if m.fact_type != "observation")
|
||||
|
||||
await client.memory.update_memory(bank_id, target.id, {"state": "invalidated", "reason": "Alice never moved"})
|
||||
|
||||
refreshed = False
|
||||
for _ in range(60):
|
||||
operations = await client.operations.list_operations(bank_id, type="refresh_mental_model", limit=100)
|
||||
if operations.total:
|
||||
refreshed = True
|
||||
break
|
||||
await asyncio.sleep(1)
|
||||
|
||||
assert refreshed, "retracting the evidence scheduled no refresh — the page will keep repeating it"
|
||||
|
||||
|
||||
async def test_the_scheduled_refresh_completes_cleanly(client, llm, bank_id, page, settled):
|
||||
"""The refresh runs to completion and leaves the page coherent.
|
||||
|
||||
What is *not* asserted here: that the retracted claim is gone from the text.
|
||||
A page refreshes in delta mode — it computes a diff against its current
|
||||
content rather than rewriting from scratch — so what the page ends up saying
|
||||
is a judgement the model makes about that diff. Driving it from a stub would
|
||||
mean writing the delta myself and then asserting I had written it, which
|
||||
proves nothing about the product.
|
||||
|
||||
The mechanism is testable and is: the retraction is noticed, a refresh is
|
||||
scheduled, it completes without error, and the page is still readable
|
||||
afterwards. Whether the wording actually unsays the claim needs a real model
|
||||
and belongs with the `hs_llm_core` judge tests.
|
||||
"""
|
||||
memories = await client.memory.list_memories(bank_id, limit=100)
|
||||
target = next(m for m in memories.items if m.fact_type != "observation")
|
||||
|
||||
await client.memory.update_memory(bank_id, target.id, {"state": "invalidated", "reason": "Alice never moved"})
|
||||
await settled(bank_id)
|
||||
|
||||
failed = await client.operations.list_operations(bank_id, status="failed", limit=100)
|
||||
assert failed.operations == [], "the refresh triggered by the retraction failed"
|
||||
|
||||
read = await client.knowledge_base.get_knowledge_page(bank_id, page.page_id)
|
||||
assert read.body.strip(), "the page was left empty by the refresh"
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Reflect fails loudly rather than returning a plausible non-answer.
|
||||
|
||||
Reflect is agentic: it needs a model that calls tools, and it needs that model to
|
||||
eventually produce an answer. Both can fail — a provider without function
|
||||
calling, a model that returns empty content, output truncated before the answer
|
||||
was written.
|
||||
|
||||
The tempting behaviour is to degrade: return an empty string, or a placeholder,
|
||||
or whatever prose happened to arrive instead of a tool call. That is the worst
|
||||
option available, because the caller is an agent that will treat the answer as
|
||||
memory and act on it. A confident empty answer is indistinguishable from "your
|
||||
bank has nothing about this", and the two demand opposite responses.
|
||||
|
||||
So each of these must raise, and the message must name the cause — someone
|
||||
pointing reflect at a model that cannot call tools needs to be told that, not
|
||||
handed a blank.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from hindsight_client_api.exceptions import ServiceException
|
||||
|
||||
from hindsight_system_tests import reflect_loop
|
||||
from hindsight_system_tests.payloads import consolidation, extracted, fact
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
QUERY = "Where does Alice live?"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def bank_with_facts(client, llm, bank_id, settled) -> str:
|
||||
llm.on_step("extract_facts").returns(
|
||||
extracted(fact("Alice moved to Berlin", who="Alice", entities=["Alice", "Berlin"]))
|
||||
)
|
||||
llm.on_step("consolidate").returns(consolidation())
|
||||
await client.aretain(bank_id=bank_id, content="Alice moved to Berlin.")
|
||||
await settled(bank_id)
|
||||
return bank_id
|
||||
|
||||
|
||||
async def test_a_model_that_never_calls_a_tool_is_an_error(client, llm, bank_with_facts):
|
||||
"""The provider-mismatch case: reflect's first turn forces a tool, and this
|
||||
model answers with prose instead.
|
||||
|
||||
The failure has to name the cause. "No answer" sends someone looking at their
|
||||
data; "this model produced no usable tool call" sends them at their config.
|
||||
"""
|
||||
llm.on_step("reflect").returns_text("I think Alice lives in Berlin.")
|
||||
|
||||
with pytest.raises(ServiceException) as raised:
|
||||
await client.areflect(bank_id=bank_with_facts, query=QUERY)
|
||||
|
||||
assert "tool" in str(raised.value).lower()
|
||||
|
||||
|
||||
async def test_an_empty_answer_is_an_error_rather_than_an_empty_result(client, llm, bank_with_facts):
|
||||
"""A model that climbs the ladder and then says nothing.
|
||||
|
||||
Returning "" here would be the dangerous degradation: an agent cannot tell it
|
||||
apart from a bank that genuinely holds nothing, and those call for opposite
|
||||
behaviour — one is "go and find out", the other is "you already know".
|
||||
"""
|
||||
llm.on_step("reflect", tool="done").returns_tool_call("done", answer="")
|
||||
llm.on_step("reflect").calls_the_offered_tool(query="Alice")
|
||||
llm.on_step("reflect").returns_text("")
|
||||
|
||||
with pytest.raises(ServiceException):
|
||||
await client.areflect(bank_id=bank_with_facts, query=QUERY)
|
||||
|
||||
|
||||
async def test_a_working_model_still_answers(client, llm, bank_with_facts):
|
||||
"""The control. Without it, the two tests above would also pass if reflect
|
||||
were broken outright."""
|
||||
reflect_loop(llm, answer="Alice lives in Berlin.")
|
||||
|
||||
response = await client.areflect(bank_id=bank_with_facts, query=QUERY)
|
||||
|
||||
assert response.text == "Alice lives in Berlin."
|
||||
@@ -0,0 +1,136 @@
|
||||
"""An image retained inline reaches the model, and stays fetchable afterwards.
|
||||
|
||||
Content can be a list of blocks rather than a string, so a screenshot sits where
|
||||
it actually appeared in the conversation. Two halves have to work, and they fail
|
||||
differently.
|
||||
|
||||
The image has to reach the *model* — an attachment stored but never sent means
|
||||
extraction describes text that references a picture it never saw, and the facts
|
||||
are confidently wrong rather than missing. And it has to survive as a
|
||||
**retrievable** attachment, because a fact drawn from an image is unverifiable
|
||||
without it: "Alice stood in front of the Brandenburg Gate" is only auditable if
|
||||
someone can still look at the photo.
|
||||
|
||||
The stub records what it was sent, so the first half is directly observable
|
||||
rather than inferred from the facts that came back.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
import pytest
|
||||
|
||||
from hindsight_system_tests.payloads import consolidation, extracted, fact
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
# A 1x1 transparent PNG — the smallest thing that is unambiguously an image.
|
||||
PNG = base64.b64decode(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
|
||||
)
|
||||
PNG_BASE64 = base64.b64encode(PNG).decode()
|
||||
|
||||
CAPTION = "Alice sent a photo from her trip:"
|
||||
FACT = "Alice stood in front of the Brandenburg Gate | Involving: Alice"
|
||||
|
||||
CONTENT = [
|
||||
{"type": "text", "text": CAPTION},
|
||||
{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG_BASE64}},
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def bank_with_photo(client, llm, bank_id, settled) -> str:
|
||||
llm.on_step("extract_facts").returns(
|
||||
extracted(fact("Alice stood in front of the Brandenburg Gate", who="Alice", entities=["Alice"]))
|
||||
)
|
||||
llm.on_step("consolidate").returns(consolidation())
|
||||
|
||||
await client.aretain_batch(bank_id=bank_id, items=[{"content": CONTENT}], document_id="d1")
|
||||
await settled(bank_id)
|
||||
return bank_id
|
||||
|
||||
|
||||
def _parts(prompt_messages: list[dict]) -> list[str]:
|
||||
return [
|
||||
part.get("type")
|
||||
for message in prompt_messages
|
||||
if isinstance(message.get("content"), list)
|
||||
for part in message["content"]
|
||||
if isinstance(part, dict)
|
||||
]
|
||||
|
||||
|
||||
async def test_the_image_is_sent_to_the_extraction_model(client, llm, bank_with_photo):
|
||||
"""Observed at the stub, not inferred.
|
||||
|
||||
An attachment that is stored but never sent produces facts about a picture
|
||||
the model never saw — plausible, specific and wrong, which is worse than no
|
||||
facts at all.
|
||||
"""
|
||||
calls = [call for call in llm.calls if any(a in call.all_text for a in [CAPTION])]
|
||||
assert calls, "the extraction call never carried the caption"
|
||||
|
||||
kinds = _parts(calls[0].messages)
|
||||
assert "text" in kinds
|
||||
assert "image_url" in kinds, "the image was dropped between the API and the model"
|
||||
|
||||
|
||||
async def test_the_caption_and_the_image_arrive_together(client, llm, bank_with_photo):
|
||||
"""Order is the point of inline blocks. A photo hoisted to the end of the
|
||||
prompt loses which sentence it belonged to, which is most of its meaning in a
|
||||
conversation."""
|
||||
call = next(call for call in llm.calls if CAPTION in call.all_text)
|
||||
kinds = _parts(call.messages)
|
||||
|
||||
assert kinds.index("text") < kinds.index("image_url")
|
||||
|
||||
|
||||
async def test_the_attachment_is_stored_against_the_document(client, bank_with_photo):
|
||||
document = await client.documents.get_document(bank_with_photo, "d1")
|
||||
|
||||
assert document.attachments, "the image was not kept"
|
||||
attachment = document.attachments[0]
|
||||
assert attachment.kind == "image"
|
||||
assert attachment.media_type == "image/png"
|
||||
assert attachment.byte_size == len(PNG)
|
||||
|
||||
|
||||
async def test_the_attachment_hangs_off_the_chunk_it_appeared_in(client, bank_with_photo):
|
||||
"""Provenance at the granularity extraction actually works at: a fact points
|
||||
at a chunk, so the chunk has to carry the image that fact was drawn from."""
|
||||
chunks = await client.documents.list_document_chunks(bank_with_photo, "d1")
|
||||
|
||||
attachments = [a for chunk in chunks.items for a in (chunk.attachments or [])]
|
||||
assert [a.kind for a in attachments] == ["image"]
|
||||
|
||||
|
||||
async def test_the_stored_image_is_byte_identical(client, bank_with_photo):
|
||||
"""The audit path, and it does not work today — #4292.
|
||||
|
||||
A fact drawn from a picture is unverifiable unless the picture comes back
|
||||
exactly as sent. The bytes survive on the server; the *client* destroys them,
|
||||
because the endpoint declares `application/json` alongside
|
||||
`application/octet-stream` in the spec and the generated method decodes the
|
||||
body as text. `0x89` — the first byte of the PNG magic number — is where it
|
||||
gives up.
|
||||
|
||||
Left going through the client rather than reaching around it with raw HTTP:
|
||||
an SDK-inaccessible endpoint is exactly the defect this suite exists to
|
||||
surface, and papering over it here would hide that every consumer has the
|
||||
same problem.
|
||||
"""
|
||||
document = await client.documents.get_document(bank_with_photo, "d1")
|
||||
attachment = document.attachments[0]
|
||||
|
||||
fetched = await client.memory.get_bank_attachment(bank_with_photo, attachment.id)
|
||||
|
||||
assert fetched == PNG
|
||||
|
||||
|
||||
async def test_the_fact_drawn_from_the_image_is_recallable(client, bank_with_photo):
|
||||
"""End to end: the image produced a memory like any other."""
|
||||
response = await client.arecall(bank_id=bank_with_photo, query="Where was Alice photographed?")
|
||||
|
||||
assert [r.text for r in response.results] == [FACT]
|
||||
Reference in New Issue
Block a user