fix(context): guard SearchAPIRetriever against None raw_content

SearchAPIRetriever sliced page.get('raw_content', '')[:MAX], but the
default only applies when the key is absent. The scraper explicitly sets
raw_content=None for pages that failed to scrape, and slicing None raises
TypeError, crashing context retrieval. Coerce a missing/None raw_content
to '' before truncating (matching the defensive str() coercion already
used in compression.py).

(cherry picked from commit 2ffe6df27b)
This commit is contained in:
Bartok9
2026-07-04 02:24:07 -04:00
committed by Assaf Elovic
parent 68a9687e6b
commit b9f1192e31
2 changed files with 36 additions and 1 deletions
+4 -1
View File
@@ -23,7 +23,10 @@ class SearchAPIRetriever(BaseRetriever):
docs = [
Document(
page_content=page.get("raw_content", "")[:_MAX_CONTENT_CHARS],
# ``raw_content`` may be explicitly None (the scraper sets it to
# None for pages that failed to scrape), and slicing None raises
# TypeError. Coerce to a string before truncating.
page_content=(page.get("raw_content") or "")[:_MAX_CONTENT_CHARS],
metadata={
"title": page.get("title", ""),
"source": page.get("url", ""),
+32
View File
@@ -0,0 +1,32 @@
"""Regression: SearchAPIRetriever must tolerate raw_content=None.
The scraper sets ``raw_content`` to ``None`` for pages that failed to scrape.
Slicing ``None`` raises ``TypeError``, so the retriever must coerce a missing
or None ``raw_content`` to an empty string.
"""
from gpt_researcher.context.retriever import SearchAPIRetriever
def test_none_raw_content_yields_empty_page_content():
retriever = SearchAPIRetriever(
pages=[{"raw_content": None, "title": "t", "url": "https://u.example"}]
)
docs = retriever.invoke("q")
assert len(docs) == 1
assert docs[0].page_content == ""
assert docs[0].metadata["source"] == "https://u.example"
def test_present_raw_content_is_preserved():
retriever = SearchAPIRetriever(
pages=[{"raw_content": "hello world", "title": "t", "url": "u"}]
)
docs = retriever.invoke("q")
assert docs[0].page_content == "hello world"
def test_missing_raw_content_key_yields_empty():
retriever = SearchAPIRetriever(pages=[{"title": "t", "url": "u"}])
docs = retriever.invoke("q")
assert docs[0].page_content == ""