mirror of
https://github.com/assafelovic/gpt-researcher.git
synced 2026-09-14 20:17:32 +08:00
fix(context): guard SearchAPIRetriever against None raw_content
SearchAPIRetriever sliced page.get('raw_content', '')[:MAX], but the
default only applies when the key is absent. The scraper explicitly sets
raw_content=None for pages that failed to scrape, and slicing None raises
TypeError, crashing context retrieval. Coerce a missing/None raw_content
to '' before truncating (matching the defensive str() coercion already
used in compression.py).
(cherry picked from commit 2ffe6df27b)
This commit is contained in:
@@ -23,7 +23,10 @@ class SearchAPIRetriever(BaseRetriever):
|
||||
|
||||
docs = [
|
||||
Document(
|
||||
page_content=page.get("raw_content", "")[:_MAX_CONTENT_CHARS],
|
||||
# ``raw_content`` may be explicitly None (the scraper sets it to
|
||||
# None for pages that failed to scrape), and slicing None raises
|
||||
# TypeError. Coerce to a string before truncating.
|
||||
page_content=(page.get("raw_content") or "")[:_MAX_CONTENT_CHARS],
|
||||
metadata={
|
||||
"title": page.get("title", ""),
|
||||
"source": page.get("url", ""),
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Regression: SearchAPIRetriever must tolerate raw_content=None.
|
||||
|
||||
The scraper sets ``raw_content`` to ``None`` for pages that failed to scrape.
|
||||
Slicing ``None`` raises ``TypeError``, so the retriever must coerce a missing
|
||||
or None ``raw_content`` to an empty string.
|
||||
"""
|
||||
|
||||
from gpt_researcher.context.retriever import SearchAPIRetriever
|
||||
|
||||
|
||||
def test_none_raw_content_yields_empty_page_content():
|
||||
retriever = SearchAPIRetriever(
|
||||
pages=[{"raw_content": None, "title": "t", "url": "https://u.example"}]
|
||||
)
|
||||
docs = retriever.invoke("q")
|
||||
assert len(docs) == 1
|
||||
assert docs[0].page_content == ""
|
||||
assert docs[0].metadata["source"] == "https://u.example"
|
||||
|
||||
|
||||
def test_present_raw_content_is_preserved():
|
||||
retriever = SearchAPIRetriever(
|
||||
pages=[{"raw_content": "hello world", "title": "t", "url": "u"}]
|
||||
)
|
||||
docs = retriever.invoke("q")
|
||||
assert docs[0].page_content == "hello world"
|
||||
|
||||
|
||||
def test_missing_raw_content_key_yields_empty():
|
||||
retriever = SearchAPIRetriever(pages=[{"title": "t", "url": "u"}])
|
||||
docs = retriever.invoke("q")
|
||||
assert docs[0].page_content == ""
|
||||
Reference in New Issue
Block a user