fix(retriever): guard GoogleSearch against malformed CSE content items

Prefer .get for title/snippet/link so a partial CSE item cannot raise and
is not silently discarded beyond missing-link cases. Non-dict JSON and
non-dict row items return/ skip cleanly to an empty or partial list.

(cherry picked from commit 48cbf35a7d)

gpt_researcher/retrievers/google/google.py

@@ -71,30 +71,32 @@ class GoogleSearch:
             print("Google search: unexpected response status: ", resp.status_code)
 
         if resp is None:
-            return
+            return []
         try:
             search_results = json.loads(resp.text)
         except Exception:
-            return
-        if search_results is None:
-            return
+            return []
+        if not isinstance(search_results, dict):
+            return []
 
-        results = search_results.get("items", [])
-        search_results = []
+        results = search_results.get("items", []) or []
+        search_response = []
 
-        # Normalizing results to match the format of the other search APIs
+        # Normalizing results to match the format of the other search APIs.
+        # Use .get so a missing title/snippet cannot drop a valid link; skip
+        # non-dict rows and empty links outright.
         for result in results:
-            # skip youtube results
-            if "youtube.com" in result["link"]:
+            if not isinstance(result, dict):
                 continue
-            try:
-                search_result = {
-                    "title": result["title"],
-                    "href": result["link"],
-                    "body": result["snippet"],
+            link = result.get("link") or ""
+            if not link or "youtube.com" in link:
+                continue
+            search_response.append(
+                {
+                    "title": result.get("title") or "",
+                    "href": link,
+                    "body": result.get("snippet") or "",
                 }
-            except Exception:
-                continue
-            search_results.append(search_result)
+            )
 
-        return search_results[:max_results]
+        return search_response[:max_results]

tests/test_google_malformed.py

@@ -0,0 +1,53 @@
+"""GoogleSearch must tolerate malformed CSE items / response shapes."""
+
+from __future__ import annotations
+
+from unittest.mock import MagicMock, patch
+
+from gpt_researcher.retrievers.google.google import GoogleSearch
+
+
+def _searcher():
+    g = GoogleSearch.__new__(GoogleSearch)
+    g.query = "q"
+    g.headers = {}
+    g.query_domains = None
+    g.api_key = "k"
+    g.cx_key = "cx"
+    return g
+
+
+def test_google_skips_non_dict_and_missing_link():
+    g = _searcher()
+    payload = {
+        "items": [
+            {"title": "A", "link": "https://a.example", "snippet": "sa"},
+            "not-a-dict",
+            {"title": "No link"},
+            {"title": "YT", "link": "https://youtube.com/watch?v=1", "snippet": "y"},
+            {"link": "https://b.example"},  # title/snippet optional
+        ]
+    }
+    resp = MagicMock(status_code=200, text='{}')
+    with patch(
+        "gpt_researcher.retrievers.google.google.requests.get", return_value=resp
+    ), patch(
+        "gpt_researcher.retrievers.google.google.json.loads", return_value=payload
+    ):
+        out = g.search(max_results=10)
+
+    assert out == [
+        {"title": "A", "href": "https://a.example", "body": "sa"},
+        {"title": "", "href": "https://b.example", "body": ""},
+    ]
+
+
+def test_google_returns_empty_list_on_non_dict_json():
+    g = _searcher()
+    resp = MagicMock(status_code=200, text='[]')
+    with patch(
+        "gpt_researcher.retrievers.google.google.requests.get", return_value=resp
+    ), patch(
+        "gpt_researcher.retrievers.google.google.json.loads", return_value=[]
+    ):
+        assert g.search() == []