diff --git a/baidu/parse_html.go b/baidu/parse_html.go
index 6f48d24..d42cf2f 100644
--- a/baidu/parse_html.go
+++ b/baidu/parse_html.go
@@ -19,22 +19,61 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
}
func parseBaiduDocument(doc *goquery.Document) []core.SearchResult {
+ for _, selector := range baiduResultSelectors() {
+ if results := parseBaiduSelection(doc.Find(selector)); len(results) > 0 {
+ return results
+ }
+ }
+ return nil
+}
+
+func baiduResultSelectors() []string {
+ selectors := make([]string, 0, len(Selectors.ResultsAlt)+1)
+ selectors = append(selectors, Selectors.Results)
+ selectors = append(selectors, Selectors.ResultsAlt...)
+ return selectors
+}
+
+func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
var results []core.SearchResult
rank := 1
- doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
- linkTag := item.Find(Selectors.Link).First()
- if linkTag.Length() == 0 {
+ sel.Each(func(_ int, item *goquery.Selection) {
+ // h3-first: organic results always carry a heading; this filters out
+ // non-result blocks that may share the wrapper class.
+ titleTag := item.Find("h3").First()
+ var (
+ title string
+ linkTag *goquery.Selection
+ )
+ if titleTag.Length() > 0 {
+ title = strings.TrimSpace(titleTag.Text())
+ if child := titleTag.Find("a[href]").First(); child.Length() > 0 {
+ linkTag = child
+ } else if closest := titleTag.Closest("a[href]"); closest.Length() > 0 {
+ linkTag = closest
+ }
+ }
+ if linkTag == nil || linkTag.Length() == 0 {
+ first := item.Find(Selectors.Link).First()
+ if first.Length() == 0 {
+ return
+ }
+ linkTag = first
+ }
+ if title == "" {
+ title = strings.TrimSpace(linkTag.Text())
+ }
+ if title == "" {
return
}
href, exists := linkTag.Attr("href")
- if !exists || href == "" || href == "#" || !strings.HasPrefix(href, "http") {
+ if !exists {
return
}
-
- title := strings.TrimSpace(linkTag.Text())
- if title == "" {
+ href = strings.TrimSpace(href)
+ if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
return
}
@@ -42,6 +81,16 @@ func parseBaiduDocument(doc *goquery.Document) []core.SearchResult {
if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 {
desc = strings.TrimSpace(descTag.Text())
}
+ if desc == "" {
+ for _, alt := range Selectors.DescAlt {
+ if descTag := item.Find(alt).First(); descTag.Length() > 0 {
+ if t := strings.TrimSpace(descTag.Text()); t != "" {
+ desc = t
+ break
+ }
+ }
+ }
+ }
if desc == "" {
full := strings.TrimSpace(item.Text())
desc = strings.TrimSpace(strings.Replace(full, title, "", 1))
@@ -56,5 +105,12 @@ func parseBaiduDocument(doc *goquery.Document) []core.SearchResult {
rank++
})
- return core.DeduplicateResults(results)
+ // Re-rank sequentially after dedup so callers get a clean 1..N sequence
+ // (dedup may drop intermediate ranks when the same URL appears in
+ // multiple Baidu result-card variants on the same SERP).
+ deduped := core.DeduplicateResults(results)
+ for i := range deduped {
+ deduped[i].Rank = i + 1
+ }
+ return deduped
}
diff --git a/baidu/parse_html_test.go b/baidu/parse_html_test.go
index 1d076b4..9823a4d 100644
--- a/baidu/parse_html_test.go
+++ b/baidu/parse_html_test.go
@@ -51,3 +51,56 @@ func TestParseBaiduHTMLEmpty(t *testing.T) {
t.Fatalf("expected zero results for empty HTML, got %d", len(results))
}
}
+
+func TestParseBaiduHTMLFallbackSelectors(t *testing.T) {
+ t.Parallel()
+
+ html := `
+
+
+
+
Fallback description
+
+
+
+
+
+
Parseable description
+
+
+ -
+
+
Fallback description
+
+
`
+
+ results, err := ParseHTML(bytes.NewReader([]byte(html)))
+ if err != nil {
+ t.Fatalf("ParseHTML() error = %v", err)
+ }
+ if len(results) != 1 {
+ t.Fatalf("expected 1 result, got %d", len(results))
+ }
+ if results[0].URL != "https://example.com/result" {
+ t.Fatalf("unexpected URL: %s", results[0].URL)
+ }
+ if results[0].Title != "Fallback Title" {
+ t.Fatalf("unexpected title: %s", results[0].Title)
+ }
+ if results[0].Description != "Fallback description" {
+ t.Fatalf("unexpected description: %s", results[0].Description)
+ }
+}
diff --git a/yandex/search_raw.go b/yandex/search_raw.go
index d63c5de..a65be83 100644
--- a/yandex/search_raw.go
+++ b/yandex/search_raw.go
@@ -1,96 +1,27 @@
package yandex
import (
+ "bytes"
"context"
+ "errors"
"fmt"
- "net/http"
- "strings"
"github.com/PuerkitoBio/goquery"
- "github.com/corpix/uarand"
"github.com/karust/openserp/core"
- "github.com/sirupsen/logrus"
)
-func yandexRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) {
- baseClient, err := core.NewRawHTTPClient(query)
+func classifyYandexRawHTML(body []byte) error {
+ doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
if err != nil {
- return nil, err
+ return err
}
-
- req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil)
- if err != nil {
- return nil, err
+ if doc.Find(Selectors.Captcha).Length() > 0 {
+ return core.ErrCaptcha
}
- req.Header.Set("User-Agent", uarand.GetRandom())
- core.SetAcceptLanguageHeader(req, query.LangCode)
-
- res, err := baseClient.Do(req)
- if err != nil {
- return nil, err
+ if doc.Find(Selectors.NoResults).Length() > 0 {
+ return core.ErrEmptyResult
}
- return res, nil
-}
-
-func yandexResultParser(response *http.Response) ([]core.SearchResult, error) {
- doc, err := goquery.NewDocumentFromReader(response.Body)
- if err != nil {
- return nil, err
- }
-
- results := []core.SearchResult{}
- rank := 1
-
- // Prefer stable container + attributes and keep legacy fallback.
- sel := doc.Find("#search-result > li[data-fast], li.serp-item")
-
- for i := range sel.Nodes {
- item := sel.Eq(i)
-
- // Skip blocks without a result heading.
- titleTag := item.Find("h2").First()
- if titleTag.Length() == 0 {
- continue
- }
-
- // Find URL
- linkTag := item.Find("a.OrganicTitle-Link").First()
- if linkTag.Length() == 0 {
- linkTag = titleTag.Closest("a")
- }
- if linkTag.Length() == 0 {
- linkTag = item.Find("a").First()
- }
- link, _ := linkTag.Attr("href")
- link = strings.Trim(link, " ")
-
- // Find title
- title := strings.TrimSpace(titleTag.Text())
-
- // Find description
- descTag := item.Find(`span.OrganicTextContentSpan`).First()
- if descTag.Length() == 0 {
- descTag = item.Find("div.OrganicText").First()
- }
- desc := strings.TrimSpace(descTag.Text())
-
- if link != "" && link != "#" && title != "" {
- result := core.SearchResult{
- Rank: rank,
- URL: link,
- Title: title,
- Description: desc,
- }
-
- results = append(results, result)
- rank++
- }
- }
-
- logrus.WithField("document_size", len(doc.Text())).Trace(
- fmt.Sprintf("Yandex search document size: %d", len(doc.Text())),
- )
- return core.DeduplicateResults(results), err
+ return nil
}
func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -109,13 +40,13 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
return nil, err
}
- googleURL, err := BuildURL(query, startPage)
+ searchURL, err := BuildURL(query, startPage)
if err != nil {
return nil, err
}
- core.WithRequest(ctx).WithField("url", googleURL).Debug(fmt.Sprintf("Yandex URL built: %s", googleURL))
+ core.WithRequest(ctx).WithField("url", searchURL).Debug(fmt.Sprintf("Yandex URL built: %s", searchURL))
- res, err := yandexRequest(ctx, googleURL, query)
+ res, err := core.RawSearchRequest(ctx, searchURL, query)
if err != nil {
return nil, err
}
@@ -124,10 +55,25 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
fmt.Sprintf("Yandex Raw response: code=%d", res.StatusCode),
)
- parsedResults, err := yandexResultParser(res)
+ body, err := core.ReadRawSearchBody(res)
if err != nil {
return nil, err
}
+ htmlStatus := classifyYandexRawHTML(body)
+ if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) {
+ return nil, htmlStatus
+ }
+
+ parsedResults, err := ParseHTML(bytes.NewReader(body))
+ if err != nil {
+ return nil, err
+ }
+ if len(parsedResults) == 0 {
+ if errors.Is(htmlStatus, core.ErrEmptyResult) {
+ return []core.SearchResult{}, nil
+ }
+ return nil, fmt.Errorf("%w: yandex raw search returned no parseable results", core.ErrParser)
+ }
if skipOnFirstPage > 0 {
if skipOnFirstPage >= len(parsedResults) {
diff --git a/yandex/search_raw_test.go b/yandex/search_raw_test.go
index 7be7444..a504137 100644
--- a/yandex/search_raw_test.go
+++ b/yandex/search_raw_test.go
@@ -1,76 +1,69 @@
package yandex
import (
+ "errors"
+ "io"
"testing"
+ "github.com/karust/openserp/core"
"github.com/karust/openserp/testutil"
)
-func TestYandexResultParserSnapshots(t *testing.T) {
+// TestYandexParseHTMLFixtures covers the no-results and captcha fixtures.
+// The happy path is covered in TestParseYandexHTML; this file ensures the
+// shared parser (used by both raw mode and the /yandex/parse endpoint) does
+// not over-extract on captcha or empty SERPs.
+func TestYandexParseHTMLFixtures(t *testing.T) {
t.Parallel()
tests := []struct {
- name string
- fixture string
- minResultCount int
- maxResultCount int
- wantZero bool
+ name string
+ fixture string
+ wantZero bool
}{
- {
- name: "search results",
- fixture: "search_results.html",
- minResultCount: 1,
- maxResultCount: 30,
- },
- {
- name: "no results",
- fixture: "search_no_results.html",
- wantZero: true,
- },
- {
- name: "captcha page",
- fixture: "search_captcha.html",
- wantZero: true,
- },
+ {name: "no results", fixture: "search_no_results.html", wantZero: true},
+ {name: "captcha page", fixture: "search_captcha.html", wantZero: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
- results, err := yandexResultParser(testutil.ResponseFromFixture(t, tt.fixture))
+ results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body)
if err != nil {
- t.Fatalf("yandexResultParser() error = %v", err)
+ t.Fatalf("ParseHTML() error = %v", err)
}
-
- if tt.wantZero {
- if len(results) != 0 {
- t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
- }
- return
+ if tt.wantZero && len(results) != 0 {
+ t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results))
}
-
- if len(results) < tt.minResultCount || len(results) > tt.maxResultCount {
- t.Fatalf(
- "unexpected result count for %s: got %d, want range [%d,%d]",
- tt.fixture, len(results), tt.minResultCount, tt.maxResultCount,
- )
- }
-
- testutil.AssertSequentialRanks(t, results)
- testutil.AssertFirstResultFilled(t, results)
})
}
}
-func TestYandexResultParserEmptyHTML(t *testing.T) {
+func TestYandexClassifyRawHTML(t *testing.T) {
t.Parallel()
- results, err := yandexResultParser(testutil.ResponseFromString(""))
- if err != nil {
- t.Fatalf("yandexResultParser() error = %v", err)
+ tests := []struct {
+ name string
+ fixture string
+ want error
+ }{
+ {name: "no results", fixture: "search_no_results.html", want: core.ErrEmptyResult},
+ {name: "captcha page", fixture: "search_captcha.html", want: core.ErrCaptcha},
}
- if len(results) != 0 {
- t.Fatalf("expected zero results for empty HTML, got %d", len(results))
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ body, err := io.ReadAll(testutil.ResponseFromFixture(t, tt.fixture).Body)
+ if err != nil {
+ t.Fatalf("read fixture body: %v", err)
+ }
+ err = classifyYandexRawHTML(body)
+ if !errors.Is(err, tt.want) {
+ t.Fatalf("expected %v for %s, got %v", tt.want, tt.fixture, err)
+ }
+ })
}
}
diff --git a/yandex/selectors.go b/yandex/selectors.go
index 9cb2f60..1913b94 100644
--- a/yandex/selectors.go
+++ b/yandex/selectors.go
@@ -5,9 +5,13 @@ var Selectors = struct {
Captcha string
NoResults string
Results string
+ // LinkPrimary is preferred over a generic ; falls back to title.Closest("a")
+ // then the first in the result block when absent.
+ LinkPrimary string
Link string
Title string
Desc string
+ DescFallback string
ImageItems string
ImageItemsAlt []string
ImageStateAll string
@@ -15,9 +19,11 @@ var Selectors = struct {
Captcha: "div.CheckboxCaptcha",
NoResults: "div.EmptySearchResults",
Results: "li[data-fast], li.serp-item",
+ LinkPrimary: "a.OrganicTitle-Link",
Link: "a",
Title: "h2",
Desc: "span.OrganicTextContentSpan",
+ DescFallback: "div.OrganicText",
ImageItems: "div[role='main'] div[data-state]",
ImageItemsAlt: []string{"div[data-state*='serpList']"},
ImageStateAll: "div[data-state]",