From 88337cd7ce972a93386ef37356f667a11bc58af2 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Fri, 8 May 2026 00:00:33 +0300 Subject: [PATCH] refactor: Speed up Baidu. Fix raw client + reuse HTML parsers for raw search --- baidu/parse_html.go | 72 +++++++++++++++++++++--- baidu/parse_html_test.go | 53 ++++++++++++++++++ baidu/search.go | 115 ++++++++++++++++++-------------------- baidu/search_raw.go | 113 ++++++++++--------------------------- baidu/search_raw_test.go | 85 +++++++++++++--------------- baidu/selectors.go | 8 ++- cmd/root.go | 2 +- core/http_client.go | 67 +++++++++++++++++++++- core/http_client_test.go | 30 ++++++++++ core/proxy_test.go | 28 ++++++++++ ecosia/search_raw.go | 105 ++++++++++++---------------------- ecosia/search_raw_test.go | 48 ++++------------ google/search_raw.go | 60 +++++++++++--------- google/search_raw_test.go | 43 ++++++++++++-- yandex/parse_html.go | 32 +++++++++-- yandex/parse_html_test.go | 29 ++++++++++ yandex/search_raw.go | 112 ++++++++++--------------------------- yandex/search_raw_test.go | 85 +++++++++++++--------------- yandex/selectors.go | 6 ++ 19 files changed, 618 insertions(+), 475 deletions(-) diff --git a/baidu/parse_html.go b/baidu/parse_html.go index 6f48d24..d42cf2f 100644 --- a/baidu/parse_html.go +++ b/baidu/parse_html.go @@ -19,22 +19,61 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { } func parseBaiduDocument(doc *goquery.Document) []core.SearchResult { + for _, selector := range baiduResultSelectors() { + if results := parseBaiduSelection(doc.Find(selector)); len(results) > 0 { + return results + } + } + return nil +} + +func baiduResultSelectors() []string { + selectors := make([]string, 0, len(Selectors.ResultsAlt)+1) + selectors = append(selectors, Selectors.Results) + selectors = append(selectors, Selectors.ResultsAlt...) + return selectors +} + +func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult { var results []core.SearchResult rank := 1 - doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) { - linkTag := item.Find(Selectors.Link).First() - if linkTag.Length() == 0 { + sel.Each(func(_ int, item *goquery.Selection) { + // h3-first: organic results always carry a heading; this filters out + // non-result blocks that may share the wrapper class. + titleTag := item.Find("h3").First() + var ( + title string + linkTag *goquery.Selection + ) + if titleTag.Length() > 0 { + title = strings.TrimSpace(titleTag.Text()) + if child := titleTag.Find("a[href]").First(); child.Length() > 0 { + linkTag = child + } else if closest := titleTag.Closest("a[href]"); closest.Length() > 0 { + linkTag = closest + } + } + if linkTag == nil || linkTag.Length() == 0 { + first := item.Find(Selectors.Link).First() + if first.Length() == 0 { + return + } + linkTag = first + } + if title == "" { + title = strings.TrimSpace(linkTag.Text()) + } + if title == "" { return } href, exists := linkTag.Attr("href") - if !exists || href == "" || href == "#" || !strings.HasPrefix(href, "http") { + if !exists { return } - - title := strings.TrimSpace(linkTag.Text()) - if title == "" { + href = strings.TrimSpace(href) + if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") { return } @@ -42,6 +81,16 @@ func parseBaiduDocument(doc *goquery.Document) []core.SearchResult { if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 { desc = strings.TrimSpace(descTag.Text()) } + if desc == "" { + for _, alt := range Selectors.DescAlt { + if descTag := item.Find(alt).First(); descTag.Length() > 0 { + if t := strings.TrimSpace(descTag.Text()); t != "" { + desc = t + break + } + } + } + } if desc == "" { full := strings.TrimSpace(item.Text()) desc = strings.TrimSpace(strings.Replace(full, title, "", 1)) @@ -56,5 +105,12 @@ func parseBaiduDocument(doc *goquery.Document) []core.SearchResult { rank++ }) - return core.DeduplicateResults(results) + // Re-rank sequentially after dedup so callers get a clean 1..N sequence + // (dedup may drop intermediate ranks when the same URL appears in + // multiple Baidu result-card variants on the same SERP). + deduped := core.DeduplicateResults(results) + for i := range deduped { + deduped[i].Rank = i + 1 + } + return deduped } diff --git a/baidu/parse_html_test.go b/baidu/parse_html_test.go index 1d076b4..9823a4d 100644 --- a/baidu/parse_html_test.go +++ b/baidu/parse_html_test.go @@ -51,3 +51,56 @@ func TestParseBaiduHTMLEmpty(t *testing.T) { t.Fatalf("expected zero results for empty HTML, got %d", len(results)) } } + +func TestParseBaiduHTMLFallbackSelectors(t *testing.T) { + t.Parallel() + + html := ` +
+
+

Fallback Title

+
Fallback description
+
+
` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if results[0].URL != "https://example.com/result" { + t.Fatalf("unexpected URL: %s", results[0].URL) + } + if results[0].Title != "Fallback Title" { + t.Fatalf("unexpected title: %s", results[0].Title) + } + if results[0].Description != "Fallback description" { + t.Fatalf("unexpected description: %s", results[0].Description) + } +} + +func TestParseBaiduHTMLFallsBackWhenEarlierSelectorHasNoResult(t *testing.T) { + t.Parallel() + + html := ` +
+
+
+

Parseable Title

+
Parseable description
+
+
` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if results[0].URL != "https://example.com/parseable" { + t.Fatalf("unexpected URL: %s", results[0].URL) + } +} diff --git a/baidu/search.go b/baidu/search.go index bf0f2c2..abaa229 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -91,6 +91,9 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query)) scoped := *baid scoped.logger = baid.logger.WithRequest(ctx) + if scoped.Browser.WaitLoadTime == 0 || scoped.Browser.WaitLoadTime > 250*time.Millisecond { + scoped.Browser.WaitLoadTime = 250 * time.Millisecond + } baid = &scoped baid.logger.Debug("Starting search, query: %+v", query) @@ -101,8 +104,6 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core } }() - searchResults := []core.SearchResult{} - // Build URL from query struct to open in browser url, err := BuildURL(query) if err != nil { @@ -122,69 +123,63 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core } } - resultElements, _, err := core.WaitForElements( - ctx, - page, - append([]string{Selectors.Results}, Selectors.ResultsAlt...), - baid.GetSelectorTimeout(), - ) + searchResults, err := baid.waitForParsedSearchResults(ctx, page, url) if err != nil { - if blockErr := baid.classifyBlockPage(page, url); blockErr != nil { - closePage() - return nil, blockErr - } closePage() - baid.logger.Error("Cannot parse search results: %s", err) - return nil, core.ErrSearchTimeout + return nil, err } - - if len(resultElements) == 0 { - if blockErr := baid.classifyBlockPage(page, url); blockErr != nil { - closePage() - return nil, blockErr - } - closePage() - return nil, nil - } - - for i, r := range resultElements { - // Get URL - link, err := r.Element(Selectors.Link) - if err != nil { - // Element detached mid-iteration (page navigated/refreshed): the rest - // of the slice is also stale, no point continuing. - if core.IsRodObjectNotFound(err) { - break - } - continue - } - linkText, err := link.Property("href") - if err != nil { - baid.logger.Debug("Missing href tag") - continue - } - - // Get title - title, err := link.Text() - if err != nil { - baid.logger.Debug("Failed to extract title") - title = "No title" - } - - // Get description - desc, err := r.Text() - if err != nil { - desc = "" - } - desc = strings.ReplaceAll(desc, title, "") - - gR := core.SearchResult{Rank: query.Start + i + 1, URL: linkText.String(), Title: title, Description: desc} - searchResults = append(searchResults, gR) - } - closePage() - return core.DeduplicateResults(searchResults), nil + for i := range searchResults { + searchResults[i].Rank = query.Start + i + 1 + } + return searchResults, nil +} + +func (baid *Baidu) waitForParsedSearchResults(ctx context.Context, page *rod.Page, url string) ([]core.SearchResult, error) { + timeout := baid.GetSelectorTimeout() + if timeout <= 0 { + timeout = 5 * time.Second + } + deadline := time.Now().Add(timeout) + var sawResultContainer bool + var lastErr error + + for { + html, err := page.HTML() + if err == nil { + results, parseErr := ParseHTML(strings.NewReader(html)) + if parseErr == nil && len(results) > 0 { + return results, nil + } + lastErr = parseErr + } else { + lastErr = err + } + + if core.HasAnySelector(page, baiduResultSelectors()) { + sawResultContainer = true + } + if blockErr := baid.classifyBlockPage(page, url); blockErr != nil { + return nil, blockErr + } + if !time.Now().Before(deadline) { + break + } + if err := core.SleepContext(ctx, 120*time.Millisecond); err != nil { + return nil, err + } + } + + if sawResultContainer { + if lastErr != nil { + baid.logger.Debug("Baidu result containers found but HTML parsing failed: %v", lastErr) + } else { + baid.logger.Debug("Baidu result containers found but no parseable organic results") + } + return nil, core.ErrParser + } + return nil, nil } // SearchImage executes a Baidu image search and returns normalized image diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 2d526a5..ae4b9ac 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -1,97 +1,27 @@ package baidu import ( + "bytes" "context" + "errors" "fmt" - "net/http" - "strings" "github.com/PuerkitoBio/goquery" - "github.com/corpix/uarand" "github.com/karust/openserp/core" - "github.com/sirupsen/logrus" ) -func baiduRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) { - baseClient, err := core.NewRawHTTPClient(query) +func classifyBaiduRawHTML(body []byte) error { + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body)) if err != nil { - return nil, err + return err } - - req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) - if err != nil { - return nil, err + if doc.Find(Selectors.Captcha).Length() > 0 || doc.Find(Selectors.Timeout).Length() > 0 { + return core.ErrCaptcha } - req.Header.Set("User-Agent", uarand.GetRandom()) - core.SetAcceptLanguageHeader(req, query.LangCode) - - res, err := baseClient.Do(req) - if err != nil { - return nil, err + if doc.Find("div.content_none, div.nors").Length() > 0 { + return core.ErrEmptyResult } - return res, nil -} - -func baiduResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromReader(response.Body) - if err != nil { - return nil, err - } - - results := []core.SearchResult{} - rank := 1 - - // Prefer organic result blocks from the main result column. - sel := doc.Find("#content_left .result.c-container") - if sel.Length() == 0 { - sel = doc.Find("div.c-container.new-pmd") - } - - for i := range sel.Nodes { - item := sel.Eq(i) - - // Find URL - titleTag := item.Find("h3").First() - if titleTag.Length() == 0 { - continue - } - - linkTag := titleTag.Closest("a") - if linkTag.Length() == 0 { - linkTag = item.Find("a").First() - } - link, _ := linkTag.Attr("href") - link = strings.TrimSpace(link) - - // Find title - title := strings.TrimSpace(titleTag.Text()) - - // Find description - descTag := item.Find(".c-abstract, .content-right_8Zs40, .summary-gap_3Jb4I").First() - desc := strings.TrimSpace(descTag.Text()) - if desc == "" { - desc = strings.TrimSpace(item.Text()) - } - desc = strings.ReplaceAll(desc, title, "") - desc = strings.TrimSpace(desc) - - if link != "" && link != "#" && title != "" { - result := core.SearchResult{ - Rank: rank, - URL: link, - Title: title, - Description: desc, - } - - results = append(results, result) - rank++ - } - } - - logrus.WithField("document_size", len(doc.Text())).Trace( - fmt.Sprintf("Baidu search document size: %d", len(doc.Text())), - ) - return results, err + return nil } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { @@ -105,13 +35,13 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, } }() - googleURL, err := BuildURL(query) + searchURL, err := BuildURL(query) if err != nil { return nil, err } - core.WithRequest(ctx).WithField("url", googleURL).Debug(fmt.Sprintf("Baidu URL built: %s", googleURL)) + core.WithRequest(ctx).WithField("url", searchURL).Debug(fmt.Sprintf("Baidu URL built: %s", searchURL)) - res, err := baiduRequest(ctx, googleURL, query) + res, err := core.RawSearchRequest(ctx, searchURL, query) if err != nil { return nil, err } @@ -120,10 +50,25 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, fmt.Sprintf("Baidu Raw response: code=%d", res.StatusCode), ) - parsedResults, err := baiduResultParser(res) + body, err := core.ReadRawSearchBody(res) if err != nil { return nil, err } + htmlStatus := classifyBaiduRawHTML(body) + if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) { + return nil, htmlStatus + } + + parsedResults, err := ParseHTML(bytes.NewReader(body)) + if err != nil { + return nil, err + } + if len(parsedResults) == 0 { + if errors.Is(htmlStatus, core.ErrEmptyResult) { + return []core.SearchResult{}, nil + } + return nil, fmt.Errorf("%w: baidu raw search returned no parseable results", core.ErrParser) + } if query.Start > 0 { for i := range parsedResults { parsedResults[i].Rank = query.Start + i + 1 diff --git a/baidu/search_raw_test.go b/baidu/search_raw_test.go index 0b306c9..2d1bb49 100644 --- a/baidu/search_raw_test.go +++ b/baidu/search_raw_test.go @@ -1,76 +1,69 @@ package baidu import ( + "errors" + "io" "testing" + "github.com/karust/openserp/core" "github.com/karust/openserp/testutil" ) -func TestBaiduResultParserSnapshots(t *testing.T) { +// TestBaiduParseHTMLFixtures covers the no-results and captcha fixtures. +// The happy path is covered in TestParseBaiduHTML; this file ensures the +// shared parser (used by both raw mode and the /baidu/parse endpoint) does +// not over-extract on captcha or empty SERPs. +func TestBaiduParseHTMLFixtures(t *testing.T) { t.Parallel() tests := []struct { - name string - fixture string - minResultCount int - maxResultCount int - wantZero bool + name string + fixture string + wantZero bool }{ - { - name: "search results", - fixture: "search_results.html", - minResultCount: 5, - maxResultCount: 30, - }, - { - name: "no results", - fixture: "search_no_results.html", - wantZero: true, - }, - { - name: "captcha page", - fixture: "search_captcha.html", - wantZero: true, - }, + {name: "no results", fixture: "search_no_results.html", wantZero: true}, + {name: "captcha page", fixture: "search_captcha.html", wantZero: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - results, err := baiduResultParser(testutil.ResponseFromFixture(t, tt.fixture)) + results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body) if err != nil { - t.Fatalf("baiduResultParser() error = %v", err) + t.Fatalf("ParseHTML() error = %v", err) } - - if tt.wantZero { - if len(results) != 0 { - t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results)) - } - return + if tt.wantZero && len(results) != 0 { + t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results)) } - - if len(results) < tt.minResultCount || len(results) > tt.maxResultCount { - t.Fatalf( - "unexpected result count for %s: got %d, want range [%d,%d]", - tt.fixture, len(results), tt.minResultCount, tt.maxResultCount, - ) - } - - testutil.AssertSequentialRanks(t, results) - testutil.AssertFirstResultFilled(t, results) }) } } -func TestBaiduResultParserEmptyHTML(t *testing.T) { +func TestBaiduClassifyRawHTML(t *testing.T) { t.Parallel() - results, err := baiduResultParser(testutil.ResponseFromString("")) - if err != nil { - t.Fatalf("baiduResultParser() error = %v", err) + tests := []struct { + name string + fixture string + want error + }{ + {name: "no results", fixture: "search_no_results.html", want: core.ErrEmptyResult}, + {name: "captcha page", fixture: "search_captcha.html", want: core.ErrCaptcha}, } - if len(results) != 0 { - t.Fatalf("expected zero results for empty HTML, got %d", len(results)) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + body, err := io.ReadAll(testutil.ResponseFromFixture(t, tt.fixture).Body) + if err != nil { + t.Fatalf("read fixture body: %v", err) + } + err = classifyBaiduRawHTML(body) + if !errors.Is(err, tt.want) { + t.Fatalf("expected %v for %s, got %v", tt.want, tt.fixture, err) + } + }) } } diff --git a/baidu/selectors.go b/baidu/selectors.go index 4fc1405..d22a972 100644 --- a/baidu/selectors.go +++ b/baidu/selectors.go @@ -9,12 +9,16 @@ var Selectors = struct { ImageJSONRoot []string Link string Desc string + // DescAlt are additional description containers tried when Desc misses. + // Baidu varies abstract markup across feature blocks (info cards, news rows). + DescAlt []string }{ Captcha: "div.passMod_dialog-wrapper", Timeout: "button.timeout-button", - Results: "div.c-container.new-pmd", - ResultsAlt: []string{"#content_left div.result.c-container", "#content_left div.result-op.c-container"}, + Results: "#content_left div.result.c-container", + ResultsAlt: []string{"#content_left div.result-op.c-container", "div.c-container.new-pmd"}, ImageJSONRoot: []string{"body > pre", "pre"}, Link: "a", Desc: "div.c-abstract", + DescAlt: []string{".content-right_8Zs40", ".summary-gap_3Jb4I"}, } diff --git a/cmd/root.go b/cmd/root.go index e4241d8..2bbc746 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ import ( ) const ( - version = "0.7.9" + version = "0.7.10" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/http_client.go b/core/http_client.go index 4c0f99c..48bf828 100644 --- a/core/http_client.go +++ b/core/http_client.go @@ -3,6 +3,7 @@ package core import ( "context" "crypto/tls" + "fmt" "io" "net" "net/http" @@ -10,10 +11,11 @@ import ( "strings" "time" + "github.com/corpix/uarand" utls "github.com/refraction-networking/utls" ) -const rawHTTPTimeout = 10 * time.Second +const rawHTTPTimeout = 30 * time.Second // SetAcceptLanguageHeader sets the Accept-Language header from a lang code. // No-op when the code has no language subtag. @@ -37,6 +39,53 @@ func DrainAndCloseResponse(resp *http.Response) { _ = resp.Body.Close() } +// RawSearchRequest builds and executes a raw-mode SERP HTTP GET. It uses the +// shared raw HTTP client (TLS fingerprinting, network usage tracking, proxy +// support), randomizes the User-Agent, and applies the Accept-Language header +// derived from the query locale. The caller owns the returned response and +// must drain/close it (see DrainAndCloseResponse). +func RawSearchRequest(ctx context.Context, searchURL string, query Query) (*http.Response, error) { + client, err := NewRawHTTPClient(query) + if err != nil { + return nil, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", uarand.GetRandom()) + SetAcceptLanguageHeader(req, query.LangCode) + return client.Do(req) +} + +func ReadRawSearchBody(resp *http.Response) ([]byte, error) { + if resp == nil { + return nil, fmt.Errorf("%w: nil raw search response", ErrEngineInternal) + } + if err := ClassifySearchHTTPStatus(resp.StatusCode); err != nil { + return nil, err + } + return io.ReadAll(resp.Body) +} + +func ClassifySearchHTTPStatus(status int) error { + switch status { + case 0: + return nil + case http.StatusForbidden, http.StatusUnauthorized: + return ErrBlocked + case http.StatusTooManyRequests: + return ErrRateLimited + } + if status >= 500 { + return fmt.Errorf("%w: search engine returned HTTP %d", ErrBlocked, status) + } + if status < 200 || status >= 300 { + return fmt.Errorf("%w: search engine returned HTTP %d", ErrParser, status) + } + return nil +} + func NewRawHTTPClient(query Query) (*http.Client, error) { transport, err := newRawTransport(query) if err != nil { @@ -87,9 +136,15 @@ func newRawTransport(query Query) (*http.Transport, error) { config := &utls.Config{ ServerName: hostname, InsecureSkipVerify: query.Insecure, + NextProtos: []string{"http/1.1"}, } uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto) + if err := uconn.BuildHandshakeState(); err != nil { + rawConn.Close() + return nil, err + } + forceHTTP1ALPN(uconn) if err := uconn.Handshake(); err != nil { rawConn.Close() return nil, err @@ -101,6 +156,16 @@ func newRawTransport(query Query) (*http.Transport, error) { return transport, nil } +func forceHTTP1ALPN(conn *utls.UConn) { + for _, ext := range conn.Extensions { + if alpn, ok := ext.(*utls.ALPNExtension); ok { + alpn.AlpnProtocols = []string{"http/1.1"} + return + } + } + conn.Extensions = append(conn.Extensions, &utls.ALPNExtension{AlpnProtocols: []string{"http/1.1"}}) +} + func dialNetworkUsageConn(ctx context.Context, network, addr string) (net.Conn, error) { dialer := &net.Dialer{} conn, err := dialer.DialContext(ctx, network, addr) diff --git a/core/http_client_test.go b/core/http_client_test.go index 4ec725c..88f92cd 100644 --- a/core/http_client_test.go +++ b/core/http_client_test.go @@ -36,6 +36,36 @@ func TestDrainAndCloseResponseDrainsAndCloses(t *testing.T) { } } +func TestClassifySearchHTTPStatus(t *testing.T) { + tests := []struct { + name string + status int + want error + }{ + {name: "unknown browser status", status: 0, want: nil}, + {name: "ok", status: http.StatusOK, want: nil}, + {name: "blocked", status: http.StatusForbidden, want: ErrBlocked}, + {name: "rate limited", status: http.StatusTooManyRequests, want: ErrRateLimited}, + {name: "server error", status: http.StatusBadGateway, want: ErrBlocked}, + {name: "unexpected status", status: http.StatusNotFound, want: ErrParser}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ClassifySearchHTTPStatus(tt.status) + if tt.want == nil { + if err != nil { + t.Fatalf("expected nil error, got %v", err) + } + return + } + if !errors.Is(err, tt.want) { + t.Fatalf("expected %v, got %v", tt.want, err) + } + }) + } +} + func TestDrainAndCloseResponseNilSafe(t *testing.T) { DrainAndCloseResponse(nil) DrainAndCloseResponse(&http.Response{}) diff --git a/core/proxy_test.go b/core/proxy_test.go index 3b32f6c..62cfb30 100644 --- a/core/proxy_test.go +++ b/core/proxy_test.go @@ -344,6 +344,34 @@ func TestNewRawHTTPClientSocks5hUsesProxyDNS(t *testing.T) { } } +func TestNewRawHTTPClientDirectTLSUsesHTTP1(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(r.Proto)) + })) + server.EnableHTTP2 = true + server.StartTLS() + defer server.Close() + + client, err := NewRawHTTPClient(Query{Insecure: true}) + if err != nil { + t.Fatalf("new raw http client: %v", err) + } + + resp, err := client.Get(server.URL) + if err != nil { + t.Fatalf("expected direct TLS request to succeed, got %v", err) + } + defer DrainAndCloseResponse(resp) + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read response body: %v", err) + } + if string(body) != "HTTP/1.1" { + t.Fatalf("expected raw client to use HTTP/1.1, got %q", string(body)) + } +} + func TestClassifyProxyNetworkError(t *testing.T) { tests := []struct { name string diff --git a/ecosia/search_raw.go b/ecosia/search_raw.go index 04074f1..aede483 100644 --- a/ecosia/search_raw.go +++ b/ecosia/search_raw.go @@ -1,84 +1,32 @@ package ecosia import ( + "bytes" "context" + "errors" "fmt" "net/http" "strings" "github.com/PuerkitoBio/goquery" - "github.com/corpix/uarand" "github.com/karust/openserp/core" ) -func ecosiaRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) { - baseClient, err := core.NewRawHTTPClient(query) +func classifyEcosiaRawHTML(body []byte) error { + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body)) if err != nil { - return nil, err + return err } - - req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) - if err != nil { - return nil, err + if strings.Contains(strings.ToLower(doc.Text()), "captcha") { + return core.ErrCaptcha } - req.Header.Set("User-Agent", uarand.GetRandom()) - core.SetAcceptLanguageHeader(req, query.LangCode) - - res, err := baseClient.Do(req) - if err != nil { - return nil, err + if doc.Find("[data-test-id='web-no-results']").Length() > 0 || + (doc.Find(Selectors.Mainline).Length() > 0 && + doc.Find(Selectors.Result).Length() == 0 && + doc.Find(Selectors.Ad).Length() == 0) { + return core.ErrEmptyResult } - return res, nil -} - -// resultParser parses an Ecosia SERP HTML response into search results -// using goquery (no browser required). -func resultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromReader(response.Body) - if err != nil { - return nil, err - } - var ( - results []core.SearchResult - rank = 1 - ) - doc.Find(Selectors.Result).Each(func(_ int, s *goquery.Selection) { - href, ok := s.Find(Selectors.ResultLink).Attr("href") - if !ok || strings.TrimSpace(href) == "" { - return - } - var ( - title = strings.TrimSpace(s.Find(Selectors.Title).Text()) - desc = strings.TrimSpace(s.Find(Selectors.Desc).Text()) - ) - results = append(results, core.SearchResult{ - Rank: rank, - URL: href, - Title: title, - Description: desc, - }) - rank++ - }) - adRank := -1 - doc.Find(Selectors.Ad).Each(func(_ int, s *goquery.Selection) { - href, ok := s.Find(Selectors.ResultLink).Attr("href") - if !ok || strings.TrimSpace(href) == "" { - return - } - var ( - title = strings.TrimSpace(s.Find(Selectors.Title).Text()) - desc = strings.TrimSpace(s.Find(Selectors.Desc).Text()) - ) - results = append(results, core.SearchResult{ - Rank: adRank, - URL: href, - Title: title, - Description: desc, - Ad: true, - }) - adRank-- - }) - return results, nil + return nil } // imageResultParser parses an Ecosia image SERP HTML response into search @@ -144,7 +92,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, } core.WithRequest(ctx).WithField("url", ecosiaURL).Debug(fmt.Sprintf("Ecosia URL built: %s", ecosiaURL)) - res, err := ecosiaRequest(ctx, ecosiaURL, query) + res, err := core.RawSearchRequest(ctx, ecosiaURL, query) if err != nil { return nil, err } @@ -153,15 +101,36 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, fmt.Sprintf("Ecosia Raw response: code=%d", res.StatusCode), ) - parsedResults, err := resultParser(res) + body, err := core.ReadRawSearchBody(res) if err != nil { return nil, err } + htmlStatus := classifyEcosiaRawHTML(body) + if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) { + return nil, htmlStatus + } + + parsedResults, err := ParseHTML(bytes.NewReader(body)) + if err != nil { + return nil, err + } + if len(parsedResults) == 0 { + if errors.Is(htmlStatus, core.ErrEmptyResult) { + return []core.SearchResult{}, nil + } + return nil, fmt.Errorf("%w: ecosia raw search returned no parseable results", core.ErrParser) + } // Ecosia paginates by page index, not result offset, so re-rank from // the page boundary rather than query.Start (off-grid offsets round down). + // Skip ads (rank<0) so organic ranks stay sequential from startRank. + organicIdx := 0 for i := range parsedResults { - parsedResults[i].Rank = startRank + i + if parsedResults[i].Ad { + continue + } + parsedResults[i].Rank = startRank + organicIdx + organicIdx++ } core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug( diff --git a/ecosia/search_raw_test.go b/ecosia/search_raw_test.go index 9154d1c..2c8f7ba 100644 --- a/ecosia/search_raw_test.go +++ b/ecosia/search_raw_test.go @@ -1,44 +1,14 @@ package ecosia import ( + "errors" + "io" "testing" + "github.com/karust/openserp/core" "github.com/karust/openserp/testutil" ) -func TestEcosiaResultParser(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - fixture string - wantCount int - }{ - {name: "search results", fixture: "search_results.html", wantCount: 10}, - {name: "no results", fixture: "search_no_results.html", wantCount: 0}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - results, err := resultParser(testutil.ResponseFromFixture(t, tt.fixture)) - if err != nil { - t.Fatalf("resultParser() error = %v", err) - } - if len(results) != tt.wantCount { - t.Fatalf("expected %d results for %s, got %d", tt.wantCount, tt.fixture, len(results)) - } - if tt.wantCount == 0 { - return - } - - testutil.AssertSequentialRanks(t, results) - testutil.AssertFirstResultFilled(t, results) - }) - } -} - func TestEcosiaImageResultParser(t *testing.T) { t.Parallel() @@ -71,14 +41,16 @@ func TestEcosiaImageResultParser(t *testing.T) { } } -func TestEcosiaResultParserEmptyHTML(t *testing.T) { +func TestEcosiaClassifyRawHTML(t *testing.T) { t.Parallel() - results, err := resultParser(testutil.ResponseFromString("")) + body, err := io.ReadAll(testutil.ResponseFromFixture(t, "search_no_results.html").Body) if err != nil { - t.Fatalf("resultParser() error = %v", err) + t.Fatalf("read fixture body: %v", err) } - if len(results) != 0 { - t.Fatalf("expected zero results for empty HTML, got %d", len(results)) + + err = classifyEcosiaRawHTML(body) + if !errors.Is(err, core.ErrEmptyResult) { + t.Fatalf("expected %v for search_no_results.html, got %v", core.ErrEmptyResult, err) } } diff --git a/google/search_raw.go b/google/search_raw.go index 73550ba..afa2a2b 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -1,38 +1,18 @@ package google import ( + "bytes" "context" + "errors" "fmt" "io" - "net/http" "strings" "github.com/PuerkitoBio/goquery" - "github.com/corpix/uarand" "github.com/karust/openserp/core" "github.com/sirupsen/logrus" ) -func googleRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) { - baseClient, err := core.NewRawHTTPClient(query) - if err != nil { - return nil, err - } - - req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) - if err != nil { - return nil, err - } - req.Header.Set("User-Agent", uarand.GetRandom()) - core.SetAcceptLanguageHeader(req, query.LangCode) - - res, err := baseClient.Do(req) - if err != nil { - return nil, err - } - return res, nil -} - // ParseHTML parses a Google SERP HTML document and returns search results. // It is the pure parser used by both raw HTTP search and parse endpoints. func ParseHTML(r io.Reader) ([]core.SearchResult, error) { @@ -111,8 +91,21 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult { return core.DeduplicateResults(results) } -func googleResultParser(response *http.Response) ([]core.SearchResult, error) { - return ParseHTML(response.Body) +func classifyGoogleRawHTML(body []byte) error { + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body)) + if err != nil { + return err + } + if doc.Find(Selectors.Captcha).Length() > 0 { + return core.ErrCaptcha + } + + text := strings.ToLower(doc.Text()) + if strings.Contains(text, "did not match any documents") || + strings.Contains(text, "about 0 results") { + return core.ErrEmptyResult + } + return nil } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { @@ -132,7 +125,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, } core.WithRequest(ctx).WithField("url", googleURL).Debug(fmt.Sprintf("Google URL built: %s", googleURL)) - res, err := googleRequest(ctx, googleURL, query) + res, err := core.RawSearchRequest(ctx, googleURL, query) if err != nil { return nil, err } @@ -141,10 +134,25 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, fmt.Sprintf("Google Raw response: code=%d", res.StatusCode), ) - parsedResults, err := googleResultParser(res) + body, err := core.ReadRawSearchBody(res) if err != nil { return nil, err } + htmlStatus := classifyGoogleRawHTML(body) + if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) { + return nil, htmlStatus + } + + parsedResults, err := ParseHTML(bytes.NewReader(body)) + if err != nil { + return nil, err + } + if len(parsedResults) == 0 { + if errors.Is(htmlStatus, core.ErrEmptyResult) { + return []core.SearchResult{}, nil + } + return nil, fmt.Errorf("%w: google raw search returned no parseable results", core.ErrParser) + } if query.Start > 0 { for i := range parsedResults { diff --git a/google/search_raw_test.go b/google/search_raw_test.go index 5a35f1f..436bcad 100644 --- a/google/search_raw_test.go +++ b/google/search_raw_test.go @@ -1,12 +1,15 @@ package google import ( + "errors" + "io" "testing" + "github.com/karust/openserp/core" "github.com/karust/openserp/testutil" ) -func TestGoogleResultParserSnapshots(t *testing.T) { +func TestGoogleParseHTMLFixtures(t *testing.T) { t.Parallel() tests := []struct { @@ -38,9 +41,9 @@ func TestGoogleResultParserSnapshots(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - results, err := googleResultParser(testutil.ResponseFromFixture(t, tt.fixture)) + results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body) if err != nil { - t.Fatalf("googleResultParser() error = %v", err) + t.Fatalf("ParseHTML() error = %v", err) } if tt.wantZero { @@ -63,14 +66,42 @@ func TestGoogleResultParserSnapshots(t *testing.T) { } } -func TestGoogleResultParserEmptyHTML(t *testing.T) { +func TestGoogleParseHTMLEmptyHTML(t *testing.T) { t.Parallel() - results, err := googleResultParser(testutil.ResponseFromString("")) + results, err := ParseHTML(testutil.ResponseFromString("").Body) if err != nil { - t.Fatalf("googleResultParser() error = %v", err) + t.Fatalf("ParseHTML() error = %v", err) } if len(results) != 0 { t.Fatalf("expected zero results for empty HTML, got %d", len(results)) } } + +func TestGoogleClassifyRawHTML(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + fixture string + want error + }{ + {name: "no results", fixture: "search_no_results.html", want: core.ErrEmptyResult}, + {name: "captcha page", fixture: "search_captcha.html", want: core.ErrCaptcha}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + body, err := io.ReadAll(testutil.ResponseFromFixture(t, tt.fixture).Body) + if err != nil { + t.Fatalf("read fixture body: %v", err) + } + err = classifyGoogleRawHTML(body) + if !errors.Is(err, tt.want) { + t.Fatalf("expected %v for %s, got %v", tt.want, tt.fixture, err) + } + }) + } +} diff --git a/yandex/parse_html.go b/yandex/parse_html.go index a1182a3..896a384 100644 --- a/yandex/parse_html.go +++ b/yandex/parse_html.go @@ -23,20 +23,35 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult { rank := 1 doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) { - linkTag := item.Find("a").First() + // Skip blocks without a result heading (filters out non-organic blocks + // that share the result-row container). + titleTag := item.Find(Selectors.Title).First() + if titleTag.Length() == 0 { + return + } + + // Prefer the canonical organic-title link, then the closest wrapping + // the title, then any in the block. + linkTag := item.Find(Selectors.LinkPrimary).First() + if linkTag.Length() == 0 { + linkTag = titleTag.Closest("a") + } + if linkTag.Length() == 0 { + linkTag = item.Find(Selectors.Link).First() + } if linkTag.Length() == 0 { return } href, exists := linkTag.Attr("href") - if !exists || href == "" || href == "#" || strings.HasPrefix(href, "javascript:") { + if !exists { + return + } + href = strings.TrimSpace(href) + if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") { return } - titleTag := item.Find(Selectors.Title).First() - if titleTag.Length() == 0 { - return - } title := strings.TrimSpace(titleTag.Text()) if title == "" { return @@ -46,6 +61,11 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult { if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 { desc = strings.TrimSpace(descTag.Text()) } + if desc == "" { + if descTag := item.Find(Selectors.DescFallback).First(); descTag.Length() > 0 { + desc = strings.TrimSpace(descTag.Text()) + } + } results = append(results, core.SearchResult{ Rank: rank, diff --git a/yandex/parse_html_test.go b/yandex/parse_html_test.go index 5c7691f..149ee00 100644 --- a/yandex/parse_html_test.go +++ b/yandex/parse_html_test.go @@ -55,3 +55,32 @@ func TestParseYandexHTMLEmpty(t *testing.T) { t.Fatalf("expected zero results for empty HTML, got %d", len(results)) } } + +func TestParseYandexHTMLFallbackSelectors(t *testing.T) { + t.Parallel() + + html := ` +` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if results[0].URL != "https://example.com/result" { + t.Fatalf("unexpected URL: %s", results[0].URL) + } + if results[0].Title != "Fallback Title" { + t.Fatalf("unexpected title: %s", results[0].Title) + } + if results[0].Description != "Fallback description" { + t.Fatalf("unexpected description: %s", results[0].Description) + } +} diff --git a/yandex/search_raw.go b/yandex/search_raw.go index d63c5de..a65be83 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -1,96 +1,27 @@ package yandex import ( + "bytes" "context" + "errors" "fmt" - "net/http" - "strings" "github.com/PuerkitoBio/goquery" - "github.com/corpix/uarand" "github.com/karust/openserp/core" - "github.com/sirupsen/logrus" ) -func yandexRequest(ctx context.Context, searchURL string, query core.Query) (*http.Response, error) { - baseClient, err := core.NewRawHTTPClient(query) +func classifyYandexRawHTML(body []byte) error { + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body)) if err != nil { - return nil, err + return err } - - req, err := http.NewRequestWithContext(ctx, "GET", searchURL, nil) - if err != nil { - return nil, err + if doc.Find(Selectors.Captcha).Length() > 0 { + return core.ErrCaptcha } - req.Header.Set("User-Agent", uarand.GetRandom()) - core.SetAcceptLanguageHeader(req, query.LangCode) - - res, err := baseClient.Do(req) - if err != nil { - return nil, err + if doc.Find(Selectors.NoResults).Length() > 0 { + return core.ErrEmptyResult } - return res, nil -} - -func yandexResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromReader(response.Body) - if err != nil { - return nil, err - } - - results := []core.SearchResult{} - rank := 1 - - // Prefer stable container + attributes and keep legacy fallback. - sel := doc.Find("#search-result > li[data-fast], li.serp-item") - - for i := range sel.Nodes { - item := sel.Eq(i) - - // Skip blocks without a result heading. - titleTag := item.Find("h2").First() - if titleTag.Length() == 0 { - continue - } - - // Find URL - linkTag := item.Find("a.OrganicTitle-Link").First() - if linkTag.Length() == 0 { - linkTag = titleTag.Closest("a") - } - if linkTag.Length() == 0 { - linkTag = item.Find("a").First() - } - link, _ := linkTag.Attr("href") - link = strings.Trim(link, " ") - - // Find title - title := strings.TrimSpace(titleTag.Text()) - - // Find description - descTag := item.Find(`span.OrganicTextContentSpan`).First() - if descTag.Length() == 0 { - descTag = item.Find("div.OrganicText").First() - } - desc := strings.TrimSpace(descTag.Text()) - - if link != "" && link != "#" && title != "" { - result := core.SearchResult{ - Rank: rank, - URL: link, - Title: title, - Description: desc, - } - - results = append(results, result) - rank++ - } - } - - logrus.WithField("document_size", len(doc.Text())).Trace( - fmt.Sprintf("Yandex search document size: %d", len(doc.Text())), - ) - return core.DeduplicateResults(results), err + return nil } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { @@ -109,13 +40,13 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, return nil, err } - googleURL, err := BuildURL(query, startPage) + searchURL, err := BuildURL(query, startPage) if err != nil { return nil, err } - core.WithRequest(ctx).WithField("url", googleURL).Debug(fmt.Sprintf("Yandex URL built: %s", googleURL)) + core.WithRequest(ctx).WithField("url", searchURL).Debug(fmt.Sprintf("Yandex URL built: %s", searchURL)) - res, err := yandexRequest(ctx, googleURL, query) + res, err := core.RawSearchRequest(ctx, searchURL, query) if err != nil { return nil, err } @@ -124,10 +55,25 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, fmt.Sprintf("Yandex Raw response: code=%d", res.StatusCode), ) - parsedResults, err := yandexResultParser(res) + body, err := core.ReadRawSearchBody(res) if err != nil { return nil, err } + htmlStatus := classifyYandexRawHTML(body) + if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) { + return nil, htmlStatus + } + + parsedResults, err := ParseHTML(bytes.NewReader(body)) + if err != nil { + return nil, err + } + if len(parsedResults) == 0 { + if errors.Is(htmlStatus, core.ErrEmptyResult) { + return []core.SearchResult{}, nil + } + return nil, fmt.Errorf("%w: yandex raw search returned no parseable results", core.ErrParser) + } if skipOnFirstPage > 0 { if skipOnFirstPage >= len(parsedResults) { diff --git a/yandex/search_raw_test.go b/yandex/search_raw_test.go index 7be7444..a504137 100644 --- a/yandex/search_raw_test.go +++ b/yandex/search_raw_test.go @@ -1,76 +1,69 @@ package yandex import ( + "errors" + "io" "testing" + "github.com/karust/openserp/core" "github.com/karust/openserp/testutil" ) -func TestYandexResultParserSnapshots(t *testing.T) { +// TestYandexParseHTMLFixtures covers the no-results and captcha fixtures. +// The happy path is covered in TestParseYandexHTML; this file ensures the +// shared parser (used by both raw mode and the /yandex/parse endpoint) does +// not over-extract on captcha or empty SERPs. +func TestYandexParseHTMLFixtures(t *testing.T) { t.Parallel() tests := []struct { - name string - fixture string - minResultCount int - maxResultCount int - wantZero bool + name string + fixture string + wantZero bool }{ - { - name: "search results", - fixture: "search_results.html", - minResultCount: 1, - maxResultCount: 30, - }, - { - name: "no results", - fixture: "search_no_results.html", - wantZero: true, - }, - { - name: "captcha page", - fixture: "search_captcha.html", - wantZero: true, - }, + {name: "no results", fixture: "search_no_results.html", wantZero: true}, + {name: "captcha page", fixture: "search_captcha.html", wantZero: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - results, err := yandexResultParser(testutil.ResponseFromFixture(t, tt.fixture)) + results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body) if err != nil { - t.Fatalf("yandexResultParser() error = %v", err) + t.Fatalf("ParseHTML() error = %v", err) } - - if tt.wantZero { - if len(results) != 0 { - t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results)) - } - return + if tt.wantZero && len(results) != 0 { + t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results)) } - - if len(results) < tt.minResultCount || len(results) > tt.maxResultCount { - t.Fatalf( - "unexpected result count for %s: got %d, want range [%d,%d]", - tt.fixture, len(results), tt.minResultCount, tt.maxResultCount, - ) - } - - testutil.AssertSequentialRanks(t, results) - testutil.AssertFirstResultFilled(t, results) }) } } -func TestYandexResultParserEmptyHTML(t *testing.T) { +func TestYandexClassifyRawHTML(t *testing.T) { t.Parallel() - results, err := yandexResultParser(testutil.ResponseFromString("")) - if err != nil { - t.Fatalf("yandexResultParser() error = %v", err) + tests := []struct { + name string + fixture string + want error + }{ + {name: "no results", fixture: "search_no_results.html", want: core.ErrEmptyResult}, + {name: "captcha page", fixture: "search_captcha.html", want: core.ErrCaptcha}, } - if len(results) != 0 { - t.Fatalf("expected zero results for empty HTML, got %d", len(results)) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + body, err := io.ReadAll(testutil.ResponseFromFixture(t, tt.fixture).Body) + if err != nil { + t.Fatalf("read fixture body: %v", err) + } + err = classifyYandexRawHTML(body) + if !errors.Is(err, tt.want) { + t.Fatalf("expected %v for %s, got %v", tt.want, tt.fixture, err) + } + }) } } diff --git a/yandex/selectors.go b/yandex/selectors.go index 9cb2f60..1913b94 100644 --- a/yandex/selectors.go +++ b/yandex/selectors.go @@ -5,9 +5,13 @@ var Selectors = struct { Captcha string NoResults string Results string + // LinkPrimary is preferred over a generic ; falls back to title.Closest("a") + // then the first in the result block when absent. + LinkPrimary string Link string Title string Desc string + DescFallback string ImageItems string ImageItemsAlt []string ImageStateAll string @@ -15,9 +19,11 @@ var Selectors = struct { Captcha: "div.CheckboxCaptcha", NoResults: "div.EmptySearchResults", Results: "li[data-fast], li.serp-item", + LinkPrimary: "a.OrganicTitle-Link", Link: "a", Title: "h2", Desc: "span.OrganicTextContentSpan", + DescFallback: "div.OrganicText", ImageItems: "div[role='main'] div[data-state]", ImageItemsAlt: []string{"div[data-state*='serpList']"}, ImageStateAll: "div[data-state]",