diff --git a/baidu/search.go b/baidu/search.go
index 47b2e9b..bf0f2c2 100644
--- a/baidu/search.go
+++ b/baidu/search.go
@@ -33,7 +33,6 @@ type imageDataJson struct {
}
}
-
// Baidu implements core.SearchEngine for Baidu SERP pages.
type Baidu struct {
core.Browser
@@ -123,7 +122,12 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
}
}
- searchRes, err := page.Timeout(baid.Timeout).Search(Selectors.Results)
+ resultElements, _, err := core.WaitForElements(
+ ctx,
+ page,
+ append([]string{Selectors.Results}, Selectors.ResultsAlt...),
+ baid.GetSelectorTimeout(),
+ )
if err != nil {
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
closePage()
@@ -131,11 +135,10 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
}
closePage()
baid.logger.Error("Cannot parse search results: %s", err)
- return nil, core.ErrParser
+ return nil, core.ErrSearchTimeout
}
- // Check why no results, maybe captcha?
- if searchRes == nil {
+ if len(resultElements) == 0 {
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
closePage()
return nil, blockErr
@@ -144,16 +147,12 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
return nil, nil
}
- resultElements, err := searchRes.All()
- if err != nil {
- closePage()
- return nil, err
- }
-
for i, r := range resultElements {
// Get URL
link, err := r.Element(Selectors.Link)
if err != nil {
+ // Element detached mid-iteration (page navigated/refreshed): the rest
+ // of the slice is also stale, no point continuing.
if core.IsRodObjectNotFound(err) {
break
}
@@ -210,7 +209,8 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
return nil, err
}
- // Get anti-crawler cookies first, then reload page
+ // First load often seeds anti-crawler cookies; results tend to become
+ // available after one explicit reload.
page, err := baid.Navigate(ctx, url)
if err != nil {
return nil, err
@@ -223,18 +223,14 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
baid.logger.Debug("Page close error: %v", closeErr)
}
}
- if err := page.Reload(); err != nil {
+
+ jsonWaitTimeout := baid.GetSelectorTimeout()
+ if reloadErr := page.Reload(); reloadErr != nil {
closePage()
- baid.logger.Error("Page reload failed: %s", err)
- return nil, core.ErrSearchTimeout
- }
- if err := page.WaitLoad(); err != nil {
- closePage()
- baid.logger.Error("Page load wait failed: %s", err)
return nil, core.ErrSearchTimeout
}
- result, err := page.Timeout(baid.Timeout).Search("body > pre")
+ preElements, _, err := core.WaitForElements(ctx, page, Selectors.ImageJSONRoot, jsonWaitTimeout)
if err != nil {
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
closePage()
@@ -242,11 +238,10 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
}
closePage()
baid.logger.Error("Cannot parse search results: %s", err)
- return nil, core.ErrParser
+ return nil, core.ErrSearchTimeout
}
- // Check why no results, maybe captcha?
- if result == nil {
+ if len(preElements) == 0 {
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
closePage()
return nil, blockErr
@@ -255,7 +250,7 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
return nil, nil
}
- jsonText, err := result.First.Text()
+ jsonText, err := preElements[0].Text()
if err != nil {
closePage()
return nil, err
@@ -286,6 +281,10 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
baid.logger.Error("Failed to unmarshal JSON: %v", err)
return nil, core.ErrParser
}
+ if len(data.Data) == 0 {
+ closePage()
+ break
+ }
for i, img := range data.Data {
if len(img.URL) == 0 {
@@ -314,6 +313,9 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
}(),
}
searchResults = append(searchResults, res)
+ if query.Limit > 0 && len(searchResults) >= query.Limit {
+ break
+ }
}
searchPage += 1
@@ -321,5 +323,9 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
closePage()
}
- return core.DeduplicateResults(searchResults), nil
+ deduped := core.DeduplicateResults(searchResults)
+ if query.Limit > 0 && len(deduped) > query.Limit {
+ deduped = deduped[:query.Limit]
+ }
+ return deduped, nil
}
diff --git a/baidu/selectors.go b/baidu/selectors.go
index 578d7d8..4fc1405 100644
--- a/baidu/selectors.go
+++ b/baidu/selectors.go
@@ -2,15 +2,19 @@ package baidu
// Selectors is the single source of truth for Baidu SERP CSS selectors.
var Selectors = struct {
- Captcha string
- Timeout string
- Results string
- Link string
- Desc string
+ Captcha string
+ Timeout string
+ Results string
+ ResultsAlt []string
+ ImageJSONRoot []string
+ Link string
+ Desc string
}{
- Captcha: "div.passMod_dialog-wrapper",
- Timeout: "button.timeout-button",
- Results: "div.c-container.new-pmd",
- Link: "a",
- Desc: "div.c-abstract",
+ Captcha: "div.passMod_dialog-wrapper",
+ Timeout: "button.timeout-button",
+ Results: "div.c-container.new-pmd",
+ ResultsAlt: []string{"#content_left div.result.c-container", "#content_left div.result-op.c-container"},
+ ImageJSONRoot: []string{"body > pre", "pre"},
+ Link: "a",
+ Desc: "div.c-abstract",
}
diff --git a/bing/parse_html.go b/bing/parse_html.go
index 5f56308..536e92b 100644
--- a/bing/parse_html.go
+++ b/bing/parse_html.go
@@ -64,7 +64,7 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
title := titleTag.Text()
desc := ""
- if descTag := item.Find(Selectors.AdDesc).First(); descTag.Length() > 0 {
+ if descTag := item.Find("p").First(); descTag.Length() > 0 {
desc = descTag.Text()
}
@@ -93,7 +93,7 @@ func descriptionFromItem(item *goquery.Selection, title string) string {
return text
}
}
- if descTag := item.Find(Selectors.DescLast).First(); descTag.Length() > 0 {
+ if descTag := item.Find("p").First(); descTag.Length() > 0 {
if text := strings.TrimSpace(descTag.Text()); text != "" {
return text
}
diff --git a/bing/search.go b/bing/search.go
index 244b4ab..297852b 100644
--- a/bing/search.go
+++ b/bing/search.go
@@ -122,11 +122,6 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
}
}()
- if err := page.WaitLoad(); err != nil {
- bing.logger.Error("Initial page load wait failed: %s", err)
- return nil, core.ErrSearchTimeout
- }
-
if bing.checkCaptcha(page) {
bing.logger.Error("Captcha detected: %s", url)
return nil, core.ErrCaptcha
@@ -135,18 +130,19 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
if err := bing.acceptCookies(ctx, page); err != nil {
return nil, err
}
- if err := page.WaitLoad(); err != nil {
- bing.logger.Error("Post-consent page load wait failed: %s", err)
+
+ organicElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, bing.GetSelectorTimeout())
+ if err != nil {
+ // Re-check captcha on timeout - Bing interstitials can render after WaitLoad.
+ if bing.checkCaptcha(page) {
+ bing.logger.Error("Captcha detected: %s", url)
+ return nil, core.ErrCaptcha
+ }
+ bing.logger.Error("Cannot parse organic results: %s", err)
return nil, core.ErrSearchTimeout
}
- organicElements, err := page.Timeout(bing.Timeout).Elements(Selectors.Results)
- if err != nil {
- bing.logger.Error("Cannot parse organic results: %s", err)
- return nil, core.ErrParser
- }
-
- adElements, err := page.Timeout(bing.Timeout).Elements(Selectors.Ads)
+ adElements, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Ads)
if err != nil {
bing.logger.Debug("No ads found")
}
@@ -166,7 +162,6 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
bing.logger.Debug("Missing title")
continue
}
- srchRes.Title, _ = titleElem.Text()
href, err := titleElem.Property("href")
if err != nil {
@@ -174,13 +169,26 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
continue
}
srchRes.URL = href.String()
+ if strings.TrimSpace(srchRes.URL) == "" || strings.HasPrefix(srchRes.URL, "javascript:") {
+ continue
+ }
+
+ srchRes.Title, _ = titleElem.Text()
+ srchRes.Title = strings.TrimSpace(srchRes.Title)
+ if srchRes.Title == "" {
+ srchRes.Title = core.FirstNonEmptyText(result, Selectors.TitleFallbacks...)
+ }
+ if srchRes.Title == "" {
+ bing.logger.Debug("Missing title text")
+ continue
+ }
var desc string
if descElem, err := result.Element(Selectors.DescPrimary); err == nil {
desc, _ = descElem.Text()
} else if descElem, err := result.Element(Selectors.DescFallback); err == nil {
desc, _ = descElem.Text()
- } else if descElem, err := result.Element(Selectors.DescLast); err == nil {
+ } else if descElem, err := result.Element("p"); err == nil {
desc, _ = descElem.Text()
} else {
fullText, _ := result.Text()
@@ -204,6 +212,13 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
continue
}
srchRes.Title, _ = titleElem.Text()
+ srchRes.Title = strings.TrimSpace(srchRes.Title)
+ if srchRes.Title == "" {
+ srchRes.Title = core.FirstNonEmptyText(adResult, "h2", "a[aria-label]")
+ }
+ if srchRes.Title == "" {
+ continue
+ }
href, err := titleElem.Property("href")
if err != nil {
@@ -212,7 +227,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
}
srchRes.URL = href.String()
- if descElem, err := adResult.Element(Selectors.AdDesc); err == nil {
+ if descElem, err := adResult.Element("p"); err == nil {
srchRes.Description, _ = descElem.Text()
}
@@ -261,6 +276,16 @@ type BingImageData struct {
MURL string `json:"murl"` // Image URL
}
+func resolveImageLinkElement(container *rod.Element) (*rod.Element, error) {
+ if container == nil {
+ return nil, errors.New("nil image container")
+ }
+ if core.HasAttribute(container, "m") {
+ return container, nil
+ }
+ return container.Element("a")
+}
+
// SearchImage executes a Bing image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
@@ -294,11 +319,6 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
}
}()
- if err := page.WaitLoad(); err != nil {
- bing.logger.Error("Initial image page load wait failed: %s", err)
- return nil, core.ErrSearchTimeout
- }
-
// Check for captcha
if bing.checkCaptcha(page) {
bing.logger.Error("Captcha detected during image search: %s", url)
@@ -310,18 +330,16 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
return nil, err
}
- // Wait for image results to load
- if err := page.WaitLoad(); err != nil {
- bing.logger.Error("Image results load wait failed: %s", err)
- return nil, core.ErrSearchTimeout
- }
- if err := core.SleepContext(ctx, 2*time.Second); err != nil {
- return nil, err
- }
-
- // Find all image result containers using CSS selector
- imageContainers, err := page.Timeout(bing.Timeout).Elements(Selectors.ImageResults)
+ imageContainers, _, err := core.WaitForElements(
+ ctx,
+ page,
+ []string{Selectors.ImageResults},
+ bing.GetSelectorTimeout(),
+ )
if err != nil {
+ if bing.checkCaptcha(page) {
+ return nil, core.ErrCaptcha
+ }
bing.logger.Error("Cannot parse image results: %s", err)
return nil, core.ErrSearchTimeout
}
@@ -336,10 +354,9 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
for _, c := range imageContainers {
srchRes := core.SearchResult{}
- // Get the element inside the div
- linkElem, err := c.Element("a")
+ linkElem, err := resolveImageLinkElement(c)
if err != nil {
- bing.logger.Debug("Missing element")
+ bing.logger.Debug("Missing image link element")
continue
}
@@ -386,6 +403,9 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
srchRes.Rank = rank
searchResults = append(searchResults, srchRes)
+ if query.Limit > 0 && len(searchResults) >= query.Limit {
+ break
+ }
}
return searchResults, nil
diff --git a/bing/selectors.go b/bing/selectors.go
index 4d9f477..8c88862 100644
--- a/bing/selectors.go
+++ b/bing/selectors.go
@@ -2,27 +2,28 @@ package bing
// Selectors is the single source of truth for Bing SERP CSS selectors.
var Selectors = struct {
- Captcha []string
- CookieBtn string
- Results string
- Ads string
- ImageResults string
- Title string
- DescPrimary string
- DescFallback string
- DescLast string
- AdTitle string
- AdDesc string
+ Captcha []string
+ CookieBtn string
+ Results string
+ Ads string
+ ImageResults string
+ Title string
+ TitleFallbacks []string
+ DescPrimary string
+ DescFallback string
+ AdTitle string
}{
Captcha: []string{"div.captcha", "div.captcha_header"},
CookieBtn: "button#bnp_btn_accept",
Results: "li.b_algo",
Ads: "li.b_ad",
- ImageResults: "div.iuscp, div.isv",
- Title: "a",
- DescPrimary: "div.b_caption p",
- DescFallback: "div.b_caption div",
- DescLast: "p",
- AdTitle: "h2 a",
- AdDesc: "p",
+ ImageResults: "a.iusc, div.iuscp, div.isv",
+ Title: "h2 a",
+ // TitleFallbacks are tried by FirstNonEmptyText when the primary Title
+ // selector matches but yields empty text (Bing occasionally renders an
+ // empty
while the visible label sits in aria-label or h2).
+ TitleFallbacks: []string{"h2", "a[aria-label]"},
+ DescPrimary: "div.b_caption p",
+ DescFallback: "div.b_caption div",
+ AdTitle: "h2 a",
}
diff --git a/cmd/root.go b/cmd/root.go
index 05674f3..e4241d8 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -16,7 +16,7 @@ import (
)
const (
- version = "0.7.8"
+ version = "0.7.9"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)
diff --git a/config.yaml b/config.yaml
index 6a8906b..a4ad9fd 100644
--- a/config.yaml
+++ b/config.yaml
@@ -41,7 +41,7 @@ cache:
max_size: 1000 # Maximum cached dedicated responses before oldest-entry eviction
resilience:
- max_retries: 2 # Retry attempts per engine request (0 disables retries)
+ max_retries: 1 # Retry attempts per engine request (0 disables retries)
allow_endpoint_fallback: false # Keep dedicated endpoints engine-pure by default
# circuit_breaker:
diff --git a/core/page_helpers.go b/core/page_helpers.go
new file mode 100644
index 0000000..3cfe6b1
--- /dev/null
+++ b/core/page_helpers.go
@@ -0,0 +1,133 @@
+package core
+
+import (
+ "context"
+ "strings"
+ "time"
+
+ "github.com/go-rod/rod"
+)
+
+// pollInterval is how often WaitForElements re-probes selectors while the
+// page hydrates. Short enough to feel snappy, long enough not to hammer CDP.
+const pollInterval = 120 * time.Millisecond
+
+// WaitForElements probes the supplied CSS selectors until one returns at least
+// one matching element or timeout elapses. It exists because rod's
+// page.Search/Elements and the surrounding WaitLoad/WaitStable do not wait for
+// a *specific* selector to hydrate — modern SPA SERPs (DDG, Bing, Google)
+// regularly fire `load` and even reach DOM-stable before result rows render,
+// causing parsers to see an empty page on the first probe and forcing the
+// caller's retry layer to reload.
+//
+// The probe loop returns as soon as a selector matches, returning the matched
+// elements and the selector that hit. On timeout it returns ErrSearchTimeout
+// so callers can disambiguate between "no results" / "captcha" by inspecting
+// the page directly.
+func WaitForElements(ctx context.Context, page *rod.Page, selectors []string, timeout time.Duration) (rod.Elements, string, error) {
+ if page == nil {
+ return nil, "", ErrSearchTimeout
+ }
+ ctx = EnsureContext(ctx)
+ if timeout <= 0 {
+ timeout = 2 * time.Second
+ }
+
+ probe := func() (rod.Elements, string) {
+ for _, selector := range selectors {
+ elements, err := page.Elements(selector)
+ if err != nil || len(elements) == 0 {
+ continue
+ }
+ return elements, selector
+ }
+ return nil, ""
+ }
+
+ if elements, selector := probe(); len(elements) > 0 {
+ return elements, selector, nil
+ }
+
+ deadline := time.Now().Add(timeout)
+ for time.Now().Before(deadline) {
+ if err := ctx.Err(); err != nil {
+ return nil, "", err
+ }
+ if elements, selector := probe(); len(elements) > 0 {
+ return elements, selector, nil
+ }
+ if err := SleepContext(ctx, pollInterval); err != nil {
+ return nil, "", err
+ }
+ }
+ return nil, "", ErrSearchTimeout
+}
+
+// HasAnySelector returns true if at least one of the supplied selectors
+// currently matches in the page DOM. It does not wait — pair with
+// WaitForElements when hydration may be in flight.
+func HasAnySelector(page *rod.Page, selectors []string) bool {
+ if page == nil {
+ return false
+ }
+ for _, selector := range selectors {
+ has, _, err := page.Has(selector)
+ if err == nil && has {
+ return true
+ }
+ }
+ return false
+}
+
+// HasAttribute reports whether el carries attr (regardless of value).
+func HasAttribute(el *rod.Element, attr string) bool {
+ if el == nil {
+ return false
+ }
+ v, err := el.Attribute(attr)
+ return err == nil && v != nil
+}
+
+// FirstNonEmptyText returns the trimmed text of the first selector under root
+// that yields non-empty content. Empty string if none match.
+func FirstNonEmptyText(root *rod.Element, selectors ...string) string {
+ if root == nil {
+ return ""
+ }
+ for _, selector := range selectors {
+ el, err := root.Element(selector)
+ if err != nil {
+ continue
+ }
+ text, err := el.Text()
+ if err != nil {
+ continue
+ }
+ if trimmed := strings.TrimSpace(text); trimmed != "" {
+ return trimmed
+ }
+ }
+ return ""
+}
+
+// FirstNonEmptyAttribute returns the trimmed value of attr from the first
+// selector under root whose attribute is non-empty.
+func FirstNonEmptyAttribute(root *rod.Element, attr string, selectors ...string) string {
+ if root == nil {
+ return ""
+ }
+ for _, selector := range selectors {
+ el, err := root.Element(selector)
+ if err != nil {
+ continue
+ }
+ value, err := el.Attribute(attr)
+ if err != nil || value == nil {
+ continue
+ }
+ if trimmed := strings.TrimSpace(*value); trimmed != "" {
+ return trimmed
+ }
+ }
+ return ""
+}
diff --git a/core/server.go b/core/server.go
index 68d667e..f1f67e9 100644
--- a/core/server.go
+++ b/core/server.go
@@ -189,10 +189,16 @@ func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngin
continue
}
locParser := parser
- serv.app.Post(fmt.Sprintf("/%s/parse", strings.ToLower(parser.Name())),
+ parserEndpointName := strings.ToLower(parser.Name())
+ serv.app.Post(fmt.Sprintf("/%s/parse", parserEndpointName),
func(c *fiber.Ctx) error {
return serv.handleParseEndpoint(c, locParser)
})
+ if parserEndpointName == "duckduckgo" {
+ serv.app.Post("/duck/parse", func(c *fiber.Ctx) error {
+ return serv.handleParseEndpoint(c, locParser)
+ })
+ }
}
serv.app.Get("/mega/search", serv.handleMegaSearch)
diff --git a/core/server_parse_test.go b/core/server_parse_test.go
index c5b83d1..ec05034 100644
--- a/core/server_parse_test.go
+++ b/core/server_parse_test.go
@@ -116,3 +116,13 @@ func TestParseEndpointMarkdownFormat(t *testing.T) {
t.Fatalf("expected markdown content type, got %s", ct)
}
}
+
+func TestParseEndpointDuckAlias(t *testing.T) {
+ engine := &parserMock{engineMock: engineMock{name: "duckduckgo", initialized: true}}
+ srv := NewServerWithOptions("127.0.0.1", 7125, DefaultServerOptions(), engine)
+
+ resp := postHTML(t, srv, "/duck/parse", "sample serp")
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("expected 200, got %d", resp.StatusCode)
+ }
+}
diff --git a/duckduckgo/search.go b/duckduckgo/search.go
index 1f6ad03..d90ce3a 100644
--- a/duckduckgo/search.go
+++ b/duckduckgo/search.go
@@ -11,10 +11,6 @@ import (
"golang.org/x/time/rate"
)
-// captchaBodyText is matched against the raw page HTML because DDG returns
-// a plain-text 202 rate-limit response rather than a structured captcha page.
-const captchaBodyText = "bots user"
-
// DuckDuckGo implements core.SearchEngine for DuckDuckGo SERP pages.
type DuckDuckGo struct {
core.Browser
@@ -46,16 +42,36 @@ func (ddg *DuckDuckGo) GetRateLimiter() *rate.Limiter {
}
func (ddg *DuckDuckGo) isCaptcha(page *rod.Page) bool {
- html, err := page.Timeout(ddg.GetSelectorTimeout()).HTML()
+ if core.HasAnySelector(page, Selectors.CaptchaSelectors) {
+ return true
+ }
+
+ if info, err := page.Info(); err == nil {
+ url := strings.ToLower(info.URL)
+ if strings.Contains(url, "anomaly") || strings.Contains(url, "captcha") || strings.Contains(url, "challenge") {
+ return true
+ }
+ }
+
+ htmlTimeout := ddg.GetSelectorTimeout() / 2
+ if htmlTimeout <= 0 || htmlTimeout > 1500*time.Millisecond {
+ htmlTimeout = 1500 * time.Millisecond
+ }
+ html, err := page.Timeout(htmlTimeout).HTML()
if err != nil {
return false
}
- return strings.Contains(html, captchaBodyText)
+ html = strings.ToLower(html)
+ for _, marker := range Selectors.CaptchaMarkers {
+ if strings.Contains(html, marker) {
+ return true
+ }
+ }
+ return false
}
func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool {
- has, _, _ := page.Has(Selectors.NoResults)
- return has
+ return core.HasAnySelector(page, Selectors.NoResults)
}
func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
@@ -75,6 +91,8 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
if err != nil {
ddg.logger.Debug("Missing link")
+ // If the result element itself was detached (page navigated mid-iteration),
+ // the rest of the slice is also stale and further work is wasted.
if core.IsRodObjectNotFound(err) {
break
}
@@ -94,29 +112,12 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
continue
}
- // Get title - try multiple selectors
- var titleTag *rod.Element
- for _, selector := range Selectors.Title {
- titleTag, err = r.Element(selector)
- if err == nil {
- break
- }
+ title := core.FirstNonEmptyText(r, Selectors.Title...)
+ if title == "" {
+ title = "No title"
}
- title := "No title"
- if titleTag != nil {
- title, _ = titleTag.Text()
- }
-
- // Get description - try multiple selectors
- desc := ""
- for _, selector := range Selectors.Desc {
- descTag, err := r.Element(selector)
- if err == nil {
- desc, _ = descTag.Text()
- break
- }
- }
+ desc := core.FirstNonEmptyText(r, Selectors.Desc...)
// Check if it's an ad
isAd := false
@@ -181,50 +182,30 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
}
}
- // Get all search results in page - try multiple selectors
- var searchRes *rod.SearchResult
- var searchErr error
-
- for _, selector := range Selectors.Results {
- searchRes, searchErr = page.Timeout(ddg.GetSelectorTimeout()).Search(selector)
- if searchErr == nil && searchRes != nil {
- ddg.logger.Debug("Found results with selector: %s", selector)
- break
- }
- }
-
- if searchErr != nil {
- closePage()
- ddg.logger.Error("Cannot parse search results: %s", searchErr)
- return nil, core.ErrParser
- }
-
- // Check why no results, maybe captcha?
- if searchRes == nil {
- closePage()
-
+ elements, selector, err := core.WaitForElements(ctx, page, Selectors.Results, ddg.GetSelectorTimeout())
+ if err != nil {
if ddg.isNoResults(page) {
ddg.logger.Warn("No results found")
- } else if ddg.isCaptcha(page) {
+ closePage()
+ break
+ }
+ if ddg.isCaptcha(page) {
ddg.logger.Error("Captcha detected: %s", url)
+ closePage()
return nil, core.ErrCaptcha
}
- break
- }
-
- elements, err := searchRes.All()
- if err != nil {
- ddg.logger.Error("Cannot get search elements: %s", err)
closePage()
- break
+ ddg.logger.Error("Cannot parse search results: %s", err)
+ return nil, core.ErrSearchTimeout
}
+ ddg.logger.Debug("Found results with selector: %s", selector)
r := ddg.parseResults(elements, searchPage)
if len(r) == 0 {
ddg.logger.Debug("No valid results found on page %d", searchPage)
closePage()
- return nil, core.ErrParser
+ return nil, core.ErrSearchTimeout
}
allResults = append(allResults, r...)
@@ -287,36 +268,8 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor
}()
}
- // Wait for page to load
- if err := page.WaitLoad(); err != nil {
- ddg.logger.Error("Wait load failed: %s", err)
- return searchResults, core.ErrSearchTimeout
- }
- if err := core.SleepContext(ctx, 2*time.Second); err != nil {
- return searchResults, err
- }
-
- // Try multiple selectors for DuckDuckGo image results
- var searchRes *rod.SearchResult
- var searchErr error
-
- for _, selector := range Selectors.ImageResult {
- searchRes, searchErr = page.Timeout(ddg.GetSelectorTimeout()).Search(selector)
- if searchErr == nil && searchRes != nil {
- ddg.logger.Debug("Found image results with selector: %s", selector)
- break
- } else {
- ddg.logger.Debug("Selector '%s' not found: %v", selector, searchErr)
- }
- }
-
- if searchErr != nil {
- ddg.logger.Error("Cannot find image results: %s", searchErr)
- return searchResults, core.ErrSearchTimeout
- }
-
- // Check why no results
- if searchRes == nil {
+ elements, selector, err := core.WaitForElements(ctx, page, Selectors.ImageResult, ddg.GetSelectorTimeout())
+ if err != nil {
if ddg.isCaptcha(page) {
ddg.logger.Error("Captcha detected: %s", url)
return searchResults, core.ErrCaptcha
@@ -325,12 +278,7 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor
}
return searchResults, core.ErrSearchTimeout
}
-
- elements, err := searchRes.All()
- if err != nil {
- ddg.logger.Error("Cannot get search elements: %s", err)
- return searchResults, err
- }
+ ddg.logger.Debug("Found image results with selector: %s", selector)
ddg.logger.Info("Found %d image elements", len(elements))
@@ -357,20 +305,9 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor
continue
}
- // Get title - try multiple selectors based on the HTML structure
- var titleTag *rod.Element
- var titleErr error
-
- for _, selector := range Selectors.ImageTitle {
- titleTag, titleErr = r.Element(selector)
- if titleErr == nil {
- break
- }
- }
-
- title := "No title"
- if titleTag != nil {
- title, _ = titleTag.Text()
+ title := core.FirstNonEmptyText(r, Selectors.ImageTitle...)
+ if title == "" {
+ title = "No title"
}
// Get source page URL - try multiple selectors
diff --git a/duckduckgo/selectors.go b/duckduckgo/selectors.go
index c2a9987..1de5fbb 100644
--- a/duckduckgo/selectors.go
+++ b/duckduckgo/selectors.go
@@ -1,23 +1,52 @@
package duckduckgo
// Selectors is the single source of truth for DuckDuckGo SERP CSS selectors.
+//
+// Order matters: WaitForElements / parseResults try selectors in declared order
+// and stop on the first hit. Put the most specific / current variants first;
+// keep older fallbacks last to absorb DOM rewrites without losing coverage.
var Selectors = struct {
- NoResults string
- Results []string
- Title []string
- Desc []string
- Link []string
- AdBadge []string
- ImageResult []string
- ImageImg []string
- ImageTitle []string
- ImageLink []string
+ NoResults []string
+ CaptchaSelectors []string
+ CaptchaMarkers []string
+ Results []string
+ Title []string
+ Desc []string
+ Link []string
+ AdBadge []string
+ ImageResult []string
+ ImageImg []string
+ ImageTitle []string
+ ImageLink []string
}{
- NoResults: "div[class*='no-results']",
+ NoResults: []string{
+ "div[class*='no-results']",
+ "[data-testid='no-results']",
+ "div[data-result='no-results']",
+ },
+ // CaptchaSelectors detect DDG's anomaly/challenge interstitials via DOM.
+ CaptchaSelectors: []string{
+ "form[action*='anomaly']",
+ "input[name='challenge']",
+ "div[id*='anomaly']",
+ "div[class*='captcha']",
+ },
+ // CaptchaMarkers are case-insensitive substrings used for full-HTML scan
+ // fallback when the selectors above don't match. DDG sometimes returns a
+ // plain-text 202 rate-limit page rather than a structured form, so the
+ // HTML scan is the only reliable signal in those cases.
+ CaptchaMarkers: []string{
+ "bots user",
+ "bots use duckduckgo too",
+ "human verification",
+ "unusual traffic",
+ "anomaly",
+ },
Results: []string{
"article[data-testid='result']",
- "div.result",
+ "li[data-layout='organic']",
"div[data-testid='result']",
+ "div.result",
},
Title: []string{
"h2",
@@ -41,11 +70,11 @@ var Selectors = struct {
".result--ad",
},
ImageResult: []string{
+ "figure[data-testid='image-result']",
"figure",
},
ImageImg: []string{
"img",
- "img[src*='duckduckgo.com']",
},
ImageTitle: []string{
"figcaption a p span",
diff --git a/ecosia/parse_html.go b/ecosia/parse_html.go
index cf45678..3435d3b 100644
--- a/ecosia/parse_html.go
+++ b/ecosia/parse_html.go
@@ -1,7 +1,6 @@
package ecosia
import (
- "fmt"
"io"
"strings"
@@ -85,50 +84,50 @@ func parseEcosiaItem(item *goquery.Selection, rank int, ad bool) (core.SearchRes
}, true
}
-// parseEcosiaImageItem extracts a single image card from a goquery Selection.
-func parseEcosiaImageItem(item *goquery.Selection, rank int) (core.SearchResult, bool) {
- linkTag := item.Find(Selectors.ImageLink).First()
- if linkTag.Length() == 0 {
- return core.SearchResult{}, false
- }
- href, exists := linkTag.Attr("href")
- if !exists {
- return core.SearchResult{}, false
- }
- imgURL := strings.TrimSpace(href)
- if imgURL == "" {
- return core.SearchResult{}, false
- }
+// // parseEcosiaImageItem extracts a single image card from a goquery Selection.
+// func parseEcosiaImageItem(item *goquery.Selection, rank int) (core.SearchResult, bool) {
+// linkTag := item.Find(Selectors.ImageLink).First()
+// if linkTag.Length() == 0 {
+// return core.SearchResult{}, false
+// }
+// href, exists := linkTag.Attr("href")
+// if !exists {
+// return core.SearchResult{}, false
+// }
+// imgURL := strings.TrimSpace(href)
+// if imgURL == "" {
+// return core.SearchResult{}, false
+// }
- title := ""
- if img := linkTag.Find("img").First(); img.Length() > 0 {
- if alt, err := img.Attr("alt"); err {
- title = strings.TrimSpace(alt)
- }
- }
+// title := ""
+// if img := linkTag.Find("img").First(); img.Length() > 0 {
+// if alt, err := img.Attr("alt"); err {
+// title = strings.TrimSpace(alt)
+// }
+// }
- source := ""
- if s := item.Find(Selectors.ImageSource).First(); s.Length() > 0 {
- source = strings.TrimSpace(s.Text())
- }
- dims := ""
- if d := item.Find(Selectors.ImageDims).First(); d.Length() > 0 {
- dims = strings.TrimSpace(d.Text())
- }
+// source := ""
+// if s := item.Find(Selectors.ImageSource).First(); s.Length() > 0 {
+// source = strings.TrimSpace(s.Text())
+// }
+// dims := ""
+// if d := item.Find(Selectors.ImageDims).First(); d.Length() > 0 {
+// dims = strings.TrimSpace(d.Text())
+// }
- desc := source
- if dims != "" {
- if source != "" {
- desc = fmt.Sprintf("%s (%s)", source, dims)
- } else {
- desc = dims
- }
- }
+// desc := source
+// if dims != "" {
+// if source != "" {
+// desc = fmt.Sprintf("%s (%s)", source, dims)
+// } else {
+// desc = dims
+// }
+// }
- return core.SearchResult{
- Rank: rank,
- URL: imgURL,
- Title: title,
- Description: desc,
- }, true
-}
+// return core.SearchResult{
+// Rank: rank,
+// URL: imgURL,
+// Title: title,
+// Description: desc,
+// }, true
+// }
diff --git a/ecosia/search_integration_test.go b/ecosia/search_integration_test.go
new file mode 100644
index 0000000..65841cb
--- /dev/null
+++ b/ecosia/search_integration_test.go
@@ -0,0 +1,55 @@
+//go:build integration
+// +build integration
+
+package ecosia
+
+import (
+ "context"
+ "testing"
+
+ "github.com/karust/openserp/core"
+ "github.com/karust/openserp/testutil"
+ "github.com/karust/openserp/testutil/ithelper"
+)
+
+func TestSearchEcosia(t *testing.T) {
+ testutil.RequireIntegration(t)
+
+ browser := ithelper.CreateBrowser(t)
+ engine := New(*browser, core.SearchEngineOptions{})
+
+ query := core.Query{Text: "golang programming", Limit: 10}
+ results, err := engine.Search(context.Background(), query)
+ ithelper.HandleError(t, "ecosia web search", err)
+
+ if len(results) == 0 {
+ t.Fatal("returned empty results")
+ }
+ if results[0].URL == "" {
+ t.Fatal("first result URL is empty")
+ }
+ if results[0].Title == "" {
+ t.Fatal("first result title is empty")
+ }
+}
+
+func TestImageSearchEcosia(t *testing.T) {
+ testutil.RequireIntegration(t)
+
+ browser := ithelper.CreateBrowser(t)
+ engine := New(*browser, core.SearchEngineOptions{})
+
+ query := core.Query{Text: "golden retriever puppy", Limit: 10}
+ results, err := engine.SearchImage(context.Background(), query)
+ ithelper.HandleError(t, "ecosia image search", err)
+
+ if len(results) == 0 {
+ t.Fatal("returned empty image results")
+ }
+ if results[0].URL == "" {
+ t.Fatal("first image result URL is empty")
+ }
+ if results[0].Title == "" {
+ t.Fatal("first image result title is empty")
+ }
+}
diff --git a/google/search.go b/google/search.go
index a279a33..7c24b8f 100644
--- a/google/search.go
+++ b/google/search.go
@@ -220,15 +220,23 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
gogl.acceptCookies(page)
}
- // Find all results using stable attributes
- searchRes, err := page.Timeout(gogl.Timeout).Search(Selectors.Results)
+ // Wait for result containers (data-hveid + data-ved) to hydrate. WaitLoad in
+ // Navigate fires before Google's right-rail/answers script attaches these
+ // attributes, so a one-shot Search races the DOM and frequently sees nothing.
+ searchResultElems, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, gogl.GetSelectorTimeout())
if err != nil {
- gogl.logger.Error("Cannot parse search results: %s", err)
- return nil, core.ErrParser
- }
-
- if searchRes == nil {
- return nil, nil
+ if gogl.checkCaptcha(page, query.ProxyURL) {
+ gogl.logger.Error("Captcha detected: %s", url)
+ return nil, core.ErrCaptcha
+ }
+ if core.IsContextDone(err) {
+ return nil, err
+ }
+ // Keep empty-SERP behavior for selector timeout only.
+ if errors.Is(err, core.ErrSearchTimeout) {
+ return nil, nil
+ }
+ return nil, core.ErrSearchTimeout
}
totalResults, err := gogl.getTotalResults(page)
@@ -237,26 +245,15 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
}
gogl.logger.Info("Found %d total results", totalResults)
- searchResultElems, err := searchRes.All()
- if err != nil {
- return nil, err
- }
-
rank := query.Start
for _, resEl := range searchResultElems {
srchRes := core.SearchResult{}
- describe, err := resEl.Describe(1, false)
- if err != nil {
- gogl.logger.Debug("Result describe failed: %s", err)
- if core.IsRodObjectNotFound(err) {
- break
- }
- continue
- }
- attrs := strings.Join(describe.Attributes, " ")
+ isAd := core.HasAttribute(resEl, "data-text-ad")
+ isAnswerBox := query.Answers && core.HasAttribute(resEl, "data-ulkwtsb") && !core.HasAttribute(resEl, "data-ispaa")
+ isResultCandidate := core.HasAttribute(resEl, "data-ved")
- if strings.Contains(attrs, "data-text-ad") {
+ if isAd {
// 1. Parse ads
srchRes.Ad = true
@@ -293,10 +290,14 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
continue
}
textSliced := strings.Split(text, "\n")
- srchRes.Description = strings.Join(textSliced[4:], "\n")
+ if len(textSliced) > 4 {
+ srchRes.Description = strings.Join(textSliced[4:], "\n")
+ } else {
+ srchRes.Description = strings.TrimSpace(text)
+ }
rank += 1
- } else if query.Answers && strings.Contains(attrs, "data-ulkwtsb") && !strings.Contains(attrs, "data-ispaa") {
+ } else if isAnswerBox {
// 2. Parse answer boxes
answerEls, err := resEl.Page().Search(Selectors.AnswerBox)
if err != nil {
@@ -321,7 +322,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
if err := answ.Focus(); err != nil {
gogl.logger.Debug("Answer focus failed: %s", err)
}
- //answ.Page().WaitRepaint()
+
}
if err := core.SleepContext(ctx, 2*time.Second); err != nil {
return nil, err
@@ -361,7 +362,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
searchResults = append(searchResults, srchRes)
}
continue
- } else if strings.Contains(attrs, "data-ved") {
+ } else if isResultCandidate {
// Parse regular search results
// Get title from h3
titleTag, err := resEl.Element(Selectors.Title)
@@ -419,7 +420,6 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
continue
} else {
- //fmt.Println(i, attrs)
continue
}
@@ -427,7 +427,20 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
searchResults = append(searchResults, srchRes)
}
- return core.DeduplicateResults(searchResults), nil
+ deduped := core.DeduplicateResults(searchResults)
+ if len(deduped) == 0 {
+ if gogl.checkCaptcha(page, query.ProxyURL) {
+ return nil, core.ErrCaptcha
+ }
+ // Result candidates were found by Selectors.Results but none parsed
+ // into usable rows: treat as a genuine no-results SERP rather than a
+ // timeout, so callers don't retry pointlessly.
+ if len(searchResultElems) == 0 {
+ return nil, nil
+ }
+ return nil, core.ErrSearchTimeout
+ }
+ return deduped, nil
}
// SearchImage executes a Google image search and returns normalized image
@@ -456,111 +469,127 @@ func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.S
defer gogl.close(ctx, page)
- for len(searchResultsMap) < query.Limit {
- if err := page.WaitLoad(); err != nil {
- gogl.logger.Error("Image page load wait failed: %s", err)
- return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
+ // Hard cap on outer scroll/parse passes. Google's image grid is virtualized
+ // and infinite-scroll: WaitLoad after scroll never settles and individual
+ // cells can hang on right-click. Cap iterations as a last-resort guard.
+ const maxImagePasses = 20
+ stagnant := 0
+ for pass := 0; pass < maxImagePasses && len(searchResultsMap) < query.Limit; pass++ {
+ if err := ctx.Err(); err != nil {
+ return *core.ConvertSearchResultsMap(searchResultsMap), err
}
- if err := page.Mouse.Scroll(0, 1000000, 1); err != nil {
- gogl.logger.Error("Image page scroll failed: %s", err)
- return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
+ if _, err := page.Eval(`() => window.scrollBy(0, 1000000)`); err != nil {
+ gogl.logger.Debug("Image page scroll failed: %s", err)
}
- if err := page.WaitLoad(); err != nil {
- gogl.logger.Error("Image results load wait failed: %s", err)
- return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
+ if err := core.SleepContext(ctx, 600*time.Millisecond); err != nil {
+ return *core.ConvertSearchResultsMap(searchResultsMap), err
}
- results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
+ resultElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.ImageResults}, gogl.GetSelectorTimeout())
if err != nil {
- gogl.logger.Error("Cannot parse search results: %s", err)
- return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
- }
-
- // Check why no results
- if results == nil {
if gogl.checkCaptcha(page, query.ProxyURL) {
gogl.logger.Error("Captcha detected: %s", url)
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
}
- return *core.ConvertSearchResultsMap(searchResultsMap), err
+ break
}
- resultElements, err := results.All()
- if err != nil {
- return *core.ConvertSearchResultsMap(searchResultsMap), err
- }
-
- if len(resultElements) < len(searchResultsMap) {
- continue
- }
-
- for i, r := range resultElements {
- // TODO: parse AF_initDataCallback to optimize instead of this?
- err := r.Click(proto.InputMouseButtonRight, 1)
- if err != nil {
- gogl.logger.Error("Click failed")
- continue
+ before := len(searchResultsMap)
+ for _, r := range resultElements {
+ if err := ctx.Err(); err != nil {
+ return *core.ConvertSearchResultsMap(searchResultsMap), err
}
-
- dataVed, err := r.Attribute("data-ved")
- if err != nil {
- gogl.logger.Error("Missing data-ved attribute")
- continue
- }
-
- // If already have image with this ID
- if _, ok := searchResultsMap[*dataVed]; ok {
- continue
- }
-
- // Get URLs
- link, err := r.Element("a:not([ping])")
- if err != nil {
- continue
- }
-
- linkText, err := link.Property("href")
- if err != nil {
- gogl.logger.Debug("Missing href")
- continue
- }
-
- imgSrc, err := parseSourceImageURL(linkText.String())
- if err != nil {
- gogl.logger.Error("Failed to parse image URL: %v", err)
- continue
- }
-
- // Get title
- titleTag, err := r.Element("h3")
- if err != nil {
- gogl.logger.Error("Missing h3 tag")
- continue
- }
-
- title, err := titleTag.Text()
- if err != nil {
- gogl.logger.Error("Failed to extract title")
- title = "No title"
- }
-
- gR := core.SearchResult{
- Rank: i + 1,
- URL: imgSrc.OriginalURL,
- Title: title,
- Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
- }
- searchResultsMap[*dataVed] = gR
-
+ gogl.parseImageCell(r, searchResultsMap)
+ // Always remove the cell so the next outer iteration only sees
+ // freshly-scrolled elements. Without this we re-iterate the same
+ // already-parsed cells and the loop scales O(n²) — or worse,
+ // hangs entirely when right-click on a stale node never returns.
if err := r.Remove(); err != nil {
gogl.logger.Debug("Failed to remove parsed image element: %s", err)
}
+ if len(searchResultsMap) >= query.Limit {
+ break
+ }
+ }
+
+ if len(searchResultsMap) == before {
+ stagnant++
+ if stagnant >= 2 {
+ break
+ }
+ } else {
+ stagnant = 0
}
}
return *core.ConvertSearchResultsMap(searchResultsMap), nil
}
+// parseImageCell extracts one image result from a Google image grid cell and
+// stores it in dst keyed by the cell's data-ved. Returns silently on any
+// failure — the surrounding loop calls r.Remove() unconditionally so a
+// problem cell can't stall the outer scroll loop.
+func (gogl *Google) parseImageCell(r *rod.Element, dst map[string]core.SearchResult) {
+ // Right-clicking a Google image cell forces it to materialize its
+ // `imgres` link (the grid is virtualized; href is absent until the
+ // cell is interacted with).
+ if err := r.Click(proto.InputMouseButtonRight, 1); err != nil {
+ gogl.logger.Debug("Right-click on image cell failed: %s", err)
+ return
+ }
+
+ dataVed, err := r.Attribute("data-ved")
+ if err != nil || dataVed == nil || strings.TrimSpace(*dataVed) == "" {
+ gogl.logger.Debug("Missing data-ved attribute")
+ return
+ }
+ resultKey := *dataVed
+ if _, ok := dst[resultKey]; ok {
+ return
+ }
+
+ link, err := r.Element(Selectors.ImageLink)
+ if err != nil {
+ link, err = r.Element(Selectors.ImageLinkFallback)
+ if err != nil {
+ return
+ }
+ }
+
+ linkText, err := link.Property("href")
+ if err != nil {
+ gogl.logger.Debug("Missing href")
+ return
+ }
+
+ imgSrc, err := parseSourceImageURL(linkText.String())
+ if err != nil {
+ gogl.logger.Error("Failed to parse image URL: %v", err)
+ return
+ }
+ if strings.TrimSpace(imgSrc.OriginalURL) == "" {
+ return
+ }
+
+ title := core.FirstNonEmptyText(r, Selectors.ImageTitle...)
+ if title == "" {
+ title = core.FirstNonEmptyAttribute(r, "alt", "img")
+ }
+ if title == "" {
+ title = core.FirstNonEmptyAttribute(r, "aria-label", "a")
+ }
+ if title == "" {
+ title = "No title"
+ }
+
+ dst[resultKey] = core.SearchResult{
+ Rank: len(dst) + 1,
+ URL: imgSrc.OriginalURL,
+ Title: title,
+ Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
+ }
+}
+
func (gogl *Google) close(ctx context.Context, page *rod.Page) {
if !gogl.Browser.LeavePageOpen {
err := core.ClosePageWithTimeout(ctx, page, time.Second)
diff --git a/google/selectors.go b/google/selectors.go
index ad03d45..c5c4c06 100644
--- a/google/selectors.go
+++ b/google/selectors.go
@@ -13,6 +13,12 @@ var Selectors = struct {
DescFallback string
AnswerBox string
AnswerItem string
+
+ // Image search.
+ ImageResults string
+ ImageLink string
+ ImageLinkFallback string
+ ImageTitle []string
}{
Captcha: "div[data-sitekey]",
ResultStats: "div#result-stats",
@@ -23,4 +29,14 @@ var Selectors = struct {
DescFallback: "div.VwiC3b",
AnswerBox: "div[data-hveid][data-ulkwtsb] div[data-q]",
AnswerItem: "a",
+
+ // ImageResults selects each image cell in the image SERP grid.
+ ImageResults: "div[data-hveid][data-ved][jsaction]",
+ // ImageLink: the canonical href of an image cell. The :not([ping])
+ // variant excludes Google's click-tracking hops; the imgres fallback is
+ // only present after the cell has been right-clicked to materialize.
+ ImageLink: "a:not([ping])",
+ ImageLinkFallback: "a[href*='imgres']",
+ // ImageTitle selectors are tried in order to recover a human-readable title.
+ ImageTitle: []string{"h3", "a"},
}
diff --git a/testutil/ithelper/ithelper.go b/testutil/ithelper/ithelper.go
index 7877fd1..404f027 100644
--- a/testutil/ithelper/ithelper.go
+++ b/testutil/ithelper/ithelper.go
@@ -1,6 +1,8 @@
package ithelper
import (
+ "context"
+ "errors"
"testing"
"time"
@@ -30,6 +32,12 @@ func HandleError(t *testing.T, operation string, err error) {
}
t.Skipf("skipping flaky live %s due to timeout: %v", operation, err)
}
+ if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || core.IsContextDone(err) {
+ if testutil.IntegrationStrict() {
+ t.Fatalf("%s failed (strict mode): %v", operation, err)
+ }
+ t.Skipf("skipping flaky live %s due to context deadline: %v", operation, err)
+ }
t.Fatalf("%s failed: %v", operation, err)
}
@@ -42,7 +50,7 @@ func CreateBrowser(t *testing.T) *core.Browser {
opts := core.BrowserOpts{
IsHeadless: !headful,
IsLeakless: false,
- Timeout: time.Second * 15,
+ Timeout: time.Second * 30,
LeavePageOpen: headful,
}
b, err := core.NewBrowser(opts)
diff --git a/yandex/search.go b/yandex/search.go
index 9b72d96..7dbb94a 100644
--- a/yandex/search.go
+++ b/yandex/search.go
@@ -36,7 +36,6 @@ type ImageData struct {
} `json:"initialState"`
}
-
// Yandex implements core.SearchEngine for Yandex SERP pages.
type Yandex struct {
core.Browser
@@ -124,6 +123,26 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
return searchResults
}
+func (yand *Yandex) parseImageEntities(items rod.Elements) map[string]ImageEntity {
+ entities := make(map[string]ImageEntity)
+ for _, item := range items {
+ state, err := item.Attribute("data-state")
+ if err != nil || state == nil || *state == "" {
+ continue
+ }
+
+ var imgData ImageData
+ if err := json.Unmarshal([]byte(*state), &imgData); err != nil {
+ continue
+ }
+
+ for id, entity := range imgData.InitalState.SerpList.Items.Entities {
+ entities[id] = entity
+ }
+ }
+ return entities
+}
+
// Search executes a Yandex web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -172,32 +191,21 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
}
}
- // Get all search results in page
- searchRes, err := page.Timeout(yand.Timeout).Search(Selectors.Results)
+ elements, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, yand.GetSelectorTimeout())
if err != nil {
- closePage()
- yand.logger.Error("Cannot parse search results: %s", err)
- return nil, core.ErrParser
- }
-
- // Check why no results, maybe captcha?
- if searchRes == nil {
- closePage()
-
- if yand.isNoResults(page) {
- yand.logger.Warn("No results found")
- } else if yand.isCaptcha(page) {
+ if yand.isCaptcha(page) {
yand.logger.Error("Captcha detected: %s", url)
+ closePage()
return nil, core.ErrCaptcha
}
- break
- }
-
- elements, err := searchRes.All()
- if err != nil {
- yand.logger.Error("Cannot get search elements: %s", err)
+ if yand.isNoResults(page) {
+ yand.logger.Warn("No results found")
+ closePage()
+ break
+ }
closePage()
- break
+ yand.logger.Error("Cannot parse search results: %s", err)
+ return nil, core.ErrSearchTimeout
}
r := yand.parseResults(elements, searchPage)
@@ -237,6 +245,7 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S
yand.logger.Debug("Starting image search, query: %+v", query)
searchResults := []core.SearchResult{}
+ allowPagination := query.Limit > 30
searchPage := 0
for len(searchResults) < query.Limit {
@@ -259,43 +268,45 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S
}
}
- //page.Keyboard.Press(input.End)
- //page.WaitLoad()
- //time.Sleep(time.Duration(time.Second * 2))
-
- results, err := page.Timeout(yand.Timeout).Search(Selectors.ImageItems)
+ results, _, err := core.WaitForElements(
+ ctx,
+ page,
+ append([]string{Selectors.ImageItems}, Selectors.ImageItemsAlt...),
+ yand.GetSelectorTimeout(),
+ )
if err != nil {
- closePage()
- yand.logger.Error("Cannot find search results: %s", err)
- return searchResults, core.ErrParser
+ if allStateNodes, allErr := page.Elements(Selectors.ImageStateAll); allErr == nil && len(allStateNodes) > 0 {
+ results = allStateNodes
+ err = nil
+ }
}
-
- // Check why no results
- if results == nil {
- closePage()
+ if err != nil {
if yand.isCaptcha(page) {
yand.logger.Error("Captcha detected: %s", url)
+ closePage()
return searchResults, core.ErrCaptcha
- } else if yand.isNoResults(page) {
+ }
+ if yand.isNoResults(page) {
yand.logger.Warn("No results found")
}
+ closePage()
return searchResults, core.ErrSearchTimeout
}
- data, err := results.First.Attribute("data-state")
- if err != nil {
+ pageEntities := yand.parseImageEntities(results)
+ if len(pageEntities) == 0 {
+ allStateNodes, allErr := page.Elements(Selectors.ImageStateAll)
+ if allErr == nil && len(allStateNodes) > 0 {
+ pageEntities = yand.parseImageEntities(allStateNodes)
+ }
+ }
+ if len(pageEntities) == 0 {
closePage()
- return nil, err
+ break
}
- var imgData ImageData
- if err := json.Unmarshal([]byte(*data), &imgData); err != nil {
- closePage()
- return nil, err
- }
-
- for id := range imgData.InitalState.SerpList.Items.Entities {
- img := imgData.InitalState.SerpList.Items.Entities[id]
+ for id := range pageEntities {
+ img := pageEntities[id]
res := core.SearchResult{
Rank: img.Rank + 1,
URL: img.OrigURL,
@@ -305,6 +316,14 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S
searchResults = append(searchResults, res)
}
+ if len(searchResults) >= query.Limit {
+ closePage()
+ break
+ }
+ if searchPage == 1 && !allowPagination {
+ closePage()
+ break
+ }
closePage()
}
diff --git a/yandex/selectors.go b/yandex/selectors.go
index 0f011a5..9cb2f60 100644
--- a/yandex/selectors.go
+++ b/yandex/selectors.go
@@ -2,19 +2,23 @@ package yandex
// Selectors is the single source of truth for Yandex SERP CSS selectors.
var Selectors = struct {
- Captcha string
- NoResults string
- Results string
- Link string
- Title string
- Desc string
- ImageItems string
+ Captcha string
+ NoResults string
+ Results string
+ Link string
+ Title string
+ Desc string
+ ImageItems string
+ ImageItemsAlt []string
+ ImageStateAll string
}{
- Captcha: "div.CheckboxCaptcha",
- NoResults: "div.EmptySearchResults",
- Results: "li[data-fast], li.serp-item",
- Link: "a",
- Title: "h2",
- Desc: "span.OrganicTextContentSpan",
- ImageItems: "div[role='main'] div[data-state]",
+ Captcha: "div.CheckboxCaptcha",
+ NoResults: "div.EmptySearchResults",
+ Results: "li[data-fast], li.serp-item",
+ Link: "a",
+ Title: "h2",
+ Desc: "span.OrganicTextContentSpan",
+ ImageItems: "div[role='main'] div[data-state]",
+ ImageItemsAlt: []string{"div[data-state*='serpList']"},
+ ImageStateAll: "div[data-state]",
}