zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-05 16:53:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: speed up SERP parsing and centralize selectors

Browse Source
This commit is contained in:
Rustem Kamalov
2026-05-06 04:12:15 +03:00
parent ed23af7acc
commit 471e5ad7fe
19 changed files with 711 additions and 435 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
baidu/search.go
View File

@@ -33,7 +33,6 @@ type imageDataJson struct {
}
}
// Baidu implements core.SearchEngine for Baidu SERP pages.
type Baidu struct {
core.Browser
@@ -123,7 +122,12 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
}
}
searchRes, err := page.Timeout(baid.Timeout).Search(Selectors.Results)
resultElements, _, err := core.WaitForElements(
ctx,
page,
append([]string{Selectors.Results}, Selectors.ResultsAlt...),
baid.GetSelectorTimeout(),
)
if err != nil {
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
closePage()
@@ -131,11 +135,10 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
}
closePage()
baid.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrParser
return nil, core.ErrSearchTimeout
}
// Check why no results, maybe captcha?
if searchRes == nil {
if len(resultElements) == 0 {
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
closePage()
return nil, blockErr
@@ -144,16 +147,12 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
return nil, nil
}
resultElements, err := searchRes.All()
if err != nil {
closePage()
return nil, err
}
for i, r := range resultElements {
// Get URL
link, err := r.Element(Selectors.Link)
if err != nil {
// Element detached mid-iteration (page navigated/refreshed): the rest
// of the slice is also stale, no point continuing.
if core.IsRodObjectNotFound(err) {
break
}
@@ -210,7 +209,8 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
return nil, err
}
// Get anti-crawler cookies first, then reload page
// First load often seeds anti-crawler cookies; results tend to become
// available after one explicit reload.
page, err := baid.Navigate(ctx, url)
if err != nil {
return nil, err
@@ -223,18 +223,14 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
baid.logger.Debug("Page close error: %v", closeErr)
}
}
if err := page.Reload(); err != nil {
jsonWaitTimeout := baid.GetSelectorTimeout()
if reloadErr := page.Reload(); reloadErr != nil {
closePage()
baid.logger.Error("Page reload failed: %s", err)
return nil, core.ErrSearchTimeout
}
if err := page.WaitLoad(); err != nil {
closePage()
baid.logger.Error("Page load wait failed: %s", err)
return nil, core.ErrSearchTimeout
}
result, err := page.Timeout(baid.Timeout).Search("body > pre")
preElements, _, err := core.WaitForElements(ctx, page, Selectors.ImageJSONRoot, jsonWaitTimeout)
if err != nil {
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
closePage()
@@ -242,11 +238,10 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
}
closePage()
baid.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrParser
return nil, core.ErrSearchTimeout
}
// Check why no results, maybe captcha?
if result == nil {
if len(preElements) == 0 {
if blockErr := baid.classifyBlockPage(page, url); blockErr != nil {
closePage()
return nil, blockErr
@@ -255,7 +250,7 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
return nil, nil
}
jsonText, err := result.First.Text()
jsonText, err := preElements[0].Text()
if err != nil {
closePage()
return nil, err
@@ -286,6 +281,10 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
baid.logger.Error("Failed to unmarshal JSON: %v", err)
return nil, core.ErrParser
}
if len(data.Data) == 0 {
closePage()
break
}
for i, img := range data.Data {
if len(img.URL) == 0 {
@@ -314,6 +313,9 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
}(),
}
searchResults = append(searchResults, res)
if query.Limit > 0 && len(searchResults) >= query.Limit {
break
}
}
searchPage += 1
@@ -321,5 +323,9 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se
closePage()
}
return core.DeduplicateResults(searchResults), nil
deduped := core.DeduplicateResults(searchResults)
if query.Limit > 0 && len(deduped) > query.Limit {
deduped = deduped[:query.Limit]
}
return deduped, nil
}

24
baidu/selectors.go
View File

@@ -2,15 +2,19 @@ package baidu
// Selectors is the single source of truth for Baidu SERP CSS selectors.
var Selectors = struct {
Captcha string
Timeout string
Results string
Link string
Desc string
Captcha string
Timeout string
Results string
ResultsAlt []string
ImageJSONRoot []string
Link string
Desc string
}{
Captcha: "div.passMod_dialog-wrapper",
Timeout: "button.timeout-button",
Results: "div.c-container.new-pmd",
Link: "a",
Desc: "div.c-abstract",
Captcha: "div.passMod_dialog-wrapper",
Timeout: "button.timeout-button",
Results: "div.c-container.new-pmd",
ResultsAlt: []string{"#content_left div.result.c-container", "#content_left div.result-op.c-container"},
ImageJSONRoot: []string{"body > pre", "pre"},
Link: "a",
Desc: "div.c-abstract",
}

4
bing/parse_html.go
View File

@@ -64,7 +64,7 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
title := titleTag.Text()
desc := ""
if descTag := item.Find(Selectors.AdDesc).First(); descTag.Length() > 0 {
if descTag := item.Find("p").First(); descTag.Length() > 0 {
desc = descTag.Text()
}
@@ -93,7 +93,7 @@ func descriptionFromItem(item *goquery.Selection, title string) string {
return text
}
}
if descTag := item.Find(Selectors.DescLast).First(); descTag.Length() > 0 {
if descTag := item.Find("p").First(); descTag.Length() > 0 {
if text := strings.TrimSpace(descTag.Text()); text != "" {
return text
}

92
bing/search.go
View File

@@ -122,11 +122,6 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
}
}()
if err := page.WaitLoad(); err != nil {
bing.logger.Error("Initial page load wait failed: %s", err)
return nil, core.ErrSearchTimeout
}
if bing.checkCaptcha(page) {
bing.logger.Error("Captcha detected: %s", url)
return nil, core.ErrCaptcha
@@ -135,18 +130,19 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
if err := bing.acceptCookies(ctx, page); err != nil {
return nil, err
}
if err := page.WaitLoad(); err != nil {
bing.logger.Error("Post-consent page load wait failed: %s", err)
organicElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, bing.GetSelectorTimeout())
if err != nil {
// Re-check captcha on timeout - Bing interstitials can render after WaitLoad.
if bing.checkCaptcha(page) {
bing.logger.Error("Captcha detected: %s", url)
return nil, core.ErrCaptcha
}
bing.logger.Error("Cannot parse organic results: %s", err)
return nil, core.ErrSearchTimeout
}
organicElements, err := page.Timeout(bing.Timeout).Elements(Selectors.Results)
if err != nil {
bing.logger.Error("Cannot parse organic results: %s", err)
return nil, core.ErrParser
}
adElements, err := page.Timeout(bing.Timeout).Elements(Selectors.Ads)
adElements, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Ads)
if err != nil {
bing.logger.Debug("No ads found")
}
@@ -166,7 +162,6 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
bing.logger.Debug("Missing title")
continue
}
srchRes.Title, _ = titleElem.Text()
href, err := titleElem.Property("href")
if err != nil {
@@ -174,13 +169,26 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
continue
}
srchRes.URL = href.String()
if strings.TrimSpace(srchRes.URL) == "" || strings.HasPrefix(srchRes.URL, "javascript:") {
continue
}
srchRes.Title, _ = titleElem.Text()
srchRes.Title = strings.TrimSpace(srchRes.Title)
if srchRes.Title == "" {
srchRes.Title = core.FirstNonEmptyText(result, Selectors.TitleFallbacks...)
}
if srchRes.Title == "" {
bing.logger.Debug("Missing title text")
continue
}
var desc string
if descElem, err := result.Element(Selectors.DescPrimary); err == nil {
desc, _ = descElem.Text()
} else if descElem, err := result.Element(Selectors.DescFallback); err == nil {
desc, _ = descElem.Text()
} else if descElem, err := result.Element(Selectors.DescLast); err == nil {
} else if descElem, err := result.Element("p"); err == nil {
desc, _ = descElem.Text()
} else {
fullText, _ := result.Text()
@@ -204,6 +212,13 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
continue
}
srchRes.Title, _ = titleElem.Text()
srchRes.Title = strings.TrimSpace(srchRes.Title)
if srchRes.Title == "" {
srchRes.Title = core.FirstNonEmptyText(adResult, "h2", "a[aria-label]")
}
if srchRes.Title == "" {
continue
}
href, err := titleElem.Property("href")
if err != nil {
@@ -212,7 +227,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
}
srchRes.URL = href.String()
if descElem, err := adResult.Element(Selectors.AdDesc); err == nil {
if descElem, err := adResult.Element("p"); err == nil {
srchRes.Description, _ = descElem.Text()
}
@@ -261,6 +276,16 @@ type BingImageData struct {
MURL string `json:"murl"` // Image URL
}
func resolveImageLinkElement(container *rod.Element) (*rod.Element, error) {
if container == nil {
return nil, errors.New("nil image container")
}
if core.HasAttribute(container, "m") {
return container, nil
}
return container.Element("a")
}
// SearchImage executes a Bing image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
@@ -294,11 +319,6 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
}
}()
if err := page.WaitLoad(); err != nil {
bing.logger.Error("Initial image page load wait failed: %s", err)
return nil, core.ErrSearchTimeout
}
// Check for captcha
if bing.checkCaptcha(page) {
bing.logger.Error("Captcha detected during image search: %s", url)
@@ -310,18 +330,16 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
return nil, err
}
// Wait for image results to load
if err := page.WaitLoad(); err != nil {
bing.logger.Error("Image results load wait failed: %s", err)
return nil, core.ErrSearchTimeout
}
if err := core.SleepContext(ctx, 2*time.Second); err != nil {
return nil, err
}
// Find all image result containers using CSS selector
imageContainers, err := page.Timeout(bing.Timeout).Elements(Selectors.ImageResults)
imageContainers, _, err := core.WaitForElements(
ctx,
page,
[]string{Selectors.ImageResults},
bing.GetSelectorTimeout(),
)
if err != nil {
if bing.checkCaptcha(page) {
return nil, core.ErrCaptcha
}
bing.logger.Error("Cannot parse image results: %s", err)
return nil, core.ErrSearchTimeout
}
@@ -336,10 +354,9 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
for _, c := range imageContainers {
srchRes := core.SearchResult{}
// Get the <a> element inside the div
linkElem, err := c.Element("a")
linkElem, err := resolveImageLinkElement(c)
if err != nil {
bing.logger.Debug("Missing <a> element")
bing.logger.Debug("Missing image link element")
continue
}
@@ -386,6 +403,9 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
srchRes.Rank = rank
searchResults = append(searchResults, srchRes)
if query.Limit > 0 && len(searchResults) >= query.Limit {
break
}
}
return searchResults, nil

37
bing/selectors.go
View File

@@ -2,27 +2,28 @@ package bing
// Selectors is the single source of truth for Bing SERP CSS selectors.
var Selectors = struct {
Captcha []string
CookieBtn string
Results string
Ads string
ImageResults string
Title string
DescPrimary string
DescFallback string
DescLast string
AdTitle string
AdDesc string
Captcha []string
CookieBtn string
Results string
Ads string
ImageResults string
Title string
TitleFallbacks []string
DescPrimary string
DescFallback string
AdTitle string
}{
Captcha: []string{"div.captcha", "div.captcha_header"},
CookieBtn: "button#bnp_btn_accept",
Results: "li.b_algo",
Ads: "li.b_ad",
ImageResults: "div.iuscp, div.isv",
Title: "a",
DescPrimary: "div.b_caption p",
DescFallback: "div.b_caption div",
DescLast: "p",
AdTitle: "h2 a",
AdDesc: "p",
ImageResults: "a.iusc, div.iuscp, div.isv",
Title: "h2 a",
// TitleFallbacks are tried by FirstNonEmptyText when the primary Title
// selector matches but yields empty text (Bing occasionally renders an
// empty <h2><a/></h2> while the visible label sits in aria-label or h2).
TitleFallbacks: []string{"h2", "a[aria-label]"},
DescPrimary: "div.b_caption p",
DescFallback: "div.b_caption div",
AdTitle: "h2 a",
}

2
cmd/root.go
View File

@@ -16,7 +16,7 @@ import (
)
const (
version = "0.7.8"
version = "0.7.9"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)

2
config.yaml
View File

@@ -41,7 +41,7 @@ cache:
max_size: 1000 # Maximum cached dedicated responses before oldest-entry eviction
resilience:
max_retries: 2 # Retry attempts per engine request (0 disables retries)
max_retries: 1 # Retry attempts per engine request (0 disables retries)
allow_endpoint_fallback: false # Keep dedicated endpoints engine-pure by default
# circuit_breaker:

133
core/page_helpers.go Normal file
View File

@@ -0,0 +1,133 @@
package core
import (
"context"
"strings"
"time"
"github.com/go-rod/rod"
)
// pollInterval is how often WaitForElements re-probes selectors while the
// page hydrates. Short enough to feel snappy, long enough not to hammer CDP.
const pollInterval = 120 * time.Millisecond
// WaitForElements probes the supplied CSS selectors until one returns at least
// one matching element or timeout elapses. It exists because rod's
// page.Search/Elements and the surrounding WaitLoad/WaitStable do not wait for
// a *specific* selector to hydrate — modern SPA SERPs (DDG, Bing, Google)
// regularly fire `load` and even reach DOM-stable before result rows render,
// causing parsers to see an empty page on the first probe and forcing the
// caller's retry layer to reload.
//
// The probe loop returns as soon as a selector matches, returning the matched
// elements and the selector that hit. On timeout it returns ErrSearchTimeout
// so callers can disambiguate between "no results" / "captcha" by inspecting
// the page directly.
func WaitForElements(ctx context.Context, page *rod.Page, selectors []string, timeout time.Duration) (rod.Elements, string, error) {
if page == nil {
return nil, "", ErrSearchTimeout
}
ctx = EnsureContext(ctx)
if timeout <= 0 {
timeout = 2 * time.Second
}
probe := func() (rod.Elements, string) {
for _, selector := range selectors {
elements, err := page.Elements(selector)
if err != nil || len(elements) == 0 {
continue
}
return elements, selector
}
return nil, ""
}
if elements, selector := probe(); len(elements) > 0 {
return elements, selector, nil
}
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if err := ctx.Err(); err != nil {
return nil, "", err
}
if elements, selector := probe(); len(elements) > 0 {
return elements, selector, nil
}
if err := SleepContext(ctx, pollInterval); err != nil {
return nil, "", err
}
}
return nil, "", ErrSearchTimeout
}
// HasAnySelector returns true if at least one of the supplied selectors
// currently matches in the page DOM. It does not wait — pair with
// WaitForElements when hydration may be in flight.
func HasAnySelector(page *rod.Page, selectors []string) bool {
if page == nil {
return false
}
for _, selector := range selectors {
has, _, err := page.Has(selector)
if err == nil && has {
return true
}
}
return false
}
// HasAttribute reports whether el carries attr (regardless of value).
func HasAttribute(el *rod.Element, attr string) bool {
if el == nil {
return false
}
v, err := el.Attribute(attr)
return err == nil && v != nil
}
// FirstNonEmptyText returns the trimmed text of the first selector under root
// that yields non-empty content. Empty string if none match.
func FirstNonEmptyText(root *rod.Element, selectors ...string) string {
if root == nil {
return ""
}
for _, selector := range selectors {
el, err := root.Element(selector)
if err != nil {
continue
}
text, err := el.Text()
if err != nil {
continue
}
if trimmed := strings.TrimSpace(text); trimmed != "" {
return trimmed
}
}
return ""
}
// FirstNonEmptyAttribute returns the trimmed value of attr from the first
// selector under root whose attribute is non-empty.
func FirstNonEmptyAttribute(root *rod.Element, attr string, selectors ...string) string {
if root == nil {
return ""
}
for _, selector := range selectors {
el, err := root.Element(selector)
if err != nil {
continue
}
value, err := el.Attribute(attr)
if err != nil || value == nil {
continue
}
if trimmed := strings.TrimSpace(*value); trimmed != "" {
return trimmed
}
}
return ""
}

8
core/server.go
View File

@@ -189,10 +189,16 @@ func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngin
continue
}
locParser := parser
serv.app.Post(fmt.Sprintf("/%s/parse", strings.ToLower(parser.Name())),
parserEndpointName := strings.ToLower(parser.Name())
serv.app.Post(fmt.Sprintf("/%s/parse", parserEndpointName),
func(c *fiber.Ctx) error {
return serv.handleParseEndpoint(c, locParser)
})
if parserEndpointName == "duckduckgo" {
serv.app.Post("/duck/parse", func(c *fiber.Ctx) error {
return serv.handleParseEndpoint(c, locParser)
})
}
}
serv.app.Get("/mega/search", serv.handleMegaSearch)

10
core/server_parse_test.go
View File

@@ -116,3 +116,13 @@ func TestParseEndpointMarkdownFormat(t *testing.T) {
t.Fatalf("expected markdown content type, got %s", ct)
}
}
func TestParseEndpointDuckAlias(t *testing.T) {
engine := &parserMock{engineMock: engineMock{name: "duckduckgo", initialized: true}}
srv := NewServerWithOptions("127.0.0.1", 7125, DefaultServerOptions(), engine)
resp := postHTML(t, srv, "/duck/parse", "<html><body>sample serp</body></html>")
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected 200, got %d", resp.StatusCode)
}
}

157
duckduckgo/search.go
View File

@@ -11,10 +11,6 @@ import (
"golang.org/x/time/rate"
)
// captchaBodyText is matched against the raw page HTML because DDG returns
// a plain-text 202 rate-limit response rather than a structured captcha page.
const captchaBodyText = "bots user"
// DuckDuckGo implements core.SearchEngine for DuckDuckGo SERP pages.
type DuckDuckGo struct {
core.Browser
@@ -46,16 +42,36 @@ func (ddg *DuckDuckGo) GetRateLimiter() *rate.Limiter {
}
func (ddg *DuckDuckGo) isCaptcha(page *rod.Page) bool {
html, err := page.Timeout(ddg.GetSelectorTimeout()).HTML()
if core.HasAnySelector(page, Selectors.CaptchaSelectors) {
return true
}
if info, err := page.Info(); err == nil {
url := strings.ToLower(info.URL)
if strings.Contains(url, "anomaly") || strings.Contains(url, "captcha") || strings.Contains(url, "challenge") {
return true
}
}
htmlTimeout := ddg.GetSelectorTimeout() / 2
if htmlTimeout <= 0 || htmlTimeout > 1500*time.Millisecond {
htmlTimeout = 1500 * time.Millisecond
}
html, err := page.Timeout(htmlTimeout).HTML()
if err != nil {
return false
}
return strings.Contains(html, captchaBodyText)
html = strings.ToLower(html)
for _, marker := range Selectors.CaptchaMarkers {
if strings.Contains(html, marker) {
return true
}
}
return false
}
func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool {
has, _, _ := page.Has(Selectors.NoResults)
return has
return core.HasAnySelector(page, Selectors.NoResults)
}
func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
@@ -75,6 +91,8 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
if err != nil {
ddg.logger.Debug("Missing link")
// If the result element itself was detached (page navigated mid-iteration),
// the rest of the slice is also stale and further work is wasted.
if core.IsRodObjectNotFound(err) {
break
}
@@ -94,29 +112,12 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
continue
}
// Get title - try multiple selectors
var titleTag *rod.Element
for _, selector := range Selectors.Title {
titleTag, err = r.Element(selector)
if err == nil {
break
}
title := core.FirstNonEmptyText(r, Selectors.Title...)
if title == "" {
title = "No title"
}
title := "No title"
if titleTag != nil {
title, _ = titleTag.Text()
}
// Get description - try multiple selectors
desc := ""
for _, selector := range Selectors.Desc {
descTag, err := r.Element(selector)
if err == nil {
desc, _ = descTag.Text()
break
}
}
desc := core.FirstNonEmptyText(r, Selectors.Desc...)
// Check if it's an ad
isAd := false
@@ -181,50 +182,30 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
}
}
// Get all search results in page - try multiple selectors
var searchRes *rod.SearchResult
var searchErr error
for _, selector := range Selectors.Results {
searchRes, searchErr = page.Timeout(ddg.GetSelectorTimeout()).Search(selector)
if searchErr == nil && searchRes != nil {
ddg.logger.Debug("Found results with selector: %s", selector)
break
}
}
if searchErr != nil {
closePage()
ddg.logger.Error("Cannot parse search results: %s", searchErr)
return nil, core.ErrParser
}
// Check why no results, maybe captcha?
if searchRes == nil {
closePage()
elements, selector, err := core.WaitForElements(ctx, page, Selectors.Results, ddg.GetSelectorTimeout())
if err != nil {
if ddg.isNoResults(page) {
ddg.logger.Warn("No results found")
} else if ddg.isCaptcha(page) {
closePage()
break
}
if ddg.isCaptcha(page) {
ddg.logger.Error("Captcha detected: %s", url)
closePage()
return nil, core.ErrCaptcha
}
break
}
elements, err := searchRes.All()
if err != nil {
ddg.logger.Error("Cannot get search elements: %s", err)
closePage()
break
ddg.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrSearchTimeout
}
ddg.logger.Debug("Found results with selector: %s", selector)
r := ddg.parseResults(elements, searchPage)
if len(r) == 0 {
ddg.logger.Debug("No valid results found on page %d", searchPage)
closePage()
return nil, core.ErrParser
return nil, core.ErrSearchTimeout
}
allResults = append(allResults, r...)
@@ -287,36 +268,8 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor
}()
}
// Wait for page to load
if err := page.WaitLoad(); err != nil {
ddg.logger.Error("Wait load failed: %s", err)
return searchResults, core.ErrSearchTimeout
}
if err := core.SleepContext(ctx, 2*time.Second); err != nil {
return searchResults, err
}
// Try multiple selectors for DuckDuckGo image results
var searchRes *rod.SearchResult
var searchErr error
for _, selector := range Selectors.ImageResult {
searchRes, searchErr = page.Timeout(ddg.GetSelectorTimeout()).Search(selector)
if searchErr == nil && searchRes != nil {
ddg.logger.Debug("Found image results with selector: %s", selector)
break
} else {
ddg.logger.Debug("Selector '%s' not found: %v", selector, searchErr)
}
}
if searchErr != nil {
ddg.logger.Error("Cannot find image results: %s", searchErr)
return searchResults, core.ErrSearchTimeout
}
// Check why no results
if searchRes == nil {
elements, selector, err := core.WaitForElements(ctx, page, Selectors.ImageResult, ddg.GetSelectorTimeout())
if err != nil {
if ddg.isCaptcha(page) {
ddg.logger.Error("Captcha detected: %s", url)
return searchResults, core.ErrCaptcha
@@ -325,12 +278,7 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor
}
return searchResults, core.ErrSearchTimeout
}
elements, err := searchRes.All()
if err != nil {
ddg.logger.Error("Cannot get search elements: %s", err)
return searchResults, err
}
ddg.logger.Debug("Found image results with selector: %s", selector)
ddg.logger.Info("Found %d image elements", len(elements))
@@ -357,20 +305,9 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor
continue
}
// Get title - try multiple selectors based on the HTML structure
var titleTag *rod.Element
var titleErr error
for _, selector := range Selectors.ImageTitle {
titleTag, titleErr = r.Element(selector)
if titleErr == nil {
break
}
}
title := "No title"
if titleTag != nil {
title, _ = titleTag.Text()
title := core.FirstNonEmptyText(r, Selectors.ImageTitle...)
if title == "" {
title = "No title"
}
// Get source page URL - try multiple selectors

55
duckduckgo/selectors.go
View File

@@ -1,23 +1,52 @@
package duckduckgo
// Selectors is the single source of truth for DuckDuckGo SERP CSS selectors.
//
// Order matters: WaitForElements / parseResults try selectors in declared order
// and stop on the first hit. Put the most specific / current variants first;
// keep older fallbacks last to absorb DOM rewrites without losing coverage.
var Selectors = struct {
NoResults string
Results []string
Title []string
Desc []string
Link []string
AdBadge []string
ImageResult []string
ImageImg []string
ImageTitle []string
ImageLink []string
NoResults []string
CaptchaSelectors []string
CaptchaMarkers []string
Results []string
Title []string
Desc []string
Link []string
AdBadge []string
ImageResult []string
ImageImg []string
ImageTitle []string
ImageLink []string
}{
NoResults: "div[class*='no-results']",
NoResults: []string{
"div[class*='no-results']",
"[data-testid='no-results']",
"div[data-result='no-results']",
},
// CaptchaSelectors detect DDG's anomaly/challenge interstitials via DOM.
CaptchaSelectors: []string{
"form[action*='anomaly']",
"input[name='challenge']",
"div[id*='anomaly']",
"div[class*='captcha']",
},
// CaptchaMarkers are case-insensitive substrings used for full-HTML scan
// fallback when the selectors above don't match. DDG sometimes returns a
// plain-text 202 rate-limit page rather than a structured form, so the
// HTML scan is the only reliable signal in those cases.
CaptchaMarkers: []string{
"bots user",
"bots use duckduckgo too",
"human verification",
"unusual traffic",
"anomaly",
},
Results: []string{
"article[data-testid='result']",
"div.result",
"li[data-layout='organic']",
"div[data-testid='result']",
"div.result",
},
Title: []string{
"h2",
@@ -41,11 +70,11 @@ var Selectors = struct {
".result--ad",
},
ImageResult: []string{
"figure[data-testid='image-result']",
"figure",
},
ImageImg: []string{
"img",
"img[src*='duckduckgo.com']",
},
ImageTitle: []string{
"figcaption a p span",

87
ecosia/parse_html.go
View File

@@ -1,7 +1,6 @@
package ecosia
import (
"fmt"
"io"
"strings"
@@ -85,50 +84,50 @@ func parseEcosiaItem(item *goquery.Selection, rank int, ad bool) (core.SearchRes
}, true
}
// parseEcosiaImageItem extracts a single image card from a goquery Selection.
func parseEcosiaImageItem(item *goquery.Selection, rank int) (core.SearchResult, bool) {
linkTag := item.Find(Selectors.ImageLink).First()
if linkTag.Length() == 0 {
return core.SearchResult{}, false
}
href, exists := linkTag.Attr("href")
if !exists {
return core.SearchResult{}, false
}
imgURL := strings.TrimSpace(href)
if imgURL == "" {
return core.SearchResult{}, false
}
// // parseEcosiaImageItem extracts a single image card from a goquery Selection.
// func parseEcosiaImageItem(item *goquery.Selection, rank int) (core.SearchResult, bool) {
// linkTag := item.Find(Selectors.ImageLink).First()
// if linkTag.Length() == 0 {
// return core.SearchResult{}, false
// }
// href, exists := linkTag.Attr("href")
// if !exists {
// return core.SearchResult{}, false
// }
// imgURL := strings.TrimSpace(href)
// if imgURL == "" {
// return core.SearchResult{}, false
// }
title := ""
if img := linkTag.Find("img").First(); img.Length() > 0 {
if alt, err := img.Attr("alt"); err {
title = strings.TrimSpace(alt)
}
}
// title := ""
// if img := linkTag.Find("img").First(); img.Length() > 0 {
// if alt, err := img.Attr("alt"); err {
// title = strings.TrimSpace(alt)
// }
// }
source := ""
if s := item.Find(Selectors.ImageSource).First(); s.Length() > 0 {
source = strings.TrimSpace(s.Text())
}
dims := ""
if d := item.Find(Selectors.ImageDims).First(); d.Length() > 0 {
dims = strings.TrimSpace(d.Text())
}
// source := ""
// if s := item.Find(Selectors.ImageSource).First(); s.Length() > 0 {
// source = strings.TrimSpace(s.Text())
// }
// dims := ""
// if d := item.Find(Selectors.ImageDims).First(); d.Length() > 0 {
// dims = strings.TrimSpace(d.Text())
// }
desc := source
if dims != "" {
if source != "" {
desc = fmt.Sprintf("%s (%s)", source, dims)
} else {
desc = dims
}
}
// desc := source
// if dims != "" {
// if source != "" {
// desc = fmt.Sprintf("%s (%s)", source, dims)
// } else {
// desc = dims
// }
// }
return core.SearchResult{
Rank: rank,
URL: imgURL,
Title: title,
Description: desc,
}, true
}
// return core.SearchResult{
// Rank: rank,
// URL: imgURL,
// Title: title,
// Description: desc,
// }, true
// }

55
ecosia/search_integration_test.go Normal file
View File

@@ -0,0 +1,55 @@
//go:build integration
// +build integration
package ecosia
import (
"context"
"testing"
"github.com/karust/openserp/core"
"github.com/karust/openserp/testutil"
"github.com/karust/openserp/testutil/ithelper"
)
func TestSearchEcosia(t *testing.T) {
testutil.RequireIntegration(t)
browser := ithelper.CreateBrowser(t)
engine := New(*browser, core.SearchEngineOptions{})
query := core.Query{Text: "golang programming", Limit: 10}
results, err := engine.Search(context.Background(), query)
ithelper.HandleError(t, "ecosia web search", err)
if len(results) == 0 {
t.Fatal("returned empty results")
}
if results[0].URL == "" {
t.Fatal("first result URL is empty")
}
if results[0].Title == "" {
t.Fatal("first result title is empty")
}
}
func TestImageSearchEcosia(t *testing.T) {
testutil.RequireIntegration(t)
browser := ithelper.CreateBrowser(t)
engine := New(*browser, core.SearchEngineOptions{})
query := core.Query{Text: "golden retriever puppy", Limit: 10}
results, err := engine.SearchImage(context.Background(), query)
ithelper.HandleError(t, "ecosia image search", err)
if len(results) == 0 {
t.Fatal("returned empty image results")
}
if results[0].URL == "" {
t.Fatal("first image result URL is empty")
}
if results[0].Title == "" {
t.Fatal("first image result title is empty")
}
}

255
google/search.go
View File

@@ -220,15 +220,23 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
gogl.acceptCookies(page)
}
// Find all results using stable attributes
searchRes, err := page.Timeout(gogl.Timeout).Search(Selectors.Results)
// Wait for result containers (data-hveid + data-ved) to hydrate. WaitLoad in
// Navigate fires before Google's right-rail/answers script attaches these
// attributes, so a one-shot Search races the DOM and frequently sees nothing.
searchResultElems, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, gogl.GetSelectorTimeout())
if err != nil {
gogl.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrParser
}
if searchRes == nil {
return nil, nil
if gogl.checkCaptcha(page, query.ProxyURL) {
gogl.logger.Error("Captcha detected: %s", url)
return nil, core.ErrCaptcha
}
if core.IsContextDone(err) {
return nil, err
}
// Keep empty-SERP behavior for selector timeout only.
if errors.Is(err, core.ErrSearchTimeout) {
return nil, nil
}
return nil, core.ErrSearchTimeout
}
totalResults, err := gogl.getTotalResults(page)
@@ -237,26 +245,15 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
}
gogl.logger.Info("Found %d total results", totalResults)
searchResultElems, err := searchRes.All()
if err != nil {
return nil, err
}
rank := query.Start
for _, resEl := range searchResultElems {
srchRes := core.SearchResult{}
describe, err := resEl.Describe(1, false)
if err != nil {
gogl.logger.Debug("Result describe failed: %s", err)
if core.IsRodObjectNotFound(err) {
break
}
continue
}
attrs := strings.Join(describe.Attributes, " ")
isAd := core.HasAttribute(resEl, "data-text-ad")
isAnswerBox := query.Answers && core.HasAttribute(resEl, "data-ulkwtsb") && !core.HasAttribute(resEl, "data-ispaa")
isResultCandidate := core.HasAttribute(resEl, "data-ved")
if strings.Contains(attrs, "data-text-ad") {
if isAd {
// 1. Parse ads
srchRes.Ad = true
@@ -293,10 +290,14 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
continue
}
textSliced := strings.Split(text, "\n")
srchRes.Description = strings.Join(textSliced[4:], "\n")
if len(textSliced) > 4 {
srchRes.Description = strings.Join(textSliced[4:], "\n")
} else {
srchRes.Description = strings.TrimSpace(text)
}
rank += 1
} else if query.Answers && strings.Contains(attrs, "data-ulkwtsb") && !strings.Contains(attrs, "data-ispaa") {
} else if isAnswerBox {
// 2. Parse answer boxes
answerEls, err := resEl.Page().Search(Selectors.AnswerBox)
if err != nil {
@@ -321,7 +322,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
if err := answ.Focus(); err != nil {
gogl.logger.Debug("Answer focus failed: %s", err)
}
//answ.Page().WaitRepaint()
}
if err := core.SleepContext(ctx, 2*time.Second); err != nil {
return nil, err
@@ -361,7 +362,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
searchResults = append(searchResults, srchRes)
}
continue
} else if strings.Contains(attrs, "data-ved") {
} else if isResultCandidate {
// Parse regular search results
// Get title from h3
titleTag, err := resEl.Element(Selectors.Title)
@@ -419,7 +420,6 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
continue
} else {
//fmt.Println(i, attrs)
continue
}
@@ -427,7 +427,20 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
searchResults = append(searchResults, srchRes)
}
return core.DeduplicateResults(searchResults), nil
deduped := core.DeduplicateResults(searchResults)
if len(deduped) == 0 {
if gogl.checkCaptcha(page, query.ProxyURL) {
return nil, core.ErrCaptcha
}
// Result candidates were found by Selectors.Results but none parsed
// into usable rows: treat as a genuine no-results SERP rather than a
// timeout, so callers don't retry pointlessly.
if len(searchResultElems) == 0 {
return nil, nil
}
return nil, core.ErrSearchTimeout
}
return deduped, nil
}
// SearchImage executes a Google image search and returns normalized image
@@ -456,111 +469,127 @@ func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.S
defer gogl.close(ctx, page)
for len(searchResultsMap) < query.Limit {
if err := page.WaitLoad(); err != nil {
gogl.logger.Error("Image page load wait failed: %s", err)
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
// Hard cap on outer scroll/parse passes. Google's image grid is virtualized
// and infinite-scroll: WaitLoad after scroll never settles and individual
// cells can hang on right-click. Cap iterations as a last-resort guard.
const maxImagePasses = 20
stagnant := 0
for pass := 0; pass < maxImagePasses && len(searchResultsMap) < query.Limit; pass++ {
if err := ctx.Err(); err != nil {
return *core.ConvertSearchResultsMap(searchResultsMap), err
}
if err := page.Mouse.Scroll(0, 1000000, 1); err != nil {
gogl.logger.Error("Image page scroll failed: %s", err)
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
if _, err := page.Eval(`() => window.scrollBy(0, 1000000)`); err != nil {
gogl.logger.Debug("Image page scroll failed: %s", err)
}
if err := page.WaitLoad(); err != nil {
gogl.logger.Error("Image results load wait failed: %s", err)
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
if err := core.SleepContext(ctx, 600*time.Millisecond); err != nil {
return *core.ConvertSearchResultsMap(searchResultsMap), err
}
results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
resultElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.ImageResults}, gogl.GetSelectorTimeout())
if err != nil {
gogl.logger.Error("Cannot parse search results: %s", err)
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
}
// Check why no results
if results == nil {
if gogl.checkCaptcha(page, query.ProxyURL) {
gogl.logger.Error("Captcha detected: %s", url)
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
}
return *core.ConvertSearchResultsMap(searchResultsMap), err
break
}
resultElements, err := results.All()
if err != nil {
return *core.ConvertSearchResultsMap(searchResultsMap), err
}
if len(resultElements) < len(searchResultsMap) {
continue
}
for i, r := range resultElements {
// TODO: parse AF_initDataCallback to optimize instead of this?
err := r.Click(proto.InputMouseButtonRight, 1)
if err != nil {
gogl.logger.Error("Click failed")
continue
before := len(searchResultsMap)
for _, r := range resultElements {
if err := ctx.Err(); err != nil {
return *core.ConvertSearchResultsMap(searchResultsMap), err
}
dataVed, err := r.Attribute("data-ved")
if err != nil {
gogl.logger.Error("Missing data-ved attribute")
continue
}
// If already have image with this ID
if _, ok := searchResultsMap[*dataVed]; ok {
continue
}
// Get URLs
link, err := r.Element("a:not([ping])")
if err != nil {
continue
}
linkText, err := link.Property("href")
if err != nil {
gogl.logger.Debug("Missing href")
continue
}
imgSrc, err := parseSourceImageURL(linkText.String())
if err != nil {
gogl.logger.Error("Failed to parse image URL: %v", err)
continue
}
// Get title
titleTag, err := r.Element("h3")
if err != nil {
gogl.logger.Error("Missing h3 tag")
continue
}
title, err := titleTag.Text()
if err != nil {
gogl.logger.Error("Failed to extract title")
title = "No title"
}
gR := core.SearchResult{
Rank: i + 1,
URL: imgSrc.OriginalURL,
Title: title,
Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
}
searchResultsMap[*dataVed] = gR
gogl.parseImageCell(r, searchResultsMap)
// Always remove the cell so the next outer iteration only sees
// freshly-scrolled elements. Without this we re-iterate the same
// already-parsed cells and the loop scales O(n²) — or worse,
// hangs entirely when right-click on a stale node never returns.
if err := r.Remove(); err != nil {
gogl.logger.Debug("Failed to remove parsed image element: %s", err)
}
if len(searchResultsMap) >= query.Limit {
break
}
}
if len(searchResultsMap) == before {
stagnant++
if stagnant >= 2 {
break
}
} else {
stagnant = 0
}
}
return *core.ConvertSearchResultsMap(searchResultsMap), nil
}
// parseImageCell extracts one image result from a Google image grid cell and
// stores it in dst keyed by the cell's data-ved. Returns silently on any
// failure — the surrounding loop calls r.Remove() unconditionally so a
// problem cell can't stall the outer scroll loop.
func (gogl *Google) parseImageCell(r *rod.Element, dst map[string]core.SearchResult) {
// Right-clicking a Google image cell forces it to materialize its
// `imgres` link (the grid is virtualized; href is absent until the
// cell is interacted with).
if err := r.Click(proto.InputMouseButtonRight, 1); err != nil {
gogl.logger.Debug("Right-click on image cell failed: %s", err)
return
}
dataVed, err := r.Attribute("data-ved")
if err != nil || dataVed == nil || strings.TrimSpace(*dataVed) == "" {
gogl.logger.Debug("Missing data-ved attribute")
return
}
resultKey := *dataVed
if _, ok := dst[resultKey]; ok {
return
}
link, err := r.Element(Selectors.ImageLink)
if err != nil {
link, err = r.Element(Selectors.ImageLinkFallback)
if err != nil {
return
}
}
linkText, err := link.Property("href")
if err != nil {
gogl.logger.Debug("Missing href")
return
}
imgSrc, err := parseSourceImageURL(linkText.String())
if err != nil {
gogl.logger.Error("Failed to parse image URL: %v", err)
return
}
if strings.TrimSpace(imgSrc.OriginalURL) == "" {
return
}
title := core.FirstNonEmptyText(r, Selectors.ImageTitle...)
if title == "" {
title = core.FirstNonEmptyAttribute(r, "alt", "img")
}
if title == "" {
title = core.FirstNonEmptyAttribute(r, "aria-label", "a")
}
if title == "" {
title = "No title"
}
dst[resultKey] = core.SearchResult{
Rank: len(dst) + 1,
URL: imgSrc.OriginalURL,
Title: title,
Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
}
}
func (gogl *Google) close(ctx context.Context, page *rod.Page) {
if !gogl.Browser.LeavePageOpen {
err := core.ClosePageWithTimeout(ctx, page, time.Second)

16
google/selectors.go
View File

@@ -13,6 +13,12 @@ var Selectors = struct {
DescFallback string
AnswerBox string
AnswerItem string
// Image search.
ImageResults string
ImageLink string
ImageLinkFallback string
ImageTitle []string
}{
Captcha: "div[data-sitekey]",
ResultStats: "div#result-stats",
@@ -23,4 +29,14 @@ var Selectors = struct {
DescFallback: "div.VwiC3b",
AnswerBox: "div[data-hveid][data-ulkwtsb] div[data-q]",
AnswerItem: "a",
// ImageResults selects each image cell in the image SERP grid.
ImageResults: "div[data-hveid][data-ved][jsaction]",
// ImageLink: the canonical href of an image cell. The :not([ping])
// variant excludes Google's click-tracking hops; the imgres fallback is
// only present after the cell has been right-clicked to materialize.
ImageLink: "a:not([ping])",
ImageLinkFallback: "a[href*='imgres']",
// ImageTitle selectors are tried in order to recover a human-readable title.
ImageTitle: []string{"h3", "a"},
}

10
testutil/ithelper/ithelper.go
View File

@@ -1,6 +1,8 @@
package ithelper
import (
"context"
"errors"
"testing"
"time"
@@ -30,6 +32,12 @@ func HandleError(t *testing.T, operation string, err error) {
}
t.Skipf("skipping flaky live %s due to timeout: %v", operation, err)
}
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) || core.IsContextDone(err) {
if testutil.IntegrationStrict() {
t.Fatalf("%s failed (strict mode): %v", operation, err)
}
t.Skipf("skipping flaky live %s due to context deadline: %v", operation, err)
}
t.Fatalf("%s failed: %v", operation, err)
}
@@ -42,7 +50,7 @@ func CreateBrowser(t *testing.T) *core.Browser {
opts := core.BrowserOpts{
IsHeadless: !headful,
IsLeakless: false,
Timeout: time.Second * 15,
Timeout: time.Second * 30,
LeavePageOpen: headful,
}
b, err := core.NewBrowser(opts)

111
yandex/search.go
View File

@@ -36,7 +36,6 @@ type ImageData struct {
} `json:"initialState"`
}
// Yandex implements core.SearchEngine for Yandex SERP pages.
type Yandex struct {
core.Browser
@@ -124,6 +123,26 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
return searchResults
}
func (yand *Yandex) parseImageEntities(items rod.Elements) map[string]ImageEntity {
entities := make(map[string]ImageEntity)
for _, item := range items {
state, err := item.Attribute("data-state")
if err != nil || state == nil || *state == "" {
continue
}
var imgData ImageData
if err := json.Unmarshal([]byte(*state), &imgData); err != nil {
continue
}
for id, entity := range imgData.InitalState.SerpList.Items.Entities {
entities[id] = entity
}
}
return entities
}
// Search executes a Yandex web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -172,32 +191,21 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
}
}
// Get all search results in page
searchRes, err := page.Timeout(yand.Timeout).Search(Selectors.Results)
elements, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, yand.GetSelectorTimeout())
if err != nil {
closePage()
yand.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrParser
}
// Check why no results, maybe captcha?
if searchRes == nil {
closePage()
if yand.isNoResults(page) {
yand.logger.Warn("No results found")
} else if yand.isCaptcha(page) {
if yand.isCaptcha(page) {
yand.logger.Error("Captcha detected: %s", url)
closePage()
return nil, core.ErrCaptcha
}
break
}
elements, err := searchRes.All()
if err != nil {
yand.logger.Error("Cannot get search elements: %s", err)
if yand.isNoResults(page) {
yand.logger.Warn("No results found")
closePage()
break
}
closePage()
break
yand.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrSearchTimeout
}
r := yand.parseResults(elements, searchPage)
@@ -237,6 +245,7 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S
yand.logger.Debug("Starting image search, query: %+v", query)
searchResults := []core.SearchResult{}
allowPagination := query.Limit > 30
searchPage := 0
for len(searchResults) < query.Limit {
@@ -259,43 +268,45 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S
}
}
//page.Keyboard.Press(input.End)
//page.WaitLoad()
//time.Sleep(time.Duration(time.Second * 2))
results, err := page.Timeout(yand.Timeout).Search(Selectors.ImageItems)
results, _, err := core.WaitForElements(
ctx,
page,
append([]string{Selectors.ImageItems}, Selectors.ImageItemsAlt...),
yand.GetSelectorTimeout(),
)
if err != nil {
closePage()
yand.logger.Error("Cannot find search results: %s", err)
return searchResults, core.ErrParser
if allStateNodes, allErr := page.Elements(Selectors.ImageStateAll); allErr == nil && len(allStateNodes) > 0 {
results = allStateNodes
err = nil
}
}
// Check why no results
if results == nil {
closePage()
if err != nil {
if yand.isCaptcha(page) {
yand.logger.Error("Captcha detected: %s", url)
closePage()
return searchResults, core.ErrCaptcha
} else if yand.isNoResults(page) {
}
if yand.isNoResults(page) {
yand.logger.Warn("No results found")
}
closePage()
return searchResults, core.ErrSearchTimeout
}
data, err := results.First.Attribute("data-state")
if err != nil {
pageEntities := yand.parseImageEntities(results)
if len(pageEntities) == 0 {
allStateNodes, allErr := page.Elements(Selectors.ImageStateAll)
if allErr == nil && len(allStateNodes) > 0 {
pageEntities = yand.parseImageEntities(allStateNodes)
}
}
if len(pageEntities) == 0 {
closePage()
return nil, err
break
}
var imgData ImageData
if err := json.Unmarshal([]byte(*data), &imgData); err != nil {
closePage()
return nil, err
}
for id := range imgData.InitalState.SerpList.Items.Entities {
img := imgData.InitalState.SerpList.Items.Entities[id]
for id := range pageEntities {
img := pageEntities[id]
res := core.SearchResult{
Rank: img.Rank + 1,
URL: img.OrigURL,
@@ -305,6 +316,14 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S
searchResults = append(searchResults, res)
}
if len(searchResults) >= query.Limit {
closePage()
break
}
if searchPage == 1 && !allowPagination {
closePage()
break
}
closePage()
}

32
yandex/selectors.go
View File

@@ -2,19 +2,23 @@ package yandex
// Selectors is the single source of truth for Yandex SERP CSS selectors.
var Selectors = struct {
Captcha string
NoResults string
Results string
Link string
Title string
Desc string
ImageItems string
Captcha string
NoResults string
Results string
Link string
Title string
Desc string
ImageItems string
ImageItemsAlt []string
ImageStateAll string
}{
Captcha: "div.CheckboxCaptcha",
NoResults: "div.EmptySearchResults",
Results: "li[data-fast], li.serp-item",
Link: "a",
Title: "h2",
Desc: "span.OrganicTextContentSpan",
ImageItems: "div[role='main'] div[data-state]",
Captcha: "div.CheckboxCaptcha",
NoResults: "div.EmptySearchResults",
Results: "li[data-fast], li.serp-item",
Link: "a",
Title: "h2",
Desc: "span.OrganicTextContentSpan",
ImageItems: "div[role='main'] div[data-state]",
ImageItemsAlt: []string{"div[data-state*='serpList']"},
ImageStateAll: "div[data-state]",
}