From 24f9bdab30cd9dcb885ded6ad53df74b8e858e80 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Thu, 23 Apr 2026 22:04:28 +0300 Subject: [PATCH] perf(browser): smart waits, fix captcha detection selectors + selectors single source of truth --- baidu/captcha_selector_test.go | 58 +++++++++++++++++++++++++++++++++ baidu/search.go | 20 +++++++++--- bing/search.go | 40 +++++++++++++---------- core/browser.go | 10 +++--- duckduckgo/search.go | 23 ++++++++++--- google/captcha_selector_test.go | 55 +++++++++++++++++++++++++++++++ google/search.go | 35 +++++++++++++++----- yandex/captcha_selector_test.go | 58 +++++++++++++++++++++++++++++++++ yandex/search.go | 25 ++++++++++---- 9 files changed, 277 insertions(+), 47 deletions(-) create mode 100644 baidu/captcha_selector_test.go create mode 100644 google/captcha_selector_test.go create mode 100644 yandex/captcha_selector_test.go diff --git a/baidu/captcha_selector_test.go b/baidu/captcha_selector_test.go new file mode 100644 index 0000000..d08885d --- /dev/null +++ b/baidu/captcha_selector_test.go @@ -0,0 +1,58 @@ +package baidu + +import ( + "testing" + + "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/testutil" +) + +// TestBaiduPageTypeSelectors verifies that the selectors defined in selectors.go +// match (or don't match) real fixture HTML without needing a browser. +func TestBaiduPageTypeSelectors(t *testing.T) { + t.Parallel() + + tests := []struct { + fixture string + selector string + wantHit bool + }{ + {"search_captcha.html", sel.Captcha, true}, + {"search_captcha.html", sel.Timeout, true}, + {"search_captcha.html", sel.Results, false}, + + {"search_results.html", sel.Results, true}, + {"search_results.html", sel.Captcha, false}, + {"search_results.html", sel.Timeout, false}, + + {"search_no_results.html", sel.Captcha, false}, + {"search_no_results.html", sel.Timeout, false}, + {"search_no_results.html", sel.Results, false}, + } + + for _, tt := range tests { + t.Run(tt.fixture+"/"+tt.selector, func(t *testing.T) { + t.Parallel() + assertSelector(t, tt.fixture, tt.selector, tt.wantHit) + }) + } +} + +func assertSelector(t *testing.T, fixture, selector string, wantHit bool) { + t.Helper() + + resp := testutil.ResponseFromFixture(t, fixture) + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + t.Fatalf("parse fixture: %v", err) + } + + got := doc.Find(selector).Length() > 0 + if got != wantHit { + if wantHit { + t.Fatalf("selector %q not found in %s — update selectors.go", selector, fixture) + } else { + t.Fatalf("selector %q unexpectedly present in %s", selector, fixture) + } + } +} diff --git a/baidu/search.go b/baidu/search.go index 6db5537..a363d9e 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -33,6 +33,16 @@ type imageDataJson struct { } } +var sel = struct { + Captcha string + Timeout string + Results string +}{ + Captcha: "div.passMod_dialog-body", + Timeout: "button.timeout-button", + Results: "div.c-container.new-pmd", +} + // Baidu implements core.SearchEngine for Baidu SERP pages. type Baidu struct { core.Browser @@ -61,13 +71,13 @@ func (baid *Baidu) GetRateLimiter() *rate.Limiter { } func (baid *Baidu) isCaptcha(page *rod.Page) bool { - _, err := page.Timeout(baid.GetSelectorTimeout()).Search("div.passMod_dialog-body") - return err == nil + has, _, _ := page.Has(sel.Captcha) + return has } func (baid *Baidu) isTimeout(page *rod.Page) bool { - _, err := page.Timeout(baid.GetSelectorTimeout()).Search("button.timeout-button") - return err == nil + has, _, _ := page.Has(sel.Timeout) + return has } // Search executes a Baidu web search and returns normalized search results. @@ -109,7 +119,7 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core } } - searchRes, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd") + searchRes, err := page.Timeout(baid.Timeout).Search(sel.Results) if err != nil { closePage() baid.logger.Error("Cannot parse search results: %s", err) diff --git a/bing/search.go b/bing/search.go index 392b636..643099a 100644 --- a/bing/search.go +++ b/bing/search.go @@ -14,6 +14,20 @@ import ( "golang.org/x/time/rate" ) +var sel = struct { + Captcha []string + CookieBtn string + Results string + Ads string + ImageResults string +}{ + Captcha: []string{"div.captcha", "div.captcha_header"}, + CookieBtn: "button#bnp_btn_accept", + Results: "li.b_algo", + Ads: "li.b_ad", + ImageResults: "div.iuscp, div.isv", +} + // Bing implements core.SearchEngine for Bing SERP pages. type Bing struct { core.Browser @@ -42,7 +56,7 @@ func (bing *Bing) GetRateLimiter() *rate.Limiter { } func (bing *Bing) getTotalResults(page *rod.Page) (int, error) { - results, err := page.Timeout(bing.GetSelectorTimeout()).Elements("li.b_algo") + results, err := page.Timeout(bing.GetSelectorTimeout()).Elements(sel.Results) if err != nil { return 0, errors.New("Cannot find result elements: " + err.Error()) } @@ -61,19 +75,9 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool { } } - timeout := bing.GetSelectorTimeout() / 2 - if timeout <= 0 { - timeout = time.Second * 2 - } - - selectors := []string{ - "div.captcha", - "div.captcha_header", - } - - for _, selector := range selectors { - has, err, _ := page.Timeout(timeout).Has(selector) - if err == nil && has { + for _, selector := range sel.Captcha { + has, _, _ := page.Has(selector) + if has { bing.logger.Debug("Captcha detected: %s", selector) return true } @@ -83,7 +87,7 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool { } func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error { - consentBtn, err := page.Timeout(bing.Timeout / 10).Element("button#bnp_btn_accept") + consentBtn, err := page.Timeout(bing.Timeout / 10).Element(sel.CookieBtn) if err != nil { return nil } @@ -150,13 +154,13 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. return nil, core.ErrSearchTimeout } - organicElements, err := page.Timeout(bing.Timeout).Elements("li.b_algo") + organicElements, err := page.Timeout(bing.Timeout).Elements(sel.Results) if err != nil { bing.logger.Error("Cannot parse organic results: %s", err) return nil, core.ErrParser } - adElements, err := page.Timeout(bing.Timeout).Elements("li.b_ad") + adElements, err := page.Timeout(bing.Timeout).Elements(sel.Ads) if err != nil { bing.logger.Debug("No ads found") } @@ -330,7 +334,7 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea } // Find all image result containers using CSS selector - imageContainers, err := page.Timeout(bing.Timeout).Elements("div.iuscp, div.isv") + imageContainers, err := page.Timeout(bing.Timeout).Elements(sel.ImageResults) if err != nil { bing.logger.Error("Cannot parse image results: %s", err) return nil, core.ErrSearchTimeout diff --git a/core/browser.go b/core/browser.go index 4f73a33..6116d29 100644 --- a/core/browser.go +++ b/core/browser.go @@ -35,7 +35,8 @@ type BrowserOpts struct { WaitRequests bool // LeavePageOpen keeps pages open after search operations. LeavePageOpen bool - // WaitLoadTime is an additional fixed wait after load/idle checks. + // WaitLoadTime is kept for config backwards-compatibility but no longer used; + // Navigate now calls WaitStable instead. WaitLoadTime time.Duration // CaptchaSolverApiKey enables 2Captcha integration for supported engines. CaptchaSolverApiKey string @@ -702,9 +703,10 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) { wait() } - if err := SleepContext(ctx, b.WaitLoadTime); err != nil { - closeOnErr() - return nil, err + // WaitStable blocks until no layout changes occur for 800 ms, or falls + // through on error so engine-specific selector timeouts handle the result. + if err := page.Context(ctx).WaitStable(800 * time.Millisecond); err != nil { + WithRequest(ctx).WithError(err).Debug("WaitStable returned early; continuing") } return page, nil } diff --git a/duckduckgo/search.go b/duckduckgo/search.go index 082276b..d0645e6 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -11,6 +11,22 @@ import ( "golang.org/x/time/rate" ) +// captchaBodyText is matched against the raw page HTML because DDG returns +// a plain-text 202 rate-limit response rather than a structured captcha page. +const captchaBodyText = "bots user" + +var sel = struct { + NoResults string + Results []string +}{ + NoResults: "div[class*='no-results']", + Results: []string{ + "article[data-testid='result']", + "div.result", + "div[data-testid='result']", + }, +} + // DuckDuckGo implements core.SearchEngine for DuckDuckGo SERP pages. type DuckDuckGo struct { core.Browser @@ -46,13 +62,12 @@ func (ddg *DuckDuckGo) isCaptcha(page *rod.Page) bool { if err != nil { return false } - return strings.Contains(html, "bots user") + return strings.Contains(html, captchaBodyText) } -// Check if no results are found func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool { - _, err := page.Timeout(ddg.GetSelectorTimeout()).Search("div[class*='no-results']") - return err == nil + has, _, _ := page.Has(sel.NoResults) + return has } func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.SearchResult { diff --git a/google/captcha_selector_test.go b/google/captcha_selector_test.go new file mode 100644 index 0000000..4704446 --- /dev/null +++ b/google/captcha_selector_test.go @@ -0,0 +1,55 @@ +package google + +import ( + "testing" + + "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/testutil" +) + +// TestGooglePageTypeSelectors verifies that the selectors defined in selectors.go +// match (or don't match) real fixture HTML without needing a browser. +func TestGooglePageTypeSelectors(t *testing.T) { + t.Parallel() + + tests := []struct { + fixture string + selector string + wantHit bool + }{ + {"search_captcha.html", sel.Captcha, true}, + {"search_captcha.html", sel.ResultStats, false}, + + {"search_results.html", sel.ResultStats, true}, + {"search_results.html", sel.Captcha, false}, + + {"search_no_results.html", sel.ResultStats, true}, + {"search_no_results.html", sel.Captcha, false}, + } + + for _, tt := range tests { + t.Run(tt.fixture+"/"+tt.selector, func(t *testing.T) { + t.Parallel() + assertSelector(t, tt.fixture, tt.selector, tt.wantHit) + }) + } +} + +func assertSelector(t *testing.T, fixture, selector string, wantHit bool) { + t.Helper() + + resp := testutil.ResponseFromFixture(t, fixture) + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + t.Fatalf("parse fixture: %v", err) + } + + got := doc.Find(selector).Length() > 0 + if got != wantHit { + if wantHit { + t.Fatalf("selector %q not found in %s — update selectors.go", selector, fixture) + } else { + t.Fatalf("selector %q unexpectedly present in %s", selector, fixture) + } + } +} diff --git a/google/search.go b/google/search.go index a7231bd..a8b33be 100644 --- a/google/search.go +++ b/google/search.go @@ -15,6 +15,18 @@ import ( "golang.org/x/time/rate" ) +var sel = struct { + Captcha string + ResultStats string + CookieBtn string + Results string +}{ + Captcha: "div[data-sitekey]", + ResultStats: "div#result-stats", + CookieBtn: "div[role='dialog'][aria-modal] button", + Results: "div[data-hveid][data-ved]", +} + // Google implements core.SearchEngine for Google SERP pages. type Google struct { core.Browser @@ -49,7 +61,7 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) { return 0, core.ErrParser } - resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search("div#result-stats") + resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search(sel.ResultStats) if err != nil { return 0, errors.New("Result stats not found: " + err.Error()) } @@ -112,21 +124,26 @@ func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas, proxyURL string } func (gogl *Google) checkCaptcha(page *rod.Page, queryProxyURL string) bool { - captchaDiv, err := page.Timeout(gogl.GetSelectorTimeout()).Search("div[data-sitekey]") - if err != nil { + has, _, _ := page.Has(sel.Captcha) + if !has { return false } - sitekey, err := captchaDiv.First.Attribute("data-sitekey") + captchaDiv, err := page.Element(sel.Captcha) + if err != nil { + return true + } + + sitekey, err := captchaDiv.Attribute("data-sitekey") if err != nil { gogl.logger.Error("Cannot get captcha sitekey: %s", err) - return false + return true } - dataS, err := captchaDiv.First.Attribute("data-s") + dataS, err := captchaDiv.Attribute("data-s") if err != nil { gogl.logger.Error("Cannot get captcha datas: %s", err) - return false + return true } if gogl.IsSolveCaptcha && gogl.CaptchaSolverEnabled { @@ -150,7 +167,7 @@ func (gogl *Google) preparePage(page *rod.Page) { } func (gogl *Google) acceptCookies(page *rod.Page) { - diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search("div[role='dialog'][aria-modal] button") + diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search(sel.CookieBtn) if err != nil { gogl.logger.Debug("Cookie consent not found: %s", err) return @@ -215,7 +232,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } // Find all results using stable attributes - searchRes, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved]") + searchRes, err := page.Timeout(gogl.Timeout).Search(sel.Results) if err != nil { gogl.logger.Error("Cannot parse search results: %s", err) return nil, core.ErrParser diff --git a/yandex/captcha_selector_test.go b/yandex/captcha_selector_test.go new file mode 100644 index 0000000..f833a46 --- /dev/null +++ b/yandex/captcha_selector_test.go @@ -0,0 +1,58 @@ +package yandex + +import ( + "testing" + + "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/testutil" +) + +// TestYandexPageTypeSelectors verifies that the selectors defined in selectors.go +// match (or don't match) real fixture HTML without needing a browser. +func TestYandexPageTypeSelectors(t *testing.T) { + t.Parallel() + + tests := []struct { + fixture string + selector string + wantHit bool + }{ + {"search_captcha.html", sel.Captcha, true}, + {"search_captcha.html", sel.NoResults, false}, + {"search_captcha.html", "li[data-fast]", false}, + + {"search_results.html", "li[data-fast]", true}, + {"search_results.html", sel.Captcha, false}, + {"search_results.html", sel.NoResults, false}, + + {"search_no_results.html", sel.NoResults, true}, + {"search_no_results.html", sel.Captcha, false}, + {"search_no_results.html", "li[data-fast]", false}, + } + + for _, tt := range tests { + t.Run(tt.fixture+"/"+tt.selector, func(t *testing.T) { + t.Parallel() + assertSelector(t, tt.fixture, tt.selector, tt.wantHit) + }) + } +} + +func assertSelector(t *testing.T, fixture, selector string, wantHit bool) { + t.Helper() + + resp := testutil.ResponseFromFixture(t, fixture) + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + t.Fatalf("parse fixture: %v", err) + } + + got := doc.Find(selector).Length() > 0 + if got != wantHit { + if wantHit { + t.Fatalf("selector %q not found in %s — update selectors.go", selector, fixture) + } else { + t.Fatalf("selector %q unexpectedly present in %s", selector, fixture) + } + } +} diff --git a/yandex/search.go b/yandex/search.go index 2756a4f..1800681 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -36,6 +36,18 @@ type ImageData struct { } `json:"initialState"` } +var sel = struct { + Captcha string + NoResults string + Results string + ImageItems string +}{ + Captcha: "div.CheckboxCaptcha", + NoResults: "div.EmptySearchResults", + Results: "li[data-fast], li.serp-item", + ImageItems: "div[role='main'] div[data-state]", +} + // Yandex implements core.SearchEngine for Yandex SERP pages. type Yandex struct { core.Browser @@ -67,14 +79,13 @@ func (yand *Yandex) GetRateLimiter() *rate.Limiter { } func (yand *Yandex) isCaptcha(page *rod.Page) bool { - _, err := page.Timeout(yand.GetSelectorTimeout()).Search("form#checkbox-captcha-form") - return err == nil + has, _, _ := page.Has(sel.Captcha) + return has } -// Check if nothig is found func (yand *Yandex) isNoResults(page *rod.Page) bool { - _, err := page.Timeout(yand.GetSelectorTimeout()).Search("div.Correction.SearchCorrection") - return err == nil + has, _, _ := page.Has(sel.NoResults) + return has } func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.SearchResult { @@ -173,7 +184,7 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor } // Get all search results in page - searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item") + searchRes, err := page.Timeout(yand.Timeout).Search(sel.Results) if err != nil { closePage() yand.logger.Error("Cannot parse search results: %s", err) @@ -263,7 +274,7 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S //page.WaitLoad() //time.Sleep(time.Duration(time.Second * 2)) - results, err := page.Timeout(yand.Timeout).Search("div[role='main'] div[data-state]") + results, err := page.Timeout(yand.Timeout).Search(sel.ImageItems) if err != nil { closePage() yand.logger.Error("Cannot find search results: %s", err)