zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-15 05:04:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

perf(browser): smart waits, fix captcha detection selectors + selectors single source of truth

Browse Source
This commit is contained in:
Rustem Kamalov
2026-04-23 22:04:28 +03:00
parent d052c30bd5
commit 24f9bdab30
9 changed files with 277 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
baidu/captcha_selector_test.go Normal file
View File

@@ -0,0 +1,58 @@
package baidu
import (
"testing"
"github.com/PuerkitoBio/goquery"
"github.com/karust/openserp/testutil"
)
// TestBaiduPageTypeSelectors verifies that the selectors defined in selectors.go
// match (or don't match) real fixture HTML without needing a browser.
func TestBaiduPageTypeSelectors(t *testing.T) {
t.Parallel()
tests := []struct {
fixture string
selector string
wantHit bool
}{
{"search_captcha.html", sel.Captcha, true},
{"search_captcha.html", sel.Timeout, true},
{"search_captcha.html", sel.Results, false},
{"search_results.html", sel.Results, true},
{"search_results.html", sel.Captcha, false},
{"search_results.html", sel.Timeout, false},
{"search_no_results.html", sel.Captcha, false},
{"search_no_results.html", sel.Timeout, false},
{"search_no_results.html", sel.Results, false},
}
for _, tt := range tests {
t.Run(tt.fixture+"/"+tt.selector, func(t *testing.T) {
t.Parallel()
assertSelector(t, tt.fixture, tt.selector, tt.wantHit)
})
}
}
func assertSelector(t *testing.T, fixture, selector string, wantHit bool) {
t.Helper()
resp := testutil.ResponseFromFixture(t, fixture)
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
t.Fatalf("parse fixture: %v", err)
}
got := doc.Find(selector).Length() > 0
if got != wantHit {
if wantHit {
t.Fatalf("selector %q not found in %s — update selectors.go", selector, fixture)
} else {
t.Fatalf("selector %q unexpectedly present in %s", selector, fixture)
}
}
}

20
baidu/search.go
View File

@@ -33,6 +33,16 @@ type imageDataJson struct {
}
}
var sel = struct {
Captcha string
Timeout string
Results string
}{
Captcha: "div.passMod_dialog-body",
Timeout: "button.timeout-button",
Results: "div.c-container.new-pmd",
}
// Baidu implements core.SearchEngine for Baidu SERP pages.
type Baidu struct {
core.Browser
@@ -61,13 +71,13 @@ func (baid *Baidu) GetRateLimiter() *rate.Limiter {
}
func (baid *Baidu) isCaptcha(page *rod.Page) bool {
_, err := page.Timeout(baid.GetSelectorTimeout()).Search("div.passMod_dialog-body")
return err == nil
has, _, _ := page.Has(sel.Captcha)
return has
}
func (baid *Baidu) isTimeout(page *rod.Page) bool {
_, err := page.Timeout(baid.GetSelectorTimeout()).Search("button.timeout-button")
return err == nil
has, _, _ := page.Has(sel.Timeout)
return has
}
// Search executes a Baidu web search and returns normalized search results.
@@ -109,7 +119,7 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
}
}
searchRes, err := page.Timeout(baid.Timeout).Search("div.c-container.new-pmd")
searchRes, err := page.Timeout(baid.Timeout).Search(sel.Results)
if err != nil {
closePage()
baid.logger.Error("Cannot parse search results: %s", err)

40
bing/search.go
View File

@@ -14,6 +14,20 @@ import (
"golang.org/x/time/rate"
)
var sel = struct {
Captcha []string
CookieBtn string
Results string
Ads string
ImageResults string
}{
Captcha: []string{"div.captcha", "div.captcha_header"},
CookieBtn: "button#bnp_btn_accept",
Results: "li.b_algo",
Ads: "li.b_ad",
ImageResults: "div.iuscp, div.isv",
}
// Bing implements core.SearchEngine for Bing SERP pages.
type Bing struct {
core.Browser
@@ -42,7 +56,7 @@ func (bing *Bing) GetRateLimiter() *rate.Limiter {
}
func (bing *Bing) getTotalResults(page *rod.Page) (int, error) {
results, err := page.Timeout(bing.GetSelectorTimeout()).Elements("li.b_algo")
results, err := page.Timeout(bing.GetSelectorTimeout()).Elements(sel.Results)
if err != nil {
return 0, errors.New("Cannot find result elements: " + err.Error())
}
@@ -61,19 +75,9 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool {
}
}
timeout := bing.GetSelectorTimeout() / 2
if timeout <= 0 {
timeout = time.Second * 2
}
selectors := []string{
"div.captcha",
"div.captcha_header",
}
for _, selector := range selectors {
has, err, _ := page.Timeout(timeout).Has(selector)
if err == nil && has {
for _, selector := range sel.Captcha {
has, _, _ := page.Has(selector)
if has {
bing.logger.Debug("Captcha detected: %s", selector)
return true
}
@@ -83,7 +87,7 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool {
}
func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
consentBtn, err := page.Timeout(bing.Timeout / 10).Element("button#bnp_btn_accept")
consentBtn, err := page.Timeout(bing.Timeout / 10).Element(sel.CookieBtn)
if err != nil {
return nil
}
@@ -150,13 +154,13 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
return nil, core.ErrSearchTimeout
}
organicElements, err := page.Timeout(bing.Timeout).Elements("li.b_algo")
organicElements, err := page.Timeout(bing.Timeout).Elements(sel.Results)
if err != nil {
bing.logger.Error("Cannot parse organic results: %s", err)
return nil, core.ErrParser
}
adElements, err := page.Timeout(bing.Timeout).Elements("li.b_ad")
adElements, err := page.Timeout(bing.Timeout).Elements(sel.Ads)
if err != nil {
bing.logger.Debug("No ads found")
}
@@ -330,7 +334,7 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea
}
// Find all image result containers using CSS selector
imageContainers, err := page.Timeout(bing.Timeout).Elements("div.iuscp, div.isv")
imageContainers, err := page.Timeout(bing.Timeout).Elements(sel.ImageResults)
if err != nil {
bing.logger.Error("Cannot parse image results: %s", err)
return nil, core.ErrSearchTimeout

10
core/browser.go
View File

@@ -35,7 +35,8 @@ type BrowserOpts struct {
WaitRequests bool
// LeavePageOpen keeps pages open after search operations.
LeavePageOpen bool
// WaitLoadTime is an additional fixed wait after load/idle checks.
// WaitLoadTime is kept for config backwards-compatibility but no longer used;
// Navigate now calls WaitStable instead.
WaitLoadTime time.Duration
// CaptchaSolverApiKey enables 2Captcha integration for supported engines.
CaptchaSolverApiKey string
@@ -702,9 +703,10 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
wait()
}
if err := SleepContext(ctx, b.WaitLoadTime); err != nil {
closeOnErr()
return nil, err
// WaitStable blocks until no layout changes occur for 800 ms, or falls
// through on error so engine-specific selector timeouts handle the result.
if err := page.Context(ctx).WaitStable(800 * time.Millisecond); err != nil {
WithRequest(ctx).WithError(err).Debug("WaitStable returned early; continuing")
}
return page, nil
}

23
duckduckgo/search.go
View File

@@ -11,6 +11,22 @@ import (
"golang.org/x/time/rate"
)
// captchaBodyText is matched against the raw page HTML because DDG returns
// a plain-text 202 rate-limit response rather than a structured captcha page.
const captchaBodyText = "bots user"
var sel = struct {
NoResults string
Results []string
}{
NoResults: "div[class*='no-results']",
Results: []string{
"article[data-testid='result']",
"div.result",
"div[data-testid='result']",
},
}
// DuckDuckGo implements core.SearchEngine for DuckDuckGo SERP pages.
type DuckDuckGo struct {
core.Browser
@@ -46,13 +62,12 @@ func (ddg *DuckDuckGo) isCaptcha(page *rod.Page) bool {
if err != nil {
return false
}
return strings.Contains(html, "bots user")
return strings.Contains(html, captchaBodyText)
}
// Check if no results are found
func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool {
_, err := page.Timeout(ddg.GetSelectorTimeout()).Search("div[class*='no-results']")
return err == nil
has, _, _ := page.Has(sel.NoResults)
return has
}
func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.SearchResult {

55
google/captcha_selector_test.go Normal file
View File

@@ -0,0 +1,55 @@
package google
import (
"testing"
"github.com/PuerkitoBio/goquery"
"github.com/karust/openserp/testutil"
)
// TestGooglePageTypeSelectors verifies that the selectors defined in selectors.go
// match (or don't match) real fixture HTML without needing a browser.
func TestGooglePageTypeSelectors(t *testing.T) {
t.Parallel()
tests := []struct {
fixture string
selector string
wantHit bool
}{
{"search_captcha.html", sel.Captcha, true},
{"search_captcha.html", sel.ResultStats, false},
{"search_results.html", sel.ResultStats, true},
{"search_results.html", sel.Captcha, false},
{"search_no_results.html", sel.ResultStats, true},
{"search_no_results.html", sel.Captcha, false},
}
for _, tt := range tests {
t.Run(tt.fixture+"/"+tt.selector, func(t *testing.T) {
t.Parallel()
assertSelector(t, tt.fixture, tt.selector, tt.wantHit)
})
}
}
func assertSelector(t *testing.T, fixture, selector string, wantHit bool) {
t.Helper()
resp := testutil.ResponseFromFixture(t, fixture)
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
t.Fatalf("parse fixture: %v", err)
}
got := doc.Find(selector).Length() > 0
if got != wantHit {
if wantHit {
t.Fatalf("selector %q not found in %s — update selectors.go", selector, fixture)
} else {
t.Fatalf("selector %q unexpectedly present in %s", selector, fixture)
}
}
}

35
google/search.go
View File

@@ -15,6 +15,18 @@ import (
"golang.org/x/time/rate"
)
var sel = struct {
Captcha string
ResultStats string
CookieBtn string
Results string
}{
Captcha: "div[data-sitekey]",
ResultStats: "div#result-stats",
CookieBtn: "div[role='dialog'][aria-modal] button",
Results: "div[data-hveid][data-ved]",
}
// Google implements core.SearchEngine for Google SERP pages.
type Google struct {
core.Browser
@@ -49,7 +61,7 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) {
return 0, core.ErrParser
}
resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search("div#result-stats")
resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search(sel.ResultStats)
if err != nil {
return 0, errors.New("Result stats not found: " + err.Error())
}
@@ -112,21 +124,26 @@ func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas, proxyURL string
}
func (gogl *Google) checkCaptcha(page *rod.Page, queryProxyURL string) bool {
captchaDiv, err := page.Timeout(gogl.GetSelectorTimeout()).Search("div[data-sitekey]")
if err != nil {
has, _, _ := page.Has(sel.Captcha)
if !has {
return false
}
sitekey, err := captchaDiv.First.Attribute("data-sitekey")
captchaDiv, err := page.Element(sel.Captcha)
if err != nil {
return true
}
sitekey, err := captchaDiv.Attribute("data-sitekey")
if err != nil {
gogl.logger.Error("Cannot get captcha sitekey: %s", err)
return false
return true
}
dataS, err := captchaDiv.First.Attribute("data-s")
dataS, err := captchaDiv.Attribute("data-s")
if err != nil {
gogl.logger.Error("Cannot get captcha datas: %s", err)
return false
return true
}
if gogl.IsSolveCaptcha && gogl.CaptchaSolverEnabled {
@@ -150,7 +167,7 @@ func (gogl *Google) preparePage(page *rod.Page) {
}
func (gogl *Google) acceptCookies(page *rod.Page) {
diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search("div[role='dialog'][aria-modal] button")
diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search(sel.CookieBtn)
if err != nil {
gogl.logger.Debug("Cookie consent not found: %s", err)
return
@@ -215,7 +232,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
}
// Find all results using stable attributes
searchRes, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved]")
searchRes, err := page.Timeout(gogl.Timeout).Search(sel.Results)
if err != nil {
gogl.logger.Error("Cannot parse search results: %s", err)
return nil, core.ErrParser

58
yandex/captcha_selector_test.go Normal file
View File

@@ -0,0 +1,58 @@
package yandex
import (
"testing"
"github.com/PuerkitoBio/goquery"
"github.com/karust/openserp/testutil"
)
// TestYandexPageTypeSelectors verifies that the selectors defined in selectors.go
// match (or don't match) real fixture HTML without needing a browser.
func TestYandexPageTypeSelectors(t *testing.T) {
t.Parallel()
tests := []struct {
fixture string
selector string
wantHit bool
}{
{"search_captcha.html", sel.Captcha, true},
{"search_captcha.html", sel.NoResults, false},
{"search_captcha.html", "li[data-fast]", false},
{"search_results.html", "li[data-fast]", true},
{"search_results.html", sel.Captcha, false},
{"search_results.html", sel.NoResults, false},
{"search_no_results.html", sel.NoResults, true},
{"search_no_results.html", sel.Captcha, false},
{"search_no_results.html", "li[data-fast]", false},
}
for _, tt := range tests {
t.Run(tt.fixture+"/"+tt.selector, func(t *testing.T) {
t.Parallel()
assertSelector(t, tt.fixture, tt.selector, tt.wantHit)
})
}
}
func assertSelector(t *testing.T, fixture, selector string, wantHit bool) {
t.Helper()
resp := testutil.ResponseFromFixture(t, fixture)
doc, err := goquery.NewDocumentFromReader(resp.Body)
if err != nil {
t.Fatalf("parse fixture: %v", err)
}
got := doc.Find(selector).Length() > 0
if got != wantHit {
if wantHit {
t.Fatalf("selector %q not found in %s — update selectors.go", selector, fixture)
} else {
t.Fatalf("selector %q unexpectedly present in %s", selector, fixture)
}
}
}

25
yandex/search.go
View File

@@ -36,6 +36,18 @@ type ImageData struct {
} `json:"initialState"`
}
var sel = struct {
Captcha string
NoResults string
Results string
ImageItems string
}{
Captcha: "div.CheckboxCaptcha",
NoResults: "div.EmptySearchResults",
Results: "li[data-fast], li.serp-item",
ImageItems: "div[role='main'] div[data-state]",
}
// Yandex implements core.SearchEngine for Yandex SERP pages.
type Yandex struct {
core.Browser
@@ -67,14 +79,13 @@ func (yand *Yandex) GetRateLimiter() *rate.Limiter {
}
func (yand *Yandex) isCaptcha(page *rod.Page) bool {
_, err := page.Timeout(yand.GetSelectorTimeout()).Search("form#checkbox-captcha-form")
return err == nil
has, _, _ := page.Has(sel.Captcha)
return has
}
// Check if nothig is found
func (yand *Yandex) isNoResults(page *rod.Page) bool {
_, err := page.Timeout(yand.GetSelectorTimeout()).Search("div.Correction.SearchCorrection")
return err == nil
has, _, _ := page.Has(sel.NoResults)
return has
}
func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
@@ -173,7 +184,7 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
}
// Get all search results in page
searchRes, err := page.Timeout(yand.Timeout).Search("li.serp-item")
searchRes, err := page.Timeout(yand.Timeout).Search(sel.Results)
if err != nil {
closePage()
yand.logger.Error("Cannot parse search results: %s", err)
@@ -263,7 +274,7 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S
//page.WaitLoad()
//time.Sleep(time.Duration(time.Second * 2))
results, err := page.Timeout(yand.Timeout).Search("div[role='main'] div[data-state]")
results, err := page.Timeout(yand.Timeout).Search(sel.ImageItems)
if err != nil {
closePage()
yand.logger.Error("Cannot find search results: %s", err)