From 3d5eddfeed3e7689e3cd4965540494ed476994ec Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Wed, 1 Jul 2026 01:38:03 +0300 Subject: [PATCH] fix(engines): better classify captcha/blocked/no-results pages instead of returning empty results, also enable it to `/parse` endpoints. Enable `allow_request_proxy_url` by default --- .gitignore | 1 + baidu/parse_html.go | 15 ++++++ baidu/search.go | 5 ++ baidu/search_raw.go | 8 +-- baidu/search_raw_test.go | 18 ++++--- baidu/selectors.go | 2 + bing/parse_html.go | 16 ++++++ bing/parse_html_test.go | 26 +++++++++ bing/search.go | 31 ++++++++++- bing/selectors.go | 40 +++++++++----- cmd/root.go | 2 +- config.yaml | 2 +- core/browser.go | 14 ++++- core/page_helpers.go | 50 ++++++++++++++++++ core/server.go | 20 +++++++ core/server_parse_test.go | 23 ++++++++ duckduckgo/parse_html.go | 16 ++++++ duckduckgo/parse_html_test.go | 26 +++++++++ ecosia/parse_html.go | 21 ++++++++ ecosia/search_raw.go | 26 +++------ ecosia/search_raw_test.go | 32 +++++++++++ ecosia/selectors.go | 2 + google/captcha_selector_test.go | 13 +++++ google/parse_html_test.go | 36 +++++++++++++ google/search.go | 70 ++++++++++++++++++++++--- google/search_raw.go | 62 ++++++++++++++++++++-- google/search_raw_test.go | 25 +++++++-- google/selectors.go | 42 ++++++++++----- google/testdata/search_captcha_new.html | 1 + google/testdata/search_soft_block.html | 1 + yandex/parse_html.go | 15 ++++++ yandex/search_raw.go | 8 +-- yandex/search_raw_test.go | 18 ++++--- yandex/url.go | 8 ++- yandex/url_test.go | 14 +++-- 35 files changed, 605 insertions(+), 104 deletions(-) create mode 100644 google/testdata/search_captcha_new.html create mode 100644 google/testdata/search_soft_block.html diff --git a/.gitignore b/.gitignore index 55be0d2..99adb03 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ core/testdata/* .tmp-gocache .tmpcache/ google/data/geotargets-2026-05-28.csv +demo.yml diff --git a/baidu/parse_html.go b/baidu/parse_html.go index caa78b4..fd86160 100644 --- a/baidu/parse_html.go +++ b/baidu/parse_html.go @@ -1,6 +1,7 @@ package baidu import ( + "errors" "io" "strings" @@ -15,9 +16,23 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { if err != nil { return nil, err } + pageStatus := classifyBaiduDocument(doc) + if errors.Is(pageStatus, core.ErrEmptyResult) { + return []core.SearchResult{}, nil + } + if pageStatus != nil { + return nil, pageStatus + } return parseBaiduDocument(doc), nil } +func classifyBaiduDocument(doc *goquery.Document) error { + return core.ClassifyChallengeDocument(doc, core.DocSignals{ + CaptchaSelectors: []string{Selectors.Captcha, Selectors.Timeout}, + EmptySelectors: []string{Selectors.NoResults}, + }) +} + func parseBaiduDocument(doc *goquery.Document) []core.SearchResult { features := extractBaiduFeatures(doc) // Match all result-card variants in one pass so DOM order is preserved and diff --git a/baidu/search.go b/baidu/search.go index 35c271e..d1114f4 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -3,6 +3,7 @@ package baidu import ( "context" "encoding/json" + "errors" "fmt" "regexp" "strings" @@ -134,6 +135,10 @@ func (baid *Baidu) waitForParsedSearchResults(ctx context.Context, page *rod.Pag if parseErr == nil && len(results) > 0 { return results, nil } + if errors.Is(parseErr, core.ErrCaptcha) { + baid.logger.Error("Captcha detected: %s", url) + return nil, core.ErrCaptcha + } lastErr = parseErr } else { lastErr = err diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 2986869..78c986d 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -15,13 +15,7 @@ func classifyBaiduRawHTML(body []byte) error { if err != nil { return err } - if doc.Find(Selectors.Captcha).Length() > 0 || doc.Find(Selectors.Timeout).Length() > 0 { - return core.ErrCaptcha - } - if doc.Find("div.content_none, div.nors").Length() > 0 { - return core.ErrEmptyResult - } - return nil + return classifyBaiduDocument(doc) } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { diff --git a/baidu/search_raw_test.go b/baidu/search_raw_test.go index 2d1bb49..4525402 100644 --- a/baidu/search_raw_test.go +++ b/baidu/search_raw_test.go @@ -17,12 +17,12 @@ func TestBaiduParseHTMLFixtures(t *testing.T) { t.Parallel() tests := []struct { - name string - fixture string - wantZero bool + name string + fixture string + wantErr error }{ - {name: "no results", fixture: "search_no_results.html", wantZero: true}, - {name: "captcha page", fixture: "search_captcha.html", wantZero: true}, + {name: "no results", fixture: "search_no_results.html"}, + {name: "captcha page", fixture: "search_captcha.html", wantErr: core.ErrCaptcha}, } for _, tt := range tests { @@ -30,10 +30,16 @@ func TestBaiduParseHTMLFixtures(t *testing.T) { t.Parallel() results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("expected %v for %s, got %v", tt.wantErr, tt.fixture, err) + } + return + } if err != nil { t.Fatalf("ParseHTML() error = %v", err) } - if tt.wantZero && len(results) != 0 { + if len(results) != 0 { t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results)) } }) diff --git a/baidu/selectors.go b/baidu/selectors.go index f9628a0..6e4f365 100644 --- a/baidu/selectors.go +++ b/baidu/selectors.go @@ -4,6 +4,7 @@ package baidu var Selectors = struct { Captcha string Timeout string + NoResults string Results string ResultsAlt []string AdMarkers []string @@ -16,6 +17,7 @@ var Selectors = struct { }{ Captcha: "div.passMod_dialog-wrapper", Timeout: "button.timeout-button", + NoResults: "div.content_none, div.nors", Results: "#content_left div.result.c-container", ResultsAlt: []string{"#content_left div.result-op.c-container", "div.c-container.new-pmd"}, AdMarkers: []string{"[data-tuiguang]", "[data-click*='tuiguang']", ".ec-tuiguang", ".c-icon-bear-p"}, diff --git a/bing/parse_html.go b/bing/parse_html.go index 32ac356..9ddb574 100644 --- a/bing/parse_html.go +++ b/bing/parse_html.go @@ -1,6 +1,7 @@ package bing import ( + "errors" "io" "strings" @@ -16,9 +17,24 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { if err != nil { return nil, err } + pageStatus := classifyBingDocument(doc) + if errors.Is(pageStatus, core.ErrEmptyResult) { + return []core.SearchResult{}, nil + } + if pageStatus != nil { + return nil, pageStatus + } return parseBingDocument(doc), nil } +func classifyBingDocument(doc *goquery.Document) error { + return core.ClassifyChallengeDocument(doc, core.DocSignals{ + CaptchaSelectors: Selectors.Captcha, + CaptchaMarkers: Selectors.CaptchaMarkers, + EmptyMarkers: Selectors.NoResultsMarkers, + }) +} + func parseBingDocument(doc *goquery.Document) []core.SearchResult { var results []core.SearchResult rank := core.NewRankState(0) diff --git a/bing/parse_html_test.go b/bing/parse_html_test.go index c55dab3..2c4197b 100644 --- a/bing/parse_html_test.go +++ b/bing/parse_html_test.go @@ -2,9 +2,12 @@ package bing import ( "bytes" + "errors" "os" "strings" "testing" + + "github.com/karust/openserp/core" ) func TestParseBingHTML(t *testing.T) { @@ -56,6 +59,29 @@ func TestParseBingHTMLEmpty(t *testing.T) { } } +func TestParseBingHTMLCaptcha(t *testing.T) { + t.Parallel() + + html := `
Enter the characters you see
` + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if !errors.Is(err, core.ErrCaptcha) { + t.Fatalf("expected ErrCaptcha, got results=%d err=%v", len(results), err) + } +} + +func TestParseBingHTMLNoResults(t *testing.T) { + t.Parallel() + + html := `
There are no results for this search.
` + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results, got %d", len(results)) + } +} + func TestParseBingHTMLAds(t *testing.T) { t.Parallel() diff --git a/bing/search.go b/bing/search.go index b0b1df4..c6fdc85 100644 --- a/bing/search.go +++ b/bing/search.go @@ -42,6 +42,8 @@ func (bing *Bing) getTotalResults(page *rod.Page) (int, error) { return len(results), nil } +// checkCaptcha classifies the live page the same way classifyBingDocument +// (parse_html.go) classifies raw HTML, so /bing/search and /bing/parse agree. func (bing *Bing) checkCaptcha(page *rod.Page) bool { if page == nil { return false @@ -62,6 +64,29 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool { } } + return pageTextContainsAny(page, Selectors.CaptchaMarkers) +} + +// checkNoResults reports whether the page text matches Bing's no-results +// phrasing, mirroring classifyBingDocument's text-marker check. +func (bing *Bing) checkNoResults(page *rod.Page) bool { + return pageTextContainsAny(page, Selectors.NoResultsMarkers) +} + +func pageTextContainsAny(page *rod.Page, markers []string) bool { + if page == nil || len(markers) == 0 { + return false + } + html, err := page.HTML() + if err != nil { + return false + } + text := strings.ToLower(html) + for _, marker := range markers { + if strings.Contains(text, marker) { + return true + } + } return false } @@ -161,11 +186,15 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. resultElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.ResultItems, Selectors.Results}, bing.GetSelectorTimeout()) if err != nil { - // Re-check captcha on timeout - Bing interstitials can render after WaitLoad. + // Re-check captcha/no-results on timeout - Bing interstitials and + // no-results pages can both render after WaitLoad. if bing.checkCaptcha(page) { bing.logger.Error("Captcha detected: %s", url) return nil, core.ErrCaptcha } + if bing.checkNoResults(page) { + return []core.SearchResult{}, nil + } bing.logger.Error("Cannot parse organic results: %s", err) return nil, core.ErrSearchTimeout } diff --git a/bing/selectors.go b/bing/selectors.go index 7b6d5cd..22946ca 100644 --- a/bing/selectors.go +++ b/bing/selectors.go @@ -2,21 +2,33 @@ package bing // Selectors is the single source of truth for Bing SERP CSS selectors. var Selectors = struct { - Captcha []string - CookieBtn string - ResultItems string - Results string - Ads string - ImageResults string - Title string - TitleFallbacks []string - DescPrimary string - DescFallback string - DescAny string - AdTitle string + Captcha []string + CaptchaMarkers []string + NoResultsMarkers []string + CookieBtn string + ResultItems string + Results string + Ads string + ImageResults string + Title string + TitleFallbacks []string + DescPrimary string + DescFallback string + DescAny string + AdTitle string }{ - Captcha: []string{"div.captcha", "div.captcha_header"}, - CookieBtn: "button#bnp_btn_accept", + Captcha: []string{"div.captcha", "div.captcha_header"}, + // CaptchaMarkers/NoResultsMarkers are checked against lowercased page text + // as a fallback when the CSS selectors above don't match a challenge page. + CaptchaMarkers: []string{ + "verify that you are not a robot", + "enter the characters you see", + }, + NoResultsMarkers: []string{ + "there are no results for", + "no results found for", + }, + CookieBtn: "button#bnp_btn_accept", // ResultItems matches the main-column children only, so carousels and // "related searches" cards that reuse b_algo-style markup are excluded. ResultItems: "#b_results > li.b_algo, #b_results > li.b_ad", diff --git a/cmd/root.go b/cmd/root.go index c30b604..133db56 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ import ( ) const ( - version = "0.8.6" + version = "0.8.7" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/config.yaml b/config.yaml index 27e4639..1f6ad8d 100644 --- a/config.yaml +++ b/config.yaml @@ -27,7 +27,7 @@ extract: max_concurrent: 2 proxies: - allow_request_proxy_url: false + allow_request_proxy_url: true # Force a single proxy for all engines. # Same behavior as passing --proxy on the CLI. #global: http://127.0.0.1:8080 diff --git a/core/browser.go b/core/browser.go index 6f137c1..c412e9c 100644 --- a/core/browser.go +++ b/core/browser.go @@ -1446,6 +1446,16 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) { WithRequest(ctx).WithError(derr).Debug("Dispose browser context after navigate error failed") } } + // LeavePageOpen keeps the Chrome tab open for manual inspection, but the + // background watchers attached to it must still stop or they leak goroutines. + stopWatchersOnErr := func() { + if b.LeavePageOpen { + stopWorkerPatchWatcher(page) + stopNetworkUsageWatcher(page) + } else { + closeOnErr() + } + } profile, laneKey := b.laneProfile(ctx, browser) SetBrowserProfileID(ctx, profile.ID) @@ -1493,7 +1503,7 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) { timedPage := page.Timeout(b.Timeout) if err := timedPage.Navigate(URL); err != nil { - closeOnErr() + stopWatchersOnErr() return nil, classifyProxyNetworkError(err) } @@ -1519,7 +1529,7 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) { } if err := classifyMainDocumentStatus(statusWatcher.Status()); err != nil { - closeOnErr() + stopWatchersOnErr() return nil, err } b.saveLaneCookies(ctx, page, URL) diff --git a/core/page_helpers.go b/core/page_helpers.go index 8529740..9266561 100644 --- a/core/page_helpers.go +++ b/core/page_helpers.go @@ -64,6 +64,56 @@ func WaitForElements(ctx context.Context, page *rod.Page, selectors []string, ti return nil, "", ErrSearchTimeout } +// DocSignals is the per-engine selector/text-marker config for +// ClassifyChallengeDocument. +type DocSignals struct { + CaptchaSelectors []string + CaptchaMarkers []string + EmptySelectors []string + EmptyMarkers []string +} + +// ClassifyChallengeDocument is the shared captcha/empty-result check behind +// every engine's classify*Document. Selectors are checked before markers +// since they're cheaper (no doc.Text() walk). +func ClassifyChallengeDocument(doc *goquery.Document, s DocSignals) error { + if anySelectorMatches(doc, s.CaptchaSelectors) { + return ErrCaptcha + } + var text string + if len(s.CaptchaMarkers)+len(s.EmptyMarkers) > 0 { + text = strings.ToLower(doc.Text()) + } + if anyMarkerMatches(text, s.CaptchaMarkers) { + return ErrCaptcha + } + if anySelectorMatches(doc, s.EmptySelectors) { + return ErrEmptyResult + } + if anyMarkerMatches(text, s.EmptyMarkers) { + return ErrEmptyResult + } + return nil +} + +func anySelectorMatches(doc *goquery.Document, selectors []string) bool { + for _, selector := range selectors { + if doc.Find(selector).Length() > 0 { + return true + } + } + return false +} + +func anyMarkerMatches(text string, markers []string) bool { + for _, marker := range markers { + if strings.Contains(text, marker) { + return true + } + } + return false +} + // HasAnySelector returns true if at least one of the supplied selectors // currently matches in the page DOM. It does not wait — pair with // WaitForElements when hydration may be in flight. diff --git a/core/server.go b/core/server.go index 3751dba..7deba6b 100644 --- a/core/server.go +++ b/core/server.go @@ -399,6 +399,18 @@ func (s *Server) handleParseEndpoint(c *fiber.Ctx, parser HTMLParser) error { results, err := parser.ParseHTML(bytes.NewReader(body)) if err != nil { + if isParseEndpointSearchError(err) { + spec := mapSearchError(err) + return &APIError{ + HTTPStatus: spec.status, + ErrorCode: spec.code, + Message: spec.message, + Meta: map[string]any{ + "engine": parser.Name(), + "error_detail": err.Error(), + }, + } + } return &APIError{ HTTPStatus: fiber.StatusBadRequest, ErrorCode: "parser_failure", @@ -418,6 +430,14 @@ func (s *Server) handleParseEndpoint(c *fiber.Ctx, parser HTMLParser) error { return sendEnvelope(c, format, env) } +func isParseEndpointSearchError(err error) bool { + return errors.Is(err, ErrCaptcha) || + errors.Is(err, ErrBlocked) || + errors.Is(err, ErrRateLimited) || + errors.Is(err, ErrSearchTimeout) || + errors.Is(err, ErrParser) +} + type searchErrorSpec struct { status int code string diff --git a/core/server_parse_test.go b/core/server_parse_test.go index ec05034..7f6a4f7 100644 --- a/core/server_parse_test.go +++ b/core/server_parse_test.go @@ -87,6 +87,29 @@ func TestParseEndpointParserErrorReturns400(t *testing.T) { } } +func TestParseEndpointSearchErrorReturnsTypedError(t *testing.T) { + engine := &parserMock{ + engineMock: engineMock{name: "google", initialized: true}, + parseHTMLFn: func(_ io.Reader) ([]SearchResult, error) { + return nil, ErrCaptcha + }, + } + srv := NewServerWithOptions("127.0.0.1", 7126, DefaultServerOptions(), engine) + + resp := postHTML(t, srv, "/google/parse", "captcha") + if resp.StatusCode != http.StatusTooManyRequests { + t.Fatalf("expected 429, got %d", resp.StatusCode) + } + + var body map[string]any + if err := json.NewDecoder(resp.Body).Decode(&body); err != nil { + t.Fatalf("decode error body: %v", err) + } + if body["error"] != "captcha_detected" { + t.Fatalf("expected captcha_detected, got %#v", body["error"]) + } +} + func TestParseEndpointNotRegisteredForNonParserEngine(t *testing.T) { // engineMock does NOT implement HTMLParser so no /mock/parse route is registered. engine := &engineMock{name: "mock", initialized: true} diff --git a/duckduckgo/parse_html.go b/duckduckgo/parse_html.go index 7daa8fa..9d18d20 100644 --- a/duckduckgo/parse_html.go +++ b/duckduckgo/parse_html.go @@ -1,6 +1,7 @@ package duckduckgo import ( + "errors" "io" "strings" @@ -15,9 +16,24 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { if err != nil { return nil, err } + pageStatus := classifyDDGDocument(doc) + if errors.Is(pageStatus, core.ErrEmptyResult) { + return []core.SearchResult{}, nil + } + if pageStatus != nil { + return nil, pageStatus + } return parseDDGDocument(doc), nil } +func classifyDDGDocument(doc *goquery.Document) error { + return core.ClassifyChallengeDocument(doc, core.DocSignals{ + CaptchaSelectors: Selectors.CaptchaSelectors, + CaptchaMarkers: Selectors.CaptchaMarkers, + EmptySelectors: Selectors.NoResults, + }) +} + func parseDDGDocument(doc *goquery.Document) []core.SearchResult { var results []core.SearchResult rank := core.NewRankState(0) diff --git a/duckduckgo/parse_html_test.go b/duckduckgo/parse_html_test.go index ac2bf98..6e9a4da 100644 --- a/duckduckgo/parse_html_test.go +++ b/duckduckgo/parse_html_test.go @@ -2,9 +2,12 @@ package duckduckgo import ( "bytes" + "errors" "os" "strings" "testing" + + "github.com/karust/openserp/core" ) func TestParseDDGHTML(t *testing.T) { @@ -56,6 +59,29 @@ func TestParseDDGHTMLEmpty(t *testing.T) { } } +func TestParseDDGHTMLCaptcha(t *testing.T) { + t.Parallel() + + html := `
` + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if !errors.Is(err, core.ErrCaptcha) { + t.Fatalf("expected ErrCaptcha, got results=%d err=%v", len(results), err) + } +} + +func TestParseDDGHTMLNoResults(t *testing.T) { + t.Parallel() + + html := `
No results found.
` + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results, got %d", len(results)) + } +} + func TestParseDDGHTMLAdsDoNotConsumeOrganicRank(t *testing.T) { t.Parallel() diff --git a/ecosia/parse_html.go b/ecosia/parse_html.go index d83f9c2..e778420 100644 --- a/ecosia/parse_html.go +++ b/ecosia/parse_html.go @@ -1,6 +1,7 @@ package ecosia import ( + "errors" "fmt" "io" "strings" @@ -16,9 +17,29 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { if err != nil { return nil, err } + pageStatus := classifyEcosiaDocument(doc) + if errors.Is(pageStatus, core.ErrEmptyResult) { + return []core.SearchResult{}, nil + } + if pageStatus != nil { + return nil, pageStatus + } return parseEcosiaDocument(doc), nil } +func classifyEcosiaDocument(doc *goquery.Document) error { + if isCaptchaDoc(doc) { + return core.ErrCaptcha + } + if doc.Find(Selectors.NoResults).Length() > 0 || + (doc.Find(Selectors.Mainline).Length() > 0 && + doc.Find(Selectors.Result).Length() == 0 && + doc.Find(Selectors.Ad).Length() == 0) { + return core.ErrEmptyResult + } + return nil +} + func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult { var results []core.SearchResult rank := 1 diff --git a/ecosia/search_raw.go b/ecosia/search_raw.go index 5735383..e28899f 100644 --- a/ecosia/search_raw.go +++ b/ecosia/search_raw.go @@ -16,16 +16,11 @@ import ( // challenge rather than a SERP. Prefers the hidden Turnstile input, then falls // back to the shared challenge-page body phrases (see cfBodyMarkers). func isCaptchaDoc(doc *goquery.Document) bool { - if doc.Find(Selectors.Captcha).Length() > 0 { - return true - } - text := strings.ToLower(doc.Text()) - for _, m := range cfBodyMarkers { - if strings.Contains(text, m) { - return true - } - } - return false + err := core.ClassifyChallengeDocument(doc, core.DocSignals{ + CaptchaSelectors: []string{Selectors.Captcha}, + CaptchaMarkers: cfBodyMarkers, + }) + return errors.Is(err, core.ErrCaptcha) } func classifyEcosiaRawHTML(body []byte) error { @@ -33,16 +28,7 @@ func classifyEcosiaRawHTML(body []byte) error { if err != nil { return err } - if isCaptchaDoc(doc) { - return core.ErrCaptcha - } - if doc.Find("[data-test-id='web-no-results']").Length() > 0 || - (doc.Find(Selectors.Mainline).Length() > 0 && - doc.Find(Selectors.Result).Length() == 0 && - doc.Find(Selectors.Ad).Length() == 0) { - return core.ErrEmptyResult - } - return nil + return classifyEcosiaDocument(doc) } // imageResultParser parses an Ecosia image SERP HTML response into search diff --git a/ecosia/search_raw_test.go b/ecosia/search_raw_test.go index daebb45..3044a91 100644 --- a/ecosia/search_raw_test.go +++ b/ecosia/search_raw_test.go @@ -75,3 +75,35 @@ func TestEcosiaClassifyRawHTML(t *testing.T) { }) } } + +func TestEcosiaParseHTMLClassifiesCaptchaAndNoResults(t *testing.T) { + t.Parallel() + + tests := []struct { + fixture string + wantErr error + }{ + {"search_captcha.html", core.ErrCaptcha}, + {"search_no_results.html", nil}, + } + + for _, tt := range tests { + t.Run(tt.fixture, func(t *testing.T) { + t.Parallel() + + results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("expected %v for %s, got %v", tt.wantErr, tt.fixture, err) + } + return + } + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results)) + } + }) + } +} diff --git a/ecosia/selectors.go b/ecosia/selectors.go index bf0baca..8f52676 100644 --- a/ecosia/selectors.go +++ b/ecosia/selectors.go @@ -3,6 +3,7 @@ package ecosia // Selectors is the single source of truth for Ecosia SERP CSS selectors. var Selectors = struct { Captcha string + NoResults string Mainline string Result string Ad string @@ -18,6 +19,7 @@ var Selectors = struct { // cf-turnstile-response input is present on every challenge page and never // on a real SERP, so it's a precise marker for the raw (browserless) path. Captcha: "input[name='cf-turnstile-response']", + NoResults: "[data-test-id='web-no-results']", Mainline: "[data-test-id='mainline']", Result: "[data-test-id='mainline-result-web']", Ad: "[data-test-id='mainline-result-ad']", diff --git a/google/captcha_selector_test.go b/google/captcha_selector_test.go index a6eea7a..4cdd0a6 100644 --- a/google/captcha_selector_test.go +++ b/google/captcha_selector_test.go @@ -18,13 +18,26 @@ func TestGooglePageTypeSelectors(t *testing.T) { wantHit bool }{ {"search_captcha.html", Selectors.Captcha, true}, + {"search_captcha.html", Selectors.CaptchaPage, true}, {"search_captcha.html", Selectors.ResultStats, false}, + {"search_captcha_new.html", Selectors.Captcha, true}, + {"search_captcha_new.html", Selectors.CaptchaPage, true}, + {"search_captcha_new.html", Selectors.ResultStats, false}, + + {"search_soft_block.html", Selectors.SoftBlock, true}, + {"search_soft_block.html", Selectors.Captcha, false}, + {"search_soft_block.html", Selectors.CaptchaPage, false}, + {"search_soft_block.html", Selectors.ResultStats, false}, + {"search_results.html", Selectors.ResultStats, true}, {"search_results.html", Selectors.Captcha, false}, + {"search_results.html", Selectors.CaptchaPage, false}, {"search_no_results.html", Selectors.ResultStats, true}, {"search_no_results.html", Selectors.Captcha, false}, + {"search_no_results.html", Selectors.CaptchaPage, false}, + {"search_no_results.html", Selectors.NoResults, true}, } for _, tt := range tests { diff --git a/google/parse_html_test.go b/google/parse_html_test.go index 560379c..938b180 100644 --- a/google/parse_html_test.go +++ b/google/parse_html_test.go @@ -2,11 +2,13 @@ package google import ( "bytes" + "errors" "os" "strings" "testing" "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/core" ) func TestParseHTML(t *testing.T) { @@ -126,6 +128,40 @@ func TestParseHTMLNoResults(t *testing.T) { } } +func TestParseHTMLCaptcha(t *testing.T) { + t.Parallel() + + for _, fixture := range []string{"search_captcha.html", "search_captcha_new.html"} { + t.Run(fixture, func(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/" + fixture) + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if !errors.Is(err, core.ErrCaptcha) { + t.Fatalf("expected ErrCaptcha, got results=%d err=%v", len(results), err) + } + }) + } +} + +func TestParseHTMLSoftBlock(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_soft_block.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if !errors.Is(err, core.ErrBlocked) { + t.Fatalf("expected ErrBlocked, got results=%d err=%v", len(results), err) + } +} + func TestParseHTMLAdsDoNotConsumeOrganicRank(t *testing.T) { t.Parallel() diff --git a/google/search.go b/google/search.go index d5f6193..1f46ddb 100644 --- a/google/search.go +++ b/google/search.go @@ -111,7 +111,7 @@ func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas, proxyURL string } func (gogl *Google) checkCaptcha(page *rod.Page, queryProxyURL string) bool { - has, _, _ := page.Has(Selectors.Captcha) + has, _, _ := page.Has(Selectors.CaptchaPage) if !has { return false } @@ -122,14 +122,14 @@ func (gogl *Google) checkCaptcha(page *rod.Page, queryProxyURL string) bool { } sitekey, err := captchaDiv.Attribute("data-sitekey") - if err != nil { - gogl.logger.Error("Cannot get captcha sitekey: %s", err) + if err != nil || sitekey == nil { + gogl.logger.Error("Cannot get captcha sitekey: %v", err) return true } dataS, err := captchaDiv.Attribute("data-s") - if err != nil { - gogl.logger.Error("Cannot get captcha datas: %s", err) + if err != nil || dataS == nil { + gogl.logger.Error("Cannot get captcha datas: %v", err) return true } @@ -143,6 +143,46 @@ func (gogl *Google) checkCaptcha(page *rod.Page, queryProxyURL string) bool { return true } +func (gogl *Google) checkNoResults(page *rod.Page) bool { + if has, _, err := page.Has(Selectors.ResultStats); err == nil && has { + statsEl, err := page.Element(Selectors.ResultStats) + if err == nil { + stats, _ := statsEl.Text() + if isZeroResultStats(strings.ToLower(stats)) { + return true + } + } + } + if has, _, err := page.Has(Selectors.NoResults); err == nil && has { + noResultsEl, err := page.Element(Selectors.NoResults) + if err == nil { + text, _ := noResultsEl.Text() + text = strings.ToLower(text) + return strings.Contains(text, "did not match any documents") || isZeroResultStats(text) + } + } + return false +} + +func (gogl *Google) checkSoftBlock(page *rod.Page) bool { + if has, _, err := page.Has(Selectors.Results); err == nil && has { + return false + } + if has, _, err := page.Has(Selectors.ResultsBroad); err == nil && has { + return false + } + + htmlTimeout := gogl.GetSelectorTimeout() / 2 + if htmlTimeout <= 0 || htmlTimeout > time.Second { + htmlTimeout = time.Second + } + html, err := page.Timeout(htmlTimeout).HTML() + if err != nil { + return false + } + return strings.Contains(strings.ToLower(html), "/httpservice/retry/enablejs") +} + func (gogl *Google) preparePage(page *rod.Page) { // Remove "similar queries" lists _, err := page.Eval(`() => { @@ -241,6 +281,10 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor gogl.logger.Error("Captcha detected: %s", url) return nil, core.ErrCaptcha } + if gogl.checkSoftBlock(page) { + gogl.logger.Error("Google soft block detected: %s", url) + return nil, core.ErrBlocked + } // Accept cookie consent so Google renders its SERP feature modules if query.Features { @@ -256,12 +300,18 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor gogl.logger.Error("Captcha detected: %s", url) return nil, core.ErrCaptcha } + if gogl.checkSoftBlock(page) { + gogl.logger.Error("Google soft block detected: %s", url) + return nil, core.ErrBlocked + } if core.IsContextDone(err) { return nil, err } - // Keep empty-SERP behavior for selector timeout only. if errors.Is(err, core.ErrSearchTimeout) { - return nil, nil + if gogl.checkNoResults(page) { + return nil, nil + } + return nil, core.ErrSearchTimeout } return nil, core.ErrSearchTimeout } @@ -463,6 +513,12 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor if gogl.checkCaptcha(page, query.ProxyURL) { return nil, core.ErrCaptcha } + if gogl.checkSoftBlock(page) { + return nil, core.ErrBlocked + } + if gogl.checkNoResults(page) { + return nil, nil + } // Result candidates were found by Selectors.Results but none parsed // into usable rows: treat as a genuine no-results SERP rather than a // timeout, so callers don't retry pointlessly. diff --git a/google/search_raw.go b/google/search_raw.go index 7530d9d..29e7a72 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "regexp" "strings" "github.com/PuerkitoBio/goquery" @@ -13,6 +14,10 @@ import ( "github.com/sirupsen/logrus" ) +// Matches a literal zero result count, e.g. "about 0 results". Requires "0" +// as its own word so it doesn't match large counts like "10,500,000 results". +var zeroResultsPattern = regexp.MustCompile(`\b0 results\b`) + // ParseHTML parses a Google SERP HTML document and returns search results. // It is the pure parser used by both raw HTTP search and parse endpoints. func ParseHTML(r io.Reader) ([]core.SearchResult, error) { @@ -20,6 +25,13 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { if err != nil { return nil, err } + pageStatus := classifyGoogleDocument(doc) + if errors.Is(pageStatus, core.ErrEmptyResult) { + return []core.SearchResult{}, nil + } + if pageStatus != nil { + return nil, pageStatus + } return parseGoogleDocument(doc), nil } @@ -104,18 +116,58 @@ func classifyGoogleRawHTML(body []byte) error { if err != nil { return err } - if doc.Find(Selectors.Captcha).Length() > 0 { + return classifyGoogleDocument(doc) +} + +func classifyGoogleDocument(doc *goquery.Document) error { + if isGoogleCaptchaDocument(doc) { return core.ErrCaptcha } - - text := strings.ToLower(doc.Text()) - if strings.Contains(text, "did not match any documents") || - strings.Contains(text, "about 0 results") { + if isGoogleSoftBlockDocument(doc) { + return core.ErrBlocked + } + if isGoogleNoResultsDocument(doc) { return core.ErrEmptyResult } return nil } +func isGoogleCaptchaDocument(doc *goquery.Document) bool { + err := core.ClassifyChallengeDocument(doc, core.DocSignals{ + CaptchaSelectors: []string{Selectors.CaptchaPage}, + CaptchaMarkers: Selectors.CaptchaMarkers, + }) + return errors.Is(err, core.ErrCaptcha) +} + +// isGoogleSoftBlockDocument reports the JS-retry interstitial Google serves +// instead of a SERP. +func isGoogleSoftBlockDocument(doc *goquery.Document) bool { + if doc.Find(Selectors.Results).Length() > 0 || doc.Find(Selectors.ResultsBroad).Length() > 0 { + return false + } + return strings.Contains(strings.ToLower(doc.Find(Selectors.SoftBlock).Text()), "/httpservice/retry/enablejs") +} + +func isGoogleNoResultsDocument(doc *goquery.Document) bool { + stats := strings.ToLower(doc.Find(Selectors.ResultStats).Text()) + if isZeroResultStats(stats) { + return true + } + text := strings.ToLower(doc.Find(Selectors.NoResults).Text()) + if text == "" { + text = strings.ToLower(doc.Text()) + } + return strings.Contains(text, "did not match any documents") || isZeroResultStats(text) +} + +// isZeroResultStats reports whether s states a literal zero result count, +// e.g. "about 0 results". A plain substring check on "0 results" also matches +// large counts like "10,500,000 results", so require "0" as its own word. +func isZeroResultStats(s string) bool { + return zeroResultsPattern.MatchString(s) +} + func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { ctx = core.PrepareEngineContext(ctx, query, "google", false) diff --git a/google/search_raw_test.go b/google/search_raw_test.go index 436bcad..ef28616 100644 --- a/google/search_raw_test.go +++ b/google/search_raw_test.go @@ -18,6 +18,7 @@ func TestGoogleParseHTMLFixtures(t *testing.T) { minResultCount int maxResultCount int wantZero bool + wantErr error }{ { name: "search results", @@ -31,9 +32,19 @@ func TestGoogleParseHTMLFixtures(t *testing.T) { wantZero: true, }, { - name: "captcha page", - fixture: "search_captcha.html", - wantZero: true, + name: "captcha page", + fixture: "search_captcha.html", + wantErr: core.ErrCaptcha, + }, + { + name: "new captcha page", + fixture: "search_captcha_new.html", + wantErr: core.ErrCaptcha, + }, + { + name: "soft block page", + fixture: "search_soft_block.html", + wantErr: core.ErrBlocked, }, } @@ -42,6 +53,12 @@ func TestGoogleParseHTMLFixtures(t *testing.T) { t.Parallel() results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("expected %v for %s, got %v", tt.wantErr, tt.fixture, err) + } + return + } if err != nil { t.Fatalf("ParseHTML() error = %v", err) } @@ -88,6 +105,8 @@ func TestGoogleClassifyRawHTML(t *testing.T) { }{ {name: "no results", fixture: "search_no_results.html", want: core.ErrEmptyResult}, {name: "captcha page", fixture: "search_captcha.html", want: core.ErrCaptcha}, + {name: "new captcha page", fixture: "search_captcha_new.html", want: core.ErrCaptcha}, + {name: "soft block page", fixture: "search_soft_block.html", want: core.ErrBlocked}, } for _, tt := range tests { diff --git a/google/selectors.go b/google/selectors.go index 7ed19d5..7bb61b9 100644 --- a/google/selectors.go +++ b/google/selectors.go @@ -4,19 +4,23 @@ package google // Both the browser parser (search.go, rod) and HTML parser (search_raw.go, // goquery) reference these. When Google changes their DOM, edit here only. var Selectors = struct { - Captcha string - ResultStats string - CookieBtn string - Results string - ResultsBroad string - Ad string - Link string - Title string - DescPrimary string - DescFallback string - DescAny string - AnswerBox string - AnswerItem string + Captcha string + CaptchaPage string + CaptchaMarkers []string + SoftBlock string + NoResults string + ResultStats string + CookieBtn string + Results string + ResultsBroad string + Ad string + Link string + Title string + DescPrimary string + DescFallback string + DescAny string + AnswerBox string + AnswerItem string // Image search. ImageResults string @@ -24,7 +28,17 @@ var Selectors = struct { ImageLinkFallback string ImageTitle []string }{ - Captcha: "div[data-sitekey]", + Captcha: "[data-sitekey]", + CaptchaPage: "form#captcha-form, [data-sitekey], .g-recaptcha, script[src*='recaptcha']", + // CaptchaMarkers is the page-text fallback for captcha variants whose + // markup doesn't match CaptchaPage. + CaptchaMarkers: []string{ + "our systems have detected unusual traffic", + "not a robot", + "solve the captcha", + }, + SoftBlock: "noscript", + NoResults: "#botstuff, #topstuff, .mnr-c", ResultStats: "div#result-stats", CookieBtn: "div[role='dialog'][aria-modal] button", // Results targets the canonical organic result block. div.tF2Cxc is the diff --git a/google/testdata/search_captcha_new.html b/google/testdata/search_captcha_new.html new file mode 100644 index 0000000..3280d75 --- /dev/null +++ b/google/testdata/search_captcha_new.html @@ -0,0 +1 @@ +https://www.google.com/search?ie=UTF-8&oq=weather&pws=0&q=weather&sourceid=chrome

About this page

Our systems have detected unusual traffic from your computer network. This page checks to see if it's really you sending the requests, and not a robot.Why did this happen?


IP address: 107.219.149.58
Time: 2026-06-30T19:48:20Z
URL: https://www.google.com/search?ie=UTF-8&oq=weather&pws=0&q=weather&sourceid=chrome
\ No newline at end of file diff --git a/google/testdata/search_soft_block.html b/google/testdata/search_soft_block.html new file mode 100644 index 0000000..29164d7 --- /dev/null +++ b/google/testdata/search_soft_block.html @@ -0,0 +1 @@ +weather - Google Search
diff --git a/yandex/parse_html.go b/yandex/parse_html.go index 921ec08..34e23a8 100644 --- a/yandex/parse_html.go +++ b/yandex/parse_html.go @@ -1,6 +1,7 @@ package yandex import ( + "errors" "io" "net/url" "strings" @@ -16,9 +17,23 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { if err != nil { return nil, err } + pageStatus := classifyYandexDocument(doc) + if errors.Is(pageStatus, core.ErrEmptyResult) { + return []core.SearchResult{}, nil + } + if pageStatus != nil { + return nil, pageStatus + } return parseYandexDocument(doc), nil } +func classifyYandexDocument(doc *goquery.Document) error { + return core.ClassifyChallengeDocument(doc, core.DocSignals{ + CaptchaSelectors: []string{Selectors.Captcha}, + EmptySelectors: []string{Selectors.NoResults}, + }) +} + func parseYandexDocument(doc *goquery.Document) []core.SearchResult { var results []core.SearchResult rank := core.NewRankState(0) diff --git a/yandex/search_raw.go b/yandex/search_raw.go index dd2fbd6..c2a83b0 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -15,13 +15,7 @@ func classifyYandexRawHTML(body []byte) error { if err != nil { return err } - if doc.Find(Selectors.Captcha).Length() > 0 { - return core.ErrCaptcha - } - if doc.Find(Selectors.NoResults).Length() > 0 { - return core.ErrEmptyResult - } - return nil + return classifyYandexDocument(doc) } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { diff --git a/yandex/search_raw_test.go b/yandex/search_raw_test.go index a504137..9fe8180 100644 --- a/yandex/search_raw_test.go +++ b/yandex/search_raw_test.go @@ -17,12 +17,12 @@ func TestYandexParseHTMLFixtures(t *testing.T) { t.Parallel() tests := []struct { - name string - fixture string - wantZero bool + name string + fixture string + wantErr error }{ - {name: "no results", fixture: "search_no_results.html", wantZero: true}, - {name: "captcha page", fixture: "search_captcha.html", wantZero: true}, + {name: "no results", fixture: "search_no_results.html"}, + {name: "captcha page", fixture: "search_captcha.html", wantErr: core.ErrCaptcha}, } for _, tt := range tests { @@ -30,10 +30,16 @@ func TestYandexParseHTMLFixtures(t *testing.T) { t.Parallel() results, err := ParseHTML(testutil.ResponseFromFixture(t, tt.fixture).Body) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("expected %v for %s, got %v", tt.wantErr, tt.fixture, err) + } + return + } if err != nil { t.Fatalf("ParseHTML() error = %v", err) } - if tt.wantZero && len(results) != 0 { + if len(results) != 0 { t.Fatalf("expected zero results for %s, got %d", tt.fixture, len(results)) } }) diff --git a/yandex/url.go b/yandex/url.go index 2adaaea..2f71616 100644 --- a/yandex/url.go +++ b/yandex/url.go @@ -44,7 +44,9 @@ func BuildURL(q core.Query, page int) (string, error) { if lr := yandexLR(q.Region); lr != "" { params.Add("lr", lr) - params.Add("rstr", "true") + // rstr (strict region) dropped - it makes Yandex captcha far more + // often. lr alone still ranks toward the region, just less precisely. + // params.Add("rstr", "true") } base.RawQuery = params.Encode() @@ -83,7 +85,9 @@ func BuildImageURL(q core.Query, page int) (string, error) { if lr := yandexLR(q.Region); lr != "" { params.Add("lr", lr) - params.Add("rstr", "true") + // rstr (strict region) dropped - it makes Yandex captcha far more + // often. lr alone still ranks toward the region, just less precisely. + // params.Add("rstr", "true") } base.RawQuery = params.Encode() diff --git a/yandex/url_test.go b/yandex/url_test.go index e80d2e4..545b264 100644 --- a/yandex/url_test.go +++ b/yandex/url_test.go @@ -73,12 +73,10 @@ func TestBuildURLRegionLR(t *testing.T) { if gotLR := parsed.Query().Get("lr"); gotLR != tt.wantLR { t.Fatalf("unexpected lr value: %q want %q", gotLR, tt.wantLR) } - wantRstr := "" - if tt.wantLR != "" { - wantRstr = "true" - } - if gotRstr := parsed.Query().Get("rstr"); gotRstr != wantRstr { - t.Fatalf("unexpected rstr value: %q for lr %q", gotRstr, tt.wantLR) + // rstr (strict region) is intentionally never set - it makes Yandex + // captcha far more often. lr alone still ranks toward the region. + if gotRstr := parsed.Query().Get("rstr"); gotRstr != "" { + t.Fatalf("rstr should never be set, got %q for lr %q", gotRstr, tt.wantLR) } }) } @@ -97,7 +95,7 @@ func TestBuildImageURLRegionLR(t *testing.T) { if gotLR := parsed.Query().Get("lr"); gotLR != "2" { t.Fatalf("unexpected lr value: %q", gotLR) } - if gotRstr := parsed.Query().Get("rstr"); gotRstr != "true" { - t.Fatalf("unexpected rstr value: %q", gotRstr) + if gotRstr := parsed.Query().Get("rstr"); gotRstr != "" { + t.Fatalf("rstr should never be set, got %q", gotRstr) } }