From 9cc69da758d9e4ae2ea4e034c9d011d7df2a1377 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Tue, 16 Jun 2026 03:51:37 +0300 Subject: [PATCH] feat(cli): clean search, add extract/format flags; harden engines & proxy rotation; fix bugs; update docs - Add structured `search [engine] [query]` CLI: --limit/--lang/--region/--site/--file, --format (json|text|markdown|ndjson), --extract N, --search-timeout; Envelope and route logs to stderr with a --quiet default (fixes stdout pollution) - Unify engines behind a single engineSpec registry (CLI + serve share it) - Unify the extract knob to bool-or-int `extract=N` (drop extract_top); CLI and HTTP share core batch extraction, raw/rendered fetch, and clamp helpers - Engines: Ecosia CF captcha detection (raw + browser), Yandex progressive-result wait, Google PAA poll + Has() existence probes, Bing title/desc attribute fallbacks - Proxy: rotate challenged proxies out of the tag pool for one retry (X-Proxy-Attempts); browser health-ping skip window; opt-in WaitStable --- README.md | 255 ++++++++++-- bing/parse_html.go | 32 +- bing/parse_html_test.go | 46 +++ bing/search.go | 25 +- cmd/engines.go | 78 ++++ cmd/root.go | 22 +- cmd/search.go | 376 ++++++++++++++---- cmd/search_test.go | 82 ++++ cmd/serve.go | 78 +--- cmd/serve_test.go | 9 + config.yaml | 4 +- core/browser.go | 47 ++- core/common.go | 107 +++-- core/logger.go | 22 +- core/page_helpers.go | 53 ++- core/proxy.go | 87 +++- core/proxy_rotation_test.go | 150 +++++++ core/query_extract_param_test.go | 73 ++++ core/resilient.go | 130 +++--- core/server.go | 3 + core/server_extract.go | 60 ++- docs/openapi.yaml | 30 +- ecosia/captcha_selector_test.go | 54 +++ ecosia/search.go | 23 +- ecosia/search_raw.go | 18 +- ecosia/search_raw_test.go | 33 +- ecosia/selectors.go | 5 + ecosia/testdata/search_captcha.html | 1 + .../content/js-search-with-extract/README.md | 2 +- .../content/js-search-with-extract/index.js | 9 +- google/search.go | 35 +- main.go | 2 +- yandex/search.go | 45 ++- 33 files changed, 1613 insertions(+), 383 deletions(-) create mode 100644 cmd/engines.go create mode 100644 cmd/search_test.go create mode 100644 core/proxy_rotation_test.go create mode 100644 core/query_extract_param_test.go create mode 100644 ecosia/captcha_selector_test.go create mode 100644 ecosia/testdata/search_captcha.html diff --git a/README.md b/README.md index 0e0095b..9931058 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ![OpenSERP](./logo.svg) -# OpenSERP (Search Engine Results) +# OpenSERP [![Go Report Card](https://goreportcard.com/badge/github.com/karust/openserp)](https://goreportcard.com/report/github.com/karust/openserp) [![Go Reference](https://pkg.go.dev/badge/github/karust/openserp?style=for-the-badge)](https://pkg.go.dev/github.com/karust/openserp) @@ -8,42 +8,47 @@ [![Docker Pulls](https://img.shields.io/docker/v/karust/openserp)](https://hub.docker.com/r/karust/openserp) [![CI](https://github.com/karust/openserp/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/karust/openserp/actions/workflows/ci.yml) -**OpenSERP** is a free, open-source API and CLI for accessing normalized search engine results from **Google, Yandex, Baidu, Bing, DuckDuckGo, and Ecosia**. +**OpenSERP** is a free, open-source SERP API and CLI for live search data from **Google, Yandex, Baidu, Bing, DuckDuckGo, and Ecosia**. -Run it locally, self-host it, or use the optional hosted API when you do not want to manage infrastructure. +Use it as a search tool for **LLMs, agents, and RAG pipelines**, or as a scraper backend for **SEO rank tracking across Google, Yandex, Baidu, and more**. It is especially useful when your workflow needs RU/CN web coverage instead of another Google-only API. + +Run it locally, self-host it, or use [OpenSERP Cloud](https://openserp.org/cloud) when you want the same public API shape without operating the server. **Official website:** [openserp.org](https://openserp.org) -**Feedback:** [GitHub Issues](https://github.com/karust/openserp/issues) or [feedback@openserp.org](mailto:feedback@openserp.org) - -**Latest updates, usage examples**: [Telegram](https://t.me/+RJEKspw3mUlhZDMy) - -> πŸ’‘ OpenSERP is free and open-source. Only links listed in this repository and on the official website are associated with the project. - ## Features -- πŸ” **Multi-engine** - search with dedicated endpoints for each engine -- 🌐 **Megasearch** - cross-engine aggregation with deduplication +- πŸ” **Multi-engine** - dedicated endpoints for Google, Yandex, Baidu, Bing, DuckDuckGo, and Ecosia, with stable JSON for SEO rank pipelines +- 🌐 **Megasearch** - `/mega/search` runs one query across every selected engine, then merges and dedupes results +- πŸ“„ **URL extraction** - return search results plus clean markdown/text target-page content in one call, for grounding and automation +- ✨ **SERP features** - AI summaries, answer boxes, people-also-ask, and related searches in a response - πŸ–Ό **Images** - image search is also available - 🎯 **Advanced filters** - language, date range, file type, and site queries -- ✨ **SERP features** - AI summaries, answer boxes, people-also-ask, and related searches in a response -- πŸ“„ **URL extraction** - turn target pages into clean markdown/text for grounding and automation +- πŸ“ **Data formats** - JSON, Markdown, Text, NdJSON response formats - 🌍 **Configurable** - proxy, cache, and resilient mode - 🐳 **Docker-ready** - local and container deployment -- πŸ“ **Data Formats** - JSON, Markdown, Text, NdJSON response formats ## ⚑ Quick Start ### Docker +Prebuilt images are published to [docker hub: `karust/openserp`](https://hub.docker.com/r/karust/openserp). + ```sh # Run the API server via prebuilt image docker run --rm -p 127.0.0.1:7000:7000 karust/openserp:latest serve -a 0.0.0.0 -p 7000 -# Or use docker-compose (pulls the prebuilt image) +# Or docker compose up ``` +### Go install + +```sh +go install github.com/karust/openserp@latest +openserp search duckduckgo "open source serp api" --format markdown +``` + ### From Source ```sh @@ -53,10 +58,117 @@ go build -o openserp . ./openserp serve ``` +### First request + +```sh +curl "http://127.0.0.1:7000/mega/search?engines=bing,google&text=golang+vs+rust&extract=1&mode=any" +``` + +
+Example JSON response + +```json +{ + "query": { + "text": "golang vs rust", + "engines_requested": ["bing", "google"] + }, + "meta": { + "request_id": "019ecdc0-a66d-79a4-9d2b-9e9b480d495e", + "requested_at": "2026-06-16T00:06:55Z", + "took_ms": 720, + "engines_responded": ["bing"], + "engines_failed": [], + "version": "2.1" + }, + "results": [ + { + "id": "s_5a8273f16b19ab64", + "rank": 1, + "type": "organic", + "title": "The Go Programming Language", + "url": "https://go.dev/", + "display_url": "go.dev", + "snippet": "Get Started Playground Tour Stack Overflow Help Packages Standard Library About Go Packages About Download Blog Issue Tracker Release Notes Brand Guidelines Code of Conduct Connect …", + "domain": "go.dev", + "favicon": "https://go.dev/favicon.ico", + "position": { + "absolute": 1 + }, + "engine": "bing", + "domain_info": { + "tld": "dev", + "sld": "go", + "category": "" + }, + "extracted": { + "title": "Build simple, secure, scalable systems with Go", + "format": "markdown", + "content": "## Build simple, secure, scalable systems with Go\n\n![Go Gopher climbing a ladder.](https://go.dev/images/gophers/ladder.svg)\n\n- β€œAt the time, no single team member knew Go, but **within a month, everyone was writing in Go** and we were building out the endpoints. It was the flexibility, how easy it was to use, and the really cool concept behind Go (how Go handles native concurrency, garbage collection, and of course safety+speed.) that helped engage us during the build. Also, who can beat that cute mascot!”\n ........", + "mode_used": "fast", + "fetched_at": "2026-06-16T00:06:56Z" + } + }, + { + "id": "s_1a364ebcb3035539", + "rank": 2, + "type": "organic", + "title": "Go (programming language) - Wikipedia", + "url": "https://en.wikipedia.org/wiki/Go_(programming_language)", + "display_url": "en.wikipedia.org β€Ί wiki β€Ί Go_(programming_language)", + "snippet": "In Go's package system, each package has a path (e.g., \"compress/bzip2\" or \"golang.org/x/net/html\") and a name (e.g., bzip2 or html). By default other packages' definitions must always be prefixed with …", + "domain": "en.wikipedia.org", + "favicon": "https://en.wikipedia.org/favicon.ico", + "position": { + "absolute": 2 + }, + "engine": "bing", + "domain_info": { + "tld": "org", + "sld": "wikipedia", + "category": "" + }, + "classification": { + "content_type": "article", + "source_hint": "encyclopedia" + } + }, + ... + ], + "serp_features": [], + "pagination": { + "page": 1, + "has_more": false, + "next_start": 10 + }, + "clusters": [ + { + "id": "c_f20b23a020101dce", + "canonical_url": "https://go.dev/", + "domain": "go.dev", + "title": "The Go Programming Language", + "occurrences": [ + { + "engine": "bing", + "rank": 1, + "result_id": "s_5a8273f16b19ab64" + } + ], + "engines_count": 1, + "best_rank": 1, + "score": 0.5 + }, + ... + ] +} +``` + +
+ ## Deployment Options - **Self-hosted (this repo)** - free, MIT-licensed, with full control over runtime, proxies, cache, and scaling. -- **[Hosted API](https://openserp.org/cloud)** - optional managed version from the project maintainers, with the same API shape. +- **[OpenSERP Cloud](https://openserp.org/cloud)** - optional managed version from the project maintainers, with the same API shape. The hosted API helps fund continued development of the open-source project. Same endpoints, same response schema, and client code can migrate either direction. @@ -109,16 +221,26 @@ curl "http://127.0.0.1:7000/bing/image?text=golang+logo&limit=10" Megasearch: ```bash -# Search all configured engines curl "http://127.0.0.1:7000/mega/search?text=golang&limit=10" +``` -# Fast mode: only one fastest engine is queried +| Mode | Best for | Behavior | +| ---------- | ------------------------------------ | ---------------------------------------------- | +| `balanced` | Most multi-engine SERP workflows | Queries engines in parallel and merges results | +| `fast` | Lowest latency | Uses the fastest available engine | +| `any` | Fallback-style availability checking | Tries engines sequentially until one responds | + +
+More megasearch examples + +```bash +# Fast mode curl "http://127.0.0.1:7000/mega/search?text=golang&mode=fast&engines=google,bing,yandex" -# Any mode: sequential fallback in provided order (default order if none provided) +# Any mode curl "http://127.0.0.1:7000/mega/search?text=golang&mode=any&engines=google,yandex,bing" -# Balanced mode (default): parallel all engines with aggregation controls +# Balanced mode with aggregation controls curl "http://127.0.0.1:7000/mega/search?text=golang&mode=balanced&dedupe=true&merge=true" # Advanced filtering @@ -128,6 +250,8 @@ curl "http://127.0.0.1:7000/mega/search?text=golang&engines=google,bing&limit=20 curl "http://127.0.0.1:7000/mega/image?text=golang+logo&limit=20" ``` +
+ List engines: ```bash @@ -144,27 +268,84 @@ curl "http://127.0.0.1:7000/extract?url=https://example.com&mode=auto" curl "http://127.0.0.1:7000/extract?url=https://example.com&format=markdown" # Embed extracted content under the top search results -curl "http://127.0.0.1:7000/google/search?text=llm+observability&extract=true&extract_top=2&format=markdown" +curl "http://127.0.0.1:7000/google/search?text=llm+observability&extract=2&format=markdown" ``` +## πŸ–₯ CLI Search + +No server required - query an engine straight from the terminal. The CLI shares the same engines, formats, and filters as the API. + +```sh +openserp search duckduckgo "free open source serp" --format markdown +``` + +
+CLI output and more examples + +```markdown +# Search results for "free open source serp" + +**Query:** free open source serp - **Engines:** duckduckgo - **Took:** 1794ms + +## Results + +### 1. OpenSERP: Open-Source, Self-Hosted & Free SERP API + +**openserp.org** - organic + +OpenSERP is a free, open-source and self-hosted SERP API for Google, Bing, Yandex, Baidu, DuckDuckGo and Ecosia, with an optional managed Cloud path. + +-> https://openserp.org/ + +### 2. GitHub - karust/openserp: Open-source SERP API for AI, SEO & automation ... + +**github.com β€Ί karust β€Ί openserp** - organic + +OpenSERP is a free, open-source API and CLI for accessing normalized search engine results from Google, Yandex, Baidu, Bing, DuckDuckGo, and Ecosia. Run it locally, self-host it, or use the optional hosted API when you do not want to manage infrastructure. + +-> https://github.com/karust/openserp +``` + +More CLI examples: + +```sh +# JSON is the default format +openserp search google "golang generics" --limit 20 + +# Plain text, German results +openserp search yandex "wetter berlin" --format text --lang DE --region DE + +# Restrict to a site and stream NdJSON +openserp search bing "release notes" --site github.com --format ndjson + +# Embed clean page content from the top 2 results +openserp search google "llm observability" --extract 2 --format markdown + +# Browserless (raw HTTP) mode through a proxy +openserp search duckduckgo "free open source serp" --raw --proxy http://user:pass@127.0.0.1:8080 +``` + +
+ +Run `openserp search --help` for the full flag list. Engine names: `google`, `yandex`, `baidu`, `bing`, `duckduckgo`, `ecosia`. + ## πŸ” Query Parameters Common parameters: -| Parameter | Description | Example | -| -------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------ | -| `text` | Search query | `golang programming` | -| `lang` | Language code | `EN`, `DE`, `RU`, `ES` | -| `region` | Market/location hint. Countries/locales work across engines; Google also accepts city names via `uule`; Yandex accepts numeric `lr`. | `DE`, `en-GB`, `Berlin`, `213` | -| `date` | Date range | `20250101..20251231` | -| `file` | File extension | `pdf`, `doc`, `xls` | -| `site` | Site-specific search | `github.com` | -| `limit` | Number of organic results, max 100. When omitted or `<=10`, only the first SERP page is parsed. | `25`, `50` | -| `start` | Pagination offset | `0`, `10`, `20` | -| `format` | Output format | `json`, `markdown`, `text`, `ndjson` | -| `extract` | Fetch and embed target-page content for top web results | `true` | -| `extract_top` | Number of top web results to extract, clamped to 1-5 | `3` | -| `extract_mode` | Extraction strategy: raw HTTP first, raw only, or browser-rendered | `auto`, `fast`, `rendered` | +| Parameter | Description | Example | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------ | +| `text` | Search query | `golang programming` | +| `lang` | Language code | `EN`, `DE`, `RU`, `ES` | +| `region` | Market/location hint. Countries/locales work across engines; Google also accepts city names via `uule`; Yandex accepts numeric `lr`. | `DE`, `en-GB`, `Berlin`, `213` | +| `date` | Date range | `20250101..20251231` | +| `file` | File extension | `pdf`, `doc`, `xls` | +| `site` | Site-specific search | `github.com` | +| `limit` | Number of organic results, max 100. When omitted or `<=10`, only the first SERP page is parsed. | `25`, `50` | +| `start` | Pagination offset | `0`, `10`, `20` | +| `format` | Output format | `json`, `markdown`, `text`, `ndjson` | +| `extract` | Fetch and embed target-page content for top web results. Bool or int depth: `0`/`false` off, `true`/`1` top result, `N` top N (1-5). `extract_mode`/`min_runes` imply `extract=true` unless `extract=0` | `1`, `3`, `true` | +| `extract_mode` | Extraction strategy: raw HTTP first, raw only, or browser-rendered | `auto`, `fast`, `rendered` | Engine-specific parameters: @@ -335,7 +516,7 @@ Contributions are welcome. See [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md). ## Feedback & Updates - [GitHub Issues](https://github.com/karust/openserp/issues) - bugs, feature ideas, and reproducible issues. -- [Telegram channel](https://t.me/openserp_cloud) - OpenSERP news, release notes, and project updates. Direct messages are open for quick feedback and hosted API questions. - [feedback@openserp.org](mailto:feedback@openserp.org) - private notes, longer feedback, or anything that does not fit GitHub Issues. +- [Telegram Channel](https://t.me/+RJEKspw3mUlhZDMy) - OpenSERP news, release notes, and project updates. Direct messages are open for quick feedback and hosted API questions. -###### _"OpenSERP" is the name of this open-source project. The official website is [openserp.org](https://openserp.org). Resources not linked on this page are not affiliated with the project._ +> OpenSERP is free and open-source. Only links listed in this repository and on [openserp.org](https://openserp.org) are associated with the project. diff --git a/bing/parse_html.go b/bing/parse_html.go index e176ba1..1050f7b 100644 --- a/bing/parse_html.go +++ b/bing/parse_html.go @@ -47,7 +47,10 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult { return } - title := titleTag.Text() + title := firstNonEmptyAttr(titleTag, "aria-label", "title") + if title == "" { + title = normalizeWhitespace(titleTag.Text()) + } if title == "" { title = extractFirstText(item, Selectors.TitleFallbacks) } @@ -82,19 +85,30 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult { func extractFirstText(item *goquery.Selection, selectors []string) string { for _, selector := range selectors { if tag := item.Find(selector).First(); tag.Length() > 0 { - if text := strings.TrimSpace(tag.Text()); text != "" { + if text := normalizeWhitespace(tag.Text()); text != "" { return text } - if label, exists := tag.Attr("aria-label"); exists { - if label = strings.TrimSpace(label); label != "" { - return label - } + if label := firstNonEmptyAttr(tag, "aria-label", "title"); label != "" { + return label } } } return "" } +func firstNonEmptyAttr(item *goquery.Selection, attrs ...string) string { + for _, attr := range attrs { + value, exists := item.Attr(attr) + if !exists { + continue + } + if value = normalizeWhitespace(value); value != "" { + return value + } + } + return "" +} + // descriptionFromItem extracts a description using the same 4-step fallback // chain as the rod-based browser parser. Bing renders snippet text with heavy // source-indentation whitespace, so each candidate is whitespace-collapsed. @@ -118,8 +132,8 @@ func descriptionFromItem(item *goquery.Selection, title string) string { return normalizeWhitespace(strings.Replace(item.Text(), title, "", 1)) } -// normalizeWhitespace collapses runs of whitespace (including the newlines and -// indentation Bing leaves in snippet markup) into single spaces. +// normalizeWhitespace collapses Bing's snippet-markup whitespace into single +// spaces (see core.NormalizeWhitespace). func normalizeWhitespace(s string) string { - return strings.Join(strings.Fields(s), " ") + return core.NormalizeWhitespace(s) } diff --git a/bing/parse_html_test.go b/bing/parse_html_test.go index 3456a37..c55dab3 100644 --- a/bing/parse_html_test.go +++ b/bing/parse_html_test.go @@ -145,3 +145,49 @@ func TestParseBingHTMLTitleFallback(t *testing.T) { t.Fatalf("title = %q, want fallback", results[0].Title) } } + +func TestParseBingHTMLPrefersTitleAttribute(t *testing.T) { + t.Parallel() + + html := ` +
    +
  1. +

    example.com

    +

    Snippet

    +
    2. +
` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if results[0].Title != "Real SERP Title" { + t.Fatalf("title = %q, want attribute title", results[0].Title) + } +} + +func TestParseBingHTMLDescriptionFallsThroughEmptyPrimary(t *testing.T) { + t.Parallel() + + html := ` +
    +
  1. +

    Result title

    +

    Useful snippet text
    +
    2. +
` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if results[0].Description != "Useful snippet text" { + t.Fatalf("description = %q, want fallback snippet", results[0].Description) + } +} diff --git a/bing/search.go b/bing/search.go index f519abe..47b1e03 100644 --- a/bing/search.go +++ b/bing/search.go @@ -66,6 +66,11 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool { } func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error { + // Probe first so a banner-less SERP returns immediately instead of blocking + // .Element for the full Timeout/10. + if has, _, err := page.Has(Selectors.CookieBtn); err != nil || !has { + return nil + } consentBtn, err := page.Timeout(bing.Timeout / 10).Element(Selectors.CookieBtn) if err != nil { return nil @@ -107,8 +112,10 @@ func (bing *Bing) parseResultElement(el *rod.Element, isAd bool, rank, absoluteR return core.SearchResult{}, false } - title, _ := titleElem.Text() - title = strings.TrimSpace(title) + title := core.ElementAttribute(titleElem, "aria-label", "title") + if title == "" { + title = core.ElementText(titleElem) + } if title == "" { title = core.FirstNonEmptyText(el, Selectors.TitleFallbacks...) } @@ -120,16 +127,10 @@ func (bing *Bing) parseResultElement(el *rod.Element, isAd bool, rank, absoluteR return core.SearchResult{}, false } - desc := "" - if descElem, err := el.Element(Selectors.DescPrimary); err == nil { - desc, _ = descElem.Text() - } else if descElem, err := el.Element(Selectors.DescFallback); err == nil { - desc, _ = descElem.Text() - } else if descElem, err := el.Element(Selectors.DescAny); err == nil { - desc, _ = descElem.Text() - } else { + desc := core.FirstNonEmptyText(el, Selectors.DescPrimary, Selectors.DescFallback, Selectors.DescAny) + if desc == "" { fullText, _ := el.Text() - desc = strings.TrimSpace(strings.Replace(fullText, title, "", 1)) + desc = core.NormalizeWhitespace(strings.Replace(fullText, title, "", 1)) } return core.SearchResult{ @@ -137,7 +138,7 @@ func (bing *Bing) parseResultElement(el *rod.Element, isAd bool, rank, absoluteR AbsoluteRank: absoluteRank, URL: url, Title: title, - Description: strings.TrimSpace(desc), + Description: desc, Ad: isAd, }, true } diff --git a/cmd/engines.go b/cmd/engines.go new file mode 100644 index 0000000..2843bc0 --- /dev/null +++ b/cmd/engines.go @@ -0,0 +1,78 @@ +package cmd + +import ( + "context" + "io" + + "github.com/karust/openserp/baidu" + "github.com/karust/openserp/bing" + "github.com/karust/openserp/core" + "github.com/karust/openserp/duckduckgo" + "github.com/karust/openserp/ecosia" + "github.com/karust/openserp/google" + "github.com/karust/openserp/yandex" +) + +// engineSpec is the single registry row for a search engine, driving CLI search, +// raw dispatch, serve's browserEngineSpecs, and the alias/validation strings. +// cfg points into the live config global; rawSearchFn is nil when an engine has +// no browserless mode. +type engineSpec struct { + name string + aliases []string + factory func(core.Browser, core.SearchEngineOptions) core.SearchEngine + rawSearchFn func(context.Context, core.Query) ([]core.SearchResult, error) + parseHTMLFn func(io.Reader) ([]core.SearchResult, error) + cfg *EngineConfig +} + +func (s engineSpec) opts() core.SearchEngineOptions { + return s.cfg.SearchEngineOptions +} + +func engineSpecs() []engineSpec { + return []engineSpec{ + {name: "google", factory: newEngine(google.New), rawSearchFn: google.Search, parseHTMLFn: google.ParseHTML, cfg: &config.GoogleConfig}, + {name: "yandex", factory: newEngine(yandex.New), rawSearchFn: yandex.Search, parseHTMLFn: yandex.ParseHTML, cfg: &config.YandexConfig}, + {name: "baidu", factory: newEngine(baidu.New), rawSearchFn: baidu.Search, parseHTMLFn: baidu.ParseHTML, cfg: &config.BaiduConfig}, + {name: "bing", factory: newEngine(bing.New), parseHTMLFn: bing.ParseHTML, cfg: &config.BingConfig}, + {name: "duckduckgo", aliases: []string{"duck", "ddg"}, factory: newEngine(duckduckgo.New), parseHTMLFn: duckduckgo.ParseHTML, cfg: &config.DuckDuckGoConfig}, + {name: "ecosia", factory: newEngine(ecosia.New), rawSearchFn: ecosia.Search, parseHTMLFn: ecosia.ParseHTML, cfg: &config.EcosiaConfig}, + } +} + +// newEngine adapts a concrete pkg.New (returning *Engine) to the +// core.SearchEngine-typed factory the registry stores. +func newEngine[T core.SearchEngine](ctor func(core.Browser, core.SearchEngineOptions) T) func(core.Browser, core.SearchEngineOptions) core.SearchEngine { + return func(b core.Browser, o core.SearchEngineOptions) core.SearchEngine { + return ctor(b, o) + } +} + +// engineValidArgs returns every accepted engine token (canonical names + +// aliases) for cobra's OnlyValidArgs validation. +func engineValidArgs() []string { + specs := engineSpecs() + args := make([]string, 0, len(specs)) + for _, s := range specs { + args = append(args, s.name) + args = append(args, s.aliases...) + } + return args +} + +// resolveEngineSpec returns the spec whose canonical name or alias matches raw +// (case/space already normalized by the caller), or false when unknown. +func resolveEngineSpec(raw string) (engineSpec, bool) { + for _, s := range engineSpecs() { + if s.name == raw { + return s, true + } + for _, alias := range s.aliases { + if alias == raw { + return s, true + } + } + } + return engineSpec{}, false +} diff --git a/cmd/root.go b/cmd/root.go index 482931a..703bbdb 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ import ( ) const ( - version = "0.8.3" + version = "0.8.4" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) @@ -51,6 +51,7 @@ type ServerConfig struct { ConfigPath string `mapstructure:"config_path"` IsDebug bool `mapstructure:"debug"` IsVerbose bool `mapstructure:"verbose"` + IsQuiet bool `mapstructure:"quiet"` IsRawRequests bool `mapstructure:"raw_requests"` Insecure bool `mapstructure:"insecure"` } @@ -115,6 +116,7 @@ var flagToConfigKey = map[string]string{ "profiles-json": "app.profiles", "verbose": "server.verbose", "debug": "server.debug", + "quiet": "server.quiet", "head": "app.head", "leakless": "app.leakless", "raw": "server.raw_requests", @@ -154,12 +156,24 @@ var RootCmd = &cobra.Command{ } config.App.LogFormat = logFormat - core.InitLogger(config.Server.IsVerbose, config.Server.IsDebug, config.App.LogFormat) + // One-shot CLI commands default to quiet so stdout is payload-only. + // Server mode keeps request logs unless server.quiet is set. + quiet := config.Server.IsQuiet + if commandDefaultsToQuiet(cmd) && !cmd.Flags().Changed("quiet") { + quiet = true + } + config.Server.IsQuiet = quiet + + core.InitLogger(config.Server.IsVerbose, config.Server.IsDebug, quiet, config.App.LogFormat) logrus.WithField("config", sanitizedConfigForLog(config)).Debug("Final config") return nil }, } +func commandDefaultsToQuiet(cmd *cobra.Command) bool { + return cmd != nil && cmd.Name() != serveCMD.Name() +} + func sanitizedConfigForLog(cfg Config) map[string]interface{} { return map[string]interface{}{ "server": cfg.Server, @@ -380,6 +394,7 @@ func setConfigDefaults(v *viper.Viper) { v.SetDefault("server.port", 7070) v.SetDefault("server.debug", false) v.SetDefault("server.verbose", false) + v.SetDefault("server.quiet", false) v.SetDefault("server.raw_requests", false) v.SetDefault("server.insecure", false) v.SetDefault("app.log_format", "") @@ -436,8 +451,9 @@ func init() { RootCmd.PersistentFlags().StringVar(&config.App.ProfilesJSON, "profiles", "", "Path to browser profile catalog JSON") RootCmd.PersistentFlags().BoolVarP(&config.Server.IsVerbose, "verbose", "v", false, "Use verbose output") RootCmd.PersistentFlags().BoolVarP(&config.Server.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser") + RootCmd.PersistentFlags().BoolVarP(&config.Server.IsQuiet, "quiet", "q", false, "Suppress info logs on stderr (default for CLI commands)") RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI") - RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search") + RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to ensure browser instances are closed after search") RootCmd.PersistentFlags().BoolVarP(&config.Server.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests") RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made") RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key") diff --git a/cmd/search.go b/cmd/search.go index 2e67ecb..b59f77a 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -4,42 +4,91 @@ import ( "context" "encoding/json" "fmt" - "os" + "net/url" "strings" + "sync" "time" - "github.com/karust/openserp/baidu" - "github.com/karust/openserp/bing" + "github.com/google/uuid" "github.com/karust/openserp/core" - "github.com/karust/openserp/duckduckgo" - "github.com/karust/openserp/ecosia" - "github.com/karust/openserp/google" - "github.com/karust/openserp/yandex" + extractpkg "github.com/karust/openserp/extract" "github.com/sirupsen/logrus" "github.com/spf13/cobra" ) -var searchCMD = &cobra.Command{ - Use: "search", - Aliases: []string{"find"}, - Short: "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo, ecosia)", - Args: cobra.MatchAll(cobra.OnlyValidArgs, cobra.ExactArgs(2)), - Run: search, +// searchFlags holds the per-invocation CLI flags for the search command. +type searchFlags struct { + limit int + lang string + region string + start int + site string + filetype string + format string + full bool + features bool + extract int + timeout int } -func search(cmd *cobra.Command, args []string) { +var searchOpts searchFlags + +var searchCMD = &cobra.Command{ + Use: "search [engine] [query]", + Aliases: []string{"find"}, + Short: "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo, ecosia)", + // Validate the engine ourselves; cobra.OnlyValidArgs would also reject the + // query arg. ValidArgs still feeds shell completion. + Args: cobra.MatchAll(cobra.ExactArgs(2), validateEngineArg), + ValidArgs: engineValidArgs(), + RunE: search, +} + +// validateEngineArg checks args[0] against the registry with a clear error, +// without rejecting the query arg. +func validateEngineArg(cmd *cobra.Command, args []string) error { + if len(args) == 0 { + return nil + } + if _, ok := resolveEngineSpec(normalizeEngineArg(args[0])); !ok { + return fmt.Errorf("unknown engine %q; valid: %s", args[0], strings.Join(engineValidArgs(), ", ")) + } + return nil +} + +func search(cmd *cobra.Command, args []string) error { + // Already validated by validateEngineArg, so this can't miss. engineType := normalizeEngineArg(args[0]) + spec, _ := resolveEngineSpec(engineType) + + format, err := normalizeSearchFormat(searchOpts.format) + if err != nil { + return err + } + + limit := searchOpts.limit + if limit <= 0 { + limit = 10 + } query := core.Query{ Text: args[1], - Limit: 10, + LangCode: searchOpts.lang, + Region: searchOpts.region, + Site: searchOpts.site, + Filetype: searchOpts.filetype, + Limit: limit, + Start: searchOpts.start, Filter: true, + Features: searchOpts.features, Insecure: config.Server.Insecure, } + if err := applyCLIExtractFlag(&query, searchOpts.extract); err != nil { + return err + } captchaSolverEnabled, captchaSolverAPIKey, err := resolveCaptchaSolverConfig() if err != nil { - logrus.WithError(err).Error(fmt.Sprintf("Error validating captcha solver config: %v", err)) - os.Exit(1) + return fmt.Errorf("validate captcha solver config: %w", err) } proxyRuntime := core.ProxyRuntimeBrowser @@ -49,39 +98,45 @@ func search(cmd *cobra.Command, args []string) { proxyCfg, err := buildNormalizedProxyConfig(proxyRuntime) if err != nil { - logrus.WithError(err).Error(fmt.Sprintf("Error validating proxy config: %v", err)) - return + return fmt.Errorf("validate proxy config: %w", err) } policy := resolveEngineProxyPolicy(proxyCfg, engineType) selectedProxy, err := selectCLIProxy(proxyCfg, policy) if err != nil { - logrus.WithError(err).Error(fmt.Sprintf("Error selecting proxy for %s: %v", engineType, err)) - return + return fmt.Errorf("select proxy for %s: %w", engineType, err) } if config.Server.IsRawRequests { query.ProxyURL = selectedProxy } + // Bound the whole search so a wedged Chrome can't hang the CLI forever. + timeoutSec := searchOpts.timeout + if timeoutSec <= 0 { + timeoutSec = 60 + } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSec)*time.Second) + defer cancel() + logrus.WithFields(logrus.Fields{ "engine": engineType, "query_hash": core.QueryHashFromQuery(query), }).Info(fmt.Sprintf("Starting SERP search request using %s engine for query: %s", engineType, query.Text)) + startedAt := time.Now() var results []core.SearchResult if config.Server.IsRawRequests { logrus.WithField("engine", engineType).Info(fmt.Sprintf("Using raw requests mode for %s search", engineType)) - results, err = searchRaw(engineType, query) + results, err = searchRaw(ctx, spec, query) } else { logrus.WithField("engine", engineType).Info(fmt.Sprintf("Using browser mode for %s search", engineType)) - results, err = searchBrowser(engineType, query, selectedProxy, captchaSolverEnabled, captchaSolverAPIKey) + results, err = searchBrowser(ctx, spec, query, selectedProxy, captchaSolverEnabled, captchaSolverAPIKey) } if err != nil { - logrus.WithError(err).WithField("engine", engineType).Error(fmt.Sprintf("Error during %s search: %s", engineType, err)) - return + return fmt.Errorf("%s search: %w", engineType, err) } logrus.WithFields(logrus.Fields{ @@ -89,17 +144,203 @@ func search(cmd *cobra.Command, args []string) { "results_count": len(results), }).Info(fmt.Sprintf("Successfully completed SERP search using %s engine, returned %d results", engineType, len(results))) - b, err := json.MarshalIndent(results, "", " ") - if err != nil { - logrus.Error(err) - return + env := buildCLIEnvelope(spec.name, query, results, startedAt) + if query.Extract { + if err := enrichCLIEnvelopeWithExtraction(ctx, env, query, format, selectedProxy, captchaSolverEnabled, captchaSolverAPIKey); err != nil { + return fmt.Errorf("extract search results: %w", err) + } } - - fmt.Println(string(b)) + payload := renderCLIEnvelope(env, format, searchOpts.full) + fmt.Println(strings.TrimRight(string(payload), "\n")) + return nil } -func searchBrowser(engineType string, query core.Query, browserProxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) ([]core.SearchResult, error) { - var engine core.SearchEngine +func buildCLIEnvelope(engineName string, query core.Query, results []core.SearchResult, startedAt time.Time) *core.Envelope { + env := core.NewEnvelope(query, uuid.NewString(), startedAt, []string{engineName}) + ectx := core.EnrichContext{Engine: engineName, Query: query} + for _, r := range results { + core.AppendEnrichedSearchResult(env, r, ectx, startedAt) + } + env.Finalize(startedAt, query) + return env +} + +// renderCLIEnvelope renders a v2.1 envelope. JSON/ndjson always carry the full +// envelope; text/markdown omit serp_features unless --full. +func renderCLIEnvelope(env *core.Envelope, format string, full bool) []byte { + if !full && format != "json" && format != "ndjson" { + env.SerpFeatures = nil + } + + switch format { + case "text": + return core.RenderText(env) + case "markdown": + return core.RenderMarkdown(env) + case "ndjson": + return core.RenderNDJSON(env) + default: // json + b, err := json.MarshalIndent(env, "", " ") + if err != nil { + logrus.WithError(err).Error("marshal envelope") + return nil + } + return b + } +} + +const maxCLIExtractTop = 5 + +func applyCLIExtractFlag(query *core.Query, extractTop int) error { + top, err := normalizeCLIExtractTop(extractTop) + if err != nil { + return err + } + if top == 0 { + return nil + } + if !config.Extract.Enabled { + return fmt.Errorf("extraction is disabled in config") + } + query.Extract = true + query.ExtractTop = top + query.ExtractMode = string(extractpkg.ModeAuto) + return nil +} + +func normalizeCLIExtractTop(raw int) (int, error) { + if raw < 0 { + return 0, fmt.Errorf("--extract must be a non-negative integer") + } + if raw > maxCLIExtractTop { + return maxCLIExtractTop, nil + } + return raw, nil +} + +func enrichCLIEnvelopeWithExtraction(ctx context.Context, env *core.Envelope, query core.Query, format string, proxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) error { + if env == nil || !query.Extract { + return nil + } + query.ProxyURL = proxyURL + extractor, closeExtractor, err := newCLIExtractor(captchaSolverEnabled, captchaSolverAPIKey) + if err != nil { + return err + } + defer closeExtractor() + + // Same depth bounds, batch deadline, and candidate fill-in as the HTTP server. + core.EnrichEnvelopeWithExtraction(ctx, env, query, format, extractor, config.Extract) + return nil +} + +// newCLIExtractor builds an Extractor backed by a lazily-created, single-use +// browser. The raw path delegates to core.RawExtractFetch; the rendered path +// validates the target, gates auth'd SOCKS, then reuses core.RenderExtractHTML. +func newCLIExtractor(captchaSolverEnabled bool, captchaSolverAPIKey string) (extractpkg.Extractor, func(), error) { + cfg := config.Extract.Normalized() + var browserMu sync.Mutex + var browser *core.Browser + + closeExtractor := func() { + browserMu.Lock() + defer browserMu.Unlock() + if browser == nil { + return + } + if err := browser.Close(); err != nil { + logrus.WithError(err).Debug("Extraction browser close error") + } + browser = nil + } + + extractor := extractpkg.Extractor{ + Cfg: cfg, + RawFetch: func(ctx context.Context, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) { + return core.RawExtractFetch(ctx, req, cfg, config.Server.Insecure) + }, + RenderedFetch: func(ctx context.Context, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) { + if err := validateCLIExtractTargetURL(ctx, req.URL, cfg.AllowPrivateNetworks); err != nil { + return nil, err + } + if core.IsAuthenticatedSocksProxyURL(req.ProxyURL) { + return nil, fmt.Errorf( + "%w: browser runtime does not support authenticated SOCKS proxy %s", + core.ErrProxyUnavailable, + core.MaskProxyURL(req.ProxyURL), + ) + } + + browserMu.Lock() + if browser == nil { + created, err := newCLIExtractBrowser(cfg, req.ProxyURL, captchaSolverEnabled, captchaSolverAPIKey) + if err != nil { + browserMu.Unlock() + return nil, err + } + browser = created + } + current := browser + browserMu.Unlock() + + return core.RenderExtractHTML(ctx, current, req) + }, + } + return extractor, closeExtractor, nil +} + +func newCLIExtractBrowser(cfg extractpkg.Config, proxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) (*core.Browser, error) { + blockedResourceTypes, err := core.ParseBlockedResourceTypes(config.App.BlockResources) + if err != nil { + return nil, fmt.Errorf("invalid block_resources config: %w", err) + } + opts := core.BrowserOpts{ + IsHeadless: !config.App.IsBrowserHead && !config.Server.IsDebug, + IsLeakless: config.App.IsLeakless, + Timeout: cfg.Timeout, + LeavePageOpen: false, + CaptchaSolverEnabled: captchaSolverEnabled, + CaptchaSolverApiKey: captchaSolverAPIKey, + BrowserPath: config.App.BrowserPath, + ProxyURL: proxyURL, + Insecure: config.Server.Insecure, + BlockResourceTypes: blockedResourceTypes, + BlockTrackers: config.App.BlockTrackers, + } + return core.NewBrowser(opts) +} + +func validateCLIExtractTargetURL(ctx context.Context, rawURL string, allowPrivateNetworks bool) error { + targetURL := extractpkg.NormalizeURL(strings.TrimSpace(rawURL)) + if allowPrivateNetworks { + parsed, err := url.ParseRequestURI(targetURL) + if err != nil { + return fmt.Errorf("invalid url: %w", err) + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return fmt.Errorf("url must use http or https") + } + return nil + } + return core.ValidatePublicHTTPURL(ctx, targetURL) +} + +func normalizeSearchFormat(raw string) (string, error) { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "", "json": + return "json", nil + case "text", "txt": + return "text", nil + case "markdown", "md": + return "markdown", nil + case "ndjson", "jsonl": + return "ndjson", nil + default: + return "", fmt.Errorf("invalid --format %q; valid: json, text, markdown, ndjson", raw) + } +} + +func searchBrowser(ctx context.Context, spec engineSpec, query core.Query, browserProxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) ([]core.SearchResult, error) { blockedResourceTypes, err := core.ParseBlockedResourceTypes(config.App.BlockResources) if err != nil { return nil, fmt.Errorf("invalid block_resources config: %w", err) @@ -134,49 +375,24 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string, if err != nil { return nil, err } + // Close the browser so Chromium never outlives the CLI run. + defer func() { + if closeErr := browser.Close(); closeErr != nil { + logrus.WithError(closeErr).Debug("Browser close error") + } + }() - switch strings.ToLower(engineType) { - case "yandex": - engine = yandex.New(*browser, config.YandexConfig.SearchEngineOptions) - case "google": - engine = google.New(*browser, config.GoogleConfig.SearchEngineOptions) - case "baidu": - engine = baidu.New(*browser, config.BaiduConfig.SearchEngineOptions) - case "bing": - engine = bing.New(*browser, config.BingConfig.SearchEngineOptions) - case "duckduckgo": - engine = duckduckgo.New(*browser, config.DuckDuckGoConfig.SearchEngineOptions) - case "ecosia": - engine = ecosia.New(*browser, config.EcosiaConfig.SearchEngineOptions) - default: - return nil, fmt.Errorf("no %q search engine found", engineType) - } - - return engine.Search(context.Background(), query) + engine := spec.factory(*browser, spec.opts()) + return engine.Search(ctx, query) } -func searchRaw(engineType string, query core.Query) ([]core.SearchResult, error) { +func searchRaw(ctx context.Context, spec engineSpec, query core.Query) ([]core.SearchResult, error) { logrus.Warn("Browserless results are very inconsistent or may not even work!") - ctx := context.Background() - - switch strings.ToLower(engineType) { - case "yandex": - return yandex.Search(ctx, query) - case "google": - return google.Search(ctx, query) - case "baidu": - return baidu.Search(ctx, query) - case "ecosia": - return ecosia.Search(ctx, query) - case "bing": - logrus.Warn("Bing does not support raw HTTP requests mode. Please use browser mode instead.") - return nil, fmt.Errorf("bing does not support raw requests mode") - case "duckduckgo": - logrus.Warn("DuckDuckGo does not support raw HTTP requests mode. Please use browser mode instead.") - return nil, fmt.Errorf("duckduckgo does not support raw requests mode") - default: - return nil, fmt.Errorf("no %q search engine found", engineType) + if spec.rawSearchFn == nil { + logrus.Warnf("%s does not support raw HTTP requests mode. Please use browser mode instead.", spec.name) + return nil, fmt.Errorf("%s does not support raw requests mode", spec.name) } + return spec.rawSearchFn(ctx, query) } func selectCLIProxy(proxyCfg core.ProxyConfig, policy core.ProxyPolicy) (string, error) { @@ -201,14 +417,20 @@ func selectCLIProxy(proxyCfg core.ProxyConfig, policy core.ProxyPolicy) (string, } func normalizeEngineArg(raw string) string { - switch strings.ToLower(strings.TrimSpace(raw)) { - case "duck": - return "duckduckgo" - default: - return strings.ToLower(strings.TrimSpace(raw)) - } + return strings.ToLower(strings.TrimSpace(raw)) } func init() { + searchCMD.Flags().IntVar(&searchOpts.limit, "limit", 10, "Maximum number of results") + searchCMD.Flags().StringVar(&searchOpts.lang, "lang", "", "Language hint (e.g. EN, DE, RU)") + searchCMD.Flags().StringVar(&searchOpts.region, "region", "", "Region/market hint (e.g. RU, en-US)") + searchCMD.Flags().IntVar(&searchOpts.start, "start", 0, "Pagination start offset") + searchCMD.Flags().StringVar(&searchOpts.site, "site", "", "Restrict results to a domain (e.g. github.com)") + searchCMD.Flags().StringVar(&searchOpts.filetype, "file", "", "File type filter (e.g. pdf)") + searchCMD.Flags().StringVar(&searchOpts.format, "format", "json", "Output format: json, text, markdown, ndjson") + searchCMD.Flags().BoolVar(&searchOpts.full, "full", false, "Include SERP features in text/markdown output") + searchCMD.Flags().BoolVar(&searchOpts.features, "features", false, "Parse SERP feature modules (browser mode)") + searchCMD.Flags().IntVar(&searchOpts.extract, "extract", 0, "Extract clean content from the top N results using auto mode (1-5)") + searchCMD.Flags().IntVar(&searchOpts.timeout, "search-timeout", 60, "Overall search timeout in seconds") RootCmd.AddCommand(searchCMD) } diff --git a/cmd/search_test.go b/cmd/search_test.go new file mode 100644 index 0000000..5ee7e4a --- /dev/null +++ b/cmd/search_test.go @@ -0,0 +1,82 @@ +package cmd + +import ( + "strings" + "testing" + + "github.com/karust/openserp/core" + extractpkg "github.com/karust/openserp/extract" +) + +func TestNormalizeCLIExtractTop(t *testing.T) { + tests := []struct { + name string + raw int + want int + wantErr bool + }{ + {name: "disabled", raw: 0, want: 0}, + {name: "one", raw: 1, want: 1}, + {name: "clamped", raw: 20, want: maxCLIExtractTop}, + {name: "negative", raw: -1, wantErr: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := normalizeCLIExtractTop(tc.raw) + if tc.wantErr { + if err == nil { + t.Fatal("expected error") + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tc.want { + t.Fatalf("normalizeCLIExtractTop(%d) = %d, want %d", tc.raw, got, tc.want) + } + }) + } +} + +func TestApplyCLIExtractFlagSetsAutoMode(t *testing.T) { + previous := config + config.Extract = extractpkg.DefaultConfig() + defer func() { config = previous }() + + query := core.Query{Text: "weather today"} + if err := applyCLIExtractFlag(&query, 2); err != nil { + t.Fatalf("applyCLIExtractFlag() error = %v", err) + } + if !query.Extract { + t.Fatal("expected query.Extract") + } + if query.ExtractTop != 2 { + t.Fatalf("ExtractTop = %d, want 2", query.ExtractTop) + } + if query.ExtractMode != string(extractpkg.ModeAuto) { + t.Fatalf("ExtractMode = %q, want auto", query.ExtractMode) + } +} + +func TestApplyCLIExtractFlagRequiresEnabledConfig(t *testing.T) { + previous := config + config.Extract = extractpkg.Config{Enabled: false} + defer func() { config = previous }() + + query := core.Query{Text: "weather today"} + err := applyCLIExtractFlag(&query, 1) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "disabled") { + t.Fatalf("error = %q, want disabled message", err.Error()) + } +} + +func TestSearchCommandHasExtractFlag(t *testing.T) { + if searchCMD.Flags().Lookup("extract") == nil { + t.Fatal("expected search command to expose --extract") + } +} diff --git a/cmd/serve.go b/cmd/serve.go index aa0d067..d39f385 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -13,13 +13,7 @@ import ( "syscall" "time" - "github.com/karust/openserp/baidu" - "github.com/karust/openserp/bing" "github.com/karust/openserp/core" - "github.com/karust/openserp/duckduckgo" - "github.com/karust/openserp/ecosia" - "github.com/karust/openserp/google" - "github.com/karust/openserp/yandex" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/time/rate" @@ -35,18 +29,11 @@ type rawEngine struct { func (r *rawEngine) Search(ctx context.Context, q core.Query) ([]core.SearchResult, error) { q.Insecure = config.Server.Insecure - switch r.name { - case "google": - return google.Search(ctx, q) - case "yandex": - return yandex.Search(ctx, q) - case "baidu": - return baidu.Search(ctx, q) - case "ecosia": - return ecosia.Search(ctx, q) - default: + spec, ok := resolveEngineSpec(r.name) + if !ok || spec.rawSearchFn == nil { return nil, fmt.Errorf("unsupported engine: %s", r.name) } + return spec.rawSearchFn(ctx, q) } func (r *rawEngine) SearchImage(_ context.Context, _ core.Query) ([]core.SearchResult, error) { @@ -626,56 +613,17 @@ type browserEngineSpec struct { } func browserEngineSpecs() []browserEngineSpec { - return []browserEngineSpec{ - { - name: "google", - opts: config.GoogleConfig.SearchEngineOptions, - factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { - return google.New(browser, opts) - }, - parseHTMLFn: google.ParseHTML, - }, - { - name: "yandex", - opts: config.YandexConfig.SearchEngineOptions, - factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { - return yandex.New(browser, opts) - }, - parseHTMLFn: yandex.ParseHTML, - }, - { - name: "baidu", - opts: config.BaiduConfig.SearchEngineOptions, - factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { - return baidu.New(browser, opts) - }, - parseHTMLFn: baidu.ParseHTML, - }, - { - name: "bing", - opts: config.BingConfig.SearchEngineOptions, - factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { - return bing.New(browser, opts) - }, - parseHTMLFn: bing.ParseHTML, - }, - { - name: "duckduckgo", - opts: config.DuckDuckGoConfig.SearchEngineOptions, - factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { - return duckduckgo.New(browser, opts) - }, - parseHTMLFn: duckduckgo.ParseHTML, - }, - { - name: "ecosia", - opts: config.EcosiaConfig.SearchEngineOptions, - factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { - return ecosia.New(browser, opts) - }, - parseHTMLFn: ecosia.ParseHTML, - }, + specs := engineSpecs() + out := make([]browserEngineSpec, 0, len(specs)) + for _, s := range specs { + out = append(out, browserEngineSpec{ + name: s.name, + opts: s.opts(), + factory: s.factory, + parseHTMLFn: s.parseHTMLFn, + }) } + return out } func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ([]core.SearchEngine, func() error, core.BrowserResolver, error) { diff --git a/cmd/serve_test.go b/cmd/serve_test.go index 81e3390..65c3e42 100644 --- a/cmd/serve_test.go +++ b/cmd/serve_test.go @@ -19,6 +19,15 @@ func TestRawEngineCachesRateLimiter(t *testing.T) { } } +func TestCommandDefaultsToQuiet(t *testing.T) { + if !commandDefaultsToQuiet(searchCMD) { + t.Fatal("expected search command to default to quiet") + } + if commandDefaultsToQuiet(serveCMD) { + t.Fatal("expected serve command to keep server logging by default") + } +} + func TestBrowserPoolKey(t *testing.T) { cases := []struct { name string diff --git a/config.yaml b/config.yaml index bd0f051..27e4639 100644 --- a/config.yaml +++ b/config.yaml @@ -2,7 +2,7 @@ server: host: 0.0.0.0 # API host to bind port: 7000 # API port to bind debug: false # Enable debug logs and force browser UI mode - verbose: true # Enable info-level request logs + verbose: false # Enable debug-level request logs raw_requests: false # true = raw HTTP mode, false = browser mode insecure: true # Allow insecure TLS connections @@ -18,8 +18,6 @@ app: idle_ttl: 5m # close a Chrome that has not served traffic for this long mega_timeout: 90s # max total wait for /mega/* requests; slow engines return partial results - block_trackers: true - block_resources: "image,font,css,media" extract: enabled: true diff --git a/core/browser.go b/core/browser.go index 408b9e2..6f137c1 100644 --- a/core/browser.go +++ b/core/browser.go @@ -252,8 +252,19 @@ type browserConnection struct { laneProfiles map[string]browserprofile.Profile authCancel context.CancelFunc authStopped chan struct{} + // lastOK is when a CDP call last succeeded; the health ping is skipped while + // it is within healthPingSkipWindow. + lastOK time.Time } +const ( + // healthPingTimeout bounds the per-call connection ping so a wedged Chrome + // can't stall navigations while holding the connection lock. + healthPingTimeout = 3 * time.Second + // healthPingSkipWindow skips the ping when a CDP call succeeded this recently. + healthPingSkipWindow = 5 * time.Second +) + // NewBrowser launches a new Chromium process via Rod launcher and returns a // Browser wrapper configured with proxy and captcha solver settings. func NewBrowser(opts BrowserOpts) (*Browser, error) { @@ -272,6 +283,9 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) { // headless=new uses the full Chrome renderer; legacy --headless disables the // GPU process entirely, making WebGL context creation fail even with SwiftShader. // use-angle=swiftshader-webgl (Chrome β‰₯112) enables a software WebGL renderer. + // Rod enables leakless by default, so always pass the configured value + // through. OpenSERP defaults it to false because the helper binary is + // commonly flagged by antivirus on Windows. l := launcher.New().Leakless(opts.IsLeakless). Set("disable-blink-features", "AutomationControlled"). Delete("enable-automation"). @@ -557,18 +571,18 @@ func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect boo return nil, err } state.browser = connected + state.lastOK = time.Now() return state.browser, nil } - // Bound just this health-ping with a per-call timeout so a wedged browser - // can't block the connection lock forever. Use a fresh derived context each - // call (not browser.Timeout, which would bake a permanent deadline into the - // persistent connection β€” see newRodBrowser). - pingTimeout := b.Timeout - if pingTimeout <= 0 { - pingTimeout = 30 * time.Second + // A recent successful CDP call means the connection is alive; skip the ping. + if !state.lastOK.IsZero() && time.Since(state.lastOK) < healthPingSkipWindow { + return state.browser, nil } - pingCtx, cancelPing := context.WithTimeout(EnsureContext(ctx), pingTimeout) + + // Fresh derived context per call (not browser.Timeout, which would bake a + // permanent deadline into the persistent connection β€” see newRodBrowser). + pingCtx, cancelPing := context.WithTimeout(EnsureContext(ctx), healthPingTimeout) _, pingErr := state.browser.Context(pingCtx).Version() cancelPing() if pingErr != nil { @@ -579,6 +593,7 @@ func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect boo } state.browser = connected } + state.lastOK = time.Now() return state.browser, nil } @@ -1503,20 +1518,24 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) { wait() } - // WaitStable internally waits for page load too, so cap it separately. - // Selector parsing still decides whether a partially loaded page is usable. - stableWaitTimeout := minPositiveDuration(b.Timeout, b.WaitLoadTime+time.Second) - if err := page.Timeout(stableWaitTimeout).WaitStable(800 * time.Millisecond); err != nil { - WithRequest(ctx).WithError(err).Debug("WaitStable returned early; continuing") - } if err := classifyMainDocumentStatus(statusWatcher.Status()); err != nil { closeOnErr() return nil, err } b.saveLaneCookies(ctx, page, URL) + b.markConnectionOK() return page, nil } +// markConnectionOK records a successful CDP round trip so the next +// ensureConnectedBrowser can skip its health ping (see healthPingSkipWindow). +func (b *Browser) markConnectionOK() { + state := b.connectionState() + state.mu.Lock() + state.lastOK = time.Now() + state.mu.Unlock() +} + // Close closes the active browser connection. func (b *Browser) Close() error { if b == nil || b.browserAddr == "" { diff --git a/core/common.go b/core/common.go index 6ce02e9..80f096b 100644 --- a/core/common.go +++ b/core/common.go @@ -15,6 +15,14 @@ import ( "golang.org/x/time/rate" ) +// Extraction depth bounds for the unified extract=N query param. The default +// is 1 (extract=true == extract=1 == "extract one result"); callers raise it up +// to maxExtractTop. These mirror the CLI's --extract flag limits. +const ( + defaultExtractTop = 1 + maxExtractTop = 5 +) + // ErrCaptcha is returned when the engine detects a captcha challenge page. // This error is treated as non-retryable by resilient search policies. var ErrCaptcha = errors.New("captcha detected") @@ -385,33 +393,12 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { if err != nil { return errInvalidParam(fmt.Sprintf("features: %v", err)) } - searchQuery.Extract, err = strconv.ParseBool(reqCtx.Query("extract", "0")) - if err != nil { - return errInvalidParam(fmt.Sprintf("extract: %v", err)) - } - searchQuery.ExtractTop = 3 - if raw := strings.TrimSpace(reqCtx.Query("extract_top")); raw != "" { - extractTop, err := strconv.Atoi(raw) - if err != nil { - return errInvalidParam("extract_top must be an integer") - } - if extractTop < 1 { - extractTop = 1 - } - if extractTop > 5 { - extractTop = 5 - } - searchQuery.ExtractTop = extractTop - } - searchQuery.ExtractMode = strings.ToLower(strings.TrimSpace(reqCtx.Query("extract_mode", "auto"))) - switch searchQuery.ExtractMode { - case "auto", "fast", "rendered": - default: - return errInvalidParam("extract_mode must be one of auto, fast, rendered") - } - searchQuery.ExtractMinRunes, err = parseNonNegativeIntQuery(reqCtx.Query("min_runes"), 0) - if err != nil { - return errInvalidParam("min_runes must be a non-negative integer") + // extract is a unified bool-or-int knob: extract=0/false disables, extract=N + // (or true/1) extracts the top N results. The tuning params extract_mode and + // min_runes also imply extraction (extract=0 still overrides them). The + // default depth is 1 β€” true == 1 == "extract one result". + if err := parseExtractParams(reqCtx, searchQuery); err != nil { + return err } searchQuery.ProxyOverride, err = NormalizeProxyRequestOverride(reqCtx.Get("X-Use-Proxy")) @@ -437,6 +424,72 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { return nil } +// parseExtractParams reads the unified extract knob plus its tuning params onto +// q. The extract param is bool-or-int: +// +// extract=0 / extract=false β†’ extraction off +// extract=true / extract=1 β†’ on, top 1 +// extract=N (1..5) β†’ on, top N (clamped to maxExtractTop) +// +// extract_mode and min_runes tune how extraction runs and imply extraction when +// present, unless extract is explicitly set (extract=0 wins over them). When +// extraction is on but no depth is given, ExtractTop defaults to 1. +func parseExtractParams(reqCtx *fiber.Ctx, q *Query) error { + q.ExtractTop = defaultExtractTop + + // extract accepts both bool spellings (true/false/1/0) and an integer depth. + // Try bool first so legacy true/false keep working, then fall back to int. + extractExplicit := false + if raw := strings.TrimSpace(reqCtx.Query("extract")); raw != "" { + extractExplicit = true + if b, err := strconv.ParseBool(raw); err == nil { + q.Extract = b + if b { + q.ExtractTop = 1 + } + } else if n, err := strconv.Atoi(raw); err == nil { + q.Extract = n > 0 + if n > 0 { + q.ExtractTop = clampExtractTop(n) + } + } else { + return errInvalidParam("extract must be a boolean or an integer (0 disables, N extracts top N)") + } + } + + q.ExtractMode = strings.ToLower(strings.TrimSpace(reqCtx.Query("extract_mode", "auto"))) + switch q.ExtractMode { + case "auto", "fast", "rendered": + default: + return errInvalidParam("extract_mode must be one of auto, fast, rendered") + } + if !extractExplicit && strings.TrimSpace(reqCtx.Query("extract_mode")) != "" { + q.Extract = true + } + + minRunes, err := parseNonNegativeIntQuery(reqCtx.Query("min_runes"), 0) + if err != nil { + return errInvalidParam("min_runes must be a non-negative integer") + } + q.ExtractMinRunes = minRunes + if !extractExplicit && strings.TrimSpace(reqCtx.Query("min_runes")) != "" { + q.Extract = true + } + + return nil +} + +// clampExtractTop bounds a requested extraction depth to [1, maxExtractTop]. +func clampExtractTop(n int) int { + if n < 1 { + return 1 + } + if n > maxExtractTop { + return maxExtractTop + } + return n +} + // SearchEngineOptions controls engine pacing, selector waits, and captcha // handling behavior shared by browser and raw implementations. type SearchEngineOptions struct { diff --git a/core/logger.go b/core/logger.go index 342f4f7..48fd374 100644 --- a/core/logger.go +++ b/core/logger.go @@ -251,7 +251,7 @@ func quoteIfNeeded(s string) string { return s } -func InitLogger(isVerbose, isDebug bool, format string) { +func InitLogger(isVerbose, isDebug, isQuiet bool, format string) { switch format { case LogFormatText: logrus.SetFormatter(&bracketFormatter{TimestampFormat: "2006-01-02 15:04:05"}) @@ -261,16 +261,22 @@ func InitLogger(isVerbose, isDebug bool, format string) { }) } - if isDebug { - logrus.SetOutput(io.MultiWriter(os.Stdout)) + // Logs go to stderr (+ optional file) so stdout carries only the payload. + switch { + case isDebug: + logrus.SetOutput(io.MultiWriter(os.Stderr)) logrus.SetReportCaller(true) - } else { + case isQuiet: + // One-shot CLI default: stderr only, no ./logs.txt in the user's CWD. + logrus.SetOutput(os.Stderr) + logrus.SetReportCaller(false) + default: f, err := os.OpenFile("./logs.txt", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) if err != nil { fmt.Fprintf(os.Stderr, "Failed to open logs file ./logs.txt: %v\n", err) - logrus.SetOutput(io.MultiWriter(os.Stdout)) + logrus.SetOutput(io.MultiWriter(os.Stderr)) } else { - logrus.SetOutput(io.MultiWriter(f, os.Stdout)) + logrus.SetOutput(io.MultiWriter(f, os.Stderr)) } logrus.SetReportCaller(false) } @@ -282,5 +288,9 @@ func InitLogger(isVerbose, isDebug bool, format string) { if isDebug { level = logrus.TraceLevel } + if isQuiet && !isVerbose && !isDebug { + // Quiet keeps only warnings/errors on stderr. + level = logrus.WarnLevel + } logrus.SetLevel(level) } diff --git a/core/page_helpers.go b/core/page_helpers.go index d6e81f0..8e2ccdb 100644 --- a/core/page_helpers.go +++ b/core/page_helpers.go @@ -102,8 +102,49 @@ func HasAttribute(el *rod.Element, attr string) bool { return err == nil && v != nil } -// FirstNonEmptyText returns the trimmed text of the first selector under root -// that yields non-empty content. Empty string if none match. +// NormalizeWhitespace collapses runs of whitespace (newlines, source +// indentation) into single spaces and trims the result. +func NormalizeWhitespace(s string) string { + return strings.Join(strings.Fields(s), " ") +} + +// ElementText returns el's visible text, falling back to textContent (for nodes +// rod's Text() leaves empty), normalized. Empty string if el is nil or blank. +func ElementText(el *rod.Element) string { + if el == nil { + return "" + } + if text, err := el.Text(); err == nil { + if normalized := NormalizeWhitespace(text); normalized != "" { + return normalized + } + } + if value, err := el.Property("textContent"); err == nil { + return NormalizeWhitespace(value.String()) + } + return "" +} + +// ElementAttribute returns the first non-empty value among attrs on el, +// normalized. Empty string if el is nil or none are set. +func ElementAttribute(el *rod.Element, attrs ...string) string { + if el == nil { + return "" + } + for _, attr := range attrs { + value, err := el.Attribute(attr) + if err != nil || value == nil { + continue + } + if normalized := NormalizeWhitespace(*value); normalized != "" { + return normalized + } + } + return "" +} + +// FirstNonEmptyText returns the text (see ElementText) of the first selector +// under root that yields non-empty content. Empty string if none match. func FirstNonEmptyText(root *rod.Element, selectors ...string) string { if root == nil { return "" @@ -113,12 +154,8 @@ func FirstNonEmptyText(root *rod.Element, selectors ...string) string { if err != nil { continue } - text, err := el.Text() - if err != nil { - continue - } - if trimmed := strings.TrimSpace(text); trimmed != "" { - return trimmed + if text := ElementText(el); text != "" { + return text } } return "" diff --git a/core/proxy.go b/core/proxy.go index 3201bbe..9a223c5 100644 --- a/core/proxy.go +++ b/core/proxy.go @@ -24,6 +24,10 @@ const ( // ProxyPoolQuarantineDuration is how long an exhausted tag pool stays quarantined // before a single probe proxy is re-enabled for recovery testing. ProxyPoolQuarantineDuration = 5 * time.Minute + // ProxyChallengeCooldown is how long a captcha/blocked proxy is deprioritized + // in rotation. It does not degrade health, so the proxy is still served if + // it's the only one left. + ProxyChallengeCooldown = 2 * time.Minute ) var supportedProxySchemes = map[string]struct{}{ @@ -99,6 +103,9 @@ type proxyState struct { tags []string failures int disabled bool + // challengedUntil deprioritizes (but does not disable) this proxy in + // rotation after a captcha/block. See ReportChallenged. + challengedUntil time.Time } type ProxyRegistry struct { @@ -411,21 +418,29 @@ func (r *ProxyRegistry) NextByTagWithContext(ctx context.Context, tag string) st } } + now := time.Now() start := r.nextByTag[tag] - for i := 0; i < len(urls); i++ { - idx := (start + i) % len(urls) - proxyURL := urls[idx] - state := r.states[proxyURL] - if state.disabled { - continue - } + // First pass skips challenged proxies; second pass relaxes that so a + // challenged-but-healthy proxy is still served rather than failing. + for _, skipChallenged := range []bool{true, false} { + for i := 0; i < len(urls); i++ { + idx := (start + i) % len(urls) + proxyURL := urls[idx] + state := r.states[proxyURL] + if state.disabled { + continue + } + if skipChallenged && now.Before(state.challengedUntil) { + continue + } - r.nextByTag[tag] = (idx + 1) % len(urls) - WithRequest(ctx).WithFields(logrus.Fields{ - "proxy_tag": tag, - "proxy": MaskProxyURL(proxyURL), - }).Debugf("Selected proxy for tag=%s: %s", tag, MaskProxyURL(proxyURL)) - return proxyURL + r.nextByTag[tag] = (idx + 1) % len(urls) + WithRequest(ctx).WithFields(logrus.Fields{ + "proxy_tag": tag, + "proxy": MaskProxyURL(proxyURL), + }).Debugf("Selected proxy for tag=%s: %s", tag, MaskProxyURL(proxyURL)) + return proxyURL + } } // All proxies are disabled and no probe could be selected. @@ -498,22 +513,50 @@ func (r *ProxyRegistry) ReportSuccess(_ context.Context, proxyURL string) { } } -func (r *ProxyRegistry) HasHealthyProxyForTag(tag string) bool { - tag = normalizeTag(tag) - if tag == "" { - return false +// ReportChallenged deprioritizes a captcha/blocked proxy for +// ProxyChallengeCooldown without degrading its health (unlike ReportFailure, it +// never disables the proxy or trips quarantine), so the next attempt prefers a +// different IP. +func (r *ProxyRegistry) ReportChallenged(ctx context.Context, proxyURL string) { + proxyURL, err := NormalizeProxyURL(proxyURL) + if err != nil || proxyURL == "" { + return } r.mu.Lock() defer r.mu.Unlock() - for _, proxyURL := range r.tagIndex[tag] { - if state, ok := r.states[proxyURL]; ok && !state.disabled { - return true - } + state, ok := r.states[proxyURL] + if !ok { + return + } + state.challengedUntil = time.Now().Add(ProxyChallengeCooldown) + WithRequest(ctx).WithField("proxy", MaskProxyURL(proxyURL)). + Debugf("Deprioritized challenged proxy for %s: %s", ProxyChallengeCooldown, MaskProxyURL(proxyURL)) +} + +func (r *ProxyRegistry) HasHealthyProxyForTag(tag string) bool { + return r.HealthyCountForTag(tag) > 0 +} + +// HealthyCountForTag returns how many non-disabled proxies the tag pool holds +// (a challenged proxy still counts β€” it's usable, just deprioritized). +func (r *ProxyRegistry) HealthyCountForTag(tag string) int { + tag = normalizeTag(tag) + if tag == "" { + return 0 } - return false + r.mu.Lock() + defer r.mu.Unlock() + + count := 0 + for _, proxyURL := range r.tagIndex[tag] { + if state, ok := r.states[proxyURL]; ok && !state.disabled { + count++ + } + } + return count } func (r *ProxyRegistry) BuildStats() ProxyStats { diff --git a/core/proxy_rotation_test.go b/core/proxy_rotation_test.go new file mode 100644 index 0000000..cc23c16 --- /dev/null +++ b/core/proxy_rotation_test.go @@ -0,0 +1,150 @@ +package core + +import ( + "context" + "sync" + "testing" + + "golang.org/x/time/rate" +) + +// captchaThenSuccessEngine returns ErrCaptcha for every proxy URL except the +// one designated as healthy, where it succeeds. It records the proxy URL of +// each attempt so the test can assert rotation happened. +type captchaThenSuccessEngine struct { + name string + goodProxy string + mu sync.Mutex + seenProxies []string +} + +func (e *captchaThenSuccessEngine) Name() string { return e.name } +func (e *captchaThenSuccessEngine) IsInitialized() bool { return true } +func (e *captchaThenSuccessEngine) GetRateLimiter() *rate.Limiter { return nil } + +func (e *captchaThenSuccessEngine) Search(ctx context.Context, q Query) ([]SearchResult, error) { + e.mu.Lock() + e.seenProxies = append(e.seenProxies, q.ProxyURL) + e.mu.Unlock() + if q.ProxyURL == e.goodProxy { + return []SearchResult{{Title: "ok", URL: "https://example.com", Rank: 1}}, nil + } + return nil, ErrCaptcha +} + +func (e *captchaThenSuccessEngine) SearchImage(ctx context.Context, q Query) ([]SearchResult, error) { + return e.Search(ctx, q) +} + +func (e *captchaThenSuccessEngine) attempts() int { + e.mu.Lock() + defer e.mu.Unlock() + return len(e.seenProxies) +} + +func tagPoolSearcher(t *testing.T, engine SearchEngine, entries []ProxyEntryConfig) *ResilientSearcher { + t.Helper() + cfg := DefaultResilientConfig() + cfg.Retry.MaxRetries = 0 + cfg.CircuitBreaker.FailureThreshold = 5 + cfg.Proxy = ProxyConfig{ + Runtime: ProxyRuntimeBrowser, + Proxies: ProxiesConfig{Entries: entries}, + EnginePolicies: map[string]string{engine.Name(): "rot"}, + } + return NewResilientSearcher([]SearchEngine{engine}, cfg) +} + +func TestReportChallengedDeprioritizesWithoutDisabling(t *testing.T) { + registry, err := NewProxyRegistry([]ProxyEntryConfig{ + {URL: "http://proxy1:8080", Tags: []string{"rot"}}, + {URL: "http://proxy2:8080", Tags: []string{"rot"}}, + }, 3) + if err != nil { + t.Fatalf("new proxy registry: %v", err) + } + ctx := context.Background() + + // Challenge proxy1 from a fresh index; the next selection must skip it. + registry.ReportChallenged(ctx, "http://proxy1:8080") + if got := registry.NextByTag("rot"); got != "http://proxy2:8080" { + t.Fatalf("expected challenged proxy to be skipped, got %q", got) + } + + // Health is untouched: both proxies still count as healthy. + if n := registry.HealthyCountForTag("rot"); n != 2 { + t.Fatalf("challenge must not degrade health, healthy=%d", n) + } + + // When both are challenged, rotation still serves one (relaxed second pass). + registry.ReportChallenged(ctx, "http://proxy2:8080") + if got := registry.NextByTag("rot"); got == "" { + t.Fatal("expected a proxy even when all are challenged") + } +} + +func TestSearchWithProtection_RotatesProxyOnCaptcha(t *testing.T) { + // Two proxies in the same tag pool; the second one is the one that works. + // NextByTag serves proxy1 first, so the first attempt gets a captcha and the + // retry should pick proxy2 and succeed. + good := "http://proxy2:8080" + engine := &captchaThenSuccessEngine{name: "google", goodProxy: good} + rs := tagPoolSearcher(t, engine, []ProxyEntryConfig{ + {URL: "http://proxy1:8080", Tags: []string{"rot"}}, + {URL: good, Tags: []string{"rot"}}, + }) + + results, _, meta, err := rs.SearchPrimary(context.Background(), engine, Query{Text: "rotate"}) + if err != nil { + t.Fatalf("expected success after rotation, got %v", err) + } + if len(results) != 1 { + t.Fatalf("expected 1 result, got %d", len(results)) + } + if meta.Attempts != 2 { + t.Fatalf("expected 2 proxy attempts, got %d", meta.Attempts) + } + if got := engine.attempts(); got != 2 { + t.Fatalf("expected engine called twice, got %d", got) + } +} + +func TestSearchWithProtection_NoRotationWithSingleProxy(t *testing.T) { + // Only one proxy in the pool: captcha must fail fast without a second + // attempt (HealthyCountForTag < 2). + engine := &captchaThenSuccessEngine{name: "google", goodProxy: "http://unused:8080"} + rs := tagPoolSearcher(t, engine, []ProxyEntryConfig{ + {URL: "http://proxy1:8080", Tags: []string{"rot"}}, + }) + + _, _, meta, err := rs.SearchPrimary(context.Background(), engine, Query{Text: "single"}) + if err == nil { + t.Fatal("expected captcha failure with a single proxy") + } + if meta.Attempts != 1 { + t.Fatalf("expected exactly 1 attempt with single proxy, got %d", meta.Attempts) + } + if got := engine.attempts(); got != 1 { + t.Fatalf("expected engine called once, got %d", got) + } +} + +func TestSearchWithProtection_DirectModeFailsFastOnCaptcha(t *testing.T) { + // Direct mode (no proxy config): captcha is non-retryable and rotation must + // not kick in. + engine := &captchaThenSuccessEngine{name: "google", goodProxy: "http://never:8080"} + cfg := DefaultResilientConfig() + cfg.Retry.MaxRetries = 0 + rs := NewResilientSearcher([]SearchEngine{engine}, cfg) + + _, _, meta, err := rs.SearchPrimary(context.Background(), engine, Query{Text: "direct"}) + if err == nil { + t.Fatal("expected captcha failure in direct mode") + } + if meta.Attempts > 1 { + t.Fatalf("direct mode must not rotate proxies, attempts=%d", meta.Attempts) + } + if got := engine.attempts(); got != 1 { + t.Fatalf("expected engine called once in direct mode, got %d", got) + } +} diff --git a/core/query_extract_param_test.go b/core/query_extract_param_test.go new file mode 100644 index 0000000..ed5678a --- /dev/null +++ b/core/query_extract_param_test.go @@ -0,0 +1,73 @@ +package core + +import ( + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/gofiber/fiber/v2" +) + +// TestInitFromContextExtractParams verifies how the unified extract knob and its +// tuning params map onto Query.Extract / Query.ExtractTop. Key behaviors: +// - extract is bool-or-int: extract=0/false off, extract=true/1 β†’ top 1, +// extract=N β†’ top N (clamped to [1,5]). +// - extract_mode/min_runes imply extraction (top defaults to 1), but an +// explicit extract=0 still wins over them. +func TestInitFromContextExtractParams(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + query string + wantExtract bool + wantTop int + }{ + {"no params defaults off", "?text=q", false, 1}, + {"extract=true means top 1", "?text=q&extract=true", true, 1}, + {"extract=1 means top 1", "?text=q&extract=1", true, 1}, + {"extract=3 means top 3", "?text=q&extract=3", true, 3}, + {"extract=N clamps high", "?text=q&extract=99", true, 5}, + {"extract=0 disables", "?text=q&extract=0", false, 1}, + {"extract=false disables", "?text=q&extract=false", false, 1}, + {"extract_mode implies extract", "?text=q&extract_mode=fast", true, 1}, + {"min_runes implies extract", "?text=q&min_runes=200", true, 1}, + {"explicit extract=0 overrides tuning", "?text=q&extract=0&extract_mode=fast", false, 1}, + } + + app := fiber.New() + app.Get("/probe", func(c *fiber.Ctx) error { + q := Query{} + if err := q.InitFromContext(c); err != nil { + return c.Status(http.StatusBadRequest).SendString(err.Error()) + } + extract := "0" + if q.Extract { + extract = "1" + } + c.Set("X-Extract", extract) + c.Set("X-Extract-Top", strconv.Itoa(q.ExtractTop)) + return c.SendStatus(http.StatusOK) + }) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/probe"+tt.query, nil) + resp, err := app.Test(req, -1) + if err != nil { + t.Fatalf("request failed: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("unexpected status %d for %s", resp.StatusCode, tt.query) + } + gotExtract := resp.Header.Get("X-Extract") == "1" + if gotExtract != tt.wantExtract { + t.Errorf("%s: Extract = %v, want %v", tt.query, gotExtract, tt.wantExtract) + } + if got := resp.Header.Get("X-Extract-Top"); got != strconv.Itoa(tt.wantTop) { + t.Errorf("%s: ExtractTop = %s, want %d", tt.query, got, tt.wantTop) + } + }) + } +} diff --git a/core/resilient.go b/core/resilient.go index 6b0fc79..6e59d5c 100644 --- a/core/resilient.go +++ b/core/resilient.go @@ -28,6 +28,9 @@ type ProxyExecutionMeta struct { Mode string `json:"mode"` Tag string `json:"tag,omitempty"` Used string `json:"used"` + // Attempts is how many proxies were tried; >1 means a challenged proxy was + // rotated out in tag-pool mode. + Attempts int `json:"attempts,omitempty"` } type ResilientConfig struct { @@ -171,56 +174,83 @@ func (rs *ResilientSearcher) searchWithProtection(ctx context.Context, engine Se attemptMeta := rs.baseProxyMeta(policy) startedAt := time.Now() - result := RetryableSearch(ctx, rs.retryCfg, engine.Name(), func(callCtx context.Context) ([]SearchResult, error) { - limiter := engine.GetRateLimiter() - if limiter != nil { - if err := limiter.Wait(callCtx); err != nil { - return nil, normalizeLimiterWaitErr(callCtx, err) - } - } - attemptQuery := q - proxyURL := "" - reportToRegistry := false - attemptMeta = rs.baseProxyMeta(policy) - - switch policy.Mode { - case ProxyModeOff: - attemptQuery.ProxyURL = "" - attemptMeta.Used = "direct" - case ProxyModeRequestURL: - proxyURL = q.ProxyURL - attemptQuery.ProxyURL = proxyURL - attemptMeta.Used = MaskProxyURL(proxyURL) - case ProxyModeTagPool: - proxyURL = rs.selectProxyForQuery(policy, q, engineCtx) - if proxyURL == "" { - return nil, fmt.Errorf("%w: no healthy proxy available for tag %q", ErrProxyUnavailable, policy.Tag) - } - attemptQuery.ProxyURL = proxyURL - reportToRegistry = policy.Tag != "" - attemptMeta.Used = MaskProxyURL(proxyURL) - } - - requestCtx := proxyRequestContext(callCtx, engine.Name(), attemptQuery) - results, err := invokeEngine(requestCtx, engine, attemptQuery, isImage) - - if reportToRegistry { - rs.reportProxyAttempt(engineCtx, proxyURL, err) - } - if err != nil && errors.Is(err, ErrCaptcha) && rs.proxyCfg.Proxies.Lanes.DropCookiesOnChallenge { - // Recompute lane key only to gate the call: empty key means we have no - // session to drop cookies for. The dropper recomputes the key itself - // when it actually needs to mutate lane state. - if !ProxyLaneKeyForTenant(engine.Name(), TenantFromContext(callCtx), attemptQuery, attemptQuery.ProxyURL).Empty() { - if dropper, ok := engine.(proxyLaneCookieDropper); ok { - dropper.DropProxyLaneCookies(callCtx, attemptQuery) + // lastProxyURL is the unmasked proxy of the last attempt, for rotation below. + lastProxyURL := "" + runOnce := func() RetryResult { + return RetryableSearch(ctx, rs.retryCfg, engine.Name(), func(callCtx context.Context) ([]SearchResult, error) { + limiter := engine.GetRateLimiter() + if limiter != nil { + if err := limiter.Wait(callCtx); err != nil { + return nil, normalizeLimiterWaitErr(callCtx, err) } } - } - return results, err - }) + attemptQuery := q + proxyURL := "" + reportToRegistry := false + attemptMeta = rs.baseProxyMeta(policy) + + switch policy.Mode { + case ProxyModeOff: + attemptQuery.ProxyURL = "" + attemptMeta.Used = "direct" + case ProxyModeRequestURL: + proxyURL = q.ProxyURL + attemptQuery.ProxyURL = proxyURL + attemptMeta.Used = MaskProxyURL(proxyURL) + case ProxyModeTagPool: + proxyURL = rs.selectProxyForQuery(policy, q, engineCtx) + if proxyURL == "" { + return nil, fmt.Errorf("%w: no healthy proxy available for tag %q", ErrProxyUnavailable, policy.Tag) + } + attemptQuery.ProxyURL = proxyURL + reportToRegistry = policy.Tag != "" + attemptMeta.Used = MaskProxyURL(proxyURL) + } + lastProxyURL = proxyURL + + requestCtx := proxyRequestContext(callCtx, engine.Name(), attemptQuery) + results, err := invokeEngine(requestCtx, engine, attemptQuery, isImage) + + if reportToRegistry { + rs.reportProxyAttempt(engineCtx, proxyURL, err) + } + if err != nil && errors.Is(err, ErrCaptcha) && rs.proxyCfg.Proxies.Lanes.DropCookiesOnChallenge { + // Recompute lane key only to gate the call: empty key means we have no + // session to drop cookies for. The dropper recomputes the key itself + // when it actually needs to mutate lane state. + if !ProxyLaneKeyForTenant(engine.Name(), TenantFromContext(callCtx), attemptQuery, attemptQuery.ProxyURL).Empty() { + if dropper, ok := engine.(proxyLaneCookieDropper); ok { + dropper.DropProxyLaneCookies(callCtx, attemptQuery) + } + } + } + + return results, err + }) + } + + result := runOnce() + attemptMeta.Attempts = 1 + + // On a captcha/block/rate-limit (non-retryable inside RetryableSearch), if + // the tag pool has another healthy proxy, deprioritize the burned one and + // retry once with the next. Tag-pool only β€” direct/request-url/global can't + // rotate. + if result.Err != nil && + policy.Mode == ProxyModeTagPool && + policy.Tag != "" && + rs.proxyRegistry != nil && + IsProxyChallengeError(result.Err) && + rs.proxyRegistry.HealthyCountForTag(policy.Tag) >= 2 && + ctx.Err() == nil { + rs.proxyRegistry.ReportChallenged(engineCtx, lastProxyURL) + WithRequestEngine(ctx, engine.Name()).WithError(result.Err). + Debug("Challenged proxy rotated out, retrying once with next proxy") + result = runOnce() + attemptMeta.Attempts = 2 + } if result.Err != nil { if shouldRecordCircuitFailure(result.Err) { @@ -233,6 +263,14 @@ func (rs *ResilientSearcher) searchWithProtection(ctx context.Context, engine Se return result.Results, attemptMeta, nil } +// IsProxyChallengeError reports whether err is a captcha/block/rate-limit β€” an +// IP-reputation problem another proxy might dodge. +func IsProxyChallengeError(err error) bool { + return errors.Is(err, ErrCaptcha) || + errors.Is(err, ErrBlocked) || + errors.Is(err, ErrRateLimited) +} + func shouldRecordCircuitFailure(err error) bool { return err != nil && !IsContextDone(err) && diff --git a/core/server.go b/core/server.go index 8009fff..5922acc 100644 --- a/core/server.go +++ b/core/server.go @@ -1451,6 +1451,9 @@ func (s *Server) applyProxyHeaders(c *fiber.Ctx, meta ProxyExecutionMeta) { c.Set("X-Proxy-Tag", tag) } c.Set("X-Proxy-Used", used) + if meta.Attempts > 1 { + c.Set("X-Proxy-Attempts", strconv.Itoa(meta.Attempts)) + } } func setNetworkBytesHeader(c *fiber.Ctx, ctx context.Context) { diff --git a/core/server_extract.go b/core/server_extract.go index 2cd0b85..a3ef971 100644 --- a/core/server_extract.go +++ b/core/server_extract.go @@ -116,14 +116,21 @@ func (s *Server) newExtractor() extractpkg.Extractor { } func (s *Server) rawExtractFetch(ctx context.Context, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) { - cfg := s.opts.Extract.Normalized() + return RawExtractFetch(ctx, req, s.opts.Extract, s.opts.FingerprintBrowserOpts.Insecure) +} + +// RawExtractFetch performs the browserless extraction fetch: validate the +// target, issue a guarded HTTP GET, classify the status, and return the body +// capped to the byte budget. Shared by the HTTP server and the CLI. +func RawExtractFetch(ctx context.Context, req extractpkg.ExtractRequest, cfg extractpkg.Config, insecure bool) (*extractpkg.FetchResponse, error) { + cfg = cfg.Normalized() if err := validateExtractTargetURL(ctx, req.URL, cfg.AllowPrivateNetworks); err != nil { return nil, err } resp, err := RawSearchRequest(ctx, req.URL, Query{ ProxyURL: req.ProxyURL, LangCode: req.LangCode, - Insecure: s.opts.FingerprintBrowserOpts.Insecure, + Insecure: insecure, GuardPrivateNetworks: !cfg.AllowPrivateNetworks, }) if err != nil { @@ -135,7 +142,7 @@ func (s *Server) rawExtractFetch(ctx context.Context, req extractpkg.ExtractRequ } limit := int64(req.MaxBytes) if limit <= 0 { - limit = int64(s.opts.Extract.Normalized().MaxBytes) + limit = int64(cfg.MaxBytes) } body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) if err != nil { @@ -159,7 +166,15 @@ func (s *Server) renderedExtractFetch(ctx context.Context, req extractpkg.Extrac if err != nil { return nil, err } - page, err := browser.Navigate(WithRequestProxyURL(ctx, req.ProxyURL), req.URL) + return RenderExtractHTML(WithRequestProxyURL(ctx, req.ProxyURL), browser, req) +} + +// RenderExtractHTML navigates an already-resolved browser to the target, +// returns its rendered HTML capped to the byte budget, and always closes the +// page. Shared by the HTTP server's BrowserResolver path and the CLI's +// one-shot browser. Callers own target validation and proxy gating. +func RenderExtractHTML(ctx context.Context, browser *Browser, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) { + page, err := browser.Navigate(ctx, req.URL) if err != nil { return nil, err } @@ -220,7 +235,17 @@ func validateExtractTargetURL(ctx context.Context, rawURL string, allowPrivateNe } func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope, q Query, format string) { - cfg := s.opts.Extract.Normalized() + EnrichEnvelopeWithExtraction(ctx, env, q, format, s.newExtractor(), s.opts.Extract) +} + +// EnrichEnvelopeWithExtraction fills env.Results[*].Extracted by running the +// extractor over the top organic results, with candidate fill-in when a top +// result fails. It is shared by the HTTP search handler and the CLI so both +// apply the same depth bounds, batch deadline, and result selection. The +// extractor and cfg are supplied by the caller (the server reuses its +// long-lived browser pool; the CLI builds a one-shot browser). +func EnrichEnvelopeWithExtraction(ctx context.Context, env *Envelope, q Query, format string, extractor extractpkg.Extractor, cfg extractpkg.Config) { + cfg = cfg.Normalized() if env == nil || !q.Extract || !cfg.Enabled { return } @@ -231,11 +256,7 @@ func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope if format == "text" { contentFormat = "text" } - extractor := s.newExtractor() - limit := q.ExtractTop - if limit <= 0 || limit > 5 { - limit = 3 - } + limit := clampExtractTop(q.ExtractTop) if limit > len(env.Results) { limit = len(env.Results) } @@ -245,7 +266,7 @@ func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope } // Per-fetch timeouts bound a single URL; this aggregate deadline bounds the - // whole batch so a few slow/hanging targets can't stretch the search request + // whole batch so a few slow/hanging targets can't stretch the request // open-endedly. The ceiling is derived from the per-URL budget (see // Config.BatchTimeout) rather than a separate knob. When it fires, in-flight // fetches are cancelled and any not yet started record a timeout error instead @@ -256,7 +277,7 @@ func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope extractOne := func(idx int) { // Skip the fetch entirely if the batch budget is already spent. if err := ctx.Err(); err != nil { - env.Results[idx].Extracted = &ExtractedContent{Error: sanitizeExtractError(err)} + env.Results[idx].Extracted = &ExtractedContent{Error: SanitizeExtractError(err)} return } req := extractpkg.ExtractRequest{ @@ -270,14 +291,14 @@ func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope } result, err := extractor.Extract(ctx, req) if err != nil { - env.Results[idx].Extracted = &ExtractedContent{Error: sanitizeExtractError(err)} + env.Results[idx].Extracted = &ExtractedContent{Error: SanitizeExtractError(err)} return } content := result.Markdown if contentFormat == "text" { content = result.Text } - if !extractedContentLooksUseful(content) { + if !ExtractedContentLooksUseful(content) { env.Results[idx].Extracted = &ExtractedContent{Error: "empty extracted content"} return } @@ -320,7 +341,10 @@ func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope const minUsefulExtractRunes = 80 -func extractedContentLooksUseful(content string) bool { +// ExtractedContentLooksUseful reports whether extracted page content is long +// enough to keep, rather than an empty/boilerplate shell. Shared by the HTTP +// server and the CLI so both apply the same threshold. +func ExtractedContentLooksUseful(content string) bool { return len([]rune(strings.TrimSpace(content))) >= minUsefulExtractRunes } @@ -337,7 +361,7 @@ func extractedSuccessCount(results []Result) int { func extractedResultSucceeded(result Result) bool { return result.Extracted != nil && result.Extracted.Error == "" && - extractedContentLooksUseful(result.Extracted.Content) + ExtractedContentLooksUseful(result.Extracted.Content) } func sendExtractResult(c *fiber.Ctx, format string, result *extractpkg.ExtractResult) error { @@ -379,7 +403,9 @@ func parseBoolDefault(raw string, fallback bool) bool { return raw == "1" || strings.EqualFold(raw, "true") || strings.EqualFold(raw, "yes") } -func sanitizeExtractError(err error) string { +// SanitizeExtractError trims and length-bounds an extraction error for safe +// inclusion in a response payload. Shared by the HTTP server and the CLI. +func SanitizeExtractError(err error) string { if err == nil { return "" } diff --git a/docs/openapi.yaml b/docs/openapi.yaml index c68427b..01715a8 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -48,7 +48,6 @@ paths: - $ref: "#/components/parameters/FilterQuery" - $ref: "#/components/parameters/FeaturesQuery" - $ref: "#/components/parameters/ExtractQuery" - - $ref: "#/components/parameters/ExtractTopQuery" - $ref: "#/components/parameters/ExtractModeQuery" - $ref: "#/components/parameters/MinRunesQuery" - $ref: "#/components/parameters/FormatQuery" @@ -330,7 +329,6 @@ paths: - $ref: "#/components/parameters/MegaDedupeQuery" - $ref: "#/components/parameters/MegaMergeQuery" - $ref: "#/components/parameters/ExtractQuery" - - $ref: "#/components/parameters/ExtractTopQuery" - $ref: "#/components/parameters/ExtractModeQuery" - $ref: "#/components/parameters/MinRunesQuery" - $ref: "#/components/parameters/FormatQuery" @@ -834,25 +832,27 @@ components: name: extract in: query required: false - description: Fetch and embed cleaned target-page content for top web results. + description: > + Fetch and embed cleaned target-page content for the top web results. + Accepts a boolean or an integer depth: `extract=0`/`false` disables + extraction; `extract=true`/`1` enriches the top result; `extract=N` + (1-5) enriches the top N results. The tuning params `extract_mode` and + `min_runes` imply `extract=true` (top 1) when present, unless + `extract=0` is set explicitly. schema: - type: boolean + oneOf: + - type: boolean + - type: integer + minimum: 0 + maximum: 5 default: false - ExtractTopQuery: - name: extract_top - in: query - required: false - description: Number of top organic results to enrich when `extract=true`. - schema: - type: integer - minimum: 1 - maximum: 5 - default: 3 ExtractModeQuery: name: extract_mode in: query required: false - description: Extraction strategy for target pages. + description: > + Extraction strategy for target pages. Its presence implies + `extract=true` unless `extract=0` is set explicitly. schema: type: string enum: [auto, fast, rendered] diff --git a/ecosia/captcha_selector_test.go b/ecosia/captcha_selector_test.go new file mode 100644 index 0000000..8c34f41 --- /dev/null +++ b/ecosia/captcha_selector_test.go @@ -0,0 +1,54 @@ +package ecosia + +import ( + "testing" + + "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/testutil" +) + +// TestEcosiaPageTypeSelectors verifies that the selectors defined in selectors.go +// match (or don't match) real fixture HTML without needing a browser. +func TestEcosiaPageTypeSelectors(t *testing.T) { + t.Parallel() + + tests := []struct { + fixture string + selector string + wantHit bool + }{ + {"search_captcha.html", Selectors.Captcha, true}, + {"search_captcha.html", Selectors.Mainline, false}, + + {"search_results.html", Selectors.Mainline, true}, + {"search_results.html", Selectors.Captcha, false}, + + {"search_no_results.html", Selectors.Captcha, false}, + } + + for _, tt := range tests { + t.Run(tt.fixture+"/"+tt.selector, func(t *testing.T) { + t.Parallel() + assertSelector(t, tt.fixture, tt.selector, tt.wantHit) + }) + } +} + +func assertSelector(t *testing.T, fixture, selector string, wantHit bool) { + t.Helper() + + resp := testutil.ResponseFromFixture(t, fixture) + doc, err := goquery.NewDocumentFromReader(resp.Body) + if err != nil { + t.Fatalf("parse fixture: %v", err) + } + + got := doc.Find(selector).Length() > 0 + if got != wantHit { + if wantHit { + t.Fatalf("selector %q not found in %s β€” update selectors.go", selector, fixture) + } else { + t.Fatalf("selector %q unexpectedly present in %s", selector, fixture) + } + } +} diff --git a/ecosia/search.go b/ecosia/search.go index ce6c38a..eaa74cc 100644 --- a/ecosia/search.go +++ b/ecosia/search.go @@ -35,13 +35,17 @@ func startPage(start int) (pageNum, startRank int, err error) { return pageNum, startRank, nil } -// Cloudflare interstitial markers. URL/title are CF defaults. +// Cloudflare interstitial markers. URL/title are CF defaults; cfBodyMarkers are +// challenge-page phrases absent from a real SERP, shared by the browser +// (isCaptcha) and raw (isCaptchaDoc) paths. Bare "captcha" is omitted β€” too +// broad (appears in SERP snippets) and not on the interstitial anyway. const ( - cfURLPath = "cdn-cgi" - cfPageTitle = "just a moment" - cfBodyMarker = "not a bot" + cfURLPath = "cdn-cgi" + cfPageTitle = "just a moment" ) +var cfBodyMarkers = []string{"not a robot", "not a bot", "unusual traffic"} + // Ecosia implements core.SearchEngine for Ecosia SERP pages. Additional // documentation at https://support.ecosia.org/article/447-search-features. type Ecosia struct { @@ -80,7 +84,16 @@ func (e *Ecosia) isCaptcha(page *rod.Page) bool { if err != nil { return false } - return strings.Contains(strings.ToLower(html), cfBodyMarker) + lower := strings.ToLower(html) + if strings.Contains(lower, "cf-turnstile-response") { + return true + } + for _, m := range cfBodyMarkers { + if strings.Contains(lower, m) { + return true + } + } + return false } func (e *Ecosia) parseResult(elem *rod.Element, rank int, ad bool) (core.SearchResult, bool) { diff --git a/ecosia/search_raw.go b/ecosia/search_raw.go index 774c9de..63f44df 100644 --- a/ecosia/search_raw.go +++ b/ecosia/search_raw.go @@ -12,12 +12,28 @@ import ( "github.com/karust/openserp/core" ) +// isCaptchaDoc reports whether a parsed Ecosia document is a Cloudflare +// challenge rather than a SERP. Prefers the hidden Turnstile input, then falls +// back to the shared challenge-page body phrases (see cfBodyMarkers). +func isCaptchaDoc(doc *goquery.Document) bool { + if doc.Find(Selectors.Captcha).Length() > 0 { + return true + } + text := strings.ToLower(doc.Text()) + for _, m := range cfBodyMarkers { + if strings.Contains(text, m) { + return true + } + } + return false +} + func classifyEcosiaRawHTML(body []byte) error { doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body)) if err != nil { return err } - if strings.Contains(strings.ToLower(doc.Text()), "captcha") { + if isCaptchaDoc(doc) { return core.ErrCaptcha } if doc.Find("[data-test-id='web-no-results']").Length() > 0 || diff --git a/ecosia/search_raw_test.go b/ecosia/search_raw_test.go index 2c8f7ba..daebb45 100644 --- a/ecosia/search_raw_test.go +++ b/ecosia/search_raw_test.go @@ -44,13 +44,34 @@ func TestEcosiaImageResultParser(t *testing.T) { func TestEcosiaClassifyRawHTML(t *testing.T) { t.Parallel() - body, err := io.ReadAll(testutil.ResponseFromFixture(t, "search_no_results.html").Body) - if err != nil { - t.Fatalf("read fixture body: %v", err) + tests := []struct { + fixture string + want error + }{ + {"search_no_results.html", core.ErrEmptyResult}, + {"search_captcha.html", core.ErrCaptcha}, + {"search_results.html", nil}, } - err = classifyEcosiaRawHTML(body) - if !errors.Is(err, core.ErrEmptyResult) { - t.Fatalf("expected %v for search_no_results.html, got %v", core.ErrEmptyResult, err) + for _, tt := range tests { + t.Run(tt.fixture, func(t *testing.T) { + t.Parallel() + + body, err := io.ReadAll(testutil.ResponseFromFixture(t, tt.fixture).Body) + if err != nil { + t.Fatalf("read fixture body: %v", err) + } + + got := classifyEcosiaRawHTML(body) + if tt.want == nil { + if got != nil { + t.Fatalf("expected nil for %s, got %v", tt.fixture, got) + } + return + } + if !errors.Is(got, tt.want) { + t.Fatalf("expected %v for %s, got %v", tt.want, tt.fixture, got) + } + }) } } diff --git a/ecosia/selectors.go b/ecosia/selectors.go index d9fad99..bf0baca 100644 --- a/ecosia/selectors.go +++ b/ecosia/selectors.go @@ -2,6 +2,7 @@ package ecosia // Selectors is the single source of truth for Ecosia SERP CSS selectors. var Selectors = struct { + Captcha string Mainline string Result string Ad string @@ -13,6 +14,10 @@ var Selectors = struct { ImageSource string ImageDims string }{ + // Captcha matches Ecosia's Cloudflare Turnstile interstitial. The hidden + // cf-turnstile-response input is present on every challenge page and never + // on a real SERP, so it's a precise marker for the raw (browserless) path. + Captcha: "input[name='cf-turnstile-response']", Mainline: "[data-test-id='mainline']", Result: "[data-test-id='mainline-result-web']", Ad: "[data-test-id='mainline-result-ad']", diff --git a/ecosia/testdata/search_captcha.html b/ecosia/testdata/search_captcha.html new file mode 100644 index 0000000..f4ae0d9 --- /dev/null +++ b/ecosia/testdata/search_captcha.html @@ -0,0 +1 @@ +Just a moment...

Confirm you’re not a robot

Our system has detected unusual traffic from your network. Please solve the challenge below to show you’re not a robot.

Verification successful. Waiting for www.ecosia.org to respond
diff --git a/examples/content/js-search-with-extract/README.md b/examples/content/js-search-with-extract/README.md index a9c8d43..b87cd77 100644 --- a/examples/content/js-search-with-extract/README.md +++ b/examples/content/js-search-with-extract/README.md @@ -8,4 +8,4 @@ npm install node index.js ``` -Edit the `text`, `extractTop`, or `extractMode` in [index.js](index.js) to tune it. +Edit the `text`, `extract` depth, or `extractMode` in [index.js](index.js) to tune it. diff --git a/examples/content/js-search-with-extract/index.js b/examples/content/js-search-with-extract/index.js index 2009ec9..aba3d57 100644 --- a/examples/content/js-search-with-extract/index.js +++ b/examples/content/js-search-with-extract/index.js @@ -4,14 +4,13 @@ import { OpenSERP } from "@openserp/sdk"; // const client = new OpenSERP({ apiKey: "", timeoutMs: 60_000 }); const client = new OpenSERP({ baseUrl: "http://localhost:7000", timeoutMs: 60_000 }); -// `extract: true` fetches the top pages and returns their cleaned content -// alongside each result, so you get the page text in a single request. -// `extractTop` (max 5) controls how many results are enriched. +// `extract: N` fetches the top N pages (max 5) and returns their cleaned +// content alongside each result, so you get the page text in a single request. +// `extract: true` is shorthand for the top result. const { results } = await client.search({ engine: "ecosia", text: "what is a serp api", - extract: true, - extractTop: 2, + extract: 2, extractMode: "auto", }); diff --git a/google/search.go b/google/search.go index 4b97267..2af564b 100644 --- a/google/search.go +++ b/google/search.go @@ -42,6 +42,12 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) { return 0, core.ErrParser } + // Stats div is absent on many locales; probe first so .Search doesn't block + // the full selector timeout. + if has, _, err := page.Has(Selectors.ResultStats); err != nil || !has { + return 0, nil + } + resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search(Selectors.ResultStats) if err != nil { return 0, errors.New("Result stats not found: " + err.Error()) @@ -147,7 +153,33 @@ func (gogl *Google) preparePage(page *rod.Page) { } } +// waitAnswersExpanded polls until the first PAA entry has expanded to a +// title+body (β‰₯2 text lines) or maxWait elapses, replacing a flat 2s sleep. +func (gogl *Google) waitAnswersExpanded(ctx context.Context, answers rod.Elements, maxWait time.Duration) error { + if len(answers) == 0 || maxWait <= 0 { + return nil + } + deadline := time.Now().Add(maxWait) + for { + text, err := answers[0].Text() + if err == nil && len(strings.Split(text, "\n")) >= 2 { + return nil + } + if !time.Now().Before(deadline) { + return nil + } + if err := core.SleepContext(ctx, 100*time.Millisecond); err != nil { + return err + } + } +} + func (gogl *Google) acceptCookies(page *rod.Page) { + // Probe with Has first so a banner-less SERP doesn't block on .Search's full + // timeout (AGENTS.md: use Has for existence). + if has, _, err := page.Has(Selectors.CookieBtn); err != nil || !has { + return + } diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search(Selectors.CookieBtn) if err != nil { gogl.logger.Debug("Cookie consent not found: %s", err) @@ -331,7 +363,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } } - if err := core.SleepContext(ctx, 2*time.Second); err != nil { + // Poll for expansion (usually 200-400ms) instead of a flat 2s sleep. + if err := gogl.waitAnswersExpanded(ctx, answers, 2*time.Second); err != nil { return nil, err } diff --git a/main.go b/main.go index 25107d7..119ecab 100644 --- a/main.go +++ b/main.go @@ -11,7 +11,7 @@ func main() { defer recoverPanic() if err := cmd.RootCmd.Execute(); err != nil { - logrus.Info(err) + // Cobra already prints the error to stderr; just set the exit code. os.Exit(1) } } diff --git a/yandex/search.go b/yandex/search.go index 5ea406b..60e712d 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -140,6 +140,41 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc return searchResults } +// Yandex hydrates the results list progressively, so the first parse can be +// short. After the list selector appears, re-poll briefly until we have the +// requested number of organic results or the grace period elapses. +const ( + resultHydrationGrace = 2 * time.Second + resultPollInterval = 120 * time.Millisecond +) + +func (yand *Yandex) waitForParsedResults(ctx context.Context, page *rod.Page, pageNum, wantOrganic int) ([]core.SearchResult, error) { + elements, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, yand.GetSelectorTimeout()) + if err != nil { + return nil, err + } + + results := yand.parseResults(elements, pageNum) + if wantOrganic <= 0 { + return results, nil + } + + deadline := time.Now().Add(resultHydrationGrace) + for core.CountOrganicResults(results) < wantOrganic && time.Now().Before(deadline) { + if err := core.SleepContext(ctx, resultPollInterval); err != nil { + return results, err + } + nextElements, eerr := page.Elements(Selectors.Results) + if eerr != nil || len(nextElements) <= len(elements) { + continue + } + elements = nextElements + results = yand.parseResults(nextElements, pageNum) + } + + return results, nil +} + func yandexElementHasAdMarker(el *rod.Element) bool { if el == nil { return false @@ -212,7 +247,14 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor } defer core.DeferClosePage(ctx, page, &yand.Browser)() - elements, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, yand.GetSelectorTimeout()) + wantOrganic := query.Limit + if wantOrganic <= 0 || wantOrganic > pageSize { + wantOrganic = pageSize + } + if searchPage == startPage && skipOnFirstPage > 0 { + wantOrganic += skipOnFirstPage + } + r, err := yand.waitForParsedResults(ctx, page, searchPage, wantOrganic) if err != nil { if yand.isCaptcha(page) { yand.logger.Error("Captcha detected: %s", url) @@ -226,7 +268,6 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor return false, core.ErrSearchTimeout } - r := yand.parseResults(elements, searchPage) if searchPage == startPage && skipOnFirstPage > 0 { r = skipOrganicResults(r, skipOnFirstPage) }