feat(cli): clean search, add extract/format flags; harden engines & proxy rotation; fix bugs; update docs

- Add structured `search [engine] [query]` CLI: --limit/--lang/--region/--site/--file, --format (json|text|markdown|ndjson), --extract N, --search-timeout; Envelope and route logs to stderr with a --quiet default (fixes stdout pollution)
- Unify engines behind a single engineSpec registry (CLI + serve share it)
- Unify the extract knob to bool-or-int `extract=N` (drop extract_top); CLI and HTTP share core batch extraction, raw/rendered fetch, and clamp helpers
- Engines: Ecosia CF captcha detection (raw + browser), Yandex progressive-result wait, Google PAA poll + Has() existence probes, Bing title/desc attribute fallbacks
- Proxy: rotate challenged proxies out of the tag pool for one retry (X-Proxy-Attempts); browser health-ping skip window; opt-in WaitStable
This commit is contained in:
Rustem Kamalov
2026-06-16 03:51:37 +03:00
parent ca3143ccd1
commit 9cc69da758
33 changed files with 1613 additions and 383 deletions

View File

@@ -145,3 +145,49 @@ func TestParseBingHTMLTitleFallback(t *testing.T) {
t.Fatalf("title = %q, want fallback", results[0].Title)
}
}
func TestParseBingHTMLPrefersTitleAttribute(t *testing.T) {
t.Parallel()
html := `
<ol id="b_results">
<li class="b_algo">
<h2><a aria-label="Real SERP Title" href="https://example.com/result">example.com</a></h2>
<div class="b_caption"><p>Snippet</p></div>
</li>
</ol>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
if results[0].Title != "Real SERP Title" {
t.Fatalf("title = %q, want attribute title", results[0].Title)
}
}
func TestParseBingHTMLDescriptionFallsThroughEmptyPrimary(t *testing.T) {
t.Parallel()
html := `
<ol id="b_results">
<li class="b_algo">
<h2><a href="https://example.com/result">Result title</a></h2>
<div class="b_caption"><p> </p><div>Useful snippet text</div></div>
</li>
</ol>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
if results[0].Description != "Useful snippet text" {
t.Fatalf("description = %q, want fallback snippet", results[0].Description)
}
}