mirror of
https://github.com/karust/openserp.git
synced 2026-08-16 05:16:02 +08:00
feat(cli): clean search, add extract/format flags; harden engines & proxy rotation; fix bugs; update docs
- Add structured `search [engine] [query]` CLI: --limit/--lang/--region/--site/--file, --format (json|text|markdown|ndjson), --extract N, --search-timeout; Envelope and route logs to stderr with a --quiet default (fixes stdout pollution) - Unify engines behind a single engineSpec registry (CLI + serve share it) - Unify the extract knob to bool-or-int `extract=N` (drop extract_top); CLI and HTTP share core batch extraction, raw/rendered fetch, and clamp helpers - Engines: Ecosia CF captcha detection (raw + browser), Yandex progressive-result wait, Google PAA poll + Has() existence probes, Bing title/desc attribute fallbacks - Proxy: rotate challenged proxies out of the tag pool for one retry (X-Proxy-Attempts); browser health-ping skip window; opt-in WaitStable
This commit is contained in:
@@ -145,3 +145,49 @@ func TestParseBingHTMLTitleFallback(t *testing.T) {
|
||||
t.Fatalf("title = %q, want fallback", results[0].Title)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBingHTMLPrefersTitleAttribute(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
html := `
|
||||
<ol id="b_results">
|
||||
<li class="b_algo">
|
||||
<h2><a aria-label="Real SERP Title" href="https://example.com/result">example.com</a></h2>
|
||||
<div class="b_caption"><p>Snippet</p></div>
|
||||
</li>
|
||||
</ol>`
|
||||
|
||||
results, err := ParseHTML(bytes.NewReader([]byte(html)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML() error = %v", err)
|
||||
}
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("expected 1 result, got %d", len(results))
|
||||
}
|
||||
if results[0].Title != "Real SERP Title" {
|
||||
t.Fatalf("title = %q, want attribute title", results[0].Title)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBingHTMLDescriptionFallsThroughEmptyPrimary(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
html := `
|
||||
<ol id="b_results">
|
||||
<li class="b_algo">
|
||||
<h2><a href="https://example.com/result">Result title</a></h2>
|
||||
<div class="b_caption"><p> </p><div>Useful snippet text</div></div>
|
||||
</li>
|
||||
</ol>`
|
||||
|
||||
results, err := ParseHTML(bytes.NewReader([]byte(html)))
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML() error = %v", err)
|
||||
}
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("expected 1 result, got %d", len(results))
|
||||
}
|
||||
if results[0].Description != "Useful snippet text" {
|
||||
t.Fatalf("description = %q, want fallback snippet", results[0].Description)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user