mirror of
https://github.com/karust/openserp.git
synced 2026-08-18 14:18:52 +08:00
9cc69da758
- Add structured `search [engine] [query]` CLI: --limit/--lang/--region/--site/--file, --format (json|text|markdown|ndjson), --extract N, --search-timeout; Envelope and route logs to stderr with a --quiet default (fixes stdout pollution) - Unify engines behind a single engineSpec registry (CLI + serve share it) - Unify the extract knob to bool-or-int `extract=N` (drop extract_top); CLI and HTTP share core batch extraction, raw/rendered fetch, and clamp helpers - Engines: Ecosia CF captcha detection (raw + browser), Yandex progressive-result wait, Google PAA poll + Has() existence probes, Bing title/desc attribute fallbacks - Proxy: rotate challenged proxies out of the tag pool for one retry (X-Proxy-Attempts); browser health-ping skip window; opt-in WaitStable
55 lines
1.3 KiB
Go
55 lines
1.3 KiB
Go
package ecosia
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/karust/openserp/testutil"
|
|
)
|
|
|
|
// TestEcosiaPageTypeSelectors verifies that the selectors defined in selectors.go
|
|
// match (or don't match) real fixture HTML without needing a browser.
|
|
func TestEcosiaPageTypeSelectors(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
fixture string
|
|
selector string
|
|
wantHit bool
|
|
}{
|
|
{"search_captcha.html", Selectors.Captcha, true},
|
|
{"search_captcha.html", Selectors.Mainline, false},
|
|
|
|
{"search_results.html", Selectors.Mainline, true},
|
|
{"search_results.html", Selectors.Captcha, false},
|
|
|
|
{"search_no_results.html", Selectors.Captcha, false},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.fixture+"/"+tt.selector, func(t *testing.T) {
|
|
t.Parallel()
|
|
assertSelector(t, tt.fixture, tt.selector, tt.wantHit)
|
|
})
|
|
}
|
|
}
|
|
|
|
func assertSelector(t *testing.T, fixture, selector string, wantHit bool) {
|
|
t.Helper()
|
|
|
|
resp := testutil.ResponseFromFixture(t, fixture)
|
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
|
if err != nil {
|
|
t.Fatalf("parse fixture: %v", err)
|
|
}
|
|
|
|
got := doc.Find(selector).Length() > 0
|
|
if got != wantHit {
|
|
if wantHit {
|
|
t.Fatalf("selector %q not found in %s — update selectors.go", selector, fixture)
|
|
} else {
|
|
t.Fatalf("selector %q unexpectedly present in %s", selector, fixture)
|
|
}
|
|
}
|
|
}
|