feat(cli): clean search, add extract/format flags; harden engines & proxy rotation; fix bugs; update docs

- Add structured `search [engine] [query]` CLI: --limit/--lang/--region/--site/--file, --format (json|text|markdown|ndjson), --extract N, --search-timeout; Envelope and route logs to stderr with a --quiet default (fixes stdout pollution)
- Unify engines behind a single engineSpec registry (CLI + serve share it)
- Unify the extract knob to bool-or-int `extract=N` (drop extract_top); CLI and HTTP share core batch extraction, raw/rendered fetch, and clamp helpers
- Engines: Ecosia CF captcha detection (raw + browser), Yandex progressive-result wait, Google PAA poll + Has() existence probes, Bing title/desc attribute fallbacks
- Proxy: rotate challenged proxies out of the tag pool for one retry (X-Proxy-Attempts); browser health-ping skip window; opt-in WaitStable

core/browser.go

@@ -252,8 +252,19 @@ type browserConnection struct {
 	laneProfiles map[string]browserprofile.Profile
 	authCancel   context.CancelFunc
 	authStopped  chan struct{}
+	// lastOK is when a CDP call last succeeded; the health ping is skipped while
+	// it is within healthPingSkipWindow.
+	lastOK time.Time
 }
 
+const (
+	// healthPingTimeout bounds the per-call connection ping so a wedged Chrome
+	// can't stall navigations while holding the connection lock.
+	healthPingTimeout = 3 * time.Second
+	// healthPingSkipWindow skips the ping when a CDP call succeeded this recently.
+	healthPingSkipWindow = 5 * time.Second
+)
+
 // NewBrowser launches a new Chromium process via Rod launcher and returns a
 // Browser wrapper configured with proxy and captcha solver settings.
 func NewBrowser(opts BrowserOpts) (*Browser, error) {
@@ -272,6 +283,9 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) {
 	// headless=new uses the full Chrome renderer; legacy --headless disables the
 	// GPU process entirely, making WebGL context creation fail even with SwiftShader.
 	// use-angle=swiftshader-webgl (Chrome ≥112) enables a software WebGL renderer.
+	// Rod enables leakless by default, so always pass the configured value
+	// through. OpenSERP defaults it to false because the helper binary is
+	// commonly flagged by antivirus on Windows.
 	l := launcher.New().Leakless(opts.IsLeakless).
 		Set("disable-blink-features", "AutomationControlled").
 		Delete("enable-automation").
@@ -557,18 +571,18 @@ func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect boo
 			return nil, err
 		}
 		state.browser = connected
+		state.lastOK = time.Now()
 		return state.browser, nil
 	}
 
-	// Bound just this health-ping with a per-call timeout so a wedged browser
-	// can't block the connection lock forever. Use a fresh derived context each
-	// call (not browser.Timeout, which would bake a permanent deadline into the
-	// persistent connection — see newRodBrowser).
-	pingTimeout := b.Timeout
-	if pingTimeout <= 0 {
-		pingTimeout = 30 * time.Second
+	// A recent successful CDP call means the connection is alive; skip the ping.
+	if !state.lastOK.IsZero() && time.Since(state.lastOK) < healthPingSkipWindow {
+		return state.browser, nil
 	}
-	pingCtx, cancelPing := context.WithTimeout(EnsureContext(ctx), pingTimeout)
+
+	// Fresh derived context per call (not browser.Timeout, which would bake a
+	// permanent deadline into the persistent connection — see newRodBrowser).
+	pingCtx, cancelPing := context.WithTimeout(EnsureContext(ctx), healthPingTimeout)
 	_, pingErr := state.browser.Context(pingCtx).Version()
 	cancelPing()
 	if pingErr != nil {
@@ -579,6 +593,7 @@ func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect boo
 		}
 		state.browser = connected
 	}
+	state.lastOK = time.Now()
 
 	return state.browser, nil
 }
@@ -1503,20 +1518,24 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
 		wait()
 	}
 
-	// WaitStable internally waits for page load too, so cap it separately.
-	// Selector parsing still decides whether a partially loaded page is usable.
-	stableWaitTimeout := minPositiveDuration(b.Timeout, b.WaitLoadTime+time.Second)
-	if err := page.Timeout(stableWaitTimeout).WaitStable(800 * time.Millisecond); err != nil {
-		WithRequest(ctx).WithError(err).Debug("WaitStable returned early; continuing")
-	}
 	if err := classifyMainDocumentStatus(statusWatcher.Status()); err != nil {
 		closeOnErr()
 		return nil, err
 	}
 	b.saveLaneCookies(ctx, page, URL)
+	b.markConnectionOK()
 	return page, nil
 }
 
+// markConnectionOK records a successful CDP round trip so the next
+// ensureConnectedBrowser can skip its health ping (see healthPingSkipWindow).
+func (b *Browser) markConnectionOK() {
+	state := b.connectionState()
+	state.mu.Lock()
+	state.lastOK = time.Now()
+	state.mu.Unlock()
+}
+
 // Close closes the active browser connection.
 func (b *Browser) Close() error {
 	if b == nil || b.browserAddr == "" {

core/common.go

@@ -15,6 +15,14 @@ import (
 	"golang.org/x/time/rate"
 )
 
+// Extraction depth bounds for the unified extract=N query param. The default
+// is 1 (extract=true == extract=1 == "extract one result"); callers raise it up
+// to maxExtractTop. These mirror the CLI's --extract flag limits.
+const (
+	defaultExtractTop = 1
+	maxExtractTop     = 5
+)
+
 // ErrCaptcha is returned when the engine detects a captcha challenge page.
 // This error is treated as non-retryable by resilient search policies.
 var ErrCaptcha = errors.New("captcha detected")
@@ -385,33 +393,12 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
 	if err != nil {
 		return errInvalidParam(fmt.Sprintf("features: %v", err))
 	}
-	searchQuery.Extract, err = strconv.ParseBool(reqCtx.Query("extract", "0"))
-	if err != nil {
-		return errInvalidParam(fmt.Sprintf("extract: %v", err))
-	}
-	searchQuery.ExtractTop = 3
-	if raw := strings.TrimSpace(reqCtx.Query("extract_top")); raw != "" {
-		extractTop, err := strconv.Atoi(raw)
-		if err != nil {
-			return errInvalidParam("extract_top must be an integer")
-		}
-		if extractTop < 1 {
-			extractTop = 1
-		}
-		if extractTop > 5 {
-			extractTop = 5
-		}
-		searchQuery.ExtractTop = extractTop
-	}
-	searchQuery.ExtractMode = strings.ToLower(strings.TrimSpace(reqCtx.Query("extract_mode", "auto")))
-	switch searchQuery.ExtractMode {
-	case "auto", "fast", "rendered":
-	default:
-		return errInvalidParam("extract_mode must be one of auto, fast, rendered")
-	}
-	searchQuery.ExtractMinRunes, err = parseNonNegativeIntQuery(reqCtx.Query("min_runes"), 0)
-	if err != nil {
-		return errInvalidParam("min_runes must be a non-negative integer")
+	// extract is a unified bool-or-int knob: extract=0/false disables, extract=N
+	// (or true/1) extracts the top N results. The tuning params extract_mode and
+	// min_runes also imply extraction (extract=0 still overrides them). The
+	// default depth is 1 — true == 1 == "extract one result".
+	if err := parseExtractParams(reqCtx, searchQuery); err != nil {
+		return err
 	}
 
 	searchQuery.ProxyOverride, err = NormalizeProxyRequestOverride(reqCtx.Get("X-Use-Proxy"))
@@ -437,6 +424,72 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
 	return nil
 }
 
+// parseExtractParams reads the unified extract knob plus its tuning params onto
+// q. The extract param is bool-or-int:
+//
+//	extract=0 / extract=false  → extraction off
+//	extract=true / extract=1   → on, top 1
+//	extract=N (1..5)           → on, top N (clamped to maxExtractTop)
+//
+// extract_mode and min_runes tune how extraction runs and imply extraction when
+// present, unless extract is explicitly set (extract=0 wins over them). When
+// extraction is on but no depth is given, ExtractTop defaults to 1.
+func parseExtractParams(reqCtx *fiber.Ctx, q *Query) error {
+	q.ExtractTop = defaultExtractTop
+
+	// extract accepts both bool spellings (true/false/1/0) and an integer depth.
+	// Try bool first so legacy true/false keep working, then fall back to int.
+	extractExplicit := false
+	if raw := strings.TrimSpace(reqCtx.Query("extract")); raw != "" {
+		extractExplicit = true
+		if b, err := strconv.ParseBool(raw); err == nil {
+			q.Extract = b
+			if b {
+				q.ExtractTop = 1
+			}
+		} else if n, err := strconv.Atoi(raw); err == nil {
+			q.Extract = n > 0
+			if n > 0 {
+				q.ExtractTop = clampExtractTop(n)
+			}
+		} else {
+			return errInvalidParam("extract must be a boolean or an integer (0 disables, N extracts top N)")
+		}
+	}
+
+	q.ExtractMode = strings.ToLower(strings.TrimSpace(reqCtx.Query("extract_mode", "auto")))
+	switch q.ExtractMode {
+	case "auto", "fast", "rendered":
+	default:
+		return errInvalidParam("extract_mode must be one of auto, fast, rendered")
+	}
+	if !extractExplicit && strings.TrimSpace(reqCtx.Query("extract_mode")) != "" {
+		q.Extract = true
+	}
+
+	minRunes, err := parseNonNegativeIntQuery(reqCtx.Query("min_runes"), 0)
+	if err != nil {
+		return errInvalidParam("min_runes must be a non-negative integer")
+	}
+	q.ExtractMinRunes = minRunes
+	if !extractExplicit && strings.TrimSpace(reqCtx.Query("min_runes")) != "" {
+		q.Extract = true
+	}
+
+	return nil
+}
+
+// clampExtractTop bounds a requested extraction depth to [1, maxExtractTop].
+func clampExtractTop(n int) int {
+	if n < 1 {
+		return 1
+	}
+	if n > maxExtractTop {
+		return maxExtractTop
+	}
+	return n
+}
+
 // SearchEngineOptions controls engine pacing, selector waits, and captcha
 // handling behavior shared by browser and raw implementations.
 type SearchEngineOptions struct {

core/logger.go

@@ -251,7 +251,7 @@ func quoteIfNeeded(s string) string {
 	return s
 }
 
-func InitLogger(isVerbose, isDebug bool, format string) {
+func InitLogger(isVerbose, isDebug, isQuiet bool, format string) {
 	switch format {
 	case LogFormatText:
 		logrus.SetFormatter(&bracketFormatter{TimestampFormat: "2006-01-02 15:04:05"})
@@ -261,16 +261,22 @@ func InitLogger(isVerbose, isDebug bool, format string) {
 		})
 	}
 
-	if isDebug {
-		logrus.SetOutput(io.MultiWriter(os.Stdout))
+	// Logs go to stderr (+ optional file) so stdout carries only the payload.
+	switch {
+	case isDebug:
+		logrus.SetOutput(io.MultiWriter(os.Stderr))
 		logrus.SetReportCaller(true)
-	} else {
+	case isQuiet:
+		// One-shot CLI default: stderr only, no ./logs.txt in the user's CWD.
+		logrus.SetOutput(os.Stderr)
+		logrus.SetReportCaller(false)
+	default:
 		f, err := os.OpenFile("./logs.txt", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "Failed to open logs file ./logs.txt: %v\n", err)
-			logrus.SetOutput(io.MultiWriter(os.Stdout))
+			logrus.SetOutput(io.MultiWriter(os.Stderr))
 		} else {
-			logrus.SetOutput(io.MultiWriter(f, os.Stdout))
+			logrus.SetOutput(io.MultiWriter(f, os.Stderr))
 		}
 		logrus.SetReportCaller(false)
 	}
@@ -282,5 +288,9 @@ func InitLogger(isVerbose, isDebug bool, format string) {
 	if isDebug {
 		level = logrus.TraceLevel
 	}
+	if isQuiet && !isVerbose && !isDebug {
+		// Quiet keeps only warnings/errors on stderr.
+		level = logrus.WarnLevel
+	}
 	logrus.SetLevel(level)
 }

core/page_helpers.go

@@ -102,8 +102,49 @@ func HasAttribute(el *rod.Element, attr string) bool {
 	return err == nil && v != nil
 }
 
-// FirstNonEmptyText returns the trimmed text of the first selector under root
-// that yields non-empty content. Empty string if none match.
+// NormalizeWhitespace collapses runs of whitespace (newlines, source
+// indentation) into single spaces and trims the result.
+func NormalizeWhitespace(s string) string {
+	return strings.Join(strings.Fields(s), " ")
+}
+
+// ElementText returns el's visible text, falling back to textContent (for nodes
+// rod's Text() leaves empty), normalized. Empty string if el is nil or blank.
+func ElementText(el *rod.Element) string {
+	if el == nil {
+		return ""
+	}
+	if text, err := el.Text(); err == nil {
+		if normalized := NormalizeWhitespace(text); normalized != "" {
+			return normalized
+		}
+	}
+	if value, err := el.Property("textContent"); err == nil {
+		return NormalizeWhitespace(value.String())
+	}
+	return ""
+}
+
+// ElementAttribute returns the first non-empty value among attrs on el,
+// normalized. Empty string if el is nil or none are set.
+func ElementAttribute(el *rod.Element, attrs ...string) string {
+	if el == nil {
+		return ""
+	}
+	for _, attr := range attrs {
+		value, err := el.Attribute(attr)
+		if err != nil || value == nil {
+			continue
+		}
+		if normalized := NormalizeWhitespace(*value); normalized != "" {
+			return normalized
+		}
+	}
+	return ""
+}
+
+// FirstNonEmptyText returns the text (see ElementText) of the first selector
+// under root that yields non-empty content. Empty string if none match.
 func FirstNonEmptyText(root *rod.Element, selectors ...string) string {
 	if root == nil {
 		return ""
@@ -113,12 +154,8 @@ func FirstNonEmptyText(root *rod.Element, selectors ...string) string {
 		if err != nil {
 			continue
 		}
-		text, err := el.Text()
-		if err != nil {
-			continue
-		}
-		if trimmed := strings.TrimSpace(text); trimmed != "" {
-			return trimmed
+		if text := ElementText(el); text != "" {
+			return text
 		}
 	}
 	return ""

core/proxy.go

@@ -24,6 +24,10 @@ const (
 	// ProxyPoolQuarantineDuration is how long an exhausted tag pool stays quarantined
 	// before a single probe proxy is re-enabled for recovery testing.
 	ProxyPoolQuarantineDuration = 5 * time.Minute
+	// ProxyChallengeCooldown is how long a captcha/blocked proxy is deprioritized
+	// in rotation. It does not degrade health, so the proxy is still served if
+	// it's the only one left.
+	ProxyChallengeCooldown = 2 * time.Minute
 )
 
 var supportedProxySchemes = map[string]struct{}{
@@ -99,6 +103,9 @@ type proxyState struct {
 	tags     []string
 	failures int
 	disabled bool
+	// challengedUntil deprioritizes (but does not disable) this proxy in
+	// rotation after a captcha/block. See ReportChallenged.
+	challengedUntil time.Time
 }
 
 type ProxyRegistry struct {
@@ -411,21 +418,29 @@ func (r *ProxyRegistry) NextByTagWithContext(ctx context.Context, tag string) st
 		}
 	}
 
+	now := time.Now()
 	start := r.nextByTag[tag]
-	for i := 0; i < len(urls); i++ {
-		idx := (start + i) % len(urls)
-		proxyURL := urls[idx]
-		state := r.states[proxyURL]
-		if state.disabled {
-			continue
-		}
+	// First pass skips challenged proxies; second pass relaxes that so a
+	// challenged-but-healthy proxy is still served rather than failing.
+	for _, skipChallenged := range []bool{true, false} {
+		for i := 0; i < len(urls); i++ {
+			idx := (start + i) % len(urls)
+			proxyURL := urls[idx]
+			state := r.states[proxyURL]
+			if state.disabled {
+				continue
+			}
+			if skipChallenged && now.Before(state.challengedUntil) {
+				continue
+			}
 
-		r.nextByTag[tag] = (idx + 1) % len(urls)
-		WithRequest(ctx).WithFields(logrus.Fields{
-			"proxy_tag": tag,
-			"proxy":     MaskProxyURL(proxyURL),
-		}).Debugf("Selected proxy for tag=%s: %s", tag, MaskProxyURL(proxyURL))
-		return proxyURL
+			r.nextByTag[tag] = (idx + 1) % len(urls)
+			WithRequest(ctx).WithFields(logrus.Fields{
+				"proxy_tag": tag,
+				"proxy":     MaskProxyURL(proxyURL),
+			}).Debugf("Selected proxy for tag=%s: %s", tag, MaskProxyURL(proxyURL))
+			return proxyURL
+		}
 	}
 
 	// All proxies are disabled and no probe could be selected.
@@ -498,22 +513,50 @@ func (r *ProxyRegistry) ReportSuccess(_ context.Context, proxyURL string) {
 	}
 }
 
-func (r *ProxyRegistry) HasHealthyProxyForTag(tag string) bool {
-	tag = normalizeTag(tag)
-	if tag == "" {
-		return false
+// ReportChallenged deprioritizes a captcha/blocked proxy for
+// ProxyChallengeCooldown without degrading its health (unlike ReportFailure, it
+// never disables the proxy or trips quarantine), so the next attempt prefers a
+// different IP.
+func (r *ProxyRegistry) ReportChallenged(ctx context.Context, proxyURL string) {
+	proxyURL, err := NormalizeProxyURL(proxyURL)
+	if err != nil || proxyURL == "" {
+		return
 	}
 
 	r.mu.Lock()
 	defer r.mu.Unlock()
 
-	for _, proxyURL := range r.tagIndex[tag] {
-		if state, ok := r.states[proxyURL]; ok && !state.disabled {
-			return true
-		}
+	state, ok := r.states[proxyURL]
+	if !ok {
+		return
+	}
+	state.challengedUntil = time.Now().Add(ProxyChallengeCooldown)
+	WithRequest(ctx).WithField("proxy", MaskProxyURL(proxyURL)).
+		Debugf("Deprioritized challenged proxy for %s: %s", ProxyChallengeCooldown, MaskProxyURL(proxyURL))
+}
+
+func (r *ProxyRegistry) HasHealthyProxyForTag(tag string) bool {
+	return r.HealthyCountForTag(tag) > 0
+}
+
+// HealthyCountForTag returns how many non-disabled proxies the tag pool holds
+// (a challenged proxy still counts — it's usable, just deprioritized).
+func (r *ProxyRegistry) HealthyCountForTag(tag string) int {
+	tag = normalizeTag(tag)
+	if tag == "" {
+		return 0
 	}
 
-	return false
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	count := 0
+	for _, proxyURL := range r.tagIndex[tag] {
+		if state, ok := r.states[proxyURL]; ok && !state.disabled {
+			count++
+		}
+	}
+	return count
 }
 
 func (r *ProxyRegistry) BuildStats() ProxyStats {

core/proxy_rotation_test.go

@@ -0,0 +1,150 @@
+package core
+
+import (
+	"context"
+	"sync"
+	"testing"
+
+	"golang.org/x/time/rate"
+)
+
+// captchaThenSuccessEngine returns ErrCaptcha for every proxy URL except the
+// one designated as healthy, where it succeeds. It records the proxy URL of
+// each attempt so the test can assert rotation happened.
+type captchaThenSuccessEngine struct {
+	name        string
+	goodProxy   string
+	mu          sync.Mutex
+	seenProxies []string
+}
+
+func (e *captchaThenSuccessEngine) Name() string                  { return e.name }
+func (e *captchaThenSuccessEngine) IsInitialized() bool           { return true }
+func (e *captchaThenSuccessEngine) GetRateLimiter() *rate.Limiter { return nil }
+
+func (e *captchaThenSuccessEngine) Search(ctx context.Context, q Query) ([]SearchResult, error) {
+	e.mu.Lock()
+	e.seenProxies = append(e.seenProxies, q.ProxyURL)
+	e.mu.Unlock()
+	if q.ProxyURL == e.goodProxy {
+		return []SearchResult{{Title: "ok", URL: "https://example.com", Rank: 1}}, nil
+	}
+	return nil, ErrCaptcha
+}
+
+func (e *captchaThenSuccessEngine) SearchImage(ctx context.Context, q Query) ([]SearchResult, error) {
+	return e.Search(ctx, q)
+}
+
+func (e *captchaThenSuccessEngine) attempts() int {
+	e.mu.Lock()
+	defer e.mu.Unlock()
+	return len(e.seenProxies)
+}
+
+func tagPoolSearcher(t *testing.T, engine SearchEngine, entries []ProxyEntryConfig) *ResilientSearcher {
+	t.Helper()
+	cfg := DefaultResilientConfig()
+	cfg.Retry.MaxRetries = 0
+	cfg.CircuitBreaker.FailureThreshold = 5
+	cfg.Proxy = ProxyConfig{
+		Runtime:        ProxyRuntimeBrowser,
+		Proxies:        ProxiesConfig{Entries: entries},
+		EnginePolicies: map[string]string{engine.Name(): "rot"},
+	}
+	return NewResilientSearcher([]SearchEngine{engine}, cfg)
+}
+
+func TestReportChallengedDeprioritizesWithoutDisabling(t *testing.T) {
+	registry, err := NewProxyRegistry([]ProxyEntryConfig{
+		{URL: "http://proxy1:8080", Tags: []string{"rot"}},
+		{URL: "http://proxy2:8080", Tags: []string{"rot"}},
+	}, 3)
+	if err != nil {
+		t.Fatalf("new proxy registry: %v", err)
+	}
+	ctx := context.Background()
+
+	// Challenge proxy1 from a fresh index; the next selection must skip it.
+	registry.ReportChallenged(ctx, "http://proxy1:8080")
+	if got := registry.NextByTag("rot"); got != "http://proxy2:8080" {
+		t.Fatalf("expected challenged proxy to be skipped, got %q", got)
+	}
+
+	// Health is untouched: both proxies still count as healthy.
+	if n := registry.HealthyCountForTag("rot"); n != 2 {
+		t.Fatalf("challenge must not degrade health, healthy=%d", n)
+	}
+
+	// When both are challenged, rotation still serves one (relaxed second pass).
+	registry.ReportChallenged(ctx, "http://proxy2:8080")
+	if got := registry.NextByTag("rot"); got == "" {
+		t.Fatal("expected a proxy even when all are challenged")
+	}
+}
+
+func TestSearchWithProtection_RotatesProxyOnCaptcha(t *testing.T) {
+	// Two proxies in the same tag pool; the second one is the one that works.
+	// NextByTag serves proxy1 first, so the first attempt gets a captcha and the
+	// retry should pick proxy2 and succeed.
+	good := "http://proxy2:8080"
+	engine := &captchaThenSuccessEngine{name: "google", goodProxy: good}
+	rs := tagPoolSearcher(t, engine, []ProxyEntryConfig{
+		{URL: "http://proxy1:8080", Tags: []string{"rot"}},
+		{URL: good, Tags: []string{"rot"}},
+	})
+
+	results, _, meta, err := rs.SearchPrimary(context.Background(), engine, Query{Text: "rotate"})
+	if err != nil {
+		t.Fatalf("expected success after rotation, got %v", err)
+	}
+	if len(results) != 1 {
+		t.Fatalf("expected 1 result, got %d", len(results))
+	}
+	if meta.Attempts != 2 {
+		t.Fatalf("expected 2 proxy attempts, got %d", meta.Attempts)
+	}
+	if got := engine.attempts(); got != 2 {
+		t.Fatalf("expected engine called twice, got %d", got)
+	}
+}
+
+func TestSearchWithProtection_NoRotationWithSingleProxy(t *testing.T) {
+	// Only one proxy in the pool: captcha must fail fast without a second
+	// attempt (HealthyCountForTag < 2).
+	engine := &captchaThenSuccessEngine{name: "google", goodProxy: "http://unused:8080"}
+	rs := tagPoolSearcher(t, engine, []ProxyEntryConfig{
+		{URL: "http://proxy1:8080", Tags: []string{"rot"}},
+	})
+
+	_, _, meta, err := rs.SearchPrimary(context.Background(), engine, Query{Text: "single"})
+	if err == nil {
+		t.Fatal("expected captcha failure with a single proxy")
+	}
+	if meta.Attempts != 1 {
+		t.Fatalf("expected exactly 1 attempt with single proxy, got %d", meta.Attempts)
+	}
+	if got := engine.attempts(); got != 1 {
+		t.Fatalf("expected engine called once, got %d", got)
+	}
+}
+
+func TestSearchWithProtection_DirectModeFailsFastOnCaptcha(t *testing.T) {
+	// Direct mode (no proxy config): captcha is non-retryable and rotation must
+	// not kick in.
+	engine := &captchaThenSuccessEngine{name: "google", goodProxy: "http://never:8080"}
+	cfg := DefaultResilientConfig()
+	cfg.Retry.MaxRetries = 0
+	rs := NewResilientSearcher([]SearchEngine{engine}, cfg)
+
+	_, _, meta, err := rs.SearchPrimary(context.Background(), engine, Query{Text: "direct"})
+	if err == nil {
+		t.Fatal("expected captcha failure in direct mode")
+	}
+	if meta.Attempts > 1 {
+		t.Fatalf("direct mode must not rotate proxies, attempts=%d", meta.Attempts)
+	}
+	if got := engine.attempts(); got != 1 {
+		t.Fatalf("expected engine called once in direct mode, got %d", got)
+	}
+}

core/query_extract_param_test.go

@@ -0,0 +1,73 @@
+package core
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+
+	"github.com/gofiber/fiber/v2"
+)
+
+// TestInitFromContextExtractParams verifies how the unified extract knob and its
+// tuning params map onto Query.Extract / Query.ExtractTop. Key behaviors:
+//   - extract is bool-or-int: extract=0/false off, extract=true/1 → top 1,
+//     extract=N → top N (clamped to [1,5]).
+//   - extract_mode/min_runes imply extraction (top defaults to 1), but an
+//     explicit extract=0 still wins over them.
+func TestInitFromContextExtractParams(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		query       string
+		wantExtract bool
+		wantTop     int
+	}{
+		{"no params defaults off", "?text=q", false, 1},
+		{"extract=true means top 1", "?text=q&extract=true", true, 1},
+		{"extract=1 means top 1", "?text=q&extract=1", true, 1},
+		{"extract=3 means top 3", "?text=q&extract=3", true, 3},
+		{"extract=N clamps high", "?text=q&extract=99", true, 5},
+		{"extract=0 disables", "?text=q&extract=0", false, 1},
+		{"extract=false disables", "?text=q&extract=false", false, 1},
+		{"extract_mode implies extract", "?text=q&extract_mode=fast", true, 1},
+		{"min_runes implies extract", "?text=q&min_runes=200", true, 1},
+		{"explicit extract=0 overrides tuning", "?text=q&extract=0&extract_mode=fast", false, 1},
+	}
+
+	app := fiber.New()
+	app.Get("/probe", func(c *fiber.Ctx) error {
+		q := Query{}
+		if err := q.InitFromContext(c); err != nil {
+			return c.Status(http.StatusBadRequest).SendString(err.Error())
+		}
+		extract := "0"
+		if q.Extract {
+			extract = "1"
+		}
+		c.Set("X-Extract", extract)
+		c.Set("X-Extract-Top", strconv.Itoa(q.ExtractTop))
+		return c.SendStatus(http.StatusOK)
+	})
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			req := httptest.NewRequest(http.MethodGet, "/probe"+tt.query, nil)
+			resp, err := app.Test(req, -1)
+			if err != nil {
+				t.Fatalf("request failed: %v", err)
+			}
+			if resp.StatusCode != http.StatusOK {
+				t.Fatalf("unexpected status %d for %s", resp.StatusCode, tt.query)
+			}
+			gotExtract := resp.Header.Get("X-Extract") == "1"
+			if gotExtract != tt.wantExtract {
+				t.Errorf("%s: Extract = %v, want %v", tt.query, gotExtract, tt.wantExtract)
+			}
+			if got := resp.Header.Get("X-Extract-Top"); got != strconv.Itoa(tt.wantTop) {
+				t.Errorf("%s: ExtractTop = %s, want %d", tt.query, got, tt.wantTop)
+			}
+		})
+	}
+}

core/resilient.go

@@ -28,6 +28,9 @@ type ProxyExecutionMeta struct {
 	Mode string `json:"mode"`
 	Tag  string `json:"tag,omitempty"`
 	Used string `json:"used"`
+	// Attempts is how many proxies were tried; >1 means a challenged proxy was
+	// rotated out in tag-pool mode.
+	Attempts int `json:"attempts,omitempty"`
 }
 
 type ResilientConfig struct {
@@ -171,56 +174,83 @@ func (rs *ResilientSearcher) searchWithProtection(ctx context.Context, engine Se
 	attemptMeta := rs.baseProxyMeta(policy)
 
 	startedAt := time.Now()
-	result := RetryableSearch(ctx, rs.retryCfg, engine.Name(), func(callCtx context.Context) ([]SearchResult, error) {
-		limiter := engine.GetRateLimiter()
-		if limiter != nil {
-			if err := limiter.Wait(callCtx); err != nil {
-				return nil, normalizeLimiterWaitErr(callCtx, err)
-			}
-		}
 
-		attemptQuery := q
-		proxyURL := ""
-		reportToRegistry := false
-		attemptMeta = rs.baseProxyMeta(policy)
-
-		switch policy.Mode {
-		case ProxyModeOff:
-			attemptQuery.ProxyURL = ""
-			attemptMeta.Used = "direct"
-		case ProxyModeRequestURL:
-			proxyURL = q.ProxyURL
-			attemptQuery.ProxyURL = proxyURL
-			attemptMeta.Used = MaskProxyURL(proxyURL)
-		case ProxyModeTagPool:
-			proxyURL = rs.selectProxyForQuery(policy, q, engineCtx)
-			if proxyURL == "" {
-				return nil, fmt.Errorf("%w: no healthy proxy available for tag %q", ErrProxyUnavailable, policy.Tag)
-			}
-			attemptQuery.ProxyURL = proxyURL
-			reportToRegistry = policy.Tag != ""
-			attemptMeta.Used = MaskProxyURL(proxyURL)
-		}
-
-		requestCtx := proxyRequestContext(callCtx, engine.Name(), attemptQuery)
-		results, err := invokeEngine(requestCtx, engine, attemptQuery, isImage)
-
-		if reportToRegistry {
-			rs.reportProxyAttempt(engineCtx, proxyURL, err)
-		}
-		if err != nil && errors.Is(err, ErrCaptcha) && rs.proxyCfg.Proxies.Lanes.DropCookiesOnChallenge {
-			// Recompute lane key only to gate the call: empty key means we have no
-			// session to drop cookies for. The dropper recomputes the key itself
-			// when it actually needs to mutate lane state.
-			if !ProxyLaneKeyForTenant(engine.Name(), TenantFromContext(callCtx), attemptQuery, attemptQuery.ProxyURL).Empty() {
-				if dropper, ok := engine.(proxyLaneCookieDropper); ok {
-					dropper.DropProxyLaneCookies(callCtx, attemptQuery)
+	// lastProxyURL is the unmasked proxy of the last attempt, for rotation below.
+	lastProxyURL := ""
+	runOnce := func() RetryResult {
+		return RetryableSearch(ctx, rs.retryCfg, engine.Name(), func(callCtx context.Context) ([]SearchResult, error) {
+			limiter := engine.GetRateLimiter()
+			if limiter != nil {
+				if err := limiter.Wait(callCtx); err != nil {
+					return nil, normalizeLimiterWaitErr(callCtx, err)
 				}
 			}
-		}
 
-		return results, err
-	})
+			attemptQuery := q
+			proxyURL := ""
+			reportToRegistry := false
+			attemptMeta = rs.baseProxyMeta(policy)
+
+			switch policy.Mode {
+			case ProxyModeOff:
+				attemptQuery.ProxyURL = ""
+				attemptMeta.Used = "direct"
+			case ProxyModeRequestURL:
+				proxyURL = q.ProxyURL
+				attemptQuery.ProxyURL = proxyURL
+				attemptMeta.Used = MaskProxyURL(proxyURL)
+			case ProxyModeTagPool:
+				proxyURL = rs.selectProxyForQuery(policy, q, engineCtx)
+				if proxyURL == "" {
+					return nil, fmt.Errorf("%w: no healthy proxy available for tag %q", ErrProxyUnavailable, policy.Tag)
+				}
+				attemptQuery.ProxyURL = proxyURL
+				reportToRegistry = policy.Tag != ""
+				attemptMeta.Used = MaskProxyURL(proxyURL)
+			}
+			lastProxyURL = proxyURL
+
+			requestCtx := proxyRequestContext(callCtx, engine.Name(), attemptQuery)
+			results, err := invokeEngine(requestCtx, engine, attemptQuery, isImage)
+
+			if reportToRegistry {
+				rs.reportProxyAttempt(engineCtx, proxyURL, err)
+			}
+			if err != nil && errors.Is(err, ErrCaptcha) && rs.proxyCfg.Proxies.Lanes.DropCookiesOnChallenge {
+				// Recompute lane key only to gate the call: empty key means we have no
+				// session to drop cookies for. The dropper recomputes the key itself
+				// when it actually needs to mutate lane state.
+				if !ProxyLaneKeyForTenant(engine.Name(), TenantFromContext(callCtx), attemptQuery, attemptQuery.ProxyURL).Empty() {
+					if dropper, ok := engine.(proxyLaneCookieDropper); ok {
+						dropper.DropProxyLaneCookies(callCtx, attemptQuery)
+					}
+				}
+			}
+
+			return results, err
+		})
+	}
+
+	result := runOnce()
+	attemptMeta.Attempts = 1
+
+	// On a captcha/block/rate-limit (non-retryable inside RetryableSearch), if
+	// the tag pool has another healthy proxy, deprioritize the burned one and
+	// retry once with the next. Tag-pool only — direct/request-url/global can't
+	// rotate.
+	if result.Err != nil &&
+		policy.Mode == ProxyModeTagPool &&
+		policy.Tag != "" &&
+		rs.proxyRegistry != nil &&
+		IsProxyChallengeError(result.Err) &&
+		rs.proxyRegistry.HealthyCountForTag(policy.Tag) >= 2 &&
+		ctx.Err() == nil {
+		rs.proxyRegistry.ReportChallenged(engineCtx, lastProxyURL)
+		WithRequestEngine(ctx, engine.Name()).WithError(result.Err).
+			Debug("Challenged proxy rotated out, retrying once with next proxy")
+		result = runOnce()
+		attemptMeta.Attempts = 2
+	}
 
 	if result.Err != nil {
 		if shouldRecordCircuitFailure(result.Err) {
@@ -233,6 +263,14 @@ func (rs *ResilientSearcher) searchWithProtection(ctx context.Context, engine Se
 	return result.Results, attemptMeta, nil
 }
 
+// IsProxyChallengeError reports whether err is a captcha/block/rate-limit — an
+// IP-reputation problem another proxy might dodge.
+func IsProxyChallengeError(err error) bool {
+	return errors.Is(err, ErrCaptcha) ||
+		errors.Is(err, ErrBlocked) ||
+		errors.Is(err, ErrRateLimited)
+}
+
 func shouldRecordCircuitFailure(err error) bool {
 	return err != nil &&
 		!IsContextDone(err) &&

core/server.go

@@ -1451,6 +1451,9 @@ func (s *Server) applyProxyHeaders(c *fiber.Ctx, meta ProxyExecutionMeta) {
 		c.Set("X-Proxy-Tag", tag)
 	}
 	c.Set("X-Proxy-Used", used)
+	if meta.Attempts > 1 {
+		c.Set("X-Proxy-Attempts", strconv.Itoa(meta.Attempts))
+	}
 }
 
 func setNetworkBytesHeader(c *fiber.Ctx, ctx context.Context) {

core/server_extract.go

@@ -116,14 +116,21 @@ func (s *Server) newExtractor() extractpkg.Extractor {
 }
 
 func (s *Server) rawExtractFetch(ctx context.Context, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) {
-	cfg := s.opts.Extract.Normalized()
+	return RawExtractFetch(ctx, req, s.opts.Extract, s.opts.FingerprintBrowserOpts.Insecure)
+}
+
+// RawExtractFetch performs the browserless extraction fetch: validate the
+// target, issue a guarded HTTP GET, classify the status, and return the body
+// capped to the byte budget. Shared by the HTTP server and the CLI.
+func RawExtractFetch(ctx context.Context, req extractpkg.ExtractRequest, cfg extractpkg.Config, insecure bool) (*extractpkg.FetchResponse, error) {
+	cfg = cfg.Normalized()
 	if err := validateExtractTargetURL(ctx, req.URL, cfg.AllowPrivateNetworks); err != nil {
 		return nil, err
 	}
 	resp, err := RawSearchRequest(ctx, req.URL, Query{
 		ProxyURL:             req.ProxyURL,
 		LangCode:             req.LangCode,
-		Insecure:             s.opts.FingerprintBrowserOpts.Insecure,
+		Insecure:             insecure,
 		GuardPrivateNetworks: !cfg.AllowPrivateNetworks,
 	})
 	if err != nil {
@@ -135,7 +142,7 @@ func (s *Server) rawExtractFetch(ctx context.Context, req extractpkg.ExtractRequ
 	}
 	limit := int64(req.MaxBytes)
 	if limit <= 0 {
-		limit = int64(s.opts.Extract.Normalized().MaxBytes)
+		limit = int64(cfg.MaxBytes)
 	}
 	body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
 	if err != nil {
@@ -159,7 +166,15 @@ func (s *Server) renderedExtractFetch(ctx context.Context, req extractpkg.Extrac
 	if err != nil {
 		return nil, err
 	}
-	page, err := browser.Navigate(WithRequestProxyURL(ctx, req.ProxyURL), req.URL)
+	return RenderExtractHTML(WithRequestProxyURL(ctx, req.ProxyURL), browser, req)
+}
+
+// RenderExtractHTML navigates an already-resolved browser to the target,
+// returns its rendered HTML capped to the byte budget, and always closes the
+// page. Shared by the HTTP server's BrowserResolver path and the CLI's
+// one-shot browser. Callers own target validation and proxy gating.
+func RenderExtractHTML(ctx context.Context, browser *Browser, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) {
+	page, err := browser.Navigate(ctx, req.URL)
 	if err != nil {
 		return nil, err
 	}
@@ -220,7 +235,17 @@ func validateExtractTargetURL(ctx context.Context, rawURL string, allowPrivateNe
 }
 
 func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope, q Query, format string) {
-	cfg := s.opts.Extract.Normalized()
+	EnrichEnvelopeWithExtraction(ctx, env, q, format, s.newExtractor(), s.opts.Extract)
+}
+
+// EnrichEnvelopeWithExtraction fills env.Results[*].Extracted by running the
+// extractor over the top organic results, with candidate fill-in when a top
+// result fails. It is shared by the HTTP search handler and the CLI so both
+// apply the same depth bounds, batch deadline, and result selection. The
+// extractor and cfg are supplied by the caller (the server reuses its
+// long-lived browser pool; the CLI builds a one-shot browser).
+func EnrichEnvelopeWithExtraction(ctx context.Context, env *Envelope, q Query, format string, extractor extractpkg.Extractor, cfg extractpkg.Config) {
+	cfg = cfg.Normalized()
 	if env == nil || !q.Extract || !cfg.Enabled {
 		return
 	}
@@ -231,11 +256,7 @@ func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope
 	if format == "text" {
 		contentFormat = "text"
 	}
-	extractor := s.newExtractor()
-	limit := q.ExtractTop
-	if limit <= 0 || limit > 5 {
-		limit = 3
-	}
+	limit := clampExtractTop(q.ExtractTop)
 	if limit > len(env.Results) {
 		limit = len(env.Results)
 	}
@@ -245,7 +266,7 @@ func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope
 	}
 
 	// Per-fetch timeouts bound a single URL; this aggregate deadline bounds the
-	// whole batch so a few slow/hanging targets can't stretch the search request
+	// whole batch so a few slow/hanging targets can't stretch the request
 	// open-endedly. The ceiling is derived from the per-URL budget (see
 	// Config.BatchTimeout) rather than a separate knob. When it fires, in-flight
 	// fetches are cancelled and any not yet started record a timeout error instead
@@ -256,7 +277,7 @@ func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope
 	extractOne := func(idx int) {
 		// Skip the fetch entirely if the batch budget is already spent.
 		if err := ctx.Err(); err != nil {
-			env.Results[idx].Extracted = &ExtractedContent{Error: sanitizeExtractError(err)}
+			env.Results[idx].Extracted = &ExtractedContent{Error: SanitizeExtractError(err)}
 			return
 		}
 		req := extractpkg.ExtractRequest{
@@ -270,14 +291,14 @@ func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope
 		}
 		result, err := extractor.Extract(ctx, req)
 		if err != nil {
-			env.Results[idx].Extracted = &ExtractedContent{Error: sanitizeExtractError(err)}
+			env.Results[idx].Extracted = &ExtractedContent{Error: SanitizeExtractError(err)}
 			return
 		}
 		content := result.Markdown
 		if contentFormat == "text" {
 			content = result.Text
 		}
-		if !extractedContentLooksUseful(content) {
+		if !ExtractedContentLooksUseful(content) {
 			env.Results[idx].Extracted = &ExtractedContent{Error: "empty extracted content"}
 			return
 		}
@@ -320,7 +341,10 @@ func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope
 
 const minUsefulExtractRunes = 80
 
-func extractedContentLooksUseful(content string) bool {
+// ExtractedContentLooksUseful reports whether extracted page content is long
+// enough to keep, rather than an empty/boilerplate shell. Shared by the HTTP
+// server and the CLI so both apply the same threshold.
+func ExtractedContentLooksUseful(content string) bool {
 	return len([]rune(strings.TrimSpace(content))) >= minUsefulExtractRunes
 }
 
@@ -337,7 +361,7 @@ func extractedSuccessCount(results []Result) int {
 func extractedResultSucceeded(result Result) bool {
 	return result.Extracted != nil &&
 		result.Extracted.Error == "" &&
-		extractedContentLooksUseful(result.Extracted.Content)
+		ExtractedContentLooksUseful(result.Extracted.Content)
 }
 
 func sendExtractResult(c *fiber.Ctx, format string, result *extractpkg.ExtractResult) error {
@@ -379,7 +403,9 @@ func parseBoolDefault(raw string, fallback bool) bool {
 	return raw == "1" || strings.EqualFold(raw, "true") || strings.EqualFold(raw, "yes")
 }
 
-func sanitizeExtractError(err error) string {
+// SanitizeExtractError trims and length-bounds an extraction error for safe
+// inclusion in a response payload. Shared by the HTTP server and the CLI.
+func SanitizeExtractError(err error) string {
 	if err == nil {
 		return ""
 	}