feat(cli): clean search, add extract/format flags; harden engines & proxy rotation; fix bugs; update docs

- Add structured `search [engine] [query]` CLI: --limit/--lang/--region/--site/--file, --format (json|text|markdown|ndjson), --extract N, --search-timeout; Envelope and route logs to stderr with a --quiet default (fixes stdout pollution)
- Unify engines behind a single engineSpec registry (CLI + serve share it)
- Unify the extract knob to bool-or-int `extract=N` (drop extract_top); CLI and HTTP share core batch extraction, raw/rendered fetch, and clamp helpers
- Engines: Ecosia CF captcha detection (raw + browser), Yandex progressive-result wait, Google PAA poll + Has() existence probes, Bing title/desc attribute fallbacks
- Proxy: rotate challenged proxies out of the tag pool for one retry (X-Proxy-Attempts); browser health-ping skip window; opt-in WaitStable

cmd/engines.go

@@ -0,0 +1,78 @@
+package cmd
+
+import (
+	"context"
+	"io"
+
+	"github.com/karust/openserp/baidu"
+	"github.com/karust/openserp/bing"
+	"github.com/karust/openserp/core"
+	"github.com/karust/openserp/duckduckgo"
+	"github.com/karust/openserp/ecosia"
+	"github.com/karust/openserp/google"
+	"github.com/karust/openserp/yandex"
+)
+
+// engineSpec is the single registry row for a search engine, driving CLI search,
+// raw dispatch, serve's browserEngineSpecs, and the alias/validation strings.
+// cfg points into the live config global; rawSearchFn is nil when an engine has
+// no browserless mode.
+type engineSpec struct {
+	name        string
+	aliases     []string
+	factory     func(core.Browser, core.SearchEngineOptions) core.SearchEngine
+	rawSearchFn func(context.Context, core.Query) ([]core.SearchResult, error)
+	parseHTMLFn func(io.Reader) ([]core.SearchResult, error)
+	cfg         *EngineConfig
+}
+
+func (s engineSpec) opts() core.SearchEngineOptions {
+	return s.cfg.SearchEngineOptions
+}
+
+func engineSpecs() []engineSpec {
+	return []engineSpec{
+		{name: "google", factory: newEngine(google.New), rawSearchFn: google.Search, parseHTMLFn: google.ParseHTML, cfg: &config.GoogleConfig},
+		{name: "yandex", factory: newEngine(yandex.New), rawSearchFn: yandex.Search, parseHTMLFn: yandex.ParseHTML, cfg: &config.YandexConfig},
+		{name: "baidu", factory: newEngine(baidu.New), rawSearchFn: baidu.Search, parseHTMLFn: baidu.ParseHTML, cfg: &config.BaiduConfig},
+		{name: "bing", factory: newEngine(bing.New), parseHTMLFn: bing.ParseHTML, cfg: &config.BingConfig},
+		{name: "duckduckgo", aliases: []string{"duck", "ddg"}, factory: newEngine(duckduckgo.New), parseHTMLFn: duckduckgo.ParseHTML, cfg: &config.DuckDuckGoConfig},
+		{name: "ecosia", factory: newEngine(ecosia.New), rawSearchFn: ecosia.Search, parseHTMLFn: ecosia.ParseHTML, cfg: &config.EcosiaConfig},
+	}
+}
+
+// newEngine adapts a concrete pkg.New (returning *Engine) to the
+// core.SearchEngine-typed factory the registry stores.
+func newEngine[T core.SearchEngine](ctor func(core.Browser, core.SearchEngineOptions) T) func(core.Browser, core.SearchEngineOptions) core.SearchEngine {
+	return func(b core.Browser, o core.SearchEngineOptions) core.SearchEngine {
+		return ctor(b, o)
+	}
+}
+
+// engineValidArgs returns every accepted engine token (canonical names +
+// aliases) for cobra's OnlyValidArgs validation.
+func engineValidArgs() []string {
+	specs := engineSpecs()
+	args := make([]string, 0, len(specs))
+	for _, s := range specs {
+		args = append(args, s.name)
+		args = append(args, s.aliases...)
+	}
+	return args
+}
+
+// resolveEngineSpec returns the spec whose canonical name or alias matches raw
+// (case/space already normalized by the caller), or false when unknown.
+func resolveEngineSpec(raw string) (engineSpec, bool) {
+	for _, s := range engineSpecs() {
+		if s.name == raw {
+			return s, true
+		}
+		for _, alias := range s.aliases {
+			if alias == raw {
+				return s, true
+			}
+		}
+	}
+	return engineSpec{}, false
+}

cmd/root.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	version               = "0.8.3"
+	version               = "0.8.4"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )
@@ -51,6 +51,7 @@ type ServerConfig struct {
 	ConfigPath    string `mapstructure:"config_path"`
 	IsDebug       bool   `mapstructure:"debug"`
 	IsVerbose     bool   `mapstructure:"verbose"`
+	IsQuiet       bool   `mapstructure:"quiet"`
 	IsRawRequests bool   `mapstructure:"raw_requests"`
 	Insecure      bool   `mapstructure:"insecure"`
 }
@@ -115,6 +116,7 @@ var flagToConfigKey = map[string]string{
 	"profiles-json":           "app.profiles",
 	"verbose":                 "server.verbose",
 	"debug":                   "server.debug",
+	"quiet":                   "server.quiet",
 	"head":                    "app.head",
 	"leakless":                "app.leakless",
 	"raw":                     "server.raw_requests",
@@ -154,12 +156,24 @@ var RootCmd = &cobra.Command{
 		}
 		config.App.LogFormat = logFormat
 
-		core.InitLogger(config.Server.IsVerbose, config.Server.IsDebug, config.App.LogFormat)
+		// One-shot CLI commands default to quiet so stdout is payload-only.
+		// Server mode keeps request logs unless server.quiet is set.
+		quiet := config.Server.IsQuiet
+		if commandDefaultsToQuiet(cmd) && !cmd.Flags().Changed("quiet") {
+			quiet = true
+		}
+		config.Server.IsQuiet = quiet
+
+		core.InitLogger(config.Server.IsVerbose, config.Server.IsDebug, quiet, config.App.LogFormat)
 		logrus.WithField("config", sanitizedConfigForLog(config)).Debug("Final config")
 		return nil
 	},
 }
 
+func commandDefaultsToQuiet(cmd *cobra.Command) bool {
+	return cmd != nil && cmd.Name() != serveCMD.Name()
+}
+
 func sanitizedConfigForLog(cfg Config) map[string]interface{} {
 	return map[string]interface{}{
 		"server": cfg.Server,
@@ -380,6 +394,7 @@ func setConfigDefaults(v *viper.Viper) {
 	v.SetDefault("server.port", 7070)
 	v.SetDefault("server.debug", false)
 	v.SetDefault("server.verbose", false)
+	v.SetDefault("server.quiet", false)
 	v.SetDefault("server.raw_requests", false)
 	v.SetDefault("server.insecure", false)
 	v.SetDefault("app.log_format", "")
@@ -436,8 +451,9 @@ func init() {
 	RootCmd.PersistentFlags().StringVar(&config.App.ProfilesJSON, "profiles", "", "Path to browser profile catalog JSON")
 	RootCmd.PersistentFlags().BoolVarP(&config.Server.IsVerbose, "verbose", "v", false, "Use verbose output")
 	RootCmd.PersistentFlags().BoolVarP(&config.Server.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
+	RootCmd.PersistentFlags().BoolVarP(&config.Server.IsQuiet, "quiet", "q", false, "Suppress info logs on stderr (default for CLI commands)")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI")
-	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search")
+	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to ensure browser instances are closed after search")
 	RootCmd.PersistentFlags().BoolVarP(&config.Server.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
 	RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
 	RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")

cmd/search.go

@@ -4,42 +4,91 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"os"
+	"net/url"
 	"strings"
+	"sync"
 	"time"
 
-	"github.com/karust/openserp/baidu"
-	"github.com/karust/openserp/bing"
+	"github.com/google/uuid"
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/duckduckgo"
-	"github.com/karust/openserp/ecosia"
-	"github.com/karust/openserp/google"
-	"github.com/karust/openserp/yandex"
+	extractpkg "github.com/karust/openserp/extract"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
-var searchCMD = &cobra.Command{
-	Use:     "search",
-	Aliases: []string{"find"},
-	Short:   "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo, ecosia)",
-	Args:    cobra.MatchAll(cobra.OnlyValidArgs, cobra.ExactArgs(2)),
-	Run:     search,
+// searchFlags holds the per-invocation CLI flags for the search command.
+type searchFlags struct {
+	limit    int
+	lang     string
+	region   string
+	start    int
+	site     string
+	filetype string
+	format   string
+	full     bool
+	features bool
+	extract  int
+	timeout  int
 }
 
-func search(cmd *cobra.Command, args []string) {
+var searchOpts searchFlags
+
+var searchCMD = &cobra.Command{
+	Use:     "search [engine] [query]",
+	Aliases: []string{"find"},
+	Short:   "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo, ecosia)",
+	// Validate the engine ourselves; cobra.OnlyValidArgs would also reject the
+	// query arg. ValidArgs still feeds shell completion.
+	Args:      cobra.MatchAll(cobra.ExactArgs(2), validateEngineArg),
+	ValidArgs: engineValidArgs(),
+	RunE:      search,
+}
+
+// validateEngineArg checks args[0] against the registry with a clear error,
+// without rejecting the query arg.
+func validateEngineArg(cmd *cobra.Command, args []string) error {
+	if len(args) == 0 {
+		return nil
+	}
+	if _, ok := resolveEngineSpec(normalizeEngineArg(args[0])); !ok {
+		return fmt.Errorf("unknown engine %q; valid: %s", args[0], strings.Join(engineValidArgs(), ", "))
+	}
+	return nil
+}
+
+func search(cmd *cobra.Command, args []string) error {
+	// Already validated by validateEngineArg, so this can't miss.
 	engineType := normalizeEngineArg(args[0])
+	spec, _ := resolveEngineSpec(engineType)
+
+	format, err := normalizeSearchFormat(searchOpts.format)
+	if err != nil {
+		return err
+	}
+
+	limit := searchOpts.limit
+	if limit <= 0 {
+		limit = 10
+	}
 	query := core.Query{
 		Text:     args[1],
-		Limit:    10,
+		LangCode: searchOpts.lang,
+		Region:   searchOpts.region,
+		Site:     searchOpts.site,
+		Filetype: searchOpts.filetype,
+		Limit:    limit,
+		Start:    searchOpts.start,
 		Filter:   true,
+		Features: searchOpts.features,
 		Insecure: config.Server.Insecure,
 	}
+	if err := applyCLIExtractFlag(&query, searchOpts.extract); err != nil {
+		return err
+	}
 
 	captchaSolverEnabled, captchaSolverAPIKey, err := resolveCaptchaSolverConfig()
 	if err != nil {
-		logrus.WithError(err).Error(fmt.Sprintf("Error validating captcha solver config: %v", err))
-		os.Exit(1)
+		return fmt.Errorf("validate captcha solver config: %w", err)
 	}
 
 	proxyRuntime := core.ProxyRuntimeBrowser
@@ -49,39 +98,45 @@ func search(cmd *cobra.Command, args []string) {
 
 	proxyCfg, err := buildNormalizedProxyConfig(proxyRuntime)
 	if err != nil {
-		logrus.WithError(err).Error(fmt.Sprintf("Error validating proxy config: %v", err))
-		return
+		return fmt.Errorf("validate proxy config: %w", err)
 	}
 
 	policy := resolveEngineProxyPolicy(proxyCfg, engineType)
 
 	selectedProxy, err := selectCLIProxy(proxyCfg, policy)
 	if err != nil {
-		logrus.WithError(err).Error(fmt.Sprintf("Error selecting proxy for %s: %v", engineType, err))
-		return
+		return fmt.Errorf("select proxy for %s: %w", engineType, err)
 	}
 
 	if config.Server.IsRawRequests {
 		query.ProxyURL = selectedProxy
 	}
 
+	// Bound the whole search so a wedged Chrome can't hang the CLI forever.
+	timeoutSec := searchOpts.timeout
+	if timeoutSec <= 0 {
+		timeoutSec = 60
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(timeoutSec)*time.Second)
+	defer cancel()
+
 	logrus.WithFields(logrus.Fields{
 		"engine":     engineType,
 		"query_hash": core.QueryHashFromQuery(query),
 	}).Info(fmt.Sprintf("Starting SERP search request using %s engine for query: %s", engineType, query.Text))
 
+	startedAt := time.Now()
 	var results []core.SearchResult
 	if config.Server.IsRawRequests {
 		logrus.WithField("engine", engineType).Info(fmt.Sprintf("Using raw requests mode for %s search", engineType))
-		results, err = searchRaw(engineType, query)
+		results, err = searchRaw(ctx, spec, query)
 	} else {
 		logrus.WithField("engine", engineType).Info(fmt.Sprintf("Using browser mode for %s search", engineType))
-		results, err = searchBrowser(engineType, query, selectedProxy, captchaSolverEnabled, captchaSolverAPIKey)
+		results, err = searchBrowser(ctx, spec, query, selectedProxy, captchaSolverEnabled, captchaSolverAPIKey)
 	}
 
 	if err != nil {
-		logrus.WithError(err).WithField("engine", engineType).Error(fmt.Sprintf("Error during %s search: %s", engineType, err))
-		return
+		return fmt.Errorf("%s search: %w", engineType, err)
 	}
 
 	logrus.WithFields(logrus.Fields{
@@ -89,17 +144,203 @@ func search(cmd *cobra.Command, args []string) {
 		"results_count": len(results),
 	}).Info(fmt.Sprintf("Successfully completed SERP search using %s engine, returned %d results", engineType, len(results)))
 
-	b, err := json.MarshalIndent(results, "", " ")
-	if err != nil {
-		logrus.Error(err)
-		return
+	env := buildCLIEnvelope(spec.name, query, results, startedAt)
+	if query.Extract {
+		if err := enrichCLIEnvelopeWithExtraction(ctx, env, query, format, selectedProxy, captchaSolverEnabled, captchaSolverAPIKey); err != nil {
+			return fmt.Errorf("extract search results: %w", err)
+		}
 	}
-
-	fmt.Println(string(b))
+	payload := renderCLIEnvelope(env, format, searchOpts.full)
+	fmt.Println(strings.TrimRight(string(payload), "\n"))
+	return nil
 }
 
-func searchBrowser(engineType string, query core.Query, browserProxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) ([]core.SearchResult, error) {
-	var engine core.SearchEngine
+func buildCLIEnvelope(engineName string, query core.Query, results []core.SearchResult, startedAt time.Time) *core.Envelope {
+	env := core.NewEnvelope(query, uuid.NewString(), startedAt, []string{engineName})
+	ectx := core.EnrichContext{Engine: engineName, Query: query}
+	for _, r := range results {
+		core.AppendEnrichedSearchResult(env, r, ectx, startedAt)
+	}
+	env.Finalize(startedAt, query)
+	return env
+}
+
+// renderCLIEnvelope renders a v2.1 envelope. JSON/ndjson always carry the full
+// envelope; text/markdown omit serp_features unless --full.
+func renderCLIEnvelope(env *core.Envelope, format string, full bool) []byte {
+	if !full && format != "json" && format != "ndjson" {
+		env.SerpFeatures = nil
+	}
+
+	switch format {
+	case "text":
+		return core.RenderText(env)
+	case "markdown":
+		return core.RenderMarkdown(env)
+	case "ndjson":
+		return core.RenderNDJSON(env)
+	default: // json
+		b, err := json.MarshalIndent(env, "", "  ")
+		if err != nil {
+			logrus.WithError(err).Error("marshal envelope")
+			return nil
+		}
+		return b
+	}
+}
+
+const maxCLIExtractTop = 5
+
+func applyCLIExtractFlag(query *core.Query, extractTop int) error {
+	top, err := normalizeCLIExtractTop(extractTop)
+	if err != nil {
+		return err
+	}
+	if top == 0 {
+		return nil
+	}
+	if !config.Extract.Enabled {
+		return fmt.Errorf("extraction is disabled in config")
+	}
+	query.Extract = true
+	query.ExtractTop = top
+	query.ExtractMode = string(extractpkg.ModeAuto)
+	return nil
+}
+
+func normalizeCLIExtractTop(raw int) (int, error) {
+	if raw < 0 {
+		return 0, fmt.Errorf("--extract must be a non-negative integer")
+	}
+	if raw > maxCLIExtractTop {
+		return maxCLIExtractTop, nil
+	}
+	return raw, nil
+}
+
+func enrichCLIEnvelopeWithExtraction(ctx context.Context, env *core.Envelope, query core.Query, format string, proxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) error {
+	if env == nil || !query.Extract {
+		return nil
+	}
+	query.ProxyURL = proxyURL
+	extractor, closeExtractor, err := newCLIExtractor(captchaSolverEnabled, captchaSolverAPIKey)
+	if err != nil {
+		return err
+	}
+	defer closeExtractor()
+
+	// Same depth bounds, batch deadline, and candidate fill-in as the HTTP server.
+	core.EnrichEnvelopeWithExtraction(ctx, env, query, format, extractor, config.Extract)
+	return nil
+}
+
+// newCLIExtractor builds an Extractor backed by a lazily-created, single-use
+// browser. The raw path delegates to core.RawExtractFetch; the rendered path
+// validates the target, gates auth'd SOCKS, then reuses core.RenderExtractHTML.
+func newCLIExtractor(captchaSolverEnabled bool, captchaSolverAPIKey string) (extractpkg.Extractor, func(), error) {
+	cfg := config.Extract.Normalized()
+	var browserMu sync.Mutex
+	var browser *core.Browser
+
+	closeExtractor := func() {
+		browserMu.Lock()
+		defer browserMu.Unlock()
+		if browser == nil {
+			return
+		}
+		if err := browser.Close(); err != nil {
+			logrus.WithError(err).Debug("Extraction browser close error")
+		}
+		browser = nil
+	}
+
+	extractor := extractpkg.Extractor{
+		Cfg: cfg,
+		RawFetch: func(ctx context.Context, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) {
+			return core.RawExtractFetch(ctx, req, cfg, config.Server.Insecure)
+		},
+		RenderedFetch: func(ctx context.Context, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) {
+			if err := validateCLIExtractTargetURL(ctx, req.URL, cfg.AllowPrivateNetworks); err != nil {
+				return nil, err
+			}
+			if core.IsAuthenticatedSocksProxyURL(req.ProxyURL) {
+				return nil, fmt.Errorf(
+					"%w: browser runtime does not support authenticated SOCKS proxy %s",
+					core.ErrProxyUnavailable,
+					core.MaskProxyURL(req.ProxyURL),
+				)
+			}
+
+			browserMu.Lock()
+			if browser == nil {
+				created, err := newCLIExtractBrowser(cfg, req.ProxyURL, captchaSolverEnabled, captchaSolverAPIKey)
+				if err != nil {
+					browserMu.Unlock()
+					return nil, err
+				}
+				browser = created
+			}
+			current := browser
+			browserMu.Unlock()
+
+			return core.RenderExtractHTML(ctx, current, req)
+		},
+	}
+	return extractor, closeExtractor, nil
+}
+
+func newCLIExtractBrowser(cfg extractpkg.Config, proxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) (*core.Browser, error) {
+	blockedResourceTypes, err := core.ParseBlockedResourceTypes(config.App.BlockResources)
+	if err != nil {
+		return nil, fmt.Errorf("invalid block_resources config: %w", err)
+	}
+	opts := core.BrowserOpts{
+		IsHeadless:           !config.App.IsBrowserHead && !config.Server.IsDebug,
+		IsLeakless:           config.App.IsLeakless,
+		Timeout:              cfg.Timeout,
+		LeavePageOpen:        false,
+		CaptchaSolverEnabled: captchaSolverEnabled,
+		CaptchaSolverApiKey:  captchaSolverAPIKey,
+		BrowserPath:          config.App.BrowserPath,
+		ProxyURL:             proxyURL,
+		Insecure:             config.Server.Insecure,
+		BlockResourceTypes:   blockedResourceTypes,
+		BlockTrackers:        config.App.BlockTrackers,
+	}
+	return core.NewBrowser(opts)
+}
+
+func validateCLIExtractTargetURL(ctx context.Context, rawURL string, allowPrivateNetworks bool) error {
+	targetURL := extractpkg.NormalizeURL(strings.TrimSpace(rawURL))
+	if allowPrivateNetworks {
+		parsed, err := url.ParseRequestURI(targetURL)
+		if err != nil {
+			return fmt.Errorf("invalid url: %w", err)
+		}
+		if parsed.Scheme != "http" && parsed.Scheme != "https" {
+			return fmt.Errorf("url must use http or https")
+		}
+		return nil
+	}
+	return core.ValidatePublicHTTPURL(ctx, targetURL)
+}
+
+func normalizeSearchFormat(raw string) (string, error) {
+	switch strings.ToLower(strings.TrimSpace(raw)) {
+	case "", "json":
+		return "json", nil
+	case "text", "txt":
+		return "text", nil
+	case "markdown", "md":
+		return "markdown", nil
+	case "ndjson", "jsonl":
+		return "ndjson", nil
+	default:
+		return "", fmt.Errorf("invalid --format %q; valid: json, text, markdown, ndjson", raw)
+	}
+}
+
+func searchBrowser(ctx context.Context, spec engineSpec, query core.Query, browserProxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) ([]core.SearchResult, error) {
 	blockedResourceTypes, err := core.ParseBlockedResourceTypes(config.App.BlockResources)
 	if err != nil {
 		return nil, fmt.Errorf("invalid block_resources config: %w", err)
@@ -134,49 +375,24 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string,
 	if err != nil {
 		return nil, err
 	}
+	// Close the browser so Chromium never outlives the CLI run.
+	defer func() {
+		if closeErr := browser.Close(); closeErr != nil {
+			logrus.WithError(closeErr).Debug("Browser close error")
+		}
+	}()
 
-	switch strings.ToLower(engineType) {
-	case "yandex":
-		engine = yandex.New(*browser, config.YandexConfig.SearchEngineOptions)
-	case "google":
-		engine = google.New(*browser, config.GoogleConfig.SearchEngineOptions)
-	case "baidu":
-		engine = baidu.New(*browser, config.BaiduConfig.SearchEngineOptions)
-	case "bing":
-		engine = bing.New(*browser, config.BingConfig.SearchEngineOptions)
-	case "duckduckgo":
-		engine = duckduckgo.New(*browser, config.DuckDuckGoConfig.SearchEngineOptions)
-	case "ecosia":
-		engine = ecosia.New(*browser, config.EcosiaConfig.SearchEngineOptions)
-	default:
-		return nil, fmt.Errorf("no %q search engine found", engineType)
-	}
-
-	return engine.Search(context.Background(), query)
+	engine := spec.factory(*browser, spec.opts())
+	return engine.Search(ctx, query)
 }
 
-func searchRaw(engineType string, query core.Query) ([]core.SearchResult, error) {
+func searchRaw(ctx context.Context, spec engineSpec, query core.Query) ([]core.SearchResult, error) {
 	logrus.Warn("Browserless results are very inconsistent or may not even work!")
-	ctx := context.Background()
-
-	switch strings.ToLower(engineType) {
-	case "yandex":
-		return yandex.Search(ctx, query)
-	case "google":
-		return google.Search(ctx, query)
-	case "baidu":
-		return baidu.Search(ctx, query)
-	case "ecosia":
-		return ecosia.Search(ctx, query)
-	case "bing":
-		logrus.Warn("Bing does not support raw HTTP requests mode. Please use browser mode instead.")
-		return nil, fmt.Errorf("bing does not support raw requests mode")
-	case "duckduckgo":
-		logrus.Warn("DuckDuckGo does not support raw HTTP requests mode. Please use browser mode instead.")
-		return nil, fmt.Errorf("duckduckgo does not support raw requests mode")
-	default:
-		return nil, fmt.Errorf("no %q search engine found", engineType)
+	if spec.rawSearchFn == nil {
+		logrus.Warnf("%s does not support raw HTTP requests mode. Please use browser mode instead.", spec.name)
+		return nil, fmt.Errorf("%s does not support raw requests mode", spec.name)
 	}
+	return spec.rawSearchFn(ctx, query)
 }
 
 func selectCLIProxy(proxyCfg core.ProxyConfig, policy core.ProxyPolicy) (string, error) {
@@ -201,14 +417,20 @@ func selectCLIProxy(proxyCfg core.ProxyConfig, policy core.ProxyPolicy) (string,
 }
 
 func normalizeEngineArg(raw string) string {
-	switch strings.ToLower(strings.TrimSpace(raw)) {
-	case "duck":
-		return "duckduckgo"
-	default:
-		return strings.ToLower(strings.TrimSpace(raw))
-	}
+	return strings.ToLower(strings.TrimSpace(raw))
 }
 
 func init() {
+	searchCMD.Flags().IntVar(&searchOpts.limit, "limit", 10, "Maximum number of results")
+	searchCMD.Flags().StringVar(&searchOpts.lang, "lang", "", "Language hint (e.g. EN, DE, RU)")
+	searchCMD.Flags().StringVar(&searchOpts.region, "region", "", "Region/market hint (e.g. RU, en-US)")
+	searchCMD.Flags().IntVar(&searchOpts.start, "start", 0, "Pagination start offset")
+	searchCMD.Flags().StringVar(&searchOpts.site, "site", "", "Restrict results to a domain (e.g. github.com)")
+	searchCMD.Flags().StringVar(&searchOpts.filetype, "file", "", "File type filter (e.g. pdf)")
+	searchCMD.Flags().StringVar(&searchOpts.format, "format", "json", "Output format: json, text, markdown, ndjson")
+	searchCMD.Flags().BoolVar(&searchOpts.full, "full", false, "Include SERP features in text/markdown output")
+	searchCMD.Flags().BoolVar(&searchOpts.features, "features", false, "Parse SERP feature modules (browser mode)")
+	searchCMD.Flags().IntVar(&searchOpts.extract, "extract", 0, "Extract clean content from the top N results using auto mode (1-5)")
+	searchCMD.Flags().IntVar(&searchOpts.timeout, "search-timeout", 60, "Overall search timeout in seconds")
 	RootCmd.AddCommand(searchCMD)
 }

cmd/search_test.go

@@ -0,0 +1,82 @@
+package cmd
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/karust/openserp/core"
+	extractpkg "github.com/karust/openserp/extract"
+)
+
+func TestNormalizeCLIExtractTop(t *testing.T) {
+	tests := []struct {
+		name    string
+		raw     int
+		want    int
+		wantErr bool
+	}{
+		{name: "disabled", raw: 0, want: 0},
+		{name: "one", raw: 1, want: 1},
+		{name: "clamped", raw: 20, want: maxCLIExtractTop},
+		{name: "negative", raw: -1, wantErr: true},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := normalizeCLIExtractTop(tc.raw)
+			if tc.wantErr {
+				if err == nil {
+					t.Fatal("expected error")
+				}
+				return
+			}
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+			if got != tc.want {
+				t.Fatalf("normalizeCLIExtractTop(%d) = %d, want %d", tc.raw, got, tc.want)
+			}
+		})
+	}
+}
+
+func TestApplyCLIExtractFlagSetsAutoMode(t *testing.T) {
+	previous := config
+	config.Extract = extractpkg.DefaultConfig()
+	defer func() { config = previous }()
+
+	query := core.Query{Text: "weather today"}
+	if err := applyCLIExtractFlag(&query, 2); err != nil {
+		t.Fatalf("applyCLIExtractFlag() error = %v", err)
+	}
+	if !query.Extract {
+		t.Fatal("expected query.Extract")
+	}
+	if query.ExtractTop != 2 {
+		t.Fatalf("ExtractTop = %d, want 2", query.ExtractTop)
+	}
+	if query.ExtractMode != string(extractpkg.ModeAuto) {
+		t.Fatalf("ExtractMode = %q, want auto", query.ExtractMode)
+	}
+}
+
+func TestApplyCLIExtractFlagRequiresEnabledConfig(t *testing.T) {
+	previous := config
+	config.Extract = extractpkg.Config{Enabled: false}
+	defer func() { config = previous }()
+
+	query := core.Query{Text: "weather today"}
+	err := applyCLIExtractFlag(&query, 1)
+	if err == nil {
+		t.Fatal("expected error")
+	}
+	if !strings.Contains(err.Error(), "disabled") {
+		t.Fatalf("error = %q, want disabled message", err.Error())
+	}
+}
+
+func TestSearchCommandHasExtractFlag(t *testing.T) {
+	if searchCMD.Flags().Lookup("extract") == nil {
+		t.Fatal("expected search command to expose --extract")
+	}
+}

cmd/serve.go

@@ -13,13 +13,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/karust/openserp/baidu"
-	"github.com/karust/openserp/bing"
 	"github.com/karust/openserp/core"
-	"github.com/karust/openserp/duckduckgo"
-	"github.com/karust/openserp/ecosia"
-	"github.com/karust/openserp/google"
-	"github.com/karust/openserp/yandex"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"golang.org/x/time/rate"
@@ -35,18 +29,11 @@ type rawEngine struct {
 func (r *rawEngine) Search(ctx context.Context, q core.Query) ([]core.SearchResult, error) {
 	q.Insecure = config.Server.Insecure
 
-	switch r.name {
-	case "google":
-		return google.Search(ctx, q)
-	case "yandex":
-		return yandex.Search(ctx, q)
-	case "baidu":
-		return baidu.Search(ctx, q)
-	case "ecosia":
-		return ecosia.Search(ctx, q)
-	default:
+	spec, ok := resolveEngineSpec(r.name)
+	if !ok || spec.rawSearchFn == nil {
 		return nil, fmt.Errorf("unsupported engine: %s", r.name)
 	}
+	return spec.rawSearchFn(ctx, q)
 }
 
 func (r *rawEngine) SearchImage(_ context.Context, _ core.Query) ([]core.SearchResult, error) {
@@ -626,56 +613,17 @@ type browserEngineSpec struct {
 }
 
 func browserEngineSpecs() []browserEngineSpec {
-	return []browserEngineSpec{
-		{
-			name: "google",
-			opts: config.GoogleConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return google.New(browser, opts)
-			},
-			parseHTMLFn: google.ParseHTML,
-		},
-		{
-			name: "yandex",
-			opts: config.YandexConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return yandex.New(browser, opts)
-			},
-			parseHTMLFn: yandex.ParseHTML,
-		},
-		{
-			name: "baidu",
-			opts: config.BaiduConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return baidu.New(browser, opts)
-			},
-			parseHTMLFn: baidu.ParseHTML,
-		},
-		{
-			name: "bing",
-			opts: config.BingConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return bing.New(browser, opts)
-			},
-			parseHTMLFn: bing.ParseHTML,
-		},
-		{
-			name: "duckduckgo",
-			opts: config.DuckDuckGoConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return duckduckgo.New(browser, opts)
-			},
-			parseHTMLFn: duckduckgo.ParseHTML,
-		},
-		{
-			name: "ecosia",
-			opts: config.EcosiaConfig.SearchEngineOptions,
-			factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine {
-				return ecosia.New(browser, opts)
-			},
-			parseHTMLFn: ecosia.ParseHTML,
-		},
+	specs := engineSpecs()
+	out := make([]browserEngineSpec, 0, len(specs))
+	for _, s := range specs {
+		out = append(out, browserEngineSpec{
+			name:        s.name,
+			opts:        s.opts(),
+			factory:     s.factory,
+			parseHTMLFn: s.parseHTMLFn,
+		})
 	}
+	return out
 }
 
 func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ([]core.SearchEngine, func() error, core.BrowserResolver, error) {

cmd/serve_test.go

@@ -19,6 +19,15 @@ func TestRawEngineCachesRateLimiter(t *testing.T) {
 	}
 }
 
+func TestCommandDefaultsToQuiet(t *testing.T) {
+	if !commandDefaultsToQuiet(searchCMD) {
+		t.Fatal("expected search command to default to quiet")
+	}
+	if commandDefaultsToQuiet(serveCMD) {
+		t.Fatal("expected serve command to keep server logging by default")
+	}
+}
+
 func TestBrowserPoolKey(t *testing.T) {
 	cases := []struct {
 		name string