openserp/cmd/search.go

package cmd

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/karust/openserp/baidu"
	"github.com/karust/openserp/bing"
	"github.com/karust/openserp/core"
	"github.com/karust/openserp/duckduckgo"
	"github.com/karust/openserp/ecosia"
	"github.com/karust/openserp/google"
	"github.com/karust/openserp/yandex"
	"github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
)

var searchCMD = &cobra.Command{
	Use:     "search",
	Aliases: []string{"find"},
	Short:   "Search results using chosen web search engine (google, yandex, baidu, bing, duckduckgo, ecosia)",
	Args:    cobra.MatchAll(cobra.OnlyValidArgs, cobra.ExactArgs(2)),
	Run:     search,
}

func search(cmd *cobra.Command, args []string) {
	engineType := normalizeEngineArg(args[0])
	query := core.Query{
		Text:     args[1],
		Limit:    10,
		Filter:   true,
		Insecure: config.Server.Insecure,
	}

	captchaSolverEnabled, captchaSolverAPIKey, err := resolveCaptchaSolverConfig()
	if err != nil {
		logrus.WithError(err).Error(fmt.Sprintf("Error validating captcha solver config: %v", err))
		os.Exit(1)
	}

	proxyRuntime := core.ProxyRuntimeBrowser
	if config.Server.IsRawRequests {
		proxyRuntime = core.ProxyRuntimeRaw
	}

	proxyCfg, err := buildNormalizedProxyConfig(proxyRuntime)
	if err != nil {
		logrus.WithError(err).Error(fmt.Sprintf("Error validating proxy config: %v", err))
		return
	}

	policy := resolveEngineProxyPolicy(proxyCfg, engineType)

	selectedProxy, err := selectCLIProxy(proxyCfg, policy)
	if err != nil {
		logrus.WithError(err).Error(fmt.Sprintf("Error selecting proxy for %s: %v", engineType, err))
		return
	}

	if config.Server.IsRawRequests {
		query.ProxyURL = selectedProxy
	}

	logrus.WithFields(logrus.Fields{
		"engine":     engineType,
		"query_hash": core.QueryHashFromQuery(query),
	}).Info(fmt.Sprintf("Starting SERP search request using %s engine for query: %s", engineType, query.Text))

	var results []core.SearchResult
	if config.Server.IsRawRequests {
		logrus.WithField("engine", engineType).Info(fmt.Sprintf("Using raw requests mode for %s search", engineType))
		results, err = searchRaw(engineType, query)
	} else {
		logrus.WithField("engine", engineType).Info(fmt.Sprintf("Using browser mode for %s search", engineType))
		results, err = searchBrowser(engineType, query, selectedProxy, captchaSolverEnabled, captchaSolverAPIKey)
	}

	if err != nil {
		logrus.WithError(err).WithField("engine", engineType).Error(fmt.Sprintf("Error during %s search: %s", engineType, err))
		return
	}

	logrus.WithFields(logrus.Fields{
		"engine":        engineType,
		"results_count": len(results),
	}).Info(fmt.Sprintf("Successfully completed SERP search using %s engine, returned %d results", engineType, len(results)))

	b, err := json.MarshalIndent(results, "", " ")
	if err != nil {
		logrus.Error(err)
		return
	}

	fmt.Println(string(b))
}

func searchBrowser(engineType string, query core.Query, browserProxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) ([]core.SearchResult, error) {
	var engine core.SearchEngine
	blockedResourceTypes := core.MustParseBlockedResourceTypes(config.App.BlockResources)
	if core.IsAuthenticatedSocksProxyURL(browserProxyURL) {
		return nil, fmt.Errorf(
			"%w: browser runtime does not support authenticated SOCKS proxy %s",
			core.ErrProxyUnavailable,
			core.MaskProxyURL(browserProxyURL),
		)
	}

	opts := core.BrowserOpts{
		IsHeadless:           !config.App.IsBrowserHead,
		IsLeakless:           config.App.IsLeakless,
		Timeout:              time.Second * time.Duration(config.App.Timeout),
		LeavePageOpen:        config.App.IsLeaveHead,
		CaptchaSolverEnabled: captchaSolverEnabled,
		CaptchaSolverApiKey:  captchaSolverAPIKey,
		BrowserPath:          config.App.BrowserPath,
		ProxyURL:             browserProxyURL,
		Insecure:             config.Server.Insecure,
		BlockResourceTypes:   blockedResourceTypes,
		BlockTrackers:        config.App.BlockTrackers,
	}

	if config.Server.IsDebug {
		opts.IsHeadless = false
	}

	browser, err := core.NewBrowser(opts)
	if err != nil {
		return nil, err
	}

	switch strings.ToLower(engineType) {
	case "yandex":
		engine = yandex.New(*browser, config.YandexConfig.SearchEngineOptions)
	case "google":
		engine = google.New(*browser, config.GoogleConfig.SearchEngineOptions)
	case "baidu":
		engine = baidu.New(*browser, config.BaiduConfig.SearchEngineOptions)
	case "bing":
		engine = bing.New(*browser, config.BingConfig.SearchEngineOptions)
	case "duckduckgo":
		engine = duckduckgo.New(*browser, config.DuckDuckGoConfig.SearchEngineOptions)
	case "ecosia":
		engine = ecosia.New(*browser, config.EcosiaConfig.SearchEngineOptions)
	default:
		return nil, fmt.Errorf("no %q search engine found", engineType)
	}

	return engine.Search(context.Background(), query)
}

func searchRaw(engineType string, query core.Query) ([]core.SearchResult, error) {
	logrus.Warn("Browserless results are very inconsistent or may not even work!")
	ctx := context.Background()

	switch strings.ToLower(engineType) {
	case "yandex":
		return yandex.Search(ctx, query)
	case "google":
		return google.Search(ctx, query)
	case "baidu":
		return baidu.Search(ctx, query)
	case "ecosia":
		return ecosia.Search(ctx, query)
	case "bing":
		logrus.Warn("Bing does not support raw HTTP requests mode. Please use browser mode instead.")
		return nil, fmt.Errorf("bing does not support raw requests mode")
	case "duckduckgo":
		logrus.Warn("DuckDuckGo does not support raw HTTP requests mode. Please use browser mode instead.")
		return nil, fmt.Errorf("duckduckgo does not support raw requests mode")
	default:
		return nil, fmt.Errorf("no %q search engine found", engineType)
	}
}

func selectCLIProxy(proxyCfg core.ProxyConfig, policy core.ProxyPolicy) (string, error) {
	if policy.Mode == core.ProxyModeOff {
		return "", nil
	}

	if global := strings.TrimSpace(proxyCfg.Proxies.Global); global != "" {
		return global, nil
	}

	if proxyCfg.Registry == nil {
		return "", fmt.Errorf("%w: no proxy registry configured", core.ErrProxyUnavailable)
	}

	selected := proxyCfg.Registry.NextByTag(policy.Tag)
	if selected == "" {
		return "", fmt.Errorf("%w: no healthy proxy available for tag %q", core.ErrProxyUnavailable, policy.Tag)
	}

	return selected, nil
}

func normalizeEngineArg(raw string) string {
	switch strings.ToLower(strings.TrimSpace(raw)) {
	case "duck":
		return "duckduckgo"
	default:
		return strings.ToLower(strings.TrimSpace(raw))
	}
}

func init() {
	RootCmd.AddCommand(searchCMD)
}