mirror of
https://github.com/karust/openserp.git
synced 2026-08-05 16:53:54 +08:00
447 lines
18 KiB
Go
447 lines
18 KiB
Go
package cmd
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/karust/openserp/core"
|
|
browserprofile "github.com/karust/openserp/core/browser"
|
|
"github.com/sirupsen/logrus"
|
|
"github.com/spf13/cobra"
|
|
"github.com/spf13/pflag"
|
|
"github.com/spf13/viper"
|
|
)
|
|
|
|
const (
|
|
version = "0.7.16"
|
|
defaultConfigFilename = "config"
|
|
envPrefix = "OPENSERP"
|
|
)
|
|
|
|
type Config struct {
|
|
Server ServerConfig `mapstructure:"server"`
|
|
App AppConfig `mapstructure:"app"`
|
|
Proxies core.ProxiesConfig `mapstructure:"proxies"`
|
|
Cache CacheConfig `mapstructure:"cache"`
|
|
Resilience ResilienceConfig `mapstructure:"resilience"`
|
|
CircuitBreaker CircuitBreakerConfig `mapstructure:"circuit_breaker"`
|
|
CORS CORSConfig `mapstructure:"cors"`
|
|
Captcha CaptchaConfig `mapstructure:"captcha"`
|
|
Config2Capcha Config2Captcha `mapstructure:"2captcha"`
|
|
GoogleConfig EngineConfig `mapstructure:"google"`
|
|
YandexConfig EngineConfig `mapstructure:"yandex"`
|
|
BaiduConfig EngineConfig `mapstructure:"baidu"`
|
|
BingConfig EngineConfig `mapstructure:"bing"`
|
|
DuckDuckGoConfig EngineConfig `mapstructure:"duckduckgo"`
|
|
EcosiaConfig EngineConfig `mapstructure:"ecosia"`
|
|
}
|
|
|
|
type Config2Captcha struct {
|
|
ApiKey string `mapstructure:"apikey"`
|
|
}
|
|
|
|
type ServerConfig struct {
|
|
Host string `mapstructure:"host"`
|
|
Port int `mapstructure:"port"`
|
|
ConfigPath string `mapstructure:"config_path"`
|
|
IsDebug bool `mapstructure:"debug"`
|
|
IsVerbose bool `mapstructure:"verbose"`
|
|
IsRawRequests bool `mapstructure:"raw_requests"`
|
|
Insecure bool `mapstructure:"insecure"`
|
|
}
|
|
|
|
type AppConfig struct {
|
|
Timeout int `mapstructure:"timeout"`
|
|
BrowserPath string `mapstructure:"browser_path"`
|
|
ProfilesJSON string `mapstructure:"profiles"`
|
|
IsBrowserHead bool `mapstructure:"head"`
|
|
IsLeaveHead bool `mapstructure:"leave_head"`
|
|
IsLeakless bool `mapstructure:"leakless"`
|
|
BlockResources string `mapstructure:"block_resources"`
|
|
BlockTrackers bool `mapstructure:"block_trackers"`
|
|
DebugEndpoints bool `mapstructure:"debug_endpoints"`
|
|
LogFormat string `mapstructure:"log_format"`
|
|
MaxProcesses int `mapstructure:"max_processes"`
|
|
IdleTTL time.Duration `mapstructure:"idle_ttl"`
|
|
MegaTimeout time.Duration `mapstructure:"mega_timeout"`
|
|
}
|
|
|
|
type EngineConfig struct {
|
|
core.SearchEngineOptions `mapstructure:",squash"`
|
|
Proxy string `mapstructure:"proxy"`
|
|
}
|
|
|
|
type CacheConfig struct {
|
|
TTLSeconds int `mapstructure:"ttl_seconds"`
|
|
MaxSize int `mapstructure:"max_size"`
|
|
}
|
|
|
|
type ResilienceConfig struct {
|
|
MaxRetries int `mapstructure:"max_retries"`
|
|
AllowEndpointFallback bool `mapstructure:"allow_endpoint_fallback"`
|
|
}
|
|
|
|
type CircuitBreakerConfig struct {
|
|
Failures int `mapstructure:"failures"`
|
|
RecoverySeconds int `mapstructure:"recovery_seconds"`
|
|
Successes int `mapstructure:"successes"`
|
|
}
|
|
|
|
type CORSConfig struct {
|
|
Enabled bool `mapstructure:"enabled"`
|
|
AllowOrigins string `mapstructure:"allow_origins"`
|
|
AllowMethods string `mapstructure:"allow_methods"`
|
|
AllowHeaders string `mapstructure:"allow_headers"`
|
|
MaxAge int `mapstructure:"max_age"`
|
|
}
|
|
|
|
type CaptchaConfig struct {
|
|
SolverEnabled bool `mapstructure:"solver_enabled"`
|
|
}
|
|
|
|
var config = Config{}
|
|
|
|
var flagToConfigKey = map[string]string{
|
|
"host": "server.host",
|
|
"port": "server.port",
|
|
"timeout": "app.timeout",
|
|
"config": "server.config_path",
|
|
"browser-path": "app.browser_path",
|
|
"profiles-json": "app.profiles",
|
|
"verbose": "server.verbose",
|
|
"debug": "server.debug",
|
|
"head": "app.head",
|
|
"leakless": "app.leakless",
|
|
"raw": "server.raw_requests",
|
|
"leave": "app.leave_head",
|
|
"2captcha_key": "2captcha.apikey",
|
|
"proxy": "proxies.global",
|
|
"debug-endpoints": "app.debug_endpoints",
|
|
"insecure": "server.insecure",
|
|
"cache_ttl": "cache.ttl_seconds",
|
|
"cache_max_size": "cache.max_size",
|
|
"max_retries": "resilience.max_retries",
|
|
"allow_endpoint_fallback": "resilience.allow_endpoint_fallback",
|
|
"cb_failures": "circuit_breaker.failures",
|
|
"cb_recovery": "circuit_breaker.recovery_seconds",
|
|
"cb_successes": "circuit_breaker.successes",
|
|
"log_format": "app.log_format",
|
|
}
|
|
|
|
var RootCmd = &cobra.Command{
|
|
Use: "openserp",
|
|
Short: "Open SERP",
|
|
Long: `Get [Google, Yandex, Baidu] search engine results via API or CLI.`,
|
|
Version: version,
|
|
SilenceUsage: true,
|
|
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
|
|
err := initializeConfig(cmd)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := browserprofile.LoadProfilesFromJSON(config.App.ProfilesJSON); err != nil {
|
|
return fmt.Errorf("load app.profiles: %w", err)
|
|
}
|
|
|
|
logFormat, err := core.NormalizeLogFormat(config.App.LogFormat)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
config.App.LogFormat = logFormat
|
|
|
|
core.InitLogger(config.Server.IsVerbose, config.Server.IsDebug, config.App.LogFormat)
|
|
logrus.WithField("config", sanitizedConfigForLog(config)).Debug("Final config")
|
|
return nil
|
|
},
|
|
}
|
|
|
|
func sanitizedConfigForLog(cfg Config) map[string]interface{} {
|
|
return map[string]interface{}{
|
|
"server": cfg.Server,
|
|
"app": map[string]interface{}{
|
|
"timeout": cfg.App.Timeout,
|
|
"browser_path": cfg.App.BrowserPath != "",
|
|
"profiles": cfg.App.ProfilesJSON != "",
|
|
"head": cfg.App.IsBrowserHead,
|
|
"leave_head": cfg.App.IsLeaveHead,
|
|
"leakless": cfg.App.IsLeakless,
|
|
"block_resources": cfg.App.BlockResources,
|
|
"block_trackers": cfg.App.BlockTrackers,
|
|
"debug_endpoints": cfg.App.DebugEndpoints,
|
|
"log_format": cfg.App.LogFormat,
|
|
"max_processes": cfg.App.MaxProcesses,
|
|
"idle_ttl": cfg.App.IdleTTL.String(),
|
|
"mega_timeout": cfg.App.MegaTimeout.String(),
|
|
},
|
|
"proxies": map[string]interface{}{
|
|
"global": maskedProxyForLog(cfg.Proxies.Global),
|
|
"entries": len(cfg.Proxies.Entries),
|
|
"allow_request_proxy_url": cfg.Proxies.AllowRequestProxyURL,
|
|
"health": cfg.Proxies.Health,
|
|
"lanes": cfg.Proxies.Lanes,
|
|
},
|
|
"cache": cfg.Cache,
|
|
"resilience": cfg.Resilience,
|
|
"circuit_breaker": cfg.CircuitBreaker,
|
|
"cors": cfg.CORS,
|
|
"captcha": cfg.Captcha,
|
|
"2captcha": map[string]interface{}{
|
|
"apikey_configured": strings.TrimSpace(cfg.Config2Capcha.ApiKey) != "",
|
|
},
|
|
"google": cfg.GoogleConfig,
|
|
"yandex": cfg.YandexConfig,
|
|
"baidu": cfg.BaiduConfig,
|
|
"bing": cfg.BingConfig,
|
|
"duckduckgo": cfg.DuckDuckGoConfig,
|
|
"ecosia": cfg.EcosiaConfig,
|
|
}
|
|
}
|
|
|
|
func maskedProxyForLog(proxyURL string) string {
|
|
if strings.TrimSpace(proxyURL) == "" {
|
|
return ""
|
|
}
|
|
return core.MaskProxyURL(proxyURL)
|
|
}
|
|
|
|
// Bind each cobra flag to its associated viper configuration (config file and environment variable)
|
|
func bindFlags(cmd *cobra.Command, vpr *viper.Viper) {
|
|
cmd.Flags().VisitAll(func(flg *pflag.Flag) {
|
|
configName, ok := flagToConfigKey[flg.Name]
|
|
if !ok {
|
|
configName = "app." + flg.Name
|
|
}
|
|
|
|
if err := vpr.BindPFlag(configName, flg); err != nil {
|
|
logrus.WithError(err).Error(fmt.Sprintf("Unable to bind flag %s: %v", flg.Name, err))
|
|
}
|
|
|
|
if flg.Changed {
|
|
val, err := parseFlagValue(flg)
|
|
if err != nil {
|
|
logrus.WithError(err).Error(fmt.Sprintf("Unable to parse flag %s: %v", flg.Name, err))
|
|
return
|
|
}
|
|
vpr.Set(configName, val)
|
|
}
|
|
})
|
|
}
|
|
|
|
func parseFlagValue(flg *pflag.Flag) (interface{}, error) {
|
|
switch flg.Value.Type() {
|
|
case "string":
|
|
return flg.Value.String(), nil
|
|
case "bool":
|
|
return strconv.ParseBool(flg.Value.String())
|
|
case "int":
|
|
return strconv.Atoi(flg.Value.String())
|
|
default:
|
|
return flg.Value.String(), nil
|
|
}
|
|
}
|
|
|
|
// Initialize Viper
|
|
func initializeConfig(cmd *cobra.Command) error {
|
|
v := viper.New()
|
|
setConfigDefaults(v)
|
|
|
|
explicitConfigPath := strings.TrimSpace(cmd.Flag("config").Value.String())
|
|
if explicitConfigPath == "" {
|
|
explicitConfigPath = strings.TrimSpace(os.Getenv(envPrefix + "_SERVER_CONFIG_PATH"))
|
|
}
|
|
|
|
if explicitConfigPath != "" {
|
|
v.SetConfigFile(explicitConfigPath)
|
|
} else {
|
|
// Base name of the config file, without the file extension
|
|
v.SetConfigName(defaultConfigFilename)
|
|
v.AddConfigPath(".")
|
|
}
|
|
|
|
// 1. Config file (lowest priority). Return an error if we cannot parse the config file.
|
|
err := v.ReadInConfig()
|
|
if err != nil {
|
|
if explicitConfigPath != "" {
|
|
return fmt.Errorf("cannot read config %q: %w", explicitConfigPath, err)
|
|
}
|
|
err = fmt.Errorf("cannot read config: %v", err)
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
// 2. Environment variables (medium priority). Bind environment variables to their equivalent keys with underscores
|
|
for _, key := range v.AllKeys() {
|
|
envKey := envPrefix + "_" + strings.ToUpper(strings.ReplaceAll(key, ".", "_"))
|
|
err := v.BindEnv(key, envKey)
|
|
if err != nil {
|
|
logrus.WithError(err).Error(fmt.Sprintf("Unable to bind ENV valye: %v", err))
|
|
}
|
|
}
|
|
|
|
// 3. Command flags (highest priority). Bind the current command's flags to viper
|
|
bindFlags(cmd, v)
|
|
|
|
// Keep compatibility with historical typo in local configs. Runs after all
|
|
// sources are merged so CLI flags and env vars take precedence over the typo key.
|
|
if v.IsSet("app.block_resorces") && !v.IsSet("app.block_resources") {
|
|
v.Set("app.block_resources", v.Get("app.block_resorces"))
|
|
logrus.Warn(`config key "app.block_resorces" is deprecated, use "app.block_resources"`)
|
|
}
|
|
|
|
if err := validateRemovedConfigPaths(v); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Dump Viper values to config struct
|
|
if err := validateEngineProxyTags(v); err != nil {
|
|
return err
|
|
}
|
|
|
|
err = v.Unmarshal(&config)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot unmarshall config: %v", err)
|
|
}
|
|
|
|
if _, err := core.ParseBlockedResourceTypes(config.App.BlockResources); err != nil {
|
|
return fmt.Errorf("invalid app.block_resources: %w", err)
|
|
}
|
|
|
|
config.Proxies, err = core.NormalizeProxiesConfig(config.Proxies)
|
|
if err != nil {
|
|
return fmt.Errorf("invalid proxies config: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateEngineProxyTags(v *viper.Viper) error {
|
|
for _, engineName := range []string{"google", "yandex", "baidu", "bing", "duckduckgo", "ecosia"} {
|
|
key := engineName + ".proxy"
|
|
if !v.IsSet(key) {
|
|
continue
|
|
}
|
|
|
|
raw := v.Get(key)
|
|
tag, ok := raw.(string)
|
|
if !ok {
|
|
return fmt.Errorf("invalid %s.proxy config: proxy must be a string tag", engineName)
|
|
}
|
|
|
|
if _, err := core.NormalizeProxyTag(tag); err != nil {
|
|
return fmt.Errorf("invalid %s.proxy config: %w", engineName, err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func validateRemovedConfigPaths(v *viper.Viper) error {
|
|
legacyKeys := map[string]string{
|
|
"app.proxy": "use proxies.global or proxies.entries with per-engine proxy tags instead",
|
|
"proxy_pool": "use proxies.entries and proxies.health.failure_threshold instead",
|
|
"proxy_pool.urls": "use proxies.entries instead",
|
|
"proxy_pool.failure_threshold": "use proxies.health.failure_threshold instead",
|
|
"app.host": "move to server.host",
|
|
"app.port": "move to server.port",
|
|
"app.debug": "move to server.debug",
|
|
"app.verbose": "move to server.verbose",
|
|
"app.raw_requests": "move to server.raw_requests",
|
|
"app.insecure": "move to server.insecure",
|
|
"proxies.defaults": "use proxies.global or per-engine proxy tags instead",
|
|
"proxies.defaults.mode": "use proxies.global or per-engine proxy tags instead",
|
|
"proxies.defaults.tag": "use per-engine proxy tags on each engine instead",
|
|
"google.proxy.mode": "use google.proxy: <tag> or omit it for direct mode",
|
|
"google.proxy.tag": "use google.proxy: <tag>",
|
|
"yandex.proxy.mode": "use yandex.proxy: <tag> or omit it for direct mode",
|
|
"yandex.proxy.tag": "use yandex.proxy: <tag>",
|
|
"baidu.proxy.mode": "use baidu.proxy: <tag> or omit it for direct mode",
|
|
"baidu.proxy.tag": "use baidu.proxy: <tag>",
|
|
"bing.proxy.mode": "use bing.proxy: <tag> or omit it for direct mode",
|
|
"bing.proxy.tag": "use bing.proxy: <tag>",
|
|
"duckduckgo.proxy.mode": "use duckduckgo.proxy: <tag> or omit it for direct mode",
|
|
"duckduckgo.proxy.tag": "use duckduckgo.proxy: <tag>",
|
|
}
|
|
|
|
for key, hint := range legacyKeys {
|
|
if v.IsSet(key) {
|
|
return fmt.Errorf("config key %q is removed in proxy v2: %s", key, hint)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func setConfigDefaults(v *viper.Viper) {
|
|
v.SetDefault("server.host", "127.0.0.1")
|
|
v.SetDefault("server.port", 7070)
|
|
v.SetDefault("server.debug", false)
|
|
v.SetDefault("server.verbose", false)
|
|
v.SetDefault("server.raw_requests", false)
|
|
v.SetDefault("server.insecure", false)
|
|
v.SetDefault("app.log_format", "")
|
|
|
|
v.SetDefault("app.timeout", 30)
|
|
v.SetDefault("app.browser_path", "")
|
|
v.SetDefault("app.profiles", "")
|
|
v.SetDefault("app.head", false)
|
|
v.SetDefault("app.leave_head", false)
|
|
v.SetDefault("app.leakless", false)
|
|
v.SetDefault("app.block_resources", "")
|
|
v.SetDefault("app.block_trackers", false)
|
|
v.SetDefault("app.debug_endpoints", false)
|
|
v.SetDefault("app.max_processes", 4)
|
|
v.SetDefault("app.idle_ttl", "10m")
|
|
v.SetDefault("app.mega_timeout", "90s")
|
|
|
|
v.SetDefault("proxies.entries", []interface{}{})
|
|
v.SetDefault("proxies.global", "")
|
|
v.SetDefault("proxies.allow_request_proxy_url", false)
|
|
v.SetDefault("proxies.health.failure_threshold", core.DefaultProxyFailureThreshold)
|
|
v.SetDefault("proxies.lanes.enabled", true)
|
|
v.SetDefault("proxies.lanes.max_lanes", core.DefaultProxyLaneMaxLanes)
|
|
v.SetDefault("proxies.lanes.drop_cookies_on_challenge", true)
|
|
|
|
v.SetDefault("cache.ttl_seconds", 300)
|
|
v.SetDefault("cache.max_size", 1000)
|
|
// Keep stage2 defaults stable even when config file is absent.
|
|
v.SetDefault("resilience.max_retries", 3)
|
|
v.SetDefault("resilience.allow_endpoint_fallback", false)
|
|
v.SetDefault("circuit_breaker.failures", 5)
|
|
v.SetDefault("circuit_breaker.recovery_seconds", 60)
|
|
v.SetDefault("circuit_breaker.successes", 2)
|
|
v.SetDefault("cors.enabled", true)
|
|
v.SetDefault("cors.allow_origins", "*")
|
|
v.SetDefault("cors.allow_methods", "GET, POST, OPTIONS")
|
|
v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization, X-Use-Proxy, X-Proxy-URL, X-Proxy-Country, X-Proxy-Class, X-Proxy-Provider, X-Proxy-Session-ID, X-Request-ID, X-Tenant")
|
|
v.SetDefault("cors.max_age", 86400)
|
|
v.SetDefault("captcha.solver_enabled", false)
|
|
}
|
|
|
|
func init() {
|
|
RootCmd.PersistentFlags().IntVarP(&config.Server.Port, "port", "p", 7070, "Port number to run server")
|
|
RootCmd.PersistentFlags().StringVarP(&config.Server.Host, "host", "a", "127.0.0.1", "Host address to run server")
|
|
RootCmd.PersistentFlags().IntVarP(&config.App.Timeout, "timeout", "t", 30, "Timeout to fail request")
|
|
RootCmd.PersistentFlags().StringVarP(&config.Server.ConfigPath, "config", "c", "", "Configuration file path")
|
|
RootCmd.PersistentFlags().StringVarP(&config.App.BrowserPath, "browser-path", "", "", "Custom browser binary path (Chrome/Chromium/Edge/Brave..)")
|
|
RootCmd.PersistentFlags().StringVar(&config.App.ProfilesJSON, "profiles", "", "Path to browser profile catalog JSON")
|
|
RootCmd.PersistentFlags().BoolVarP(&config.Server.IsVerbose, "verbose", "v", false, "Use verbose output")
|
|
RootCmd.PersistentFlags().BoolVarP(&config.Server.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
|
|
RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI")
|
|
RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeakless, "leakless", "l", false, "Use leakless mode to insure browser instances are closed after search")
|
|
RootCmd.PersistentFlags().BoolVarP(&config.Server.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
|
|
RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
|
|
RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
|
|
RootCmd.PersistentFlags().StringVarP(&config.Proxies.Global, "proxy", "x", "", "Force a single proxy for all engines (same as proxies.global)")
|
|
RootCmd.PersistentFlags().BoolVar(&config.App.DebugEndpoints, "debug-endpoints", false, "Enable debug-only HTTP endpoints")
|
|
RootCmd.PersistentFlags().BoolVarP(&config.Server.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
|
|
RootCmd.PersistentFlags().IntVar(&config.Cache.TTLSeconds, "cache_ttl", 300, "Cache TTL in seconds (0 to disable)")
|
|
RootCmd.PersistentFlags().IntVar(&config.Cache.MaxSize, "cache_max_size", 1000, "Maximum number of cached responses")
|
|
RootCmd.PersistentFlags().IntVar(&config.Resilience.MaxRetries, "max_retries", 3, "Max retry attempts per search engine (0 to disable)")
|
|
RootCmd.PersistentFlags().BoolVar(&config.Resilience.AllowEndpointFallback, "allow_endpoint_fallback", false, "Allow dedicated endpoints to fallback to other engines")
|
|
RootCmd.PersistentFlags().IntVar(&config.CircuitBreaker.Failures, "cb_failures", 5, "Consecutive failures before circuit breaker opens")
|
|
RootCmd.PersistentFlags().IntVar(&config.CircuitBreaker.RecoverySeconds, "cb_recovery", 60, "Seconds before retrying an engine with open circuit")
|
|
RootCmd.PersistentFlags().IntVar(&config.CircuitBreaker.Successes, "cb_successes", 2, "Consecutive successful half-open checks needed to close circuit")
|
|
RootCmd.PersistentFlags().StringVar(&config.App.LogFormat, "log_format", "", "Log format: json or text (default: json in production, text in debug)")
|
|
}
|