zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-05 16:53:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implement proxy support with configuration, pooling, and integration tests

Browse Source 
- Added ProxyConfig and ProxyPool structures to manage proxy settings and rotation.
- Implemented normalization functions for proxy URLs and configurations.
- Created a new ProxyPool that supports failure tracking and round-robin selection.
- Integrated proxy handling into the ResilientSearcher for search queries.
- Added integration tests for various proxy scenarios including SOCKS5 and HTTP proxies.
- Updated server and resilience stats to include proxy information.
- Refactored search functions to utilize the new proxy client.
- Removed direct proxy handling from search_raw.go and yandex/search_raw.go, using the new core proxy client instead.
- Added unit tests for proxy configuration and pool behavior.
This commit is contained in:
Rustem Kamalov
2026-03-30 22:46:03 +03:00
parent 0146891076
commit 8bec5578c0
18 changed files with 1118 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
cmd/root.go
View File

@@ -13,13 +13,14 @@ import (
)
const (
version = "0.5.6"
version = "0.5.9"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)
type Config struct {
App AppConfig `mapstructure:"app"`
ProxyPool ProxyPoolConfig `mapstructure:"proxy_pool"`
Cache CacheConfig `mapstructure:"cache"`
Resilience ResilienceConfig `mapstructure:"resilience"`
CircuitBreaker CircuitBreakerConfig `mapstructure:"circuit_breaker"`
@@ -53,6 +54,11 @@ type AppConfig struct {
IsStealth bool `mapstructure:"stealth"`
}
type ProxyPoolConfig struct {
URLs []string `mapstructure:"urls"`
FailureThreshold int `mapstructure:"failure_threshold"`
}
type CacheConfig struct {
TTLSeconds int `mapstructure:"ttl_seconds"`
MaxSize int `mapstructure:"max_size"`
@@ -189,6 +195,19 @@ func initializeConfig(cmd *cobra.Command) error {
return fmt.Errorf("cannot unmarshall config: %v", err)
}
config.App.ProxyURL, err = core.NormalizeProxyURL(config.App.ProxyURL)
if err != nil {
return fmt.Errorf("invalid app.proxy: %w", err)
}
config.ProxyPool.URLs, err = core.NormalizeProxyURLs(config.ProxyPool.URLs)
if err != nil {
return fmt.Errorf("invalid proxy_pool.urls: %w", err)
}
if config.ProxyPool.FailureThreshold <= 0 {
config.ProxyPool.FailureThreshold = core.DefaultProxyPoolFailureThreshold
}
if config.App.IsDebug {
logrus.Debug("Viper config:")
v.Debug()
@@ -197,6 +216,8 @@ func initializeConfig(cmd *cobra.Command) error {
}
func setConfigDefaults(v *viper.Viper) {
v.SetDefault("proxy_pool.urls", []string{})
v.SetDefault("proxy_pool.failure_threshold", core.DefaultProxyPoolFailureThreshold)
v.SetDefault("cache.ttl_seconds", 300)
v.SetDefault("cache.max_size", 1000)
// Keep stage2 defaults stable even when config file is absent.
@@ -225,7 +246,7 @@ func init() {
RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "x", "", "HTTP or Socks5 proxy URL (e.g. http://user:pass@127.0.0.1:8080)")
RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "x", "", "HTTP/HTTPS/SOCKS5/SOCKS5H proxy URL (e.g. socks5h://127.0.0.1:1080)")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsStealth, "stealth", "s", false, "Use stealth browser plugin")
RootCmd.PersistentFlags().BoolVarP(&config.App.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
RootCmd.PersistentFlags().IntVar(&config.Cache.TTLSeconds, "cache_ttl", 300, "Cache TTL in seconds (0 to disable)")