diff --git a/.gitignore b/.gitignore index bc833d7..87393cb 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ core/test/ .aider* .gocache/ openserp +.gomodcache/ diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 421768f..0a81543 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -1,65 +1,20 @@ package baidu import ( - "context" - "crypto/tls" "fmt" - "net" "net/http" - "net/url" "strings" - "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" "github.com/karust/openserp/core" "github.com/sirupsen/logrus" - - utls "github.com/refraction-networking/utls" ) func baiduRequest(searchURL string, query core.Query) (*http.Response, error) { - // Create HTTP transport with proxy - transport := &http.Transport{} - if query.ProxyURL != "" { - proxyUrl, err := url.Parse(query.ProxyURL) - if err != nil { - return nil, err - } - transport.Proxy = http.ProxyURL(proxyUrl) - } - - // Set insecure TLS - if query.Insecure { - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - } - - transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - dialer := &net.Dialer{} - rawConn, err := dialer.DialContext(ctx, network, addr) - if err != nil { - return nil, err - } - - hostname := strings.Split(addr, ":")[0] - config := &utls.Config{ - ServerName: hostname, - InsecureSkipVerify: query.Insecure, - } - - uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto) - - if err := uconn.Handshake(); err != nil { - rawConn.Close() - return nil, err - } - - return uconn, nil - } - - baseClient := &http.Client{ - Transport: transport, - Timeout: time.Second * 10, + baseClient, err := core.NewRawHTTPClient(query) + if err != nil { + return nil, err } req, err := http.NewRequest("GET", searchURL, nil) diff --git a/cmd/root.go b/cmd/root.go index 4ffeb49..a0afc98 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,13 +13,14 @@ import ( ) const ( - version = "0.5.6" + version = "0.5.9" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) type Config struct { App AppConfig `mapstructure:"app"` + ProxyPool ProxyPoolConfig `mapstructure:"proxy_pool"` Cache CacheConfig `mapstructure:"cache"` Resilience ResilienceConfig `mapstructure:"resilience"` CircuitBreaker CircuitBreakerConfig `mapstructure:"circuit_breaker"` @@ -53,6 +54,11 @@ type AppConfig struct { IsStealth bool `mapstructure:"stealth"` } +type ProxyPoolConfig struct { + URLs []string `mapstructure:"urls"` + FailureThreshold int `mapstructure:"failure_threshold"` +} + type CacheConfig struct { TTLSeconds int `mapstructure:"ttl_seconds"` MaxSize int `mapstructure:"max_size"` @@ -189,6 +195,19 @@ func initializeConfig(cmd *cobra.Command) error { return fmt.Errorf("cannot unmarshall config: %v", err) } + config.App.ProxyURL, err = core.NormalizeProxyURL(config.App.ProxyURL) + if err != nil { + return fmt.Errorf("invalid app.proxy: %w", err) + } + + config.ProxyPool.URLs, err = core.NormalizeProxyURLs(config.ProxyPool.URLs) + if err != nil { + return fmt.Errorf("invalid proxy_pool.urls: %w", err) + } + if config.ProxyPool.FailureThreshold <= 0 { + config.ProxyPool.FailureThreshold = core.DefaultProxyPoolFailureThreshold + } + if config.App.IsDebug { logrus.Debug("Viper config:") v.Debug() @@ -197,6 +216,8 @@ func initializeConfig(cmd *cobra.Command) error { } func setConfigDefaults(v *viper.Viper) { + v.SetDefault("proxy_pool.urls", []string{}) + v.SetDefault("proxy_pool.failure_threshold", core.DefaultProxyPoolFailureThreshold) v.SetDefault("cache.ttl_seconds", 300) v.SetDefault("cache.max_size", 1000) // Keep stage2 defaults stable even when config file is absent. @@ -225,7 +246,7 @@ func init() { RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests") RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made") RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key") - RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "x", "", "HTTP or Socks5 proxy URL (e.g. http://user:pass@127.0.0.1:8080)") + RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "x", "", "HTTP/HTTPS/SOCKS5/SOCKS5H proxy URL (e.g. socks5h://127.0.0.1:1080)") RootCmd.PersistentFlags().BoolVarP(&config.App.IsStealth, "stealth", "s", false, "Use stealth browser plugin") RootCmd.PersistentFlags().BoolVarP(&config.App.Insecure, "insecure", "k", false, "Allow insecure TLS connections") RootCmd.PersistentFlags().IntVar(&config.Cache.TTLSeconds, "cache_ttl", 300, "Cache TTL in seconds (0 to disable)") diff --git a/cmd/serve.go b/cmd/serve.go index da0071f..54e0079 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -21,8 +21,6 @@ type rawEngine struct { } func (r *rawEngine) Search(q core.Query) ([]core.SearchResult, error) { - // Inject proxy settings from config - q.ProxyURL = config.App.ProxyURL q.Insecure = config.App.Insecure switch r.name { @@ -69,6 +67,16 @@ func serve(cmd *cobra.Command, args []string) { corsCfg.AllowHeaders = config.CORS.AllowHeaders corsCfg.MaxAge = config.CORS.MaxAge + proxyCfg := core.ProxyConfig{ + Runtime: core.ProxyRuntimeBrowser, + StaticURL: config.App.ProxyURL, + PoolURLs: config.ProxyPool.URLs, + PoolFailureThreshold: config.ProxyPool.FailureThreshold, + } + if config.App.IsRawRequests { + proxyCfg.Runtime = core.ProxyRuntimeRaw + } + serverOpts := core.ServerOptions{ CacheTTL: time.Duration(config.Cache.TTLSeconds) * time.Second, CacheMaxSize: config.Cache.MaxSize, @@ -87,6 +95,7 @@ func serve(cmd *cobra.Command, args []string) { RecoveryTimeout: time.Duration(config.CircuitBreaker.RecoverySeconds) * time.Second, SuccessThreshold: config.CircuitBreaker.Successes, }, + Proxy: proxyCfg, }, } diff --git a/config.yaml b/config.yaml index 852b965..9efc33e 100644 --- a/config.yaml +++ b/config.yaml @@ -15,6 +15,13 @@ app: # Custom browser binary path (chrome/chromium/edge..) #browser_path: "C:/Program Files/BraveSoftware/Brave-Browser/Application/brave.exe" +proxy_pool: + # List of proxy URLs for raw-mode resilient rotation. + # Browser mode still uses only app.proxy in stage 4. + #urls: + # - "socks5h://127.0.0.1:1080" + failure_threshold: 3 # Disable a proxy after this many consecutive request failures + cache: ttl_seconds: 60 # Dedicated endpoint cache TTL in seconds (0 disables cache) max_size: 1000 # Maximum cached dedicated responses before oldest-entry eviction diff --git a/core/browser.go b/core/browser.go index 2aab456..9f05182 100644 --- a/core/browser.go +++ b/core/browser.go @@ -68,6 +68,12 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) { // Configure proxy if specified if opts.ProxyURL != "" { + normalizedProxyURL, err := NormalizeProxyURL(opts.ProxyURL) + if err != nil { + return nil, fmt.Errorf("invalid proxy URL: %v", err) + } + opts.ProxyURL = normalizedProxyURL + proxyUrl, err := url.Parse(opts.ProxyURL) if err != nil { return nil, fmt.Errorf("invalid proxy URL: %v", err) diff --git a/core/http_client.go b/core/http_client.go new file mode 100644 index 0000000..1c16a4d --- /dev/null +++ b/core/http_client.go @@ -0,0 +1,74 @@ +package core + +import ( + "context" + "crypto/tls" + "net" + "net/http" + "net/url" + "strings" + "time" + + utls "github.com/refraction-networking/utls" +) + +const rawHTTPTimeout = 10 * time.Second + +func NewRawHTTPClient(query Query) (*http.Client, error) { + transport, err := newRawTransport(query) + if err != nil { + return nil, err + } + + return &http.Client{ + Transport: transport, + Timeout: rawHTTPTimeout, + }, nil +} + +func newRawTransport(query Query) (*http.Transport, error) { + transport := &http.Transport{} + if query.Insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + proxyURL, err := NormalizeProxyURL(query.ProxyURL) + if err != nil { + return nil, err + } + if proxyURL != "" { + parsed, err := url.Parse(proxyURL) + if err != nil { + return nil, err + } + + // Keep proxied requests on the standard transport path so SOCKS5/SOCKS5H + // resolution and routing are handled by the configured proxy correctly. + transport.Proxy = http.ProxyURL(parsed) + return transport, nil + } + + transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + dialer := &net.Dialer{} + rawConn, err := dialer.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + + hostname := strings.Split(addr, ":")[0] + config := &utls.Config{ + ServerName: hostname, + InsecureSkipVerify: query.Insecure, + } + + uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto) + if err := uconn.Handshake(); err != nil { + rawConn.Close() + return nil, err + } + + return uconn, nil + } + + return transport, nil +} diff --git a/core/proxy.go b/core/proxy.go new file mode 100644 index 0000000..01e3e20 --- /dev/null +++ b/core/proxy.go @@ -0,0 +1,276 @@ +package core + +import ( + "fmt" + "net/url" + "strings" + "sync" + + "github.com/sirupsen/logrus" +) + +const ( + ProxyRuntimeBrowser = "browser" + ProxyRuntimeRaw = "raw" + ProxyModeDisabled = "disabled" + ProxyModeStatic = "static" + ProxyModePool = "pool" + DefaultProxyPoolFailureThreshold = 3 +) + +var supportedProxySchemes = map[string]struct{}{ + "http": {}, + "https": {}, + "socks5": {}, + "socks5h": {}, +} + +type ProxyConfig struct { + Runtime string + StaticURL string + PoolURLs []string + PoolFailureThreshold int +} + +type ProxyPool struct { + mu sync.Mutex + proxies []ProxyEntry + next int + failureThreshold int +} + +type ProxyEntry struct { + URL string + FailCount int + IsDisabled bool +} + +type ProxyPoolStats struct { + FailureThreshold int + Total int + Active int + Disabled int +} + +func DefaultProxyConfig() ProxyConfig { + return ProxyConfig{ + Runtime: ProxyRuntimeBrowser, + PoolFailureThreshold: DefaultProxyPoolFailureThreshold, + } +} + +func NormalizeProxyConfig(cfg ProxyConfig) (ProxyConfig, error) { + cfg.Runtime = normalizeProxyRuntime(cfg.Runtime) + if cfg.PoolFailureThreshold <= 0 { + cfg.PoolFailureThreshold = DefaultProxyPoolFailureThreshold + } + + staticURL, err := NormalizeProxyURL(cfg.StaticURL) + if err != nil { + return cfg, fmt.Errorf("invalid static proxy: %w", err) + } + poolURLs, err := NormalizeProxyURLs(cfg.PoolURLs) + if err != nil { + return cfg, fmt.Errorf("invalid proxy pool: %w", err) + } + + cfg.StaticURL = staticURL + cfg.PoolURLs = poolURLs + return cfg, nil +} + +func NormalizeProxyURL(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", nil + } + + parsed, err := url.Parse(raw) + if err != nil { + return "", err + } + if parsed.Scheme == "" { + return "", fmt.Errorf("proxy URL must include a scheme") + } + if parsed.Host == "" { + return "", fmt.Errorf("proxy URL must include a host") + } + + parsed.Scheme = strings.ToLower(parsed.Scheme) + if _, ok := supportedProxySchemes[parsed.Scheme]; !ok { + return "", fmt.Errorf("unsupported proxy scheme %q", parsed.Scheme) + } + + return parsed.String(), nil +} + +func NormalizeProxyURLs(rawURLs []string) ([]string, error) { + normalized := make([]string, 0, len(rawURLs)) + seen := make(map[string]struct{}, len(rawURLs)) + + for _, raw := range rawURLs { + proxyURL, err := NormalizeProxyURL(raw) + if err != nil { + return nil, err + } + if proxyURL == "" { + continue + } + if _, ok := seen[proxyURL]; ok { + continue + } + seen[proxyURL] = struct{}{} + normalized = append(normalized, proxyURL) + } + + return normalized, nil +} + +func MaskProxyURL(raw string) string { + proxyURL, err := NormalizeProxyURL(raw) + if err != nil || proxyURL == "" { + return "invalid-proxy" + } + + parsed, err := url.Parse(proxyURL) + if err != nil { + return "invalid-proxy" + } + + return fmt.Sprintf("%s://%s", parsed.Scheme, parsed.Host) +} + +func NewProxyPool(proxyURLs []string, failureThreshold int) (*ProxyPool, error) { + normalizedURLs, err := NormalizeProxyURLs(proxyURLs) + if err != nil { + return nil, err + } + if failureThreshold <= 0 { + failureThreshold = DefaultProxyPoolFailureThreshold + } + + entries := make([]ProxyEntry, 0, len(normalizedURLs)) + for _, proxyURL := range normalizedURLs { + entries = append(entries, ProxyEntry{URL: proxyURL}) + } + + return &ProxyPool{ + proxies: entries, + failureThreshold: failureThreshold, + }, nil +} + +func (p *ProxyPool) Next() string { + p.mu.Lock() + defer p.mu.Unlock() + + if len(p.proxies) == 0 { + return "" + } + + if p.allDisabledLocked() { + logrus.Warn("Proxy pool exhausted, re-enabling all configured proxies") + for i := range p.proxies { + p.proxies[i].IsDisabled = false + p.proxies[i].FailCount = 0 + } + } + + for i := 0; i < len(p.proxies); i++ { + idx := (p.next + i) % len(p.proxies) + if p.proxies[idx].IsDisabled { + continue + } + + p.next = (idx + 1) % len(p.proxies) + selected := p.proxies[idx].URL + logrus.Debugf("Selected proxy from pool: %s", MaskProxyURL(selected)) + return selected + } + + return "" +} + +func (p *ProxyPool) ReportFailure(proxyURL string) { + p.mu.Lock() + defer p.mu.Unlock() + + for i := range p.proxies { + if p.proxies[i].URL != proxyURL { + continue + } + + p.proxies[i].FailCount++ + if p.proxies[i].FailCount >= p.failureThreshold { + p.proxies[i].IsDisabled = true + logrus.Warnf( + "Disabled proxy after %d failures: %s", + p.proxies[i].FailCount, + MaskProxyURL(proxyURL), + ) + } + return + } +} + +func (p *ProxyPool) ReportSuccess(proxyURL string) { + p.mu.Lock() + defer p.mu.Unlock() + + for i := range p.proxies { + if p.proxies[i].URL != proxyURL { + continue + } + + p.proxies[i].FailCount = 0 + p.proxies[i].IsDisabled = false + return + } +} + +func (p *ProxyPool) Size() int { + p.mu.Lock() + defer p.mu.Unlock() + + return len(p.proxies) +} + +func (p *ProxyPool) Stats() ProxyPoolStats { + p.mu.Lock() + defer p.mu.Unlock() + + stats := ProxyPoolStats{ + FailureThreshold: p.failureThreshold, + Total: len(p.proxies), + } + for _, proxy := range p.proxies { + if proxy.IsDisabled { + stats.Disabled++ + continue + } + stats.Active++ + } + + return stats +} + +func (p *ProxyPool) allDisabledLocked() bool { + if len(p.proxies) == 0 { + return false + } + for _, proxy := range p.proxies { + if !proxy.IsDisabled { + return false + } + } + return true +} + +func normalizeProxyRuntime(runtime string) string { + switch strings.ToLower(strings.TrimSpace(runtime)) { + case ProxyRuntimeRaw: + return ProxyRuntimeRaw + default: + return ProxyRuntimeBrowser + } +} diff --git a/core/proxy_integration_test.go b/core/proxy_integration_test.go new file mode 100644 index 0000000..620b299 --- /dev/null +++ b/core/proxy_integration_test.go @@ -0,0 +1,226 @@ +//go:build integration +// +build integration + +package core + +import ( + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "testing" + + "golang.org/x/time/rate" +) + +const proxyIntegrationEnabledEnv = "OPENSERP_PROXY_TESTS" + +type proxyIntegrationURLs struct { + targetURL string + socks5hAuthURL string + socks5hPlainURL string + httpAuthURL string + httpPlainURL string + badSocks5URL string + badHTTPURL string +} + +func TestIntegrationSocks5hAuthProxyDNS(t *testing.T) { + cfg := proxyIntegrationConfig(t) + assertProxyFetchesTarget(t, cfg.targetURL, cfg.socks5hAuthURL) +} + +func TestIntegrationSocks5hPlainProxyDNS(t *testing.T) { + cfg := proxyIntegrationConfig(t) + assertProxyFetchesTarget(t, cfg.targetURL, cfg.socks5hPlainURL) +} + +func TestIntegrationHTTPAuthProxy(t *testing.T) { + cfg := proxyIntegrationConfig(t) + assertProxyFetchesTarget(t, cfg.targetURL, cfg.httpAuthURL) +} + +func TestIntegrationHTTPPlainProxy(t *testing.T) { + cfg := proxyIntegrationConfig(t) + assertProxyFetchesTarget(t, cfg.targetURL, cfg.httpPlainURL) +} + +func TestIntegrationRawSOCKSProxyPoolRotation(t *testing.T) { + cfg := proxyIntegrationConfig(t) + assertProxyPoolRotation(t, cfg.targetURL, []string{cfg.badSocks5URL, cfg.socks5hAuthURL}) +} + +func TestIntegrationRawHTTPProxyPoolRotation(t *testing.T) { + cfg := proxyIntegrationConfig(t) + assertProxyPoolRotation(t, cfg.targetURL, []string{cfg.badHTTPURL, cfg.httpAuthURL}) +} + +type proxyIntegrationEngine struct { + targetURL string + limiter *rate.Limiter + proxies []string +} + +func (e *proxyIntegrationEngine) Name() string { + return "google" +} + +func (e *proxyIntegrationEngine) IsInitialized() bool { + return true +} + +func (e *proxyIntegrationEngine) GetRateLimiter() *rate.Limiter { + return e.limiter +} + +func (e *proxyIntegrationEngine) Search(q Query) ([]SearchResult, error) { + e.proxies = append(e.proxies, q.ProxyURL) + + body, err := fetchViaRawProxy(q.ProxyURL, q.Insecure, e.targetURL) + if err != nil { + return nil, err + } + + return []SearchResult{{ + Rank: 1, + URL: e.targetURL, + Title: "proxy-ok", + Description: body, + }}, nil +} + +func (e *proxyIntegrationEngine) SearchImage(q Query) ([]SearchResult, error) { + return nil, ErrSearchTimeout +} + +func proxyIntegrationConfig(t *testing.T) proxyIntegrationURLs { + t.Helper() + + if os.Getenv(proxyIntegrationEnabledEnv) != "1" { + t.Skipf("set %s=1 to run proxy integration tests", proxyIntegrationEnabledEnv) + } + + return proxyIntegrationURLs{ + targetURL: envOrDefault("OPENSERP_PROXY_TEST_TARGET_URL", "http://proxy-target:8080/"), + socks5hAuthURL: envOrDefault("OPENSERP_PROXY_TEST_SOCKS5H_AUTH_URL", "socks5h://test:test@127.0.0.1:19080"), + socks5hPlainURL: envOrDefault("OPENSERP_PROXY_TEST_SOCKS5H_PLAIN_URL", "socks5h://127.0.0.1:19082"), + httpAuthURL: envOrDefault("OPENSERP_PROXY_TEST_HTTP_AUTH_URL", "http://test:test@127.0.0.1:18888"), + httpPlainURL: envOrDefault("OPENSERP_PROXY_TEST_HTTP_PLAIN_URL", "http://127.0.0.1:18889"), + badSocks5URL: envOrDefault("OPENSERP_PROXY_TEST_BAD_SOCKS5_URL", "socks5://127.0.0.1:19081"), + badHTTPURL: envOrDefault("OPENSERP_PROXY_TEST_BAD_HTTP_URL", "http://127.0.0.1:18890"), + } +} + +func assertProxyFetchesTarget(t *testing.T, targetURL, proxyURL string) { + t.Helper() + + target := mustParseURL(t, targetURL) + assertHostCannotResolveTarget(t, target.Hostname()) + + body, err := fetchViaRawProxy(proxyURL, false, targetURL) + if err != nil { + t.Fatalf("expected proxied request via %s to succeed, got %v", proxyURL, err) + } + if !strings.Contains(body, "proxy-ok") { + t.Fatalf("expected proxy target response, got %q", body) + } +} + +func assertProxyPoolRotation(t *testing.T, targetURL string, pool []string) { + t.Helper() + + engine := &proxyIntegrationEngine{ + targetURL: targetURL, + limiter: rate.NewLimiter(rate.Inf, 1), + } + + opts := DefaultServerOptions() + opts.Resilience.Retry.MaxRetries = 1 + opts.Resilience.Retry.InitialBackoff = 0 + opts.Resilience.Retry.MaxBackoff = 0 + opts.Resilience.Retry.BackoffFactor = 1 + opts.Resilience.Proxy = ProxyConfig{ + Runtime: ProxyRuntimeRaw, + PoolURLs: pool, + PoolFailureThreshold: 1, + } + + srv := NewServerWithOptions("127.0.0.1", 7190, opts, engine) + resp := request(t, srv, "/google/search?text=proxy") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected rotated proxy request to succeed, got %d", resp.StatusCode) + } + + if len(engine.proxies) != 2 { + t.Fatalf("expected 2 proxy attempts, got %d", len(engine.proxies)) + } + if engine.proxies[0] != pool[0] || engine.proxies[1] != pool[1] { + t.Fatalf("unexpected proxy rotation order: %#v", engine.proxies) + } + + statsResp := request(t, srv, "/resilience/stats") + var stats map[string]interface{} + if err := json.NewDecoder(statsResp.Body).Decode(&stats); err != nil { + t.Fatalf("decode stats: %v", err) + } + + proxyStats := stats["proxy"].(map[string]interface{}) + poolStats := proxyStats["pool"].(map[string]interface{}) + if got := poolStats["active"].(float64); got != 1 { + t.Fatalf("expected 1 active proxy, got %v", got) + } + if got := poolStats["disabled"].(float64); got != 1 { + t.Fatalf("expected 1 disabled proxy, got %v", got) + } +} + +func fetchViaRawProxy(proxyURL string, insecure bool, targetURL string) (string, error) { + client, err := NewRawHTTPClient(Query{ + ProxyURL: proxyURL, + Insecure: insecure, + }) + if err != nil { + return "", err + } + + resp, err := client.Get(targetURL) + if err != nil { + return "", err + } + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(bodyBytes), nil +} + +func assertHostCannotResolveTarget(t *testing.T, hostname string) { + t.Helper() + + if _, err := net.DefaultResolver.LookupHost(context.Background(), hostname); err == nil { + t.Fatalf("expected direct host-side DNS lookup for %q to fail", hostname) + } +} + +func mustParseURL(t *testing.T, raw string) *url.URL { + t.Helper() + + parsed, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse target URL %q: %v", raw, err) + } + return parsed +} + +func envOrDefault(key, fallback string) string { + if value := strings.TrimSpace(os.Getenv(key)); value != "" { + return value + } + return fallback +} diff --git a/core/proxy_test.go b/core/proxy_test.go new file mode 100644 index 0000000..6fd5bf8 --- /dev/null +++ b/core/proxy_test.go @@ -0,0 +1,201 @@ +package core + +import ( + "fmt" + "io" + "log" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + socks5 "github.com/armon/go-socks5" + xcontext "golang.org/x/net/context" +) + +func TestNormalizeProxyURL(t *testing.T) { + tests := []struct { + name string + raw string + want string + wantErr bool + }{ + {name: "empty", raw: "", want: ""}, + {name: "http", raw: "http://127.0.0.1:8080", want: "http://127.0.0.1:8080"}, + {name: "https", raw: "https://127.0.0.1:8443", want: "https://127.0.0.1:8443"}, + {name: "socks5", raw: "socks5://127.0.0.1:1080", want: "socks5://127.0.0.1:1080"}, + {name: "socks5h upper", raw: "SOCKS5H://127.0.0.1:1080", want: "socks5h://127.0.0.1:1080"}, + {name: "missing scheme", raw: "127.0.0.1:8080", wantErr: true}, + {name: "missing host", raw: "http://", wantErr: true}, + {name: "unsupported scheme", raw: "ftp://127.0.0.1:21", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NormalizeProxyURL(tt.raw) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error for %q", tt.raw) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != tt.want { + t.Fatalf("expected %q, got %q", tt.want, got) + } + }) + } +} + +func TestNormalizeProxyConfigDefaultsAndDeduplicates(t *testing.T) { + cfg, err := NormalizeProxyConfig(ProxyConfig{ + Runtime: "RAW", + StaticURL: " socks5h://127.0.0.1:1080 ", + PoolURLs: []string{ + "", + "http://proxy-one:8080", + "http://proxy-one:8080", + "socks5://proxy-two:1080", + }, + }) + if err != nil { + t.Fatalf("normalize config: %v", err) + } + + if cfg.Runtime != ProxyRuntimeRaw { + t.Fatalf("expected raw runtime, got %s", cfg.Runtime) + } + if cfg.StaticURL != "socks5h://127.0.0.1:1080" { + t.Fatalf("unexpected static proxy: %s", cfg.StaticURL) + } + if cfg.PoolFailureThreshold != DefaultProxyPoolFailureThreshold { + t.Fatalf("expected default threshold %d, got %d", DefaultProxyPoolFailureThreshold, cfg.PoolFailureThreshold) + } + if len(cfg.PoolURLs) != 2 { + t.Fatalf("expected 2 deduplicated pool URLs, got %d", len(cfg.PoolURLs)) + } +} + +func TestProxyPoolRoundRobinAndFailureRecovery(t *testing.T) { + pool, err := NewProxyPool([]string{"http://proxy1:8080", "http://proxy2:8080"}, 2) + if err != nil { + t.Fatalf("new proxy pool: %v", err) + } + + if got := pool.Next(); got != "http://proxy1:8080" { + t.Fatalf("expected first proxy1, got %s", got) + } + if got := pool.Next(); got != "http://proxy2:8080" { + t.Fatalf("expected second proxy2, got %s", got) + } + + pool.ReportFailure("http://proxy1:8080") + pool.ReportFailure("http://proxy1:8080") + if got := pool.Next(); got != "http://proxy2:8080" { + t.Fatalf("expected proxy2 while proxy1 disabled, got %s", got) + } + + pool.ReportFailure("http://proxy2:8080") + pool.ReportFailure("http://proxy2:8080") + if got := pool.Next(); got != "http://proxy1:8080" { + t.Fatalf("expected pool reset to proxy1 after exhaustion, got %s", got) + } + + pool.ReportFailure("http://proxy1:8080") + pool.ReportSuccess("http://proxy1:8080") + stats := pool.Stats() + if stats.Disabled != 0 { + t.Fatalf("expected no disabled proxies after recovery, got %d", stats.Disabled) + } + if stats.Active != 2 { + t.Fatalf("expected both proxies active, got %d", stats.Active) + } +} + +func TestMaskProxyURLRedactsCredentials(t *testing.T) { + if got := MaskProxyURL("http://user:pass@127.0.0.1:8080"); got != "http://127.0.0.1:8080" { + t.Fatalf("unexpected masked proxy value: %s", got) + } +} + +func TestNewRawHTTPClientSocks5hUsesProxyDNS(t *testing.T) { + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("proxied")) + })) + defer target.Close() + + targetAddr, err := net.ResolveTCPAddr("tcp", target.Listener.Addr().String()) + if err != nil { + t.Fatalf("resolve target listener: %v", err) + } + + const proxyOnlyHost = "proxy-target.invalid" + proxyAddr := startSOCKS5TestServer(t, proxyOnlyHost, targetAddr.IP) + + directClient := &http.Client{Timeout: 500 * time.Millisecond} + targetURL := fmt.Sprintf("http://%s:%d/", proxyOnlyHost, targetAddr.Port) + if _, err := directClient.Get(targetURL); err == nil { + t.Fatal("expected direct request to fail without proxy DNS") + } + + client, err := NewRawHTTPClient(Query{ProxyURL: "socks5h://" + proxyAddr}) + if err != nil { + t.Fatalf("new raw http client: %v", err) + } + + resp, err := client.Get(targetURL) + if err != nil { + t.Fatalf("expected proxied request to succeed, got %v", err) + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read proxied body: %v", err) + } + if string(body) != "proxied" { + t.Fatalf("unexpected proxied body: %q", string(body)) + } +} + +type staticResolver struct { + host string + ip net.IP +} + +func (r staticResolver) Resolve(ctx xcontext.Context, name string) (xcontext.Context, net.IP, error) { + if name == r.host { + return ctx, r.ip, nil + } + return ctx, nil, net.UnknownNetworkError(name) +} + +func startSOCKS5TestServer(t *testing.T, host string, ip net.IP) string { + t.Helper() + + server, err := socks5.New(&socks5.Config{ + Resolver: staticResolver{host: host, ip: ip}, + Logger: log.New(io.Discard, "", 0), + }) + if err != nil { + t.Fatalf("create socks5 server: %v", err) + } + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen socks5: %v", err) + } + t.Cleanup(func() { + _ = listener.Close() + }) + + go func() { + _ = server.Serve(listener) + }() + + return listener.Addr().String() +} diff --git a/core/resilient.go b/core/resilient.go index 01bb22e..a8e356b 100644 --- a/core/resilient.go +++ b/core/resilient.go @@ -13,26 +13,53 @@ type ResilientSearcher struct { engines []SearchEngine cbManager *CircuitBreakerManager retryCfg RetryConfig + proxyCfg ProxyConfig + proxyPool *ProxyPool } type ResilientConfig struct { Retry RetryConfig CircuitBreaker CircuitBreakerConfig + Proxy ProxyConfig } func DefaultResilientConfig() ResilientConfig { return ResilientConfig{ Retry: DefaultRetryConfig(), CircuitBreaker: DefaultCircuitBreakerConfig(), + Proxy: DefaultProxyConfig(), } } func NewResilientSearcher(engines []SearchEngine, cfg ResilientConfig) *ResilientSearcher { - return &ResilientSearcher{ + proxyCfg, err := NormalizeProxyConfig(cfg.Proxy) + if err != nil { + logrus.Errorf("Invalid proxy config, disabling proxy support: %v", err) + proxyCfg = DefaultProxyConfig() + } + + rs := &ResilientSearcher{ engines: engines, cbManager: NewCircuitBreakerManager(cfg.CircuitBreaker), retryCfg: cfg.Retry, + proxyCfg: proxyCfg, } + + if len(proxyCfg.PoolURLs) > 0 { + pool, err := NewProxyPool(proxyCfg.PoolURLs, proxyCfg.PoolFailureThreshold) + if err != nil { + logrus.Errorf("Invalid proxy pool config, disabling proxy rotation: %v", err) + } else { + rs.proxyPool = pool + if proxyCfg.Runtime == ProxyRuntimeBrowser { + logrus.Warn("Proxy pool configured in browser runtime; pool remains observability-only until browser proxy rotation is implemented") + } else { + logrus.Infof("Proxy rotation enabled with %d proxies", pool.Size()) + } + } + } + + return rs } // SearchPrimary keeps dedicated endpoints engine-pure (no fallback). @@ -111,7 +138,11 @@ func (rs *ResilientSearcher) searchWithProtection(engine SearchEngine, q Query) return nil, err } } - return engine.Search(q) + + attemptQuery, attemptProxy, reportToPool := rs.prepareAttemptQuery(q) + results, err := engine.Search(attemptQuery) + rs.reportProxyAttempt(attemptProxy, reportToPool, err) + return results, err }) if result.Err != nil { @@ -136,7 +167,11 @@ func (rs *ResilientSearcher) searchImageWithProtection(engine SearchEngine, q Qu return nil, err } } - return engine.SearchImage(q) + + attemptQuery, attemptProxy, reportToPool := rs.prepareAttemptQuery(q) + results, err := engine.SearchImage(attemptQuery) + rs.reportProxyAttempt(attemptProxy, reportToPool, err) + return results, err }) if result.Err != nil { @@ -229,4 +264,88 @@ func (rs *ResilientSearcher) GetCircuitBreakerStats() []map[string]interface{} { return rs.cbManager.AllStats() } +func (rs *ResilientSearcher) GetProxyPool() *ProxyPool { + return rs.proxyPool +} + +func (rs *ResilientSearcher) GetProxyStats() map[string]interface{} { + mode, source, rotationActive := rs.proxyMode() + + stats := map[string]interface{}{ + "mode": mode, + "runtime": rs.proxyCfg.Runtime, + "source": source, + "rotation_active": rotationActive, + } + + if rs.proxyPool != nil { + poolStats := rs.proxyPool.Stats() + stats["pool"] = map[string]interface{}{ + "failure_threshold": poolStats.FailureThreshold, + "total": poolStats.Total, + "active": poolStats.Active, + "disabled": poolStats.Disabled, + } + } + + return stats +} + +func (rs *ResilientSearcher) prepareAttemptQuery(q Query) (Query, string, bool) { + attemptQuery := q + if rs.proxyCfg.Runtime != ProxyRuntimeRaw { + return attemptQuery, "", false + } + + if rs.proxyPool != nil { + proxyURL := rs.proxyPool.Next() + if proxyURL != "" { + attemptQuery.ProxyURL = proxyURL + return attemptQuery, proxyURL, true + } + } + + attemptQuery.ProxyURL = rs.proxyCfg.StaticURL + return attemptQuery, "", false +} + +func (rs *ResilientSearcher) reportProxyAttempt(proxyURL string, reportToPool bool, err error) { + if !reportToPool || rs.proxyPool == nil || proxyURL == "" { + return + } + + if err != nil { + rs.proxyPool.ReportFailure(proxyURL) + return + } + + rs.proxyPool.ReportSuccess(proxyURL) +} + +func (rs *ResilientSearcher) proxyMode() (string, string, bool) { + hasStatic := rs.proxyCfg.StaticURL != "" + hasPool := rs.proxyPool != nil + + switch rs.proxyCfg.Runtime { + case ProxyRuntimeRaw: + switch { + case hasPool: + return ProxyModePool, "proxy_pool.urls", true + case hasStatic: + return ProxyModeStatic, "app.proxy", false + default: + return ProxyModeDisabled, "none", false + } + default: + switch { + case hasStatic: + return ProxyModeStatic, "app.proxy", false + case hasPool: + return ProxyModePool, "proxy_pool.urls", false + default: + return ProxyModeDisabled, "none", false + } + } +} + var ErrAllEnginesFailed = fmt.Errorf("all search engines failed") diff --git a/core/server.go b/core/server.go index 418aec5..eefce58 100644 --- a/core/server.go +++ b/core/server.go @@ -276,6 +276,7 @@ func (s *Server) handleHealthCheck(c *fiber.Ctx) error { func (s *Server) handleResilienceStats(c *fiber.Ctx) error { return c.JSON(map[string]interface{}{ "circuit_breakers": s.resilient.GetCircuitBreakerStats(), + "proxy": s.resilient.GetProxyStats(), }) } diff --git a/core/server_test.go b/core/server_test.go index 00e201e..213b789 100644 --- a/core/server_test.go +++ b/core/server_test.go @@ -402,6 +402,164 @@ func TestResilienceStatsContainsRetryInWhenCircuitOpen(t *testing.T) { } } +func TestResilienceStatsReportProxyModes(t *testing.T) { + engine := &engineMock{name: "google", initialized: true} + + tests := []struct { + name string + proxyCfg ProxyConfig + wantMode string + wantRuntime string + wantSource string + wantRotation bool + wantPool bool + }{ + { + name: "disabled raw", + proxyCfg: ProxyConfig{Runtime: ProxyRuntimeRaw}, + wantMode: ProxyModeDisabled, + wantRuntime: ProxyRuntimeRaw, + wantSource: "none", + wantRotation: false, + wantPool: false, + }, + { + name: "static raw", + proxyCfg: ProxyConfig{ + Runtime: ProxyRuntimeRaw, + StaticURL: "socks5h://127.0.0.1:1080", + }, + wantMode: ProxyModeStatic, + wantRuntime: ProxyRuntimeRaw, + wantSource: "app.proxy", + wantRotation: false, + wantPool: false, + }, + { + name: "pool raw", + proxyCfg: ProxyConfig{ + Runtime: ProxyRuntimeRaw, + PoolURLs: []string{"http://proxy1:8080", "http://proxy2:8080"}, + PoolFailureThreshold: 2, + }, + wantMode: ProxyModePool, + wantRuntime: ProxyRuntimeRaw, + wantSource: "proxy_pool.urls", + wantRotation: true, + wantPool: true, + }, + { + name: "pool browser inactive", + proxyCfg: ProxyConfig{ + Runtime: ProxyRuntimeBrowser, + PoolURLs: []string{"http://proxy1:8080", "http://proxy2:8080"}, + PoolFailureThreshold: 2, + }, + wantMode: ProxyModePool, + wantRuntime: ProxyRuntimeBrowser, + wantSource: "proxy_pool.urls", + wantRotation: false, + wantPool: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + opts := DefaultServerOptions() + opts.Resilience.Proxy = tt.proxyCfg + + srv := NewServerWithOptions("127.0.0.1", 7090, opts, engine) + resp := request(t, srv, "/resilience/stats") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var stats map[string]interface{} + if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil { + t.Fatalf("decode stats response: %v", err) + } + + proxy, ok := stats["proxy"].(map[string]interface{}) + if !ok { + t.Fatalf("expected proxy stats object, got %#v", stats["proxy"]) + } + if got := proxy["mode"]; got != tt.wantMode { + t.Fatalf("expected mode %q, got %#v", tt.wantMode, got) + } + if got := proxy["runtime"]; got != tt.wantRuntime { + t.Fatalf("expected runtime %q, got %#v", tt.wantRuntime, got) + } + if got := proxy["source"]; got != tt.wantSource { + t.Fatalf("expected source %q, got %#v", tt.wantSource, got) + } + if got := proxy["rotation_active"]; got != tt.wantRotation { + t.Fatalf("expected rotation_active=%v, got %#v", tt.wantRotation, got) + } + + _, hasPool := proxy["pool"] + if hasPool != tt.wantPool { + t.Fatalf("expected pool presence=%v, got %v", tt.wantPool, hasPool) + } + }) + } +} + +func TestResilientRawProxyPoolRotatesOnRetry(t *testing.T) { + var attemptedProxies []string + engine := &engineMock{ + name: "google", + initialized: true, + searchFn: func(q Query) ([]SearchResult, error) { + attemptedProxies = append(attemptedProxies, q.ProxyURL) + if q.ProxyURL == "http://bad-proxy:8080" { + return nil, errors.New("proxy failed") + } + return []SearchResult{{Rank: 1, URL: "https://example.com/success", Title: "ok"}}, nil + }, + } + + opts := DefaultServerOptions() + opts.Resilience.Retry.MaxRetries = 1 + opts.Resilience.Retry.InitialBackoff = 0 + opts.Resilience.Retry.MaxBackoff = 0 + opts.Resilience.Retry.BackoffFactor = 1 + opts.Resilience.Proxy = ProxyConfig{ + Runtime: ProxyRuntimeRaw, + PoolURLs: []string{"http://bad-proxy:8080", "http://good-proxy:8080"}, + PoolFailureThreshold: 1, + } + + srv := NewServerWithOptions("127.0.0.1", 7091, opts, engine) + resp := request(t, srv, "/google/search?text=golang") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected search to recover through rotated proxy, got %d", resp.StatusCode) + } + if len(attemptedProxies) != 2 { + t.Fatalf("expected 2 attempts, got %d", len(attemptedProxies)) + } + if attemptedProxies[0] != "http://bad-proxy:8080" || attemptedProxies[1] != "http://good-proxy:8080" { + t.Fatalf("unexpected proxy rotation order: %#v", attemptedProxies) + } + + statsResp := request(t, srv, "/resilience/stats") + var stats map[string]interface{} + if err := json.NewDecoder(statsResp.Body).Decode(&stats); err != nil { + t.Fatalf("decode stats response: %v", err) + } + proxy := stats["proxy"].(map[string]interface{}) + if got := proxy["mode"]; got != ProxyModePool { + t.Fatalf("expected pool mode, got %#v", got) + } + + pool := proxy["pool"].(map[string]interface{}) + if got := pool["active"].(float64); got != 1 { + t.Fatalf("expected 1 active proxy, got %v", got) + } + if got := pool["disabled"].(float64); got != 1 { + t.Fatalf("expected 1 disabled proxy, got %v", got) + } +} + func TestRetryAppliesRateLimiterOnEachAttempt(t *testing.T) { engine := &engineMock{ name: "google", diff --git a/go.mod b/go.mod index b44e076..b2bb06d 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( require ( github.com/andybalholm/brotli v1.2.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/google/uuid v1.6.0 // indirect diff --git a/go.sum b/go.sum index e3ba533..0a3c1bf 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwTo github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= diff --git a/google/search_raw.go b/google/search_raw.go index 599bc5f..6fedf9f 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -1,64 +1,19 @@ package google import ( - "context" - "crypto/tls" - "net" "net/http" - "net/url" "strings" - "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" "github.com/karust/openserp/core" "github.com/sirupsen/logrus" - - utls "github.com/refraction-networking/utls" ) func googleRequest(searchURL string, query core.Query) (*http.Response, error) { - // Create HTTP transport with proxy - transport := &http.Transport{} - if query.ProxyURL != "" { - proxyUrl, err := url.Parse(query.ProxyURL) - if err != nil { - return nil, err - } - transport.Proxy = http.ProxyURL(proxyUrl) - } - - // Set insecure TLS - if query.Insecure { - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - } - - transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - dialer := &net.Dialer{} - rawConn, err := dialer.DialContext(ctx, network, addr) - if err != nil { - return nil, err - } - - hostname := strings.Split(addr, ":")[0] - config := &utls.Config{ - ServerName: hostname, - InsecureSkipVerify: query.Insecure, - } - - uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto) - - if err := uconn.Handshake(); err != nil { - rawConn.Close() - return nil, err - } - - return uconn, nil - } - - baseClient := &http.Client{ - Transport: transport, - Timeout: time.Second * 10, + baseClient, err := core.NewRawHTTPClient(query) + if err != nil { + return nil, err } req, err := http.NewRequest("GET", searchURL, nil) diff --git a/google/search_test.go b/google/search_test.go index 03e5dbd..83ea7d8 100644 --- a/google/search_test.go +++ b/google/search_test.go @@ -2,32 +2,8 @@ package google import ( "testing" - "time" - - "github.com/karust/openserp/core" ) -var browser *core.Browser - -func init() { - opts := core.BrowserOpts{IsHeadless: true, IsLeakless: false, Timeout: time.Second * 5, LeavePageOpen: false} - browser, _ = core.NewBrowser(opts) -} - -func TestSearchGoogle(t *testing.T) { - gogl := New(*browser, core.SearchEngineOptions{}) - - query := core.Query{Text: "HEY", Limit: 10} - results, err := gogl.Search(query) - if err != nil { - t.Fatalf("Cannot [SearchGoogle]: %s", err) - } - - if len(results) == 0 { - t.Fatalf("[SearchGoogle] returned empty result") - } -} - func TestParseSourceImageURL(t *testing.T) { //href1 := `/imgres?imgurl=https%3A%2F%2Fupload.wikimedia.org%2Fwikipedia%2Fcommons%2F2%2F26%2FMarmota_marmota_Alpes2.jpg&tbnid=Be_RycOe8xzlpM&vet=12ahUKEwjkh6WzwIeAAxWV_yoKHRzHC9wQMygAegUIARD0AQ..i&imgrefurl=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FAlpine_marmot&docid=7miWbc2QiSw9uM&w=801&h=599&q=alpine%20marmot&ved=2ahUKEwjkh6WzwIeAAxWV_yoKHRzHC9wQMygAegUIARD0AQ` href2 := `/imgres?imgurl=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fnaturerules1%2Fimages%2Ff%2Ff2%2F13d79d934ccf6f7919777fcb6dbb6e6c.jpg%2Frevision%2Flatest%3Fcb%3D20210218225522&tbnid=JxC8NUyBjdNbdM&vet=12ahUKEwiHrJnN1YeAAxXvEBAIHfRADAAQMygCegUIARD4AQ..i&imgrefurl=https%3A%2F%2Fnaturerules1.fandom.com%2Fwiki%2FAlpine_Marmot&docid=XXYeDjL67badNM&w=1600&h=1200&q=alpine%20marmot&ved=2ahUKEwiHrJnN1YeAAxXvEBAIHfRADAAQMygCegUIARD4AQ` @@ -47,21 +23,3 @@ func TestParseSourceImageURL(t *testing.T) { t.Fatalf("Want: %v, Got: %v", want, got) } } - -func TestImageSearch(t *testing.T) { - gogl := New(*browser, core.SearchEngineOptions{}) - query := core.Query{Text: "Ferrari Testarossa", Limit: 77} - results, err := gogl.SearchImage(query) - - if err != nil { - t.Fatalf("Cannot search images: %s", err) - } - - if len(results) < 77 { - t.Fatalf("Returned not full result") - } - - if results[0].URL == "" { - t.Fatalf("First result doesn't contain URL, %v+", results[0]) - } -} diff --git a/yandex/search_raw.go b/yandex/search_raw.go index 65b54f3..8052539 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -1,64 +1,19 @@ package yandex import ( - "context" - "crypto/tls" - "net" "net/http" - "net/url" "strings" - "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" "github.com/karust/openserp/core" "github.com/sirupsen/logrus" - - utls "github.com/refraction-networking/utls" ) func yandexRequest(searchURL string, query core.Query) (*http.Response, error) { - // Create HTTP transport with proxy - transport := &http.Transport{} - if query.ProxyURL != "" { - proxyUrl, err := url.Parse(query.ProxyURL) - if err != nil { - return nil, err - } - transport.Proxy = http.ProxyURL(proxyUrl) - } - - // Set insecure TLS - if query.Insecure { - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} - } - - transport.DialTLSContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - dialer := &net.Dialer{} - rawConn, err := dialer.DialContext(ctx, network, addr) - if err != nil { - return nil, err - } - - hostname := strings.Split(addr, ":")[0] - config := &utls.Config{ - ServerName: hostname, - InsecureSkipVerify: query.Insecure, - } - - uconn := utls.UClient(rawConn, config, utls.HelloChrome_Auto) - - if err := uconn.Handshake(); err != nil { - rawConn.Close() - return nil, err - } - - return uconn, nil - } - - baseClient := &http.Client{ - Transport: transport, - Timeout: time.Second * 10, + baseClient, err := core.NewRawHTTPClient(query) + if err != nil { + return nil, err } req, err := http.NewRequest("GET", searchURL, nil)