From 3c07c7789af0d2c5b552b0e19babc05ce0602968 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Wed, 1 Apr 2026 17:38:13 +0300 Subject: [PATCH] Runtime proxy override --- cmd/root.go | 4 +- config.yaml | 32 ++++----- core/cache.go | 3 +- core/common.go | 28 ++++---- core/middleware.go | 2 +- core/middleware_test.go | 8 +++ core/proxy.go | 12 ++++ core/proxy_test.go | 26 ++++++++ core/resilient.go | 35 +++++++--- core/server.go | 2 +- core/server_test.go | 142 ++++++++++++++++++++++++++++++++++++++++ 11 files changed, 252 insertions(+), 42 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 12c2730..5102cd9 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -13,7 +13,7 @@ import ( ) const ( - version = "0.5.9" + version = "0.5.10" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) @@ -307,7 +307,7 @@ func setConfigDefaults(v *viper.Viper) { v.SetDefault("cors.enabled", true) v.SetDefault("cors.allow_origins", "*") v.SetDefault("cors.allow_methods", "GET, POST, OPTIONS") - v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization") + v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization, X-Use-Proxy") v.SetDefault("cors.max_age", 86400) } diff --git a/config.yaml b/config.yaml index ae9f7d3..95e2edd 100644 --- a/config.yaml +++ b/config.yaml @@ -14,19 +14,19 @@ app: leave_head: false # Keep tabs open after request for debugging stealth: false # Enable stealth browser plugin -#proxies: -# # Force a single proxy for all engines. -# # Same behavior as passing --proxy on the CLI. -# global: http://test:test@127.0.0.1:18888 -# -# # Advanced mode: define tagged proxy pools and opt engines in with `proxy: `. -# entries: -# - url: http://test:test@127.0.0.1:18888 -# tags: [default, us] -# - url: socks5://127.0.0.1:19080 # Chrome doesn't support authenticated SOCKS proxies and hostname resolution -# tags: [default, eu] -# health: -# failure_threshold: 3 # Disable proxy after this many consecutive failures +proxies: + # Force a single proxy for all engines. + # Same behavior as passing --proxy on the CLI. + #global: socks5h://127.0.0.1:19080 + + # Advanced mode: define tagged proxy pools and opt engines in with `proxy: `. + entries: + - url: http://test:test@127.0.0.1:18888 + tags: [default, us] + - url: socks5://127.0.0.1:19080 # Browser mode fails fast on authenticated SOCKS proxies + tags: [default, eu] + health: + failure_threshold: 3 # Disable proxy after this many consecutive failures cache: ttl_seconds: 60 # Dedicated endpoint cache TTL in seconds (0 disables cache) @@ -55,12 +55,12 @@ google: rate_requests: 4 # Allowed average requests per minute rate_burst: 2 # Burst requests before limiter applies captcha: true # Enable captcha solver path - #proxy: us + proxy: us yandex: rate_requests: 4 rate_burst: 2 - #proxy: eu + proxy: eu baidu: rate_requests: 4 @@ -75,4 +75,4 @@ bing: duckduckgo: rate_requests: 4 rate_burst: 2 - #proxy: default + proxy: default diff --git a/core/cache.go b/core/cache.go index 0e6fe94..e7b67c4 100644 --- a/core/cache.go +++ b/core/cache.go @@ -36,7 +36,7 @@ func NewResponseCache(ttl time.Duration, maxSize int) *ResponseCache { func BuildCacheKey(engine string, action string, q Query) string { raw := fmt.Sprintf( - "%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t", + "%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t|%s", engine, action, q.Text, @@ -48,6 +48,7 @@ func BuildCacheKey(engine string, action string, q Query) string { q.Start, q.Filter, q.Answers, + q.ProxyOverride, ) hash := sha256.Sum256([]byte(raw)) return hex.EncodeToString(hash[:]) diff --git a/core/common.go b/core/common.go index 82e5793..ff9689b 100644 --- a/core/common.go +++ b/core/common.go @@ -54,17 +54,18 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search } type Query struct { - Text string - LangCode string // eg. EN, ES, RU... - DateInterval string // format: YYYYMMDD..YYYMMDD - 20181010..20231010 - Filetype string // File extension to search. - Site string // Search site - Limit int // Limit the number of results - Start int // Search offset for pagination (Google uses 0, 10, 20...) - Filter bool // Filter duplicates (google) (false: include similar, true: hide similar) - Answers bool // Include question and answers from SERP page to results with negative indexes - ProxyURL string // Proxy URL for raw requests - Insecure bool // Allow insecure TLS connections + Text string + LangCode string // eg. EN, ES, RU... + DateInterval string // format: YYYYMMDD..YYYMMDD - 20181010..20231010 + Filetype string // File extension to search. + Site string // Search site + Limit int // Limit the number of results + Start int // Search offset for pagination (Google uses 0, 10, 20...) + Filter bool // Filter duplicates (google) (false: include similar, true: hide similar) + Answers bool // Include question and answers from SERP page to results with negative indexes + ProxyURL string // Proxy URL for raw requests + ProxyOverride string // Request-scoped proxy override: tag or direct + Insecure bool // Allow insecure TLS connections } func ComputePagination(start int, pageSize int) (int, int, error) { @@ -116,6 +117,11 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { return err } + searchQuery.ProxyOverride, err = NormalizeProxyRequestOverride(reqCtx.Get("X-Use-Proxy")) + if err != nil { + return err + } + if searchQuery.IsEmpty() { return errors.New("Query cannot be empty") } diff --git a/core/middleware.go b/core/middleware.go index a5aa0fc..a087e36 100644 --- a/core/middleware.go +++ b/core/middleware.go @@ -26,7 +26,7 @@ func DefaultCORSConfig() CORSConfig { return CORSConfig{ AllowOrigins: "*", AllowMethods: "GET, POST, OPTIONS", - AllowHeaders: "Origin, Content-Type, Accept, Authorization", + AllowHeaders: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy", MaxAge: 86400, } } diff --git a/core/middleware_test.go b/core/middleware_test.go index d4271d0..1bf10f1 100644 --- a/core/middleware_test.go +++ b/core/middleware_test.go @@ -3,6 +3,7 @@ package core import ( "net/http" "net/http/httptest" + "strings" "testing" "github.com/gofiber/fiber/v2" @@ -99,3 +100,10 @@ func TestNormalizeCORSConfig_FillsMissingValues(t *testing.T) { t.Fatalf("expected default max_age, got %d", cfg.MaxAge) } } + +func TestDefaultCORSConfig_IncludesProxyOverrideHeader(t *testing.T) { + cfg := DefaultCORSConfig() + if got := cfg.AllowHeaders; !strings.Contains(got, "X-Use-Proxy") { + t.Fatalf("expected allow_headers to include X-Use-Proxy, got %q", got) + } +} diff --git a/core/proxy.go b/core/proxy.go index 987b8dd..5a9edbd 100644 --- a/core/proxy.go +++ b/core/proxy.go @@ -17,6 +17,7 @@ const ( ProxyModeOff = "off" ProxyModeTagPool = "tag_pool" DefaultProxyFailureThreshold = 3 + ProxyOverrideDirect = "direct" ) var supportedProxySchemes = map[string]struct{}{ @@ -281,6 +282,17 @@ func NormalizeProxyTag(raw string) (string, error) { return tag, nil } +func NormalizeProxyRequestOverride(raw string) (string, error) { + override := normalizeTag(raw) + if override == "" { + return "", nil + } + if override == ProxyOverrideDirect { + return ProxyOverrideDirect, nil + } + return NormalizeProxyTag(override) +} + func IsAuthenticatedSocksProxyURL(raw string) bool { normalized, err := NormalizeProxyURL(raw) if err != nil || normalized == "" { diff --git a/core/proxy_test.go b/core/proxy_test.go index 63b69b6..f504af2 100644 --- a/core/proxy_test.go +++ b/core/proxy_test.go @@ -227,6 +227,32 @@ func TestNormalizeProxyTag(t *testing.T) { } } +func TestNormalizeProxyRequestOverride(t *testing.T) { + override, err := NormalizeProxyRequestOverride(" direct ") + if err != nil { + t.Fatalf("normalize direct proxy override: %v", err) + } + if override != ProxyOverrideDirect { + t.Fatalf("expected direct override, got %q", override) + } + + override, err = NormalizeProxyRequestOverride(" US ") + if err != nil { + t.Fatalf("normalize tagged proxy override: %v", err) + } + if override != "us" { + t.Fatalf("expected normalized tag override us, got %q", override) + } + + override, err = NormalizeProxyRequestOverride("") + if err != nil { + t.Fatalf("normalize empty override: %v", err) + } + if override != "" { + t.Fatalf("expected empty override, got %q", override) + } +} + func TestIsAuthenticatedSocksProxyURL(t *testing.T) { if !IsAuthenticatedSocksProxyURL("socks5h://user:pass@127.0.0.1:1080") { t.Fatal("expected authenticated socks proxy to be detected") diff --git a/core/resilient.go b/core/resilient.go index c8bebb2..8fee752 100644 --- a/core/resilient.go +++ b/core/resilient.go @@ -139,7 +139,7 @@ func (rs *ResilientSearcher) searchWithProtection(engine SearchEngine, q Query, return nil, ProxyExecutionMeta{}, ErrCircuitOpen } - policy := rs.effectivePolicyForEngine(engine.Name()) + policy := rs.effectivePolicyForQuery(engine.Name(), q) attemptMeta := rs.baseProxyMeta(policy) result := RetryableSearch(rs.retryCfg, engine.Name(), func() ([]SearchResult, error) { @@ -160,7 +160,7 @@ func (rs *ResilientSearcher) searchWithProtection(engine SearchEngine, q Query, attemptQuery.ProxyURL = "" attemptMeta.Used = "direct" case ProxyModeTagPool: - proxyURL = rs.selectProxyForPolicy(policy) + proxyURL = rs.selectProxyForQuery(policy, q) if proxyURL == "" { return nil, fmt.Errorf("%w: no healthy proxy available for tag %q", ErrProxyUnavailable, policy.Tag) } @@ -318,7 +318,7 @@ func (rs *ResilientSearcher) GetProxyStats() ProxyStats { return stats } -func (rs *ResilientSearcher) ResolveMegaProxyMeta(engines []SearchEngine) ProxyExecutionMeta { +func (rs *ResilientSearcher) ResolveMegaProxyMeta(q Query, engines []SearchEngine) ProxyExecutionMeta { if len(engines) == 0 { return ProxyExecutionMeta{Mode: ProxyModeOff, Used: "direct"} } @@ -328,7 +328,7 @@ func (rs *ResilientSearcher) ResolveMegaProxyMeta(engines []SearchEngine) ProxyE hasOff := false for _, engine := range engines { - policy := rs.effectivePolicyForEngine(engine.Name()) + policy := rs.effectivePolicyForQuery(engine.Name(), q) if policy.Mode == ProxyModeOff { hasOff = true continue @@ -351,9 +351,11 @@ func (rs *ResilientSearcher) ResolveMegaProxyMeta(engines []SearchEngine) ProxyE } } - if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" && !hasOff { - meta.Used = MaskProxyURL(global) - return meta + if q.ProxyOverride == "" { + if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" && !hasOff { + meta.Used = MaskProxyURL(global) + return meta + } } if rs.proxyRuntime == ProxyRuntimeRaw { @@ -388,6 +390,17 @@ func (rs *ResilientSearcher) effectivePolicyForEngine(engineName string) ProxyPo return rs.proxyDefaults } +func (rs *ResilientSearcher) effectivePolicyForQuery(engineName string, q Query) ProxyPolicy { + switch q.ProxyOverride { + case "": + return rs.effectivePolicyForEngine(engineName) + case ProxyOverrideDirect: + return ProxyPolicy{Mode: ProxyModeOff} + default: + return ProxyPolicy{Mode: ProxyModeTagPool, Tag: q.ProxyOverride} + } +} + func (rs *ResilientSearcher) selectProxyForTag(tag string) string { if rs.proxyRegistry == nil { return "" @@ -408,12 +421,14 @@ func (rs *ResilientSearcher) reportProxyAttempt(proxyURL string, err error) { rs.proxyRegistry.ReportSuccess(proxyURL) } -func (rs *ResilientSearcher) selectProxyForPolicy(policy ProxyPolicy) string { +func (rs *ResilientSearcher) selectProxyForQuery(policy ProxyPolicy, q Query) string { if policy.Mode != ProxyModeTagPool { return "" } - if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" { - return global + if q.ProxyOverride == "" { + if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" { + return global + } } return rs.selectProxyForTag(policy.Tag) } diff --git a/core/server.go b/core/server.go index 888573c..54172f9 100644 --- a/core/server.go +++ b/core/server.go @@ -334,7 +334,7 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string, run func(Query, engineNames[i] = engine.Name() } engineNamesJoined := strings.Join(engineNames, ",") - s.applyProxyHeaders(c, s.resilient.ResolveMegaProxyMeta(enginesToUse)) + s.applyProxyHeaders(c, s.resilient.ResolveMegaProxyMeta(q, enginesToUse)) logrus.Infof("Starting SERP mega %s request using engines: %s for query: %s", action, engineNamesJoined, q.Text) cacheHitCandidates := []cacheHitCandidate{ diff --git a/core/server_test.go b/core/server_test.go index df16edb..9f2c62e 100644 --- a/core/server_test.go +++ b/core/server_test.go @@ -60,6 +60,17 @@ func request(t *testing.T, s *Server, path string) *http.Response { return resp } +func requestWithHeader(t *testing.T, s *Server, path string, header string, value string) *http.Response { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + req.Header.Set(header, value) + resp, err := s.app.Test(req, -1) + if err != nil { + t.Fatalf("request failed for %s: %v", path, err) + } + return resp +} + func TestHealthEndpointStatusSemantics(t *testing.T) { ready := &engineMock{name: "google", initialized: true, limiter: rate.NewLimiter(rate.Every(time.Second), 1)} notReady := &engineMock{name: "yandex", initialized: false, limiter: rate.NewLimiter(rate.Every(time.Second), 1)} @@ -663,6 +674,79 @@ func TestGlobalProxyForcesAllEnginesRaw(t *testing.T) { } } +func TestRequestProxyOverrideDirectBeatsGlobal(t *testing.T) { + var googleProxy string + engine := &engineMock{ + name: "google", + initialized: true, + searchFn: func(q Query) ([]SearchResult, error) { + googleProxy = q.ProxyURL + return []SearchResult{{Rank: 1, URL: "https://example.com/google", Title: "google"}}, nil + }, + } + + opts := DefaultServerOptions() + opts.Resilience.Proxy = ProxyConfig{ + Runtime: ProxyRuntimeRaw, + Proxies: ProxiesConfig{ + Global: "http://global-proxy:8080", + }, + } + + srv := NewServerWithOptions("127.0.0.1", 7099, opts, engine) + resp := requestWithHeader(t, srv, "/google/search?text=golang", "X-Use-Proxy", "direct") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected direct override request to succeed, got %d", resp.StatusCode) + } + if googleProxy != "" { + t.Fatalf("expected direct override to disable proxy, got %q", googleProxy) + } + if got := resp.Header.Get("X-Proxy-Mode"); got != ProxyModeOff { + t.Fatalf("expected X-Proxy-Mode=%s, got %q", ProxyModeOff, got) + } + if got := resp.Header.Get("X-Proxy-Used"); got != "direct" { + t.Fatalf("expected X-Proxy-Used=direct, got %q", got) + } +} + +func TestRequestProxyOverrideTagBeatsGlobal(t *testing.T) { + var googleProxy string + engine := &engineMock{ + name: "google", + initialized: true, + searchFn: func(q Query) ([]SearchResult, error) { + googleProxy = q.ProxyURL + return []SearchResult{{Rank: 1, URL: "https://example.com/google", Title: "google"}}, nil + }, + } + + opts := DefaultServerOptions() + opts.Resilience.Proxy = ProxyConfig{ + Runtime: ProxyRuntimeRaw, + Proxies: ProxiesConfig{ + Global: "http://global-proxy:8080", + Entries: []ProxyEntryConfig{ + {URL: "http://proxy-us:8080", Tags: []string{"us"}}, + }, + }, + } + + srv := NewServerWithOptions("127.0.0.1", 7100, opts, engine) + resp := requestWithHeader(t, srv, "/google/search?text=golang", "X-Use-Proxy", "us") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected tagged override request to succeed, got %d", resp.StatusCode) + } + if googleProxy != "http://proxy-us:8080" { + t.Fatalf("expected tagged override to use pool proxy, got %q", googleProxy) + } + if got := resp.Header.Get("X-Proxy-Tag"); got != "us" { + t.Fatalf("expected X-Proxy-Tag=us, got %q", got) + } + if got := resp.Header.Get("X-Proxy-Used"); got != "http://proxy-us:8080" { + t.Fatalf("expected X-Proxy-Used to reflect override proxy, got %q", got) + } +} + func TestBrowserProxyPoolRotatesPerRequest(t *testing.T) { var attemptedProxies []string engine := &engineMock{ @@ -702,6 +786,64 @@ func TestBrowserProxyPoolRotatesPerRequest(t *testing.T) { } } +func TestRequestProxyOverrideMissingTagFailsClosed(t *testing.T) { + engine := &engineMock{name: "google", initialized: true} + opts := DefaultServerOptions() + opts.Resilience.Proxy = ProxyConfig{ + Runtime: ProxyRuntimeRaw, + Proxies: ProxiesConfig{ + Global: "http://global-proxy:8080", + }, + } + + srv := NewServerWithOptions("127.0.0.1", 7101, opts, engine) + resp := requestWithHeader(t, srv, "/google/search?text=golang", "X-Use-Proxy", "missing") + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("expected missing tag override to fail closed, got %d", resp.StatusCode) + } + if got := resp.Header.Get("X-Proxy-Mode"); got != ProxyModeTagPool { + t.Fatalf("expected X-Proxy-Mode=%s on missing tag response, got %q", ProxyModeTagPool, got) + } + if got := resp.Header.Get("X-Proxy-Tag"); got != "missing" { + t.Fatalf("expected X-Proxy-Tag=missing, got %q", got) + } +} + +func TestMegaProxyOverrideHeaderBeatsGlobal(t *testing.T) { + var googleProxy string + engine := &engineMock{ + name: "google", + initialized: true, + searchFn: func(q Query) ([]SearchResult, error) { + googleProxy = q.ProxyURL + return []SearchResult{{Rank: 1, URL: "https://example.com/google", Title: "google"}}, nil + }, + } + + opts := DefaultServerOptions() + opts.Resilience.Proxy = ProxyConfig{ + Runtime: ProxyRuntimeRaw, + Proxies: ProxiesConfig{ + Global: "http://global-proxy:8080", + Entries: []ProxyEntryConfig{ + {URL: "http://proxy-us:8080", Tags: []string{"us"}}, + }, + }, + } + + srv := NewServerWithOptions("127.0.0.1", 7102, opts, engine) + resp := requestWithHeader(t, srv, "/mega/search?text=golang&engines=google", "X-Use-Proxy", "us") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected mega override request to succeed, got %d", resp.StatusCode) + } + if googleProxy != "http://proxy-us:8080" { + t.Fatalf("expected mega override to use pool proxy, got %q", googleProxy) + } + if got := resp.Header.Get("X-Proxy-Tag"); got != "us" { + t.Fatalf("expected mega X-Proxy-Tag=us, got %q", got) + } +} + func TestProxyFailClosedWhenNoHealthyProxy(t *testing.T) { engine := &engineMock{name: "google", initialized: true} opts := DefaultServerOptions()