zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-16 21:36:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Runtime proxy override

Browse Source
This commit is contained in:
Rustem Kamalov
2026-04-01 17:38:13 +03:00
parent 3daa48e6a3
commit 3c07c7789a
11 changed files with 252 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/root.go
View File

@@ -13,7 +13,7 @@ import (
)
const (
version = "0.5.9"
version = "0.5.10"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)
@@ -307,7 +307,7 @@ func setConfigDefaults(v *viper.Viper) {
v.SetDefault("cors.enabled", true)
v.SetDefault("cors.allow_origins", "*")
v.SetDefault("cors.allow_methods", "GET, POST, OPTIONS")
v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization")
v.SetDefault("cors.allow_headers", "Origin, Content-Type, Accept, Authorization, X-Use-Proxy")
v.SetDefault("cors.max_age", 86400)
}

32
config.yaml
View File

@@ -14,19 +14,19 @@ app:
leave_head: false # Keep tabs open after request for debugging
stealth: false # Enable stealth browser plugin
#proxies:
# # Force a single proxy for all engines.
# # Same behavior as passing --proxy on the CLI.
# global: http://test:test@127.0.0.1:18888
#
# # Advanced mode: define tagged proxy pools and opt engines in with `proxy: <tag>`.
# entries:
# - url: http://test:test@127.0.0.1:18888
# tags: [default, us]
# - url: socks5://127.0.0.1:19080 # Chrome doesn't support authenticated SOCKS proxies and hostname resolution
# tags: [default, eu]
# health:
# failure_threshold: 3 # Disable proxy after this many consecutive failures
proxies:
# Force a single proxy for all engines.
# Same behavior as passing --proxy on the CLI.
#global: socks5h://127.0.0.1:19080
# Advanced mode: define tagged proxy pools and opt engines in with `proxy: <tag>`.
entries:
- url: http://test:test@127.0.0.1:18888
tags: [default, us]
- url: socks5://127.0.0.1:19080 # Browser mode fails fast on authenticated SOCKS proxies
tags: [default, eu]
health:
failure_threshold: 3 # Disable proxy after this many consecutive failures
cache:
ttl_seconds: 60 # Dedicated endpoint cache TTL in seconds (0 disables cache)
@@ -55,12 +55,12 @@ google:
rate_requests: 4 # Allowed average requests per minute
rate_burst: 2 # Burst requests before limiter applies
captcha: true # Enable captcha solver path
#proxy: us
proxy: us
yandex:
rate_requests: 4
rate_burst: 2
#proxy: eu
proxy: eu
baidu:
rate_requests: 4
@@ -75,4 +75,4 @@ bing:
duckduckgo:
rate_requests: 4
rate_burst: 2
#proxy: default
proxy: default

3
core/cache.go
View File

@@ -36,7 +36,7 @@ func NewResponseCache(ttl time.Duration, maxSize int) *ResponseCache {
func BuildCacheKey(engine string, action string, q Query) string {
raw := fmt.Sprintf(
"%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t",
"%s|%s|%s|%s|%s|%s|%s|%d|%d|%t|%t|%s",
engine,
action,
q.Text,
@@ -48,6 +48,7 @@ func BuildCacheKey(engine string, action string, q Query) string {
q.Start,
q.Filter,
q.Answers,
q.ProxyOverride,
)
hash := sha256.Sum256([]byte(raw))
return hex.EncodeToString(hash[:])

28
core/common.go
View File

@@ -54,17 +54,18 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search
}
type Query struct {
Text string
LangCode string // eg. EN, ES, RU...
DateInterval string // format: YYYYMMDD..YYYMMDD - 20181010..20231010
Filetype string // File extension to search.
Site string // Search site
Limit int // Limit the number of results
Start int // Search offset for pagination (Google uses 0, 10, 20...)
Filter bool // Filter duplicates (google) (false: include similar, true: hide similar)
Answers bool // Include question and answers from SERP page to results with negative indexes
ProxyURL string // Proxy URL for raw requests
Insecure bool // Allow insecure TLS connections
Text string
LangCode string // eg. EN, ES, RU...
DateInterval string // format: YYYYMMDD..YYYMMDD - 20181010..20231010
Filetype string // File extension to search.
Site string // Search site
Limit int // Limit the number of results
Start int // Search offset for pagination (Google uses 0, 10, 20...)
Filter bool // Filter duplicates (google) (false: include similar, true: hide similar)
Answers bool // Include question and answers from SERP page to results with negative indexes
ProxyURL string // Proxy URL for raw requests
ProxyOverride string // Request-scoped proxy override: tag or direct
Insecure bool // Allow insecure TLS connections
}
func ComputePagination(start int, pageSize int) (int, int, error) {
@@ -116,6 +117,11 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error {
return err
}
searchQuery.ProxyOverride, err = NormalizeProxyRequestOverride(reqCtx.Get("X-Use-Proxy"))
if err != nil {
return err
}
if searchQuery.IsEmpty() {
return errors.New("Query cannot be empty")
}

2
core/middleware.go
View File

@@ -26,7 +26,7 @@ func DefaultCORSConfig() CORSConfig {
return CORSConfig{
AllowOrigins: "*",
AllowMethods: "GET, POST, OPTIONS",
AllowHeaders: "Origin, Content-Type, Accept, Authorization",
AllowHeaders: "Origin, Content-Type, Accept, Authorization, X-Use-Proxy",
MaxAge: 86400,
}
}

8
core/middleware_test.go
View File

@@ -3,6 +3,7 @@ package core
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gofiber/fiber/v2"
@@ -99,3 +100,10 @@ func TestNormalizeCORSConfig_FillsMissingValues(t *testing.T) {
t.Fatalf("expected default max_age, got %d", cfg.MaxAge)
}
}
func TestDefaultCORSConfig_IncludesProxyOverrideHeader(t *testing.T) {
cfg := DefaultCORSConfig()
if got := cfg.AllowHeaders; !strings.Contains(got, "X-Use-Proxy") {
t.Fatalf("expected allow_headers to include X-Use-Proxy, got %q", got)
}
}

12
core/proxy.go
View File

@@ -17,6 +17,7 @@ const (
ProxyModeOff = "off"
ProxyModeTagPool = "tag_pool"
DefaultProxyFailureThreshold = 3
ProxyOverrideDirect = "direct"
)
var supportedProxySchemes = map[string]struct{}{
@@ -281,6 +282,17 @@ func NormalizeProxyTag(raw string) (string, error) {
return tag, nil
}
func NormalizeProxyRequestOverride(raw string) (string, error) {
override := normalizeTag(raw)
if override == "" {
return "", nil
}
if override == ProxyOverrideDirect {
return ProxyOverrideDirect, nil
}
return NormalizeProxyTag(override)
}
func IsAuthenticatedSocksProxyURL(raw string) bool {
normalized, err := NormalizeProxyURL(raw)
if err != nil || normalized == "" {

26
core/proxy_test.go
View File

@@ -227,6 +227,32 @@ func TestNormalizeProxyTag(t *testing.T) {
}
}
func TestNormalizeProxyRequestOverride(t *testing.T) {
override, err := NormalizeProxyRequestOverride(" direct ")
if err != nil {
t.Fatalf("normalize direct proxy override: %v", err)
}
if override != ProxyOverrideDirect {
t.Fatalf("expected direct override, got %q", override)
}
override, err = NormalizeProxyRequestOverride(" US ")
if err != nil {
t.Fatalf("normalize tagged proxy override: %v", err)
}
if override != "us" {
t.Fatalf("expected normalized tag override us, got %q", override)
}
override, err = NormalizeProxyRequestOverride("")
if err != nil {
t.Fatalf("normalize empty override: %v", err)
}
if override != "" {
t.Fatalf("expected empty override, got %q", override)
}
}
func TestIsAuthenticatedSocksProxyURL(t *testing.T) {
if !IsAuthenticatedSocksProxyURL("socks5h://user:pass@127.0.0.1:1080") {
t.Fatal("expected authenticated socks proxy to be detected")

35
core/resilient.go
View File

@@ -139,7 +139,7 @@ func (rs *ResilientSearcher) searchWithProtection(engine SearchEngine, q Query,
return nil, ProxyExecutionMeta{}, ErrCircuitOpen
}
policy := rs.effectivePolicyForEngine(engine.Name())
policy := rs.effectivePolicyForQuery(engine.Name(), q)
attemptMeta := rs.baseProxyMeta(policy)
result := RetryableSearch(rs.retryCfg, engine.Name(), func() ([]SearchResult, error) {
@@ -160,7 +160,7 @@ func (rs *ResilientSearcher) searchWithProtection(engine SearchEngine, q Query,
attemptQuery.ProxyURL = ""
attemptMeta.Used = "direct"
case ProxyModeTagPool:
proxyURL = rs.selectProxyForPolicy(policy)
proxyURL = rs.selectProxyForQuery(policy, q)
if proxyURL == "" {
return nil, fmt.Errorf("%w: no healthy proxy available for tag %q", ErrProxyUnavailable, policy.Tag)
}
@@ -318,7 +318,7 @@ func (rs *ResilientSearcher) GetProxyStats() ProxyStats {
return stats
}
func (rs *ResilientSearcher) ResolveMegaProxyMeta(engines []SearchEngine) ProxyExecutionMeta {
func (rs *ResilientSearcher) ResolveMegaProxyMeta(q Query, engines []SearchEngine) ProxyExecutionMeta {
if len(engines) == 0 {
return ProxyExecutionMeta{Mode: ProxyModeOff, Used: "direct"}
}
@@ -328,7 +328,7 @@ func (rs *ResilientSearcher) ResolveMegaProxyMeta(engines []SearchEngine) ProxyE
hasOff := false
for _, engine := range engines {
policy := rs.effectivePolicyForEngine(engine.Name())
policy := rs.effectivePolicyForQuery(engine.Name(), q)
if policy.Mode == ProxyModeOff {
hasOff = true
continue
@@ -351,9 +351,11 @@ func (rs *ResilientSearcher) ResolveMegaProxyMeta(engines []SearchEngine) ProxyE
}
}
if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" && !hasOff {
meta.Used = MaskProxyURL(global)
return meta
if q.ProxyOverride == "" {
if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" && !hasOff {
meta.Used = MaskProxyURL(global)
return meta
}
}
if rs.proxyRuntime == ProxyRuntimeRaw {
@@ -388,6 +390,17 @@ func (rs *ResilientSearcher) effectivePolicyForEngine(engineName string) ProxyPo
return rs.proxyDefaults
}
func (rs *ResilientSearcher) effectivePolicyForQuery(engineName string, q Query) ProxyPolicy {
switch q.ProxyOverride {
case "":
return rs.effectivePolicyForEngine(engineName)
case ProxyOverrideDirect:
return ProxyPolicy{Mode: ProxyModeOff}
default:
return ProxyPolicy{Mode: ProxyModeTagPool, Tag: q.ProxyOverride}
}
}
func (rs *ResilientSearcher) selectProxyForTag(tag string) string {
if rs.proxyRegistry == nil {
return ""
@@ -408,12 +421,14 @@ func (rs *ResilientSearcher) reportProxyAttempt(proxyURL string, err error) {
rs.proxyRegistry.ReportSuccess(proxyURL)
}
func (rs *ResilientSearcher) selectProxyForPolicy(policy ProxyPolicy) string {
func (rs *ResilientSearcher) selectProxyForQuery(policy ProxyPolicy, q Query) string {
if policy.Mode != ProxyModeTagPool {
return ""
}
if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" {
return global
if q.ProxyOverride == "" {
if global := strings.TrimSpace(rs.proxyCfg.Proxies.Global); global != "" {
return global
}
}
return rs.selectProxyForTag(policy.Tag)
}

2
core/server.go
View File

@@ -334,7 +334,7 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string, run func(Query,
engineNames[i] = engine.Name()
}
engineNamesJoined := strings.Join(engineNames, ",")
s.applyProxyHeaders(c, s.resilient.ResolveMegaProxyMeta(enginesToUse))
s.applyProxyHeaders(c, s.resilient.ResolveMegaProxyMeta(q, enginesToUse))
logrus.Infof("Starting SERP mega %s request using engines: %s for query: %s", action, engineNamesJoined, q.Text)
cacheHitCandidates := []cacheHitCandidate{

142
core/server_test.go
View File

@@ -60,6 +60,17 @@ func request(t *testing.T, s *Server, path string) *http.Response {
return resp
}
func requestWithHeader(t *testing.T, s *Server, path string, header string, value string) *http.Response {
t.Helper()
req := httptest.NewRequest(http.MethodGet, path, nil)
req.Header.Set(header, value)
resp, err := s.app.Test(req, -1)
if err != nil {
t.Fatalf("request failed for %s: %v", path, err)
}
return resp
}
func TestHealthEndpointStatusSemantics(t *testing.T) {
ready := &engineMock{name: "google", initialized: true, limiter: rate.NewLimiter(rate.Every(time.Second), 1)}
notReady := &engineMock{name: "yandex", initialized: false, limiter: rate.NewLimiter(rate.Every(time.Second), 1)}
@@ -663,6 +674,79 @@ func TestGlobalProxyForcesAllEnginesRaw(t *testing.T) {
}
}
func TestRequestProxyOverrideDirectBeatsGlobal(t *testing.T) {
var googleProxy string
engine := &engineMock{
name: "google",
initialized: true,
searchFn: func(q Query) ([]SearchResult, error) {
googleProxy = q.ProxyURL
return []SearchResult{{Rank: 1, URL: "https://example.com/google", Title: "google"}}, nil
},
}
opts := DefaultServerOptions()
opts.Resilience.Proxy = ProxyConfig{
Runtime: ProxyRuntimeRaw,
Proxies: ProxiesConfig{
Global: "http://global-proxy:8080",
},
}
srv := NewServerWithOptions("127.0.0.1", 7099, opts, engine)
resp := requestWithHeader(t, srv, "/google/search?text=golang", "X-Use-Proxy", "direct")
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected direct override request to succeed, got %d", resp.StatusCode)
}
if googleProxy != "" {
t.Fatalf("expected direct override to disable proxy, got %q", googleProxy)
}
if got := resp.Header.Get("X-Proxy-Mode"); got != ProxyModeOff {
t.Fatalf("expected X-Proxy-Mode=%s, got %q", ProxyModeOff, got)
}
if got := resp.Header.Get("X-Proxy-Used"); got != "direct" {
t.Fatalf("expected X-Proxy-Used=direct, got %q", got)
}
}
func TestRequestProxyOverrideTagBeatsGlobal(t *testing.T) {
var googleProxy string
engine := &engineMock{
name: "google",
initialized: true,
searchFn: func(q Query) ([]SearchResult, error) {
googleProxy = q.ProxyURL
return []SearchResult{{Rank: 1, URL: "https://example.com/google", Title: "google"}}, nil
},
}
opts := DefaultServerOptions()
opts.Resilience.Proxy = ProxyConfig{
Runtime: ProxyRuntimeRaw,
Proxies: ProxiesConfig{
Global: "http://global-proxy:8080",
Entries: []ProxyEntryConfig{
{URL: "http://proxy-us:8080", Tags: []string{"us"}},
},
},
}
srv := NewServerWithOptions("127.0.0.1", 7100, opts, engine)
resp := requestWithHeader(t, srv, "/google/search?text=golang", "X-Use-Proxy", "us")
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected tagged override request to succeed, got %d", resp.StatusCode)
}
if googleProxy != "http://proxy-us:8080" {
t.Fatalf("expected tagged override to use pool proxy, got %q", googleProxy)
}
if got := resp.Header.Get("X-Proxy-Tag"); got != "us" {
t.Fatalf("expected X-Proxy-Tag=us, got %q", got)
}
if got := resp.Header.Get("X-Proxy-Used"); got != "http://proxy-us:8080" {
t.Fatalf("expected X-Proxy-Used to reflect override proxy, got %q", got)
}
}
func TestBrowserProxyPoolRotatesPerRequest(t *testing.T) {
var attemptedProxies []string
engine := &engineMock{
@@ -702,6 +786,64 @@ func TestBrowserProxyPoolRotatesPerRequest(t *testing.T) {
}
}
func TestRequestProxyOverrideMissingTagFailsClosed(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
opts.Resilience.Proxy = ProxyConfig{
Runtime: ProxyRuntimeRaw,
Proxies: ProxiesConfig{
Global: "http://global-proxy:8080",
},
}
srv := NewServerWithOptions("127.0.0.1", 7101, opts, engine)
resp := requestWithHeader(t, srv, "/google/search?text=golang", "X-Use-Proxy", "missing")
if resp.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("expected missing tag override to fail closed, got %d", resp.StatusCode)
}
if got := resp.Header.Get("X-Proxy-Mode"); got != ProxyModeTagPool {
t.Fatalf("expected X-Proxy-Mode=%s on missing tag response, got %q", ProxyModeTagPool, got)
}
if got := resp.Header.Get("X-Proxy-Tag"); got != "missing" {
t.Fatalf("expected X-Proxy-Tag=missing, got %q", got)
}
}
func TestMegaProxyOverrideHeaderBeatsGlobal(t *testing.T) {
var googleProxy string
engine := &engineMock{
name: "google",
initialized: true,
searchFn: func(q Query) ([]SearchResult, error) {
googleProxy = q.ProxyURL
return []SearchResult{{Rank: 1, URL: "https://example.com/google", Title: "google"}}, nil
},
}
opts := DefaultServerOptions()
opts.Resilience.Proxy = ProxyConfig{
Runtime: ProxyRuntimeRaw,
Proxies: ProxiesConfig{
Global: "http://global-proxy:8080",
Entries: []ProxyEntryConfig{
{URL: "http://proxy-us:8080", Tags: []string{"us"}},
},
},
}
srv := NewServerWithOptions("127.0.0.1", 7102, opts, engine)
resp := requestWithHeader(t, srv, "/mega/search?text=golang&engines=google", "X-Use-Proxy", "us")
if resp.StatusCode != http.StatusOK {
t.Fatalf("expected mega override request to succeed, got %d", resp.StatusCode)
}
if googleProxy != "http://proxy-us:8080" {
t.Fatalf("expected mega override to use pool proxy, got %q", googleProxy)
}
if got := resp.Header.Get("X-Proxy-Tag"); got != "us" {
t.Fatalf("expected mega X-Proxy-Tag=us, got %q", got)
}
}
func TestProxyFailClosedWhenNoHealthyProxy(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()