diff --git a/cmd/root.go b/cmd/root.go index 9497605..1a8975c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,7 +15,7 @@ import ( ) const ( - version = "0.6.6" + version = "0.6.7" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) @@ -58,6 +58,8 @@ type AppConfig struct { IsBrowserHead bool `mapstructure:"head"` IsLeaveHead bool `mapstructure:"leave_head"` IsLeakless bool `mapstructure:"leakless"` + BlockResources string `mapstructure:"block_resources"` + BlockTrackers bool `mapstructure:"block_trackers"` DebugEndpoints bool `mapstructure:"debug_endpoints"` LogFormat string `mapstructure:"log_format"` } @@ -227,6 +229,13 @@ func initializeConfig(cmd *cobra.Command) error { // 3. Command flags (highest priority). Bind the current command's flags to viper bindFlags(cmd, v) + // Keep compatibility with historical typo in local configs. Runs after all + // sources are merged so CLI flags and env vars take precedence over the typo key. + if v.IsSet("app.block_resorces") && !v.IsSet("app.block_resources") { + v.Set("app.block_resources", v.Get("app.block_resorces")) + logrus.Warn(`config key "app.block_resorces" is deprecated, use "app.block_resources"`) + } + if err := validateRemovedConfigPaths(v); err != nil { return err } @@ -241,6 +250,10 @@ func initializeConfig(cmd *cobra.Command) error { return fmt.Errorf("cannot unmarshall config: %v", err) } + if _, err := core.ParseBlockedResourceTypes(config.App.BlockResources); err != nil { + return fmt.Errorf("invalid app.block_resources: %w", err) + } + config.Proxies, err = core.NormalizeProxiesConfig(config.Proxies) if err != nil { return fmt.Errorf("invalid proxies config: %w", err) @@ -324,6 +337,8 @@ func setConfigDefaults(v *viper.Viper) { v.SetDefault("app.head", false) v.SetDefault("app.leave_head", false) v.SetDefault("app.leakless", false) + v.SetDefault("app.block_resources", "") + v.SetDefault("app.block_trackers", false) v.SetDefault("app.debug_endpoints", false) v.SetDefault("proxies.entries", []interface{}{}) diff --git a/cmd/search.go b/cmd/search.go index 564f0e8..66483f5 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -99,6 +99,7 @@ func search(cmd *cobra.Command, args []string) { func searchBrowser(engineType string, query core.Query, browserProxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) ([]core.SearchResult, error) { var engine core.SearchEngine + blockedResourceTypes := core.MustParseBlockedResourceTypes(config.App.BlockResources) if core.IsAuthenticatedSocksProxyURL(browserProxyURL) { return nil, fmt.Errorf( "%w: browser runtime does not support authenticated SOCKS proxy %s", @@ -117,6 +118,8 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string, BrowserPath: config.App.BrowserPath, ProxyURL: browserProxyURL, Insecure: config.Server.Insecure, + BlockResourceTypes: blockedResourceTypes, + BlockTrackers: config.App.BlockTrackers, } if config.Server.IsDebug { diff --git a/cmd/serve.go b/cmd/serve.go index ac99b4b..3548d44 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -126,12 +126,16 @@ func serve(cmd *cobra.Command, args []string) { } func buildFingerprintBrowserOptions() core.BrowserOpts { + blockedResourceTypes := core.MustParseBlockedResourceTypes(config.App.BlockResources) + opts := core.BrowserOpts{ - IsHeadless: !config.App.IsBrowserHead, - IsLeakless: config.App.IsLeakless, - Timeout: time.Second * time.Duration(config.App.Timeout), - BrowserPath: config.App.BrowserPath, - Insecure: config.Server.Insecure, + IsHeadless: !config.App.IsBrowserHead, + IsLeakless: config.App.IsLeakless, + Timeout: time.Second * time.Duration(config.App.Timeout), + BrowserPath: config.App.BrowserPath, + Insecure: config.Server.Insecure, + BlockResourceTypes: blockedResourceTypes, + BlockTrackers: config.App.BlockTrackers, } if config.Server.IsDebug { opts.IsHeadless = false diff --git a/cmd/serve_test.go b/cmd/serve_test.go index 78f6186..ea624da 100644 --- a/cmd/serve_test.go +++ b/cmd/serve_test.go @@ -169,3 +169,42 @@ func TestResolveCaptchaSolverConfigEnabledWithKey(t *testing.T) { t.Fatalf("expected configured API key, got %q", key) } } + +func TestBuildFingerprintBrowserOptionsRespectsBlockConfig(t *testing.T) { + origBlockResources := config.App.BlockResources + origBlockTrackers := config.App.BlockTrackers + origHead := config.App.IsBrowserHead + origLeakless := config.App.IsLeakless + origTimeout := config.App.Timeout + origBrowserPath := config.App.BrowserPath + origInsecure := config.Server.Insecure + origDebug := config.Server.IsDebug + defer func() { + config.App.BlockResources = origBlockResources + config.App.BlockTrackers = origBlockTrackers + config.App.IsBrowserHead = origHead + config.App.IsLeakless = origLeakless + config.App.Timeout = origTimeout + config.App.BrowserPath = origBrowserPath + config.Server.Insecure = origInsecure + config.Server.IsDebug = origDebug + }() + + config.App.BlockResources = "image,font,css,media" + config.App.BlockTrackers = true + config.App.IsBrowserHead = false + config.App.IsLeakless = false + config.App.Timeout = 15 + config.App.BrowserPath = "" + config.Server.Insecure = false + config.Server.IsDebug = false + + opts := buildFingerprintBrowserOptions() + if len(opts.BlockResourceTypes) != 4 { + t.Fatalf("expected 4 blocked resource types, got %d", len(opts.BlockResourceTypes)) + } + if !opts.BlockTrackers { + t.Fatal("expected tracker blocking to be enabled") + } + +} diff --git a/core/browser.go b/core/browser.go index 6116d29..7fc2438 100644 --- a/core/browser.go +++ b/core/browser.go @@ -50,6 +50,11 @@ type BrowserOpts struct { Insecure bool // UserAgent optionally overrides browser-reported user agent during emulation. UserAgent string + // BlockResourceTypes are blocked during page navigation when non-empty. + // Typical tokens map to these types: image, font, css(stylesheet), js(script), media. + BlockResourceTypes []proto.NetworkResourceType + // BlockTrackers toggles static tracker-domain blocking. + BlockTrackers bool } // Check applies default option values when optional fields are unset. @@ -63,6 +68,127 @@ func (o *BrowserOpts) Check() { } } +var alwaysBlockedTrackingDomains = []string{ + "google-analytics.com", + "googletagmanager.com", + "doubleclick.net", + "connect.facebook.net", +} + +var alwaysBlockedTrackingURLPatterns = buildTrackingDomainURLPatterns(alwaysBlockedTrackingDomains) + +var blockedResourceTypeTokenMap = map[string]proto.NetworkResourceType{ + "image": proto.NetworkResourceTypeImage, + "font": proto.NetworkResourceTypeFont, + "media": proto.NetworkResourceTypeMedia, + "css": proto.NetworkResourceTypeStylesheet, + "js": proto.NetworkResourceTypeScript, +} + +// ParseBlockedResourceTypes parses a comma-separated config value into +// NetworkResourceType values accepted by the request blocker. +func ParseBlockedResourceTypes(raw string) ([]proto.NetworkResourceType, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return nil, nil + } + + parts := strings.Split(raw, ",") + seen := make(map[proto.NetworkResourceType]struct{}, len(parts)) + out := make([]proto.NetworkResourceType, 0, len(parts)) + + for _, part := range parts { + token := strings.TrimSpace(strings.ToLower(part)) + if token == "" { + continue + } + + resourceType, ok := blockedResourceTypeTokenMap[token] + if !ok { + return nil, fmt.Errorf("unsupported resource type %q", token) + } + + if _, exists := seen[resourceType]; exists { + continue + } + seen[resourceType] = struct{}{} + out = append(out, resourceType) + } + + return out, nil +} + +// MustParseBlockedResourceTypes is like ParseBlockedResourceTypes but panics on error. +// Only call this after the value has already been validated by ParseBlockedResourceTypes. +func MustParseBlockedResourceTypes(raw string) []proto.NetworkResourceType { + types, err := ParseBlockedResourceTypes(raw) + if err != nil { + panic(fmt.Sprintf("MustParseBlockedResourceTypes: %v", err)) + } + return types +} + +func buildTrackingDomainURLPatterns(domains []string) []string { + patterns := make([]string, 0, len(domains)*2) + for _, domain := range domains { + domain = strings.TrimSpace(strings.ToLower(domain)) + if domain == "" { + continue + } + patterns = append(patterns, "*://"+domain+"/*", "*://*."+domain+"/*") + } + return patterns +} + +func blockedResourceTypeSet(types []proto.NetworkResourceType) map[proto.NetworkResourceType]struct{} { + out := make(map[proto.NetworkResourceType]struct{}, len(types)) + for _, t := range types { + if t != "" { + out[t] = struct{}{} + } + } + return out +} + +func (b *Browser) configureRequestBlocking(ctx context.Context, page *rod.Page) error { + if !b.BlockTrackers && len(b.BlockResourceTypes) == 0 { + return nil + } + + if b.BlockTrackers && len(alwaysBlockedTrackingURLPatterns) > 0 { + if err := (proto.NetworkEnable{}).Call(page); err != nil { + return fmt.Errorf("enable network domain for tracker blocking: %w", err) + } + if err := (proto.NetworkSetBlockedURLs{Urls: alwaysBlockedTrackingURLPatterns}).Call(page); err != nil { + return fmt.Errorf("set blocked tracking URLs: %w", err) + } + } + + if len(b.BlockResourceTypes) == 0 { + return nil + } + + blocked := blockedResourceTypeSet(b.BlockResourceTypes) + router := page.HijackRequests() + router.MustAdd("*", func(h *rod.Hijack) { + if _, ok := blocked[h.Request.Type()]; ok { + h.Response.Fail(proto.NetworkErrorReasonBlockedByClient) + return + } + h.ContinueRequest(&proto.FetchContinueRequest{}) + }) + + go router.Run() + + // Stop the router when the page context is done to avoid goroutine leak. + go func() { + <-ctx.Done() + router.MustStop() + }() + + return nil +} + // Browser wraps a launched Chromium instance used by engine implementations. type Browser struct { BrowserOpts @@ -677,6 +803,10 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) { } page = page.Context(ctx) + if err := b.configureRequestBlocking(ctx, page); err != nil { + closeOnErr() + return nil, fmt.Errorf("configure request blocking failed: %w", err) + } timedPage := page.Timeout(b.Timeout) if err := timedPage.Navigate(URL); err != nil { diff --git a/core/browser_resource_blocking_test.go b/core/browser_resource_blocking_test.go new file mode 100644 index 0000000..9c82419 --- /dev/null +++ b/core/browser_resource_blocking_test.go @@ -0,0 +1,82 @@ +package core + +import ( + "testing" + + "github.com/go-rod/rod/lib/proto" +) + +func TestBuildTrackingDomainURLPatterns(t *testing.T) { + patterns := buildTrackingDomainURLPatterns([]string{"google-analytics.com"}) + if len(patterns) != 2 { + t.Fatalf("expected 2 URL patterns, got %d", len(patterns)) + } + if patterns[0] != "*://google-analytics.com/*" { + t.Fatalf("unexpected root pattern: %q", patterns[0]) + } + if patterns[1] != "*://*.google-analytics.com/*" { + t.Fatalf("unexpected subdomain pattern: %q", patterns[1]) + } +} + +func TestShouldBlockResourceType(t *testing.T) { + blockedTypes := blockedResourceTypeSet([]proto.NetworkResourceType{ + proto.NetworkResourceTypeImage, + proto.NetworkResourceTypeFont, + proto.NetworkResourceTypeMedia, + proto.NetworkResourceTypeStylesheet, + proto.NetworkResourceTypeScript, + }) + + tests := []struct { + resourceType proto.NetworkResourceType + wantBlocked bool + }{ + {resourceType: proto.NetworkResourceTypeImage, wantBlocked: true}, + {resourceType: proto.NetworkResourceTypeFont, wantBlocked: true}, + {resourceType: proto.NetworkResourceTypeMedia, wantBlocked: true}, + {resourceType: proto.NetworkResourceTypeStylesheet, wantBlocked: true}, + {resourceType: proto.NetworkResourceTypeScript, wantBlocked: true}, + {resourceType: proto.NetworkResourceTypeDocument, wantBlocked: false}, + {resourceType: proto.NetworkResourceTypeXHR, wantBlocked: false}, + } + + for _, tt := range tests { + t.Run(string(tt.resourceType), func(t *testing.T) { + _, got := blockedTypes[tt.resourceType] + if got != tt.wantBlocked { + t.Fatalf("resource type %s: got blocked=%t want %t", tt.resourceType, got, tt.wantBlocked) + } + }) + } +} + +func TestParseBlockedResourceTypes(t *testing.T) { + got, err := ParseBlockedResourceTypes("image,font,css,js,media") + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + + expectedSet := map[proto.NetworkResourceType]struct{}{ + proto.NetworkResourceTypeImage: {}, + proto.NetworkResourceTypeFont: {}, + proto.NetworkResourceTypeStylesheet: {}, + proto.NetworkResourceTypeScript: {}, + proto.NetworkResourceTypeMedia: {}, + } + gotSet := blockedResourceTypeSet(got) + if len(gotSet) != len(expectedSet) { + t.Fatalf("expected %d unique resource types, got %d", len(expectedSet), len(gotSet)) + } + for resourceType := range expectedSet { + if _, ok := gotSet[resourceType]; !ok { + t.Fatalf("expected resource type %s to be present", resourceType) + } + } +} + +func TestParseBlockedResourceTypesInvalid(t *testing.T) { + if _, err := ParseBlockedResourceTypes("image,unknown"); err == nil { + t.Fatal("expected invalid token to return error") + } +}