feat(browser): make configurable resource/tracker blocking

cmd/root.go

@@ -15,7 +15,7 @@ import (
 )
 
 const (
-	version               = "0.6.6"
+	version               = "0.6.7"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )
@@ -58,6 +58,8 @@ type AppConfig struct {
 	IsBrowserHead  bool   `mapstructure:"head"`
 	IsLeaveHead    bool   `mapstructure:"leave_head"`
 	IsLeakless     bool   `mapstructure:"leakless"`
+	BlockResources string `mapstructure:"block_resources"`
+	BlockTrackers  bool   `mapstructure:"block_trackers"`
 	DebugEndpoints bool   `mapstructure:"debug_endpoints"`
 	LogFormat      string `mapstructure:"log_format"`
 }
@@ -227,6 +229,13 @@ func initializeConfig(cmd *cobra.Command) error {
 	// 3. Command flags (highest priority). Bind the current command's flags to viper
 	bindFlags(cmd, v)
 
+	// Keep compatibility with historical typo in local configs. Runs after all
+	// sources are merged so CLI flags and env vars take precedence over the typo key.
+	if v.IsSet("app.block_resorces") && !v.IsSet("app.block_resources") {
+		v.Set("app.block_resources", v.Get("app.block_resorces"))
+		logrus.Warn(`config key "app.block_resorces" is deprecated, use "app.block_resources"`)
+	}
+
 	if err := validateRemovedConfigPaths(v); err != nil {
 		return err
 	}
@@ -241,6 +250,10 @@ func initializeConfig(cmd *cobra.Command) error {
 		return fmt.Errorf("cannot unmarshall config: %v", err)
 	}
 
+	if _, err := core.ParseBlockedResourceTypes(config.App.BlockResources); err != nil {
+		return fmt.Errorf("invalid app.block_resources: %w", err)
+	}
+
 	config.Proxies, err = core.NormalizeProxiesConfig(config.Proxies)
 	if err != nil {
 		return fmt.Errorf("invalid proxies config: %w", err)
@@ -324,6 +337,8 @@ func setConfigDefaults(v *viper.Viper) {
 	v.SetDefault("app.head", false)
 	v.SetDefault("app.leave_head", false)
 	v.SetDefault("app.leakless", false)
+	v.SetDefault("app.block_resources", "")
+	v.SetDefault("app.block_trackers", false)
 	v.SetDefault("app.debug_endpoints", false)
 
 	v.SetDefault("proxies.entries", []interface{}{})

cmd/search.go

@@ -99,6 +99,7 @@ func search(cmd *cobra.Command, args []string) {
 
 func searchBrowser(engineType string, query core.Query, browserProxyURL string, captchaSolverEnabled bool, captchaSolverAPIKey string) ([]core.SearchResult, error) {
 	var engine core.SearchEngine
+	blockedResourceTypes := core.MustParseBlockedResourceTypes(config.App.BlockResources)
 	if core.IsAuthenticatedSocksProxyURL(browserProxyURL) {
 		return nil, fmt.Errorf(
 			"%w: browser runtime does not support authenticated SOCKS proxy %s",
@@ -117,6 +118,8 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string,
 		BrowserPath:          config.App.BrowserPath,
 		ProxyURL:             browserProxyURL,
 		Insecure:             config.Server.Insecure,
+		BlockResourceTypes:   blockedResourceTypes,
+		BlockTrackers:        config.App.BlockTrackers,
 	}
 
 	if config.Server.IsDebug {

cmd/serve.go

@@ -126,12 +126,16 @@ func serve(cmd *cobra.Command, args []string) {
 }
 
 func buildFingerprintBrowserOptions() core.BrowserOpts {
+	blockedResourceTypes := core.MustParseBlockedResourceTypes(config.App.BlockResources)
+
 	opts := core.BrowserOpts{
-		IsHeadless:  !config.App.IsBrowserHead,
-		IsLeakless:  config.App.IsLeakless,
-		Timeout:     time.Second * time.Duration(config.App.Timeout),
-		BrowserPath: config.App.BrowserPath,
-		Insecure:    config.Server.Insecure,
+		IsHeadless:         !config.App.IsBrowserHead,
+		IsLeakless:         config.App.IsLeakless,
+		Timeout:            time.Second * time.Duration(config.App.Timeout),
+		BrowserPath:        config.App.BrowserPath,
+		Insecure:           config.Server.Insecure,
+		BlockResourceTypes: blockedResourceTypes,
+		BlockTrackers:      config.App.BlockTrackers,
 	}
 	if config.Server.IsDebug {
 		opts.IsHeadless = false

cmd/serve_test.go

@@ -169,3 +169,42 @@ func TestResolveCaptchaSolverConfigEnabledWithKey(t *testing.T) {
 		t.Fatalf("expected configured API key, got %q", key)
 	}
 }
+
+func TestBuildFingerprintBrowserOptionsRespectsBlockConfig(t *testing.T) {
+	origBlockResources := config.App.BlockResources
+	origBlockTrackers := config.App.BlockTrackers
+	origHead := config.App.IsBrowserHead
+	origLeakless := config.App.IsLeakless
+	origTimeout := config.App.Timeout
+	origBrowserPath := config.App.BrowserPath
+	origInsecure := config.Server.Insecure
+	origDebug := config.Server.IsDebug
+	defer func() {
+		config.App.BlockResources = origBlockResources
+		config.App.BlockTrackers = origBlockTrackers
+		config.App.IsBrowserHead = origHead
+		config.App.IsLeakless = origLeakless
+		config.App.Timeout = origTimeout
+		config.App.BrowserPath = origBrowserPath
+		config.Server.Insecure = origInsecure
+		config.Server.IsDebug = origDebug
+	}()
+
+	config.App.BlockResources = "image,font,css,media"
+	config.App.BlockTrackers = true
+	config.App.IsBrowserHead = false
+	config.App.IsLeakless = false
+	config.App.Timeout = 15
+	config.App.BrowserPath = ""
+	config.Server.Insecure = false
+	config.Server.IsDebug = false
+
+	opts := buildFingerprintBrowserOptions()
+	if len(opts.BlockResourceTypes) != 4 {
+		t.Fatalf("expected 4 blocked resource types, got %d", len(opts.BlockResourceTypes))
+	}
+	if !opts.BlockTrackers {
+		t.Fatal("expected tracker blocking to be enabled")
+	}
+
+}

core/browser.go

@@ -50,6 +50,11 @@ type BrowserOpts struct {
 	Insecure bool
 	// UserAgent optionally overrides browser-reported user agent during emulation.
 	UserAgent string
+	// BlockResourceTypes are blocked during page navigation when non-empty.
+	// Typical tokens map to these types: image, font, css(stylesheet), js(script), media.
+	BlockResourceTypes []proto.NetworkResourceType
+	// BlockTrackers toggles static tracker-domain blocking.
+	BlockTrackers bool
 }
 
 // Check applies default option values when optional fields are unset.
@@ -63,6 +68,127 @@ func (o *BrowserOpts) Check() {
 	}
 }
 
+var alwaysBlockedTrackingDomains = []string{
+	"google-analytics.com",
+	"googletagmanager.com",
+	"doubleclick.net",
+	"connect.facebook.net",
+}
+
+var alwaysBlockedTrackingURLPatterns = buildTrackingDomainURLPatterns(alwaysBlockedTrackingDomains)
+
+var blockedResourceTypeTokenMap = map[string]proto.NetworkResourceType{
+	"image": proto.NetworkResourceTypeImage,
+	"font":  proto.NetworkResourceTypeFont,
+	"media": proto.NetworkResourceTypeMedia,
+	"css":   proto.NetworkResourceTypeStylesheet,
+	"js":    proto.NetworkResourceTypeScript,
+}
+
+// ParseBlockedResourceTypes parses a comma-separated config value into
+// NetworkResourceType values accepted by the request blocker.
+func ParseBlockedResourceTypes(raw string) ([]proto.NetworkResourceType, error) {
+	raw = strings.TrimSpace(raw)
+	if raw == "" {
+		return nil, nil
+	}
+
+	parts := strings.Split(raw, ",")
+	seen := make(map[proto.NetworkResourceType]struct{}, len(parts))
+	out := make([]proto.NetworkResourceType, 0, len(parts))
+
+	for _, part := range parts {
+		token := strings.TrimSpace(strings.ToLower(part))
+		if token == "" {
+			continue
+		}
+
+		resourceType, ok := blockedResourceTypeTokenMap[token]
+		if !ok {
+			return nil, fmt.Errorf("unsupported resource type %q", token)
+		}
+
+		if _, exists := seen[resourceType]; exists {
+			continue
+		}
+		seen[resourceType] = struct{}{}
+		out = append(out, resourceType)
+	}
+
+	return out, nil
+}
+
+// MustParseBlockedResourceTypes is like ParseBlockedResourceTypes but panics on error.
+// Only call this after the value has already been validated by ParseBlockedResourceTypes.
+func MustParseBlockedResourceTypes(raw string) []proto.NetworkResourceType {
+	types, err := ParseBlockedResourceTypes(raw)
+	if err != nil {
+		panic(fmt.Sprintf("MustParseBlockedResourceTypes: %v", err))
+	}
+	return types
+}
+
+func buildTrackingDomainURLPatterns(domains []string) []string {
+	patterns := make([]string, 0, len(domains)*2)
+	for _, domain := range domains {
+		domain = strings.TrimSpace(strings.ToLower(domain))
+		if domain == "" {
+			continue
+		}
+		patterns = append(patterns, "*://"+domain+"/*", "*://*."+domain+"/*")
+	}
+	return patterns
+}
+
+func blockedResourceTypeSet(types []proto.NetworkResourceType) map[proto.NetworkResourceType]struct{} {
+	out := make(map[proto.NetworkResourceType]struct{}, len(types))
+	for _, t := range types {
+		if t != "" {
+			out[t] = struct{}{}
+		}
+	}
+	return out
+}
+
+func (b *Browser) configureRequestBlocking(ctx context.Context, page *rod.Page) error {
+	if !b.BlockTrackers && len(b.BlockResourceTypes) == 0 {
+		return nil
+	}
+
+	if b.BlockTrackers && len(alwaysBlockedTrackingURLPatterns) > 0 {
+		if err := (proto.NetworkEnable{}).Call(page); err != nil {
+			return fmt.Errorf("enable network domain for tracker blocking: %w", err)
+		}
+		if err := (proto.NetworkSetBlockedURLs{Urls: alwaysBlockedTrackingURLPatterns}).Call(page); err != nil {
+			return fmt.Errorf("set blocked tracking URLs: %w", err)
+		}
+	}
+
+	if len(b.BlockResourceTypes) == 0 {
+		return nil
+	}
+
+	blocked := blockedResourceTypeSet(b.BlockResourceTypes)
+	router := page.HijackRequests()
+	router.MustAdd("*", func(h *rod.Hijack) {
+		if _, ok := blocked[h.Request.Type()]; ok {
+			h.Response.Fail(proto.NetworkErrorReasonBlockedByClient)
+			return
+		}
+		h.ContinueRequest(&proto.FetchContinueRequest{})
+	})
+
+	go router.Run()
+
+	// Stop the router when the page context is done to avoid goroutine leak.
+	go func() {
+		<-ctx.Done()
+		router.MustStop()
+	}()
+
+	return nil
+}
+
 // Browser wraps a launched Chromium instance used by engine implementations.
 type Browser struct {
 	BrowserOpts
@@ -677,6 +803,10 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
 	}
 
 	page = page.Context(ctx)
+	if err := b.configureRequestBlocking(ctx, page); err != nil {
+		closeOnErr()
+		return nil, fmt.Errorf("configure request blocking failed: %w", err)
+	}
 	timedPage := page.Timeout(b.Timeout)
 
 	if err := timedPage.Navigate(URL); err != nil {

core/browser_resource_blocking_test.go

@@ -0,0 +1,82 @@
+package core
+
+import (
+	"testing"
+
+	"github.com/go-rod/rod/lib/proto"
+)
+
+func TestBuildTrackingDomainURLPatterns(t *testing.T) {
+	patterns := buildTrackingDomainURLPatterns([]string{"google-analytics.com"})
+	if len(patterns) != 2 {
+		t.Fatalf("expected 2 URL patterns, got %d", len(patterns))
+	}
+	if patterns[0] != "*://google-analytics.com/*" {
+		t.Fatalf("unexpected root pattern: %q", patterns[0])
+	}
+	if patterns[1] != "*://*.google-analytics.com/*" {
+		t.Fatalf("unexpected subdomain pattern: %q", patterns[1])
+	}
+}
+
+func TestShouldBlockResourceType(t *testing.T) {
+	blockedTypes := blockedResourceTypeSet([]proto.NetworkResourceType{
+		proto.NetworkResourceTypeImage,
+		proto.NetworkResourceTypeFont,
+		proto.NetworkResourceTypeMedia,
+		proto.NetworkResourceTypeStylesheet,
+		proto.NetworkResourceTypeScript,
+	})
+
+	tests := []struct {
+		resourceType proto.NetworkResourceType
+		wantBlocked  bool
+	}{
+		{resourceType: proto.NetworkResourceTypeImage, wantBlocked: true},
+		{resourceType: proto.NetworkResourceTypeFont, wantBlocked: true},
+		{resourceType: proto.NetworkResourceTypeMedia, wantBlocked: true},
+		{resourceType: proto.NetworkResourceTypeStylesheet, wantBlocked: true},
+		{resourceType: proto.NetworkResourceTypeScript, wantBlocked: true},
+		{resourceType: proto.NetworkResourceTypeDocument, wantBlocked: false},
+		{resourceType: proto.NetworkResourceTypeXHR, wantBlocked: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(string(tt.resourceType), func(t *testing.T) {
+			_, got := blockedTypes[tt.resourceType]
+			if got != tt.wantBlocked {
+				t.Fatalf("resource type %s: got blocked=%t want %t", tt.resourceType, got, tt.wantBlocked)
+			}
+		})
+	}
+}
+
+func TestParseBlockedResourceTypes(t *testing.T) {
+	got, err := ParseBlockedResourceTypes("image,font,css,js,media")
+	if err != nil {
+		t.Fatalf("expected no error, got %v", err)
+	}
+
+	expectedSet := map[proto.NetworkResourceType]struct{}{
+		proto.NetworkResourceTypeImage:      {},
+		proto.NetworkResourceTypeFont:       {},
+		proto.NetworkResourceTypeStylesheet: {},
+		proto.NetworkResourceTypeScript:     {},
+		proto.NetworkResourceTypeMedia:      {},
+	}
+	gotSet := blockedResourceTypeSet(got)
+	if len(gotSet) != len(expectedSet) {
+		t.Fatalf("expected %d unique resource types, got %d", len(expectedSet), len(gotSet))
+	}
+	for resourceType := range expectedSet {
+		if _, ok := gotSet[resourceType]; !ok {
+			t.Fatalf("expected resource type %s to be present", resourceType)
+		}
+	}
+}
+
+func TestParseBlockedResourceTypesInvalid(t *testing.T) {
+	if _, err := ParseBlockedResourceTypes("image,unknown"); err == nil {
+		t.Fatal("expected invalid token to return error")
+	}
+}