mirror of
https://github.com/karust/openserp.git
synced 2026-08-05 16:53:54 +08:00
feat(browser): replace stealth with custom profile functionality
This commit is contained in:
@@ -74,6 +74,7 @@ func (baid *Baidu) isTimeout(page *rod.Page) bool {
|
||||
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
|
||||
ctx = core.WithEngine(core.EnsureContext(ctx), baid.Name())
|
||||
ctx = core.WithProfileRegion(ctx, query.LangCode)
|
||||
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
|
||||
scoped := *baid
|
||||
scoped.logger = baid.logger.WithRequest(ctx)
|
||||
@@ -177,6 +178,7 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
|
||||
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
|
||||
ctx = core.WithEngine(core.EnsureContext(ctx), baid.Name())
|
||||
ctx = core.WithProfileRegion(ctx, query.LangCode)
|
||||
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
|
||||
scoped := *baid
|
||||
scoped.logger = baid.logger.WithRequest(ctx)
|
||||
|
||||
@@ -98,6 +98,7 @@ func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
|
||||
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
|
||||
ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name())
|
||||
ctx = core.WithProfileRegion(ctx, query.LangCode)
|
||||
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
|
||||
scoped := *bing
|
||||
scoped.logger = bing.logger.WithRequest(ctx)
|
||||
@@ -274,6 +275,7 @@ type BingImageData struct {
|
||||
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
|
||||
ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name())
|
||||
ctx = core.WithProfileRegion(ctx, query.LangCode)
|
||||
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
|
||||
scoped := *bing
|
||||
scoped.logger = bing.logger.WithRequest(ctx)
|
||||
|
||||
14
cmd/root.go
14
cmd/root.go
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/karust/openserp/core"
|
||||
browserprofile "github.com/karust/openserp/core/browser"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/pflag"
|
||||
@@ -14,7 +15,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
version = "0.6.5"
|
||||
version = "0.6.6"
|
||||
defaultConfigFilename = "config"
|
||||
envPrefix = "OPENSERP"
|
||||
)
|
||||
@@ -53,10 +54,10 @@ type ServerConfig struct {
|
||||
type AppConfig struct {
|
||||
Timeout int `mapstructure:"timeout"`
|
||||
BrowserPath string `mapstructure:"browser_path"`
|
||||
ProfilesJSON string `mapstructure:"profiles"`
|
||||
IsBrowserHead bool `mapstructure:"head"`
|
||||
IsLeaveHead bool `mapstructure:"leave_head"`
|
||||
IsLeakless bool `mapstructure:"leakless"`
|
||||
IsStealth bool `mapstructure:"stealth"`
|
||||
DebugEndpoints bool `mapstructure:"debug_endpoints"`
|
||||
LogFormat string `mapstructure:"log_format"`
|
||||
}
|
||||
@@ -102,6 +103,7 @@ var flagToConfigKey = map[string]string{
|
||||
"timeout": "app.timeout",
|
||||
"config": "server.config_path",
|
||||
"browser-path": "app.browser_path",
|
||||
"profiles-json": "app.profiles",
|
||||
"verbose": "server.verbose",
|
||||
"debug": "server.debug",
|
||||
"head": "app.head",
|
||||
@@ -110,7 +112,6 @@ var flagToConfigKey = map[string]string{
|
||||
"leave": "app.leave_head",
|
||||
"2captcha_key": "2captcha.apikey",
|
||||
"proxy": "proxies.global",
|
||||
"stealth": "app.stealth",
|
||||
"debug-endpoints": "app.debug_endpoints",
|
||||
"insecure": "server.insecure",
|
||||
"cache_ttl": "cache.ttl_seconds",
|
||||
@@ -134,6 +135,9 @@ var RootCmd = &cobra.Command{
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := browserprofile.LoadProfilesFromJSON(config.App.ProfilesJSON); err != nil {
|
||||
return fmt.Errorf("load app.profiles: %w", err)
|
||||
}
|
||||
|
||||
logFormat, err := core.NormalizeLogFormat(config.App.LogFormat)
|
||||
if err != nil {
|
||||
@@ -316,10 +320,10 @@ func setConfigDefaults(v *viper.Viper) {
|
||||
|
||||
v.SetDefault("app.timeout", 30)
|
||||
v.SetDefault("app.browser_path", "")
|
||||
v.SetDefault("app.profiles", "")
|
||||
v.SetDefault("app.head", false)
|
||||
v.SetDefault("app.leave_head", false)
|
||||
v.SetDefault("app.leakless", false)
|
||||
v.SetDefault("app.stealth", false)
|
||||
v.SetDefault("app.debug_endpoints", false)
|
||||
|
||||
v.SetDefault("proxies.entries", []interface{}{})
|
||||
@@ -348,6 +352,7 @@ func init() {
|
||||
RootCmd.PersistentFlags().IntVarP(&config.App.Timeout, "timeout", "t", 30, "Timeout to fail request")
|
||||
RootCmd.PersistentFlags().StringVarP(&config.Server.ConfigPath, "config", "c", "", "Configuration file path")
|
||||
RootCmd.PersistentFlags().StringVarP(&config.App.BrowserPath, "browser-path", "", "", "Custom browser binary path (Chrome/Chromium/Edge/Brave..)")
|
||||
RootCmd.PersistentFlags().StringVar(&config.App.ProfilesJSON, "profiles", "", "Path to browser profile catalog JSON")
|
||||
RootCmd.PersistentFlags().BoolVarP(&config.Server.IsVerbose, "verbose", "v", false, "Use verbose output")
|
||||
RootCmd.PersistentFlags().BoolVarP(&config.Server.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
|
||||
RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI")
|
||||
@@ -356,7 +361,6 @@ func init() {
|
||||
RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
|
||||
RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
|
||||
RootCmd.PersistentFlags().StringVarP(&config.Proxies.Global, "proxy", "x", "", "Force a single proxy for all engines (same as proxies.global)")
|
||||
RootCmd.PersistentFlags().BoolVarP(&config.App.IsStealth, "stealth", "s", false, "Use stealth browser plugin")
|
||||
RootCmd.PersistentFlags().BoolVar(&config.App.DebugEndpoints, "debug-endpoints", false, "Enable debug-only HTTP endpoints")
|
||||
RootCmd.PersistentFlags().BoolVarP(&config.Server.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
|
||||
RootCmd.PersistentFlags().IntVar(&config.Cache.TTLSeconds, "cache_ttl", 300, "Cache TTL in seconds (0 to disable)")
|
||||
|
||||
@@ -117,7 +117,6 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string,
|
||||
BrowserPath: config.App.BrowserPath,
|
||||
ProxyURL: browserProxyURL,
|
||||
Insecure: config.Server.Insecure,
|
||||
UseStealth: config.App.IsStealth,
|
||||
}
|
||||
|
||||
if config.Server.IsDebug {
|
||||
|
||||
@@ -132,7 +132,6 @@ func buildFingerprintBrowserOptions() core.BrowserOpts {
|
||||
Timeout: time.Second * time.Duration(config.App.Timeout),
|
||||
BrowserPath: config.App.BrowserPath,
|
||||
Insecure: config.Server.Insecure,
|
||||
UseStealth: config.App.IsStealth,
|
||||
}
|
||||
if config.Server.IsDebug {
|
||||
opts.IsHeadless = false
|
||||
|
||||
@@ -11,10 +11,10 @@ app:
|
||||
log_format: "text"
|
||||
timeout: 15 # Browser/search timeout in seconds
|
||||
browser_path: "" # Custom browser binary path (chrome/chromium/edge..)
|
||||
profiles: "" # Optional JSON file path overriding built-in browser profiles
|
||||
head: false # Show browser UI (headful mode)
|
||||
leakless: false # Force browser process cleanup after request
|
||||
leave_head: false # Keep tabs open after request for debugging
|
||||
stealth: false # Enable stealth browser plugin
|
||||
|
||||
proxies:
|
||||
# Force a single proxy for all engines.
|
||||
|
||||
392
core/browser.go
392
core/browser.go
@@ -2,21 +2,23 @@ package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/go-rod/rod/lib/devices"
|
||||
"github.com/go-rod/rod/lib/launcher"
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
"github.com/go-rod/stealth"
|
||||
browserprofile "github.com/karust/openserp/core/browser"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/ysmood/gson"
|
||||
)
|
||||
|
||||
// BrowserOpts configures Chromium launch and navigation behavior.
|
||||
@@ -45,8 +47,6 @@ type BrowserOpts struct {
|
||||
ProxyURL string
|
||||
// Insecure allows invalid TLS certificates for browser requests.
|
||||
Insecure bool
|
||||
// UseStealth enables go-rod stealth page creation.
|
||||
UseStealth bool
|
||||
// UserAgent optionally overrides browser-reported user agent during emulation.
|
||||
UserAgent string
|
||||
}
|
||||
@@ -71,15 +71,18 @@ type Browser struct {
|
||||
}
|
||||
|
||||
type browserConnection struct {
|
||||
mu sync.Mutex
|
||||
browser *rod.Browser
|
||||
cachedUserAgent string
|
||||
mu sync.Mutex
|
||||
browser *rod.Browser
|
||||
laneProfiles map[string]browserprofile.Profile
|
||||
}
|
||||
|
||||
// NewBrowser launches a new Chromium process via Rod launcher and returns a
|
||||
// Browser wrapper configured with proxy and captcha solver settings.
|
||||
func NewBrowser(opts BrowserOpts) (*Browser, error) {
|
||||
opts.Check()
|
||||
if strings.TrimSpace(opts.UserAgent) != "" {
|
||||
logrus.Warn("custom user_agent override can reduce profile coherence; use only for diagnostics")
|
||||
}
|
||||
logrus.WithField("browser_options", fmt.Sprintf("%+v", opts)).Debug("Browser options")
|
||||
|
||||
path, err := resolveBrowserBinaryPath(opts.BrowserPath, launcher.LookPath)
|
||||
@@ -87,9 +90,20 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Create launcher
|
||||
l := launcher.New().Leakless(opts.IsLeakless).Headless(opts.IsHeadless).Set("disable-blink-features", "AutomationControlled").
|
||||
Delete("enable-automation")
|
||||
// Create launcher.
|
||||
// headless=new uses the full Chrome renderer; legacy --headless disables the
|
||||
// GPU process entirely, making WebGL context creation fail even with SwiftShader.
|
||||
// use-angle=swiftshader-webgl (Chrome ≥112) enables a software WebGL renderer.
|
||||
l := launcher.New().Leakless(opts.IsLeakless).
|
||||
Set("disable-blink-features", "AutomationControlled").
|
||||
Delete("enable-automation").
|
||||
Set("use-angle", "swiftshader-webgl").
|
||||
Set("ignore-gpu-blocklist")
|
||||
if opts.IsHeadless {
|
||||
l = l.HeadlessNew(true)
|
||||
} else {
|
||||
l = l.Headless(false)
|
||||
}
|
||||
if path != "" {
|
||||
logrus.WithField("browser_path", path).Debug("Using browser binary")
|
||||
l = l.Bin(path)
|
||||
@@ -204,35 +218,26 @@ func (b *Browser) newRodBrowser() *rod.Browser {
|
||||
return browser
|
||||
}
|
||||
|
||||
func (b *Browser) connectBrowser() (*rod.Browser, string, error) {
|
||||
func (b *Browser) connectBrowser() (*rod.Browser, error) {
|
||||
browser := b.newRodBrowser()
|
||||
if err := browser.Connect(); err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Keep cert handling on the persistent browser session.
|
||||
// Proxy runtime can surface MITM certs, and insecure mode is explicit opt-in.
|
||||
if b.ProxyURL != "" || b.Insecure {
|
||||
if err := browser.IgnoreCertErrors(true); err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
version, err := browser.Version()
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("read browser version: %w", err)
|
||||
}
|
||||
ua := strings.ReplaceAll(version.UserAgent, "HeadlessChrome/", "Chrome/")
|
||||
if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" {
|
||||
ua = overrideUA
|
||||
}
|
||||
|
||||
return browser, ua, nil
|
||||
return browser, nil
|
||||
}
|
||||
|
||||
func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect bool) (*rod.Browser, string, error) {
|
||||
func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect bool) (*rod.Browser, error) {
|
||||
if b == nil || b.browserAddr == "" {
|
||||
return nil, "", fmt.Errorf("browser is not initialized")
|
||||
return nil, fmt.Errorf("browser is not initialized")
|
||||
}
|
||||
|
||||
state := b.connectionState()
|
||||
@@ -240,26 +245,24 @@ func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect boo
|
||||
defer state.mu.Unlock()
|
||||
|
||||
if state.browser == nil || forceReconnect {
|
||||
connected, ua, err := b.connectBrowser()
|
||||
connected, err := b.connectBrowser()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
return nil, err
|
||||
}
|
||||
state.browser = connected
|
||||
state.cachedUserAgent = ua
|
||||
return state.browser, ua, nil
|
||||
return state.browser, nil
|
||||
}
|
||||
|
||||
if _, err := state.browser.Version(); err != nil {
|
||||
WithRequest(ctx).WithError(err).Debug("Browser ping failed, reconnecting")
|
||||
connected, ua, reconnectErr := b.connectBrowser()
|
||||
connected, reconnectErr := b.connectBrowser()
|
||||
if reconnectErr != nil {
|
||||
return nil, "", reconnectErr
|
||||
return nil, reconnectErr
|
||||
}
|
||||
state.browser = connected
|
||||
state.cachedUserAgent = ua
|
||||
}
|
||||
|
||||
return state.browser, state.cachedUserAgent, nil
|
||||
return state.browser, nil
|
||||
}
|
||||
|
||||
func createIsolatedPage(browser *rod.Browser) (*rod.Page, proto.BrowserBrowserContextID, error) {
|
||||
@@ -329,7 +332,294 @@ func (b *Browser) startProxyAuthHandler(ctx context.Context, browser *rod.Browse
|
||||
return cancel, nil
|
||||
}
|
||||
|
||||
// Navigate connects to Chromium, creates a page, applies stealth/emulation and
|
||||
var chromeVersionPattern = regexp.MustCompile(`(?:HeadlessChrome|Chrome)/([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`)
|
||||
|
||||
func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browserprofile.Profile, string) {
|
||||
engine := engineFromContext(ctx)
|
||||
region := profileRegionFromContext(ctx)
|
||||
if region == "" {
|
||||
region = strings.TrimSpace(b.LanguageCode)
|
||||
}
|
||||
laneKey := browserprofile.LaneKey(engine, region)
|
||||
|
||||
state := b.connectionState()
|
||||
state.mu.Lock()
|
||||
if state.laneProfiles == nil {
|
||||
state.laneProfiles = make(map[string]browserprofile.Profile)
|
||||
}
|
||||
if profile, ok := state.laneProfiles[laneKey]; ok {
|
||||
state.mu.Unlock()
|
||||
return profile, laneKey
|
||||
}
|
||||
state.mu.Unlock()
|
||||
|
||||
// Resolve profile outside the lock: SelectProfile reads from a separate
|
||||
// RWMutex-guarded catalog, and applyRuntimeBrowserVersion makes a CDP
|
||||
// round-trip (browser.Version). Holding state.mu over network I/O would
|
||||
// serialize all concurrent Navigate calls.
|
||||
profile := browserprofile.SelectProfile(engine, region)
|
||||
profile = applyRuntimeBrowserVersion(profile, browser)
|
||||
if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" {
|
||||
profile.UserAgent = overrideUA
|
||||
}
|
||||
|
||||
state.mu.Lock()
|
||||
if state.laneProfiles == nil {
|
||||
state.laneProfiles = make(map[string]browserprofile.Profile)
|
||||
}
|
||||
if _, exists := state.laneProfiles[laneKey]; !exists {
|
||||
state.laneProfiles[laneKey] = profile
|
||||
} else {
|
||||
profile = state.laneProfiles[laneKey]
|
||||
}
|
||||
state.mu.Unlock()
|
||||
|
||||
return profile, laneKey
|
||||
}
|
||||
|
||||
func applyRuntimeBrowserVersion(profile browserprofile.Profile, browser *rod.Browser) browserprofile.Profile {
|
||||
fullVersion := ""
|
||||
if browser != nil {
|
||||
version, err := browser.Version()
|
||||
if err == nil && version != nil {
|
||||
fullVersion = extractChromeVersion(version.UserAgent)
|
||||
if fullVersion == "" {
|
||||
fullVersion = extractChromeVersion(version.Product)
|
||||
}
|
||||
}
|
||||
}
|
||||
if fullVersion == "" {
|
||||
fullVersion = extractChromeVersion(profile.UserAgent)
|
||||
}
|
||||
if fullVersion == "" {
|
||||
return profile
|
||||
}
|
||||
|
||||
major := chromeMajorVersion(fullVersion)
|
||||
if major == "" {
|
||||
return profile
|
||||
}
|
||||
|
||||
profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, major+".0.0.0")
|
||||
profile.UACHBrands = patchBrandVersions(profile.UACHBrands, major, false)
|
||||
profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, fullVersion, true)
|
||||
return profile
|
||||
}
|
||||
|
||||
func extractChromeVersion(value string) string {
|
||||
matches := chromeVersionPattern.FindStringSubmatch(strings.TrimSpace(value))
|
||||
if len(matches) < 2 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(matches[1])
|
||||
}
|
||||
|
||||
func chromeMajorVersion(fullVersion string) string {
|
||||
fullVersion = strings.TrimSpace(fullVersion)
|
||||
if fullVersion == "" {
|
||||
return ""
|
||||
}
|
||||
parts := strings.Split(fullVersion, ".")
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(parts[0])
|
||||
}
|
||||
|
||||
func replaceChromeUserAgentVersion(userAgent string, replacement string) string {
|
||||
replacement = strings.TrimSpace(replacement)
|
||||
if replacement == "" {
|
||||
return strings.ReplaceAll(strings.TrimSpace(userAgent), "HeadlessChrome/", "Chrome/")
|
||||
}
|
||||
normalized := strings.ReplaceAll(strings.TrimSpace(userAgent), "HeadlessChrome/", "Chrome/")
|
||||
return chromeVersionPattern.ReplaceAllString(normalized, "Chrome/"+replacement)
|
||||
}
|
||||
|
||||
func patchBrandVersions(values []browserprofile.BrandVersion, version string, patchNotABrand bool) []browserprofile.BrandVersion {
|
||||
version = strings.TrimSpace(version)
|
||||
if version == "" {
|
||||
return values
|
||||
}
|
||||
|
||||
out := make([]browserprofile.BrandVersion, 0, len(values))
|
||||
for _, value := range values {
|
||||
item := value
|
||||
brandLower := strings.ToLower(strings.TrimSpace(item.Brand))
|
||||
if brandLower == "chromium" || brandLower == "google chrome" {
|
||||
item.Version = version
|
||||
} else if patchNotABrand && strings.Contains(brandLower, "not_a brand") && strings.Count(version, ".") == 3 {
|
||||
item.Version = "24.0.0.0"
|
||||
}
|
||||
out = append(out, item)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func navigatorPlatformForProfile(profile browserprofile.Profile) string {
|
||||
switch strings.ToLower(strings.TrimSpace(profile.Platform)) {
|
||||
case "windows":
|
||||
return "Win32"
|
||||
case "macos":
|
||||
return "MacIntel"
|
||||
case "linux":
|
||||
return "Linux x86_64"
|
||||
default:
|
||||
return strings.TrimSpace(profile.Platform)
|
||||
}
|
||||
}
|
||||
|
||||
func applyProfile(page *rod.Page, profile browserprofile.Profile) error {
|
||||
if page == nil {
|
||||
return fmt.Errorf("page is nil")
|
||||
}
|
||||
|
||||
navigatorLangs := profileNavigatorLanguages(profile)
|
||||
acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
|
||||
if acceptLanguage == "" {
|
||||
acceptLanguage = navigatorLangs[0]
|
||||
}
|
||||
locale := strings.TrimSpace(profile.Locale)
|
||||
if locale == "" {
|
||||
locale = navigatorLangs[0]
|
||||
}
|
||||
|
||||
width := profile.Viewport.Width
|
||||
height := profile.Viewport.Height
|
||||
if width <= 0 {
|
||||
width = 1920
|
||||
}
|
||||
if height <= 0 {
|
||||
height = 1080
|
||||
}
|
||||
|
||||
metadata := &proto.EmulationUserAgentMetadata{
|
||||
Brands: toProtoBrandVersions(profile.UACHBrands),
|
||||
FullVersionList: toProtoBrandVersions(profile.UACHFullVerList),
|
||||
Platform: strings.TrimSpace(profile.Platform),
|
||||
PlatformVersion: strings.TrimSpace(profile.PlatformVersion),
|
||||
Architecture: strings.TrimSpace(profile.Architecture),
|
||||
Bitness: strings.TrimSpace(profile.Bitness),
|
||||
Mobile: profile.Mobile,
|
||||
}
|
||||
|
||||
if err := (proto.NetworkSetUserAgentOverride{
|
||||
UserAgent: strings.TrimSpace(profile.UserAgent),
|
||||
AcceptLanguage: acceptLanguage,
|
||||
Platform: navigatorPlatformForProfile(profile),
|
||||
UserAgentMetadata: metadata,
|
||||
}).Call(page); err != nil {
|
||||
return fmt.Errorf("set user agent override failed: %w", err)
|
||||
}
|
||||
|
||||
if err := (proto.EmulationSetLocaleOverride{
|
||||
Locale: locale,
|
||||
}).Call(page); err != nil {
|
||||
return fmt.Errorf("set locale override failed: %w", err)
|
||||
}
|
||||
|
||||
if err := (proto.EmulationSetTimezoneOverride{
|
||||
TimezoneID: strings.TrimSpace(profile.Timezone),
|
||||
}).Call(page); err != nil {
|
||||
return fmt.Errorf("set timezone override failed: %w", err)
|
||||
}
|
||||
|
||||
if err := (proto.EmulationSetDeviceMetricsOverride{
|
||||
Width: width,
|
||||
Height: height,
|
||||
DeviceScaleFactor: 1,
|
||||
Mobile: profile.Mobile,
|
||||
ScreenWidth: &width,
|
||||
ScreenHeight: &height,
|
||||
}).Call(page); err != nil {
|
||||
return fmt.Errorf("set device metrics failed: %w", err)
|
||||
}
|
||||
|
||||
if err := (proto.NetworkSetExtraHTTPHeaders{
|
||||
Headers: proto.NetworkHeaders{
|
||||
"Accept-Language": gson.New(acceptLanguage),
|
||||
},
|
||||
}).Call(page); err != nil {
|
||||
return fmt.Errorf("set extra headers failed: %w", err)
|
||||
}
|
||||
|
||||
if err := evalPatchScript(page, navigatorLangs, width, height); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func profileNavigatorLanguages(profile browserprofile.Profile) []string {
|
||||
langs := make([]string, 0, len(profile.NavigatorLangs))
|
||||
for _, language := range profile.NavigatorLangs {
|
||||
trimmed := strings.TrimSpace(language)
|
||||
if trimmed == "" {
|
||||
continue
|
||||
}
|
||||
langs = append(langs, trimmed)
|
||||
}
|
||||
if len(langs) > 0 {
|
||||
return langs
|
||||
}
|
||||
|
||||
acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
|
||||
if acceptLanguage != "" {
|
||||
parts := strings.Split(acceptLanguage, ",")
|
||||
langs = make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
if idx := strings.Index(part, ";"); idx >= 0 {
|
||||
part = strings.TrimSpace(part[:idx])
|
||||
}
|
||||
if part != "" {
|
||||
langs = append(langs, part)
|
||||
}
|
||||
}
|
||||
if len(langs) > 0 {
|
||||
return langs
|
||||
}
|
||||
}
|
||||
|
||||
if locale := strings.TrimSpace(profile.Locale); locale != "" {
|
||||
return []string{locale}
|
||||
}
|
||||
return []string{"en-US"}
|
||||
}
|
||||
|
||||
func toProtoBrandVersions(values []browserprofile.BrandVersion) []*proto.EmulationUserAgentBrandVersion {
|
||||
out := make([]*proto.EmulationUserAgentBrandVersion, 0, len(values))
|
||||
for _, value := range values {
|
||||
brand := strings.TrimSpace(value.Brand)
|
||||
version := strings.TrimSpace(value.Version)
|
||||
if brand == "" || version == "" {
|
||||
continue
|
||||
}
|
||||
out = append(out, &proto.EmulationUserAgentBrandVersion{
|
||||
Brand: brand,
|
||||
Version: version,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func evalPatchScript(page *rod.Page, langs []string, width, height int) error {
|
||||
langsJSON, err := json.Marshal(langs)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal navigator languages: %w", err)
|
||||
}
|
||||
args := fmt.Sprintf("const __langs = %s;\nconst __w = %d;\nconst __h = %d;\n",
|
||||
string(langsJSON), width, height)
|
||||
_, err = page.EvalOnNewDocument(args + string(browserprofile.PatchJS))
|
||||
if err != nil {
|
||||
return fmt.Errorf("eval patch script: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Navigate connects to Chromium, creates a page, applies a coherent profile and
|
||||
// proxy auth, then navigates to URL. It returns an initialized page ready for
|
||||
// selector queries, or an error when browser setup/navigation fails.
|
||||
func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
|
||||
@@ -340,7 +630,7 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
|
||||
|
||||
WithRequest(ctx).WithField("url", URL).Debug("Navigate")
|
||||
|
||||
browser, ua, err := b.ensureConnectedBrowser(ctx, false)
|
||||
browser, err := b.ensureConnectedBrowser(ctx, false)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("browser connect failed: %w", err)
|
||||
}
|
||||
@@ -348,7 +638,7 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
|
||||
page, browserContextID, err := createIsolatedPage(browser)
|
||||
if err != nil {
|
||||
// Single-shot reconnect for stale websocket sessions.
|
||||
browser, ua, err = b.ensureConnectedBrowser(ctx, true)
|
||||
browser, err = b.ensureConnectedBrowser(ctx, true)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create isolated page failed, reconnect also failed: %w", err)
|
||||
}
|
||||
@@ -379,33 +669,10 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
|
||||
defer cancelProxyAuth()
|
||||
}
|
||||
|
||||
if b.UseStealth {
|
||||
if _, err := page.EvalOnNewDocument(stealth.JS); err != nil {
|
||||
closeOnErr()
|
||||
return nil, fmt.Errorf("create stealth page failed: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := page.Emulate(devices.Device{
|
||||
AcceptLanguage: b.LanguageCode,
|
||||
UserAgent: ua,
|
||||
}); err != nil {
|
||||
profile, laneKey := b.laneProfile(ctx, browser)
|
||||
if err := applyProfile(page, profile); err != nil {
|
||||
closeOnErr()
|
||||
return nil, fmt.Errorf("emulate page failed: %w", err)
|
||||
}
|
||||
|
||||
if !b.UseStealth {
|
||||
if err := (proto.EmulationSetDeviceMetricsOverride{
|
||||
Width: 1920,
|
||||
Height: 1080,
|
||||
DeviceScaleFactor: 1,
|
||||
Mobile: false,
|
||||
ScreenWidth: &[]int{1920}[0],
|
||||
ScreenHeight: &[]int{1080}[0],
|
||||
}).Call(page); err != nil {
|
||||
closeOnErr()
|
||||
return nil, fmt.Errorf("set device metrics failed: %w", err)
|
||||
}
|
||||
return nil, fmt.Errorf("apply profile %s (%s) failed: %w", profile.ID, laneKey, err)
|
||||
}
|
||||
|
||||
page = page.Context(ctx)
|
||||
@@ -464,6 +731,7 @@ func (b *Browser) Close() error {
|
||||
}
|
||||
|
||||
state.browser = nil
|
||||
state.laneProfiles = nil
|
||||
if err := browser.Close(); err != nil && !isBrowserClosedError(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
73
core/browser/patch.js
Normal file
73
core/browser/patch.js
Normal file
@@ -0,0 +1,73 @@
|
||||
// Stealth patches injected via EvalOnNewDocument.
|
||||
// Arguments are injected as a leading const block by the Go caller:
|
||||
// const __langs = [...]; // navigator_langs from profile
|
||||
// const __w = 1920; // viewport width
|
||||
// const __h = 1080; // viewport height
|
||||
(() => {
|
||||
'use strict';
|
||||
|
||||
// --- navigator.language / navigator.languages ---
|
||||
//
|
||||
// CDP Network.setUserAgentOverride(acceptLanguage) sets the HTTP header but
|
||||
// does NOT update navigator.language / navigator.languages in JS. Those are
|
||||
// read from the browser profile at context creation. On Linux headless they
|
||||
// reflect the system ICU locale (usually "en-US" regardless of profile).
|
||||
//
|
||||
// We patch them here. The key to being undetectable: define with a getter
|
||||
// first (requires configurable:true), then immediately lock the descriptor
|
||||
// back to configurable:false so it looks exactly like a native property.
|
||||
const primary = __langs.length ? __langs[0] : 'en-US';
|
||||
|
||||
const sealGetter = (target, prop, fn) => {
|
||||
try {
|
||||
Object.defineProperty(target, prop, {
|
||||
get: fn,
|
||||
set: undefined,
|
||||
enumerable: true,
|
||||
configurable: true, // must be true to set a getter
|
||||
});
|
||||
Object.defineProperty(target, prop, {
|
||||
configurable: false, // seal: now indistinguishable from native
|
||||
});
|
||||
} catch (_) {}
|
||||
};
|
||||
|
||||
const patchLangs = (target) => {
|
||||
if (!target) return;
|
||||
sealGetter(target, 'language', () => primary);
|
||||
sealGetter(target, 'languages', () => Object.freeze(__langs.slice()));
|
||||
};
|
||||
|
||||
// Patch the navigator instance (Linux headless stores own-props here)
|
||||
// and Navigator.prototype (other platforms / future Chrome versions).
|
||||
patchLangs(navigator);
|
||||
patchLangs(Object.getPrototypeOf(navigator));
|
||||
if (typeof WorkerNavigator !== 'undefined') {
|
||||
patchLangs(WorkerNavigator.prototype);
|
||||
}
|
||||
|
||||
// --- screen dimensions ---
|
||||
//
|
||||
// EmulationSetDeviceMetricsOverride sets the CSS viewport but leaves
|
||||
// window.screen.* at headless defaults. Checkers compare screen size
|
||||
// against viewport and flag the mismatch as automation.
|
||||
//
|
||||
// Screen properties live on Screen.prototype as non-configurable getters.
|
||||
// Patching the prototype makes them look native.
|
||||
const patchScreen = () => {
|
||||
const proto = typeof Screen !== 'undefined' ? Screen.prototype : null;
|
||||
if (!proto) return;
|
||||
sealGetter(proto, 'width', () => __w);
|
||||
sealGetter(proto, 'height', () => __h);
|
||||
sealGetter(proto, 'availWidth', () => __w);
|
||||
sealGetter(proto, 'availHeight', () => __h);
|
||||
sealGetter(proto, 'availLeft', () => 0);
|
||||
sealGetter(proto, 'availTop', () => 0);
|
||||
};
|
||||
patchScreen();
|
||||
|
||||
// window.outerWidth/Height are own configurable properties; Chrome normally
|
||||
// sets them to match the OS window size. In headless they are 0.
|
||||
sealGetter(window, 'outerWidth', () => __w);
|
||||
sealGetter(window, 'outerHeight', () => __h);
|
||||
})();
|
||||
294
core/browser/profile.go
Normal file
294
core/browser/profile.go
Normal file
@@ -0,0 +1,294 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type BrandVersion struct {
|
||||
Brand string `json:"brand"`
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
type Viewport struct {
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
}
|
||||
|
||||
type Profile struct {
|
||||
ID string `json:"id"`
|
||||
UserAgent string `json:"user_agent"`
|
||||
UACHBrands []BrandVersion `json:"uach_brands"`
|
||||
UACHFullVerList []BrandVersion `json:"uach_full_version_list"`
|
||||
Platform string `json:"platform"`
|
||||
PlatformVersion string `json:"platform_version"`
|
||||
Architecture string `json:"architecture"`
|
||||
Bitness string `json:"bitness"`
|
||||
Mobile bool `json:"mobile"`
|
||||
AcceptLanguage string `json:"accept_language"`
|
||||
NavigatorLangs []string `json:"navigator_langs"`
|
||||
Locale string `json:"locale"`
|
||||
Timezone string `json:"timezone"`
|
||||
Viewport Viewport `json:"viewport"`
|
||||
}
|
||||
|
||||
type catalogConfig struct {
|
||||
Profiles []Profile `json:"profiles"`
|
||||
LaneProfileIDs map[string]string `json:"lane_profile_ids"`
|
||||
DefaultRegionByEngine map[string]string `json:"default_region_by_engine"`
|
||||
}
|
||||
|
||||
const (
|
||||
ProfileChromeWinUS = "chrome-win-us"
|
||||
ProfileChromeWinRU = "chrome-win-ru"
|
||||
ProfileChromeMacUS = "chrome-macos-us"
|
||||
ProfileChromeLinuxUS = "chrome-linux-us"
|
||||
ProfileChromeLinuxRU = "chrome-linux-ru"
|
||||
)
|
||||
|
||||
//go:embed profiles.json
|
||||
var defaultProfilesJSON []byte
|
||||
|
||||
//go:embed patch.js
|
||||
var PatchJS []byte
|
||||
|
||||
var profileCatalogMu sync.RWMutex
|
||||
|
||||
var catalog = map[string]Profile{}
|
||||
|
||||
var laneProfileIDs = map[string]string{}
|
||||
|
||||
var defaultRegionByEngine = map[string]string{}
|
||||
|
||||
func init() {
|
||||
if err := loadProfilesFromJSONBytes(defaultProfilesJSON); err != nil {
|
||||
panic(fmt.Sprintf("load embedded browser profiles: %v", err))
|
||||
}
|
||||
}
|
||||
|
||||
func LoadProfilesFromJSON(path string) error {
|
||||
path = strings.TrimSpace(path)
|
||||
if path == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read profiles json %q: %w", path, err)
|
||||
}
|
||||
|
||||
if err := loadProfilesFromJSONBytes(data); err != nil {
|
||||
return fmt.Errorf("parse profiles json %q: %w", path, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadProfilesFromJSONBytes(data []byte) error {
|
||||
var cfg catalogConfig
|
||||
if err := json.Unmarshal(data, &cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(cfg.Profiles) == 0 {
|
||||
return fmt.Errorf("profiles list is empty")
|
||||
}
|
||||
|
||||
nextCatalog := make(map[string]Profile, len(cfg.Profiles))
|
||||
for i, profile := range cfg.Profiles {
|
||||
profile.ID = strings.TrimSpace(profile.ID)
|
||||
if profile.ID == "" {
|
||||
return fmt.Errorf("profiles[%d].id is empty", i)
|
||||
}
|
||||
if strings.TrimSpace(profile.UserAgent) == "" {
|
||||
return fmt.Errorf("profiles[%d].user_agent is empty", i)
|
||||
}
|
||||
if _, exists := nextCatalog[profile.ID]; exists {
|
||||
return fmt.Errorf("duplicate profile id %q", profile.ID)
|
||||
}
|
||||
nextCatalog[profile.ID] = profile
|
||||
}
|
||||
|
||||
nextLaneProfileIDs := make(map[string]string, len(cfg.LaneProfileIDs))
|
||||
for rawLaneKey, profileID := range cfg.LaneProfileIDs {
|
||||
laneKey, err := normalizeLaneKey(rawLaneKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
profileID = strings.TrimSpace(profileID)
|
||||
if profileID == "" {
|
||||
return fmt.Errorf("lane profile id for %q is empty", laneKey)
|
||||
}
|
||||
if _, exists := nextCatalog[profileID]; !exists {
|
||||
return fmt.Errorf("lane %q references unknown profile id %q", laneKey, profileID)
|
||||
}
|
||||
nextLaneProfileIDs[laneKey] = profileID
|
||||
}
|
||||
|
||||
nextDefaultRegionByEngine := map[string]string{
|
||||
"yandex": "ru",
|
||||
}
|
||||
for engine, region := range cfg.DefaultRegionByEngine {
|
||||
engine = NormalizeEngine(engine)
|
||||
if engine == "" {
|
||||
return fmt.Errorf("default_region_by_engine contains empty engine key")
|
||||
}
|
||||
nextDefaultRegionByEngine[engine] = normalizeConfiguredRegion(region)
|
||||
}
|
||||
|
||||
profileCatalogMu.Lock()
|
||||
catalog = nextCatalog
|
||||
laneProfileIDs = nextLaneProfileIDs
|
||||
defaultRegionByEngine = nextDefaultRegionByEngine
|
||||
profileCatalogMu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeLaneKey(value string) (string, error) {
|
||||
value = strings.TrimSpace(value)
|
||||
parts := strings.SplitN(value, ":", 2)
|
||||
if len(parts) != 2 {
|
||||
return "", fmt.Errorf("invalid lane key %q, expected engine:region", value)
|
||||
}
|
||||
engine := NormalizeEngine(parts[0])
|
||||
if engine == "" {
|
||||
return "", fmt.Errorf("invalid lane key %q, engine is empty", value)
|
||||
}
|
||||
region := normalizeConfiguredRegion(parts[1])
|
||||
return engine + ":" + region, nil
|
||||
}
|
||||
|
||||
func normalizeConfiguredRegion(region string) string {
|
||||
region = strings.TrimSpace(region)
|
||||
if region == "" {
|
||||
return "us"
|
||||
}
|
||||
return NormalizeRegion(region)
|
||||
}
|
||||
|
||||
func Catalog() []Profile {
|
||||
profileCatalogMu.RLock()
|
||||
defer profileCatalogMu.RUnlock()
|
||||
|
||||
out := make([]Profile, 0, len(catalog))
|
||||
for _, profile := range catalog {
|
||||
out = append(out, profile)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func SelectProfile(engine string, region string) Profile {
|
||||
engine = NormalizeEngine(engine)
|
||||
region = NormalizeRegion(region)
|
||||
if region == "" {
|
||||
region = DefaultRegionForEngine(engine)
|
||||
}
|
||||
laneKey := engine + ":" + region
|
||||
|
||||
profileCatalogMu.RLock()
|
||||
profileID, ok := laneProfileIDs[laneKey]
|
||||
profileCatalogMu.RUnlock()
|
||||
if ok {
|
||||
return profileByID(profileID)
|
||||
}
|
||||
return profileByID(defaultProfileID(region))
|
||||
}
|
||||
|
||||
func LaneKey(engine string, region string) string {
|
||||
engine = NormalizeEngine(engine)
|
||||
if engine == "" {
|
||||
engine = "unknown"
|
||||
}
|
||||
|
||||
region = NormalizeRegion(region)
|
||||
if region == "" {
|
||||
region = DefaultRegionForEngine(engine)
|
||||
}
|
||||
|
||||
return engine + ":" + region
|
||||
}
|
||||
|
||||
func DefaultRegionForEngine(engine string) string {
|
||||
engine = NormalizeEngine(engine)
|
||||
|
||||
profileCatalogMu.RLock()
|
||||
defer profileCatalogMu.RUnlock()
|
||||
|
||||
if region, ok := defaultRegionByEngine[engine]; ok {
|
||||
return region
|
||||
}
|
||||
return "us"
|
||||
}
|
||||
|
||||
func NormalizeEngine(engine string) string {
|
||||
return strings.ToLower(strings.TrimSpace(engine))
|
||||
}
|
||||
|
||||
func NormalizeRegion(region string) string {
|
||||
region = strings.TrimSpace(strings.ToLower(region))
|
||||
if region == "" {
|
||||
return ""
|
||||
}
|
||||
|
||||
if idx := strings.Index(region, ","); idx >= 0 {
|
||||
region = region[:idx]
|
||||
}
|
||||
if idx := strings.Index(region, ";"); idx >= 0 {
|
||||
region = region[:idx]
|
||||
}
|
||||
|
||||
region = strings.ReplaceAll(region, "_", "-")
|
||||
if idx := strings.Index(region, "-"); idx >= 0 {
|
||||
region = region[:idx]
|
||||
}
|
||||
|
||||
switch region {
|
||||
case "ru", "be", "kz", "ky":
|
||||
return "ru"
|
||||
default:
|
||||
return "us"
|
||||
}
|
||||
}
|
||||
|
||||
func profileByID(profileID string) Profile {
|
||||
profileCatalogMu.RLock()
|
||||
defer profileCatalogMu.RUnlock()
|
||||
|
||||
if profile, ok := catalog[profileID]; ok {
|
||||
return profile
|
||||
}
|
||||
if fallback, ok := catalog[defaultProfileID("us")]; ok {
|
||||
return fallback
|
||||
}
|
||||
for _, profile := range catalog {
|
||||
return profile
|
||||
}
|
||||
return Profile{}
|
||||
}
|
||||
|
||||
func defaultProfileID(region string) string {
|
||||
region = NormalizeRegion(region)
|
||||
if region == "" {
|
||||
region = "us"
|
||||
}
|
||||
|
||||
switch runtime.GOOS {
|
||||
case "windows":
|
||||
if region == "ru" {
|
||||
return ProfileChromeWinRU
|
||||
}
|
||||
return ProfileChromeWinUS
|
||||
case "darwin":
|
||||
return ProfileChromeMacUS
|
||||
default:
|
||||
if region == "ru" {
|
||||
return ProfileChromeLinuxRU
|
||||
}
|
||||
return ProfileChromeLinuxUS
|
||||
}
|
||||
}
|
||||
200
core/browser/profile_coherence_test.go
Normal file
200
core/browser/profile_coherence_test.go
Normal file
@@ -0,0 +1,200 @@
|
||||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package browser_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
"github.com/karust/openserp/core"
|
||||
browserprofile "github.com/karust/openserp/core/browser"
|
||||
"github.com/karust/openserp/testutil"
|
||||
)
|
||||
|
||||
func TestProfileCoherence(t *testing.T) {
|
||||
testutil.RequireIntegration(t)
|
||||
|
||||
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_, _ = w.Write([]byte(`<!doctype html><html><head><meta charset="utf-8"><title>coherence</title></head><body>ok</body></html>`))
|
||||
}))
|
||||
defer fixture.Close()
|
||||
|
||||
browser, err := core.NewBrowser(core.BrowserOpts{
|
||||
IsHeadless: true,
|
||||
IsLeakless: false,
|
||||
Timeout: 20 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("create browser: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := browser.Close(); closeErr != nil {
|
||||
t.Fatalf("close browser: %v", closeErr)
|
||||
}
|
||||
}()
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
engine string
|
||||
region string
|
||||
}{
|
||||
{
|
||||
name: "windows lane",
|
||||
engine: "google",
|
||||
region: "ru",
|
||||
},
|
||||
{
|
||||
name: "mac lane",
|
||||
engine: "bing",
|
||||
region: "en-US",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
expected := browserprofile.SelectProfile(tc.engine, tc.region)
|
||||
ctx := core.WithEngine(context.Background(), tc.engine)
|
||||
ctx = core.WithProfileRegion(ctx, tc.region)
|
||||
|
||||
page, err := browser.Navigate(ctx, fixture.URL)
|
||||
if err != nil {
|
||||
t.Fatalf("navigate fixture: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
if closeErr := core.ClosePageWithTimeout(context.Background(), page, time.Second); closeErr != nil {
|
||||
t.Fatalf("close page: %v", closeErr)
|
||||
}
|
||||
}()
|
||||
|
||||
got, err := browserProfileSurface(page)
|
||||
if err != nil {
|
||||
t.Fatalf("collect profile surfaces: %v", err)
|
||||
}
|
||||
|
||||
if got.UserAgent != expected.UserAgent {
|
||||
t.Fatalf("navigator.userAgent mismatch:\nexpected: %s\nactual: %s", expected.UserAgent, got.UserAgent)
|
||||
}
|
||||
if got.Platform != expected.Platform {
|
||||
t.Fatalf("navigator.userAgentData.platform mismatch: expected %q got %q", expected.Platform, got.Platform)
|
||||
}
|
||||
if got.NavigatorPlatform != expectedNavigatorPlatform(expected.Platform) {
|
||||
t.Fatalf("navigator.platform mismatch: expected %q got %q", expectedNavigatorPlatform(expected.Platform), got.NavigatorPlatform)
|
||||
}
|
||||
if got.Locale != expected.Locale {
|
||||
t.Fatalf("Intl locale mismatch: expected %q got %q", expected.Locale, got.Locale)
|
||||
}
|
||||
if got.Timezone != expected.Timezone {
|
||||
t.Fatalf("Intl timezone mismatch: expected %q got %q", expected.Timezone, got.Timezone)
|
||||
}
|
||||
if len(got.NavigatorLanguages) == 0 {
|
||||
t.Fatal("navigator.languages is empty")
|
||||
}
|
||||
if got.NavigatorLanguages[0] != expected.NavigatorLangs[0] {
|
||||
t.Fatalf("navigator.languages[0] mismatch: expected %q got %q", expected.NavigatorLangs[0], got.NavigatorLanguages[0])
|
||||
}
|
||||
if got.WebdriverType != "undefined" {
|
||||
t.Fatalf("navigator.webdriver expected undefined, got %q", got.WebdriverType)
|
||||
}
|
||||
if got.WebdriverOwnPropPresent {
|
||||
t.Fatal("navigator own property 'webdriver' should not be present")
|
||||
}
|
||||
if got.WorkerUserAgent != got.UserAgent {
|
||||
t.Fatalf("worker userAgent mismatch: main %q worker %q", got.UserAgent, got.WorkerUserAgent)
|
||||
}
|
||||
if got.WorkerPlatform != got.NavigatorPlatform {
|
||||
t.Fatalf("worker platform mismatch: main %q worker %q", got.NavigatorPlatform, got.WorkerPlatform)
|
||||
}
|
||||
if len(got.WorkerNavigatorLangs) == 0 {
|
||||
t.Fatal("worker navigator.languages is empty")
|
||||
}
|
||||
if got.WorkerNavigatorLangs[0] != got.NavigatorLanguages[0] {
|
||||
t.Fatalf("worker navigator.languages[0] mismatch: main %q worker %q", got.NavigatorLanguages[0], got.WorkerNavigatorLangs[0])
|
||||
}
|
||||
if got.WorkerTimezone != got.Timezone {
|
||||
t.Fatalf("worker timezone mismatch: main %q worker %q", got.Timezone, got.WorkerTimezone)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type profileSurface struct {
|
||||
UserAgent string `json:"userAgent"`
|
||||
Platform string `json:"platform"`
|
||||
NavigatorPlatform string `json:"navigatorPlatform"`
|
||||
NavigatorLanguages []string `json:"navigatorLanguages"`
|
||||
Timezone string `json:"timezone"`
|
||||
Locale string `json:"locale"`
|
||||
WebdriverType string `json:"webdriverType"`
|
||||
WebdriverOwnPropPresent bool `json:"webdriverOwnPropPresent"`
|
||||
WorkerUserAgent string `json:"workerUserAgent"`
|
||||
WorkerPlatform string `json:"workerPlatform"`
|
||||
WorkerNavigatorLangs []string `json:"workerNavigatorLangs"`
|
||||
WorkerTimezone string `json:"workerTimezone"`
|
||||
}
|
||||
|
||||
func browserProfileSurface(page *rod.Page) (profileSurface, error) {
|
||||
result, err := page.Eval(`async () => {
|
||||
const workerData = await new Promise((resolve) => {
|
||||
try {
|
||||
const source = "self.onmessage=()=>{self.postMessage({userAgent:self.navigator.userAgent||'',platform:self.navigator.platform||'',navigatorLanguages:Array.from(self.navigator.languages||[]),timezone:Intl.DateTimeFormat().resolvedOptions().timeZone||''});};";
|
||||
const blob = new Blob([source], { type: 'application/javascript' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const worker = new Worker(url);
|
||||
worker.onmessage = (event) => {
|
||||
resolve(event.data || {});
|
||||
worker.terminate();
|
||||
URL.revokeObjectURL(url);
|
||||
};
|
||||
worker.onerror = () => {
|
||||
resolve({});
|
||||
worker.terminate();
|
||||
URL.revokeObjectURL(url);
|
||||
};
|
||||
worker.postMessage('run');
|
||||
} catch (_) {
|
||||
resolve({});
|
||||
}
|
||||
});
|
||||
return {
|
||||
userAgent: navigator.userAgent || "",
|
||||
platform: navigator.userAgentData ? (navigator.userAgentData.platform || "") : "",
|
||||
navigatorPlatform: navigator.platform || "",
|
||||
navigatorLanguages: Array.from(navigator.languages || []),
|
||||
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || "",
|
||||
locale: Intl.DateTimeFormat().resolvedOptions().locale || "",
|
||||
webdriverType: typeof navigator.webdriver,
|
||||
webdriverOwnPropPresent: Object.getOwnPropertyNames(navigator).includes('webdriver'),
|
||||
workerUserAgent: workerData.userAgent || "",
|
||||
workerPlatform: workerData.platform || "",
|
||||
workerNavigatorLangs: Array.from(workerData.navigatorLanguages || []),
|
||||
workerTimezone: workerData.timezone || "",
|
||||
};
|
||||
}`)
|
||||
if err != nil {
|
||||
return profileSurface{}, err
|
||||
}
|
||||
|
||||
var out profileSurface
|
||||
if err := result.Value.Unmarshal(&out); err != nil {
|
||||
return profileSurface{}, fmt.Errorf("decode eval payload: %w", err)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func expectedNavigatorPlatform(platform string) string {
|
||||
switch platform {
|
||||
case "Windows":
|
||||
return "Win32"
|
||||
case "macOS":
|
||||
return "MacIntel"
|
||||
default:
|
||||
return "Linux x86_64"
|
||||
}
|
||||
}
|
||||
169
core/browser/profile_test.go
Normal file
169
core/browser/profile_test.go
Normal file
@@ -0,0 +1,169 @@
|
||||
package browser
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSelectProfile(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
engine string
|
||||
region string
|
||||
wantLocale string
|
||||
wantTimezone string
|
||||
}{
|
||||
{
|
||||
name: "google ru lane",
|
||||
engine: "google",
|
||||
region: "ru",
|
||||
wantLocale: "ru-RU",
|
||||
wantTimezone: "Europe/Moscow",
|
||||
},
|
||||
{
|
||||
name: "yandex defaults to ru",
|
||||
engine: "yandex",
|
||||
region: "",
|
||||
wantLocale: "ru-RU",
|
||||
wantTimezone: "Europe/Moscow",
|
||||
},
|
||||
{
|
||||
name: "google default lane uses us profile",
|
||||
engine: "google",
|
||||
region: "en",
|
||||
wantLocale: "en-US",
|
||||
wantTimezone: "America/New_York",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
profile := SelectProfile(tt.engine, tt.region)
|
||||
if profile.ID == "" {
|
||||
t.Fatal("expected non-empty profile ID")
|
||||
}
|
||||
if profile.Locale != tt.wantLocale {
|
||||
t.Fatalf("expected locale %q, got %q", tt.wantLocale, profile.Locale)
|
||||
}
|
||||
if profile.Timezone != tt.wantTimezone {
|
||||
t.Fatalf("expected timezone %q, got %q", tt.wantTimezone, profile.Timezone)
|
||||
}
|
||||
if profile.Platform == "" {
|
||||
t.Fatal("expected non-empty platform")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRegion(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{input: "", want: ""},
|
||||
{input: "ru", want: "ru"},
|
||||
{input: "RU", want: "ru"},
|
||||
{input: "ru-RU", want: "ru"},
|
||||
{input: "ru_RU", want: "ru"},
|
||||
{input: "ru-RU,ru;q=0.9", want: "ru"},
|
||||
{input: "en-US,en;q=0.9", want: "us"},
|
||||
{input: "de", want: "us"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
if got := NormalizeRegion(tt.input); got != tt.want {
|
||||
t.Fatalf("NormalizeRegion(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadProfilesFromJSON(t *testing.T) {
|
||||
originalCatalog, originalLaneProfiles, originalDefaultRegions := snapshotProfileState()
|
||||
t.Cleanup(func() {
|
||||
restoreProfileState(originalCatalog, originalLaneProfiles, originalDefaultRegions)
|
||||
})
|
||||
|
||||
path := filepath.Join(t.TempDir(), "profiles.json")
|
||||
payload := `{
|
||||
"profiles": [
|
||||
{
|
||||
"id": "custom-ru",
|
||||
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
|
||||
"uach_brands": [
|
||||
{"brand": "Chromium", "version": "136"}
|
||||
],
|
||||
"uach_full_version_list": [
|
||||
{"brand": "Chromium", "version": "136.0.0.0"}
|
||||
],
|
||||
"platform": "Linux",
|
||||
"platform_version": "6.0.0",
|
||||
"architecture": "x86",
|
||||
"mobile": false,
|
||||
"accept_language": "ru-RU,ru;q=0.9",
|
||||
"navigator_langs": ["ru-RU"],
|
||||
"locale": "ru-RU",
|
||||
"timezone": "Europe/Moscow",
|
||||
"viewport": {"width": 1920, "height": 1080}
|
||||
}
|
||||
],
|
||||
"lane_profile_ids": {
|
||||
"google:ru-RU": "custom-ru"
|
||||
},
|
||||
"default_region_by_engine": {
|
||||
"google": "ru-RU"
|
||||
}
|
||||
}`
|
||||
if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
|
||||
t.Fatalf("write profiles json: %v", err)
|
||||
}
|
||||
|
||||
if err := LoadProfilesFromJSON(path); err != nil {
|
||||
t.Fatalf("load profiles json: %v", err)
|
||||
}
|
||||
|
||||
if got := DefaultRegionForEngine("google"); got != "ru" {
|
||||
t.Fatalf("expected google default region ru, got %q", got)
|
||||
}
|
||||
|
||||
profile := SelectProfile("google", "")
|
||||
if profile.ID != "custom-ru" {
|
||||
t.Fatalf("expected custom profile id custom-ru, got %q", profile.ID)
|
||||
}
|
||||
if profile.Timezone != "Europe/Moscow" {
|
||||
t.Fatalf("expected timezone Europe/Moscow, got %q", profile.Timezone)
|
||||
}
|
||||
}
|
||||
|
||||
func snapshotProfileState() (map[string]Profile, map[string]string, map[string]string) {
|
||||
profileCatalogMu.RLock()
|
||||
defer profileCatalogMu.RUnlock()
|
||||
|
||||
catalogCopy := make(map[string]Profile, len(catalog))
|
||||
for k, v := range catalog {
|
||||
catalogCopy[k] = v
|
||||
}
|
||||
|
||||
laneProfilesCopy := make(map[string]string, len(laneProfileIDs))
|
||||
for k, v := range laneProfileIDs {
|
||||
laneProfilesCopy[k] = v
|
||||
}
|
||||
|
||||
defaultRegionsCopy := make(map[string]string, len(defaultRegionByEngine))
|
||||
for k, v := range defaultRegionByEngine {
|
||||
defaultRegionsCopy[k] = v
|
||||
}
|
||||
|
||||
return catalogCopy, laneProfilesCopy, defaultRegionsCopy
|
||||
}
|
||||
|
||||
func restoreProfileState(catalogState map[string]Profile, laneProfiles map[string]string, defaultRegions map[string]string) {
|
||||
profileCatalogMu.Lock()
|
||||
defer profileCatalogMu.Unlock()
|
||||
|
||||
catalog = catalogState
|
||||
laneProfileIDs = laneProfiles
|
||||
defaultRegionByEngine = defaultRegions
|
||||
}
|
||||
243
core/browser/profiles.json
Normal file
243
core/browser/profiles.json
Normal file
@@ -0,0 +1,243 @@
|
||||
{
|
||||
"profiles": [
|
||||
{
|
||||
"id": "chrome-win-us",
|
||||
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
|
||||
"uach_brands": [
|
||||
{
|
||||
"brand": "Not_A Brand",
|
||||
"version": "24"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "136"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "136"
|
||||
}
|
||||
],
|
||||
"uach_full_version_list": [
|
||||
{
|
||||
"brand": "Not_A Brand",
|
||||
"version": "24.0.0.0"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "136.0.0.0"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "136.0.0.0"
|
||||
}
|
||||
],
|
||||
"platform": "Windows",
|
||||
"platform_version": "15.0.0",
|
||||
"architecture": "x86",
|
||||
"bitness": "64",
|
||||
"mobile": false,
|
||||
"accept_language": "en-US,en;q=0.9",
|
||||
"navigator_langs": [
|
||||
"en-US"
|
||||
],
|
||||
"locale": "en-US",
|
||||
"timezone": "America/New_York",
|
||||
"viewport": {
|
||||
"width": 1920,
|
||||
"height": 1080
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "chrome-win-ru",
|
||||
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
|
||||
"uach_brands": [
|
||||
{
|
||||
"brand": "Not_A Brand",
|
||||
"version": "24"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "136"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "136"
|
||||
}
|
||||
],
|
||||
"uach_full_version_list": [
|
||||
{
|
||||
"brand": "Not_A Brand",
|
||||
"version": "24.0.0.0"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "136.0.0.0"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "136.0.0.0"
|
||||
}
|
||||
],
|
||||
"platform": "Windows",
|
||||
"platform_version": "15.0.0",
|
||||
"architecture": "x86",
|
||||
"bitness": "64",
|
||||
"mobile": false,
|
||||
"accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"navigator_langs": [
|
||||
"ru-RU"
|
||||
],
|
||||
"locale": "ru-RU",
|
||||
"timezone": "Europe/Moscow",
|
||||
"viewport": {
|
||||
"width": 1920,
|
||||
"height": 1080
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "chrome-macos-us",
|
||||
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
|
||||
"uach_brands": [
|
||||
{
|
||||
"brand": "Not_A Brand",
|
||||
"version": "24"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "136"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "136"
|
||||
}
|
||||
],
|
||||
"uach_full_version_list": [
|
||||
{
|
||||
"brand": "Not_A Brand",
|
||||
"version": "24.0.0.0"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "136.0.0.0"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "136.0.0.0"
|
||||
}
|
||||
],
|
||||
"platform": "macOS",
|
||||
"platform_version": "14.0.0",
|
||||
"architecture": "x86",
|
||||
"bitness": "64",
|
||||
"mobile": false,
|
||||
"accept_language": "en-US,en;q=0.9",
|
||||
"navigator_langs": [
|
||||
"en-US"
|
||||
],
|
||||
"locale": "en-US",
|
||||
"timezone": "America/Los_Angeles",
|
||||
"viewport": {
|
||||
"width": 1680,
|
||||
"height": 1050
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "chrome-linux-us",
|
||||
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
|
||||
"uach_brands": [
|
||||
{
|
||||
"brand": "Not_A Brand",
|
||||
"version": "24"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "136"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "136"
|
||||
}
|
||||
],
|
||||
"uach_full_version_list": [
|
||||
{
|
||||
"brand": "Not_A Brand",
|
||||
"version": "24.0.0.0"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "136.0.0.0"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "136.0.0.0"
|
||||
}
|
||||
],
|
||||
"platform": "Linux",
|
||||
"platform_version": "6.0.0",
|
||||
"architecture": "x86",
|
||||
"bitness": "64",
|
||||
"mobile": false,
|
||||
"accept_language": "en-US,en;q=0.9",
|
||||
"navigator_langs": [
|
||||
"en-US"
|
||||
],
|
||||
"locale": "en-US",
|
||||
"timezone": "America/New_York",
|
||||
"viewport": {
|
||||
"width": 1920,
|
||||
"height": 1080
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "chrome-linux-ru",
|
||||
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
|
||||
"uach_brands": [
|
||||
{
|
||||
"brand": "Not_A Brand",
|
||||
"version": "24"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "136"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "136"
|
||||
}
|
||||
],
|
||||
"uach_full_version_list": [
|
||||
{
|
||||
"brand": "Not_A Brand",
|
||||
"version": "24.0.0.0"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "136.0.0.0"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "136.0.0.0"
|
||||
}
|
||||
],
|
||||
"platform": "Linux",
|
||||
"platform_version": "6.0.0",
|
||||
"architecture": "x86",
|
||||
"bitness": "64",
|
||||
"mobile": false,
|
||||
"accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"navigator_langs": [
|
||||
"ru-RU"
|
||||
],
|
||||
"locale": "ru-RU",
|
||||
"timezone": "Europe/Moscow",
|
||||
"viewport": {
|
||||
"width": 1920,
|
||||
"height": 1080
|
||||
}
|
||||
}
|
||||
],
|
||||
"lane_profile_ids": {},
|
||||
"default_region_by_engine": {
|
||||
"yandex": "ru"
|
||||
}
|
||||
}
|
||||
@@ -113,15 +113,13 @@ func TestFingerprintDetectors(t *testing.T) {
|
||||
criticalFailures := make([]string, 0)
|
||||
|
||||
for _, detector := range detectors.All() {
|
||||
for _, useStealth := range []bool{false, true} {
|
||||
report := runFingerprintDetector(t, detector, useStealth)
|
||||
key := fmt.Sprintf("%s_%s", detector.Name(), stealthModeLabel(useStealth))
|
||||
reports[key] = report
|
||||
report := runFingerprintDetector(t, detector)
|
||||
key := detector.Name()
|
||||
reports[key] = report
|
||||
|
||||
if len(report.Summary.Critical) > 0 {
|
||||
for _, critical := range report.Summary.Critical {
|
||||
criticalFailures = append(criticalFailures, fmt.Sprintf("%s:%s", key, critical))
|
||||
}
|
||||
if len(report.Summary.Critical) > 0 {
|
||||
for _, critical := range report.Summary.Critical {
|
||||
criticalFailures = append(criticalFailures, fmt.Sprintf("%s:%s", key, critical))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -142,31 +140,28 @@ func TestFingerprintDetectors(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func runFingerprintDetector(t *testing.T, detector fpcheck.Detector, useStealth bool) fpcheck.Report {
|
||||
func runFingerprintDetector(t *testing.T, detector fpcheck.Detector) fpcheck.Report {
|
||||
t.Helper()
|
||||
|
||||
label := stealthModeLabel(useStealth)
|
||||
opts := BrowserOpts{
|
||||
IsHeadless: true,
|
||||
IsLeakless: false,
|
||||
Timeout: 20 * time.Second,
|
||||
UseStealth: useStealth,
|
||||
}
|
||||
browser, err := NewBrowser(opts)
|
||||
if err != nil {
|
||||
t.Fatalf("create browser (%s): %v", label, err)
|
||||
t.Fatalf("create browser: %v", err)
|
||||
}
|
||||
defer closeTestBrowser(t, browser)
|
||||
|
||||
report, err := fpcheck.Run(context.Background(), browser, detector, useStealth, botFingerprintArtifactDir)
|
||||
report, err := fpcheck.Run(context.Background(), browser, detector, botFingerprintArtifactDir)
|
||||
if err != nil {
|
||||
t.Fatalf("run detector %s (%s): %v", detector.Name(), label, err)
|
||||
t.Fatalf("run detector %s: %v", detector.Name(), err)
|
||||
}
|
||||
|
||||
t.Logf(
|
||||
"Fingerprint %s (%s): passed=%d failed=%d critical=%d",
|
||||
"Fingerprint %s: passed=%d failed=%d critical=%d",
|
||||
detector.Name(),
|
||||
label,
|
||||
report.Summary.Passed,
|
||||
report.Summary.Failed,
|
||||
len(report.Summary.Critical),
|
||||
@@ -191,13 +186,6 @@ func writeFingerprintReport(path string, reports map[string]fpcheck.Report) erro
|
||||
return nil
|
||||
}
|
||||
|
||||
func stealthModeLabel(useStealth bool) string {
|
||||
if useStealth {
|
||||
return fpcheck.ModeStealthOn
|
||||
}
|
||||
return fpcheck.ModeStealthOff
|
||||
}
|
||||
|
||||
func closeTestBrowser(t *testing.T, browser *Browser) {
|
||||
t.Helper()
|
||||
if browser == nil {
|
||||
|
||||
@@ -25,7 +25,6 @@ type Summary struct {
|
||||
type Report struct {
|
||||
DetectorName string `json:"detector_name"`
|
||||
URL string `json:"url"`
|
||||
UseStealth bool `json:"use_stealth"`
|
||||
CapturedAtUTC string `json:"captured_at_utc"`
|
||||
Screenshot string `json:"screenshot_path"`
|
||||
Detections map[string]Detection `json:"detections"`
|
||||
|
||||
@@ -36,7 +36,7 @@ func (c Custom) URL() string {
|
||||
|
||||
func (c Custom) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
|
||||
err := waitFor(ctx, 15*time.Second, 200*time.Millisecond, func() (bool, error) {
|
||||
hasBody, _, err := page.Has("body")
|
||||
hasBody, _, err := page.Has("pre")
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
@@ -88,6 +88,13 @@ func classifyStatus(status string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
if strings.Contains(value, "🔴") {
|
||||
return true
|
||||
}
|
||||
if strings.Contains(value, "🟢") || strings.Contains(value, "⚪") {
|
||||
return false
|
||||
}
|
||||
|
||||
notDetected := []string{"not detected", "not found", "clean", "clear", "pass", "passed", "ok", "safe", "green", "false", "no"}
|
||||
for _, marker := range notDetected {
|
||||
if strings.Contains(value, marker) {
|
||||
|
||||
@@ -2,7 +2,9 @@ package detectors
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-rod/rod"
|
||||
@@ -25,6 +27,14 @@ func (Rebrowser) URL() string {
|
||||
return rebrowserURL
|
||||
}
|
||||
|
||||
type rebrowserCheck struct {
|
||||
Type string `json:"type"`
|
||||
Icon string `json:"icon"`
|
||||
Rating float64 `json:"rating"`
|
||||
Note string `json:"note"`
|
||||
Debug string `json:"debug"`
|
||||
}
|
||||
|
||||
func (Rebrowser) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
|
||||
err := waitFor(ctx, 20*time.Second, 250*time.Millisecond, func() (bool, error) {
|
||||
hasBody, _, err := page.Has("body")
|
||||
@@ -36,8 +46,14 @@ func (Rebrowser) Extract(ctx context.Context, page *rod.Page) (map[string]fpchec
|
||||
}
|
||||
|
||||
res, err := page.Eval(`() => {
|
||||
const text = (document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
|
||||
return text.includes("runtimeenableleak") || text.includes("sourceurlleak") || text.includes("mainworldexecution") || text.includes("webdriver");
|
||||
const output = document.querySelector('#detections-json');
|
||||
if (!output || !output.value) return false;
|
||||
try {
|
||||
const parsed = JSON.parse(output.value);
|
||||
return Array.isArray(parsed) && parsed.length > 0;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}`)
|
||||
if err != nil {
|
||||
return false, nil
|
||||
@@ -53,25 +69,124 @@ func (Rebrowser) Extract(ctx context.Context, page *rod.Page) (map[string]fpchec
|
||||
return nil, "", fmt.Errorf("rebrowser readiness: %w", err)
|
||||
}
|
||||
|
||||
rows, err := parseRows(page)
|
||||
res, err := page.Eval(`() => {
|
||||
const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
|
||||
const stripHTML = (value) => {
|
||||
const div = document.createElement('div');
|
||||
div.innerHTML = value || '';
|
||||
return normalize(div.textContent || div.innerText || '');
|
||||
};
|
||||
const checks = new Map();
|
||||
|
||||
const put = (item) => {
|
||||
const type = normalize(item.type);
|
||||
if (!type) return;
|
||||
|
||||
const current = checks.get(type) || { type, icon: "", rating: 0, note: "", debug: "" };
|
||||
const next = {
|
||||
type,
|
||||
icon: normalize(item.icon || current.icon),
|
||||
rating: Number.isFinite(item.rating) ? item.rating : current.rating,
|
||||
note: normalize(item.note || current.note),
|
||||
debug: normalize(item.debug || current.debug),
|
||||
};
|
||||
checks.set(type, next);
|
||||
};
|
||||
|
||||
try {
|
||||
const raw = document.querySelector('#detections-json')?.value || '[]';
|
||||
const parsed = JSON.parse(raw);
|
||||
if (Array.isArray(parsed)) {
|
||||
for (const item of parsed) {
|
||||
put({
|
||||
type: item.type || '',
|
||||
rating: Number(item.rating),
|
||||
note: stripHTML(item.note || ''),
|
||||
debug: typeof item.debug === 'string' ? normalize(item.debug) : normalize(JSON.stringify(item.debug || {})),
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (_) {}
|
||||
|
||||
for (const row of Array.from(document.querySelectorAll('#detections-table tbody tr'))) {
|
||||
const cells = Array.from(row.querySelectorAll('td'));
|
||||
if (cells.length < 1) continue;
|
||||
const rawName = normalize(cells[0].innerText || cells[0].textContent || "");
|
||||
if (!rawName) continue;
|
||||
const chars = Array.from(rawName);
|
||||
const icon = chars.length > 0 ? chars[0] : "";
|
||||
const type = normalize(rawName.replace(icon, ""));
|
||||
const note = cells.length > 2 ? normalize(cells[2].innerText || cells[2].textContent || "") : "";
|
||||
put({
|
||||
type,
|
||||
icon,
|
||||
note,
|
||||
});
|
||||
}
|
||||
|
||||
return Array.from(checks.values());
|
||||
}`)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
return nil, "", fmt.Errorf("rebrowser detector rows not found")
|
||||
return nil, "", fmt.Errorf("rebrowser extraction failed: %w", err)
|
||||
}
|
||||
|
||||
detections := rowsToDetections(rows, []string{
|
||||
"runtimeenableleak",
|
||||
"sourceurlleak",
|
||||
"mainworldexecution",
|
||||
"webdriver",
|
||||
"automation",
|
||||
})
|
||||
var checks []rebrowserCheck
|
||||
if err := res.Value.Unmarshal(&checks); err != nil {
|
||||
return nil, "", fmt.Errorf("decode rebrowser checks: %w", err)
|
||||
}
|
||||
if len(checks) == 0 {
|
||||
return nil, "", fmt.Errorf("rebrowser detector checks not found")
|
||||
}
|
||||
|
||||
detections := rebrowserChecksToDetections(checks)
|
||||
if len(detections) == 0 {
|
||||
return nil, "", fmt.Errorf("rebrowser detections are empty")
|
||||
}
|
||||
|
||||
return detections, "", nil
|
||||
raw, _ := json.MarshalIndent(checks, "", " ")
|
||||
return detections, string(raw), nil
|
||||
}
|
||||
|
||||
func rebrowserChecksToDetections(checks []rebrowserCheck) map[string]fpcheck.Detection {
|
||||
detections := make(map[string]fpcheck.Detection, len(checks))
|
||||
for _, check := range checks {
|
||||
key := normalizeKey(check.Type)
|
||||
if key == "unknown" {
|
||||
continue
|
||||
}
|
||||
|
||||
detected := false
|
||||
switch check.Icon {
|
||||
case "🔴":
|
||||
detected = true
|
||||
case "🟢", "🟡", "⚪️", "⚪":
|
||||
detected = false
|
||||
default:
|
||||
detected = check.Rating >= 1
|
||||
}
|
||||
|
||||
description := strings.TrimSpace(check.Note)
|
||||
if strings.TrimSpace(check.Debug) != "" {
|
||||
if description != "" {
|
||||
description = description + " | " + strings.TrimSpace(check.Debug)
|
||||
} else {
|
||||
description = strings.TrimSpace(check.Debug)
|
||||
}
|
||||
}
|
||||
if description == "" {
|
||||
description = fmt.Sprintf("rating=%.2f", check.Rating)
|
||||
}
|
||||
|
||||
severity := ""
|
||||
if detected {
|
||||
severity = "critical"
|
||||
}
|
||||
|
||||
detections[key] = fpcheck.Detection{
|
||||
Detected: detected,
|
||||
Description: description,
|
||||
Severity: severity,
|
||||
}
|
||||
}
|
||||
return detections
|
||||
}
|
||||
|
||||
38
core/fpcheck/detectors/rebrowser_test.go
Normal file
38
core/fpcheck/detectors/rebrowser_test.go
Normal file
@@ -0,0 +1,38 @@
|
||||
package detectors
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestRebrowserChecksToDetections_UsesIconAndRating(t *testing.T) {
|
||||
checks := []rebrowserCheck{
|
||||
{
|
||||
Type: "navigatorWebdriver",
|
||||
Icon: "🔴",
|
||||
Rating: -1,
|
||||
Note: "Own properties detected",
|
||||
},
|
||||
{
|
||||
Type: "runtimeEnableLeak",
|
||||
Icon: "",
|
||||
Rating: 1,
|
||||
Note: "runtime leak",
|
||||
},
|
||||
{
|
||||
Type: "viewport",
|
||||
Icon: "🟢",
|
||||
Rating: 1,
|
||||
Note: "looks fine",
|
||||
},
|
||||
}
|
||||
|
||||
detections := rebrowserChecksToDetections(checks)
|
||||
|
||||
if !detections["navigatorwebdriver"].Detected {
|
||||
t.Fatal("expected red icon check to be detected")
|
||||
}
|
||||
if !detections["runtimeenableleak"].Detected {
|
||||
t.Fatal("expected rating>=1 check to be detected")
|
||||
}
|
||||
if detections["viewport"].Detected {
|
||||
t.Fatal("expected green icon check to be not detected")
|
||||
}
|
||||
}
|
||||
@@ -13,23 +13,16 @@ import (
|
||||
"github.com/go-rod/rod/lib/proto"
|
||||
)
|
||||
|
||||
const (
|
||||
ModeStealthOn = "stealth-on"
|
||||
ModeStealthOff = "stealth-off"
|
||||
)
|
||||
|
||||
// RunOptions controls detector run behavior.
|
||||
type RunOptions struct {
|
||||
UseStealth bool
|
||||
ArtifactDir string
|
||||
WaitBeforeClose time.Duration
|
||||
}
|
||||
|
||||
// Run navigates the given browser to detector URL, extracts verdicts,
|
||||
// captures a screenshot artifact, and returns a normalized report.
|
||||
func Run(ctx context.Context, browser BrowserNavigator, detector Detector, useStealth bool, artifactDir string) (Report, error) {
|
||||
func Run(ctx context.Context, browser BrowserNavigator, detector Detector, artifactDir string) (Report, error) {
|
||||
return RunWithOptions(ctx, browser, detector, RunOptions{
|
||||
UseStealth: useStealth,
|
||||
ArtifactDir: artifactDir,
|
||||
})
|
||||
}
|
||||
@@ -41,7 +34,6 @@ func RunWithOptions(ctx context.Context, browser BrowserNavigator, detector Dete
|
||||
report := Report{
|
||||
DetectorName: detector.Name(),
|
||||
URL: detector.URL(),
|
||||
UseStealth: options.UseStealth,
|
||||
Detections: map[string]Detection{},
|
||||
}
|
||||
|
||||
@@ -50,7 +42,7 @@ func RunWithOptions(ctx context.Context, browser BrowserNavigator, detector Dete
|
||||
artifactDir = "testdata"
|
||||
}
|
||||
|
||||
screenshotPath := filepath.Join(artifactDir, fmt.Sprintf("fpcheck_%s_%s.png", sanitizeFilePart(detector.Name()), modeLabel(options.UseStealth)))
|
||||
screenshotPath := filepath.Join(artifactDir, fmt.Sprintf("fpcheck_%s.png", sanitizeFilePart(detector.Name())))
|
||||
report.Screenshot = filepath.ToSlash(screenshotPath)
|
||||
|
||||
page, err := browser.Navigate(ctx, detector.URL())
|
||||
@@ -152,13 +144,6 @@ func closePageWithTimeout(ctx context.Context, page *rod.Page, timeout time.Dura
|
||||
}
|
||||
}
|
||||
|
||||
func modeLabel(useStealth bool) string {
|
||||
if useStealth {
|
||||
return ModeStealthOn
|
||||
}
|
||||
return ModeStealthOff
|
||||
}
|
||||
|
||||
func sanitizeFilePart(value string) string {
|
||||
value = strings.TrimSpace(strings.ToLower(value))
|
||||
if value == "" {
|
||||
|
||||
28
core/profile_context.go
Normal file
28
core/profile_context.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package core
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type profileContextKey string
|
||||
|
||||
const profileRegionContextKey profileContextKey = "profile_region"
|
||||
|
||||
func WithProfileRegion(ctx context.Context, region string) context.Context {
|
||||
region = strings.TrimSpace(region)
|
||||
if region == "" {
|
||||
return EnsureContext(ctx)
|
||||
}
|
||||
return context.WithValue(EnsureContext(ctx), profileRegionContextKey, region)
|
||||
}
|
||||
|
||||
func profileRegionFromContext(ctx context.Context) string {
|
||||
value, _ := EnsureContext(ctx).Value(profileRegionContextKey).(string)
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
|
||||
func engineFromContext(ctx context.Context) string {
|
||||
value, _ := EnsureContext(ctx).Value(engineContextKey).(string)
|
||||
return strings.TrimSpace(value)
|
||||
}
|
||||
@@ -428,7 +428,6 @@ func (s *Server) handleFingerprintCheck(c *fiber.Ctx) error {
|
||||
reports := make([]fpcheck.Report, 0, len(req.detectors))
|
||||
for idx, detector := range req.detectors {
|
||||
runOpts := fpcheck.RunOptions{
|
||||
UseStealth: req.browserOpts.UseStealth,
|
||||
ArtifactDir: artifactDir,
|
||||
}
|
||||
if req.waitMs > 0 && idx == len(req.detectors)-1 {
|
||||
@@ -466,11 +465,6 @@ func (s *Server) parseFingerprintCheckRequest(c *fiber.Ctx) (fingerprintCheckReq
|
||||
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, err.Error())
|
||||
}
|
||||
|
||||
useStealth, err := parseOptionalBoolQuery(c.Query("stealth", ""), s.opts.FingerprintBrowserOpts.UseStealth)
|
||||
if err != nil {
|
||||
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("invalid stealth query value: %v", err))
|
||||
}
|
||||
|
||||
headless, err := parseOptionalBoolQuery(c.Query("headless", ""), true)
|
||||
if err != nil {
|
||||
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("invalid headless query value: %v", err))
|
||||
@@ -491,7 +485,6 @@ func (s *Server) parseFingerprintCheckRequest(c *fiber.Ctx) (fingerprintCheckReq
|
||||
|
||||
browserOpts := s.opts.FingerprintBrowserOpts
|
||||
browserOpts.IsHeadless = headless
|
||||
browserOpts.UseStealth = useStealth
|
||||
browserOpts.Timeout = time.Duration(timeoutMs) * time.Millisecond
|
||||
browserOpts.LeavePageOpen = false
|
||||
browserOpts.UserAgent = strings.TrimSpace(c.Query("user_agent", browserOpts.UserAgent))
|
||||
|
||||
@@ -186,14 +186,14 @@ Defaults below are the shipped defaults in `config.yaml` (if present). If the co
|
||||
|
||||
### `app`
|
||||
|
||||
| Key | Default | Description |
|
||||
| ------------------ | ------- | ----------------------------- |
|
||||
| `app.timeout` | `15` | Request timeout in seconds |
|
||||
| `app.browser_path` | `""` | Custom browser binary path |
|
||||
| `app.head` | `false` | Headful browser UI |
|
||||
| `app.leakless` | `false` | Force browser process cleanup |
|
||||
| `app.leave_head` | `false` | Keep browser tabs open |
|
||||
| `app.stealth` | `false` | Enable stealth plugin |
|
||||
| Key | Default | Description |
|
||||
| ------------------ | ------- | ------------------------------ |
|
||||
| `app.timeout` | `15` | Request timeout in seconds |
|
||||
| `app.browser_path` | `""` | Custom browser binary path |
|
||||
| `app.profiles` | `""` | Override browser profiles JSON |
|
||||
| `app.head` | `false` | Headful browser UI |
|
||||
| `app.leakless` | `false` | Force browser process cleanup |
|
||||
| `app.leave_head` | `false` | Keep browser tabs open |
|
||||
|
||||
### `proxies`
|
||||
|
||||
|
||||
@@ -175,6 +175,7 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
|
||||
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
|
||||
ctx = core.WithEngine(core.EnsureContext(ctx), ddg.Name())
|
||||
ctx = core.WithProfileRegion(ctx, query.LangCode)
|
||||
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
|
||||
scoped := *ddg
|
||||
scoped.logger = ddg.logger.WithRequest(ctx)
|
||||
@@ -298,6 +299,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
|
||||
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
|
||||
ctx = core.WithEngine(core.EnsureContext(ctx), ddg.Name())
|
||||
ctx = core.WithProfileRegion(ctx, query.LangCode)
|
||||
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
|
||||
scoped := *ddg
|
||||
scoped.logger = ddg.logger.WithRequest(ctx)
|
||||
|
||||
9
go.mod
9
go.mod
@@ -7,25 +7,26 @@ toolchain go1.24.6
|
||||
require (
|
||||
github.com/2captcha/2captcha-go v1.1.10
|
||||
github.com/PuerkitoBio/goquery v1.10.3
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
|
||||
github.com/corpix/uarand v0.2.0
|
||||
github.com/go-rod/rod v0.116.2
|
||||
github.com/go-rod/stealth v0.4.9
|
||||
github.com/gofiber/fiber/v2 v2.52.9
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/refraction-networking/utls v1.8.0
|
||||
github.com/sirupsen/logrus v1.9.3
|
||||
github.com/spf13/cobra v1.9.1
|
||||
github.com/spf13/pflag v1.0.7
|
||||
github.com/spf13/viper v1.20.1
|
||||
github.com/ysmood/gson v0.7.3
|
||||
golang.org/x/net v0.43.0
|
||||
golang.org/x/time v0.12.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/andybalholm/brotli v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
@@ -44,10 +45,8 @@ require (
|
||||
github.com/ysmood/fetchup v0.3.0 // indirect
|
||||
github.com/ysmood/goob v0.4.0 // indirect
|
||||
github.com/ysmood/got v0.41.0 // indirect
|
||||
github.com/ysmood/gson v0.7.3 // indirect
|
||||
github.com/ysmood/leakless v0.9.0 // indirect
|
||||
golang.org/x/crypto v0.41.0 // indirect
|
||||
golang.org/x/net v0.43.0 // indirect
|
||||
golang.org/x/sys v0.35.0 // indirect
|
||||
golang.org/x/text v0.28.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
|
||||
|
||||
7
go.sum
7
go.sum
@@ -18,11 +18,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
|
||||
github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
|
||||
github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
|
||||
github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4=
|
||||
github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
|
||||
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/gofiber/fiber/v2 v2.52.9 h1:YjKl5DOiyP3j0mO61u3NTmK7or8GzzWzCFzkboyP5cw=
|
||||
@@ -89,22 +86,18 @@ github.com/valyala/fasthttp v1.65.0 h1:j/u3uzFEGFfRxw79iYzJN+TteTJwbYkru9uDp3d0Y
|
||||
github.com/valyala/fasthttp v1.65.0/go.mod h1:P/93/YkKPMsKSnATEeELUCkG8a7Y+k99uxNHVbKINr4=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
|
||||
github.com/ysmood/fetchup v0.3.0 h1:UhYz9xnLEVn2ukSuK3KCgcznWpHMdrmbsPpllcylyu8=
|
||||
github.com/ysmood/fetchup v0.3.0/go.mod h1:hbysoq65PXL0NQeNzUczNYIKpwpkwFL4LXMDEvIQq9A=
|
||||
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
|
||||
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
|
||||
github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
|
||||
github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg=
|
||||
github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
|
||||
github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM=
|
||||
github.com/ysmood/got v0.41.0 h1:XiFH311ltTSGyxjeKcNvy7dzbJjjTzn6DBgK313JHBs=
|
||||
github.com/ysmood/got v0.41.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
|
||||
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
|
||||
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
|
||||
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
|
||||
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
|
||||
github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
|
||||
github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
|
||||
@@ -174,6 +174,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) {
|
||||
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (gogl *Google) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
|
||||
ctx = core.WithEngine(core.EnsureContext(ctx), gogl.Name())
|
||||
ctx = core.WithProfileRegion(ctx, query.LangCode)
|
||||
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
|
||||
scoped := *gogl
|
||||
scoped.logger = gogl.logger.WithRequest(ctx)
|
||||
@@ -427,6 +428,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
|
||||
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
|
||||
ctx = core.WithEngine(core.EnsureContext(ctx), gogl.Name())
|
||||
ctx = core.WithProfileRegion(ctx, query.LangCode)
|
||||
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
|
||||
scoped := *gogl
|
||||
scoped.logger = gogl.logger.WithRequest(ctx)
|
||||
|
||||
@@ -128,6 +128,7 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
|
||||
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
|
||||
ctx = core.WithEngine(core.EnsureContext(ctx), yand.Name())
|
||||
ctx = core.WithProfileRegion(ctx, query.LangCode)
|
||||
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
|
||||
scoped := *yand
|
||||
scoped.logger = yand.logger.WithRequest(ctx)
|
||||
@@ -227,6 +228,7 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
|
||||
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
|
||||
func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
|
||||
ctx = core.WithEngine(core.EnsureContext(ctx), yand.Name())
|
||||
ctx = core.WithProfileRegion(ctx, query.LangCode)
|
||||
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
|
||||
scoped := *yand
|
||||
scoped.logger = yand.logger.WithRequest(ctx)
|
||||
|
||||
Reference in New Issue
Block a user