feat(browser): replace stealth with custom profile functionality

This commit is contained in:
Rustem Kamalov
2026-04-23 03:06:49 +03:00
parent e02b374699
commit d052c30bd5
27 changed files with 1558 additions and 154 deletions

View File

@@ -74,6 +74,7 @@ func (baid *Baidu) isTimeout(page *rod.Page) bool {
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.WithEngine(core.EnsureContext(ctx), baid.Name())
ctx = core.WithProfileRegion(ctx, query.LangCode)
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *baid
scoped.logger = baid.logger.WithRequest(ctx)
@@ -177,6 +178,7 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.WithEngine(core.EnsureContext(ctx), baid.Name())
ctx = core.WithProfileRegion(ctx, query.LangCode)
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *baid
scoped.logger = baid.logger.WithRequest(ctx)

View File

@@ -98,6 +98,7 @@ func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name())
ctx = core.WithProfileRegion(ctx, query.LangCode)
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *bing
scoped.logger = bing.logger.WithRequest(ctx)
@@ -274,6 +275,7 @@ type BingImageData struct {
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name())
ctx = core.WithProfileRegion(ctx, query.LangCode)
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *bing
scoped.logger = bing.logger.WithRequest(ctx)

View File

@@ -7,6 +7,7 @@ import (
"strings"
"github.com/karust/openserp/core"
browserprofile "github.com/karust/openserp/core/browser"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
@@ -14,7 +15,7 @@ import (
)
const (
version = "0.6.5"
version = "0.6.6"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)
@@ -53,10 +54,10 @@ type ServerConfig struct {
type AppConfig struct {
Timeout int `mapstructure:"timeout"`
BrowserPath string `mapstructure:"browser_path"`
ProfilesJSON string `mapstructure:"profiles"`
IsBrowserHead bool `mapstructure:"head"`
IsLeaveHead bool `mapstructure:"leave_head"`
IsLeakless bool `mapstructure:"leakless"`
IsStealth bool `mapstructure:"stealth"`
DebugEndpoints bool `mapstructure:"debug_endpoints"`
LogFormat string `mapstructure:"log_format"`
}
@@ -102,6 +103,7 @@ var flagToConfigKey = map[string]string{
"timeout": "app.timeout",
"config": "server.config_path",
"browser-path": "app.browser_path",
"profiles-json": "app.profiles",
"verbose": "server.verbose",
"debug": "server.debug",
"head": "app.head",
@@ -110,7 +112,6 @@ var flagToConfigKey = map[string]string{
"leave": "app.leave_head",
"2captcha_key": "2captcha.apikey",
"proxy": "proxies.global",
"stealth": "app.stealth",
"debug-endpoints": "app.debug_endpoints",
"insecure": "server.insecure",
"cache_ttl": "cache.ttl_seconds",
@@ -134,6 +135,9 @@ var RootCmd = &cobra.Command{
if err != nil {
return err
}
if err := browserprofile.LoadProfilesFromJSON(config.App.ProfilesJSON); err != nil {
return fmt.Errorf("load app.profiles: %w", err)
}
logFormat, err := core.NormalizeLogFormat(config.App.LogFormat)
if err != nil {
@@ -316,10 +320,10 @@ func setConfigDefaults(v *viper.Viper) {
v.SetDefault("app.timeout", 30)
v.SetDefault("app.browser_path", "")
v.SetDefault("app.profiles", "")
v.SetDefault("app.head", false)
v.SetDefault("app.leave_head", false)
v.SetDefault("app.leakless", false)
v.SetDefault("app.stealth", false)
v.SetDefault("app.debug_endpoints", false)
v.SetDefault("proxies.entries", []interface{}{})
@@ -348,6 +352,7 @@ func init() {
RootCmd.PersistentFlags().IntVarP(&config.App.Timeout, "timeout", "t", 30, "Timeout to fail request")
RootCmd.PersistentFlags().StringVarP(&config.Server.ConfigPath, "config", "c", "", "Configuration file path")
RootCmd.PersistentFlags().StringVarP(&config.App.BrowserPath, "browser-path", "", "", "Custom browser binary path (Chrome/Chromium/Edge/Brave..)")
RootCmd.PersistentFlags().StringVar(&config.App.ProfilesJSON, "profiles", "", "Path to browser profile catalog JSON")
RootCmd.PersistentFlags().BoolVarP(&config.Server.IsVerbose, "verbose", "v", false, "Use verbose output")
RootCmd.PersistentFlags().BoolVarP(&config.Server.IsDebug, "debug", "d", false, "Use debug output. Disable headless browser")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsBrowserHead, "head", "", false, "Enable browser UI")
@@ -356,7 +361,6 @@ func init() {
RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made")
RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
RootCmd.PersistentFlags().StringVarP(&config.Proxies.Global, "proxy", "x", "", "Force a single proxy for all engines (same as proxies.global)")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsStealth, "stealth", "s", false, "Use stealth browser plugin")
RootCmd.PersistentFlags().BoolVar(&config.App.DebugEndpoints, "debug-endpoints", false, "Enable debug-only HTTP endpoints")
RootCmd.PersistentFlags().BoolVarP(&config.Server.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
RootCmd.PersistentFlags().IntVar(&config.Cache.TTLSeconds, "cache_ttl", 300, "Cache TTL in seconds (0 to disable)")

View File

@@ -117,7 +117,6 @@ func searchBrowser(engineType string, query core.Query, browserProxyURL string,
BrowserPath: config.App.BrowserPath,
ProxyURL: browserProxyURL,
Insecure: config.Server.Insecure,
UseStealth: config.App.IsStealth,
}
if config.Server.IsDebug {

View File

@@ -132,7 +132,6 @@ func buildFingerprintBrowserOptions() core.BrowserOpts {
Timeout: time.Second * time.Duration(config.App.Timeout),
BrowserPath: config.App.BrowserPath,
Insecure: config.Server.Insecure,
UseStealth: config.App.IsStealth,
}
if config.Server.IsDebug {
opts.IsHeadless = false

View File

@@ -11,10 +11,10 @@ app:
log_format: "text"
timeout: 15 # Browser/search timeout in seconds
browser_path: "" # Custom browser binary path (chrome/chromium/edge..)
profiles: "" # Optional JSON file path overriding built-in browser profiles
head: false # Show browser UI (headful mode)
leakless: false # Force browser process cleanup after request
leave_head: false # Keep tabs open after request for debugging
stealth: false # Enable stealth browser plugin
proxies:
# Force a single proxy for all engines.

View File

@@ -2,21 +2,23 @@ package core
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"regexp"
"runtime/debug"
"strings"
"sync"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/devices"
"github.com/go-rod/rod/lib/launcher"
"github.com/go-rod/rod/lib/proto"
"github.com/go-rod/stealth"
browserprofile "github.com/karust/openserp/core/browser"
"github.com/sirupsen/logrus"
"github.com/ysmood/gson"
)
// BrowserOpts configures Chromium launch and navigation behavior.
@@ -45,8 +47,6 @@ type BrowserOpts struct {
ProxyURL string
// Insecure allows invalid TLS certificates for browser requests.
Insecure bool
// UseStealth enables go-rod stealth page creation.
UseStealth bool
// UserAgent optionally overrides browser-reported user agent during emulation.
UserAgent string
}
@@ -71,15 +71,18 @@ type Browser struct {
}
type browserConnection struct {
mu sync.Mutex
browser *rod.Browser
cachedUserAgent string
mu sync.Mutex
browser *rod.Browser
laneProfiles map[string]browserprofile.Profile
}
// NewBrowser launches a new Chromium process via Rod launcher and returns a
// Browser wrapper configured with proxy and captcha solver settings.
func NewBrowser(opts BrowserOpts) (*Browser, error) {
opts.Check()
if strings.TrimSpace(opts.UserAgent) != "" {
logrus.Warn("custom user_agent override can reduce profile coherence; use only for diagnostics")
}
logrus.WithField("browser_options", fmt.Sprintf("%+v", opts)).Debug("Browser options")
path, err := resolveBrowserBinaryPath(opts.BrowserPath, launcher.LookPath)
@@ -87,9 +90,20 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) {
return nil, err
}
// Create launcher
l := launcher.New().Leakless(opts.IsLeakless).Headless(opts.IsHeadless).Set("disable-blink-features", "AutomationControlled").
Delete("enable-automation")
// Create launcher.
// headless=new uses the full Chrome renderer; legacy --headless disables the
// GPU process entirely, making WebGL context creation fail even with SwiftShader.
// use-angle=swiftshader-webgl (Chrome ≥112) enables a software WebGL renderer.
l := launcher.New().Leakless(opts.IsLeakless).
Set("disable-blink-features", "AutomationControlled").
Delete("enable-automation").
Set("use-angle", "swiftshader-webgl").
Set("ignore-gpu-blocklist")
if opts.IsHeadless {
l = l.HeadlessNew(true)
} else {
l = l.Headless(false)
}
if path != "" {
logrus.WithField("browser_path", path).Debug("Using browser binary")
l = l.Bin(path)
@@ -204,35 +218,26 @@ func (b *Browser) newRodBrowser() *rod.Browser {
return browser
}
func (b *Browser) connectBrowser() (*rod.Browser, string, error) {
func (b *Browser) connectBrowser() (*rod.Browser, error) {
browser := b.newRodBrowser()
if err := browser.Connect(); err != nil {
return nil, "", err
return nil, err
}
// Keep cert handling on the persistent browser session.
// Proxy runtime can surface MITM certs, and insecure mode is explicit opt-in.
if b.ProxyURL != "" || b.Insecure {
if err := browser.IgnoreCertErrors(true); err != nil {
return nil, "", err
return nil, err
}
}
version, err := browser.Version()
if err != nil {
return nil, "", fmt.Errorf("read browser version: %w", err)
}
ua := strings.ReplaceAll(version.UserAgent, "HeadlessChrome/", "Chrome/")
if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" {
ua = overrideUA
}
return browser, ua, nil
return browser, nil
}
func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect bool) (*rod.Browser, string, error) {
func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect bool) (*rod.Browser, error) {
if b == nil || b.browserAddr == "" {
return nil, "", fmt.Errorf("browser is not initialized")
return nil, fmt.Errorf("browser is not initialized")
}
state := b.connectionState()
@@ -240,26 +245,24 @@ func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect boo
defer state.mu.Unlock()
if state.browser == nil || forceReconnect {
connected, ua, err := b.connectBrowser()
connected, err := b.connectBrowser()
if err != nil {
return nil, "", err
return nil, err
}
state.browser = connected
state.cachedUserAgent = ua
return state.browser, ua, nil
return state.browser, nil
}
if _, err := state.browser.Version(); err != nil {
WithRequest(ctx).WithError(err).Debug("Browser ping failed, reconnecting")
connected, ua, reconnectErr := b.connectBrowser()
connected, reconnectErr := b.connectBrowser()
if reconnectErr != nil {
return nil, "", reconnectErr
return nil, reconnectErr
}
state.browser = connected
state.cachedUserAgent = ua
}
return state.browser, state.cachedUserAgent, nil
return state.browser, nil
}
func createIsolatedPage(browser *rod.Browser) (*rod.Page, proto.BrowserBrowserContextID, error) {
@@ -329,7 +332,294 @@ func (b *Browser) startProxyAuthHandler(ctx context.Context, browser *rod.Browse
return cancel, nil
}
// Navigate connects to Chromium, creates a page, applies stealth/emulation and
var chromeVersionPattern = regexp.MustCompile(`(?:HeadlessChrome|Chrome)/([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`)
func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browserprofile.Profile, string) {
engine := engineFromContext(ctx)
region := profileRegionFromContext(ctx)
if region == "" {
region = strings.TrimSpace(b.LanguageCode)
}
laneKey := browserprofile.LaneKey(engine, region)
state := b.connectionState()
state.mu.Lock()
if state.laneProfiles == nil {
state.laneProfiles = make(map[string]browserprofile.Profile)
}
if profile, ok := state.laneProfiles[laneKey]; ok {
state.mu.Unlock()
return profile, laneKey
}
state.mu.Unlock()
// Resolve profile outside the lock: SelectProfile reads from a separate
// RWMutex-guarded catalog, and applyRuntimeBrowserVersion makes a CDP
// round-trip (browser.Version). Holding state.mu over network I/O would
// serialize all concurrent Navigate calls.
profile := browserprofile.SelectProfile(engine, region)
profile = applyRuntimeBrowserVersion(profile, browser)
if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" {
profile.UserAgent = overrideUA
}
state.mu.Lock()
if state.laneProfiles == nil {
state.laneProfiles = make(map[string]browserprofile.Profile)
}
if _, exists := state.laneProfiles[laneKey]; !exists {
state.laneProfiles[laneKey] = profile
} else {
profile = state.laneProfiles[laneKey]
}
state.mu.Unlock()
return profile, laneKey
}
func applyRuntimeBrowserVersion(profile browserprofile.Profile, browser *rod.Browser) browserprofile.Profile {
fullVersion := ""
if browser != nil {
version, err := browser.Version()
if err == nil && version != nil {
fullVersion = extractChromeVersion(version.UserAgent)
if fullVersion == "" {
fullVersion = extractChromeVersion(version.Product)
}
}
}
if fullVersion == "" {
fullVersion = extractChromeVersion(profile.UserAgent)
}
if fullVersion == "" {
return profile
}
major := chromeMajorVersion(fullVersion)
if major == "" {
return profile
}
profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, major+".0.0.0")
profile.UACHBrands = patchBrandVersions(profile.UACHBrands, major, false)
profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, fullVersion, true)
return profile
}
func extractChromeVersion(value string) string {
matches := chromeVersionPattern.FindStringSubmatch(strings.TrimSpace(value))
if len(matches) < 2 {
return ""
}
return strings.TrimSpace(matches[1])
}
func chromeMajorVersion(fullVersion string) string {
fullVersion = strings.TrimSpace(fullVersion)
if fullVersion == "" {
return ""
}
parts := strings.Split(fullVersion, ".")
if len(parts) == 0 {
return ""
}
return strings.TrimSpace(parts[0])
}
func replaceChromeUserAgentVersion(userAgent string, replacement string) string {
replacement = strings.TrimSpace(replacement)
if replacement == "" {
return strings.ReplaceAll(strings.TrimSpace(userAgent), "HeadlessChrome/", "Chrome/")
}
normalized := strings.ReplaceAll(strings.TrimSpace(userAgent), "HeadlessChrome/", "Chrome/")
return chromeVersionPattern.ReplaceAllString(normalized, "Chrome/"+replacement)
}
func patchBrandVersions(values []browserprofile.BrandVersion, version string, patchNotABrand bool) []browserprofile.BrandVersion {
version = strings.TrimSpace(version)
if version == "" {
return values
}
out := make([]browserprofile.BrandVersion, 0, len(values))
for _, value := range values {
item := value
brandLower := strings.ToLower(strings.TrimSpace(item.Brand))
if brandLower == "chromium" || brandLower == "google chrome" {
item.Version = version
} else if patchNotABrand && strings.Contains(brandLower, "not_a brand") && strings.Count(version, ".") == 3 {
item.Version = "24.0.0.0"
}
out = append(out, item)
}
return out
}
func navigatorPlatformForProfile(profile browserprofile.Profile) string {
switch strings.ToLower(strings.TrimSpace(profile.Platform)) {
case "windows":
return "Win32"
case "macos":
return "MacIntel"
case "linux":
return "Linux x86_64"
default:
return strings.TrimSpace(profile.Platform)
}
}
func applyProfile(page *rod.Page, profile browserprofile.Profile) error {
if page == nil {
return fmt.Errorf("page is nil")
}
navigatorLangs := profileNavigatorLanguages(profile)
acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
if acceptLanguage == "" {
acceptLanguage = navigatorLangs[0]
}
locale := strings.TrimSpace(profile.Locale)
if locale == "" {
locale = navigatorLangs[0]
}
width := profile.Viewport.Width
height := profile.Viewport.Height
if width <= 0 {
width = 1920
}
if height <= 0 {
height = 1080
}
metadata := &proto.EmulationUserAgentMetadata{
Brands: toProtoBrandVersions(profile.UACHBrands),
FullVersionList: toProtoBrandVersions(profile.UACHFullVerList),
Platform: strings.TrimSpace(profile.Platform),
PlatformVersion: strings.TrimSpace(profile.PlatformVersion),
Architecture: strings.TrimSpace(profile.Architecture),
Bitness: strings.TrimSpace(profile.Bitness),
Mobile: profile.Mobile,
}
if err := (proto.NetworkSetUserAgentOverride{
UserAgent: strings.TrimSpace(profile.UserAgent),
AcceptLanguage: acceptLanguage,
Platform: navigatorPlatformForProfile(profile),
UserAgentMetadata: metadata,
}).Call(page); err != nil {
return fmt.Errorf("set user agent override failed: %w", err)
}
if err := (proto.EmulationSetLocaleOverride{
Locale: locale,
}).Call(page); err != nil {
return fmt.Errorf("set locale override failed: %w", err)
}
if err := (proto.EmulationSetTimezoneOverride{
TimezoneID: strings.TrimSpace(profile.Timezone),
}).Call(page); err != nil {
return fmt.Errorf("set timezone override failed: %w", err)
}
if err := (proto.EmulationSetDeviceMetricsOverride{
Width: width,
Height: height,
DeviceScaleFactor: 1,
Mobile: profile.Mobile,
ScreenWidth: &width,
ScreenHeight: &height,
}).Call(page); err != nil {
return fmt.Errorf("set device metrics failed: %w", err)
}
if err := (proto.NetworkSetExtraHTTPHeaders{
Headers: proto.NetworkHeaders{
"Accept-Language": gson.New(acceptLanguage),
},
}).Call(page); err != nil {
return fmt.Errorf("set extra headers failed: %w", err)
}
if err := evalPatchScript(page, navigatorLangs, width, height); err != nil {
return err
}
return nil
}
func profileNavigatorLanguages(profile browserprofile.Profile) []string {
langs := make([]string, 0, len(profile.NavigatorLangs))
for _, language := range profile.NavigatorLangs {
trimmed := strings.TrimSpace(language)
if trimmed == "" {
continue
}
langs = append(langs, trimmed)
}
if len(langs) > 0 {
return langs
}
acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
if acceptLanguage != "" {
parts := strings.Split(acceptLanguage, ",")
langs = make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "" {
continue
}
if idx := strings.Index(part, ";"); idx >= 0 {
part = strings.TrimSpace(part[:idx])
}
if part != "" {
langs = append(langs, part)
}
}
if len(langs) > 0 {
return langs
}
}
if locale := strings.TrimSpace(profile.Locale); locale != "" {
return []string{locale}
}
return []string{"en-US"}
}
func toProtoBrandVersions(values []browserprofile.BrandVersion) []*proto.EmulationUserAgentBrandVersion {
out := make([]*proto.EmulationUserAgentBrandVersion, 0, len(values))
for _, value := range values {
brand := strings.TrimSpace(value.Brand)
version := strings.TrimSpace(value.Version)
if brand == "" || version == "" {
continue
}
out = append(out, &proto.EmulationUserAgentBrandVersion{
Brand: brand,
Version: version,
})
}
return out
}
func evalPatchScript(page *rod.Page, langs []string, width, height int) error {
langsJSON, err := json.Marshal(langs)
if err != nil {
return fmt.Errorf("marshal navigator languages: %w", err)
}
args := fmt.Sprintf("const __langs = %s;\nconst __w = %d;\nconst __h = %d;\n",
string(langsJSON), width, height)
_, err = page.EvalOnNewDocument(args + string(browserprofile.PatchJS))
if err != nil {
return fmt.Errorf("eval patch script: %w", err)
}
return nil
}
// Navigate connects to Chromium, creates a page, applies a coherent profile and
// proxy auth, then navigates to URL. It returns an initialized page ready for
// selector queries, or an error when browser setup/navigation fails.
func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
@@ -340,7 +630,7 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
WithRequest(ctx).WithField("url", URL).Debug("Navigate")
browser, ua, err := b.ensureConnectedBrowser(ctx, false)
browser, err := b.ensureConnectedBrowser(ctx, false)
if err != nil {
return nil, fmt.Errorf("browser connect failed: %w", err)
}
@@ -348,7 +638,7 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
page, browserContextID, err := createIsolatedPage(browser)
if err != nil {
// Single-shot reconnect for stale websocket sessions.
browser, ua, err = b.ensureConnectedBrowser(ctx, true)
browser, err = b.ensureConnectedBrowser(ctx, true)
if err != nil {
return nil, fmt.Errorf("create isolated page failed, reconnect also failed: %w", err)
}
@@ -379,33 +669,10 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
defer cancelProxyAuth()
}
if b.UseStealth {
if _, err := page.EvalOnNewDocument(stealth.JS); err != nil {
closeOnErr()
return nil, fmt.Errorf("create stealth page failed: %w", err)
}
}
if err := page.Emulate(devices.Device{
AcceptLanguage: b.LanguageCode,
UserAgent: ua,
}); err != nil {
profile, laneKey := b.laneProfile(ctx, browser)
if err := applyProfile(page, profile); err != nil {
closeOnErr()
return nil, fmt.Errorf("emulate page failed: %w", err)
}
if !b.UseStealth {
if err := (proto.EmulationSetDeviceMetricsOverride{
Width: 1920,
Height: 1080,
DeviceScaleFactor: 1,
Mobile: false,
ScreenWidth: &[]int{1920}[0],
ScreenHeight: &[]int{1080}[0],
}).Call(page); err != nil {
closeOnErr()
return nil, fmt.Errorf("set device metrics failed: %w", err)
}
return nil, fmt.Errorf("apply profile %s (%s) failed: %w", profile.ID, laneKey, err)
}
page = page.Context(ctx)
@@ -464,6 +731,7 @@ func (b *Browser) Close() error {
}
state.browser = nil
state.laneProfiles = nil
if err := browser.Close(); err != nil && !isBrowserClosedError(err) {
return err
}

73
core/browser/patch.js Normal file
View File

@@ -0,0 +1,73 @@
// Stealth patches injected via EvalOnNewDocument.
// Arguments are injected as a leading const block by the Go caller:
// const __langs = [...]; // navigator_langs from profile
// const __w = 1920; // viewport width
// const __h = 1080; // viewport height
(() => {
'use strict';
// --- navigator.language / navigator.languages ---
//
// CDP Network.setUserAgentOverride(acceptLanguage) sets the HTTP header but
// does NOT update navigator.language / navigator.languages in JS. Those are
// read from the browser profile at context creation. On Linux headless they
// reflect the system ICU locale (usually "en-US" regardless of profile).
//
// We patch them here. The key to being undetectable: define with a getter
// first (requires configurable:true), then immediately lock the descriptor
// back to configurable:false so it looks exactly like a native property.
const primary = __langs.length ? __langs[0] : 'en-US';
const sealGetter = (target, prop, fn) => {
try {
Object.defineProperty(target, prop, {
get: fn,
set: undefined,
enumerable: true,
configurable: true, // must be true to set a getter
});
Object.defineProperty(target, prop, {
configurable: false, // seal: now indistinguishable from native
});
} catch (_) {}
};
const patchLangs = (target) => {
if (!target) return;
sealGetter(target, 'language', () => primary);
sealGetter(target, 'languages', () => Object.freeze(__langs.slice()));
};
// Patch the navigator instance (Linux headless stores own-props here)
// and Navigator.prototype (other platforms / future Chrome versions).
patchLangs(navigator);
patchLangs(Object.getPrototypeOf(navigator));
if (typeof WorkerNavigator !== 'undefined') {
patchLangs(WorkerNavigator.prototype);
}
// --- screen dimensions ---
//
// EmulationSetDeviceMetricsOverride sets the CSS viewport but leaves
// window.screen.* at headless defaults. Checkers compare screen size
// against viewport and flag the mismatch as automation.
//
// Screen properties live on Screen.prototype as non-configurable getters.
// Patching the prototype makes them look native.
const patchScreen = () => {
const proto = typeof Screen !== 'undefined' ? Screen.prototype : null;
if (!proto) return;
sealGetter(proto, 'width', () => __w);
sealGetter(proto, 'height', () => __h);
sealGetter(proto, 'availWidth', () => __w);
sealGetter(proto, 'availHeight', () => __h);
sealGetter(proto, 'availLeft', () => 0);
sealGetter(proto, 'availTop', () => 0);
};
patchScreen();
// window.outerWidth/Height are own configurable properties; Chrome normally
// sets them to match the OS window size. In headless they are 0.
sealGetter(window, 'outerWidth', () => __w);
sealGetter(window, 'outerHeight', () => __h);
})();

294
core/browser/profile.go Normal file
View File

@@ -0,0 +1,294 @@
package browser
import (
_ "embed"
"encoding/json"
"fmt"
"os"
"runtime"
"strings"
"sync"
)
type BrandVersion struct {
Brand string `json:"brand"`
Version string `json:"version"`
}
type Viewport struct {
Width int `json:"width"`
Height int `json:"height"`
}
type Profile struct {
ID string `json:"id"`
UserAgent string `json:"user_agent"`
UACHBrands []BrandVersion `json:"uach_brands"`
UACHFullVerList []BrandVersion `json:"uach_full_version_list"`
Platform string `json:"platform"`
PlatformVersion string `json:"platform_version"`
Architecture string `json:"architecture"`
Bitness string `json:"bitness"`
Mobile bool `json:"mobile"`
AcceptLanguage string `json:"accept_language"`
NavigatorLangs []string `json:"navigator_langs"`
Locale string `json:"locale"`
Timezone string `json:"timezone"`
Viewport Viewport `json:"viewport"`
}
type catalogConfig struct {
Profiles []Profile `json:"profiles"`
LaneProfileIDs map[string]string `json:"lane_profile_ids"`
DefaultRegionByEngine map[string]string `json:"default_region_by_engine"`
}
const (
ProfileChromeWinUS = "chrome-win-us"
ProfileChromeWinRU = "chrome-win-ru"
ProfileChromeMacUS = "chrome-macos-us"
ProfileChromeLinuxUS = "chrome-linux-us"
ProfileChromeLinuxRU = "chrome-linux-ru"
)
//go:embed profiles.json
var defaultProfilesJSON []byte
//go:embed patch.js
var PatchJS []byte
var profileCatalogMu sync.RWMutex
var catalog = map[string]Profile{}
var laneProfileIDs = map[string]string{}
var defaultRegionByEngine = map[string]string{}
func init() {
if err := loadProfilesFromJSONBytes(defaultProfilesJSON); err != nil {
panic(fmt.Sprintf("load embedded browser profiles: %v", err))
}
}
func LoadProfilesFromJSON(path string) error {
path = strings.TrimSpace(path)
if path == "" {
return nil
}
data, err := os.ReadFile(path)
if err != nil {
return fmt.Errorf("read profiles json %q: %w", path, err)
}
if err := loadProfilesFromJSONBytes(data); err != nil {
return fmt.Errorf("parse profiles json %q: %w", path, err)
}
return nil
}
func loadProfilesFromJSONBytes(data []byte) error {
var cfg catalogConfig
if err := json.Unmarshal(data, &cfg); err != nil {
return err
}
if len(cfg.Profiles) == 0 {
return fmt.Errorf("profiles list is empty")
}
nextCatalog := make(map[string]Profile, len(cfg.Profiles))
for i, profile := range cfg.Profiles {
profile.ID = strings.TrimSpace(profile.ID)
if profile.ID == "" {
return fmt.Errorf("profiles[%d].id is empty", i)
}
if strings.TrimSpace(profile.UserAgent) == "" {
return fmt.Errorf("profiles[%d].user_agent is empty", i)
}
if _, exists := nextCatalog[profile.ID]; exists {
return fmt.Errorf("duplicate profile id %q", profile.ID)
}
nextCatalog[profile.ID] = profile
}
nextLaneProfileIDs := make(map[string]string, len(cfg.LaneProfileIDs))
for rawLaneKey, profileID := range cfg.LaneProfileIDs {
laneKey, err := normalizeLaneKey(rawLaneKey)
if err != nil {
return err
}
profileID = strings.TrimSpace(profileID)
if profileID == "" {
return fmt.Errorf("lane profile id for %q is empty", laneKey)
}
if _, exists := nextCatalog[profileID]; !exists {
return fmt.Errorf("lane %q references unknown profile id %q", laneKey, profileID)
}
nextLaneProfileIDs[laneKey] = profileID
}
nextDefaultRegionByEngine := map[string]string{
"yandex": "ru",
}
for engine, region := range cfg.DefaultRegionByEngine {
engine = NormalizeEngine(engine)
if engine == "" {
return fmt.Errorf("default_region_by_engine contains empty engine key")
}
nextDefaultRegionByEngine[engine] = normalizeConfiguredRegion(region)
}
profileCatalogMu.Lock()
catalog = nextCatalog
laneProfileIDs = nextLaneProfileIDs
defaultRegionByEngine = nextDefaultRegionByEngine
profileCatalogMu.Unlock()
return nil
}
func normalizeLaneKey(value string) (string, error) {
value = strings.TrimSpace(value)
parts := strings.SplitN(value, ":", 2)
if len(parts) != 2 {
return "", fmt.Errorf("invalid lane key %q, expected engine:region", value)
}
engine := NormalizeEngine(parts[0])
if engine == "" {
return "", fmt.Errorf("invalid lane key %q, engine is empty", value)
}
region := normalizeConfiguredRegion(parts[1])
return engine + ":" + region, nil
}
func normalizeConfiguredRegion(region string) string {
region = strings.TrimSpace(region)
if region == "" {
return "us"
}
return NormalizeRegion(region)
}
func Catalog() []Profile {
profileCatalogMu.RLock()
defer profileCatalogMu.RUnlock()
out := make([]Profile, 0, len(catalog))
for _, profile := range catalog {
out = append(out, profile)
}
return out
}
func SelectProfile(engine string, region string) Profile {
engine = NormalizeEngine(engine)
region = NormalizeRegion(region)
if region == "" {
region = DefaultRegionForEngine(engine)
}
laneKey := engine + ":" + region
profileCatalogMu.RLock()
profileID, ok := laneProfileIDs[laneKey]
profileCatalogMu.RUnlock()
if ok {
return profileByID(profileID)
}
return profileByID(defaultProfileID(region))
}
func LaneKey(engine string, region string) string {
engine = NormalizeEngine(engine)
if engine == "" {
engine = "unknown"
}
region = NormalizeRegion(region)
if region == "" {
region = DefaultRegionForEngine(engine)
}
return engine + ":" + region
}
func DefaultRegionForEngine(engine string) string {
engine = NormalizeEngine(engine)
profileCatalogMu.RLock()
defer profileCatalogMu.RUnlock()
if region, ok := defaultRegionByEngine[engine]; ok {
return region
}
return "us"
}
func NormalizeEngine(engine string) string {
return strings.ToLower(strings.TrimSpace(engine))
}
func NormalizeRegion(region string) string {
region = strings.TrimSpace(strings.ToLower(region))
if region == "" {
return ""
}
if idx := strings.Index(region, ","); idx >= 0 {
region = region[:idx]
}
if idx := strings.Index(region, ";"); idx >= 0 {
region = region[:idx]
}
region = strings.ReplaceAll(region, "_", "-")
if idx := strings.Index(region, "-"); idx >= 0 {
region = region[:idx]
}
switch region {
case "ru", "be", "kz", "ky":
return "ru"
default:
return "us"
}
}
func profileByID(profileID string) Profile {
profileCatalogMu.RLock()
defer profileCatalogMu.RUnlock()
if profile, ok := catalog[profileID]; ok {
return profile
}
if fallback, ok := catalog[defaultProfileID("us")]; ok {
return fallback
}
for _, profile := range catalog {
return profile
}
return Profile{}
}
func defaultProfileID(region string) string {
region = NormalizeRegion(region)
if region == "" {
region = "us"
}
switch runtime.GOOS {
case "windows":
if region == "ru" {
return ProfileChromeWinRU
}
return ProfileChromeWinUS
case "darwin":
return ProfileChromeMacUS
default:
if region == "ru" {
return ProfileChromeLinuxRU
}
return ProfileChromeLinuxUS
}
}

View File

@@ -0,0 +1,200 @@
//go:build integration
// +build integration
package browser_test
import (
"context"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/go-rod/rod"
"github.com/karust/openserp/core"
browserprofile "github.com/karust/openserp/core/browser"
"github.com/karust/openserp/testutil"
)
func TestProfileCoherence(t *testing.T) {
testutil.RequireIntegration(t)
fixture := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html; charset=utf-8")
_, _ = w.Write([]byte(`<!doctype html><html><head><meta charset="utf-8"><title>coherence</title></head><body>ok</body></html>`))
}))
defer fixture.Close()
browser, err := core.NewBrowser(core.BrowserOpts{
IsHeadless: true,
IsLeakless: false,
Timeout: 20 * time.Second,
})
if err != nil {
t.Fatalf("create browser: %v", err)
}
defer func() {
if closeErr := browser.Close(); closeErr != nil {
t.Fatalf("close browser: %v", closeErr)
}
}()
cases := []struct {
name string
engine string
region string
}{
{
name: "windows lane",
engine: "google",
region: "ru",
},
{
name: "mac lane",
engine: "bing",
region: "en-US",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
expected := browserprofile.SelectProfile(tc.engine, tc.region)
ctx := core.WithEngine(context.Background(), tc.engine)
ctx = core.WithProfileRegion(ctx, tc.region)
page, err := browser.Navigate(ctx, fixture.URL)
if err != nil {
t.Fatalf("navigate fixture: %v", err)
}
defer func() {
if closeErr := core.ClosePageWithTimeout(context.Background(), page, time.Second); closeErr != nil {
t.Fatalf("close page: %v", closeErr)
}
}()
got, err := browserProfileSurface(page)
if err != nil {
t.Fatalf("collect profile surfaces: %v", err)
}
if got.UserAgent != expected.UserAgent {
t.Fatalf("navigator.userAgent mismatch:\nexpected: %s\nactual: %s", expected.UserAgent, got.UserAgent)
}
if got.Platform != expected.Platform {
t.Fatalf("navigator.userAgentData.platform mismatch: expected %q got %q", expected.Platform, got.Platform)
}
if got.NavigatorPlatform != expectedNavigatorPlatform(expected.Platform) {
t.Fatalf("navigator.platform mismatch: expected %q got %q", expectedNavigatorPlatform(expected.Platform), got.NavigatorPlatform)
}
if got.Locale != expected.Locale {
t.Fatalf("Intl locale mismatch: expected %q got %q", expected.Locale, got.Locale)
}
if got.Timezone != expected.Timezone {
t.Fatalf("Intl timezone mismatch: expected %q got %q", expected.Timezone, got.Timezone)
}
if len(got.NavigatorLanguages) == 0 {
t.Fatal("navigator.languages is empty")
}
if got.NavigatorLanguages[0] != expected.NavigatorLangs[0] {
t.Fatalf("navigator.languages[0] mismatch: expected %q got %q", expected.NavigatorLangs[0], got.NavigatorLanguages[0])
}
if got.WebdriverType != "undefined" {
t.Fatalf("navigator.webdriver expected undefined, got %q", got.WebdriverType)
}
if got.WebdriverOwnPropPresent {
t.Fatal("navigator own property 'webdriver' should not be present")
}
if got.WorkerUserAgent != got.UserAgent {
t.Fatalf("worker userAgent mismatch: main %q worker %q", got.UserAgent, got.WorkerUserAgent)
}
if got.WorkerPlatform != got.NavigatorPlatform {
t.Fatalf("worker platform mismatch: main %q worker %q", got.NavigatorPlatform, got.WorkerPlatform)
}
if len(got.WorkerNavigatorLangs) == 0 {
t.Fatal("worker navigator.languages is empty")
}
if got.WorkerNavigatorLangs[0] != got.NavigatorLanguages[0] {
t.Fatalf("worker navigator.languages[0] mismatch: main %q worker %q", got.NavigatorLanguages[0], got.WorkerNavigatorLangs[0])
}
if got.WorkerTimezone != got.Timezone {
t.Fatalf("worker timezone mismatch: main %q worker %q", got.Timezone, got.WorkerTimezone)
}
})
}
}
type profileSurface struct {
UserAgent string `json:"userAgent"`
Platform string `json:"platform"`
NavigatorPlatform string `json:"navigatorPlatform"`
NavigatorLanguages []string `json:"navigatorLanguages"`
Timezone string `json:"timezone"`
Locale string `json:"locale"`
WebdriverType string `json:"webdriverType"`
WebdriverOwnPropPresent bool `json:"webdriverOwnPropPresent"`
WorkerUserAgent string `json:"workerUserAgent"`
WorkerPlatform string `json:"workerPlatform"`
WorkerNavigatorLangs []string `json:"workerNavigatorLangs"`
WorkerTimezone string `json:"workerTimezone"`
}
func browserProfileSurface(page *rod.Page) (profileSurface, error) {
result, err := page.Eval(`async () => {
const workerData = await new Promise((resolve) => {
try {
const source = "self.onmessage=()=>{self.postMessage({userAgent:self.navigator.userAgent||'',platform:self.navigator.platform||'',navigatorLanguages:Array.from(self.navigator.languages||[]),timezone:Intl.DateTimeFormat().resolvedOptions().timeZone||''});};";
const blob = new Blob([source], { type: 'application/javascript' });
const url = URL.createObjectURL(blob);
const worker = new Worker(url);
worker.onmessage = (event) => {
resolve(event.data || {});
worker.terminate();
URL.revokeObjectURL(url);
};
worker.onerror = () => {
resolve({});
worker.terminate();
URL.revokeObjectURL(url);
};
worker.postMessage('run');
} catch (_) {
resolve({});
}
});
return {
userAgent: navigator.userAgent || "",
platform: navigator.userAgentData ? (navigator.userAgentData.platform || "") : "",
navigatorPlatform: navigator.platform || "",
navigatorLanguages: Array.from(navigator.languages || []),
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || "",
locale: Intl.DateTimeFormat().resolvedOptions().locale || "",
webdriverType: typeof navigator.webdriver,
webdriverOwnPropPresent: Object.getOwnPropertyNames(navigator).includes('webdriver'),
workerUserAgent: workerData.userAgent || "",
workerPlatform: workerData.platform || "",
workerNavigatorLangs: Array.from(workerData.navigatorLanguages || []),
workerTimezone: workerData.timezone || "",
};
}`)
if err != nil {
return profileSurface{}, err
}
var out profileSurface
if err := result.Value.Unmarshal(&out); err != nil {
return profileSurface{}, fmt.Errorf("decode eval payload: %w", err)
}
return out, nil
}
func expectedNavigatorPlatform(platform string) string {
switch platform {
case "Windows":
return "Win32"
case "macOS":
return "MacIntel"
default:
return "Linux x86_64"
}
}

View File

@@ -0,0 +1,169 @@
package browser
import (
"os"
"path/filepath"
"testing"
)
func TestSelectProfile(t *testing.T) {
tests := []struct {
name string
engine string
region string
wantLocale string
wantTimezone string
}{
{
name: "google ru lane",
engine: "google",
region: "ru",
wantLocale: "ru-RU",
wantTimezone: "Europe/Moscow",
},
{
name: "yandex defaults to ru",
engine: "yandex",
region: "",
wantLocale: "ru-RU",
wantTimezone: "Europe/Moscow",
},
{
name: "google default lane uses us profile",
engine: "google",
region: "en",
wantLocale: "en-US",
wantTimezone: "America/New_York",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
profile := SelectProfile(tt.engine, tt.region)
if profile.ID == "" {
t.Fatal("expected non-empty profile ID")
}
if profile.Locale != tt.wantLocale {
t.Fatalf("expected locale %q, got %q", tt.wantLocale, profile.Locale)
}
if profile.Timezone != tt.wantTimezone {
t.Fatalf("expected timezone %q, got %q", tt.wantTimezone, profile.Timezone)
}
if profile.Platform == "" {
t.Fatal("expected non-empty platform")
}
})
}
}
func TestNormalizeRegion(t *testing.T) {
tests := []struct {
input string
want string
}{
{input: "", want: ""},
{input: "ru", want: "ru"},
{input: "RU", want: "ru"},
{input: "ru-RU", want: "ru"},
{input: "ru_RU", want: "ru"},
{input: "ru-RU,ru;q=0.9", want: "ru"},
{input: "en-US,en;q=0.9", want: "us"},
{input: "de", want: "us"},
}
for _, tt := range tests {
t.Run(tt.input, func(t *testing.T) {
if got := NormalizeRegion(tt.input); got != tt.want {
t.Fatalf("NormalizeRegion(%q) = %q, want %q", tt.input, got, tt.want)
}
})
}
}
func TestLoadProfilesFromJSON(t *testing.T) {
originalCatalog, originalLaneProfiles, originalDefaultRegions := snapshotProfileState()
t.Cleanup(func() {
restoreProfileState(originalCatalog, originalLaneProfiles, originalDefaultRegions)
})
path := filepath.Join(t.TempDir(), "profiles.json")
payload := `{
"profiles": [
{
"id": "custom-ru",
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Chromium", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Chromium", "version": "136.0.0.0"}
],
"platform": "Linux",
"platform_version": "6.0.0",
"architecture": "x86",
"mobile": false,
"accept_language": "ru-RU,ru;q=0.9",
"navigator_langs": ["ru-RU"],
"locale": "ru-RU",
"timezone": "Europe/Moscow",
"viewport": {"width": 1920, "height": 1080}
}
],
"lane_profile_ids": {
"google:ru-RU": "custom-ru"
},
"default_region_by_engine": {
"google": "ru-RU"
}
}`
if err := os.WriteFile(path, []byte(payload), 0o644); err != nil {
t.Fatalf("write profiles json: %v", err)
}
if err := LoadProfilesFromJSON(path); err != nil {
t.Fatalf("load profiles json: %v", err)
}
if got := DefaultRegionForEngine("google"); got != "ru" {
t.Fatalf("expected google default region ru, got %q", got)
}
profile := SelectProfile("google", "")
if profile.ID != "custom-ru" {
t.Fatalf("expected custom profile id custom-ru, got %q", profile.ID)
}
if profile.Timezone != "Europe/Moscow" {
t.Fatalf("expected timezone Europe/Moscow, got %q", profile.Timezone)
}
}
func snapshotProfileState() (map[string]Profile, map[string]string, map[string]string) {
profileCatalogMu.RLock()
defer profileCatalogMu.RUnlock()
catalogCopy := make(map[string]Profile, len(catalog))
for k, v := range catalog {
catalogCopy[k] = v
}
laneProfilesCopy := make(map[string]string, len(laneProfileIDs))
for k, v := range laneProfileIDs {
laneProfilesCopy[k] = v
}
defaultRegionsCopy := make(map[string]string, len(defaultRegionByEngine))
for k, v := range defaultRegionByEngine {
defaultRegionsCopy[k] = v
}
return catalogCopy, laneProfilesCopy, defaultRegionsCopy
}
func restoreProfileState(catalogState map[string]Profile, laneProfiles map[string]string, defaultRegions map[string]string) {
profileCatalogMu.Lock()
defer profileCatalogMu.Unlock()
catalog = catalogState
laneProfileIDs = laneProfiles
defaultRegionByEngine = defaultRegions
}

243
core/browser/profiles.json Normal file
View File

@@ -0,0 +1,243 @@
{
"profiles": [
{
"id": "chrome-win-us",
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{
"brand": "Not_A Brand",
"version": "24"
},
{
"brand": "Chromium",
"version": "136"
},
{
"brand": "Google Chrome",
"version": "136"
}
],
"uach_full_version_list": [
{
"brand": "Not_A Brand",
"version": "24.0.0.0"
},
{
"brand": "Chromium",
"version": "136.0.0.0"
},
{
"brand": "Google Chrome",
"version": "136.0.0.0"
}
],
"platform": "Windows",
"platform_version": "15.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": [
"en-US"
],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {
"width": 1920,
"height": 1080
}
},
{
"id": "chrome-win-ru",
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{
"brand": "Not_A Brand",
"version": "24"
},
{
"brand": "Chromium",
"version": "136"
},
{
"brand": "Google Chrome",
"version": "136"
}
],
"uach_full_version_list": [
{
"brand": "Not_A Brand",
"version": "24.0.0.0"
},
{
"brand": "Chromium",
"version": "136.0.0.0"
},
{
"brand": "Google Chrome",
"version": "136.0.0.0"
}
],
"platform": "Windows",
"platform_version": "15.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
"navigator_langs": [
"ru-RU"
],
"locale": "ru-RU",
"timezone": "Europe/Moscow",
"viewport": {
"width": 1920,
"height": 1080
}
},
{
"id": "chrome-macos-us",
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{
"brand": "Not_A Brand",
"version": "24"
},
{
"brand": "Chromium",
"version": "136"
},
{
"brand": "Google Chrome",
"version": "136"
}
],
"uach_full_version_list": [
{
"brand": "Not_A Brand",
"version": "24.0.0.0"
},
{
"brand": "Chromium",
"version": "136.0.0.0"
},
{
"brand": "Google Chrome",
"version": "136.0.0.0"
}
],
"platform": "macOS",
"platform_version": "14.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": [
"en-US"
],
"locale": "en-US",
"timezone": "America/Los_Angeles",
"viewport": {
"width": 1680,
"height": 1050
}
},
{
"id": "chrome-linux-us",
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{
"brand": "Not_A Brand",
"version": "24"
},
{
"brand": "Chromium",
"version": "136"
},
{
"brand": "Google Chrome",
"version": "136"
}
],
"uach_full_version_list": [
{
"brand": "Not_A Brand",
"version": "24.0.0.0"
},
{
"brand": "Chromium",
"version": "136.0.0.0"
},
{
"brand": "Google Chrome",
"version": "136.0.0.0"
}
],
"platform": "Linux",
"platform_version": "6.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": [
"en-US"
],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {
"width": 1920,
"height": 1080
}
},
{
"id": "chrome-linux-ru",
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{
"brand": "Not_A Brand",
"version": "24"
},
{
"brand": "Chromium",
"version": "136"
},
{
"brand": "Google Chrome",
"version": "136"
}
],
"uach_full_version_list": [
{
"brand": "Not_A Brand",
"version": "24.0.0.0"
},
{
"brand": "Chromium",
"version": "136.0.0.0"
},
{
"brand": "Google Chrome",
"version": "136.0.0.0"
}
],
"platform": "Linux",
"platform_version": "6.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
"navigator_langs": [
"ru-RU"
],
"locale": "ru-RU",
"timezone": "Europe/Moscow",
"viewport": {
"width": 1920,
"height": 1080
}
}
],
"lane_profile_ids": {},
"default_region_by_engine": {
"yandex": "ru"
}
}

View File

@@ -113,15 +113,13 @@ func TestFingerprintDetectors(t *testing.T) {
criticalFailures := make([]string, 0)
for _, detector := range detectors.All() {
for _, useStealth := range []bool{false, true} {
report := runFingerprintDetector(t, detector, useStealth)
key := fmt.Sprintf("%s_%s", detector.Name(), stealthModeLabel(useStealth))
reports[key] = report
report := runFingerprintDetector(t, detector)
key := detector.Name()
reports[key] = report
if len(report.Summary.Critical) > 0 {
for _, critical := range report.Summary.Critical {
criticalFailures = append(criticalFailures, fmt.Sprintf("%s:%s", key, critical))
}
if len(report.Summary.Critical) > 0 {
for _, critical := range report.Summary.Critical {
criticalFailures = append(criticalFailures, fmt.Sprintf("%s:%s", key, critical))
}
}
}
@@ -142,31 +140,28 @@ func TestFingerprintDetectors(t *testing.T) {
}
}
func runFingerprintDetector(t *testing.T, detector fpcheck.Detector, useStealth bool) fpcheck.Report {
func runFingerprintDetector(t *testing.T, detector fpcheck.Detector) fpcheck.Report {
t.Helper()
label := stealthModeLabel(useStealth)
opts := BrowserOpts{
IsHeadless: true,
IsLeakless: false,
Timeout: 20 * time.Second,
UseStealth: useStealth,
}
browser, err := NewBrowser(opts)
if err != nil {
t.Fatalf("create browser (%s): %v", label, err)
t.Fatalf("create browser: %v", err)
}
defer closeTestBrowser(t, browser)
report, err := fpcheck.Run(context.Background(), browser, detector, useStealth, botFingerprintArtifactDir)
report, err := fpcheck.Run(context.Background(), browser, detector, botFingerprintArtifactDir)
if err != nil {
t.Fatalf("run detector %s (%s): %v", detector.Name(), label, err)
t.Fatalf("run detector %s: %v", detector.Name(), err)
}
t.Logf(
"Fingerprint %s (%s): passed=%d failed=%d critical=%d",
"Fingerprint %s: passed=%d failed=%d critical=%d",
detector.Name(),
label,
report.Summary.Passed,
report.Summary.Failed,
len(report.Summary.Critical),
@@ -191,13 +186,6 @@ func writeFingerprintReport(path string, reports map[string]fpcheck.Report) erro
return nil
}
func stealthModeLabel(useStealth bool) string {
if useStealth {
return fpcheck.ModeStealthOn
}
return fpcheck.ModeStealthOff
}
func closeTestBrowser(t *testing.T, browser *Browser) {
t.Helper()
if browser == nil {

View File

@@ -25,7 +25,6 @@ type Summary struct {
type Report struct {
DetectorName string `json:"detector_name"`
URL string `json:"url"`
UseStealth bool `json:"use_stealth"`
CapturedAtUTC string `json:"captured_at_utc"`
Screenshot string `json:"screenshot_path"`
Detections map[string]Detection `json:"detections"`

View File

@@ -36,7 +36,7 @@ func (c Custom) URL() string {
func (c Custom) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
err := waitFor(ctx, 15*time.Second, 200*time.Millisecond, func() (bool, error) {
hasBody, _, err := page.Has("body")
hasBody, _, err := page.Has("pre")
if err != nil {
return false, err
}

View File

@@ -88,6 +88,13 @@ func classifyStatus(status string) bool {
return false
}
if strings.Contains(value, "🔴") {
return true
}
if strings.Contains(value, "🟢") || strings.Contains(value, "⚪") {
return false
}
notDetected := []string{"not detected", "not found", "clean", "clear", "pass", "passed", "ok", "safe", "green", "false", "no"}
for _, marker := range notDetected {
if strings.Contains(value, marker) {

View File

@@ -2,7 +2,9 @@ package detectors
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/go-rod/rod"
@@ -25,6 +27,14 @@ func (Rebrowser) URL() string {
return rebrowserURL
}
type rebrowserCheck struct {
Type string `json:"type"`
Icon string `json:"icon"`
Rating float64 `json:"rating"`
Note string `json:"note"`
Debug string `json:"debug"`
}
func (Rebrowser) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
err := waitFor(ctx, 20*time.Second, 250*time.Millisecond, func() (bool, error) {
hasBody, _, err := page.Has("body")
@@ -36,8 +46,14 @@ func (Rebrowser) Extract(ctx context.Context, page *rod.Page) (map[string]fpchec
}
res, err := page.Eval(`() => {
const text = (document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
return text.includes("runtimeenableleak") || text.includes("sourceurlleak") || text.includes("mainworldexecution") || text.includes("webdriver");
const output = document.querySelector('#detections-json');
if (!output || !output.value) return false;
try {
const parsed = JSON.parse(output.value);
return Array.isArray(parsed) && parsed.length > 0;
} catch (_) {
return false;
}
}`)
if err != nil {
return false, nil
@@ -53,25 +69,124 @@ func (Rebrowser) Extract(ctx context.Context, page *rod.Page) (map[string]fpchec
return nil, "", fmt.Errorf("rebrowser readiness: %w", err)
}
rows, err := parseRows(page)
res, err := page.Eval(`() => {
const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
const stripHTML = (value) => {
const div = document.createElement('div');
div.innerHTML = value || '';
return normalize(div.textContent || div.innerText || '');
};
const checks = new Map();
const put = (item) => {
const type = normalize(item.type);
if (!type) return;
const current = checks.get(type) || { type, icon: "", rating: 0, note: "", debug: "" };
const next = {
type,
icon: normalize(item.icon || current.icon),
rating: Number.isFinite(item.rating) ? item.rating : current.rating,
note: normalize(item.note || current.note),
debug: normalize(item.debug || current.debug),
};
checks.set(type, next);
};
try {
const raw = document.querySelector('#detections-json')?.value || '[]';
const parsed = JSON.parse(raw);
if (Array.isArray(parsed)) {
for (const item of parsed) {
put({
type: item.type || '',
rating: Number(item.rating),
note: stripHTML(item.note || ''),
debug: typeof item.debug === 'string' ? normalize(item.debug) : normalize(JSON.stringify(item.debug || {})),
});
}
}
} catch (_) {}
for (const row of Array.from(document.querySelectorAll('#detections-table tbody tr'))) {
const cells = Array.from(row.querySelectorAll('td'));
if (cells.length < 1) continue;
const rawName = normalize(cells[0].innerText || cells[0].textContent || "");
if (!rawName) continue;
const chars = Array.from(rawName);
const icon = chars.length > 0 ? chars[0] : "";
const type = normalize(rawName.replace(icon, ""));
const note = cells.length > 2 ? normalize(cells[2].innerText || cells[2].textContent || "") : "";
put({
type,
icon,
note,
});
}
return Array.from(checks.values());
}`)
if err != nil {
return nil, "", err
}
if len(rows) == 0 {
return nil, "", fmt.Errorf("rebrowser detector rows not found")
return nil, "", fmt.Errorf("rebrowser extraction failed: %w", err)
}
detections := rowsToDetections(rows, []string{
"runtimeenableleak",
"sourceurlleak",
"mainworldexecution",
"webdriver",
"automation",
})
var checks []rebrowserCheck
if err := res.Value.Unmarshal(&checks); err != nil {
return nil, "", fmt.Errorf("decode rebrowser checks: %w", err)
}
if len(checks) == 0 {
return nil, "", fmt.Errorf("rebrowser detector checks not found")
}
detections := rebrowserChecksToDetections(checks)
if len(detections) == 0 {
return nil, "", fmt.Errorf("rebrowser detections are empty")
}
return detections, "", nil
raw, _ := json.MarshalIndent(checks, "", " ")
return detections, string(raw), nil
}
func rebrowserChecksToDetections(checks []rebrowserCheck) map[string]fpcheck.Detection {
detections := make(map[string]fpcheck.Detection, len(checks))
for _, check := range checks {
key := normalizeKey(check.Type)
if key == "unknown" {
continue
}
detected := false
switch check.Icon {
case "🔴":
detected = true
case "🟢", "🟡", "⚪️", "⚪":
detected = false
default:
detected = check.Rating >= 1
}
description := strings.TrimSpace(check.Note)
if strings.TrimSpace(check.Debug) != "" {
if description != "" {
description = description + " | " + strings.TrimSpace(check.Debug)
} else {
description = strings.TrimSpace(check.Debug)
}
}
if description == "" {
description = fmt.Sprintf("rating=%.2f", check.Rating)
}
severity := ""
if detected {
severity = "critical"
}
detections[key] = fpcheck.Detection{
Detected: detected,
Description: description,
Severity: severity,
}
}
return detections
}

View File

@@ -0,0 +1,38 @@
package detectors
import "testing"
func TestRebrowserChecksToDetections_UsesIconAndRating(t *testing.T) {
checks := []rebrowserCheck{
{
Type: "navigatorWebdriver",
Icon: "🔴",
Rating: -1,
Note: "Own properties detected",
},
{
Type: "runtimeEnableLeak",
Icon: "",
Rating: 1,
Note: "runtime leak",
},
{
Type: "viewport",
Icon: "🟢",
Rating: 1,
Note: "looks fine",
},
}
detections := rebrowserChecksToDetections(checks)
if !detections["navigatorwebdriver"].Detected {
t.Fatal("expected red icon check to be detected")
}
if !detections["runtimeenableleak"].Detected {
t.Fatal("expected rating>=1 check to be detected")
}
if detections["viewport"].Detected {
t.Fatal("expected green icon check to be not detected")
}
}

View File

@@ -13,23 +13,16 @@ import (
"github.com/go-rod/rod/lib/proto"
)
const (
ModeStealthOn = "stealth-on"
ModeStealthOff = "stealth-off"
)
// RunOptions controls detector run behavior.
type RunOptions struct {
UseStealth bool
ArtifactDir string
WaitBeforeClose time.Duration
}
// Run navigates the given browser to detector URL, extracts verdicts,
// captures a screenshot artifact, and returns a normalized report.
func Run(ctx context.Context, browser BrowserNavigator, detector Detector, useStealth bool, artifactDir string) (Report, error) {
func Run(ctx context.Context, browser BrowserNavigator, detector Detector, artifactDir string) (Report, error) {
return RunWithOptions(ctx, browser, detector, RunOptions{
UseStealth: useStealth,
ArtifactDir: artifactDir,
})
}
@@ -41,7 +34,6 @@ func RunWithOptions(ctx context.Context, browser BrowserNavigator, detector Dete
report := Report{
DetectorName: detector.Name(),
URL: detector.URL(),
UseStealth: options.UseStealth,
Detections: map[string]Detection{},
}
@@ -50,7 +42,7 @@ func RunWithOptions(ctx context.Context, browser BrowserNavigator, detector Dete
artifactDir = "testdata"
}
screenshotPath := filepath.Join(artifactDir, fmt.Sprintf("fpcheck_%s_%s.png", sanitizeFilePart(detector.Name()), modeLabel(options.UseStealth)))
screenshotPath := filepath.Join(artifactDir, fmt.Sprintf("fpcheck_%s.png", sanitizeFilePart(detector.Name())))
report.Screenshot = filepath.ToSlash(screenshotPath)
page, err := browser.Navigate(ctx, detector.URL())
@@ -152,13 +144,6 @@ func closePageWithTimeout(ctx context.Context, page *rod.Page, timeout time.Dura
}
}
func modeLabel(useStealth bool) string {
if useStealth {
return ModeStealthOn
}
return ModeStealthOff
}
func sanitizeFilePart(value string) string {
value = strings.TrimSpace(strings.ToLower(value))
if value == "" {

28
core/profile_context.go Normal file
View File

@@ -0,0 +1,28 @@
package core
import (
"context"
"strings"
)
type profileContextKey string
const profileRegionContextKey profileContextKey = "profile_region"
func WithProfileRegion(ctx context.Context, region string) context.Context {
region = strings.TrimSpace(region)
if region == "" {
return EnsureContext(ctx)
}
return context.WithValue(EnsureContext(ctx), profileRegionContextKey, region)
}
func profileRegionFromContext(ctx context.Context) string {
value, _ := EnsureContext(ctx).Value(profileRegionContextKey).(string)
return strings.TrimSpace(value)
}
func engineFromContext(ctx context.Context) string {
value, _ := EnsureContext(ctx).Value(engineContextKey).(string)
return strings.TrimSpace(value)
}

View File

@@ -428,7 +428,6 @@ func (s *Server) handleFingerprintCheck(c *fiber.Ctx) error {
reports := make([]fpcheck.Report, 0, len(req.detectors))
for idx, detector := range req.detectors {
runOpts := fpcheck.RunOptions{
UseStealth: req.browserOpts.UseStealth,
ArtifactDir: artifactDir,
}
if req.waitMs > 0 && idx == len(req.detectors)-1 {
@@ -466,11 +465,6 @@ func (s *Server) parseFingerprintCheckRequest(c *fiber.Ctx) (fingerprintCheckReq
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, err.Error())
}
useStealth, err := parseOptionalBoolQuery(c.Query("stealth", ""), s.opts.FingerprintBrowserOpts.UseStealth)
if err != nil {
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("invalid stealth query value: %v", err))
}
headless, err := parseOptionalBoolQuery(c.Query("headless", ""), true)
if err != nil {
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("invalid headless query value: %v", err))
@@ -491,7 +485,6 @@ func (s *Server) parseFingerprintCheckRequest(c *fiber.Ctx) (fingerprintCheckReq
browserOpts := s.opts.FingerprintBrowserOpts
browserOpts.IsHeadless = headless
browserOpts.UseStealth = useStealth
browserOpts.Timeout = time.Duration(timeoutMs) * time.Millisecond
browserOpts.LeavePageOpen = false
browserOpts.UserAgent = strings.TrimSpace(c.Query("user_agent", browserOpts.UserAgent))

View File

@@ -186,14 +186,14 @@ Defaults below are the shipped defaults in `config.yaml` (if present). If the co
### `app`
| Key | Default | Description |
| ------------------ | ------- | ----------------------------- |
| `app.timeout` | `15` | Request timeout in seconds |
| `app.browser_path` | `""` | Custom browser binary path |
| `app.head` | `false` | Headful browser UI |
| `app.leakless` | `false` | Force browser process cleanup |
| `app.leave_head` | `false` | Keep browser tabs open |
| `app.stealth` | `false` | Enable stealth plugin |
| Key | Default | Description |
| ------------------ | ------- | ------------------------------ |
| `app.timeout` | `15` | Request timeout in seconds |
| `app.browser_path` | `""` | Custom browser binary path |
| `app.profiles` | `""` | Override browser profiles JSON |
| `app.head` | `false` | Headful browser UI |
| `app.leakless` | `false` | Force browser process cleanup |
| `app.leave_head` | `false` | Keep browser tabs open |
### `proxies`

View File

@@ -175,6 +175,7 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.WithEngine(core.EnsureContext(ctx), ddg.Name())
ctx = core.WithProfileRegion(ctx, query.LangCode)
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *ddg
scoped.logger = ddg.logger.WithRequest(ctx)
@@ -298,6 +299,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.WithEngine(core.EnsureContext(ctx), ddg.Name())
ctx = core.WithProfileRegion(ctx, query.LangCode)
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *ddg
scoped.logger = ddg.logger.WithRequest(ctx)

9
go.mod
View File

@@ -7,25 +7,26 @@ toolchain go1.24.6
require (
github.com/2captcha/2captcha-go v1.1.10
github.com/PuerkitoBio/goquery v1.10.3
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
github.com/corpix/uarand v0.2.0
github.com/go-rod/rod v0.116.2
github.com/go-rod/stealth v0.4.9
github.com/gofiber/fiber/v2 v2.52.9
github.com/google/uuid v1.6.0
github.com/refraction-networking/utls v1.8.0
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.9.1
github.com/spf13/pflag v1.0.7
github.com/spf13/viper v1.20.1
github.com/ysmood/gson v0.7.3
golang.org/x/net v0.43.0
golang.org/x/time v0.12.0
)
require (
github.com/andybalholm/brotli v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
@@ -44,10 +45,8 @@ require (
github.com/ysmood/fetchup v0.3.0 // indirect
github.com/ysmood/goob v0.4.0 // indirect
github.com/ysmood/got v0.41.0 // indirect
github.com/ysmood/gson v0.7.3 // indirect
github.com/ysmood/leakless v0.9.0 // indirect
golang.org/x/crypto v0.41.0 // indirect
golang.org/x/net v0.43.0 // indirect
golang.org/x/sys v0.35.0 // indirect
golang.org/x/text v0.28.0 // indirect
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect

7
go.sum
View File

@@ -18,11 +18,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-rod/rod v0.113.0/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA=
github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg=
github.com/go-rod/stealth v0.4.9 h1:X2PmQk4DUF2wzw6GOsWjW/glb8K5ebnftbEvLh7MlZ4=
github.com/go-rod/stealth v0.4.9/go.mod h1:eAzyvw8c0iAd5nJJsSWeh0fQ5z94vCIfdi1hUmYDimc=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/gofiber/fiber/v2 v2.52.9 h1:YjKl5DOiyP3j0mO61u3NTmK7or8GzzWzCFzkboyP5cw=
@@ -89,22 +86,18 @@ github.com/valyala/fasthttp v1.65.0 h1:j/u3uzFEGFfRxw79iYzJN+TteTJwbYkru9uDp3d0Y
github.com/valyala/fasthttp v1.65.0/go.mod h1:P/93/YkKPMsKSnATEeELUCkG8a7Y+k99uxNHVbKINr4=
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
github.com/ysmood/fetchup v0.3.0 h1:UhYz9xnLEVn2ukSuK3KCgcznWpHMdrmbsPpllcylyu8=
github.com/ysmood/fetchup v0.3.0/go.mod h1:hbysoq65PXL0NQeNzUczNYIKpwpkwFL4LXMDEvIQq9A=
github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
github.com/ysmood/gop v0.2.0 h1:+tFrG0TWPxT6p9ZaZs+VY+opCvHU8/3Fk6BaNv6kqKg=
github.com/ysmood/gop v0.2.0/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM=
github.com/ysmood/got v0.41.0 h1:XiFH311ltTSGyxjeKcNvy7dzbJjjTzn6DBgK313JHBs=
github.com/ysmood/got v0.41.0/go.mod h1:W7DdpuX6skL3NszLmAsC5hT7JAhuLZhByVzHTq874Qg=
github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU=
github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=

View File

@@ -174,6 +174,7 @@ func (gogl *Google) acceptCookies(page *rod.Page) {
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.WithEngine(core.EnsureContext(ctx), gogl.Name())
ctx = core.WithProfileRegion(ctx, query.LangCode)
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *gogl
scoped.logger = gogl.logger.WithRequest(ctx)
@@ -427,6 +428,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.WithEngine(core.EnsureContext(ctx), gogl.Name())
ctx = core.WithProfileRegion(ctx, query.LangCode)
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *gogl
scoped.logger = gogl.logger.WithRequest(ctx)

View File

@@ -128,6 +128,7 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.WithEngine(core.EnsureContext(ctx), yand.Name())
ctx = core.WithProfileRegion(ctx, query.LangCode)
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *yand
scoped.logger = yand.logger.WithRequest(ctx)
@@ -227,6 +228,7 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.WithEngine(core.EnsureContext(ctx), yand.Name())
ctx = core.WithProfileRegion(ctx, query.LangCode)
ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
scoped := *yand
scoped.logger = yand.logger.WithRequest(ctx)