From e7827cc03d68edbe5eab0f3ba23cf3c3ff29931f Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Mon, 13 Jul 2026 06:07:37 +0300 Subject: [PATCH] refactor(browser): native CDP fingerprinting, drop patch.js + localized profiles. Update docker chrome --- Dockerfile | 2 +- baidu/search.go | 4 +- baidu/search_raw.go | 2 +- bing/search.go | 4 +- cmd/root.go | 2 +- core/browser.go | 384 ++++++++--------------- core/browser/patch.js | 273 ---------------- core/browser/profile.go | 123 ++++---- core/browser/profile_coherence_test.go | 99 ++++-- core/browser/profile_surface_test.js | 19 ++ core/browser/profile_test.go | 65 +++- core/browser/profiles.json | 411 ++++--------------------- core/browser_unit_test.go | 84 +++++ core/context.go | 5 +- core/http_client.go | 31 +- core/profile_context.go | 10 - core/proxy_lane.go | 3 + core/proxy_lane_test.go | 10 + duckduckgo/search.go | 4 +- ecosia/search.go | 4 +- ecosia/search_raw.go | 2 +- google/search.go | 4 +- google/search_raw.go | 2 +- yandex/search.go | 4 +- yandex/search_raw.go | 2 +- 25 files changed, 531 insertions(+), 1022 deletions(-) delete mode 100644 core/browser/patch.js diff --git a/Dockerfile b/Dockerfile index 284d1ef..81bea43 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ARG TARGETARCH RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build -trimpath -ldflags="-s -w" -o /app/openserp . # `chromedp/headless-shell:stable` also works here -FROM chromedp/headless-shell:stable@sha256:aac539266027f91cf47610da1129dce360d23f45f8f150683cca94223fa2f1e2 +FROM chromedp/headless-shell:stable@sha256:f7e7ac721b023cb8717f8108aef8b3e49995fb1e5a912f41e570c29e45d24961 WORKDIR /usr/src/app diff --git a/baidu/search.go b/baidu/search.go index 4fea7ed..67a4955 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -72,7 +72,7 @@ func (baid *Baidu) classifyBlockPage(page *rod.Page, url string) error { // Search executes a Baidu web search and returns normalized search results. // It may return core.ErrCaptcha or core.ErrSearchTimeout. func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, baid.Name(), true) + ctx = core.PrepareEngineContext(ctx, query, baid.Name()) scoped := *baid scoped.logger = baid.logger.WithRequest(ctx) if scoped.Browser.WaitLoadTime == 0 || scoped.Browser.WaitLoadTime > 250*time.Millisecond { @@ -170,7 +170,7 @@ func (baid *Baidu) waitForParsedSearchResults(ctx context.Context, page *rod.Pag // SearchImage executes a Baidu image search and returns normalized image // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) { - ctx = core.PrepareEngineContext(ctx, query, baid.Name(), true) + ctx = core.PrepareEngineContext(ctx, query, baid.Name()) scoped := *baid scoped.logger = baid.logger.WithRequest(ctx) baid = &scoped diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 78c986d..a772579 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -19,7 +19,7 @@ func classifyBaiduRawHTML(body []byte) error { } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, "baidu", false) + ctx = core.PrepareEngineContext(ctx, query, "baidu") searchURL, err := BuildURL(query) if err != nil { diff --git a/bing/search.go b/bing/search.go index c6fdc85..e8cb4e1 100644 --- a/bing/search.go +++ b/bing/search.go @@ -155,7 +155,7 @@ func (bing *Bing) parseResultElement(el *rod.Element, isAd bool, rank *core.Rank // Search executes a Bing web search and returns normalized search results. // It may return core.ErrCaptcha or core.ErrSearchTimeout. func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, bing.Name(), false) + ctx = core.PrepareEngineContext(ctx, query, bing.Name()) scoped := *bing scoped.logger = bing.logger.WithRequest(ctx) bing = &scoped @@ -256,7 +256,7 @@ func resolveImageLinkElement(container *rod.Element) (*rod.Element, error) { // SearchImage executes a Bing image search and returns normalized image // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) { - ctx = core.PrepareEngineContext(ctx, query, bing.Name(), false) + ctx = core.PrepareEngineContext(ctx, query, bing.Name()) scoped := *bing scoped.logger = bing.logger.WithRequest(ctx) bing = &scoped diff --git a/cmd/root.go b/cmd/root.go index 6cf7754..59c4182 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -17,7 +17,7 @@ import ( ) const ( - version = "0.8.9" + version = "0.8.10" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/browser.go b/core/browser.go index c412e9c..540d8a4 100644 --- a/core/browser.go +++ b/core/browser.go @@ -2,13 +2,13 @@ package core import ( "context" - "encoding/json" "errors" "fmt" "net/http" "net/url" "os" "regexp" + "runtime" "runtime/debug" "strings" "sync" @@ -30,7 +30,7 @@ type BrowserOpts struct { IsLeakless bool // Timeout is applied to browser connect and page navigation operations. Timeout time.Duration - // LanguageCode sets Accept-Language for emulated requests. + // LanguageCode selects the regional browser lane and search locale. LanguageCode string // WaitRequests waits for request-idle state after navigation. WaitRequests bool @@ -279,20 +279,17 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) { return nil, err } - // Create launcher. - // headless=new uses the full Chrome renderer; legacy --headless disables the - // GPU process entirely, making WebGL context creation fail even with SwiftShader. - // use-angle=swiftshader-webgl (Chrome ≥112) enables a software WebGL renderer. - // Rod enables leakless by default, so always pass the configured value - // through. OpenSERP defaults it to false because the helper binary is - // commonly flagged by antivirus on Windows. + // headless=new keeps the full renderer, so WebGL falls back to SwiftShader + // when there's no GPU - which is what the swiftshader profiles describe. + // Legacy --headless kills the GPU process and breaks WebGL entirely. + // Leakless defaults to false: the helper binary trips antivirus on Windows. l := launcher.New().Leakless(opts.IsLeakless). + Set("lang", browserLaunchLanguage(opts)). Set("disable-blink-features", "AutomationControlled"). - Delete("enable-automation"). - Set("use-angle", "swiftshader-webgl"). - Set("ignore-gpu-blocklist") + Delete("enable-automation") if opts.IsHeadless { - l = l.HeadlessNew(true) + // Initial window only; per-page setDeviceMetricsOverride is what pages see. Sized for the 1920x1080 swiftshader profiles that run on Docker. + l = l.HeadlessNew(true).Set("window-size", "1920,1040") } else { l = l.Headless(false) } @@ -365,6 +362,12 @@ func browserOptsLogFields(opts BrowserOpts) logrus.Fields { } } +func browserLaunchLanguage(_ BrowserOpts) string { + // Speech voices are per-process and CDP can't change them, so pin the + // launch locale to the catalog rather than the request hint. + return "en-US" +} + func maskedProxyLogValue(proxyURL string) string { if strings.TrimSpace(proxyURL) == "" { return "" @@ -653,7 +656,6 @@ func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browse profile, ok := browserprofile.ProfileByID(forcedID) if ok { profile = applyRuntimeBrowserVersion(profile, browser) - profile = applyProfileLanguageHint(profile, region) if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" { profile.UserAgent = overrideUA } @@ -663,9 +665,8 @@ func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browse if laneKey := proxyLaneKeyFromContext(ctx); !laneKey.Empty() && b.ProxyLaneStore != nil { profile := b.ProxyLaneStore.Profile(laneKey, func() browserprofile.Profile { - selected := browserprofile.SelectProfileForSession(engine, region, laneKey.SessionID) + selected := browserprofile.SelectProfileForSessionHeadless(engine, region, laneKey.SessionID, b.IsHeadless) selected = applyRuntimeBrowserVersion(selected, browser) - selected = applyProfileLanguageHint(selected, region) if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" { selected.UserAgent = overrideUA } @@ -691,9 +692,8 @@ func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browse // RWMutex-guarded catalog, and applyRuntimeBrowserVersion makes a CDP // round-trip (browser.Version). Holding state.mu over network I/O would // serialize all concurrent Navigate calls. - profile := browserprofile.SelectProfileForSession(engine, region, laneKey) + profile := browserprofile.SelectProfileForSessionHeadless(engine, region, laneKey, b.IsHeadless) profile = applyRuntimeBrowserVersion(profile, browser) - profile = applyProfileLanguageHint(profile, region) if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" { profile.UserAgent = overrideUA } @@ -712,17 +712,46 @@ func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browse return profile, laneKey } +// removeChromeBrand drops the "Google Chrome" client-hint brand from a profile, +// used when the launched binary is Chromium rather than Google Chrome. +func removeChromeBrand(profile browserprofile.Profile) browserprofile.Profile { + profile.UACHBrands = removeBrand(profile.UACHBrands, "google chrome") + profile.UACHFullVerList = removeBrand(profile.UACHFullVerList, "google chrome") + return profile +} + +func removeBrand(values []browserprofile.BrandVersion, unwanted string) []browserprofile.BrandVersion { + out := make([]browserprofile.BrandVersion, 0, len(values)) + for _, value := range values { + if strings.EqualFold(strings.TrimSpace(value.Brand), unwanted) { + continue + } + out = append(out, value) + } + return out +} + func applyRuntimeBrowserVersion(profile browserprofile.Profile, browser *rod.Browser) browserprofile.Profile { fullVersion := "" + product := "" if browser != nil { version, err := browser.Version() if err == nil && version != nil { + product = strings.TrimSpace(version.Product) fullVersion = extractChromeVersion(version.UserAgent) if fullVersion == "" { - fullVersion = extractChromeVersion(version.Product) + fullVersion = extractChromeVersion(product) } } } + + // version.Product tells us the real binary: "Chrome/..." for genuine Chrome, + // "Chromium/..." or "HeadlessChrome/..." otherwise. On non-Chrome builds we + // drop the "Google Chrome" UA-CH brand so the hints match the binary. + return applyRuntimeBrowserVersionValues(profile, product, fullVersion) +} + +func applyRuntimeBrowserVersionValues(profile browserprofile.Profile, product, fullVersion string) browserprofile.Profile { if fullVersion == "" { fullVersion = extractChromeVersion(profile.UserAgent) } @@ -735,12 +764,39 @@ func applyRuntimeBrowserVersion(profile browserprofile.Profile, browser *rod.Bro return profile } - profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, major+".0.0.0") - profile.UACHBrands = patchBrandVersions(profile.UACHBrands, major, false) - profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, fullVersion, true) + if template := strings.TrimSpace(profile.UserAgentTemplate); template != "" { + profile.UserAgent = strings.ReplaceAll(template, "{chrome_major}", major) + } else { + profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, major+".0.0.0") + } + if len(profile.UACHBrands) == 0 { + profile.UACHBrands = runtimeUACHBrands(major, false) + } else { + profile.UACHBrands = patchBrandVersions(profile.UACHBrands, major, false) + } + if len(profile.UACHFullVerList) == 0 { + profile.UACHFullVerList = runtimeUACHBrands(fullVersion, true) + } else { + profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, fullVersion, true) + } + if product != "" && !strings.HasPrefix(product, "Chrome/") { + profile = removeChromeBrand(profile) + } return profile } +func runtimeUACHBrands(version string, full bool) []browserprofile.BrandVersion { + notABrandVersion := "24" + if full { + notABrandVersion = "24.0.0.0" + } + return []browserprofile.BrandVersion{ + {Brand: "Not_A Brand", Version: notABrandVersion}, + {Brand: "Chromium", Version: version}, + {Brand: "Google Chrome", Version: version}, + } +} + func extractChromeVersion(value string) string { matches := chromeVersionPattern.FindStringSubmatch(strings.TrimSpace(value)) if len(matches) < 2 { @@ -808,9 +864,6 @@ type profileDisplayMetrics struct { ViewportHeight int ScreenWidth int ScreenHeight int - AvailWidth int - AvailHeight int - AvailTop int OuterWidth int OuterHeight int PositionX int @@ -849,9 +902,6 @@ func profileDisplayMetricsFor(profile browserprofile.Profile) profileDisplayMetr ViewportHeight: viewportHeight, ScreenWidth: screenWidth, ScreenHeight: screenHeight, - AvailWidth: screenWidth, - AvailHeight: availHeight, - AvailTop: availTop, OuterWidth: screenWidth, OuterHeight: availHeight, PositionX: 0, @@ -897,12 +947,12 @@ func applyProfileLanguageHint(profile browserprofile.Profile, langCode string) b return profile } -func applyProfile(page *rod.Page, profile browserprofile.Profile, minimal bool) error { +func applyProfile(page *rod.Page, profile browserprofile.Profile, headless bool) error { if page == nil { return fmt.Errorf("page is nil") } - navigatorLangs := profileNavigatorLanguages(profile) + navigatorLangs := profileNavigatorLanguagesForRuntime(profile, runtime.GOOS, headless) acceptLanguage := strings.TrimSpace(profile.AcceptLanguage) if acceptLanguage == "" { acceptLanguage = navigatorLangs[0] @@ -913,6 +963,18 @@ func applyProfile(page *rod.Page, profile browserprofile.Profile, minimal bool) } metrics := profileDisplayMetricsFor(profile) + // setWindowBounds can close the target under headless, and there's no real + // window anyway - setDeviceMetricsOverride below covers those dimensions. + if !headless { + if err := page.SetWindow(&proto.BrowserBounds{ + Left: &metrics.PositionX, + Top: &metrics.PositionY, + Width: &metrics.OuterWidth, + Height: &metrics.OuterHeight, + }); err != nil { + logrus.WithError(err).Debug("set browser window unsupported") + } + } metadata := &proto.EmulationUserAgentMetadata{ Brands: toProtoBrandVersions(profile.UACHBrands), @@ -924,9 +986,11 @@ func applyProfile(page *rod.Page, profile browserprofile.Profile, minimal bool) Mobile: profile.Mobile, } + // This only seeds navigator.languages (Chrome strips q-values), so it's the + // plain tag list. The wire header with q-weights is set below. if err := (proto.NetworkSetUserAgentOverride{ UserAgent: strings.TrimSpace(profile.UserAgent), - AcceptLanguage: acceptLanguage, + AcceptLanguage: strings.Join(navigatorLangs, ","), Platform: navigatorPlatformForProfile(profile), UserAgentMetadata: metadata, }).Call(page); err != nil { @@ -958,6 +1022,14 @@ func applyProfile(page *rod.Page, profile browserprofile.Profile, minimal bool) return fmt.Errorf("set device metrics failed: %w", err) } + if err := (proto.EmulationSetEmulatedMedia{Features: []*proto.EmulationMediaFeature{ + {Name: "prefers-reduced-motion", Value: "no-preference"}, + {Name: "prefers-color-scheme", Value: "light"}, + {Name: "forced-colors", Value: "none"}, + }}).Call(page); err != nil { + return fmt.Errorf("set emulated media failed: %w", err) + } + if err := (proto.NetworkSetExtraHTTPHeaders{ Headers: proto.NetworkHeaders{ "Accept-Language": gson.New(acceptLanguage), @@ -966,14 +1038,6 @@ func applyProfile(page *rod.Page, profile browserprofile.Profile, minimal bool) return fmt.Errorf("set extra headers failed: %w", err) } - if minimal { - return nil - } - - if err := evalPatchScript(page, profile, navigatorLangs, metrics); err != nil { - return err - } - return nil } @@ -1023,14 +1087,7 @@ type networkUsageWatcher struct { done chan struct{} } -type workerPatchWatcher struct { - cancel context.CancelFunc - done chan struct{} - page *rod.Page -} - var pageNetworkUsageWatchers sync.Map -var pageWorkerPatchWatchers sync.Map func startMainDocumentStatusWatcher(ctx context.Context, page *rod.Page) *mainDocumentStatusWatcher { watchCtx, cancel := context.WithCancel(EnsureContext(ctx)) @@ -1098,95 +1155,6 @@ func startNetworkUsageWatcher(ctx context.Context, page *rod.Page) *networkUsage return watcher } -func startWorkerPatchWatcher(ctx context.Context, page *rod.Page, script string) (*workerPatchWatcher, error) { - if page == nil || strings.TrimSpace(script) == "" { - return nil, nil - } - - watchCtx, cancel := context.WithCancel(EnsureContext(ctx)) - watcher := &workerPatchWatcher{ - cancel: cancel, - done: make(chan struct{}), - page: page, - } - - scopedPage := page.Context(watchCtx) - started := make(chan struct{}) - go func() { - defer close(watcher.done) - wait := scopedPage.EachEvent(func(e *proto.TargetAttachedToTarget) bool { - if e == nil || e.SessionID == "" { - return false - } - go injectWorkerPatch(watchCtx, scopedPage.Browser(), e, script) - return false - }) - close(started) - wait() - }() - - <-started - if err := (proto.TargetSetAutoAttach{ - AutoAttach: true, - WaitForDebuggerOnStart: true, - Flatten: true, - Filter: workerTargetFilter(), - }).Call(scopedPage); err != nil { - cancel() - <-watcher.done - return nil, fmt.Errorf("enable worker auto-attach: %w", err) - } - - return watcher, nil -} - -func workerTargetFilter() proto.TargetTargetFilter { - return proto.TargetTargetFilter{ - {Type: "worker"}, - {Type: string(proto.TargetTargetInfoTypeSharedWorker)}, - {Type: string(proto.TargetTargetInfoTypeServiceWorker)}, - } -} - -func injectWorkerPatch(ctx context.Context, browser *rod.Browser, e *proto.TargetAttachedToTarget, script string) { - if browser == nil || e == nil || e.SessionID == "" { - return - } - - injectCtx, cancel := context.WithTimeout(EnsureContext(ctx), 3*time.Second) - defer cancel() - - if isPatchableWorkerTarget(e.TargetInfo) { - eval := proto.RuntimeEvaluate{ - Expression: script, - Silent: true, - AllowUnsafeEvalBlockedByCSP: true, - } - if _, err := browser.Call(injectCtx, string(e.SessionID), eval.ProtoReq(), eval); err != nil && !errors.Is(err, context.Canceled) { - logrus.WithError(err).Debug("Worker profile patch failed") - } - } - - if e.WaitingForDebugger { - run := proto.RuntimeRunIfWaitingForDebugger{} - if _, err := browser.Call(injectCtx, string(e.SessionID), run.ProtoReq(), run); err != nil && !errors.Is(err, context.Canceled) { - logrus.WithError(err).Debug("Resume worker after profile patch failed") - } - } -} - -func isPatchableWorkerTarget(info *proto.TargetTargetInfo) bool { - if info == nil { - return true - } - switch string(info.Type) { - case "worker", string(proto.TargetTargetInfoTypeSharedWorker), string(proto.TargetTargetInfoTypeServiceWorker): - return true - default: - return false - } -} - func rememberNetworkUsageWatcher(page *rod.Page, watcher *networkUsageWatcher) { if page == nil || watcher == nil { return @@ -1194,13 +1162,6 @@ func rememberNetworkUsageWatcher(page *rod.Page, watcher *networkUsageWatcher) { pageNetworkUsageWatchers.Store(page, watcher) } -func rememberWorkerPatchWatcher(page *rod.Page, watcher *workerPatchWatcher) { - if page == nil || watcher == nil { - return - } - pageWorkerPatchWatchers.Store(page, watcher) -} - func stopNetworkUsageWatcher(page *rod.Page) { if page == nil { return @@ -1214,19 +1175,6 @@ func stopNetworkUsageWatcher(page *rod.Page) { } } -func stopWorkerPatchWatcher(page *rod.Page) { - if page == nil { - return - } - raw, ok := pageWorkerPatchWatchers.LoadAndDelete(page) - if !ok { - return - } - if watcher, ok := raw.(*workerPatchWatcher); ok { - watcher.Stop() - } -} - func (w *networkUsageWatcher) Stop() { if w == nil { return @@ -1238,27 +1186,6 @@ func (w *networkUsageWatcher) Stop() { } } -func (w *workerPatchWatcher) Stop() { - if w == nil { - return - } - if w.page != nil { - disableCtx, cancel := context.WithTimeout(context.Background(), time.Second) - _ = (proto.TargetSetAutoAttach{ - AutoAttach: false, - WaitForDebuggerOnStart: false, - Flatten: true, - Filter: workerTargetFilter(), - }).Call(w.page.Context(disableCtx)) - cancel() - } - w.cancel() - select { - case <-w.done: - case <-time.After(100 * time.Millisecond): - } -} - func (w *mainDocumentStatusWatcher) Status() int { if w == nil { return 0 @@ -1281,21 +1208,22 @@ func classifyMainDocumentStatus(status int) error { func profileNavigatorLanguages(profile browserprofile.Profile) []string { langs := make([]string, 0, len(profile.NavigatorLangs)) + seen := make(map[string]struct{}, len(profile.NavigatorLangs)) for _, language := range profile.NavigatorLangs { trimmed := strings.TrimSpace(language) if trimmed == "" { continue } + if _, ok := seen[trimmed]; ok { + continue + } + seen[trimmed] = struct{}{} langs = append(langs, trimmed) } - if len(langs) > 0 { - return langs - } acceptLanguage := strings.TrimSpace(profile.AcceptLanguage) if acceptLanguage != "" { parts := strings.Split(acceptLanguage, ",") - langs = make([]string, 0, len(parts)) for _, part := range parts { part = strings.TrimSpace(part) if part == "" { @@ -1305,12 +1233,16 @@ func profileNavigatorLanguages(profile browserprofile.Profile) []string { part = strings.TrimSpace(part[:idx]) } if part != "" { + if _, ok := seen[part]; ok { + continue + } + seen[part] = struct{}{} langs = append(langs, part) } } - if len(langs) > 0 { - return langs - } + } + if len(langs) > 0 { + return langs } if locale := strings.TrimSpace(profile.Locale); locale != "" { @@ -1319,6 +1251,15 @@ func profileNavigatorLanguages(profile browserprofile.Profile) []string { return []string{"en-US"} } +func profileNavigatorLanguagesForRuntime(profile browserprofile.Profile, goos string, headless bool) []string { + langs := profileNavigatorLanguages(profile) + // Linux headless-shell keeps workers at the process locale only. + if headless && goos == "linux" { + return langs[:1] + } + return langs +} + func toProtoBrandVersions(values []browserprofile.BrandVersion) []*proto.EmulationUserAgentBrandVersion { out := make([]*proto.EmulationUserAgentBrandVersion, 0, len(values)) for _, value := range values { @@ -1335,60 +1276,6 @@ func toProtoBrandVersions(values []browserprofile.BrandVersion) []*proto.Emulati return out } -func buildProfilePatchScript(profile browserprofile.Profile, langs []string, metrics profileDisplayMetrics) (string, error) { - langsJSON, err := json.Marshal(langs) - if err != nil { - return "", fmt.Errorf("marshal navigator languages: %w", err) - } - - webGLVendor := strings.TrimSpace(profile.WebGLVendor) - if webGLVendor == "" { - webGLVendor = "Intel Inc." - } - webGLRenderer := strings.TrimSpace(profile.WebGLRenderer) - if webGLRenderer == "" { - webGLRenderer = "Intel Iris OpenGL Engine" - } - - webGLVendorJSON, err := json.Marshal(webGLVendor) - if err != nil { - return "", fmt.Errorf("marshal webgl vendor: %w", err) - } - webGLRendererJSON, err := json.Marshal(webGLRenderer) - if err != nil { - return "", fmt.Errorf("marshal webgl renderer: %w", err) - } - - return fmt.Sprintf("(() => {\nconst __langs = %s;\nconst __w = %d;\nconst __h = %d;\nconst __screenW = %d;\nconst __screenH = %d;\nconst __availW = %d;\nconst __availH = %d;\nconst __availTop = %d;\nconst __outerW = %d;\nconst __outerH = %d;\nconst __webglVendor = %s;\nconst __webglRenderer = %s;\n%s\n})();", - string(langsJSON), - metrics.ViewportWidth, - metrics.ViewportHeight, - metrics.ScreenWidth, - metrics.ScreenHeight, - metrics.AvailWidth, - metrics.AvailHeight, - metrics.AvailTop, - metrics.OuterWidth, - metrics.OuterHeight, - string(webGLVendorJSON), - string(webGLRendererJSON), - string(browserprofile.PatchJS), - ), nil -} - -func evalPatchScript(page *rod.Page, profile browserprofile.Profile, langs []string, metrics profileDisplayMetrics) error { - script, err := buildProfilePatchScript(profile, langs, metrics) - if err != nil { - return err - } - - _, err = page.EvalOnNewDocument(script) - if err != nil { - return fmt.Errorf("eval patch script: %w", err) - } - return nil -} - // Navigate connects to Chromium, creates a page, applies a coherent profile and // proxy auth, then navigates to URL. It returns an initialized page ready for // selector queries, or an error when browser setup/navigation fails. @@ -1437,7 +1324,6 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) { // context first causes Chrome to kill the page target before our Close call, // producing a spurious "target closed" error on the page.Close() that follows. closeOnErr := func() { - stopWorkerPatchWatcher(page) stopNetworkUsageWatcher(page) if cerr := page.Close(); cerr != nil && !isBrowserClosedError(cerr) { WithRequest(ctx).WithError(cerr).Debug("Close page after navigate error failed") @@ -1450,7 +1336,6 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) { // background watchers attached to it must still stop or they leak goroutines. stopWatchersOnErr := func() { if b.LeavePageOpen { - stopWorkerPatchWatcher(page) stopNetworkUsageWatcher(page) } else { closeOnErr() @@ -1459,12 +1344,10 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) { profile, laneKey := b.laneProfile(ctx, browser) SetBrowserProfileID(ctx, profile.ID) - minimalProfile := minimalBrowserProfileFromContext(ctx) WithRequest(ctx).WithFields(logrus.Fields{ - "lane_id": laneKey, - "minimal_profile": minimalProfile, + "lane_id": laneKey, }).Info("Browser profile selected") - if err := applyProfile(page, profile, minimalProfile); err != nil { + if err := applyProfile(page, profile, b.IsHeadless); err != nil { closeOnErr() return nil, fmt.Errorf("apply profile %s (%s) failed: %w", profile.ID, laneKey, err) } @@ -1474,20 +1357,6 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) { } page = page.Context(ctx) - if !minimalProfile { - metrics := profileDisplayMetricsFor(profile) - patchScript, err := buildProfilePatchScript(profile, profileNavigatorLanguages(profile), metrics) - if err != nil { - closeOnErr() - return nil, err - } - workerPatchWatcher, err := startWorkerPatchWatcher(ctx, page, patchScript) - if err != nil { - closeOnErr() - return nil, err - } - rememberWorkerPatchWatcher(page, workerPatchWatcher) - } if err := b.configureRequestBlocking(ctx, page); err != nil { closeOnErr() return nil, fmt.Errorf("configure request blocking failed: %w", err) @@ -1603,7 +1472,6 @@ func ClosePageWithTimeout(ctx context.Context, page *rod.Page, timeout time.Dura if page == nil { return nil } - stopWorkerPatchWatcher(page) stopNetworkUsageWatcher(page) if timeout <= 0 { timeout = time.Second diff --git a/core/browser/patch.js b/core/browser/patch.js deleted file mode 100644 index fa3529d..0000000 --- a/core/browser/patch.js +++ /dev/null @@ -1,273 +0,0 @@ -// Stealth patches injected via EvalOnNewDocument. -// Arguments are injected as a leading const block by the Go caller: -// const __langs = [...]; // navigator_langs from profile -// const __w = 1920; // viewport/content width -// const __h = 955; // viewport/content height -// const __screenW = 1920; // screen.width -// const __screenH = 1080; // screen.height -// const __availW = 1920; // screen.availWidth -// const __availH = 1040; // screen.availHeight -// const __availTop = 0; // screen.availTop -// const __outerW = 1920; // window.outerWidth -// const __outerH = 1040; // window.outerHeight -// const __webglVendor = "..."; // UNMASKED_VENDOR_WEBGL spoof -// const __webglRenderer = "..."; // UNMASKED_RENDERER_WEBGL spoof -// -// Scope: only patches that fix detectors with high ROI and low introspection -// surface. Notably absent: Function.prototype.toString proxy, custom Worker -// constructor, mass plugin/mimeType arrays, iframe contentWindow patching. -// Those triggered server-side detection on Google in commit 612e0dc. -(() => { - 'use strict'; - - const sealGetter = (target, prop, fn) => { - try { - Object.defineProperty(target, prop, { - get: fn, - set: undefined, - enumerable: true, - configurable: true, - }); - Object.defineProperty(target, prop, { configurable: false }); - } catch (_) {} - }; - - // --- navigator.webdriver --- - // headless Chrome sets this to true. Delete it so getter returns undefined. - // (We rely on `disable-blink-features=AutomationControlled` already turning - // this off via the launcher; this is belt-and-braces in case it leaks.) - try { - if (typeof navigator.webdriver !== 'undefined') { - delete Object.getPrototypeOf(navigator).webdriver; - } - } catch (_) {} - - // --- navigator.language / navigator.languages --- - // CDP setUserAgentOverride sets the HTTP header but not these JS props. - const primary = __langs.length ? __langs[0] : 'en-US'; - const patchLangs = (target) => { - if (!target) return; - sealGetter(target, 'language', () => primary); - sealGetter(target, 'languages', () => Object.freeze(__langs.slice())); - }; - patchLangs(Object.getPrototypeOf(navigator)); - if (typeof WorkerNavigator !== 'undefined') { - patchLangs(WorkerNavigator.prototype); - } - - // --- screen dimensions --- - // EmulationSetDeviceMetricsOverride sets the CSS viewport but leaves - // window.screen.* at headless defaults. - const screenProto = typeof Screen !== 'undefined' ? Screen.prototype : null; - if (screenProto) { - sealGetter(screenProto, 'width', () => __screenW); - sealGetter(screenProto, 'height', () => __screenH); - sealGetter(screenProto, 'availWidth', () => __availW); - sealGetter(screenProto, 'availHeight', () => __availH); - sealGetter(screenProto, 'availLeft', () => 0); - sealGetter(screenProto, 'availTop', () => __availTop); - // Headless reports colorDepth/pixelDepth=24 already in most builds, but - // some checks see 0 in WSL/Docker. Lock to 24 which matches real Chrome. - sealGetter(screenProto, 'colorDepth', () => 24); - sealGetter(screenProto, 'pixelDepth', () => 24); - } - if (typeof window !== 'undefined') { - sealGetter(window, 'outerWidth', () => __outerW); - sealGetter(window, 'outerHeight', () => __outerH); - } - - // --- WebGL vendor/renderer spoof --- - // Patches getParameter(37445=UNMASKED_VENDOR_WEBGL, 37446=UNMASKED_RENDERER_WEBGL) - // on WebGLRenderingContext.prototype and WebGL2RenderingContext.prototype. - // Only delegates to the original for all other params, so behavioral - // signals (extension list, real pixel rendering) still pass through. - const patchWebGLProto = (proto) => { - if (!proto) return; - const original = proto.getParameter; - if (typeof original !== 'function') return; - const replacement = function getParameter(parameter) { - if (parameter === 37445) return __webglVendor; - if (parameter === 37446) return __webglRenderer; - return original.apply(this, arguments); - }; - try { - Object.defineProperty(proto, 'getParameter', { - value: replacement, - writable: true, - enumerable: false, - configurable: true, - }); - } catch (_) {} - }; - if (typeof WebGLRenderingContext !== 'undefined') { - patchWebGLProto(WebGLRenderingContext.prototype); - } - if (typeof WebGL2RenderingContext !== 'undefined') { - patchWebGLProto(WebGL2RenderingContext.prototype); - } - - // --- navigator.plugins / mimeTypes --- - // Match real Chrome 136 exactly: 5 plugins (all "internal-pdf-viewer"), each - // exposing application/pdf + text/pdf. navigator.mimeTypes dedupes to 2. - // Each MimeType.enabledPlugin must back-reference the FIRST plugin that owns - // that type (Chrome's invariant). Plugin order is fixed. - try { - const PluginArrayProto = typeof PluginArray !== 'undefined' ? PluginArray.prototype : null; - const PluginProto = typeof Plugin !== 'undefined' ? Plugin.prototype : null; - const MimeTypeArrayProto = typeof MimeTypeArray !== 'undefined' ? MimeTypeArray.prototype : null; - const MimeTypeProto = typeof MimeType !== 'undefined' ? MimeType.prototype : null; - - if (PluginArrayProto && PluginProto && MimeTypeArrayProto && MimeTypeProto) { - const pluginNames = [ - 'PDF Viewer', - 'Chrome PDF Viewer', - 'Chromium PDF Viewer', - 'Microsoft Edge PDF Viewer', - 'WebKit built-in PDF', - ]; - const mimeSpecs = [ - { type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format' }, - { type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format' }, - ]; - - // Two MimeType instances, each enabledPlugin points to the first plugin - // (PDF Viewer) per Chrome's invariant: navigator.mimeTypes[i].enabledPlugin - // === navigator.plugins[0] for every PDF mime. - const sharedMimes = mimeSpecs.map((spec) => { - const m = Object.create(MimeTypeProto); - Object.defineProperty(m, 'type', { value: spec.type, enumerable: true }); - Object.defineProperty(m, 'suffixes', { value: spec.suffixes, enumerable: true }); - Object.defineProperty(m, 'description', { value: spec.description, enumerable: true }); - return m; - }); - - const plugins = pluginNames.map((name) => { - const p = Object.create(PluginProto); - Object.defineProperty(p, 'name', { value: name, enumerable: true }); - Object.defineProperty(p, 'filename', { value: 'internal-pdf-viewer', enumerable: true }); - Object.defineProperty(p, 'description', { value: 'Portable Document Format', enumerable: true }); - Object.defineProperty(p, 'length', { value: sharedMimes.length, enumerable: true }); - sharedMimes.forEach((m, i) => { - Object.defineProperty(p, String(i), { value: m, enumerable: true }); - Object.defineProperty(p, m.type, { value: m }); - }); - return p; - }); - - // Set enabledPlugin AFTER plugins are constructed, pointing to plugins[0]. - sharedMimes.forEach((m) => { - Object.defineProperty(m, 'enabledPlugin', { value: plugins[0], enumerable: true }); - }); - - const pluginArr = Object.create(PluginArrayProto); - Object.defineProperty(pluginArr, 'length', { value: plugins.length, enumerable: true }); - plugins.forEach((p, i) => { - Object.defineProperty(pluginArr, String(i), { value: p, enumerable: true }); - Object.defineProperty(pluginArr, p.name, { value: p }); - }); - - const mimeArr = Object.create(MimeTypeArrayProto); - Object.defineProperty(mimeArr, 'length', { value: sharedMimes.length, enumerable: true }); - sharedMimes.forEach((m, i) => { - Object.defineProperty(mimeArr, String(i), { value: m, enumerable: true }); - Object.defineProperty(mimeArr, m.type, { value: m }); - }); - - sealGetter(Object.getPrototypeOf(navigator), 'plugins', () => pluginArr); - sealGetter(Object.getPrototypeOf(navigator), 'mimeTypes', () => mimeArr); - } - } catch (_) {} - - // --- navigator.permissions.query notifications fix --- - // headless returns 'denied' for notifications when Notification.permission is - // 'default'. Real Chrome returns 'prompt' in that case. Sannysoft checks this - // mismatch (permissions_new / headchr_permissions). - try { - if (navigator.permissions && typeof navigator.permissions.query === 'function') { - const proto = Object.getPrototypeOf(navigator.permissions); - const desc = Object.getOwnPropertyDescriptor(proto, 'query'); - if (desc && typeof desc.value === 'function') { - const original = desc.value; - const replacement = function query(parameters) { - if (parameters && parameters.name === 'notifications' && - typeof Notification !== 'undefined' && Notification.permission === 'default') { - return Promise.resolve({ state: 'prompt', onchange: null }); - } - return original.apply(this, arguments); - }; - Object.defineProperty(proto, 'query', { - value: replacement, - writable: desc.writable, - enumerable: desc.enumerable, - configurable: desc.configurable, - }); - } - } - } catch (_) {} - - // --- getBoundingClientRect / getClientRects subpixel jitter --- - // Headless Chrome returns integer-valued rects; real Chrome returns subpixel - // floats due to CSS layout fractions. Fingerprinters hash rect tuples; even - // a sub-pixel offset breaks the canonical "headless rect" hash. - // Jitter is deterministic per-element (based on element identity) so the - // same element returns the same value across calls within the page lifetime. - try { - const rectProto = typeof DOMRect !== 'undefined' ? DOMRect.prototype : null; - const elProto = typeof Element !== 'undefined' ? Element.prototype : null; - if (rectProto && elProto) { - const wmJitter = new WeakMap(); - const jitterFor = (el) => { - let j = wmJitter.get(el); - if (!j) { - // Tiny noise in [-0.05, +0.05) — well below visual threshold but - // changes hash output. Generated once per element. - j = { - x: (Math.random() - 0.5) * 0.1, - y: (Math.random() - 0.5) * 0.1, - }; - wmJitter.set(el, j); - } - return j; - }; - const origGBCR = elProto.getBoundingClientRect; - Object.defineProperty(elProto, 'getBoundingClientRect', { - value: function getBoundingClientRect() { - const r = origGBCR.apply(this, arguments); - const j = jitterFor(this); - // DOMRect is mutable; nudge x/y. width/height left intact so layout - // calculations don't drift. - try { r.x = r.x + j.x; r.y = r.y + j.y; } catch (_) {} - return r; - }, - writable: true, - enumerable: false, - configurable: true, - }); - } - } catch (_) {} - - // --- window.chrome.runtime stub --- - // Real Chrome exposes window.chrome with a .runtime sub-object. - // headless leaves window.chrome empty, which sannysoft (chrome_new, - // headchr_chrome_obj) flags. A minimal runtime stub satisfies the check - // without touching method behavior. - try { - if (typeof window !== 'undefined') { - if (!window.chrome) { - Object.defineProperty(window, 'chrome', { value: {}, writable: true, configurable: true }); - } - if (window.chrome && !window.chrome.runtime) { - Object.defineProperty(window.chrome, 'runtime', { - value: { - OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', UPDATE: 'update' }, - OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' }, - PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', WIN: 'win' }, - }, - writable: true, - enumerable: true, - configurable: true, - }); - } - } - } catch (_) {} -})(); diff --git a/core/browser/profile.go b/core/browser/profile.go index f97b3bc..3517417 100644 --- a/core/browser/profile.go +++ b/core/browser/profile.go @@ -23,24 +23,23 @@ type Viewport struct { } type Profile struct { - ID string `json:"id"` - UserAgent string `json:"user_agent"` - UACHBrands []BrandVersion `json:"uach_brands"` - UACHFullVerList []BrandVersion `json:"uach_full_version_list"` - Platform string `json:"platform"` - PlatformVersion string `json:"platform_version"` - Architecture string `json:"architecture"` - Bitness string `json:"bitness"` - Mobile bool `json:"mobile"` - AcceptLanguage string `json:"accept_language"` - NavigatorLangs []string `json:"navigator_langs"` - Locale string `json:"locale"` - Timezone string `json:"timezone"` - Viewport Viewport `json:"viewport"` - WebGLVendor string `json:"webgl_vendor"` - WebGLRenderer string `json:"webgl_renderer"` - Tags []string `json:"tags"` - Weight int `json:"weight"` + ID string `json:"id"` + UserAgentTemplate string `json:"user_agent_template"` + UserAgent string `json:"user_agent,omitempty"` + UACHBrands []BrandVersion `json:"uach_brands,omitempty"` + UACHFullVerList []BrandVersion `json:"uach_full_version_list,omitempty"` + Platform string `json:"platform"` + PlatformVersion string `json:"platform_version"` + Architecture string `json:"architecture"` + Bitness string `json:"bitness"` + Mobile bool `json:"mobile"` + AcceptLanguage string `json:"accept_language"` + NavigatorLangs []string `json:"navigator_langs"` + Locale string `json:"locale"` + Timezone string `json:"timezone"` + Viewport Viewport `json:"viewport"` + Tags []string `json:"tags"` + Weight int `json:"weight"` } type catalogConfig struct { @@ -51,18 +50,13 @@ type catalogConfig struct { const ( ProfileChromeWinUS = "chrome-win-uhd620" - ProfileChromeWinRU = "chrome-win-ru" ProfileChromeMacUS = "chrome-macos-intel-iris" ProfileChromeLinuxUS = "chrome-linux-mesa-uhd620" - ProfileChromeLinuxRU = "chrome-linux-ru" ) //go:embed profiles.json var defaultProfilesJSON []byte -//go:embed patch.js -var PatchJS []byte - var profileCatalogMu sync.RWMutex var catalog = map[string]Profile{} @@ -110,8 +104,8 @@ func loadProfilesFromJSONBytes(data []byte) error { if profile.ID == "" { return fmt.Errorf("profiles[%d].id is empty", i) } - if strings.TrimSpace(profile.UserAgent) == "" { - return fmt.Errorf("profiles[%d].user_agent is empty", i) + if strings.TrimSpace(profile.UserAgentTemplate) == "" && strings.TrimSpace(profile.UserAgent) == "" { + return fmt.Errorf("profiles[%d] has no user_agent_template or user_agent", i) } if _, exists := nextCatalog[profile.ID]; exists { return fmt.Errorf("duplicate profile id %q", profile.ID) @@ -201,6 +195,13 @@ func SelectProfile(engine string, region string) Profile { // Non-empty salt uses weighted selection seeded by FNV-1a hash of salt, // giving each session a stable but varied profile. func SelectProfileForSession(engine, region, salt string) Profile { + return SelectProfileForSessionHeadless(engine, region, salt, false) +} + +// SelectProfileForSessionHeadless is SelectProfileForSession, but headless Linux +// (the Docker deployment) renders WebGL via SwiftShader, so it only picks +// swiftshader-tagged profiles. Runtimes with a real GPU exclude them. +func SelectProfileForSessionHeadless(engine, region, salt string, headless bool) Profile { engine = NormalizeEngine(engine) region = NormalizeRegion(region) if region == "" { @@ -215,7 +216,7 @@ func SelectProfileForSession(engine, region, salt string) Profile { return profileByID(profileID) } - pool := eligibleProfiles(engine, region) + pool := eligibleProfiles(runtime.GOOS, headless) return pickWeighted(pool, salt) } @@ -224,10 +225,11 @@ type weightedProfile struct { weight int } -// eligibleProfiles builds the weighted pool for (engine, region). -// Linux profiles are preferred 4x on linux runtime; Windows 4x on windows; macOS 4x on darwin. -// Profiles tagged "ru" are included only when region == "ru"; "ru"-tagged profiles are excluded otherwise. -func eligibleProfiles(engine, region string) []weightedProfile { +// eligibleProfiles builds the pool for the runtime platform. Headless Linux +// keeps only swiftshader-tagged profiles (SwiftShader WebGL); a real GPU +// excludes them. We no longer spoof WebGL, so the GPU sub-tag (nvidia/amd/mesa) +// only steers selection - it matches reality on Docker, cosmetic on a headful box. +func eligibleProfiles(goos string, headless bool) []weightedProfile { profileCatalogMu.RLock() snap := make([]Profile, 0, len(catalog)) for _, p := range catalog { @@ -240,50 +242,52 @@ func eligibleProfiles(engine, region string) []weightedProfile { return strings.Compare(a.ID, b.ID) }) - goos := runtime.GOOS + platformTag := runtimePlatformTag(goos) + wantSwiftShader := headless && goos == "linux" var pool []weightedProfile for _, p := range snap { - isRu := slices.Contains(p.Tags, "ru") - if region == "ru" && !isRu { + if platformTag != "" && !slices.Contains(p.Tags, platformTag) { continue } - if region != "ru" && isRu { + if slices.Contains(p.Tags, "swiftshader") != wantSwiftShader { continue } - w := p.Weight if w <= 0 { w = 1 } - platformLower := strings.ToLower(p.Platform) - switch goos { - case "linux": - if platformLower == "linux" { - w *= 4 - } - case "windows": - if platformLower == "windows" { - w *= 4 - } - case "darwin": - if platformLower == "macos" { - w *= 4 - } - } - pool = append(pool, weightedProfile{profile: p, weight: w}) } + // Fall back to the platform pool if no swiftshader profile exists yet, so + // selection never returns empty. + if len(pool) == 0 && wantSwiftShader { + return eligibleProfiles(goos, false) + } + return pool } +func runtimePlatformTag(goos string) string { + switch strings.ToLower(strings.TrimSpace(goos)) { + case "windows": + return "windows" + case "darwin": + return "macos" + case "linux": + return "linux" + default: + return "" + } +} + // pickWeighted selects a profile from pool using FNV-1a hash of salt modulo total weight. // Empty salt returns the first profile in the pool (deterministic for tests). func pickWeighted(pool []weightedProfile, salt string) Profile { if len(pool) == 0 { - return profileByID(defaultProfileID("us")) + return profileByID(defaultProfileID()) } if salt == "" { return pool[0].profile @@ -384,7 +388,7 @@ func profileByID(profileID string) Profile { if profile, ok := catalog[profileID]; ok { return profile } - if fallback, ok := catalog[defaultProfileID("us")]; ok { + if fallback, ok := catalog[defaultProfileID()]; ok { return fallback } for _, profile := range catalog { @@ -393,24 +397,13 @@ func profileByID(profileID string) Profile { return Profile{} } -func defaultProfileID(region string) string { - region = NormalizeRegion(region) - if region == "" { - region = "us" - } - +func defaultProfileID() string { switch runtime.GOOS { case "windows": - if region == "ru" { - return ProfileChromeWinRU - } return ProfileChromeWinUS case "darwin": return ProfileChromeMacUS default: - if region == "ru" { - return ProfileChromeLinuxRU - } return ProfileChromeLinuxUS } } diff --git a/core/browser/profile_coherence_test.go b/core/browser/profile_coherence_test.go index bf3e5df..c63c1c1 100644 --- a/core/browser/profile_coherence_test.go +++ b/core/browser/profile_coherence_test.go @@ -51,12 +51,12 @@ func TestProfileCoherence(t *testing.T) { region string }{ { - name: "windows lane", + name: "ru lane", engine: "google", region: "ru", }, { - name: "mac lane", + name: "us lane", engine: "bing", region: "en-US", }, @@ -84,7 +84,7 @@ func TestProfileCoherence(t *testing.T) { } expected := selectedProfileFromContext(t, ctx) - expected.UserAgent = expectedUserAgentForRuntime(expected.UserAgent, got.UserAgent) + expected.UserAgent = expectedUserAgentForRuntime(expectedProfileUserAgent(expected, got.UserAgent), got.UserAgent) if got.UserAgent != expected.UserAgent { t.Fatalf("navigator.userAgent mismatch:\nexpected: %s\nactual: %s", expected.UserAgent, got.UserAgent) } @@ -106,11 +106,26 @@ func TestProfileCoherence(t *testing.T) { if got.NavigatorLanguages[0] != expected.NavigatorLangs[0] { t.Fatalf("navigator.languages[0] mismatch: expected %q got %q", expected.NavigatorLangs[0], got.NavigatorLanguages[0]) } - if got.WebdriverType != "undefined" { - t.Fatalf("navigator.webdriver expected undefined, got %q", got.WebdriverType) + // Real Chrome exposes navigator.webdriver as an inherited boolean + // that reads false; --disable-blink-features=AutomationControlled + // gives us that. Deleting the property (typeof undefined) is itself a + // bot tell, so we assert the genuine-browser shape instead. + if got.WebdriverType != "boolean" { + t.Fatalf("navigator.webdriver expected boolean, got %q", got.WebdriverType) + } + if got.WebdriverValue { + t.Fatal("navigator.webdriver should read false") } if got.WebdriverOwnPropPresent { - t.Fatal("navigator own property 'webdriver' should not be present") + t.Fatal("navigator.webdriver should be inherited, not an own property") + } + // hardwareConcurrency is left native (no override), so main and + // worker must simply agree on the machine's real core count. + if got.HardwareConcurrency <= 0 { + t.Fatal("navigator.hardwareConcurrency should be reported") + } + if got.WorkerHardwareConcurrency != got.HardwareConcurrency { + t.Fatalf("worker hardwareConcurrency mismatch: main %d worker %d", got.HardwareConcurrency, got.WorkerHardwareConcurrency) } if got.WorkerUserAgent != got.UserAgent { t.Fatalf("worker userAgent mismatch: main %q worker %q", got.UserAgent, got.WorkerUserAgent) @@ -127,11 +142,11 @@ func TestProfileCoherence(t *testing.T) { if got.WorkerTimezone != got.Timezone { t.Fatalf("worker timezone mismatch: main %q worker %q", got.Timezone, got.WorkerTimezone) } - if got.WorkerWebGLVendor != expected.WebGLVendor { - t.Fatalf("worker WebGL vendor mismatch: expected %q got %q", expected.WebGLVendor, got.WorkerWebGLVendor) + if got.WorkerWebGLVendor != got.WebGLVendor { + t.Fatalf("worker WebGL vendor mismatch: main %q worker %q", got.WebGLVendor, got.WorkerWebGLVendor) } - if got.WorkerWebGLRenderer != expected.WebGLRenderer { - t.Fatalf("worker WebGL renderer mismatch: expected %q got %q", expected.WebGLRenderer, got.WorkerWebGLRenderer) + if got.WorkerWebGLRenderer != got.WebGLRenderer { + t.Fatalf("worker WebGL renderer mismatch: main %q worker %q", got.WebGLRenderer, got.WorkerWebGLRenderer) } if got.InnerHeight >= got.OuterHeight { t.Fatalf("innerHeight should be smaller than outerHeight, got inner=%d outer=%d", got.InnerHeight, got.OuterHeight) @@ -139,32 +154,37 @@ func TestProfileCoherence(t *testing.T) { if got.OuterHeight > got.ScreenAvailHeight { t.Fatalf("outerHeight should fit in screen.availHeight, got outer=%d avail=%d", got.OuterHeight, got.ScreenAvailHeight) } - if got.ScreenAvailHeight >= got.ScreenHeight { - t.Fatalf("screen.availHeight should be smaller than screen.height, got avail=%d screen=%d", got.ScreenAvailHeight, got.ScreenHeight) + if got.ScreenAvailHeight > got.ScreenHeight { + t.Fatalf("screen.availHeight should fit in screen.height, got avail=%d screen=%d", got.ScreenAvailHeight, got.ScreenHeight) } }) } } type profileSurface struct { - UserAgent string `json:"userAgent"` - Platform string `json:"platform"` - NavigatorPlatform string `json:"navigatorPlatform"` - NavigatorLanguages []string `json:"navigatorLanguages"` - Timezone string `json:"timezone"` - Locale string `json:"locale"` - WebdriverType string `json:"webdriverType"` - WebdriverOwnPropPresent bool `json:"webdriverOwnPropPresent"` - WorkerUserAgent string `json:"workerUserAgent"` - WorkerPlatform string `json:"workerPlatform"` - WorkerNavigatorLangs []string `json:"workerNavigatorLangs"` - WorkerTimezone string `json:"workerTimezone"` - WorkerWebGLVendor string `json:"workerWebGLVendor"` - WorkerWebGLRenderer string `json:"workerWebGLRenderer"` - InnerHeight int `json:"innerHeight"` - OuterHeight int `json:"outerHeight"` - ScreenHeight int `json:"screenHeight"` - ScreenAvailHeight int `json:"screenAvailHeight"` + UserAgent string `json:"userAgent"` + Platform string `json:"platform"` + NavigatorPlatform string `json:"navigatorPlatform"` + NavigatorLanguages []string `json:"navigatorLanguages"` + Timezone string `json:"timezone"` + Locale string `json:"locale"` + WebdriverType string `json:"webdriverType"` + WebdriverValue bool `json:"webdriverValue"` + WebdriverOwnPropPresent bool `json:"webdriverOwnPropPresent"` + HardwareConcurrency int `json:"hardwareConcurrency"` + WorkerHardwareConcurrency int `json:"workerHardwareConcurrency"` + WorkerUserAgent string `json:"workerUserAgent"` + WorkerPlatform string `json:"workerPlatform"` + WorkerNavigatorLangs []string `json:"workerNavigatorLangs"` + WorkerTimezone string `json:"workerTimezone"` + WebGLVendor string `json:"webGLVendor"` + WebGLRenderer string `json:"webGLRenderer"` + WorkerWebGLVendor string `json:"workerWebGLVendor"` + WorkerWebGLRenderer string `json:"workerWebGLRenderer"` + InnerHeight int `json:"innerHeight"` + OuterHeight int `json:"outerHeight"` + ScreenHeight int `json:"screenHeight"` + ScreenAvailHeight int `json:"screenAvailHeight"` } func browserProfileSurface(page *rod.Page) (profileSurface, error) { @@ -205,6 +225,25 @@ func selectedProfileFromContext(t *testing.T, ctx context.Context) browserprofil return profile } +// expectedProfileUserAgent resolves the profile's user agent, expanding the +// {chrome_major} template with the runtime's major when the profile carries a +// template rather than a literal UA. +func expectedProfileUserAgent(profile browserprofile.Profile, runtimeUserAgent string) string { + if ua := strings.TrimSpace(profile.UserAgent); ua != "" { + return ua + } + template := strings.TrimSpace(profile.UserAgentTemplate) + if template == "" { + return "" + } + major := chromeToken(runtimeUserAgent) + major = strings.TrimPrefix(major, "Chrome/") + if idx := strings.IndexByte(major, '.'); idx >= 0 { + major = major[:idx] + } + return strings.ReplaceAll(template, "{chrome_major}", major) +} + func expectedUserAgentForRuntime(profileUserAgent, runtimeUserAgent string) string { runtimeChrome := chromeToken(runtimeUserAgent) if runtimeChrome == "" { diff --git a/core/browser/profile_surface_test.js b/core/browser/profile_surface_test.js index c6204ad..95464d8 100644 --- a/core/browser/profile_surface_test.js +++ b/core/browser/profile_surface_test.js @@ -1,4 +1,17 @@ async () => { + const readWebGL = (canvas) => { + try { + const gl = canvas ? (canvas.getContext('webgl') || canvas.getContext('experimental-webgl') || canvas.getContext('webgl2')) : null; + const debugInfo = gl && gl.getExtension('WEBGL_debug_renderer_info'); + return { + vendor: gl && debugInfo ? (gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL) || '') : '', + renderer: gl && debugInfo ? (gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL) || '') : '', + }; + } catch (_) { + return { vendor: '', renderer: '' }; + } + }; + const webGL = readWebGL(document.createElement('canvas')); const workerData = await new Promise((resolve) => { try { const source = [ @@ -19,6 +32,7 @@ async () => { "platform: self.navigator.platform || '',", "navigatorLanguages: Array.from(self.navigator.languages || []),", "timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || '',", + "hardwareConcurrency: self.navigator.hardwareConcurrency || 0,", "webGLVendor,", "webGLRenderer,", "});", @@ -51,11 +65,16 @@ async () => { timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || "", locale: Intl.DateTimeFormat().resolvedOptions().locale || "", webdriverType: typeof navigator.webdriver, + webdriverValue: navigator.webdriver === true, webdriverOwnPropPresent: Object.getOwnPropertyNames(navigator).includes("webdriver"), + hardwareConcurrency: navigator.hardwareConcurrency || 0, + workerHardwareConcurrency: workerData.hardwareConcurrency || 0, workerUserAgent: workerData.userAgent || "", workerPlatform: workerData.platform || "", workerNavigatorLangs: Array.from(workerData.navigatorLanguages || []), workerTimezone: workerData.timezone || "", + webGLVendor: webGL.vendor, + webGLRenderer: webGL.renderer, workerWebGLVendor: workerData.webGLVendor || "", workerWebGLRenderer: workerData.webGLRenderer || "", innerHeight: window.innerHeight || 0, diff --git a/core/browser/profile_test.go b/core/browser/profile_test.go index 08e37fd..ae5ea30 100644 --- a/core/browser/profile_test.go +++ b/core/browser/profile_test.go @@ -69,14 +69,73 @@ func TestSelectProfileForSession(t *testing.T) { } }) - t.Run("ru region returns ru-tagged profile", func(t *testing.T) { + t.Run("locale does not change hardware profile pool", func(t *testing.T) { p := SelectProfileForSession("yandex", "ru", "some-session") - if !slices.Contains(p.Tags, "ru") { - t.Fatalf("expected ru-tagged profile, got ID=%q tags=%v", p.ID, p.Tags) + if slices.Contains(p.Tags, "ru") { + t.Fatalf("expected locale-neutral profile, got ID=%q tags=%v", p.ID, p.Tags) } }) } +func TestEligibleProfilesMatchRuntimePlatform(t *testing.T) { + tests := []struct { + goos string + tag string + }{ + {goos: "linux", tag: "linux"}, + {goos: "windows", tag: "windows"}, + {goos: "darwin", tag: "macos"}, + } + + for _, tt := range tests { + t.Run(tt.goos, func(t *testing.T) { + pool := eligibleProfiles(tt.goos, false) + if len(pool) == 0 { + t.Fatal("expected eligible profiles") + } + for _, candidate := range pool { + if !slices.Contains(candidate.profile.Tags, tt.tag) { + t.Fatalf("profile %q does not match %s", candidate.profile.ID, tt.goos) + } + } + }) + } +} + +// Headless Linux (the Docker deployment) has no real GPU, so only +// swiftshader-tagged profiles are eligible; headful Linux excludes them. +func TestEligibleProfilesHeadlessLinuxUsesSwiftShader(t *testing.T) { + headless := eligibleProfiles("linux", true) + if len(headless) == 0 { + t.Fatal("expected a Linux SwiftShader profile") + } + for _, candidate := range headless { + if !slices.Contains(candidate.profile.Tags, "swiftshader") { + t.Fatalf("headless profile %q is not a SwiftShader profile", candidate.profile.ID) + } + } + if len(headless) < 2 { + t.Fatalf("expected multiple SwiftShader profiles so Docker is not a single fingerprint, got %d", len(headless)) + } + timezones := make(map[string]struct{}, len(headless)) + for _, candidate := range headless { + timezones[candidate.profile.Timezone] = struct{}{} + } + if len(timezones) < 2 { + t.Fatalf("expected distinct CDP-visible SwiftShader profiles, got timezones %v", timezones) + } + + headful := eligibleProfiles("linux", false) + if len(headful) == 0 { + t.Fatal("expected headful Linux profiles") + } + for _, candidate := range headful { + if slices.Contains(candidate.profile.Tags, "swiftshader") { + t.Fatalf("headful profile %q should not be a SwiftShader profile", candidate.profile.ID) + } + } +} + func TestNormalizeRegion(t *testing.T) { tests := []struct { input string diff --git a/core/browser/profiles.json b/core/browser/profiles.json index 7287287..5d0fffa 100644 --- a/core/browser/profiles.json +++ b/core/browser/profiles.json @@ -2,399 +2,96 @@ "profiles": [ { "id": "chrome-linux-mesa-uhd620", - "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Linux", - "platform_version": "6.1.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1920, "height": 1080}, - "webgl_vendor": "Intel Inc.", - "webgl_renderer": "Mesa Intel(R) UHD Graphics 620 (KBL GT2)", - "tags": ["linux", "mesa", "integrated"], - "weight": 3 + "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "mesa", "integrated"], "weight": 3 }, { "id": "chrome-linux-mesa-iris-xe", - "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Linux", - "platform_version": "6.1.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1920, "height": 1080}, - "webgl_vendor": "Intel Inc.", - "webgl_renderer": "Mesa Intel(R) Graphics (RPL-S)", - "tags": ["linux", "mesa", "integrated"], - "weight": 2 + "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "mesa", "integrated"], "weight": 2 }, { "id": "chrome-linux-nvidia", - "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Linux", - "platform_version": "6.1.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1920, "height": 1080}, - "webgl_vendor": "Google Inc. (NVIDIA Corporation)", - "webgl_renderer": "ANGLE (NVIDIA Corporation, NVIDIA GeForce GTX 1660 SUPER/PCIe/SSE2, OpenGL 4.5.0 NVIDIA 535.86.05)", - "tags": ["linux", "nvidia"], - "weight": 2 + "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "nvidia"], "weight": 2 }, { "id": "chrome-linux-amd", - "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Linux", - "platform_version": "6.1.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1920, "height": 1080}, - "webgl_vendor": "Google Inc. (AMD)", - "webgl_renderer": "ANGLE (AMD, AMD Radeon RX 6600 (radeonsi, navi23, ACO, DRM 3.42.0, 5.15.0-91-generic), OpenGL 4.6 (Core Profile))", - "tags": ["linux", "amd"], - "weight": 1 + "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "amd"], "weight": 1 }, { "id": "chrome-linux-swiftshader", - "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Linux", - "platform_version": "6.1.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1366, "height": 768}, - "webgl_vendor": "Google Inc. (Google)", - "webgl_renderer": "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device (Subzero) (0x0000C0DE)), SwiftShader driver)", - "tags": ["linux", "swiftshader", "headless"], - "weight": 1 + "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "swiftshader", "headless"], "weight": 1 }, { - "id": "chrome-linux-ru", - "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Linux", - "platform_version": "6.0.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7", - "navigator_langs": ["ru-RU"], - "locale": "ru-RU", - "timezone": "Europe/Moscow", - "viewport": {"width": 1920, "height": 1080}, - "webgl_vendor": "Intel Inc.", - "webgl_renderer": "Mesa Intel(R) UHD Graphics 620 (KBL GT2)", - "tags": ["linux", "ru"], - "weight": 1 + "id": "chrome-linux-swiftshader-fhd", + "user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/Chicago", + "viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "swiftshader", "headless"], "weight": 1 }, { "id": "chrome-win-uhd620", - "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Windows", - "platform_version": "15.0.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1920, "height": 1080}, - "webgl_vendor": "Intel Inc.", - "webgl_renderer": "ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D11 vs_5_0 ps_5_0, D3D11)", - "tags": ["windows"], - "weight": 2 + "user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1920, "height": 1080}, "tags": ["windows"], "weight": 2 }, { "id": "chrome-win-nvidia", - "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Windows", - "platform_version": "15.0.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1920, "height": 1080}, - "webgl_vendor": "Google Inc. (NVIDIA)", - "webgl_renderer": "ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)", - "tags": ["windows", "nvidia"], - "weight": 2 + "user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1920, "height": 1080}, "tags": ["windows", "nvidia"], "weight": 2 }, { "id": "chrome-win-iris-xe", - "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Windows", - "platform_version": "15.0.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1536, "height": 864}, - "webgl_vendor": "Google Inc. (Intel)", - "webgl_renderer": "ANGLE (Intel, Intel(R) Iris(R) Xe Graphics Direct3D11 vs_5_0 ps_5_0, D3D11)", - "tags": ["windows", "integrated"], - "weight": 1 + "user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1920, "height": 1080}, "tags": ["windows", "integrated"], "weight": 1 }, { "id": "chrome-win-amd", - "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Windows", - "platform_version": "15.0.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1920, "height": 1080}, - "webgl_vendor": "Google Inc. (AMD)", - "webgl_renderer": "ANGLE (AMD, AMD Radeon RX 6600 Direct3D11 vs_5_0 ps_5_0, D3D11)", - "tags": ["windows", "amd"], - "weight": 1 - }, - { - "id": "chrome-win-ru", - "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "Windows", - "platform_version": "15.0.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7", - "navigator_langs": ["ru-RU"], - "locale": "ru-RU", - "timezone": "Europe/Moscow", - "viewport": {"width": 1920, "height": 1080}, - "webgl_vendor": "Intel Inc.", - "webgl_renderer": "ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D11 vs_5_0 ps_5_0, D3D11)", - "tags": ["windows", "ru"], - "weight": 1 + "user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1920, "height": 1080}, "tags": ["windows", "amd"], "weight": 1 }, { "id": "chrome-macos-m1", - "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "macOS", - "platform_version": "13.0.0", - "architecture": "arm", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1440, "height": 900}, - "webgl_vendor": "Google Inc. (Apple)", - "webgl_renderer": "ANGLE (Apple, ANGLE Metal Renderer: Apple M1, Unspecified Version)", - "tags": ["macos", "apple-silicon"], - "weight": 2 + "user_agent_template": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "macOS", "platform_version": "13.0.0", "architecture": "arm", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1440, "height": 900}, "tags": ["macos", "apple-silicon"], "weight": 2 }, { "id": "chrome-macos-m2", - "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "macOS", - "platform_version": "14.0.0", - "architecture": "arm", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1512, "height": 982}, - "webgl_vendor": "Google Inc. (Apple)", - "webgl_renderer": "ANGLE (Apple, ANGLE Metal Renderer: Apple M2, Unspecified Version)", - "tags": ["macos", "apple-silicon"], - "weight": 2 + "user_agent_template": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "macOS", "platform_version": "14.0.0", "architecture": "arm", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1512, "height": 982}, "tags": ["macos", "apple-silicon"], "weight": 2 }, { "id": "chrome-macos-intel-iris", - "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36", - "uach_brands": [ - {"brand": "Not_A Brand", "version": "24"}, - {"brand": "Chromium", "version": "136"}, - {"brand": "Google Chrome", "version": "136"} - ], - "uach_full_version_list": [ - {"brand": "Not_A Brand", "version": "24.0.0.0"}, - {"brand": "Chromium", "version": "136.0.0.0"}, - {"brand": "Google Chrome", "version": "136.0.0.0"} - ], - "platform": "macOS", - "platform_version": "14.0.0", - "architecture": "x86", - "bitness": "64", - "mobile": false, - "accept_language": "en-US,en;q=0.9", - "navigator_langs": ["en-US"], - "locale": "en-US", - "timezone": "America/New_York", - "viewport": {"width": 1680, "height": 1050}, - "webgl_vendor": "Intel Inc.", - "webgl_renderer": "Intel Iris OpenGL Engine", - "tags": ["macos", "integrated"], - "weight": 1 + "user_agent_template": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36", + "platform": "macOS", "platform_version": "14.0.0", "architecture": "x86", "bitness": "64", "mobile": false, + "accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York", + "viewport": {"width": 1680, "height": 1050}, "tags": ["macos", "integrated"], "weight": 1 } ], "lane_profile_ids": {}, - "default_region_by_engine": { - "yandex": "ru" - } + "default_region_by_engine": {"yandex": "ru"} } diff --git a/core/browser_unit_test.go b/core/browser_unit_test.go index 1834196..b1aecce 100644 --- a/core/browser_unit_test.go +++ b/core/browser_unit_test.go @@ -61,6 +61,61 @@ func TestResolveBrowserBinaryPathRejectsInvalidExplicit(t *testing.T) { } } +func TestBrowserLaunchLanguageIsProcessStable(t *testing.T) { + tests := []struct { + name string + opts BrowserOpts + want string + }{ + {name: "default locale", opts: BrowserOpts{}, want: "en-US"}, + {name: "request hint does not change process locale", opts: BrowserOpts{LanguageCode: "de"}, want: "en-US"}, + {name: "regional hint does not change process locale", opts: BrowserOpts{LanguageCode: "en-GB"}, want: "en-US"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := browserLaunchLanguage(tt.opts); got != tt.want { + t.Fatalf("browserLaunchLanguage() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestProfileNavigatorLanguagesStripsHeaderWeights(t *testing.T) { + profile := browserprofile.Profile{ + AcceptLanguage: "en-US,en;q=0.9", + NavigatorLangs: []string{"en-US"}, + } + + got := profileNavigatorLanguages(profile) + want := []string{"en-US", "en"} + if len(got) != len(want) { + t.Fatalf("profileNavigatorLanguages() = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("profileNavigatorLanguages() = %v, want %v", got, want) + } + } +} + +func TestProfileNavigatorLanguagesForRuntime(t *testing.T) { + profile := browserprofile.Profile{ + AcceptLanguage: "en-US,en;q=0.9", + NavigatorLangs: []string{"en-US"}, + } + + linuxHeadless := profileNavigatorLanguagesForRuntime(profile, "linux", true) + if len(linuxHeadless) != 1 || linuxHeadless[0] != "en-US" { + t.Fatalf("linux headless languages = %v, want [en-US]", linuxHeadless) + } + + windowsHeadless := profileNavigatorLanguagesForRuntime(profile, "windows", true) + if len(windowsHeadless) != 2 || windowsHeadless[0] != "en-US" || windowsHeadless[1] != "en" { + t.Fatalf("windows headless languages = %v, want [en-US en]", windowsHeadless) + } +} + func TestMinPositiveDuration(t *testing.T) { tests := []struct { name string @@ -95,6 +150,35 @@ func TestApplyProfileLanguageHintRewritesTimezone(t *testing.T) { } } +func TestRemoveChromeBrand(t *testing.T) { + profile := browserprofile.Profile{ + UACHBrands: []browserprofile.BrandVersion{ + {Brand: "Not_A Brand", Version: "24"}, + {Brand: "Chromium", Version: "136"}, + {Brand: "Google Chrome", Version: "136"}, + }, + UACHFullVerList: []browserprofile.BrandVersion{ + {Brand: "Chromium", Version: "136.0.0.0"}, + {Brand: "Google Chrome", Version: "136.0.0.0"}, + }, + } + + got := removeChromeBrand(profile) + for _, brand := range got.UACHBrands { + if brand.Brand == "Google Chrome" { + t.Fatal("expected Google Chrome brand to be removed from UACHBrands") + } + } + for _, brand := range got.UACHFullVerList { + if brand.Brand == "Google Chrome" { + t.Fatal("expected Google Chrome brand to be removed from UACHFullVerList") + } + } + if len(got.UACHBrands) != 2 || len(got.UACHFullVerList) != 1 { + t.Fatalf("unexpected brand counts: brands=%d fullList=%d", len(got.UACHBrands), len(got.UACHFullVerList)) + } +} + func TestApplyProfileLanguageHint(t *testing.T) { base := browserprofile.Profile{ AcceptLanguage: "en-US,en;q=0.9", diff --git a/core/context.go b/core/context.go index b286364..be75c5b 100644 --- a/core/context.go +++ b/core/context.go @@ -39,12 +39,9 @@ func IsContextDone(err error) bool { // PrepareEngineContext applies request-scoped metadata expected by all engine // search implementations. -func PrepareEngineContext(ctx context.Context, query Query, engineName string, minimalBrowserProfile bool) context.Context { +func PrepareEngineContext(ctx context.Context, query Query, engineName string) context.Context { ctx = WithEngine(EnsureContext(ctx), engineName) ctx = WithProfileRegion(ctx, profileRegionHint(query)) - if minimalBrowserProfile { - ctx = WithMinimalBrowserProfile(ctx) - } return WithQueryHash(ctx, QueryHashFromQuery(query)) } diff --git a/core/http_client.go b/core/http_client.go index ad73d54..616aae8 100644 --- a/core/http_client.go +++ b/core/http_client.go @@ -419,15 +419,38 @@ func rawRequestProfileFor(ctx context.Context, query Query) rawRequestProfile { func applyRawChromeMajor(profile browserprofile.Profile, major int) browserprofile.Profile { version := strconv.Itoa(major) - if extractChromeVersion(profile.UserAgent) == "" { + if template := strings.TrimSpace(profile.UserAgentTemplate); template != "" { + profile.UserAgent = strings.ReplaceAll(template, "{chrome_major}", version) + } else if extractChromeVersion(profile.UserAgent) == "" { profile.UserAgent = fallbackRawUserAgent + } else { + profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, version+".0.0.0") + } + if len(profile.UACHBrands) == 0 { + profile.UACHBrands = rawUACHBrands(version, false) + } else { + profile.UACHBrands = patchBrandVersions(profile.UACHBrands, version, false) + } + if len(profile.UACHFullVerList) == 0 { + profile.UACHFullVerList = rawUACHBrands(version+".0.0.0", true) + } else { + profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, version+".0.0.0", true) } - profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, version+".0.0.0") - profile.UACHBrands = patchBrandVersions(profile.UACHBrands, version, false) - profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, version+".0.0.0", true) return profile } +func rawUACHBrands(version string, full bool) []browserprofile.BrandVersion { + notABrandVersion := "24" + if full { + notABrandVersion = "24.0.0.0" + } + return []browserprofile.BrandVersion{ + {Brand: "Not_A Brand", Version: notABrandVersion}, + {Brand: "Chromium", Version: version}, + {Brand: "Google Chrome", Version: version}, + } +} + func rawProfileRegion(ctx context.Context, query Query) string { if region := profileRegionFromContext(ctx); region != "" { return region diff --git a/core/profile_context.go b/core/profile_context.go index 4c2e764..2eb254e 100644 --- a/core/profile_context.go +++ b/core/profile_context.go @@ -9,7 +9,6 @@ type profileContextKey string const profileRegionContextKey profileContextKey = "profile_region" const forcedProfileIDContextKey profileContextKey = "forced_profile_id" -const minimalProfileContextKey profileContextKey = "minimal_profile" func WithProfileRegion(ctx context.Context, region string) context.Context { region = strings.TrimSpace(region) @@ -41,12 +40,3 @@ func forcedProfileIDFromContext(ctx context.Context) string { value, _ := EnsureContext(ctx).Value(forcedProfileIDContextKey).(string) return strings.TrimSpace(value) } - -func WithMinimalBrowserProfile(ctx context.Context) context.Context { - return context.WithValue(EnsureContext(ctx), minimalProfileContextKey, true) -} - -func minimalBrowserProfileFromContext(ctx context.Context) bool { - value, _ := EnsureContext(ctx).Value(minimalProfileContextKey).(bool) - return value -} diff --git a/core/proxy_lane.go b/core/proxy_lane.go index 6e7f7f1..e0fad82 100644 --- a/core/proxy_lane.go +++ b/core/proxy_lane.go @@ -224,6 +224,9 @@ func ProxyLaneKeyForTenant(engine string, tenant string, q Query, proxyURL strin if sessionID == "" { sessionID = proxyLaneIDFromProxyURL(proxyURL) } + if sessionID == "" { + sessionID = "direct" + } return NormalizeProxyLaneKey(ProxyLaneKey{Tenant: tenant, Engine: engine, SessionID: sessionID}) } diff --git a/core/proxy_lane_test.go b/core/proxy_lane_test.go index b544f62..e9d1178 100644 --- a/core/proxy_lane_test.go +++ b/core/proxy_lane_test.go @@ -103,3 +103,13 @@ func TestProxyLaneKeyIncludesTenant(t *testing.T) { t.Fatalf("unexpected tenant lane id: %q", got) } } + +func TestProxyLaneKeyUsesStableDirectSession(t *testing.T) { + key := ProxyLaneKeyForTenant("google", "tenant-a", Query{}, "") + if key.Empty() { + t.Fatal("expected a direct lane") + } + if key.SessionID != "direct" { + t.Fatalf("SessionID = %q, want direct", key.SessionID) + } +} diff --git a/duckduckgo/search.go b/duckduckgo/search.go index bdb3010..205a79b 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -149,7 +149,7 @@ func ddgElementHasAdMarker(el *rod.Element) bool { // Search executes a DuckDuckGo web search and returns normalized search // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, ddg.Name(), false) + ctx = core.PrepareEngineContext(ctx, query, ddg.Name()) scoped := *ddg scoped.logger = ddg.logger.WithRequest(ctx) ddg = &scoped @@ -229,7 +229,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] // SearchImage executes a DuckDuckGo image search and returns normalized image // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) { - ctx = core.PrepareEngineContext(ctx, query, ddg.Name(), false) + ctx = core.PrepareEngineContext(ctx, query, ddg.Name()) scoped := *ddg scoped.logger = ddg.logger.WithRequest(ctx) ddg = &scoped diff --git a/ecosia/search.go b/ecosia/search.go index 61aca86..cf452a7 100644 --- a/ecosia/search.go +++ b/ecosia/search.go @@ -105,7 +105,7 @@ func (e *Ecosia) parseResult(elem *rod.Element, rank int, ad bool) (core.SearchR // Search executes an Ecosia web search and returns normalized search results. // It may return core.ErrCaptcha or core.ErrSearchTimeout. func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, e.Name(), false) + ctx = core.PrepareEngineContext(ctx, query, e.Name()) scoped := *e scoped.logger = e.logger.WithRequest(ctx) e = &scoped @@ -237,7 +237,7 @@ func elementText(el *rod.Element, selector string) string { // query.Start is ignored: per-page card count varies, so callers should // drive depth through query.Limit alone. func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, e.Name(), false) + ctx = core.PrepareEngineContext(ctx, query, e.Name()) scoped := *e scoped.logger = e.logger.WithRequest(ctx) e = &scoped diff --git a/ecosia/search_raw.go b/ecosia/search_raw.go index e28899f..519d2d8 100644 --- a/ecosia/search_raw.go +++ b/ecosia/search_raw.go @@ -57,7 +57,7 @@ func imageResultParser(response *http.Response) ([]core.SearchResult, error) { } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, "ecosia", false) + ctx = core.PrepareEngineContext(ctx, query, "ecosia") pageNum, startRank, err := startPage(query.Start) if err != nil { diff --git a/google/search.go b/google/search.go index 25573b2..5b9b2da 100644 --- a/google/search.go +++ b/google/search.go @@ -228,7 +228,7 @@ func googleElementHasAdMarker(el *rod.Element) bool { // Search executes a Google web search and returns normalized search results. // It may return core.ErrCaptcha or core.ErrSearchTimeout. func (gogl *Google) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, gogl.Name(), true) + ctx = core.PrepareEngineContext(ctx, query, gogl.Name()) scoped := *gogl scoped.logger = gogl.logger.WithRequest(ctx) gogl = &scoped @@ -494,7 +494,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor // SearchImage executes a Google image search and returns normalized image // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) { - ctx = core.PrepareEngineContext(ctx, query, gogl.Name(), true) + ctx = core.PrepareEngineContext(ctx, query, gogl.Name()) scoped := *gogl scoped.logger = gogl.logger.WithRequest(ctx) gogl = &scoped diff --git a/google/search_raw.go b/google/search_raw.go index 69b1433..e87ac09 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -173,7 +173,7 @@ func isZeroResultStats(s string) bool { } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, "google", false) + ctx = core.PrepareEngineContext(ctx, query, "google") googleURL, err := BuildURL(query) if err != nil { diff --git a/yandex/search.go b/yandex/search.go index bdc3853..e61f3d0 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -191,7 +191,7 @@ func (yand *Yandex) parseImageEntities(items rod.Elements) map[string]ImageEntit // Search executes a Yandex web search and returns normalized search results. // It may return core.ErrCaptcha or core.ErrSearchTimeout. func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, yand.Name(), false) + ctx = core.PrepareEngineContext(ctx, query, yand.Name()) scoped := *yand scoped.logger = yand.logger.WithRequest(ctx) yand = &scoped @@ -286,7 +286,7 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor // SearchImage executes a Yandex image search and returns normalized image // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) { - ctx = core.PrepareEngineContext(ctx, query, yand.Name(), false) + ctx = core.PrepareEngineContext(ctx, query, yand.Name()) scoped := *yand scoped.logger = yand.logger.WithRequest(ctx) yand = &scoped diff --git a/yandex/search_raw.go b/yandex/search_raw.go index c2a83b0..5fd8bbe 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -19,7 +19,7 @@ func classifyYandexRawHTML(body []byte) error { } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { - ctx = core.PrepareEngineContext(ctx, query, "yandex", false) + ctx = core.PrepareEngineContext(ctx, query, "yandex") startPage, skipOnFirstPage, err := core.ComputePagination(query.Start, 10) if err != nil {