refactor(browser): native CDP fingerprinting, drop patch.js + localized profiles. Update docker chrome

This commit is contained in:
Rustem Kamalov
2026-07-13 06:07:37 +03:00
parent cf21bf5150
commit e7827cc03d
25 changed files with 531 additions and 1022 deletions

View File

@@ -12,7 +12,7 @@ ARG TARGETARCH
RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH:-amd64} go build -trimpath -ldflags="-s -w" -o /app/openserp .
# `chromedp/headless-shell:stable` also works here
FROM chromedp/headless-shell:stable@sha256:aac539266027f91cf47610da1129dce360d23f45f8f150683cca94223fa2f1e2
FROM chromedp/headless-shell:stable@sha256:f7e7ac721b023cb8717f8108aef8b3e49995fb1e5a912f41e570c29e45d24961
WORKDIR /usr/src/app

View File

@@ -72,7 +72,7 @@ func (baid *Baidu) classifyBlockPage(page *rod.Page, url string) error {
// Search executes a Baidu web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, baid.Name(), true)
ctx = core.PrepareEngineContext(ctx, query, baid.Name())
scoped := *baid
scoped.logger = baid.logger.WithRequest(ctx)
if scoped.Browser.WaitLoadTime == 0 || scoped.Browser.WaitLoadTime > 250*time.Millisecond {
@@ -170,7 +170,7 @@ func (baid *Baidu) waitForParsedSearchResults(ctx context.Context, page *rod.Pag
// SearchImage executes a Baidu image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.PrepareEngineContext(ctx, query, baid.Name(), true)
ctx = core.PrepareEngineContext(ctx, query, baid.Name())
scoped := *baid
scoped.logger = baid.logger.WithRequest(ctx)
baid = &scoped

View File

@@ -19,7 +19,7 @@ func classifyBaiduRawHTML(body []byte) error {
}
func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, "baidu", false)
ctx = core.PrepareEngineContext(ctx, query, "baidu")
searchURL, err := BuildURL(query)
if err != nil {

View File

@@ -155,7 +155,7 @@ func (bing *Bing) parseResultElement(el *rod.Element, isAd bool, rank *core.Rank
// Search executes a Bing web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, bing.Name(), false)
ctx = core.PrepareEngineContext(ctx, query, bing.Name())
scoped := *bing
scoped.logger = bing.logger.WithRequest(ctx)
bing = &scoped
@@ -256,7 +256,7 @@ func resolveImageLinkElement(container *rod.Element) (*rod.Element, error) {
// SearchImage executes a Bing image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.PrepareEngineContext(ctx, query, bing.Name(), false)
ctx = core.PrepareEngineContext(ctx, query, bing.Name())
scoped := *bing
scoped.logger = bing.logger.WithRequest(ctx)
bing = &scoped

View File

@@ -17,7 +17,7 @@ import (
)
const (
version = "0.8.9"
version = "0.8.10"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)

View File

@@ -2,13 +2,13 @@ package core
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"net/url"
"os"
"regexp"
"runtime"
"runtime/debug"
"strings"
"sync"
@@ -30,7 +30,7 @@ type BrowserOpts struct {
IsLeakless bool
// Timeout is applied to browser connect and page navigation operations.
Timeout time.Duration
// LanguageCode sets Accept-Language for emulated requests.
// LanguageCode selects the regional browser lane and search locale.
LanguageCode string
// WaitRequests waits for request-idle state after navigation.
WaitRequests bool
@@ -279,20 +279,17 @@ func NewBrowser(opts BrowserOpts) (*Browser, error) {
return nil, err
}
// Create launcher.
// headless=new uses the full Chrome renderer; legacy --headless disables the
// GPU process entirely, making WebGL context creation fail even with SwiftShader.
// use-angle=swiftshader-webgl (Chrome ≥112) enables a software WebGL renderer.
// Rod enables leakless by default, so always pass the configured value
// through. OpenSERP defaults it to false because the helper binary is
// commonly flagged by antivirus on Windows.
// headless=new keeps the full renderer, so WebGL falls back to SwiftShader
// when there's no GPU - which is what the swiftshader profiles describe.
// Legacy --headless kills the GPU process and breaks WebGL entirely.
// Leakless defaults to false: the helper binary trips antivirus on Windows.
l := launcher.New().Leakless(opts.IsLeakless).
Set("lang", browserLaunchLanguage(opts)).
Set("disable-blink-features", "AutomationControlled").
Delete("enable-automation").
Set("use-angle", "swiftshader-webgl").
Set("ignore-gpu-blocklist")
Delete("enable-automation")
if opts.IsHeadless {
l = l.HeadlessNew(true)
// Initial window only; per-page setDeviceMetricsOverride is what pages see. Sized for the 1920x1080 swiftshader profiles that run on Docker.
l = l.HeadlessNew(true).Set("window-size", "1920,1040")
} else {
l = l.Headless(false)
}
@@ -365,6 +362,12 @@ func browserOptsLogFields(opts BrowserOpts) logrus.Fields {
}
}
func browserLaunchLanguage(_ BrowserOpts) string {
// Speech voices are per-process and CDP can't change them, so pin the
// launch locale to the catalog rather than the request hint.
return "en-US"
}
func maskedProxyLogValue(proxyURL string) string {
if strings.TrimSpace(proxyURL) == "" {
return ""
@@ -653,7 +656,6 @@ func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browse
profile, ok := browserprofile.ProfileByID(forcedID)
if ok {
profile = applyRuntimeBrowserVersion(profile, browser)
profile = applyProfileLanguageHint(profile, region)
if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" {
profile.UserAgent = overrideUA
}
@@ -663,9 +665,8 @@ func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browse
if laneKey := proxyLaneKeyFromContext(ctx); !laneKey.Empty() && b.ProxyLaneStore != nil {
profile := b.ProxyLaneStore.Profile(laneKey, func() browserprofile.Profile {
selected := browserprofile.SelectProfileForSession(engine, region, laneKey.SessionID)
selected := browserprofile.SelectProfileForSessionHeadless(engine, region, laneKey.SessionID, b.IsHeadless)
selected = applyRuntimeBrowserVersion(selected, browser)
selected = applyProfileLanguageHint(selected, region)
if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" {
selected.UserAgent = overrideUA
}
@@ -691,9 +692,8 @@ func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browse
// RWMutex-guarded catalog, and applyRuntimeBrowserVersion makes a CDP
// round-trip (browser.Version). Holding state.mu over network I/O would
// serialize all concurrent Navigate calls.
profile := browserprofile.SelectProfileForSession(engine, region, laneKey)
profile := browserprofile.SelectProfileForSessionHeadless(engine, region, laneKey, b.IsHeadless)
profile = applyRuntimeBrowserVersion(profile, browser)
profile = applyProfileLanguageHint(profile, region)
if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" {
profile.UserAgent = overrideUA
}
@@ -712,17 +712,46 @@ func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browse
return profile, laneKey
}
// removeChromeBrand drops the "Google Chrome" client-hint brand from a profile,
// used when the launched binary is Chromium rather than Google Chrome.
func removeChromeBrand(profile browserprofile.Profile) browserprofile.Profile {
profile.UACHBrands = removeBrand(profile.UACHBrands, "google chrome")
profile.UACHFullVerList = removeBrand(profile.UACHFullVerList, "google chrome")
return profile
}
func removeBrand(values []browserprofile.BrandVersion, unwanted string) []browserprofile.BrandVersion {
out := make([]browserprofile.BrandVersion, 0, len(values))
for _, value := range values {
if strings.EqualFold(strings.TrimSpace(value.Brand), unwanted) {
continue
}
out = append(out, value)
}
return out
}
func applyRuntimeBrowserVersion(profile browserprofile.Profile, browser *rod.Browser) browserprofile.Profile {
fullVersion := ""
product := ""
if browser != nil {
version, err := browser.Version()
if err == nil && version != nil {
product = strings.TrimSpace(version.Product)
fullVersion = extractChromeVersion(version.UserAgent)
if fullVersion == "" {
fullVersion = extractChromeVersion(version.Product)
fullVersion = extractChromeVersion(product)
}
}
}
// version.Product tells us the real binary: "Chrome/..." for genuine Chrome,
// "Chromium/..." or "HeadlessChrome/..." otherwise. On non-Chrome builds we
// drop the "Google Chrome" UA-CH brand so the hints match the binary.
return applyRuntimeBrowserVersionValues(profile, product, fullVersion)
}
func applyRuntimeBrowserVersionValues(profile browserprofile.Profile, product, fullVersion string) browserprofile.Profile {
if fullVersion == "" {
fullVersion = extractChromeVersion(profile.UserAgent)
}
@@ -735,12 +764,39 @@ func applyRuntimeBrowserVersion(profile browserprofile.Profile, browser *rod.Bro
return profile
}
profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, major+".0.0.0")
profile.UACHBrands = patchBrandVersions(profile.UACHBrands, major, false)
profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, fullVersion, true)
if template := strings.TrimSpace(profile.UserAgentTemplate); template != "" {
profile.UserAgent = strings.ReplaceAll(template, "{chrome_major}", major)
} else {
profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, major+".0.0.0")
}
if len(profile.UACHBrands) == 0 {
profile.UACHBrands = runtimeUACHBrands(major, false)
} else {
profile.UACHBrands = patchBrandVersions(profile.UACHBrands, major, false)
}
if len(profile.UACHFullVerList) == 0 {
profile.UACHFullVerList = runtimeUACHBrands(fullVersion, true)
} else {
profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, fullVersion, true)
}
if product != "" && !strings.HasPrefix(product, "Chrome/") {
profile = removeChromeBrand(profile)
}
return profile
}
func runtimeUACHBrands(version string, full bool) []browserprofile.BrandVersion {
notABrandVersion := "24"
if full {
notABrandVersion = "24.0.0.0"
}
return []browserprofile.BrandVersion{
{Brand: "Not_A Brand", Version: notABrandVersion},
{Brand: "Chromium", Version: version},
{Brand: "Google Chrome", Version: version},
}
}
func extractChromeVersion(value string) string {
matches := chromeVersionPattern.FindStringSubmatch(strings.TrimSpace(value))
if len(matches) < 2 {
@@ -808,9 +864,6 @@ type profileDisplayMetrics struct {
ViewportHeight int
ScreenWidth int
ScreenHeight int
AvailWidth int
AvailHeight int
AvailTop int
OuterWidth int
OuterHeight int
PositionX int
@@ -849,9 +902,6 @@ func profileDisplayMetricsFor(profile browserprofile.Profile) profileDisplayMetr
ViewportHeight: viewportHeight,
ScreenWidth: screenWidth,
ScreenHeight: screenHeight,
AvailWidth: screenWidth,
AvailHeight: availHeight,
AvailTop: availTop,
OuterWidth: screenWidth,
OuterHeight: availHeight,
PositionX: 0,
@@ -897,12 +947,12 @@ func applyProfileLanguageHint(profile browserprofile.Profile, langCode string) b
return profile
}
func applyProfile(page *rod.Page, profile browserprofile.Profile, minimal bool) error {
func applyProfile(page *rod.Page, profile browserprofile.Profile, headless bool) error {
if page == nil {
return fmt.Errorf("page is nil")
}
navigatorLangs := profileNavigatorLanguages(profile)
navigatorLangs := profileNavigatorLanguagesForRuntime(profile, runtime.GOOS, headless)
acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
if acceptLanguage == "" {
acceptLanguage = navigatorLangs[0]
@@ -913,6 +963,18 @@ func applyProfile(page *rod.Page, profile browserprofile.Profile, minimal bool)
}
metrics := profileDisplayMetricsFor(profile)
// setWindowBounds can close the target under headless, and there's no real
// window anyway - setDeviceMetricsOverride below covers those dimensions.
if !headless {
if err := page.SetWindow(&proto.BrowserBounds{
Left: &metrics.PositionX,
Top: &metrics.PositionY,
Width: &metrics.OuterWidth,
Height: &metrics.OuterHeight,
}); err != nil {
logrus.WithError(err).Debug("set browser window unsupported")
}
}
metadata := &proto.EmulationUserAgentMetadata{
Brands: toProtoBrandVersions(profile.UACHBrands),
@@ -924,9 +986,11 @@ func applyProfile(page *rod.Page, profile browserprofile.Profile, minimal bool)
Mobile: profile.Mobile,
}
// This only seeds navigator.languages (Chrome strips q-values), so it's the
// plain tag list. The wire header with q-weights is set below.
if err := (proto.NetworkSetUserAgentOverride{
UserAgent: strings.TrimSpace(profile.UserAgent),
AcceptLanguage: acceptLanguage,
AcceptLanguage: strings.Join(navigatorLangs, ","),
Platform: navigatorPlatformForProfile(profile),
UserAgentMetadata: metadata,
}).Call(page); err != nil {
@@ -958,6 +1022,14 @@ func applyProfile(page *rod.Page, profile browserprofile.Profile, minimal bool)
return fmt.Errorf("set device metrics failed: %w", err)
}
if err := (proto.EmulationSetEmulatedMedia{Features: []*proto.EmulationMediaFeature{
{Name: "prefers-reduced-motion", Value: "no-preference"},
{Name: "prefers-color-scheme", Value: "light"},
{Name: "forced-colors", Value: "none"},
}}).Call(page); err != nil {
return fmt.Errorf("set emulated media failed: %w", err)
}
if err := (proto.NetworkSetExtraHTTPHeaders{
Headers: proto.NetworkHeaders{
"Accept-Language": gson.New(acceptLanguage),
@@ -966,14 +1038,6 @@ func applyProfile(page *rod.Page, profile browserprofile.Profile, minimal bool)
return fmt.Errorf("set extra headers failed: %w", err)
}
if minimal {
return nil
}
if err := evalPatchScript(page, profile, navigatorLangs, metrics); err != nil {
return err
}
return nil
}
@@ -1023,14 +1087,7 @@ type networkUsageWatcher struct {
done chan struct{}
}
type workerPatchWatcher struct {
cancel context.CancelFunc
done chan struct{}
page *rod.Page
}
var pageNetworkUsageWatchers sync.Map
var pageWorkerPatchWatchers sync.Map
func startMainDocumentStatusWatcher(ctx context.Context, page *rod.Page) *mainDocumentStatusWatcher {
watchCtx, cancel := context.WithCancel(EnsureContext(ctx))
@@ -1098,95 +1155,6 @@ func startNetworkUsageWatcher(ctx context.Context, page *rod.Page) *networkUsage
return watcher
}
func startWorkerPatchWatcher(ctx context.Context, page *rod.Page, script string) (*workerPatchWatcher, error) {
if page == nil || strings.TrimSpace(script) == "" {
return nil, nil
}
watchCtx, cancel := context.WithCancel(EnsureContext(ctx))
watcher := &workerPatchWatcher{
cancel: cancel,
done: make(chan struct{}),
page: page,
}
scopedPage := page.Context(watchCtx)
started := make(chan struct{})
go func() {
defer close(watcher.done)
wait := scopedPage.EachEvent(func(e *proto.TargetAttachedToTarget) bool {
if e == nil || e.SessionID == "" {
return false
}
go injectWorkerPatch(watchCtx, scopedPage.Browser(), e, script)
return false
})
close(started)
wait()
}()
<-started
if err := (proto.TargetSetAutoAttach{
AutoAttach: true,
WaitForDebuggerOnStart: true,
Flatten: true,
Filter: workerTargetFilter(),
}).Call(scopedPage); err != nil {
cancel()
<-watcher.done
return nil, fmt.Errorf("enable worker auto-attach: %w", err)
}
return watcher, nil
}
func workerTargetFilter() proto.TargetTargetFilter {
return proto.TargetTargetFilter{
{Type: "worker"},
{Type: string(proto.TargetTargetInfoTypeSharedWorker)},
{Type: string(proto.TargetTargetInfoTypeServiceWorker)},
}
}
func injectWorkerPatch(ctx context.Context, browser *rod.Browser, e *proto.TargetAttachedToTarget, script string) {
if browser == nil || e == nil || e.SessionID == "" {
return
}
injectCtx, cancel := context.WithTimeout(EnsureContext(ctx), 3*time.Second)
defer cancel()
if isPatchableWorkerTarget(e.TargetInfo) {
eval := proto.RuntimeEvaluate{
Expression: script,
Silent: true,
AllowUnsafeEvalBlockedByCSP: true,
}
if _, err := browser.Call(injectCtx, string(e.SessionID), eval.ProtoReq(), eval); err != nil && !errors.Is(err, context.Canceled) {
logrus.WithError(err).Debug("Worker profile patch failed")
}
}
if e.WaitingForDebugger {
run := proto.RuntimeRunIfWaitingForDebugger{}
if _, err := browser.Call(injectCtx, string(e.SessionID), run.ProtoReq(), run); err != nil && !errors.Is(err, context.Canceled) {
logrus.WithError(err).Debug("Resume worker after profile patch failed")
}
}
}
func isPatchableWorkerTarget(info *proto.TargetTargetInfo) bool {
if info == nil {
return true
}
switch string(info.Type) {
case "worker", string(proto.TargetTargetInfoTypeSharedWorker), string(proto.TargetTargetInfoTypeServiceWorker):
return true
default:
return false
}
}
func rememberNetworkUsageWatcher(page *rod.Page, watcher *networkUsageWatcher) {
if page == nil || watcher == nil {
return
@@ -1194,13 +1162,6 @@ func rememberNetworkUsageWatcher(page *rod.Page, watcher *networkUsageWatcher) {
pageNetworkUsageWatchers.Store(page, watcher)
}
func rememberWorkerPatchWatcher(page *rod.Page, watcher *workerPatchWatcher) {
if page == nil || watcher == nil {
return
}
pageWorkerPatchWatchers.Store(page, watcher)
}
func stopNetworkUsageWatcher(page *rod.Page) {
if page == nil {
return
@@ -1214,19 +1175,6 @@ func stopNetworkUsageWatcher(page *rod.Page) {
}
}
func stopWorkerPatchWatcher(page *rod.Page) {
if page == nil {
return
}
raw, ok := pageWorkerPatchWatchers.LoadAndDelete(page)
if !ok {
return
}
if watcher, ok := raw.(*workerPatchWatcher); ok {
watcher.Stop()
}
}
func (w *networkUsageWatcher) Stop() {
if w == nil {
return
@@ -1238,27 +1186,6 @@ func (w *networkUsageWatcher) Stop() {
}
}
func (w *workerPatchWatcher) Stop() {
if w == nil {
return
}
if w.page != nil {
disableCtx, cancel := context.WithTimeout(context.Background(), time.Second)
_ = (proto.TargetSetAutoAttach{
AutoAttach: false,
WaitForDebuggerOnStart: false,
Flatten: true,
Filter: workerTargetFilter(),
}).Call(w.page.Context(disableCtx))
cancel()
}
w.cancel()
select {
case <-w.done:
case <-time.After(100 * time.Millisecond):
}
}
func (w *mainDocumentStatusWatcher) Status() int {
if w == nil {
return 0
@@ -1281,21 +1208,22 @@ func classifyMainDocumentStatus(status int) error {
func profileNavigatorLanguages(profile browserprofile.Profile) []string {
langs := make([]string, 0, len(profile.NavigatorLangs))
seen := make(map[string]struct{}, len(profile.NavigatorLangs))
for _, language := range profile.NavigatorLangs {
trimmed := strings.TrimSpace(language)
if trimmed == "" {
continue
}
if _, ok := seen[trimmed]; ok {
continue
}
seen[trimmed] = struct{}{}
langs = append(langs, trimmed)
}
if len(langs) > 0 {
return langs
}
acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
if acceptLanguage != "" {
parts := strings.Split(acceptLanguage, ",")
langs = make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "" {
@@ -1305,12 +1233,16 @@ func profileNavigatorLanguages(profile browserprofile.Profile) []string {
part = strings.TrimSpace(part[:idx])
}
if part != "" {
if _, ok := seen[part]; ok {
continue
}
seen[part] = struct{}{}
langs = append(langs, part)
}
}
if len(langs) > 0 {
return langs
}
}
if len(langs) > 0 {
return langs
}
if locale := strings.TrimSpace(profile.Locale); locale != "" {
@@ -1319,6 +1251,15 @@ func profileNavigatorLanguages(profile browserprofile.Profile) []string {
return []string{"en-US"}
}
func profileNavigatorLanguagesForRuntime(profile browserprofile.Profile, goos string, headless bool) []string {
langs := profileNavigatorLanguages(profile)
// Linux headless-shell keeps workers at the process locale only.
if headless && goos == "linux" {
return langs[:1]
}
return langs
}
func toProtoBrandVersions(values []browserprofile.BrandVersion) []*proto.EmulationUserAgentBrandVersion {
out := make([]*proto.EmulationUserAgentBrandVersion, 0, len(values))
for _, value := range values {
@@ -1335,60 +1276,6 @@ func toProtoBrandVersions(values []browserprofile.BrandVersion) []*proto.Emulati
return out
}
func buildProfilePatchScript(profile browserprofile.Profile, langs []string, metrics profileDisplayMetrics) (string, error) {
langsJSON, err := json.Marshal(langs)
if err != nil {
return "", fmt.Errorf("marshal navigator languages: %w", err)
}
webGLVendor := strings.TrimSpace(profile.WebGLVendor)
if webGLVendor == "" {
webGLVendor = "Intel Inc."
}
webGLRenderer := strings.TrimSpace(profile.WebGLRenderer)
if webGLRenderer == "" {
webGLRenderer = "Intel Iris OpenGL Engine"
}
webGLVendorJSON, err := json.Marshal(webGLVendor)
if err != nil {
return "", fmt.Errorf("marshal webgl vendor: %w", err)
}
webGLRendererJSON, err := json.Marshal(webGLRenderer)
if err != nil {
return "", fmt.Errorf("marshal webgl renderer: %w", err)
}
return fmt.Sprintf("(() => {\nconst __langs = %s;\nconst __w = %d;\nconst __h = %d;\nconst __screenW = %d;\nconst __screenH = %d;\nconst __availW = %d;\nconst __availH = %d;\nconst __availTop = %d;\nconst __outerW = %d;\nconst __outerH = %d;\nconst __webglVendor = %s;\nconst __webglRenderer = %s;\n%s\n})();",
string(langsJSON),
metrics.ViewportWidth,
metrics.ViewportHeight,
metrics.ScreenWidth,
metrics.ScreenHeight,
metrics.AvailWidth,
metrics.AvailHeight,
metrics.AvailTop,
metrics.OuterWidth,
metrics.OuterHeight,
string(webGLVendorJSON),
string(webGLRendererJSON),
string(browserprofile.PatchJS),
), nil
}
func evalPatchScript(page *rod.Page, profile browserprofile.Profile, langs []string, metrics profileDisplayMetrics) error {
script, err := buildProfilePatchScript(profile, langs, metrics)
if err != nil {
return err
}
_, err = page.EvalOnNewDocument(script)
if err != nil {
return fmt.Errorf("eval patch script: %w", err)
}
return nil
}
// Navigate connects to Chromium, creates a page, applies a coherent profile and
// proxy auth, then navigates to URL. It returns an initialized page ready for
// selector queries, or an error when browser setup/navigation fails.
@@ -1437,7 +1324,6 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
// context first causes Chrome to kill the page target before our Close call,
// producing a spurious "target closed" error on the page.Close() that follows.
closeOnErr := func() {
stopWorkerPatchWatcher(page)
stopNetworkUsageWatcher(page)
if cerr := page.Close(); cerr != nil && !isBrowserClosedError(cerr) {
WithRequest(ctx).WithError(cerr).Debug("Close page after navigate error failed")
@@ -1450,7 +1336,6 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
// background watchers attached to it must still stop or they leak goroutines.
stopWatchersOnErr := func() {
if b.LeavePageOpen {
stopWorkerPatchWatcher(page)
stopNetworkUsageWatcher(page)
} else {
closeOnErr()
@@ -1459,12 +1344,10 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
profile, laneKey := b.laneProfile(ctx, browser)
SetBrowserProfileID(ctx, profile.ID)
minimalProfile := minimalBrowserProfileFromContext(ctx)
WithRequest(ctx).WithFields(logrus.Fields{
"lane_id": laneKey,
"minimal_profile": minimalProfile,
"lane_id": laneKey,
}).Info("Browser profile selected")
if err := applyProfile(page, profile, minimalProfile); err != nil {
if err := applyProfile(page, profile, b.IsHeadless); err != nil {
closeOnErr()
return nil, fmt.Errorf("apply profile %s (%s) failed: %w", profile.ID, laneKey, err)
}
@@ -1474,20 +1357,6 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
}
page = page.Context(ctx)
if !minimalProfile {
metrics := profileDisplayMetricsFor(profile)
patchScript, err := buildProfilePatchScript(profile, profileNavigatorLanguages(profile), metrics)
if err != nil {
closeOnErr()
return nil, err
}
workerPatchWatcher, err := startWorkerPatchWatcher(ctx, page, patchScript)
if err != nil {
closeOnErr()
return nil, err
}
rememberWorkerPatchWatcher(page, workerPatchWatcher)
}
if err := b.configureRequestBlocking(ctx, page); err != nil {
closeOnErr()
return nil, fmt.Errorf("configure request blocking failed: %w", err)
@@ -1603,7 +1472,6 @@ func ClosePageWithTimeout(ctx context.Context, page *rod.Page, timeout time.Dura
if page == nil {
return nil
}
stopWorkerPatchWatcher(page)
stopNetworkUsageWatcher(page)
if timeout <= 0 {
timeout = time.Second

View File

@@ -1,273 +0,0 @@
// Stealth patches injected via EvalOnNewDocument.
// Arguments are injected as a leading const block by the Go caller:
// const __langs = [...]; // navigator_langs from profile
// const __w = 1920; // viewport/content width
// const __h = 955; // viewport/content height
// const __screenW = 1920; // screen.width
// const __screenH = 1080; // screen.height
// const __availW = 1920; // screen.availWidth
// const __availH = 1040; // screen.availHeight
// const __availTop = 0; // screen.availTop
// const __outerW = 1920; // window.outerWidth
// const __outerH = 1040; // window.outerHeight
// const __webglVendor = "..."; // UNMASKED_VENDOR_WEBGL spoof
// const __webglRenderer = "..."; // UNMASKED_RENDERER_WEBGL spoof
//
// Scope: only patches that fix detectors with high ROI and low introspection
// surface. Notably absent: Function.prototype.toString proxy, custom Worker
// constructor, mass plugin/mimeType arrays, iframe contentWindow patching.
// Those triggered server-side detection on Google in commit 612e0dc.
(() => {
'use strict';
const sealGetter = (target, prop, fn) => {
try {
Object.defineProperty(target, prop, {
get: fn,
set: undefined,
enumerable: true,
configurable: true,
});
Object.defineProperty(target, prop, { configurable: false });
} catch (_) {}
};
// --- navigator.webdriver ---
// headless Chrome sets this to true. Delete it so getter returns undefined.
// (We rely on `disable-blink-features=AutomationControlled` already turning
// this off via the launcher; this is belt-and-braces in case it leaks.)
try {
if (typeof navigator.webdriver !== 'undefined') {
delete Object.getPrototypeOf(navigator).webdriver;
}
} catch (_) {}
// --- navigator.language / navigator.languages ---
// CDP setUserAgentOverride sets the HTTP header but not these JS props.
const primary = __langs.length ? __langs[0] : 'en-US';
const patchLangs = (target) => {
if (!target) return;
sealGetter(target, 'language', () => primary);
sealGetter(target, 'languages', () => Object.freeze(__langs.slice()));
};
patchLangs(Object.getPrototypeOf(navigator));
if (typeof WorkerNavigator !== 'undefined') {
patchLangs(WorkerNavigator.prototype);
}
// --- screen dimensions ---
// EmulationSetDeviceMetricsOverride sets the CSS viewport but leaves
// window.screen.* at headless defaults.
const screenProto = typeof Screen !== 'undefined' ? Screen.prototype : null;
if (screenProto) {
sealGetter(screenProto, 'width', () => __screenW);
sealGetter(screenProto, 'height', () => __screenH);
sealGetter(screenProto, 'availWidth', () => __availW);
sealGetter(screenProto, 'availHeight', () => __availH);
sealGetter(screenProto, 'availLeft', () => 0);
sealGetter(screenProto, 'availTop', () => __availTop);
// Headless reports colorDepth/pixelDepth=24 already in most builds, but
// some checks see 0 in WSL/Docker. Lock to 24 which matches real Chrome.
sealGetter(screenProto, 'colorDepth', () => 24);
sealGetter(screenProto, 'pixelDepth', () => 24);
}
if (typeof window !== 'undefined') {
sealGetter(window, 'outerWidth', () => __outerW);
sealGetter(window, 'outerHeight', () => __outerH);
}
// --- WebGL vendor/renderer spoof ---
// Patches getParameter(37445=UNMASKED_VENDOR_WEBGL, 37446=UNMASKED_RENDERER_WEBGL)
// on WebGLRenderingContext.prototype and WebGL2RenderingContext.prototype.
// Only delegates to the original for all other params, so behavioral
// signals (extension list, real pixel rendering) still pass through.
const patchWebGLProto = (proto) => {
if (!proto) return;
const original = proto.getParameter;
if (typeof original !== 'function') return;
const replacement = function getParameter(parameter) {
if (parameter === 37445) return __webglVendor;
if (parameter === 37446) return __webglRenderer;
return original.apply(this, arguments);
};
try {
Object.defineProperty(proto, 'getParameter', {
value: replacement,
writable: true,
enumerable: false,
configurable: true,
});
} catch (_) {}
};
if (typeof WebGLRenderingContext !== 'undefined') {
patchWebGLProto(WebGLRenderingContext.prototype);
}
if (typeof WebGL2RenderingContext !== 'undefined') {
patchWebGLProto(WebGL2RenderingContext.prototype);
}
// --- navigator.plugins / mimeTypes ---
// Match real Chrome 136 exactly: 5 plugins (all "internal-pdf-viewer"), each
// exposing application/pdf + text/pdf. navigator.mimeTypes dedupes to 2.
// Each MimeType.enabledPlugin must back-reference the FIRST plugin that owns
// that type (Chrome's invariant). Plugin order is fixed.
try {
const PluginArrayProto = typeof PluginArray !== 'undefined' ? PluginArray.prototype : null;
const PluginProto = typeof Plugin !== 'undefined' ? Plugin.prototype : null;
const MimeTypeArrayProto = typeof MimeTypeArray !== 'undefined' ? MimeTypeArray.prototype : null;
const MimeTypeProto = typeof MimeType !== 'undefined' ? MimeType.prototype : null;
if (PluginArrayProto && PluginProto && MimeTypeArrayProto && MimeTypeProto) {
const pluginNames = [
'PDF Viewer',
'Chrome PDF Viewer',
'Chromium PDF Viewer',
'Microsoft Edge PDF Viewer',
'WebKit built-in PDF',
];
const mimeSpecs = [
{ type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format' },
{ type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format' },
];
// Two MimeType instances, each enabledPlugin points to the first plugin
// (PDF Viewer) per Chrome's invariant: navigator.mimeTypes[i].enabledPlugin
// === navigator.plugins[0] for every PDF mime.
const sharedMimes = mimeSpecs.map((spec) => {
const m = Object.create(MimeTypeProto);
Object.defineProperty(m, 'type', { value: spec.type, enumerable: true });
Object.defineProperty(m, 'suffixes', { value: spec.suffixes, enumerable: true });
Object.defineProperty(m, 'description', { value: spec.description, enumerable: true });
return m;
});
const plugins = pluginNames.map((name) => {
const p = Object.create(PluginProto);
Object.defineProperty(p, 'name', { value: name, enumerable: true });
Object.defineProperty(p, 'filename', { value: 'internal-pdf-viewer', enumerable: true });
Object.defineProperty(p, 'description', { value: 'Portable Document Format', enumerable: true });
Object.defineProperty(p, 'length', { value: sharedMimes.length, enumerable: true });
sharedMimes.forEach((m, i) => {
Object.defineProperty(p, String(i), { value: m, enumerable: true });
Object.defineProperty(p, m.type, { value: m });
});
return p;
});
// Set enabledPlugin AFTER plugins are constructed, pointing to plugins[0].
sharedMimes.forEach((m) => {
Object.defineProperty(m, 'enabledPlugin', { value: plugins[0], enumerable: true });
});
const pluginArr = Object.create(PluginArrayProto);
Object.defineProperty(pluginArr, 'length', { value: plugins.length, enumerable: true });
plugins.forEach((p, i) => {
Object.defineProperty(pluginArr, String(i), { value: p, enumerable: true });
Object.defineProperty(pluginArr, p.name, { value: p });
});
const mimeArr = Object.create(MimeTypeArrayProto);
Object.defineProperty(mimeArr, 'length', { value: sharedMimes.length, enumerable: true });
sharedMimes.forEach((m, i) => {
Object.defineProperty(mimeArr, String(i), { value: m, enumerable: true });
Object.defineProperty(mimeArr, m.type, { value: m });
});
sealGetter(Object.getPrototypeOf(navigator), 'plugins', () => pluginArr);
sealGetter(Object.getPrototypeOf(navigator), 'mimeTypes', () => mimeArr);
}
} catch (_) {}
// --- navigator.permissions.query notifications fix ---
// headless returns 'denied' for notifications when Notification.permission is
// 'default'. Real Chrome returns 'prompt' in that case. Sannysoft checks this
// mismatch (permissions_new / headchr_permissions).
try {
if (navigator.permissions && typeof navigator.permissions.query === 'function') {
const proto = Object.getPrototypeOf(navigator.permissions);
const desc = Object.getOwnPropertyDescriptor(proto, 'query');
if (desc && typeof desc.value === 'function') {
const original = desc.value;
const replacement = function query(parameters) {
if (parameters && parameters.name === 'notifications' &&
typeof Notification !== 'undefined' && Notification.permission === 'default') {
return Promise.resolve({ state: 'prompt', onchange: null });
}
return original.apply(this, arguments);
};
Object.defineProperty(proto, 'query', {
value: replacement,
writable: desc.writable,
enumerable: desc.enumerable,
configurable: desc.configurable,
});
}
}
} catch (_) {}
// --- getBoundingClientRect / getClientRects subpixel jitter ---
// Headless Chrome returns integer-valued rects; real Chrome returns subpixel
// floats due to CSS layout fractions. Fingerprinters hash rect tuples; even
// a sub-pixel offset breaks the canonical "headless rect" hash.
// Jitter is deterministic per-element (based on element identity) so the
// same element returns the same value across calls within the page lifetime.
try {
const rectProto = typeof DOMRect !== 'undefined' ? DOMRect.prototype : null;
const elProto = typeof Element !== 'undefined' ? Element.prototype : null;
if (rectProto && elProto) {
const wmJitter = new WeakMap();
const jitterFor = (el) => {
let j = wmJitter.get(el);
if (!j) {
// Tiny noise in [-0.05, +0.05) — well below visual threshold but
// changes hash output. Generated once per element.
j = {
x: (Math.random() - 0.5) * 0.1,
y: (Math.random() - 0.5) * 0.1,
};
wmJitter.set(el, j);
}
return j;
};
const origGBCR = elProto.getBoundingClientRect;
Object.defineProperty(elProto, 'getBoundingClientRect', {
value: function getBoundingClientRect() {
const r = origGBCR.apply(this, arguments);
const j = jitterFor(this);
// DOMRect is mutable; nudge x/y. width/height left intact so layout
// calculations don't drift.
try { r.x = r.x + j.x; r.y = r.y + j.y; } catch (_) {}
return r;
},
writable: true,
enumerable: false,
configurable: true,
});
}
} catch (_) {}
// --- window.chrome.runtime stub ---
// Real Chrome exposes window.chrome with a .runtime sub-object.
// headless leaves window.chrome empty, which sannysoft (chrome_new,
// headchr_chrome_obj) flags. A minimal runtime stub satisfies the check
// without touching method behavior.
try {
if (typeof window !== 'undefined') {
if (!window.chrome) {
Object.defineProperty(window, 'chrome', { value: {}, writable: true, configurable: true });
}
if (window.chrome && !window.chrome.runtime) {
Object.defineProperty(window.chrome, 'runtime', {
value: {
OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', UPDATE: 'update' },
OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' },
PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', WIN: 'win' },
},
writable: true,
enumerable: true,
configurable: true,
});
}
}
} catch (_) {}
})();

View File

@@ -23,24 +23,23 @@ type Viewport struct {
}
type Profile struct {
ID string `json:"id"`
UserAgent string `json:"user_agent"`
UACHBrands []BrandVersion `json:"uach_brands"`
UACHFullVerList []BrandVersion `json:"uach_full_version_list"`
Platform string `json:"platform"`
PlatformVersion string `json:"platform_version"`
Architecture string `json:"architecture"`
Bitness string `json:"bitness"`
Mobile bool `json:"mobile"`
AcceptLanguage string `json:"accept_language"`
NavigatorLangs []string `json:"navigator_langs"`
Locale string `json:"locale"`
Timezone string `json:"timezone"`
Viewport Viewport `json:"viewport"`
WebGLVendor string `json:"webgl_vendor"`
WebGLRenderer string `json:"webgl_renderer"`
Tags []string `json:"tags"`
Weight int `json:"weight"`
ID string `json:"id"`
UserAgentTemplate string `json:"user_agent_template"`
UserAgent string `json:"user_agent,omitempty"`
UACHBrands []BrandVersion `json:"uach_brands,omitempty"`
UACHFullVerList []BrandVersion `json:"uach_full_version_list,omitempty"`
Platform string `json:"platform"`
PlatformVersion string `json:"platform_version"`
Architecture string `json:"architecture"`
Bitness string `json:"bitness"`
Mobile bool `json:"mobile"`
AcceptLanguage string `json:"accept_language"`
NavigatorLangs []string `json:"navigator_langs"`
Locale string `json:"locale"`
Timezone string `json:"timezone"`
Viewport Viewport `json:"viewport"`
Tags []string `json:"tags"`
Weight int `json:"weight"`
}
type catalogConfig struct {
@@ -51,18 +50,13 @@ type catalogConfig struct {
const (
ProfileChromeWinUS = "chrome-win-uhd620"
ProfileChromeWinRU = "chrome-win-ru"
ProfileChromeMacUS = "chrome-macos-intel-iris"
ProfileChromeLinuxUS = "chrome-linux-mesa-uhd620"
ProfileChromeLinuxRU = "chrome-linux-ru"
)
//go:embed profiles.json
var defaultProfilesJSON []byte
//go:embed patch.js
var PatchJS []byte
var profileCatalogMu sync.RWMutex
var catalog = map[string]Profile{}
@@ -110,8 +104,8 @@ func loadProfilesFromJSONBytes(data []byte) error {
if profile.ID == "" {
return fmt.Errorf("profiles[%d].id is empty", i)
}
if strings.TrimSpace(profile.UserAgent) == "" {
return fmt.Errorf("profiles[%d].user_agent is empty", i)
if strings.TrimSpace(profile.UserAgentTemplate) == "" && strings.TrimSpace(profile.UserAgent) == "" {
return fmt.Errorf("profiles[%d] has no user_agent_template or user_agent", i)
}
if _, exists := nextCatalog[profile.ID]; exists {
return fmt.Errorf("duplicate profile id %q", profile.ID)
@@ -201,6 +195,13 @@ func SelectProfile(engine string, region string) Profile {
// Non-empty salt uses weighted selection seeded by FNV-1a hash of salt,
// giving each session a stable but varied profile.
func SelectProfileForSession(engine, region, salt string) Profile {
return SelectProfileForSessionHeadless(engine, region, salt, false)
}
// SelectProfileForSessionHeadless is SelectProfileForSession, but headless Linux
// (the Docker deployment) renders WebGL via SwiftShader, so it only picks
// swiftshader-tagged profiles. Runtimes with a real GPU exclude them.
func SelectProfileForSessionHeadless(engine, region, salt string, headless bool) Profile {
engine = NormalizeEngine(engine)
region = NormalizeRegion(region)
if region == "" {
@@ -215,7 +216,7 @@ func SelectProfileForSession(engine, region, salt string) Profile {
return profileByID(profileID)
}
pool := eligibleProfiles(engine, region)
pool := eligibleProfiles(runtime.GOOS, headless)
return pickWeighted(pool, salt)
}
@@ -224,10 +225,11 @@ type weightedProfile struct {
weight int
}
// eligibleProfiles builds the weighted pool for (engine, region).
// Linux profiles are preferred 4x on linux runtime; Windows 4x on windows; macOS 4x on darwin.
// Profiles tagged "ru" are included only when region == "ru"; "ru"-tagged profiles are excluded otherwise.
func eligibleProfiles(engine, region string) []weightedProfile {
// eligibleProfiles builds the pool for the runtime platform. Headless Linux
// keeps only swiftshader-tagged profiles (SwiftShader WebGL); a real GPU
// excludes them. We no longer spoof WebGL, so the GPU sub-tag (nvidia/amd/mesa)
// only steers selection - it matches reality on Docker, cosmetic on a headful box.
func eligibleProfiles(goos string, headless bool) []weightedProfile {
profileCatalogMu.RLock()
snap := make([]Profile, 0, len(catalog))
for _, p := range catalog {
@@ -240,50 +242,52 @@ func eligibleProfiles(engine, region string) []weightedProfile {
return strings.Compare(a.ID, b.ID)
})
goos := runtime.GOOS
platformTag := runtimePlatformTag(goos)
wantSwiftShader := headless && goos == "linux"
var pool []weightedProfile
for _, p := range snap {
isRu := slices.Contains(p.Tags, "ru")
if region == "ru" && !isRu {
if platformTag != "" && !slices.Contains(p.Tags, platformTag) {
continue
}
if region != "ru" && isRu {
if slices.Contains(p.Tags, "swiftshader") != wantSwiftShader {
continue
}
w := p.Weight
if w <= 0 {
w = 1
}
platformLower := strings.ToLower(p.Platform)
switch goos {
case "linux":
if platformLower == "linux" {
w *= 4
}
case "windows":
if platformLower == "windows" {
w *= 4
}
case "darwin":
if platformLower == "macos" {
w *= 4
}
}
pool = append(pool, weightedProfile{profile: p, weight: w})
}
// Fall back to the platform pool if no swiftshader profile exists yet, so
// selection never returns empty.
if len(pool) == 0 && wantSwiftShader {
return eligibleProfiles(goos, false)
}
return pool
}
func runtimePlatformTag(goos string) string {
switch strings.ToLower(strings.TrimSpace(goos)) {
case "windows":
return "windows"
case "darwin":
return "macos"
case "linux":
return "linux"
default:
return ""
}
}
// pickWeighted selects a profile from pool using FNV-1a hash of salt modulo total weight.
// Empty salt returns the first profile in the pool (deterministic for tests).
func pickWeighted(pool []weightedProfile, salt string) Profile {
if len(pool) == 0 {
return profileByID(defaultProfileID("us"))
return profileByID(defaultProfileID())
}
if salt == "" {
return pool[0].profile
@@ -384,7 +388,7 @@ func profileByID(profileID string) Profile {
if profile, ok := catalog[profileID]; ok {
return profile
}
if fallback, ok := catalog[defaultProfileID("us")]; ok {
if fallback, ok := catalog[defaultProfileID()]; ok {
return fallback
}
for _, profile := range catalog {
@@ -393,24 +397,13 @@ func profileByID(profileID string) Profile {
return Profile{}
}
func defaultProfileID(region string) string {
region = NormalizeRegion(region)
if region == "" {
region = "us"
}
func defaultProfileID() string {
switch runtime.GOOS {
case "windows":
if region == "ru" {
return ProfileChromeWinRU
}
return ProfileChromeWinUS
case "darwin":
return ProfileChromeMacUS
default:
if region == "ru" {
return ProfileChromeLinuxRU
}
return ProfileChromeLinuxUS
}
}

View File

@@ -51,12 +51,12 @@ func TestProfileCoherence(t *testing.T) {
region string
}{
{
name: "windows lane",
name: "ru lane",
engine: "google",
region: "ru",
},
{
name: "mac lane",
name: "us lane",
engine: "bing",
region: "en-US",
},
@@ -84,7 +84,7 @@ func TestProfileCoherence(t *testing.T) {
}
expected := selectedProfileFromContext(t, ctx)
expected.UserAgent = expectedUserAgentForRuntime(expected.UserAgent, got.UserAgent)
expected.UserAgent = expectedUserAgentForRuntime(expectedProfileUserAgent(expected, got.UserAgent), got.UserAgent)
if got.UserAgent != expected.UserAgent {
t.Fatalf("navigator.userAgent mismatch:\nexpected: %s\nactual: %s", expected.UserAgent, got.UserAgent)
}
@@ -106,11 +106,26 @@ func TestProfileCoherence(t *testing.T) {
if got.NavigatorLanguages[0] != expected.NavigatorLangs[0] {
t.Fatalf("navigator.languages[0] mismatch: expected %q got %q", expected.NavigatorLangs[0], got.NavigatorLanguages[0])
}
if got.WebdriverType != "undefined" {
t.Fatalf("navigator.webdriver expected undefined, got %q", got.WebdriverType)
// Real Chrome exposes navigator.webdriver as an inherited boolean
// that reads false; --disable-blink-features=AutomationControlled
// gives us that. Deleting the property (typeof undefined) is itself a
// bot tell, so we assert the genuine-browser shape instead.
if got.WebdriverType != "boolean" {
t.Fatalf("navigator.webdriver expected boolean, got %q", got.WebdriverType)
}
if got.WebdriverValue {
t.Fatal("navigator.webdriver should read false")
}
if got.WebdriverOwnPropPresent {
t.Fatal("navigator own property 'webdriver' should not be present")
t.Fatal("navigator.webdriver should be inherited, not an own property")
}
// hardwareConcurrency is left native (no override), so main and
// worker must simply agree on the machine's real core count.
if got.HardwareConcurrency <= 0 {
t.Fatal("navigator.hardwareConcurrency should be reported")
}
if got.WorkerHardwareConcurrency != got.HardwareConcurrency {
t.Fatalf("worker hardwareConcurrency mismatch: main %d worker %d", got.HardwareConcurrency, got.WorkerHardwareConcurrency)
}
if got.WorkerUserAgent != got.UserAgent {
t.Fatalf("worker userAgent mismatch: main %q worker %q", got.UserAgent, got.WorkerUserAgent)
@@ -127,11 +142,11 @@ func TestProfileCoherence(t *testing.T) {
if got.WorkerTimezone != got.Timezone {
t.Fatalf("worker timezone mismatch: main %q worker %q", got.Timezone, got.WorkerTimezone)
}
if got.WorkerWebGLVendor != expected.WebGLVendor {
t.Fatalf("worker WebGL vendor mismatch: expected %q got %q", expected.WebGLVendor, got.WorkerWebGLVendor)
if got.WorkerWebGLVendor != got.WebGLVendor {
t.Fatalf("worker WebGL vendor mismatch: main %q worker %q", got.WebGLVendor, got.WorkerWebGLVendor)
}
if got.WorkerWebGLRenderer != expected.WebGLRenderer {
t.Fatalf("worker WebGL renderer mismatch: expected %q got %q", expected.WebGLRenderer, got.WorkerWebGLRenderer)
if got.WorkerWebGLRenderer != got.WebGLRenderer {
t.Fatalf("worker WebGL renderer mismatch: main %q worker %q", got.WebGLRenderer, got.WorkerWebGLRenderer)
}
if got.InnerHeight >= got.OuterHeight {
t.Fatalf("innerHeight should be smaller than outerHeight, got inner=%d outer=%d", got.InnerHeight, got.OuterHeight)
@@ -139,32 +154,37 @@ func TestProfileCoherence(t *testing.T) {
if got.OuterHeight > got.ScreenAvailHeight {
t.Fatalf("outerHeight should fit in screen.availHeight, got outer=%d avail=%d", got.OuterHeight, got.ScreenAvailHeight)
}
if got.ScreenAvailHeight >= got.ScreenHeight {
t.Fatalf("screen.availHeight should be smaller than screen.height, got avail=%d screen=%d", got.ScreenAvailHeight, got.ScreenHeight)
if got.ScreenAvailHeight > got.ScreenHeight {
t.Fatalf("screen.availHeight should fit in screen.height, got avail=%d screen=%d", got.ScreenAvailHeight, got.ScreenHeight)
}
})
}
}
type profileSurface struct {
UserAgent string `json:"userAgent"`
Platform string `json:"platform"`
NavigatorPlatform string `json:"navigatorPlatform"`
NavigatorLanguages []string `json:"navigatorLanguages"`
Timezone string `json:"timezone"`
Locale string `json:"locale"`
WebdriverType string `json:"webdriverType"`
WebdriverOwnPropPresent bool `json:"webdriverOwnPropPresent"`
WorkerUserAgent string `json:"workerUserAgent"`
WorkerPlatform string `json:"workerPlatform"`
WorkerNavigatorLangs []string `json:"workerNavigatorLangs"`
WorkerTimezone string `json:"workerTimezone"`
WorkerWebGLVendor string `json:"workerWebGLVendor"`
WorkerWebGLRenderer string `json:"workerWebGLRenderer"`
InnerHeight int `json:"innerHeight"`
OuterHeight int `json:"outerHeight"`
ScreenHeight int `json:"screenHeight"`
ScreenAvailHeight int `json:"screenAvailHeight"`
UserAgent string `json:"userAgent"`
Platform string `json:"platform"`
NavigatorPlatform string `json:"navigatorPlatform"`
NavigatorLanguages []string `json:"navigatorLanguages"`
Timezone string `json:"timezone"`
Locale string `json:"locale"`
WebdriverType string `json:"webdriverType"`
WebdriverValue bool `json:"webdriverValue"`
WebdriverOwnPropPresent bool `json:"webdriverOwnPropPresent"`
HardwareConcurrency int `json:"hardwareConcurrency"`
WorkerHardwareConcurrency int `json:"workerHardwareConcurrency"`
WorkerUserAgent string `json:"workerUserAgent"`
WorkerPlatform string `json:"workerPlatform"`
WorkerNavigatorLangs []string `json:"workerNavigatorLangs"`
WorkerTimezone string `json:"workerTimezone"`
WebGLVendor string `json:"webGLVendor"`
WebGLRenderer string `json:"webGLRenderer"`
WorkerWebGLVendor string `json:"workerWebGLVendor"`
WorkerWebGLRenderer string `json:"workerWebGLRenderer"`
InnerHeight int `json:"innerHeight"`
OuterHeight int `json:"outerHeight"`
ScreenHeight int `json:"screenHeight"`
ScreenAvailHeight int `json:"screenAvailHeight"`
}
func browserProfileSurface(page *rod.Page) (profileSurface, error) {
@@ -205,6 +225,25 @@ func selectedProfileFromContext(t *testing.T, ctx context.Context) browserprofil
return profile
}
// expectedProfileUserAgent resolves the profile's user agent, expanding the
// {chrome_major} template with the runtime's major when the profile carries a
// template rather than a literal UA.
func expectedProfileUserAgent(profile browserprofile.Profile, runtimeUserAgent string) string {
if ua := strings.TrimSpace(profile.UserAgent); ua != "" {
return ua
}
template := strings.TrimSpace(profile.UserAgentTemplate)
if template == "" {
return ""
}
major := chromeToken(runtimeUserAgent)
major = strings.TrimPrefix(major, "Chrome/")
if idx := strings.IndexByte(major, '.'); idx >= 0 {
major = major[:idx]
}
return strings.ReplaceAll(template, "{chrome_major}", major)
}
func expectedUserAgentForRuntime(profileUserAgent, runtimeUserAgent string) string {
runtimeChrome := chromeToken(runtimeUserAgent)
if runtimeChrome == "" {

View File

@@ -1,4 +1,17 @@
async () => {
const readWebGL = (canvas) => {
try {
const gl = canvas ? (canvas.getContext('webgl') || canvas.getContext('experimental-webgl') || canvas.getContext('webgl2')) : null;
const debugInfo = gl && gl.getExtension('WEBGL_debug_renderer_info');
return {
vendor: gl && debugInfo ? (gl.getParameter(debugInfo.UNMASKED_VENDOR_WEBGL) || '') : '',
renderer: gl && debugInfo ? (gl.getParameter(debugInfo.UNMASKED_RENDERER_WEBGL) || '') : '',
};
} catch (_) {
return { vendor: '', renderer: '' };
}
};
const webGL = readWebGL(document.createElement('canvas'));
const workerData = await new Promise((resolve) => {
try {
const source = [
@@ -19,6 +32,7 @@ async () => {
"platform: self.navigator.platform || '',",
"navigatorLanguages: Array.from(self.navigator.languages || []),",
"timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || '',",
"hardwareConcurrency: self.navigator.hardwareConcurrency || 0,",
"webGLVendor,",
"webGLRenderer,",
"});",
@@ -51,11 +65,16 @@ async () => {
timezone: Intl.DateTimeFormat().resolvedOptions().timeZone || "",
locale: Intl.DateTimeFormat().resolvedOptions().locale || "",
webdriverType: typeof navigator.webdriver,
webdriverValue: navigator.webdriver === true,
webdriverOwnPropPresent: Object.getOwnPropertyNames(navigator).includes("webdriver"),
hardwareConcurrency: navigator.hardwareConcurrency || 0,
workerHardwareConcurrency: workerData.hardwareConcurrency || 0,
workerUserAgent: workerData.userAgent || "",
workerPlatform: workerData.platform || "",
workerNavigatorLangs: Array.from(workerData.navigatorLanguages || []),
workerTimezone: workerData.timezone || "",
webGLVendor: webGL.vendor,
webGLRenderer: webGL.renderer,
workerWebGLVendor: workerData.webGLVendor || "",
workerWebGLRenderer: workerData.webGLRenderer || "",
innerHeight: window.innerHeight || 0,

View File

@@ -69,14 +69,73 @@ func TestSelectProfileForSession(t *testing.T) {
}
})
t.Run("ru region returns ru-tagged profile", func(t *testing.T) {
t.Run("locale does not change hardware profile pool", func(t *testing.T) {
p := SelectProfileForSession("yandex", "ru", "some-session")
if !slices.Contains(p.Tags, "ru") {
t.Fatalf("expected ru-tagged profile, got ID=%q tags=%v", p.ID, p.Tags)
if slices.Contains(p.Tags, "ru") {
t.Fatalf("expected locale-neutral profile, got ID=%q tags=%v", p.ID, p.Tags)
}
})
}
func TestEligibleProfilesMatchRuntimePlatform(t *testing.T) {
tests := []struct {
goos string
tag string
}{
{goos: "linux", tag: "linux"},
{goos: "windows", tag: "windows"},
{goos: "darwin", tag: "macos"},
}
for _, tt := range tests {
t.Run(tt.goos, func(t *testing.T) {
pool := eligibleProfiles(tt.goos, false)
if len(pool) == 0 {
t.Fatal("expected eligible profiles")
}
for _, candidate := range pool {
if !slices.Contains(candidate.profile.Tags, tt.tag) {
t.Fatalf("profile %q does not match %s", candidate.profile.ID, tt.goos)
}
}
})
}
}
// Headless Linux (the Docker deployment) has no real GPU, so only
// swiftshader-tagged profiles are eligible; headful Linux excludes them.
func TestEligibleProfilesHeadlessLinuxUsesSwiftShader(t *testing.T) {
headless := eligibleProfiles("linux", true)
if len(headless) == 0 {
t.Fatal("expected a Linux SwiftShader profile")
}
for _, candidate := range headless {
if !slices.Contains(candidate.profile.Tags, "swiftshader") {
t.Fatalf("headless profile %q is not a SwiftShader profile", candidate.profile.ID)
}
}
if len(headless) < 2 {
t.Fatalf("expected multiple SwiftShader profiles so Docker is not a single fingerprint, got %d", len(headless))
}
timezones := make(map[string]struct{}, len(headless))
for _, candidate := range headless {
timezones[candidate.profile.Timezone] = struct{}{}
}
if len(timezones) < 2 {
t.Fatalf("expected distinct CDP-visible SwiftShader profiles, got timezones %v", timezones)
}
headful := eligibleProfiles("linux", false)
if len(headful) == 0 {
t.Fatal("expected headful Linux profiles")
}
for _, candidate := range headful {
if slices.Contains(candidate.profile.Tags, "swiftshader") {
t.Fatalf("headful profile %q should not be a SwiftShader profile", candidate.profile.ID)
}
}
}
func TestNormalizeRegion(t *testing.T) {
tests := []struct {
input string

View File

@@ -2,399 +2,96 @@
"profiles": [
{
"id": "chrome-linux-mesa-uhd620",
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Linux",
"platform_version": "6.1.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080},
"webgl_vendor": "Intel Inc.",
"webgl_renderer": "Mesa Intel(R) UHD Graphics 620 (KBL GT2)",
"tags": ["linux", "mesa", "integrated"],
"weight": 3
"user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "mesa", "integrated"], "weight": 3
},
{
"id": "chrome-linux-mesa-iris-xe",
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Linux",
"platform_version": "6.1.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080},
"webgl_vendor": "Intel Inc.",
"webgl_renderer": "Mesa Intel(R) Graphics (RPL-S)",
"tags": ["linux", "mesa", "integrated"],
"weight": 2
"user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "mesa", "integrated"], "weight": 2
},
{
"id": "chrome-linux-nvidia",
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Linux",
"platform_version": "6.1.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080},
"webgl_vendor": "Google Inc. (NVIDIA Corporation)",
"webgl_renderer": "ANGLE (NVIDIA Corporation, NVIDIA GeForce GTX 1660 SUPER/PCIe/SSE2, OpenGL 4.5.0 NVIDIA 535.86.05)",
"tags": ["linux", "nvidia"],
"weight": 2
"user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "nvidia"], "weight": 2
},
{
"id": "chrome-linux-amd",
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Linux",
"platform_version": "6.1.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080},
"webgl_vendor": "Google Inc. (AMD)",
"webgl_renderer": "ANGLE (AMD, AMD Radeon RX 6600 (radeonsi, navi23, ACO, DRM 3.42.0, 5.15.0-91-generic), OpenGL 4.6 (Core Profile))",
"tags": ["linux", "amd"],
"weight": 1
"user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "amd"], "weight": 1
},
{
"id": "chrome-linux-swiftshader",
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Linux",
"platform_version": "6.1.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1366, "height": 768},
"webgl_vendor": "Google Inc. (Google)",
"webgl_renderer": "ANGLE (Google, Vulkan 1.3.0 (SwiftShader Device (Subzero) (0x0000C0DE)), SwiftShader driver)",
"tags": ["linux", "swiftshader", "headless"],
"weight": 1
"user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "swiftshader", "headless"], "weight": 1
},
{
"id": "chrome-linux-ru",
"user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Linux",
"platform_version": "6.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
"navigator_langs": ["ru-RU"],
"locale": "ru-RU",
"timezone": "Europe/Moscow",
"viewport": {"width": 1920, "height": 1080},
"webgl_vendor": "Intel Inc.",
"webgl_renderer": "Mesa Intel(R) UHD Graphics 620 (KBL GT2)",
"tags": ["linux", "ru"],
"weight": 1
"id": "chrome-linux-swiftshader-fhd",
"user_agent_template": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "Linux", "platform_version": "6.1.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/Chicago",
"viewport": {"width": 1920, "height": 1080}, "tags": ["linux", "swiftshader", "headless"], "weight": 1
},
{
"id": "chrome-win-uhd620",
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Windows",
"platform_version": "15.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080},
"webgl_vendor": "Intel Inc.",
"webgl_renderer": "ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D11 vs_5_0 ps_5_0, D3D11)",
"tags": ["windows"],
"weight": 2
"user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080}, "tags": ["windows"], "weight": 2
},
{
"id": "chrome-win-nvidia",
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Windows",
"platform_version": "15.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080},
"webgl_vendor": "Google Inc. (NVIDIA)",
"webgl_renderer": "ANGLE (NVIDIA, NVIDIA GeForce RTX 3060 Direct3D11 vs_5_0 ps_5_0, D3D11)",
"tags": ["windows", "nvidia"],
"weight": 2
"user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080}, "tags": ["windows", "nvidia"], "weight": 2
},
{
"id": "chrome-win-iris-xe",
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Windows",
"platform_version": "15.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1536, "height": 864},
"webgl_vendor": "Google Inc. (Intel)",
"webgl_renderer": "ANGLE (Intel, Intel(R) Iris(R) Xe Graphics Direct3D11 vs_5_0 ps_5_0, D3D11)",
"tags": ["windows", "integrated"],
"weight": 1
"user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080}, "tags": ["windows", "integrated"], "weight": 1
},
{
"id": "chrome-win-amd",
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Windows",
"platform_version": "15.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080},
"webgl_vendor": "Google Inc. (AMD)",
"webgl_renderer": "ANGLE (AMD, AMD Radeon RX 6600 Direct3D11 vs_5_0 ps_5_0, D3D11)",
"tags": ["windows", "amd"],
"weight": 1
},
{
"id": "chrome-win-ru",
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "Windows",
"platform_version": "15.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
"navigator_langs": ["ru-RU"],
"locale": "ru-RU",
"timezone": "Europe/Moscow",
"viewport": {"width": 1920, "height": 1080},
"webgl_vendor": "Intel Inc.",
"webgl_renderer": "ANGLE (Intel, Intel(R) UHD Graphics 620 Direct3D11 vs_5_0 ps_5_0, D3D11)",
"tags": ["windows", "ru"],
"weight": 1
"user_agent_template": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "Windows", "platform_version": "15.0.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1920, "height": 1080}, "tags": ["windows", "amd"], "weight": 1
},
{
"id": "chrome-macos-m1",
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "macOS",
"platform_version": "13.0.0",
"architecture": "arm",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1440, "height": 900},
"webgl_vendor": "Google Inc. (Apple)",
"webgl_renderer": "ANGLE (Apple, ANGLE Metal Renderer: Apple M1, Unspecified Version)",
"tags": ["macos", "apple-silicon"],
"weight": 2
"user_agent_template": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "macOS", "platform_version": "13.0.0", "architecture": "arm", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1440, "height": 900}, "tags": ["macos", "apple-silicon"], "weight": 2
},
{
"id": "chrome-macos-m2",
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "macOS",
"platform_version": "14.0.0",
"architecture": "arm",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1512, "height": 982},
"webgl_vendor": "Google Inc. (Apple)",
"webgl_renderer": "ANGLE (Apple, ANGLE Metal Renderer: Apple M2, Unspecified Version)",
"tags": ["macos", "apple-silicon"],
"weight": 2
"user_agent_template": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "macOS", "platform_version": "14.0.0", "architecture": "arm", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1512, "height": 982}, "tags": ["macos", "apple-silicon"], "weight": 2
},
{
"id": "chrome-macos-intel-iris",
"user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/136.0.0.0 Safari/537.36",
"uach_brands": [
{"brand": "Not_A Brand", "version": "24"},
{"brand": "Chromium", "version": "136"},
{"brand": "Google Chrome", "version": "136"}
],
"uach_full_version_list": [
{"brand": "Not_A Brand", "version": "24.0.0.0"},
{"brand": "Chromium", "version": "136.0.0.0"},
{"brand": "Google Chrome", "version": "136.0.0.0"}
],
"platform": "macOS",
"platform_version": "14.0.0",
"architecture": "x86",
"bitness": "64",
"mobile": false,
"accept_language": "en-US,en;q=0.9",
"navigator_langs": ["en-US"],
"locale": "en-US",
"timezone": "America/New_York",
"viewport": {"width": 1680, "height": 1050},
"webgl_vendor": "Intel Inc.",
"webgl_renderer": "Intel Iris OpenGL Engine",
"tags": ["macos", "integrated"],
"weight": 1
"user_agent_template": "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{chrome_major}.0.0.0 Safari/537.36",
"platform": "macOS", "platform_version": "14.0.0", "architecture": "x86", "bitness": "64", "mobile": false,
"accept_language": "en-US,en;q=0.9", "navigator_langs": ["en-US"], "locale": "en-US", "timezone": "America/New_York",
"viewport": {"width": 1680, "height": 1050}, "tags": ["macos", "integrated"], "weight": 1
}
],
"lane_profile_ids": {},
"default_region_by_engine": {
"yandex": "ru"
}
"default_region_by_engine": {"yandex": "ru"}
}

View File

@@ -61,6 +61,61 @@ func TestResolveBrowserBinaryPathRejectsInvalidExplicit(t *testing.T) {
}
}
func TestBrowserLaunchLanguageIsProcessStable(t *testing.T) {
tests := []struct {
name string
opts BrowserOpts
want string
}{
{name: "default locale", opts: BrowserOpts{}, want: "en-US"},
{name: "request hint does not change process locale", opts: BrowserOpts{LanguageCode: "de"}, want: "en-US"},
{name: "regional hint does not change process locale", opts: BrowserOpts{LanguageCode: "en-GB"}, want: "en-US"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := browserLaunchLanguage(tt.opts); got != tt.want {
t.Fatalf("browserLaunchLanguage() = %q, want %q", got, tt.want)
}
})
}
}
func TestProfileNavigatorLanguagesStripsHeaderWeights(t *testing.T) {
profile := browserprofile.Profile{
AcceptLanguage: "en-US,en;q=0.9",
NavigatorLangs: []string{"en-US"},
}
got := profileNavigatorLanguages(profile)
want := []string{"en-US", "en"}
if len(got) != len(want) {
t.Fatalf("profileNavigatorLanguages() = %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("profileNavigatorLanguages() = %v, want %v", got, want)
}
}
}
func TestProfileNavigatorLanguagesForRuntime(t *testing.T) {
profile := browserprofile.Profile{
AcceptLanguage: "en-US,en;q=0.9",
NavigatorLangs: []string{"en-US"},
}
linuxHeadless := profileNavigatorLanguagesForRuntime(profile, "linux", true)
if len(linuxHeadless) != 1 || linuxHeadless[0] != "en-US" {
t.Fatalf("linux headless languages = %v, want [en-US]", linuxHeadless)
}
windowsHeadless := profileNavigatorLanguagesForRuntime(profile, "windows", true)
if len(windowsHeadless) != 2 || windowsHeadless[0] != "en-US" || windowsHeadless[1] != "en" {
t.Fatalf("windows headless languages = %v, want [en-US en]", windowsHeadless)
}
}
func TestMinPositiveDuration(t *testing.T) {
tests := []struct {
name string
@@ -95,6 +150,35 @@ func TestApplyProfileLanguageHintRewritesTimezone(t *testing.T) {
}
}
func TestRemoveChromeBrand(t *testing.T) {
profile := browserprofile.Profile{
UACHBrands: []browserprofile.BrandVersion{
{Brand: "Not_A Brand", Version: "24"},
{Brand: "Chromium", Version: "136"},
{Brand: "Google Chrome", Version: "136"},
},
UACHFullVerList: []browserprofile.BrandVersion{
{Brand: "Chromium", Version: "136.0.0.0"},
{Brand: "Google Chrome", Version: "136.0.0.0"},
},
}
got := removeChromeBrand(profile)
for _, brand := range got.UACHBrands {
if brand.Brand == "Google Chrome" {
t.Fatal("expected Google Chrome brand to be removed from UACHBrands")
}
}
for _, brand := range got.UACHFullVerList {
if brand.Brand == "Google Chrome" {
t.Fatal("expected Google Chrome brand to be removed from UACHFullVerList")
}
}
if len(got.UACHBrands) != 2 || len(got.UACHFullVerList) != 1 {
t.Fatalf("unexpected brand counts: brands=%d fullList=%d", len(got.UACHBrands), len(got.UACHFullVerList))
}
}
func TestApplyProfileLanguageHint(t *testing.T) {
base := browserprofile.Profile{
AcceptLanguage: "en-US,en;q=0.9",

View File

@@ -39,12 +39,9 @@ func IsContextDone(err error) bool {
// PrepareEngineContext applies request-scoped metadata expected by all engine
// search implementations.
func PrepareEngineContext(ctx context.Context, query Query, engineName string, minimalBrowserProfile bool) context.Context {
func PrepareEngineContext(ctx context.Context, query Query, engineName string) context.Context {
ctx = WithEngine(EnsureContext(ctx), engineName)
ctx = WithProfileRegion(ctx, profileRegionHint(query))
if minimalBrowserProfile {
ctx = WithMinimalBrowserProfile(ctx)
}
return WithQueryHash(ctx, QueryHashFromQuery(query))
}

View File

@@ -419,15 +419,38 @@ func rawRequestProfileFor(ctx context.Context, query Query) rawRequestProfile {
func applyRawChromeMajor(profile browserprofile.Profile, major int) browserprofile.Profile {
version := strconv.Itoa(major)
if extractChromeVersion(profile.UserAgent) == "" {
if template := strings.TrimSpace(profile.UserAgentTemplate); template != "" {
profile.UserAgent = strings.ReplaceAll(template, "{chrome_major}", version)
} else if extractChromeVersion(profile.UserAgent) == "" {
profile.UserAgent = fallbackRawUserAgent
} else {
profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, version+".0.0.0")
}
if len(profile.UACHBrands) == 0 {
profile.UACHBrands = rawUACHBrands(version, false)
} else {
profile.UACHBrands = patchBrandVersions(profile.UACHBrands, version, false)
}
if len(profile.UACHFullVerList) == 0 {
profile.UACHFullVerList = rawUACHBrands(version+".0.0.0", true)
} else {
profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, version+".0.0.0", true)
}
profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, version+".0.0.0")
profile.UACHBrands = patchBrandVersions(profile.UACHBrands, version, false)
profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, version+".0.0.0", true)
return profile
}
func rawUACHBrands(version string, full bool) []browserprofile.BrandVersion {
notABrandVersion := "24"
if full {
notABrandVersion = "24.0.0.0"
}
return []browserprofile.BrandVersion{
{Brand: "Not_A Brand", Version: notABrandVersion},
{Brand: "Chromium", Version: version},
{Brand: "Google Chrome", Version: version},
}
}
func rawProfileRegion(ctx context.Context, query Query) string {
if region := profileRegionFromContext(ctx); region != "" {
return region

View File

@@ -9,7 +9,6 @@ type profileContextKey string
const profileRegionContextKey profileContextKey = "profile_region"
const forcedProfileIDContextKey profileContextKey = "forced_profile_id"
const minimalProfileContextKey profileContextKey = "minimal_profile"
func WithProfileRegion(ctx context.Context, region string) context.Context {
region = strings.TrimSpace(region)
@@ -41,12 +40,3 @@ func forcedProfileIDFromContext(ctx context.Context) string {
value, _ := EnsureContext(ctx).Value(forcedProfileIDContextKey).(string)
return strings.TrimSpace(value)
}
func WithMinimalBrowserProfile(ctx context.Context) context.Context {
return context.WithValue(EnsureContext(ctx), minimalProfileContextKey, true)
}
func minimalBrowserProfileFromContext(ctx context.Context) bool {
value, _ := EnsureContext(ctx).Value(minimalProfileContextKey).(bool)
return value
}

View File

@@ -224,6 +224,9 @@ func ProxyLaneKeyForTenant(engine string, tenant string, q Query, proxyURL strin
if sessionID == "" {
sessionID = proxyLaneIDFromProxyURL(proxyURL)
}
if sessionID == "" {
sessionID = "direct"
}
return NormalizeProxyLaneKey(ProxyLaneKey{Tenant: tenant, Engine: engine, SessionID: sessionID})
}

View File

@@ -103,3 +103,13 @@ func TestProxyLaneKeyIncludesTenant(t *testing.T) {
t.Fatalf("unexpected tenant lane id: %q", got)
}
}
func TestProxyLaneKeyUsesStableDirectSession(t *testing.T) {
key := ProxyLaneKeyForTenant("google", "tenant-a", Query{}, "")
if key.Empty() {
t.Fatal("expected a direct lane")
}
if key.SessionID != "direct" {
t.Fatalf("SessionID = %q, want direct", key.SessionID)
}
}

View File

@@ -149,7 +149,7 @@ func ddgElementHasAdMarker(el *rod.Element) bool {
// Search executes a DuckDuckGo web search and returns normalized search
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, ddg.Name(), false)
ctx = core.PrepareEngineContext(ctx, query, ddg.Name())
scoped := *ddg
scoped.logger = ddg.logger.WithRequest(ctx)
ddg = &scoped
@@ -229,7 +229,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
// SearchImage executes a DuckDuckGo image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.PrepareEngineContext(ctx, query, ddg.Name(), false)
ctx = core.PrepareEngineContext(ctx, query, ddg.Name())
scoped := *ddg
scoped.logger = ddg.logger.WithRequest(ctx)
ddg = &scoped

View File

@@ -105,7 +105,7 @@ func (e *Ecosia) parseResult(elem *rod.Element, rank int, ad bool) (core.SearchR
// Search executes an Ecosia web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, e.Name(), false)
ctx = core.PrepareEngineContext(ctx, query, e.Name())
scoped := *e
scoped.logger = e.logger.WithRequest(ctx)
e = &scoped
@@ -237,7 +237,7 @@ func elementText(el *rod.Element, selector string) string {
// query.Start is ignored: per-page card count varies, so callers should
// drive depth through query.Limit alone.
func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, e.Name(), false)
ctx = core.PrepareEngineContext(ctx, query, e.Name())
scoped := *e
scoped.logger = e.logger.WithRequest(ctx)
e = &scoped

View File

@@ -57,7 +57,7 @@ func imageResultParser(response *http.Response) ([]core.SearchResult, error) {
}
func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, "ecosia", false)
ctx = core.PrepareEngineContext(ctx, query, "ecosia")
pageNum, startRank, err := startPage(query.Start)
if err != nil {

View File

@@ -228,7 +228,7 @@ func googleElementHasAdMarker(el *rod.Element) bool {
// Search executes a Google web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, gogl.Name(), true)
ctx = core.PrepareEngineContext(ctx, query, gogl.Name())
scoped := *gogl
scoped.logger = gogl.logger.WithRequest(ctx)
gogl = &scoped
@@ -494,7 +494,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
// SearchImage executes a Google image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.PrepareEngineContext(ctx, query, gogl.Name(), true)
ctx = core.PrepareEngineContext(ctx, query, gogl.Name())
scoped := *gogl
scoped.logger = gogl.logger.WithRequest(ctx)
gogl = &scoped

View File

@@ -173,7 +173,7 @@ func isZeroResultStats(s string) bool {
}
func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, "google", false)
ctx = core.PrepareEngineContext(ctx, query, "google")
googleURL, err := BuildURL(query)
if err != nil {

View File

@@ -191,7 +191,7 @@ func (yand *Yandex) parseImageEntities(items rod.Elements) map[string]ImageEntit
// Search executes a Yandex web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, yand.Name(), false)
ctx = core.PrepareEngineContext(ctx, query, yand.Name())
scoped := *yand
scoped.logger = yand.logger.WithRequest(ctx)
yand = &scoped
@@ -286,7 +286,7 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
// SearchImage executes a Yandex image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
ctx = core.PrepareEngineContext(ctx, query, yand.Name(), false)
ctx = core.PrepareEngineContext(ctx, query, yand.Name())
scoped := *yand
scoped.logger = yand.logger.WithRequest(ctx)
yand = &scoped

View File

@@ -19,7 +19,7 @@ func classifyYandexRawHTML(body []byte) error {
}
func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
ctx = core.PrepareEngineContext(ctx, query, "yandex", false)
ctx = core.PrepareEngineContext(ctx, query, "yandex")
startPage, skipOnFirstPage, err := core.ComputePagination(query.Start, 10)
if err != nil {