diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 7aa58a5..2d526a5 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -23,6 +23,7 @@ func baiduRequest(ctx context.Context, searchURL string, query core.Query) (*htt return nil, err } req.Header.Set("User-Agent", uarand.GetRandom()) + core.SetAcceptLanguageHeader(req, query.LangCode) res, err := baseClient.Do(req) if err != nil { diff --git a/cmd/root.go b/cmd/root.go index 666be7f..40c31ac 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,7 +15,7 @@ import ( ) const ( - version = "0.7.1" + version = "0.7.2" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/browser.go b/core/browser.go index 7fc2438..92c44b0 100644 --- a/core/browser.go +++ b/core/browser.go @@ -486,6 +486,7 @@ func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browse // serialize all concurrent Navigate calls. profile := browserprofile.SelectProfile(engine, region) profile = applyRuntimeBrowserVersion(profile, browser) + profile = applyProfileLanguageHint(profile, region) if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" { profile.UserAgent = overrideUA } @@ -595,6 +596,32 @@ func navigatorPlatformForProfile(profile browserprofile.Profile) string { } } +// applyProfileLanguageHint overrides the profile's locale-derived fields when +// the requested language differs from the cached profile's. A bare language +// hint that already matches the profile language is treated as a no-op so that +// an explicit profile region (e.g. en-GB) isn't clobbered by a default (en-US). +func applyProfileLanguageHint(profile browserprofile.Profile, langCode string) browserprofile.Profile { + hint := ParseLocale(langCode) + if hint.Language == "" { + return profile + } + + current := ParseLocale(profile.Locale) + if current.Language == hint.Language && (hint.Country == "" || current.Country == hint.Country) { + return profile + } + + primary := PrimaryLanguageTag(langCode) + if primary == "" { + return profile + } + + profile.AcceptLanguage = BuildAcceptLanguageHeader(langCode) + profile.NavigatorLangs = []string{primary} + profile.Locale = primary + return profile +} + func applyProfile(page *rod.Page, profile browserprofile.Profile) error { if page == nil { return fmt.Errorf("page is nil") diff --git a/core/browser_unit_test.go b/core/browser_unit_test.go index 0a7db9d..559d7e2 100644 --- a/core/browser_unit_test.go +++ b/core/browser_unit_test.go @@ -4,6 +4,8 @@ import ( "os" "path/filepath" "testing" + + browserprofile "github.com/karust/openserp/core/browser" ) func TestResolveBrowserBinaryPathPrefersExplicit(t *testing.T) { @@ -57,3 +59,58 @@ func TestResolveBrowserBinaryPathRejectsInvalidExplicit(t *testing.T) { t.Fatalf("expected error when explicit browser_path points to a directory") } } + +func TestApplyProfileLanguageHint(t *testing.T) { + base := browserprofile.Profile{ + AcceptLanguage: "en-US,en;q=0.9", + NavigatorLangs: []string{"en-US"}, + Locale: "en-US", + } + + tests := []struct { + name string + lang string + wantAL string + wantL string + }{ + { + name: "empty hint keeps profile", + lang: "", + wantAL: "en-US,en;q=0.9", + wantL: "en-US", + }, + { + name: "same language without region keeps profile", + lang: "en", + wantAL: "en-US,en;q=0.9", + wantL: "en-US", + }, + { + name: "new language overrides locale headers", + lang: "de", + wantAL: "de-DE,de;q=0.9", + wantL: "de-DE", + }, + { + name: "explicit region overrides locale headers", + lang: "en-GB", + wantAL: "en-GB,en;q=0.9", + wantL: "en-GB", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := applyProfileLanguageHint(base, tt.lang) + if got.AcceptLanguage != tt.wantAL { + t.Fatalf("AcceptLanguage = %q, want %q", got.AcceptLanguage, tt.wantAL) + } + if got.Locale != tt.wantL { + t.Fatalf("Locale = %q, want %q", got.Locale, tt.wantL) + } + if len(got.NavigatorLangs) != 1 || got.NavigatorLangs[0] != tt.wantL { + t.Fatalf("NavigatorLangs = %v, want [%q]", got.NavigatorLangs, tt.wantL) + } + }) + } +} diff --git a/core/http_client.go b/core/http_client.go index 9f25f5c..83dcc85 100644 --- a/core/http_client.go +++ b/core/http_client.go @@ -15,6 +15,17 @@ import ( const rawHTTPTimeout = 10 * time.Second +// SetAcceptLanguageHeader sets the Accept-Language header from a lang code. +// No-op when the code has no language subtag. +func SetAcceptLanguageHeader(req *http.Request, langCode string) { + if req == nil { + return + } + if value := BuildAcceptLanguageHeader(langCode); value != "" { + req.Header.Set("Accept-Language", value) + } +} + // DrainAndCloseResponse drains unread bytes before closing so HTTP transports // can safely reuse connections when callers don't consume the full body. func DrainAndCloseResponse(resp *http.Response) { diff --git a/core/locale.go b/core/locale.go index d7d7ab9..216ea5a 100644 --- a/core/locale.go +++ b/core/locale.go @@ -11,6 +11,23 @@ type Locale struct { Country string } +var defaultLocaleCountryByLanguage = map[string]string{ + "en": "US", + "de": "DE", + "ru": "RU", + "fr": "FR", + "es": "ES", + "it": "IT", + "pt": "BR", + "zh": "CN", + "ja": "JP", + "ko": "KR", + "nl": "NL", + "pl": "PL", + "tr": "TR", + "ar": "SA", +} + // ParseLocale parses a language code such as "en", "EN-us", or "de_AT" into a // Locale. Returns the zero value when the input is empty or has no language // subtag. Country is uppercased; Language is lowercased. @@ -33,3 +50,35 @@ func ParseLocale(code string) Locale { } return Locale{Language: language, Country: country} } + +// PrimaryLanguageTag returns the BCP47 primary tag for a lang code, filling in +// a default country for bare languages (e.g. "de" -> "de-DE"). Returns "" when +// the input has no language subtag. +func PrimaryLanguageTag(langCode string) string { + locale := ParseLocale(langCode) + if locale.Language == "" { + return "" + } + country := locale.Country + if country == "" { + country = defaultLocaleCountryByLanguage[locale.Language] + } + if country == "" { + return locale.Language + } + return locale.Language + "-" + country +} + +// BuildAcceptLanguageHeader formats an Accept-Language value from a lang code. +// Example: "de" -> "de-DE,de;q=0.9", "en-GB" -> "en-GB,en;q=0.9", "sw" -> "sw". +func BuildAcceptLanguageHeader(langCode string) string { + primary := PrimaryLanguageTag(langCode) + if primary == "" { + return "" + } + language := ParseLocale(langCode).Language + if primary == language { + return language + } + return primary + "," + language + ";q=0.9" +} diff --git a/core/locale_test.go b/core/locale_test.go index 388ba30..d167d37 100644 --- a/core/locale_test.go +++ b/core/locale_test.go @@ -31,3 +31,27 @@ func TestParseLocale(t *testing.T) { }) } } + +func TestBuildAcceptLanguageHeader(t *testing.T) { + tests := []struct { + name string + in string + want string + }{ + {name: "empty", in: "", want: ""}, + {name: "language only with default country", in: "de", want: "de-DE,de;q=0.9"}, + {name: "language only with mapped country", in: "pt", want: "pt-BR,pt;q=0.9"}, + {name: "explicit region", in: "en-GB", want: "en-GB,en;q=0.9"}, + {name: "unknown language emits bare tag", in: "sw", want: "sw"}, + {name: "invalid locale", in: "-US", want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := BuildAcceptLanguageHeader(tt.in) + if got != tt.want { + t.Fatalf("BuildAcceptLanguageHeader(%q) = %q, want %q", tt.in, got, tt.want) + } + }) + } +} diff --git a/google/search_raw.go b/google/search_raw.go index e114579..39cacaa 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -23,6 +23,7 @@ func googleRequest(ctx context.Context, searchURL string, query core.Query) (*ht return nil, err } req.Header.Set("User-Agent", uarand.GetRandom()) + core.SetAcceptLanguageHeader(req, query.LangCode) res, err := baseClient.Do(req) if err != nil { diff --git a/yandex/search_raw.go b/yandex/search_raw.go index f4bc2c4..d63c5de 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -23,6 +23,7 @@ func yandexRequest(ctx context.Context, searchURL string, query core.Query) (*ht return nil, err } req.Header.Set("User-Agent", uarand.GetRandom()) + core.SetAcceptLanguageHeader(req, query.LangCode) res, err := baseClient.Do(req) if err != nil {