feat: propagate query locale to accept-language header and browser profile

baidu/search_raw.go

@@ -23,6 +23,7 @@ func baiduRequest(ctx context.Context, searchURL string, query core.Query) (*htt
 		return nil, err
 	}
 	req.Header.Set("User-Agent", uarand.GetRandom())
+	core.SetAcceptLanguageHeader(req, query.LangCode)
 
 	res, err := baseClient.Do(req)
 	if err != nil {

cmd/root.go

@@ -15,7 +15,7 @@ import (
 )
 
 const (
-	version               = "0.7.1"
+	version               = "0.7.2"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )

core/browser.go

@@ -486,6 +486,7 @@ func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browse
 	// serialize all concurrent Navigate calls.
 	profile := browserprofile.SelectProfile(engine, region)
 	profile = applyRuntimeBrowserVersion(profile, browser)
+	profile = applyProfileLanguageHint(profile, region)
 	if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" {
 		profile.UserAgent = overrideUA
 	}
@@ -595,6 +596,32 @@ func navigatorPlatformForProfile(profile browserprofile.Profile) string {
 	}
 }
 
+// applyProfileLanguageHint overrides the profile's locale-derived fields when
+// the requested language differs from the cached profile's. A bare language
+// hint that already matches the profile language is treated as a no-op so that
+// an explicit profile region (e.g. en-GB) isn't clobbered by a default (en-US).
+func applyProfileLanguageHint(profile browserprofile.Profile, langCode string) browserprofile.Profile {
+	hint := ParseLocale(langCode)
+	if hint.Language == "" {
+		return profile
+	}
+
+	current := ParseLocale(profile.Locale)
+	if current.Language == hint.Language && (hint.Country == "" || current.Country == hint.Country) {
+		return profile
+	}
+
+	primary := PrimaryLanguageTag(langCode)
+	if primary == "" {
+		return profile
+	}
+
+	profile.AcceptLanguage = BuildAcceptLanguageHeader(langCode)
+	profile.NavigatorLangs = []string{primary}
+	profile.Locale = primary
+	return profile
+}
+
 func applyProfile(page *rod.Page, profile browserprofile.Profile) error {
 	if page == nil {
 		return fmt.Errorf("page is nil")

core/browser_unit_test.go

@@ -4,6 +4,8 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+
+	browserprofile "github.com/karust/openserp/core/browser"
 )
 
 func TestResolveBrowserBinaryPathPrefersExplicit(t *testing.T) {
@@ -57,3 +59,58 @@ func TestResolveBrowserBinaryPathRejectsInvalidExplicit(t *testing.T) {
 		t.Fatalf("expected error when explicit browser_path points to a directory")
 	}
 }
+
+func TestApplyProfileLanguageHint(t *testing.T) {
+	base := browserprofile.Profile{
+		AcceptLanguage: "en-US,en;q=0.9",
+		NavigatorLangs: []string{"en-US"},
+		Locale:         "en-US",
+	}
+
+	tests := []struct {
+		name   string
+		lang   string
+		wantAL string
+		wantL  string
+	}{
+		{
+			name:   "empty hint keeps profile",
+			lang:   "",
+			wantAL: "en-US,en;q=0.9",
+			wantL:  "en-US",
+		},
+		{
+			name:   "same language without region keeps profile",
+			lang:   "en",
+			wantAL: "en-US,en;q=0.9",
+			wantL:  "en-US",
+		},
+		{
+			name:   "new language overrides locale headers",
+			lang:   "de",
+			wantAL: "de-DE,de;q=0.9",
+			wantL:  "de-DE",
+		},
+		{
+			name:   "explicit region overrides locale headers",
+			lang:   "en-GB",
+			wantAL: "en-GB,en;q=0.9",
+			wantL:  "en-GB",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := applyProfileLanguageHint(base, tt.lang)
+			if got.AcceptLanguage != tt.wantAL {
+				t.Fatalf("AcceptLanguage = %q, want %q", got.AcceptLanguage, tt.wantAL)
+			}
+			if got.Locale != tt.wantL {
+				t.Fatalf("Locale = %q, want %q", got.Locale, tt.wantL)
+			}
+			if len(got.NavigatorLangs) != 1 || got.NavigatorLangs[0] != tt.wantL {
+				t.Fatalf("NavigatorLangs = %v, want [%q]", got.NavigatorLangs, tt.wantL)
+			}
+		})
+	}
+}

core/http_client.go

@@ -15,6 +15,17 @@ import (
 
 const rawHTTPTimeout = 10 * time.Second
 
+// SetAcceptLanguageHeader sets the Accept-Language header from a lang code.
+// No-op when the code has no language subtag.
+func SetAcceptLanguageHeader(req *http.Request, langCode string) {
+	if req == nil {
+		return
+	}
+	if value := BuildAcceptLanguageHeader(langCode); value != "" {
+		req.Header.Set("Accept-Language", value)
+	}
+}
+
 // DrainAndCloseResponse drains unread bytes before closing so HTTP transports
 // can safely reuse connections when callers don't consume the full body.
 func DrainAndCloseResponse(resp *http.Response) {

core/locale.go

@@ -11,6 +11,23 @@ type Locale struct {
 	Country  string
 }
 
+var defaultLocaleCountryByLanguage = map[string]string{
+	"en": "US",
+	"de": "DE",
+	"ru": "RU",
+	"fr": "FR",
+	"es": "ES",
+	"it": "IT",
+	"pt": "BR",
+	"zh": "CN",
+	"ja": "JP",
+	"ko": "KR",
+	"nl": "NL",
+	"pl": "PL",
+	"tr": "TR",
+	"ar": "SA",
+}
+
 // ParseLocale parses a language code such as "en", "EN-us", or "de_AT" into a
 // Locale. Returns the zero value when the input is empty or has no language
 // subtag. Country is uppercased; Language is lowercased.
@@ -33,3 +50,35 @@ func ParseLocale(code string) Locale {
 	}
 	return Locale{Language: language, Country: country}
 }
+
+// PrimaryLanguageTag returns the BCP47 primary tag for a lang code, filling in
+// a default country for bare languages (e.g. "de" -> "de-DE"). Returns "" when
+// the input has no language subtag.
+func PrimaryLanguageTag(langCode string) string {
+	locale := ParseLocale(langCode)
+	if locale.Language == "" {
+		return ""
+	}
+	country := locale.Country
+	if country == "" {
+		country = defaultLocaleCountryByLanguage[locale.Language]
+	}
+	if country == "" {
+		return locale.Language
+	}
+	return locale.Language + "-" + country
+}
+
+// BuildAcceptLanguageHeader formats an Accept-Language value from a lang code.
+// Example: "de" -> "de-DE,de;q=0.9", "en-GB" -> "en-GB,en;q=0.9", "sw" -> "sw".
+func BuildAcceptLanguageHeader(langCode string) string {
+	primary := PrimaryLanguageTag(langCode)
+	if primary == "" {
+		return ""
+	}
+	language := ParseLocale(langCode).Language
+	if primary == language {
+		return language
+	}
+	return primary + "," + language + ";q=0.9"
+}

core/locale_test.go

@@ -31,3 +31,27 @@ func TestParseLocale(t *testing.T) {
 		})
 	}
 }
+
+func TestBuildAcceptLanguageHeader(t *testing.T) {
+	tests := []struct {
+		name string
+		in   string
+		want string
+	}{
+		{name: "empty", in: "", want: ""},
+		{name: "language only with default country", in: "de", want: "de-DE,de;q=0.9"},
+		{name: "language only with mapped country", in: "pt", want: "pt-BR,pt;q=0.9"},
+		{name: "explicit region", in: "en-GB", want: "en-GB,en;q=0.9"},
+		{name: "unknown language emits bare tag", in: "sw", want: "sw"},
+		{name: "invalid locale", in: "-US", want: ""},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := BuildAcceptLanguageHeader(tt.in)
+			if got != tt.want {
+				t.Fatalf("BuildAcceptLanguageHeader(%q) = %q, want %q", tt.in, got, tt.want)
+			}
+		})
+	}
+}

google/search_raw.go

@@ -23,6 +23,7 @@ func googleRequest(ctx context.Context, searchURL string, query core.Query) (*ht
 		return nil, err
 	}
 	req.Header.Set("User-Agent", uarand.GetRandom())
+	core.SetAcceptLanguageHeader(req, query.LangCode)
 
 	res, err := baseClient.Do(req)
 	if err != nil {

yandex/search_raw.go

@@ -23,6 +23,7 @@ func yandexRequest(ctx context.Context, searchURL string, query core.Query) (*ht
 		return nil, err
 	}
 	req.Header.Set("User-Agent", uarand.GetRandom())
+	core.SetAcceptLanguageHeader(req, query.LangCode)
 
 	res, err := baseClient.Do(req)
 	if err != nil {