From 2b10aca4ac2085fe33b769d9cb7b65ffd1906bfd Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Mon, 27 Apr 2026 04:07:28 +0300 Subject: [PATCH] feat: enhance query handling with locale support and date filtering for Bing, DuckDuckGo, Google, and Yandex --- bing/bing_test.go | 77 +++++++++++++++++- bing/url.go | 166 +++++++++++++++++++++++++++++++------- cmd/root.go | 2 +- core/locale.go | 35 ++++++++ core/locale_test.go | 33 ++++++++ duckduckgo/search_test.go | 46 ++++++++++- duckduckgo/url.go | 74 +++++++++++++++-- google/search_test.go | 33 +++++++- google/url.go | 77 ++++++++++++++++-- yandex/url.go | 6 +- yandex/url_test.go | 47 +++++++++++ 11 files changed, 544 insertions(+), 52 deletions(-) create mode 100644 core/locale.go create mode 100644 core/locale_test.go create mode 100644 yandex/url_test.go diff --git a/bing/bing_test.go b/bing/bing_test.go index ba4eb37..84cadaa 100644 --- a/bing/bing_test.go +++ b/bing/bing_test.go @@ -30,7 +30,7 @@ func TestBuildURL(t *testing.T) { if host != "www.bing.com" { t.Fatalf("unexpected host: %s", host) } - if got := params.Get("q"); got != "golang тест site:example.com filetype:pdf after:2024-01-01 before:2024-01-31" { + if got := params.Get("q"); got != "golang тест site:example.com filetype:pdf" { t.Fatalf("unexpected q value: %q", got) } if got := params.Get("pq"); got != params.Get("q") { @@ -39,6 +39,15 @@ func TestBuildURL(t *testing.T) { if got := params.Get("setlang"); got != "ru" { t.Fatalf("unexpected setlang value: %q", got) } + if got := params.Get("mkt"); got != "ru-RU" { + t.Fatalf("unexpected mkt value: %q", got) + } + if got := params.Get("cc"); got != "RU" { + t.Fatalf("unexpected cc value: %q", got) + } + if got := params.Get("filters"); got != `ex1:"ez5_19723_19753"` { + t.Fatalf("unexpected filters value: %q", got) + } if got := params.Get("count"); got != "30" { t.Fatalf("unexpected count value: %q", got) } @@ -56,6 +65,58 @@ func TestBuildURL(t *testing.T) { } }, }, + { + name: "date operators in text are converted to filters", + query: core.Query{ + Text: "megadeth tickets after:2026-01-01 before:2026-04-27", + Limit: 10, + }, + check: func(t *testing.T, params url.Values, _ string) { + t.Helper() + if got := params.Get("q"); got != "megadeth tickets" { + t.Fatalf("unexpected q value: %q", got) + } + if got := params.Get("pq"); got != "megadeth tickets" { + t.Fatalf("unexpected pq value: %q", got) + } + if got := params.Get("filters"); got != `ex1:"ez5_20454_20570"` { + t.Fatalf("unexpected filters value: %q", got) + } + // LangCode unset → no locale params; let Bing pick defaults + // from the request rather than biasing toward en-US. + for _, key := range []string{"mkt", "setlang", "cc"} { + if got := params.Get(key); got != "" { + t.Fatalf("expected %s to be empty, got %q", key, got) + } + } + }, + }, + { + name: "date param overrides date operators in text", + query: core.Query{ + Text: "megadeth tickets after:2026-01-01 before:2026-04-27", + DateInterval: "20240101..20240131", + LangCode: "en-DE", + }, + check: func(t *testing.T, params url.Values, _ string) { + t.Helper() + if got := params.Get("q"); got != "megadeth tickets" { + t.Fatalf("unexpected q value: %q", got) + } + if got := params.Get("filters"); got != `ex1:"ez5_19723_19753"` { + t.Fatalf("unexpected filters value: %q", got) + } + if got := params.Get("mkt"); got != "en-DE" { + t.Fatalf("unexpected mkt value: %q", got) + } + if got := params.Get("setlang"); got != "en" { + t.Fatalf("unexpected setlang value: %q", got) + } + if got := params.Get("cc"); got != "DE" { + t.Fatalf("unexpected cc value: %q", got) + } + }, + }, { name: "very large start", query: core.Query{ @@ -81,6 +142,14 @@ func TestBuildURL(t *testing.T) { }, wantErr: true, }, + { + name: "reversed date interval returns error", + query: core.Query{ + Text: "golang", + DateInterval: "20240131..20240101", + }, + wantErr: true, + }, { name: "empty fields return error", query: core.Query{}, @@ -145,6 +214,12 @@ func TestBuildImageURL(t *testing.T) { if got := params.Get("setlang"); got != "en" { t.Fatalf("unexpected setlang value: %q", got) } + if got := params.Get("mkt"); got != "en-US" { + t.Fatalf("unexpected mkt value: %q", got) + } + if got := params.Get("cc"); got != "US" { + t.Fatalf("unexpected cc value: %q", got) + } if got := params.Get("form"); got != "HDRSC2" { t.Fatalf("unexpected form value: %q", got) } diff --git a/bing/url.go b/bing/url.go index e224d84..d6b91c0 100644 --- a/bing/url.go +++ b/bing/url.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "net/url" + "regexp" "strconv" "strings" "time" @@ -12,6 +13,29 @@ import ( "github.com/sirupsen/logrus" ) +var bingDateOperatorRE = regexp.MustCompile(`(?i)\b(after|before):(\d{4}-\d{2}-\d{2})\b`) + +// defaultBingCountryByLanguage maps a language subtag to the country Bing +// pairs with it for the "mkt" parameter when the caller did not specify one. +// Languages outside this map fall back to "US" rather than echoing the +// language code, since Bing rejects unknown markets like "ja-JA". +var defaultBingCountryByLanguage = map[string]string{ + "en": "US", + "de": "DE", + "ru": "RU", + "fr": "FR", + "es": "ES", + "it": "IT", + "pt": "BR", + "zh": "CN", + "ja": "JP", + "ko": "KR", + "nl": "NL", + "pl": "PL", + "tr": "TR", + "ar": "SA", +} + // BuildURL builds a Bing web search URL from Query fields. // It returns an error when query text or date parameters are invalid. func BuildURL(q core.Query) (string, error) { @@ -25,7 +49,13 @@ func BuildURL(q core.Query) (string, error) { // Set search query text with operators if q.Text != "" || q.Site != "" || q.Filetype != "" { - text := q.Text + text, textDateInterval, err := normalizeBingQueryText(q.Text) + if err != nil { + return "", err + } + if q.DateInterval == "" { + q.DateInterval = textDateInterval + } if q.Site != "" { text += " site:" + q.Site } @@ -41,8 +71,10 @@ func BuildURL(q core.Query) (string, error) { return "", errors.New("empty query built") } - if q.LangCode != "" { - params.Add("setlang", strings.ToLower(q.LangCode)) + if locale, ok := bingLocale(q.LangCode); ok { + params.Add("mkt", locale.market) + params.Add("setlang", locale.language) + params.Add("cc", locale.country) } // Set result offset (pagination) - Bing uses "first" parameter. @@ -57,32 +89,12 @@ func BuildURL(q core.Query) (string, error) { params.Add("count", strconv.Itoa(q.Limit)) } - // Set search date range - Bing supports date filtering via query text if q.DateInterval != "" { - intervals := strings.Split(q.DateInterval, "..") - if len(intervals) != 2 { - return "", errors.New("incorrect date interval provided, expected format: YYYYMMDD..YYYYMMDD") - } - - // Convert YYYYMMDD to YYYY-MM-DD format for Bing - startDate, err := time.Parse("20060102", intervals[0]) + filter, err := buildBingDateFilter(q.DateInterval) if err != nil { - return "", errors.New("invalid start date format, expected YYYYMMDD") + return "", err } - - endDate, err := time.Parse("20060102", intervals[1]) - if err != nil { - return "", errors.New("invalid end date format, expected YYYYMMDD") - } - - // Add date range to the search query text (Bing supports this format) - dateRange := fmt.Sprintf(" after:%s before:%s", - startDate.Format("2006-01-02"), - endDate.Format("2006-01-02")) - - // Update the query text to include date range - currentQuery := params.Get("q") - params.Set("q", currentQuery+dateRange) + params.Add("filters", filter) } // Bing-specific parameters for consistent results @@ -95,6 +107,101 @@ func BuildURL(q core.Query) (string, error) { return base.String(), nil } +type bingLocaleParams struct { + language string + country string + market string +} + +// bingLocale resolves a Bing market triplet (language, country, mkt) from a +// caller-supplied language code. It returns ok=false when the input is empty +// so callers can omit Bing's locale parameters entirely instead of forcing a +// default market that biases results toward en-US. +func bingLocale(langCode string) (bingLocaleParams, bool) { + parsed := core.ParseLocale(langCode) + if parsed.Language == "" { + return bingLocaleParams{}, false + } + + country := parsed.Country + if country == "" { + country = defaultBingCountry(parsed.Language) + } + return bingLocaleParams{ + language: parsed.Language, + country: country, + market: parsed.Language + "-" + country, + }, true +} + +func defaultBingCountry(language string) string { + if country, ok := defaultBingCountryByLanguage[language]; ok { + return country + } + return "US" +} + +func normalizeBingQueryText(text string) (string, string, error) { + matches := bingDateOperatorRE.FindAllStringSubmatch(text, -1) + if len(matches) == 0 { + return text, "", nil + } + + var after, before string + for _, match := range matches { + if len(match) != 3 { + continue + } + switch strings.ToLower(match[1]) { + case "after": + after = strings.ReplaceAll(match[2], "-", "") + case "before": + before = strings.ReplaceAll(match[2], "-", "") + } + } + + cleaned := bingDateOperatorRE.ReplaceAllString(text, "") + cleaned = strings.Join(strings.Fields(cleaned), " ") + + if after == "" && before == "" { + return cleaned, "", nil + } + if after == "" || before == "" { + return cleaned, "", nil + } + if _, err := buildBingDateFilter(after + ".." + before); err != nil { + return "", "", err + } + return cleaned, after + ".." + before, nil +} + +func buildBingDateFilter(dateInterval string) (string, error) { + intervals := strings.Split(dateInterval, "..") + if len(intervals) != 2 { + return "", errors.New("incorrect date interval provided, expected format: YYYYMMDD..YYYYMMDD") + } + + startDate, err := time.Parse("20060102", intervals[0]) + if err != nil { + return "", errors.New("invalid start date format, expected YYYYMMDD") + } + + endDate, err := time.Parse("20060102", intervals[1]) + if err != nil { + return "", errors.New("invalid end date format, expected YYYYMMDD") + } + + if startDate.After(endDate) { + return "", errors.New("start date must not be after end date") + } + + const secondsPerDay = int64(24 * 60 * 60) + startDay := startDate.Unix() / secondsPerDay + endDay := endDate.Unix() / secondsPerDay + + return fmt.Sprintf(`ex1:"ez5_%d_%d"`, startDay, endDay), nil +} + // BuildImageURL builds a Bing image search URL from Query fields. // It returns an error when the resulting query text is empty. func BuildImageURL(q core.Query) (string, error) { @@ -118,9 +225,10 @@ func BuildImageURL(q core.Query) (string, error) { return "", errors.New("empty query built") } - // Add common parameters - if q.LangCode != "" { - params.Add("setlang", strings.ToLower(q.LangCode)) + if locale, ok := bingLocale(q.LangCode); ok { + params.Add("mkt", locale.market) + params.Add("setlang", locale.language) + params.Add("cc", locale.country) } // Image-specific parameters diff --git a/cmd/root.go b/cmd/root.go index 8b7f30d..666be7f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,7 +15,7 @@ import ( ) const ( - version = "0.7.0" + version = "0.7.1" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/locale.go b/core/locale.go new file mode 100644 index 0000000..d7d7ab9 --- /dev/null +++ b/core/locale.go @@ -0,0 +1,35 @@ +package core + +import "strings" + +// Locale is a parsed language/region pair derived from a BCP47-style code. +// Language is the lowercase 2-letter language subtag (e.g. "en", "de"). +// Country is the uppercase 2-letter region subtag (e.g. "US", "DE"); it may be +// empty when the input had no region and the caller did not request a default. +type Locale struct { + Language string + Country string +} + +// ParseLocale parses a language code such as "en", "EN-us", or "de_AT" into a +// Locale. Returns the zero value when the input is empty or has no language +// subtag. Country is uppercased; Language is lowercased. +func ParseLocale(code string) Locale { + code = strings.TrimSpace(code) + if code == "" { + return Locale{} + } + code = strings.ReplaceAll(code, "_", "-") + + parts := strings.Split(code, "-") + language := strings.ToLower(strings.TrimSpace(parts[0])) + if language == "" { + return Locale{} + } + + country := "" + if len(parts) > 1 { + country = strings.ToUpper(strings.TrimSpace(parts[1])) + } + return Locale{Language: language, Country: country} +} diff --git a/core/locale_test.go b/core/locale_test.go new file mode 100644 index 0000000..388ba30 --- /dev/null +++ b/core/locale_test.go @@ -0,0 +1,33 @@ +package core + +import "testing" + +func TestParseLocale(t *testing.T) { + tests := []struct { + name string + in string + wantLang string + wantCC string + }{ + {"empty", "", "", ""}, + {"whitespace only", " ", "", ""}, + {"language only", "EN", "en", ""}, + {"language with region dash", "en-US", "en", "US"}, + {"language with region underscore", "de_AT", "de", "AT"}, + {"mixed casing", "Pt-bR", "pt", "BR"}, + {"trailing whitespace", " fr-CA ", "fr", "CA"}, + {"empty region after dash", "ru-", "ru", ""}, + {"language only after split", "-US", "", ""}, + {"extra subtags ignored", "en-US-x-private", "en", "US"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ParseLocale(tt.in) + if got.Language != tt.wantLang || got.Country != tt.wantCC { + t.Fatalf("ParseLocale(%q) = {%q, %q}, want {%q, %q}", + tt.in, got.Language, got.Country, tt.wantLang, tt.wantCC) + } + }) + } +} diff --git a/duckduckgo/search_test.go b/duckduckgo/search_test.go index 75ca74f..40cbc65 100644 --- a/duckduckgo/search_test.go +++ b/duckduckgo/search_test.go @@ -44,7 +44,7 @@ func TestBuildURL(t *testing.T) { Text: "поиск", Site: "github.com", Filetype: "pdf", - LangCode: "RU-ru", + LangCode: "RU", DateInterval: "20240101..20240131", }, page: 0, @@ -151,7 +151,7 @@ func TestBuildImageURL(t *testing.T) { if got := params.Get("df"); got != "2024-02-01..2024-02-29" { t.Fatalf("unexpected df value: %q", got) } - if got := params.Get("kl"); got != "ru" { + if got := params.Get("kl"); got != "ru-ru" { t.Fatalf("unexpected kl value: %q", got) } }, @@ -191,3 +191,45 @@ func TestBuildImageURL(t *testing.T) { }) } } + +func TestDuckDuckGoLanguageMapping(t *testing.T) { + tests := []struct { + name string + langCode string + wantKL string + }{ + { + name: "language only maps to default region", + langCode: "DE", + wantKL: "de-de", + }, + { + name: "regional language maps to duckduckgo region", + langCode: "de-AT", + wantKL: "at-de", + }, + { + name: "unknown language omits kl", + langCode: "xx", + wantKL: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := BuildURL(core.Query{Text: "golang", LangCode: tt.langCode}, 0) + if err != nil { + t.Fatalf("BuildURL() error = %v", err) + } + + parsed, err := url.Parse(got) + if err != nil { + t.Fatalf("BuildURL() returned invalid URL: %v", err) + } + + if gotKL := parsed.Query().Get("kl"); gotKL != tt.wantKL { + t.Fatalf("unexpected kl value: %q", gotKL) + } + }) + } +} diff --git a/duckduckgo/url.go b/duckduckgo/url.go index e78d6e3..76cc7d2 100644 --- a/duckduckgo/url.go +++ b/duckduckgo/url.go @@ -12,6 +12,69 @@ import ( const baseURL = "https://duckduckgo.com" +// ddgKLByLocale maps lowercase BCP47 codes ("en", "en-gb", "zh-tw") to +// DuckDuckGo's "kl" parameter, which uses an inverted region-language form +// (e.g. "uk-en"). Lookup is locale-first, then language-only as a fallback. +var ddgKLByLocale = map[string]string{ + "en": "us-en", + "en-us": "us-en", + "en-gb": "uk-en", + "en-au": "au-en", + "en-ca": "ca-en", + "de": "de-de", + "de-at": "at-de", + "de-ch": "ch-de", + "fr": "fr-fr", + "fr-ca": "ca-fr", + "fr-be": "be-fr", + "fr-ch": "ch-fr", + "es": "es-es", + "es-mx": "mx-es", + "es-ar": "ar-es", + "it": "it-it", + "nl": "nl-nl", + "nl-be": "be-nl", + "pt": "pt-pt", + "pt-br": "br-pt", + "ru": "ru-ru", + "pl": "pl-pl", + "cs": "cz-cs", + "sk": "sk-sk", + "hu": "hu-hu", + "ro": "ro-ro", + "da": "dk-da", + "sv": "se-sv", + "no": "no-no", + "fi": "fi-fi", + "tr": "tr-tr", + "el": "gr-el", + "he": "il-he", + "ar": "xa-ar", + "zh": "cn-zh", + "zh-cn": "cn-zh", + "zh-tw": "tw-zh", + "ja": "jp-ja", + "ko": "kr-ko", +} + +// duckDuckGoKL resolves a DuckDuckGo "kl" value for the supplied language code. +// Returns "" when the input has no known mapping so callers can omit the +// parameter rather than send an unrecognized region. +func duckDuckGoKL(langCode string) string { + locale := core.ParseLocale(langCode) + if locale.Language == "" { + return "" + } + + if locale.Country != "" { + key := locale.Language + "-" + strings.ToLower(locale.Country) + if kl, ok := ddgKLByLocale[key]; ok { + return kl + } + } + return ddgKLByLocale[locale.Language] +} + // BuildURL builds a DuckDuckGo web search URL for the provided query and page // index. It returns an error when query text or date parameters are invalid. func BuildURL(q core.Query, page int) (string, error) { @@ -65,9 +128,8 @@ func BuildURL(q core.Query, page int) (string, error) { params.Add("df", dateRange) } - // Set language - if q.LangCode != "" { - params.Add("kl", strings.ToLower(q.LangCode)) + if kl := duckDuckGoKL(q.LangCode); kl != "" { + params.Add("kl", kl) } // DuckDuckGo specific parameters @@ -142,11 +204,9 @@ func BuildImageURL(q core.Query) (string, error) { params.Add("df", dateRange) } - // Set language - if q.LangCode != "" { - params.Add("kl", strings.ToLower(q.LangCode)) + if kl := duckDuckGoKL(q.LangCode); kl != "" { + params.Add("kl", kl) } - base.RawQuery = params.Encode() return base.String(), nil } diff --git a/google/search_test.go b/google/search_test.go index 6c06264..9d420be 100644 --- a/google/search_test.go +++ b/google/search_test.go @@ -49,9 +49,12 @@ func TestBuildSearchURL(t *testing.T) { if got := params.Get("filter"); got != "0" { t.Fatalf("unexpected filter value: %q", got) } - if got := params.Get("hl"); got != "RU" { + if got := params.Get("hl"); got != "ru" { t.Fatalf("unexpected hl value: %q", got) } + if got := params.Get("gl"); got != "ru" { + t.Fatalf("unexpected gl value: %q", got) + } if got := params.Get("lr"); got != "lang_ru" { t.Fatalf("unexpected lr value: %q", got) } @@ -63,6 +66,29 @@ func TestBuildSearchURL(t *testing.T) { } }, }, + { + name: "german language sets matching google locale params", + query: core.Query{ + Text: "megadeth tickets", + Filter: true, + LangCode: "DE", + }, + check: func(t *testing.T, params url.Values, host string) { + t.Helper() + if host != "www.google.de" { + t.Fatalf("unexpected host: %s", host) + } + if got := params.Get("hl"); got != "de" { + t.Fatalf("unexpected hl value: %q", got) + } + if got := params.Get("gl"); got != "de" { + t.Fatalf("unexpected gl value: %q", got) + } + if got := params.Get("lr"); got != "lang_de" { + t.Fatalf("unexpected lr value: %q", got) + } + }, + }, { name: "site and filetype without text", query: core.Query{ @@ -167,9 +193,12 @@ func TestBuildImageSearchURL(t *testing.T) { if got := params.Get("tbs"); got != "cdr:1,cd_min:20240301,cd_max:20240315" { t.Fatalf("unexpected tbs value: %q", got) } - if got := params.Get("hl"); got != "EN" { + if got := params.Get("hl"); got != "en" { t.Fatalf("unexpected hl value: %q", got) } + if got := params.Get("gl"); got != "us" { + t.Fatalf("unexpected gl value: %q", got) + } if got := params.Get("lr"); got != "lang_en" { t.Fatalf("unexpected lr value: %q", got) } diff --git a/google/url.go b/google/url.go index a4c2a39..6da9a87 100644 --- a/google/url.go +++ b/google/url.go @@ -215,10 +215,21 @@ var GoogleDomains = map[string]string{ "zw": "co.zw", } +// googleDefaultCountryByLanguage maps a language subtag to the country Google +// associates with it when the user did not provide an explicit region. +var googleDefaultCountryByLanguage = map[string]string{ + "en": "us", + "pt": "br", + "zh": "cn", + "ja": "jp", + "ko": "kr", +} + // BuildURL builds a Google web search URL from Query fields. // It returns an error when the resulting query text is empty or invalid. func BuildURL(q core.Query) (string, error) { - googleBase := GoogleDomains[strings.ToLower(q.LangCode)] + locale := googleLocale(q.LangCode) + googleBase := googleDomain(locale) base, err := url.Parse(fmt.Sprintf("https://www.google.%s", googleBase)) if err != nil { return "", err @@ -275,9 +286,10 @@ func BuildURL(q core.Query) (string, error) { params.Add("filter", "0") } - if q.LangCode != "" { - params.Add("hl", q.LangCode) - params.Add("lr", "lang_"+strings.ToLower(q.LangCode)) + if locale.language != "" { + params.Add("hl", locale.language) + params.Add("gl", locale.country) + params.Add("lr", "lang_"+locale.language) } params.Add("pws", "0") // Do not personalize earch results @@ -293,7 +305,8 @@ func BuildURL(q core.Query) (string, error) { // It returns an error when the resulting query text is empty or invalid. func BuildImageURL(q core.Query) (string, error) { // TODO: Add new params - googleBase := GoogleDomains[strings.ToLower(q.LangCode)] + locale := googleLocale(q.LangCode) + googleBase := googleDomain(locale) base, err := url.Parse(fmt.Sprintf("https://www.google.%s", googleBase)) if err != nil { return "", err @@ -337,9 +350,10 @@ func BuildImageURL(q core.Query) (string, error) { params.Add("num", strconv.Itoa(q.Limit)) } - if q.LangCode != "" { - params.Add("hl", q.LangCode) - params.Add("lr", "lang_"+strings.ToLower(q.LangCode)) + if locale.language != "" { + params.Add("hl", locale.language) + params.Add("gl", locale.country) + params.Add("lr", "lang_"+locale.language) } params.Add("pws", "0") // Do not personalize earch results @@ -349,6 +363,53 @@ func BuildImageURL(q core.Query) (string, error) { return base.String(), nil } +type googleLocaleParams struct { + language string + country string +} + +// googleLocale parses langCode and fills in a default country when one is not +// supplied so callers always get a usable (gl, hl) pair. +func googleLocale(langCode string) googleLocaleParams { + parsed := core.ParseLocale(langCode) + if parsed.Language == "" { + return googleLocaleParams{} + } + + country := strings.ToLower(parsed.Country) + if country == "" { + country = defaultGoogleCountry(parsed.Language) + } + return googleLocaleParams{ + language: parsed.Language, + country: country, + } +} + +func defaultGoogleCountry(language string) string { + if country, ok := googleDefaultCountryByLanguage[language]; ok { + return country + } + // Many ISO 639-1 codes also exist as ccTLDs in GoogleDomains + // (e.g. "de", "fr", "ru"); fall back to that mapping when present. + if _, ok := GoogleDomains[language]; ok { + return language + } + return "us" +} + +// googleDomain picks a Google ccTLD for the resolved locale, preferring the +// country, then the language, then the global "com" default. +func googleDomain(locale googleLocaleParams) string { + if domain, ok := GoogleDomains[locale.country]; ok { + return domain + } + if domain, ok := GoogleDomains[locale.language]; ok { + return domain + } + return GoogleDomains[""] +} + // SourceImage contains parsed Google image metadata extracted from result links. type SourceImage struct { PageURL string diff --git a/yandex/url.go b/yandex/url.go index dea0028..0ecbb0f 100644 --- a/yandex/url.go +++ b/yandex/url.go @@ -28,8 +28,10 @@ func BuildURL(q core.Query, page int) (string, error) { if q.DateInterval != "" { text += " date:" + q.DateInterval } - if q.LangCode != "" { - text += " lang:" + q.LangCode + if locale := core.ParseLocale(q.LangCode); locale.Language != "" { + // Yandex's lang: operator accepts the lowercase language subtag + // only; region modifiers must go through the lr= parameter. + text += " lang:" + locale.Language } params.Add("text", text) diff --git a/yandex/url_test.go b/yandex/url_test.go new file mode 100644 index 0000000..037061f --- /dev/null +++ b/yandex/url_test.go @@ -0,0 +1,47 @@ +package yandex + +import ( + "net/url" + "strings" + "testing" + + "github.com/karust/openserp/core" +) + +func TestBuildURLLanguageOperator(t *testing.T) { + tests := []struct { + name string + langCode string + wantOp string // expected substring in text query, "" means no lang: operator + }{ + {"empty lang adds no operator", "", ""}, + {"uppercase language is normalized", "RU", "lang:ru"}, + {"region is dropped from lang operator", "en-US", "lang:en"}, + {"underscore form is accepted", "DE_at", "lang:de"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := BuildURL(core.Query{Text: "golang", LangCode: tt.langCode}, 0) + if err != nil { + t.Fatalf("BuildURL() error = %v", err) + } + + parsed, err := url.Parse(got) + if err != nil { + t.Fatalf("BuildURL() returned invalid URL: %v", err) + } + + text := parsed.Query().Get("text") + if tt.wantOp == "" { + if strings.Contains(text, "lang:") { + t.Fatalf("expected no lang: operator, got text=%q", text) + } + return + } + if !strings.Contains(text, tt.wantOp) { + t.Fatalf("expected text to contain %q, got %q", tt.wantOp, text) + } + }) + } +}