diff --git a/README.md b/README.md index 73864f6..2810d14 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,9 @@ Run it locally, self-host it, or use the optional hosted API when you do not wan **Official website:** [openserp.org](https://openserp.org) -**Feedback:** [GitHub Issues](https://github.com/karust/openserp/issues) ยท [Telegram](https://t.me/+RJEKspw3mUlhZDMy) ยท [feedback@openserp.org](mailto:feedback@openserp.org) +**Feedback:** [GitHub Issues](https://github.com/karust/openserp/issues) or [feedback@openserp.org](mailto:feedback@openserp.org) + +**Latest updates, usage examples**: [Telegram](https://t.me/+RJEKspw3mUlhZDMy) > ๐Ÿ’ก OpenSERP is free and open-source. Only links listed in this repository and on the official website are associated with the project. @@ -114,17 +116,17 @@ curl "http://127.0.0.1:7000/mega/engines" Common parameters: -| Parameter | Description | Example | -| --------- | ---------------------------------------------------------------------------- | ------------------------------------ | -| `text` | Search query | `golang programming` | -| `lang` | Language code | `EN`, `DE`, `RU`, `ES` | -| `region` | Market/location hint. Yandex accepts numeric `lr`; others use country hints. | `213`, `RU`, `en-US` | -| `date` | Date range | `20250101..20251231` | -| `file` | File extension | `pdf`, `doc`, `xls` | -| `site` | Site-specific search | `github.com` | -| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` | -| `start` | Pagination offset | `0`, `10`, `20` | -| `format` | Output format | `json`, `markdown`, `text`, `ndjson` | +| Parameter | Description | Example | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------ | +| `text` | Search query | `golang programming` | +| `lang` | Language code | `EN`, `DE`, `RU`, `ES` | +| `region` | Market/location hint. Countries/locales work across engines; Google also accepts city names via `uule`; Yandex accepts numeric `lr`. | `DE`, `en-GB`, `Berlin`, `213` | +| `date` | Date range | `20250101..20251231` | +| `file` | File extension | `pdf`, `doc`, `xls` | +| `site` | Site-specific search | `github.com` | +| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` | +| `start` | Pagination offset | `0`, `10`, `20` | +| `format` | Output format | `json`, `markdown`, `text`, `ndjson` | Engine-specific parameters: @@ -298,4 +300,4 @@ Contributions are welcome. See [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md). - [Telegram channel](https://t.me/openserp_cloud) - OpenSERP news, release notes, and project updates. Direct messages are open for quick feedback and hosted API questions. - [feedback@openserp.org](mailto:feedback@openserp.org) - private notes, longer feedback, or anything that does not fit GitHub Issues. -###### _"OpenSERP" is the name of this open-source project. The official website is [openserp.org](https://openserp.org). Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._ +###### _"OpenSERP" is the name of this open-source project. The official website is [openserp.org](https://openserp.org). Resources not linked on this page are not affiliated with the project._ diff --git a/cmd/root.go b/cmd/root.go index b1572cd..2d10f11 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ import ( ) const ( - version = "0.7.14" + version = "0.7.15" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/region.go b/core/region.go new file mode 100644 index 0000000..8f5e2ed --- /dev/null +++ b/core/region.go @@ -0,0 +1,174 @@ +package core + +import "strings" + +// RegionTarget is the resolved, per-engine targeting for a user-supplied region +// hint. Engines read the field relevant to them: Google uses GoogleCanonical to +// build a UULE, Yandex uses YandexLR. Country is the ISO 3166-1 alpha-2 code +// when one could be derived, useful as a coarse market signal. +// +// A field left empty means "no better signal than the raw input" โ€” callers +// should fall back to their previous behavior (e.g. gl= from locale, or +// dropping the parameter entirely). Resolution never fails: an unrecognized +// region simply yields empty engine fields rather than an error. +type RegionTarget struct { + // Raw is the trimmed original input, preserved for engines that pass it + // through (e.g. Yandex numeric lr IDs). + Raw string + // Country is the ISO 3166-1 alpha-2 code (uppercase) when derivable, else "". + Country string + // GoogleCanonical is the exact Google geotargets canonical location name + // (e.g. "Berlin,Berlin,Germany") suitable for UULE v1 encoding, else "". + GoogleCanonical string + // YandexLR is the Yandex lr region ID (e.g. "213"), else "". + YandexLR string +} + +// yandexLRByCountry maps an ISO country code to a Yandex lr region ID. Yandex +// only exposes a limited set of country-level regions; cities require numeric +// lr IDs passed through verbatim. +var yandexLRByCountry = map[string]string{ + "AT": "113", "AU": "211", "BE": "114", "BR": "94", "CA": "95", + "CH": "126", "DE": "96", "DK": "203", "ES": "204", "FI": "123", + "FR": "124", "GB": "102", "IE": "10063", "IN": "994", "IT": "205", + "JP": "137", "KR": "135", "MX": "20271", "NL": "118", "NO": "119", + "PL": "120", "RU": "225", "SE": "127", "SG": "10105", "TR": "983", + "UA": "187", "UK": "102", "US": "84", "ZA": "10021", +} + +// cityCanonical maps a normalized bare city name to its exact Google geotargets +// canonical name (used to build a UULE). Only UULE-bearing city targeting needs +// a name table โ€” country/state targeting rides on gl= and never needs one. +// +// The list is deliberately small and hand-curated: bare city names are +// ambiguous (e.g. "London" exists in CA/GB/US), so we only auto-resolve a +// prominence list to its "obvious" match. Canonical names below are copied +// verbatim from Google's geotargets data; any other city can still be targeted +// by passing its full canonical name ("City,Region,Country") directly. +var cityCanonical = map[string]string{ + "amsterdam": "Amsterdam,North Holland,Netherlands", + "athens": "Athens,Athens,Attica,Greece", + "austin": "Austin,Texas,United States", + "bangalore": "Bengaluru,Karnataka,India", + "barcelona": "Barcelona,Barcelona,Catalonia,Spain", + "beijing": "Beijing,Beijing,China", + "berlin": "Berlin,Berlin,Germany", + "birmingham": "Birmingham,West Midlands,England,United Kingdom", + "boston": "Boston,Massachusetts,United States", + "brussels": "Brussels,Brussels,Belgium", + "buenos aires": "Buenos Aires,Buenos Aires,Argentina", + "cairo": "Cairo,Cairo Governorate,Egypt", + "chicago": "Chicago,Illinois,United States", + "copenhagen": "Copenhagen,Capital Region of Denmark,Denmark", + "dallas": "Dallas,Texas,United States", + "delhi": "Delhi,Delhi,India", + "dubai": "Dubai,Dubai,United Arab Emirates", + "dublin": "Dublin,County Dublin,Ireland", + "frankfurt": "Frankfurt am Main,Hessen,Germany", + "hamburg": "Hamburg,Hamburg,Germany", + "helsinki": "Helsinki,Helsinki,Uusimaa,Finland", + "hong kong": "Hong Kong,Hong Kong", + "istanbul": "Istanbul,Istanbul,Turkiye", + "johannesburg": "Johannesburg,Gauteng,South Africa", + "kyiv": "Kyiv,Kyiv city,Ukraine", + "lisbon": "Lisbon,Lisbon,Lisbon,Portugal", + "london": "London,England,United Kingdom", + "los angeles": "Los Angeles,California,United States", + "lyon": "Lyon,Auvergne-Rhone-Alpes,France", + "madrid": "Madrid,Community of Madrid,Spain", + "manchester": "Manchester,England,United Kingdom", + "marseille": "Marseille,Provence-Alpes-Cote d'Azur,France", + "melbourne": "Melbourne,Victoria,Australia", + "mexico city": "Mexico City,Mexico City,Mexico", + "miami": "Miami,Florida,United States", + "milan": "Milan,Milan,Lombardy,Italy", + "montreal": "Montreal,Montreal,Quebec,Canada", + "moscow": "Moscow,Moscow,Russia", + "mumbai": "Mumbai,Maharashtra,India", + "munich": "Munich,Bavaria,Germany", + "new york": "New York,New York,United States", + "osaka": "Osaka,Osaka,Japan", + "oslo": "Oslo,Oslo,Norway", + "paris": "Paris,Paris,Ile-de-France,France", + "prague": "Prague,Prague,Czechia", + "rio de janeiro": "Rio de Janeiro,State of Rio de Janeiro,Brazil", + "rome": "Rome,Rome,Lazio,Italy", + "san francisco": "San Francisco,California,United States", + "sao paulo": "Sao Paulo,State of Sao Paulo,Brazil", + "seattle": "Seattle,Washington,United States", + "seoul": "Seoul,Seoul,South Korea", + "shanghai": "Shanghai,Shanghai,China", + "singapore": "Singapore,Singapore", + "stockholm": "Stockholm,Stockholm County,Sweden", + "sydney": "Sydney,New South Wales,Australia", + "tokyo": "Tokyo,Tokyo,Japan", + "toronto": "Toronto,Toronto,Ontario,Canada", + "vancouver": "Vancouver,British Columbia,Canada", + "vienna": "Vienna,Vienna,Vienna,Austria", + "warsaw": "Warsaw,Warsaw,Masovian Voivodeship,Poland", + "washington": "Washington,District of Columbia,United States", + "zurich": "Zurich,Zurich,Switzerland", +} + +// ResolveRegion turns a free-text region hint into per-engine targeting. It +// never errors: unrecognized input yields a RegionTarget with empty engine +// fields, leaving callers to fall back to their defaults. +// +// Accepted inputs, in priority order: +// - Numeric (e.g. "213"): a Yandex-native lr ID. Passed through as YandexLR. +// - A 2-letter country or BCP47-style locale (e.g. "DE", "en-GB"): resolved to +// a country and its Yandex lr. No Google canonical โ€” country targeting rides +// on gl=, not UULE. +// - A bare curated city name (e.g. "Berlin"): resolved to its canonical name. +// - A full "City,Region,Country" canonical name typed verbatim (>=2 commas): +// passed through to Google as-is. +func ResolveRegion(region string) RegionTarget { + region = strings.TrimSpace(region) + t := RegionTarget{Raw: region} + if region == "" { + return t + } + + // Yandex-native numeric lr IDs: pass through, nothing else to derive. + if isDigitsOnly(region) { + t.YandexLR = region + return t + } + + // Country / locale code (e.g. "DE", "en-GB"). Country-level targeting is + // conveyed via Country (Google uses gl=, Yandex the country lr); we + // deliberately do NOT emit a Google canonical/UULE for a whole country. + if cc := CountryFromRegion(region); cc != "" { + t.Country = cc + t.YandexLR = yandexLRByCountry[cc] + return t + } + + // Bare curated city name (e.g. "Berlin", "New York"). + if c := cityCanonical[strings.ToLower(region)]; c != "" { + t.GoogleCanonical = c + return t + } + + // Looks like a full "City,Region,Country" canonical name the caller typed + // verbatim (>=2 commas). Pass it through to Google as-is; Google ignores it + // if it isn't a real canonical name, which is the caller's responsibility. + if strings.Count(region, ",") >= 2 { + t.GoogleCanonical = region + return t + } + + return t +} + +func isDigitsOnly(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} diff --git a/core/region_test.go b/core/region_test.go new file mode 100644 index 0000000..e023e46 --- /dev/null +++ b/core/region_test.go @@ -0,0 +1,74 @@ +package core + +import "testing" + +func TestResolveRegion(t *testing.T) { + tests := []struct { + name string + region string + country string + canonical string + lr string + }{ + { + name: "bare curated city resolves to canonical name", + region: "Berlin", + canonical: "Berlin,Berlin,Germany", + }, + { + name: "curated city is case-insensitive", + region: " berlin ", + canonical: "Berlin,Berlin,Germany", + }, + { + name: "ambiguous curated city picks obvious match", + region: "London", + canonical: "London,England,United Kingdom", + }, + { + name: "full canonical name passes through verbatim", + region: "Smalltown,Some Region,Faraway", + canonical: "Smalltown,Some Region,Faraway", + }, + { + name: "bare country code sets country and yandex lr but no uule", + region: "DE", + country: "DE", + lr: "96", + }, + { + name: "locale code resolves country", + region: "en-GB", + country: "GB", + lr: "102", + }, + { + name: "numeric region is a yandex lr passthrough", + region: "213", + lr: "213", + }, + { + name: "empty region resolves to nothing", + region: "", + }, + { + name: "unknown bare name resolves to nothing", + region: "Nowhereville", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ResolveRegion(tt.region) + if got.Country != tt.country { + t.Errorf("Country = %q, want %q", got.Country, tt.country) + } + if got.GoogleCanonical != tt.canonical { + t.Errorf("GoogleCanonical = %q, want %q", got.GoogleCanonical, tt.canonical) + } + if got.YandexLR != tt.lr { + t.Errorf("YandexLR = %q, want %q", got.YandexLR, tt.lr) + } + }) + } +} diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 4d1b0a9..b6b007e 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -594,9 +594,11 @@ components: in: query required: false description: > - Market/location hint. Yandex accepts numeric `lr` region IDs such as - `213`; Google, Bing, and DuckDuckGo use country/locale-style hints such - as `RU`, `US`, or `en-GB` where supported. + Market/location hint. Country or locale-style values such as `US`, + `DE`, or `en-GB` are shared by engines that support them. Google also + accepts city names such as `Berlin` or `New York` and sends them as + `uule`. Yandex accepts numeric `lr` region IDs such as `213`; those IDs + are engine-specific and are ignored by other engines. schema: type: string examples: @@ -604,6 +606,8 @@ components: value: "213" country: value: RU + google_city: + value: Berlin DateQuery: name: date in: query diff --git a/google/search_test.go b/google/search_test.go index d8a42dc..867dff4 100644 --- a/google/search_test.go +++ b/google/search_test.go @@ -133,6 +133,68 @@ func TestBuildSearchURL(t *testing.T) { } }, }, + { + name: "curated city region resolves to canonical google uule without country market", + query: core.Query{ + Text: "weather", + Region: "New York", + Filter: true, + }, + check: func(t *testing.T, params url.Values, host string) { + t.Helper() + if host != "www.google.com" { + t.Fatalf("unexpected host: %s", host) + } + // "New York" resolves to the exact Google canonical name + // "New York,New York,United States", which is what Google + // actually honors (a bare city name is ignored). + if got := params.Get("uule"); got != "w+CAIQICIfTmV3IFlvcmssTmV3IFlvcmssVW5pdGVkIFN0YXRlcw==" { + t.Fatalf("unexpected uule value: %q", got) + } + if got := params.Get("gl"); got != "" { + t.Fatalf("city region should not set gl, got %q", got) + } + }, + }, + { + name: "bare city Berlin resolves to canonical google uule", + query: core.Query{ + Text: "weather", + Region: "Berlin", + Filter: true, + }, + check: func(t *testing.T, params url.Values, host string) { + t.Helper() + // region=Berlin -> "Berlin,Berlin,Germany" canonical, which is + // the form Google honors. A bare "Berlin" UULE is ignored. + if got := params.Get("uule"); got != "w+CAIQICIVQmVybGluLEJlcmxpbixHZXJtYW55" { + t.Fatalf("unexpected uule value: %q", got) + } + if got := params.Get("gl"); got != "" { + t.Fatalf("city region should not set gl, got %q", got) + } + }, + }, + { + name: "numeric yandex region does not set google market", + query: core.Query{ + Text: "weather", + Region: "213", + Filter: true, + }, + check: func(t *testing.T, params url.Values, host string) { + t.Helper() + if host != "www.google.com" { + t.Fatalf("unexpected host: %s", host) + } + if got := params.Get("gl"); got != "" { + t.Fatalf("numeric region should not set gl, got %q", got) + } + if got := params.Get("uule"); got != "" { + t.Fatalf("numeric region should not set uule, got %q", got) + } + }, + }, { name: "site and filetype without text", query: core.Query{ diff --git a/google/url.go b/google/url.go index db237e4..aeda40c 100644 --- a/google/url.go +++ b/google/url.go @@ -1,6 +1,7 @@ package google import ( + "encoding/base64" "errors" "fmt" "net/url" @@ -225,6 +226,10 @@ var googleDefaultCountryByLanguage = map[string]string{ "ko": "kr", } +const googleUULEPrefix = "w+CAIQICI" + +var googleUULELengthAlphabet = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_") + // BuildURL builds a Google web search URL from Query fields. // It returns an error when the resulting query text is empty or invalid. func BuildURL(q core.Query) (string, error) { @@ -289,6 +294,9 @@ func BuildURL(q core.Query) (string, error) { if locale.country != "" { params.Add("gl", locale.country) } + if uule := googleUULE(q.Region); uule != "" { + params.Add("uule", uule) + } if locale.language != "" { params.Add("hl", locale.language) params.Add("lr", "lang_"+locale.language) @@ -355,6 +363,9 @@ func BuildImageURL(q core.Query) (string, error) { if locale.country != "" { params.Add("gl", locale.country) } + if uule := googleUULE(q.Region); uule != "" { + params.Add("uule", uule) + } if locale.language != "" { params.Add("hl", locale.language) params.Add("lr", "lang_"+locale.language) @@ -420,6 +431,43 @@ func googleDomain(locale googleLocaleParams) string { return GoogleDomains[""] } +// googleUULE builds a Google UULE v1 value for a region hint. +// +// UULE only takes effect when the encoded string is an exact Google geotargets +// canonical name; a bare free-text city is silently ignored by Google. We +// therefore resolve the region first (core.ResolveRegion) and encode the +// resolved canonical name when one is found. Country-level hints intentionally +// produce no UULE โ€” those are conveyed via gl= instead. +func googleUULE(region string) string { + target := core.ResolveRegion(region) + canonical := target.GoogleCanonical + if canonical == "" { + // No resolved canonical name. A bare country code (e.g. "DE") or a + // numeric Yandex lr ID must not become a UULE; both are handled + // elsewhere (gl=) or are not applicable to Google. + if target.Country != "" || target.YandexLR != "" { + return "" + } + // Best-effort: encode the raw free-text region. This may be ignored by + // Google if it isn't a canonical name, matching prior behavior. + canonical = strings.TrimSpace(region) + } + if canonical == "" { + return "" + } + return encodeGoogleUULE(canonical) +} + +// encodeGoogleUULE encodes a canonical location name into a UULE v1 value: +// a fixed prefix, a single length-tag character, then base64(name). +func encodeGoogleUULE(canonical string) string { + length := len([]rune(canonical)) + if length <= 0 || length >= len(googleUULELengthAlphabet) { + return "" + } + return googleUULEPrefix + string(googleUULELengthAlphabet[length]) + base64.StdEncoding.EncodeToString([]byte(canonical)) +} + // SourceImage contains parsed Google image metadata extracted from result links. type SourceImage struct { PageURL string diff --git a/yandex/url.go b/yandex/url.go index 63c4253..a8ca50e 100644 --- a/yandex/url.go +++ b/yandex/url.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "net/url" - "strings" "github.com/karust/openserp/core" ) @@ -45,6 +44,7 @@ func BuildURL(q core.Query, page int) (string, error) { if lr := yandexLR(q.Region); lr != "" { params.Add("lr", lr) + params.Add("rstr", "true") } base.RawQuery = params.Encode() @@ -84,6 +84,7 @@ func BuildImageURL(q core.Query, page int) (string, error) { if lr := yandexLR(q.Region); lr != "" { params.Add("lr", lr) + params.Add("rstr", "true") } base.RawQuery = params.Encode() @@ -91,14 +92,5 @@ func BuildImageURL(q core.Query, page int) (string, error) { } func yandexLR(region string) string { - region = strings.TrimSpace(region) - if region == "" { - return "" - } - for i := 0; i < len(region); i++ { - if region[i] < '0' || region[i] > '9' { - return "" - } - } - return region + return core.ResolveRegion(region).YandexLR } diff --git a/yandex/url_test.go b/yandex/url_test.go index 8746c78..e80d2e4 100644 --- a/yandex/url_test.go +++ b/yandex/url_test.go @@ -54,7 +54,8 @@ func TestBuildURLRegionLR(t *testing.T) { }{ {name: "numeric yandex region is lr", region: "213", wantLR: "213"}, {name: "whitespace is trimmed", region: " 1 ", wantLR: "1"}, - {name: "country region is ignored by yandex lr", region: "RU", wantLR: ""}, + {name: "country region maps to yandex lr", region: "DE", wantLR: "96"}, + {name: "locale region maps to yandex lr", region: "en-GB", wantLR: "102"}, {name: "empty region omits lr", region: "", wantLR: ""}, } @@ -72,6 +73,13 @@ func TestBuildURLRegionLR(t *testing.T) { if gotLR := parsed.Query().Get("lr"); gotLR != tt.wantLR { t.Fatalf("unexpected lr value: %q want %q", gotLR, tt.wantLR) } + wantRstr := "" + if tt.wantLR != "" { + wantRstr = "true" + } + if gotRstr := parsed.Query().Get("rstr"); gotRstr != wantRstr { + t.Fatalf("unexpected rstr value: %q for lr %q", gotRstr, tt.wantLR) + } }) } } @@ -89,4 +97,7 @@ func TestBuildImageURLRegionLR(t *testing.T) { if gotLR := parsed.Query().Get("lr"); gotLR != "2" { t.Fatalf("unexpected lr value: %q", gotLR) } + if gotRstr := parsed.Query().Get("rstr"); gotRstr != "true" { + t.Fatalf("unexpected rstr value: %q", gotRstr) + } }