Add google uule query param support for regional search

2026-08-05 16:53:54 +08:00 · 2026-06-02 03:54:55 +03:00
parent 2b96a8330f
commit 7ed0780192
9 changed files with 396 additions and 29 deletions
--- a/README.md
+++ b/README.md
@@ -14,7 +14,9 @@ Run it locally, self-host it, or use the optional hosted API when you do not wan

 **Official website:** [openserp.org](https://openserp.org)

-**Feedback:** [GitHub Issues](https://github.com/karust/openserp/issues) · [Telegram](https://t.me/+RJEKspw3mUlhZDMy) · [feedback@openserp.org](mailto:feedback@openserp.org)
+**Feedback:** [GitHub Issues](https://github.com/karust/openserp/issues) or [feedback@openserp.org](mailto:feedback@openserp.org)
+
+**Latest updates, usage examples**: [Telegram](https://t.me/+RJEKspw3mUlhZDMy)

 > 💡 OpenSERP is free and open-source. Only links listed in this repository and on the official website are associated with the project.

@@ -114,17 +116,17 @@ curl "http://127.0.0.1:7000/mega/engines"

 Common parameters:

-| Parameter | Description                                                                  | Example                              |
-| --------- | ---------------------------------------------------------------------------- | ------------------------------------ |
-| `text`    | Search query                                                                 | `golang programming`                 |
-| `lang`    | Language code                                                                | `EN`, `DE`, `RU`, `ES`               |
-| `region`  | Market/location hint. Yandex accepts numeric `lr`; others use country hints. | `213`, `RU`, `en-US`                 |
-| `date`    | Date range                                                                   | `20250101..20251231`                 |
-| `file`    | File extension                                                               | `pdf`, `doc`, `xls`                  |
-| `site`    | Site-specific search                                                         | `github.com`                         |
-| `limit`   | Number of organic results, max 100. Ads may be returned in addition.         | `10`, `25`, `50`                     |
-| `start`   | Pagination offset                                                            | `0`, `10`, `20`                      |
-| `format`  | Output format                                                                | `json`, `markdown`, `text`, `ndjson` |
+| Parameter | Description                                                                                                                          | Example                              |
+| --------- | ------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------ |
+| `text`    | Search query                                                                                                                         | `golang programming`                 |
+| `lang`    | Language code                                                                                                                        | `EN`, `DE`, `RU`, `ES`               |
+| `region`  | Market/location hint. Countries/locales work across engines; Google also accepts city names via `uule`; Yandex accepts numeric `lr`. | `DE`, `en-GB`, `Berlin`, `213`       |
+| `date`    | Date range                                                                                                                           | `20250101..20251231`                 |
+| `file`    | File extension                                                                                                                       | `pdf`, `doc`, `xls`                  |
+| `site`    | Site-specific search                                                                                                                 | `github.com`                         |
+| `limit`   | Number of organic results, max 100. Ads may be returned in addition.                                                                 | `10`, `25`, `50`                     |
+| `start`   | Pagination offset                                                                                                                    | `0`, `10`, `20`                      |
+| `format`  | Output format                                                                                                                        | `json`, `markdown`, `text`, `ndjson` |

 Engine-specific parameters:

@@ -298,4 +300,4 @@ Contributions are welcome. See [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md).
 - [Telegram channel](https://t.me/openserp_cloud) - OpenSERP news, release notes, and project updates. Direct messages are open for quick feedback and hosted API questions.
 - [feedback@openserp.org](mailto:feedback@openserp.org) - private notes, longer feedback, or anything that does not fit GitHub Issues.

-###### _"OpenSERP" is the name of this open-source project. The official website is [openserp.org](https://openserp.org). Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._
+###### _"OpenSERP" is the name of this open-source project. The official website is [openserp.org](https://openserp.org). Resources not linked on this page are not affiliated with the project._
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -16,7 +16,7 @@ import (
 )

 const (
-	version               = "0.7.14"
+	version               = "0.7.15"
 	defaultConfigFilename = "config"
 	envPrefix             = "OPENSERP"
 )
--- a/core/region.go
+++ b/core/region.go
@@ -0,0 +1,174 @@
+package core
+
+import "strings"
+
+// RegionTarget is the resolved, per-engine targeting for a user-supplied region
+// hint. Engines read the field relevant to them: Google uses GoogleCanonical to
+// build a UULE, Yandex uses YandexLR. Country is the ISO 3166-1 alpha-2 code
+// when one could be derived, useful as a coarse market signal.
+//
+// A field left empty means "no better signal than the raw input" — callers
+// should fall back to their previous behavior (e.g. gl= from locale, or
+// dropping the parameter entirely). Resolution never fails: an unrecognized
+// region simply yields empty engine fields rather than an error.
+type RegionTarget struct {
+	// Raw is the trimmed original input, preserved for engines that pass it
+	// through (e.g. Yandex numeric lr IDs).
+	Raw string
+	// Country is the ISO 3166-1 alpha-2 code (uppercase) when derivable, else "".
+	Country string
+	// GoogleCanonical is the exact Google geotargets canonical location name
+	// (e.g. "Berlin,Berlin,Germany") suitable for UULE v1 encoding, else "".
+	GoogleCanonical string
+	// YandexLR is the Yandex lr region ID (e.g. "213"), else "".
+	YandexLR string
+}
+
+// yandexLRByCountry maps an ISO country code to a Yandex lr region ID. Yandex
+// only exposes a limited set of country-level regions; cities require numeric
+// lr IDs passed through verbatim.
+var yandexLRByCountry = map[string]string{
+	"AT": "113", "AU": "211", "BE": "114", "BR": "94", "CA": "95",
+	"CH": "126", "DE": "96", "DK": "203", "ES": "204", "FI": "123",
+	"FR": "124", "GB": "102", "IE": "10063", "IN": "994", "IT": "205",
+	"JP": "137", "KR": "135", "MX": "20271", "NL": "118", "NO": "119",
+	"PL": "120", "RU": "225", "SE": "127", "SG": "10105", "TR": "983",
+	"UA": "187", "UK": "102", "US": "84", "ZA": "10021",
+}
+
+// cityCanonical maps a normalized bare city name to its exact Google geotargets
+// canonical name (used to build a UULE). Only UULE-bearing city targeting needs
+// a name table — country/state targeting rides on gl= and never needs one.
+//
+// The list is deliberately small and hand-curated: bare city names are
+// ambiguous (e.g. "London" exists in CA/GB/US), so we only auto-resolve a
+// prominence list to its "obvious" match. Canonical names below are copied
+// verbatim from Google's geotargets data; any other city can still be targeted
+// by passing its full canonical name ("City,Region,Country") directly.
+var cityCanonical = map[string]string{
+	"amsterdam":      "Amsterdam,North Holland,Netherlands",
+	"athens":         "Athens,Athens,Attica,Greece",
+	"austin":         "Austin,Texas,United States",
+	"bangalore":      "Bengaluru,Karnataka,India",
+	"barcelona":      "Barcelona,Barcelona,Catalonia,Spain",
+	"beijing":        "Beijing,Beijing,China",
+	"berlin":         "Berlin,Berlin,Germany",
+	"birmingham":     "Birmingham,West Midlands,England,United Kingdom",
+	"boston":         "Boston,Massachusetts,United States",
+	"brussels":       "Brussels,Brussels,Belgium",
+	"buenos aires":   "Buenos Aires,Buenos Aires,Argentina",
+	"cairo":          "Cairo,Cairo Governorate,Egypt",
+	"chicago":        "Chicago,Illinois,United States",
+	"copenhagen":     "Copenhagen,Capital Region of Denmark,Denmark",
+	"dallas":         "Dallas,Texas,United States",
+	"delhi":          "Delhi,Delhi,India",
+	"dubai":          "Dubai,Dubai,United Arab Emirates",
+	"dublin":         "Dublin,County Dublin,Ireland",
+	"frankfurt":      "Frankfurt am Main,Hessen,Germany",
+	"hamburg":        "Hamburg,Hamburg,Germany",
+	"helsinki":       "Helsinki,Helsinki,Uusimaa,Finland",
+	"hong kong":      "Hong Kong,Hong Kong",
+	"istanbul":       "Istanbul,Istanbul,Turkiye",
+	"johannesburg":   "Johannesburg,Gauteng,South Africa",
+	"kyiv":           "Kyiv,Kyiv city,Ukraine",
+	"lisbon":         "Lisbon,Lisbon,Lisbon,Portugal",
+	"london":         "London,England,United Kingdom",
+	"los angeles":    "Los Angeles,California,United States",
+	"lyon":           "Lyon,Auvergne-Rhone-Alpes,France",
+	"madrid":         "Madrid,Community of Madrid,Spain",
+	"manchester":     "Manchester,England,United Kingdom",
+	"marseille":      "Marseille,Provence-Alpes-Cote d'Azur,France",
+	"melbourne":      "Melbourne,Victoria,Australia",
+	"mexico city":    "Mexico City,Mexico City,Mexico",
+	"miami":          "Miami,Florida,United States",
+	"milan":          "Milan,Milan,Lombardy,Italy",
+	"montreal":       "Montreal,Montreal,Quebec,Canada",
+	"moscow":         "Moscow,Moscow,Russia",
+	"mumbai":         "Mumbai,Maharashtra,India",
+	"munich":         "Munich,Bavaria,Germany",
+	"new york":       "New York,New York,United States",
+	"osaka":          "Osaka,Osaka,Japan",
+	"oslo":           "Oslo,Oslo,Norway",
+	"paris":          "Paris,Paris,Ile-de-France,France",
+	"prague":         "Prague,Prague,Czechia",
+	"rio de janeiro": "Rio de Janeiro,State of Rio de Janeiro,Brazil",
+	"rome":           "Rome,Rome,Lazio,Italy",
+	"san francisco":  "San Francisco,California,United States",
+	"sao paulo":      "Sao Paulo,State of Sao Paulo,Brazil",
+	"seattle":        "Seattle,Washington,United States",
+	"seoul":          "Seoul,Seoul,South Korea",
+	"shanghai":       "Shanghai,Shanghai,China",
+	"singapore":      "Singapore,Singapore",
+	"stockholm":      "Stockholm,Stockholm County,Sweden",
+	"sydney":         "Sydney,New South Wales,Australia",
+	"tokyo":          "Tokyo,Tokyo,Japan",
+	"toronto":        "Toronto,Toronto,Ontario,Canada",
+	"vancouver":      "Vancouver,British Columbia,Canada",
+	"vienna":         "Vienna,Vienna,Vienna,Austria",
+	"warsaw":         "Warsaw,Warsaw,Masovian Voivodeship,Poland",
+	"washington":     "Washington,District of Columbia,United States",
+	"zurich":         "Zurich,Zurich,Switzerland",
+}
+
+// ResolveRegion turns a free-text region hint into per-engine targeting. It
+// never errors: unrecognized input yields a RegionTarget with empty engine
+// fields, leaving callers to fall back to their defaults.
+//
+// Accepted inputs, in priority order:
+//   - Numeric (e.g. "213"): a Yandex-native lr ID. Passed through as YandexLR.
+//   - A 2-letter country or BCP47-style locale (e.g. "DE", "en-GB"): resolved to
+//     a country and its Yandex lr. No Google canonical — country targeting rides
+//     on gl=, not UULE.
+//   - A bare curated city name (e.g. "Berlin"): resolved to its canonical name.
+//   - A full "City,Region,Country" canonical name typed verbatim (>=2 commas):
+//     passed through to Google as-is.
+func ResolveRegion(region string) RegionTarget {
+	region = strings.TrimSpace(region)
+	t := RegionTarget{Raw: region}
+	if region == "" {
+		return t
+	}
+
+	// Yandex-native numeric lr IDs: pass through, nothing else to derive.
+	if isDigitsOnly(region) {
+		t.YandexLR = region
+		return t
+	}
+
+	// Country / locale code (e.g. "DE", "en-GB"). Country-level targeting is
+	// conveyed via Country (Google uses gl=, Yandex the country lr); we
+	// deliberately do NOT emit a Google canonical/UULE for a whole country.
+	if cc := CountryFromRegion(region); cc != "" {
+		t.Country = cc
+		t.YandexLR = yandexLRByCountry[cc]
+		return t
+	}
+
+	// Bare curated city name (e.g. "Berlin", "New York").
+	if c := cityCanonical[strings.ToLower(region)]; c != "" {
+		t.GoogleCanonical = c
+		return t
+	}
+
+	// Looks like a full "City,Region,Country" canonical name the caller typed
+	// verbatim (>=2 commas). Pass it through to Google as-is; Google ignores it
+	// if it isn't a real canonical name, which is the caller's responsibility.
+	if strings.Count(region, ",") >= 2 {
+		t.GoogleCanonical = region
+		return t
+	}
+
+	return t
+}
+
+func isDigitsOnly(s string) bool {
+	if s == "" {
+		return false
+	}
+	for _, r := range s {
+		if r < '0' || r > '9' {
+			return false
+		}
+	}
+	return true
+}
--- a/core/region_test.go
+++ b/core/region_test.go
@@ -0,0 +1,74 @@
+package core
+
+import "testing"
+
+func TestResolveRegion(t *testing.T) {
+	tests := []struct {
+		name      string
+		region    string
+		country   string
+		canonical string
+		lr        string
+	}{
+		{
+			name:      "bare curated city resolves to canonical name",
+			region:    "Berlin",
+			canonical: "Berlin,Berlin,Germany",
+		},
+		{
+			name:      "curated city is case-insensitive",
+			region:    "  berlin ",
+			canonical: "Berlin,Berlin,Germany",
+		},
+		{
+			name:      "ambiguous curated city picks obvious match",
+			region:    "London",
+			canonical: "London,England,United Kingdom",
+		},
+		{
+			name:      "full canonical name passes through verbatim",
+			region:    "Smalltown,Some Region,Faraway",
+			canonical: "Smalltown,Some Region,Faraway",
+		},
+		{
+			name:    "bare country code sets country and yandex lr but no uule",
+			region:  "DE",
+			country: "DE",
+			lr:      "96",
+		},
+		{
+			name:    "locale code resolves country",
+			region:  "en-GB",
+			country: "GB",
+			lr:      "102",
+		},
+		{
+			name:   "numeric region is a yandex lr passthrough",
+			region: "213",
+			lr:     "213",
+		},
+		{
+			name:   "empty region resolves to nothing",
+			region: "",
+		},
+		{
+			name:   "unknown bare name resolves to nothing",
+			region: "Nowhereville",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := ResolveRegion(tt.region)
+			if got.Country != tt.country {
+				t.Errorf("Country = %q, want %q", got.Country, tt.country)
+			}
+			if got.GoogleCanonical != tt.canonical {
+				t.Errorf("GoogleCanonical = %q, want %q", got.GoogleCanonical, tt.canonical)
+			}
+			if got.YandexLR != tt.lr {
+				t.Errorf("YandexLR = %q, want %q", got.YandexLR, tt.lr)
+			}
+		})
+	}
+}
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -594,9 +594,11 @@ components:
      in: query
      required: false
      description: >
-        Market/location hint. Yandex accepts numeric `lr` region IDs such as
-        `213`; Google, Bing, and DuckDuckGo use country/locale-style hints such
-        as `RU`, `US`, or `en-GB` where supported.
+        Market/location hint. Country or locale-style values such as `US`,
+        `DE`, or `en-GB` are shared by engines that support them. Google also
+        accepts city names such as `Berlin` or `New York` and sends them as
+        `uule`. Yandex accepts numeric `lr` region IDs such as `213`; those IDs
+        are engine-specific and are ignored by other engines.
      schema:
        type: string
      examples:
@@ -604,6 +606,8 @@ components:
          value: "213"
        country:
          value: RU
+        google_city:
+          value: Berlin
    DateQuery:
      name: date
      in: query
--- a/google/search_test.go
+++ b/google/search_test.go
@@ -133,6 +133,68 @@ func TestBuildSearchURL(t *testing.T) {
 				}
 			},
 		},
+		{
+			name: "curated city region resolves to canonical google uule without country market",
+			query: core.Query{
+				Text:   "weather",
+				Region: "New York",
+				Filter: true,
+			},
+			check: func(t *testing.T, params url.Values, host string) {
+				t.Helper()
+				if host != "www.google.com" {
+					t.Fatalf("unexpected host: %s", host)
+				}
+				// "New York" resolves to the exact Google canonical name
+				// "New York,New York,United States", which is what Google
+				// actually honors (a bare city name is ignored).
+				if got := params.Get("uule"); got != "w+CAIQICIfTmV3IFlvcmssTmV3IFlvcmssVW5pdGVkIFN0YXRlcw==" {
+					t.Fatalf("unexpected uule value: %q", got)
+				}
+				if got := params.Get("gl"); got != "" {
+					t.Fatalf("city region should not set gl, got %q", got)
+				}
+			},
+		},
+		{
+			name: "bare city Berlin resolves to canonical google uule",
+			query: core.Query{
+				Text:   "weather",
+				Region: "Berlin",
+				Filter: true,
+			},
+			check: func(t *testing.T, params url.Values, host string) {
+				t.Helper()
+				// region=Berlin -> "Berlin,Berlin,Germany" canonical, which is
+				// the form Google honors. A bare "Berlin" UULE is ignored.
+				if got := params.Get("uule"); got != "w+CAIQICIVQmVybGluLEJlcmxpbixHZXJtYW55" {
+					t.Fatalf("unexpected uule value: %q", got)
+				}
+				if got := params.Get("gl"); got != "" {
+					t.Fatalf("city region should not set gl, got %q", got)
+				}
+			},
+		},
+		{
+			name: "numeric yandex region does not set google market",
+			query: core.Query{
+				Text:   "weather",
+				Region: "213",
+				Filter: true,
+			},
+			check: func(t *testing.T, params url.Values, host string) {
+				t.Helper()
+				if host != "www.google.com" {
+					t.Fatalf("unexpected host: %s", host)
+				}
+				if got := params.Get("gl"); got != "" {
+					t.Fatalf("numeric region should not set gl, got %q", got)
+				}
+				if got := params.Get("uule"); got != "" {
+					t.Fatalf("numeric region should not set uule, got %q", got)
+				}
+			},
+		},
 		{
 			name: "site and filetype without text",
 			query: core.Query{
--- a/google/url.go
+++ b/google/url.go
@@ -1,6 +1,7 @@
 package google

 import (
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"net/url"
@@ -225,6 +226,10 @@ var googleDefaultCountryByLanguage = map[string]string{
 	"ko": "kr",
 }

+const googleUULEPrefix = "w+CAIQICI"
+
+var googleUULELengthAlphabet = []byte("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_")
+
 // BuildURL builds a Google web search URL from Query fields.
 // It returns an error when the resulting query text is empty or invalid.
 func BuildURL(q core.Query) (string, error) {
@@ -289,6 +294,9 @@ func BuildURL(q core.Query) (string, error) {
 	if locale.country != "" {
 		params.Add("gl", locale.country)
 	}
+	if uule := googleUULE(q.Region); uule != "" {
+		params.Add("uule", uule)
+	}
 	if locale.language != "" {
 		params.Add("hl", locale.language)
 		params.Add("lr", "lang_"+locale.language)
@@ -355,6 +363,9 @@ func BuildImageURL(q core.Query) (string, error) {
 	if locale.country != "" {
 		params.Add("gl", locale.country)
 	}
+	if uule := googleUULE(q.Region); uule != "" {
+		params.Add("uule", uule)
+	}
 	if locale.language != "" {
 		params.Add("hl", locale.language)
 		params.Add("lr", "lang_"+locale.language)
@@ -420,6 +431,43 @@ func googleDomain(locale googleLocaleParams) string {
 	return GoogleDomains[""]
 }

+// googleUULE builds a Google UULE v1 value for a region hint.
+//
+// UULE only takes effect when the encoded string is an exact Google geotargets
+// canonical name; a bare free-text city is silently ignored by Google. We
+// therefore resolve the region first (core.ResolveRegion) and encode the
+// resolved canonical name when one is found. Country-level hints intentionally
+// produce no UULE — those are conveyed via gl= instead.
+func googleUULE(region string) string {
+	target := core.ResolveRegion(region)
+	canonical := target.GoogleCanonical
+	if canonical == "" {
+		// No resolved canonical name. A bare country code (e.g. "DE") or a
+		// numeric Yandex lr ID must not become a UULE; both are handled
+		// elsewhere (gl=) or are not applicable to Google.
+		if target.Country != "" || target.YandexLR != "" {
+			return ""
+		}
+		// Best-effort: encode the raw free-text region. This may be ignored by
+		// Google if it isn't a canonical name, matching prior behavior.
+		canonical = strings.TrimSpace(region)
+	}
+	if canonical == "" {
+		return ""
+	}
+	return encodeGoogleUULE(canonical)
+}
+
+// encodeGoogleUULE encodes a canonical location name into a UULE v1 value:
+// a fixed prefix, a single length-tag character, then base64(name).
+func encodeGoogleUULE(canonical string) string {
+	length := len([]rune(canonical))
+	if length <= 0 || length >= len(googleUULELengthAlphabet) {
+		return ""
+	}
+	return googleUULEPrefix + string(googleUULELengthAlphabet[length]) + base64.StdEncoding.EncodeToString([]byte(canonical))
+}
+
 // SourceImage contains parsed Google image metadata extracted from result links.
 type SourceImage struct {
 	PageURL     string
--- a/yandex/url.go
+++ b/yandex/url.go
@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"net/url"
-	"strings"

 	"github.com/karust/openserp/core"
 )
@@ -45,6 +44,7 @@ func BuildURL(q core.Query, page int) (string, error) {

 	if lr := yandexLR(q.Region); lr != "" {
 		params.Add("lr", lr)
+		params.Add("rstr", "true")
 	}

 	base.RawQuery = params.Encode()
@@ -84,6 +84,7 @@ func BuildImageURL(q core.Query, page int) (string, error) {

 	if lr := yandexLR(q.Region); lr != "" {
 		params.Add("lr", lr)
+		params.Add("rstr", "true")
 	}

 	base.RawQuery = params.Encode()
@@ -91,14 +92,5 @@ func BuildImageURL(q core.Query, page int) (string, error) {
 }

 func yandexLR(region string) string {
-	region = strings.TrimSpace(region)
-	if region == "" {
-		return ""
-	}
-	for i := 0; i < len(region); i++ {
-		if region[i] < '0' || region[i] > '9' {
-			return ""
-		}
-	}
-	return region
+	return core.ResolveRegion(region).YandexLR
 }
--- a/yandex/url_test.go
+++ b/yandex/url_test.go
@@ -54,7 +54,8 @@ func TestBuildURLRegionLR(t *testing.T) {
 	}{
 		{name: "numeric yandex region is lr", region: "213", wantLR: "213"},
 		{name: "whitespace is trimmed", region: " 1 ", wantLR: "1"},
-		{name: "country region is ignored by yandex lr", region: "RU", wantLR: ""},
+		{name: "country region maps to yandex lr", region: "DE", wantLR: "96"},
+		{name: "locale region maps to yandex lr", region: "en-GB", wantLR: "102"},
 		{name: "empty region omits lr", region: "", wantLR: ""},
 	}

@@ -72,6 +73,13 @@ func TestBuildURLRegionLR(t *testing.T) {
 			if gotLR := parsed.Query().Get("lr"); gotLR != tt.wantLR {
 				t.Fatalf("unexpected lr value: %q want %q", gotLR, tt.wantLR)
 			}
+			wantRstr := ""
+			if tt.wantLR != "" {
+				wantRstr = "true"
+			}
+			if gotRstr := parsed.Query().Get("rstr"); gotRstr != wantRstr {
+				t.Fatalf("unexpected rstr value: %q for lr %q", gotRstr, tt.wantLR)
+			}
 		})
 	}
 }
@@ -89,4 +97,7 @@ func TestBuildImageURLRegionLR(t *testing.T) {
 	if gotLR := parsed.Query().Get("lr"); gotLR != "2" {
 		t.Fatalf("unexpected lr value: %q", gotLR)
 	}
+	if gotRstr := parsed.Query().Get("rstr"); gotRstr != "true" {
+		t.Fatalf("unexpected rstr value: %q", gotRstr)
+	}
 }