package google import ( "errors" "fmt" "net/url" "strconv" "strings" "github.com/karust/openserp/core" regionpkg "github.com/karust/openserp/core/region" "github.com/sirupsen/logrus" ) // GoogleDomains maps language/country hints to Google TLD suffixes used for // URL construction. var GoogleDomains = map[string]string{ "": "com", "en": "com", "ac": "ac", "ad": "ad", "ae": "ae", "af": "com.af", "ag": "com.ag", "ai": "com.ai", "al": "al", "am": "am", "ao": "co.ao", "ar": "com.ar", "as": "as", "at": "at", "au": "com.au", "az": "az", "ba": "ba", "bd": "com.bd", "be": "be", "bf": "bf", "bg": "bg", "bh": "com.bh", "bi": "bi", "bj": "bj", "bn": "com.bn", "bo": "com.bo", "br": "com.br", "bs": "bs", "bt": "bt", "bw": "co.bw", "by": "by", "bz": "com.bz", "ca": "ca", "kh": "com.kh", "cc": "cc", "cd": "cd", "cf": "cf", "cat": "cat", "cg": "cg", "ch": "ch", "ci": "ci", "ck": "co.ck", "cl": "cl", "cm": "cm", "cn": "cn", "co": "com.co", "cr": "co.cr", "cu": "com.cu", "cv": "cv", "cy": "com.cy", "cz": "cz", "de": "de", "dj": "dj", "dk": "dk", "dm": "dm", "do": "com.do", "dz": "dz", "ec": "com.ec", "ee": "ee", "eg": "com.eg", "es": "es", "et": "com.et", "fi": "fi", "fj": "com.fj", "fm": "fm", "fr": "fr", "ga": "ga", "gb": "co.uk", "ge": "ge", "gf": "gf", "gg": "gg", "gh": "com.gh", "gi": "com.gi", "gl": "gl", "gm": "gm", "gp": "gp", "gr": "gr", "gt": "com.gt", "gy": "gy", "hk": "com.hk", "hn": "hn", "hr": "hr", "ht": "ht", "hu": "hu", "id": "co.id", "iq": "iq", "ie": "ie", "il": "co.il", "im": "im", "in": "co.in", "io": "io", "is": "is", "it": "it", "je": "je", "jm": "com.jm", "jo": "jo", "jp": "co.jp", "ke": "co.ke", "ki": "ki", "kg": "kg", "kr": "co.kr", "kw": "com.kw", "kz": "kz", "la": "la", "lb": "com.lb", "lc": "com.lc", "li": "li", "lk": "lk", "ls": "co.ls", "lt": "lt", "lu": "lu", "lv": "lv", "ly": "com.ly", "ma": "co.ma", "md": "md", "me": "me", "mg": "mg", "mk": "mk", "ml": "ml", "mm": "com.mm", "mn": "mn", "ms": "ms", "mt": "com.mt", "mu": "mu", "mv": "mv", "mw": "mw", "mx": "com.mx", "my": "com.my", "mz": "co.mz", "na": "com.na", "ne": "ne", "nf": "com.nf", "ng": "com.ng", "ni": "com.ni", "nl": "nl", "no": "no", "np": "com.np", "nr": "nr", "nu": "nu", "nz": "co.nz", "om": "com.om", "pa": "com.pa", "pe": "com.pe", "ph": "com.ph", "pk": "com.pk", "pl": "pl", "pg": "com.pg", "pn": "pn", "pr": "com.pr", "ps": "ps", "pt": "pt", "py": "com.py", "qa": "com.qa", "ro": "ro", "rs": "rs", "ru": "ru", "rw": "rw", "sa": "com.sa", "sb": "com.sb", "sc": "sc", "se": "se", "sg": "com.sg", "sh": "sh", "si": "si", "sk": "sk", "sl": "com.sl", "sn": "sn", "sm": "sm", "so": "so", "st": "st", "sv": "com.sv", "td": "td", "tg": "tg", "th": "co.th", "tj": "com.tj", "tk": "tk", "tl": "tl", "tm": "tm", "to": "to", "tn": "tn", "tr": "com.tr", "tt": "tt", "tw": "com.tw", "tz": "co.tz", "ua": "com.ua", "ug": "co.ug", "uk": "co.uk", "uy": "com.uy", "uz": "co.uz", "vc": "com.vc", "ve": "co.ve", "vg": "vg", "vi": "co.vi", "vn": "com.vn", "vu": "vu", "ws": "ws", "za": "co.za", "zm": "co.zm", "zw": "co.zw", } // googleDefaultCountryByLanguage maps a language subtag to the country Google // associates with it when the user did not provide an explicit region. var googleDefaultCountryByLanguage = map[string]string{ "en": "us", "pt": "br", "zh": "cn", "ja": "jp", "ko": "kr", } // BuildURL builds a Google web search URL from Query fields. // It returns an error when the resulting query text is empty or invalid. func BuildURL(q core.Query) (string, error) { locale := googleLocale(q.LangCode, q.Region) googleBase := googleDomain(locale) base, err := url.Parse(fmt.Sprintf("https://www.google.%s", googleBase)) if err != nil { return "", err } base.Path += "search" params := url.Values{} // Set request text if q.Text != "" || q.Site != "" || q.Filetype != "" { text := q.Text if q.Site != "" { text += " site:" + q.Site } if q.Filetype != "" { text += " filetype:" + q.Filetype } logrus.WithField("query_hash", core.QueryHash(text)).Trace(fmt.Sprintf("Query text: %s", text)) params.Add("q", text) params.Add("oq", text) } if len(params.Get("q")) == 0 { return "", errors.New("empty query built") } // Set search date range if q.DateInterval != "" { intervals := strings.Split(q.DateInterval, "..") if len(intervals) != 2 { return "", errors.New("incorrect date interval provided") } dataParam := fmt.Sprintf("cdr:1,cd_min:%s,cd_max:%s", intervals[0], intervals[1]) params.Add("tbs", dataParam) } // Limit number of results if q.Limit > 10 { params.Add("num", strconv.Itoa(q.Limit)) } // Set result offset for pagination if q.Start < 0 { return "", errors.New("incorrect start param provided") } if q.Start > 0 { params.Add("start", strconv.Itoa(q.Start)) } // Google default is filter=1; send only when user asks to include similar results. if !q.Filter { params.Add("filter", "0") } if locale.country != "" { params.Add("gl", locale.country) } if uule := googleUULE(q.Region); uule != "" { params.Add("uule", uule) } if locale.language != "" { params.Add("hl", locale.language) params.Add("lr", "lang_"+locale.language) } params.Add("pws", "0") // Do not personalize search results params.Add("sourceid", "chrome") params.Add("ie", "UTF-8") base.RawQuery = params.Encode() return base.String(), nil } // BuildImageURL builds a Google image search URL from Query fields. // It returns an error when the resulting query text is empty or invalid. func BuildImageURL(q core.Query) (string, error) { // TODO: Add new params locale := googleLocale(q.LangCode, q.Region) googleBase := googleDomain(locale) base, err := url.Parse(fmt.Sprintf("https://www.google.%s", googleBase)) if err != nil { return "", err } base.Path += "search" params := url.Values{} params.Add("tbm", "isch") // Search images // Set request text if q.Text != "" || q.Site != "" || q.Filetype != "" { text := q.Text if q.Site != "" { text += " site:" + q.Site } if q.Filetype != "" { text += " filetype:" + q.Filetype } params.Add("q", text) params.Add("oq", text) } if len(params.Get("q")) == 0 { return "", errors.New("empty query built") } // Set search date range if q.DateInterval != "" { intervals := strings.Split(q.DateInterval, "..") if len(intervals) != 2 { return "", errors.New("incorrect date interval provided") } dataParam := fmt.Sprintf("cdr:1,cd_min:%s,cd_max:%s", intervals[0], intervals[1]) params.Add("tbs", dataParam) } // Limit number of results if q.Limit > 10 { params.Add("num", strconv.Itoa(q.Limit)) } if locale.country != "" { params.Add("gl", locale.country) } if uule := googleUULE(q.Region); uule != "" { params.Add("uule", uule) } if locale.language != "" { params.Add("hl", locale.language) params.Add("lr", "lang_"+locale.language) } params.Add("pws", "0") // Do not personalize search results base.RawQuery = params.Encode() return base.String(), nil } type googleLocaleParams struct { language string country string } // googleLocale parses langCode and fills in a default country when one is not // supplied so callers always get a usable (gl, hl) pair. func googleLocale(langCode, region string) googleLocaleParams { parsed := core.ParseLocale(langCode) country := strings.ToLower(core.CountryFromRegion(region)) if parsed.Language == "" { if country != "" { return googleLocaleParams{country: country} } return googleLocaleParams{} } if country == "" { country = strings.ToLower(parsed.Country) if country == "" { country = defaultGoogleCountry(parsed.Language) } } return googleLocaleParams{ language: parsed.Language, country: country, } } func defaultGoogleCountry(language string) string { if country, ok := googleDefaultCountryByLanguage[language]; ok { return country } // Many ISO 639-1 codes also exist as ccTLDs in GoogleDomains // (e.g. "de", "fr", "ru"); fall back to that mapping when present. if _, ok := GoogleDomains[language]; ok { return language } return "us" } // googleDomain picks a Google ccTLD for the resolved locale, preferring the // country, then the language, then the global "com" default. func googleDomain(locale googleLocaleParams) string { if domain, ok := GoogleDomains[locale.country]; ok { return domain } if domain, ok := GoogleDomains[locale.language]; ok { return domain } return GoogleDomains[""] } // googleUULE builds a Google UULE v1 value for a region hint. The resolution // and encoding live in the dependency-free core/region subpackage. func googleUULE(region string) string { return regionpkg.GoogleUULE(region) } // SourceImage contains parsed Google image metadata extracted from result links. type SourceImage struct { PageURL string OriginalURL string Width string Height string } func parseSourceImageURL(href string) (SourceImage, error) { source := SourceImage{} href = strings.ReplaceAll(href, ";", "&") parsed, err := url.QueryUnescape(href) if err != nil { return source, err } u, err := url.Parse(parsed) if err != nil { return source, err } queryMap, err := url.ParseQuery(u.RawQuery) if err != nil { return source, err } val, ok := queryMap["h"] if ok && len(val) > 0 { source.Height = val[0] } val, ok = queryMap["w"] if ok && len(val) > 0 { source.Width = val[0] } val, ok = queryMap["imgrefurl"] if ok && len(val) > 0 { source.PageURL = val[0] } val, ok = queryMap["imgurl"] if ok && len(val) > 0 { source.OriginalURL = val[0] } return source, nil }