From d47bf08523e800c1cd6c1ff610d31df2f4a61789 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Sun, 26 Apr 2026 02:40:08 +0300 Subject: [PATCH] fix api response format issues --- baidu/search.go | 17 ++- bing/search.go | 14 ++- cmd/root.go | 2 +- core/clusters.go | 16 +-- core/enrichment_domain.go | 214 ++++++++++++++++++----------------- core/enrichment_domains.yaml | 103 +++++++++++++++++ core/format_markdown.go | 10 +- core/format_text.go | 2 +- core/response.go | 41 +++---- core/response_builder.go | 123 ++++++++++++++++---- core/result.go | 78 +++++-------- core/server.go | 100 ++++++++++++---- core/server_test.go | 173 ++++++++++++++++++++++++++++ docs/openapi.yaml | 56 ++------- go.mod | 2 +- yandex/search.go | 2 +- 16 files changed, 663 insertions(+), 290 deletions(-) create mode 100644 core/enrichment_domains.yaml diff --git a/baidu/search.go b/baidu/search.go index a363d9e..d062d87 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -287,10 +287,19 @@ func (baid *Baidu) SearchImage(ctx context.Context, query core.Query) ([]core.Se continue } res := core.SearchResult{ - Rank: (searchPage * 30) + (i + 1), - URL: img.URL[0].Original, - Title: img.Title, - Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright), + Rank: (searchPage * 30) + (i + 1), + URL: img.URL[0].Original, + Title: img.Title, + Description: fmt.Sprintf( + "Source Page: %s, thumb_url:%s, %dx%d, date:%v, type:%v, copyright:%v", + img.URL[0].SourcePage, + img.ThumbURL, + img.Width, + img.Height, + img.PictureDate, + img.Type, + img.IsCopyright, + ), Ad: func() bool { if img.AdType != "0" { return true diff --git a/bing/search.go b/bing/search.go index 643099a..86ff06e 100644 --- a/bing/search.go +++ b/bing/search.go @@ -382,12 +382,14 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea // Extract information from the parsed data srchRes.Title = imgData.T srchRes.URL = imgData.IMGURL - srchRes.Description = imgData.Desc - - // Add dimensions to description if available - if imgData.W > 0 && imgData.H > 0 { - srchRes.Description += fmt.Sprintf(" (%dx%d)", imgData.W, imgData.H) - } + srchRes.Description = fmt.Sprintf( + "%s Source Page: %s, thumb_url:%s, %dx%d", + imgData.Desc, + imgData.PURL, + imgData.TURL, + imgData.W, + imgData.H, + ) // Get the page URL if imgData.MURL != "" { diff --git a/cmd/root.go b/cmd/root.go index 1a8975c..8b7f30d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,7 +15,7 @@ import ( ) const ( - version = "0.6.7" + version = "0.7.0" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/clusters.go b/core/clusters.go index 9807606..f96ba63 100644 --- a/core/clusters.go +++ b/core/clusters.go @@ -1,7 +1,7 @@ package core import ( - "crypto/sha256" + "crypto/md5" "encoding/hex" "sort" ) @@ -17,12 +17,12 @@ func BuildClusters(results []Result, enginesQueried int) []Cluster { } type clusterAccum struct { - occurrences []ClusterOccurrence - scoreSum float64 - bestRank int - title string + occurrences []ClusterOccurrence + scoreSum float64 + bestRank int + title string canonicalURL string - domain string + domain string } // Group by result ID (which is derived from normalized URL + engine). @@ -95,8 +95,8 @@ func BuildClusters(results []Result, enginesQueried int) []Cluster { } func buildClusterID(normalizedURL string) string { - h := sha256.Sum256([]byte(normalizedURL)) - return "c_" + hex.EncodeToString(h[:12]) + h := md5.Sum([]byte(normalizedURL)) + return "c_" + hex.EncodeToString(h[:responseIDBytes]) } func roundScore(s float64) float64 { diff --git a/core/enrichment_domain.go b/core/enrichment_domain.go index 3e933c4..e1dedd9 100644 --- a/core/enrichment_domain.go +++ b/core/enrichment_domain.go @@ -1,7 +1,37 @@ package core import ( + _ "embed" + "os" "strings" + "sync" + + "golang.org/x/net/publicsuffix" + "gopkg.in/yaml.v3" +) + +//go:embed enrichment_domains.yaml +var defaultEnrichmentDomainsYAML []byte + +type enrichmentDomainsFile struct { + DomainSourceHints map[string]string `yaml:"domain_source_hints"` + NewsDomains []string `yaml:"news_domains"` + ForumDomains []string `yaml:"forum_domains"` + MarketplaceDomains []string `yaml:"marketplace_domains"` + SocialDomains []string `yaml:"social_domains"` +} + +type enrichmentDomainsConfig struct { + DomainSourceHints map[string]string + NewsDomains map[string]bool + ForumDomains map[string]bool + MarketplaceDomains map[string]bool + SocialDomains map[string]bool +} + +var ( + enrichmentDomainsOnce sync.Once + enrichmentDomains enrichmentDomainsConfig ) // EnrichDomainInfo derives TLD/category signals from a bare hostname. @@ -10,7 +40,9 @@ func EnrichDomainInfo(domain string) *DomainInfo { return nil } + domain = normalizeDomain(domain) tld, sld := splitDomain(domain) + cfg := loadEnrichmentDomains() info := &DomainInfo{ TLD: tld, @@ -18,16 +50,16 @@ func EnrichDomainInfo(domain string) *DomainInfo { IsGov: isGovTLD(domain, tld), IsEdu: isEduTLD(domain, tld), IsMil: isMilTLD(tld), - IsNews: newsDomains[domain], - IsForum: forumDomains[domain], - IsMarketplace: marketplaceDomains[domain], - IsSocial: socialDomains[domain], + IsNews: cfg.NewsDomains[domain], + IsForum: cfg.ForumDomains[domain], + IsMarketplace: cfg.MarketplaceDomains[domain], + IsSocial: cfg.SocialDomains[domain], } return info } // ClassifyURL returns a rough content-type and source hint derived from the -// URL path alone — no network calls. +// URL path alone; no network calls. func ClassifyURL(rawURL, domain string) *Classification { if rawURL == "" && domain == "" { return nil @@ -42,46 +74,42 @@ func ClassifyURL(rawURL, domain string) *Classification { } } -// splitDomain returns (tld, sld) for a bare hostname. -// Uses a simple heuristic: last label is TLD, second-to-last is SLD. -// For compound TLDs like co.uk the full suffix is returned as TLD. +// splitDomain returns (public suffix, registrable domain label). func splitDomain(domain string) (tld, sld string) { - parts := strings.Split(domain, ".") - if len(parts) < 2 { - return domain, "" + domain = normalizeDomain(domain) + if domain == "" { + return "", "" } - // Known compound TLDs. - compoundTLDs := map[string]bool{ - "co.uk": true, "co.jp": true, "co.in": true, "co.nz": true, - "co.za": true, "com.au": true, "com.br": true, "com.mx": true, - "gov.uk": true, "ac.uk": true, "edu.au": true, "gov.au": true, - "or.jp": true, "ne.jp": true, - } - if len(parts) >= 3 { - compound := parts[len(parts)-2] + "." + parts[len(parts)-1] - if compoundTLDs[compound] { - return compound, parts[len(parts)-3] + + suffix, icann := publicsuffix.PublicSuffix(domain) + if suffix == "" || !icann { + parts := strings.Split(domain, ".") + if len(parts) < 2 { + return domain, "" } + return parts[len(parts)-1], parts[len(parts)-2] } - return parts[len(parts)-1], parts[len(parts)-2] + + registrable, err := publicsuffix.EffectiveTLDPlusOne(domain) + if err != nil { + parts := strings.Split(domain, ".") + if len(parts) < 2 { + return suffix, "" + } + return suffix, parts[len(parts)-2] + } + + sld = strings.TrimSuffix(registrable, "."+suffix) + return suffix, sld } func isGovTLD(domain, tld string) bool { - if tld == "gov" || tld == "gov.uk" || tld == "gov.au" { - return true - } - return strings.HasSuffix(domain, ".gov") || - strings.HasSuffix(domain, ".gov.uk") || - strings.HasSuffix(domain, ".gov.au") + return tld == "gov" || strings.HasSuffix(tld, ".gov") || strings.HasSuffix(domain, ".gov") } func isEduTLD(domain, tld string) bool { - if tld == "edu" || tld == "ac.uk" || tld == "edu.au" { - return true - } - return strings.HasSuffix(domain, ".edu") || - strings.HasSuffix(domain, ".ac.uk") || - strings.HasSuffix(domain, ".edu.au") + return tld == "edu" || strings.HasSuffix(tld, ".edu") || tld == "ac.uk" || + strings.HasSuffix(domain, ".edu") || strings.HasSuffix(domain, ".ac.uk") } func isMilTLD(tld string) bool { @@ -111,81 +139,65 @@ func classifyContentType(rawURL string) string { } func classifySourceHint(domain string) string { - if hint, ok := domainSourceHints[domain]; ok { + cfg := loadEnrichmentDomains() + if hint, ok := cfg.DomainSourceHints[normalizeDomain(domain)]; ok { return hint } return "" } -// domainSourceHints maps known domains to a descriptive source hint. -var domainSourceHints = map[string]string{ - "wikipedia.org": "encyclopedia", - "en.wikipedia.org": "encyclopedia", - "github.com": "code_repository", - "gitlab.com": "code_repository", - "stackoverflow.com": "qa_forum", - "stackexchange.com": "qa_forum", - "reddit.com": "social_forum", - "nytimes.com": "news", - "bbc.com": "news", - "bbc.co.uk": "news", - "reuters.com": "news", - "theguardian.com": "news", - "washingtonpost.com": "news", - "forbes.com": "news", - "techcrunch.com": "news", - "medium.com": "blog_platform", - "scholar.google.com": "academic", - "arxiv.org": "academic", - "pubmed.ncbi.nlm.nih.gov": "academic", - "amazon.com": "marketplace", - "ebay.com": "marketplace", - "etsy.com": "marketplace", - "docs.google.com": "document", - "youtube.com": "video_platform", - "vimeo.com": "video_platform", - "twitter.com": "social_media", - "x.com": "social_media", - "facebook.com": "social_media", - "linkedin.com": "professional_network", - "instagram.com": "social_media", +func loadEnrichmentDomains() enrichmentDomainsConfig { + enrichmentDomainsOnce.Do(func() { + enrichmentDomains = parseEnrichmentDomains(defaultEnrichmentDomainsYAML) + if path := strings.TrimSpace(os.Getenv("OPENSERP_ENRICHMENT_DOMAINS_FILE")); path != "" { + if data, err := os.ReadFile(path); err == nil { + enrichmentDomains = parseEnrichmentDomains(data) + } + } + }) + return enrichmentDomains } -// newsDomains is the set of known news publisher domains. -var newsDomains = map[string]bool{ - "nytimes.com": true, "bbc.com": true, "bbc.co.uk": true, - "reuters.com": true, "apnews.com": true, "theguardian.com": true, - "washingtonpost.com": true, "forbes.com": true, "techcrunch.com": true, - "wired.com": true, "bloomberg.com": true, "cnn.com": true, - "nbcnews.com": true, "cbsnews.com": true, "abcnews.go.com": true, - "foxnews.com": true, "theverge.com": true, "engadget.com": true, - "arstechnica.com": true, "zdnet.com": true, "venturebeat.com": true, - "axios.com": true, "politico.com": true, "theatlantic.com": true, - "economist.com": true, "ft.com": true, "wsj.com": true, - "usatoday.com": true, "latimes.com": true, "nypost.com": true, +func parseEnrichmentDomains(data []byte) enrichmentDomainsConfig { + cfg := enrichmentDomainsConfig{ + DomainSourceHints: map[string]string{}, + NewsDomains: map[string]bool{}, + ForumDomains: map[string]bool{}, + MarketplaceDomains: map[string]bool{}, + SocialDomains: map[string]bool{}, + } + + var file enrichmentDomainsFile + if err := yaml.Unmarshal(data, &file); err != nil { + return cfg + } + + for domain, hint := range file.DomainSourceHints { + domain = normalizeDomain(domain) + hint = strings.TrimSpace(hint) + if domain != "" && hint != "" { + cfg.DomainSourceHints[domain] = hint + } + } + fillDomainSet(cfg.NewsDomains, file.NewsDomains) + fillDomainSet(cfg.ForumDomains, file.ForumDomains) + fillDomainSet(cfg.MarketplaceDomains, file.MarketplaceDomains) + fillDomainSet(cfg.SocialDomains, file.SocialDomains) + + return cfg } -// forumDomains is the set of known community/forum domains. -var forumDomains = map[string]bool{ - "reddit.com": true, "news.ycombinator.com": true, - "stackoverflow.com": true, "stackexchange.com": true, - "superuser.com": true, "serverfault.com": true, - "quora.com": true, "discourse.org": true, - "boards.4chan.org": true, "hackernews.com": true, +func fillDomainSet(dst map[string]bool, domains []string) { + for _, domain := range domains { + domain = normalizeDomain(domain) + if domain != "" { + dst[domain] = true + } + } } -// marketplaceDomains is the set of known e-commerce/marketplace domains. -var marketplaceDomains = map[string]bool{ - "amazon.com": true, "amazon.co.uk": true, "amazon.de": true, - "ebay.com": true, "etsy.com": true, "walmart.com": true, - "target.com": true, "bestbuy.com": true, "newegg.com": true, - "aliexpress.com": true, "alibaba.com": true, "shopify.com": true, -} - -// socialDomains is the set of known social media platform domains. -var socialDomains = map[string]bool{ - "twitter.com": true, "x.com": true, "facebook.com": true, - "instagram.com": true, "tiktok.com": true, "snapchat.com": true, - "pinterest.com": true, "tumblr.com": true, "linkedin.com": true, - "youtube.com": true, "twitch.tv": true, "discord.com": true, +func normalizeDomain(domain string) string { + domain = strings.ToLower(strings.TrimSpace(domain)) + domain = strings.TrimPrefix(domain, "www.") + return strings.TrimSuffix(domain, ".") } diff --git a/core/enrichment_domains.yaml b/core/enrichment_domains.yaml new file mode 100644 index 0000000..56aad43 --- /dev/null +++ b/core/enrichment_domains.yaml @@ -0,0 +1,103 @@ +domain_source_hints: + wikipedia.org: encyclopedia + en.wikipedia.org: encyclopedia + github.com: code_repository + gitlab.com: code_repository + stackoverflow.com: qa_forum + stackexchange.com: qa_forum + reddit.com: social_forum + nytimes.com: news + bbc.com: news + bbc.co.uk: news + reuters.com: news + theguardian.com: news + washingtonpost.com: news + forbes.com: news + techcrunch.com: news + medium.com: blog_platform + scholar.google.com: academic + arxiv.org: academic + pubmed.ncbi.nlm.nih.gov: academic + amazon.com: marketplace + ebay.com: marketplace + etsy.com: marketplace + docs.google.com: document + youtube.com: video_platform + vimeo.com: video_platform + twitter.com: social_media + x.com: social_media + facebook.com: social_media + linkedin.com: professional_network + instagram.com: social_media + +news_domains: + - nytimes.com + - bbc.com + - bbc.co.uk + - reuters.com + - apnews.com + - theguardian.com + - washingtonpost.com + - forbes.com + - techcrunch.com + - wired.com + - bloomberg.com + - cnn.com + - nbcnews.com + - cbsnews.com + - abcnews.go.com + - foxnews.com + - theverge.com + - engadget.com + - arstechnica.com + - zdnet.com + - venturebeat.com + - axios.com + - politico.com + - theatlantic.com + - economist.com + - ft.com + - wsj.com + - usatoday.com + - latimes.com + - nypost.com + +forum_domains: + - reddit.com + - news.ycombinator.com + - stackoverflow.com + - stackexchange.com + - superuser.com + - serverfault.com + - quora.com + - discourse.org + - boards.4chan.org + - hackernews.com + +marketplace_domains: + - amazon.com + - amazon.co.uk + - amazon.de + - ebay.com + - etsy.com + - walmart.com + - target.com + - bestbuy.com + - newegg.com + - aliexpress.com + - alibaba.com + - shopify.com + +social_domains: + - twitter.com + - x.com + - facebook.com + - instagram.com + - tiktok.com + - snapchat.com + - pinterest.com + - tumblr.com + - linkedin.com + - youtube.com + - twitch.tv + - discord.com diff --git a/core/format_markdown.go b/core/format_markdown.go index 9b003a3..343ce54 100644 --- a/core/format_markdown.go +++ b/core/format_markdown.go @@ -10,10 +10,7 @@ import ( func RenderMarkdown(env *Envelope) []byte { var b strings.Builder - enginesStr := strings.Join(env.Meta.EnginesResponded, ", ") - if enginesStr == "" { - enginesStr = strings.Join(env.Query.EnginesRequested, ", ") - } + enginesStr := strings.Join(env.Query.EnginesRequested, ", ") fmt.Fprintf(&b, "# Search results for %q\n\n", env.Query.Text) fmt.Fprintf(&b, "**Query:** %s · **Engines:** %s · **Took:** %dms\n\n", env.Query.Text, enginesStr, env.Meta.TookMs) @@ -42,10 +39,7 @@ func RenderMarkdown(env *Envelope) []byte { func RenderMarkdownImage(env *ImageEnvelope) []byte { var b strings.Builder - enginesStr := strings.Join(env.Meta.EnginesResponded, ", ") - if enginesStr == "" { - enginesStr = strings.Join(env.Query.EnginesRequested, ", ") - } + enginesStr := strings.Join(env.Query.EnginesRequested, ", ") fmt.Fprintf(&b, "# Image results for %q\n\n", env.Query.Text) fmt.Fprintf(&b, "**Query:** %s · **Engines:** %s · **Took:** %dms\n\n", env.Query.Text, enginesStr, env.Meta.TookMs) diff --git a/core/format_text.go b/core/format_text.go index 9b3cea3..8118aca 100644 --- a/core/format_text.go +++ b/core/format_text.go @@ -12,7 +12,7 @@ func RenderText(env *Envelope) []byte { var b strings.Builder fmt.Fprintf(&b, "Search: %s\n", env.Query.Text) - enginesStr := strings.Join(env.Meta.EnginesResponded, ", ") + enginesStr := strings.Join(env.Query.EnginesRequested, ", ") if enginesStr != "" { fmt.Fprintf(&b, "Engines: %s\n", enginesStr) } diff --git a/core/response.go b/core/response.go index e2f93c8..f2664a8 100644 --- a/core/response.go +++ b/core/response.go @@ -6,19 +6,16 @@ import "time" type QueryEcho struct { Text string `json:"text"` Lang string `json:"lang,omitempty"` - Location *string `json:"location"` - Device string `json:"device"` EnginesRequested []string `json:"engines_requested"` } // ResponseMeta carries request-level metadata for observability and debugging. type ResponseMeta struct { - RequestID string `json:"request_id"` - Timestamp string `json:"timestamp"` - TookMs int64 `json:"took_ms"` - EnginesResponded []string `json:"engines_responded"` - EnginesFailed []string `json:"engines_failed"` - Version string `json:"version"` + RequestID string `json:"request_id"` + RequestedAt string `json:"requested_at"` + TookMs int64 `json:"took_ms"` + EnginesFailed []string `json:"engines_failed"` + Version string `json:"version"` } // Pagination carries cursor information for client-side loop termination. @@ -60,10 +57,10 @@ type ClusterOccurrence struct { // ImageEnvelope is the top-level v1 response wrapper for image search endpoints. type ImageEnvelope struct { - Query QueryEcho `json:"query"` - Meta ResponseMeta `json:"meta"` + Query QueryEcho `json:"query"` + Meta ResponseMeta `json:"meta"` Results []ImageResult `json:"results"` - Pagination Pagination `json:"pagination"` + Pagination Pagination `json:"pagination"` } const apiVersion = "1.0" @@ -75,16 +72,13 @@ func NewEnvelope(q Query, requestID string, startedAt time.Time, engines []strin Query: QueryEcho{ Text: q.Text, Lang: q.LangCode, - Location: nil, - Device: "desktop", EnginesRequested: engines, }, Meta: ResponseMeta{ - RequestID: requestID, - Timestamp: startedAt.UTC().Format(time.RFC3339), - EnginesResponded: []string{}, - EnginesFailed: []string{}, - Version: apiVersion, + RequestID: requestID, + RequestedAt: startedAt.UTC().Format(time.RFC3339), + EnginesFailed: []string{}, + Version: apiVersion, }, Results: []Result{}, Pagination: Pagination{}, @@ -97,16 +91,13 @@ func NewImageEnvelope(q Query, requestID string, startedAt time.Time, engines [] Query: QueryEcho{ Text: q.Text, Lang: q.LangCode, - Location: nil, - Device: "desktop", EnginesRequested: engines, }, Meta: ResponseMeta{ - RequestID: requestID, - Timestamp: startedAt.UTC().Format(time.RFC3339), - EnginesResponded: []string{}, - EnginesFailed: []string{}, - Version: apiVersion, + RequestID: requestID, + RequestedAt: startedAt.UTC().Format(time.RFC3339), + EnginesFailed: []string{}, + Version: apiVersion, }, Results: []ImageResult{}, Pagination: Pagination{}, diff --git a/core/response_builder.go b/core/response_builder.go index d9c23a8..df483cb 100644 --- a/core/response_builder.go +++ b/core/response_builder.go @@ -1,14 +1,23 @@ package core import ( - "crypto/sha256" + "crypto/md5" "encoding/base64" "encoding/hex" "fmt" "net/url" + "regexp" + "strconv" "strings" ) +const responseIDBytes = 8 + +var imageDimensionPatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?i)height:\s*(\d+),\s*width:\s*(\d+)`), + regexp.MustCompile(`(?i)\b(\d+)x(\d+)\b`), +} + // EnrichContext carries request-scoped values needed to enrich a raw result. type EnrichContext struct { Engine string @@ -38,12 +47,8 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result { if limit <= 0 { limit = 25 } - onPage := raw.Rank - if onPage < 0 { - onPage = 0 - } - absolute := ctx.Query.Start + onPage page := ctx.Query.Start/limit + 1 + absolute, onPage := computeResultPosition(raw.Rank, ctx.Query.Start) result := Result{ ID: buildResultID(ctx.Engine, normalizedURL), @@ -62,10 +67,6 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result { OnPage: onPage, }, Engine: ctx.Engine, - Rich: nil, - EngineMeta: map[string]any{ - "raw_rank": raw.Rank, - }, } result.DomainInfo = EnrichDomainInfo(domain) @@ -77,12 +78,15 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result { // EnrichImageResult converts a raw engine result into the v1 ImageResult shape. func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult { imageURL := normalizeURL(raw.URL) - // raw.Description may hold the page URL for image results in some engines. - pageURL := raw.Description + meta := parseImageDescription(raw.Description) + + pageURL := meta.PageURL if pageURL == "" { pageURL = imageURL } + pageURL = normalizeURL(pageURL) sourceDomain := extractDomain(pageURL) + imageWidth, imageHeight := meta.Width, meta.Height return ImageResult{ ID: buildImageID(ctx.Engine, imageURL), @@ -90,27 +94,42 @@ func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult { Type: ResultTypeImage, Title: raw.Title, Image: ImageData{ - URL: imageURL, + URL: imageURL, + Thumbnail: meta.ThumbnailURL, + Width: imageWidth, + Height: imageHeight, }, Source: ImageSource{ PageURL: pageURL, Domain: sourceDomain, }, - Engine: ctx.Engine, - EngineMeta: map[string]any{"raw_rank": raw.Rank}, + Engine: ctx.Engine, } } -// buildResultID returns a stable "r_" ID for web results. +// buildResultID returns a stable "s_" ID for web results. func buildResultID(engine, normalizedURL string) string { - h := sha256.Sum256([]byte(engine + "|" + normalizedURL)) - return "r_" + hex.EncodeToString(h[:12]) + return "s_" + shortMD5(engine+"|"+normalizedURL) } // buildImageID returns a stable "i_" ID for image results. func buildImageID(engine, imageURL string) string { - h := sha256.Sum256([]byte(engine + "|" + imageURL)) - return "i_" + hex.EncodeToString(h[:12]) + return "i_" + shortMD5(engine+"|"+imageURL) +} + +func shortMD5(value string) string { + h := md5.Sum([]byte(value)) + return hex.EncodeToString(h[:responseIDBytes]) +} + +func computeResultPosition(rank, start int) (absolute, onPage int) { + if rank <= 0 { + return 0, 0 + } + if start > 0 && rank > start { + return rank, rank - start + } + return start + rank, rank } // normalizeURL lowercases scheme+host, strips trailing slash, and removes @@ -235,6 +254,70 @@ func decodeBase64String(s string) ([]byte, error) { return base64.StdEncoding.DecodeString(s) } +type imageDescriptionMeta struct { + PageURL string + ThumbnailURL string + Width int + Height int +} + +func parseImageDescription(desc string) imageDescriptionMeta { + meta := imageDescriptionMeta{} + trimmed := strings.TrimSpace(desc) + if trimmed == "" { + return meta + } + + lower := strings.ToLower(trimmed) + if idx := strings.Index(lower, "source page:"); idx >= 0 { + meta.PageURL = strings.TrimSpace(trimmed[idx+len("source page:"):]) + if comma := strings.Index(meta.PageURL, ","); comma >= 0 { + meta.PageURL = strings.TrimSpace(meta.PageURL[:comma]) + } + } else if strings.HasPrefix(lower, "source:") { + meta.PageURL = strings.TrimSpace(trimmed[len("source:"):]) + } + + if idx := strings.Index(lower, "thumb_url:"); idx >= 0 { + meta.ThumbnailURL = strings.TrimSpace(trimmed[idx+len("thumb_url:"):]) + if comma := strings.Index(meta.ThumbnailURL, ","); comma >= 0 { + meta.ThumbnailURL = strings.TrimSpace(meta.ThumbnailURL[:comma]) + } + } + + for _, pattern := range imageDimensionPatterns { + match := pattern.FindStringSubmatch(trimmed) + if len(match) != 3 { + continue + } + first, firstErr := strconv.Atoi(match[1]) + second, secondErr := strconv.Atoi(match[2]) + if firstErr != nil || secondErr != nil { + continue + } + if strings.Contains(strings.ToLower(match[0]), "height") { + meta.Height = first + meta.Width = second + } else { + meta.Width = first + meta.Height = second + } + break + } + + if !isHTTPURL(meta.PageURL) { + meta.PageURL = "" + } + if !isHTTPURL(meta.ThumbnailURL) { + meta.ThumbnailURL = "" + } + return meta +} + +func isHTTPURL(value string) bool { + return strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://") +} + // NormalizeURLForClustering returns a URL suitable for cross-engine grouping // (same as normalizeURL but exported for use in cluster building). func NormalizeURLForClustering(rawURL string) string { diff --git a/core/result.go b/core/result.go index afa073e..2da4c97 100644 --- a/core/result.go +++ b/core/result.go @@ -4,17 +4,17 @@ package core type ResultType string const ( - ResultTypeOrganic ResultType = "organic" - ResultTypeAd ResultType = "ad" + ResultTypeOrganic ResultType = "organic" + ResultTypeAd ResultType = "ad" ResultTypeFeaturedSnippet ResultType = "featured_snippet" - ResultTypeKnowledgePanel ResultType = "knowledge_panel" - ResultTypePeopleAlsoAsk ResultType = "people_also_ask" - ResultTypeVideo ResultType = "video" - ResultTypeImage ResultType = "image" - ResultTypeNews ResultType = "news" - ResultTypeShopping ResultType = "shopping" - ResultTypeLocal ResultType = "local" - ResultTypeAnswerBox ResultType = "answer_box" + ResultTypeKnowledgePanel ResultType = "knowledge_panel" + ResultTypePeopleAlsoAsk ResultType = "people_also_ask" + ResultTypeVideo ResultType = "video" + ResultTypeImage ResultType = "image" + ResultTypeNews ResultType = "news" + ResultTypeShopping ResultType = "shopping" + ResultTypeLocal ResultType = "local" + ResultTypeAnswerBox ResultType = "answer_box" ) // Position describes where a result sits in the overall result stream. @@ -27,21 +27,6 @@ type Position struct { OnPage int `json:"on_page"` } -// RichData is a placeholder for future structured SERP features such as star -// ratings, prices, or sitelinks. It is null in v1.0. -type RichData struct { - Stars *float64 `json:"stars,omitempty"` - Reviews *int `json:"reviews,omitempty"` - Price *string `json:"price,omitempty"` - Sitelinks []RichSitelink `json:"sitelinks,omitempty"` -} - -// RichSitelink is one navigational sub-link shown below a result. -type RichSitelink struct { - Title string `json:"title"` - URL string `json:"url"` -} - // DomainInfo carries TLD-derived category signals for a result domain. type DomainInfo struct { TLD string `json:"tld"` @@ -63,21 +48,19 @@ type Classification struct { // Result is the v1 normalized result returned in every search response. type Result struct { - ID string `json:"id"` - Rank int `json:"rank"` - Type ResultType `json:"type"` - Title string `json:"title"` - URL string `json:"url"` - DisplayURL string `json:"display_url"` - Snippet string `json:"snippet"` - Domain string `json:"domain"` - Favicon string `json:"favicon"` - IsAd bool `json:"is_ad"` - Position Position `json:"position"` - Engine string `json:"engine"` - Rich *RichData `json:"rich"` - EngineMeta map[string]any `json:"engine_meta"` - DomainInfo *DomainInfo `json:"domain_info,omitempty"` + ID string `json:"id"` + Rank int `json:"rank"` + Type ResultType `json:"type"` + Title string `json:"title"` + URL string `json:"url"` + DisplayURL string `json:"display_url"` + Snippet string `json:"snippet"` + Domain string `json:"domain"` + Favicon string `json:"favicon"` + IsAd bool `json:"is_ad"` + Position Position `json:"position"` + Engine string `json:"engine"` + DomainInfo *DomainInfo `json:"domain_info,omitempty"` Classification *Classification `json:"classification,omitempty"` } @@ -97,12 +80,11 @@ type ImageSource struct { // ImageResult is the v1 shape for image search results. type ImageResult struct { - ID string `json:"id"` - Rank int `json:"rank"` - Type ResultType `json:"type"` - Title string `json:"title"` - Image ImageData `json:"image"` - Source ImageSource `json:"source"` - Engine string `json:"engine"` - EngineMeta map[string]any `json:"engine_meta"` + ID string `json:"id"` + Rank int `json:"rank"` + Type ResultType `json:"type"` + Title string `json:"title"` + Image ImageData `json:"image"` + Source ImageSource `json:"source"` + Engine string `json:"engine"` } diff --git a/core/server.go b/core/server.go index 87b7c9e..d77071a 100644 --- a/core/server.go +++ b/core/server.go @@ -206,14 +206,17 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm WithField("action", action). Debugf("Starting %s request for query: %s", action, q.Text) - if hit, err := s.tryServeCacheHit( - c, - cacheHitCandidate{ - key: BuildCacheKey(engine.Name(), action, q), - logMessage: fmt.Sprintf("Cache hit for %s %s: %s", engine.Name(), action, q.Text), - }, - ); hit || err != nil { - return err + if format == "json" { + if hit, err := s.tryServeCacheHit( + c, + startedAt, + cacheHitCandidate{ + key: BuildCacheKey(engine.Name(), action, q), + logMessage: fmt.Sprintf("Cache hit for %s %s: %s", engine.Name(), action, q.Text), + }, + ); hit || err != nil { + return err + } } engineNames := []string{engine.Name()} @@ -237,7 +240,9 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm } env := NewImageEnvelope(q, requestID, startedAt, engineNames) - env.Meta.EnginesResponded = []string{usedEngine} + if usedEngine != "" && usedEngine != engine.Name() { + env.Meta.EnginesFailed = []string{engine.Name()} + } ectx := EnrichContext{Engine: usedEngine, Query: q} for _, r := range res { env.Results = append(env.Results, EnrichImageResult(r, ectx)) @@ -276,7 +281,9 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm } env := NewEnvelope(q, requestID, startedAt, engineNames) - env.Meta.EnginesResponded = []string{usedEngine} + if usedEngine != "" && usedEngine != engine.Name() { + env.Meta.EnginesFailed = []string{engine.Name()} + } ectx := EnrichContext{Engine: usedEngine, Query: q} for _, r := range res { env.Results = append(env.Results, EnrichResult(r, ectx)) @@ -687,15 +694,16 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string, run func(contex logMessage: fmt.Sprintf("Cache hit for mega %s partial set: engines=%s query=%s", action, engineNamesJoined, q.Text), }) } - if hit, err := s.tryServeCacheHit(c, cacheHitCandidates...); hit || err != nil { - return err + if format == "json" { + if hit, err := s.tryServeCacheHit(c, startedAt, cacheHitCandidates...); hit || err != nil { + return err + } } - rawResults, enginesResponded, enginesFailed := run(requestCtx, q, enginesToUse) + rawResults, _, enginesFailed := run(requestCtx, q, enginesToUse) if action == "image" { env := NewImageEnvelope(q, requestID, startedAt, engineNames) - env.Meta.EnginesResponded = enginesResponded env.Meta.EnginesFailed = enginesFailed for _, r := range rawResults { ectx := EnrichContext{Engine: r.Engine, Query: q} @@ -724,7 +732,6 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string, run func(contex // Deduplicate the flat results list by normalized URL (keep best-ranked occurrence). dedupedRaw := s.deduplicateMegaResults(rawResults) env := NewEnvelope(q, requestID, startedAt, engineNames) - env.Meta.EnginesResponded = enginesResponded env.Meta.EnginesFailed = enginesFailed for _, r := range dedupedRaw { ectx := EnrichContext{Engine: r.Engine, Query: q} @@ -800,33 +807,60 @@ func (s *Server) resolveEngines(enginesParam string) []SearchEngine { func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearchResult { urlMap := make(map[string]MegaSearchResult) + order := []string{} for _, result := range results { if result.URL == "" { continue } - if _, exists := urlMap[result.URL]; !exists { - urlMap[result.URL] = result + key := NormalizeURLForClustering(result.URL) + if key == "" { + continue + } + existing, exists := urlMap[key] + if !exists { + urlMap[key] = result + order = append(order, key) + continue + } + if betterMegaResult(result, existing) { + urlMap[key] = result } } - var deduped []MegaSearchResult - for _, result := range urlMap { - deduped = append(deduped, result) + deduped := make([]MegaSearchResult, 0, len(urlMap)) + for _, key := range order { + deduped = append(deduped, urlMap[key]) } sort.Slice(deduped, func(i, j int) bool { - return deduped[i].Rank < deduped[j].Rank + if deduped[i].Rank != deduped[j].Rank { + return deduped[i].Rank < deduped[j].Rank + } + if deduped[i].Engine != deduped[j].Engine { + return deduped[i].Engine < deduped[j].Engine + } + return NormalizeURLForClustering(deduped[i].URL) < NormalizeURLForClustering(deduped[j].URL) }) return deduped } +func betterMegaResult(candidate, current MegaSearchResult) bool { + if candidate.Rank > 0 && (current.Rank <= 0 || candidate.Rank < current.Rank) { + return true + } + if candidate.Rank == current.Rank && candidate.Engine < current.Engine { + return true + } + return false +} + type cacheHitCandidate struct { key string logMessage string } -func (s *Server) tryServeCacheHit(c *fiber.Ctx, candidates ...cacheHitCandidate) (bool, error) { +func (s *Server) tryServeCacheHit(c *fiber.Ctx, startedAt time.Time, candidates ...cacheHitCandidate) (bool, error) { if s.cache == nil { return false, nil } @@ -835,6 +869,7 @@ func (s *Server) tryServeCacheHit(c *fiber.Ctx, candidates ...cacheHitCandidate) if !ok { continue } + cached = refreshCachedMeta(cached, RequestIDFromContext(c.UserContext()), startedAt) c.Set("Content-Type", "application/json") c.Set("X-Cache", "HIT") WithRequest(c.UserContext()).Debug(candidate.logMessage) @@ -843,6 +878,27 @@ func (s *Server) tryServeCacheHit(c *fiber.Ctx, candidates ...cacheHitCandidate) return false, nil } +func refreshCachedMeta(data []byte, requestID string, startedAt time.Time) []byte { + var payload map[string]any + if err := json.Unmarshal(data, &payload); err != nil { + return data + } + meta, ok := payload["meta"].(map[string]any) + if !ok { + return data + } + meta["request_id"] = requestID + meta["requested_at"] = startedAt.UTC().Format(time.RFC3339) + delete(meta, "timestamp") + meta["took_ms"] = time.Since(startedAt).Milliseconds() + + refreshed, err := json.Marshal(payload) + if err != nil { + return data + } + return refreshed +} + func (s *Server) cacheJSON(cacheKey string, payload interface{}) bool { if s.cache == nil { return false diff --git a/core/server_test.go b/core/server_test.go index 9c557f8..537fe04 100644 --- a/core/server_test.go +++ b/core/server_test.go @@ -645,6 +645,13 @@ func TestDedicatedEndpointFallbackBypassesCache(t *testing.T) { if got := first.Header.Get("X-Fallback-Engine"); got != "yandex" { t.Fatalf("expected fallback engine header, got %q", got) } + var env Envelope + if err := json.NewDecoder(first.Body).Decode(&env); err != nil { + t.Fatalf("decode fallback envelope: %v", err) + } + if len(env.Meta.EnginesFailed) != 1 || env.Meta.EnginesFailed[0] != "google" { + t.Fatalf("expected primary engine in engines_failed, got %v", env.Meta.EnginesFailed) + } second := request(t, srv, "/google/search?text=golang") if second.StatusCode != http.StatusOK { @@ -1653,6 +1660,172 @@ func TestFormatParamReturnsCorrectContentType(t *testing.T) { } } +func TestFormatParamBypassesJSONCache(t *testing.T) { + engine := &engineMock{name: "google", initialized: true} + opts := DefaultServerOptions() + opts.Resilience.Retry.MaxRetries = 0 + srv := NewServerWithOptions("127.0.0.1", 7205, opts, engine) + + first := request(t, srv, "/google/search?text=cache-format&format=json") + if first.StatusCode != http.StatusOK { + t.Fatalf("expected first request 200, got %d", first.StatusCode) + } + + second := request(t, srv, "/google/search?text=cache-format&format=markdown") + if second.StatusCode != http.StatusOK { + t.Fatalf("expected second request 200, got %d", second.StatusCode) + } + if ct := second.Header.Get("Content-Type"); !strings.Contains(ct, "text/markdown") { + t.Fatalf("expected markdown content type, got %q", ct) + } + body, _ := io.ReadAll(second.Body) + if !strings.Contains(string(body), "# Search results") { + t.Fatalf("expected markdown body, got %q", string(body)) + } + + engine.mu.Lock() + searchCalls := engine.searchCalls + engine.mu.Unlock() + if searchCalls != 2 { + t.Fatalf("expected markdown request to bypass JSON cache, got %d search calls", searchCalls) + } +} + +func TestCachedEnvelopeRefreshesRequestID(t *testing.T) { + engine := &engineMock{name: "google", initialized: true} + opts := DefaultServerOptions() + opts.Resilience.Retry.MaxRetries = 0 + srv := NewServerWithOptions("127.0.0.1", 7206, opts, engine) + + first := requestWithHeader(t, srv, "/google/search?text=cache-id", "X-Request-ID", "req-first") + if first.StatusCode != http.StatusOK { + t.Fatalf("expected first request 200, got %d", first.StatusCode) + } + second := requestWithHeader(t, srv, "/google/search?text=cache-id", "X-Request-ID", "req-second") + if second.StatusCode != http.StatusOK { + t.Fatalf("expected second request 200, got %d", second.StatusCode) + } + if got := second.Header.Get("X-Cache"); got != "HIT" { + t.Fatalf("expected cache hit, got %q", got) + } + + var env Envelope + if err := json.NewDecoder(second.Body).Decode(&env); err != nil { + t.Fatalf("decode envelope: %v", err) + } + if env.Meta.RequestID != "req-second" { + t.Fatalf("expected refreshed meta.request_id, got %q", env.Meta.RequestID) + } + if got := second.Header.Get("X-Request-ID"); got != env.Meta.RequestID { + t.Fatalf("expected header/body request IDs to match, header=%q body=%q", got, env.Meta.RequestID) + } +} + +func TestPaginatedPositionUsesAbsoluteRank(t *testing.T) { + engine := &engineMock{ + name: "google", + initialized: true, + searchFn: func(_ context.Context, q Query) ([]SearchResult, error) { + return []SearchResult{{Rank: 11, URL: "https://example.com/page", Title: "Page"}}, nil + }, + } + opts := DefaultServerOptions() + opts.Resilience.Retry.MaxRetries = 0 + srv := NewServerWithOptions("127.0.0.1", 7207, opts, engine) + + resp := request(t, srv, "/google/search?text=page&start=10&limit=10") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var env Envelope + if err := json.NewDecoder(resp.Body).Decode(&env); err != nil { + t.Fatalf("decode envelope: %v", err) + } + if len(env.Results) != 1 { + t.Fatalf("expected one result, got %d", len(env.Results)) + } + pos := env.Results[0].Position + if pos.Absolute != 11 || pos.OnPage != 1 || pos.Page != 2 { + t.Fatalf("unexpected position: %+v", pos) + } +} + +func TestImageEnvelopeEnrichesMetadataFromDescription(t *testing.T) { + engine := &engineMock{ + name: "google", + initialized: true, + imageFn: func(_ context.Context, q Query) ([]SearchResult, error) { + return []SearchResult{{ + Rank: 1, + URL: "https://cdn.example.com/image.jpg", + Title: "Image", + Description: "Height:800, Width:1200, Source Page: https://example.com/article", + }}, nil + }, + } + opts := DefaultServerOptions() + opts.Resilience.Retry.MaxRetries = 0 + srv := NewServerWithOptions("127.0.0.1", 7208, opts, engine) + + resp := request(t, srv, "/google/image?text=image") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var env ImageEnvelope + if err := json.NewDecoder(resp.Body).Decode(&env); err != nil { + t.Fatalf("decode envelope: %v", err) + } + if len(env.Results) != 1 { + t.Fatalf("expected one image result, got %d", len(env.Results)) + } + got := env.Results[0] + if got.Image.Width != 1200 || got.Image.Height != 800 { + t.Fatalf("expected image dimensions 1200x800, got %dx%d", got.Image.Width, got.Image.Height) + } + if got.Source.PageURL != "https://example.com/article" || got.Source.Domain != "example.com" { + t.Fatalf("unexpected source: %+v", got.Source) + } +} + +func TestMegaSearchDeduplicatesByNormalizedURLDeterministically(t *testing.T) { + google := &engineMock{ + name: "google", + initialized: true, + searchFn: func(_ context.Context, q Query) ([]SearchResult, error) { + return []SearchResult{{Rank: 2, URL: "https://example.com/page?utm_source=test", Title: "Google"}}, nil + }, + } + bing := &engineMock{ + name: "bing", + initialized: true, + searchFn: func(_ context.Context, q Query) ([]SearchResult, error) { + return []SearchResult{{Rank: 1, URL: "https://example.com/page", Title: "Bing"}}, nil + }, + } + + opts := DefaultServerOptions() + opts.Resilience.Retry.MaxRetries = 0 + srv := NewServerWithOptions("127.0.0.1", 7209, opts, google, bing) + + resp := request(t, srv, "/mega/search?text=dedupe&engines=google,bing") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var env Envelope + if err := json.NewDecoder(resp.Body).Decode(&env); err != nil { + t.Fatalf("decode envelope: %v", err) + } + if len(env.Results) != 1 { + t.Fatalf("expected one deduped result, got %d", len(env.Results)) + } + if env.Results[0].Engine != "bing" || env.Results[0].Rank != 1 { + t.Fatalf("expected best-ranked bing result, got engine=%q rank=%d", env.Results[0].Engine, env.Results[0].Rank) + } +} + func min(a, b int) int { if a < b { return a diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 150fa8d..b14a47a 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -74,18 +74,15 @@ paths: query: text: golang lang: EN - location: null - device: desktop engines_requested: [google] meta: request_id: 01HXYZ... - timestamp: "2026-04-24T12:00:00Z" + requested_at: "2026-04-24T12:00:00Z" took_ms: 842 - engines_responded: [google] engines_failed: [] version: "1.0" results: - - id: r_a1b2c3d4e5f6a1b2c3d4e5f6 + - id: s_a1b2c3d4e5f6a1b2 rank: 1 type: organic title: The Go Programming Language @@ -100,9 +97,6 @@ paths: page: 1 on_page: 1 engine: google - rich: null - engine_meta: - raw_rank: 1 domain_info: tld: dev sld: go @@ -600,7 +594,7 @@ components: # ── v1 envelope ────────────────────────────────────────────────── QueryEcho: type: object - required: [text, location, device, engines_requested] + required: [text, engines_requested] properties: text: type: string @@ -608,12 +602,6 @@ components: lang: type: string example: EN - location: - type: string - nullable: true - device: - type: string - example: desktop engines_requested: type: array items: @@ -621,23 +609,18 @@ components: example: [google] ResponseMeta: type: object - required: [request_id, timestamp, took_ms, engines_responded, engines_failed, version] + required: [request_id, requested_at, took_ms, engines_failed, version] properties: request_id: type: string example: 01HXYZ... - timestamp: + requested_at: type: string format: date-time example: "2026-04-24T12:00:00Z" took_ms: type: integer example: 842 - engines_responded: - type: array - items: - type: string - example: [google, bing] engines_failed: type: array items: @@ -743,16 +726,14 @@ components: - is_ad - position - engine - - rich - - engine_meta properties: id: type: string description: > - Stable identifier: `r_` + hex(first 12 bytes of SHA-256(engine|normalized_url)). + Stable identifier: `s_` + hex(first 8 bytes of MD5(engine|normalized_url)). Normalized URL: lowercase scheme+host, trailing slash stripped, utm_*/fbclid/gclid tracking params removed. Same URL → same ID across requests. - example: r_a1b2c3d4e5f6a1b2c3d4e5f6 + example: s_a1b2c3d4e5f6a1b2 rank: type: integer example: 1 @@ -787,16 +768,6 @@ components: engine: type: string example: google - rich: - type: object - nullable: true - description: Reserved for future structured SERP data (stars, price, sitelinks). - engine_meta: - type: object - additionalProperties: true - description: Engine-specific data. Always contains `raw_rank`. - example: - raw_rank: 1 domain_info: $ref: "#/components/schemas/DomainInfo" classification: @@ -830,12 +801,12 @@ components: example: example.com ImageResult: type: object - required: [id, rank, type, title, image, source, engine, engine_meta] + required: [id, rank, type, title, image, source, engine] properties: id: type: string description: Stable identifier prefixed with `i_`. - example: i_a1b2c3d4e5f6a1b2c3d4e5f6 + example: i_a1b2c3d4e5f6a1b2 rank: type: integer example: 1 @@ -850,9 +821,6 @@ components: $ref: "#/components/schemas/ImageSource" engine: type: string - engine_meta: - type: object - additionalProperties: true # ── Clusters (mega only) ────────────────────────────────────────── ClusterOccurrence: type: object @@ -866,7 +834,7 @@ components: example: 1 result_id: type: string - example: r_a1b2c3d4e5f6a1b2c3d4e5f6 + example: s_a1b2c3d4e5f6a1b2 Cluster: type: object required: [id, canonical_url, domain, title, occurrences, engines_count, best_rank, score] @@ -874,8 +842,8 @@ components: id: type: string description: > - Stable identifier: `c_` + hex(first 12 bytes of SHA-256(normalized_url)). - example: c_a1b2c3d4e5f6a1b2c3d4e5f6 + Stable identifier: `c_` + hex(first 8 bytes of MD5(normalized_url)). + example: c_a1b2c3d4e5f6a1b2 canonical_url: type: string example: https://go.dev/ diff --git a/go.mod b/go.mod index 655e1e0..f68a87a 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/ysmood/gson v0.7.3 golang.org/x/net v0.43.0 golang.org/x/time v0.12.0 + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -50,5 +51,4 @@ require ( golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.28.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/yandex/search.go b/yandex/search.go index 1800681..c609137 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -311,7 +311,7 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S Rank: img.Rank + 1, URL: img.OrigURL, Title: img.Title, - Description: fmt.Sprintf("%dx%d, freshness:%s, thumb_url:%s", img.Height, img.Width, img.Freshness, img.ThumbURL), + Description: fmt.Sprintf("%dx%d, freshness:%s, thumb_url:%s", img.Width, img.Height, img.Freshness, img.ThumbURL), } searchResults = append(searchResults, res)