diff --git a/README.md b/README.md index 3b2d729..f157213 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Common parameters: | `date` | Date range | `20250101..20251231` | | `file` | File extension | `pdf`, `doc`, `xls` | | `site` | Site-specific search | `github.com` | -| `limit` | Number of results, max 100 | `10`, `25`, `50` | +| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` | | `start` | Pagination offset | `0`, `10`, `20` | | `format` | Output format | `json`, `markdown`, `text`, `ndjson` | @@ -130,7 +130,7 @@ Engine-specific parameters: "requested_at": "2026-04-25T22:27:52Z", "took_ms": 6410, "engines_failed": [], - "version": "1.0" + "version": "2.0" }, "results": [ { @@ -143,22 +143,14 @@ Engine-specific parameters: "snippet": "Official Go documentation, tutorials, references, and release notes.", "domain": "go.dev", "favicon": "https://go.dev/favicon.ico", - "is_ad": false, "position": { - "absolute": 1, - "page": 1, - "on_page": 1 + "absolute": 1 }, "engine": "google", "domain_info": { "tld": "dev", "sld": "go", - "is_gov": false, - "is_edu": false, - "is_social": false - }, - "classification": { - "content_type": "webpage" + "category": "" } } ], diff --git a/baidu/parse_html.go b/baidu/parse_html.go index d42cf2f..a9c0649 100644 --- a/baidu/parse_html.go +++ b/baidu/parse_html.go @@ -37,8 +37,12 @@ func baiduResultSelectors() []string { func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult { var results []core.SearchResult rank := 1 + adRank := 1 + absoluteRank := 1 sel.Each(func(_ int, item *goquery.Selection) { + isAd := baiduSelectionHasAdMarker(item) + // h3-first: organic results always carry a heading; this filters out // non-result blocks that may share the wrapper class. titleTag := item.Find("h3").First() @@ -96,21 +100,55 @@ func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult { desc = strings.TrimSpace(strings.Replace(full, title, "", 1)) } + resultRank := rank + if isAd { + resultRank = adRank + adRank++ + } else { + rank++ + } + results = append(results, core.SearchResult{ - Rank: rank, - URL: href, - Title: title, - Description: desc, + Rank: resultRank, + AbsoluteRank: absoluteRank, + URL: href, + Title: title, + Description: desc, + Ad: isAd, }) - rank++ + absoluteRank++ }) // Re-rank sequentially after dedup so callers get a clean 1..N sequence // (dedup may drop intermediate ranks when the same URL appears in // multiple Baidu result-card variants on the same SERP). deduped := core.DeduplicateResults(results) + organicIdx := 0 for i := range deduped { - deduped[i].Rank = i + 1 + if deduped[i].Ad { + continue + } + organicIdx++ + deduped[i].Rank = organicIdx } return deduped } + +func baiduSelectionHasAdMarker(item *goquery.Selection) bool { + for _, selector := range Selectors.AdMarkers { + if item.Is(selector) || item.Find(selector).Length() > 0 { + return true + } + } + + isAd := false + item.Find("span, i, em").EachWithBreak(func(_ int, marker *goquery.Selection) bool { + text := strings.TrimSpace(marker.Text()) + if text == "广告" || text == "推广" || text == "商业推广" { + isAd = true + return false + } + return true + }) + return isAd +} diff --git a/baidu/parse_html_test.go b/baidu/parse_html_test.go index 9823a4d..50331f5 100644 --- a/baidu/parse_html_test.go +++ b/baidu/parse_html_test.go @@ -104,3 +104,53 @@ func TestParseBaiduHTMLFallsBackWhenEarlierSelectorHasNoResult(t *testing.T) { t.Fatalf("unexpected URL: %s", results[0].URL) } } + +func TestParseBaiduHTMLAdsDoNotConsumeOrganicRank(t *testing.T) { + t.Parallel() + + html := ` +
+
+

Sponsored Result

+
Paid snippet
+
+
+

Organic One

+
Organic snippet one
+
+
+

Organic Two

+
Organic snippet two
+
+
` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 3 { + t.Fatalf("expected 3 results, got %d", len(results)) + } + + organicRank := 0 + adRank := 0 + for _, r := range results { + if r.Ad { + adRank++ + if r.Rank != adRank { + t.Fatalf("ad rank = %d, want %d", r.Rank, adRank) + } + continue + } + organicRank++ + if r.Rank != organicRank { + t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank) + } + } + if organicRank != 2 { + t.Fatalf("organic count = %d, want 2", organicRank) + } + if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 { + t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank) + } +} diff --git a/baidu/search.go b/baidu/search.go index 763e9b7..13cd3d6 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -112,7 +112,13 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core } for i := range searchResults { - searchResults[i].Rank = query.Start + i + 1 + if searchResults[i].AbsoluteRank > 0 { + searchResults[i].AbsoluteRank += query.Start + } + if searchResults[i].Ad { + continue + } + searchResults[i].Rank = query.Start + searchResults[i].Rank } return searchResults, nil } diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 2c020f7..26afdb0 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -69,7 +69,13 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, } if query.Start > 0 { for i := range parsedResults { - parsedResults[i].Rank = query.Start + i + 1 + if parsedResults[i].AbsoluteRank > 0 { + parsedResults[i].AbsoluteRank += query.Start + } + if parsedResults[i].Ad { + continue + } + parsedResults[i].Rank = query.Start + parsedResults[i].Rank } } core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug( diff --git a/baidu/selectors.go b/baidu/selectors.go index d22a972..5c56e93 100644 --- a/baidu/selectors.go +++ b/baidu/selectors.go @@ -6,6 +6,7 @@ var Selectors = struct { Timeout string Results string ResultsAlt []string + AdMarkers []string ImageJSONRoot []string Link string Desc string @@ -17,6 +18,7 @@ var Selectors = struct { Timeout: "button.timeout-button", Results: "#content_left div.result.c-container", ResultsAlt: []string{"#content_left div.result-op.c-container", "div.c-container.new-pmd"}, + AdMarkers: []string{"[data-tuiguang]", "[data-click*='tuiguang']", ".ec-tuiguang", ".c-icon-bear-p"}, ImageJSONRoot: []string{"body > pre", "pre"}, Link: "a", Desc: "div.c-abstract", diff --git a/bing/parse_html.go b/bing/parse_html.go index 536e92b..b7b68b9 100644 --- a/bing/parse_html.go +++ b/bing/parse_html.go @@ -22,6 +22,7 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { func parseBingDocument(doc *goquery.Document) []core.SearchResult { var results []core.SearchResult rank := 1 + adRank := 1 doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) { titleTag := item.Find(Selectors.Title).First() @@ -69,17 +70,37 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult { } results = append(results, core.SearchResult{ - Rank: -1, + Rank: adRank, URL: link, Title: title, Description: desc, Ad: true, }) + adRank++ }) + setSeparatedAdAbsoluteRanks(results, 0) return core.DeduplicateResults(results) } +func setSeparatedAdAbsoluteRanks(results []core.SearchResult, start int) { + adCount := 0 + for i := range results { + if results[i].Ad { + adCount++ + results[i].AbsoluteRank = start + results[i].Rank + } + } + organicAbsoluteRank := start + adCount + 1 + for i := range results { + if results[i].Ad { + continue + } + results[i].AbsoluteRank = organicAbsoluteRank + organicAbsoluteRank++ + } +} + // descriptionFromItem extracts a description using the same 4-step fallback // chain as the rod-based browser parser. func descriptionFromItem(item *goquery.Selection, title string) string { diff --git a/bing/search.go b/bing/search.go index 06ec701..a39b78f 100644 --- a/bing/search.go +++ b/bing/search.go @@ -138,6 +138,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements)) rank := query.Start + adRank := 1 for _, result := range organicElements { srchRes := core.SearchResult{} @@ -215,35 +216,17 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. srchRes.Description, _ = descElem.Text() } - // Mark ads with negative rank - srchRes.Rank = -1 + srchRes.Rank = adRank + adRank++ searchResults = append(searchResults, srchRes) } + setSeparatedAdAbsoluteRanks(searchResults, query.Start) + // Deduplicate results deduped := core.DeduplicateResults(searchResults) - // Trim to exact limit if necessary (only organic results, not ads) - if query.Limit > 0 { - organicResults := []core.SearchResult{} - adResults := []core.SearchResult{} - - for _, result := range deduped { - if result.Ad { - adResults = append(adResults, result) - } else { - organicResults = append(organicResults, result) - } - } - - // Trim organic results to limit - if len(organicResults) > query.Limit { - organicResults = organicResults[:query.Limit] - } - - // Combine back: organic results + ads - deduped = append(organicResults, adResults...) - } + deduped = core.LimitOrganicResults(deduped, query.Limit) return deduped, nil } diff --git a/cmd/root.go b/cmd/root.go index 2bbc746..a45a1d5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -16,7 +16,7 @@ import ( ) const ( - version = "0.7.10" + version = "0.7.11" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) diff --git a/core/common.go b/core/common.go index 1b4cd24..7236dd8 100644 --- a/core/common.go +++ b/core/common.go @@ -95,9 +95,11 @@ func classifyProxyNetworkError(err error) error { // SearchResult represents one normalized result item returned by any engine. type SearchResult struct { - // Rank is a 1-based position in engine output. Some engines use negative - // ranks for non-organic blocks such as ads or instant answers. + // Rank is the 1-based position within this result type. For SEO callers, + // organic rank must not be shifted by ads. Rank int `json:"rank"` + // AbsoluteRank is the 1-based position in the mixed SERP stream. + AbsoluteRank int `json:"absolute_rank,omitempty"` // URL is the canonical result URL. URL string `json:"url"` // Title is the result headline shown on the SERP. @@ -118,14 +120,15 @@ func DeduplicateResults(results []SearchResult) []SearchResult { if result.URL == "" { continue } - if !unique[result.URL] { - unique[result.URL] = true + key := resultDedupKey(result) + if !unique[key] { + unique[key] = true deduped = append(deduped, result) } } sort.Slice(deduped, func(i, j int) bool { - return deduped[i].Rank < deduped[j].Rank + return resultLess(deduped[i], deduped[j]) }) return deduped } @@ -140,11 +143,79 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search } sort.Slice(searchResults, func(i, j int) bool { - return searchResults[i].Rank < searchResults[j].Rank + return resultLess(searchResults[i], searchResults[j]) }) return &searchResults } +// CountOrganicResults returns the number of non-ad results in a mixed SERP. +func CountOrganicResults(results []SearchResult) int { + count := 0 + for _, result := range results { + if !result.Ad { + count++ + } + } + return count +} + +// LimitOrganicResults keeps all ads and at most limit non-ad results. +func LimitOrganicResults(results []SearchResult, limit int) []SearchResult { + if limit <= 0 { + return results + } + out := make([]SearchResult, 0, len(results)) + organicCount := 0 + for _, result := range results { + if result.Ad { + out = append(out, result) + continue + } + if organicCount >= limit { + continue + } + organicCount++ + out = append(out, result) + } + return out +} + +func resultDedupKey(result SearchResult) string { + resultType := "organic" + if result.Ad { + resultType = "ad" + } + return resultType + "\x00" + result.URL +} + +func resultLess(left, right SearchResult) bool { + leftPos := resultSortPosition(left) + rightPos := resultSortPosition(right) + if leftPos != rightPos { + return leftPos < rightPos + } + if left.Ad != right.Ad { + return left.Ad + } + if left.Rank != right.Rank { + return left.Rank < right.Rank + } + return left.URL < right.URL +} + +func resultSortPosition(result SearchResult) int { + if result.AbsoluteRank > 0 { + return result.AbsoluteRank + } + if result.Rank < 0 { + return -result.Rank + } + if result.Rank > 0 { + return result.Rank + } + return int(^uint(0) >> 1) +} + // Query holds request parameters used by HTTP handlers and search engines. // Example minimal query: Query{Text: "golang", Limit: 10}. type Query struct { @@ -168,7 +239,7 @@ type Query struct { // For Google, false includes similar results and true hides them. Filter bool // Answers enables parsing answer modules when supported by the engine. - // Such entries may be returned with negative rank values. + // Such entries may be returned with non-positive internal rank values. Answers bool // ProxyURL is a direct proxy URL used by raw HTTP search paths. ProxyURL string diff --git a/core/enrichment_domain.go b/core/enrichment_domain.go index e1dedd9..0729664 100644 --- a/core/enrichment_domain.go +++ b/core/enrichment_domain.go @@ -45,15 +45,9 @@ func EnrichDomainInfo(domain string) *DomainInfo { cfg := loadEnrichmentDomains() info := &DomainInfo{ - TLD: tld, - SLD: sld, - IsGov: isGovTLD(domain, tld), - IsEdu: isEduTLD(domain, tld), - IsMil: isMilTLD(tld), - IsNews: cfg.NewsDomains[domain], - IsForum: cfg.ForumDomains[domain], - IsMarketplace: cfg.MarketplaceDomains[domain], - IsSocial: cfg.SocialDomains[domain], + TLD: tld, + SLD: sld, + Category: domainCategory(domain, tld, cfg), } return info } @@ -67,6 +61,9 @@ func ClassifyURL(rawURL, domain string) *Classification { contentType := classifyContentType(rawURL) sourceHint := classifySourceHint(domain) + if contentType == "webpage" && sourceHint == "" { + return nil + } return &Classification{ ContentType: contentType, @@ -74,6 +71,27 @@ func ClassifyURL(rawURL, domain string) *Classification { } } +func domainCategory(domain, tld string, cfg enrichmentDomainsConfig) string { + switch { + case isGovTLD(domain, tld): + return "gov" + case isEduTLD(domain, tld): + return "edu" + case isMilTLD(tld): + return "mil" + case cfg.NewsDomains[domain]: + return "news" + case cfg.ForumDomains[domain]: + return "forum" + case cfg.MarketplaceDomains[domain]: + return "marketplace" + case cfg.SocialDomains[domain]: + return "social" + default: + return "" + } +} + // splitDomain returns (public suffix, registrable domain label). func splitDomain(domain string) (tld, sld string) { domain = normalizeDomain(domain) diff --git a/core/format_markdown.go b/core/format_markdown.go index 343ce54..eda9dfe 100644 --- a/core/format_markdown.go +++ b/core/format_markdown.go @@ -22,9 +22,6 @@ func RenderMarkdown(env *Envelope) []byte { for i, r := range env.Results { fmt.Fprintf(&b, "## %d. %s\n\n", i+1, escapeMarkdown(r.Title)) typeLabel := string(r.Type) - if r.IsAd { - typeLabel = "ad" - } fmt.Fprintf(&b, "**%s** · %s\n\n", r.DisplayURL, typeLabel) if r.Snippet != "" { fmt.Fprintf(&b, "%s\n\n", r.Snippet) diff --git a/core/response.go b/core/response.go index 4460525..2af4156 100644 --- a/core/response.go +++ b/core/response.go @@ -11,13 +11,13 @@ type QueryEcho struct { // ResponseMeta carries request-level metadata for observability and debugging. type ResponseMeta struct { - RequestID string `json:"request_id"` - RequestedAt string `json:"requested_at"` - TookMs int64 `json:"took_ms"` - EnginesResponded []string `json:"engines_responded,omitempty"` - EnginesFailed []string `json:"engines_failed"` - EngineErrors []EngineErrorDetail `json:"engine_errors,omitempty"` - Version string `json:"version"` + RequestID string `json:"request_id"` + RequestedAt string `json:"requested_at"` + TookMs int64 `json:"took_ms"` + EnginesResponded []string `json:"engines_responded,omitempty"` + EnginesFailed []string `json:"engines_failed"` + EngineErrors []EngineErrorDetail `json:"engine_errors,omitempty"` + Version string `json:"version"` } // EngineErrorDetail is a client-facing, sanitized per-engine failure summary. @@ -34,7 +34,7 @@ type Pagination struct { NextStart int `json:"next_start"` } -// Envelope is the top-level v1 response wrapper for all search endpoints. +// Envelope is the top-level v2 response wrapper for all search endpoints. type Envelope struct { Query QueryEcho `json:"query"` Meta ResponseMeta `json:"meta"` @@ -64,7 +64,7 @@ type ClusterOccurrence struct { ResultID string `json:"result_id"` } -// ImageEnvelope is the top-level v1 response wrapper for image search endpoints. +// ImageEnvelope is the top-level v2 response wrapper for image search endpoints. type ImageEnvelope struct { Query QueryEcho `json:"query"` Meta ResponseMeta `json:"meta"` @@ -72,7 +72,7 @@ type ImageEnvelope struct { Pagination Pagination `json:"pagination"` } -const apiVersion = "1.0" +const apiVersion = "2.0" // NewEnvelope builds a fresh Envelope pre-filled with query echo and an open // meta block. Call Finalize before serializing. @@ -124,11 +124,21 @@ func (e *Envelope) Finalize(startedAt time.Time, q Query) { page := q.Start/limit + 1 e.Pagination = Pagination{ Page: page, - HasMore: len(e.Results) >= limit, + HasMore: countNonAdResults(e.Results) >= limit, NextStart: q.Start + limit, } } +func countNonAdResults(results []Result) int { + count := 0 + for _, result := range results { + if result.Type != ResultTypeAd { + count++ + } + } + return count +} + // Finalize stamps the elapsed time and computes pagination fields. func (e *ImageEnvelope) Finalize(startedAt time.Time, q Query) { e.Meta.TookMs = time.Since(startedAt).Milliseconds() diff --git a/core/response_builder.go b/core/response_builder.go index df483cb..296bb40 100644 --- a/core/response_builder.go +++ b/core/response_builder.go @@ -24,7 +24,7 @@ type EnrichContext struct { Query Query } -// EnrichResult converts a raw engine result into the v1 Result shape. +// EnrichResult converts a raw engine result into the v2 Result shape. func EnrichResult(raw SearchResult, ctx EnrichContext) Result { normalizedURL := normalizeURL(raw.URL) domain := extractDomain(normalizedURL) @@ -42,17 +42,23 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result { if raw.Rank <= 0 && !raw.Ad { resultType = ResultTypeAnswerBox } - - limit := ctx.Query.Limit - if limit <= 0 { - limit = 25 + rank := raw.Rank + if rank < 0 { + if raw.Ad { + rank = -rank + } else { + rank = 0 + } + } + + absolute := computeResultPosition(raw, ctx.Query.Start) + if absolute <= 0 { + absolute = rank } - page := ctx.Query.Start/limit + 1 - absolute, onPage := computeResultPosition(raw.Rank, ctx.Query.Start) result := Result{ ID: buildResultID(ctx.Engine, normalizedURL), - Rank: raw.Rank, + Rank: rank, Type: resultType, Title: raw.Title, URL: normalizedURL, @@ -60,13 +66,10 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result { Snippet: raw.Description, Domain: domain, Favicon: favicon, - IsAd: raw.Ad, - Position: Position{ - Absolute: absolute, - Page: page, - OnPage: onPage, - }, - Engine: ctx.Engine, + Engine: ctx.Engine, + } + if absolute > 0 { + result.Position = &Position{Absolute: absolute} } result.DomainInfo = EnrichDomainInfo(domain) @@ -75,7 +78,7 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result { return result } -// EnrichImageResult converts a raw engine result into the v1 ImageResult shape. +// EnrichImageResult converts a raw engine result into the v2 ImageResult shape. func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult { imageURL := normalizeURL(raw.URL) meta := parseImageDescription(raw.Description) @@ -122,14 +125,22 @@ func shortMD5(value string) string { return hex.EncodeToString(h[:responseIDBytes]) } -func computeResultPosition(rank, start int) (absolute, onPage int) { - if rank <= 0 { - return 0, 0 +func computeResultPosition(raw SearchResult, start int) int { + if raw.AbsoluteRank > 0 { + return raw.AbsoluteRank + } + + rank := raw.Rank + if rank < 0 { + rank = -rank + } + if rank == 0 { + return 0 } if start > 0 && rank > start { - return rank, rank - start + return rank } - return start + rank, rank + return start + rank } // normalizeURL lowercases scheme+host, strips trailing slash, and removes diff --git a/core/result.go b/core/result.go index 2da4c97..e0d0a4f 100644 --- a/core/result.go +++ b/core/result.go @@ -19,34 +19,29 @@ const ( // Position describes where a result sits in the overall result stream. type Position struct { - // Absolute is the 1-based rank counting from the first result of the first page. + // Absolute is the 1-based rank counting from the first result of the first page, + // across both organic and ad blocks. Always emitted so SEO callers can plot + // rank vs. on-page position without inferring it from the result order. Absolute int `json:"absolute"` - // Page is the 1-based page number derived from start/limit. - Page int `json:"page"` - // OnPage is the 1-based rank within this page. - OnPage int `json:"on_page"` } // DomainInfo carries TLD-derived category signals for a result domain. type DomainInfo struct { - TLD string `json:"tld"` - SLD string `json:"sld"` - IsGov bool `json:"is_gov"` - IsEdu bool `json:"is_edu"` - IsMil bool `json:"is_mil"` - IsNews bool `json:"is_news"` - IsForum bool `json:"is_forum"` - IsMarketplace bool `json:"is_marketplace"` - IsSocial bool `json:"is_social"` + TLD string `json:"tld,omitempty"` + SLD string `json:"sld,omitempty"` + // Category is one of "gov", "edu", "mil", "news", "forum", "marketplace", + // "social", or "" when the domain does not match any known category. + Category string `json:"category"` } // Classification holds URL-path heuristic hints for downstream consumers. type Classification struct { - ContentType string `json:"content_type"` - SourceHint string `json:"source_hint"` + ContentType string `json:"content_type,omitempty"` + SourceHint string `json:"source_hint,omitempty"` } -// Result is the v1 normalized result returned in every search response. +// Result is the v2 normalized result returned in search responses. Optional +// fields (Position, DomainInfo, Classification) are omitted when empty. type Result struct { ID string `json:"id"` Rank int `json:"rank"` @@ -57,8 +52,7 @@ type Result struct { Snippet string `json:"snippet"` Domain string `json:"domain"` Favicon string `json:"favicon"` - IsAd bool `json:"is_ad"` - Position Position `json:"position"` + Position *Position `json:"position,omitempty"` Engine string `json:"engine"` DomainInfo *DomainInfo `json:"domain_info,omitempty"` Classification *Classification `json:"classification,omitempty"` @@ -78,7 +72,7 @@ type ImageSource struct { Domain string `json:"domain"` } -// ImageResult is the v1 shape for image search results. +// ImageResult is the v2 shape for image search results. type ImageResult struct { ID string `json:"id"` Rank int `json:"rank"` diff --git a/core/result_rank_test.go b/core/result_rank_test.go new file mode 100644 index 0000000..1a58afc --- /dev/null +++ b/core/result_rank_test.go @@ -0,0 +1,104 @@ +package core + +import ( + "testing" + "time" +) + +func TestDeduplicateResultsOrdersByAbsoluteRank(t *testing.T) { + t.Parallel() + + results := DeduplicateResults([]SearchResult{ + {Rank: 1, AbsoluteRank: 3, URL: "https://organic.example.com/one"}, + {Rank: 1, AbsoluteRank: 1, URL: "https://ads.example.com/one", Ad: true}, + {Rank: 2, AbsoluteRank: 2, URL: "https://ads.example.com/two", Ad: true}, + {Rank: 2, AbsoluteRank: 4, URL: "https://organic.example.com/two"}, + }) + + wantRanks := []int{1, 2, 1, 2} + wantAbsoluteRanks := []int{1, 2, 3, 4} + if len(results) != len(wantRanks) { + t.Fatalf("len(results) = %d, want %d", len(results), len(wantRanks)) + } + for i, want := range wantRanks { + if results[i].Rank != want { + t.Fatalf("result %d rank = %d, want %d", i, results[i].Rank, want) + } + if results[i].AbsoluteRank != wantAbsoluteRanks[i] { + t.Fatalf("result %d absolute rank = %d, want %d", i, results[i].AbsoluteRank, wantAbsoluteRanks[i]) + } + } +} + +func TestDeduplicateResultsKeepsAdAndOrganicForSameURL(t *testing.T) { + t.Parallel() + + results := DeduplicateResults([]SearchResult{ + {Rank: 1, AbsoluteRank: 1, URL: "https://example.com/page", Ad: true}, + {Rank: 1, AbsoluteRank: 2, URL: "https://example.com/page"}, + }) + + if len(results) != 2 { + t.Fatalf("len(results) = %d, want 2", len(results)) + } + if !results[0].Ad || results[1].Ad { + t.Fatalf("expected ad and organic rows to be preserved separately: %+v", results) + } +} + +func TestLimitOrganicResultsDoesNotCountAds(t *testing.T) { + t.Parallel() + + results := LimitOrganicResults([]SearchResult{ + {Rank: 1, URL: "https://ads.example.com/one", Ad: true}, + {Rank: 1, URL: "https://organic.example.com/one"}, + {Rank: 2, URL: "https://ads.example.com/two", Ad: true}, + {Rank: 2, URL: "https://organic.example.com/two"}, + }, 1) + + if len(results) != 3 { + t.Fatalf("len(results) = %d, want 3", len(results)) + } + if CountOrganicResults(results) != 1 { + t.Fatalf("organic count = %d, want 1", CountOrganicResults(results)) + } +} + +func TestEnrichResultUsesAdRankAndAbsolutePosition(t *testing.T) { + t.Parallel() + + result := EnrichResult(SearchResult{ + Rank: 1, + AbsoluteRank: 2, + URL: "https://ads.example.com/", + Title: "Ad", + Ad: true, + }, EnrichContext{Engine: "google", Query: Query{Limit: 10}}) + + if result.Rank != 1 { + t.Fatalf("rank = %d, want 1", result.Rank) + } + if result.Position == nil || result.Position.Absolute != 2 { + t.Fatalf("unexpected position: %+v", result.Position) + } + if result.Type != ResultTypeAd { + t.Fatalf("unexpected result type: %q", result.Type) + } +} + +func TestEnvelopePaginationCountsOrganicResults(t *testing.T) { + t.Parallel() + + env := NewEnvelope(Query{Text: "ads", Limit: 2}, "req-1", time.Now(), []string{"bing"}) + env.Results = []Result{ + {Rank: 1, Type: ResultTypeAd}, + {Rank: 2, Type: ResultTypeAd}, + {Rank: 1, Type: ResultTypeOrganic}, + } + + env.Finalize(time.Now(), Query{Text: "ads", Limit: 2}) + + if env.Pagination.HasMore { + t.Fatalf("has_more = true, want false when organic count is below limit: %+v", env.Pagination) + } +} diff --git a/core/server.go b/core/server.go index f1f67e9..d92cff6 100644 --- a/core/server.go +++ b/core/server.go @@ -1198,10 +1198,11 @@ func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearch if result.URL == "" { continue } - key := NormalizeURLForClustering(result.URL) - if key == "" { + normalizedURL := NormalizeURLForClustering(result.URL) + if normalizedURL == "" { continue } + key := resultDedupKey(SearchResult{URL: normalizedURL, Ad: result.Ad}) existing, exists := urlMap[key] if !exists { urlMap[key] = result @@ -1219,8 +1220,11 @@ func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearch } sort.Slice(deduped, func(i, j int) bool { - if deduped[i].Rank != deduped[j].Rank { - return deduped[i].Rank < deduped[j].Rank + if resultLess(deduped[i].SearchResult, deduped[j].SearchResult) { + return true + } + if resultLess(deduped[j].SearchResult, deduped[i].SearchResult) { + return false } if deduped[i].Engine != deduped[j].Engine { return deduped[i].Engine < deduped[j].Engine diff --git a/core/server_test.go b/core/server_test.go index 411eaff..179c870 100644 --- a/core/server_test.go +++ b/core/server_test.go @@ -2208,7 +2208,7 @@ func TestServerOptions_DisableCORSMiddleware(t *testing.T) { } } -func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) { +func TestDedicatedEndpointReturnsV2Envelope(t *testing.T) { engine := &engineMock{name: "google", initialized: true} opts := DefaultServerOptions() opts.Resilience.Retry.MaxRetries = 0 @@ -2223,8 +2223,8 @@ func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) { if err := json.NewDecoder(resp.Body).Decode(&env); err != nil { t.Fatalf("decode envelope: %v", err) } - if env.Meta.Version != "1.0" { - t.Fatalf("expected meta.version=1.0, got %q", env.Meta.Version) + if env.Meta.Version != "2.0" { + t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version) } if env.Meta.RequestID == "" { t.Fatal("expected non-empty meta.request_id") @@ -2248,15 +2248,12 @@ func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) { if r.Snippet == "" && r.Title == "" { t.Fatal("expected result to have title or snippet") } - if r.Position.Page < 1 { - t.Fatalf("expected position.page >= 1, got %d", r.Position.Page) - } if env.Pagination.Page < 1 { t.Fatalf("expected pagination.page >= 1, got %d", env.Pagination.Page) } } -func TestMegaSearchReturnsV1EnvelopeWithEnginesFailed(t *testing.T) { +func TestMegaSearchReturnsV2EnvelopeWithEnginesFailed(t *testing.T) { good := &engineMock{name: "google", initialized: true} bad := &engineMock{ name: "bing", @@ -2279,8 +2276,8 @@ func TestMegaSearchReturnsV1EnvelopeWithEnginesFailed(t *testing.T) { if err := json.NewDecoder(resp.Body).Decode(&env); err != nil { t.Fatalf("decode envelope: %v", err) } - if env.Meta.Version != "1.0" { - t.Fatalf("expected meta.version=1.0, got %q", env.Meta.Version) + if env.Meta.Version != "2.0" { + t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version) } if len(env.Meta.EnginesFailed) == 0 { t.Fatal("expected engines_failed to contain bing") @@ -2446,6 +2443,49 @@ func TestFormatParamReturnsCorrectContentType(t *testing.T) { } } +func TestJSONOmitsRedundantResultFields(t *testing.T) { + engine := &engineMock{ + name: "google", + initialized: true, + searchFn: func(_ context.Context, _ Query) ([]SearchResult, error) { + return []SearchResult{{ + Rank: 1, + AbsoluteRank: 2, + URL: "https://example.gov/page", + Title: "Gov Result", + Description: "Snippet", + }}, nil + }, + } + opts := DefaultServerOptions() + opts.Resilience.Retry.MaxRetries = 0 + srv := NewServerWithOptions("127.0.0.1", 7210, opts, engine) + + resp := request(t, srv, "/google/search?text=clean-json") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + body, _ := io.ReadAll(resp.Body) + payload := string(body) + for _, omitted := range []string{`"is_ad"`, `"on_page"`, `"is_gov"`} { + if strings.Contains(payload, omitted) { + t.Fatalf("json payload should not contain %s: %s", omitted, payload) + } + } + if strings.Contains(payload, `"position":{"absolute":2,"page"`) { + t.Fatalf("json payload should not contain per-result page: %s", payload) + } + if !strings.Contains(payload, `"type":"organic"`) { + t.Fatalf("json payload should keep result type: %s", payload) + } + if !strings.Contains(payload, `"category":"gov"`) { + t.Fatalf("json payload should collapse domain category: %s", payload) + } + if !strings.Contains(payload, `"absolute":2`) { + t.Fatalf("json payload should keep meaningful absolute position: %s", payload) + } +} + func TestFormatParamBypassesJSONCache(t *testing.T) { engine := &engineMock{name: "google", initialized: true} opts := DefaultServerOptions() @@ -2531,9 +2571,8 @@ func TestPaginatedPositionUsesAbsoluteRank(t *testing.T) { if len(env.Results) != 1 { t.Fatalf("expected one result, got %d", len(env.Results)) } - pos := env.Results[0].Position - if pos.Absolute != 11 || pos.OnPage != 1 || pos.Page != 2 { - t.Fatalf("unexpected position: %+v", pos) + if env.Results[0].Position == nil || env.Results[0].Position.Absolute != 11 { + t.Fatalf("expected absolute position 11, got %+v", env.Results[0].Position) } } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 43286c9..ee50d15 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -83,7 +83,7 @@ Engine parsers return the older internal shape: - `Description` - `Ad` -HTTP handlers convert this into the public v1 response through `core/response_builder.go`. +HTTP handlers convert this into the public v2 response through `core/response_builder.go`. ## HTTP Request Flow @@ -118,7 +118,7 @@ HTTP request ## Public API Response -JSON endpoints return a v1 envelope. +JSON endpoints return a v2 envelope. Top-level fields: @@ -192,7 +192,7 @@ Only JSON responses use the response cache. Cached JSON refreshes request-scoped `core/enrichment_domain.go` derives: -- `domain_info`: public suffix, SLD, and category booleans +- `domain_info`: public suffix, SLD, and collapsed category - `classification`: content type and known source hint Public suffix parsing uses `golang.org/x/net/publicsuffix`. diff --git a/docs/openapi.yaml b/docs/openapi.yaml index e0e9c7d..5f8b570 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1,10 +1,10 @@ openapi: 3.0.3 info: title: OpenSERP API - version: 1.0.0 + version: 2.0.0 description: > OpenSERP provides dedicated and multi-engine search endpoints for Google, Yandex, - Baidu, Bing, and DuckDuckGo. All responses are wrapped in a v1 envelope with + Baidu, Bing, and DuckDuckGo. Search responses are wrapped in a v2 envelope with query echo, metadata, normalized results, and pagination. Invalid client input returns 400 with a machine-readable `reason` code. license: @@ -90,7 +90,7 @@ paths: requested_at: "2026-04-24T12:00:00Z" took_ms: 842 engines_failed: [] - version: "1.0" + version: "2.0" results: - id: s_a1b2c3d4e5f6a1b2 rank: 1 @@ -101,25 +101,13 @@ paths: snippet: Go is an open source programming language... domain: go.dev favicon: https://go.dev/favicon.ico - is_ad: false position: absolute: 1 - page: 1 - on_page: 1 engine: google domain_info: tld: dev sld: go - is_gov: false - is_edu: false - is_mil: false - is_news: false - is_forum: false - is_marketplace: false - is_social: false - classification: - content_type: webpage - source_hint: "" + category: "" pagination: page: 1 has_more: true @@ -612,7 +600,7 @@ components: name: limit in: query required: false - description: Maximum results to return (1–100). + description: Maximum organic results to return (1–100). Ads may be returned in addition. schema: type: integer minimum: 1 @@ -690,7 +678,7 @@ components: in: query required: false description: > - Output format. `json` (default) returns the full envelope. `markdown` returns a + Output format. `json` (default) returns the envelope. `markdown` returns a Markdown document suitable for Slack/email. `text` returns a minimal plain-text block optimised for LLM context windows. `ndjson` returns one result object per line with no envelope. The `Accept` header is also checked @@ -993,7 +981,7 @@ components: schema: $ref: "#/components/schemas/ErrorResponse" schemas: - # ── v1 envelope ────────────────────────────────────────────────── + # ── v2 envelope ────────────────────────────────────────────────── QueryEcho: type: object required: [text, engines_requested] @@ -1035,7 +1023,7 @@ components: $ref: "#/components/schemas/EngineErrorDetail" version: type: string - example: "1.0" + example: "2.0" EngineErrorDetail: type: object required: [engine, error] @@ -1067,21 +1055,18 @@ components: # ── Result ─────────────────────────────────────────────────────── Position: type: object - required: [absolute, page, on_page] + required: [absolute] properties: absolute: type: integer - description: 1-based rank counting from the first result of the first page. - example: 1 - page: - type: integer - example: 1 - on_page: - type: integer - example: 1 + description: > + 1-based rank in the mixed SERP stream, across both organic and ad blocks. + Always present so SEO callers can plot rank vs. on-page position without + inferring it from result order. + example: 2 DomainInfo: type: object - required: [tld, sld, is_gov, is_edu, is_mil, is_news, is_forum, is_marketplace, is_social] + required: [category] properties: tld: type: string @@ -1089,23 +1074,12 @@ components: sld: type: string example: wikipedia - is_gov: - type: boolean - is_edu: - type: boolean - is_mil: - type: boolean - is_news: - type: boolean - is_forum: - type: boolean - is_marketplace: - type: boolean - is_social: - type: boolean + category: + type: string + description: Empty string when the domain matches no known category. + enum: ["", gov, edu, mil, news, forum, marketplace, social] Classification: type: object - required: [content_type, source_hint] properties: content_type: type: string @@ -1117,8 +1091,8 @@ components: ResultType: type: string description: > - SERP block type. New values require a minor version bump (`meta.version: "1.1"`). - In v1.0, engines emit `organic`, `ad`, or `answer_box`; other enum values are + SERP block type. New values require a minor version bump (`meta.version: "2.1"`). + In v2.0, engines emit `organic`, `ad`, or `answer_box`; other enum values are reserved for future parser upgrades. enum: - organic @@ -1144,7 +1118,6 @@ components: - snippet - domain - favicon - - is_ad - position - engine properties: @@ -1181,9 +1154,6 @@ components: type: string description: Constructed as `https://{domain}/favicon.ico`. Not probed. example: https://go.dev/favicon.ico - is_ad: - type: boolean - example: false position: $ref: "#/components/schemas/Position" engine: diff --git a/duckduckgo/parse_html.go b/duckduckgo/parse_html.go index a169170..6ef35a8 100644 --- a/duckduckgo/parse_html.go +++ b/duckduckgo/parse_html.go @@ -21,6 +21,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { func parseDDGDocument(doc *goquery.Document) []core.SearchResult { var results []core.SearchResult rank := 1 + adRank := 1 + absoluteRank := 1 resultSel := firstMatchingSelector(doc, Selectors.Results) if resultSel == "" { @@ -40,32 +42,38 @@ func parseDDGDocument(doc *goquery.Document) []core.SearchResult { desc := extractFirstText(item, Selectors.Desc) - isAd := false - for _, sel := range Selectors.AdBadge { - if item.Find(sel).Length() > 0 { - isAd = true - break - } - } + isAd := duckduckgoSelectionHasAdMarker(item) r := core.SearchResult{ - Rank: rank, - URL: href, - Title: title, - Description: desc, - Ad: isAd, + Rank: rank, + AbsoluteRank: absoluteRank, + URL: href, + Title: title, + Description: desc, + Ad: isAd, } if !isAd { rank++ } else { - r.Rank = -1 + r.Rank = adRank + adRank++ } results = append(results, r) + absoluteRank++ }) return core.DeduplicateResults(results) } +func duckduckgoSelectionHasAdMarker(item *goquery.Selection) bool { + for _, sel := range Selectors.AdBadge { + if item.Is(sel) || item.Find(sel).Length() > 0 { + return true + } + } + return false +} + // firstMatchingSelector returns the first selector from the list that matches // at least one element in the document. func firstMatchingSelector(doc *goquery.Document, selectors []string) string { diff --git a/duckduckgo/parse_html_test.go b/duckduckgo/parse_html_test.go index 7eac92f..ac2bf98 100644 --- a/duckduckgo/parse_html_test.go +++ b/duckduckgo/parse_html_test.go @@ -55,3 +55,51 @@ func TestParseDDGHTMLEmpty(t *testing.T) { t.Fatalf("expected zero results for empty HTML, got %d", len(results)) } } + +func TestParseDDGHTMLAdsDoNotConsumeOrganicRank(t *testing.T) { + t.Parallel() + + html := ` +
+

Sponsored Result

+
Paid snippet
+
+
+

Organic One

+
Organic snippet one
+
+
+

Organic Two

+
Organic snippet two
+
` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 3 { + t.Fatalf("expected 3 results, got %d", len(results)) + } + + organicRank := 0 + adRank := 0 + for _, r := range results { + if r.Ad { + adRank++ + if r.Rank != adRank { + t.Fatalf("ad rank = %d, want %d", r.Rank, adRank) + } + continue + } + organicRank++ + if r.Rank != organicRank { + t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank) + } + } + if organicRank != 2 { + t.Fatalf("organic count = %d, want 2", organicRank) + } + if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 { + t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank) + } +} diff --git a/duckduckgo/search.go b/duckduckgo/search.go index cd138e8..8cef4c3 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -69,8 +69,11 @@ func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool { func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.SearchResult { searchResults := []core.SearchResult{} + organicRank := pageNum * 10 + adRank := 1 + absoluteRank := pageNum*10 + 1 - for i, r := range results { + for _, r := range results { // Get URL - try multiple selectors var link *rod.Element var err error @@ -113,28 +116,47 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se desc := core.FirstNonEmptyText(r, Selectors.Desc...) // Check if it's an ad - isAd := false - for _, selector := range Selectors.AdBadge { - adIndicator, err := r.Element(selector) - if err == nil && adIndicator != nil { - isAd = true - break - } + isAd := duckduckgoElementHasAdMarker(r) + resultRank := 0 + if isAd { + resultRank = adRank + adRank++ + } else { + organicRank++ + resultRank = organicRank } result := core.SearchResult{ - Rank: (pageNum * 10) + (i + 1), - URL: hrefStr, - Title: title, - Description: desc, - Ad: isAd, + Rank: resultRank, + AbsoluteRank: absoluteRank, + URL: hrefStr, + Title: title, + Description: desc, + Ad: isAd, } searchResults = append(searchResults, result) + absoluteRank++ } return searchResults } +func duckduckgoElementHasAdMarker(el *rod.Element) bool { + if el == nil { + return false + } + for _, selector := range Selectors.AdBadge { + matches, err := el.Matches(selector) + if err == nil && matches { + return true + } + if adIndicator, err := el.Element(selector); err == nil && adIndicator != nil { + return true + } + } + return false +} + // Search executes a DuckDuckGo web search and returns normalized search // results. It may return core.ErrCaptcha or core.ErrSearchTimeout. func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { @@ -193,13 +215,13 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] return false, nil } - for len(allResults) < query.Limit { + for query.Limit <= 0 || core.CountOrganicResults(allResults) < query.Limit { done, err := fetchPage() if err != nil { return nil, err } searchPage++ - if done || len(allResults) >= query.Limit { + if done || (query.Limit > 0 && core.CountOrganicResults(allResults) >= query.Limit) { break } if err := core.SleepContext(ctx, ddg.pageSleep); err != nil { @@ -211,9 +233,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] deduped := core.DeduplicateResults(allResults) // Trim to exact limit if necessary - if len(deduped) > query.Limit { - deduped = deduped[:query.Limit] - } + deduped = core.LimitOrganicResults(deduped, query.Limit) ddg.logger.Info("Search completed: %d results", len(deduped)) return deduped, nil diff --git a/duckduckgo/selectors.go b/duckduckgo/selectors.go index 1de5fbb..c86d0dc 100644 --- a/duckduckgo/selectors.go +++ b/duckduckgo/selectors.go @@ -43,9 +43,13 @@ var Selectors = struct { "anomaly", }, Results: []string{ + "article[data-testid='result'], article[data-testid='ad'], li[data-layout='organic'], li[data-layout='ad'], div[data-testid='result'], div[data-testid='ad']", "article[data-testid='result']", + "article[data-testid='ad']", "li[data-layout='organic']", + "li[data-layout='ad']", "div[data-testid='result']", + "div[data-testid='ad']", "div.result", }, Title: []string{ @@ -65,6 +69,9 @@ var Selectors = struct { "h3 a", }, AdBadge: []string{ + "article[data-testid='ad']", + "li[data-layout='ad']", + "div[data-testid='ad']", "[data-testid='ad-badge']", ".ad-badge", ".result--ad", diff --git a/ecosia/parse_html.go b/ecosia/parse_html.go index 3435d3b..a363bc2 100644 --- a/ecosia/parse_html.go +++ b/ecosia/parse_html.go @@ -21,6 +21,7 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult { var results []core.SearchResult rank := 1 + adRank := 1 doc.Find(Selectors.Result).Each(func(_ int, item *goquery.Selection) { res, ok := parseEcosiaItem(item, rank, false) @@ -32,13 +33,15 @@ func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult { }) doc.Find(Selectors.Ad).Each(func(_ int, item *goquery.Selection) { - res, ok := parseEcosiaItem(item, -1, true) + res, ok := parseEcosiaItem(item, adRank, true) if !ok { return } results = append(results, res) + adRank++ }) + setSeparatedAdAbsoluteRanks(results, 0) return core.DeduplicateResults(results) } @@ -84,6 +87,24 @@ func parseEcosiaItem(item *goquery.Selection, rank int, ad bool) (core.SearchRes }, true } +func setSeparatedAdAbsoluteRanks(results []core.SearchResult, start int) { + adCount := 0 + for i := range results { + if results[i].Ad { + adCount++ + results[i].AbsoluteRank = start + results[i].Rank + } + } + organicAbsoluteRank := start + adCount + 1 + for i := range results { + if results[i].Ad { + continue + } + results[i].AbsoluteRank = organicAbsoluteRank + organicAbsoluteRank++ + } +} + // // parseEcosiaImageItem extracts a single image card from a goquery Selection. // func parseEcosiaImageItem(item *goquery.Selection, rank int) (core.SearchResult, bool) { // linkTag := item.Find(Selectors.ImageLink).First() diff --git a/ecosia/search.go b/ecosia/search.go index 8961e7e..e9fcca8 100644 --- a/ecosia/search.go +++ b/ecosia/search.go @@ -139,13 +139,13 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S }() // nextRank counts up across pages for organic results; nextAdRank counts - // down from -1 so ads keep unique, order-preserving negative ranks. + // up within sponsored results so ad rank stays separate from SEO rank. all := []core.SearchResult{} pageNum, nextRank, err := startPage(query.Start) if err != nil { return nil, err } - nextAdRank := -1 + nextAdRank := 1 // fetchPage loads one SERP page and appends parsed results. // Returns (done, error): done=true ends the outer loop without error. fetchPage := func() (bool, error) { @@ -192,13 +192,13 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S for _, r := range ads { if res, ok := e.parseResult(r, nextAdRank, true); ok { all = append(all, res) - nextAdRank-- + nextAdRank++ } } return false, nil } - for query.Limit <= 0 || len(all) < query.Limit { + for query.Limit <= 0 || core.CountOrganicResults(all) < query.Limit { done, err := fetchPage() if err != nil { return nil, err @@ -207,7 +207,7 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S if done { break } - if query.Limit > 0 && len(all) >= query.Limit { + if query.Limit > 0 && core.CountOrganicResults(all) >= query.Limit { break } if err := core.SleepContext(ctx, e.pageSleep); err != nil { @@ -215,13 +215,10 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S } } + setSeparatedAdAbsoluteRanks(all, query.Start) deduped := core.DeduplicateResults(all) if query.Limit > 0 { - organic, ads := splitAds(deduped) - if len(organic) > query.Limit { - organic = organic[:query.Limit] - } - deduped = append(organic, ads...) + deduped = core.LimitOrganicResults(deduped, query.Limit) } e.logger.Info("Search completed: %d results", len(deduped)) return deduped, nil @@ -369,14 +366,3 @@ func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []c e.logger.Info("Image search completed: %d results", len(deduped)) return deduped, nil } - -func splitAds(in []core.SearchResult) (organic, ads []core.SearchResult) { - for _, r := range in { - if r.Ad { - ads = append(ads, r) - } else { - organic = append(organic, r) - } - } - return -} diff --git a/ecosia/search_raw.go b/ecosia/search_raw.go index 780607e..c1d8de3 100644 --- a/ecosia/search_raw.go +++ b/ecosia/search_raw.go @@ -121,7 +121,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, // Ecosia paginates by page index, not result offset, so re-rank from // the page boundary rather than query.Start (off-grid offsets round down). - // Skip ads (rank<0) so organic ranks stay sequential from startRank. + // Skip ads so organic ranks stay sequential from startRank. organicIdx := 0 for i := range parsedResults { if parsedResults[i].Ad { @@ -130,6 +130,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, parsedResults[i].Rank = startRank + organicIdx organicIdx++ } + setSeparatedAdAbsoluteRanks(parsedResults, pageNum*10) core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug( fmt.Sprintf("Ecosia Raw results : %v", parsedResults), diff --git a/google/parse_html_test.go b/google/parse_html_test.go index 06da9a3..8f4c1f5 100644 --- a/google/parse_html_test.go +++ b/google/parse_html_test.go @@ -68,3 +68,53 @@ func TestParseHTMLNoResults(t *testing.T) { t.Fatalf("expected zero results, got %d", len(results)) } } + +func TestParseHTMLAdsDoNotConsumeOrganicRank(t *testing.T) { + t.Parallel() + + html := ` +
+
+

Sponsored Result

+
Paid snippet
+
+
+

Organic One

+
Organic snippet one
+
+
+

Organic Two

+
Organic snippet two
+
+
` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 3 { + t.Fatalf("expected 3 results, got %d", len(results)) + } + + organicRank := 0 + adRank := 0 + for _, r := range results { + if r.Ad { + adRank++ + if r.Rank != adRank { + t.Fatalf("ad rank = %d, want %d", r.Rank, adRank) + } + continue + } + organicRank++ + if r.Rank != organicRank { + t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank) + } + } + if organicRank != 2 { + t.Fatalf("organic count = %d, want 2", organicRank) + } + if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 { + t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank) + } +} diff --git a/google/search.go b/google/search.go index d315942..f294a42 100644 --- a/google/search.go +++ b/google/search.go @@ -236,6 +236,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor gogl.logger.Info("Found %d total results", totalResults) rank := query.Start + adRank := 1 + absoluteRank := query.Start + 1 for _, resEl := range searchResultElems { srchRes := core.SearchResult{} @@ -285,7 +287,10 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } else { srchRes.Description = strings.TrimSpace(text) } - rank += 1 + srchRes.Rank = adRank + srchRes.AbsoluteRank = absoluteRank + adRank++ + absoluteRank++ } else if isAnswerBox { // 2. Parse answer boxes @@ -406,6 +411,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor rank += 1 srchRes.Rank = rank + srchRes.AbsoluteRank = absoluteRank + absoluteRank++ searchResults = append(searchResults, srchRes) continue @@ -413,7 +420,16 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor continue } - srchRes.Rank = rank + if srchRes.Ad && srchRes.Rank == 0 { + srchRes.Rank = adRank + adRank++ + } else if !srchRes.Ad { + srchRes.Rank = rank + } + if srchRes.AbsoluteRank == 0 { + srchRes.AbsoluteRank = absoluteRank + absoluteRank++ + } searchResults = append(searchResults, srchRes) } diff --git a/google/search_raw.go b/google/search_raw.go index 39c33e4..c5258ab 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -26,6 +26,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { func parseGoogleDocument(doc *goquery.Document) []core.SearchResult { results := []core.SearchResult{} rank := 1 + adRank := 1 + absoluteRank := 1 // Use data attributes instead of class names to find results // Both old and new DOM have data-hveid and data-ved attributes @@ -55,6 +57,8 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult { titleTag := item.Find(Selectors.Title) title := titleTag.Text() + isAd := item.Is(Selectors.Ad) || item.Find(Selectors.Ad).Length() > 0 + // Find description - find div with text content after the heading // Using attribute selectors that match the description container descTag := item.Find(Selectors.DescPrimary).First() @@ -73,15 +77,25 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult { desc := descTag.Text() if link != "" && link != "#" { + resultRank := rank + if isAd { + resultRank = adRank + adRank++ + } else { + rank++ + } + result := core.SearchResult{ - Rank: rank, - URL: link, - Title: title, - Description: desc, + Rank: resultRank, + AbsoluteRank: absoluteRank, + URL: link, + Title: title, + Description: desc, + Ad: isAd, } results = append(results, result) - rank++ + absoluteRank++ } } @@ -153,8 +167,18 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, } if query.Start > 0 { + organicIdx := 0 for i := range parsedResults { - parsedResults[i].Rank = query.Start + i + 1 + if parsedResults[i].Ad { + continue + } + organicIdx++ + parsedResults[i].Rank = query.Start + organicIdx + } + for i := range parsedResults { + if parsedResults[i].AbsoluteRank > 0 { + parsedResults[i].AbsoluteRank += query.Start + } } } core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug( diff --git a/google/selectors.go b/google/selectors.go index c5c4c06..dde4ed2 100644 --- a/google/selectors.go +++ b/google/selectors.go @@ -8,6 +8,7 @@ var Selectors = struct { ResultStats string CookieBtn string Results string + Ad string Title string DescPrimary string DescFallback string @@ -24,6 +25,7 @@ var Selectors = struct { ResultStats: "div#result-stats", CookieBtn: "div[role='dialog'][aria-modal] button", Results: "div[data-hveid][data-ved]", + Ad: "div[data-text-ad], [data-text-ad]", Title: "h3", DescPrimary: "div[data-sncf='1'] div", DescFallback: "div.VwiC3b", diff --git a/yandex/parse_html.go b/yandex/parse_html.go index 896a384..2cf5e8b 100644 --- a/yandex/parse_html.go +++ b/yandex/parse_html.go @@ -2,6 +2,7 @@ package yandex import ( "io" + "net/url" "strings" "github.com/PuerkitoBio/goquery" @@ -21,6 +22,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) { func parseYandexDocument(doc *goquery.Document) []core.SearchResult { var results []core.SearchResult rank := 1 + adRank := 1 + absoluteRank := 1 doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) { // Skip blocks without a result heading (filters out non-organic blocks @@ -51,6 +54,7 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult { if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") { return } + isAd := yandexSelectionHasAdMarker(item) || yandexURLLooksAd(href) title := strings.TrimSpace(titleTag.Text()) if title == "" { @@ -67,14 +71,84 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult { } } + resultRank := rank + if isAd { + resultRank = adRank + adRank++ + } else { + rank++ + } + results = append(results, core.SearchResult{ - Rank: rank, - URL: href, - Title: title, - Description: desc, + Rank: resultRank, + AbsoluteRank: absoluteRank, + URL: href, + Title: title, + Description: desc, + Ad: isAd, }) - rank++ + absoluteRank++ }) return core.DeduplicateResults(results) } + +func yandexSelectionHasAdMarker(item *goquery.Selection) bool { + for _, selector := range Selectors.AdMarkers { + if item.Is(selector) || item.Find(selector).Length() > 0 { + return true + } + } + return false +} + +func yandexURLLooksAd(raw string) bool { + u, err := url.Parse(raw) + if err != nil { + return false + } + host := strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.") + return strings.HasPrefix(host, "yabs.yandex.") || + host == "an.yandex.ru" || + strings.HasSuffix(host, ".yandexadexchange.net") +} + +func skipOrganicResults(results []core.SearchResult, skip int) []core.SearchResult { + if skip <= 0 { + return results + } + out := results[:0] + for _, result := range results { + if !result.Ad && skip > 0 { + skip-- + continue + } + out = append(out, result) + } + return out +} + +func rebaseOrganicRanks(results []core.SearchResult, start int) { + if start <= 0 { + return + } + organicIdx := 0 + for i := range results { + if results[i].Ad { + continue + } + organicIdx++ + results[i].Rank = start + organicIdx + } +} + +func offsetAbsoluteRanks(results []core.SearchResult, start int) { + if start <= 0 { + return + } + for i := range results { + if results[i].AbsoluteRank > 0 { + results[i].AbsoluteRank += start + } + } +} diff --git a/yandex/parse_html_test.go b/yandex/parse_html_test.go index 149ee00..36c39b3 100644 --- a/yandex/parse_html_test.go +++ b/yandex/parse_html_test.go @@ -84,3 +84,83 @@ func TestParseYandexHTMLFallbackSelectors(t *testing.T) { t.Fatalf("unexpected description: %s", results[0].Description) } } + +func TestParseYandexHTMLAdsDoNotConsumeOrganicRank(t *testing.T) { + t.Parallel() + + html := ` +` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 3 { + t.Fatalf("expected 3 results, got %d", len(results)) + } + + organicRank := 0 + adRank := 0 + for _, r := range results { + if r.Ad { + adRank++ + if r.Rank != adRank { + t.Fatalf("ad rank = %d, want %d", r.Rank, adRank) + } + continue + } + organicRank++ + if r.Rank != organicRank { + t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank) + } + } + if organicRank != 2 { + t.Fatalf("organic count = %d, want 2", organicRank) + } + if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 { + t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank) + } +} + +func TestParseYandexHTMLYabsURLIsAd(t *testing.T) { + t.Parallel() + + html := ` +` + + results, err := ParseHTML(bytes.NewReader([]byte(html))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 results, got %d", len(results)) + } + if !results[0].Ad || results[0].Rank != 1 { + t.Fatalf("first result should be ad rank 1: %+v", results[0]) + } + if results[1].Ad || results[1].Rank != 1 { + t.Fatalf("second result should be organic rank 1: %+v", results[1]) + } +} diff --git a/yandex/search.go b/yandex/search.go index 14f1e2f..0c0a596 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -71,8 +71,11 @@ func (yand *Yandex) isNoResults(page *rod.Page) bool { func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.SearchResult { searchResults := []core.SearchResult{} + organicRank := pageNum * 10 + adRank := 1 + absoluteRank := pageNum*10 + 1 - for i, r := range results { + for _, r := range results { // Get URL link, err := r.Element(Selectors.Link) if err != nil { @@ -109,13 +112,48 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc desc, _ = descTag.Text() } - r := core.SearchResult{Rank: (pageNum * 10) + (i + 1), URL: linkText.String(), Title: title, Description: desc} - searchResults = append(searchResults, r) + hrefStr := linkText.String() + isAd := yandexElementHasAdMarker(r) || yandexURLLooksAd(hrefStr) + resultRank := 0 + if isAd { + resultRank = adRank + adRank++ + } else { + organicRank++ + resultRank = organicRank + } + + res := core.SearchResult{ + Rank: resultRank, + AbsoluteRank: absoluteRank, + URL: hrefStr, + Title: title, + Description: desc, + Ad: isAd, + } + searchResults = append(searchResults, res) + absoluteRank++ } return searchResults } +func yandexElementHasAdMarker(el *rod.Element) bool { + if el == nil { + return false + } + for _, selector := range Selectors.AdMarkers { + matches, err := el.Matches(selector) + if err == nil && matches { + return true + } + if child, err := el.Element(selector); err == nil && child != nil { + return true + } + } + return false +} + func (yand *Yandex) parseImageEntities(items rod.Elements) map[string]ImageEntity { entities := make(map[string]ImageEntity) for _, item := range items { @@ -193,17 +231,13 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor r := yand.parseResults(elements, searchPage) if searchPage == startPage && skipOnFirstPage > 0 { - if skipOnFirstPage >= len(r) { - r = []core.SearchResult{} - } else { - r = r[skipOnFirstPage:] - } + r = skipOrganicResults(r, skipOnFirstPage) } allResults = append(allResults, r...) return false, nil } - for len(allResults) < query.Limit { + for query.Limit <= 0 || core.CountOrganicResults(allResults) < query.Limit { done, err := fetchPage() if err != nil { return nil, err @@ -212,13 +246,16 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor if done { break } + if query.Limit > 0 && core.CountOrganicResults(allResults) >= query.Limit { + break + } if err := core.SleepContext(ctx, yand.pageSleep); err != nil { return nil, err } } yand.logger.Info("Search completed: %d results", len(allResults)) - return core.DeduplicateResults(allResults), nil + return core.LimitOrganicResults(core.DeduplicateResults(allResults), query.Limit), nil } // SearchImage executes a Yandex image search and returns normalized image diff --git a/yandex/search_raw.go b/yandex/search_raw.go index d613e3e..9af173c 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -74,17 +74,10 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult, } if skipOnFirstPage > 0 { - if skipOnFirstPage >= len(parsedResults) { - parsedResults = []core.SearchResult{} - } else { - parsedResults = parsedResults[skipOnFirstPage:] - } - } - if query.Start > 0 { - for i := range parsedResults { - parsedResults[i].Rank = query.Start + i + 1 - } + parsedResults = skipOrganicResults(parsedResults, skipOnFirstPage) } + rebaseOrganicRanks(parsedResults, query.Start) + offsetAbsoluteRanks(parsedResults, startPage*10) core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug( fmt.Sprintf("Yandex Raw results : %v", parsedResults), ) diff --git a/yandex/selectors.go b/yandex/selectors.go index 1913b94..c4fa149 100644 --- a/yandex/selectors.go +++ b/yandex/selectors.go @@ -2,9 +2,10 @@ package yandex // Selectors is the single source of truth for Yandex SERP CSS selectors. var Selectors = struct { - Captcha string - NoResults string - Results string + Captcha string + NoResults string + Results string + AdMarkers []string // LinkPrimary is preferred over a generic ; falls back to title.Closest("a") // then the first in the result block when absent. LinkPrimary string @@ -16,9 +17,18 @@ var Selectors = struct { ImageItemsAlt []string ImageStateAll string }{ - Captcha: "div.CheckboxCaptcha", - NoResults: "div.EmptySearchResults", - Results: "li[data-fast], li.serp-item", + Captcha: "div.CheckboxCaptcha", + NoResults: "div.EmptySearchResults", + Results: "li[data-fast], li.serp-item", + AdMarkers: []string{ + "[data-fast-name='direct']", + "[data-fast-name='serp-adv']", + "[data-bem*='serp-adv']", + ".serp-adv-item", + ".serp-adv__found", + "[aria-label='Реклама']", + "[title='Реклама']", + }, LinkPrimary: "a.OrganicTitle-Link", Link: "a", Title: "h2",