diff --git a/README.md b/README.md
index 3b2d729..f157213 100644
--- a/README.md
+++ b/README.md
@@ -106,7 +106,7 @@ Common parameters:
| `date` | Date range | `20250101..20251231` |
| `file` | File extension | `pdf`, `doc`, `xls` |
| `site` | Site-specific search | `github.com` |
-| `limit` | Number of results, max 100 | `10`, `25`, `50` |
+| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` |
| `start` | Pagination offset | `0`, `10`, `20` |
| `format` | Output format | `json`, `markdown`, `text`, `ndjson` |
@@ -130,7 +130,7 @@ Engine-specific parameters:
"requested_at": "2026-04-25T22:27:52Z",
"took_ms": 6410,
"engines_failed": [],
- "version": "1.0"
+ "version": "2.0"
},
"results": [
{
@@ -143,22 +143,14 @@ Engine-specific parameters:
"snippet": "Official Go documentation, tutorials, references, and release notes.",
"domain": "go.dev",
"favicon": "https://go.dev/favicon.ico",
- "is_ad": false,
"position": {
- "absolute": 1,
- "page": 1,
- "on_page": 1
+ "absolute": 1
},
"engine": "google",
"domain_info": {
"tld": "dev",
"sld": "go",
- "is_gov": false,
- "is_edu": false,
- "is_social": false
- },
- "classification": {
- "content_type": "webpage"
+ "category": ""
}
}
],
diff --git a/baidu/parse_html.go b/baidu/parse_html.go
index d42cf2f..a9c0649 100644
--- a/baidu/parse_html.go
+++ b/baidu/parse_html.go
@@ -37,8 +37,12 @@ func baiduResultSelectors() []string {
func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
var results []core.SearchResult
rank := 1
+ adRank := 1
+ absoluteRank := 1
sel.Each(func(_ int, item *goquery.Selection) {
+ isAd := baiduSelectionHasAdMarker(item)
+
// h3-first: organic results always carry a heading; this filters out
// non-result blocks that may share the wrapper class.
titleTag := item.Find("h3").First()
@@ -96,21 +100,55 @@ func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
desc = strings.TrimSpace(strings.Replace(full, title, "", 1))
}
+ resultRank := rank
+ if isAd {
+ resultRank = adRank
+ adRank++
+ } else {
+ rank++
+ }
+
results = append(results, core.SearchResult{
- Rank: rank,
- URL: href,
- Title: title,
- Description: desc,
+ Rank: resultRank,
+ AbsoluteRank: absoluteRank,
+ URL: href,
+ Title: title,
+ Description: desc,
+ Ad: isAd,
})
- rank++
+ absoluteRank++
})
// Re-rank sequentially after dedup so callers get a clean 1..N sequence
// (dedup may drop intermediate ranks when the same URL appears in
// multiple Baidu result-card variants on the same SERP).
deduped := core.DeduplicateResults(results)
+ organicIdx := 0
for i := range deduped {
- deduped[i].Rank = i + 1
+ if deduped[i].Ad {
+ continue
+ }
+ organicIdx++
+ deduped[i].Rank = organicIdx
}
return deduped
}
+
+func baiduSelectionHasAdMarker(item *goquery.Selection) bool {
+ for _, selector := range Selectors.AdMarkers {
+ if item.Is(selector) || item.Find(selector).Length() > 0 {
+ return true
+ }
+ }
+
+ isAd := false
+ item.Find("span, i, em").EachWithBreak(func(_ int, marker *goquery.Selection) bool {
+ text := strings.TrimSpace(marker.Text())
+ if text == "广告" || text == "推广" || text == "商业推广" {
+ isAd = true
+ return false
+ }
+ return true
+ })
+ return isAd
+}
diff --git a/baidu/parse_html_test.go b/baidu/parse_html_test.go
index 9823a4d..50331f5 100644
--- a/baidu/parse_html_test.go
+++ b/baidu/parse_html_test.go
@@ -104,3 +104,53 @@ func TestParseBaiduHTMLFallsBackWhenEarlierSelectorHasNoResult(t *testing.T) {
t.Fatalf("unexpected URL: %s", results[0].URL)
}
}
+
+func TestParseBaiduHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
+ t.Parallel()
+
+ html := `
+
+
+
+
+
Organic snippet one
+
+
+
+
Organic snippet two
+
+
`
+
+ results, err := ParseHTML(bytes.NewReader([]byte(html)))
+ if err != nil {
+ t.Fatalf("ParseHTML() error = %v", err)
+ }
+ if len(results) != 3 {
+ t.Fatalf("expected 3 results, got %d", len(results))
+ }
+
+ organicRank := 0
+ adRank := 0
+ for _, r := range results {
+ if r.Ad {
+ adRank++
+ if r.Rank != adRank {
+ t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
+ }
+ continue
+ }
+ organicRank++
+ if r.Rank != organicRank {
+ t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
+ }
+ }
+ if organicRank != 2 {
+ t.Fatalf("organic count = %d, want 2", organicRank)
+ }
+ if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
+ t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
+ }
+}
diff --git a/baidu/search.go b/baidu/search.go
index 763e9b7..13cd3d6 100644
--- a/baidu/search.go
+++ b/baidu/search.go
@@ -112,7 +112,13 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core
}
for i := range searchResults {
- searchResults[i].Rank = query.Start + i + 1
+ if searchResults[i].AbsoluteRank > 0 {
+ searchResults[i].AbsoluteRank += query.Start
+ }
+ if searchResults[i].Ad {
+ continue
+ }
+ searchResults[i].Rank = query.Start + searchResults[i].Rank
}
return searchResults, nil
}
diff --git a/baidu/search_raw.go b/baidu/search_raw.go
index 2c020f7..26afdb0 100644
--- a/baidu/search_raw.go
+++ b/baidu/search_raw.go
@@ -69,7 +69,13 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
}
if query.Start > 0 {
for i := range parsedResults {
- parsedResults[i].Rank = query.Start + i + 1
+ if parsedResults[i].AbsoluteRank > 0 {
+ parsedResults[i].AbsoluteRank += query.Start
+ }
+ if parsedResults[i].Ad {
+ continue
+ }
+ parsedResults[i].Rank = query.Start + parsedResults[i].Rank
}
}
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
diff --git a/baidu/selectors.go b/baidu/selectors.go
index d22a972..5c56e93 100644
--- a/baidu/selectors.go
+++ b/baidu/selectors.go
@@ -6,6 +6,7 @@ var Selectors = struct {
Timeout string
Results string
ResultsAlt []string
+ AdMarkers []string
ImageJSONRoot []string
Link string
Desc string
@@ -17,6 +18,7 @@ var Selectors = struct {
Timeout: "button.timeout-button",
Results: "#content_left div.result.c-container",
ResultsAlt: []string{"#content_left div.result-op.c-container", "div.c-container.new-pmd"},
+ AdMarkers: []string{"[data-tuiguang]", "[data-click*='tuiguang']", ".ec-tuiguang", ".c-icon-bear-p"},
ImageJSONRoot: []string{"body > pre", "pre"},
Link: "a",
Desc: "div.c-abstract",
diff --git a/bing/parse_html.go b/bing/parse_html.go
index 536e92b..b7b68b9 100644
--- a/bing/parse_html.go
+++ b/bing/parse_html.go
@@ -22,6 +22,7 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
func parseBingDocument(doc *goquery.Document) []core.SearchResult {
var results []core.SearchResult
rank := 1
+ adRank := 1
doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
titleTag := item.Find(Selectors.Title).First()
@@ -69,17 +70,37 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
}
results = append(results, core.SearchResult{
- Rank: -1,
+ Rank: adRank,
URL: link,
Title: title,
Description: desc,
Ad: true,
})
+ adRank++
})
+ setSeparatedAdAbsoluteRanks(results, 0)
return core.DeduplicateResults(results)
}
+func setSeparatedAdAbsoluteRanks(results []core.SearchResult, start int) {
+ adCount := 0
+ for i := range results {
+ if results[i].Ad {
+ adCount++
+ results[i].AbsoluteRank = start + results[i].Rank
+ }
+ }
+ organicAbsoluteRank := start + adCount + 1
+ for i := range results {
+ if results[i].Ad {
+ continue
+ }
+ results[i].AbsoluteRank = organicAbsoluteRank
+ organicAbsoluteRank++
+ }
+}
+
// descriptionFromItem extracts a description using the same 4-step fallback
// chain as the rod-based browser parser.
func descriptionFromItem(item *goquery.Selection, title string) string {
diff --git a/bing/search.go b/bing/search.go
index 06ec701..a39b78f 100644
--- a/bing/search.go
+++ b/bing/search.go
@@ -138,6 +138,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements))
rank := query.Start
+ adRank := 1
for _, result := range organicElements {
srchRes := core.SearchResult{}
@@ -215,35 +216,17 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
srchRes.Description, _ = descElem.Text()
}
- // Mark ads with negative rank
- srchRes.Rank = -1
+ srchRes.Rank = adRank
+ adRank++
searchResults = append(searchResults, srchRes)
}
+ setSeparatedAdAbsoluteRanks(searchResults, query.Start)
+
// Deduplicate results
deduped := core.DeduplicateResults(searchResults)
- // Trim to exact limit if necessary (only organic results, not ads)
- if query.Limit > 0 {
- organicResults := []core.SearchResult{}
- adResults := []core.SearchResult{}
-
- for _, result := range deduped {
- if result.Ad {
- adResults = append(adResults, result)
- } else {
- organicResults = append(organicResults, result)
- }
- }
-
- // Trim organic results to limit
- if len(organicResults) > query.Limit {
- organicResults = organicResults[:query.Limit]
- }
-
- // Combine back: organic results + ads
- deduped = append(organicResults, adResults...)
- }
+ deduped = core.LimitOrganicResults(deduped, query.Limit)
return deduped, nil
}
diff --git a/cmd/root.go b/cmd/root.go
index 2bbc746..a45a1d5 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -16,7 +16,7 @@ import (
)
const (
- version = "0.7.10"
+ version = "0.7.11"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)
diff --git a/core/common.go b/core/common.go
index 1b4cd24..7236dd8 100644
--- a/core/common.go
+++ b/core/common.go
@@ -95,9 +95,11 @@ func classifyProxyNetworkError(err error) error {
// SearchResult represents one normalized result item returned by any engine.
type SearchResult struct {
- // Rank is a 1-based position in engine output. Some engines use negative
- // ranks for non-organic blocks such as ads or instant answers.
+ // Rank is the 1-based position within this result type. For SEO callers,
+ // organic rank must not be shifted by ads.
Rank int `json:"rank"`
+ // AbsoluteRank is the 1-based position in the mixed SERP stream.
+ AbsoluteRank int `json:"absolute_rank,omitempty"`
// URL is the canonical result URL.
URL string `json:"url"`
// Title is the result headline shown on the SERP.
@@ -118,14 +120,15 @@ func DeduplicateResults(results []SearchResult) []SearchResult {
if result.URL == "" {
continue
}
- if !unique[result.URL] {
- unique[result.URL] = true
+ key := resultDedupKey(result)
+ if !unique[key] {
+ unique[key] = true
deduped = append(deduped, result)
}
}
sort.Slice(deduped, func(i, j int) bool {
- return deduped[i].Rank < deduped[j].Rank
+ return resultLess(deduped[i], deduped[j])
})
return deduped
}
@@ -140,11 +143,79 @@ func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]Search
}
sort.Slice(searchResults, func(i, j int) bool {
- return searchResults[i].Rank < searchResults[j].Rank
+ return resultLess(searchResults[i], searchResults[j])
})
return &searchResults
}
+// CountOrganicResults returns the number of non-ad results in a mixed SERP.
+func CountOrganicResults(results []SearchResult) int {
+ count := 0
+ for _, result := range results {
+ if !result.Ad {
+ count++
+ }
+ }
+ return count
+}
+
+// LimitOrganicResults keeps all ads and at most limit non-ad results.
+func LimitOrganicResults(results []SearchResult, limit int) []SearchResult {
+ if limit <= 0 {
+ return results
+ }
+ out := make([]SearchResult, 0, len(results))
+ organicCount := 0
+ for _, result := range results {
+ if result.Ad {
+ out = append(out, result)
+ continue
+ }
+ if organicCount >= limit {
+ continue
+ }
+ organicCount++
+ out = append(out, result)
+ }
+ return out
+}
+
+func resultDedupKey(result SearchResult) string {
+ resultType := "organic"
+ if result.Ad {
+ resultType = "ad"
+ }
+ return resultType + "\x00" + result.URL
+}
+
+func resultLess(left, right SearchResult) bool {
+ leftPos := resultSortPosition(left)
+ rightPos := resultSortPosition(right)
+ if leftPos != rightPos {
+ return leftPos < rightPos
+ }
+ if left.Ad != right.Ad {
+ return left.Ad
+ }
+ if left.Rank != right.Rank {
+ return left.Rank < right.Rank
+ }
+ return left.URL < right.URL
+}
+
+func resultSortPosition(result SearchResult) int {
+ if result.AbsoluteRank > 0 {
+ return result.AbsoluteRank
+ }
+ if result.Rank < 0 {
+ return -result.Rank
+ }
+ if result.Rank > 0 {
+ return result.Rank
+ }
+ return int(^uint(0) >> 1)
+}
+
// Query holds request parameters used by HTTP handlers and search engines.
// Example minimal query: Query{Text: "golang", Limit: 10}.
type Query struct {
@@ -168,7 +239,7 @@ type Query struct {
// For Google, false includes similar results and true hides them.
Filter bool
// Answers enables parsing answer modules when supported by the engine.
- // Such entries may be returned with negative rank values.
+ // Such entries may be returned with non-positive internal rank values.
Answers bool
// ProxyURL is a direct proxy URL used by raw HTTP search paths.
ProxyURL string
diff --git a/core/enrichment_domain.go b/core/enrichment_domain.go
index e1dedd9..0729664 100644
--- a/core/enrichment_domain.go
+++ b/core/enrichment_domain.go
@@ -45,15 +45,9 @@ func EnrichDomainInfo(domain string) *DomainInfo {
cfg := loadEnrichmentDomains()
info := &DomainInfo{
- TLD: tld,
- SLD: sld,
- IsGov: isGovTLD(domain, tld),
- IsEdu: isEduTLD(domain, tld),
- IsMil: isMilTLD(tld),
- IsNews: cfg.NewsDomains[domain],
- IsForum: cfg.ForumDomains[domain],
- IsMarketplace: cfg.MarketplaceDomains[domain],
- IsSocial: cfg.SocialDomains[domain],
+ TLD: tld,
+ SLD: sld,
+ Category: domainCategory(domain, tld, cfg),
}
return info
}
@@ -67,6 +61,9 @@ func ClassifyURL(rawURL, domain string) *Classification {
contentType := classifyContentType(rawURL)
sourceHint := classifySourceHint(domain)
+ if contentType == "webpage" && sourceHint == "" {
+ return nil
+ }
return &Classification{
ContentType: contentType,
@@ -74,6 +71,27 @@ func ClassifyURL(rawURL, domain string) *Classification {
}
}
+func domainCategory(domain, tld string, cfg enrichmentDomainsConfig) string {
+ switch {
+ case isGovTLD(domain, tld):
+ return "gov"
+ case isEduTLD(domain, tld):
+ return "edu"
+ case isMilTLD(tld):
+ return "mil"
+ case cfg.NewsDomains[domain]:
+ return "news"
+ case cfg.ForumDomains[domain]:
+ return "forum"
+ case cfg.MarketplaceDomains[domain]:
+ return "marketplace"
+ case cfg.SocialDomains[domain]:
+ return "social"
+ default:
+ return ""
+ }
+}
+
// splitDomain returns (public suffix, registrable domain label).
func splitDomain(domain string) (tld, sld string) {
domain = normalizeDomain(domain)
diff --git a/core/format_markdown.go b/core/format_markdown.go
index 343ce54..eda9dfe 100644
--- a/core/format_markdown.go
+++ b/core/format_markdown.go
@@ -22,9 +22,6 @@ func RenderMarkdown(env *Envelope) []byte {
for i, r := range env.Results {
fmt.Fprintf(&b, "## %d. %s\n\n", i+1, escapeMarkdown(r.Title))
typeLabel := string(r.Type)
- if r.IsAd {
- typeLabel = "ad"
- }
fmt.Fprintf(&b, "**%s** · %s\n\n", r.DisplayURL, typeLabel)
if r.Snippet != "" {
fmt.Fprintf(&b, "%s\n\n", r.Snippet)
diff --git a/core/response.go b/core/response.go
index 4460525..2af4156 100644
--- a/core/response.go
+++ b/core/response.go
@@ -11,13 +11,13 @@ type QueryEcho struct {
// ResponseMeta carries request-level metadata for observability and debugging.
type ResponseMeta struct {
- RequestID string `json:"request_id"`
- RequestedAt string `json:"requested_at"`
- TookMs int64 `json:"took_ms"`
- EnginesResponded []string `json:"engines_responded,omitempty"`
- EnginesFailed []string `json:"engines_failed"`
- EngineErrors []EngineErrorDetail `json:"engine_errors,omitempty"`
- Version string `json:"version"`
+ RequestID string `json:"request_id"`
+ RequestedAt string `json:"requested_at"`
+ TookMs int64 `json:"took_ms"`
+ EnginesResponded []string `json:"engines_responded,omitempty"`
+ EnginesFailed []string `json:"engines_failed"`
+ EngineErrors []EngineErrorDetail `json:"engine_errors,omitempty"`
+ Version string `json:"version"`
}
// EngineErrorDetail is a client-facing, sanitized per-engine failure summary.
@@ -34,7 +34,7 @@ type Pagination struct {
NextStart int `json:"next_start"`
}
-// Envelope is the top-level v1 response wrapper for all search endpoints.
+// Envelope is the top-level v2 response wrapper for all search endpoints.
type Envelope struct {
Query QueryEcho `json:"query"`
Meta ResponseMeta `json:"meta"`
@@ -64,7 +64,7 @@ type ClusterOccurrence struct {
ResultID string `json:"result_id"`
}
-// ImageEnvelope is the top-level v1 response wrapper for image search endpoints.
+// ImageEnvelope is the top-level v2 response wrapper for image search endpoints.
type ImageEnvelope struct {
Query QueryEcho `json:"query"`
Meta ResponseMeta `json:"meta"`
@@ -72,7 +72,7 @@ type ImageEnvelope struct {
Pagination Pagination `json:"pagination"`
}
-const apiVersion = "1.0"
+const apiVersion = "2.0"
// NewEnvelope builds a fresh Envelope pre-filled with query echo and an open
// meta block. Call Finalize before serializing.
@@ -124,11 +124,21 @@ func (e *Envelope) Finalize(startedAt time.Time, q Query) {
page := q.Start/limit + 1
e.Pagination = Pagination{
Page: page,
- HasMore: len(e.Results) >= limit,
+ HasMore: countNonAdResults(e.Results) >= limit,
NextStart: q.Start + limit,
}
}
+func countNonAdResults(results []Result) int {
+ count := 0
+ for _, result := range results {
+ if result.Type != ResultTypeAd {
+ count++
+ }
+ }
+ return count
+}
+
// Finalize stamps the elapsed time and computes pagination fields.
func (e *ImageEnvelope) Finalize(startedAt time.Time, q Query) {
e.Meta.TookMs = time.Since(startedAt).Milliseconds()
diff --git a/core/response_builder.go b/core/response_builder.go
index df483cb..296bb40 100644
--- a/core/response_builder.go
+++ b/core/response_builder.go
@@ -24,7 +24,7 @@ type EnrichContext struct {
Query Query
}
-// EnrichResult converts a raw engine result into the v1 Result shape.
+// EnrichResult converts a raw engine result into the v2 Result shape.
func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
normalizedURL := normalizeURL(raw.URL)
domain := extractDomain(normalizedURL)
@@ -42,17 +42,23 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
if raw.Rank <= 0 && !raw.Ad {
resultType = ResultTypeAnswerBox
}
-
- limit := ctx.Query.Limit
- if limit <= 0 {
- limit = 25
+ rank := raw.Rank
+ if rank < 0 {
+ if raw.Ad {
+ rank = -rank
+ } else {
+ rank = 0
+ }
+ }
+
+ absolute := computeResultPosition(raw, ctx.Query.Start)
+ if absolute <= 0 {
+ absolute = rank
}
- page := ctx.Query.Start/limit + 1
- absolute, onPage := computeResultPosition(raw.Rank, ctx.Query.Start)
result := Result{
ID: buildResultID(ctx.Engine, normalizedURL),
- Rank: raw.Rank,
+ Rank: rank,
Type: resultType,
Title: raw.Title,
URL: normalizedURL,
@@ -60,13 +66,10 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
Snippet: raw.Description,
Domain: domain,
Favicon: favicon,
- IsAd: raw.Ad,
- Position: Position{
- Absolute: absolute,
- Page: page,
- OnPage: onPage,
- },
- Engine: ctx.Engine,
+ Engine: ctx.Engine,
+ }
+ if absolute > 0 {
+ result.Position = &Position{Absolute: absolute}
}
result.DomainInfo = EnrichDomainInfo(domain)
@@ -75,7 +78,7 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
return result
}
-// EnrichImageResult converts a raw engine result into the v1 ImageResult shape.
+// EnrichImageResult converts a raw engine result into the v2 ImageResult shape.
func EnrichImageResult(raw SearchResult, ctx EnrichContext) ImageResult {
imageURL := normalizeURL(raw.URL)
meta := parseImageDescription(raw.Description)
@@ -122,14 +125,22 @@ func shortMD5(value string) string {
return hex.EncodeToString(h[:responseIDBytes])
}
-func computeResultPosition(rank, start int) (absolute, onPage int) {
- if rank <= 0 {
- return 0, 0
+func computeResultPosition(raw SearchResult, start int) int {
+ if raw.AbsoluteRank > 0 {
+ return raw.AbsoluteRank
+ }
+
+ rank := raw.Rank
+ if rank < 0 {
+ rank = -rank
+ }
+ if rank == 0 {
+ return 0
}
if start > 0 && rank > start {
- return rank, rank - start
+ return rank
}
- return start + rank, rank
+ return start + rank
}
// normalizeURL lowercases scheme+host, strips trailing slash, and removes
diff --git a/core/result.go b/core/result.go
index 2da4c97..e0d0a4f 100644
--- a/core/result.go
+++ b/core/result.go
@@ -19,34 +19,29 @@ const (
// Position describes where a result sits in the overall result stream.
type Position struct {
- // Absolute is the 1-based rank counting from the first result of the first page.
+ // Absolute is the 1-based rank counting from the first result of the first page,
+ // across both organic and ad blocks. Always emitted so SEO callers can plot
+ // rank vs. on-page position without inferring it from the result order.
Absolute int `json:"absolute"`
- // Page is the 1-based page number derived from start/limit.
- Page int `json:"page"`
- // OnPage is the 1-based rank within this page.
- OnPage int `json:"on_page"`
}
// DomainInfo carries TLD-derived category signals for a result domain.
type DomainInfo struct {
- TLD string `json:"tld"`
- SLD string `json:"sld"`
- IsGov bool `json:"is_gov"`
- IsEdu bool `json:"is_edu"`
- IsMil bool `json:"is_mil"`
- IsNews bool `json:"is_news"`
- IsForum bool `json:"is_forum"`
- IsMarketplace bool `json:"is_marketplace"`
- IsSocial bool `json:"is_social"`
+ TLD string `json:"tld,omitempty"`
+ SLD string `json:"sld,omitempty"`
+ // Category is one of "gov", "edu", "mil", "news", "forum", "marketplace",
+ // "social", or "" when the domain does not match any known category.
+ Category string `json:"category"`
}
// Classification holds URL-path heuristic hints for downstream consumers.
type Classification struct {
- ContentType string `json:"content_type"`
- SourceHint string `json:"source_hint"`
+ ContentType string `json:"content_type,omitempty"`
+ SourceHint string `json:"source_hint,omitempty"`
}
-// Result is the v1 normalized result returned in every search response.
+// Result is the v2 normalized result returned in search responses. Optional
+// fields (Position, DomainInfo, Classification) are omitted when empty.
type Result struct {
ID string `json:"id"`
Rank int `json:"rank"`
@@ -57,8 +52,7 @@ type Result struct {
Snippet string `json:"snippet"`
Domain string `json:"domain"`
Favicon string `json:"favicon"`
- IsAd bool `json:"is_ad"`
- Position Position `json:"position"`
+ Position *Position `json:"position,omitempty"`
Engine string `json:"engine"`
DomainInfo *DomainInfo `json:"domain_info,omitempty"`
Classification *Classification `json:"classification,omitempty"`
@@ -78,7 +72,7 @@ type ImageSource struct {
Domain string `json:"domain"`
}
-// ImageResult is the v1 shape for image search results.
+// ImageResult is the v2 shape for image search results.
type ImageResult struct {
ID string `json:"id"`
Rank int `json:"rank"`
diff --git a/core/result_rank_test.go b/core/result_rank_test.go
new file mode 100644
index 0000000..1a58afc
--- /dev/null
+++ b/core/result_rank_test.go
@@ -0,0 +1,104 @@
+package core
+
+import (
+ "testing"
+ "time"
+)
+
+func TestDeduplicateResultsOrdersByAbsoluteRank(t *testing.T) {
+ t.Parallel()
+
+ results := DeduplicateResults([]SearchResult{
+ {Rank: 1, AbsoluteRank: 3, URL: "https://organic.example.com/one"},
+ {Rank: 1, AbsoluteRank: 1, URL: "https://ads.example.com/one", Ad: true},
+ {Rank: 2, AbsoluteRank: 2, URL: "https://ads.example.com/two", Ad: true},
+ {Rank: 2, AbsoluteRank: 4, URL: "https://organic.example.com/two"},
+ })
+
+ wantRanks := []int{1, 2, 1, 2}
+ wantAbsoluteRanks := []int{1, 2, 3, 4}
+ if len(results) != len(wantRanks) {
+ t.Fatalf("len(results) = %d, want %d", len(results), len(wantRanks))
+ }
+ for i, want := range wantRanks {
+ if results[i].Rank != want {
+ t.Fatalf("result %d rank = %d, want %d", i, results[i].Rank, want)
+ }
+ if results[i].AbsoluteRank != wantAbsoluteRanks[i] {
+ t.Fatalf("result %d absolute rank = %d, want %d", i, results[i].AbsoluteRank, wantAbsoluteRanks[i])
+ }
+ }
+}
+
+func TestDeduplicateResultsKeepsAdAndOrganicForSameURL(t *testing.T) {
+ t.Parallel()
+
+ results := DeduplicateResults([]SearchResult{
+ {Rank: 1, AbsoluteRank: 1, URL: "https://example.com/page", Ad: true},
+ {Rank: 1, AbsoluteRank: 2, URL: "https://example.com/page"},
+ })
+
+ if len(results) != 2 {
+ t.Fatalf("len(results) = %d, want 2", len(results))
+ }
+ if !results[0].Ad || results[1].Ad {
+ t.Fatalf("expected ad and organic rows to be preserved separately: %+v", results)
+ }
+}
+
+func TestLimitOrganicResultsDoesNotCountAds(t *testing.T) {
+ t.Parallel()
+
+ results := LimitOrganicResults([]SearchResult{
+ {Rank: 1, URL: "https://ads.example.com/one", Ad: true},
+ {Rank: 1, URL: "https://organic.example.com/one"},
+ {Rank: 2, URL: "https://ads.example.com/two", Ad: true},
+ {Rank: 2, URL: "https://organic.example.com/two"},
+ }, 1)
+
+ if len(results) != 3 {
+ t.Fatalf("len(results) = %d, want 3", len(results))
+ }
+ if CountOrganicResults(results) != 1 {
+ t.Fatalf("organic count = %d, want 1", CountOrganicResults(results))
+ }
+}
+
+func TestEnrichResultUsesAdRankAndAbsolutePosition(t *testing.T) {
+ t.Parallel()
+
+ result := EnrichResult(SearchResult{
+ Rank: 1,
+ AbsoluteRank: 2,
+ URL: "https://ads.example.com/",
+ Title: "Ad",
+ Ad: true,
+ }, EnrichContext{Engine: "google", Query: Query{Limit: 10}})
+
+ if result.Rank != 1 {
+ t.Fatalf("rank = %d, want 1", result.Rank)
+ }
+ if result.Position == nil || result.Position.Absolute != 2 {
+ t.Fatalf("unexpected position: %+v", result.Position)
+ }
+ if result.Type != ResultTypeAd {
+ t.Fatalf("unexpected result type: %q", result.Type)
+ }
+}
+
+func TestEnvelopePaginationCountsOrganicResults(t *testing.T) {
+ t.Parallel()
+
+ env := NewEnvelope(Query{Text: "ads", Limit: 2}, "req-1", time.Now(), []string{"bing"})
+ env.Results = []Result{
+ {Rank: 1, Type: ResultTypeAd},
+ {Rank: 2, Type: ResultTypeAd},
+ {Rank: 1, Type: ResultTypeOrganic},
+ }
+
+ env.Finalize(time.Now(), Query{Text: "ads", Limit: 2})
+
+ if env.Pagination.HasMore {
+ t.Fatalf("has_more = true, want false when organic count is below limit: %+v", env.Pagination)
+ }
+}
diff --git a/core/server.go b/core/server.go
index f1f67e9..d92cff6 100644
--- a/core/server.go
+++ b/core/server.go
@@ -1198,10 +1198,11 @@ func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearch
if result.URL == "" {
continue
}
- key := NormalizeURLForClustering(result.URL)
- if key == "" {
+ normalizedURL := NormalizeURLForClustering(result.URL)
+ if normalizedURL == "" {
continue
}
+ key := resultDedupKey(SearchResult{URL: normalizedURL, Ad: result.Ad})
existing, exists := urlMap[key]
if !exists {
urlMap[key] = result
@@ -1219,8 +1220,11 @@ func (s *Server) deduplicateMegaResults(results []MegaSearchResult) []MegaSearch
}
sort.Slice(deduped, func(i, j int) bool {
- if deduped[i].Rank != deduped[j].Rank {
- return deduped[i].Rank < deduped[j].Rank
+ if resultLess(deduped[i].SearchResult, deduped[j].SearchResult) {
+ return true
+ }
+ if resultLess(deduped[j].SearchResult, deduped[i].SearchResult) {
+ return false
}
if deduped[i].Engine != deduped[j].Engine {
return deduped[i].Engine < deduped[j].Engine
diff --git a/core/server_test.go b/core/server_test.go
index 411eaff..179c870 100644
--- a/core/server_test.go
+++ b/core/server_test.go
@@ -2208,7 +2208,7 @@ func TestServerOptions_DisableCORSMiddleware(t *testing.T) {
}
}
-func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) {
+func TestDedicatedEndpointReturnsV2Envelope(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
opts.Resilience.Retry.MaxRetries = 0
@@ -2223,8 +2223,8 @@ func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) {
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
- if env.Meta.Version != "1.0" {
- t.Fatalf("expected meta.version=1.0, got %q", env.Meta.Version)
+ if env.Meta.Version != "2.0" {
+ t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version)
}
if env.Meta.RequestID == "" {
t.Fatal("expected non-empty meta.request_id")
@@ -2248,15 +2248,12 @@ func TestDedicatedEndpointReturnsV1Envelope(t *testing.T) {
if r.Snippet == "" && r.Title == "" {
t.Fatal("expected result to have title or snippet")
}
- if r.Position.Page < 1 {
- t.Fatalf("expected position.page >= 1, got %d", r.Position.Page)
- }
if env.Pagination.Page < 1 {
t.Fatalf("expected pagination.page >= 1, got %d", env.Pagination.Page)
}
}
-func TestMegaSearchReturnsV1EnvelopeWithEnginesFailed(t *testing.T) {
+func TestMegaSearchReturnsV2EnvelopeWithEnginesFailed(t *testing.T) {
good := &engineMock{name: "google", initialized: true}
bad := &engineMock{
name: "bing",
@@ -2279,8 +2276,8 @@ func TestMegaSearchReturnsV1EnvelopeWithEnginesFailed(t *testing.T) {
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
- if env.Meta.Version != "1.0" {
- t.Fatalf("expected meta.version=1.0, got %q", env.Meta.Version)
+ if env.Meta.Version != "2.0" {
+ t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version)
}
if len(env.Meta.EnginesFailed) == 0 {
t.Fatal("expected engines_failed to contain bing")
@@ -2446,6 +2443,49 @@ func TestFormatParamReturnsCorrectContentType(t *testing.T) {
}
}
+func TestJSONOmitsRedundantResultFields(t *testing.T) {
+ engine := &engineMock{
+ name: "google",
+ initialized: true,
+ searchFn: func(_ context.Context, _ Query) ([]SearchResult, error) {
+ return []SearchResult{{
+ Rank: 1,
+ AbsoluteRank: 2,
+ URL: "https://example.gov/page",
+ Title: "Gov Result",
+ Description: "Snippet",
+ }}, nil
+ },
+ }
+ opts := DefaultServerOptions()
+ opts.Resilience.Retry.MaxRetries = 0
+ srv := NewServerWithOptions("127.0.0.1", 7210, opts, engine)
+
+ resp := request(t, srv, "/google/search?text=clean-json")
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("expected 200, got %d", resp.StatusCode)
+ }
+ body, _ := io.ReadAll(resp.Body)
+ payload := string(body)
+ for _, omitted := range []string{`"is_ad"`, `"on_page"`, `"is_gov"`} {
+ if strings.Contains(payload, omitted) {
+ t.Fatalf("json payload should not contain %s: %s", omitted, payload)
+ }
+ }
+ if strings.Contains(payload, `"position":{"absolute":2,"page"`) {
+ t.Fatalf("json payload should not contain per-result page: %s", payload)
+ }
+ if !strings.Contains(payload, `"type":"organic"`) {
+ t.Fatalf("json payload should keep result type: %s", payload)
+ }
+ if !strings.Contains(payload, `"category":"gov"`) {
+ t.Fatalf("json payload should collapse domain category: %s", payload)
+ }
+ if !strings.Contains(payload, `"absolute":2`) {
+ t.Fatalf("json payload should keep meaningful absolute position: %s", payload)
+ }
+}
+
func TestFormatParamBypassesJSONCache(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
@@ -2531,9 +2571,8 @@ func TestPaginatedPositionUsesAbsoluteRank(t *testing.T) {
if len(env.Results) != 1 {
t.Fatalf("expected one result, got %d", len(env.Results))
}
- pos := env.Results[0].Position
- if pos.Absolute != 11 || pos.OnPage != 1 || pos.Page != 2 {
- t.Fatalf("unexpected position: %+v", pos)
+ if env.Results[0].Position == nil || env.Results[0].Position.Absolute != 11 {
+ t.Fatalf("expected absolute position 11, got %+v", env.Results[0].Position)
}
}
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 43286c9..ee50d15 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -83,7 +83,7 @@ Engine parsers return the older internal shape:
- `Description`
- `Ad`
-HTTP handlers convert this into the public v1 response through `core/response_builder.go`.
+HTTP handlers convert this into the public v2 response through `core/response_builder.go`.
## HTTP Request Flow
@@ -118,7 +118,7 @@ HTTP request
## Public API Response
-JSON endpoints return a v1 envelope.
+JSON endpoints return a v2 envelope.
Top-level fields:
@@ -192,7 +192,7 @@ Only JSON responses use the response cache. Cached JSON refreshes request-scoped
`core/enrichment_domain.go` derives:
-- `domain_info`: public suffix, SLD, and category booleans
+- `domain_info`: public suffix, SLD, and collapsed category
- `classification`: content type and known source hint
Public suffix parsing uses `golang.org/x/net/publicsuffix`.
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index e0e9c7d..5f8b570 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -1,10 +1,10 @@
openapi: 3.0.3
info:
title: OpenSERP API
- version: 1.0.0
+ version: 2.0.0
description: >
OpenSERP provides dedicated and multi-engine search endpoints for Google, Yandex,
- Baidu, Bing, and DuckDuckGo. All responses are wrapped in a v1 envelope with
+ Baidu, Bing, and DuckDuckGo. Search responses are wrapped in a v2 envelope with
query echo, metadata, normalized results, and pagination. Invalid client input
returns 400 with a machine-readable `reason` code.
license:
@@ -90,7 +90,7 @@ paths:
requested_at: "2026-04-24T12:00:00Z"
took_ms: 842
engines_failed: []
- version: "1.0"
+ version: "2.0"
results:
- id: s_a1b2c3d4e5f6a1b2
rank: 1
@@ -101,25 +101,13 @@ paths:
snippet: Go is an open source programming language...
domain: go.dev
favicon: https://go.dev/favicon.ico
- is_ad: false
position:
absolute: 1
- page: 1
- on_page: 1
engine: google
domain_info:
tld: dev
sld: go
- is_gov: false
- is_edu: false
- is_mil: false
- is_news: false
- is_forum: false
- is_marketplace: false
- is_social: false
- classification:
- content_type: webpage
- source_hint: ""
+ category: ""
pagination:
page: 1
has_more: true
@@ -612,7 +600,7 @@ components:
name: limit
in: query
required: false
- description: Maximum results to return (1–100).
+ description: Maximum organic results to return (1–100). Ads may be returned in addition.
schema:
type: integer
minimum: 1
@@ -690,7 +678,7 @@ components:
in: query
required: false
description: >
- Output format. `json` (default) returns the full envelope. `markdown` returns a
+ Output format. `json` (default) returns the envelope. `markdown` returns a
Markdown document suitable for Slack/email. `text` returns a minimal plain-text
block optimised for LLM context windows. `ndjson` returns one result object per
line with no envelope. The `Accept` header is also checked
@@ -993,7 +981,7 @@ components:
schema:
$ref: "#/components/schemas/ErrorResponse"
schemas:
- # ── v1 envelope ──────────────────────────────────────────────────
+ # ── v2 envelope ──────────────────────────────────────────────────
QueryEcho:
type: object
required: [text, engines_requested]
@@ -1035,7 +1023,7 @@ components:
$ref: "#/components/schemas/EngineErrorDetail"
version:
type: string
- example: "1.0"
+ example: "2.0"
EngineErrorDetail:
type: object
required: [engine, error]
@@ -1067,21 +1055,18 @@ components:
# ── Result ───────────────────────────────────────────────────────
Position:
type: object
- required: [absolute, page, on_page]
+ required: [absolute]
properties:
absolute:
type: integer
- description: 1-based rank counting from the first result of the first page.
- example: 1
- page:
- type: integer
- example: 1
- on_page:
- type: integer
- example: 1
+ description: >
+ 1-based rank in the mixed SERP stream, across both organic and ad blocks.
+ Always present so SEO callers can plot rank vs. on-page position without
+ inferring it from result order.
+ example: 2
DomainInfo:
type: object
- required: [tld, sld, is_gov, is_edu, is_mil, is_news, is_forum, is_marketplace, is_social]
+ required: [category]
properties:
tld:
type: string
@@ -1089,23 +1074,12 @@ components:
sld:
type: string
example: wikipedia
- is_gov:
- type: boolean
- is_edu:
- type: boolean
- is_mil:
- type: boolean
- is_news:
- type: boolean
- is_forum:
- type: boolean
- is_marketplace:
- type: boolean
- is_social:
- type: boolean
+ category:
+ type: string
+ description: Empty string when the domain matches no known category.
+ enum: ["", gov, edu, mil, news, forum, marketplace, social]
Classification:
type: object
- required: [content_type, source_hint]
properties:
content_type:
type: string
@@ -1117,8 +1091,8 @@ components:
ResultType:
type: string
description: >
- SERP block type. New values require a minor version bump (`meta.version: "1.1"`).
- In v1.0, engines emit `organic`, `ad`, or `answer_box`; other enum values are
+ SERP block type. New values require a minor version bump (`meta.version: "2.1"`).
+ In v2.0, engines emit `organic`, `ad`, or `answer_box`; other enum values are
reserved for future parser upgrades.
enum:
- organic
@@ -1144,7 +1118,6 @@ components:
- snippet
- domain
- favicon
- - is_ad
- position
- engine
properties:
@@ -1181,9 +1154,6 @@ components:
type: string
description: Constructed as `https://{domain}/favicon.ico`. Not probed.
example: https://go.dev/favicon.ico
- is_ad:
- type: boolean
- example: false
position:
$ref: "#/components/schemas/Position"
engine:
diff --git a/duckduckgo/parse_html.go b/duckduckgo/parse_html.go
index a169170..6ef35a8 100644
--- a/duckduckgo/parse_html.go
+++ b/duckduckgo/parse_html.go
@@ -21,6 +21,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
func parseDDGDocument(doc *goquery.Document) []core.SearchResult {
var results []core.SearchResult
rank := 1
+ adRank := 1
+ absoluteRank := 1
resultSel := firstMatchingSelector(doc, Selectors.Results)
if resultSel == "" {
@@ -40,32 +42,38 @@ func parseDDGDocument(doc *goquery.Document) []core.SearchResult {
desc := extractFirstText(item, Selectors.Desc)
- isAd := false
- for _, sel := range Selectors.AdBadge {
- if item.Find(sel).Length() > 0 {
- isAd = true
- break
- }
- }
+ isAd := duckduckgoSelectionHasAdMarker(item)
r := core.SearchResult{
- Rank: rank,
- URL: href,
- Title: title,
- Description: desc,
- Ad: isAd,
+ Rank: rank,
+ AbsoluteRank: absoluteRank,
+ URL: href,
+ Title: title,
+ Description: desc,
+ Ad: isAd,
}
if !isAd {
rank++
} else {
- r.Rank = -1
+ r.Rank = adRank
+ adRank++
}
results = append(results, r)
+ absoluteRank++
})
return core.DeduplicateResults(results)
}
+func duckduckgoSelectionHasAdMarker(item *goquery.Selection) bool {
+ for _, sel := range Selectors.AdBadge {
+ if item.Is(sel) || item.Find(sel).Length() > 0 {
+ return true
+ }
+ }
+ return false
+}
+
// firstMatchingSelector returns the first selector from the list that matches
// at least one element in the document.
func firstMatchingSelector(doc *goquery.Document, selectors []string) string {
diff --git a/duckduckgo/parse_html_test.go b/duckduckgo/parse_html_test.go
index 7eac92f..ac2bf98 100644
--- a/duckduckgo/parse_html_test.go
+++ b/duckduckgo/parse_html_test.go
@@ -55,3 +55,51 @@ func TestParseDDGHTMLEmpty(t *testing.T) {
t.Fatalf("expected zero results for empty HTML, got %d", len(results))
}
}
+
+func TestParseDDGHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
+ t.Parallel()
+
+ html := `
+
+
+ Paid snippet
+
+
+
+ Organic snippet one
+
+
+
+ Organic snippet two
+`
+
+ results, err := ParseHTML(bytes.NewReader([]byte(html)))
+ if err != nil {
+ t.Fatalf("ParseHTML() error = %v", err)
+ }
+ if len(results) != 3 {
+ t.Fatalf("expected 3 results, got %d", len(results))
+ }
+
+ organicRank := 0
+ adRank := 0
+ for _, r := range results {
+ if r.Ad {
+ adRank++
+ if r.Rank != adRank {
+ t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
+ }
+ continue
+ }
+ organicRank++
+ if r.Rank != organicRank {
+ t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
+ }
+ }
+ if organicRank != 2 {
+ t.Fatalf("organic count = %d, want 2", organicRank)
+ }
+ if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
+ t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
+ }
+}
diff --git a/duckduckgo/search.go b/duckduckgo/search.go
index cd138e8..8cef4c3 100644
--- a/duckduckgo/search.go
+++ b/duckduckgo/search.go
@@ -69,8 +69,11 @@ func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool {
func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
searchResults := []core.SearchResult{}
+ organicRank := pageNum * 10
+ adRank := 1
+ absoluteRank := pageNum*10 + 1
- for i, r := range results {
+ for _, r := range results {
// Get URL - try multiple selectors
var link *rod.Element
var err error
@@ -113,28 +116,47 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se
desc := core.FirstNonEmptyText(r, Selectors.Desc...)
// Check if it's an ad
- isAd := false
- for _, selector := range Selectors.AdBadge {
- adIndicator, err := r.Element(selector)
- if err == nil && adIndicator != nil {
- isAd = true
- break
- }
+ isAd := duckduckgoElementHasAdMarker(r)
+ resultRank := 0
+ if isAd {
+ resultRank = adRank
+ adRank++
+ } else {
+ organicRank++
+ resultRank = organicRank
}
result := core.SearchResult{
- Rank: (pageNum * 10) + (i + 1),
- URL: hrefStr,
- Title: title,
- Description: desc,
- Ad: isAd,
+ Rank: resultRank,
+ AbsoluteRank: absoluteRank,
+ URL: hrefStr,
+ Title: title,
+ Description: desc,
+ Ad: isAd,
}
searchResults = append(searchResults, result)
+ absoluteRank++
}
return searchResults
}
+func duckduckgoElementHasAdMarker(el *rod.Element) bool {
+ if el == nil {
+ return false
+ }
+ for _, selector := range Selectors.AdBadge {
+ matches, err := el.Matches(selector)
+ if err == nil && matches {
+ return true
+ }
+ if adIndicator, err := el.Element(selector); err == nil && adIndicator != nil {
+ return true
+ }
+ }
+ return false
+}
+
// Search executes a DuckDuckGo web search and returns normalized search
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -193,13 +215,13 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
return false, nil
}
- for len(allResults) < query.Limit {
+ for query.Limit <= 0 || core.CountOrganicResults(allResults) < query.Limit {
done, err := fetchPage()
if err != nil {
return nil, err
}
searchPage++
- if done || len(allResults) >= query.Limit {
+ if done || (query.Limit > 0 && core.CountOrganicResults(allResults) >= query.Limit) {
break
}
if err := core.SleepContext(ctx, ddg.pageSleep); err != nil {
@@ -211,9 +233,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results []
deduped := core.DeduplicateResults(allResults)
// Trim to exact limit if necessary
- if len(deduped) > query.Limit {
- deduped = deduped[:query.Limit]
- }
+ deduped = core.LimitOrganicResults(deduped, query.Limit)
ddg.logger.Info("Search completed: %d results", len(deduped))
return deduped, nil
diff --git a/duckduckgo/selectors.go b/duckduckgo/selectors.go
index 1de5fbb..c86d0dc 100644
--- a/duckduckgo/selectors.go
+++ b/duckduckgo/selectors.go
@@ -43,9 +43,13 @@ var Selectors = struct {
"anomaly",
},
Results: []string{
+ "article[data-testid='result'], article[data-testid='ad'], li[data-layout='organic'], li[data-layout='ad'], div[data-testid='result'], div[data-testid='ad']",
"article[data-testid='result']",
+ "article[data-testid='ad']",
"li[data-layout='organic']",
+ "li[data-layout='ad']",
"div[data-testid='result']",
+ "div[data-testid='ad']",
"div.result",
},
Title: []string{
@@ -65,6 +69,9 @@ var Selectors = struct {
"h3 a",
},
AdBadge: []string{
+ "article[data-testid='ad']",
+ "li[data-layout='ad']",
+ "div[data-testid='ad']",
"[data-testid='ad-badge']",
".ad-badge",
".result--ad",
diff --git a/ecosia/parse_html.go b/ecosia/parse_html.go
index 3435d3b..a363bc2 100644
--- a/ecosia/parse_html.go
+++ b/ecosia/parse_html.go
@@ -21,6 +21,7 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult {
var results []core.SearchResult
rank := 1
+ adRank := 1
doc.Find(Selectors.Result).Each(func(_ int, item *goquery.Selection) {
res, ok := parseEcosiaItem(item, rank, false)
@@ -32,13 +33,15 @@ func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult {
})
doc.Find(Selectors.Ad).Each(func(_ int, item *goquery.Selection) {
- res, ok := parseEcosiaItem(item, -1, true)
+ res, ok := parseEcosiaItem(item, adRank, true)
if !ok {
return
}
results = append(results, res)
+ adRank++
})
+ setSeparatedAdAbsoluteRanks(results, 0)
return core.DeduplicateResults(results)
}
@@ -84,6 +87,24 @@ func parseEcosiaItem(item *goquery.Selection, rank int, ad bool) (core.SearchRes
}, true
}
+func setSeparatedAdAbsoluteRanks(results []core.SearchResult, start int) {
+ adCount := 0
+ for i := range results {
+ if results[i].Ad {
+ adCount++
+ results[i].AbsoluteRank = start + results[i].Rank
+ }
+ }
+ organicAbsoluteRank := start + adCount + 1
+ for i := range results {
+ if results[i].Ad {
+ continue
+ }
+ results[i].AbsoluteRank = organicAbsoluteRank
+ organicAbsoluteRank++
+ }
+}
+
// // parseEcosiaImageItem extracts a single image card from a goquery Selection.
// func parseEcosiaImageItem(item *goquery.Selection, rank int) (core.SearchResult, bool) {
// linkTag := item.Find(Selectors.ImageLink).First()
diff --git a/ecosia/search.go b/ecosia/search.go
index 8961e7e..e9fcca8 100644
--- a/ecosia/search.go
+++ b/ecosia/search.go
@@ -139,13 +139,13 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
}()
// nextRank counts up across pages for organic results; nextAdRank counts
- // down from -1 so ads keep unique, order-preserving negative ranks.
+ // up within sponsored results so ad rank stays separate from SEO rank.
all := []core.SearchResult{}
pageNum, nextRank, err := startPage(query.Start)
if err != nil {
return nil, err
}
- nextAdRank := -1
+ nextAdRank := 1
// fetchPage loads one SERP page and appends parsed results.
// Returns (done, error): done=true ends the outer loop without error.
fetchPage := func() (bool, error) {
@@ -192,13 +192,13 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
for _, r := range ads {
if res, ok := e.parseResult(r, nextAdRank, true); ok {
all = append(all, res)
- nextAdRank--
+ nextAdRank++
}
}
return false, nil
}
- for query.Limit <= 0 || len(all) < query.Limit {
+ for query.Limit <= 0 || core.CountOrganicResults(all) < query.Limit {
done, err := fetchPage()
if err != nil {
return nil, err
@@ -207,7 +207,7 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
if done {
break
}
- if query.Limit > 0 && len(all) >= query.Limit {
+ if query.Limit > 0 && core.CountOrganicResults(all) >= query.Limit {
break
}
if err := core.SleepContext(ctx, e.pageSleep); err != nil {
@@ -215,13 +215,10 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S
}
}
+ setSeparatedAdAbsoluteRanks(all, query.Start)
deduped := core.DeduplicateResults(all)
if query.Limit > 0 {
- organic, ads := splitAds(deduped)
- if len(organic) > query.Limit {
- organic = organic[:query.Limit]
- }
- deduped = append(organic, ads...)
+ deduped = core.LimitOrganicResults(deduped, query.Limit)
}
e.logger.Info("Search completed: %d results", len(deduped))
return deduped, nil
@@ -369,14 +366,3 @@ func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []c
e.logger.Info("Image search completed: %d results", len(deduped))
return deduped, nil
}
-
-func splitAds(in []core.SearchResult) (organic, ads []core.SearchResult) {
- for _, r := range in {
- if r.Ad {
- ads = append(ads, r)
- } else {
- organic = append(organic, r)
- }
- }
- return
-}
diff --git a/ecosia/search_raw.go b/ecosia/search_raw.go
index 780607e..c1d8de3 100644
--- a/ecosia/search_raw.go
+++ b/ecosia/search_raw.go
@@ -121,7 +121,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
// Ecosia paginates by page index, not result offset, so re-rank from
// the page boundary rather than query.Start (off-grid offsets round down).
- // Skip ads (rank<0) so organic ranks stay sequential from startRank.
+ // Skip ads so organic ranks stay sequential from startRank.
organicIdx := 0
for i := range parsedResults {
if parsedResults[i].Ad {
@@ -130,6 +130,7 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
parsedResults[i].Rank = startRank + organicIdx
organicIdx++
}
+ setSeparatedAdAbsoluteRanks(parsedResults, pageNum*10)
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
fmt.Sprintf("Ecosia Raw results : %v", parsedResults),
diff --git a/google/parse_html_test.go b/google/parse_html_test.go
index 06da9a3..8f4c1f5 100644
--- a/google/parse_html_test.go
+++ b/google/parse_html_test.go
@@ -68,3 +68,53 @@ func TestParseHTMLNoResults(t *testing.T) {
t.Fatalf("expected zero results, got %d", len(results))
}
}
+
+func TestParseHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
+ t.Parallel()
+
+ html := `
+`
+
+ results, err := ParseHTML(bytes.NewReader([]byte(html)))
+ if err != nil {
+ t.Fatalf("ParseHTML() error = %v", err)
+ }
+ if len(results) != 3 {
+ t.Fatalf("expected 3 results, got %d", len(results))
+ }
+
+ organicRank := 0
+ adRank := 0
+ for _, r := range results {
+ if r.Ad {
+ adRank++
+ if r.Rank != adRank {
+ t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
+ }
+ continue
+ }
+ organicRank++
+ if r.Rank != organicRank {
+ t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
+ }
+ }
+ if organicRank != 2 {
+ t.Fatalf("organic count = %d, want 2", organicRank)
+ }
+ if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
+ t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
+ }
+}
diff --git a/google/search.go b/google/search.go
index d315942..f294a42 100644
--- a/google/search.go
+++ b/google/search.go
@@ -236,6 +236,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
gogl.logger.Info("Found %d total results", totalResults)
rank := query.Start
+ adRank := 1
+ absoluteRank := query.Start + 1
for _, resEl := range searchResultElems {
srchRes := core.SearchResult{}
@@ -285,7 +287,10 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
} else {
srchRes.Description = strings.TrimSpace(text)
}
- rank += 1
+ srchRes.Rank = adRank
+ srchRes.AbsoluteRank = absoluteRank
+ adRank++
+ absoluteRank++
} else if isAnswerBox {
// 2. Parse answer boxes
@@ -406,6 +411,8 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
rank += 1
srchRes.Rank = rank
+ srchRes.AbsoluteRank = absoluteRank
+ absoluteRank++
searchResults = append(searchResults, srchRes)
continue
@@ -413,7 +420,16 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
continue
}
- srchRes.Rank = rank
+ if srchRes.Ad && srchRes.Rank == 0 {
+ srchRes.Rank = adRank
+ adRank++
+ } else if !srchRes.Ad {
+ srchRes.Rank = rank
+ }
+ if srchRes.AbsoluteRank == 0 {
+ srchRes.AbsoluteRank = absoluteRank
+ absoluteRank++
+ }
searchResults = append(searchResults, srchRes)
}
diff --git a/google/search_raw.go b/google/search_raw.go
index 39c33e4..c5258ab 100644
--- a/google/search_raw.go
+++ b/google/search_raw.go
@@ -26,6 +26,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
results := []core.SearchResult{}
rank := 1
+ adRank := 1
+ absoluteRank := 1
// Use data attributes instead of class names to find results
// Both old and new DOM have data-hveid and data-ved attributes
@@ -55,6 +57,8 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
titleTag := item.Find(Selectors.Title)
title := titleTag.Text()
+ isAd := item.Is(Selectors.Ad) || item.Find(Selectors.Ad).Length() > 0
+
// Find description - find div with text content after the heading
// Using attribute selectors that match the description container
descTag := item.Find(Selectors.DescPrimary).First()
@@ -73,15 +77,25 @@ func parseGoogleDocument(doc *goquery.Document) []core.SearchResult {
desc := descTag.Text()
if link != "" && link != "#" {
+ resultRank := rank
+ if isAd {
+ resultRank = adRank
+ adRank++
+ } else {
+ rank++
+ }
+
result := core.SearchResult{
- Rank: rank,
- URL: link,
- Title: title,
- Description: desc,
+ Rank: resultRank,
+ AbsoluteRank: absoluteRank,
+ URL: link,
+ Title: title,
+ Description: desc,
+ Ad: isAd,
}
results = append(results, result)
- rank++
+ absoluteRank++
}
}
@@ -153,8 +167,18 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
}
if query.Start > 0 {
+ organicIdx := 0
for i := range parsedResults {
- parsedResults[i].Rank = query.Start + i + 1
+ if parsedResults[i].Ad {
+ continue
+ }
+ organicIdx++
+ parsedResults[i].Rank = query.Start + organicIdx
+ }
+ for i := range parsedResults {
+ if parsedResults[i].AbsoluteRank > 0 {
+ parsedResults[i].AbsoluteRank += query.Start
+ }
}
}
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
diff --git a/google/selectors.go b/google/selectors.go
index c5c4c06..dde4ed2 100644
--- a/google/selectors.go
+++ b/google/selectors.go
@@ -8,6 +8,7 @@ var Selectors = struct {
ResultStats string
CookieBtn string
Results string
+ Ad string
Title string
DescPrimary string
DescFallback string
@@ -24,6 +25,7 @@ var Selectors = struct {
ResultStats: "div#result-stats",
CookieBtn: "div[role='dialog'][aria-modal] button",
Results: "div[data-hveid][data-ved]",
+ Ad: "div[data-text-ad], [data-text-ad]",
Title: "h3",
DescPrimary: "div[data-sncf='1'] div",
DescFallback: "div.VwiC3b",
diff --git a/yandex/parse_html.go b/yandex/parse_html.go
index 896a384..2cf5e8b 100644
--- a/yandex/parse_html.go
+++ b/yandex/parse_html.go
@@ -2,6 +2,7 @@ package yandex
import (
"io"
+ "net/url"
"strings"
"github.com/PuerkitoBio/goquery"
@@ -21,6 +22,8 @@ func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
var results []core.SearchResult
rank := 1
+ adRank := 1
+ absoluteRank := 1
doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
// Skip blocks without a result heading (filters out non-organic blocks
@@ -51,6 +54,7 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
return
}
+ isAd := yandexSelectionHasAdMarker(item) || yandexURLLooksAd(href)
title := strings.TrimSpace(titleTag.Text())
if title == "" {
@@ -67,14 +71,84 @@ func parseYandexDocument(doc *goquery.Document) []core.SearchResult {
}
}
+ resultRank := rank
+ if isAd {
+ resultRank = adRank
+ adRank++
+ } else {
+ rank++
+ }
+
results = append(results, core.SearchResult{
- Rank: rank,
- URL: href,
- Title: title,
- Description: desc,
+ Rank: resultRank,
+ AbsoluteRank: absoluteRank,
+ URL: href,
+ Title: title,
+ Description: desc,
+ Ad: isAd,
})
- rank++
+ absoluteRank++
})
return core.DeduplicateResults(results)
}
+
+func yandexSelectionHasAdMarker(item *goquery.Selection) bool {
+ for _, selector := range Selectors.AdMarkers {
+ if item.Is(selector) || item.Find(selector).Length() > 0 {
+ return true
+ }
+ }
+ return false
+}
+
+func yandexURLLooksAd(raw string) bool {
+ u, err := url.Parse(raw)
+ if err != nil {
+ return false
+ }
+ host := strings.TrimPrefix(strings.ToLower(u.Hostname()), "www.")
+ return strings.HasPrefix(host, "yabs.yandex.") ||
+ host == "an.yandex.ru" ||
+ strings.HasSuffix(host, ".yandexadexchange.net")
+}
+
+func skipOrganicResults(results []core.SearchResult, skip int) []core.SearchResult {
+ if skip <= 0 {
+ return results
+ }
+ out := results[:0]
+ for _, result := range results {
+ if !result.Ad && skip > 0 {
+ skip--
+ continue
+ }
+ out = append(out, result)
+ }
+ return out
+}
+
+func rebaseOrganicRanks(results []core.SearchResult, start int) {
+ if start <= 0 {
+ return
+ }
+ organicIdx := 0
+ for i := range results {
+ if results[i].Ad {
+ continue
+ }
+ organicIdx++
+ results[i].Rank = start + organicIdx
+ }
+}
+
+func offsetAbsoluteRanks(results []core.SearchResult, start int) {
+ if start <= 0 {
+ return
+ }
+ for i := range results {
+ if results[i].AbsoluteRank > 0 {
+ results[i].AbsoluteRank += start
+ }
+ }
+}
diff --git a/yandex/parse_html_test.go b/yandex/parse_html_test.go
index 149ee00..36c39b3 100644
--- a/yandex/parse_html_test.go
+++ b/yandex/parse_html_test.go
@@ -84,3 +84,83 @@ func TestParseYandexHTMLFallbackSelectors(t *testing.T) {
t.Fatalf("unexpected description: %s", results[0].Description)
}
}
+
+func TestParseYandexHTMLAdsDoNotConsumeOrganicRank(t *testing.T) {
+ t.Parallel()
+
+ html := `
+
+ -
+
+
Paid snippet
+
+ -
+
+
Organic snippet one
+
+ -
+
+
Organic snippet two
+
+
`
+
+ results, err := ParseHTML(bytes.NewReader([]byte(html)))
+ if err != nil {
+ t.Fatalf("ParseHTML() error = %v", err)
+ }
+ if len(results) != 3 {
+ t.Fatalf("expected 3 results, got %d", len(results))
+ }
+
+ organicRank := 0
+ adRank := 0
+ for _, r := range results {
+ if r.Ad {
+ adRank++
+ if r.Rank != adRank {
+ t.Fatalf("ad rank = %d, want %d", r.Rank, adRank)
+ }
+ continue
+ }
+ organicRank++
+ if r.Rank != organicRank {
+ t.Fatalf("organic rank = %d, want %d", r.Rank, organicRank)
+ }
+ }
+ if organicRank != 2 {
+ t.Fatalf("organic count = %d, want 2", organicRank)
+ }
+ if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
+ t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
+ }
+}
+
+func TestParseYandexHTMLYabsURLIsAd(t *testing.T) {
+ t.Parallel()
+
+ html := `
+
+ -
+
+
Paid snippet
+
+ -
+
+
Organic snippet one
+
+
`
+
+ results, err := ParseHTML(bytes.NewReader([]byte(html)))
+ if err != nil {
+ t.Fatalf("ParseHTML() error = %v", err)
+ }
+ if len(results) != 2 {
+ t.Fatalf("expected 2 results, got %d", len(results))
+ }
+ if !results[0].Ad || results[0].Rank != 1 {
+ t.Fatalf("first result should be ad rank 1: %+v", results[0])
+ }
+ if results[1].Ad || results[1].Rank != 1 {
+ t.Fatalf("second result should be organic rank 1: %+v", results[1])
+ }
+}
diff --git a/yandex/search.go b/yandex/search.go
index 14f1e2f..0c0a596 100644
--- a/yandex/search.go
+++ b/yandex/search.go
@@ -71,8 +71,11 @@ func (yand *Yandex) isNoResults(page *rod.Page) bool {
func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.SearchResult {
searchResults := []core.SearchResult{}
+ organicRank := pageNum * 10
+ adRank := 1
+ absoluteRank := pageNum*10 + 1
- for i, r := range results {
+ for _, r := range results {
// Get URL
link, err := r.Element(Selectors.Link)
if err != nil {
@@ -109,13 +112,48 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
desc, _ = descTag.Text()
}
- r := core.SearchResult{Rank: (pageNum * 10) + (i + 1), URL: linkText.String(), Title: title, Description: desc}
- searchResults = append(searchResults, r)
+ hrefStr := linkText.String()
+ isAd := yandexElementHasAdMarker(r) || yandexURLLooksAd(hrefStr)
+ resultRank := 0
+ if isAd {
+ resultRank = adRank
+ adRank++
+ } else {
+ organicRank++
+ resultRank = organicRank
+ }
+
+ res := core.SearchResult{
+ Rank: resultRank,
+ AbsoluteRank: absoluteRank,
+ URL: hrefStr,
+ Title: title,
+ Description: desc,
+ Ad: isAd,
+ }
+ searchResults = append(searchResults, res)
+ absoluteRank++
}
return searchResults
}
+func yandexElementHasAdMarker(el *rod.Element) bool {
+ if el == nil {
+ return false
+ }
+ for _, selector := range Selectors.AdMarkers {
+ matches, err := el.Matches(selector)
+ if err == nil && matches {
+ return true
+ }
+ if child, err := el.Element(selector); err == nil && child != nil {
+ return true
+ }
+ }
+ return false
+}
+
func (yand *Yandex) parseImageEntities(items rod.Elements) map[string]ImageEntity {
entities := make(map[string]ImageEntity)
for _, item := range items {
@@ -193,17 +231,13 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
r := yand.parseResults(elements, searchPage)
if searchPage == startPage && skipOnFirstPage > 0 {
- if skipOnFirstPage >= len(r) {
- r = []core.SearchResult{}
- } else {
- r = r[skipOnFirstPage:]
- }
+ r = skipOrganicResults(r, skipOnFirstPage)
}
allResults = append(allResults, r...)
return false, nil
}
- for len(allResults) < query.Limit {
+ for query.Limit <= 0 || core.CountOrganicResults(allResults) < query.Limit {
done, err := fetchPage()
if err != nil {
return nil, err
@@ -212,13 +246,16 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor
if done {
break
}
+ if query.Limit > 0 && core.CountOrganicResults(allResults) >= query.Limit {
+ break
+ }
if err := core.SleepContext(ctx, yand.pageSleep); err != nil {
return nil, err
}
}
yand.logger.Info("Search completed: %d results", len(allResults))
- return core.DeduplicateResults(allResults), nil
+ return core.LimitOrganicResults(core.DeduplicateResults(allResults), query.Limit), nil
}
// SearchImage executes a Yandex image search and returns normalized image
diff --git a/yandex/search_raw.go b/yandex/search_raw.go
index d613e3e..9af173c 100644
--- a/yandex/search_raw.go
+++ b/yandex/search_raw.go
@@ -74,17 +74,10 @@ func Search(ctx context.Context, query core.Query) (results []core.SearchResult,
}
if skipOnFirstPage > 0 {
- if skipOnFirstPage >= len(parsedResults) {
- parsedResults = []core.SearchResult{}
- } else {
- parsedResults = parsedResults[skipOnFirstPage:]
- }
- }
- if query.Start > 0 {
- for i := range parsedResults {
- parsedResults[i].Rank = query.Start + i + 1
- }
+ parsedResults = skipOrganicResults(parsedResults, skipOnFirstPage)
}
+ rebaseOrganicRanks(parsedResults, query.Start)
+ offsetAbsoluteRanks(parsedResults, startPage*10)
core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
fmt.Sprintf("Yandex Raw results : %v", parsedResults),
)
diff --git a/yandex/selectors.go b/yandex/selectors.go
index 1913b94..c4fa149 100644
--- a/yandex/selectors.go
+++ b/yandex/selectors.go
@@ -2,9 +2,10 @@ package yandex
// Selectors is the single source of truth for Yandex SERP CSS selectors.
var Selectors = struct {
- Captcha string
- NoResults string
- Results string
+ Captcha string
+ NoResults string
+ Results string
+ AdMarkers []string
// LinkPrimary is preferred over a generic ; falls back to title.Closest("a")
// then the first in the result block when absent.
LinkPrimary string
@@ -16,9 +17,18 @@ var Selectors = struct {
ImageItemsAlt []string
ImageStateAll string
}{
- Captcha: "div.CheckboxCaptcha",
- NoResults: "div.EmptySearchResults",
- Results: "li[data-fast], li.serp-item",
+ Captcha: "div.CheckboxCaptcha",
+ NoResults: "div.EmptySearchResults",
+ Results: "li[data-fast], li.serp-item",
+ AdMarkers: []string{
+ "[data-fast-name='direct']",
+ "[data-fast-name='serp-adv']",
+ "[data-bem*='serp-adv']",
+ ".serp-adv-item",
+ ".serp-adv__found",
+ "[aria-label='Реклама']",
+ "[title='Реклама']",
+ },
LinkPrimary: "a.OrganicTitle-Link",
Link: "a",
Title: "h2",