diff --git a/README.md b/README.md
index f157213..a20a721 100644
--- a/README.md
+++ b/README.md
@@ -99,16 +99,16 @@ curl "http://127.0.0.1:7000/mega/engines"
Common parameters:
-| Parameter | Description | Example |
-| --------- | -------------------------- | ------------------------------------ |
-| `text` | Search query | `golang programming` |
-| `lang` | Language code | `EN`, `DE`, `RU`, `ES` |
-| `date` | Date range | `20250101..20251231` |
-| `file` | File extension | `pdf`, `doc`, `xls` |
-| `site` | Site-specific search | `github.com` |
+| Parameter | Description | Example |
+| --------- | -------------------------------------------------------------------- | ------------------------------------ |
+| `text` | Search query | `golang programming` |
+| `lang` | Language code | `EN`, `DE`, `RU`, `ES` |
+| `date` | Date range | `20250101..20251231` |
+| `file` | File extension | `pdf`, `doc`, `xls` |
+| `site` | Site-specific search | `github.com` |
| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` |
-| `start` | Pagination offset | `0`, `10`, `20` |
-| `format` | Output format | `json`, `markdown`, `text`, `ndjson` |
+| `start` | Pagination offset | `0`, `10`, `20` |
+| `format` | Output format | `json`, `markdown`, `text`, `ndjson` |
Engine-specific parameters:
@@ -130,7 +130,7 @@ Engine-specific parameters:
"requested_at": "2026-04-25T22:27:52Z",
"took_ms": 6410,
"engines_failed": [],
- "version": "2.0"
+ "version": "2.1"
},
"results": [
{
@@ -255,8 +255,12 @@ curl "http://127.0.0.1:7000/stats/cb"
This project is licensed under the MIT License. See [LICENSE](LICENSE).
-## 🤝 Contributing
+## Contributing
Contributions are welcome. See [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md).
-###### _"OpenSERP" is the name of this open-source project. The official [website](https://openserp.org) and [hosted](https://openserp.org/cloud) solution. Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._
+## Updates
+
+If you want to follow updates to the hosted version — status, changes, and occasional notes on the OSS project — join the [Telegram channel](https://t.me/+RJEKspw3mUlhZDMy).
+
+###### _"OpenSERP" is the name of this open-source project. The official [website](https://openserp.org). Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._
diff --git a/bing/parse_html.go b/bing/parse_html.go
index b7b68b9..58be47d 100644
--- a/bing/parse_html.go
+++ b/bing/parse_html.go
@@ -23,9 +23,21 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
var results []core.SearchResult
rank := 1
adRank := 1
+ absoluteRank := 1
- doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
- titleTag := item.Find(Selectors.Title).First()
+ doc.Find(Selectors.ResultItems).Each(func(_ int, item *goquery.Selection) {
+ isAd := item.Is(Selectors.Ads)
+ isOrganic := item.Is(Selectors.Results)
+ if !isAd && !isOrganic {
+ return
+ }
+
+ titleSelector := Selectors.Title
+ if isAd {
+ titleSelector = Selectors.AdTitle
+ }
+
+ titleTag := item.Find(titleSelector).First()
if titleTag.Length() == 0 {
return
}
@@ -36,69 +48,51 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
}
title := titleTag.Text()
+ if title == "" {
+ title = extractFirstText(item, Selectors.TitleFallbacks)
+ }
if title == "" {
return
}
desc := descriptionFromItem(item, title)
- results = append(results, core.SearchResult{
- Rank: rank,
- URL: link,
- Title: title,
- Description: desc,
- Ad: false,
- })
- rank++
- })
-
- doc.Find(Selectors.Ads).Each(func(_ int, item *goquery.Selection) {
- titleTag := item.Find(Selectors.AdTitle).First()
- if titleTag.Length() == 0 {
- return
- }
-
- link, exists := titleTag.Attr("href")
- if !exists || link == "" {
- return
- }
-
- title := titleTag.Text()
- desc := ""
- if descTag := item.Find("p").First(); descTag.Length() > 0 {
- desc = descTag.Text()
+ resultRank := rank
+ if isAd {
+ resultRank = adRank
+ adRank++
+ } else {
+ rank++
}
results = append(results, core.SearchResult{
- Rank: adRank,
- URL: link,
- Title: title,
- Description: desc,
- Ad: true,
+ Rank: resultRank,
+ AbsoluteRank: absoluteRank,
+ URL: link,
+ Title: title,
+ Description: desc,
+ Ad: isAd,
})
- adRank++
+ absoluteRank++
})
- setSeparatedAdAbsoluteRanks(results, 0)
return core.DeduplicateResults(results)
}
-func setSeparatedAdAbsoluteRanks(results []core.SearchResult, start int) {
- adCount := 0
- for i := range results {
- if results[i].Ad {
- adCount++
- results[i].AbsoluteRank = start + results[i].Rank
+func extractFirstText(item *goquery.Selection, selectors []string) string {
+ for _, selector := range selectors {
+ if tag := item.Find(selector).First(); tag.Length() > 0 {
+ if text := strings.TrimSpace(tag.Text()); text != "" {
+ return text
+ }
+ if label, exists := tag.Attr("aria-label"); exists {
+ if label = strings.TrimSpace(label); label != "" {
+ return label
+ }
+ }
}
}
- organicAbsoluteRank := start + adCount + 1
- for i := range results {
- if results[i].Ad {
- continue
- }
- results[i].AbsoluteRank = organicAbsoluteRank
- organicAbsoluteRank++
- }
+ return ""
}
// descriptionFromItem extracts a description using the same 4-step fallback
@@ -114,7 +108,7 @@ func descriptionFromItem(item *goquery.Selection, title string) string {
return text
}
}
- if descTag := item.Find("p").First(); descTag.Length() > 0 {
+ if descTag := item.Find(Selectors.DescAny).First(); descTag.Length() > 0 {
if text := strings.TrimSpace(descTag.Text()); text != "" {
return text
}
diff --git a/bing/parse_html_test.go b/bing/parse_html_test.go
index 3fa5ee3..3456a37 100644
--- a/bing/parse_html_test.go
+++ b/bing/parse_html_test.go
@@ -85,3 +85,63 @@ func TestParseBingHTMLAds(t *testing.T) {
}
}
}
+
+func TestParseBingHTMLMixedAdsKeepAbsoluteOrder(t *testing.T) {
+ t.Parallel()
+
+ html := `
+
+ -
+
+
+
+ -
+
+
Paid snippet
+
+ -
+
+
+
+
`
+
+ results, err := ParseHTML(bytes.NewReader([]byte(html)))
+ if err != nil {
+ t.Fatalf("ParseHTML() error = %v", err)
+ }
+ if len(results) != 3 {
+ t.Fatalf("expected 3 results, got %d", len(results))
+ }
+ if results[0].Ad || !results[1].Ad || results[2].Ad {
+ t.Fatalf("unexpected ad ordering: %+v", results)
+ }
+ if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
+ t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
+ }
+ if results[2].Rank != 2 {
+ t.Fatalf("second organic rank = %d, want 2", results[2].Rank)
+ }
+}
+
+func TestParseBingHTMLTitleFallback(t *testing.T) {
+ t.Parallel()
+
+ html := `
+
+ -
+
+
+
+
`
+
+ results, err := ParseHTML(bytes.NewReader([]byte(html)))
+ if err != nil {
+ t.Fatalf("ParseHTML() error = %v", err)
+ }
+ if len(results) != 1 {
+ t.Fatalf("expected 1 result, got %d", len(results))
+ }
+ if results[0].Title != "Fallback Title" {
+ t.Fatalf("title = %q, want fallback", results[0].Title)
+ }
+}
diff --git a/bing/search.go b/bing/search.go
index a39b78f..fa58fa9 100644
--- a/bing/search.go
+++ b/bing/search.go
@@ -77,6 +77,71 @@ func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
return core.SleepContext(ctx, 500*time.Millisecond)
}
+func bingElementMatches(el *rod.Element, selector string) bool {
+ if el == nil {
+ return false
+ }
+ matches, err := el.Matches(selector)
+ return err == nil && matches
+}
+
+func (bing *Bing) parseResultElement(el *rod.Element, isAd bool, rank, absoluteRank int) (core.SearchResult, bool) {
+ titleSelector := Selectors.Title
+ if isAd {
+ titleSelector = Selectors.AdTitle
+ }
+
+ titleElem, err := el.Element(titleSelector)
+ if err != nil {
+ bing.logger.Debug("Missing title")
+ return core.SearchResult{}, false
+ }
+
+ href, err := titleElem.Property("href")
+ if err != nil {
+ bing.logger.Debug("Missing URL")
+ return core.SearchResult{}, false
+ }
+ url := strings.TrimSpace(href.String())
+ if url == "" || url == "#" || strings.HasPrefix(url, "javascript:") {
+ return core.SearchResult{}, false
+ }
+
+ title, _ := titleElem.Text()
+ title = strings.TrimSpace(title)
+ if title == "" {
+ title = core.FirstNonEmptyText(el, Selectors.TitleFallbacks...)
+ }
+ if title == "" {
+ title = core.FirstNonEmptyAttribute(el, "aria-label", Selectors.TitleFallbacks...)
+ }
+ if title == "" {
+ bing.logger.Debug("Missing title text")
+ return core.SearchResult{}, false
+ }
+
+ desc := ""
+ if descElem, err := el.Element(Selectors.DescPrimary); err == nil {
+ desc, _ = descElem.Text()
+ } else if descElem, err := el.Element(Selectors.DescFallback); err == nil {
+ desc, _ = descElem.Text()
+ } else if descElem, err := el.Element(Selectors.DescAny); err == nil {
+ desc, _ = descElem.Text()
+ } else {
+ fullText, _ := el.Text()
+ desc = strings.TrimSpace(strings.Replace(fullText, title, "", 1))
+ }
+
+ return core.SearchResult{
+ Rank: rank,
+ AbsoluteRank: absoluteRank,
+ URL: url,
+ Title: title,
+ Description: strings.TrimSpace(desc),
+ Ad: isAd,
+ }, true
+}
+
// Search executes a Bing web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -115,7 +180,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
return nil, err
}
- organicElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, bing.GetSelectorTimeout())
+ resultElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.ResultItems, Selectors.Results}, bing.GetSelectorTimeout())
if err != nil {
// Re-check captcha on timeout - Bing interstitials can render after WaitLoad.
if bing.checkCaptcha(page) {
@@ -126,103 +191,39 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
return nil, core.ErrSearchTimeout
}
- adElements, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Ads)
- if err != nil {
- bing.logger.Debug("No ads found")
- }
-
totalResults, err := bing.getTotalResults(page)
if err != nil {
bing.logger.Debug("Failed to get total results: %v", err)
}
- bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements))
+ bing.logger.Info("Found %d organic result containers", totalResults)
rank := query.Start
adRank := 1
- for _, result := range organicElements {
- srchRes := core.SearchResult{}
-
- titleElem, err := result.Element(Selectors.Title)
- if err != nil {
- bing.logger.Debug("Missing title")
+ absoluteRank := query.Start + 1
+ for _, result := range resultElements {
+ isAd := bingElementMatches(result, Selectors.Ads)
+ isOrganic := bingElementMatches(result, Selectors.Results)
+ if !isAd && !isOrganic {
continue
}
- href, err := titleElem.Property("href")
- if err != nil {
- bing.logger.Debug("Missing URL")
+ resultRank := rank + 1
+ if isAd {
+ resultRank = adRank
+ }
+ srchRes, ok := bing.parseResultElement(result, isAd, resultRank, absoluteRank)
+ if !ok {
continue
}
- srchRes.URL = href.String()
- if strings.TrimSpace(srchRes.URL) == "" || strings.HasPrefix(srchRes.URL, "javascript:") {
- continue
- }
-
- srchRes.Title, _ = titleElem.Text()
- srchRes.Title = strings.TrimSpace(srchRes.Title)
- if srchRes.Title == "" {
- srchRes.Title = core.FirstNonEmptyText(result, Selectors.TitleFallbacks...)
- }
- if srchRes.Title == "" {
- bing.logger.Debug("Missing title text")
- continue
- }
-
- var desc string
- if descElem, err := result.Element(Selectors.DescPrimary); err == nil {
- desc, _ = descElem.Text()
- } else if descElem, err := result.Element(Selectors.DescFallback); err == nil {
- desc, _ = descElem.Text()
- } else if descElem, err := result.Element("p"); err == nil {
- desc, _ = descElem.Text()
+ searchResults = append(searchResults, srchRes)
+ absoluteRank++
+ if isAd {
+ adRank++
} else {
- fullText, _ := result.Text()
- desc = strings.TrimSpace(strings.Replace(fullText, srchRes.Title, "", 1))
+ rank++
}
- srchRes.Description = desc
-
- rank++
- srchRes.Rank = rank
- srchRes.Ad = false
-
- searchResults = append(searchResults, srchRes)
}
- for _, adResult := range adElements {
- srchRes := core.SearchResult{Ad: true}
-
- titleElem, err := adResult.Element(Selectors.AdTitle)
- if err != nil {
- bing.logger.Debug("Ad missing title")
- continue
- }
- srchRes.Title, _ = titleElem.Text()
- srchRes.Title = strings.TrimSpace(srchRes.Title)
- if srchRes.Title == "" {
- srchRes.Title = core.FirstNonEmptyText(adResult, "h2", "a[aria-label]")
- }
- if srchRes.Title == "" {
- continue
- }
-
- href, err := titleElem.Property("href")
- if err != nil {
- bing.logger.Debug("Ad missing URL")
- continue
- }
- srchRes.URL = href.String()
-
- if descElem, err := adResult.Element("p"); err == nil {
- srchRes.Description, _ = descElem.Text()
- }
-
- srchRes.Rank = adRank
- adRank++
- searchResults = append(searchResults, srchRes)
- }
-
- setSeparatedAdAbsoluteRanks(searchResults, query.Start)
-
// Deduplicate results
deduped := core.DeduplicateResults(searchResults)
diff --git a/bing/selectors.go b/bing/selectors.go
index 8c88862..7b6d5cd 100644
--- a/bing/selectors.go
+++ b/bing/selectors.go
@@ -4,6 +4,7 @@ package bing
var Selectors = struct {
Captcha []string
CookieBtn string
+ ResultItems string
Results string
Ads string
ImageResults string
@@ -11,19 +12,24 @@ var Selectors = struct {
TitleFallbacks []string
DescPrimary string
DescFallback string
+ DescAny string
AdTitle string
}{
Captcha: []string{"div.captcha", "div.captcha_header"},
CookieBtn: "button#bnp_btn_accept",
+ // ResultItems matches the main-column children only, so carousels and
+ // "related searches" cards that reuse b_algo-style markup are excluded.
+ ResultItems: "#b_results > li.b_algo, #b_results > li.b_ad",
Results: "li.b_algo",
Ads: "li.b_ad",
ImageResults: "a.iusc, div.iuscp, div.isv",
Title: "h2 a",
- // TitleFallbacks are tried by FirstNonEmptyText when the primary Title
- // selector matches but yields empty text (Bing occasionally renders an
- // empty
while the visible label sits in aria-label or h2).
+ // TitleFallbacks are tried when the primary Title selector matches but
+ // yields empty text (Bing occasionally renders an empty
+ // while the visible label sits in aria-label or h2).
TitleFallbacks: []string{"h2", "a[aria-label]"},
DescPrimary: "div.b_caption p",
DescFallback: "div.b_caption div",
+ DescAny: "p",
AdTitle: "h2 a",
}
diff --git a/cmd/root.go b/cmd/root.go
index a45a1d5..0441ce1 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -16,7 +16,7 @@ import (
)
const (
- version = "0.7.11"
+ version = "0.7.12"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)
diff --git a/core/common.go b/core/common.go
index 7236dd8..7a331de 100644
--- a/core/common.go
+++ b/core/common.go
@@ -100,6 +100,9 @@ type SearchResult struct {
Rank int `json:"rank"`
// AbsoluteRank is the 1-based position in the mixed SERP stream.
AbsoluteRank int `json:"absolute_rank,omitempty"`
+ // Type is the SERP block type when an engine can classify a non-standard
+ // SERP module without changing the public SearchEngine interface.
+ Type ResultType `json:"type,omitempty"`
// URL is the canonical result URL.
URL string `json:"url"`
// Title is the result headline shown on the SERP.
diff --git a/core/page_helpers.go b/core/page_helpers.go
index 8a9f1ab..2b644a6 100644
--- a/core/page_helpers.go
+++ b/core/page_helpers.go
@@ -123,6 +123,28 @@ func FirstNonEmptyText(root *rod.Element, selectors ...string) string {
return ""
}
+// ClosestMatching walks up the ancestor chain (including el itself) and returns
+// the first element matching selector, or nil if none is found within maxHops.
+// rod has no native Closest helper, so this is a bounded walk used by parsers
+// that need to recover a wrapping from a nested title node.
+func ClosestMatching(el *rod.Element, selector string, maxHops int) *rod.Element {
+ if el == nil || selector == "" {
+ return nil
+ }
+ current := el
+ for hop := 0; hop <= maxHops; hop++ {
+ if matches, err := current.Matches(selector); err == nil && matches {
+ return current
+ }
+ parent, err := current.Parent()
+ if err != nil || parent == nil {
+ return nil
+ }
+ current = parent
+ }
+ return nil
+}
+
// FirstNonEmptyAttribute returns the trimmed value of attr from the first
// selector under root whose attribute is non-empty.
func FirstNonEmptyAttribute(root *rod.Element, attr string, selectors ...string) string {
diff --git a/core/response.go b/core/response.go
index 2af4156..a2a5483 100644
--- a/core/response.go
+++ b/core/response.go
@@ -72,7 +72,7 @@ type ImageEnvelope struct {
Pagination Pagination `json:"pagination"`
}
-const apiVersion = "2.0"
+const apiVersion = "2.1"
// NewEnvelope builds a fresh Envelope pre-filled with query echo and an open
// meta block. Call Finalize before serializing.
diff --git a/core/response_builder.go b/core/response_builder.go
index 296bb40..928b037 100644
--- a/core/response_builder.go
+++ b/core/response_builder.go
@@ -35,11 +35,17 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
}
resultType := ResultTypeOrganic
+ if raw.Type != "" {
+ if validType, warning := ValidateResultType(raw.Type); warning == "" {
+ resultType = validType
+ }
+ }
if raw.Ad {
resultType = ResultTypeAd
}
- // Google answer boxes use negative rank; promote to answer_box type.
- if raw.Rank <= 0 && !raw.Ad {
+ // Backward compatibility for older parsers: Google answer boxes used
+ // negative rank before SearchResult carried an explicit type hint.
+ if raw.Rank <= 0 && !raw.Ad && raw.Type == "" {
resultType = ResultTypeAnswerBox
}
rank := raw.Rank
diff --git a/core/result_rank_test.go b/core/result_rank_test.go
index 1a58afc..47fc03f 100644
--- a/core/result_rank_test.go
+++ b/core/result_rank_test.go
@@ -86,6 +86,21 @@ func TestEnrichResultUsesAdRankAndAbsolutePosition(t *testing.T) {
}
}
+func TestEnrichResultUsesExplicitResultType(t *testing.T) {
+ t.Parallel()
+
+ result := EnrichResult(SearchResult{
+ Rank: 1,
+ Type: ResultTypePeopleAlsoAsk,
+ URL: "https://example.com/question",
+ Title: "Question",
+ }, EnrichContext{Engine: "google", Query: Query{Limit: 10}})
+
+ if result.Type != ResultTypePeopleAlsoAsk {
+ t.Fatalf("unexpected result type: %q", result.Type)
+ }
+}
+
func TestEnvelopePaginationCountsOrganicResults(t *testing.T) {
t.Parallel()
diff --git a/core/server_test.go b/core/server_test.go
index 179c870..38cc470 100644
--- a/core/server_test.go
+++ b/core/server_test.go
@@ -2223,8 +2223,8 @@ func TestDedicatedEndpointReturnsV2Envelope(t *testing.T) {
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
- if env.Meta.Version != "2.0" {
- t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version)
+ if env.Meta.Version != "2.1" {
+ t.Fatalf("expected meta.version=2.1, got %q", env.Meta.Version)
}
if env.Meta.RequestID == "" {
t.Fatal("expected non-empty meta.request_id")
@@ -2276,8 +2276,8 @@ func TestMegaSearchReturnsV2EnvelopeWithEnginesFailed(t *testing.T) {
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
- if env.Meta.Version != "2.0" {
- t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version)
+ if env.Meta.Version != "2.1" {
+ t.Fatalf("expected meta.version=2.1, got %q", env.Meta.Version)
}
if len(env.Meta.EnginesFailed) == 0 {
t.Fatal("expected engines_failed to contain bing")
diff --git a/docs/openapi.yaml b/docs/openapi.yaml
index 5f8b570..91b08f2 100644
--- a/docs/openapi.yaml
+++ b/docs/openapi.yaml
@@ -1,7 +1,7 @@
openapi: 3.0.3
info:
title: OpenSERP API
- version: 2.0.0
+ version: 2.1.0
description: >
OpenSERP provides dedicated and multi-engine search endpoints for Google, Yandex,
Baidu, Bing, and DuckDuckGo. Search responses are wrapped in a v2 envelope with
@@ -90,7 +90,7 @@ paths:
requested_at: "2026-04-24T12:00:00Z"
took_ms: 842
engines_failed: []
- version: "2.0"
+ version: "2.1"
results:
- id: s_a1b2c3d4e5f6a1b2
rank: 1
@@ -1023,7 +1023,7 @@ components:
$ref: "#/components/schemas/EngineErrorDetail"
version:
type: string
- example: "2.0"
+ example: "2.1"
EngineErrorDetail:
type: object
required: [engine, error]
@@ -1092,8 +1092,8 @@ components:
type: string
description: >
SERP block type. New values require a minor version bump (`meta.version: "2.1"`).
- In v2.0, engines emit `organic`, `ad`, or `answer_box`; other enum values are
- reserved for future parser upgrades.
+ In v2.1, engines may emit specialized modules such as `people_also_ask`
+ when a parser can classify them reliably.
enum:
- organic
- ad
diff --git a/google/search.go b/google/search.go
index f294a42..01fdabd 100644
--- a/google/search.go
+++ b/google/search.go
@@ -168,6 +168,18 @@ func (gogl *Google) acceptCookies(page *rod.Page) {
}
+func googleElementHasAdMarker(el *rod.Element) bool {
+ if el == nil {
+ return false
+ }
+ matches, err := el.Matches(Selectors.Ad)
+ if err == nil && matches {
+ return true
+ }
+ child, err := el.Element(Selectors.Ad)
+ return err == nil && child != nil
+}
+
// Search executes a Google web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -241,7 +253,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
for _, resEl := range searchResultElems {
srchRes := core.SearchResult{}
- isAd := core.HasAttribute(resEl, "data-text-ad")
+ isAd := googleElementHasAdMarker(resEl)
isAnswerBox := query.Answers && core.HasAttribute(resEl, "data-ulkwtsb") && !core.HasAttribute(resEl, "data-ispaa")
isResultCandidate := core.HasAttribute(resEl, "data-ved")
@@ -251,7 +263,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
srchRes.Ad = true
// Get URL
- link, err := resEl.Element("a")
+ link, err := resEl.Element(Selectors.Link)
if err != nil {
gogl.logger.Debug("Missing link")
continue
@@ -354,6 +366,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
srchRes.Title = answerText[0]
srchRes.Description = strings.Join(answerText[1:len(answerText)-2], "\n")
srchRes.Rank = -1 * (i + 1)
+ srchRes.Type = core.ResultTypePeopleAlsoAsk
searchResults = append(searchResults, srchRes)
}
continue
@@ -369,7 +382,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
// Get URL from parent link of h3
link, err := titleTag.Parent()
if err == nil {
- isLink, matchErr := link.Matches("a")
+ isLink, matchErr := link.Matches(Selectors.Link)
if matchErr != nil {
gogl.logger.Debug("Failed to match link selector: %s", matchErr)
}
@@ -399,7 +412,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
parent, err = parent.Parent()
if err == nil {
if descTag, err := parent.Next(); err == nil {
- if descDiv, err := descTag.Element("div"); err == nil {
+ if descDiv, err := descTag.Element(Selectors.DescAny); err == nil {
desc, _ = descDiv.Text()
}
}
diff --git a/google/selectors.go b/google/selectors.go
index dde4ed2..15b3bc3 100644
--- a/google/selectors.go
+++ b/google/selectors.go
@@ -9,9 +9,11 @@ var Selectors = struct {
CookieBtn string
Results string
Ad string
+ Link string
Title string
DescPrimary string
DescFallback string
+ DescAny string
AnswerBox string
AnswerItem string
@@ -26,9 +28,11 @@ var Selectors = struct {
CookieBtn: "div[role='dialog'][aria-modal] button",
Results: "div[data-hveid][data-ved]",
Ad: "div[data-text-ad], [data-text-ad]",
+ Link: "a",
Title: "h3",
DescPrimary: "div[data-sncf='1'] div",
DescFallback: "div.VwiC3b",
+ DescAny: "div",
AnswerBox: "div[data-hveid][data-ulkwtsb] div[data-q]",
AnswerItem: "a",
diff --git a/yandex/search.go b/yandex/search.go
index 0c0a596..e464aeb 100644
--- a/yandex/search.go
+++ b/yandex/search.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"sort"
+ "strings"
"time"
"github.com/go-rod/rod"
@@ -76,43 +77,44 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
absoluteRank := pageNum*10 + 1
for _, r := range results {
- // Get URL
- link, err := r.Element(Selectors.Link)
+ titleTag, err := r.Element(Selectors.Title)
if err != nil {
- if core.IsRodObjectNotFound(err) {
- break
- }
+ yand.logger.Debug("Missing h2 title")
continue
}
+ title, err := titleTag.Text()
+ if err != nil || strings.TrimSpace(title) == "" {
+ yand.logger.Debug("Failed to extract title")
+ continue
+ }
+ title = strings.TrimSpace(title)
+
+ link, err := r.Element(Selectors.LinkPrimary)
+ if err != nil {
+ if closest := core.ClosestMatching(titleTag, Selectors.Link, 4); closest != nil {
+ link = closest
+ err = nil
+ }
+ }
+ if err != nil {
+ link, err = r.Element(Selectors.Link)
+ if err != nil {
+ yand.logger.Debug("Missing link")
+ continue
+ }
+ }
linkText, err := link.Property("href")
if err != nil {
yand.logger.Debug("Missing href")
continue
}
-
- // Get title
- titleTag, err := link.Element(Selectors.Title)
- if err != nil {
- yand.logger.Debug("Missing h2 title")
+ hrefStr := strings.TrimSpace(linkText.String())
+ if hrefStr == "" || hrefStr == "#" || strings.HasPrefix(hrefStr, "javascript:") {
continue
}
- title, err := titleTag.Text()
- if err != nil {
- yand.logger.Debug("Failed to extract title")
- title = "No title"
- }
+ desc := core.FirstNonEmptyText(r, Selectors.Desc, Selectors.DescFallback)
- // Get description
- descTag, err := r.Element(Selectors.Desc)
- desc := ""
- if err != nil {
- yand.logger.Debug("No description")
- } else {
- desc, _ = descTag.Text()
- }
-
- hrefStr := linkText.String()
isAd := yandexElementHasAdMarker(r) || yandexURLLooksAd(hrefStr)
resultRank := 0
if isAd {