zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-16 21:36:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Improve SERP result classification and ad positioning

Browse Source
This commit is contained in:
Rustem Kamalov
2026-05-19 00:32:11 +03:00
parent b3ca4e0803
commit 82abe57aae
16 changed files with 318 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
README.md
View File

@@ -99,16 +99,16 @@ curl "http://127.0.0.1:7000/mega/engines"
Common parameters:
| Parameter | Description | Example |
| --------- | -------------------------- | ------------------------------------ |
| `text` | Search query | `golang programming` |
| `lang` | Language code | `EN`, `DE`, `RU`, `ES` |
| `date` | Date range | `20250101..20251231` |
| `file` | File extension | `pdf`, `doc`, `xls` |
| `site` | Site-specific search | `github.com` |
| Parameter | Description | Example |
| --------- | -------------------------------------------------------------------- | ------------------------------------ |
| `text` | Search query | `golang programming` |
| `lang` | Language code | `EN`, `DE`, `RU`, `ES` |
| `date` | Date range | `20250101..20251231` |
| `file` | File extension | `pdf`, `doc`, `xls` |
| `site` | Site-specific search | `github.com` |
| `limit` | Number of organic results, max 100. Ads may be returned in addition. | `10`, `25`, `50` |
| `start` | Pagination offset | `0`, `10`, `20` |
| `format` | Output format | `json`, `markdown`, `text`, `ndjson` |
| `start` | Pagination offset | `0`, `10`, `20` |
| `format` | Output format | `json`, `markdown`, `text`, `ndjson` |
Engine-specific parameters:
@@ -130,7 +130,7 @@ Engine-specific parameters:
"requested_at": "2026-04-25T22:27:52Z",
"took_ms": 6410,
"engines_failed": [],
"version": "2.0"
"version": "2.1"
},
"results": [
{
@@ -255,8 +255,12 @@ curl "http://127.0.0.1:7000/stats/cb"
This project is licensed under the MIT License. See [LICENSE](LICENSE).
## 🤝 Contributing
## Contributing
Contributions are welcome. See [docs/CONTRIBUTING.md](./docs/CONTRIBUTING.md).
###### _"OpenSERP" is the name of this open-source project. The official [website](https://openserp.org) and [hosted](https://openserp.org/cloud) solution. Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._
## Updates
If you want to follow updates to the hosted version — status, changes, and occasional notes on the OSS project — join the [Telegram channel](https://t.me/+RJEKspw3mUlhZDMy).
###### _"OpenSERP" is the name of this open-source project. The official [website](https://openserp.org). Use of the name in a way that implies affiliation, endorsement, or official status is not permitted._

92
bing/parse_html.go
View File

@@ -23,9 +23,21 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
var results []core.SearchResult
rank := 1
adRank := 1
absoluteRank := 1
doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) {
titleTag := item.Find(Selectors.Title).First()
doc.Find(Selectors.ResultItems).Each(func(_ int, item *goquery.Selection) {
isAd := item.Is(Selectors.Ads)
isOrganic := item.Is(Selectors.Results)
if !isAd && !isOrganic {
return
}
titleSelector := Selectors.Title
if isAd {
titleSelector = Selectors.AdTitle
}
titleTag := item.Find(titleSelector).First()
if titleTag.Length() == 0 {
return
}
@@ -36,69 +48,51 @@ func parseBingDocument(doc *goquery.Document) []core.SearchResult {
}
title := titleTag.Text()
if title == "" {
title = extractFirstText(item, Selectors.TitleFallbacks)
}
if title == "" {
return
}
desc := descriptionFromItem(item, title)
results = append(results, core.SearchResult{
Rank: rank,
URL: link,
Title: title,
Description: desc,
Ad: false,
})
rank++
})
doc.Find(Selectors.Ads).Each(func(_ int, item *goquery.Selection) {
titleTag := item.Find(Selectors.AdTitle).First()
if titleTag.Length() == 0 {
return
}
link, exists := titleTag.Attr("href")
if !exists || link == "" {
return
}
title := titleTag.Text()
desc := ""
if descTag := item.Find("p").First(); descTag.Length() > 0 {
desc = descTag.Text()
resultRank := rank
if isAd {
resultRank = adRank
adRank++
} else {
rank++
}
results = append(results, core.SearchResult{
Rank: adRank,
URL: link,
Title: title,
Description: desc,
Ad: true,
Rank: resultRank,
AbsoluteRank: absoluteRank,
URL: link,
Title: title,
Description: desc,
Ad: isAd,
})
adRank++
absoluteRank++
})
setSeparatedAdAbsoluteRanks(results, 0)
return core.DeduplicateResults(results)
}
func setSeparatedAdAbsoluteRanks(results []core.SearchResult, start int) {
adCount := 0
for i := range results {
if results[i].Ad {
adCount++
results[i].AbsoluteRank = start + results[i].Rank
func extractFirstText(item *goquery.Selection, selectors []string) string {
for _, selector := range selectors {
if tag := item.Find(selector).First(); tag.Length() > 0 {
if text := strings.TrimSpace(tag.Text()); text != "" {
return text
}
if label, exists := tag.Attr("aria-label"); exists {
if label = strings.TrimSpace(label); label != "" {
return label
}
}
}
}
organicAbsoluteRank := start + adCount + 1
for i := range results {
if results[i].Ad {
continue
}
results[i].AbsoluteRank = organicAbsoluteRank
organicAbsoluteRank++
}
return ""
}
// descriptionFromItem extracts a description using the same 4-step fallback
@@ -114,7 +108,7 @@ func descriptionFromItem(item *goquery.Selection, title string) string {
return text
}
}
if descTag := item.Find("p").First(); descTag.Length() > 0 {
if descTag := item.Find(Selectors.DescAny).First(); descTag.Length() > 0 {
if text := strings.TrimSpace(descTag.Text()); text != "" {
return text
}

60
bing/parse_html_test.go
View File

@@ -85,3 +85,63 @@ func TestParseBingHTMLAds(t *testing.T) {
}
}
}
func TestParseBingHTMLMixedAdsKeepAbsoluteOrder(t *testing.T) {
t.Parallel()
html := `
<ol id="b_results">
<li class="b_algo">
<h2><a href="https://organic.example.com/one">Organic One</a></h2>
<div class="b_caption"><p>Organic snippet one</p></div>
</li>
<li class="b_ad">
<h2><a href="https://ads.example.com">Sponsored Result</a></h2>
<p>Paid snippet</p>
</li>
<li class="b_algo">
<h2><a href="https://organic.example.com/two">Organic Two</a></h2>
<div class="b_caption"><p>Organic snippet two</p></div>
</li>
</ol>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 3 {
t.Fatalf("expected 3 results, got %d", len(results))
}
if results[0].Ad || !results[1].Ad || results[2].Ad {
t.Fatalf("unexpected ad ordering: %+v", results)
}
if results[0].AbsoluteRank != 1 || results[1].AbsoluteRank != 2 || results[2].AbsoluteRank != 3 {
t.Fatalf("unexpected absolute ranks: %d, %d, %d", results[0].AbsoluteRank, results[1].AbsoluteRank, results[2].AbsoluteRank)
}
if results[2].Rank != 2 {
t.Fatalf("second organic rank = %d, want 2", results[2].Rank)
}
}
func TestParseBingHTMLTitleFallback(t *testing.T) {
t.Parallel()
html := `
<ol id="b_results">
<li class="b_algo">
<h2><a aria-label="Fallback Title" href="https://example.com/fallback"></a></h2>
<div class="b_caption"><p>Snippet</p></div>
</li>
</ol>`
results, err := ParseHTML(bytes.NewReader([]byte(html)))
if err != nil {
t.Fatalf("ParseHTML() error = %v", err)
}
if len(results) != 1 {
t.Fatalf("expected 1 result, got %d", len(results))
}
if results[0].Title != "Fallback Title" {
t.Fatalf("title = %q, want fallback", results[0].Title)
}
}

165
bing/search.go
View File

@@ -77,6 +77,71 @@ func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
return core.SleepContext(ctx, 500*time.Millisecond)
}
func bingElementMatches(el *rod.Element, selector string) bool {
if el == nil {
return false
}
matches, err := el.Matches(selector)
return err == nil && matches
}
func (bing *Bing) parseResultElement(el *rod.Element, isAd bool, rank, absoluteRank int) (core.SearchResult, bool) {
titleSelector := Selectors.Title
if isAd {
titleSelector = Selectors.AdTitle
}
titleElem, err := el.Element(titleSelector)
if err != nil {
bing.logger.Debug("Missing title")
return core.SearchResult{}, false
}
href, err := titleElem.Property("href")
if err != nil {
bing.logger.Debug("Missing URL")
return core.SearchResult{}, false
}
url := strings.TrimSpace(href.String())
if url == "" || url == "#" || strings.HasPrefix(url, "javascript:") {
return core.SearchResult{}, false
}
title, _ := titleElem.Text()
title = strings.TrimSpace(title)
if title == "" {
title = core.FirstNonEmptyText(el, Selectors.TitleFallbacks...)
}
if title == "" {
title = core.FirstNonEmptyAttribute(el, "aria-label", Selectors.TitleFallbacks...)
}
if title == "" {
bing.logger.Debug("Missing title text")
return core.SearchResult{}, false
}
desc := ""
if descElem, err := el.Element(Selectors.DescPrimary); err == nil {
desc, _ = descElem.Text()
} else if descElem, err := el.Element(Selectors.DescFallback); err == nil {
desc, _ = descElem.Text()
} else if descElem, err := el.Element(Selectors.DescAny); err == nil {
desc, _ = descElem.Text()
} else {
fullText, _ := el.Text()
desc = strings.TrimSpace(strings.Replace(fullText, title, "", 1))
}
return core.SearchResult{
Rank: rank,
AbsoluteRank: absoluteRank,
URL: url,
Title: title,
Description: strings.TrimSpace(desc),
Ad: isAd,
}, true
}
// Search executes a Bing web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -115,7 +180,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
return nil, err
}
organicElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, bing.GetSelectorTimeout())
resultElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.ResultItems, Selectors.Results}, bing.GetSelectorTimeout())
if err != nil {
// Re-check captcha on timeout - Bing interstitials can render after WaitLoad.
if bing.checkCaptcha(page) {
@@ -126,103 +191,39 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.
return nil, core.ErrSearchTimeout
}
adElements, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Ads)
if err != nil {
bing.logger.Debug("No ads found")
}
totalResults, err := bing.getTotalResults(page)
if err != nil {
bing.logger.Debug("Failed to get total results: %v", err)
}
bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements))
bing.logger.Info("Found %d organic result containers", totalResults)
rank := query.Start
adRank := 1
for _, result := range organicElements {
srchRes := core.SearchResult{}
titleElem, err := result.Element(Selectors.Title)
if err != nil {
bing.logger.Debug("Missing title")
absoluteRank := query.Start + 1
for _, result := range resultElements {
isAd := bingElementMatches(result, Selectors.Ads)
isOrganic := bingElementMatches(result, Selectors.Results)
if !isAd && !isOrganic {
continue
}
href, err := titleElem.Property("href")
if err != nil {
bing.logger.Debug("Missing URL")
resultRank := rank + 1
if isAd {
resultRank = adRank
}
srchRes, ok := bing.parseResultElement(result, isAd, resultRank, absoluteRank)
if !ok {
continue
}
srchRes.URL = href.String()
if strings.TrimSpace(srchRes.URL) == "" || strings.HasPrefix(srchRes.URL, "javascript:") {
continue
}
srchRes.Title, _ = titleElem.Text()
srchRes.Title = strings.TrimSpace(srchRes.Title)
if srchRes.Title == "" {
srchRes.Title = core.FirstNonEmptyText(result, Selectors.TitleFallbacks...)
}
if srchRes.Title == "" {
bing.logger.Debug("Missing title text")
continue
}
var desc string
if descElem, err := result.Element(Selectors.DescPrimary); err == nil {
desc, _ = descElem.Text()
} else if descElem, err := result.Element(Selectors.DescFallback); err == nil {
desc, _ = descElem.Text()
} else if descElem, err := result.Element("p"); err == nil {
desc, _ = descElem.Text()
searchResults = append(searchResults, srchRes)
absoluteRank++
if isAd {
adRank++
} else {
fullText, _ := result.Text()
desc = strings.TrimSpace(strings.Replace(fullText, srchRes.Title, "", 1))
rank++
}
srchRes.Description = desc
rank++
srchRes.Rank = rank
srchRes.Ad = false
searchResults = append(searchResults, srchRes)
}
for _, adResult := range adElements {
srchRes := core.SearchResult{Ad: true}
titleElem, err := adResult.Element(Selectors.AdTitle)
if err != nil {
bing.logger.Debug("Ad missing title")
continue
}
srchRes.Title, _ = titleElem.Text()
srchRes.Title = strings.TrimSpace(srchRes.Title)
if srchRes.Title == "" {
srchRes.Title = core.FirstNonEmptyText(adResult, "h2", "a[aria-label]")
}
if srchRes.Title == "" {
continue
}
href, err := titleElem.Property("href")
if err != nil {
bing.logger.Debug("Ad missing URL")
continue
}
srchRes.URL = href.String()
if descElem, err := adResult.Element("p"); err == nil {
srchRes.Description, _ = descElem.Text()
}
srchRes.Rank = adRank
adRank++
searchResults = append(searchResults, srchRes)
}
setSeparatedAdAbsoluteRanks(searchResults, query.Start)
// Deduplicate results
deduped := core.DeduplicateResults(searchResults)

12
bing/selectors.go
View File

@@ -4,6 +4,7 @@ package bing
var Selectors = struct {
Captcha []string
CookieBtn string
ResultItems string
Results string
Ads string
ImageResults string
@@ -11,19 +12,24 @@ var Selectors = struct {
TitleFallbacks []string
DescPrimary string
DescFallback string
DescAny string
AdTitle string
}{
Captcha: []string{"div.captcha", "div.captcha_header"},
CookieBtn: "button#bnp_btn_accept",
// ResultItems matches the main-column children only, so carousels and
// "related searches" cards that reuse b_algo-style markup are excluded.
ResultItems: "#b_results > li.b_algo, #b_results > li.b_ad",
Results: "li.b_algo",
Ads: "li.b_ad",
ImageResults: "a.iusc, div.iuscp, div.isv",
Title: "h2 a",
// TitleFallbacks are tried by FirstNonEmptyText when the primary Title
// selector matches but yields empty text (Bing occasionally renders an
// empty <h2><a/></h2> while the visible label sits in aria-label or h2).
// TitleFallbacks are tried when the primary Title selector matches but
// yields empty text (Bing occasionally renders an empty <h2><a/></h2>
// while the visible label sits in aria-label or h2).
TitleFallbacks: []string{"h2", "a[aria-label]"},
DescPrimary: "div.b_caption p",
DescFallback: "div.b_caption div",
DescAny: "p",
AdTitle: "h2 a",
}

2
cmd/root.go
View File

@@ -16,7 +16,7 @@ import (
)
const (
version = "0.7.11"
version = "0.7.12"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
)

3
core/common.go
View File

@@ -100,6 +100,9 @@ type SearchResult struct {
Rank int `json:"rank"`
// AbsoluteRank is the 1-based position in the mixed SERP stream.
AbsoluteRank int `json:"absolute_rank,omitempty"`
// Type is the SERP block type when an engine can classify a non-standard
// SERP module without changing the public SearchEngine interface.
Type ResultType `json:"type,omitempty"`
// URL is the canonical result URL.
URL string `json:"url"`
// Title is the result headline shown on the SERP.

22
core/page_helpers.go
View File

@@ -123,6 +123,28 @@ func FirstNonEmptyText(root *rod.Element, selectors ...string) string {
return ""
}
// ClosestMatching walks up the ancestor chain (including el itself) and returns
// the first element matching selector, or nil if none is found within maxHops.
// rod has no native Closest helper, so this is a bounded walk used by parsers
// that need to recover a wrapping <a> from a nested title node.
func ClosestMatching(el *rod.Element, selector string, maxHops int) *rod.Element {
if el == nil || selector == "" {
return nil
}
current := el
for hop := 0; hop <= maxHops; hop++ {
if matches, err := current.Matches(selector); err == nil && matches {
return current
}
parent, err := current.Parent()
if err != nil || parent == nil {
return nil
}
current = parent
}
return nil
}
// FirstNonEmptyAttribute returns the trimmed value of attr from the first
// selector under root whose attribute is non-empty.
func FirstNonEmptyAttribute(root *rod.Element, attr string, selectors ...string) string {

2
core/response.go
View File

@@ -72,7 +72,7 @@ type ImageEnvelope struct {
Pagination Pagination `json:"pagination"`
}
const apiVersion = "2.0"
const apiVersion = "2.1"
// NewEnvelope builds a fresh Envelope pre-filled with query echo and an open
// meta block. Call Finalize before serializing.

10
core/response_builder.go
View File

@@ -35,11 +35,17 @@ func EnrichResult(raw SearchResult, ctx EnrichContext) Result {
}
resultType := ResultTypeOrganic
if raw.Type != "" {
if validType, warning := ValidateResultType(raw.Type); warning == "" {
resultType = validType
}
}
if raw.Ad {
resultType = ResultTypeAd
}
// Google answer boxes use negative rank; promote to answer_box type.
if raw.Rank <= 0 && !raw.Ad {
// Backward compatibility for older parsers: Google answer boxes used
// negative rank before SearchResult carried an explicit type hint.
if raw.Rank <= 0 && !raw.Ad && raw.Type == "" {
resultType = ResultTypeAnswerBox
}
rank := raw.Rank

15
core/result_rank_test.go
View File

@@ -86,6 +86,21 @@ func TestEnrichResultUsesAdRankAndAbsolutePosition(t *testing.T) {
}
}
func TestEnrichResultUsesExplicitResultType(t *testing.T) {
t.Parallel()
result := EnrichResult(SearchResult{
Rank: 1,
Type: ResultTypePeopleAlsoAsk,
URL: "https://example.com/question",
Title: "Question",
}, EnrichContext{Engine: "google", Query: Query{Limit: 10}})
if result.Type != ResultTypePeopleAlsoAsk {
t.Fatalf("unexpected result type: %q", result.Type)
}
}
func TestEnvelopePaginationCountsOrganicResults(t *testing.T) {
t.Parallel()

8
core/server_test.go
View File

@@ -2223,8 +2223,8 @@ func TestDedicatedEndpointReturnsV2Envelope(t *testing.T) {
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
if env.Meta.Version != "2.0" {
t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version)
if env.Meta.Version != "2.1" {
t.Fatalf("expected meta.version=2.1, got %q", env.Meta.Version)
}
if env.Meta.RequestID == "" {
t.Fatal("expected non-empty meta.request_id")
@@ -2276,8 +2276,8 @@ func TestMegaSearchReturnsV2EnvelopeWithEnginesFailed(t *testing.T) {
if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
t.Fatalf("decode envelope: %v", err)
}
if env.Meta.Version != "2.0" {
t.Fatalf("expected meta.version=2.0, got %q", env.Meta.Version)
if env.Meta.Version != "2.1" {
t.Fatalf("expected meta.version=2.1, got %q", env.Meta.Version)
}
if len(env.Meta.EnginesFailed) == 0 {
t.Fatal("expected engines_failed to contain bing")

10
docs/openapi.yaml
View File

@@ -1,7 +1,7 @@
openapi: 3.0.3
info:
title: OpenSERP API
version: 2.0.0
version: 2.1.0
description: >
OpenSERP provides dedicated and multi-engine search endpoints for Google, Yandex,
Baidu, Bing, and DuckDuckGo. Search responses are wrapped in a v2 envelope with
@@ -90,7 +90,7 @@ paths:
requested_at: "2026-04-24T12:00:00Z"
took_ms: 842
engines_failed: []
version: "2.0"
version: "2.1"
results:
- id: s_a1b2c3d4e5f6a1b2
rank: 1
@@ -1023,7 +1023,7 @@ components:
$ref: "#/components/schemas/EngineErrorDetail"
version:
type: string
example: "2.0"
example: "2.1"
EngineErrorDetail:
type: object
required: [engine, error]
@@ -1092,8 +1092,8 @@ components:
type: string
description: >
SERP block type. New values require a minor version bump (`meta.version: "2.1"`).
In v2.0, engines emit `organic`, `ad`, or `answer_box`; other enum values are
reserved for future parser upgrades.
In v2.1, engines may emit specialized modules such as `people_also_ask`
when a parser can classify them reliably.
enum:
- organic
- ad

21
google/search.go
View File

@@ -168,6 +168,18 @@ func (gogl *Google) acceptCookies(page *rod.Page) {
}
func googleElementHasAdMarker(el *rod.Element) bool {
if el == nil {
return false
}
matches, err := el.Matches(Selectors.Ad)
if err == nil && matches {
return true
}
child, err := el.Element(Selectors.Ad)
return err == nil && child != nil
}
// Search executes a Google web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
@@ -241,7 +253,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
for _, resEl := range searchResultElems {
srchRes := core.SearchResult{}
isAd := core.HasAttribute(resEl, "data-text-ad")
isAd := googleElementHasAdMarker(resEl)
isAnswerBox := query.Answers && core.HasAttribute(resEl, "data-ulkwtsb") && !core.HasAttribute(resEl, "data-ispaa")
isResultCandidate := core.HasAttribute(resEl, "data-ved")
@@ -251,7 +263,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
srchRes.Ad = true
// Get URL
link, err := resEl.Element("a")
link, err := resEl.Element(Selectors.Link)
if err != nil {
gogl.logger.Debug("Missing link")
continue
@@ -354,6 +366,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
srchRes.Title = answerText[0]
srchRes.Description = strings.Join(answerText[1:len(answerText)-2], "\n")
srchRes.Rank = -1 * (i + 1)
srchRes.Type = core.ResultTypePeopleAlsoAsk
searchResults = append(searchResults, srchRes)
}
continue
@@ -369,7 +382,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
// Get URL from parent link of h3
link, err := titleTag.Parent()
if err == nil {
isLink, matchErr := link.Matches("a")
isLink, matchErr := link.Matches(Selectors.Link)
if matchErr != nil {
gogl.logger.Debug("Failed to match link selector: %s", matchErr)
}
@@ -399,7 +412,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor
parent, err = parent.Parent()
if err == nil {
if descTag, err := parent.Next(); err == nil {
if descDiv, err := descTag.Element("div"); err == nil {
if descDiv, err := descTag.Element(Selectors.DescAny); err == nil {
desc, _ = descDiv.Text()
}
}

4
google/selectors.go
View File

@@ -9,9 +9,11 @@ var Selectors = struct {
CookieBtn string
Results string
Ad string
Link string
Title string
DescPrimary string
DescFallback string
DescAny string
AnswerBox string
AnswerItem string
@@ -26,9 +28,11 @@ var Selectors = struct {
CookieBtn: "div[role='dialog'][aria-modal] button",
Results: "div[data-hveid][data-ved]",
Ad: "div[data-text-ad], [data-text-ad]",
Link: "a",
Title: "h3",
DescPrimary: "div[data-sncf='1'] div",
DescFallback: "div.VwiC3b",
DescAny: "div",
AnswerBox: "div[data-hveid][data-ulkwtsb] div[data-q]",
AnswerItem: "a",

52
yandex/search.go
View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"sort"
"strings"
"time"
"github.com/go-rod/rod"
@@ -76,43 +77,44 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc
absoluteRank := pageNum*10 + 1
for _, r := range results {
// Get URL
link, err := r.Element(Selectors.Link)
titleTag, err := r.Element(Selectors.Title)
if err != nil {
if core.IsRodObjectNotFound(err) {
break
}
yand.logger.Debug("Missing h2 title")
continue
}
title, err := titleTag.Text()
if err != nil || strings.TrimSpace(title) == "" {
yand.logger.Debug("Failed to extract title")
continue
}
title = strings.TrimSpace(title)
link, err := r.Element(Selectors.LinkPrimary)
if err != nil {
if closest := core.ClosestMatching(titleTag, Selectors.Link, 4); closest != nil {
link = closest
err = nil
}
}
if err != nil {
link, err = r.Element(Selectors.Link)
if err != nil {
yand.logger.Debug("Missing link")
continue
}
}
linkText, err := link.Property("href")
if err != nil {
yand.logger.Debug("Missing href")
continue
}
// Get title
titleTag, err := link.Element(Selectors.Title)
if err != nil {
yand.logger.Debug("Missing h2 title")
hrefStr := strings.TrimSpace(linkText.String())
if hrefStr == "" || hrefStr == "#" || strings.HasPrefix(hrefStr, "javascript:") {
continue
}
title, err := titleTag.Text()
if err != nil {
yand.logger.Debug("Failed to extract title")
title = "No title"
}
desc := core.FirstNonEmptyText(r, Selectors.Desc, Selectors.DescFallback)
// Get description
descTag, err := r.Element(Selectors.Desc)
desc := ""
if err != nil {
yand.logger.Debug("No description")
} else {
desc, _ = descTag.Text()
}
hrefStr := linkText.String()
isAd := yandexElementHasAdMarker(r) || yandexURLLooksAd(hrefStr)
resultRank := 0
if isAd {