diff --git a/baidu/captcha_selector_test.go b/baidu/captcha_selector_test.go index d08885d..30f4a58 100644 --- a/baidu/captcha_selector_test.go +++ b/baidu/captcha_selector_test.go @@ -17,17 +17,17 @@ func TestBaiduPageTypeSelectors(t *testing.T) { selector string wantHit bool }{ - {"search_captcha.html", sel.Captcha, true}, - {"search_captcha.html", sel.Timeout, true}, - {"search_captcha.html", sel.Results, false}, + {"search_captcha.html", Selectors.Captcha, true}, + {"search_captcha.html", Selectors.Timeout, true}, + {"search_captcha.html", Selectors.Results, false}, - {"search_results.html", sel.Results, true}, - {"search_results.html", sel.Captcha, false}, - {"search_results.html", sel.Timeout, false}, + {"search_results.html", Selectors.Results, true}, + {"search_results.html", Selectors.Captcha, false}, + {"search_results.html", Selectors.Timeout, false}, - {"search_no_results.html", sel.Captcha, false}, - {"search_no_results.html", sel.Timeout, false}, - {"search_no_results.html", sel.Results, false}, + {"search_no_results.html", Selectors.Captcha, false}, + {"search_no_results.html", Selectors.Timeout, false}, + {"search_no_results.html", Selectors.Results, false}, } for _, tt := range tests { diff --git a/baidu/parse_html.go b/baidu/parse_html.go new file mode 100644 index 0000000..6f48d24 --- /dev/null +++ b/baidu/parse_html.go @@ -0,0 +1,60 @@ +package baidu + +import ( + "io" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/core" +) + +// ParseHTML parses a Baidu SERP HTML document and returns search results. +// No network I/O. +func ParseHTML(r io.Reader) ([]core.SearchResult, error) { + doc, err := goquery.NewDocumentFromReader(r) + if err != nil { + return nil, err + } + return parseBaiduDocument(doc), nil +} + +func parseBaiduDocument(doc *goquery.Document) []core.SearchResult { + var results []core.SearchResult + rank := 1 + + doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) { + linkTag := item.Find(Selectors.Link).First() + if linkTag.Length() == 0 { + return + } + + href, exists := linkTag.Attr("href") + if !exists || href == "" || href == "#" || !strings.HasPrefix(href, "http") { + return + } + + title := strings.TrimSpace(linkTag.Text()) + if title == "" { + return + } + + desc := "" + if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 { + desc = strings.TrimSpace(descTag.Text()) + } + if desc == "" { + full := strings.TrimSpace(item.Text()) + desc = strings.TrimSpace(strings.Replace(full, title, "", 1)) + } + + results = append(results, core.SearchResult{ + Rank: rank, + URL: href, + Title: title, + Description: desc, + }) + rank++ + }) + + return core.DeduplicateResults(results) +} diff --git a/baidu/parse_html_test.go b/baidu/parse_html_test.go new file mode 100644 index 0000000..1d076b4 --- /dev/null +++ b/baidu/parse_html_test.go @@ -0,0 +1,53 @@ +package baidu + +import ( + "bytes" + "os" + "testing" +) + +func TestParseBaiduHTML(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + + rank := 0 + for i, r := range results { + if r.Ad { + continue + } + rank++ + if r.Rank != rank { + t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, rank) + } + if r.URL == "" { + t.Fatalf("result %d: empty URL", i) + } + if r.Title == "" { + t.Fatalf("result %d: empty Title", i) + } + } + if rank == 0 { + t.Fatal("expected at least one organic result") + } +} + +func TestParseBaiduHTMLEmpty(t *testing.T) { + t.Parallel() + + results, err := ParseHTML(bytes.NewReader([]byte(""))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results for empty HTML, got %d", len(results)) + } +} diff --git a/baidu/search.go b/baidu/search.go index c59de7e..47b2e9b 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -33,15 +33,6 @@ type imageDataJson struct { } } -var sel = struct { - Captcha string - Timeout string - Results string -}{ - Captcha: "div.passMod_dialog-wrapper", - Timeout: "button.timeout-button", - Results: "div.c-container.new-pmd", -} // Baidu implements core.SearchEngine for Baidu SERP pages. type Baidu struct { @@ -71,12 +62,12 @@ func (baid *Baidu) GetRateLimiter() *rate.Limiter { } func (baid *Baidu) isCaptcha(page *rod.Page) bool { - has, _, _ := page.Has(sel.Captcha) + has, _, _ := page.Has(Selectors.Captcha) return has } func (baid *Baidu) isTimeout(page *rod.Page) bool { - has, _, _ := page.Has(sel.Timeout) + has, _, _ := page.Has(Selectors.Timeout) return has } @@ -132,7 +123,7 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core } } - searchRes, err := page.Timeout(baid.Timeout).Search(sel.Results) + searchRes, err := page.Timeout(baid.Timeout).Search(Selectors.Results) if err != nil { if blockErr := baid.classifyBlockPage(page, url); blockErr != nil { closePage() @@ -161,7 +152,7 @@ func (baid *Baidu) Search(ctx context.Context, query core.Query) (results []core for i, r := range resultElements { // Get URL - link, err := r.Element("a") + link, err := r.Element(Selectors.Link) if err != nil { if core.IsRodObjectNotFound(err) { break diff --git a/baidu/selectors.go b/baidu/selectors.go new file mode 100644 index 0000000..578d7d8 --- /dev/null +++ b/baidu/selectors.go @@ -0,0 +1,16 @@ +package baidu + +// Selectors is the single source of truth for Baidu SERP CSS selectors. +var Selectors = struct { + Captcha string + Timeout string + Results string + Link string + Desc string +}{ + Captcha: "div.passMod_dialog-wrapper", + Timeout: "button.timeout-button", + Results: "div.c-container.new-pmd", + Link: "a", + Desc: "div.c-abstract", +} diff --git a/bing/parse_html.go b/bing/parse_html.go new file mode 100644 index 0000000..5f56308 --- /dev/null +++ b/bing/parse_html.go @@ -0,0 +1,103 @@ +package bing + +import ( + "io" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/core" +) + +// ParseHTML parses a Bing SERP HTML document and returns search results. +// Mirrors the rod-based parser in search.go but operates on a goquery doc. +// No network I/O. +func ParseHTML(r io.Reader) ([]core.SearchResult, error) { + doc, err := goquery.NewDocumentFromReader(r) + if err != nil { + return nil, err + } + return parseBingDocument(doc), nil +} + +func parseBingDocument(doc *goquery.Document) []core.SearchResult { + var results []core.SearchResult + rank := 1 + + doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) { + titleTag := item.Find(Selectors.Title).First() + if titleTag.Length() == 0 { + return + } + + link, exists := titleTag.Attr("href") + if !exists || link == "" || link == "#" { + return + } + + title := titleTag.Text() + if title == "" { + return + } + + desc := descriptionFromItem(item, title) + + results = append(results, core.SearchResult{ + Rank: rank, + URL: link, + Title: title, + Description: desc, + Ad: false, + }) + rank++ + }) + + doc.Find(Selectors.Ads).Each(func(_ int, item *goquery.Selection) { + titleTag := item.Find(Selectors.AdTitle).First() + if titleTag.Length() == 0 { + return + } + + link, exists := titleTag.Attr("href") + if !exists || link == "" { + return + } + + title := titleTag.Text() + desc := "" + if descTag := item.Find(Selectors.AdDesc).First(); descTag.Length() > 0 { + desc = descTag.Text() + } + + results = append(results, core.SearchResult{ + Rank: -1, + URL: link, + Title: title, + Description: desc, + Ad: true, + }) + }) + + return core.DeduplicateResults(results) +} + +// descriptionFromItem extracts a description using the same 4-step fallback +// chain as the rod-based browser parser. +func descriptionFromItem(item *goquery.Selection, title string) string { + if descTag := item.Find(Selectors.DescPrimary).First(); descTag.Length() > 0 { + if text := strings.TrimSpace(descTag.Text()); text != "" { + return text + } + } + if descTag := item.Find(Selectors.DescFallback).First(); descTag.Length() > 0 { + if text := strings.TrimSpace(descTag.Text()); text != "" { + return text + } + } + if descTag := item.Find(Selectors.DescLast).First(); descTag.Length() > 0 { + if text := strings.TrimSpace(descTag.Text()); text != "" { + return text + } + } + // Structural fallback: strip title from full text + return strings.TrimSpace(strings.Replace(item.Text(), title, "", 1)) +} diff --git a/bing/parse_html_test.go b/bing/parse_html_test.go new file mode 100644 index 0000000..3fa5ee3 --- /dev/null +++ b/bing/parse_html_test.go @@ -0,0 +1,87 @@ +package bing + +import ( + "bytes" + "os" + "strings" + "testing" +) + +func TestParseBingHTML(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + + rank := 0 + for i, r := range results { + if r.Ad { + continue + } + rank++ + if r.Rank != rank { + t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, rank) + } + if r.URL == "" { + t.Fatalf("result %d: empty URL", i) + } + if r.Title == "" { + t.Fatalf("result %d: empty Title", i) + } + if !strings.HasPrefix(r.URL, "http") { + t.Fatalf("result %d: URL not absolute: %s", i, r.URL) + } + } + if rank == 0 { + t.Fatal("expected at least one organic result") + } +} + +func TestParseBingHTMLEmpty(t *testing.T) { + t.Parallel() + + results, err := ParseHTML(bytes.NewReader([]byte(""))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results for empty HTML, got %d", len(results)) + } +} + +func TestParseBingHTMLAds(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + + var ads []struct{ url, title string } + for _, r := range results { + if r.Ad { + ads = append(ads, struct{ url, title string }{r.URL, r.Title}) + } + } + + for i, ad := range ads { + if ad.url == "" { + t.Fatalf("ad result %d: empty URL", i) + } + if ad.title == "" { + t.Fatalf("ad result %d: empty Title", i) + } + } +} diff --git a/bing/search.go b/bing/search.go index 86ff06e..244b4ab 100644 --- a/bing/search.go +++ b/bing/search.go @@ -14,20 +14,6 @@ import ( "golang.org/x/time/rate" ) -var sel = struct { - Captcha []string - CookieBtn string - Results string - Ads string - ImageResults string -}{ - Captcha: []string{"div.captcha", "div.captcha_header"}, - CookieBtn: "button#bnp_btn_accept", - Results: "li.b_algo", - Ads: "li.b_ad", - ImageResults: "div.iuscp, div.isv", -} - // Bing implements core.SearchEngine for Bing SERP pages. type Bing struct { core.Browser @@ -56,7 +42,7 @@ func (bing *Bing) GetRateLimiter() *rate.Limiter { } func (bing *Bing) getTotalResults(page *rod.Page) (int, error) { - results, err := page.Timeout(bing.GetSelectorTimeout()).Elements(sel.Results) + results, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Results) if err != nil { return 0, errors.New("Cannot find result elements: " + err.Error()) } @@ -75,7 +61,7 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool { } } - for _, selector := range sel.Captcha { + for _, selector := range Selectors.Captcha { has, _, _ := page.Has(selector) if has { bing.logger.Debug("Captcha detected: %s", selector) @@ -87,7 +73,7 @@ func (bing *Bing) checkCaptcha(page *rod.Page) bool { } func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error { - consentBtn, err := page.Timeout(bing.Timeout / 10).Element(sel.CookieBtn) + consentBtn, err := page.Timeout(bing.Timeout / 10).Element(Selectors.CookieBtn) if err != nil { return nil } @@ -154,13 +140,13 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. return nil, core.ErrSearchTimeout } - organicElements, err := page.Timeout(bing.Timeout).Elements(sel.Results) + organicElements, err := page.Timeout(bing.Timeout).Elements(Selectors.Results) if err != nil { bing.logger.Error("Cannot parse organic results: %s", err) return nil, core.ErrParser } - adElements, err := page.Timeout(bing.Timeout).Elements(sel.Ads) + adElements, err := page.Timeout(bing.Timeout).Elements(Selectors.Ads) if err != nil { bing.logger.Debug("No ads found") } @@ -175,7 +161,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. for _, result := range organicElements { srchRes := core.SearchResult{} - titleElem, err := result.Element("a") + titleElem, err := result.Element(Selectors.Title) if err != nil { bing.logger.Debug("Missing title") continue @@ -190,11 +176,11 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. srchRes.URL = href.String() var desc string - if descElem, err := result.Element("div.b_caption p"); err == nil { + if descElem, err := result.Element(Selectors.DescPrimary); err == nil { desc, _ = descElem.Text() - } else if descElem, err := result.Element("div.b_caption div"); err == nil { + } else if descElem, err := result.Element(Selectors.DescFallback); err == nil { desc, _ = descElem.Text() - } else if descElem, err := result.Element("p"); err == nil { + } else if descElem, err := result.Element(Selectors.DescLast); err == nil { desc, _ = descElem.Text() } else { fullText, _ := result.Text() @@ -212,7 +198,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. for _, adResult := range adElements { srchRes := core.SearchResult{Ad: true} - titleElem, err := adResult.Element("h2 a") + titleElem, err := adResult.Element(Selectors.AdTitle) if err != nil { bing.logger.Debug("Ad missing title") continue @@ -226,7 +212,7 @@ func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core. } srchRes.URL = href.String() - if descElem, err := adResult.Element("p"); err == nil { + if descElem, err := adResult.Element(Selectors.AdDesc); err == nil { srchRes.Description, _ = descElem.Text() } @@ -334,7 +320,7 @@ func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.Sea } // Find all image result containers using CSS selector - imageContainers, err := page.Timeout(bing.Timeout).Elements(sel.ImageResults) + imageContainers, err := page.Timeout(bing.Timeout).Elements(Selectors.ImageResults) if err != nil { bing.logger.Error("Cannot parse image results: %s", err) return nil, core.ErrSearchTimeout diff --git a/bing/selectors.go b/bing/selectors.go new file mode 100644 index 0000000..4d9f477 --- /dev/null +++ b/bing/selectors.go @@ -0,0 +1,28 @@ +package bing + +// Selectors is the single source of truth for Bing SERP CSS selectors. +var Selectors = struct { + Captcha []string + CookieBtn string + Results string + Ads string + ImageResults string + Title string + DescPrimary string + DescFallback string + DescLast string + AdTitle string + AdDesc string +}{ + Captcha: []string{"div.captcha", "div.captcha_header"}, + CookieBtn: "button#bnp_btn_accept", + Results: "li.b_algo", + Ads: "li.b_ad", + ImageResults: "div.iuscp, div.isv", + Title: "a", + DescPrimary: "div.b_caption p", + DescFallback: "div.b_caption div", + DescLast: "p", + AdTitle: "h2 a", + AdDesc: "p", +} diff --git a/bing/testdata/search_results.html b/bing/testdata/search_results.html new file mode 100644 index 0000000..2f3b49e --- /dev/null +++ b/bing/testdata/search_results.html @@ -0,0 +1 @@ +`\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=" >http://test.test>\x3C!--pc-->pizza delivery - Search\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=" >http://test.test>\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>\x3Cscript type="importmap" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74="> { "imports": {"rms-answers-SharedStaticAssets-mdast-util-from-markdown":"http://test.test","rms-answers-SharedStaticAssets-mdast-util-gfm-table":"http://test.test","rms-answers-SharedStaticAssets-micromark-extension-gfm-table":"http://test.test","rms-answers-SharedStaticAssets-markdown-it":"http://test.test","rms-answers-SharedStaticAssets-katex":"http://test.test","rms-answers-SharedStaticAssets-docx":"http://test.test","rms-answers-SharedStaticAssets-xlsx":"http://test.test"} } \x3C/script>
About 46,900 results
Pentre, Rhondda Cynon Taf
Open links in new tab
  1. Domino's Pizza
    http://test.test

    Order Pizza Delivery & Takeaway Near You | Domino's Pizza

    Enter your postcode to find the nearest Domino's store and order pizza online. Enjoy the Tasty Trio deal with a Medium Pizza, Side and Dessert for £19.99.

    • Visit Domino's Pizza for a tasty pizza delivery or takeaway near me. Order …

    • Visit Domino's Pizza for a tasty pizza delivery near you. Order online today for …

    • I want to contact the Customer Care Team If you feel that your local store will be …

    • Discover Domino’s UK and Ireland – your favorite pizza delivery brand. From our …

    • 9. One type of base and one type of crust per whole pizza. Premium crusts, bases …

    • Looking for the best pizza discount codes and deals? We’ve got a load of …

    • At Domino’s, our vision is to be the favourite food delivery and collection brand, with …

    • Snag the Domino's student discount: Buy one pizza, get one free. Perfect for …

    See results only from dominos.co.uk
  2. Just Eat
    http://test.test › takeaway › nearme › pizza

    Pizza Near Me: Takeaways & Delivery from best …

    Order Pizza near me for delivery & takeaway. Find a wide selection of delicious …

    • 7.5/10
      (331.4K)
    • Papa Johns
      http://test.test

      Pizza Delivery & Takeaway Near You | Papa Johns

      Treat yourself to a delicious Papa Johns pizza, and browse our range of sides and desserts. Available for delivery and collection.

    • Pizza delivery

      pizza delivery

      Domino's Pizza - Treorchy

      Food delivery service
      11 High St, Treorchy
      Closed· Opens 11:00·01443 777888

      Papa John's Pizza

      Pizza
      Porthcawl
      Closed· Opens 17:00·01656 774394

      Domino's Pizza - Bridgend - Tremains Road

      Food delivery service
      1 Tremains Rd, Bridgend
      Closed· Opens 11:30·01656 668877
      feedback
    • Pizza Hut UK
      http://test.test

      Pizza Hut | Pizza Delivery

      This site is protected by reCAPTCHA and the Google Privacy Policy and Terms of Service apply. By clicking "Accept", you agree to the storing of cookies on your device, including third-party cookies, to …

    • Uber Eats
      http://test.test › gb › category › pentre-wls › pizza

      Pizza delivery in Pentre - Uber Eats

      Craving Pizza? Get it fast with your Uber account. Order online from top Pizza restaurants in Pentre.

    • thepizzaguysdeeside.com
      http://test.test › pizza-delivery-pentre

      Pizza Delivery - Pentre - The Pizza Guys Deeside

      Order online from The Pizza Guys Deeside. Pizza Delivery in Pentre. Our mouth-watering dishes are prepared with care and fresh ingredients. Go straight to our online menu and place your order. You'll …

    • rhonddatakeawayystrad.co.uk
      http://test.test › contact

      Rhondda Takeaway Ystrad - Pizza Delivery,Kebab …

      Every Monday & Tuesday Buy One Get One Free Pizza! Bon Appetit. Download our applications or use our webpage to place order.

    • PizzaExpress
      http://test.test

      Italian Restaurants Near You | Book a Table or Order …

      Discover PizzaExpress – serving handcrafted Italian pizzas across the UK. Book a table or order online for delivery or collection. Find your nearest restaurant now!

    • mrpizzakebabs.co.uk
      http://test.test

      Mr. Pizza Pentre · Online Ordering

      10" pizza, doner kebab, 6pcs hot wings, chips, salad & 2 pot of sauce. All in same box.

    • marmarischarcoalgrill.co.uk
      http://test.test

      Pizza, Fried Chicken & Kebab Delivery in Pentre | Order Online ...

      Order pizza, fried chicken & kebab in Pentre from Marmaris Grill. Fast delivery and collection available.

      • \x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>\x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=">http://test.test>
    Content was generated with AI.Learn more
      \x3Cscript type="text/javascript" nonce="r9MaibFfHKlfkoChlJGqLXJLcR/zPx2WCXADWx3zM74=" >http://test.test>` diff --git a/cmd/serve.go b/cmd/serve.go index 7162abc..ba09b0f 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "io" "net/url" "os" "os/signal" @@ -535,6 +536,17 @@ type pooledBrowserEngine struct { reportLaneStats bool } +// parsableEngine wraps pooledBrowserEngine and additionally satisfies +// core.HTMLParser for engines that have a stateless HTML parse function. +type parsableEngine struct { + *pooledBrowserEngine + parseHTMLFn func(io.Reader) ([]core.SearchResult, error) +} + +func (e *parsableEngine) ParseHTML(r io.Reader) ([]core.SearchResult, error) { + return e.parseHTMLFn(r) +} + func (e *pooledBrowserEngine) Search(ctx context.Context, q core.Query) ([]core.SearchResult, error) { engine, err := e.resolveEngine(q) if err != nil { @@ -595,9 +607,10 @@ func (e *pooledBrowserEngine) resolveEngine(q core.Query) (core.SearchEngine, er } type browserEngineSpec struct { - name string - opts core.SearchEngineOptions - factory func(core.Browser, core.SearchEngineOptions) core.SearchEngine + name string + opts core.SearchEngineOptions + factory func(core.Browser, core.SearchEngineOptions) core.SearchEngine + parseHTMLFn func(io.Reader) ([]core.SearchResult, error) } func browserEngineSpecs() []browserEngineSpec { @@ -608,6 +621,7 @@ func browserEngineSpecs() []browserEngineSpec { factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { return google.New(browser, opts) }, + parseHTMLFn: google.ParseHTML, }, { name: "yandex", @@ -615,6 +629,7 @@ func browserEngineSpecs() []browserEngineSpec { factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { return yandex.New(browser, opts) }, + parseHTMLFn: yandex.ParseHTML, }, { name: "baidu", @@ -622,6 +637,7 @@ func browserEngineSpecs() []browserEngineSpec { factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { return baidu.New(browser, opts) }, + parseHTMLFn: baidu.ParseHTML, }, { name: "bing", @@ -629,6 +645,7 @@ func browserEngineSpecs() []browserEngineSpec { factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { return bing.New(browser, opts) }, + parseHTMLFn: bing.ParseHTML, }, { name: "duckduckgo", @@ -636,6 +653,7 @@ func browserEngineSpecs() []browserEngineSpec { factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { return duckduckgo.New(browser, opts) }, + parseHTMLFn: duckduckgo.ParseHTML, }, { name: "ecosia", @@ -643,6 +661,7 @@ func browserEngineSpecs() []browserEngineSpec { factory: func(browser core.Browser, opts core.SearchEngineOptions) core.SearchEngine { return ecosia.New(browser, opts) }, + parseHTMLFn: ecosia.ParseHTML, }, } } @@ -676,14 +695,19 @@ func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ( opts := spec.opts opts.Init() - engines = append(engines, &pooledBrowserEngine{ + base := &pooledBrowserEngine{ name: spec.name, limiter: rate.NewLimiter(rate.Every(opts.GetRatelimit()), opts.RateBurst), opts: opts, factory: spec.factory, pool: pool, reportLaneStats: idx == 0, - }) + } + if spec.parseHTMLFn != nil { + engines = append(engines, &parsableEngine{pooledBrowserEngine: base, parseHTMLFn: spec.parseHTMLFn}) + } else { + engines = append(engines, base) + } } return engines, pool.close, nil diff --git a/core/html_parser.go b/core/html_parser.go new file mode 100644 index 0000000..28e6eec --- /dev/null +++ b/core/html_parser.go @@ -0,0 +1,10 @@ +package core + +import "io" + +// HTMLParser is implemented by engines that can parse a SERP HTML document +// without a live browser. Used to expose POST /parse/{engine} endpoints. +type HTMLParser interface { + Name() string + ParseHTML(io.Reader) ([]SearchResult, error) +} diff --git a/core/server.go b/core/server.go index 9069ff5..68d667e 100644 --- a/core/server.go +++ b/core/server.go @@ -1,6 +1,7 @@ package core import ( + "bytes" "context" "encoding/json" "errors" @@ -122,6 +123,7 @@ func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngin addr := fmt.Sprintf("%s:%d", host, port) app := fiber.New(fiber.Config{ ErrorHandler: JSONErrorMiddleware(), + BodyLimit: 10 * 1024 * 1024, }) serv := Server{ @@ -181,6 +183,18 @@ func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngin }) } + for _, engine := range searchEngines { + parser, ok := engine.(HTMLParser) + if !ok { + continue + } + locParser := parser + serv.app.Post(fmt.Sprintf("/%s/parse", strings.ToLower(parser.Name())), + func(c *fiber.Ctx) error { + return serv.handleParseEndpoint(c, locParser) + }) + } + serv.app.Get("/mega/search", serv.handleMegaSearch) serv.app.Get("/mega/image", serv.handleMegaImage) serv.app.Get("/mega/engines", serv.handleListEngines) @@ -333,6 +347,42 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm return sendEnvelope(c, format, env) } +func (s *Server) handleParseEndpoint(c *fiber.Ctx, parser HTMLParser) error { + startedAt := time.Now() + requestCtx := withRequestUsage(c.UserContext(), parser.Name()) + c.SetUserContext(requestCtx) + + body := c.Body() + if len(body) == 0 { + return errInvalidParam("request body is empty") + } + + format, err := resolveFormat(c) + if err != nil { + return err + } + + results, err := parser.ParseHTML(bytes.NewReader(body)) + if err != nil { + return &APIError{ + HTTPStatus: fiber.StatusBadRequest, + ErrorCode: "parser_failure", + Message: fmt.Sprintf("failed to parse HTML: %v", err), + } + } + + requestID := RequestIDFromContext(requestCtx) + q := Query{} + env := NewEnvelope(q, requestID, startedAt, []string{parser.Name()}) + ectx := EnrichContext{Engine: parser.Name(), Query: q} + for _, r := range results { + env.Results = append(env.Results, EnrichResult(r, ectx)) + } + env.Finalize(startedAt, q) + + return sendEnvelope(c, format, env) +} + type searchErrorSpec struct { status int code string diff --git a/core/server_parse_test.go b/core/server_parse_test.go new file mode 100644 index 0000000..c5b83d1 --- /dev/null +++ b/core/server_parse_test.go @@ -0,0 +1,118 @@ +package core + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +type parserMock struct { + engineMock + parseHTMLFn func(io.Reader) ([]SearchResult, error) +} + +func (p *parserMock) ParseHTML(r io.Reader) ([]SearchResult, error) { + if p.parseHTMLFn != nil { + return p.parseHTMLFn(r) + } + return []SearchResult{ + {Rank: 1, URL: "https://example.com/1", Title: "Result One", Description: "Desc one"}, + {Rank: 2, URL: "https://example.com/2", Title: "Result Two", Description: "Desc two"}, + }, nil +} + +func postHTML(t *testing.T, s *Server, path, body string) *http.Response { + t.Helper() + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "text/html") + resp, err := s.app.Test(req, -1) + if err != nil { + t.Fatalf("POST %s failed: %v", path, err) + } + return resp +} + +func TestParseEndpointReturnsEnvelope(t *testing.T) { + engine := &parserMock{engineMock: engineMock{name: "google", initialized: true}} + srv := NewServerWithOptions("127.0.0.1", 7120, DefaultServerOptions(), engine) + + resp := postHTML(t, srv, "/google/parse", "sample serp") + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + var env Envelope + if err := json.NewDecoder(resp.Body).Decode(&env); err != nil { + t.Fatalf("decode envelope: %v", err) + } + if len(env.Results) != 2 { + t.Fatalf("expected 2 results, got %d", len(env.Results)) + } + if env.Results[0].URL != "https://example.com/1" { + t.Fatalf("unexpected first result URL: %s", env.Results[0].URL) + } +} + +func TestParseEndpointEmptyBodyReturns400(t *testing.T) { + engine := &parserMock{engineMock: engineMock{name: "google", initialized: true}} + srv := NewServerWithOptions("127.0.0.1", 7121, DefaultServerOptions(), engine) + + req := httptest.NewRequest(http.MethodPost, "/google/parse", bytes.NewReader(nil)) + req.Header.Set("Content-Type", "text/html") + resp, err := srv.app.Test(req, -1) + if err != nil { + t.Fatalf("request failed: %v", err) + } + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400, got %d", resp.StatusCode) + } +} + +func TestParseEndpointParserErrorReturns400(t *testing.T) { + engine := &parserMock{ + engineMock: engineMock{name: "bing", initialized: true}, + parseHTMLFn: func(_ io.Reader) ([]SearchResult, error) { + return nil, io.ErrUnexpectedEOF + }, + } + srv := NewServerWithOptions("127.0.0.1", 7122, DefaultServerOptions(), engine) + + resp := postHTML(t, srv, "/bing/parse", "bad") + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400, got %d", resp.StatusCode) + } +} + +func TestParseEndpointNotRegisteredForNonParserEngine(t *testing.T) { + // engineMock does NOT implement HTMLParser so no /mock/parse route is registered. + engine := &engineMock{name: "mock", initialized: true} + srv := NewServerWithOptions("127.0.0.1", 7123, DefaultServerOptions(), engine) + + resp := postHTML(t, srv, "/mock/parse", "") + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected 404 for non-parser engine, got %d", resp.StatusCode) + } +} + +func TestParseEndpointMarkdownFormat(t *testing.T) { + engine := &parserMock{engineMock: engineMock{name: "google", initialized: true}} + srv := NewServerWithOptions("127.0.0.1", 7124, DefaultServerOptions(), engine) + + req := httptest.NewRequest(http.MethodPost, "/google/parse?format=markdown", strings.NewReader("serp")) + req.Header.Set("Content-Type", "text/html") + resp, err := srv.app.Test(req, -1) + if err != nil { + t.Fatalf("request failed: %v", err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + ct := resp.Header.Get("Content-Type") + if !strings.Contains(ct, "text/markdown") { + t.Fatalf("expected markdown content type, got %s", ct) + } +} diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 696372f..e0e9c7d 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -213,6 +213,84 @@ paths: $ref: "#/components/responses/NotFoundError" "500": $ref: "#/components/responses/InternalServerError" + /google/parse: + post: + tags: [Search] + operationId: parseGoogleHTML + summary: Parse a Google SERP HTML document into structured results + description: > + Accepts raw Google SERP HTML in the request body and returns a standard + search envelope. Useful when an upstream provider delivers raw HTML + rather than JSON. No browser is used; parsing is done with goquery. + The body size limit is 10 MB. + requestBody: + required: true + content: + text/html: + schema: + type: string + description: Raw Google SERP HTML page + parameters: + - $ref: "#/components/parameters/FormatQuery" + responses: + "200": + description: Parsed search results envelope + content: + application/json: + schema: + $ref: "#/components/schemas/SearchEnvelope" + text/markdown: + schema: + type: string + text/plain: + schema: + type: string + application/x-ndjson: + schema: + type: string + "400": + $ref: "#/components/responses/BadRequestError" + "500": + $ref: "#/components/responses/InternalServerError" + /bing/parse: + post: + tags: [Search] + operationId: parseBingHTML + summary: Parse a Bing SERP HTML document into structured results + description: > + Accepts raw Bing SERP HTML in the request body and returns a standard + search envelope. Useful when an upstream provider delivers raw HTML + rather than JSON. No browser is used; parsing is done with goquery. + The body size limit is 10 MB. + requestBody: + required: true + content: + text/html: + schema: + type: string + description: Raw Bing SERP HTML page + parameters: + - $ref: "#/components/parameters/FormatQuery" + responses: + "200": + description: Parsed search results envelope + content: + application/json: + schema: + $ref: "#/components/schemas/SearchEnvelope" + text/markdown: + schema: + type: string + text/plain: + schema: + type: string + application/x-ndjson: + schema: + type: string + "400": + $ref: "#/components/responses/BadRequestError" + "500": + $ref: "#/components/responses/InternalServerError" /mega/search: get: tags: [Mega] diff --git a/duckduckgo/parse_html.go b/duckduckgo/parse_html.go new file mode 100644 index 0000000..a169170 --- /dev/null +++ b/duckduckgo/parse_html.go @@ -0,0 +1,109 @@ +package duckduckgo + +import ( + "io" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/core" +) + +// ParseHTML parses a DuckDuckGo SERP HTML document and returns search results. +// No network I/O. +func ParseHTML(r io.Reader) ([]core.SearchResult, error) { + doc, err := goquery.NewDocumentFromReader(r) + if err != nil { + return nil, err + } + return parseDDGDocument(doc), nil +} + +func parseDDGDocument(doc *goquery.Document) []core.SearchResult { + var results []core.SearchResult + rank := 1 + + resultSel := firstMatchingSelector(doc, Selectors.Results) + if resultSel == "" { + return results + } + + doc.Find(resultSel).Each(func(_ int, item *goquery.Selection) { + href := extractFirstAttr(item, Selectors.Link, "href") + if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") { + return + } + + title := extractFirstText(item, Selectors.Title) + if title == "" { + return + } + + desc := extractFirstText(item, Selectors.Desc) + + isAd := false + for _, sel := range Selectors.AdBadge { + if item.Find(sel).Length() > 0 { + isAd = true + break + } + } + + r := core.SearchResult{ + Rank: rank, + URL: href, + Title: title, + Description: desc, + Ad: isAd, + } + if !isAd { + rank++ + } else { + r.Rank = -1 + } + results = append(results, r) + }) + + return core.DeduplicateResults(results) +} + +// firstMatchingSelector returns the first selector from the list that matches +// at least one element in the document. +func firstMatchingSelector(doc *goquery.Document, selectors []string) string { + for _, sel := range selectors { + if doc.Find(sel).Length() > 0 { + return sel + } + } + return "" +} + +// extractFirstAttr tries each selector in order and returns the named attribute +// of the first match, or "". +func extractFirstAttr(item *goquery.Selection, selectors []string, attr string) string { + for _, sel := range selectors { + tag := item.Find(sel).First() + if tag.Length() == 0 { + continue + } + val, exists := tag.Attr(attr) + if exists && val != "" { + return strings.TrimSpace(val) + } + } + return "" +} + +// extractFirstText tries each selector in order and returns the trimmed text of +// the first match, or "". +func extractFirstText(item *goquery.Selection, selectors []string) string { + for _, sel := range selectors { + tag := item.Find(sel).First() + if tag.Length() == 0 { + continue + } + if text := strings.TrimSpace(tag.Text()); text != "" { + return text + } + } + return "" +} diff --git a/duckduckgo/parse_html_test.go b/duckduckgo/parse_html_test.go new file mode 100644 index 0000000..7eac92f --- /dev/null +++ b/duckduckgo/parse_html_test.go @@ -0,0 +1,57 @@ +package duckduckgo + +import ( + "bytes" + "os" + "strings" + "testing" +) + +func TestParseDDGHTML(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + + rank := 0 + for i, r := range results { + if r.Ad { + continue + } + rank++ + if r.Rank != rank { + t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, rank) + } + if r.URL == "" { + t.Fatalf("result %d: empty URL", i) + } + if r.Title == "" { + t.Fatalf("result %d: empty Title", i) + } + if !strings.HasPrefix(r.URL, "http") { + t.Fatalf("result %d: URL not absolute: %s", i, r.URL) + } + } + if rank == 0 { + t.Fatal("expected at least one organic result") + } +} + +func TestParseDDGHTMLEmpty(t *testing.T) { + t.Parallel() + + results, err := ParseHTML(bytes.NewReader([]byte(""))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results for empty HTML, got %d", len(results)) + } +} diff --git a/duckduckgo/search.go b/duckduckgo/search.go index d0645e6..1f6ad03 100644 --- a/duckduckgo/search.go +++ b/duckduckgo/search.go @@ -15,18 +15,6 @@ import ( // a plain-text 202 rate-limit response rather than a structured captcha page. const captchaBodyText = "bots user" -var sel = struct { - NoResults string - Results []string -}{ - NoResults: "div[class*='no-results']", - Results: []string{ - "article[data-testid='result']", - "div.result", - "div[data-testid='result']", - }, -} - // DuckDuckGo implements core.SearchEngine for DuckDuckGo SERP pages. type DuckDuckGo struct { core.Browser @@ -66,7 +54,7 @@ func (ddg *DuckDuckGo) isCaptcha(page *rod.Page) bool { } func (ddg *DuckDuckGo) isNoResults(page *rod.Page) bool { - has, _, _ := page.Has(sel.NoResults) + has, _, _ := page.Has(Selectors.NoResults) return has } @@ -78,17 +66,7 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se var link *rod.Element var err error - linkSelectors := []string{ - "a[data-testid='result-title-a']", - "a.result__a", - "a.result__url", - "h2 a", - "h3 a", - "a[href]", - "a", - } - - for _, selector := range linkSelectors { + for _, selector := range Selectors.Link { link, err = r.Element(selector) if err == nil { break @@ -118,15 +96,7 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se // Get title - try multiple selectors var titleTag *rod.Element - titleSelectors := []string{ - "h2", - ".result__title", - ".result__a", - "span", - "div", - } - - for _, selector := range titleSelectors { + for _, selector := range Selectors.Title { titleTag, err = r.Element(selector) if err == nil { break @@ -140,16 +110,7 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se // Get description - try multiple selectors desc := "" - descSelectors := []string{ - "div[data-result='snippet']", - ".result__snippet", - ".result__body", - "span[class*='snippet']", - "div[class*='snippet']", - "p", - } - - for _, selector := range descSelectors { + for _, selector := range Selectors.Desc { descTag, err := r.Element(selector) if err == nil { desc, _ = descTag.Text() @@ -159,13 +120,7 @@ func (ddg *DuckDuckGo) parseResults(results rod.Elements, pageNum int) []core.Se // Check if it's an ad isAd := false - adSelectors := []string{ - "[data-testid='ad-badge']", - ".ad-badge", - ".result--ad", - } - - for _, selector := range adSelectors { + for _, selector := range Selectors.AdBadge { adIndicator, err := r.Element(selector) if err == nil && adIndicator != nil { isAd = true @@ -230,17 +185,7 @@ func (ddg *DuckDuckGo) Search(ctx context.Context, query core.Query) (results [] var searchRes *rod.SearchResult var searchErr error - // Try different selectors for DuckDuckGo results - selectors := []string{ - "article[data-testid='result']", - "div[data-testid='result']", - "div.result", - "div.web-result", - ".result", - "[data-testid='result']", - } - - for _, selector := range selectors { + for _, selector := range Selectors.Results { searchRes, searchErr = page.Timeout(ddg.GetSelectorTimeout()).Search(selector) if searchErr == nil && searchRes != nil { ddg.logger.Debug("Found results with selector: %s", selector) @@ -355,19 +300,7 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor var searchRes *rod.SearchResult var searchErr error - selectors := []string{ - "figure", - // "figure.nsogf_Hpj9UUxfhcwQd5", - // "div[data-testid='result']", - // "div.tile--img", - // "div.tile.tile--img", - // "div.js-images-show-more", - // "div.img-result", - } - - ddg.logger.Debug("Trying selectors: %v", selectors) - - for _, selector := range selectors { + for _, selector := range Selectors.ImageResult { searchRes, searchErr = page.Timeout(ddg.GetSelectorTimeout()).Search(selector) if searchErr == nil && searchRes != nil { ddg.logger.Debug("Found image results with selector: %s", selector) @@ -406,13 +339,7 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor var imgTag *rod.Element var imgErr error - imgSelectors := []string{ - "img", - "div.SZ76bwIlqO8BBoqOLqYV img", - "img[src*='duckduckgo.com']", - } - - for _, selector := range imgSelectors { + for _, selector := range Selectors.ImageImg { imgTag, imgErr = r.Element(selector) if imgErr == nil { break @@ -434,17 +361,7 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor var titleTag *rod.Element var titleErr error - titleSelectors := []string{ - "figcaption a p span", - "figcaption span", - "figcaption p span", - "span.EKtkFWMYpwzMKOYr0GYm", - "h3", - "span", - "p", - } - - for _, selector := range titleSelectors { + for _, selector := range Selectors.ImageTitle { titleTag, titleErr = r.Element(selector) if titleErr == nil { break @@ -460,12 +377,7 @@ func (ddg *DuckDuckGo) SearchImage(ctx context.Context, query core.Query) ([]cor var linkTag *rod.Element var linkErr error - linkSelectors := []string{ - "figcaption a", - "a", - } - - for _, selector := range linkSelectors { + for _, selector := range Selectors.ImageLink { linkTag, linkErr = r.Element(selector) if linkErr == nil { break diff --git a/duckduckgo/selectors.go b/duckduckgo/selectors.go new file mode 100644 index 0000000..c2a9987 --- /dev/null +++ b/duckduckgo/selectors.go @@ -0,0 +1,62 @@ +package duckduckgo + +// Selectors is the single source of truth for DuckDuckGo SERP CSS selectors. +var Selectors = struct { + NoResults string + Results []string + Title []string + Desc []string + Link []string + AdBadge []string + ImageResult []string + ImageImg []string + ImageTitle []string + ImageLink []string +}{ + NoResults: "div[class*='no-results']", + Results: []string{ + "article[data-testid='result']", + "div.result", + "div[data-testid='result']", + }, + Title: []string{ + "h2", + ".result__title", + ".result__a", + }, + Desc: []string{ + "div[data-result='snippet']", + ".result__snippet", + ".result__body", + }, + Link: []string{ + "a[data-testid='result-title-a']", + "a.result__a", + "h2 a", + "h3 a", + }, + AdBadge: []string{ + "[data-testid='ad-badge']", + ".ad-badge", + ".result--ad", + }, + ImageResult: []string{ + "figure", + }, + ImageImg: []string{ + "img", + "img[src*='duckduckgo.com']", + }, + ImageTitle: []string{ + "figcaption a p span", + "figcaption span", + "figcaption p span", + "h3", + "span", + "p", + }, + ImageLink: []string{ + "figcaption a", + "a", + }, +} diff --git a/duckduckgo/testdata/search_results.html b/duckduckgo/testdata/search_results.html new file mode 100644 index 0000000..f86589b --- /dev/null +++ b/duckduckgo/testdata/search_results.html @@ -0,0 +1 @@ +open serp at DuckDuckGo

      2. Github

        http://test.test› karust › openserp

        OpenSERP (Search Engine Results) - GitHub

        OpenSERPis an API and CLI for accessing search engine results from Google, Yandex, Baidu, Bing, and DuckDuckGo. A developer-friendly alternative to paidSERPAPI services! Official website:openserp.org 💡OpenSerpis free andopen-source. Only links listed in this repository and on the official website are associated with the project.
        1. Request parameters
        2. Search
        3. Example request
        4. Get 20 Google results for hello world, only in English: You can replace google to yandex or baidu in query to change search engine. |
        5. Example response
        6. Images
        7. Example request
        8. Get 100 Google results for golden puppy:
        9. Example response
        See more on github.com
      11. Generate answer foropen serp

      Get Our Windows Browser

      Protect your data as you search, browse, and use AI.

      Download
      Custom date rangeX
      diff --git a/ecosia/parse_html.go b/ecosia/parse_html.go new file mode 100644 index 0000000..cf45678 --- /dev/null +++ b/ecosia/parse_html.go @@ -0,0 +1,134 @@ +package ecosia + +import ( + "fmt" + "io" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/core" +) + +// ParseHTML parses an Ecosia SERP HTML document and returns search results. +// No network I/O. +func ParseHTML(r io.Reader) ([]core.SearchResult, error) { + doc, err := goquery.NewDocumentFromReader(r) + if err != nil { + return nil, err + } + return parseEcosiaDocument(doc), nil +} + +func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult { + var results []core.SearchResult + rank := 1 + + doc.Find(Selectors.Result).Each(func(_ int, item *goquery.Selection) { + res, ok := parseEcosiaItem(item, rank, false) + if !ok { + return + } + results = append(results, res) + rank++ + }) + + doc.Find(Selectors.Ad).Each(func(_ int, item *goquery.Selection) { + res, ok := parseEcosiaItem(item, -1, true) + if !ok { + return + } + results = append(results, res) + }) + + return core.DeduplicateResults(results) +} + +func parseEcosiaItem(item *goquery.Selection, rank int, ad bool) (core.SearchResult, bool) { + linkTag := item.Find(Selectors.ResultLink).First() + if linkTag.Length() == 0 { + linkTag = item.Find("a[href]").First() + } + if linkTag.Length() == 0 { + return core.SearchResult{}, false + } + + href, exists := linkTag.Attr("href") + if !exists { + return core.SearchResult{}, false + } + href = strings.TrimSpace(href) + if href == "" || strings.HasPrefix(href, "javascript:") { + return core.SearchResult{}, false + } + + title := "" + if t := item.Find(Selectors.Title).First(); t.Length() > 0 { + title = strings.TrimSpace(t.Text()) + } + if title == "" { + if t := item.Find("h2, h3").First(); t.Length() > 0 { + title = strings.TrimSpace(t.Text()) + } + } + + desc := "" + if d := item.Find(Selectors.Desc).First(); d.Length() > 0 { + desc = strings.TrimSpace(d.Text()) + } + + return core.SearchResult{ + Rank: rank, + URL: href, + Title: title, + Description: desc, + Ad: ad, + }, true +} + +// parseEcosiaImageItem extracts a single image card from a goquery Selection. +func parseEcosiaImageItem(item *goquery.Selection, rank int) (core.SearchResult, bool) { + linkTag := item.Find(Selectors.ImageLink).First() + if linkTag.Length() == 0 { + return core.SearchResult{}, false + } + href, exists := linkTag.Attr("href") + if !exists { + return core.SearchResult{}, false + } + imgURL := strings.TrimSpace(href) + if imgURL == "" { + return core.SearchResult{}, false + } + + title := "" + if img := linkTag.Find("img").First(); img.Length() > 0 { + if alt, err := img.Attr("alt"); err { + title = strings.TrimSpace(alt) + } + } + + source := "" + if s := item.Find(Selectors.ImageSource).First(); s.Length() > 0 { + source = strings.TrimSpace(s.Text()) + } + dims := "" + if d := item.Find(Selectors.ImageDims).First(); d.Length() > 0 { + dims = strings.TrimSpace(d.Text()) + } + + desc := source + if dims != "" { + if source != "" { + desc = fmt.Sprintf("%s (%s)", source, dims) + } else { + desc = dims + } + } + + return core.SearchResult{ + Rank: rank, + URL: imgURL, + Title: title, + Description: desc, + }, true +} diff --git a/ecosia/parse_html_test.go b/ecosia/parse_html_test.go new file mode 100644 index 0000000..149859f --- /dev/null +++ b/ecosia/parse_html_test.go @@ -0,0 +1,83 @@ +package ecosia + +import ( + "bytes" + "os" + "strings" + "testing" +) + +func TestParseEcosiaHTML(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + + rank := 0 + for i, r := range results { + if r.Ad { + continue + } + rank++ + if r.Rank != rank { + t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, rank) + } + if r.URL == "" { + t.Fatalf("result %d: empty URL", i) + } + if r.Title == "" { + t.Fatalf("result %d: empty Title", i) + } + if !strings.HasPrefix(r.URL, "http") { + t.Fatalf("result %d: URL not absolute: %s", i, r.URL) + } + } + if rank == 0 { + t.Fatal("expected at least one organic result") + } +} + +func TestParseEcosiaHTMLEmpty(t *testing.T) { + t.Parallel() + + results, err := ParseHTML(bytes.NewReader([]byte(""))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results for empty HTML, got %d", len(results)) + } +} + +func TestParseEcosiaHTMLAds(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + + for i, r := range results { + if !r.Ad { + continue + } + if r.URL == "" { + t.Fatalf("ad result %d: empty URL", i) + } + if r.Title == "" { + t.Fatalf("ad result %d: empty Title", i) + } + } +} diff --git a/ecosia/search.go b/ecosia/search.go index f958c7d..041dde4 100644 --- a/ecosia/search.go +++ b/ecosia/search.go @@ -43,32 +43,6 @@ const ( cfBodyMarker = "not a bot" ) -// Ecosia's SERP DOM shape varies by the underlying provider chosen per market -// (Bing, Google, or EUSP per Ecosia's search-features doc). The data-test-id -// attributes are the most stable surface across providers; class names drift. -var sel = struct { - Mainline string - Result string - Ad string - ResultLink string - Title string - Desc string - ImageResult string - ImageLink string - ImageSource string - ImageDims string -}{ - Mainline: "[data-test-id='mainline']", - Result: "[data-test-id='mainline-result-web']", - Ad: "[data-test-id='mainline-result-ad']", - ResultLink: "[data-test-id='result-link']", - Title: "[data-test-id='result-title']", - Desc: "[data-test-id='result-description']", - ImageResult: "[data-test-id='images-result']", - ImageLink: "[data-test-id='image-result-link']", - ImageSource: "[data-test-id='image-result-source']", - ImageDims: "[data-test-id='image-result-dimensions']", -} // Ecosia implements core.SearchEngine for Ecosia SERP pages. Additional // documentation at https://support.ecosia.org/article/447-search-features. @@ -117,7 +91,7 @@ func (e *Ecosia) isCaptcha(page *rod.Page) bool { } func (e *Ecosia) parseResult(elem *rod.Element, rank int, ad bool) (core.SearchResult, bool) { - link, err := elem.Element(sel.ResultLink) + link, err := elem.Element(Selectors.ResultLink) if err != nil { // Fall back to the first anchor when the test-id selector is absent. link, err = elem.Element("a[href]") @@ -135,14 +109,14 @@ func (e *Ecosia) parseResult(elem *rod.Element, rank int, ad bool) (core.SearchR } title := "" - if t, err := elem.Element(sel.Title); err == nil { + if t, err := elem.Element(Selectors.Title); err == nil { title, _ = t.Text() } else if t, err := elem.Element("h2, h3"); err == nil { title, _ = t.Text() } desc := "" - if d, err := elem.Element(sel.Desc); err == nil { + if d, err := elem.Element(Selectors.Desc); err == nil { desc, _ = d.Text() } @@ -206,7 +180,7 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S return nil, core.ErrSearchTimeout } - if _, err := page.Timeout(e.GetSelectorTimeout()).Element(sel.Mainline); err != nil { + if _, err := page.Timeout(e.GetSelectorTimeout()).Element(Selectors.Mainline); err != nil { if e.isCaptcha(page) { closePage() e.logger.Error("Captcha detected: %s", u) @@ -217,8 +191,8 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S break } - organic, _ := page.Elements(sel.Result) - ads, _ := page.Elements(sel.Ad) + organic, _ := page.Elements(Selectors.Result) + ads, _ := page.Elements(Selectors.Ad) if len(organic) == 0 && len(ads) == 0 { // Empty mainline = zero-result query or end of pagination, not // a parser failure; don't trip the retry path. @@ -266,7 +240,7 @@ func (e *Ecosia) Search(ctx context.Context, query core.Query) (results []core.S // parseImageResult extracts a single image card into a SearchResult, // returning (_, false) if the card lacks a usable image URL. func (e *Ecosia) parseImageResult(el *rod.Element, rank int) (core.SearchResult, bool) { - link, err := el.Element(sel.ImageLink) + link, err := el.Element(Selectors.ImageLink) if err != nil { return core.SearchResult{}, false } @@ -287,12 +261,12 @@ func (e *Ecosia) parseImageResult(el *rod.Element, rank int) (core.SearchResult, } source := "" - if s, err := el.Element(sel.ImageSource); err == nil { + if s, err := el.Element(Selectors.ImageSource); err == nil { source, _ = s.Text() source = strings.TrimSpace(source) } dims := "" - if d, err := el.Element(sel.ImageDims); err == nil { + if d, err := el.Element(Selectors.ImageDims); err == nil { dims, _ = d.Text() dims = strings.TrimSpace(dims) } @@ -362,7 +336,7 @@ func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []c return nil, core.ErrSearchTimeout } - if _, err := page.Timeout(e.GetSelectorTimeout()).Element(sel.ImageResult); err != nil { + if _, err := page.Timeout(e.GetSelectorTimeout()).Element(Selectors.ImageResult); err != nil { if e.isCaptcha(page) { closePage() e.logger.Error("Captcha detected: %s", u) @@ -373,7 +347,7 @@ func (e *Ecosia) SearchImage(ctx context.Context, query core.Query) (results []c break } - elements, err := page.Elements(sel.ImageResult) + elements, err := page.Elements(Selectors.ImageResult) if err != nil { closePage() e.logger.Error("Cannot collect image results: %s", err) diff --git a/ecosia/search_raw.go b/ecosia/search_raw.go index 8c3383b..04074f1 100644 --- a/ecosia/search_raw.go +++ b/ecosia/search_raw.go @@ -42,14 +42,14 @@ func resultParser(response *http.Response) ([]core.SearchResult, error) { results []core.SearchResult rank = 1 ) - doc.Find(sel.Result).Each(func(_ int, s *goquery.Selection) { - href, ok := s.Find(sel.ResultLink).Attr("href") + doc.Find(Selectors.Result).Each(func(_ int, s *goquery.Selection) { + href, ok := s.Find(Selectors.ResultLink).Attr("href") if !ok || strings.TrimSpace(href) == "" { return } var ( - title = strings.TrimSpace(s.Find(sel.Title).Text()) - desc = strings.TrimSpace(s.Find(sel.Desc).Text()) + title = strings.TrimSpace(s.Find(Selectors.Title).Text()) + desc = strings.TrimSpace(s.Find(Selectors.Desc).Text()) ) results = append(results, core.SearchResult{ Rank: rank, @@ -60,14 +60,14 @@ func resultParser(response *http.Response) ([]core.SearchResult, error) { rank++ }) adRank := -1 - doc.Find(sel.Ad).Each(func(_ int, s *goquery.Selection) { - href, ok := s.Find(sel.ResultLink).Attr("href") + doc.Find(Selectors.Ad).Each(func(_ int, s *goquery.Selection) { + href, ok := s.Find(Selectors.ResultLink).Attr("href") if !ok || strings.TrimSpace(href) == "" { return } var ( - title = strings.TrimSpace(s.Find(sel.Title).Text()) - desc = strings.TrimSpace(s.Find(sel.Desc).Text()) + title = strings.TrimSpace(s.Find(Selectors.Title).Text()) + desc = strings.TrimSpace(s.Find(Selectors.Desc).Text()) ) results = append(results, core.SearchResult{ Rank: adRank, @@ -92,16 +92,16 @@ func imageResultParser(response *http.Response) ([]core.SearchResult, error) { results []core.SearchResult rank = 1 ) - doc.Find(sel.ImageResult).Each(func(_ int, s *goquery.Selection) { - href, ok := s.Find(sel.ImageLink).Attr("href") + doc.Find(Selectors.ImageResult).Each(func(_ int, s *goquery.Selection) { + href, ok := s.Find(Selectors.ImageLink).Attr("href") if !ok || strings.TrimSpace(href) == "" { return } - title, _ := s.Find(sel.ImageLink).Find("img").Attr("alt") + title, _ := s.Find(Selectors.ImageLink).Find("img").Attr("alt") title = strings.TrimSpace(title) var ( - source = strings.TrimSpace(s.Find(sel.ImageSource).Text()) - dims = strings.TrimSpace(s.Find(sel.ImageDims).Text()) + source = strings.TrimSpace(s.Find(Selectors.ImageSource).Text()) + dims = strings.TrimSpace(s.Find(Selectors.ImageDims).Text()) desc = source ) if dims != "" { diff --git a/ecosia/selectors.go b/ecosia/selectors.go new file mode 100644 index 0000000..d9fad99 --- /dev/null +++ b/ecosia/selectors.go @@ -0,0 +1,26 @@ +package ecosia + +// Selectors is the single source of truth for Ecosia SERP CSS selectors. +var Selectors = struct { + Mainline string + Result string + Ad string + ResultLink string + Title string + Desc string + ImageResult string + ImageLink string + ImageSource string + ImageDims string +}{ + Mainline: "[data-test-id='mainline']", + Result: "[data-test-id='mainline-result-web']", + Ad: "[data-test-id='mainline-result-ad']", + ResultLink: "[data-test-id='result-link']", + Title: "[data-test-id='result-title']", + Desc: "[data-test-id='result-description']", + ImageResult: "[data-test-id='images-result']", + ImageLink: "[data-test-id='image-result-link']", + ImageSource: "[data-test-id='image-result-source']", + ImageDims: "[data-test-id='image-result-dimensions']", +} diff --git a/google/captcha_selector_test.go b/google/captcha_selector_test.go index 4704446..a6eea7a 100644 --- a/google/captcha_selector_test.go +++ b/google/captcha_selector_test.go @@ -17,14 +17,14 @@ func TestGooglePageTypeSelectors(t *testing.T) { selector string wantHit bool }{ - {"search_captcha.html", sel.Captcha, true}, - {"search_captcha.html", sel.ResultStats, false}, + {"search_captcha.html", Selectors.Captcha, true}, + {"search_captcha.html", Selectors.ResultStats, false}, - {"search_results.html", sel.ResultStats, true}, - {"search_results.html", sel.Captcha, false}, + {"search_results.html", Selectors.ResultStats, true}, + {"search_results.html", Selectors.Captcha, false}, - {"search_no_results.html", sel.ResultStats, true}, - {"search_no_results.html", sel.Captcha, false}, + {"search_no_results.html", Selectors.ResultStats, true}, + {"search_no_results.html", Selectors.Captcha, false}, } for _, tt := range tests { diff --git a/google/parse_html_test.go b/google/parse_html_test.go new file mode 100644 index 0000000..06da9a3 --- /dev/null +++ b/google/parse_html_test.go @@ -0,0 +1,70 @@ +package google + +import ( + "bytes" + "os" + "strings" + "testing" +) + +func TestParseHTML(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + + if len(results) == 0 { + t.Fatal("expected at least one result") + } + + for i, r := range results { + if r.Rank != i+1 { + t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, i+1) + } + if r.URL == "" { + t.Fatalf("result %d: empty URL", i) + } + if r.Title == "" { + t.Fatalf("result %d: empty Title", i) + } + if !strings.HasPrefix(r.URL, "http") { + t.Fatalf("result %d: URL not absolute: %s", i, r.URL) + } + } +} + +func TestParseHTMLEmpty(t *testing.T) { + t.Parallel() + + results, err := ParseHTML(bytes.NewReader([]byte(""))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results for empty HTML, got %d", len(results)) + } +} + +func TestParseHTMLNoResults(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_no_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results, got %d", len(results)) + } +} diff --git a/google/search.go b/google/search.go index eeb7c67..a279a33 100644 --- a/google/search.go +++ b/google/search.go @@ -15,18 +15,6 @@ import ( "golang.org/x/time/rate" ) -var sel = struct { - Captcha string - ResultStats string - CookieBtn string - Results string -}{ - Captcha: "div[data-sitekey]", - ResultStats: "div#result-stats", - CookieBtn: "div[role='dialog'][aria-modal] button", - Results: "div[data-hveid][data-ved]", -} - // Google implements core.SearchEngine for Google SERP pages. type Google struct { core.Browser @@ -61,7 +49,7 @@ func (gogl *Google) getTotalResults(page *rod.Page) (int, error) { return 0, core.ErrParser } - resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search(sel.ResultStats) + resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search(Selectors.ResultStats) if err != nil { return 0, errors.New("Result stats not found: " + err.Error()) } @@ -124,12 +112,12 @@ func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas, proxyURL string } func (gogl *Google) checkCaptcha(page *rod.Page, queryProxyURL string) bool { - has, _, _ := page.Has(sel.Captcha) + has, _, _ := page.Has(Selectors.Captcha) if !has { return false } - captchaDiv, err := page.Element(sel.Captcha) + captchaDiv, err := page.Element(Selectors.Captcha) if err != nil { return true } @@ -167,7 +155,7 @@ func (gogl *Google) preparePage(page *rod.Page) { } func (gogl *Google) acceptCookies(page *rod.Page) { - diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search(sel.CookieBtn) + diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search(Selectors.CookieBtn) if err != nil { gogl.logger.Debug("Cookie consent not found: %s", err) return @@ -233,7 +221,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } // Find all results using stable attributes - searchRes, err := page.Timeout(gogl.Timeout).Search(sel.Results) + searchRes, err := page.Timeout(gogl.Timeout).Search(Selectors.Results) if err != nil { gogl.logger.Error("Cannot parse search results: %s", err) return nil, core.ErrParser @@ -310,7 +298,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } else if query.Answers && strings.Contains(attrs, "data-ulkwtsb") && !strings.Contains(attrs, "data-ispaa") { // 2. Parse answer boxes - answerEls, err := resEl.Page().Search("div[data-hveid][data-ulkwtsb] div[data-q]") + answerEls, err := resEl.Page().Search(Selectors.AnswerBox) if err != nil { gogl.logger.Debug("Answer parsing failed: %s", err.Error()) continue @@ -352,7 +340,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } // Get URL - link, err := answ.Element("a") + link, err := answ.Element(Selectors.AnswerItem) if err != nil { gogl.logger.Debug("Missing answer link") continue @@ -376,7 +364,7 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor } else if strings.Contains(attrs, "data-ved") { // Parse regular search results // Get title from h3 - titleTag, err := resEl.Element("h3") + titleTag, err := resEl.Element(Selectors.Title) if err != nil { continue } @@ -402,9 +390,9 @@ func (gogl *Google) Search(ctx context.Context, query core.Query) (results []cor // Get description using multiple fallback strategies desc := "" - if descTag, err := resEl.Element("div[data-sncf='1'] div"); err == nil { + if descTag, err := resEl.Element(Selectors.DescPrimary); err == nil { desc, _ = descTag.Text() - } else if descTag, err := resEl.Element("div.VwiC3b"); err == nil { + } else if descTag, err := resEl.Element(Selectors.DescFallback); err == nil { desc, _ = descTag.Text() } else { // Structural fallback diff --git a/google/search_raw.go b/google/search_raw.go index 39cacaa..73550ba 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -3,6 +3,7 @@ package google import ( "context" "fmt" + "io" "net/http" "strings" @@ -32,31 +33,36 @@ func googleRequest(ctx context.Context, searchURL string, query core.Query) (*ht return res, nil } -func googleResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromReader(response.Body) +// ParseHTML parses a Google SERP HTML document and returns search results. +// It is the pure parser used by both raw HTTP search and parse endpoints. +func ParseHTML(r io.Reader) ([]core.SearchResult, error) { + doc, err := goquery.NewDocumentFromReader(r) if err != nil { return nil, err } + return parseGoogleDocument(doc), nil +} +func parseGoogleDocument(doc *goquery.Document) []core.SearchResult { results := []core.SearchResult{} rank := 1 // Use data attributes instead of class names to find results // Both old and new DOM have data-hveid and data-ved attributes - sel := doc.Find("div[data-hveid][data-ved]") + sel := doc.Find(Selectors.Results) for i := range sel.Nodes { item := sel.Eq(i) // Skip items without an h3 element (which indicates a search result) - if item.Find("h3").Length() == 0 { + if item.Find(Selectors.Title).Length() == 0 { continue } // Find URL - look for the anchor that contains the h3 title - linkTag := item.Find("h3").Parent() + linkTag := item.Find(Selectors.Title).Parent() if !linkTag.Is("a") { - linkTag = item.Find("h3").Closest("a") + linkTag = item.Find(Selectors.Title).Closest("a") } link, exists := linkTag.Attr("href") @@ -66,15 +72,15 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) { link = strings.Trim(link, " ") // Find title - this is inside the h3 element - titleTag := item.Find("h3") + titleTag := item.Find(Selectors.Title) title := titleTag.Text() // Find description - find div with text content after the heading // Using attribute selectors that match the description container - descTag := item.Find("div[data-sncf='1']").Find("div").First() + descTag := item.Find(Selectors.DescPrimary).First() if descTag.Length() == 0 { // Try another selector approach if the first one fails - descTag = item.Find("div.VwiC3b") + descTag = item.Find(Selectors.DescFallback) if descTag.Length() == 0 { // As a last resort, look for any div after the title that might contain description titleParent := titleTag.Parent() @@ -102,7 +108,11 @@ func googleResultParser(response *http.Response) ([]core.SearchResult, error) { logrus.WithField("document_size", len(doc.Text())).Trace( fmt.Sprintf("Google search document size: %d", len(doc.Text())), ) - return core.DeduplicateResults(results), err + return core.DeduplicateResults(results) +} + +func googleResultParser(response *http.Response) ([]core.SearchResult, error) { + return ParseHTML(response.Body) } func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) { diff --git a/google/selectors.go b/google/selectors.go new file mode 100644 index 0000000..ad03d45 --- /dev/null +++ b/google/selectors.go @@ -0,0 +1,26 @@ +package google + +// Selectors is the single source of truth for Google SERP CSS selectors. +// Both the browser parser (search.go, rod) and HTML parser (search_raw.go, +// goquery) reference these. When Google changes their DOM, edit here only. +var Selectors = struct { + Captcha string + ResultStats string + CookieBtn string + Results string + Title string + DescPrimary string + DescFallback string + AnswerBox string + AnswerItem string +}{ + Captcha: "div[data-sitekey]", + ResultStats: "div#result-stats", + CookieBtn: "div[role='dialog'][aria-modal] button", + Results: "div[data-hveid][data-ved]", + Title: "h3", + DescPrimary: "div[data-sncf='1'] div", + DescFallback: "div.VwiC3b", + AnswerBox: "div[data-hveid][data-ulkwtsb] div[data-q]", + AnswerItem: "a", +} diff --git a/yandex/captcha_selector_test.go b/yandex/captcha_selector_test.go index f833a46..d6c425b 100644 --- a/yandex/captcha_selector_test.go +++ b/yandex/captcha_selector_test.go @@ -17,16 +17,16 @@ func TestYandexPageTypeSelectors(t *testing.T) { selector string wantHit bool }{ - {"search_captcha.html", sel.Captcha, true}, - {"search_captcha.html", sel.NoResults, false}, + {"search_captcha.html", Selectors.Captcha, true}, + {"search_captcha.html", Selectors.NoResults, false}, {"search_captcha.html", "li[data-fast]", false}, {"search_results.html", "li[data-fast]", true}, - {"search_results.html", sel.Captcha, false}, - {"search_results.html", sel.NoResults, false}, + {"search_results.html", Selectors.Captcha, false}, + {"search_results.html", Selectors.NoResults, false}, - {"search_no_results.html", sel.NoResults, true}, - {"search_no_results.html", sel.Captcha, false}, + {"search_no_results.html", Selectors.NoResults, true}, + {"search_no_results.html", Selectors.Captcha, false}, {"search_no_results.html", "li[data-fast]", false}, } diff --git a/yandex/parse_html.go b/yandex/parse_html.go new file mode 100644 index 0000000..a1182a3 --- /dev/null +++ b/yandex/parse_html.go @@ -0,0 +1,60 @@ +package yandex + +import ( + "io" + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/karust/openserp/core" +) + +// ParseHTML parses a Yandex SERP HTML document and returns search results. +// No network I/O. +func ParseHTML(r io.Reader) ([]core.SearchResult, error) { + doc, err := goquery.NewDocumentFromReader(r) + if err != nil { + return nil, err + } + return parseYandexDocument(doc), nil +} + +func parseYandexDocument(doc *goquery.Document) []core.SearchResult { + var results []core.SearchResult + rank := 1 + + doc.Find(Selectors.Results).Each(func(_ int, item *goquery.Selection) { + linkTag := item.Find("a").First() + if linkTag.Length() == 0 { + return + } + + href, exists := linkTag.Attr("href") + if !exists || href == "" || href == "#" || strings.HasPrefix(href, "javascript:") { + return + } + + titleTag := item.Find(Selectors.Title).First() + if titleTag.Length() == 0 { + return + } + title := strings.TrimSpace(titleTag.Text()) + if title == "" { + return + } + + desc := "" + if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 { + desc = strings.TrimSpace(descTag.Text()) + } + + results = append(results, core.SearchResult{ + Rank: rank, + URL: href, + Title: title, + Description: desc, + }) + rank++ + }) + + return core.DeduplicateResults(results) +} diff --git a/yandex/parse_html_test.go b/yandex/parse_html_test.go new file mode 100644 index 0000000..5c7691f --- /dev/null +++ b/yandex/parse_html_test.go @@ -0,0 +1,57 @@ +package yandex + +import ( + "bytes" + "os" + "strings" + "testing" +) + +func TestParseYandexHTML(t *testing.T) { + t.Parallel() + + data, err := os.ReadFile("testdata/search_results.html") + if err != nil { + t.Fatalf("read fixture: %v", err) + } + + results, err := ParseHTML(bytes.NewReader(data)) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + + rank := 0 + for i, r := range results { + if r.Ad { + continue + } + rank++ + if r.Rank != rank { + t.Fatalf("rank sequence broken at index %d: got %d, want %d", i, r.Rank, rank) + } + if r.URL == "" { + t.Fatalf("result %d: empty URL", i) + } + if r.Title == "" { + t.Fatalf("result %d: empty Title", i) + } + if !strings.HasPrefix(r.URL, "http") { + t.Fatalf("result %d: URL not absolute: %s", i, r.URL) + } + } + if rank == 0 { + t.Fatal("expected at least one organic result") + } +} + +func TestParseYandexHTMLEmpty(t *testing.T) { + t.Parallel() + + results, err := ParseHTML(bytes.NewReader([]byte(""))) + if err != nil { + t.Fatalf("ParseHTML() error = %v", err) + } + if len(results) != 0 { + t.Fatalf("expected zero results for empty HTML, got %d", len(results)) + } +} diff --git a/yandex/search.go b/yandex/search.go index c609137..9b72d96 100644 --- a/yandex/search.go +++ b/yandex/search.go @@ -36,17 +36,6 @@ type ImageData struct { } `json:"initialState"` } -var sel = struct { - Captcha string - NoResults string - Results string - ImageItems string -}{ - Captcha: "div.CheckboxCaptcha", - NoResults: "div.EmptySearchResults", - Results: "li[data-fast], li.serp-item", - ImageItems: "div[role='main'] div[data-state]", -} // Yandex implements core.SearchEngine for Yandex SERP pages. type Yandex struct { @@ -79,12 +68,12 @@ func (yand *Yandex) GetRateLimiter() *rate.Limiter { } func (yand *Yandex) isCaptcha(page *rod.Page) bool { - has, _, _ := page.Has(sel.Captcha) + has, _, _ := page.Has(Selectors.Captcha) return has } func (yand *Yandex) isNoResults(page *rod.Page) bool { - has, _, _ := page.Has(sel.NoResults) + has, _, _ := page.Has(Selectors.NoResults) return has } @@ -93,7 +82,7 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc for i, r := range results { // Get URL - link, err := r.Element("a") + link, err := r.Element(Selectors.Link) if err != nil { if core.IsRodObjectNotFound(err) { break @@ -107,7 +96,7 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc } // Get title - titleTag, err := link.Element("h2") + titleTag, err := link.Element(Selectors.Title) if err != nil { yand.logger.Debug("Missing h2 title") continue @@ -120,7 +109,7 @@ func (yand *Yandex) parseResults(results rod.Elements, pageNum int) []core.Searc } // Get description - descTag, err := r.Element(`span.OrganicTextContentSpan`) + descTag, err := r.Element(Selectors.Desc) desc := "" if err != nil { yand.logger.Debug("No description") @@ -184,7 +173,7 @@ func (yand *Yandex) Search(ctx context.Context, query core.Query) (results []cor } // Get all search results in page - searchRes, err := page.Timeout(yand.Timeout).Search(sel.Results) + searchRes, err := page.Timeout(yand.Timeout).Search(Selectors.Results) if err != nil { closePage() yand.logger.Error("Cannot parse search results: %s", err) @@ -274,7 +263,7 @@ func (yand *Yandex) SearchImage(ctx context.Context, query core.Query) ([]core.S //page.WaitLoad() //time.Sleep(time.Duration(time.Second * 2)) - results, err := page.Timeout(yand.Timeout).Search(sel.ImageItems) + results, err := page.Timeout(yand.Timeout).Search(Selectors.ImageItems) if err != nil { closePage() yand.logger.Error("Cannot find search results: %s", err) diff --git a/yandex/selectors.go b/yandex/selectors.go new file mode 100644 index 0000000..0f011a5 --- /dev/null +++ b/yandex/selectors.go @@ -0,0 +1,20 @@ +package yandex + +// Selectors is the single source of truth for Yandex SERP CSS selectors. +var Selectors = struct { + Captcha string + NoResults string + Results string + Link string + Title string + Desc string + ImageItems string +}{ + Captcha: "div.CheckboxCaptcha", + NoResults: "div.EmptySearchResults", + Results: "li[data-fast], li.serp-item", + Link: "a", + Title: "h2", + Desc: "span.OrganicTextContentSpan", + ImageItems: "div[role='main'] div[data-state]", +}