package ecosia import ( "errors" "fmt" "io" "strings" "github.com/PuerkitoBio/goquery" "github.com/karust/openserp/core" ) // ParseHTML parses an Ecosia SERP HTML document and returns search results. // No network I/O. func ParseHTML(r io.Reader) ([]core.SearchResult, error) { doc, err := goquery.NewDocumentFromReader(r) if err != nil { return nil, err } pageStatus := classifyEcosiaDocument(doc) if errors.Is(pageStatus, core.ErrEmptyResult) { return []core.SearchResult{}, nil } if pageStatus != nil { return nil, pageStatus } return parseEcosiaDocument(doc), nil } func classifyEcosiaDocument(doc *goquery.Document) error { if isCaptchaDoc(doc) { return core.ErrCaptcha } if doc.Find(Selectors.NoResults).Length() > 0 || (doc.Find(Selectors.Mainline).Length() > 0 && doc.Find(Selectors.Result).Length() == 0 && doc.Find(Selectors.Ad).Length() == 0) { return core.ErrEmptyResult } return nil } func parseEcosiaDocument(doc *goquery.Document) []core.SearchResult { var results []core.SearchResult rank := 1 adRank := 1 doc.Find(Selectors.Result).Each(func(_ int, item *goquery.Selection) { if res, ok := parseEcosiaSelectionRow(item, rank, false); ok { results = append(results, res) rank++ } }) doc.Find(Selectors.Ad).Each(func(_ int, item *goquery.Selection) { if res, ok := parseEcosiaSelectionRow(item, adRank, true); ok { results = append(results, res) adRank++ } }) core.SetSeparatedAdAbsoluteRanks(results, 0) return core.AttachFeaturesToFirstResult(core.DeduplicateResults(results), extractEcosiaFeatures(doc)) } // parseEcosiaSelectionRow extracts a web row from a goquery selection. func parseEcosiaSelectionRow(item *goquery.Selection, rank int, ad bool) (core.SearchResult, bool) { link := item.Find(Selectors.ResultLink).First() if link.Length() == 0 { // Fall back to the first anchor when the test-id selector is absent. link = item.Find("a[href]").First() } href, _ := link.Attr("href") title := selectionText(item, Selectors.Title) if title == "" { title = selectionText(item, "h2, h3") } desc := selectionText(item, Selectors.Desc) return assembleEcosiaRow(href, title, desc, rank, ad) } // assembleEcosiaRow validates an already-extracted web row and builds the // result. Shared by the rod (browser) and goquery (raw / parse) parsers. func assembleEcosiaRow(href, title, desc string, rank int, ad bool) (core.SearchResult, bool) { href = strings.TrimSpace(href) if href == "" || strings.HasPrefix(href, "javascript:") { return core.SearchResult{}, false } return core.SearchResult{ Rank: rank, URL: href, Title: strings.TrimSpace(title), Description: strings.TrimSpace(desc), Ad: ad, }, true } // assembleEcosiaImageRow validates an already-extracted image card and builds // the result (formatting the source/dimensions description). Shared by both // parser backends. func assembleEcosiaImageRow(href, title, source, dims string, rank int) (core.SearchResult, bool) { href = strings.TrimSpace(href) if href == "" { return core.SearchResult{}, false } desc := source if dims != "" { if source != "" { desc = fmt.Sprintf("%s (%s)", source, dims) } else { desc = dims } } return core.SearchResult{ Rank: rank, URL: href, Title: title, Description: desc, }, true } // selectionText returns the trimmed text of the first selector that matches, // the goquery counterpart of the rod element's selector-fallback text lookup. func selectionText(item *goquery.Selection, selectors ...string) string { for _, selector := range selectors { if tag := item.Find(selector).First(); tag.Length() > 0 { return strings.TrimSpace(tag.Text()) } } return "" }