openserp/baidu/parse_html.go

package baidu

import (
	"io"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/karust/openserp/core"
)

// ParseHTML parses a Baidu SERP HTML document and returns search results.
// No network I/O.
func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		return nil, err
	}
	return parseBaiduDocument(doc), nil
}

func parseBaiduDocument(doc *goquery.Document) []core.SearchResult {
	for _, selector := range baiduResultSelectors() {
		if results := parseBaiduSelection(doc.Find(selector)); len(results) > 0 {
			return results
		}
	}
	return nil
}

func baiduResultSelectors() []string {
	selectors := make([]string, 0, len(Selectors.ResultsAlt)+1)
	selectors = append(selectors, Selectors.Results)
	selectors = append(selectors, Selectors.ResultsAlt...)
	return selectors
}

func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
	var results []core.SearchResult
	rank := 1
	adRank := 1
	absoluteRank := 1

	sel.Each(func(_ int, item *goquery.Selection) {
		isAd := baiduSelectionHasAdMarker(item)

		// h3-first: organic results always carry a heading; this filters out
		// non-result blocks that may share the wrapper class.
		titleTag := item.Find("h3").First()
		var (
			title   string
			linkTag *goquery.Selection
		)
		if titleTag.Length() > 0 {
			title = strings.TrimSpace(titleTag.Text())
			if child := titleTag.Find("a[href]").First(); child.Length() > 0 {
				linkTag = child
			} else if closest := titleTag.Closest("a[href]"); closest.Length() > 0 {
				linkTag = closest
			}
		}
		if linkTag == nil || linkTag.Length() == 0 {
			first := item.Find(Selectors.Link).First()
			if first.Length() == 0 {
				return
			}
			linkTag = first
		}
		if title == "" {
			title = strings.TrimSpace(linkTag.Text())
		}
		if title == "" {
			return
		}

		href, exists := linkTag.Attr("href")
		if !exists {
			return
		}
		href = strings.TrimSpace(href)
		if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
			return
		}

		desc := ""
		if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 {
			desc = strings.TrimSpace(descTag.Text())
		}
		if desc == "" {
			for _, alt := range Selectors.DescAlt {
				if descTag := item.Find(alt).First(); descTag.Length() > 0 {
					if t := strings.TrimSpace(descTag.Text()); t != "" {
						desc = t
						break
					}
				}
			}
		}
		if desc == "" {
			full := strings.TrimSpace(item.Text())
			desc = strings.TrimSpace(strings.Replace(full, title, "", 1))
		}

		resultRank := rank
		if isAd {
			resultRank = adRank
			adRank++
		} else {
			rank++
		}

		results = append(results, core.SearchResult{
			Rank:         resultRank,
			AbsoluteRank: absoluteRank,
			URL:          href,
			Title:        title,
			Description:  desc,
			Ad:           isAd,
		})
		absoluteRank++
	})

	// Re-rank sequentially after dedup so callers get a clean 1..N sequence
	// (dedup may drop intermediate ranks when the same URL appears in
	// multiple Baidu result-card variants on the same SERP).
	deduped := core.DeduplicateResults(results)
	organicIdx := 0
	for i := range deduped {
		if deduped[i].Ad {
			continue
		}
		organicIdx++
		deduped[i].Rank = organicIdx
	}
	return deduped
}

func baiduSelectionHasAdMarker(item *goquery.Selection) bool {
	for _, selector := range Selectors.AdMarkers {
		if item.Is(selector) || item.Find(selector).Length() > 0 {
			return true
		}
	}

	isAd := false
	item.Find("span, i, em").EachWithBreak(func(_ int, marker *goquery.Selection) bool {
		text := strings.TrimSpace(marker.Text())
		if text == "广告" || text == "推广" || text == "商业推广" {
			isAd = true
			return false
		}
		return true
	})
	return isAd
}