openserp/baidu/parse_html.go

package baidu

import (
	"errors"
	"io"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/karust/openserp/core"
)

// ParseHTML parses a Baidu SERP HTML document and returns search results.
// No network I/O.
func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		return nil, err
	}
	pageStatus := classifyBaiduDocument(doc)
	if errors.Is(pageStatus, core.ErrEmptyResult) {
		return []core.SearchResult{}, nil
	}
	if pageStatus != nil {
		return nil, pageStatus
	}
	return parseBaiduDocument(doc), nil
}

func classifyBaiduDocument(doc *goquery.Document) error {
	if doc.Find(Selectors.Captcha).Length() > 0 {
		return core.ErrCaptcha
	}
	if doc.Find(Selectors.Timeout).Length() > 0 {
		return core.ErrSearchTimeout
	}
	return core.ClassifyChallengeDocument(doc, core.DocSignals{
		EmptySelectors: []string{Selectors.NoResults},
	})
}

func parseBaiduDocument(doc *goquery.Document) []core.SearchResult {
	features := extractBaiduFeatures(doc)
	// Match all result-card variants in one pass so DOM order is preserved and
	// every card type (organic www_index, baike/encyclopedia, op cards) is
	// collected. Selecting one variant at a time and returning on the first hit
	// dropped baike and other result-op cards that interleave with organic rows.
	results := parseBaiduSelection(doc.Find(baiduResultSelector()))
	return core.AttachFeaturesToFirstResult(results, features)
}

func baiduResultSelectors() []string {
	selectors := make([]string, 0, len(Selectors.ResultsAlt)+1)
	selectors = append(selectors, Selectors.Results)
	selectors = append(selectors, Selectors.ResultsAlt...)
	return selectors
}

func baiduResultSelector() string {
	return strings.Join(baiduResultSelectors(), ", ")
}

func parseBaiduSelection(sel *goquery.Selection) []core.SearchResult {
	var results []core.SearchResult
	rank := core.NewRankState(0)

	sel.Each(func(_ int, item *goquery.Selection) {
		isAd := baiduSelectionHasAdMarker(item)

		// h3-first: organic results always carry a heading; this filters out
		// non-result blocks that may share the wrapper class.
		titleTag := item.Find("h3").First()
		var (
			title   string
			linkTag *goquery.Selection
		)
		if titleTag.Length() > 0 {
			title = strings.TrimSpace(titleTag.Text())
			if child := titleTag.Find("a[href]").First(); child.Length() > 0 {
				linkTag = child
			} else if closest := titleTag.Closest("a[href]"); closest.Length() > 0 {
				linkTag = closest
			}
		}
		if linkTag == nil || linkTag.Length() == 0 {
			first := item.Find(Selectors.Link).First()
			if first.Length() == 0 {
				return
			}
			linkTag = first
		}
		if title == "" {
			title = strings.TrimSpace(linkTag.Text())
		}
		if title == "" {
			return
		}

		href, exists := linkTag.Attr("href")
		if !exists {
			return
		}
		href = strings.TrimSpace(href)
		if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
			return
		}
		// Organic Baidu results link out through an absolute redirect
		// (http://www.baidu.com/link?url=...). Op cards like "People also search"
		// (tpl=recommend_list) instead carry relative on-site search links
		// (/s?wd=...); treat those as related-search modules, not organic rows.
		if strings.HasPrefix(href, "/") {
			return
		}
		// Baidu result cards carry the canonical destination in the mu= attribute
		// (e.g. baike.baidu.com, britannica.com), while the visible link is an
		// opaque www.baidu.com/link?url= redirect. Prefer mu= so callers get the
		// real URL, which also enables domain-based classification (encyclopedia,
		// news, etc.) downstream.
		if mu := canonicalBaiduURL(item); mu != "" {
			href = mu
		}

		desc := ""
		if descTag := item.Find(Selectors.Desc).First(); descTag.Length() > 0 {
			desc = strings.TrimSpace(descTag.Text())
		}
		if desc == "" {
			for _, alt := range Selectors.DescAlt {
				if descTag := item.Find(alt).First(); descTag.Length() > 0 {
					if t := strings.TrimSpace(descTag.Text()); t != "" {
						desc = t
						break
					}
				}
			}
		}
		if desc == "" {
			full := strings.TrimSpace(item.Text())
			desc = strings.TrimSpace(strings.Replace(full, title, "", 1))
		}

		resultRank, absoluteRank := rank.Next(isAd)
		results = append(results, core.SearchResult{
			Rank:         resultRank,
			AbsoluteRank: absoluteRank,
			URL:          href,
			Title:        title,
			Description:  desc,
			Ad:           isAd,
		})
	})

	// Re-rank sequentially after dedup so callers get a clean 1..N sequence
	// (dedup may drop intermediate ranks when the same URL appears in
	// multiple Baidu result-card variants on the same SERP).
	deduped := core.DeduplicateResults(results)
	organicIdx := 0
	for i := range deduped {
		if deduped[i].Ad {
			continue
		}
		organicIdx++
		deduped[i].Rank = organicIdx
	}
	return deduped
}

// canonicalBaiduURL returns the card's mu= destination when it is an absolute
// http(s) URL. The attribute lives on the result-card container; when the title
// link is nested, walk up to the nearest ancestor that carries it.
func canonicalBaiduURL(item *goquery.Selection) string {
	mu := strings.TrimSpace(firstAttrValue(item, "mu"))
	if mu == "" {
		if host := item.Closest("[mu]"); host.Length() > 0 {
			mu = strings.TrimSpace(firstAttrValue(host, "mu"))
		}
	}
	if strings.HasPrefix(mu, "http://") || strings.HasPrefix(mu, "https://") {
		return mu
	}
	return ""
}

func firstAttrValue(item *goquery.Selection, name string) string {
	if value, ok := item.Attr(name); ok {
		return value
	}
	return ""
}

func baiduSelectionHasAdMarker(item *goquery.Selection) bool {
	for _, selector := range Selectors.AdMarkers {
		if item.Is(selector) || item.Find(selector).Length() > 0 {
			return true
		}
	}

	isAd := false
	item.Find("span, i, em").EachWithBreak(func(_ int, marker *goquery.Selection) bool {
		text := strings.TrimSpace(marker.Text())
		if text == "广告" || text == "推广" || text == "商业推广" {
			isAd = true
			return false
		}
		return true
	})
	return isAd
}