openserp/duckduckgo/parse_html.go

package duckduckgo

import (
	"io"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/karust/openserp/core"
)

// ParseHTML parses a DuckDuckGo SERP HTML document and returns search results.
// No network I/O.
func ParseHTML(r io.Reader) ([]core.SearchResult, error) {
	doc, err := goquery.NewDocumentFromReader(r)
	if err != nil {
		return nil, err
	}
	return parseDDGDocument(doc), nil
}

func parseDDGDocument(doc *goquery.Document) []core.SearchResult {
	var results []core.SearchResult
	rank := core.NewRankState(0)

	resultSel := firstMatchingSelector(doc, Selectors.Results)
	if resultSel == "" {
		return core.AttachFeaturesToFirstResult(results, extractDDGFeatures(doc))
	}

	doc.Find(resultSel).Each(func(_ int, item *goquery.Selection) {
		href := ddgDocumentHref(item)
		if href == "" || href == "#" || strings.HasPrefix(href, "javascript:") {
			return
		}

		title := firstText(item, Selectors.Title...)
		if title == "" {
			return
		}

		desc := firstText(item, Selectors.Desc...)
		isAd := ddgSelectionHasAdMarker(item)

		resultRank, absoluteRank := rank.Next(isAd)
		results = append(results, core.SearchResult{
			Rank:         resultRank,
			AbsoluteRank: absoluteRank,
			URL:          href,
			Title:        title,
			Description:  desc,
			Ad:           isAd,
		})
	})

	return core.AttachFeaturesToFirstResult(core.DeduplicateResults(results), extractDDGFeatures(doc))
}

// firstText returns the normalized text of the first selector that matches a
// non-empty element.
func firstText(item *goquery.Selection, selectors ...string) string {
	for _, sel := range selectors {
		if tag := item.Find(sel).First(); tag.Length() > 0 {
			if text := core.NormalizeWhitespace(tag.Text()); text != "" {
				return text
			}
		}
	}
	return ""
}

// ddgSelectionHasAdMarker reports whether the row self-or-descendant matches any
// DuckDuckGo ad-badge selector.
func ddgSelectionHasAdMarker(item *goquery.Selection) bool {
	for _, sel := range Selectors.AdBadge {
		if item.Is(sel) || item.Find(sel).Length() > 0 {
			return true
		}
	}
	return false
}

// ddgDocumentHref returns the first non-empty href among the link selectors,
// trimmed. It skips a selector whose anchor has an absent/empty href and tries
// the next (the pre-refactor extractFirstAttr behavior).
func ddgDocumentHref(item *goquery.Selection) string {
	for _, sel := range Selectors.Link {
		tag := item.Find(sel).First()
		if tag.Length() == 0 {
			continue
		}
		if val, exists := tag.Attr("href"); exists && val != "" {
			return strings.TrimSpace(val)
		}
	}
	return ""
}

// firstMatchingSelector returns the first selector from the list that matches
// at least one element in the document.
func firstMatchingSelector(doc *goquery.Document, selectors []string) string {
	for _, sel := range selectors {
		if doc.Find(sel).Length() > 0 {
			return sel
		}
	}
	return ""
}