openserp/baidu/search_raw.go

package baidu

import (
	"bytes"
	"context"
	"errors"
	"fmt"

	"github.com/PuerkitoBio/goquery"
	"github.com/karust/openserp/core"
)

func classifyBaiduRawHTML(body []byte) error {
	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
	if err != nil {
		return err
	}
	if doc.Find(Selectors.Captcha).Length() > 0 || doc.Find(Selectors.Timeout).Length() > 0 {
		return core.ErrCaptcha
	}
	if doc.Find("div.content_none, div.nors").Length() > 0 {
		return core.ErrEmptyResult
	}
	return nil
}

func Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
	ctx = core.PrepareEngineContext(ctx, query, "baidu", false)
	defer func() {
		if recovered := recover(); recovered != nil {
			err = core.RecoverEnginePanicWithContext(ctx, "baidu", recovered, nil)
			results = nil
		}
	}()

	searchURL, err := BuildURL(query)
	if err != nil {
		return nil, err
	}
	core.WithRequest(ctx).WithField("url", searchURL).Debug(fmt.Sprintf("Baidu URL built: %s", searchURL))

	res, err := core.RawSearchRequest(ctx, searchURL, query)
	if err != nil {
		return nil, err
	}
	defer core.DrainAndCloseResponse(res)
	core.WithRequest(ctx).WithField("status_code", res.StatusCode).Debug(
		fmt.Sprintf("Baidu Raw response: code=%d", res.StatusCode),
	)

	body, err := core.ReadRawSearchBody(res)
	if err != nil {
		return nil, err
	}
	htmlStatus := classifyBaiduRawHTML(body)
	if htmlStatus != nil && !errors.Is(htmlStatus, core.ErrEmptyResult) {
		return nil, htmlStatus
	}

	parsedResults, err := ParseHTML(bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	if len(parsedResults) == 0 {
		if errors.Is(htmlStatus, core.ErrEmptyResult) {
			return []core.SearchResult{}, nil
		}
		return nil, fmt.Errorf("%w: baidu raw search returned no parseable results", core.ErrParser)
	}
	if query.Start > 0 {
		for i := range parsedResults {
			if parsedResults[i].AbsoluteRank > 0 {
				parsedResults[i].AbsoluteRank += query.Start
			}
			if parsedResults[i].Ad {
				continue
			}
			parsedResults[i].Rank = query.Start + parsedResults[i].Rank
		}
	}
	core.WithRequest(ctx).WithField("results_count", len(parsedResults)).Debug(
		fmt.Sprintf("Baidu Raw results : %v", parsedResults),
	)

	return core.DeduplicateResults(parsedResults), nil
}