openserp/bing/search.go

package bing

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"time"

	"github.com/go-rod/rod"
	"github.com/go-rod/rod/lib/proto"
	"github.com/karust/openserp/core"
	"golang.org/x/time/rate"
)

// Bing implements core.SearchEngine for Bing SERP pages.
type Bing struct {
	core.Browser
	core.SearchEngineOptions
	logger *core.EngineLogger
}

// New creates a Bing engine instance with browser/runtime options applied.
func New(browser core.Browser, opts core.SearchEngineOptions) *Bing {
	bing := Bing{Browser: browser}
	opts.Init()
	bing.SearchEngineOptions = opts
	bing.logger = core.NewEngineLogger("Bing")
	return &bing
}

// Name returns the stable engine identifier.
func (bing *Bing) Name() string {
	return "bing"
}

// GetRateLimiter returns a limiter configured from SearchEngineOptions.
func (bing *Bing) GetRateLimiter() *rate.Limiter {
	ratelimit := rate.Every(bing.GetRatelimit())
	return rate.NewLimiter(ratelimit, bing.RateBurst)
}

func (bing *Bing) getTotalResults(page *rod.Page) (int, error) {
	results, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Results)
	if err != nil {
		return 0, errors.New("Cannot find result elements: " + err.Error())
	}
	return len(results), nil
}

func (bing *Bing) checkCaptcha(page *rod.Page) bool {
	if page == nil {
		return false
	}

	if info, err := page.Info(); err == nil {
		url := strings.ToLower(info.URL)
		if strings.Contains(url, "turing") || strings.Contains(url, "captcha") {
			return true
		}
	}

	for _, selector := range Selectors.Captcha {
		has, _, _ := page.Has(selector)
		if has {
			bing.logger.Debug("Captcha detected: %s", selector)
			return true
		}
	}

	return false
}

func (bing *Bing) acceptCookies(ctx context.Context, page *rod.Page) error {
	consentBtn, err := page.Timeout(bing.Timeout / 10).Element(Selectors.CookieBtn)
	if err != nil {
		return nil
	}
	if err := consentBtn.Click(proto.InputMouseButtonLeft, 1); err != nil {
		bing.logger.Debug("Cookie consent click failed: %v", err)
	}

	return core.SleepContext(ctx, 500*time.Millisecond)
}

// Search executes a Bing web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
	ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name())
	ctx = core.WithProfileRegion(ctx, query.LangCode)
	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
	scoped := *bing
	scoped.logger = bing.logger.WithRequest(ctx)
	bing = &scoped

	bing.logger.Debug("Starting search, query: %+v", query)
	defer func() {
		if recovered := recover(); recovered != nil {
			err = core.RecoverEnginePanicWithContext(ctx, bing.Name(), recovered, bing.logger)
			results = nil
		}
	}()

	searchResults := []core.SearchResult{}

	url, err := BuildURL(query)
	if err != nil {
		return nil, err
	}

	page, err := bing.Navigate(ctx, url)
	if err != nil {
		return nil, err
	}
	defer func() {
		if bing.Browser.LeavePageOpen {
			return
		}
		if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil {
			bing.logger.Debug("Page close error: %v", closeErr)
		}
	}()

	if bing.checkCaptcha(page) {
		bing.logger.Error("Captcha detected: %s", url)
		return nil, core.ErrCaptcha
	}

	if err := bing.acceptCookies(ctx, page); err != nil {
		return nil, err
	}

	organicElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, bing.GetSelectorTimeout())
	if err != nil {
		// Re-check captcha on timeout - Bing interstitials can render after WaitLoad.
		if bing.checkCaptcha(page) {
			bing.logger.Error("Captcha detected: %s", url)
			return nil, core.ErrCaptcha
		}
		bing.logger.Error("Cannot parse organic results: %s", err)
		return nil, core.ErrSearchTimeout
	}

	adElements, err := page.Timeout(bing.GetSelectorTimeout()).Elements(Selectors.Ads)
	if err != nil {
		bing.logger.Debug("No ads found")
	}

	totalResults, err := bing.getTotalResults(page)
	if err != nil {
		bing.logger.Debug("Failed to get total results: %v", err)
	}
	bing.logger.Info("Found %d results (%d ads)", totalResults, len(adElements))

	rank := query.Start
	for _, result := range organicElements {
		srchRes := core.SearchResult{}

		titleElem, err := result.Element(Selectors.Title)
		if err != nil {
			bing.logger.Debug("Missing title")
			continue
		}

		href, err := titleElem.Property("href")
		if err != nil {
			bing.logger.Debug("Missing URL")
			continue
		}
		srchRes.URL = href.String()
		if strings.TrimSpace(srchRes.URL) == "" || strings.HasPrefix(srchRes.URL, "javascript:") {
			continue
		}

		srchRes.Title, _ = titleElem.Text()
		srchRes.Title = strings.TrimSpace(srchRes.Title)
		if srchRes.Title == "" {
			srchRes.Title = core.FirstNonEmptyText(result, Selectors.TitleFallbacks...)
		}
		if srchRes.Title == "" {
			bing.logger.Debug("Missing title text")
			continue
		}

		var desc string
		if descElem, err := result.Element(Selectors.DescPrimary); err == nil {
			desc, _ = descElem.Text()
		} else if descElem, err := result.Element(Selectors.DescFallback); err == nil {
			desc, _ = descElem.Text()
		} else if descElem, err := result.Element("p"); err == nil {
			desc, _ = descElem.Text()
		} else {
			fullText, _ := result.Text()
			desc = strings.TrimSpace(strings.Replace(fullText, srchRes.Title, "", 1))
		}
		srchRes.Description = desc

		rank++
		srchRes.Rank = rank
		srchRes.Ad = false

		searchResults = append(searchResults, srchRes)
	}

	for _, adResult := range adElements {
		srchRes := core.SearchResult{Ad: true}

		titleElem, err := adResult.Element(Selectors.AdTitle)
		if err != nil {
			bing.logger.Debug("Ad missing title")
			continue
		}
		srchRes.Title, _ = titleElem.Text()
		srchRes.Title = strings.TrimSpace(srchRes.Title)
		if srchRes.Title == "" {
			srchRes.Title = core.FirstNonEmptyText(adResult, "h2", "a[aria-label]")
		}
		if srchRes.Title == "" {
			continue
		}

		href, err := titleElem.Property("href")
		if err != nil {
			bing.logger.Debug("Ad missing URL")
			continue
		}
		srchRes.URL = href.String()

		if descElem, err := adResult.Element("p"); err == nil {
			srchRes.Description, _ = descElem.Text()
		}

		// Mark ads with negative rank
		srchRes.Rank = -1
		searchResults = append(searchResults, srchRes)
	}

	// Deduplicate results
	deduped := core.DeduplicateResults(searchResults)

	// Trim to exact limit if necessary (only organic results, not ads)
	if query.Limit > 0 {
		organicResults := []core.SearchResult{}
		adResults := []core.SearchResult{}

		for _, result := range deduped {
			if result.Ad {
				adResults = append(adResults, result)
			} else {
				organicResults = append(organicResults, result)
			}
		}

		// Trim organic results to limit
		if len(organicResults) > query.Limit {
			organicResults = organicResults[:query.Limit]
		}

		// Combine back: organic results + ads
		deduped = append(organicResults, adResults...)
	}

	return deduped, nil
}

// BingImageData represents metadata encoded in the image result `m` attribute.
type BingImageData struct {
	T      string `json:"t"`      // Title
	Desc   string `json:"desc"`   // Description
	IMGURL string `json:"imgurl"` // Original image URL
	W      int    `json:"w"`      // Width
	H      int    `json:"h"`      // Height
	PURL   string `json:"purl"`   // Page URL
	TURL   string `json:"turl"`   // Thumbnail URL
	MURL   string `json:"murl"`   // Image URL
}

func resolveImageLinkElement(container *rod.Element) (*rod.Element, error) {
	if container == nil {
		return nil, errors.New("nil image container")
	}
	if core.HasAttribute(container, "m") {
		return container, nil
	}
	return container.Element("a")
}

// SearchImage executes a Bing image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (bing *Bing) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
	ctx = core.WithEngine(core.EnsureContext(ctx), bing.Name())
	ctx = core.WithProfileRegion(ctx, query.LangCode)
	ctx = core.WithQueryHash(ctx, core.QueryHashFromQuery(query))
	scoped := *bing
	scoped.logger = bing.logger.WithRequest(ctx)
	bing = &scoped

	bing.logger.Debug("Starting image search, query: %+v", query)

	searchResults := []core.SearchResult{}

	// Build Bing image search URL
	url, err := BuildImageURL(query)
	if err != nil {
		return nil, err
	}

	page, err := bing.Navigate(ctx, url)
	if err != nil {
		return nil, err
	}
	defer func() {
		if bing.Browser.LeavePageOpen {
			return
		}
		if closeErr := core.ClosePageWithTimeout(ctx, page, time.Second); closeErr != nil {
			bing.logger.Debug("Page close error: %v", closeErr)
		}
	}()

	// Check for captcha
	if bing.checkCaptcha(page) {
		bing.logger.Error("Captcha detected during image search: %s", url)
		return nil, core.ErrCaptcha
	}

	// Accept cookies if present
	if err := bing.acceptCookies(ctx, page); err != nil {
		return nil, err
	}

	imageContainers, _, err := core.WaitForElements(
		ctx,
		page,
		[]string{Selectors.ImageResults},
		bing.GetSelectorTimeout(),
	)
	if err != nil {
		if bing.checkCaptcha(page) {
			return nil, core.ErrCaptcha
		}
		bing.logger.Error("Cannot parse image results: %s", err)
		return nil, core.ErrSearchTimeout
	}

	if len(imageContainers) == 0 {
		return nil, errors.New("no image results found")
	}

	bing.logger.Info("Found %d image elements", len(imageContainers))

	rank := 0
	for _, c := range imageContainers {
		srchRes := core.SearchResult{}

		linkElem, err := resolveImageLinkElement(c)
		if err != nil {
			bing.logger.Debug("Missing image link element")
			continue
		}

		// Extract image metadata from m attribute (contains JSON)
		mAttr, err := linkElem.Attribute("m")
		if err != nil || mAttr == nil {
			bing.logger.Debug("Missing m attribute")
			continue
		}

		// Ensure we have valid JSON data to unmarshal
		jsonData := []byte(*mAttr)
		if len(jsonData) == 0 {
			bing.logger.Debug("Empty JSON data")
			continue
		}

		// Initialize imgData properly
		var imgData BingImageData
		err = json.Unmarshal(jsonData, &imgData)
		if err != nil {
			bing.logger.Debug("Failed to parse JSON: %s", err)
			continue
		}

		// Extract information from the parsed data
		srchRes.Title = imgData.T
		srchRes.URL = imgData.IMGURL
		srchRes.Description = fmt.Sprintf(
			"%s Source Page: %s, thumb_url:%s, %dx%d",
			imgData.Desc,
			imgData.PURL,
			imgData.TURL,
			imgData.W,
			imgData.H,
		)

		// Get the page URL
		if imgData.MURL != "" {
			srchRes.URL = imgData.MURL
		}

		rank++
		srchRes.Rank = rank

		searchResults = append(searchResults, srchRes)
		if query.Limit > 0 && len(searchResults) >= query.Limit {
			break
		}
	}

	return searchResults, nil
}