openserp/google/search.go

package google

import (
	"context"
	"errors"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/go-rod/rod"
	"github.com/go-rod/rod/lib/proto"
	"github.com/karust/openserp/core"
)

// Google implements core.SearchEngine for Google SERP pages.
type Google struct {
	core.Browser
	core.SearchEngineOptions
	rgxpGetDigits *regexp.Regexp
	logger        *core.EngineLogger
}

// New creates a Google engine instance with browser/runtime options applied.
func New(browser core.Browser, opts core.SearchEngineOptions) *Google {
	gogl := Google{Browser: browser}
	opts.Init()
	gogl.SearchEngineOptions = opts
	gogl.logger = core.NewEngineLogger("Google")
	gogl.rgxpGetDigits = regexp.MustCompile(`\d`)
	return &gogl
}

// Name returns the stable engine identifier.
func (gogl *Google) Name() string {
	return "google"
}

func (gogl *Google) getTotalResults(page *rod.Page) (int, error) {
	if gogl.rgxpGetDigits == nil {
		return 0, core.ErrParser
	}

	resultsStats, err := page.Timeout(gogl.GetSelectorTimeout()).Search(Selectors.ResultStats)
	if err != nil {
		return 0, errors.New("Result stats not found: " + err.Error())
	}

	statsText, err := resultsStats.First.Text()
	if err != nil {
		return 0, errors.New("Cannot extract result stats text: " + err.Error())
	}

	if len(statsText) == 0 {
		return 0, nil
	}

	// Remove search time seconds info from the end
	if len(statsText) > 15 {
		statsText = statsText[:len(statsText)-15]
	}

	foundDigits := gogl.rgxpGetDigits.FindAllString(statsText, -1)
	totalNum := strings.Join(foundDigits, "")
	total, err := strconv.Atoi(totalNum)
	if err != nil {
		return 0, err
	}
	return total, nil
}

func (gogl *Google) solveCaptcha(page *rod.Page, sitekey, datas, proxyURL string) bool {
	gogl.logger.Debug("Solve captcha: sitekey=%s", sitekey)

	if gogl.CaptchaSolver == nil {
		gogl.logger.Error("Captcha solver is not configured")
		return false
	}
	if page == nil {
		gogl.logger.Error("Captcha page context is missing")
		return false
	}

	info, err := page.Info()
	if err != nil {
		gogl.logger.Error("Cannot read page info for captcha solve: %s", err)
		return false
	}

	resp, _, err := gogl.CaptchaSolver.SolveReCaptcha2(sitekey, info.URL, datas, proxyURL)
	if err != nil {
		gogl.logger.Error("Captcha solve failed: %s", err)
		return false
	}

	gogl.logger.Debug("Captcha response received")
	_, err = page.Eval(fmt.Sprintf(`;(() => { document.getElementById("g-recaptcha-response").innerHTML="%s"; submitCallback(); })();`, resp))
	if err != nil {
		gogl.logger.Error("Failed to set captcha response: %s", err)
		return false
	}

	return true
}

func (gogl *Google) checkCaptcha(page *rod.Page, queryProxyURL string) bool {
	has, _, _ := page.Has(Selectors.Captcha)
	if !has {
		return false
	}

	captchaDiv, err := page.Element(Selectors.Captcha)
	if err != nil {
		return true
	}

	sitekey, err := captchaDiv.Attribute("data-sitekey")
	if err != nil {
		gogl.logger.Error("Cannot get captcha sitekey: %s", err)
		return true
	}

	dataS, err := captchaDiv.Attribute("data-s")
	if err != nil {
		gogl.logger.Error("Cannot get captcha datas: %s", err)
		return true
	}

	if gogl.IsSolveCaptcha && gogl.CaptchaSolverEnabled {
		proxyURL := queryProxyURL
		if strings.TrimSpace(proxyURL) == "" {
			proxyURL = gogl.ProxyURL
		}
		return !gogl.solveCaptcha(page, *sitekey, *dataS, proxyURL)
	}
	return true
}

func (gogl *Google) preparePage(page *rod.Page) {
	// Remove "similar queries" lists
	_, err := page.Eval(`() => {
		document.querySelectorAll("div[data-initq]").forEach((el) => el.remove())
	}`)
	if err != nil {
		gogl.logger.Debug("Page preparation skipped: %s", err)
	}
}

func (gogl *Google) acceptCookies(page *rod.Page) {
	diaglogBtns, err := page.Timeout(gogl.Timeout / 10).Search(Selectors.CookieBtn)
	if err != nil {
		gogl.logger.Debug("Cookie consent not found: %s", err)
		return
	}
	btnElms, err := diaglogBtns.All()
	if err != nil {
		gogl.logger.Debug("Cannot get cookie consent buttons: %s", err)
		return
	}
	if len(btnElms) < 4 {
		gogl.logger.Debug("Cookie consent buttons unavailable")
		return
	}
	if err := btnElms[3].Click(proto.InputMouseButtonLeft, 1); err != nil {
		gogl.logger.Debug("Cookie consent click failed: %s", err)
	}

}

func googleElementHasAdMarker(el *rod.Element) bool {
	if el == nil {
		return false
	}
	matches, err := el.Matches(Selectors.Ad)
	if err == nil && matches {
		return true
	}
	child, err := el.Element(Selectors.Ad)
	return err == nil && child != nil
}

// Search executes a Google web search and returns normalized search results.
// It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) Search(ctx context.Context, query core.Query) (results []core.SearchResult, err error) {
	ctx = core.PrepareEngineContext(ctx, query, gogl.Name(), true)
	scoped := *gogl
	scoped.logger = gogl.logger.WithRequest(ctx)
	gogl = &scoped

	gogl.logger.Debug("Starting search, query: %+v", query)
	defer func() {
		if recovered := recover(); recovered != nil {
			err = core.RecoverEnginePanicWithContext(ctx, gogl.Name(), recovered, gogl.logger)
			results = nil
		}
	}()

	searchResults := []core.SearchResult{}

	// Build URL from query struct to open in browser
	url, err := BuildURL(query)
	if err != nil {
		return nil, err
	}
	page, err := gogl.Navigate(ctx, url)
	if err != nil {
		return nil, err
	}

	defer gogl.close(ctx, page)
	gogl.preparePage(page)

	// Check first if there captcha
	if gogl.checkCaptcha(page, query.ProxyURL) {
		gogl.logger.Error("Captcha detected: %s", url)
		return nil, core.ErrCaptcha
	}

	// Accept cookie consent so Google renders its SERP feature modules
	if query.Features {
		gogl.acceptCookies(page)
	}

	// Wait for result containers (data-hveid + data-ved) to hydrate. WaitLoad in
	// Navigate fires before Google's right-rail/answers script attaches these
	// attributes, so a one-shot Search races the DOM and frequently sees nothing.
	searchResultElems, _, err := core.WaitForElements(ctx, page, []string{Selectors.Results}, gogl.GetSelectorTimeout())
	if err != nil {
		if gogl.checkCaptcha(page, query.ProxyURL) {
			gogl.logger.Error("Captcha detected: %s", url)
			return nil, core.ErrCaptcha
		}
		if core.IsContextDone(err) {
			return nil, err
		}
		// Keep empty-SERP behavior for selector timeout only.
		if errors.Is(err, core.ErrSearchTimeout) {
			return nil, nil
		}
		return nil, core.ErrSearchTimeout
	}

	totalResults, err := gogl.getTotalResults(page)
	if err != nil {
		gogl.logger.Debug("Failed to get total results: %v", err)
	}
	gogl.logger.Info("Found %d total results", totalResults)

	rank := query.Start
	adRank := 1
	absoluteRank := query.Start + 1
	for _, resEl := range searchResultElems {
		srchRes := core.SearchResult{}

		isAd := googleElementHasAdMarker(resEl)
		isAnswerBox := query.Features && core.HasAttribute(resEl, "data-ulkwtsb") && !core.HasAttribute(resEl, "data-ispaa")
		isResultCandidate := core.HasAttribute(resEl, "data-ved")

		if isAd {
			// 1. Parse ads

			srchRes.Ad = true

			// Get URL
			link, err := resEl.Element(Selectors.Link)
			if err != nil {
				gogl.logger.Debug("Missing link")
				continue
			}
			if err := link.MoveMouseOut(); err != nil {
				gogl.logger.Debug("Move mouse out failed: %s", err)
			}

			href, err := link.Property("href")
			if err != nil {
				gogl.logger.Debug("Missing href")
				continue
			}
			srchRes.URL = href.String()

			// Get title
			titleText, err := link.Text()
			if err != nil {
				gogl.logger.Debug("Missing ad title text")
				continue
			}
			srchRes.Title = titleText

			// Get description
			text, err := resEl.Text()
			if err != nil {
				gogl.logger.Debug("Missing ad description text")
				continue
			}
			textSliced := strings.Split(text, "\n")
			if len(textSliced) > 4 {
				srchRes.Description = strings.Join(textSliced[4:], "\n")
			} else {
				srchRes.Description = strings.TrimSpace(text)
			}
			srchRes.Rank = adRank
			srchRes.AbsoluteRank = absoluteRank
			adRank++
			absoluteRank++

		} else if isAnswerBox {
			// 2. Parse answer boxes
			answerEls, err := resEl.Page().Search(Selectors.AnswerBox)
			if err != nil {
				gogl.logger.Debug("Answer parsing failed: %s", err.Error())
				continue
			}

			answers, err := answerEls.All()
			if err != nil {
				gogl.logger.Debug("Answer parsing failed: %s", err.Error())
				continue
			}

			gogl.logger.Info("Found %d answers", len(answers))

			// Unvail answer contents
			for _, answ := range answers {
				if err := answ.Click(proto.InputMouseButtonLeft, 1); err != nil {
					gogl.logger.Debug("Answer expand click failed: %s", err)
					continue
				}
				if err := answ.Focus(); err != nil {
					gogl.logger.Debug("Answer focus failed: %s", err)
				}

			}
			if err := core.SleepContext(ctx, 2*time.Second); err != nil {
				return nil, err
			}

			for i, answ := range answers {
				answerRawText, err := answ.Text()
				if err != nil {
					gogl.logger.Debug("Missing answer text")
					continue
				}
				answerText := strings.Split(answerRawText, "\n")
				if len(answerText) < 2 {
					gogl.logger.Debug("Short answer text: %s", answerText)
					continue
				}

				// Get URL
				link, err := answ.Element(Selectors.AnswerItem)
				if err != nil {
					gogl.logger.Debug("Missing answer link")
					continue
				}
				if err := link.MoveMouseOut(); err != nil {
					gogl.logger.Debug("Move mouse out failed: %s", err)
				}

				href, err := link.Property("href")
				if err != nil {
					gogl.logger.Debug("Missing answer href")
					continue
				}
				srchRes.URL = href.String()
				srchRes.Title = answerText[0]
				srchRes.Description = strings.Join(answerText[1:len(answerText)-2], "\n")
				srchRes.Rank = -1 * (i + 1)
				srchRes.Type = core.ResultTypePeopleAlsoAsk
				searchResults = append(searchResults, srchRes)
			}
			continue
		} else if isResultCandidate {
			// Parse regular search results
			// Get title from h3
			titleTag, err := resEl.Element(Selectors.Title)
			if err != nil {
				continue
			}
			srchRes.Title, _ = titleTag.Text()

			// Get URL from parent link of h3
			link, err := titleTag.Parent()
			if err == nil {
				isLink, matchErr := link.Matches(Selectors.Link)
				if matchErr != nil {
					gogl.logger.Debug("Failed to match link selector: %s", matchErr)
				}
				if isLink {
					href, _ := link.Property("href")
					srchRes.URL = href.String()
				}
			}

			// Skip if URL is empty
			if srchRes.URL == "" {
				continue
			}

			// Get description using multiple fallback strategies
			desc := ""
			if descTag, err := resEl.Element(Selectors.DescPrimary); err == nil {
				desc, _ = descTag.Text()
			} else if descTag, err := resEl.Element(Selectors.DescFallback); err == nil {
				desc, _ = descTag.Text()
			} else {
				// Structural fallback
				parent, err := titleTag.Parent()
				if err == nil {
					parent, err = parent.Parent()
					if err == nil {
						parent, err = parent.Parent()
						if err == nil {
							if descTag, err := parent.Next(); err == nil {
								if descDiv, err := descTag.Element(Selectors.DescAny); err == nil {
									desc, _ = descDiv.Text()
								}
							}
						}
					}
				}
			}
			srchRes.Description = desc

			rank += 1
			srchRes.Rank = rank
			srchRes.AbsoluteRank = absoluteRank
			absoluteRank++
			searchResults = append(searchResults, srchRes)
			continue

		} else {
			continue
		}

		if srchRes.Ad && srchRes.Rank == 0 {
			srchRes.Rank = adRank
			adRank++
		} else if !srchRes.Ad {
			srchRes.Rank = rank
		}
		if srchRes.AbsoluteRank == 0 {
			srchRes.AbsoluteRank = absoluteRank
			absoluteRank++
		}
		searchResults = append(searchResults, srchRes)
	}

	deduped := core.DeduplicateResults(searchResults)
	if len(deduped) == 0 {
		if gogl.checkCaptcha(page, query.ProxyURL) {
			return nil, core.ErrCaptcha
		}
		// Result candidates were found by Selectors.Results but none parsed
		// into usable rows: treat as a genuine no-results SERP rather than a
		// timeout, so callers don't retry pointlessly.
		if len(searchResultElems) == 0 {
			return nil, nil
		}
		return nil, core.ErrSearchTimeout
	}
	if query.Features {
		deduped = core.AttachFeaturesToFirstResult(deduped, extractGoogleFeaturesFromPage(page))
	}
	return deduped, nil
}

// SearchImage executes a Google image search and returns normalized image
// results. It may return core.ErrCaptcha or core.ErrSearchTimeout.
func (gogl *Google) SearchImage(ctx context.Context, query core.Query) ([]core.SearchResult, error) {
	ctx = core.PrepareEngineContext(ctx, query, gogl.Name(), true)
	scoped := *gogl
	scoped.logger = gogl.logger.WithRequest(ctx)
	gogl = &scoped

	gogl.logger.Debug("Starting image search, query: %+v", query)

	searchResultsMap := map[string]core.SearchResult{}
	url, err := BuildImageURL(query)
	if err != nil {
		return nil, err
	}

	page, err := gogl.Navigate(ctx, url)
	if err != nil {
		return nil, err
	}

	defer gogl.close(ctx, page)

	// Hard cap on outer scroll/parse passes. Google's image grid is virtualized
	// and infinite-scroll: WaitLoad after scroll never settles and individual
	// cells can hang on right-click. Cap iterations as a last-resort guard.
	const maxImagePasses = 20
	stagnant := 0
	for pass := 0; pass < maxImagePasses && core.ShouldFetchResultPage(len(searchResultsMap), query.Limit, pass); pass++ {
		if err := ctx.Err(); err != nil {
			return *core.ConvertSearchResultsMap(searchResultsMap), err
		}
		if _, err := page.Eval(`() => window.scrollBy(0, 1000000)`); err != nil {
			gogl.logger.Debug("Image page scroll failed: %s", err)
		}
		if err := core.SleepContext(ctx, 600*time.Millisecond); err != nil {
			return *core.ConvertSearchResultsMap(searchResultsMap), err
		}

		resultElements, _, err := core.WaitForElements(ctx, page, []string{Selectors.ImageResults}, gogl.GetSelectorTimeout())
		if err != nil {
			if gogl.checkCaptcha(page, query.ProxyURL) {
				gogl.logger.Error("Captcha detected: %s", url)
				return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
			}
			break
		}

		before := len(searchResultsMap)
		for _, r := range resultElements {
			if err := ctx.Err(); err != nil {
				return *core.ConvertSearchResultsMap(searchResultsMap), err
			}
			gogl.parseImageCell(r, searchResultsMap)
			// Always remove the cell so the next outer iteration only sees
			// freshly-scrolled elements. Without this we re-iterate the same
			// already-parsed cells and the loop scales O(n²) — or worse,
			// hangs entirely when right-click on a stale node never returns.
			if err := r.Remove(); err != nil {
				gogl.logger.Debug("Failed to remove parsed image element: %s", err)
			}
			if query.Limit > 0 && len(searchResultsMap) >= query.Limit {
				break
			}
		}

		if len(searchResultsMap) == before {
			stagnant++
			if stagnant >= 2 {
				break
			}
		} else {
			stagnant = 0
		}
	}

	return *core.ConvertSearchResultsMap(searchResultsMap), nil
}

// parseImageCell extracts one image result from a Google image grid cell and
// stores it in dst keyed by the cell's data-ved. Returns silently on any
// failure — the surrounding loop calls r.Remove() unconditionally so a
// problem cell can't stall the outer scroll loop.
func (gogl *Google) parseImageCell(r *rod.Element, dst map[string]core.SearchResult) {
	// Right-clicking a Google image cell forces it to materialize its
	// `imgres` link (the grid is virtualized; href is absent until the
	// cell is interacted with).
	if err := r.Click(proto.InputMouseButtonRight, 1); err != nil {
		gogl.logger.Debug("Right-click on image cell failed: %s", err)
		return
	}

	dataVed, err := r.Attribute("data-ved")
	if err != nil || dataVed == nil || strings.TrimSpace(*dataVed) == "" {
		gogl.logger.Debug("Missing data-ved attribute")
		return
	}
	resultKey := *dataVed
	if _, ok := dst[resultKey]; ok {
		return
	}

	link, err := r.Element(Selectors.ImageLink)
	if err != nil {
		link, err = r.Element(Selectors.ImageLinkFallback)
		if err != nil {
			return
		}
	}

	linkText, err := link.Property("href")
	if err != nil {
		gogl.logger.Debug("Missing href")
		return
	}

	imgSrc, err := parseSourceImageURL(linkText.String())
	if err != nil {
		gogl.logger.Error("Failed to parse image URL: %v", err)
		return
	}
	if strings.TrimSpace(imgSrc.OriginalURL) == "" {
		return
	}

	title := core.FirstNonEmptyText(r, Selectors.ImageTitle...)
	if title == "" {
		title = core.FirstNonEmptyAttribute(r, "alt", "img")
	}
	if title == "" {
		title = core.FirstNonEmptyAttribute(r, "aria-label", "a")
	}
	if title == "" {
		title = "No title"
	}

	dst[resultKey] = core.SearchResult{
		Rank:        len(dst) + 1,
		URL:         imgSrc.OriginalURL,
		Title:       title,
		Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
	}
}

func (gogl *Google) close(ctx context.Context, page *rod.Page) {
	core.DeferClosePage(ctx, page, &gogl.Browser)()
}