Google image search test

google/search.go

@@ -2,11 +2,13 @@ package google
 
 import (
 	"errors"
+	"fmt"
 	"regexp"
 	"strconv"
 	"strings"
 
 	"github.com/go-rod/rod"
+	"github.com/go-rod/rod/lib/proto"
 	"github.com/karust/openserp/core"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/time/rate"
@@ -161,3 +163,103 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 
 	return searchResults, nil
 }
+
+func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
+	logrus.Tracef("Start Google Image search, query: %+v", query)
+
+	searchResults := []core.SearchResult{}
+
+	// Build URL from query struct to open in browser
+	url, err := BuildImageURL(query)
+	if err != nil {
+		return nil, err
+	}
+
+	page := gogl.Navigate(url)
+	if !gogl.LeavePageOpen {
+		defer page.Close()
+	}
+	page.WaitLoad()
+
+	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
+	if err != nil {
+		defer page.Close()
+		logrus.Errorf("Cannot parse search results: %s", err)
+		return nil, core.ErrSearchTimeout
+	}
+
+	// Check why no results, maybe captcha?
+	if results == nil {
+		defer page.Close()
+
+		if gogl.isCaptcha(page) {
+			logrus.Errorf("Google captcha occurred during: %s", url)
+			return nil, core.ErrCaptcha
+		}
+		return nil, err
+	}
+
+	resultElements, err := results.All()
+	if err != nil {
+		return nil, err
+	}
+
+	for i, r := range resultElements {
+		// TODO: parse AF_initDataCallback to optimize instead of this?
+		err := r.Click(proto.InputMouseButtonRight, 1)
+		if err != nil {
+			logrus.Error("Error clicking")
+			continue
+		}
+
+		// dataID, err := r.Attribute("data-id")
+		// if err != nil {
+		// 	continue
+		// }
+		// fmt.Println(*dataID)
+
+		// Get URLs
+		link, err := r.Element("a[tabindex][role]")
+		if err != nil {
+			continue
+		}
+
+		linkText, err := link.Property("href")
+		if err != nil {
+			logrus.Error("No `href` tag found")
+		}
+
+		imgSrc, err := parseSourceImageURL(linkText.String())
+		if err != nil {
+			logrus.Errorf("Cannot parse image href: %v", err)
+			continue
+		}
+
+		// Get title
+		titleTag, err := r.Element("h3")
+		if err != nil {
+			logrus.Error("No `h3` tag found")
+			continue
+		}
+
+		title, err := titleTag.Text()
+		if err != nil {
+			logrus.Error("Cannot extract text from title")
+			title = "No title"
+		}
+
+		gR := core.SearchResult{
+			Rank:        i + 1,
+			URL:         imgSrc.OriginalURL,
+			Title:       title,
+			Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
+		}
+		searchResults = append(searchResults, gR)
+	}
+
+	if !gogl.LeavePageOpen {
+		page.Close()
+	}
+
+	return searchResults, nil
+}

google/search_test.go

@@ -10,7 +10,7 @@ import (
 var browser *core.Browser
 
 func init() {
-	opts := core.BrowserOpts{IsHeadless: true, IsLeakless: true, Timeout: time.Second * 2, WaitRequests: false}
+	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 5}
 	browser, _ = core.NewBrowser(opts)
 }
 
@@ -27,3 +27,81 @@ func TestSearchGoogle(t *testing.T) {
 		t.Fatalf("[SearchGoogle] returned empty result")
 	}
 }
+
+func TestParseSourceImageURL(t *testing.T) {
+	//href1 := `/imgres?imgurl=https%3A%2F%2Fupload.wikimedia.org%2Fwikipedia%2Fcommons%2F2%2F26%2FMarmota_marmota_Alpes2.jpg&tbnid=Be_RycOe8xzlpM&vet=12ahUKEwjkh6WzwIeAAxWV_yoKHRzHC9wQMygAegUIARD0AQ..i&imgrefurl=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FAlpine_marmot&docid=7miWbc2QiSw9uM&w=801&h=599&q=alpine%20marmot&ved=2ahUKEwjkh6WzwIeAAxWV_yoKHRzHC9wQMygAegUIARD0AQ`
+	href2 := `/imgres?imgurl=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fnaturerules1%2Fimages%2Ff%2Ff2%2F13d79d934ccf6f7919777fcb6dbb6e6c.jpg%2Frevision%2Flatest%3Fcb%3D20210218225522&tbnid=JxC8NUyBjdNbdM&vet=12ahUKEwiHrJnN1YeAAxXvEBAIHfRADAAQMygCegUIARD4AQ..i&imgrefurl=https%3A%2F%2Fnaturerules1.fandom.com%2Fwiki%2FAlpine_Marmot&docid=XXYeDjL67badNM&w=1600&h=1200&q=alpine%20marmot&ved=2ahUKEwiHrJnN1YeAAxXvEBAIHfRADAAQMygCegUIARD4AQ`
+	want := SourceImage{
+		OriginalURL: "https://static.wikia.nocookie.net/naturerules1/images/f/f2/13d79d934ccf6f7919777fcb6dbb6e6c.jpg/revision/latest?cb=20210218225522",
+		PageURL:     "https://naturerules1.fandom.com/wiki/Alpine_Marmot",
+		Width:       "1600",
+		Height:      "1200",
+	}
+
+	got, err := parseSourceImageURL(href2)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if want != got {
+		t.Fatalf("Want: %v, Got: %v", want, got)
+	}
+}
+
+// 	// TODO: Remove
+// func TestImageScript(t *testing.T) {
+// 	gogl := New(*browser, core.SearchEngineOptions{})
+// 	query := core.Query{Text: "alpine marmot"}
+
+// 	url, err := BuildImageURL(query)
+// 	if err != nil {
+// 		t.Fatalf("[ImageGoogle] Cannot build query")
+// 	}
+
+// 	page := gogl.Navigate(url)
+// 	result, err := page.Search("script")
+// 	if err != nil {
+// 		t.Fatalf("[ImageGoogle] Cannot find script")
+// 	}
+
+// 	scripts, err := result.All()
+// 	if err != nil {
+// 		t.Fatalf("[ImageGoogle] Cannot get all results")
+// 	}
+
+// 	rgxp, err := regexp.Compile(`http[^\[]*?\.(?:jpg|jpeg|png|bmp|gif)`)
+// 	if err != nil {
+// 		t.Fatalf("[ImageGoogle] Cannot compile regexp")
+// 	}
+
+// 	for _, s := range scripts {
+// 		if !strings.Contains(s.MustText(), "AF_initDataCallback") {
+// 			continue
+// 		}
+// 		if strings.Contains(s.MustText(), "ds:0") || !strings.Contains(s.MustText(), "ds:1") {
+// 			continue
+// 		}
+
+// 		uris := rgxp.FindAllString(s.MustText(), -1)
+// 		fmt.Println(len(uris))
+// 		fmt.Println(uris)
+// 	}
+// }
+
+func TestImageSearch(t *testing.T) {
+	gogl := New(*browser, core.SearchEngineOptions{})
+	query := core.Query{Text: "Ferrari Testarossa"}
+	results, err := gogl.SearchImage(query)
+
+	if err != nil {
+		t.Fatalf("Cannot search images: %s", err)
+	}
+
+	if len(results) == 0 {
+		t.Fatalf("Returned empty result")
+	}
+
+	if results[0].URL == "" {
+		t.Fatalf("First result doesn't contain URL, %v+", results[0])
+	}
+}

google/url.go

@@ -269,3 +269,111 @@ func BuildURL(q core.Query) (string, error) {
 	base.RawQuery = params.Encode()
 	return base.String(), nil
 }
+
+func BuildImageURL(q core.Query) (string, error) {
+	// TODO: Add new params
+	googleBase := GoogleDomains[strings.ToLower(q.LangCode)]
+	base, err := url.Parse(fmt.Sprintf("https://www.google.%s", googleBase))
+	if err != nil {
+		return "", err
+	}
+
+	base.Path += "search"
+	params := url.Values{}
+	params.Add("tbm", "isch") // Search images
+
+	// Set request text
+	if q.Text != "" || q.Site != "" || q.Filetype != "" {
+		text := q.Text
+		if q.Site != "" {
+			text += " site:" + q.Site
+		}
+		if q.Filetype != "" {
+			text += " filetype:" + q.Filetype
+		}
+
+		logrus.Tracef("Query text: %s", text)
+		params.Add("q", text)
+		params.Add("oq", text)
+	}
+
+	if len(params.Get("q")) == 0 {
+		return "", errors.New("Empty query built")
+	}
+
+	// Set search date range
+	if q.DateInterval != "" {
+		intervals := strings.Split(q.DateInterval, "..")
+		if len(intervals) != 2 {
+			return "", errors.New("Incorrect data interval provided")
+		}
+
+		dataParam := fmt.Sprintf("cdr:1,cd_min:%s,cd_max:%s", intervals[0], intervals[1])
+		params.Add("tbs", dataParam)
+	}
+
+	// Limit number of results
+	if q.Limit != 0 {
+		params.Add("num", strconv.Itoa(q.Limit))
+	}
+
+	if q.LangCode != "" {
+		params.Add("hl", q.LangCode)
+		params.Add("lr", "lang_"+strings.ToLower(q.LangCode))
+	}
+
+	params.Add("pws", "0")  // Do not personalize earch results
+	params.Add("nfpr", "1") // Do not auto correct search queries
+
+	base.RawQuery = params.Encode()
+	return base.String(), nil
+}
+
+type SourceImage struct {
+	PageURL     string
+	OriginalURL string
+	Width       string
+	Height      string
+}
+
+func parseSourceImageURL(href string) (SourceImage, error) {
+	source := SourceImage{}
+
+	href = strings.ReplaceAll(href, ";", "&")
+	parsed, err := url.QueryUnescape(href)
+	if err != nil {
+		return source, err
+	}
+
+	u, err := url.Parse(parsed)
+	if err != nil {
+		return source, err
+	}
+
+	queryMap, err := url.ParseQuery(u.RawQuery)
+	if err != nil {
+		return source, err
+	}
+
+	val, ok := queryMap["h"]
+	if ok && len(val) > 0 {
+		source.Height = val[0]
+	}
+
+	val, ok = queryMap["w"]
+	if ok && len(val) > 0 {
+		source.Width = val[0]
+	}
+
+	val, ok = queryMap["imgrefurl"]
+	if ok && len(val) > 0 {
+		source.PageURL = val[0]
+	}
+
+	val, ok = queryMap["imgurl"]
+	if ok && len(val) > 0 {
+		source.OriginalURL = val[0]
+	}
+
+	return source, nil
+}