Google images scroll parsing

core/common.go

@@ -2,6 +2,7 @@ package core
 
 import (
 	"errors"
+	"sort"
 	"strconv"
 	"time"
 
@@ -18,6 +19,19 @@ type SearchResult struct {
 	Description string `json:"description"`
 }
 
+func ConvertSearchResultsMap(searchResultsMap map[string]SearchResult) *[]SearchResult {
+	searchResults := []SearchResult{}
+
+	for _, v := range searchResultsMap {
+		searchResults = append(searchResults, v)
+	}
+
+	sort.Slice(searchResults, func(i, j int) bool {
+		return searchResults[i].Rank < searchResults[j].Rank
+	})
+	return &searchResults
+}
+
 type Query struct {
 	Text         string
 	LangCode     string // eg. EN, ES, RU...

google/search.go

@@ -167,9 +167,7 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) {
 func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 	logrus.Tracef("Start Google Image search, query: %+v", query)
 
-	searchResults := []core.SearchResult{}
-
-	// Build URL from query struct to open in browser
+	searchResultsMap := map[string]core.SearchResult{}
 	url, err := BuildImageURL(query)
 	if err != nil {
 		return nil, err
@@ -179,87 +177,106 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
 	if !gogl.LeavePageOpen {
 		defer page.Close()
 	}
-	page.WaitLoad()
 
-	results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
-	if err != nil {
-		defer page.Close()
-		logrus.Errorf("Cannot parse search results: %s", err)
-		return nil, core.ErrSearchTimeout
-	}
+	//// TODO: Case with cookie accept (appears with VPN)
+	// if page.MustInfo().URL != url {
+	// 	results, _ := page.Search("button[aria-label][jsaction]")
+	// 	if results != nil {
+	// 		//buttons, _ := results.All()
+	// 		//buttons[1].Click(proto.InputMouseButtonLeft, 1)
+	// 	}
+	// }
 
-	// Check why no results, maybe captcha?
-	if results == nil {
-		defer page.Close()
+	for len(searchResultsMap) < query.Limit {
+		page.WaitLoad()
+		page.Mouse.Scroll(0, 1000000, 1)
+		page.WaitLoad()
 
-		if gogl.isCaptcha(page) {
-			logrus.Errorf("Google captcha occurred during: %s", url)
-			return nil, core.ErrCaptcha
-		}
-		return nil, err
-	}
-
-	resultElements, err := results.All()
-	if err != nil {
-		return nil, err
-	}
-
-	for i, r := range resultElements {
-		// TODO: parse AF_initDataCallback to optimize instead of this?
-		err := r.Click(proto.InputMouseButtonRight, 1)
+		results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
 		if err != nil {
-			logrus.Error("Error clicking")
+			logrus.Errorf("Cannot parse search results: %s", err)
+			return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
+		}
+
+		// Check why no results
+		if results == nil {
+			if gogl.isCaptcha(page) {
+				logrus.Errorf("Google captcha occurred during: %s", url)
+				return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrCaptcha
+			}
+			return *core.ConvertSearchResultsMap(searchResultsMap), err
+		}
+
+		resultElements, err := results.All()
+		if err != nil {
+			return *core.ConvertSearchResultsMap(searchResultsMap), err
+		}
+
+		if len(resultElements) < len(searchResultsMap) {
 			continue
 		}
 
-		// dataID, err := r.Attribute("data-id")
-		// if err != nil {
-		// 	continue
-		// }
-		// fmt.Println(*dataID)
+		for i, r := range resultElements[len(searchResultsMap):] {
+			// TODO: parse AF_initDataCallback to optimize instead of this?
+			err := r.Click(proto.InputMouseButtonRight, 1)
+			if err != nil {
+				logrus.Error("Error clicking")
+				continue
+			}
 
-		// Get URLs
-		link, err := r.Element("a[tabindex][role]")
-		if err != nil {
-			continue
+			dataID, err := r.Attribute("data-id")
+			if err != nil {
+				continue
+			}
+
+			// If already have image with this ID
+			if _, ok := searchResultsMap[*dataID]; ok {
+				continue
+			}
+
+			// Get URLs
+			link, err := r.Element("a[tabindex][role]")
+			if err != nil {
+				continue
+			}
+
+			linkText, err := link.Property("href")
+			if err != nil {
+				logrus.Error("No `href` tag found")
+			}
+
+			imgSrc, err := parseSourceImageURL(linkText.String())
+			if err != nil {
+				logrus.Errorf("Cannot parse image href: %v", err)
+				continue
+			}
+
+			// Get title
+			titleTag, err := r.Element("h3")
+			if err != nil {
+				logrus.Error("No `h3` tag found")
+				continue
+			}
+
+			title, err := titleTag.Text()
+			if err != nil {
+				logrus.Error("Cannot extract text from title")
+				title = "No title"
+			}
+
+			gR := core.SearchResult{
+				Rank:        i + 1,
+				URL:         imgSrc.OriginalURL,
+				Title:       title,
+				Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
+			}
+			searchResultsMap[*dataID] = gR
 		}
 
-		linkText, err := link.Property("href")
-		if err != nil {
-			logrus.Error("No `href` tag found")
+		if !gogl.LeavePageOpen {
+			page.Close()
 		}
-
-		imgSrc, err := parseSourceImageURL(linkText.String())
-		if err != nil {
-			logrus.Errorf("Cannot parse image href: %v", err)
-			continue
-		}
-
-		// Get title
-		titleTag, err := r.Element("h3")
-		if err != nil {
-			logrus.Error("No `h3` tag found")
-			continue
-		}
-
-		title, err := titleTag.Text()
-		if err != nil {
-			logrus.Error("Cannot extract text from title")
-			title = "No title"
-		}
-
-		gR := core.SearchResult{
-			Rank:        i + 1,
-			URL:         imgSrc.OriginalURL,
-			Title:       title,
-			Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
-		}
-		searchResults = append(searchResults, gR)
 	}
 
-	if !gogl.LeavePageOpen {
-		page.Close()
-	}
-
-	return searchResults, nil
+	return *core.ConvertSearchResultsMap(searchResultsMap), nil
 }

google/search_test.go

@@ -10,7 +10,7 @@ import (
 var browser *core.Browser
 
 func init() {
-	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 5}
+	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 5, LeavePageOpen: true}
 	browser, _ = core.NewBrowser(opts)
 }
 
@@ -48,57 +48,17 @@ func TestParseSourceImageURL(t *testing.T) {
 	}
 }
 
-// 	// TODO: Remove
-// func TestImageScript(t *testing.T) {
-// 	gogl := New(*browser, core.SearchEngineOptions{})
-// 	query := core.Query{Text: "alpine marmot"}
-
-// 	url, err := BuildImageURL(query)
-// 	if err != nil {
-// 		t.Fatalf("[ImageGoogle] Cannot build query")
-// 	}
-
-// 	page := gogl.Navigate(url)
-// 	result, err := page.Search("script")
-// 	if err != nil {
-// 		t.Fatalf("[ImageGoogle] Cannot find script")
-// 	}
-
-// 	scripts, err := result.All()
-// 	if err != nil {
-// 		t.Fatalf("[ImageGoogle] Cannot get all results")
-// 	}
-
-// 	rgxp, err := regexp.Compile(`http[^\[]*?\.(?:jpg|jpeg|png|bmp|gif)`)
-// 	if err != nil {
-// 		t.Fatalf("[ImageGoogle] Cannot compile regexp")
-// 	}
-
-// 	for _, s := range scripts {
-// 		if !strings.Contains(s.MustText(), "AF_initDataCallback") {
-// 			continue
-// 		}
-// 		if strings.Contains(s.MustText(), "ds:0") || !strings.Contains(s.MustText(), "ds:1") {
-// 			continue
-// 		}
-
-// 		uris := rgxp.FindAllString(s.MustText(), -1)
-// 		fmt.Println(len(uris))
-// 		fmt.Println(uris)
-// 	}
-// }
-
 func TestImageSearch(t *testing.T) {
 	gogl := New(*browser, core.SearchEngineOptions{})
-	query := core.Query{Text: "Ferrari Testarossa"}
+	query := core.Query{Text: "Ferrari Testarossa", Limit: 77}
 	results, err := gogl.SearchImage(query)
 
 	if err != nil {
 		t.Fatalf("Cannot search images: %s", err)
 	}
 
-	if len(results) == 0 {
-		t.Fatalf("Returned empty result")
+	if len(results) < 77 {
+		t.Fatalf("Returned not full result")
 	}
 
 	if results[0].URL == "" {

google/url.go

@@ -292,7 +292,6 @@ func BuildImageURL(q core.Query) (string, error) {
 			text += " filetype:" + q.Filetype
 		}
 
-		logrus.Tracef("Query text: %s", text)
 		params.Add("q", text)
 		params.Add("oq", text)
 	}