From e12fff8313b598f264470d2b38fe40cd7997d990 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Wed, 12 Jul 2023 02:17:04 +0300 Subject: [PATCH] Google image search test --- google/search.go | 102 +++++++++++++++++++++++++++++++++++++++ google/search_test.go | 80 ++++++++++++++++++++++++++++++- google/url.go | 108 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 289 insertions(+), 1 deletion(-) diff --git a/google/search.go b/google/search.go index 94bdb4a..11a74db 100644 --- a/google/search.go +++ b/google/search.go @@ -2,11 +2,13 @@ package google import ( "errors" + "fmt" "regexp" "strconv" "strings" "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/proto" "github.com/karust/openserp/core" "github.com/sirupsen/logrus" "golang.org/x/time/rate" @@ -161,3 +163,103 @@ func (gogl *Google) Search(query core.Query) ([]core.SearchResult, error) { return searchResults, nil } + +func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) { + logrus.Tracef("Start Google Image search, query: %+v", query) + + searchResults := []core.SearchResult{} + + // Build URL from query struct to open in browser + url, err := BuildImageURL(query) + if err != nil { + return nil, err + } + + page := gogl.Navigate(url) + if !gogl.LeavePageOpen { + defer page.Close() + } + page.WaitLoad() + + results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]") + if err != nil { + defer page.Close() + logrus.Errorf("Cannot parse search results: %s", err) + return nil, core.ErrSearchTimeout + } + + // Check why no results, maybe captcha? + if results == nil { + defer page.Close() + + if gogl.isCaptcha(page) { + logrus.Errorf("Google captcha occurred during: %s", url) + return nil, core.ErrCaptcha + } + return nil, err + } + + resultElements, err := results.All() + if err != nil { + return nil, err + } + + for i, r := range resultElements { + // TODO: parse AF_initDataCallback to optimize instead of this? + err := r.Click(proto.InputMouseButtonRight, 1) + if err != nil { + logrus.Error("Error clicking") + continue + } + + // dataID, err := r.Attribute("data-id") + // if err != nil { + // continue + // } + // fmt.Println(*dataID) + + // Get URLs + link, err := r.Element("a[tabindex][role]") + if err != nil { + continue + } + + linkText, err := link.Property("href") + if err != nil { + logrus.Error("No `href` tag found") + } + + imgSrc, err := parseSourceImageURL(linkText.String()) + if err != nil { + logrus.Errorf("Cannot parse image href: %v", err) + continue + } + + // Get title + titleTag, err := r.Element("h3") + if err != nil { + logrus.Error("No `h3` tag found") + continue + } + + title, err := titleTag.Text() + if err != nil { + logrus.Error("Cannot extract text from title") + title = "No title" + } + + gR := core.SearchResult{ + Rank: i + 1, + URL: imgSrc.OriginalURL, + Title: title, + Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL), + } + searchResults = append(searchResults, gR) + } + + if !gogl.LeavePageOpen { + page.Close() + } + + return searchResults, nil +} diff --git a/google/search_test.go b/google/search_test.go index eb4344e..d383a1b 100644 --- a/google/search_test.go +++ b/google/search_test.go @@ -10,7 +10,7 @@ import ( var browser *core.Browser func init() { - opts := core.BrowserOpts{IsHeadless: true, IsLeakless: true, Timeout: time.Second * 2, WaitRequests: false} + opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 5} browser, _ = core.NewBrowser(opts) } @@ -27,3 +27,81 @@ func TestSearchGoogle(t *testing.T) { t.Fatalf("[SearchGoogle] returned empty result") } } + +func TestParseSourceImageURL(t *testing.T) { + //href1 := `/imgres?imgurl=https%3A%2F%2Fupload.wikimedia.org%2Fwikipedia%2Fcommons%2F2%2F26%2FMarmota_marmota_Alpes2.jpg&tbnid=Be_RycOe8xzlpM&vet=12ahUKEwjkh6WzwIeAAxWV_yoKHRzHC9wQMygAegUIARD0AQ..i&imgrefurl=https%3A%2F%2Fen.wikipedia.org%2Fwiki%2FAlpine_marmot&docid=7miWbc2QiSw9uM&w=801&h=599&q=alpine%20marmot&ved=2ahUKEwjkh6WzwIeAAxWV_yoKHRzHC9wQMygAegUIARD0AQ` + href2 := `/imgres?imgurl=https%3A%2F%2Fstatic.wikia.nocookie.net%2Fnaturerules1%2Fimages%2Ff%2Ff2%2F13d79d934ccf6f7919777fcb6dbb6e6c.jpg%2Frevision%2Flatest%3Fcb%3D20210218225522&tbnid=JxC8NUyBjdNbdM&vet=12ahUKEwiHrJnN1YeAAxXvEBAIHfRADAAQMygCegUIARD4AQ..i&imgrefurl=https%3A%2F%2Fnaturerules1.fandom.com%2Fwiki%2FAlpine_Marmot&docid=XXYeDjL67badNM&w=1600&h=1200&q=alpine%20marmot&ved=2ahUKEwiHrJnN1YeAAxXvEBAIHfRADAAQMygCegUIARD4AQ` + want := SourceImage{ + OriginalURL: "https://static.wikia.nocookie.net/naturerules1/images/f/f2/13d79d934ccf6f7919777fcb6dbb6e6c.jpg/revision/latest?cb=20210218225522", + PageURL: "https://naturerules1.fandom.com/wiki/Alpine_Marmot", + Width: "1600", + Height: "1200", + } + + got, err := parseSourceImageURL(href2) + if err != nil { + t.Fatal(err) + } + + if want != got { + t.Fatalf("Want: %v, Got: %v", want, got) + } +} + +// // TODO: Remove +// func TestImageScript(t *testing.T) { +// gogl := New(*browser, core.SearchEngineOptions{}) +// query := core.Query{Text: "alpine marmot"} + +// url, err := BuildImageURL(query) +// if err != nil { +// t.Fatalf("[ImageGoogle] Cannot build query") +// } + +// page := gogl.Navigate(url) +// result, err := page.Search("script") +// if err != nil { +// t.Fatalf("[ImageGoogle] Cannot find script") +// } + +// scripts, err := result.All() +// if err != nil { +// t.Fatalf("[ImageGoogle] Cannot get all results") +// } + +// rgxp, err := regexp.Compile(`http[^\[]*?\.(?:jpg|jpeg|png|bmp|gif)`) +// if err != nil { +// t.Fatalf("[ImageGoogle] Cannot compile regexp") +// } + +// for _, s := range scripts { +// if !strings.Contains(s.MustText(), "AF_initDataCallback") { +// continue +// } +// if strings.Contains(s.MustText(), "ds:0") || !strings.Contains(s.MustText(), "ds:1") { +// continue +// } + +// uris := rgxp.FindAllString(s.MustText(), -1) +// fmt.Println(len(uris)) +// fmt.Println(uris) +// } +// } + +func TestImageSearch(t *testing.T) { + gogl := New(*browser, core.SearchEngineOptions{}) + query := core.Query{Text: "Ferrari Testarossa"} + results, err := gogl.SearchImage(query) + + if err != nil { + t.Fatalf("Cannot search images: %s", err) + } + + if len(results) == 0 { + t.Fatalf("Returned empty result") + } + + if results[0].URL == "" { + t.Fatalf("First result doesn't contain URL, %v+", results[0]) + } +} diff --git a/google/url.go b/google/url.go index 1738b03..1cf25bc 100644 --- a/google/url.go +++ b/google/url.go @@ -269,3 +269,111 @@ func BuildURL(q core.Query) (string, error) { base.RawQuery = params.Encode() return base.String(), nil } + +func BuildImageURL(q core.Query) (string, error) { + // TODO: Add new params + googleBase := GoogleDomains[strings.ToLower(q.LangCode)] + base, err := url.Parse(fmt.Sprintf("https://www.google.%s", googleBase)) + if err != nil { + return "", err + } + + base.Path += "search" + params := url.Values{} + params.Add("tbm", "isch") // Search images + + // Set request text + if q.Text != "" || q.Site != "" || q.Filetype != "" { + text := q.Text + if q.Site != "" { + text += " site:" + q.Site + } + if q.Filetype != "" { + text += " filetype:" + q.Filetype + } + + logrus.Tracef("Query text: %s", text) + params.Add("q", text) + params.Add("oq", text) + } + + if len(params.Get("q")) == 0 { + return "", errors.New("Empty query built") + } + + // Set search date range + if q.DateInterval != "" { + intervals := strings.Split(q.DateInterval, "..") + if len(intervals) != 2 { + return "", errors.New("Incorrect data interval provided") + } + + dataParam := fmt.Sprintf("cdr:1,cd_min:%s,cd_max:%s", intervals[0], intervals[1]) + params.Add("tbs", dataParam) + } + + // Limit number of results + if q.Limit != 0 { + params.Add("num", strconv.Itoa(q.Limit)) + } + + if q.LangCode != "" { + params.Add("hl", q.LangCode) + params.Add("lr", "lang_"+strings.ToLower(q.LangCode)) + } + + params.Add("pws", "0") // Do not personalize earch results + params.Add("nfpr", "1") // Do not auto correct search queries + + base.RawQuery = params.Encode() + return base.String(), nil +} + +type SourceImage struct { + PageURL string + OriginalURL string + Width string + Height string +} + +func parseSourceImageURL(href string) (SourceImage, error) { + source := SourceImage{} + + href = strings.ReplaceAll(href, ";", "&") + parsed, err := url.QueryUnescape(href) + if err != nil { + return source, err + } + + u, err := url.Parse(parsed) + if err != nil { + return source, err + } + + queryMap, err := url.ParseQuery(u.RawQuery) + if err != nil { + return source, err + } + + val, ok := queryMap["h"] + if ok && len(val) > 0 { + source.Height = val[0] + } + + val, ok = queryMap["w"] + if ok && len(val) > 0 { + source.Width = val[0] + } + + val, ok = queryMap["imgrefurl"] + if ok && len(val) > 0 { + source.PageURL = val[0] + } + + val, ok = queryMap["imgurl"] + if ok && len(val) > 0 { + source.OriginalURL = val[0] + } + + return source, nil +}