Baidu image search test

baidu/all_test.go

@@ -13,7 +13,7 @@ var testQuery = core.Query{Text: "go", Site: "tutorialspoint.com", DateInterval:
 func init() {
 	core.InitLogger(true, true)
 
-	opts := core.BrowserOpts{IsHeadless: true, IsLeakless: false, Timeout: time.Second * 2, WaitRequests: false}
+	opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 10}
 	browser, _ = core.NewBrowser(opts)
 }
 
@@ -30,16 +30,7 @@ func TestUrlBuild(t *testing.T) {
 	}
 }
 
-// func TestSearchRaw(t *testing.T) {
-// 	results, err := Search(testQuery)
-// 	if err != nil {
-// 		t.Fatal(err)
-// 	}
-
-// 	fmt.Println(results)
-// }
-
-func TestSearchBaidu(t *testing.T) {
+func TestSearch(t *testing.T) {
 	baid := New(*browser, core.SearchEngineOptions{})
 	results, err := baid.Search(testQuery)
 	if err != nil {
@@ -50,3 +41,37 @@ func TestSearchBaidu(t *testing.T) {
 		t.Fatal("No results got from Baidu search")
 	}
 }
+
+func TestImageUrlBuild(t *testing.T) {
+	query := core.Query{Text: "金毛猎犬", Limit: 10}
+
+	res, err := BuildImageURL(query, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	want := "https://image.baidu.com/search/acjson?cl=2&ie=utf-8&ipn=r&tn=resultjson_com&word=%E9%87%91%E6%AF%9B%E7%8C%8E%E7%8A%AC"
+
+	if want != res {
+		t.Fatalf("Wanted result `%s` doesn't match to resulted `%s`", want, res)
+	}
+}
+
+func TestImageSearch(t *testing.T) {
+
+	baid := New(*browser, core.SearchEngineOptions{})
+
+	query := core.Query{Text: "金毛猎犬", Limit: 31}
+	results, err := baid.SearchImage(query)
+	if err != nil {
+		t.Fatalf("Cannot [ImageBaidu]: %s", err)
+	}
+
+	if len(results) == 0 {
+		t.Fatalf("[ImageBaidu] returned empty result")
+	}
+
+	// for _, r := range results {
+	// 	fmt.Printf("%+v\n", r)
+	// }
+}

baidu/search.go

@@ -1,6 +1,8 @@
 package baidu
 
 import (
+	"encoding/json"
+	"fmt"
 	"strings"
 
 	"github.com/go-rod/rod"
@@ -9,6 +11,34 @@ import (
 	"golang.org/x/time/rate"
 )
 
+type imageDataJson struct {
+	Query        string `json:"queryExt"`
+	TotalResults int    `json:"displayNum"`
+
+	Data []struct {
+		Title       string `json:"fromPageTitle"`
+		PictureDate string `json:"bdImgnewsDate"`
+		ThumbURL    string `json:"thumbURL"`
+		Type        string
+		Height      int
+		Width       int
+		IsCopyright int
+
+		URL []struct {
+			SourcePage string `json:"FromURL"`
+			Original   string `json:"ObjURL"`
+		} `json:"replaceUrl"`
+
+		// Versions []struct {
+		// 	Height        int
+		// 	Width         int
+		// 	ImgSourcePage string `json:"fromURL"`
+		// 	URL           string `json:"objURL"`
+		// 	Type          string
+		// } `json:"setList"`
+	}
+}
+
 type Baidu struct {
 	core.Browser
 	core.SearchEngineOptions
@@ -123,3 +153,79 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
 
 	return searchResults, nil
 }
+
+func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
+	logrus.Tracef("Start Baidu Image search, query: %+v", query)
+
+	searchResults := []core.SearchResult{}
+	searchPage := 0
+
+	for len(searchResults) < query.Limit {
+		url, err := BuildImageURL(query, searchPage)
+		if err != nil {
+			return nil, err
+		}
+
+		// Get anti-crawler cookies first, then reload page
+		page := baid.Navigate(url)
+		if !baid.LeavePageOpen {
+			defer page.Close()
+		}
+		page.Reload()
+		page.WaitLoad()
+
+		result, err := page.Timeout(baid.Timeout).Search("body > pre")
+		if err != nil {
+			defer page.Close()
+			logrus.Errorf("Cannot parse search results: %s", err)
+			return nil, core.ErrSearchTimeout
+		}
+
+		// Check why no results, maybe captcha?
+		if result == nil {
+			defer page.Close()
+
+			if baid.isCaptcha(page) {
+				logrus.Errorf("Baidu captcha occurred during: %s", url)
+				return nil, core.ErrCaptcha
+			} else if baid.isTimeout(page) {
+				logrus.Errorf("Baidu timeout occurred during: %s", url)
+				return nil, core.ErrCaptcha
+			}
+			return nil, nil
+		}
+
+		jsonText, err := result.First.Text()
+		if err != nil {
+			return nil, err
+		}
+
+		var data imageDataJson
+
+		err = json.Unmarshal([]byte(jsonText), &data)
+		if err != nil {
+			logrus.Errorf("Cannot unmarshal Baidu image JSON: %v\nData: %v", err, jsonText)
+			return nil, err
+		}
+
+		for i, img := range data.Data {
+			if len(img.URL) == 0 {
+				continue
+			}
+			res := core.SearchResult{
+				Rank:        (searchPage * 30) + (i + 1),
+				URL:         img.URL[0].Original,
+				Title:       img.Title,
+				Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright)}
+			searchResults = append(searchResults, res)
+		}
+
+		searchPage += 1
+
+		if !baid.LeavePageOpen {
+			page.Close()
+		}
+	}
+
+	return searchResults, nil
+}

baidu/url.go

@@ -76,3 +76,32 @@ func BuildURL(q core.Query) (string, error) {
 	base.RawQuery = params.Encode()
 	return base.String(), nil
 }
+
+func BuildImageURL(q core.Query, pageNum int) (string, error) {
+	base, _ := url.Parse("https://image.baidu.com/")
+	base.Path += "search/acjson"
+
+	params := url.Values{}
+	params.Add("tn", "resultjson_com")
+	params.Add("cl", "2") // Cl = 2 indicates image search
+
+	if q.Text != "" {
+		params.Add("word", q.Text)
+	}
+
+	if len(params.Get("word")) == 0 {
+		return "", errors.New("Empty query built")
+	}
+
+	if q.Limit != 0 {
+		params.Add("rn", "30")                     // Results per page
+		params.Add("pn", strconv.Itoa(pageNum*30)) // Offset
+	}
+
+	params.Add("fp", "result")
+	params.Add("ipn", "rj")
+	params.Add("ie", "utf-8")
+	params.Add("oe", "utf-8")
+	base.RawQuery = params.Encode()
+	return base.String(), nil
+}