From 284aebbfb273a25a45d2912d8bfad492f7c063a8 Mon Sep 17 00:00:00 2001 From: Rustem Kamalov Date: Sat, 8 Jul 2023 01:13:56 +0300 Subject: [PATCH] Baidu image search test --- baidu/all_test.go | 47 +++++++++++++++----- baidu/search.go | 106 ++++++++++++++++++++++++++++++++++++++++++++++ baidu/url.go | 29 +++++++++++++ 3 files changed, 171 insertions(+), 11 deletions(-) diff --git a/baidu/all_test.go b/baidu/all_test.go index 4ce6902..bc9dc56 100644 --- a/baidu/all_test.go +++ b/baidu/all_test.go @@ -13,7 +13,7 @@ var testQuery = core.Query{Text: "go", Site: "tutorialspoint.com", DateInterval: func init() { core.InitLogger(true, true) - opts := core.BrowserOpts{IsHeadless: true, IsLeakless: false, Timeout: time.Second * 2, WaitRequests: false} + opts := core.BrowserOpts{IsHeadless: false, IsLeakless: false, Timeout: time.Second * 10} browser, _ = core.NewBrowser(opts) } @@ -30,16 +30,7 @@ func TestUrlBuild(t *testing.T) { } } -// func TestSearchRaw(t *testing.T) { -// results, err := Search(testQuery) -// if err != nil { -// t.Fatal(err) -// } - -// fmt.Println(results) -// } - -func TestSearchBaidu(t *testing.T) { +func TestSearch(t *testing.T) { baid := New(*browser, core.SearchEngineOptions{}) results, err := baid.Search(testQuery) if err != nil { @@ -50,3 +41,37 @@ func TestSearchBaidu(t *testing.T) { t.Fatal("No results got from Baidu search") } } + +func TestImageUrlBuild(t *testing.T) { + query := core.Query{Text: "金毛猎犬", Limit: 10} + + res, err := BuildImageURL(query, 0) + if err != nil { + t.Fatal(err) + } + + want := "https://image.baidu.com/search/acjson?cl=2&ie=utf-8&ipn=r&tn=resultjson_com&word=%E9%87%91%E6%AF%9B%E7%8C%8E%E7%8A%AC" + + if want != res { + t.Fatalf("Wanted result `%s` doesn't match to resulted `%s`", want, res) + } +} + +func TestImageSearch(t *testing.T) { + + baid := New(*browser, core.SearchEngineOptions{}) + + query := core.Query{Text: "金毛猎犬", Limit: 31} + results, err := baid.SearchImage(query) + if err != nil { + t.Fatalf("Cannot [ImageBaidu]: %s", err) + } + + if len(results) == 0 { + t.Fatalf("[ImageBaidu] returned empty result") + } + + // for _, r := range results { + // fmt.Printf("%+v\n", r) + // } +} diff --git a/baidu/search.go b/baidu/search.go index c09a3b7..d8b3ec9 100644 --- a/baidu/search.go +++ b/baidu/search.go @@ -1,6 +1,8 @@ package baidu import ( + "encoding/json" + "fmt" "strings" "github.com/go-rod/rod" @@ -9,6 +11,34 @@ import ( "golang.org/x/time/rate" ) +type imageDataJson struct { + Query string `json:"queryExt"` + TotalResults int `json:"displayNum"` + + Data []struct { + Title string `json:"fromPageTitle"` + PictureDate string `json:"bdImgnewsDate"` + ThumbURL string `json:"thumbURL"` + Type string + Height int + Width int + IsCopyright int + + URL []struct { + SourcePage string `json:"FromURL"` + Original string `json:"ObjURL"` + } `json:"replaceUrl"` + + // Versions []struct { + // Height int + // Width int + // ImgSourcePage string `json:"fromURL"` + // URL string `json:"objURL"` + // Type string + // } `json:"setList"` + } +} + type Baidu struct { core.Browser core.SearchEngineOptions @@ -123,3 +153,79 @@ func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) { return searchResults, nil } + +func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) { + logrus.Tracef("Start Baidu Image search, query: %+v", query) + + searchResults := []core.SearchResult{} + searchPage := 0 + + for len(searchResults) < query.Limit { + url, err := BuildImageURL(query, searchPage) + if err != nil { + return nil, err + } + + // Get anti-crawler cookies first, then reload page + page := baid.Navigate(url) + if !baid.LeavePageOpen { + defer page.Close() + } + page.Reload() + page.WaitLoad() + + result, err := page.Timeout(baid.Timeout).Search("body > pre") + if err != nil { + defer page.Close() + logrus.Errorf("Cannot parse search results: %s", err) + return nil, core.ErrSearchTimeout + } + + // Check why no results, maybe captcha? + if result == nil { + defer page.Close() + + if baid.isCaptcha(page) { + logrus.Errorf("Baidu captcha occurred during: %s", url) + return nil, core.ErrCaptcha + } else if baid.isTimeout(page) { + logrus.Errorf("Baidu timeout occurred during: %s", url) + return nil, core.ErrCaptcha + } + return nil, nil + } + + jsonText, err := result.First.Text() + if err != nil { + return nil, err + } + + var data imageDataJson + + err = json.Unmarshal([]byte(jsonText), &data) + if err != nil { + logrus.Errorf("Cannot unmarshal Baidu image JSON: %v\nData: %v", err, jsonText) + return nil, err + } + + for i, img := range data.Data { + if len(img.URL) == 0 { + continue + } + res := core.SearchResult{ + Rank: (searchPage * 30) + (i + 1), + URL: img.URL[0].Original, + Title: img.Title, + Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright)} + searchResults = append(searchResults, res) + } + + searchPage += 1 + + if !baid.LeavePageOpen { + page.Close() + } + } + + return searchResults, nil +} diff --git a/baidu/url.go b/baidu/url.go index 7aa39e2..666534b 100644 --- a/baidu/url.go +++ b/baidu/url.go @@ -76,3 +76,32 @@ func BuildURL(q core.Query) (string, error) { base.RawQuery = params.Encode() return base.String(), nil } + +func BuildImageURL(q core.Query, pageNum int) (string, error) { + base, _ := url.Parse("https://image.baidu.com/") + base.Path += "search/acjson" + + params := url.Values{} + params.Add("tn", "resultjson_com") + params.Add("cl", "2") // Cl = 2 indicates image search + + if q.Text != "" { + params.Add("word", q.Text) + } + + if len(params.Get("word")) == 0 { + return "", errors.New("Empty query built") + } + + if q.Limit != 0 { + params.Add("rn", "30") // Results per page + params.Add("pn", strconv.Itoa(pageNum*30)) // Offset + } + + params.Add("fp", "result") + params.Add("ipn", "rj") + params.Add("ie", "utf-8") + params.Add("oe", "utf-8") + base.RawQuery = params.Encode() + return base.String(), nil +}