zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-12 20:03:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

Fix google images selector + baidu images ads tag

Browse Source
This commit is contained in:
Rustem Kamalov
2024-05-12 02:57:39 +03:00
parent fc49deba67
commit aeb0616249
2 changed files with 19 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
baidu/search.go
View File

@@ -24,19 +24,11 @@ type imageDataJson struct {
Height int
Width int
IsCopyright int
URL []struct {
AdType string `json:"adType"`
URL []struct {
SourcePage string `json:"FromURL"`
Original string `json:"ObjURL"`
} `json:"replaceUrl"`
// Versions []struct {
// Height int
// Width int
// ImgSourcePage string `json:"fromURL"`
// URL string `json:"objURL"`
// Type string
// } `json:"setList"`
}
}
@@ -63,18 +55,12 @@ func (baid *Baidu) GetRateLimiter() *rate.Limiter {
func (baid *Baidu) isCaptcha(page *rod.Page) bool {
_, err := page.Timeout(baid.GetSelectorTimeout()).Search("div.passMod_dialog-body")
if err != nil {
return false
}
return true
return err == nil
}
func (baid *Baidu) isTimeout(page *rod.Page) bool {
_, err := page.Timeout(baid.GetSelectorTimeout()).Search("button.timeout-button")
if err != nil {
return false
}
return true
return err == nil
}
func (baid *Baidu) Search(query core.Query) ([]core.SearchResult, error) {
@@ -226,7 +212,15 @@ func (baid *Baidu) SearchImage(query core.Query) ([]core.SearchResult, error) {
Rank: (searchPage * 30) + (i + 1),
URL: img.URL[0].Original,
Title: img.Title,
Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright)}
Description: fmt.Sprintf("%v,%v,%vx%x,copyright:%v", img.PictureDate, img.Type, img.Height, img.Width, img.IsCopyright),
Ad: func() bool {
if img.AdType != "0" {
return true
} else {
return false
}
}(),
}
searchResults = append(searchResults, res)
}

20
google/search.go
View File

@@ -319,21 +319,12 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
page := gogl.Navigate(url)
defer gogl.close(page)
//// TODO: Case with cookie accept (appears with VPN)
// if page.MustInfo().URL != url {
// results, _ := page.Search("button[aria-label][jsaction]")
// if results != nil {
// //buttons, _ := results.All()
// //buttons[1].Click(proto.InputMouseButtonLeft, 1)
// }
// }
for len(searchResultsMap) < query.Limit {
page.WaitLoad()
page.Mouse.Scroll(0, 1000000, 1)
page.WaitLoad()
results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction]")
results, err := page.Timeout(gogl.Timeout).Search("div[data-hveid][data-ved][jsaction][jsdata]")
if err != nil {
logrus.Errorf("Cannot parse search results: %s", err)
return *core.ConvertSearchResultsMap(searchResultsMap), core.ErrSearchTimeout
@@ -365,18 +356,19 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
continue
}
dataID, err := r.Attribute("data-id")
dataVed, err := r.Attribute("data-ved")
if err != nil {
logrus.Error("Cannot find `data-ved` attr")
continue
}
// If already have image with this ID
if _, ok := searchResultsMap[*dataID]; ok {
if _, ok := searchResultsMap[*dataVed]; ok {
continue
}
// Get URLs
link, err := r.Element("a[tabindex][role]")
link, err := r.Element("a:not([ping])")
if err != nil {
continue
}
@@ -411,7 +403,7 @@ func (gogl *Google) SearchImage(query core.Query) ([]core.SearchResult, error) {
Title: title,
Description: fmt.Sprintf("Height:%v, Width:%v, Source Page: %v", imgSrc.Height, imgSrc.Width, imgSrc.PageURL),
}
searchResultsMap[*dataID] = gR
searchResultsMap[*dataVed] = gR
r.Remove()
}