zlei6/openserp
1
0
Fork 0
mirror of https://github.com/karust/openserp.git synced 2026-08-14 20:54:15 +08:00
Code Issues Packages Projects Releases Wiki Activity

Default ratelimeter

Browse Source
This commit is contained in:
Rustem Kamalov
2023-06-28 22:20:19 +03:00
parent 72bddf7fc9
commit dd5e0a2069
14 changed files with 105 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
README.md
View File

@@ -35,14 +35,14 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine
| lang | Search pages in selected language (`EN`, `DE`, `RU`...) |
| date | Date in `YYYYMMDD..YYYYMMDD` format (e.g. 20181010..20231010) |
| file | File extension to search (e.g. `PDF`, `DOC`) |
| site | Search only in selected site |
| site | Search within a specific website |
| limit | Limit the number of results |
### *Example response*
```JSON
[
{
"rank": 0,
"rank": 1,
"url": "https://en.wikipedia.org/wiki/%22Hello,_World!%22_program",
"title": "\"Hello, World!\" program",
"description": "A \"Hello, World!\" program is generally a computer program that ignores any input, and outputs or displays a message similar to \"Hello, World!\"."
@@ -50,8 +50,6 @@ You can replace `google` to `yandex` or `baidu` in query to change search engine
]
```
## CLI <a name="cli"></a> ⌨️
* Use `-h` flag to see commands.
* You can use `serve` command to serve API:
@@ -66,7 +64,7 @@ As a result you should get JSON output containting search results:
```json
[
{
"rank": 0,
"rank": 1,
"url": "https://www.cyberoptik.net/blog/6-sure-fire-ways-to-get-banned-from-google/",
"title": "11 Sure-Fire Ways to Get Banned From Google | CyberOptik",
"description": "How To Get Banned From Google · 1. Cloaking: The Art of Deception · 2. Plagiarism: Because Originality is Overrated · 3. Keyword Stuffing: More is Always Better · 4 ..."

18
baidu/search.go
View File

@@ -7,24 +7,32 @@ import (
"github.com/go-rod/rod"
"github.com/karust/openserp/core"
"github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
type Baidu struct {
core.Browser
checkTimeout time.Duration
core.SearchEngineOptions
}
func New(browser core.Browser) *Baidu {
func New(browser core.Browser, opts core.SearchEngineOptions) *Baidu {
baid := Baidu{Browser: browser}
baid.checkTimeout = time.Second * 2
opts.Init()
baid.SearchEngineOptions = opts
return &baid
}
func (baid *Baidu) Name() string {
return "baidu"
}
func (baid *Baidu) GetRateLimiter() *rate.Limiter {
ratelimit := rate.Every(baid.RateTime / time.Duration(baid.RateRequests))
return rate.NewLimiter(ratelimit, baid.RateBurst)
}
func (baid *Baidu) isCaptcha(page *rod.Page) bool {
_, err := page.Timeout(baid.checkTimeout).Search("div.passMod_dialog-body")
_, err := page.Timeout(baid.SelectorTimeout).Search("div.passMod_dialog-body")
if err != nil {
return false
}
@@ -32,7 +40,7 @@ func (baid *Baidu) isCaptcha(page *rod.Page) bool {
}
func (baid *Baidu) isTimeout(page *rod.Page) bool {
_, err := page.Timeout(baid.checkTimeout).Search("button.timeout-button")
_, err := page.Timeout(baid.SelectorTimeout).Search("button.timeout-button")
if err != nil {
return false
}

17
cmd/root.go
View File

@@ -11,7 +11,7 @@ import (
)
const (
version = "0.1.2"
version = "0.2.1"
defaultConfigFilename = "config"
envPrefix = "OPENSERP"
replaceHyphenWithCamelCase = false
@@ -22,12 +22,15 @@ type AppConfig struct {
Port int
Timeout int
ConfigPath string
IsBrowserHead bool `mapstructure:"head"`
IsLeaveHead bool `mapstructure:"leave_head"`
IsLeakless bool `mapstructure:"leakless"`
IsDebug bool `mapstructure:"debug"`
IsVerbose bool `mapstructure:"verbose"`
IsRawRequests bool `mapstructure:"raw_requests"`
IsBrowserHead bool `mapstructure:"head"`
IsLeaveHead bool `mapstructure:"leave_head"`
IsLeakless bool `mapstructure:"leakless"`
IsDebug bool `mapstructure:"debug"`
IsVerbose bool `mapstructure:"verbose"`
IsRawRequests bool `mapstructure:"raw_requests"`
GoogleConfig core.SearchEngineOptions `mapstructure:"google"`
YandexConfig core.SearchEngineOptions `mapstructure:"yandex"`
BaiduConfig core.SearchEngineOptions `mapstructure:"baidu"`
}
var appConf = AppConfig{}

6
cmd/search.go
View File

@@ -71,11 +71,11 @@ func searchBrowser(engineType string, query core.Query) ([]core.SearchResult, er
switch strings.ToLower(engineType) {
case "yandex":
engine = yandex.New(*browser)
engine = yandex.New(*browser, appConf.YandexConfig)
case "google":
engine = google.New(*browser)
engine = google.New(*browser, appConf.GoogleConfig)
case "baidu":
engine = baidu.New(*browser)
engine = baidu.New(*browser, appConf.BaiduConfig)
default:
logrus.Infof("No `%s` search engine found", engineType)
}

6
cmd/serve.go
View File

@@ -36,9 +36,9 @@ func serve(cmd *cobra.Command, args []string) {
logrus.Error(err)
}
yand := yandex.New(*browser)
gogl := google.New(*browser)
baidu := baidu.New(*browser)
yand := yandex.New(*browser, appConf.YandexConfig)
gogl := google.New(*browser, appConf.GoogleConfig)
baidu := baidu.New(*browser, appConf.BaiduConfig)
serv := core.NewServer(appConf.Host, appConf.Port, gogl, yand, baidu)
serv.Listen()

23
core/common.go
View File

@@ -3,6 +3,7 @@ package core
import (
"errors"
"strconv"
"time"
"github.com/gofiber/fiber/v2"
)
@@ -52,3 +53,25 @@ func (q *Query) InitFromContext(c *fiber.Ctx) error {
return nil
}
type SearchEngineOptions struct {
RateRequests int `mapstructure:"rate_requests"`
RateTime time.Duration `mapstructure:"rate_seconds"`
RateBurst int `mapstructure:"rate_burst"`
SelectorTimeout time.Duration `mapstructure:"selector_timeout"`
}
func (o *SearchEngineOptions) Init() {
if o.RateRequests == 0 {
o.RateRequests = 1
}
if o.RateTime == 0 {
o.RateTime = time.Second * 10
}
if o.RateBurst == 0 {
o.RateBurst = 1
}
if o.SelectorTimeout == 0 {
o.SelectorTimeout = time.Second * 5
}
}

11
core/server.go
View File

@@ -1,18 +1,21 @@
package core
import (
"context"
"errors"
"fmt"
"strings"
"github.com/gofiber/fiber/v2"
"github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
type SearchEngine interface {
Search(Query) ([]SearchResult, error)
IsInitialized() bool
Name() string
GetRateLimiter() *rate.Limiter
}
type Server struct {
@@ -29,15 +32,21 @@ func NewServer(host string, port int, searchEngines ...SearchEngine) *Server {
for _, engine := range searchEngines {
locEngine := engine
limiter := engine.GetRateLimiter()
serv.app.Get(fmt.Sprintf("/%s/search", strings.ToLower(locEngine.Name())), func(c *fiber.Ctx) error {
q := Query{}
err := q.InitFromContext(c)
if err != nil {
logrus.Errorf("Error while setting %s query: %s", locEngine.Name(), err)
return err
}
err = limiter.Wait(context.Background())
if err != nil {
logrus.Errorf("Ratelimiter error during %s query: %s", locEngine.Name(), err)
}
res, err := locEngine.Search(q)
if err != nil {
switch err {

5
core/server_test.go
View File

@@ -4,6 +4,8 @@ import (
"fmt"
"testing"
"time"
"golang.org/x/time/rate"
)
var (
@@ -25,6 +27,9 @@ func (SeMock) IsInitialized() bool {
func (s SeMock) Search(q Query) (res []SearchResult, err error) {
return []SearchResult{{Title: s.EngineName}}, nil
}
func (s SeMock) GetRateLimiter() *rate.Limiter {
return nil
}
func TestCreateServer(t *testing.T) {
se1 := SeMock{"mock_engine_1"}

1
go.mod
View File

@@ -12,6 +12,7 @@ require (
github.com/spf13/cobra v1.7.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.16.0
golang.org/x/time v0.3.0
)
require (

2
go.sum
View File

@@ -421,6 +421,8 @@ golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=

21
google/search.go
View File

@@ -10,26 +10,35 @@ import (
"github.com/go-rod/rod"
"github.com/karust/openserp/core"
"github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
type Google struct {
core.Browser
findNumRgxp *regexp.Regexp
checkTimeout time.Duration
core.SearchEngineOptions
findNumRgxp *regexp.Regexp
}
func New(browser core.Browser) *Google {
func New(browser core.Browser, opts core.SearchEngineOptions) *Google {
gogl := Google{Browser: browser}
gogl.checkTimeout = time.Second * 5
opts.Init()
gogl.SearchEngineOptions = opts
gogl.findNumRgxp = regexp.MustCompile("\\d")
return &gogl
}
func (gogl *Google) Name() string {
return "google"
}
func (gogl *Google) GetRateLimiter() *rate.Limiter {
ratelimit := rate.Every(gogl.RateTime / time.Duration(gogl.RateRequests))
return rate.NewLimiter(ratelimit, gogl.RateBurst)
}
func (gogl *Google) FindTotalResults(page *rod.Page) (int, error) {
resultsStats, err := page.Timeout(gogl.checkTimeout).Search("div#result-stats")
resultsStats, err := page.Timeout(gogl.SelectorTimeout).Search("div#result-stats")
if err != nil {
return 0, errors.New("Result stats not found: " + err.Error())
}
@@ -50,7 +59,7 @@ func (gogl *Google) FindTotalResults(page *rod.Page) (int, error) {
}
func (gogl *Google) isCaptcha(page *rod.Page) bool {
_, err := page.Timeout(gogl.checkTimeout).Search("form#captcha-form")
_, err := page.Timeout(gogl.SelectorTimeout).Search("form#captcha-form")
if err != nil {
return false
}

2
google/search_test.go
View File

@@ -15,7 +15,7 @@ func init() {
}
func TestSearchGoogle(t *testing.T) {
gogl := New(*browser)
gogl := New(*browser, core.SearchEngineOptions{})
query := core.Query{Text: "HEY", Limit: 10}
results, err := gogl.Search(query)

22
yandex/search.go
View File

@@ -6,17 +6,20 @@ import (
"github.com/go-rod/rod"
"github.com/karust/openserp/core"
"github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
type Yandex struct {
core.Browser
checkTimeout time.Duration // Timeout for secondary elements check
pageSleep time.Duration // Sleep between pages
core.SearchEngineOptions
pageSleep time.Duration // Sleep between pages
}
func New(browser core.Browser) *Yandex {
func New(browser core.Browser, opts core.SearchEngineOptions) *Yandex {
yand := Yandex{Browser: browser}
yand.checkTimeout = time.Second * 2
opts.Init()
yand.SearchEngineOptions = opts
yand.pageSleep = time.Second * 1
return &yand
}
@@ -25,8 +28,13 @@ func (yand *Yandex) Name() string {
return "yandex"
}
func (yand *Yandex) GetRateLimiter() *rate.Limiter {
ratelimit := rate.Every(yand.RateTime / time.Duration(yand.RateRequests))
return rate.NewLimiter(ratelimit, yand.RateBurst)
}
func (yand *Yandex) isCaptcha(page *rod.Page) bool {
_, err := page.Timeout(yand.checkTimeout).Search("form#checkbox-captcha-form")
_, err := page.Timeout(yand.SelectorTimeout).Search("form#checkbox-captcha-form")
if err != nil {
return false
}
@@ -37,12 +45,12 @@ func (yand *Yandex) isCaptcha(page *rod.Page) bool {
func (yand *Yandex) isNoResults(page *rod.Page) bool {
noResFound := false
_, err := page.Timeout(yand.checkTimeout).Search("div.EmptySearchResults-Title")
_, err := page.Timeout(yand.SelectorTimeout).Search("div.EmptySearchResults-Title")
if err == nil {
noResFound = true
}
_, err = page.Timeout(yand.checkTimeout).Search("div>div.RequestMeta-Message")
_, err = page.Timeout(yand.SelectorTimeout).Search("div>div.RequestMeta-Message")
if err == nil {
noResFound = true
}

2
yandex/search_test.go
View File

@@ -16,7 +16,7 @@ func init() {
func TestSearchYandex(t *testing.T) {
yand := New(*browser)
yand := New(*browser, core.SearchEngineOptions{})
query := core.Query{Text: "HEY", Limit: 10}
results, err := yand.Search(query)