diff --git a/baidu/search_raw.go b/baidu/search_raw.go index 695586d..188fd7b 100644 --- a/baidu/search_raw.go +++ b/baidu/search_raw.go @@ -1,9 +1,12 @@ package baidu import ( + "crypto/tls" "fmt" "net/http" + "net/url" "strings" + "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" @@ -11,8 +14,27 @@ import ( "github.com/sirupsen/logrus" ) -func baiduRequest(searchURL string) (*http.Response, error) { - baseClient := &http.Client{} +func baiduRequest(searchURL string, query core.Query) (*http.Response, error) { + // Create HTTP transport with proxy + transport := &http.Transport{} + if query.ProxyURL != "" { + proxyUrl, err := url.Parse(query.ProxyURL) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyUrl) + } + + // Set insecure TLS + if query.Insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + baseClient := &http.Client{ + Transport: transport, + Timeout: time.Second * 10, + } + req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -27,7 +49,7 @@ func baiduRequest(searchURL string) (*http.Response, error) { } func baiduResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromResponse(response) + doc, err := goquery.NewDocumentFromReader(response.Body) if err != nil { return nil, err } @@ -79,7 +101,7 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Baidu URL built: %s", googleURL) - res, err := baiduRequest(googleURL) + res, err := baiduRequest(googleURL, query) if err != nil { return nil, err } diff --git a/cmd/root.go b/cmd/root.go index a799355..ed72e3c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -40,6 +40,8 @@ type AppConfig struct { IsDebug bool `mapstructure:"debug"` IsVerbose bool `mapstructure:"verbose"` IsRawRequests bool `mapstructure:"raw_requests"` + ProxyURL string `mapstructure:"proxy"` + Insecure bool `mapstructure:"insecure"` } var config = Config{} @@ -133,4 +135,6 @@ func init() { RootCmd.PersistentFlags().BoolVarP(&config.App.IsRawRequests, "raw", "r", false, "Disable browser usage, use HTTP requests") RootCmd.PersistentFlags().BoolVarP(&config.App.IsLeaveHead, "leave", "", false, "Leave browser and tabs opened after search is made") RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key") + RootCmd.PersistentFlags().StringVarP(&config.App.ProxyURL, "proxy", "", "", "HTTP proxy URL (e.g. http://user:pass@proxy:8080)") + RootCmd.PersistentFlags().BoolVarP(&config.App.Insecure, "insecure", "k", false, "Allow insecure TLS connections") } diff --git a/cmd/search.go b/cmd/search.go index c137735..823d694 100644 --- a/cmd/search.go +++ b/cmd/search.go @@ -26,8 +26,10 @@ func search(cmd *cobra.Command, args []string) { var err error engineType := args[0] query := core.Query{ - Text: args[1], - Limit: 10, + Text: args[1], + Limit: 10, + ProxyURL: config.App.ProxyURL, + Insecure: config.App.Insecure, } results := []core.SearchResult{} diff --git a/core/common.go b/core/common.go index 7a52a66..bf0cfad 100644 --- a/core/common.go +++ b/core/common.go @@ -41,6 +41,8 @@ type Query struct { Site string // Search site Limit int // Limit the number of results Answers bool // Include question and answers from SERP page to results with negative indexes + ProxyURL string // Proxy URL for raw requests + Insecure bool // Allow insecure TLS connections } func (q Query) IsEmpty() bool { diff --git a/google/search_raw.go b/google/search_raw.go index 3e482ff..287a355 100644 --- a/google/search_raw.go +++ b/google/search_raw.go @@ -1,8 +1,11 @@ package google import ( + "crypto/tls" "net/http" + "net/url" "strings" + "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" @@ -10,8 +13,26 @@ import ( "github.com/sirupsen/logrus" ) -func googleRequest(searchURL string) (*http.Response, error) { - baseClient := &http.Client{} +func googleRequest(searchURL string, query core.Query) (*http.Response, error) { + // Create HTTP transport with proxy + transport := &http.Transport{} + if query.ProxyURL != "" { + proxyUrl, err := url.Parse(query.ProxyURL) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyUrl) + } + + // Set insecure TLS + if query.Insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + baseClient := &http.Client{ + Transport: transport, + Timeout: time.Second * 10, + } req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -26,7 +47,7 @@ func googleRequest(searchURL string) (*http.Response, error) { } func googleResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromResponse(response) + doc, err := goquery.NewDocumentFromReader(response.Body) if err != nil { return nil, err } @@ -77,7 +98,7 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Google URL built: %s", googleURL) - res, err := googleRequest(googleURL) + res, err := googleRequest(googleURL, query) if err != nil { return nil, err } diff --git a/yandex/search_raw.go b/yandex/search_raw.go index 8779050..102a721 100644 --- a/yandex/search_raw.go +++ b/yandex/search_raw.go @@ -1,8 +1,11 @@ package yandex import ( + "crypto/tls" "net/http" + "net/url" "strings" + "time" "github.com/PuerkitoBio/goquery" "github.com/corpix/uarand" @@ -10,8 +13,27 @@ import ( "github.com/sirupsen/logrus" ) -func yandexRequest(searchURL string) (*http.Response, error) { - baseClient := &http.Client{} +func yandexRequest(searchURL string, query core.Query) (*http.Response, error) { + // Create HTTP transport with proxy + transport := &http.Transport{} + if query.ProxyURL != "" { + proxyUrl, err := url.Parse(query.ProxyURL) + if err != nil { + return nil, err + } + transport.Proxy = http.ProxyURL(proxyUrl) + } + + // Set insecure TLS + if query.Insecure { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } + + baseClient := &http.Client{ + Transport: transport, + Timeout: time.Second * 10, + } + req, err := http.NewRequest("GET", searchURL, nil) if err != nil { return nil, err @@ -26,7 +48,7 @@ func yandexRequest(searchURL string) (*http.Response, error) { } func yandexResultParser(response *http.Response) ([]core.SearchResult, error) { - doc, err := goquery.NewDocumentFromResponse(response) + doc, err := goquery.NewDocumentFromReader(response.Body) if err != nil { return nil, err } @@ -77,7 +99,7 @@ func Search(query core.Query) ([]core.SearchResult, error) { } logrus.Debugf("Yandex URL built: %s", googleURL) - res, err := yandexRequest(googleURL) + res, err := yandexRequest(googleURL, query) if err != nil { return nil, err }