mirror of
https://github.com/gosom/google-maps-scraper.git
synced 2026-09-19 07:27:12 +08:00
Add FastMode to fetch results much faster (#102)
Utilizing scrapemate's stealth mode to try to fetch directly the results without JS rendering. This version is much faster. However the version is still in BETA. Results are not extracted in full and there is the possibility of the client being blocked
This commit is contained in:
Georgios Komninos
committed by
GitHub
GitHub
parent
1015b4c978
commit
00bb13a087
@@ -42,8 +42,6 @@
|
||||
---
|
||||
## Try it
|
||||
|
||||
|
||||
|
||||
A command line and web based google maps scraper build using
|
||||
|
||||
[scrapemate](https://github.com/gosom/scrapemate) web crawling framework.
|
||||
@@ -53,11 +51,8 @@ customize it to your needs
|
||||
|
||||
|
||||
|
||||
|
||||
### Web UI:
|
||||
|
||||
|
||||
|
||||
```
|
||||
mkdir -p gmapsdata && docker run -v $PWD/gmapsdata:/gmapsdata -p 8080:8080 gosom/google-maps-scraper -data-folder /gmapsdata
|
||||
```
|
||||
@@ -98,6 +93,7 @@ Your support helps ensure continued improvement and maintenance.
|
||||
- Optionally extracts emails from the website of the business
|
||||
- SOCKS5/HTTP/HTTPS proxy support
|
||||
- Serverless execution via AWS Lambda functions (experimental & no documentation yet)
|
||||
- Fast Mode (BETA)
|
||||
|
||||
## Notes on email extraction
|
||||
|
||||
@@ -113,6 +109,22 @@ For the moment it only checks only one page of the website (the one that is regi
|
||||
Keep in mind that enabling email extraction results to larger processing time, since more
|
||||
pages are scraped.
|
||||
|
||||
## Fast Mode
|
||||
|
||||
Fast mode returns you at most 21 search results per query ordered by distance from the **latitude** and **longitude** provided.
|
||||
All the results are within the specificied **radius**
|
||||
|
||||
It does not contain all the data points but basic ones.
|
||||
However it provides the ability to extract data really fast.
|
||||
|
||||
When you use the fast mode ensure that you have provided:
|
||||
- zoom
|
||||
- radius (in meters)
|
||||
- latitude
|
||||
- longitude
|
||||
|
||||
|
||||
**Fast mode is Beta, you may experience blocking**
|
||||
|
||||
## Extracted Data Points
|
||||
|
||||
@@ -195,7 +207,6 @@ The results are written when they arrive in the `results` file you specified
|
||||
### Command line options
|
||||
|
||||
try `./google-maps-scraper -h` to see the command line options available:
|
||||
|
||||
```
|
||||
-aws-access-key string
|
||||
AWS access key
|
||||
@@ -225,6 +236,8 @@ try `./google-maps-scraper -h` to see the command line options available:
|
||||
extract emails from websites
|
||||
-exit-on-inactivity duration
|
||||
exit after inactivity duration (e.g., '5m')
|
||||
-fast-mode
|
||||
fast mode (reduced data collection)
|
||||
-function-name string
|
||||
AWS Lambda function name
|
||||
-geo string
|
||||
@@ -239,6 +252,8 @@ try `./google-maps-scraper -h` to see the command line options available:
|
||||
produce seed jobs only (requires dsn)
|
||||
-proxies string
|
||||
comma separated list of proxies to use in the format protocol://user:pass@host:port example: socks5://localhost:9050 or http://user:pass@localhost:9050
|
||||
-radius float
|
||||
search radius in meters. Default is 10000 meters (default 10000)
|
||||
-results string
|
||||
path to the results file [default: stdout] (default "stdout")
|
||||
-s3-bucket string
|
||||
@@ -248,7 +263,7 @@ try `./google-maps-scraper -h` to see the command line options available:
|
||||
-writer string
|
||||
use custom writer plugin (format: 'dir:pluginName')
|
||||
-zoom int
|
||||
set zoom level (0-21) for search
|
||||
set zoom level (0-21) for search (default 15)
|
||||
```
|
||||
|
||||
## Using a custom writer
|
||||
@@ -445,3 +460,4 @@ banner is generated using OpenAI's DALE
|
||||
|
||||
|
||||
If you register via the links on my page I may get a commission. This is another way to support my work
|
||||
|
||||
|
||||
@@ -3,7 +3,10 @@ package gmaps
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"iter"
|
||||
"math"
|
||||
"runtime/debug"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
@@ -91,6 +94,33 @@ type Entry struct {
|
||||
Emails []string `json:"emails"`
|
||||
}
|
||||
|
||||
func (e *Entry) haversineDistance(lat, lon float64) float64 {
|
||||
const R = 6371e3 // earth radius in meters
|
||||
|
||||
clat := lat * math.Pi / 180
|
||||
clon := lon * math.Pi / 180
|
||||
|
||||
elat := e.Latitude * math.Pi / 180
|
||||
elon := e.Longtitude * math.Pi / 180
|
||||
|
||||
dlat := elat - clat
|
||||
dlon := elon - clon
|
||||
|
||||
a := math.Sin(dlat/2)*math.Sin(dlat/2) +
|
||||
math.Cos(clat)*math.Cos(elat)*
|
||||
math.Sin(dlon/2)*math.Sin(dlon/2)
|
||||
|
||||
c := 2 * math.Atan2(math.Sqrt(a), math.Sqrt(1-a))
|
||||
|
||||
return R * c
|
||||
}
|
||||
|
||||
func (e *Entry) isWithinRadius(lat, lon, radius float64) bool {
|
||||
distance := e.haversineDistance(lat, lon)
|
||||
|
||||
return distance <= radius
|
||||
}
|
||||
|
||||
func (e *Entry) IsWebsiteValidForEmail() bool {
|
||||
if e.WebSite == "" {
|
||||
return false
|
||||
@@ -555,3 +585,44 @@ func decodeURL(url string) (string, error) {
|
||||
|
||||
return unquoted, nil
|
||||
}
|
||||
|
||||
type EntryWithDistance struct {
|
||||
Entry *Entry
|
||||
Distance float64
|
||||
}
|
||||
|
||||
func filterAndSortEntriesWithinRadius(entries []*Entry, lat, lon, radius float64) []*Entry {
|
||||
withinRadiusIterator := func(yield func(EntryWithDistance) bool) {
|
||||
for _, entry := range entries {
|
||||
distance := entry.haversineDistance(lat, lon)
|
||||
if distance <= radius {
|
||||
if !yield(EntryWithDistance{Entry: entry, Distance: distance}) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
entriesWithDistance := slices.Collect(iter.Seq[EntryWithDistance](withinRadiusIterator))
|
||||
|
||||
slices.SortFunc(entriesWithDistance, func(a, b EntryWithDistance) int {
|
||||
switch {
|
||||
case a.Distance < b.Distance:
|
||||
return -1
|
||||
case a.Distance > b.Distance:
|
||||
return 1
|
||||
default:
|
||||
return 0
|
||||
}
|
||||
})
|
||||
|
||||
resultIterator := func(yield func(*Entry) bool) {
|
||||
for _, e := range entriesWithDistance {
|
||||
if !yield(e.Entry) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return slices.Collect(iter.Seq[*Entry](resultIterator))
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package gmaps_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
@@ -200,3 +201,18 @@ func Test_EntryFromJSONRaw2(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.Greater(t, len(entry.About), 0)
|
||||
}
|
||||
|
||||
func Test_EntryFromJsonC(t *testing.T) {
|
||||
raw, err := os.ReadFile("../testdata/output.json")
|
||||
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, raw)
|
||||
|
||||
entries, err := gmaps.ParseSearchResults(raw)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, entry := range entries {
|
||||
fmt.Printf("%+v\n", entry)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
package gmaps
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
olc "github.com/google/open-location-code/go"
|
||||
)
|
||||
|
||||
func ParseSearchResults(raw []byte) ([]*Entry, error) {
|
||||
var data []any
|
||||
if err := json.Unmarshal(raw, &data); err != nil {
|
||||
return nil, fmt.Errorf("failed to unmarshal JSON: %w", err)
|
||||
}
|
||||
|
||||
if len(data) == 0 {
|
||||
return nil, fmt.Errorf("empty JSON data")
|
||||
}
|
||||
|
||||
container, ok := data[0].([]any)
|
||||
if !ok || len(container) == 0 {
|
||||
return nil, fmt.Errorf("invalid business list structure")
|
||||
}
|
||||
|
||||
items := getNthElementAndCast[[]any](container, 1)
|
||||
if len(items) < 2 {
|
||||
return nil, fmt.Errorf("empty business list")
|
||||
}
|
||||
|
||||
entries := make([]*Entry, 0, len(items)-1)
|
||||
|
||||
for i := 1; i < len(items); i++ {
|
||||
arr, ok := items[i].([]any)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
|
||||
business := getNthElementAndCast[[]any](arr, 14)
|
||||
|
||||
var entry Entry
|
||||
|
||||
entry.ID = getNthElementAndCast[string](business, 0)
|
||||
entry.Title = getNthElementAndCast[string](business, 11)
|
||||
entry.Categories = toStringSlice(getNthElementAndCast[[]any](business, 13))
|
||||
entry.WebSite = getNthElementAndCast[string](business, 7, 0)
|
||||
|
||||
entry.ReviewRating = getNthElementAndCast[float64](business, 4, 7)
|
||||
entry.ReviewCount = int(getNthElementAndCast[float64](business, 4, 8))
|
||||
|
||||
fullAddress := getNthElementAndCast[[]any](business, 2)
|
||||
|
||||
entry.Address = func() string {
|
||||
sb := strings.Builder{}
|
||||
|
||||
for i, part := range fullAddress {
|
||||
if i > 0 {
|
||||
sb.WriteString(", ")
|
||||
}
|
||||
|
||||
sb.WriteString(fmt.Sprintf("%v", part))
|
||||
}
|
||||
|
||||
return sb.String()
|
||||
}()
|
||||
|
||||
entry.Latitude = getNthElementAndCast[float64](business, 9, 2)
|
||||
entry.Longtitude = getNthElementAndCast[float64](business, 9, 3)
|
||||
entry.Phone = strings.ReplaceAll(getNthElementAndCast[string](business, 178, 0, 0), " ", "")
|
||||
entry.OpenHours = getHours(business)
|
||||
entry.Status = getNthElementAndCast[string](business, 34, 4, 4)
|
||||
entry.Timezone = getNthElementAndCast[string](business, 30)
|
||||
entry.DataID = getNthElementAndCast[string](business, 10)
|
||||
|
||||
entry.PlusCode = olc.Encode(entry.Latitude, entry.Longtitude, 10)
|
||||
|
||||
entries = append(entries, &entry)
|
||||
}
|
||||
|
||||
return entries, nil
|
||||
}
|
||||
|
||||
func toStringSlice(arr []any) []string {
|
||||
ans := make([]string, 0, len(arr))
|
||||
for _, v := range arr {
|
||||
ans = append(ans, fmt.Sprintf("%v", v))
|
||||
}
|
||||
|
||||
return ans
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package gmaps
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/gosom/google-maps-scraper/exiter"
|
||||
"github.com/gosom/scrapemate"
|
||||
)
|
||||
|
||||
type SearchJobOptions func(*SearchJob)
|
||||
|
||||
type MapLocation struct {
|
||||
Lat float64
|
||||
Lon float64
|
||||
ZoomLvl float64
|
||||
Radius float64
|
||||
}
|
||||
|
||||
type MapSearchParams struct {
|
||||
Location MapLocation
|
||||
Query string
|
||||
ViewportW int
|
||||
ViewportH int
|
||||
Hl string
|
||||
}
|
||||
|
||||
type SearchJob struct {
|
||||
scrapemate.Job
|
||||
|
||||
params *MapSearchParams
|
||||
ExitMonitor exiter.Exiter
|
||||
}
|
||||
|
||||
func NewSearchJob(params *MapSearchParams, opts ...SearchJobOptions) *SearchJob {
|
||||
const (
|
||||
defaultPrio = scrapemate.PriorityMedium
|
||||
defaultMaxRetries = 3
|
||||
baseURL = "https://maps.google.com/search"
|
||||
)
|
||||
|
||||
job := SearchJob{
|
||||
Job: scrapemate.Job{
|
||||
ID: uuid.New().String(),
|
||||
Method: http.MethodGet,
|
||||
URL: baseURL,
|
||||
URLParams: buildGoogleMapsParams(params),
|
||||
MaxRetries: defaultMaxRetries,
|
||||
Priority: defaultPrio,
|
||||
},
|
||||
}
|
||||
|
||||
job.params = params
|
||||
|
||||
for _, opt := range opts {
|
||||
opt(&job)
|
||||
}
|
||||
|
||||
return &job
|
||||
}
|
||||
|
||||
func WithSearchJobExitMonitor(exitMonitor exiter.Exiter) SearchJobOptions {
|
||||
return func(j *SearchJob) {
|
||||
j.ExitMonitor = exitMonitor
|
||||
}
|
||||
}
|
||||
|
||||
func (j *SearchJob) Process(_ context.Context, resp *scrapemate.Response) (any, []scrapemate.IJob, error) {
|
||||
defer func() {
|
||||
resp.Document = nil
|
||||
resp.Body = nil
|
||||
resp.Meta = nil
|
||||
}()
|
||||
|
||||
body := removeFirstLine(resp.Body)
|
||||
if len(body) == 0 {
|
||||
return nil, nil, fmt.Errorf("empty response body")
|
||||
}
|
||||
|
||||
entries, err := ParseSearchResults(body)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("failed to parse search results: %w", err)
|
||||
}
|
||||
|
||||
entries = filterAndSortEntriesWithinRadius(entries,
|
||||
j.params.Location.Lat,
|
||||
j.params.Location.Lon,
|
||||
j.params.Location.Radius,
|
||||
)
|
||||
|
||||
if j.ExitMonitor != nil {
|
||||
j.ExitMonitor.IncrSeedCompleted(1)
|
||||
j.ExitMonitor.IncrPlacesFound(len(entries))
|
||||
j.ExitMonitor.IncrPlacesCompleted(len(entries))
|
||||
}
|
||||
|
||||
return entries, nil, nil
|
||||
}
|
||||
|
||||
func removeFirstLine(data []byte) []byte {
|
||||
if len(data) == 0 {
|
||||
return data
|
||||
}
|
||||
|
||||
index := bytes.IndexByte(data, '\n')
|
||||
if index == -1 {
|
||||
return []byte{}
|
||||
}
|
||||
|
||||
return data[index+1:]
|
||||
}
|
||||
|
||||
func buildGoogleMapsParams(params *MapSearchParams) map[string]string {
|
||||
params.ViewportH = 800
|
||||
params.ViewportW = 600
|
||||
|
||||
ans := map[string]string{
|
||||
"tbm": "map",
|
||||
"authuser": "0",
|
||||
"hl": params.Hl,
|
||||
"q": params.Query,
|
||||
}
|
||||
|
||||
pb := fmt.Sprintf("!4m12!1m3!1d3826.902183192154!2d%.4f!3d%.4f!2m3!1f0!2f0!3f0!3m2!1i%d!2i%d!4f%.1f!7i20!8i0"+
|
||||
"!10b1!12m22!1m3!18b1!30b1!34e1!2m3!5m1!6e2!20e3!4b0!10b1!12b1!13b1!16b1!17m1!3e1!20m3!5e2!6b1!14b1!46m1!1b0"+
|
||||
"!96b1!19m4!2m3!1i360!2i120!4i8",
|
||||
params.Location.Lon,
|
||||
params.Location.Lat,
|
||||
params.ViewportW,
|
||||
params.ViewportH,
|
||||
params.Location.ZoomLvl,
|
||||
)
|
||||
|
||||
ans["pb"] = pb
|
||||
|
||||
return ans
|
||||
}
|
||||
@@ -11,8 +11,9 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/service/lambda v1.64.1
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.66.2
|
||||
github.com/golangci/golangci-lint v1.61.0
|
||||
github.com/google/open-location-code/go v0.0.0-20241213145606-bf601ad90a45
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/gosom/scrapemate v0.8.2
|
||||
github.com/gosom/scrapemate v0.8.3
|
||||
github.com/jackc/pgx/v5 v5.7.1
|
||||
github.com/mattn/go-runewidth v0.0.16
|
||||
github.com/mcnijman/go-emailaddress v1.1.1
|
||||
@@ -21,8 +22,8 @@ require (
|
||||
github.com/shirou/gopsutil/v4 v4.24.9
|
||||
github.com/stretchr/testify v1.9.0
|
||||
golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c
|
||||
golang.org/x/sync v0.8.0
|
||||
golang.org/x/term v0.25.0
|
||||
golang.org/x/sync v0.10.0
|
||||
golang.org/x/term v0.27.0
|
||||
modernc.org/sqlite v1.33.1
|
||||
)
|
||||
|
||||
@@ -39,11 +40,16 @@ require (
|
||||
github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect
|
||||
github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0 // indirect
|
||||
github.com/Masterminds/semver/v3 v3.3.0 // indirect
|
||||
github.com/Noooste/azuretls-client v1.5.11 // indirect
|
||||
github.com/Noooste/fhttp v1.0.12 // indirect
|
||||
github.com/Noooste/utls v1.2.12 // indirect
|
||||
github.com/Noooste/websocket v1.0.3 // indirect
|
||||
github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect
|
||||
github.com/alecthomas/go-check-sumtype v0.1.4 // indirect
|
||||
github.com/alexkohler/nakedret/v2 v2.0.4 // indirect
|
||||
github.com/alexkohler/prealloc v1.0.0 // indirect
|
||||
github.com/alingse/asasalint v0.0.11 // indirect
|
||||
github.com/andybalholm/brotli v1.1.1 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/ashanbrown/forbidigo v1.6.0 // indirect
|
||||
github.com/ashanbrown/makezero v1.1.1 // indirect
|
||||
@@ -75,6 +81,7 @@ require (
|
||||
github.com/charithe/durationcheck v0.0.10 // indirect
|
||||
github.com/chavacava/garif v0.1.0 // indirect
|
||||
github.com/ckaznocha/intrange v0.2.0 // indirect
|
||||
github.com/cloudflare/circl v1.5.0 // indirect
|
||||
github.com/curioswitch/go-reassign v0.2.0 // indirect
|
||||
github.com/daixiang0/gci v0.13.5 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
@@ -83,7 +90,7 @@ require (
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.8.0 // indirect
|
||||
github.com/ettle/strcase v0.2.0 // indirect
|
||||
github.com/fatih/color v1.17.0 // indirect
|
||||
github.com/fatih/color v1.18.0 // indirect
|
||||
github.com/fatih/structtag v1.2.0 // indirect
|
||||
github.com/firefart/nonamedreturns v1.0.5 // indirect
|
||||
github.com/fsnotify/fsnotify v1.5.4 // indirect
|
||||
@@ -141,6 +148,7 @@ require (
|
||||
github.com/karamaru-alpha/copyloopvar v1.1.0 // indirect
|
||||
github.com/kisielk/errcheck v1.7.0 // indirect
|
||||
github.com/kkHAIKE/contextcheck v1.1.5 // indirect
|
||||
github.com/klauspost/compress v1.17.11 // indirect
|
||||
github.com/kulti/thelper v0.6.3 // indirect
|
||||
github.com/kunwardeep/paralleltest v1.0.10 // indirect
|
||||
github.com/kyoh86/exportloopref v0.1.11 // indirect
|
||||
@@ -233,12 +241,12 @@ require (
|
||||
go.uber.org/automaxprocs v1.5.3 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.24.0 // indirect
|
||||
golang.org/x/crypto v0.28.0 // indirect
|
||||
golang.org/x/crypto v0.31.0 // indirect
|
||||
golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect
|
||||
golang.org/x/mod v0.21.0 // indirect
|
||||
golang.org/x/net v0.30.0 // indirect
|
||||
golang.org/x/sys v0.26.0 // indirect
|
||||
golang.org/x/text v0.19.0 // indirect
|
||||
golang.org/x/net v0.32.0 // indirect
|
||||
golang.org/x/sys v0.28.0 // indirect
|
||||
golang.org/x/text v0.21.0 // indirect
|
||||
golang.org/x/tools v0.26.0 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
@@ -254,5 +262,3 @@ require (
|
||||
mvdan.cc/gofumpt v0.7.0 // indirect
|
||||
mvdan.cc/unparam v0.0.0-20240528143540-8a5130ca722f // indirect
|
||||
)
|
||||
|
||||
///replace github.com/gosom/scrapemate v0.8.2 => ../scrapemate
|
||||
|
||||
@@ -57,6 +57,14 @@ github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0 h1:/fTUt5vmbkAcMBt4YQiuC
|
||||
github.com/GaijinEntertainment/go-exhaustruct/v3 v3.3.0/go.mod h1:ONJg5sxcbsdQQ4pOW8TGdTidT2TMAUy/2Xhr8mrYaao=
|
||||
github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0=
|
||||
github.com/Masterminds/semver/v3 v3.3.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM=
|
||||
github.com/Noooste/azuretls-client v1.5.11 h1:adoAsHjKbxcxDxineOMMS2z2qwauHXvNhqOC3+N2pYU=
|
||||
github.com/Noooste/azuretls-client v1.5.11/go.mod h1:EmuuM4FlELR5XlTqJQfe02dIP6Fi92bhcEPtnjJg0SU=
|
||||
github.com/Noooste/fhttp v1.0.12 h1:2N15bIATKaC6q+LVyRGyxPyuqEPvwAS3Uk1peC3YVHU=
|
||||
github.com/Noooste/fhttp v1.0.12/go.mod h1:CMVxKOhNheqJN5HYE4Rlvz2SRdV8Uv7YWmi6OwmB/Bk=
|
||||
github.com/Noooste/utls v1.2.12 h1:Zcm/7OB6W4Ro1q2OV1BrFb3qBI7uqYeC21wHYX+Ez9I=
|
||||
github.com/Noooste/utls v1.2.12/go.mod h1:CJaLzDHOhjuKESY3/wTSEzs3N2QgdXTrNQE3sW2632M=
|
||||
github.com/Noooste/websocket v1.0.3 h1:drW7tvZ3YqzqI9wApnaH1Q0syFMXO7gbLlsBWjZvMNA=
|
||||
github.com/Noooste/websocket v1.0.3/go.mod h1:Qhw0Rtuju/fPPbcb3R5XGq7poa51qPDL462jTltl9nQ=
|
||||
github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA=
|
||||
github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ=
|
||||
github.com/PuerkitoBio/goquery v1.10.0 h1:6fiXdLuUvYs2OJSvNRqlNPoBm6YABE226xrbavY5Wv4=
|
||||
@@ -78,6 +86,8 @@ github.com/alexkohler/prealloc v1.0.0 h1:Hbq0/3fJPQhNkN0dR95AVrr6R7tou91y0uHG5pO
|
||||
github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE=
|
||||
github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw=
|
||||
github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I=
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
|
||||
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
|
||||
github.com/ashanbrown/forbidigo v1.6.0 h1:D3aewfM37Yb3pxHujIPSpTf6oQk9sc9WZi8gerOIVIY=
|
||||
@@ -162,6 +172,8 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
|
||||
github.com/ckaznocha/intrange v0.2.0 h1:FykcZuJ8BD7oX93YbO1UY9oZtkRbp+1/kJcDjkefYLs=
|
||||
github.com/ckaznocha/intrange v0.2.0/go.mod h1:r5I7nUlAAG56xmkOpw4XVr16BXhwYTUdcuRFeevn1oE=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys=
|
||||
github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
|
||||
@@ -186,8 +198,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q=
|
||||
github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A=
|
||||
github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
|
||||
github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
|
||||
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
|
||||
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
|
||||
github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4=
|
||||
github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
|
||||
github.com/firefart/nonamedreturns v1.0.5 h1:tM+Me2ZaXs8tfdDw3X6DOX++wMCOqzYUho6tUTYIdRA=
|
||||
@@ -332,6 +344,8 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
|
||||
github.com/google/open-location-code/go v0.0.0-20241213145606-bf601ad90a45 h1:+2LSGdn52FkvjGv/BPigVlhtKXQQO1MKUxG14AiKUAw=
|
||||
github.com/google/open-location-code/go v0.0.0-20241213145606-bf601ad90a45/go.mod h1:eJfRN6aj+kR/rnua/rw9jAgYhqoMHldQkdTi+sePRKk=
|
||||
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
|
||||
github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
|
||||
@@ -350,8 +364,8 @@ github.com/gordonklaus/ineffassign v0.1.0 h1:y2Gd/9I7MdY1oEIt+n+rowjBNDcLQq3RsH5
|
||||
github.com/gordonklaus/ineffassign v0.1.0/go.mod h1:Qcp2HIAYhR7mNUVSIxZww3Guk4it82ghYcEXIAk+QT0=
|
||||
github.com/gosom/kit v0.0.0-20230309082109-543b32ac686a h1:5tcB33GTXm0pFUiEFpmE91tMsHQj+I+W7zubT8J/ugI=
|
||||
github.com/gosom/kit v0.0.0-20230309082109-543b32ac686a/go.mod h1:ngnWSsuBEpCA5Y43kZRa3x8RBYZZ4LDtvZHO4N5dHZ0=
|
||||
github.com/gosom/scrapemate v0.8.2 h1:QdzMtYwjSIRqC6aFvZAXppKlzmGGJ6FetIE420+hqec=
|
||||
github.com/gosom/scrapemate v0.8.2/go.mod h1:0EuH67Lz16HlyxQfoSOY46zpLNq/75/qlarYstMPHiQ=
|
||||
github.com/gosom/scrapemate v0.8.3 h1:okEpdLKkdaivKlZt0GzCVouwRnWGMei5OU4okAuHVYA=
|
||||
github.com/gosom/scrapemate v0.8.3/go.mod h1:k6nFr9vq78/JHPAq6MgLKo2lh3eMaYnRNmv/CrxNY0s=
|
||||
github.com/gostaticanalysis/analysisutil v0.7.1 h1:ZMCjoue3DtDWQ5WyU16YbjbQEQ3VuzwxALrpYd+HeKk=
|
||||
github.com/gostaticanalysis/analysisutil v0.7.1/go.mod h1:v21E3hY37WKMGSnbsw2S/ojApNWb6C1//mXO48CXbVc=
|
||||
github.com/gostaticanalysis/comment v1.4.1/go.mod h1:ih6ZxzTHLdadaiSnF5WY3dxUoXfXAlTaRzuaNDlSado=
|
||||
@@ -418,6 +432,8 @@ github.com/kisielk/errcheck v1.7.0/go.mod h1:1kLL+jV4e+CFfueBmI1dSK2ADDyQnlrnrY/
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/kkHAIKE/contextcheck v1.1.5 h1:CdnJh63tcDe53vG+RebdpdXJTc9atMgGqdx8LXxiilg=
|
||||
github.com/kkHAIKE/contextcheck v1.1.5/go.mod h1:O930cpht4xb1YQpK+1+AgoM3mFsvxr7uyFptcnWTYUA=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
|
||||
@@ -677,6 +693,8 @@ github.com/uudashr/gocognit v1.1.3 h1:l+a111VcDbKfynh+airAy/DJQKaXh2m9vkoysMPSZy
|
||||
github.com/uudashr/gocognit v1.1.3/go.mod h1:aKH8/e8xbTRBwjbCkwZ8qt4l2EpKXl31KMHgSS+lZ2U=
|
||||
github.com/xen0n/gosmopolitan v1.2.2 h1:/p2KTnMzwRexIW8GlKawsTWOxn7UHA+jCMF/V8HHtvU=
|
||||
github.com/xen0n/gosmopolitan v1.2.2/go.mod h1:7XX7Mj61uLYrj0qmeN0zi7XDon9JRAEhYQqAPLVNTeg=
|
||||
github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
|
||||
github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
|
||||
github.com/yagipy/maintidx v1.0.0 h1:h5NvIsCz+nRDapQ0exNv4aJ0yXSI0420omVANTv3GJM=
|
||||
github.com/yagipy/maintidx v1.0.0/go.mod h1:0qNf/I/CCZXSMhsRsrEPDZ+DkekpKLXAJfsTACwgXLk=
|
||||
github.com/yeya24/promlinter v0.3.0 h1:JVDbMp08lVCP7Y6NP3qHroGAO6z2yGKQtS5JsjqtoFs=
|
||||
@@ -726,8 +744,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
|
||||
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
|
||||
@@ -816,8 +834,8 @@ golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
|
||||
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
|
||||
golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
|
||||
golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
@@ -837,8 +855,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ=
|
||||
golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
@@ -900,8 +918,8 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
|
||||
golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -911,8 +929,8 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24=
|
||||
golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M=
|
||||
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
@@ -925,8 +943,8 @@ golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
|
||||
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
|
||||
+36
-1
@@ -25,8 +25,18 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9 h1:VpgP7xuJadIUu
|
||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802 h1:1BDTz0u9nC3//pOCMdNH+CiXJVYJh5UQNCOBG7jbELc=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
|
||||
github.com/Noooste/azuretls-client v1.5.11 h1:adoAsHjKbxcxDxineOMMS2z2qwauHXvNhqOC3+N2pYU=
|
||||
github.com/Noooste/azuretls-client v1.5.11/go.mod h1:EmuuM4FlELR5XlTqJQfe02dIP6Fi92bhcEPtnjJg0SU=
|
||||
github.com/Noooste/fhttp v1.0.12 h1:2N15bIATKaC6q+LVyRGyxPyuqEPvwAS3Uk1peC3YVHU=
|
||||
github.com/Noooste/fhttp v1.0.12/go.mod h1:CMVxKOhNheqJN5HYE4Rlvz2SRdV8Uv7YWmi6OwmB/Bk=
|
||||
github.com/Noooste/utls v1.2.12 h1:Zcm/7OB6W4Ro1q2OV1BrFb3qBI7uqYeC21wHYX+Ez9I=
|
||||
github.com/Noooste/utls v1.2.12/go.mod h1:CJaLzDHOhjuKESY3/wTSEzs3N2QgdXTrNQE3sW2632M=
|
||||
github.com/Noooste/websocket v1.0.3 h1:drW7tvZ3YqzqI9wApnaH1Q0syFMXO7gbLlsBWjZvMNA=
|
||||
github.com/Noooste/websocket v1.0.3/go.mod h1:Qhw0Rtuju/fPPbcb3R5XGq7poa51qPDL462jTltl9nQ=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E=
|
||||
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
|
||||
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
|
||||
github.com/armon/go-metrics v0.3.10 h1:FR+drcQStOe+32sYyJYyZ7FIdgoGGBnwLl+flodp8Uo=
|
||||
github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
|
||||
@@ -34,6 +44,8 @@ github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8=
|
||||
github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
|
||||
github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys=
|
||||
github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
|
||||
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f h1:WBZRG4aNOuI15bLRrCgN8fCq8E5Xuty6jGbmSNEvSsU=
|
||||
github.com/confluentinc/confluent-kafka-go v1.9.2 h1:gV/GxhMBUb03tFWkN+7kdhg+zf+QUM+wVkI9zwh770Q=
|
||||
github.com/confluentinc/confluent-kafka-go v1.9.2/go.mod h1:ptXNqsuDfYbAE/LBW6pnwWZElUoWxHoV8E43DCrliyo=
|
||||
@@ -45,6 +57,8 @@ github.com/cristalhq/acmd v0.12.0 h1:RdlKnxjN+txbQosg8p/TRNZ+J1Rdne43MVQZ1zDhGWk
|
||||
github.com/cristalhq/acmd v0.12.0/go.mod h1:LG5oa43pE/BbxtfMoImHCQN++0Su7dzipdgBjMCBVDQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4 h1:rEvIZUSZ3fx39WIi3JkQqQBitGwpELBIYWeBVh6wn+E=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A=
|
||||
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
|
||||
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0=
|
||||
@@ -89,6 +103,8 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/
|
||||
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
|
||||
github.com/gosom/scrapemate v0.8.1 h1:xyKeZTQ+mrnjCA6W7KnsXLuyUBEcbiChhpJtC8haI+8=
|
||||
github.com/gosom/scrapemate v0.8.1/go.mod h1:0EuH67Lz16HlyxQfoSOY46zpLNq/75/qlarYstMPHiQ=
|
||||
github.com/gosom/scrapemate v0.8.3 h1:okEpdLKkdaivKlZt0GzCVouwRnWGMei5OU4okAuHVYA=
|
||||
github.com/gosom/scrapemate v0.8.3/go.mod h1:k6nFr9vq78/JHPAq6MgLKo2lh3eMaYnRNmv/CrxNY0s=
|
||||
github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=
|
||||
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
|
||||
github.com/hashicorp/consul/api v1.12.0 h1:k3y1FYv6nuKyNTqj6w9gXOx5r5CfLj/k/euUeBXj1OY=
|
||||
@@ -112,7 +128,6 @@ github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpT
|
||||
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c=
|
||||
github.com/ismurov/swaggerui v0.2.0 h1:rx/BTbufsCUMq0G2a0Cmd045nkrRmHBa7T249wqnVBM=
|
||||
github.com/ismurov/swaggerui v0.2.0/go.mod h1:EaaariTC2xXLMsKU9v3MdYT62/akXBvRFxmuY9zyqF0=
|
||||
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
|
||||
github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g=
|
||||
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
|
||||
github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
|
||||
@@ -122,6 +137,8 @@ github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4d
|
||||
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
|
||||
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
|
||||
github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
|
||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
@@ -205,16 +222,34 @@ go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2
|
||||
go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=
|
||||
go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=
|
||||
go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=
|
||||
golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug=
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG1In4gJY5YRhtpDNeDeHWs=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
|
||||
golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
|
||||
golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
|
||||
golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
|
||||
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457 h1:zf5N6UOrA487eEFacMePxjXAJctxKmyjKUsjA11Uzuk=
|
||||
golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
|
||||
golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U=
|
||||
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
google.golang.org/api v0.196.0 h1:k/RafYqebaIJBO3+SMnfEGtFVlvp5vSgqTUF54UN/zg=
|
||||
|
||||
@@ -65,13 +65,17 @@ func New(cfg *runner.Config) (runner.Runner, error) {
|
||||
)
|
||||
}
|
||||
|
||||
if cfg.Debug {
|
||||
opts = append(opts, scrapemateapp.WithJS(
|
||||
scrapemateapp.Headfull(),
|
||||
scrapemateapp.DisableImages(),
|
||||
))
|
||||
if !cfg.FastMode {
|
||||
if cfg.Debug {
|
||||
opts = append(opts, scrapemateapp.WithJS(
|
||||
scrapemateapp.Headfull(),
|
||||
scrapemateapp.DisableImages(),
|
||||
))
|
||||
} else {
|
||||
opts = append(opts, scrapemateapp.WithJS(scrapemateapp.DisableImages()))
|
||||
}
|
||||
} else {
|
||||
opts = append(opts, scrapemateapp.WithJS(scrapemateapp.DisableImages()))
|
||||
opts = append(opts, scrapemateapp.WithStealth())
|
||||
}
|
||||
|
||||
matecfg, err := scrapemateapp.NewConfig(
|
||||
@@ -130,12 +134,14 @@ func (d *dbrunner) produceSeedJobs(ctx context.Context) error {
|
||||
}
|
||||
|
||||
jobs, err := runner.CreateSeedJobs(
|
||||
d.cfg.FastMode,
|
||||
d.cfg.LangCode,
|
||||
input,
|
||||
d.cfg.MaxDepth,
|
||||
d.cfg.Email,
|
||||
d.cfg.GeoCoordinates,
|
||||
d.cfg.Zoom,
|
||||
d.cfg.Radius,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
|
||||
@@ -76,12 +76,14 @@ func (r *fileRunner) Run(ctx context.Context) (err error) {
|
||||
exitMonitor := exiter.New()
|
||||
|
||||
seedJobs, err = runner.CreateSeedJobs(
|
||||
r.cfg.FastMode,
|
||||
r.cfg.LangCode,
|
||||
r.input,
|
||||
r.cfg.MaxDepth,
|
||||
r.cfg.Email,
|
||||
r.cfg.GeoCoordinates,
|
||||
r.cfg.Zoom,
|
||||
r.cfg.Radius,
|
||||
dedup,
|
||||
exitMonitor,
|
||||
)
|
||||
@@ -194,14 +196,18 @@ func (r *fileRunner) setApp() error {
|
||||
)
|
||||
}
|
||||
|
||||
if r.cfg.Debug {
|
||||
opts = append(opts, scrapemateapp.WithJS(
|
||||
scrapemateapp.Headfull(),
|
||||
scrapemateapp.DisableImages(),
|
||||
),
|
||||
)
|
||||
if !r.cfg.FastMode {
|
||||
if r.cfg.Debug {
|
||||
opts = append(opts, scrapemateapp.WithJS(
|
||||
scrapemateapp.Headfull(),
|
||||
scrapemateapp.DisableImages(),
|
||||
),
|
||||
)
|
||||
} else {
|
||||
opts = append(opts, scrapemateapp.WithJS(scrapemateapp.DisableImages()))
|
||||
}
|
||||
} else {
|
||||
opts = append(opts, scrapemateapp.WithJS(scrapemateapp.DisableImages()))
|
||||
opts = append(opts, scrapemateapp.WithStealth())
|
||||
}
|
||||
|
||||
matecfg, err := scrapemateapp.NewConfig(
|
||||
|
||||
+76
-9
@@ -7,6 +7,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"plugin"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/gosom/google-maps-scraper/deduper"
|
||||
@@ -16,15 +17,56 @@ import (
|
||||
)
|
||||
|
||||
func CreateSeedJobs(
|
||||
fastmode bool,
|
||||
langCode string,
|
||||
r io.Reader,
|
||||
maxDepth int,
|
||||
email bool,
|
||||
geoCoordinates string,
|
||||
zoom int,
|
||||
radius float64,
|
||||
dedup deduper.Deduper,
|
||||
exitMonitor exiter.Exiter,
|
||||
) (jobs []scrapemate.IJob, err error) {
|
||||
var lat, lon float64
|
||||
|
||||
if fastmode {
|
||||
if geoCoordinates == "" {
|
||||
return nil, fmt.Errorf("geo coordinates are required in fast mode")
|
||||
}
|
||||
|
||||
parts := strings.Split(geoCoordinates, ",")
|
||||
if len(parts) != 2 {
|
||||
return nil, fmt.Errorf("invalid geo coordinates: %s", geoCoordinates)
|
||||
}
|
||||
|
||||
lat, err = strconv.ParseFloat(parts[0], 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid latitude: %w", err)
|
||||
}
|
||||
|
||||
lon, err = strconv.ParseFloat(parts[1], 64)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid longitude: %w", err)
|
||||
}
|
||||
|
||||
if lat < -90 || lat > 90 {
|
||||
return nil, fmt.Errorf("invalid latitude: %f", lat)
|
||||
}
|
||||
|
||||
if lon < -180 || lon > 180 {
|
||||
return nil, fmt.Errorf("invalid longitude: %f", lon)
|
||||
}
|
||||
|
||||
if zoom < 1 || zoom > 21 {
|
||||
return nil, fmt.Errorf("invalid zoom level: %d", zoom)
|
||||
}
|
||||
|
||||
if radius < 0 {
|
||||
return nil, fmt.Errorf("invalid radius: %f", radius)
|
||||
}
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(r)
|
||||
|
||||
for scanner.Scan() {
|
||||
@@ -40,18 +82,43 @@ func CreateSeedJobs(
|
||||
id = strings.TrimSpace(after)
|
||||
}
|
||||
|
||||
opts := []gmaps.GmapJobOptions{}
|
||||
var job scrapemate.IJob
|
||||
|
||||
if dedup != nil {
|
||||
opts = append(opts, gmaps.WithDeduper(dedup))
|
||||
if !fastmode {
|
||||
opts := []gmaps.GmapJobOptions{}
|
||||
|
||||
if dedup != nil {
|
||||
opts = append(opts, gmaps.WithDeduper(dedup))
|
||||
}
|
||||
|
||||
if exitMonitor != nil {
|
||||
opts = append(opts, gmaps.WithExitMonitor(exitMonitor))
|
||||
}
|
||||
|
||||
job = gmaps.NewGmapJob(id, langCode, query, maxDepth, email, geoCoordinates, zoom, opts...)
|
||||
} else {
|
||||
jparams := gmaps.MapSearchParams{
|
||||
Location: gmaps.MapLocation{
|
||||
Lat: lat,
|
||||
Lon: lon,
|
||||
ZoomLvl: float64(zoom),
|
||||
Radius: radius,
|
||||
},
|
||||
Query: query,
|
||||
ViewportW: 1920,
|
||||
ViewportH: 450,
|
||||
Hl: langCode,
|
||||
}
|
||||
|
||||
opts := []gmaps.SearchJobOptions{}
|
||||
|
||||
if exitMonitor != nil {
|
||||
opts = append(opts, gmaps.WithSearchJobExitMonitor(exitMonitor))
|
||||
}
|
||||
|
||||
job = gmaps.NewSearchJob(&jparams, opts...)
|
||||
}
|
||||
|
||||
if exitMonitor != nil {
|
||||
opts = append(opts, gmaps.WithExitMonitor(exitMonitor))
|
||||
}
|
||||
|
||||
job := gmaps.NewGmapJob(id, langCode, query, maxDepth, email, geoCoordinates, zoom, opts...)
|
||||
|
||||
jobs = append(jobs, job)
|
||||
}
|
||||
|
||||
|
||||
@@ -79,12 +79,14 @@ func (l *lambdaAwsRunner) handler(ctx context.Context, input lInput) error {
|
||||
exitMonitor := exiter.New()
|
||||
|
||||
seedJobs, err = runner.CreateSeedJobs(
|
||||
false, // TODO supoort fast mode
|
||||
input.Language,
|
||||
in,
|
||||
input.Depth,
|
||||
false,
|
||||
"",
|
||||
0,
|
||||
10000, // TODO support radius
|
||||
nil,
|
||||
exitMonitor,
|
||||
)
|
||||
|
||||
+5
-1
@@ -74,6 +74,8 @@ type Config struct {
|
||||
AwsLambdaInvoker bool
|
||||
FunctionName string
|
||||
AwsLambdaChunkSize int
|
||||
FastMode bool
|
||||
Radius float64
|
||||
}
|
||||
|
||||
func ParseConfig() *Config {
|
||||
@@ -103,7 +105,7 @@ func ParseConfig() *Config {
|
||||
flag.BoolVar(&cfg.Email, "email", false, "extract emails from websites")
|
||||
flag.StringVar(&cfg.CustomWriter, "writer", "", "use custom writer plugin (format: 'dir:pluginName')")
|
||||
flag.StringVar(&cfg.GeoCoordinates, "geo", "", "set geo coordinates for search (e.g., '37.7749,-122.4194')")
|
||||
flag.IntVar(&cfg.Zoom, "zoom", 0, "set zoom level (0-21) for search")
|
||||
flag.IntVar(&cfg.Zoom, "zoom", 15, "set zoom level (0-21) for search")
|
||||
flag.BoolVar(&cfg.WebRunner, "web", false, "run web server instead of crawling")
|
||||
flag.StringVar(&cfg.DataFolder, "data-folder", "webdata", "data folder for web runner")
|
||||
flag.StringVar(&proxies, "proxies", "", "comma separated list of proxies to use in the format protocol://user:pass@host:port example: socks5://localhost:9050 or http://user:pass@localhost:9050")
|
||||
@@ -115,6 +117,8 @@ func ParseConfig() *Config {
|
||||
flag.StringVar(&cfg.AwsRegion, "aws-region", "", "AWS region")
|
||||
flag.StringVar(&cfg.S3Bucket, "s3-bucket", "", "S3 bucket name")
|
||||
flag.IntVar(&cfg.AwsLambdaChunkSize, "aws-lambda-chunk-size", 100, "AWS Lambda chunk size")
|
||||
flag.BoolVar(&cfg.FastMode, "fast-mode", false, "fast mode (reduced data collection)")
|
||||
flag.Float64Var(&cfg.Radius, "radius", 10000, "search radius in meters. Default is 10000 meters")
|
||||
|
||||
flag.Parse()
|
||||
|
||||
|
||||
@@ -178,12 +178,20 @@ func (w *webrunner) scrapeJob(ctx context.Context, job *web.Job) error {
|
||||
exitMonitor := exiter.New()
|
||||
|
||||
seedJobs, err := runner.CreateSeedJobs(
|
||||
job.Data.FastMode,
|
||||
job.Data.Lang,
|
||||
strings.NewReader(strings.Join(job.Data.Keywords, "\n")),
|
||||
job.Data.Depth,
|
||||
job.Data.Email,
|
||||
coords,
|
||||
job.Data.Zoom,
|
||||
func() float64 {
|
||||
if job.Data.Radius <= 0 {
|
||||
return 10000 // 10 km
|
||||
}
|
||||
|
||||
return float64(job.Data.Radius)
|
||||
}(),
|
||||
dedup,
|
||||
exitMonitor,
|
||||
)
|
||||
@@ -243,10 +251,19 @@ func (w *webrunner) scrapeJob(ctx context.Context, job *web.Job) error {
|
||||
func (w *webrunner) setupMate(_ context.Context, writer io.Writer, job *web.Job) (*scrapemateapp.ScrapemateApp, error) {
|
||||
opts := []func(*scrapemateapp.Config) error{
|
||||
scrapemateapp.WithConcurrency(w.cfg.Concurrency),
|
||||
scrapemateapp.WithJS(scrapemateapp.DisableImages()),
|
||||
scrapemateapp.WithExitOnInactivity(time.Minute * 3),
|
||||
}
|
||||
|
||||
if !job.Data.FastMode {
|
||||
opts = append(opts,
|
||||
scrapemateapp.WithJS(scrapemateapp.DisableImages()),
|
||||
)
|
||||
} else {
|
||||
opts = append(opts,
|
||||
scrapemateapp.WithStealth(),
|
||||
)
|
||||
}
|
||||
|
||||
hasProxy := false
|
||||
|
||||
if len(w.cfg.Proxies) > 0 {
|
||||
|
||||
Vendored
+1
File diff suppressed because one or more lines are too long
@@ -66,6 +66,8 @@ type JobData struct {
|
||||
Zoom int `json:"zoom"`
|
||||
Lat string `json:"lat"`
|
||||
Lon string `json:"lon"`
|
||||
FastMode bool `json:"fast_mode"`
|
||||
Radius int `json:"radius"`
|
||||
Depth int `json:"depth"`
|
||||
Email bool `json:"email"`
|
||||
MaxTime time.Duration `json:"max_time"`
|
||||
@@ -93,5 +95,9 @@ func (d *JobData) Validate() error {
|
||||
return errors.New("missing max time")
|
||||
}
|
||||
|
||||
if d.FastMode && (d.Lat == "" || d.Lon == "") {
|
||||
return errors.New("missing geo coordinates")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -71,11 +71,11 @@
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="latitude">Latitude:</label>
|
||||
<input type="number" step="0.000001" id="latitude" name="latitude" value="{{.Lat}}">
|
||||
<input type="number" step="0.000000000000001" id="latitude" name="latitude" value="{{.Lat}}">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="longitude">Longitude:</label>
|
||||
<input type="number" step="0.000001" id="longitude" name="longitude" value="{{.Lon}}">
|
||||
<input type="number" step="0.000000000000001" id="longitude" name="longitude" value="{{.Lon}}">
|
||||
</div>
|
||||
</fieldset>
|
||||
</details>
|
||||
@@ -83,6 +83,14 @@
|
||||
<details class="expandable-section">
|
||||
<summary>Advanced Options</summary>
|
||||
<fieldset>
|
||||
<div class="form-group">
|
||||
<label for="fastmode">Fast Mode (BETA):</label>
|
||||
<input type="checkbox" id="fastmode" name="fastmode" {{if .FastMode}}checked{{end}}>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="radius">Radius (BETA):</label>
|
||||
<input type="number" id="radius" name="radius" value="{{.Radius}}">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label for="depth">Depth:</label>
|
||||
<input type="number" step="1" id="depth" name="depth" value="{{.Depth}}">
|
||||
|
||||
+16
-1
@@ -102,6 +102,8 @@ type formData struct {
|
||||
Keywords []string
|
||||
Language string
|
||||
Zoom int
|
||||
FastMode bool
|
||||
Radius int
|
||||
Lat string
|
||||
Lon string
|
||||
Depth int
|
||||
@@ -138,7 +140,9 @@ func (s *Server) index(w http.ResponseWriter, r *http.Request) {
|
||||
MaxTime: "10m",
|
||||
Keywords: []string{},
|
||||
Language: "en",
|
||||
Zoom: 0,
|
||||
Zoom: 15,
|
||||
FastMode: false,
|
||||
Radius: 10000,
|
||||
Lat: "0",
|
||||
Lon: "0",
|
||||
Depth: 10,
|
||||
@@ -213,6 +217,17 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if r.Form.Get("fastmode") == "on" {
|
||||
newJob.Data.FastMode = true
|
||||
}
|
||||
|
||||
newJob.Data.Radius, err = strconv.Atoi(r.Form.Get("radius"))
|
||||
if err != nil {
|
||||
http.Error(w, "invalid radius", http.StatusUnprocessableEntity)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
newJob.Data.Lat = r.Form.Get("latitude")
|
||||
newJob.Data.Lon = r.Form.Get("longitude")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user