mirror of
https://github.com/gosom/google-maps-scraper.git
synced 2026-09-19 07:27:12 +08:00
adds build tag
This commit is contained in:
Giorgos Komninos
@@ -1,5 +1,5 @@
|
||||
APP_NAME := google_maps_scraper
|
||||
VERSION := 1.2.3
|
||||
VERSION := 1.2.4
|
||||
|
||||
default: help
|
||||
|
||||
|
||||
@@ -198,6 +198,44 @@ try `./google-maps-scraper -h` to see the command line options available:
|
||||
produce seed jobs only (only valid with dsn)
|
||||
-results string
|
||||
is the path to the file where the results will be written (default "stdout")
|
||||
-writer string
|
||||
Use a custom writer utilizing the go plugin system.
|
||||
The plugin must implement the scrapemate.ResultWriter interface.
|
||||
The plugin must be a shared library (a file with .so or .dll extension).
|
||||
The plugin must be compiled with the following build tags: go build -buildmode=plugin plugins/example.go.
|
||||
Example: If you have your shared library in a folder /home/user/myplugins and it exposesa DummyPrinter symbol the -writer /home/user/myplugins:DummyPrinter
|
||||
```
|
||||
|
||||
## Using a custom writer
|
||||
|
||||
In cases the results need to be written in a custom format or in another system like a db a message queue or basically anything the Go plugin system can be utilized.
|
||||
|
||||
Write a Go plugin (see an example in examples/plugins/example_writeR.go)
|
||||
|
||||
compile it using (for linux:
|
||||
|
||||
```
|
||||
go build -buildmode=plugin -tags=plugin -o ~/mytest/plugins/example_writer.so examples/plugins/example_writer.go
|
||||
```
|
||||
|
||||
and then run the program using the `-writer` argument.
|
||||
|
||||
See an example:
|
||||
|
||||
1. Write your plugin (use the examples/plugins/example_writer.go as a reference)
|
||||
2. Build your plugin `go build -buildmode=plugin -tags=plugin -o ~/myplugins/example_writer.so plugins/example_writer.go`
|
||||
3. Download the lastes [release](https://github.com/gosom/google-maps-scraper/releases/) or build the program
|
||||
4. Run the program like `./google-maps-scraper -writer ~/myplugins:DummyPrinter -input example-queries.txt`
|
||||
|
||||
|
||||
### Plugins and Docker
|
||||
|
||||
It is possible to use the docker image and use tha plugins.
|
||||
In such case make sure that the shared library is build using a compatible GLIB version with the docker image.
|
||||
otherwise you will encounter an error like:
|
||||
|
||||
```
|
||||
/lib/x86_64-linux-gnu/libc.so.6: version `GLIBC_2.32' not found (required by /plugins/example_writer.so)
|
||||
```
|
||||
|
||||
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:15.2-alpine
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
//go:build plugin
|
||||
// +build plugin
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/gosom/google-maps-scraper/gmaps"
|
||||
"github.com/gosom/scrapemate"
|
||||
)
|
||||
|
||||
var _ scrapemate.ResultWriter = (*exampleWriter)(nil)
|
||||
|
||||
var DummyPrinter scrapemate.ResultWriter = newWriter("dummy.txt")
|
||||
|
||||
type exampleWriter struct {
|
||||
w *bufio.Writer
|
||||
}
|
||||
|
||||
func newWriter(fname string) scrapemate.ResultWriter {
|
||||
fd, err := os.Create(fname)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
return &exampleWriter{
|
||||
w: bufio.NewWriter(fd),
|
||||
}
|
||||
}
|
||||
|
||||
// Run is the main function of the writer
|
||||
// we we write the job id and the title of the entries in a file
|
||||
// notice the asSlice function that converts the data to a slice of *gmaps.Entry
|
||||
func (e *exampleWriter) Run(_ context.Context, in <-chan scrapemate.Result) error {
|
||||
defer e.w.Flush()
|
||||
|
||||
for result := range in {
|
||||
job, ok := result.Job.(scrapemate.IJob)
|
||||
if !ok {
|
||||
return fmt.Errorf("cannot cast %T to IJob", result.Job)
|
||||
}
|
||||
|
||||
items, err := asSlice(result.Data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
_, err := fmt.Fprintf(e.w, "Job %s: %s\n", job.GetID(), item.Title)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func asSlice(t any) ([]*gmaps.Entry, error) {
|
||||
var elements []*gmaps.Entry
|
||||
|
||||
isSlice, ok := t.([]any)
|
||||
if ok {
|
||||
elements := make([]*gmaps.Entry, len(isSlice))
|
||||
for i, v := range isSlice {
|
||||
elements[i], ok = v.(*gmaps.Entry)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("cannot cast %T to *gmaps.Entry", v)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
element, ok := t.(*gmaps.Entry)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("cannot cast %T to *gmaps.Entry", t)
|
||||
}
|
||||
|
||||
elements = append(elements, element)
|
||||
}
|
||||
|
||||
return elements, nil
|
||||
}
|
||||
+54
@@ -1,29 +1,74 @@
|
||||
cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc=
|
||||
cloud.google.com/go/ai v0.8.0/go.mod h1:t3Dfk4cM61sytiggo2UyGsDVW3RF1qGZaUKDrZFyqkE=
|
||||
cloud.google.com/go/auth v0.9.3/go.mod h1:7z6VY+7h3KUdRov5F1i8NDP5ZzWKYmEPO842BgCsmTk=
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc=
|
||||
cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI=
|
||||
cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY=
|
||||
cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY=
|
||||
cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8=
|
||||
cloud.google.com/go/longrunning v0.5.7/go.mod h1:8GClkudohy1Fxm3owmBGid8W0pSgodEMwEAztp38Xng=
|
||||
cloud.google.com/go/storage v1.35.1/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8=
|
||||
github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
|
||||
github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
|
||||
github.com/confluentinc/confluent-kafka-go v1.9.2/go.mod h1:ptXNqsuDfYbAE/LBW6pnwWZElUoWxHoV8E43DCrliyo=
|
||||
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
|
||||
github.com/cristalhq/acmd v0.12.0/go.mod h1:LG5oa43pE/BbxtfMoImHCQN++0Su7dzipdgBjMCBVDQ=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang-migrate/migrate/v4 v4.15.2/go.mod h1:f2toGLkYqD3JH+Todi4aZ2ZdbeUNx4sIwiOK96rE9Lw=
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
|
||||
github.com/google/generative-ai-go v0.17.0/go.mod h1:JYolL13VG7j79kM5BtHz4qwONHkeJQzOCkKXnpqtS/E=
|
||||
github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA=
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.3/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA=
|
||||
github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A=
|
||||
github.com/googleapis/google-cloud-go-testing v0.0.0-20210719221736-1c9a4c676720/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g=
|
||||
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
|
||||
github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
|
||||
github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=
|
||||
github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
|
||||
github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ=
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
|
||||
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||
github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
|
||||
github.com/ismurov/swaggerui v0.2.0/go.mod h1:EaaariTC2xXLMsKU9v3MdYT62/akXBvRFxmuY9zyqF0=
|
||||
github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ=
|
||||
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
|
||||
github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg=
|
||||
github.com/mozilla/tls-observatory v0.0.0-20210609171429-7bc42856d2e5/go.mod h1:FUqVoUPHSEdDR0MnFM3Dh8AU0pZHLXUD127SAJGER/s=
|
||||
github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNsTT1QQ=
|
||||
github.com/orisano/pixelmatch v0.0.0-20230914042517-fa304d1dc785/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw=
|
||||
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/quasilyte/go-ruleguard/rules v0.0.0-20211022131956-028d6511ab71/go.mod h1:4cgAphtvu7Ftv7vOT2ZOYhC6CvBxZixcasr8qIOTA50=
|
||||
github.com/realclientip/realclientip-go v1.0.0/go.mod h1:CXnUdVwFRcXFJIRb/dTYqbT7ud48+Pi2pFm80bxDmcI=
|
||||
github.com/rollbar/rollbar-go v1.4.5/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w=
|
||||
github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
|
||||
github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8=
|
||||
github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
|
||||
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
|
||||
github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
|
||||
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/quicktemplate v1.8.0/go.mod h1:qIqW8/igXt8fdrUln5kOSb+KWMaJ4Y8QUsfd1k6L2jM=
|
||||
github.com/wagslane/go-password-validator v0.3.0/go.mod h1:TI1XJ6T5fRdRnHqHt14pvy1tNVnrwe7m3/f1f2fDphQ=
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
|
||||
go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU=
|
||||
@@ -41,10 +86,19 @@ golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSO
|
||||
golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
|
||||
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
|
||||
golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
|
||||
golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
|
||||
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
|
||||
golang.org/x/term v0.24.0/go.mod h1:lOBK/LVxemqiMij05LGJ0tzNr8xlmwBRJ81PX6wVLH8=
|
||||
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
|
||||
golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
|
||||
google.golang.org/api v0.196.0/go.mod h1:g9IL21uGkYgvQ5BZg6BAtoGJQIm8r6EgaAbpNey5wBE=
|
||||
google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
|
||||
google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY=
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20240725223205-93522f1f2a9f/go.mod h1:AHT0dDg3SoMOgZGnZk29b5xTbPHMoEC8qthmBLJCpys=
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
|
||||
google.golang.org/grpc v1.66.0/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y=
|
||||
|
||||
@@ -6,8 +6,11 @@ import (
|
||||
"database/sql"
|
||||
"encoding/csv"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"plugin"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -74,30 +77,46 @@ func runFromLocalFile(ctx context.Context, args *arguments) error {
|
||||
input = f
|
||||
}
|
||||
|
||||
var resultsWriter io.Writer
|
||||
writers := []scrapemate.ResultWriter{}
|
||||
|
||||
switch args.resultsFile {
|
||||
case "stdout":
|
||||
resultsWriter = os.Stdout
|
||||
default:
|
||||
f, err := os.Create(args.resultsFile)
|
||||
if args.customWriter != "" {
|
||||
parts := strings.Split(args.customWriter, ":")
|
||||
if len(parts) != 2 {
|
||||
return fmt.Errorf("invalid custom writer format: %s", args.customWriter)
|
||||
}
|
||||
|
||||
dir, pluginName := parts[0], parts[1]
|
||||
|
||||
customWriter, err := loadCustomWriter(dir, pluginName)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
resultsWriter = f
|
||||
}
|
||||
|
||||
csvWriter := csvwriter.NewCsvWriter(csv.NewWriter(resultsWriter))
|
||||
|
||||
writers := []scrapemate.ResultWriter{}
|
||||
|
||||
if args.json {
|
||||
writers = append(writers, jsonwriter.NewJSONWriter(resultsWriter))
|
||||
writers = append(writers, customWriter)
|
||||
} else {
|
||||
writers = append(writers, csvWriter)
|
||||
var resultsWriter io.Writer
|
||||
|
||||
switch args.resultsFile {
|
||||
case "stdout":
|
||||
resultsWriter = os.Stdout
|
||||
default:
|
||||
f, err := os.Create(args.resultsFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer f.Close()
|
||||
|
||||
resultsWriter = f
|
||||
}
|
||||
|
||||
csvWriter := csvwriter.NewCsvWriter(csv.NewWriter(resultsWriter))
|
||||
|
||||
if args.json {
|
||||
writers = append(writers, jsonwriter.NewJSONWriter(resultsWriter))
|
||||
} else {
|
||||
writers = append(writers, csvWriter)
|
||||
}
|
||||
}
|
||||
|
||||
opts := []func(*scrapemateapp.Config) error{
|
||||
@@ -254,6 +273,7 @@ type arguments struct {
|
||||
produceOnly bool
|
||||
exitOnInactivityDuration time.Duration
|
||||
email bool
|
||||
customWriter string
|
||||
}
|
||||
|
||||
func parseArgs() (args arguments) {
|
||||
@@ -279,6 +299,12 @@ func parseArgs() (args arguments) {
|
||||
flag.DurationVar(&args.exitOnInactivityDuration, "exit-on-inactivity", 0, "program exits after this duration of inactivity(example value '5m')")
|
||||
flag.BoolVar(&args.json, "json", false, "Use this to produce a json file instead of csv (not available when using db)")
|
||||
flag.BoolVar(&args.email, "email", false, "Use this to extract emails from the websites")
|
||||
flag.StringVar(&args.customWriter, "writer", "",
|
||||
`Use a custom writer utilizing the go plugin system.
|
||||
The plugin must implement the scrapemate.ResultWriter interface.
|
||||
The plugin must be a shared library (a file with .so extension).
|
||||
The plugin must be compiled with the following build tags: go build -buildmode=plugin plugins/example.go.
|
||||
The plugins must be placed in the same directory as the binary in a directory called plugins.`)
|
||||
|
||||
flag.Parse()
|
||||
|
||||
@@ -295,3 +321,41 @@ func openPsqlConn(dsn string) (conn *sql.DB, err error) {
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func loadCustomWriter(pluginDir, pluginName string) (scrapemate.ResultWriter, error) {
|
||||
files, err := os.ReadDir(pluginDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read plugin directory: %w", err)
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
if file.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
if filepath.Ext(file.Name()) != ".so" && filepath.Ext(file.Name()) != ".dll" {
|
||||
continue
|
||||
}
|
||||
|
||||
pluginPath := filepath.Join(pluginDir, file.Name())
|
||||
|
||||
p, err := plugin.Open(pluginPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open plugin %s: %w", file.Name(), err)
|
||||
}
|
||||
|
||||
symWriter, err := p.Lookup(pluginName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to lookup symbol %s: %w", pluginName, err)
|
||||
}
|
||||
|
||||
writer, ok := symWriter.(*scrapemate.ResultWriter)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unexpected type %T from writer symbol in plugin %s", symWriter, file.Name())
|
||||
}
|
||||
|
||||
return *writer, nil
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no plugin found in %s", pluginDir)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user