mirror of
https://github.com/gosom/google-maps-scraper.git
synced 2026-09-19 07:27:12 +08:00
Add telementry support to collect anonymous statistics
This commit is contained in:
Giorgos Komninos
@@ -311,6 +311,10 @@ Please replace the values or the command args accordingly
|
||||
Note: Keep in mind that because the application starts a headless browser it requires CPU and memory.
|
||||
Use an appropriate kubernetes cluster
|
||||
|
||||
## Telemetry
|
||||
|
||||
Anonymous usage statistics are collected for debug and improvement reasons.
|
||||
You can opt out by setting the env variable `DISABLE_TELEMETRY=1`
|
||||
|
||||
## Perfomance
|
||||
|
||||
|
||||
@@ -10,7 +10,10 @@ require (
|
||||
github.com/jackc/pgx/v5 v5.7.1
|
||||
github.com/mcnijman/go-emailaddress v1.1.1
|
||||
github.com/playwright-community/playwright-go v0.4701.0
|
||||
github.com/posthog/posthog-go v1.2.24
|
||||
github.com/shirou/gopsutil/v4 v4.24.9
|
||||
github.com/stretchr/testify v1.9.0
|
||||
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -53,6 +56,7 @@ require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
|
||||
github.com/denis-tingaikin/go-header v0.5.0 // indirect
|
||||
github.com/ebitengine/purego v0.8.0 // indirect
|
||||
github.com/ettle/strcase v0.2.0 // indirect
|
||||
github.com/fatih/color v1.17.0 // indirect
|
||||
github.com/fatih/structtag v1.2.0 // indirect
|
||||
@@ -63,6 +67,7 @@ require (
|
||||
github.com/ghostiam/protogetter v0.3.6 // indirect
|
||||
github.com/go-critic/go-critic v0.11.4 // indirect
|
||||
github.com/go-jose/go-jose/v3 v3.0.3 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.22.1 // indirect
|
||||
@@ -118,6 +123,7 @@ require (
|
||||
github.com/leodido/go-urn v1.4.0 // indirect
|
||||
github.com/leonklingele/grouper v1.1.2 // indirect
|
||||
github.com/lufeee/execinquery v1.2.1 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/macabu/inamedparam v0.1.3 // indirect
|
||||
github.com/magiconair/properties v1.8.6 // indirect
|
||||
github.com/maratori/testableexamples v1.0.0 // indirect
|
||||
@@ -140,6 +146,7 @@ require (
|
||||
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/polyfloyd/go-errorlint v1.6.0 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/prometheus/client_golang v1.12.1 // indirect
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
github.com/prometheus/common v0.32.1 // indirect
|
||||
@@ -178,6 +185,8 @@ require (
|
||||
github.com/tetafro/godot v1.4.17 // indirect
|
||||
github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 // indirect
|
||||
github.com/timonwong/loggercheck v0.9.4 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.12 // indirect
|
||||
github.com/tklauser/numcpus v0.6.1 // indirect
|
||||
github.com/tomarrell/wrapcheck/v2 v2.9.0 // indirect
|
||||
github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect
|
||||
github.com/ultraware/funlen v0.1.0 // indirect
|
||||
@@ -187,6 +196,7 @@ require (
|
||||
github.com/yagipy/maintidx v1.0.0 // indirect
|
||||
github.com/yeya24/promlinter v0.3.0 // indirect
|
||||
github.com/ykadowak/zerologlint v0.1.5 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.4 // indirect
|
||||
gitlab.com/bosi/decorder v0.4.2 // indirect
|
||||
go-simpler.org/musttag v0.12.2 // indirect
|
||||
go-simpler.org/sloglint v0.7.2 // indirect
|
||||
@@ -195,7 +205,6 @@ require (
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.24.0 // indirect
|
||||
golang.org/x/crypto v0.27.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
|
||||
golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect
|
||||
golang.org/x/mod v0.21.0 // indirect
|
||||
golang.org/x/net v0.29.0 // indirect
|
||||
|
||||
@@ -136,6 +136,8 @@ github.com/deckarep/golang-set/v2 v2.6.0 h1:XfcQbWM1LlMB8BsJ8N9vW5ehnnPVIw0je80N
|
||||
github.com/deckarep/golang-set/v2 v2.6.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4=
|
||||
github.com/denis-tingaikin/go-header v0.5.0 h1:SRdnP5ZKvcO9KKRP1KJrhFR3RrlGuD+42t4429eC9k8=
|
||||
github.com/denis-tingaikin/go-header v0.5.0/go.mod h1:mMenU5bWrok6Wl2UsZjy+1okegmwQ3UgWl4V1D8gjlY=
|
||||
github.com/ebitengine/purego v0.8.0 h1:JbqvnEzRvPpxhCJzJJ2y0RbiZ8nyjccVUrSM3q+GvvE=
|
||||
github.com/ebitengine/purego v0.8.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
@@ -174,6 +176,8 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V
|
||||
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
|
||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
@@ -396,6 +400,8 @@ github.com/leonklingele/grouper v1.1.2 h1:o1ARBDLOmmasUaNDesWqWCIFH3u7hoFlM84Yrj
|
||||
github.com/leonklingele/grouper v1.1.2/go.mod h1:6D0M/HVkhs2yRKRFZUoGjeDy7EZTfFBE9gl4kjmIGkA=
|
||||
github.com/lufeee/execinquery v1.2.1 h1:hf0Ems4SHcUGBxpGN7Jz78z1ppVkP/837ZlETPCEtOM=
|
||||
github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/macabu/inamedparam v0.1.3 h1:2tk/phHkMlEL/1GNe/Yf6kkR/hkcUdAEY3L0hjYV1Mk=
|
||||
github.com/macabu/inamedparam v0.1.3/go.mod h1:93FLICAIk/quk7eaPPQvbzihUdn/QkGDwIZEoLtpH6I=
|
||||
github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
|
||||
@@ -476,6 +482,10 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/polyfloyd/go-errorlint v1.6.0 h1:tftWV9DE7txiFzPpztTAwyoRLKNj9gpVm2cg8/OwcYY=
|
||||
github.com/polyfloyd/go-errorlint v1.6.0/go.mod h1:HR7u8wuP1kb1NeN1zqTd1ZMlqUKPPHF+Id4vIPvDqVw=
|
||||
github.com/posthog/posthog-go v1.2.24 h1:A+iG4saBJemo++VDlcWovbYf8KFFNUfrCoJtsc40RPA=
|
||||
github.com/posthog/posthog-go v1.2.24/go.mod h1:uYC2l1Yktc8E+9FAHJ9QZG4vQf/NHJPD800Hsm7DzoM=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
|
||||
github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
@@ -533,6 +543,8 @@ github.com/securego/gosec/v2 v2.21.2 h1:deZp5zmYf3TWwU7A7cR2+SolbTpZ3HQiwFqnzQyE
|
||||
github.com/securego/gosec/v2 v2.21.2/go.mod h1:au33kg78rNseF5PwPnTWhuYBFf534bvJRvOrgZ/bFzU=
|
||||
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c h1:W65qqJCIOVP4jpqPQ0YvHYKwcMEMVWIzWC5iNQQfBTU=
|
||||
github.com/shazow/go-diff v0.0.0-20160112020656-b6b7b6733b8c/go.mod h1:/PevMnwAxekIXwN8qQyfc5gl2NlkB3CQlkizAbOkeBs=
|
||||
github.com/shirou/gopsutil/v4 v4.24.9 h1:KIV+/HaHD5ka5f570RZq+2SaeFsb/pq+fp2DGNWYoOI=
|
||||
github.com/shirou/gopsutil/v4 v4.24.9/go.mod h1:3fkaHNeYsUFCGZ8+9vZVWtbyM1k2eRnlL+bWO8Bxa/Q=
|
||||
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
|
||||
github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
|
||||
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
|
||||
@@ -596,6 +608,10 @@ github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 h1:quvGphlmUVU+n
|
||||
github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966/go.mod h1:27bSVNWSBOHm+qRp1T9qzaIpsWEP6TbUnei/43HK+PQ=
|
||||
github.com/timonwong/loggercheck v0.9.4 h1:HKKhqrjcVj8sxL7K77beXh0adEm6DLjV/QOGeMXEVi4=
|
||||
github.com/timonwong/loggercheck v0.9.4/go.mod h1:caz4zlPcgvpEkXgVnAJGowHAMW2NwHaNlpS8xDbVhTg=
|
||||
github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
|
||||
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
|
||||
github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
|
||||
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
|
||||
github.com/tomarrell/wrapcheck/v2 v2.9.0 h1:801U2YCAjLhdN8zhZ/7tdjB3EnAoRlJHt/s+9hijLQ4=
|
||||
github.com/tomarrell/wrapcheck/v2 v2.9.0/go.mod h1:g9vNIyhb5/9TQgumxQyOEqDHsmGYcGsVMOx/xGkqdMo=
|
||||
github.com/tommy-muehle/go-mnd/v2 v2.5.1 h1:NowYhSdyE/1zwK9QCLeRb6USWdoif80Ie+v+yU8u1Zw=
|
||||
@@ -621,6 +637,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
gitlab.com/bosi/decorder v0.4.2 h1:qbQaV3zgwnBZ4zPMhGLW4KZe7A7NwxEhJx39R3shffo=
|
||||
gitlab.com/bosi/decorder v0.4.2/go.mod h1:muuhHoaJkA9QLcYHq4Mj8FJUwDZ+EirSHRiaTcTf6T8=
|
||||
go-simpler.org/assert v0.9.0 h1:PfpmcSvL7yAnWyChSjOz6Sp6m9j5lyK8Ok9pEL31YkQ=
|
||||
@@ -779,6 +797,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -800,6 +819,7 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@@ -822,6 +842,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
|
||||
|
||||
+3
-8
@@ -16,7 +16,6 @@ github.com/cristalhq/acmd v0.12.0/go.mod h1:LG5oa43pE/BbxtfMoImHCQN++0Su7dzipdgB
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
|
||||
github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang-migrate/migrate/v4 v4.15.2/go.mod h1:f2toGLkYqD3JH+Todi4aZ2ZdbeUNx4sIwiOK96rE9Lw=
|
||||
@@ -43,7 +42,6 @@ github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Cc
|
||||
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/magefile/mage v1.14.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
|
||||
github.com/mgechev/dots v0.0.0-20210922191527-e955255bf517/go.mod h1:KQ7+USdGKfpPjXk4Ga+5XxQM4Lm4e3gAogrreFAYpOg=
|
||||
github.com/mozilla/tls-observatory v0.0.0-20210609171429-7bc42856d2e5/go.mod h1:FUqVoUPHSEdDR0MnFM3Dh8AU0pZHLXUD127SAJGER/s=
|
||||
@@ -51,24 +49,21 @@ github.com/oklog/ulid/v2 v2.1.0/go.mod h1:rcEKHmBBKfef9DhnvX7y1HZBYxjXb0cP5ExxNs
|
||||
github.com/orisano/pixelmatch v0.0.0-20230914042517-fa304d1dc785/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
|
||||
github.com/phayes/checkstyle v0.0.0-20170904204023-bfd46e6a821d/go.mod h1:3OzsM7FXDQlpCiw2j81fOmAwQLnZnLGXVKUzeKQXIAw=
|
||||
github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk=
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
|
||||
github.com/quasilyte/go-ruleguard/rules v0.0.0-20211022131956-028d6511ab71/go.mod h1:4cgAphtvu7Ftv7vOT2ZOYhC6CvBxZixcasr8qIOTA50=
|
||||
github.com/realclientip/realclientip-go v1.0.0/go.mod h1:CXnUdVwFRcXFJIRb/dTYqbT7ud48+Pi2pFm80bxDmcI=
|
||||
github.com/rollbar/rollbar-go v1.4.5/go.mod h1:kLQ9gP3WCRGrvJmF0ueO3wK9xWocej8GRX98D8sa39w=
|
||||
github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
|
||||
github.com/sagikazarmark/crypt v0.6.0/go.mod h1:U8+INwJo3nBv1m6A/8OBXAq7Jnpspk5AxSgDyEQcea8=
|
||||
github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
|
||||
github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
|
||||
github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
|
||||
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
|
||||
github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
|
||||
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
|
||||
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
|
||||
github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
|
||||
github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
|
||||
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
|
||||
github.com/valyala/quicktemplate v1.8.0/go.mod h1:qIqW8/igXt8fdrUln5kOSb+KWMaJ4Y8QUsfd1k6L2jM=
|
||||
github.com/wagslane/go-password-validator v0.3.0/go.mod h1:TI1XJ6T5fRdRnHqHt14pvy1tNVnrwe7m3/f1f2fDphQ=
|
||||
github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
|
||||
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A=
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g=
|
||||
go.etcd.io/etcd/client/v2 v2.305.4/go.mod h1:Ud+VUwIi9/uQHOMA+4ekToJ12lTxlv0zB/+DHwTGEbU=
|
||||
|
||||
@@ -19,6 +19,8 @@ func main() {
|
||||
if err != nil {
|
||||
os.Stderr.WriteString(err.Error() + "\n")
|
||||
|
||||
runner.Telemetry().Close()
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -26,11 +28,13 @@ func main() {
|
||||
os.Stderr.WriteString(err.Error() + "\n")
|
||||
|
||||
_ = runnerInstance.Close(ctx)
|
||||
runner.Telemetry().Close()
|
||||
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
_ = runnerInstance.Close(ctx)
|
||||
runner.Telemetry().Close()
|
||||
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"github.com/gosom/google-maps-scraper/postgres"
|
||||
"github.com/gosom/google-maps-scraper/runner"
|
||||
"github.com/gosom/google-maps-scraper/tlmt"
|
||||
"github.com/gosom/scrapemate"
|
||||
"github.com/gosom/scrapemate/scrapemateapp"
|
||||
)
|
||||
@@ -84,6 +85,8 @@ func New(cfg *runner.Config) (runner.Runner, error) {
|
||||
}
|
||||
|
||||
func (d *dbrunner) Run(ctx context.Context) error {
|
||||
_ = runner.Telemetry().Send(ctx, tlmt.NewEvent("databaserunner.Run", nil))
|
||||
|
||||
if d.produce {
|
||||
return d.produceSeedJobs(ctx)
|
||||
}
|
||||
@@ -138,6 +141,10 @@ func (d *dbrunner) produceSeedJobs(ctx context.Context) error {
|
||||
}
|
||||
}
|
||||
|
||||
_ = runner.Telemetry().Send(ctx, tlmt.NewEvent("databaserunner.produceSeedJobs", map[string]any{
|
||||
"jobs": len(jobs),
|
||||
}))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -7,8 +7,10 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gosom/google-maps-scraper/runner"
|
||||
"github.com/gosom/google-maps-scraper/tlmt"
|
||||
"github.com/gosom/scrapemate"
|
||||
"github.com/gosom/scrapemate/adapters/writers/csvwriter"
|
||||
"github.com/gosom/scrapemate/adapters/writers/jsonwriter"
|
||||
@@ -47,8 +49,28 @@ func New(cfg *runner.Config) (runner.Runner, error) {
|
||||
return ans, nil
|
||||
}
|
||||
|
||||
func (r *fileRunner) Run(ctx context.Context) error {
|
||||
seedJobs, err := runner.CreateSeedJobs(
|
||||
func (r *fileRunner) Run(ctx context.Context) (err error) {
|
||||
var seedJobs []scrapemate.IJob
|
||||
|
||||
t0 := time.Now().UTC()
|
||||
|
||||
defer func() {
|
||||
elapsed := time.Now().UTC().Sub(t0)
|
||||
params := map[string]any{
|
||||
"job_count": len(seedJobs),
|
||||
"duration": elapsed.String(),
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
params["error"] = err.Error()
|
||||
}
|
||||
|
||||
evt := tlmt.NewEvent("file_runner", params)
|
||||
|
||||
_ = runner.Telemetry().Send(ctx, evt)
|
||||
}()
|
||||
|
||||
seedJobs, err = runner.CreateSeedJobs(
|
||||
r.cfg.LangCode,
|
||||
r.input,
|
||||
r.cfg.MaxDepth,
|
||||
@@ -60,7 +82,9 @@ func (r *fileRunner) Run(ctx context.Context) error {
|
||||
return err
|
||||
}
|
||||
|
||||
return r.app.Start(ctx, seedJobs...)
|
||||
err = r.app.Start(ctx, seedJobs...)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *fileRunner) Close(context.Context) error {
|
||||
|
||||
@@ -6,7 +6,12 @@ import (
|
||||
"flag"
|
||||
"os"
|
||||
"runtime"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gosom/google-maps-scraper/tlmt"
|
||||
"github.com/gosom/google-maps-scraper/tlmt/gonoop"
|
||||
"github.com/gosom/google-maps-scraper/tlmt/goposthog"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -42,6 +47,7 @@ type Config struct {
|
||||
GeoCoordinates string
|
||||
Zoom int
|
||||
RunMode int
|
||||
DisableTelemetry bool
|
||||
}
|
||||
|
||||
func ParseConfig() *Config {
|
||||
@@ -100,3 +106,33 @@ func ParseConfig() *Config {
|
||||
|
||||
return &cfg
|
||||
}
|
||||
|
||||
var (
|
||||
telemetryOnce sync.Once
|
||||
telemetry tlmt.Telemetry
|
||||
)
|
||||
|
||||
func Telemetry() tlmt.Telemetry {
|
||||
telemetryOnce.Do(func() {
|
||||
disableTel := func() bool {
|
||||
return os.Getenv("DISABLE_TELEMETRY") == "1"
|
||||
}()
|
||||
|
||||
if disableTel {
|
||||
telemetry = gonoop.New()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
val, err := goposthog.New("phc_CHYBGEd1eJZzDE7ZWhyiSFuXa9KMLRnaYN47aoIAY2A", "https://eu.i.posthog.com")
|
||||
if err != nil || val == nil {
|
||||
telemetry = gonoop.New()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
telemetry = val
|
||||
})
|
||||
|
||||
return telemetry
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package gonoop
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gosom/google-maps-scraper/tlmt"
|
||||
)
|
||||
|
||||
type service struct {
|
||||
}
|
||||
|
||||
func New() tlmt.Telemetry {
|
||||
return &service{}
|
||||
}
|
||||
|
||||
func (s *service) Send(context.Context, tlmt.Event) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *service) Close() error {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
package goposthog
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/gosom/google-maps-scraper/tlmt"
|
||||
"github.com/posthog/posthog-go"
|
||||
)
|
||||
|
||||
type service struct {
|
||||
client posthog.Client
|
||||
}
|
||||
|
||||
func New(publicAPIKEY, endpointURL string) (tlmt.Telemetry, error) {
|
||||
client, err := posthog.NewWithConfig(publicAPIKEY, posthog.Config{Endpoint: endpointURL})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ans := service{
|
||||
client: client,
|
||||
}
|
||||
|
||||
return &ans, nil
|
||||
}
|
||||
|
||||
func (s *service) Send(_ context.Context, event tlmt.Event) error {
|
||||
capture := posthog.Capture{
|
||||
DistinctId: event.AnonymousID,
|
||||
Event: event.Name,
|
||||
Properties: event.Properties,
|
||||
}
|
||||
|
||||
if err := capture.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return s.client.Enqueue(capture)
|
||||
}
|
||||
|
||||
func (s *service) Close() error {
|
||||
if s.client != nil {
|
||||
return s.client.Close()
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
package tlmt
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/shirou/gopsutil/v4/host"
|
||||
"golang.org/x/exp/rand"
|
||||
)
|
||||
|
||||
var (
|
||||
once sync.Once
|
||||
identifier machineIdentifier
|
||||
)
|
||||
|
||||
type Event struct {
|
||||
AnonymousID string
|
||||
Name string
|
||||
Properties map[string]any
|
||||
}
|
||||
|
||||
func NewEvent(name string, props map[string]any) Event {
|
||||
ev := Event{
|
||||
AnonymousID: generateMachineID().id,
|
||||
Name: name,
|
||||
Properties: generateMachineID().meta,
|
||||
}
|
||||
|
||||
for k, v := range props {
|
||||
ev.Properties[k] = v
|
||||
}
|
||||
|
||||
return ev
|
||||
}
|
||||
|
||||
type Telemetry interface {
|
||||
Send(ctx context.Context, event Event) error
|
||||
Close() error
|
||||
}
|
||||
|
||||
type machineIdentifier struct {
|
||||
id string
|
||||
meta map[string]any
|
||||
}
|
||||
|
||||
func generateMachineID() machineIdentifier {
|
||||
once.Do(func() {
|
||||
ip := fetchExternalIP()
|
||||
if ip == "" {
|
||||
ip = uuid.New().String()
|
||||
}
|
||||
|
||||
hash := sha256.New()
|
||||
hash.Write([]byte(ip))
|
||||
hash.Write([]byte(runtime.GOARCH))
|
||||
hash.Write([]byte(runtime.GOOS))
|
||||
hash.Write([]byte(runtime.Version()))
|
||||
|
||||
id := fmt.Sprintf("%x", hash.Sum(nil))
|
||||
|
||||
meta := make(map[string]any)
|
||||
|
||||
info, err := host.Info()
|
||||
if err == nil {
|
||||
meta["os"] = info.OS
|
||||
meta["platform"] = info.Platform
|
||||
meta["platform_family"] = info.PlatformFamily
|
||||
meta["platform_version"] = info.PlatformVersion
|
||||
}
|
||||
|
||||
identifier.id = id
|
||||
identifier.meta = meta
|
||||
})
|
||||
|
||||
return identifier
|
||||
}
|
||||
|
||||
func fetchExternalIP() string {
|
||||
endpoints := []string{
|
||||
"https://api.ipify.org",
|
||||
"https://ifconfig.me",
|
||||
"https://icanhazip.com",
|
||||
"https://ident.me",
|
||||
"https://ifconfig.co",
|
||||
}
|
||||
|
||||
r := rand.New(rand.NewSource(uint64(time.Now().UnixNano()))) //nolint:gosec // no need for secure random here
|
||||
|
||||
r.Shuffle(len(endpoints), func(i, j int) {
|
||||
endpoints[i], endpoints[j] = endpoints[j], endpoints[i]
|
||||
})
|
||||
|
||||
client := http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
}
|
||||
|
||||
for _, endpoint := range endpoints {
|
||||
ip := func(u string) string {
|
||||
req, err := http.NewRequest(http.MethodGet, u, http.NoBody)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_, _ = io.Copy(io.Discard, resp.Body)
|
||||
resp.Body.Close()
|
||||
}()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return ""
|
||||
}
|
||||
|
||||
ip, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return strings.TrimSpace(string(ip))
|
||||
}(endpoint)
|
||||
|
||||
if ip != "" {
|
||||
return ip
|
||||
}
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
Reference in New Issue
Block a user