Add fingerprints debug tests and endpoint

This commit is contained in:
Rustem Kamalov
2026-04-23 00:53:30 +03:00
parent cfd8c418dc
commit e02b374699
17 changed files with 1706 additions and 342 deletions

View File

@@ -51,13 +51,14 @@ type ServerConfig struct {
}
type AppConfig struct {
Timeout int `mapstructure:"timeout"`
BrowserPath string `mapstructure:"browser_path"`
IsBrowserHead bool `mapstructure:"head"`
IsLeaveHead bool `mapstructure:"leave_head"`
IsLeakless bool `mapstructure:"leakless"`
IsStealth bool `mapstructure:"stealth"`
LogFormat string `mapstructure:"log_format"`
Timeout int `mapstructure:"timeout"`
BrowserPath string `mapstructure:"browser_path"`
IsBrowserHead bool `mapstructure:"head"`
IsLeaveHead bool `mapstructure:"leave_head"`
IsLeakless bool `mapstructure:"leakless"`
IsStealth bool `mapstructure:"stealth"`
DebugEndpoints bool `mapstructure:"debug_endpoints"`
LogFormat string `mapstructure:"log_format"`
}
type EngineConfig struct {
@@ -110,6 +111,7 @@ var flagToConfigKey = map[string]string{
"2captcha_key": "2captcha.apikey",
"proxy": "proxies.global",
"stealth": "app.stealth",
"debug-endpoints": "app.debug_endpoints",
"insecure": "server.insecure",
"cache_ttl": "cache.ttl_seconds",
"cache_max_size": "cache.max_size",
@@ -318,6 +320,7 @@ func setConfigDefaults(v *viper.Viper) {
v.SetDefault("app.leave_head", false)
v.SetDefault("app.leakless", false)
v.SetDefault("app.stealth", false)
v.SetDefault("app.debug_endpoints", false)
v.SetDefault("proxies.entries", []interface{}{})
v.SetDefault("proxies.global", "")
@@ -354,6 +357,7 @@ func init() {
RootCmd.PersistentFlags().StringVarP(&config.Config2Capcha.ApiKey, "2captcha_key", "", "", "2 captcha api key")
RootCmd.PersistentFlags().StringVarP(&config.Proxies.Global, "proxy", "x", "", "Force a single proxy for all engines (same as proxies.global)")
RootCmd.PersistentFlags().BoolVarP(&config.App.IsStealth, "stealth", "s", false, "Use stealth browser plugin")
RootCmd.PersistentFlags().BoolVar(&config.App.DebugEndpoints, "debug-endpoints", false, "Enable debug-only HTTP endpoints")
RootCmd.PersistentFlags().BoolVarP(&config.Server.Insecure, "insecure", "k", false, "Allow insecure TLS connections")
RootCmd.PersistentFlags().IntVar(&config.Cache.TTLSeconds, "cache_ttl", 300, "Cache TTL in seconds (0 to disable)")
RootCmd.PersistentFlags().IntVar(&config.Cache.MaxSize, "cache_max_size", 1000, "Maximum number of cached responses")

View File

@@ -91,9 +91,11 @@ func serve(cmd *cobra.Command, args []string) {
return
}
fingerprintBrowserOpts := buildFingerprintBrowserOptions()
if config.Server.IsRawRequests {
logrus.Warn("Browserless results are very inconsistent or may not even work!")
serverOpts := buildServerOptions(corsCfg, proxyCfg)
serverOpts := buildServerOptions(corsCfg, proxyCfg, fingerprintBrowserOpts)
serv := core.NewServerWithOptions(config.Server.Host, config.Server.Port, serverOpts,
&rawEngine{name: "google"},
&rawEngine{name: "yandex"},
@@ -105,20 +107,10 @@ func serve(cmd *cobra.Command, args []string) {
return
}
baseOpts := core.BrowserOpts{
IsHeadless: !config.App.IsBrowserHead,
IsLeakless: config.App.IsLeakless,
Timeout: time.Second * time.Duration(config.App.Timeout),
LeavePageOpen: config.App.IsLeaveHead,
CaptchaSolverEnabled: captchaSolverEnabled,
CaptchaSolverApiKey: captchaSolverAPIKey,
BrowserPath: config.App.BrowserPath,
Insecure: config.Server.Insecure,
UseStealth: config.App.IsStealth,
}
if config.Server.IsDebug {
baseOpts.IsHeadless = false
}
baseOpts := fingerprintBrowserOpts
baseOpts.LeavePageOpen = config.App.IsLeaveHead
baseOpts.CaptchaSolverEnabled = captchaSolverEnabled
baseOpts.CaptchaSolverApiKey = captchaSolverAPIKey
engines, closeBrowsers, err := buildBrowserEngines(baseOpts, proxyCfg)
if err != nil {
@@ -126,20 +118,38 @@ func serve(cmd *cobra.Command, args []string) {
return
}
serverOpts := buildServerOptions(corsCfg, proxyCfg)
serverOpts := buildServerOptions(corsCfg, proxyCfg, fingerprintBrowserOpts)
serv := core.NewServerWithOptions(config.Server.Host, config.Server.Port, serverOpts, engines...)
if err := listenWithGracefulShutdown(serv, closeBrowsers); err != nil {
logrus.Error(err)
}
}
func buildServerOptions(corsCfg core.CORSConfig, proxyCfg core.ProxyConfig) core.ServerOptions {
func buildFingerprintBrowserOptions() core.BrowserOpts {
opts := core.BrowserOpts{
IsHeadless: !config.App.IsBrowserHead,
IsLeakless: config.App.IsLeakless,
Timeout: time.Second * time.Duration(config.App.Timeout),
BrowserPath: config.App.BrowserPath,
Insecure: config.Server.Insecure,
UseStealth: config.App.IsStealth,
}
if config.Server.IsDebug {
opts.IsHeadless = false
}
return opts
}
func buildServerOptions(corsCfg core.CORSConfig, proxyCfg core.ProxyConfig, fingerprintBrowserOpts core.BrowserOpts) core.ServerOptions {
return core.ServerOptions{
CacheTTL: time.Duration(config.Cache.TTLSeconds) * time.Second,
CacheMaxSize: config.Cache.MaxSize,
EnableCORS: config.CORS.Enabled,
CORS: corsCfg,
AllowEndpointFallback: config.Resilience.AllowEndpointFallback,
CacheTTL: time.Duration(config.Cache.TTLSeconds) * time.Second,
CacheMaxSize: config.Cache.MaxSize,
EnableCORS: config.CORS.Enabled,
CORS: corsCfg,
AllowEndpointFallback: config.Resilience.AllowEndpointFallback,
EnableDebugEndpoints: config.App.DebugEndpoints,
FingerprintArtifactDir: core.DefaultFingerprintArtifactDir,
FingerprintBrowserOpts: fingerprintBrowserOpts,
Resilience: core.ResilientConfig{
Retry: core.RetryConfig{
MaxRetries: config.Resilience.MaxRetries,

View File

@@ -47,6 +47,8 @@ type BrowserOpts struct {
Insecure bool
// UseStealth enables go-rod stealth page creation.
UseStealth bool
// UserAgent optionally overrides browser-reported user agent during emulation.
UserAgent string
}
// Check applies default option values when optional fields are unset.
@@ -69,8 +71,8 @@ type Browser struct {
}
type browserConnection struct {
mu sync.Mutex
browser *rod.Browser
mu sync.Mutex
browser *rod.Browser
cachedUserAgent string
}
@@ -221,6 +223,9 @@ func (b *Browser) connectBrowser() (*rod.Browser, string, error) {
return nil, "", fmt.Errorf("read browser version: %w", err)
}
ua := strings.ReplaceAll(version.UserAgent, "HeadlessChrome/", "Chrome/")
if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" {
ua = overrideUA
}
return browser, ua, nil
}
@@ -333,7 +338,7 @@ func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
return nil, err
}
WithRequest(ctx).WithField("url", URL).Debug("Navigate to")
WithRequest(ctx).WithField("url", URL).Debug("Navigate")
browser, ua, err := b.ensureConnectedBrowser(ctx, false)
if err != nil {

View File

@@ -11,23 +11,18 @@ import (
"net/http/httptest"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"testing"
"time"
"github.com/go-rod/rod"
"github.com/karust/openserp/core/fpcheck"
"github.com/karust/openserp/core/fpcheck/detectors"
"github.com/karust/openserp/testutil"
)
const botFingerprintTestsEnv = "OPENSERP_BOT_TESTS"
const botFingerprintArtifactDir = "testdata"
var criticalSannysoftChecks = []string{"webdriver"}
const sannysoftURL = "https://bot.sannysoft.com"
func TestCreateBrowser(t *testing.T) {
testutil.RequireIntegration(t)
@@ -108,83 +103,53 @@ func TestNavigateUsesIsolatedBrowserContext(t *testing.T) {
}
}
func TestFingerprintSannysoft(t *testing.T) {
func TestFingerprintDetectors(t *testing.T) {
testutil.RequireIntegration(t)
if strings.TrimSpace(os.Getenv(botFingerprintTestsEnv)) != "1" {
t.Skipf("set %s=1 to run fingerprint tests", botFingerprintTestsEnv)
}
stealthOn := runSannysoftFingerprint(t, true)
stealthOff := runSannysoftFingerprint(t, false)
reports := make(map[string]fpcheck.Report)
criticalFailures := make([]string, 0)
improved, regressed, stillDetected := compareSannysoftRuns(stealthOff, stealthOn)
t.Logf("Stealth comparison (OFF -> ON): improved=%d regressed=%d still_detected=%d", len(improved), len(regressed), len(stillDetected))
if len(improved) > 0 {
t.Logf("Stealth fixed: %s", strings.Join(improved, ", "))
}
if len(regressed) > 0 {
t.Logf("WARN: stealth regressions: %s", strings.Join(regressed, ", "))
}
if len(stillDetected) > 0 {
t.Logf("WARN: checks still detected with stealth: %s", strings.Join(stillDetected, ", "))
for _, detector := range detectors.All() {
for _, useStealth := range []bool{false, true} {
report := runFingerprintDetector(t, detector, useStealth)
key := fmt.Sprintf("%s_%s", detector.Name(), stealthModeLabel(useStealth))
reports[key] = report
if len(report.Summary.Critical) > 0 {
for _, critical := range report.Summary.Critical {
criticalFailures = append(criticalFailures, fmt.Sprintf("%s:%s", key, critical))
}
}
}
}
reportPath := filepath.Join(botFingerprintArtifactDir, "fingerprint_sannysoft_report.json")
report := sannysoftReport{
GeneratedAtUTC: time.Now().UTC().Format(time.RFC3339),
URL: sannysoftURL,
Runs: []sannysoftRunSummary{stealthOff, stealthOn},
Comparison: sannysoftComparison{
Baseline: stealthModeLabel(false),
Candidate: stealthModeLabel(true),
Improved: improved,
Regressed: regressed,
StillDetected: stillDetected,
},
}
if err := writeSannysoftReport(reportPath, report); err != nil {
t.Fatalf("write sannysoft report: %v", err)
reportPath := filepath.Join(botFingerprintArtifactDir, "fingerprint_report.json")
if err := writeFingerprintReport(reportPath, reports); err != nil {
t.Fatalf("write fingerprint report: %v", err)
}
absReportPath, err := filepath.Abs(reportPath)
if err == nil {
t.Logf("Sannysoft report artifact: %s", absReportPath)
t.Logf("Fingerprint report artifact: %s", absReportPath)
} else {
t.Logf("Sannysoft report artifact: core/%s", filepath.ToSlash(reportPath))
t.Logf("Fingerprint report artifact: core/%s", filepath.ToSlash(reportPath))
}
for _, run := range []sannysoftRunSummary{stealthOff, stealthOn} {
if len(run.CriticalFailures) > 0 {
t.Errorf("CRITICAL fingerprint failures (%s): %v", stealthModeLabel(run.UseStealth), run.CriticalFailures)
}
if len(criticalFailures) > 0 {
t.Fatalf("critical fingerprint detections found: %s", strings.Join(criticalFailures, ", "))
}
}
type sannysoftCheck struct {
Name string `json:"name"`
Status string `json:"status"`
}
type sannysoftRunSummary struct {
UseStealth bool `json:"use_stealth"`
ScreenshotPath string `json:"screenshot_path"`
Checks []sannysoftCheck `json:"checks"`
Passed int `json:"passed"`
Failed int `json:"failed"`
Unknown int `json:"unknown"`
FailedChecks []string `json:"failed_checks"`
CriticalFailures []string `json:"critical_failures"`
ChecksByName map[string]sannysoftCheck `json:"-"`
}
func runSannysoftFingerprint(t *testing.T, useStealth bool) sannysoftRunSummary {
func runFingerprintDetector(t *testing.T, detector fpcheck.Detector, useStealth bool) fpcheck.Report {
t.Helper()
label := stealthModeLabel(useStealth)
opts := BrowserOpts{
IsHeadless: true,
IsLeakless: false,
Timeout: 15 * time.Second,
Timeout: 20 * time.Second,
UseStealth: useStealth,
}
browser, err := NewBrowser(opts)
@@ -193,91 +158,28 @@ func runSannysoftFingerprint(t *testing.T, useStealth bool) sannysoftRunSummary
}
defer closeTestBrowser(t, browser)
artifactPath := filepath.Join(botFingerprintArtifactDir, fmt.Sprintf("fingerprint_sannysoft_%s.png", label))
page, err := browser.Navigate(context.Background(), sannysoftURL)
report, err := fpcheck.Run(context.Background(), browser, detector, useStealth, botFingerprintArtifactDir)
if err != nil {
t.Fatalf("navigate to sannysoft (%s): %v", label, err)
}
defer func() {
if err := ClosePageWithTimeout(context.Background(), page, time.Second); err != nil {
t.Logf("close page (%s): %v", label, err)
}
}()
// Keep screenshot defer after page close defer so it runs first (LIFO).
defer saveSannysoftScreenshot(t, page, artifactPath, label)
if err := waitForSannysoftResults(page, 20*time.Second); err != nil {
t.Fatalf("waiting for sannysoft results (%s): %v", label, err)
t.Fatalf("run detector %s (%s): %v", detector.Name(), label, err)
}
checks, err := extractSannysoftResults(page)
if err != nil {
t.Fatalf("extracting sannysoft results (%s): %v", label, err)
}
if len(checks) == 0 {
t.Fatalf("sannysoft did not return any fingerprint check rows (%s)", label)
}
summary := sannysoftRunSummary{
UseStealth: useStealth,
ScreenshotPath: filepath.ToSlash(filepath.Join("core", artifactPath)),
ChecksByName: make(map[string]sannysoftCheck, len(checks)),
Checks: make([]sannysoftCheck, 0, len(checks)),
}
for _, check := range checks {
summary.Checks = append(summary.Checks, check)
switch check.Status {
case "pass":
summary.Passed++
case "fail":
summary.Failed++
t.Logf("WARN (%s): detected by: %s", label, check.Name)
summary.FailedChecks = append(summary.FailedChecks, check.Name)
if isCriticalFingerprintFailure(check.Name) {
summary.CriticalFailures = append(summary.CriticalFailures, check.Name)
}
default:
summary.Unknown++
}
summary.ChecksByName[normalizeFingerprintCheckName(check.Name)] = check
}
sort.Slice(summary.Checks, func(i, j int) bool {
return normalizeFingerprintCheckName(summary.Checks[i].Name) < normalizeFingerprintCheckName(summary.Checks[j].Name)
})
sort.Strings(summary.FailedChecks)
sort.Strings(summary.CriticalFailures)
total := len(checks)
t.Logf("Sannysoft (%s): %d/%d checks passed (%d failed, %d unknown)", label, summary.Passed, total, summary.Failed, summary.Unknown)
return summary
t.Logf(
"Fingerprint %s (%s): passed=%d failed=%d critical=%d",
detector.Name(),
label,
report.Summary.Passed,
report.Summary.Failed,
len(report.Summary.Critical),
)
return report
}
type sannysoftComparison struct {
Baseline string `json:"baseline"`
Candidate string `json:"candidate"`
Improved []string `json:"improved"`
Regressed []string `json:"regressed"`
StillDetected []string `json:"still_detected"`
}
type sannysoftReport struct {
GeneratedAtUTC string `json:"generated_at_utc"`
URL string `json:"url"`
Runs []sannysoftRunSummary `json:"runs"`
Comparison sannysoftComparison `json:"comparison"`
}
func writeSannysoftReport(path string, report sannysoftReport) error {
func writeFingerprintReport(path string, reports map[string]fpcheck.Report) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return fmt.Errorf("create artifact directory: %w", err)
}
data, err := json.MarshalIndent(report, "", " ")
data, err := json.MarshalIndent(reports, "", " ")
if err != nil {
return fmt.Errorf("marshal report: %w", err)
}
@@ -289,176 +191,11 @@ func writeSannysoftReport(path string, report sannysoftReport) error {
return nil
}
func saveSannysoftScreenshot(t *testing.T, page *rod.Page, path, label string) {
t.Helper()
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
t.Logf("WARN: create screenshot directory (%s): %v", label, err)
return
}
bytes, err := page.Screenshot(true, nil)
if err != nil {
t.Logf("WARN: capture screenshot (%s): %v", label, err)
return
}
if err := os.WriteFile(path, bytes, 0o644); err != nil {
t.Logf("WARN: write screenshot (%s): %v", label, err)
return
}
absPath, err := filepath.Abs(path)
if err == nil {
t.Logf("Sannysoft screenshot artifact (%s): %s", label, absPath)
return
}
t.Logf("Sannysoft screenshot artifact (%s): core/%s", label, filepath.ToSlash(path))
}
func stealthModeLabel(useStealth bool) string {
if useStealth {
return "stealth-on"
return fpcheck.ModeStealthOn
}
return "stealth-off"
}
func compareSannysoftRuns(baseline, candidate sannysoftRunSummary) (improved, regressed, stillDetected []string) {
names := make(map[string]struct{}, len(baseline.ChecksByName)+len(candidate.ChecksByName))
for key := range baseline.ChecksByName {
names[key] = struct{}{}
}
for key := range candidate.ChecksByName {
names[key] = struct{}{}
}
for key := range names {
before, hasBefore := baseline.ChecksByName[key]
after, hasAfter := candidate.ChecksByName[key]
if !hasBefore || !hasAfter {
continue
}
switch {
case before.Status == "fail" && after.Status == "pass":
improved = append(improved, after.Name)
case before.Status == "pass" && after.Status == "fail":
regressed = append(regressed, after.Name)
case before.Status == "fail" && after.Status == "fail":
stillDetected = append(stillDetected, after.Name)
}
}
sort.Strings(improved)
sort.Strings(regressed)
sort.Strings(stillDetected)
return improved, regressed, stillDetected
}
func normalizeFingerprintCheckName(name string) string {
return strings.ToLower(strings.TrimSpace(name))
}
func waitForSannysoftResults(page *rod.Page, timeout time.Duration) error {
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
hasRows, _, err := page.Has("table tr")
if err != nil {
return fmt.Errorf("table probe failed: %w", err)
}
if hasRows {
checks, err := extractSannysoftResults(page)
if err == nil && len(checks) >= 5 {
return nil
}
}
time.Sleep(250 * time.Millisecond)
}
return fmt.Errorf("results table not ready after %s", timeout)
}
func extractSannysoftResults(page *rod.Page) ([]sannysoftCheck, error) {
res, err := page.Eval(`() => {
const parseRGB = (value) => {
const match = (value || "").match(/rgba?\((\d+),\s*(\d+),\s*(\d+)/i);
if (!match) return null;
return [parseInt(match[1], 10), parseInt(match[2], 10), parseInt(match[3], 10)];
};
const classify = (text, className, bgColor) => {
const normalizedText = (text || "").toLowerCase();
const normalizedClass = (className || "").toLowerCase();
const rgb = parseRGB(bgColor);
if (/\b(fail(?:ed)?|detected|bot)\b/.test(normalizedText)) return "fail";
if (/\b(pass(?:ed)?|ok|success)\b/.test(normalizedText)) return "pass";
if (/\b(fail(?:ed)?|error|danger|bad|red)\b/.test(normalizedClass)) return "fail";
if (/\b(pass(?:ed)?|success|ok|good|green)\b/.test(normalizedClass)) return "pass";
if (rgb) {
const [r, g, b] = rgb;
if (r > g + 35 && r > b + 35) return "fail";
if (g > r + 20 && g > b + 20) return "pass";
}
return "unknown";
};
const rows = Array.from(document.querySelectorAll("table tr"));
const seen = new Set();
const checks = [];
for (const row of rows) {
const cells = Array.from(row.querySelectorAll("th, td"));
if (cells.length < 2) continue;
const nameCell = cells[0];
const name = (nameCell.innerText || nameCell.textContent || "").replace(/\s+/g, " ").trim();
if (!name) continue;
if (/^(test(\s+name)?|property|status|result)$/i.test(name)) continue;
const resultCells = cells.slice(1);
const statusCell =
resultCells.find((cell) => /\b(result|pass|fail|success|ok)\b/i.test(cell.className || "")) ||
resultCells.find((cell) => {
const bg = window.getComputedStyle(cell).backgroundColor || "";
return bg !== "" && bg !== "transparent" && bg !== "rgba(0, 0, 0, 0)";
}) ||
resultCells[0];
if (!statusCell) continue;
const statusText = (statusCell.innerText || statusCell.textContent || "").replace(/\s+/g, " ").trim();
const className = (row.className || "") + " " + (statusCell.className || "");
const bgColor = window.getComputedStyle(statusCell).backgroundColor || "";
const dedupeKey = name.toLowerCase();
if (seen.has(dedupeKey)) continue;
seen.add(dedupeKey);
checks.push({
name,
status: classify(statusText, className, bgColor),
});
}
return checks;
}`)
if err != nil {
return nil, err
}
var checks []sannysoftCheck
if err := res.Value.Unmarshal(&checks); err != nil {
return nil, fmt.Errorf("decode sannysoft results: %w", err)
}
return checks, nil
}
func isCriticalFingerprintFailure(checkName string) bool {
name := strings.ToLower(strings.TrimSpace(checkName))
return slices.ContainsFunc(criticalSannysoftChecks, func(critical string) bool {
return strings.Contains(name, critical)
})
return fpcheck.ModeStealthOff
}
func closeTestBrowser(t *testing.T, browser *Browser) {

46
core/fpcheck/detector.go Normal file
View File

@@ -0,0 +1,46 @@
package fpcheck
import (
"context"
"github.com/go-rod/rod"
)
// Detection represents a single anti-bot signal verdict from a detector page.
type Detection struct {
Detected bool `json:"detected"`
Description string `json:"description"`
Severity string `json:"severity,omitempty"`
Numeric *float64 `json:"numeric,omitempty"`
}
// Summary contains aggregate counters for a detector report.
type Summary struct {
Passed int `json:"passed"`
Failed int `json:"failed"`
Critical []string `json:"critical,omitempty"`
}
// Report is the normalized output for one detector run.
type Report struct {
DetectorName string `json:"detector_name"`
URL string `json:"url"`
UseStealth bool `json:"use_stealth"`
CapturedAtUTC string `json:"captured_at_utc"`
Screenshot string `json:"screenshot_path"`
Detections map[string]Detection `json:"detections"`
Summary Summary `json:"summary"`
RawNotes string `json:"raw_notes,omitempty"`
}
// Detector knows how to extract normalized anti-bot verdicts from one site.
type Detector interface {
Name() string
URL() string
Extract(ctx context.Context, page *rod.Page) (map[string]Detection, string, error)
}
// BrowserNavigator is the minimal browser contract used by fpcheck runner.
type BrowserNavigator interface {
Navigate(ctx context.Context, URL string) (*rod.Page, error)
}

View File

@@ -0,0 +1,62 @@
package detectors
import (
"context"
"fmt"
"time"
"github.com/go-rod/rod"
"github.com/karust/openserp/core/fpcheck"
)
const browserscanURL = "https://www.browserscan.net/bot-detection"
type BrowserScan struct{}
func NewBrowserScan() fpcheck.Detector {
return BrowserScan{}
}
func (BrowserScan) Name() string {
return "browserscan"
}
func (BrowserScan) URL() string {
return browserscanURL
}
func (BrowserScan) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
err := waitFor(ctx, 25*time.Second, 250*time.Millisecond, func() (bool, error) {
res, err := page.Eval(`() => {
const text = (document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
return text.includes("bot") && (text.includes("detected") || text.includes("pass") || text.includes("fail"));
}`)
if err != nil {
return false, nil
}
var ready bool
if err := res.Value.Unmarshal(&ready); err != nil {
return false, nil
}
return ready, nil
})
if err != nil {
return nil, "", fmt.Errorf("browserscan readiness: %w", err)
}
rows, err := parseRows(page)
if err != nil {
return nil, "", err
}
if len(rows) == 0 {
return nil, "", fmt.Errorf("browserscan detector rows not found")
}
detections := rowsToDetections(rows, []string{"bot", "webdriver", "automation", "headless", "fingerprint"})
if len(detections) == 0 {
return nil, "", fmt.Errorf("browserscan detections are empty")
}
return detections, "", nil
}

View File

@@ -0,0 +1,108 @@
package detectors
import (
"context"
"encoding/json"
"fmt"
"net/url"
"strings"
"time"
"github.com/go-rod/rod"
"github.com/karust/openserp/core/fpcheck"
)
const customDetectorName = "custom"
type Custom struct {
targetURL string
}
func NewCustom(rawURL string) (fpcheck.Detector, error) {
normalized, err := normalizeCustomURL(rawURL)
if err != nil {
return nil, err
}
return Custom{targetURL: normalized}, nil
}
func (c Custom) Name() string {
return customDetectorName
}
func (c Custom) URL() string {
return c.targetURL
}
func (c Custom) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
err := waitFor(ctx, 15*time.Second, 200*time.Millisecond, func() (bool, error) {
hasBody, _, err := page.Has("body")
if err != nil {
return false, err
}
return hasBody, nil
})
if err != nil {
return nil, "", fmt.Errorf("custom page readiness: %w", err)
}
res, err := page.Eval(`() => {
const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
return {
title: document.title || "",
url: location.href || "",
readyState: document.readyState || "",
bodyText: normalize(document.body ? document.body.innerText || document.body.textContent || "" : ""),
html: document.documentElement ? document.documentElement.outerHTML || "" : "",
};
}`)
if err != nil {
return nil, "", err
}
var payload struct {
Title string `json:"title"`
URL string `json:"url"`
ReadyState string `json:"readyState"`
BodyText string `json:"bodyText"`
HTML string `json:"html"`
}
if err := res.Value.Unmarshal(&payload); err != nil {
return nil, "", fmt.Errorf("decode custom detector payload: %w", err)
}
payload.BodyText = strings.TrimSpace(payload.BodyText)
payload.HTML = strings.TrimSpace(payload.HTML)
rawOut, err := json.MarshalIndent(payload, "", " ")
if err != nil {
return nil, "", fmt.Errorf("encode custom detector payload: %w", err)
}
return map[string]fpcheck.Detection{
"raw_page_output": {
Detected: false,
Description: "raw page payload captured",
},
}, string(rawOut), nil
}
func normalizeCustomURL(rawURL string) (string, error) {
trimmed := strings.TrimSpace(rawURL)
if trimmed == "" {
return "", fmt.Errorf("custom detector requires non-empty url query parameter")
}
parsed, err := url.Parse(trimmed)
if err != nil {
return "", fmt.Errorf("invalid custom detector URL: %w", err)
}
if parsed.Scheme != "http" && parsed.Scheme != "https" {
return "", fmt.Errorf("invalid custom detector URL scheme %q: use http or https", parsed.Scheme)
}
if strings.TrimSpace(parsed.Host) == "" {
return "", fmt.Errorf("invalid custom detector URL: host is required")
}
return parsed.String(), nil
}

View File

@@ -0,0 +1,171 @@
package detectors
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"github.com/go-rod/rod"
"github.com/karust/openserp/core/fpcheck"
)
const deviceAndBrowserURL = "https://deviceandbrowserinfo.com/are_you_a_bot"
type DeviceAndBrowser struct{}
func NewDeviceAndBrowser() fpcheck.Detector {
return DeviceAndBrowser{}
}
func (DeviceAndBrowser) Name() string {
return "deviceandbrowser"
}
func (DeviceAndBrowser) URL() string {
return deviceAndBrowserURL
}
func (DeviceAndBrowser) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
err := waitFor(ctx, 25*time.Second, 250*time.Millisecond, func() (bool, error) {
res, err := page.Eval(`() => {
const hasJson = !!document.querySelector("#jsonResult");
const hasCard = !!document.querySelector("#resultsBotTest");
const text = (document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
return hasJson || hasCard || text.includes("are you a bot");
}`)
if err != nil {
return false, nil
}
var ready bool
if err := res.Value.Unmarshal(&ready); err != nil {
return false, nil
}
return ready, nil
})
if err != nil {
return nil, "", fmt.Errorf("deviceandbrowser readiness: %w", err)
}
res, err := page.Eval(`() => {
const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
const decodeHTML = (value) => {
const textarea = document.createElement("textarea");
textarea.innerHTML = value;
return textarea.value;
};
const out = { isBot: null, details: {}, rawJson: "", cardText: "", body: "" };
const card = document.querySelector("#resultsBotTest");
if (card) {
out.cardText = normalize(card.innerText || card.textContent || "");
const low = out.cardText.toLowerCase();
if (low.includes("you are a bot")) out.isBot = true;
if (low.includes("not a bot")) out.isBot = false;
}
const jsonNode = document.querySelector("#jsonResult");
if (jsonNode) {
let raw = (jsonNode.textContent || jsonNode.innerText || "").replace(/\u00a0/g, " ").trim();
if (!raw && jsonNode.innerHTML) {
raw = decodeHTML(jsonNode.innerHTML.replace(/<br\s*\/?>/gi, "\n")).replace(/\u00a0/g, " ").trim();
}
out.rawJson = raw;
try {
const parsed = JSON.parse(raw);
if (typeof parsed.isBot === "boolean") out.isBot = parsed.isBot;
if (parsed.details && typeof parsed.details === "object") {
for (const [k, v] of Object.entries(parsed.details)) {
if (typeof v === "boolean") out.details[k] = v;
}
}
} catch (_) {}
}
out.body = normalize(document.body && document.body.innerText ? document.body.innerText : "").slice(0, 6000);
return out;
}`)
if err != nil {
return nil, "", err
}
var payload struct {
IsBot *bool `json:"isBot"`
Details map[string]bool `json:"details"`
RawJSON string `json:"rawJson"`
CardText string `json:"cardText"`
Body string `json:"body"`
}
if err := res.Value.Unmarshal(&payload); err != nil {
return nil, "", fmt.Errorf("decode deviceandbrowser payload: %w", err)
}
detections := make(map[string]fpcheck.Detection)
if payload.IsBot != nil {
severity := ""
if *payload.IsBot {
severity = "critical"
}
detections["overall_is_bot"] = fpcheck.Detection{
Detected: *payload.IsBot,
Description: strings.TrimSpace(payload.CardText),
Severity: severity,
}
}
for key, value := range payload.Details {
norm := normalizeKey(key)
if norm == "unknown" {
continue
}
severity := ""
if value && hasKeyword(norm, []string{"webdriver", "cdp", "headless", "bot", "playwright", "selenium"}) {
severity = "critical"
}
detections[norm] = fpcheck.Detection{
Detected: value,
Description: fmt.Sprintf("%t", value),
Severity: severity,
}
}
if len(detections) == 0 && strings.TrimSpace(payload.RawJSON) != "" {
// Fallback parse when JSON was extracted but JS-side parser missed fields.
var decoded struct {
IsBot *bool `json:"isBot"`
Details map[string]bool `json:"details"`
}
if err := json.Unmarshal([]byte(payload.RawJSON), &decoded); err == nil {
if decoded.IsBot != nil {
detections["overall_is_bot"] = fpcheck.Detection{
Detected: *decoded.IsBot,
Description: strings.TrimSpace(payload.CardText),
}
}
for key, value := range decoded.Details {
norm := normalizeKey(key)
if norm == "unknown" {
continue
}
detections[norm] = fpcheck.Detection{
Detected: value,
Description: fmt.Sprintf("%t", value),
}
}
}
}
if len(detections) == 0 {
return nil, payload.Body, fmt.Errorf("deviceandbrowser detections are empty")
}
rawNotes := strings.TrimSpace(payload.RawJSON)
if rawNotes == "" {
rawNotes = strings.TrimSpace(payload.Body)
}
return detections, rawNotes, nil
}

View File

@@ -0,0 +1,241 @@
package detectors
import (
"context"
"fmt"
"regexp"
"strconv"
"strings"
"time"
"github.com/go-rod/rod"
"github.com/karust/openserp/core/fpcheck"
)
var (
reNonAlphanumUnderscore = regexp.MustCompile(`[^a-z0-9_]+`)
reMultiUnderscore = regexp.MustCompile(`_+`)
reExtractScore = regexp.MustCompile(`(?i)(score|overall|risk)[^\d]{0,20}(\d+(?:\.\d+)?)`)
)
type detectorRow struct {
Name string `json:"name"`
Status string `json:"status"`
Detail string `json:"detail"`
}
func waitFor(ctx context.Context, timeout time.Duration, poll time.Duration, probe func() (bool, error)) error {
if timeout <= 0 {
timeout = 20 * time.Second
}
if poll <= 0 {
poll = 250 * time.Millisecond
}
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if err := ctx.Err(); err != nil {
return err
}
ok, err := probe()
if err != nil {
return err
}
if ok {
return nil
}
timer := time.NewTimer(poll)
select {
case <-timer.C:
case <-ctx.Done():
timer.Stop()
return ctx.Err()
}
}
return fmt.Errorf("ready condition not met after %s", timeout)
}
func normalizeKey(name string) string {
name = strings.ToLower(strings.TrimSpace(name))
if name == "" {
return "unknown"
}
replacer := strings.NewReplacer(
" ", "_",
"-", "_",
"/", "_",
"\\", "_",
":", "_",
".", "_",
)
name = replacer.Replace(name)
name = reNonAlphanumUnderscore.ReplaceAllString(name, "")
name = reMultiUnderscore.ReplaceAllString(name, "_")
name = strings.Trim(name, "_")
if name == "" {
return "unknown"
}
return name
}
func classifyStatus(status string) bool {
value := strings.ToLower(strings.TrimSpace(status))
if value == "" {
return false
}
notDetected := []string{"not detected", "not found", "clean", "clear", "pass", "passed", "ok", "safe", "green", "false", "no"}
for _, marker := range notDetected {
if strings.Contains(value, marker) {
return false
}
}
detected := []string{"detected", "fail", "failed", "bot", "leak", "warning", "critical", "red", "true", "yes"}
for _, marker := range detected {
if strings.Contains(value, marker) {
return true
}
}
return false
}
func parseRows(page *rod.Page) ([]detectorRow, error) {
res, err := page.Eval(`() => {
const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
const rows = [];
const seen = new Set();
const tableRows = Array.from(document.querySelectorAll("table tr"));
for (const row of tableRows) {
const cells = Array.from(row.querySelectorAll("th, td"));
if (cells.length < 2) continue;
const name = normalize(cells[0].innerText || cells[0].textContent || "");
if (!name) continue;
if (/^(test(\s+name)?|property|status|result|check)$/i.test(name)) continue;
const valueCell = cells[cells.length - 1];
const status = normalize(valueCell.innerText || valueCell.textContent || "");
if (!status) continue;
const key = name.toLowerCase();
if (seen.has(key)) continue;
seen.add(key);
rows.push({
name,
status,
detail: normalize(row.innerText || row.textContent || ""),
});
}
const candidates = Array.from(document.querySelectorAll("[data-test], [data-testid], [data-check], [data-name], .check, .result, li"));
for (const node of candidates) {
const text = normalize(node.innerText || node.textContent || "");
if (!text || text.length > 260) continue;
if (!/(pass|fail|detected|not detected|warning|critical|true|false|yes|no|leak|bot)/i.test(text)) continue;
let name = normalize(node.getAttribute("data-check") || node.getAttribute("data-name") || node.getAttribute("data-testid") || "");
let status = "";
if (!name) {
const parts = text.split(/[:\-|]/).map(normalize).filter(Boolean);
if (parts.length >= 2) {
name = parts[0];
status = parts.slice(1).join(" ");
}
}
if (!name) {
const lines = text.split(/\n+/).map(normalize).filter(Boolean);
if (lines.length >= 2) {
name = lines[0];
status = lines.slice(1).join(" ");
}
}
if (!name) continue;
if (!status) {
status = text;
}
const key = name.toLowerCase();
if (seen.has(key)) continue;
seen.add(key);
rows.push({ name, status, detail: text });
}
return rows;
}`)
if err != nil {
return nil, err
}
var rows []detectorRow
if err := res.Value.Unmarshal(&rows); err != nil {
return nil, fmt.Errorf("decode detector rows: %w", err)
}
return rows, nil
}
func rowsToDetections(rows []detectorRow, criticalKeywords []string) map[string]fpcheck.Detection {
out := make(map[string]fpcheck.Detection, len(rows))
for _, row := range rows {
key := normalizeKey(row.Name)
if key == "unknown" {
continue
}
detected := classifyStatus(row.Status)
severity := ""
if detected && hasKeyword(key+" "+strings.ToLower(row.Detail), criticalKeywords) {
severity = "critical"
}
description := strings.TrimSpace(row.Status)
if description == "" {
description = strings.TrimSpace(row.Detail)
}
out[key] = fpcheck.Detection{
Detected: detected,
Description: description,
Severity: severity,
}
}
return out
}
func hasKeyword(value string, keywords []string) bool {
if len(keywords) == 0 {
return false
}
for _, keyword := range keywords {
k := strings.ToLower(strings.TrimSpace(keyword))
if k == "" {
continue
}
if strings.Contains(value, k) {
return true
}
}
return false
}
func extractScore(text string) (float64, bool) {
matches := reExtractScore.FindStringSubmatch(text)
if len(matches) < 3 {
return 0, false
}
score, err := strconv.ParseFloat(matches[2], 64)
if err != nil {
return 0, false
}
return score, true
}

View File

@@ -0,0 +1,204 @@
package detectors
import (
"context"
"fmt"
"strings"
"time"
"github.com/go-rod/rod"
"github.com/karust/openserp/core/fpcheck"
)
const (
pixelscanURL = "https://pixelscan.net/bot-check"
)
type PixelScan struct{}
func NewPixelScan() fpcheck.Detector {
return PixelScan{}
}
func (PixelScan) Name() string {
return "pixelscan"
}
func (PixelScan) URL() string {
return pixelscanURL
}
func (PixelScan) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
err := waitFor(ctx, 25*time.Second, 250*time.Millisecond, func() (bool, error) {
res, err := page.Eval(`() => {
const hasSummary = !!document.querySelector(".bot-check-summary, .bot-check__summary, .bot-check-accordion__row");
const hasState = !!document.querySelector(".state-success, .state-error");
const body = (document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
return hasSummary || hasState || body.includes("running bot detection") || body.includes("definitely a human") || body.includes("bot behavior detected");
}`)
if err != nil {
return false, nil
}
var ready bool
if err := res.Value.Unmarshal(&ready); err != nil {
return false, nil
}
return ready, nil
})
if err != nil {
return nil, "", fmt.Errorf("pixelscan readiness: %w", err)
}
res, err := page.Eval(`() => {
const normalize = (value) => (value || "").replace(/\s+/g, " ").trim();
const isVisible = (node) => !!node && !!(node.offsetParent || node.getClientRects().length) &&
window.getComputedStyle(node).display !== "none" &&
window.getComputedStyle(node).visibility !== "hidden" &&
window.getComputedStyle(node).opacity !== "0";
const successNode = document.querySelector(".state-success");
const errorNode = document.querySelector(".state-error");
let state = "unknown";
if (isVisible(successNode)) state = "human";
if (isVisible(errorNode)) state = "bot";
if (state === "unknown") {
const text = normalize(document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
if (text.includes("you're definitely a human")) state = "human";
if (text.includes("bot behavior detected")) state = "bot";
}
const summary = [];
for (const section of Array.from(document.querySelectorAll(".summary-section"))) {
const full = normalize(section.innerText || section.textContent || "");
if (!full) continue;
const statusNode = section.querySelector(".summary-section__status");
const status = normalize(statusNode ? (statusNode.innerText || statusNode.textContent || "") : "");
let name = full;
if (status) {
name = normalize(full.replace(new RegExp("\\\\s*" + status + "\\\\s*\\\\d*\\\\s*parameters?$", "i"), ""));
}
if (!name || !status) continue;
summary.push({name, status});
}
const rows = [];
for (const row of Array.from(document.querySelectorAll(".bot-check-accordion__row"))) {
const statusNode = row.querySelector(".bot-check-accordion__status");
const labelNode = row.querySelector(".bot-check-accordion__label");
const status = normalize(statusNode ? (statusNode.innerText || statusNode.textContent || "") : "");
if (!status) continue;
let name = normalize(labelNode ? (labelNode.innerText || labelNode.textContent || "") : "");
if (!name) {
const full = normalize(row.innerText || row.textContent || "");
name = normalize(full.replace(new RegExp("\\\\s*" + status + "\\\\s*$", "i"), ""));
}
if (!name) continue;
rows.push({name, status});
}
const body = normalize(document.body && document.body.innerText ? document.body.innerText : "");
return {
state,
summary,
rows,
body: body.slice(0, 6000),
};
}`)
if err != nil {
return nil, "", err
}
var payload struct {
State string `json:"state"`
Summary []struct {
Name string `json:"name"`
Status string `json:"status"`
} `json:"summary"`
Rows []struct {
Name string `json:"name"`
Status string `json:"status"`
} `json:"rows"`
Body string `json:"body"`
}
if err := res.Value.Unmarshal(&payload); err != nil {
return nil, "", fmt.Errorf("decode pixelscan payload: %w", err)
}
payload.Body = strings.TrimSpace(payload.Body)
if payload.Body == "" {
return nil, "", fmt.Errorf("pixelscan body is empty")
}
detections := make(map[string]fpcheck.Detection)
if payload.State != "" {
overallDetected := strings.EqualFold(strings.TrimSpace(payload.State), "bot")
overallStatus := strings.TrimSpace(payload.State)
if overallStatus == "" {
overallStatus = "unknown"
}
overallSeverity := ""
if overallDetected {
overallSeverity = "critical"
}
detections["overall_verdict"] = fpcheck.Detection{
Detected: overallDetected,
Description: overallStatus,
Severity: overallSeverity,
}
}
for _, item := range payload.Summary {
key := "summary_" + normalizeKey(item.Name)
if key == "summary_unknown" {
continue
}
detected := classifyStatus(item.Status)
severity := ""
if detected && hasKeyword(strings.ToLower(item.Name), []string{"webdriver", "cdp", "bot"}) {
severity = "critical"
}
detections[key] = fpcheck.Detection{
Detected: detected,
Description: strings.TrimSpace(item.Status),
Severity: severity,
}
}
for _, row := range payload.Rows {
key := normalizeKey(row.Name)
if key == "unknown" {
continue
}
detected := classifyStatus(row.Status)
severity := ""
if detected && hasKeyword(strings.ToLower(row.Name), []string{"webdriver", "cdp", "headless", "automation"}) {
severity = "critical"
}
detections[key] = fpcheck.Detection{
Detected: detected,
Description: strings.TrimSpace(row.Status),
Severity: severity,
}
}
if len(detections) == 0 {
if score, ok := extractScore(payload.Body); ok {
scoreValue := score
detections["overall_score"] = fpcheck.Detection{
Detected: false,
Description: fmt.Sprintf("score %.2f", score),
Numeric: &scoreValue,
}
}
}
if len(detections) == 0 {
return nil, payload.Body, fmt.Errorf("pixelscan detections not found")
}
return detections, payload.Body, nil
}

View File

@@ -0,0 +1,77 @@
package detectors
import (
"context"
"fmt"
"time"
"github.com/go-rod/rod"
"github.com/karust/openserp/core/fpcheck"
)
const rebrowserURL = "https://bot-detector.rebrowser.net/"
type Rebrowser struct{}
func NewRebrowser() fpcheck.Detector {
return Rebrowser{}
}
func (Rebrowser) Name() string {
return "rebrowser"
}
func (Rebrowser) URL() string {
return rebrowserURL
}
func (Rebrowser) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
err := waitFor(ctx, 20*time.Second, 250*time.Millisecond, func() (bool, error) {
hasBody, _, err := page.Has("body")
if err != nil {
return false, err
}
if !hasBody {
return false, nil
}
res, err := page.Eval(`() => {
const text = (document.body && document.body.innerText ? document.body.innerText : "").toLowerCase();
return text.includes("runtimeenableleak") || text.includes("sourceurlleak") || text.includes("mainworldexecution") || text.includes("webdriver");
}`)
if err != nil {
return false, nil
}
var ready bool
if err := res.Value.Unmarshal(&ready); err != nil {
return false, nil
}
return ready, nil
})
if err != nil {
return nil, "", fmt.Errorf("rebrowser readiness: %w", err)
}
rows, err := parseRows(page)
if err != nil {
return nil, "", err
}
if len(rows) == 0 {
return nil, "", fmt.Errorf("rebrowser detector rows not found")
}
detections := rowsToDetections(rows, []string{
"runtimeenableleak",
"sourceurlleak",
"mainworldexecution",
"webdriver",
"automation",
})
if len(detections) == 0 {
return nil, "", fmt.Errorf("rebrowser detections are empty")
}
return detections, "", nil
}

View File

@@ -0,0 +1,65 @@
package detectors
import (
"fmt"
"sort"
"strings"
"github.com/karust/openserp/core/fpcheck"
)
var standardDetectorFactories = []struct {
name string
new func() fpcheck.Detector
}{
{name: "sannysoft", new: NewSannysoft},
{name: "rebrowser", new: NewRebrowser},
{name: "browserscan", new: NewBrowserScan},
{name: "pixelscan", new: NewPixelScan},
{name: "deviceandbrowser", new: NewDeviceAndBrowser},
}
func All() []fpcheck.Detector {
detectors := make([]fpcheck.Detector, 0, len(standardDetectorFactories))
for _, item := range standardDetectorFactories {
detectors = append(detectors, item.new())
}
return detectors
}
func Select(name string, customURL string) ([]fpcheck.Detector, error) {
trimmed := strings.ToLower(strings.TrimSpace(name))
if trimmed == "" || trimmed == "all" {
return All(), nil
}
if IsCustom(trimmed) {
customDetector, err := NewCustom(customURL)
if err != nil {
return nil, err
}
return []fpcheck.Detector{customDetector}, nil
}
for _, item := range standardDetectorFactories {
if strings.EqualFold(item.name, trimmed) {
return []fpcheck.Detector{item.new()}, nil
}
}
return nil, fmt.Errorf("unknown detector %q (allowed: %s)", name, strings.Join(Names(), ","))
}
func Names() []string {
names := make([]string, 0, len(standardDetectorFactories)+1)
for _, item := range standardDetectorFactories {
names = append(names, item.name)
}
names = append(names, customDetectorName)
sort.Strings(names)
return names
}
func IsCustom(name string) bool {
return strings.EqualFold(strings.TrimSpace(name), customDetectorName)
}

View File

@@ -0,0 +1,40 @@
package detectors
import "testing"
func TestSelectCustomRequiresURL(t *testing.T) {
_, err := Select("custom", "")
if err == nil {
t.Fatal("expected custom detector selection without URL to fail")
}
}
func TestSelectCustomAcceptsHTTPSURL(t *testing.T) {
detectorList, err := Select("custom", "https://localhost:9000")
if err != nil {
t.Fatalf("expected custom detector URL to be accepted, got %v", err)
}
if len(detectorList) != 1 {
t.Fatalf("expected one detector, got %d", len(detectorList))
}
if detectorList[0].Name() != "custom" {
t.Fatalf("expected custom detector, got %q", detectorList[0].Name())
}
if detectorList[0].URL() != "https://localhost:9000" {
t.Fatalf("expected normalized custom URL to be preserved, got %q", detectorList[0].URL())
}
}
func TestNamesIncludesCustom(t *testing.T) {
names := Names()
found := false
for _, name := range names {
if name == "custom" {
found = true
break
}
}
if !found {
t.Fatalf("expected names to include custom detector, got %#v", names)
}
}

View File

@@ -0,0 +1,162 @@
package detectors
import (
"context"
"fmt"
"strings"
"time"
"github.com/go-rod/rod"
"github.com/karust/openserp/core/fpcheck"
)
const sannysoftURL = "https://bot.sannysoft.com"
type Sannysoft struct{}
func NewSannysoft() fpcheck.Detector {
return Sannysoft{}
}
func (Sannysoft) Name() string {
return "sannysoft"
}
func (Sannysoft) URL() string {
return sannysoftURL
}
func (Sannysoft) Extract(ctx context.Context, page *rod.Page) (map[string]fpcheck.Detection, string, error) {
var checks []sannysoftRow
err := waitFor(ctx, 20*time.Second, 250*time.Millisecond, func() (bool, error) {
hasRows, _, err := page.Has("table tr")
if err != nil {
return false, fmt.Errorf("table probe failed: %w", err)
}
if !hasRows {
return false, nil
}
rows, err := extractSannysoftRows(page)
if err != nil {
return false, nil
}
checks = rows
return len(checks) >= 5, nil
})
if err != nil {
return nil, "", err
}
if len(checks) == 0 {
return nil, "", fmt.Errorf("sannysoft did not return any fingerprint check rows")
}
detections := make(map[string]fpcheck.Detection, len(checks))
for _, check := range checks {
key := normalizeKey(check.Name)
if key == "unknown" {
continue
}
detected := check.Status == "fail"
severity := ""
if detected && strings.Contains(key, "webdriver") {
severity = "critical"
}
description := check.Status
if description == "" {
description = "unknown"
}
detections[key] = fpcheck.Detection{
Detected: detected,
Description: description,
Severity: severity,
}
}
return detections, "", nil
}
type sannysoftRow struct {
Name string `json:"name"`
Status string `json:"status"`
}
func extractSannysoftRows(page *rod.Page) ([]sannysoftRow, error) {
res, err := page.Eval(`() => {
const parseRGB = (value) => {
const match = (value || "").match(/rgba?\((\d+),\s*(\d+),\s*(\d+)/i);
if (!match) return null;
return [parseInt(match[1], 10), parseInt(match[2], 10), parseInt(match[3], 10)];
};
const classify = (text, className, bgColor) => {
const normalizedText = (text || "").toLowerCase();
const normalizedClass = (className || "").toLowerCase();
const rgb = parseRGB(bgColor);
if (/\b(fail(?:ed)?|detected|bot)\b/.test(normalizedText)) return "fail";
if (/\b(pass(?:ed)?|ok|success)\b/.test(normalizedText)) return "pass";
if (/\b(fail(?:ed)?|error|danger|bad|red)\b/.test(normalizedClass)) return "fail";
if (/\b(pass(?:ed)?|success|ok|good|green)\b/.test(normalizedClass)) return "pass";
if (rgb) {
const [r, g, b] = rgb;
if (r > g + 35 && r > b + 35) return "fail";
if (g > r + 20 && g > b + 20) return "pass";
}
return "unknown";
};
const rows = Array.from(document.querySelectorAll("table tr"));
const seen = new Set();
const checks = [];
for (const row of rows) {
const cells = Array.from(row.querySelectorAll("th, td"));
if (cells.length < 2) continue;
const nameCell = cells[0];
const name = (nameCell.innerText || nameCell.textContent || "").replace(/\s+/g, " ").trim();
if (!name) continue;
if (/^(test(\s+name)?|property|status|result)$/i.test(name)) continue;
const resultCells = cells.slice(1);
const statusCell =
resultCells.find((cell) => /\b(result|pass|fail|success|ok)\b/i.test(cell.className || "")) ||
resultCells.find((cell) => {
const bg = window.getComputedStyle(cell).backgroundColor || "";
return bg !== "" && bg !== "transparent" && bg !== "rgba(0, 0, 0, 0)";
}) ||
resultCells[0];
if (!statusCell) continue;
const statusText = (statusCell.innerText || statusCell.textContent || "").replace(/\s+/g, " ").trim();
const className = (row.className || "") + " " + (statusCell.className || "");
const bgColor = window.getComputedStyle(statusCell).backgroundColor || "";
const dedupeKey = name.toLowerCase();
if (seen.has(dedupeKey)) continue;
seen.add(dedupeKey);
checks.push({
name,
status: classify(statusText, className, bgColor),
});
}
return checks;
}`)
if err != nil {
return nil, err
}
var checks []sannysoftRow
if err := res.Value.Unmarshal(&checks); err != nil {
return nil, fmt.Errorf("decode sannysoft results: %w", err)
}
return checks, nil
}

176
core/fpcheck/runner.go Normal file
View File

@@ -0,0 +1,176 @@
package fpcheck
import (
"context"
"fmt"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/proto"
)
const (
ModeStealthOn = "stealth-on"
ModeStealthOff = "stealth-off"
)
// RunOptions controls detector run behavior.
type RunOptions struct {
UseStealth bool
ArtifactDir string
WaitBeforeClose time.Duration
}
// Run navigates the given browser to detector URL, extracts verdicts,
// captures a screenshot artifact, and returns a normalized report.
func Run(ctx context.Context, browser BrowserNavigator, detector Detector, useStealth bool, artifactDir string) (Report, error) {
return RunWithOptions(ctx, browser, detector, RunOptions{
UseStealth: useStealth,
ArtifactDir: artifactDir,
})
}
// RunWithOptions navigates the given browser to detector URL, extracts
// verdicts, captures a screenshot artifact, optionally waits, and returns a
// normalized report.
func RunWithOptions(ctx context.Context, browser BrowserNavigator, detector Detector, options RunOptions) (Report, error) {
report := Report{
DetectorName: detector.Name(),
URL: detector.URL(),
UseStealth: options.UseStealth,
Detections: map[string]Detection{},
}
artifactDir := strings.TrimSpace(options.ArtifactDir)
if artifactDir == "" {
artifactDir = "testdata"
}
screenshotPath := filepath.Join(artifactDir, fmt.Sprintf("fpcheck_%s_%s.png", sanitizeFilePart(detector.Name()), modeLabel(options.UseStealth)))
report.Screenshot = filepath.ToSlash(screenshotPath)
page, err := browser.Navigate(ctx, detector.URL())
if err != nil {
return report, fmt.Errorf("navigate %s: %w", detector.Name(), err)
}
defer func() {
if options.WaitBeforeClose > 0 {
_ = sleepWithContext(ctx, options.WaitBeforeClose)
}
closePageWithTimeout(context.Background(), page, time.Second)
}()
detections, rawNotes, err := detector.Extract(ctx, page)
if err != nil {
_ = saveScreenshot(page, screenshotPath)
return report, fmt.Errorf("extract %s: %w", detector.Name(), err)
}
if err := saveScreenshot(page, screenshotPath); err != nil {
return report, fmt.Errorf("capture screenshot %s: %w", detector.Name(), err)
}
report.CapturedAtUTC = time.Now().UTC().Format(time.RFC3339)
report.Detections = detections
report.Summary = summarize(detections)
report.RawNotes = strings.TrimSpace(rawNotes)
return report, nil
}
func sleepWithContext(ctx context.Context, d time.Duration) error {
if d <= 0 {
return nil
}
if ctx == nil {
ctx = context.Background()
}
timer := time.NewTimer(d)
defer timer.Stop()
select {
case <-timer.C:
return nil
case <-ctx.Done():
return ctx.Err()
}
}
func summarize(detections map[string]Detection) Summary {
summary := Summary{}
for key, detection := range detections {
if detection.Detected {
summary.Failed++
if strings.EqualFold(strings.TrimSpace(detection.Severity), "critical") {
summary.Critical = append(summary.Critical, key)
}
continue
}
summary.Passed++
}
sort.Strings(summary.Critical)
return summary
}
func saveScreenshot(page *rod.Page, path string) error {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return fmt.Errorf("create screenshot directory: %w", err)
}
bytes, err := page.Screenshot(true, nil)
if err != nil {
return fmt.Errorf("capture screenshot: %w", err)
}
if err := os.WriteFile(path, bytes, 0o644); err != nil {
return fmt.Errorf("write screenshot file %s: %w", path, err)
}
return nil
}
func closePageWithTimeout(ctx context.Context, page *rod.Page, timeout time.Duration) {
if page == nil {
return
}
if timeout <= 0 {
timeout = time.Second
}
closeCtx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
pageWithTimeout := page.Context(closeCtx)
info, _ := pageWithTimeout.Info()
_ = pageWithTimeout.Close()
if info != nil && info.BrowserContextID != "" {
_ = (proto.TargetDisposeBrowserContext{BrowserContextID: info.BrowserContextID}).Call(page.Browser().Context(closeCtx))
}
}
func modeLabel(useStealth bool) string {
if useStealth {
return ModeStealthOn
}
return ModeStealthOff
}
func sanitizeFilePart(value string) string {
value = strings.TrimSpace(strings.ToLower(value))
if value == "" {
return "detector"
}
parts := strings.FieldsFunc(value, func(r rune) bool {
return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9')
})
if len(parts) == 0 {
return "detector"
}
return strings.Join(parts, "_")
}

View File

@@ -6,18 +6,27 @@ import (
"errors"
"fmt"
"html"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/gofiber/fiber/v2"
"github.com/karust/openserp/core/fpcheck"
"github.com/karust/openserp/core/fpcheck/detectors"
apidocs "github.com/karust/openserp/docs"
"github.com/sirupsen/logrus"
"golang.org/x/time/rate"
)
// DefaultFingerprintArtifactDir is the artifact directory used when none is
// configured. It is relative to the server's working directory at start time.
var DefaultFingerprintArtifactDir = filepath.Join("core", "testdata")
// SearchEngine defines the contract required by the HTTP server and resilient
// search pipeline.
type SearchEngine interface {
@@ -60,6 +69,12 @@ type ServerOptions struct {
// AllowEndpointFallback allows dedicated engine routes to fall back to other
// healthy engines when the primary engine fails.
AllowEndpointFallback bool
// EnableDebugEndpoints enables debug-only routes such as fingerprint checks.
EnableDebugEndpoints bool
// FingerprintArtifactDir is where debug fingerprint screenshots are written.
FingerprintArtifactDir string
// FingerprintBrowserOpts are the defaults for debug fingerprint runs.
FingerprintBrowserOpts BrowserOpts
// Resilience defines retry/circuit-breaker/proxy strategy settings.
Resilience ResilientConfig
}
@@ -68,12 +83,18 @@ type ServerOptions struct {
// and resilient search policies.
func DefaultServerOptions() ServerOptions {
return ServerOptions{
CacheTTL: 5 * time.Minute,
CacheMaxSize: 1000,
EnableCORS: true,
CORS: DefaultCORSConfig(),
AllowEndpointFallback: false,
Resilience: DefaultResilientConfig(),
CacheTTL: 5 * time.Minute,
CacheMaxSize: 1000,
EnableCORS: true,
CORS: DefaultCORSConfig(),
AllowEndpointFallback: false,
EnableDebugEndpoints: false,
FingerprintArtifactDir: DefaultFingerprintArtifactDir,
FingerprintBrowserOpts: BrowserOpts{
IsHeadless: true,
Timeout: 30 * time.Second,
},
Resilience: DefaultResilientConfig(),
}
}
@@ -128,6 +149,9 @@ func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngin
app.Get("/stats/cache", serv.handleCacheStats)
app.Get("/stats/proxy", serv.handleProxyStats)
app.Get("/stats/cb", serv.handleCircuitBreakerStats)
if opts.EnableDebugEndpoints {
app.Get("/debug/fingerprint-check", serv.handleFingerprintCheck)
}
for _, engine := range searchEngines {
locEngine := engine
@@ -380,6 +404,168 @@ func (s *Server) handleCircuitBreakerStats(c *fiber.Ctx) error {
})
}
func (s *Server) handleFingerprintCheck(c *fiber.Ctx) error {
req, err := s.parseFingerprintCheckRequest(c)
if err != nil {
return err
}
runCtx, cancel := context.WithTimeout(c.UserContext(), time.Duration(req.timeoutMs)*time.Millisecond)
defer cancel()
browser, err := NewBrowser(req.browserOpts)
if err != nil {
return fiber.NewError(fiber.StatusServiceUnavailable, fmt.Sprintf("failed to create debug browser: %v", err))
}
defer func() {
if closeErr := browser.Close(); closeErr != nil {
WithRequest(c.UserContext()).WithError(closeErr).Warn("failed to close debug fingerprint browser")
}
}()
artifactDir := defaultFingerprintArtifactDir(s.opts.FingerprintArtifactDir)
reports := make([]fpcheck.Report, 0, len(req.detectors))
for idx, detector := range req.detectors {
runOpts := fpcheck.RunOptions{
UseStealth: req.browserOpts.UseStealth,
ArtifactDir: artifactDir,
}
if req.waitMs > 0 && idx == len(req.detectors)-1 {
runOpts.WaitBeforeClose = time.Duration(req.waitMs) * time.Millisecond
}
report, runErr := fpcheck.RunWithOptions(runCtx, browser, detector, runOpts)
if runErr != nil {
if errors.Is(runCtx.Err(), context.DeadlineExceeded) {
return fiber.NewError(fiber.StatusGatewayTimeout, fmt.Sprintf("fingerprint check timed out after %dms", req.timeoutMs))
}
return fiber.NewError(
fiber.StatusServiceUnavailable,
fmt.Sprintf("detector %s failed: %v", detector.Name(), runErr),
)
}
reports = append(reports, report)
}
return c.JSON(reports)
}
type fingerprintCheckRequest struct {
detectors []fpcheck.Detector
timeoutMs int
waitMs int
browserOpts BrowserOpts
}
func (s *Server) parseFingerprintCheckRequest(c *fiber.Ctx) (fingerprintCheckRequest, error) {
detectorName := strings.TrimSpace(c.Query("detector", "all"))
customURL := strings.TrimSpace(c.Query("url", ""))
selectedDetectors, err := detectors.Select(detectorName, customURL)
if err != nil {
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, err.Error())
}
useStealth, err := parseOptionalBoolQuery(c.Query("stealth", ""), s.opts.FingerprintBrowserOpts.UseStealth)
if err != nil {
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("invalid stealth query value: %v", err))
}
headless, err := parseOptionalBoolQuery(c.Query("headless", ""), true)
if err != nil {
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("invalid headless query value: %v", err))
}
if !headless && strings.TrimSpace(os.Getenv("DISPLAY")) == "" {
WithRequest(c.UserContext()).Warn("headless=false ignored because DISPLAY is not set; forcing headless mode")
headless = true
}
timeoutMs, err := parsePositiveIntQuery(c.Query("timeout_ms", ""), 150000)
if err != nil {
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("invalid timeout_ms query value: %v", err))
}
waitMs, err := parseNonNegativeIntQuery(c.Query("wait_ms", ""), 0)
if err != nil {
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("invalid wait_ms query value: %v", err))
}
browserOpts := s.opts.FingerprintBrowserOpts
browserOpts.IsHeadless = headless
browserOpts.UseStealth = useStealth
browserOpts.Timeout = time.Duration(timeoutMs) * time.Millisecond
browserOpts.LeavePageOpen = false
browserOpts.UserAgent = strings.TrimSpace(c.Query("user_agent", browserOpts.UserAgent))
browserOpts.ProxyURL = strings.TrimSpace(c.Query("proxy", browserOpts.ProxyURL))
browserOpts.LanguageCode = strings.TrimSpace(c.Query("language", browserOpts.LanguageCode))
insecureDefault := browserOpts.Insecure
if detectors.IsCustom(detectorName) {
insecureDefault = true
}
insecure, err := parseOptionalBoolQuery(c.Query("insecure", ""), insecureDefault)
if err != nil {
return fingerprintCheckRequest{}, fiber.NewError(fiber.StatusBadRequest, fmt.Sprintf("invalid insecure query value: %v", err))
}
browserOpts.Insecure = insecure
return fingerprintCheckRequest{
detectors: selectedDetectors,
timeoutMs: timeoutMs,
waitMs: waitMs,
browserOpts: browserOpts,
}, nil
}
func defaultFingerprintArtifactDir(value string) string {
trimmed := strings.TrimSpace(value)
if trimmed == "" {
return DefaultFingerprintArtifactDir
}
return trimmed
}
func parseOptionalBoolQuery(raw string, defaultValue bool) (bool, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return defaultValue, nil
}
value, err := strconv.ParseBool(raw)
if err != nil {
return false, err
}
return value, nil
}
func parsePositiveIntQuery(raw string, defaultValue int) (int, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return defaultValue, nil
}
value, err := strconv.Atoi(raw)
if err != nil {
return 0, err
}
if value <= 0 {
return 0, fmt.Errorf("must be > 0")
}
return value, nil
}
func parseNonNegativeIntQuery(raw string, defaultValue int) (int, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return defaultValue, nil
}
value, err := strconv.Atoi(raw)
if err != nil {
return 0, err
}
if value < 0 {
return 0, fmt.Errorf("must be >= 0")
}
return value, nil
}
// MegaSearchResult extends SearchResult with the engine source name.
type MegaSearchResult struct {
SearchResult

View File

@@ -151,6 +151,76 @@ func TestDocsEndpointServesSwaggerUI(t *testing.T) {
}
}
func TestDebugFingerprintEndpointDisabledByDefault(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
srv := NewServerWithOptions("127.0.0.1", 7109, DefaultServerOptions(), engine)
resp := request(t, srv, "/debug/fingerprint-check")
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("expected disabled debug endpoint to return 404, got %d", resp.StatusCode)
}
}
func TestDebugFingerprintEndpointValidatesDetectorParam(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
opts.EnableDebugEndpoints = true
srv := NewServerWithOptions("127.0.0.1", 7112, opts, engine)
resp := request(t, srv, "/debug/fingerprint-check?detector=unknown")
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected invalid detector to return 400, got %d", resp.StatusCode)
}
}
func TestDebugFingerprintEndpointValidatesWaitParam(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
opts.EnableDebugEndpoints = true
srv := NewServerWithOptions("127.0.0.1", 7113, opts, engine)
resp := request(t, srv, "/debug/fingerprint-check?detector=sannysoft&wait_ms=-1")
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected invalid wait_ms to return 400, got %d", resp.StatusCode)
}
}
func TestDebugFingerprintEndpointCustomDetectorRequiresURL(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
opts.EnableDebugEndpoints = true
srv := NewServerWithOptions("127.0.0.1", 7114, opts, engine)
resp := request(t, srv, "/debug/fingerprint-check?detector=custom")
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected custom detector without url to return 400, got %d", resp.StatusCode)
}
}
func TestDebugFingerprintEndpointCustomDetectorValidatesInsecureParam(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
opts.EnableDebugEndpoints = true
srv := NewServerWithOptions("127.0.0.1", 7115, opts, engine)
resp := request(t, srv, "/debug/fingerprint-check?detector=custom&url=https://localhost:9000&insecure=notabool")
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected invalid insecure query value to return 400, got %d", resp.StatusCode)
}
}
func TestDebugFingerprintEndpointCustomDetectorValidatesURL(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
opts := DefaultServerOptions()
opts.EnableDebugEndpoints = true
srv := NewServerWithOptions("127.0.0.1", 7116, opts, engine)
resp := request(t, srv, "/debug/fingerprint-check?detector=custom&url=not-a-url")
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("expected invalid custom URL to return 400, got %d", resp.StatusCode)
}
}
func TestInvalidQueryParametersReturnJSONError(t *testing.T) {
engine := &engineMock{name: "google", initialized: true}
srv := NewServerWithOptions("127.0.0.1", 7104, DefaultServerOptions(), engine)