Files
openserp/core/browser.go

811 lines
23 KiB
Go

package core
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"regexp"
"runtime/debug"
"strings"
"sync"
"time"
"github.com/go-rod/rod"
"github.com/go-rod/rod/lib/launcher"
"github.com/go-rod/rod/lib/proto"
browserprofile "github.com/karust/openserp/core/browser"
"github.com/sirupsen/logrus"
"github.com/ysmood/gson"
)
// BrowserOpts configures Chromium launch and navigation behavior.
type BrowserOpts struct {
// IsHeadless runs Chromium without visible UI.
IsHeadless bool
// IsLeakless forces child browser process cleanup when the parent exits.
IsLeakless bool
// Timeout is applied to browser connect and page navigation operations.
Timeout time.Duration
// LanguageCode sets Accept-Language for emulated requests.
LanguageCode string
// WaitRequests waits for request-idle state after navigation.
WaitRequests bool
// LeavePageOpen keeps pages open after search operations.
LeavePageOpen bool
// WaitLoadTime is an additional fixed wait after load/idle checks.
WaitLoadTime time.Duration
// CaptchaSolverApiKey enables 2Captcha integration for supported engines.
CaptchaSolverApiKey string
// CaptchaSolverEnabled gates solver invocation regardless of engine flags.
CaptchaSolverEnabled bool
// BrowserPath optionally points to a specific browser executable.
BrowserPath string
// ProxyURL defines the upstream proxy for browser traffic.
ProxyURL string
// Insecure allows invalid TLS certificates for browser requests.
Insecure bool
// UserAgent optionally overrides browser-reported user agent during emulation.
UserAgent string
}
// Check applies default option values when optional fields are unset.
func (o *BrowserOpts) Check() {
if o.Timeout == 0 {
o.Timeout = time.Second * 30
}
if o.WaitLoadTime == 0 {
o.WaitLoadTime = time.Second * 2
}
}
// Browser wraps a launched Chromium instance used by engine implementations.
type Browser struct {
BrowserOpts
browserAddr string
conn *browserConnection
CaptchaSolver *CaptchaSolver
}
type browserConnection struct {
mu sync.Mutex
browser *rod.Browser
laneProfiles map[string]browserprofile.Profile
}
// NewBrowser launches a new Chromium process via Rod launcher and returns a
// Browser wrapper configured with proxy and captcha solver settings.
func NewBrowser(opts BrowserOpts) (*Browser, error) {
opts.Check()
if strings.TrimSpace(opts.UserAgent) != "" {
logrus.Warn("custom user_agent override can reduce profile coherence; use only for diagnostics")
}
logrus.WithField("browser_options", fmt.Sprintf("%+v", opts)).Debug("Browser options")
path, err := resolveBrowserBinaryPath(opts.BrowserPath, launcher.LookPath)
if err != nil {
return nil, err
}
// Create launcher.
// headless=new uses the full Chrome renderer; legacy --headless disables the
// GPU process entirely, making WebGL context creation fail even with SwiftShader.
// use-angle=swiftshader-webgl (Chrome ≥112) enables a software WebGL renderer.
l := launcher.New().Leakless(opts.IsLeakless).
Set("disable-blink-features", "AutomationControlled").
Delete("enable-automation").
Set("use-angle", "swiftshader-webgl").
Set("ignore-gpu-blocklist")
if opts.IsHeadless {
l = l.HeadlessNew(true)
} else {
l = l.Headless(false)
}
if path != "" {
logrus.WithField("browser_path", path).Debug("Using browser binary")
l = l.Bin(path)
}
// Configure proxy if specified
if opts.ProxyURL != "" {
normalizedProxyURL, err := NormalizeProxyURL(opts.ProxyURL)
if err != nil {
return nil, fmt.Errorf("invalid proxy URL: %v", err)
}
opts.ProxyURL = normalizedProxyURL
proxyUrl, err := url.Parse(opts.ProxyURL)
if err != nil {
return nil, fmt.Errorf("invalid proxy URL: %v", err)
}
// Chrome's proxy-server flag must not contain credentials.
// Auth (if needed) is handled separately via DevTools auth callbacks.
proxyStr := proxyURLForBrowserLaunch(proxyUrl)
logrus.WithField("proxy", MaskProxyURL(proxyStr)).Debug("Setting up proxy")
l = l.Proxy(proxyStr)
// Check if proxy has auth credentials
if proxyUrl.User != nil {
username := proxyUrl.User.Username()
logrus.WithFields(logrus.Fields{
"proxy_scheme": proxyUrl.Scheme,
"proxy_username": username,
}).Debugf("Proxy credentials configured for %s proxy: %s:****", proxyUrl.Scheme, username)
}
}
b := Browser{
BrowserOpts: opts,
conn: &browserConnection{},
}
b.browserAddr, err = l.Launch()
if opts.CaptchaSolverEnabled && opts.CaptchaSolverApiKey != "" {
b.CaptchaSolver = NewSolver(opts.CaptchaSolverApiKey)
logrus.Debug("Captcha solver initialized")
}
return &b, err
}
func proxyURLForBrowserLaunch(u *url.URL) string {
if u == nil {
return ""
}
clone := *u
// Chrome expects socks5 scheme in --proxy-server; socks5h is not accepted.
if clone.Scheme == "socks5h" {
clone.Scheme = "socks5"
}
clone.User = nil
clone.Path = ""
clone.RawPath = ""
clone.RawQuery = ""
clone.Fragment = ""
return clone.String()
}
func validateBrowserBinaryPath(path string) error {
info, err := os.Stat(path)
if err != nil {
return err
}
if info.IsDir() {
return fmt.Errorf("path points to a directory")
}
return nil
}
// resolveBrowserBinaryPath prefers an explicit browser path. If no explicit path is provided,
// it falls back to launcher autodiscovery and lets Rod handle auto-download when no binary is found.
func resolveBrowserBinaryPath(browserPath string, lookPath func() (string, bool)) (string, error) {
if browserPath != "" {
if err := validateBrowserBinaryPath(browserPath); err != nil {
return "", fmt.Errorf("invalid browser_path %q: %w", browserPath, err)
}
return browserPath, nil
}
path, has := lookPath()
if has {
return path, nil
}
return "", nil
}
// IsInitialized reports whether the browser launcher has been created.
func (b *Browser) IsInitialized() bool {
return b.browserAddr != ""
}
func (b *Browser) connectionState() *browserConnection {
if b.conn == nil {
b.conn = &browserConnection{}
}
return b.conn
}
func (b *Browser) newRodBrowser() *rod.Browser {
browser := rod.New().NoDefaultDevice().ControlURL(b.browserAddr)
if b.Timeout > 0 {
browser = browser.Timeout(b.Timeout)
}
return browser
}
func (b *Browser) connectBrowser() (*rod.Browser, error) {
browser := b.newRodBrowser()
if err := browser.Connect(); err != nil {
return nil, err
}
// Keep cert handling on the persistent browser session.
// Proxy runtime can surface MITM certs, and insecure mode is explicit opt-in.
if b.ProxyURL != "" || b.Insecure {
if err := browser.IgnoreCertErrors(true); err != nil {
return nil, err
}
}
return browser, nil
}
func (b *Browser) ensureConnectedBrowser(ctx context.Context, forceReconnect bool) (*rod.Browser, error) {
if b == nil || b.browserAddr == "" {
return nil, fmt.Errorf("browser is not initialized")
}
state := b.connectionState()
state.mu.Lock()
defer state.mu.Unlock()
if state.browser == nil || forceReconnect {
connected, err := b.connectBrowser()
if err != nil {
return nil, err
}
state.browser = connected
return state.browser, nil
}
if _, err := state.browser.Version(); err != nil {
WithRequest(ctx).WithError(err).Debug("Browser ping failed, reconnecting")
connected, reconnectErr := b.connectBrowser()
if reconnectErr != nil {
return nil, reconnectErr
}
state.browser = connected
}
return state.browser, nil
}
func createIsolatedPage(browser *rod.Browser) (*rod.Page, proto.BrowserBrowserContextID, error) {
browserContext, err := (proto.TargetCreateBrowserContext{}).Call(browser)
if err != nil {
return nil, "", err
}
target, err := (proto.TargetCreateTarget{
URL: "about:blank",
BrowserContextID: browserContext.BrowserContextID,
}).Call(browser)
if err != nil {
_ = disposeBrowserContext(browser, browserContext.BrowserContextID)
return nil, "", err
}
page, err := browser.PageFromTarget(target.TargetID)
if err != nil {
_ = disposeBrowserContext(browser, browserContext.BrowserContextID)
return nil, "", err
}
return page, browserContext.BrowserContextID, nil
}
func disposeBrowserContext(browser *rod.Browser, browserContextID proto.BrowserBrowserContextID) error {
if browser == nil || browserContextID == "" {
return nil
}
return (proto.TargetDisposeBrowserContext{
BrowserContextID: browserContextID,
}).Call(browser)
}
func (b *Browser) startProxyAuthHandler(ctx context.Context, browser *rod.Browser) (context.CancelFunc, error) {
if browser == nil || b.ProxyURL == "" {
return nil, nil
}
proxyURL, err := url.Parse(b.ProxyURL)
if err != nil {
return nil, fmt.Errorf("parse proxy URL: %w", err)
}
if proxyURL.User == nil {
return nil, nil
}
if proxyURL.Scheme != "http" && proxyURL.Scheme != "https" {
if proxyURL.Scheme == "socks5" || proxyURL.Scheme == "socks5h" {
logrus.Debug("SOCKS proxy credentials are not handled by browser auth callback")
}
return nil, nil
}
username := proxyURL.User.Username()
password, _ := proxyURL.User.Password()
authCtx, cancel := context.WithCancel(EnsureContext(ctx))
go func() {
if err := browser.Context(authCtx).HandleAuth(username, password)(); err != nil && !errors.Is(err, context.Canceled) {
WithRequest(ctx).WithError(err).Debug("Proxy auth handler stopped")
}
}()
return cancel, nil
}
var chromeVersionPattern = regexp.MustCompile(`(?:HeadlessChrome|Chrome)/([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)`)
func (b *Browser) laneProfile(ctx context.Context, browser *rod.Browser) (browserprofile.Profile, string) {
engine := engineFromContext(ctx)
region := profileRegionFromContext(ctx)
if region == "" {
region = strings.TrimSpace(b.LanguageCode)
}
laneKey := browserprofile.LaneKey(engine, region)
state := b.connectionState()
state.mu.Lock()
if state.laneProfiles == nil {
state.laneProfiles = make(map[string]browserprofile.Profile)
}
if profile, ok := state.laneProfiles[laneKey]; ok {
state.mu.Unlock()
return profile, laneKey
}
state.mu.Unlock()
// Resolve profile outside the lock: SelectProfile reads from a separate
// RWMutex-guarded catalog, and applyRuntimeBrowserVersion makes a CDP
// round-trip (browser.Version). Holding state.mu over network I/O would
// serialize all concurrent Navigate calls.
profile := browserprofile.SelectProfile(engine, region)
profile = applyRuntimeBrowserVersion(profile, browser)
if overrideUA := strings.TrimSpace(b.UserAgent); overrideUA != "" {
profile.UserAgent = overrideUA
}
state.mu.Lock()
if state.laneProfiles == nil {
state.laneProfiles = make(map[string]browserprofile.Profile)
}
if _, exists := state.laneProfiles[laneKey]; !exists {
state.laneProfiles[laneKey] = profile
} else {
profile = state.laneProfiles[laneKey]
}
state.mu.Unlock()
return profile, laneKey
}
func applyRuntimeBrowserVersion(profile browserprofile.Profile, browser *rod.Browser) browserprofile.Profile {
fullVersion := ""
if browser != nil {
version, err := browser.Version()
if err == nil && version != nil {
fullVersion = extractChromeVersion(version.UserAgent)
if fullVersion == "" {
fullVersion = extractChromeVersion(version.Product)
}
}
}
if fullVersion == "" {
fullVersion = extractChromeVersion(profile.UserAgent)
}
if fullVersion == "" {
return profile
}
major := chromeMajorVersion(fullVersion)
if major == "" {
return profile
}
profile.UserAgent = replaceChromeUserAgentVersion(profile.UserAgent, major+".0.0.0")
profile.UACHBrands = patchBrandVersions(profile.UACHBrands, major, false)
profile.UACHFullVerList = patchBrandVersions(profile.UACHFullVerList, fullVersion, true)
return profile
}
func extractChromeVersion(value string) string {
matches := chromeVersionPattern.FindStringSubmatch(strings.TrimSpace(value))
if len(matches) < 2 {
return ""
}
return strings.TrimSpace(matches[1])
}
func chromeMajorVersion(fullVersion string) string {
fullVersion = strings.TrimSpace(fullVersion)
if fullVersion == "" {
return ""
}
parts := strings.Split(fullVersion, ".")
if len(parts) == 0 {
return ""
}
return strings.TrimSpace(parts[0])
}
func replaceChromeUserAgentVersion(userAgent string, replacement string) string {
replacement = strings.TrimSpace(replacement)
if replacement == "" {
return strings.ReplaceAll(strings.TrimSpace(userAgent), "HeadlessChrome/", "Chrome/")
}
normalized := strings.ReplaceAll(strings.TrimSpace(userAgent), "HeadlessChrome/", "Chrome/")
return chromeVersionPattern.ReplaceAllString(normalized, "Chrome/"+replacement)
}
func patchBrandVersions(values []browserprofile.BrandVersion, version string, patchNotABrand bool) []browserprofile.BrandVersion {
version = strings.TrimSpace(version)
if version == "" {
return values
}
out := make([]browserprofile.BrandVersion, 0, len(values))
for _, value := range values {
item := value
brandLower := strings.ToLower(strings.TrimSpace(item.Brand))
if brandLower == "chromium" || brandLower == "google chrome" {
item.Version = version
} else if patchNotABrand && strings.Contains(brandLower, "not_a brand") && strings.Count(version, ".") == 3 {
item.Version = "24.0.0.0"
}
out = append(out, item)
}
return out
}
func navigatorPlatformForProfile(profile browserprofile.Profile) string {
switch strings.ToLower(strings.TrimSpace(profile.Platform)) {
case "windows":
return "Win32"
case "macos":
return "MacIntel"
case "linux":
return "Linux x86_64"
default:
return strings.TrimSpace(profile.Platform)
}
}
func applyProfile(page *rod.Page, profile browserprofile.Profile) error {
if page == nil {
return fmt.Errorf("page is nil")
}
navigatorLangs := profileNavigatorLanguages(profile)
acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
if acceptLanguage == "" {
acceptLanguage = navigatorLangs[0]
}
locale := strings.TrimSpace(profile.Locale)
if locale == "" {
locale = navigatorLangs[0]
}
width := profile.Viewport.Width
height := profile.Viewport.Height
if width <= 0 {
width = 1920
}
if height <= 0 {
height = 1080
}
metadata := &proto.EmulationUserAgentMetadata{
Brands: toProtoBrandVersions(profile.UACHBrands),
FullVersionList: toProtoBrandVersions(profile.UACHFullVerList),
Platform: strings.TrimSpace(profile.Platform),
PlatformVersion: strings.TrimSpace(profile.PlatformVersion),
Architecture: strings.TrimSpace(profile.Architecture),
Bitness: strings.TrimSpace(profile.Bitness),
Mobile: profile.Mobile,
}
if err := (proto.NetworkSetUserAgentOverride{
UserAgent: strings.TrimSpace(profile.UserAgent),
AcceptLanguage: acceptLanguage,
Platform: navigatorPlatformForProfile(profile),
UserAgentMetadata: metadata,
}).Call(page); err != nil {
return fmt.Errorf("set user agent override failed: %w", err)
}
if err := (proto.EmulationSetLocaleOverride{
Locale: locale,
}).Call(page); err != nil {
return fmt.Errorf("set locale override failed: %w", err)
}
if err := (proto.EmulationSetTimezoneOverride{
TimezoneID: strings.TrimSpace(profile.Timezone),
}).Call(page); err != nil {
return fmt.Errorf("set timezone override failed: %w", err)
}
if err := (proto.EmulationSetDeviceMetricsOverride{
Width: width,
Height: height,
DeviceScaleFactor: 1,
Mobile: profile.Mobile,
ScreenWidth: &width,
ScreenHeight: &height,
}).Call(page); err != nil {
return fmt.Errorf("set device metrics failed: %w", err)
}
if err := (proto.NetworkSetExtraHTTPHeaders{
Headers: proto.NetworkHeaders{
"Accept-Language": gson.New(acceptLanguage),
},
}).Call(page); err != nil {
return fmt.Errorf("set extra headers failed: %w", err)
}
if err := evalPatchScript(page, navigatorLangs, width, height); err != nil {
return err
}
return nil
}
func profileNavigatorLanguages(profile browserprofile.Profile) []string {
langs := make([]string, 0, len(profile.NavigatorLangs))
for _, language := range profile.NavigatorLangs {
trimmed := strings.TrimSpace(language)
if trimmed == "" {
continue
}
langs = append(langs, trimmed)
}
if len(langs) > 0 {
return langs
}
acceptLanguage := strings.TrimSpace(profile.AcceptLanguage)
if acceptLanguage != "" {
parts := strings.Split(acceptLanguage, ",")
langs = make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part == "" {
continue
}
if idx := strings.Index(part, ";"); idx >= 0 {
part = strings.TrimSpace(part[:idx])
}
if part != "" {
langs = append(langs, part)
}
}
if len(langs) > 0 {
return langs
}
}
if locale := strings.TrimSpace(profile.Locale); locale != "" {
return []string{locale}
}
return []string{"en-US"}
}
func toProtoBrandVersions(values []browserprofile.BrandVersion) []*proto.EmulationUserAgentBrandVersion {
out := make([]*proto.EmulationUserAgentBrandVersion, 0, len(values))
for _, value := range values {
brand := strings.TrimSpace(value.Brand)
version := strings.TrimSpace(value.Version)
if brand == "" || version == "" {
continue
}
out = append(out, &proto.EmulationUserAgentBrandVersion{
Brand: brand,
Version: version,
})
}
return out
}
func evalPatchScript(page *rod.Page, langs []string, width, height int) error {
langsJSON, err := json.Marshal(langs)
if err != nil {
return fmt.Errorf("marshal navigator languages: %w", err)
}
args := fmt.Sprintf("const __langs = %s;\nconst __w = %d;\nconst __h = %d;\n",
string(langsJSON), width, height)
_, err = page.EvalOnNewDocument(args + string(browserprofile.PatchJS))
if err != nil {
return fmt.Errorf("eval patch script: %w", err)
}
return nil
}
// Navigate connects to Chromium, creates a page, applies a coherent profile and
// proxy auth, then navigates to URL. It returns an initialized page ready for
// selector queries, or an error when browser setup/navigation fails.
func (b *Browser) Navigate(ctx context.Context, URL string) (*rod.Page, error) {
ctx = EnsureContext(ctx)
if err := ctx.Err(); err != nil {
return nil, err
}
WithRequest(ctx).WithField("url", URL).Debug("Navigate")
browser, err := b.ensureConnectedBrowser(ctx, false)
if err != nil {
return nil, fmt.Errorf("browser connect failed: %w", err)
}
page, browserContextID, err := createIsolatedPage(browser)
if err != nil {
// Single-shot reconnect for stale websocket sessions.
browser, err = b.ensureConnectedBrowser(ctx, true)
if err != nil {
return nil, fmt.Errorf("create isolated page failed, reconnect also failed: %w", err)
}
page, browserContextID, err = createIsolatedPage(browser)
if err != nil {
return nil, fmt.Errorf("create isolated page failed after reconnect: %w", err)
}
}
// closeOnErr closes page then disposes context. Order matters: disposing the
// context first causes Chrome to kill the page target before our Close call,
// producing a spurious "target closed" error on the page.Close() that follows.
closeOnErr := func() {
if cerr := page.Close(); cerr != nil && !isBrowserClosedError(cerr) {
WithRequest(ctx).WithError(cerr).Debug("Close page after navigate error failed")
}
if derr := disposeBrowserContext(browser, browserContextID); derr != nil && !isBrowserClosedError(derr) {
WithRequest(ctx).WithError(derr).Debug("Dispose browser context after navigate error failed")
}
}
cancelProxyAuth, err := b.startProxyAuthHandler(ctx, browser)
if err != nil {
closeOnErr()
return nil, fmt.Errorf("proxy auth setup failed: %w", err)
}
if cancelProxyAuth != nil {
defer cancelProxyAuth()
}
profile, laneKey := b.laneProfile(ctx, browser)
if err := applyProfile(page, profile); err != nil {
closeOnErr()
return nil, fmt.Errorf("apply profile %s (%s) failed: %w", profile.ID, laneKey, err)
}
page = page.Context(ctx)
timedPage := page.Timeout(b.Timeout)
if err := timedPage.Navigate(URL); err != nil {
closeOnErr()
return nil, err
}
// Avoid panics from MustWaitLoad when the target navigates/closes mid-wait
if werr := timedPage.WaitLoad(); werr != nil {
if errors.Is(werr, context.DeadlineExceeded) {
// Some engines keep loading background resources while the DOM is already usable.
// Treat load timeout as non-fatal and let engine-specific selector timeouts decide.
WithRequest(ctx).WithField("timeout", b.Timeout.String()).Debug(
fmt.Sprintf("WaitLoad timed out after %s; continuing with partial page state", b.Timeout),
)
} else {
WithRequest(ctx).WithError(werr).Debug("WaitLoad returned early")
}
}
// may cause bugs with google
if b.WaitRequests {
wait := timedPage.WaitRequestIdle(300*time.Millisecond, nil, nil, nil)
wait()
}
if err := SleepContext(ctx, b.WaitLoadTime); err != nil {
closeOnErr()
return nil, err
}
return page, nil
}
// Close closes the active browser connection.
func (b *Browser) Close() error {
if b == nil || b.browserAddr == "" {
return nil
}
state := b.connectionState()
state.mu.Lock()
defer state.mu.Unlock()
browser := state.browser
if browser == nil {
browser = b.newRodBrowser()
if err := browser.Connect(); err != nil {
if isBrowserClosedError(err) {
return nil
}
return err
}
}
state.browser = nil
state.laneProfiles = nil
if err := browser.Close(); err != nil && !isBrowserClosedError(err) {
return err
}
return nil
}
func isBrowserClosedError(err error) bool {
if err == nil {
return false
}
msg := strings.ToLower(err.Error())
return strings.Contains(msg, "connection refused") ||
strings.Contains(msg, "closed network connection") ||
strings.Contains(msg, "target closed") ||
strings.Contains(msg, "eof")
}
// ClosePageWithTimeout bounds page close calls so shutdown paths don't hang.
func ClosePageWithTimeout(ctx context.Context, page *rod.Page, timeout time.Duration) error {
if page == nil {
return nil
}
if timeout <= 0 {
timeout = time.Second
}
baseCtx := EnsureContext(ctx)
if baseCtx.Err() != nil {
baseCtx = context.Background()
}
closeCtx, cancel := context.WithTimeout(baseCtx, timeout)
defer cancel()
pageWithTimeout := page.Context(closeCtx)
info, _ := pageWithTimeout.Info()
if err := pageWithTimeout.Close(); err != nil && !isBrowserClosedError(err) {
return err
}
if info != nil && info.BrowserContextID != "" {
if derr := disposeBrowserContext(page.Browser().Context(closeCtx), info.BrowserContextID); derr != nil && !isBrowserClosedError(derr) {
return derr
}
}
return nil
}
// RecoverEnginePanic converts recovered panics to a typed engine error and
// logs stack trace with engine context.
func RecoverEnginePanic(engine string, recovered interface{}, logger *EngineLogger) error {
return RecoverEnginePanicWithContext(context.TODO(), engine, recovered, logger)
}
func RecoverEnginePanicWithContext(ctx context.Context, engine string, recovered interface{}, logger *EngineLogger) error {
stack := debug.Stack()
if logger != nil {
logger.Error("Recovered panic in %s Search: panic=%v\n%s", engine, recovered, string(stack))
} else {
WithRequestEngine(ctx, engine).Errorf("Recovered panic in %s Search: panic=%v\n%s", engine, recovered, string(stack))
}
return fmt.Errorf("%w: %s", ErrEngineInternal, engine)
}
// IsRodObjectNotFound reports element/object lookup misses across rod error
// variants used by selector calls.
func IsRodObjectNotFound(err error) bool {
if err == nil {
return false
}
var objectErr *rod.ObjectNotFoundError
if errors.As(err, &objectErr) {
return true
}
var elementErr *rod.ElementNotFoundError
return errors.As(err, &elementErr)
}