diff --git a/README.md b/README.md index 30363de..758186a 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Run it locally, self-host it, or use the optional hosted API when you do not wan - 🖼 **Images** - image search is also available - 🎯 **Advanced filters** - language, date range, file type, and site queries - ✨ **SERP features** - AI summaries, answer boxes, people-also-ask, and related searches in a response +- 📄 **URL extraction** - turn target pages into clean markdown/text for grounding and automation - 🌍 **Configurable** - proxy, cache, and resilient mode - 🐳 **Docker-ready** - local and container deployment - 📝 **Data Formats** - JSON, Markdown, Text, NdJSON response formats @@ -112,21 +113,37 @@ List engines: curl "http://127.0.0.1:7000/mega/engines" ``` +URL extraction: + +```bash +# Extract one URL as JSON +curl "http://127.0.0.1:7000/extract?url=https://example.com&mode=auto" + +# Return clean page markdown +curl "http://127.0.0.1:7000/extract?url=https://example.com&format=markdown" + +# Embed extracted content under the top search results +curl "http://127.0.0.1:7000/google/search?text=llm+observability&extract=true&extract_top=2&format=markdown" +``` + ## 🔍 Query Parameters Common parameters: -| Parameter | Description | Example | -| --------- | ------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------ | -| `text` | Search query | `golang programming` | -| `lang` | Language code | `EN`, `DE`, `RU`, `ES` | -| `region` | Market/location hint. Countries/locales work across engines; Google also accepts city names via `uule`; Yandex accepts numeric `lr`. | `DE`, `en-GB`, `Berlin`, `213` | -| `date` | Date range | `20250101..20251231` | -| `file` | File extension | `pdf`, `doc`, `xls` | -| `site` | Site-specific search | `github.com` | -| `limit` | Number of organic results, max 100. When omitted or `<=10`, only the first SERP page is parsed. | `25`, `50` | -| `start` | Pagination offset | `0`, `10`, `20` | -| `format` | Output format | `json`, `markdown`, `text`, `ndjson` | +| Parameter | Description | Example | +| -------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------ | +| `text` | Search query | `golang programming` | +| `lang` | Language code | `EN`, `DE`, `RU`, `ES` | +| `region` | Market/location hint. Countries/locales work across engines; Google also accepts city names via `uule`; Yandex accepts numeric `lr`. | `DE`, `en-GB`, `Berlin`, `213` | +| `date` | Date range | `20250101..20251231` | +| `file` | File extension | `pdf`, `doc`, `xls` | +| `site` | Site-specific search | `github.com` | +| `limit` | Number of organic results, max 100. When omitted or `<=10`, only the first SERP page is parsed. | `25`, `50` | +| `start` | Pagination offset | `0`, `10`, `20` | +| `format` | Output format | `json`, `markdown`, `text`, `ndjson` | +| `extract` | Fetch and embed target-page content for top web results | `true` | +| `extract_top` | Number of top web results to extract, clamped to 1-5 | `3` | +| `extract_mode` | Extraction strategy: raw HTTP first, raw only, or browser-rendered | `auto`, `fast`, `rendered` | Engine-specific parameters: diff --git a/cmd/root.go b/cmd/root.go index 6420723..1b92950 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,6 +9,7 @@ import ( "github.com/karust/openserp/core" browserprofile "github.com/karust/openserp/core/browser" + extractpkg "github.com/karust/openserp/extract" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -16,7 +17,7 @@ import ( ) const ( - version = "0.7.16" + version = "0.8.0" defaultConfigFilename = "config" envPrefix = "OPENSERP" ) @@ -26,6 +27,7 @@ type Config struct { App AppConfig `mapstructure:"app"` Proxies core.ProxiesConfig `mapstructure:"proxies"` Cache CacheConfig `mapstructure:"cache"` + Extract extractpkg.Config `mapstructure:"extract"` Resilience ResilienceConfig `mapstructure:"resilience"` CircuitBreaker CircuitBreakerConfig `mapstructure:"circuit_breaker"` CORS CORSConfig `mapstructure:"cors"` @@ -184,6 +186,7 @@ func sanitizedConfigForLog(cfg Config) map[string]interface{} { "lanes": cfg.Proxies.Lanes, }, "cache": cfg.Cache, + "extract": cfg.Extract, "resilience": cfg.Resilience, "circuit_breaker": cfg.CircuitBreaker, "cors": cfg.CORS, @@ -404,6 +407,11 @@ func setConfigDefaults(v *viper.Viper) { v.SetDefault("cache.ttl_seconds", 300) v.SetDefault("cache.max_size", 1000) + v.SetDefault("extract.enabled", true) + v.SetDefault("extract.default_mode", "auto") + v.SetDefault("extract.timeout", "20s") + v.SetDefault("extract.max_bytes", 2*1024*1024) + v.SetDefault("extract.max_concurrent", 2) // Keep stage2 defaults stable even when config file is absent. v.SetDefault("resilience.max_retries", 3) v.SetDefault("resilience.allow_endpoint_fallback", false) diff --git a/cmd/serve.go b/cmd/serve.go index ba09b0f..a58817d 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -118,13 +118,14 @@ func serve(cmd *cobra.Command, args []string) { baseOpts.CaptchaSolverEnabled = captchaSolverEnabled baseOpts.CaptchaSolverApiKey = captchaSolverAPIKey - engines, closeBrowsers, err := buildBrowserEngines(baseOpts, proxyCfg) + engines, closeBrowsers, browserResolver, err := buildBrowserEngines(baseOpts, proxyCfg) if err != nil { logrus.Error(err) return } serverOpts := buildServerOptions(corsCfg, proxyCfg, fingerprintBrowserOpts) + serverOpts.BrowserResolver = browserResolver serv := core.NewServerWithOptions(config.Server.Host, config.Server.Port, serverOpts, engines...) if err := listenWithGracefulShutdown(serv, closeBrowsers); err != nil { logrus.Error(err) @@ -160,6 +161,7 @@ func buildServerOptions(corsCfg core.CORSConfig, proxyCfg core.ProxyConfig, fing FingerprintArtifactDir: core.DefaultFingerprintArtifactDir, FingerprintBrowserOpts: fingerprintBrowserOpts, MegaTimeout: config.App.MegaTimeout, + Extract: config.Extract, Resilience: core.ResilientConfig{ Retry: core.RetryConfig{ MaxRetries: config.Resilience.MaxRetries, @@ -666,7 +668,7 @@ func browserEngineSpecs() []browserEngineSpec { } } -func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ([]core.SearchEngine, func() error, error) { +func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ([]core.SearchEngine, func() error, core.BrowserResolver, error) { launchProxyURL := "" if strings.TrimSpace(proxyCfg.Proxies.Global) != "" && !proxyCfg.Proxies.AllowRequestProxyURL { launchProxyURL = proxyCfg.Proxies.Global @@ -690,7 +692,7 @@ func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ( for idx, spec := range specs { policy := resolveEngineProxyPolicy(proxyCfg, spec.name) if err := validateBrowserProxyPolicy(proxyCfg, policy); err != nil { - return nil, nil, fmt.Errorf("browser proxy validation failed for engine %s: %w", spec.name, err) + return nil, nil, nil, fmt.Errorf("browser proxy validation failed for engine %s: %w", spec.name, err) } opts := spec.opts @@ -710,7 +712,7 @@ func buildBrowserEngines(baseOpts core.BrowserOpts, proxyCfg core.ProxyConfig) ( } } - return engines, pool.close, nil + return engines, pool.close, pool.get, nil } func validateBrowserProxyPolicy(proxyCfg core.ProxyConfig, policy core.ProxyPolicy) error { diff --git a/config.yaml b/config.yaml index 1ebfaa3..bac3be4 100644 --- a/config.yaml +++ b/config.yaml @@ -18,6 +18,14 @@ app: idle_ttl: 5m # close a Chrome that has not served traffic for this long mega_timeout: 90s # max total wait for /mega/* requests; slow engines return partial results + +extract: + enabled: true + default_mode: auto # auto|fast|rendered + timeout: 20s + max_bytes: 2000000 + max_concurrent: 2 + proxies: allow_request_proxy_url: false # Force a single proxy for all engines. diff --git a/core/common.go b/core/common.go index 2f71269..f56e81c 100644 --- a/core/common.go +++ b/core/common.go @@ -271,6 +271,14 @@ type Query struct { // supported by the engine. Such entries may be returned with non-positive // internal rank values. Features bool + // Extract fetches and embeds cleaned target-page content for top results. + Extract bool + // ExtractTop limits how many top results are enriched when Extract is true. + ExtractTop int + // ExtractMode selects auto, fast, or rendered extraction. + ExtractMode string + // ExtractMinRunes overrides the auto-mode escalation floor (0 = default). + ExtractMinRunes int // ProxyURL is a direct proxy URL used by raw HTTP search paths. ProxyURL string // ProxyCountry identifies the proxy market country for cache/error metadata. @@ -297,9 +305,9 @@ func (q Query) String() string { maskedProxyURL = MaskProxyURL(q.ProxyURL) } return fmt.Sprintf( - "{Text:%s LangCode:%s Region:%s DateInterval:%s Filetype:%s Site:%s Limit:%d Start:%d Filter:%t Features:%t ProxyURL:%s ProxyCountry:%s ProxyClass:%s ProxyProvider:%s ProxySessionID:%s ProxyOverride:%s Insecure:%t}", + "{Text:%s LangCode:%s Region:%s DateInterval:%s Filetype:%s Site:%s Limit:%d Start:%d Filter:%t Features:%t Extract:%t ExtractTop:%d ExtractMode:%s ProxyURL:%s ProxyCountry:%s ProxyClass:%s ProxyProvider:%s ProxySessionID:%s ProxyOverride:%s Insecure:%t}", q.Text, q.LangCode, q.Region, q.DateInterval, q.Filetype, q.Site, - q.Limit, q.Start, q.Filter, q.Features, + q.Limit, q.Start, q.Filter, q.Features, q.Extract, q.ExtractTop, q.ExtractMode, maskedProxyURL, q.ProxyCountry, q.ProxyClass, q.ProxyProvider, q.ProxySessionID, q.ProxyOverride, q.Insecure, ) @@ -373,6 +381,34 @@ func (searchQuery *Query) InitFromContext(reqCtx *fiber.Ctx) error { if err != nil { return errInvalidParam(fmt.Sprintf("features: %v", err)) } + searchQuery.Extract, err = strconv.ParseBool(reqCtx.Query("extract", "0")) + if err != nil { + return errInvalidParam(fmt.Sprintf("extract: %v", err)) + } + searchQuery.ExtractTop = 3 + if raw := strings.TrimSpace(reqCtx.Query("extract_top")); raw != "" { + extractTop, err := strconv.Atoi(raw) + if err != nil { + return errInvalidParam("extract_top must be an integer") + } + if extractTop < 1 { + extractTop = 1 + } + if extractTop > 5 { + extractTop = 5 + } + searchQuery.ExtractTop = extractTop + } + searchQuery.ExtractMode = strings.ToLower(strings.TrimSpace(reqCtx.Query("extract_mode", "auto"))) + switch searchQuery.ExtractMode { + case "auto", "fast", "rendered": + default: + return errInvalidParam("extract_mode must be one of auto, fast, rendered") + } + searchQuery.ExtractMinRunes, err = parseNonNegativeIntQuery(reqCtx.Query("min_runes"), 0) + if err != nil { + return errInvalidParam("min_runes must be a non-negative integer") + } searchQuery.ProxyOverride, err = NormalizeProxyRequestOverride(reqCtx.Get("X-Use-Proxy")) if err != nil { diff --git a/core/format_markdown.go b/core/format_markdown.go index bfea18f..bf9d4bd 100644 --- a/core/format_markdown.go +++ b/core/format_markdown.go @@ -32,6 +32,11 @@ func RenderMarkdown(env *Envelope) []byte { fmt.Fprintf(&b, "%s\n\n", r.Snippet) } fmt.Fprintf(&b, "-> %s\n\n", r.URL) + if r.Extracted != nil && r.Extracted.Content != "" { + b.WriteString("#### Extracted content\n\n") + b.WriteString(shiftMarkdownHeadings(r.Extracted.Content, 4)) + b.WriteString("\n\n") + } } renderMarkdownFeatures(&b, env.SerpFeatures, featureRenderOrderAfterResults(env.SerpFeatures)) @@ -39,6 +44,30 @@ func RenderMarkdown(env *Envelope) []byte { return []byte(b.String()) } +func shiftMarkdownHeadings(markdown string, minLevel int) string { + lines := strings.Split(markdown, "\n") + for i, line := range lines { + trimmed := strings.TrimLeft(line, " ") + indent := line[:len(line)-len(trimmed)] + if !strings.HasPrefix(trimmed, "#") { + continue + } + count := 0 + for count < len(trimmed) && trimmed[count] == '#' { + count++ + } + if count == 0 || count >= len(trimmed) || trimmed[count] != ' ' { + continue + } + target := count + minLevel + if target > 6 { + target = 6 + } + lines[i] = indent + strings.Repeat("#", target) + trimmed[count:] + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} + // RenderMarkdownImage formats an ImageEnvelope as Markdown. func RenderMarkdownImage(env *ImageEnvelope) []byte { var b strings.Builder diff --git a/core/format_text.go b/core/format_text.go index 3adf6ff..868d0ab 100644 --- a/core/format_text.go +++ b/core/format_text.go @@ -32,6 +32,11 @@ func RenderText(env *Envelope) []byte { fmt.Fprintf(&b, "%s\n", r.Snippet) } fmt.Fprintf(&b, "URL: %s\n\n", r.URL) + if r.Extracted != nil && r.Extracted.Content != "" { + b.WriteString("Extracted content:\n") + b.WriteString(r.Extracted.Content) + b.WriteString("\n\n") + } } renderTextFeatures(&b, env.SerpFeatures, featureRenderOrderAfterResults(env.SerpFeatures)) diff --git a/core/result.go b/core/result.go index 1fd2346..7ff033e 100644 --- a/core/result.go +++ b/core/result.go @@ -49,6 +49,18 @@ type Classification struct { SourceHint string `json:"source_hint,omitempty"` } +// ExtractedContent carries one enriched target page. Content holds a single +// representation chosen by the response format (markdown for json/ndjson/markdown, +// plain text for text), with Format naming which it is — no duplicated blobs. +type ExtractedContent struct { + Title string `json:"title,omitempty"` + Format string `json:"format,omitempty"` + Content string `json:"content,omitempty"` + ModeUsed string `json:"mode_used,omitempty"` + FetchedAt string `json:"fetched_at,omitempty"` + Error string `json:"error,omitempty"` +} + // FeatureItem is one child entry inside a grouped SERP feature. type FeatureItem struct { Title string `json:"title,omitempty"` @@ -81,19 +93,20 @@ type SerpFeature struct { // Result is the v2 normalized result returned in search responses. Optional // fields (Position, DomainInfo, Classification) are omitted when empty. type Result struct { - ID string `json:"id"` - Rank int `json:"rank"` - Type ResultType `json:"type"` - Title string `json:"title"` - URL string `json:"url"` - DisplayURL string `json:"display_url"` - Snippet string `json:"snippet"` - Domain string `json:"domain"` - Favicon string `json:"favicon"` - Position *Position `json:"position,omitempty"` - Engine string `json:"engine"` - DomainInfo *DomainInfo `json:"domain_info,omitempty"` - Classification *Classification `json:"classification,omitempty"` + ID string `json:"id"` + Rank int `json:"rank"` + Type ResultType `json:"type"` + Title string `json:"title"` + URL string `json:"url"` + DisplayURL string `json:"display_url"` + Snippet string `json:"snippet"` + Domain string `json:"domain"` + Favicon string `json:"favicon"` + Position *Position `json:"position,omitempty"` + Engine string `json:"engine"` + DomainInfo *DomainInfo `json:"domain_info,omitempty"` + Classification *Classification `json:"classification,omitempty"` + Extracted *ExtractedContent `json:"extracted,omitempty"` } // ImageData holds image-specific URL and dimension fields. diff --git a/core/server.go b/core/server.go index 307d18b..c4dec82 100644 --- a/core/server.go +++ b/core/server.go @@ -22,6 +22,7 @@ import ( "github.com/karust/openserp/core/fpcheck" "github.com/karust/openserp/core/fpcheck/detectors" apidocs "github.com/karust/openserp/docs" + extractpkg "github.com/karust/openserp/extract" "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) @@ -88,8 +89,14 @@ type ServerOptions struct { // as failed with a context-deadline error. Zero disables the bound // (legacy behavior — wait until the slowest engine finishes). MegaTimeout time.Duration + // BrowserResolver returns a pooled browser for rendered extraction. + BrowserResolver BrowserResolver + // Extract configures the URL extraction endpoint and search enrichment. + Extract extractpkg.Config } +type BrowserResolver func(proxyURL string) (*Browser, error) + // DefaultServerOptions returns production-oriented defaults for cache, CORS, // and resilient search policies. func DefaultServerOptions() ServerOptions { @@ -107,6 +114,7 @@ func DefaultServerOptions() ServerOptions { }, Resilience: DefaultResilientConfig(), MegaTimeout: 90 * time.Second, + Extract: extractpkg.DefaultConfig(), } } @@ -204,6 +212,8 @@ func NewServerWithOptions(host string, port int, opts ServerOptions, searchEngin serv.app.Get("/mega/search", serv.handleMegaSearch) serv.app.Get("/mega/image", serv.handleMegaImage) serv.app.Get("/mega/engines", serv.handleListEngines) + serv.app.Get("/extract", serv.handleExtract) + serv.app.Post("/extract", serv.handleExtract) return &serv } @@ -251,7 +261,7 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm WithField("action", action). Debugf("Starting %s request for query: %s", action, q.Text) - if format == "json" && !ShouldBypassCacheForProxyMarket(q) { + if format == "json" && !q.Extract && !ShouldBypassCacheForProxyMarket(q) { if hit, err := s.tryServeCacheHit( c, startedAt, @@ -335,7 +345,11 @@ func (s *Server) handleDedicatedEndpoint(c *fiber.Ctx, engine SearchEngine, isIm } env.Finalize(startedAt, q) - if format == "json" { + if q.Extract { + s.enrichEnvelopeWithExtraction(requestCtx, env, q, format) + } + + if format == "json" && !q.Extract { cacheStatus := s.cacheEnvelopeIfEligible(engine.Name(), usedEngine, action, q, env) if cacheStatus != "" { c.Set("X-Cache", cacheStatus) @@ -928,7 +942,7 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string) error { "mode": runCfg.Mode, }).Debugf("Starting mega %s request for query: %s", action, q.Text) - if format == "json" && !ShouldBypassCacheForProxyMarket(q) && runCfg.Mode != megaModeFast { + if format == "json" && !q.Extract && !ShouldBypassCacheForProxyMarket(q) && runCfg.Mode != megaModeFast { cacheHitCandidates := []cacheHitCandidate{ { key: s.buildMegaCacheKey(action, enginesToUse, q, runCfg), @@ -1024,6 +1038,9 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string) error { AppendEnrichedSearchResult(env, r.SearchResult, ectx, startedAt) } env.Finalize(startedAt, q) + if q.Extract { + s.enrichEnvelopeWithExtraction(requestCtx, env, q, format) + } if runCfg.Merge { allEnriched := make([]Result, 0, len(rawResults)) @@ -1037,7 +1054,7 @@ func (s *Server) handleMegaEndpoint(c *fiber.Ctx, action string) error { } } - if format == "json" && s.cache != nil && runCfg.Mode != megaModeFast { + if format == "json" && s.cache != nil && !q.Extract && runCfg.Mode != megaModeFast { c.Set("X-Cache", s.cacheMegaEnvelopeResults(action, enginesToUse, q, env, runCfg)) } diff --git a/core/server_extract.go b/core/server_extract.go new file mode 100644 index 0000000..e94be94 --- /dev/null +++ b/core/server_extract.go @@ -0,0 +1,290 @@ +package core + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" + + "github.com/gofiber/fiber/v2" + extractpkg "github.com/karust/openserp/extract" +) + +type extractPayload struct { + URL string `json:"url"` + Mode string `json:"mode"` + // Clean defaults to true (article-only). Pointer so we can tell "omitted" + // (use default) from an explicit false (full-page extraction). + Clean *bool `json:"clean"` + UseLLMSTxt bool `json:"use_llms_txt"` + MinRunes int `json:"min_runes"` +} + +func (s *Server) handleExtract(c *fiber.Ctx) error { + startedAt := time.Now() + requestCtx := withRequestUsage(c.UserContext(), "extract") + c.SetUserContext(requestCtx) + defer setNetworkBytesHeader(c, requestCtx) + defer setBrowserProfileHeader(c, requestCtx) + + cfg := s.opts.Extract.Normalized() + if !cfg.Enabled { + return &APIError{HTTPStatus: fiber.StatusNotFound, ErrorCode: "not_found", Message: "Extraction is disabled"} + } + format, err := resolveFormat(c) + if err != nil { + return err + } + req, err := s.extractRequestFromFiber(c, cfg) + if err != nil { + return err + } + extractor := s.newExtractor() + result, err := extractor.Extract(requestCtx, req) + if err != nil { + WithRequest(requestCtx).WithError(err).Warn("Extract failed") + return &APIError{HTTPStatus: fiber.StatusBadGateway, ErrorCode: "extract_failed", Message: "Failed to extract URL content"} + } + result.Meta.TookMs = time.Since(startedAt).Milliseconds() + return sendExtractResult(c, format, result) +} + +func (s *Server) extractRequestFromFiber(c *fiber.Ctx, cfg extractpkg.Config) (extractpkg.ExtractRequest, error) { + var body extractPayload + if len(c.Body()) > 0 { + _ = c.BodyParser(&body) + } + proxyOverride, err := NormalizeProxyRequestOverride(c.Get("X-Use-Proxy")) + if err != nil { + return extractpkg.ExtractRequest{}, errInvalidParam(fmt.Sprintf("X-Use-Proxy: %v", err)) + } + proxyURL := strings.TrimSpace(c.Get("X-Proxy-URL")) + if proxyURL != "" { + normalized, err := NormalizeProxyURL(proxyURL) + if err != nil { + return extractpkg.ExtractRequest{}, errInvalidParam(fmt.Sprintf("X-Proxy-URL: %v", err)) + } + proxyURL = normalized + } + q := Query{ProxyURL: proxyURL, ProxyOverride: proxyOverride} + if err := s.validateRequestProxyURL(&q); err != nil { + return extractpkg.ExtractRequest{}, err + } + mode := firstNonEmpty(c.Query("mode"), body.Mode, cfg.DefaultMode) + // Default clean=true (article-only). FullPage is the inverse: full-readable-body + // extraction, opted in via clean=false on the query string or body. + bodyClean := true + if body.Clean != nil { + bodyClean = *body.Clean + } + clean := parseBoolDefault(c.Query("clean"), bodyClean) + minRunes, err := parseNonNegativeIntQuery(c.Query("min_runes"), body.MinRunes) + if err != nil { + return extractpkg.ExtractRequest{}, errInvalidParam("min_runes must be a non-negative integer") + } + return extractpkg.ExtractRequest{ + URL: firstNonEmpty(c.Query("url"), body.URL), + Mode: extractpkg.Mode(mode), + ProxyURL: proxyURL, + LangCode: strings.TrimSpace(c.Query("lang")), + Timeout: cfg.Timeout, + MaxBytes: cfg.MaxBytes, + FullPage: !clean, + UseLLMSTxt: parseBoolDefault(c.Query("use_llms_txt"), body.UseLLMSTxt), + MinRunes: minRunes, + }, nil +} + +func (s *Server) newExtractor() extractpkg.Extractor { + return extractpkg.Extractor{ + RawFetch: s.rawExtractFetch, + RenderedFetch: s.renderedExtractFetch, + Cfg: s.opts.Extract, + } +} + +func (s *Server) rawExtractFetch(ctx context.Context, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) { + resp, err := RawSearchRequest(ctx, req.URL, Query{ + ProxyURL: req.ProxyURL, + LangCode: req.LangCode, + Insecure: s.opts.FingerprintBrowserOpts.Insecure, + }) + if err != nil { + return nil, err + } + defer DrainAndCloseResponse(resp) + if err := ClassifySearchHTTPStatus(resp.StatusCode); err != nil { + return nil, err + } + limit := int64(req.MaxBytes) + if limit <= 0 { + limit = int64(s.opts.Extract.Normalized().MaxBytes) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil { + return nil, err + } + if int64(len(body)) > limit { + body = body[:limit] + } + return &extractpkg.FetchResponse{StatusCode: resp.StatusCode, Body: body}, nil +} + +func (s *Server) renderedExtractFetch(ctx context.Context, req extractpkg.ExtractRequest) (*extractpkg.FetchResponse, error) { + if s.opts.BrowserResolver == nil { + return nil, fmt.Errorf("rendered extraction is unavailable") + } + browser, err := s.opts.BrowserResolver(req.ProxyURL) + if err != nil { + return nil, err + } + page, err := browser.Navigate(WithRequestProxyURL(ctx, req.ProxyURL), req.URL) + if err != nil { + return nil, err + } + defer func() { + _ = browser.ClosePage(ctx, page, time.Second) + }() + html, err := page.HTML() + if err != nil { + return nil, err + } + body := []byte(html) + if req.MaxBytes > 0 && len(body) > req.MaxBytes { + body = body[:req.MaxBytes] + } + return &extractpkg.FetchResponse{StatusCode: http.StatusOK, Body: body}, nil +} + +func (s *Server) enrichEnvelopeWithExtraction(ctx context.Context, env *Envelope, q Query, format string) { + cfg := s.opts.Extract.Normalized() + if env == nil || !q.Extract || !cfg.Enabled { + return + } + // One representation per result, chosen by the response format: plain text for + // format=text, markdown for everything else (json/ndjson/markdown). This keeps + // the format-specific renderers fed without serializing two near-identical blobs. + contentFormat := "markdown" + if format == "text" { + contentFormat = "text" + } + extractor := s.newExtractor() + limit := q.ExtractTop + if limit <= 0 || limit > 5 { + limit = 3 + } + if limit > len(env.Results) { + limit = len(env.Results) + } + + // Per-fetch timeouts bound a single URL; this aggregate deadline bounds the + // whole batch so a few slow/hanging targets can't stretch the search request + // open-endedly. The ceiling is derived from the per-URL budget (see + // Config.BatchTimeout) rather than a separate knob. When it fires, in-flight + // fetches are cancelled and any not yet started record a timeout error instead + // of a result — never a 500. + ctx, cancel := context.WithTimeout(ctx, cfg.BatchTimeout(limit)) + defer cancel() + + sem := make(chan struct{}, cfg.MaxConcurrent) + var wg sync.WaitGroup + for i := 0; i < limit; i++ { + if strings.TrimSpace(env.Results[i].URL) == "" { + continue + } + wg.Add(1) + sem <- struct{}{} + go func(idx int) { + defer wg.Done() + defer func() { <-sem }() + // Skip the fetch entirely if the batch budget is already spent. + if err := ctx.Err(); err != nil { + env.Results[idx].Extracted = &ExtractedContent{Error: sanitizeExtractError(err)} + return + } + req := extractpkg.ExtractRequest{ + URL: env.Results[idx].URL, + Mode: extractpkg.Mode(q.ExtractMode), + ProxyURL: q.ProxyURL, + LangCode: q.LangCode, + Timeout: cfg.Timeout, + MaxBytes: cfg.MaxBytes, + MinRunes: q.ExtractMinRunes, + } + result, err := extractor.Extract(ctx, req) + if err != nil { + env.Results[idx].Extracted = &ExtractedContent{Error: sanitizeExtractError(err)} + return + } + content := result.Markdown + if contentFormat == "text" { + content = result.Text + } + env.Results[idx].Extracted = &ExtractedContent{ + Title: result.Title, + Format: contentFormat, + Content: content, + ModeUsed: result.Meta.ModeUsed, + FetchedAt: result.Meta.FetchedAt, + } + }(i) + } + wg.Wait() +} + +func sendExtractResult(c *fiber.Ctx, format string, result *extractpkg.ExtractResult) error { + switch format { + case "json": + return c.JSON(result) + case "markdown": + c.Set("Content-Type", "text/markdown; charset=utf-8") + var b strings.Builder + if result.Title != "" { + fmt.Fprintf(&b, "# %s\n\n", result.Title) + } + if result.URL != "" { + fmt.Fprintf(&b, "<%s>\n\n", result.URL) + } + b.WriteString(result.Markdown) + b.WriteString("\n") + return c.SendString(b.String()) + case "text": + c.Set("Content-Type", "text/plain; charset=utf-8") + return c.SendString(result.Text + "\n") + case "ndjson": + c.Set("Content-Type", "application/x-ndjson; charset=utf-8") + data, err := json.Marshal(map[string]any{"kind": "extract", "result": result}) + if err != nil { + return err + } + return c.Send(append(data, '\n')) + default: + return errInvalidParam("format must be one of json, markdown, text, ndjson") + } +} + +func parseBoolDefault(raw string, fallback bool) bool { + raw = strings.TrimSpace(raw) + if raw == "" { + return fallback + } + return raw == "1" || strings.EqualFold(raw, "true") || strings.EqualFold(raw, "yes") +} + +func sanitizeExtractError(err error) string { + if err == nil { + return "" + } + msg := strings.TrimSpace(err.Error()) + if msg == "" { + return "extract failed" + } + if len(msg) > 180 { + msg = msg[:180] + } + return msg +} diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 958d64a..c68427b 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: OpenSERP API - version: 2.1.0 + version: 2.2.0 description: > OpenSERP provides dedicated and multi-engine search endpoints for Google, Yandex, Baidu, Bing, and DuckDuckGo. Search responses are wrapped in a v2 envelope with @@ -47,6 +47,10 @@ paths: - $ref: "#/components/parameters/StartQuery" - $ref: "#/components/parameters/FilterQuery" - $ref: "#/components/parameters/FeaturesQuery" + - $ref: "#/components/parameters/ExtractQuery" + - $ref: "#/components/parameters/ExtractTopQuery" + - $ref: "#/components/parameters/ExtractModeQuery" + - $ref: "#/components/parameters/MinRunesQuery" - $ref: "#/components/parameters/FormatQuery" - $ref: "#/components/parameters/UseProxyHeader" - $ref: "#/components/parameters/ProxyURLHeader" @@ -325,6 +329,10 @@ paths: - $ref: "#/components/parameters/MegaModeQuery" - $ref: "#/components/parameters/MegaDedupeQuery" - $ref: "#/components/parameters/MegaMergeQuery" + - $ref: "#/components/parameters/ExtractQuery" + - $ref: "#/components/parameters/ExtractTopQuery" + - $ref: "#/components/parameters/ExtractModeQuery" + - $ref: "#/components/parameters/MinRunesQuery" - $ref: "#/components/parameters/FormatQuery" - $ref: "#/components/parameters/UseProxyHeader" - $ref: "#/components/parameters/ProxyURLHeader" @@ -454,6 +462,93 @@ paths: application/json: schema: $ref: "#/components/schemas/MegaEnginesResponse" + /extract: + get: + tags: [Search] + operationId: extractURL + summary: Extract clean content from one URL + parameters: + - $ref: "#/components/parameters/URLQuery" + - $ref: "#/components/parameters/ExtractModeShortQuery" + - $ref: "#/components/parameters/MinRunesQuery" + - $ref: "#/components/parameters/LangQuery" + - $ref: "#/components/parameters/CleanQuery" + - $ref: "#/components/parameters/UseLLMSTxtQuery" + - $ref: "#/components/parameters/FormatQuery" + - $ref: "#/components/parameters/UseProxyHeader" + - $ref: "#/components/parameters/ProxyURLHeader" + responses: + "200": + description: Extracted URL content + content: + application/json: + schema: + $ref: "#/components/schemas/ExtractResult" + text/markdown: + schema: + type: string + text/plain: + schema: + type: string + application/x-ndjson: + schema: + type: string + "400": + $ref: "#/components/responses/BadRequestError" + "502": + $ref: "#/components/responses/BadGatewayError" + post: + tags: [Search] + operationId: extractURLPost + summary: Extract clean content from one URL + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [url] + properties: + url: + type: string + format: uri + mode: + type: string + enum: [auto, fast, rendered] + default: auto + clean: + type: boolean + default: true + description: > + Article-only extraction (default). Set `false` for + whole-readable-body extraction that keeps nav/feature/landing + content trafilatura would otherwise strip. + use_llms_txt: + type: boolean + default: false + description: > + When the URL is a site root, probe `/llms-full.txt` then + `/llms.txt` and return that LLM-optimized markdown instead of + scraping HTML. Falls through to normal extraction when absent. + min_runes: + type: integer + minimum: 0 + description: > + Auto-mode escalation floor: if the fast (raw) pass yields fewer + extracted-text runes than this, escalate to a browser render. + `0` (default) uses the built-in floor. Ignored in `fast` and + `rendered` modes. + responses: + "200": + description: Extracted URL content + content: + application/json: + schema: + $ref: "#/components/schemas/ExtractResult" + "400": + $ref: "#/components/responses/BadRequestError" + "502": + $ref: "#/components/responses/BadGatewayError" /health: get: tags: [Health] @@ -727,6 +822,86 @@ components: type: string enum: [json, markdown, text, ndjson] default: json + URLQuery: + name: url + in: query + required: true + description: Absolute URL to fetch and extract. + schema: + type: string + format: uri + ExtractQuery: + name: extract + in: query + required: false + description: Fetch and embed cleaned target-page content for top web results. + schema: + type: boolean + default: false + ExtractTopQuery: + name: extract_top + in: query + required: false + description: Number of top organic results to enrich when `extract=true`. + schema: + type: integer + minimum: 1 + maximum: 5 + default: 3 + ExtractModeQuery: + name: extract_mode + in: query + required: false + description: Extraction strategy for target pages. + schema: + type: string + enum: [auto, fast, rendered] + default: auto + ExtractModeShortQuery: + name: mode + in: query + required: false + description: Extraction strategy for one URL. + schema: + type: string + enum: [auto, fast, rendered] + default: auto + MinRunesQuery: + name: min_runes + in: query + required: false + description: > + Auto-mode escalation floor: if the fast (raw) pass yields fewer + extracted-text runes than this, escalate to a browser render. `0` + (default) uses the built-in floor. Ignored in `fast` and `rendered` + modes. + schema: + type: integer + minimum: 0 + CleanQuery: + name: clean + in: query + required: false + description: > + Article-only extraction (default). Set `false` for whole-readable-body + extraction that keeps nav/feature/landing content trafilatura would + otherwise strip — useful for landing pages, doc indexes, and dashboards. + schema: + type: boolean + default: true + UseLLMSTxtQuery: + name: use_llms_txt + in: query + required: false + description: > + When the URL is a site root, probe `/llms-full.txt` then `/llms.txt` and + return that LLM-optimized markdown instead of scraping HTML + (see https://llmstxt.org/). Falls through to normal extraction when + absent. Ignored for non-root URLs, where it would miss the requested + page's own content. + schema: + type: boolean + default: false UseProxyHeader: name: X-Use-Proxy in: header @@ -1215,6 +1390,102 @@ components: $ref: "#/components/schemas/DomainInfo" classification: $ref: "#/components/schemas/Classification" + extracted: + $ref: "#/components/schemas/ExtractedContent" + ExtractedContent: + type: object + description: > + One enriched target page. `content` holds a single representation chosen + by the response `format` (plain text for `format=text`, markdown + otherwise), named by `format`. No duplicated markdown/text blobs. + properties: + title: + type: string + format: + type: string + enum: [markdown, text] + content: + type: string + mode_used: + type: string + enum: [fast, rendered, llms_txt] + fetched_at: + type: string + format: date-time + error: + type: string + ExtractResult: + type: object + properties: + url: + type: string + format: uri + title: + type: string + description: + type: string + markdown: + type: string + text: + type: string + headings: + type: array + items: + $ref: "#/components/schemas/ExtractHeading" + links: + type: array + items: + $ref: "#/components/schemas/ExtractLink" + canonical: + type: string + format: uri + lang: + type: string + schema_org: + type: array + items: + type: object + additionalProperties: true + og_tags: + type: object + additionalProperties: + type: string + meta: + $ref: "#/components/schemas/ExtractMeta" + ExtractHeading: + type: object + properties: + level: + type: integer + minimum: 1 + maximum: 6 + text: + type: string + ExtractLink: + type: object + properties: + text: + type: string + url: + type: string + format: uri + ExtractMeta: + type: object + properties: + mode_used: + type: string + description: > + Extraction strategy that produced the result. `llms_txt` means the + site's LLM-optimized markdown was served from `/llms-full.txt` or + `/llms.txt` instead of scraping HTML. + enum: [fast, rendered, llms_txt] + fetched_at: + type: string + format: date-time + bytes: + type: integer + took_ms: + type: integer FeatureItem: type: object properties: diff --git a/extract/config.go b/extract/config.go new file mode 100644 index 0000000..04a50c5 --- /dev/null +++ b/extract/config.go @@ -0,0 +1,52 @@ +package extract + +import "time" + +type Config struct { + Enabled bool `json:"enabled" mapstructure:"enabled"` + DefaultMode string `json:"default_mode" mapstructure:"default_mode"` + Timeout time.Duration `json:"timeout" mapstructure:"timeout"` + MaxBytes int `json:"max_bytes" mapstructure:"max_bytes"` + MaxConcurrent int `json:"max_concurrent" mapstructure:"max_concurrent"` +} + +func DefaultConfig() Config { + return Config{ + Enabled: true, + DefaultMode: string(ModeAuto), + Timeout: 20 * time.Second, + MaxBytes: 2 * 1024 * 1024, + MaxConcurrent: 2, + } +} + +// BatchTimeout derives the wall-clock ceiling for enriching one search response +// (extract=true) from the per-URL budget. Workers run in ceil(count/MaxConcurrent) +// waves; each worker's worst case is a raw fetch plus a rendered escalation, so a +// single Extract is bounded by 2*Timeout. This keeps the batch bound an explicit +// consequence of Timeout rather than a separate knob that can drift out of sync. +func (c Config) BatchTimeout(count int) time.Duration { + c = c.Normalized() + if count <= 0 { + return c.Timeout + } + waves := (count + c.MaxConcurrent - 1) / c.MaxConcurrent + return time.Duration(waves) * 2 * c.Timeout +} + +func (c Config) Normalized() Config { + def := DefaultConfig() + if c.DefaultMode == "" { + c.DefaultMode = def.DefaultMode + } + if c.Timeout <= 0 { + c.Timeout = def.Timeout + } + if c.MaxBytes <= 0 { + c.MaxBytes = def.MaxBytes + } + if c.MaxConcurrent <= 0 { + c.MaxConcurrent = def.MaxConcurrent + } + return c +} diff --git a/extract/content.go b/extract/content.go new file mode 100644 index 0000000..c8fc852 --- /dev/null +++ b/extract/content.go @@ -0,0 +1,164 @@ +package extract + +import ( + "bytes" + "net/url" + "regexp" + "strings" + + "github.com/JohannesKaufmann/html-to-markdown/v2/converter" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/strikethrough" + "github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table" + "github.com/PuerkitoBio/goquery" + "github.com/markusmobius/go-trafilatura" + "golang.org/x/net/html" +) + +var blankLineRun = regexp.MustCompile(`\n{3,}`) + +type contentResult struct { + HTML string + Text string + Markdown string + Title string + Description string + Lang string +} + +// minCleanTextRunes is the trimmed-text floor below which a clean (article-only) +// extraction is treated as too thin. trafilatura strips everything it classifies +// as boilerplate, which guts landing pages, doc indexes, and dashboards where the +// "chrome" is the actual information. When the cleaned text falls under this and +// the page itself carried real content, we fall back to whole-readable-body +// extraction so callers (and LLM agents) get the full page instead of a husk. +const minCleanTextRunes = 250 + +// extractContent turns a page into markdown/text. When clean is true (the +// default) it uses trafilatura to keep only the main article body; if that pass +// comes back too thin for a page that clearly had content, it transparently +// falls back to full-body extraction. When clean is false it skips article +// detection entirely and converts the whole readable . +func extractContent(htmlBytes []byte, baseURL string, clean bool) (contentResult, error) { + if !clean { + return extractFullBody(htmlBytes, baseURL) + } + + var out contentResult + opts := trafilatura.Options{ + EnableFallback: true, + Focus: trafilatura.Balanced, + ExcludeComments: true, + IncludeImages: true, + IncludeLinks: true, + Deduplicate: true, + } + if parsed, err := url.Parse(baseURL); err == nil { + opts.OriginalURL = parsed + } + extracted, err := trafilatura.Extract(bytes.NewReader(htmlBytes), opts) + if err != nil { + return out, err + } + if extracted == nil || extracted.ContentNode == nil { + return extractFullBody(htmlBytes, baseURL) + } + + var htmlBuf bytes.Buffer + if err := html.Render(&htmlBuf, extracted.ContentNode); err != nil { + return out, err + } + out.HTML = strings.TrimSpace(htmlBuf.String()) + out.Text = strings.TrimSpace(extracted.ContentText) + out.Title = strings.TrimSpace(extracted.Metadata.Title) + out.Description = strings.TrimSpace(extracted.Metadata.Description) + out.Lang = strings.TrimSpace(extracted.Metadata.Language) + + markdown, err := htmlToMarkdown(out.HTML, baseURL) + if err != nil { + return out, err + } + out.Markdown = normalizeMarkdown(markdown) + + // trafilatura was too aggressive: the cleaned article is near-empty but the + // raw page had real visible text. Prefer the fuller readable-body pass. + if len([]rune(out.Text)) < minCleanTextRunes { + if full, ferr := extractFullBody(htmlBytes, baseURL); ferr == nil && + len([]rune(full.Text)) > len([]rune(out.Text)) { + // Keep trafilatura's metadata (title/description/lang) when present; + // it is usually cleaner than what we derive from the full body. + full.Title = firstNonEmpty(out.Title, full.Title) + full.Description = firstNonEmpty(out.Description, full.Description) + full.Lang = firstNonEmpty(out.Lang, full.Lang) + return full, nil + } + } + return out, nil +} + +// extractFullBody converts the whole readable to markdown, stripping only +// non-content elements (scripts, styles, nav/header/footer chrome is kept since +// for landing pages and indexes that "chrome" is the information). This is the +// raw-er extraction used for clean=false and as the thin-output fallback. +func extractFullBody(htmlBytes []byte, baseURL string) (contentResult, error) { + var out contentResult + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes)) + if err != nil { + return out, err + } + doc.Find("script,style,noscript,template,svg,iframe").Remove() + + body := doc.Find("body").First() + if body.Length() == 0 { + body = doc.Selection + } + bodyHTML, err := body.Html() + if err != nil { + return out, err + } + out.HTML = strings.TrimSpace(bodyHTML) + out.Text = collapseBlankLines(strings.TrimSpace(body.Text())) + out.Title = strings.TrimSpace(doc.Find("title").First().Text()) + if lang, ok := doc.Find("html").First().Attr("lang"); ok { + out.Lang = strings.TrimSpace(lang) + } + + markdown, err := htmlToMarkdown(out.HTML, baseURL) + if err != nil { + return out, err + } + out.Markdown = normalizeMarkdown(markdown) + return out, nil +} + +func htmlToMarkdown(htmlStr, baseURL string) (string, error) { + conv := converter.NewConverter( + converter.WithPlugins( + base.NewBasePlugin(), + commonmark.NewCommonmarkPlugin(), + ), + ) + conv.Register.Plugin(table.NewTablePlugin(table.WithSkipEmptyRows(true), table.WithHeaderPromotion(true))) + conv.Register.Plugin(strikethrough.NewStrikethroughPlugin()) + return conv.ConvertString(htmlStr, converter.WithDomain(baseURL)) +} + +var whitespaceRun = regexp.MustCompile(`[ \t]+`) + +// collapseBlankLines tidies the raw .Text() of a full body: each line is trimmed, +// intra-line whitespace runs collapse to a single space, and runs of blank lines +// collapse to one (via normalizeMarkdown). +func collapseBlankLines(text string) string { + lines := strings.Split(text, "\n") + for i, line := range lines { + lines[i] = strings.TrimSpace(whitespaceRun.ReplaceAllString(line, " ")) + } + return normalizeMarkdown(strings.Join(lines, "\n")) +} + +func normalizeMarkdown(markdown string) string { + markdown = strings.ReplaceAll(markdown, "\r\n", "\n") + markdown = blankLineRun.ReplaceAllString(markdown, "\n\n") + return strings.TrimSpace(markdown) +} diff --git a/extract/extractor.go b/extract/extractor.go new file mode 100644 index 0000000..ee29948 --- /dev/null +++ b/extract/extractor.go @@ -0,0 +1,209 @@ +package extract + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/PuerkitoBio/goquery" +) + +type Extractor struct { + RawFetch RawFetcher + RenderedFetch RenderedFetcher + Cfg Config +} + +func (e *Extractor) Extract(ctx context.Context, req ExtractRequest) (*ExtractResult, error) { + startedAt := time.Now() + cfg := e.Cfg.Normalized() + req = normalizeRequest(req, cfg) + if req.URL == "" { + return nil, errors.New("url is required") + } + if _, err := url.ParseRequestURI(req.URL); err != nil { + return nil, fmt.Errorf("invalid url: %w", err) + } + if req.Mode == "" { + req.Mode = Mode(cfg.DefaultMode) + } + if req.Mode != ModeAuto && req.Mode != ModeFast && req.Mode != ModeRendered { + return nil, fmt.Errorf("invalid mode %q", req.Mode) + } + + // A site's llms.txt is purpose-built markdown — when the caller opts in and + // the URL is a root, it beats anything we can scrape. Falls through silently + // to normal extraction when absent. + if req.UseLLMSTxt { + if result, ok := e.tryLLMSTxt(ctx, req, startedAt); ok { + return result, nil + } + } + + var rawResult *ExtractResult + var rawErr error + if req.Mode != ModeRendered && e.RawFetch != nil { + rawResult, rawErr = e.extractFast(ctx, req, startedAt) + if req.Mode == ModeFast || goodEnough(rawResult, rawErr, req.MinRunes) { + return rawResult, rawErr + } + } + + if e.RenderedFetch == nil { + return rawResult, rawErr + } + renderedResult, renderedErr := e.extractRendered(ctx, req, startedAt) + if renderedErr == nil && renderedResult != nil { + // In auto mode the raw pass already produced something usable but below + // the quality threshold. Only prefer the rendered pass when it actually + // recovered more content — a bot wall or consent page can render shorter + // than the raw HTML, and falling back to it would be a regression. + if rawErr == nil && rawResult != nil && textLength(rawResult) > textLength(renderedResult) { + return rawResult, nil + } + return renderedResult, nil + } + if rawResult != nil || rawErr != nil { + return rawResult, rawErr + } + return nil, renderedErr +} + +func (e *Extractor) extractFast(ctx context.Context, req ExtractRequest, startedAt time.Time) (*ExtractResult, error) { + fetchCtx, cancel := context.WithTimeout(ctx, req.Timeout) + defer cancel() + resp, err := e.RawFetch(fetchCtx, req) + if err != nil { + return nil, err + } + if resp == nil { + return nil, errors.New("empty raw response") + } + return buildResult(req, resp, string(ModeFast), startedAt) +} + +func (e *Extractor) extractRendered(ctx context.Context, req ExtractRequest, startedAt time.Time) (*ExtractResult, error) { + fetchCtx, cancel := context.WithTimeout(ctx, req.Timeout) + defer cancel() + resp, err := e.RenderedFetch(fetchCtx, req) + if err != nil { + return nil, err + } + if resp == nil { + return nil, errors.New("empty rendered response") + } + return buildResult(req, resp, string(ModeRendered), startedAt) +} + +func normalizeRequest(req ExtractRequest, cfg Config) ExtractRequest { + req.URL = normalizeURL(strings.TrimSpace(req.URL)) + if req.Timeout <= 0 { + req.Timeout = cfg.Timeout + } + if req.MaxBytes <= 0 { + req.MaxBytes = cfg.MaxBytes + } + if req.MaxBytes > 0 && req.MaxBytes < 64*1024 { + req.MaxBytes = 64 * 1024 + } + req.Mode = Mode(strings.ToLower(strings.TrimSpace(string(req.Mode)))) + return req +} + +// normalizeURL defaults a missing scheme to https so callers can pass a bare host +// (e.g. "kamaloff.ru"). URLs that already carry a scheme are left untouched. +func normalizeURL(raw string) string { + if raw == "" || strings.Contains(raw, "://") || strings.HasPrefix(raw, "//") { + return raw + } + return "https://" + raw +} + +func buildResult(req ExtractRequest, resp *FetchResponse, mode string, startedAt time.Time) (*ExtractResult, error) { + if err := classifyStatus(resp.StatusCode); err != nil { + return nil, err + } + body := resp.Body + if req.MaxBytes > 0 && len(body) > req.MaxBytes { + body = body[:req.MaxBytes] + } + doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + metadata := parseMetadata(doc, req.URL) + content, contentErr := extractContent(body, req.URL, !req.FullPage) + result := &ExtractResult{ + URL: req.URL, + Title: firstNonEmpty(content.Title, metadata.Title), + Description: firstNonEmpty(content.Description, metadata.Description), + Markdown: content.Markdown, + Text: content.Text, + Headings: metadata.Headings, + Links: metadata.Links, + Canonical: metadata.Canonical, + Lang: firstNonEmpty(content.Lang, metadata.Lang), + SchemaOrg: metadata.SchemaOrg, + OGTags: metadata.OGTags, + Meta: ExtractMeta{ + ModeUsed: mode, + FetchedAt: time.Now().UTC().Format(time.RFC3339), + Bytes: len(body), + TookMs: time.Since(startedAt).Milliseconds(), + }, + } + if contentErr != nil && strings.TrimSpace(result.Text) == "" { + return result, contentErr + } + return result, nil +} + +func classifyStatus(status int) error { + if status == 0 { + return nil + } + if status == http.StatusForbidden || status == http.StatusUnauthorized { + return fmt.Errorf("blocked: HTTP %d", status) + } + if status == http.StatusTooManyRequests { + return fmt.Errorf("rate limited: HTTP %d", status) + } + if status < 200 || status >= 300 { + return fmt.Errorf("unexpected HTTP status %d", status) + } + return nil +} + +// defaultMinRunes is the built-in auto-mode escalation floor. +const defaultMinRunes = 200 + +// goodEnough reports whether the raw pass is worth keeping instead of escalating +// to a render. minRunes overrides the floor (<= 0 uses defaultMinRunes); the +// headings shortcut accepts shorter structured pages, never below 120 runes. +func goodEnough(result *ExtractResult, err error, minRunes int) bool { + if err != nil || result == nil { + return false + } + if minRunes <= 0 { + minRunes = defaultMinRunes + } + textLen := textLength(result) + if textLen >= minRunes { + return true + } + return textLen >= 120 && textLen >= minRunes-80 && len(result.Headings) > 0 +} + +// textLength returns the rune count of the trimmed extracted text, used both to +// gate auto-mode escalation and to compare raw vs rendered output quality. +func textLength(result *ExtractResult) int { + if result == nil { + return 0 + } + return len([]rune(strings.TrimSpace(result.Text))) +} diff --git a/extract/extractor_test.go b/extract/extractor_test.go new file mode 100644 index 0000000..a8cd101 --- /dev/null +++ b/extract/extractor_test.go @@ -0,0 +1,310 @@ +package extract + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/PuerkitoBio/goquery" +) + +// staticRaw returns a RawFetcher that always serves the same HTML with HTTP 200. +func staticRaw(html string) RawFetcher { + return func(context.Context, ExtractRequest) (*FetchResponse, error) { + return &FetchResponse{StatusCode: 200, Body: []byte(html)}, nil + } +} + +// runExtract builds an Extractor from the given fetchers and runs Extract, +// failing the test on error. +func runExtract(t *testing.T, raw RawFetcher, rendered RenderedFetcher, req ExtractRequest) *ExtractResult { + t.Helper() + extractor := Extractor{RawFetch: raw, RenderedFetch: rendered, Cfg: DefaultConfig()} + result, err := extractor.Extract(context.Background(), req) + if err != nil { + t.Fatalf("Extract() error = %v", err) + } + return result +} + +func TestExtractFastStaticArticle(t *testing.T) { + html := ` +Static Article + + +
+

Static Article

+

This article has enough body text to be treated as useful extracted content. It explains how OpenSERP extracts target pages into markdown for grounding workflows, with clear paragraphs and useful links for downstream automation. The text is intentionally longer than the quality threshold.

+Docs +
+` + result := runExtract(t, staticRaw(html), nil, ExtractRequest{URL: "https://example.com/post", Mode: ModeFast}) + if result.Title != "Static Article" { + t.Fatalf("title = %q", result.Title) + } + if !strings.Contains(result.Markdown, "OpenSERP extracts target pages") { + t.Fatalf("markdown missing article body: %q", result.Markdown) + } + if len(result.Links) != 1 || result.Links[0].URL != "https://example.com/docs" { + t.Fatalf("links = %#v", result.Links) + } + if result.Meta.ModeUsed != string(ModeFast) { + t.Fatalf("mode_used = %q", result.Meta.ModeUsed) + } +} + +func TestExtractAutoEscalatesThinShell(t *testing.T) { + raw := staticRaw(`
`) + rendered := func(context.Context, ExtractRequest) (*FetchResponse, error) { + return &FetchResponse{StatusCode: 200, Body: []byte(`

Rendered

This rendered page contains enough meaningful article text after JavaScript execution to pass the extraction threshold and avoid returning an empty shell to callers.

`)}, nil + } + result := runExtract(t, raw, rendered, ExtractRequest{URL: "https://example.com/app", Mode: ModeAuto}) + if result.Meta.ModeUsed != string(ModeRendered) { + t.Fatalf("mode_used = %q", result.Meta.ModeUsed) + } +} + +func TestNormalizeURL(t *testing.T) { + cases := map[string]string{ + "kamaloff.ru": "https://kamaloff.ru", + "example.com/path?q=1": "https://example.com/path?q=1", + "http://example.com": "http://example.com", + "https://example.com": "https://example.com", + "//example.com": "//example.com", + "": "", + "socks5h://127.0.0.1:80": "socks5h://127.0.0.1:80", + } + for in, want := range cases { + if got := normalizeURL(in); got != want { + t.Errorf("normalizeURL(%q) = %q, want %q", in, got, want) + } + } +} + +func TestExtractBareHostGetsScheme(t *testing.T) { + var fetched string + raw := func(_ context.Context, req ExtractRequest) (*FetchResponse, error) { + fetched = req.URL + return &FetchResponse{StatusCode: 200, Body: []byte(`Home

` + + strings.Repeat("Bare hostnames must resolve to https and extract normally. ", 5) + `

`)}, nil + } + result := runExtract(t, raw, nil, ExtractRequest{URL: "kamaloff.ru", Mode: ModeFast}) + if fetched != "https://kamaloff.ru" { + t.Fatalf("fetched %q, want https://kamaloff.ru", fetched) + } + if result.Title != "Home" { + t.Fatalf("title = %q", result.Title) + } +} + +func TestExtractAutoMinRunesForcesEscalation(t *testing.T) { + // Raw body clears the default floor (kept as-is), but a high MinRunes raises + // the bar above the raw yield, forcing escalation to the richer rendered pass. + rawHTML := `Summary +

Summary

This raw article body is comfortably over the default two hundred rune floor, so without a higher threshold the auto pass would accept it and never render.

+` + rendered := func(context.Context, ExtractRequest) (*FetchResponse, error) { + return &FetchResponse{StatusCode: 200, Body: []byte(`

Full

` + + strings.Repeat("The rendered pass returns the complete article with substantially more prose than the server-sent summary, which is exactly what a caller raising the content floor is asking for. ", 4) + + `

`)}, nil + } + + // Without the override, auto keeps the raw summary. + base := runExtract(t, staticRaw(rawHTML), rendered, ExtractRequest{URL: "https://example.com/x", Mode: ModeAuto}) + if base.Meta.ModeUsed != string(ModeFast) { + t.Fatalf("baseline mode_used = %q, want fast (raw clears default floor)", base.Meta.ModeUsed) + } + + // With a floor above the raw yield, auto escalates to rendered. + raised := runExtract(t, staticRaw(rawHTML), rendered, ExtractRequest{URL: "https://example.com/x", Mode: ModeAuto, MinRunes: 1000}) + if raised.Meta.ModeUsed != string(ModeRendered) { + t.Fatalf("raised-floor mode_used = %q, want rendered", raised.Meta.ModeUsed) + } +} + +func TestExtractAutoKeepsRawWhenRenderedThinner(t *testing.T) { + rawHTML := `Raw +

Raw

This raw HTML response already carries a substantial article body that sits just under the auto-mode quality threshold, yet still holds far more useful prose than the consent wall the rendered pass returns for this page.

+` + rendered := func(context.Context, ExtractRequest) (*FetchResponse, error) { + return &FetchResponse{StatusCode: 200, Body: []byte(`

Accept cookies to continue.

`)}, nil + } + result := runExtract(t, staticRaw(rawHTML), rendered, ExtractRequest{URL: "https://example.com/wall", Mode: ModeAuto}) + if result.Meta.ModeUsed != string(ModeFast) { + t.Fatalf("expected raw result to win, mode_used = %q", result.Meta.ModeUsed) + } +} + +func TestExtractCleanFallsBackOnThinArticle(t *testing.T) { + // A landing page: trafilatura strips the feature/nav chrome down to almost + // nothing, so the thin-output guard should fall back to full-body extraction + // and recover the visible text. + landing := `OpenSERP +
+
+

OpenSERP — Free SERP API

+
+

Google

Scrape Google results with one call.

+

Bing

Bing and Yandex supported out of the box.

+

Yandex

Region targeting and UULE handling built in.

+
+ +
+` + result := runExtract(t, staticRaw(landing), nil, ExtractRequest{URL: "https://openserp.org", Mode: ModeFast}) + for _, want := range []string{"Google", "Bing", "Yandex", "Region targeting"} { + if !strings.Contains(result.Text, want) { + t.Fatalf("full-body fallback dropped %q; text = %q", want, result.Text) + } + } +} + +func TestExtractFullPageKeepsChrome(t *testing.T) { + page := `Dash + +

Article

A genuine article body that easily clears the extraction quality threshold so the clean path would normally keep only this and discard the navigation links above it.

+` + // FullPage must retain the nav chrome that clean mode would strip. + result := runExtract(t, staticRaw(page), nil, ExtractRequest{URL: "https://example.com/dash", Mode: ModeFast, FullPage: true}) + if !strings.Contains(result.Text, "Alpha") || !strings.Contains(result.Text, "Beta") { + t.Fatalf("full-page extraction dropped nav chrome; text = %q", result.Text) + } +} + +func TestExtractLLMSTxtRootHit(t *testing.T) { + const llmsFull = `# Example Docs + +This is the full LLM-optimized corpus for the site. It contains far more useful, structured prose than scraping the rendered HTML landing page would ever surface, which is exactly why agents should prefer it when present at the site root.` + var fetched []string + raw := func(_ context.Context, req ExtractRequest) (*FetchResponse, error) { + fetched = append(fetched, req.URL) + if strings.HasSuffix(req.URL, "/llms-full.txt") { + return &FetchResponse{StatusCode: 200, Body: []byte(llmsFull)}, nil + } + return &FetchResponse{StatusCode: 404}, nil + } + result := runExtract(t, raw, nil, ExtractRequest{URL: "https://example.com", Mode: ModeFast, UseLLMSTxt: true}) + if result.Meta.ModeUsed != "llms_txt" { + t.Fatalf("mode_used = %q, want llms_txt", result.Meta.ModeUsed) + } + if result.Title != "Example Docs" { + t.Fatalf("title = %q", result.Title) + } + if fetched[0] != "https://example.com/llms-full.txt" { + t.Fatalf("first probe = %q, want .../llms-full.txt", fetched[0]) + } +} + +func TestExtractLLMSTxtSkippedForDeepURL(t *testing.T) { + article := `Post

Post

` + + strings.Repeat("This deep article page has its own substantial body content that must win over any site-level llms index. ", 4) + + `

` + var probedLLMS bool + raw := func(_ context.Context, req ExtractRequest) (*FetchResponse, error) { + if strings.Contains(req.URL, "llms") { + probedLLMS = true + } + return &FetchResponse{StatusCode: 200, Body: []byte(article)}, nil + } + result := runExtract(t, raw, nil, ExtractRequest{URL: "https://example.com/blog/post", Mode: ModeFast, UseLLMSTxt: true}) + if probedLLMS { + t.Fatal("llms.txt was probed for a deep (non-root) URL") + } + if result.Meta.ModeUsed == "llms_txt" { + t.Fatal("deep URL must not resolve via llms.txt") + } +} + +func TestExtractLLMSTxtRejectsHTMLShell(t *testing.T) { + // A site that answers unknown paths with its SPA index.html (200, but HTML). + const spaShell = `App
` + raw := func(_ context.Context, req ExtractRequest) (*FetchResponse, error) { + if strings.Contains(req.URL, "llms") { + return &FetchResponse{StatusCode: 200, Body: []byte(spaShell)}, nil + } + return &FetchResponse{StatusCode: 200, Body: []byte(`Home

` + + strings.Repeat("Real homepage article content that should be extracted normally. ", 5) + `

`)}, nil + } + result := runExtract(t, raw, nil, ExtractRequest{URL: "https://example.com", Mode: ModeFast, UseLLMSTxt: true}) + if result.Meta.ModeUsed == "llms_txt" { + t.Fatal("HTML shell served at /llms.txt must be rejected") + } +} + +// TestExtractHonorsContextCancellation proves a fetch aborts when the parent +// context is cancelled. The derived batch deadline in the search-enrichment +// path (Config.BatchTimeout) relies on exactly this: once the budget is spent, +// in-flight Extract calls must return promptly with the context error rather +// than running to their own per-fetch timeout. +func TestExtractHonorsContextCancellation(t *testing.T) { + // A raw fetcher that hangs until its context is cancelled, mimicking a slow + // or unresponsive target. + hangingRaw := func(ctx context.Context, _ ExtractRequest) (*FetchResponse, error) { + <-ctx.Done() + return nil, ctx.Err() + } + extractor := Extractor{RawFetch: hangingRaw, Cfg: DefaultConfig()} + + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := extractor.Extract(ctx, ExtractRequest{URL: "https://example.com/slow", Mode: ModeFast}) + if err == nil { + t.Fatal("expected a context error, got nil") + } + // Must abort near the parent deadline, not run to the 20s per-fetch timeout. + if elapsed := time.Since(start); elapsed > 5*time.Second { + t.Fatalf("Extract ignored context cancellation; took %s", elapsed) + } +} + +func TestBatchTimeoutDerivation(t *testing.T) { + cfg := DefaultConfig() // Timeout=20s, MaxConcurrent=2 + cases := []struct { + count int + want time.Duration + }{ + {count: 0, want: 20 * time.Second}, // degenerate: one per-URL budget + {count: 1, want: 40 * time.Second}, // 1 wave * 2 * 20s + {count: 2, want: 40 * time.Second}, // 1 wave * 2 * 20s + {count: 3, want: 80 * time.Second}, // 2 waves * 2 * 20s + {count: 5, want: 120 * time.Second}, // 3 waves * 2 * 20s + } + for _, tc := range cases { + if got := cfg.BatchTimeout(tc.count); got != tc.want { + t.Errorf("BatchTimeout(%d) = %s, want %s", tc.count, got, tc.want) + } + } +} + +func TestParseMetadataRichTags(t *testing.T) { + doc, err := documentFromString(` + + + + + +

Heading

A link`) + if err != nil { + t.Fatal(err) + } + meta := parseMetadata(doc, "https://example.com/page") + if meta.Title != "OG title" || meta.Description != "OG description" { + t.Fatalf("metadata title/description = %q / %q", meta.Title, meta.Description) + } + if meta.Canonical != "https://example.com/canonical" { + t.Fatalf("canonical = %q", meta.Canonical) + } + if len(meta.SchemaOrg) != 2 { + t.Fatalf("schema_org count = %d", len(meta.SchemaOrg)) + } + if meta.OGTags["twitter:card"] != "summary" { + t.Fatalf("og_tags = %#v", meta.OGTags) + } +} + +func documentFromString(raw string) (*goquery.Document, error) { + return goquery.NewDocumentFromReader(strings.NewReader(raw)) +} diff --git a/extract/llmstxt.go b/extract/llmstxt.go new file mode 100644 index 0000000..494019c --- /dev/null +++ b/extract/llmstxt.go @@ -0,0 +1,109 @@ +package extract + +import ( + "context" + "net/url" + "strings" + "time" +) + +// llmsTxtCandidates are the well-known LLM-optimized markdown files, tried in +// order of richness. /llms-full.txt is the concatenated full corpus; +// /llms.txt is the curated index. See https://llmstxt.org/. +var llmsTxtCandidates = []string{"/llms-full.txt", "/llms.txt"} + +// minLLMSTxtRunes guards against a site answering an unknown path with its SPA +// index.html (HTTP 200, but HTML, not markdown). Anything shorter than this, or +// that sniffs as HTML, is rejected so we fall through to normal extraction. +const minLLMSTxtRunes = 200 + +// isSiteRoot reports whether the URL points at a site root, where /llms.txt is +// meaningful. We only probe roots because /llms.txt describes the whole site — +// for a deep page (e.g. /blog/post) it would return the site index and miss the +// content the caller actually asked for. +func isSiteRoot(rawURL string) bool { + parsed, err := url.Parse(rawURL) + if err != nil { + return false + } + path := strings.Trim(parsed.Path, "/") + return path == "" +} + +// tryLLMSTxt probes the well-known llms.txt files at the site root using the raw +// fetcher. It returns (result, true) on the first usable hit, or (nil, false) to +// signal the caller should fall through to normal HTML extraction. Errors are +// swallowed deliberately: a missing/!200/HTML llms.txt is the common case and +// must never fail the extract. +func (e *Extractor) tryLLMSTxt(ctx context.Context, req ExtractRequest, startedAt time.Time) (*ExtractResult, bool) { + if e.RawFetch == nil || !isSiteRoot(req.URL) { + return nil, false + } + base, err := url.Parse(req.URL) + if err != nil { + return nil, false + } + for _, candidate := range llmsTxtCandidates { + ref, err := url.Parse(candidate) + if err != nil { + continue + } + probe := req + probe.URL = base.ResolveReference(ref).String() + + fetchCtx, cancel := context.WithTimeout(ctx, req.Timeout) + resp, ferr := e.RawFetch(fetchCtx, probe) + cancel() + if ferr != nil || resp == nil || resp.StatusCode != 200 { + continue + } + body := resp.Body + if req.MaxBytes > 0 && len(body) > req.MaxBytes { + body = body[:req.MaxBytes] + } + text := strings.TrimSpace(string(body)) + if len([]rune(text)) < minLLMSTxtRunes || looksLikeHTML(text) { + continue + } + return &ExtractResult{ + URL: req.URL, + Title: firstNonEmpty(llmsTxtTitle(text), req.URL), + Markdown: normalizeMarkdown(text), + Text: text, + Meta: ExtractMeta{ + ModeUsed: "llms_txt", + FetchedAt: time.Now().UTC().Format(time.RFC3339), + Bytes: len(body), + TookMs: time.Since(startedAt).Milliseconds(), + }, + }, true + } + return nil, false +} + +// looksLikeHTML rejects bodies that are actually HTML (common when a site serves +// its SPA shell for unknown paths) rather than the markdown we want. +func looksLikeHTML(text string) bool { + head := strings.ToLower(strings.TrimSpace(text)) + if len(head) > 256 { + head = head[:256] + } + return strings.HasPrefix(head, "") || + strings.Contains(head, "= 1 && level <= 6 && text != "" { + meta.Headings = append(meta.Headings, Heading{Level: level, Text: collapseWhitespace(text)}) + } + }) + + doc.Find("a[href]").Each(func(_ int, sel *goquery.Selection) { + if len(meta.Links) >= 100 { + return + } + href, _ := sel.Attr("href") + resolved := resolveURL(baseURL, href) + if resolved == "" { + return + } + text := collapseWhitespace(strings.TrimSpace(sel.Text())) + meta.Links = append(meta.Links, Link{Text: text, URL: resolved}) + }) + return meta +} + +func metaContent(doc *goquery.Document, selector string) string { + value, _ := doc.Find(selector).First().Attr("content") + return strings.TrimSpace(value) +} + +func firstNonEmpty(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func appendJSONLD(dst *[]json.RawMessage, raw string) { + if raw == "" { + return + } + var decoded any + if err := json.Unmarshal([]byte(raw), &decoded); err != nil { + return + } + switch value := decoded.(type) { + case map[string]any: + if graph, ok := value["@graph"].([]any); ok { + for _, item := range graph { + if data, err := json.Marshal(item); err == nil { + *dst = append(*dst, data) + } + } + return + } + case []any: + for _, item := range value { + if data, err := json.Marshal(item); err == nil { + *dst = append(*dst, data) + } + } + return + } + if data, err := json.Marshal(decoded); err == nil { + *dst = append(*dst, data) + } +} + +func resolveURL(baseURL string, raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" || strings.HasPrefix(raw, "#") || strings.HasPrefix(strings.ToLower(raw), "javascript:") { + return "" + } + parsed, err := url.Parse(raw) + if err != nil { + return "" + } + if parsed.IsAbs() { + return parsed.String() + } + base, err := url.Parse(baseURL) + if err != nil { + return "" + } + return base.ResolveReference(parsed).String() +} + +func collapseWhitespace(value string) string { + return strings.Join(strings.Fields(value), " ") +} diff --git a/extract/types.go b/extract/types.go new file mode 100644 index 0000000..8e6acf6 --- /dev/null +++ b/extract/types.go @@ -0,0 +1,76 @@ +package extract + +import ( + "context" + "encoding/json" + "time" +) + +type Mode string + +const ( + ModeAuto Mode = "auto" + ModeFast Mode = "fast" + ModeRendered Mode = "rendered" +) + +type ExtractRequest struct { + URL string + Mode Mode + ProxyURL string + LangCode string + Timeout time.Duration + MaxBytes int + // FullPage selects whole-readable-body extraction instead of the default + // article-only (trafilatura) extraction. LLM agents fetching arbitrary URLs + // often want the full page; FullPage keeps nav/feature/landing content that + // trafilatura strips. The zero value (false) preserves the cleaned default. + FullPage bool + // UseLLMSTxt, when set and the URL is a site root, probes /llms-full.txt then + // /llms.txt and returns that LLM-optimized markdown instead of scraping HTML. + UseLLMSTxt bool + // MinRunes is the per-request auto-mode escalation floor: raw output below + // this many extracted-text runes escalates to a render. 0 uses defaultMinRunes. + MinRunes int +} + +type ExtractResult struct { + URL string `json:"url"` + Title string `json:"title"` + Description string `json:"description"` + Markdown string `json:"markdown"` + Text string `json:"text"` + Headings []Heading `json:"headings,omitempty"` + Links []Link `json:"links,omitempty"` + Canonical string `json:"canonical,omitempty"` + Lang string `json:"lang,omitempty"` + SchemaOrg []json.RawMessage `json:"schema_org,omitempty"` + OGTags map[string]string `json:"og_tags,omitempty"` + Meta ExtractMeta `json:"meta"` +} + +type Heading struct { + Level int `json:"level"` + Text string `json:"text"` +} + +type Link struct { + Text string `json:"text"` + URL string `json:"url"` +} + +type ExtractMeta struct { + ModeUsed string `json:"mode_used"` + FetchedAt string `json:"fetched_at"` + Bytes int `json:"bytes"` + TookMs int64 `json:"took_ms"` +} + +type FetchResponse struct { + StatusCode int + Body []byte +} + +type RawFetcher func(ctx context.Context, req ExtractRequest) (*FetchResponse, error) + +type RenderedFetcher func(ctx context.Context, req ExtractRequest) (*FetchResponse, error) diff --git a/go.mod b/go.mod index f68a87a..5195d6d 100644 --- a/go.mod +++ b/go.mod @@ -6,12 +6,14 @@ toolchain go1.24.6 require ( github.com/2captcha/2captcha-go v1.1.10 + github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 github.com/PuerkitoBio/goquery v1.10.3 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/corpix/uarand v0.2.0 github.com/go-rod/rod v0.116.2 github.com/gofiber/fiber/v2 v2.52.9 github.com/google/uuid v1.6.0 + github.com/markusmobius/go-trafilatura v1.12.2 github.com/refraction-networking/utls v1.8.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.9.1 @@ -24,30 +26,50 @@ require ( ) require ( + github.com/JohannesKaufmann/dom v0.2.0 // indirect + github.com/RadhiFadlillah/whatlanggo v0.0.0-20240916001553-aac1f0f737fc // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/andybalholm/cascadia v1.3.3 // indirect + github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect + github.com/elliotchance/pie/v2 v2.9.0 // indirect + github.com/forPelevin/gomoji v1.2.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect + github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect + github.com/hablullah/go-hijri v1.0.2 // indirect + github.com/hablullah/go-juliandays v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 // indirect github.com/klauspost/compress v1.18.0 // indirect + github.com/markusmobius/go-dateparser v1.2.3 // indirect + github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4 // indirect + github.com/markusmobius/go-htmldate v1.9.1 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rs/zerolog v1.33.0 // indirect github.com/sagikazarmark/locafero v0.10.0 // indirect github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.14.0 // indirect github.com/spf13/cast v1.9.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tetratelabs/wazero v1.8.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/fasthttp v1.65.0 // indirect + github.com/wasilibs/go-re2 v1.7.0 // indirect + github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect + github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 // indirect github.com/ysmood/fetchup v0.3.0 // indirect github.com/ysmood/goob v0.4.0 // indirect github.com/ysmood/got v0.41.0 // indirect github.com/ysmood/leakless v0.9.0 // indirect golang.org/x/crypto v0.41.0 // indirect + golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.28.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/go.sum b/go.sum index 9f4ce54..3febe48 100644 --- a/go.sum +++ b/go.sum @@ -1,35 +1,61 @@ github.com/2captcha/2captcha-go v1.1.10 h1:U3Y7VLwR9z5XpCMijB+FkhHRt6QikKnHDEy1uLQVCD8= github.com/2captcha/2captcha-go v1.1.10/go.mod h1:TsupeToBP0BPHfZOQpNDb61NKqtbBJ2ddptqSK2p9/M= +github.com/JohannesKaufmann/dom v0.2.0 h1:1bragmEb19K8lHAqgFgqCpiPCFEZMTXzOIEjuxkUfLQ= +github.com/JohannesKaufmann/dom v0.2.0/go.mod h1:57iSUl5RKric4bUkgos4zu6Xt5LMHUnw3TF1l5CbGZo= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0 h1:C0/TerKdQX9Y9pbYi1EsLr5LDNANsqunyI/btpyfCg8= +github.com/JohannesKaufmann/html-to-markdown/v2 v2.4.0/go.mod h1:OLaKh+giepO8j7teevrNwiy/fwf8LXgoc9g7rwaE1jk= github.com/PuerkitoBio/goquery v1.10.3 h1:pFYcNSqHxBD06Fpj/KsbStFRsgRATgnf3LeXiUkhzPo= github.com/PuerkitoBio/goquery v1.10.3/go.mod h1:tMUX0zDMHXYlAQk6p35XxQMqMweEKB7iK7iLNd4RH4Y= +github.com/RadhiFadlillah/whatlanggo v0.0.0-20240916001553-aac1f0f737fc h1:6aA31zw7fnfJ/G1ebisIesCDl44slkIVFqk3YTSadd8= +github.com/RadhiFadlillah/whatlanggo v0.0.0-20240916001553-aac1f0f737fc/go.mod h1:PgrPWaMBxL1lyq1k5DEMqC0Y67R3pG1vEsHzxFXeDxc= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM= github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/corpix/uarand v0.2.0 h1:U98xXwud/AVuCpkpgfPF7J5TQgr7R5tqT8VZP5KWbzE= github.com/corpix/uarand v0.2.0/go.mod h1:/3Z1QIqWkDIhf6XWn/08/uMHoQ8JUoTIKc2iPchBOmM= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/elliotchance/pie/v2 v2.9.0 h1:BkEhh8b/avGCSpXpABSjNuytxlI/S2snkjT3vtVORjw= +github.com/elliotchance/pie/v2 v2.9.0/go.mod h1:18t0dgGFH006g4eVdDtWfgFZPQEgl10IoEO8YWEq3Og= +github.com/forPelevin/gomoji v1.2.0 h1:9k4WVSSkE1ARO/BWywxgEUBvR/jMnao6EZzrql5nxJ8= +github.com/forPelevin/gomoji v1.2.0/go.mod h1:8+Z3KNGkdslmeGZBC3tCrwMrcPy5GRzAD+gL9NAwMXg= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/go-rod/rod v0.116.2 h1:A5t2Ky2A+5eD/ZJQr1EfsQSe5rms5Xof/qj296e+ZqA= github.com/go-rod/rod v0.116.2/go.mod h1:H+CMO9SCNc2TJ2WfrG+pKhITz57uGNYU43qYHh438Mg= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w= +github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM= +github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f h1:cypj7SJh+47G9J3VCPdMzT3uWcXWAWDJA54ErTfOigI= +github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f/go.mod h1:YWa00ashoPZMAOElrSn4E1cJErhDVU6PWAll4Hxzn+w= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofiber/fiber/v2 v2.52.9 h1:YjKl5DOiyP3j0mO61u3NTmK7or8GzzWzCFzkboyP5cw= github.com/gofiber/fiber/v2 v2.52.9/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs= +github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hablullah/go-hijri v1.0.2 h1:drT/MZpSZJQXo7jftf5fthArShcaMtsal0Zf/dnmp6k= +github.com/hablullah/go-hijri v1.0.2/go.mod h1:OS5qyYLDjORXzK4O1adFw9Q5WfhOcMdAKglDkcTxgWQ= +github.com/hablullah/go-juliandays v1.0.0 h1:A8YM7wIj16SzlKT0SRJc9CD29iiaUzpBLzh5hr0/5p0= +github.com/hablullah/go-juliandays v1.0.0/go.mod h1:0JOYq4oFOuDja+oospuc61YoX+uNEn7Z6uHYTbBzdGc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958 h1:qxLoi6CAcXVzjfvu+KXIXJOAsQB62LXjsfbOaErsVzE= +github.com/jalaali/go-jalaali v0.0.0-20210801064154-80525e88d958/go.mod h1:Wqfu7mjUHj9WDzSSPI5KfBclTTEnLveRUFr/ujWnTgE= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -39,26 +65,50 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a h1:tdPcGgyiH0K+SbsJBBm2oPyEIOTAvLBwD9TuUwVtZho= +github.com/magefile/mage v1.15.1-0.20230912152418-9f54e0f83e2a/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/markusmobius/go-dateparser v1.2.3 h1:TvrsIvr5uk+3v6poDjaicnAFJ5IgtFHgLiuMY2Eb7Nw= +github.com/markusmobius/go-dateparser v1.2.3/go.mod h1:cMwQRrBUQlK1UI5TIFHEcvpsMbkWrQLXuaPNMFzuYLk= +github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4 h1:+7kfF1+dmSXV469sqjeNC+eKJF7xDuS5mvZA3DFVLLY= +github.com/markusmobius/go-domdistiller v0.0.0-20240926050704-25b8d046ffb4/go.mod h1:E7PoeC3nd4GqtxP1A64v7JDBxpAbpTSnhlq9/DHmQ28= +github.com/markusmobius/go-htmldate v1.9.1 h1:0kfVz0wdxGCBaotWNzdtIZKhy7+8ClBlzvANQ67Rlt8= +github.com/markusmobius/go-htmldate v1.9.1/go.mod h1:fLls4rjQDxYR+Pxhf0YR6Ht8dEeHd4SxK/NPaVqhMa8= +github.com/markusmobius/go-trafilatura v1.12.2 h1:JgEto0kDjwTuyXFl6TB+psrs1QGJqTdYJEbLhDy1vrw= +github.com/markusmobius/go-trafilatura v1.12.2/go.mod h1:2WnYLuvGBgJAarHaAQnsvofihEojt2xDDrtVJU5UXZI= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/refraction-networking/utls v1.8.0 h1:L38krhiTAyj9EeiQQa2sg+hYb4qwLCqdMcpZrRfbONE= github.com/refraction-networking/utls v1.8.0/go.mod h1:jkSOEkLqn+S/jtpEHPOsVv/4V4EVnelwbMQl4vCWXAM= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.10.0 h1:FM8Cv6j2KqIhM2ZK7HZjm4mpj9NBktLgowT1aN9q5Cc= github.com/sagikazarmark/locafero v0.10.0/go.mod h1:Ieo3EUsjifvQu4NZwV5sPd4dwvu0OCgEQV7vjc9yDjw= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= +github.com/sebdah/goldie/v2 v2.7.1 h1:PkBHymaYdtvEkZV7TmyqKxdmn5/Vcj+8TpATWZjnG5E= +github.com/sebdah/goldie/v2 v2.7.1/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= +github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= +github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= @@ -80,12 +130,22 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tetratelabs/wazero v1.8.1 h1:NrcgVbWfkWvVc4UtT4LRLDf91PsOzDzefMdwhLfA550= +github.com/tetratelabs/wazero v1.8.1/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.65.0 h1:j/u3uzFEGFfRxw79iYzJN+TteTJwbYkru9uDp3d0Yf8= github.com/valyala/fasthttp v1.65.0/go.mod h1:P/93/YkKPMsKSnATEeELUCkG8a7Y+k99uxNHVbKINr4= +github.com/wasilibs/go-re2 v1.7.0 h1:bYhl8gn+a9h01dxwotNycxkiFPTiSgwUrIz8KZJ90Lc= +github.com/wasilibs/go-re2 v1.7.0/go.mod h1:sUsZMLflgl+LNivDE229omtmvjICmOseT9xOy199VDU= +github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ= +github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo= +github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 h1:OvLBa8SqJnZ6P+mjlzc2K7PM22rRUPE1x32G9DTPrC4= +github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4 h1:0sw0nJM544SpsihWx1bkXdYLQDlzRflMgFJQ4Yih9ts= +github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4/go.mod h1:+ccdNT0xMY1dtc5XBxumbYfOUhmduiGudqaDgD2rVRE= github.com/ysmood/fetchup v0.3.0 h1:UhYz9xnLEVn2ukSuK3KCgcznWpHMdrmbsPpllcylyu8= github.com/ysmood/fetchup v0.3.0/go.mod h1:hbysoq65PXL0NQeNzUczNYIKpwpkwFL4LXMDEvIQq9A= github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ= @@ -101,6 +161,8 @@ github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3R github.com/ysmood/leakless v0.9.0 h1:qxCG5VirSBvmi3uynXFkcnLMzkphdh3xx5FtrORwDCU= github.com/ysmood/leakless v0.9.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/goldmark v1.7.13 h1:GPddIs617DnBLFFVJFgpo1aBfe/4xcvMc3SB5t/D0pA= +github.com/yuin/goldmark v1.7.13/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= @@ -109,6 +171,8 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY= +golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -138,6 +202,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=