Add new URL extraction endpoint and search feature

2026-08-05 16:53:54 +08:00 · 2026-06-04 02:22:38 +03:00
parent 2b4a80fcb2
commit aa397eb2c1
20 changed files with 1894 additions and 36 deletions
--- a/extract/config.go
+++ b/extract/config.go
@@ -0,0 +1,52 @@
+package extract
+
+import "time"
+
+type Config struct {
+	Enabled       bool          `json:"enabled" mapstructure:"enabled"`
+	DefaultMode   string        `json:"default_mode" mapstructure:"default_mode"`
+	Timeout       time.Duration `json:"timeout" mapstructure:"timeout"`
+	MaxBytes      int           `json:"max_bytes" mapstructure:"max_bytes"`
+	MaxConcurrent int           `json:"max_concurrent" mapstructure:"max_concurrent"`
+}
+
+func DefaultConfig() Config {
+	return Config{
+		Enabled:       true,
+		DefaultMode:   string(ModeAuto),
+		Timeout:       20 * time.Second,
+		MaxBytes:      2 * 1024 * 1024,
+		MaxConcurrent: 2,
+	}
+}
+
+// BatchTimeout derives the wall-clock ceiling for enriching one search response
+// (extract=true) from the per-URL budget. Workers run in ceil(count/MaxConcurrent)
+// waves; each worker's worst case is a raw fetch plus a rendered escalation, so a
+// single Extract is bounded by 2*Timeout. This keeps the batch bound an explicit
+// consequence of Timeout rather than a separate knob that can drift out of sync.
+func (c Config) BatchTimeout(count int) time.Duration {
+	c = c.Normalized()
+	if count <= 0 {
+		return c.Timeout
+	}
+	waves := (count + c.MaxConcurrent - 1) / c.MaxConcurrent
+	return time.Duration(waves) * 2 * c.Timeout
+}
+
+func (c Config) Normalized() Config {
+	def := DefaultConfig()
+	if c.DefaultMode == "" {
+		c.DefaultMode = def.DefaultMode
+	}
+	if c.Timeout <= 0 {
+		c.Timeout = def.Timeout
+	}
+	if c.MaxBytes <= 0 {
+		c.MaxBytes = def.MaxBytes
+	}
+	if c.MaxConcurrent <= 0 {
+		c.MaxConcurrent = def.MaxConcurrent
+	}
+	return c
+}
--- a/extract/content.go
+++ b/extract/content.go
@@ -0,0 +1,164 @@
+package extract
+
+import (
+	"bytes"
+	"net/url"
+	"regexp"
+	"strings"
+
+	"github.com/JohannesKaufmann/html-to-markdown/v2/converter"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/base"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/commonmark"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/strikethrough"
+	"github.com/JohannesKaufmann/html-to-markdown/v2/plugin/table"
+	"github.com/PuerkitoBio/goquery"
+	"github.com/markusmobius/go-trafilatura"
+	"golang.org/x/net/html"
+)
+
+var blankLineRun = regexp.MustCompile(`\n{3,}`)
+
+type contentResult struct {
+	HTML        string
+	Text        string
+	Markdown    string
+	Title       string
+	Description string
+	Lang        string
+}
+
+// minCleanTextRunes is the trimmed-text floor below which a clean (article-only)
+// extraction is treated as too thin. trafilatura strips everything it classifies
+// as boilerplate, which guts landing pages, doc indexes, and dashboards where the
+// "chrome" is the actual information. When the cleaned text falls under this and
+// the page itself carried real content, we fall back to whole-readable-body
+// extraction so callers (and LLM agents) get the full page instead of a husk.
+const minCleanTextRunes = 250
+
+// extractContent turns a page into markdown/text. When clean is true (the
+// default) it uses trafilatura to keep only the main article body; if that pass
+// comes back too thin for a page that clearly had content, it transparently
+// falls back to full-body extraction. When clean is false it skips article
+// detection entirely and converts the whole readable <body>.
+func extractContent(htmlBytes []byte, baseURL string, clean bool) (contentResult, error) {
+	if !clean {
+		return extractFullBody(htmlBytes, baseURL)
+	}
+
+	var out contentResult
+	opts := trafilatura.Options{
+		EnableFallback:  true,
+		Focus:           trafilatura.Balanced,
+		ExcludeComments: true,
+		IncludeImages:   true,
+		IncludeLinks:    true,
+		Deduplicate:     true,
+	}
+	if parsed, err := url.Parse(baseURL); err == nil {
+		opts.OriginalURL = parsed
+	}
+	extracted, err := trafilatura.Extract(bytes.NewReader(htmlBytes), opts)
+	if err != nil {
+		return out, err
+	}
+	if extracted == nil || extracted.ContentNode == nil {
+		return extractFullBody(htmlBytes, baseURL)
+	}
+
+	var htmlBuf bytes.Buffer
+	if err := html.Render(&htmlBuf, extracted.ContentNode); err != nil {
+		return out, err
+	}
+	out.HTML = strings.TrimSpace(htmlBuf.String())
+	out.Text = strings.TrimSpace(extracted.ContentText)
+	out.Title = strings.TrimSpace(extracted.Metadata.Title)
+	out.Description = strings.TrimSpace(extracted.Metadata.Description)
+	out.Lang = strings.TrimSpace(extracted.Metadata.Language)
+
+	markdown, err := htmlToMarkdown(out.HTML, baseURL)
+	if err != nil {
+		return out, err
+	}
+	out.Markdown = normalizeMarkdown(markdown)
+
+	// trafilatura was too aggressive: the cleaned article is near-empty but the
+	// raw page had real visible text. Prefer the fuller readable-body pass.
+	if len([]rune(out.Text)) < minCleanTextRunes {
+		if full, ferr := extractFullBody(htmlBytes, baseURL); ferr == nil &&
+			len([]rune(full.Text)) > len([]rune(out.Text)) {
+			// Keep trafilatura's metadata (title/description/lang) when present;
+			// it is usually cleaner than what we derive from the full body.
+			full.Title = firstNonEmpty(out.Title, full.Title)
+			full.Description = firstNonEmpty(out.Description, full.Description)
+			full.Lang = firstNonEmpty(out.Lang, full.Lang)
+			return full, nil
+		}
+	}
+	return out, nil
+}
+
+// extractFullBody converts the whole readable <body> to markdown, stripping only
+// non-content elements (scripts, styles, nav/header/footer chrome is kept since
+// for landing pages and indexes that "chrome" is the information). This is the
+// raw-er extraction used for clean=false and as the thin-output fallback.
+func extractFullBody(htmlBytes []byte, baseURL string) (contentResult, error) {
+	var out contentResult
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(htmlBytes))
+	if err != nil {
+		return out, err
+	}
+	doc.Find("script,style,noscript,template,svg,iframe").Remove()
+
+	body := doc.Find("body").First()
+	if body.Length() == 0 {
+		body = doc.Selection
+	}
+	bodyHTML, err := body.Html()
+	if err != nil {
+		return out, err
+	}
+	out.HTML = strings.TrimSpace(bodyHTML)
+	out.Text = collapseBlankLines(strings.TrimSpace(body.Text()))
+	out.Title = strings.TrimSpace(doc.Find("title").First().Text())
+	if lang, ok := doc.Find("html").First().Attr("lang"); ok {
+		out.Lang = strings.TrimSpace(lang)
+	}
+
+	markdown, err := htmlToMarkdown(out.HTML, baseURL)
+	if err != nil {
+		return out, err
+	}
+	out.Markdown = normalizeMarkdown(markdown)
+	return out, nil
+}
+
+func htmlToMarkdown(htmlStr, baseURL string) (string, error) {
+	conv := converter.NewConverter(
+		converter.WithPlugins(
+			base.NewBasePlugin(),
+			commonmark.NewCommonmarkPlugin(),
+		),
+	)
+	conv.Register.Plugin(table.NewTablePlugin(table.WithSkipEmptyRows(true), table.WithHeaderPromotion(true)))
+	conv.Register.Plugin(strikethrough.NewStrikethroughPlugin())
+	return conv.ConvertString(htmlStr, converter.WithDomain(baseURL))
+}
+
+var whitespaceRun = regexp.MustCompile(`[ \t]+`)
+
+// collapseBlankLines tidies the raw .Text() of a full body: each line is trimmed,
+// intra-line whitespace runs collapse to a single space, and runs of blank lines
+// collapse to one (via normalizeMarkdown).
+func collapseBlankLines(text string) string {
+	lines := strings.Split(text, "\n")
+	for i, line := range lines {
+		lines[i] = strings.TrimSpace(whitespaceRun.ReplaceAllString(line, " "))
+	}
+	return normalizeMarkdown(strings.Join(lines, "\n"))
+}
+
+func normalizeMarkdown(markdown string) string {
+	markdown = strings.ReplaceAll(markdown, "\r\n", "\n")
+	markdown = blankLineRun.ReplaceAllString(markdown, "\n\n")
+	return strings.TrimSpace(markdown)
+}
--- a/extract/extractor.go
+++ b/extract/extractor.go
@@ -0,0 +1,209 @@
+package extract
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type Extractor struct {
+	RawFetch      RawFetcher
+	RenderedFetch RenderedFetcher
+	Cfg           Config
+}
+
+func (e *Extractor) Extract(ctx context.Context, req ExtractRequest) (*ExtractResult, error) {
+	startedAt := time.Now()
+	cfg := e.Cfg.Normalized()
+	req = normalizeRequest(req, cfg)
+	if req.URL == "" {
+		return nil, errors.New("url is required")
+	}
+	if _, err := url.ParseRequestURI(req.URL); err != nil {
+		return nil, fmt.Errorf("invalid url: %w", err)
+	}
+	if req.Mode == "" {
+		req.Mode = Mode(cfg.DefaultMode)
+	}
+	if req.Mode != ModeAuto && req.Mode != ModeFast && req.Mode != ModeRendered {
+		return nil, fmt.Errorf("invalid mode %q", req.Mode)
+	}
+
+	// A site's llms.txt is purpose-built markdown — when the caller opts in and
+	// the URL is a root, it beats anything we can scrape. Falls through silently
+	// to normal extraction when absent.
+	if req.UseLLMSTxt {
+		if result, ok := e.tryLLMSTxt(ctx, req, startedAt); ok {
+			return result, nil
+		}
+	}
+
+	var rawResult *ExtractResult
+	var rawErr error
+	if req.Mode != ModeRendered && e.RawFetch != nil {
+		rawResult, rawErr = e.extractFast(ctx, req, startedAt)
+		if req.Mode == ModeFast || goodEnough(rawResult, rawErr, req.MinRunes) {
+			return rawResult, rawErr
+		}
+	}
+
+	if e.RenderedFetch == nil {
+		return rawResult, rawErr
+	}
+	renderedResult, renderedErr := e.extractRendered(ctx, req, startedAt)
+	if renderedErr == nil && renderedResult != nil {
+		// In auto mode the raw pass already produced something usable but below
+		// the quality threshold. Only prefer the rendered pass when it actually
+		// recovered more content — a bot wall or consent page can render shorter
+		// than the raw HTML, and falling back to it would be a regression.
+		if rawErr == nil && rawResult != nil && textLength(rawResult) > textLength(renderedResult) {
+			return rawResult, nil
+		}
+		return renderedResult, nil
+	}
+	if rawResult != nil || rawErr != nil {
+		return rawResult, rawErr
+	}
+	return nil, renderedErr
+}
+
+func (e *Extractor) extractFast(ctx context.Context, req ExtractRequest, startedAt time.Time) (*ExtractResult, error) {
+	fetchCtx, cancel := context.WithTimeout(ctx, req.Timeout)
+	defer cancel()
+	resp, err := e.RawFetch(fetchCtx, req)
+	if err != nil {
+		return nil, err
+	}
+	if resp == nil {
+		return nil, errors.New("empty raw response")
+	}
+	return buildResult(req, resp, string(ModeFast), startedAt)
+}
+
+func (e *Extractor) extractRendered(ctx context.Context, req ExtractRequest, startedAt time.Time) (*ExtractResult, error) {
+	fetchCtx, cancel := context.WithTimeout(ctx, req.Timeout)
+	defer cancel()
+	resp, err := e.RenderedFetch(fetchCtx, req)
+	if err != nil {
+		return nil, err
+	}
+	if resp == nil {
+		return nil, errors.New("empty rendered response")
+	}
+	return buildResult(req, resp, string(ModeRendered), startedAt)
+}
+
+func normalizeRequest(req ExtractRequest, cfg Config) ExtractRequest {
+	req.URL = normalizeURL(strings.TrimSpace(req.URL))
+	if req.Timeout <= 0 {
+		req.Timeout = cfg.Timeout
+	}
+	if req.MaxBytes <= 0 {
+		req.MaxBytes = cfg.MaxBytes
+	}
+	if req.MaxBytes > 0 && req.MaxBytes < 64*1024 {
+		req.MaxBytes = 64 * 1024
+	}
+	req.Mode = Mode(strings.ToLower(strings.TrimSpace(string(req.Mode))))
+	return req
+}
+
+// normalizeURL defaults a missing scheme to https so callers can pass a bare host
+// (e.g. "kamaloff.ru"). URLs that already carry a scheme are left untouched.
+func normalizeURL(raw string) string {
+	if raw == "" || strings.Contains(raw, "://") || strings.HasPrefix(raw, "//") {
+		return raw
+	}
+	return "https://" + raw
+}
+
+func buildResult(req ExtractRequest, resp *FetchResponse, mode string, startedAt time.Time) (*ExtractResult, error) {
+	if err := classifyStatus(resp.StatusCode); err != nil {
+		return nil, err
+	}
+	body := resp.Body
+	if req.MaxBytes > 0 && len(body) > req.MaxBytes {
+		body = body[:req.MaxBytes]
+	}
+	doc, err := goquery.NewDocumentFromReader(bytes.NewReader(body))
+	if err != nil {
+		return nil, err
+	}
+	metadata := parseMetadata(doc, req.URL)
+	content, contentErr := extractContent(body, req.URL, !req.FullPage)
+	result := &ExtractResult{
+		URL:         req.URL,
+		Title:       firstNonEmpty(content.Title, metadata.Title),
+		Description: firstNonEmpty(content.Description, metadata.Description),
+		Markdown:    content.Markdown,
+		Text:        content.Text,
+		Headings:    metadata.Headings,
+		Links:       metadata.Links,
+		Canonical:   metadata.Canonical,
+		Lang:        firstNonEmpty(content.Lang, metadata.Lang),
+		SchemaOrg:   metadata.SchemaOrg,
+		OGTags:      metadata.OGTags,
+		Meta: ExtractMeta{
+			ModeUsed:  mode,
+			FetchedAt: time.Now().UTC().Format(time.RFC3339),
+			Bytes:     len(body),
+			TookMs:    time.Since(startedAt).Milliseconds(),
+		},
+	}
+	if contentErr != nil && strings.TrimSpace(result.Text) == "" {
+		return result, contentErr
+	}
+	return result, nil
+}
+
+func classifyStatus(status int) error {
+	if status == 0 {
+		return nil
+	}
+	if status == http.StatusForbidden || status == http.StatusUnauthorized {
+		return fmt.Errorf("blocked: HTTP %d", status)
+	}
+	if status == http.StatusTooManyRequests {
+		return fmt.Errorf("rate limited: HTTP %d", status)
+	}
+	if status < 200 || status >= 300 {
+		return fmt.Errorf("unexpected HTTP status %d", status)
+	}
+	return nil
+}
+
+// defaultMinRunes is the built-in auto-mode escalation floor.
+const defaultMinRunes = 200
+
+// goodEnough reports whether the raw pass is worth keeping instead of escalating
+// to a render. minRunes overrides the floor (<= 0 uses defaultMinRunes); the
+// headings shortcut accepts shorter structured pages, never below 120 runes.
+func goodEnough(result *ExtractResult, err error, minRunes int) bool {
+	if err != nil || result == nil {
+		return false
+	}
+	if minRunes <= 0 {
+		minRunes = defaultMinRunes
+	}
+	textLen := textLength(result)
+	if textLen >= minRunes {
+		return true
+	}
+	return textLen >= 120 && textLen >= minRunes-80 && len(result.Headings) > 0
+}
+
+// textLength returns the rune count of the trimmed extracted text, used both to
+// gate auto-mode escalation and to compare raw vs rendered output quality.
+func textLength(result *ExtractResult) int {
+	if result == nil {
+		return 0
+	}
+	return len([]rune(strings.TrimSpace(result.Text)))
+}
--- a/extract/extractor_test.go
+++ b/extract/extractor_test.go
@@ -0,0 +1,310 @@
+package extract
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// staticRaw returns a RawFetcher that always serves the same HTML with HTTP 200.
+func staticRaw(html string) RawFetcher {
+	return func(context.Context, ExtractRequest) (*FetchResponse, error) {
+		return &FetchResponse{StatusCode: 200, Body: []byte(html)}, nil
+	}
+}
+
+// runExtract builds an Extractor from the given fetchers and runs Extract,
+// failing the test on error.
+func runExtract(t *testing.T, raw RawFetcher, rendered RenderedFetcher, req ExtractRequest) *ExtractResult {
+	t.Helper()
+	extractor := Extractor{RawFetch: raw, RenderedFetch: rendered, Cfg: DefaultConfig()}
+	result, err := extractor.Extract(context.Background(), req)
+	if err != nil {
+		t.Fatalf("Extract() error = %v", err)
+	}
+	return result
+}
+
+func TestExtractFastStaticArticle(t *testing.T) {
+	html := `<!doctype html><html lang="en"><head>
+<title>Static Article</title>
+<meta name="description" content="A focused article">
+</head><body>
+<article>
+<h1>Static Article</h1>
+<p>This article has enough body text to be treated as useful extracted content. It explains how OpenSERP extracts target pages into markdown for grounding workflows, with clear paragraphs and useful links for downstream automation. The text is intentionally longer than the quality threshold.</p>
+<a href="/docs">Docs</a>
+</article>
+</body></html>`
+	result := runExtract(t, staticRaw(html), nil, ExtractRequest{URL: "https://example.com/post", Mode: ModeFast})
+	if result.Title != "Static Article" {
+		t.Fatalf("title = %q", result.Title)
+	}
+	if !strings.Contains(result.Markdown, "OpenSERP extracts target pages") {
+		t.Fatalf("markdown missing article body: %q", result.Markdown)
+	}
+	if len(result.Links) != 1 || result.Links[0].URL != "https://example.com/docs" {
+		t.Fatalf("links = %#v", result.Links)
+	}
+	if result.Meta.ModeUsed != string(ModeFast) {
+		t.Fatalf("mode_used = %q", result.Meta.ModeUsed)
+	}
+}
+
+func TestExtractAutoEscalatesThinShell(t *testing.T) {
+	raw := staticRaw(`<!doctype html><div id="root"></div><script src="/app.js"></script>`)
+	rendered := func(context.Context, ExtractRequest) (*FetchResponse, error) {
+		return &FetchResponse{StatusCode: 200, Body: []byte(`<!doctype html><article><h1>Rendered</h1><p>This rendered page contains enough meaningful article text after JavaScript execution to pass the extraction threshold and avoid returning an empty shell to callers.</p></article>`)}, nil
+	}
+	result := runExtract(t, raw, rendered, ExtractRequest{URL: "https://example.com/app", Mode: ModeAuto})
+	if result.Meta.ModeUsed != string(ModeRendered) {
+		t.Fatalf("mode_used = %q", result.Meta.ModeUsed)
+	}
+}
+
+func TestNormalizeURL(t *testing.T) {
+	cases := map[string]string{
+		"kamaloff.ru":            "https://kamaloff.ru",
+		"example.com/path?q=1":   "https://example.com/path?q=1",
+		"http://example.com":     "http://example.com",
+		"https://example.com":    "https://example.com",
+		"//example.com":          "//example.com",
+		"":                       "",
+		"socks5h://127.0.0.1:80": "socks5h://127.0.0.1:80",
+	}
+	for in, want := range cases {
+		if got := normalizeURL(in); got != want {
+			t.Errorf("normalizeURL(%q) = %q, want %q", in, got, want)
+		}
+	}
+}
+
+func TestExtractBareHostGetsScheme(t *testing.T) {
+	var fetched string
+	raw := func(_ context.Context, req ExtractRequest) (*FetchResponse, error) {
+		fetched = req.URL
+		return &FetchResponse{StatusCode: 200, Body: []byte(`<!doctype html><html><head><title>Home</title></head><body><article><p>` +
+			strings.Repeat("Bare hostnames must resolve to https and extract normally. ", 5) + `</p></article></body></html>`)}, nil
+	}
+	result := runExtract(t, raw, nil, ExtractRequest{URL: "kamaloff.ru", Mode: ModeFast})
+	if fetched != "https://kamaloff.ru" {
+		t.Fatalf("fetched %q, want https://kamaloff.ru", fetched)
+	}
+	if result.Title != "Home" {
+		t.Fatalf("title = %q", result.Title)
+	}
+}
+
+func TestExtractAutoMinRunesForcesEscalation(t *testing.T) {
+	// Raw body clears the default floor (kept as-is), but a high MinRunes raises
+	// the bar above the raw yield, forcing escalation to the richer rendered pass.
+	rawHTML := `<!doctype html><html lang="en"><head><title>Summary</title></head><body>
+<article><h1>Summary</h1><p>This raw article body is comfortably over the default two hundred rune floor, so without a higher threshold the auto pass would accept it and never render.</p></article>
+</body></html>`
+	rendered := func(context.Context, ExtractRequest) (*FetchResponse, error) {
+		return &FetchResponse{StatusCode: 200, Body: []byte(`<!doctype html><article><h1>Full</h1><p>` +
+			strings.Repeat("The rendered pass returns the complete article with substantially more prose than the server-sent summary, which is exactly what a caller raising the content floor is asking for. ", 4) +
+			`</p></article>`)}, nil
+	}
+
+	// Without the override, auto keeps the raw summary.
+	base := runExtract(t, staticRaw(rawHTML), rendered, ExtractRequest{URL: "https://example.com/x", Mode: ModeAuto})
+	if base.Meta.ModeUsed != string(ModeFast) {
+		t.Fatalf("baseline mode_used = %q, want fast (raw clears default floor)", base.Meta.ModeUsed)
+	}
+
+	// With a floor above the raw yield, auto escalates to rendered.
+	raised := runExtract(t, staticRaw(rawHTML), rendered, ExtractRequest{URL: "https://example.com/x", Mode: ModeAuto, MinRunes: 1000})
+	if raised.Meta.ModeUsed != string(ModeRendered) {
+		t.Fatalf("raised-floor mode_used = %q, want rendered", raised.Meta.ModeUsed)
+	}
+}
+
+func TestExtractAutoKeepsRawWhenRenderedThinner(t *testing.T) {
+	rawHTML := `<!doctype html><html lang="en"><head><title>Raw</title></head><body>
+<article><h1>Raw</h1><p>This raw HTML response already carries a substantial article body that sits just under the auto-mode quality threshold, yet still holds far more useful prose than the consent wall the rendered pass returns for this page.</p></article>
+</body></html>`
+	rendered := func(context.Context, ExtractRequest) (*FetchResponse, error) {
+		return &FetchResponse{StatusCode: 200, Body: []byte(`<!doctype html><article><p>Accept cookies to continue.</p></article>`)}, nil
+	}
+	result := runExtract(t, staticRaw(rawHTML), rendered, ExtractRequest{URL: "https://example.com/wall", Mode: ModeAuto})
+	if result.Meta.ModeUsed != string(ModeFast) {
+		t.Fatalf("expected raw result to win, mode_used = %q", result.Meta.ModeUsed)
+	}
+}
+
+func TestExtractCleanFallsBackOnThinArticle(t *testing.T) {
+	// A landing page: trafilatura strips the feature/nav chrome down to almost
+	// nothing, so the thin-output guard should fall back to full-body extraction
+	// and recover the visible text.
+	landing := `<!doctype html><html lang="en"><head><title>OpenSERP</title></head><body>
+<header><nav><a href="/docs">Docs</a><a href="/pricing">Pricing</a></nav></header>
+<main>
+<h1>OpenSERP — Free SERP API</h1>
+<section class="features">
+<div class="card"><h3>Google</h3><p>Scrape Google results with one call.</p></div>
+<div class="card"><h3>Bing</h3><p>Bing and Yandex supported out of the box.</p></div>
+<div class="card"><h3>Yandex</h3><p>Region targeting and UULE handling built in.</p></div>
+</section>
+<footer><p>Free and open source SERP scraping for everyone.</p></footer>
+</main>
+</body></html>`
+	result := runExtract(t, staticRaw(landing), nil, ExtractRequest{URL: "https://openserp.org", Mode: ModeFast})
+	for _, want := range []string{"Google", "Bing", "Yandex", "Region targeting"} {
+		if !strings.Contains(result.Text, want) {
+			t.Fatalf("full-body fallback dropped %q; text = %q", want, result.Text)
+		}
+	}
+}
+
+func TestExtractFullPageKeepsChrome(t *testing.T) {
+	page := `<!doctype html><html lang="en"><head><title>Dash</title></head><body>
+<nav><a href="/a">Alpha</a><a href="/b">Beta</a></nav>
+<article><h1>Article</h1><p>A genuine article body that easily clears the extraction quality threshold so the clean path would normally keep only this and discard the navigation links above it.</p></article>
+</body></html>`
+	// FullPage must retain the nav chrome that clean mode would strip.
+	result := runExtract(t, staticRaw(page), nil, ExtractRequest{URL: "https://example.com/dash", Mode: ModeFast, FullPage: true})
+	if !strings.Contains(result.Text, "Alpha") || !strings.Contains(result.Text, "Beta") {
+		t.Fatalf("full-page extraction dropped nav chrome; text = %q", result.Text)
+	}
+}
+
+func TestExtractLLMSTxtRootHit(t *testing.T) {
+	const llmsFull = `# Example Docs
+
+This is the full LLM-optimized corpus for the site. It contains far more useful, structured prose than scraping the rendered HTML landing page would ever surface, which is exactly why agents should prefer it when present at the site root.`
+	var fetched []string
+	raw := func(_ context.Context, req ExtractRequest) (*FetchResponse, error) {
+		fetched = append(fetched, req.URL)
+		if strings.HasSuffix(req.URL, "/llms-full.txt") {
+			return &FetchResponse{StatusCode: 200, Body: []byte(llmsFull)}, nil
+		}
+		return &FetchResponse{StatusCode: 404}, nil
+	}
+	result := runExtract(t, raw, nil, ExtractRequest{URL: "https://example.com", Mode: ModeFast, UseLLMSTxt: true})
+	if result.Meta.ModeUsed != "llms_txt" {
+		t.Fatalf("mode_used = %q, want llms_txt", result.Meta.ModeUsed)
+	}
+	if result.Title != "Example Docs" {
+		t.Fatalf("title = %q", result.Title)
+	}
+	if fetched[0] != "https://example.com/llms-full.txt" {
+		t.Fatalf("first probe = %q, want .../llms-full.txt", fetched[0])
+	}
+}
+
+func TestExtractLLMSTxtSkippedForDeepURL(t *testing.T) {
+	article := `<!doctype html><html><head><title>Post</title></head><body><article><h1>Post</h1><p>` +
+		strings.Repeat("This deep article page has its own substantial body content that must win over any site-level llms index. ", 4) +
+		`</p></article></body></html>`
+	var probedLLMS bool
+	raw := func(_ context.Context, req ExtractRequest) (*FetchResponse, error) {
+		if strings.Contains(req.URL, "llms") {
+			probedLLMS = true
+		}
+		return &FetchResponse{StatusCode: 200, Body: []byte(article)}, nil
+	}
+	result := runExtract(t, raw, nil, ExtractRequest{URL: "https://example.com/blog/post", Mode: ModeFast, UseLLMSTxt: true})
+	if probedLLMS {
+		t.Fatal("llms.txt was probed for a deep (non-root) URL")
+	}
+	if result.Meta.ModeUsed == "llms_txt" {
+		t.Fatal("deep URL must not resolve via llms.txt")
+	}
+}
+
+func TestExtractLLMSTxtRejectsHTMLShell(t *testing.T) {
+	// A site that answers unknown paths with its SPA index.html (200, but HTML).
+	const spaShell = `<!doctype html><html><head><title>App</title></head><body><div id="root"></div></body></html>`
+	raw := func(_ context.Context, req ExtractRequest) (*FetchResponse, error) {
+		if strings.Contains(req.URL, "llms") {
+			return &FetchResponse{StatusCode: 200, Body: []byte(spaShell)}, nil
+		}
+		return &FetchResponse{StatusCode: 200, Body: []byte(`<!doctype html><html><head><title>Home</title></head><body><article><p>` +
+			strings.Repeat("Real homepage article content that should be extracted normally. ", 5) + `</p></article></body></html>`)}, nil
+	}
+	result := runExtract(t, raw, nil, ExtractRequest{URL: "https://example.com", Mode: ModeFast, UseLLMSTxt: true})
+	if result.Meta.ModeUsed == "llms_txt" {
+		t.Fatal("HTML shell served at /llms.txt must be rejected")
+	}
+}
+
+// TestExtractHonorsContextCancellation proves a fetch aborts when the parent
+// context is cancelled. The derived batch deadline in the search-enrichment
+// path (Config.BatchTimeout) relies on exactly this: once the budget is spent,
+// in-flight Extract calls must return promptly with the context error rather
+// than running to their own per-fetch timeout.
+func TestExtractHonorsContextCancellation(t *testing.T) {
+	// A raw fetcher that hangs until its context is cancelled, mimicking a slow
+	// or unresponsive target.
+	hangingRaw := func(ctx context.Context, _ ExtractRequest) (*FetchResponse, error) {
+		<-ctx.Done()
+		return nil, ctx.Err()
+	}
+	extractor := Extractor{RawFetch: hangingRaw, Cfg: DefaultConfig()}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+
+	start := time.Now()
+	_, err := extractor.Extract(ctx, ExtractRequest{URL: "https://example.com/slow", Mode: ModeFast})
+	if err == nil {
+		t.Fatal("expected a context error, got nil")
+	}
+	// Must abort near the parent deadline, not run to the 20s per-fetch timeout.
+	if elapsed := time.Since(start); elapsed > 5*time.Second {
+		t.Fatalf("Extract ignored context cancellation; took %s", elapsed)
+	}
+}
+
+func TestBatchTimeoutDerivation(t *testing.T) {
+	cfg := DefaultConfig() // Timeout=20s, MaxConcurrent=2
+	cases := []struct {
+		count int
+		want  time.Duration
+	}{
+		{count: 0, want: 20 * time.Second},  // degenerate: one per-URL budget
+		{count: 1, want: 40 * time.Second},  // 1 wave  * 2 * 20s
+		{count: 2, want: 40 * time.Second},  // 1 wave  * 2 * 20s
+		{count: 3, want: 80 * time.Second},  // 2 waves * 2 * 20s
+		{count: 5, want: 120 * time.Second}, // 3 waves * 2 * 20s
+	}
+	for _, tc := range cases {
+		if got := cfg.BatchTimeout(tc.count); got != tc.want {
+			t.Errorf("BatchTimeout(%d) = %s, want %s", tc.count, got, tc.want)
+		}
+	}
+}
+
+func TestParseMetadataRichTags(t *testing.T) {
+	doc, err := documentFromString(`<!doctype html><html lang="en"><head>
+<meta property="og:title" content="OG title">
+<meta property="og:description" content="OG description">
+<meta name="twitter:card" content="summary">
+<link rel="canonical" href="/canonical">
+<script type="application/ld+json">{"@graph":[{"@type":"Article","headline":"One"},{"@type":"BreadcrumbList"}]}</script>
+</head><body><h1>Heading</h1><a href="/a">A link</a></body></html>`)
+	if err != nil {
+		t.Fatal(err)
+	}
+	meta := parseMetadata(doc, "https://example.com/page")
+	if meta.Title != "OG title" || meta.Description != "OG description" {
+		t.Fatalf("metadata title/description = %q / %q", meta.Title, meta.Description)
+	}
+	if meta.Canonical != "https://example.com/canonical" {
+		t.Fatalf("canonical = %q", meta.Canonical)
+	}
+	if len(meta.SchemaOrg) != 2 {
+		t.Fatalf("schema_org count = %d", len(meta.SchemaOrg))
+	}
+	if meta.OGTags["twitter:card"] != "summary" {
+		t.Fatalf("og_tags = %#v", meta.OGTags)
+	}
+}
+
+func documentFromString(raw string) (*goquery.Document, error) {
+	return goquery.NewDocumentFromReader(strings.NewReader(raw))
+}
--- a/extract/llmstxt.go
+++ b/extract/llmstxt.go
@@ -0,0 +1,109 @@
+package extract
+
+import (
+	"context"
+	"net/url"
+	"strings"
+	"time"
+)
+
+// llmsTxtCandidates are the well-known LLM-optimized markdown files, tried in
+// order of richness. /llms-full.txt is the concatenated full corpus;
+// /llms.txt is the curated index. See https://llmstxt.org/.
+var llmsTxtCandidates = []string{"/llms-full.txt", "/llms.txt"}
+
+// minLLMSTxtRunes guards against a site answering an unknown path with its SPA
+// index.html (HTTP 200, but HTML, not markdown). Anything shorter than this, or
+// that sniffs as HTML, is rejected so we fall through to normal extraction.
+const minLLMSTxtRunes = 200
+
+// isSiteRoot reports whether the URL points at a site root, where /llms.txt is
+// meaningful. We only probe roots because /llms.txt describes the whole site —
+// for a deep page (e.g. /blog/post) it would return the site index and miss the
+// content the caller actually asked for.
+func isSiteRoot(rawURL string) bool {
+	parsed, err := url.Parse(rawURL)
+	if err != nil {
+		return false
+	}
+	path := strings.Trim(parsed.Path, "/")
+	return path == ""
+}
+
+// tryLLMSTxt probes the well-known llms.txt files at the site root using the raw
+// fetcher. It returns (result, true) on the first usable hit, or (nil, false) to
+// signal the caller should fall through to normal HTML extraction. Errors are
+// swallowed deliberately: a missing/!200/HTML llms.txt is the common case and
+// must never fail the extract.
+func (e *Extractor) tryLLMSTxt(ctx context.Context, req ExtractRequest, startedAt time.Time) (*ExtractResult, bool) {
+	if e.RawFetch == nil || !isSiteRoot(req.URL) {
+		return nil, false
+	}
+	base, err := url.Parse(req.URL)
+	if err != nil {
+		return nil, false
+	}
+	for _, candidate := range llmsTxtCandidates {
+		ref, err := url.Parse(candidate)
+		if err != nil {
+			continue
+		}
+		probe := req
+		probe.URL = base.ResolveReference(ref).String()
+
+		fetchCtx, cancel := context.WithTimeout(ctx, req.Timeout)
+		resp, ferr := e.RawFetch(fetchCtx, probe)
+		cancel()
+		if ferr != nil || resp == nil || resp.StatusCode != 200 {
+			continue
+		}
+		body := resp.Body
+		if req.MaxBytes > 0 && len(body) > req.MaxBytes {
+			body = body[:req.MaxBytes]
+		}
+		text := strings.TrimSpace(string(body))
+		if len([]rune(text)) < minLLMSTxtRunes || looksLikeHTML(text) {
+			continue
+		}
+		return &ExtractResult{
+			URL:      req.URL,
+			Title:    firstNonEmpty(llmsTxtTitle(text), req.URL),
+			Markdown: normalizeMarkdown(text),
+			Text:     text,
+			Meta: ExtractMeta{
+				ModeUsed:  "llms_txt",
+				FetchedAt: time.Now().UTC().Format(time.RFC3339),
+				Bytes:     len(body),
+				TookMs:    time.Since(startedAt).Milliseconds(),
+			},
+		}, true
+	}
+	return nil, false
+}
+
+// looksLikeHTML rejects bodies that are actually HTML (common when a site serves
+// its SPA shell for unknown paths) rather than the markdown we want.
+func looksLikeHTML(text string) bool {
+	head := strings.ToLower(strings.TrimSpace(text))
+	if len(head) > 256 {
+		head = head[:256]
+	}
+	return strings.HasPrefix(head, "<!doctype html") ||
+		strings.HasPrefix(head, "<html") ||
+		strings.Contains(head, "<head>") ||
+		strings.Contains(head, "<body")
+}
+
+// llmsTxtTitle pulls a title from the first markdown H1 ("# Title"), if present.
+func llmsTxtTitle(text string) string {
+	for _, line := range strings.Split(text, "\n") {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, "# ") {
+			return strings.TrimSpace(strings.TrimPrefix(line, "# "))
+		}
+		if line != "" {
+			break
+		}
+	}
+	return ""
+}
--- a/extract/metadata.go
+++ b/extract/metadata.go
@@ -0,0 +1,155 @@
+package extract
+
+import (
+	"encoding/json"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+type pageMetadata struct {
+	Title       string
+	Description string
+	Canonical   string
+	Lang        string
+	OGTags      map[string]string
+	SchemaOrg   []json.RawMessage
+	Headings    []Heading
+	Links       []Link
+}
+
+func parseMetadata(doc *goquery.Document, baseURL string) pageMetadata {
+	var meta pageMetadata
+	if doc == nil {
+		return meta
+	}
+	meta.Title = firstNonEmpty(
+		strings.TrimSpace(doc.Find("title").First().Text()),
+		metaContent(doc, `meta[property="og:title"]`),
+		metaContent(doc, `meta[name="twitter:title"]`),
+	)
+	meta.Description = firstNonEmpty(
+		metaContent(doc, `meta[name="description"]`),
+		metaContent(doc, `meta[property="og:description"]`),
+		metaContent(doc, `meta[name="twitter:description"]`),
+	)
+	if canonical, ok := doc.Find(`link[rel="canonical"]`).First().Attr("href"); ok {
+		meta.Canonical = resolveURL(baseURL, canonical)
+	}
+	if lang, ok := doc.Find("html").First().Attr("lang"); ok {
+		meta.Lang = strings.TrimSpace(lang)
+	}
+
+	meta.OGTags = map[string]string{}
+	doc.Find(`meta[property^="og:"], meta[name^="twitter:"]`).Each(func(_ int, sel *goquery.Selection) {
+		key, _ := sel.Attr("property")
+		if key == "" {
+			key, _ = sel.Attr("name")
+		}
+		value, _ := sel.Attr("content")
+		key = strings.TrimSpace(key)
+		value = strings.TrimSpace(value)
+		if key != "" && value != "" {
+			meta.OGTags[key] = value
+		}
+	})
+	if len(meta.OGTags) == 0 {
+		meta.OGTags = nil
+	}
+
+	doc.Find(`script[type="application/ld+json"]`).Each(func(_ int, sel *goquery.Selection) {
+		appendJSONLD(&meta.SchemaOrg, strings.TrimSpace(sel.Text()))
+	})
+
+	doc.Find("h1,h2,h3,h4,h5,h6").Each(func(_ int, sel *goquery.Selection) {
+		level, _ := strconv.Atoi(strings.TrimPrefix(strings.ToLower(goquery.NodeName(sel)), "h"))
+		text := strings.TrimSpace(sel.Text())
+		if level >= 1 && level <= 6 && text != "" {
+			meta.Headings = append(meta.Headings, Heading{Level: level, Text: collapseWhitespace(text)})
+		}
+	})
+
+	doc.Find("a[href]").Each(func(_ int, sel *goquery.Selection) {
+		if len(meta.Links) >= 100 {
+			return
+		}
+		href, _ := sel.Attr("href")
+		resolved := resolveURL(baseURL, href)
+		if resolved == "" {
+			return
+		}
+		text := collapseWhitespace(strings.TrimSpace(sel.Text()))
+		meta.Links = append(meta.Links, Link{Text: text, URL: resolved})
+	})
+	return meta
+}
+
+func metaContent(doc *goquery.Document, selector string) string {
+	value, _ := doc.Find(selector).First().Attr("content")
+	return strings.TrimSpace(value)
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, value := range values {
+		if strings.TrimSpace(value) != "" {
+			return strings.TrimSpace(value)
+		}
+	}
+	return ""
+}
+
+func appendJSONLD(dst *[]json.RawMessage, raw string) {
+	if raw == "" {
+		return
+	}
+	var decoded any
+	if err := json.Unmarshal([]byte(raw), &decoded); err != nil {
+		return
+	}
+	switch value := decoded.(type) {
+	case map[string]any:
+		if graph, ok := value["@graph"].([]any); ok {
+			for _, item := range graph {
+				if data, err := json.Marshal(item); err == nil {
+					*dst = append(*dst, data)
+				}
+			}
+			return
+		}
+	case []any:
+		for _, item := range value {
+			if data, err := json.Marshal(item); err == nil {
+				*dst = append(*dst, data)
+			}
+		}
+		return
+	}
+	if data, err := json.Marshal(decoded); err == nil {
+		*dst = append(*dst, data)
+	}
+}
+
+func resolveURL(baseURL string, raw string) string {
+	raw = strings.TrimSpace(raw)
+	if raw == "" || strings.HasPrefix(raw, "#") || strings.HasPrefix(strings.ToLower(raw), "javascript:") {
+		return ""
+	}
+	parsed, err := url.Parse(raw)
+	if err != nil {
+		return ""
+	}
+	if parsed.IsAbs() {
+		return parsed.String()
+	}
+	base, err := url.Parse(baseURL)
+	if err != nil {
+		return ""
+	}
+	return base.ResolveReference(parsed).String()
+}
+
+func collapseWhitespace(value string) string {
+	return strings.Join(strings.Fields(value), " ")
+}
--- a/extract/types.go
+++ b/extract/types.go
@@ -0,0 +1,76 @@
+package extract
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+)
+
+type Mode string
+
+const (
+	ModeAuto     Mode = "auto"
+	ModeFast     Mode = "fast"
+	ModeRendered Mode = "rendered"
+)
+
+type ExtractRequest struct {
+	URL      string
+	Mode     Mode
+	ProxyURL string
+	LangCode string
+	Timeout  time.Duration
+	MaxBytes int
+	// FullPage selects whole-readable-body extraction instead of the default
+	// article-only (trafilatura) extraction. LLM agents fetching arbitrary URLs
+	// often want the full page; FullPage keeps nav/feature/landing content that
+	// trafilatura strips. The zero value (false) preserves the cleaned default.
+	FullPage bool
+	// UseLLMSTxt, when set and the URL is a site root, probes /llms-full.txt then
+	// /llms.txt and returns that LLM-optimized markdown instead of scraping HTML.
+	UseLLMSTxt bool
+	// MinRunes is the per-request auto-mode escalation floor: raw output below
+	// this many extracted-text runes escalates to a render. 0 uses defaultMinRunes.
+	MinRunes int
+}
+
+type ExtractResult struct {
+	URL         string            `json:"url"`
+	Title       string            `json:"title"`
+	Description string            `json:"description"`
+	Markdown    string            `json:"markdown"`
+	Text        string            `json:"text"`
+	Headings    []Heading         `json:"headings,omitempty"`
+	Links       []Link            `json:"links,omitempty"`
+	Canonical   string            `json:"canonical,omitempty"`
+	Lang        string            `json:"lang,omitempty"`
+	SchemaOrg   []json.RawMessage `json:"schema_org,omitempty"`
+	OGTags      map[string]string `json:"og_tags,omitempty"`
+	Meta        ExtractMeta       `json:"meta"`
+}
+
+type Heading struct {
+	Level int    `json:"level"`
+	Text  string `json:"text"`
+}
+
+type Link struct {
+	Text string `json:"text"`
+	URL  string `json:"url"`
+}
+
+type ExtractMeta struct {
+	ModeUsed  string `json:"mode_used"`
+	FetchedAt string `json:"fetched_at"`
+	Bytes     int    `json:"bytes"`
+	TookMs    int64  `json:"took_ms"`
+}
+
+type FetchResponse struct {
+	StatusCode int
+	Body       []byte
+}
+
+type RawFetcher func(ctx context.Context, req ExtractRequest) (*FetchResponse, error)
+
+type RenderedFetcher func(ctx context.Context, req ExtractRequest) (*FetchResponse, error)