openserp/extract/metadata.go

package extract

import (
	"encoding/json"
	"net/url"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

type pageMetadata struct {
	Title       string
	Description string
	Canonical   string
	Lang        string
	OGTags      map[string]string
	SchemaOrg   []json.RawMessage
	Headings    []Heading
	Links       []Link
}

func parseMetadata(doc *goquery.Document, baseURL string) pageMetadata {
	var meta pageMetadata
	if doc == nil {
		return meta
	}
	meta.Title = firstNonEmpty(
		strings.TrimSpace(doc.Find("title").First().Text()),
		metaContent(doc, `meta[property="og:title"]`),
		metaContent(doc, `meta[name="twitter:title"]`),
	)
	meta.Description = firstNonEmpty(
		metaContent(doc, `meta[name="description"]`),
		metaContent(doc, `meta[property="og:description"]`),
		metaContent(doc, `meta[name="twitter:description"]`),
	)
	if canonical, ok := doc.Find(`link[rel="canonical"]`).First().Attr("href"); ok {
		meta.Canonical = resolveURL(baseURL, canonical)
	}
	if lang, ok := doc.Find("html").First().Attr("lang"); ok {
		meta.Lang = strings.TrimSpace(lang)
	}

	meta.OGTags = map[string]string{}
	doc.Find(`meta[property^="og:"], meta[name^="twitter:"]`).Each(func(_ int, sel *goquery.Selection) {
		key, _ := sel.Attr("property")
		if key == "" {
			key, _ = sel.Attr("name")
		}
		value, _ := sel.Attr("content")
		key = strings.TrimSpace(key)
		value = strings.TrimSpace(value)
		if key != "" && value != "" {
			meta.OGTags[key] = value
		}
	})
	if len(meta.OGTags) == 0 {
		meta.OGTags = nil
	}

	doc.Find(`script[type="application/ld+json"]`).Each(func(_ int, sel *goquery.Selection) {
		appendJSONLD(&meta.SchemaOrg, strings.TrimSpace(sel.Text()))
	})

	doc.Find("h1,h2,h3,h4,h5,h6").Each(func(_ int, sel *goquery.Selection) {
		level, _ := strconv.Atoi(strings.TrimPrefix(strings.ToLower(goquery.NodeName(sel)), "h"))
		text := strings.TrimSpace(sel.Text())
		if level >= 1 && level <= 6 && text != "" {
			meta.Headings = append(meta.Headings, Heading{Level: level, Text: collapseWhitespace(text)})
		}
	})

	doc.Find("a[href]").Each(func(_ int, sel *goquery.Selection) {
		if len(meta.Links) >= 100 {
			return
		}
		href, _ := sel.Attr("href")
		resolved := resolveURL(baseURL, href)
		if resolved == "" {
			return
		}
		text := collapseWhitespace(strings.TrimSpace(sel.Text()))
		meta.Links = append(meta.Links, Link{Text: text, URL: resolved})
	})
	return meta
}

func metaContent(doc *goquery.Document, selector string) string {
	value, _ := doc.Find(selector).First().Attr("content")
	return strings.TrimSpace(value)
}

func firstNonEmpty(values ...string) string {
	for _, value := range values {
		if strings.TrimSpace(value) != "" {
			return strings.TrimSpace(value)
		}
	}
	return ""
}

func appendJSONLD(dst *[]json.RawMessage, raw string) {
	if raw == "" {
		return
	}
	var decoded any
	if err := json.Unmarshal([]byte(raw), &decoded); err != nil {
		return
	}
	switch value := decoded.(type) {
	case map[string]any:
		if graph, ok := value["@graph"].([]any); ok {
			for _, item := range graph {
				if data, err := json.Marshal(item); err == nil {
					*dst = append(*dst, data)
				}
			}
			return
		}
	case []any:
		for _, item := range value {
			if data, err := json.Marshal(item); err == nil {
				*dst = append(*dst, data)
			}
		}
		return
	}
	if data, err := json.Marshal(decoded); err == nil {
		*dst = append(*dst, data)
	}
}

func resolveURL(baseURL string, raw string) string {
	raw = strings.TrimSpace(raw)
	if raw == "" || strings.HasPrefix(raw, "#") || strings.HasPrefix(strings.ToLower(raw), "javascript:") {
		return ""
	}
	parsed, err := url.Parse(raw)
	if err != nil {
		return ""
	}
	if parsed.IsAbs() {
		return parsed.String()
	}
	base, err := url.Parse(baseURL)
	if err != nil {
		return ""
	}
	return base.ResolveReference(parsed).String()
}

func collapseWhitespace(value string) string {
	return strings.Join(strings.Fields(value), " ")
}