openserp/core/enrichment_domain.go

package core

import (
	_ "embed"
	"os"
	"strings"
	"sync"

	"golang.org/x/net/publicsuffix"
	"gopkg.in/yaml.v3"
)

//go:embed enrichment_domains.yaml
var defaultEnrichmentDomainsYAML []byte

type enrichmentDomainsFile struct {
	DomainSourceHints  map[string]string `yaml:"domain_source_hints"`
	NewsDomains        []string          `yaml:"news_domains"`
	ForumDomains       []string          `yaml:"forum_domains"`
	MarketplaceDomains []string          `yaml:"marketplace_domains"`
	SocialDomains      []string          `yaml:"social_domains"`
}

type enrichmentDomainsConfig struct {
	DomainSourceHints  map[string]string
	NewsDomains        map[string]bool
	ForumDomains       map[string]bool
	MarketplaceDomains map[string]bool
	SocialDomains      map[string]bool
}

var (
	enrichmentDomainsOnce sync.Once
	enrichmentDomains     enrichmentDomainsConfig
)

// EnrichDomainInfo derives TLD/category signals from a bare hostname.
func EnrichDomainInfo(domain string) *DomainInfo {
	if domain == "" {
		return nil
	}

	domain = normalizeDomain(domain)
	tld, sld := splitDomain(domain)
	cfg := loadEnrichmentDomains()

	info := &DomainInfo{
		TLD:      tld,
		SLD:      sld,
		Category: domainCategory(domain, tld, cfg),
	}
	return info
}

// ClassifyURL returns a rough content-type and source hint derived from the
// URL path alone; no network calls.
func ClassifyURL(rawURL, domain string) *Classification {
	if rawURL == "" && domain == "" {
		return nil
	}

	contentType := classifyContentType(rawURL)
	sourceHint := classifySourceHint(domain)
	if contentType == "webpage" && sourceHint == "" {
		return nil
	}

	return &Classification{
		ContentType: contentType,
		SourceHint:  sourceHint,
	}
}

func domainCategory(domain, tld string, cfg enrichmentDomainsConfig) string {
	switch {
	case isGovTLD(domain, tld):
		return "gov"
	case isEduTLD(domain, tld):
		return "edu"
	case isMilTLD(tld):
		return "mil"
	case cfg.NewsDomains[domain]:
		return "news"
	case cfg.ForumDomains[domain]:
		return "forum"
	case cfg.MarketplaceDomains[domain]:
		return "marketplace"
	case cfg.SocialDomains[domain]:
		return "social"
	default:
		return ""
	}
}

// splitDomain returns (public suffix, registrable domain label).
func splitDomain(domain string) (tld, sld string) {
	domain = normalizeDomain(domain)
	if domain == "" {
		return "", ""
	}

	suffix, icann := publicsuffix.PublicSuffix(domain)
	if suffix == "" || !icann {
		parts := strings.Split(domain, ".")
		if len(parts) < 2 {
			return domain, ""
		}
		return parts[len(parts)-1], parts[len(parts)-2]
	}

	registrable, err := publicsuffix.EffectiveTLDPlusOne(domain)
	if err != nil {
		parts := strings.Split(domain, ".")
		if len(parts) < 2 {
			return suffix, ""
		}
		return suffix, parts[len(parts)-2]
	}

	sld = strings.TrimSuffix(registrable, "."+suffix)
	return suffix, sld
}

func isGovTLD(domain, tld string) bool {
	return tld == "gov" || strings.HasSuffix(tld, ".gov") || strings.HasSuffix(domain, ".gov")
}

func isEduTLD(domain, tld string) bool {
	return tld == "edu" || strings.HasSuffix(tld, ".edu") || tld == "ac.uk" ||
		strings.HasSuffix(domain, ".edu") || strings.HasSuffix(domain, ".ac.uk")
}

func isMilTLD(tld string) bool {
	return tld == "mil"
}

func classifyContentType(rawURL string) string {
	lower := strings.ToLower(rawURL)
	switch {
	case strings.Contains(lower, "/wiki/"):
		return "article"
	case strings.HasSuffix(lower, ".pdf") || strings.Contains(lower, ".pdf?"):
		return "document"
	case strings.Contains(lower, "/watch?v=") || strings.Contains(lower, "/video/") ||
		strings.Contains(lower, "/videos/"):
		return "video"
	case strings.Contains(lower, "/forum/") || strings.Contains(lower, "/thread/") ||
		strings.Contains(lower, "/discussion/") || strings.Contains(lower, "/t/") ||
		strings.Contains(lower, "/questions/") || strings.Contains(lower, "/q/"):
		return "forum_thread"
	case strings.Contains(lower, "/blog/") || strings.Contains(lower, "/post/") ||
		strings.Contains(lower, "/article/") || strings.Contains(lower, "/news/"):
		return "article"
	default:
		return "webpage"
	}
}

func classifySourceHint(domain string) string {
	cfg := loadEnrichmentDomains()
	domain = normalizeDomain(domain)
	if hint, ok := cfg.DomainSourceHints[domain]; ok {
		return hint
	}
	// Fall back to the registrable domain so subdomain hosts (e.g.
	// megadeth.fandom.com, zh.m.wikipedia.org) match a hint keyed on the
	// registrable domain (fandom.com, wikipedia.org).
	if _, sld := splitDomain(domain); sld != "" {
		if registrable, err := publicsuffix.EffectiveTLDPlusOne(domain); err == nil {
			if hint, ok := cfg.DomainSourceHints[registrable]; ok {
				return hint
			}
		}
	}
	return ""
}

func loadEnrichmentDomains() enrichmentDomainsConfig {
	enrichmentDomainsOnce.Do(func() {
		enrichmentDomains = parseEnrichmentDomains(defaultEnrichmentDomainsYAML)
		if path := strings.TrimSpace(os.Getenv("OPENSERP_ENRICHMENT_DOMAINS_FILE")); path != "" {
			if data, err := os.ReadFile(path); err == nil {
				enrichmentDomains = parseEnrichmentDomains(data)
			}
		}
	})
	return enrichmentDomains
}

func parseEnrichmentDomains(data []byte) enrichmentDomainsConfig {
	cfg := enrichmentDomainsConfig{
		DomainSourceHints:  map[string]string{},
		NewsDomains:        map[string]bool{},
		ForumDomains:       map[string]bool{},
		MarketplaceDomains: map[string]bool{},
		SocialDomains:      map[string]bool{},
	}

	var file enrichmentDomainsFile
	if err := yaml.Unmarshal(data, &file); err != nil {
		return cfg
	}

	for domain, hint := range file.DomainSourceHints {
		domain = normalizeDomain(domain)
		hint = strings.TrimSpace(hint)
		if domain != "" && hint != "" {
			cfg.DomainSourceHints[domain] = hint
		}
	}
	fillDomainSet(cfg.NewsDomains, file.NewsDomains)
	fillDomainSet(cfg.ForumDomains, file.ForumDomains)
	fillDomainSet(cfg.MarketplaceDomains, file.MarketplaceDomains)
	fillDomainSet(cfg.SocialDomains, file.SocialDomains)

	return cfg
}

func fillDomainSet(dst map[string]bool, domains []string) {
	for _, domain := range domains {
		domain = normalizeDomain(domain)
		if domain != "" {
			dst[domain] = true
		}
	}
}

func normalizeDomain(domain string) string {
	domain = strings.ToLower(strings.TrimSpace(domain))
	domain = strings.TrimPrefix(domain, "www.")
	return strings.TrimSuffix(domain, ".")
}