openserp/core/clusters.go

package core

import (
	"crypto/md5"
	"encoding/hex"
	"sort"
)

// BuildClusters groups results by normalized URL and scores them by cross-engine
// agreement. enginesQueried is the total number of engines that were asked
// (denominator for the score formula).
//
// Score = sum(1/rank for each occurrence) / enginesQueried, capped at 1.0.
func BuildClusters(results []Result, enginesQueried int) []Cluster {
	if enginesQueried <= 0 {
		enginesQueried = 1
	}

	type clusterAccum struct {
		occurrences  []ClusterOccurrence
		scoreSum     float64
		bestRank     int
		title        string
		canonicalURL string
		domain       string
	}

	// Group by result ID (which is derived from normalized URL + engine).
	// For clustering we group by normalized URL regardless of engine, so we
	// re-key on NormalizeURLForClustering(result.URL).
	byURL := map[string]*clusterAccum{}
	urlOrder := []string{}

	for _, r := range results {
		norm := NormalizeURLForClustering(r.URL)
		if norm == "" {
			continue
		}
		acc, exists := byURL[norm]
		if !exists {
			acc = &clusterAccum{
				bestRank:     r.Rank,
				title:        r.Title,
				canonicalURL: r.URL,
				domain:       r.Domain,
			}
			byURL[norm] = acc
			urlOrder = append(urlOrder, norm)
		}
		rank := r.Rank
		if rank <= 0 {
			rank = 1
		}
		acc.scoreSum += 1.0 / float64(rank)
		acc.occurrences = append(acc.occurrences, ClusterOccurrence{
			Engine:   r.Engine,
			Rank:     r.Rank,
			ResultID: r.ID,
		})
		if r.Rank > 0 && (acc.bestRank <= 0 || r.Rank < acc.bestRank) {
			acc.bestRank = r.Rank
			acc.title = r.Title
			acc.canonicalURL = r.URL
			acc.domain = r.Domain
		}
	}

	clusters := make([]Cluster, 0, len(byURL))
	for _, norm := range urlOrder {
		acc := byURL[norm]
		score := acc.scoreSum / float64(enginesQueried)
		if score > 1.0 {
			score = 1.0
		}
		clusters = append(clusters, Cluster{
			ID:           buildClusterID(norm),
			CanonicalURL: acc.canonicalURL,
			Domain:       acc.domain,
			Title:        acc.title,
			Occurrences:  acc.occurrences,
			EnginesCount: len(acc.occurrences),
			BestRank:     acc.bestRank,
			Score:        roundScore(score),
		})
	}

	// Sort by score descending, then best_rank ascending as tiebreak.
	sort.Slice(clusters, func(i, j int) bool {
		if clusters[i].Score != clusters[j].Score {
			return clusters[i].Score > clusters[j].Score
		}
		return clusters[i].BestRank < clusters[j].BestRank
	})
	return clusters
}

func buildClusterID(normalizedURL string) string {
	h := md5.Sum([]byte(normalizedURL))
	return "c_" + hex.EncodeToString(h[:responseIDBytes])
}

func roundScore(s float64) float64 {
	return float64(int(s*100+0.5)) / 100
}