[turbopack] Add clusters to chunking heuristics (#95157)

The goal of this PR increase the expected benefit of merging two chunks
when their overlapping chunk groups are in a "cluster" (a cluster is a
set of routes that are commonly visited together). This is because in
these cases the Z+Z case (one of six in the N=2 set of cases) is more
likely than others. To this I have increased the probability of the Z+Z
case.

At the moment, to do this I've done the following to the probabilities:

```rust
/*
MERGED CASE (N = 2):
case X + X (p = (a_rem/groups) * ((a_rem - 1)/rem_g)): size = a_size, requests = 1
case Y + Y (p = (b_rem/groups) * ((b_rem - 1)/rem_g)): size = b_size, requests = 1
case Z + Z (p = (o_groups/groups) * (o_groups - 1)/rem_g): size = (a_size + b_size), requests = 1
case X + Y (p = (a_rem/groups) * (b_rem/rem_g) + (b_rem/groups) * (a_rem/rem_g)): size = a_size + b_size, requests = 2
case X + Z (p = (a_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (a_rem/rem_g)): size = a_size + (a_size + b_size), requests = 2
case Y + Z (p = (b_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (b_rem/rem_g)): size = b_size + (a_size + b_size), requests = 2

Request count is different in this case: Z + Z (better)
Requests size is different (worse) in these cases: X + Z, Y + Z

Each cost / benefit is weighted by the probabilities above. There are cases when
we know that Z + Z is more likely due to common user behaviour. This is based on
the "cluster" chunking heuristic we provide. We increase P(Z + Z) when two or more
routes in a cluster request both chunk items together (ie. request Z). P(X + Z) and
P(Y + Z) are therefore less likely. To increase P(Z + Z) while maintaining a total
probability of 1, we do the following (when chunk items overlap in a cluster):

P'(X + Z) = (1/2) * P(X + Z)
P'(Y + Z) = (1/2) * P(Y + Z)
P'(Z + Z) = P(Z + Z) + (1/2) * P(X + Z) + (1/2) * P(Y + Z)

Otherwise:

P'(X + Z) = P(X + Z)
P'(Y + Z) = P(Y + Z)
P'(Z + Z) = P(Z + Z)
*/
```

This increases `P'(Z + Z)` while reducing `P'(X + Z)` and `P'(Y + Z)`.
It is a bit of a blunt tool for doing this, however. So if anyone has
other suggestions for how to change this probabilities while maintaining
existing behaviour - let me know!

crates/next-core/src/next_config.rs

@@ -7,7 +7,7 @@ use either::Either;
 use rustc_hash::FxHashSet;
 use serde::{Deserialize, Deserializer, Serialize};
 use serde_json::Value as JsonValue;
-use turbo_esregex::EsRegex;
+use turbo_esregex::{EsRegex, EsRegexSet};
 use turbo_rcstr::{RcStr, rcstr};
 use turbo_tasks::{
     FxIndexMap, NonLocalValue, OperationValue, ResolvedVc, TryJoinIterExt, Vc,
@@ -1150,6 +1150,9 @@ const DEFAULT_WEIGHT_DISTRIBUTION: f32 = 0.1;
 )]
 #[serde(rename_all = "camelCase")]
 pub struct TurbopackChunkingConfig {
+    /// Groups of pages commonly visited together, each defined by a list of regular expressions
+    /// matched against the route pathname. The cluster ID is the index in this list.
+    clusters: Option<Vec<Vec<RegexComponents>>>,
     /// A number between `0.0..=1.0`. Higher values weight the benefit of merging
     /// chunks for a single page load more heavily. A site's bounce rate is a good
     /// approximation if you don't have a better value.
@@ -1183,10 +1186,12 @@ pub struct TurbopackChunkingConfig {
 
 #[turbo_tasks::value]
 pub struct TurbopackChunking {
+    /// The route-matching regexes for each user-defined cluster.
+    clusters: Vec<EsRegexSet>,
     /// First-page-load priority as an integer percentage (`0..=100`), or `None` if unset.
     pub first_page_load_priority: Option<u32>,
     /// Route-matching regexes for priority routes.
-    priority_routes: Vec<EsRegex>,
+    priority_routes: EsRegexSet,
     /// Priority-route boost as an integer percentage (e.g. `150` for a 1.5x boost), or
     /// `None` to use the default.
     pub priority_boost_percent: Option<u32>,
@@ -1206,33 +1211,35 @@ pub struct TurbopackChunking {
 
 impl TurbopackChunking {
     /// Compute the [`EntryHeuristics`] for a route `pathname` by matching it against the configured
-    /// priority-route regexes.
+    /// cluster and priority-route regexes.
     pub fn entry_heuristics_for(&self, pathname: &str) -> EntryHeuristics {
-        let high_priority = self
-            .priority_routes
+        let clusters = self
+            .clusters
             .iter()
-            .filter(|regex| regex.as_regex_str().is_none())
-            .any(|regex| regex.is_match(pathname))
-            || regex::RegexSet::new(
-                self.priority_routes
-                    .iter()
-                    .filter_map(|regex| regex.as_regex_str()),
-            )
-            .is_ok_and(|set| set.is_match(pathname));
-        EntryHeuristics { high_priority }
+            .enumerate()
+            .filter(|(_, regexes)| regexes.is_match(pathname))
+            .map(|(index, _)| index as u16)
+            .collect();
+        let high_priority = self.priority_routes.is_match(pathname);
+        EntryHeuristics {
+            clusters,
+            high_priority,
+        }
     }
 }
 
-/// Compile a list of route-matching [`RegexComponents`] into [`EsRegex`]es.
-fn parse_route_regexes(patterns: &[RegexComponents]) -> Result<Vec<EsRegex>> {
-    patterns
+/// Compile a list of route-matching [`RegexComponents`] into an [`EsRegexSet`], which builds the
+/// combined [`regex::RegexSet`] up front so that matching a route doesn't have to.
+fn parse_route_regexes(patterns: &[RegexComponents]) -> Result<EsRegexSet> {
+    let regexes = patterns
         .iter()
         .cloned()
         .map(|pattern| {
             EsRegex::try_from(pattern)
                 .context("Invalid route pattern in `experimental.turbopackChunking`")
         })
-        .collect()
+        .collect::<Result<Vec<_>>>()?;
+    Ok(EsRegexSet::new(regexes))
 }
 
 /// Resolve `experimental.cssChunking` to the [`StyleGroupsAlgorithm`] Turbopack should use.
@@ -2165,12 +2172,19 @@ impl NextConfig {
     #[turbo_tasks::function]
     pub fn turbopack_chunking(&self) -> Result<Vc<TurbopackChunking>> {
         let config = self.experimental.turbopack_chunking.as_ref();
+        let clusters = config
+            .and_then(|c| c.clusters.as_deref())
+            .unwrap_or_default()
+            .iter()
+            .map(|patterns| parse_route_regexes(patterns))
+            .collect::<Result<Vec<_>>>()?;
         let priority_routes = parse_route_regexes(
             config
                 .and_then(|c| c.priority_routes.as_deref())
                 .unwrap_or_default(),
         )?;
         Ok(TurbopackChunking {
+            clusters,
             first_page_load_priority: config
                 .and_then(|c| c.first_page_load_priority)
                 .map(|priority| (priority.clamp(0.0, 1.0) * 100.0).round() as u32),

packages/next/src/build/swc/index.ts

@@ -1050,6 +1050,9 @@ function bindingToApi(
         ...nextConfigSerializable.experimental,
         turbopackChunking: {
           ...chunkingConfig,
+          clusters: chunkingConfig.clusters?.map((cluster: RegExp[]) =>
+            cluster.map(regexComponents)
+          ),
           priorityRoutes: chunkingConfig.priorityRoutes?.map(regexComponents),
         },
       }

packages/next/src/server/config-schema.ts

@@ -396,6 +396,7 @@ export const experimentalSchema = {
   turbopackSharedRuntime: z.boolean().optional(),
   turbopackChunking: z
     .object({
+      clusters: z.array(z.array(z.instanceof(RegExp))).optional(),
       firstPageLoadPriority: z.number().min(0).max(1).optional(),
       priorityRoutes: z.array(z.instanceof(RegExp)).optional(),
       priorityBoost: z.number().min(1).optional(),

packages/next/src/server/config-shared.ts

@@ -805,6 +805,19 @@ export interface ExperimentalConfig {
    * making chunk merging decisions and the raw size thresholds it uses.
    */
   turbopackChunking?: {
+    /**
+     * Groups of pages commonly visited together, each defined by a list of regular
+     * expressions matched against the route pathname.
+     *
+     * @example
+     * ```js
+     * clusters: [
+     *   [/^\/dashboard/, /^\/dashboard\/settings/],
+     *   [/^\/blog/, /^\/blog\/[^/]+$/],
+     * ]
+     * ```
+     */
+    clusters?: RegExp[][]
     /**
      * This is a number between `0..1`, when higher, we weight the benefits of
      * merging chunks for a signal page load higher. If you don't know a good

turbopack/crates/turbo-esregex/src/lib.rs

@@ -161,6 +161,89 @@ impl EsRegex {
     }
 }
 
+/// A group of [`EsRegex`]es matched against a haystack as a unit.
+///
+/// The members backed by the `regex` crate are compiled into a single [`regex::RegexSet`] once,
+/// when the group is built, rather than on every match. The remainder (those that fall back to
+/// `regress`, e.g. for lookahead) are matched one at a time.
+#[derive(Debug, Clone)]
+#[turbo_tasks::value(eq = "manual", shared, serialization = "custom")]
+pub struct EsRegexSet {
+    /// The members, in the order they were given. Also the source of truth for equality and
+    /// serialization, since [`regex::RegexSet`] supports neither.
+    regexes: Vec<EsRegex>,
+    /// The combined members, or `None` if the combined program couldn't be built.
+    #[turbo_tasks(trace_ignore)]
+    set: Option<regex::RegexSet>,
+    /// Indices into `regexes` of the members `set` doesn't cover. Usually empty.
+    individual: Vec<u32>,
+}
+
+impl PartialEq for EsRegexSet {
+    fn eq(&self, other: &Self) -> bool {
+        self.regexes == other.regexes
+    }
+}
+impl Eq for EsRegexSet {}
+
+impl Encode for EsRegexSet {
+    fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
+        self.regexes.encode(encoder)
+    }
+}
+
+impl<Context> Decode<Context> for EsRegexSet {
+    fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
+        let regexes: Vec<EsRegex> = Decode::decode(decoder)?;
+        Ok(EsRegexSet::new(regexes))
+    }
+}
+
+impl_borrow_decode!(EsRegexSet);
+
+impl Default for EsRegexSet {
+    fn default() -> Self {
+        Self::new(Vec::new())
+    }
+}
+
+impl EsRegexSet {
+    /// Builds the combined matcher. Members backed by `regress` can't join a
+    /// [`regex::RegexSet`], and the combined program has its own size limit; either way the
+    /// leftovers are recorded up front and matched one at a time.
+    pub fn new(regexes: Vec<EsRegex>) -> Self {
+        let set = regex::RegexSet::new(regexes.iter().filter_map(EsRegex::as_regex_str)).ok();
+        let individual = regexes
+            .iter()
+            .enumerate()
+            .filter(|(_, regex)| set.is_none() || regex.as_regex_str().is_none())
+            .map(|(index, _)| index as u32)
+            .collect();
+        Self {
+            regexes,
+            set,
+            individual,
+        }
+    }
+
+    /// Returns true if any member matches somewhere in the `haystack`.
+    pub fn is_match(&self, haystack: &str) -> bool {
+        if let Some(set) = &self.set
+            && set.is_match(haystack)
+        {
+            return true;
+        }
+        self.individual
+            .iter()
+            .any(|&index| self.regexes[index as usize].is_match(haystack))
+    }
+
+    /// Returns true if the group has no members.
+    pub fn is_empty(&self) -> bool {
+        self.regexes.is_empty()
+    }
+}
+
 pub struct Captures<'h> {
     delegate: CapturesImpl<'h>,
 }
@@ -216,7 +299,75 @@ impl<'h> Iterator for Captures<'h> {
 
 #[cfg(test)]
 mod tests {
-    use super::{EsRegex, EsRegexImpl};
+    use super::{EsRegex, EsRegexImpl, EsRegexSet};
+
+    #[test]
+    fn es_regex_set_matches_either_delegate() {
+        // `a(?!b)` needs regress; `^/docs` is handled by the shared `RegexSet`.
+        let set = EsRegexSet::new(vec![
+            EsRegex::new("^/docs", "").unwrap(),
+            EsRegex::new("a(?!b)", "").unwrap(),
+        ]);
+        assert_eq!(set.individual, vec![1]);
+        assert!(set.is_match("/docs/getting-started"));
+        assert!(set.is_match("ac"));
+        assert!(!set.is_match("/blog"));
+        assert!(!set.is_match("ab"));
+    }
+
+    #[test]
+    fn es_regex_set_combines_every_member_when_it_can() {
+        let set = EsRegexSet::new(vec![
+            EsRegex::new("^/docs", "").unwrap(),
+            EsRegex::new("^/blog", "").unwrap(),
+        ]);
+        // A miss only queries the combined set, not every member again.
+        assert!(set.individual.is_empty());
+        assert!(set.is_match("/docs"));
+        assert!(set.is_match("/blog"));
+        assert!(!set.is_match("/about"));
+    }
+
+    #[test]
+    fn empty_es_regex_set_never_matches() {
+        let set = EsRegexSet::default();
+        assert!(set.is_empty());
+        assert!(!set.is_match(""));
+        assert!(!set.is_match("/docs"));
+    }
+
+    #[test]
+    fn oversized_es_regex_set_falls_back_to_matching_individually() {
+        // Each of these compiles on its own but together they blow the combined size limit.
+        const N: usize = 60_000;
+        let regexes = vec![
+            EsRegex::new(&format!("^/docs/[0-9a-zA-Z]{{{N}}}"), "").unwrap(),
+            EsRegex::new(&format!("^/blog/[0-9a-zA-Z]{{{N}}}"), "").unwrap(),
+        ];
+        assert!(regexes.iter().all(|regex| regex.as_regex_str().is_some()));
+        let set = EsRegexSet::new(regexes);
+        assert!(set.set.is_none());
+        assert_eq!(set.individual, vec![0, 1]);
+        assert!(set.is_match(&format!("/docs/{}", "a".repeat(N))));
+        assert!(set.is_match(&format!("/blog/{}", "a".repeat(N))));
+        assert!(!set.is_match("/about"));
+    }
+
+    #[test]
+    fn es_regex_set_round_trip_bincode() {
+        let set = EsRegexSet::new(vec![
+            EsRegex::new("^/docs", "").unwrap(),
+            EsRegex::new("a(?!b)", "").unwrap(),
+        ]);
+        let config = bincode::config::standard();
+        let encoded = bincode::encode_to_vec(&set, config).unwrap();
+        let (decoded, len) = bincode::decode_from_slice::<EsRegexSet, _>(&encoded, config).unwrap();
+        assert_eq!(set, decoded);
+        assert_eq!(len, encoded.len());
+        // The `RegexSet` is rebuilt on decode, not carried in the encoding.
+        assert!(decoded.is_match("/docs"));
+        assert!(decoded.is_match("ac"));
+    }
 
     #[test]
     fn round_trip_bincode() {

turbopack/crates/turbopack-core/src/chunk/chunking/chunk_merging_cost_benefit.md

@@ -0,0 +1,435 @@
+# Should we merge a chunk?
+
+By default, for our calculations we assume that there is a probability of 2/3 that
+we request exactly 1 chunk group (`N = 1`) and a probability of 1/3 that we request
+2 chunk groups (`N = 2`). This is a simplification, but it should be good enough
+for our purposes and it is configurable using the chunking heuristics.
+
+**UNMERGED CASE**
+
+from the total of $\text{groups}$ chunk groups
+
+- $a_{\text{groups}}$ chunk groups request a $a_{\text{size}}$ chunk
+- $b_{\text{groups}}$ chunk groups request a $b_{\text{size}}$ chunk
+
+but there is an overlapy of $o_{\text{groups}}$ between them, which request both chunks.
+
+**MERGED CASE**
+
+from the total of $\text{groups}$ chunk groups
+
+- $a_{\text{rem}}$ chunk groups request a $a_{\text{size}}$ chunk
+- $b_{\text{rem}}$ chunk groups request a $b_{\text{size}}$ chunk
+- $o_{\text{groups}}$ chunk groups request the merged chunk of size $(a_{\text{size}} + b_{\text{size}})$
+
+We want to compute the expected request count $e_{\text{req}}$ and the expected total requested size $e_{\text{size}}$ for the unmerged and merged case.
+This will allow us to compare the two.
+
+To compute that we compute the two cases $N = 1$ and $N = 2$ and combine them
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+e_{\text{size}} &= P(N=1) \cdot e_{\text{size}}(N=1) + P(N=2) \cdot e_{\text{size}}(N=2) \\
+e_{\text{req}} &= P(N=1) \cdot e_{\text{req}}(N=1) + P(N=2) \cdot e_{\text{req}}(N=2)
+\end{aligned} &
+\end{flalign*}
+$$
+
+We combine $e_{\text{size}}$ with $e_{\text{req}}$ using this formula:
+
+$$
+\begin{flalign*}
+& e_{\text{cost}} = e_{\text{req}} \cdot c_{\text{req}} + e_{\text{size}} &
+\end{flalign*}
+$$
+
+The constant $c_{\text{req}}$ is the cost of a single request in transferred bytes. We have to choose a good value for that since there is no real value of that.
+This way we can compute a cost for both cases ($e_{\text{unmerged cost}}$ and $e_{\text{merged cost}}$).
+
+With both costs we can compute the cost benefit $d$ of merging the two chunks:
+
+$$
+\begin{flalign*}
+& d = e_{\text{unmerged cost}} - e_{\text{merged cost}} &
+\end{flalign*}
+$$
+
+We can also split the formula into two parts:
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+d &= d_{\text{req}} \cdot c_{\text{req}} + d_{\text{size}} \\
+d_{\text{size}} &= e_{\text{unmerged size}} - e_{\text{merged size}} \\
+d_{\text{req}} &= e_{\text{unmerged req}} - e_{\text{merged req}}
+\end{aligned} &
+\end{flalign*}
+$$
+
+And we can split it further for every $N$:
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+d_{\text{size}} &= P(N=1) \cdot d_{\text{size}}(N=1) + P(N=2) \cdot d_{\text{size}}(N=2) \\
+d_{\text{req}} &= P(N=1) \cdot d_{\text{req}}(N=1) + P(N=2) \cdot d_{\text{req}}(N=2)
+\end{aligned} &
+\end{flalign*}
+$$
+
+---
+
+To compute $e_{\text{size}}$ and $e_{\text{req}}$ we need to determine all cases and their probabilities.
+
+**UNMERGED CASE (N = 1):**
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+& \textbf{case X} \\
+& p = a_{\text{rem}}/\text{groups} \\
+& \text{size} = a_{\text{size}} \\
+& \text{requests} = 1 \\
+& \\
+& \textbf{case Y} \\
+& p = b_{\text{rem}}/\text{groups} \\
+& \text{size} = b_{\text{size}} \\
+& \text{requests} = 1 \\
+& \\
+& \textbf{case Z} \\
+& p = o_{\text{groups}}/\text{groups} \\
+& \text{size} = a_{\text{size}} + b_{\text{size}} \\
+& \text{requests} = 2
+\end{aligned} &
+\end{flalign*}
+$$
+
+**MERGED CASE (N = 1):**
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+& \textbf{case X} \\
+& p = a_{\text{rem}}/\text{groups} \\
+& \text{size} = a_{\text{size}} \\
+& \text{requests} = 1 \\
+& \\
+& \textbf{case Y} \\
+& p = b_{\text{rem}}/\text{groups} \\
+& \text{size} = b_{\text{size}} \\
+& \text{requests} = 1 \\
+& \\
+& \textbf{case Z} \\
+& p = o_{\text{groups}}/\text{groups} \\
+& \text{size} = a_{\text{size}} + b_{\text{size}} \\
+& \text{requests} = 1
+\end{aligned} &
+\end{flalign*}
+$$
+
+There is no difference in the sizes at all, so that means:
+
+$$
+\begin{flalign*}
+& d_{\text{size}}(N=1) = 0 &
+\end{flalign*}
+$$
+
+The only difference is in case $Z$ in the request count. That case has $p = o_{\text{groups}}/\text{groups}$:
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+d_{\text{req}}(N=1) &= (o_{\text{groups}} / \text{groups}) \cdot (2 - 1) \\
+d_{\text{req}}(N=1) &= o_{\text{groups}} / \text{groups} \\
+d(N=1) &= d_{\text{req}}(N=1) \cdot c_{\text{req}} + d_{\text{size}}(N=1) \\
+&= (o_{\text{groups}} \cdot c_{\text{req}}) / \text{groups}
+\end{aligned} &
+\end{flalign*}
+$$
+
+---
+
+Each $N = 2$ case's probability $p$ is the probability of navigating
+to a page in the first group, then from that page to the second
+group. That second probability is the transition probability,
+written $\text{trans}(1 \to 2)$.
+
+Route "clusters" can be configured to signal that users are more
+likely to navigate between pages in the same cluster. This changes
+the transition probability. When no clusters are configured:
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+\text{trans}(X \to X) &= (a_{\text{rem}} - 1)/(\text{groups} - 1) \\
+\text{trans}(X \to Y) &= b_{\text{rem}}/(\text{groups} - 1) \\
+\text{trans}(X \to Z) &= o_{\text{groups}}/(\text{groups} - 1) \\
+\text{trans}(Y \to X) &= a_{\text{rem}}/(\text{groups} - 1) \\
+\text{trans}(Y \to Y) &= (b_{\text{rem}} - 1)/(\text{groups} - 1) \\
+\text{trans}(Y \to Z) &= o_{\text{groups}}/(\text{groups} - 1) \\
+\text{trans}(Z \to X) &= a_{\text{rem}}/(\text{groups} - 1) \\
+\text{trans}(Z \to Y) &= b_{\text{rem}}/(\text{groups} - 1) \\
+\text{trans}(Z \to Z) &= (o_{\text{groups}} - 1)/(\text{groups} - 1)
+\end{aligned} &
+\end{flalign*}
+$$
+
+and the $N = 2$ cases reduce to:
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+& \textbf{case X + X} \\
+& p = (a_{\text{rem}}/\text{groups}) \cdot ((a_{\text{rem}} - 1)/(\text{groups} - 1)) \\
+& \\
+& \textbf{case Y + Y} \\
+& p = (b_{\text{rem}}/\text{groups}) \cdot ((b_{\text{rem}} - 1)/(\text{groups} - 1)) \\
+& \\
+& \textbf{case Z + Z} \\
+& p = (o_{\text{groups}}/\text{groups}) \cdot ((o_{\text{groups}} - 1)/(\text{groups} - 1)) \\
+& \\
+& \textbf{case X + Y} \\
+& p = (a_{\text{rem}}/\text{groups}) \cdot (b_{\text{rem}}/(\text{groups} - 1)) + (b_{\text{rem}}/\text{groups}) \cdot (a_{\text{rem}}/(\text{groups} - 1)) \\
+& \\
+& \textbf{case X + Z} \\
+& p = (a_{\text{rem}}/\text{groups}) \cdot (o_{\text{groups}}/(\text{groups} - 1)) + (o_{\text{groups}}/\text{groups}) \cdot (a_{\text{rem}}/(\text{groups} - 1)) \\
+& \\
+& \textbf{case Y + Z} \\
+& p = (b_{\text{rem}}/\text{groups}) \cdot (o_{\text{groups}}/(\text{groups} - 1)) + (o_{\text{groups}}/\text{groups}) \cdot (b_{\text{rem}}/(\text{groups} - 1))
+\end{aligned} &
+\end{flalign*}
+$$
+
+$X$, $Y$ and $Z$ are three sets of chunk groups:
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+X &= \text{the } a_{\text{rem}} \text{ groups that load only chunk A} \\
+Y &= \text{the } b_{\text{rem}} \text{ groups that load only chunk B} \\
+Z &= \text{the } o_{\text{groups}} \text{ groups that load both}
+\end{aligned} &
+\end{flalign*}
+$$
+
+Now, when clusters are configured, it is more complicated.
+
+Two groups that sit in the same cluster form a "pair". The table below
+counts the number of pairs that exist between groups. Each cell says
+how many pairs have one group in one set and the other group in another
+set. The diagonal cells ($c_{xx}$, $c_{yy}$, $c_{zz}$) count pairs where
+the paired groups are in the same set.
+
+$$
+\begin{flalign*}
+& \begin{array}{c|ccc}
+ & X\\,(a_{\text{rem}}) & Y\\,(b_{\text{rem}}) & Z\\,(\text{overlap}) \\
+\hline
+X\\,(a_{\text{rem}})   & c_{xx} & c_{xy} & c_{xz} \\
+Y\\,(b_{\text{rem}})   & c_{xy} & c_{yy} & c_{yz} \\
+Z\\,(\text{overlap})   & c_{xz} & c_{yz} & c_{zz}
+\end{array} &
+\end{flalign*}
+$$
+
+Each row sum is the total number of pairs leaving that set:
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+\text{paired}\_x &= c\_{xx} + c\_{xy} + c\_{xz} \\
+\text{paired}\_y &= c\_{xy} + c\_{yy} + c\_{yz} \\
+\text{paired}\_z &= c\_{xz} + c\_{yz} + c\_{zz}
+\end{aligned} &
+\end{flalign*}
+$$
+
+`CLUSTER_NAVIGATION_PROBABILITY` ($= 0.6$, written as $\text{cnp}$ below) is the chance a
+navigation stays within a cluster.
+
+For a first group in set 1, $\text{trans}(1 \to 2)$ can be calculated using
+the following probability tree:
+
+```
+                    Will the navigation stay
+                       within a cluster?
+                    /                    \
+             yes (cnp)                 no (1 - cnp)
+                  /                        \
+       Does it go to set 2?         Does it go to set 2?
+                |                            |
+         c_12 / paired_1      non_paired_12 / non_paired_1
+```
+
+$\text{non\\_paired}\_{12}$ is the amount of navigations from set 1 into set 2
+that are not within a cluster, and $\text{non\\_paired}\_1$ is the total
+number of unpaired navigations leaving set 1:
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+\text{non\\_paired}\_{12} &= |S_1| \cdot |S_2| - c\_{12} \\
+\text{non\\_paired}\_1 &= (\text{groups} - 1) \cdot |S_1| - \text{paired}\_1
+\end{aligned} &
+\end{flalign*}
+$$
+
+When set 1 and set 2 are the same set, the $|S_1|$ in $\text{non\\_paired}\_{12}$
+becomes $|S_1| - 1$, since a group cannot navigate to itself. The $|S_1|$ in
+$\text{non\\_paired}\_1$ is unchanged, as every group is still a possible
+starting point.
+
+Therefore:
+
+$$
+\begin{flalign*}
+& \text{trans}(1 \to 2) = \text{cnp} \cdot (c\_{12} / \text{paired}\_1) + (1 - \text{cnp}) \cdot (\text{non\\_paired}\_{12} / \text{non\\_paired}\_1) &
+\end{flalign*}
+$$
+
+To prevent dividing by zero, if $\text{paired}\_1 = 0$ then
+$\text{trans}(1 \to 2) = |S_2| / (\text{groups} - 1)$, and if $\text{non\\_paired}\_1 = 0$
+then $\text{trans}(1 \to 2) = c\_{12} / \text{paired}\_1$.
+
+---
+
+In terms of transition probabilities, $d$ is:
+
+**UNMERGED CASE (N = 2):**
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+& \textbf{case X + X} \\
+& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to X) \\
+& \text{size} = a_{\text{size}} \\
+& \text{requests} = 1 \\
+& \\
+& \textbf{case Y + Y} \\
+& p = (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to Y) \\
+& \text{size} = b_{\text{size}} \\
+& \text{requests} = 1 \\
+& \\
+& \textbf{case Z + Z} \\
+& p = (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Z) \\
+& \text{size} = a_{\text{size}} + b_{\text{size}} \\
+& \text{requests} = 2 \\
+& \\
+& \textbf{case X + Y} \\
+& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Y) + (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to X) \\
+& \text{size} = a_{\text{size}} + b_{\text{size}} \\
+& \text{requests} = 2 \\
+& \\
+& \textbf{case X + Z} \\
+& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to X) \\
+& \text{size} = a_{\text{size}} + b_{\text{size}} \\
+& \text{requests} = 2 \\
+& \\
+& \textbf{case Y + Z} \\
+& p = (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Y) \\
+& \text{size} = a_{\text{size}} + b_{\text{size}} \\
+& \text{requests} = 2
+\end{aligned} &
+\end{flalign*}
+$$
+
+**MERGED CASE (N = 2):**
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+& \textbf{case X + X} \\
+& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to X) \\
+& \text{size} = a_{\text{size}} \\
+& \text{requests} = 1 \\
+& \\
+& \textbf{case Y + Y} \\
+& p = (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to Y) \\
+& \text{size} = b_{\text{size}} \\
+& \text{requests} = 1 \\
+& \\
+& \textbf{case Z + Z} \\
+& p = (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Z) \\
+& \text{size} = (a_{\text{size}} + b_{\text{size}}) \\
+& \text{requests} = 1 \\
+& \\
+& \textbf{case X + Y} \\
+& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Y) + (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to X) \\
+& \text{size} = a_{\text{size}} + b_{\text{size}} \\
+& \text{requests} = 2 \\
+& \\
+& \textbf{case X + Z} \\
+& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to X) \\
+& \text{size} = a_{\text{size}} + (a_{\text{size}} + b_{\text{size}}) \\
+& \text{requests} = 2 \\
+& \\
+& \textbf{case Y + Z} \\
+& p = (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Y) \\
+& \text{size} = b_{\text{size}} + (a_{\text{size}} + b_{\text{size}}) \\
+& \text{requests} = 2
+\end{aligned} &
+\end{flalign*}
+$$
+
+Request count is different in this case: $Z + Z$ (better)
+
+Requests size is different (worse) in these cases: $X + Z$, $Y + Z$
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+d_{\text{req } Z+Z} &= ((o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Z)) \cdot (2 - 1) \\
+&= (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Z)
+\end{aligned} &
+\end{flalign*}
+$$
+
+$$
+\begin{flalign*}
+& d_{\text{req}}(N=2) = (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Z) &
+\end{flalign*}
+$$
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+d_{\text{size } X+Z} &= ((a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to X)) \cdot (a_{\text{size}} + b_{\text{size}} - (a_{\text{size}} + (a_{\text{size}} + b_{\text{size}}))) \\
+&= ((a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to X)) \cdot (-a_{\text{size}}) \\
+&= -a_{\text{size}} \cdot (a_{\text{rem}} \cdot \text{trans}(X \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to X)) / \text{groups} \\
+& \\
+d_{\text{size } Y+Z} &= -b_{\text{size}} \cdot (b_{\text{rem}} \cdot \text{trans}(Y \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to Y)) / \text{groups}
+\end{aligned} &
+\end{flalign*}
+$$
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+d_{\text{size}}(N=2) &= -(a_{\text{size}} \cdot (a_{\text{rem}} \cdot \text{trans}(X \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to X)) \\
+&\quad + b_{\text{size}} \cdot (b_{\text{rem}} \cdot \text{trans}(Y \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to Y))) / \text{groups}
+\end{aligned} &
+\end{flalign*}
+$$
+
+$$
+\begin{flalign*}
+& \begin{aligned}
+d(N=2) &= d_{\text{req}}(N=2) \cdot c_{\text{req}} + d_{\text{size}}(N=2) \\
+&= (o_{\text{groups}} \cdot \text{trans}(Z \to Z) \cdot c_{\text{req}} \\
+&\quad - a_{\text{size}} \cdot (a_{\text{rem}} \cdot \text{trans}(X \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to X)) \\
+&\quad - b_{\text{size}} \cdot (b_{\text{rem}} \cdot \text{trans}(Y \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to Y))) / \text{groups}
+\end{aligned} &
+\end{flalign*}
+$$
+
+---
+
+Finally,
+
+$$
+\begin{flalign*}
+& d = P(N=1) \cdot d(N=1) + P(N=2) \cdot d(N=2) &
+\end{flalign*}
+$$

turbopack/crates/turbopack-core/src/chunk/chunking/production.rs

@@ -1,7 +1,8 @@
 use std::{borrow::Cow, collections::BinaryHeap, hash::BuildHasherDefault, mem::take};
 
 use anyhow::{Context, Result};
-use rustc_hash::FxHasher;
+use roaring::RoaringBitmap;
+use rustc_hash::{FxHashMap, FxHasher};
 use smallvec::SmallVec;
 use tracing::{Instrument, field::Empty};
 use turbo_prehash::BuildHasherExt;
@@ -22,6 +23,9 @@ use crate::{
 /// Default estimated cost of an additional request, in bytes (200 KB).
 const DEFAULT_ESTIMATED_REQUEST_COST_BYTES: u64 = 200_000;
 
+/// Probability that a navigation stays within a cluster.
+const CLUSTER_NAVIGATION_PROBABILITY: f64 = 0.6;
+
 pub async fn make_production_chunks(
     chunk_items: Vec<&ChunkItemOrBatchWithInfo>,
     batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
@@ -254,6 +258,10 @@ pub async fn make_production_chunks(
                 let priority_boost =
                     priority_boost_percent.map_or(1.5, |percent| percent as f64 / 100.0);
 
+                // If chunk group clusters are configured in `next.config.js` and the patterns
+                // match at least one route.
+                let has_clusters = heuristics.clusters.iter().any(|c| !c.is_empty());
+
                 let mut iterations = 0;
                 while chunks_to_merge.len() > 1 {
                     // Find best candidate
@@ -306,158 +314,14 @@ pub async fn make_production_chunks(
                             let a_rem = a_groups - o_groups;
                             let b_rem = b_groups - o_groups;
 
-                            /*
-                                UNMERGED CASE
+                            // See ./chunk_merging_cost_benefit.md for a description of how
+                            // this works.
 
-                                from the total of `groups` chunk groups
-                                - `a_groups` chunk groups request a `a_size` chunk
-                                - `b_groups` chunk groups request a `b_size` chunk
-                                but there is an overlapy of `o_groups` between them, which request both chunks.
-
-                                MERGED CASE
-
-                                from the total of `groups` chunk groups
-                                - `a_rem` chunk groups request a `a_size` chunk
-                                - `b_rem` chunk groups request a `b_size` chunk
-                                - `o_groups` chunk groups request the merged chunk of size `(a_size + b_size)`
-                            */
-
-                            /*
-                                By default, for our calculations we assume that there is a probability of 2/3 that
-                                we request exactly 1 chunk group (`N = 1`) and a probability of 1/3 that we request
-                                2 chunk groups (`N = 2`). This is a simplification, but it should be good enough
-                                for our purposes and it is configurable using the chunking heuristics.
-
-                                We want to compute the expected request count `e_req` and the expected total requested size `e_size` for the unmerged and merged case.
-
-                                To compute that we compute the two cases `N = 1` and `N = 2` and combine them
-                                e_size = P(N = 1) * e_size(N = 1) + P(N = 2) * e_size(N = 2)
-                                e_req = P(N = 1) * e_req(N = 1) + P(N = 2) * e_req(N = 2)
-
-                                We combine `e_size` with `e_req` using this formula:
-                                e_cost = e_req * c_req + e_size
-
-                                The constant `c_req` is the cost of a single request in transferred bytes. We have to choose a good value for that since there is no real value of that.
-                                This way we can compute a cost for both cases (`e_cost_unmerged` and `e_cost_merged`).
-
-                                With both costs we can compute the cost benefit `d` of merging the two chunks:
-                                d = e_cost_unmerged - e_cost_merged
-
-                                We can also split the formula into two parts:
-                                d = d_req * c_req + d_size
-                                d_size = e_size_unmerged - e_size_merged
-                                d_req = e_req_unmerged - e_req_merged
-
-                                And we can split it further for every N:
-                                d_size = P(N = 1) * d_size(N = 1) + P(N = 2) * d_size(N = 2)
-                                d_req = P(N = 1) * d_req(N = 1) + P(N = 2) * d_req(N = 2)
-                            */
-
-                            /*
-                                To compute `e_size` and `e_req` we need to determine all cases and their probabilities.
-
-                                UNMERGED CASE (N = 1):
-
-                                case X (p = a_rem/groups): size = a_size, requests = 1
-                                case Y (p = b_rem/groups): size = b_size, requests = 1
-                                case Z (p = o_groups/groups): size = a_size + b_size, requests = 2
-
-                                MERGED CASE (N = 1):
-
-                                case X (p = a_rem/groups): size = a_size, requests = 1
-                                case Y (p = b_rem/groups): size = b_size, requests = 1
-                                case Z (p = o_groups/groups): size = a_size + b_size, requests = 1
-                            */
-
-                            /*
-                                There is no difference in the sizes at all, so that means:
-
-                                d_size(N = 1) = 0
-
-                                The only difference is in case Z in the request count. That case has `p = o_groups/groups`:
-
-                                d_req(N = 1) = (o_groups / groups) * (2 - 1)
-                                d_req(N = 1) = o_groups / groups
-
-                                d(N = 1) = d_req(N = 1) * c_req + d_size(N = 1)
-                                         = (o_groups * c_req) / groups
-                            */
-
-                            /*
-                                The N = 2 case is more complicated, since we have to consider all possible combinations of the cases X, Y and Z for the two chunk groups:
-
-                                p_x = a_rem/groups
-                                p_y = b_rem/groups
-                                p_z = o_groups/groups
-
-                                The chunk groups remaining after the first one has been picked
-                                rem_g = groups - 1
-
-                                UNMERGED CASE (N = 2):
-                                case X + X (p = (a_rem/groups) * ((a_rem - 1)/rem_g)): size = a_size, requests = 1
-                                case Y + Y (p = (b_rem/groups) * ((b_rem - 1)/rem_g)): size = b_size, requests = 1
-                                case Z + Z (p = (o_groups/groups) * (o_groups - 1)/rem_g): size = a_size + b_size, requests = 2
-                                case X + Y (p = (a_rem/groups) * (b_rem/rem_g) + (b_rem/groups) * (a_rem/rem_g)): size = a_size + b_size, requests = 2
-                                case X + Z (p = (a_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (a_rem/rem_g)): size = a_size + b_size, requests = 2
-                                case Y + Z (p = (b_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (b_rem/rem_g)): size = a_size + b_size, requests = 2
-
-                                MERGED CASE (N = 2):
-                                case X + X (p = (a_rem/groups) * ((a_rem - 1)/rem_g)): size = a_size, requests = 1
-                                case Y + Y (p = (b_rem/groups) * ((b_rem - 1)/rem_g)): size = b_size, requests = 1
-                                case Z + Z (p = (o_groups/groups) * (o_groups - 1)/rem_g): size = (a_size + b_size), requests = 1
-                                case X + Y (p = (a_rem/groups) * (b_rem/rem_g) + (b_rem/groups) * (a_rem/rem_g)): size = a_size + b_size, requests = 2
-                                case X + Z (p = (a_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (a_rem/rem_g)): size = a_size + (a_size + b_size), requests = 2
-                                case Y + Z (p = (b_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (b_rem/rem_g)): size = b_size + (a_size + b_size), requests = 2
-
-                                Request count is different in this case: Z + Z (better)
-                                Requests size is different (worse) in these cases: X + Z, Y + Z
-
-                                d_req_z_z = ((o_groups/groups) * (o_groups - 1)/rem_g) * (2 - 1)
-                                          = o_groups * (o_groups - 1) / (groups * rem_g)
-                                d_req_x_z = ((a_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (a_rem/rem_g)) * (2 - 2)
-                                          = 0
-                                d_req_y_z = ((b_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (b_rem/rem_g)) * (2 - 2)
-                                          = 0
-
-                                d_req(N = 2) = o_groups * (o_groups - 1) / (groups * rem_g)
-
-                                d_size_x_z = ((a_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (a_rem/rem_g)) * (a_size + b_size - (a_size + (a_size + b_size)))
-                                           = ((2 * a_rem * o_groups) / (groups * rem_g)) * (-a_size)
-                                           = -2 * a_rem * a_size * o_groups / (groups * rem_g)
-                                d_size_y_z = -2 * b_rem * b_size * o_groups / (groups * rem_g)
-
-                                d_size(N = 2) = -2 * (a_rem * a_size + b_rem * b_size) * o_groups / (groups * rem_g)
-
-                                d(N = 2) = d_req(N = 2) * c_req + d_size(N = 2)
-                                         = (o_groups * (o_groups - 1) * c_req) / (groups * rem_g) - (2 * (a_rem * a_size + b_rem * b_size) * o_groups) / (groups * rem_g)
-                                         = (o_groups * (o_groups - 1) * c_req - 2 * (a_rem * a_size + b_rem * b_size) * o_groups) / (groups * rem_g)
-                            */
-
-                            /*
-                                d  = P(N = 1) * d(N = 1) + P(N = 2) * d(N = 2)
-                            */
-
-                            /*
-                               Recall from above:
-                               d = d_req * c_req + d_size
-                               d_req = P(N = 1) * d_req(N = 1) + P(N = 2) * d_req(N = 2)
-
-                               `d_size` is always <= 0, so for d > 0, d_req * c_req must be
-                               positive:
-
-                               d > 0
-                               d_req * c_req > 0
-                               (P(N = 1) * o_groups / groups + P(N = 2) * o_groups * (o_groups - 1) / (groups * rem_g)) * c_req > 0
-                               P(N = 1) * o_groups / groups + P(N = 2) * o_groups * (o_groups - 1) / (groups * rem_g) > 0
-                               P(N = 1) * o_groups * rem_g + P(N = 2) * o_groups * (o_groups - 1) > 0
-                               o_groups * (P(N = 1) * rem_g + P(N = 2) * (o_groups - 1)) > 0
-                               o_groups > 0 && P(N = 1) * rem_g + P(N = 2) * (o_groups - 1) > 0
-                               o_groups > 0 && P(N = 1) * (groups - 1) + P(N = 2) * (o_groups - 1) > 0
-                               o_groups > 0 && groups >= 2
-                            */
-
-                            // It need to have some request count benefit, the
-                            // check for that has been derived above:
+                            // If there are no overlapping groups, there is no benefit to
+                            // merging - skip this process. Also, our code assumes that
+                            // more than one group requests these chunks. If it was just
+                            // one group requesting both it should already have been merged
+                            // in `grouped_chunk_items` above.
                             if o_groups == 0 || groups < 2 {
                                 continue;
                             }
@@ -475,17 +339,70 @@ pub async fn make_production_chunks(
                             // in `o_groups` would be a chunk group that requests both chunk items.
 
                             let mut is_priority_route = false;
+
+                            // Distinct pairs between the sets X (a_rem), Y (b_rem) and Z (overlap)
+                            // that are both in a cluster.
+                            let (mut c_xx, mut c_xy, mut c_xz, mut c_yy, mut c_yz, mut c_zz) =
+                                (0.0, 0.0, 0.0, 0.0, 0.0, 0.0);
                             if let (Some(a), Some(b)) =
                                 (&candidate.chunk_groups, &other.chunk_groups)
                             {
-                                let o = &***a & &***b; // `o_groups`
+                                let o = &***a & &***b; // `o_groups` (Z)
 
                                 // if there is one chunk group in `o_groups` that is used by a
                                 // priority route, we should prioritise merging these two chunk
                                 // items.
                                 is_priority_route = !o.is_disjoint(&heuristics.priority_routes);
+
+                                if has_clusters {
+                                    let x = &***a - &o; // a_rem groups: load only chunk A
+                                    let y = &***b - &o; // b_rem groups: load only chunk B
+
+                                    // Map each cluster to the candidate groups it contains.
+                                    let mut cluster_groups: FxHashMap<u16, RoaringBitmap> =
+                                        FxHashMap::default();
+                                    for set in [&x, &y, &o] {
+                                        for index in set.iter() {
+                                            for &c in &heuristics.clusters[index as usize] {
+                                                cluster_groups.entry(c).or_default().insert(index);
+                                            }
+                                        }
+                                    }
+
+                                    // Groups sharing >= 1 cluster with `index`, deduped across
+                                    // clusters (excluding `index` itself) so each pair counts once.
+                                    let pairs_with = |index: u32| {
+                                        let mut p = RoaringBitmap::new();
+                                        for &c in &heuristics.clusters[index as usize] {
+                                            if let Some(groups) = cluster_groups.get(&c) {
+                                                p |= groups;
+                                            }
+                                        }
+                                        p.remove(index);
+                                        p
+                                    };
+
+                                    for index in x.iter() {
+                                        let p = pairs_with(index);
+                                        c_xx += p.intersection_len(&x) as f64;
+                                        c_xy += p.intersection_len(&y) as f64;
+                                        c_xz += p.intersection_len(&o) as f64;
+                                    }
+                                    for index in y.iter() {
+                                        let p = pairs_with(index);
+                                        c_yy += p.intersection_len(&y) as f64;
+                                        c_yz += p.intersection_len(&o) as f64;
+                                    }
+                                    for index in o.iter() {
+                                        c_zz += pairs_with(index).intersection_len(&o) as f64;
+                                    }
+                                }
                             }
 
+                            let paired_x = c_xx + c_xy + c_xz;
+                            let paired_y = c_xy + c_yy + c_yz;
+                            let paired_z = c_xz + c_yz + c_zz;
+
                             let p1 = if is_priority_route {
                                 (default_p1 * priority_boost).min(1.0)
                             } else {
@@ -497,14 +414,55 @@ pub async fn make_production_chunks(
                             let o = o_groups as f64;
                             let groups = groups as f64;
                             let rem_g = rem_g as f64;
+                            let a_rem = a_rem as f64;
+                            let b_rem = b_rem as f64;
+                            let a_size = a_size as f64;
+                            let b_size = b_size as f64;
+
+                            /* transition_probability(source -> dest): probability that, after landing on a page
+                            in the `source` set, the next navigation goes to the `dest` set.
+                            `CLUSTER_NAVIGATION_PROBABILITY` of the time it stays within a cluster
+                            (split across the source's pairs); the
+                            rest spreads over the non-paired groups. With no pairs it is a uniform hop.
+
+                            - pairs_to_dest: co-clustered pairs from source to dest
+                            - source_pairs: all co-clustered pairs leaving source (its row sum)
+                            - source_groups: number of groups in the source set
+                            - dest_groups: groups in the dest set (minus 1 if source == dest) */
+                            let transition_probability =
+                                |pairs_to_dest: f64,
+                                 source_pairs: f64,
+                                 source_groups: f64,
+                                 dest_groups: f64| {
+                                    if source_pairs == 0.0 {
+                                        // Source has no pairs: navigate uniformly.
+                                        return dest_groups / rem_g;
+                                    }
+                                    let non_paired_from_source =
+                                        rem_g * source_groups - source_pairs;
+                                    if non_paired_from_source <= 0.0 {
+                                        // Every other group is paired: all weight on the pairs.
+                                        return pairs_to_dest / source_pairs;
+                                    }
+                                    let non_paired_from_source_to_dest =
+                                        dest_groups * source_groups - pairs_to_dest;
+                                    CLUSTER_NAVIGATION_PROBABILITY * (pairs_to_dest / source_pairs)
+                                        + (1.0 - CLUSTER_NAVIGATION_PROBABILITY)
+                                            * (non_paired_from_source_to_dest
+                                                / non_paired_from_source)
+                                };
+
+                            let p_zz = transition_probability(c_zz, paired_z, o, o - 1.0);
+                            let p_zx = transition_probability(c_xz, paired_z, o, a_rem);
+                            let p_zy = transition_probability(c_yz, paired_z, o, b_rem);
+                            let p_xz = transition_probability(c_xz, paired_x, a_rem, o);
+                            let p_yz = transition_probability(c_yz, paired_y, b_rem, o);
 
                             let d1 = o / groups * c_req;
-                            let d2 = o
-                                * ((o - 1.0) * c_req
-                                    - 2.0
-                                        * (a_rem as f64 * a_size as f64
-                                            + b_rem as f64 * b_size as f64))
-                                / (groups * rem_g);
+                            let d2 = (o * p_zz * c_req
+                                - a_size * (a_rem * p_xz + o * p_zx)
+                                - b_size * (b_rem * p_yz + o * p_zy))
+                                / groups;
 
                             let value = p1 * d1 + p2 * d2;
                             // It need to have some runtime benefit of merging the chunks

turbopack/crates/turbopack-core/src/module_graph/chunk_group_info.rs

@@ -97,8 +97,9 @@ pub struct ChunkGroupInfo {
     pub chunking_heuristics: ChunkingHeuristicsInfo,
 }
 
-/// Chunking heuristics computed by [`compute_chunk_group_info`]. `priority_routes` is a set of
-/// chunk-group indices (same indexing as [`ChunkGroupInfo::chunk_groups`]).
+/// Chunking heuristics computed by [`compute_chunk_group_info`]. `clusters` is indexed by
+/// chunk-group index (same length and order as [`ChunkGroupInfo::chunk_groups`]); `priority_routes`
+/// is a set of those indices.
 #[derive(
     Debug,
     Default,
@@ -112,6 +113,12 @@ pub struct ChunkGroupInfo {
     Decode,
 )]
 pub struct ChunkingHeuristicsInfo {
+    /// For each chunk group (by index), the set of cluster IDs it belongs to. A cluster ID is the
+    /// index of a configured cluster. A route's chunk group carries that route's clusters; chunk
+    /// groups it pulls in inherit them.
+    ///
+    /// Example: `clusters[5] = [0, 2]` — chunk group 5 is part of clusters 0 and 2.
+    pub clusters: Vec<Vec<u16>>,
     /// The set of chunk-group indices that belong to a priority route: the priority
     /// routes themselves, plus every chunk group they pull in.
     ///
@@ -155,13 +162,17 @@ impl ChunkGroupInfo {
 #[turbo_tasks::task_input]
 #[derive(Debug, Default, Clone, Hash, PartialEq, Eq, TraceRawVcs, Encode, Decode)]
 pub struct EntryHeuristics {
+    /// Cluster indices this route belongs to.
+    pub clusters: Vec<u16>,
     pub high_priority: bool,
 }
 
 impl EntryHeuristics {
-    /// Heuristics for an entry that is a high-priority route.
+    /// Heuristics for an entry that is a high-priority route: belongs to no clusters and is marked
+    /// as high priority.
     pub fn high_priority() -> Self {
         Self {
+            clusters: Vec::new(),
             high_priority: true,
         }
     }
@@ -813,9 +824,10 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
             }
         }
 
-        // Resolve per-chunk-group chunking heuristics. Entry chunk groups carry their route's
-        // priority-route flag; other chunk groups inherit it (OR) from their referencing chunk
-        // groups.
+        // Resolve per-chunk-group chunking heuristics. Entry
+        // chunk groups carry their route's clusters / priority-route flag; other chunk groups
+        // inherit the union of clusters (and OR of the flag) from their referencing chunk groups.
+        let mut clusters: Vec<RoaringBitmap> = vec![RoaringBitmap::new(); chunk_groups_map.len()];
         let mut priority_routes = RoaringBitmap::new();
 
         let mut worklist: Vec<usize> = Vec::new();
@@ -828,18 +840,24 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
             else {
                 continue;
             };
-            if !heuristics.high_priority {
+            if heuristics.clusters.is_empty() && !heuristics.high_priority {
                 continue;
             }
             if let Some(index) =
                 chunk_groups_map.get_index_of(&ChunkGroupKey::Entry(modules.clone()))
-                && priority_routes.insert(index as u32)
             {
-                worklist.push(index);
+                if clusters[index].is_empty() && !priority_routes.contains(index as u32) {
+                    worklist.push(index);
+                }
+                clusters[index].extend(heuristics.clusters.iter().map(|&c| c as u32));
+                if heuristics.high_priority {
+                    priority_routes.insert(index as u32);
+                }
             }
         }
 
         while let Some(source) = worklist.pop() {
+            let source_priority_route = priority_routes.contains(source as u32);
             let Some(targets) = inherits_from.get(&(source as u32)) else {
                 continue;
             };
@@ -848,18 +866,29 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result<Vc<ChunkGro
                 if target == source {
                     continue;
                 }
-                if priority_routes.insert(target as u32) {
+                let [source_clusters, target_clusters] =
+                    clusters.get_disjoint_mut([source, target]).unwrap();
+                let previous_target_clusters_len = target_clusters.len();
+                *target_clusters |= &*source_clusters;
+                let changed = (source_priority_route && priority_routes.insert(target as u32))
+                    || previous_target_clusters_len != target_clusters.len();
+                if changed {
                     worklist.push(target);
                 }
             }
         }
 
+        let chunk_group_clusters: Vec<Vec<u16>> = clusters
+            .into_iter()
+            .map(|bm| bm.iter().map(|id| id as u16).collect())
+            .collect();
         let chunk_group_priority_routes = RoaringBitmapWrapper(priority_routes);
 
         Ok(ChunkGroupInfo {
             module_chunk_groups: ResolvedVc::cell(module_chunk_groups),
             chunk_group_keys: chunk_groups_map.keys().cloned().collect(),
             chunking_heuristics: ChunkingHeuristicsInfo {
+                clusters: chunk_group_clusters,
                 priority_routes: chunk_group_priority_routes,
             },
             chunk_groups: chunk_groups_map