From 3c2f9be2cba4aa621a4c5c8777b60ae6e25cf8ca Mon Sep 17 00:00:00 2001 From: Sam Poder Date: Wed, 12 Aug 2026 14:23:02 -0700 Subject: [PATCH] [turbopack] Add `clusters` to chunking heuristics (#95157) The goal of this PR increase the expected benefit of merging two chunks when their overlapping chunk groups are in a "cluster" (a cluster is a set of routes that are commonly visited together). This is because in these cases the Z+Z case (one of six in the N=2 set of cases) is more likely than others. To this I have increased the probability of the Z+Z case. At the moment, to do this I've done the following to the probabilities: ```rust /* MERGED CASE (N = 2): case X + X (p = (a_rem/groups) * ((a_rem - 1)/rem_g)): size = a_size, requests = 1 case Y + Y (p = (b_rem/groups) * ((b_rem - 1)/rem_g)): size = b_size, requests = 1 case Z + Z (p = (o_groups/groups) * (o_groups - 1)/rem_g): size = (a_size + b_size), requests = 1 case X + Y (p = (a_rem/groups) * (b_rem/rem_g) + (b_rem/groups) * (a_rem/rem_g)): size = a_size + b_size, requests = 2 case X + Z (p = (a_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (a_rem/rem_g)): size = a_size + (a_size + b_size), requests = 2 case Y + Z (p = (b_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (b_rem/rem_g)): size = b_size + (a_size + b_size), requests = 2 Request count is different in this case: Z + Z (better) Requests size is different (worse) in these cases: X + Z, Y + Z Each cost / benefit is weighted by the probabilities above. There are cases when we know that Z + Z is more likely due to common user behaviour. This is based on the "cluster" chunking heuristic we provide. We increase P(Z + Z) when two or more routes in a cluster request both chunk items together (ie. request Z). P(X + Z) and P(Y + Z) are therefore less likely. To increase P(Z + Z) while maintaining a total probability of 1, we do the following (when chunk items overlap in a cluster): P'(X + Z) = (1/2) * P(X + Z) P'(Y + Z) = (1/2) * P(Y + Z) P'(Z + Z) = P(Z + Z) + (1/2) * P(X + Z) + (1/2) * P(Y + Z) Otherwise: P'(X + Z) = P(X + Z) P'(Y + Z) = P(Y + Z) P'(Z + Z) = P(Z + Z) */ ``` This increases `P'(Z + Z)` while reducing `P'(X + Z)` and `P'(Y + Z)`. It is a bit of a blunt tool for doing this, however. So if anyone has other suggestions for how to change this probabilities while maintaining existing behaviour - let me know! --- crates/next-core/src/next_config.rs | 50 +- packages/next/src/build/swc/index.ts | 3 + packages/next/src/server/config-schema.ts | 1 + packages/next/src/server/config-shared.ts | 13 + turbopack/crates/turbo-esregex/src/lib.rs | 153 +++++- .../chunking/chunk_merging_cost_benefit.md | 435 ++++++++++++++++++ .../src/chunk/chunking/production.rs | 276 +++++------ .../src/module_graph/chunk_group_info.rs | 49 +- 8 files changed, 792 insertions(+), 188 deletions(-) create mode 100644 turbopack/crates/turbopack-core/src/chunk/chunking/chunk_merging_cost_benefit.md diff --git a/crates/next-core/src/next_config.rs b/crates/next-core/src/next_config.rs index edd9ae7ee02..a6d35687e79 100644 --- a/crates/next-core/src/next_config.rs +++ b/crates/next-core/src/next_config.rs @@ -7,7 +7,7 @@ use either::Either; use rustc_hash::FxHashSet; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::Value as JsonValue; -use turbo_esregex::EsRegex; +use turbo_esregex::{EsRegex, EsRegexSet}; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ FxIndexMap, NonLocalValue, OperationValue, ResolvedVc, TryJoinIterExt, Vc, @@ -1150,6 +1150,9 @@ const DEFAULT_WEIGHT_DISTRIBUTION: f32 = 0.1; )] #[serde(rename_all = "camelCase")] pub struct TurbopackChunkingConfig { + /// Groups of pages commonly visited together, each defined by a list of regular expressions + /// matched against the route pathname. The cluster ID is the index in this list. + clusters: Option>>, /// A number between `0.0..=1.0`. Higher values weight the benefit of merging /// chunks for a single page load more heavily. A site's bounce rate is a good /// approximation if you don't have a better value. @@ -1183,10 +1186,12 @@ pub struct TurbopackChunkingConfig { #[turbo_tasks::value] pub struct TurbopackChunking { + /// The route-matching regexes for each user-defined cluster. + clusters: Vec, /// First-page-load priority as an integer percentage (`0..=100`), or `None` if unset. pub first_page_load_priority: Option, /// Route-matching regexes for priority routes. - priority_routes: Vec, + priority_routes: EsRegexSet, /// Priority-route boost as an integer percentage (e.g. `150` for a 1.5x boost), or /// `None` to use the default. pub priority_boost_percent: Option, @@ -1206,33 +1211,35 @@ pub struct TurbopackChunking { impl TurbopackChunking { /// Compute the [`EntryHeuristics`] for a route `pathname` by matching it against the configured - /// priority-route regexes. + /// cluster and priority-route regexes. pub fn entry_heuristics_for(&self, pathname: &str) -> EntryHeuristics { - let high_priority = self - .priority_routes + let clusters = self + .clusters .iter() - .filter(|regex| regex.as_regex_str().is_none()) - .any(|regex| regex.is_match(pathname)) - || regex::RegexSet::new( - self.priority_routes - .iter() - .filter_map(|regex| regex.as_regex_str()), - ) - .is_ok_and(|set| set.is_match(pathname)); - EntryHeuristics { high_priority } + .enumerate() + .filter(|(_, regexes)| regexes.is_match(pathname)) + .map(|(index, _)| index as u16) + .collect(); + let high_priority = self.priority_routes.is_match(pathname); + EntryHeuristics { + clusters, + high_priority, + } } } -/// Compile a list of route-matching [`RegexComponents`] into [`EsRegex`]es. -fn parse_route_regexes(patterns: &[RegexComponents]) -> Result> { - patterns +/// Compile a list of route-matching [`RegexComponents`] into an [`EsRegexSet`], which builds the +/// combined [`regex::RegexSet`] up front so that matching a route doesn't have to. +fn parse_route_regexes(patterns: &[RegexComponents]) -> Result { + let regexes = patterns .iter() .cloned() .map(|pattern| { EsRegex::try_from(pattern) .context("Invalid route pattern in `experimental.turbopackChunking`") }) - .collect() + .collect::>>()?; + Ok(EsRegexSet::new(regexes)) } /// Resolve `experimental.cssChunking` to the [`StyleGroupsAlgorithm`] Turbopack should use. @@ -2165,12 +2172,19 @@ impl NextConfig { #[turbo_tasks::function] pub fn turbopack_chunking(&self) -> Result> { let config = self.experimental.turbopack_chunking.as_ref(); + let clusters = config + .and_then(|c| c.clusters.as_deref()) + .unwrap_or_default() + .iter() + .map(|patterns| parse_route_regexes(patterns)) + .collect::>>()?; let priority_routes = parse_route_regexes( config .and_then(|c| c.priority_routes.as_deref()) .unwrap_or_default(), )?; Ok(TurbopackChunking { + clusters, first_page_load_priority: config .and_then(|c| c.first_page_load_priority) .map(|priority| (priority.clamp(0.0, 1.0) * 100.0).round() as u32), diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 3fa7bbefc13..25fc665b22d 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -1050,6 +1050,9 @@ function bindingToApi( ...nextConfigSerializable.experimental, turbopackChunking: { ...chunkingConfig, + clusters: chunkingConfig.clusters?.map((cluster: RegExp[]) => + cluster.map(regexComponents) + ), priorityRoutes: chunkingConfig.priorityRoutes?.map(regexComponents), }, } diff --git a/packages/next/src/server/config-schema.ts b/packages/next/src/server/config-schema.ts index 44c2011a504..31dad2d2710 100644 --- a/packages/next/src/server/config-schema.ts +++ b/packages/next/src/server/config-schema.ts @@ -396,6 +396,7 @@ export const experimentalSchema = { turbopackSharedRuntime: z.boolean().optional(), turbopackChunking: z .object({ + clusters: z.array(z.array(z.instanceof(RegExp))).optional(), firstPageLoadPriority: z.number().min(0).max(1).optional(), priorityRoutes: z.array(z.instanceof(RegExp)).optional(), priorityBoost: z.number().min(1).optional(), diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index 88af303e89c..e58e4c413b8 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -805,6 +805,19 @@ export interface ExperimentalConfig { * making chunk merging decisions and the raw size thresholds it uses. */ turbopackChunking?: { + /** + * Groups of pages commonly visited together, each defined by a list of regular + * expressions matched against the route pathname. + * + * @example + * ```js + * clusters: [ + * [/^\/dashboard/, /^\/dashboard\/settings/], + * [/^\/blog/, /^\/blog\/[^/]+$/], + * ] + * ``` + */ + clusters?: RegExp[][] /** * This is a number between `0..1`, when higher, we weight the benefits of * merging chunks for a signal page load higher. If you don't know a good diff --git a/turbopack/crates/turbo-esregex/src/lib.rs b/turbopack/crates/turbo-esregex/src/lib.rs index 3517f8e34d2..ef0a3572b3a 100644 --- a/turbopack/crates/turbo-esregex/src/lib.rs +++ b/turbopack/crates/turbo-esregex/src/lib.rs @@ -161,6 +161,89 @@ impl EsRegex { } } +/// A group of [`EsRegex`]es matched against a haystack as a unit. +/// +/// The members backed by the `regex` crate are compiled into a single [`regex::RegexSet`] once, +/// when the group is built, rather than on every match. The remainder (those that fall back to +/// `regress`, e.g. for lookahead) are matched one at a time. +#[derive(Debug, Clone)] +#[turbo_tasks::value(eq = "manual", shared, serialization = "custom")] +pub struct EsRegexSet { + /// The members, in the order they were given. Also the source of truth for equality and + /// serialization, since [`regex::RegexSet`] supports neither. + regexes: Vec, + /// The combined members, or `None` if the combined program couldn't be built. + #[turbo_tasks(trace_ignore)] + set: Option, + /// Indices into `regexes` of the members `set` doesn't cover. Usually empty. + individual: Vec, +} + +impl PartialEq for EsRegexSet { + fn eq(&self, other: &Self) -> bool { + self.regexes == other.regexes + } +} +impl Eq for EsRegexSet {} + +impl Encode for EsRegexSet { + fn encode(&self, encoder: &mut E) -> Result<(), EncodeError> { + self.regexes.encode(encoder) + } +} + +impl Decode for EsRegexSet { + fn decode>(decoder: &mut D) -> Result { + let regexes: Vec = Decode::decode(decoder)?; + Ok(EsRegexSet::new(regexes)) + } +} + +impl_borrow_decode!(EsRegexSet); + +impl Default for EsRegexSet { + fn default() -> Self { + Self::new(Vec::new()) + } +} + +impl EsRegexSet { + /// Builds the combined matcher. Members backed by `regress` can't join a + /// [`regex::RegexSet`], and the combined program has its own size limit; either way the + /// leftovers are recorded up front and matched one at a time. + pub fn new(regexes: Vec) -> Self { + let set = regex::RegexSet::new(regexes.iter().filter_map(EsRegex::as_regex_str)).ok(); + let individual = regexes + .iter() + .enumerate() + .filter(|(_, regex)| set.is_none() || regex.as_regex_str().is_none()) + .map(|(index, _)| index as u32) + .collect(); + Self { + regexes, + set, + individual, + } + } + + /// Returns true if any member matches somewhere in the `haystack`. + pub fn is_match(&self, haystack: &str) -> bool { + if let Some(set) = &self.set + && set.is_match(haystack) + { + return true; + } + self.individual + .iter() + .any(|&index| self.regexes[index as usize].is_match(haystack)) + } + + /// Returns true if the group has no members. + pub fn is_empty(&self) -> bool { + self.regexes.is_empty() + } +} + pub struct Captures<'h> { delegate: CapturesImpl<'h>, } @@ -216,7 +299,75 @@ impl<'h> Iterator for Captures<'h> { #[cfg(test)] mod tests { - use super::{EsRegex, EsRegexImpl}; + use super::{EsRegex, EsRegexImpl, EsRegexSet}; + + #[test] + fn es_regex_set_matches_either_delegate() { + // `a(?!b)` needs regress; `^/docs` is handled by the shared `RegexSet`. + let set = EsRegexSet::new(vec![ + EsRegex::new("^/docs", "").unwrap(), + EsRegex::new("a(?!b)", "").unwrap(), + ]); + assert_eq!(set.individual, vec![1]); + assert!(set.is_match("/docs/getting-started")); + assert!(set.is_match("ac")); + assert!(!set.is_match("/blog")); + assert!(!set.is_match("ab")); + } + + #[test] + fn es_regex_set_combines_every_member_when_it_can() { + let set = EsRegexSet::new(vec![ + EsRegex::new("^/docs", "").unwrap(), + EsRegex::new("^/blog", "").unwrap(), + ]); + // A miss only queries the combined set, not every member again. + assert!(set.individual.is_empty()); + assert!(set.is_match("/docs")); + assert!(set.is_match("/blog")); + assert!(!set.is_match("/about")); + } + + #[test] + fn empty_es_regex_set_never_matches() { + let set = EsRegexSet::default(); + assert!(set.is_empty()); + assert!(!set.is_match("")); + assert!(!set.is_match("/docs")); + } + + #[test] + fn oversized_es_regex_set_falls_back_to_matching_individually() { + // Each of these compiles on its own but together they blow the combined size limit. + const N: usize = 60_000; + let regexes = vec![ + EsRegex::new(&format!("^/docs/[0-9a-zA-Z]{{{N}}}"), "").unwrap(), + EsRegex::new(&format!("^/blog/[0-9a-zA-Z]{{{N}}}"), "").unwrap(), + ]; + assert!(regexes.iter().all(|regex| regex.as_regex_str().is_some())); + let set = EsRegexSet::new(regexes); + assert!(set.set.is_none()); + assert_eq!(set.individual, vec![0, 1]); + assert!(set.is_match(&format!("/docs/{}", "a".repeat(N)))); + assert!(set.is_match(&format!("/blog/{}", "a".repeat(N)))); + assert!(!set.is_match("/about")); + } + + #[test] + fn es_regex_set_round_trip_bincode() { + let set = EsRegexSet::new(vec![ + EsRegex::new("^/docs", "").unwrap(), + EsRegex::new("a(?!b)", "").unwrap(), + ]); + let config = bincode::config::standard(); + let encoded = bincode::encode_to_vec(&set, config).unwrap(); + let (decoded, len) = bincode::decode_from_slice::(&encoded, config).unwrap(); + assert_eq!(set, decoded); + assert_eq!(len, encoded.len()); + // The `RegexSet` is rebuilt on decode, not carried in the encoding. + assert!(decoded.is_match("/docs")); + assert!(decoded.is_match("ac")); + } #[test] fn round_trip_bincode() { diff --git a/turbopack/crates/turbopack-core/src/chunk/chunking/chunk_merging_cost_benefit.md b/turbopack/crates/turbopack-core/src/chunk/chunking/chunk_merging_cost_benefit.md new file mode 100644 index 00000000000..36da6702eea --- /dev/null +++ b/turbopack/crates/turbopack-core/src/chunk/chunking/chunk_merging_cost_benefit.md @@ -0,0 +1,435 @@ +# Should we merge a chunk? + +By default, for our calculations we assume that there is a probability of 2/3 that +we request exactly 1 chunk group (`N = 1`) and a probability of 1/3 that we request +2 chunk groups (`N = 2`). This is a simplification, but it should be good enough +for our purposes and it is configurable using the chunking heuristics. + +**UNMERGED CASE** + +from the total of $\text{groups}$ chunk groups + +- $a_{\text{groups}}$ chunk groups request a $a_{\text{size}}$ chunk +- $b_{\text{groups}}$ chunk groups request a $b_{\text{size}}$ chunk + +but there is an overlapy of $o_{\text{groups}}$ between them, which request both chunks. + +**MERGED CASE** + +from the total of $\text{groups}$ chunk groups + +- $a_{\text{rem}}$ chunk groups request a $a_{\text{size}}$ chunk +- $b_{\text{rem}}$ chunk groups request a $b_{\text{size}}$ chunk +- $o_{\text{groups}}$ chunk groups request the merged chunk of size $(a_{\text{size}} + b_{\text{size}})$ + +We want to compute the expected request count $e_{\text{req}}$ and the expected total requested size $e_{\text{size}}$ for the unmerged and merged case. +This will allow us to compare the two. + +To compute that we compute the two cases $N = 1$ and $N = 2$ and combine them + +$$ +\begin{flalign*} +& \begin{aligned} +e_{\text{size}} &= P(N=1) \cdot e_{\text{size}}(N=1) + P(N=2) \cdot e_{\text{size}}(N=2) \\ +e_{\text{req}} &= P(N=1) \cdot e_{\text{req}}(N=1) + P(N=2) \cdot e_{\text{req}}(N=2) +\end{aligned} & +\end{flalign*} +$$ + +We combine $e_{\text{size}}$ with $e_{\text{req}}$ using this formula: + +$$ +\begin{flalign*} +& e_{\text{cost}} = e_{\text{req}} \cdot c_{\text{req}} + e_{\text{size}} & +\end{flalign*} +$$ + +The constant $c_{\text{req}}$ is the cost of a single request in transferred bytes. We have to choose a good value for that since there is no real value of that. +This way we can compute a cost for both cases ($e_{\text{unmerged cost}}$ and $e_{\text{merged cost}}$). + +With both costs we can compute the cost benefit $d$ of merging the two chunks: + +$$ +\begin{flalign*} +& d = e_{\text{unmerged cost}} - e_{\text{merged cost}} & +\end{flalign*} +$$ + +We can also split the formula into two parts: + +$$ +\begin{flalign*} +& \begin{aligned} +d &= d_{\text{req}} \cdot c_{\text{req}} + d_{\text{size}} \\ +d_{\text{size}} &= e_{\text{unmerged size}} - e_{\text{merged size}} \\ +d_{\text{req}} &= e_{\text{unmerged req}} - e_{\text{merged req}} +\end{aligned} & +\end{flalign*} +$$ + +And we can split it further for every $N$: + +$$ +\begin{flalign*} +& \begin{aligned} +d_{\text{size}} &= P(N=1) \cdot d_{\text{size}}(N=1) + P(N=2) \cdot d_{\text{size}}(N=2) \\ +d_{\text{req}} &= P(N=1) \cdot d_{\text{req}}(N=1) + P(N=2) \cdot d_{\text{req}}(N=2) +\end{aligned} & +\end{flalign*} +$$ + +--- + +To compute $e_{\text{size}}$ and $e_{\text{req}}$ we need to determine all cases and their probabilities. + +**UNMERGED CASE (N = 1):** + +$$ +\begin{flalign*} +& \begin{aligned} +& \textbf{case X} \\ +& p = a_{\text{rem}}/\text{groups} \\ +& \text{size} = a_{\text{size}} \\ +& \text{requests} = 1 \\ +& \\ +& \textbf{case Y} \\ +& p = b_{\text{rem}}/\text{groups} \\ +& \text{size} = b_{\text{size}} \\ +& \text{requests} = 1 \\ +& \\ +& \textbf{case Z} \\ +& p = o_{\text{groups}}/\text{groups} \\ +& \text{size} = a_{\text{size}} + b_{\text{size}} \\ +& \text{requests} = 2 +\end{aligned} & +\end{flalign*} +$$ + +**MERGED CASE (N = 1):** + +$$ +\begin{flalign*} +& \begin{aligned} +& \textbf{case X} \\ +& p = a_{\text{rem}}/\text{groups} \\ +& \text{size} = a_{\text{size}} \\ +& \text{requests} = 1 \\ +& \\ +& \textbf{case Y} \\ +& p = b_{\text{rem}}/\text{groups} \\ +& \text{size} = b_{\text{size}} \\ +& \text{requests} = 1 \\ +& \\ +& \textbf{case Z} \\ +& p = o_{\text{groups}}/\text{groups} \\ +& \text{size} = a_{\text{size}} + b_{\text{size}} \\ +& \text{requests} = 1 +\end{aligned} & +\end{flalign*} +$$ + +There is no difference in the sizes at all, so that means: + +$$ +\begin{flalign*} +& d_{\text{size}}(N=1) = 0 & +\end{flalign*} +$$ + +The only difference is in case $Z$ in the request count. That case has $p = o_{\text{groups}}/\text{groups}$: + +$$ +\begin{flalign*} +& \begin{aligned} +d_{\text{req}}(N=1) &= (o_{\text{groups}} / \text{groups}) \cdot (2 - 1) \\ +d_{\text{req}}(N=1) &= o_{\text{groups}} / \text{groups} \\ +d(N=1) &= d_{\text{req}}(N=1) \cdot c_{\text{req}} + d_{\text{size}}(N=1) \\ +&= (o_{\text{groups}} \cdot c_{\text{req}}) / \text{groups} +\end{aligned} & +\end{flalign*} +$$ + +--- + +Each $N = 2$ case's probability $p$ is the probability of navigating +to a page in the first group, then from that page to the second +group. That second probability is the transition probability, +written $\text{trans}(1 \to 2)$. + +Route "clusters" can be configured to signal that users are more +likely to navigate between pages in the same cluster. This changes +the transition probability. When no clusters are configured: + +$$ +\begin{flalign*} +& \begin{aligned} +\text{trans}(X \to X) &= (a_{\text{rem}} - 1)/(\text{groups} - 1) \\ +\text{trans}(X \to Y) &= b_{\text{rem}}/(\text{groups} - 1) \\ +\text{trans}(X \to Z) &= o_{\text{groups}}/(\text{groups} - 1) \\ +\text{trans}(Y \to X) &= a_{\text{rem}}/(\text{groups} - 1) \\ +\text{trans}(Y \to Y) &= (b_{\text{rem}} - 1)/(\text{groups} - 1) \\ +\text{trans}(Y \to Z) &= o_{\text{groups}}/(\text{groups} - 1) \\ +\text{trans}(Z \to X) &= a_{\text{rem}}/(\text{groups} - 1) \\ +\text{trans}(Z \to Y) &= b_{\text{rem}}/(\text{groups} - 1) \\ +\text{trans}(Z \to Z) &= (o_{\text{groups}} - 1)/(\text{groups} - 1) +\end{aligned} & +\end{flalign*} +$$ + +and the $N = 2$ cases reduce to: + +$$ +\begin{flalign*} +& \begin{aligned} +& \textbf{case X + X} \\ +& p = (a_{\text{rem}}/\text{groups}) \cdot ((a_{\text{rem}} - 1)/(\text{groups} - 1)) \\ +& \\ +& \textbf{case Y + Y} \\ +& p = (b_{\text{rem}}/\text{groups}) \cdot ((b_{\text{rem}} - 1)/(\text{groups} - 1)) \\ +& \\ +& \textbf{case Z + Z} \\ +& p = (o_{\text{groups}}/\text{groups}) \cdot ((o_{\text{groups}} - 1)/(\text{groups} - 1)) \\ +& \\ +& \textbf{case X + Y} \\ +& p = (a_{\text{rem}}/\text{groups}) \cdot (b_{\text{rem}}/(\text{groups} - 1)) + (b_{\text{rem}}/\text{groups}) \cdot (a_{\text{rem}}/(\text{groups} - 1)) \\ +& \\ +& \textbf{case X + Z} \\ +& p = (a_{\text{rem}}/\text{groups}) \cdot (o_{\text{groups}}/(\text{groups} - 1)) + (o_{\text{groups}}/\text{groups}) \cdot (a_{\text{rem}}/(\text{groups} - 1)) \\ +& \\ +& \textbf{case Y + Z} \\ +& p = (b_{\text{rem}}/\text{groups}) \cdot (o_{\text{groups}}/(\text{groups} - 1)) + (o_{\text{groups}}/\text{groups}) \cdot (b_{\text{rem}}/(\text{groups} - 1)) +\end{aligned} & +\end{flalign*} +$$ + +$X$, $Y$ and $Z$ are three sets of chunk groups: + +$$ +\begin{flalign*} +& \begin{aligned} +X &= \text{the } a_{\text{rem}} \text{ groups that load only chunk A} \\ +Y &= \text{the } b_{\text{rem}} \text{ groups that load only chunk B} \\ +Z &= \text{the } o_{\text{groups}} \text{ groups that load both} +\end{aligned} & +\end{flalign*} +$$ + +Now, when clusters are configured, it is more complicated. + +Two groups that sit in the same cluster form a "pair". The table below +counts the number of pairs that exist between groups. Each cell says +how many pairs have one group in one set and the other group in another +set. The diagonal cells ($c_{xx}$, $c_{yy}$, $c_{zz}$) count pairs where +the paired groups are in the same set. + +$$ +\begin{flalign*} +& \begin{array}{c|ccc} + & X\\,(a_{\text{rem}}) & Y\\,(b_{\text{rem}}) & Z\\,(\text{overlap}) \\ +\hline +X\\,(a_{\text{rem}}) & c_{xx} & c_{xy} & c_{xz} \\ +Y\\,(b_{\text{rem}}) & c_{xy} & c_{yy} & c_{yz} \\ +Z\\,(\text{overlap}) & c_{xz} & c_{yz} & c_{zz} +\end{array} & +\end{flalign*} +$$ + +Each row sum is the total number of pairs leaving that set: + +$$ +\begin{flalign*} +& \begin{aligned} +\text{paired}\_x &= c\_{xx} + c\_{xy} + c\_{xz} \\ +\text{paired}\_y &= c\_{xy} + c\_{yy} + c\_{yz} \\ +\text{paired}\_z &= c\_{xz} + c\_{yz} + c\_{zz} +\end{aligned} & +\end{flalign*} +$$ + +`CLUSTER_NAVIGATION_PROBABILITY` ($= 0.6$, written as $\text{cnp}$ below) is the chance a +navigation stays within a cluster. + +For a first group in set 1, $\text{trans}(1 \to 2)$ can be calculated using +the following probability tree: + +``` + Will the navigation stay + within a cluster? + / \ + yes (cnp) no (1 - cnp) + / \ + Does it go to set 2? Does it go to set 2? + | | + c_12 / paired_1 non_paired_12 / non_paired_1 +``` + +$\text{non\\_paired}\_{12}$ is the amount of navigations from set 1 into set 2 +that are not within a cluster, and $\text{non\\_paired}\_1$ is the total +number of unpaired navigations leaving set 1: + +$$ +\begin{flalign*} +& \begin{aligned} +\text{non\\_paired}\_{12} &= |S_1| \cdot |S_2| - c\_{12} \\ +\text{non\\_paired}\_1 &= (\text{groups} - 1) \cdot |S_1| - \text{paired}\_1 +\end{aligned} & +\end{flalign*} +$$ + +When set 1 and set 2 are the same set, the $|S_1|$ in $\text{non\\_paired}\_{12}$ +becomes $|S_1| - 1$, since a group cannot navigate to itself. The $|S_1|$ in +$\text{non\\_paired}\_1$ is unchanged, as every group is still a possible +starting point. + +Therefore: + +$$ +\begin{flalign*} +& \text{trans}(1 \to 2) = \text{cnp} \cdot (c\_{12} / \text{paired}\_1) + (1 - \text{cnp}) \cdot (\text{non\\_paired}\_{12} / \text{non\\_paired}\_1) & +\end{flalign*} +$$ + +To prevent dividing by zero, if $\text{paired}\_1 = 0$ then +$\text{trans}(1 \to 2) = |S_2| / (\text{groups} - 1)$, and if $\text{non\\_paired}\_1 = 0$ +then $\text{trans}(1 \to 2) = c\_{12} / \text{paired}\_1$. + +--- + +In terms of transition probabilities, $d$ is: + +**UNMERGED CASE (N = 2):** + +$$ +\begin{flalign*} +& \begin{aligned} +& \textbf{case X + X} \\ +& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to X) \\ +& \text{size} = a_{\text{size}} \\ +& \text{requests} = 1 \\ +& \\ +& \textbf{case Y + Y} \\ +& p = (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to Y) \\ +& \text{size} = b_{\text{size}} \\ +& \text{requests} = 1 \\ +& \\ +& \textbf{case Z + Z} \\ +& p = (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Z) \\ +& \text{size} = a_{\text{size}} + b_{\text{size}} \\ +& \text{requests} = 2 \\ +& \\ +& \textbf{case X + Y} \\ +& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Y) + (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to X) \\ +& \text{size} = a_{\text{size}} + b_{\text{size}} \\ +& \text{requests} = 2 \\ +& \\ +& \textbf{case X + Z} \\ +& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to X) \\ +& \text{size} = a_{\text{size}} + b_{\text{size}} \\ +& \text{requests} = 2 \\ +& \\ +& \textbf{case Y + Z} \\ +& p = (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Y) \\ +& \text{size} = a_{\text{size}} + b_{\text{size}} \\ +& \text{requests} = 2 +\end{aligned} & +\end{flalign*} +$$ + +**MERGED CASE (N = 2):** + +$$ +\begin{flalign*} +& \begin{aligned} +& \textbf{case X + X} \\ +& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to X) \\ +& \text{size} = a_{\text{size}} \\ +& \text{requests} = 1 \\ +& \\ +& \textbf{case Y + Y} \\ +& p = (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to Y) \\ +& \text{size} = b_{\text{size}} \\ +& \text{requests} = 1 \\ +& \\ +& \textbf{case Z + Z} \\ +& p = (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Z) \\ +& \text{size} = (a_{\text{size}} + b_{\text{size}}) \\ +& \text{requests} = 1 \\ +& \\ +& \textbf{case X + Y} \\ +& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Y) + (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to X) \\ +& \text{size} = a_{\text{size}} + b_{\text{size}} \\ +& \text{requests} = 2 \\ +& \\ +& \textbf{case X + Z} \\ +& p = (a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to X) \\ +& \text{size} = a_{\text{size}} + (a_{\text{size}} + b_{\text{size}}) \\ +& \text{requests} = 2 \\ +& \\ +& \textbf{case Y + Z} \\ +& p = (b_{\text{rem}}/\text{groups}) \cdot \text{trans}(Y \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Y) \\ +& \text{size} = b_{\text{size}} + (a_{\text{size}} + b_{\text{size}}) \\ +& \text{requests} = 2 +\end{aligned} & +\end{flalign*} +$$ + +Request count is different in this case: $Z + Z$ (better) + +Requests size is different (worse) in these cases: $X + Z$, $Y + Z$ + +$$ +\begin{flalign*} +& \begin{aligned} +d_{\text{req } Z+Z} &= ((o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Z)) \cdot (2 - 1) \\ +&= (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Z) +\end{aligned} & +\end{flalign*} +$$ + +$$ +\begin{flalign*} +& d_{\text{req}}(N=2) = (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to Z) & +\end{flalign*} +$$ + +$$ +\begin{flalign*} +& \begin{aligned} +d_{\text{size } X+Z} &= ((a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to X)) \cdot (a_{\text{size}} + b_{\text{size}} - (a_{\text{size}} + (a_{\text{size}} + b_{\text{size}}))) \\ +&= ((a_{\text{rem}}/\text{groups}) \cdot \text{trans}(X \to Z) + (o_{\text{groups}}/\text{groups}) \cdot \text{trans}(Z \to X)) \cdot (-a_{\text{size}}) \\ +&= -a_{\text{size}} \cdot (a_{\text{rem}} \cdot \text{trans}(X \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to X)) / \text{groups} \\ +& \\ +d_{\text{size } Y+Z} &= -b_{\text{size}} \cdot (b_{\text{rem}} \cdot \text{trans}(Y \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to Y)) / \text{groups} +\end{aligned} & +\end{flalign*} +$$ + +$$ +\begin{flalign*} +& \begin{aligned} +d_{\text{size}}(N=2) &= -(a_{\text{size}} \cdot (a_{\text{rem}} \cdot \text{trans}(X \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to X)) \\ +&\quad + b_{\text{size}} \cdot (b_{\text{rem}} \cdot \text{trans}(Y \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to Y))) / \text{groups} +\end{aligned} & +\end{flalign*} +$$ + +$$ +\begin{flalign*} +& \begin{aligned} +d(N=2) &= d_{\text{req}}(N=2) \cdot c_{\text{req}} + d_{\text{size}}(N=2) \\ +&= (o_{\text{groups}} \cdot \text{trans}(Z \to Z) \cdot c_{\text{req}} \\ +&\quad - a_{\text{size}} \cdot (a_{\text{rem}} \cdot \text{trans}(X \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to X)) \\ +&\quad - b_{\text{size}} \cdot (b_{\text{rem}} \cdot \text{trans}(Y \to Z) + o_{\text{groups}} \cdot \text{trans}(Z \to Y))) / \text{groups} +\end{aligned} & +\end{flalign*} +$$ + +--- + +Finally, + +$$ +\begin{flalign*} +& d = P(N=1) \cdot d(N=1) + P(N=2) \cdot d(N=2) & +\end{flalign*} +$$ diff --git a/turbopack/crates/turbopack-core/src/chunk/chunking/production.rs b/turbopack/crates/turbopack-core/src/chunk/chunking/production.rs index 81538e202c4..5b661b70411 100644 --- a/turbopack/crates/turbopack-core/src/chunk/chunking/production.rs +++ b/turbopack/crates/turbopack-core/src/chunk/chunking/production.rs @@ -1,7 +1,8 @@ use std::{borrow::Cow, collections::BinaryHeap, hash::BuildHasherDefault, mem::take}; use anyhow::{Context, Result}; -use rustc_hash::FxHasher; +use roaring::RoaringBitmap; +use rustc_hash::{FxHashMap, FxHasher}; use smallvec::SmallVec; use tracing::{Instrument, field::Empty}; use turbo_prehash::BuildHasherExt; @@ -22,6 +23,9 @@ use crate::{ /// Default estimated cost of an additional request, in bytes (200 KB). const DEFAULT_ESTIMATED_REQUEST_COST_BYTES: u64 = 200_000; +/// Probability that a navigation stays within a cluster. +const CLUSTER_NAVIGATION_PROBABILITY: f64 = 0.6; + pub async fn make_production_chunks( chunk_items: Vec<&ChunkItemOrBatchWithInfo>, batch_groups: Vec>, @@ -254,6 +258,10 @@ pub async fn make_production_chunks( let priority_boost = priority_boost_percent.map_or(1.5, |percent| percent as f64 / 100.0); + // If chunk group clusters are configured in `next.config.js` and the patterns + // match at least one route. + let has_clusters = heuristics.clusters.iter().any(|c| !c.is_empty()); + let mut iterations = 0; while chunks_to_merge.len() > 1 { // Find best candidate @@ -306,158 +314,14 @@ pub async fn make_production_chunks( let a_rem = a_groups - o_groups; let b_rem = b_groups - o_groups; - /* - UNMERGED CASE + // See ./chunk_merging_cost_benefit.md for a description of how + // this works. - from the total of `groups` chunk groups - - `a_groups` chunk groups request a `a_size` chunk - - `b_groups` chunk groups request a `b_size` chunk - but there is an overlapy of `o_groups` between them, which request both chunks. - - MERGED CASE - - from the total of `groups` chunk groups - - `a_rem` chunk groups request a `a_size` chunk - - `b_rem` chunk groups request a `b_size` chunk - - `o_groups` chunk groups request the merged chunk of size `(a_size + b_size)` - */ - - /* - By default, for our calculations we assume that there is a probability of 2/3 that - we request exactly 1 chunk group (`N = 1`) and a probability of 1/3 that we request - 2 chunk groups (`N = 2`). This is a simplification, but it should be good enough - for our purposes and it is configurable using the chunking heuristics. - - We want to compute the expected request count `e_req` and the expected total requested size `e_size` for the unmerged and merged case. - - To compute that we compute the two cases `N = 1` and `N = 2` and combine them - e_size = P(N = 1) * e_size(N = 1) + P(N = 2) * e_size(N = 2) - e_req = P(N = 1) * e_req(N = 1) + P(N = 2) * e_req(N = 2) - - We combine `e_size` with `e_req` using this formula: - e_cost = e_req * c_req + e_size - - The constant `c_req` is the cost of a single request in transferred bytes. We have to choose a good value for that since there is no real value of that. - This way we can compute a cost for both cases (`e_cost_unmerged` and `e_cost_merged`). - - With both costs we can compute the cost benefit `d` of merging the two chunks: - d = e_cost_unmerged - e_cost_merged - - We can also split the formula into two parts: - d = d_req * c_req + d_size - d_size = e_size_unmerged - e_size_merged - d_req = e_req_unmerged - e_req_merged - - And we can split it further for every N: - d_size = P(N = 1) * d_size(N = 1) + P(N = 2) * d_size(N = 2) - d_req = P(N = 1) * d_req(N = 1) + P(N = 2) * d_req(N = 2) - */ - - /* - To compute `e_size` and `e_req` we need to determine all cases and their probabilities. - - UNMERGED CASE (N = 1): - - case X (p = a_rem/groups): size = a_size, requests = 1 - case Y (p = b_rem/groups): size = b_size, requests = 1 - case Z (p = o_groups/groups): size = a_size + b_size, requests = 2 - - MERGED CASE (N = 1): - - case X (p = a_rem/groups): size = a_size, requests = 1 - case Y (p = b_rem/groups): size = b_size, requests = 1 - case Z (p = o_groups/groups): size = a_size + b_size, requests = 1 - */ - - /* - There is no difference in the sizes at all, so that means: - - d_size(N = 1) = 0 - - The only difference is in case Z in the request count. That case has `p = o_groups/groups`: - - d_req(N = 1) = (o_groups / groups) * (2 - 1) - d_req(N = 1) = o_groups / groups - - d(N = 1) = d_req(N = 1) * c_req + d_size(N = 1) - = (o_groups * c_req) / groups - */ - - /* - The N = 2 case is more complicated, since we have to consider all possible combinations of the cases X, Y and Z for the two chunk groups: - - p_x = a_rem/groups - p_y = b_rem/groups - p_z = o_groups/groups - - The chunk groups remaining after the first one has been picked - rem_g = groups - 1 - - UNMERGED CASE (N = 2): - case X + X (p = (a_rem/groups) * ((a_rem - 1)/rem_g)): size = a_size, requests = 1 - case Y + Y (p = (b_rem/groups) * ((b_rem - 1)/rem_g)): size = b_size, requests = 1 - case Z + Z (p = (o_groups/groups) * (o_groups - 1)/rem_g): size = a_size + b_size, requests = 2 - case X + Y (p = (a_rem/groups) * (b_rem/rem_g) + (b_rem/groups) * (a_rem/rem_g)): size = a_size + b_size, requests = 2 - case X + Z (p = (a_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (a_rem/rem_g)): size = a_size + b_size, requests = 2 - case Y + Z (p = (b_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (b_rem/rem_g)): size = a_size + b_size, requests = 2 - - MERGED CASE (N = 2): - case X + X (p = (a_rem/groups) * ((a_rem - 1)/rem_g)): size = a_size, requests = 1 - case Y + Y (p = (b_rem/groups) * ((b_rem - 1)/rem_g)): size = b_size, requests = 1 - case Z + Z (p = (o_groups/groups) * (o_groups - 1)/rem_g): size = (a_size + b_size), requests = 1 - case X + Y (p = (a_rem/groups) * (b_rem/rem_g) + (b_rem/groups) * (a_rem/rem_g)): size = a_size + b_size, requests = 2 - case X + Z (p = (a_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (a_rem/rem_g)): size = a_size + (a_size + b_size), requests = 2 - case Y + Z (p = (b_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (b_rem/rem_g)): size = b_size + (a_size + b_size), requests = 2 - - Request count is different in this case: Z + Z (better) - Requests size is different (worse) in these cases: X + Z, Y + Z - - d_req_z_z = ((o_groups/groups) * (o_groups - 1)/rem_g) * (2 - 1) - = o_groups * (o_groups - 1) / (groups * rem_g) - d_req_x_z = ((a_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (a_rem/rem_g)) * (2 - 2) - = 0 - d_req_y_z = ((b_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (b_rem/rem_g)) * (2 - 2) - = 0 - - d_req(N = 2) = o_groups * (o_groups - 1) / (groups * rem_g) - - d_size_x_z = ((a_rem/groups) * (o_groups/rem_g) + (o_groups/groups) * (a_rem/rem_g)) * (a_size + b_size - (a_size + (a_size + b_size))) - = ((2 * a_rem * o_groups) / (groups * rem_g)) * (-a_size) - = -2 * a_rem * a_size * o_groups / (groups * rem_g) - d_size_y_z = -2 * b_rem * b_size * o_groups / (groups * rem_g) - - d_size(N = 2) = -2 * (a_rem * a_size + b_rem * b_size) * o_groups / (groups * rem_g) - - d(N = 2) = d_req(N = 2) * c_req + d_size(N = 2) - = (o_groups * (o_groups - 1) * c_req) / (groups * rem_g) - (2 * (a_rem * a_size + b_rem * b_size) * o_groups) / (groups * rem_g) - = (o_groups * (o_groups - 1) * c_req - 2 * (a_rem * a_size + b_rem * b_size) * o_groups) / (groups * rem_g) - */ - - /* - d = P(N = 1) * d(N = 1) + P(N = 2) * d(N = 2) - */ - - /* - Recall from above: - d = d_req * c_req + d_size - d_req = P(N = 1) * d_req(N = 1) + P(N = 2) * d_req(N = 2) - - `d_size` is always <= 0, so for d > 0, d_req * c_req must be - positive: - - d > 0 - d_req * c_req > 0 - (P(N = 1) * o_groups / groups + P(N = 2) * o_groups * (o_groups - 1) / (groups * rem_g)) * c_req > 0 - P(N = 1) * o_groups / groups + P(N = 2) * o_groups * (o_groups - 1) / (groups * rem_g) > 0 - P(N = 1) * o_groups * rem_g + P(N = 2) * o_groups * (o_groups - 1) > 0 - o_groups * (P(N = 1) * rem_g + P(N = 2) * (o_groups - 1)) > 0 - o_groups > 0 && P(N = 1) * rem_g + P(N = 2) * (o_groups - 1) > 0 - o_groups > 0 && P(N = 1) * (groups - 1) + P(N = 2) * (o_groups - 1) > 0 - o_groups > 0 && groups >= 2 - */ - - // It need to have some request count benefit, the - // check for that has been derived above: + // If there are no overlapping groups, there is no benefit to + // merging - skip this process. Also, our code assumes that + // more than one group requests these chunks. If it was just + // one group requesting both it should already have been merged + // in `grouped_chunk_items` above. if o_groups == 0 || groups < 2 { continue; } @@ -475,17 +339,70 @@ pub async fn make_production_chunks( // in `o_groups` would be a chunk group that requests both chunk items. let mut is_priority_route = false; + + // Distinct pairs between the sets X (a_rem), Y (b_rem) and Z (overlap) + // that are both in a cluster. + let (mut c_xx, mut c_xy, mut c_xz, mut c_yy, mut c_yz, mut c_zz) = + (0.0, 0.0, 0.0, 0.0, 0.0, 0.0); if let (Some(a), Some(b)) = (&candidate.chunk_groups, &other.chunk_groups) { - let o = &***a & &***b; // `o_groups` + let o = &***a & &***b; // `o_groups` (Z) // if there is one chunk group in `o_groups` that is used by a // priority route, we should prioritise merging these two chunk // items. is_priority_route = !o.is_disjoint(&heuristics.priority_routes); + + if has_clusters { + let x = &***a - &o; // a_rem groups: load only chunk A + let y = &***b - &o; // b_rem groups: load only chunk B + + // Map each cluster to the candidate groups it contains. + let mut cluster_groups: FxHashMap = + FxHashMap::default(); + for set in [&x, &y, &o] { + for index in set.iter() { + for &c in &heuristics.clusters[index as usize] { + cluster_groups.entry(c).or_default().insert(index); + } + } + } + + // Groups sharing >= 1 cluster with `index`, deduped across + // clusters (excluding `index` itself) so each pair counts once. + let pairs_with = |index: u32| { + let mut p = RoaringBitmap::new(); + for &c in &heuristics.clusters[index as usize] { + if let Some(groups) = cluster_groups.get(&c) { + p |= groups; + } + } + p.remove(index); + p + }; + + for index in x.iter() { + let p = pairs_with(index); + c_xx += p.intersection_len(&x) as f64; + c_xy += p.intersection_len(&y) as f64; + c_xz += p.intersection_len(&o) as f64; + } + for index in y.iter() { + let p = pairs_with(index); + c_yy += p.intersection_len(&y) as f64; + c_yz += p.intersection_len(&o) as f64; + } + for index in o.iter() { + c_zz += pairs_with(index).intersection_len(&o) as f64; + } + } } + let paired_x = c_xx + c_xy + c_xz; + let paired_y = c_xy + c_yy + c_yz; + let paired_z = c_xz + c_yz + c_zz; + let p1 = if is_priority_route { (default_p1 * priority_boost).min(1.0) } else { @@ -497,14 +414,55 @@ pub async fn make_production_chunks( let o = o_groups as f64; let groups = groups as f64; let rem_g = rem_g as f64; + let a_rem = a_rem as f64; + let b_rem = b_rem as f64; + let a_size = a_size as f64; + let b_size = b_size as f64; + + /* transition_probability(source -> dest): probability that, after landing on a page + in the `source` set, the next navigation goes to the `dest` set. + `CLUSTER_NAVIGATION_PROBABILITY` of the time it stays within a cluster + (split across the source's pairs); the + rest spreads over the non-paired groups. With no pairs it is a uniform hop. + + - pairs_to_dest: co-clustered pairs from source to dest + - source_pairs: all co-clustered pairs leaving source (its row sum) + - source_groups: number of groups in the source set + - dest_groups: groups in the dest set (minus 1 if source == dest) */ + let transition_probability = + |pairs_to_dest: f64, + source_pairs: f64, + source_groups: f64, + dest_groups: f64| { + if source_pairs == 0.0 { + // Source has no pairs: navigate uniformly. + return dest_groups / rem_g; + } + let non_paired_from_source = + rem_g * source_groups - source_pairs; + if non_paired_from_source <= 0.0 { + // Every other group is paired: all weight on the pairs. + return pairs_to_dest / source_pairs; + } + let non_paired_from_source_to_dest = + dest_groups * source_groups - pairs_to_dest; + CLUSTER_NAVIGATION_PROBABILITY * (pairs_to_dest / source_pairs) + + (1.0 - CLUSTER_NAVIGATION_PROBABILITY) + * (non_paired_from_source_to_dest + / non_paired_from_source) + }; + + let p_zz = transition_probability(c_zz, paired_z, o, o - 1.0); + let p_zx = transition_probability(c_xz, paired_z, o, a_rem); + let p_zy = transition_probability(c_yz, paired_z, o, b_rem); + let p_xz = transition_probability(c_xz, paired_x, a_rem, o); + let p_yz = transition_probability(c_yz, paired_y, b_rem, o); let d1 = o / groups * c_req; - let d2 = o - * ((o - 1.0) * c_req - - 2.0 - * (a_rem as f64 * a_size as f64 - + b_rem as f64 * b_size as f64)) - / (groups * rem_g); + let d2 = (o * p_zz * c_req + - a_size * (a_rem * p_xz + o * p_zx) + - b_size * (b_rem * p_yz + o * p_zy)) + / groups; let value = p1 * d1 + p2 * d2; // It need to have some runtime benefit of merging the chunks diff --git a/turbopack/crates/turbopack-core/src/module_graph/chunk_group_info.rs b/turbopack/crates/turbopack-core/src/module_graph/chunk_group_info.rs index 6821869ed0d..876e1eb58b3 100644 --- a/turbopack/crates/turbopack-core/src/module_graph/chunk_group_info.rs +++ b/turbopack/crates/turbopack-core/src/module_graph/chunk_group_info.rs @@ -97,8 +97,9 @@ pub struct ChunkGroupInfo { pub chunking_heuristics: ChunkingHeuristicsInfo, } -/// Chunking heuristics computed by [`compute_chunk_group_info`]. `priority_routes` is a set of -/// chunk-group indices (same indexing as [`ChunkGroupInfo::chunk_groups`]). +/// Chunking heuristics computed by [`compute_chunk_group_info`]. `clusters` is indexed by +/// chunk-group index (same length and order as [`ChunkGroupInfo::chunk_groups`]); `priority_routes` +/// is a set of those indices. #[derive( Debug, Default, @@ -112,6 +113,12 @@ pub struct ChunkGroupInfo { Decode, )] pub struct ChunkingHeuristicsInfo { + /// For each chunk group (by index), the set of cluster IDs it belongs to. A cluster ID is the + /// index of a configured cluster. A route's chunk group carries that route's clusters; chunk + /// groups it pulls in inherit them. + /// + /// Example: `clusters[5] = [0, 2]` — chunk group 5 is part of clusters 0 and 2. + pub clusters: Vec>, /// The set of chunk-group indices that belong to a priority route: the priority /// routes themselves, plus every chunk group they pull in. /// @@ -155,13 +162,17 @@ impl ChunkGroupInfo { #[turbo_tasks::task_input] #[derive(Debug, Default, Clone, Hash, PartialEq, Eq, TraceRawVcs, Encode, Decode)] pub struct EntryHeuristics { + /// Cluster indices this route belongs to. + pub clusters: Vec, pub high_priority: bool, } impl EntryHeuristics { - /// Heuristics for an entry that is a high-priority route. + /// Heuristics for an entry that is a high-priority route: belongs to no clusters and is marked + /// as high priority. pub fn high_priority() -> Self { Self { + clusters: Vec::new(), high_priority: true, } } @@ -813,9 +824,10 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result = vec![RoaringBitmap::new(); chunk_groups_map.len()]; let mut priority_routes = RoaringBitmap::new(); let mut worklist: Vec = Vec::new(); @@ -828,18 +840,24 @@ pub async fn compute_chunk_group_info(graph: &ModuleGraph) -> Result Result> = clusters + .into_iter() + .map(|bm| bm.iter().map(|id| id as u16).collect()) + .collect(); let chunk_group_priority_routes = RoaringBitmapWrapper(priority_routes); Ok(ChunkGroupInfo { module_chunk_groups: ResolvedVc::cell(module_chunk_groups), chunk_group_keys: chunk_groups_map.keys().cloned().collect(), chunking_heuristics: ChunkingHeuristicsInfo { + clusters: chunk_group_clusters, priority_routes: chunk_group_priority_routes, }, chunk_groups: chunk_groups_map