diff --git a/api/db/init_data/compilation_templates/tree.yaml b/api/db/init_data/compilation_templates/tree.yaml
index e241c929a1..1b4c6ffc7d 100644
--- a/api/db/init_data/compilation_templates/tree.yaml
+++ b/api/db/init_data/compilation_templates/tree.yaml
@@ -8,11 +8,17 @@ config:
kind: tree
raptor:
prompt: |-
- Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:
- {cluster_content}
- The above is the content you need to summarize.
+ Summarize the paragraphs below without inventing facts or changing numbers.
+ Output exactly two parts in the same language as the source:
+ 1. First line: a concise title only.
+ 2. Following lines: a concise summary of the content.
+ Do not output labels, Markdown headings, bullet points, or any other commentary.
+
+ Paragraphs:
+ {cluster_content}
max_token: 512
- threshold: 0.1
+ clustering_threshold: 0.3
+ clustering_ratio: 0.5
# Entity / relation collections are not used by the tree kind but kept
# as empty stubs so the form schema's shared validators see a stable
# shape regardless of kind.
diff --git a/api/utils/api_utils.py b/api/utils/api_utils.py
index 0cde1eac2f..e9fd8af972 100644
--- a/api/utils/api_utils.py
+++ b/api/utils/api_utils.py
@@ -345,9 +345,10 @@ def get_parser_config(chunk_method, parser_config):
"topn_tags": 3,
"raptor": {
"use_raptor": True,
- "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
- "max_token": 256,
- "threshold": 0.1,
+ "prompt": "Summarize the paragraphs below without inventing facts or changing numbers.\nOutput exactly two parts in the same language as the source:\n1. First line: a concise title only.\n2. Following lines: a concise summary of the content.\nDo not output labels, Markdown headings, bullet points, or any other commentary.\n\nParagraphs:\n{cluster_content}",
+ "max_token": 512,
+ "clustering_threshold": 0.3,
+ "clustering_ratio": 0.5,
"max_cluster": 64,
"random_seed": 0,
},
diff --git a/api/utils/validation_utils.py b/api/utils/validation_utils.py
index 450ac95045..70b3a3f47b 100644
--- a/api/utils/validation_utils.py
+++ b/api/utils/validation_utils.py
@@ -360,18 +360,16 @@ class RaptorConfig(Base):
str,
StringConstraints(strip_whitespace=True, min_length=1),
Field(
- default="Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize."
+ default="Summarize the paragraphs below without inventing facts or changing numbers.\nOutput exactly two parts in the same language as the source:\n1. First line: a concise title only.\n2. Following lines: a concise summary of the content.\nDo not output labels, Markdown headings, bullet points, or any other commentary.\n\nParagraphs:\n{cluster_content}"
),
]
- max_token: Annotated[int, Field(default=256, ge=1, le=2048)]
- threshold: Annotated[float, Field(default=0.1, ge=0.0, le=1.0)]
+ max_token: Annotated[int, Field(default=512, ge=512, le=2048)]
+ clustering_threshold: Annotated[float, Field(default=0.3, ge=0.0, le=1.0)]
+ clustering_ratio: Annotated[float, Field(default=0.5, ge=0.0, le=1.0)]
max_cluster: Annotated[int, Field(default=64, ge=1, le=1024)]
random_seed: Annotated[int, Field(default=0, ge=0)]
scope: Annotated[Literal["file", "dataset"], Field(default="file")]
- clustering_method: Annotated[Literal["gmm", "ahc"], Field(default="gmm")]
- tree_builder: Annotated[Literal["raptor", "psi"], Field(default="raptor")]
auto_disable_for_structured_data: Annotated[bool, Field(default=True)]
- ext: Annotated[dict, Field(default={})]
class GraphragConfig(Base):
diff --git a/rag/advanced_rag/knowlege_compile/raptor.py b/rag/advanced_rag/knowlege_compile/raptor.py
index d68b265af3..48723a6844 100644
--- a/rag/advanced_rag/knowlege_compile/raptor.py
+++ b/rag/advanced_rag/knowlege_compile/raptor.py
@@ -14,14 +14,11 @@
# limitations under the License.
#
import asyncio
-from dataclasses import dataclass, field
import logging
import re
import numpy as np
-from sklearn.mixture import GaussianMixture
-
from api.db.services.task_service import has_canceled
from common.connection_utils import timeout
from common.exceptions import TaskCanceledException
@@ -34,132 +31,6 @@ from rag.graphrag.utils import (
set_llm_cache,
)
from common.misc_utils import thread_pool_exec
-from rag.utils.raptor_utils import (
- AHC_CLUSTERING_METHOD,
- GMM_CLUSTERING_METHOD,
- PSI_TREE_BUILDER,
- RAPTOR_TREE_BUILDER,
- SUPPORTED_CLUSTERING_METHODS,
- SUPPORTED_TREE_BUILDERS,
-)
-
-# Regularization added to GMM covariance diagonals; keeps components
-# from collapsing on singleton/near-identical reduced points.
-_GMM_REG_COVAR = 1e-4
-
-
-@dataclass
-class _PsiTreeNode:
- """Node used to represent the in-memory Psi merge tree."""
-
- index: int
- text: str = ""
- embedding: np.ndarray | None = None
- children: list["_PsiTreeNode"] = field(default_factory=list)
- parent: "_PsiTreeNode | None" = None
- # Original (leaf-level) chunk ids that contributed to this node. On
- # a leaf this is a single-element list with the leaf's own id; on an
- # internal node it's the order-preserving deduped union of its
- # children's lists. Carried up through the merge tree so each
- # produced summary knows which source chunks it covers.
- source_chunk_ids: list[str] = field(default_factory=list)
-
-
-class _PsiUnionFind:
- """Build parent links for the Psi merge tree from ranked leaf pairs."""
-
- def __init__(self, n: int):
- """Initialize the union-find state for n leaf nodes."""
- self._rank = [0 for _ in range(n)]
- self._parent_chains = [[] for _ in range(n)]
- self._node_ids = [[i] for i in range(n)]
- self._tree = [-1 for _ in range(max(1, 2 * n - 1))]
- self._next_id = n
-
- @staticmethod
- def _ordered_extend(target: list[int], values: list[int]):
- """Append unseen values while preserving their original order."""
- for value in values:
- if value not in target:
- target.append(value)
-
- def _find(self, i: int) -> list[int]:
- """Return the parent chain for a leaf, extending it lazily."""
- chain = self._parent_chains[i]
- if not chain or (len(chain) == 1 and chain[0] == i):
- return [i]
- if chain[0] == i:
- self._ordered_extend(chain, self._find(chain[1]))
- else:
- self._ordered_extend(chain, self._find(chain[0]))
- return chain
-
- def _rank_bisect_right(self, chain: list[int], rank: int) -> int:
- """Return the first chain index whose rank is greater than rank."""
- idx = 0
- while idx < len(chain) and self._rank[chain[idx]] <= rank:
- idx += 1
- return idx
-
- def _build(self, i: int, j: int, insert_point: int | None = None):
- """Record a merge edge in the compact parent array."""
- if insert_point is not None:
- parent_ids = self._node_ids[insert_point]
- parent_rank_idx = self._rank[i] + 1
- if parent_rank_idx >= len(parent_ids):
- logging.warning(
- "RAPTOR Psi union fallback: rank index %d is out of bounds for node %d with %d parent ids",
- parent_rank_idx,
- insert_point,
- len(parent_ids),
- )
- parent_rank_idx = len(parent_ids) - 1
- self._tree[self._node_ids[i][-1]] = parent_ids[parent_rank_idx]
- return
- self._tree[self._node_ids[i][-1]] = self._next_id
- self._tree[self._node_ids[j][-1]] = self._next_id
- self._node_ids[i].append(self._next_id)
- self._next_id += 1
-
- def union(self, i: int, j: int) -> bool:
- """Merge two ranked leaves and return whether a new edge was added."""
- root_i = self._find(i)[-1]
- root_j = self._find(j)[-1]
- if root_i == root_j:
- return False
-
- if self._rank[root_i] < self._rank[root_j]:
- if not self._parent_chains[root_j]:
- self._parent_chains[root_j].append(root_j)
- chain = self._parent_chains[j]
- higher_rank_idx = self._rank_bisect_right(chain, self._rank[root_i])
- if higher_rank_idx >= len(chain):
- higher_rank_idx = len(chain) - 1
- insert_point = chain[higher_rank_idx]
- self._ordered_extend(self._parent_chains[root_i], chain[higher_rank_idx:])
- self._build(root_i, root_j, insert_point=insert_point)
- elif self._rank[root_i] > self._rank[root_j]:
- if not self._parent_chains[root_i]:
- self._parent_chains[root_i].append(root_i)
- chain = self._parent_chains[i]
- higher_rank_idx = self._rank_bisect_right(chain, self._rank[root_j])
- if higher_rank_idx >= len(chain):
- higher_rank_idx = len(chain) - 1
- insert_point = chain[higher_rank_idx]
- self._ordered_extend(self._parent_chains[root_j], chain[higher_rank_idx:])
- self._build(root_j, root_i, insert_point=insert_point)
- else:
- if not self._parent_chains[root_i]:
- self._parent_chains[root_i].append(root_i)
- self._ordered_extend(self._parent_chains[root_j], self._parent_chains[i][-1:])
- self._rank[root_i] += 1
- self._build(root_i, root_j)
- return True
-
- @property
- def tree(self) -> list[int]:
- """Return the compact child-to-parent array for constructed nodes."""
- return self._tree[: self._next_id]
class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:
@@ -172,42 +43,33 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:
embd_model,
prompt,
max_token=512,
- threshold=0.1,
small_layer_collapse=8,
max_errors=3,
- tree_builder=RAPTOR_TREE_BUILDER,
- clustering_method=GMM_CLUSTERING_METHOD,
- psi_exact_max_leaves=4096,
- psi_bucket_size=1024,
- cluster_percentile=30,
+ clustering_threshold=0.3,
+ clustering_ratio=0.5,
):
- """Configure RAPTOR summarization, clustering, and Psi limits.
+ """Configure RAPTOR summarization and clustering.
Args:
- cluster_percentile: AHC distance threshold is set to this
- percentile of all pairwise cosine distances in each
- layer. A lower value produces finer (more) clusters.
- Default 30 means the threshold excludes the top 70%
- most dissimilar pairs.
+ clustering_threshold: Adjacent chunks with cosine similarity
+ below this value become cluster boundaries. Default 0.3.
+ clustering_ratio: Maximum number of clusters as a fraction of
+ chunk count (e.g. 0.5 means at most 50% of chunks become
+ cluster representatives). If the threshold-based watershed
+ produces more clusters than this cap, the threshold is
+ lowered using the distribution of recorded adjacent
+ similarities.
"""
self._max_cluster = max_cluster
self._small_layer_collapse = small_layer_collapse
- self._cluster_percentile = cluster_percentile
+ self._clustering_threshold = clustering_threshold
+ self._clustering_ratio = clustering_ratio
self._llm_model = llm_model
self._embd_model = embd_model
- self._threshold = threshold
self._prompt = prompt
- self._max_token = max_token
+ self._max_token = min(max(int(max_token or 512), 512), 2048)
self._max_errors = max(1, max_errors)
self._error_count = 0
- self._tree_builder = tree_builder or RAPTOR_TREE_BUILDER
- if self._tree_builder not in SUPPORTED_TREE_BUILDERS:
- raise ValueError(f"Unsupported RAPTOR tree builder: {self._tree_builder}")
- self._clustering_method = clustering_method or GMM_CLUSTERING_METHOD
- if self._clustering_method not in SUPPORTED_CLUSTERING_METHODS:
- raise ValueError(f"Unsupported RAPTOR clustering method: {self._clustering_method}")
- self._psi_exact_max_leaves = max(2, int(psi_exact_max_leaves or 4096))
- self._psi_bucket_size = min(max(2, int(psi_bucket_size or 1024)), self._psi_exact_max_leaves)
def _check_task_canceled(self, task_id: str, message: str = ""):
"""Raise if the current document task was canceled."""
@@ -253,118 +115,80 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:
await thread_pool_exec(set_embed_cache, self._embd_model.llm_name, txt, embds)
return embds
- def _get_optimal_clusters(self, embeddings: np.ndarray, random_state: int, task_id: str = ""):
- """Choose the GMM cluster count with the lowest BIC score."""
- max_clusters = min(self._max_cluster, len(embeddings))
- if max_clusters <= 1:
- logging.info(
- "RAPTOR GMM: _get_optimal_clusters returning 1 (max_clusters=%s, embeddings=%d)",
- max_clusters,
- len(embeddings),
- )
- return 1
- n_clusters = np.arange(1, max_clusters + 1)
- bics = []
- for n in n_clusters:
- self._check_task_canceled(task_id, "get optimal clusters")
-
- gm = GaussianMixture(n_components=n, random_state=random_state, covariance_type="diag", reg_covar=_GMM_REG_COVAR)
- gm.fit(embeddings)
- bics.append(gm.bic(embeddings))
- optimal_clusters = n_clusters[np.argmin(bics)]
- return int(optimal_clusters)
-
def _get_clusters_ahc(self, embeddings: np.ndarray, task_id: str = "") -> np.ndarray:
- """Sequential clustering of adjacent embeddings (cosine similarity).
+ """1D-watershed segmentation over adjacent cosine similarities.
- Only compares **adjacent** pairs (chunk i vs chunk i+1), not all
- pairwise — ``O(N)`` complexity per layer instead of ``O(N²)``.
+ Only adjacent embeddings are compared (O(N) instead of O(N²)).
- The similarity threshold is the ``p``-th percentile of all
- adjacent-pair similarities in the current layer, so it adapts
- to each layer's data distribution automatically.
-
- Returns an array of cluster labels (contiguous 0..K-1).
+ The split threshold is taken from the ``clustering_threshold``
+ percentile of the adjacent-similarity distribution. If the resulting
+ cluster count exceeds the ``clustering_ratio`` cap, the threshold is
+ further lowered.
"""
n = len(embeddings)
if n <= 1:
return np.zeros(n, dtype=int)
- if n == 2:
- return np.arange(n)
self._check_task_canceled(task_id, "_get_clusters_ahc")
- # L2-normalize embeddings so dot product = cosine similarity
+ # L2-normalize
norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
norms = np.where(norms == 0, 1.0, norms)
normalized = embeddings / norms
# Adjacent cosine similarities (n-1 pairs)
adj_sims = np.sum(normalized[:-1] * normalized[1:], axis=1)
- if len(adj_sims) == 0:
- return np.zeros(n, dtype=int)
+ sorted_sims = np.sort(adj_sims) # ascending
- # Adaptive threshold from adjacent distribution
- threshold = float(np.percentile(adj_sims, self._cluster_percentile))
- labels = np.zeros(n, dtype=int)
- cluster_id = 0
- for i in range(1, n):
- if adj_sims[i - 1] >= threshold:
- labels[i] = cluster_id
- else:
- cluster_id += 1
- labels[i] = cluster_id
+ # Max clusters allowed by the ratio cap
+ max_clusters = max(1, int(round(n * self._clustering_ratio)))
+
+ def _watershed(th: float) -> np.ndarray:
+ lbl = np.zeros(n, dtype=int)
+ cid = 0
+ for i in range(1, n):
+ if adj_sims[i - 1] >= th:
+ lbl[i] = cid
+ else:
+ cid += 1
+ lbl[i] = cid
+ return lbl
+
+ # ---- Phase 1: watershed at percentile-based threshold ----
+ # clustering_threshold (e.g. 0.3) denotes the percentile of the
+ # adjacent-similarity distribution to use as the split threshold.
+ # This adapts to each layer's similarity range automatically.
+ pct = max(1, min(99, int(round(self._clustering_threshold * 100))))
+ threshold = float(np.percentile(adj_sims, pct))
+ labels = _watershed(threshold)
+ n_clusters = int(np.unique(labels).size)
+
+ # ---- Phase 2: adjust threshold if we still exceed the cap ----
+ if n_clusters > max_clusters and len(sorted_sims) >= max_clusters:
+ adjusted = float(sorted_sims[min(max_clusters - 1, len(sorted_sims) - 1)])
+ if adjusted < threshold:
+ threshold = adjusted
+ labels = _watershed(threshold)
+ n_clusters = int(np.unique(labels).size)
logging.info(
- "RAPTOR seq-clus: p=%d threshold=%.4f n_clusters=%d for %d embeddings (adj pairs=%d)",
- self._cluster_percentile,
+ "RAPTOR seq-clus: pct=%d threshold=%.4f n_clusters=%d/%d (%d chunks) cluster_ratio=%.2f",
+ pct,
threshold,
- int(np.unique(labels).size),
+ n_clusters,
+ max_clusters,
n,
- len(adj_sims),
+ self._clustering_ratio,
)
return labels
def clustering(self, embeddings, random_state: int, task_id: str = "") -> tuple[int, list[int]]:
- """Cluster one RAPTOR layer and return contiguous labels."""
+ """Cluster one RAPTOR layer using 1D-watershed and return contiguous labels."""
if len(embeddings) == 0:
return 0, []
- if self._clustering_method == AHC_CLUSTERING_METHOD:
- # AHC: cluster on raw embeddings with cosine distance.
- # UMAP is skipped because it discards semantic information
- # that average-linkage + cosine can leverage directly.
- logging.info("RAPTOR: using clustering_method=%s on raw embeddings (dim=%d)", self._clustering_method, len(embeddings[0]) if hasattr(embeddings[0], "__len__") else "?")
- asarray = np.asarray(embeddings, dtype=np.float64)
- raw_labels = self._get_clusters_ahc(asarray, task_id=task_id)
- raw_cluster_count = np.unique(raw_labels).size
- logging.info("RAPTOR AHC: _get_clusters_ahc produced n_clusters=%d", raw_cluster_count)
- labels = raw_labels
- else:
- # GMM: reduce dimensionality first (UMAP, 12D) so the
- # Gaussian mixture can find meaningful clusters.
- if len(embeddings) == 0:
- return 0, []
- reduced = np.asarray(embeddings, dtype=np.float64)
- n_neighbors = min(int((len(embeddings) - 1) ** 0.8), 100)
- import umap
-
- reduced = umap.UMAP(
- n_neighbors=max(2, n_neighbors),
- n_components=min(12, len(embeddings) - 2),
- metric="cosine",
- ).fit_transform(embeddings)
- n_clusters = int(self._get_optimal_clusters(reduced, random_state, task_id=task_id))
- if n_clusters <= 1:
- labels = [0 for _ in range(len(reduced))]
- else:
- gm = GaussianMixture(n_components=n_clusters, random_state=random_state, covariance_type="diag", reg_covar=_GMM_REG_COVAR)
- gm.fit(reduced)
- probs = gm.predict_proba(reduced)
- labels = []
- for prob in probs:
- candidates = np.where(prob > self._threshold)[0]
- labels.append(int(candidates[0]) if len(candidates) else int(np.argmax(prob)))
+ asarray = np.asarray(embeddings, dtype=np.float64)
+ labels = self._get_clusters_ahc(asarray, task_id=task_id)
normalized_labels: list[int] = []
for label in labels:
@@ -425,328 +249,6 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:
raise RuntimeError(f"RAPTOR aborted after {self._error_count} errors. Last error: {exc}") from exc
return None
- @staticmethod
- def _root(node: _PsiTreeNode) -> _PsiTreeNode:
- """Return the current root for a Psi tree node."""
- while node.parent is not None:
- node = node.parent
- return node
-
- def _rank_leaf_pairs(self, leaves: list[_PsiTreeNode]) -> np.ndarray:
- """Rank all leaf pairs by original embedding-space cosine similarity."""
- node_embeddings = np.asarray([leaf.embedding for leaf in leaves], dtype=np.float64)
- node_embeddings = self._normalize_embeddings(node_embeddings)
- similarities = node_embeddings @ node_embeddings.T
- lower = np.tril_indices(len(leaves), -1)
- ordered = np.argsort(similarities[lower], axis=0)[::-1]
- return np.stack([lower[0][ordered], lower[1][ordered]], axis=-1)
-
- @staticmethod
- def _normalize_embeddings(node_embeddings: np.ndarray) -> np.ndarray:
- """Normalize embeddings for cosine operations while tolerating zero vectors."""
- node_embeddings = np.asarray(node_embeddings, dtype=np.float64)
- norms = np.linalg.norm(node_embeddings, axis=1, keepdims=True)
- return node_embeddings / np.maximum(norms, 1e-12)
-
- def _split_psi_buckets(self, nodes: list[_PsiTreeNode]) -> list[list[_PsiTreeNode]]:
- """Split large Psi inputs so exact pair ranking is bounded per bucket."""
- if len(nodes) <= self._psi_bucket_size:
- return [nodes]
-
- node_embeddings = self._normalize_embeddings(np.asarray([node.embedding for node in nodes], dtype=np.float64))
- groups = [np.arange(len(nodes), dtype=int)]
- buckets = []
-
- while groups:
- group = np.asarray(groups.pop(), dtype=int)
- if len(group) <= self._psi_bucket_size:
- buckets.append(group.tolist())
- continue
-
- fanout = min(max(2, int(np.ceil(len(group) / self._psi_bucket_size))), len(group), 32)
- group_embeddings = node_embeddings[group]
- center_idx = np.linspace(0, len(group_embeddings) - 1, num=fanout, dtype=int)
- centers = group_embeddings[center_idx].copy()
-
- for _ in range(5):
- labels = np.argmax(group_embeddings @ centers.T, axis=1)
- for center_id in range(fanout):
- mask = labels == center_id
- if not np.any(mask):
- continue
- center = group_embeddings[mask].mean(axis=0)
- norm = np.linalg.norm(center)
- centers[center_id] = center / norm if norm > 0 else center
-
- labels = np.argmax(group_embeddings @ centers.T, axis=1)
- split_groups = [group[labels == center_id].tolist() for center_id in range(fanout)]
- split_groups = [bucket for bucket in split_groups if bucket]
- if len(split_groups) <= 1:
- split_groups = [group[start : start + self._psi_bucket_size].tolist() for start in range(0, len(group), self._psi_bucket_size)]
- groups.extend(split_groups)
-
- buckets = [bucket for bucket in buckets if bucket]
- buckets.sort(key=lambda bucket: (len(bucket), bucket[0]))
- return [[nodes[idx] for idx in bucket] for bucket in buckets]
-
- def _assign_prototype_embeddings(self, node: _PsiTreeNode) -> np.ndarray:
- """Assign mean child embeddings to internal Psi nodes for bucket-level ranking."""
- if not node.children:
- return np.asarray(node.embedding, dtype=np.float64)
- embeddings = np.asarray([self._assign_prototype_embeddings(child) for child in node.children], dtype=np.float64)
- node.embedding = embeddings.mean(axis=0)
- return node.embedding
-
- @staticmethod
- def _iter_nodes(root: _PsiTreeNode):
- """Yield nodes in a Psi tree using a stack traversal."""
- stack = [root]
- while stack:
- node = stack.pop()
- yield node
- stack.extend(node.children)
-
- def _create_psi_parent(self, index: int, children: list[_PsiTreeNode]) -> _PsiTreeNode:
- """Create a parent node and attach the provided children to it."""
- parent = _PsiTreeNode(index=index, children=children)
- for child in children:
- child.parent = parent
- return parent
-
- def _rebalance_psi_tree(self, root: _PsiTreeNode, next_index: int) -> tuple[_PsiTreeNode, int]:
- """Group oversized Psi tree nodes so fanout stays within max_cluster."""
- max_children = max(2, int(self._max_cluster or 2))
-
- def rebalance(node: _PsiTreeNode):
- """Recursively group children when a Psi node exceeds fanout."""
- nonlocal next_index
-
- for child in list(node.children):
- rebalance(child)
-
- while len(node.children) > max_children:
- original_children = len(node.children)
- grouped_children = []
- for start in range(0, len(node.children), max_children):
- batch = node.children[start : start + max_children]
- if len(batch) == 1:
- grouped_children.append(batch[0])
- batch[0].parent = node
- else:
- grouped_children.append(self._create_psi_parent(next_index, batch))
- grouped_children[-1].parent = node
- next_index += 1
- node.children = grouped_children
- logging.info(
- "RAPTOR Psi rebalance: node=%s children=%d grouped_to=%d max_cluster=%d",
- node.index,
- original_children,
- len(grouped_children),
- max_children,
- )
-
- rebalance(root)
- return self._root(root), next_index
-
- def _build_exact_psi_structure(
- self,
- nodes: list[_PsiTreeNode],
- next_index: int,
- task_id: str = "",
- ) -> tuple[_PsiTreeNode, int, int]:
- """Build an exact Psi subtree for a bounded node set."""
- if len(nodes) == 1:
- return nodes[0], next_index, 0
-
- ranked_pairs = self._rank_leaf_pairs(nodes)
- union_find = _PsiUnionFind(len(nodes))
- merges = 0
- for left_idx, right_idx in ranked_pairs:
- self._check_task_canceled(task_id, "Psi tree construction")
- if union_find.union(int(left_idx), int(right_idx)):
- merges += 1
- if merges == len(nodes) - 1:
- break
-
- local_nodes = {idx: node for idx, node in enumerate(nodes)}
- tree = union_find.tree
- children_by_parent = {}
- for child_idx, parent_idx in enumerate(tree):
- if child_idx not in local_nodes:
- local_nodes[child_idx] = _PsiTreeNode(index=next_index)
- next_index += 1
- if parent_idx == -1:
- continue
- children_by_parent.setdefault(parent_idx, []).append(child_idx)
- if parent_idx not in local_nodes:
- local_nodes[parent_idx] = _PsiTreeNode(index=next_index)
- next_index += 1
-
- for parent_idx, child_indices in children_by_parent.items():
- parent = local_nodes[parent_idx]
- parent.children = [local_nodes[child_idx] for child_idx in child_indices]
- for child in parent.children:
- child.parent = parent
-
- roots = [local_nodes[idx] for idx, parent_idx in enumerate(tree) if parent_idx == -1 and idx in local_nodes]
- root = max(roots, key=lambda node: node.index)
- return root, next_index, merges
-
- def _build_bucketed_psi_structure(
- self,
- nodes: list[_PsiTreeNode],
- next_index: int,
- task_id: str = "",
- ) -> tuple[_PsiTreeNode, int, int]:
- """Build large Psi trees by exact-ranking bounded buckets, then bucket roots."""
- buckets = self._split_psi_buckets(nodes)
- logging.info(
- "RAPTOR Psi bucketed build: nodes=%d buckets=%d bucket_size=%d exact_max_leaves=%d",
- len(nodes),
- len(buckets),
- self._psi_bucket_size,
- self._psi_exact_max_leaves,
- )
-
- bucket_roots = []
- merges = 0
- for bucket in buckets:
- bucket_root, next_index, bucket_merges = self._build_psi_structure_from_nodes(bucket, next_index, task_id)
- self._assign_prototype_embeddings(bucket_root)
- bucket_roots.append(bucket_root)
- merges += bucket_merges
-
- if len(bucket_roots) == 1:
- return bucket_roots[0], next_index, merges
-
- root, next_index, root_merges = self._build_psi_structure_from_nodes(bucket_roots, next_index, task_id)
- return root, next_index, merges + root_merges
-
- def _build_psi_structure_from_nodes(
- self,
- nodes: list[_PsiTreeNode],
- next_index: int,
- task_id: str = "",
- ) -> tuple[_PsiTreeNode, int, int]:
- """Build Psi structure exactly for small sets and bucket large sets."""
- if len(nodes) <= self._psi_exact_max_leaves:
- return self._build_exact_psi_structure(nodes, next_index, task_id)
- return self._build_bucketed_psi_structure(nodes, next_index, task_id)
-
- def _build_psi_structure(self, chunks, task_id: str = "") -> tuple[_PsiTreeNode, list[_PsiTreeNode]]:
- """Build the Psi merge tree from original chunk embeddings.
-
- ``chunks`` is expected in the normalized 3-tuple shape
- ``(text, vec, source_chunk_ids)`` — leaves are seeded with
- their own source ids, internal nodes get their ids set during
- layer materialization in ``_build_psi_layers``.
- """
- leaves = [
- _PsiTreeNode(
- index=i,
- text=item[0],
- embedding=np.asarray(item[1]),
- source_chunk_ids=list(item[2] if len(item) > 2 else []),
- )
- for i, item in enumerate(chunks)
- ]
- if len(leaves) == 1:
- return leaves[0], leaves
-
- root, next_index, merges = self._build_psi_structure_from_nodes(leaves, len(leaves), task_id)
- root, _ = self._rebalance_psi_tree(root, next_index)
- logging.info(
- "RAPTOR Psi tree built: leaves=%d merges=%d root_fanout=%d",
- len(leaves),
- merges,
- len(root.children),
- )
- return root, leaves
-
- @staticmethod
- def _psi_layers(root: _PsiTreeNode) -> dict[int, list[_PsiTreeNode]]:
- """Collect non-leaf Psi nodes by height for bottom-up summarization."""
- layers = {}
-
- def height(node: _PsiTreeNode) -> int:
- """Return node height while collecting internal nodes by layer."""
- if not node.children:
- return 0
- node_height = max(height(child) for child in node.children) + 1
- layers.setdefault(node_height, []).append(node)
- return node_height
-
- height(root)
- return layers
-
- async def _build_psi_layers(self, chunks, callback=None, task_id: str = ""):
- """Materialize Psi tree layers as summary chunks."""
- layers = [(0, len(chunks))]
- root, _ = self._build_psi_structure(chunks, task_id=task_id)
-
- for layer_idx, (_, nodes) in enumerate(sorted(self._psi_layers(root).items()), start=1):
- layer_start = len(chunks)
-
- async def summarize_node(node: _PsiTreeNode):
- """Summarize one Psi internal node if its children have text.
-
- Also propagates leaf provenance: the node's
- ``source_chunk_ids`` becomes the order-preserving deduped
- union of every child's ``source_chunk_ids``. Because
- children at this layer have already been processed (leaves
- first, then bottom-up), each child carries the full set
- of leaf ids underneath it — so the union here is the
- complete leaf set this summary covers.
- """
- texts = [child.text for child in node.children if child.text]
- if not texts:
- logging.warning("RAPTOR Psi node %s skipped because it has no child text to summarize", node.index)
- return None
- result = await self._summarize_texts(texts, callback, task_id)
- if result is None:
- logging.warning("RAPTOR Psi node %s skipped because summarization failed", node.index)
- return None
- _, node.text, node.embedding = result
- merged_ids: list[str] = []
- seen: set[str] = set()
- for child in node.children:
- for src in child.source_chunk_ids:
- if src and src not in seen:
- seen.add(src)
- merged_ids.append(src)
- node.source_chunk_ids = merged_ids
- return node
-
- tasks = [asyncio.create_task(summarize_node(node)) for node in nodes]
- try:
- summarized_nodes = await asyncio.gather(*tasks, return_exceptions=False)
- except Exception as e:
- logging.error(f"Error in RAPTOR Psi tree processing: {e}")
- for task in tasks:
- task.cancel()
- await asyncio.gather(*tasks, return_exceptions=True)
- raise
-
- summarized_nodes = [node for node in summarized_nodes if node is not None]
- for node in summarized_nodes:
- chunks.append((node.text, node.embedding, list(node.source_chunk_ids)))
-
- if len(chunks) > layer_start:
- layers.append((layer_start, len(chunks)))
- logging.info(
- "RAPTOR Psi layer materialized: layer=%d nodes=%d summaries=%d",
- layer_idx,
- len(nodes),
- len(chunks) - layer_start,
- )
- if callback:
- callback(msg="Build one Psi-RAG layer: {} -> {}".format(len(nodes), len(chunks) - layer_start))
- else:
- logging.warning("RAPTOR Psi layer %d produced no summaries; stopping materialization", layer_idx)
- break
-
- return chunks, layers
-
async def __call__(
self,
chunks,
@@ -803,14 +305,6 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:
return (None, None) if is_tree else (normalized, [(0, len(normalized))])
chunks = normalized
- if self._tree_builder == PSI_TREE_BUILDER:
- if is_tree:
- raise NotImplementedError(
- "is_tree=True is not supported for PSI_TREE_BUILDER",
- )
- logging.info("RAPTOR: using %s tree builder for %d chunks", self._tree_builder, len(chunks))
- return await self._build_psi_layers(chunks, callback, task_id)
-
# ``parent_child_map`` records each summary's immediate
# children so ``_materialize_tree`` can walk back into a tree
# when ``is_tree`` is set. Always populated (cheap) so the
@@ -972,18 +466,17 @@ class RecursiveAbstractiveProcessing4TreeOrganizedRetrieval:
def _build_node(idx: int) -> dict:
children_idx = parent_child_map.get(idx, [])
- # If every immediate child is a layer-0 original, this
- # node is a "leaf" in the tree contract — collapse to
- # source_chunk_ids.
+ # If every immediate child is a layer-0 original, collapse the
+ # cluster into one leaf node and retain all source chunk IDs.
if children_idx and all(c < n_originals for c in children_idx):
- ids: list[str] = []
+ source_chunk_ids: list[str] = []
seen: set[str] = set()
for c in children_idx:
for s in chunks[c][2]:
if s and s not in seen:
seen.add(s)
- ids.append(s)
- return {"title": _title_at(idx), "source_chunk_ids": ids, "description": _desc_at(idx)}
+ source_chunk_ids.append(s)
+ return {"title": _title_at(idx), "source_chunk_ids": source_chunk_ids, "description": _desc_at(idx)}
return {"children": [_build_node(c) for c in children_idx], "title": _title_at(idx), "description": _desc_at(idx)}
top_nodes = [_build_node(i) for i in range(top_start, top_end)]
diff --git a/rag/flow/compiler/compiler.py b/rag/flow/compiler/compiler.py
index ebc544e601..a92ffdd383 100644
--- a/rag/flow/compiler/compiler.py
+++ b/rag/flow/compiler/compiler.py
@@ -419,12 +419,12 @@ class Compiler(ProcessBase, LLM):
for idx, (template_id, parser_cfg) in enumerate(templates):
raptor_cfg = (parser_cfg or {}).get("raptor") or {}
raptor_config = {
- "prompt": raptor_cfg.get("prompt") or "Please write a concise summary of the following texts:\n{cluster_content}",
+ "prompt": raptor_cfg.get("prompt")
+ or "Summarize the paragraphs below without inventing facts or changing numbers.\nOutput exactly two parts in the same language as the source:\n1. First line: a concise title only.\n2. Following lines: a concise summary of the content.\nDo not output labels, Markdown headings, bullet points, or any other commentary.\n\nParagraphs:\n{cluster_content}",
"max_token": int(raptor_cfg.get("max_token") or 512),
- "threshold": float(raptor_cfg.get("threshold") or 0.1),
"random_seed": int(raptor_cfg.get("random_seed") or 0),
- "max_cluster": int(raptor_cfg.get("max_cluster") or 64),
- "ext": raptor_cfg.get("ext") or {},
+ "clustering_threshold": float(0.3 if raptor_cfg.get("clustering_threshold") is None else raptor_cfg["clustering_threshold"]),
+ "clustering_ratio": float(0.5 if raptor_cfg.get("clustering_ratio") is None else raptor_cfg["clustering_ratio"]),
}
self._compile_progress(msg=f"tree-template ({idx + 1}/{len(templates)}): building tree for doc={doc_id}")
try:
@@ -433,8 +433,6 @@ class Compiler(ProcessBase, LLM):
raptor_config=raptor_config,
chat_mdl=chat_mdl_by_tid[template_id],
embd_mdl=embedding_model,
- tree_builder="raptor",
- clustering_method="ahc",
max_errors=3,
)
except Exception:
@@ -619,9 +617,9 @@ class Compiler(ProcessBase, LLM):
self.set_output("chunks", chunks)
return
- for ck in chunks:
+ for idx, ck in enumerate(chunks):
ck["doc_id"] = doc_id
- ck["id"] = xxhash.xxh64((ck["text"] + str(ck["doc_id"])).encode("utf-8")).hexdigest()
+ ck["id"] = xxhash.xxh64(f"{ck['text']}\x00{ck['doc_id']}\x00{idx}".encode("utf-8")).hexdigest()
if self._canvas._kb_id:
e, kb = KnowledgebaseService.get_by_id(self._canvas._kb_id)
diff --git a/rag/svr/task_executor.py b/rag/svr/task_executor.py
index b4d53808fe..23db1fab30 100644
--- a/rag/svr/task_executor.py
+++ b/rag/svr/task_executor.py
@@ -46,10 +46,9 @@ from common.connection_utils import timeout
from common.metadata_utils import turn2jsonschema, update_metadata_to
from rag.utils.base64_image import image2id
from rag.utils.raptor_utils import (
+ RAPTOR_TREE_BUILDER,
collect_raptor_chunk_ids,
collect_raptor_methods,
- get_raptor_clustering_method,
- get_raptor_tree_builder,
get_skip_reason,
make_raptor_summary_chunk_id,
should_skip_raptor,
@@ -84,9 +83,7 @@ from common.versions import get_ragflow_version
from api.db.db_models import close_connection
from rag.app import laws, paper, presentation, manual, qa, table, book, resume, picture, naive, one, audio, email, tag
from rag.nlp import search, rag_tokenizer, add_positions
-from rag.advanced_rag.knowlege_compile.raptor import (
- RAPTOR_TREE_BUILDER,
-)
+
from common.token_utils import num_tokens_from_string, truncate
from rag.utils.redis_conn import REDIS_CONN, RedisDistributedLock
from rag.graphrag.utils import chat_limiter
@@ -1062,15 +1059,14 @@ async def delete_raptor_chunks(doc_id: str, tenant_id: str, kb_id: str, keep_met
@timeout(3600)
async def run_raptor_for_kb(row, kb_parser_config, chat_mdl, embd_mdl, vector_size, callback=None, doc_ids=[]):
+ tree_builder = "raptor"
+ clustering_method = "watershed"
"""Generate RAPTOR summaries for selected documents in a knowledge base."""
fake_doc_id = GRAPH_RAPTOR_FAKE_DOC_ID
rag_tokenizer.tokenizer.set_language(row.get("language", "English"))
raptor_config = kb_parser_config.get("raptor", {})
- raptor_ext_config = raptor_config.get("ext") or {}
- tree_builder = get_raptor_tree_builder(raptor_config)
- clustering_method = get_raptor_clustering_method(raptor_config)
vctr_nm = "q_%d_vec" % vector_size
res = []
@@ -1121,12 +1117,9 @@ async def run_raptor_for_kb(row, kb_parser_config, chat_mdl, embd_mdl, vector_si
embd_mdl,
raptor_config["prompt"],
raptor_config["max_token"],
- raptor_config["threshold"],
max_errors=max_errors,
- tree_builder=tree_builder,
- clustering_method=clustering_method,
- psi_exact_max_leaves=raptor_ext_config.get("psi_exact_max_leaves", 4096),
- psi_bucket_size=raptor_ext_config.get("psi_bucket_size", 1024),
+ clustering_threshold=float(raptor_config.get("clustering_threshold", 0.3)),
+ clustering_ratio=float(raptor_config.get("clustering_ratio", 0.5)),
)
original_length = len(chunks)
chunks, layers = await raptor(chunks, kb_parser_config["raptor"]["random_seed"], callback, row["id"])
@@ -1482,14 +1475,13 @@ async def do_handle_task(task):
{
"raptor": {
"use_raptor": True,
- "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
- "max_token": 256,
- "threshold": 0.1,
+ "prompt": "Summarize the paragraphs below without inventing facts or changing numbers.\nOutput exactly two parts in the same language as the source:\n1. First line: a concise title only.\n2. Following lines: a concise summary of the content.\nDo not output labels, Markdown headings, bullet points, or any other commentary.\n\nParagraphs:\n{cluster_content}",
+ "max_token": 512,
+ "clustering_threshold": 0.3,
+ "clustering_ratio": 0.5,
"max_cluster": 64,
"random_seed": 0,
"scope": "file",
- "clustering_method": "gmm",
- "tree_builder": "raptor",
},
}
)
diff --git a/rag/svr/task_executor_refactor/chunk_post_processor.py b/rag/svr/task_executor_refactor/chunk_post_processor.py
index 484ff20920..ed33f7d813 100644
--- a/rag/svr/task_executor_refactor/chunk_post_processor.py
+++ b/rag/svr/task_executor_refactor/chunk_post_processor.py
@@ -944,8 +944,6 @@ async def run_tree_templates(
raptor_config=raptor_config,
chat_mdl=chat_mdl_by_tid[template_id],
embd_mdl=embedding_model,
- tree_builder="raptor",
- clustering_method="ahc",
max_errors=3,
)
except Exception:
diff --git a/rag/svr/task_executor_refactor/raptor_service.py b/rag/svr/task_executor_refactor/raptor_service.py
index b72c952275..a095b64ef3 100644
--- a/rag/svr/task_executor_refactor/raptor_service.py
+++ b/rag/svr/task_executor_refactor/raptor_service.py
@@ -41,8 +41,6 @@ from rag.nlp import rag_tokenizer, search
from rag.utils.raptor_utils import (
collect_raptor_chunk_ids,
collect_raptor_methods,
- get_raptor_clustering_method,
- get_raptor_tree_builder,
get_skip_reason,
make_raptor_summary_chunk_id,
should_skip_raptor,
@@ -118,8 +116,6 @@ class RaptorService:
Tuple of (chunks, token_count, cleanup_raptor_chunks).
"""
raptor_config = kb_parser_config.get("raptor", {})
- tree_builder = get_raptor_tree_builder(raptor_config)
- clustering_method = get_raptor_clustering_method(raptor_config)
vctr_nm = "q_%d_vec" % vector_size
res = []
@@ -132,13 +128,9 @@ class RaptorService:
# Determine scope
if raptor_config.get("scope", "file") == "file":
- res, tk_count = await self._run_file_level_raptor(
- raptor_config, tree_builder, clustering_method, chat_mdl, embd_mdl, vctr_nm, doc_ids, doc_info_by_id, max_errors, res, tk_count, cleanup_raptor_chunks
- )
+ res, tk_count = await self._run_file_level_raptor(raptor_config, chat_mdl, embd_mdl, vctr_nm, doc_ids, doc_info_by_id, max_errors, res, tk_count, cleanup_raptor_chunks)
else:
- res, tk_count = await self._run_dataset_level_raptor(
- raptor_config, tree_builder, clustering_method, chat_mdl, embd_mdl, vctr_nm, doc_ids, doc_info_by_id, max_errors, res, tk_count, cleanup_raptor_chunks
- )
+ res, tk_count = await self._run_dataset_level_raptor(raptor_config, chat_mdl, embd_mdl, vctr_nm, doc_ids, doc_info_by_id, max_errors, res, tk_count, cleanup_raptor_chunks)
return res, tk_count, cleanup_raptor_chunks
@@ -158,7 +150,8 @@ class RaptorService:
}
return doc_info_by_id
- async def _run_file_level_raptor(self, raptor_config, tree_builder, clustering_method, chat_mdl, embd_mdl, vctr_nm, doc_ids, doc_info_by_id, max_errors, res, tk_count, cleanup_raptor_chunks):
+ async def _run_file_level_raptor(self, raptor_config, chat_mdl, embd_mdl, vctr_nm, doc_ids, doc_info_by_id, max_errors, res, tk_count, cleanup_raptor_chunks):
+ tree_builder = "raptor"
"""Run RAPTOR at file level (per document)."""
ctx = self._task_context
fake_doc_id = GRAPH_RAPTOR_FAKE_DOC_ID
@@ -197,7 +190,7 @@ class RaptorService:
continue
before_generate = len(res)
- new_chunks, new_tk_count = await self._generate_raptor(chunks, doc_id, raptor_config, chat_mdl, embd_mdl, tree_builder, clustering_method, max_errors, doc_info_by_id)
+ new_chunks, new_tk_count = await self._generate_raptor(chunks, doc_id, raptor_config, chat_mdl, embd_mdl, max_errors, doc_info_by_id)
res.extend(new_chunks)
tk_count += new_tk_count
@@ -215,7 +208,8 @@ class RaptorService:
return res, tk_count
- async def _run_dataset_level_raptor(self, raptor_config, tree_builder, clustering_method, chat_mdl, embd_mdl, vctr_nm, doc_ids, doc_info_by_id, max_errors, res, tk_count, cleanup_raptor_chunks):
+ async def _run_dataset_level_raptor(self, raptor_config, chat_mdl, embd_mdl, vctr_nm, doc_ids, doc_info_by_id, max_errors, res, tk_count, cleanup_raptor_chunks):
+ tree_builder = "raptor"
"""Run RAPTOR at dataset level (all documents combined)."""
ctx = self._task_context
fake_doc_id = GRAPH_RAPTOR_FAKE_DOC_ID
@@ -264,7 +258,7 @@ class RaptorService:
return res, tk_count
before_generate = len(res)
- new_chunks, new_tk_count = await self._generate_raptor(chunks, fake_doc_id, raptor_config, chat_mdl, embd_mdl, tree_builder, clustering_method, max_errors, doc_info_by_id)
+ new_chunks, new_tk_count = await self._generate_raptor(chunks, fake_doc_id, raptor_config, chat_mdl, embd_mdl, max_errors, doc_info_by_id)
res.extend(new_chunks)
tk_count += new_tk_count
@@ -354,8 +348,6 @@ class RaptorService:
raptor_config: Dict,
chat_mdl,
embd_mdl,
- tree_builder: str,
- clustering_method: str,
max_errors: int,
doc_info_by_id: Dict,
is_tree: bool = False,
@@ -372,7 +364,6 @@ class RaptorService:
ctx = self._task_context
from rag.advanced_rag.knowlege_compile.raptor import RecursiveAbstractiveProcessing4TreeOrganizedRetrieval as Raptor
- raptor_ext_config = raptor_config.get("ext") or {}
assert chunks, "_generate_raptor must not be called with empty chunks"
vctr_nm = "q_%d_vec" % len(chunks[0][1])
@@ -382,12 +373,9 @@ class RaptorService:
embd_mdl,
raptor_config["prompt"],
raptor_config["max_token"],
- raptor_config["threshold"],
max_errors=max_errors,
- tree_builder=tree_builder,
- clustering_method=clustering_method,
- psi_exact_max_leaves=raptor_ext_config.get("psi_exact_max_leaves", 4096),
- psi_bucket_size=raptor_ext_config.get("psi_bucket_size", 1024),
+ clustering_threshold=float(raptor_config.get("clustering_threshold", 0.3)),
+ clustering_ratio=float(raptor_config.get("clustering_ratio", 0.5)),
)
# Seed each leaf with its own id as the start of its
@@ -420,7 +408,6 @@ class RaptorService:
raptor_config,
doc_id,
effective_doc_name,
- tree_builder,
vctr_nm,
)
@@ -432,7 +419,7 @@ class RaptorService:
"docnm_kwd": effective_doc_name,
"title_tks": rag_tokenizer.tokenize(effective_doc_name),
"raptor_kwd": "raptor",
- "extra": {"raptor_method": tree_builder},
+ "extra": {"raptor_method": "raptor"},
"create_time": str(datetime.now()).replace("T", " ")[:19],
"create_timestamp_flt": datetime.now().timestamp(),
}
@@ -465,7 +452,7 @@ class RaptorService:
return res, tk_count
row_id = xxhash.xxh64(
- f"raptor_tree:{doc_id}:{tree_builder}".encode("utf-8", "surrogatepass"),
+ f"raptor_tree:{doc_id}:raptor".encode("utf-8", "surrogatepass"),
).hexdigest()
row = {
**doc,
@@ -482,8 +469,6 @@ class RaptorService:
raptor_config: Dict,
chat_mdl,
embd_mdl,
- tree_builder: str,
- clustering_method: str,
max_errors: int,
) -> Optional[Dict]:
"""Build a RAPTOR tree dict for one document — no ES IO.
@@ -497,19 +482,15 @@ class RaptorService:
return None
from rag.advanced_rag.knowlege_compile.raptor import RecursiveAbstractiveProcessing4TreeOrganizedRetrieval as Raptor
- raptor_ext_config = raptor_config.get("ext") or {}
raptor = Raptor(
raptor_config.get("max_cluster", 64),
chat_mdl,
embd_mdl,
raptor_config["prompt"],
raptor_config["max_token"],
- raptor_config["threshold"],
max_errors=max_errors,
- tree_builder=tree_builder,
- clustering_method=clustering_method,
- psi_exact_max_leaves=raptor_ext_config.get("psi_exact_max_leaves", 4096),
- psi_bucket_size=raptor_ext_config.get("psi_bucket_size", 1024),
+ clustering_threshold=float(raptor_config.get("clustering_threshold", 0.3)),
+ clustering_ratio=float(raptor_config.get("clustering_ratio", 0.5)),
)
raptor_input = [(content, vctr, [chunk_id] if chunk_id else []) for content, vctr, chunk_id in chunks]
@@ -537,7 +518,6 @@ class RaptorService:
raptor_config,
doc_id,
effective_doc_name,
- tree_builder,
vctr_nm,
) -> Tuple[List[Dict], int]:
"""Legacy per-summary materialization, kept only for PSI builds.
@@ -563,7 +543,7 @@ class RaptorService:
"docnm_kwd": effective_doc_name,
"title_tks": rag_tokenizer.tokenize(effective_doc_name),
"raptor_kwd": "raptor",
- "extra": {"raptor_method": tree_builder},
+ "extra": {"raptor_method": "raptor"},
}
if ctx.pagerank:
doc[PAGERANK_FLD] = int(ctx.pagerank)
diff --git a/rag/svr/task_executor_refactor/task_handler.py b/rag/svr/task_executor_refactor/task_handler.py
index c160b9f440..4416eafc6c 100644
--- a/rag/svr/task_executor_refactor/task_handler.py
+++ b/rag/svr/task_executor_refactor/task_handler.py
@@ -395,14 +395,13 @@ class TaskHandler:
{
"raptor": {
"use_raptor": True,
- "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
- "max_token": 256,
- "threshold": 0.1,
+ "prompt": "Summarize the paragraphs below without inventing facts or changing numbers.\nOutput exactly two parts in the same language as the source:\n1. First line: a concise title only.\n2. Following lines: a concise summary of the content.\nDo not output labels, Markdown headings, bullet points, or any other commentary.\n\nParagraphs:\n{cluster_content}",
+ "max_token": 512,
+ "clustering_threshold": 0.3,
+ "clustering_ratio": 0.5,
"max_cluster": 64,
"random_seed": 0,
"scope": "file",
- "clustering_method": "gmm",
- "tree_builder": "raptor",
},
}
)
diff --git a/rag/utils/raptor_utils.py b/rag/utils/raptor_utils.py
index 083d42b242..0ddfa46f20 100644
--- a/rag/utils/raptor_utils.py
+++ b/rag/utils/raptor_utils.py
@@ -25,11 +25,6 @@ from typing import Optional
import xxhash
RAPTOR_TREE_BUILDER = "raptor"
-PSI_TREE_BUILDER = "psi"
-SUPPORTED_TREE_BUILDERS = {RAPTOR_TREE_BUILDER, PSI_TREE_BUILDER}
-GMM_CLUSTERING_METHOD = "gmm"
-AHC_CLUSTERING_METHOD = "ahc"
-SUPPORTED_CLUSTERING_METHODS = {GMM_CLUSTERING_METHOD, AHC_CLUSTERING_METHOD}
# File extensions for structured data types
EXCEL_EXTENSIONS = {".xls", ".xlsx", ".xlsm", ".xlsb"}
@@ -37,26 +32,6 @@ CSV_EXTENSIONS = {".csv", ".tsv"}
STRUCTURED_EXTENSIONS = EXCEL_EXTENSIONS | CSV_EXTENSIONS
-def get_raptor_tree_builder(raptor_config: dict | None) -> str:
- """Return the configured RAPTOR tree builder with legacy ext fallback."""
- raptor_config = raptor_config or {}
- ext = raptor_config.get("ext") or {}
- tree_builder = ext.get("tree_builder") or raptor_config.get("tree_builder") or RAPTOR_TREE_BUILDER
- if tree_builder not in SUPPORTED_TREE_BUILDERS:
- raise ValueError(f"Unsupported RAPTOR tree builder: {tree_builder}")
- return tree_builder
-
-
-def get_raptor_clustering_method(raptor_config: dict | None) -> str:
- """Return the configured RAPTOR clustering method with legacy ext fallback."""
- raptor_config = raptor_config or {}
- ext = raptor_config.get("ext") or {}
- clustering_method = ext.get("clustering_method") or raptor_config.get("clustering_method") or GMM_CLUSTERING_METHOD
- if clustering_method not in SUPPORTED_CLUSTERING_METHODS:
- raise ValueError(f"Unsupported RAPTOR clustering method: {clustering_method}")
- return clustering_method
-
-
def _as_extra_dict(extra) -> dict:
"""Normalize a chunk extra payload into a dictionary."""
if isinstance(extra, dict):
diff --git a/test/testcases/configs.py b/test/testcases/configs.py
index e29a99ac2e..608b25d59e 100644
--- a/test/testcases/configs.py
+++ b/test/testcases/configs.py
@@ -88,9 +88,10 @@ DEFAULT_PARSER_CONFIG = {
"llm_id": "glm-4-flash@CI@ZHIPU-AI",
"raptor": {
"use_raptor": True,
- "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
- "max_token": 256,
- "threshold": 0.1,
+ "prompt": "Summarize the paragraphs below without inventing facts or changing numbers.\nOutput exactly two parts in the same language as the source:\n1. First line: a concise title only.\n2. Following lines: a concise summary of the content.\nDo not output labels, Markdown headings, bullet points, or any other commentary.\n\nParagraphs:\n{cluster_content}",
+ "max_token": 512,
+ "clustering_threshold": 0.3,
+ "clustering_ratio": 0.5,
"max_cluster": 64,
"random_seed": 0,
},
diff --git a/test/testcases/restful_api/test_datasets.py b/test/testcases/restful_api/test_datasets.py
index 8609daea22..f4f8d22145 100644
--- a/test/testcases/restful_api/test_datasets.py
+++ b/test/testcases/restful_api/test_datasets.py
@@ -272,20 +272,16 @@ def test_dataset_update_chunk_method_contract(rest_client, clear_datasets, chunk
("raptor_true", {"raptor": {"use_raptor": True}}),
("raptor_false", {"raptor": {"use_raptor": False}}),
("raptor_prompt", {"raptor": {"prompt": "Who are you?"}}),
- ("raptor_max_token_min", {"raptor": {"max_token": 1}}),
+ ("raptor_max_token_min", {"raptor": {"max_token": 512}}),
("raptor_max_token_mid", {"raptor": {"max_token": 1024}}),
("raptor_max_token_max", {"raptor": {"max_token": 2048}}),
- ("raptor_threshold_min", {"raptor": {"threshold": 0.0}}),
- ("raptor_threshold_mid", {"raptor": {"threshold": 0.5}}),
- ("raptor_threshold_max", {"raptor": {"threshold": 1.0}}),
+ ("raptor_clustering_threshold_min", {"raptor": {"clustering_threshold": 0.0}}),
+ ("raptor_clustering_threshold_mid", {"raptor": {"clustering_threshold": 0.5}}),
+ ("raptor_clustering_threshold_max", {"raptor": {"clustering_threshold": 1.0}}),
("raptor_max_cluster_min", {"raptor": {"max_cluster": 1}}),
("raptor_max_cluster_mid", {"raptor": {"max_cluster": 512}}),
("raptor_max_cluster_max", {"raptor": {"max_cluster": 1024}}),
("raptor_random_seed_min", {"raptor": {"random_seed": 0}}),
- ("raptor_clustering_method_gmm", {"raptor": {"clustering_method": "gmm"}}),
- ("raptor_clustering_method_ahc", {"raptor": {"clustering_method": "ahc"}}),
- ("raptor_tree_builder_raptor", {"raptor": {"tree_builder": "raptor"}}),
- ("raptor_tree_builder_psi", {"raptor": {"tree_builder": "psi"}}),
],
ids=[
"auto_keywords_min",
@@ -329,17 +325,13 @@ def test_dataset_update_chunk_method_contract(rest_client, clear_datasets, chunk
"raptor_max_token_min",
"raptor_max_token_mid",
"raptor_max_token_max",
- "raptor_threshold_min",
- "raptor_threshold_mid",
- "raptor_threshold_max",
+ "raptor_clustering_threshold_min",
+ "raptor_clustering_threshold_mid",
+ "raptor_clustering_threshold_max",
"raptor_max_cluster_min",
"raptor_max_cluster_mid",
"raptor_max_cluster_max",
"raptor_random_seed_min",
- "raptor_clustering_method_gmm",
- "raptor_clustering_method_ahc",
- "raptor_tree_builder_raptor",
- "raptor_tree_builder_psi",
],
)
def test_dataset_update_parser_config_valid_matrix_contract(rest_client, clear_datasets, name, parser_config):
@@ -1028,13 +1020,13 @@ def test_dataset_update_parser_config_invalid_contract(rest_client, clear_datase
({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),
({"raptor": {"prompt": ""}}, "String should have at least 1 character"),
({"raptor": {"prompt": " "}}, "String should have at least 1 character"),
- ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 1"),
+ ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),
({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
({"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
- ({"raptor": {"threshold": -0.1}}, "Input should be greater than or equal to 0"),
- ({"raptor": {"threshold": 1.1}}, "Input should be less than or equal to 1"),
- ({"raptor": {"threshold": "string"}}, "Input should be a valid number"),
+ ({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
+ ({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
+ ({"raptor": {"clustering_threshold": "string"}}, "Input should be a valid number"),
({"raptor": {"max_cluster": 0}}, "Input should be greater than or equal to 1"),
({"raptor": {"max_cluster": 1025}}, "Input should be less than or equal to 1024"),
({"raptor": {"max_cluster": 3.14}}, "Input should be a valid integer"),
@@ -1042,10 +1034,6 @@ def test_dataset_update_parser_config_invalid_contract(rest_client, clear_datase
({"raptor": {"random_seed": -1}}, "Input should be greater than or equal to 0"),
({"raptor": {"random_seed": 3.14}}, "Input should be a valid integer"),
({"raptor": {"random_seed": "string"}}, "Input should be a valid integer"),
- ({"raptor": {"clustering_method": "unknown"}}, "Input should be 'gmm' or 'ahc'"),
- ({"raptor": {"clustering_method": None}}, "Input should be 'gmm' or 'ahc'"),
- ({"raptor": {"tree_builder": "ahc"}}, "Input should be 'raptor' or 'psi'"),
- ({"raptor": {"tree_builder": None}}, "Input should be 'raptor' or 'psi'"),
({"delimiter": "a" * 65536}, "Parser config exceeds size limit (max 65,535 characters)"),
]
for parser_config, expected_message in invalid_cases:
@@ -1375,12 +1363,12 @@ def test_dataset_create_concurrent_contract(rest_client, clear_datasets):
("raptor_true", {"raptor": {"use_raptor": True}}),
("raptor_false", {"raptor": {"use_raptor": False}}),
("raptor_prompt", {"raptor": {"prompt": "Who are you?"}}),
- ("raptor_max_token_min", {"raptor": {"max_token": 1}}),
+ ("raptor_max_token_min", {"raptor": {"max_token": 512}}),
("raptor_max_token_mid", {"raptor": {"max_token": 1024}}),
("raptor_max_token_max", {"raptor": {"max_token": 2048}}),
- ("raptor_threshold_min", {"raptor": {"threshold": 0.0}}),
- ("raptor_threshold_mid", {"raptor": {"threshold": 0.5}}),
- ("raptor_threshold_max", {"raptor": {"threshold": 1.0}}),
+ ("raptor_clustering_threshold_min", {"raptor": {"clustering_threshold": 0.0}}),
+ ("raptor_clustering_threshold_mid", {"raptor": {"clustering_threshold": 0.5}}),
+ ("raptor_clustering_threshold_max", {"raptor": {"clustering_threshold": 1.0}}),
("raptor_max_cluster_min", {"raptor": {"max_cluster": 1}}),
("raptor_max_cluster_mid", {"raptor": {"max_cluster": 512}}),
("raptor_max_cluster_max", {"raptor": {"max_cluster": 1024}}),
@@ -1432,9 +1420,9 @@ def test_dataset_create_concurrent_contract(rest_client, clear_datasets):
"raptor_max_token_min",
"raptor_max_token_mid",
"raptor_max_token_max",
- "raptor_threshold_min",
- "raptor_threshold_mid",
- "raptor_threshold_max",
+ "raptor_clustering_threshold_min",
+ "raptor_clustering_threshold_mid",
+ "raptor_clustering_threshold_max",
"raptor_max_cluster_min",
"raptor_max_cluster_mid",
"raptor_max_cluster_max",
@@ -1757,13 +1745,13 @@ def test_dataset_create_parser_config_invalid_contract(rest_client, clear_datase
("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),
("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"),
("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"),
- ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 1"),
+ ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),
("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
- ("raptor_threshold_min_limit", {"raptor": {"threshold": -0.1}}, "Input should be greater than or equal to 0"),
- ("raptor_threshold_max_limit", {"raptor": {"threshold": 1.1}}, "Input should be less than or equal to 1"),
- ("raptor_threshold_type_invalid", {"raptor": {"threshold": "string"}}, "Input should be a valid number"),
+ ("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
+ ("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
+ ("raptor_clustering_threshold_type_invalid", {"raptor": {"clustering_threshold": "string"}}, "Input should be a valid number"),
("raptor_max_cluster_min_limit", {"raptor": {"max_cluster": 0}}, "Input should be greater than or equal to 1"),
("raptor_max_cluster_max_limit", {"raptor": {"max_cluster": 1025}}, "Input should be less than or equal to 1024"),
("raptor_max_cluster_float_not_allowed", {"raptor": {"max_cluster": 3.14}}, "Input should be a valid integer"),
diff --git a/test/testcases/restful_api/test_documents.py b/test/testcases/restful_api/test_documents.py
index aaa1301b57..238322778f 100644
--- a/test/testcases/restful_api/test_documents.py
+++ b/test/testcases/restful_api/test_documents.py
@@ -754,9 +754,10 @@ def test_documents_update_parser_config_contract(rest_client, create_dataset, tm
"topn_tags": 3,
"raptor": {
"use_raptor": True,
- "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
- "max_token": 256,
- "threshold": 0.1,
+ "prompt": "Summarize the paragraphs below without inventing facts or changing numbers.\nOutput exactly two parts in the same language as the source:\n1. First line: a concise title only.\n2. Following lines: a concise summary of the content.\nDo not output labels, Markdown headings, bullet points, or any other commentary.\n\nParagraphs:\n{cluster_content}",
+ "max_token": 512,
+ "clustering_threshold": 0.3,
+ "clustering_ratio": 0.5,
"max_cluster": 64,
"random_seed": 0,
},
diff --git a/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py b/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py
index e56910009f..ed2c4319cf 100644
--- a/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py
+++ b/test/testcases/test_http_api/test_dataset_management/test_create_dataset.py
@@ -439,12 +439,12 @@ class TestDatasetCreate:
("raptor_true", {"raptor": {"use_raptor": True}}),
("raptor_false", {"raptor": {"use_raptor": False}}),
("raptor_prompt", {"raptor": {"prompt": "Who are you?"}}),
- ("raptor_max_token_min", {"raptor": {"max_token": 1}}),
+ ("raptor_max_token_min", {"raptor": {"max_token": 512}}),
("raptor_max_token_mid", {"raptor": {"max_token": 1024}}),
("raptor_max_token_max", {"raptor": {"max_token": 2048}}),
- ("raptor_threshold_min", {"raptor": {"threshold": 0.0}}),
- ("raptor_threshold_mid", {"raptor": {"threshold": 0.5}}),
- ("raptor_threshold_max", {"raptor": {"threshold": 1.0}}),
+ ("raptor_clustering_threshold_min", {"raptor": {"clustering_threshold": 0.0}}),
+ ("raptor_clustering_threshold_mid", {"raptor": {"clustering_threshold": 0.5}}),
+ ("raptor_clustering_threshold_max", {"raptor": {"clustering_threshold": 1.0}}),
("raptor_max_cluster_min", {"raptor": {"max_cluster": 1}}),
("raptor_max_cluster_mid", {"raptor": {"max_cluster": 512}}),
("raptor_max_cluster_max", {"raptor": {"max_cluster": 1024}}),
@@ -496,9 +496,9 @@ class TestDatasetCreate:
"raptor_max_token_min",
"raptor_max_token_mid",
"raptor_max_token_max",
- "raptor_threshold_min",
- "raptor_threshold_mid",
- "raptor_threshold_max",
+ "raptor_clustering_threshold_min",
+ "raptor_clustering_threshold_mid",
+ "raptor_clustering_threshold_max",
"raptor_max_cluster_min",
"raptor_max_cluster_mid",
"raptor_max_cluster_max",
@@ -567,9 +567,9 @@ class TestDatasetCreate:
("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
- ("raptor_threshold_min_limit", {"raptor": {"threshold": -0.1}}, "Input should be greater than or equal to 0"),
- ("raptor_threshold_max_limit", {"raptor": {"threshold": 1.1}}, "Input should be less than or equal to 1"),
- ("raptor_threshold_type_invalid", {"raptor": {"threshold": "string"}}, "Input should be a valid number"),
+ ("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
+ ("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
+ ("raptor_clustering_threshold_type_invalid", {"raptor": {"clustering_threshold": "string"}}, "Input should be a valid number"),
("raptor_max_cluster_min_limit", {"raptor": {"max_cluster": 0}}, "Input should be greater than or equal to 1"),
("raptor_max_cluster_max_limit", {"raptor": {"max_cluster": 1025}}, "Input should be less than or equal to 1024"),
("raptor_max_cluster_float_not_allowed", {"raptor": {"max_cluster": 3.14}}, "Input should be a valid integer"),
@@ -625,9 +625,9 @@ class TestDatasetCreate:
"raptor_max_token_max_limit",
"raptor_max_token_float_not_allowed",
"raptor_max_token_type_invalid",
- "raptor_threshold_min_limit",
- "raptor_threshold_max_limit",
- "raptor_threshold_type_invalid",
+ "raptor_clustering_threshold_min_limit",
+ "raptor_clustering_threshold_max_limit",
+ "raptor_clustering_threshold_type_invalid",
"raptor_max_cluster_min_limit",
"raptor_max_cluster_max_limit",
"raptor_max_cluster_float_not_allowed",
diff --git a/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py b/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py
index e18e99073c..d216b35793 100644
--- a/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py
+++ b/test/testcases/test_http_api/test_dataset_management/test_update_dataset.py
@@ -573,20 +573,16 @@ class TestDatasetUpdate:
{"raptor": {"use_raptor": True}},
{"raptor": {"use_raptor": False}},
{"raptor": {"prompt": "Who are you?"}},
- {"raptor": {"max_token": 1}},
+ {"raptor": {"max_token": 512}},
{"raptor": {"max_token": 1024}},
{"raptor": {"max_token": 2048}},
- {"raptor": {"threshold": 0.0}},
- {"raptor": {"threshold": 0.5}},
- {"raptor": {"threshold": 1.0}},
+ {"raptor": {"clustering_threshold": 0.0}},
+ {"raptor": {"clustering_threshold": 0.5}},
+ {"raptor": {"clustering_threshold": 1.0}},
{"raptor": {"max_cluster": 1}},
{"raptor": {"max_cluster": 512}},
{"raptor": {"max_cluster": 1024}},
{"raptor": {"random_seed": 0}},
- {"raptor": {"clustering_method": "gmm"}},
- {"raptor": {"clustering_method": "ahc"}},
- {"raptor": {"tree_builder": "raptor"}},
- {"raptor": {"tree_builder": "psi"}},
],
ids=[
"auto_keywords_min",
@@ -630,17 +626,13 @@ class TestDatasetUpdate:
"raptor_max_token_min",
"raptor_max_token_mid",
"raptor_max_token_max",
- "raptor_threshold_min",
- "raptor_threshold_mid",
- "raptor_threshold_max",
+ "raptor_clustering_threshold_min",
+ "raptor_clustering_threshold_mid",
+ "raptor_clustering_threshold_max",
"raptor_max_cluster_min",
"raptor_max_cluster_mid",
"raptor_max_cluster_max",
"raptor_random_seed_min",
- "raptor_clustering_method_gmm",
- "raptor_clustering_method_ahc",
- "raptor_tree_builder_raptor",
- "raptor_tree_builder_psi",
],
)
def test_parser_config(self, HttpApiAuth, add_dataset_func, parser_config):
@@ -705,9 +697,9 @@ class TestDatasetUpdate:
({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
({"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
- ({"raptor": {"threshold": -0.1}}, "Input should be greater than or equal to 0"),
- ({"raptor": {"threshold": 1.1}}, "Input should be less than or equal to 1"),
- ({"raptor": {"threshold": "string"}}, "Input should be a valid number"),
+ ({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
+ ({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
+ ({"raptor": {"clustering_threshold": "string"}}, "Input should be a valid number"),
({"raptor": {"max_cluster": 0}}, "Input should be greater than or equal to 1"),
({"raptor": {"max_cluster": 1025}}, "Input should be less than or equal to 1024"),
({"raptor": {"max_cluster": 3.14}}, "Input should be a valid integer"),
@@ -715,10 +707,6 @@ class TestDatasetUpdate:
({"raptor": {"random_seed": -1}}, "Input should be greater than or equal to 0"),
({"raptor": {"random_seed": 3.14}}, "Input should be a valid integer"),
({"raptor": {"random_seed": "string"}}, "Input should be a valid integer"),
- ({"raptor": {"clustering_method": "unknown"}}, "Input should be 'gmm' or 'ahc'"),
- ({"raptor": {"clustering_method": None}}, "Input should be 'gmm' or 'ahc'"),
- ({"raptor": {"tree_builder": "ahc"}}, "Input should be 'raptor' or 'psi'"),
- ({"raptor": {"tree_builder": None}}, "Input should be 'raptor' or 'psi'"),
({"delimiter": "a" * 65536}, "Parser config exceeds size limit (max 65,535 characters)"),
],
ids=[
@@ -765,9 +753,9 @@ class TestDatasetUpdate:
"raptor_max_token_max_limit",
"raptor_max_token_float_not_allowed",
"raptor_max_token_type_invalid",
- "raptor_threshold_min_limit",
- "raptor_threshold_max_limit",
- "raptor_threshold_type_invalid",
+ "raptor_clustering_threshold_min_limit",
+ "raptor_clustering_threshold_max_limit",
+ "raptor_clustering_threshold_type_invalid",
"raptor_max_cluster_min_limit",
"raptor_max_cluster_max_limit",
"raptor_max_cluster_float_not_allowed",
@@ -775,10 +763,6 @@ class TestDatasetUpdate:
"raptor_random_seed_min_limit",
"raptor_random_seed_float_not_allowed",
"raptor_random_seed_type_invalid",
- "raptor_clustering_method_invalid",
- "raptor_clustering_method_none_invalid",
- "raptor_tree_builder_invalid",
- "raptor_tree_builder_none_invalid",
"parser_config_type_invalid",
],
)
diff --git a/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py b/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
index 3cfdae2296..f1fe170b93 100644
--- a/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
+++ b/test/testcases/test_http_api/test_file_management_within_dataset/test_update_document.py
@@ -371,9 +371,10 @@ DEFAULT_PARSER_CONFIG_FOR_TEST = {
"topn_tags": 3,
"raptor": {
"use_raptor": True,
- "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
- "max_token": 256,
- "threshold": 0.1,
+ "prompt": "Summarize the paragraphs below without inventing facts or changing numbers.\nOutput exactly two parts in the same language as the source:\n1. First line: a concise title only.\n2. Following lines: a concise summary of the content.\nDo not output labels, Markdown headings, bullet points, or any other commentary.\n\nParagraphs:\n{cluster_content}",
+ "max_token": 512,
+ "clustering_threshold": 0.3,
+ "clustering_ratio": 0.5,
"max_cluster": 64,
"random_seed": 0,
},
diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py
index 7d0618b2b9..ab626f0b48 100644
--- a/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py
+++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_create_dataset.py
@@ -396,12 +396,12 @@ class TestDatasetCreate:
("raptor_true", {"raptor": {"use_raptor": True}}),
("raptor_false", {"raptor": {"use_raptor": False}}),
("raptor_prompt", {"raptor": {"prompt": "Who are you?"}}),
- ("raptor_max_token_min", {"raptor": {"max_token": 1}}),
+ ("raptor_max_token_min", {"raptor": {"max_token": 512}}),
("raptor_max_token_mid", {"raptor": {"max_token": 1024}}),
("raptor_max_token_max", {"raptor": {"max_token": 2048}}),
- ("raptor_threshold_min", {"raptor": {"threshold": 0.0}}),
- ("raptor_threshold_mid", {"raptor": {"threshold": 0.5}}),
- ("raptor_threshold_max", {"raptor": {"threshold": 1.0}}),
+ ("raptor_clustering_threshold_min", {"raptor": {"clustering_threshold": 0.0}}),
+ ("raptor_clustering_threshold_mid", {"raptor": {"clustering_threshold": 0.5}}),
+ ("raptor_clustering_threshold_max", {"raptor": {"clustering_threshold": 1.0}}),
("raptor_max_cluster_min", {"raptor": {"max_cluster": 1}}),
("raptor_max_cluster_mid", {"raptor": {"max_cluster": 512}}),
("raptor_max_cluster_max", {"raptor": {"max_cluster": 1024}}),
@@ -449,9 +449,9 @@ class TestDatasetCreate:
"raptor_max_token_min",
"raptor_max_token_mid",
"raptor_max_token_max",
- "raptor_threshold_min",
- "raptor_threshold_mid",
- "raptor_threshold_max",
+ "raptor_clustering_threshold_min",
+ "raptor_clustering_threshold_mid",
+ "raptor_clustering_threshold_max",
"raptor_max_cluster_min",
"raptor_max_cluster_mid",
"raptor_max_cluster_max",
@@ -512,13 +512,13 @@ class TestDatasetCreate:
("raptor_type_invalid", {"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),
("raptor_prompt_empty", {"raptor": {"prompt": ""}}, "String should have at least 1 character"),
("raptor_prompt_space", {"raptor": {"prompt": " "}}, "String should have at least 1 character"),
- ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 1"),
+ ("raptor_max_token_min_limit", {"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),
("raptor_max_token_max_limit", {"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
("raptor_max_token_float_not_allowed", {"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
("raptor_max_token_type_invalid", {"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
- ("raptor_threshold_min_limit", {"raptor": {"threshold": -0.1}}, "Input should be greater than or equal to 0"),
- ("raptor_threshold_max_limit", {"raptor": {"threshold": 1.1}}, "Input should be less than or equal to 1"),
- ("raptor_threshold_type_invalid", {"raptor": {"threshold": "string"}}, "Input should be a valid number"),
+ ("raptor_clustering_threshold_min_limit", {"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
+ ("raptor_clustering_threshold_max_limit", {"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
+ ("raptor_clustering_threshold_type_invalid", {"raptor": {"clustering_threshold": "string"}}, "Input should be a valid number"),
("raptor_max_cluster_min_limit", {"raptor": {"max_cluster": 0}}, "Input should be greater than or equal to 1"),
("raptor_max_cluster_max_limit", {"raptor": {"max_cluster": 1025}}, "Input should be less than or equal to 1024"),
("raptor_max_cluster_float_not_allowed", {"raptor": {"max_cluster": 3.14}}, "Input should be a valid integer"),
@@ -572,9 +572,9 @@ class TestDatasetCreate:
"raptor_max_token_max_limit",
"raptor_max_token_float_not_allowed",
"raptor_max_token_type_invalid",
- "raptor_threshold_min_limit",
- "raptor_threshold_max_limit",
- "raptor_threshold_type_invalid",
+ "raptor_clustering_threshold_min_limit",
+ "raptor_clustering_threshold_max_limit",
+ "raptor_clustering_threshold_type_invalid",
"raptor_max_cluster_min_limit",
"raptor_max_cluster_max_limit",
"raptor_max_cluster_float_not_allowed",
diff --git a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py
index 7fe6a69ea5..daba719c6c 100644
--- a/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py
+++ b/test/testcases/test_sdk_api/test_dataset_mangement/test_update_dataset.py
@@ -458,12 +458,12 @@ class TestDatasetUpdate:
{"raptor": {"use_raptor": True}},
{"raptor": {"use_raptor": False}},
{"raptor": {"prompt": "Who are you?"}},
- {"raptor": {"max_token": 1}},
+ {"raptor": {"max_token": 512}},
{"raptor": {"max_token": 1024}},
{"raptor": {"max_token": 2048}},
- {"raptor": {"threshold": 0.0}},
- {"raptor": {"threshold": 0.5}},
- {"raptor": {"threshold": 1.0}},
+ {"raptor": {"clustering_threshold": 0.0}},
+ {"raptor": {"clustering_threshold": 0.5}},
+ {"raptor": {"clustering_threshold": 1.0}},
{"raptor": {"max_cluster": 1}},
{"raptor": {"max_cluster": 512}},
{"raptor": {"max_cluster": 1024}},
@@ -511,9 +511,9 @@ class TestDatasetUpdate:
"raptor_max_token_min",
"raptor_max_token_mid",
"raptor_max_token_max",
- "raptor_threshold_min",
- "raptor_threshold_mid",
- "raptor_threshold_max",
+ "raptor_clustering_threshold_min",
+ "raptor_clustering_threshold_mid",
+ "raptor_clustering_threshold_max",
"raptor_max_cluster_min",
"raptor_max_cluster_mid",
"raptor_max_cluster_max",
@@ -581,13 +581,13 @@ class TestDatasetUpdate:
({"raptor": {"use_raptor": "string"}}, "Input should be a valid boolean"),
({"raptor": {"prompt": ""}}, "String should have at least 1 character"),
({"raptor": {"prompt": " "}}, "String should have at least 1 character"),
- ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 1"),
+ ({"raptor": {"max_token": 0}}, "Input should be greater than or equal to 512"),
({"raptor": {"max_token": 2049}}, "Input should be less than or equal to 2048"),
({"raptor": {"max_token": 3.14}}, "Input should be a valid integer"),
({"raptor": {"max_token": "string"}}, "Input should be a valid integer"),
- ({"raptor": {"threshold": -0.1}}, "Input should be greater than or equal to 0"),
- ({"raptor": {"threshold": 1.1}}, "Input should be less than or equal to 1"),
- ({"raptor": {"threshold": "string"}}, "Input should be a valid number"),
+ ({"raptor": {"clustering_threshold": -0.1}}, "Input should be greater than or equal to 0"),
+ ({"raptor": {"clustering_threshold": 1.1}}, "Input should be less than or equal to 1"),
+ ({"raptor": {"clustering_threshold": "string"}}, "Input should be a valid number"),
({"raptor": {"max_cluster": 0}}, "Input should be greater than or equal to 1"),
({"raptor": {"max_cluster": 1025}}, "Input should be less than or equal to 1024"),
({"raptor": {"max_cluster": 3.14}}, "Input should be a valid integer"),
@@ -641,9 +641,9 @@ class TestDatasetUpdate:
"raptor_max_token_max_limit",
"raptor_max_token_float_not_allowed",
"raptor_max_token_type_invalid",
- "raptor_threshold_min_limit",
- "raptor_threshold_max_limit",
- "raptor_threshold_type_invalid",
+ "raptor_clustering_threshold_min_limit",
+ "raptor_clustering_threshold_max_limit",
+ "raptor_clustering_threshold_type_invalid",
"raptor_max_cluster_min_limit",
"raptor_max_cluster_max_limit",
"raptor_max_cluster_float_not_allowed",
diff --git a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
index f29be2c384..e79244d4d0 100644
--- a/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
+++ b/test/testcases/test_sdk_api/test_file_management_within_dataset/test_update_document.py
@@ -298,9 +298,10 @@ DEFAULT_PARSER_CONFIG_FOR_TEST = {
"topn_tags": 3,
"raptor": {
"use_raptor": True,
- "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
- "max_token": 256,
- "threshold": 0.1,
+ "prompt": "Summarize the paragraphs below without inventing facts or changing numbers.\nOutput exactly two parts in the same language as the source:\n1. First line: a concise title only.\n2. Following lines: a concise summary of the content.\nDo not output labels, Markdown headings, bullet points, or any other commentary.\n\nParagraphs:\n{cluster_content}",
+ "max_token": 512,
+ "clustering_threshold": 0.3,
+ "clustering_ratio": 0.5,
"max_cluster": 64,
"random_seed": 0,
},
@@ -402,9 +403,10 @@ class TestUpdateDocumentParserConfig:
{
"raptor": {
"use_raptor": True,
- "prompt": "Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:\n {cluster_content}\nThe above is the content you need to summarize.",
- "max_token": 256,
- "threshold": 0.1,
+ "prompt": "Summarize the paragraphs below without inventing facts or changing numbers.\nOutput exactly two parts in the same language as the source:\n1. First line: a concise title only.\n2. Following lines: a concise summary of the content.\nDo not output labels, Markdown headings, bullet points, or any other commentary.\n\nParagraphs:\n{cluster_content}",
+ "max_token": 512,
+ "clustering_threshold": 0.3,
+ "clustering_ratio": 0.5,
"max_cluster": 64,
"random_seed": 0,
}
diff --git a/test/unit_test/rag/svr/task_executor_refactor/test_raptor_service.py b/test/unit_test/rag/svr/task_executor_refactor/test_raptor_service.py
index c44a3c50dc..2d35d45ab4 100644
--- a/test/unit_test/rag/svr/task_executor_refactor/test_raptor_service.py
+++ b/test/unit_test/rag/svr/task_executor_refactor/test_raptor_service.py
@@ -188,12 +188,11 @@ class TestRaptorServiceRunRaptorForKb:
"""RAPTOR config with file-level scope."""
return {
"raptor": {
- "tree_builder": "raptor",
- "clustering_method": "gmm",
"scope": "file",
"prompt": "summarize",
"max_token": 512,
- "threshold": 0.5,
+ "clustering_threshold": 0.5,
+ "clustering_ratio": 0.5,
"max_cluster": 64,
"random_seed": 42,
}
@@ -204,12 +203,11 @@ class TestRaptorServiceRunRaptorForKb:
"""RAPTOR config with dataset-level scope."""
return {
"raptor": {
- "tree_builder": "raptor",
- "clustering_method": "gmm",
"scope": "dataset",
"prompt": "summarize",
"max_token": 512,
- "threshold": 0.5,
+ "clustering_threshold": 0.5,
+ "clustering_ratio": 0.5,
"max_cluster": 64,
"random_seed": 42,
}
@@ -321,7 +319,7 @@ class TestRaptorServiceRunRaptorForKb:
):
async def mock_run_file(*args, **kwargs):
- cleanup_list = args[11]
+ cleanup_list = args[9]
cleanup_list.append(("doc_1", "tree_builder_a"))
return [{"id": "c1"}], 10
@@ -383,10 +381,10 @@ class TestRaptorServiceRunRaptorForKb:
await svc.run_raptor_for_kb(raptor_config_file_scope, chat_mdl, embd_mdl, vector_size, doc_ids)
# Verify _run_file_level_raptor received vctr_nm with the correct vector size
- # Positional args: 0=raptor_config, 1=tree_builder, 2=clustering_method,
- # 3=chat_mdl, 4=embd_mdl, 5=vctr_nm
+ # Positional args: 0=raptor_config, 1=chat_mdl, 2=embd_mdl,
+ # 3=vctr_nm, 4=doc_ids, 5=doc_info_by_id
positional_args = mock_file.call_args[0]
- assert positional_args[5] == "q_256_vec"
+ assert positional_args[3] == "q_256_vec"
# ---- Document info collection through public API ----
@@ -405,9 +403,9 @@ class TestRaptorServiceRunRaptorForKb:
await svc.run_raptor_for_kb(raptor_config_file_scope, chat_mdl, embd_mdl, 128, doc_ids)
mock_collect.assert_called_once_with(doc_ids)
- # Verify doc_info_by_id was passed as positional arg[7] to _run_file_level_raptor
+ # Verify doc_info_by_id was passed as positional arg[5] to _run_file_level_raptor
positional_args = mock_file.call_args[0]
- assert positional_args[7] == expected_info
+ assert positional_args[5] == expected_info
class TestRaptorServiceFileLevelRaptorCheckpoint:
@@ -429,12 +427,10 @@ class TestRaptorServiceFileLevelRaptorCheckpoint:
"scope": "file",
"max_cluster": 64,
"prompt": "test prompt",
- "max_token": 256,
- "threshold": 0.1,
+ "max_token": 512,
+ "clustering_threshold": 0.3,
+ "clustering_ratio": 0.5,
"random_seed": 0,
- "clustering_method": "gmm",
- "tree_builder": "raptor",
- "ext": {},
}
with patch.object(svc, "_get_raptor_chunk_methods", new_callable=AsyncMock) as mock_methods, patch.object(svc, "_should_skip_raptor", return_value=False):
@@ -442,8 +438,6 @@ class TestRaptorServiceFileLevelRaptorCheckpoint:
result = await svc._run_file_level_raptor(
raptor_config=raptor_config,
- tree_builder="raptor",
- clustering_method="gmm",
chat_mdl=MagicMock(),
embd_mdl=MagicMock(),
vctr_nm="q_128_vec",
diff --git a/test/unit_test/rag/test_raptor_psi_tree_builder.py b/test/unit_test/rag/test_raptor_psi_tree_builder.py
deleted file mode 100644
index a2c2178bda..0000000000
--- a/test/unit_test/rag/test_raptor_psi_tree_builder.py
+++ /dev/null
@@ -1,434 +0,0 @@
-#
-# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import importlib
-import os
-import sys
-import types
-import pytest
-
-np = pytest.importorskip("numpy")
-
-from api.utils.validation_utils import RaptorConfig
-from pydantic import ValidationError
-
-
-@pytest.fixture()
-def raptor_module(monkeypatch):
- class TaskCanceledException(Exception):
- pass
-
- class DummyLimiter:
- async def __aenter__(self):
- return self
-
- async def __aexit__(self, exc_type, exc, tb):
- return False
-
- class DummyGaussianMixture:
- def __init__(self, *args, **kwargs):
- pass
-
- def fit(self, embeddings):
- return self
-
- def bic(self, embeddings):
- return 0
-
- def predict_proba(self, embeddings):
- return np.ones((len(embeddings), 1))
-
- class DummyAgglomerativeClustering:
- def __init__(self, n_clusters=None, distance_threshold=None, compute_distances=False, linkage="ward", metric="euclidean"):
- self.n_clusters = n_clusters
- self.distance_threshold = distance_threshold
- self.compute_distances = compute_distances
- self.linkage = linkage
- self.metric = metric
- self.distances_ = np.array([0.1, 0.2, 1.0])
-
- def fit(self, embeddings):
- self.labels_ = self.fit_predict(embeddings)
- return self
-
- def fit_predict(self, embeddings):
- if self.n_clusters is None:
- return np.zeros(len(embeddings), dtype=int)
- return np.array([idx % self.n_clusters for idx in range(len(embeddings))])
-
- class DummyUMAP:
- def __init__(self, *args, **kwargs):
- pass
-
- def fit_transform(self, embeddings):
- raise AssertionError("Psi tree builder must use original embeddings, not UMAP")
-
- sklearn_module = types.ModuleType("sklearn")
- mixture_module = types.ModuleType("sklearn.mixture")
- mixture_module.GaussianMixture = DummyGaussianMixture
- cluster_module = types.ModuleType("sklearn.cluster")
- cluster_module.AgglomerativeClustering = DummyAgglomerativeClustering
- umap_module = types.ModuleType("umap")
- umap_module.UMAP = DummyUMAP
- task_service_module = types.ModuleType("api.db.services.task_service")
- task_service_module.has_canceled = lambda task_id: False
- connection_utils_module = types.ModuleType("common.connection_utils")
- connection_utils_module.timeout = lambda seconds: lambda fn: fn
- exceptions_module = types.ModuleType("common.exceptions")
- exceptions_module.TaskCanceledException = TaskCanceledException
- token_utils_module = types.ModuleType("common.token_utils")
- token_utils_module.truncate = lambda text, max_len: text[:max_len]
- graphrag_utils_module = types.ModuleType("rag.graphrag.utils")
- graphrag_utils_module.chat_limiter = DummyLimiter()
- graphrag_utils_module.get_embed_cache = lambda *args, **kwargs: None
- graphrag_utils_module.get_llm_cache = lambda *args, **kwargs: None
- graphrag_utils_module.set_embed_cache = lambda *args, **kwargs: None
- graphrag_utils_module.set_llm_cache = lambda *args, **kwargs: None
-
- async def thread_pool_exec(fn, *args, **kwargs):
- return fn(*args, **kwargs)
-
- misc_utils_module = types.ModuleType("common.misc_utils")
- misc_utils_module.thread_pool_exec = thread_pool_exec
-
- monkeypatch.setitem(sys.modules, "sklearn", sklearn_module)
- monkeypatch.setitem(sys.modules, "sklearn.mixture", mixture_module)
- monkeypatch.setitem(sys.modules, "sklearn.cluster", cluster_module)
- monkeypatch.setitem(sys.modules, "umap", umap_module)
- monkeypatch.setitem(sys.modules, "api.db.services.task_service", task_service_module)
- monkeypatch.setitem(sys.modules, "common.connection_utils", connection_utils_module)
- monkeypatch.setitem(sys.modules, "common.exceptions", exceptions_module)
- monkeypatch.setitem(sys.modules, "common.token_utils", token_utils_module)
- monkeypatch.setitem(sys.modules, "rag.graphrag.utils", graphrag_utils_module)
- monkeypatch.setitem(sys.modules, "common.misc_utils", misc_utils_module)
- # Create stub parent packages and load raptor directly via spec_from_file_location
- # to bypass rag/advanced_rag/__init__.py (which triggers ES connection etc.).
- _test_dir = os.path.dirname(__file__)
- _rag_adv_kc_dir = os.path.normpath(os.path.join(_test_dir, "../../../rag/advanced_rag/knowlege_compile"))
- _rag_adv = types.ModuleType("rag.advanced_rag")
- _rag_adv.__path__ = [os.path.normpath(os.path.join(_test_dir, "../../../rag/advanced_rag"))]
- _rag_adv.__package__ = "rag.advanced_rag"
- monkeypatch.setitem(sys.modules, "rag.advanced_rag", _rag_adv)
- _rag_adv_kc = types.ModuleType("rag.advanced_rag.knowlege_compile")
- _rag_adv_kc.__path__ = [_rag_adv_kc_dir]
- _rag_adv_kc.__package__ = "rag.advanced_rag.knowlege_compile"
- monkeypatch.setitem(sys.modules, "rag.advanced_rag.knowlege_compile", _rag_adv_kc)
- monkeypatch.delitem(sys.modules, "rag.advanced_rag.knowlege_compile.raptor", raising=False)
- _raptor_spec = importlib.util.spec_from_file_location(
- "rag.advanced_rag.knowlege_compile.raptor",
- os.path.join(_rag_adv_kc_dir, "raptor.py"),
- )
- module = importlib.util.module_from_spec(_raptor_spec)
- sys.modules["rag.advanced_rag.knowlege_compile.raptor"] = module
- _raptor_spec.loader.exec_module(module)
- yield module
- monkeypatch.delitem(sys.modules, "rag.advanced_rag.knowlege_compile.raptor", raising=False)
-
-
-class FakeChatModel:
- llm_name = "fake-chat"
- max_length = 4096
-
- def __init__(self):
- self.calls = []
-
- async def async_chat(self, system, history, gen_conf):
- self.calls.append(history[0]["content"])
- return f"summary-{len(self.calls)}"
-
-
-class FakeEmbeddingModel:
- llm_name = "fake-embedding"
-
- def encode(self, texts):
- embeddings = []
- for text in texts:
- checksum = sum(ord(ch) for ch in text)
- embeddings.append(np.array([len(text), checksum % 17 + 1], dtype=float))
- return embeddings, len(texts)
-
-
-_DEFAULT_TREE_BUILDER = object()
-
-
-def _make_raptor(raptor_module, max_cluster=64, tree_builder=_DEFAULT_TREE_BUILDER, **kwargs):
- if tree_builder is _DEFAULT_TREE_BUILDER:
- kwargs["tree_builder"] = raptor_module.PSI_TREE_BUILDER
- else:
- kwargs["tree_builder"] = tree_builder
- return raptor_module.RecursiveAbstractiveProcessing4TreeOrganizedRetrieval(
- max_cluster,
- FakeChatModel(),
- FakeEmbeddingModel(),
- "{cluster_content}",
- max_token=32,
- threshold=0.1,
- **kwargs,
- )
-
-
-def _chunks():
- return [
- ("alpha first", np.array([1.0, 0.0])),
- ("alpha second", np.array([0.99, 0.01])),
- ("alpha third", np.array([0.98, 0.02])),
- ]
-
-
-def test_default_tree_builder_remains_original_raptor(raptor_module):
- raptor = _make_raptor(raptor_module, tree_builder=None)
-
- assert raptor._tree_builder == raptor_module.RAPTOR_TREE_BUILDER
-
-
-def test_unknown_tree_builder_is_rejected(raptor_module):
- with pytest.raises(ValueError, match="Unsupported RAPTOR tree builder"):
- _make_raptor(raptor_module, tree_builder="ahc")
-
-
-def test_raptor_config_accepts_hidden_psi_tree_builder():
- assert RaptorConfig().tree_builder == "raptor"
- assert RaptorConfig().clustering_method == "gmm"
- assert RaptorConfig(clustering_method="ahc").clustering_method == "ahc"
- assert RaptorConfig(tree_builder="psi").tree_builder == "psi"
-
- with pytest.raises(ValidationError):
- RaptorConfig(tree_builder="ahc")
- with pytest.raises(ValidationError):
- RaptorConfig(clustering_method="psi")
-
-
-def test_ahc_clustering_method_is_supported_in_original_tree_builder(raptor_module):
- raptor = _make_raptor(raptor_module, tree_builder=raptor_module.RAPTOR_TREE_BUILDER, clustering_method="ahc")
-
- labels = raptor._get_clusters_ahc(np.array([[0.0, 0.0], [0.1, 0.0], [10.0, 10.0], [10.1, 10.0]]))
-
- assert raptor._tree_builder == raptor_module.RAPTOR_TREE_BUILDER
- assert raptor._clustering_method == "ahc"
- assert len(labels) == 4
-
-
-def test_unknown_clustering_method_is_rejected(raptor_module):
- with pytest.raises(ValueError, match="Unsupported RAPTOR clustering method"):
- _make_raptor(raptor_module, clustering_method="psi")
-
-
-@pytest.mark.p2
-def test_get_optimal_clusters_handles_max_cluster_equal_one(raptor_module):
- raptor = _make_raptor(raptor_module, max_cluster=1)
-
- optimal = raptor._get_optimal_clusters(
- np.array([[0.0, 0.0], [0.1, 0.0], [1.0, 1.0], [1.1, 1.1]]),
- random_state=0,
- )
-
- assert optimal == 1
-
-
-@pytest.mark.p2
-def test_get_optimal_clusters_evaluates_upper_bound_candidate(monkeypatch, raptor_module):
- raptor = _make_raptor(raptor_module, max_cluster=3)
- evaluated = []
-
- class RecordingGaussianMixture:
- def __init__(self, n_components, random_state=None, **kwargs):
- self.n_components = n_components
- evaluated.append(n_components)
-
- def fit(self, embeddings):
- return self
-
- def bic(self, embeddings):
- scores = {1: 30.0, 2: 20.0, 3: 10.0}
- return scores[self.n_components]
-
- monkeypatch.setattr(raptor_module, "GaussianMixture", RecordingGaussianMixture)
-
- optimal = raptor._get_optimal_clusters(
- np.array([[0.0, 0.0], [0.1, 0.0], [1.0, 1.0], [1.1, 1.1]]),
- random_state=0,
- )
-
- assert optimal == 3
- assert evaluated == [1, 2, 3]
-
-
-def test_psi_tree_builder_ranks_all_leaf_pairs_by_original_cosine_similarity(raptor_module):
- raptor = _make_raptor(raptor_module)
- leaves = [
- raptor_module._PsiTreeNode(index=0, embedding=np.array([1.0, 0.0])),
- raptor_module._PsiTreeNode(index=1, embedding=np.array([0.0, 1.0])),
- raptor_module._PsiTreeNode(index=2, embedding=np.array([0.99, 0.01])),
- raptor_module._PsiTreeNode(index=3, embedding=np.array([-1.0, 0.0])),
- ]
-
- ranked_pairs = raptor._rank_leaf_pairs(leaves)
-
- assert len(ranked_pairs) == 6
- assert tuple(ranked_pairs[0]) == (2, 0)
-
-
-def test_psi_tree_builder_uses_cosine_similarity_not_vector_magnitude(raptor_module):
- raptor = _make_raptor(raptor_module)
- leaves = [
- raptor_module._PsiTreeNode(index=0, embedding=np.array([100.0, 0.0])),
- raptor_module._PsiTreeNode(index=1, embedding=np.array([1.0, 1.0])),
- raptor_module._PsiTreeNode(index=2, embedding=np.array([0.1, 0.0])),
- ]
-
- ranked_pairs = raptor._rank_leaf_pairs(leaves)
-
- assert tuple(ranked_pairs[0]) == (2, 0)
-
-
-def test_psi_tree_builder_handles_zero_vectors_in_cosine_ranking(raptor_module):
- raptor = _make_raptor(raptor_module)
- leaves = [
- raptor_module._PsiTreeNode(index=0, embedding=np.array([0.0, 0.0])),
- raptor_module._PsiTreeNode(index=1, embedding=np.array([1.0, 0.0])),
- raptor_module._PsiTreeNode(index=2, embedding=np.array([0.9, 0.1])),
- ]
-
- ranked_pairs = raptor._rank_leaf_pairs(leaves)
-
- assert tuple(ranked_pairs[0]) == (2, 1)
-
-
-def test_psi_tree_builder_collapses_leaf_into_ranked_pair_parent(raptor_module):
- raptor = _make_raptor(raptor_module, max_cluster=64)
-
- root, leaves = raptor._build_psi_structure(_chunks())
-
- assert len(root.children) == 3
- assert {child.index for child in root.children} == {0, 1, 2}
- assert all(leaf.parent is root for leaf in leaves)
-
-
-def test_psi_tree_builder_collapses_leaf_at_matching_rank(monkeypatch, raptor_module):
- raptor = _make_raptor(raptor_module, max_cluster=64)
- chunks = [
- ("node 0", np.array([1.0, 0.0])),
- ("node 1", np.array([0.9, 0.1])),
- ("node 2", np.array([-1.0, 0.0])),
- ("node 3", np.array([-0.9, -0.1])),
- ("node 4", np.array([0.8, 0.2])),
- ]
- monkeypatch.setattr(
- raptor,
- "_rank_leaf_pairs",
- lambda _leaves: np.array([[0, 1], [2, 3], [0, 2], [4, 0]]),
- )
-
- root, leaves = raptor._build_psi_structure(chunks)
-
- assert leaves[4].parent is leaves[0].parent
- assert leaves[4].parent is not root
- assert len(root.children) == 2
-
-
-def test_psi_union_find_clamps_out_of_bounds_parent_rank(caplog, raptor_module):
- union_find = raptor_module._PsiUnionFind(2)
- union_find._node_ids[1] = [1]
- union_find._rank[0] = 2
-
- with caplog.at_level("WARNING"):
- union_find._build(0, 1, insert_point=1)
-
- assert union_find.tree[0] == 1
- assert "rank index" in caplog.text
-
-
-def test_psi_tree_builder_rebalances_nodes_over_max_children(raptor_module):
- raptor = _make_raptor(raptor_module, max_cluster=2)
-
- root, _ = raptor._build_psi_structure(_chunks())
-
- assert all(len(node.children) <= 2 for node in raptor._iter_nodes(root))
- assert len(root.children) == 2
- assert any(child.children for child in root.children)
-
-
-def test_psi_tree_builder_uses_bucketed_structure_for_large_inputs(monkeypatch, raptor_module):
- chunks = [(f"node {idx}", np.array([float(idx), float(idx % 3 + 1)])) for idx in range(8)]
- raptor = _make_raptor(
- raptor_module,
- max_cluster=3,
- psi_exact_max_leaves=3,
- psi_bucket_size=2,
- )
- ranked_sizes = []
- original_rank = raptor._rank_leaf_pairs
-
- def track_rank(nodes):
- ranked_sizes.append(len(nodes))
- return original_rank(nodes)
-
- monkeypatch.setattr(raptor, "_rank_leaf_pairs", track_rank)
-
- root, leaves = raptor._build_psi_structure(chunks)
-
- assert len(leaves) == len(chunks)
- assert all(leaf.parent is not None for leaf in leaves)
- assert all(len(node.children) <= 3 for node in raptor._iter_nodes(root))
- assert max(ranked_sizes) <= 3
-
-
-@pytest.mark.asyncio
-async def test_psi_tree_builder_materializes_rebalanced_summary_layers_without_umap(monkeypatch, raptor_module):
- def fail_umap(*args, **kwargs):
- raise AssertionError("Psi tree builder must use original embeddings, not UMAP")
-
- monkeypatch.setattr("umap.UMAP", fail_umap)
- raptor = _make_raptor(raptor_module, max_cluster=2)
-
- chunks, layers = await raptor(_chunks(), random_state=0)
-
- assert len(chunks) == 5
- assert layers == [(0, 3), (3, 4), (4, 5)]
- assert [chunk[0] for chunk in chunks[3:]] == ["summary-1", "summary-2"]
-
-
-@pytest.mark.asyncio
-async def test_psi_tree_builder_skips_failed_node_summary(monkeypatch, raptor_module):
- raptor = _make_raptor(raptor_module, max_cluster=2)
-
- async def fail_summary(*args, **kwargs):
- return None
-
- monkeypatch.setattr(raptor, "_summarize_texts", fail_summary)
-
- chunks, layers = await raptor(_chunks(), random_state=0)
-
- assert len(chunks) == 3
- assert [chunk[0] for chunk in chunks] == [chunk[0] for chunk in _chunks()]
- assert layers == [(0, 3)]
-
-
-@pytest.mark.asyncio
-async def test_original_raptor_stops_when_transient_summary_fails(monkeypatch, raptor_module):
- raptor = _make_raptor(raptor_module, tree_builder=raptor_module.RAPTOR_TREE_BUILDER)
-
- async def fail_summary(*args, **kwargs):
- return None
-
- monkeypatch.setattr(raptor, "_summarize_texts", fail_summary)
-
- input_chunks = _chunks()[:2]
- chunks, layers = await raptor(input_chunks, random_state=0)
-
- assert len(chunks) == 2
- assert [chunk[0] for chunk in chunks] == [chunk[0] for chunk in input_chunks]
- assert layers == [(0, 2)]
diff --git a/test/unit_test/rag/utils/test_raptor_utils.py b/test/unit_test/rag/utils/test_raptor_utils.py
index f9c5f0b060..22e86be6a2 100644
--- a/test/unit_test/rag/utils/test_raptor_utils.py
+++ b/test/unit_test/rag/utils/test_raptor_utils.py
@@ -17,14 +17,8 @@
Unit tests for rag/utils/raptor_utils.py module.
"""
-import pytest
from rag.utils.raptor_utils import (
RAPTOR_TREE_BUILDER,
- PSI_TREE_BUILDER,
- GMM_CLUSTERING_METHOD,
- AHC_CLUSTERING_METHOD,
- get_raptor_tree_builder,
- get_raptor_clustering_method,
_as_extra_dict,
_has_raptor_marker,
_raptor_methods_from_fields,
@@ -38,65 +32,6 @@ from rag.utils.raptor_utils import (
)
-class TestGetRaptorTreeBuilder:
- """Tests for get_raptor_tree_builder function."""
-
- def test_returns_default_raptor_tree_builder(self):
- """Test that default tree builder is 'raptor'."""
- result = get_raptor_tree_builder(None)
- assert result == RAPTOR_TREE_BUILDER
-
- def test_returns_default_with_empty_config(self):
- """Test that empty config returns default."""
- result = get_raptor_tree_builder({})
- assert result == RAPTOR_TREE_BUILDER
-
- def test_returns_configured_tree_builder(self):
- """Test that configured tree builder is returned."""
- config = {"tree_builder": PSI_TREE_BUILDER}
- result = get_raptor_tree_builder(config)
- assert result == PSI_TREE_BUILDER
-
- def test_returns_ext_tree_builder(self):
- """Test that ext.tree_builder takes precedence."""
- config = {"tree_builder": "old", "ext": {"tree_builder": PSI_TREE_BUILDER}}
- result = get_raptor_tree_builder(config)
- assert result == PSI_TREE_BUILDER
-
- def test_raises_error_for_unsupported_tree_builder(self):
- """Test that unsupported tree builder raises ValueError."""
- config = {"tree_builder": "unknown"}
- with pytest.raises(ValueError, match="Unsupported RAPTOR tree builder"):
- get_raptor_tree_builder(config)
-
-
-class TestGetRaptorClusteringMethod:
- """Tests for get_raptor_clustering_method function."""
-
- def test_returns_default_gmm(self):
- """Test that default clustering method is 'gmm'."""
- result = get_raptor_clustering_method(None)
- assert result == GMM_CLUSTERING_METHOD
-
- def test_returns_configured_clustering_method(self):
- """Test that configured clustering method is returned."""
- config = {"clustering_method": AHC_CLUSTERING_METHOD}
- result = get_raptor_clustering_method(config)
- assert result == AHC_CLUSTERING_METHOD
-
- def test_returns_ext_clustering_method(self):
- """Test that ext.clustering_method takes precedence."""
- config = {"clustering_method": "old", "ext": {"clustering_method": AHC_CLUSTERING_METHOD}}
- result = get_raptor_clustering_method(config)
- assert result == AHC_CLUSTERING_METHOD
-
- def test_raises_error_for_unsupported_clustering_method(self):
- """Test that unsupported clustering method raises ValueError."""
- config = {"clustering_method": "unknown"}
- with pytest.raises(ValueError, match="Unsupported RAPTOR clustering method"):
- get_raptor_clustering_method(config)
-
-
class TestAsExtraDict:
"""Tests for _as_extra_dict function."""
@@ -129,14 +64,14 @@ class TestAsExtraDict:
assert result == {}
def test_parses_python_dict_literal(self):
- """Test that Python dict literal is parsed."""
+ """Test that Python dict literal string is parsed correctly."""
input_str = "{'key': 'value'}"
result = _as_extra_dict(input_str)
assert result == {"key": "value"}
def test_returns_empty_dict_for_malformed_string(self):
"""Test that malformed string returns empty dict."""
- input_str = "{invalid json}"
+ input_str = "not a dict at all"
result = _as_extra_dict(input_str)
assert result == {}
@@ -162,7 +97,7 @@ class TestHasRaptorMarker:
def test_returns_false_for_list_without_raptor(self):
"""Test that list without 'raptor' returns False."""
- assert _has_raptor_marker(["psi", "other"]) is False
+ assert _has_raptor_marker(["other", "unknown"]) is False
class TestRaptorMethodsFromFields:
@@ -173,23 +108,23 @@ class TestRaptorMethodsFromFields:
result = _raptor_methods_from_fields({})
assert result == {RAPTOR_TREE_BUILDER}
- def test_returns_method_from_extra_dict(self):
- """Test that method is extracted from extra dict."""
- fields = {"extra": {"raptor_method": PSI_TREE_BUILDER}}
+ def test_returns_raptor_method_from_extra_dict(self):
+ """Test that the RAPTOR method is extracted from extra dict."""
+ fields = {"extra": {"raptor_method": RAPTOR_TREE_BUILDER}}
result = _raptor_methods_from_fields(fields)
- assert result == {PSI_TREE_BUILDER}
+ assert result == {RAPTOR_TREE_BUILDER}
def test_returns_method_from_extra_field(self):
"""Test that method is extracted from extra field directly."""
- fields = {"extra": "{'raptor_method': 'psi'}"}
+ fields = {"extra": "{'raptor_method': 'raptor'}"}
result = _raptor_methods_from_fields(fields)
- assert result == {PSI_TREE_BUILDER}
+ assert result == {RAPTOR_TREE_BUILDER}
def test_handles_list_method(self):
"""Test that list method is converted to set."""
- fields = {"extra": {"raptor_method": ["raptor", "psi"]}}
+ fields = {"extra": {"raptor_method": ["raptor", "other"]}}
result = _raptor_methods_from_fields(fields)
- assert result == {RAPTOR_TREE_BUILDER, PSI_TREE_BUILDER}
+ assert result == {RAPTOR_TREE_BUILDER, "other"}
def test_handles_empty_method(self):
"""Test that empty method returns default."""
@@ -208,21 +143,21 @@ class TestCollectRaptorMethods:
def test_collects_methods_from_raptor_chunks(self):
"""Test that methods are collected from RAPTOR chunks."""
- field_map = {"chunk_1": {"raptor_kwd": "raptor", "extra": {"raptor_method": PSI_TREE_BUILDER}}}
+ field_map = {"chunk_1": {"raptor_kwd": "raptor", "extra": {"raptor_method": RAPTOR_TREE_BUILDER}}}
result = collect_raptor_methods(field_map)
- assert result == {PSI_TREE_BUILDER}
+ assert result == {RAPTOR_TREE_BUILDER}
def test_skips_non_raptor_chunks(self):
"""Test that non-RAPTOR chunks are skipped."""
- field_map = {"chunk_1": {"raptor_kwd": "other", "extra": {"raptor_method": PSI_TREE_BUILDER}}}
+ field_map = {"chunk_1": {"raptor_kwd": "other", "extra": {"raptor_method": RAPTOR_TREE_BUILDER}}}
result = collect_raptor_methods(field_map)
assert result == set()
def test_collects_multiple_methods(self):
"""Test that multiple methods are collected."""
- field_map = {"chunk_1": {"raptor_kwd": "raptor", "extra": {"raptor_method": "raptor"}}, "chunk_2": {"raptor_kwd": "raptor", "extra": {"raptor_method": "psi"}}}
+ field_map = {"chunk_1": {"raptor_kwd": "raptor", "extra": {"raptor_method": "raptor"}}, "chunk_2": {"raptor_kwd": "raptor", "extra": {"raptor_method": "other"}}}
result = collect_raptor_methods(field_map)
- assert result == {RAPTOR_TREE_BUILDER, PSI_TREE_BUILDER}
+ assert result == {RAPTOR_TREE_BUILDER, "other"}
class TestCollectRaptorChunkIds:
@@ -241,7 +176,7 @@ class TestCollectRaptorChunkIds:
def test_excludes_specified_methods(self):
"""Test that specified methods are excluded."""
- field_map = {"chunk_1": {"raptor_kwd": "raptor", "extra": {"raptor_method": "raptor"}}, "chunk_2": {"raptor_kwd": "raptor", "extra": {"raptor_method": "psi"}}}
+ field_map = {"chunk_1": {"raptor_kwd": "raptor", "extra": {"raptor_method": "raptor"}}, "chunk_2": {"raptor_kwd": "raptor", "extra": {"raptor_method": "other"}}}
result = collect_raptor_chunk_ids(field_map, exclude_methods={"raptor"})
assert result == {"chunk_2"}
diff --git a/web/src/interfaces/database/compilation-template.ts b/web/src/interfaces/database/compilation-template.ts
index b29876e69e..8310a8f2ad 100644
--- a/web/src/interfaces/database/compilation-template.ts
+++ b/web/src/interfaces/database/compilation-template.ts
@@ -13,7 +13,8 @@ export interface ICompilationTemplateSection {
export interface ICompilationTemplateRaptorConfig {
prompt?: string;
max_token?: number;
- threshold?: number;
+ clustering_threshold?: number;
+ clustering_ratio?: number;
rechunk?: boolean;
}
diff --git a/web/src/interfaces/request/compilation-template.ts b/web/src/interfaces/request/compilation-template.ts
index 6ab69cb280..9b36a00a9c 100644
--- a/web/src/interfaces/request/compilation-template.ts
+++ b/web/src/interfaces/request/compilation-template.ts
@@ -13,7 +13,8 @@ export interface ICompilationTemplateSectionRequest {
export interface ICompilationTemplateRaptorConfigRequest {
prompt?: string;
max_token?: number;
- threshold?: number;
+ clustering_threshold?: number;
+ clustering_ratio?: number;
rechunk?: boolean;
}
diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts
index cd4ac840a7..b157c4a063 100644
--- a/web/src/locales/en.ts
+++ b/web/src/locales/en.ts
@@ -932,9 +932,14 @@ Example: A 1 KB message with 1024-dim embedding uses ~9 KB. The 5 MB default lim
promptTip:
'Use the system prompt to describe the task for the LLM, specify how it should respond, and outline other miscellaneous requirements. The system prompt is often used in conjunction with keys (variables), which serve as various data inputs for the LLM. Use a forward slash `/` or the (x) button to show the keys to use.',
promptMessage: 'Prompt is required',
- promptText: `Please summarize the following paragraphs. Be careful with the numbers, do not make things up. Paragraphs as following:
- {cluster_content}
-The above is the content you need to summarize.`,
+ promptText: `Summarize the paragraphs below without inventing facts or changing numbers.
+Output exactly two parts in the same language as the source:
+1. First line: a concise title only.
+2. Following lines: a concise summary of the content.
+Do not output labels, Markdown headings, bullet points, or any other commentary.
+
+Paragraphs:
+{cluster_content}`,
maxToken: 'Max token',
maxTokenTip: 'The maximum number of tokens per generated summary chunk.',
maxTokenMessage: 'Max token is required',
@@ -1900,6 +1905,12 @@ Example: Virtual Hosted Style`,
maxToken: 'Max token',
maxTokenRequired: 'Please input max token',
threshold: 'Threshold',
+ clusteringThreshold: 'Clustering threshold',
+ clusteringThresholdTip:
+ 'Sets the percentile used to split clusters by adjacent chunk similarity. Higher values create more cluster boundaries.',
+ clusteringRatio: 'Clustering ratio',
+ clusteringRatioTip:
+ 'Sets the maximum number of clusters as a fraction of the input chunks. Lower values produce fewer clusters.',
rechunkByTreeLeaves: 'Re-chunk by tree leaves',
rechunkByTreeLeavesTip:
"Merge each leaf cluster's source chunks into a single replacement chunk. Originals are kept but marked unavailable for retrieval. Only one tree template per group may enable this.",
diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts
index 44816e32a0..4f1d5995a7 100644
--- a/web/src/locales/zh.ts
+++ b/web/src/locales/zh.ts
@@ -841,9 +841,14 @@ export default {
'RAPTOR 常应用于复杂的多跳问答任务。如需打开,请跳转至知识库的文件页面,点击生成 > RAPTOR 开启。详见: https://ragflow.io/docs/dev/enable_raptor。',
prompt: '提示词',
promptMessage: '提示词是必填项',
- promptText: `请总结以下段落。 小心数字,不要编造。 段落如下:
- {cluster_content}
-以上就是你需要总结的内容。`,
+ promptText: `请在不编造事实、不改变数字的前提下总结以下段落。
+请用与原文相同的语言严格输出两部分:
+1. 第一行:仅输出简洁标题。
+2. 后续行:输出内容的简洁摘要。
+不要输出标签、Markdown 标题、项目符号或其他说明。
+
+段落:
+{cluster_content}`,
maxToken: '最大token数',
maxTokenMessage: '最大token数是必填项',
threshold: '阈值',
@@ -1590,6 +1595,12 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取实体和关系
maxToken: '最大 token 数',
maxTokenRequired: '请输入最大 token 数',
threshold: '阈值',
+ clusteringThreshold: '聚类阈值',
+ clusteringThresholdTip:
+ '按相邻数据块相似度分布的百分位决定聚类边界。数值越高,产生的聚类边界越多。',
+ clusteringRatio: '聚类比例',
+ clusteringRatioTip:
+ '设置聚类数量相对于输入数据块数量的最大比例。数值越低,聚类数量越少。',
rechunkByTreeLeaves: '按树叶重新分块',
rechunkByTreeLeavesTip:
'将每个叶簇的源数据块合并为单个替换数据块。原始数据块保留但标记为不可检索。每个分组最多只能有一个树模板启用此功能。',
diff --git a/web/src/pages/user-setting/compilation-templates/create-next/components/tree-template-fields.tsx b/web/src/pages/user-setting/compilation-templates/create-next/components/tree-template-fields.tsx
index d18f49a05f..25b675e97c 100644
--- a/web/src/pages/user-setting/compilation-templates/create-next/components/tree-template-fields.tsx
+++ b/web/src/pages/user-setting/compilation-templates/create-next/components/tree-template-fields.tsx
@@ -21,7 +21,8 @@ export function TreeTemplateFields({ index }: TreeTemplateFieldsProps) {
>
@@ -29,12 +30,21 @@ export function TreeTemplateFields({ index }: TreeTemplateFieldsProps) {
name={`templates.${index}.config.raptor.max_token`}
label={t('setting.maxToken')}
max={2048}
- min={0}
+ min={512}
step={1}
/>
+ string) =>
export const buildRaptorConfigSchema = (t: (key: string) => string) =>
z.object({
prompt: z.string().optional(),
- max_token: z.number().min(1, t('setting.maxTokenRequired')),
- threshold: z.number().min(0).max(1),
+ max_token: z.number().min(512, t('setting.maxTokenRequired')).max(2048),
+ clustering_threshold: z.number().min(0).max(1),
+ clustering_ratio: z.number().min(0).max(1),
rechunk: z.boolean().optional(),
});
diff --git a/web/src/pages/user-setting/compilation-templates/create-next/utils.ts b/web/src/pages/user-setting/compilation-templates/create-next/utils.ts
index 9befdf11b4..82140d5c44 100644
--- a/web/src/pages/user-setting/compilation-templates/create-next/utils.ts
+++ b/web/src/pages/user-setting/compilation-templates/create-next/utils.ts
@@ -139,7 +139,8 @@ export const buildConfigFromBuiltin = (
raptor: {
prompt: builtinRaptor.prompt ?? '',
max_token: builtinRaptor.max_token ?? 512,
- threshold: builtinRaptor.threshold ?? 0.1,
+ clustering_threshold: builtinRaptor.clustering_threshold ?? 0.3,
+ clustering_ratio: builtinRaptor.clustering_ratio ?? 0.5,
rechunk: builtinRaptor.rechunk ?? false,
},
};
@@ -205,7 +206,8 @@ export const transformDetailToForm = (
raptor: {
prompt: raptor.prompt ?? '',
max_token: raptor.max_token ?? 512,
- threshold: raptor.threshold ?? 0.1,
+ clustering_threshold: raptor.clustering_threshold ?? 0.3,
+ clustering_ratio: raptor.clustering_ratio ?? 0.5,
rechunk: raptor.rechunk ?? false,
},
},
diff --git a/web/src/pages/user-setting/compilation-templates/edit-next/components/tree-template-fields.tsx b/web/src/pages/user-setting/compilation-templates/edit-next/components/tree-template-fields.tsx
index d18f49a05f..25b675e97c 100644
--- a/web/src/pages/user-setting/compilation-templates/edit-next/components/tree-template-fields.tsx
+++ b/web/src/pages/user-setting/compilation-templates/edit-next/components/tree-template-fields.tsx
@@ -21,7 +21,8 @@ export function TreeTemplateFields({ index }: TreeTemplateFieldsProps) {
>
@@ -29,12 +30,21 @@ export function TreeTemplateFields({ index }: TreeTemplateFieldsProps) {
name={`templates.${index}.config.raptor.max_token`}
label={t('setting.maxToken')}
max={2048}
- min={0}
+ min={512}
step={1}
/>
+ string) =>
export const buildRaptorConfigSchema = (t: (key: string) => string) =>
z.object({
prompt: z.string().optional(),
- max_token: z.number().min(1, t('setting.maxTokenRequired')),
- threshold: z.number().min(0).max(1),
+ max_token: z.number().min(512, t('setting.maxTokenRequired')).max(2048),
+ clustering_threshold: z.number().min(0).max(1),
+ clustering_ratio: z.number().min(0).max(1),
rechunk: z.boolean().optional(),
});
diff --git a/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts b/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts
index b35132fd1b..4178431a84 100644
--- a/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts
+++ b/web/src/pages/user-setting/compilation-templates/edit-next/utils.ts
@@ -135,7 +135,8 @@ export const buildConfigFromBuiltin = (
raptor: {
prompt: builtinRaptor.prompt ?? '',
max_token: builtinRaptor.max_token ?? 512,
- threshold: builtinRaptor.threshold ?? 0.1,
+ clustering_threshold: builtinRaptor.clustering_threshold ?? 0.3,
+ clustering_ratio: builtinRaptor.clustering_ratio ?? 0.5,
rechunk: builtinRaptor.rechunk ?? false,
},
};
@@ -201,7 +202,8 @@ export const transformDetailToForm = (
raptor: {
prompt: raptor.prompt ?? '',
max_token: raptor.max_token ?? 512,
- threshold: raptor.threshold ?? 0.1,
+ clustering_threshold: raptor.clustering_threshold ?? 0.3,
+ clustering_ratio: raptor.clustering_ratio ?? 0.5,
rechunk: raptor.rechunk ?? false,
},
},