From add00924e79ee348301440409025795ed1877aa3 Mon Sep 17 00:00:00 2001 From: Charles <74543653+anneheartrecord@users.noreply.github.com> Date: Tue, 11 Aug 2026 21:44:36 +0800 Subject: [PATCH] fix: replace editdistance with rapidfuzz for python 3.13 wheel support (#18132) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Fixes #18107. `editdistance==0.8.1` (the only recent release on PyPI) has no cp313 wheels for any platform. Since this project requires exactly Python 3.13, `uv`/`pip`/`poetry` fall back to building it from source (Cython), which fails on Windows for anyone without a working C build toolchain — that's the PEP 517 build error in the issue. Swapped `editdistance` for `rapidfuzz`, which ships full cp313 wheels (win32/win_amd64/win_arm64 included) and has no build-from-source step on any of our target platforms. The only call site was `EntityResolution.is_similarity` in `rag/graphrag/entity_resolution.py`, using `editdistance.eval(a, b)` to get the unweighted Levenshtein distance between two entity names. `rapidfuzz.distance.Levenshtein.distance(a, b)` computes the same thing (verified identical output on several string pairs) and is used as a direct replacement. --- pyproject.toml | 2 +- rag/graphrag/entity_resolution.py | 5 ++- .../rag/graphrag/test_entity_resolution.py | 34 ++++++++++++++++ uv.lock | 40 +++++++++++++++---- 4 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 test/unit_test/rag/graphrag/test_entity_resolution.py diff --git a/pyproject.toml b/pyproject.toml index 476b94da66..b897bca32c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,6 @@ dependencies = [ "discord-py==2.3.2", "dropbox==12.0.2", "duckduckgo-search>=7.2.0,<8.0.0", - "editdistance==0.8.1", "elasticsearch-dsl==8.12.0", "exceptiongroup>=1.3.0,<2.0.0", "feedparser>=6.0.11,<7.0.0", @@ -105,6 +104,7 @@ dependencies = [ # fixed in >=3.1.7. Pin so uv.lock cannot resolve back to the buggy release. "werkzeug>=3.1.7,<4", "ranx==0.3.20", + "rapidfuzz==3.14.5", "readability-lxml>=0.8.4,<1.0.0", "replicate==0.31.0", "reportlab>=4.4.1", diff --git a/rag/graphrag/entity_resolution.py b/rag/graphrag/entity_resolution.py index 1efd7a2afd..0b900464fc 100644 --- a/rag/graphrag/entity_resolution.py +++ b/rag/graphrag/entity_resolution.py @@ -23,9 +23,10 @@ from typing import Any, Awaitable, Callable import networkx as nx +from rapidfuzz.distance import Levenshtein + from rag.graphrag.general.extractor import Extractor from rag.nlp import is_english -import editdistance from rag.graphrag.entity_resolution_prompt import ENTITY_RESOLUTION_PROMPT from rag.graphrag.checkpoints import resolution_checkpoint_key from rag.llm.chat_model import Base as CompletionLLM @@ -276,7 +277,7 @@ class EntityResolution(Extractor): return False if is_english(a) and is_english(b): - if editdistance.eval(a, b) <= min(len(a), len(b)) // 2: + if Levenshtein.distance(a, b) <= min(len(a), len(b)) // 2: return True return False diff --git a/test/unit_test/rag/graphrag/test_entity_resolution.py b/test/unit_test/rag/graphrag/test_entity_resolution.py new file mode 100644 index 0000000000..c282569589 --- /dev/null +++ b/test/unit_test/rag/graphrag/test_entity_resolution.py @@ -0,0 +1,34 @@ +# +# Copyright 2026 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from rag.graphrag.entity_resolution import EntityResolution + + +def _resolver(): + return EntityResolution(llm_invoker=None) + + +def test_is_similarity_matches_within_half_edit_distance(): + resolver = _resolver() + + assert resolver.is_similarity("microsoft", "microsfot") is True + assert resolver.is_similarity("apple", "orange") is False + + +def test_is_similarity_identical_strings(): + resolver = _resolver() + + assert resolver.is_similarity("openai", "openai") is True diff --git a/uv.lock b/uv.lock index 6e38b0448e..2c4104bd88 100644 --- a/uv.lock +++ b/uv.lock @@ -1828,12 +1828,6 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/0d/2f/633031205333bee5f9f93761af8268746aa75f38754823aabb8570eb245b/ebcdic-1.1.1-py2.py3-none-any.whl", hash = "sha256:33b4cb729bc2d0bf46cc1847b0e5946897cb8d3f53520c5b9aa5fa98d7e735f1" }, ] -[[package]] -name = "editdistance" -version = "0.8.1" -source = { registry = "https://mirrors.aliyun.com/pypi/simple" } -sdist = { url = "https://mirrors.aliyun.com/pypi/packages/d5/18/9f4f975ca87a390832b1c22478f3702fcdf739f83211e24d054b7551270d/editdistance-0.8.1.tar.gz", hash = "sha256:d1cdf80a5d5014b0c9126a69a42ce55a457b457f6986ff69ca98e4fe4d2d8fed" } - [[package]] name = "elastic-transport" version = "8.17.1" @@ -8131,7 +8125,6 @@ dependencies = [ { name = "discord-py" }, { name = "dropbox" }, { name = "duckduckgo-search" }, - { name = "editdistance" }, { name = "elasticsearch-dsl" }, { name = "en-core-web-sm" }, { name = "es-core-news-sm" }, @@ -8211,6 +8204,7 @@ dependencies = [ { name = "quart-cors" }, { name = "quart-schema" }, { name = "ranx" }, + { name = "rapidfuzz" }, { name = "readability-lxml" }, { name = "replicate" }, { name = "reportlab" }, @@ -8295,7 +8289,6 @@ requires-dist = [ { name = "discord-py", specifier = ">=2.3.0" }, { name = "dropbox", specifier = "==12.0.2" }, { name = "duckduckgo-search", specifier = ">=7.2.0,<8.0.0" }, - { name = "editdistance", specifier = "==0.8.1" }, { name = "elasticsearch-dsl", specifier = "==8.12.0" }, { name = "en-core-web-sm", url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl" }, { name = "es-core-news-sm", url = "https://github.com/explosion/spacy-models/releases/download/es_core_news_sm-3.8.0/es_core_news_sm-3.8.0-py3-none-any.whl" }, @@ -8375,6 +8368,7 @@ requires-dist = [ { name = "quart-cors", specifier = "==0.8.0" }, { name = "quart-schema", specifier = "==0.23.0" }, { name = "ranx", specifier = "==0.3.20" }, + { name = "rapidfuzz", specifier = "==3.14.5" }, { name = "readability-lxml", specifier = ">=0.8.4,<1.0.0" }, { name = "replicate", specifier = "==0.31.0" }, { name = "reportlab", specifier = ">=4.4.1" }, @@ -8462,6 +8456,36 @@ wheels = [ { url = "https://mirrors.aliyun.com/pypi/packages/1e/30/53f41b7b728a48da8974075f56c57200d7b11e4e9fa93be3cabf8218dc0c/ranx-0.3.20-py3-none-any.whl", hash = "sha256:e056e4d5981b0328b045868cc7064fc57a545f36009fbe9bb602295ec33335de" }, ] +[[package]] +name = "rapidfuzz" +version = "3.14.5" +source = { registry = "https://mirrors.aliyun.com/pypi/simple" } +sdist = { url = "https://mirrors.aliyun.com/pypi/packages/2c/21/ef6157213316e85790041254259907eb722e00b03480256c0545d98acd33/rapidfuzz-3.14.5.tar.gz", hash = "sha256:ba10ac57884ce82112f7ed910b67e7fb6072d8ef2c06e30dc63c0f604a112e0e" } +wheels = [ + { url = "https://mirrors.aliyun.com/pypi/packages/11/66/252803f2010ba699618cdc048b6e1f7cc1f433c08b4a9a17579b92ab0142/rapidfuzz-3.14.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ebd8fd343bf8492a1e60bcb6dc99f90f74f65d98d8241a6b3e1fed225b76ecd6" }, + { url = "https://mirrors.aliyun.com/pypi/packages/ea/59/b2afd98e41af9cd54554a4c1c423d84cdd60e6b1c0a09496f033b55f60ec/rapidfuzz-3.14.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6737b35d5af7479c5bf9710f7b17edd9d2c43128d974d25fb4ea653e42c64609" }, + { url = "https://mirrors.aliyun.com/pypi/packages/a3/31/7aa7e62c4c516a7af322ed0c4f0774208b72d457d0cfec808bad0df12f4a/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b002c7994cc9f2bc9d9856f0fbaee6e8072c983873846c92f25cefba5b2a925f" }, + { url = "https://mirrors.aliyun.com/pypi/packages/90/79/2fc252a63bc91d3c3b234d0a3a6ad4ebc460037a23cdcdaf9285f986e6c9/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:17a34330cd2a538c1ce5d400b61ba358c5b72c654b928ff87b362e88f8b864c7" }, + { url = "https://mirrors.aliyun.com/pypi/packages/17/54/0c83508f2683ea70e2d05f8527eb07328acf7bb1e9d97a3bece5702378e7/rapidfuzz-3.14.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:95d937e74c1a7a1287dfb03b62a827be08ede10a155cf1af73bbf47f2b73ee6e" }, + { url = "https://mirrors.aliyun.com/pypi/packages/71/1b/070175e873177814d58850a01ebe80e20ae11e93eb4da894d563988660fa/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:46b92a9970dcc34f0096901c792644094cab49554ac3547f35e3aebbdf0a3610" }, + { url = "https://mirrors.aliyun.com/pypi/packages/c9/dd/77caf7aaf9c2be050ad1f128d7c24ff0f59079aa62c5f62f9df41c0af45e/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:e012177c8e8a8a0754ae0d6027d63042aa5ff036d9f40f07cb3466a6082e21b8" }, + { url = "https://mirrors.aliyun.com/pypi/packages/2c/e2/dd7e1f2aa31a8fbbfc16b0610af1d770ffaf1287490f3c8c5b1c52da264f/rapidfuzz-3.14.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a2ae6f53f99c9a0eca7a0afc5b4e45fc73bc1dd4ac74c00509031d76df80ed98" }, + { url = "https://mirrors.aliyun.com/pypi/packages/9c/0a/ac99e1ba347ba0e85e0bb60b74231d55fb93c0eff43f2920ccb413d0be08/rapidfuzz-3.14.5-cp313-cp313-win32.whl", hash = "sha256:4a60f0057231188e3bd30216f7b4e0f279b11fa4ec818bb6c1d9f014d1562fbc" }, + { url = "https://mirrors.aliyun.com/pypi/packages/cf/cb/0e251d731b3166378644238e8f0cf9e89858c024e19f75ca9f7e3ae83fd5/rapidfuzz-3.14.5-cp313-cp313-win_amd64.whl", hash = "sha256:11bfc2ed8fbe4ab86bd516fadefab126f90e6dcadffa761739fcb304707dfd35" }, + { url = "https://mirrors.aliyun.com/pypi/packages/30/6f/4548132acc947db6d5346a248e44a8b3a22d608ef30e770fb578caaf2d00/rapidfuzz-3.14.5-cp313-cp313-win_arm64.whl", hash = "sha256:b486b5218808f6f4dc471b114b1054e63553db69705c97da0271f47bd706aedd" }, + { url = "https://mirrors.aliyun.com/pypi/packages/00/60/69b177577290c5eab892c6f75fe89c3aff3f9ae80298a78d9372b1cecb9a/rapidfuzz-3.14.5-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:39ef8658aaf67d51667e7bdaf7096f432333377d8302ac43c70b5df8a4cf89b8" }, + { url = "https://mirrors.aliyun.com/pypi/packages/48/38/2fd790052659cc4e2907b63c25433f0987864b445c1aeec1a302ef5ad948/rapidfuzz-3.14.5-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:9ad37a0be705b544af6296da8edddc260d10a8ae5462530fc9991f66498bb1f9" }, + { url = "https://mirrors.aliyun.com/pypi/packages/80/f4/28430ad8472fc3536e8ebd51a864a226e979cfe924c6e3f83d111373aa74/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d45e06f60729e07d9b20c205f7e5cff90b6ef2584e852eecf46e045aea69627d" }, + { url = "https://mirrors.aliyun.com/pypi/packages/77/7e/9aeacabcfd1e77397968362e5b98fe14248b8307011136b17daf99752a8e/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e52da10236aa6212de71b9e170bace65b64b129c0dea7fc243d6c9ce976f5074" }, + { url = "https://mirrors.aliyun.com/pypi/packages/56/f4/db4dd7be0cd2f2022117ac5407d905f435d60e48baaea313a567ad27e865/rapidfuzz-3.14.5-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:440d30faaf682ca496170a7f0cc5453ec942e3e079f0fd802c9a7f938dfb50a3" }, + { url = "https://mirrors.aliyun.com/pypi/packages/a4/99/0e9f6aa57f3e32a767216f797e56dc96b720fcecfb9d8ee907ecc82f8d66/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:56227a61fd3d17b0cd9793132431f3a3d07c8654be96794ba9f89fe0fc8b2d09" }, + { url = "https://mirrors.aliyun.com/pypi/packages/60/94/44a78e39ffce17cbdd3e2b53b696acc751d5d153be0f499d052b07a4d904/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:2e83cd2e25bb4edd97b689d9979d9c3acccdaaf26ceac08212ceece202febcfa" }, + { url = "https://mirrors.aliyun.com/pypi/packages/dd/df/454311469a09a507e9d784a35796742bec22e4cebe75551e2da4e0e290fd/rapidfuzz-3.14.5-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:af3b859726cd3374287e405e14b9634563c078c5531a4f62375508addebddad1" }, + { url = "https://mirrors.aliyun.com/pypi/packages/fc/01/175465a9ab3e3b70ba669058372f009d1d49c1746e2dcd56b69df188d3a5/rapidfuzz-3.14.5-cp313-cp313t-win32.whl", hash = "sha256:8ce1d850b3c0178440efde9e884d98421b5e87ff925f364d6d79e23910d7593f" }, + { url = "https://mirrors.aliyun.com/pypi/packages/1b/a0/a9b84a47af06ebed94a1439eb2f02adebfb8628bcd30af1fe3e02f5ef56c/rapidfuzz-3.14.5-cp313-cp313t-win_amd64.whl", hash = "sha256:c84af70bcf34e99aee894e46a0f1ac77f17d0ef828179c387407642e2466d28a" }, + { url = "https://mirrors.aliyun.com/pypi/packages/1e/f1/5937800238b3f8248e70860d79f69ba8f73e764fff47e36bc9e2f26dbcc6/rapidfuzz-3.14.5-cp313-cp313t-win_arm64.whl", hash = "sha256:aac0ad28c686a5e72b81668b906c030ee28050b244544b8af68e12fb32543895" }, +] + [[package]] name = "readability-lxml" version = "0.8.4.1"