mirror of
https://github.com/infiniflow/ragflow.git
synced 2026-08-16 21:50:58 +08:00
fix(mineru): honor dataset language in figure prompts (#18188)
### Summary Closes #17885. MinerU figure enrichment now passes the resolved dataset language to `vision_llm_figure_describe_prompt`. Missing and empty language values use `English`, matching the other figure-description paths. This change is limited to MinerU. PR #18021 already fixed the Mistral path.
This commit is contained in:
@@ -876,7 +876,7 @@ class MinerUParser(RAGFlowPdfParser):
|
||||
def _transfer_to_tables(self, outputs: list[dict[str, Any]]):
|
||||
return []
|
||||
|
||||
def _enhance_images_with_vlm(self, outputs: list[dict[str, Any]], vision_model, callback: Optional[Callable] = None):
|
||||
def _enhance_images_with_vlm(self, outputs: list[dict[str, Any]], vision_model, callback: Optional[Callable] = None, language: str = "English"):
|
||||
"""Generate semantic descriptions for image blocks via the tenant's
|
||||
VISION model, mirroring deepdoc's VisionFigureParser. Each
|
||||
IMAGE block with a readable img_path gets a ``vlm_description``
|
||||
@@ -894,7 +894,7 @@ class MinerUParser(RAGFlowPdfParser):
|
||||
if callback:
|
||||
callback(0.78, f"[MinerU] Generating VLM descriptions for {len(image_jobs)} images...")
|
||||
|
||||
prompt = vision_llm_figure_describe_prompt()
|
||||
prompt = vision_llm_figure_describe_prompt(language=language or "English")
|
||||
|
||||
def worker(idx, item):
|
||||
try:
|
||||
@@ -935,7 +935,7 @@ class MinerUParser(RAGFlowPdfParser):
|
||||
created_tmp_dir = False
|
||||
|
||||
parser_cfg = kwargs.get("parser_config", {})
|
||||
lang = parser_cfg.get("mineru_lang") or kwargs.get("lang", "English")
|
||||
lang = parser_cfg.get("mineru_lang") or kwargs.get("lang") or "English"
|
||||
mineru_lang_code = LANGUAGE_TO_MINERU_MAP.get(lang, "ch") # Defaults to Chinese if not matched
|
||||
mineru_method_raw_str = parser_cfg.get("mineru_parse_method", "auto")
|
||||
enable_formula = parser_cfg.get("mineru_formula_enable", True)
|
||||
@@ -998,7 +998,7 @@ class MinerUParser(RAGFlowPdfParser):
|
||||
vision_model = kwargs.get("vision_model")
|
||||
if vision_model is not None:
|
||||
try:
|
||||
self._enhance_images_with_vlm(outputs, vision_model, callback=callback)
|
||||
self._enhance_images_with_vlm(outputs, vision_model, callback=callback, language=lang)
|
||||
except Exception as e:
|
||||
self.logger.warning(f"[MinerU] VLM image enhancement failed: {e}. Continuing without descriptions.")
|
||||
|
||||
|
||||
@@ -4,8 +4,11 @@ import sys
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from unittest.mock import Mock
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _load_mineru_parser(monkeypatch):
|
||||
repo_root = Path(__file__).resolve().parents[4]
|
||||
@@ -39,6 +42,73 @@ def _load_mineru_parser(monkeypatch):
|
||||
return module
|
||||
|
||||
|
||||
@pytest.mark.p1
|
||||
@pytest.mark.parametrize(
|
||||
("language", "expected_language"),
|
||||
[
|
||||
("Japanese", "Japanese"),
|
||||
("", "English"),
|
||||
(None, "English"),
|
||||
],
|
||||
)
|
||||
def test_enhance_images_with_vlm_passes_dataset_language_to_prompt(monkeypatch, tmp_path, language, expected_language):
|
||||
module = _load_mineru_parser(monkeypatch)
|
||||
parser = module.MinerUParser()
|
||||
image_path = tmp_path / "figure.png"
|
||||
module.Image.new("RGB", (1, 1)).save(image_path)
|
||||
|
||||
picture_module = ModuleType("rag.app.picture")
|
||||
picture_module.vision_llm_chunk = Mock(return_value="description")
|
||||
prompt = Mock(return_value="prompt")
|
||||
generator_module = ModuleType("rag.prompts.generator")
|
||||
generator_module.vision_llm_figure_describe_prompt = prompt
|
||||
monkeypatch.setitem(sys.modules, "rag.app.picture", picture_module)
|
||||
monkeypatch.setitem(sys.modules, "rag.prompts.generator", generator_module)
|
||||
|
||||
outputs = [{"type": module.MinerUContentType.IMAGE, "img_path": str(image_path)}]
|
||||
parser._enhance_images_with_vlm(outputs, vision_model=object(), language=language)
|
||||
|
||||
prompt.assert_called_once_with(language=expected_language)
|
||||
assert outputs[0]["vlm_description"] == "description"
|
||||
|
||||
|
||||
@pytest.mark.p1
|
||||
@pytest.mark.parametrize(
|
||||
("language", "expected_language"),
|
||||
[
|
||||
("Japanese", "Japanese"),
|
||||
("", "English"),
|
||||
(None, "English"),
|
||||
],
|
||||
)
|
||||
def test_parse_pdf_forwards_normalized_dataset_language_to_image_enhancement(monkeypatch, tmp_path, language, expected_language):
|
||||
module = _load_mineru_parser(monkeypatch)
|
||||
parser = module.MinerUParser()
|
||||
pdf_path = tmp_path / "document.pdf"
|
||||
pdf_path.write_bytes(b"%PDF-1.4 fake")
|
||||
output_dir = tmp_path / "output"
|
||||
vision_model = object()
|
||||
|
||||
monkeypatch.setattr(module, "extract_pdf_outlines", Mock(return_value=[]))
|
||||
monkeypatch.setattr(parser, "__images__", Mock())
|
||||
monkeypatch.setattr(parser, "_run_mineru", Mock(return_value=output_dir))
|
||||
monkeypatch.setattr(parser, "_read_output", Mock(return_value=[]))
|
||||
enhance = Mock()
|
||||
monkeypatch.setattr(parser, "_enhance_images_with_vlm", enhance)
|
||||
|
||||
language_kwargs = {} if language is None else {"lang": language}
|
||||
parser.parse_pdf(
|
||||
filepath=pdf_path,
|
||||
binary=None,
|
||||
output_dir=str(output_dir),
|
||||
delete_output=False,
|
||||
vision_model=vision_model,
|
||||
**language_kwargs,
|
||||
)
|
||||
|
||||
enhance.assert_called_once_with([], vision_model, callback=None, language=expected_language)
|
||||
|
||||
|
||||
def test_sanitize_section_text_removes_escaped_html_tags(monkeypatch):
|
||||
module = _load_mineru_parser(monkeypatch)
|
||||
text = "<table><tr><td>Alpha</td><td>Beta</td></tr></table>"
|
||||
|
||||
@@ -125,3 +125,30 @@ def test_mistral_ocr_forwards_language_to_parser(monkeypatch):
|
||||
assert tables == []
|
||||
assert returned_parser is parser
|
||||
assert parser.parse_pdf.call_args.kwargs["lang"] == "Japanese"
|
||||
|
||||
|
||||
@pytest.mark.p1
|
||||
def test_mineru_forwards_dataset_language_to_parser(monkeypatch):
|
||||
parser = Mock()
|
||||
parser.parse_pdf.return_value = (["section"], [])
|
||||
ocr_model = Mock(mdl=parser)
|
||||
monkeypatch.setattr(naive, "resolve_model_config", Mock(return_value={"llm_name": "mineru"}))
|
||||
monkeypatch.setattr(naive, "LLMBundle", Mock(return_value=ocr_model))
|
||||
|
||||
sections, tables, returned_parser = naive.by_mineru(
|
||||
"document.pdf",
|
||||
binary=b"pdf",
|
||||
from_page=2,
|
||||
to_page=5,
|
||||
lang="Japanese",
|
||||
callback=lambda *_args, **_kwargs: None,
|
||||
parse_method="raw",
|
||||
mineru_llm_name="mineru",
|
||||
tenant_id="tenant-id",
|
||||
vision_model=object(),
|
||||
)
|
||||
|
||||
assert sections == ["section"]
|
||||
assert tables == []
|
||||
assert returned_parser is parser
|
||||
assert parser.parse_pdf.call_args.kwargs["lang"] == "Japanese"
|
||||
|
||||
Reference in New Issue
Block a user