fix(mistral): honor dataset language in figure prompts (#18021)

### Summary

Refs #17885.

Mistral figure enrichment now receives the dataset language through the
production parsing path. `by_mistral_ocr` forwards `lang` to
`MistralParser.parse_pdf`; the parser stores the normalized language and
passes it to the figure-description prompt. Empty or missing values
still fall back to English.
This commit is contained in:
Ziyang Guo
2026-08-11 20:38:24 +08:00
committed by GitHub
parent ad6fdfd7b4
commit 75363af54b
4 changed files with 38 additions and 3 deletions

View File

@@ -78,6 +78,7 @@ class MistralParser(RAGFlowPdfParser):
# Optional tenant vision model (LLMBundle) for figure description;
# set from parse_pdf's vision_model kwarg. None -> no enrichment.
self.vision_model = None
self.language = "English"
# ------------------------------------------------------------------
# Page image rendering
@@ -326,7 +327,8 @@ class MistralParser(RAGFlowPdfParser):
# vision_llm_chunk expects a PIL Image (it calls img.size / img.save),
# not raw bytes — pass the crop directly.
desc = vision_llm_chunk(binary=img, vision_model=self.vision_model, prompt=vision_llm_figure_describe_prompt())
self.logger.debug("[Mistral OCR] describing figure with language=%s", self.language)
desc = vision_llm_chunk(binary=img, vision_model=self.vision_model, prompt=vision_llm_figure_describe_prompt(language=self.language))
return (desc or "").strip()
except Exception:
self.logger.info("[Mistral OCR] figure description skipped", exc_info=True)
@@ -457,6 +459,7 @@ class MistralParser(RAGFlowPdfParser):
def parse_pdf(self, filepath: str | PathLike[str], binary=None, callback=None, parse_method: str = "raw", from_page: int = 0, to_page: int = MAXIMUM_PAGE_NUMBER, **kwargs) -> tuple[list, list]:
# Optional tenant vision model for figure description (best-effort).
self.vision_model = kwargs.pop("vision_model", None)
self.language = kwargs.pop("lang", None) or "English"
# Load bytes.
if binary is not None:

View File

@@ -396,6 +396,7 @@ def by_mistral_ocr(
parse_method=parse_method,
from_page=from_page,
to_page=to_page,
lang=lang,
**kwargs,
)
return sections, tables, pdf_parser

View File

@@ -539,8 +539,11 @@ def test_parse_pdf_consumes_vision_model_kwarg(monkeypatch, tmp_path):
_patch_render(m, p, 2)
pdf = tmp_path / "x.pdf"
pdf.write_bytes(b"%PDF-1.4 minimal")
p.parse_pdf(str(pdf), vision_model="VM")
p.parse_pdf(str(pdf), vision_model="VM", lang="Japanese")
assert p.vision_model == "VM" # popped from kwargs into self, not forwarded to _call_ocr
assert p.language == "Japanese"
p.parse_pdf(str(pdf), vision_model="VM", lang="")
assert p.language == "English"
def test_describe_image_passes_pil_image_not_bytes(monkeypatch):
@@ -562,10 +565,11 @@ def test_describe_image_passes_pil_image_not_bytes(monkeypatch):
pic = ModuleType("rag.app.picture")
pic.vision_llm_chunk = lambda binary, vision_model, prompt=None, callback=None: (captured.update(kind=type(binary).__name__), "a white square")[1]
gen = ModuleType("rag.prompts.generator")
gen.vision_llm_figure_describe_prompt = lambda: "describe"
gen.vision_llm_figure_describe_prompt = lambda language: (captured.update(language=language), "describe")[1]
monkeypatch.setitem(_sys.modules, "rag.app.picture", pic)
monkeypatch.setitem(_sys.modules, "rag.prompts.generator", gen)
out = p._describe_image("@@1\t0\t0\t40\t40##")
assert out == "a white square"
assert captured["kind"] == "Image" # PIL Image, not 'bytes'
assert captured["language"] == "English"

View File

@@ -98,3 +98,30 @@ def test_markdown_chunk_forwards_language_to_model_and_figure_parser(monkeypatch
assert parser_factory.call_args.kwargs["vision_model"] is vision_model
assert parser_factory.call_args.kwargs["lang"] == "Japanese"
parser_instance.assert_called_once()
@pytest.mark.p1
def test_mistral_ocr_forwards_language_to_parser(monkeypatch):
parser = Mock()
parser.parse_pdf.return_value = (["section"], [])
ocr_model = Mock(mdl=parser)
monkeypatch.setattr(naive, "resolve_model_config", Mock(return_value={"llm_name": "mistral-ocr"}))
monkeypatch.setattr(naive, "LLMBundle", Mock(return_value=ocr_model))
sections, tables, returned_parser = naive.by_mistral_ocr(
"document.pdf",
binary=b"pdf",
from_page=2,
to_page=5,
lang="Japanese",
callback=lambda *_args, **_kwargs: None,
parse_method="raw",
mistral_ocr_llm_name="mistral-ocr",
tenant_id="tenant-id",
vision_model=object(),
)
assert sections == ["section"]
assert tables == []
assert returned_parser is parser
assert parser.parse_pdf.call_args.kwargs["lang"] == "Japanese"