feat(embeddings): add ONNX local provider (#1970)

* feat(embeddings): add ONNX local provider

* fix(embeddings): download ONNX external data sidecars

* fix(embeddings): address ONNX provider review feedback

.env.example

@@ -80,10 +80,23 @@ HINDSIGHT_API_LOG_LEVEL=info
 # HINDSIGHT_API_TEXT_SEARCH_EXTENSION_PG_SEARCH_TOKENIZER=
 
 # Embeddings Configuration (Optional - uses local by default)
-# Provider: "local" (default), "tei", "openai", "cohere", "google", "openrouter", "zeroentropy", "litellm", or "litellm-sdk"
+# Provider: "local" (default), "onnx", "tei", "openai", "cohere", "google", "openrouter", "zeroentropy", "litellm", or "litellm-sdk"
 # HINDSIGHT_API_EMBEDDINGS_PROVIDER=local
 # For local provider:
 # HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL=BAAI/bge-small-en-v1.5
+# For ONNX provider (local CPU embeddings without an Ollama/TEI sidecar):
+# HINDSIGHT_API_EMBEDDINGS_PROVIDER=onnx
+# HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID=intfloat/multilingual-e5-small
+# HINDSIGHT_API_EMBEDDINGS_ONNX_FILE=onnx/model.onnx
+# HINDSIGHT_API_EMBEDDINGS_ONNX_DIMENSIONS=384
+# HINDSIGHT_API_EMBEDDINGS_ONNX_MAX_TOKENS=512
+# HINDSIGHT_API_EMBEDDINGS_ONNX_POOLING=mean
+# HINDSIGHT_API_EMBEDDINGS_ONNX_NORMALIZE=true
+# HINDSIGHT_API_EMBEDDINGS_ONNX_QUERY_PREFIX="query: "
+# HINDSIGHT_API_EMBEDDINGS_ONNX_PASSAGE_PREFIX="passage: "
+# Optional for local model paths or pre-downloaded artifacts:
+# HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_PATH=/models/multilingual-e5-small/onnx/model.onnx
+# HINDSIGHT_API_EMBEDDINGS_ONNX_TOKENIZER_NAME_OR_PATH=/models/multilingual-e5-small
 # Optional for China network / restricted HF access:
 # HF_ENDPOINT=https://hf-mirror.com
 # For TEI provider:

docker/standalone/Dockerfile

@@ -50,6 +50,8 @@ WORKDIR /app/api
 # Sync dependencies using appropriate extras based on INCLUDE_LOCAL_MODELS
 # local-ml: torch, sentence-transformers, transformers, einops, flashrank, mlx (optional)
 # embedded-db: pg0-embedded (always included for embedded PostgreSQL support)
+# ONNX Runtime embeddings are intentionally not bundled into the official
+# standalone image; install the local-onnx extra in custom images when needed.
 RUN if [ "$INCLUDE_LOCAL_MODELS" = "true" ]; then \
         uv sync --extra local-ml --extra embedded-db; \
     else \

hindsight-api-slim/hindsight_api/config.py

@@ -209,6 +209,17 @@ ENV_EMBEDDINGS_PROVIDER = "HINDSIGHT_API_EMBEDDINGS_PROVIDER"
 ENV_EMBEDDINGS_LOCAL_MODEL = "HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL"
 ENV_EMBEDDINGS_LOCAL_FORCE_CPU = "HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU"
 ENV_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE = "HINDSIGHT_API_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE"
+ENV_EMBEDDINGS_ONNX_MODEL_ID = "HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID"
+ENV_EMBEDDINGS_ONNX_MODEL_PATH = "HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_PATH"
+ENV_EMBEDDINGS_ONNX_TOKENIZER_NAME_OR_PATH = "HINDSIGHT_API_EMBEDDINGS_ONNX_TOKENIZER_NAME_OR_PATH"
+ENV_EMBEDDINGS_ONNX_FILE = "HINDSIGHT_API_EMBEDDINGS_ONNX_FILE"
+ENV_EMBEDDINGS_ONNX_DIMENSIONS = "HINDSIGHT_API_EMBEDDINGS_ONNX_DIMENSIONS"
+ENV_EMBEDDINGS_ONNX_MAX_TOKENS = "HINDSIGHT_API_EMBEDDINGS_ONNX_MAX_TOKENS"
+ENV_EMBEDDINGS_ONNX_POOLING = "HINDSIGHT_API_EMBEDDINGS_ONNX_POOLING"
+ENV_EMBEDDINGS_ONNX_NORMALIZE = "HINDSIGHT_API_EMBEDDINGS_ONNX_NORMALIZE"
+ENV_EMBEDDINGS_ONNX_QUERY_PREFIX = "HINDSIGHT_API_EMBEDDINGS_ONNX_QUERY_PREFIX"
+ENV_EMBEDDINGS_ONNX_PASSAGE_PREFIX = "HINDSIGHT_API_EMBEDDINGS_ONNX_PASSAGE_PREFIX"
+ENV_EMBEDDINGS_ONNX_OUTPUT_NAME = "HINDSIGHT_API_EMBEDDINGS_ONNX_OUTPUT_NAME"
 ENV_EMBEDDINGS_TEI_URL = "HINDSIGHT_API_EMBEDDINGS_TEI_URL"
 ENV_EMBEDDINGS_OPENAI_API_KEY = "HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY"
 ENV_EMBEDDINGS_OPENAI_MODEL = "HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL"
@@ -597,6 +608,13 @@ DEFAULT_EMBEDDINGS_PROVIDER = "local"
 DEFAULT_EMBEDDINGS_LOCAL_MODEL = "BAAI/bge-small-en-v1.5"
 DEFAULT_EMBEDDINGS_LOCAL_FORCE_CPU = False  # Force CPU mode for local embeddings
 DEFAULT_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE = False  # Security: disabled by default, required for some models
+DEFAULT_EMBEDDINGS_ONNX_MODEL_ID = "intfloat/multilingual-e5-small"
+DEFAULT_EMBEDDINGS_ONNX_FILE = "onnx/model.onnx"
+DEFAULT_EMBEDDINGS_ONNX_MAX_TOKENS = 512
+DEFAULT_EMBEDDINGS_ONNX_POOLING = "mean"
+DEFAULT_EMBEDDINGS_ONNX_NORMALIZE = True
+DEFAULT_EMBEDDINGS_ONNX_QUERY_PREFIX = "query: "
+DEFAULT_EMBEDDINGS_ONNX_PASSAGE_PREFIX = "passage: "
 DEFAULT_EMBEDDINGS_OPENAI_MODEL = "text-embedding-3-small"
 DEFAULT_EMBEDDINGS_OPENAI_BATCH_SIZE = 100
 DEFAULT_EMBEDDINGS_GEMINI_MODEL = "gemini-embedding-001"
@@ -1235,6 +1253,17 @@ class HindsightConfig:
     embeddings_local_model: str
     embeddings_local_force_cpu: bool
     embeddings_local_trust_remote_code: bool
+    embeddings_onnx_model_id: str
+    embeddings_onnx_model_path: str | None
+    embeddings_onnx_tokenizer_name_or_path: str | None
+    embeddings_onnx_file: str
+    embeddings_onnx_dimensions: int | None
+    embeddings_onnx_max_tokens: int
+    embeddings_onnx_pooling: str
+    embeddings_onnx_normalize: bool
+    embeddings_onnx_query_prefix: str
+    embeddings_onnx_passage_prefix: str
+    embeddings_onnx_output_name: str | None
     embeddings_tei_url: str | None
     embeddings_openai_base_url: str | None
     embeddings_cohere_api_key: str | None
@@ -1727,6 +1756,21 @@ class HindsightConfig:
                     " and ".join(missing),
                 )
 
+        if self.embeddings_provider == "onnx":
+            try:
+                import importlib
+
+                importlib.import_module("onnxruntime")
+                importlib.import_module("transformers")
+            except ImportError:
+                logger.warning(
+                    "ONNX embeddings provider configured, but 'onnxruntime' and/or "
+                    "'transformers' is not installed. The API will fail at model init time. Either:\n"
+                    "  1. Install ONNX deps: pip install hindsight-api-slim[local-onnx]\n"
+                    "  2. Use a different embeddings provider, e.g. HINDSIGHT_API_EMBEDDINGS_PROVIDER=local "
+                    "or openai"
+                )
+
         # Validate that sum of per-operation slot reservations does not exceed max_slots
         total_reserved = sum(self.worker_slot_reservations.values())
         if total_reserved > self.worker_max_slots:
@@ -1889,6 +1933,36 @@ class HindsightConfig:
                 ENV_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE, str(DEFAULT_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE)
             ).lower()
             in ("true", "1"),
+            embeddings_onnx_model_id=os.getenv(ENV_EMBEDDINGS_ONNX_MODEL_ID, DEFAULT_EMBEDDINGS_ONNX_MODEL_ID),
+            embeddings_onnx_model_path=os.getenv(ENV_EMBEDDINGS_ONNX_MODEL_PATH) or None,
+            embeddings_onnx_tokenizer_name_or_path=os.getenv(ENV_EMBEDDINGS_ONNX_TOKENIZER_NAME_OR_PATH) or None,
+            embeddings_onnx_file=os.getenv(ENV_EMBEDDINGS_ONNX_FILE, DEFAULT_EMBEDDINGS_ONNX_FILE),
+            embeddings_onnx_dimensions=_parse_optional_positive_int(
+                ENV_EMBEDDINGS_ONNX_DIMENSIONS,
+                os.getenv(ENV_EMBEDDINGS_ONNX_DIMENSIONS),
+            ),
+            embeddings_onnx_max_tokens=_parse_positive_int(
+                ENV_EMBEDDINGS_ONNX_MAX_TOKENS,
+                os.getenv(ENV_EMBEDDINGS_ONNX_MAX_TOKENS),
+                DEFAULT_EMBEDDINGS_ONNX_MAX_TOKENS,
+            ),
+            embeddings_onnx_pooling=_parse_optional_choice(
+                ENV_EMBEDDINGS_ONNX_POOLING,
+                os.getenv(ENV_EMBEDDINGS_ONNX_POOLING),
+                frozenset({"mean", "cls"}),
+            )
+            or DEFAULT_EMBEDDINGS_ONNX_POOLING,
+            embeddings_onnx_normalize=os.getenv(
+                ENV_EMBEDDINGS_ONNX_NORMALIZE, str(DEFAULT_EMBEDDINGS_ONNX_NORMALIZE)
+            ).lower()
+            in ("true", "1"),
+            embeddings_onnx_query_prefix=os.getenv(
+                ENV_EMBEDDINGS_ONNX_QUERY_PREFIX, DEFAULT_EMBEDDINGS_ONNX_QUERY_PREFIX
+            ),
+            embeddings_onnx_passage_prefix=os.getenv(
+                ENV_EMBEDDINGS_ONNX_PASSAGE_PREFIX, DEFAULT_EMBEDDINGS_ONNX_PASSAGE_PREFIX
+            ),
+            embeddings_onnx_output_name=os.getenv(ENV_EMBEDDINGS_ONNX_OUTPUT_NAME) or None,
             embeddings_tei_url=os.getenv(ENV_EMBEDDINGS_TEI_URL),
             embeddings_openai_base_url=os.getenv(ENV_EMBEDDINGS_OPENAI_BASE_URL) or None,
             embeddings_openai_batch_size=_parse_positive_int(

hindsight-api-slim/hindsight_api/engine/embeddings.py

@@ -43,6 +43,10 @@ from ..config import (
     ENV_EMBEDDINGS_LOCAL_FORCE_CPU,
     ENV_EMBEDDINGS_LOCAL_MODEL,
     ENV_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE,
+    ENV_EMBEDDINGS_ONNX_DIMENSIONS,
+    ENV_EMBEDDINGS_ONNX_MODEL_ID,
+    ENV_EMBEDDINGS_ONNX_MODEL_PATH,
+    ENV_EMBEDDINGS_ONNX_TOKENIZER_NAME_OR_PATH,
     ENV_EMBEDDINGS_OPENAI_API_KEY,
     ENV_EMBEDDINGS_OPENAI_BASE_URL,
     ENV_EMBEDDINGS_OPENAI_MODEL,
@@ -252,6 +256,172 @@ class LocalSTEmbeddings(Embeddings):
         return [emb.tolist() for emb in embeddings]
 
 
+class OnnxEmbeddings(Embeddings):
+    """Local ONNX Runtime embeddings provider.
+
+    This provider runs transformer embedding models in-process with ONNX Runtime,
+    avoiding a sidecar Ollama/TEI server or a remote embeddings API. It supports
+    sentence-transformer style mean pooling and E5-style asymmetric prefixes.
+    """
+
+    def __init__(
+        self,
+        model_id: str,
+        model_path: str | None = None,
+        tokenizer_name_or_path: str | None = None,
+        onnx_file: str = "onnx/model.onnx",
+        dimensions: int | None = None,
+        max_tokens: int = 512,
+        pooling: str = "mean",
+        normalize: bool = True,
+        query_prefix: str = "query: ",
+        passage_prefix: str = "passage: ",
+        output_name: str | None = None,
+    ):
+        self.model_id = model_id
+        self.model_path = model_path
+        if model_path and tokenizer_name_or_path is None:
+            logger.warning(
+                "Embeddings: ONNX model_path is set without tokenizer_name_or_path; "
+                "falling back to tokenizer from model_id %s. Set "
+                "HINDSIGHT_API_EMBEDDINGS_ONNX_TOKENIZER_NAME_OR_PATH when using local ONNX artifacts.",
+                model_id,
+            )
+        self.tokenizer_name_or_path = tokenizer_name_or_path or model_id
+        self.onnx_file = onnx_file
+        self.configured_dimensions = dimensions
+        self.max_tokens = max_tokens
+        self.pooling = pooling.lower()
+        if self.pooling not in {"mean", "cls"}:
+            raise ValueError("ONNX embeddings pooling must be 'mean' or 'cls'")
+        self.normalize = normalize
+        self.query_prefix = query_prefix
+        self.passage_prefix = passage_prefix
+        self.output_name = output_name
+        self._session = None
+        self._tokenizer = None
+        self._dimension: int | None = dimensions
+
+    @property
+    def provider_name(self) -> str:
+        return "onnx"
+
+    @property
+    def dimension(self) -> int:
+        if self._dimension is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+        return self._dimension
+
+    async def initialize(self) -> None:
+        if self._session is not None and self._tokenizer is not None:
+            return
+
+        try:
+            import onnxruntime as ort
+            from transformers import AutoTokenizer
+        except ImportError as exc:
+            raise ImportError(
+                "onnxruntime and transformers are required for OnnxEmbeddings. "
+                "Install with: pip install 'hindsight-api-slim[local-onnx]'"
+            ) from exc
+
+        model_path = self.model_path
+        if not model_path:
+            try:
+                from huggingface_hub import snapshot_download
+            except ImportError as exc:
+                raise ImportError(
+                    "huggingface-hub is required to download ONNX embedding models. "
+                    "Set HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_PATH or install local-onnx."
+                ) from exc
+            # Some large ONNX exports, for example BAAI/bge-m3, store weights in
+            # an external sidecar file next to model.onnx. Download both the
+            # requested graph and its conventional *_data sidecar when present.
+            snapshot_dir = snapshot_download(
+                repo_id=self.model_id,
+                allow_patterns=[self.onnx_file, f"{self.onnx_file}_data"],
+            )
+            model_path = os.path.join(snapshot_dir, self.onnx_file)
+
+        logger.info(
+            "Embeddings: initializing ONNX provider with model %s (%s)",
+            self.model_id,
+            model_path,
+        )
+        logger.info(
+            "Embeddings: ONNX query_prefix=%r passage_prefix=%r pooling=%s normalize=%s",
+            self.query_prefix,
+            self.passage_prefix,
+            self.pooling,
+            self.normalize,
+        )
+        self._tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_name_or_path)
+        self._session = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
+
+        detected = len(self.encode(["test"])[0])
+        if self.configured_dimensions is not None and detected != self.configured_dimensions:
+            raise ValueError(
+                f"Configured ONNX embedding dimension {self.configured_dimensions} does not match model output {detected}"
+            )
+        self._dimension = detected
+        logger.info("Embeddings: ONNX provider initialized (dim: %s)", self._dimension)
+
+    def _encode_prefixed(self, texts: list[str], prefix: str) -> list[list[float]]:
+        if prefix:
+            return self.encode([f"{prefix}{text}" for text in texts])
+        return self.encode(texts)
+
+    def encode_query(self, texts: list[str]) -> list[list[float]]:
+        return self._encode_prefixed(texts, self.query_prefix)
+
+    def encode_documents(self, texts: list[str]) -> list[list[float]]:
+        return self._encode_prefixed(texts, self.passage_prefix)
+
+    def encode(self, texts: list[str]) -> list[list[float]]:
+        if self._session is None or self._tokenizer is None:
+            raise RuntimeError("Embeddings not initialized. Call initialize() first.")
+        if not texts:
+            return []
+
+        import numpy as np
+
+        encoded = self._tokenizer(
+            texts,
+            padding=True,
+            truncation=True,
+            max_length=self.max_tokens,
+            return_tensors="np",
+        )
+        input_names = {inp.name for inp in self._session.get_inputs()}
+        ort_inputs = {name: value for name, value in encoded.items() if name in input_names}
+        if "token_type_ids" in input_names and "token_type_ids" not in ort_inputs:
+            ort_inputs["token_type_ids"] = np.zeros_like(encoded["input_ids"])
+
+        outputs = self._session.run([self.output_name] if self.output_name else None, ort_inputs)
+        token_embeddings = outputs[0]
+
+        # Some exported models expose a pooled 2-D embedding as their first output.
+        if getattr(token_embeddings, "ndim", 0) == 2:
+            embeddings = token_embeddings
+        elif self.pooling == "cls":
+            embeddings = token_embeddings[:, 0]
+        else:
+            attention_mask = encoded.get("attention_mask")
+            if attention_mask is None:
+                attention_mask = np.ones(token_embeddings.shape[:2], dtype=np.float32)
+            mask = attention_mask[..., None].astype(np.float32)
+            summed = (token_embeddings * mask).sum(axis=1)
+            counts = np.clip(mask.sum(axis=1), a_min=1e-9, a_max=None)
+            embeddings = summed / counts
+
+        if self.normalize:
+            norms = np.linalg.norm(embeddings, axis=1, keepdims=True)
+            norms[norms == 0] = 1
+            embeddings = embeddings / norms
+
+        return embeddings.astype(float).tolist()
+
+
 class RemoteTEIEmbeddings(Embeddings):
     """
     Remote embeddings implementation using HuggingFace Text Embeddings Inference (TEI) HTTP API.
@@ -1391,6 +1561,20 @@ def create_embeddings_from_env() -> Embeddings:
             force_cpu=config.embeddings_local_force_cpu,
             trust_remote_code=config.embeddings_local_trust_remote_code,
         )
+    elif provider == "onnx":
+        return OnnxEmbeddings(
+            model_id=config.embeddings_onnx_model_id,
+            model_path=config.embeddings_onnx_model_path,
+            tokenizer_name_or_path=config.embeddings_onnx_tokenizer_name_or_path,
+            onnx_file=config.embeddings_onnx_file,
+            dimensions=config.embeddings_onnx_dimensions,
+            max_tokens=config.embeddings_onnx_max_tokens,
+            pooling=config.embeddings_onnx_pooling,
+            normalize=config.embeddings_onnx_normalize,
+            query_prefix=config.embeddings_onnx_query_prefix,
+            passage_prefix=config.embeddings_onnx_passage_prefix,
+            output_name=config.embeddings_onnx_output_name,
+        )
     elif provider == "openai":
         # Use dedicated embeddings API key, or fall back to LLM API key
         api_key = os.environ.get(ENV_EMBEDDINGS_OPENAI_API_KEY) or os.environ.get(ENV_LLM_API_KEY)
@@ -1492,6 +1676,6 @@ def create_embeddings_from_env() -> Embeddings:
     else:
         raise ValueError(
             f"Unknown embeddings provider: {provider}. "
-            f"Supported: 'local', 'tei', 'openai', 'openai-codex', 'openrouter', 'cohere', 'google', "
+            f"Supported: 'local', 'onnx', 'tei', 'openai', 'openai-codex', 'openrouter', 'cohere', 'google', "
             f"'zeroentropy', 'litellm', 'litellm-sdk'"
         )

hindsight-api-slim/pyproject.toml

@@ -96,6 +96,13 @@ local-llm = [
     "llama-cpp-python[server]>=0.3.0",
     "huggingface-hub>=0.20.0",
 ]
+local-onnx = [
+    # In-process ONNX Runtime embeddings without an Ollama/TEI sidecar
+    "onnxruntime>=1.17.0",
+    "transformers>=4.53.0",
+    "huggingface-hub>=0.20.0",
+    "numpy>=1.26.0",
+]
 embedded-db = [
     "pg0-embedded>=0.14.2",
 ]
@@ -103,7 +110,7 @@ oracle = [
     "oracledb>=2.5.0",
 ]
 all = [
-    "hindsight-api-slim[local-ml,embedded-db]",
+    "hindsight-api-slim[local-ml,local-onnx,embedded-db]",
 ]
 test = [
     "pytest>=7.0.0",

hindsight-api-slim/tests/test_onnx_embeddings.py

@@ -0,0 +1,199 @@
+"""Tests for the ONNX Runtime embeddings provider."""
+
+import sys
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pytest
+
+from hindsight_api.engine.embeddings import OnnxEmbeddings, create_embeddings_from_env
+
+
+class FakeTokenizer:
+    def __init__(self):
+        self.calls = []
+
+    def __call__(self, texts, padding, truncation, max_length, return_tensors):
+        self.calls.append(
+            {
+                "texts": texts,
+                "padding": padding,
+                "truncation": truncation,
+                "max_length": max_length,
+                "return_tensors": return_tensors,
+            }
+        )
+        batch = len(texts)
+        return {
+            "input_ids": np.ones((batch, 3), dtype=np.int64),
+            "attention_mask": np.array([[1, 1, 0]] * batch, dtype=np.int64),
+            "token_type_ids": np.zeros((batch, 3), dtype=np.int64),
+        }
+
+
+class FakeOnnxSession:
+    def get_inputs(self):
+        return [SimpleNamespace(name="input_ids"), SimpleNamespace(name="attention_mask")]
+
+    def run(self, output_names, inputs):
+        batch = inputs["input_ids"].shape[0]
+        # Last token is masked out. Mean pooling should average first two tokens:
+        # ([3, 4] + [0, 0]) / 2 = [1.5, 2.0], then normalize to [0.6, 0.8].
+        token_embeddings = np.array([[[3.0, 4.0], [0.0, 0.0], [100.0, 100.0]]] * batch, dtype=np.float32)
+        return [token_embeddings]
+
+
+class FakePooledOnnxSession:
+    def get_inputs(self):
+        return [SimpleNamespace(name="input_ids"), SimpleNamespace(name="attention_mask")]
+
+    def run(self, output_names, inputs):
+        batch = inputs["input_ids"].shape[0]
+        assert output_names == ["sentence_embedding"]
+        return [np.array([[3.0, 4.0]] * batch, dtype=np.float32)]
+
+
+def test_onnx_embeddings_mean_pooling_normalizes_and_filters_inputs():
+    emb = OnnxEmbeddings(model_id="intfloat/multilingual-e5-small", dimensions=2, max_tokens=17)
+    emb._tokenizer = FakeTokenizer()
+    emb._session = FakeOnnxSession()
+    emb._dimension = 2
+
+    result = emb.encode(["hello"])
+
+    assert result == [pytest.approx([0.6, 0.8])]
+    assert emb._tokenizer.calls[-1]["max_length"] == 17
+
+
+def test_onnx_embeddings_cls_pooling_and_normalize_false():
+    emb = OnnxEmbeddings(
+        model_id="intfloat/multilingual-e5-small",
+        dimensions=2,
+        pooling="cls",
+        normalize=False,
+    )
+    emb._tokenizer = FakeTokenizer()
+    emb._session = FakeOnnxSession()
+    emb._dimension = 2
+
+    result = emb.encode(["hello"])
+
+    assert result == [pytest.approx([3.0, 4.0])]
+
+
+def test_onnx_embeddings_output_name_uses_pre_pooled_2d_output():
+    emb = OnnxEmbeddings(
+        model_id="intfloat/multilingual-e5-small",
+        dimensions=2,
+        output_name="sentence_embedding",
+    )
+    emb._tokenizer = FakeTokenizer()
+    emb._session = FakePooledOnnxSession()
+    emb._dimension = 2
+
+    result = emb.encode(["hello"])
+
+    assert result == [pytest.approx([0.6, 0.8])]
+
+
+def test_onnx_embeddings_rejects_invalid_pooling_before_initialize():
+    with pytest.raises(ValueError, match="pooling"):
+        OnnxEmbeddings(model_id="intfloat/multilingual-e5-small", pooling="max")
+
+
+def test_onnx_embeddings_warns_when_local_model_path_has_no_tokenizer(caplog):
+    emb = OnnxEmbeddings(
+        model_id="intfloat/multilingual-e5-small",
+        model_path="/models/custom/onnx/model.onnx",
+    )
+
+    assert emb.tokenizer_name_or_path == "intfloat/multilingual-e5-small"
+    assert "model_path is set without tokenizer_name_or_path" in caplog.text
+
+
+def test_onnx_embeddings_query_and_document_prefixes_are_asymmetric():
+    tokenizer = FakeTokenizer()
+    emb = OnnxEmbeddings(
+        model_id="intfloat/multilingual-e5-small",
+        dimensions=2,
+        query_prefix="query: ",
+        passage_prefix="passage: ",
+    )
+    emb._tokenizer = tokenizer
+    emb._session = FakeOnnxSession()
+    emb._dimension = 2
+
+    emb.encode_query(["weather"])
+    emb.encode_documents(["weather"])
+
+    assert tokenizer.calls[0]["texts"] == ["query: weather"]
+    assert tokenizer.calls[1]["texts"] == ["passage: weather"]
+
+
+@pytest.mark.asyncio
+async def test_onnx_embeddings_dimension_mismatch_raises_value_error():
+    emb = OnnxEmbeddings(
+        model_id="intfloat/multilingual-e5-small",
+        model_path="/models/e5/onnx/model.onnx",
+        tokenizer_name_or_path="/models/e5",
+        dimensions=3,
+    )
+    fake_transformers = SimpleNamespace(AutoTokenizer=SimpleNamespace(from_pretrained=MagicMock(return_value=FakeTokenizer())))
+    fake_onnxruntime = SimpleNamespace(InferenceSession=MagicMock(return_value=FakeOnnxSession()))
+
+    with patch.dict(sys.modules, {"transformers": fake_transformers, "onnxruntime": fake_onnxruntime}):
+        with pytest.raises(ValueError, match="does not match model output"):
+            await emb.initialize()
+
+
+@pytest.mark.asyncio
+async def test_onnx_embeddings_downloads_external_data_sidecar_when_needed():
+    emb = OnnxEmbeddings(model_id="BAAI/bge-m3", onnx_file="onnx/model.onnx")
+    download = MagicMock(return_value="/hf/bge-m3")
+    session = MagicMock(return_value=FakeOnnxSession())
+    fake_hf = SimpleNamespace(snapshot_download=download)
+    fake_transformers = SimpleNamespace(AutoTokenizer=SimpleNamespace(from_pretrained=MagicMock(return_value=FakeTokenizer())))
+    fake_onnxruntime = SimpleNamespace(InferenceSession=session)
+
+    with patch.dict(
+        sys.modules,
+        {
+            "huggingface_hub": fake_hf,
+            "transformers": fake_transformers,
+            "onnxruntime": fake_onnxruntime,
+        },
+    ):
+        await emb.initialize()
+
+    download.assert_called_once_with(
+        repo_id="BAAI/bge-m3",
+        allow_patterns=["onnx/model.onnx", "onnx/model.onnx_data"],
+    )
+    session.assert_called_once_with("/hf/bge-m3/onnx/model.onnx", providers=["CPUExecutionProvider"])
+
+
+def test_create_embeddings_from_env_supports_onnx_provider():
+    mock_config = MagicMock()
+    mock_config.embeddings_provider = "onnx"
+    mock_config.embeddings_onnx_model_id = "intfloat/multilingual-e5-small"
+    mock_config.embeddings_onnx_model_path = "/models/e5/onnx/model.onnx"
+    mock_config.embeddings_onnx_tokenizer_name_or_path = "/models/e5"
+    mock_config.embeddings_onnx_file = "onnx/model.onnx"
+    mock_config.embeddings_onnx_dimensions = 384
+    mock_config.embeddings_onnx_max_tokens = 512
+    mock_config.embeddings_onnx_pooling = "mean"
+    mock_config.embeddings_onnx_normalize = True
+    mock_config.embeddings_onnx_query_prefix = "query: "
+    mock_config.embeddings_onnx_passage_prefix = "passage: "
+    mock_config.embeddings_onnx_output_name = None
+
+    with patch("hindsight_api.config.get_config", return_value=mock_config):
+        emb = create_embeddings_from_env()
+
+    assert isinstance(emb, OnnxEmbeddings)
+    assert emb.provider_name == "onnx"
+    assert emb.model_id == "intfloat/multilingual-e5-small"
+    assert emb.model_path == "/models/e5/onnx/model.onnx"
+    assert emb.tokenizer_name_or_path == "/models/e5"
+    assert emb.dimension == 384

hindsight-docs/docs/developer/configuration.md

@@ -459,10 +459,21 @@ two slots that retain/consolidation cannot consume.
 
 | Variable | Description | Default |
 |----------|-------------|---------|
-| `HINDSIGHT_API_EMBEDDINGS_PROVIDER` | Provider: `local`, `tei`, `openai`, `openai-codex`, `openrouter`, `cohere`, `google`, `zeroentropy`, `litellm`, or `litellm-sdk` | `local` |
+| `HINDSIGHT_API_EMBEDDINGS_PROVIDER` | Provider: `local`, `onnx`, `tei`, `openai`, `openai-codex`, `openrouter`, `cohere`, `google`, `zeroentropy`, `litellm`, or `litellm-sdk` | `local` |
 | `HINDSIGHT_API_EMBEDDINGS_LOCAL_MODEL` | Model for local provider | `BAAI/bge-small-en-v1.5` |
 | `HINDSIGHT_API_EMBEDDINGS_LOCAL_TRUST_REMOTE_CODE` | Allow loading models with custom code (security risk, disabled by default) | `false` |
 | `HINDSIGHT_API_EMBEDDINGS_LOCAL_FORCE_CPU` | Force CPU mode for local embeddings (avoids MPS/XPC issues on macOS) | `false` |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID` | Hugging Face model repo for the ONNX provider. Used for auto-download and as the tokenizer fallback. | `intfloat/multilingual-e5-small` |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_PATH` | Local path to the ONNX graph. When unset, Hindsight downloads `HINDSIGHT_API_EMBEDDINGS_ONNX_FILE` from `HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID`. | - |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_TOKENIZER_NAME_OR_PATH` | Hugging Face tokenizer repo or local tokenizer directory. Set this when using `HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_PATH`. | Falls back to `HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID` |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_FILE` | ONNX file path inside the Hugging Face repo. Hindsight also downloads the conventional external-data sidecar with `_data` suffix when present. | `onnx/model.onnx` |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_DIMENSIONS` | Expected embedding dimensions. Startup fails if the loaded model returns a different size. | Auto-detected |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_MAX_TOKENS` | Max tokenizer length for ONNX embeddings. | `512` |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_POOLING` | Pooling strategy for token embeddings: `mean` or `cls`. Ignored when the ONNX graph returns a pre-pooled 2-D embedding output. | `mean` |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_NORMALIZE` | L2-normalize ONNX vectors before storage. | `true` |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_QUERY_PREFIX` | Prefix applied to query/search text before ONNX embedding. Keep `query: ` for E5 models; set to empty for non-E5 models such as MiniLM or BGE. | `query: ` |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_PASSAGE_PREFIX` | Prefix applied to stored memory/document text before ONNX embedding. Keep `passage: ` for E5 models; set to empty for non-E5 models such as MiniLM or BGE. | `passage: ` |
+| `HINDSIGHT_API_EMBEDDINGS_ONNX_OUTPUT_NAME` | Optional ONNX output name to request when an exported graph exposes a pooled embedding output. | - |
 | `HINDSIGHT_API_EMBEDDINGS_TEI_URL` | TEI server URL | - |
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY` | OpenAI API key (falls back to `HINDSIGHT_API_LLM_API_KEY`) | - |
 | `HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL` | OpenAI embedding model | `text-embedding-3-small` |
@@ -498,7 +509,107 @@ two slots that retain/consolidation cannot consume.
 | `HINDSIGHT_API_EMBEDDINGS_VERTEXAI_REGION` | Vertex AI region for embeddings (falls back to `HINDSIGHT_API_LLM_VERTEXAI_REGION`) | - |
 | `HINDSIGHT_API_EMBEDDINGS_VERTEXAI_SERVICE_ACCOUNT_KEY` | Service account key for Vertex AI embeddings (falls back to `HINDSIGHT_API_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY`) | - |
 
-Embedding provider selection, credentials, base URLs, model choices, dimensions, encoding format, batch sizes, and latency modes are static server-level settings. They are not hierarchical per-bank overrides.
+Embedding provider selection, credentials, base URLs, model choices, dimensions, encoding format, batch sizes, and latency modes are static server-level settings. They are not hierarchical per-bank overrides. The ONNX settings above are also static, matching the existing `embeddings_local_*` settings.
+
+#### Local ONNX embeddings
+
+The ONNX provider runs embedding models in-process with ONNX Runtime. Install the optional deps when building your own API environment:
+
+```bash
+pip install 'hindsight-api-slim[local-onnx]'
+# or, in this repository:
+uv sync --project hindsight-api-slim --extra local-onnx
+```
+
+You can either let Hindsight download the model from Hugging Face at startup by setting `HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID`, or pre-download the ONNX graph and tokenizer files under the Hindsight repository root.
+
+```bash
+cd /path/to/hindsight
+mkdir -p models
+
+MODEL_ID=intfloat/multilingual-e5-small
+MODEL_DIR=models/intfloat__multilingual-e5-small
+
+uv run --project hindsight-api-slim --extra local-onnx python - <<'PY'
+import os
+from huggingface_hub import snapshot_download
+
+snapshot_download(
+    repo_id=os.environ["MODEL_ID"],
+    local_dir=os.environ["MODEL_DIR"],
+    allow_patterns=[
+        "onnx/model.onnx",
+        "onnx/model.onnx_data",
+        "*.json",
+        "*.txt",
+        "*.model",
+    ],
+)
+PY
+```
+
+Then start Hindsight with paths relative to the repository root:
+
+```bash
+export HINDSIGHT_API_EMBEDDINGS_PROVIDER=onnx
+export HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_PATH=./models/intfloat__multilingual-e5-small/onnx/model.onnx
+export HINDSIGHT_API_EMBEDDINGS_ONNX_TOKENIZER_NAME_OR_PATH=./models/intfloat__multilingual-e5-small
+export HINDSIGHT_API_EMBEDDINGS_ONNX_DIMENSIONS=384
+export HINDSIGHT_API_EMBEDDINGS_ONNX_QUERY_PREFIX="query: "
+export HINDSIGHT_API_EMBEDDINGS_ONNX_PASSAGE_PREFIX="passage: "
+```
+
+For Docker deployments, mount the same model directory and use container paths:
+
+```yaml
+services:
+  hindsight:
+    volumes:
+      - ./models:/app/models:ro
+    environment:
+      HINDSIGHT_API_EMBEDDINGS_PROVIDER: onnx
+      HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_PATH: /app/models/intfloat__multilingual-e5-small/onnx/model.onnx
+      HINDSIGHT_API_EMBEDDINGS_ONNX_TOKENIZER_NAME_OR_PATH: /app/models/intfloat__multilingual-e5-small
+      HINDSIGHT_API_EMBEDDINGS_ONNX_DIMENSIONS: "384"
+      HINDSIGHT_API_EMBEDDINGS_ONNX_QUERY_PREFIX: "query: "
+      HINDSIGHT_API_EMBEDDINGS_ONNX_PASSAGE_PREFIX: "passage: "
+```
+
+Model-specific examples:
+
+```bash
+# sentence-transformers/all-MiniLM-L6-v2: 384 dimensions, no E5 prefixes
+export HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID=sentence-transformers/all-MiniLM-L6-v2
+export HINDSIGHT_API_EMBEDDINGS_ONNX_DIMENSIONS=384
+export HINDSIGHT_API_EMBEDDINGS_ONNX_QUERY_PREFIX=""
+export HINDSIGHT_API_EMBEDDINGS_ONNX_PASSAGE_PREFIX=""
+
+# intfloat/multilingual-e5-small: 384 dimensions, keep E5 prefixes
+export HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID=intfloat/multilingual-e5-small
+export HINDSIGHT_API_EMBEDDINGS_ONNX_DIMENSIONS=384
+export HINDSIGHT_API_EMBEDDINGS_ONNX_QUERY_PREFIX="query: "
+export HINDSIGHT_API_EMBEDDINGS_ONNX_PASSAGE_PREFIX="passage: "
+
+# sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2: 384 dimensions, no E5 prefixes
+export HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+export HINDSIGHT_API_EMBEDDINGS_ONNX_DIMENSIONS=384
+export HINDSIGHT_API_EMBEDDINGS_ONNX_QUERY_PREFIX=""
+export HINDSIGHT_API_EMBEDDINGS_ONNX_PASSAGE_PREFIX=""
+
+# BAAI/bge-m3: 1024 dimensions, no E5 prefixes; keep onnx/model.onnx_data next to model.onnx
+export HINDSIGHT_API_EMBEDDINGS_ONNX_MODEL_ID=BAAI/bge-m3
+export HINDSIGHT_API_EMBEDDINGS_ONNX_DIMENSIONS=1024
+export HINDSIGHT_API_EMBEDDINGS_ONNX_QUERY_PREFIX=""
+export HINDSIGHT_API_EMBEDDINGS_ONNX_PASSAGE_PREFIX=""
+```
+
+:::warning
+Do not mix embeddings from different models in the same vector index. Switching from `local` to `onnx`, or changing ONNX models, requires re-embedding existing memories/documents even when the vector dimensions happen to match. For example, `BAAI/bge-small-en-v1.5` and `intfloat/multilingual-e5-small` both produce 384-dimensional vectors, but their embedding spaces are not semantically comparable.
+:::
+
+:::warning
+The default ONNX query/document prefixes (`query: ` and `passage: `) are for E5 models. Clear both prefix variables for non-E5 models such as MiniLM or BGE, otherwise Hindsight will prepend E5-style text to models that were not trained with that format.
+:::
 
 #### Common Pitfall: Provider-Specific Embedding Env Var Names
 

uv.lock

@@ -1722,8 +1722,11 @@ dependencies = [
 all = [
     { name = "einops" },
     { name = "flashrank" },
+    { name = "huggingface-hub" },
     { name = "mlx", marker = "sys_platform != 'win32'" },
     { name = "mlx-lm", marker = "sys_platform != 'win32'" },
+    { name = "numpy" },
+    { name = "onnxruntime" },
     { name = "pg0-embedded" },
     { name = "safetensors" },
     { name = "sentence-transformers" },
@@ -1749,6 +1752,12 @@ local-ml = [
     { name = "torch", version = "2.10.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" },
     { name = "transformers" },
 ]
+local-onnx = [
+    { name = "huggingface-hub" },
+    { name = "numpy" },
+    { name = "onnxruntime" },
+    { name = "transformers" },
+]
 oracle = [
     { name = "oracledb" },
 ]
@@ -1796,9 +1805,10 @@ requires-dist = [
     { name = "google-auth", specifier = ">=2.0.0" },
     { name = "google-genai", specifier = ">=1.0.0" },
     { name = "greenlet", specifier = ">=3.2.4,<3.4.0" },
-    { name = "hindsight-api-slim", extras = ["local-ml", "embedded-db"], marker = "extra == 'all'" },
+    { name = "hindsight-api-slim", extras = ["local-ml", "local-onnx", "embedded-db"], marker = "extra == 'all'" },
     { name = "httpx", specifier = ">=0.27.0" },
     { name = "huggingface-hub", marker = "extra == 'local-llm'", specifier = ">=0.20.0" },
+    { name = "huggingface-hub", marker = "extra == 'local-onnx'", specifier = ">=0.20.0" },
     { name = "langchain-core", specifier = ">=1.2.22" },
     { name = "langchain-text-splitters", specifier = ">=0.3.0" },
     { name = "langsmith", specifier = ">=0.6.3" },
@@ -1807,7 +1817,9 @@ requires-dist = [
     { name = "markitdown", extras = ["pdf", "docx", "pptx", "xlsx", "xls"], specifier = ">=0.1.4" },
     { name = "mlx", marker = "sys_platform != 'win32' and extra == 'local-ml'", specifier = ">=0.31.0" },
     { name = "mlx-lm", marker = "sys_platform != 'win32' and extra == 'local-ml'", specifier = ">=0.31.1" },
+    { name = "numpy", marker = "extra == 'local-onnx'", specifier = ">=1.26.0" },
     { name = "obstore", specifier = ">=0.4.0" },
+    { name = "onnxruntime", marker = "extra == 'local-onnx'", specifier = ">=1.17.0" },
     { name = "openai", specifier = ">=1.0.0" },
     { name = "opentelemetry-api", specifier = ">=1.41.0" },
     { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.41.0" },
@@ -1843,6 +1855,7 @@ requires-dist = [
     { name = "torch", marker = "extra == 'local-ml'", specifier = ">=2.6.0", index = "https://download.pytorch.org/whl/cpu" },
     { name = "tornado", specifier = ">=6.5.5" },
     { name = "transformers", marker = "extra == 'local-ml'", specifier = ">=4.53.0" },
+    { name = "transformers", marker = "extra == 'local-onnx'", specifier = ">=4.53.0" },
     { name = "typer", specifier = ">=0.9.0" },
     { name = "urllib3", specifier = ">=2.7.0" },
     { name = "uvicorn", specifier = ">=0.38.0" },
@@ -1850,7 +1863,7 @@ requires-dist = [
     { name = "winloop", marker = "sys_platform == 'win32'", specifier = ">=0.1.0" },
     { name = "wsproto", specifier = ">=1.0.0" },
 ]
-provides-extras = ["local-ml", "local-llm", "embedded-db", "oracle", "all", "test"]
+provides-extras = ["local-ml", "local-llm", "local-onnx", "embedded-db", "oracle", "all", "test"]
 
 [package.metadata.requires-dev]
 dev = [