Mvp/certification round 5 fact embedding

.env.example

@@ -9,12 +9,17 @@ MESA_MODEL_ENABLED=false
 MESA_EXTERNAL_PROVIDER_ENABLED=false
 MESA_V4_REBUILD_ENABLED=false
 MESA_EMBEDDING_VERSION=v1
-MESA_LLM_PROVIDER=openai_compatible
-LLM_EMBEDDING_MODEL=sentence-transformers/all-MiniLM-L6-v2
-MESA_EMBEDDING_DIMENSION=384
+MESA_LOCAL_EMBEDDING_MODEL=magibu/embeddingmagibu-200m
+MESA_EMBEDDING_DIMENSION=768
+MESA_EMBEDDING_NORMALIZED=true
+MESA_EXTRACTION_PROVIDER=ollama
+MESA_EXTRACTION_MODEL=qwen3:1.7b
+MESA_EXTRACTION_THINKING=false
 
 # External embedding profile example (OpenAI-compatible text-embedding-3-small)
-# LLM_EMBEDDING_MODEL=text-embedding-3-small
+# MESA_EXTERNAL_PROVIDER_ENABLED=true
+# MESA_EMBEDDING_PROVIDER=openai_compatible
+# MESA_EXTERNAL_EMBEDDING_MODEL=text-embedding-3-small
 # MESA_EMBEDDING_DIMENSION=1536
 # Validation Policy:
 # 0 = Deterministic validation only (zero validation LLMs required)

.github/workflows/benchmark-quality.yml

@@ -116,7 +116,7 @@ jobs:
         with:
           name: v4-rrf-ablation
           path: v4-rrf-ablation.json
-          if-no-files-found: error
+          if-no-files-found: warn
           retention-days: 30
 
   nightly-external-sync:

ARCHITECTURE.md

@@ -90,7 +90,7 @@ FTS5 provides **zero-VRAM lexical pre-filtering** — enabling fast keyword/pref
 
 ### 3.3 LanceDB — Vector Store
 
-Vector embeddings are stored in LanceDB with **multi-dimensional table routing**. Upon ingestion, the engine detects the embedding dimensionality and routes to a dedicated table (`mesa_vectors_384`, `mesa_vectors_1536`, etc.), preserving full semantic integrity across embedding providers.
+Vector embeddings are stored in LanceDB by the canonical `EmbeddingService`. An active projection generation is fenced to one full embedding-space identity (provider, model, revision, dimension, and normalization); a model change requires an explicit rebuild and atomic generation cutover.
 
 All LanceDB operations are offloaded from the event loop via `ThreadPoolExecutor` + `asyncio.run_in_executor()`. The vector engine supports:
 
@@ -231,9 +231,7 @@ The retriever (`mesa_memory/retriever.py`) implements bi-temporal awareness: mem
 
 ## 6. Multi-Dimensional Vector Routing
 
-MESA natively supports multi-model embedding pipelines (e.g., OpenAI `1536` dimensions, local MiniLM `384` dimensions). The `VectorEngine` utilizes mathematical projections like Procrustes rotation to dynamically align and isolate vector spaces.
-
-Upon ingestion, MESA analyzes the incoming tensor dimension and routes the vector to a dedicated, dimension-specific LanceDB table (e.g., `mesa_vectors_1536` or `mesa_vectors_384`). This ensures absolute semantic integrity while allowing real-time switching between cloud and local SLMs.
+MESA does not align or mix embedding spaces by dimension. `EmbeddingService` owns document/query generation and truthful identity; `VectorEngine` only stores and searches vectors. Changing local Magibu 768D to another 768D model is still an incompatible migration, not a live switch.
 
 ---
 

docker-compose.v4.yml

@@ -27,11 +27,17 @@ x-mesa-v4-runtime: &mesa-v4-runtime
     MESA_LOAD_DOTENV: "false"
     MESA_MODEL_ENABLED: ${MESA_MODEL_ENABLED:?MESA_MODEL_ENABLED must be explicitly enabled for v4}
     MESA_EXTERNAL_PROVIDER_ENABLED: ${MESA_EXTERNAL_PROVIDER_ENABLED:?MESA_EXTERNAL_PROVIDER_ENABLED must be explicit}
-    MESA_LLM_PROVIDER: ${MESA_LLM_PROVIDER:-openai_compatible}
     LLM_BASE_URL: ${LLM_BASE_URL:-}
     LLM_API_KEY: ${LLM_API_KEY:-}
     LLM_MODEL_NAME: ${LLM_MODEL_NAME:-llama-3.1-8b-instant}
-    LLM_EMBEDDING_MODEL: ${LLM_EMBEDDING_MODEL:-sentence-transformers/all-MiniLM-L6-v2}
+    MESA_LOCAL_EMBEDDING_MODEL: ${MESA_LOCAL_EMBEDDING_MODEL:-magibu/embeddingmagibu-200m}
+    MESA_EMBEDDING_DIMENSION: ${MESA_EMBEDDING_DIMENSION:-768}
+    MESA_EMBEDDING_NORMALIZED: ${MESA_EMBEDDING_NORMALIZED:-true}
+    MESA_EMBEDDING_PROVIDER: ${MESA_EMBEDDING_PROVIDER:-openai_compatible}
+    MESA_EXTERNAL_EMBEDDING_MODEL: ${MESA_EXTERNAL_EMBEDDING_MODEL:-text-embedding-3-small}
+    MESA_EXTRACTION_PROVIDER: ${MESA_EXTRACTION_PROVIDER:-ollama}
+    MESA_EXTRACTION_MODEL: ${MESA_EXTRACTION_MODEL:-qwen3:1.7b}
+    MESA_EXTRACTION_THINKING: ${MESA_EXTRACTION_THINKING:-false}
     MESA_TIER3_MODE: ${MESA_TIER3_MODE:-}
     MESA_TIER3_LLM_PROVIDER_A: ${MESA_TIER3_LLM_PROVIDER_A:-}
     MESA_TIER3_LLM_MODEL_A: ${MESA_TIER3_LLM_MODEL_A:-}

mesa_api/v4_router.py

@@ -953,6 +953,9 @@ async def insert_memory(
                 embedding_model=embedding_identity.model,
                 embedding_version=embedding_identity.version,
                 embedding_dimension=embedding_identity.dimension,
+                embedding_space_id=embedding_identity.embedding_space_id,
+                embedding_model_revision=embedding_identity.model_revision,
+                embedding_normalized=embedding_identity.normalized,
                 policy=config.queue_admission_policy,
                 validation_mode=(
                     validation_policy.mode

mesa_memory/adapter/live.py

@@ -179,11 +179,10 @@ def embed(self, text: str, **kwargs) -> list[float]:
                 input=text,
             )
             return response.data[0].embedding
-        except _OPENAI_NOT_FOUND_ERRORS:
-            logger.debug("Using local embedding fallback for Groq")
-            from mesa_memory.adapter.claude import _local_embed
-
-            return _local_embed(text)
+        except _OPENAI_NOT_FOUND_ERRORS as exc:
+            raise RuntimeError(
+                "configured external embedding model is unavailable"
+            ) from exc
         except _OPENAI_RATE_LIMIT_ERRORS as e:
             logger.error("Rate limit error during embedding: %s", e)
             raise
@@ -204,16 +203,10 @@ async def aembed(self, text: str, **kwargs) -> list[float]:
                 input=text,
             )
             return response.data[0].embedding
-        except _OPENAI_NOT_FOUND_ERRORS:
-            import asyncio
-            import functools
-
-            from mesa_memory.adapter.claude import _local_embed
-
-            loop = asyncio.get_running_loop()
-            return await loop.run_in_executor(
-                None, functools.partial(_local_embed, text)
-            )
+        except _OPENAI_NOT_FOUND_ERRORS as exc:
+            raise RuntimeError(
+                "configured external embedding model is unavailable"
+            ) from exc
         except _OPENAI_RATE_LIMIT_ERRORS as e:
             logger.error("Rate limit error during async embedding: %s", e)
             raise
@@ -236,10 +229,10 @@ def embed_batch(self, texts: list[str], **kwargs) -> list[list[float]]:  # type:
             return [
                 data.embedding for data in sorted(response.data, key=lambda x: x.index)
             ]
-        except _OPENAI_NOT_FOUND_ERRORS:
-            from mesa_memory.adapter.claude import _local_embed_batch
-
-            return _local_embed_batch(texts)
+        except _OPENAI_NOT_FOUND_ERRORS as exc:
+            raise RuntimeError(
+                "configured external embedding model is unavailable"
+            ) from exc
         except _OPENAI_RATE_LIMIT_ERRORS as e:
             logger.error("Rate limit error during batch embedding: %s", e)
             raise
@@ -262,16 +255,10 @@ async def aembed_batch(self, texts: list[str], **kwargs) -> list[list[float]]:
             return [
                 data.embedding for data in sorted(response.data, key=lambda x: x.index)
             ]
-        except _OPENAI_NOT_FOUND_ERRORS:
-            import asyncio
-            import functools
-
-            from mesa_memory.adapter.claude import _local_embed_batch
-
-            loop = asyncio.get_running_loop()
-            return await loop.run_in_executor(
-                None, functools.partial(_local_embed_batch, texts)
-            )
+        except _OPENAI_NOT_FOUND_ERRORS as exc:
+            raise RuntimeError(
+                "configured external embedding model is unavailable"
+            ) from exc
         except _OPENAI_RATE_LIMIT_ERRORS as e:
             logger.error("Rate limit error during async batch embedding: %s", e)
             raise

mesa_memory/api/server.py

@@ -331,6 +331,9 @@ async def _runtime_lifespan(app: FastAPI, runtime: RuntimeProfileConfig):
         allow_model_loading=runtime.model_enabled,
         force_refresh=True,
     )
+    await generation_repository.assert_active_embedding_identity(
+        embedding_service.identity().as_dict()
+    )
     state.vector_engine = VectorEngine(
         uri=str(_VECTOR_PATH),
         max_workers=config.vector_worker_limit,

mesa_memory/config.py

@@ -390,6 +390,12 @@ class MesaConfig(BaseSettings):
     llm_embedding_model_name: str = Field(
         "sentence-transformers/all-MiniLM-L6-v2", validation_alias="LLM_EMBEDDING_MODEL"
     )
+    embedding_provider: str = Field(
+        "openai_compatible", validation_alias="MESA_EMBEDDING_PROVIDER"
+    )
+    external_embedding_model: str = Field(
+        "text-embedding-3-small", validation_alias="MESA_EXTERNAL_EMBEDDING_MODEL"
+    )
     llm_timeout_seconds: float = Field(20.0, validation_alias="LLM_TIMEOUT_SECONDS")
     model_enabled: bool = Field(False, validation_alias="MESA_MODEL_ENABLED")
     tier3_mode: int | None = Field(None, validation_alias="MESA_TIER3_MODE")
@@ -580,8 +586,8 @@ def vector_worker_limit(self) -> int:
     # When MESA_ZERO_COST_MODE=true, the system reconfigures itself to use
     # exclusively local resources:
     #   - LLM provider  → OllamaAdapter (localhost:11434)
-    #   - Embeddings    → sentence-transformers/all-MiniLM-L6-v2 (local)
-    #   - REBEL         → enabled (local, zero API cost)
+    #   - Embeddings    → configured local EmbeddingService model (no fallback)
+    #   - REBEL         → disabled
     #   - Validation    → preserves the explicitly selected assurance mode;
     #                      Mode 2 still requires two distinct validators.
     # This mode requires Ollama to be running locally with a pulled model.
@@ -697,7 +703,8 @@ def validate_embedding_fallback(self) -> "MesaConfig":
 
             logging.getLogger("MESA_Config").warning(
                 "OPENAI_API_KEY not set for Claude provider. "
-                "Embeddings will use local model '%s' as fallback.",
+                "No external embedding fallback will be selected automatically; "
+                "the configured embedding provider remains authoritative (%s).",
                 self.local_embedding_model,
             )
         return self
@@ -850,9 +857,9 @@ def configured_embedding_identity(
         default=False,
     )
     return EmbeddingIdentity(
-        provider=config.mesa_llm_provider if external else "local",
+        provider=config.embedding_provider if external else "local",
         model=(
-            config.llm_embedding_model_name
+            config.external_embedding_model
             if external
             else config.local_embedding_model
         ),

mesa_memory/consolidation/schemas.py

@@ -50,6 +50,9 @@ class MemoryCandidate(BaseModel):
     embedding_model: str | None = None
     embedding_version: str | None = None
     embedding_dimension: int | None = Field(default=None, ge=1)
+    embedding_space_id: str | None = None
+    embedding_model_revision: str | None = None
+    embedding_normalized: bool | None = None
     created_artifact_ids: list[str] = Field(default_factory=list)
     validation_mode: int | None = None
     validation_policy: str | None = None
@@ -76,6 +79,9 @@ def from_raw_log(
         embedding_model: str | None = None,
         embedding_version: str | None = None,
         embedding_dimension: int | None = None,
+        embedding_space_id: str | None = None,
+        embedding_model_revision: str | None = None,
+        embedding_normalized: bool | None = None,
         validation_mode: int | None = None,
         validation_policy: str | None = None,
     ) -> "MemoryCandidate":
@@ -119,6 +125,9 @@ def from_raw_log(
             embedding_model=embedding_model,
             embedding_version=embedding_version,
             embedding_dimension=embedding_dimension,
+            embedding_space_id=embedding_space_id,
+            embedding_model_revision=embedding_model_revision,
+            embedding_normalized=embedding_normalized,
             validation_mode=mode,
             validation_policy=policy,
             pipeline_run_id=pipeline_run_id
@@ -127,6 +136,28 @@ def from_raw_log(
 
     def as_consolidation_record(self) -> dict[str, Any]:
         """Return the compatibility projection consumed by the existing loop."""
+        metadata = dict(self.metadata)
+        if (
+            self.embedding_provider
+            and self.embedding_model
+            and self.embedding_version
+            and self.embedding_dimension
+        ):
+            revision = self.embedding_model_revision or self.embedding_version
+            metadata["_mesa_embedding_identity"] = {
+                "embedding_space_id": self.embedding_space_id
+                or f"{self.embedding_provider}:{self.embedding_model}:{revision}:{self.embedding_dimension}:norm={str(bool(self.embedding_normalized if self.embedding_normalized is not None else True)).lower()}",
+                "provider": self.embedding_provider,
+                "model": self.embedding_model,
+                "model_revision": self.embedding_model_revision,
+                "version": self.embedding_version,
+                "dimension": self.embedding_dimension,
+                "normalized": (
+                    self.embedding_normalized
+                    if self.embedding_normalized is not None
+                    else True
+                ),
+            }
         return {
             "cmb_id": self.candidate_id,
             "candidate_id": self.candidate_id,
@@ -143,7 +174,7 @@ def as_consolidation_record(self) -> dict[str, Any]:
             "content_payload": self.content_payload,
             "source_ref": self.source_ref,
             "evidence_span": self.evidence_span,
-            "metadata": self.metadata,
+            "metadata": metadata,
             "source": self.source,
             "performative": self.performative,
             "pipeline_run_id": self.pipeline_run_id,
@@ -152,6 +183,9 @@ def as_consolidation_record(self) -> dict[str, Any]:
             "embedding_model": self.embedding_model,
             "embedding_version": self.embedding_version,
             "embedding_dimension": self.embedding_dimension,
+            "embedding_space_id": self.embedding_space_id,
+            "embedding_model_revision": self.embedding_model_revision,
+            "embedding_normalized": self.embedding_normalized,
             "created_artifact_ids": self.created_artifact_ids,
             "validation_mode": self.validation_mode,
             "validation_policy": self.validation_policy,