chore: promote main to staging (progressive visual ingestion)

AGENTS.md

@@ -535,7 +535,7 @@ Core retrieval internals are grouped by ownership:
 - `execution/`: request shaping, route selection (classic / agent_explore / small_corpus), and public response projection.
 - `search/`: `map_unit_discovery` (persisted map-unit BM25 discovery; incomplete or incompatible indexes raise), scoring, section filters, candidate ranking.
 - `hydration/`: row/path/reference hydration, inline assets, and result assembly.
-- `agent_explore/` + `agent_tools/`: agent retrieval when `use_agentic` is on. Harness is `cursor_sdk` (Cursor) or `openai` (Knowhere's own). Map-nav is archived under `deprecated/mapnav/`.
+- `agent_explore/` + `agent_tools/`: agent retrieval when `use_agentic` is on. Harness is `cursor_sdk` (Cursor) or `openai` (Knowhere's own).
 - `trace/`: `DecisionTraceStep` mapping and `TraceRecorder`.
 - `graph/`: document graph publication/query support.
 - `stats/`: retrieval hit recording.
@@ -571,8 +571,7 @@ There is no scored term channel in this primary path. Chunk
 
 Default agentic path is `agent_explore`: a `corpus.*` tool loop run by the
 resolved harness (`AGENT_EXPLORE_HARNESS`, default `cursor_sdk`). Finish refs
-are resolved to chunks, then assembled like classic. Map-nav is archived at
-`deprecated/mapnav/` and is no longer a live route.
+are resolved to chunks, then assembled like classic.
 
 ### Result Assembly
 

CONTEXT.md

@@ -371,7 +371,7 @@ scope, filters, data type, ranking options, and agentic toggle.
 
 ### Retrieval Run
 
-One execution of a Retrieval Query, recorded across classic, map-nav,
+One execution of a Retrieval Query, recorded across classic, agent_explore,
 small-corpus, cache-hit, failed, and cancelled outcomes with route, timing,
 and terminal-status metadata.
 

apps/api/.env.example

@@ -85,6 +85,7 @@ GPT_API_KEY=
 GLM_API_KEY=
 ALI_API_KEYS=
 ARK_API_KEY=
+VERCEL_KEY=
 
 # Optional overrides:
 # DS_URL=https://api.deepseek.com/v1
@@ -97,6 +98,9 @@ ARK_API_KEY=
 # ASSET_MODEL is optional; empty follows IMAGE_MODEL.
 # IMAGE_MODEL=deepseek-flash
 # ASSET_MODEL=
+# Page-image and query embeddings use VERCEL_KEY.
+# EMBEDDING_MODEL=google/gemini-embedding-2
+# EMBEDDING_DIM=3072
 
 # Optional retrieval overrides have code defaults. Retrieval is evidence-only:
 # evidence is the primary output; evidence_text is deprecated and always empty;

apps/api/alembic/versions/0c1d2e3f4a5b_add_chunk_revision_section_order_index.py

@@ -1,4 +1,4 @@
-"""Add the index used by lazy map-nav section loads."""
+"""Add the index used by revision-section chunk loads."""
 
 from __future__ import annotations
 

apps/api/alembic/versions/1d2e3f4a5b6c_add_document_map_units.py

@@ -1,4 +1,4 @@
-"""Add revision-pinned map-nav score units."""
+"""Add revision-pinned map-unit score rows."""
 
 from __future__ import annotations
 

apps/api/alembic/versions/2e3f4a5b6c7d_add_map_unit_token_covering_index.py

@@ -1,4 +1,4 @@
-"""Add a covering index for persisted map-nav token lookups."""
+"""Add a covering index for persisted map-unit token lookups."""
 
 from __future__ import annotations
 

apps/api/alembic/versions/3f4a5b6c7d8e_add_section_snapshot_order_index.py

@@ -1,4 +1,4 @@
-"""Add the index used by lazy map-nav section pagination."""
+"""Add the index used by section snapshot-order pagination."""
 
 from __future__ import annotations
 

apps/api/alembic/versions/6f7a8b9c0d1e_add_document_scope_enrichments.py

@@ -0,0 +1,51 @@
+"""add document scope enrichments
+
+Revision ID: 6f7a8b9c0d1e
+Revises: 5e6f7a8b9c0d
+Create Date: 2026-10-26 09:30:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "6f7a8b9c0d1e"
+down_revision: str = "5e6f7a8b9c0d"
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "document_scope_enrichments",
+        sa.Column("id", sa.String(length=36), nullable=False),
+        sa.Column("document_id", sa.String(length=36), nullable=False),
+        sa.Column("job_result_id", sa.String(length=36), nullable=False),
+        sa.Column("scope_id", sa.String(length=64), nullable=False),
+        sa.Column("page_start", sa.Integer(), nullable=False),
+        sa.Column("page_end", sa.Integer(), nullable=False),
+        sa.Column("state", sa.String(length=32), nullable=False),
+        sa.Column("priority", sa.String(length=32), nullable=True),
+        sa.Column("attempts", sa.Integer(), nullable=False),
+        sa.Column("last_hit_at", sa.DateTime(), nullable=True),
+        sa.Column("refined_at", sa.DateTime(), nullable=True),
+        sa.Column("updated_at", sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id"], ["documents.document_id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["job_result_id"], ["job_results.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "job_result_id",
+            "scope_id",
+            name="uq_document_scope_enrichments_revision_scope",
+        ),
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("document_scope_enrichments")

apps/api/alembic/versions/7a8b9c0d1e2f_add_document_page_embeddings.py

@@ -0,0 +1,136 @@
+"""add document page embeddings
+
+Revision ID: 7a8b9c0d1e2f
+Revises: 6f7a8b9c0d1e
+Create Date: 2026-10-26 17:10:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+from pgvector.sqlalchemy import Vector
+
+revision: str = "7a8b9c0d1e2f"
+down_revision: str = "6f7a8b9c0d1e"
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+
+def upgrade() -> None:
+    op.execute("CREATE EXTENSION IF NOT EXISTS vector")
+
+    op.create_table(
+        "document_page_embeddings",
+        sa.Column("id", sa.String(length=36), nullable=False),
+        sa.Column("user_id", sa.Text(), nullable=False),
+        sa.Column("namespace", sa.String(length=255), nullable=False),
+        sa.Column("document_id", sa.String(length=36), nullable=False),
+        sa.Column("job_result_id", sa.String(length=36), nullable=False),
+        sa.Column("chunk_id", sa.String(length=64), nullable=False),
+        sa.Column("page_num", sa.Integer(), nullable=False),
+        sa.Column("modality", sa.String(length=16), nullable=False),
+        sa.Column("model", sa.String(length=64), nullable=False),
+        sa.Column("dims", sa.Integer(), nullable=False),
+        sa.Column("embedding", Vector(1024), nullable=False),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id"], ["documents.document_id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["job_result_id"], ["job_results.id"], ondelete="CASCADE"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "document_id",
+            "job_result_id",
+            "page_num",
+            name="uq_document_page_embeddings_revision_page",
+        ),
+    )
+    op.create_index(
+        "idx_document_page_embeddings_scope",
+        "document_page_embeddings",
+        ["user_id", "namespace", "job_result_id"],
+    )
+    op.create_index(
+        "idx_document_page_embeddings_revision",
+        "document_page_embeddings",
+        ["document_id", "job_result_id", "page_num"],
+    )
+    op.create_index(
+        "idx_document_page_embeddings_hnsw_cosine",
+        "document_page_embeddings",
+        ["embedding"],
+        postgresql_using="hnsw",
+        postgresql_ops={"embedding": "vector_cosine_ops"},
+    )
+
+    op.create_table(
+        "demo_document_page_embeddings",
+        sa.Column("id", sa.String(length=36), nullable=False),
+        sa.Column("user_id", sa.Text(), nullable=False),
+        sa.Column("namespace", sa.String(length=255), nullable=False),
+        sa.Column("document_id", sa.String(length=36), nullable=False),
+        sa.Column("job_result_id", sa.String(length=36), nullable=False),
+        sa.Column("chunk_id", sa.String(length=64), nullable=False),
+        sa.Column("page_num", sa.Integer(), nullable=False),
+        sa.Column("modality", sa.String(length=16), nullable=False),
+        sa.Column("model", sa.String(length=64), nullable=False),
+        sa.Column("dims", sa.Integer(), nullable=False),
+        sa.Column("embedding", Vector(1024), nullable=False),
+        sa.Column("created_at", sa.DateTime(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ["document_id"], ["demo_documents.document_id"], ondelete="CASCADE"
+        ),
+        sa.ForeignKeyConstraint(
+            ["job_result_id"], ["job_results.id"], ondelete="RESTRICT"
+        ),
+        sa.ForeignKeyConstraint(
+            ["job_result_id", "document_id"],
+            ["job_results.id", "job_results.demo_document_id"],
+            ondelete="RESTRICT",
+            name="fk_demo_documentpageembedding_revision",
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint(
+            "document_id",
+            "job_result_id",
+            "page_num",
+            name="uq_demo_document_page_embeddings_revision_page",
+        ),
+    )
+    op.create_index(
+        "idx_demo_document_page_embeddings_scope",
+        "demo_document_page_embeddings",
+        ["user_id", "namespace", "job_result_id"],
+    )
+    op.create_index(
+        "idx_demo_document_page_embeddings_revision",
+        "demo_document_page_embeddings",
+        ["document_id", "job_result_id", "page_num"],
+    )
+    op.create_index(
+        "idx_demo_document_page_embeddings_hnsw_cosine",
+        "demo_document_page_embeddings",
+        ["embedding"],
+        postgresql_using="hnsw",
+        postgresql_ops={"embedding": "vector_cosine_ops"},
+    )
+    op.execute("ALTER TABLE demo_document_page_embeddings ENABLE ROW LEVEL SECURITY")
+    op.execute("ALTER TABLE demo_document_page_embeddings FORCE ROW LEVEL SECURITY")
+    op.execute(
+        "CREATE POLICY demo_maintainer_write ON demo_document_page_embeddings "
+        "FOR ALL USING (COALESCE(current_setting('knowhere.demo_maintainer', true), '') <> '') "
+        "WITH CHECK (COALESCE(current_setting('knowhere.demo_maintainer', true), '') <> '')"
+    )
+    op.execute(
+        "CREATE POLICY demo_public_read ON demo_document_page_embeddings "
+        "FOR SELECT USING (is_demo_revision_ready(document_id, job_result_id))"
+    )
+
+
+def downgrade() -> None:
+    op.drop_table("demo_document_page_embeddings")
+    op.drop_table("document_page_embeddings")

apps/api/alembic/versions/8b9c0d1e2f3a_resize_page_embeddings_to_3072.py

@@ -0,0 +1,60 @@
+"""resize page embeddings to 3072
+
+Revision ID: 8b9c0d1e2f3a
+Revises: 7a8b9c0d1e2f
+Create Date: 2026-10-26 21:40:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "8b9c0d1e2f3a"
+down_revision: str = "7a8b9c0d1e2f"
+branch_labels: Sequence[str] | None = None
+depends_on: Sequence[str] | None = None
+
+_TABLES = (
+    "document_page_embeddings",
+    "demo_document_page_embeddings",
+)
+
+
+def _resize_embedding_column(
+    table_name: str,
+    *,
+    column_type: str,
+    index_ops: str,
+) -> None:
+    index_name = f"idx_{table_name}_hnsw_cosine"
+    op.drop_index(index_name, table_name=table_name)
+    op.execute(f"DELETE FROM {table_name}")
+    op.execute(
+        f"ALTER TABLE {table_name} ALTER COLUMN embedding TYPE {column_type}"
+    )
+    op.create_index(
+        index_name,
+        table_name,
+        ["embedding"],
+        postgresql_using="hnsw",
+        postgresql_ops={"embedding": index_ops},
+    )
+
+
+def upgrade() -> None:
+    for table_name in _TABLES:
+        _resize_embedding_column(
+            table_name,
+            column_type="halfvec(3072)",
+            index_ops="halfvec_cosine_ops",
+        )
+
+
+def downgrade() -> None:
+    for table_name in reversed(_TABLES):
+        _resize_embedding_column(
+            table_name,
+            column_type="vector(1024)",
+            index_ops="vector_cosine_ops",
+        )

apps/api/app/services/document_ingestion/command.py

@@ -5,7 +5,7 @@
 from dataclasses import dataclass
 from typing import Literal
 
-from shared.models.schemas.job import JobCreateBase
+from shared.models.schemas.job import IngestionMode, JobCreateBase
 from shared.models.schemas.page_memory_config import PageMemoryConfig
 
 ApiVersion = Literal["v1", "v2"]
@@ -20,6 +20,7 @@ class DocumentIngestionCommand:
     api_version: ApiVersion
     parse_track: ParseTrack
     processing_generation: ProcessingGeneration
+    ingestion_mode: IngestionMode = "full"
     page_memory_config: PageMemoryConfig | None = None
 
 
@@ -57,11 +58,13 @@ def _build_ingestion_command(
     page_memory_config = (
         PageMemoryConfig.default() if parse_track == "page_memory" else None
     )
+    ingestion_mode: IngestionMode = getattr(payload, "ingestion_mode", "full")
     return DocumentIngestionCommand(
         payload=payload,
         api_version=api_version,
         parse_track=parse_track,
         processing_generation=processing_generation,
+        ingestion_mode=ingestion_mode,
         page_memory_config=page_memory_config,
     )
 

apps/api/app/services/document_ingestion/service.py

@@ -122,6 +122,19 @@ async def _create_job_with_command_factory(
                 api_version=api_version,
             )
             command = build_command(file_extension)
+            if command.ingestion_mode == "progressive" and command.parse_track != "page_memory":
+                raise ValidationException(
+                    user_message="Progressive ingestion is only supported for page-memory formats (.pdf, .pptx) on /v2/jobs",
+                    violations=[
+                        {
+                            "field": "ingestion_mode",
+                            "description": (
+                                "ingestion_mode=progressive requires a page_memory "
+                                "document (.pdf, .pptx) submitted via /v2/jobs"
+                            ),
+                        }
+                    ],
+                )
             scope = await self._resolve_scope(
                 db,
                 command=command,
@@ -284,6 +297,7 @@ async def _resolve_scope(
                 parse_track=command.parse_track,
                 processing_generation=command.processing_generation,
                 page_memory_config=command.page_memory_config,
+                ingestion_mode=command.ingestion_mode,
             ),
         )
         if CorpusStorage.resolve_namespace(payload.namespace).is_demo:

apps/api/scripts/backfill_map_unit_indexes.py

@@ -1,4 +1,4 @@
-"""Backfill persisted MAP-NAV lexical indexes for existing revisions.
+"""Backfill persisted map-unit lexical indexes for existing revisions.
 
 Rebuilds, per active revision: the map-unit index, the revision serving
 manifest, that document's subtree in the namespace MAP snapshot, and the
@@ -88,7 +88,7 @@ def _bootstrap_python_path() -> None:
 
 def _build_parser() -> argparse.ArgumentParser:
     parser = argparse.ArgumentParser(
-        description="Backfill MAP-NAV indexes for current document revisions."
+        description="Backfill map-unit indexes for current document revisions."
     )
     parser.add_argument(
         "--apply",