From 752b7db77a18dec5d741530da02a778a5c9adf7f Mon Sep 17 00:00:00 2001
From: chengke <404835780@qq.com>
Date: Fri, 16 Oct 2026 10:22:15 +0800
Subject: [PATCH 1/2] refactor: enhance asset connection handling and metadata
 structure

- Removed the `chunk_stats.json` reference from the documentation as it is no longer used.
- Updated the `importance` field in the metadata schema to improve clarity on its usage.
- Refactored the asset connection logic to streamline the addition of `embedded_by` relationships for images and tables.
- Introduced a new `asset_region` metadata structure to encapsulate asset positioning and dimensions.
- Adjusted tests to validate the new asset metadata and connection behaviors, ensuring consistency across chunk and zip formats.
---
 AGENTS.md                                     |  29 +---
 .../services/page_memory/node_assembler.py    |  16 +--
 .../app/services/page_memory/page_assets.py   |  17 ++-
 ...est_page_memory_node_assembler_contract.py |  15 +-
 .../test_page_memory_output_contract.py       | 133 ++++++++++++++++++
 .../test_table_embedded_images_contract.py    |   7 +
 .../services/chunks/chunk_connections.py      |  76 +++++++---
 .../chunks/dataframe_chunk_converter.py       |   7 +-
 .../services/retrieval/hydration/row_utils.py |   6 +
 .../services/storage/zip_chunk_schema.py      |  69 ++++++++-
 .../shared/tests/test_chunk_relations.py      | 133 ++++++++++++++++++
 11 files changed, 445 insertions(+), 63 deletions(-)
 create mode 100644 packages/shared-python/shared/tests/test_chunk_relations.py

diff --git a/AGENTS.md b/AGENTS.md
index 43f64487c..d3a5898d4 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -237,7 +237,6 @@ DataFrame is an internal detail. Below is the complete schema.
 ```text
 ~/.knowhere/{corpus_name}/
 ├── knowledge_graph.json           # corpus-wide graph: file metadata + cross-doc edges
-├── chunk_stats.json               # Per-chunk retrieval hit analytics {chunk_id → stats}
 ├── {source_file_name}/            # One directory per ingested document
 │   ├── chunks.json                # All parsed chunks for this document
 │   ├── doc_nav.json               # Hierarchical navigation tree for agentic retrieval
@@ -264,7 +263,6 @@ DataFrame is an internal detail. Below is the complete schema.
       "types": { "image": 13, "table": 1, "text": 141 },
       "top_keywords": ["model", "security", "ai", "operations", "artificial_intelligence"],
       "top_summary": "This document includes: Legal Notice, Foreword, 1. Overview, ...",
-      "importance": 0.3,
       "created_at": "2026-05-09T09:14:12.422208+00:00"
     }
   },
@@ -276,25 +274,8 @@ DataFrame is an internal detail. Below is the complete schema.
 |:---|:---|
 | `files.{name}.top_keywords` | TF-IDF top keywords across all chunks (used for cross-doc edge scoring) |
 | `files.{name}.top_summary` | Auto-generated outline of top-level headings (injected by `load_nav_top_summary()`) |
-| `files.{name}.importance` | Base importance score (feeds `compute_importance_score()` in ranking) |
 | `edges[]` | Cross-document edges with `{source, target, weight, shared_keywords}` when keyword overlap ≥ 0.8 |
 
-### `chunk_stats.json` — Retrieval Hit Analytics
-
-```json
-{
-  "2e2beffc-90b2-5429-8ee7-3c49260a1204": {
-    "hit_count": 0,
-    "first_hit": null,
-    "last_hit": null,
-    "created_at": "2026-05-09T09:14:12.423011+00:00"
-  }
-}
-```
-
-Keyed by `chunk_id`. `hit_count` and `last_hit` feed into `importance_norm_score`
-for retrieval ranking boost.
-
 ### `chunks.json` — Per-Document Chunk Records
 
 The core persisted data. Contains `{"chunks": [...]}` — an ordered array of
@@ -340,7 +321,10 @@ chunk objects. Three chunk types exist:
     "page_nums": [],
     "file_path": "images/image-1 ai_model.png",
     "keywords": [],
-    "tokens": []
+    "tokens": [],
+    "connect_to": [
+      { "target": "d88e4c47-3c48-5bdf-b849-693c00453021", "relation": "embedded_by" }
+    ]
   }
 }
 ```
@@ -378,9 +362,10 @@ chunk objects. Three chunk types exist:
 | `metadata.keywords` | `list[str]` | LLM-extracted keywords (semicolon-split from DataFrame) |
 | `metadata.page_nums` | `list[int]` | Source page numbers (PDF only) |
 | `metadata.file_path` | `str` | Relative asset path for images/tables |
-| `metadata.connect_to` | `list[ConnectionValue]` | Cross-chunk references (text→image/table embeddings) |
+| `metadata.connect_to` | `list[ConnectionValue]` | Cross-chunk relations (body↔image/table embeddings) |
 | `metadata.connect_to[].target` | `str` | Target chunk_id |
-| `metadata.connect_to[].relation` | `str` | `"embeds"` (inline asset) or `"related"` |
+| `metadata.connect_to[].target_document_id` | `str` | Reserved for cross-document targets; omitted for same-document edges |
+| `metadata.connect_to[].relation` | `str` | `"embeds"` (body → inline asset), `"embedded_by"` (asset → owning body), `"related"` (same-page body → asset); `"cross_doc_entity"` / `"intra_doc_ref"` are reserved and not generated yet |
 | `metadata.connect_to[].ref` | `str` | Original reference string: `"[images/image-1.png]"` |
 | `metadata.connect_to[].position` | `{start, end}` | Character offset of the reference in content |
 
diff --git a/apps/worker/app/services/page_memory/node_assembler.py b/apps/worker/app/services/page_memory/node_assembler.py
index c4dc46257..1a2da30a8 100644
--- a/apps/worker/app/services/page_memory/node_assembler.py
+++ b/apps/worker/app/services/page_memory/node_assembler.py
@@ -630,14 +630,14 @@ def _attach_asset_connections(
                 row = rows_by_path.get(leaf.section_path)
                 if row is None:
                     continue
-                connection: dict[str, Any] = {
-                    "target": uri,
-                    "relation": "related",
-                    "ref": ref,
-                }
-                if owner_leaf is not None:
-                    connection["same_as_owner"] = owner_leaf.section_path
-                _append_connect_to(row, connection)
+                _append_connect_to(
+                    row,
+                    {
+                        "target": uri,
+                        "relation": "related",
+                        "ref": ref,
+                    },
+                )
 
 
 def _append_connect_to(row: dict[str, Any], connection: dict[str, Any]) -> None:
diff --git a/apps/worker/app/services/page_memory/page_assets.py b/apps/worker/app/services/page_memory/page_assets.py
index 1379aaf5e..19c828510 100644
--- a/apps/worker/app/services/page_memory/page_assets.py
+++ b/apps/worker/app/services/page_memory/page_assets.py
@@ -23,6 +23,7 @@
 from shared.services.ai.prompt_service import build_prompt
 
 _GRID_SIZE = 1000
+_ASSET_CROP_MARGIN_PX = 4
 _VALID_KINDS = {"table", "figure"}
 _ASSET_ANNOTATE_DIR = "asset_annotate"
 
@@ -194,7 +195,7 @@ def crop_page_asset(
     asset: PageAsset,
     page_image_path: str,
     output_dir: str,
-    margin_px: int = 4,
+    margin_px: int = _ASSET_CROP_MARGIN_PX,
 ) -> PageAsset:
     image_dir = Path(output_dir) / "images"
     image_dir.mkdir(parents=True, exist_ok=True)
@@ -543,10 +544,18 @@ def build_asset_rows(
 
 
 def _asset_extra_metadata(asset: PageAsset) -> dict[str, Any]:
-    metadata: dict[str, Any] = {
-        "image_uri": asset.image_uri,
+    # bbox is the detected (VLM) box on the rendered page image, not the cropped box.
+    return {
+        "asset_region": {
+            "page_num": asset.page_index,
+            "bbox": list(asset.bbox_px),
+            "coord_origin": "top_left",
+            "coord_system": "page_image_px",
+            "page_width": asset.width_px,
+            "page_height": asset.height_px,
+            "crop_margin_px": _ASSET_CROP_MARGIN_PX,
+        }
     }
-    return metadata
 
 
 def asset_reference(asset: PageAsset) -> str:
diff --git a/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py b/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py
index 773f14a2e..d0300fb8e 100644
--- a/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py
+++ b/apps/worker/tests/contract/test_page_memory_node_assembler_contract.py
@@ -421,6 +421,16 @@ def test_build_node_rows_prepends_asset_rows_and_links_page_nodes() -> None:
     assert "asset_kind" not in rows[0]["extra_metadata"]
     assert "title" not in rows[0]["extra_metadata"]
     assert "html_uri" not in rows[0]["extra_metadata"]
+    assert "image_uri" not in rows[0]["extra_metadata"]
+    assert rows[0]["extra_metadata"]["asset_region"] == {
+        "page_num": 231,
+        "bbox": [10, 20, 200, 120],
+        "coord_origin": "top_left",
+        "coord_system": "page_image_px",
+        "page_width": 1000,
+        "page_height": 1400,
+        "crop_margin_px": 4,
+    }
 
     by_path = {row["path"]: row for row in rows}
     owner = by_path["demo.pdf/3 基本规定/3.1 职责"]
@@ -428,7 +438,7 @@ def test_build_node_rows_prepends_asset_rows_and_links_page_nodes() -> None:
     assert '"relation": "embeds"' in owner["connectto"]
     assert '"target": "tables/table_page_231_1.html"' in owner["connectto"]
     assert '"relation": "related"' in shared["connectto"]
-    assert '"same_as_owner": "demo.pdf/3 基本规定/3.1 职责"' in shared["connectto"]
+    assert "same_as_owner" not in shared["connectto"]
 
 
 def test_page_connectto_normalizes_to_asset_chunk_id() -> None:
@@ -481,3 +491,6 @@ def test_page_connectto_normalizes_to_asset_chunk_id() -> None:
             "ref": "[tables/table_page_1_1.html]",
         }
     ]
+    assert asset_chunk["metadata"]["connect_to"] == [
+        {"target": "page_1", "relation": "embedded_by"}
+    ]
diff --git a/apps/worker/tests/contract/test_page_memory_output_contract.py b/apps/worker/tests/contract/test_page_memory_output_contract.py
index f86eb7084..2c7889541 100644
--- a/apps/worker/tests/contract/test_page_memory_output_contract.py
+++ b/apps/worker/tests/contract/test_page_memory_output_contract.py
@@ -12,11 +12,144 @@
 os.environ.setdefault("S3_SECRET_ACCESS_KEY", "test")
 os.environ.setdefault("S3_TEMP_PATH", "/tmp")
 
+from app.services.document_parser.support.parser_rows import PARSER_ROW_COLUMNS
 from app.services.page_memory import memory_service
+from app.services.page_memory.page_assets import PageAsset, build_asset_rows
 from shared.services.chunks.dataframe_chunk_converter import dataframe_to_chunks
 from shared.services.storage.zip_chunk_schema import ZipChunkSchemaBuilder
 
 
+def _page_asset(*, kind: str, asset_index: int, **uris: str) -> PageAsset:
+    return PageAsset(
+        asset_id=f"asset_{kind}_{asset_index}",
+        page_index=5,
+        asset_index=asset_index,
+        kind=kind,
+        bbox_px=[10, 20, 200, 120],
+        width_px=1000,
+        height_px=1400,
+        width_pt=500,
+        height_pt=700,
+        title=f"{kind} title",
+        summary=f"{kind} summary",
+        keywords=["k1", "k2"],
+        entities=[{"text": "Toolformer", "type": "model"}],
+        **uris,
+    )
+
+
+def test_asset_region_and_metadata_match_between_chunks_and_zip() -> None:
+    figure = _page_asset(
+        kind="figure",
+        asset_index=1,
+        image_uri="images/image_page_5_figure_1.png",
+    )
+    table = _page_asset(
+        kind="table",
+        asset_index=2,
+        image_uri="images/image_page_5_table_2.png",
+        html_uri="tables/table_page_5_2.html",
+    )
+    rows = build_asset_rows({5: [figure, table]})
+    df = pd.DataFrame(rows, columns=pd.Index([*PARSER_ROW_COLUMNS, "extra_metadata"]))
+
+    chunks = dataframe_to_chunks(df)
+    formatted = ZipChunkSchemaBuilder().format_chunks(
+        chunks,
+        image_files_map={
+            "asset_figure_1": {"file_path": "images/image_page_5_figure_1.png"}
+        },
+        table_files_map={"asset_table_2": {"file_path": "tables/table_page_5_2.html"}},
+    )
+
+    chunk_by_id = {chunk["chunk_id"]: chunk["metadata"] for chunk in chunks}
+    zip_by_id = {chunk["chunk_id"]: chunk["metadata"] for chunk in formatted}
+    expected_region = {
+        "page_num": 5,
+        "bbox": [10, 20, 200, 120],
+        "coord_origin": "top_left",
+        "coord_system": "page_image_px",
+        "page_width": 1000,
+        "page_height": 1400,
+        "crop_margin_px": 4,
+    }
+    for asset_id in ("asset_figure_1", "asset_table_2"):
+        chunk_metadata = chunk_by_id[asset_id]
+        zip_metadata = zip_by_id[asset_id]
+        assert chunk_metadata["asset_region"] == expected_region
+        assert "image_uri" not in chunk_metadata
+        assert set(zip_metadata) <= set(chunk_metadata)
+        for key in (
+            "asset_region",
+            "entities",
+            "asset_title",
+            "tokens",
+            "keywords",
+            "summary",
+            "page_nums",
+            "length",
+        ):
+            assert zip_metadata[key] == chunk_metadata[key], key
+    assert zip_by_id["asset_figure_1"]["original_name"] == (
+        chunk_by_id["asset_figure_1"]["original_name"]
+    )
+    assert zip_by_id["asset_figure_1"]["entities"] == [
+        {"text": "Toolformer", "type": "model"}
+    ]
+
+
+def test_zip_chunk_schema_omits_malformed_asset_region() -> None:
+    chunks = [
+        {
+            "chunk_id": "asset_figure_1",
+            "type": "image",
+            "content": "figure",
+            "path": "images/a.png",
+            "metadata": {
+                "file_path": "images/a.png",
+                "asset_region": {"page_num": 5, "bbox": [1, 2, 3]},
+            },
+        }
+    ]
+
+    formatted = ZipChunkSchemaBuilder().format_chunks(
+        chunks,
+        image_files_map={"asset_figure_1": {"file_path": "images/a.png"}},
+        table_files_map={},
+    )
+
+    assert "asset_region" not in formatted[0]["metadata"]
+
+
+def test_zip_text_tokens_stay_a_list_when_empty() -> None:
+    chunks = dataframe_to_chunks(
+        pd.DataFrame(
+            [
+                {
+                    "content": "body",
+                    "path": "demo.pdf/Root",
+                    "type": "PTXT",
+                    "length": 4,
+                    "keywords": "",
+                    "summary": "",
+                    "know_id": "text-1",
+                    "tokens": "",
+                    "connectto": "",
+                    "addtime": "2026-06-11 00:00:00",
+                    "page_nums": "1",
+                }
+            ]
+        )
+    )
+
+    formatted = ZipChunkSchemaBuilder().format_chunks(
+        chunks, image_files_map={}, table_files_map={}
+    )
+
+    assert chunks[0]["metadata"]["tokens"] == []
+    assert formatted[0]["metadata"]["tokens"] == []
+
+
 def test_dataframe_converter_accepts_page_chunks_with_extra_metadata() -> None:
     df = pd.DataFrame(
         [
diff --git a/apps/worker/tests/contract/test_table_embedded_images_contract.py b/apps/worker/tests/contract/test_table_embedded_images_contract.py
index 79cff2e3e..145c15ad9 100644
--- a/apps/worker/tests/contract/test_table_embedded_images_contract.py
+++ b/apps/worker/tests/contract/test_table_embedded_images_contract.py
@@ -178,6 +178,13 @@ def test_table_embedded_images_are_extracted_rewritten_and_linked(
         for item in table_connect
     )
 
+    text_id = str(by_type["text"]["chunk_id"])
+    image_connect = by_type["image"]["metadata"]["connect_to"]
+    assert len(image_connect) == 2
+    assert all(item["relation"] == "embedded_by" for item in image_connect)
+    assert {item["target"] for item in image_connect} == {text_id, table_id}
+    assert {"target": text_id, "relation": "embedded_by"} in table_connect
+
 def test_table_embedded_images_skip_below_img_min_size(tmp_path: Path) -> None:
     output_dir = tmp_path / "doc"
     images_dir = output_dir / "images"
diff --git a/packages/shared-python/shared/services/chunks/chunk_connections.py b/packages/shared-python/shared/services/chunks/chunk_connections.py
index dcfa865ec..e8bc0d3d2 100644
--- a/packages/shared-python/shared/services/chunks/chunk_connections.py
+++ b/packages/shared-python/shared/services/chunks/chunk_connections.py
@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from collections.abc import Mapping, Sequence
-from typing import Any, TypeAlias, TypedDict
+from typing import Any, Final, TypeAlias, TypedDict
 
 from shared.utils.chunk_refs import ChunkRefSpan, extract_chunk_ref_spans
 
@@ -13,14 +13,27 @@ class PositionPayload(TypedDict):
     end: int
 
 
+RELATION_EMBEDS: Final = "embeds"
+RELATION_EMBEDDED_BY: Final = "embedded_by"
+RELATION_RELATED: Final = "related"
+# Reserved for future producers; no code path emits these yet.
+RELATION_CROSS_DOC_ENTITY: Final = "cross_doc_entity"
+RELATION_INTRA_DOC_REF: Final = "intra_doc_ref"
+
+# Relations whose targets are expanded when hydrating or deduplicating a
+# body chunk's connected assets. Reverse and future relations are excluded so
+# they stay purely additive for existing consumers.
+FORWARD_RELATIONS: Final = frozenset({RELATION_EMBEDS, RELATION_RELATED})
+
+_ASSET_CHUNK_TYPES: Final = frozenset({"image", "table"})
+
+
 class ConnectionPayload(TypedDict, total=False):
     target: str
+    target_document_id: str
     relation: str
     ref: str
     position: PositionPayload
-    score: float
-    keywords: list[str]
-    same_as_owner: str
 
 
 RelationshipRef: TypeAlias = str | ChunkRefSpan
@@ -97,7 +110,7 @@ def convert_refs_to_embed_connections(
 
         connection: ConnectionPayload = {
             "target": target_id,
-            "relation": "embeds",
+            "relation": RELATION_EMBEDS,
             "ref": ref_text,
         }
         if isinstance(start, int) and isinstance(end, int):
@@ -129,20 +142,14 @@ def normalize_connect_to_targets(
 
             normalized_item: ConnectionPayload = {
                 "target": normalized_target,
-                "relation": str(item.get("relation") or "related"),
+                "relation": str(item.get("relation") or RELATION_RELATED),
             }
-            score = item.get("score")
-            if isinstance(score, (int, float)):
-                normalized_item["score"] = float(score)
-            keywords = item.get("keywords")
-            if isinstance(keywords, list):
-                normalized_item["keywords"] = [str(keyword) for keyword in keywords]
+            target_document_id = item.get("target_document_id")
+            if target_document_id:
+                normalized_item["target_document_id"] = str(target_document_id)
             ref = item.get("ref")
             if ref:
                 normalized_item["ref"] = str(ref)
-            same_as_owner = item.get("same_as_owner")
-            if same_as_owner:
-                normalized_item["same_as_owner"] = str(same_as_owner)
             position = item.get("position")
             if isinstance(position, dict):
                 start = position.get("start")
@@ -158,9 +165,7 @@ def normalize_connect_to_targets(
             normalized.append(
                 {
                     "target": normalized_target,
-                    "relation": "related",
-                    "score": 1.0,
-                    "keywords": [],
+                    "relation": RELATION_RELATED,
                 }
             )
     return normalized
@@ -197,6 +202,37 @@ def merge_connections(
     return merged
 
 
+def attach_embedded_by_connections(chunks: Sequence[Mapping[str, Any]]) -> None:
+    """Give every embedded image/table an ``embedded_by`` edge to its owner.
+
+    Mutates each asset chunk's ``metadata["connect_to"]`` in place. ``chunks``
+    must already carry their final forward ``embeds`` edges, with targets
+    normalized to chunk ids.
+    """
+    assets_by_chunk_id: dict[str, list[Mapping[str, Any]]] = {}
+    for chunk in chunks:
+        if chunk.get("type") in _ASSET_CHUNK_TYPES:
+            assets_by_chunk_id.setdefault(str(chunk["chunk_id"]), []).append(chunk)
+
+    for owner in chunks:
+        owner_id = str(owner["chunk_id"])
+        for connection in owner["metadata"].get("connect_to") or []:
+            if (
+                not isinstance(connection, dict)
+                or connection.get("relation") != RELATION_EMBEDS
+            ):
+                continue
+            target_id = str(connection.get("target") or "")
+            if target_id == owner_id:
+                continue
+            for asset in assets_by_chunk_id.get(target_id, ()):
+                asset_metadata = asset["metadata"]
+                asset_metadata["connect_to"] = merge_connections(
+                    asset_metadata.get("connect_to") or [],
+                    [{"target": owner_id, "relation": RELATION_EMBEDDED_BY}],
+                )
+
+
 def _parse_type_relationship_refs(type_value: object) -> list[str]:
     if not isinstance(type_value, str) or "\n" not in type_value:
         return []
@@ -206,10 +242,10 @@ def _parse_type_relationship_refs(type_value: object) -> list[str]:
 
 def _get_connection_key(item: ConnectionValue) -> ConnectionKey:
     if not isinstance(item, dict):
-        return ("", "related", "")
+        return ("", RELATION_RELATED, "")
     return (
         str(item.get("target") or ""),
-        str(item.get("relation") or "related"),
+        str(item.get("relation") or RELATION_RELATED),
         str(item.get("ref") or ""),
     )
 
diff --git a/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py b/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py
index e5ad3931f..c7920a61d 100644
--- a/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py
+++ b/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py
@@ -12,8 +12,10 @@
 from loguru import logger
 
 from shared.services.chunks.chunk_connections import (
+    RELATION_RELATED,
     ConnectionValue,
     RelationshipRef,
+    attach_embedded_by_connections,
     build_resource_target_map,
     convert_refs_to_embed_connections,
     merge_connections,
@@ -203,9 +205,7 @@ def _parse_connect_to(value: object) -> list[ConnectionValue]:
     return [
         {
             "target": connects_text,
-            "relation": "related",
-            "score": 1.0,
-            "keywords": [],
+            "relation": RELATION_RELATED,
         }
     ]
 
@@ -384,6 +384,7 @@ def dataframe_to_chunks(df: _ParserDataFrame | None) -> list[Dict[str, JsonValue
                 resource_target_map,
             ),
         )
+    attach_embedded_by_connections(chunks)
 
     logger.debug(f"DataFrame conversion completed: chunk count={len(chunks)}")
     return cast(list[Dict[str, JsonValue]], chunks)
diff --git a/packages/shared-python/shared/services/retrieval/hydration/row_utils.py b/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
index eaff7cb70..f4707a060 100644
--- a/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
+++ b/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
@@ -1,5 +1,9 @@
 from __future__ import annotations
 
+from shared.services.chunks.chunk_connections import (
+    FORWARD_RELATIONS,
+    RELATION_RELATED,
+)
 from shared.services.retrieval.document_scope import DocumentScope
 
 from typing import Any
@@ -95,6 +99,8 @@ def iter_connected_target_ids(row: dict[str, Any]) -> list[str]:
     for item in metadata.get('connect_to') or []:
         if not isinstance(item, dict):
             continue
+        if str(item.get('relation') or RELATION_RELATED) not in FORWARD_RELATIONS:
+            continue
         target_id = str(item.get('target') or '').strip()
         if target_id:
             target_ids.append(target_id)
diff --git a/packages/shared-python/shared/services/storage/zip_chunk_schema.py b/packages/shared-python/shared/services/storage/zip_chunk_schema.py
index 7dc0be09e..6d3a93d85 100644
--- a/packages/shared-python/shared/services/storage/zip_chunk_schema.py
+++ b/packages/shared-python/shared/services/storage/zip_chunk_schema.py
@@ -79,8 +79,7 @@ def format_chunks(
             if chunk_type == "text":
                 metadata.update(
                     {
-                        "tokens": existing_metadata.get("tokens")
-                        or chunk.get("tokens", 0),
+                        "tokens": existing_metadata.get("tokens") or [],
                         "keywords": existing_metadata.get("keywords")
                         or chunk.get("keywords", []),
                         "connect_to": _build_embed_connect_to(
@@ -95,10 +94,17 @@ def format_chunks(
             elif chunk_type == "image":
                 if image_info:
                     metadata["file_path"] = image_info["file_path"]
+                if original_name := existing_metadata.get("original_name"):
+                    metadata["original_name"] = original_name
                 metadata["keywords"] = existing_metadata.get("keywords") or chunk.get(
                     "keywords", []
                 )
-                metadata["tokens"] = []
+                metadata["tokens"] = existing_metadata.get("tokens") or []
+                metadata["connect_to"] = existing_metadata.get("connect_to") or []
+                if asset_region := _normalize_asset_region(
+                    existing_metadata.get("asset_region")
+                ):
+                    metadata["asset_region"] = asset_region
             elif chunk_type == "table":
                 metadata["file_path"] = _resolve_table_file_path(
                     chunk=chunk,
@@ -110,7 +116,11 @@ def format_chunks(
                 metadata["keywords"] = existing_metadata.get("keywords") or chunk.get(
                     "keywords", []
                 )
-                metadata["tokens"] = []
+                metadata["tokens"] = existing_metadata.get("tokens") or []
+                if asset_region := _normalize_asset_region(
+                    existing_metadata.get("asset_region")
+                ):
+                    metadata["asset_region"] = asset_region
                 metadata["connect_to"] = _build_embed_connect_to(
                     chunk=chunk,
                     chunk_type_str=chunk_type_str,
@@ -153,6 +163,10 @@ def _base_chunk_metadata(
         "summary": existing_metadata.get("summary") or chunk.get("summary", ""),
         "page_nums": existing_metadata.get("page_nums", []),
     }
+    if entities := existing_metadata.get("entities"):
+        metadata["entities"] = entities
+    if asset_title := existing_metadata.get("asset_title"):
+        metadata["asset_title"] = asset_title
     return metadata
 
 
@@ -230,9 +244,54 @@ def _normalize_page_assets(value: Any) -> list[dict[str, Any]]:
     return assets
 
 
+def _normalize_asset_region(value: Any) -> dict[str, Any] | None:
+    if not isinstance(value, dict):
+        return None
+    page_num = _positive_int_or_none(value.get("page_num"))
+    bbox = _bbox_or_none(value.get("bbox"))
+    coord_origin = str(value.get("coord_origin") or "").strip()
+    coord_system = str(value.get("coord_system") or "").strip()
+    page_width = _positive_int_or_none(value.get("page_width"))
+    page_height = _positive_int_or_none(value.get("page_height"))
+    crop_margin_px = _non_negative_int_or_none(value.get("crop_margin_px"))
+    if (
+        page_num is None
+        or bbox is None
+        or not coord_origin
+        or not coord_system
+        or page_width is None
+        or page_height is None
+        or crop_margin_px is None
+    ):
+        return None
+    return {
+        "page_num": page_num,
+        "bbox": bbox,
+        "coord_origin": coord_origin,
+        "coord_system": coord_system,
+        "page_width": page_width,
+        "page_height": page_height,
+        "crop_margin_px": crop_margin_px,
+    }
+
+
+def _bbox_or_none(value: Any) -> list[int] | None:
+    if not isinstance(value, list) or len(value) != 4:
+        return None
+    try:
+        return [int(item) for item in value]
+    except (TypeError, ValueError):
+        return None
+
+
 def _positive_int_or_none(value: Any) -> int | None:
+    number = _non_negative_int_or_none(value)
+    return number if number is not None and number > 0 else None
+
+
+def _non_negative_int_or_none(value: Any) -> int | None:
     try:
         number = int(value)
     except (TypeError, ValueError):
         return None
-    return number if number > 0 else None
+    return number if number >= 0 else None
diff --git a/packages/shared-python/shared/tests/test_chunk_relations.py b/packages/shared-python/shared/tests/test_chunk_relations.py
new file mode 100644
index 000000000..80f5da14b
--- /dev/null
+++ b/packages/shared-python/shared/tests/test_chunk_relations.py
@@ -0,0 +1,133 @@
+"""Contract tests for the unified chunk relation edges (embeds / embedded_by)."""
+
+from __future__ import annotations
+
+import asyncio
+
+import pandas as pd
+
+from shared.services.chunks.dataframe_chunk_converter import dataframe_to_chunks
+from shared.services.retrieval.hydration.result_assembly import (
+    assemble_retrieval_results,
+)
+from shared.services.retrieval.hydration.row_utils import iter_connected_target_ids
+from shared.services.storage.zip_chunk_schema import ZipChunkSchemaBuilder
+
+_COLUMNS = [
+    "content",
+    "path",
+    "type",
+    "length",
+    "keywords",
+    "summary",
+    "know_id",
+    "tokens",
+    "connectto",
+    "addtime",
+    "page_nums",
+]
+
+
+def _row(*, know_id: str, path: str, chunk_type: str, content: str) -> list[object]:
+    return [content, path, chunk_type, len(content), "", "", know_id, "", "", "", ""]
+
+
+def _chunks_with_two_embeds_of_one_image() -> list[dict]:
+    df = pd.DataFrame(
+        [
+            _row(
+                know_id="img-1",
+                path="images/a.jpg",
+                chunk_type="image",
+                content="a chart",
+            ),
+            _row(
+                know_id="body-1",
+                path="doc.md/Intro",
+                chunk_type="PTXT",
+                content="before [images/a.jpg] middle [images/a.jpg] after",
+            ),
+        ],
+        columns=pd.Index(_COLUMNS),
+    )
+    return dataframe_to_chunks(df)  # pyright: ignore[reportArgumentType]
+
+
+def test_embedded_asset_points_back_to_owner_once() -> None:
+    chunks = {chunk["chunk_id"]: chunk for chunk in _chunks_with_two_embeds_of_one_image()}
+
+    assert [
+        item["relation"] for item in chunks["body-1"]["metadata"]["connect_to"]
+    ] == ["embeds", "embeds"]
+    assert chunks["img-1"]["metadata"]["connect_to"] == [
+        {"target": "body-1", "relation": "embedded_by"}
+    ]
+
+
+def test_zip_image_chunk_carries_the_same_connect_to_as_chunks() -> None:
+    chunks = _chunks_with_two_embeds_of_one_image()
+
+    formatted = ZipChunkSchemaBuilder().format_chunks(
+        chunks,
+        image_files_map={"img-1": {"file_path": "images/a.jpg"}},
+        table_files_map={},
+    )
+
+    zip_image = next(chunk for chunk in formatted if chunk["type"] == "image")
+    chunk_image = next(chunk for chunk in chunks if chunk["type"] == "image")
+    assert zip_image["metadata"]["connect_to"] == chunk_image["metadata"]["connect_to"]
+
+
+def test_iter_connected_target_ids_follows_only_forward_relations() -> None:
+    row = {
+        "chunk_metadata": {
+            "connect_to": [
+                {"target": "embedded", "relation": "embeds"},
+                {"target": "same-page", "relation": "related"},
+                {"target": "unlabeled"},
+                {"target": "owner", "relation": "embedded_by"},
+                {"target": "other-doc", "relation": "cross_doc_entity"},
+                {"target": "cited", "relation": "intra_doc_ref"},
+            ]
+        }
+    }
+
+    assert iter_connected_target_ids(row) == ["embedded", "same-page", "unlabeled"]
+
+
+def test_mutually_linked_body_and_asset_are_not_both_dropped() -> None:
+    rows = [
+        {
+            "document_id": "doc_a",
+            "chunk_id": "body-1",
+            "chunk_type": "text",
+            "content": "before [images/a.jpg] after",
+            "section_path": "Intro",
+            "sort_order": 1,
+            "chunk_metadata": {
+                "connect_to": [
+                    {"target": "img-1", "relation": "embeds", "ref": "[images/a.jpg]"}
+                ]
+            },
+        },
+        {
+            "document_id": "doc_a",
+            "chunk_id": "img-1",
+            "chunk_type": "image",
+            "content": "chart",
+            "file_path": "images/a.jpg",
+            "section_path": "Intro",
+            "sort_order": 2,
+            "chunk_metadata": {
+                "connect_to": [{"target": "body-1", "relation": "embedded_by"}]
+            },
+        },
+    ]
+
+    assembled = asyncio.run(
+        assemble_retrieval_results(
+            rows=rows, exclude_document_ids=[], exclude_sections=[]
+        )
+    )
+
+    assert [row["chunk_id"] for row in assembled] == ["body-1"]

From afa3c40c9ddf53e41a382def381b7fa033119fbb Mon Sep 17 00:00:00 2001
From: chengke <404835780@qq.com>
Date: Fri, 16 Oct 2026 10:34:54 +0800
Subject: [PATCH 2/2] feat: expose importance and prefer entities over legacy
 keywords/tokens

Wire retrieval importance into the public result projection, migrate
metadata consumers to entities with legacy fallbacks, and stop emitting
redundant metadata.tokens from chunk/ZIP payloads.

Co-authored-by: Cursor <cursoragent@cursor.com>
---
 AGENTS.md                                     |  2 +-
 .../test_page_memory_output_contract.py       |  7 +-
 .../chunks/dataframe_chunk_converter.py       | 23 -------
 .../retrieval/agent_tools/tools/assets.py     | 18 ++++-
 .../services/retrieval/graph/keywords.py      | 12 +++-
 .../services/retrieval/hydration/row_utils.py |  1 +
 .../services/retrieval/search/lexical_text.py | 27 +++++---
 .../services/retrieval/search/ranking.py      |  1 +
 .../services/storage/zip_chunk_schema.py      |  3 -
 .../shared/tests/test_chunk_relations.py      | 68 +++++++++++++++++++
 10 files changed, 118 insertions(+), 44 deletions(-)

diff --git a/AGENTS.md b/AGENTS.md
index d3a5898d4..9e82c3f5d 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -358,7 +358,7 @@ chunk objects. Three chunk types exist:
 | `path` | `str` | Hierarchical path: `{kb}/{file}/{section1}/{section2}/...` for text; `images/...` or `tables/...` for assets |
 | `metadata.length` | `int` | Character count of content |
 | `metadata.summary` | `str` | LLM summary (images/tables: `"image-N\n{description}"`) |
-| `metadata.tokens` | `list[str]` | Pre-tokenized Chinese terms for BM25 retrieval |
+| `metadata.entities` | `list[{text, type}]` | Typed entities extracted from content or asset VLM/LLM analysis |
 | `metadata.keywords` | `list[str]` | LLM-extracted keywords (semicolon-split from DataFrame) |
 | `metadata.page_nums` | `list[int]` | Source page numbers (PDF only) |
 | `metadata.file_path` | `str` | Relative asset path for images/tables |
diff --git a/apps/worker/tests/contract/test_page_memory_output_contract.py b/apps/worker/tests/contract/test_page_memory_output_contract.py
index 2c7889541..1f9a8801f 100644
--- a/apps/worker/tests/contract/test_page_memory_output_contract.py
+++ b/apps/worker/tests/contract/test_page_memory_output_contract.py
@@ -83,7 +83,6 @@ def test_asset_region_and_metadata_match_between_chunks_and_zip() -> None:
             "asset_region",
             "entities",
             "asset_title",
-            "tokens",
             "keywords",
             "summary",
             "page_nums",
@@ -121,7 +120,7 @@ def test_zip_chunk_schema_omits_malformed_asset_region() -> None:
     assert "asset_region" not in formatted[0]["metadata"]
 
 
-def test_zip_text_tokens_stay_a_list_when_empty() -> None:
+def test_zip_text_omits_legacy_tokens_field() -> None:
     chunks = dataframe_to_chunks(
         pd.DataFrame(
             [
@@ -146,8 +145,8 @@ def test_zip_text_tokens_stay_a_list_when_empty() -> None:
         chunks, image_files_map={}, table_files_map={}
     )
 
-    assert chunks[0]["metadata"]["tokens"] == []
-    assert formatted[0]["metadata"]["tokens"] == []
+    assert "tokens" not in chunks[0]["metadata"]
+    assert "tokens" not in formatted[0]["metadata"]
 
 
 def test_dataframe_converter_accepts_page_chunks_with_extra_metadata() -> None:
diff --git a/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py b/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py
index c7920a61d..8eb6fb760 100644
--- a/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py
+++ b/packages/shared-python/shared/services/chunks/dataframe_chunk_converter.py
@@ -49,7 +49,6 @@ class ChunkMetadata(TypedDict, total=False):
     asset_title: str
     summary: str
     length: int
-    tokens: list[str]
     connect_to: list[ConnectionValue]
     page_nums: list[int]
     file_path: str
@@ -138,26 +137,6 @@ def _safe_parse_entities(value: object) -> list[dict[str, str]]:
     return entities
 
 
-def _safe_parse_tokens(value: object) -> list[str]:
-    if _is_missing(value):
-        return []
-    token_text = str(value).strip()
-    if not token_text:
-        return []
-    if token_text.startswith("[") and token_text.endswith("]"):
-        inner_text = token_text[1:-1].strip()
-        if (inner_text.startswith("'") and inner_text.endswith("'")) or (
-            inner_text.startswith('"') and inner_text.endswith('"')
-        ):
-            inner_text = inner_text[1:-1]
-        token_text = inner_text
-    if ";" in token_text:
-        return [token.strip() for token in token_text.split(";") if token.strip()]
-    if "->" in token_text:
-        return [token.strip() for token in token_text.split("->") if token.strip()]
-    return []
-
-
 def _normalize_resource_ref(ref: RelationshipRef) -> str:
     if isinstance(ref, dict):
         ref_text = str(ref.get("ref") or "").strip()
@@ -324,8 +303,6 @@ def dataframe_to_chunks(df: _ParserDataFrame | None) -> list[Dict[str, JsonValue
         asset_title = str(row.get("asset_title", "") or "").strip()
         if asset_title:
             metadata["asset_title"] = asset_title
-        if chunk_type != "page":
-            metadata["tokens"] = _safe_parse_tokens(row.get("tokens"))
         _merge_extra_metadata(metadata, _parse_extra_metadata(row.get("extra_metadata")))
 
         if chunk_type == "image":
diff --git a/packages/shared-python/shared/services/retrieval/agent_tools/tools/assets.py b/packages/shared-python/shared/services/retrieval/agent_tools/tools/assets.py
index 83c8d9b7f..06d14eb90 100644
--- a/packages/shared-python/shared/services/retrieval/agent_tools/tools/assets.py
+++ b/packages/shared-python/shared/services/retrieval/agent_tools/tools/assets.py
@@ -153,10 +153,24 @@ async def _forward_search(
             continue
         metadata = chunk.chunk_metadata if isinstance(chunk.chunk_metadata, dict) else {}
         summary = str(metadata.get("summary") or "").strip()
-        keywords = metadata.get("keywords") or []
+        asset_title = str(metadata.get("asset_title") or "").strip()
+        entities = metadata.get("entities")
+        if isinstance(entities, list) and entities:
+            entity_terms = [
+                str(item.get("text", "")).strip()
+                for item in entities
+                if isinstance(item, dict) and str(item.get("text", "")).strip()
+            ]
+        else:
+            entity_terms = []
+        keywords = entity_terms or (metadata.get("keywords") or [])
         if query:
             haystack = " ".join(
-                [summary.lower(), " ".join(str(k).lower() for k in keywords)]
+                [
+                    summary.lower(),
+                    asset_title.lower(),
+                    " ".join(str(k).lower() for k in keywords),
+                ]
             )
             if query not in haystack:
                 continue
diff --git a/packages/shared-python/shared/services/retrieval/graph/keywords.py b/packages/shared-python/shared/services/retrieval/graph/keywords.py
index 5486769e1..7d0b6be53 100644
--- a/packages/shared-python/shared/services/retrieval/graph/keywords.py
+++ b/packages/shared-python/shared/services/retrieval/graph/keywords.py
@@ -19,10 +19,20 @@ def normalize_keyword(keyword: str) -> str:
 
 
 def extract_keywords_from_chunk_metadata(meta: dict) -> list[str]:
-    """Extract keywords from chunk metadata."""
+    """Extract keywords from chunk metadata, preferring typed entities (§4.4)."""
     if not isinstance(meta, dict):
         return []
 
+    entities = meta.get('entities')
+    if isinstance(entities, list) and entities:
+        entity_texts = [
+            str(item.get('text', '')).strip()
+            for item in entities
+            if isinstance(item, dict) and str(item.get('text', '')).strip()
+        ]
+        if entity_texts:
+            return entity_texts
+
     keywords = meta.get('keywords', [])
     if isinstance(keywords, list) and keywords:
         return [str(keyword) for keyword in keywords if keyword]
diff --git a/packages/shared-python/shared/services/retrieval/hydration/row_utils.py b/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
index f4707a060..b8f6b8d4c 100644
--- a/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
+++ b/packages/shared-python/shared/services/retrieval/hydration/row_utils.py
@@ -18,6 +18,7 @@
     'content',
     'content_source',
     'score',
+    'importance',
     'asset_url',
     'source_chunk_path',
     'file_path',
diff --git a/packages/shared-python/shared/services/retrieval/search/lexical_text.py b/packages/shared-python/shared/services/retrieval/search/lexical_text.py
index 176dbf746..fb1e7fcde 100644
--- a/packages/shared-python/shared/services/retrieval/search/lexical_text.py
+++ b/packages/shared-python/shared/services/retrieval/search/lexical_text.py
@@ -63,12 +63,14 @@ def build_content_lexical_text(chunk: dict[str, Any]) -> Optional[str]:
         return None
     metadata = chunk.get("metadata") or {}
     tokens = metadata.get("tokens") if isinstance(metadata, dict) else None
-    if isinstance(tokens, list):
+    if isinstance(tokens, list) and tokens:
         token_text = " ".join(
             str(token).strip() for token in tokens if str(token).strip()
         )
     else:
-        token_text = ""
+        token_text = " ".join(
+            PublicationPreparationCache.tokenize(content, dedupe=True)
+        )
     lexical_parts = [part for part in [content, token_text] if part]
     return "\n".join(lexical_parts) if lexical_parts else content
 
@@ -207,18 +209,23 @@ def _table_search_source_text(chunk: dict[str, Any]) -> str:
     if not isinstance(metadata, dict):
         metadata = {}
 
-    keyword_value = metadata.get("keywords") or chunk.get("keywords") or []
-    if isinstance(keyword_value, list):
-        keyword_text = " ".join(
-            str(keyword).strip()
-            for keyword in keyword_value
-            if str(keyword).strip()
-        )
+    entity_text = _chunk_entity_text(chunk)
+    if entity_text:
+        keyword_text = entity_text
     else:
-        keyword_text = str(keyword_value or "").replace(";", " ").strip()
+        keyword_value = metadata.get("keywords") or chunk.get("keywords") or []
+        if isinstance(keyword_value, list):
+            keyword_text = " ".join(
+                str(keyword).strip()
+                for keyword in keyword_value
+                if str(keyword).strip()
+            )
+        else:
+            keyword_text = str(keyword_value or "").replace(";", " ").strip()
 
     parts = [
         str(metadata.get("summary") or chunk.get("summary") or "").strip(),
+        str(metadata.get("asset_title") or chunk.get("asset_title") or "").strip(),
         keyword_text,
         str(metadata.get("caption") or chunk.get("caption") or "").strip(),
     ]
diff --git a/packages/shared-python/shared/services/retrieval/search/ranking.py b/packages/shared-python/shared/services/retrieval/search/ranking.py
index eb8d25105..7d26fa8c8 100644
--- a/packages/shared-python/shared/services/retrieval/search/ranking.py
+++ b/packages/shared-python/shared/services/retrieval/search/ranking.py
@@ -164,6 +164,7 @@ def rank_candidates_by_path(
             )
             or 0.0
         )
+        row["importance"] = row["importance_raw_score"]
     apply_importance_multiplier(list(merged.values()))
 
     has_agent_results = len(routed_rows) > 0
diff --git a/packages/shared-python/shared/services/storage/zip_chunk_schema.py b/packages/shared-python/shared/services/storage/zip_chunk_schema.py
index 6d3a93d85..0449671bd 100644
--- a/packages/shared-python/shared/services/storage/zip_chunk_schema.py
+++ b/packages/shared-python/shared/services/storage/zip_chunk_schema.py
@@ -79,7 +79,6 @@ def format_chunks(
             if chunk_type == "text":
                 metadata.update(
                     {
-                        "tokens": existing_metadata.get("tokens") or [],
                         "keywords": existing_metadata.get("keywords")
                         or chunk.get("keywords", []),
                         "connect_to": _build_embed_connect_to(
@@ -99,7 +98,6 @@ def format_chunks(
                 metadata["keywords"] = existing_metadata.get("keywords") or chunk.get(
                     "keywords", []
                 )
-                metadata["tokens"] = existing_metadata.get("tokens") or []
                 metadata["connect_to"] = existing_metadata.get("connect_to") or []
                 if asset_region := _normalize_asset_region(
                     existing_metadata.get("asset_region")
@@ -116,7 +114,6 @@ def format_chunks(
                 metadata["keywords"] = existing_metadata.get("keywords") or chunk.get(
                     "keywords", []
                 )
-                metadata["tokens"] = existing_metadata.get("tokens") or []
                 if asset_region := _normalize_asset_region(
                     existing_metadata.get("asset_region")
                 ):
diff --git a/packages/shared-python/shared/tests/test_chunk_relations.py b/packages/shared-python/shared/tests/test_chunk_relations.py
index 80f5da14b..e4b7581cf 100644
--- a/packages/shared-python/shared/tests/test_chunk_relations.py
+++ b/packages/shared-python/shared/tests/test_chunk_relations.py
@@ -7,10 +7,21 @@
 import pandas as pd
 
 from shared.services.chunks.dataframe_chunk_converter import dataframe_to_chunks
+from shared.services.retrieval.execution.response_projection import (
+    project_public_retrieval_response,
+)
+from shared.services.retrieval.graph.keywords import (
+    extract_keywords_from_chunk_metadata,
+)
 from shared.services.retrieval.hydration.result_assembly import (
     assemble_retrieval_results,
 )
 from shared.services.retrieval.hydration.row_utils import iter_connected_target_ids
+from shared.services.retrieval.search.lexical_text import (
+    build_content_lexical_text,
+    build_content_search_text,
+    build_term_search_text,
+)
 from shared.services.storage.zip_chunk_schema import ZipChunkSchemaBuilder
 
 _COLUMNS = [
@@ -131,3 +142,60 @@ def test_mutually_linked_body_and_asset_are_not_both_dropped() -> None:
     )
 
     assert [row["chunk_id"] for row in assembled] == ["body-1"]
+
+
+def test_public_retrieval_projection_exposes_importance() -> None:
+    response = asyncio.run(
+        project_public_retrieval_response(
+            {
+                "status": "success",
+                "results": [
+                    {
+                        "document_id": "doc_a",
+                        "chunk_id": "body-1",
+                        "chunk_type": "text",
+                        "content": "hello",
+                        "score": 0.85,
+                        "importance": 0.42,
+                        "source_file_name": "a.pdf",
+                        "section_path": "a.pdf / Intro",
+                    }
+                ],
+            }
+        )
+    )
+
+    assert response["results"][0]["importance"] == 0.42
+
+
+def test_metadata_consumers_prefer_entities_and_computed_tokens() -> None:
+    meta = {
+        "entities": [{"text": "Knowhere", "type": "product"}],
+        "keywords": ["legacy-kw"],
+        "tokens": ["legacy-token"],
+    }
+    assert extract_keywords_from_chunk_metadata(meta) == ["Knowhere"]
+
+    table_chunk = {
+        "type": "table",
+        "content": "<table><tr><td>Revenue</td></tr></table>",
+        "metadata": {
+            "summary": "Revenue breakdown",
+            "asset_title": "Table 1",
+            "entities": [{"text": "ARR", "type": "metric"}],
+        },
+    }
+    table_bm25 = build_content_search_text(table_chunk) or ""
+    table_term = build_term_search_text(table_chunk, path_text="Finance") or ""
+    assert "arr" in table_bm25.lower()
+    assert "table" in table_bm25.lower()
+    assert "arr" in table_term.lower()
+
+    text_chunk_no_tokens = {
+        "type": "text",
+        "content": "Revenue growth accelerated in Q4.",
+        "metadata": {},
+    }
+    lexical = build_content_lexical_text(text_chunk_no_tokens) or ""
+    assert "revenue" in lexical.lower()
+