Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions backend/app/routers/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ class SearchHit(BaseModel):
# Seconds into the asset. Null for a metadata hit, which is about the file rather
# than a moment in it.
start_time: Optional[float] = None
# Where that moment ends, so it can be cut out as a clip. Null whenever start_time
# is, and for a moment whose transcript segment no longer exists.
end_time: Optional[float] = None
segment_id: Optional[str] = None
# Which retrievers found it — "keyword", "semantic", or both.
sources: List[str] = Field(default_factory=list)
Expand Down Expand Up @@ -80,6 +83,7 @@ def search(
score=round(fused.score, 6),
snippet=fused.snippet,
start_time=fused.start_time,
end_time=fused.end_time,
segment_id=fused.segment_id,
sources=sorted(fused.sources),
other_matches=fused.other_matches,
Expand Down
4 changes: 4 additions & 0 deletions backend/app/search/hybrid.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ class FusedAsset:
# returns its single best moment is useful; one that floods the page with twenty of
# its own moments is not.
other_matches: int = 0
# Where the matched moment ends. Not carried through fusion: neither retriever has
# it to hand (the keyword index stores only the start), so it is looked up once for
# the final page — see `services/search.py::_attach_end_times`.
end_time: Optional[float] = None


def fuse(ranked_lists: dict[str, Sequence[Candidate]], *, k: int = RRF_K) -> list[Candidate]:
Expand Down
31 changes: 28 additions & 3 deletions backend/app/services/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,9 +68,34 @@ def search(
allowed = _asset_ids_of_type(session, user_id, asset_type)
fused = [a for a in fused if a.asset_id in allowed]

return SearchOutcome(
assets=fused[:limit], semantic_ran=semantic_ran, semantic_error=semantic_error
)
page = fused[:limit]
_attach_end_times(session, page)
return SearchOutcome(assets=page, semantic_ran=semantic_ran, semantic_error=semantic_error)


def _attach_end_times(session: Session, hits: list[FusedAsset]) -> None:
"""Give each moment its end, so a caller can cut it out without guessing a length.

Gecko Video Creator turns a hit into a clip ("where people talk about X"), and a
start alone leaves it to pick an out point blind. Looked up from the transcript
rather than stored in the FTS index: adding a column there means rebuilding the
whole index in a migration, for a value one indexed query fetches for a page of
thirty. Done after `limit`, so only hits that will be returned are looked up.
"""
segment_ids = [h.segment_id for h in hits if h.segment_id]
if not segment_ids:
return
rows = session.exec(
select(TranscriptSegment.id, TranscriptSegment.end_time).where(
col(TranscriptSegment.id).in_(segment_ids)
)
).all()
ends = dict(rows)
for hit in hits:
if hit.segment_id:
# A segment deleted since it was indexed (a transcript re-run) has no end to
# give; the hit keeps its start and says nothing rather than something wrong.
hit.end_time = ends.get(hit.segment_id)


def _keyword_candidates(session: Session, user_id: str, query: str) -> list[Candidate]:
Expand Down
22 changes: 22 additions & 0 deletions backend/tests/test_search_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,28 @@ def test_a_spoken_hit_returns_the_second_it_was_said(embedded_library):
assert "«nano»" in top["snippet"] or "«weapons»" in top["snippet"]


def test_a_spoken_hit_says_where_the_moment_ends(embedded_library):
"""So it can be cut out as a clip without guessing a length (Gecko Video Creator)."""
top = search(embedded_library["client"], "nano weapons")["data"][0]
assert (top["start_time"], top["end_time"]) == (412.0, 417.0)


def test_a_moment_found_only_by_meaning_has_its_end_too(embedded_library):
"""The semantic retriever reaches segments by a different path from the keyword
one; the end must not depend on which of them found the moment."""
# "armament" is said nowhere, so only the semantic half can find this moment.
top = search(embedded_library["client"], "armament")["data"][0]
assert top["asset"]["name"] == "Giordano interview"
assert top["sources"] == ["semantic"]
assert (top["start_time"], top["end_time"]) == (412.0, 417.0)


def test_a_metadata_hit_has_no_moment_to_end(embedded_library):
top = search(embedded_library["client"], "committee")["data"][0]
assert top["start_time"] is None
assert top["end_time"] is None


def test_the_snippet_is_the_line_not_the_whole_transcript(embedded_library):
top = search(embedded_library["client"], "aerosol")["data"][0]
assert len(top["snippet"]) < 200
Expand Down
6 changes: 4 additions & 2 deletions docs/plan-of-attack.md
Original file line number Diff line number Diff line change
Expand Up @@ -178,8 +178,10 @@ Two indexes over the same corpus, fused:
- **Fusion**: Reciprocal Rank Fusion over the two ranked lists. No weights to tune, and
exact-phrase queries and half-remembered ones both land.

A result carries `{asset, score, snippet, start_time}`, and the UI deep-links to
`/assets/{id}?t=412.0`.
A result carries `{asset, score, snippet, start_time, end_time}`, and the UI deep-links
to `/assets/{id}?t=412.0`. `end_time` is the matched transcript segment's end, so
Gecko Video Creator can cut the moment out as a clip rather than guess its length
(GVC's plan, GAM-1); it is looked up for the returned page, not stored in the index.

### Serving bytes — diverging from Notes on purpose

Expand Down
2 changes: 2 additions & 0 deletions frontend/src/api/search.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ export interface SearchHit {
snippet: string
/** Seconds into the asset. Null for a metadata hit, which is about the file itself. */
start_time: number | null
/** Where that moment ends, for cutting it out as a clip. Null with `start_time`. */
end_time: number | null
segment_id: string | null
/** Which retrievers found it: 'keyword', 'semantic', or both. */
sources: string[]
Expand Down
1 change: 1 addition & 0 deletions frontend/src/views/SearchView.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ function hit(overrides: Partial<SearchHit> = {}): SearchHit {
score: 0.03,
snippet: 'We are deploying «nano» «weapons» via aerosol dispersion.',
start_time: 412,
end_time: 417,
segment_id: 's1',
sources: ['keyword', 'semantic'],
other_matches: 0,
Expand Down
Loading