Skip to content

fix: gate contradiction hunt BM25 sensitivity #19

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bob-codedaptive wants to merge 1 commit into from
Closed

fix: gate contradiction hunt BM25 sensitivity #19

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions packages/kits/GeniusLocusKit/Sources/GeniusLocusKit/Brain/ContradictionHunt.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,10 @@ public extension GeniusLocusKit {
// at rank #399) and a whole-partition float scan is ~3 s/probe. BM25
// returns SOURCE (drawer) IDs directly, so no chunk→drawer remap.
// `seenPairs` keys on drawer IDs, so both lanes dedupe together.
func huntDrawerVisible(_ drawer: Drawer) -> Bool {
drawer.adjectiveSensitivity.rawValue <= AdjectiveSensitivity.elevated.rawValue
}

if let corpus = corpusKits[handle] {
// Probe drawers = the owning drawers of the recent probe chunks.
let probeChunkUUIDs = probeIDs.compactMap { UUID(uuidString: $0) }
Expand All @@ -240,12 +244,16 @@ public extension GeniusLocusKit {
if !probeDrawerIDs.isEmpty {
probeDrawers = try await estate.hydrateBodies(ids: probeDrawerIDs)
}
for probeDrawer in probeDrawers where !probeDrawer.content.isEmpty {
for probeDrawer in probeDrawers
where !probeDrawer.content.isEmpty && huntDrawerVisible(probeDrawer) {
let query = String(probeDrawer.content.prefix(Self.huntBM25QueryCharLimit))
guard let hits = try? await corpus.bm25TopKBySource(
query: query, limit: Self.huntBM25CandidateK)
else { continue }
for hit in hits where hit.sourceID != probeDrawer.id {
let hitDrawers = (try? await estate.hydrateBodies(ids: [hit.sourceID])) ?? []

@chatgpt-codex-connector chatgpt-codex-connector Bot Jul 13, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Batch BM25 hit hydration before filtering

When the corpus lane has many hits, this performs a separate hydrateBodies storage read for every BM25 result before the existing batched candidate hydration below; with the default 50 probes × 20 hits, and larger on-demand sweeps, that can turn one chunked by-id load into hundreds or thousands of round trips while re-reading the same drawers. Consider collecting the hit source IDs and applying the sensitivity check from a single batch load/map instead.

Useful? React with 👍 / 👎.

guard let hitDrawer = hitDrawers.first,
huntDrawerVisible(hitDrawer) else { continue }
let key = Self.pairKey(probeDrawer.id, hit.sourceID)
guard seenPairs.insert(key).inserted else { continue }
candidatePairs.append((min(probeDrawer.id, hit.sourceID),
Expand All @@ -272,7 +280,8 @@ public extension GeniusLocusKit {

for pair in candidatePairs {
guard let a = drawersByID[pair.a], let b = drawersByID[pair.b],
a.tombstonedAt == nil, b.tombstonedAt == nil else { continue }
a.tombstonedAt == nil, b.tombstonedAt == nil,
huntDrawerVisible(a), huntDrawerVisible(b) else { continue }
// Incremental watermark: at least one side must be new enough.
if let watermark = filedAfter,
a.filedAt <= watermark, b.filedAt <= watermark { continue }
Expand Down
18 changes: 16 additions & 2 deletions packages/kits/GeniusLocusKit/rust/src/coordinator.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2799,6 +2799,10 @@ impl EstateCoordinator {
let node_names = build_node_name_map(self.node_stores.get(handle), &all_drawers);
let drawers_by_id: std::collections::HashMap<&str, &locus_kit::drawer::Drawer> =
all_drawers.iter().map(|d| (d.id.as_str(), d)).collect();
let hunt_drawer_visible = |drawer: &locus_kit::drawer::Drawer| {
drawer.adjective_sensitivity().raw_value()
<= locus_kit::adjectives::AdjectiveSensitivity::Elevated.raw_value()
};

// Lane 2 — the corpus lane, the ONLY lane a production estate populates
// (the encode drain writes chunk-keyed rows under the corpus provider's
Expand Down Expand Up @@ -2828,7 +2832,7 @@ impl EstateCoordinator {
Some(d) => *d,
None => continue,
};
if pd.content.is_empty() {
if pd.content.is_empty() || !hunt_drawer_visible(pd) {
continue;
}
// Cap the query length so per-probe cost is independent of body size.
Expand All @@ -2842,6 +2846,12 @@ impl EstateCoordinator {
if source_id == *pd_id {
continue;
}
let Some(hit_drawer) = drawers_by_id.get(source_id.as_str()) else {
continue;
};
if !hunt_drawer_visible(hit_drawer) {
continue;
}
let key = pair_key(pd_id, &source_id);
if !seen_pairs.insert(key) {
continue;
Expand All @@ -2867,7 +2877,11 @@ impl EstateCoordinator {
(Some(a), Some(b)) => (*a, *b),
_ => continue,
};
if a.tombstoned_at.is_some() || b.tombstoned_at.is_some() {
if a.tombstoned_at.is_some()
|| b.tombstoned_at.is_some()
|| !hunt_drawer_visible(a)
|| !hunt_drawer_visible(b)
{
continue;
}
// Incremental watermark: at least one side must be new enough.
Expand Down
Loading