From 5325ce5cdacebcf73165e879a585a5b4bcf3245e Mon Sep 17 00:00:00 2001 From: Matt Dawkins Date: Sun, 27 Sep 2026 16:17:45 -0400 Subject: [PATCH] Add vision-language model text query and whole-track image query Ollama VLM backends (find objects, ask) in Text Query dialog and panel, with install guidance; all-frames jobs with optional tracking; Image Query can search with frames sampled along the selected track. --- client/dive-common/apispec.ts | 72 ++++ client/dive-common/components/EditorMenu.vue | 196 ++++++---- client/dive-common/components/Viewer.vue | 20 +- .../dive-common/components/VlmSetupHelp.vue | 63 +++ client/platform/desktop/backend/ipcService.ts | 26 +- .../desktop/backend/native/interactive.ts | 37 +- .../desktop/backend/native/videoSearch.ts | 12 +- .../desktop/backend/native/vlm.spec.ts | 64 ++++ client/platform/desktop/backend/native/vlm.ts | 78 ++++ client/platform/desktop/frontend/api.ts | 66 +++- .../desktop/frontend/components/QueryPage.vue | 11 +- .../frontend/components/TextQueryContext.vue | 286 ++++++++++++++ .../components/VideoSearchContext.spec.ts | 47 ++- .../components/VideoSearchContext.vue | 55 ++- .../frontend/components/ViewerLoader.vue | 98 +++-- .../frontend/components/VlmAskContext.vue | 362 ++++++++++++++++++ .../platform/desktop/frontend/queryLaunch.ts | 4 + .../platform/desktop/frontend/useQueryPage.ts | 17 +- .../desktop/frontend/useVideoSearch.ts | 11 +- client/platform/desktop/frontend/useVlm.ts | 78 ++++ docs/Query.md | 12 +- 21 files changed, 1495 insertions(+), 120 deletions(-) create mode 100644 client/dive-common/components/VlmSetupHelp.vue create mode 100644 client/platform/desktop/backend/native/vlm.spec.ts create mode 100644 client/platform/desktop/backend/native/vlm.ts create mode 100644 client/platform/desktop/frontend/components/TextQueryContext.vue create mode 100644 client/platform/desktop/frontend/components/VlmAskContext.vue create mode 100644 client/platform/desktop/frontend/useVlm.ts diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 977006bdd..13c7fb072 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -781,6 +781,72 @@ export interface TextQueryResponse { fallback?: boolean; } +/** + * Vision-language model (VLM) query types + */ + +/** Model offered even before it is downloaded */ +export const RECOMMENDED_VLM_MODEL = 'qwen3-vl:8b'; + +/** A locally served vision-capable model */ +export interface VlmModel { + name: string; + /** Model has a reasoning phase that can be toggled */ + thinking: boolean; + /** False for a recommended model that still needs downloading */ + installed: boolean; +} + +export interface VlmModelsResponse { + /** Whether the VLM server could be reached */ + available: boolean; + models: VlmModel[]; + error?: string; +} + +/** Backends the text query dialog can offer */ +export interface TextQueryModelOptions { + sam3: boolean; + vlm: VlmModelsResponse; +} + +/** Text-prompted grounding through a VLM; answered with a TextQueryResponse */ +export interface VlmDetectRequest { + model: string; + think?: boolean; + imagePath: string; + frameTime?: number; + text: string; + maxDetections?: number; +} + +/** One image given to a VLM question, optionally cropped to a box */ +export interface VlmImageSpec { + imagePath: string; + frameTime?: number; + box?: [number, number, number, number]; +} + +export interface VlmChatTurn { + role: 'user' | 'assistant'; + content: string; +} + +export interface VlmAskRequest { + model: string; + think?: boolean; + question: string; + images: VlmImageSpec[]; + /** Prior turns, sent as text only */ + history?: VlmChatTurn[]; +} + +export interface VlmAskResponse { + success: boolean; + error?: string; + answer?: string; +} + export interface RefineDetectionsRequest { /** Path to the image file */ imagePath: string; @@ -809,6 +875,12 @@ export interface RefineDetectionsResponse { export type VideoSearchIndexMethod = 'detections' | 'tracking' | 'existing' | 'frames'; +/** One exemplar image of a query, e.g. one frame of a track, with its region. */ +export interface VideoSearchExemplar { + imagePath: string; + box?: [number, number, number, number]; +} + /** * One indexed media stream (video/sequence identifier) in the shared search * database. All database rows key on this identifier, so a dataset can be diff --git a/client/dive-common/components/EditorMenu.vue b/client/dive-common/components/EditorMenu.vue index 2e3098952..4a645dcab 100644 --- a/client/dive-common/components/EditorMenu.vue +++ b/client/dive-common/components/EditorMenu.vue @@ -13,8 +13,10 @@ import { Mousetrap, SuppressionDisplaySettings } from 'vue-media-annotator/types import { EditAnnotationTypes, VisibleAnnotationTypes } from 'vue-media-annotator/layers'; import Recipe from 'vue-media-annotator/recipe'; import SegmentationPointClick from 'dive-common/recipes/segmentationpointclick'; +import type { TextQueryModelOptions } from 'dive-common/apispec'; import AnnotationVisibilityMenu from './AnnotationVisibilityMenu.vue'; +import VlmSetupHelp from './VlmSetupHelp.vue'; import OutlinedLabeledGroup from './OutlinedLabeledGroup.vue'; import ToolbarExpandToggle from './ToolbarExpandToggle.vue'; @@ -37,6 +39,7 @@ export default defineComponent({ AnnotationVisibilityMenu, OutlinedLabeledGroup, ToolbarExpandToggle, + VlmSetupHelp, }, props: { editingTrack: { @@ -99,9 +102,9 @@ export default defineComponent({ type: Boolean, default: false, }, - /** Re-checks whether the SAM3 add-on is installed at press time. */ - checkTextQueryAvailable: { - type: Function as PropType<() => Promise>, + /** Re-lists the text query backends, e.g. after a model download. */ + loadTextQueryModels: { + type: Function as PropType<() => Promise>, default: undefined, }, /** True while browser auto-populate (mask/points) is embedding or predicting. */ @@ -155,24 +158,63 @@ export default defineComponent({ // When on, existing annotations are removed before the query results are // applied. On by default so a query replaces rather than accumulates. const textQueryReplaceExisting = ref(true); - const sam3InfoDialogOpen = ref(false); - - const openSam3InfoDialog = () => { - sam3InfoDialogOpen.value = true; - }; + const SAM3_MODEL = 'sam3'; + const TEXT_QUERY_MODEL_KEY = 'editorMenu.textQueryModel'; + const textQueryModelOptions = ref({ + sam3: true, vlm: { available: false, models: [] }, + }); + const textQueryModelsChecking = ref(false); + const preferredTextQueryModel = ref(localStorage.getItem(TEXT_QUERY_MODEL_KEY) || SAM3_MODEL); + const textQueryModelItems = computed(() => [ + { + text: textQueryModelOptions.value.sam3 ? 'SAM3' : 'SAM3 (not installed)', + value: SAM3_MODEL, + installed: textQueryModelOptions.value.sam3, + }, + ...textQueryModelOptions.value.vlm.models.map((m) => ({ + text: m.installed ? m.name : `${m.name} (not installed)`, + value: m.name, + installed: m.installed, + })), + ]); + // The preference survives a model list that briefly lacks it (e.g. Ollama + // slow to answer); only the effective choice falls back. + const textQueryModel = computed({ + get: () => { + const items = textQueryModelItems.value; + return items.some((item) => item.value === preferredTextQueryModel.value) + ? preferredTextQueryModel.value + : (items.find((item) => item.installed) ?? items[0]).value; + }, + set: (value: string) => { + preferredTextQueryModel.value = value; + localStorage.setItem(TEXT_QUERY_MODEL_KEY, value); + }, + }); + const textQueryUsesVlm = computed(() => textQueryModel.value !== SAM3_MODEL); + const textQueryVlmMissing = computed(() => textQueryUsesVlm.value + && textQueryModelOptions.value.vlm.models.some( + (m) => m.name === textQueryModel.value && !m.installed, + )); + const textQuerySam3Missing = computed(() => !textQueryUsesVlm.value + && !textQueryModelOptions.value.sam3); + const textQueryModelMissing = computed( + () => textQueryVlmMissing.value || textQuerySam3Missing.value, + ); - const closeSam3InfoDialog = () => { - sam3InfoDialogOpen.value = false; + const recheckTextQueryModels = async () => { + if (!props.loadTextQueryModels) return; + textQueryModelsChecking.value = true; + try { + textQueryModelOptions.value = await props.loadTextQueryModels(); + } finally { + textQueryModelsChecking.value = false; + } }; - const handleTextQueryClick = async () => { - const available = props.checkTextQueryAvailable - ? await props.checkTextQueryAvailable() - : props.textQueryAvailable; - if (!available) { - openSam3InfoDialog(); - return; - } + // The dialog explains how to install whichever model is missing, so it + // opens even when nothing is installed yet. + const handleTextQueryClick = () => { openTextQueryDialog(); }; @@ -195,15 +237,22 @@ export default defineComponent({ textQueryReplaceExisting.value = true; }; - const onTextQueryServiceReady = (success: boolean, error?: string) => { + const onTextQueryServiceReady = ( + success: boolean, + error?: string, + options?: TextQueryModelOptions, + ) => { textQueryInitializing.value = false; if (!success) { textQueryServiceError.value = error || 'Text query service is not available'; } + if (options) { + textQueryModelOptions.value = options; + } }; const submitTextQuery = () => { - if (!textQueryInput.value.trim()) { + if (!textQueryInput.value.trim() || textQueryModelMissing.value) { return; } textQueryLoading.value = true; @@ -212,12 +261,14 @@ export default defineComponent({ text: textQueryInput.value.trim(), boxThreshold: textQueryThreshold.value, replaceExisting: textQueryReplaceExisting.value, + vlmModel: textQueryUsesVlm.value ? textQueryModel.value : undefined, }); } else { emit('text-query', { text: textQueryInput.value.trim(), boxThreshold: textQueryThreshold.value, replaceExisting: textQueryReplaceExisting.value, + vlmModel: textQueryUsesVlm.value ? textQueryModel.value : undefined, }); } closeTextQueryDialog(); @@ -278,7 +329,7 @@ export default defineComponent({ icon: 'mdi-text-search', active: false, unavailable: !props.textQueryAvailable, - unavailableTooltip: 'SAM3 add-on not installed. Click for more information.', + unavailableTooltip: 'No text query model installed. Click for setup instructions.', description: 'Text Query', mousetrap: [{ bind: 'q', @@ -487,12 +538,19 @@ export default defineComponent({ textQueryServiceError, textQueryAllFrames, textQueryReplaceExisting, + textQueryModel, + textQueryModelItems, + textQueryUsesVlm, + textQueryVlmMissing, + textQuerySam3Missing, + textQueryModelMissing, + textQueryModelOptions, + textQueryModelsChecking, + recheckTextQueryModels, openTextQueryDialog, closeTextQueryDialog, onTextQueryServiceReady, submitTextQuery, - sam3InfoDialogOpen, - closeSam3InfoDialog, }; }, }); @@ -789,6 +847,43 @@ export default defineComponent({

Enter a description of objects to find in the current frame.

+ + + SAM3 text query needs the SAM3 Text Query Segmentation and Tracking + Models add-on. Install it from the Add-Ons page. + + Open Add-Ons + + + -

+

+ Boxes come from a locally served vision-language model and carry no confidence score. +

+

Textual query support uses architectures derived from Meta's SAM3 project

@@ -840,7 +945,7 @@ export default defineComponent({ v-if="!textQueryInitializing && !textQueryServiceError" color="primary" :loading="textQueryLoading" - :disabled="!textQueryInput.trim() || textQueryLoading" + :disabled="!textQueryInput.trim() || textQueryLoading || textQueryModelMissing" @click="submitTextQuery" > Search @@ -848,47 +953,6 @@ export default defineComponent({ - - - - - - - mdi-package-down - - SAM3 Add-On Required - - -

- Text query requires the SAM3 Text Query Segmentation and Tracking Models - add-on to be installed in your VIAME directory. -

-

- Download and install the add-on directly from the Add-Ons page. -

- - - - - Close - - - Open Add-Ons - - - - diff --git a/client/dive-common/components/Viewer.vue b/client/dive-common/components/Viewer.vue index f09531a56..13e27a860 100644 --- a/client/dive-common/components/Viewer.vue +++ b/client/dive-common/components/Viewer.vue @@ -73,7 +73,7 @@ import type { } from 'dive-common/use/useModeManager'; import clientSettingsSetup, { clientSettings, isStereoInteractiveModeEnabled } from 'dive-common/store/settings'; import { - useApi, FrameImage, DatasetType, GlobalStyleSettings, + useApi, FrameImage, DatasetType, GlobalStyleSettings, TextQueryModelOptions, } from 'dive-common/apispec'; import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; import { @@ -186,8 +186,8 @@ export default defineComponent({ type: Boolean, default: false, }, - checkTextQueryAvailable: { - type: Function as PropType<() => Promise>, + loadTextQueryModels: { + type: Function as PropType<() => Promise>, default: undefined, }, /** Deep link: frame to seek to once the media is ready (e.g. from the review grid). */ @@ -323,9 +323,13 @@ export default defineComponent({ * Forward text query service ready status to EditorMenu * Called by ViewerLoader when text query service initialization completes */ - function onTextQueryServiceReady(success: boolean, error?: string) { + function onTextQueryServiceReady( + success: boolean, + error?: string, + options?: TextQueryModelOptions, + ) { if (editorMenuRef.value?.onTextQueryServiceReady) { - editorMenuRef.value.onTextQueryServiceReady(success, error); + editorMenuRef.value.onTextQueryServiceReady(success, error, options); } } @@ -334,7 +338,9 @@ export default defineComponent({ * injecting the current frame number the query should run against. */ function onTextQuerySubmit( - payload: { text: string; boxThreshold: number; replaceExisting?: boolean }, + payload: { + text: string; boxThreshold: number; replaceExisting?: boolean; vlmModel?: string; + }, ) { emit('text-query-submit', { ...payload, @@ -2774,7 +2780,7 @@ export default defineComponent({ lassoDrawing: !readonlyState && lassoDrawing, textQueryEnabled, textQueryAvailable, - checkTextQueryAvailable, + loadTextQueryModels, autoPopulateBusy, autoPopulateStatus, }" diff --git a/client/dive-common/components/VlmSetupHelp.vue b/client/dive-common/components/VlmSetupHelp.vue new file mode 100644 index 000000000..6dd6a5f7d --- /dev/null +++ b/client/dive-common/components/VlmSetupHelp.vue @@ -0,0 +1,63 @@ + + + diff --git a/client/platform/desktop/backend/ipcService.ts b/client/platform/desktop/backend/ipcService.ts index bbde40635..61bcd371f 100644 --- a/client/platform/desktop/backend/ipcService.ts +++ b/client/platform/desktop/backend/ipcService.ts @@ -1,4 +1,7 @@ -import type { SegmentationPolygon, Pipe, GlobalStyleSettings } from 'dive-common/apispec'; +import type { + SegmentationPolygon, Pipe, GlobalStyleSettings, VlmAskRequest, VlmDetectRequest, + VideoSearchExemplar, +} from 'dive-common/apispec'; import OS from 'os'; import http from 'http'; import fs from 'fs'; @@ -37,6 +40,7 @@ import { getInteractiveServiceManager, } from './native/interactive'; import * as videoSearch from './native/videoSearch'; +import { listVisionModels } from './native/vlm'; import { SegmentationPredictRequest, SegmentationStereoSegmentRequest, @@ -527,6 +531,20 @@ export default function register() { return response; }); + ipcMain.handle('vlm-models', () => listVisionModels()); + + ipcMain.handle('vlm-detect', async (_, args: VlmDetectRequest) => { + const service = getInteractiveServiceManager(); + await service.ensureStarted(settings.get()); + return service.vlmDetect(args); + }); + + ipcMain.handle('vlm-ask', async (_, args: VlmAskRequest) => { + const service = getInteractiveServiceManager(); + await service.ensureStarted(settings.get()); + return service.vlmAsk(args); + }); + ipcMain.handle('segmentation-refine', async (_, args: { imagePath: string; detections: { @@ -694,9 +712,11 @@ export default function register() { return { success: true, streams }; }); - ipcMain.handle('video-search-formulate', async (_, args: { imagePath: string; boxes?: number[][] }) => { + ipcMain.handle('video-search-formulate', async (_, args: { + imagePath: string; boxes?: number[][]; exemplars?: VideoSearchExemplar[]; + }) => { const manager = videoSearch.getQueryServiceManager(); - return manager.formulateQuery(args.imagePath, args.boxes); + return manager.formulateQuery(args.imagePath, args.boxes, args.exemplars); }); ipcMain.handle('video-search-query', async ( diff --git a/client/platform/desktop/backend/native/interactive.ts b/client/platform/desktop/backend/native/interactive.ts index 36127ecf2..9fd725d72 100644 --- a/client/platform/desktop/backend/native/interactive.ts +++ b/client/platform/desktop/backend/native/interactive.ts @@ -15,7 +15,9 @@ */ import fs from 'fs-extra'; -import type { SegmentationPolygon } from 'dive-common/apispec'; +import type { + SegmentationPolygon, TextQueryResponse, VlmAskRequest, VlmAskResponse, VlmDetectRequest, +} from 'dive-common/apispec'; import OS from 'os'; import { spawn, ChildProcess } from 'child_process'; import npath from 'path'; @@ -505,6 +507,39 @@ export class InteractiveServiceManager extends EventEmitter { }, 'Text query'); } + async vlmDetect(request: VlmDetectRequest): Promise { + const response = await this.sendRequest({ + command: 'vlm_detect', + model: request.model, + think: request.think, + image_path: request.imagePath, + frame_time: request.frameTime, + text: request.text, + max_detections: request.maxDetections, + }, 'VLM detect'); + return response as unknown as TextQueryResponse; + } + + async vlmAsk(request: VlmAskRequest): Promise { + const response = await this.sendRequest({ + command: 'vlm_ask', + model: request.model, + think: request.think, + question: request.question, + images: request.images.map((image) => ({ + image_path: image.imagePath, + frame_time: image.frameTime, + box: image.box, + })), + history: request.history, + }, 'VLM ask'); + return { + success: response.success ?? false, + error: response.error, + answer: response.answer, + }; + } + async refineDetections(request: { imagePath: string; detections: { diff --git a/client/platform/desktop/backend/native/videoSearch.ts b/client/platform/desktop/backend/native/videoSearch.ts index 8301bf8fd..e70dcaa8d 100644 --- a/client/platform/desktop/backend/native/videoSearch.ts +++ b/client/platform/desktop/backend/native/videoSearch.ts @@ -42,7 +42,7 @@ import { } from 'platform/desktop/constants'; import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; import type { - VideoSearchIndexStatus, VideoSearchIndexInfo, SearchIndexBackend, + VideoSearchIndexStatus, VideoSearchIndexInfo, SearchIndexBackend, VideoSearchExemplar, } from 'dive-common/apispec'; import { serialize } from 'platform/desktop/backend/serializers/viame'; import { observeChild } from './processManager'; @@ -743,11 +743,19 @@ export class QueryServiceManager extends EventEmitter { this.currentIndexDirs = indexDirs; } - async formulateQuery(imagePath: string, boxes?: number[][]): Promise { + async formulateQuery( + imagePath: string, + boxes?: number[][], + exemplars?: VideoSearchExemplar[], + ): Promise { const response = await this.sendRequest({ command: 'formulate_query', image_path: imagePath, boxes, + exemplars: exemplars?.map((e) => ({ + image_path: e.imagePath, + boxes: e.box ? [e.box] : undefined, + })), }, 'Query formulation'); return QueryServiceManager.check(response, 'Query formulation'); } diff --git a/client/platform/desktop/backend/native/vlm.spec.ts b/client/platform/desktop/backend/native/vlm.spec.ts new file mode 100644 index 000000000..cbc968a09 --- /dev/null +++ b/client/platform/desktop/backend/native/vlm.spec.ts @@ -0,0 +1,64 @@ +import { listVisionModels, resolveOllamaUrl } from './vlm'; + +describe('resolveOllamaUrl', () => { + it.each([ + [undefined, 'http://localhost:11434'], + ['', 'http://localhost:11434'], + ['0.0.0.0', 'http://localhost:11434'], + ['0.0.0.0:8080', 'http://localhost:8080'], + ['gpu-box', 'http://gpu-box:11434'], + ['https://ollama.example.com', 'https://ollama.example.com'], + ['http://gpu-box:9000/', 'http://gpu-box:9000'], + ])('%s -> %s', (host, expected) => { + expect(resolveOllamaUrl(host)).toBe(expected); + }); +}); + +function fakeFetch(capabilities: Record): typeof fetch { + return (async (url: string, init?: RequestInit) => { + const body = (payload: unknown) => ({ ok: true, json: async () => payload }); + if (url.endsWith('/api/tags')) { + return body({ models: Object.keys(capabilities).map((name) => ({ name })) }); + } + const { model } = JSON.parse(init?.body as string); + if (capabilities[model] === undefined) { + return { ok: false, status: 404, statusText: 'Not Found' }; + } + return body({ capabilities: capabilities[model] }); + }) as unknown as typeof fetch; +} + +describe('listVisionModels', () => { + it('keeps only vision models, sorted, noting thinking support', async () => { + const result = await listVisionModels('http://x', fakeFetch({ + 'qwen3-vl:8b': ['completion', 'vision', 'thinking'], + 'llama3:8b': ['completion'], + 'gemma4:26b': ['completion', 'vision'], + })); + expect(result).toEqual({ + available: true, + models: [ + { name: 'gemma4:26b', thinking: false, installed: true }, + { name: 'qwen3-vl:8b', thinking: true, installed: true }, + ], + }); + }); + + it('offers the recommended model as not installed', async () => { + const result = await listVisionModels('http://x', fakeFetch({ + 'gemma4:26b': ['completion', 'vision'], + })); + expect(result.models).toEqual([ + { name: 'gemma4:26b', thinking: false, installed: true }, + { name: 'qwen3-vl:8b', thinking: false, installed: false }, + ]); + }); + + it('reports an unreachable server', async () => { + const failing = (async () => { throw new Error('ECONNREFUSED'); }) as unknown as typeof fetch; + const result = await listVisionModels('http://x', failing); + expect(result.available).toBe(false); + expect(result.models).toEqual([{ name: 'qwen3-vl:8b', thinking: false, installed: false }]); + expect(result.error).toContain('ECONNREFUSED'); + }); +}); diff --git a/client/platform/desktop/backend/native/vlm.ts b/client/platform/desktop/backend/native/vlm.ts new file mode 100644 index 000000000..b4dbdf0de --- /dev/null +++ b/client/platform/desktop/backend/native/vlm.ts @@ -0,0 +1,78 @@ +/** + * Discovery of locally served vision-language models (Ollama). + * + * Inference runs inside the interactive service (it needs frame decoding and + * cropping); only the cheap model listing happens here so the viewer can show + * VLM options without starting the Python process. + */ + +import { RECOMMENDED_VLM_MODEL } from 'dive-common/apispec'; +import type { VlmModel, VlmModelsResponse } from 'dive-common/apispec'; + +const DEFAULT_OLLAMA_URL = 'http://localhost:11434'; +const LIST_TIMEOUT_MS = 10000; + +/** Resolve the Ollama base URL the same way the Ollama CLI reads OLLAMA_HOST. */ +export function resolveOllamaUrl(host: string | undefined = process.env.OLLAMA_HOST): string { + if (!host || !host.trim()) { + return DEFAULT_OLLAMA_URL; + } + const hasScheme = /^https?:\/\//i.test(host.trim()); + const parsed = new URL(hasScheme ? host.trim() : `http://${host.trim()}`); + if (!hasScheme && !parsed.port) { + parsed.port = '11434'; + } + // A server bound to all interfaces is reached through loopback. + if (parsed.hostname === '0.0.0.0') { + parsed.hostname = 'localhost'; + } + return parsed.toString().replace(/\/+$/, ''); +} + +async function postJson(url: string, body: unknown, fetchFn: typeof fetch): Promise { + const response = await fetchFn(url, { + method: body === undefined ? 'GET' : 'POST', + headers: { 'Content-Type': 'application/json' }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: AbortSignal.timeout(LIST_TIMEOUT_MS), + }); + if (!response.ok) { + throw new Error(`${response.status} ${response.statusText}`); + } + return response.json() as Promise; +} + +export async function listVisionModels( + baseUrl: string = resolveOllamaUrl(), + fetchFn: typeof fetch = fetch, +): Promise { + let names: string[]; + try { + const tags = await postJson<{ models?: { name: string }[] }>(`${baseUrl}/api/tags`, undefined, fetchFn); + names = (tags.models || []).map((m) => m.name); + } catch (err) { + return { + available: false, + models: [{ name: RECOMMENDED_VLM_MODEL, thinking: false, installed: false }], + error: `Could not reach Ollama at ${baseUrl} (${(err as Error).message}).`, + }; + } + const shown = await Promise.all(names.map(async (name): Promise => { + try { + const info = await postJson<{ capabilities?: string[] }>(`${baseUrl}/api/show`, { model: name }, fetchFn); + const capabilities = info.capabilities || []; + if (!capabilities.includes('vision')) { + return null; + } + return { name, thinking: capabilities.includes('thinking'), installed: true }; + } catch { + return null; + } + })); + const models = shown.filter((m): m is VlmModel => m !== null) + .sort((a, b) => a.name.localeCompare(b.name)); + if (!names.includes(RECOMMENDED_VLM_MODEL)) { + models.push({ name: RECOMMENDED_VLM_MODEL, thinking: false, installed: false }); + } + return { available: true, models }; +} diff --git a/client/platform/desktop/frontend/api.ts b/client/platform/desktop/frontend/api.ts index 74d87a1aa..a76418ac5 100644 --- a/client/platform/desktop/frontend/api.ts +++ b/client/platform/desktop/frontend/api.ts @@ -9,9 +9,11 @@ import type { SegmentationPolygonKeypointsResponse, SegmentationStereoSegmentRequest, SegmentationStereoSegmentResponse, TextQueryRequest, TextQueryResponse, RefineDetectionsRequest, RefineDetectionsResponse, + VlmModelsResponse, VlmDetectRequest, VlmAskRequest, VlmAskResponse, PipelineJobResult, ScoringDatasetSummary, ScoringJobArgs, ScoringResult, ScoringResultSummary, ScoringSourceOptions, VideoSearchIndexStatus, VideoSearchIndexMethod, VideoSearchQueryResponse, VideoSearchIndexInfo, + VideoSearchExemplar, } from 'dive-common/apispec'; import axios, { AxiosInstance } from 'axios'; import { watch } from 'vue'; @@ -587,6 +589,18 @@ async function textQuery(request: TextQueryRequest): Promise return invoke('segmentation-text-query', request); } +async function vlmModels(): Promise { + return invoke('vlm-models'); +} + +async function vlmDetect(request: VlmDetectRequest): Promise { + return invoke('vlm-detect', request); +} + +async function vlmAsk(request: VlmAskRequest): Promise { + return invoke('vlm-ask', request); +} + async function refineDetections(request: RefineDetectionsRequest): Promise { return invoke('segmentation-refine', request); } @@ -598,11 +612,14 @@ async function runTextQueryPipeline( datasetId: string, queryText: string, threshold?: number, - replaceExisting = false, + replaceExisting?: boolean, + tracked?: boolean, ): Promise { const pipeline: Pipe = { name: 'Text Query', - pipe: 'utility_text_query_default.pipe', + pipe: tracked === undefined + ? 'utility_text_query_default.pipe' + : `utility_text_query_sam3_${tracked ? 'tracking' : 'no_tracking'}.pipe`, type: 'utility', }; @@ -628,6 +645,38 @@ async function runTextQueryPipeline( gpuJobQueue.enqueue(args); } +/** + * Run a vision-language model text query on all frames + */ +async function runVlmTextQueryPipeline( + datasetId: string, + queryText: string, + model: string, + replaceExisting = false, + tracked = false, +): Promise { + const pipeline: Pipe = { + name: 'Text Query (Vision Model)', + pipe: `utility_text_query_ollama_vlm_${tracked ? 'tracking' : 'no_tracking'}.pipe`, + type: 'utility', + }; + // The tracking variant always replaces existing annotations. + const kwiverParams: Record = tracked ? { + 'detector:detector:ollama_vlm:text_query': queryText, + 'detector:detector:ollama_vlm:model': model, + } : { + 'track_refiner:refiner:ollama_vlm:text_query': queryText, + 'track_refiner:refiner:ollama_vlm:model': model, + 'track_refiner:refiner:ollama_vlm:replace_existing': replaceExisting ? 'true' : 'false', + }; + gpuJobQueue.enqueue({ + type: JobType.RunPipeline, + pipeline, + datasetId, + pipelineParams: { kwiverParams }, + }); +} + /** * Video Search / IQR API */ @@ -666,8 +715,13 @@ async function videoSearchRemoveIndex(datasetId: string): Promise<{ success: boo return window.diveDesktop.invoke('video-search-remove-index', datasetId); } -async function videoSearchFormulate(imagePath: string, boxes?: number[][]): Promise { - return window.diveDesktop.invoke('video-search-formulate', { imagePath, boxes }); +/** Exemplars, when given, replace imagePath/boxes and each join the query as a positive. */ +async function videoSearchFormulate( + imagePath: string, + boxes?: number[][], + exemplars?: VideoSearchExemplar[], +): Promise { + return window.diveDesktop.invoke('video-search-formulate', { imagePath, boxes, exemplars }); } async function videoSearchQuery( @@ -1134,8 +1188,12 @@ export { segmentationSam3Installed, /* Text Query APIs */ textQuery, + vlmModels, + vlmDetect, + vlmAsk, refineDetections, runTextQueryPipeline, + runVlmTextQueryPipeline, /* Auto Register APIs */ /* Video Search / IQR */ videoSearchInstalled, diff --git a/client/platform/desktop/frontend/components/QueryPage.vue b/client/platform/desktop/frontend/components/QueryPage.vue index 565143b42..3f140f57a 100644 --- a/client/platform/desktop/frontend/components/QueryPage.vue +++ b/client/platform/desktop/frontend/components/QueryPage.vue @@ -131,6 +131,7 @@ export default defineComponent({ if (ret.canceled || !ret.filePaths?.length) return; [page.imagePath.value] = ret.filePaths; page.imageBox.value = null; + page.imageExemplars.value = null; } async function pickVideoFile() { @@ -231,6 +232,7 @@ export default defineComponent({ page.mode.value = 'image'; page.imagePath.value = launch.imagePath; page.imageBox.value = launch.box || null; + page.imageExemplars.value = launch.exemplars?.length ? launch.exemplars : null; page.warmStartModel.value = launch.modelPath || ''; page.onlySelected.value = false; page.search.selectIndex(launch.streamName); @@ -510,8 +512,15 @@ export default defineComponent({ v-if="imageUrl" :src="imageUrl" :box="page.imageBox.value" - @update:box="page.imageBox.value = $event" + @update:box="page.imageBox.value = $event; page.imageExemplars.value = null" /> +
+ Searching with {{ page.imageExemplars.value.length }} frames of the track. + Use this frame only +
{{ page.imagePath.value }}
diff --git a/client/platform/desktop/frontend/components/TextQueryContext.vue b/client/platform/desktop/frontend/components/TextQueryContext.vue new file mode 100644 index 000000000..352e71c8a --- /dev/null +++ b/client/platform/desktop/frontend/components/TextQueryContext.vue @@ -0,0 +1,286 @@ + + + diff --git a/client/platform/desktop/frontend/components/VideoSearchContext.spec.ts b/client/platform/desktop/frontend/components/VideoSearchContext.spec.ts index e50c80ee8..612db5c9d 100644 --- a/client/platform/desktop/frontend/components/VideoSearchContext.spec.ts +++ b/client/platform/desktop/frontend/components/VideoSearchContext.spec.ts @@ -5,7 +5,12 @@ import { videoSearchListIndexes } from '../api'; import VideoSearchContext from './VideoSearchContext.vue'; import { takeQueryLaunch } from '../queryLaunch'; -const mocks = vi.hoisted(() => ({ search: null as unknown, push: vi.fn(async (location: unknown) => location) })); +const mocks = vi.hoisted(() => ({ + search: null as unknown, + push: vi.fn(async (location: unknown) => location), + cameraStore: {} as unknown, + selectedTrackId: null as number | null, +})); vi.mock('vue-router/composables', () => ({ useRouter: () => ({ push: mocks.push }) })); vi.mock('../api', () => ({ videoSearchListIndexes: vi.fn() })); vi.mock('../useVideoSearch', () => ({ useVideoSearch: () => mocks.search })); @@ -13,10 +18,10 @@ vi.mock('../useSearchChips', () => ({ createSearchChips: () => ({ chips: ref({}) vi.mock('../store/jobs', () => ({ runningJobs: ref([]) })); vi.mock('dive-common/vue-utilities/prompt-service', () => ({ usePrompt: () => ({ prompt: vi.fn() }) })); vi.mock('vue-media-annotator/provides', () => ({ - useCameraStore: () => ({}), + useCameraStore: () => mocks.cameraStore, useHandler: () => ({}), useSelectedCamera: () => ref('left'), - useSelectedTrackId: () => ref(null), + useSelectedTrackId: () => ref(mocks.selectedTrackId), useTime: () => ({ frame: ref(0) }), })); vi.mock('./VideoSearchResultsGrid.vue', () => ({ default: {} })); @@ -79,6 +84,40 @@ it('launches an image query on the Query page without querying or rendering resu expect(takeQueryLaunch(location.query.launch)).toBeUndefined(); expect(queryFromImage).not.toHaveBeenCalled(); expect(wrapper.find('.results-list').exists()).toBe(false); - expect(wrapper.findAll('.query-launch-button')).toHaveLength(3); + expect(wrapper.findAll('.query-launch-button')).toHaveLength(4); + wrapper.destroy(); +}); + +it('launches a whole-track query with frames sampled along the selected track', async () => { + mocks.selectedTrackId = 3; + mocks.cameraStore = { + getTrack: () => ({ + featureIndex: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9], + getFeature: (f: number) => [{ bounds: [f, f, f + 5, f + 5] }], + }), + }; + mocks.search = { + datasetId: 'current', + state: reactive({ + installed: true, status: { datasetCount: 1 }, selectedStream: null, results: [], busy: null, + }), + refreshStatus: vi.fn(), + selectIndex: vi.fn(), + exemplarImageForFrame: vi.fn(async (f: number) => `/f${f}.png`), + }; + vi.mocked(videoSearchListIndexes).mockResolvedValue([{ + name: 'Alpha', datasetId: 'a', streamName: 'a-stream', method: 'detections', + }]); + const wrapper = shallowMount(VideoSearchContext, { stubs: ['v-btn', 'v-select', 'v-divider', 'v-alert', 'v-progress-linear'] }); + await new Promise((resolve) => setTimeout(resolve, 0)); + await (wrapper.vm as unknown as { queryFromWholeTrack(): Promise }).queryFromWholeTrack(); + const location = mocks.push.mock.calls.at(-1)?.[0] as unknown as { query: { launch: string } }; + const launch = takeQueryLaunch(location.query.launch); + expect(launch?.exemplars?.map((e) => e.imagePath)).toEqual( + ['/f0.png', '/f2.png', '/f4.png', '/f5.png', '/f7.png', '/f9.png'], + ); + expect(launch).toMatchObject({ imagePath: '/f0.png', box: [0, 0, 5, 5] }); wrapper.destroy(); + mocks.selectedTrackId = null; + mocks.cameraStore = {}; }); diff --git a/client/platform/desktop/frontend/components/VideoSearchContext.vue b/client/platform/desktop/frontend/components/VideoSearchContext.vue index 59c4162d6..2c9d62fba 100644 --- a/client/platform/desktop/frontend/components/VideoSearchContext.vue +++ b/client/platform/desktop/frontend/components/VideoSearchContext.vue @@ -16,9 +16,14 @@ import { runningJobs } from 'platform/desktop/frontend/store/jobs'; import { useVideoSearch } from 'platform/desktop/frontend/useVideoSearch'; import { holdQueryLaunch, takeQueryLaunch, QueryLaunch } from '../queryLaunch'; +type Box = [number, number, number, number]; + +/** Frames sampled along a track; each costs one descriptor pass. */ +const TRACK_QUERY_FRAMES = 6; + export default defineComponent({ name: 'VideoSearchContext', - description: 'Video Search', + description: 'Image Query', setup() { const search = useVideoSearch(); const router = useRouter(); @@ -95,6 +100,42 @@ export default defineComponent({ } catch (err) { search.state.error = (err as Error).message; } finally { launching.value = false; } } + /** Frames of the selected track that have a box, sampled evenly along it. */ + const selectedTrackFrames = computed((): { frame: number; box: Box }[] => { + if (!search || selectedTrackId.value === null) return []; + let track; + try { + track = cameraStore.getTrack(selectedTrackId.value, selectedCamera.value); + } catch { + return []; + } + const boxed = track.featureIndex.flatMap((f) => { + const [real] = track.getFeature(f); + return real?.bounds ? [{ frame: f, box: [...real.bounds] as Box }] : []; + }); + if (boxed.length <= TRACK_QUERY_FRAMES) return boxed; + const step = (boxed.length - 1) / (TRACK_QUERY_FRAMES - 1); + return Array.from({ length: TRACK_QUERY_FRAMES }, (_, i) => boxed[Math.round(i * step)]); + }); + + async function queryFromWholeTrack() { + if (!search || selectedTrackFrames.value.length < 2) return; + launching.value = true; + try { + const exemplars = await Promise.all(selectedTrackFrames.value.map( + async ({ frame: f, box }) => ({ imagePath: await search.exemplarImageForFrame(f), box }), + )); + const shown = exemplars.find((_, i) => selectedTrackFrames.value[i].frame === frame.value) + ?? exemplars[0]; + await launchQuery({ + imagePath: shown.imagePath, + box: shown.box, + exemplars, + streamName: search.state.selectedStream, + }); + } catch (err) { search.state.error = (err as Error).message; } finally { launching.value = false; } + } + async function queryFromImageFile(warmStart = false) { if (!search) return; let modelPath: string | undefined; @@ -140,6 +181,8 @@ export default defineComponent({ selectedTrackBox, launching, queryFromSelectedTrack, + selectedTrackFrames, + queryFromWholeTrack, queryFromImageFile, }; @@ -199,6 +242,16 @@ export default defineComponent({ > Search from
selected annotation + + Search from
selected track + settings.value?.readonlyMode || !!scoringPreviewFile.value); const timeFilter: Ref<[number, number] | null> = ref(null); const textQueryAvailable = ref(false); + const sam3Installed = ref(false); + const vlm = createVlm((frameNum: number) => { + const imagePath = segmentationGetImagePath?.(frameNum) || getImagePathForFrame(frameNum); + return imagePath ? { imagePath, frameTime: segmentationGetFrameTime?.(frameNum) } : null; + }); + provideVlm(vlm); async function refreshTextQueryAvailability() { - try { - const result = await segmentationSam3Installed(); - textQueryAvailable.value = result.installed; - } catch { - textQueryAvailable.value = false; - } + const [sam3, models] = await Promise.all([ + segmentationSam3Installed().then((r) => r.installed).catch(() => false), + vlm.refreshModels(), + ]); + sam3Installed.value = sam3; + textQueryAvailable.value = sam3 || models.some((m) => m.installed); return textQueryAvailable.value; } + async function loadTextQueryModels(): Promise { + await refreshTextQueryAvailability(); + return { + sam3: sam3Installed.value, + vlm: { + available: vlm.state.available, + models: vlm.state.models, + error: vlm.state.error || undefined, + }, + }; + } + // Keep button chrome current if the add-on is installed while this // viewer stays open (e.g. manual extract, or another desktop window). onMounted(() => { @@ -446,9 +470,10 @@ export default defineComponent({ boxThreshold: number; frameNum: number; replaceExisting?: boolean; + vlmModel?: string; }) { const { - text, boxThreshold, frameNum, replaceExisting = false, + text, boxThreshold, frameNum, replaceExisting = false, vlmModel, } = params; // Ensure metadata is loaded @@ -485,13 +510,22 @@ export default defineComponent({ textQueryRunning.value = true; try { - const response = await textQuery({ - imagePath, - frameTime, - text, - boxThreshold, - maxDetections: 10, - }); + const vlmInfo = vlm.state.models.find((m) => m.name === vlmModel); + const response = vlmModel + ? await vlmDetect({ + model: vlmModel, + think: vlmInfo?.thinking ? false : undefined, + imagePath, + frameTime, + text, + }) + : await textQuery({ + imagePath, + frameTime, + text, + boxThreshold, + maxDetections: 10, + }); // Query finished; drop the loading bar before any result/error prompts. textQueryRunning.value = false; @@ -677,16 +711,17 @@ export default defineComponent({ async function handleTextQueryInit() { try { // Check if the service is already ready + const modelOptions = await loadTextQueryModels(); const status = await segmentationIsReady(); if (status.ready) { - viewerRef.value?.onTextQueryServiceReady(true); + viewerRef.value?.onTextQueryServiceReady(true, undefined, modelOptions); return; } // Start the service process only -- do NOT warm the point-segmentation // model. The text-query model loads lazily on the first query. await segmentationEnsureStarted(); - viewerRef.value?.onTextQueryServiceReady(true); + viewerRef.value?.onTextQueryServiceReady(true, undefined, modelOptions); } catch (error) { // Provide text-query specific error message instead of generic segmentation error const rawMessage = error instanceof Error ? error.message : ''; @@ -704,11 +739,19 @@ export default defineComponent({ text: string; boxThreshold: number; replaceExisting?: boolean; + vlmModel?: string; + tracked?: boolean; }) { - const { text, boxThreshold, replaceExisting = false } = params; + const { + text, boxThreshold, replaceExisting = false, vlmModel, tracked, + } = params; try { - await runTextQueryPipeline(props.id, text, boxThreshold, replaceExisting); + if (vlmModel) { + await runVlmTextQueryPipeline(props.id, text, vlmModel, replaceExisting, tracked); + } else { + await runTextQueryPipeline(props.id, text, boxThreshold, replaceExisting, tracked); + } await prompt({ title: 'Text Query Pipeline Started', text: [ @@ -724,6 +767,14 @@ export default defineComponent({ } } + provideTextQuery({ + sam3Installed, + running: textQueryRunning, + loadModels: loadTextQueryModels, + runFrame: handleTextQuerySubmit, + runAllFrames: handleTextQueryAllFrames, + }); + /** * Interactive Stereo Service */ @@ -2571,6 +2622,7 @@ export default defineComponent({ handleTextQueryAllFrames, textQueryAvailable, refreshTextQueryAvailability, + loadTextQueryModels, openLink, /* Stereo */ stereoLoadingDialog, @@ -2615,7 +2667,7 @@ export default defineComponent({ :initial-track-id="viewerFocus.trackId" :text-query-enabled="true" :text-query-available="textQueryAvailable" - :check-text-query-available="refreshTextQueryAvailability" + :load-text-query-models="loadTextQueryModels" :stereo-view-link="stereoViewLink" @return-to-current-annotations="returnToCurrentAnnotations" @change-camera="changeCamera" diff --git a/client/platform/desktop/frontend/components/VlmAskContext.vue b/client/platform/desktop/frontend/components/VlmAskContext.vue new file mode 100644 index 000000000..6b382be15 --- /dev/null +++ b/client/platform/desktop/frontend/components/VlmAskContext.vue @@ -0,0 +1,362 @@ + + + + + diff --git a/client/platform/desktop/frontend/queryLaunch.ts b/client/platform/desktop/frontend/queryLaunch.ts index 5c5454d0b..c5d792f05 100644 --- a/client/platform/desktop/frontend/queryLaunch.ts +++ b/client/platform/desktop/frontend/queryLaunch.ts @@ -1,7 +1,11 @@ +import type { VideoSearchExemplar } from 'dive-common/apispec'; + /** One-shot handoff after the viewer's normal unsaved-change navigation guard. */ export interface QueryLaunch { imagePath: string; box?: [number, number, number, number]; + /** Frames of a whole track; each joins the query as a separate positive */ + exemplars?: VideoSearchExemplar[]; modelPath?: string; streamName: string | null; } diff --git a/client/platform/desktop/frontend/useQueryPage.ts b/client/platform/desktop/frontend/useQueryPage.ts index 50ff82d2e..05e51a9d2 100644 --- a/client/platform/desktop/frontend/useQueryPage.ts +++ b/client/platform/desktop/frontend/useQueryPage.ts @@ -10,7 +10,9 @@ import { import { JobType } from 'platform/desktop/constants'; import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; import type { ScoringDatasetSummary } from 'dive-common/scoring/types'; -import type { VideoSearchIndexMethod, VideoSearchResult, VideoSearchIndexInfo } from 'dive-common/apispec'; +import type { + VideoSearchIndexMethod, VideoSearchResult, VideoSearchIndexInfo, VideoSearchExemplar, +} from 'dive-common/apispec'; import type { ReviewItem } from 'dive-common/review/types'; import { listScoringDatasets, loadConfig, segmentationSam3Installed, textQuery, @@ -69,6 +71,11 @@ export function createQueryPage() { const imagePath = ref(''); /** Exemplar box on the image, in image pixels, or null for the whole image. */ const imageBox = ref<[number, number, number, number] | null>(null); + /** + * Frames of a whole track launched from the viewer. They replace the image + * and box as the query until either is changed. + */ + const imageExemplars = ref(null); const video = reactive({ kind: 'dataset', datasetId: '', filePath: '', frame: 0, }); @@ -283,7 +290,12 @@ export function createQueryPage() { if (!imagePath.value) return; error.value = null; const boxes = imageBox.value ? [[...imageBox.value]] : undefined; - await search.queryFromImage(imagePath.value, boxes, warmStartModel.value || undefined); + await search.queryFromImage( + imagePath.value, + boxes, + warmStartModel.value || undefined, + imageExemplars.value ?? undefined, + ); } /** Resolve the chosen video frame to an image on disk (extracting it for videos). */ @@ -432,6 +444,7 @@ export function createQueryPage() { mode, imagePath, imageBox, + imageExemplars, video, videoFramePath, videoFrameBox, diff --git a/client/platform/desktop/frontend/useVideoSearch.ts b/client/platform/desktop/frontend/useVideoSearch.ts index e5d4d2da3..2887fd1d2 100644 --- a/client/platform/desktop/frontend/useVideoSearch.ts +++ b/client/platform/desktop/frontend/useVideoSearch.ts @@ -11,7 +11,7 @@ import { reactive, provide, inject } from 'vue'; import type { VideoSearchIndexStatus, VideoSearchIndexMethod, VideoSearchIndexInfo, - VideoSearchResult, VideoSearchQueryResponse, + VideoSearchResult, VideoSearchQueryResponse, VideoSearchExemplar, } from 'dive-common/apispec'; import { videoSearchInstalled, videoSearchIndexStatus, videoSearchBuildIndex, @@ -261,11 +261,16 @@ export function createVideoSearch( * boxes. Descriptors are computed once (on this dataset's primary * session) and the similarity query fans out to every open index. */ - async function queryFromImage(imagePath: string, boxes?: number[][], warmStartModelPath?: string) { + async function queryFromImage( + imagePath: string, + boxes?: number[][], + warmStartModelPath?: string, + exemplars?: VideoSearchExemplar[], + ) { await guarded('Searching...', async () => { await ensureSession(); resetQueryState(); - let response = await videoSearchFormulate(imagePath, boxes); + let response = await videoSearchFormulate(imagePath, boxes, exemplars); if (!response.results?.length || warmStartModelPath) { response = await videoSearchQuery( warmStartModelPath ? { iqrModelPath: warmStartModelPath } : {}, diff --git a/client/platform/desktop/frontend/useVlm.ts b/client/platform/desktop/frontend/useVlm.ts new file mode 100644 index 000000000..15025c31c --- /dev/null +++ b/client/platform/desktop/frontend/useVlm.ts @@ -0,0 +1,78 @@ +import { + inject, provide, reactive, Ref, +} from 'vue'; +import type { TextQueryModelOptions, VlmModel } from 'dive-common/apispec'; +import { vlmModels } from 'platform/desktop/frontend/api'; + +const VlmInjectKey = Symbol('vlm'); +const TextQueryInjectKey = Symbol('textQuery'); + +export interface VlmFrameMedia { + imagePath: string; + frameTime?: number; +} + +/** + * Shared VLM state for a desktop viewer: the locally served vision models and + * a resolver from frame number to the media the interactive service reads. + */ +export function createVlm(resolveFrame: (frame: number) => VlmFrameMedia | null) { + const state = reactive({ + available: false, + models: [] as VlmModel[], + error: '', + }); + + async function refreshModels(): Promise { + try { + const result = await vlmModels(); + state.available = result.available; + state.models = result.models; + state.error = result.error || ''; + } catch (err) { + state.available = false; + state.models = []; + state.error = (err as Error).message; + } + return state.models; + } + + return { state, refreshModels, resolveFrame }; +} + +export type VlmContextType = ReturnType; + +export function provideVlm(context: VlmContextType) { + provide(VlmInjectKey, context); +} + +export function useVlm(): VlmContextType | null { + return inject(VlmInjectKey, null); +} + +export interface TextQueryParams { + text: string; + boxThreshold: number; + replaceExisting?: boolean; + /** Unset runs SAM3 */ + vlmModel?: string; + /** All-frames runs only; unset keeps each backend's default pipe */ + tracked?: boolean; +} + +/** The viewer's text query actions, shared by the toolbar dialog and side panel. */ +export interface TextQueryActions { + sam3Installed: Readonly>; + running: Readonly>; + loadModels: () => Promise; + runFrame: (params: TextQueryParams & { frameNum: number }) => Promise; + runAllFrames: (params: TextQueryParams) => Promise; +} + +export function provideTextQuery(actions: TextQueryActions) { + provide(TextQueryInjectKey, actions); +} + +export function useTextQuery(): TextQueryActions | null { + return inject(TextQueryInjectKey, null); +} diff --git a/docs/Query.md b/docs/Query.md index 0cdc125a3..61c71a70e 100644 --- a/docs/Query.md +++ b/docs/Query.md @@ -13,7 +13,7 @@ Add datasets with the same picker as the Training and Pipelines pages: search th * *Around existing annotations*: describe the dataset's current annotations. * *Whole frames*: describe each frame as a whole, with no detector. -Indexing runs as jobs on the Jobs page; the rows update as they finish. **Build all not indexed** queues every unindexed dataset. All indexed datasets share one index, so a dataset indexed here is also searchable from the viewer's Video Search panel, and the reverse. +Indexing runs as jobs on the Jobs page; the rows update as they finish. **Build all not indexed** queues every unindexed dataset. All indexed datasets share one index, so a dataset indexed here is also searchable from the viewer's Image Query panel, and the reverse. ## Query panel @@ -35,7 +35,7 @@ Index builds appear in **Jobs** while preparing and running, with their dataset, Selecting a stereo or multicamera sequence uses its first camera in the configured display order. Query shows that camera by name and indexes its media and, for the existing-detections method, its annotations. The other cameras are not indexed automatically. Results refer to the indexed camera, keeping thumbnails and frame numbers aligned with its media. -Inside an annotation sequence, **Video Search** lists the available indexed +Inside an annotation sequence, the **Image Query** panel lists the available indexed sequences, with an option to search all of them. Selecting an indexed sequence limits the ranked results and result grid to that sequence; the underlying similarity search still uses the shared database. Changing this selection clears @@ -56,8 +56,14 @@ index** deletes one sequence's search data, while **Delete entire index** remove the whole shared index. Both ask for confirmation and keep source media and annotations. Index deletion is disabled while index builds are queued or running. -The sequence editor’s **Video Search** panel is a launch point. Its large search +The sequence editor’s **Image Query** panel is a launch point. Its large search buttons open Query and run the selected annotation, image, or saved-model search there. The chosen index and annotation crop are preserved; results and refinement appear on Query rather than in the sidebar. Leaving the editor still follows the normal unsaved-annotation checks. + +**Search from selected track** queries with the whole selected track instead +of one frame: up to six frames sampled along the track each contribute a +descriptor, and all of them join the query as separate positive examples. +Query shows the current frame's crop; redrawing the box or choosing **Use this +frame only** returns to a single-frame query.