diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 57ba46287..0ad0d0bb9 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -9,7 +9,7 @@ import { CustomStyle } from 'vue-media-annotator/StyleManager'; import { AttributeTrackFilter } from 'vue-media-annotator/AttributeTrackFilterControls'; import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements'; import type { - CameraHomographies, CameraCorrespondences, CameraTransformTypes, RegistrationSource, + CameraHomographies, CameraObservations, CameraTransformTypes, RegistrationSource, } from 'vue-media-annotator/alignedView/CameraRegistrationStore'; import type { PercentileStretch } from 'vue-media-annotator/use/useImageEnhancements'; @@ -82,6 +82,15 @@ interface PipeMetadata { interface PipelineRuntimeParams { frameRange?: [number, number] | null; + /** + * Multicam registration subset: camera name -> ordered image identifiers + * for exactly the frames the job should process. Row i of one camera's + * list pairs with row i of every other's. Identifiers are the camera's + * own image names (the platform backend resolves them to real paths) or + * `frame://N` pseudo-names for video cameras (the backend extracts those + * frames to temp images before the job). + */ + imagePairs?: Record; } interface PipelineParams { @@ -266,7 +275,12 @@ interface DatasetConfigMutable { attributeTrackFilters?: Readonly>; datasetInfo?: DatasetInfoFields; cameraHomographies?: CameraHomographies; - cameraCorrespondences?: CameraCorrespondences; + /** + * Per-image-pair correspondence observations, keyed by directional + * "left::right". Each entry lists the observations (image-pair identity, + * enabled flag, producer source, stats, and points) behind that pair's fit. + */ + cameraCorrespondences?: CameraObservations; cameraTransformTypes?: CameraTransformTypes; /** Producer provenance of the camera registration (see RegistrationSource). */ cameraRegistrationSource?: RegistrationSource | null; @@ -698,4 +712,4 @@ export type { MediaImportResponse, }; -export type { PercentileStretch, CameraCorrespondences }; +export type { PercentileStretch, CameraObservations }; diff --git a/client/dive-common/autoRegisterSelection.spec.ts b/client/dive-common/autoRegisterSelection.spec.ts new file mode 100644 index 000000000..d2f532e50 --- /dev/null +++ b/client/dive-common/autoRegisterSelection.spec.ts @@ -0,0 +1,88 @@ +import { describe, expect, it } from 'vitest'; + +import proposeRegistrationFrames from './autoRegisterSelection'; + +describe('proposeRegistrationFrames', () => { + it('spreads candidates across every temporal bin (no single-scene bias)', () => { + const frames = proposeRegistrationFrames({ + counts: [1200, 1200], + bins: 12, + perBin: 2, + }); + expect(frames.length).toBe(24); + // Every 100-frame bin contributes exactly its share: a scene-rich + // stretch can never supply all the candidates. + for (let bin = 0; bin < 12; bin += 1) { + const inBin = frames.filter((f) => f >= bin * 100 && f < (bin + 1) * 100); + expect(inBin.length).toBe(2); + } + expect(frames).toEqual([...frames].sort((a, b) => a - b)); + }); + + it('spans only the shortest camera', () => { + const frames = proposeRegistrationFrames({ + counts: [1000, 300], + bins: 10, + perBin: 1, + }); + expect(Math.max(...frames)).toBeLessThan(300); + expect(frames.length).toBe(10); + }); + + it('ranks within a bin by inter-camera timestamp skew', () => { + // Two cameras, 10 frames, one bin: frame 6 is perfectly synced, frame 3 + // is close, everything else is badly skewed. + const base = 1_700_000_000; + const camA = Array.from({ length: 10 }, (_, i) => base + i); + const camB = camA.map((t, i) => { + if (i === 6) return t; + if (i === 3) return t + 0.1; + return t + 5; + }); + const frames = proposeRegistrationFrames({ + counts: [10, 10], + timestamps: [camA, camB], + bins: 1, + perBin: 2, + maxSkewSeconds: 0.5, + }); + expect(frames).toEqual([3, 6]); + }); + + it('drops candidates whose skew exceeds the threshold entirely', () => { + const base = 1_700_000_000; + const camA = Array.from({ length: 4 }, (_, i) => base + i); + const camB = camA.map((t) => t + 10); // hopeless sync everywhere + const frames = proposeRegistrationFrames({ + counts: [4, 4], + timestamps: [camA, camB], + bins: 1, + perBin: 2, + maxSkewSeconds: 0.5, + }); + expect(frames).toEqual([]); + }); + + it('falls back to even spread for frames without timestamps', () => { + const frames = proposeRegistrationFrames({ + counts: [100, 100], + timestamps: [ + Array.from({ length: 100 }, () => undefined), + Array.from({ length: 100 }, () => undefined), + ], + bins: 2, + perBin: 2, + }); + expect(frames.length).toBe(4); + expect(frames.filter((f) => f < 50).length).toBe(2); + expect(frames.filter((f) => f >= 50).length).toBe(2); + }); + + it('handles degenerate inputs', () => { + expect(proposeRegistrationFrames({ counts: [0, 10], bins: 5, perBin: 2 })).toEqual([]); + expect(proposeRegistrationFrames({ counts: [], bins: 5, perBin: 2 })).toEqual([]); + // More bins than frames: every frame proposed once. + const tiny = proposeRegistrationFrames({ counts: [3, 3], bins: 12, perBin: 2 }); + expect(tiny).toEqual([0, 1, 2]); + }); +}); diff --git a/client/dive-common/autoRegisterSelection.ts b/client/dive-common/autoRegisterSelection.ts new file mode 100644 index 000000000..1d93124d7 --- /dev/null +++ b/client/dive-common/autoRegisterSelection.ts @@ -0,0 +1,108 @@ +/** + * Candidate-frame proposal for the auto-register pipeline (stage 0 of the + * selection contract): DIVE picks for diversity and synchronization, VIAME + * picks for image quality. + * + * Stratified, not "evenly spread" and not "most featureful": the flight is + * divided into `bins` equal time bins and `perBin` candidates are proposed + * within each, so temporal spread is guaranteed structurally -- a single + * scene-rich stretch can never supply every frame, which would reconstitute + * exactly the single-scene bias the multi-pair restructure exists to remove. + * The pipeline then keeps the best candidate per bin by image quality + * (`max_frames` is the bin count), so oversampling here is deliberate. + * + * Within a bin, candidates rank by inter-camera timestamp skew when every + * camera carries per-frame timestamps: on a survey aircraft at ~100 kt, + * 100 ms of desync is ~5 m of ground motion baked straight into the "ground + * truth" points, and RANSAC cannot reject it because it is consistent + * within the frame. Without timestamps (positional alignment), candidates + * spread evenly within the bin and skew is unknowable -- no threshold + * applies. + */ + +export interface ProposalOptions { + /** Per-camera usable frame counts; the proposal spans [0, min(counts)). */ + counts: number[]; + /** + * Optional per-camera per-frame capture timestamps (epoch seconds; + * undefined entries = unknown). Skew ranking applies only when every + * camera has a timestamp for the frame under consideration. + */ + timestamps?: (number | undefined)[][]; + /** Number of temporal bins (the pipeline's max_frames budget). */ + bins: number; + /** Candidates proposed per bin. */ + perBin: number; + /** Candidates with a larger inter-camera skew (seconds) are dropped. */ + maxSkewSeconds?: number; +} + +/** Largest pairwise timestamp difference across cameras, or null if unknowable. */ +function frameSkew( + timestamps: (number | undefined)[][], + frame: number, +): number | null { + const stamps: number[] = []; + for (let cam = 0; cam < timestamps.length; cam += 1) { + const t = timestamps[cam]?.[frame]; + if (t === undefined) { + return null; + } + stamps.push(t); + } + return Math.max(...stamps) - Math.min(...stamps); +} + +/** + * Propose candidate frame indices, sorted ascending. Returns at most + * `bins * perBin` frames; short datasets simply yield fewer. + */ +export default function proposeRegistrationFrames(options: ProposalOptions): number[] { + const usable = Math.min(...options.counts); + if (!Number.isFinite(usable) || usable <= 0 || options.bins <= 0 || options.perBin <= 0) { + return []; + } + const bins = Math.min(options.bins, usable); + const { timestamps } = options; + const maxSkew = options.maxSkewSeconds ?? 0.5; + const chosen = new Set(); + for (let bin = 0; bin < bins; bin += 1) { + const start = Math.floor((bin * usable) / bins); + const end = Math.floor(((bin + 1) * usable) / bins); // exclusive + // bins <= usable, so every bin spans at least one frame. + const size = end - start; + const perBin = Math.min(options.perBin, size); + if (timestamps && timestamps.length) { + // Rank the whole bin by skew; unknowable-skew frames rank after + // measured ones (evenly spread among themselves), over-threshold + // frames are dropped outright. + const measured: { frame: number; skew: number }[] = []; + const unknowable: number[] = []; + for (let frame = start; frame < end; frame += 1) { + const skew = frameSkew(timestamps, frame); + if (skew === null) { + unknowable.push(frame); + } else if (skew <= maxSkew) { + measured.push({ frame, skew }); + } + } + measured.sort((a, b) => a.skew - b.skew || a.frame - b.frame); + measured.slice(0, perBin).forEach(({ frame }) => chosen.add(frame)); + let still = perBin - Math.min(measured.length, perBin); + if (still > 0 && unknowable.length) { + const step = unknowable.length / still; + for (let i = 0; i < still; i += 1) { + chosen.add(unknowable[Math.floor(i * step)]); + } + still = 0; + } + } else { + // No timeline: spread evenly within the bin. + const step = size / perBin; + for (let i = 0; i < perBin; i += 1) { + chosen.add(start + Math.floor(i * step + step / 2)); + } + } + } + return [...chosen].sort((a, b) => a - b); +} diff --git a/client/dive-common/components/CameraRegistration/AutoRegisterDialog.vue b/client/dive-common/components/CameraRegistration/AutoRegisterDialog.vue new file mode 100644 index 000000000..47f141e01 --- /dev/null +++ b/client/dive-common/components/CameraRegistration/AutoRegisterDialog.vue @@ -0,0 +1,203 @@ + + + diff --git a/client/dive-common/components/CameraRegistration/RegistrationFrameList.vue b/client/dive-common/components/CameraRegistration/RegistrationFrameList.vue new file mode 100644 index 000000000..12fc7d582 --- /dev/null +++ b/client/dive-common/components/CameraRegistration/RegistrationFrameList.vue @@ -0,0 +1,191 @@ + + + + + diff --git a/client/dive-common/components/CameraRegistration/RegistrationTools.vue b/client/dive-common/components/CameraRegistration/RegistrationTools.vue index 23481ccf7..a1771a1e1 100644 --- a/client/dive-common/components/CameraRegistration/RegistrationTools.vue +++ b/client/dive-common/components/CameraRegistration/RegistrationTools.vue @@ -13,14 +13,30 @@ import { } from 'vue-media-annotator/alignedView/transform'; import { unresolvedCameras } from 'vue-media-annotator/alignedView/alignedView'; import { buildPerCameraRegistrationFiles } from 'vue-media-annotator/alignedView/cameraRegistrationFiles'; +import { MANUAL_SOURCE } from 'vue-media-annotator/alignedView/CameraRegistrationStore'; import TooltipBtn from 'vue-media-annotator/components/TooltipButton.vue'; +import { injectAggregateController } from 'vue-media-annotator/components/annotators/useMediaController'; import { useApi } from 'dive-common/apispec'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; +import { AutoRegisterRunOptions, useAutoRegisterJob } from 'dive-common/use/useAutoRegisterJob'; +import { loopClosureResidual, Matrix3 } from 'vue-media-annotator/alignedView/homography'; + +import RegistrationFrameList, { FrameRow } from './RegistrationFrameList.vue'; +import AutoRegisterDialog from './AutoRegisterDialog.vue'; +/** + * A solved triplet is "consistent" when the direct and routed transforms + * agree to within this fraction of the last camera's width. 0.001 keeps the + * behaviour the old fixed 5px threshold happened to have on a ~5000px-wide + * camera, while meaning the same thing on rigs whose cameras are far larger + * or smaller. Measured on a KAMERA calibration flight: a converged 10-100 + * frame fit sits near 0.05%, a single-frame (overfit) rig at 0.15%. + */ +const LOOP_CLOSURE_MAX_FRACTION = 0.001; export default defineComponent({ name: 'CameraRegistration', description: 'Camera Registration', - components: { TooltipBtn }, + components: { AutoRegisterDialog, TooltipBtn, RegistrationFrameList }, setup() { const cameraStore = useCameraStore(); const registration = useCameraRegistration(); @@ -28,8 +44,18 @@ export default defineComponent({ const alignedView = useAlignedView(); const { saveConfig } = useApi(); const { prompt } = usePrompt(); + const aggregateController = injectAggregateController(); - const cameras = computed(() => [...cameraStore.camMap.value.keys()]); + /** + * The rig in persisted display order. Not camMap's key order: that is + * insertion order from an awaited per-camera load loop and can carry + * entries across a dataset switch, and both the reference camera below + * and the direction of the loop-closure residual depend on which camera + * is first and last. + */ + // orderedCameraNames reads camMap and displayOrder, so the computed picks + // up both as dependencies without touching them explicitly. + const cameras = computed(() => cameraStore.orderedCameraNames()); /** * Per-camera alignment status for the whole rig, driving the status block: * the first camera (display order) is the reference (identity); every other @@ -64,7 +90,7 @@ export default defineComponent({ return { icon: complete ? 'mdi-check-circle' : 'mdi-alert', color: complete ? 'success' : 'warning', - text: `${total - unresolvedCount}/${total} cameras registered`, + text: `${total - unresolvedCount}/${total} cameras ready`, }; }); const camLeft = ref(null); @@ -79,7 +105,7 @@ export default defineComponent({ /** * Author-vs-review posture: picking defaults on for a pair that still * needs points, and off for one whose transform came from a registration - * file (review it; the "Pick points" toggle opts back in to refine). + * file (review it; the "Edit points" toggle opts back in to refine). * Re-applied whenever the active pair changes identity, so it overrides a * manual toggle on pair switch -- each pair opens in its own posture. */ @@ -137,8 +163,238 @@ export default defineComponent({ const activeKey = computed(() => registration.activePairKey()); const correspondences = computed(() => { const key = activeKey.value; - return key ? (registration.correspondences.value[key] || []) : []; + // Pooled across every enabled observation -- the fit input. + return key ? registration.enabledPoints(key) : []; + }); + /** Pooled fit quality: per-frame agreement with the pair's consensus. */ + const pairStats = computed(() => ( + activeKey.value ? registration.pairFitStats(activeKey.value) : null)); + /** The camA-space frame the viewer currently displays for this pair. */ + const currentPairFrame = computed(() => { + // Touch currentFrame so scrubbing recomputes the readouts. + // eslint-disable-next-line no-void + void registration.currentFrame.value; + const key = activeKey.value; + return key !== null ? registration.currentFrameForPair(key) : null; + }); + /** + * The registration-frames list: the multi-image-pair selector AND the + * quality readout, one row per observation with its agreement dot. + */ + const frameRows = computed(() => { + const key = activeKey.value; + if (!key) { + return []; + } + const stats = registration.pairFitStats(key); + const rmsByIdentity = new Map(stats.perObservation.map((obs) => [ + `${obs.imageA}::${obs.imageB}`, obs.rmsPx, + ])); + // camA-local drives every action in this panel, but the number shown must + // match the frame readout / scrubber, which count global slots. + const [camA] = key.split('::'); + const toSlot = (frame: number | null) => ( + frame === null ? null : aggregateController.value.cameraFrameToSlot(camA, frame) ?? frame + ); + return registration.framesForPair(key).map((row) => ({ + frame: row.frame, + displayFrame: toSlot(row.frame), + imageA: row.imageA, + imageB: row.imageB, + enabled: row.enabled, + source: row.source, + count: row.count, + rmsPx: rmsByIdentity.get(`${row.imageA}::${row.imageB}`) ?? null, + skipped: (row.stats && typeof row.stats.skipped === 'string') + ? row.stats.skipped as string : null, + current: row.frame !== null && row.frame === currentPairFrame.value, + })); + }); + /** + * Skipped rows are candidates a producer rejected -- overwhelmingly + * "pruned", the oversampling remainder from proposing candidatesPerBin per + * bin and keeping the best. They carry no points and support no action, so + * they are counted, not listed. + */ + const skippedCount = computed(() => frameRows.value.filter((row) => row.skipped).length); + const listedRows = computed(() => frameRows.value.filter((row) => !row.skipped)); + const autoRows = computed(() => listedRows.value.filter((row) => row.source !== MANUAL_SOURCE)); + const manualRows = computed( + () => listedRows.value.filter((row) => row.source === MANUAL_SOURCE), + ); + /** Enabled count + mean agreement with the pooled fit, for a section header. */ + function sectionSummary(rows: FrameRow[]) { + const enabled = rows.filter((row) => row.enabled); + const measured = enabled + .map((row) => row.rmsPx) + .filter((rms): rms is number => rms !== null); + return { + total: rows.length, + enabled: enabled.length, + rmsPx: measured.length + ? measured.reduce((sum, rms) => sum + rms, 0) / measured.length + : null, + }; + } + const autoSummary = computed(() => sectionSummary(autoRows.value)); + const manualSummary = computed(() => sectionSummary(manualRows.value)); + + /** + * Captures queued for the next matcher run, as global aligned-timeline + * slots (what the frame readout shows). The user picks the captures, so + * the run matches exactly these rather than proposing a spread. + */ + const queuedSlots = ref([]); + const currentSlot = computed(() => { + const pair = registration.activePair.value; + const frame = currentPairFrame.value; + if (!pair || frame === null) { + return null; + } + return aggregateController.value.cameraFrameToSlot(pair.camA, frame) ?? frame; }); + const currentQueued = computed(() => ( + currentSlot.value !== null && queuedSlots.value.includes(currentSlot.value) + )); + function queueCurrentFrame() { + const slot = currentSlot.value; + if (slot !== null && !queuedSlots.value.includes(slot)) { + queuedSlots.value = [...queuedSlots.value, slot].sort((a, b) => a - b); + } + } + function unqueueSlot(slot: number) { + queuedSlots.value = queuedSlots.value.filter((queued) => queued !== slot); + } + /** + * Every observation of a section, INCLUDING the skipped ones the list + * hides: they are still stored and still travel into the saved + * registration file, so a "clear all" that left them behind would leave + * invisible cruft the user has no way to see or remove. + */ + const autoAll = computed( + () => frameRows.value.filter((row) => row.source !== MANUAL_SOURCE), + ); + const manualAll = computed( + () => frameRows.value.filter((row) => row.source === MANUAL_SOURCE), + ); + async function clearRows(rows: FrameRow[], label: string) { + const key = activeKey.value; + if (!key || !rows.length) { + return; + } + const doomed = [...rows]; + const points = doomed.reduce((sum, row) => sum + row.count, 0); + const hidden = doomed.filter((row) => row.skipped).length; + const confirmed = await prompt({ + title: `Clear ${label} Frames`, + text: `Remove all ${doomed.length} ${label.toLowerCase()} frame(s)` + + `${hidden ? ` (${hidden} hidden as rejected candidates)` : ''}` + + ` and their ${points} point pair(s) from this pair's registration?`, + positiveButton: 'Remove all', + negativeButton: 'Cancel', + confirm: true, + }); + if (!confirmed) { + return; + } + doomed.forEach( + (row) => registration.removeObservation(key, row.imageA, row.imageB, row.source), + ); + } + const clearAuto = () => clearRows(autoAll.value, 'Auto'); + const clearManual = () => clearRows(manualAll.value, 'Manual'); + function clearQueue() { + queuedSlots.value = []; + } + function runQueuedFrames() { + autoRegisterJob?.run({ + maxFrames: queuedSlots.value.length, + candidatesPerBin: 1, + slots: [...queuedSlots.value], + }); + queuedSlots.value = []; + } + + /** Point pairs picked/matched on the frame currently being viewed. */ + const frameCorrespondences = computed(() => { + const key = activeKey.value; + if (!key) { + return []; + } + return registration.correspondencesForFrame(key, currentPairFrame.value); + }); + function toggleFrameRow(row: FrameRow, enabled: boolean) { + const key = activeKey.value; + if (key) { + registration.setObservationEnabled(key, row.imageA, row.imageB, enabled); + } + } + /** + * Seek a camA-local frame. markerFrames / currentPairFrame are all in the + * active pair's camA space (see CameraRegistrationStore.currentPairFrame), + * so every seek out of this panel goes through camA rather than whichever + * camera happens to be selected. + */ + function seekPairFrame(frame: number) { + const pair = registration.activePair.value; + if (!pair) { + return; + } + aggregateController.value.seekCameraFrame(pair.camA, frame); + } + function jumpToFrame(frame: number) { + // Observation frames are camA-local, but handler.seekFrame() interprets + // its argument in the SELECTED camera's local space -- wrong whenever + // the rig's cameras drop frames independently. Seek camA's own frame + // instead; seekCameraFrame translates through the aligned timeline so + // every camera lands on the same capture. + seekPairFrame(frame); + } + async function removeFrameRow(row: FrameRow) { + const key = activeKey.value; + if (!key) { + return; + } + if (row.count > 0) { + const confirmed = await prompt({ + title: 'Remove Registration Frame', + text: `Remove the ${row.count} point pair(s) from ` + + `${row.displayFrame !== null ? `frame ${row.displayFrame}` : 'this frame'}? ` + + 'To exclude the frame from the fit without deleting its points, ' + + 'uncheck it instead.', + positiveButton: 'Remove', + negativeButton: 'Cancel', + confirm: true, + }); + if (!confirmed) { + return; + } + } + registration.removeObservation(key, row.imageA, row.imageB, row.source); + } + /** Frames carrying registration points, for prev/next navigation. */ + const markerFrames = computed(() => frameRows.value + .map((row) => row.frame) + .filter((frameNum): frameNum is number => frameNum !== null) + .sort((a, b) => a - b)); + function seekPrevMarker() { + const current = currentPairFrame.value ?? 0; + const prev = [...markerFrames.value].reverse().find((frameNum) => frameNum < current); + if (prev !== undefined) { + seekPairFrame(prev); + } + } + function seekNextMarker() { + const current = currentPairFrame.value ?? 0; + const next = markerFrames.value.find((frameNum) => frameNum > current); + if (next !== undefined) { + seekPairFrame(next); + } + } + /** One-click way to start contributing the current frame: enable picking. */ + function addCurrentFrame() { + registration.pickingEnabled.value = true; + } const transformType = computed( () => (activeKey.value ? registration.transformTypeForPair(activeKey.value) @@ -202,10 +458,14 @@ export default defineComponent({ }; } if (canFit.value) { + const stats = pairStats.value; + const frames = stats ? stats.frameCount : 0; + const rms = stats && stats.rmsPx !== null ? ` — rms ${stats.rmsPx.toFixed(1)} px` : ''; return { icon: 'mdi-check-circle', color: fitQualityColor.value, - text: `Transform fit from ${correspondences.value.length} point pairs`, + text: `Transform fit from ${frames} frame${frames === 1 ? '' : 's'} / ` + + `${correspondences.value.length} point pairs${rms}`, }; } return { @@ -323,7 +583,7 @@ export default defineComponent({ const nextFiles = new Map(buildPerCameraRegistrationFiles( { homographies: registration.homographies.value, - correspondences: registration.correspondences.value, + observations: registration.observations.value, transformTypes: registration.transformTypes.value, source: registration.source.value, }, @@ -359,7 +619,7 @@ export default defineComponent({ try { await saveConfig(datasetId.value, { cameraHomographies: registration.homographies.value, - cameraCorrespondences: registration.correspondences.value, + cameraCorrespondences: registration.observations.value, cameraTransformTypes: registration.transformTypes.value, cameraRegistrationSource: registration.source.value, }); @@ -369,6 +629,104 @@ export default defineComponent({ } } + /** + * Auto Register Frames: launch the align_cameras pipeline over a + * stratified spread of candidate frames (one job registers the whole + * rig; a triplet solves up to three pairs at once). The service is + * provided by the viewer; availability tracks whether the align pipes + * are installed, which hides the button entirely when they aren't. + */ + const autoRegisterJob = useAutoRegisterJob(); + const autoRegisterAvailable = computed(() => !!autoRegisterJob?.available.value); + const autoRegistering = computed(() => !!autoRegisterJob?.running.value); + const autoRegisterError = computed(() => autoRegisterJob?.error.value ?? null); + const autoRegisterStatus = computed(() => autoRegisterJob?.status.value ?? null); + const autoRegisterDialog = ref(false); + + function openAutoRegisterDialog() { + autoRegisterDialog.value = true; + } + function runAutoRegister(options: AutoRegisterRunOptions) { + autoRegisterJob?.run(options); + } + + /** + * Rig-level consistency readout for a solved triplet: with all three + * pairs fitted, compare the direct A->C transform against the A->B->C + * route. This is the whole reason to solve the redundant third pair -- + * three individually plausible fits can still disagree as a rig. + * + * The residual comes out in the LAST camera's native pixels (the grid is + * pushed from camera 1 into camera 3), so both halves of this have to + * respect real image sizes: + * + * - Sample over camera 1's actual frame. The default nominal 1000x1000 + * grid only covers a corner of a 12768x9564 EO frame, so it measured + * agreement over a fraction of the field of view and missed exactly + * the divergence at the edges that matters. + * - Judge the result relative to camera 3's width, not against a fixed + * pixel count. The same rig error reads ~7.6x larger against a + * 4864-wide UV camera than a 640-wide IR one, so a fixed threshold + * silently means something different per rig -- and flags a rig + * inconsistent for nothing more than having a large last camera. + */ + /** + * A camera's native frame size, or null until its annotator has actually + * drawn a frame. originalBounds starts life as a 1x1 placeholder + * (useMediaController's reactive state), so "not ready" has to be + * detected by an implausibly small box rather than a zero -- a 1x1 box + * read as a real size collapses the loop-closure sample grid onto a + * single corner pixel and yields a meaningless residual. + */ + function nativeSize(camera: string): [number, number] | null { + try { + const bounds = aggregateController.value.getController(camera).originalBounds.value; + const width = bounds.right - bounds.left; + const height = bounds.bottom - bounds.top; + return (width > 1 && height > 1) ? [width, height] : null; + } catch { + return null; + } + } + const loopClosure = computed(() => { + const list = cameras.value; + if (list.length !== 3) { + return null; + } + const directed = (a: string, b: string): Matrix3 | null => { + const forward = registration.homographies.value[registration.pairKey(a, b)]; + if (forward) { + return forward.AtoB; + } + const reverse = registration.homographies.value[registration.pairKey(b, a)]; + return reverse ? reverse.BtoA : null; + }; + const h01 = directed(list[0], list[1]); + const h12 = directed(list[1], list[2]); + const h02 = directed(list[0], list[2]); + if (!h01 || !h12 || !h02) { + return null; + } + // Both ends must be measurable: the grid is sampled over the source + // camera's frame and the residual judged against the target camera's + // width, so a placeholder size on either side makes the ratio + // meaningless. Report nothing while the panes are still coming up + // rather than a confident-looking wrong verdict. + const sourceSize = nativeSize(list[0]); + const targetSize = nativeSize(list[2]); + if (!sourceSize || !targetSize) { + return null; + } + const residual = loopClosureResidual(h01, h12, h02, sourceSize); + const fraction = residual.meanPx / targetSize[0]; + return { + ...residual, + fraction, + consistent: fraction <= LOOP_CLOSURE_MAX_FRACTION, + route: `${list[0]}↔${list[2]} vs ${list[0]}↔${list[1]}↔${list[2]}`, + }; + }); + return { cameras, cameraAlignmentStatuses, @@ -383,6 +741,33 @@ export default defineComponent({ deleteSelectedCorrespondence, cursorReadout, correspondences, + pairStats, + currentPairFrame, + frameRows, + frameCorrespondences, + markerFrames, + toggleFrameRow, + jumpToFrame, + removeFrameRow, + seekPrevMarker, + seekNextMarker, + addCurrentFrame, + skippedCount, + autoRows, + manualRows, + autoSummary, + manualSummary, + queuedSlots, + currentSlot, + currentQueued, + queueCurrentFrame, + unqueueSlot, + runQueuedFrames, + autoAll, + manualAll, + clearAuto, + clearManual, + clearQueue, transformType, transformTypeItems: TRANSFORM_TYPES, minPoints, @@ -404,6 +789,14 @@ export default defineComponent({ setTransformType, setAlignmentMode, save, + autoRegisterAvailable, + autoRegistering, + autoRegisterError, + autoRegisterStatus, + autoRegisterDialog, + openAutoRegisterDialog, + runAutoRegister, + loopClosure, }; }, }); @@ -429,14 +822,23 @@ export default defineComponent({ > Source: {{ sourceReadout }} + This pair has been refined in-app since the source registration was - produced. Save, then download the camera's registration from the - Export menu to hand the refinement (and its points) back to the - producer. + produced. + +
+
+ + {{ loopClosure.consistent ? 'mdi-vector-triangle' : 'mdi-alert' }} + + + +
@@ -522,6 +946,226 @@ export default defineComponent({ points is optional: fitting {{ minPoints }} or more pairs replaces it. + + + + + {{ autoRegisterError }} + + + {{ autoRegisterStatus }} + + - Correspondences ({{ correspondences.length }}) + Correspondences on frame + {{ currentPairFrame !== null ? currentPairFrame : '—' }} + ({{ frameCorrespondences.length }})
@@ -579,7 +1225,7 @@ export default defineComponent({ />
@@ -594,7 +1240,7 @@ export default defineComponent({ - No correspondences yet. At least {{ minPoints }} required for the selected transform. + No correspondences on this frame yet + ({{ correspondences.length }} total across all frames; at least + {{ minPoints }} required for the selected transform).
diff --git a/client/dive-common/components/ControlsContainer.vue b/client/dive-common/components/ControlsContainer.vue index f80139917..3baade77c 100644 --- a/client/dive-common/components/ControlsContainer.vue +++ b/client/dive-common/components/ControlsContainer.vue @@ -14,9 +14,11 @@ import { Timeline, } from 'vue-media-annotator/components'; import { clientSettings } from 'dive-common/store/settings'; +import context from 'dive-common/store/context'; import { useHandler, useAttributesFilters, + useCameraRegistration, useCameraStore, useSelectedCamera, useTime, @@ -131,6 +133,49 @@ export default defineComponent({ const { volume, setVolume, setSpeed, speed, } = aggregateController.value; + /** + * Registration-frame markers for the Timeline work-area, shown ONLY + * while the Camera Registration panel is open (the same signal the + * viewer's registrationActive keys off) -- outside that tab the timeline + * stays exactly as it is today. + * + * Observation frames are camA-local, but the Timeline draws in the + * SELECTED camera's local frame space. Those two spaces only coincide + * when the rig drops no frames (or when camA happens to be selected): + * a rig whose cameras drop frames independently accumulates an offset, + * putting every marker a frame or two off. Translate through the aligned + * timeline, and drop markers whose capture has no frame on the selected + * camera -- there is no honest place to draw those. + */ + const cameraRegistration = useCameraRegistration(); + const registrationMarkers = computed(() => { + if (context.state.active !== 'CameraRegistration') { + return []; + } + const key = cameraRegistration.activePairKey(); + // Touch observations so edits recompute the markers. + // eslint-disable-next-line no-void + void cameraRegistration.observations.value; + if (!key) { + return []; + } + const [camA] = key.split('::'); + const target = selectedCamera.value; + return cameraRegistration.framesForPair(key) + // Only frames that actually carry points. A producer records the + // candidates it considered and discarded too (auto-register proposes + // more frames than it matches, then prunes) -- those have no points + // and nothing to toggle, so a marker for them is just noise on the + // scrubber. The frame list still lists them with their skip reason. + .filter((row) => row.frame !== null && row.count > 0) + .map((row) => ({ + frame: aggregateController.value.translateCameraFrame(camA, row.frame as number, target), + enabled: row.enabled, + })) + .filter((marker): marker is { frame: number; enabled: boolean } => ( + marker.frame !== undefined + )); + }); // The timeline charts (line/event charts) are built from trackStores in // the selected camera's own local frame space. Under an aligned timeline // (SEAL feature 5) the aggregate controller's frame/maxFrame/seek operate @@ -167,6 +212,7 @@ export default defineComponent({ ticks, hasGroups, attributeData, + registrationMarkers, timelineEnabled, activeCountSettings, clientSettings, @@ -466,6 +512,7 @@ export default defineComponent({ :display="!collapsed" :dataset-type="datasetType" :bottom-layout="bottomLayout" + :markers="registrationMarkers" @seek="seek" >