From 8bb338f05f5afbe586e765102d1b22705b6b2bf0 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Wed, 15 Jul 2026 11:27:06 -0400 Subject: [PATCH 1/9] Pass DIVE camera registrations to 2-cam/3-cam VIAME pipelines Multicam pipelines previously received no transform information from DIVE; the old h5 workflow relied on paths hardcoded inside the pipes. Desktop and web now hand the dataset's camera registration to the pipeline's warp processes, mirroring the stereo calibration flow: - One standard _to__registration.json per non-reference camera is written into the job work dir; warpN (matching the pipeline camera position) receives its file via -s warpN:transformation_file=..., with the pair and direction pinned through the reader's from_camera/to_camera config since a pair may be stored in either orientation. - Pipeline camera order is the registration reference first (the pipes warp everything onto camera 1's frame), then display order. Which detector a pipe runs on which input is the pipe's documented contract, not something DIVE infers. - Only pairs registering a camera directly onto the reference are supported; pairs between two non-reference cameras are explicitly unsupported for pipelines and never reach the job. Cameras with points-only (unfitted) pairs get no settings. - Web sends the registration as a multicam_registration job param built from dataset meta; the worker writes the files. Requires VIAME's dive transform_2d_io reader and warp_detections process. Co-Authored-By: Claude Fable 5 --- client/dive-common/multicamDisplay.ts | 13 ++ .../backend/native/cameraRegistration.spec.ts | 192 ++++++++++++++++++ .../backend/native/cameraRegistration.ts | 75 ++++++- .../desktop/backend/native/multiCamUtils.ts | 20 +- .../platform/desktop/backend/native/viame.ts | 12 +- server/dive_server/crud_rpc.py | 29 ++- server/dive_tasks/multicam_pipeline.py | 108 +++++++++- server/dive_tasks/run_pipeline.py | 9 + server/dive_utils/types.py | 12 ++ server/tests/test_multicam_pipeline.py | 111 ++++++++++ 10 files changed, 574 insertions(+), 7 deletions(-) create mode 100644 client/platform/desktop/backend/native/cameraRegistration.spec.ts diff --git a/client/dive-common/multicamDisplay.ts b/client/dive-common/multicamDisplay.ts index e51ea0819..21b1da6b8 100644 --- a/client/dive-common/multicamDisplay.ts +++ b/client/dive-common/multicamDisplay.ts @@ -57,6 +57,19 @@ export function referenceCameraName(multiCamMedia: MultiCamMediaLike | null | un return defaultDisplay && ordered.includes(defaultDisplay) ? defaultDisplay : ordered[0]; } +/** + * Camera order for 2-cam/3-cam VIAME pipelines: the registration reference + * camera feeds input1 (the per-camera registrations all map onto the + * reference, and the pipes warp everything onto camera 1's frame), remaining + * cameras keep display order. Which detector a pipe runs on which input is + * the pipe's documented contract, not something DIVE infers. + */ +export function pipelineOrderedCameraNames(multiCamMedia: MultiCamMediaLike | null | undefined): string[] { + const ordered = orderedMultiCamCameraNames(multiCamMedia); + const reference = referenceCameraName(multiCamMedia); + return reference ? [reference, ...ordered.filter((name) => name !== reference)] : ordered; +} + export function isMultiCamSubType(subType: SubType | string | null | undefined): subType is MultiCamSubType { return subType === 'stereo' || subType === 'multicam'; } diff --git a/client/platform/desktop/backend/native/cameraRegistration.spec.ts b/client/platform/desktop/backend/native/cameraRegistration.spec.ts new file mode 100644 index 000000000..3b151bb73 --- /dev/null +++ b/client/platform/desktop/backend/native/cameraRegistration.spec.ts @@ -0,0 +1,192 @@ +import mockfs from 'mock-fs'; +import npath from 'path'; +import fs from 'fs-extra'; +import { + it, expect, describe, afterAll, vi, +} from 'vitest'; + +import { Settings, JsonMeta } from 'platform/desktop/constants'; +import { buildRegistrationPipelineArgs } from './cameraRegistration'; + +// mock-fs no longer intercepts fs-extra's exists checks on newer Node; +// route them through statSync like common.spec.ts does. +vi.mock('fs-extra', async () => { + const actual = await vi.importActual('fs-extra'); + const fsNode = await import('node:fs'); + const existsByStat = (targetPath: fsNode.PathLike) => { + try { + fsNode.statSync(targetPath); + return true; + } catch { + return false; + } + }; + + const patchedDefault = { + ...actual.default, + existsSync: existsByStat, + pathExistsSync: existsByStat, + }; + + return { + ...actual, + default: patchedDefault, + existsSync: existsByStat, + pathExistsSync: existsByStat, + }; +}); + +const settings: Settings = { + version: 1, + dataPath: '/home/user/viamedata', + viamePath: '/opt/viame', + readonlyMode: false, + overrides: {}, +}; + +const identity = [[1, 0, 0], [0, 1, 0], [0, 0, 1]]; +const irToRgb = [[1, 0, 5], [0, 1, -3], [0, 0, 1]]; +const uvToRgb = [[1, 0, -8], [0, 1, 2], [0, 0, 1]]; + +function registrationFile(pairs: unknown[]) { + return JSON.stringify({ type: 'dive-camera-registration', version: 1, pairs }); +} + +const projectFiles = { 'meta.json': '{}', 'result_1.json': '{}' }; + +mockfs({ + [npath.join(settings.dataPath, 'DIVE_Projects')]: { + withreg: { + ...projectFiles, + 'ir_to_rgb_registration.json': registrationFile([ + { + left: 'ir', right: 'rgb', points: [], leftToRight: irToRgb, rightToLeft: identity, + }, + ]), + 'uv_to_rgb_registration.json': registrationFile([ + { + left: 'uv', right: 'rgb', points: [], leftToRight: uvToRgb, rightToLeft: identity, + }, + ]), + }, + partial: { + ...projectFiles, + 'ir_to_rgb_registration.json': registrationFile([ + { + left: 'ir', right: 'rgb', points: [], leftToRight: irToRgb, rightToLeft: identity, + }, + ]), + // Non-star pair (two non-reference cameras): explicitly unsupported + // for pipelines even though fitted. + 'uv_to_ir_registration.json': registrationFile([ + { + left: 'uv', right: 'ir', points: [], leftToRight: uvToRgb, rightToLeft: identity, + }, + ]), + }, + noreg: { ...projectFiles }, + seeded: { ...projectFiles }, + }, + '/home/user/job': {}, +}); + +afterAll(() => mockfs.restore()); + +function multiCamMeta(id: string, cameras: string[], defaultDisplay: string): JsonMeta { + return { + version: 1, + type: 'multi', + id, + fps: 1, + originalBasePath: '/home/user/data', + originalImageFiles: [], + originalVideoFile: '', + transcodedVideoFile: '', + transcodedImageFiles: [], + name: id, + createdAt: 'now', + subType: 'multicam', + multiCam: { + cameras: Object.fromEntries(cameras.map((name) => [name, { + type: 'image-sequence', + originalBasePath: `/home/user/data/${name}`, + originalImageFiles: [], + originalVideoFile: '', + transcodedVideoFile: '', + transcodedImageFiles: [], + }])), + defaultDisplay, + }, + } as unknown as JsonMeta; +} + +describe('buildRegistrationPipelineArgs', () => { + it('writes one file per camera pair and pins each warp pair/direction', async () => { + // Display order intentionally scrambles the input: the pipeline order is + // reference-first then display order (rgb, uv, ir -- IR displays last), + // so uv lands on input2 / warp2 and ir on input3 / warp3. + const meta = multiCamMeta('withreg', ['ir', 'rgb', 'uv'], 'rgb'); + const jobWorkDir = '/home/user/job/full'; + await fs.ensureDir(jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + + const irPath = npath.join(jobWorkDir, 'ir_to_rgb_registration.json'); + const uvPath = npath.join(jobWorkDir, 'uv_to_rgb_registration.json'); + expect(args).toStrictEqual({ + 'warp2:transformation_file': uvPath, + 'warp2:transform_reader:type': 'dive', + 'warp2:transform_reader:dive:from_camera': 'uv', + 'warp2:transform_reader:dive:to_camera': 'rgb', + 'warp3:transformation_file': irPath, + 'warp3:transform_reader:type': 'dive', + 'warp3:transform_reader:dive:from_camera': 'ir', + 'warp3:transform_reader:dive:to_camera': 'rgb', + }); + const written = await fs.readJSON(irPath); + expect(written.type).toBe('dive-camera-registration'); + expect(written.pairs).toHaveLength(1); + expect(written.pairs[0].left).toBe('ir'); + expect(written.pairs[0].right).toBe('rgb'); + expect(written.pairs[0].leftToRight).toStrictEqual(irToRgb); + expect((await fs.readJSON(uvPath)).pairs[0].left).toBe('uv'); + }); + + it('skips cameras without a fitted reference pair; non-star pairs never reach the job', async () => { + const meta = multiCamMeta('partial', ['rgb', 'ir', 'uv'], 'rgb'); + const jobWorkDir = '/home/user/job/partial'; + await fs.ensureDir(jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + // Pipeline order is rgb, uv, ir: uv (warp2) has only the unsupported + // uv-to-ir pair, so it gets nothing; ir (warp3) has a reference pair. + expect(Object.keys(args).some((key) => key.startsWith('warp2'))).toBe(false); + expect(args['warp3:transform_reader:dive:from_camera']).toBe('ir'); + const irPath = npath.join(jobWorkDir, 'ir_to_rgb_registration.json'); + expect(args['warp3:transformation_file']).toBe(irPath); + // The uv-to-ir pair is dropped from ir's job file too. + const written = await fs.readJSON(irPath); + expect(written.pairs).toHaveLength(1); + expect(written.pairs[0].right).toBe('rgb'); + expect(await fs.readdir(jobWorkDir)).toStrictEqual(['ir_to_rgb_registration.json']); + }); + + it('returns no args when the dataset has no registration', async () => { + const meta = multiCamMeta('noreg', ['rgb', 'ir'], 'rgb'); + const jobWorkDir = '/home/user/job/noreg'; + await fs.ensureDir(jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + expect(args).toStrictEqual({}); + expect(await fs.readdir(jobWorkDir)).toStrictEqual([]); + }); + + it('falls back to the import-time seed in dataset meta when no files exist', async () => { + const meta = multiCamMeta('seeded', ['rgb', 'ir'], 'rgb'); + meta.cameraHomographies = { + 'ir::rgb': { AtoB: irToRgb, BtoA: identity }, + }; + const jobWorkDir = '/home/user/job/seeded'; + await fs.ensureDir(jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + expect(args['warp2:transform_reader:dive:from_camera']).toBe('ir'); + expect(args['warp2:transform_reader:dive:to_camera']).toBe('rgb'); + }); +}); diff --git a/client/platform/desktop/backend/native/cameraRegistration.ts b/client/platform/desktop/backend/native/cameraRegistration.ts index 157e247e4..9b70e357c 100644 --- a/client/platform/desktop/backend/native/cameraRegistration.ts +++ b/client/platform/desktop/backend/native/cameraRegistration.ts @@ -12,12 +12,16 @@ import fs from 'fs-extra'; import { TransformType, DEFAULT_TRANSFORM_TYPE } from 'vue-media-annotator/alignedView/transform'; import { buildPerCameraRegistrationFiles, registrationValuesSummary, filterRegistrationValues, - mergeRegistrationValues, mergeRegistrationSources, CameraRegistrationValues, + mergeRegistrationValues, mergeRegistrationSources, registrationFileName, + CameraRegistrationValues, } from 'vue-media-annotator/alignedView/cameraRegistrationFiles'; import { readTransformMatrix } from 'vue-media-annotator/alignedView/alignedView'; import { invert3, Matrix3 } from 'vue-media-annotator/alignedView/homography'; import { DatasetConfigMutable } from 'dive-common/apispec'; -import { referenceCameraName as multicamReferenceCameraName } from 'dive-common/multicamDisplay'; +import { + referenceCameraName as multicamReferenceCameraName, + pipelineOrderedCameraNames, +} from 'dive-common/multicamDisplay'; import { RegistrationFileNamePattern, compareRegistrationCandidates, @@ -191,6 +195,73 @@ export async function loadEffectiveRegistration( }; } +/** + * Build the kwiver -s settings that hand a dataset's camera registration to + * a 2-cam/3-cam pipeline. One standard _to__registration + * file per non-reference camera is written into the job work dir; each + * camera's warp process (warp2, warp3, ... matching the pipeline's + * reference-first camera order) receives its own single-pair file. The pair + * and direction are still pinned through the reader's from_camera/to_camera + * config, since a pair may be stored in either orientation. + * + * Only pairs registering a camera directly onto the reference are + * supported: pairs between two non-reference cameras are explicitly + * unsupported here (there is no transform composition) and never reach the + * pipeline. Cameras without a fitted reference pair get no settings; a + * pipeline that needs one then fails at configure time with the file name + * it was missing. + */ +export async function buildRegistrationPipelineArgs( + settings: Settings, + meta: JsonConfig, + jobWorkDir: string, +): Promise> { + const args: Record = {}; + if (!meta.multiCam) { + return args; + } + const projectDirInfo = await getValidatedProjectDir(settings, meta.id); + const values = await loadEffectiveRegistration(projectDirInfo.basePath, meta); + const reference = multicamReferenceCameraName(meta.multiCam); + if (!reference) { + return args; + } + const files = buildPerCameraRegistrationFiles(values, reference); + const writes: Promise[] = []; + pipelineOrderedCameraNames(meta.multiCam).forEach((camera, index) => { + if (index === 0 || camera === reference) { + return; + } + // Points-only pairs have no matrix the warp could apply. + const fitted = values.homographies[`${camera}::${reference}`] + || values.homographies[`${reference}::${camera}`]; + if (!fitted) { + return; + } + const file = files.find((candidate) => candidate.camera === camera); + if (!file) { + return; + } + // Unsupported non-reference pairs are dropped from the file body too, so + // the job dir only ever holds camera-to-reference registrations. + const referencePairs = file.body.pairs.filter( + (pair) => pair.left === reference || pair.right === reference, + ); + if (!referencePairs.length) { + return; + } + const registrationPath = npath.join(jobWorkDir, registrationFileName(camera, reference)); + writes.push(writeJsonFile(registrationPath, { ...file.body, pairs: referencePairs })); + const warp = `warp${index + 1}`; + args[`${warp}:transformation_file`] = registrationPath; + args[`${warp}:transform_reader:type`] = 'dive'; + args[`${warp}:transform_reader:dive:from_camera`] = camera; + args[`${warp}:transform_reader:dive:to_camera`] = reference; + }); + await Promise.all(writes); + return args; +} + /** * Persist camera registration to standalone per-camera files in a dataset * directory, merging partial updates with whatever is already on disk. diff --git a/client/platform/desktop/backend/native/multiCamUtils.ts b/client/platform/desktop/backend/native/multiCamUtils.ts index 1ddcf1166..cd0b1134b 100644 --- a/client/platform/desktop/backend/native/multiCamUtils.ts +++ b/client/platform/desktop/backend/native/multiCamUtils.ts @@ -10,6 +10,7 @@ import { JsonConfig, Settings } from 'platform/desktop/constants'; import { loadAnnotationFile, loadJsonConfig, getValidatedProjectDir } from 'platform/desktop/backend/native/common'; import { serialize } from 'platform/desktop/backend/serializers/viame'; import { parseFrameTimestamp } from 'dive-common/frameTimestamp'; +import { pipelineOrderedCameraNames } from 'dive-common/multicamDisplay'; /** * Figure out the destination location @@ -73,11 +74,25 @@ function getTranscodedMultiCamType(imageListFile: string, jsonConfig: JsonConfig throw new Error(`No associate type for ${imageListFile} in multiCam data`); } -async function writeMultiCamStereoPipelineArgs(jobWorkDir: string, meta: JsonConfig, settings: Settings, utility = false, forceTranscoded = false) { +async function writeMultiCamStereoPipelineArgs( + jobWorkDir: string, + meta: JsonConfig, + settings: Settings, + utility = false, + forceTranscoded = false, + // 2-cam/3-cam pipes treat camera 1 as the reference frame the other + // cameras register onto, so their inputs go reference-first; stereo + // measurement keeps the stored left/right order. + referenceFirst = false, +) { const argFilePair: Record = {}; const outFiles: Record = {}; if (meta.multiCam && meta.multiCam.cameras) { - const cameraList = Object.entries(meta.multiCam.cameras); + const { cameras } = meta.multiCam; + const cameraNames = referenceFirst + ? pipelineOrderedCameraNames(meta.multiCam).filter((name) => name in cameras) + : Object.keys(cameras); + const cameraList = cameraNames.map((name) => [name, cameras[name]] as const); for (let i = 0; i < cameraList.length; i += 1) { const [key, list] = cameraList[i]; const { originalBasePath } = list; @@ -149,6 +164,7 @@ function getMultiCamUrls( } const multiCamMedia: MultiCamMedia = { cameras: {}, + cameraOrder: projectMetaData.multiCam.cameraOrder, defaultDisplay: projectMetaData.multiCam.defaultDisplay, }; diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index f9668b925..0053163b0 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -32,6 +32,7 @@ import { jobFileEchoMiddleware, createWorkingDirectory, createCustomWorkingDirectory, splitExt, buildTrainingExitManifest, } from './utils'; +import { buildRegistrationPipelineArgs } from './cameraRegistration'; import { getMultiCamImageFiles, getMultiCamVideoPath, writeMultiCamStereoPipelineArgs, @@ -366,7 +367,8 @@ async function runPipeline( let multiOutFiles: Record; if (meta.multiCam && stereoOrMultiCam) { - const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput); + const isMultiCamPipeline = multiCamPipelineMarkers.includes(pipeline.type); + const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput, false, isMultiCamPipeline); Object.entries(argFilePair).forEach(([arg, file]) => { command.push(`-s ${arg}="${file}"`); }); @@ -394,6 +396,14 @@ async function runPipeline( command.push(`-s ${key}="${meta.multiCam?.calibration}"`); }); } + if (isMultiCamPipeline) { + // Hand the camera registration (Aligned View homographies) to the + // pipeline's per-camera warp processes. + const registrationArgs = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + Object.entries(registrationArgs).forEach(([arg, value]) => { + command.push(`-s ${arg}="${value}"`); + }); + } } else if (pipeline.type === stereoPipelineMarker) { throw new Error('Attempting to run a multicam pipeline on non multicam data'); } diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index 70f18dfe2..5f1dfda97 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta import json +from pathlib import Path from typing import Dict, List, Literal, NamedTuple, Optional, Tuple, TypedDict, cast from girder.constants import AccessType @@ -18,7 +19,12 @@ from dive_server import crud, crud_annotation, crud_dataset from dive_tasks import tasks -from dive_tasks.multicam_pipeline import is_stereo_or_multicam_pipeline, pipeline_requires_input +from dive_tasks.multicam_pipeline import ( + build_registration_pairs, + is_stereo_or_multicam_pipeline, + pipeline_camera_order, + pipeline_requires_input, +) from dive_tasks.utils import choose_annotation_fps from dive_utils import ( TRUTHY_META_VALUES, @@ -320,12 +326,24 @@ def run_pipeline( multicam_default_display = '' calibration_item_id: Optional[str] = None default_camera_folder: Optional[types.GirderModel] = None + is_warp_pipeline = False + reference_camera = '' if dataset_type == constants.MultiType: multi_cam = fromMeta(folder, constants.MultiCamMarker, required=True) multicam_default_display = multi_cam['defaultDisplay'] camera_order = crud_dataset._multicam_camera_order(multi_cam) cameras_meta = multi_cam.get('cameras') or {} + is_warp_pipeline = pipeline['type'] in constants.MultiCamPipelineMarkers + if is_warp_pipeline and camera_order: + # 2-cam/3-cam pipes treat camera 1 as the reference frame the + # other cameras register onto; feed them reference-first. + reference_camera = ( + multicam_default_display + if multicam_default_display in cameras_meta + else camera_order[0] + ) + camera_order = pipeline_camera_order(camera_order, reference_camera) for name in camera_order: cam_info = cameras_meta[name] folder_id = cam_info.get('folderId') @@ -407,6 +425,15 @@ def run_pipeline( params['multicam_requires_input'] = multicam_requires_input if calibration_item_id: params['calibration_item_id'] = calibration_item_id + if is_warp_pipeline and reference_camera: + registration_pairs = build_registration_pairs(folder.get('meta') or {}) + if any( + pair.get('leftToRight') or pair.get('rightToLeft') for pair in registration_pairs + ): + params['multicam_registration'] = { + 'reference': reference_camera, + 'pairs': registration_pairs, + } if metadata_file_key and metadata_file_item_id: params['metadata_file_key'] = metadata_file_key params['metadata_file_item_id'] = metadata_file_item_id diff --git a/server/dive_tasks/multicam_pipeline.py b/server/dive_tasks/multicam_pipeline.py index 2094520e9..d94776a20 100644 --- a/server/dive_tasks/multicam_pipeline.py +++ b/server/dive_tasks/multicam_pipeline.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from pathlib import Path import re import shlex @@ -9,7 +10,7 @@ from dive_tasks.pipeline_creates_dataset import is_disparity_image_pipeline from dive_utils import constants -from dive_utils.types import MulticamCameraJob, PipelineDescription +from dive_utils.types import MulticamCameraJob, MulticamRegistrationJob, PipelineDescription _PIPELINE_INPUT_PATTERN = re.compile(r'utility_|filter_|transcode_|measurement_') @@ -89,6 +90,111 @@ def append_metadata_file_kwiver_settings( command.append(f'-s {shlex.quote(kwiver_key)}={shlex.quote(str(metadata_path))}') +def pipeline_camera_order(camera_names: List[str], reference: str) -> List[str]: + """ + Camera order for 2-cam/3-cam pipelines, matching the desktop client: the + registration reference camera feeds input1 (the per-camera registrations + all map onto the reference, and the pipes warp everything onto camera 1's + frame), remaining cameras keep display order. Which detector a pipe runs + on which input is the pipe's documented contract, not something DIVE + infers. + """ + if reference not in camera_names: + return camera_names + return [reference] + [name for name in camera_names if name != reference] + + +def build_registration_pairs(folder_meta: dict) -> List[dict]: + """ + Convert a dataset folder's camera registration meta (cameraHomographies / + cameraCorrespondences / cameraTransformTypes, keyed by directional + "left::right") into dive-camera-registration file pairs. + """ + homographies = folder_meta.get('cameraHomographies') or {} + correspondences = folder_meta.get('cameraCorrespondences') or {} + transform_types = folder_meta.get('cameraTransformTypes') or {} + keys = set(homographies) | set(correspondences) | set(transform_types) + pairs: List[dict] = [] + for key in sorted(keys): + left, _, right = key.partition('::') + homography = homographies.get(key) + pairs.append( + { + 'left': left, + 'right': right, + 'points': [ + [c['a'][0], c['a'][1], c['b'][0], c['b'][1]] + for c in correspondences.get(key) or [] + ], + 'leftToRight': homography.get('AtoB') if homography else None, + 'rightToLeft': homography.get('BtoA') if homography else None, + 'transformType': transform_types.get(key, 'similarity'), + } + ) + return pairs + + +def build_registration_kwiver_settings( + work_dir: Path, + cameras: List[MulticamCameraJob], + registration: MulticamRegistrationJob, +) -> Dict[str, str]: + """ + Build the -s settings handing the camera registration to a 2-cam/3-cam + pipeline. One standard _to__registration.json per + non-reference camera is written into the work dir; each camera's warp + process (warp2, warp3, ... matching the job camera order) gets its own + single-pair file. The pair and direction are still pinned via the + reader's from_camera/to_camera config, since a pair may be stored in + either orientation. + + Only pairs registering a camera directly onto the reference are + supported: pairs between two non-reference cameras are explicitly + unsupported here (there is no transform composition) and never reach the + pipeline. Cameras without a fitted reference pair get no settings. + """ + reference = registration.get('reference') + if not reference: + return {} + reference_pairs = [ + pair + for pair in registration.get('pairs') or [] + if reference in (pair['left'], pair['right']) and pair['left'] != pair['right'] + ] + pairs_by_camera: Dict[str, List[dict]] = {} + for pair in reference_pairs: + camera = pair['left'] if pair['right'] == reference else pair['right'] + pairs_by_camera.setdefault(camera, []).append(pair) + fitted = { + (pair['left'], pair['right']) + for pair in reference_pairs + if pair.get('leftToRight') or pair.get('rightToLeft') + } + settings: Dict[str, str] = {} + for index, camera in enumerate(cameras): + name = camera['name'] + if index == 0 or name == reference: + continue + if (name, reference) not in fitted and (reference, name) not in fitted: + continue + camera_pairs = pairs_by_camera.get(name) + if not camera_pairs: + continue + registration_path = work_dir / f'{name}_to_{reference}_registration.json' + with open(registration_path, 'w', encoding='utf-8') as registration_file: + json.dump( + {'type': 'dive-camera-registration', 'version': 1, 'pairs': camera_pairs}, + registration_file, + indent=2, + ) + warp = f'warp{index + 1}' + settings[f'{warp}:transformation_file'] = str(registration_path) + settings[f'{warp}:transform_reader:type'] = 'dive' + settings[f'{warp}:transform_reader:dive:from_camera'] = name + settings[f'{warp}:transform_reader:dive:to_camera'] = reference + return settings + + def build_multicam_kwiver_settings( work_dir: Path, cameras: List[MulticamCameraJob], diff --git a/server/dive_tasks/run_pipeline.py b/server/dive_tasks/run_pipeline.py index ab3a4aab2..f7819c263 100644 --- a/server/dive_tasks/run_pipeline.py +++ b/server/dive_tasks/run_pipeline.py @@ -17,6 +17,7 @@ append_metadata_file_kwiver_settings, append_stereo_calibration_kwiver_settings, build_multicam_kwiver_settings, + build_registration_kwiver_settings, find_downloaded_calibration_file, is_stereo_measurement_pipeline, ) @@ -327,6 +328,14 @@ def run_pipeline(self: Task, params: PipelineJob): for arg, file_name in arg_file_pair.items(): command.append(f"-s {shlex.quote(arg)}={shlex.quote(file_name)}") + multicam_registration = multicam_params.get('multicam_registration') + if multicam_registration: + registration_settings = build_registration_kwiver_settings( + _working_directory_path, multicam_cameras, multicam_registration + ) + for arg, value in registration_settings.items(): + command.append(f'-s {shlex.quote(arg)}={shlex.quote(value)}') + transcoded_video: Optional[str] = None if creates_new_dataset: video_name = None diff --git a/server/dive_utils/types.py b/server/dive_utils/types.py index 162187bcf..e6ddd6fe7 100644 --- a/server/dive_utils/types.py +++ b/server/dive_utils/types.py @@ -155,6 +155,17 @@ class PipelineJob(TypedDict): output_parent_folder_id: NotRequired[Optional[str]] +class MulticamRegistrationJob(TypedDict): + """Camera registration handed to a 2-cam/3-cam pipeline's warp processes. + + Pairs use the dive-camera-registration file layout: left/right camera + names, correspondence points, and leftToRight/rightToLeft 3x3 matrices. + """ + + reference: str + pairs: List[dict] + + class MulticamPipelineJob(PipelineJob, total=False): """Pipeline job fields set when running stereo/multicam pipelines on a multi dataset.""" @@ -162,6 +173,7 @@ class MulticamPipelineJob(PipelineJob, total=False): multicam_default_display: str calibration_item_id: Optional[str] multicam_requires_input: bool + multicam_registration: Optional[MulticamRegistrationJob] class TrainingJob(TypedDict): diff --git a/server/tests/test_multicam_pipeline.py b/server/tests/test_multicam_pipeline.py index 0595f2785..a4e50c4a7 100644 --- a/server/tests/test_multicam_pipeline.py +++ b/server/tests/test_multicam_pipeline.py @@ -1,12 +1,16 @@ +import json from pathlib import Path from dive_tasks.multicam_pipeline import ( DEFAULT_CALIBRATION_KEYS, append_stereo_calibration_kwiver_settings, build_multicam_kwiver_settings, + build_registration_kwiver_settings, + build_registration_pairs, find_downloaded_calibration_file, is_stereo_measurement_pipeline, is_stereo_or_multicam_pipeline, + pipeline_camera_order, pipeline_requires_input, stereo_calibration_keys, ) @@ -120,6 +124,113 @@ def test_build_multicam_kwiver_settings_image_sequence(tmp_path: Path): ) +def test_pipeline_camera_order(): + # Reference first, remaining display order preserved. + assert pipeline_camera_order(['ir', 'rgb', 'uv'], 'rgb') == ['rgb', 'ir', 'uv'] + assert pipeline_camera_order(['rgb', 'uv', 'ir'], 'rgb') == ['rgb', 'uv', 'ir'] + assert pipeline_camera_order(['CENT_IR', 'CENT_EO'], 'CENT_EO') == ['CENT_EO', 'CENT_IR'] + # Unknown reference leaves the order alone. + assert pipeline_camera_order(['a', 'b'], 'missing') == ['a', 'b'] + + +IR_TO_RGB = [[1, 0, 5], [0, 1, -3], [0, 0, 1]] +RGB_TO_IR = [[1, 0, -5], [0, 1, 3], [0, 0, 1]] + + +def test_build_registration_pairs(): + folder_meta = { + 'cameraHomographies': {'ir::rgb': {'AtoB': IR_TO_RGB, 'BtoA': RGB_TO_IR}}, + 'cameraCorrespondences': { + 'ir::rgb': [{'id': 1, 'a': [1, 2], 'b': [3, 4]}], + 'uv::rgb': [{'id': 1, 'a': [5, 6], 'b': [7, 8]}], + }, + 'cameraTransformTypes': {'ir::rgb': 'affine'}, + } + pairs = build_registration_pairs(folder_meta) + assert pairs == [ + { + 'left': 'ir', + 'right': 'rgb', + 'points': [[1, 2, 3, 4]], + 'leftToRight': IR_TO_RGB, + 'rightToLeft': RGB_TO_IR, + 'transformType': 'affine', + }, + { + 'left': 'uv', + 'right': 'rgb', + 'points': [[5, 6, 7, 8]], + 'leftToRight': None, + 'rightToLeft': None, + 'transformType': 'similarity', + }, + ] + assert build_registration_pairs({}) == [] + + +def test_build_registration_kwiver_settings(tmp_path: Path): + cameras = [ + {'name': 'rgb', 'folder_id': '1', 'media_type': constants.ImageSequenceType}, + {'name': 'ir', 'folder_id': '2', 'media_type': constants.ImageSequenceType}, + {'name': 'uv', 'folder_id': '3', 'media_type': constants.ImageSequenceType}, + ] + registration = { + 'reference': 'rgb', + 'pairs': [ + { + 'left': 'ir', + 'right': 'rgb', + 'points': [], + 'leftToRight': IR_TO_RGB, + 'rightToLeft': RGB_TO_IR, + 'transformType': 'similarity', + }, + # Points-only pair: uv has nothing fitted, so no warp3 settings. + { + 'left': 'uv', + 'right': 'rgb', + 'points': [[1, 2, 3, 4]], + 'leftToRight': None, + 'rightToLeft': None, + 'transformType': 'similarity', + }, + # Non-star pair (two non-reference cameras): explicitly + # unsupported, never reaches the pipeline even though fitted. + { + 'left': 'uv', + 'right': 'ir', + 'points': [], + 'leftToRight': IR_TO_RGB, + 'rightToLeft': RGB_TO_IR, + 'transformType': 'similarity', + }, + ], + } + settings = build_registration_kwiver_settings(tmp_path, cameras, registration) + # One file per camera pair; uv is points-only so it gets no file or settings. + registration_path = str(tmp_path / 'ir_to_rgb_registration.json') + assert settings == { + 'warp2:transformation_file': registration_path, + 'warp2:transform_reader:type': 'dive', + 'warp2:transform_reader:dive:from_camera': 'ir', + 'warp2:transform_reader:dive:to_camera': 'rgb', + } + written = json.loads((tmp_path / 'ir_to_rgb_registration.json').read_text(encoding='utf-8')) + assert written['type'] == 'dive-camera-registration' + assert len(written['pairs']) == 1 + assert written['pairs'][0]['left'] == 'ir' + # uv produced no file: its only fitted pair skips the reference. + assert list(tmp_path.iterdir()) == [tmp_path / 'ir_to_rgb_registration.json'] + + +def test_build_registration_kwiver_settings_empty(tmp_path: Path): + cameras = [{'name': 'rgb', 'folder_id': '1', 'media_type': constants.ImageSequenceType}] + assert ( + build_registration_kwiver_settings(tmp_path, cameras, {'reference': '', 'pairs': []}) == {} + ) + assert list(tmp_path.iterdir()) == [] + + def test_build_multicam_kwiver_settings_video(tmp_path: Path): cameras = [ {'name': 'left', 'folder_id': 'l', 'media_type': constants.VideoType}, From a4f6d3674b4bdfd432dec02916fc9d4e516725b2 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 17 Aug 2026 08:54:03 -0400 Subject: [PATCH 2/9] Write pipeline registration files at format v2 The web path built its registration files from the pre-v2 correspondence shape: a flat list of {id, a, b} per camera pair, emitted as pairs[].points and stamped version 1. Format v2 replaced that with observations -- one entry per image pair, each carrying its own points -- so this did not merely write an outdated file, it raised KeyError: 'a' on the first correspondence and failed the job in the worker. Every web multicam pipeline run on a v2-registered dataset would have died there. Build observations instead, as the inverse of registration_output._from_registration_pairs: the store's imageA/imageB become the file's imageLeft/imageRight, frame/enabled/source/stats carry through, and each point's a/b pair becomes one `leftX leftY rightX rightY` row. Stamp version 2. The desktop path was never affected -- it writes through buildPerCameraRegistrationFiles, which the format-v2 work already moved -- which is exactly why this was easy to miss: desktop produces correct v2 files while the worker crashes. VIAME's dive transform reader only consumes the matrices, so the observations travel for provenance and so a file round-trips back into DIVE without losing which frame contributed what. Tests now assert the version, which nothing did before. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_014S8CVLjQDzaBGbCSKL7FiF --- server/dive_tasks/multicam_pipeline.py | 39 +++++++++++++--- server/tests/test_multicam_pipeline.py | 63 +++++++++++++++++++++++--- 2 files changed, 89 insertions(+), 13 deletions(-) diff --git a/server/dive_tasks/multicam_pipeline.py b/server/dive_tasks/multicam_pipeline.py index d94776a20..b3650fd00 100644 --- a/server/dive_tasks/multicam_pipeline.py +++ b/server/dive_tasks/multicam_pipeline.py @@ -108,7 +108,17 @@ def build_registration_pairs(folder_meta: dict) -> List[dict]: """ Convert a dataset folder's camera registration meta (cameraHomographies / cameraCorrespondences / cameraTransformTypes, keyed by directional - "left::right") into dive-camera-registration file pairs. + "left::right") into dive-camera-registration file pairs (format v2). + + Meta stores each pair's points as observations -- one entry per image + pair, carrying its own points -- so this is the inverse of + registration_output._from_registration_pairs: the store's imageA/imageB + become the file's imageLeft/imageRight, and each point's a/b pair becomes + one `leftX leftY rightX rightY` row. + + VIAME's dive transform reader only consumes the matrices; the + observations travel for provenance and so a file round-trips back into + DIVE without losing which frame contributed what. """ homographies = folder_meta.get('cameraHomographies') or {} correspondences = folder_meta.get('cameraCorrespondences') or {} @@ -118,14 +128,31 @@ def build_registration_pairs(folder_meta: dict) -> List[dict]: for key in sorted(keys): left, _, right = key.partition('::') homography = homographies.get(key) + observations = [] + for obs in correspondences.get(key) or []: + observations.append( + { + 'imageLeft': obs.get('imageA'), + 'imageRight': obs.get('imageB'), + 'frame': obs.get('frame'), + 'enabled': obs.get('enabled', True), + 'source': obs.get('source') or 'manual', + **( + {'stats': obs['stats']} + if obs.get('stats') is not None + else {} + ), + 'points': [ + [p['a'][0], p['a'][1], p['b'][0], p['b'][1]] + for p in obs.get('points') or [] + ], + } + ) pairs.append( { 'left': left, 'right': right, - 'points': [ - [c['a'][0], c['a'][1], c['b'][0], c['b'][1]] - for c in correspondences.get(key) or [] - ], + 'observations': observations, 'leftToRight': homography.get('AtoB') if homography else None, 'rightToLeft': homography.get('BtoA') if homography else None, 'transformType': transform_types.get(key, 'similarity'), @@ -183,7 +210,7 @@ def build_registration_kwiver_settings( registration_path = work_dir / f'{name}_to_{reference}_registration.json' with open(registration_path, 'w', encoding='utf-8') as registration_file: json.dump( - {'type': 'dive-camera-registration', 'version': 1, 'pairs': camera_pairs}, + {'type': 'dive-camera-registration', 'version': 2, 'pairs': camera_pairs}, registration_file, indent=2, ) diff --git a/server/tests/test_multicam_pipeline.py b/server/tests/test_multicam_pipeline.py index a4e50c4a7..79dc86d84 100644 --- a/server/tests/test_multicam_pipeline.py +++ b/server/tests/test_multicam_pipeline.py @@ -141,8 +141,26 @@ def test_build_registration_pairs(): folder_meta = { 'cameraHomographies': {'ir::rgb': {'AtoB': IR_TO_RGB, 'BtoA': RGB_TO_IR}}, 'cameraCorrespondences': { - 'ir::rgb': [{'id': 1, 'a': [1, 2], 'b': [3, 4]}], - 'uv::rgb': [{'id': 1, 'a': [5, 6], 'b': [7, 8]}], + 'ir::rgb': [ + { + 'imageA': 'ir_0001.png', + 'imageB': 'rgb_0001.jpg', + 'frame': 1, + 'enabled': True, + 'source': 'manual', + 'points': [{'id': 1, 'a': [1, 2], 'b': [3, 4]}], + } + ], + 'uv::rgb': [ + { + 'imageA': 'uv_0002.jpg', + 'imageB': 'rgb_0002.jpg', + 'frame': 2, + 'enabled': True, + 'source': 'minima_loftr', + 'points': [{'id': 1, 'a': [5, 6], 'b': [7, 8]}], + } + ], }, 'cameraTransformTypes': {'ir::rgb': 'affine'}, } @@ -151,7 +169,16 @@ def test_build_registration_pairs(): { 'left': 'ir', 'right': 'rgb', - 'points': [[1, 2, 3, 4]], + 'observations': [ + { + 'imageLeft': 'ir_0001.png', + 'imageRight': 'rgb_0001.jpg', + 'frame': 1, + 'enabled': True, + 'source': 'manual', + 'points': [[1, 2, 3, 4]], + } + ], 'leftToRight': IR_TO_RGB, 'rightToLeft': RGB_TO_IR, 'transformType': 'affine', @@ -159,7 +186,16 @@ def test_build_registration_pairs(): { 'left': 'uv', 'right': 'rgb', - 'points': [[5, 6, 7, 8]], + 'observations': [ + { + 'imageLeft': 'uv_0002.jpg', + 'imageRight': 'rgb_0002.jpg', + 'frame': 2, + 'enabled': True, + 'source': 'minima_loftr', + 'points': [[5, 6, 7, 8]], + } + ], 'leftToRight': None, 'rightToLeft': None, 'transformType': 'similarity', @@ -180,7 +216,7 @@ def test_build_registration_kwiver_settings(tmp_path: Path): { 'left': 'ir', 'right': 'rgb', - 'points': [], + 'observations': [], 'leftToRight': IR_TO_RGB, 'rightToLeft': RGB_TO_IR, 'transformType': 'similarity', @@ -189,7 +225,16 @@ def test_build_registration_kwiver_settings(tmp_path: Path): { 'left': 'uv', 'right': 'rgb', - 'points': [[1, 2, 3, 4]], + 'observations': [ + { + 'imageLeft': 'uv_0001.jpg', + 'imageRight': 'rgb_0001.jpg', + 'frame': 1, + 'enabled': True, + 'source': 'manual', + 'points': [[1, 2, 3, 4]], + } + ], 'leftToRight': None, 'rightToLeft': None, 'transformType': 'similarity', @@ -199,7 +244,7 @@ def test_build_registration_kwiver_settings(tmp_path: Path): { 'left': 'uv', 'right': 'ir', - 'points': [], + 'observations': [], 'leftToRight': IR_TO_RGB, 'rightToLeft': RGB_TO_IR, 'transformType': 'similarity', @@ -217,6 +262,10 @@ def test_build_registration_kwiver_settings(tmp_path: Path): } written = json.loads((tmp_path / 'ir_to_rgb_registration.json').read_text(encoding='utf-8')) assert written['type'] == 'dive-camera-registration' + # VIAME's dive transform reader rejects anything but v2, and the client + # loader skips a non-v2 file rather than reading a matrix-only pair with + # its points silently dropped. + assert written['version'] == 2 assert len(written['pairs']) == 1 assert written['pairs'][0]['left'] == 'ir' # uv produced no file: its only fitted pair skips the reference. From c5d6409362f0bcbaf749ae8c8e5ada46f6cc0c52 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 17 Aug 2026 09:27:23 -0400 Subject: [PATCH 3/9] Fix typecheck and lint failures in the registration pipeline args - cameraRegistration.spec.ts: JsonMeta does not exist (use JsonConfig); type the fs-extra mock the way common.spec.ts does so vue-tsc accepts the default export and statSync parameter types. - crud_rpc.py: drop unused pathlib import. - multicam_pipeline.py: black formatting. - test_multicam_pipeline.py: VIAME's dive reader ignores the file version; it is DIVE's format-v2 loader that rejects other versions. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MD6kx1uDv4NuiKdFNnYpG2 --- .../desktop/backend/native/cameraRegistration.spec.ts | 10 +++++----- server/dive_server/crud_rpc.py | 1 - server/dive_tasks/multicam_pipeline.py | 6 +----- server/tests/test_multicam_pipeline.py | 6 +++--- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/client/platform/desktop/backend/native/cameraRegistration.spec.ts b/client/platform/desktop/backend/native/cameraRegistration.spec.ts index 3b151bb73..9e853bbda 100644 --- a/client/platform/desktop/backend/native/cameraRegistration.spec.ts +++ b/client/platform/desktop/backend/native/cameraRegistration.spec.ts @@ -5,15 +5,15 @@ import { it, expect, describe, afterAll, vi, } from 'vitest'; -import { Settings, JsonMeta } from 'platform/desktop/constants'; +import { Settings, JsonConfig } from 'platform/desktop/constants'; import { buildRegistrationPipelineArgs } from './cameraRegistration'; // mock-fs no longer intercepts fs-extra's exists checks on newer Node; // route them through statSync like common.spec.ts does. vi.mock('fs-extra', async () => { - const actual = await vi.importActual('fs-extra'); + const actual = await vi.importActual('fs-extra'); const fsNode = await import('node:fs'); - const existsByStat = (targetPath: fsNode.PathLike) => { + const existsByStat = (targetPath: Parameters[0]) => { try { fsNode.statSync(targetPath); return true; @@ -92,7 +92,7 @@ mockfs({ afterAll(() => mockfs.restore()); -function multiCamMeta(id: string, cameras: string[], defaultDisplay: string): JsonMeta { +function multiCamMeta(id: string, cameras: string[], defaultDisplay: string): JsonConfig { return { version: 1, type: 'multi', @@ -117,7 +117,7 @@ function multiCamMeta(id: string, cameras: string[], defaultDisplay: string): Js }])), defaultDisplay, }, - } as unknown as JsonMeta; + } as unknown as JsonConfig; } describe('buildRegistrationPipelineArgs', () => { diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index 5f1dfda97..af5dc81df 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -1,6 +1,5 @@ from datetime import datetime, timedelta import json -from pathlib import Path from typing import Dict, List, Literal, NamedTuple, Optional, Tuple, TypedDict, cast from girder.constants import AccessType diff --git a/server/dive_tasks/multicam_pipeline.py b/server/dive_tasks/multicam_pipeline.py index b3650fd00..79589252b 100644 --- a/server/dive_tasks/multicam_pipeline.py +++ b/server/dive_tasks/multicam_pipeline.py @@ -137,11 +137,7 @@ def build_registration_pairs(folder_meta: dict) -> List[dict]: 'frame': obs.get('frame'), 'enabled': obs.get('enabled', True), 'source': obs.get('source') or 'manual', - **( - {'stats': obs['stats']} - if obs.get('stats') is not None - else {} - ), + **({'stats': obs['stats']} if obs.get('stats') is not None else {}), 'points': [ [p['a'][0], p['a'][1], p['b'][0], p['b'][1]] for p in obs.get('points') or [] diff --git a/server/tests/test_multicam_pipeline.py b/server/tests/test_multicam_pipeline.py index 79dc86d84..ef3d4701e 100644 --- a/server/tests/test_multicam_pipeline.py +++ b/server/tests/test_multicam_pipeline.py @@ -262,9 +262,9 @@ def test_build_registration_kwiver_settings(tmp_path: Path): } written = json.loads((tmp_path / 'ir_to_rgb_registration.json').read_text(encoding='utf-8')) assert written['type'] == 'dive-camera-registration' - # VIAME's dive transform reader rejects anything but v2, and the client - # loader skips a non-v2 file rather than reading a matrix-only pair with - # its points silently dropped. + # DIVE's format-v2 loader rejects any other version rather than reading + # a matrix-only pair with its points silently dropped; VIAME's dive + # transform reader only consumes the matrices and ignores the version. assert written['version'] == 2 assert len(written['pairs']) == 1 assert written['pairs'][0]['left'] == 'ir' From b6335bcc1b4e214cdaedf0af0d80a43640577dfc Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 17 Aug 2026 10:29:44 -0400 Subject: [PATCH 4/9] Let 2-cam/3-cam pipes declare their camera order Which dataset camera feeds which inputN of a multicam pipe is the pipe's contract -- the arctic seal 3-cam pipe runs the thermal detector on input3 and projects optical boxes onto input2 -- but DIVE was inferring it from display order (reference camera first, then the rest), so a dataset imported as EO, IR, UV silently ran the thermal detector on UV. Pipes now state their slots with a `# Camera Order: EO, UV, IR` header, parsed into pipeline metadata on desktop and web. At run time each slot is matched to a dataset camera by name: an exact camera name, or a name segment sharing the slot's role (EO ~ eo/rgb/optical/color/vis, IR ~ ir/thermal/lwir/flir, UV ~ uv/ultraviolet); other tokens match literally. A slot matching no camera or several refuses the run with a message that names the pipe's slots and the dataset's cameras. Camera 1 of the resolved order is the warp target the registration files are built onto. Pipes without the header keep the reference-first fallback. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MD6kx1uDv4NuiKdFNnYpG2 --- client/dive-common/apispec.ts | 8 ++ client/dive-common/multicamDisplay.ts | 23 +++++ .../dive-common/pipelineCameraOrder.spec.ts | 52 +++++++++++ client/dive-common/pipelineCameraOrder.ts | 90 +++++++++++++++++++ .../backend/native/cameraRegistration.spec.ts | 24 ++++- .../backend/native/cameraRegistration.ts | 30 +++---- .../platform/desktop/backend/native/common.ts | 13 ++- .../desktop/backend/native/multiCamUtils.ts | 13 ++- .../platform/desktop/backend/native/viame.ts | 12 ++- docs/Pipeline-Import-Export.md | 1 + server/dive_server/crud_rpc.py | 27 ++++-- server/dive_tasks/multicam_pipeline.py | 57 ++++++++++++ server/dive_tasks/pipeline_discovery.py | 15 +++- server/dive_utils/types.py | 5 ++ server/tests/test_multicam_pipeline.py | 33 +++++++ server/tests/test_pipeline_discovery.py | 28 ++++++ 16 files changed, 391 insertions(+), 40 deletions(-) create mode 100644 client/dive-common/pipelineCameraOrder.spec.ts create mode 100644 client/dive-common/pipelineCameraOrder.ts diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 57ba46287..f7af5fa79 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -78,6 +78,14 @@ interface PipeMetadata { * the two conventional keys are used. */ calibrationKeys?: string[]; + /** + * Camera role per pipeline input for 2-cam/3-cam pipes (e.g. ["EO", "UV", "IR"]: + * input1 is optical, input2 ultraviolet, input3 thermal), parsed from a + * `# Camera Order: [cam...]` header. At run time each slot is matched to + * a dataset camera by name (dive-common/pipelineCameraOrder.ts); when unset, + * cameras are fed registration-reference first, then display order. + */ + cameraOrder?: string[]; } interface PipelineRuntimeParams { diff --git a/client/dive-common/multicamDisplay.ts b/client/dive-common/multicamDisplay.ts index 21b1da6b8..96c9277a6 100644 --- a/client/dive-common/multicamDisplay.ts +++ b/client/dive-common/multicamDisplay.ts @@ -1,5 +1,6 @@ import type { SubType } from 'dive-common/apispec'; import { preferEoIrSubfolderOrder } from 'dive-common/components/ImportMultiCamDialog/multicamSubfolderLayout'; +import { resolvePipelineCameraOrder } from 'dive-common/pipelineCameraOrder'; export type MultiCamSubType = 'stereo' | 'multicam'; @@ -70,6 +71,28 @@ export function pipelineOrderedCameraNames(multiCamMedia: MultiCamMediaLike | nu return reference ? [reference, ...ordered.filter((name) => name !== reference)] : ordered; } +/** + * The cameras to feed input1..N of a 2-cam/3-cam pipeline. A pipe that + * declares its slots (`# Camera Order:` header, parsed into + * metadata.cameraOrder) gets each slot matched to a dataset camera by name and + * throws when that is not unambiguous; a pipe without one gets + * {@link pipelineOrderedCameraNames}. Camera 1 is the frame the others' + * registrations must map onto. + */ +export function pipelineCameraNames( + multiCamMedia: MultiCamMediaLike | null | undefined, + declaredOrder?: string[] | null, +): string[] { + if (declaredOrder?.length) { + const result = resolvePipelineCameraOrder(declaredOrder, orderedMultiCamCameraNames(multiCamMedia)); + if (result.error !== undefined) { + throw new Error(result.error); + } + return result.order; + } + return pipelineOrderedCameraNames(multiCamMedia); +} + export function isMultiCamSubType(subType: SubType | string | null | undefined): subType is MultiCamSubType { return subType === 'stereo' || subType === 'multicam'; } diff --git a/client/dive-common/pipelineCameraOrder.spec.ts b/client/dive-common/pipelineCameraOrder.spec.ts new file mode 100644 index 000000000..22e221d0e --- /dev/null +++ b/client/dive-common/pipelineCameraOrder.spec.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest'; +import { + camerasMatchingSlot, parseCameraOrderHeader, resolvePipelineCameraOrder, +} from './pipelineCameraOrder'; +import { pipelineCameraNames } from './multicamDisplay'; + +describe('pipelineCameraOrder', () => { + it('parses the header value into slot tokens', () => { + expect(parseCameraOrderHeader(' EO, UV, IR ')).toStrictEqual(['EO', 'UV', 'IR']); + expect(parseCameraOrderHeader('left right')).toStrictEqual(['left', 'right']); + expect(parseCameraOrderHeader('')).toStrictEqual([]); + }); + + it('matches slots by exact name, name segment, or role alias', () => { + const cameras = ['rgb', 'CENT_IR', 'uv_cam']; + expect(camerasMatchingSlot('EO', cameras)).toStrictEqual(['rgb']); + expect(camerasMatchingSlot('IR', cameras)).toStrictEqual(['CENT_IR']); + expect(camerasMatchingSlot('ultraviolet', cameras)).toStrictEqual(['uv_cam']); + expect(camerasMatchingSlot('rgb', cameras)).toStrictEqual(['rgb']); + // Exact name wins over role matching elsewhere. + expect(camerasMatchingSlot('ir', ['ir', 'thermal'])).toStrictEqual(['ir']); + // Literal segments for non-role tokens. + expect(camerasMatchingSlot('left', ['left_cam', 'right_cam'])).toStrictEqual(['left_cam']); + }); + + it('resolves declared slots to a full, unique camera order', () => { + expect(resolvePipelineCameraOrder(['EO', 'UV', 'IR'], ['rgb', 'ir', 'uv'])) + .toStrictEqual({ order: ['rgb', 'uv', 'ir'] }); + expect(resolvePipelineCameraOrder(['EO', 'IR'], ['CENT_IR', 'CENT_EO'])) + .toStrictEqual({ order: ['CENT_EO', 'CENT_IR'] }); + }); + + it('reports count mismatch, unmatched and ambiguous slots', () => { + expect(resolvePipelineCameraOrder(['EO', 'IR'], ['rgb', 'ir', 'uv']).error) + .toMatch(/expects 2 cameras but the dataset has 3/); + expect(resolvePipelineCameraOrder(['EO', 'UV', 'IR'], ['rgb', 'ir', 'cam3']).error) + .toMatch(/"UV" \(input2\): no dataset camera matches/); + expect(resolvePipelineCameraOrder(['EO', 'IR'], ['rgb', 'color']).error) + .toMatch(/"EO" \(input1\): several dataset cameras match \(rgb, color\)/); + }); + + it('pipelineCameraNames uses the declared order or falls back to reference-first', () => { + const media = { + cameras: { ir: {}, rgb: {}, uv: {} }, + cameraOrder: ['rgb', 'ir', 'uv'], + defaultDisplay: 'rgb', + }; + expect(pipelineCameraNames(media, ['EO', 'UV', 'IR'])).toStrictEqual(['rgb', 'uv', 'ir']); + expect(pipelineCameraNames(media)).toStrictEqual(['rgb', 'ir', 'uv']); + expect(() => pipelineCameraNames(media, ['EO', 'IR'])).toThrow(/expects 2 cameras/); + }); +}); diff --git a/client/dive-common/pipelineCameraOrder.ts b/client/dive-common/pipelineCameraOrder.ts new file mode 100644 index 000000000..5bd21162c --- /dev/null +++ b/client/dive-common/pipelineCameraOrder.ts @@ -0,0 +1,90 @@ +/** + * Map a multicam pipeline's declared camera slots onto a dataset's cameras. + * + * A 2-cam/3-cam pipe wires each `inputN` to a specific role (the arctic seal + * 3-cam pipe runs the thermal detector on input3 and projects optical boxes + * onto input2), so which dataset camera lands on which input is the pipe's + * contract, not something DIVE can infer from display order. Pipes state it + * with a `# Camera Order: EO, UV, IR` header (one token per input, in + * order); this resolves each token to exactly one dataset camera by name. + * + * Kept in sync with server/dive_tasks/multicam_pipeline.py + * (resolve_pipeline_camera_order). + */ + +/** + * Role aliases: a slot token and a camera name match when they share a role, + * or when the token itself appears as a segment of the camera name (so pipes + * can name cameras literally, e.g. `# Camera Order: left, right`). + */ +export const CAMERA_ROLE_ALIASES: Record = { + eo: ['eo', 'rgb', 'optical', 'color', 'colour', 'vis', 'visible'], + ir: ['ir', 'thermal', 'lwir', 'mwir', 'flir'], + uv: ['uv', 'ultraviolet'], +}; + +function segments(name: string): string[] { + return name.toLowerCase().split(/[^a-z0-9]+/).filter((s) => s); +} + +function roleOf(token: string): string | null { + const lower = token.toLowerCase(); + const found = Object.entries(CAMERA_ROLE_ALIASES) + .find(([, aliases]) => aliases.includes(lower)); + return found ? found[0] : null; +} + +/** Cameras whose name matches a slot token, by literal segment or shared role. */ +export function camerasMatchingSlot(token: string, cameras: string[]): string[] { + const lower = token.toLowerCase(); + const exact = cameras.filter((camera) => camera.toLowerCase() === lower); + if (exact.length) { + return exact; + } + const role = roleOf(token); + const aliases = new Set(role ? CAMERA_ROLE_ALIASES[role] : [lower]); + return cameras.filter((camera) => segments(camera).some((seg) => aliases.has(seg))); +} + +export type PipelineCameraOrderResult = + | { order: string[]; error?: undefined } + | { order?: undefined; error: string }; + +/** + * Resolve declared slots to dataset cameras. Every slot must match exactly one + * camera and no camera may fill two slots; anything else is an error message + * naming the slot, the pipe's slots and the dataset's cameras so the user can + * rename cameras (or fix the header) rather than get a silently mis-wired run. + */ +export function resolvePipelineCameraOrder( + slots: string[], + cameras: string[], +): PipelineCameraOrderResult { + const context = `pipeline cameras [${slots.join(', ')}], dataset cameras [${cameras.join(', ')}]`; + if (slots.length !== cameras.length) { + return { + error: `Pipeline expects ${slots.length} cameras but the dataset has ${cameras.length}: ${context}`, + }; + } + const order: string[] = []; + for (let i = 0; i < slots.length; i += 1) { + const slot = slots[i]; + const matches = camerasMatchingSlot(slot, cameras).filter((c) => !order.includes(c)); + if (matches.length !== 1) { + const why = matches.length === 0 + ? 'no dataset camera matches' + : `several dataset cameras match (${matches.join(', ')})`; + return { + error: `Cannot place pipeline camera "${slot}" (input${i + 1}): ${why}. ${context}. ` + + 'Rename the dataset cameras so each pipeline camera matches exactly one.', + }; + } + order.push(matches[0]); + } + return { order }; +} + +/** Parse the value of a `# Camera Order:` header into slot tokens. */ +export function parseCameraOrderHeader(value: string): string[] { + return value.trim().split(/[\s,]+/).filter((token) => token); +} diff --git a/client/platform/desktop/backend/native/cameraRegistration.spec.ts b/client/platform/desktop/backend/native/cameraRegistration.spec.ts index 9e853bbda..57ef69d86 100644 --- a/client/platform/desktop/backend/native/cameraRegistration.spec.ts +++ b/client/platform/desktop/backend/native/cameraRegistration.spec.ts @@ -6,6 +6,7 @@ import { } from 'vitest'; import { Settings, JsonConfig } from 'platform/desktop/constants'; +import { pipelineOrderedCameraNames } from 'dive-common/multicamDisplay'; import { buildRegistrationPipelineArgs } from './cameraRegistration'; // mock-fs no longer intercepts fs-extra's exists checks on newer Node; @@ -128,7 +129,7 @@ describe('buildRegistrationPipelineArgs', () => { const meta = multiCamMeta('withreg', ['ir', 'rgb', 'uv'], 'rgb'); const jobWorkDir = '/home/user/job/full'; await fs.ensureDir(jobWorkDir); - const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, pipelineOrderedCameraNames(meta.multiCam)); const irPath = npath.join(jobWorkDir, 'ir_to_rgb_registration.json'); const uvPath = npath.join(jobWorkDir, 'uv_to_rgb_registration.json'); @@ -155,7 +156,7 @@ describe('buildRegistrationPipelineArgs', () => { const meta = multiCamMeta('partial', ['rgb', 'ir', 'uv'], 'rgb'); const jobWorkDir = '/home/user/job/partial'; await fs.ensureDir(jobWorkDir); - const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, pipelineOrderedCameraNames(meta.multiCam)); // Pipeline order is rgb, uv, ir: uv (warp2) has only the unsupported // uv-to-ir pair, so it gets nothing; ir (warp3) has a reference pair. expect(Object.keys(args).some((key) => key.startsWith('warp2'))).toBe(false); @@ -169,11 +170,26 @@ describe('buildRegistrationPipelineArgs', () => { expect(await fs.readdir(jobWorkDir)).toStrictEqual(['ir_to_rgb_registration.json']); }); + it('follows an explicit pipeline camera order, camera 1 being the warp target', async () => { + // Pipe declares EO, UV, IR; dataset display order is ir, rgb, uv. The + // resolved order is rgb, uv, ir so uv is warp2 and ir is warp3, and the + // files register onto rgb (camera 1) regardless of the reference camera. + const meta = multiCamMeta('withreg', ['ir', 'rgb', 'uv'], 'ir'); + const jobWorkDir = '/home/user/job/declared'; + await fs.ensureDir(jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, ['rgb', 'uv', 'ir']); + expect(args['warp2:transform_reader:dive:from_camera']).toBe('uv'); + expect(args['warp2:transform_reader:dive:to_camera']).toBe('rgb'); + expect(args['warp3:transform_reader:dive:from_camera']).toBe('ir'); + expect(args['warp3:transform_reader:dive:to_camera']).toBe('rgb'); + expect(await fs.readdir(jobWorkDir)).toStrictEqual(['ir_to_rgb_registration.json', 'uv_to_rgb_registration.json']); + }); + it('returns no args when the dataset has no registration', async () => { const meta = multiCamMeta('noreg', ['rgb', 'ir'], 'rgb'); const jobWorkDir = '/home/user/job/noreg'; await fs.ensureDir(jobWorkDir); - const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, pipelineOrderedCameraNames(meta.multiCam)); expect(args).toStrictEqual({}); expect(await fs.readdir(jobWorkDir)).toStrictEqual([]); }); @@ -185,7 +201,7 @@ describe('buildRegistrationPipelineArgs', () => { }; const jobWorkDir = '/home/user/job/seeded'; await fs.ensureDir(jobWorkDir); - const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, pipelineOrderedCameraNames(meta.multiCam)); expect(args['warp2:transform_reader:dive:from_camera']).toBe('ir'); expect(args['warp2:transform_reader:dive:to_camera']).toBe('rgb'); }); diff --git a/client/platform/desktop/backend/native/cameraRegistration.ts b/client/platform/desktop/backend/native/cameraRegistration.ts index 9b70e357c..849146921 100644 --- a/client/platform/desktop/backend/native/cameraRegistration.ts +++ b/client/platform/desktop/backend/native/cameraRegistration.ts @@ -18,10 +18,7 @@ import { import { readTransformMatrix } from 'vue-media-annotator/alignedView/alignedView'; import { invert3, Matrix3 } from 'vue-media-annotator/alignedView/homography'; import { DatasetConfigMutable } from 'dive-common/apispec'; -import { - referenceCameraName as multicamReferenceCameraName, - pipelineOrderedCameraNames, -} from 'dive-common/multicamDisplay'; +import { referenceCameraName as multicamReferenceCameraName } from 'dive-common/multicamDisplay'; import { RegistrationFileNamePattern, compareRegistrationCandidates, @@ -197,24 +194,25 @@ export async function loadEffectiveRegistration( /** * Build the kwiver -s settings that hand a dataset's camera registration to - * a 2-cam/3-cam pipeline. One standard _to__registration - * file per non-reference camera is written into the job work dir; each - * camera's warp process (warp2, warp3, ... matching the pipeline's - * reference-first camera order) receives its own single-pair file. The pair + * a 2-cam/3-cam pipeline. `cameraOrder` is the pipeline's input1..N camera + * order (see pipelineCameraNames); camera 1 is the frame the pipe warps + * everything onto, so one standard _to__registration file + * per other camera is written into the job work dir and each camera's warp + * process (warp2, warp3, ...) receives its own single-pair file. The pair * and direction are still pinned through the reader's from_camera/to_camera * config, since a pair may be stored in either orientation. * - * Only pairs registering a camera directly onto the reference are - * supported: pairs between two non-reference cameras are explicitly - * unsupported here (there is no transform composition) and never reach the - * pipeline. Cameras without a fitted reference pair get no settings; a - * pipeline that needs one then fails at configure time with the file name - * it was missing. + * Only pairs registering a camera directly onto camera 1 are supported: + * pairs between two other cameras are explicitly unsupported here (there is + * no transform composition) and never reach the pipeline. Cameras without a + * fitted pair onto camera 1 get no settings; a pipeline that needs one then + * fails at configure time with the file name it was missing. */ export async function buildRegistrationPipelineArgs( settings: Settings, meta: JsonConfig, jobWorkDir: string, + cameraOrder: string[], ): Promise> { const args: Record = {}; if (!meta.multiCam) { @@ -222,13 +220,13 @@ export async function buildRegistrationPipelineArgs( } const projectDirInfo = await getValidatedProjectDir(settings, meta.id); const values = await loadEffectiveRegistration(projectDirInfo.basePath, meta); - const reference = multicamReferenceCameraName(meta.multiCam); + const [reference] = cameraOrder; if (!reference) { return args; } const files = buildPerCameraRegistrationFiles(values, reference); const writes: Promise[] = []; - pipelineOrderedCameraNames(meta.multiCam).forEach((camera, index) => { + cameraOrder.forEach((camera, index) => { if (index === 0 || camera === reference) { return; } diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index e1c8ae6e4..54e2dbbc5 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -31,6 +31,7 @@ import { FrameMetadataSourcesResponse, } from 'dive-common/apispec'; import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; +import { parseCameraOrderHeader } from 'dive-common/pipelineCameraOrder'; import isFrameMetadataSourceName from 'dive-common/frameMetadata/naming'; import { METADATA_ATTACHMENT_UNAVAILABLE, isFrameMetadataReadableName, @@ -297,7 +298,7 @@ async function extractPipeMetadata(filePath: string): Promise { } if (inDescription) { - if (/^#\s*$/.test(line) || /^#\s*=/.test(line) || /^#\s*(Input|Output|Requires\s+Calibration|Metadata\s+File|Image\s+List\s+Keys?|Calibration\s+Keys?):/i.test(line) || !line.startsWith('#')) { + if (/^#\s*$/.test(line) || /^#\s*=/.test(line) || /^#\s*(Input|Output|Requires\s+Calibration|Metadata\s+File|Image\s+List\s+Keys?|Calibration\s+Keys?|Camera\s+Order):/i.test(line) || !line.startsWith('#')) { inDescription = false; } else { fullDescription += ` ${line.replace(/^#\s*/, '').trim()}`; @@ -353,6 +354,16 @@ async function extractPipeMetadata(filePath: string): Promise { metadata.calibrationKeys = keys; } } + + // `# Camera Order: EO, UV, IR` names the camera role fed to each inputN of + // a 2-cam/3-cam pipe; DIVE matches dataset cameras onto it by name. + const cameraOrderMatch = line.match(/^#\s*Camera\s+Order:\s*(.+)/i); + if (cameraOrderMatch) { + const slots = parseCameraOrderHeader(cameraOrderMatch[1]); + if (slots.length) { + metadata.cameraOrder = slots; + } + } }); metadata.description = fullDescription.trim() || undefined; } catch (error) { diff --git a/client/platform/desktop/backend/native/multiCamUtils.ts b/client/platform/desktop/backend/native/multiCamUtils.ts index cd0b1134b..5c2ca3c19 100644 --- a/client/platform/desktop/backend/native/multiCamUtils.ts +++ b/client/platform/desktop/backend/native/multiCamUtils.ts @@ -10,7 +10,6 @@ import { JsonConfig, Settings } from 'platform/desktop/constants'; import { loadAnnotationFile, loadJsonConfig, getValidatedProjectDir } from 'platform/desktop/backend/native/common'; import { serialize } from 'platform/desktop/backend/serializers/viame'; import { parseFrameTimestamp } from 'dive-common/frameTimestamp'; -import { pipelineOrderedCameraNames } from 'dive-common/multicamDisplay'; /** * Figure out the destination location @@ -80,17 +79,17 @@ async function writeMultiCamStereoPipelineArgs( settings: Settings, utility = false, forceTranscoded = false, - // 2-cam/3-cam pipes treat camera 1 as the reference frame the other - // cameras register onto, so their inputs go reference-first; stereo - // measurement keeps the stored left/right order. - referenceFirst = false, + // Explicit input1..N camera order for 2-cam/3-cam pipes (see + // pipelineCameraNames); stereo measurement keeps the stored left/right + // order when omitted. + cameraOrder: string[] | undefined = undefined, ) { const argFilePair: Record = {}; const outFiles: Record = {}; if (meta.multiCam && meta.multiCam.cameras) { const { cameras } = meta.multiCam; - const cameraNames = referenceFirst - ? pipelineOrderedCameraNames(meta.multiCam).filter((name) => name in cameras) + const cameraNames = cameraOrder + ? cameraOrder.filter((name) => name in cameras) : Object.keys(cameras); const cameraList = cameraNames.map((name) => [name, cameras[name]] as const); for (let i = 0; i < cameraList.length; i += 1) { diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 0053163b0..7a5ea5cd2 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -27,6 +27,7 @@ import { isTranscodePipeline, pipelineCreatesNewDataset, } from 'dive-common/pipelineCreatesDataset'; +import { pipelineCameraNames } from 'dive-common/multicamDisplay'; import * as common from './common'; import { jobFileEchoMiddleware, createWorkingDirectory, createCustomWorkingDirectory, splitExt, @@ -368,7 +369,12 @@ async function runPipeline( let multiOutFiles: Record; if (meta.multiCam && stereoOrMultiCam) { const isMultiCamPipeline = multiCamPipelineMarkers.includes(pipeline.type); - const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput, false, isMultiCamPipeline); + // 2-cam/3-cam pipes: which camera feeds which inputN is the pipe's + // contract (its `# Camera Order:` header), else reference-first. + const multiCamOrder = isMultiCamPipeline + ? pipelineCameraNames(meta.multiCam, pipeline.metadata?.cameraOrder) + : undefined; + const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput, false, multiCamOrder); Object.entries(argFilePair).forEach(([arg, file]) => { command.push(`-s ${arg}="${file}"`); }); @@ -396,10 +402,10 @@ async function runPipeline( command.push(`-s ${key}="${meta.multiCam?.calibration}"`); }); } - if (isMultiCamPipeline) { + if (multiCamOrder) { // Hand the camera registration (Aligned View homographies) to the // pipeline's per-camera warp processes. - const registrationArgs = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir); + const registrationArgs = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, multiCamOrder); Object.entries(registrationArgs).forEach(([arg, value]) => { command.push(`-s ${arg}="${value}"`); }); diff --git a/docs/Pipeline-Import-Export.md b/docs/Pipeline-Import-Export.md index 17970d0f0..6968a0689 100644 --- a/docs/Pipeline-Import-Export.md +++ b/docs/Pipeline-Import-Export.md @@ -75,6 +75,7 @@ Example: | `# Calibration Keys: [k…]` | Opt-in: binds the dataset's stereo calibration file to each listed KWIVER config key at run time (one `-s =` per key). Keys may be space- or comma-separated. Use this when the pipe's calibration consumer is not the conventional `measurer:calibration_file` / `calibration_reader:file` pair (those are used when the header is unset). Needed because `$CONFIG{global:…}` indirection cannot receive `-s` overrides (macros expand at parse time; `-s` blocks are appended last). | | `# Metadata File: :` | Opt-in: when the dataset has an attached **Metadata File**, DIVE appends a KWIVER override `-s :=` at run time. The same CSV/TXT attachment is also considered for [Frame Metadata](Frame-Metadata.md). Without this header, no metadata file is injected. | | `# Image List Keys: [k…]` | Opt-in: binds the run's per-camera input image list(s) to each listed KWIVER key. Keys may be space- or comma-separated. A key containing `{cam}` is expanded once per camera (1-based), e.g. `stabilizer:image_list{cam}` → `image_list1`, `image_list2`, …. A key without `{cam}` receives camera 1's list only. | +| `# Camera Order: [cam…]` | 2-cam/3-cam pipes only: names the camera role fed to each `inputN`, in order (e.g. `# Camera Order: EO, UV, IR` → `input1` optical, `input2` ultraviolet, `input3` thermal). At run time DIVE matches each slot to a dataset camera by name — an exact camera name, or a name segment sharing the slot's role (`EO` ≈ eo/rgb/optical/color/vis, `IR` ≈ ir/thermal/lwir/flir, `UV` ≈ uv/ultraviolet); other tokens match literally (`left`, `right`). If any slot matches no camera or several, the run is refused with a message naming the pipe's slots and the dataset's cameras. Camera 1 is the frame the pipe's warp processes map onto: each other camera's Aligned View registration onto camera 1 is written to the job as `_to__registration.json` and bound to `warpN`. Without this header, cameras are fed registration-reference (default display) first, then display order. | ### Metadata File vs Configuration File diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index af5dc81df..de2d2e679 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -23,6 +23,7 @@ is_stereo_or_multicam_pipeline, pipeline_camera_order, pipeline_requires_input, + resolve_pipeline_camera_order, ) from dive_tasks.utils import choose_annotation_fps from dive_utils import ( @@ -335,14 +336,24 @@ def run_pipeline( cameras_meta = multi_cam.get('cameras') or {} is_warp_pipeline = pipeline['type'] in constants.MultiCamPipelineMarkers if is_warp_pipeline and camera_order: - # 2-cam/3-cam pipes treat camera 1 as the reference frame the - # other cameras register onto; feed them reference-first. - reference_camera = ( - multicam_default_display - if multicam_default_display in cameras_meta - else camera_order[0] - ) - camera_order = pipeline_camera_order(camera_order, reference_camera) + # 2-cam/3-cam pipes warp everything onto camera 1. Which camera + # feeds which inputN is the pipe's contract (`# Camera Order:` + # header); a pipe without one gets the registration reference + # first, then display order. + declared_order = (pipeline.get('metadata') or {}).get('cameraOrder') + if declared_order: + try: + camera_order = resolve_pipeline_camera_order(declared_order, camera_order) + except ValueError as err: + raise RestException(str(err), code=400) from err + else: + reference_camera = ( + multicam_default_display + if multicam_default_display in cameras_meta + else camera_order[0] + ) + camera_order = pipeline_camera_order(camera_order, reference_camera) + reference_camera = camera_order[0] for name in camera_order: cam_info = cameras_meta[name] folder_id = cam_info.get('folderId') diff --git a/server/dive_tasks/multicam_pipeline.py b/server/dive_tasks/multicam_pipeline.py index 79589252b..d1796ebeb 100644 --- a/server/dive_tasks/multicam_pipeline.py +++ b/server/dive_tasks/multicam_pipeline.py @@ -90,6 +90,63 @@ def append_metadata_file_kwiver_settings( command.append(f'-s {shlex.quote(kwiver_key)}={shlex.quote(str(metadata_path))}') +# Role aliases: a slot token and a camera name match when they share a role, or +# when the token itself appears as a segment of the camera name (so pipes can +# name cameras literally, e.g. `# Camera Order: left, right`). Kept in sync with +# client/dive-common/pipelineCameraOrder.ts. +CAMERA_ROLE_ALIASES: Dict[str, Tuple[str, ...]] = { + 'eo': ('eo', 'rgb', 'optical', 'color', 'colour', 'vis', 'visible'), + 'ir': ('ir', 'thermal', 'lwir', 'mwir', 'flir'), + 'uv': ('uv', 'ultraviolet'), +} + + +def _name_segments(name: str) -> List[str]: + return [seg for seg in re.split(r'[^a-z0-9]+', name.lower()) if seg] + + +def cameras_matching_slot(token: str, cameras: List[str]) -> List[str]: + """Cameras whose name matches a slot token, by exact name, segment, or shared role.""" + lower = token.lower() + exact = [camera for camera in cameras if camera.lower() == lower] + if exact: + return exact + role = next((r for r, aliases in CAMERA_ROLE_ALIASES.items() if lower in aliases), None) + aliases = set(CAMERA_ROLE_ALIASES[role]) if role else {lower} + return [camera for camera in cameras if any(seg in aliases for seg in _name_segments(camera))] + + +def resolve_pipeline_camera_order(slots: List[str], cameras: List[str]) -> List[str]: + """ + Map a pipe's declared `# Camera Order:` slots onto dataset cameras: every + slot must match exactly one camera and no camera may fill two slots. + Raises ValueError with a message naming the slot, the pipe's slots and the + dataset's cameras otherwise, so the run fails up front instead of being + silently mis-wired. + """ + context = f'pipeline cameras [{", ".join(slots)}], dataset cameras [{", ".join(cameras)}]' + if len(slots) != len(cameras): + raise ValueError( + f'Pipeline expects {len(slots)} cameras but the dataset has {len(cameras)}: ' + f'{context}' + ) + order: List[str] = [] + for index, slot in enumerate(slots, start=1): + matches = [c for c in cameras_matching_slot(slot, cameras) if c not in order] + if len(matches) != 1: + why = ( + 'no dataset camera matches' + if not matches + else f'several dataset cameras match ({", ".join(matches)})' + ) + raise ValueError( + f'Cannot place pipeline camera "{slot}" (input{index}): {why}. {context}. ' + 'Rename the dataset cameras so each pipeline camera matches exactly one.' + ) + order.append(matches[0]) + return order + + def pipeline_camera_order(camera_names: List[str], reference: str) -> List[str]: """ Camera order for 2-cam/3-cam pipelines, matching the desktop client: the diff --git a/server/dive_tasks/pipeline_discovery.py b/server/dive_tasks/pipeline_discovery.py index 590ee844c..3fcbb3f82 100644 --- a/server/dive_tasks/pipeline_discovery.py +++ b/server/dive_tasks/pipeline_discovery.py @@ -205,7 +205,7 @@ def extract_pipe_metadata(file_path: Path) -> PipeMetadata: or re.match(r'^#\s*=', line_raw) or re.match( r'^#\s*(Input|Output|Requires\s+Calibration|Metadata\s+File' - r'|Image\s+List\s+Keys?|Calibration\s+Keys?):', + r'|Image\s+List\s+Keys?|Calibration\s+Keys?|Camera\s+Order):', line_raw, re.IGNORECASE, ) @@ -277,6 +277,19 @@ def extract_pipe_metadata(file_path: Path) -> PipeMetadata: if keys: metadata["calibrationKeys"] = keys + # `# Camera Order: EO, UV, IR` names the camera role fed to each + # inputN of a 2-cam/3-cam pipe; DIVE matches dataset cameras onto + # it by name at run time (multicam_pipeline.resolve_pipeline_camera_order). + camera_order_match = re.match( + r'^#\s*Camera\s+Order:\s*(.+)', line_raw, re.IGNORECASE + ) + if camera_order_match: + slots = [ + s for s in re.split(r'[\s,]+', camera_order_match.group(1).strip()) if s + ] + if slots: + metadata["cameraOrder"] = slots + if full_description_parts: metadata["description"] = " ".join(full_description_parts) else: diff --git a/server/dive_utils/types.py b/server/dive_utils/types.py index e6ddd6fe7..9f24cfa66 100644 --- a/server/dive_utils/types.py +++ b/server/dive_utils/types.py @@ -92,6 +92,11 @@ class PipeMetadata(TypedDict): # not the conventional `measurer`/`calibration_reader` declare their own keys # here; when unset the two conventional keys are used. calibrationKeys: NotRequired[Optional[list[str]]] + # Camera role per pipeline input for 2-cam/3-cam pipes (e.g. ["EO", "UV", "IR"]), + # parsed from `# Camera Order: [cam...]`. Each slot is matched to a dataset + # camera by name at run time; when unset cameras are fed reference-first, then + # display order. + cameraOrder: NotRequired[Optional[list[str]]] class PipelineDescription(TypedDict): diff --git a/server/tests/test_multicam_pipeline.py b/server/tests/test_multicam_pipeline.py index ef3d4701e..a75405dc3 100644 --- a/server/tests/test_multicam_pipeline.py +++ b/server/tests/test_multicam_pipeline.py @@ -1,17 +1,21 @@ import json from pathlib import Path +import pytest + from dive_tasks.multicam_pipeline import ( DEFAULT_CALIBRATION_KEYS, append_stereo_calibration_kwiver_settings, build_multicam_kwiver_settings, build_registration_kwiver_settings, build_registration_pairs, + cameras_matching_slot, find_downloaded_calibration_file, is_stereo_measurement_pipeline, is_stereo_or_multicam_pipeline, pipeline_camera_order, pipeline_requires_input, + resolve_pipeline_camera_order, stereo_calibration_keys, ) from dive_utils import constants @@ -133,6 +137,35 @@ def test_pipeline_camera_order(): assert pipeline_camera_order(['a', 'b'], 'missing') == ['a', 'b'] +def test_cameras_matching_slot(): + cameras = ['rgb', 'CENT_IR', 'uv_cam'] + assert cameras_matching_slot('EO', cameras) == ['rgb'] + assert cameras_matching_slot('IR', cameras) == ['CENT_IR'] + assert cameras_matching_slot('ultraviolet', cameras) == ['uv_cam'] + assert cameras_matching_slot('rgb', cameras) == ['rgb'] + # Exact name wins over role matching elsewhere; literal segments for non-role tokens. + assert cameras_matching_slot('ir', ['ir', 'thermal']) == ['ir'] + assert cameras_matching_slot('left', ['left_cam', 'right_cam']) == ['left_cam'] + + +def test_resolve_pipeline_camera_order(): + assert resolve_pipeline_camera_order(['EO', 'UV', 'IR'], ['rgb', 'ir', 'uv']) == [ + 'rgb', + 'uv', + 'ir', + ] + assert resolve_pipeline_camera_order(['EO', 'IR'], ['CENT_IR', 'CENT_EO']) == [ + 'CENT_EO', + 'CENT_IR', + ] + with pytest.raises(ValueError, match='expects 2 cameras but the dataset has 3'): + resolve_pipeline_camera_order(['EO', 'IR'], ['rgb', 'ir', 'uv']) + with pytest.raises(ValueError, match=r'"UV" \(input2\): no dataset camera matches'): + resolve_pipeline_camera_order(['EO', 'UV', 'IR'], ['rgb', 'ir', 'cam3']) + with pytest.raises(ValueError, match=r'"EO" \(input1\): several dataset cameras match'): + resolve_pipeline_camera_order(['EO', 'IR'], ['rgb', 'color']) + + IR_TO_RGB = [[1, 0, 5], [0, 1, -3], [0, 0, 1]] RGB_TO_IR = [[1, 0, -5], [0, 1, 3], [0, 0, 1]] diff --git a/server/tests/test_pipeline_discovery.py b/server/tests/test_pipeline_discovery.py index f9feab1df..4fcd0aec0 100644 --- a/server/tests/test_pipeline_discovery.py +++ b/server/tests/test_pipeline_discovery.py @@ -120,6 +120,34 @@ def test_extract_pipe_metadata_calibration_keys(tmp_path: Path): assert metadata['description'] == 'rectified disparity' +def test_extract_pipe_metadata_camera_order(tmp_path: Path): + pipe = tmp_path / 'detector_seal_3-cam.pipe' + pipe.write_text( + '\n'.join( + [ + '# Description: three camera detector', + '# Camera Order: EO, UV, IR', + '# Input: IMAGE (per camera)', + ] + ) + ) + + metadata = extract_pipe_metadata(pipe) + + assert metadata['cameraOrder'] == ['EO', 'UV', 'IR'] + # The header must not bleed into the multi-line description. + assert metadata['description'] == 'three camera detector' + assert 'cameraOrder' not in extract_pipe_metadata( + _write(tmp_path, 'detector_plain.pipe', ['# Description: none']) + ) + + +def _write(tmp_path: Path, name: str, lines: list) -> Path: + pipe = tmp_path / name + pipe.write_text('\n'.join(lines)) + return pipe + + def test_extract_pipe_metadata_parses_metadata_file_key(tmp_path: Path): pipe = tmp_path / 'detector_stabilize.pipe' pipe.write_text( From 3b0ae42a809851ecda7f12b0760c120bde26243e Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 17 Aug 2026 11:03:48 -0400 Subject: [PATCH 5/9] Confirm multicam camera placement before a run; carry camera roles on datasets Which dataset camera feeds which inputN of a 2-cam/3-cam pipe was decided silently -- from the pipe's `# Camera Order:` header matched against camera names, and before that from display order. Make it explicit and visible: - Datasets carry a sensor role per camera (`cameraRoles`: eo / ir / uv), inferred once at import from the camera name and, failing that, from tokens in its image file names (KAMERA style _rgb / _ir / _uv). Only a unanimous answer is recorded. Desktop and web import both set it; it is a mutable config key so it round-trips through load/save on both. - Running a 2-cam/3-cam pipe opens a camera-assignment step: one row per pipeline camera (the header's slots, or bare input1..N) with the dataset camera DIVE proposes -- by role when both sides have one, else by name -- which the user confirms or changes. Unfilled or duplicated slots block the run. Confirming role-labelled slots saves the roles back onto the dataset (opt-out checkbox), so a corrected role wins over a misleading name next time and for every other pipe. - The confirmed order travels as pipelineParams.cameraOrder; desktop and web validate it against the dataset's cameras and use it verbatim. Runs without it (CLI) still resolve the header by role then name; pipes with no header keep the reference-first fallback. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MD6kx1uDv4NuiKdFNnYpG2 --- client/dive-common/apispec.ts | 17 +- .../components/PipelineCameraAssignDialog.vue | 196 ++++++++++++++++++ .../components/RunPipelineMenu.vue | 97 ++++++++- client/dive-common/multicamDisplay.ts | 5 +- .../dive-common/pipelineCameraOrder.spec.ts | 31 ++- client/dive-common/pipelineCameraOrder.ts | 144 +++++++++++-- .../desktop/backend/native/multiCamImport.ts | 14 ++ .../platform/desktop/backend/native/viame.ts | 26 ++- docs/Pipeline-Import-Export.md | 10 +- server/dive_server/crud_dataset.py | 8 + server/dive_server/crud_rpc.py | 16 +- server/dive_tasks/multicam_pipeline.py | 68 +++++- server/dive_utils/models.py | 7 + server/dive_utils/types.py | 4 + server/tests/test_multicam_pipeline.py | 23 ++ 15 files changed, 631 insertions(+), 35 deletions(-) create mode 100644 client/dive-common/components/PipelineCameraAssignDialog.vue diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index f7af5fa79..ef1fcd718 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -11,6 +11,7 @@ import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements' import type { CameraHomographies, CameraCorrespondences, CameraTransformTypes, RegistrationSource, } from 'vue-media-annotator/alignedView/CameraRegistrationStore'; +import type { CameraRole } from 'dive-common/pipelineCameraOrder'; import type { PercentileStretch } from 'vue-media-annotator/use/useImageEnhancements'; type DatasetType = 'image-sequence' | 'video' | 'multi' | 'large-image'; @@ -95,6 +96,13 @@ interface PipelineRuntimeParams { interface PipelineParams { kwiverParams?: Record; runtimeParams?: PipelineRuntimeParams; + /** + * 2-cam/3-cam pipes: the dataset camera to feed each inputN, in order, as + * confirmed by the user before the run. When omitted the backend places + * cameras from the pipe's `# Camera Order:` header (or, without one, + * registration reference first then display order). + */ + cameraOrder?: string[]; /** Filter / transcode / disparity pipelines: name for the newly created dataset. */ outputDatasetName?: string; /** @@ -278,9 +286,16 @@ interface DatasetConfigMutable { cameraTransformTypes?: CameraTransformTypes; /** Producer provenance of the camera registration (see RegistrationSource). */ cameraRegistrationSource?: RegistrationSource | null; + /** + * Sensor role per multicam camera name (eo / ir / uv), inferred at import + * from the camera and image names and editable afterwards; used to place + * cameras onto a pipeline's declared camera slots. Cameras with no known + * role are absent. + */ + cameraRoles?: Record; error?: string; } -const DatasetConfigMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraRegistrationSource', 'typeHierarchy']; +const DatasetConfigMutableKeys = ['attributes', 'confidenceFilters', 'timeFilters', 'imageEnhancements', 'customTypeStyling', 'customGroupStyling', 'attributeTrackFilters', 'datasetInfo', 'cameraHomographies', 'cameraCorrespondences', 'cameraTransformTypes', 'cameraRegistrationSource', 'typeHierarchy', 'cameraRoles']; /** * Cross-dataset color/style overrides, reused across every dataset when the * "shared" color scope is enabled (see clientSettings.typeSettings.colorScope). diff --git a/client/dive-common/components/PipelineCameraAssignDialog.vue b/client/dive-common/components/PipelineCameraAssignDialog.vue new file mode 100644 index 000000000..223e889b1 --- /dev/null +++ b/client/dive-common/components/PipelineCameraAssignDialog.vue @@ -0,0 +1,196 @@ + + + diff --git a/client/dive-common/components/RunPipelineMenu.vue b/client/dive-common/components/RunPipelineMenu.vue index f280e5f97..5b71a4052 100644 --- a/client/dive-common/components/RunPipelineMenu.vue +++ b/client/dive-common/components/RunPipelineMenu.vue @@ -37,6 +37,13 @@ import pipelineTypeDisplay from 'dive-common/pipelineTypeDisplay'; import { useRequest } from 'dive-common/use'; import { usePrompt } from 'dive-common/vue-utilities/prompt-service'; import PipelineParamsDialog from 'dive-common/components/PipelineParamsDialog.vue'; +import PipelineCameraAssignDialog, { + PipelineCameraAssignRequest, PipelineCameraAssignResult, +} from 'dive-common/components/PipelineCameraAssignDialog.vue'; +import { orderedMultiCamCameraNames } from 'dive-common/multicamDisplay'; +import { + CameraRole, pipelineCameraSlots, prefillPipelineCameraOrder, +} from 'dive-common/pipelineCameraOrder'; import PipelineCalibrationWarningIcon from 'dive-common/components/PipelineCalibrationWarningIcon.vue'; type MenuState = 'idle' | 'configuring'; @@ -48,6 +55,7 @@ export default defineComponent({ JobLaunchDialog, JobConfigFilterTranscodeDialog, PipelineParamsDialog, + PipelineCameraAssignDialog, RunPipelineToast, PipelineCalibrationWarningIcon, }, @@ -111,7 +119,9 @@ export default defineComponent({ setup(props) { const { prompt } = usePrompt(); - const { runPipeline, getPipelineList, hasCalibrationFile } = useApi(); + const { + runPipeline, getPipelineList, hasCalibrationFile, loadConfig, saveConfig, + } = useApi(); const unsortedPipelines = ref({} as Pipelines); const { request: _runPipelineRequest, @@ -257,6 +267,74 @@ export default defineComponent({ return false; }); + // --- Multicam camera assignment ------------------------------------- + // Before a 2-cam/3-cam run the user sees which dataset camera DIVE + // proposes for each pipeline input (by camera role, else by name) and + // confirms or corrects it. Nothing about the placement is inferred at + // run time behind their back. + const cameraAssignRequest = ref(null); + let cameraAssignResolve: ((result: PipelineCameraAssignResult | null) => void) | null = null; + + function askCameraAssignment(request: PipelineCameraAssignRequest) { + return new Promise((resolve) => { + cameraAssignResolve = resolve; + cameraAssignRequest.value = request; + }); + } + function settleCameraAssignment(result: PipelineCameraAssignResult | null) { + cameraAssignRequest.value = null; + const resolve = cameraAssignResolve; + cameraAssignResolve = null; + resolve?.(result); + } + + /** + * Confirmed input1..N camera order per dataset, or null when the user + * cancelled. Persists confirmed roles onto the dataset when asked so the + * next run (and other pipelines) prefill correctly. + */ + async function confirmCameraOrders( + pipeline: Pipe, + datasetIds: string[], + ): Promise | null> { + const orders: Record = {}; + // eslint-disable-next-line no-restricted-syntax + for (const id of datasetIds) { + // eslint-disable-next-line no-await-in-loop + const config = await loadConfig(id); + const cameras = orderedMultiCamCameraNames(config.multiCamMedia); + if (!cameras.length) { + throw new Error(`${config.name} is not a multi-camera dataset`); + } + const slots = pipelineCameraSlots(pipeline.metadata?.cameraOrder, cameras.length); + if (slots.length !== cameras.length) { + throw new Error(`${pipeline.name} expects ${slots.length} cameras but ${config.name} has ${cameras.length} (${cameras.join(', ')})`); + } + const roles: Record = config.cameraRoles ?? {}; + // eslint-disable-next-line no-await-in-loop + const result = await askCameraAssignment({ + datasetName: config.name, + pipelineName: pipeline.name, + slots, + cameras, + proposed: prefillPipelineCameraOrder(slots, cameras, roles), + roles, + }); + if (!result) { + return null; + } + orders[id] = result.order; + if (result.roles) { + const merged = { ...roles, ...result.roles }; + if (JSON.stringify(merged) !== JSON.stringify(roles)) { + // eslint-disable-next-line no-await-in-loop + await saveConfig(id, { cameraRoles: merged }); + } + } + } + return orders; + } + async function _runPipelineOnSelectedItemInner( pipeline: Pipe, outputDatasetNameById?: Record, @@ -286,6 +364,14 @@ export default defineComponent({ || stereoPipelineMarker === pipeline.type) { datasetIds = props.selectedDatasetIds.map((item) => parentDatasetId(item)); } + let cameraOrderById: Record = {}; + if (multiCamPipelineMarkers.includes(pipeline.type)) { + const confirmed = await confirmCameraOrders(pipeline, datasetIds); + if (!confirmed) { + return; + } + cameraOrderById = confirmed; + } selectedPipeline.value = pipeline; const frameRange = props.timeFilter; await _runPipelineRequest(() => Promise.all( @@ -294,6 +380,7 @@ export default defineComponent({ outputDatasetName: outputDatasetNameById?.[id], outputParentFolderId, kwiverParams: kwiverParamsById?.[id], + cameraOrder: cameraOrderById[id], })), )); } @@ -373,6 +460,8 @@ export default defineComponent({ pipelineHasParams, categoryHasParams, categoryHasCalibrationWarning, + cameraAssignRequest, + settleCameraAssignment, }; }, }); @@ -598,6 +687,12 @@ export default defineComponent({ :params="pipelineParams" @confirm="confirmPipelineExecution" /> + diff --git a/client/dive-common/multicamDisplay.ts b/client/dive-common/multicamDisplay.ts index 96c9277a6..5e735dd5a 100644 --- a/client/dive-common/multicamDisplay.ts +++ b/client/dive-common/multicamDisplay.ts @@ -1,6 +1,6 @@ import type { SubType } from 'dive-common/apispec'; import { preferEoIrSubfolderOrder } from 'dive-common/components/ImportMultiCamDialog/multicamSubfolderLayout'; -import { resolvePipelineCameraOrder } from 'dive-common/pipelineCameraOrder'; +import { CameraRole, resolvePipelineCameraOrder } from 'dive-common/pipelineCameraOrder'; export type MultiCamSubType = 'stereo' | 'multicam'; @@ -82,9 +82,10 @@ export function pipelineOrderedCameraNames(multiCamMedia: MultiCamMediaLike | nu export function pipelineCameraNames( multiCamMedia: MultiCamMediaLike | null | undefined, declaredOrder?: string[] | null, + roles: Record = {}, ): string[] { if (declaredOrder?.length) { - const result = resolvePipelineCameraOrder(declaredOrder, orderedMultiCamCameraNames(multiCamMedia)); + const result = resolvePipelineCameraOrder(declaredOrder, orderedMultiCamCameraNames(multiCamMedia), roles); if (result.error !== undefined) { throw new Error(result.error); } diff --git a/client/dive-common/pipelineCameraOrder.spec.ts b/client/dive-common/pipelineCameraOrder.spec.ts index 22e221d0e..b42522027 100644 --- a/client/dive-common/pipelineCameraOrder.spec.ts +++ b/client/dive-common/pipelineCameraOrder.spec.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from 'vitest'; import { - camerasMatchingSlot, parseCameraOrderHeader, resolvePipelineCameraOrder, + camerasMatchingSlot, inferCameraRole, inferCameraRoles, parseCameraOrderHeader, + pipelineCameraSlots, prefillPipelineCameraOrder, resolvePipelineCameraOrder, } from './pipelineCameraOrder'; import { pipelineCameraNames } from './multicamDisplay'; @@ -39,6 +40,34 @@ describe('pipelineCameraOrder', () => { .toMatch(/"EO" \(input1\): several dataset cameras match \(rgb, color\)/); }); + it('infers roles from camera names, then image names, only when unanimous', () => { + expect(inferCameraRole('rgb')).toBe('eo'); + expect(inferCameraRole('CENT_IR')).toBe('ir'); + expect(inferCameraRole('uv_cam')).toBe('uv'); + expect(inferCameraRole('cam1', ['flight_0001_rgb.jpg', 'flight_0002_rgb.jpg'])).toBe('eo'); + expect(inferCameraRole('cam1', ['a_rgb.jpg', 'b_ir.tif'])).toBeNull(); + expect(inferCameraRole('eo_ir')).toBeNull(); + expect(inferCameraRole('center', ['0001.png'])).toBeNull(); + expect(inferCameraRoles({ rgb: [], center: ['x_ir.tif'], other: ['a.png'] })) + .toStrictEqual({ rgb: 'eo', center: 'ir' }); + }); + + it('assigned roles beat name matching, and prefill leaves ambiguity open', () => { + // "thermal" is named like IR but the user marked it optical. + expect(resolvePipelineCameraOrder(['EO', 'IR'], ['thermal', 'other'], { thermal: 'eo', other: 'ir' })) + .toStrictEqual({ order: ['thermal', 'other'] }); + expect(prefillPipelineCameraOrder(['EO', 'UV', 'IR'], ['rgb', 'ir', 'uv'])) + .toStrictEqual(['rgb', 'uv', 'ir']); + // Two EO-ish names, no roles: the EO slot stays open, IR still fills. + expect(prefillPipelineCameraOrder(['EO', 'IR'], ['rgb', 'color', 'ir'])) + .toStrictEqual([null, 'ir']); + // Bare positional slots: nothing to match on, all open. + expect(prefillPipelineCameraOrder(['input1', 'input2'], ['rgb', 'ir'])) + .toStrictEqual([null, null]); + expect(pipelineCameraSlots(['EO', 'IR'], 2)).toStrictEqual(['EO', 'IR']); + expect(pipelineCameraSlots(undefined, 3)).toStrictEqual(['input1', 'input2', 'input3']); + }); + it('pipelineCameraNames uses the declared order or falls back to reference-first', () => { const media = { cameras: { ir: {}, rgb: {}, uv: {} }, diff --git a/client/dive-common/pipelineCameraOrder.ts b/client/dive-common/pipelineCameraOrder.ts index 5bd21162c..9abd13346 100644 --- a/client/dive-common/pipelineCameraOrder.ts +++ b/client/dive-common/pipelineCameraOrder.ts @@ -1,23 +1,38 @@ /** - * Map a multicam pipeline's declared camera slots onto a dataset's cameras. + * Camera roles, and how a multicam pipeline's input slots are filled from a + * dataset's cameras. * * A 2-cam/3-cam pipe wires each `inputN` to a specific role (the arctic seal * 3-cam pipe runs the thermal detector on input3 and projects optical boxes * onto input2), so which dataset camera lands on which input is the pipe's * contract, not something DIVE can infer from display order. Pipes state it - * with a `# Camera Order: EO, UV, IR` header (one token per input, in - * order); this resolves each token to exactly one dataset camera by name. + * with a `# Camera Order: EO, UV, IR` header (one token per input, in order). * - * Kept in sync with server/dive_tasks/multicam_pipeline.py - * (resolve_pipeline_camera_order). + * Datasets carry a role per camera (`cameraRoles`, inferred once at import + * from the camera / image names and editable afterwards). Before a run the + * user is shown each slot with the camera DIVE proposes for it -- by role + * when both sides have one, else by name -- and confirms or corrects it; the + * confirmed order is what the job runs with. + * + * Kept in sync with server/dive_tasks/multicam_pipeline.py. */ +/** The sensor modalities DIVE knows how to match. */ +export type CameraRole = 'eo' | 'ir' | 'uv'; +export const CAMERA_ROLES: readonly CameraRole[] = ['eo', 'ir', 'uv']; + +export const CAMERA_ROLE_LABELS: Record = { + eo: 'Optical (EO)', + ir: 'Thermal (IR)', + uv: 'Ultraviolet (UV)', +}; + /** * Role aliases: a slot token and a camera name match when they share a role, * or when the token itself appears as a segment of the camera name (so pipes * can name cameras literally, e.g. `# Camera Order: left, right`). */ -export const CAMERA_ROLE_ALIASES: Record = { +export const CAMERA_ROLE_ALIASES: Record = { eo: ['eo', 'rgb', 'optical', 'color', 'colour', 'vis', 'visible'], ir: ['ir', 'thermal', 'lwir', 'mwir', 'flir'], uv: ['uv', 'ultraviolet'], @@ -27,11 +42,47 @@ function segments(name: string): string[] { return name.toLowerCase().split(/[^a-z0-9]+/).filter((s) => s); } -function roleOf(token: string): string | null { +/** The role a slot token or camera-name segment denotes, if any. */ +export function roleOfToken(token: string): CameraRole | null { const lower = token.toLowerCase(); - const found = Object.entries(CAMERA_ROLE_ALIASES) - .find(([, aliases]) => aliases.includes(lower)); - return found ? found[0] : null; + const found = CAMERA_ROLES.find((role) => CAMERA_ROLE_ALIASES[role].includes(lower)); + return found ?? null; +} + +/** + * Infer a camera's role from its name, falling back to tokens in its image + * file names (KAMERA style `..._rgb.jpg` / `_ir.tif` / `_uv.jpg`). Only a + * unanimous answer counts: a name or file set naming two roles yields null. + */ +export function inferCameraRole(cameraName: string, imageNames: string[] = []): CameraRole | null { + const fromName = new Set(segments(cameraName).map(roleOfToken).filter((r): r is CameraRole => !!r)); + if (fromName.size === 1) { + return [...fromName][0]; + } + if (fromName.size > 1) { + return null; + } + const fromImages = new Set(); + imageNames.slice(0, 50).forEach((image) => { + const base = image.split(/[\\/]/).pop() ?? image; + const stem = base.replace(/\.[^.]+$/, ''); + segments(stem).map(roleOfToken).forEach((r) => { if (r) fromImages.add(r); }); + }); + return fromImages.size === 1 ? [...fromImages][0] : null; +} + +/** Infer roles for a whole rig; cameras DIVE cannot classify are omitted. */ +export function inferCameraRoles( + cameras: Record, +): Record { + const roles: Record = {}; + Object.entries(cameras).forEach(([name, images]) => { + const role = inferCameraRole(name, images ?? []); + if (role) { + roles[name] = role; + } + }); + return roles; } /** Cameras whose name matches a slot token, by literal segment or shared role. */ @@ -41,8 +92,8 @@ export function camerasMatchingSlot(token: string, cameras: string[]): string[] if (exact.length) { return exact; } - const role = roleOf(token); - const aliases = new Set(role ? CAMERA_ROLE_ALIASES[role] : [lower]); + const role = roleOfToken(token); + const aliases = new Set(role ? CAMERA_ROLE_ALIASES[role] : [lower]); return cameras.filter((camera) => segments(camera).some((seg) => aliases.has(seg))); } @@ -51,14 +102,16 @@ export type PipelineCameraOrderResult = | { order?: undefined; error: string }; /** - * Resolve declared slots to dataset cameras. Every slot must match exactly one - * camera and no camera may fill two slots; anything else is an error message - * naming the slot, the pipe's slots and the dataset's cameras so the user can - * rename cameras (or fix the header) rather than get a silently mis-wired run. + * Resolve declared slots to dataset cameras without user interaction (CLI, + * and the fallback when no confirmed order was supplied). Every slot must + * match exactly one camera and no camera may fill two slots; anything else + * is an error message naming the slot, the pipe's slots and the dataset's + * cameras. */ export function resolvePipelineCameraOrder( slots: string[], cameras: string[], + roles: Record = {}, ): PipelineCameraOrderResult { const context = `pipeline cameras [${slots.join(', ')}], dataset cameras [${cameras.join(', ')}]`; if (slots.length !== cameras.length) { @@ -69,14 +122,14 @@ export function resolvePipelineCameraOrder( const order: string[] = []; for (let i = 0; i < slots.length; i += 1) { const slot = slots[i]; - const matches = camerasMatchingSlot(slot, cameras).filter((c) => !order.includes(c)); + const matches = candidatesForSlot(slot, cameras, roles).filter((c) => !order.includes(c)); if (matches.length !== 1) { const why = matches.length === 0 ? 'no dataset camera matches' : `several dataset cameras match (${matches.join(', ')})`; return { error: `Cannot place pipeline camera "${slot}" (input${i + 1}): ${why}. ${context}. ` - + 'Rename the dataset cameras so each pipeline camera matches exactly one.', + + 'Set the camera roles (or rename the cameras) so each pipeline camera matches exactly one.', }; } order.push(matches[0]); @@ -84,6 +137,61 @@ export function resolvePipelineCameraOrder( return { order }; } +/** + * Cameras that could fill a slot: cameras whose assigned role equals the + * slot's role take precedence over name matching, so a corrected role wins + * over a misleading name. + */ +export function candidatesForSlot( + slot: string, + cameras: string[], + roles: Record = {}, +): string[] { + const role = roleOfToken(slot); + if (role) { + const byRole = cameras.filter((camera) => roles[camera] === role); + if (byRole.length) { + return byRole; + } + } + return camerasMatchingSlot(slot, cameras); +} + +/** + * Propose a camera for every slot for the confirmation step. Unlike + * {@link resolvePipelineCameraOrder} this never fails: a slot with no unique + * candidate is proposed as null and left for the user to fill. + */ +export function prefillPipelineCameraOrder( + slots: string[], + cameras: string[], + roles: Record = {}, +): (string | null)[] { + const taken = new Set(); + const proposed: (string | null)[] = slots.map(() => null); + // Unique matches first, so an ambiguous slot cannot steal a camera another + // slot needs unambiguously. + slots.forEach((slot, index) => { + const matches = candidatesForSlot(slot, cameras, roles).filter((c) => !taken.has(c)); + if (matches.length === 1) { + [proposed[index]] = matches; + taken.add(matches[0]); + } + }); + return proposed; +} + +/** + * Slot labels for a pipe: its declared `# Camera Order:` tokens, else the + * bare `input1..N` positions. + */ +export function pipelineCameraSlots(declaredOrder: string[] | null | undefined, count: number): string[] { + if (declaredOrder?.length) { + return declaredOrder; + } + return Array.from({ length: count }, (_, i) => `input${i + 1}`); +} + /** Parse the value of a `# Camera Order:` header into slot tokens. */ export function parseCameraOrderHeader(value: string): string[] { return value.trim().split(/[\s,]+/).filter((token) => token); diff --git a/client/platform/desktop/backend/native/multiCamImport.ts b/client/platform/desktop/backend/native/multiCamImport.ts index 9119f7517..48a425e7b 100644 --- a/client/platform/desktop/backend/native/multiCamImport.ts +++ b/client/platform/desktop/backend/native/multiCamImport.ts @@ -12,6 +12,7 @@ import { MultiType, } from 'dive-common/constants'; import { preferEoIrSubfolderOrder } from 'dive-common/components/ImportMultiCamDialog/multicamSubfolderLayout'; +import { inferCameraRoles } from 'dive-common/pipelineCameraOrder'; import { JsonConfig, JsonConfigCurrentVersion, DesktopMediaImportResponse, @@ -371,6 +372,19 @@ async function beginMultiCamImport(args: MultiCamImportArgs): Promise [ + name, + camera.originalImageFiles.length ? camera.originalImageFiles : [camera.originalVideoFile], + ]), + )); + if (Object.keys(cameraRoles).length) { + jsonConfig.cameraRoles = cameraRoles; + } + return { jsonConfig, globPattern: '', diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 7a5ea5cd2..5bfbea009 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -15,6 +15,7 @@ import { observeChild } from 'platform/desktop/backend/native/processManager'; import { convertMedia } from 'platform/desktop/backend/native/mediaJobs'; import sendToRenderer from 'platform/desktop/background'; +import type { Pipe } from 'dive-common/apispec'; import { MultiType, stereoPipelineMarker, @@ -164,6 +165,24 @@ async function importNewMedia( /** * a node.js implementation of dive_tasks.tasks.run_pipeline */ +/** + * The input1..N camera order for a 2-cam/3-cam run: the confirmed order when + * one was supplied (validated against the dataset's cameras), else resolved + * from the pipe's declared slots and the dataset's camera roles/names. + */ +function resolveMultiCamOrder(meta: JsonConfig, pipeline: Pipe, confirmed?: string[]): string[] { + const cameras = Object.keys(meta.multiCam?.cameras ?? {}); + if (confirmed?.length) { + const unknown = confirmed.filter((name) => !cameras.includes(name)); + const missing = cameras.filter((name) => !confirmed.includes(name)); + if (unknown.length || missing.length || new Set(confirmed).size !== confirmed.length) { + throw new Error(`Camera assignment [${confirmed.join(', ')}] does not match the dataset cameras [${cameras.join(', ')}]`); + } + return confirmed; + } + return pipelineCameraNames(meta.multiCam, pipeline.metadata?.cameraOrder, meta.cameraRoles ?? {}); +} + async function runPipeline( settings: Settings, runPipelineArgs: RunPipeline, @@ -369,10 +388,11 @@ async function runPipeline( let multiOutFiles: Record; if (meta.multiCam && stereoOrMultiCam) { const isMultiCamPipeline = multiCamPipelineMarkers.includes(pipeline.type); - // 2-cam/3-cam pipes: which camera feeds which inputN is the pipe's - // contract (its `# Camera Order:` header), else reference-first. + // 2-cam/3-cam pipes: which camera feeds which inputN is the order the + // user confirmed before the run; without one (CLI), the pipe's + // `# Camera Order:` header matched by role/name, else reference-first. const multiCamOrder = isMultiCamPipeline - ? pipelineCameraNames(meta.multiCam, pipeline.metadata?.cameraOrder) + ? resolveMultiCamOrder(meta, pipeline, runPipelineArgs.pipelineParams?.cameraOrder) : undefined; const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput, false, multiCamOrder); Object.entries(argFilePair).forEach(([arg, file]) => { diff --git a/docs/Pipeline-Import-Export.md b/docs/Pipeline-Import-Export.md index 6968a0689..ec60ec8e1 100644 --- a/docs/Pipeline-Import-Export.md +++ b/docs/Pipeline-Import-Export.md @@ -75,7 +75,15 @@ Example: | `# Calibration Keys: [k…]` | Opt-in: binds the dataset's stereo calibration file to each listed KWIVER config key at run time (one `-s =` per key). Keys may be space- or comma-separated. Use this when the pipe's calibration consumer is not the conventional `measurer:calibration_file` / `calibration_reader:file` pair (those are used when the header is unset). Needed because `$CONFIG{global:…}` indirection cannot receive `-s` overrides (macros expand at parse time; `-s` blocks are appended last). | | `# Metadata File: :` | Opt-in: when the dataset has an attached **Metadata File**, DIVE appends a KWIVER override `-s :=` at run time. The same CSV/TXT attachment is also considered for [Frame Metadata](Frame-Metadata.md). Without this header, no metadata file is injected. | | `# Image List Keys: [k…]` | Opt-in: binds the run's per-camera input image list(s) to each listed KWIVER key. Keys may be space- or comma-separated. A key containing `{cam}` is expanded once per camera (1-based), e.g. `stabilizer:image_list{cam}` → `image_list1`, `image_list2`, …. A key without `{cam}` receives camera 1's list only. | -| `# Camera Order: [cam…]` | 2-cam/3-cam pipes only: names the camera role fed to each `inputN`, in order (e.g. `# Camera Order: EO, UV, IR` → `input1` optical, `input2` ultraviolet, `input3` thermal). At run time DIVE matches each slot to a dataset camera by name — an exact camera name, or a name segment sharing the slot's role (`EO` ≈ eo/rgb/optical/color/vis, `IR` ≈ ir/thermal/lwir/flir, `UV` ≈ uv/ultraviolet); other tokens match literally (`left`, `right`). If any slot matches no camera or several, the run is refused with a message naming the pipe's slots and the dataset's cameras. Camera 1 is the frame the pipe's warp processes map onto: each other camera's Aligned View registration onto camera 1 is written to the job as `_to__registration.json` and bound to `warpN`. Without this header, cameras are fed registration-reference (default display) first, then display order. | +| `# Camera Order: [cam…]` | 2-cam/3-cam pipes only: names the camera role fed to each `inputN`, in order (e.g. `# Camera Order: EO, UV, IR` → `input1` optical, `input2` ultraviolet, `input3` thermal). Role tokens are `EO`, `IR`, `UV` (aliases: eo/rgb/optical/color/vis, ir/thermal/lwir/flir, uv/ultraviolet); any other token (`left`, `right`) matches a camera name literally. See [Multicam camera assignment](#multicam-camera-assignment) for how DIVE places a dataset's cameras onto these slots. Camera 1 is the frame the pipe's warp processes map onto: each other camera's Aligned View registration onto camera 1 is written to the job as `_to__registration.json` and bound to `warpN`. | + +### Multicam camera assignment + +Which dataset camera feeds which input of a 2-cam/3-cam pipe is decided in the open, not inferred from display order: + +1. **Camera roles.** Each camera of a multicam dataset carries a sensor role (`eo`, `ir`, `uv`) in `cameraRoles`, inferred once at import from the camera (subfolder) name and, failing that, from tokens in its image file names (KAMERA style `…_rgb.jpg`, `…_ir.tif`, `…_uv.jpg`). Only a unanimous answer is recorded; ambiguous cameras get no role. +2. **Assignment step.** When you run a 2-cam/3-cam pipe, DIVE shows one row per pipeline camera (from the `# Camera Order:` header, or plain `input1..N` when the pipe has none) with the dataset camera it proposes — matched by role when both sides have one, else by name — and you confirm or change it before anything runs. Unfilled or duplicated slots block the run. Confirming a role-labelled slot saves the roles back onto the dataset (uncheck *Save these as the dataset's camera roles* to skip), so a corrected role wins over a misleading name next time and for every other pipeline. +3. **Job.** The confirmed order is what the job runs with (`cameraOrder` in the pipeline params; visible in the desktop job manifest). Runs started without the step (CLI) fall back to matching the header by role/name, and to registration-reference-first order for pipes with no header. ### Metadata File vs Configuration File diff --git a/server/dive_server/crud_dataset.py b/server/dive_server/crud_dataset.py index ef1dda5fb..84206d0e5 100644 --- a/server/dive_server/crud_dataset.py +++ b/server/dive_server/crud_dataset.py @@ -16,6 +16,7 @@ from dive_server import crud, crud_annotation from dive_tasks import tasks +from dive_tasks.multicam_pipeline import infer_camera_roles from dive_utils import ( TRUTHY_META_VALUES, asbool, @@ -1692,6 +1693,7 @@ def create_multicam( default_child = loaded_children[validated.defaultDisplay] parent_folder_doc = parent_folder multi_cam_cameras: Dict[str, Dict[str, str]] = {} + camera_image_names: Dict[str, List[str]] = {} for name in camera_order: child = loaded_children[name] if child['name'] != name: @@ -1701,6 +1703,11 @@ def create_multicam( 'folderId': str(child['_id']), 'type': camera_types_by_name[name], } + camera_image_names[name] = [item['name'] for item in Folder().childItems(child, limit=50)] + # Sensor role per camera, from the camera name and its media names; the + # pipeline camera-assignment step prefills from it and the user can + # correct it there. + camera_roles = infer_camera_roles(camera_image_names) calibration_source_item_id = None json_calibration_item_id = None @@ -1780,6 +1787,7 @@ def create_multicam( else {} ), }, + **({'cameraRoles': camera_roles} if camera_roles else {}), } parent_folder_doc['meta'].setdefault( constants.ConfidenceFiltersMarker, diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index de2d2e679..22cd0ec2a 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -340,10 +340,22 @@ def run_pipeline( # feeds which inputN is the pipe's contract (`# Camera Order:` # header); a pipe without one gets the registration reference # first, then display order. + confirmed_order = (pipeline_params or {}).get('cameraOrder') declared_order = (pipeline.get('metadata') or {}).get('cameraOrder') - if declared_order: + if confirmed_order: + # The order the user confirmed in the camera-assignment step. + if sorted(confirmed_order) != sorted(camera_order): + raise RestException( + f'Camera assignment [{", ".join(confirmed_order)}] does not match ' + f'the dataset cameras [{", ".join(camera_order)}]', + code=400, + ) + camera_order = list(confirmed_order) + elif declared_order: try: - camera_order = resolve_pipeline_camera_order(declared_order, camera_order) + camera_order = resolve_pipeline_camera_order( + declared_order, camera_order, (folder.get('meta') or {}).get('cameraRoles') + ) except ValueError as err: raise RestException(str(err), code=400) from err else: diff --git a/server/dive_tasks/multicam_pipeline.py b/server/dive_tasks/multicam_pipeline.py index d1796ebeb..63d9c500e 100644 --- a/server/dive_tasks/multicam_pipeline.py +++ b/server/dive_tasks/multicam_pipeline.py @@ -105,6 +105,58 @@ def _name_segments(name: str) -> List[str]: return [seg for seg in re.split(r'[^a-z0-9]+', name.lower()) if seg] +def role_of_token(token: str) -> Optional[str]: + """The role (eo / ir / uv) a slot token or camera-name segment denotes, if any.""" + lower = token.lower() + return next((role for role, aliases in CAMERA_ROLE_ALIASES.items() if lower in aliases), None) + + +def infer_camera_role(camera_name: str, image_names: Optional[List[str]] = None) -> Optional[str]: + """ + Infer a camera's sensor role from its name, falling back to tokens in its + image file names (KAMERA style `..._rgb.jpg` / `_ir.tif` / `_uv.jpg`). + Only a unanimous answer counts: a name or file set naming two roles yields + None. Kept in sync with client/dive-common/pipelineCameraOrder.ts. + """ + from_name = {r for r in (role_of_token(seg) for seg in _name_segments(camera_name)) if r} + if len(from_name) == 1: + return next(iter(from_name)) + if len(from_name) > 1: + return None + from_images = set() + for image in (image_names or [])[:50]: + base = re.split(r'[\\/]', image)[-1] + stem = re.sub(r'\.[^.]+$', '', base) + from_images.update(r for r in (role_of_token(seg) for seg in _name_segments(stem)) if r) + return next(iter(from_images)) if len(from_images) == 1 else None + + +def infer_camera_roles(cameras: Dict[str, Optional[List[str]]]) -> Dict[str, str]: + """Infer roles for a whole rig; cameras that cannot be classified are omitted.""" + roles: Dict[str, str] = {} + for name, images in cameras.items(): + role = infer_camera_role(name, images or []) + if role: + roles[name] = role + return roles + + +def candidates_for_slot( + slot: str, cameras: List[str], roles: Optional[Dict[str, str]] = None +) -> List[str]: + """ + Cameras that could fill a slot: cameras whose assigned role equals the + slot's role take precedence over name matching, so a corrected role wins + over a misleading name. + """ + role = role_of_token(slot) + if role and roles: + by_role = [camera for camera in cameras if roles.get(camera) == role] + if by_role: + return by_role + return cameras_matching_slot(slot, cameras) + + def cameras_matching_slot(token: str, cameras: List[str]) -> List[str]: """Cameras whose name matches a slot token, by exact name, segment, or shared role.""" lower = token.lower() @@ -116,11 +168,14 @@ def cameras_matching_slot(token: str, cameras: List[str]) -> List[str]: return [camera for camera in cameras if any(seg in aliases for seg in _name_segments(camera))] -def resolve_pipeline_camera_order(slots: List[str], cameras: List[str]) -> List[str]: +def resolve_pipeline_camera_order( + slots: List[str], cameras: List[str], roles: Optional[Dict[str, str]] = None +) -> List[str]: """ - Map a pipe's declared `# Camera Order:` slots onto dataset cameras: every - slot must match exactly one camera and no camera may fill two slots. - Raises ValueError with a message naming the slot, the pipe's slots and the + Map a pipe's declared `# Camera Order:` slots onto dataset cameras without + user interaction: every slot must match exactly one camera (by assigned + role first, then by name) and no camera may fill two slots. Raises + ValueError with a message naming the slot, the pipe's slots and the dataset's cameras otherwise, so the run fails up front instead of being silently mis-wired. """ @@ -132,7 +187,7 @@ def resolve_pipeline_camera_order(slots: List[str], cameras: List[str]) -> List[ ) order: List[str] = [] for index, slot in enumerate(slots, start=1): - matches = [c for c in cameras_matching_slot(slot, cameras) if c not in order] + matches = [c for c in candidates_for_slot(slot, cameras, roles) if c not in order] if len(matches) != 1: why = ( 'no dataset camera matches' @@ -141,7 +196,8 @@ def resolve_pipeline_camera_order(slots: List[str], cameras: List[str]) -> List[ ) raise ValueError( f'Cannot place pipeline camera "{slot}" (input{index}): {why}. {context}. ' - 'Rename the dataset cameras so each pipeline camera matches exactly one.' + 'Set the camera roles (or rename the cameras) so each pipeline camera ' + 'matches exactly one.' ) order.append(matches[0]) return order diff --git a/server/dive_utils/models.py b/server/dive_utils/models.py index db5039e00..7a4492e98 100644 --- a/server/dive_utils/models.py +++ b/server/dive_utils/models.py @@ -242,6 +242,9 @@ class PairHomography(BaseModel): CameraTransformType = Literal['translation', 'rigid', 'similarity', 'affine', 'homography'] TypeHierarchy = Dict[StrictStr, StrictStr] +# Sensor modality of a multicam camera; see dive_tasks.multicam_pipeline.CAMERA_ROLE_ALIASES. +CameraRole = Literal['eo', 'ir', 'uv'] + class MetadataMutable(BaseModel): version = ( @@ -271,6 +274,10 @@ class MetadataMutable(BaseModel): # DIVE; preserved verbatim so refined calibrations can be traced back to the # model version they were made against. cameraRegistrationSource: Optional[Dict[str, Any]] + # Sensor role per multicam camera name, inferred at import from the camera + # and image names and editable afterwards; used to place cameras onto a + # pipeline's declared camera slots. Cameras with no known role are absent. + cameraRoles: Optional[Dict[str, CameraRole]] fps: Optional[float] @staticmethod diff --git a/server/dive_utils/types.py b/server/dive_utils/types.py index 9f24cfa66..a88685c6f 100644 --- a/server/dive_utils/types.py +++ b/server/dive_utils/types.py @@ -119,6 +119,10 @@ class PipelineRuntimeParams(TypedDict, total=False): class PipelineParams(TypedDict, total=False): kwiverParams: Dict[str, str] runtimeParams: PipelineRuntimeParams + # 2-cam/3-cam pipes: the dataset camera to feed each inputN, in order, as + # confirmed by the user before the run. When omitted the pipe's declared + # `# Camera Order:` slots are matched by role/name, else reference-first. + cameraOrder: List[str] # Name for the newly created dataset (filter / transcode / disparity). outputDatasetName: str # Optional Girder folder that should own the new dataset (else sibling of input). diff --git a/server/tests/test_multicam_pipeline.py b/server/tests/test_multicam_pipeline.py index a75405dc3..8590ca6b8 100644 --- a/server/tests/test_multicam_pipeline.py +++ b/server/tests/test_multicam_pipeline.py @@ -11,6 +11,8 @@ build_registration_pairs, cameras_matching_slot, find_downloaded_calibration_file, + infer_camera_role, + infer_camera_roles, is_stereo_measurement_pipeline, is_stereo_or_multicam_pipeline, pipeline_camera_order, @@ -148,6 +150,27 @@ def test_cameras_matching_slot(): assert cameras_matching_slot('left', ['left_cam', 'right_cam']) == ['left_cam'] +def test_infer_camera_role(): + assert infer_camera_role('rgb') == 'eo' + assert infer_camera_role('CENT_IR') == 'ir' + assert infer_camera_role('uv_cam') == 'uv' + assert infer_camera_role('cam1', ['flight_0001_rgb.jpg', 'flight_0002_rgb.jpg']) == 'eo' + assert infer_camera_role('cam1', ['a_rgb.jpg', 'b_ir.tif']) is None + assert infer_camera_role('eo_ir') is None + assert infer_camera_role('center', ['0001.png']) is None + assert infer_camera_roles({'rgb': [], 'center': ['x_ir.tif'], 'other': ['a.png']}) == { + 'rgb': 'eo', + 'center': 'ir', + } + + +def test_resolve_pipeline_camera_order_roles_win_over_names(): + # "thermal" is named like IR but the user marked it optical. + assert resolve_pipeline_camera_order( + ['EO', 'IR'], ['thermal', 'other'], {'thermal': 'eo', 'other': 'ir'} + ) == ['thermal', 'other'] + + def test_resolve_pipeline_camera_order(): assert resolve_pipeline_camera_order(['EO', 'UV', 'IR'], ['rgb', 'ir', 'uv']) == [ 'rgb', From 7a8d89155ca6cd9ef4ff9adf1879384efcfc3b5f Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 17 Aug 2026 11:39:09 -0400 Subject: [PATCH 6/9] Check camera registrations before a multicam run, not at pipe configure time A 2-cam/3-cam pipe whose warpN camera had no registration onto camera 1 launched anyway and died inside kwiver with "Path does not exist: registration_camera2_to_camera1.json". Do the check before the job exists: - Pipeline discovery (desktop + web) records registrationWarps: the input positions of `process warpN :: warp_detections | warp_image` in the pipe body, e.g. [2, 3]. - The camera-assignment dialog shows, for each warped row, whether the chosen camera has a fitted Aligned View registration onto camera 1, and blocks Run with "Register X -> Y in Aligned View first" when it does not. - Desktop (viame.ts) and web (crud_rpc) refuse the run with the same message before creating the job, so CLI and API callers get it too. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MD6kx1uDv4NuiKdFNnYpG2 --- client/dive-common/apispec.ts | 8 +++ .../components/PipelineCameraAssignDialog.vue | 57 +++++++++++++++++-- .../components/RunPipelineMenu.vue | 2 + .../dive-common/pipelineCameraOrder.spec.ts | 17 +++++- client/dive-common/pipelineCameraOrder.ts | 44 ++++++++++++++ .../platform/desktop/backend/native/common.ts | 17 ++++++ .../platform/desktop/backend/native/viame.ts | 15 ++++- docs/Pipeline-Import-Export.md | 3 +- server/dive_server/crud_rpc.py | 23 ++++++++ server/dive_tasks/multicam_pipeline.py | 35 ++++++++++++ server/dive_tasks/pipeline_discovery.py | 17 ++++++ server/dive_utils/types.py | 4 ++ server/tests/test_multicam_pipeline.py | 13 +++++ server/tests/test_pipeline_discovery.py | 22 +++++++ 14 files changed, 269 insertions(+), 8 deletions(-) diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index ef1fcd718..874955578 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -87,6 +87,14 @@ interface PipeMetadata { * cameras are fed registration-reference first, then display order. */ cameraOrder?: string[]; + /** + * Input positions whose detections/images the pipe warps onto camera 1 + * (`process warpN :: warp_detections | warp_image` in the pipe body), e.g. + * [2, 3]. Each such camera needs a fitted Aligned View registration onto + * camera 1; DIVE checks that before the run instead of letting the pipe + * fail at configure time on a missing file. + */ + registrationWarps?: number[]; } interface PipelineRuntimeParams { diff --git a/client/dive-common/components/PipelineCameraAssignDialog.vue b/client/dive-common/components/PipelineCameraAssignDialog.vue index 223e889b1..42f7ee02a 100644 --- a/client/dive-common/components/PipelineCameraAssignDialog.vue +++ b/client/dive-common/components/PipelineCameraAssignDialog.vue @@ -3,7 +3,7 @@ import { computed, defineComponent, PropType, ref, watch, } from 'vue'; import { - CameraRole, CAMERA_ROLE_LABELS, roleOfToken, + CameraRole, CAMERA_ROLE_LABELS, describeMissingRegistration, missingRegistrations, roleOfToken, } from 'dive-common/pipelineCameraOrder'; /** @@ -21,6 +21,10 @@ export interface PipelineCameraAssignRequest { proposed: (string | null)[]; /** Current dataset roles, shown next to each camera in the pickers. */ roles: Record; + /** Input positions the pipe warps onto camera 1 (metadata.registrationWarps). */ + registrationWarps: number[]; + /** Fitted registration pair keys (`a::b`) the dataset holds. */ + fittedPairs: string[]; } export interface PipelineCameraAssignResult { @@ -91,8 +95,33 @@ export default defineComponent({ const changed = computed(() => (props.request?.proposed ?? []) .some((camera, index) => camera !== selection.value[index])); + /** Warped inputs whose chosen camera has no registration onto camera 1. */ + const missing = computed(() => missingRegistrations( + selection.value, + props.request?.registrationWarps, + props.request?.fittedPairs ?? [], + )); + const missingMessages = computed(() => missing.value + .map((entry) => describeMissingRegistration(entry))); + function rowRegistrationHint(index: number): string | null { + const warps = props.request?.registrationWarps ?? []; + const target = selection.value[0]; + const camera = selection.value[index]; + if (index === 0 || !warps.includes(index + 1) || !camera || !target) { + return null; + } + const isMissing = missing.value.some((entry) => entry.input === index + 1); + return isMissing + ? `No registration of ${camera} onto ${target}` + : `Registered onto ${target}`; + } + function rowMissing(index: number): boolean { + return missing.value.some((entry) => entry.input === index + 1); + } + const blocked = computed(() => problems.value.length > 0 || missing.value.length > 0); + function confirm() { - if (problems.value.length || !props.request) { + if (blocked.value || !props.request) { return; } const order = selection.value as string[]; @@ -116,6 +145,10 @@ export default defineComponent({ slotLabel, cameraLabel, problems, + missingMessages, + rowRegistrationHint, + rowMissing, + blocked, changed, confirm, }; @@ -147,9 +180,11 @@ export default defineComponent({ v-model="selection[index]" :items="request.cameras.map((camera) => ({ text: cameraLabel(camera), value: camera }))" :label="slotLabel(index)" + :hint="rowRegistrationHint(index) || undefined" + :persistent-hint="!!rowRegistrationHint(index)" + :error="rowMissing(index)" outlined dense - hide-details class="mb-3" /> + +
+ {{ message }} +
+ @@ -185,7 +234,7 @@ export default defineComponent({ {{ changed ? 'Run with these cameras' : 'Run' }} diff --git a/client/dive-common/components/RunPipelineMenu.vue b/client/dive-common/components/RunPipelineMenu.vue index 5b71a4052..efda23a32 100644 --- a/client/dive-common/components/RunPipelineMenu.vue +++ b/client/dive-common/components/RunPipelineMenu.vue @@ -319,6 +319,8 @@ export default defineComponent({ cameras, proposed: prefillPipelineCameraOrder(slots, cameras, roles), roles, + registrationWarps: pipeline.metadata?.registrationWarps ?? [], + fittedPairs: Object.keys(config.cameraHomographies ?? {}), }); if (!result) { return null; diff --git a/client/dive-common/pipelineCameraOrder.spec.ts b/client/dive-common/pipelineCameraOrder.spec.ts index b42522027..e52dd9a7e 100644 --- a/client/dive-common/pipelineCameraOrder.spec.ts +++ b/client/dive-common/pipelineCameraOrder.spec.ts @@ -1,7 +1,8 @@ import { describe, expect, it } from 'vitest'; import { - camerasMatchingSlot, inferCameraRole, inferCameraRoles, parseCameraOrderHeader, - pipelineCameraSlots, prefillPipelineCameraOrder, resolvePipelineCameraOrder, + camerasMatchingSlot, inferCameraRole, inferCameraRoles, missingRegistrations, + parseCameraOrderHeader, pipelineCameraSlots, prefillPipelineCameraOrder, + resolvePipelineCameraOrder, } from './pipelineCameraOrder'; import { pipelineCameraNames } from './multicamDisplay'; @@ -68,6 +69,18 @@ describe('pipelineCameraOrder', () => { expect(pipelineCameraSlots(undefined, 3)).toStrictEqual(['input1', 'input2', 'input3']); }); + it('reports warped cameras with no fitted registration onto camera 1', () => { + const order = ['rgb', 'uv', 'ir']; + // Stored in the reverse orientation still counts. + expect(missingRegistrations(order, [2, 3], ['rgb::ir'])) + .toStrictEqual([{ input: 2, camera: 'uv', target: 'rgb' }]); + expect(missingRegistrations(order, [2, 3], ['rgb::ir', 'uv::rgb'])).toStrictEqual([]); + expect(missingRegistrations(order, undefined, [])).toStrictEqual([]); + expect(missingRegistrations([null, 'uv'], [2], [])).toStrictEqual([]); + // An unfilled slot is reported by the assignment problems, not here. + expect(missingRegistrations(['rgb', null], [2], [])).toStrictEqual([]); + }); + it('pipelineCameraNames uses the declared order or falls back to reference-first', () => { const media = { cameras: { ir: {}, rgb: {}, uv: {} }, diff --git a/client/dive-common/pipelineCameraOrder.ts b/client/dive-common/pipelineCameraOrder.ts index 9abd13346..112bf4180 100644 --- a/client/dive-common/pipelineCameraOrder.ts +++ b/client/dive-common/pipelineCameraOrder.ts @@ -192,6 +192,50 @@ export function pipelineCameraSlots(declaredOrder: string[] | null | undefined, return Array.from({ length: count }, (_, i) => `input${i + 1}`); } +export interface MissingRegistration { + /** 1-based pipeline input position. */ + input: number; + camera: string; + /** Camera 1, the frame the warp maps onto. */ + target: string; +} + +/** + * Cameras a pipe will warp (its `warpN` processes) that have no fitted + * registration onto camera 1 of the given order. `fittedPairs` are the + * dataset's homography keys (`a::b`, either orientation counts). Checked + * before a run so the failure is "register camera X onto Y first" rather + * than the pipe dying at configure time on a missing file. + */ +export function missingRegistrations( + order: (string | null)[], + registrationWarps: number[] | null | undefined, + fittedPairs: string[], +): MissingRegistration[] { + const target = order[0]; + if (!target || !registrationWarps?.length) { + return []; + } + const fitted = new Set(fittedPairs); + const missing: MissingRegistration[] = []; + registrationWarps.forEach((input) => { + const camera = order[input - 1]; + if (!camera || camera === target) { + return; + } + if (!fitted.has(`${camera}::${target}`) && !fitted.has(`${target}::${camera}`)) { + missing.push({ input, camera, target }); + } + }); + return missing; +} + +export function describeMissingRegistration(missing: MissingRegistration, pipelineName?: string): string { + const where = pipelineName ? ` before running ${pipelineName}` : ''; + return `Camera "${missing.camera}" (input${missing.input}) has no registration onto camera 1 ` + + `("${missing.target}"). Register ${missing.camera} → ${missing.target} in Aligned View${where}.`; +} + /** Parse the value of a `# Camera Order:` header into slot tokens. */ export function parseCameraOrderHeader(value: string): string[] { return value.trim().split(/[\s,]+/).filter((token) => token); diff --git a/client/platform/desktop/backend/native/common.ts b/client/platform/desktop/backend/native/common.ts index 54e2dbbc5..6cc912675 100644 --- a/client/platform/desktop/backend/native/common.ts +++ b/client/platform/desktop/backend/native/common.ts @@ -285,11 +285,25 @@ async function extractPipeMetadata(filePath: string): Promise { const lines = await readLines(filePath); let inDescription = false; let fullDescription = ''; + // `process warpN` followed by `:: warp_detections|warp_image` marks an + // input whose camera must be registered onto camera 1. + let lastProcessName: string | null = null; + const registrationWarps: number[] = []; lines.forEach((line) => { const trimmed = line.trim(); if (!trimmed) return; + const processMatch = trimmed.match(/^process\s+(\S+)/); + if (processMatch) { + [, lastProcessName] = processMatch; + } else if (/^::\s*(warp_detections|warp_image)\b/.test(trimmed) && lastProcessName) { + const warpMatch = lastProcessName.match(/^warp(\d+)$/); + if (warpMatch) { + registrationWarps.push(Number.parseInt(warpMatch[1], 10)); + } + } + // --- Description extraction (Multiline) --- if (/^#\s*Description:\s*/i.test(line)) { inDescription = true; @@ -365,6 +379,9 @@ async function extractPipeMetadata(filePath: string): Promise { } } }); + if (registrationWarps.length) { + metadata.registrationWarps = [...new Set(registrationWarps)].sort((a, b) => a - b); + } metadata.description = fullDescription.trim() || undefined; } catch (error) { console.error(`Error while reading ${filePath} metadata`, error); diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 5bfbea009..840d692a8 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -29,6 +29,7 @@ import { pipelineCreatesNewDataset, } from 'dive-common/pipelineCreatesDataset'; import { pipelineCameraNames } from 'dive-common/multicamDisplay'; +import { describeMissingRegistration, missingRegistrations } from 'dive-common/pipelineCameraOrder'; import * as common from './common'; import { jobFileEchoMiddleware, createWorkingDirectory, createCustomWorkingDirectory, splitExt, @@ -424,8 +425,20 @@ async function runPipeline( } if (multiCamOrder) { // Hand the camera registration (Aligned View homographies) to the - // pipeline's per-camera warp processes. + // pipeline's per-camera warp processes. Refuse up front when a warped + // camera has no registration onto camera 1, rather than letting the + // pipe die at configure time on a missing file. const registrationArgs = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, multiCamOrder); + // buildRegistrationPipelineArgs only binds warps whose camera has a + // fitted pair onto camera 1, so the bound warps are the fitted set. + const fittedPairs = Object.keys(registrationArgs) + .map((key) => key.match(/^warp(\d+):transformation_file$/)) + .filter((match): match is RegExpMatchArray => match !== null) + .map((match) => `${multiCamOrder[Number.parseInt(match[1], 10) - 1]}::${multiCamOrder[0]}`); + const missing = missingRegistrations(multiCamOrder, pipeline.metadata?.registrationWarps, fittedPairs); + if (missing.length) { + throw new Error(missing.map((entry) => describeMissingRegistration(entry, pipeline.name)).join('\n')); + } Object.entries(registrationArgs).forEach(([arg, value]) => { command.push(`-s ${arg}="${value}"`); }); diff --git a/docs/Pipeline-Import-Export.md b/docs/Pipeline-Import-Export.md index ec60ec8e1..e925c9d40 100644 --- a/docs/Pipeline-Import-Export.md +++ b/docs/Pipeline-Import-Export.md @@ -83,7 +83,8 @@ Which dataset camera feeds which input of a 2-cam/3-cam pipe is decided in the o 1. **Camera roles.** Each camera of a multicam dataset carries a sensor role (`eo`, `ir`, `uv`) in `cameraRoles`, inferred once at import from the camera (subfolder) name and, failing that, from tokens in its image file names (KAMERA style `…_rgb.jpg`, `…_ir.tif`, `…_uv.jpg`). Only a unanimous answer is recorded; ambiguous cameras get no role. 2. **Assignment step.** When you run a 2-cam/3-cam pipe, DIVE shows one row per pipeline camera (from the `# Camera Order:` header, or plain `input1..N` when the pipe has none) with the dataset camera it proposes — matched by role when both sides have one, else by name — and you confirm or change it before anything runs. Unfilled or duplicated slots block the run. Confirming a role-labelled slot saves the roles back onto the dataset (uncheck *Save these as the dataset's camera roles* to skip), so a corrected role wins over a misleading name next time and for every other pipeline. -3. **Job.** The confirmed order is what the job runs with (`cameraOrder` in the pipeline params; visible in the desktop job manifest). Runs started without the step (CLI) fall back to matching the header by role/name, and to registration-reference-first order for pipes with no header. +3. **Registration check.** DIVE reads which inputs the pipe warps onto camera 1 (`process warpN :: warp_detections` / `warp_image` in the pipe body) and, in the same dialog, shows for each such row whether the chosen camera has a fitted Aligned View registration onto camera 1. A missing one blocks the run with "Register X → Y in Aligned View first" — the pipe never gets to fail at configure time on a missing `registration_cameraN_to_camera1.json`. The same check runs server-side (and on desktop) before the job is created, so CLI runs get the same message. +4. **Job.** The confirmed order is what the job runs with (`cameraOrder` in the pipeline params; visible in the desktop job manifest). Runs started without the step (CLI) fall back to matching the header by role/name, and to registration-reference-first order for pipes with no header. ### Metadata File vs Configuration File diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index 22cd0ec2a..58d1a5948 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -20,7 +20,9 @@ from dive_tasks import tasks from dive_tasks.multicam_pipeline import ( build_registration_pairs, + describe_missing_registration, is_stereo_or_multicam_pipeline, + missing_registrations, pipeline_camera_order, pipeline_requires_input, resolve_pipeline_camera_order, @@ -448,6 +450,27 @@ def run_pipeline( if calibration_item_id: params['calibration_item_id'] = calibration_item_id if is_warp_pipeline and reference_camera: + # Refuse up front when a warped camera has no registration onto + # camera 1, rather than letting the pipe die at configure time. + fitted_pairs = [ + key + for key, value in ( + (folder.get('meta') or {}).get('cameraHomographies') or {} + ).items() + if value and (value.get('AtoB') or value.get('BtoA')) + ] + missing = missing_registrations( + camera_order, + (pipeline.get('metadata') or {}).get('registrationWarps'), + fitted_pairs, + ) + if missing: + raise RestException( + ' '.join( + describe_missing_registration(*entry, pipeline['name']) for entry in missing + ), + code=400, + ) registration_pairs = build_registration_pairs(folder.get('meta') or {}) if any( pair.get('leftToRight') or pair.get('rightToLeft') for pair in registration_pairs diff --git a/server/dive_tasks/multicam_pipeline.py b/server/dive_tasks/multicam_pipeline.py index 63d9c500e..01c0145d9 100644 --- a/server/dive_tasks/multicam_pipeline.py +++ b/server/dive_tasks/multicam_pipeline.py @@ -270,6 +270,41 @@ def build_registration_pairs(folder_meta: dict) -> List[dict]: return pairs +def missing_registrations( + order: List[str], registration_warps: Optional[List[int]], fitted_pairs: List[str] +) -> List[Tuple[int, str, str]]: + """ + (input, camera, camera1) for each warped input whose camera has no fitted + registration onto camera 1. `fitted_pairs` are the dataset's homography + keys (`a::b`, either orientation counts). Checked before a run so the + failure is "register camera X onto Y first" rather than the pipe dying at + configure time on a missing file. + """ + if not order or not registration_warps: + return [] + target = order[0] + fitted = set(fitted_pairs) + missing = [] + for position in registration_warps: + if position < 2 or position > len(order): + continue + camera = order[position - 1] + if camera == target: + continue + if f'{camera}::{target}' not in fitted and f'{target}::{camera}' not in fitted: + missing.append((position, camera, target)) + return missing + + +def describe_missing_registration( + position: int, camera: str, target: str, pipeline_name: str +) -> str: + return ( + f'Camera "{camera}" (input{position}) has no registration onto camera 1 ("{target}"). ' + f'Register {camera} -> {target} in Aligned View before running {pipeline_name}.' + ) + + def build_registration_kwiver_settings( work_dir: Path, cameras: List[MulticamCameraJob], diff --git a/server/dive_tasks/pipeline_discovery.py b/server/dive_tasks/pipeline_discovery.py index 3fcbb3f82..9975e73a4 100644 --- a/server/dive_tasks/pipeline_discovery.py +++ b/server/dive_tasks/pipeline_discovery.py @@ -181,6 +181,10 @@ def extract_pipe_metadata(file_path: Path) -> PipeMetadata: in_description = False full_description_parts: List[str] = [] + # `process warpN` followed by `:: warp_detections|warp_image` marks an + # input whose camera must be registered onto camera 1. + last_process_name: Optional[str] = None + registration_warps: List[int] = [] try: with open(file_path, 'r', encoding='utf-8') as f: @@ -190,6 +194,16 @@ def extract_pipe_metadata(file_path: Path) -> PipeMetadata: if not trimmed: continue + process_match = re.match(r'^process\s+(\S+)', trimmed) + if process_match: + last_process_name = process_match.group(1) + elif ( + re.match(r'^::\s*(warp_detections|warp_image)\b', trimmed) and last_process_name + ): + warp_match = re.match(r'^warp(\d+)$', last_process_name) + if warp_match: + registration_warps.append(int(warp_match.group(1))) + # --- Description extraction (Multiline) --- desc_start_match = re.match(r'^#\s*Description:\s*(.*)', line_raw, re.IGNORECASE) if desc_start_match: @@ -290,6 +304,9 @@ def extract_pipe_metadata(file_path: Path) -> PipeMetadata: if slots: metadata["cameraOrder"] = slots + if registration_warps: + metadata["registrationWarps"] = sorted(set(registration_warps)) + if full_description_parts: metadata["description"] = " ".join(full_description_parts) else: diff --git a/server/dive_utils/types.py b/server/dive_utils/types.py index a88685c6f..d82c9344e 100644 --- a/server/dive_utils/types.py +++ b/server/dive_utils/types.py @@ -97,6 +97,10 @@ class PipeMetadata(TypedDict): # camera by name at run time; when unset cameras are fed reference-first, then # display order. cameraOrder: NotRequired[Optional[list[str]]] + # Input positions the pipe warps onto camera 1 (`process warpN :: warp_detections | + # warp_image`), e.g. [2, 3]; each such camera needs a fitted registration onto + # camera 1, checked before the run. + registrationWarps: NotRequired[Optional[list[int]]] class PipelineDescription(TypedDict): diff --git a/server/tests/test_multicam_pipeline.py b/server/tests/test_multicam_pipeline.py index 8590ca6b8..66f3c1027 100644 --- a/server/tests/test_multicam_pipeline.py +++ b/server/tests/test_multicam_pipeline.py @@ -15,6 +15,7 @@ infer_camera_roles, is_stereo_measurement_pipeline, is_stereo_or_multicam_pipeline, + missing_registrations, pipeline_camera_order, pipeline_requires_input, resolve_pipeline_camera_order, @@ -171,6 +172,18 @@ def test_resolve_pipeline_camera_order_roles_win_over_names(): ) == ['thermal', 'other'] +def test_missing_registrations(): + order = ['rgb', 'uv', 'ir'] + fitted = ['rgb::ir'] # stored in the reverse orientation still counts + assert missing_registrations(order, [2, 3], fitted) == [(2, 'uv', 'rgb')] + assert missing_registrations(order, [2, 3], fitted + ['uv::rgb']) == [] + # No warps declared (or no order): nothing to check. + assert missing_registrations(order, None, []) == [] + assert missing_registrations([], [2], []) == [] + # Out-of-range warp positions are ignored rather than crashing. + assert missing_registrations(['rgb', 'ir'], [3], []) == [] + + def test_resolve_pipeline_camera_order(): assert resolve_pipeline_camera_order(['EO', 'UV', 'IR'], ['rgb', 'ir', 'uv']) == [ 'rgb', diff --git a/server/tests/test_pipeline_discovery.py b/server/tests/test_pipeline_discovery.py index 4fcd0aec0..a0546116c 100644 --- a/server/tests/test_pipeline_discovery.py +++ b/server/tests/test_pipeline_discovery.py @@ -135,6 +135,7 @@ def test_extract_pipe_metadata_camera_order(tmp_path: Path): metadata = extract_pipe_metadata(pipe) assert metadata['cameraOrder'] == ['EO', 'UV', 'IR'] + assert 'registrationWarps' not in metadata # The header must not bleed into the multi-line description. assert metadata['description'] == 'three camera detector' assert 'cameraOrder' not in extract_pipe_metadata( @@ -142,6 +143,27 @@ def test_extract_pipe_metadata_camera_order(tmp_path: Path): ) +def test_extract_pipe_metadata_registration_warps(tmp_path: Path): + pipe = _write( + tmp_path, + 'detector_seal_3-cam.pipe', + [ + '# Description: three camera detector', + 'process input1', + ' :: video_input', + 'process warp3', + ' :: warp_detections', + ' :transformation_file registration_camera3_to_camera1.json', + 'process warp2', + ' :: warp_detections', + ' :inverse true', + 'process not_a_warp', + ' :: warp_image', + ], + ) + assert extract_pipe_metadata(pipe)['registrationWarps'] == [2, 3] + + def _write(tmp_path: Path, name: str, lines: list) -> Path: pipe = tmp_path / name pipe.write_text('\n'.join(lines)) From 0b607e8fb36fa651f3927e4c6c9a0396eb4cdb18 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 17 Aug 2026 11:42:10 -0400 Subject: [PATCH 7/9] Point at the Camera Registration tab, not "Aligned View", in the registration-missing message Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MD6kx1uDv4NuiKdFNnYpG2 --- client/dive-common/apispec.ts | 2 +- client/dive-common/pipelineCameraOrder.ts | 2 +- client/platform/desktop/backend/native/viame.ts | 2 +- docs/Pipeline-Import-Export.md | 4 ++-- server/dive_tasks/multicam_pipeline.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index 874955578..fdaad98de 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -90,7 +90,7 @@ interface PipeMetadata { /** * Input positions whose detections/images the pipe warps onto camera 1 * (`process warpN :: warp_detections | warp_image` in the pipe body), e.g. - * [2, 3]. Each such camera needs a fitted Aligned View registration onto + * [2, 3]. Each such camera needs a fitted registration (Camera Registration tab) onto * camera 1; DIVE checks that before the run instead of letting the pipe * fail at configure time on a missing file. */ diff --git a/client/dive-common/pipelineCameraOrder.ts b/client/dive-common/pipelineCameraOrder.ts index 112bf4180..1da0e09f4 100644 --- a/client/dive-common/pipelineCameraOrder.ts +++ b/client/dive-common/pipelineCameraOrder.ts @@ -233,7 +233,7 @@ export function missingRegistrations( export function describeMissingRegistration(missing: MissingRegistration, pipelineName?: string): string { const where = pipelineName ? ` before running ${pipelineName}` : ''; return `Camera "${missing.camera}" (input${missing.input}) has no registration onto camera 1 ` - + `("${missing.target}"). Register ${missing.camera} → ${missing.target} in Aligned View${where}.`; + + `("${missing.target}"). Register ${missing.camera} → ${missing.target} in the Camera Registration tab${where}.`; } /** Parse the value of a `# Camera Order:` header into slot tokens. */ diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 840d692a8..4e1050ff5 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -424,7 +424,7 @@ async function runPipeline( }); } if (multiCamOrder) { - // Hand the camera registration (Aligned View homographies) to the + // Hand the camera registration (Camera Registration homographies) to the // pipeline's per-camera warp processes. Refuse up front when a warped // camera has no registration onto camera 1, rather than letting the // pipe die at configure time on a missing file. diff --git a/docs/Pipeline-Import-Export.md b/docs/Pipeline-Import-Export.md index e925c9d40..b5f1b597c 100644 --- a/docs/Pipeline-Import-Export.md +++ b/docs/Pipeline-Import-Export.md @@ -75,7 +75,7 @@ Example: | `# Calibration Keys: [k…]` | Opt-in: binds the dataset's stereo calibration file to each listed KWIVER config key at run time (one `-s =` per key). Keys may be space- or comma-separated. Use this when the pipe's calibration consumer is not the conventional `measurer:calibration_file` / `calibration_reader:file` pair (those are used when the header is unset). Needed because `$CONFIG{global:…}` indirection cannot receive `-s` overrides (macros expand at parse time; `-s` blocks are appended last). | | `# Metadata File: :` | Opt-in: when the dataset has an attached **Metadata File**, DIVE appends a KWIVER override `-s :=` at run time. The same CSV/TXT attachment is also considered for [Frame Metadata](Frame-Metadata.md). Without this header, no metadata file is injected. | | `# Image List Keys: [k…]` | Opt-in: binds the run's per-camera input image list(s) to each listed KWIVER key. Keys may be space- or comma-separated. A key containing `{cam}` is expanded once per camera (1-based), e.g. `stabilizer:image_list{cam}` → `image_list1`, `image_list2`, …. A key without `{cam}` receives camera 1's list only. | -| `# Camera Order: [cam…]` | 2-cam/3-cam pipes only: names the camera role fed to each `inputN`, in order (e.g. `# Camera Order: EO, UV, IR` → `input1` optical, `input2` ultraviolet, `input3` thermal). Role tokens are `EO`, `IR`, `UV` (aliases: eo/rgb/optical/color/vis, ir/thermal/lwir/flir, uv/ultraviolet); any other token (`left`, `right`) matches a camera name literally. See [Multicam camera assignment](#multicam-camera-assignment) for how DIVE places a dataset's cameras onto these slots. Camera 1 is the frame the pipe's warp processes map onto: each other camera's Aligned View registration onto camera 1 is written to the job as `_to__registration.json` and bound to `warpN`. | +| `# Camera Order: [cam…]` | 2-cam/3-cam pipes only: names the camera role fed to each `inputN`, in order (e.g. `# Camera Order: EO, UV, IR` → `input1` optical, `input2` ultraviolet, `input3` thermal). Role tokens are `EO`, `IR`, `UV` (aliases: eo/rgb/optical/color/vis, ir/thermal/lwir/flir, uv/ultraviolet); any other token (`left`, `right`) matches a camera name literally. See [Multicam camera assignment](#multicam-camera-assignment) for how DIVE places a dataset's cameras onto these slots. Camera 1 is the frame the pipe's warp processes map onto: each other camera's registration (Camera Registration tab) onto camera 1 is written to the job as `_to__registration.json` and bound to `warpN`. | ### Multicam camera assignment @@ -83,7 +83,7 @@ Which dataset camera feeds which input of a 2-cam/3-cam pipe is decided in the o 1. **Camera roles.** Each camera of a multicam dataset carries a sensor role (`eo`, `ir`, `uv`) in `cameraRoles`, inferred once at import from the camera (subfolder) name and, failing that, from tokens in its image file names (KAMERA style `…_rgb.jpg`, `…_ir.tif`, `…_uv.jpg`). Only a unanimous answer is recorded; ambiguous cameras get no role. 2. **Assignment step.** When you run a 2-cam/3-cam pipe, DIVE shows one row per pipeline camera (from the `# Camera Order:` header, or plain `input1..N` when the pipe has none) with the dataset camera it proposes — matched by role when both sides have one, else by name — and you confirm or change it before anything runs. Unfilled or duplicated slots block the run. Confirming a role-labelled slot saves the roles back onto the dataset (uncheck *Save these as the dataset's camera roles* to skip), so a corrected role wins over a misleading name next time and for every other pipeline. -3. **Registration check.** DIVE reads which inputs the pipe warps onto camera 1 (`process warpN :: warp_detections` / `warp_image` in the pipe body) and, in the same dialog, shows for each such row whether the chosen camera has a fitted Aligned View registration onto camera 1. A missing one blocks the run with "Register X → Y in Aligned View first" — the pipe never gets to fail at configure time on a missing `registration_cameraN_to_camera1.json`. The same check runs server-side (and on desktop) before the job is created, so CLI runs get the same message. +3. **Registration check.** DIVE reads which inputs the pipe warps onto camera 1 (`process warpN :: warp_detections` / `warp_image` in the pipe body) and, in the same dialog, shows for each such row whether the chosen camera has a fitted registration onto camera 1. A missing one blocks the run with "Register X → Y in the Camera Registration tab first" — the pipe never gets to fail at configure time on a missing `registration_cameraN_to_camera1.json`. The same check runs server-side (and on desktop) before the job is created, so CLI runs get the same message. 4. **Job.** The confirmed order is what the job runs with (`cameraOrder` in the pipeline params; visible in the desktop job manifest). Runs started without the step (CLI) fall back to matching the header by role/name, and to registration-reference-first order for pipes with no header. ### Metadata File vs Configuration File diff --git a/server/dive_tasks/multicam_pipeline.py b/server/dive_tasks/multicam_pipeline.py index 01c0145d9..9da1bba84 100644 --- a/server/dive_tasks/multicam_pipeline.py +++ b/server/dive_tasks/multicam_pipeline.py @@ -301,7 +301,7 @@ def describe_missing_registration( ) -> str: return ( f'Camera "{camera}" (input{position}) has no registration onto camera 1 ("{target}"). ' - f'Register {camera} -> {target} in Aligned View before running {pipeline_name}.' + f'Register {camera} -> {target} in the Camera Registration tab before running {pipeline_name}.' ) From 541a600d0762ed5b63904ab27dac27a560365356 Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 17 Aug 2026 11:42:22 -0400 Subject: [PATCH 8/9] Wrap the registration-missing message under the line limit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MD6kx1uDv4NuiKdFNnYpG2 --- server/dive_tasks/multicam_pipeline.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/dive_tasks/multicam_pipeline.py b/server/dive_tasks/multicam_pipeline.py index 9da1bba84..634e9f81b 100644 --- a/server/dive_tasks/multicam_pipeline.py +++ b/server/dive_tasks/multicam_pipeline.py @@ -301,7 +301,8 @@ def describe_missing_registration( ) -> str: return ( f'Camera "{camera}" (input{position}) has no registration onto camera 1 ("{target}"). ' - f'Register {camera} -> {target} in the Camera Registration tab before running {pipeline_name}.' + f'Register {camera} -> {target} in the Camera Registration tab ' + f'before running {pipeline_name}.' ) From 58e195923d78cd340367a8bd14f825ee3c49a24e Mon Sep 17 00:00:00 2001 From: romleiaj Date: Mon, 17 Aug 2026 13:27:32 -0400 Subject: [PATCH 9/9] Simplify the multicam camera-order plumbing The design settled on the camera-assignment step always supplying the input1..N order, which left the earlier automatic header resolution as a dead path with a lot of code behind it. Remove it and flatten what is left: - Drop resolvePipelineCameraOrder / candidatesForSlot / camerasMatchingSlot / pipelineCameraNames / pipelineOrderedCameraNames on the client and resolve_pipeline_camera_order / candidates_for_slot / cameras_matching_slot / pipeline_camera_order on the server. Without a confirmed cameraOrder (API callers) both backends now use the dataset's stored camera order, exactly as main does. One matcher, camerasForSlot, remains for the dialog prefill. - buildRegistrationPipelineArgs takes the pipe's warp positions and throws the register-first error itself, instead of viame.ts reverse-engineering the fitted set from the -s keys it had just built; its three guards for "this camera has a fitted pair onto camera 1" become one. - Web import infers camera roles from names only (no per-folder item scan); the server's infer_camera_role drops its image-name fallback accordingly. Desktop keeps the image-name fallback since it has the list in hand. No change on the dialog path. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MD6kx1uDv4NuiKdFNnYpG2 --- client/dive-common/apispec.ts | 11 +- client/dive-common/multicamDisplay.ts | 37 ----- .../dive-common/pipelineCameraOrder.spec.ts | 47 ++----- client/dive-common/pipelineCameraOrder.ts | 79 +++-------- .../backend/native/cameraRegistration.spec.ts | 39 +++--- .../backend/native/cameraRegistration.ts | 72 ++++------ .../desktop/backend/native/multiCamUtils.ts | 5 +- .../platform/desktop/backend/native/viame.ts | 56 ++++---- docs/Pipeline-Import-Export.md | 2 +- server/dive_server/crud_dataset.py | 9 +- server/dive_server/crud_rpc.py | 25 +--- server/dive_tasks/multicam_pipeline.py | 126 ++---------------- server/dive_utils/types.py | 9 +- server/tests/test_multicam_pipeline.py | 59 +------- 14 files changed, 128 insertions(+), 448 deletions(-) diff --git a/client/dive-common/apispec.ts b/client/dive-common/apispec.ts index fdaad98de..2b6fd3bd6 100644 --- a/client/dive-common/apispec.ts +++ b/client/dive-common/apispec.ts @@ -82,9 +82,9 @@ interface PipeMetadata { /** * Camera role per pipeline input for 2-cam/3-cam pipes (e.g. ["EO", "UV", "IR"]: * input1 is optical, input2 ultraviolet, input3 thermal), parsed from a - * `# Camera Order: [cam...]` header. At run time each slot is matched to - * a dataset camera by name (dive-common/pipelineCameraOrder.ts); when unset, - * cameras are fed registration-reference first, then display order. + * `# Camera Order: [cam...]` header. Labels the slots of the pre-run + * camera-assignment step and drives its prefill (dive-common/pipelineCameraOrder.ts); + * pipes without it show bare input1..N slots. */ cameraOrder?: string[]; /** @@ -106,9 +106,8 @@ interface PipelineParams { runtimeParams?: PipelineRuntimeParams; /** * 2-cam/3-cam pipes: the dataset camera to feed each inputN, in order, as - * confirmed by the user before the run. When omitted the backend places - * cameras from the pipe's `# Camera Order:` header (or, without one, - * registration reference first then display order). + * confirmed by the user before the run. When omitted (API callers) the + * dataset's stored camera order is used. */ cameraOrder?: string[]; /** Filter / transcode / disparity pipelines: name for the newly created dataset. */ diff --git a/client/dive-common/multicamDisplay.ts b/client/dive-common/multicamDisplay.ts index 5e735dd5a..e51ea0819 100644 --- a/client/dive-common/multicamDisplay.ts +++ b/client/dive-common/multicamDisplay.ts @@ -1,6 +1,5 @@ import type { SubType } from 'dive-common/apispec'; import { preferEoIrSubfolderOrder } from 'dive-common/components/ImportMultiCamDialog/multicamSubfolderLayout'; -import { CameraRole, resolvePipelineCameraOrder } from 'dive-common/pipelineCameraOrder'; export type MultiCamSubType = 'stereo' | 'multicam'; @@ -58,42 +57,6 @@ export function referenceCameraName(multiCamMedia: MultiCamMediaLike | null | un return defaultDisplay && ordered.includes(defaultDisplay) ? defaultDisplay : ordered[0]; } -/** - * Camera order for 2-cam/3-cam VIAME pipelines: the registration reference - * camera feeds input1 (the per-camera registrations all map onto the - * reference, and the pipes warp everything onto camera 1's frame), remaining - * cameras keep display order. Which detector a pipe runs on which input is - * the pipe's documented contract, not something DIVE infers. - */ -export function pipelineOrderedCameraNames(multiCamMedia: MultiCamMediaLike | null | undefined): string[] { - const ordered = orderedMultiCamCameraNames(multiCamMedia); - const reference = referenceCameraName(multiCamMedia); - return reference ? [reference, ...ordered.filter((name) => name !== reference)] : ordered; -} - -/** - * The cameras to feed input1..N of a 2-cam/3-cam pipeline. A pipe that - * declares its slots (`# Camera Order:` header, parsed into - * metadata.cameraOrder) gets each slot matched to a dataset camera by name and - * throws when that is not unambiguous; a pipe without one gets - * {@link pipelineOrderedCameraNames}. Camera 1 is the frame the others' - * registrations must map onto. - */ -export function pipelineCameraNames( - multiCamMedia: MultiCamMediaLike | null | undefined, - declaredOrder?: string[] | null, - roles: Record = {}, -): string[] { - if (declaredOrder?.length) { - const result = resolvePipelineCameraOrder(declaredOrder, orderedMultiCamCameraNames(multiCamMedia), roles); - if (result.error !== undefined) { - throw new Error(result.error); - } - return result.order; - } - return pipelineOrderedCameraNames(multiCamMedia); -} - export function isMultiCamSubType(subType: SubType | string | null | undefined): subType is MultiCamSubType { return subType === 'stereo' || subType === 'multicam'; } diff --git a/client/dive-common/pipelineCameraOrder.spec.ts b/client/dive-common/pipelineCameraOrder.spec.ts index e52dd9a7e..b24dd5c51 100644 --- a/client/dive-common/pipelineCameraOrder.spec.ts +++ b/client/dive-common/pipelineCameraOrder.spec.ts @@ -1,10 +1,8 @@ import { describe, expect, it } from 'vitest'; import { - camerasMatchingSlot, inferCameraRole, inferCameraRoles, missingRegistrations, + camerasForSlot, inferCameraRole, inferCameraRoles, missingRegistrations, parseCameraOrderHeader, pipelineCameraSlots, prefillPipelineCameraOrder, - resolvePipelineCameraOrder, } from './pipelineCameraOrder'; -import { pipelineCameraNames } from './multicamDisplay'; describe('pipelineCameraOrder', () => { it('parses the header value into slot tokens', () => { @@ -15,30 +13,14 @@ describe('pipelineCameraOrder', () => { it('matches slots by exact name, name segment, or role alias', () => { const cameras = ['rgb', 'CENT_IR', 'uv_cam']; - expect(camerasMatchingSlot('EO', cameras)).toStrictEqual(['rgb']); - expect(camerasMatchingSlot('IR', cameras)).toStrictEqual(['CENT_IR']); - expect(camerasMatchingSlot('ultraviolet', cameras)).toStrictEqual(['uv_cam']); - expect(camerasMatchingSlot('rgb', cameras)).toStrictEqual(['rgb']); + expect(camerasForSlot('EO', cameras)).toStrictEqual(['rgb']); + expect(camerasForSlot('IR', cameras)).toStrictEqual(['CENT_IR']); + expect(camerasForSlot('ultraviolet', cameras)).toStrictEqual(['uv_cam']); + expect(camerasForSlot('rgb', cameras)).toStrictEqual(['rgb']); // Exact name wins over role matching elsewhere. - expect(camerasMatchingSlot('ir', ['ir', 'thermal'])).toStrictEqual(['ir']); + expect(camerasForSlot('ir', ['ir', 'thermal'])).toStrictEqual(['ir']); // Literal segments for non-role tokens. - expect(camerasMatchingSlot('left', ['left_cam', 'right_cam'])).toStrictEqual(['left_cam']); - }); - - it('resolves declared slots to a full, unique camera order', () => { - expect(resolvePipelineCameraOrder(['EO', 'UV', 'IR'], ['rgb', 'ir', 'uv'])) - .toStrictEqual({ order: ['rgb', 'uv', 'ir'] }); - expect(resolvePipelineCameraOrder(['EO', 'IR'], ['CENT_IR', 'CENT_EO'])) - .toStrictEqual({ order: ['CENT_EO', 'CENT_IR'] }); - }); - - it('reports count mismatch, unmatched and ambiguous slots', () => { - expect(resolvePipelineCameraOrder(['EO', 'IR'], ['rgb', 'ir', 'uv']).error) - .toMatch(/expects 2 cameras but the dataset has 3/); - expect(resolvePipelineCameraOrder(['EO', 'UV', 'IR'], ['rgb', 'ir', 'cam3']).error) - .toMatch(/"UV" \(input2\): no dataset camera matches/); - expect(resolvePipelineCameraOrder(['EO', 'IR'], ['rgb', 'color']).error) - .toMatch(/"EO" \(input1\): several dataset cameras match \(rgb, color\)/); + expect(camerasForSlot('left', ['left_cam', 'right_cam'])).toStrictEqual(['left_cam']); }); it('infers roles from camera names, then image names, only when unanimous', () => { @@ -55,8 +37,8 @@ describe('pipelineCameraOrder', () => { it('assigned roles beat name matching, and prefill leaves ambiguity open', () => { // "thermal" is named like IR but the user marked it optical. - expect(resolvePipelineCameraOrder(['EO', 'IR'], ['thermal', 'other'], { thermal: 'eo', other: 'ir' })) - .toStrictEqual({ order: ['thermal', 'other'] }); + expect(prefillPipelineCameraOrder(['EO', 'IR'], ['thermal', 'other'], { thermal: 'eo', other: 'ir' })) + .toStrictEqual(['thermal', 'other']); expect(prefillPipelineCameraOrder(['EO', 'UV', 'IR'], ['rgb', 'ir', 'uv'])) .toStrictEqual(['rgb', 'uv', 'ir']); // Two EO-ish names, no roles: the EO slot stays open, IR still fills. @@ -80,15 +62,4 @@ describe('pipelineCameraOrder', () => { // An unfilled slot is reported by the assignment problems, not here. expect(missingRegistrations(['rgb', null], [2], [])).toStrictEqual([]); }); - - it('pipelineCameraNames uses the declared order or falls back to reference-first', () => { - const media = { - cameras: { ir: {}, rgb: {}, uv: {} }, - cameraOrder: ['rgb', 'ir', 'uv'], - defaultDisplay: 'rgb', - }; - expect(pipelineCameraNames(media, ['EO', 'UV', 'IR'])).toStrictEqual(['rgb', 'uv', 'ir']); - expect(pipelineCameraNames(media)).toStrictEqual(['rgb', 'ir', 'uv']); - expect(() => pipelineCameraNames(media, ['EO', 'IR'])).toThrow(/expects 2 cameras/); - }); }); diff --git a/client/dive-common/pipelineCameraOrder.ts b/client/dive-common/pipelineCameraOrder.ts index 1da0e09f4..0cf9c6794 100644 --- a/client/dive-common/pipelineCameraOrder.ts +++ b/client/dive-common/pipelineCameraOrder.ts @@ -14,7 +14,7 @@ * when both sides have one, else by name -- and confirms or corrects it; the * confirmed order is what the job runs with. * - * Kept in sync with server/dive_tasks/multicam_pipeline.py. + * Role inference is mirrored in server/dive_tasks/multicam_pipeline.py. */ /** The sensor modalities DIVE knows how to match. */ @@ -85,64 +85,14 @@ export function inferCameraRoles( return roles; } -/** Cameras whose name matches a slot token, by literal segment or shared role. */ -export function camerasMatchingSlot(token: string, cameras: string[]): string[] { - const lower = token.toLowerCase(); - const exact = cameras.filter((camera) => camera.toLowerCase() === lower); - if (exact.length) { - return exact; - } - const role = roleOfToken(token); - const aliases = new Set(role ? CAMERA_ROLE_ALIASES[role] : [lower]); - return cameras.filter((camera) => segments(camera).some((seg) => aliases.has(seg))); -} - -export type PipelineCameraOrderResult = - | { order: string[]; error?: undefined } - | { order?: undefined; error: string }; - -/** - * Resolve declared slots to dataset cameras without user interaction (CLI, - * and the fallback when no confirmed order was supplied). Every slot must - * match exactly one camera and no camera may fill two slots; anything else - * is an error message naming the slot, the pipe's slots and the dataset's - * cameras. - */ -export function resolvePipelineCameraOrder( - slots: string[], - cameras: string[], - roles: Record = {}, -): PipelineCameraOrderResult { - const context = `pipeline cameras [${slots.join(', ')}], dataset cameras [${cameras.join(', ')}]`; - if (slots.length !== cameras.length) { - return { - error: `Pipeline expects ${slots.length} cameras but the dataset has ${cameras.length}: ${context}`, - }; - } - const order: string[] = []; - for (let i = 0; i < slots.length; i += 1) { - const slot = slots[i]; - const matches = candidatesForSlot(slot, cameras, roles).filter((c) => !order.includes(c)); - if (matches.length !== 1) { - const why = matches.length === 0 - ? 'no dataset camera matches' - : `several dataset cameras match (${matches.join(', ')})`; - return { - error: `Cannot place pipeline camera "${slot}" (input${i + 1}): ${why}. ${context}. ` - + 'Set the camera roles (or rename the cameras) so each pipeline camera matches exactly one.', - }; - } - order.push(matches[0]); - } - return { order }; -} - /** - * Cameras that could fill a slot: cameras whose assigned role equals the - * slot's role take precedence over name matching, so a corrected role wins - * over a misleading name. + * Cameras that could fill a slot. A camera whose assigned role equals the + * slot's role wins over name matching, so a corrected role beats a + * misleading name; otherwise the slot token must equal the camera name, or + * share a role (or, for non-role tokens like `left`, appear literally) with + * one of the name's segments. */ -export function candidatesForSlot( +export function camerasForSlot( slot: string, cameras: string[], roles: Record = {}, @@ -154,13 +104,18 @@ export function candidatesForSlot( return byRole; } } - return camerasMatchingSlot(slot, cameras); + const lower = slot.toLowerCase(); + const exact = cameras.filter((camera) => camera.toLowerCase() === lower); + if (exact.length) { + return exact; + } + const aliases = new Set(role ? CAMERA_ROLE_ALIASES[role] : [lower]); + return cameras.filter((camera) => segments(camera).some((seg) => aliases.has(seg))); } /** - * Propose a camera for every slot for the confirmation step. Unlike - * {@link resolvePipelineCameraOrder} this never fails: a slot with no unique - * candidate is proposed as null and left for the user to fill. + * Propose a camera for every slot for the confirmation step. Never fails: a + * slot with no unique candidate is proposed as null and left for the user. */ export function prefillPipelineCameraOrder( slots: string[], @@ -172,7 +127,7 @@ export function prefillPipelineCameraOrder( // Unique matches first, so an ambiguous slot cannot steal a camera another // slot needs unambiguously. slots.forEach((slot, index) => { - const matches = candidatesForSlot(slot, cameras, roles).filter((c) => !taken.has(c)); + const matches = camerasForSlot(slot, cameras, roles).filter((c) => !taken.has(c)); if (matches.length === 1) { [proposed[index]] = matches; taken.add(matches[0]); diff --git a/client/platform/desktop/backend/native/cameraRegistration.spec.ts b/client/platform/desktop/backend/native/cameraRegistration.spec.ts index 57ef69d86..3133f5a2c 100644 --- a/client/platform/desktop/backend/native/cameraRegistration.spec.ts +++ b/client/platform/desktop/backend/native/cameraRegistration.spec.ts @@ -6,7 +6,6 @@ import { } from 'vitest'; import { Settings, JsonConfig } from 'platform/desktop/constants'; -import { pipelineOrderedCameraNames } from 'dive-common/multicamDisplay'; import { buildRegistrationPipelineArgs } from './cameraRegistration'; // mock-fs no longer intercepts fs-extra's exists checks on newer Node; @@ -123,13 +122,12 @@ function multiCamMeta(id: string, cameras: string[], defaultDisplay: string): Js describe('buildRegistrationPipelineArgs', () => { it('writes one file per camera pair and pins each warp pair/direction', async () => { - // Display order intentionally scrambles the input: the pipeline order is - // reference-first then display order (rgb, uv, ir -- IR displays last), - // so uv lands on input2 / warp2 and ir on input3 / warp3. + // Display order is irrelevant: the given pipeline order (rgb, uv, ir) + // puts uv on input2 / warp2 and ir on input3 / warp3. const meta = multiCamMeta('withreg', ['ir', 'rgb', 'uv'], 'rgb'); const jobWorkDir = '/home/user/job/full'; await fs.ensureDir(jobWorkDir); - const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, pipelineOrderedCameraNames(meta.multiCam)); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, ['rgb', 'uv', 'ir']); const irPath = npath.join(jobWorkDir, 'ir_to_rgb_registration.json'); const uvPath = npath.join(jobWorkDir, 'uv_to_rgb_registration.json'); @@ -156,9 +154,10 @@ describe('buildRegistrationPipelineArgs', () => { const meta = multiCamMeta('partial', ['rgb', 'ir', 'uv'], 'rgb'); const jobWorkDir = '/home/user/job/partial'; await fs.ensureDir(jobWorkDir); - const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, pipelineOrderedCameraNames(meta.multiCam)); - // Pipeline order is rgb, uv, ir: uv (warp2) has only the unsupported - // uv-to-ir pair, so it gets nothing; ir (warp3) has a reference pair. + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, ['rgb', 'uv', 'ir']); + // uv (warp2) has only the unsupported uv-to-ir pair, so it gets nothing + // (no warps declared, so that is not an error); ir (warp3) has a + // reference pair. expect(Object.keys(args).some((key) => key.startsWith('warp2'))).toBe(false); expect(args['warp3:transform_reader:dive:from_camera']).toBe('ir'); const irPath = npath.join(jobWorkDir, 'ir_to_rgb_registration.json'); @@ -170,26 +169,22 @@ describe('buildRegistrationPipelineArgs', () => { expect(await fs.readdir(jobWorkDir)).toStrictEqual(['ir_to_rgb_registration.json']); }); - it('follows an explicit pipeline camera order, camera 1 being the warp target', async () => { - // Pipe declares EO, UV, IR; dataset display order is ir, rgb, uv. The - // resolved order is rgb, uv, ir so uv is warp2 and ir is warp3, and the - // files register onto rgb (camera 1) regardless of the reference camera. - const meta = multiCamMeta('withreg', ['ir', 'rgb', 'uv'], 'ir'); - const jobWorkDir = '/home/user/job/declared'; + it('refuses a warped input whose camera has no registration onto camera 1', async () => { + const meta = multiCamMeta('partial', ['rgb', 'ir', 'uv'], 'rgb'); + const jobWorkDir = '/home/user/job/partial-warps'; await fs.ensureDir(jobWorkDir); - const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, ['rgb', 'uv', 'ir']); - expect(args['warp2:transform_reader:dive:from_camera']).toBe('uv'); - expect(args['warp2:transform_reader:dive:to_camera']).toBe('rgb'); - expect(args['warp3:transform_reader:dive:from_camera']).toBe('ir'); - expect(args['warp3:transform_reader:dive:to_camera']).toBe('rgb'); - expect(await fs.readdir(jobWorkDir)).toStrictEqual(['ir_to_rgb_registration.json', 'uv_to_rgb_registration.json']); + // warp3 (ir) is fitted; warp2 (uv) is not. + await expect(buildRegistrationPipelineArgs(settings, meta, jobWorkDir, ['rgb', 'uv', 'ir'], [3], 'pipe')) + .resolves.toHaveProperty('warp3:transformation_file'); + await expect(buildRegistrationPipelineArgs(settings, meta, jobWorkDir, ['rgb', 'uv', 'ir'], [2, 3], 'pipe')) + .rejects.toThrow(/Camera "uv" \(input2\) has no registration onto camera 1 \("rgb"\)/); }); it('returns no args when the dataset has no registration', async () => { const meta = multiCamMeta('noreg', ['rgb', 'ir'], 'rgb'); const jobWorkDir = '/home/user/job/noreg'; await fs.ensureDir(jobWorkDir); - const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, pipelineOrderedCameraNames(meta.multiCam)); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, ['rgb', 'ir']); expect(args).toStrictEqual({}); expect(await fs.readdir(jobWorkDir)).toStrictEqual([]); }); @@ -201,7 +196,7 @@ describe('buildRegistrationPipelineArgs', () => { }; const jobWorkDir = '/home/user/job/seeded'; await fs.ensureDir(jobWorkDir); - const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, pipelineOrderedCameraNames(meta.multiCam)); + const args = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, ['rgb', 'ir']); expect(args['warp2:transform_reader:dive:from_camera']).toBe('ir'); expect(args['warp2:transform_reader:dive:to_camera']).toBe('rgb'); }); diff --git a/client/platform/desktop/backend/native/cameraRegistration.ts b/client/platform/desktop/backend/native/cameraRegistration.ts index 849146921..e89381751 100644 --- a/client/platform/desktop/backend/native/cameraRegistration.ts +++ b/client/platform/desktop/backend/native/cameraRegistration.ts @@ -19,6 +19,7 @@ import { readTransformMatrix } from 'vue-media-annotator/alignedView/alignedView import { invert3, Matrix3 } from 'vue-media-annotator/alignedView/homography'; import { DatasetConfigMutable } from 'dive-common/apispec'; import { referenceCameraName as multicamReferenceCameraName } from 'dive-common/multicamDisplay'; +import { describeMissingRegistration } from 'dive-common/pipelineCameraOrder'; import { RegistrationFileNamePattern, compareRegistrationCandidates, @@ -194,67 +195,52 @@ export async function loadEffectiveRegistration( /** * Build the kwiver -s settings that hand a dataset's camera registration to - * a 2-cam/3-cam pipeline. `cameraOrder` is the pipeline's input1..N camera - * order (see pipelineCameraNames); camera 1 is the frame the pipe warps - * everything onto, so one standard _to__registration file - * per other camera is written into the job work dir and each camera's warp - * process (warp2, warp3, ...) receives its own single-pair file. The pair - * and direction are still pinned through the reader's from_camera/to_camera - * config, since a pair may be stored in either orientation. - * - * Only pairs registering a camera directly onto camera 1 are supported: - * pairs between two other cameras are explicitly unsupported here (there is - * no transform composition) and never reach the pipeline. Cameras without a - * fitted pair onto camera 1 get no settings; a pipeline that needs one then - * fails at configure time with the file name it was missing. + * a 2-cam/3-cam pipeline. `cameraOrder` is the pipeline's input1..N cameras; + * camera 1 is the frame the pipe warps everything onto, so each other camera + * with a fitted pair onto camera 1 gets a single-pair + * _to__registration.json in the job work dir bound to its + * warpN, with the direction pinned through the reader's from/to config since + * a pair may be stored in either orientation. Pairs between two other + * cameras are unsupported (no transform composition) and never reach the + * job. A warped input (`registrationWarps`) whose camera has no fitted pair + * onto camera 1 is an error before the run, not a missing file at pipe + * configure time. */ export async function buildRegistrationPipelineArgs( settings: Settings, meta: JsonConfig, jobWorkDir: string, cameraOrder: string[], + registrationWarps: number[] = [], + pipelineName = '', ): Promise> { const args: Record = {}; - if (!meta.multiCam) { + const [reference] = cameraOrder; + if (!meta.multiCam || !reference) { return args; } const projectDirInfo = await getValidatedProjectDir(settings, meta.id); const values = await loadEffectiveRegistration(projectDirInfo.basePath, meta); - const [reference] = cameraOrder; - if (!reference) { - return args; - } const files = buildPerCameraRegistrationFiles(values, reference); const writes: Promise[] = []; - cameraOrder.forEach((camera, index) => { - if (index === 0 || camera === reference) { - return; - } - // Points-only pairs have no matrix the warp could apply. - const fitted = values.homographies[`${camera}::${reference}`] - || values.homographies[`${reference}::${camera}`]; - if (!fitted) { - return; - } + cameraOrder.slice(1).forEach((camera, offset) => { + const input = offset + 2; const file = files.find((candidate) => candidate.camera === camera); - if (!file) { - return; - } - // Unsupported non-reference pairs are dropped from the file body too, so - // the job dir only ever holds camera-to-reference registrations. - const referencePairs = file.body.pairs.filter( - (pair) => pair.left === reference || pair.right === reference, - ); - if (!referencePairs.length) { + const pair = file?.body.pairs.find((candidate) => ( + (candidate.left === reference || candidate.right === reference) + && (candidate.leftToRight || candidate.rightToLeft))); + if (!file || !pair) { + if (registrationWarps.includes(input)) { + throw new Error(describeMissingRegistration({ input, camera, target: reference }, pipelineName)); + } return; } const registrationPath = npath.join(jobWorkDir, registrationFileName(camera, reference)); - writes.push(writeJsonFile(registrationPath, { ...file.body, pairs: referencePairs })); - const warp = `warp${index + 1}`; - args[`${warp}:transformation_file`] = registrationPath; - args[`${warp}:transform_reader:type`] = 'dive'; - args[`${warp}:transform_reader:dive:from_camera`] = camera; - args[`${warp}:transform_reader:dive:to_camera`] = reference; + writes.push(writeJsonFile(registrationPath, { ...file.body, pairs: [pair] })); + args[`warp${input}:transformation_file`] = registrationPath; + args[`warp${input}:transform_reader:type`] = 'dive'; + args[`warp${input}:transform_reader:dive:from_camera`] = camera; + args[`warp${input}:transform_reader:dive:to_camera`] = reference; }); await Promise.all(writes); return args; diff --git a/client/platform/desktop/backend/native/multiCamUtils.ts b/client/platform/desktop/backend/native/multiCamUtils.ts index 5c2ca3c19..fcf0ceb6a 100644 --- a/client/platform/desktop/backend/native/multiCamUtils.ts +++ b/client/platform/desktop/backend/native/multiCamUtils.ts @@ -79,9 +79,8 @@ async function writeMultiCamStereoPipelineArgs( settings: Settings, utility = false, forceTranscoded = false, - // Explicit input1..N camera order for 2-cam/3-cam pipes (see - // pipelineCameraNames); stereo measurement keeps the stored left/right - // order when omitted. + // Explicit input1..N camera order for 2-cam/3-cam pipes; stereo + // measurement keeps the stored left/right order when omitted. cameraOrder: string[] | undefined = undefined, ) { const argFilePair: Record = {}; diff --git a/client/platform/desktop/backend/native/viame.ts b/client/platform/desktop/backend/native/viame.ts index 4e1050ff5..faf2524f7 100644 --- a/client/platform/desktop/backend/native/viame.ts +++ b/client/platform/desktop/backend/native/viame.ts @@ -15,7 +15,6 @@ import { observeChild } from 'platform/desktop/backend/native/processManager'; import { convertMedia } from 'platform/desktop/backend/native/mediaJobs'; import sendToRenderer from 'platform/desktop/background'; -import type { Pipe } from 'dive-common/apispec'; import { MultiType, stereoPipelineMarker, @@ -28,8 +27,6 @@ import { isTranscodePipeline, pipelineCreatesNewDataset, } from 'dive-common/pipelineCreatesDataset'; -import { pipelineCameraNames } from 'dive-common/multicamDisplay'; -import { describeMissingRegistration, missingRegistrations } from 'dive-common/pipelineCameraOrder'; import * as common from './common'; import { jobFileEchoMiddleware, createWorkingDirectory, createCustomWorkingDirectory, splitExt, @@ -167,21 +164,19 @@ async function importNewMedia( * a node.js implementation of dive_tasks.tasks.run_pipeline */ /** - * The input1..N camera order for a 2-cam/3-cam run: the confirmed order when - * one was supplied (validated against the dataset's cameras), else resolved - * from the pipe's declared slots and the dataset's camera roles/names. + * The input1..N camera order for a 2-cam/3-cam run: the order the user + * confirmed in the camera-assignment step (validated against the dataset's + * cameras), else the dataset's camera order as stored. */ -function resolveMultiCamOrder(meta: JsonConfig, pipeline: Pipe, confirmed?: string[]): string[] { +function multiCamOrderFor(meta: JsonConfig, confirmed?: string[]): string[] { const cameras = Object.keys(meta.multiCam?.cameras ?? {}); - if (confirmed?.length) { - const unknown = confirmed.filter((name) => !cameras.includes(name)); - const missing = cameras.filter((name) => !confirmed.includes(name)); - if (unknown.length || missing.length || new Set(confirmed).size !== confirmed.length) { - throw new Error(`Camera assignment [${confirmed.join(', ')}] does not match the dataset cameras [${cameras.join(', ')}]`); - } - return confirmed; + if (!confirmed?.length) { + return cameras; + } + if ([...confirmed].sort().join('\n') !== [...cameras].sort().join('\n')) { + throw new Error(`Camera assignment [${confirmed.join(', ')}] does not match the dataset cameras [${cameras.join(', ')}]`); } - return pipelineCameraNames(meta.multiCam, pipeline.metadata?.cameraOrder, meta.cameraRoles ?? {}); + return confirmed; } async function runPipeline( @@ -390,10 +385,9 @@ async function runPipeline( if (meta.multiCam && stereoOrMultiCam) { const isMultiCamPipeline = multiCamPipelineMarkers.includes(pipeline.type); // 2-cam/3-cam pipes: which camera feeds which inputN is the order the - // user confirmed before the run; without one (CLI), the pipe's - // `# Camera Order:` header matched by role/name, else reference-first. + // user confirmed before the run. const multiCamOrder = isMultiCamPipeline - ? resolveMultiCamOrder(meta, pipeline, runPipelineArgs.pipelineParams?.cameraOrder) + ? multiCamOrderFor(meta, runPipelineArgs.pipelineParams?.cameraOrder) : undefined; const { argFilePair, outFiles } = await writeMultiCamStereoPipelineArgs(jobWorkDir, meta, settings, requiresInput, false, multiCamOrder); Object.entries(argFilePair).forEach(([arg, file]) => { @@ -424,21 +418,17 @@ async function runPipeline( }); } if (multiCamOrder) { - // Hand the camera registration (Camera Registration homographies) to the - // pipeline's per-camera warp processes. Refuse up front when a warped - // camera has no registration onto camera 1, rather than letting the - // pipe die at configure time on a missing file. - const registrationArgs = await buildRegistrationPipelineArgs(settings, meta, jobWorkDir, multiCamOrder); - // buildRegistrationPipelineArgs only binds warps whose camera has a - // fitted pair onto camera 1, so the bound warps are the fitted set. - const fittedPairs = Object.keys(registrationArgs) - .map((key) => key.match(/^warp(\d+):transformation_file$/)) - .filter((match): match is RegExpMatchArray => match !== null) - .map((match) => `${multiCamOrder[Number.parseInt(match[1], 10) - 1]}::${multiCamOrder[0]}`); - const missing = missingRegistrations(multiCamOrder, pipeline.metadata?.registrationWarps, fittedPairs); - if (missing.length) { - throw new Error(missing.map((entry) => describeMissingRegistration(entry, pipeline.name)).join('\n')); - } + // Hand the camera registration to the pipeline's warp processes; a + // warped camera with no registration onto camera 1 fails here, before + // the job exists. + const registrationArgs = await buildRegistrationPipelineArgs( + settings, + meta, + jobWorkDir, + multiCamOrder, + pipeline.metadata?.registrationWarps, + pipeline.name, + ); Object.entries(registrationArgs).forEach(([arg, value]) => { command.push(`-s ${arg}="${value}"`); }); diff --git a/docs/Pipeline-Import-Export.md b/docs/Pipeline-Import-Export.md index b5f1b597c..d4250cdb4 100644 --- a/docs/Pipeline-Import-Export.md +++ b/docs/Pipeline-Import-Export.md @@ -84,7 +84,7 @@ Which dataset camera feeds which input of a 2-cam/3-cam pipe is decided in the o 1. **Camera roles.** Each camera of a multicam dataset carries a sensor role (`eo`, `ir`, `uv`) in `cameraRoles`, inferred once at import from the camera (subfolder) name and, failing that, from tokens in its image file names (KAMERA style `…_rgb.jpg`, `…_ir.tif`, `…_uv.jpg`). Only a unanimous answer is recorded; ambiguous cameras get no role. 2. **Assignment step.** When you run a 2-cam/3-cam pipe, DIVE shows one row per pipeline camera (from the `# Camera Order:` header, or plain `input1..N` when the pipe has none) with the dataset camera it proposes — matched by role when both sides have one, else by name — and you confirm or change it before anything runs. Unfilled or duplicated slots block the run. Confirming a role-labelled slot saves the roles back onto the dataset (uncheck *Save these as the dataset's camera roles* to skip), so a corrected role wins over a misleading name next time and for every other pipeline. 3. **Registration check.** DIVE reads which inputs the pipe warps onto camera 1 (`process warpN :: warp_detections` / `warp_image` in the pipe body) and, in the same dialog, shows for each such row whether the chosen camera has a fitted registration onto camera 1. A missing one blocks the run with "Register X → Y in the Camera Registration tab first" — the pipe never gets to fail at configure time on a missing `registration_cameraN_to_camera1.json`. The same check runs server-side (and on desktop) before the job is created, so CLI runs get the same message. -4. **Job.** The confirmed order is what the job runs with (`cameraOrder` in the pipeline params; visible in the desktop job manifest). Runs started without the step (CLI) fall back to matching the header by role/name, and to registration-reference-first order for pipes with no header. +4. **Job.** The confirmed order is what the job runs with (`cameraOrder` in the pipeline params; visible in the desktop job manifest). API callers that omit it get the dataset's stored camera order, as before. ### Metadata File vs Configuration File diff --git a/server/dive_server/crud_dataset.py b/server/dive_server/crud_dataset.py index 84206d0e5..694bb52f5 100644 --- a/server/dive_server/crud_dataset.py +++ b/server/dive_server/crud_dataset.py @@ -1693,7 +1693,6 @@ def create_multicam( default_child = loaded_children[validated.defaultDisplay] parent_folder_doc = parent_folder multi_cam_cameras: Dict[str, Dict[str, str]] = {} - camera_image_names: Dict[str, List[str]] = {} for name in camera_order: child = loaded_children[name] if child['name'] != name: @@ -1703,11 +1702,9 @@ def create_multicam( 'folderId': str(child['_id']), 'type': camera_types_by_name[name], } - camera_image_names[name] = [item['name'] for item in Folder().childItems(child, limit=50)] - # Sensor role per camera, from the camera name and its media names; the - # pipeline camera-assignment step prefills from it and the user can - # correct it there. - camera_roles = infer_camera_roles(camera_image_names) + # Sensor role per camera from its name; the pipeline camera-assignment + # step prefills from it and the user can correct it there. + camera_roles = infer_camera_roles(camera_order) calibration_source_item_id = None json_calibration_item_id = None diff --git a/server/dive_server/crud_rpc.py b/server/dive_server/crud_rpc.py index 58d1a5948..9ae6acebd 100644 --- a/server/dive_server/crud_rpc.py +++ b/server/dive_server/crud_rpc.py @@ -23,9 +23,7 @@ describe_missing_registration, is_stereo_or_multicam_pipeline, missing_registrations, - pipeline_camera_order, pipeline_requires_input, - resolve_pipeline_camera_order, ) from dive_tasks.utils import choose_annotation_fps from dive_utils import ( @@ -338,14 +336,11 @@ def run_pipeline( cameras_meta = multi_cam.get('cameras') or {} is_warp_pipeline = pipeline['type'] in constants.MultiCamPipelineMarkers if is_warp_pipeline and camera_order: - # 2-cam/3-cam pipes warp everything onto camera 1. Which camera - # feeds which inputN is the pipe's contract (`# Camera Order:` - # header); a pipe without one gets the registration reference - # first, then display order. + # 2-cam/3-cam pipes warp everything onto camera 1; the order the + # user confirmed in the camera-assignment step is which camera + # feeds which inputN. Without one, the dataset's stored order. confirmed_order = (pipeline_params or {}).get('cameraOrder') - declared_order = (pipeline.get('metadata') or {}).get('cameraOrder') if confirmed_order: - # The order the user confirmed in the camera-assignment step. if sorted(confirmed_order) != sorted(camera_order): raise RestException( f'Camera assignment [{", ".join(confirmed_order)}] does not match ' @@ -353,20 +348,6 @@ def run_pipeline( code=400, ) camera_order = list(confirmed_order) - elif declared_order: - try: - camera_order = resolve_pipeline_camera_order( - declared_order, camera_order, (folder.get('meta') or {}).get('cameraRoles') - ) - except ValueError as err: - raise RestException(str(err), code=400) from err - else: - reference_camera = ( - multicam_default_display - if multicam_default_display in cameras_meta - else camera_order[0] - ) - camera_order = pipeline_camera_order(camera_order, reference_camera) reference_camera = camera_order[0] for name in camera_order: cam_info = cameras_meta[name] diff --git a/server/dive_tasks/multicam_pipeline.py b/server/dive_tasks/multicam_pipeline.py index 634e9f81b..bb2cbd284 100644 --- a/server/dive_tasks/multicam_pipeline.py +++ b/server/dive_tasks/multicam_pipeline.py @@ -90,9 +90,7 @@ def append_metadata_file_kwiver_settings( command.append(f'-s {shlex.quote(kwiver_key)}={shlex.quote(str(metadata_path))}') -# Role aliases: a slot token and a camera name match when they share a role, or -# when the token itself appears as a segment of the camera name (so pipes can -# name cameras literally, e.g. `# Camera Order: left, right`). Kept in sync with +# Sensor-role aliases for camera names; mirrors CAMERA_ROLE_ALIASES in # client/dive-common/pipelineCameraOrder.ts. CAMERA_ROLE_ALIASES: Dict[str, Tuple[str, ...]] = { 'eo': ('eo', 'rgb', 'optical', 'color', 'colour', 'vis', 'visible'), @@ -101,120 +99,22 @@ def append_metadata_file_kwiver_settings( } -def _name_segments(name: str) -> List[str]: - return [seg for seg in re.split(r'[^a-z0-9]+', name.lower()) if seg] - - -def role_of_token(token: str) -> Optional[str]: - """The role (eo / ir / uv) a slot token or camera-name segment denotes, if any.""" - lower = token.lower() - return next((role for role, aliases in CAMERA_ROLE_ALIASES.items() if lower in aliases), None) - - -def infer_camera_role(camera_name: str, image_names: Optional[List[str]] = None) -> Optional[str]: - """ - Infer a camera's sensor role from its name, falling back to tokens in its - image file names (KAMERA style `..._rgb.jpg` / `_ir.tif` / `_uv.jpg`). - Only a unanimous answer counts: a name or file set naming two roles yields - None. Kept in sync with client/dive-common/pipelineCameraOrder.ts. - """ - from_name = {r for r in (role_of_token(seg) for seg in _name_segments(camera_name)) if r} - if len(from_name) == 1: - return next(iter(from_name)) - if len(from_name) > 1: - return None - from_images = set() - for image in (image_names or [])[:50]: - base = re.split(r'[\\/]', image)[-1] - stem = re.sub(r'\.[^.]+$', '', base) - from_images.update(r for r in (role_of_token(seg) for seg in _name_segments(stem)) if r) - return next(iter(from_images)) if len(from_images) == 1 else None - - -def infer_camera_roles(cameras: Dict[str, Optional[List[str]]]) -> Dict[str, str]: - """Infer roles for a whole rig; cameras that cannot be classified are omitted.""" - roles: Dict[str, str] = {} - for name, images in cameras.items(): - role = infer_camera_role(name, images or []) - if role: - roles[name] = role - return roles - - -def candidates_for_slot( - slot: str, cameras: List[str], roles: Optional[Dict[str, str]] = None -) -> List[str]: - """ - Cameras that could fill a slot: cameras whose assigned role equals the - slot's role take precedence over name matching, so a corrected role wins - over a misleading name. +def infer_camera_role(camera_name: str) -> Optional[str]: """ - role = role_of_token(slot) - if role and roles: - by_role = [camera for camera in cameras if roles.get(camera) == role] - if by_role: - return by_role - return cameras_matching_slot(slot, cameras) - - -def cameras_matching_slot(token: str, cameras: List[str]) -> List[str]: - """Cameras whose name matches a slot token, by exact name, segment, or shared role.""" - lower = token.lower() - exact = [camera for camera in cameras if camera.lower() == lower] - if exact: - return exact - role = next((r for r, aliases in CAMERA_ROLE_ALIASES.items() if lower in aliases), None) - aliases = set(CAMERA_ROLE_ALIASES[role]) if role else {lower} - return [camera for camera in cameras if any(seg in aliases for seg in _name_segments(camera))] - - -def resolve_pipeline_camera_order( - slots: List[str], cameras: List[str], roles: Optional[Dict[str, str]] = None -) -> List[str]: + The sensor role (eo / ir / uv) a camera name denotes, or None when it names + none or more than one. Set once at multicam import; the pipeline + camera-assignment step prefills from it and lets the user correct it. """ - Map a pipe's declared `# Camera Order:` slots onto dataset cameras without - user interaction: every slot must match exactly one camera (by assigned - role first, then by name) and no camera may fill two slots. Raises - ValueError with a message naming the slot, the pipe's slots and the - dataset's cameras otherwise, so the run fails up front instead of being - silently mis-wired. - """ - context = f'pipeline cameras [{", ".join(slots)}], dataset cameras [{", ".join(cameras)}]' - if len(slots) != len(cameras): - raise ValueError( - f'Pipeline expects {len(slots)} cameras but the dataset has {len(cameras)}: ' - f'{context}' - ) - order: List[str] = [] - for index, slot in enumerate(slots, start=1): - matches = [c for c in candidates_for_slot(slot, cameras, roles) if c not in order] - if len(matches) != 1: - why = ( - 'no dataset camera matches' - if not matches - else f'several dataset cameras match ({", ".join(matches)})' - ) - raise ValueError( - f'Cannot place pipeline camera "{slot}" (input{index}): {why}. {context}. ' - 'Set the camera roles (or rename the cameras) so each pipeline camera ' - 'matches exactly one.' - ) - order.append(matches[0]) - return order + segments = [seg for seg in re.split(r'[^a-z0-9]+', camera_name.lower()) if seg] + roles = { + role for role, aliases in CAMERA_ROLE_ALIASES.items() if any(s in aliases for s in segments) + } + return next(iter(roles)) if len(roles) == 1 else None -def pipeline_camera_order(camera_names: List[str], reference: str) -> List[str]: - """ - Camera order for 2-cam/3-cam pipelines, matching the desktop client: the - registration reference camera feeds input1 (the per-camera registrations - all map onto the reference, and the pipes warp everything onto camera 1's - frame), remaining cameras keep display order. Which detector a pipe runs - on which input is the pipe's documented contract, not something DIVE - infers. - """ - if reference not in camera_names: - return camera_names - return [reference] + [name for name in camera_names if name != reference] +def infer_camera_roles(camera_names: List[str]) -> Dict[str, str]: + """Roles for a whole rig; cameras that cannot be classified are omitted.""" + return {name: role for name in camera_names for role in [infer_camera_role(name)] if role} def build_registration_pairs(folder_meta: dict) -> List[dict]: diff --git a/server/dive_utils/types.py b/server/dive_utils/types.py index d82c9344e..e5c5489a1 100644 --- a/server/dive_utils/types.py +++ b/server/dive_utils/types.py @@ -93,9 +93,8 @@ class PipeMetadata(TypedDict): # here; when unset the two conventional keys are used. calibrationKeys: NotRequired[Optional[list[str]]] # Camera role per pipeline input for 2-cam/3-cam pipes (e.g. ["EO", "UV", "IR"]), - # parsed from `# Camera Order: [cam...]`. Each slot is matched to a dataset - # camera by name at run time; when unset cameras are fed reference-first, then - # display order. + # parsed from `# Camera Order: [cam...]`. Labels the slots of the client's + # pre-run camera-assignment step; pipes without it show bare input1..N slots. cameraOrder: NotRequired[Optional[list[str]]] # Input positions the pipe warps onto camera 1 (`process warpN :: warp_detections | # warp_image`), e.g. [2, 3]; each such camera needs a fitted registration onto @@ -124,8 +123,8 @@ class PipelineParams(TypedDict, total=False): kwiverParams: Dict[str, str] runtimeParams: PipelineRuntimeParams # 2-cam/3-cam pipes: the dataset camera to feed each inputN, in order, as - # confirmed by the user before the run. When omitted the pipe's declared - # `# Camera Order:` slots are matched by role/name, else reference-first. + # confirmed by the user before the run. When omitted (API callers) the + # dataset's stored camera order is used. cameraOrder: List[str] # Name for the newly created dataset (filter / transcode / disparity). outputDatasetName: str diff --git a/server/tests/test_multicam_pipeline.py b/server/tests/test_multicam_pipeline.py index 66f3c1027..19769bf95 100644 --- a/server/tests/test_multicam_pipeline.py +++ b/server/tests/test_multicam_pipeline.py @@ -1,24 +1,19 @@ import json from pathlib import Path -import pytest - from dive_tasks.multicam_pipeline import ( DEFAULT_CALIBRATION_KEYS, append_stereo_calibration_kwiver_settings, build_multicam_kwiver_settings, build_registration_kwiver_settings, build_registration_pairs, - cameras_matching_slot, find_downloaded_calibration_file, infer_camera_role, infer_camera_roles, is_stereo_measurement_pipeline, is_stereo_or_multicam_pipeline, missing_registrations, - pipeline_camera_order, pipeline_requires_input, - resolve_pipeline_camera_order, stereo_calibration_keys, ) from dive_utils import constants @@ -131,45 +126,13 @@ def test_build_multicam_kwiver_settings_image_sequence(tmp_path: Path): ) -def test_pipeline_camera_order(): - # Reference first, remaining display order preserved. - assert pipeline_camera_order(['ir', 'rgb', 'uv'], 'rgb') == ['rgb', 'ir', 'uv'] - assert pipeline_camera_order(['rgb', 'uv', 'ir'], 'rgb') == ['rgb', 'uv', 'ir'] - assert pipeline_camera_order(['CENT_IR', 'CENT_EO'], 'CENT_EO') == ['CENT_EO', 'CENT_IR'] - # Unknown reference leaves the order alone. - assert pipeline_camera_order(['a', 'b'], 'missing') == ['a', 'b'] - - -def test_cameras_matching_slot(): - cameras = ['rgb', 'CENT_IR', 'uv_cam'] - assert cameras_matching_slot('EO', cameras) == ['rgb'] - assert cameras_matching_slot('IR', cameras) == ['CENT_IR'] - assert cameras_matching_slot('ultraviolet', cameras) == ['uv_cam'] - assert cameras_matching_slot('rgb', cameras) == ['rgb'] - # Exact name wins over role matching elsewhere; literal segments for non-role tokens. - assert cameras_matching_slot('ir', ['ir', 'thermal']) == ['ir'] - assert cameras_matching_slot('left', ['left_cam', 'right_cam']) == ['left_cam'] - - def test_infer_camera_role(): assert infer_camera_role('rgb') == 'eo' assert infer_camera_role('CENT_IR') == 'ir' assert infer_camera_role('uv_cam') == 'uv' - assert infer_camera_role('cam1', ['flight_0001_rgb.jpg', 'flight_0002_rgb.jpg']) == 'eo' - assert infer_camera_role('cam1', ['a_rgb.jpg', 'b_ir.tif']) is None assert infer_camera_role('eo_ir') is None - assert infer_camera_role('center', ['0001.png']) is None - assert infer_camera_roles({'rgb': [], 'center': ['x_ir.tif'], 'other': ['a.png']}) == { - 'rgb': 'eo', - 'center': 'ir', - } - - -def test_resolve_pipeline_camera_order_roles_win_over_names(): - # "thermal" is named like IR but the user marked it optical. - assert resolve_pipeline_camera_order( - ['EO', 'IR'], ['thermal', 'other'], {'thermal': 'eo', 'other': 'ir'} - ) == ['thermal', 'other'] + assert infer_camera_role('center') is None + assert infer_camera_roles(['rgb', 'CENT_IR', 'center']) == {'rgb': 'eo', 'CENT_IR': 'ir'} def test_missing_registrations(): @@ -184,24 +147,6 @@ def test_missing_registrations(): assert missing_registrations(['rgb', 'ir'], [3], []) == [] -def test_resolve_pipeline_camera_order(): - assert resolve_pipeline_camera_order(['EO', 'UV', 'IR'], ['rgb', 'ir', 'uv']) == [ - 'rgb', - 'uv', - 'ir', - ] - assert resolve_pipeline_camera_order(['EO', 'IR'], ['CENT_IR', 'CENT_EO']) == [ - 'CENT_EO', - 'CENT_IR', - ] - with pytest.raises(ValueError, match='expects 2 cameras but the dataset has 3'): - resolve_pipeline_camera_order(['EO', 'IR'], ['rgb', 'ir', 'uv']) - with pytest.raises(ValueError, match=r'"UV" \(input2\): no dataset camera matches'): - resolve_pipeline_camera_order(['EO', 'UV', 'IR'], ['rgb', 'ir', 'cam3']) - with pytest.raises(ValueError, match=r'"EO" \(input1\): several dataset cameras match'): - resolve_pipeline_camera_order(['EO', 'IR'], ['rgb', 'color']) - - IR_TO_RGB = [[1, 0, 5], [0, 1, -3], [0, 0, 1]] RGB_TO_IR = [[1, 0, -5], [0, 1, 3], [0, 0, 1]]