From 54f00004db1992c06117163b6ff257a2cc1a330d Mon Sep 17 00:00:00 2001 From: danvi Date: Tue, 11 Aug 2026 01:09:10 +0900 Subject: [PATCH 01/12] Add offline object inventory and localize perception pipeline (DIM-1343 PR1). Text prompt to 2D masks and memory search to 3D point clouds, with OWLv2, SigLIP, MPS support, LocalizePolicy, and related dependency pins. --- dimos/models/base.py | 21 +- dimos/models/embedding/siglip.py | 101 ++ dimos/models/segmentation/edge_tam.py | 91 +- dimos/models/test_base.py | 19 +- dimos/perception/detection/detectors/owlv2.py | 112 +++ dimos/perception/detection/detectors/yoloe.py | 4 +- dimos/perception/detection/project.py | 69 +- dimos/perception/memory/gates.py | 280 ++++++ dimos/perception/memory/inventory.py | 859 ++++++++++++++++++ dimos/perception/memory/localize.py | 538 +++++++++++ dimos/perception/memory/support_plane.py | 152 ++++ dimos/perception/memory/tool_inventory.py | 95 ++ dimos/perception/memory/tool_localize.py | 329 +++---- dimos/perception/memory/types.py | 198 ++++ pyproject.toml | 24 +- uv.lock | 591 +++++++++--- 16 files changed, 3121 insertions(+), 362 deletions(-) create mode 100644 dimos/models/embedding/siglip.py create mode 100644 dimos/perception/detection/detectors/owlv2.py create mode 100644 dimos/perception/memory/gates.py create mode 100644 dimos/perception/memory/inventory.py create mode 100644 dimos/perception/memory/localize.py create mode 100644 dimos/perception/memory/support_plane.py create mode 100644 dimos/perception/memory/tool_inventory.py create mode 100644 dimos/perception/memory/types.py diff --git a/dimos/models/base.py b/dimos/models/base.py index 65393405c5..5766f72663 100644 --- a/dimos/models/base.py +++ b/dimos/models/base.py @@ -24,12 +24,21 @@ from dimos.core.resource import Resource from dimos.protocol.service.spec import BaseConfig, Configurable -# Device string type - 'cuda', 'cpu', 'cuda:0', 'cuda:1', etc. -DeviceType = Annotated[str, "Device identifier (e.g., 'cuda', 'cpu', 'cuda:0')"] +# Device string type - 'cuda', 'cpu', 'cuda:0', 'cuda:1', 'mps', etc. +DeviceType = Annotated[str, "Device identifier (e.g., 'cuda', 'cpu', 'cuda:0', 'mps')"] + + +def default_torch_device() -> str: + """Best available torch device: CUDA, then Apple Metal (MPS), then CPU.""" + if torch.cuda.is_available(): + return "cuda" + if torch.backends.mps.is_available(): + return "mps" + return "cpu" class LocalModelConfig(BaseConfig): - device: DeviceType = "cuda" if torch.cuda.is_available() else "cpu" + device: DeviceType = default_torch_device() dtype: torch.dtype = torch.float32 warmup: bool = False autostart: bool = False @@ -54,8 +63,8 @@ def __init__(self, **kwargs: object) -> None: """Initialize local model with device and dtype configuration. Args: - device: Device to run on ('cuda', 'cpu', 'cuda:0', etc.). - Auto-detects CUDA availability if None. + device: Device to run on ('cuda', 'cpu', 'cuda:0', 'mps', etc.). + Defaults to the best available device (CUDA, then MPS, then CPU). dtype: Model dtype (torch.float16, torch.bfloat16, etc.). Uses class _default_dtype if None. autostart: If True, immediately load the model. @@ -109,6 +118,8 @@ def stop(self) -> None: gc.collect() if self.config.device.startswith("cuda") and torch.cuda.is_available(): torch.cuda.empty_cache() + elif self.config.device.startswith("mps") and torch.backends.mps.is_available(): + torch.mps.empty_cache() def _ensure_cuda_initialized(self) -> None: """Initialize CUDA context to prevent cuBLAS allocation failures. diff --git a/dimos/models/embedding/siglip.py b/dimos/models/embedding/siglip.py new file mode 100644 index 0000000000..5b3ff53d32 --- /dev/null +++ b/dimos/models/embedding/siglip.py @@ -0,0 +1,101 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from functools import cached_property +from typing import overload + +from PIL import Image as PILImage +import torch +import torch.nn.functional as functional +from transformers import SiglipModel as HFSiglipModel, SiglipProcessor + +from dimos.models.base import HuggingFaceModel +from dimos.models.embedding.base import Embedding, EmbeddingModel, HuggingFaceEmbeddingModelConfig +from dimos.msgs.sensor_msgs.Image import Image + + +class SigLIPModelConfig(HuggingFaceEmbeddingModelConfig): + model_name: str = "google/siglip-base-patch16-224" + dtype: torch.dtype = torch.float32 + + +class SigLIPModel(EmbeddingModel, HuggingFaceModel): + """SigLIP image/text embedding model for frame-level semantic retrieval. + + Weights come from the Hugging Face hub cache: a miss downloads once, a + hit reuses the cache. Text inputs use ``padding="max_length"`` - SigLIP + was trained with max-length padded text, and unpadded prompts measurably + degrade the text-image alignment. + """ + + config: SigLIPModelConfig + _model_class = HFSiglipModel + + @cached_property + def _model(self) -> HFSiglipModel: + self._ensure_cuda_initialized() + return ( + HFSiglipModel.from_pretrained(self.config.model_name) + .eval() + .to(self.config.device) + ) + + @cached_property + def _processor(self) -> SiglipProcessor: + return SiglipProcessor.from_pretrained(self.config.model_name, use_fast=True) + + @overload + def embed(self, image: Image, /) -> Embedding: ... + @overload + def embed(self, *images: Image) -> list[Embedding]: ... + def embed(self, *images: Image) -> Embedding | list[Embedding]: + """Embed one or more images into the shared image-text space.""" + pil_images = [PILImage.fromarray(img.to_rgb().data) for img in images] + + with torch.inference_mode(): + inputs = self._processor(images=pil_images, return_tensors="pt").to(self.config.device) + image_features = self._model.get_image_features(**inputs) + if self.config.normalize: + image_features = functional.normalize(image_features, dim=-1) + + embeddings = [ + Embedding(vector=feat, timestamp=images[i].ts) + for i, feat in enumerate(image_features) + ] + return embeddings[0] if len(images) == 1 else embeddings + + @overload + def embed_text(self, text: str, /) -> Embedding: ... + @overload + def embed_text(self, *texts: str) -> list[Embedding]: ... + def embed_text(self, *texts: str) -> Embedding | list[Embedding]: + """Embed one or more text strings into the shared image-text space.""" + with torch.inference_mode(): + inputs = self._processor( + text=list(texts), return_tensors="pt", padding="max_length", truncation=True + ).to(self.config.device) + text_features = self._model.get_text_features(**inputs) + if self.config.normalize: + text_features = functional.normalize(text_features, dim=-1) + + embeddings = [Embedding(vector=feat) for feat in text_features] + return embeddings[0] if len(texts) == 1 else embeddings + + def stop(self) -> None: + """Release model and free GPU memory.""" + if "_processor" in self.__dict__: + del self.__dict__["_processor"] + super().stop() diff --git a/dimos/models/segmentation/edge_tam.py b/dimos/models/segmentation/edge_tam.py index 72f1484af8..e9488fc935 100644 --- a/dimos/models/segmentation/edge_tam.py +++ b/dimos/models/segmentation/edge_tam.py @@ -13,7 +13,8 @@ # limitations under the License. from collections.abc import Generator -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext +from functools import lru_cache from pathlib import Path import shutil import tempfile @@ -26,6 +27,7 @@ from PIL import Image as PILImage import torch +from dimos.models.base import default_torch_device from dimos.msgs.sensor_msgs.Image import Image from dimos.perception.detection.detectors.base import Detector from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox @@ -40,6 +42,20 @@ logger = setup_logger() +# Hole/sprinkle cleanup in SAM2 needs the optional CUDA extension sam2._C. +# Requesting it without _C triggers a per-call UserWarning and then skips. +_FILL_HOLE_AREA = 8 +_MIN_MASK_REGION_AREA = 150 + + +@lru_cache(maxsize=1) +def _sam2_cuda_postprocess_available() -> bool: + try: + from sam2 import _C # noqa: F401 + except ImportError: + return False + return True + class SAM2InferenceState(TypedDict): images: list[torch.Tensor | None] @@ -54,17 +70,19 @@ def _build_model() -> "SAM2VideoPredictor": if not local_config_path.exists(): raise FileNotFoundError(f"EdgeTAM config not found at {local_config_path}") - if not torch.cuda.is_available(): - raise RuntimeError("EdgeTAM requires a CUDA-capable GPU") + device = default_torch_device() + if device == "cpu": + raise RuntimeError("EdgeTAM requires a CUDA or MPS device") cfg = OmegaConf.load(local_config_path) + fill_hole_area = _FILL_HOLE_AREA if _sam2_cuda_postprocess_available() else 0 overrides = { "model.sam_mask_decoder_extra_args.dynamic_multimask_via_stability": True, "model.sam_mask_decoder_extra_args.dynamic_multimask_stability_delta": 0.05, "model.sam_mask_decoder_extra_args.dynamic_multimask_stability_thresh": 0.98, "model.binarize_mask_from_pts_for_mem_enc": True, - "model.fill_hole_area": 8, + "model.fill_hole_area": fill_hole_area, } for key, value in overrides.items(): @@ -90,7 +108,7 @@ def _build_model() -> "SAM2VideoPredictor": if unexpected_keys: raise RuntimeError("Unexpected keys in checkpoint") - predictor = predictor.to("cuda") + predictor = predictor.to(device) predictor.eval() return predictor @@ -102,6 +120,59 @@ def __init__(self) -> None: from sam2.sam2_image_predictor import SAM2ImagePredictor self._predictor = SAM2ImagePredictor(_build_model()) + self._mask_generators: dict[int, Any] = {} + + def propose_all( + self, + image: Image, + points_per_side: int = 24, + pred_iou_thresh: float = 0.55, + stability_score_thresh: float = 0.85, + min_mask_region_area: int = _MIN_MASK_REGION_AREA, + ) -> ImageDetections2D: + """Class-agnostic mask proposals over the whole image, no vocabulary. + + Wraps ``SAM2AutomaticMaskGenerator`` on the shared EdgeTAM model. + Proposals carry no names ("proposal") and ``confidence`` is the + predicted mask IoU. + """ + import cv2 + from sam2.automatic_mask_generator import SAM2AutomaticMaskGenerator + + if not _sam2_cuda_postprocess_available(): + min_mask_region_area = 0 + + generator = self._mask_generators.get(points_per_side) + if generator is None: + generator = SAM2AutomaticMaskGenerator( + model=self._predictor.model, + points_per_side=points_per_side, + pred_iou_thresh=pred_iou_thresh, + stability_score_thresh=stability_score_thresh, + min_mask_region_area=min_mask_region_area, + ) + self._mask_generators[points_per_side] = generator + + rgb = cv2.cvtColor(image.to_opencv(), cv2.COLOR_BGR2RGB) + amp = ( + torch.autocast("cuda", dtype=torch.bfloat16) + if self._predictor.device.type == "cuda" + else nullcontext() + ) + with torch.no_grad(), amp: + proposals = generator.generate(rgb) + + detections: list[Detection2DBBox] = [ + Detection2DSeg.from_sam2_result( + proposal["segmentation"], + i, + image, + name="proposal", + confidence=float(proposal.get("predicted_iou", 1.0)), + ) + for i, proposal in enumerate(proposals) + ] + return ImageDetections2D(image, [det for det in detections if det.is_valid()]) def segment(self, detections: ImageDetections2D) -> ImageDetections2D: """Refine box detections into mask detections (Detection2DSeg).""" @@ -113,7 +184,13 @@ def segment(self, detections: ImageDetections2D) -> ImageDetections2D: image = detections.image rgb = cv2.cvtColor(image.to_opencv(), cv2.COLOR_BGR2RGB) - with torch.no_grad(), torch.autocast("cuda", dtype=torch.bfloat16): + # bfloat16 autocast is CUDA-only; MPS runs in float32 + amp = ( + torch.autocast("cuda", dtype=torch.bfloat16) + if self._predictor.device.type == "cuda" + else nullcontext() + ) + with torch.no_grad(), amp: self._predictor.set_image(rgb) boxes = np.array([det.bbox for det in detections], dtype=np.float32) masks, _, _ = self._predictor.predict(box=boxes, multimask_output=False) @@ -168,7 +245,7 @@ def _prepare_frame(self, image: Image) -> torch.Tensor: img_np /= img_std img_tensor = torch.from_numpy(img_np).permute(2, 0, 1).float() - img_tensor = img_tensor.cuda() + img_tensor = img_tensor.to(self._predictor.device) return img_tensor diff --git a/dimos/models/test_base.py b/dimos/models/test_base.py index 5b167b7fb5..38f5db147a 100644 --- a/dimos/models/test_base.py +++ b/dimos/models/test_base.py @@ -16,9 +16,10 @@ from functools import cached_property +import pytest import torch -from dimos.models.base import HuggingFaceModel, LocalModel +from dimos.models.base import HuggingFaceModel, LocalModel, default_torch_device class ConcreteLocalModel(LocalModel): @@ -37,11 +38,17 @@ def _model(self) -> str: return f"hf_model:{self.model_name}" -def test_local_model_device_auto_detection() -> None: - """Test that device is auto-detected based on CUDA availability.""" - model = ConcreteLocalModel() - expected = "cuda" if torch.cuda.is_available() else "cpu" - assert model.device == expected +def test_default_torch_device_priority(monkeypatch: pytest.MonkeyPatch) -> None: + """CUDA wins over MPS, MPS wins over CPU, CPU is the last resort.""" + monkeypatch.setattr(torch.cuda, "is_available", lambda: True) + monkeypatch.setattr(torch.backends.mps, "is_available", lambda: True) + assert default_torch_device() == "cuda" + + monkeypatch.setattr(torch.cuda, "is_available", lambda: False) + assert default_torch_device() == "mps" + + monkeypatch.setattr(torch.backends.mps, "is_available", lambda: False) + assert default_torch_device() == "cpu" def test_local_model_explicit_device() -> None: diff --git a/dimos/perception/detection/detectors/owlv2.py b/dimos/perception/detection/detectors/owlv2.py new file mode 100644 index 0000000000..b2d85bd31e --- /dev/null +++ b/dimos/perception/detection/detectors/owlv2.py @@ -0,0 +1,112 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OWLv2 open-vocabulary detection: text prompts to boxes with calibrated scores.""" + +from __future__ import annotations + +from functools import cached_property + +from PIL import Image as PILImage +import torch + +from dimos.models.base import HuggingFaceModel, HuggingFaceModelConfig +from dimos.msgs.sensor_msgs.Image import Image +from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D + + +class Owlv2Config(HuggingFaceModelConfig): + model_name: str = "google/owlv2-base-patch16-ensemble" + dtype: torch.dtype = torch.float32 + + +class Owlv2Detector(HuggingFaceModel): + """Text-conditioned open-vocabulary detector with per-box scores. + + Unlike VLM-based proposers, the per-box score is a usable acceptance + signal: real matches on this rig score well above text-only + hallucinations, so a threshold plus refusal is meaningful. Weights come + from the Hugging Face hub cache: a miss downloads once, a hit reuses the + cache. + """ + + config: Owlv2Config + + @cached_property + def _model(self): # type: ignore[no-untyped-def] + from transformers import Owlv2ForObjectDetection + + self._ensure_cuda_initialized() + return ( + Owlv2ForObjectDetection.from_pretrained(self.config.model_name) + .eval() + .to(self.config.device) + ) + + @cached_property + def _processor(self): # type: ignore[no-untyped-def] + from transformers import Owlv2Processor + + return Owlv2Processor.from_pretrained(self.config.model_name) + + def query_detections( + self, + image: Image, + queries: list[str], + threshold: float = 0.1, + ) -> ImageDetections2D: + """Detect every query string in the image; boxes below threshold are dropped. + + Each detection's ``name`` is the query text it matched and its + ``confidence`` is the calibrated per-box score. ``class_id`` indexes + into ``queries``. + """ + pil = PILImage.fromarray(image.to_rgb().data) + with torch.inference_mode(): + inputs = self._processor(text=[queries], images=pil, return_tensors="pt").to( + self.config.device + ) + outputs = self._model(**inputs) + results = self._processor.post_process_grounded_object_detection( + outputs=outputs, + target_sizes=torch.tensor([(pil.height, pil.width)]), + threshold=threshold, + )[0] + + detections: list[Detection2DBBox] = [] + w, h = float(pil.width), float(pil.height) + for box, score, label in zip( + results["boxes"], results["scores"], results["labels"], strict=False + ): + x1, y1, x2, y2 = (float(v) for v in box) + bbox = (max(0.0, x1), max(0.0, y1), min(w, x2), min(h, y2)) + det = Detection2DBBox( + bbox=bbox, + track_id=-1, + class_id=int(label), + confidence=float(score), + name=queries[int(label)], + ts=image.ts, + image=image, + ) + if det.is_valid(): + detections.append(det) + + return ImageDetections2D(image=image, detections=detections) + + def stop(self) -> None: + if "_processor" in self.__dict__: + del self.__dict__["_processor"] + super().stop() diff --git a/dimos/perception/detection/detectors/yoloe.py b/dimos/perception/detection/detectors/yoloe.py index aa7ed19b21..fddd71bca1 100644 --- a/dimos/perception/detection/detectors/yoloe.py +++ b/dimos/perception/detection/detectors/yoloe.py @@ -18,9 +18,9 @@ import numpy as np from numpy.typing import NDArray -import torch from ultralytics import YOLOE # type: ignore[attr-defined] +from dimos.models.base import default_torch_device from dimos.msgs.sensor_msgs.Image import Image from dimos.perception.detection.detectors.base import Detector from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D @@ -81,7 +81,7 @@ def __init__( if self.max_area_ratio is not None and not (0.0 < self.max_area_ratio <= 1.0): raise ValueError("max_area_ratio must be in the range (0, 1].") - self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") + self.device = device or default_torch_device() logger.info( f"YOLO-E detector loaded model={model_name} prompt_mode={prompt_mode.value} " f"device={self.device} conf={self.conf}" diff --git a/dimos/perception/detection/project.py b/dimos/perception/detection/project.py index 0ea57730bf..1f9a8561ab 100644 --- a/dimos/perception/detection/project.py +++ b/dimos/perception/detection/project.py @@ -122,6 +122,10 @@ def sees( time_tolerance: float = 5.0, margin: float = 0.0, max_range: float | None = None, + extent: Any | None = None, + min_fraction: float = 1.0, + depth: Callable[[Observation[Any]], Image | None] | None = None, + occlusion_tolerance: float = 0.08, ) -> Callable[[Observation[Any]], bool]: """Predicate for ``Stream.filter()``: keep frames whose camera sees the world point. @@ -129,9 +133,19 @@ def sees( images.near(det.pose, radius=6.0).filter(sees(det.pose, camera_info, ...)) - ``margin`` requires the point that many pixels inside the image edges; - ``max_range`` drops frames further than that from the point. Occlusion is - not checked. + ``margin`` requires projections that many pixels inside the image edges; + ``max_range`` drops frames further than that from the point. + + ``extent`` turns the point into a support box: the center plus the eight + box corners are projected, and the frame passes when at least + ``min_fraction`` of those samples land inside the image. A half-framed + object is then no longer treated like a fully framed one. + + ``depth`` enables a cheap occlusion test: measured depth at the visible + sample pixels is compared against each sample's expected range, and the + frame is rejected when a majority of samples sit behind nearer geometry + by more than ``occlusion_tolerance`` meters. Invalid depth (0) counts as + unknown, not as occluding. """ if tf is None and base_to_optical is None: raise ValueError("sees needs either tf or base_to_optical") @@ -141,20 +155,57 @@ def sees( cx, cy = camera_info.K[2], camera_info.K[5] width, height = float(camera_info.width), float(camera_info.height) + if extent is not None: + half = 0.5 * ( + np.array([extent.x, extent.y, extent.z]) + if hasattr(extent, "x") + else np.asarray(extent, dtype=float) + ) + corners = np.array( + [[sx, sy, sz] for sx in (-1, 1) for sy in (-1, 1) for sz in (-1, 1)], dtype=float + ) + samples = np.vstack([target, target + corners * half]) + else: + samples = target.reshape(1, 3) + def _sees(obs: Observation[Any]) -> bool: transform = _world_to_optical( obs, world_frame, tf, base_to_optical, optical_frame, time_tolerance ) if transform is None: return False - p = (transform.to_matrix() @ np.append(target, 1.0))[:3] - if p[2] <= 0: + matrix = transform.to_matrix() + pts = (matrix @ np.column_stack([samples, np.ones(len(samples))]).T).T[:, :3] + in_front = pts[:, 2] > 0 + if not in_front.any(): return False - if max_range is not None and float(np.linalg.norm(p)) > max_range: + if max_range is not None and float(np.linalg.norm(pts[0])) > max_range: return False - u = float(fx * p[0] / p[2] + cx) - v = float(fy * p[1] / p[2] + cy) - return margin <= u < width - margin and margin <= v < height - margin + u = fx * pts[:, 0] / np.where(in_front, pts[:, 2], 1.0) + cx + v = fy * pts[:, 1] / np.where(in_front, pts[:, 2], 1.0) + cy + visible = ( + in_front & (u >= margin) & (u < width - margin) & (v >= margin) & (v < height - margin) + ) + if float(visible.mean()) < min_fraction: + return False + + if depth is not None and visible.any(): + depth_frame = depth(obs) + if depth_frame is not None: + depth_m = np.asarray(depth_frame.data, dtype=np.float32) + if depth_frame.data.dtype == np.uint16: + depth_m = depth_m * 0.001 + rows = np.clip(v[visible].astype(int), 0, depth_m.shape[0] - 1) + cols = np.clip(u[visible].astype(int), 0, depth_m.shape[1] - 1) + measured = depth_m[rows, cols] + expected = pts[visible][:, 2] + known = measured > 0 + if known.any(): + occluded = measured[known] + occlusion_tolerance < expected[known] + if float(occluded.mean()) > 0.5: + return False + + return True return _sees diff --git a/dimos/perception/memory/gates.py b/dimos/perception/memory/gates.py new file mode 100644 index 0000000000..2536a44a4d --- /dev/null +++ b/dimos/perception/memory/gates.py @@ -0,0 +1,280 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Per-frame gates and lookups over a memory2 recording: poses, stillness, frames. + +Two per-frame gates live here and both are required: + +* The **camera-motion gate** differences tf. It rejects frames captured while + the wrist sweeps, because a stale or interpolated transform smears the + projection. +* The **scene-motion gate** differences images, conditioned on camera + stillness at both compared instants. It rejects frames captured while the + scene itself changes - a parked camera watching hands rearrange objects is + exactly the case tf cannot see. The reference frame is anchored at the + start of the surrounding camera-still interval, so a frame is trusted only + while the scene still matches the state it had when the camera parked. + +Every function here takes the store and/or tf plus primitives; the stillness +intervals and the grayscale memo are allocated by the caller and passed in, +one per query. +""" + +from __future__ import annotations + +from bisect import bisect_right +from typing import TYPE_CHECKING, Any + +import numpy as np + +if TYPE_CHECKING: + from dimos.memory2.type.observation import Observation + from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped + from dimos.msgs.sensor_msgs.Image import Image + from dimos.protocol.tf.tf import TFLookup + +OPTICAL_FRAME = "camera_color_optical_frame" +WORLD_FRAME = "world" + +# One world-pose period (9.86 Hz measured) plus margin. The 0.5 s the sketch +# used spans five samples - up to 7.5 cm of smear at peak wrist speed. +TF_TOLERANCE = 0.12 + +SPEED_MAX = 0.02 # m/s - camera counts as still below this +STILL_ENVELOPE = 0.15 # s - stillness must hold over the whole capture envelope + +# Scene-motion gate: downscaled grayscale absolute difference. +DIFF_PIXEL_THRESHOLD = 28 # gray levels - per-pixel change floor +MOTION_THRESHOLD = 0.02 # fraction of changed pixels that flags scene motion +SHORT_DIFF_DT = 0.45 # s - bilateral diff span for active motion +DIFF_WIDTH = 212 # px - diff resolution (1/4 of 848) + + +def camera_pose( + tf: TFLookup, + ts: float, + optical_frame: str = OPTICAL_FRAME, + world_frame: str = WORLD_FRAME, + tolerance: float = TF_TOLERANCE, +) -> PoseStamped | None: + """World pose of the camera optical frame at ts - it rides the wrist.""" + transform = tf.get(optical_frame, world_frame, ts, tolerance) + return (-transform).to_pose() if transform is not None else None + + +def camera_speed( + tf: TFLookup, + ts: float, + optical_frame: str = OPTICAL_FRAME, + world_frame: str = WORLD_FRAME, + tolerance: float = TF_TOLERANCE, + dt: float = 0.06, +) -> float | None: + """Linear speed of the camera (m/s) around ts, from tf differencing.""" + a = camera_pose(tf, ts - dt, optical_frame, world_frame, tolerance) + b = camera_pose(tf, ts + dt, optical_frame, world_frame, tolerance) + if a is None or b is None: + return None + return float((b.position - a.position).magnitude() / (2 * dt)) + + +def camera_still( + tf: TFLookup, + ts: float, + optical_frame: str = OPTICAL_FRAME, + world_frame: str = WORLD_FRAME, + tolerance: float = TF_TOLERANCE, + speed_max: float = SPEED_MAX, + envelope: float = STILL_ENVELOPE, +) -> bool: + """Camera is still over the whole capture envelope, not just at ts.""" + for offset in (-envelope, 0.0, envelope): + speed = camera_speed(tf, ts + offset, optical_frame, world_frame, tolerance) + if speed is None or speed > speed_max: + return False + return True + + +def still_intervals( + tf: TFLookup, + t0: float, + t1: float, + optical_frame: str = OPTICAL_FRAME, + world_frame: str = WORLD_FRAME, + tolerance: float = TF_TOLERANCE, + speed_max: float = SPEED_MAX, +) -> list[tuple[float, float]]: + """Maximal camera-still intervals inside [t0, t1], sampled at 0.25 s. + + Computed once per query by the caller; every scene-motion query resolves + its surrounding interval from this list. + """ + step = 0.25 + times = np.arange(t0, t1 + step, step) + intervals: list[tuple[float, float]] = [] + run_start: float | None = None + for t in times: + speed = camera_speed(tf, float(t), optical_frame, world_frame, tolerance) + still = speed is not None and speed <= speed_max + if still and run_start is None: + run_start = float(t) + elif not still and run_start is not None: + intervals.append((run_start, float(t) - step)) + run_start = None + if run_start is not None: + intervals.append((run_start, float(times[-1]))) + return [(a, b) for a, b in intervals if b >= a] + + +def _interval_containing( + ts: float, intervals: list[tuple[float, float]] +) -> tuple[float, float] | None: + idx = bisect_right([a for a, _ in intervals], ts) - 1 + if idx < 0: + return None + a, b = intervals[idx] + return (a, b) if a - 0.25 <= ts <= b + 0.25 else None + + +def _gray_small(store: Any, ts: float, gray: dict[float, np.ndarray | None]) -> np.ndarray | None: + """Downscaled grayscale of the color frame nearest ts, memoized in *gray*.""" + key = round(ts, 2) + if key in gray: + return gray[key] + + import cv2 + + small_gray: np.ndarray | None = None + try: + frame = store.streams.color_image.at(ts, 0.1).first().data + except LookupError: + frame = None + if frame is not None: + img = frame.to_opencv() + h = int(img.shape[0] * DIFF_WIDTH / img.shape[1]) + small = cv2.resize(img, (DIFF_WIDTH, h), interpolation=cv2.INTER_AREA) + small_gray = cv2.cvtColor(small, cv2.COLOR_BGR2GRAY) + + gray[key] = small_gray + return small_gray + + +def _diff_fraction( + store: Any, ts_a: float, ts_b: float, gray: dict[float, np.ndarray | None] +) -> float | None: + """Fraction of pixels changed between the frames nearest the two instants.""" + a, b = _gray_small(store, ts_a, gray), _gray_small(store, ts_b, gray) + if a is None or b is None or a.shape != b.shape: + return None + delta = np.abs(a.astype(np.int16) - b.astype(np.int16)) + return float((delta > DIFF_PIXEL_THRESHOLD).mean()) + + +def scene_still( + store: Any, + ts: float, + intervals: list[tuple[float, float]], + gray: dict[float, np.ndarray | None], + motion_threshold: float = MOTION_THRESHOLD, +) -> bool: + """True when the scene around ts is static and unchanged since the camera parked. + + Requires camera stillness at every compared instant - unconditioned, + a moving camera changes every pixel and scans become indistinguishable + from manipulation. Three image terms, all below ``motion_threshold``: + + * anchored: against the start of the surrounding camera-still + interval, so anything the scene changed since the camera parked + (an object placed, moved, or removed) rejects every later frame of + that interval; + * bilateral: against frames a fixed short span before and after ts, + which catches hands actively moving through the view. + """ + interval = _interval_containing(ts, intervals) + if interval is None: + return False + a, b = interval + + anchor = min(a + 0.3, ts) + fraction = _diff_fraction(store, anchor, ts, gray) + if fraction is None or fraction > motion_threshold: + return False + + for other in (max(a, ts - SHORT_DIFF_DT), min(b, ts + SHORT_DIFF_DT)): + if abs(other - ts) < 0.05: + continue + fraction = _diff_fraction(store, other, ts, gray) + if fraction is None or fraction > motion_threshold: + return False + return True + + +def depth_at(store: Any, ts: float, tolerance: float = 0.06) -> Image | None: + """Temporal join: aligned depth frame for a color timestamp.""" + try: + return store.streams.depth_image.at(ts, tolerance).first().data + except LookupError: + return None + + +def keyframes( + store: Any, + tf: TFLookup, + t0: float, + t1: float, + stride: float, + intervals: list[tuple[float, float]], + gray: dict[float, np.ndarray | None], + motion_threshold: float = MOTION_THRESHOLD, + optical_frame: str = OPTICAL_FRAME, + world_frame: str = WORLD_FRAME, + tolerance: float = TF_TOLERANCE, +) -> list[Observation[Image]]: + """Camera-still, scene-still color frames on a coarse grid over [t0, t1]. + + For each grid point the nearest passing frame within half a stride is + selected, so a grid point landing mid-sweep snaps to the neighboring + pause instead of being lost. + """ + selected: list[Observation[Image]] = [] + seen: set[float] = set() + offsets = [0.0] + probe = 0.35 + while probe <= stride / 2: + offsets.extend([probe, -probe]) + probe += 0.35 + + t = t0 + 0.5 + while t < t1: + for offset in offsets: + ts = t + offset + if ts < t0 or ts > t1: + continue + if not camera_still(tf, ts, optical_frame, world_frame, tolerance): + continue + if not scene_still(store, ts, intervals, gray, motion_threshold): + continue + try: + obs = store.streams.color_image.at(ts, 0.1).first() + except LookupError: + continue + if obs.ts in seen: + break + if tf.get(optical_frame, world_frame, obs.ts, tolerance) is None: + continue + seen.add(obs.ts) + selected.append(obs) + break + t += stride + return selected diff --git a/dimos/perception/memory/inventory.py b/dimos/perception/memory/inventory.py new file mode 100644 index 0000000000..66ff38e602 --- /dev/null +++ b/dimos/perception/memory/inventory.py @@ -0,0 +1,859 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Query-time scene inventory: prompt-free discovery plus geometric dedup. + +``inventory()`` answers "what instances are on the table" for a time window, +computed at query time over the recording - no ingest pass, no persisted +instance table. Existence is decoupled from naming and the ordering is a +constraint, not a preference: propose (EdgeTAM automatic masks), lift +(masked depth to world supports), associate (hard constraints before any +score), and only then name (OWLv2, labels as metadata). Labels and +appearance never enter association; position and same-frame co-occurrence +decide everything, which is what keeps two identical objects two instances. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from time import perf_counter +from typing import TYPE_CHECKING, Any + +import numpy as np + +from dimos.memory2.tf import StreamTF +from dimos.perception.detection.type.detection3d.imageDetections3DPC import ImageDetections3DPC +from dimos.perception.memory import gates +from dimos.perception.memory.gates import ( + MOTION_THRESHOLD, + OPTICAL_FRAME, + TF_TOLERANCE, + WORLD_FRAME, +) +from dimos.perception.memory.support_plane import SupportPlane, fit_support_plane +from dimos.perception.memory.types import ( + Instance, + InventoryPolicy, + Support, + SupportObservation, + aabb_overlap, +) +from dimos.utils.logging_config import setup_logger + +if TYPE_CHECKING: + from dimos_lcm.sensor_msgs import CameraInfo + + from dimos.perception.detection.type.detection2d.seg import Detection2DSeg + from dimos.protocol.tf.tf import TFLookup + +logger = setup_logger() + +KEYFRAME_STRIDE = 2.5 # s - proposal keyframe grid +MAX_PROPOSALS_PER_FRAME = 40 +NAME_FRAMES_PER_INSTANCE = 5 +NAME_SCORE_FLOOR = 0.18 +# An attachment must be the detector drawing a box around this member, not a +# box that merely crosses it: a tape-grid "ruler" box overlapping a diagonal +# book's axis-aligned bbox reaches IoU 0.31, the book's own box reaches 0.95. +NAME_ATTACH_IOU = 0.45 +# Measured on the S1 run: a 31-class prompt dilutes the text tokens enough +# that a plainly visible class drops out of its own box (book: 0.39 in a +# 10-class prompt, absent at the same threshold in the 31-class one). +NAME_PROMPT_CAP = 10 +SUPPRESS_SCORE = 0.25 +SUPPRESS_OVERLAP = 0.35 +UNGROUNDED_TRACK_IOU = 0.40 + +# Naming vocabulary: a generic list of common tabletop and household objects, +# batched into one capped OWLv2 prompt. The vocabulary is a naming-pass +# concern only - existence is geometric and an instance that matches nothing +# here keeps its unknown-N name with score 0. +GENERIC_VOCABULARY = [ + "pen", + "pencil", + "marker", + "highlighter", + "eraser", + "book", + "notebook", + "sticky notes", + "sheet of paper", + "roll of tape", + "scissors", + "stapler", + "ruler", + "laptop", + "computer keyboard", + "computer mouse", + "mobile phone", + "cup", + "bottle", + "drink can", + "bowl", + "cardboard box", + "cable", + "remote control", + "glasses", + "headphones", + "wallet", + "toy block", +] +SUPPRESS_QUERIES = ["person", "human hand", "human arm"] + + +@dataclass +class _Track: + members: list[SupportObservation] = field(default_factory=list) + frame_ts: set[float] = field(default_factory=set) + labels: dict[str, float] = field(default_factory=dict) + + def add(self, obs: SupportObservation, frame_key: float) -> None: + self.members.append(obs) + self.frame_ts.add(frame_key) + + @property + def centroid(self) -> np.ndarray: + return np.median(np.stack([m.centroid for m in self.members]), axis=0) + + @property + def aabb(self) -> tuple[np.ndarray, np.ndarray]: + lo = np.median(np.stack([m.aabb_min for m in self.members]), axis=0) + hi = np.median(np.stack([m.aabb_max for m in self.members]), axis=0) + return lo, hi + + @property + def latest(self) -> SupportObservation: + return max(self.members, key=lambda m: m.ts) + + +@dataclass +class _Track2D: + """Ungrounded track: RGB detections with no valid depth, 2D identity only.""" + + members: list[Detection2DSeg] = field(default_factory=list) + frame_ts: set[float] = field(default_factory=set) + labels: dict[str, float] = field(default_factory=dict) + + +def _bbox_iou(a: tuple[float, float, float, float], b: tuple[float, float, float, float]) -> float: + ax1, ay1, ax2, ay2 = a + bx1, by1, bx2, by2 = b + ix = max(0.0, min(ax2, bx2) - max(ax1, bx1)) + iy = max(0.0, min(ay2, by2) - max(ay1, by1)) + inter = ix * iy + union = (ax2 - ax1) * (ay2 - ay1) + (bx2 - bx1) * (by2 - by1) - inter + return inter / union if union > 0 else 0.0 + + +def _proposal_passes_2d(det: Detection2DSeg, image_area: float, policy: InventoryPolicy) -> bool: + area = float((det.mask > 0).sum()) + if area < policy.min_mask_area_px: + return False + if area > policy.max_mask_area_fraction * image_area: + return False + return True + + +# A lifted cloud plainly spanning more than one object: wider than any single +# tabletop object here, or reaching table-to-well-above-hand height. +SPLIT_EXTENT_M = 0.30 +SPLIT_HEIGHT_M = 0.10 +SPLIT_EPS_M = 0.03 + + +def _split_oversized( + points: np.ndarray, plane: SupportPlane | None, policy: InventoryPolicy +) -> list[np.ndarray]: + """Re-segment a mask-bled cloud by 3D connectivity. + + Automatic masks occasionally bleed across an object onto the table and + its neighbors; the lifted cloud then violates single-object bounds. The + repair is geometric: strip the support-surface points, then split by + spatial connectivity - distinct objects on this rig are separated by + more than the cluster gap, one object's surface is not. + """ + extent = points.max(axis=0) - points.min(axis=0) + if float(extent.max()) <= SPLIT_EXTENT_M and float(extent[2]) <= SPLIT_HEIGHT_M: + return [points] + if plane is None: + return [points] + heights = plane.height_above(points) + if float((np.abs(heights) <= 0.003).mean()) < 0.15: + # No appreciable support-surface content: this is one oversized body, + # not a mask that bled across the table. Leave it to the extent cap. + return [points] + + above = heights > 0.002 + body = points[above] if above.sum() >= policy.min_depth_points else points + + import open3d as o3d + + cloud = o3d.geometry.PointCloud() + cloud.points = o3d.utility.Vector3dVector(body) + labels = np.asarray(cloud.cluster_dbscan(eps=SPLIT_EPS_M, min_points=20)) + clusters = [ + body[labels == label] + for label in range(labels.max() + 1) + if (labels == label).sum() >= policy.min_depth_points + ] + return clusters if clusters else [body] + + +def _pixel_bbox( + points: np.ndarray, camera_info: CameraInfo, transform: Any +) -> tuple[float, float, float, float]: + """Project world points back into the frame for a sub-observation's bbox.""" + matrix = transform.to_matrix() + optical = (matrix[:3, :3] @ points.T).T + matrix[:3, 3] + z = np.maximum(optical[:, 2], 1e-6) + fx, fy = camera_info.K[0], camera_info.K[4] + cx, cy = camera_info.K[2], camera_info.K[5] + u = fx * optical[:, 0] / z + cx + v = fy * optical[:, 1] / z + cy + return (float(u.min()), float(v.min()), float(u.max()), float(v.max())) + + +def _lift_frame( + detections_2d: Any, + store: Any, + tf: TFLookup, + camera_info: CameraInfo, + obs_ts: float, + camera_position: np.ndarray, + policy: InventoryPolicy, + optical_frame: str, + world_frame: str, + tf_tolerance: float, + plane: SupportPlane | None = None, +) -> tuple[list[SupportObservation], list[Detection2DSeg]]: + """Depth-lift accepted proposals of one frame; returns (grounded, ungrounded).""" + from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 + + depth = gates.depth_at(store, obs_ts) + transform = tf.get(optical_frame, world_frame, obs_ts, tf_tolerance) + if depth is None or transform is None: + return [], list(detections_2d) + + lifted = ImageDetections3DPC.from_depth(detections_2d, depth, camera_info, transform) + + grounded: list[SupportObservation] = [] + ungrounded: list[Detection2DSeg] = [] + lifted_by_track = {det3d.track_id: det3d for det3d in lifted} + for det2d in detections_2d: + det3d = lifted_by_track.get(det2d.track_id) + if det3d is None or len(det3d.pointcloud) < policy.min_depth_points: + ungrounded.append(det2d) + continue + points = np.asarray(det3d.pointcloud.pointcloud.points) + mask_area = int((det2d.mask > 0).sum()) + for piece in _split_oversized(points, plane, policy): + aabb_min, aabb_max = piece.min(axis=0), piece.max(axis=0) + extent = aabb_max - aabb_min + if float(extent.max()) > policy.max_object_extent_m: + continue + ranges = np.linalg.norm(piece - camera_position, axis=1) + if float(np.median(ranges)) < policy.min_camera_range_m: + continue + whole = len(piece) == len(points) + grounded.append( + SupportObservation( + ts=obs_ts, + cloud=det3d.pointcloud + if whole + else PointCloud2.from_numpy(piece, frame_id="world", timestamp=obs_ts), + centroid=piece.mean(axis=0), + aabb_min=aabb_min, + aabb_max=aabb_max, + n_points=len(piece), + mask_area_px=mask_area if whole else int(mask_area * len(piece) / len(points)), + camera_position=camera_position, + bbox=det2d.bbox if whole else _pixel_bbox(piece, camera_info, transform), + ) + ) + return grounded, ungrounded + + +def _in_scope(obs: SupportObservation, plane: SupportPlane | None, policy: InventoryPolicy) -> bool: + """Support-plane scope: in a band above the plane, footprint on the workspace.""" + if policy.in_scope is not None: + points = np.asarray(obs.cloud.pointcloud.points) + return bool(policy.in_scope(points)) + if plane is None: + return True + points = np.asarray(obs.cloud.pointcloud.points) + heights = plane.height_above(points) + low, high = float(np.quantile(heights, 0.05)), float(np.quantile(heights, 0.95)) + band_lo, band_hi = policy.band_above_plane_m + if low < band_lo or high > band_hi: + return False + if not policy.include_surfaces and high < policy.min_height_above_plane_m: + # A patch of the surface itself: no volume above the plane. Tape + # lines and wood-grain segments die here; every real object rises. + return False + inside = plane.footprint_contains(points[:, :2]) + return bool(inside.mean() >= 0.3) + + +def _aabb_gap(a: SupportObservation, b: SupportObservation) -> float: + """Largest per-axis separation between two world AABBs (0 when touching).""" + gap = np.maximum(a.aabb_min - b.aabb_max, b.aabb_min - a.aabb_max) + return float(gap.max()) + + +def _cloud_gap(a: SupportObservation, b: SupportObservation) -> float: + """Minimum point-to-point distance between two observation clouds. + + The AABB gap is a poor contact test for diagonal objects - an + axis-aligned box overhangs its object's true footprint and "touches" + neighbors that are centimeters of clear table away. Actual cloud + distance is the physical claim. The AABB test remains as a cheap + prefilter. + """ + if _aabb_gap(a, b) > 0.06: + return np.inf + from scipy.spatial import cKDTree + + pa = np.asarray(a.cloud.pointcloud.points) + pb = np.asarray(b.cloud.pointcloud.points) + pa = pa[:: max(1, len(pa) // 800)] + pb = pb[:: max(1, len(pb) // 800)] + distances, _ = cKDTree(pa).query(pb, k=1) + return float(distances.min()) + + +def _absorb_into(target: SupportObservation, obs: SupportObservation) -> None: + target.aabb_min = np.minimum(target.aabb_min, obs.aabb_min) + target.aabb_max = np.maximum(target.aabb_max, obs.aabb_max) + target.cloud = target.cloud + obs.cloud + points = np.asarray(target.cloud.pointcloud.points) + target.centroid = points.mean(axis=0) + target.n_points = len(points) + target.mask_area_px = max(target.mask_area_px, obs.mask_area_px) + + +def _merge_same_frame( + observations: list[SupportObservation], policy: InventoryPolicy +) -> list[SupportObservation]: + """Fuse same-frame proposals that are one physical support. + + The criterion is contact: two same-frame observations whose clouds + touch (minimum cloud distance within the gap) are one rigid body - + duplicates, whole-and-part pairs, and split halves of one object all + satisfy it, while distinct objects on the workspace, identical twins + included, sit farther apart than the gap. Runs to a fixed point so + chains of touching pieces collapse into one support. + """ + if policy.include_object_parts: + return observations + items = sorted(observations, key=lambda o: -o.n_points) + changed = True + while changed: + changed = False + for i in range(len(items)): + for j in range(i + 1, len(items)): + if _cloud_gap(items[i], items[j]) <= policy.same_frame_merge_gap_m: + _absorb_into(items[i], items[j]) + items.pop(j) + changed = True + break + if changed: + break + return items + + +def _associate( + frames: list[tuple[float, list[SupportObservation]]], policy: InventoryPolicy +) -> list[_Track]: + """Hard constraints first, geometric score second, Hungarian for the residual. + + Per frame, observations assign one-to-one to existing tracks - the + same-frame constraint is structural, no score overrides it. A pair is + forbidden outright (infinite cost) when the supports are farther apart + than the search radius, their envelopes do not overlap enough, or their + sizes are incompatible beyond measurement error. + """ + from scipy.optimize import linear_sum_assignment + + forbidden = 1e6 + tracks: list[_Track] = [] + for frame_key, observations in frames: + if not observations: + continue + if not tracks: + for obs in observations: + track = _Track() + track.add(obs, frame_key) + tracks.append(track) + continue + + cost = np.full((len(observations), len(tracks)), forbidden) + for i, obs in enumerate(observations): + for j, track in enumerate(tracks): + distance = float(np.linalg.norm(obs.centroid - track.centroid)) + if distance > policy.search_radius_m: + continue + t_lo, t_hi = track.aabb + size_gap = np.abs((t_hi - t_lo) - (obs.aabb_max - obs.aabb_min)) + if float(size_gap.max()) > 0.25: + continue + overlap = aabb_overlap( + obs.aabb_min, obs.aabb_max, t_lo, t_hi, pad=policy.envelope_pad_m + ) + if overlap < policy.overlap_accept: + continue + cost[i, j] = 1.0 - overlap + + rows, cols = linear_sum_assignment(cost) + assigned = {} + for i, j in zip(rows, cols, strict=False): + if cost[i, j] < forbidden: + assigned[i] = j + for i, obs in enumerate(observations): + j = assigned.get(i) + if j is not None: + tracks[j].add(obs, frame_key) + else: + track = _Track() + track.add(obs, frame_key) + tracks.append(track) + return tracks + + +def _tracks_are_fragments(a: _Track, b: _Track, policy: InventoryPolicy) -> bool: + """Co-observed tracks that were touching whenever seen together. + + The same-frame veto keeps coexisting objects apart, but a mask split can + put two pieces of one object into the same frame with a cloud gap just + over the per-frame merge threshold, locking a permanent duplicate. Two + rigid objects cannot occupy one volume: when the supports interpenetrate + at containment level and every shared frame shows the pair in contact, + they are pieces of one body. Identical twins never satisfy this - their + supports do not overlap at all. + """ + a_lo, a_hi = a.aabb + b_lo, b_hi = b.aabb + overlap = aabb_overlap(a_lo, a_hi, b_lo, b_hi, pad=policy.envelope_pad_m) + if overlap < 0.5: + return False + shared = a.frame_ts & b.frame_ts + for ts in shared: + pairs_gap = min( + _cloud_gap(ma, mb) for ma in a.members if ma.ts == ts for mb in b.members if mb.ts == ts + ) + if pairs_gap > 1.5 * policy.same_frame_merge_gap_m: + return False + return True + + +def _merge_tracks(tracks: list[_Track], policy: InventoryPolicy) -> list[_Track]: + """Collapse fragmented tracks of one support. + + Tracks merge when they never share a frame (the same-frame veto at + instance level) and their supports overlap within the envelope - or when + they do share frames but were demonstrably pieces of one body in every + one of them. Runs to a fixed point. + """ + changed = True + while changed: + changed = False + for i in range(len(tracks)): + for j in range(i + 1, len(tracks)): + a, b = tracks[i], tracks[j] + if float(np.linalg.norm(a.centroid - b.centroid)) > policy.search_radius_m: + continue + if a.frame_ts & b.frame_ts: + if not _tracks_are_fragments(a, b, policy): + continue + else: + a_lo, a_hi = a.aabb + b_lo, b_hi = b.aabb + overlap = aabb_overlap(a_lo, a_hi, b_lo, b_hi, pad=policy.envelope_pad_m) + if overlap < policy.overlap_accept: + continue + for obs in b.members: + a.add(obs, obs.ts) + tracks.pop(j) + changed = True + break + if changed: + break + return tracks + + +def _track_ungrounded( + frames: list[tuple[float, list[Detection2DSeg]]], +) -> list[_Track2D]: + """Greedy 2D IoU association for detections that never produced depth.""" + tracks: list[_Track2D] = [] + for frame_key, detections in frames: + for det in detections: + best, best_iou = None, UNGROUNDED_TRACK_IOU + for track in tracks: + if frame_key in track.frame_ts: + continue + iou = _bbox_iou(det.bbox, track.members[-1].bbox) + if iou > best_iou: + best, best_iou = track, iou + if best is None: + best = _Track2D() + tracks.append(best) + best.members.append(det) + best.frame_ts.add(frame_key) + return [t for t in tracks if len(t.members) >= 2] + + +def _view_coverage(members: list[SupportObservation]) -> tuple[float, tuple[bool, bool, bool]]: + """Azimuth-bin coverage of the viewpoints and which world axes were observed.""" + if not members: + return 0.0, (False, False, False) + centroid = np.median(np.stack([m.centroid for m in members]), axis=0) + directions = [] + for m in members: + v = m.camera_position - centroid + norm = np.linalg.norm(v) + if norm > 1e-6: + directions.append(v / norm) + if not directions: + return 0.0, (False, False, False) + dirs = np.stack(directions) + azimuth = np.arctan2(dirs[:, 1], dirs[:, 0]) + bins = set(((azimuth + np.pi) / (2 * np.pi) * 8).astype(int) % 8) + coverage = len(bins) / 8.0 + observed = tuple(bool((np.abs(dirs[:, i]) > 0.3).any()) for i in range(3)) + return coverage, observed # type: ignore[return-value] + + +def _build_instance(index: int, track: _Track, grounded: bool = True) -> Instance: + labels = tuple(sorted(track.labels.items(), key=lambda kv: -kv[1])) + primary = labels[0][0] if labels else None + latest = track.latest + coverage, axes_observed = _view_coverage(track.members) + lo, hi = track.aabb + extent = np.maximum(hi - lo, 0.005) + centroids = np.stack([m.centroid for m in track.members]) + sigma = centroids.std(axis=0) if len(track.members) > 1 else np.full(3, 0.01) + support = Support( + center_xyz=tuple(float(v) for v in (lo + hi) / 2), + extent_xyz_m=tuple(float(v) for v in extent), + orientation_xyzw=(0.0, 0.0, 0.0, 1.0), + sigma_xyz_m=tuple(float(v) for v in sigma), + coverage=coverage, + axes_observed=axes_observed, + frame_id="world", + ) + distinct_views = len({tuple(np.round(m.camera_position, 2)) for m in track.members}) + return Instance( + instance_id=f"obj-{index:02d}", + grounded=grounded, + primary_label=primary, + labels=labels, + state="active", + identity_confidence=min(1.0, distinct_views / 3.0), + support=support, + latest_position_xyz=tuple(float(v) for v in latest.centroid), + latest_seen_ts=latest.ts, + members=track.members, + ) + + +def _naming_picks(track: _Track) -> list[SupportObservation]: + """Members to name on: the largest view, then maximal viewpoint spread. + + Picking by mask area alone selects near-duplicate views when the sweep + keeps returning to one vantage; the label then hinges on a single + viewing angle. Greedy farthest-point selection over camera positions + guarantees the close-up passes participate. + """ + if len(track.members) <= NAME_FRAMES_PER_INSTANCE: + return list(track.members) + picks = [max(track.members, key=lambda m: m.mask_area_px)] + remaining = [m for m in track.members if m is not picks[0]] + while len(picks) < NAME_FRAMES_PER_INSTANCE and remaining: + best = max( + remaining, + key=lambda m: min( + float(np.linalg.norm(m.camera_position - p.camera_position)) for p in picks + ), + ) + picks.append(best) + remaining.remove(best) + return picks + + +def _name_and_suppress( + tracks: list[_Track], + tracks_2d: list[_Track2D], + store: Any, +) -> None: + """OWLv2 naming per instance on keyframes, person/hand suppressing observations. + + Runs after association by construction: association consumed unnamed + supports, so per-view label instability cannot starve existence or split + an instance. A naming failure degrades names, never counts. + """ + from dimos.perception.detection.detectors.owlv2 import Owlv2Detector + + frame_members: dict[float, list[tuple[_Track, SupportObservation]]] = {} + for track in tracks: + for member in _naming_picks(track): + frame_members.setdefault(member.ts, []).append((track, member)) + frame_members_2d: dict[float, list[tuple[_Track2D, Detection2DSeg]]] = {} + for track2d in tracks_2d: + for det in sorted(track2d.members, key=lambda d: -(d.mask > 0).sum())[:2]: + frame_members_2d.setdefault(det.ts, []).append((track2d, det)) + + if not frame_members and not frame_members_2d: + return + + owl = Owlv2Detector() + chunks = [ + GENERIC_VOCABULARY[i : i + NAME_PROMPT_CAP] + for i in range(0, len(GENERIC_VOCABULARY), NAME_PROMPT_CAP) + ] + chunks.append(list(SUPPRESS_QUERIES)) + suppress_set = set(SUPPRESS_QUERIES) + all_ts = sorted(set(frame_members) | set(frame_members_2d)) + logger.info( + f"naming: OWLv2 over {len(all_ts)} keyframes, " + f"{len(chunks)} prompts of <= {NAME_PROMPT_CAP} classes" + ) + for ts in all_ts: + try: + image = store.streams.color_image.at(ts, 0.05).first().data + except LookupError: + continue + detections = [ + det + for chunk in chunks + for det in owl.query_detections(image, chunk, threshold=NAME_SCORE_FLOOR) + ] + for det in detections: + if det.name in suppress_set: + if det.confidence < SUPPRESS_SCORE: + continue + for track, member in frame_members.get(ts, []): + if member.bbox is None: + continue + inside = _mask_overlap_fraction_bbox(member.bbox, det.bbox) + if inside >= SUPPRESS_OVERLAP and member in track.members: + track.members.remove(member) + continue + best_target: Any = None + best_iou = NAME_ATTACH_IOU + for track, member in frame_members.get(ts, []): + if member.bbox is None: + continue + iou = _bbox_iou(member.bbox, det.bbox) + if iou > best_iou: + best_target, best_iou = track, iou + for track2d, det2d in frame_members_2d.get(ts, []): + iou = _bbox_iou(det2d.bbox, det.bbox) + if iou > best_iou: + best_target, best_iou = track2d, iou + if best_target is not None: + previous = best_target.labels.get(det.name, 0.0) + best_target.labels[det.name] = max(previous, det.confidence) + owl.stop() + + +def _mask_overlap_fraction_bbox( + member_box: tuple[float, float, float, float], region: tuple[float, float, float, float] +) -> float: + """Fraction of the member box inside the region box.""" + mx1, my1, mx2, my2 = member_box + rx1, ry1, rx2, ry2 = region + ix = max(0.0, min(mx2, rx2) - max(mx1, rx1)) + iy = max(0.0, min(my2, ry2) - max(my1, ry1)) + area = (mx2 - mx1) * (my2 - my1) + return (ix * iy) / area if area > 0 else 0.0 + + +def inventory( + store: Any, + *, + after: float | None = None, + before: float | None = None, + include_ungrounded: bool = False, + policy: InventoryPolicy | None = None, + motion_threshold: float = MOTION_THRESHOLD, + log_progress: bool = False, + world_frame: str = WORLD_FRAME, + optical_frame: str = OPTICAL_FRAME, + tf_tolerance: float = TF_TOLERANCE, +) -> list[Instance]: + """Deduplicated object instances for the window, computed at query time. + + Reports the scene as of the window's end: an instance's position and + timestamp come from its latest member observation in this call. An + object that moved between rest positions inside the window registers + once per rest position; linking rest positions of one object is + cross-time identity and out of scope here. + + ``log_progress`` enables per-keyframe discovery lines + (``discovery: i/n ts_offset=… prop=… scope=… …s``). Off by default. + """ + policy = policy or InventoryPolicy() + tf = StreamTF.from_store(store) + if tf is None: + raise ValueError("recording has no tf stream") + camera_info = store.streams.camera_info.first().data + lo, hi = store.streams.color_image.get_time_range() + t0 = after if after is not None else lo + t1 = before if before is not None else hi + logger.info(f"inventory window: {t0 - lo:.1f}s to {t1 - lo:.1f}s ({t1 - t0:.1f}s)") + + intervals = gates.still_intervals(tf, t0, t1, optical_frame, world_frame, tf_tolerance) + gray: dict[float, Any] = {} + + keyframes = gates.keyframes( + store, + tf, + t0, + t1, + KEYFRAME_STRIDE, + intervals, + gray, + motion_threshold, + optical_frame, + world_frame, + tf_tolerance, + ) + logger.info(f"gates: {len(keyframes)} keyframes pass camera-still + scene-still") + if not keyframes: + return [] + + plane = fit_support_plane( + store, tf, camera_info, keyframes, optical_frame, world_frame, tf_tolerance + ) + if plane is not None: + logger.info(f"support plane: {plane.inlier_count} inliers") + + from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter + + segmenter = EdgeTAMImageSegmenter() + frames_grounded: list[tuple[float, list[SupportObservation]]] = [] + frames_ungrounded: list[tuple[float, list[Detection2DSeg]]] = [] + image_area = float(camera_info.width * camera_info.height) + + from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D + + n_kf = len(keyframes) + for i, obs in enumerate(keyframes, start=1): + t_frame = perf_counter() if log_progress else 0.0 + proposals = segmenter.propose_all(obs.data) + accepted = [det for det in proposals if _proposal_passes_2d(det, image_area, policy)] + accepted = sorted(accepted, key=lambda d: -(d.mask > 0).sum())[:MAX_PROPOSALS_PER_FRAME] + if not accepted: + if log_progress: + logger.info( + f"discovery: {i}/{n_kf} ts_offset={obs.ts - lo:.1f}s " + f"prop={len(proposals)}->0 scope=0 {perf_counter() - t_frame:.1f}s" + ) + continue + + pose = gates.camera_pose(tf, obs.ts, optical_frame, world_frame, tf_tolerance) + if pose is None: + if log_progress: + logger.info( + f"discovery: {i}/{n_kf} ts_offset={obs.ts - lo:.1f}s " + f"skip=no_pose {perf_counter() - t_frame:.1f}s" + ) + continue + camera_position = np.array([pose.position.x, pose.position.y, pose.position.z]) + + for j, det in enumerate(accepted): + det.track_id = j + grounded, ungrounded = _lift_frame( + ImageDetections2D(obs.data, accepted), + store, + tf, + camera_info, + obs.ts, + camera_position, + policy, + optical_frame, + world_frame, + tf_tolerance, + plane, + ) + grounded = [o for o in grounded if _in_scope(o, plane, policy)] + grounded = _merge_same_frame(grounded, policy) + frames_grounded.append((obs.ts, grounded)) + frames_ungrounded.append((obs.ts, ungrounded)) + if log_progress: + logger.info( + f"discovery: {i}/{n_kf} ts_offset={obs.ts - lo:.1f}s " + f"prop={len(proposals)}->{len(accepted)} scope={len(grounded)} " + f"{perf_counter() - t_frame:.1f}s" + ) + + del segmenter + _free_accelerator() + + total = sum(len(g) for _, g in frames_grounded) + logger.info(f"discovery: {total} in-scope supports across {len(frames_grounded)} keyframes") + + tracks = _associate(frames_grounded, policy) + tracks = _merge_tracks(tracks, policy) + tracks_2d = _track_ungrounded(frames_ungrounded) if include_ungrounded else [] + logger.info(f"association: {len(tracks)} grounded instances") + + _name_and_suppress(tracks, tracks_2d, store) + tracks = [t for t in tracks if len(t.members) >= policy.min_member_observations] + + tracks.sort(key=lambda t: min(m.ts for m in t.members)) + instances: list[Instance] = [] + unknown = 0 + for index, track in enumerate(tracks): + instance = _build_instance(index, track) + if instance.primary_label is None: + instance.primary_label = f"unknown-{unknown}" + unknown += 1 + instances.append(instance) + + if include_ungrounded: + for track2d in tracks_2d: + labels = tuple(sorted(track2d.labels.items(), key=lambda kv: -kv[1])) + primary = labels[0][0] if labels else None + if primary is None: + primary = f"unknown-{unknown}" + unknown += 1 + latest = max(track2d.members, key=lambda d: d.ts) + instances.append( + Instance( + instance_id=f"obj-{len(instances):02d}", + grounded=False, + primary_label=primary, + labels=labels, + state="active", + identity_confidence=0.3, + support=None, + latest_position_xyz=None, + latest_seen_ts=latest.ts, + members=[], + ) + ) + return instances + + +def _free_accelerator() -> None: + import gc + + import torch + + gc.collect() + if torch.cuda.is_available(): + torch.cuda.empty_cache() + elif torch.backends.mps.is_available(): + torch.mps.empty_cache() diff --git a/dimos/perception/memory/localize.py b/dimos/perception/memory/localize.py new file mode 100644 index 0000000000..c8e3fd3de2 --- /dev/null +++ b/dimos/perception/memory/localize.py @@ -0,0 +1,538 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Query-time object localization: text prompt to latest 3D pose and cloud. + +Search memory with embeddings (SigLIP, +frame-level), open-vocabulary detection (OWLv2, calibrated per-box scores), +segmentation (EdgeTAM), projection to 3D through aligned depth. Two +algorithm rules distinguish it from a best-crop search: + +* **Latest-pose semantics.** Among verified observations of the chosen + support, the greatest timestamp wins. The answer is "where is it now", + never "where did it match best". +* **Calibrated refusal.** Every stage carries a score and the answer can be + ``None``: no accept-level detection, no multi-view confirmation, or an + ambiguity between coexisting candidates below the refusal margin. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +import math +from typing import TYPE_CHECKING, Any + +import numpy as np + +from dimos.memory2.embed import EmbedImages +from dimos.memory2.tf import StreamTF +from dimos.memory2.transform import throttle +from dimos.perception.detection.project import sees +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D +from dimos.perception.detection.type.detection3d.imageDetections3DPC import ImageDetections3DPC +from dimos.perception.memory import gates +from dimos.perception.memory.gates import OPTICAL_FRAME, TF_TOLERANCE, WORLD_FRAME +from dimos.perception.memory.types import Localization, LocalizePolicy, Support +from dimos.utils.logging_config import setup_logger + +if TYPE_CHECKING: + from dimos_lcm.sensor_msgs import CameraInfo + + from dimos.memory2.stream import Stream + from dimos.perception.detection.type.detection3d.pointcloud import Detection3DPC + from dimos.protocol.tf.tf import TFLookup + +logger = setup_logger() + +EMBED_HZ = 1.0 +TOP_FRAMES = 12 +TIME_BANDS = 6 # stratify retrieval across the window so late scans always compete +BOXES_PER_FRAME = 4 +CONFIRM_FLOOR = 0.22 # geometric confirmation accept for re-detections +VERIFY_FRAMES = 20 + + +@dataclass +class _ClusterObservation: + ts: float + score: float + centroid: np.ndarray + cloud: Any + camera_position: np.ndarray + detection: Detection3DPC + + +@dataclass +class _Cluster: + center: np.ndarray + observations: list[_ClusterObservation] = field(default_factory=list) + + def add(self, obs: _ClusterObservation) -> None: + self.observations.append(obs) + self.center = np.mean(np.stack([o.centroid for o in self.observations]), axis=0) + + @property + def max_score(self) -> float: + return max(o.score for o in self.observations) + + @property + def latest(self) -> _ClusterObservation: + return max(self.observations, key=lambda o: o.ts) + + @property + def interval(self) -> tuple[float, float]: + times = [o.ts for o in self.observations] + return min(times), max(times) + + @property + def n_views(self) -> int: + return len({tuple(np.round(o.camera_position, 2)) for o in self.observations}) + + @property + def extent(self) -> np.ndarray: + points = np.concatenate([np.asarray(o.cloud.pointcloud.points) for o in self.observations]) + return points.max(axis=0) - points.min(axis=0) + + +@dataclass +class LocalizeTrace: + """Intermediate artifacts collected for rendering; filled when passed in.""" + + detection_frames: list[Any] = field(default_factory=list) # Observation[ImageDetections2D] + matched: list[tuple[float, Detection3DPC]] = field(default_factory=list) + verified: list[tuple[float, Detection3DPC]] = field(default_factory=list) + answer: Detection3DPC | None = None + backdrop_ts: float | None = None + + +def _quaternion_from_matrix(rotation: np.ndarray) -> tuple[float, float, float, float]: + from scipy.spatial.transform import Rotation + + x, y, z, w = Rotation.from_matrix(rotation).as_quat() + return (float(x), float(y), float(z), float(w)) + + +def _azimuth_coverage(observations: list[_ClusterObservation], center: np.ndarray) -> float: + directions = [] + for obs in observations: + v = obs.camera_position - center + norm = np.linalg.norm(v) + if norm > 1e-6: + directions.append(v / norm) + if not directions: + return 0.0 + dirs = np.stack(directions) + azimuth = np.arctan2(dirs[:, 1], dirs[:, 0]) + bins = set(((azimuth + math.pi) / (2 * math.pi) * 8).astype(int) % 8) + return len(bins) / 8.0 + + +def _axes_observed( + observations: list[_ClusterObservation], center: np.ndarray +) -> tuple[bool, bool, bool]: + directions = [] + for obs in observations: + v = obs.camera_position - center + norm = np.linalg.norm(v) + if norm > 1e-6: + directions.append(v / norm) + if not directions: + return (False, False, False) + dirs = np.stack(directions) + return tuple(bool((np.abs(dirs[:, i]) > 0.3).any()) for i in range(3)) # type: ignore[return-value] + + +class _DetectionCache: + """One OWLv2 + EdgeTAM pass per unique frame, shared across clusters.""" + + def __init__(self, owl: Any, segmenter: Any, query: str, floor: float) -> None: + self.owl = owl + self.segmenter = segmenter + self.query = query + self.floor = floor + self._cache: dict[float, ImageDetections2D] = {} + + def detect(self, image: Any) -> ImageDetections2D: + key = image.ts + hit = self._cache.get(key) + if hit is not None: + return hit + detections = self.owl.query_detections(image, [self.query], threshold=self.floor) + detections = ImageDetections2D( + image, + sorted(detections.detections, key=lambda d: -d.confidence)[:BOXES_PER_FRAME], + ) + if len(detections): + detections = self.segmenter.segment(detections) + self._cache[key] = detections + return detections + + +def _lift( + detections: ImageDetections2D, + store: Any, + tf: TFLookup, + camera_info: CameraInfo, + optical_frame: str, + world_frame: str, + tf_tolerance: float, + policy: LocalizePolicy, + plane: Any | None = None, +) -> list[tuple[Detection3DPC, np.ndarray]]: + """Depth-lift 2D detections; returns valid (detection3d, camera_position) pairs.""" + depth = gates.depth_at(store, detections.ts) + transform = tf.get(optical_frame, world_frame, detections.ts, tf_tolerance) + if depth is None or transform is None: + return [] + pose = gates.camera_pose(tf, detections.ts, optical_frame, world_frame, tf_tolerance) + if pose is None: + return [] + camera = np.array([pose.position.x, pose.position.y, pose.position.z]) + + lifted = ImageDetections3DPC.from_depth(detections, depth, camera_info, transform) + valid: list[tuple[Detection3DPC, np.ndarray]] = [] + for det3d in lifted: + points = np.asarray(det3d.pointcloud.pointcloud.points) + if len(points) < policy.min_depth_points: + continue + extent = points.max(axis=0) - points.min(axis=0) + if float(extent.max()) > policy.max_object_extent_m: + continue + ranges = np.linalg.norm(points - camera, axis=1) + if float(np.median(ranges)) < policy.min_camera_range_m: + continue + if plane is not None: + heights = plane.height_above(points) + low = float(np.quantile(heights, 0.05)) + high = float(np.quantile(heights, 0.95)) + if low > policy.surface_patch_min_drop_m and high < policy.surface_patch_max_rise_m: + continue + valid.append((det3d, camera)) + return valid + + +def _embed_index( + store: Any, + tf: TFLookup, + t0: float, + t1: float, + siglip: Any, + optical_frame: str, + world_frame: str, + tf_tolerance: float, +) -> Stream[Any, Any]: + """SigLIP-embedded, world-posed frame index at EMBED_HZ over the window.""" + posed = ( + store.streams.color_image.after(t0) + .before(t1) + .transform(throttle(1.0 / EMBED_HZ)) + .map( + lambda obs: obs.derive( + data=obs.data, + pose=gates.camera_pose(tf, obs.ts, optical_frame, world_frame, tf_tolerance), + ) + ) + .filter(lambda obs: obs.pose is not None) + ) + return posed.transform(EmbedImages(siglip)).materialize() + + +def _retrieve( + index: Stream[Any, Any], + tf: TFLookup, + query_embedding: Any, + t0: float, + t1: float, + optical_frame: str, + world_frame: str, + tf_tolerance: float, +) -> list[Any]: + """Top still frames by text similarity, stratified over time bands. + + Stratification is what keeps latest-pose semantics honest at the + candidate stage: the top frames of the whole window may all be early, + and a support that only exists late must still get a detection pass. + """ + ranked = [ + obs + for obs in index.search(query_embedding, k=max(index.count(), 1)) + if gates.camera_still(tf, obs.ts, optical_frame, world_frame, tf_tolerance) + ] + if not ranked: + return [] + + bands = max(1, min(TIME_BANDS, int((t1 - t0) / 20))) + per_band = max(1, TOP_FRAMES // bands) + span = (t1 - t0) / bands + selected: list[Any] = [] + chosen: set[float] = set() + for band in range(bands): + band_lo = t0 + band * span + band_hi = band_lo + span + in_band = [obs for obs in ranked if band_lo <= obs.ts < band_hi] + for obs in in_band[:per_band]: + if obs.ts not in chosen: + chosen.add(obs.ts) + selected.append(obs) + for obs in ranked: # fill remaining budget by global rank + if len(selected) >= TOP_FRAMES: + break + if obs.ts not in chosen: + chosen.add(obs.ts) + selected.append(obs) + return selected + + +def localize( + store: Any, + query: str, + *, + after: float | None = None, + before: float | None = None, + require_pose: bool = True, + policy: LocalizePolicy | None = None, + cloud_mode: str = "latest_visible", + world_frame: str = WORLD_FRAME, + optical_frame: str = OPTICAL_FRAME, + tf_tolerance: float = TF_TOLERANCE, + trace: LocalizeTrace | None = None, +) -> Localization | None: + """Latest unambiguous 3D localization of *query*, or ``None``. + + ``None`` is a first-class answer: nothing reached the accept score, no + support was confirmed from a second viewpoint, or the best candidate had + no valid depth and ``require_pose`` holds. An ambiguity between + coexisting candidates is returned with ``ambiguity_margin`` below + ``refusal_margin`` - a flagged hit, never a silent guess. + """ + policy = policy or LocalizePolicy() + tf = StreamTF.from_store(store) + if tf is None: + raise ValueError("recording has no tf stream") + camera_info = store.streams.camera_info.first().data + lo, hi = store.streams.color_image.get_time_range() + t0 = after if after is not None else lo + t1 = before if before is not None else hi + logger.info(f"localize '{query}': window {t0 - lo:.1f}s..{t1 - lo:.1f}s") + + # Pass 1 - SigLIP resident: embed the window, rank frames by the query. + from dimos.models.embedding.siglip import SigLIPModel + + siglip = SigLIPModel() + index = _embed_index(store, tf, t0, t1, siglip, optical_frame, world_frame, tf_tolerance) + query_embedding = siglip.embed_text(query) + frames = _retrieve(index, tf, query_embedding, t0, t1, optical_frame, world_frame, tf_tolerance) + siglip.stop() + logger.info(f"retrieval: {len(frames)} candidate frames of {index.count()} embedded") + if not frames: + return None + + from dimos.perception.memory.support_plane import fit_support_plane + + plane = fit_support_plane( + store, tf, camera_info, frames, optical_frame, world_frame, tf_tolerance + ) + + # Pass 2 - OWLv2 + EdgeTAM resident: detect, segment, lift, verify. + from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter + from dimos.perception.detection.detectors.owlv2 import Owlv2Detector + + owl = Owlv2Detector() + segmenter = EdgeTAMImageSegmenter() + cache = _DetectionCache(owl, segmenter, query, policy.candidate_floor) + + clusters: list[_Cluster] = [] + ungrounded_best: tuple[float, float] | None = None # (score, ts) + processed: set[float] = set() + + def _absorb(frame_obs: Any, is_verify: bool) -> None: + nonlocal ungrounded_best + if frame_obs.ts in processed: + return + processed.add(frame_obs.ts) + detections = cache.detect(frame_obs.data) + if not len(detections): + return + if trace is not None and not is_verify: + trace.detection_frames.append(frame_obs.derive(data=detections)) + lifted = _lift( + detections, + store, + tf, + camera_info, + optical_frame, + world_frame, + tf_tolerance, + policy, + plane, + ) + for det2d in detections: + if not any(d.track_id == det2d.track_id for d, _ in lifted): + best = (det2d.confidence, det2d.ts) + if ungrounded_best is None or best[0] > ungrounded_best[0]: + ungrounded_best = best + for det3d, camera in lifted: + observation = _ClusterObservation( + ts=det3d.ts, + score=det3d.confidence, + centroid=np.asarray(det3d.pointcloud.pointcloud.points).mean(axis=0), + cloud=det3d.pointcloud, + camera_position=camera, + detection=det3d, + ) + if trace is not None: + (trace.verified if is_verify else trace.matched).append((det3d.ts, det3d)) + for cluster in clusters: + distance = float(np.linalg.norm(observation.centroid - cluster.center)) + if distance <= policy.cluster_radius_m: + cluster.add(observation) + break + else: + clusters.append(_Cluster(center=observation.centroid, observations=[observation])) + + for frame_obs in frames: + _absorb(frame_obs, is_verify=False) + logger.info(f"detection: {len(clusters)} support candidates") + + # Cross-view verification: a support seen from one pose only is + # unconfirmed. Frames whose camera could observe the support are found + # geometrically (near + sees with occlusion), then re-detected. + clusters.sort(key=lambda c: -c.max_score) + for cluster in list(clusters[:4]): + predicate = sees( + cluster.center, + camera_info, + tf=tf, + world_frame=world_frame, + optical_frame=optical_frame, + time_tolerance=tf_tolerance, + extent=np.minimum(cluster.extent, 0.4), + # A large object overflows close-up frames; a third of its box in + # view is still a usable re-detection pass, and those close-ups + # are exactly the distinct viewpoints verification needs. + min_fraction=0.35, + depth=lambda obs: gates.depth_at(store, obs.ts), + max_range=1.6, + ) + observing = [ + obs + for obs in index.near(cluster.center, radius=1.6) + if obs.ts not in processed + and gates.camera_still(tf, obs.ts, optical_frame, world_frame, tf_tolerance) + and predicate(obs) + ] + if len(observing) > VERIFY_FRAMES: + # Even spread that always includes the endpoints: dropping the + # latest seeing frames would bias the latest-pose answer early. + picks = np.unique(np.linspace(0, len(observing) - 1, VERIFY_FRAMES).astype(int)) + observing = [observing[i] for i in picks] + for frame_obs in observing: + _absorb(frame_obs, is_verify=True) + + verified = [ + c for c in clusters if c.max_score >= policy.accept_score and c.n_views >= policy.min_views + ] + logger.info( + "verification: " + + ", ".join( + f"score={c.max_score:.2f} views={c.n_views} obs={len(c.observations)}" for c in clusters + ) + ) + + owl.stop() + del segmenter + + if not verified: + if ungrounded_best is not None and ungrounded_best[0] >= policy.accept_score: + if require_pose: + logger.info("best candidate has no valid depth and require_pose is set") + return None + score, ts = ungrounded_best + return Localization( + instance_id="query-0", + semantic_score=score, + identity_score=0.0, + ambiguity_margin=1.0, + position_world_xyz=None, + orientation_world_xyzw=None, + frame_id="world", + support=None, + pose_timestamp=ts, + geometry_timestamp=ts, + last_seen_timestamp=ts, + point_cloud=None, + cloud_mode=cloud_mode, + coverage=0.0, + n_views=1, + reason="no_valid_depth", + ) + return None + + winner = max(verified, key=lambda c: c.latest.ts) + w_lo, w_hi = winner.interval + rival_scores = [ + c.max_score + for c in verified + if c is not winner + and not (c.interval[1] < w_lo or c.interval[0] > w_hi) # coexisting in time + ] + margin = winner.max_score - max(rival_scores) if rival_scores else 1.0 + reason = "ambiguous_between_coexisting_candidates" if margin < policy.refusal_margin else None + + latest = winner.latest + points = np.asarray(latest.cloud.pointcloud.points) + aabb_min, aabb_max = points.min(axis=0), points.max(axis=0) + try: + orientation = _quaternion_from_matrix(np.asarray(latest.cloud.oriented_bounding_box.R)) + except Exception: + orientation = (0.0, 0.0, 0.0, 1.0) + support = Support( + center_xyz=tuple(float(v) for v in (aabb_min + aabb_max) / 2), + extent_xyz_m=tuple(float(v) for v in np.maximum(aabb_max - aabb_min, 0.005)), + orientation_xyzw=(0.0, 0.0, 0.0, 1.0), + sigma_xyz_m=tuple( + float(v) + for v in ( + np.stack([o.centroid for o in winner.observations]).std(axis=0) + if len(winner.observations) > 1 + else np.full(3, 0.01) + ) + ), + coverage=_azimuth_coverage(winner.observations, winner.center), + axes_observed=_axes_observed(winner.observations, winner.center), + frame_id="world", + ) + + if trace is not None: + trace.answer = latest.detection + trace.backdrop_ts = latest.ts + + return Localization( + instance_id="query-0", + semantic_score=winner.max_score, + identity_score=min(1.0, winner.n_views / 4.0), + ambiguity_margin=margin, + position_world_xyz=tuple(float(v) for v in latest.centroid), + orientation_world_xyzw=orientation, + frame_id="world", + support=support, + pose_timestamp=latest.ts, + geometry_timestamp=latest.ts, + last_seen_timestamp=latest.ts, + point_cloud=latest.cloud, + cloud_mode=cloud_mode, + coverage=support.coverage, + n_views=winner.n_views, + reason=reason, + ) diff --git a/dimos/perception/memory/support_plane.py b/dimos/perception/memory/support_plane.py new file mode 100644 index 0000000000..c4c3dead4f --- /dev/null +++ b/dimos/perception/memory/support_plane.py @@ -0,0 +1,152 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Support-surface fit and the scope predicate derived from it. + +The plane is RANSAC-fit from the window's own frames - nothing scene-specific +is passed in and no caller supplies coordinates. The plane's inlier footprint +is the workspace; the scope predicate accepts a support when its cloud sits in +a band above the plane and its footprint intersects the plane footprint. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import numpy as np + +from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 +from dimos.perception.memory import gates +from dimos.utils.logging_config import setup_logger + +if TYPE_CHECKING: + from dimos_lcm.sensor_msgs import CameraInfo + + from dimos.memory2.type.observation import Observation + from dimos.msgs.sensor_msgs.Image import Image + from dimos.protocol.tf.tf import TFLookup + +logger = setup_logger() + +BACKDROP_DEPTH_TRUNC = 1.5 # m - the workspace a wrist camera actually covers +PLANE_DISTANCE = 0.01 # m - RANSAC inlier distance +MIN_HORIZONTAL_DOT = 0.90 # |normal . z| for a plane to count as horizontal +FOOTPRINT_DILATE_M = 0.03 + + +@dataclass +class SupportPlane: + """A horizontal support surface: plane coefficients plus inlier footprint.""" + + # Plane (a, b, c, d): a*x + b*y + c*z + d = 0, normal pointing up (+z). + coefficients: tuple[float, float, float, float] + footprint_hull: np.ndarray # (K, 2) convex hull of inlier (x, y), world + inlier_count: int + + @property + def normal(self) -> np.ndarray: + return np.array(self.coefficients[:3]) + + def height_above(self, points: np.ndarray) -> np.ndarray: + """Signed height of (N, 3) world points above the plane.""" + a, b, c, d = self.coefficients + return points @ np.array([a, b, c]) + d + + def footprint_contains(self, points_xy: np.ndarray) -> np.ndarray: + """Boolean mask: which (N, 2) world XY points fall inside the (dilated) hull.""" + from matplotlib.path import Path as MplPath + + hull = self.footprint_hull + center = hull.mean(axis=0) + offsets = hull - center + norms = np.linalg.norm(offsets, axis=1, keepdims=True) + dilated = hull + offsets / np.maximum(norms, 1e-9) * FOOTPRINT_DILATE_M + return MplPath(dilated).contains_points(points_xy) + + +def fit_support_plane( + store: Any, + tf: TFLookup, + camera_info: CameraInfo, + keyframes: list[Observation[Image]], + optical_frame: str, + world_frame: str, + tf_tolerance: float, +) -> SupportPlane | None: + """Fit the dominant near-horizontal plane from a handful of window keyframes. + + Non-horizontal dominant planes (a wall, a screen) are peeled off and the + fit repeats on the remainder. Among horizontal candidates the one with + the most inliers wins - for a wrist camera over a workspace that is the + support surface itself. + """ + if not keyframes: + return None + + picks = keyframes[:: max(1, len(keyframes) // 5)][:5] + clouds = [] + for obs in picks: + depth = gates.depth_at(store, obs.ts) + transform = tf.get(optical_frame, world_frame, obs.ts, tf_tolerance) + if depth is None or transform is None: + continue + cloud = PointCloud2.from_rgbd( + obs.data, depth, camera_info, depth_scale=0.001, depth_trunc=BACKDROP_DEPTH_TRUNC + ).transform(-transform) + clouds.append(cloud.voxel_downsample(0.01)) + if not clouds: + return None + + merged = clouds[0] + for cloud in clouds[1:]: + merged = merged + cloud + points = np.asarray(merged.pointcloud.points) + if len(points) < 500: + return None + + import open3d as o3d + + remaining = o3d.geometry.PointCloud() + remaining.points = o3d.utility.Vector3dVector(points) + best: tuple[np.ndarray, np.ndarray] | None = None # (coefficients, inlier points) + for _ in range(4): + if len(remaining.points) < 500: + break + model, inlier_idx = remaining.segment_plane( + distance_threshold=PLANE_DISTANCE, ransac_n=3, num_iterations=1000 + ) + inliers = np.asarray(remaining.points)[inlier_idx] + normal = np.array(model[:3]) + if abs(normal[2]) >= MIN_HORIZONTAL_DOT: + if best is None or len(inliers) > len(best[1]): + best = (np.array(model), inliers) + remaining = remaining.select_by_index(inlier_idx, invert=True) + + if best is None: + logger.warning("support plane: no horizontal plane found in backdrop") + return None + + model, inliers = best + if model[2] < 0: # normal points up + model = -model + from scipy.spatial import ConvexHull + + xy = inliers[:, :2] + hull = ConvexHull(xy) + return SupportPlane( + coefficients=tuple(float(v) for v in model), + footprint_hull=xy[hull.vertices], + inlier_count=len(inliers), + ) diff --git a/dimos/perception/memory/tool_inventory.py b/dimos/perception/memory/tool_inventory.py new file mode 100644 index 0000000000..8aaaee0519 --- /dev/null +++ b/dimos/perception/memory/tool_inventory.py @@ -0,0 +1,95 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Enumerate deduplicated object instances in a recording window. + +Run: uv run python -m dimos.perception.memory.tool_inventory + [--from ] [--duration ] [--include-ungrounded] [--log-progress] + +Stdout contract: a summary line ``instances: N`` followed by one line per +instance: `` id= name= xyz=(x,y,z) ts_offset= +members=``. Exit code 0 whenever the call completes; an empty scene is +``instances: 0``, not a failure. + +The instance list reports the scene as of the window's end: position and +timestamp come from each instance's latest member observation. An object +moved between rest positions inside the window registers once per rest +position - linking rest positions of one object across time is +re-identification, which this tool does not do. +""" + +import argparse +from pathlib import Path +import sys + +from dimos.memory2.store.sqlite import SqliteStore +from dimos.perception.memory.inventory import inventory +from dimos.utils.data import get_data + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--dataset", type=Path, help="memory2 recording database") + parser.add_argument( + "--from", dest="start", type=float, default=0.0, help="start offset into the recording (s)" + ) + parser.add_argument("--duration", type=float, default=None, help="how much to parse (s)") + parser.add_argument( + "--include-ungrounded", + action="store_true", + help="also list RGB-only instances that never produced valid depth", + ) + parser.add_argument( + "--log-progress", + action="store_true", + help="per-keyframe discovery progress lines (off by default)", + ) + args = parser.parse_args() + + dataset = args.dataset or get_data( + "xarm6_worldbelief_realsense_d435i_stationery_calibrated/" + "xarm6_worldbelief_20260729_203624_161992.db" + ) + store = SqliteStore(path=dataset) + lo, _ = store.streams.color_image.get_time_range() + after = lo + args.start + before = lo + args.start + args.duration if args.duration is not None else None + + instances = inventory( + store, + after=after, + before=before, + include_ungrounded=args.include_ungrounded, + log_progress=args.log_progress, + ) + + print(f"instances: {len(instances)}") + for i, instance in enumerate(instances): + if instance.latest_position_xyz is not None: + x, y, z = instance.latest_position_xyz + xyz = f"({x:.3f},{y:.3f},{z:.3f})" + else: + xyz = "None" + print( + f"{i} id={instance.instance_id} name={instance.primary_label} " + f"xyz={xyz} ts_offset={instance.latest_seen_ts - lo:.1f} " + f"members={len(instance.members)}" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dimos/perception/memory/tool_localize.py b/dimos/perception/memory/tool_localize.py index 8f7de3050b..7151085ddb 100644 --- a/dimos/perception/memory/tool_localize.py +++ b/dimos/perception/memory/tool_localize.py @@ -16,178 +16,44 @@ Run: uv run python -m dimos.perception.memory.tool_localize [query] [out.rrd] [--from ] [--duration ] + +Exit code 0 with a printed position on a verified hit; exit code 1 with +"no verified detection of ..." when the honest answer is that the object is +not there. An ambiguous hit (identical twins in view) is printed with its +ambiguity margin flagged. """ import argparse from pathlib import Path - -import numpy as np - -from dimos.memory.embed import EmbedImages -from dimos.memory.store.sqlite import SqliteStore -from dimos.memory.tf import StreamTF -from dimos.memory.transform import throttle -from dimos.memory.utils.progress import progress -from dimos.models.embedding.clip import CLIPModel -from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter -from dimos.models.vl.moondream import MoondreamVlModel -from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 -from dimos.perception.detection.project import ProjectDepthTo3D, sees +import sys +from typing import Any + +from dimos.memory2.store.sqlite import SqliteStore +from dimos.memory2.tf import StreamTF +from dimos.memory2.transform import throttle +from dimos.perception.memory import gates +from dimos.perception.memory.localize import LocalizeTrace, localize from dimos.utils.data import get_data -from dimos.visualization.rerun.init import rerun_init - -parser = argparse.ArgumentParser(description=__doc__) -parser.add_argument("query", nargs="?", default="plant") -parser.add_argument("out", nargs="?", default="localize.rrd") -parser.add_argument("--dataset", type=Path, help="memory recording database") -parser.add_argument( - "--from", dest="start", type=float, default=0.0, help="start offset into the recording (s)" -) -parser.add_argument("--duration", type=float, default=None, help="how much to parse (s)") -args = parser.parse_args() -query, out = args.query, args.out - -dataset = args.dataset or get_data( - "xarm6_worldbelief_realsense_d435i_stationery_calibrated/xarm6_worldbelief_20260729_203624_161992.db" -) -store = SqliteStore(path=dataset) -tf = StreamTF.from_store(store) -camera_info = store.streams.camera_info.first().data -OPTICAL_FRAME = "camera_color_optical_frame" - -lo, hi = store.streams.color_image.get_time_range() -t0 = lo + args.start -t1 = min(hi, t0 + args.duration) if args.duration is not None else hi -images = store.streams.color_image.after(t0).before(t1) -span = t1 - t0 -print(f"window: {args.start:.0f}s → {args.start + span:.0f}s of the recording ({span:.0f}s)") - - -def camera_pose(ts): - """World pose of the camera optical frame at ts — it rides the wrist.""" - return (-tf.get(OPTICAL_FRAME, "world", ts, 0.5)).to_pose() - - -def camera_speed(ts, dt=0.06): - """Linear speed of the camera (m/s) around ts, from tf.""" - a, b = camera_pose(ts - dt), camera_pose(ts + dt) - return float((b.position - a.position).magnitude() / (2 * dt)) - - -def still(ts, envelope=0.15): - """Camera is still over the whole capture envelope, not just at ts.""" - return all(camera_speed(ts + o) <= SPEED_MAX for o in (-envelope, 0.0, envelope)) - - -# Embed only the requested window and keep the source recording read-only. -clip = CLIPModel() -with progress(int(span) + 1, "embed") as bar: - embedded = ( - images.transform(throttle(1.0)) - .map(lambda obs: obs.derive(data=obs.data, pose=camera_pose(obs.ts))) - .transform(EmbedImages(clip)) - .tap(bar) - .materialize() - ) - -# motion gate: only capture while the wrist is (near-)still — the arm parks -# often, and any real motion measurably blurs the frames -SPEED_MAX = 0.02 # m/s -speeds = [camera_speed(obs.ts) for obs in embedded.after(t0).before(t1)] -print(f"motion gate: > {SPEED_MAX} m/s rejects {np.mean([s > SPEED_MAX for s in speeds]):.0%}") - -# embeddings zone us into the frames worth looking at. -# rank ALL frames then window: the backend applies .search() before the time -# filters, so a windowed top-k would keep only global hits inside the window -matches = ( - embedded.search(clip.embed_text(query), k=embedded.count()) - .after(t0) - .before(t1) - .filter(lambda obs: still(obs.ts)) - .limit(12) - .materialize() -) -print(matches.summary()) -clip.stop() # free GPU memory for the VLM + segmenter - -moondream = MoondreamVlModel() -moondream.start() -segmenter = EdgeTAMImageSegmenter() - - -def detect(frames): - """VLM detections refined into masks, frames without detections dropped.""" - return ( - frames.map_data(lambda obs: moondream.query_detections(obs.data, query)) - .map_data(lambda obs: obs.data.filter(lambda det: det.bbox_2d_volume() > 3000)) - .filter(lambda obs: len(obs.data) > 0) - .map_data(lambda obs: segmenter.segment(obs.data)) - ) +REFUSAL_MARGIN = 0.15 -with progress(matches.count(), "detect") as bar: - detections = detect(matches.tap(bar)).materialize() -print(f"{detections.count()} frames with 2d detections") +def render(out: str, store: Any, trace: LocalizeTrace, t0: float, t1: float) -> None: + """Write the .rrd - rerun stays an inline import. -def depth_at(obs): - """Temporal join: aligned depth frame for a color observation. - - One frame of tolerance — a farther depth frame may come from mid-motion. + Entity contract (the acceptance color cheat sheet): ``map`` backdrop, + ``detections/matched/*`` green, ``detections/verified/*`` red, + ``detections/answer`` always blue. """ - nearest = store.streams.depth_image.at(obs.ts, 0.05).first() - return nearest.data if nearest is not None else None - - -# lift into 3D straight through the depth image — no map needed -# (tight tf tolerance: the arm moves, a stale transform smears the projection) -project = ProjectDepthTo3D( - depth_at, - camera_info, - tf=tf, - optical_frame=OPTICAL_FRAME, - time_tolerance=0.5, - filters=[], # no pointcloud filtering for now -) -with progress(detections.count(), "project 3d") as bar: - detections3d = ( - detections.tap(bar).transform(project).filter(lambda obs: len(obs.data) > 0).materialize() - ) -print(f"{detections3d.count()} frames with 3d detections") - -# every frame that was looking at the strongest detection -best = max((det for obs in detections3d for det in obs.data), key=lambda det: len(det.pointcloud)) -with progress(int(span / 0.25) + 1, "observing") as bar: - observing = ( - images.transform(throttle(0.25)) - .tap(bar) - .filter( - sees(best.pose, camera_info, tf=tf, optical_frame=OPTICAL_FRAME, time_tolerance=0.5) - ) - .filter(lambda obs: still(obs.ts)) - .materialize() - ) -print(f"{observing.count()} frames observing the detection at {best.pose.position}") - -# re-detect from observing frames and project those into 3D too -verify_frames = observing.limit(32).materialize() -with progress(verify_frames.count(), "cross-view") as bar: - verified3d = ( - detect(verify_frames.tap(bar)) - .transform(project) - .filter(lambda obs: len(obs.data) > 0) - .materialize() - ) -print(f"cross-view: {verified3d.count()} observing frames re-detect in 3d") - -# Rerun output uses the source timestamps directly. - - -def render() -> None: - """Write the .rrd — rerun stays an inline import.""" import rerun as rr import rerun.blueprint as rrb + from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 + from dimos.visualization.rerun.init import rerun_init + + tf = StreamTF.from_store(store) + camera_info = store.streams.camera_info.first().data + rerun_init("memory-localize") rr.save(out) rr.send_blueprint( @@ -200,47 +66,142 @@ def render() -> None: ) ) - GREEN, RED = [46, 204, 113], [231, 76, 60] + GREEN, RED, BLUE = [46, 204, 113], [231, 76, 60], [52, 120, 246] POINT_SIZE = 0.005 def at(ts: float) -> None: rr.set_time("ts", timestamp=ts) - # scene backdrop from one depth frame - backdrop_obs = detections3d.first() - backdrop = PointCloud2.from_rgbd( - backdrop_obs.data.image, depth_at(backdrop_obs), camera_info, depth_scale=0.001 - ).transform(-tf.get(OPTICAL_FRAME, "world", backdrop_obs.ts, 0.5)) - rr.log("map", backdrop.voxel_downsample(0.01).to_rerun(voxel_size=POINT_SIZE), static=True) + # scene backdrop from the answer frame's depth (or the first detection) + backdrop_ts = trace.backdrop_ts + if backdrop_ts is None and trace.matched: + backdrop_ts = trace.matched[0][0] + if backdrop_ts is not None: + try: + color = store.streams.color_image.at(backdrop_ts, 0.1).first().data + depth = gates.depth_at(store, backdrop_ts) + transform = tf.get( + gates.OPTICAL_FRAME, gates.WORLD_FRAME, backdrop_ts, gates.TF_TOLERANCE + ) + if depth is not None and transform is not None: + backdrop = PointCloud2.from_rgbd( + color, depth, camera_info, depth_scale=0.001 + ).transform(-transform) + rr.log( + "map", + backdrop.voxel_downsample(0.01).to_rerun(voxel_size=POINT_SIZE), + static=True, + ) + except LookupError: + pass - # live camera feed + camera frustum tracking the wrist along the timeline + # live camera feed + frustum tracking the wrist along the timeline rr.log("camera", camera_info.to_rerun(), static=True) - for obs in images.transform(throttle(0.1)): + feed_throttle = 0.1 if (t1 - t0) <= 160 else 0.4 + feed = store.streams.color_image.after(t0).before(t1).transform(throttle(feed_throttle)) + for obs in feed: + pose = gates.camera_pose(tf, obs.ts) + if pose is None: + continue at(obs.ts) rr.log("camera/image", obs.data.to_rerun()) - rr.log("camera", camera_pose(obs.ts).to_rerun()) + rr.log("camera", pose.to_rerun()) - # marked frames: into the live feed, plus a frozen frustum pinned at the - # capture pose (not the moving camera entity, which would drag them along) - for i, obs in enumerate(detections): + # marked frames: into the live feed, plus a frozen frustum at the capture pose + for i, obs in enumerate(trace.detection_frames): + pose = gates.camera_pose(tf, obs.ts) + if pose is None: + continue at(obs.ts) annotated = obs.data.annotated_image() rr.log("camera/image", annotated.to_rerun()) frame = f"detections/frames/{i}" - rr.log(frame, camera_pose(obs.ts).to_rerun()) + rr.log(frame, pose.to_rerun()) rr.log(frame, camera_info.to_rerun()) rr.log(f"{frame}/image", annotated.to_rerun()) - # 3d detections: green = embedding matches, red = cross-view re-detections - for tag, stream, rgb in [("matched", detections3d, GREEN), ("verified", verified3d, RED)]: - for i, obs in enumerate(stream): - at(obs.ts) - for j, det in enumerate(obs.data): - rr.log( - f"detections/{tag}/{i}_{j}_{det.name.replace(' ', '_')}", - det.pointcloud.to_rerun(voxel_size=POINT_SIZE, colors=rgb), - ) + # 3d detections: green = matched candidates, red = cross-view re-detections + for tag, entries, rgb in [("matched", trace.matched, GREEN), ("verified", trace.verified, RED)]: + for i, (ts, det) in enumerate(entries): + at(ts) + rr.log( + f"detections/{tag}/{i}_{det.name.replace(' ', '_')}", + det.pointcloud.to_rerun(voxel_size=POINT_SIZE, colors=rgb), + ) + + # the answer: always blue, whatever the query + if trace.answer is not None: + at(trace.answer.ts) + rr.log( + "detections/answer", + trace.answer.pointcloud.to_rerun(voxel_size=POINT_SIZE, colors=BLUE), + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("query", nargs="?", default="plant") + parser.add_argument("out", nargs="?", default="localize.rrd") + parser.add_argument("--dataset", type=Path, help="memory2 recording database") + parser.add_argument( + "--from", dest="start", type=float, default=0.0, help="start offset into the recording (s)" + ) + parser.add_argument("--duration", type=float, default=None, help="how much to parse (s)") + parser.add_argument( + "--allow-no-pose", + action="store_true", + help="return an RGB-only hit with a null position instead of refusing", + ) + args = parser.parse_args() + + dataset = args.dataset or get_data( + "xarm6_worldbelief_realsense_d435i_stationery_calibrated/" + "xarm6_worldbelief_20260729_203624_161992.db" + ) + store = SqliteStore(path=dataset) + lo, hi = store.streams.color_image.get_time_range() + after = lo + args.start + before = lo + args.start + args.duration if args.duration is not None else None + + trace = LocalizeTrace() + hit = localize( + store, + args.query, + after=after, + before=before, + require_pose=not args.allow_no_pose, + trace=trace, + ) + + if hit is None: + print(f"no verified detection of {args.query!r}") + return 1 + + offset = hit.pose_timestamp - lo + if hit.position_world_xyz is None: + print( + f"hit {args.query!r} without pose: reason={hit.reason} " + f"score={hit.semantic_score:.2f} ts_offset={offset:.1f}s" + ) + return 0 + + x, y, z = hit.position_world_xyz + cloud_points = len(hit.point_cloud) if hit.point_cloud is not None else 0 + print( + f"hit {args.query!r}: position=({x:.3f}, {y:.3f}, {z:.3f}) frame={hit.frame_id} " + f"ts_offset={offset:.1f}s points={cloud_points} views={hit.n_views} " + f"score={hit.semantic_score:.2f} margin={hit.ambiguity_margin:.2f}" + ) + if hit.ambiguity_margin < REFUSAL_MARGIN: + print( + f"ambiguity: margin {hit.ambiguity_margin:.2f} below refusal threshold " + f"{REFUSAL_MARGIN:.2f} - multiple coexisting matches, this pick is flagged" + ) + + render(args.out, store, trace, after, before if before is not None else hi) + print(f"saved {args.out}") + return 0 -render() -print(f"saved {out}") +if __name__ == "__main__": + sys.exit(main()) diff --git a/dimos/perception/memory/types.py b/dimos/perception/memory/types.py new file mode 100644 index 0000000000..24fba5863f --- /dev/null +++ b/dimos/perception/memory/types.py @@ -0,0 +1,198 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Object registration types: supports, instances, localizations, policies. + +The identity key throughout is the *support* - the object's occupied volume +in world coordinates. Labels and appearance are metadata attached to a +support, never a key. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Literal + +import numpy as np + +if TYPE_CHECKING: + from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 + + +@dataclass(frozen=True) +class Support: + """Occupied volume with an error envelope. The identity key.""" + + center_xyz: tuple[float, float, float] + extent_xyz_m: tuple[float, float, float] + orientation_xyzw: tuple[float, float, float, float] + sigma_xyz_m: tuple[float, float, float] + coverage: float + axes_observed: tuple[bool, bool, bool] + frame_id: str + + +@dataclass +class SupportObservation: + """One accepted per-frame observation of a support: a masked depth lift.""" + + ts: float + cloud: PointCloud2 + centroid: np.ndarray # (3,) world + aabb_min: np.ndarray # (3,) world + aabb_max: np.ndarray # (3,) world + n_points: int + mask_area_px: int + camera_position: np.ndarray # (3,) world + score: float = 1.0 + bbox: tuple[float, float, float, float] | None = None + + +@dataclass +class Instance: + """A deduplicated object instance computed for one inventory call.""" + + instance_id: str + grounded: bool + primary_label: str | None + labels: tuple[tuple[str, float], ...] + state: Literal["active", "occluded", "stale", "retired"] + identity_confidence: float + support: Support | None + latest_position_xyz: tuple[float, float, float] | None + latest_seen_ts: float + members: list[SupportObservation] = field(default_factory=list) + + +@dataclass +class Localization: + """Latest unambiguous localization of a queried object.""" + + instance_id: str + semantic_score: float + identity_score: float + ambiguity_margin: float + + position_world_xyz: tuple[float, float, float] | None + orientation_world_xyzw: tuple[float, float, float, float] | None + frame_id: str + + support: Support | None + pose_timestamp: float + geometry_timestamp: float + last_seen_timestamp: float + + point_cloud: PointCloud2 | None + cloud_mode: str + coverage: float + n_views: int + reason: str | None = None + + +@dataclass(frozen=True) +class LocalizePolicy: + """Score and geometry thresholds for candidate formation, lift and acceptance. + + The funnel generalizes; these numbers do not. Each was fit to the score + and height distributions of one measured scene, so a different rig, + object scale or detector vocabulary needs its own instance rather than + the defaults. + """ + + candidate_floor: float = 0.25 # form a candidate at this score + # Measured across the replay: every true positive's best view scores 0.43 + # or higher (the diagonal book is the floor); every text-only near-miss - + # gray tape rolls for "roll of black tape" at 0.31, a dark table stripe + # for "black tape" at 0.36 - stays at or under 0.36. The accept sits + # between. + accept_score: float = 0.40 + refusal_margin: float = 0.15 + min_views: int = 2 # a support seen from one pose only is unconfirmed + + cluster_radius_m: float = 0.08 # observations within this are the same support + min_depth_points: int = 60 + max_object_extent_m: float = 0.60 + min_camera_range_m: float = 0.28 + # A cloud that hugs the support surface is a patch of the surface, not an + # object: a dark wood-grain cell outlined by the tape grid reads as "black + # tape" to the detector at 0.41, but nothing about it rises above the + # table. The flattest real object here, a sticky pad, clears 5 mm at its + # 95th height percentile; surface patches stay under 2 mm. + surface_patch_max_rise_m: float = 0.003 + surface_patch_min_drop_m: float = -0.02 + + +@dataclass(frozen=True) +class InventoryPolicy: + """Physical thresholds for discovery, validity, scope and association. + + Every quantity is metric (meters, seconds, pixels, IoU) - a claim that + can be checked against the recording, unlike an appearance-similarity + threshold. + """ + + min_mask_area_px: int = 400 + max_mask_area_fraction: float = 0.25 + min_depth_points: int = 60 + max_object_extent_m: float = 0.45 + min_height_above_plane_m: float = 0.003 + band_above_plane_m: tuple[float, float] = (-0.02, 0.30) + min_camera_range_m: float = 0.28 + + envelope_pad_m: float = 0.015 + search_radius_m: float = 0.15 + overlap_accept: float = 0.20 + # Same-frame observations whose clouds touch within this gap are one + # body - rigid objects cannot interpenetrate, and distinct objects on a + # workspace sit apart by more than sensor noise. This is what fuses + # whole-and-part duplicate proposals while identical twins, centimeters + # apart, stay two. + same_frame_merge_gap_m: float = 0.02 + # A support observed in a single keyframe is unconfirmed - nothing saw it + # from a second pose or moment, so it never becomes an instance. + min_member_observations: int = 2 + + include_object_parts: bool = False + include_surfaces: bool = False + include_containers: bool = True + preserve_unknown_instances: bool = True + + in_scope: Callable[[np.ndarray], bool] | None = None + + +def aabb_overlap( + a_min: np.ndarray, + a_max: np.ndarray, + b_min: np.ndarray, + b_max: np.ndarray, + pad: float = 0.0, +) -> float: + """Intersection volume normalized by the smaller (padded) box volume. + + ``pad`` grows each box by the error envelope on every side, so the value + is an overlap of envelopes, not of raw partial-view boxes. Degenerate + axes are floored at 1 cm so thin objects (a pen, a sticky pad) do not + produce zero volumes. + """ + a_lo, a_hi = a_min - pad, a_max + pad + b_lo, b_hi = b_min - pad, b_max + pad + inter = np.minimum(a_hi, b_hi) - np.maximum(a_lo, b_lo) + if (inter <= 0).any(): + return 0.0 + floor = 0.01 + vol_a = float(np.prod(np.maximum(a_hi - a_lo, floor))) + vol_b = float(np.prod(np.maximum(b_hi - b_lo, floor))) + vol_i = float(np.prod(np.maximum(inter, 0.0))) + return vol_i / max(min(vol_a, vol_b), 1e-9) diff --git a/pyproject.toml b/pyproject.toml index 4e50ad93f8..7c4faf5afa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -261,6 +261,9 @@ perception = [ "omegaconf>=2.3.0", "hydra-core>=1.3.0", "chromadb>=1.0.0", # spatial-memory vector store (dimos/perception/spatial_*) + # SigLIP tokenizer (SiglipProcessor) requires SentencePiece; not captured by + # transformers package metadata for google/siglip-base-patch16-224. + "sentencepiece>=0.2.0", ] unitree = [ @@ -390,12 +393,15 @@ autofix = ["ruff==0.14.3"] # Project deps shared by `tests` and `lint`. project-deps = [ "dimos[web,visualization,webrtc]", - "torch", + # Blackwell (sm_120) needs torch 2.7 from pytorch-cu128. Cap below 2.8. + "torch>=2.1,<2.8", + "torchvision>=0.16,<0.23", "langchain==1.2.3", "langchain-core==1.3.3", "googlemaps>=4.10.0", "transformers[torch]==4.53.3", "einops>=0.8.1", # Florence2 trust_remote_code dep (see perception extra) + "sentencepiece>=0.2.0", # SigLIP tokenizer dep (see perception extra) "ultralytics>=8.3.70", "hydra-core>=1.3.0", "open_clip_torch==3.2.0", @@ -529,10 +535,26 @@ override-dependencies = [ "huggingface-hub>=0.30,<1", "diffusers>=0.29", "pyopengl>=3.1.5", + # Force the torch 2.7 line even if a transitive dep asks for newer. + "torch>=2.1,<2.8", + "torchvision>=0.16,<0.23", ] [tool.uv.sources] graspgenx = { git = "https://github.com/NVlabs/GraspGenX.git", rev = "b9429097728cb1c430dd78b92edf17ba318aad03" } +# CUDA builds are not published for macOS or non-x86_64 Linux. Those platforms +# fall back to PyPI. +torch = [ + { index = "pytorch-cu128", marker = "(sys_platform == 'linux' and platform_machine == 'x86_64') or sys_platform == 'win32'" }, +] +torchvision = [ + { index = "pytorch-cu128", marker = "(sys_platform == 'linux' and platform_machine == 'x86_64') or sys_platform == 'win32'" }, +] + +[[tool.uv.index]] +name = "pytorch-cu128" +url = "https://download.pytorch.org/whl/cu128" +explicit = true [tool.ruff] line-length = 100 diff --git a/uv.lock b/uv.lock index 04a30f4013..399a4a909b 100644 --- a/uv.lock +++ b/uv.lock @@ -6,11 +6,14 @@ resolution-markers = [ "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -50,6 +53,10 @@ overrides = [ { name = "pyopengl", specifier = ">=3.1.5" }, { name = "pytest", specifier = "==8.3.5" }, { name = "timm", specifier = ">=1.0.17" }, + { name = "torch", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')", specifier = ">=2.1,<2.8" }, + { name = "torch", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'", specifier = ">=2.1,<2.8", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torchvision", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')", specifier = ">=0.16,<0.23" }, + { name = "torchvision", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'", specifier = ">=0.16,<0.23", index = "https://download.pytorch.org/whl/cu128" }, { name = "trimesh", specifier = ">=4.12" }, { name = "yourdfpy", specifier = ">=0.0.60" }, ] @@ -83,7 +90,8 @@ dependencies = [ { name = "psutil" }, { name = "pyyaml" }, { name = "safetensors" }, - { name = "torch" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4a/8e/ac2a9566747a93f8be36ee08532eb0160558b07630a081a6056a9f89bf1d/accelerate-1.12.0.tar.gz", hash = "sha256:70988c352feb481887077d2ab845125024b2a137a5090d6d7a32b57d03a45df6", size = 398399, upload-time = "2025-11-21T11:27:46.973Z" } wheels = [ @@ -644,7 +652,7 @@ name = "build" version = "1.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "(os_name == 'nt' and platform_machine != 'aarch64' and sys_platform == 'linux') or (os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux')" }, + { name = "colorama", marker = "(os_name == 'nt' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (os_name == 'nt' and sys_platform != 'darwin' and sys_platform != 'linux')" }, { name = "importlib-metadata" }, { name = "packaging" }, { name = "pyproject-hooks" }, @@ -799,7 +807,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -826,9 +835,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -1171,7 +1182,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -1228,9 +1240,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -1749,6 +1763,7 @@ all = [ { name = "reportlab" }, { name = "rerun-sdk" }, { name = "roboplan" }, + { name = "sentencepiece" }, { name = "sounddevice" }, { name = "soundfile" }, { name = "sse-starlette" }, @@ -1794,6 +1809,7 @@ base = [ { name = "openevals" }, { name = "pillow" }, { name = "rerun-sdk" }, + { name = "sentencepiece" }, { name = "sounddevice" }, { name = "soundfile" }, { name = "sse-starlette" }, @@ -1819,8 +1835,10 @@ graspgenx = [ { name = "graspgenx" }, { name = "huggingface-hub" }, { name = "matplotlib" }, - { name = "torch" }, - { name = "torchvision" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "torchvision", version = "0.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, ] learning = [ { name = "h5py" }, @@ -1868,6 +1886,7 @@ perception = [ { name = "moondream" }, { name = "omegaconf" }, { name = "pillow" }, + { name = "sentencepiece" }, { name = "transformers", extra = ["torch"] }, { name = "ultralytics" }, ] @@ -1910,6 +1929,7 @@ unitree = [ { name = "openevals" }, { name = "pillow" }, { name = "rerun-sdk" }, + { name = "sentencepiece" }, { name = "sounddevice" }, { name = "soundfile" }, { name = "sse-starlette" }, @@ -1945,6 +1965,7 @@ unitree-dds = [ { name = "openevals" }, { name = "pillow" }, { name = "rerun-sdk" }, + { name = "sentencepiece" }, { name = "sounddevice" }, { name = "soundfile" }, { name = "sse-starlette" }, @@ -2007,10 +2028,14 @@ lint = [ { name = "python-socketio" }, { name = "roboplan" }, { name = "ruff" }, + { name = "sentencepiece" }, { name = "sounddevice" }, { name = "tensorboard" }, - { name = "torch" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "torchreid" }, + { name = "torchvision", version = "0.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "transformers", extra = ["torch"] }, { name = "trimesh" }, { name = "types-pyaudio" }, @@ -2036,9 +2061,13 @@ project-deps = [ { name = "ollama" }, { name = "open-clip-torch" }, { name = "openai" }, + { name = "sentencepiece" }, { name = "tensorboard" }, - { name = "torch" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "torchreid" }, + { name = "torchvision", version = "0.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "transformers", extra = ["torch"] }, { name = "ultralytics" }, { name = "xacro" }, @@ -2079,9 +2108,13 @@ tests = [ { name = "python-lsp-ruff" }, { name = "python-lsp-server", extra = ["all"] }, { name = "requests-mock" }, + { name = "sentencepiece" }, { name = "tensorboard" }, - { name = "torch" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "torchreid" }, + { name = "torchvision", version = "0.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "transformers", extra = ["torch"] }, { name = "trimesh" }, { name = "ultralytics" }, @@ -2128,9 +2161,13 @@ tests-self-hosted = [ { name = "python-lsp-ruff" }, { name = "python-lsp-server", extra = ["all"] }, { name = "requests-mock" }, + { name = "sentencepiece" }, { name = "tensorboard" }, - { name = "torch" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "torchreid" }, + { name = "torchvision", version = "0.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "transformers", extra = ["torch"] }, { name = "trimesh" }, { name = "ultralytics" }, @@ -2242,6 +2279,7 @@ requires-dist = [ { name = "rerun-sdk", marker = "extra == 'visualization'", specifier = "==0.32.0" }, { name = "roboplan", marker = "extra == 'manipulation'", specifier = ">=0.6.0,<0.7.0" }, { name = "scipy", specifier = ">=1.15.1" }, + { name = "sentencepiece", marker = "extra == 'perception'", specifier = ">=0.2.0" }, { name = "sortedcontainers", specifier = "==2.4.0" }, { name = "sounddevice", marker = "extra == 'agents'" }, { name = "soundfile", marker = "extra == 'web'" }, @@ -2254,9 +2292,11 @@ requires-dist = [ { name = "textual-serve", specifier = ">=1.1.1,<2" }, { name = "timm", marker = "extra == 'misc'", specifier = ">=1.0.15" }, { name = "toolz", specifier = ">=1.1.0" }, - { name = "torch", marker = "extra == 'graspgenx'", specifier = ">=2.1,<2.7" }, + { name = "torch", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'graspgenx') or (sys_platform == 'win32' and extra == 'graspgenx')", specifier = ">=2.1,<2.7", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torch", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux' and extra == 'graspgenx') or (sys_platform != 'linux' and sys_platform != 'win32' and extra == 'graspgenx')", specifier = ">=2.1,<2.7" }, { name = "torchreid", marker = "extra == 'misc'", specifier = "==0.2.5" }, - { name = "torchvision", marker = "extra == 'graspgenx'", specifier = ">=0.16,<0.22" }, + { name = "torchvision", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux' and extra == 'graspgenx') or (sys_platform == 'win32' and extra == 'graspgenx')", specifier = ">=0.16,<0.22", index = "https://download.pytorch.org/whl/cu128" }, + { name = "torchvision", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux' and extra == 'graspgenx') or (sys_platform != 'linux' and sys_platform != 'win32' and extra == 'graspgenx')", specifier = ">=0.16,<0.22" }, { name = "transformers", extras = ["torch"], marker = "extra == 'perception'", specifier = ">=4.53.0,<4.54" }, { name = "trimesh", marker = "extra == 'apriltag'", specifier = ">=4.0.0" }, { name = "trimesh", marker = "extra == 'manipulation'" }, @@ -2306,10 +2346,14 @@ lint = [ { name = "python-socketio", specifier = ">=5.16.1" }, { name = "roboplan", specifier = ">=0.6.0,<0.7.0" }, { name = "ruff", specifier = "==0.14.3" }, + { name = "sentencepiece", specifier = ">=0.2.0" }, { name = "sounddevice", specifier = ">=0.5.5" }, { name = "tensorboard", specifier = "==2.20.0" }, - { name = "torch" }, + { name = "torch", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')", specifier = ">=2.1,<2.8" }, + { name = "torch", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'", specifier = ">=2.1,<2.8", index = "https://download.pytorch.org/whl/cu128" }, { name = "torchreid", specifier = "==0.2.5" }, + { name = "torchvision", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')", specifier = ">=0.16,<0.23" }, + { name = "torchvision", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'", specifier = ">=0.16,<0.23", index = "https://download.pytorch.org/whl/cu128" }, { name = "transformers", extras = ["torch"], specifier = "==4.53.3" }, { name = "trimesh", specifier = ">=4.12" }, { name = "types-pyaudio" }, @@ -2335,9 +2379,13 @@ project-deps = [ { name = "ollama", specifier = ">=0.6.0" }, { name = "open-clip-torch", specifier = "==3.2.0" }, { name = "openai" }, + { name = "sentencepiece", specifier = ">=0.2.0" }, { name = "tensorboard", specifier = "==2.20.0" }, - { name = "torch" }, + { name = "torch", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')", specifier = ">=2.1,<2.8" }, + { name = "torch", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'", specifier = ">=2.1,<2.8", index = "https://download.pytorch.org/whl/cu128" }, { name = "torchreid", specifier = "==0.2.5" }, + { name = "torchvision", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')", specifier = ">=0.16,<0.23" }, + { name = "torchvision", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'", specifier = ">=0.16,<0.23", index = "https://download.pytorch.org/whl/cu128" }, { name = "transformers", extras = ["torch"], specifier = "==4.53.3" }, { name = "ultralytics", specifier = ">=8.3.70" }, { name = "xacro" }, @@ -2379,9 +2427,13 @@ tests = [ { name = "python-lsp-ruff", specifier = "==2.3.0" }, { name = "python-lsp-server", extras = ["all"], specifier = "==1.14.0" }, { name = "requests-mock", specifier = "==1.12.1" }, + { name = "sentencepiece", specifier = ">=0.2.0" }, { name = "tensorboard", specifier = "==2.20.0" }, - { name = "torch" }, + { name = "torch", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')", specifier = ">=2.1,<2.8" }, + { name = "torch", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'", specifier = ">=2.1,<2.8", index = "https://download.pytorch.org/whl/cu128" }, { name = "torchreid", specifier = "==0.2.5" }, + { name = "torchvision", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')", specifier = ">=0.16,<0.23" }, + { name = "torchvision", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'", specifier = ">=0.16,<0.23", index = "https://download.pytorch.org/whl/cu128" }, { name = "transformers", extras = ["torch"], specifier = "==4.53.3" }, { name = "trimesh", specifier = ">=4.0.0" }, { name = "ultralytics", specifier = ">=8.3.70" }, @@ -2430,9 +2482,13 @@ tests-self-hosted = [ { name = "python-lsp-ruff", specifier = "==2.3.0" }, { name = "python-lsp-server", extras = ["all"], specifier = "==1.14.0" }, { name = "requests-mock", specifier = "==1.12.1" }, + { name = "sentencepiece", specifier = ">=0.2.0" }, { name = "tensorboard", specifier = "==2.20.0" }, - { name = "torch" }, + { name = "torch", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')", specifier = ">=2.1,<2.8" }, + { name = "torch", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'", specifier = ">=2.1,<2.8", index = "https://download.pytorch.org/whl/cu128" }, { name = "torchreid", specifier = "==0.2.5" }, + { name = "torchvision", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')", specifier = ">=0.16,<0.23" }, + { name = "torchvision", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'", specifier = ">=0.16,<0.23", index = "https://download.pytorch.org/whl/cu128" }, { name = "transformers", extras = ["torch"], specifier = "==4.53.3" }, { name = "trimesh", specifier = ">=4.0.0" }, { name = "ultralytics", specifier = ">=8.3.70" }, @@ -2610,11 +2666,14 @@ version = "1.49.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ { name = "matplotlib", marker = "platform_machine != 'aarch64' and sys_platform != 'darwin'" }, @@ -2665,8 +2724,10 @@ dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pillow" }, - { name = "torch" }, - { name = "torchvision" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "torchvision", version = "0.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "tqdm" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a8/3f/6517d2cf72c5fde86254599b249986437148372b4610374b3df34d393ecc/edgetam_dimos-1.0.1.tar.gz", hash = "sha256:f194fe8a7ac2295703d2e603f381d3527f05a9cc2bdad42eba64bfe7c3eafbf5", size = 85046, upload-time = "2026-08-13T06:56:50.332Z" } @@ -2982,7 +3043,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -3013,9 +3075,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -3312,9 +3376,11 @@ dependencies = [ { name = "tensorboardx" }, { name = "tensordict" }, { name = "timm" }, - { name = "torch" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "torch-geometric" }, - { name = "torchvision" }, + { name = "torchvision", version = "0.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "tqdm" }, { name = "transformers" }, { name = "trimesh" }, @@ -3615,7 +3681,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -3653,7 +3720,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -3680,7 +3748,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -3787,7 +3856,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -3819,9 +3889,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -3886,7 +3958,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -3912,9 +3985,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -3943,7 +4018,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -3977,9 +4053,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -5177,11 +5255,14 @@ version = "11.1.2" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", ] dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin'" }, @@ -5469,7 +5550,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -5488,9 +5570,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -5546,7 +5630,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -5598,9 +5683,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -5645,120 +5732,128 @@ wheels = [ [[package]] name = "nvidia-cublas-cu12" -version = "12.4.5.8" +version = "12.8.3.14" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/71/1c91302526c45ab494c23f61c7a84aa568b8c1f9d196efa5993957faf906/nvidia_cublas_cu12-12.4.5.8-py3-none-manylinux2014_x86_64.whl", hash = "sha256:2fc8da60df463fdefa81e323eef2e36489e1c94335b5358bcb38360adf75ac9b", size = 363438805, upload-time = "2024-04-03T20:57:06.025Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/4b01f10069e23c641f116c62fc31e31e8dc361a153175d81561d15c8143b/nvidia_cublas_cu12-12.8.3.14-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:3f0e05e7293598cf61933258b73e66a160c27d59c4422670bf0b79348c04be44", size = 609620630, upload-time = "2025-01-23T17:55:00.753Z" }, ] [[package]] name = "nvidia-cuda-cupti-cu12" -version = "12.4.127" +version = "12.8.57" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/67/42/f4f60238e8194a3106d06a058d494b18e006c10bb2b915655bd9f6ea4cb1/nvidia_cuda_cupti_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:9dec60f5ac126f7bb551c055072b69d85392b13311fcc1bcda2202d172df30fb", size = 13813957, upload-time = "2024-04-03T20:55:01.564Z" }, + { url = "https://files.pythonhosted.org/packages/39/6f/3683ecf4e38931971946777d231c2df00dd5c1c4c2c914c42ad8f9f4dca6/nvidia_cuda_cupti_cu12-12.8.57-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e0b2eb847de260739bee4a3f66fac31378f4ff49538ff527a38a01a9a39f950", size = 10237547, upload-time = "2025-01-23T17:47:56.863Z" }, ] [[package]] name = "nvidia-cuda-nvrtc-cu12" -version = "12.4.127" +version = "12.8.61" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/14/91ae57cd4db3f9ef7aa99f4019cfa8d54cb4caa7e00975df6467e9725a9f/nvidia_cuda_nvrtc_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a178759ebb095827bd30ef56598ec182b85547f1508941a3d560eb7ea1fbf338", size = 24640306, upload-time = "2024-04-03T20:56:01.463Z" }, + { url = "https://files.pythonhosted.org/packages/d4/22/32029d4583f7b19cfe75c84399cbcfd23f2aaf41c66fc8db4da460104fff/nvidia_cuda_nvrtc_cu12-12.8.61-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:a0fa9c2a21583105550ebd871bd76e2037205d56f33f128e69f6d2a55e0af9ed", size = 88024585, upload-time = "2025-01-23T17:50:10.722Z" }, ] [[package]] name = "nvidia-cuda-runtime-cu12" -version = "12.4.127" +version = "12.8.57" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ea/27/1795d86fe88ef397885f2e580ac37628ed058a92ed2c39dc8eac3adf0619/nvidia_cuda_runtime_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:64403288fa2136ee8e467cdc9c9427e0434110899d07c779f25b5c068934faa5", size = 883737, upload-time = "2024-04-03T20:54:51.355Z" }, + { url = "https://files.pythonhosted.org/packages/16/f6/0e1ef31f4753a44084310ba1a7f0abaf977ccd810a604035abb43421c057/nvidia_cuda_runtime_cu12-12.8.57-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:75342e28567340b7428ce79a5d6bb6ca5ff9d07b69e7ce00d2c7b4dc23eff0be", size = 954762, upload-time = "2025-01-23T17:47:22.21Z" }, ] [[package]] name = "nvidia-cudnn-cu12" -version = "9.1.0.70" +version = "9.7.1.26" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/9f/fd/713452cd72343f682b1c7b9321e23829f00b842ceaedcda96e742ea0b0b3/nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f", size = 664752741, upload-time = "2024-04-22T15:24:15.253Z" }, + { url = "https://files.pythonhosted.org/packages/25/dc/dc825c4b1c83b538e207e34f48f86063c88deaa35d46c651c7c181364ba2/nvidia_cudnn_cu12-9.7.1.26-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:6d011159a158f3cfc47bf851aea79e31bcff60d530b70ef70474c84cac484d07", size = 726851421, upload-time = "2025-02-06T22:18:29.812Z" }, ] [[package]] name = "nvidia-cufft-cu12" -version = "11.2.1.3" +version = "11.3.3.41" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/27/94/3266821f65b92b3138631e9c8e7fe1fb513804ac934485a8d05776e1dd43/nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9", size = 211459117, upload-time = "2024-04-03T20:57:40.402Z" }, + { url = "https://files.pythonhosted.org/packages/ac/26/b53c493c38dccb1f1a42e1a21dc12cba2a77fbe36c652f7726d9ec4aba28/nvidia_cufft_cu12-11.3.3.41-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:da650080ab79fcdf7a4b06aa1b460e99860646b176a43f6208099bdc17836b6a", size = 193118795, upload-time = "2025-01-23T17:56:30.536Z" }, +] + +[[package]] +name = "nvidia-cufile-cu12" +version = "1.13.0.11" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/9c/1f3264d0a84c8a031487fb7f59780fc78fa6f1c97776233956780e3dc3ac/nvidia_cufile_cu12-1.13.0.11-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:483f434c541806936b98366f6d33caef5440572de8ddf38d453213729da3e7d4", size = 1197801, upload-time = "2025-01-23T17:57:07.247Z" }, ] [[package]] name = "nvidia-curand-cu12" -version = "10.3.5.147" +version = "10.3.9.55" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/6d/44ad094874c6f1b9c654f8ed939590bdc408349f137f9b98a3a23ccec411/nvidia_curand_cu12-10.3.5.147-py3-none-manylinux2014_x86_64.whl", hash = "sha256:a88f583d4e0bb643c49743469964103aa59f7f708d862c3ddb0fc07f851e3b8b", size = 56305206, upload-time = "2024-04-03T20:58:08.722Z" }, + { url = "https://files.pythonhosted.org/packages/bd/fc/7be5d0082507269bb04ac07cc614c84b78749efb96e8cf4100a8a1178e98/nvidia_curand_cu12-10.3.9.55-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:8387d974240c91f6a60b761b83d4b2f9b938b7e0b9617bae0f0dafe4f5c36b86", size = 63618038, upload-time = "2025-01-23T17:57:41.838Z" }, ] [[package]] name = "nvidia-cusolver-cu12" -version = "11.6.1.9" +version = "11.7.2.55" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/e1/5b9089a4b2a4790dfdea8b3a006052cfecff58139d5a4e34cb1a51df8d6f/nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260", size = 127936057, upload-time = "2024-04-03T20:58:28.735Z" }, + { url = "https://files.pythonhosted.org/packages/c2/08/953675873a136d96bb12f93b49ba045d1107bc94d2551c52b12fa6c7dec3/nvidia_cusolver_cu12-11.7.2.55-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:4d1354102f1e922cee9db51920dba9e2559877cf6ff5ad03a00d853adafb191b", size = 260373342, upload-time = "2025-01-23T17:58:56.406Z" }, ] [[package]] name = "nvidia-cusparse-cu12" -version = "12.3.1.170" +version = "12.5.7.53" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/db/f7/97a9ea26ed4bbbfc2d470994b8b4f338ef663be97b8f677519ac195e113d/nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1", size = 207454763, upload-time = "2024-04-03T20:58:59.995Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ab/31e8149c66213b846c082a3b41b1365b831f41191f9f40c6ddbc8a7d550e/nvidia_cusparse_cu12-12.5.7.53-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3c1b61eb8c85257ea07e9354606b26397612627fdcd327bfd91ccf6155e7c86d", size = 292064180, upload-time = "2025-01-23T18:00:23.233Z" }, ] [[package]] name = "nvidia-cusparselt-cu12" -version = "0.6.2" +version = "0.6.3" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/78/a8/bcbb63b53a4b1234feeafb65544ee55495e1bb37ec31b999b963cbccfd1d/nvidia_cusparselt_cu12-0.6.2-py3-none-manylinux2014_x86_64.whl", hash = "sha256:df2c24502fd76ebafe7457dbc4716b2fec071aabaed4fb7691a201cde03704d9", size = 150057751, upload-time = "2024-07-23T02:35:53.074Z" }, + { url = "https://files.pythonhosted.org/packages/3b/9a/72ef35b399b0e183bc2e8f6f558036922d453c4d8237dab26c666a04244b/nvidia_cusparselt_cu12-0.6.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:e5c8a26c36445dd2e6812f1177978a24e2d37cacce7e090f297a688d1ec44f46", size = 156785796, upload-time = "2024-10-15T21:29:17.709Z" }, ] [[package]] name = "nvidia-nccl-cu12" -version = "2.21.5" +version = "2.26.2" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/df/99/12cd266d6233f47d00daf3a72739872bdc10267d0383508b0b9c84a18bb6/nvidia_nccl_cu12-2.21.5-py3-none-manylinux2014_x86_64.whl", hash = "sha256:8579076d30a8c24988834445f8d633c697d42397e92ffc3f63fa26766d25e0a0", size = 188654414, upload-time = "2024-04-03T15:32:57.427Z" }, + { url = "https://files.pythonhosted.org/packages/67/ca/f42388aed0fddd64ade7493dbba36e1f534d4e6fdbdd355c6a90030ae028/nvidia_nccl_cu12-2.26.2-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:694cf3879a206553cc9d7dbda76b13efaf610fdb70a50cba303de1b0d1530ac6", size = 201319755, upload-time = "2025-03-13T00:29:55.296Z" }, ] [[package]] name = "nvidia-nvjitlink-cu12" -version = "12.4.127" +version = "12.8.61" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ff/ff/847841bacfbefc97a00036e0fce5a0f086b640756dc38caea5e1bb002655/nvidia_nvjitlink_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:06b3b9b25bf3f8af351d664978ca26a16d2c5127dbd53c0497e28d1fb9611d57", size = 21066810, upload-time = "2024-04-03T20:59:46.957Z" }, + { url = "https://files.pythonhosted.org/packages/03/f8/9d85593582bd99b8d7c65634d2304780aefade049b2b94d96e44084be90b/nvidia_nvjitlink_cu12-12.8.61-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:45fd79f2ae20bd67e8bc411055939049873bfd8fac70ff13bd4865e0b9bdab17", size = 39243473, upload-time = "2025-01-23T18:03:03.509Z" }, ] [[package]] name = "nvidia-nvtx-cu12" -version = "12.4.127" +version = "12.8.55" source = { registry = "https://pypi.org/simple" } wheels = [ - { url = "https://files.pythonhosted.org/packages/87/20/199b8713428322a2f22b722c62b8cc278cc53dffa9705d744484b5035ee9/nvidia_nvtx_cu12-12.4.127-py3-none-manylinux2014_x86_64.whl", hash = "sha256:781e950d9b9f60d8241ccea575b32f5105a5baf4c2351cab5256a24869f12a1a", size = 99144, upload-time = "2024-04-03T20:56:12.406Z" }, + { url = "https://files.pythonhosted.org/packages/8d/cd/0e8c51b2ae3a58f054f2e7fe91b82d201abfb30167f2431e9bd92d532f42/nvidia_nvtx_cu12-12.8.55-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dd0780f1a55c21d8e06a743de5bd95653de630decfff40621dbde78cc307102", size = 89896, upload-time = "2025-01-23T17:50:44.487Z" }, ] [[package]] @@ -5806,7 +5901,8 @@ dependencies = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "packaging" }, { name = "protobuf" }, - { name = "sympy" }, + { name = "sympy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "sympy", version = "1.14.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/d2/88/d9757c62a0f96b5193f8d447a141eefd14498c404cc5caf1a6f3233cf102/onnxruntime-1.24.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:79b3119ab9f4f3817062e6dbe7f4a44937de93905e3a31ba34313d18cb49e7be", size = 17212018, upload-time = "2026-02-05T17:32:13.986Z" }, @@ -5829,7 +5925,8 @@ dependencies = [ { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and platform_machine != 'aarch64'" }, { name = "packaging", marker = "platform_machine != 'aarch64'" }, { name = "protobuf", marker = "platform_machine != 'aarch64'" }, - { name = "sympy", marker = "platform_machine != 'aarch64'" }, + { name = "sympy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (platform_machine != 'aarch64' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "sympy", version = "1.14.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'aarch64' and sys_platform == 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ca/c7/07d06175f1124fc89e8b7da30d70eb8e0e1400d90961ae1cbea9da69e69b/onnxruntime_gpu-1.24.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ac4bfc90c376516b13d709764ab257e4e3d78639bf6a2ccfc826e9db4a5c7ddf", size = 252616647, upload-time = "2026-02-05T17:24:02.993Z" }, @@ -5848,8 +5945,10 @@ dependencies = [ { name = "regex" }, { name = "safetensors" }, { name = "timm" }, - { name = "torch" }, - { name = "torchvision" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "torchvision", version = "0.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "tqdm" }, ] sdist = { url = "https://files.pythonhosted.org/packages/30/46/fb8be250fa7fcfc56fbeb41583645e18d868268f67fbbbeb8ed62a8ff18a/open_clip_torch-3.2.0.tar.gz", hash = "sha256:62b7743012ccc40fb7c64819fa762fba0a13dd74585ac733babe58c2974c2506", size = 1502853, upload-time = "2025-09-21T17:32:08.289Z" } @@ -5943,9 +6042,11 @@ dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "tiktoken" }, - { name = "torch" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "tqdm" }, - { name = "triton", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine != 'aarch64' and sys_platform == 'linux2')" }, + { name = "triton", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 'aarch64' and sys_platform == 'linux2'" }, + { name = "triton", version = "3.3.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "triton", version = "3.6.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'aarch64' and sys_platform == 'linux2'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/35/8e/d36f8880bcf18ec026a55807d02fe4c7357da9f25aebd92f85178000c0dc/openai_whisper-20250625.tar.gz", hash = "sha256:37a91a3921809d9f44748ffc73c0a55c9f366c85a3ef5c2ae0cc09540432eb96", size = 803191, upload-time = "2025-06-26T01:06:13.34Z" } @@ -5974,8 +6075,8 @@ name = "opencv-python" version = "4.13.0.92" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, - { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'aarch64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')" }, ] [[package]] @@ -6261,7 +6362,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -6306,9 +6408,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -8426,7 +8530,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -8465,9 +8570,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -8506,7 +8613,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -8554,9 +8662,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -8593,6 +8703,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/21/38165845392cae67b61843a52c6455d47d0cc2a40dd495c89f4362944654/scipy-1.17.0-cp312-cp312-win_arm64.whl", hash = "sha256:f603d8a5518c7426414d1d8f82e253e454471de682ce5e39c29adb0df1efb86b", size = 24314368, upload-time = "2026-01-10T21:26:23.087Z" }, ] +[[package]] +name = "sentencepiece" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/33/ea3cb3839607eb175da835244a798f797f478c5ddf0e8ecdf57ea85a4c70/sentencepiece-0.2.2.tar.gz", hash = "sha256:3d2b5e824b5622038dc7b490897efe05ebbbb9e7350fc142f3ecc8789ef9bdf6", size = 8218435, upload-time = "2026-07-12T08:39:34.701Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/1b/e6c69e4c2026ed575d68dda2847a404468ca7b5fa684bb0b19f71d82d29d/sentencepiece-0.2.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:bc7b0b1da20f856bfac5f84b2673fe534b167e41980b27442ca8f78c2b7eb77e", size = 2180607, upload-time = "2026-07-12T08:38:01.018Z" }, + { url = "https://files.pythonhosted.org/packages/36/5a/2a1d84c87dc075d4f8cf1a2470a95399e59834e219ffb5f4285533e750d0/sentencepiece-0.2.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8b2db2056c97224e122054fd794543cde5d24b7cae28424f6e3eb79bbe08e42b", size = 1437502, upload-time = "2026-07-12T08:38:02.899Z" }, + { url = "https://files.pythonhosted.org/packages/1b/39/3d43a75dd5a22503ca5074d0d37707cabb2e4a71b4bc6e6c61be3643cc7a/sentencepiece-0.2.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f1f61592e7cabd45d49ce8cc0ef42ca655c091e037153754fb3fa59725b5914", size = 1345667, upload-time = "2026-07-12T08:38:04.657Z" }, + { url = "https://files.pythonhosted.org/packages/90/d5/a69a8cc896e7de3fe2061b08c2f33e28656f243bed8af6a2df9f5d8c3124/sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c798f0b327bac10dc95cdac77b9a197ab2bd7dd1e60ebd7586a12d918d4be711", size = 1322864, upload-time = "2026-07-12T08:38:06.49Z" }, + { url = "https://files.pythonhosted.org/packages/e4/79/dd1836df32971d4eb14ff5cb4a8b3fe4419adbeada8e81d09dc53c5c0ef0/sentencepiece-0.2.2-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44284adc6fbe9d5bdd480541431a3d93f674fa44736714d3ad4bcee8283ace7d", size = 1392757, upload-time = "2026-07-12T08:38:08.559Z" }, + { url = "https://files.pythonhosted.org/packages/26/83/c3547715c29b7e4c84a180a240267f7685dde6f9b981396f16b95405ec9d/sentencepiece-0.2.2-cp310-cp310-win_amd64.whl", hash = "sha256:1120e0791540615e650b2e9bea835bf38a7362455d8ab62dee7968219c2d79a0", size = 1245044, upload-time = "2026-07-12T08:38:10.21Z" }, + { url = "https://files.pythonhosted.org/packages/1f/55/7da03b35582a4eb276f99051109f3e3e8f176835b6d6837422e4c3a013dd/sentencepiece-0.2.2-cp310-cp310-win_arm64.whl", hash = "sha256:524e2a85c028a0d2f9935191fa751e5ef9d9bcc39616f70ab14b28d0369c9936", size = 1190467, upload-time = "2026-07-12T08:38:12.07Z" }, + { url = "https://files.pythonhosted.org/packages/20/31/f23a2efaa0210b883574001b88fa64e499f798f0848a0b610fb9b384d162/sentencepiece-0.2.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:69e9dc8078e128286ed3b975e37c837ba96e215a50c3ef9f3f8b7ab9e5a832a0", size = 2184255, upload-time = "2026-07-12T08:38:14.855Z" }, + { url = "https://files.pythonhosted.org/packages/96/f2/1ee0ccb772d71e822f625d6cb5f0ea825835e877f28a9ef299a1291df19e/sentencepiece-0.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6dd76f3e5c8b2eb8a3a3efee787bbf5b9a66e52a048fe09cab85eca33fec6790", size = 1438545, upload-time = "2026-07-12T08:38:16.674Z" }, + { url = "https://files.pythonhosted.org/packages/2a/92/3a6ea4a2c6dd9e7062698a5a33534ca0e20844883338ae9c6b9c122c1a9f/sentencepiece-0.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:443ac618c7a2a1377cf5c82581fbb849591d14e656d5e5a3e4682d4e36a34e4e", size = 1346997, upload-time = "2026-07-12T08:38:18.499Z" }, + { url = "https://files.pythonhosted.org/packages/f3/3a/7839048997c7bc0c34c57526f539f835e20c7a57dc2a99f99579b11cdbef/sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e2aae42960392d6dcb9a72d8e1e65a97294c965071b43c7b3429a42f350250e", size = 1324282, upload-time = "2026-07-12T08:38:20.342Z" }, + { url = "https://files.pythonhosted.org/packages/06/5f/9117bf854aef817ad0d0ee9310eed0308a7e529e7eaf2e80ad9cd281ef82/sentencepiece-0.2.2-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1416b92f2f010333786fe6306ed2631121d5ea492219b0841e967b6765e64107", size = 1394242, upload-time = "2026-07-12T08:38:22.976Z" }, + { url = "https://files.pythonhosted.org/packages/ab/62/9e2569867e3dcff7ad6d89642a9615b9801b5cd698abe7df3b490361f66e/sentencepiece-0.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:70d4ca6f4d06df7f0ccab6fe4f49c8a712c8c8b6847b4f0af9a0e1dbb0e0337e", size = 1246268, upload-time = "2026-07-12T08:38:24.857Z" }, + { url = "https://files.pythonhosted.org/packages/96/c9/5d781d4ef1124564a45c98b9ff25d531c10cdf568ec6314a2d1946f9251c/sentencepiece-0.2.2-cp311-cp311-win_arm64.whl", hash = "sha256:252908153eeec06c3ca3a32077e64a49d572e3d89881475b4e0f02d99d9fcc7c", size = 1190702, upload-time = "2026-07-12T08:38:26.789Z" }, + { url = "https://files.pythonhosted.org/packages/b8/13/7a562289c8d5b49ebdf3f9c1e8ab67cf14a8743b1d90c8f406bfdec36b72/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1edb10e520e4bddf74d85b0f5ae74cc2d60c2b448885080bfb618bc2b3a49f6b", size = 2188384, upload-time = "2026-07-12T08:38:28.486Z" }, + { url = "https://files.pythonhosted.org/packages/85/d1/912f14fd5eae168aba726ffb6a9a2dc1c71fe7676c53da6f5c442b886d4a/sentencepiece-0.2.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f7c06c751c19d923435a54bff4f7e66e728fad160e8da28254f133abc9725820", size = 1441553, upload-time = "2026-07-12T08:38:30.552Z" }, + { url = "https://files.pythonhosted.org/packages/bd/44/caa9cab5f261a019e2808bc5046152775dc57352ba9cbae7525e9e7a1ed4/sentencepiece-0.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:38111ed1f79268f399c505028023d5eaaf0ab4e5eafceb709468b0d3323e7838", size = 1347176, upload-time = "2026-07-12T08:38:32.211Z" }, + { url = "https://files.pythonhosted.org/packages/19/90/cd798935668cff71d309d8ff10385844ecf216b1fe454f1993ed8bf2cb91/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cbce24284f51f71d10a42b7b9c964dcb9048b28f1c8e5db40bcbcb6f428cba6a", size = 1325200, upload-time = "2026-07-12T08:38:33.689Z" }, + { url = "https://files.pythonhosted.org/packages/b6/2d/37e3da037318a70066ded0d51bc2a7f35491ae6338dd993d5eb1503fc3b5/sentencepiece-0.2.2-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c8a168b040bc61681293f79a949b5d911c8e25086f4260285b8d97ab5f1195da", size = 1397736, upload-time = "2026-07-12T08:38:35.771Z" }, + { url = "https://files.pythonhosted.org/packages/8d/11/753fca2e6b109be3ab7867abf357dfe48677fe726ae5a5363d0b54ca9450/sentencepiece-0.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:7c6e7bf684dc12145bfa685d3060beaea55139134ba848289bee514ed42e7383", size = 1248030, upload-time = "2026-07-12T08:38:37.604Z" }, + { url = "https://files.pythonhosted.org/packages/e2/0a/70efbe861ca182d7d4b6e1a20f58e043400848fa9f2915229f082e221648/sentencepiece-0.2.2-cp312-cp312-win_arm64.whl", hash = "sha256:76ff5814db72e7462dece042d7593cdf102b8ec82c2b1cc201a2add34ee3050d", size = 1187325, upload-time = "2026-07-12T08:38:39.348Z" }, +] + [[package]] name = "service-identity" version = "24.2.0" @@ -8897,14 +9036,54 @@ wheels = [ name = "sympy" version = "1.13.1" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", + "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", +] dependencies = [ - { name = "mpmath" }, + { name = "mpmath", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ca/99/5a5b6f19ff9f083671ddf7b9632028436167cd3d33e11015754e41b249a4/sympy-1.13.1.tar.gz", hash = "sha256:9cebf7e04ff162015ce31c9c6c9144daa34a93bd082f54fd8f12deca4f47515f", size = 7533040, upload-time = "2024-07-19T09:26:51.238Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/b2/fe/81695a1aa331a842b582453b605175f419fe8540355886031328089d840a/sympy-1.13.1-py3-none-any.whl", hash = "sha256:db36cdc64bf61b9b24578b6f7bab1ecdd2452cf008f34faa33776680c26d66f8", size = 6189177, upload-time = "2024-07-19T09:26:48.863Z" }, ] +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'win32'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", +] +dependencies = [ + { name = "mpmath", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + [[package]] name = "tenacity" version = "9.1.4" @@ -8972,7 +9151,8 @@ dependencies = [ { name = "orjson" }, { name = "packaging" }, { name = "pyvers" }, - { name = "torch" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/fb/65/81c5bfd5e410e908f183eb683c5d6fe284b98d3f6fd961f77065adb0f632/tensordict-0.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:5e76410e72525c47f668324d377c612ae75dcfb089555d529b88a92ece1bbd76", size = 908794, upload-time = "2026-06-04T14:46:45.983Z" }, @@ -8996,7 +9176,8 @@ source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -9033,9 +9214,11 @@ resolution-markers = [ "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", @@ -9153,8 +9336,10 @@ dependencies = [ { name = "huggingface-hub" }, { name = "pyyaml" }, { name = "safetensors" }, - { name = "torch" }, - { name = "torchvision" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "torchvision", version = "0.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f4/9d/0ea45640be447445c8664ce2b10c74f763b0b0b9ed11620d41a4d4baa10c/timm-1.0.24.tar.gz", hash = "sha256:c7b909f43fe2ef8fe62c505e270cd4f1af230dfbc37f2ee93e3608492b9d9a40", size = 2412239, upload-time = "2026-01-07T00:26:17.541Z" } wheels = [ @@ -9235,18 +9420,73 @@ wheels = [ name = "torch" version = "2.6.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", + "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", +] dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "filelock", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "fsspec", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "jinja2", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "setuptools", marker = "(python_full_version >= '3.12' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "sympy", version = "1.13.1", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "typing-extensions", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/81/aa9ab58ec10264c1abe62c8b73f5086c3c558885d6beecebf699f0dbeaeb/torch-2.6.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:6860df13d9911ac158f4c44031609700e1eba07916fff62e21e6ffa0a9e01961", size = 766685561, upload-time = "2025-01-29T16:19:12.12Z" }, + { url = "https://files.pythonhosted.org/packages/86/86/e661e229df2f5bfc6eab4c97deb1286d598bbeff31ab0cdb99b3c0d53c6f/torch-2.6.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c4f103a49830ce4c7561ef4434cc7926e5a5fe4e5eb100c19ab36ea1e2b634ab", size = 95751887, upload-time = "2025-01-29T16:27:50.77Z" }, + { url = "https://files.pythonhosted.org/packages/e5/16/ea1b7842413a7b8a5aaa5e99e8eaf3da3183cc3ab345ad025a07ff636301/torch-2.6.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:09e06f9949e1a0518c5b09fe95295bc9661f219d9ecb6f9893e5123e10696628", size = 66520221, upload-time = "2025-01-29T16:22:18.862Z" }, + { url = "https://files.pythonhosted.org/packages/78/a9/97cbbc97002fff0de394a2da2cdfa859481fdca36996d7bd845d50aa9d8d/torch-2.6.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:7979834102cd5b7a43cc64e87f2f3b14bd0e1458f06e9f88ffa386d07c7446e1", size = 766715424, upload-time = "2025-01-29T16:25:15.874Z" }, + { url = "https://files.pythonhosted.org/packages/6d/fa/134ce8f8a7ea07f09588c9cc2cea0d69249efab977707cf67669431dcf5c/torch-2.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:ccbd0320411fe1a3b3fec7b4d3185aa7d0c52adac94480ab024b5c8f74a0bf1d", size = 95759416, upload-time = "2025-01-29T16:27:38.429Z" }, + { url = "https://files.pythonhosted.org/packages/0b/fa/f33a4148c6fb46ca2a3f8de39c24d473822d5774d652b66ed9b1214da5f7/torch-2.6.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:94fc63b3b4bedd327af588696559f68c264440e2503cc9e6954019473d74ae21", size = 66530713, upload-time = "2025-01-29T16:26:38.881Z" }, + { url = "https://files.pythonhosted.org/packages/e5/35/0c52d708144c2deb595cd22819a609f78fdd699b95ff6f0ebcd456e3c7c1/torch-2.6.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:2bb8987f3bb1ef2675897034402373ddfc8f5ef0e156e2d8cfc47cacafdda4a9", size = 766624563, upload-time = "2025-01-29T16:23:19.084Z" }, + { url = "https://files.pythonhosted.org/packages/01/d6/455ab3fbb2c61c71c8842753b566012e1ed111e7a4c82e0e1c20d0c76b62/torch-2.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b789069020c5588c70d5c2158ac0aa23fd24a028f34a8b4fcb8fcb4d7efcf5fb", size = 95607867, upload-time = "2025-01-29T16:25:55.649Z" }, + { url = "https://files.pythonhosted.org/packages/81/b4/605ae4173aa37fb5aa14605d100ff31f4f5d49f617928c9f486bb3aaec08/torch-2.6.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:9a610afe216a85a8b9bc9f8365ed561535c93e804c2a317ef7fabcc5deda0989", size = 66532538, upload-time = "2025-01-29T16:24:18.976Z" }, +] + +[[package]] +name = "torch" +version = "2.7.1+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'win32'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", +] +dependencies = [ + { name = "filelock", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "fsspec", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "jinja2", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform == 'win32')" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufile-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, @@ -9254,24 +9494,18 @@ dependencies = [ { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "setuptools", marker = "python_full_version >= '3.12'" }, - { name = "sympy" }, - { name = "triton", version = "3.2.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "typing-extensions" }, + { name = "setuptools", marker = "(python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and sys_platform == 'win32')" }, + { name = "sympy", version = "1.14.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "triton", version = "3.3.1", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "typing-extensions", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/37/81/aa9ab58ec10264c1abe62c8b73f5086c3c558885d6beecebf699f0dbeaeb/torch-2.6.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:6860df13d9911ac158f4c44031609700e1eba07916fff62e21e6ffa0a9e01961", size = 766685561, upload-time = "2025-01-29T16:19:12.12Z" }, - { url = "https://files.pythonhosted.org/packages/86/86/e661e229df2f5bfc6eab4c97deb1286d598bbeff31ab0cdb99b3c0d53c6f/torch-2.6.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:c4f103a49830ce4c7561ef4434cc7926e5a5fe4e5eb100c19ab36ea1e2b634ab", size = 95751887, upload-time = "2025-01-29T16:27:50.77Z" }, - { url = "https://files.pythonhosted.org/packages/20/e0/5cb2f8493571f0a5a7273cd7078f191ac252a402b5fb9cb6091f14879109/torch-2.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:56eeaf2ecac90da5d9e35f7f35eb286da82673ec3c582e310a8d1631a1c02341", size = 204165139, upload-time = "2025-01-29T16:27:11.63Z" }, - { url = "https://files.pythonhosted.org/packages/e5/16/ea1b7842413a7b8a5aaa5e99e8eaf3da3183cc3ab345ad025a07ff636301/torch-2.6.0-cp310-none-macosx_11_0_arm64.whl", hash = "sha256:09e06f9949e1a0518c5b09fe95295bc9661f219d9ecb6f9893e5123e10696628", size = 66520221, upload-time = "2025-01-29T16:22:18.862Z" }, - { url = "https://files.pythonhosted.org/packages/78/a9/97cbbc97002fff0de394a2da2cdfa859481fdca36996d7bd845d50aa9d8d/torch-2.6.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:7979834102cd5b7a43cc64e87f2f3b14bd0e1458f06e9f88ffa386d07c7446e1", size = 766715424, upload-time = "2025-01-29T16:25:15.874Z" }, - { url = "https://files.pythonhosted.org/packages/6d/fa/134ce8f8a7ea07f09588c9cc2cea0d69249efab977707cf67669431dcf5c/torch-2.6.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:ccbd0320411fe1a3b3fec7b4d3185aa7d0c52adac94480ab024b5c8f74a0bf1d", size = 95759416, upload-time = "2025-01-29T16:27:38.429Z" }, - { url = "https://files.pythonhosted.org/packages/11/c5/2370d96b31eb1841c3a0883a492c15278a6718ccad61bb6a649c80d1d9eb/torch-2.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:46763dcb051180ce1ed23d1891d9b1598e07d051ce4c9d14307029809c4d64f7", size = 204164970, upload-time = "2025-01-29T16:26:16.182Z" }, - { url = "https://files.pythonhosted.org/packages/0b/fa/f33a4148c6fb46ca2a3f8de39c24d473822d5774d652b66ed9b1214da5f7/torch-2.6.0-cp311-none-macosx_11_0_arm64.whl", hash = "sha256:94fc63b3b4bedd327af588696559f68c264440e2503cc9e6954019473d74ae21", size = 66530713, upload-time = "2025-01-29T16:26:38.881Z" }, - { url = "https://files.pythonhosted.org/packages/e5/35/0c52d708144c2deb595cd22819a609f78fdd699b95ff6f0ebcd456e3c7c1/torch-2.6.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:2bb8987f3bb1ef2675897034402373ddfc8f5ef0e156e2d8cfc47cacafdda4a9", size = 766624563, upload-time = "2025-01-29T16:23:19.084Z" }, - { url = "https://files.pythonhosted.org/packages/01/d6/455ab3fbb2c61c71c8842753b566012e1ed111e7a4c82e0e1c20d0c76b62/torch-2.6.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b789069020c5588c70d5c2158ac0aa23fd24a028f34a8b4fcb8fcb4d7efcf5fb", size = 95607867, upload-time = "2025-01-29T16:25:55.649Z" }, - { url = "https://files.pythonhosted.org/packages/18/cf/ae99bd066571656185be0d88ee70abc58467b76f2f7c8bfeb48735a71fe6/torch-2.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:7e1448426d0ba3620408218b50aa6ada88aeae34f7a239ba5431f6c8774b1239", size = 204120469, upload-time = "2025-01-29T16:24:01.821Z" }, - { url = "https://files.pythonhosted.org/packages/81/b4/605ae4173aa37fb5aa14605d100ff31f4f5d49f617928c9f486bb3aaec08/torch-2.6.0-cp312-none-macosx_11_0_arm64.whl", hash = "sha256:9a610afe216a85a8b9bc9f8365ed561535c93e804c2a317ef7fabcc5deda0989", size = 66532538, upload-time = "2025-01-29T16:24:18.976Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:d6c3cba198dc93f93422a8545f48a6697890366e4b9701f54351fc27e2304bd3", upload-time = "2025-06-03T18:30:47Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp310-cp310-win_amd64.whl", hash = "sha256:5174f02de8ca14df87c8e333c4c39cf3ce93a323c9d470d690301d110a053b3c", upload-time = "2025-06-03T18:30:50Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c301dc280458afd95450af794924c98fe07522dd148ff384739b810e3e3179f2", upload-time = "2025-06-03T18:31:06Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp311-cp311-win_amd64.whl", hash = "sha256:138c66dcd0ed2f07aafba3ed8b7958e2bed893694990e0b4b55b6b2b4a336aa6", upload-time = "2025-06-03T18:31:13Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0b64f7d0a6f2a739ed052ba959f7b67c677028c9566ce51997f9f90fe573ddaa", upload-time = "2025-06-03T18:31:26Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torch-2.7.1%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:2bb8c05d48ba815b316879a18195d53a6472a03e297d971e916753f8e1053d30", upload-time = "2025-06-03T18:31:46Z" }, ] [[package]] @@ -9305,11 +9539,28 @@ sdist = { url = "https://files.pythonhosted.org/packages/62/9a/d3d8da1d1a8a189b2 name = "torchvision" version = "0.21.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'darwin'", + "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'darwin'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'darwin'", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'darwin'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'darwin'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32'", +] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "pillow" }, - { name = "torch" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "pillow", marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a9/20/72eb0b5b08fa293f20fc41c374e37cf899f0033076f0144d2cdc48f9faee/torchvision-0.21.0-1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:5568c5a1ff1b2ec33127b629403adb530fab81378d9018ca4ed6508293f76e2b", size = 2327643, upload-time = "2025-03-18T17:25:51.165Z" }, @@ -9318,15 +9569,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/0d/143bd264876fad17c82096b6c2d433f1ac9b29cdc69ee45023096976ee3d/torchvision-0.21.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:044ea420b8c6c3162a234cada8e2025b9076fa82504758cd11ec5d0f8cd9fa37", size = 1784140, upload-time = "2025-01-29T16:28:54.122Z" }, { url = "https://files.pythonhosted.org/packages/5e/44/32e2d2d174391374d5ff3c4691b802e8efda9ae27ab9062eca2255b006af/torchvision-0.21.0-cp310-cp310-manylinux1_x86_64.whl", hash = "sha256:b0c0b264b89ab572888244f2e0bad5b7eaf5b696068fc0b93e96f7c3c198953f", size = 7237187, upload-time = "2025-01-29T16:28:47.156Z" }, { url = "https://files.pythonhosted.org/packages/0e/6b/4fca9373eda42c1b04096758306b7bd55f7d8f78ba273446490855a0f25d/torchvision-0.21.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:54815e0a56dde95cc6ec952577f67e0dc151eadd928e8d9f6a7f821d69a4a734", size = 14699067, upload-time = "2025-01-29T16:28:36.086Z" }, - { url = "https://files.pythonhosted.org/packages/aa/f7/799ddd538b21017cbf80294c92e9efbf6db08dff6efee37c3be114a81845/torchvision-0.21.0-cp310-cp310-win_amd64.whl", hash = "sha256:abbf1d7b9d52c00d2af4afa8dac1fb3e2356f662a4566bd98dfaaa3634f4eb34", size = 1560542, upload-time = "2025-01-29T16:28:52.608Z" }, { url = "https://files.pythonhosted.org/packages/29/88/00c69db213ee2443ada8886ec60789b227e06bb869d85ee324578221a7f7/torchvision-0.21.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:110d115333524d60e9e474d53c7d20f096dbd8a080232f88dddb90566f90064c", size = 1784141, upload-time = "2025-01-29T16:28:51.207Z" }, { url = "https://files.pythonhosted.org/packages/be/a2/b0cedf0a411f1a5d75cfc0b87cde56dd1ddc1878be46a42c905cd8580220/torchvision-0.21.0-cp311-cp311-manylinux1_x86_64.whl", hash = "sha256:3891cd086c5071bda6b4ee9d266bb2ac39c998c045c2ebcd1e818b8316fb5d41", size = 7237719, upload-time = "2025-01-29T16:28:20.724Z" }, { url = "https://files.pythonhosted.org/packages/8c/a1/ee962ef9d0b2bf7a6f8b14cb95acb70e05cd2101af521032a09e43f8582f/torchvision-0.21.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:54454923a50104c66a9ab6bd8b73a11c2fc218c964b1006d5d1fe5b442c3dcb6", size = 14700617, upload-time = "2025-01-29T16:28:30.247Z" }, - { url = "https://files.pythonhosted.org/packages/88/53/4ad334b9b1d8dd99836869fec139cb74a27781298360b91b9506c53f1d10/torchvision-0.21.0-cp311-cp311-win_amd64.whl", hash = "sha256:49bcfad8cfe2c27dee116c45d4f866d7974bcf14a5a9fbef893635deae322f2f", size = 1560523, upload-time = "2025-01-29T16:28:48.751Z" }, { url = "https://files.pythonhosted.org/packages/6e/1b/28f527b22d5e8800184d0bc847f801ae92c7573a8c15979d92b7091c0751/torchvision-0.21.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:97a5814a93c793aaf0179cfc7f916024f4b63218929aee977b645633d074a49f", size = 1784140, upload-time = "2025-01-29T16:28:44.694Z" }, { url = "https://files.pythonhosted.org/packages/36/63/0722e153fd27d64d5b0af45b5c8cb0e80b35a68cf0130303bc9a8bb095c7/torchvision-0.21.0-cp312-cp312-manylinux1_x86_64.whl", hash = "sha256:b578bcad8a4083b40d34f689b19ca9f7c63e511758d806510ea03c29ac568f7b", size = 7238673, upload-time = "2025-01-29T16:28:27.631Z" }, { url = "https://files.pythonhosted.org/packages/bb/ea/03541ed901cdc30b934f897060d09bbf7a98466a08ad1680320f9ce0cbe0/torchvision-0.21.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:5083a5b1fec2351bf5ea9900a741d54086db75baec4b1d21e39451e00977f1b1", size = 14701186, upload-time = "2025-01-29T16:28:16.491Z" }, - { url = "https://files.pythonhosted.org/packages/4c/6a/c7752603060d076dfed95135b78b047dc71792630cbcb022e3693d6f32ef/torchvision-0.21.0-cp312-cp312-win_amd64.whl", hash = "sha256:6eb75d41e3bbfc2f7642d0abba9383cc9ae6c5a4ca8d6b00628c225e1eaa63b3", size = 1560520, upload-time = "2025-01-29T16:28:42.122Z" }, +] + +[[package]] +name = "torchvision" +version = "0.22.1+cu128" +source = { registry = "https://download.pytorch.org/whl/cu128" } +resolution-markers = [ + "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform == 'win32'", + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform == 'win32'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version >= '3.12' and platform_machine == 'aarch64' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and platform_machine == 'aarch64' and sys_platform == 'win32'", + "python_full_version < '3.11' and platform_machine == 'aarch64' and sys_platform == 'win32'", +] +dependencies = [ + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform == 'win32')" }, + { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.11' and sys_platform == 'win32')" }, + { name = "pillow", marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, +] +wheels = [ + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.22.1%2Bcu128-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:538f4db667286d939b4eee0a66d31ed21b51186668006b0e0ffe20338ecc7e00", upload-time = "2025-06-03T18:37:28Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.22.1%2Bcu128-cp310-cp310-win_amd64.whl", hash = "sha256:ad48ba3c3ffd48027e3a8de42fcea131a53a524ee9416ca4efb22f9ac6b7328d", upload-time = "2025-06-03T18:37:28Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.22.1%2Bcu128-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:92568ac46b13a8c88b61589800b1b9c4629be091ea7ce080fc6fc622e11e0915", upload-time = "2025-06-03T18:37:28Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.22.1%2Bcu128-cp311-cp311-win_amd64.whl", hash = "sha256:85ecd729c947151eccea502853be6efc2c0029dc26e6e5148e04684aed008390", upload-time = "2025-06-03T18:37:28Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.22.1%2Bcu128-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f64ef9bb91d71ab35d8384912a19f7419e35928685bc67544d58f45148334373", upload-time = "2025-06-03T18:37:28Z" }, + { url = "https://download-r2.pytorch.org/whl/cu128/torchvision-0.22.1%2Bcu128-cp312-cp312-win_amd64.whl", hash = "sha256:650561ba326d21021243f5e064133dc62dc64d52f79623db5cd76637a9665f96", upload-time = "2025-06-03T18:37:28Z" }, ] [[package]] @@ -9393,7 +9671,8 @@ wheels = [ [package.optional-dependencies] torch = [ { name = "accelerate" }, - { name = "torch" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, ] [[package]] @@ -9453,14 +9732,27 @@ name = "triton" version = "3.2.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", - "python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", - "python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'win32'", + "(python_full_version >= '3.12' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version >= '3.12' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "(python_full_version == '3.11.*' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version == '3.11.*' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", + "(python_full_version < '3.11' and platform_machine != 'aarch64' and platform_machine != 'x86_64' and sys_platform == 'linux') or (python_full_version < '3.11' and platform_machine != 'aarch64' and sys_platform != 'darwin' and sys_platform != 'linux' and sys_platform != 'win32')", +] + +[[package]] +name = "triton" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and platform_machine == 'x86_64' and sys_platform == 'linux'", + "python_full_version < '3.11' and platform_machine == 'x86_64' and sys_platform == 'linux'", +] +dependencies = [ + { name = "setuptools", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/01/65/3ffa90e158a2c82f0716eee8d26a725d241549b7d7aaf7e4f44ac03ebd89/triton-3.2.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3e54983cd51875855da7c68ec05c05cf8bb08df361b1d5b69e05e40b0c9bd62", size = 253090354, upload-time = "2025-01-22T19:12:21.872Z" }, - { url = "https://files.pythonhosted.org/packages/a7/2e/757d2280d4fefe7d33af7615124e7e298ae7b8e3bc4446cdb8e88b0f9bab/triton-3.2.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8009a1fb093ee8546495e96731336a33fb8856a38e45bb4ab6affd6dbc3ba220", size = 253157636, upload-time = "2025-01-22T19:12:51.322Z" }, - { url = "https://files.pythonhosted.org/packages/06/00/59500052cb1cf8cf5316be93598946bc451f14072c6ff256904428eaf03c/triton-3.2.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d9b215efc1c26fa7eefb9a157915c92d52e000d2bf83e5f69704047e63f125c", size = 253159365, upload-time = "2025-01-22T19:13:24.648Z" }, + { url = "https://files.pythonhosted.org/packages/8d/a9/549e51e9b1b2c9b854fd761a1d23df0ba2fbc60bd0c13b489ffa518cfcb7/triton-3.3.1-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b74db445b1c562844d3cfad6e9679c72e93fdfb1a90a24052b03bb5c49d1242e", size = 155600257, upload-time = "2025-05-29T23:39:36.085Z" }, + { url = "https://files.pythonhosted.org/packages/21/2f/3e56ea7b58f80ff68899b1dbe810ff257c9d177d288c6b0f55bf2fe4eb50/triton-3.3.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b31e3aa26f8cb3cc5bf4e187bf737cbacf17311e1112b781d4a059353dfd731b", size = 155689937, upload-time = "2025-05-29T23:39:44.182Z" }, + { url = "https://files.pythonhosted.org/packages/24/5f/950fb373bf9c01ad4eb5a8cd5eaf32cdf9e238c02f9293557a2129b9c4ac/triton-3.3.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9999e83aba21e1a78c1f36f21bce621b77bcaa530277a50484a7cb4a822f6e43", size = 155669138, upload-time = "2025-05-29T23:39:51.771Z" }, ] [[package]] @@ -9654,8 +9946,10 @@ dependencies = [ { name = "requests" }, { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "scipy", version = "1.17.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "torch" }, - { name = "torchvision" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, + { name = "torchvision", version = "0.21.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torchvision", version = "0.22.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, { name = "ultralytics-thop" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3c/dc/7947df41679c009bc33b61e10d6274a8ec885206b726ebb6027d5f204b35/ultralytics-8.4.14.tar.gz", hash = "sha256:360dff28ecb6cc7bf561aadf5bfe208c3900380bf1d4b2b190cb8db60e7b7626", size = 1014432, upload-time = "2026-02-10T11:31:51.342Z" } @@ -9670,7 +9964,8 @@ source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.5", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "torch" }, + { name = "torch", version = "2.6.0", source = { registry = "https://pypi.org/simple" }, marker = "(platform_machine != 'x86_64' and sys_platform == 'linux') or (sys_platform != 'linux' and sys_platform != 'win32')" }, + { name = "torch", version = "2.7.1+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "(platform_machine == 'x86_64' and sys_platform == 'linux') or sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1f/63/21a32e1facfeee245dbdfb7b4669faf7a36ff7c00b50987932bdab126f4b/ultralytics_thop-2.0.18.tar.gz", hash = "sha256:21103bcd39cc9928477dc3d9374561749b66a1781b35f46256c8d8c4ac01d9cf", size = 34557, upload-time = "2025-10-29T16:58:13.526Z" } wheels = [ From c54ae9a983fc1b3d2ede0068b348a285c5fcdee1 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 17:56:36 +0000 Subject: [PATCH 02/12] [autofix.ci] apply automated fixes --- dimos/models/embedding/siglip.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/dimos/models/embedding/siglip.py b/dimos/models/embedding/siglip.py index 5b3ff53d32..225c98a04f 100644 --- a/dimos/models/embedding/siglip.py +++ b/dimos/models/embedding/siglip.py @@ -47,11 +47,7 @@ class SigLIPModel(EmbeddingModel, HuggingFaceModel): @cached_property def _model(self) -> HFSiglipModel: self._ensure_cuda_initialized() - return ( - HFSiglipModel.from_pretrained(self.config.model_name) - .eval() - .to(self.config.device) - ) + return HFSiglipModel.from_pretrained(self.config.model_name).eval().to(self.config.device) @cached_property def _processor(self) -> SiglipProcessor: @@ -72,8 +68,7 @@ def embed(self, *images: Image) -> Embedding | list[Embedding]: image_features = functional.normalize(image_features, dim=-1) embeddings = [ - Embedding(vector=feat, timestamp=images[i].ts) - for i, feat in enumerate(image_features) + Embedding(vector=feat, timestamp=images[i].ts) for i, feat in enumerate(image_features) ] return embeddings[0] if len(images) == 1 else embeddings From 122ff2d7b2ef760b6384aa116133d9dffb09563b Mon Sep 17 00:00:00 2001 From: danvi Date: Wed, 12 Aug 2026 03:48:17 +0900 Subject: [PATCH 03/12] fix mypy; update comments; use SEED for fit_support_plane --- dimos/perception/memory/gates.py | 6 ++-- dimos/perception/memory/inventory.py | 35 ++++++++++++---------- dimos/perception/memory/localize.py | 36 ++++++++++++++--------- dimos/perception/memory/support_plane.py | 11 +++++-- dimos/perception/memory/tool_inventory.py | 23 +++++++++++++-- dimos/perception/memory/types.py | 10 +------ 6 files changed, 73 insertions(+), 48 deletions(-) diff --git a/dimos/perception/memory/gates.py b/dimos/perception/memory/gates.py index 2536a44a4d..50821a1dcc 100644 --- a/dimos/perception/memory/gates.py +++ b/dimos/perception/memory/gates.py @@ -47,8 +47,7 @@ OPTICAL_FRAME = "camera_color_optical_frame" WORLD_FRAME = "world" -# One world-pose period (9.86 Hz measured) plus margin. The 0.5 s the sketch -# used spans five samples - up to 7.5 cm of smear at peak wrist speed. +# One world-pose period plus margin. TF_TOLERANCE = 0.12 SPEED_MAX = 0.02 # m/s - camera counts as still below this @@ -223,9 +222,10 @@ def scene_still( def depth_at(store: Any, ts: float, tolerance: float = 0.06) -> Image | None: """Temporal join: aligned depth frame for a color timestamp.""" try: - return store.streams.depth_image.at(ts, tolerance).first().data + depth: Image = store.streams.depth_image.at(ts, tolerance).first().data except LookupError: return None + return depth def keyframes( diff --git a/dimos/perception/memory/inventory.py b/dimos/perception/memory/inventory.py index 66ff38e602..e108da2fe1 100644 --- a/dimos/perception/memory/inventory.py +++ b/dimos/perception/memory/inventory.py @@ -64,12 +64,10 @@ NAME_FRAMES_PER_INSTANCE = 5 NAME_SCORE_FLOOR = 0.18 # An attachment must be the detector drawing a box around this member, not a -# box that merely crosses it: a tape-grid "ruler" box overlapping a diagonal -# book's axis-aligned bbox reaches IoU 0.31, the book's own box reaches 0.95. +# box that merely crosses it. NAME_ATTACH_IOU = 0.45 -# Measured on the S1 run: a 31-class prompt dilutes the text tokens enough -# that a plainly visible class drops out of its own box (book: 0.39 in a -# 10-class prompt, absent at the same threshold in the 31-class one). +# Post-processing reports one label per box per request, so chunking is what +# lets a box carry more than one label. It changes no score. NAME_PROMPT_CAP = 10 SUPPRESS_SCORE = 0.25 SUPPRESS_OVERLAP = 0.35 @@ -124,7 +122,8 @@ def add(self, obs: SupportObservation, frame_key: float) -> None: @property def centroid(self) -> np.ndarray: - return np.median(np.stack([m.centroid for m in self.members]), axis=0) + median: np.ndarray = np.median(np.stack([m.centroid for m in self.members]), axis=0) + return median @property def aabb(self) -> tuple[np.ndarray, np.ndarray]: @@ -298,8 +297,7 @@ def _in_scope(obs: SupportObservation, plane: SupportPlane | None, policy: Inven if low < band_lo or high > band_hi: return False if not policy.include_surfaces and high < policy.min_height_above_plane_m: - # A patch of the surface itself: no volume above the plane. Tape - # lines and wood-grain segments die here; every real object rises. + # A patch of the surface itself: no volume above the plane. return False inside = plane.footprint_contains(points[:, :2]) return bool(inside.mean() >= 0.3) @@ -415,14 +413,14 @@ def _associate( cost[i, j] = 1.0 - overlap rows, cols = linear_sum_assignment(cost) - assigned = {} + assigned: dict[int, int] = {} for i, j in zip(rows, cols, strict=False): if cost[i, j] < forbidden: assigned[i] = j for i, obs in enumerate(observations): - j = assigned.get(i) - if j is not None: - tracks[j].add(obs, frame_key) + match = assigned.get(i) + if match is not None: + tracks[match].add(obs, frame_key) else: track = _Track() track.add(obs, frame_key) @@ -540,14 +538,15 @@ def _build_instance(index: int, track: _Track, grounded: bool = True) -> Instanc latest = track.latest coverage, axes_observed = _view_coverage(track.members) lo, hi = track.aabb + center = (lo + hi) / 2 extent = np.maximum(hi - lo, 0.005) centroids = np.stack([m.centroid for m in track.members]) sigma = centroids.std(axis=0) if len(track.members) > 1 else np.full(3, 0.01) support = Support( - center_xyz=tuple(float(v) for v in (lo + hi) / 2), - extent_xyz_m=tuple(float(v) for v in extent), + center_xyz=(float(center[0]), float(center[1]), float(center[2])), + extent_xyz_m=(float(extent[0]), float(extent[1]), float(extent[2])), orientation_xyzw=(0.0, 0.0, 0.0, 1.0), - sigma_xyz_m=tuple(float(v) for v in sigma), + sigma_xyz_m=(float(sigma[0]), float(sigma[1]), float(sigma[2])), coverage=coverage, axes_observed=axes_observed, frame_id="world", @@ -561,7 +560,11 @@ def _build_instance(index: int, track: _Track, grounded: bool = True) -> Instanc state="active", identity_confidence=min(1.0, distinct_views / 3.0), support=support, - latest_position_xyz=tuple(float(v) for v in latest.centroid), + latest_position_xyz=( + float(latest.centroid[0]), + float(latest.centroid[1]), + float(latest.centroid[2]), + ), latest_seen_ts=latest.ts, members=track.members, ) diff --git a/dimos/perception/memory/localize.py b/dimos/perception/memory/localize.py index c8e3fd3de2..cb13f6f167 100644 --- a/dimos/perception/memory/localize.py +++ b/dimos/perception/memory/localize.py @@ -102,7 +102,8 @@ def n_views(self) -> int: @property def extent(self) -> np.ndarray: points = np.concatenate([np.asarray(o.cloud.pointcloud.points) for o in self.observations]) - return points.max(axis=0) - points.min(axis=0) + extent: np.ndarray = points.max(axis=0) - points.min(axis=0) + return extent @dataclass @@ -168,7 +169,9 @@ def detect(self, image: Any) -> ImageDetections2D: hit = self._cache.get(key) if hit is not None: return hit - detections = self.owl.query_detections(image, [self.query], threshold=self.floor) + detections: ImageDetections2D = self.owl.query_detections( + image, [self.query], threshold=self.floor + ) detections = ImageDetections2D( image, sorted(detections.detections, key=lambda d: -d.confidence)[:BOXES_PER_FRAME], @@ -245,7 +248,8 @@ def _embed_index( ) .filter(lambda obs: obs.pose is not None) ) - return posed.transform(EmbedImages(siglip)).materialize() + embedded: Stream[Any, Any] = posed.transform(EmbedImages(siglip)).materialize() + return embedded def _retrieve( @@ -497,18 +501,18 @@ def _absorb(frame_obs: Any, is_verify: bool) -> None: orientation = _quaternion_from_matrix(np.asarray(latest.cloud.oriented_bounding_box.R)) except Exception: orientation = (0.0, 0.0, 0.0, 1.0) + center = (aabb_min + aabb_max) / 2 + extent = np.maximum(aabb_max - aabb_min, 0.005) + sigma = ( + np.stack([o.centroid for o in winner.observations]).std(axis=0) + if len(winner.observations) > 1 + else np.full(3, 0.01) + ) support = Support( - center_xyz=tuple(float(v) for v in (aabb_min + aabb_max) / 2), - extent_xyz_m=tuple(float(v) for v in np.maximum(aabb_max - aabb_min, 0.005)), + center_xyz=(float(center[0]), float(center[1]), float(center[2])), + extent_xyz_m=(float(extent[0]), float(extent[1]), float(extent[2])), orientation_xyzw=(0.0, 0.0, 0.0, 1.0), - sigma_xyz_m=tuple( - float(v) - for v in ( - np.stack([o.centroid for o in winner.observations]).std(axis=0) - if len(winner.observations) > 1 - else np.full(3, 0.01) - ) - ), + sigma_xyz_m=(float(sigma[0]), float(sigma[1]), float(sigma[2])), coverage=_azimuth_coverage(winner.observations, winner.center), axes_observed=_axes_observed(winner.observations, winner.center), frame_id="world", @@ -523,7 +527,11 @@ def _absorb(frame_obs: Any, is_verify: bool) -> None: semantic_score=winner.max_score, identity_score=min(1.0, winner.n_views / 4.0), ambiguity_margin=margin, - position_world_xyz=tuple(float(v) for v in latest.centroid), + position_world_xyz=( + float(latest.centroid[0]), + float(latest.centroid[1]), + float(latest.centroid[2]), + ), orientation_world_xyzw=orientation, frame_id="world", support=support, diff --git a/dimos/perception/memory/support_plane.py b/dimos/perception/memory/support_plane.py index c4c3dead4f..5ddffda9e3 100644 --- a/dimos/perception/memory/support_plane.py +++ b/dimos/perception/memory/support_plane.py @@ -43,6 +43,7 @@ BACKDROP_DEPTH_TRUNC = 1.5 # m - the workspace a wrist camera actually covers PLANE_DISTANCE = 0.01 # m - RANSAC inlier distance MIN_HORIZONTAL_DOT = 0.90 # |normal . z| for a plane to count as horizontal +PLANE_SEED = 0 # RANSAC is seeded so one window always fits the same plane FOOTPRINT_DILATE_M = 0.03 @@ -62,7 +63,8 @@ def normal(self) -> np.ndarray: def height_above(self, points: np.ndarray) -> np.ndarray: """Signed height of (N, 3) world points above the plane.""" a, b, c, d = self.coefficients - return points @ np.array([a, b, c]) + d + heights: np.ndarray = points @ np.array([a, b, c]) + d + return heights def footprint_contains(self, points_xy: np.ndarray) -> np.ndarray: """Boolean mask: which (N, 2) world XY points fall inside the (dilated) hull.""" @@ -118,6 +120,9 @@ def fit_support_plane( import open3d as o3d + # Unseeded, the plane fit lands on a different set of inliers each run + o3d.utility.random.seed(PLANE_SEED) + remaining = o3d.geometry.PointCloud() remaining.points = o3d.utility.Vector3dVector(points) best: tuple[np.ndarray, np.ndarray] | None = None # (coefficients, inlier points) @@ -125,7 +130,7 @@ def fit_support_plane( if len(remaining.points) < 500: break model, inlier_idx = remaining.segment_plane( - distance_threshold=PLANE_DISTANCE, ransac_n=3, num_iterations=1000 + distance_threshold=PLANE_DISTANCE, ransac_n=3, num_iterations=1000, probability=1.0 ) inliers = np.asarray(remaining.points)[inlier_idx] normal = np.array(model[:3]) @@ -146,7 +151,7 @@ def fit_support_plane( xy = inliers[:, :2] hull = ConvexHull(xy) return SupportPlane( - coefficients=tuple(float(v) for v in model), + coefficients=(float(model[0]), float(model[1]), float(model[2]), float(model[3])), footprint_hull=xy[hull.vertices], inlier_count=len(inliers), ) diff --git a/dimos/perception/memory/tool_inventory.py b/dimos/perception/memory/tool_inventory.py index 8aaaee0519..94d5fd3c2f 100644 --- a/dimos/perception/memory/tool_inventory.py +++ b/dimos/perception/memory/tool_inventory.py @@ -19,8 +19,15 @@ Stdout contract: a summary line ``instances: N`` followed by one line per instance: `` id= name= xyz=(x,y,z) ts_offset= -members=``. Exit code 0 whenever the call completes; an empty scene is -``instances: 0``, not a failure. +members= extent=(x,y,z) sigma=(x,y,z) coverage=``. Exit code 0 +whenever the call completes; an empty scene is ``instances: 0``, not a +failure. + +``extent`` is the instance's bounding size in meters and ``sigma`` the +spread of its member centroids, so a caller can check an object against a +gripper without a second query. Both are what the views actually saw: +``coverage`` is the fraction of viewing azimuth covered, and an instance +seen from one side reports the extent of that side. The instance list reports the scene as of the window's end: position and timestamp come from each instance's latest member observation. An object @@ -83,10 +90,20 @@ def main() -> int: xyz = f"({x:.3f},{y:.3f},{z:.3f})" else: xyz = "None" + if instance.support is not None: + ex, ey, ez = instance.support.extent_xyz_m + sx, sy, sz = instance.support.sigma_xyz_m + geometry = ( + f" extent=({ex:.3f},{ey:.3f},{ez:.3f})" + f" sigma=({sx:.3f},{sy:.3f},{sz:.3f})" + f" coverage={instance.support.coverage:.2f}" + ) + else: + geometry = "" print( f"{i} id={instance.instance_id} name={instance.primary_label} " f"xyz={xyz} ts_offset={instance.latest_seen_ts - lo:.1f} " - f"members={len(instance.members)}" + f"members={len(instance.members)}{geometry}" ) return 0 diff --git a/dimos/perception/memory/types.py b/dimos/perception/memory/types.py index 24fba5863f..809cd2bdcf 100644 --- a/dimos/perception/memory/types.py +++ b/dimos/perception/memory/types.py @@ -112,11 +112,6 @@ class LocalizePolicy: """ candidate_floor: float = 0.25 # form a candidate at this score - # Measured across the replay: every true positive's best view scores 0.43 - # or higher (the diagonal book is the floor); every text-only near-miss - - # gray tape rolls for "roll of black tape" at 0.31, a dark table stripe - # for "black tape" at 0.36 - stays at or under 0.36. The accept sits - # between. accept_score: float = 0.40 refusal_margin: float = 0.15 min_views: int = 2 # a support seen from one pose only is unconfirmed @@ -126,10 +121,7 @@ class LocalizePolicy: max_object_extent_m: float = 0.60 min_camera_range_m: float = 0.28 # A cloud that hugs the support surface is a patch of the surface, not an - # object: a dark wood-grain cell outlined by the tape grid reads as "black - # tape" to the detector at 0.41, but nothing about it rises above the - # table. The flattest real object here, a sticky pad, clears 5 mm at its - # 95th height percentile; surface patches stay under 2 mm. + # object: every real object rises above the plane, a surface patch does not. surface_patch_max_rise_m: float = 0.003 surface_patch_min_drop_m: float = -0.02 From d08cf7e70282eb78b204444772005532175ce090 Mon Sep 17 00:00:00 2001 From: bogwi Date: Wed, 12 Aug 2026 16:23:39 +0900 Subject: [PATCH 04/12] add NamingVocabulary that can be passed as an argument for inventory() --- dimos/perception/detection/detectors/owlv2.py | 37 ++++ dimos/perception/memory/inventory.py | 203 +++++++++++------- dimos/perception/memory/tool_inventory.py | 51 ++++- dimos/perception/memory/types.py | 16 +- 4 files changed, 227 insertions(+), 80 deletions(-) diff --git a/dimos/perception/detection/detectors/owlv2.py b/dimos/perception/detection/detectors/owlv2.py index b2d85bd31e..d88ee75b8d 100644 --- a/dimos/perception/detection/detectors/owlv2.py +++ b/dimos/perception/detection/detectors/owlv2.py @@ -18,6 +18,7 @@ from functools import cached_property +import numpy as np from PIL import Image as PILImage import torch @@ -106,6 +107,42 @@ def query_detections( return ImageDetections2D(image=image, detections=detections) + def query_score_rows( + self, + image: Image, + queries: list[str], + threshold: float = 0.1, + ) -> tuple[np.ndarray, np.ndarray]: + """Score every query against every kept box; no argmax, no label. + + Same forward pass and same per-box threshold as + ``query_detections()``, which reports one label per box because + post-processing maxes over the query axis. Here the whole + ``(n_boxes, n_queries)`` block survives, so a caller can rank + queries and refuse. Returns pixel ``(x1, y1, x2, y2)`` boxes and + their score rows. + """ + pil = PILImage.fromarray(image.to_rgb().data) + with torch.inference_mode(): + inputs = self._processor(text=[queries], images=pil, return_tensors="pt").to( + self.config.device + ) + outputs = self._model(**inputs) + results = self._processor.post_process_grounded_object_detection( + outputs=outputs, + target_sizes=torch.tensor([(pil.height, pil.width)]), + threshold=threshold, + )[0] + # sigmoid is monotonic, so this mask is the one post-processing + # applied to its max-over-queries scores: the same boxes, in order. + scores = torch.sigmoid(outputs.logits[0]) + kept = scores[scores.max(dim=-1).values > threshold].float().cpu().numpy() + + boxes = results["boxes"].float().cpu().numpy() + boxes[:, 0::2] = boxes[:, 0::2].clip(0.0, float(pil.width)) + boxes[:, 1::2] = boxes[:, 1::2].clip(0.0, float(pil.height)) + return boxes, kept + def stop(self) -> None: if "_processor" in self.__dict__: del self.__dict__["_processor"] diff --git a/dimos/perception/memory/inventory.py b/dimos/perception/memory/inventory.py index e108da2fe1..3f00a7989f 100644 --- a/dimos/perception/memory/inventory.py +++ b/dimos/perception/memory/inventory.py @@ -22,6 +22,12 @@ score), and only then name (OWLv2, labels as metadata). Labels and appearance never enter association; position and same-frame co-occurrence decide everything, which is what keeps two identical objects two instances. + +Naming is passed by a caller and abstains: it reads the full per-box score row, +groups surface strings under a canonical label, and reports that label only +when it beats the runner-up group by a margin, at the frame and again over +the frames of a track. The word list is not what makes the output safe, so +it can be domain-specific, generic, or empty. """ from __future__ import annotations @@ -62,51 +68,50 @@ KEYFRAME_STRIDE = 2.5 # s - proposal keyframe grid MAX_PROPOSALS_PER_FRAME = 40 NAME_FRAMES_PER_INSTANCE = 5 -NAME_SCORE_FLOOR = 0.18 # An attachment must be the detector drawing a box around this member, not a # box that merely crosses it. NAME_ATTACH_IOU = 0.45 -# Post-processing reports one label per box per request, so chunking is what -# lets a box carry more than one label. It changes no score. -NAME_PROMPT_CAP = 10 SUPPRESS_SCORE = 0.25 SUPPRESS_OVERLAP = 0.35 UNGROUNDED_TRACK_IOU = 0.40 -# Naming vocabulary: a generic list of common tabletop and household objects, -# batched into one capped OWLv2 prompt. The vocabulary is a naming-pass -# concern only - existence is geometric and an instance that matches nothing -# here keeps its unknown-N name with score 0. -GENERIC_VOCABULARY = [ - "pen", - "pencil", - "marker", - "highlighter", - "eraser", - "book", - "notebook", - "sticky notes", - "sheet of paper", - "roll of tape", - "scissors", - "stapler", - "ruler", - "laptop", - "computer keyboard", - "computer mouse", - "mobile phone", - "cup", - "bottle", - "drink can", - "bowl", - "cardboard box", - "cable", - "remote control", - "glasses", - "headphones", - "wallet", - "toy block", -] +# Groups of surface strings for one thing, canonical label first. Only groups +# compete, so near-synonyms reinforce instead of splitting a box's score. A +# string in two groups pins their margin at zero and both refuse. +NamingVocabulary = tuple[tuple[str, ...], ...] + +# Candidate names for callers with no domain list of their own. +DEFAULT_VOCABULARY: NamingVocabulary = ( + ("pen", "ballpoint pen", "ink pen"), + ("pencil", "wooden pencil", "mechanical pencil"), + ("marker", "marker pen", "felt-tip pen", "permanent marker"), + ("highlighter", "highlighter pen"), + ("eraser", "rubber eraser"), + ("book", "hardcover book", "paperback book", "textbook"), + ("notebook", "spiral notebook", "notepad", "writing pad"), + ("sticky notes", "post-it notes", "sticky note pad", "pad of sticky notes"), + ("sheet of paper", "piece of paper", "printed page", "document"), + ("roll of tape", "adhesive tape", "sticky tape", "roll of duct tape"), + ("scissors", "pair of scissors", "shears"), + ("stapler", "desk stapler"), + ("ruler", "measuring ruler", "straightedge"), + ("laptop", "laptop computer", "notebook computer"), + ("computer keyboard", "keyboard", "laptop keyboard"), + ("computer mouse", "mouse", "wireless mouse"), + ("mobile phone", "smartphone", "cell phone"), + ("cup", "mug", "coffee mug", "drinking cup"), + ("bottle", "water bottle", "plastic bottle"), + ("drink can", "soda can", "aluminum can"), + ("bowl", "small bowl", "cereal bowl"), + ("cardboard box", "carton", "small box"), + ("cable", "power cable", "usb cable", "cord"), + ("remote control", "tv remote", "remote"), + ("glasses", "eyeglasses", "pair of glasses", "spectacles"), + ("headphones", "earphones", "headset"), + ("wallet", "billfold", "leather wallet"), + ("toy block", "building block", "foam block"), +) +# An existence policy, not a candidate name: its own request, its own accept. SUPPRESS_QUERIES = ["person", "human hand", "human arm"] @@ -532,9 +537,25 @@ def _view_coverage(members: list[SupportObservation]) -> tuple[float, tuple[bool return coverage, observed # type: ignore[return-value] -def _build_instance(index: int, track: _Track, grounded: bool = True) -> Instance: +def _aggregated_label(labels: tuple[tuple[str, float], ...], policy: InventoryPolicy) -> str | None: + """The track's name: its best canonical group, if the margin holds again. + + The per-frame margin referees only candidates competing inside one view. + Two views can each accept a different group cleanly, and that + disagreement would otherwise reach ``primary_label`` unrefereed. + """ + if not labels: + return None + if len(labels) > 1 and labels[0][1] - labels[1][1] < policy.name_refusal_margin: + return None + return labels[0][0] + + +def _build_instance( + index: int, track: _Track, policy: InventoryPolicy, grounded: bool = True +) -> Instance: labels = tuple(sorted(track.labels.items(), key=lambda kv: -kv[1])) - primary = labels[0][0] if labels else None + primary = _aggregated_label(labels, policy) latest = track.latest coverage, axes_observed = _view_coverage(track.members) lo, hi = track.aabb @@ -594,16 +615,47 @@ def _naming_picks(track: _Track) -> list[SupportObservation]: return picks +def _flatten(vocabulary: NamingVocabulary) -> tuple[list[str], np.ndarray, list[str]]: + """Group table to a query list, the group start offsets, and the canonicals.""" + queries = [surface for group in vocabulary for surface in group] + starts = np.cumsum([0] + [len(group) for group in vocabulary[:-1]]) + return queries, starts, [group[0] for group in vocabulary] + + +def _accepted_groups( + scores: np.ndarray, starts: np.ndarray, policy: InventoryPolicy +) -> tuple[np.ndarray, np.ndarray]: + """Per box, the winning group and its score - or -1 where the margin refuses. + + A group's claim is its best surface string, since strings of one group + have unequal detector affinity. The accept floor is already applied: it + is the threshold the request was made with. + """ + groups = np.maximum.reduceat(scores, starts, axis=1) + best = groups.argmax(axis=1) + top = groups[np.arange(len(groups)), best] + if groups.shape[1] == 1: + return best, top + runner_up = np.partition(groups, -2, axis=1)[:, -2] + return np.where(top - runner_up >= policy.name_refusal_margin, best, -1), top + + def _name_and_suppress( tracks: list[_Track], tracks_2d: list[_Track2D], store: Any, + vocabulary: NamingVocabulary, + policy: InventoryPolicy, ) -> None: """OWLv2 naming per instance on keyframes, person/hand suppressing observations. Runs after association by construction: association consumed unnamed supports, so per-view label instability cannot starve existence or split an instance. A naming failure degrades names, never counts. + + Naming reads whole score rows, so one request per frame carries every + candidate name. Suppression is an existence decision and keeps its own + request, which runs whether or not there is a vocabulary. """ from dimos.perception.detection.detectors.owlv2 import Owlv2Detector @@ -619,54 +671,51 @@ def _name_and_suppress( if not frame_members and not frame_members_2d: return + queries, starts, canonical = _flatten(vocabulary) owl = Owlv2Detector() - chunks = [ - GENERIC_VOCABULARY[i : i + NAME_PROMPT_CAP] - for i in range(0, len(GENERIC_VOCABULARY), NAME_PROMPT_CAP) - ] - chunks.append(list(SUPPRESS_QUERIES)) - suppress_set = set(SUPPRESS_QUERIES) all_ts = sorted(set(frame_members) | set(frame_members_2d)) logger.info( f"naming: OWLv2 over {len(all_ts)} keyframes, " - f"{len(chunks)} prompts of <= {NAME_PROMPT_CAP} classes" + f"{len(canonical)} groups of {len(queries)} queries" ) for ts in all_ts: try: image = store.streams.color_image.at(ts, 0.05).first().data except LookupError: continue - detections = [ - det - for chunk in chunks - for det in owl.query_detections(image, chunk, threshold=NAME_SCORE_FLOOR) - ] - for det in detections: - if det.name in suppress_set: - if det.confidence < SUPPRESS_SCORE: + + if queries: + boxes, scores = owl.query_score_rows(image, queries, threshold=policy.name_accept_score) + winners, top = _accepted_groups(scores, starts, policy) + for box, group, score in zip(boxes, winners, top, strict=True): + if group < 0: continue + bbox = (float(box[0]), float(box[1]), float(box[2]), float(box[3])) + best_target: Any = None + best_iou = NAME_ATTACH_IOU for track, member in frame_members.get(ts, []): if member.bbox is None: continue - inside = _mask_overlap_fraction_bbox(member.bbox, det.bbox) - if inside >= SUPPRESS_OVERLAP and member in track.members: - track.members.remove(member) - continue - best_target: Any = None - best_iou = NAME_ATTACH_IOU + iou = _bbox_iou(member.bbox, bbox) + if iou > best_iou: + best_target, best_iou = track, iou + for track2d, det2d in frame_members_2d.get(ts, []): + iou = _bbox_iou(det2d.bbox, bbox) + if iou > best_iou: + best_target, best_iou = track2d, iou + if best_target is not None: + label = canonical[group] + best_target.labels[label] = max( + best_target.labels.get(label, 0.0), float(score) + ) + + for det in owl.query_detections(image, SUPPRESS_QUERIES, threshold=SUPPRESS_SCORE): for track, member in frame_members.get(ts, []): if member.bbox is None: continue - iou = _bbox_iou(member.bbox, det.bbox) - if iou > best_iou: - best_target, best_iou = track, iou - for track2d, det2d in frame_members_2d.get(ts, []): - iou = _bbox_iou(det2d.bbox, det.bbox) - if iou > best_iou: - best_target, best_iou = track2d, iou - if best_target is not None: - previous = best_target.labels.get(det.name, 0.0) - best_target.labels[det.name] = max(previous, det.confidence) + inside = _mask_overlap_fraction_bbox(member.bbox, det.bbox) + if inside >= SUPPRESS_OVERLAP and member in track.members: + track.members.remove(member) owl.stop() @@ -685,6 +734,7 @@ def _mask_overlap_fraction_bbox( def inventory( store: Any, *, + naming_vocabulary: NamingVocabulary, after: float | None = None, before: float | None = None, include_ungrounded: bool = False, @@ -703,6 +753,11 @@ def inventory( once per rest position; linking rest positions of one object is cross-time identity and out of scope here. + ``naming_vocabulary`` supplies candidate names and nothing else: an + instance whose best canonical group misses the refusal margin keeps its + ``unknown-N`` name. The empty tuple is a supported mode - discovery and + suppression run and every instance is ``unknown-N``. + ``log_progress`` enables per-keyframe discovery lines (``discovery: i/n ts_offset=… prop=… scope=… …s``). Off by default. """ @@ -812,14 +867,14 @@ def inventory( tracks_2d = _track_ungrounded(frames_ungrounded) if include_ungrounded else [] logger.info(f"association: {len(tracks)} grounded instances") - _name_and_suppress(tracks, tracks_2d, store) + _name_and_suppress(tracks, tracks_2d, store, naming_vocabulary, policy) tracks = [t for t in tracks if len(t.members) >= policy.min_member_observations] tracks.sort(key=lambda t: min(m.ts for m in t.members)) instances: list[Instance] = [] unknown = 0 for index, track in enumerate(tracks): - instance = _build_instance(index, track) + instance = _build_instance(index, track, policy) if instance.primary_label is None: instance.primary_label = f"unknown-{unknown}" unknown += 1 @@ -828,7 +883,7 @@ def inventory( if include_ungrounded: for track2d in tracks_2d: labels = tuple(sorted(track2d.labels.items(), key=lambda kv: -kv[1])) - primary = labels[0][0] if labels else None + primary = _aggregated_label(labels, policy) if primary is None: primary = f"unknown-{unknown}" unknown += 1 diff --git a/dimos/perception/memory/tool_inventory.py b/dimos/perception/memory/tool_inventory.py index 94d5fd3c2f..5418e62683 100644 --- a/dimos/perception/memory/tool_inventory.py +++ b/dimos/perception/memory/tool_inventory.py @@ -15,7 +15,15 @@ """Enumerate deduplicated object instances in a recording window. Run: uv run python -m dimos.perception.memory.tool_inventory - [--from ] [--duration ] [--include-ungrounded] [--log-progress] + [--from ] [--duration ] [--labels ...] [--no-vocabulary] + [--include-ungrounded] [--log-progress] + +``--labels`` replaces the candidate names with the caller's own. Each token +is one group: the first phrase is the canonical label, and ``|`` separates +synonyms (``"pen|ballpoint pen|ink pen"``). With none given the run uses +``DEFAULT_VOCABULARY``, and ``--no-vocabulary`` names nothing. Naming +abstains either way: a name is reported only when it beats its runner-up by +``InventoryPolicy.name_refusal_margin``, else the instance is ``unknown-N``. Stdout contract: a summary line ``instances: N`` followed by one line per instance: `` id= name= xyz=(x,y,z) ts_offset= @@ -41,10 +49,25 @@ import sys from dimos.memory2.store.sqlite import SqliteStore -from dimos.perception.memory.inventory import inventory +from dimos.perception.memory.inventory import DEFAULT_VOCABULARY, NamingVocabulary, inventory from dimos.utils.data import get_data +def labels_to_vocabulary(tokens: list[str]) -> NamingVocabulary: + """Parse ``--labels`` tokens into synonym groups. + + Each token is one group. Phrases inside a token are separated by ``|``. + The first non-empty phrase is the canonical label. + """ + groups: list[tuple[str, ...]] = [] + for token in tokens: + surfaces = tuple(part.strip() for part in token.split("|") if part.strip()) + if not surfaces: + raise ValueError(f"empty --labels group: {token!r}") + groups.append(surfaces) + return tuple(groups) + + def main() -> int: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter @@ -54,6 +77,22 @@ def main() -> int: "--from", dest="start", type=float, default=0.0, help="start offset into the recording (s)" ) parser.add_argument("--duration", type=float, default=None, help="how much to parse (s)") + vocab = parser.add_mutually_exclusive_group() + vocab.add_argument( + "--labels", + nargs="+", + metavar="GROUP", + help=( + "candidate name groups; each token is one group, " + "'|' separates synonyms (first phrase is canonical); " + "default is DEFAULT_VOCABULARY" + ), + ) + vocab.add_argument( + "--no-vocabulary", + action="store_true", + help="name nothing: every instance stays unknown-N", + ) parser.add_argument( "--include-ungrounded", action="store_true", @@ -75,8 +114,16 @@ def main() -> int: after = lo + args.start before = lo + args.start + args.duration if args.duration is not None else None + if args.no_vocabulary: + naming_vocabulary: NamingVocabulary = () + elif args.labels: + naming_vocabulary = labels_to_vocabulary(args.labels) + else: + naming_vocabulary = DEFAULT_VOCABULARY + instances = inventory( store, + naming_vocabulary=naming_vocabulary, after=after, before=before, include_ungrounded=args.include_ungrounded, diff --git a/dimos/perception/memory/types.py b/dimos/perception/memory/types.py index 809cd2bdcf..328590d71d 100644 --- a/dimos/perception/memory/types.py +++ b/dimos/perception/memory/types.py @@ -128,11 +128,13 @@ class LocalizePolicy: @dataclass(frozen=True) class InventoryPolicy: - """Physical thresholds for discovery, validity, scope and association. + """Thresholds for discovery, validity, scope, association and naming. - Every quantity is metric (meters, seconds, pixels, IoU) - a claim that - can be checked against the recording, unlike an appearance-similarity - threshold. + Every geometric quantity is metric (meters, seconds, pixels, IoU) - a + claim that can be checked against the recording. The two naming numbers + are detector scores and decide whether a name is reported, never whether + an instance exists. The candidate names are data and travel as a call + argument, not as policy. """ min_mask_area_px: int = 400 @@ -156,6 +158,12 @@ class InventoryPolicy: # from a second pose or moment, so it never becomes an instance. min_member_observations: int = 2 + # Naming abstention: a name is reported only when it clears the accept + # floor and beats the runner-up canonical group by the margin, at the + # frame and again over a track. Otherwise the instance stays unknown-N. + name_accept_score: float = 0.18 + name_refusal_margin: float = 0.06 + include_object_parts: bool = False include_surfaces: bool = False include_containers: bool = True From 0da73845a6c396f764e717c7481fcb45ecc89c13 Mon Sep 17 00:00:00 2001 From: bogwi Date: Thu, 13 Aug 2026 21:54:11 +0900 Subject: [PATCH 05/12] make `tool_inventory` to record the obseravation window to an .rrd file; also improve `inventory()` --- dimos/perception/memory/inventory.py | 40 +++++- dimos/perception/memory/tool_inventory.py | 141 +++++++++++++++++++++- 2 files changed, 174 insertions(+), 7 deletions(-) diff --git a/dimos/perception/memory/inventory.py b/dimos/perception/memory/inventory.py index 3f00a7989f..65ad676504 100644 --- a/dimos/perception/memory/inventory.py +++ b/dimos/perception/memory/inventory.py @@ -74,6 +74,11 @@ SUPPRESS_SCORE = 0.25 SUPPRESS_OVERLAP = 0.35 UNGROUNDED_TRACK_IOU = 0.40 +# The majority of a candidate's points must lie within the error envelope of +# the track's accumulated support. Partial and newly revealed views of one +# object satisfy this; a different object placed at a vacated rest position +# does not, which is what AABB overlap cannot express at tabletop scale. +SUPPORT_EXPLAINED = 0.5 # Groups of surface strings for one thing, canonical label first. Only groups # compete, so near-synonyms reinforce instead of splitting a box's score. A @@ -120,10 +125,13 @@ class _Track: members: list[SupportObservation] = field(default_factory=list) frame_ts: set[float] = field(default_factory=set) labels: dict[str, float] = field(default_factory=dict) + support_pts: np.ndarray = field(default_factory=lambda: np.empty((0, 3))) def add(self, obs: SupportObservation, frame_key: float) -> None: self.members.append(obs) self.frame_ts.add(frame_key) + points = np.asarray(obs.cloud.pointcloud.points) + self.support_pts = np.vstack([self.support_pts, points[:: max(1, len(points) // 400)]]) @property def centroid(self) -> np.ndarray: @@ -335,6 +343,16 @@ def _cloud_gap(a: SupportObservation, b: SupportObservation) -> float: return float(distances.min()) +def _support_explained(points: np.ndarray, support: np.ndarray, pad: float) -> float: + """Fraction of ``points`` lying within ``pad`` of the accumulated support.""" + from scipy.spatial import cKDTree + + sample = points[:: max(1, len(points) // 800)] + tree = cKDTree(support[:: max(1, len(support) // 4000)]) + distances, _ = tree.query(sample, k=1) + return float((distances <= pad).mean()) + + def _absorb_into(target: SupportObservation, obs: SupportObservation) -> None: target.aabb_min = np.minimum(target.aabb_min, obs.aabb_min) target.aabb_max = np.maximum(target.aabb_max, obs.aabb_max) @@ -383,8 +401,9 @@ def _associate( Per frame, observations assign one-to-one to existing tracks - the same-frame constraint is structural, no score overrides it. A pair is forbidden outright (infinite cost) when the supports are farther apart - than the search radius, their envelopes do not overlap enough, or their - sizes are incompatible beyond measurement error. + than the search radius, their envelopes do not overlap enough, their + sizes are incompatible beyond measurement error, or the observation is + not majority-explained by the track's accumulated support. """ from scipy.optimize import linear_sum_assignment @@ -402,6 +421,7 @@ def _associate( cost = np.full((len(observations), len(tracks)), forbidden) for i, obs in enumerate(observations): + obs_points = np.asarray(obs.cloud.pointcloud.points) for j, track in enumerate(tracks): distance = float(np.linalg.norm(obs.centroid - track.centroid)) if distance > policy.search_radius_m: @@ -415,6 +435,9 @@ def _associate( ) if overlap < policy.overlap_accept: continue + explained = _support_explained(obs_points, track.support_pts, policy.envelope_pad_m) + if explained < SUPPORT_EXPLAINED: + continue cost[i, j] = 1.0 - overlap rows, cols = linear_sum_assignment(cost) @@ -463,9 +486,10 @@ def _merge_tracks(tracks: list[_Track], policy: InventoryPolicy) -> list[_Track] """Collapse fragmented tracks of one support. Tracks merge when they never share a frame (the same-frame veto at - instance level) and their supports overlap within the envelope - or when - they do share frames but were demonstrably pieces of one body in every - one of them. Runs to a fixed point. + instance level), their supports overlap within the envelope and either + accumulated support majority-explains the other - or when they do share + frames but were demonstrably pieces of one body in every one of them. + Runs to a fixed point. """ changed = True while changed: @@ -484,6 +508,12 @@ def _merge_tracks(tracks: list[_Track], policy: InventoryPolicy) -> list[_Track] overlap = aabb_overlap(a_lo, a_hi, b_lo, b_hi, pad=policy.envelope_pad_m) if overlap < policy.overlap_accept: continue + explained = max( + _support_explained(a.support_pts, b.support_pts, policy.envelope_pad_m), + _support_explained(b.support_pts, a.support_pts, policy.envelope_pad_m), + ) + if explained < SUPPORT_EXPLAINED: + continue for obs in b.members: a.add(obs, obs.ts) tracks.pop(j) diff --git a/dimos/perception/memory/tool_inventory.py b/dimos/perception/memory/tool_inventory.py index 5418e62683..2ebd79dfbc 100644 --- a/dimos/perception/memory/tool_inventory.py +++ b/dimos/perception/memory/tool_inventory.py @@ -14,7 +14,7 @@ """Enumerate deduplicated object instances in a recording window. -Run: uv run python -m dimos.perception.memory.tool_inventory +Run: uv run python -m dimos.perception.memory.tool_inventory [out.rrd] [--from ] [--duration ] [--labels ...] [--no-vocabulary] [--include-ungrounded] [--log-progress] @@ -42,14 +42,24 @@ moved between rest positions inside the window registers once per rest position - linking rest positions of one object across time is re-identification, which this tool does not do. + +The .rrd holds the same instances the stdout lines report, no second +perception pass: the member clouds on the timeline, one labeled box per +instance in 3D, and each member's pixel box on the camera view at the +keyframe it came from. """ import argparse from pathlib import Path import sys +from typing import Any, cast from dimos.memory2.store.sqlite import SqliteStore +from dimos.memory2.tf import StreamTF +from dimos.memory2.transform import throttle +from dimos.perception.memory import gates from dimos.perception.memory.inventory import DEFAULT_VOCABULARY, NamingVocabulary, inventory +from dimos.perception.memory.types import Instance, SupportObservation from dimos.utils.data import get_data @@ -68,10 +78,134 @@ def labels_to_vocabulary(tokens: list[str]) -> NamingVocabulary: return tuple(groups) +def instance_label(instance: Instance) -> str: + """``obj-NN name score`` - the score only when that name won the instance.""" + if instance.labels and instance.labels[0][0] == instance.primary_label: + return f"{instance.instance_id} {instance.primary_label} {instance.labels[0][1]:.2f}" + return f"{instance.instance_id} {instance.primary_label}" + + +def render(out: str, store: Any, instances: list[Instance], t0: float, t1: float) -> None: + """Write the .rrd - rerun stays an inline import. + + Entity contract: ``map`` backdrop, ``camera/image`` the live feed carrying + the per-keyframe pixel boxes, ``instances/_`` the member clouds + with a static labeled box. One color per instance across both views. + """ + import rerun as rr + import rerun.blueprint as rrb + + from dimos.memory2.vis.color import Color + from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 + from dimos.visualization.rerun.init import rerun_init + + tf = StreamTF.from_store(store) + assert tf is not None + camera_info = store.streams.camera_info.first().data + + rerun_init("memory-inventory") + rr.save(out) + rr.send_blueprint( + rrb.Blueprint( + rrb.Horizontal( + rrb.Spatial3DView(origin="/", name="Scene"), + rrb.Spatial2DView(origin="camera", name="Live"), + column_shares=[2, 1], + ) + ) + ) + + POINT_SIZE = 0.005 + + def at(ts: float) -> None: + rr.set_time("ts", timestamp=ts) + + grounded = [instance for instance in instances if instance.support is not None] + colors = [ + list(Color.from_cmap("turbo", i / max(len(grounded) - 1, 1)).rgb_u8()) + for i in range(len(grounded)) + ] + labels = [instance_label(instance) for instance in grounded] + paths = [ + f"instances/{instance.instance_id}_{cast('str', instance.primary_label).replace(' ', '_')}" + for instance in grounded + ] + + frames: dict[float, list[tuple[int, SupportObservation]]] = {} + for i, instance in enumerate(grounded): + for member in instance.members: + frames.setdefault(member.ts, []).append((i, member)) + + # scene backdrop from the last keyframe that carried an instance + backdrop_ts = max(frames, default=None) + if backdrop_ts is not None: + color = store.streams.color_image.at(backdrop_ts, 0.1).first().data + depth = gates.depth_at(store, backdrop_ts) + transform = tf.get(gates.OPTICAL_FRAME, gates.WORLD_FRAME, backdrop_ts, gates.TF_TOLERANCE) + assert depth is not None and transform is not None + backdrop = PointCloud2.from_rgbd(color, depth, camera_info, depth_scale=0.001).transform( + -transform + ) + rr.log("map", backdrop.voxel_downsample(0.01).to_rerun(voxel_size=POINT_SIZE), static=True) + + # live camera feed + frustum; the empty box clears the overlay off non-keyframes + rr.log("camera", camera_info.to_rerun(), static=True) + feed_throttle = 0.1 if (t1 - t0) <= 160 else 0.4 + feed = store.streams.color_image.after(t0).before(t1).transform(throttle(feed_throttle)) + for obs in feed: + pose = gates.camera_pose(tf, obs.ts) + if pose is None: + continue + at(obs.ts) + rr.log("camera/image", obs.data.to_rerun()) + rr.log("camera", pose.to_rerun()) + rr.log("camera/image/instances", rr.Boxes2D(array=[], array_format=rr.Box2DFormat.XYXY)) + + # keyframes, logged after the feed so their boxes win the shared timestamps + for ts, entries in sorted(frames.items()): + at(ts) + keyframe_pose = gates.camera_pose(tf, ts) + assert keyframe_pose is not None + rr.log("camera/image", store.streams.color_image.at(ts, 0.05).first().data.to_rerun()) + rr.log("camera", keyframe_pose.to_rerun()) + rr.log( + "camera/image/instances", + rr.Boxes2D( + array=[ + cast("tuple[float, float, float, float]", member.bbox) for _, member in entries + ], + array_format=rr.Box2DFormat.XYXY, + labels=[labels[i] for i, _ in entries], + colors=[colors[i] for i, _ in entries], + ), + ) + for i, member in entries: + rr.log(paths[i], member.cloud.to_rerun(voxel_size=POINT_SIZE, colors=colors[i])) + + # the reported instance: one labeled box, static so it holds over the whole timeline + for i, instance in enumerate(grounded): + support = instance.support + assert support is not None + cx, cy, cz = support.center_xyz + ex, ey, ez = support.extent_xyz_m + rr.log( + f"{paths[i]}/box", + rr.Boxes3D( + centers=[(cx, cy, cz)], + half_sizes=[(ex / 2, ey / 2, ez / 2)], + colors=[colors[i]], + labels=[labels[i]], + fill_mode=rr.components.FillMode.MajorWireframe, + ), + static=True, + ) + + def main() -> int: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) + parser.add_argument("out", nargs="?", default="inventory.rrd") parser.add_argument("--dataset", type=Path, help="memory2 recording database") parser.add_argument( "--from", dest="start", type=float, default=0.0, help="start offset into the recording (s)" @@ -110,7 +244,7 @@ def main() -> int: "xarm6_worldbelief_20260729_203624_161992.db" ) store = SqliteStore(path=dataset) - lo, _ = store.streams.color_image.get_time_range() + lo, hi = store.streams.color_image.get_time_range() after = lo + args.start before = lo + args.start + args.duration if args.duration is not None else None @@ -152,6 +286,9 @@ def main() -> int: f"xyz={xyz} ts_offset={instance.latest_seen_ts - lo:.1f} " f"members={len(instance.members)}{geometry}" ) + + render(args.out, store, instances, after, before if before is not None else hi) + print(f"saved {args.out}") return 0 From 32c8e9e565d0bb86e6a9c233378a5ac08951470f Mon Sep 17 00:00:00 2001 From: bogwi Date: Fri, 14 Aug 2026 00:23:31 +0900 Subject: [PATCH 06/12] improve localize() --- dimos/perception/memory/localize.py | 64 ++++---- dimos/perception/memory/tool_localize.py | 187 +++++++++++++---------- 2 files changed, 142 insertions(+), 109 deletions(-) diff --git a/dimos/perception/memory/localize.py b/dimos/perception/memory/localize.py index cb13f6f167..165442d755 100644 --- a/dimos/perception/memory/localize.py +++ b/dimos/perception/memory/localize.py @@ -50,6 +50,9 @@ from dimos_lcm.sensor_msgs import CameraInfo from dimos.memory2.stream import Stream + from dimos.models.embedding.siglip import SigLIPModel + from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter + from dimos.perception.detection.detectors.owlv2 import Owlv2Detector from dimos.perception.detection.type.detection3d.pointcloud import Detection3DPC from dimos.protocol.tf.tf import TFLookup @@ -225,17 +228,24 @@ def _lift( return valid -def _embed_index( +def embed_index( store: Any, - tf: TFLookup, + siglip: SigLIPModel, t0: float, t1: float, - siglip: Any, - optical_frame: str, - world_frame: str, - tf_tolerance: float, + *, + optical_frame: str = OPTICAL_FRAME, + world_frame: str = WORLD_FRAME, + tf_tolerance: float = TF_TOLERANCE, ) -> Stream[Any, Any]: - """SigLIP-embedded, world-posed frame index at EMBED_HZ over the window.""" + """SigLIP-embedded, world-posed frame index at EMBED_HZ over the window. + + Built once per window and handed to every ``localize`` call on it: the + embed forwards are what a second query would otherwise repeat. + """ + tf = StreamTF.from_store(store) + if tf is None: + raise ValueError("recording has no tf stream") posed = ( store.streams.color_image.after(t0) .before(t1) @@ -249,6 +259,7 @@ def _embed_index( .filter(lambda obs: obs.pose is not None) ) embedded: Stream[Any, Any] = posed.transform(EmbedImages(siglip)).materialize() + logger.info(f"index: {embedded.count()} frames embedded over {t1 - t0:.1f}s") return embedded @@ -256,8 +267,6 @@ def _retrieve( index: Stream[Any, Any], tf: TFLookup, query_embedding: Any, - t0: float, - t1: float, optical_frame: str, world_frame: str, tf_tolerance: float, @@ -276,6 +285,7 @@ def _retrieve( if not ranked: return [] + t0, t1 = index.get_time_range() bands = max(1, min(TIME_BANDS, int((t1 - t0) / 20))) per_band = max(1, TOP_FRAMES // bands) span = (t1 - t0) / bands @@ -302,8 +312,10 @@ def localize( store: Any, query: str, *, - after: float | None = None, - before: float | None = None, + index: Stream[Any, Any], + siglip: SigLIPModel, + owl: Owlv2Detector, + segmenter: EdgeTAMImageSegmenter, require_pose: bool = True, policy: LocalizePolicy | None = None, cloud_mode: str = "latest_visible", @@ -319,26 +331,22 @@ def localize( no valid depth and ``require_pose`` holds. An ambiguity between coexisting candidates is returned with ``ambiguity_margin`` below ``refusal_margin`` - a flagged hit, never a silent guess. + + The index and the three models belong to the caller: nothing here is + loaded or stopped, so one process can call this repeatedly on warm + weights, and every query on one window reuses the same embeddings. The + window is the index's - build it with :func:`embed_index`. """ policy = policy or LocalizePolicy() tf = StreamTF.from_store(store) if tf is None: raise ValueError("recording has no tf stream") camera_info = store.streams.camera_info.first().data - lo, hi = store.streams.color_image.get_time_range() - t0 = after if after is not None else lo - t1 = before if before is not None else hi - logger.info(f"localize '{query}': window {t0 - lo:.1f}s..{t1 - lo:.1f}s") - # Pass 1 - SigLIP resident: embed the window, rank frames by the query. - from dimos.models.embedding.siglip import SigLIPModel - - siglip = SigLIPModel() - index = _embed_index(store, tf, t0, t1, siglip, optical_frame, world_frame, tf_tolerance) + # Pass 1 - SigLIP: rank the indexed frames by the query. query_embedding = siglip.embed_text(query) - frames = _retrieve(index, tf, query_embedding, t0, t1, optical_frame, world_frame, tf_tolerance) - siglip.stop() - logger.info(f"retrieval: {len(frames)} candidate frames of {index.count()} embedded") + frames = _retrieve(index, tf, query_embedding, optical_frame, world_frame, tf_tolerance) + logger.info(f"localize '{query}': {len(frames)} candidate frames of {index.count()} embedded") if not frames: return None @@ -348,12 +356,7 @@ def localize( store, tf, camera_info, frames, optical_frame, world_frame, tf_tolerance ) - # Pass 2 - OWLv2 + EdgeTAM resident: detect, segment, lift, verify. - from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter - from dimos.perception.detection.detectors.owlv2 import Owlv2Detector - - owl = Owlv2Detector() - segmenter = EdgeTAMImageSegmenter() + # Pass 2 - OWLv2 + EdgeTAM: detect, segment, lift, verify. cache = _DetectionCache(owl, segmenter, query, policy.candidate_floor) clusters: list[_Cluster] = [] @@ -454,9 +457,6 @@ def _absorb(frame_obs: Any, is_verify: bool) -> None: ) ) - owl.stop() - del segmenter - if not verified: if ungrounded_best is not None and ungrounded_best[0] >= policy.accept_score: if require_pose: diff --git a/dimos/perception/memory/tool_localize.py b/dimos/perception/memory/tool_localize.py index 7151085ddb..b37a8215b6 100644 --- a/dimos/perception/memory/tool_localize.py +++ b/dimos/perception/memory/tool_localize.py @@ -12,38 +12,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Query memory for an object, localize it in 3D via depth, render in rerun. +"""Query memory for objects, localize them in 3D via depth, render in rerun. -Run: uv run python -m dimos.perception.memory.tool_localize [query] [out.rrd] +Run: uv run python -m dimos.perception.memory.tool_localize [query ...] [out.rrd] [--from ] [--duration ] -Exit code 0 with a printed position on a verified hit; exit code 1 with -"no verified detection of ..." when the honest answer is that the object is -not there. An ambiguous hit (identical twins in view) is printed with its +Queries share one model load and one .rrd. Exit code 0 with a printed +position per verified hit; exit code 1 when no query is verified, with +"no verified detection of ..." per miss - the honest answer that the object +is not there. An ambiguous hit (identical twins in view) is printed with its ambiguity margin flagged. """ import argparse from pathlib import Path import sys -from typing import Any +from typing import Any, cast from dimos.memory2.store.sqlite import SqliteStore from dimos.memory2.tf import StreamTF from dimos.memory2.transform import throttle from dimos.perception.memory import gates -from dimos.perception.memory.localize import LocalizeTrace, localize +from dimos.perception.memory.localize import LocalizeTrace, embed_index, localize from dimos.utils.data import get_data REFUSAL_MARGIN = 0.15 -def render(out: str, store: Any, trace: LocalizeTrace, t0: float, t1: float) -> None: +def render( + out: str, store: Any, traces: list[tuple[str, LocalizeTrace]], t0: float, t1: float +) -> None: """Write the .rrd - rerun stays an inline import. Entity contract (the acceptance color cheat sheet): ``map`` backdrop, - ``detections/matched/*`` green, ``detections/verified/*`` red, - ``detections/answer`` always blue. + then one subtree per query - ``detections//matched/*`` green, + ``detections//verified/*`` red, ``detections//answer`` + always blue. """ import rerun as rr import rerun.blueprint as rrb @@ -51,7 +55,7 @@ def render(out: str, store: Any, trace: LocalizeTrace, t0: float, t1: float) -> from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 from dimos.visualization.rerun.init import rerun_init - tf = StreamTF.from_store(store) + tf = cast("StreamTF", StreamTF.from_store(store)) camera_info = store.streams.camera_info.first().data rerun_init("memory-localize") @@ -72,10 +76,10 @@ def render(out: str, store: Any, trace: LocalizeTrace, t0: float, t1: float) -> def at(ts: float) -> None: rr.set_time("ts", timestamp=ts) - # scene backdrop from the answer frame's depth (or the first detection) - backdrop_ts = trace.backdrop_ts - if backdrop_ts is None and trace.matched: - backdrop_ts = trace.matched[0][0] + # scene backdrop from an answer frame's depth (or the first detection) + backdrop_ts = next((t.backdrop_ts for _, t in traces if t.backdrop_ts is not None), None) + if backdrop_ts is None: + backdrop_ts = next((t.matched[0][0] for _, t in traces if t.matched), None) if backdrop_ts is not None: try: color = store.streams.color_image.at(backdrop_ts, 0.1).first().data @@ -107,41 +111,47 @@ def at(ts: float) -> None: rr.log("camera/image", obs.data.to_rerun()) rr.log("camera", pose.to_rerun()) - # marked frames: into the live feed, plus a frozen frustum at the capture pose - for i, obs in enumerate(trace.detection_frames): - pose = gates.camera_pose(tf, obs.ts) - if pose is None: - continue - at(obs.ts) - annotated = obs.data.annotated_image() - rr.log("camera/image", annotated.to_rerun()) - frame = f"detections/frames/{i}" - rr.log(frame, pose.to_rerun()) - rr.log(frame, camera_info.to_rerun()) - rr.log(f"{frame}/image", annotated.to_rerun()) - - # 3d detections: green = matched candidates, red = cross-view re-detections - for tag, entries, rgb in [("matched", trace.matched, GREEN), ("verified", trace.verified, RED)]: - for i, (ts, det) in enumerate(entries): - at(ts) + for query, trace in traces: + root = f"detections/{query.replace(' ', '_')}" + + # marked frames: into the live feed, plus a frozen frustum at the capture pose + for i, obs in enumerate(trace.detection_frames): + pose = gates.camera_pose(tf, obs.ts) + if pose is None: + continue + at(obs.ts) + annotated = obs.data.annotated_image() + rr.log("camera/image", annotated.to_rerun()) + frame = f"{root}/frames/{i}" + rr.log(frame, pose.to_rerun()) + rr.log(frame, camera_info.to_rerun()) + rr.log(f"{frame}/image", annotated.to_rerun()) + + # 3d detections: green = matched candidates, red = cross-view re-detections + for tag, entries, rgb in [ + ("matched", trace.matched, GREEN), + ("verified", trace.verified, RED), + ]: + for i, (ts, det) in enumerate(entries): + at(ts) + rr.log( + f"{root}/{tag}/{i}_{det.name.replace(' ', '_')}", + det.pointcloud.to_rerun(voxel_size=POINT_SIZE, colors=rgb), + ) + + # the answer: always blue, whatever the query + if trace.answer is not None: + at(trace.answer.ts) rr.log( - f"detections/{tag}/{i}_{det.name.replace(' ', '_')}", - det.pointcloud.to_rerun(voxel_size=POINT_SIZE, colors=rgb), + f"{root}/answer", + trace.answer.pointcloud.to_rerun(voxel_size=POINT_SIZE, colors=BLUE), ) - # the answer: always blue, whatever the query - if trace.answer is not None: - at(trace.answer.ts) - rr.log( - "detections/answer", - trace.answer.pointcloud.to_rerun(voxel_size=POINT_SIZE, colors=BLUE), - ) - def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("query", nargs="?", default="plant") - parser.add_argument("out", nargs="?", default="localize.rrd") + parser.add_argument("queries", nargs="+", help="one or more object queries") + parser.add_argument("out", help="rerun recording to write") parser.add_argument("--dataset", type=Path, help="memory2 recording database") parser.add_argument( "--from", dest="start", type=float, default=0.0, help="start offset into the recording (s)" @@ -161,44 +171,67 @@ def main() -> int: store = SqliteStore(path=dataset) lo, hi = store.streams.color_image.get_time_range() after = lo + args.start - before = lo + args.start + args.duration if args.duration is not None else None - - trace = LocalizeTrace() - hit = localize( - store, - args.query, - after=after, - before=before, - require_pose=not args.allow_no_pose, - trace=trace, - ) + before = lo + args.start + args.duration if args.duration is not None else hi + + from dimos.models.embedding.siglip import SigLIPModel + from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter + from dimos.perception.detection.detectors.owlv2 import Owlv2Detector + + siglip = SigLIPModel() + owl = Owlv2Detector() + segmenter = EdgeTAMImageSegmenter() + index = embed_index(store, siglip, after, before) + + traces: list[tuple[str, LocalizeTrace]] = [] + hits = 0 + for query in args.queries: + trace = LocalizeTrace() + hit = localize( + store, + query, + index=index, + siglip=siglip, + owl=owl, + segmenter=segmenter, + require_pose=not args.allow_no_pose, + trace=trace, + ) - if hit is None: - print(f"no verified detection of {args.query!r}") - return 1 + if hit is None: + print(f"no verified detection of {query!r}") + continue + hits += 1 + traces.append((query, trace)) + + offset = hit.pose_timestamp - lo + if hit.position_world_xyz is None: + print( + f"hit {query!r} without pose: reason={hit.reason} " + f"score={hit.semantic_score:.2f} ts_offset={offset:.1f}s" + ) + continue - offset = hit.pose_timestamp - lo - if hit.position_world_xyz is None: + x, y, z = hit.position_world_xyz + cloud_points = len(hit.point_cloud) if hit.point_cloud is not None else 0 print( - f"hit {args.query!r} without pose: reason={hit.reason} " - f"score={hit.semantic_score:.2f} ts_offset={offset:.1f}s" - ) - return 0 - - x, y, z = hit.position_world_xyz - cloud_points = len(hit.point_cloud) if hit.point_cloud is not None else 0 - print( - f"hit {args.query!r}: position=({x:.3f}, {y:.3f}, {z:.3f}) frame={hit.frame_id} " - f"ts_offset={offset:.1f}s points={cloud_points} views={hit.n_views} " - f"score={hit.semantic_score:.2f} margin={hit.ambiguity_margin:.2f}" - ) - if hit.ambiguity_margin < REFUSAL_MARGIN: - print( - f"ambiguity: margin {hit.ambiguity_margin:.2f} below refusal threshold " - f"{REFUSAL_MARGIN:.2f} - multiple coexisting matches, this pick is flagged" + f"hit {query!r}: position=({x:.3f}, {y:.3f}, {z:.3f}) frame={hit.frame_id} " + f"ts_offset={offset:.1f}s points={cloud_points} views={hit.n_views} " + f"score={hit.semantic_score:.2f} margin={hit.ambiguity_margin:.2f}" ) + if hit.ambiguity_margin < REFUSAL_MARGIN: + print( + f"ambiguity: margin {hit.ambiguity_margin:.2f} below refusal threshold " + f"{REFUSAL_MARGIN:.2f} - multiple coexisting matches, this pick is flagged" + ) + + siglip.stop() + owl.stop() + del segmenter + + if not hits: + return 1 - render(args.out, store, trace, after, before if before is not None else hi) + render(args.out, store, traces, after, before) print(f"saved {args.out}") return 0 From b3b0b00a5b0995ce3cfa6bfe58dfb68921538bef Mon Sep 17 00:00:00 2001 From: bogwi Date: Fri, 14 Aug 2026 12:58:01 +0900 Subject: [PATCH 07/12] improve invetory() api call --- dimos/perception/memory/inventory.py | 19 ++++++++++--------- dimos/perception/memory/tool_inventory.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/dimos/perception/memory/inventory.py b/dimos/perception/memory/inventory.py index 65ad676504..aa5dbfdd66 100644 --- a/dimos/perception/memory/inventory.py +++ b/dimos/perception/memory/inventory.py @@ -60,6 +60,8 @@ if TYPE_CHECKING: from dimos_lcm.sensor_msgs import CameraInfo + from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter + from dimos.perception.detection.detectors.owlv2 import Owlv2Detector from dimos.perception.detection.type.detection2d.seg import Detection2DSeg from dimos.protocol.tf.tf import TFLookup @@ -674,6 +676,7 @@ def _name_and_suppress( tracks: list[_Track], tracks_2d: list[_Track2D], store: Any, + owl: Owlv2Detector, vocabulary: NamingVocabulary, policy: InventoryPolicy, ) -> None: @@ -687,8 +690,6 @@ def _name_and_suppress( candidate name. Suppression is an existence decision and keeps its own request, which runs whether or not there is a vocabulary. """ - from dimos.perception.detection.detectors.owlv2 import Owlv2Detector - frame_members: dict[float, list[tuple[_Track, SupportObservation]]] = {} for track in tracks: for member in _naming_picks(track): @@ -702,7 +703,6 @@ def _name_and_suppress( return queries, starts, canonical = _flatten(vocabulary) - owl = Owlv2Detector() all_ts = sorted(set(frame_members) | set(frame_members_2d)) logger.info( f"naming: OWLv2 over {len(all_ts)} keyframes, " @@ -746,7 +746,6 @@ def _name_and_suppress( inside = _mask_overlap_fraction_bbox(member.bbox, det.bbox) if inside >= SUPPRESS_OVERLAP and member in track.members: track.members.remove(member) - owl.stop() def _mask_overlap_fraction_bbox( @@ -764,6 +763,8 @@ def _mask_overlap_fraction_bbox( def inventory( store: Any, *, + segmenter: EdgeTAMImageSegmenter, + owl: Owlv2Detector, naming_vocabulary: NamingVocabulary, after: float | None = None, before: float | None = None, @@ -790,6 +791,10 @@ def inventory( ``log_progress`` enables per-keyframe discovery lines (``discovery: i/n ts_offset=… prop=… scope=… …s``). Off by default. + + Both models belong to the caller: nothing here is loaded or stopped, so + one process can call this repeatedly on warm weights, over as many + windows as it wants. """ policy = policy or InventoryPolicy() tf = StreamTF.from_store(store) @@ -827,9 +832,6 @@ def inventory( if plane is not None: logger.info(f"support plane: {plane.inlier_count} inliers") - from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter - - segmenter = EdgeTAMImageSegmenter() frames_grounded: list[tuple[float, list[SupportObservation]]] = [] frames_ungrounded: list[tuple[float, list[Detection2DSeg]]] = [] image_area = float(camera_info.width * camera_info.height) @@ -886,7 +888,6 @@ def inventory( f"{perf_counter() - t_frame:.1f}s" ) - del segmenter _free_accelerator() total = sum(len(g) for _, g in frames_grounded) @@ -897,7 +898,7 @@ def inventory( tracks_2d = _track_ungrounded(frames_ungrounded) if include_ungrounded else [] logger.info(f"association: {len(tracks)} grounded instances") - _name_and_suppress(tracks, tracks_2d, store, naming_vocabulary, policy) + _name_and_suppress(tracks, tracks_2d, store, owl, naming_vocabulary, policy) tracks = [t for t in tracks if len(t.members) >= policy.min_member_observations] tracks.sort(key=lambda t: min(m.ts for m in t.members)) diff --git a/dimos/perception/memory/tool_inventory.py b/dimos/perception/memory/tool_inventory.py index 2ebd79dfbc..c301e0c72d 100644 --- a/dimos/perception/memory/tool_inventory.py +++ b/dimos/perception/memory/tool_inventory.py @@ -255,8 +255,16 @@ def main() -> int: else: naming_vocabulary = DEFAULT_VOCABULARY + from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter + from dimos.perception.detection.detectors.owlv2 import Owlv2Detector + + segmenter = EdgeTAMImageSegmenter() + owl = Owlv2Detector() + instances = inventory( store, + segmenter=segmenter, + owl=owl, naming_vocabulary=naming_vocabulary, after=after, before=before, @@ -264,6 +272,9 @@ def main() -> int: log_progress=args.log_progress, ) + owl.stop() + del segmenter + print(f"instances: {len(instances)}") for i, instance in enumerate(instances): if instance.latest_position_xyz is not None: From 13bf27745effbaadcb354deca61b55e71fea2487 Mon Sep 17 00:00:00 2001 From: bogwi Date: Fri, 14 Aug 2026 13:55:21 +0900 Subject: [PATCH 08/12] fix positional cli args in tool_localize | invetory --- dimos/perception/memory/tool_inventory.py | 13 ++++++++++--- dimos/perception/memory/tool_localize.py | 20 +++++++++++++++----- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/dimos/perception/memory/tool_inventory.py b/dimos/perception/memory/tool_inventory.py index c301e0c72d..be0d3e1043 100644 --- a/dimos/perception/memory/tool_inventory.py +++ b/dimos/perception/memory/tool_inventory.py @@ -205,7 +205,9 @@ def main() -> int: parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter ) - parser.add_argument("out", nargs="?", default="inventory.rrd") + parser.add_argument( + "out", nargs="?", default=None, help="rerun recording to write; omitted writes none" + ) parser.add_argument("--dataset", type=Path, help="memory2 recording database") parser.add_argument( "--from", dest="start", type=float, default=0.0, help="start offset into the recording (s)" @@ -239,6 +241,10 @@ def main() -> int: ) args = parser.parse_args() + out = args.out + if args.labels and args.labels[-1].endswith(".rrd"): + out = args.labels.pop() + dataset = args.dataset or get_data( "xarm6_worldbelief_realsense_d435i_stationery_calibrated/" "xarm6_worldbelief_20260729_203624_161992.db" @@ -298,8 +304,9 @@ def main() -> int: f"members={len(instance.members)}{geometry}" ) - render(args.out, store, instances, after, before if before is not None else hi) - print(f"saved {args.out}") + if out is not None: + render(out, store, instances, after, before if before is not None else hi) + print(f"saved {out}") return 0 diff --git a/dimos/perception/memory/tool_localize.py b/dimos/perception/memory/tool_localize.py index b37a8215b6..33fe108459 100644 --- a/dimos/perception/memory/tool_localize.py +++ b/dimos/perception/memory/tool_localize.py @@ -150,8 +150,12 @@ def at(ts: float) -> None: def main() -> int: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("queries", nargs="+", help="one or more object queries") - parser.add_argument("out", help="rerun recording to write") + parser.add_argument( + "positionals", + nargs="+", + metavar="query", + help="one or more object queries, optionally followed by an out.rrd to write", + ) parser.add_argument("--dataset", type=Path, help="memory2 recording database") parser.add_argument( "--from", dest="start", type=float, default=0.0, help="start offset into the recording (s)" @@ -164,6 +168,11 @@ def main() -> int: ) args = parser.parse_args() + queries = list(args.positionals) + out = queries.pop() if queries[-1].endswith(".rrd") else None + if not queries: + parser.error("no queries given") + dataset = args.dataset or get_data( "xarm6_worldbelief_realsense_d435i_stationery_calibrated/" "xarm6_worldbelief_20260729_203624_161992.db" @@ -184,7 +193,7 @@ def main() -> int: traces: list[tuple[str, LocalizeTrace]] = [] hits = 0 - for query in args.queries: + for query in queries: trace = LocalizeTrace() hit = localize( store, @@ -231,8 +240,9 @@ def main() -> int: if not hits: return 1 - render(args.out, store, traces, after, before) - print(f"saved {args.out}") + if out is not None: + render(out, store, traces, after, before) + print(f"saved {out}") return 0 From 58b76a813e87a38837b8aead9c71c2f27bd84f0a Mon Sep 17 00:00:00 2001 From: bogwi Date: Fri, 14 Aug 2026 15:42:32 +0900 Subject: [PATCH 09/12] localize(): swap OWLv2 for OmDet --- dimos/models/embedding/siglip.py | 5 + dimos/perception/detection/detectors/omdet.py | 118 ++++++++++++++++++ dimos/perception/memory/localize.py | 84 +++++++++---- dimos/perception/memory/tool_localize.py | 8 +- 4 files changed, 187 insertions(+), 28 deletions(-) create mode 100644 dimos/perception/detection/detectors/omdet.py diff --git a/dimos/models/embedding/siglip.py b/dimos/models/embedding/siglip.py index 225c98a04f..1b36cf090e 100644 --- a/dimos/models/embedding/siglip.py +++ b/dimos/models/embedding/siglip.py @@ -53,6 +53,11 @@ def _model(self) -> HFSiglipModel: def _processor(self) -> SiglipProcessor: return SiglipProcessor.from_pretrained(self.config.model_name, use_fast=True) + @property + def logit_scale(self) -> float: + """Trained sigmoid temperature: exp of the model's raw logit scale.""" + return float(self._model.logit_scale.exp()) + @overload def embed(self, image: Image, /) -> Embedding: ... @overload diff --git a/dimos/perception/detection/detectors/omdet.py b/dimos/perception/detection/detectors/omdet.py new file mode 100644 index 0000000000..80d8fc530d --- /dev/null +++ b/dimos/perception/detection/detectors/omdet.py @@ -0,0 +1,118 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OmDet-Turbo open-vocabulary detection: text prompts to boxes with per-box scores.""" + +from __future__ import annotations + +from functools import cached_property + +from PIL import Image as PILImage +import torch + +from dimos.models.base import HuggingFaceModel, HuggingFaceModelConfig +from dimos.msgs.sensor_msgs.Image import Image +from dimos.perception.detection.type.detection2d.bbox import Detection2DBBox +from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D + + +class OmDetConfig(HuggingFaceModelConfig): + model_name: str = "omlab/omdet-turbo-swin-tiny-hf" + dtype: torch.dtype = torch.float32 + + +class OmDetDetector(HuggingFaceModel): + """Real-time text-conditioned open-vocabulary detector (RT-DETR style). + + Same query contract as ``Owlv2Detector.query_detections`` at a fraction + of the forward cost: Swin-T backbone at 640px against OWLv2's ensemble + at 960px. Post-processing runs NMS, so returned boxes are already + deduplicated. Weights come from the Hugging Face hub cache. + """ + + config: OmDetConfig + + @cached_property + def _model(self): # type: ignore[no-untyped-def] + from huggingface_hub import hf_hub_download + from safetensors.torch import load_file + from transformers import OmDetTurboConfig, OmDetTurboForObjectDetection + + self._ensure_cuda_initialized() + # from_pretrained instantiates on the meta device, which leaves the + # timm Swin backbone's computed non-persistent buffers uninitialized; + # construct for real and load the checkpoint directly. + config = OmDetTurboConfig.from_pretrained(self.config.model_name) + model = OmDetTurboForObjectDetection(config) + model.load_state_dict( + load_file(hf_hub_download(self.config.model_name, "model.safetensors")) + ) + return model.eval().to(self.config.device) + + @cached_property + def _processor(self): # type: ignore[no-untyped-def] + from transformers import AutoProcessor + + return AutoProcessor.from_pretrained(self.config.model_name) + + def query_detections( + self, + image: Image, + queries: list[str], + threshold: float = 0.3, + ) -> ImageDetections2D: + """Detect every query string in the image; boxes below threshold are dropped. + + Each detection's ``name`` is the query text it matched and its + ``confidence`` is the per-box score. ``class_id`` indexes into + ``queries``. + """ + pil = PILImage.fromarray(image.to_rgb().data) + with torch.inference_mode(): + inputs = self._processor(images=pil, text=queries, return_tensors="pt").to( + self.config.device + ) + outputs = self._model(**inputs) + results = self._processor.post_process_grounded_object_detection( + outputs, + text_labels=queries, + threshold=threshold, + target_sizes=[(pil.height, pil.width)], + )[0] + + detections: list[Detection2DBBox] = [] + w, h = float(pil.width), float(pil.height) + for box, score, label in zip( + results["boxes"], results["scores"], results["labels"], strict=False + ): + x1, y1, x2, y2 = (float(v) for v in box) + bbox = (max(0.0, x1), max(0.0, y1), min(w, x2), min(h, y2)) + det = Detection2DBBox( + bbox=bbox, + track_id=-1, + class_id=int(label), + confidence=float(score), + name=queries[int(label)], + ts=image.ts, + image=image, + ) + if det.is_valid(): + detections.append(det) + + return ImageDetections2D(image=image, detections=detections) + + def stop(self) -> None: + if "_processor" in self.__dict__: + del self.__dict__["_processor"] + super().stop() diff --git a/dimos/perception/memory/localize.py b/dimos/perception/memory/localize.py index 165442d755..b04d866def 100644 --- a/dimos/perception/memory/localize.py +++ b/dimos/perception/memory/localize.py @@ -15,8 +15,9 @@ """Query-time object localization: text prompt to latest 3D pose and cloud. Search memory with embeddings (SigLIP, -frame-level), open-vocabulary detection (OWLv2, calibrated per-box scores), -segmentation (EdgeTAM), projection to 3D through aligned depth. Two +frame-level), open-vocabulary detection (OmDet-Turbo, per-box scores), +segmentation (EdgeTAM), projection to 3D through aligned depth, referent +identity on the observation crops (SigLIP, pairwise margins). Two algorithm rules distinguish it from a best-crop search: * **Latest-pose semantics.** Among verified observations of the chosen @@ -40,7 +41,7 @@ from dimos.memory2.transform import throttle from dimos.perception.detection.project import sees from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D -from dimos.perception.detection.type.detection3d.imageDetections3DPC import ImageDetections3DPC +from dimos.perception.detection.type.detection3d.pointcloud import Detection3DPC from dimos.perception.memory import gates from dimos.perception.memory.gates import OPTICAL_FRAME, TF_TOLERANCE, WORLD_FRAME from dimos.perception.memory.types import Localization, LocalizePolicy, Support @@ -50,10 +51,10 @@ from dimos_lcm.sensor_msgs import CameraInfo from dimos.memory2.stream import Stream + from dimos.models.embedding.base import Embedding from dimos.models.embedding.siglip import SigLIPModel from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter - from dimos.perception.detection.detectors.owlv2 import Owlv2Detector - from dimos.perception.detection.type.detection3d.pointcloud import Detection3DPC + from dimos.perception.detection.detectors.omdet import OmDetDetector from dimos.protocol.tf.tf import TFLookup logger = setup_logger() @@ -74,6 +75,7 @@ class _ClusterObservation: cloud: Any camera_position: np.ndarray detection: Detection3DPC + crop: Any # Image cut to the 2D box, for crop-level identity @dataclass @@ -158,10 +160,10 @@ def _axes_observed( class _DetectionCache: - """One OWLv2 + EdgeTAM pass per unique frame, shared across clusters.""" + """One OmDet + EdgeTAM pass per unique frame, shared across clusters.""" - def __init__(self, owl: Any, segmenter: Any, query: str, floor: float) -> None: - self.owl = owl + def __init__(self, detector: Any, segmenter: Any, query: str, floor: float) -> None: + self.detector = detector self.segmenter = segmenter self.query = query self.floor = floor @@ -172,7 +174,7 @@ def detect(self, image: Any) -> ImageDetections2D: hit = self._cache.get(key) if hit is not None: return hit - detections: ImageDetections2D = self.owl.query_detections( + detections: ImageDetections2D = self.detector.query_detections( image, [self.query], threshold=self.floor ) detections = ImageDetections2D( @@ -195,8 +197,8 @@ def _lift( tf_tolerance: float, policy: LocalizePolicy, plane: Any | None = None, -) -> list[tuple[Detection3DPC, np.ndarray]]: - """Depth-lift 2D detections; returns valid (detection3d, camera_position) pairs.""" +) -> list[tuple[Detection3DPC, np.ndarray, Any]]: + """Depth-lift 2D detections; returns valid (detection3d, camera_position, detection2d).""" depth = gates.depth_at(store, detections.ts) transform = tf.get(optical_frame, world_frame, detections.ts, tf_tolerance) if depth is None or transform is None: @@ -206,9 +208,11 @@ def _lift( return [] camera = np.array([pose.position.x, pose.position.y, pose.position.z]) - lifted = ImageDetections3DPC.from_depth(detections, depth, camera_info, transform) - valid: list[tuple[Detection3DPC, np.ndarray]] = [] - for det3d in lifted: + valid: list[tuple[Detection3DPC, np.ndarray, Any]] = [] + for det2d in detections: + det3d = Detection3DPC.from_depth(det2d, depth, camera_info, transform) + if det3d is None: + continue points = np.asarray(det3d.pointcloud.pointcloud.points) if len(points) < policy.min_depth_points: continue @@ -224,10 +228,18 @@ def _lift( high = float(np.quantile(heights, 0.95)) if low > policy.surface_patch_min_drop_m and high < policy.surface_patch_max_rise_m: continue - valid.append((det3d, camera)) + valid.append((det3d, camera, det2d)) return valid +def _cluster_identity(cluster: _Cluster, siglip: SigLIPModel, query_embedding: Embedding) -> float: + """Best SigLIP crop-to-query cosine over the cluster's observations.""" + embeddings = siglip.embed(*[o.crop for o in cluster.observations]) + if not isinstance(embeddings, list): + embeddings = [embeddings] + return max(e @ query_embedding for e in embeddings) + + def embed_index( store: Any, siglip: SigLIPModel, @@ -314,7 +326,7 @@ def localize( *, index: Stream[Any, Any], siglip: SigLIPModel, - owl: Owlv2Detector, + detector: OmDetDetector, segmenter: EdgeTAMImageSegmenter, require_pose: bool = True, policy: LocalizePolicy | None = None, @@ -356,8 +368,8 @@ def localize( store, tf, camera_info, frames, optical_frame, world_frame, tf_tolerance ) - # Pass 2 - OWLv2 + EdgeTAM: detect, segment, lift, verify. - cache = _DetectionCache(owl, segmenter, query, policy.candidate_floor) + # Pass 2 - OmDet + EdgeTAM: detect, segment, lift, verify. + cache = _DetectionCache(detector, segmenter, query, policy.candidate_floor) clusters: list[_Cluster] = [] ungrounded_best: tuple[float, float] | None = None # (score, ts) @@ -385,11 +397,12 @@ def _absorb(frame_obs: Any, is_verify: bool) -> None: plane, ) for det2d in detections: - if not any(d.track_id == det2d.track_id for d, _ in lifted): + if not any(d.track_id == det2d.track_id for d, _, _ in lifted): best = (det2d.confidence, det2d.ts) if ungrounded_best is None or best[0] > ungrounded_best[0]: ungrounded_best = best - for det3d, camera in lifted: + for det3d, camera, det2d in lifted: + x1, y1, x2, y2 = det2d.bbox observation = _ClusterObservation( ts=det3d.ts, score=det3d.confidence, @@ -397,6 +410,7 @@ def _absorb(frame_obs: Any, is_verify: bool) -> None: cloud=det3d.pointcloud, camera_position=camera, detection=det3d, + crop=det2d.image.crop(int(x1), int(y1), int(x2 - x1) + 1, int(y2 - y1) + 1), ) if trace is not None: (trace.verified if is_verify else trace.matched).append((det3d.ts, det3d)) @@ -483,15 +497,37 @@ def _absorb(frame_obs: Any, is_verify: bool) -> None: ) return None - winner = max(verified, key=lambda c: c.latest.ts) + # Detector scores verify supports but do not bind attributes ("red + # marker" ranks a black pen high): the referent is chosen by SigLIP crop + # identity, compared pairwise through the trained sigmoid temperature so + # margins are probability differences and the bias term cancels. + identity = {id(c): _cluster_identity(c, siglip, query_embedding) for c in verified} + scale = siglip.logit_scale + + def dominance(a: float, b: float) -> float: + return math.tanh(scale * (a - b) / 2) + + best_identity = max(identity.values()) + candidates = [ + c for c in verified if dominance(best_identity, identity[id(c)]) < policy.refusal_margin + ] + winner = max(candidates, key=lambda c: c.latest.ts) w_lo, w_hi = winner.interval - rival_scores = [ - c.max_score + rival_identities = [ + identity[id(c)] for c in verified if c is not winner and not (c.interval[1] < w_lo or c.interval[0] > w_hi) # coexisting in time ] - margin = winner.max_score - max(rival_scores) if rival_scores else 1.0 + margin = min( + (dominance(identity[id(winner)], r) for r in rival_identities), + default=1.0, + ) + logger.info( + "identity: " + + ", ".join(f"cos={identity[id(c)]:.3f} score={c.max_score:.2f}" for c in verified) + + f" margin={margin:.2f}" + ) reason = "ambiguous_between_coexisting_candidates" if margin < policy.refusal_margin else None latest = winner.latest diff --git a/dimos/perception/memory/tool_localize.py b/dimos/perception/memory/tool_localize.py index 33fe108459..1d25f0e67c 100644 --- a/dimos/perception/memory/tool_localize.py +++ b/dimos/perception/memory/tool_localize.py @@ -184,10 +184,10 @@ def main() -> int: from dimos.models.embedding.siglip import SigLIPModel from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter - from dimos.perception.detection.detectors.owlv2 import Owlv2Detector + from dimos.perception.detection.detectors.omdet import OmDetDetector siglip = SigLIPModel() - owl = Owlv2Detector() + detector = OmDetDetector() segmenter = EdgeTAMImageSegmenter() index = embed_index(store, siglip, after, before) @@ -200,7 +200,7 @@ def main() -> int: query, index=index, siglip=siglip, - owl=owl, + detector=detector, segmenter=segmenter, require_pose=not args.allow_no_pose, trace=trace, @@ -234,7 +234,7 @@ def main() -> int: ) siglip.stop() - owl.stop() + detector.stop() del segmenter if not hits: From c69a4a502b68a7b1c3003a742ec31b20198ffa5e Mon Sep 17 00:00:00 2001 From: bogwi Date: Sat, 15 Aug 2026 02:58:39 +0900 Subject: [PATCH 10/12] inventory(): swap OWLv2 for OmDet --- dimos/perception/detection/detectors/omdet.py | 45 +++++++++++++++++++ dimos/perception/detection/detectors/yoloe.py | 4 +- dimos/perception/memory/inventory.py | 20 +++++---- dimos/perception/memory/tool_inventory.py | 8 ++-- 4 files changed, 62 insertions(+), 15 deletions(-) diff --git a/dimos/perception/detection/detectors/omdet.py b/dimos/perception/detection/detectors/omdet.py index 80d8fc530d..aeee0f7433 100644 --- a/dimos/perception/detection/detectors/omdet.py +++ b/dimos/perception/detection/detectors/omdet.py @@ -18,6 +18,7 @@ from functools import cached_property +import numpy as np from PIL import Image as PILImage import torch @@ -112,6 +113,50 @@ def query_detections( return ImageDetections2D(image=image, detections=detections) + def query_score_rows( + self, + image: Image, + queries: list[str], + threshold: float = 0.3, + ) -> tuple[np.ndarray, np.ndarray]: + """Score every query against every kept box; no argmax, no label. + + Same selection as the stock post-processing behind + ``query_detections()``: top-k over the (proposal, query) score + matrix, the per-box threshold, then class-wise NMS. A proposal + surviving under several queries returns once with its whole + ``n_queries`` score row, so a caller can rank queries and refuse. + The task prompt is fixed instead of the processor default, which + lists every query in one sentence and truncates it at 77 tokens, + silently conditioning the decoder on a prefix of a large + vocabulary. Returns pixel ``(x1, y1, x2, y2)`` boxes and their + score rows. + """ + from torchvision.ops.boxes import batched_nms # type: ignore[import-untyped] + from transformers.image_transforms import center_to_corners_format + + pil = PILImage.fromarray(image.to_rgb().data) + with torch.inference_mode(): + inputs = self._processor( + images=pil, text=queries, task="Detect all objects.", return_tensors="pt" + ).to(self.config.device) + outputs = self._model(**inputs) + rows = torch.sigmoid(outputs.decoder_class_logits[0]) + n_proposals, n_queries = rows.shape + scores, pairs = rows.flatten().topk(n_proposals, sorted=False) + passing = scores > threshold + scores, pairs = scores[passing], pairs[passing] + scale = torch.tensor([pil.width, pil.height, pil.width, pil.height], device=rows.device) + boxes = center_to_corners_format(outputs.decoder_coord_logits[0]) * scale + survivors = batched_nms(boxes[pairs // n_queries], scores, pairs % n_queries, 0.5) + proposals = (pairs[survivors] // n_queries).unique() + kept_boxes = boxes[proposals].float().cpu().numpy() + kept_rows = rows[proposals].float().cpu().numpy() + + kept_boxes[:, 0::2] = kept_boxes[:, 0::2].clip(0.0, float(pil.width)) + kept_boxes[:, 1::2] = kept_boxes[:, 1::2].clip(0.0, float(pil.height)) + return kept_boxes, kept_rows + def stop(self) -> None: if "_processor" in self.__dict__: del self.__dict__["_processor"] diff --git a/dimos/perception/detection/detectors/yoloe.py b/dimos/perception/detection/detectors/yoloe.py index fddd71bca1..aa7ed19b21 100644 --- a/dimos/perception/detection/detectors/yoloe.py +++ b/dimos/perception/detection/detectors/yoloe.py @@ -18,9 +18,9 @@ import numpy as np from numpy.typing import NDArray +import torch from ultralytics import YOLOE # type: ignore[attr-defined] -from dimos.models.base import default_torch_device from dimos.msgs.sensor_msgs.Image import Image from dimos.perception.detection.detectors.base import Detector from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D @@ -81,7 +81,7 @@ def __init__( if self.max_area_ratio is not None and not (0.0 < self.max_area_ratio <= 1.0): raise ValueError("max_area_ratio must be in the range (0, 1].") - self.device = device or default_torch_device() + self.device = device or ("cuda" if torch.cuda.is_available() else "cpu") logger.info( f"YOLO-E detector loaded model={model_name} prompt_mode={prompt_mode.value} " f"device={self.device} conf={self.conf}" diff --git a/dimos/perception/memory/inventory.py b/dimos/perception/memory/inventory.py index aa5dbfdd66..ac7347412f 100644 --- a/dimos/perception/memory/inventory.py +++ b/dimos/perception/memory/inventory.py @@ -19,7 +19,7 @@ instance table. Existence is decoupled from naming and the ordering is a constraint, not a preference: propose (EdgeTAM automatic masks), lift (masked depth to world supports), associate (hard constraints before any -score), and only then name (OWLv2, labels as metadata). Labels and +score), and only then name (OmDet-Turbo, labels as metadata). Labels and appearance never enter association; position and same-frame co-occurrence decide everything, which is what keeps two identical objects two instances. @@ -61,7 +61,7 @@ from dimos_lcm.sensor_msgs import CameraInfo from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter - from dimos.perception.detection.detectors.owlv2 import Owlv2Detector + from dimos.perception.detection.detectors.omdet import OmDetDetector from dimos.perception.detection.type.detection2d.seg import Detection2DSeg from dimos.protocol.tf.tf import TFLookup @@ -676,11 +676,11 @@ def _name_and_suppress( tracks: list[_Track], tracks_2d: list[_Track2D], store: Any, - owl: Owlv2Detector, + detector: OmDetDetector, vocabulary: NamingVocabulary, policy: InventoryPolicy, ) -> None: - """OWLv2 naming per instance on keyframes, person/hand suppressing observations. + """OmDet naming per instance on keyframes, person/hand suppressing observations. Runs after association by construction: association consumed unnamed supports, so per-view label instability cannot starve existence or split @@ -705,7 +705,7 @@ def _name_and_suppress( queries, starts, canonical = _flatten(vocabulary) all_ts = sorted(set(frame_members) | set(frame_members_2d)) logger.info( - f"naming: OWLv2 over {len(all_ts)} keyframes, " + f"naming: OmDet over {len(all_ts)} keyframes, " f"{len(canonical)} groups of {len(queries)} queries" ) for ts in all_ts: @@ -715,7 +715,9 @@ def _name_and_suppress( continue if queries: - boxes, scores = owl.query_score_rows(image, queries, threshold=policy.name_accept_score) + boxes, scores = detector.query_score_rows( + image, queries, threshold=policy.name_accept_score + ) winners, top = _accepted_groups(scores, starts, policy) for box, group, score in zip(boxes, winners, top, strict=True): if group < 0: @@ -739,7 +741,7 @@ def _name_and_suppress( best_target.labels.get(label, 0.0), float(score) ) - for det in owl.query_detections(image, SUPPRESS_QUERIES, threshold=SUPPRESS_SCORE): + for det in detector.query_detections(image, SUPPRESS_QUERIES, threshold=SUPPRESS_SCORE): for track, member in frame_members.get(ts, []): if member.bbox is None: continue @@ -764,7 +766,7 @@ def inventory( store: Any, *, segmenter: EdgeTAMImageSegmenter, - owl: Owlv2Detector, + detector: OmDetDetector, naming_vocabulary: NamingVocabulary, after: float | None = None, before: float | None = None, @@ -898,7 +900,7 @@ def inventory( tracks_2d = _track_ungrounded(frames_ungrounded) if include_ungrounded else [] logger.info(f"association: {len(tracks)} grounded instances") - _name_and_suppress(tracks, tracks_2d, store, owl, naming_vocabulary, policy) + _name_and_suppress(tracks, tracks_2d, store, detector, naming_vocabulary, policy) tracks = [t for t in tracks if len(t.members) >= policy.min_member_observations] tracks.sort(key=lambda t: min(m.ts for m in t.members)) diff --git a/dimos/perception/memory/tool_inventory.py b/dimos/perception/memory/tool_inventory.py index be0d3e1043..2eebe2cd73 100644 --- a/dimos/perception/memory/tool_inventory.py +++ b/dimos/perception/memory/tool_inventory.py @@ -262,15 +262,15 @@ def main() -> int: naming_vocabulary = DEFAULT_VOCABULARY from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter - from dimos.perception.detection.detectors.owlv2 import Owlv2Detector + from dimos.perception.detection.detectors.omdet import OmDetDetector segmenter = EdgeTAMImageSegmenter() - owl = Owlv2Detector() + detector = OmDetDetector() instances = inventory( store, segmenter=segmenter, - owl=owl, + detector=detector, naming_vocabulary=naming_vocabulary, after=after, before=before, @@ -278,7 +278,7 @@ def main() -> int: log_progress=args.log_progress, ) - owl.stop() + detector.stop() del segmenter print(f"instances: {len(instances)}") From 216ffedfcc922b171c243c376808db2e2588b417 Mon Sep 17 00:00:00 2001 From: bogwi Date: Sat, 15 Aug 2026 11:11:53 +0900 Subject: [PATCH 11/12] localize: support a list queries --- dimos/perception/memory/localize.py | 98 ++++++++++++++++++------ dimos/perception/memory/tool_localize.py | 94 +++++++++++++++-------- 2 files changed, 135 insertions(+), 57 deletions(-) diff --git a/dimos/perception/memory/localize.py b/dimos/perception/memory/localize.py index b04d866def..31eed5354a 100644 --- a/dimos/perception/memory/localize.py +++ b/dimos/perception/memory/localize.py @@ -160,31 +160,35 @@ def _axes_observed( class _DetectionCache: - """One OmDet + EdgeTAM pass per unique frame, shared across clusters.""" + """One OmDet + EdgeTAM pass per unique frame, shared across queries and clusters.""" - def __init__(self, detector: Any, segmenter: Any, query: str, floor: float) -> None: + def __init__(self, detector: Any, segmenter: Any, queries: list[str], floor: float) -> None: self.detector = detector self.segmenter = segmenter - self.query = query + self.queries = queries self.floor = floor self._cache: dict[float, ImageDetections2D] = {} - def detect(self, image: Any) -> ImageDetections2D: + def detect(self, image: Any, query: str) -> ImageDetections2D: key = image.ts - hit = self._cache.get(key) - if hit is not None: - return hit - detections: ImageDetections2D = self.detector.query_detections( - image, [self.query], threshold=self.floor - ) - detections = ImageDetections2D( - image, - sorted(detections.detections, key=lambda d: -d.confidence)[:BOXES_PER_FRAME], - ) - if len(detections): - detections = self.segmenter.segment(detections) - self._cache[key] = detections - return detections + cached = self._cache.get(key) + if cached is None: + detections: ImageDetections2D = self.detector.query_detections( + image, self.queries, threshold=self.floor + ) + ranked = sorted(detections.detections, key=lambda d: -d.confidence) + cached = ImageDetections2D( + image, + [ + det + for label in self.queries + for det in [d for d in ranked if d.name == label][:BOXES_PER_FRAME] + ], + ) + if len(cached): + cached = self.segmenter.segment(cached) + self._cache[key] = cached + return cached.filter(lambda d: d.name == query) def _lift( @@ -322,7 +326,7 @@ def _retrieve( def localize( store: Any, - query: str, + query: str | list[str], *, index: Stream[Any, Any], siglip: SigLIPModel, @@ -334,8 +338,8 @@ def localize( world_frame: str = WORLD_FRAME, optical_frame: str = OPTICAL_FRAME, tf_tolerance: float = TF_TOLERANCE, - trace: LocalizeTrace | None = None, -) -> Localization | None: + trace: LocalizeTrace | list[LocalizeTrace] | None = None, +) -> Localization | list[Localization | None] | None: """Latest unambiguous 3D localization of *query*, or ``None``. ``None`` is a first-class answer: nothing reached the accept score, no @@ -344,6 +348,10 @@ def localize( coexisting candidates is returned with ``ambiguity_margin`` below ``refusal_margin`` - a flagged hit, never a silent guess. + A list *query* runs every label through one shared detection cache - + OmDet takes the whole list per frame - and returns one result per label, + in input order; ``trace`` then takes a list of the same length. + The index and the three models belong to the caller: nothing here is loaded or stopped, so one process can call this repeatedly on warm weights, and every query on one window reuses the same embeddings. The @@ -355,6 +363,50 @@ def localize( raise ValueError("recording has no tf stream") camera_info = store.streams.camera_info.first().data + queries = [query] if isinstance(query, str) else query + traces: list[LocalizeTrace | None] = ( + list(trace) if isinstance(trace, list) else [trace] * len(queries) + ) + cache = _DetectionCache(detector, segmenter, queries, policy.candidate_floor) + results = [ + _localize_one( + store, + q, + index=index, + siglip=siglip, + cache=cache, + tf=tf, + camera_info=camera_info, + require_pose=require_pose, + policy=policy, + cloud_mode=cloud_mode, + world_frame=world_frame, + optical_frame=optical_frame, + tf_tolerance=tf_tolerance, + trace=t, + ) + for q, t in zip(queries, traces, strict=True) + ] + return results[0] if isinstance(query, str) else results + + +def _localize_one( + store: Any, + query: str, + *, + index: Stream[Any, Any], + siglip: SigLIPModel, + cache: _DetectionCache, + tf: TFLookup, + camera_info: CameraInfo, + require_pose: bool, + policy: LocalizePolicy, + cloud_mode: str, + world_frame: str, + optical_frame: str, + tf_tolerance: float, + trace: LocalizeTrace | None, +) -> Localization | None: # Pass 1 - SigLIP: rank the indexed frames by the query. query_embedding = siglip.embed_text(query) frames = _retrieve(index, tf, query_embedding, optical_frame, world_frame, tf_tolerance) @@ -369,8 +421,6 @@ def localize( ) # Pass 2 - OmDet + EdgeTAM: detect, segment, lift, verify. - cache = _DetectionCache(detector, segmenter, query, policy.candidate_floor) - clusters: list[_Cluster] = [] ungrounded_best: tuple[float, float] | None = None # (score, ts) processed: set[float] = set() @@ -380,7 +430,7 @@ def _absorb(frame_obs: Any, is_verify: bool) -> None: if frame_obs.ts in processed: return processed.add(frame_obs.ts) - detections = cache.detect(frame_obs.data) + detections = cache.detect(frame_obs.data, query) if not len(detections): return if trace is not None and not is_verify: diff --git a/dimos/perception/memory/tool_localize.py b/dimos/perception/memory/tool_localize.py index 1d25f0e67c..100c92b5e8 100644 --- a/dimos/perception/memory/tool_localize.py +++ b/dimos/perception/memory/tool_localize.py @@ -15,9 +15,11 @@ """Query memory for objects, localize them in 3D via depth, render in rerun. Run: uv run python -m dimos.perception.memory.tool_localize [query ...] [out.rrd] - [--from ] [--duration ] + [--from ] [--duration ] [--multi] -Queries share one model load and one .rrd. Exit code 0 with a printed +Queries share one model load and one .rrd; with --multi they go to +localize() as one list, sharing a single detection pass per frame. +Exit code 0 with a printed position per verified hit; exit code 1 when no query is verified, with "no verified detection of ..." per miss - the honest answer that the object is not there. An ambiguous hit (identical twins in view) is printed with its @@ -34,6 +36,7 @@ from dimos.memory2.transform import throttle from dimos.perception.memory import gates from dimos.perception.memory.localize import LocalizeTrace, embed_index, localize +from dimos.perception.memory.types import Localization from dimos.utils.data import get_data REFUSAL_MARGIN = 0.15 @@ -148,6 +151,33 @@ def at(ts: float) -> None: ) +def report(query: str, hit: Localization | None, lo: float) -> bool: + """Print one query's outcome; True when it counts as a hit.""" + if hit is None: + print(f"no verified detection of {query!r}") + return False + offset = hit.pose_timestamp - lo + if hit.position_world_xyz is None: + print( + f"hit {query!r} without pose: reason={hit.reason} " + f"score={hit.semantic_score:.2f} ts_offset={offset:.1f}s" + ) + return True + x, y, z = hit.position_world_xyz + cloud_points = len(hit.point_cloud) if hit.point_cloud is not None else 0 + print( + f"hit {query!r}: position=({x:.3f}, {y:.3f}, {z:.3f}) frame={hit.frame_id} " + f"ts_offset={offset:.1f}s points={cloud_points} views={hit.n_views} " + f"score={hit.semantic_score:.2f} margin={hit.ambiguity_margin:.2f}" + ) + if hit.ambiguity_margin < REFUSAL_MARGIN: + print( + f"ambiguity: margin {hit.ambiguity_margin:.2f} below refusal threshold " + f"{REFUSAL_MARGIN:.2f} - multiple coexisting matches, this pick is flagged" + ) + return True + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -166,6 +196,11 @@ def main() -> int: action="store_true", help="return an RGB-only hit with a null position instead of refusing", ) + parser.add_argument( + "--multi", + action="store_true", + help="pass all queries to localize() as one list (one shared detection pass per frame)", + ) args = parser.parse_args() queries = list(args.positionals) @@ -193,45 +228,38 @@ def main() -> int: traces: list[tuple[str, LocalizeTrace]] = [] hits = 0 - for query in queries: - trace = LocalizeTrace() - hit = localize( + if args.multi: + qtraces = [LocalizeTrace() for _ in queries] + results = localize( store, - query, + queries, index=index, siglip=siglip, detector=detector, segmenter=segmenter, require_pose=not args.allow_no_pose, - trace=trace, - ) - - if hit is None: - print(f"no verified detection of {query!r}") - continue - hits += 1 - traces.append((query, trace)) - - offset = hit.pose_timestamp - lo - if hit.position_world_xyz is None: - print( - f"hit {query!r} without pose: reason={hit.reason} " - f"score={hit.semantic_score:.2f} ts_offset={offset:.1f}s" - ) - continue - - x, y, z = hit.position_world_xyz - cloud_points = len(hit.point_cloud) if hit.point_cloud is not None else 0 - print( - f"hit {query!r}: position=({x:.3f}, {y:.3f}, {z:.3f}) frame={hit.frame_id} " - f"ts_offset={offset:.1f}s points={cloud_points} views={hit.n_views} " - f"score={hit.semantic_score:.2f} margin={hit.ambiguity_margin:.2f}" + trace=qtraces, ) - if hit.ambiguity_margin < REFUSAL_MARGIN: - print( - f"ambiguity: margin {hit.ambiguity_margin:.2f} below refusal threshold " - f"{REFUSAL_MARGIN:.2f} - multiple coexisting matches, this pick is flagged" + for query, qtrace, hit in zip(queries, qtraces, results, strict=True): + if report(query, hit, lo): + hits += 1 + traces.append((query, qtrace)) + else: + for query in queries: + trace = LocalizeTrace() + hit = localize( + store, + query, + index=index, + siglip=siglip, + detector=detector, + segmenter=segmenter, + require_pose=not args.allow_no_pose, + trace=trace, ) + if report(query, hit, lo): + hits += 1 + traces.append((query, trace)) siglip.stop() detector.stop() From a4287239522a4d31ff228d1d0df6341b33cd8c5a Mon Sep 17 00:00:00 2001 From: bogwi Date: Sat, 15 Aug 2026 13:21:10 +0900 Subject: [PATCH 12/12] Retarget perception.memory imports after memory2 was renamed to memory. --- dimos/perception/memory/gates.py | 4 ++-- dimos/perception/memory/inventory.py | 2 +- dimos/perception/memory/localize.py | 8 ++++---- dimos/perception/memory/support_plane.py | 2 +- dimos/perception/memory/tool_inventory.py | 10 +++++----- dimos/perception/memory/tool_localize.py | 8 ++++---- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/dimos/perception/memory/gates.py b/dimos/perception/memory/gates.py index 50821a1dcc..99ade4ccdd 100644 --- a/dimos/perception/memory/gates.py +++ b/dimos/perception/memory/gates.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Per-frame gates and lookups over a memory2 recording: poses, stillness, frames. +"""Per-frame gates and lookups over a memory recording: poses, stillness, frames. Two per-frame gates live here and both are required: @@ -39,7 +39,7 @@ import numpy as np if TYPE_CHECKING: - from dimos.memory2.type.observation import Observation + from dimos.memory.type.observation import Observation from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.sensor_msgs.Image import Image from dimos.protocol.tf.tf import TFLookup diff --git a/dimos/perception/memory/inventory.py b/dimos/perception/memory/inventory.py index ac7347412f..13a0e81734 100644 --- a/dimos/perception/memory/inventory.py +++ b/dimos/perception/memory/inventory.py @@ -38,7 +38,7 @@ import numpy as np -from dimos.memory2.tf import StreamTF +from dimos.memory.tf import StreamTF from dimos.perception.detection.type.detection3d.imageDetections3DPC import ImageDetections3DPC from dimos.perception.memory import gates from dimos.perception.memory.gates import ( diff --git a/dimos/perception/memory/localize.py b/dimos/perception/memory/localize.py index 31eed5354a..7442e16d0a 100644 --- a/dimos/perception/memory/localize.py +++ b/dimos/perception/memory/localize.py @@ -36,9 +36,9 @@ import numpy as np -from dimos.memory2.embed import EmbedImages -from dimos.memory2.tf import StreamTF -from dimos.memory2.transform import throttle +from dimos.memory.embed import EmbedImages +from dimos.memory.tf import StreamTF +from dimos.memory.transform import throttle from dimos.perception.detection.project import sees from dimos.perception.detection.type.detection2d.imageDetections2D import ImageDetections2D from dimos.perception.detection.type.detection3d.pointcloud import Detection3DPC @@ -50,7 +50,7 @@ if TYPE_CHECKING: from dimos_lcm.sensor_msgs import CameraInfo - from dimos.memory2.stream import Stream + from dimos.memory.stream import Stream from dimos.models.embedding.base import Embedding from dimos.models.embedding.siglip import SigLIPModel from dimos.models.segmentation.edge_tam import EdgeTAMImageSegmenter diff --git a/dimos/perception/memory/support_plane.py b/dimos/perception/memory/support_plane.py index 5ddffda9e3..f3382f74e9 100644 --- a/dimos/perception/memory/support_plane.py +++ b/dimos/perception/memory/support_plane.py @@ -34,7 +34,7 @@ if TYPE_CHECKING: from dimos_lcm.sensor_msgs import CameraInfo - from dimos.memory2.type.observation import Observation + from dimos.memory.type.observation import Observation from dimos.msgs.sensor_msgs.Image import Image from dimos.protocol.tf.tf import TFLookup diff --git a/dimos/perception/memory/tool_inventory.py b/dimos/perception/memory/tool_inventory.py index 2eebe2cd73..715789b004 100644 --- a/dimos/perception/memory/tool_inventory.py +++ b/dimos/perception/memory/tool_inventory.py @@ -54,9 +54,9 @@ import sys from typing import Any, cast -from dimos.memory2.store.sqlite import SqliteStore -from dimos.memory2.tf import StreamTF -from dimos.memory2.transform import throttle +from dimos.memory.store.sqlite import SqliteStore +from dimos.memory.tf import StreamTF +from dimos.memory.transform import throttle from dimos.perception.memory import gates from dimos.perception.memory.inventory import DEFAULT_VOCABULARY, NamingVocabulary, inventory from dimos.perception.memory.types import Instance, SupportObservation @@ -95,7 +95,7 @@ def render(out: str, store: Any, instances: list[Instance], t0: float, t1: float import rerun as rr import rerun.blueprint as rrb - from dimos.memory2.vis.color import Color + from dimos.memory.vis.color import Color from dimos.msgs.sensor_msgs.PointCloud2 import PointCloud2 from dimos.visualization.rerun.init import rerun_init @@ -208,7 +208,7 @@ def main() -> int: parser.add_argument( "out", nargs="?", default=None, help="rerun recording to write; omitted writes none" ) - parser.add_argument("--dataset", type=Path, help="memory2 recording database") + parser.add_argument("--dataset", type=Path, help="memory recording database") parser.add_argument( "--from", dest="start", type=float, default=0.0, help="start offset into the recording (s)" ) diff --git a/dimos/perception/memory/tool_localize.py b/dimos/perception/memory/tool_localize.py index 100c92b5e8..f706ea9f1a 100644 --- a/dimos/perception/memory/tool_localize.py +++ b/dimos/perception/memory/tool_localize.py @@ -31,9 +31,9 @@ import sys from typing import Any, cast -from dimos.memory2.store.sqlite import SqliteStore -from dimos.memory2.tf import StreamTF -from dimos.memory2.transform import throttle +from dimos.memory.store.sqlite import SqliteStore +from dimos.memory.tf import StreamTF +from dimos.memory.transform import throttle from dimos.perception.memory import gates from dimos.perception.memory.localize import LocalizeTrace, embed_index, localize from dimos.perception.memory.types import Localization @@ -186,7 +186,7 @@ def main() -> int: metavar="query", help="one or more object queries, optionally followed by an out.rrd to write", ) - parser.add_argument("--dataset", type=Path, help="memory2 recording database") + parser.add_argument("--dataset", type=Path, help="memory recording database") parser.add_argument( "--from", dest="start", type=float, default=0.0, help="start offset into the recording (s)" )