feat(benchmark): add point-cloud VQA pipeline

[ruthwikdasyam]

No description provided.

[codecov]

❌ 4 Tests Failed:

Tests completed	Failed	Passed	Skipped
3841	4	3837	175

View the top 3 failed test(s) by shortest run time

dimos.benchmark.vqa.generation.test_oracle::test_ground_plane_estimator_fits_visible_lower_band

Stack Traces | 0.003s run time

def test_ground_plane_estimator_fits_visible_lower_band() -> None:
        fit = estimate_ground_plane(_measurement_frame())
    
        assert fit.rejection_reason is None
        assert fit.estimate is not None
        assert fit.estimate.inlier_count >= 12
        assert fit.estimate.residual_m < 0.001
        assert np.allclose(fit.estimate.normal, (0.0, -1.0, 0.0), atol=0.001)
>       assert fit.estimate.offset_m == 1.0
E       AssertionError: assert 0.9999999999999998 == 1.0
E        +  where 0.9999999999999998 = GroundPlaneEstimate(normal=(0.0, -0.9999999999999998, 0.0), offset_m=0.9999999999999998, sample_count=16, inlier_count=15, residual_m=0.0).offset_m
E        +    where GroundPlaneEstimate(normal=(0.0, -0.9999999999999998, 0.0), offset_m=0.9999999999999998, sample_count=16, inlier_count=15, residual_m=0.0) = PlaneFitResult(estimate=GroundPlaneEstimate(normal=(0.0, -0.9999999999999998, 0.0), offset_m=0.9999999999999998, sample_count=16, inlier_count=15, residual_m=0.0), quality_flags=('ground_band_visible', 'ransac_inliers_accepted'), rejection_reason=None).estimate

fit        = PlaneFitResult(estimate=GroundPlaneEstimate(normal=(0.0, -0.9999999999999998, 0.0), offset_m=0.9999999999999998, sampl...ier_count=15, residual_m=0.0), quality_flags=('ground_band_visible', 'ransac_inliers_accepted'), rejection_reason=None)

.../vqa/generation/test_oracle.py:327: AssertionError

dimos.cli.test_vqa::test_vqa_generation_cli_has_no_explicit_query_or_model_options

Stack Traces | 0.016s run time

def test_vqa_generation_cli_has_no_explicit_query_or_model_options() -> None:
        output = CliRunner().invoke(vqa.app, ["generate", "--help"]).output
    
        assert "--query" not in output
        assert "--propose-questions" not in output
        assert "--question-model" not in output
        assert "--oracle-model" not in output
>       assert "--spec" in output
E       AssertionError: assert '--spec' in '\x1b[1m                                                                                \x1b[0m\n\x1b[1m \x1b[0m\x1b[1;33mUsage: \x1b[0m\x1b[1mroot generate [OPTIONS]\x1b[0m\x1b[1m                                                \x1b[0m\x1b[1m \x1b[0m\n\x1b[1m                                                                                \x1b[0m\n Generate a resumable VQA dataset from sampled Go2 recording frames.            \n                                                                                \n\x1b[2m╭─\x1b[0m\x1b[2m Options \x1b[0m\x1b[2m───────────────────────────────────────────────────────────────────\x1b[0m\x1b[2m─╮\x1b[0m\n\x1b[2m│\x1b[0m \x1b[1;36m-\x1b[0m\x1b[1;36m-recording\x1b[0m                    \x1b[1;33mTEXT   \x1b[0m                                       \x1b[2m│\x1b[0m\n\x1b[2m│\x1b[0m \x1b[1;36m-\x1b[0m\x1b[1;36m-start\x1b[0m\x1b[1;36m-index\x1b[0m                  \x1b[1;33mINTEGER\x1b[0m                                       \x1b[2m│\x1b[0m\n\x1b[2m│\x1b[0m \x1b[1;36m-\x1b[0m\x1b[1;36m-stop\x1b[0m\x1b[1;36m-index\x1b[0m                   \x1b[1;33mINTEGER\x1b[0m                                       \x1b[2m│\x1b[0m\n\x1b[2m│\x1b[0m \x1b[1;36m-\x1b...   \x1b[1;33mINTEGER\x1b[0m                                       \x1b[2m│\x1b[0m\n\x1b[2m│\x1b[0m \x1b[1;36m-\x1b[0m\x1b[1;36m-question\x1b[0m\x1b[1;36m-mode\x1b[0m                \x1b[1;33mTEXT   \x1b[0m                                       \x1b[2m│\x1b[0m\n\x1b[2m│\x1b[0m \x1b[1;36m-\x1b[0m\x1b[1;36m-min\x1b[0m\x1b[1;36m-mask-area-px\x1b[0m             \x1b[1;33mINTEGER\x1b[0m                                       \x1b[2m│\x1b[0m\n\x1b[2m│\x1b[0m \x1b[1;36m-\x1b[0m\x1b[1;36m-min\x1b[0m\x1b[1;36m-foreground-points\x1b[0m        \x1b[1;33mINTEGER\x1b[0m                                       \x1b[2m│\x1b[0m\n\x1b[2m│\x1b[0m \x1b[1;36m-\x1b[0m\x1b[1;36m-output\x1b[0m                       \x1b[1;33mPATH   \x1b[0m                                       \x1b[2m│\x1b[0m\n\x1b[2m│\x1b[0m \x1b[1;36m-\x1b[0m\x1b[1;36m-spec\x1b[0m                         \x1b[1;33mFILE   \x1b[0m                                       \x1b[2m│\x1b[0m\n\x1b[2m│\x1b[0m \x1b[1;36m-\x1b[0m\x1b[1;36m-help\x1b[0m                         \x1b[1;33m       \x1b[0m  Show this message and exit.          \x1b[2m│\x1b[0m\n\x1b[2m╰──────────────────────────────────────────────────────────────────────────────╯\x1b[0m\n\n'

output     = '\x1b[1m                                                                                \x1b[0m\n\x1b[1m \x1b[0m\x1b[1...   \x1b[2m│\x1b[0m\n\x1b[2m╰──────────────────────────────────────────────────────────────────────────────╯\x1b[0m\n\n'

dimos/cli/test_vqa.py:19: AssertionError

dimos.codebase_checks.test_inline_heavy_imports::test_heavy_imports_are_inline

Stack Traces | 2.32s run time

def test_heavy_imports_are_inline() -> None:
        """Fail if any file imports cv2/open3d/rerun at module level."""
        hits = find_eager_heavy_imports()
        if hits:
            listing = "\n".join(
                f"  - dimos/{f}:{line}: `{module}`"
                for f, lines in sorted(hits.items())
                for line, module in lines
            )
>           raise AssertionError(
                f"Found module-level import(s) of {'/'.join(HEAVY_MODULES)}:\n{listing}\n\n"
                "These libraries load large native extensions into every process that "
                "transitively imports the module. Import them inside the function or "
                "method that uses them; imports needed only for type annotations go "
                "under `if TYPE_CHECKING:`."
            )
E           AssertionError: Found module-level import(s) of cv2/open3d/rerun:
E             - .../benchmark/vqa/evaluation.py:11: `cv2`
E             - .../vqa/generation/dataset.py:11: `cv2`
E             - .../generation/primitives/geometry.py:7: `cv2`
E           
E           These libraries load large native extensions into every process that transitively imports the module. Import them inside the function or method that uses them; imports needed only for type annotations go under `if TYPE_CHECKING:`.

hits       = {'benchmark/vqa/evaluation.py': [(11, 'cv2')], '.../vqa/generation/dataset.py': [(11, 'cv2')], '.../generation/primitives/geometry.py': [(7, 'cv2')]}
listing    = '  - .../benchmark/vqa/evaluation.py:11: `cv2`\n  - .../vqa/generation/dataset.py:11: `cv2`\n  - .../generation/primitives/geometry.py:7: `cv2`'

dimos/codebase_checks/test_inline_heavy_imports.py:92: AssertionError

dimos.cli.test_cli_startup::test_help_startup_time

Stack Traces | 10.9s run time

def test_help_startup_time() -> None:
        """`dimos --help` must finish in under {HELP_TIMEOUT_SECONDS}s."""
        start = time.monotonic()
        result = subprocess.run(
            [sys.executable, "-m", "dimos.cli.dimos", "--help"],
            capture_output=True,
            text=True,
            timeout=HELP_TIMEOUT_SECONDS + 5,  # hard kill safety margin
        )
        elapsed = time.monotonic() - start
        assert result.returncode == 0, f"dimos --help failed:\n{result.stderr}"
>       assert elapsed < HELP_TIMEOUT_SECONDS, (
            f"dimos --help took {elapsed:.1f}s (limit: {HELP_TIMEOUT_SECONDS}s). "
            f"Check for heavy imports in the CLI entrypoint or GlobalConfig."
        )
E       AssertionError: dimos --help took 10.9s (limit: 8s). Check for heavy imports in the CLI entrypoint or GlobalConfig.
E       assert 10.897608509000065 < 8

elapsed    = 10.897608509000065
result     = CompletedProcess(args=['.../dimos/dimos/.venv/bin/python', '-m', 'dimos.cli.dimos', '--help'], returncod...x1b[0m\n\x1b[2m╰──────────────────────────────────────────────────────────────────────────────╯\x1b[0m\n\n', stderr='')
start      = 942.938795319

dimos/cli/test_cli_startup.py:60: AssertionError

To view more test analytics, go to the Test Analytics Dashboard
_{📋 Got 3 mins? Take this short survey to help us improve Test Analytics.}