diff --git a/inference/core/version.py b/inference/core/version.py index d868670e6d..822b9c2b0f 100644 --- a/inference/core/version.py +++ b/inference/core/version.py @@ -1,4 +1,4 @@ -__version__ = "1.7.3" +__version__ = "1.7.3-post1" if __name__ == "__main__": diff --git a/requirements/requirements.workflows.txt b/requirements/requirements.workflows.txt index fdb9242e75..398a41552e 100644 --- a/requirements/requirements.workflows.txt +++ b/requirements/requirements.workflows.txt @@ -1,2 +1,2 @@ # keep in sync with workflows/pyproject.toml version -roboflow-workflows[enterprise]==0.2.4 +roboflow-workflows[enterprise]==0.2.4.post1 diff --git a/tests/workflows/integration_tests/execution/test_workflow_with_mistral_vlm_v1.py b/tests/workflows/integration_tests/execution/test_workflow_with_mistral_vlm_v1.py new file mode 100644 index 0000000000..a079c90f1e --- /dev/null +++ b/tests/workflows/integration_tests/execution/test_workflow_with_mistral_vlm_v1.py @@ -0,0 +1,59 @@ +from inference.core.env import WORKFLOWS_MAX_CONCURRENT_STEPS +from inference.core.managers.base import ModelManager +from inference.core.workflows.core_steps.common.entities import StepExecutionMode +from inference.core.workflows.execution_engine.core import ExecutionEngine + +OBJECT_DETECTION_WORKFLOW = { + "version": "1.0", + "inputs": [ + {"type": "WorkflowImage", "name": "image"}, + {"type": "WorkflowParameter", "name": "api_key"}, + {"type": "WorkflowParameter", "name": "classes"}, + ], + "steps": [ + { + "type": "roboflow_core/mistral_vlm@v1", + "name": "mistral", + "images": "$inputs.image", + "model_version": "Mistral Large 4", + "task_type": "object-detection", + "classes": "$inputs.classes", + "api_key": "$inputs.api_key", + }, + { + "type": "roboflow_core/bounding_box_visualization@v1", + "name": "visualization", + "image": "$inputs.image", + "predictions": "$steps.mistral.predictions", + }, + ], + "outputs": [ + { + "type": "JsonField", + "name": "mistral_result", + "selector": "$steps.mistral.output", + }, + { + "type": "JsonField", + "name": "predictions", + "selector": "$steps.mistral.predictions", + }, + { + "type": "JsonField", + "name": "visualization", + "selector": "$steps.visualization.image", + }, + ], +} + + +def test_object_detection_workflow_compiles(model_manager: ModelManager) -> None: + execution_engine = ExecutionEngine.init( + workflow_definition=OBJECT_DETECTION_WORKFLOW, + init_parameters={ + "workflows_core.model_manager": model_manager, + "workflows_core.step_execution_mode": StepExecutionMode.LOCAL, + }, + max_concurrent_steps=WORKFLOWS_MAX_CONCURRENT_STEPS, + ) + assert execution_engine is not None diff --git a/workflows/CHANGELOG.md b/workflows/CHANGELOG.md index 2adace3b5a..971602c7d5 100644 --- a/workflows/CHANGELOG.md +++ b/workflows/CHANGELOG.md @@ -16,6 +16,18 @@ for contributor and maintainer responsibilities. ## Unreleased + +## `0.2.4-post1` + +Bundled execution engine: `1.16.1`. + +### Added + +- Anthropic Claude block (`anthropic_claude@v5`): `claude-haiku-5-5` model option. +- Mistral AI block (`roboflow_core/mistral_vlm@v1`): runs Mistral Large 4 (`mistralai/mistral-large-4-0`) via OpenRouter with the vlm-exam request contract (image-first user message, reasoning off by default with a `high` option, `max_tokens` unset by default) and in-block decoding of detections and classifications. +- VLM detection box format `xyxy_0_999`: `box_2d` integers normalized to 0-999, Mistral's documented grounding convention. Used by the Mistral AI block and selectable as `detection_format` on the OpenRouter block (`openrouter@v3`). +- The shared OpenRouter executor accepts `max_tokens=None`: the direct path omits the parameter so the provider default applies, the Roboflow-proxied path sends the proxy ceiling (16384) because the proxy requires the field and otherwise applies a 500-token default. + ## `0.2.4` Bundled execution engine: `1.16.1`. diff --git a/workflows/pyproject.toml b/workflows/pyproject.toml index 6a84d102ee..a8d2cfed73 100644 --- a/workflows/pyproject.toml +++ b/workflows/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "roboflow-workflows" -version = "0.2.4" +version = "0.2.4.post1" description = "Roboflow Workflows execution engine and block library" readme = "README.md" license = {file = "LICENSE"} diff --git a/workflows/roboflow_workflows/core_steps/common/openrouter.py b/workflows/roboflow_workflows/core_steps/common/openrouter.py index 2c306e591c..e6d2dada50 100644 --- a/workflows/roboflow_workflows/core_steps/common/openrouter.py +++ b/workflows/roboflow_workflows/core_steps/common/openrouter.py @@ -57,6 +57,12 @@ logger = get_logger(__name__) +# Upper bound the Roboflow proxy accepts for `max_tokens` (observed live: +# "Parameter 'max_tokens' must be an integer in [1, 16384]"). Sent when a +# block leaves `max_tokens` unset, since omitting it makes the proxy apply +# its own 500-token default. +PROXY_MAX_TOKENS_CEILING = 16384 + # --------------------------------------------------------------------------- # Privacy level # --------------------------------------------------------------------------- @@ -274,7 +280,7 @@ def execute_openrouter_batch( openrouter_api_key: str, model: str, prompts: List[List[dict]], - max_tokens: int, + max_tokens: Optional[int], temperature: Optional[float], privacy_level: str, max_concurrent_requests: Optional[int], @@ -310,7 +316,7 @@ def execute_openrouter_batch_with_usage( openrouter_api_key: str, model: str, prompts: List[List[dict]], - max_tokens: int, + max_tokens: Optional[int], temperature: Optional[float], privacy_level: str, max_concurrent_requests: Optional[int], @@ -325,7 +331,10 @@ def execute_openrouter_batch_with_usage( on the returned :class:`OpenRouterResult` objects. ``temperature`` set to ``None`` omits the parameter so the provider - default applies. ``reasoning`` is an optional OpenRouter reasoning + default applies. ``max_tokens`` set to ``None`` does the same on the + direct path; the proxied path sends :data:`PROXY_MAX_TOKENS_CEILING` + instead because the proxy requires the field. + ``reasoning`` is an optional OpenRouter reasoning config object (e.g. ``{"effort": "low"}`` or ``{"enabled": False}``) forwarded verbatim; when the target model rejects the config, the request is retried once without it. @@ -469,7 +478,7 @@ def _execute_proxied_openrouter_request( openrouter_api_key: str, model: str, messages: List[dict], - max_tokens: int, + max_tokens: Optional[int], temperature: Optional[float], privacy_level: str, reasoning: Optional[dict] = None, @@ -479,7 +488,11 @@ def _execute_proxied_openrouter_request( "openrouter_api_key": openrouter_api_key, "model": model, "messages": messages, - "max_tokens": max_tokens, + # The proxy requires the field (absent -> its own 500-token default, + # which truncates detection JSON) and rejects values above the cap. + "max_tokens": ( + max_tokens if max_tokens is not None else PROXY_MAX_TOKENS_CEILING + ), "privacy_level": privacy_level, } if temperature is not None: @@ -558,7 +571,7 @@ def _execute_direct_openrouter_request( api_key: str, model: str, messages: List[dict], - max_tokens: int, + max_tokens: Optional[int], temperature: Optional[float], privacy_level: str, reasoning: Optional[dict] = None, @@ -574,8 +587,9 @@ def _execute_direct_openrouter_request( request_kwargs: Dict[str, Any] = { "model": model, "messages": messages, - "max_tokens": max_tokens, } + if max_tokens is not None: + request_kwargs["max_tokens"] = max_tokens if temperature is not None: request_kwargs["temperature"] = temperature try: diff --git a/workflows/roboflow_workflows/core_steps/common/vlm_decoding/detection_formats.py b/workflows/roboflow_workflows/core_steps/common/vlm_decoding/detection_formats.py index 3ec03260e3..0adab744cc 100644 --- a/workflows/roboflow_workflows/core_steps/common/vlm_decoding/detection_formats.py +++ b/workflows/roboflow_workflows/core_steps/common/vlm_decoding/detection_formats.py @@ -50,6 +50,19 @@ "Only use these labels: {class_list}" ) +# Same wording as `XYXY_0_1000_PROMPT_TEMPLATE` on a 0-999 grid, the +# convention Mistral documents for its models. +XYXY_0_999_PROMPT_TEMPLATE = ( + "Detect all objects in this image. " + "Output a JSON list where each entry contains the 2D bounding box " + 'in the key "box_2d" and the text label in the key "label". ' + 'The "box_2d" value must be [x_min, y_min, x_max, y_max]: the ' + "top-left and bottom-right corners as integers between 0 and 999, " + "normalized to the image width (x) and height (y). " + "Return only the JSON list, with no extra text. " + "Only use these labels: {class_list}" +) + YXYX_0_1000_PROMPT_TEMPLATE = ( "Detect all objects in this image. " "Output a JSON list where each entry contains the 2D bounding box " @@ -82,6 +95,7 @@ # Coordinate scales. NORMALIZED_0_1000_SCALE = 1000.0 +NORMALIZED_0_999_SCALE = 999.0 PERCENT_SCALE = 100.0 # Entry shape vocabulary. Models drift between the prompted keys and their @@ -196,6 +210,24 @@ def _convert_xyxy_0_1000( return [x_min * scale_x, y_min * scale_y, x_max * scale_x, y_max * scale_y] +def _convert_xyxy_0_999( + entry: dict, + image_width: int, + image_height: int, + upload_width: Optional[int], + upload_height: Optional[int], +) -> Optional[List[float]]: + box = _read_box_2d(entry) + if box is None: + return None + x_min, y_min, x_max, y_max = ( + _clamp(value, NORMALIZED_0_999_SCALE) for value in box + ) + scale_x = image_width / NORMALIZED_0_999_SCALE + scale_y = image_height / NORMALIZED_0_999_SCALE + return [x_min * scale_x, y_min * scale_y, x_max * scale_x, y_max * scale_y] + + def _convert_yxyx_0_1000( entry: dict, image_width: int, @@ -365,6 +397,13 @@ def to_pixel_xyxy( converter=_convert_xyxy_0_1000, reads_confidence=False, ), + "xyxy_0_999": DetectionBoxFormat( + name="xyxy_0_999", + prompt_template=XYXY_0_999_PROMPT_TEMPLATE, + requires_upload_dimensions=False, + converter=_convert_xyxy_0_999, + reads_confidence=False, + ), "yxyx_0_1000": DetectionBoxFormat( name="yxyx_0_1000", prompt_template=YXYX_0_1000_PROMPT_TEMPLATE, @@ -399,6 +438,7 @@ def to_pixel_xyxy( BoxFormatName = Literal[ "xyxy_absolute", "xyxy_0_1000", + "xyxy_0_999", "yxyx_0_1000", "xyxy_percent", "named_0_1000", diff --git a/workflows/roboflow_workflows/core_steps/loader.py b/workflows/roboflow_workflows/core_steps/loader.py index 8572cede83..c9f5e6952f 100644 --- a/workflows/roboflow_workflows/core_steps/loader.py +++ b/workflows/roboflow_workflows/core_steps/loader.py @@ -586,6 +586,9 @@ from roboflow_workflows.core_steps.models.foundation.meta_vlm.v1 import MetaVlmBlockV1 from roboflow_workflows.core_steps.models.foundation.meta_vlm.v2 import MetaVlmBlockV2 from roboflow_workflows.core_steps.models.foundation.meta_vlm.v3 import MetaVlmBlockV3 +from roboflow_workflows.core_steps.models.foundation.mistral_vlm.v1 import ( + MistralVlmBlockV1, +) if not ENABLE_TENSOR_DATA_REPRESENTATION: from roboflow_workflows.core_steps.models.foundation.moondream2.v1 import ( @@ -1878,6 +1881,7 @@ def load_blocks() -> List[Type[WorkflowBlock]]: MetaVlmBlockV1, MetaVlmBlockV2, MetaVlmBlockV3, + MistralVlmBlockV1, GoogleGemmaBlockV1, GoogleGemmaBlockV2, GoogleGemmaBlockV3, diff --git a/workflows/roboflow_workflows/core_steps/models/foundation/anthropic_claude/v5.py b/workflows/roboflow_workflows/core_steps/models/foundation/anthropic_claude/v5.py index a7286dee3a..cb275ebacd 100644 --- a/workflows/roboflow_workflows/core_steps/models/foundation/anthropic_claude/v5.py +++ b/workflows/roboflow_workflows/core_steps/models/foundation/anthropic_claude/v5.py @@ -103,6 +103,12 @@ "exact_version": "claude-sonnet-5-5", "max_output_tokens": 128000, }, + { + "id": "claude-haiku-5-5", + "name": "Claude Haiku 5.5", + "exact_version": "claude-haiku-5-5", + "max_output_tokens": 128000, + }, { "id": "claude-fable-5-1", "name": "Claude Fable 5.1", diff --git a/workflows/roboflow_workflows/core_steps/models/foundation/mistral_vlm/__init__.py b/workflows/roboflow_workflows/core_steps/models/foundation/mistral_vlm/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/workflows/roboflow_workflows/core_steps/models/foundation/mistral_vlm/v1.py b/workflows/roboflow_workflows/core_steps/models/foundation/mistral_vlm/v1.py new file mode 100644 index 0000000000..a08c2cccb9 --- /dev/null +++ b/workflows/roboflow_workflows/core_steps/models/foundation/mistral_vlm/v1.py @@ -0,0 +1,734 @@ +"""Mistral AI vision-language block, v1. + +OpenRouter-only. Uses the request contract validated for Mistral Large 4 in +the vlm-exam benchmarks: a single user message with the image part before +the instruction text, no system role, extended reasoning disabled unless the +user asks for ``high`` (the only level Mistral exposes), and an +``[x_min, y_min, x_max, y_max]`` ``box_2d`` detection prompt on a 0-999 +grid, the normalized convention Mistral documents for its models. Detection +and classification answers are decoded in-block, so ``predictions``, +``error_status`` and ``inference_id`` sit next to the raw ``output`` string. +""" + +import base64 +import json +from typing import Any, Dict, List, Literal, Optional, Type, Union +from uuid import uuid4 + +import cv2 +import numpy as np +from pydantic import ConfigDict, Field, field_validator, model_validator +from roboflow_workflows.core_steps.common.openrouter import ( + PRIVACY_LEVEL_LITERAL, + PRIVACY_LEVEL_METADATA, + RELEVANT_TASKS_METADATA, + SUPPORTED_TASK_TYPES_LIST, + OpenRouterBlockManifestMixin, + OpenRouterWorkflowBlockBase, + validate_task_type_required_fields, +) +from roboflow_workflows.core_steps.common.reasoning import ( + attach_reasoning_levels, + build_openrouter_reasoning_config, + validate_reasoning_level, +) +from roboflow_workflows.core_steps.common.token_usage import TOKEN_OUTPUT_DEFINITIONS +from roboflow_workflows.core_steps.common.utils import scale_dimensions_to_max_edge +from roboflow_workflows.core_steps.common.vlm_decoding import ( + actual_vlm_prediction_outputs, + build_object_detection_prompt, + decode_vlm_output, + describe_vlm_prediction_outputs, +) +from roboflow_workflows.execution_engine.entities.base import ( + Batch, + OutputDefinition, + WorkflowImageData, +) +from roboflow_workflows.execution_engine.entities.types import ( + FLOAT_KIND, + IMAGE_KIND, + LANGUAGE_MODEL_OUTPUT_KIND, + LIST_OF_VALUES_KIND, + ROBOFLOW_MANAGED_KEY, + SECRET_KIND, + STRING_KIND, + ImageInputField, + Selector, +) +from roboflow_workflows.execution_engine.entities.workload import ( + Discovery, + RuntimeRestriction, + WorkOperation, +) +from roboflow_workflows.prototypes.block import ( + AirGappedAvailability, + BlockResult, + DependentResource, + WorkflowBlockManifest, + is_workflow_selector, + third_party_model, +) +from roboflow_workflows.utils.images import encode_image_to_jpeg_bytes + +# Detection and classification answers are decoded in-block, so only +# `structured-answering` still points at a downstream parser. +RECOMMENDED_PARSERS = { + "structured-answering": "roboflow_core/json_parser@v1", +} + +# vlm-exam benchmarked Mistral Large 4 with the shared xyxy / 0-1000 prompt; +# Mistral's documented convention is 0-999, so the block uses the 0-999 +# sibling of that format for both the prompt and the coordinate conversion. +DETECTION_BOX_FORMAT = "xyxy_0_999" + +# Mistral Large 4 has two reasoning states. OpenRouter advertises +# `supported_efforts: ["high", "none"]` and accepts the other effort names, +# but a probe (18 detection calls, 3 per level) showed minimal/low/medium/ +# high/xhigh all spend the same 8k-26k reasoning tokens, so only the two +# real states are exposed. +REASONING_EFFORT_OPTIONS = ["none", "high"] +ReasoningEffort = Literal[tuple(REASONING_EFFORT_OPTIONS)] +DEFAULT_REASONING_EFFORT = "none" + +REASONING_EFFORT_METADATA = { + "none": { + "name": "Disabled (recommended)", + "description": ( + "Turns extended reasoning off. Detection answers take about " + "600 output tokens and under 15 seconds per image; this is the " + "configuration validated in the vlm-exam benchmarks." + ), + }, + "high": { + "name": "High", + "description": ( + "Turns extended reasoning on. Mistral does not scale the budget " + "by level: expect roughly 8k-26k reasoning tokens and 1.5-6 " + "minutes per detection image. If you set `max_tokens`, keep it " + "well above that or the answer comes back empty." + ), + }, +} + +# One row per model; add future Mistral models here. A row may override +# `reasoning_levels` if a model diverges from the shared set. +MODEL_VARIANTS: Dict[str, Dict[str, Any]] = { + "Mistral Large 4": { + "model_id": "mistralai/mistral-large-4-0", + }, +} + +MODEL_IDS = {label: variant["model_id"] for label, variant in MODEL_VARIANTS.items()} + +ModelVersion = Literal[tuple(MODEL_VARIANTS.keys())] +DEFAULT_MODEL_VERSION = "Mistral Large 4" + +TaskType = Literal[tuple(SUPPORTED_TASK_TYPES_LIST)] + +MODEL_REASONING_LEVELS = { + label: variant.get("reasoning_levels", REASONING_EFFORT_OPTIONS) + for label, variant in MODEL_VARIANTS.items() +} + +MODEL_VERSION_METADATA = attach_reasoning_levels( + {label: {"name": label} for label in MODEL_VARIANTS}, + MODEL_REASONING_LEVELS, +) + +OPENROUTER_MAX_BASE64_BYTES = 9_500_000 +OPENROUTER_JPEG_QUALITY = 90 + +_TASK_CLASSIFICATION = ( + "You act as single-class classification model. You must provide reasonable " + "predictions. You are only allowed to produce JSON document in Markdown " + "```json [...]``` markers. Expected structure of json: " + '{"class_name": "class-name", "confidence": 0.4}. `class-name` must be one ' + "of the class names defined by user. You are only allowed to return single " + "JSON document, even if there are potentially multiple classes. You are not " + "allowed to return list. You cannot discuss the result, you are only " + "allowed to return JSON document." +) + +_TASK_MULTI_LABEL = ( + "You act as multi-label classification model. You must provide reasonable " + "predictions. You are only allowed to produce JSON document in Markdown " + '```json``` markers. Expected structure of json: {"predicted_classes": ' + '[{"class": "class-name-1", "confidence": 0.9}, {"class": "class-name-2", ' + '"confidence": 0.7}]}. `class-name-X` must be one of the class names ' + "defined by user and `confidence` is a float value in range 0.0-1.0 that " + "represent how sure you are that the class is present in the image. Only " + "return class names that are visible. You cannot discuss the result, you " + "are only allowed to return JSON document." +) + +_TASK_VQA = ( + "You act as Visual Question Answering model. Your task is to provide " + "answer to question submitted by user. If this is open-question - answer " + "with few sentences, for ABCD question, return only the indicator of " + "the answer." +) + +_TASK_OCR = ( + "You act as OCR model. Your task is to read text from the image and " + "return it in paragraphs representing the structure of texts in the " + "image. You should only return recognised text, nothing else." +) + +_TASK_STRUCTURED = ( + "You are supposed to produce responses in JSON wrapped in Markdown " + "markers: ```json\nyour-response\n```. User is to provide you " + "dictionary with keys and values. Each key must be present in your " + "response. Values in user dictionary represent descriptions for JSON " + "fields to be generated. Provide only JSON Markdown in response." +) + + +def encode_image_for_mistral_openrouter(numpy_image: np.ndarray) -> str: + """Encode a BGR image as base64 JPEG under the OpenRouter payload cap. + + Encodes at fixed JPEG quality and, when the base64 payload exceeds + ``OPENROUTER_MAX_BASE64_BYTES``, iteratively downscales the longest + edge by 10% until it fits. Detection coordinates are normalized to + 0-999, so downscaling does not affect decoding. + + Args: + numpy_image: Image in BGR channel order. + + Returns: + Base64-encoded JPEG payload. + """ + working = numpy_image + while True: + jpeg_bytes = encode_image_to_jpeg_bytes( + working, jpeg_quality=OPENROUTER_JPEG_QUALITY + ) + base64_image = base64.b64encode(jpeg_bytes).decode("ascii") + if len(base64_image) <= OPENROUTER_MAX_BASE64_BYTES: + return base64_image + + height, width = working.shape[:2] + if max(height, width) <= 1: + return base64_image + + target_max_edge = max(int(max(height, width) * 0.9), 1) + target_width, target_height = scale_dimensions_to_max_edge( + width, height, target_max_edge + ) + working = cv2.resize( + working, (target_width, target_height), interpolation=cv2.INTER_AREA + ) + + +def _user_message(base64_image: str, text: str) -> List[dict]: + return [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": f"data:image/jpeg;base64,{base64_image}", + }, + }, + {"type": "text", "text": text}, + ], + } + ] + + +def _prepare_unconstrained_prompt(base64_image: str, prompt: str, **_) -> List[dict]: + return _user_message(base64_image=base64_image, text=prompt) + + +def _prepare_ocr_prompt(base64_image: str, **_) -> List[dict]: + return _user_message(base64_image=base64_image, text=_TASK_OCR) + + +def _prepare_vqa_prompt(base64_image: str, prompt: str, **_) -> List[dict]: + return _user_message( + base64_image=base64_image, text=f"{_TASK_VQA}\n\nQuestion: {prompt}" + ) + + +def _prepare_caption_prompt( + base64_image: str, short_description: bool, **_ +) -> List[dict]: + detail = ( + "Caption should be short." + if short_description + else "Caption should be extensive." + ) + text = ( + "You act as image caption model. Your task is to provide " + f"description of the image. {detail}" + ) + return _user_message(base64_image=base64_image, text=text) + + +def _prepare_classification_prompt( + base64_image: str, classes: List[str], **_ +) -> List[dict]: + text = ( + f"{_TASK_CLASSIFICATION}\n\n" + f"List of all classes to be recognised by model: {', '.join(classes)}" + ) + return _user_message(base64_image=base64_image, text=text) + + +def _prepare_multi_label_classification_prompt( + base64_image: str, classes: List[str], **_ +) -> List[dict]: + text = ( + f"{_TASK_MULTI_LABEL}\n\n" + f"List of all classes to be recognised by model: {', '.join(classes)}" + ) + return _user_message(base64_image=base64_image, text=text) + + +def _prepare_structured_answering_prompt( + base64_image: str, output_structure: Dict[str, str], **_ +) -> List[dict]: + text = ( + f"{_TASK_STRUCTURED}\n\n" + "Specification of requirements regarding output fields: \n" + f"{json.dumps(output_structure, indent=4)}" + ) + return _user_message(base64_image=base64_image, text=text) + + +def _prepare_object_detection_prompt( + base64_image: str, classes: List[str], **_ +) -> List[dict]: + text = build_object_detection_prompt( + box_format=DETECTION_BOX_FORMAT, classes=classes + ) + return _user_message(base64_image=base64_image, text=text) + + +PROMPT_BUILDERS = { + "unconstrained": _prepare_unconstrained_prompt, + "ocr": _prepare_ocr_prompt, + "visual-question-answering": _prepare_vqa_prompt, + "caption": lambda **kwargs: _prepare_caption_prompt( + short_description=True, **kwargs + ), + "detailed-caption": lambda **kwargs: _prepare_caption_prompt( + short_description=False, **kwargs + ), + "classification": _prepare_classification_prompt, + "multi-label-classification": _prepare_multi_label_classification_prompt, + "structured-answering": _prepare_structured_answering_prompt, + "object-detection": _prepare_object_detection_prompt, +} + + +def build_mistral_openrouter_prompts( + images: List[np.ndarray], + task_type: str, + prompt: Optional[str], + output_structure: Optional[Dict[str, str]], + classes: Optional[List[str]], +) -> List[List[dict]]: + """Build one OpenRouter ``messages`` array per input image. + + Every task sends a single user message with the image part first and + the instruction text second, with no system role, matching the + vlm-exam Mistral request contract. + + Args: + images: BGR numpy images. + task_type: One of the supported VLM task types. + prompt: User prompt for unconstrained / VQA tasks. + output_structure: Output spec for structured-answering. + classes: Class list for classification / detection tasks. + + Returns: + List of ``messages`` arrays, one per image. + + Raises: + ValueError: If the task type is not supported. + """ + if task_type not in PROMPT_BUILDERS: + raise ValueError(f"Task type: {task_type} not supported.") + + builder = PROMPT_BUILDERS[task_type] + built: List[List[dict]] = [] + for image in images: + built.append( + builder( + base64_image=encode_image_for_mistral_openrouter(numpy_image=image), + prompt=prompt, + output_structure=output_structure, + classes=classes, + ) + ) + + return built + + +RELEVANT_TASKS_DOCS_DESCRIPTION = "\n\n".join( + f"* **{v['name']}** (`{k}`) - {v['description']}" + for k, v in RELEVANT_TASKS_METADATA.items() +) + +LONG_DESCRIPTION = f""" +Run Mistral AI vision-language models via [OpenRouter](https://openrouter.ai/). + +Supported models: Mistral Large 4. + +You can specify arbitrary text prompts or predefined ones. The block supports: + +{RELEVANT_TASKS_DOCS_DESCRIPTION} + +Object detection asks for a JSON list of `box_2d` / `label` entries with +`[x_min, y_min, x_max, y_max]` integer coordinates normalized to 0-999, the +convention Mistral documents for its models, and decodes them on the same +0-999 grid. + +The block decodes the model answer inside the block, adding `predictions`, +`error_status` and `inference_id` outputs next to the raw `output` string: + +* `predictions` holds object detections for the `object-detection` task and a +classification prediction for the `classification` / +`multi-label-classification` tasks - the kind of the output follows the +selected task type. +* `predictions` is `None` for every other task (unconstrained prompting, OCR, +captioning, structured answering, visual question answering). +* `error_status` is `True` when the answer could not be decoded. +* `inference_id` is generated per image. + +Every request sends the image before the instruction text in a single user +message. Mistral exposes extended reasoning as on or off only; the block +disables it by default (the benchmarked configuration) and `reasoning_effort` +set to `high` turns it on. `max_tokens` is unset by default: your own +OpenRouter key gets the model's own output limit, the managed key gets the +Roboflow proxy maximum of 16384. When you do cap it with reasoning on, leave +room for a thinking trace of 8k-26k tokens. + +By default the block uses the Roboflow-managed OpenRouter key and bills +your Roboflow credits. Paste your own `sk-or-...` key to call OpenRouter +directly. +""" + + +class BlockManifest(OpenRouterBlockManifestMixin): + model_config = ConfigDict( + json_schema_extra={ + "name": "Mistral AI", + "version": "v1", + "short_description": "Run Mistral AI vision models via OpenRouter.", + "long_description": LONG_DESCRIPTION, + "license": "Apache-2.0", + "block_type": "model", + "search_keywords": [ + "LMM", + "VLM", + "Mistral", + "Mistral Large", + "Mistral AI", + "OpenRouter", + ], + "is_vlm_block": True, + "task_type_property": "task_type", + "ui_manifest": { + "section": "model", + "icon": "fal fa-atom", + "blockPriority": 5.57, + }, + }, + protected_namespaces=(), + ) + type: Literal["roboflow_core/mistral_vlm@v1"] + images: Selector(kind=[IMAGE_KIND]) = ImageInputField + model_version: Union[Selector(kind=[STRING_KIND]), ModelVersion] = Field( + default=DEFAULT_MODEL_VERSION, + description="Mistral AI model to run.", + examples=[DEFAULT_MODEL_VERSION], + json_schema_extra={ + "values_metadata": MODEL_VERSION_METADATA, + }, + ) + task_type: TaskType = Field( + default="unconstrained", + description=( + "Task type to be performed by model. Value determines required " + "parameters and output response." + ), + json_schema_extra={ + "values_metadata": RELEVANT_TASKS_METADATA, + "recommended_parsers": RECOMMENDED_PARSERS, + "always_visible": True, + }, + ) + prompt: Optional[Union[Selector(kind=[STRING_KIND]), str]] = Field( + default=None, + description="Text prompt to send to the model.", + examples=["my prompt", "$inputs.prompt"], + json_schema_extra={ + "relevant_for": { + "task_type": { + "values": ["unconstrained", "visual-question-answering"], + "required": True, + }, + }, + "multiline": True, + }, + ) + output_structure: Optional[Dict[str, str]] = Field( + default=None, + description="Dictionary with structure of expected JSON response.", + examples=[{"my_key": "description"}, "$inputs.output_structure"], + json_schema_extra={ + "relevant_for": { + "task_type": {"values": ["structured-answering"], "required": True}, + }, + }, + ) + classes: Optional[Union[Selector(kind=[LIST_OF_VALUES_KIND]), List[str]]] = Field( + default=None, + description="List of classes to be used.", + examples=[["class-a", "class-b"], "$inputs.classes"], + json_schema_extra={ + "relevant_for": { + "task_type": { + "values": [ + "classification", + "multi-label-classification", + "object-detection", + ], + "required": True, + }, + }, + }, + ) + reasoning_effort: ReasoningEffort = Field( + default=DEFAULT_REASONING_EFFORT, + description=( + "Extended reasoning. Mistral exposes it as on or off only: " + "`none` (the default) disables it, matching the configuration " + "validated in the vlm-exam benches; `high` turns it on." + ), + json_schema_extra={"values_metadata": REASONING_EFFORT_METADATA}, + ) + max_tokens: Optional[int] = Field( + default=None, + description=( + "Maximum number of tokens the model can generate in its response. " + "If not specified, the model's own limit applies with your own " + "OpenRouter key and 16384 (the Roboflow proxy maximum) with the " + "managed key. When set with `reasoning_effort` `high`, keep it " + "well above the reasoning trace (8k-26k tokens) or the answer " + "comes back empty." + ), + gt=1, + ) + temperature: Optional[Union[float, Selector(kind=[FLOAT_KIND])]] = Field( + default=None, + description=( + "Sampling temperature. Left unset by default so the provider " + "default applies. Range 0.0-2.0." + ), + ) + api_key: Union[ + Selector(kind=[STRING_KIND, SECRET_KIND, ROBOFLOW_MANAGED_KEY]), str + ] = Field( + default="rf_key:account", + description=( + "OpenRouter API key. Defaults to Roboflow's managed key. Provide " + "your own `sk-or-...` key to call OpenRouter directly without " + "Roboflow billing." + ), + examples=["rf_key:account", "sk-or-...", "$inputs.openrouter_api_key"], + private=True, + ) + privacy_level: PRIVACY_LEVEL_LITERAL = Field( + default="deny", + description=( + "Provider privacy filter. Stricter levels reduce the pool of " + "providers and may increase per-call cost on the managed key." + ), + json_schema_extra={"values_metadata": PRIVACY_LEVEL_METADATA}, + ) + + @model_validator(mode="after") + def validate(self) -> "BlockManifest": + validate_task_type_required_fields( + task_type=self.task_type, + prompt=self.prompt, + classes=self.classes, + output_structure=self.output_structure, + ) + validate_reasoning_level( + model=self.model_version, + level=self.reasoning_effort, + levels_by_model=MODEL_REASONING_LEVELS, + ) + return self + + @field_validator("temperature") + @classmethod + def validate_temperature( + cls, value: Optional[Union[str, float]] + ) -> Optional[Union[str, float]]: + if value is None or isinstance(value, str): + return value + if value < 0.0 or value > 2.0: + raise ValueError( + "'temperature' parameter required to be in range [0.0, 2.0]" + ) + return value + + @classmethod + def get_air_gapped_availability(cls) -> AirGappedAvailability: + return AirGappedAvailability(available=False, reason="requires_internet") + + @classmethod + def get_parameters_accepting_batches(cls) -> List[str]: + return ["images"] + + @classmethod + def describe_outputs(cls) -> List[OutputDefinition]: + return [ + *cls._describe_raw_outputs(), + *describe_vlm_prediction_outputs(), + ] + + def get_actual_outputs(self) -> List[OutputDefinition]: + return [ + *self._describe_raw_outputs(), + *actual_vlm_prediction_outputs(self.task_type), + ] + + @classmethod + def _describe_raw_outputs(cls) -> List[OutputDefinition]: + return [ + OutputDefinition( + name="output", kind=[STRING_KIND, LANGUAGE_MODEL_OUTPUT_KIND] + ), + OutputDefinition(name="classes", kind=[LIST_OF_VALUES_KIND]), + OutputDefinition( + name="thinking", + kind=[STRING_KIND], + description=( + "Reasoning trace from the model when OpenRouter returns " + "one. Empty string otherwise." + ), + ), + *TOKEN_OUTPUT_DEFINITIONS, + ] + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + def discover_dependent_resources(self) -> Optional[List[DependentResource]]: + if is_workflow_selector(self.model_version): + return [ + third_party_model( + provider="openrouter", + model_id=self.model_version, + model_id_resolver=lambda label: MODEL_IDS.get(label), + ) + ] + return [ + third_party_model( + provider="openrouter", + model_id=MODEL_IDS[self.model_version], + ) + ] + + def discover_work_operations(self) -> List[WorkOperation]: + return [ + WorkOperation.MODEL_INFERENCE, + WorkOperation.EXTERNAL_REQUEST, + WorkOperation.IMAGE_ENCODING, + ] + + def get_actual_restrictions( + self, *, ignore_environment_restrictions: bool = False + ) -> Discovery[RuntimeRestriction]: + return Discovery[RuntimeRestriction]( + items=[], complete=True, unknown_reasons=[] + ) + + +class MistralVlmBlockV1(OpenRouterWorkflowBlockBase): + @classmethod + def get_manifest(cls) -> Type[WorkflowBlockManifest]: + return BlockManifest + + @classmethod + def get_execution_engine_compatibility(cls) -> Optional[str]: + return ">=1.3.0,<2.0.0" + + def run( + self, + images: Batch[WorkflowImageData], + model_version: str, + task_type: str, + prompt: Optional[str], + output_structure: Optional[Dict[str, str]], + classes: Optional[List[str]], + reasoning_effort: str, + api_key: str, + privacy_level: str, + max_tokens: Optional[int], + temperature: Optional[float], + max_concurrent_requests: Optional[int], + ) -> BlockResult: + variant = MODEL_VARIANTS.get(model_version) + if variant is None: + raise ValueError( + f"Unknown Mistral AI model '{model_version}'. " + f"Pick one of: {list(MODEL_VARIANTS)}" + ) + validate_reasoning_level( + model=model_version, + level=reasoning_effort, + levels_by_model=MODEL_REASONING_LEVELS, + ) + + prompts = build_mistral_openrouter_prompts( + images=[image.numpy_image for image in images], + task_type=task_type, + prompt=prompt, + output_structure=output_structure, + classes=classes, + ) + results = self.execute_openrouter_batch_with_usage( + openrouter_api_key=api_key, + model=variant["model_id"], + prompts=prompts, + max_tokens=max_tokens, + temperature=temperature, + privacy_level=privacy_level, + max_concurrent_requests=max_concurrent_requests, + reasoning=build_openrouter_reasoning_config(reasoning_effort), + ) + + outputs = [] + for image, result in zip(images, results): + inference_id = str(uuid4()) + error_status, predictions = decode_vlm_output( + task_type=task_type, + raw_output=result.content, + image=image, + classes=classes, + inference_id=inference_id, + box_format=DETECTION_BOX_FORMAT, + ) + outputs.append( + { + "output": result.content, + "classes": classes, + "thinking": result.reasoning_trace, + "input_tokens": result.input_tokens, + "output_tokens": result.output_tokens, + "predictions": predictions, + "error_status": error_status, + "inference_id": inference_id, + } + ) + + return outputs diff --git a/workflows/roboflow_workflows/core_steps/models/foundation/openrouter/v3.py b/workflows/roboflow_workflows/core_steps/models/foundation/openrouter/v3.py index 34e5d805a4..7297e54c52 100644 --- a/workflows/roboflow_workflows/core_steps/models/foundation/openrouter/v3.py +++ b/workflows/roboflow_workflows/core_steps/models/foundation/openrouter/v3.py @@ -84,6 +84,10 @@ "name": "xyxy, 0-1000", "description": "`box_2d` list of integers normalized to 0-1000.", }, + "xyxy_0_999": { + "name": "xyxy, 0-999", + "description": "`box_2d` list of integers normalized to 0-999.", + }, "yxyx_0_1000": { "name": "yxyx, 0-1000", "description": "`box_2d` list of integers normalized to 0-1000, y first.", diff --git a/workflows/tests/unit_tests/core_steps/common/test_openrouter_max_tokens.py b/workflows/tests/unit_tests/core_steps/common/test_openrouter_max_tokens.py new file mode 100644 index 0000000000..5f12cb9fa3 --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/common/test_openrouter_max_tokens.py @@ -0,0 +1,86 @@ +"""How `max_tokens=None` reaches OpenRouter on each path. + +Blocks that follow the provider-default convention (``max_tokens`` unset) +pass ``None`` through the shared executor. The direct OpenAI-SDK path must +omit the key. The Roboflow proxy requires it (absent -> its own 500-token +default, which truncated detection JSON live) and caps it at 16384, so the +proxied path sends that ceiling instead. +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import pytest +from roboflow_workflows.core_steps.common.openrouter import ( + PROXY_MAX_TOKENS_CEILING, + _execute_direct_openrouter_request, + _execute_proxied_openrouter_request, +) + +from tests.unit_tests.prototypes.platform_client_double import RecordingPlatformClient + +MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] +PROXY_RESPONSE = { + "choices": [{"message": {"content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1}, +} + + +def _sdk_response() -> SimpleNamespace: + message = SimpleNamespace(content="ok", reasoning=None) + choice = SimpleNamespace(message=message, finish_reason="stop") + usage = SimpleNamespace(prompt_tokens=3, completion_tokens=1) + return SimpleNamespace(choices=[choice], usage=usage) + + +@pytest.mark.parametrize( + "max_tokens,expected_sent", + [(None, PROXY_MAX_TOKENS_CEILING), (4096, 4096)], +) +def test_proxied_request_always_sends_an_integer_max_tokens( + max_tokens, expected_sent +) -> None: + client = RecordingPlatformClient(post_response=PROXY_RESPONSE) + + result = _execute_proxied_openrouter_request( + roboflow_api_key="workspace-key", + platform_client=client, + openrouter_api_key="rf_key:account", + model="mistralai/mistral-large-4-0", + messages=MESSAGES, + max_tokens=max_tokens, + temperature=None, + privacy_level="deny", + ) + + payload = client.posts[0]["payload"] + assert result.content == "ok" + assert payload["max_tokens"] == expected_sent + assert 1 <= payload["max_tokens"] <= 16384 + assert "temperature" not in payload + + +@pytest.mark.parametrize("max_tokens", [None, 4096]) +def test_direct_request_only_sends_max_tokens_when_set(max_tokens) -> None: + client = MagicMock() + client.chat.completions.create.return_value = _sdk_response() + + with patch( + "roboflow_workflows.core_steps.common.openrouter.OpenAI", return_value=client + ): + result = _execute_direct_openrouter_request( + api_key="sk-or-test", + model="mistralai/mistral-large-4-0", + messages=MESSAGES, + max_tokens=max_tokens, + temperature=None, + privacy_level="deny", + ) + + kwargs = client.chat.completions.create.call_args.kwargs + assert result.content == "ok" + if max_tokens is None: + assert "max_tokens" not in kwargs + else: + assert kwargs["max_tokens"] == max_tokens + assert "temperature" not in kwargs diff --git a/workflows/tests/unit_tests/core_steps/common/test_reasoning_contract.py b/workflows/tests/unit_tests/core_steps/common/test_reasoning_contract.py index 59ef54500a..73902f8650 100644 --- a/workflows/tests/unit_tests/core_steps/common/test_reasoning_contract.py +++ b/workflows/tests/unit_tests/core_steps/common/test_reasoning_contract.py @@ -10,6 +10,9 @@ v5 as gemini_v5, ) from roboflow_workflows.core_steps.models.foundation.meta_vlm import v2 as meta_vlm_v2 +from roboflow_workflows.core_steps.models.foundation.mistral_vlm import ( + v1 as mistral_vlm_v1, +) from roboflow_workflows.core_steps.models.foundation.openai import v6 as openai_v6 from roboflow_workflows.core_steps.models.foundation.qwen_vlm import v3 as qwen_vlm_v3 from roboflow_workflows.core_steps.models.foundation.spacexai import v2 as spacexai_v2 @@ -96,6 +99,12 @@ def test_validate_reasoning_level_rejects_model_without_levels(): zai_vlm_v1.MODEL_VERSION_METADATA, zai_vlm_v1.DEFAULT_REASONING_EFFORT, ), + "mistral_vlm@v1": ( + mistral_vlm_v1.MODEL_REASONING_LEVELS, + mistral_vlm_v1.REASONING_EFFORT_OPTIONS, + mistral_vlm_v1.MODEL_VERSION_METADATA, + mistral_vlm_v1.DEFAULT_REASONING_EFFORT, + ), } diff --git a/workflows/tests/unit_tests/core_steps/common/test_vlm_decoding.py b/workflows/tests/unit_tests/core_steps/common/test_vlm_decoding.py index a76c4a0627..edb97cc3f9 100644 --- a/workflows/tests/unit_tests/core_steps/common/test_vlm_decoding.py +++ b/workflows/tests/unit_tests/core_steps/common/test_vlm_decoding.py @@ -204,6 +204,11 @@ def test_small_utils() -> None: {"bbox_2d": [100, 250, 500, 750], "label": "cat"}, id="xyxy_0_1000-bbox-alias", ), + pytest.param( + "xyxy_0_999", + {"box_2d": [99.9, 249.75, 499.5, 749.25], "label": "cat"}, + id="xyxy_0_999", + ), pytest.param( "yxyx_0_1000", {"box_2d": [250, 100, 750, 500], "label": "cat"}, @@ -303,6 +308,24 @@ def test_decode_object_detections_clamps_out_of_range_coordinates() -> None: ] +def test_decode_object_detections_xyxy_0_999_scales_by_999_not_1000() -> None: + # 1998x999 image: every 0-999 coordinate maps onto exactly 2x / 1x pixels, + # so a 0-1000 decoder would land 0.1% short on the far corner. + image = _build_image(width=1998, height=999) + + error_status, detections = decode_object_detections( + raw_output='[{"box_2d": [100, 250, 999, 1200], "label": "cat"}]', + box_format="xyxy_0_999", + image=image, + classes=["cat"], + inference_id="inference-id", + ) + + assert error_status is False + assert detections.xyxy.tolist() == [[200.0, 250.0, 1998.0, 999.0]] + assert detections.confidence.tolist() == [1.0] + + # --------------------------------------------------------------------------- # Shared decoding behavior # --------------------------------------------------------------------------- diff --git a/workflows/tests/unit_tests/core_steps/dependent_resources/test_mistral_vlm.py b/workflows/tests/unit_tests/core_steps/dependent_resources/test_mistral_vlm.py new file mode 100644 index 0000000000..320f70b876 --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/dependent_resources/test_mistral_vlm.py @@ -0,0 +1,50 @@ +""" +Dependent-resources discovery tests for ``roboflow_core/mistral_vlm@v1``, +served via OpenRouter. Friendly labels map through ``MODEL_IDS`` to +OpenRouter slugs, the same mapping ``run()`` resolves before calling +OpenRouter. +""" + +from roboflow_workflows.core_steps.models.foundation.mistral_vlm.v1 import ( + MODEL_IDS, + BlockManifest, +) +from roboflow_workflows.prototypes.block import third_party_model + + +def _build_payload(**overrides) -> dict: + payload = { + "type": "roboflow_core/mistral_vlm@v1", + "name": "vlm", + "images": "$inputs.image", + "task_type": "caption", + } + payload.update(overrides) + return payload + + +def test_mistral_vlm_v1_default_label_resolves_to_catalog_slug() -> None: + manifest = BlockManifest.model_validate(_build_payload()) + + assert MODEL_IDS["Mistral Large 4"] == "mistralai/mistral-large-4-0" + assert manifest.discover_dependent_resources() == [ + third_party_model( + provider="openrouter", model_id="mistralai/mistral-large-4-0" + ), + ] + + +def test_mistral_vlm_v1_selector_fed_label_is_returned_verbatim() -> None: + manifest = BlockManifest.model_validate( + _build_payload(model_version="$inputs.mistral_model") + ) + + resources = manifest.discover_dependent_resources() + + assert resources == [ + third_party_model(provider="openrouter", model_id="$inputs.mistral_model"), + ] + resolver = resources[0].metadata.model_id_resolver + assert resolver is not None + assert resolver("Mistral Large 4") == "mistralai/mistral-large-4-0" + assert resolver("not-a-model") is None diff --git a/workflows/tests/unit_tests/core_steps/models/foundation/test_mistral_vlm_v1.py b/workflows/tests/unit_tests/core_steps/models/foundation/test_mistral_vlm_v1.py new file mode 100644 index 0000000000..2efb47d855 --- /dev/null +++ b/workflows/tests/unit_tests/core_steps/models/foundation/test_mistral_vlm_v1.py @@ -0,0 +1,409 @@ +"""Contract tests for the Mistral AI VLM v1 block. + +Covers what would ship wrong boxes or dead calls if broken: manifest +defaults, the outbound OpenRouter request (slug, reasoning, token budget, +temperature, message layout), the 0-999 detection prompt actually sent, and +in-block decoding on the 0-999 grid. +""" + +import itertools +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest +from pydantic import ValidationError +from roboflow_workflows.core_steps.common.openrouter import ( + SUPPORTED_TASK_TYPES_LIST, + OpenRouterResult, +) +from roboflow_workflows.core_steps.models.foundation.mistral_vlm.v1 import ( + DEFAULT_MODEL_VERSION, + DEFAULT_REASONING_EFFORT, + REASONING_EFFORT_OPTIONS, + BlockManifest, + MistralVlmBlockV1, +) +from roboflow_workflows.execution_engine.entities.base import WorkflowImageData + +from tests.unit_tests.core_steps._vlm_prediction_readers import ( + classification_top_class, + classification_top_confidence, + detection_boxes, + detection_class_ids, + detection_class_names, + is_detection_prediction, +) + +OPENROUTER_SEAM = ( + "roboflow_workflows.core_steps.models.foundation.mistral_vlm.v1." + "OpenRouterWorkflowBlockBase.execute_openrouter_batch_with_usage" +) + +# vlm-exam `_NORMALIZED_XYXY_PROMPT_TEMPLATE` with the grid changed from +# 0-1000 to Mistral's documented 0-999, copied here verbatim so an edit to +# the shared template fails this test. +EXPECTED_DETECTION_PROMPT = ( + "Detect all objects in this image. " + "Output a JSON list where each entry contains the 2D bounding box " + 'in the key "box_2d" and the text label in the key "label". ' + 'The "box_2d" value must be [x_min, y_min, x_max, y_max]: the ' + "top-left and bottom-right corners as integers between 0 and 999, " + "normalized to the image width (x) and height (y). " + "Return only the JSON list, with no extra text. " + "Only use these labels: cat, dog" +) + +# Image is 1998 wide x 999 high, so every 0-999 coordinate maps onto exactly +# 2x (x axis) / 1x (y axis) pixels. A 0-1000 decoder would land the far +# corner at 1996x998 instead of the full image. +IMAGE_WIDTH = 1998 +IMAGE_HEIGHT = 999 +DETECTION_OUTPUT = '[{"box_2d": [100, 250, 999, 999], "label": "cat"}]' +DETECTION_OUTPUT_BBOX_ALIAS = '[{"bbox_2d": [100, 250, 999, 999], "label": "cat"}]' +EXPECTED_XYXY = [[200.0, 250.0, 1998.0, 999.0]] + +CLASSIFICATION_OUTPUT = '{"class_name": "cat", "confidence": 0.9}' + +# Effort names OpenRouter accepts for other models; Mistral Large 4 maps all +# of them onto the same budget as `high`, so the block refuses them. +UNEXPOSED_REASONING_LEVELS = ["minimal", "low", "medium", "xhigh"] + +# Minimum extra inputs each task needs to pass manifest validation and build +# a prompt; `output_structure` / `classes` double as the decode inputs. +TASK_INPUTS = { + "unconstrained": {"prompt": "describe"}, + "visual-question-answering": {"prompt": "what is it?"}, + "structured-answering": {"output_structure": {"animal": "name"}}, + "classification": {"classes": ["cat", "dog"]}, + "multi-label-classification": {"classes": ["cat", "dog"]}, + "object-detection": {"classes": ["cat", "dog"]}, + "ocr": {}, + "caption": {}, + "detailed-caption": {}, +} +DECODING_TASKS = {"object-detection", "classification", "multi-label-classification"} + + +def _stub_image() -> WorkflowImageData: + return WorkflowImageData( + parent_metadata=MagicMock( + parent_id="root", workflow_root_ancestor_metadata=None + ), + numpy_image=np.zeros((IMAGE_HEIGHT, IMAGE_WIDTH, 3), dtype=np.uint8), + ) + + +def _base_run_kwargs(**overrides): + kwargs = dict( + images=[_stub_image()], + model_version=DEFAULT_MODEL_VERSION, + task_type="caption", + prompt=None, + output_structure=None, + classes=None, + reasoning_effort=DEFAULT_REASONING_EFFORT, + api_key="rf_key:account", + privacy_level="deny", + max_tokens=None, + temperature=None, + max_concurrent_requests=None, + ) + kwargs.update(overrides) + return kwargs + + +def _manifest(**overrides) -> BlockManifest: + payload = { + "type": "roboflow_core/mistral_vlm@v1", + "name": "mistral", + "images": "$inputs.image", + "task_type": "caption", + } + payload.update(overrides) + return BlockManifest.model_validate(payload) + + +def _result(content: str) -> list: + return [ + OpenRouterResult( + content=content, reasoning_trace="", input_tokens=20, output_tokens=8 + ) + ] + + +def _kinds(outputs, name): + return [k.name for o in outputs if o.name == name for k in o.kind] + + +def test_manifest_defaults(): + manifest = _manifest() + + assert manifest.model_version == "Mistral Large 4" + assert manifest.reasoning_effort == "none" + assert manifest.max_tokens is None + assert manifest.temperature is None + assert manifest.api_key == "rf_key:account" + assert manifest.privacy_level == "deny" + assert {o.name for o in BlockManifest.describe_outputs()} >= { + "output", + "thinking", + "input_tokens", + "output_tokens", + "predictions", + "error_status", + "inference_id", + } + + +@pytest.mark.parametrize("reasoning_effort", REASONING_EFFORT_OPTIONS) +def test_manifest_accepts_every_exposed_reasoning_level(reasoning_effort): + assert _manifest(reasoning_effort=reasoning_effort).reasoning_effort == ( + reasoning_effort + ) + + +@pytest.mark.parametrize("reasoning_effort", UNEXPOSED_REASONING_LEVELS) +def test_manifest_rejects_reasoning_levels_mistral_does_not_expose(reasoning_effort): + with pytest.raises(ValidationError): + _manifest(reasoning_effort=reasoning_effort) + + +def test_manifest_accepts_selector_fed_model_version_without_level_check(): + manifest = _manifest(model_version="$inputs.model", reasoning_effort="high") + + assert manifest.model_version == "$inputs.model" + assert manifest.reasoning_effort == "high" + + +def test_manifest_rejects_unknown_model_version(): + with pytest.raises(ValidationError): + _manifest(model_version="Pixtral Large") + + +@pytest.mark.parametrize("max_tokens", [0, 1, -5]) +def test_manifest_rejects_max_tokens_at_or_below_one(max_tokens): + with pytest.raises(ValidationError): + _manifest(max_tokens=max_tokens) + + +@pytest.mark.parametrize("max_tokens", [None, 2, 4096]) +def test_manifest_accepts_unset_or_positive_max_tokens(max_tokens): + assert _manifest(max_tokens=max_tokens).max_tokens == max_tokens + + +@pytest.mark.parametrize("temperature", [-0.1, 2.1]) +def test_manifest_rejects_temperature_outside_range(temperature): + with pytest.raises(ValidationError): + _manifest(temperature=temperature) + + +def test_manifest_recommends_parser_only_for_structured_answering(): + recommended = BlockManifest.model_fields["task_type"].json_schema_extra[ + "recommended_parsers" + ] + + assert recommended == {"structured-answering": "roboflow_core/json_parser@v1"} + + +def test_get_actual_outputs_narrows_predictions_kind_per_task(): + detection = _manifest( + task_type="object-detection", classes=["cat"] + ).get_actual_outputs() + classification = _manifest( + task_type="classification", classes=["cat"] + ).get_actual_outputs() + caption = _manifest().get_actual_outputs() + + assert _kinds(detection, "predictions") == ["object_detection_prediction"] + assert _kinds(classification, "predictions") == ["classification_prediction"] + assert _kinds(caption, "predictions") == [ + "object_detection_prediction", + "classification_prediction", + ] + + +@patch(OPENROUTER_SEAM) +def test_run_sends_vlm_exam_request_contract(mock_or): + mock_or.return_value = _result(DETECTION_OUTPUT) + block = MistralVlmBlockV1(model_manager=MagicMock(), api_key="rf_key") + + block.run(**_base_run_kwargs(task_type="object-detection", classes=["cat", "dog"])) + + kwargs = mock_or.call_args.kwargs + assert kwargs["model"] == "mistralai/mistral-large-4-0" + assert kwargs["reasoning"] == {"enabled": False} + assert kwargs["max_tokens"] is None + assert kwargs["temperature"] is None + assert kwargs["privacy_level"] == "deny" + messages = kwargs["prompts"][0] + assert [message["role"] for message in messages] == ["user"] + content = messages[0]["content"] + assert content[0]["type"] == "image_url" + assert content[0]["image_url"]["url"].startswith("data:image/jpeg;base64,") + assert content[1] == {"type": "text", "text": EXPECTED_DETECTION_PROMPT} + + +EXPECTED_REASONING_CONFIG = {"none": {"enabled": False}, "high": {"effort": "high"}} + + +@pytest.mark.parametrize( + "reasoning_effort,max_tokens,temperature", + list(itertools.product(REASONING_EFFORT_OPTIONS, [None, 4096], [None, 0.2])), +) +@patch(OPENROUTER_SEAM) +def test_run_forwards_every_generation_setting_combination( + mock_or, reasoning_effort, max_tokens, temperature +): + mock_or.return_value = _result("answer") + block = MistralVlmBlockV1(model_manager=MagicMock(), api_key="rf_key") + + block.run( + **_base_run_kwargs( + reasoning_effort=reasoning_effort, + max_tokens=max_tokens, + temperature=temperature, + api_key="sk-or-user-key", + privacy_level="allow", + max_concurrent_requests=3, + ) + ) + + kwargs = mock_or.call_args.kwargs + assert kwargs["reasoning"] == EXPECTED_REASONING_CONFIG[reasoning_effort] + assert kwargs["max_tokens"] == max_tokens + assert kwargs["temperature"] == temperature + assert kwargs["openrouter_api_key"] == "sk-or-user-key" + assert kwargs["privacy_level"] == "allow" + assert kwargs["max_concurrent_requests"] == 3 + + +@pytest.mark.parametrize( + "task_type,reasoning_effort", + list(itertools.product(SUPPORTED_TASK_TYPES_LIST, REASONING_EFFORT_OPTIONS)), +) +@patch(OPENROUTER_SEAM) +def test_run_keeps_message_layout_and_decoding_per_task( + mock_or, task_type, reasoning_effort +): + mock_or.return_value = _result( + { + "object-detection": DETECTION_OUTPUT, + "classification": CLASSIFICATION_OUTPUT, + "multi-label-classification": ( + '{"predicted_classes": [{"class": "cat", "confidence": 0.8}]}' + ), + }.get(task_type, "free text answer") + ) + block = MistralVlmBlockV1(model_manager=MagicMock(), api_key="rf_key") + + result = block.run( + **_base_run_kwargs( + task_type=task_type, + reasoning_effort=reasoning_effort, + **TASK_INPUTS[task_type], + ) + ) + + kwargs = mock_or.call_args.kwargs + assert kwargs["reasoning"] == EXPECTED_REASONING_CONFIG[reasoning_effort] + messages = kwargs["prompts"][0] + assert [message["role"] for message in messages] == ["user"] + assert [part["type"] for part in messages[0]["content"]] == ["image_url", "text"] + assert result[0]["error_status"] is False + if task_type in DECODING_TASKS: + assert result[0]["predictions"] is not None + else: + assert result[0]["predictions"] is None + + +@pytest.mark.parametrize("task_type", SUPPORTED_TASK_TYPES_LIST) +def test_manifest_validates_each_task_with_its_required_inputs(task_type): + manifest = _manifest(task_type=task_type, **TASK_INPUTS[task_type]) + + assert manifest.task_type == task_type + + +@pytest.mark.parametrize("reasoning_effort", UNEXPOSED_REASONING_LEVELS) +def test_run_rejects_reasoning_levels_mistral_does_not_expose(reasoning_effort): + block = MistralVlmBlockV1(model_manager=MagicMock(), api_key="rf_key") + + with pytest.raises(ValueError, match="supports reasoning_effort values"): + block.run(**_base_run_kwargs(reasoning_effort=reasoning_effort)) + + +@pytest.mark.parametrize("raw_output", [DETECTION_OUTPUT, DETECTION_OUTPUT_BBOX_ALIAS]) +@patch(OPENROUTER_SEAM) +def test_run_decodes_detections_on_the_0_999_grid(mock_or, raw_output): + mock_or.return_value = _result(raw_output) + block = MistralVlmBlockV1(model_manager=MagicMock(), api_key="rf_key") + + result = block.run( + **_base_run_kwargs(task_type="object-detection", classes=["cat", "dog"]) + ) + + predictions = result[0]["predictions"] + assert is_detection_prediction(predictions) + assert detection_boxes(predictions) == EXPECTED_XYXY + assert detection_class_names(predictions) == ["cat"] + assert detection_class_ids(predictions) == [0] + assert result[0]["error_status"] is False + assert result[0]["inference_id"] + assert result[0]["output"] == raw_output + assert result[0]["classes"] == ["cat", "dog"] + assert result[0]["input_tokens"] == 20 + assert result[0]["output_tokens"] == 8 + + +@patch(OPENROUTER_SEAM) +def test_run_keeps_unknown_label_with_negative_class_id(mock_or): + mock_or.return_value = _result( + '[{"box_2d": [100, 250, 999, 999], "label": "giraffe"}]' + ) + block = MistralVlmBlockV1(model_manager=MagicMock(), api_key="rf_key") + + result = block.run( + **_base_run_kwargs(task_type="object-detection", classes=["cat"]) + ) + + assert detection_class_names(result[0]["predictions"]) == ["giraffe"] + assert detection_class_ids(result[0]["predictions"]) == [-1] + assert result[0]["error_status"] is False + + +@patch(OPENROUTER_SEAM) +def test_run_decodes_classification(mock_or): + mock_or.return_value = _result(CLASSIFICATION_OUTPUT) + block = MistralVlmBlockV1(model_manager=MagicMock(), api_key="rf_key") + + result = block.run( + **_base_run_kwargs(task_type="classification", classes=["cat", "dog"]) + ) + + assert classification_top_class(result[0]["predictions"]) == "cat" + assert classification_top_confidence(result[0]["predictions"]) == pytest.approx(0.9) + assert result[0]["error_status"] is False + + +@patch(OPENROUTER_SEAM) +def test_run_leaves_predictions_none_for_non_decoding_task(mock_or): + mock_or.return_value = _result("a cat on a mat") + block = MistralVlmBlockV1(model_manager=MagicMock(), api_key="rf_key") + + result = block.run(**_base_run_kwargs(task_type="caption")) + + assert result[0]["predictions"] is None + assert result[0]["error_status"] is False + assert result[0]["output"] == "a cat on a mat" + + +@patch(OPENROUTER_SEAM) +def test_run_reports_error_status_for_undecodable_detection_output(mock_or): + mock_or.return_value = _result("I am afraid I cannot help with that.") + block = MistralVlmBlockV1(model_manager=MagicMock(), api_key="rf_key") + + result = block.run( + **_base_run_kwargs(task_type="object-detection", classes=["cat"]) + ) + + assert result[0]["predictions"] is None + assert result[0]["error_status"] is True diff --git a/workflows/tests/unit_tests/core_steps/models/foundation/test_openrouter_v3.py b/workflows/tests/unit_tests/core_steps/models/foundation/test_openrouter_v3.py index aceda8ae6a..8ec5da673a 100644 --- a/workflows/tests/unit_tests/core_steps/models/foundation/test_openrouter_v3.py +++ b/workflows/tests/unit_tests/core_steps/models/foundation/test_openrouter_v3.py @@ -226,6 +226,9 @@ def test_run_flags_error_status_on_garbage_detection_output() -> None: FORMAT_OUTPUTS = { "named_normalized": DETECTION_OUTPUT, "xyxy_0_1000": json.dumps([{"box_2d": [100, 250, 500, 750], "label": "cat"}]), + "xyxy_0_999": json.dumps( + [{"box_2d": [99.9, 249.75, 499.5, 749.25], "label": "cat"}] + ), "yxyx_0_1000": json.dumps([{"box_2d": [250, 100, 750, 500], "label": "cat"}]), "xyxy_absolute": json.dumps([{"box_2d": [80, 100, 400, 300], "label": "cat"}]), "xyxy_percent": json.dumps([{"box_2d": [10, 25, 50, 75], "label": "cat"}]), @@ -238,6 +241,7 @@ def test_run_flags_error_status_on_garbage_detection_output() -> None: "named_normalized": "All coordinates must be in range 0.0-1.0", "xyxy_0_1000": "[x_min, y_min, x_max, y_max]: the top-left and bottom-right " "corners as integers between 0 and 1000", + "xyxy_0_999": "corners as integers between 0 and 999", "yxyx_0_1000": "[y_min, x_min, y_max, x_max]", "xyxy_absolute": f"of the {IMAGE_WIDTH}x{IMAGE_HEIGHT} pixel image", "xyxy_percent": "as percentages of image width and height", diff --git a/workflows/uv.lock b/workflows/uv.lock index eafe57604a..f28481727b 100644 --- a/workflows/uv.lock +++ b/workflows/uv.lock @@ -4062,7 +4062,7 @@ wheels = [ [[package]] name = "roboflow-workflows" -version = "0.2.4" +version = "0.2.4.post1" source = { editable = "." } dependencies = [ { name = "anthropic" },