diff --git a/decision_protocol.py b/decision_protocol.py index ed198e8..7d21b9f 100644 --- a/decision_protocol.py +++ b/decision_protocol.py @@ -3,7 +3,8 @@ import math import re -from decision_providers import DecisionError, probability, validate_answer +from decision_providers import (DecisionError, probability, validate_answer, + probability_decimals_for_model, probability_sum_valid) MAX_REQUEST_BYTES = 32000 MAX_RESPONSE_BYTES = 131072 @@ -63,6 +64,7 @@ def validate_response(data, payload): if not isinstance(data, dict) or not isinstance(data.get('model'), str) or not data['model']: raise ValueError('Missing decision model.') answers = data.get('answers') + decimals = probability_decimals_for_model(data['model']) if not isinstance(answers, dict) or set(answers) != set(payload['questions']): raise ValueError('Missing or unexpected answers.') for key, question in payload['questions'].items(): @@ -74,7 +76,7 @@ def validate_response(data, payload): raise ValueError('Invalid confidence.') if kind == 'choice': try: - validate_answer(answer, list(question['criteria'])) + validate_answer(answer, list(question['criteria']), probability_decimals=decimals) except DecisionError as exc: raise ValueError('Invalid choice answer.') from exc elif kind == 'noul': @@ -87,7 +89,8 @@ def validate_response(data, payload): probs = answer.get('probabilities') if probs is not None: if (not isinstance(probs, dict) or set(probs) != {str(i) for i in range(len(question['criteria']))} - or any(not probability(p) for p in probs.values()) or abs(sum(probs.values()) - 1) > 1e-6): + or any(not probability(p) for p in probs.values()) + or not probability_sum_valid(probs.values(), decimal_places=decimals)): raise ValueError('Invalid score distribution.') usage = data.get('usage') or {} if not isinstance(usage, dict): diff --git a/decision_providers/__init__.py b/decision_providers/__init__.py index ab25311..0030558 100644 --- a/decision_providers/__init__.py +++ b/decision_providers/__init__.py @@ -59,7 +59,30 @@ def probability(value): return type(value) in (int, float) and math.isfinite(value) and 0 <= value <= 1 -def validate_answer(answer, choices): +def probability_decimals_for_model(model): + """TypeSafe displays probabilities to two decimals; other models stay strict.""" + if isinstance(model, str) and re.fullmatch(r"jev-(?:latest|\d[\w.-]*)", model.rsplit('/', 1)[-1], re.I): + return 2 + return None + + +def probability_sum_valid(values, *, decimal_places=None): + values = list(values) + if not values or any(not probability(p) for p in values): + return False + if abs(math.fsum(values) - 1) <= 1e-6: + return True + # Permit only the declared display precision. Do not apply a blanket loose + # tolerance to arbitrary high-precision or malformed distributions. + if decimal_places != 2 or any(abs(p * 100 - round(p * 100)) > 1e-8 for p in values): + return False + half_step = .005 + lower = math.fsum(max(0., p - half_step) for p in values) + upper = math.fsum(min(1., p + half_step) for p in values) + return lower <= 1 + 1e-12 and upper >= 1 - 1e-12 + + +def validate_answer(answer, choices, *, probability_decimals=None): if not isinstance(answer, dict) or answer.get("type") != "choice": raise DecisionError("invalid_response") selected = answer.get("choice") @@ -73,10 +96,12 @@ def validate_answer(answer, choices): if distribution is not None: if (not isinstance(distribution, dict) or set(distribution) != set(choices) or any(not probability(p) for p in distribution.values()) - or abs(sum(distribution.values()) - 1) > 1e-6 + or not probability_sum_valid(distribution.values(), decimal_places=probability_decimals) or distribution[selected] + 1e-9 < max(distribution.values())): raise DecisionError("invalid_probabilities") - entropy = (-sum(p * math.log(p) for p in distribution.values() if p > 0) + # Preserve the reported probabilities; normalize only the derived entropy. + total = math.fsum(distribution.values()) + entropy = (-sum((p / total) * math.log(p / total) for p in distribution.values() if p > 0) / math.log(len(choices))) if len(choices) > 1 else 0.0 entropy = max(0.0, min(1.0, entropy)) return selected, distribution, confidence, entropy diff --git a/decision_providers/jev.py b/decision_providers/jev.py index f7757f0..76814ec 100644 --- a/decision_providers/jev.py +++ b/decision_providers/jev.py @@ -12,7 +12,7 @@ import httpx -from . import DecisionError, DecisionResult, validate_answer +from . import DecisionError, DecisionResult, validate_answer, probability_decimals_for_model ENDPOINTS = {"openrouter": "https://openrouter.ai/api/alpha/decisions", "typesafe": "https://api.typesafe.ai/v1/systemone"} @@ -108,7 +108,8 @@ def _parse(self, raw, request, started, attempts, had_unmetered_attempt): if not isinstance(model, str) or not model or len(model) > 160: raise ValueError("Missing model") selected, probs, confidence, entropy = validate_answer( - data.get("answers", {}).get("selection"), [c.id for c in request.choices]) + data.get("answers", {}).get("selection"), [c.id for c in request.choices], + probability_decimals=probability_decimals_for_model(model)) return DecisionResult(selected, probs, confidence, entropy, self.transport, model, (time.monotonic() - started) * 1000, None if had_unmetered_attempt else cost, tokens, attempts) except (ValueError, AttributeError, TypeError, DecisionError) as exc: diff --git a/docs/decision-rounding.md b/docs/decision-rounding.md new file mode 100644 index 0000000..19cfee3 --- /dev/null +++ b/docs/decision-rounding.md @@ -0,0 +1,24 @@ +# Jev probability display precision + +TypeSafe displays probabilities and scores to two decimal places. The maintained +[AI SDK TypeSafe adapter](https://github.com/vercel/ai/blob/main/packages/typesafe-ai/src/typesafe-ai-evaluation-model.ts) +declares `probabilityDecimals: 2` and preserves the native numbers. Consequently, +a valid distribution can arrive as `0.33, 0.33, 0.33`, summing to `0.99`. + +The router previously required a sum within `1e-6` of one. That can reject a valid +HTTP200 decision response and expose a public502. Live architecture tests found +`bad_response` failures; their discarded response bodies do not establish that +every observed failure had this cause. This patch fixes the independently +reproducible compatibility defect, not generative-provider timeouts. + +For versioned Jev model identifiers and the mutable `jev-latest` alias +(including vendor-prefixed forms of both), distributions whose values are displayed at two decimal places are accepted when their clipped +rounding intervals jointly contain a unit mass. Each value still must be finite +and within0..1; keys must match all requested choices; the selected choice must +have maximal displayed probability. Unexplained mass errors and distributions +with other precision retain strict validation. Other model families remain strict. + +Raw probabilities, scores, confidence and usage are unchanged. Only the derived +normalized entropy uses the normalized displayed weights. The rule is shared by +public Choice/Score responses, direct Jev selection and automatic-routing +revalidation. Requests, deadlines, model policies and retry behavior do not change. diff --git a/policy_selection.py b/policy_selection.py index 39666af..fa0feac 100644 --- a/policy_selection.py +++ b/policy_selection.py @@ -11,7 +11,8 @@ import uuid from auto_policies import CATALOG_VERSION, build_bundle, catalog, validate_constraints -from decision_providers import DecisionChoice, DecisionError, DecisionRequest, validate_answer +from decision_providers import (DecisionChoice, DecisionError, DecisionRequest, validate_answer, + probability_decimals_for_model) from decision_providers.jev import JevDecisionProvider log = logging.getLogger(__name__) @@ -188,7 +189,8 @@ def legal_policies(): result = await provider.decide(request, deadline=started + config["decision_timeout_ms"] / 1000) trace.update(cost_usd=result.cost_usd, attempts=result.attempts) selected_id, _, confidence, entropy = validate_answer({"type": "choice", "choice": result.selected_id, - "probabilities": result.probabilities, "confidence": result.confidence}, legal) + "probabilities": result.probabilities, "confidence": result.confidence}, legal, + probability_decimals=probability_decimals_for_model(result.model)) trace.update(provider=result.provider, model=result.model, cost_usd=result.cost_usd, confidence=confidence, normalized_entropy=entropy, attempts=result.attempts, proposed_id=selected_id, disagrees_with_default=selected_id != "general") diff --git a/tests/test_automatic_routing.py b/tests/test_automatic_routing.py index 1576eb6..889fdf0 100644 --- a/tests/test_automatic_routing.py +++ b/tests/test_automatic_routing.py @@ -84,6 +84,21 @@ async def test_uncertainty_and_unknown_selection_fall_back(automatic_host): assert trace["selected_id"] == "general" and trace["fallback_reason"] == reason +@pytest.mark.asyncio +@pytest.mark.parametrize('model,selected', [('typesafe/jev-1.13-20260917', 'coding-agent'), + ('another-decision-model', 'general')]) +async def test_automatic_revalidation_respects_only_jev_display_precision(automatic_host, model, selected): + class Rounded: + async def decide(self, req, *, deadline): + probs = {c.id: .95 if c.id == 'coding-agent' else .02 for c in req.choices} + assert len(probs) == 3 + return DecisionResult('coding-agent', probs, .95, None, 'test', model, 1, .001, 10, 1) + execution = {**compile_automatic(automatic_host, intent())["execution"], "automatic_mode": "active"} + _, trace = await select_policy(automatic_host, {}, execution, provider=Rounded()) + assert trace['selected_id'] == selected + if selected == 'general':assert trace['fallback_reason'] == 'invalid_probabilities' + + @pytest.mark.asyncio async def test_mutated_policy_or_description_is_rejected_before_network(automatic_host): execution = compile_automatic(automatic_host, intent())["execution"] diff --git a/tests/test_decision_rounding.py b/tests/test_decision_rounding.py new file mode 100644 index 0000000..409b05c --- /dev/null +++ b/tests/test_decision_rounding.py @@ -0,0 +1,90 @@ +import json +import random +import time + +import httpx +import pytest + +from decision_protocol import validate_response +from decision_providers import (DecisionChoice, DecisionError, DecisionRequest, + probability_sum_valid, validate_answer) +from decision_providers.jev import JevDecisionProvider + + +def response(probabilities, model='typesafe/jev-1.13-20260917', kind='choice'): + choice = max(probabilities, key=probabilities.get) + answer = {'type': kind, 'probabilities': probabilities, 'confidence': .2} + answer.update({'choice': choice} if kind == 'choice' else {'score': 1.0}) + return {'model': model, 'answers': {'q': answer}, 'usage': {'input_tokens': 10, 'cost': .000001}} + + +def payload(keys, kind='choice'): + return {'state': 'Select an option', 'questions': {'q': {'type': kind, 'instructions': 'Select', + 'criteria': {k: k for k in keys} if kind == 'choice' else list(keys)}}} + + +@pytest.mark.parametrize('probs', [dict(a=.33, b=.33, c=.33), dict(a=.34, b=.34, c=.33), + {str(i): .03 for i in range(32)}]) +def test_rounded_jev_distribution_survives_without_modifying_values(probs): + data = response(probs) + accepted = validate_response(data, payload(probs)) + assert accepted['answers']['q']['probabilities'] == probs + selected, raw, confidence, entropy = validate_answer(data['answers']['q'], list(probs), probability_decimals=2) + assert raw == probs and confidence == .2 and selected in probs + assert 0 <= entropy <= 1 + + +def test_rounding_does_not_relax_other_models_or_recorded_choices(): + probs = dict(a=.33, b=.33, c=.33) + with pytest.raises(ValueError): + validate_response(response(probs, model='other-decision-model'), payload(probs)) + with pytest.raises(DecisionError): + validate_answer(response(probs)['answers']['q'], list(probs)) + + +@pytest.mark.parametrize('probs', [dict(a=.3, b=.3, c=.3), dict(a=.4, b=.4, c=.4), + dict(a=.331, b=.331, c=.331), dict(a=0, b=0), dict(a=.99), + dict(a=-.01, b=1.01), dict(a=True, b=0), dict(a=float('nan'), b=1)]) +def test_unexplained_mass_missing_precision_and_invalid_values_still_rejected(probs): + with pytest.raises(ValueError): + validate_response(response(probs), payload(probs)) + + +def test_unknown_choice_missing_option_and_non_argmax_still_rejected(): + probs = dict(a=.5, b=.25, c=.25) + for choice in ['missing', 'b']: + data = response(probs);data['answers']['q']['choice'] = choice + with pytest.raises(ValueError):validate_response(data, payload(probs)) + with pytest.raises(ValueError):validate_response(response(dict(a=.5, b=.5)), payload(['a', 'b', 'c'])) + + +def test_rounded_score_distribution_has_the_same_mass_rule(): + probs = {'0': .33, '1': .33, '2': .33} + result = validate_response(response(probs, kind='score'), payload(probs, kind='score')) + assert result['answers']['q']['probabilities'] == probs + with pytest.raises(ValueError): + validate_response(response({'0': .3, '1': .3, '2': .3}, kind='score'), payload(probs, kind='score')) + + +def test_real_normalized_distributions_remain_valid_after_display_rounding(): + rng = random.Random(921) + for count in range(1, 33): + for _ in range(40): + weights = [rng.random() for _ in range(count)] + total = sum(weights) + displayed = [round(value / total, 2) for value in weights] + assert probability_sum_valid(displayed, decimal_places=2) + assert not probability_sum_valid([0.] * 32, decimal_places=2) + assert not probability_sum_valid([.7] + [0.] * 31, decimal_places=2) + + +@pytest.mark.asyncio +async def test_direct_jev_provider_preserves_rounded_distribution_and_normalizes_only_entropy(): + req = DecisionRequest('q', 'Choose', {}, tuple(DecisionChoice(k, k) for k in ['a', 'b', 'c'])) + data = response(dict(a=.33, b=.33, c=.33)) + data['answers']['selection'] = data['answers'].pop('q') + async with httpx.AsyncClient(transport=httpx.MockTransport(lambda request: httpx.Response(200, json=data))) as client: + result = await JevDecisionProvider('test-only', client=client).decide(req, deadline=time.monotonic() + 1) + assert result.probabilities == dict(a=.33, b=.33, c=.33) + assert result.normalized_entropy == pytest.approx(1.) + assert result.confidence == .2 and result.cost_usd == .000001 diff --git a/tests/test_decision_routing.py b/tests/test_decision_routing.py index 834113e..88824cd 100644 --- a/tests/test_decision_routing.py +++ b/tests/test_decision_routing.py @@ -98,6 +98,23 @@ def test_primary_antseed_success_does_not_call_openrouter(routed): assert len(calls) == 1 +@pytest.mark.parametrize('antseed_status', [200, 503]) +def test_public_decisions_accept_jev_display_rounding(routed, monkeypatch, antseed_status): + import sys + payload = copy.deepcopy(PAYLOAD) + payload['questions']['team']['criteria']['operations'] = 'Operations' + answer = copy.deepcopy(ANSWER) + answer['answers']['team']['probabilities'] = {'billing': .33, 'technical': .33, 'operations': .33} + monkeypatch.setattr(sys.modules[__name__], 'PAYLOAD', payload) + monkeypatch.setattr(sys.modules[__name__], 'ANSWER', answer) + host, _, calls, behavior = routed + behavior['antseed_status'] = antseed_status + response = TestClient(create_app(host)).post('/v1/decisions', json={**payload, 'policy_ir': POLICY}) + assert response.status_code == 200, response.text + assert response.json()['answers'] == answer['answers'] + assert len(calls) == (1 if antseed_status == 200 else 2) + + def test_decision_state_and_null_criteria_survive_lua_and_fallback(routed, monkeypatch): import sys payload = copy.deepcopy(PAYLOAD)