From 5148b13bf39bc12a12677f6b079fafd59c17fcad Mon Sep 17 00:00:00 2001 From: Yuqiang Wang <10815391+schchit@users.noreply.github.com> Date: Sat, 26 Sep 2026 21:40:21 -0500 Subject: [PATCH] fix: reject lossy JSON inputs before CLI transport; prepare 0.7.2 --- RELEASE-NOTES.md | 13 +++--- VERSION | 2 +- jep_cli/__init__.py | 4 +- jep_cli/client.py | 8 ++-- jep_cli/main.py | 19 ++++++--- jep_cli/strict_json.py | 78 +++++++++++++++++++++++++++++++++++ pyproject.toml | 4 +- tests/test_strict_json.py | 87 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 195 insertions(+), 20 deletions(-) create mode 100644 jep_cli/strict_json.py create mode 100644 tests/test_strict_json.py diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index db32642..59c6bfc 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,9 +1,8 @@ -# Release 0.7.1 +# Release 0.7.2 -Release the current JEP Core 0.7 CLI after repairing the acceptance regression gate. +- Reject duplicate members, invalid UTF-8/Unicode, non-finite numbers and unsupported precision-losing integers before parsing or forwarding event input. The same boundary applies to inline JSON, files and standard input, including explicit legacy commands. +- Preserve supported large JCS numbers and every signed member without normalization. Signature, profile and authorization checks remain with the API. +- Reject repeated extension arguments and malformed JSON-looking input instead of silently replacing a value or treating a damaged object as plain text. Plain-text creation arguments remain supported. +- Validate outgoing request values and successful JSON responses with the same boundary. No automatic legacy fallback is introduced. -- Default creation and verification use `/v0.7/events/*`; audience is optional unless a selected profile requires it. -- Explicit `--legacy` / `--legacy-format` options preserve the historical decoder contract. No fallback after current validation fails. -- Invalid and indeterminate verification return exit code 2; input or transport errors return 1. -- Large inline JSON is parsed before filesystem probing. -- Package, import, user-agent, tag, and built distribution versions are aligned and checked before publication. +Protocol remains Core 0.7 / wire 1. Published protocol drafts and historical signed artifacts are unchanged. diff --git a/VERSION b/VERSION index 39e898a..7486fdb 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.1 +0.7.2 diff --git a/jep_cli/__init__.py b/jep_cli/__init__.py index 87ee72b..0277160 100644 --- a/jep_cli/__init__.py +++ b/jep_cli/__init__.py @@ -1,3 +1,3 @@ -"""JEP CLI v0.6.""" +"""JEP Core 0.7 CLI.""" -__version__ = "0.7.1" +__version__ = "0.7.2" diff --git a/jep_cli/client.py b/jep_cli/client.py index c6be8b8..cb70c35 100644 --- a/jep_cli/client.py +++ b/jep_cli/client.py @@ -6,6 +6,8 @@ from typing import Any, Dict, Optional from urllib import request, error +from . import strict_json + class JEPCLIError(Exception): pass @@ -44,19 +46,19 @@ def _json(self, method: str, path: str, payload: Optional[Dict[str, Any]]) -> Di data = None headers = { "content-type": "application/json", - "user-agent": "JEP-CLI/0.7.1", + "user-agent": "JEP-CLI/0.7.2", } if self.api_key: headers["authorization"] = f"Bearer {self.api_key}" headers["x-api-key"] = self.api_key if payload is not None: - data = json.dumps(payload).encode("utf-8") + data = strict_json.dumps(payload).encode("utf-8") req = request.Request(self.base_url + path, data=data, headers=headers, method=method) try: with request.urlopen(req, timeout=self.timeout) as resp: body = resp.read().decode("utf-8") - return json.loads(body) if body else {} + return strict_json.loads(body) if body else {} except error.HTTPError as exc: body = exc.read().decode("utf-8") try: diff --git a/jep_cli/main.py b/jep_cli/main.py index 1ab7052..262de09 100644 --- a/jep_cli/main.py +++ b/jep_cli/main.py @@ -10,6 +10,7 @@ from typing import Any, Dict from .client import JEPClient, JEPCLIError +from . import strict_json VALID_VERBS = {"J", "D", "T", "V"} @@ -17,12 +18,16 @@ def parse_json_value(value: str) -> Any: if value == "-": - return json.load(sys.stdin) + # Read bytes when available: a text stream may replace malformed UTF-8. + stream = getattr(sys.stdin, "buffer", sys.stdin) + return strict_json.loads(stream.read()) try: - return json.loads(value) + return strict_json.loads(value) except json.JSONDecodeError: - pass + # JSON-looking input must not become a plain string after a parse error. + if value.lstrip().startswith(("{", "[", '"')): + raise # Inline event JSON may be longer than the filesystem filename limit. path = Path(value) try: @@ -30,7 +35,8 @@ def parse_json_value(value: str) -> Any: except OSError: is_file = False if is_file: - return json.loads(path.read_text(encoding="utf-8")) + return strict_json.loads(path.read_bytes()) + strict_json.validate(value) return value @@ -40,12 +46,15 @@ def parse_ext_items(items: list[str]) -> Dict[str, Any]: if "=" not in item: raise ValueError(f"extension must be key=json_or_string, got: {item}") key, raw = item.split("=", 1) + if key in ext: + raise ValueError(f"duplicate extension: {key!r}") + strict_json.validate(key) ext[key] = parse_json_value(raw) return ext def print_json(data: Any) -> None: - print(json.dumps(data, indent=2, ensure_ascii=False)) + print(strict_json.dumps(data)) def build_client(args: argparse.Namespace) -> JEPClient: diff --git a/jep_cli/strict_json.py b/jep_cli/strict_json.py new file mode 100644 index 0000000..529051c --- /dev/null +++ b/jep_cli/strict_json.py @@ -0,0 +1,78 @@ +"""Strict JSON input boundary; signature/profile validation stays with the API.""" +from __future__ import annotations + +import json +import math +from typing import Any + +import rfc8785 + + +def _pairs(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON member: {key!r}") + result[key] = value + return result + + +def _constant(token): + raise ValueError(f"non-JSON numeric constant: {token}") + + +def validate(value: Any) -> None: + """Reject lossy/invalid input without rewriting values or signed members. + + Large integer policy matches Core 0.7.4: exact binary64 integers and their + JCS shortest-decimal spellings are supported. This is not signature checking. + """ + if value is None or isinstance(value, bool): + return + if isinstance(value, str): + value.encode("utf-8", errors="strict") + elif isinstance(value, int): + if abs(value) > 2**53 - 1: + try: + number = float(value) + if not math.isfinite(number): + raise ValueError("integer exceeds binary64 range") + if int(number) != value and rfc8785.dumps(number) != str(value).encode("ascii"): + raise ValueError("integer is neither exact binary64 nor its JCS spelling") + except OverflowError as exc: + raise ValueError("integer exceeds binary64 range") from exc + elif isinstance(value, float): + if not math.isfinite(value): + raise ValueError("non-finite number") + elif isinstance(value, list): + for item in value: + validate(item) + elif isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise ValueError("JSON member names must be strings") + validate(key) + validate(item) + else: + raise ValueError("unsupported JSON value") + + +def loads(text: str | bytes) -> Any: + if isinstance(text, bytes): + text = text.decode("utf-8", errors="strict") + # Reject raw surrogate code points even before JSON parsing. + text.encode("utf-8", errors="strict") + try: + value = json.loads(text, object_pairs_hook=_pairs, parse_constant=_constant) + validate(value) + return value + except RecursionError as exc: + raise ValueError("JSON nesting exceeds parser capacity") from exc + + +def dumps(value: Any) -> str: + try: + validate(value) + return json.dumps(value, ensure_ascii=False, allow_nan=False) + except RecursionError as exc: + raise ValueError("JSON nesting exceeds parser capacity") from exc diff --git a/pyproject.toml b/pyproject.toml index 9a2a1f2..5b265d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "jep-cli" -version = "0.7.1" +version = "0.7.2" description = "Command-line client for JEP Core 0.7 with explicit legacy compatibility" readme = "README.md" license = { text = "MIT" } @@ -12,7 +12,7 @@ authors = [ { name = "HJS Spec", email = "signal@humanjudgment.org" } ] requires-python = ">=3.9" -dependencies = [] +dependencies = ["rfc8785==0.1.4"] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", diff --git a/tests/test_strict_json.py b/tests/test_strict_json.py new file mode 100644 index 0000000..0f26b7d --- /dev/null +++ b/tests/test_strict_json.py @@ -0,0 +1,87 @@ +import io +import json + +import pytest + +from jep_cli import main as cli +from jep_cli import strict_json + +BAD = [ + '{"x":1,"x":2}', + '{"x":{"a":1,"\\u0061":2}}', + '{"x":NaN}', '{"x":Infinity}', '{"x":-Infinity}', + '{"x":1e400}', '{"x":9007199254740993}', + '{"x":"\\ud800"}', '{"\\udfff":1}', + '{"x":1,}', '{"x":1} {}', +] + + +@pytest.mark.parametrize('raw', BAD) +@pytest.mark.parametrize('source', ['inline', 'file', 'stdin']) +def test_all_input_sources_reject(raw, source, tmp_path, monkeypatch): + argument = raw + if source == 'file': + path = tmp_path / 'event.json' + path.write_bytes(raw.encode('utf-8')) + argument = str(path) + elif source == 'stdin': + monkeypatch.setattr(cli.sys, 'stdin', io.StringIO(raw)) + argument = '-' + with pytest.raises(ValueError): + cli.parse_json_value(argument) + + +@pytest.mark.parametrize('source', ['file', 'stdin']) +def test_bad_utf8_rejected_before_text_replacement(source, tmp_path, monkeypatch): + raw = b'{"x":"\xff"}' + if source == 'file': + path = tmp_path / 'event.json' + path.write_bytes(raw) + argument = str(path) + else: + stream = io.TextIOWrapper(io.BytesIO(raw), encoding='utf-8', errors='replace') + monkeypatch.setattr(cli.sys, 'stdin', stream) + argument = '-' + with pytest.raises(ValueError): + cli.parse_json_value(argument) + + +@pytest.mark.parametrize('raw', [ + '{"x":1000000000000000100}', '{"x":1000000000000000128}', + '{"x":-1000000000000000100}', '{"x":100000000000000000000}', + '{"x":"\\ud83d\\ude00","ext":{},"ext_crit":[],"sig":"keep..me"}', + '{"x":"\\\\ud800","__proto__":{"ok":true}}', +]) +def test_supported_values_preserve_members(raw): + value = strict_json.loads(raw) + assert value == json.loads(raw) + assert strict_json.loads(strict_json.dumps(value)) == value + + +def test_duplicate_extension_is_not_silently_overwritten(): + with pytest.raises(ValueError): + cli.parse_ext_items(['urn:test=1', 'urn:test=2']) + + +@pytest.mark.parametrize('legacy', [False, True]) +def test_rejection_happens_before_network_and_never_falls_back(monkeypatch, legacy): + def no_client(*args): + pytest.fail('invalid input reached the client') + monkeypatch.setattr(cli, 'build_client', no_client) + args = ['verify', '{"who":"a","who":"b"}'] + if legacy: + args.append('--legacy') + assert cli.main(args) == 1 + + +def test_outgoing_values_are_checked(): + with pytest.raises(ValueError): + strict_json.dumps({'x': float('nan')}) + with pytest.raises(ValueError): + strict_json.dumps({'x': '\ud800'}) + + +def test_plain_text_and_long_inline_json_still_work(): + assert cli.parse_json_value('approve') == 'approve' + raw = json.dumps({'x': 'a' * 10000}) + assert cli.parse_json_value(raw) == json.loads(raw)