From 29c8732707a64aac788d1bac59e4cab691a49dd7 Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Mon, 31 Aug 2026 14:39:37 +0100 Subject: [PATCH] Refuse a date past the year 9999 before the arithmetic Versions 2 and 3 carry the date as an unsigned 32 bit count of minutes since 2020-01-01, which runs to 4,294,967,295 and lands on 15 February 10186. datetime stops at the end of 9999, so the addition raised OverflowError for any count from 4,197,074,400 upwards, and a read that promises never to raise raised on caller data. The same bytes read fine in Java, PHP and JavaScript, so this is the runtime's limit rather than a fault in the data. The reader now compares the count with the largest one datetime can hold, derived from datetime.max, and reports IMPLEMENTATION_CAPACITY_EXCEEDED before the arithmetic on both reading contracts. Two bytes of hours in version 1 reach June 2027, so that field needs no guard, and the reader and tests say so. test_date_range pins the boundary on both contracts, covering the maximum count, the first count past the runtime, the last count inside it, and the version 1 maximum. The coverage test that named IMPLEMENTATION_CAPACITY_EXCEEDED unreachable now produces it, and MALFORMED_ENVELOPE keeps its own test as the one still unreachable. With the guard removed, five tests error with the OverflowError the guard prevents. --- README.md | 2 +- owid/io.py | 10 +++ owid/parse.py | 18 ++++- owid/status.py | 6 ++ tests/test_date_range.py | 129 +++++++++++++++++++++++++++++++++++ tests/test_parse_contract.py | 35 +++++++--- 6 files changed, 188 insertions(+), 12 deletions(-) create mode 100644 tests/test_date_range.py diff --git a/README.md b/README.md index fe5a0bc..772c0b9 100644 --- a/README.md +++ b/README.md @@ -206,7 +206,7 @@ The reasons a read can give are named by `ParseStatus`. | `UNEXPECTED_END` | The data stopped in the middle of a field. | | `INVALID_DOMAIN_ENCODING` | The creator domain is not terminated, or is longer than the published maximum. | | `BYTE_COUNT_MISMATCH` | The declared payload byte count disagrees with the bytes actually present. | -| `IMPLEMENTATION_CAPACITY_EXCEEDED` | The envelope is consistent but larger than this runtime can hold, so the same bytes may be readable elsewhere. | +| `IMPLEMENTATION_CAPACITY_EXCEEDED` | The envelope is consistent but larger than this runtime can hold, or dated past the end of 9999 where `datetime` stops, so the same bytes may be readable elsewhere. | | `ABSENT_NODE` | The one byte marker standing for a node that is not there. | | `MALFORMED_ENVELOPE` | Malformed in a way none of the above describes. | diff --git a/owid/io.py b/owid/io.py index 0e152cb..7916653 100644 --- a/owid/io.py +++ b/owid/io.py @@ -40,6 +40,16 @@ #: number of hours or minutes after this instant. BASE_DATE = datetime(2020, 1, 1, tzinfo=timezone.utc) +#: The largest count of minutes after BASE_DATE that a datetime can hold, +#: which is 4,197,074,399 and lands on 9999-12-31 23:59. The four byte count +#: in versions 2 and 3 runs to 4,294,967,295, which is 15 February 10186, so +#: a count above this is one the wire format allows and this runtime cannot +#: represent. Derived from datetime.max rather than written as a number so it +#: cannot drift from the runtime, and worked out on naive values because +#: datetime.max cannot take part in time zone arithmetic without overflowing. +_SPAN_TO_MAX = datetime.max - BASE_DATE.replace(tzinfo=None) +MAXIMUM_MINUTES = _SPAN_TO_MAX.days * 1440 + _SPAN_TO_MAX.seconds // 60 + #: The longest creator domain the reader will accept, in characters. RFC 1035 #: section 2.3.4, "Size limits", restricts the total length of a domain name, #: being the label octets and the label length octets, to 255 octets or less. diff --git a/owid/parse.py b/owid/parse.py index 6a61963..39284c3 100644 --- a/owid/parse.py +++ b/owid/parse.py @@ -21,7 +21,12 @@ from datetime import timedelta from typing import TYPE_CHECKING, NamedTuple, Optional -from .io import BASE_DATE, MAXIMUM_DOMAIN_LENGTH, SIGNATURE_LENGTH +from .io import ( + BASE_DATE, + MAXIMUM_DOMAIN_LENGTH, + MAXIMUM_MINUTES, + SIGNATURE_LENGTH, +) from .status import ParseStatus from .version import Version @@ -185,6 +190,9 @@ def _parse(buffer, exact: bool) -> ParseResult: if version == Version.VERSION1: if total - at < 2: return _failed(ParseStatus.UNEXPECTED_END) + # Two bytes of hours reach 65,535 hours, which is June 2027, so this + # arithmetic cannot leave the runtime's range and there is nothing to + # guard. hours = (data[at] << 8) | data[at + 1] at += 2 date = BASE_DATE + timedelta(hours=hours) @@ -193,6 +201,14 @@ def _parse(buffer, exact: bool) -> ParseResult: return _failed(ParseStatus.UNEXPECTED_END) minutes = int.from_bytes(data[at:at + 4], "little") at += 4 + # The wire allows 4,294,967,295 minutes, which is the year 10186, and + # datetime stops at the end of 9999. A count past that is judged + # before the arithmetic, because the addition would raise + # OverflowError on it and this read promises not to raise. The same + # bytes read fine where the date type is wider, so this is the + # runtime's limit rather than a fault in the data. + if minutes > MAXIMUM_MINUTES: + return _failed(ParseStatus.IMPLEMENTATION_CAPACITY_EXCEEDED) date = BASE_DATE + timedelta(minutes=minutes) if total - at < 4: diff --git a/owid/status.py b/owid/status.py index df18916..860dd2d 100644 --- a/owid/status.py +++ b/owid/status.py @@ -69,6 +69,12 @@ class ParseStatus(Enum): #: The envelope is structurally consistent but larger than this runtime #: can hold. Not a fault in the data, and deliberately distinct from the #: data being wrong, because the same bytes may be readable elsewhere. + #: Produced by a date past the end of the year 9999, where datetime stops + #: while the four byte minute count of versions 2 and 3 runs to 15 + #: February 10186. The date is judged before the arithmetic, so a read + #: never raises on it. A payload count cannot produce it here, because + #: Python integers are unbounded and the count check refuses a + #: declaration that disagrees with the bytes present first. IMPLEMENTATION_CAPACITY_EXCEEDED = "ImplementationCapacityExceeded" #: The version 0 marker, which stands for an absent node inside a stream. diff --git a/tests/test_date_range.py b/tests/test_date_range.py new file mode 100644 index 0000000..c663c3d --- /dev/null +++ b/tests/test_date_range.py @@ -0,0 +1,129 @@ +# **************************************************************************** +# Copyright 2026 51 Degrees Mobile Experts Limited (51degrees.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# **************************************************************************** +"""Dates the wire format can carry but this runtime cannot. + +Versions 2 and 3 carry the date as an unsigned 32 bit count of minutes since +2020-01-01, which runs to 4,294,967,295 and lands on 15 February 10186. +datetime.max is the last moment of the year 9999, so the last count the +runtime can hold is 4,197,074,399 and the next cannot be represented. Before +the guard the addition raised OverflowError on that count, so a read that +promises never to raise raised on caller data. The same bytes read fine in +Java, PHP and JavaScript, so the finding is the runtime's limit and not a +fault in the data, which is what IMPLEMENTATION_CAPACITY_EXCEEDED means. Both +reading contracts are checked. +""" + +from __future__ import annotations + +import unittest +from datetime import datetime, timedelta, timezone + +from owid import Creator, Crypto, Owid +from owid.io import BASE_DATE, MAXIMUM_MINUTES, SIGNATURE_LENGTH +from owid.status import ParseStatus + +DOMAIN = "example.com" + +#: The last count a datetime can hold, which is 9999-12-31 23:59. Written as +#: a number here, and derived from datetime.max in the library, so the two +#: are checked against each other below. +LAST_INSIDE = 4_197_074_399 + +#: The first count the runtime cannot hold, one minute into the year 10000. +FIRST_BEYOND = LAST_INSIDE + 1 + + +def _with_minutes(minutes: int) -> bytes: + """A signed version 3 envelope with its date bytes replaced. The date + follows the version byte and the terminated domain, and is four little + endian bytes. The signature no longer matches, which does not matter, + because parsing and verifying are separate questions.""" + raw = bytearray( + Creator(DOMAIN, Crypto.new()).create(b"\x01\x02\x03").as_byte_array()) + at = 1 + len(DOMAIN) + 1 + raw[at:at + 4] = minutes.to_bytes(4, "little") + return bytes(raw) + + +def _version1_with_hours(hours: int) -> bytes: + """A version 1 envelope built by hand, because no creator writes that + version any more. Two big endian bytes of hours, then the payload count, + payload and a signature of the right length.""" + return ( + bytes([1]) + + DOMAIN.encode("ascii") + b"\x00" + + hours.to_bytes(2, "big") + + (1).to_bytes(4, "little") + b"\x07" + + bytes(SIGNATURE_LENGTH) + ) + + +class DateRangeTests(unittest.TestCase): + + def _refused_on_both_contracts(self, raw: bytes) -> None: + for read in (Owid.parse_bytes, Owid.parse_prefix): + with self.subTest(read=read.__name__): + result = read(raw) + self.assertFalse(result.ok) + self.assertIsNone(result.owid, "no value on failure") + self.assertEqual( + ParseStatus.IMPLEMENTATION_CAPACITY_EXCEEDED, + result.status) + self.assertEqual(0, result.consumed, "nothing is consumed") + + def _parsed_on_both_contracts(self, raw: bytes) -> datetime: + whole = Owid.parse_bytes(raw) + self.assertTrue(whole.ok, whole.status) + framed = Owid.parse_prefix(raw) + self.assertTrue(framed.ok, framed.status) + self.assertEqual(whole.owid.date, framed.owid.date) + self.assertEqual(len(raw), framed.consumed) + return whole.owid.date + + def test_maximum_count_is_capacity_exceeded(self) -> None: + """The largest count the wire can carry.""" + self._refused_on_both_contracts(_with_minutes(0xFFFFFFFF)) + + def test_first_count_beyond_the_runtime_is_capacity_exceeded(self) -> None: + """One minute into the year 10000.""" + self._refused_on_both_contracts(_with_minutes(FIRST_BEYOND)) + + def test_last_count_inside_the_runtime_parses(self) -> None: + date = self._parsed_on_both_contracts(_with_minutes(LAST_INSIDE)) + self.assertEqual( + datetime(9999, 12, 31, 23, 59, tzinfo=timezone.utc), date) + + def test_the_boundary_is_the_last_whole_minute_before_max(self) -> None: + """Pins the boundary to the runtime rather than to a number someone + worked out once. The library derives it and this test states it, and + the arithmetic the guard prevents is shown to raise.""" + self.assertEqual(LAST_INSIDE, MAXIMUM_MINUTES) + BASE_DATE + timedelta(minutes=LAST_INSIDE) + with self.assertRaises(OverflowError): + BASE_DATE + timedelta(minutes=FIRST_BEYOND) + + def test_version1_maximum_hours_parses(self) -> None: + """Version 1 counts hours in two bytes, so its largest count is 65,535 + hours, which is 23 June 2027 and under eight years from the base + date. The arithmetic cannot leave the runtime's range, so the reader + has no guard for it and this shows none is needed.""" + date = self._parsed_on_both_contracts(_version1_with_hours(0xFFFF)) + self.assertEqual( + datetime(2027, 6, 23, 15, 0, tzinfo=timezone.utc), date) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parse_contract.py b/tests/test_parse_contract.py index fb645a4..76753cc 100644 --- a/tests/test_parse_contract.py +++ b/tests/test_parse_contract.py @@ -244,17 +244,32 @@ def test_byte_count_mismatch_when_signature_is_short(self) -> None: self.assertEqual( ParseStatus.BYTE_COUNT_MISMATCH, Owid.parse_bytes(raw).status) - def test_implementation_capacity_and_malformed_are_unreachable(self) -> None: - """IMPLEMENTATION_CAPACITY_EXCEEDED cannot be reached in Python: + def test_implementation_capacity_is_a_date_the_runtime_cannot_hold( + self) -> None: + """IMPLEMENTATION_CAPACITY_EXCEEDED is produced by a date past the end + of 9999, which datetime cannot hold while the wire's four byte minute + count runs to the year 10186. The boundary is pinned in + test_date_range. A payload count cannot produce it here, because integers are unbounded and a declaration large enough to matter can - never agree with the bytes actually present, so the count check refuses - it first. MALFORMED_ENVELOPE is likewise unreachable while the count - check holds, and both are kept as backstops so that a future change to - that arithmetic cannot pass silently. - - This test exists to record that, so the gap is a stated decision rather - than something nobody noticed.""" - self.assertIn(ParseStatus.IMPLEMENTATION_CAPACITY_EXCEEDED, ParseStatus) + never agree with the bytes actually present, so the count check + refuses it first. The date bytes follow the version byte and the + terminated domain.""" + raw = bytearray(_creator().create(b"x").as_byte_array()) + at = 1 + len("example.com") + 1 + raw[at:at + 4] = b"\xff\xff\xff\xff" + + result = Owid.parse_bytes(bytes(raw)) + + self.assertFalse(result.ok) + self.assertIsNone(result.owid) + self.assertEqual( + ParseStatus.IMPLEMENTATION_CAPACITY_EXCEEDED, result.status) + + def test_malformed_envelope_is_unreachable(self) -> None: + """MALFORMED_ENVELOPE is unreachable while the count check holds, and + is kept as a backstop so that a future change to that arithmetic + cannot pass silently. This test exists to record that, so the gap is + a stated decision rather than something nobody noticed.""" self.assertIn(ParseStatus.MALFORMED_ENVELOPE, ParseStatus)