From d35f56f3b2ff2051ab0ab87b198d0b3ac217a3f4 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 25 Sep 2026 11:22:09 +0100 Subject: [PATCH 1/5] fix: connect the websocket on the port the client options name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The realtime transport built its URL from the scheme and the host alone, so `port` and `tlsPort` (TO3k4, TO3k5) reached the REST layer, which interpolates `Defaults.get_port(options)` into every base URL, and were dropped on the way to the websocket. Against Ably that goes unnoticed, since the defaults 80 and 443 are the ports the scheme implies. Against anything else — a proxy, a local server — the connection went to the wrong port and never opened, which is what `uts/realtime/integration/proxy/*` needs and cannot otherwise get. Co-Authored-By: Claude Opus 5 (1M context) --- ably/transport/websockettransport.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ably/transport/websockettransport.py b/ably/transport/websockettransport.py index 3381c974..cc510a4f 100644 --- a/ably/transport/websockettransport.py +++ b/ably/transport/websockettransport.py @@ -11,6 +11,7 @@ import msgpack from ably.http.httputils import HttpUtils +from ably.transport.defaults import Defaults from ably.types.connectiondetails import ConnectionDetails from ably.types.operations import PublishResult from ably.util.eventemitter import EventEmitter @@ -90,7 +91,8 @@ def connect(self): headers = HttpUtils.default_headers() query_params = urllib.parse.urlencode(self.params) scheme = 'wss' if self.options.tls else 'ws' - ws_url = f'{scheme}://{self.host}?{query_params}' + port = Defaults.get_port(self.options) + ws_url = f'{scheme}://{self.host}:{port}?{query_params}' log.info(f'connect(): attempting to connect to {ws_url}') self.ws_connect_task = asyncio.create_task(self.ws_connect(ws_url, headers)) self.ws_connect_task.add_done_callback(self.on_ws_connect_done) From 389e0426403cfd961b12ec45555c0d378aff75a2 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 25 Sep 2026 11:23:24 +0100 Subject: [PATCH 2/5] test: derive the realtime integration specifications Thirteen specifications under `uts/realtime/integration`, 43 Test IDs, run against the sandbox with no mock in front of anything. The package provisions its own app under `realtime_sandbox`, separate from the REST tier's, so a realtime test entering presence cannot be seen by a REST test reading the same channel name. Five of the twenty specifications in the tier carry a `## Protocol Variants` section and run every test twice; three of them are here. Co-Authored-By: Claude Opus 5 (1M context) --- test/uts/realtime/integration/__init__.py | 0 .../uts/realtime/integration/auth/__init__.py | 0 .../integration/auth/token_renewal_test.py | 58 +++ .../integration/auth/token_request_test.py | 64 +++ test/uts/realtime/integration/auth_test.py | 138 ++++++ .../integration/channel_history_test.py | 69 +++ .../realtime/integration/channels/__init__.py | 0 .../channels/channel_attach_test.py | 90 ++++ .../channels/channel_publish_test.py | 187 ++++++++ .../channels/channel_subscribe_test.py | 157 +++++++ test/uts/realtime/integration/conftest.py | 66 +++ .../integration/connection/__init__.py | 0 .../connection/connection_failures_test.py | 51 ++ .../integration/connection_lifecycle_test.py | 109 +++++ .../integration/delta_decoding_test.py | 314 +++++++++++++ .../integration/mutable_messages_test.py | 439 ++++++++++++++++++ .../realtime/integration/presence/__init__.py | 0 .../presence/presence_sync_test.py | 90 ++++ .../integration/presence_lifecycle_test.py | 167 +++++++ 19 files changed, 1999 insertions(+) create mode 100644 test/uts/realtime/integration/__init__.py create mode 100644 test/uts/realtime/integration/auth/__init__.py create mode 100644 test/uts/realtime/integration/auth/token_renewal_test.py create mode 100644 test/uts/realtime/integration/auth/token_request_test.py create mode 100644 test/uts/realtime/integration/auth_test.py create mode 100644 test/uts/realtime/integration/channel_history_test.py create mode 100644 test/uts/realtime/integration/channels/__init__.py create mode 100644 test/uts/realtime/integration/channels/channel_attach_test.py create mode 100644 test/uts/realtime/integration/channels/channel_publish_test.py create mode 100644 test/uts/realtime/integration/channels/channel_subscribe_test.py create mode 100644 test/uts/realtime/integration/conftest.py create mode 100644 test/uts/realtime/integration/connection/__init__.py create mode 100644 test/uts/realtime/integration/connection/connection_failures_test.py create mode 100644 test/uts/realtime/integration/connection_lifecycle_test.py create mode 100644 test/uts/realtime/integration/delta_decoding_test.py create mode 100644 test/uts/realtime/integration/mutable_messages_test.py create mode 100644 test/uts/realtime/integration/presence/__init__.py create mode 100644 test/uts/realtime/integration/presence/presence_sync_test.py create mode 100644 test/uts/realtime/integration/presence_lifecycle_test.py diff --git a/test/uts/realtime/integration/__init__.py b/test/uts/realtime/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/realtime/integration/auth/__init__.py b/test/uts/realtime/integration/auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/realtime/integration/auth/token_renewal_test.py b/test/uts/realtime/integration/auth/token_renewal_test.py new file mode 100644 index 00000000..4274f052 --- /dev/null +++ b/test/uts/realtime/integration/auth/token_renewal_test.py @@ -0,0 +1,58 @@ +"""Derived from uts/realtime/integration/auth/token_renewal_test.md in ably/specification. + +Spec points: RSA4b, RTN14b + +The client is given a JWT that lives five seconds and then long-lived ones. Measured against +the sandbox, the server answers the expiry with DISCONNECTED carrying `40142`, and the +client is back in CONNECTED about a tenth of a second later, having called the auth callback +a second time — so the specification's thirty-second poll has a wide margin. + +The specification records `initial_connection_id` before the expiry and never asserts on +it, so nothing here reads it: ably-python has no `Connection#id` to read it from, and +adding the adaptation described in [deviations.md](../../../deviations.md) would only +introduce a value no assertion consumes. +""" + +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt + +# The two lifetimes the specification issues: one short enough for the server to expire +# during the test, and one that outlives it. +SHORT_TTL = 5000 +LONG_TTL = 3600000 + +# The specification's `poll_until(interval: 1000ms, timeout: 30s)` and the fifteen seconds +# it then gives the reconnection. +RENEWAL_TIMEOUT = 30.0 +RENEWAL_INTERVAL = 1.0 +RECONNECT_TIMEOUT = 15.0 + + +# UTS: realtime/integration/RSA4b/token-renewal-on-expiry-0 +async def test_rsa4b_token_renewal_on_expiry(realtime_sandbox): + api_key = realtime_sandbox.key_str + key_name = extract_key_name(api_key) + key_secret = extract_key_secret(api_key) + callback_count = [] + + async def auth_callback(params): + callback_count.append(params) + ttl = SHORT_TTL if len(callback_count) == 1 else LONG_TTL + return generate_jwt(key_name=key_name, key_secret=key_secret, ttl=ttl) + + client = sandbox_realtime_client(auth_callback=auth_callback, auto_connect=False) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, RECONNECT_TIMEOUT) + + assert len(callback_count) == 1 + + await wall_clock_poll_until( + lambda: len(callback_count) >= 2, timeout=RENEWAL_TIMEOUT, interval=RENEWAL_INTERVAL, + description='the auth callback to be invoked for a renewed token') + + await await_connection_state(client, ConnectionState.CONNECTED, RECONNECT_TIMEOUT) + + assert len(callback_count) >= 2 + assert client.connection.state is ConnectionState.CONNECTED diff --git a/test/uts/realtime/integration/auth/token_request_test.py b/test/uts/realtime/integration/auth/token_request_test.py new file mode 100644 index 00000000..285c8e26 --- /dev/null +++ b/test/uts/realtime/integration/auth/token_request_test.py @@ -0,0 +1,64 @@ +"""Derived from uts/realtime/integration/auth/token_request_test.md in ably/specification. + +Spec points: RSA9, RSA9a, RSA9g + +Both tests split the credentials in two: a REST client holding the API key signs +TokenRequests, and a realtime client with no key of its own connects with whatever that +callback hands it. Reaching CONNECTED is therefore the server's verdict on the HMAC the +creator computed, which is what RSA9g is about. + +`Connection#id` is not a member of ably-python's `Connection`, so `connection_id` below +reads the value off the connection manager. See +[deviations.md](../../../deviations.md) for the house ruling on a missing accessor. + +`create_token_request` takes the specification's `TokenParams` as a plain dict in snake +case, so `TokenParams(clientId: x)` is `{'client_id': x}`. +""" + +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, sandbox_rest_client +from test.uts.helpers.sandbox import random_id + +# The wait the specification gives each connection. +CONNECT_TIMEOUT = 15.0 + + +def connection_id(client): + """The specification's `connection.id`, held on the connection manager.""" + return client.connection.connection_manager.connection_id + + +# UTS: realtime/integration/RSA9a/token-request-server-accepted-0 +async def test_rsa9a_token_request_server_accepted(realtime_sandbox): + creator = sandbox_rest_client(realtime_sandbox.key_str) + + async def auth_callback(params): + return await creator.auth.create_token_request() + + client = sandbox_realtime_client(auth_callback=auth_callback, auto_connect=False) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + assert client.connection.state is ConnectionState.CONNECTED + assert connection_id(client) is not None + assert client.connection.error_reason is None + + +# UTS: realtime/integration/RSA9/token-request-with-clientid-0 +async def test_rsa9_token_request_with_clientid(realtime_sandbox): + test_client_id = 'token-request-client-' + random_id() + + creator = sandbox_rest_client(realtime_sandbox.key_str) + + async def auth_callback(params): + return await creator.auth.create_token_request({'client_id': test_client_id}) + + client = sandbox_realtime_client( + auth_callback=auth_callback, client_id=test_client_id, auto_connect=False) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + assert client.connection.state is ConnectionState.CONNECTED + assert client.auth.client_id == test_client_id diff --git a/test/uts/realtime/integration/auth_test.py b/test/uts/realtime/integration/auth_test.py new file mode 100644 index 00000000..7f82d260 --- /dev/null +++ b/test/uts/realtime/integration/auth_test.py @@ -0,0 +1,138 @@ +"""Derived from uts/realtime/integration/auth.md in ably/specification. + +Spec points: RTC8, RTC8a, RTC8c, RSA8, RSA7 + +Every client here authenticates through an `auth_callback` returning an Ably JWT, which is +what the specification's third-party JWT library produces; `generate_jwt` signs it HS256 +over the key secret. + +`Connection#id` is not a member of ably-python's `Connection`, so `connection_id` below +reads the value off the connection manager. See +[deviations.md](../../deviations.md) for the house ruling on a missing accessor. + +`authorize()` on a CONNECTED connection sends AUTH and then awaits the next state change +before it returns, so the UPDATE it provokes has already been delivered to a listener +registered beforehand by the time the call completes. The reauth test needs no settle. +""" + +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client +from test.uts.helpers.deviations import deviation +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id + +# The lifetime every JWT here is issued with, as the specification's `ttl: 3600000`. +TOKEN_TTL = 3600000 + +# The wait for the mismatched-clientId connection to be failed by the server. The SDK spends +# its `disconnected_retry_timeout` — 15 seconds by default — between the first attempt and +# the one the server rejects, so the specification's unstated wait has to clear that. +FAIL_TIMEOUT = 20.0 + + +def connection_id(client): + """The specification's `connection.id`, held on the connection manager.""" + return client.connection.connection_manager.connection_id + + +def jwt_callback(api_key, client_id=None): + """The specification's `auth_callback`, answering with a freshly signed Ably JWT.""" + key_name = extract_key_name(api_key) + key_secret = extract_key_secret(api_key) + + async def auth_callback(params): + return generate_jwt(key_name=key_name, key_secret=key_secret, ttl=TOKEN_TTL, client_id=client_id) + + return auth_callback + + +# UTS: realtime/integration/RTC8a/in-band-reauth-connected-0 +async def test_rtc8a_in_band_reauth_connected(realtime_sandbox): + client = sandbox_realtime_client( + auth_callback=jwt_callback(realtime_sandbox.key_str), auto_connect=False) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED) + + connection_id_before = connection_id(client) + + state_changes = [] + + def record(change): + state_changes.append(change) + + client.connection.on(record) + + token = await client.auth.authorize() + + connection_id_after = connection_id(client) + + assert token is not None + assert isinstance(token.token, str) + + assert connection_id_after == connection_id_before + + state_transitions = [change for change in state_changes if change.current != change.previous] + assert state_transitions == [] + + +# UTS: realtime/integration/RTC8c/authorize-initiates-connection-0 +async def test_rtc8c_authorize_initiates_connection(realtime_sandbox): + client = sandbox_realtime_client( + auth_callback=jwt_callback(realtime_sandbox.key_str), auto_connect=False) + + assert client.connection.state is ConnectionState.INITIALIZED + + token = await client.auth.authorize() + + await await_connection_state(client, ConnectionState.CONNECTED) + + assert token is not None + assert client.connection.state is ConnectionState.CONNECTED + assert connection_id(client) is not None + + +# UTS: realtime/integration/RSA8/token-auth-connect-0 +async def test_rsa8_token_auth_connect(realtime_sandbox): + client = sandbox_realtime_client( + auth_callback=jwt_callback(realtime_sandbox.key_str), auto_connect=False) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED) + + assert client.connection.state is ConnectionState.CONNECTED + assert connection_id(client) is not None + assert client.connection.error_reason is None + + +# UTS: realtime/integration/RSA7/matching-clientid-succeeds-0 +@deviation +async def test_rsa7_matching_clientid_succeeds(realtime_sandbox): + test_client_id = 'test-client-' + random_id() + + client = sandbox_realtime_client( + auth_callback=jwt_callback(realtime_sandbox.key_str, client_id=test_client_id), + client_id=test_client_id, auto_connect=False) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED) + + assert client.connection.state is ConnectionState.CONNECTED + assert client.auth.client_id == test_client_id + + +# UTS: realtime/integration/RSA7/mismatched-clientid-fails-1 +async def test_rsa7_mismatched_clientid_fails(realtime_sandbox): + # UTS SPEC ERROR: the test step expects the `Realtime` constructor to throw, while the + # assertions below it say the key assertion is that the connection enters FAILED with + # 40102. Both cannot hold — a constructor has no token to compare a clientId against, + # and the specification's own note concedes the point — so the client is constructed + # and the FAILED assertion the specification names is the one made. + client = sandbox_realtime_client( + auth_callback=jwt_callback(realtime_sandbox.key_str, client_id='token-client-id'), + client_id='wrong-client-id', auto_connect=False) + + client.connect() + await await_connection_state(client, ConnectionState.FAILED, FAIL_TIMEOUT) + + assert client.connection.state is ConnectionState.FAILED + assert client.connection.error_reason.code == 40102 diff --git a/test/uts/realtime/integration/channel_history_test.py b/test/uts/realtime/integration/channel_history_test.py new file mode 100644 index 00000000..d49eb95a --- /dev/null +++ b/test/uts/realtime/integration/channel_history_test.py @@ -0,0 +1,69 @@ +"""Derived from uts/realtime/integration/channel_history_test.md in ably/specification. + +Spec points: RTL10d + +The specification carries a `## Protocol Variants` section, so the test runs once per +protocol and passes `use_binary_protocol` to both clients. + +Its setup leaves `autoConnect` at the library default and calls `connect()` anyway, which +is what is derived here: the client is already CONNECTING by the time `connect()` is +called and the call is a no-op. + +`RealtimeChannel` inherits `history()` from the REST `Channel`, so the read goes over +HTTP rather than the connection. A message does not reach history the instant its publish +is acknowledged, so the fetch is a `wall_clock_poll_until` answering `None` until the +page holds all three — a `PaginatedResult` is truthy whether or not it holds anything, +so returning the page directly would be satisfied by the first empty one. + +`RealtimeChannel.publish()` takes its arguments positionally; the keyword form the +specification writes raises `ValueError`. +""" + +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import random_id + + +# UTS: realtime/integration/RTL10d/history-cross-client-0 +async def test_rtl10d_history_cross_client(realtime_sandbox, use_binary_protocol): + channel_name = 'history-RTL10d-' + random_id() + + publisher = sandbox_realtime_client( + realtime_sandbox.key_str, use_binary_protocol=use_binary_protocol) + subscriber = sandbox_realtime_client( + realtime_sandbox.key_str, use_binary_protocol=use_binary_protocol) + + publisher.connect() + subscriber.connect() + + await await_connection_state(publisher, ConnectionState.CONNECTED, timeout=10) + await await_connection_state(subscriber, ConnectionState.CONNECTED, timeout=10) + + pub_channel = publisher.channels.get(channel_name) + sub_channel = subscriber.channels.get(channel_name) + + await pub_channel.attach() + await sub_channel.attach() + + await pub_channel.publish('event1', 'data1') + await pub_channel.publish('event2', 'data2') + await pub_channel.publish('event3', 'data3') + + async def all_three_messages(): + page = await sub_channel.history() + return page if len(page.items) == 3 else None + + history = await wall_clock_poll_until( + all_three_messages, description='all three messages to reach history') + + assert len(history.items) == 3 + + # The default order is backwards: newest first. + assert history.items[0].name == 'event3' + assert history.items[0].data == 'data3' + + assert history.items[1].name == 'event2' + assert history.items[1].data == 'data2' + + assert history.items[2].name == 'event1' + assert history.items[2].data == 'data1' diff --git a/test/uts/realtime/integration/channels/__init__.py b/test/uts/realtime/integration/channels/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/realtime/integration/channels/channel_attach_test.py b/test/uts/realtime/integration/channels/channel_attach_test.py new file mode 100644 index 00000000..68f31243 --- /dev/null +++ b/test/uts/realtime/integration/channels/channel_attach_test.py @@ -0,0 +1,90 @@ +"""Derived from uts/realtime/integration/channels/channel_attach_test.md in ably/specification. + +Spec points: RTL4, RTL4c, RTL5, RTL5d, RTL14 + +There is no `## Protocol Variants` section, so these run against JSON only and take no +`use_binary_protocol`; `sandbox_realtime_client` already defaults to it. + +`RealtimeChannel.publish()` takes its arguments positionally. The keyword form the +specification writes, which `RestChannel.publish()` does accept, raises +`ValueError: publish() expects either (name, data) or a message object or array of +messages` before anything reaches the server. + +The specification's `AWAIT_STATE` for CONNECTED is given ten seconds here. `await_connection_state` +defaults to five, which is the budget a mock-backed test needs; a real connect to the sandbox +opens a websocket over the network, and ten is the figure the sibling `channel_history_test.md` +spells out for the same wait. +""" + +import pytest + +from ably.realtime.channel import ChannelState +from ably.realtime.connection import ConnectionState +from ably.util.exceptions import AblyException +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client +from test.uts.helpers.sandbox import random_id + + +async def connected_client(key): + """A realtime client built as the specification's setup builds one, already CONNECTED.""" + client = sandbox_realtime_client(key, auto_connect=False) + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=10) + return client + + +# UTS: realtime/integration/RTL4c/attach-succeeds-0 +async def test_rtl4c_attach_succeeds(realtime_sandbox): + client = await connected_client(realtime_sandbox.key_str) + + channel = client.channels.get('attach-RTL4c-' + random_id()) + assert channel.state == ChannelState.INITIALIZED + + await channel.attach() + + assert channel.state == ChannelState.ATTACHED + assert channel.error_reason is None + + +# UTS: realtime/integration/RTL5d/detach-succeeds-0 +async def test_rtl5d_detach_succeeds(realtime_sandbox): + client = await connected_client(realtime_sandbox.key_str) + + channel = client.channels.get('detach-RTL5d-' + random_id()) + await channel.attach() + assert channel.state == ChannelState.ATTACHED + + await channel.detach() + + assert channel.state == ChannelState.DETACHED + + +# UTS SPEC ERROR: the section heading and its prose say the channel transitions to FAILED +# on a channel-scoped ERROR, but the test steps below say the opposite — a subscribe-only +# key attaches successfully to any channel — and the assertions never read the channel +# state. The steps are what the server does, so they are what is derived here. +# UTS: realtime/integration/RTL14/insufficient-capability-failed-0 +async def test_rtl14_insufficient_capability_failed(realtime_sandbox): + # keys[3] is the subscribe-only key, {"*": ["subscribe"]}. Read off the key rather + # than assumed, so that a change to the app setup shows up here rather than turning + # the test into one that proves nothing. + subscribe_only_key = realtime_sandbox.key(3) + assert subscribe_only_key.capability == {'*': ['subscribe']} + + client = await connected_client(subscribe_only_key.key_str) + + channel = client.channels.get('publish-not-allowed-' + random_id()) + + # Attach succeeds: a subscribe-only key can attach to any channel. + await channel.attach() + assert channel.state == ChannelState.ATTACHED + + # The publish is refused: the key lacks the publish capability. + with pytest.raises(AblyException) as excinfo: + await channel.publish('test', 'data') + + assert excinfo.value.code == 40160 + assert excinfo.value.status_code == 401 + + # The error is channel-scoped, so the connection is left alone. + assert client.connection.state == ConnectionState.CONNECTED diff --git a/test/uts/realtime/integration/channels/channel_publish_test.py b/test/uts/realtime/integration/channels/channel_publish_test.py new file mode 100644 index 00000000..53b9800a --- /dev/null +++ b/test/uts/realtime/integration/channels/channel_publish_test.py @@ -0,0 +1,187 @@ +"""Derived from uts/realtime/integration/channels/channel_publish_test.md in ably/specification. + +Spec points: RTL6, RTL6f, RSL4, RSL6, RSL6a2 + +The specification carries a `## Protocol Variants` section, so every test here runs once +per protocol and passes `use_binary_protocol` to both clients it builds. + +`RealtimeChannel.publish()` takes its arguments positionally; the keyword form the +specification writes, which `RestChannel.publish()` does accept, raises `ValueError`. + +A binary payload arrives as a `bytearray` rather than `bytes` under either protocol, so +the type assertion reads both. `bytearray` compares equal to the `bytes` that was +published, so the equality the specification asks for is unaffected. + +`Connection#id` is not a public member of ably-python's `Connection`; `connection_id` +below reads the same value off the connection manager, per the house ruling on missing +accessors in [deviations.md](../../../deviations.md). + +The specification's `AWAIT_STATE` for CONNECTED is given ten seconds here. `await_connection_state` +defaults to five, which is the budget a mock-backed test needs; a real connect to the sandbox +opens a websocket over the network, and ten is the figure the sibling `channel_history_test.md` +spells out for the same wait. +""" + +from ably.realtime.connection import ConnectionState +from ably.types.message import Message +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import random_id + + +def connection_id(client): + """The specification's `client.connection.id`.""" + return client.connection.connection_manager.connection_id + + +async def connected_pair(key, use_binary_protocol): + """The publisher and subscriber every test here opens with, both CONNECTED.""" + publisher = sandbox_realtime_client( + key, auto_connect=False, use_binary_protocol=use_binary_protocol) + subscriber = sandbox_realtime_client( + key, auto_connect=False, use_binary_protocol=use_binary_protocol) + publisher.connect() + subscriber.connect() + await await_connection_state(publisher, ConnectionState.CONNECTED, timeout=10) + await await_connection_state(subscriber, ConnectionState.CONNECTED, timeout=10) + return publisher, subscriber + + +# UTS: realtime/integration/RTL6/string-data-roundtrip-0 +async def test_rtl6_string_data_roundtrip(realtime_sandbox, use_binary_protocol): + publisher, subscriber = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + channel_name = 'publish-string-' + random_id() + pub_channel = publisher.channels.get(channel_name) + sub_channel = subscriber.channels.get(channel_name) + + received = [] + + def on_message(message): + received.append(message) + + await sub_channel.subscribe(on_message) + await pub_channel.attach() + + await pub_channel.publish('string-event', 'hello world') + + await wall_clock_poll_until( + lambda: len(received) >= 1, description='the published string to be delivered') + + assert len(received) == 1 + assert received[0].name == 'string-event' + assert received[0].data == 'hello world' + assert isinstance(received[0].data, str) + + +# UTS: realtime/integration/RTL6/json-data-roundtrip-1 +async def test_rtl6_json_data_roundtrip(realtime_sandbox, use_binary_protocol): + publisher, subscriber = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + channel_name = 'publish-json-' + random_id() + pub_channel = publisher.channels.get(channel_name) + sub_channel = subscriber.channels.get(channel_name) + + json_data = {'key': 'value', 'nested': {'count': 42}, 'list': [1, 2, 3]} + + received = [] + + def on_message(message): + received.append(message) + + await sub_channel.subscribe(on_message) + await pub_channel.attach() + + await pub_channel.publish('json-event', json_data) + + await wall_clock_poll_until( + lambda: len(received) >= 1, description='the published object to be delivered') + + assert len(received) == 1 + assert received[0].name == 'json-event' + assert received[0].data['key'] == 'value' + assert received[0].data['nested']['count'] == 42 + assert received[0].data['list'] == [1, 2, 3] + + +# UTS: realtime/integration/RTL6/binary-data-roundtrip-2 +async def test_rtl6_binary_data_roundtrip(realtime_sandbox, use_binary_protocol): + publisher, subscriber = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + channel_name = 'publish-binary-' + random_id() + pub_channel = publisher.channels.get(channel_name) + sub_channel = subscriber.channels.get(channel_name) + + binary_data = bytes([0, 1, 2, 255, 128, 64]) + + received = [] + + def on_message(message): + received.append(message) + + await sub_channel.subscribe(on_message) + await pub_channel.attach() + + await pub_channel.publish('binary-event', binary_data) + + await wall_clock_poll_until( + lambda: len(received) >= 1, description='the published bytes to be delivered') + + assert len(received) == 1 + assert received[0].name == 'binary-event' + assert isinstance(received[0].data, (bytes, bytearray)) + assert received[0].data == binary_data + + +# UTS: realtime/integration/RTL6f/connectionid-matches-publisher-0 +async def test_rtl6f_connectionid_matches_publisher(realtime_sandbox, use_binary_protocol): + publisher, subscriber = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + channel_name = 'publish-connid-' + random_id() + pub_channel = publisher.channels.get(channel_name) + sub_channel = subscriber.channels.get(channel_name) + + publisher_connection_id = connection_id(publisher) + + received = [] + + def on_message(message): + received.append(message) + + await sub_channel.subscribe(on_message) + await pub_channel.attach() + + await pub_channel.publish('connid-test', 'data') + + await wall_clock_poll_until( + lambda: len(received) >= 1, description='the published message to be delivered') + + assert received[0].connection_id == publisher_connection_id + assert received[0].connection_id != connection_id(subscriber) + + +# UTS: realtime/integration/RSL6a2/message-extras-roundtrip-0 +async def test_rsl6a2_message_extras_roundtrip(realtime_sandbox, use_binary_protocol): + publisher, subscriber = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + # The `pushenabled:` namespace is what lets a message carry push extras. + channel_name = 'pushenabled:publish-extras-' + random_id() + pub_channel = publisher.channels.get(channel_name) + sub_channel = subscriber.channels.get(channel_name) + + extras = {'push': {'notification': {'title': 'Testing'}}} + + received = [] + + def on_message(message): + received.append(message) + + await sub_channel.subscribe(on_message) + await pub_channel.attach() + + await pub_channel.publish(Message(name='extras-test', data='payload', extras=extras)) + + await wall_clock_poll_until( + lambda: len(received) >= 1, description='the message carrying extras to be delivered') + + assert received[0].extras is not None + assert received[0].extras['push']['notification']['title'] == 'Testing' diff --git a/test/uts/realtime/integration/channels/channel_subscribe_test.py b/test/uts/realtime/integration/channels/channel_subscribe_test.py new file mode 100644 index 00000000..c8850930 --- /dev/null +++ b/test/uts/realtime/integration/channels/channel_subscribe_test.py @@ -0,0 +1,157 @@ +"""Derived from uts/realtime/integration/channels/channel_subscribe_test.md in ably/specification. + +Spec points: RTL7, RTL7a, RTL7b, RTL7d + +There is no `## Protocol Variants` section, so these run against JSON only and take no +`use_binary_protocol`. + +`RealtimeChannel.subscribe()` is a coroutine that attaches the channel (RTL7c) and +returns nothing, so every registration here is awaited, including the second one in +RTL7b which the specification writes without `AWAIT`. Each listener is its own `def`: +`EventEmitter` keys its wrapper registry on the listener object alone, so registering +one function against two events would overwrite the first registration. + +`RealtimeChannel.publish()` takes its arguments positionally; the keyword form the +specification writes raises `ValueError`. + +The specification's `AWAIT_STATE` for CONNECTED is given ten seconds here. `await_connection_state` +defaults to five, which is the budget a mock-backed test needs; a real connect to the sandbox +opens a websocket over the network, and ten is the figure the sibling `channel_history_test.md` +spells out for the same wait. +""" + +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import random_id + + +async def connected_pair(key, **kwargs): + """Two realtime clients built as the specification's setup builds them, both CONNECTED. + + `kwargs` reaches each client, which is how RTL7 gives its two their client ids. Both + clients otherwise take the same options, so a single mapping serves. + """ + first = sandbox_realtime_client(key, auto_connect=False, **kwargs) + second = sandbox_realtime_client(key, auto_connect=False, **kwargs) + first.connect() + second.connect() + await await_connection_state(first, ConnectionState.CONNECTED, timeout=10) + await await_connection_state(second, ConnectionState.CONNECTED, timeout=10) + return first, second + + +# UTS: realtime/integration/RTL7a/subscribe-all-messages-0 +async def test_rtl7a_subscribe_all_messages(realtime_sandbox): + publisher, subscriber = await connected_pair(realtime_sandbox.key_str) + + channel_name = 'subscribe-all-' + random_id() + pub_channel = publisher.channels.get(channel_name) + sub_channel = subscriber.channels.get(channel_name) + + received = [] + + def on_message(message): + received.append(message) + + await sub_channel.subscribe(on_message) + await pub_channel.attach() + + await pub_channel.publish('event-a', 'data-a') + await pub_channel.publish('event-b', 'data-b') + await pub_channel.publish('event-c', 'data-c') + + await wall_clock_poll_until( + lambda: len(received) >= 3, description='all three messages to be delivered') + + assert len(received) == 3 + + names = [message.name for message in received] + assert 'event-a' in names + assert 'event-b' in names + assert 'event-c' in names + + +# UTS: realtime/integration/RTL7b/subscribe-filtered-by-name-0 +async def test_rtl7b_subscribe_filtered_by_name(realtime_sandbox): + publisher, subscriber = await connected_pair(realtime_sandbox.key_str) + + channel_name = 'subscribe-filtered-' + random_id() + pub_channel = publisher.channels.get(channel_name) + sub_channel = subscriber.channels.get(channel_name) + + target_received = [] + all_received = [] + + def on_target(message): + target_received.append(message) + + def on_any(message): + all_received.append(message) + + await sub_channel.subscribe('target', on_target) + + # Subscribing to every event as well gives the poll below something to wait on that + # covers the messages the filtered subscription is meant to drop. + await sub_channel.subscribe(on_any) + + await pub_channel.attach() + + await pub_channel.publish('other', 'ignored') + await pub_channel.publish('target', 'wanted-1') + await pub_channel.publish('other', 'ignored') + await pub_channel.publish('target', 'wanted-2') + + await wall_clock_poll_until( + lambda: len(all_received) >= 4, description='all four messages to be delivered') + + assert len(all_received) == 4 + + assert len(target_received) == 2 + assert target_received[0].name == 'target' + assert target_received[0].data == 'wanted-1' + assert target_received[1].name == 'target' + assert target_received[1].data == 'wanted-2' + + +# UTS: realtime/integration/RTL7/bidirectional-message-flow-0 +async def test_rtl7_bidirectional_message_flow(realtime_sandbox): + client_a = sandbox_realtime_client( + realtime_sandbox.key_str, auto_connect=False, client_id='client-a') + client_b = sandbox_realtime_client( + realtime_sandbox.key_str, auto_connect=False, client_id='client-b') + client_a.connect() + client_b.connect() + await await_connection_state(client_a, ConnectionState.CONNECTED, timeout=10) + await await_connection_state(client_b, ConnectionState.CONNECTED, timeout=10) + + channel_name = 'subscribe-bidir-' + random_id() + channel_a = client_a.channels.get(channel_name) + channel_b = client_b.channels.get(channel_name) + + received_by_a = [] + received_by_b = [] + + def on_message_for_a(message): + received_by_a.append(message) + + def on_message_for_b(message): + received_by_b.append(message) + + await channel_a.subscribe(on_message_for_a) + await channel_b.subscribe(on_message_for_b) + + await channel_a.publish('from-a', 'hello from a') + await channel_b.publish('from-b', 'hello from b') + + await wall_clock_poll_until( + lambda: len(received_by_a) >= 2 and len(received_by_b) >= 2, + description='both clients to receive both messages') + + # Each client receives both publishers' messages, its own echo included. + a_names = [message.name for message in received_by_a] + b_names = [message.name for message in received_by_b] + + assert 'from-a' in a_names + assert 'from-b' in a_names + assert 'from-a' in b_names + assert 'from-b' in b_names diff --git a/test/uts/realtime/integration/conftest.py b/test/uts/realtime/integration/conftest.py new file mode 100644 index 00000000..60e6ebfd --- /dev/null +++ b/test/uts/realtime/integration/conftest.py @@ -0,0 +1,66 @@ +"""Fixtures the realtime integration specifications share. + +The sandbox app is provisioned once and read by every test that asks for it, +as it is for the REST integration tier: a specification's `BEFORE ALL TESTS` +provisions an app, and twenty specifications each provisioning their own would +make the tier several times slower and would invite the sandbox's rate +limiting. +""" + +import os + +import pytest +import pytest_asyncio + +from test.uts.helpers.sandbox import delete_app, provision_app + +# `integration-testing.md` puts a suite of this size at 120 seconds. The +# individual operations these specifications wait on are bounded at 10 to 30, +# and a realtime test spends longer than a REST one: it opens a connection, +# attaches a channel and waits on state changes before it asserts anything. +# The repository default in `pyproject.toml` is 30 seconds, which suits a test +# served from a mock. The marker applies to this package alone, so the unit +# tiers keep the tighter default. +SUITE_TIMEOUT = 120 + +__package_dir = os.path.dirname(os.path.abspath(__file__)) + + +def pytest_collection_modifyitems(items): + for item in items: + if os.path.abspath(str(item.fspath)).startswith(__package_dir + os.sep): + item.add_marker(pytest.mark.timeout(SUITE_TIMEOUT)) + + +@pytest_asyncio.fixture(scope='session') +async def realtime_sandbox(): + """The provisioned sandbox app, as a specification's `app_config`. + + This is the specifications' `BEFORE ALL TESTS` / `AFTER ALL TESTS` pair. + `realtime_sandbox.key(0).key_str` is the full-access key they call + `api_key`, and the other indices are the capabilities named in each + specification's app provisioning section. + + It is a separate app from the REST tier's `sandbox`, and separately named, + so that a realtime test entering presence or publishing to a channel cannot + be seen by a REST test reading the same channel name. + """ + app = await provision_app() + yield app + await delete_app(app) + + +@pytest.fixture(params=[False, True], ids=['json', 'msgpack']) +def use_binary_protocol(request): + """Runs the test once per protocol, which is the specifications' `PROTOCOL`. + + A specification carrying a `## Protocol Variants` section runs against both + json and msgpack, and passes this straight to its clients: + + client = sandbox_realtime_client(api_key, use_binary_protocol=use_binary_protocol) + + Only a test that asks for it is parametrised. The specifications without + that section are json only, and their clients take the JSON default + `sandbox_realtime_client` already applies. + """ + return request.param diff --git a/test/uts/realtime/integration/connection/__init__.py b/test/uts/realtime/integration/connection/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/realtime/integration/connection/connection_failures_test.py b/test/uts/realtime/integration/connection/connection_failures_test.py new file mode 100644 index 00000000..3e31a24e --- /dev/null +++ b/test/uts/realtime/integration/connection/connection_failures_test.py @@ -0,0 +1,51 @@ +"""Derived from uts/realtime/integration/connection/connection_failures_test.md in ably/specification. + +Spec points: RTN14a, RTN14g + +Both tests present the sandbox with a key naming an application that does not exist, and +both are answered the same way: the server closes the websocket with a policy violation and +the SDK reports `40101 / 401 "unable to handle request; no application id found in +request"`. Each specification admits that code — RTN14a as one of `40005` or `40101`, RTN14g +as anything outside the token-error range `40140`-`40149` — so the two tests differ in what +they assert about one shared server response rather than in the response they provoke. + +Neither test asks for the `realtime_sandbox` fixture the specification's `BEFORE ALL +TESTS` provisions: the credentials under test name an application that was never +created, so the provisioned app has nothing to do with either connection. + +A failed connect leaves a `Task exception was never retrieved` line behind it: +`WebSocketTransport.close` sends a CLOSE over a socket the server has already closed with +1008. It is noise from teardown, not a failure of either test. +""" + +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client + +# The wait the specification gives each connect attempt. +FAIL_TIMEOUT = 15.0 + + +# UTS: realtime/integration/RTN14a/invalid-key-failed-0 +async def test_rtn14a_invalid_key_failed(): + client = sandbox_realtime_client('invalid.key:secret', auto_connect=False) + + client.connect() + await await_connection_state(client, ConnectionState.FAILED, FAIL_TIMEOUT) + + assert client.connection.state is ConnectionState.FAILED + assert client.connection.error_reason is not None + assert client.connection.error_reason.code in (40005, 40101) + assert client.connection.error_reason.status_code in (401, 404) + + +# UTS: realtime/integration/RTN14g/revoked-key-failed-0 +async def test_rtn14g_revoked_key_failed(): + client = sandbox_realtime_client('nonexistent.keyname:keysecret', auto_connect=False) + + client.connect() + await await_connection_state(client, ConnectionState.FAILED, FAIL_TIMEOUT) + + assert client.connection.state is ConnectionState.FAILED + assert client.connection.error_reason is not None + # Outside the token-error range, so RTN14g and not RTN14b applies. + assert client.connection.error_reason.code < 40140 or client.connection.error_reason.code >= 40150 diff --git a/test/uts/realtime/integration/connection_lifecycle_test.py b/test/uts/realtime/integration/connection_lifecycle_test.py new file mode 100644 index 00000000..ffa0c126 --- /dev/null +++ b/test/uts/realtime/integration/connection_lifecycle_test.py @@ -0,0 +1,109 @@ +"""Derived from uts/realtime/integration/connection_lifecycle_test.md in ably/specification. + +Spec points: RTN4b, RTN4c, RTN11, RTN12, RTN12a, RTN21 + +`Connection#id` and `Connection#key` do not exist in ably-python, so the two readers below +stand in for them; see [deviations.md](../../deviations.md) for the house ruling on a +missing accessor. Both answer `None` once the connection is closed, which is what the +graceful-close assertions read. + +`connection.close()` is a coroutine that returns once the connection has reached CLOSED, so +CLOSING is already gone by the time it does and cannot be waited for afterwards. The close +test therefore records the state changes with a listener registered beforehand and asserts +the sequence on the recording, which is what the specification's two consecutive +`AWAIT_STATE` steps describe. +""" + +import re + +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client + +# The waits the specification's `Integration Test Notes` give each leg: auth plus transport +# setup for the connect, and a CLOSE message round trip for the close. +CONNECT_TIMEOUT = 10.0 +CLOSE_TIMEOUT = 5.0 + + +def connection_id(client): + """The specification's `connection.id`, held on the connection manager.""" + return client.connection.connection_manager.connection_id + + +def connection_key(client): + """The specification's `connection.key`, held on the connection details.""" + details = client.connection.connection_details + return details.connection_key if details is not None else None + + +# UTS: realtime/integration/RTN4b/successful-connection-0 +async def test_rtn4b_successful_connection(realtime_sandbox): + # UTS SPEC ERROR: the setup leaves autoConnect at its default and the first step then + # asserts INITIALIZED. RTN3 makes that default true, so a client built as the setup has + # it is already CONNECTING when the constructor returns. The fixture is corrected here + # rather than the assertion dropped. + client = sandbox_realtime_client(realtime_sandbox.key_str, auto_connect=False) + + assert client.connection.state is ConnectionState.INITIALIZED + + client.connect() + + await await_connection_state(client, ConnectionState.CONNECTING, CONNECT_TIMEOUT) + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + assert connection_id(client) is not None + assert connection_key(client) is not None + + assert client.connection.state is ConnectionState.CONNECTED + assert re.fullmatch(r'[a-zA-Z0-9_-]+', connection_id(client)) + assert re.fullmatch(r'[a-zA-Z0-9_!-]+', connection_key(client)) + assert client.connection.error_reason is None + + +# UTS: realtime/integration/RTN4c/graceful-close-0 +async def test_rtn4c_graceful_close(realtime_sandbox): + client = sandbox_realtime_client(realtime_sandbox.key_str) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + states = [] + + def record(change): + states.append(change.current) + + client.connection.on(record) + + await client.connection.close() + + assert ConnectionState.CLOSING in states + assert states[states.index(ConnectionState.CLOSING) + 1] is ConnectionState.CLOSED + + assert client.connection.state is ConnectionState.CLOSED + assert client.connection.error_reason is None + assert connection_id(client) is None + assert connection_key(client) is None + + +# UTS: realtime/integration/RTN11/connect-reconnect-cycle-0 +async def test_rtn11_connect_reconnect_cycle(realtime_sandbox): + client = sandbox_realtime_client(realtime_sandbox.key_str, auto_connect=False) + + assert client.connection.state is ConnectionState.INITIALIZED + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + first_connection_id = connection_id(client) + + await client.connection.close() + await await_connection_state(client, ConnectionState.CLOSED, CLOSE_TIMEOUT) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + second_connection_id = connection_id(client) + + assert second_connection_id is not None + assert first_connection_id != second_connection_id + assert client.connection.error_reason is None diff --git a/test/uts/realtime/integration/delta_decoding_test.py b/test/uts/realtime/integration/delta_decoding_test.py new file mode 100644 index 00000000..6538d84b --- /dev/null +++ b/test/uts/realtime/integration/delta_decoding_test.py @@ -0,0 +1,314 @@ +"""Derived from uts/realtime/integration/delta_decoding.md in ably/specification. + +Spec points: PC3, PC3a, RTL18, RTL18b, RTL18c, RTL19b, RTL20 + +The full delta pipeline against the sandbox: a channel attached with +`params={'delta': 'vcdiff'}` has the server send every message after the first as a +vcdiff delta, and the SDK decodes it against the payload it stored for the previous one. + +The specification's `plugins: { vcdiff: decoder }` is the `vcdiff_decoder` client option +here, and its `VCDiffDecoder` interface is `ably.types.options.VCDiffDecoder` — +`decode(delta, base) -> bytes`, the VD2a argument order. `AblyVCDiffDecoder` is the real +implementation, backed by the `vcdiff-decoder` library; `CountingDecoder` wraps it so a +test can assert how many deltas the server actually sent. + +`clear_last_message_id` reaches into the channel's decoding context, which is what the +specification means by its implementation-specific `CLEAR channel._lastPayload.messageId`. +The stored id is the RTL20 base reference: clearing it makes the next delta fail its +check without any change to what the server sends. +""" + +import os + +from ably import AblyVCDiffDecoder +from ably.realtime.connection import ConnectionState +from ably.types.channeloptions import ChannelOptions +from ably.types.channelstate import ChannelState +from ably.types.options import VCDiffDecoder +from test.uts.helpers.client import ( + await_channel_state, + await_connection_state, + sandbox_realtime_client, + wall_clock_poll_until, +) +from test.uts.helpers.deviations import deviation +from test.uts.helpers.sandbox import random_id + +# The specification's `test_data`. The messages are deliberately similar, so that the +# server generates small vcdiff deltas rather than sending each one whole. +TEST_DATA = [ + {'foo': 'bar', 'count': 1, 'status': 'active'}, + {'foo': 'bar', 'count': 2, 'status': 'active'}, + {'foo': 'bar', 'count': 2, 'status': 'inactive'}, + {'foo': 'bar', 'count': 3, 'status': 'inactive'}, + {'foo': 'bar', 'count': 3, 'status': 'active'}, +] + +# The channel options that ask the server for deltas. +DELTA_PARAMS = ChannelOptions(params={'delta': 'vcdiff'}) + +# The specification's waits: 15 seconds for a straight delivery, 30 for one that has to +# go through the RTL18 recovery and a reattach first. +DELIVERY_TIMEOUT = 15.0 +RECOVERY_TIMEOUT = 30.0 + + +class CountingDecoder(VCDiffDecoder): + """A real vcdiff decoder that records how many deltas it was given.""" + + def __init__(self): + self.decode_count = 0 + self.__decoder = AblyVCDiffDecoder() + + def decode(self, delta: bytes, base: bytes) -> bytes: + self.decode_count += 1 + return self.__decoder.decode(delta, base) + + +class FailingDecoder(VCDiffDecoder): + """A decoder that always fails, for the RTL18 decode-failure path.""" + + def decode(self, delta: bytes, base: bytes) -> bytes: + raise Exception('vcdiff decode failure') + + +def clear_last_message_id(channel): + """Simulates a message gap by clearing the channel's stored last message id. + + The id lives on the channel's decoding context, which the channel builds in its + constructor and holds privately. `DecodingContext.last_message_id` is the base + reference RTL20 compares a delta's `extras.delta.from` against, so clearing it makes + the next delta fail that check exactly as a dropped message would. + """ + channel._RealtimeChannel__decoding_context.last_message_id = None + + +def record_attaching(channel): + """Collects the state change behind every ATTACHING the channel enters from here. + + The specification watches this to see whether a decode failure sent the channel back + round the RTL18 recovery: a test that expects no recovery asserts the list stayed + empty, and one that expects recovery reads the error off the first entry. + """ + changes = [] + channel.on(ChannelState.ATTACHING, lambda change: changes.append(change)) + return changes + + +async def delta_client(api_key, use_binary_protocol, decoder=None): + """A CONNECTED client carrying the specification's vcdiff plugin.""" + client = sandbox_realtime_client( + api_key, use_binary_protocol=use_binary_protocol, vcdiff_decoder=decoder) + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=10) + return client + + +async def publish_all(channel, payloads): + """Publishes the dataset one message at a time, as the specification does.""" + for i, payload in enumerate(payloads): + await channel.publish(str(i), payload) + + +# UTS: realtime/integration/PC3/delta-decode-end-to-end-0 +async def test_pc3_delta_decode_end_to_end(realtime_sandbox, use_binary_protocol): + channel_name = 'delta-PC3-' + random_id() + counting_decoder = CountingDecoder() + + client = await delta_client(realtime_sandbox.key_str, use_binary_protocol, counting_decoder) + + channel = client.channels.get(channel_name, DELTA_PARAMS) + await channel.attach() + + received_messages = [] + reattaches = record_attaching(channel) + + await channel.subscribe(lambda msg: received_messages.append(msg)) + + await publish_all(channel, TEST_DATA) + + await wall_clock_poll_until( + lambda: len(received_messages) == len(TEST_DATA) or reattaches, + timeout=DELIVERY_TIMEOUT, interval=0.2, description='every message to be received') + + assert not reattaches, f'Channel reattaching due to decode failure: {reattaches[0].reason}' + + for i, payload in enumerate(TEST_DATA): + assert received_messages[i].name == str(i) + assert received_messages[i].data == payload + + # The first message is sent as a full payload, the rest as deltas. + assert counting_decoder.decode_count == len(TEST_DATA) - 1 + + +# UTS: realtime/integration/RTL19b/dissimilar-payloads-no-delta-0 +async def test_rtl19b_dissimilar_payloads_no_delta(realtime_sandbox, use_binary_protocol): + channel_name = 'delta-dissimilar-' + random_id() + message_count = 5 + counting_decoder = CountingDecoder() + + client = await delta_client(realtime_sandbox.key_str, use_binary_protocol, counting_decoder) + + # Random binary payloads, 1KB each and completely dissimilar, so that a delta would + # be no smaller than the message it encodes. + payloads = [os.urandom(1024) for _ in range(message_count)] + + channel = client.channels.get(channel_name, DELTA_PARAMS) + await channel.attach() + + received_messages = [] + reattaches = record_attaching(channel) + + await channel.subscribe(lambda msg: received_messages.append(msg)) + + await publish_all(channel, payloads) + + await wall_clock_poll_until( + lambda: len(received_messages) == message_count or reattaches, + timeout=DELIVERY_TIMEOUT, interval=0.2, description='every message to be received') + + assert not reattaches, f'Channel reattaching due to decode failure: {reattaches[0].reason}' + + for i, payload in enumerate(payloads): + assert received_messages[i].name == str(i) + assert received_messages[i].data == payload + + # The server is expected to send full messages for dissimilar random binary payloads, + # but it is free to generate deltas anyway, so the count is reported rather than + # asserted. The assertions above hold either way: a delta that was sent was decoded + # back to the payload that was published. + print(f'Decoder was called {counting_decoder.decode_count} times ' + f'for {message_count} dissimilar messages') + + +# UTS: realtime/integration/PC3/no-deltas-without-param-1 +async def test_pc3_no_deltas_without_param(realtime_sandbox, use_binary_protocol): + channel_name = 'delta-no-param-' + random_id() + counting_decoder = CountingDecoder() + + client = await delta_client(realtime_sandbox.key_str, use_binary_protocol, counting_decoder) + + # Attached without the delta param, so the server has not been asked for deltas. + channel = client.channels.get(channel_name) + await channel.attach() + + received_messages = [] + await channel.subscribe(lambda msg: received_messages.append(msg)) + + await publish_all(channel, TEST_DATA) + + await wall_clock_poll_until( + lambda: len(received_messages) == len(TEST_DATA), + timeout=DELIVERY_TIMEOUT, interval=0.2, description='every message to be received') + + for i, payload in enumerate(TEST_DATA): + assert received_messages[i].name == str(i) + assert received_messages[i].data == payload + + assert counting_decoder.decode_count == 0 + + +# UTS: realtime/integration/RTL18/recovery-message-id-mismatch-0 +async def test_rtl18_recovery_message_id_mismatch(realtime_sandbox, use_binary_protocol): + channel_name = 'delta-recovery-mismatch-' + random_id() + counting_decoder = CountingDecoder() + + client = await delta_client(realtime_sandbox.key_str, use_binary_protocol, counting_decoder) + + channel = client.channels.get(channel_name, DELTA_PARAMS) + await channel.attach() + + received_messages = [] + attaching_changes = record_attaching(channel) + + await channel.subscribe(lambda msg: received_messages.append(msg)) + + # Publishing in two batches makes sure the server has sent and the client has + # processed the first batch before the stored id is cleared. Published all at once + # they could arrive in a single ProtocolMessage, decoded before the clear takes + # effect. + await publish_all(channel, TEST_DATA[:3]) + + await wall_clock_poll_until( + lambda: len(received_messages) >= 3, + timeout=DELIVERY_TIMEOUT, interval=0.2, description='the first batch to be received') + + clear_last_message_id(channel) + + for i in range(3, len(TEST_DATA)): + await channel.publish(str(i), TEST_DATA[i]) + + # Recovery reattaches and the server resends from the channelSerial, so a message may + # arrive twice; what is waited for is every name having arrived at least once. + expected_names = {str(i) for i in range(len(TEST_DATA))} + await wall_clock_poll_until( + lambda: {msg.name for msg in received_messages} >= expected_names, + timeout=RECOVERY_TIMEOUT, interval=0.2, description='every message to be received') + + for i, payload in enumerate(TEST_DATA): + msg = next((m for m in received_messages if m.name == str(i)), None) + assert msg is not None + assert msg.data == payload + + # RTL18c: recovery was triggered, carrying the delta decode failure. + assert len(attaching_changes) >= 1 + assert attaching_changes[0].reason.code == 40018 + + +# UTS: realtime/integration/RTL18/recovery-decode-failure-1 +async def test_rtl18_recovery_decode_failure(realtime_sandbox, use_binary_protocol): + channel_name = 'delta-recovery-decode-' + random_id() + + client = await delta_client( + realtime_sandbox.key_str, use_binary_protocol, FailingDecoder()) + + channel = client.channels.get(channel_name, DELTA_PARAMS) + await channel.attach() + + received_messages = [] + attaching_changes = record_attaching(channel) + + await channel.subscribe(lambda msg: received_messages.append(msg)) + + await publish_all(channel, TEST_DATA) + + # The first message arrives as a non-delta, the second fails to decode and triggers + # recovery, and the rest arrive after the reattach — as non-deltas, since the decode + # context is gone. + await wall_clock_poll_until( + lambda: len(received_messages) >= len(TEST_DATA), + timeout=RECOVERY_TIMEOUT, interval=0.2, description='every message to be received') + + for i, payload in enumerate(TEST_DATA): + msg = next((m for m in received_messages if m.name == str(i)), None) + assert msg is not None + assert msg.data == payload + + # RTL18c: at least one recovery was triggered. + assert len(attaching_changes) >= 1 + assert attaching_changes[0].reason.code == 40018 + + +# UTS: realtime/integration/PC3/no-plugin-causes-failed-2 +@deviation +async def test_pc3_no_plugin_causes_failed(realtime_sandbox, use_binary_protocol): + channel_name = 'delta-no-plugin-' + random_id() + + # The subscriber asks for deltas with no decoder to apply them with. The publisher is + # a separate connection so that the subscriber's channel going FAILED cannot fail the + # publishes or their pending ACKs. + subscriber = await delta_client(realtime_sandbox.key_str, use_binary_protocol) + publisher = await delta_client(realtime_sandbox.key_str, use_binary_protocol) + + sub_channel = subscriber.channels.get(channel_name, DELTA_PARAMS) + await sub_channel.attach() + + pub_channel = publisher.channels.get(channel_name) + await pub_channel.attach() + + await publish_all(pub_channel, TEST_DATA) + + await await_channel_state(sub_channel, ChannelState.FAILED, timeout=DELIVERY_TIMEOUT) + + assert sub_channel.state == ChannelState.FAILED + assert sub_channel.error_reason.code == 40019 diff --git a/test/uts/realtime/integration/mutable_messages_test.py b/test/uts/realtime/integration/mutable_messages_test.py new file mode 100644 index 00000000..8d19691a --- /dev/null +++ b/test/uts/realtime/integration/mutable_messages_test.py @@ -0,0 +1,439 @@ +"""Derived from uts/realtime/integration/mutable_messages.md in ably/specification. + +Spec points: RTL28, RTL31, RTL32, RTAN1, RTAN2, RTAN4 + +The realtime counterpart of `rest/integration/mutable_messages_test.py`: the mutations +travel as MESSAGE and ANNOTATION ProtocolMessages over the websocket rather than as HTTP +requests, and a second connection watches them arrive. + +Every channel name carries the `mutable:` prefix. `test-app-setup.json` configures that +namespace with `mutableMessages: true`, and the mutation and annotation operations are +refused on a channel outside it. + +`RealtimeChannel.publish()` takes its name and data positionally — the keyword form the +REST channel accepts raises `ValueError` here — and both `channel.subscribe()` and +`annotations.subscribe()` are coroutines, because each carries out an implicit attach. +`get_message` and `get_message_versions` on a realtime channel delegate to the REST +implementation, so they read the message store over HTTP and see it only once it is +consistent; the two polls below are the specification's `poll_until_success` around them. +""" + +from ably.http.paginatedresult import PaginatedResult +from ably.realtime.connection import ConnectionState +from ably.types.annotation import Annotation, AnnotationAction +from ably.types.channelmode import ChannelMode +from ably.types.channeloptions import ChannelOptions +from ably.types.channelstate import ChannelState +from ably.types.message import Message, MessageAction +from ably.types.operations import MessageOperation, UpdateDeleteResult +from ably.util.exceptions import AblyException +from test.uts.helpers.client import ( + await_channel_state, + await_connection_state, + sandbox_realtime_client, + wall_clock_poll_until, +) +from test.uts.helpers.sandbox import random_id + +# The specification's channel prefix, and the annotation type it annotates with. +MUTABLE_NAMESPACE = 'mutable:' +REACTION_TYPE = 'com.ably.reactions' + +# What the specification's `poll_until_success` gives a message store read. The store is +# eventually consistent, so a serial-scoped read answers 404, or answers with the version +# before the one being waited for, until the write has landed. +STORE_TIMEOUT = 20.0 + +# The modes the specification gives the two ends of the annotation tests. +PUBLISHER_MODES = [ + ChannelMode.PUBLISH, + ChannelMode.SUBSCRIBE, + ChannelMode.ANNOTATION_PUBLISH, + ChannelMode.ANNOTATION_SUBSCRIBE, +] +SUBSCRIBER_MODES = [ChannelMode.SUBSCRIBE, ChannelMode.ANNOTATION_SUBSCRIBE] + + +def mutable_channel_name(name): + """The channel a specification builds as `"mutable:rt-..." + random_id()`.""" + return f'{MUTABLE_NAMESPACE}{name}-{random_id()}' + + +async def connected_pair(api_key, use_binary_protocol): + """The two clients every observing test in the specification opens with.""" + client_a = sandbox_realtime_client(api_key, use_binary_protocol=use_binary_protocol) + client_b = sandbox_realtime_client(api_key, use_binary_protocol=use_binary_protocol) + + client_a.connect() + client_b.connect() + + await await_connection_state(client_a, ConnectionState.CONNECTED, timeout=10) + await await_connection_state(client_b, ConnectionState.CONNECTED, timeout=10) + + return client_a, client_b + + +async def await_messages(received_messages, count, description): + """The specification's `poll_until(received_messages.length >= count)`.""" + await wall_clock_poll_until( + lambda: len(received_messages) >= count, interval=0.2, description=description) + + +# UTS: realtime/integration/RTL32/update-message-observed-0 +async def test_rtl32_update_message_observed(realtime_sandbox, use_binary_protocol): + channel_name = mutable_channel_name('rt-update') + client_a, client_b = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + channel_a = client_a.channels.get(channel_name) + channel_b = client_b.channels.get(channel_name) + + await channel_b.attach() + + received_messages = [] + await channel_b.subscribe(lambda msg: received_messages.append(msg)) + + await channel_a.attach() + + await channel_a.publish('original', 'v1') + + await await_messages(received_messages, 1, 'client B to receive the original') + + serial = received_messages[0].serial + + update_result = await channel_a.update_message( + Message(serial=serial, name='updated', data='v2'), + operation=MessageOperation(description='edited')) + + await await_messages(received_messages, 2, 'client B to receive the update') + + assert isinstance(update_result, UpdateDeleteResult) + assert isinstance(update_result.version_serial, str) + assert len(update_result.version_serial) > 0 + + assert received_messages[0].action == MessageAction.MESSAGE_CREATE + assert received_messages[0].name == 'original' + assert received_messages[0].data == 'v1' + assert isinstance(received_messages[0].serial, str) + assert len(received_messages[0].serial) > 0 + + update_msg = received_messages[1] + assert update_msg.action == MessageAction.MESSAGE_UPDATE + assert update_msg.name == 'updated' + assert update_msg.data == 'v2' + assert update_msg.serial == serial + + +# UTS: realtime/integration/RTL32/delete-message-observed-1 +async def test_rtl32_delete_message_observed(realtime_sandbox, use_binary_protocol): + channel_name = mutable_channel_name('rt-delete') + client_a, client_b = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + channel_a = client_a.channels.get(channel_name) + channel_b = client_b.channels.get(channel_name) + + await channel_b.attach() + + received_messages = [] + await channel_b.subscribe(lambda msg: received_messages.append(msg)) + + await channel_a.attach() + + await channel_a.publish('to-delete', 'ephemeral') + + await await_messages(received_messages, 1, 'client B to receive the original') + + serial = received_messages[0].serial + + delete_result = await channel_a.delete_message(Message(serial=serial)) + + await await_messages(received_messages, 2, 'client B to receive the delete') + + assert isinstance(delete_result, UpdateDeleteResult) + assert isinstance(delete_result.version_serial, str) + assert len(delete_result.version_serial) > 0 + + delete_msg = received_messages[1] + assert delete_msg.action == MessageAction.MESSAGE_DELETE + assert delete_msg.serial == serial + + +# UTS: realtime/integration/RTL32/append-message-observed-2 +async def test_rtl32_append_message_observed(realtime_sandbox, use_binary_protocol): + channel_name = mutable_channel_name('rt-append') + client_a, client_b = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + channel_a = client_a.channels.get(channel_name) + channel_b = client_b.channels.get(channel_name) + + await channel_b.attach() + + received_messages = [] + await channel_b.subscribe(lambda msg: received_messages.append(msg)) + + await channel_a.attach() + + await channel_a.publish('appendable', 'original') + + await await_messages(received_messages, 1, 'client B to receive the original') + + serial = received_messages[0].serial + + append_result = await channel_a.append_message( + Message(serial=serial, data='appended-data'), + operation=MessageOperation(description='thread reply')) + + await await_messages(received_messages, 2, 'client B to receive the append') + + assert isinstance(append_result, UpdateDeleteResult) + assert isinstance(append_result.version_serial, str) + assert len(append_result.version_serial) > 0 + + append_msg = received_messages[1] + assert append_msg.action == MessageAction.MESSAGE_APPEND + assert append_msg.data == 'appended-data' + assert append_msg.serial == serial + + +# UTS: realtime/integration/RTL32/full-mutation-lifecycle-3 +async def test_rtl32_full_mutation_lifecycle(realtime_sandbox, use_binary_protocol): + channel_name = mutable_channel_name('rt-lifecycle') + client_a, client_b = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + channel_a = client_a.channels.get(channel_name) + channel_b = client_b.channels.get(channel_name) + + await channel_b.attach() + + received_messages = [] + await channel_b.subscribe(lambda msg: received_messages.append(msg)) + + await channel_a.attach() + + await channel_a.publish('lifecycle', 'v1') + await await_messages(received_messages, 1, 'client B to receive the create') + + serial = received_messages[0].serial + + await channel_a.update_message( + Message(serial=serial, name='lifecycle', data='v2'), + operation=MessageOperation(description='edit 1')) + await await_messages(received_messages, 2, 'client B to receive the update') + + await channel_a.append_message( + Message(serial=serial, data='reply-data'), + operation=MessageOperation(description='thread reply')) + await await_messages(received_messages, 3, 'client B to receive the append') + + await channel_a.delete_message(Message(serial=serial)) + await await_messages(received_messages, 4, 'client B to receive the delete') + + assert len(received_messages) == 4 + + assert received_messages[0].action == MessageAction.MESSAGE_CREATE + assert received_messages[0].name == 'lifecycle' + assert received_messages[0].data == 'v1' + assert received_messages[0].serial == serial + + assert received_messages[1].action == MessageAction.MESSAGE_UPDATE + assert received_messages[1].name == 'lifecycle' + assert received_messages[1].data == 'v2' + assert received_messages[1].serial == serial + + assert received_messages[2].action == MessageAction.MESSAGE_APPEND + assert received_messages[2].data == 'reply-data' + assert received_messages[2].serial == serial + + assert received_messages[3].action == MessageAction.MESSAGE_DELETE + assert received_messages[3].serial == serial + + +# UTS: realtime/integration/RTL28/get-message-and-versions-0 +async def test_rtl28_get_message_and_versions(realtime_sandbox, use_binary_protocol): + channel_name = mutable_channel_name('rt-get-versions') + client = sandbox_realtime_client( + realtime_sandbox.key_str, use_binary_protocol=use_binary_protocol) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=10) + + channel = client.channels.get(channel_name) + await channel.attach() + + received_messages = [] + await channel.subscribe(lambda msg: received_messages.append(msg)) + + await channel.publish('versioned', 'v1') + + await await_messages(received_messages, 1, 'the published message') + + serial = received_messages[0].serial + + await channel.update_message( + Message(serial=serial, data='v2'), operation=MessageOperation(description='first edit')) + await channel.update_message( + Message(serial=serial, data='v3'), operation=MessageOperation(description='second edit')) + + # The message store is eventually consistent, so the two reads poll rather than + # sleeping a fixed interval: a message that is already readable may still be a stale + # version, and a version listing may still be short. Only the store's 404 is + # swallowed — every other error means the request was wrong rather than early, and + # swallowing it would leave the test timing out on a message it could never explain. + async def latest_version(): + try: + message = await channel.get_message(serial) + except AblyException as error: + if error.status_code == 404: + return None + raise + if message.action == MessageAction.MESSAGE_UPDATE and message.data == 'v3': + return message + return None + + msg = await wall_clock_poll_until( + latest_version, timeout=STORE_TIMEOUT, description='the second update to be readable') + + async def full_history(): + try: + result = await channel.get_message_versions(serial) + except AblyException as error: + if error.status_code == 404: + return None + raise + return result if len(result.items) >= 3 else None + + versions = await wall_clock_poll_until( + full_history, timeout=STORE_TIMEOUT, description='the full version history') + + assert isinstance(msg, Message) + assert msg.serial == serial + assert msg.data == 'v3' + assert msg.action == MessageAction.MESSAGE_UPDATE + + assert isinstance(versions, PaginatedResult) + assert len(versions.items) >= 3 # original + 2 updates + + for item in versions.items: + assert isinstance(item, Message) + assert item.serial == serial + + +# UTS: realtime/integration/RTAN1/annotation-publish-delete-0 +async def test_rtan1_annotation_publish_delete(realtime_sandbox, use_binary_protocol): + channel_name = mutable_channel_name('rt-annotations') + client_a, client_b = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + channel_a = client_a.channels.get(channel_name, ChannelOptions(modes=PUBLISHER_MODES)) + channel_b = client_b.channels.get(channel_name, ChannelOptions(modes=SUBSCRIBER_MODES)) + + await channel_b.attach() + + received_annotations = [] + await channel_b.annotations.subscribe(lambda ann: received_annotations.append(ann)) + + # Client A subscribes to its own messages to capture the serial to annotate. + received_messages = [] + await channel_a.subscribe(lambda msg: received_messages.append(msg)) + + await channel_a.attach() + + await channel_a.publish('annotatable', 'content') + + await await_messages(received_messages, 1, 'the message to annotate') + + serial = received_messages[0].serial + + await channel_a.annotations.publish(serial, Annotation(type=REACTION_TYPE, name='like')) + + await wall_clock_poll_until( + lambda: len(received_annotations) >= 1, interval=0.2, + description='the annotation to arrive on client B') + + await channel_a.annotations.delete(serial, Annotation(type=REACTION_TYPE, name='like')) + + await wall_clock_poll_until( + lambda: len(received_annotations) >= 2, interval=0.2, + description='the annotation delete to arrive on client B') + + assert len(received_annotations) == 2 + + create_ann = received_annotations[0] + assert create_ann.action == AnnotationAction.ANNOTATION_CREATE + assert create_ann.type == REACTION_TYPE + assert create_ann.name == 'like' + assert create_ann.message_serial == serial + + delete_ann = received_annotations[1] + assert delete_ann.action == AnnotationAction.ANNOTATION_DELETE + assert delete_ann.type == REACTION_TYPE + assert delete_ann.name == 'like' + assert delete_ann.message_serial == serial + + +# UTS: realtime/integration/RTAN4c/annotation-type-filtering-0 +async def test_rtan4c_annotation_type_filtering(realtime_sandbox, use_binary_protocol): + channel_name = mutable_channel_name('rt-ann-filter') + client_a, client_b = await connected_pair(realtime_sandbox.key_str, use_binary_protocol) + + channel_a = client_a.channels.get(channel_name, ChannelOptions(modes=PUBLISHER_MODES)) + channel_b = client_b.channels.get(channel_name, ChannelOptions(modes=SUBSCRIBER_MODES)) + + await channel_b.attach() + + filtered_annotations = [] + await channel_b.annotations.subscribe( + REACTION_TYPE, lambda ann: filtered_annotations.append(ann)) + + # The unfiltered listener is what tells the test when all three have been delivered, + # so that a filtered listener which wrongly received the third type is caught rather + # than raced past. + all_annotations = [] + await channel_b.annotations.subscribe(lambda ann: all_annotations.append(ann)) + + received_messages = [] + await channel_a.subscribe(lambda msg: received_messages.append(msg)) + + await channel_a.attach() + + await channel_a.publish('multi-type', 'content') + + await await_messages(received_messages, 1, 'the message to annotate') + + serial = received_messages[0].serial + + await channel_a.annotations.publish(serial, Annotation(type=REACTION_TYPE, name='like')) + await channel_a.annotations.publish( + serial, Annotation(type='com.example.comments', name='comment')) + await channel_a.annotations.publish(serial, Annotation(type=REACTION_TYPE, name='heart')) + + await wall_clock_poll_until( + lambda: len(all_annotations) >= 3, interval=0.2, + description='all three annotations to arrive on client B') + + assert len(all_annotations) == 3 + + assert len(filtered_annotations) == 2 + assert filtered_annotations[0].type == REACTION_TYPE + assert filtered_annotations[0].name == 'like' + assert filtered_annotations[1].type == REACTION_TYPE + assert filtered_annotations[1].name == 'heart' + + +# UTS: realtime/integration/RTAN4d/annotation-implicit-attach-0 +async def test_rtan4d_annotation_implicit_attach(realtime_sandbox, use_binary_protocol): + channel_name = mutable_channel_name('rt-ann-implicit-attach') + client = sandbox_realtime_client( + realtime_sandbox.key_str, use_binary_protocol=use_binary_protocol) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=10) + + channel = client.channels.get( + channel_name, ChannelOptions(modes=[ChannelMode.ANNOTATION_SUBSCRIBE])) + + assert channel.state == ChannelState.INITIALIZED + + await channel.annotations.subscribe(lambda ann: None) + + await await_channel_state(channel, ChannelState.ATTACHED, timeout=10) + + assert channel.state == ChannelState.ATTACHED diff --git a/test/uts/realtime/integration/presence/__init__.py b/test/uts/realtime/integration/presence/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/realtime/integration/presence/presence_sync_test.py b/test/uts/realtime/integration/presence/presence_sync_test.py new file mode 100644 index 00000000..6a78481d --- /dev/null +++ b/test/uts/realtime/integration/presence/presence_sync_test.py @@ -0,0 +1,90 @@ +"""Derived from uts/realtime/integration/presence/presence_sync.md in ably/specification. + +Spec points: RTP2, RTP11a + +Client B attaches to a channel where client A is already present, so its members arrive +through the server-initiated SYNC rather than through live PRESENCE messages. +`presence.get()` waits for the SYNC to complete (RTP11a), which is why neither test polls: +a member delivered by SYNC is in the map by the time `get()` answers, and a `get()` that +answered early would show an empty set rather than a late one. + +The specification is json only — it has no `## Protocol Variants` section — so these two +take the JSON default `sandbox_realtime_client` applies rather than the +`use_binary_protocol` fixture. + +`enter_client` on a connection authenticated by key alone is refused 40012, a root cause +recorded in [deviations.md](../../../deviations.md), so the client entering members on +behalf of others is built with `client_id='*'`. +""" + +from ably.realtime.connection import ConnectionState +from ably.types.presence import PresenceAction +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client +from test.uts.helpers.sandbox import random_id + +# The specification's `member_count` for the multiple-member case. +MEMBER_COUNT = 10 + + +# UTS: realtime/integration/RTP2/sync-delivers-members-0 +async def test_rtp2_sync_delivers_members(realtime_sandbox): + api_key = realtime_sandbox.key_str + channel_name = 'presence-sync-' + random_id() + + client_a = sandbox_realtime_client(api_key, client_id='sync-member-a', auto_connect=False) + client_b = sandbox_realtime_client(api_key, auto_connect=False) + + client_a.connect() + await await_connection_state(client_a, ConnectionState.CONNECTED, timeout=10) + + channel_a = client_a.channels.get(channel_name) + await channel_a.attach() + await channel_a.presence.enter(data='sync-data') + + # Client A is already present by the time client B connects. + client_b.connect() + await await_connection_state(client_b, ConnectionState.CONNECTED, timeout=10) + + channel_b = client_b.channels.get(channel_name) + await channel_b.attach() + + members = await channel_b.presence.get() + + assert len(members) == 1 + assert members[0].client_id == 'sync-member-a' + assert members[0].data == 'sync-data' + assert members[0].action == PresenceAction.PRESENT + + +# UTS: realtime/integration/RTP2/sync-multiple-members-1 +async def test_rtp2_sync_multiple_members(realtime_sandbox): + api_key = realtime_sandbox.key_str + channel_name = 'presence-sync-multi-' + random_id() + + client_a = sandbox_realtime_client(api_key, auto_connect=False, client_id='*') + client_b = sandbox_realtime_client(api_key, auto_connect=False) + + client_a.connect() + await await_connection_state(client_a, ConnectionState.CONNECTED, timeout=10) + + channel_a = client_a.channels.get(channel_name) + await channel_a.attach() + + for i in range(MEMBER_COUNT): + await channel_a.presence.enter_client(f'sync-user-{i}', data=f'data-{i}') + + client_b.connect() + await await_connection_state(client_b, ConnectionState.CONNECTED, timeout=10) + + channel_b = client_b.channels.get(channel_name) + await channel_b.attach() + + members = await channel_b.presence.get() + + assert len(members) == MEMBER_COUNT + + members_by_client_id = {member.client_id: member for member in members} + for i in range(MEMBER_COUNT): + member = members_by_client_id.get(f'sync-user-{i}') + assert member is not None + assert member.data == f'data-{i}' diff --git a/test/uts/realtime/integration/presence_lifecycle_test.py b/test/uts/realtime/integration/presence_lifecycle_test.py new file mode 100644 index 00000000..a6a74603 --- /dev/null +++ b/test/uts/realtime/integration/presence_lifecycle_test.py @@ -0,0 +1,167 @@ +"""Derived from uts/realtime/integration/presence_lifecycle.md in ably/specification. + +Spec points: RTP4, RTP6, RTP8, RTP9, RTP10, RTP11a + +Two connections against the sandbox: client A drives the presence set, client B watches +it. Everything client B asserts is read back from the server, so each phase waits for the +event to arrive before it calls `presence.get()`. + +`RealtimePresence.subscribe()` is a coroutine, because it carries out the RTP6d implicit +attach, and a presence action is named by its lowercase wire name — `'enter'`, +`'present'`, `'update'`, `'leave'` — which is what `set_presence` emits. + +Two adaptations, both of them root causes already recorded in +[deviations.md](../../deviations.md): + +- Subscribing to several actions at once (RTP6b) raises `TypeError: unhashable type: + 'list'`, because the list reaches pyee as a dict key. The one listener is registered + once per action instead, which is what the array form means. +- `enter_client` on a connection authenticated by key alone is refused 40012, so the + client entering members on behalf of others is built with `client_id='*'`. +""" + +import asyncio + +from ably.realtime.connection import ConnectionState +from ably.types.presence import PresenceAction +from test.uts.helpers.client import ( + await_connection_state, + sandbox_realtime_client, + wall_clock_poll_until, +) +from test.uts.helpers.sandbox import random_id + +# The specification's `member_count`. It notes that RTP4 says 250 and that 50 validates +# the same behaviour without the runtime. +MEMBER_COUNT = 50 + +# What the specification gives the bulk enter to be observed, against the 10 seconds it +# gives a single transition. +BULK_TIMEOUT = 15.0 + + +# UTS: realtime/integration/RTP4/bulk-enter-observed-0 +async def test_rtp4_bulk_enter_observed(realtime_sandbox, use_binary_protocol): + api_key = realtime_sandbox.key_str + channel_name = 'presence-bulk-' + random_id() + + client_a = sandbox_realtime_client( + api_key, use_binary_protocol=use_binary_protocol, client_id='*') + client_b = sandbox_realtime_client(api_key, use_binary_protocol=use_binary_protocol) + + client_a.connect() + await await_connection_state(client_a, ConnectionState.CONNECTED, timeout=10) + client_b.connect() + await await_connection_state(client_b, ConnectionState.CONNECTED, timeout=10) + + channel_a = client_a.channels.get(channel_name) + channel_b = client_b.channels.get(channel_name) + await channel_b.attach() + + # Subscribe on client B before client A enters. Members are counted by clientId from + # both ENTER and PRESENT events: if client B's connection drops mid-test, members it + # missed arrive via the presence re-sync as PRESENT events rather than ENTER, and an + # ENTER-only count would undercount forever. + entered_client_ids = set() + + def on_member(event): + entered_client_ids.add(event.client_id) + + await channel_b.presence.subscribe('enter', on_member) + await channel_b.presence.subscribe('present', on_member) + + await channel_a.attach() + + await asyncio.gather(*[ + channel_a.presence.enter_client(f'user-{i}', data=f'data-{i}') + for i in range(MEMBER_COUNT) + ]) + + await wall_clock_poll_until( + lambda: len(entered_client_ids) >= MEMBER_COUNT, + timeout=BULK_TIMEOUT, + interval=0.2, + description='client B to observe every member entering') + + members = await channel_b.presence.get() + + assert len(entered_client_ids) == MEMBER_COUNT + + assert len(members) == MEMBER_COUNT + + members_by_client_id = {member.client_id: member for member in members} + for i in range(MEMBER_COUNT): + member = members_by_client_id.get(f'user-{i}') + assert member is not None + assert member.data == f'data-{i}' + + +# UTS: realtime/integration/RTP8/enter-update-leave-lifecycle-0 +async def test_rtp8_enter_update_leave_lifecycle(realtime_sandbox, use_binary_protocol): + api_key = realtime_sandbox.key_str + channel_name = 'presence-lifecycle-' + random_id() + + client_a = sandbox_realtime_client( + api_key, use_binary_protocol=use_binary_protocol, client_id='lifecycle-client') + client_b = sandbox_realtime_client(api_key, use_binary_protocol=use_binary_protocol) + + client_a.connect() + await await_connection_state(client_a, ConnectionState.CONNECTED, timeout=10) + client_b.connect() + await await_connection_state(client_b, ConnectionState.CONNECTED, timeout=10) + + channel_a = client_a.channels.get(channel_name) + channel_b = client_b.channels.get(channel_name) + await channel_b.attach() + + all_events = [] + await channel_b.presence.subscribe(lambda event: all_events.append(event)) + + await channel_a.attach() + + # --- Phase 1: Enter --- + await channel_a.presence.enter(data='hello') + + await wall_clock_poll_until( + lambda: len(all_events) >= 1, interval=0.2, description='the ENTER event on client B') + + members_after_enter = await channel_b.presence.get() + assert len(members_after_enter) == 1 + assert members_after_enter[0].client_id == 'lifecycle-client' + assert members_after_enter[0].data == 'hello' + + # --- Phase 2: Update --- + await channel_a.presence.update(data='world') + + await wall_clock_poll_until( + lambda: len(all_events) >= 2, interval=0.2, description='the UPDATE event on client B') + + members_after_update = await channel_b.presence.get() + assert len(members_after_update) == 1 + assert members_after_update[0].data == 'world' + + # --- Phase 3: Leave --- + await channel_a.presence.leave(data='goodbye') + + await wall_clock_poll_until( + lambda: len(all_events) >= 3, interval=0.2, description='the LEAVE event on client B') + + members_after_leave = await channel_b.presence.get() + assert len(members_after_leave) == 0 + + assert len(all_events) >= 3 + + enter_event = all_events[0] + assert enter_event.action == PresenceAction.ENTER + assert enter_event.client_id == 'lifecycle-client' + assert enter_event.data == 'hello' + + update_event = all_events[1] + assert update_event.action == PresenceAction.UPDATE + assert update_event.client_id == 'lifecycle-client' + assert update_event.data == 'world' + + leave_event = all_events[2] + assert leave_event.action == PresenceAction.LEAVE + assert leave_event.client_id == 'lifecycle-client' + assert leave_event.data == 'goodbye' From 8d0022884345344f1b9fb252d41b3d4f38d4fc03 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 25 Sep 2026 11:23:24 +0100 Subject: [PATCH 3/5] test: run the realtime proxy specifications through uts-proxy Seven specifications, 30 Test IDs, each faulting one frame or one request between the client and the sandbox and asserting on what the SDK does next. The session, the rules and the event log come from `test/uts/helpers/proxy.py`, which the REST proxy package already uses; the package's own fixtures give a session per test and close it however the test ends. Co-Authored-By: Claude Opus 5 (1M context) --- .../realtime/integration/proxy/__init__.py | 0 .../integration/proxy/auth_reauth_test.py | 133 +++ .../integration/proxy/channel_faults_test.py | 507 ++++++++++ .../realtime/integration/proxy/conftest.py | 80 ++ .../proxy/connection_open_failures_test.py | 344 +++++++ .../proxy/connection_resume_test.py | 884 ++++++++++++++++++ .../integration/proxy/heartbeat_test.py | 153 +++ .../proxy/presence_reentry_test.py | 253 +++++ .../integration/proxy/rest_faults_test.py | 243 +++++ 9 files changed, 2597 insertions(+) create mode 100644 test/uts/realtime/integration/proxy/__init__.py create mode 100644 test/uts/realtime/integration/proxy/auth_reauth_test.py create mode 100644 test/uts/realtime/integration/proxy/channel_faults_test.py create mode 100644 test/uts/realtime/integration/proxy/conftest.py create mode 100644 test/uts/realtime/integration/proxy/connection_open_failures_test.py create mode 100644 test/uts/realtime/integration/proxy/connection_resume_test.py create mode 100644 test/uts/realtime/integration/proxy/heartbeat_test.py create mode 100644 test/uts/realtime/integration/proxy/presence_reentry_test.py create mode 100644 test/uts/realtime/integration/proxy/rest_faults_test.py diff --git a/test/uts/realtime/integration/proxy/__init__.py b/test/uts/realtime/integration/proxy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/uts/realtime/integration/proxy/auth_reauth_test.py b/test/uts/realtime/integration/proxy/auth_reauth_test.py new file mode 100644 index 00000000..847f66f1 --- /dev/null +++ b/test/uts/realtime/integration/proxy/auth_reauth_test.py @@ -0,0 +1,133 @@ +"""Derived from uts/realtime/integration/proxy/auth_reauth.md in ably/specification. + +Spec points: RTN22, RTC8a + +The session carries no rules. The fault is imperative instead: once the connection +is up, `trigger_action({'type': 'inject_to_client', 'message': {'action': 17}})` +plants an AUTH ProtocolMessage in the stream as though the sandbox had asked the +client to re-authenticate, and everything else on the connection is the sandbox's +own traffic. + +`test/uts/realtime/unit/connection/server_initiated_reauth_test.py` covers RTN22 and +RTN22a against `MockWebSocket`, where the mock answers the client's AUTH itself and +the test can read the outgoing ProtocolMessage directly — it pins that the token in +`auth.accessToken` is the one the callback just returned and that the reauth surfaces +as a single UPDATE event. Here the AUTH the SDK sends reaches the real server, which +answers it with a CONNECTED of its own; the outgoing frame is read back out of the +proxy's event log rather than off a mock, and what the test shows is that a reauth +carried out against the sandbox leaves the connection and its identity intact. + +The injected AUTH is not a genuine server request, so the specification is careful +about what it asserts: that the SDK's auth machinery ran, that an AUTH frame carrying +an `auth` attribute left the client, and that the connection was never disturbed. +""" + +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt + +# The specification's `AWAIT_STATE ... WITH timeout` and its `pollUntil` timeout, +# in seconds. +CONNECT_TIMEOUT = 15 +REAUTH_TIMEOUT = 15 + +# AUTH, from the action table in `uts/docs/proxy.md`. +AUTH_ACTION = 17 + + +def connection_id(client): + """The specification's `client.connection.id`, which `Connection` does not expose. + + See the house ruling on missing accessors in `test/uts/deviations.md`. + """ + return client.connection.connection_manager.connection_id + + +def jwt_auth_callback(api_key, calls): + """The specification's `authCallback`, counting its invocations in `calls`. + + It signs a JWT from the two halves of the sandbox key rather than asking for a + token, so re-authentication costs no round trip and puts nothing of its own in + the session's event log. + """ + key_name = extract_key_name(api_key) + key_secret = extract_key_secret(api_key) + + async def auth_callback(params): + calls.append(params) + return generate_jwt(key_name, key_secret) + + return auth_callback + + +# UTS: realtime/proxy/RTN22/server-initiated-reauth-0 +async def test_rtn22_server_initiated_reauth(realtime_sandbox, proxy_session): + auth_callback_calls = [] + + session = await proxy_session(rules=[]) + + client = sandbox_realtime_client( + auth_callback=jwt_auth_callback(realtime_sandbox.key_str, auth_callback_calls), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + ) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # Record identity and auth state before injection + original_connection_id = connection_id(client) + original_auth_callback_count = len(auth_callback_calls) + assert original_connection_id is not None + assert original_auth_callback_count >= 1 + + # Record state changes from this point + state_changes = [] + + def on_change(change): + state_changes.append(change.current) + + client.connection.on(on_change) + + # Inject a server-initiated AUTH ProtocolMessage (action 17), as though Ably + # were asking the client to re-authenticate + await session.trigger_action({'type': 'inject_to_client', 'message': {'action': AUTH_ACTION}}) + + # Wait for the SDK to process the AUTH and invoke the callback again + await wall_clock_poll_until( + lambda: len(auth_callback_calls) > original_auth_callback_count, + timeout=REAUTH_TIMEOUT, + description='the auth callback to be invoked again') + + # authCallback was called again (re-authentication triggered) + assert len(auth_callback_calls) == original_auth_callback_count + 1 + + # Connection remains CONNECTED (re-auth does not disrupt the connection) + assert client.connection.state == ConnectionState.CONNECTED + + # Connection ID is unchanged (no reconnection occurred) + assert connection_id(client) == original_connection_id + + # No state transitions away from CONNECTED occurred + assert [state for state in state_changes if state != ConnectionState.CONNECTED] == [] + + # Proxy log shows the SDK sent an AUTH frame (action 17) from client to server. + # The callback returns before the frame is written, so the log is polled for it + # rather than read once: the wait above is satisfied by the token alone. + async def client_auth_frames_in_log(): + log = await session.get_log() + frames = [event for event in log + if event['type'] == 'ws_frame' + and event.get('direction') == 'client_to_server' + and (event.get('message') or {}).get('action') in (AUTH_ACTION, 'AUTH') + and (event.get('message') or {}).get('auth') is not None] + return frames or None + + client_auth_frames = await wall_clock_poll_until( + client_auth_frames_in_log, + timeout=REAUTH_TIMEOUT, + description="the SDK's AUTH frame to reach the proxy") + assert len(client_auth_frames) >= 1 diff --git a/test/uts/realtime/integration/proxy/channel_faults_test.py b/test/uts/realtime/integration/proxy/channel_faults_test.py new file mode 100644 index 00000000..dbcdd378 --- /dev/null +++ b/test/uts/realtime/integration/proxy/channel_faults_test.py @@ -0,0 +1,507 @@ +"""Derived from uts/realtime/integration/proxy/channel_faults.md in ably/specification. + +Spec points: RTL4f, RTL5f, RTL13a, RTL14, RTL12, RTL3d + +Each test opens a `uts-proxy` session, points a realtime client at it and faults one +frame: an ATTACH or a DETACH suppressed so the server never answers it, an ATTACHED +replaced by an ERROR, or a DETACHED, an ERROR or an ATTACHED injected onto a channel +that is already attached. Everything the rule does not name reaches the sandbox, so the +re-attach a fault provokes completes against the real server. + +A frame rule matches `action` as a string — `'ATTACH'`, `'ATTACHED'`, `'DETACH'` — and +the proxy records the frame it matched before applying the rule, so a suppressed frame +still appears in the event log with `ruleMatched` carrying the rule's `comment`. A +replaced frame is logged as the frame the server sent, not as the replacement. + +Two of the specification's `AWAIT_STATE` steps wait for a state the channel already +holds: RTL13a and RTL3d both re-attach an attached channel, so waiting on ATTACHED +would return at once and assert nothing. Those steps are taken on the recorded state +sequence instead — the listener is registered before the fault is triggered, and the +wait is for ATTACHING followed by ATTACHED to appear in it, which is the transition the +specification is about and the same thing its `CONTAINS_IN_ORDER` assertion checks. + +There is no `## Protocol Variants` section, so these run against JSON only, which the +proxy tier requires in any case. +""" + +import asyncio + +import pytest + +from ably.realtime.connection import ConnectionState +from ably.types.channelstate import ChannelState +from ably.util.exceptions import AblyException +from test.uts.helpers.client import ( + await_channel_state, + await_connection_state, + sandbox_realtime_client, + wall_clock_poll_until, +) +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id + +# The specification's `realtimeRequestTimeout: 3000`, in the milliseconds the client +# option takes. It is the deadline on both the attach and the detach the two timeout +# tests provoke (TO3l11). +REALTIME_REQUEST_TIMEOUT_MS = 3000 + +# The specification's `AWAIT_STATE ... WITH timeout` values, as wall-clock seconds. +CONNECT_TIMEOUT = 15.0 +CHANNEL_TIMEOUT = 15.0 +FAILED_TIMEOUT = 10.0 +PENDING_TIMEOUT = 5.0 + +# ATTACH, as the event log reports a protocol message's `action`. +ATTACH_ACTION = 10 + + +def jwt_auth_callback(api_key): + """The specification's `authCallback`, signing an Ably JWT for the app's key. + + The callback makes no request of its own: the JWT is signed locally from the key + name and secret, so nothing the client authenticates with reaches the session and + lands in the event log beside the frames a test counts. + """ + async def auth_callback(params): + return generate_jwt(extract_key_name(api_key), extract_key_secret(api_key)) + + return auth_callback + + +def proxied_client(api_key, session, **kwargs): + """The `ClientOptions` every test in this specification builds its client with.""" + return sandbox_realtime_client( + auth_callback=jwt_auth_callback(api_key), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + **kwargs) + + +def attach_frames(log, channel_name): + """The ATTACH frames the proxy recorded from the client for `channel_name`.""" + return [event for event in log + if event['type'] == 'ws_frame' + and event.get('direction') == 'client_to_server' + and (event.get('message') or {}).get('action') == ATTACH_ACTION + and (event.get('message') or {}).get('channel') == channel_name] + + +def contains_in_order(recorded, expected): + """The specifications' `CONTAINS_IN_ORDER`: `expected` as a subsequence of `recorded`.""" + remaining = list(expected) + for item in recorded: + if remaining and item == remaining[0]: + remaining.pop(0) + return not remaining + + +def state_recorder(emitter): + """Records every state `emitter` enters, from the moment this is called.""" + recorded = [] + + def record(change): + recorded.append(change.current) + + emitter.on(record) + return recorded + + +# UTS: realtime/proxy/RTL4f/attach-timeout-suppressed-0 +async def test_rtl4f_attach_timeout_suppressed(realtime_sandbox, proxy_session): + channel_name = f'test-RTL4f-{random_id()}' + + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_server', 'action': 'ATTACH', 'channel': channel_name}, + 'action': {'type': 'suppress'}, + 'comment': 'RTL4f: Suppress ATTACH so server never responds', + }]) + + client = proxied_client(realtime_sandbox.key_str, session, + realtime_request_timeout=REALTIME_REQUEST_TIMEOUT_MS) + channel = client.channels.get(channel_name) + + # Record channel state changes for sequence verification + channel_state_changes = state_recorder(channel) + + # Connect through proxy -- connection itself is not faulted + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + # Start attach -- proxy will suppress the ATTACH, so server never responds + attach_future = asyncio.ensure_future(channel.attach()) + + # Channel should enter ATTACHING immediately + await await_channel_state(channel, ChannelState.ATTACHING, PENDING_TIMEOUT) + + # Wait for the channel to transition to SUSPENDED after realtimeRequestTimeout + await await_channel_state(channel, ChannelState.SUSPENDED, CHANNEL_TIMEOUT) + + # The attach() call should have failed with a timeout error + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(attach_future, PENDING_TIMEOUT) + + # Channel transitioned to SUSPENDED + assert channel.state == ChannelState.SUSPENDED + + # Error indicates timeout + assert excinfo.value is not None + + # State sequence: ATTACHING -> SUSPENDED + assert contains_in_order(channel_state_changes, [ChannelState.ATTACHING, ChannelState.SUSPENDED]) + + # Connection remains CONNECTED (attach timeout is channel-scoped) + assert client.connection.state == ConnectionState.CONNECTED + + # Proxy log confirms the ATTACH frames were received but suppressed. + # The proxy logs frames before applying rules, so suppressed frames still appear in + # the log with `ruleMatched` set. + log = await session.get_log() + frames = attach_frames(log, channel_name) + assert len(frames) >= 1 + + # All ATTACH frames were caught by the suppress rule + for frame in frames: + assert frame.get('ruleMatched') is not None + + +# UTS: realtime/proxy/RTL14/error-on-attach-0 +async def test_rtl14_error_on_attach(realtime_sandbox, proxy_session): + channel_name = f'test-RTL14-error-on-attach-{random_id()}' + + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': 'ATTACHED', 'channel': channel_name}, + 'action': { + 'type': 'replace', + 'message': { + 'action': 9, + 'channel': channel_name, + 'error': {'code': 40160, 'statusCode': 403, 'message': 'Not permitted'}, + }, + }, + 'times': 1, + 'comment': 'RTL14: Replace ATTACHED with channel ERROR', + }]) + + client = proxied_client(realtime_sandbox.key_str, session) + channel = client.channels.get(channel_name) + + # Record channel state changes for sequence verification + channel_state_changes = state_recorder(channel) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + # Attach -- proxy replaces ATTACHED with ERROR + with pytest.raises(AblyException) as excinfo: + await channel.attach() + + await await_channel_state(channel, ChannelState.FAILED, FAILED_TIMEOUT) + + # Channel transitioned to FAILED + assert channel.state == ChannelState.FAILED + + # Error reason matches the injected error + assert channel.error_reason is not None + assert channel.error_reason.code == 40160 + assert channel.error_reason.status_code == 403 + + # The error returned from attach() matches + assert excinfo.value is not None + assert excinfo.value.code == 40160 + + # State sequence: ATTACHING -> FAILED + assert contains_in_order(channel_state_changes, [ChannelState.ATTACHING, ChannelState.FAILED]) + + # Connection remains CONNECTED (channel error does not affect connection) + assert client.connection.state == ConnectionState.CONNECTED + + +# UTS: realtime/proxy/RTL5f/detach-timeout-suppressed-0 +async def test_rtl5f_detach_timeout_suppressed(realtime_sandbox, proxy_session): + channel_name = f'test-RTL5f-{random_id()}' + + # Phase 1: a session with no fault rules, so the attach passes through + session = await proxy_session(rules=[]) + + client = proxied_client(realtime_sandbox.key_str, session, + realtime_request_timeout=REALTIME_REQUEST_TIMEOUT_MS) + channel = client.channels.get(channel_name) + + # Record channel state changes for sequence verification + channel_state_changes = state_recorder(channel) + + # Phase 1: Connect and attach normally through proxy + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + await channel.attach() + assert channel.state == ChannelState.ATTACHED + + # Clear state change history from the attach phase + channel_state_changes.clear() + + # Phase 2: Add rule to suppress DETACH messages + await session.add_rules([{ + 'match': {'type': 'ws_frame_to_server', 'action': 'DETACH', 'channel': channel_name}, + 'action': {'type': 'suppress'}, + 'comment': 'RTL5f: Suppress DETACH so server never responds', + }], position='prepend') + + # Phase 3: Try to detach -- proxy suppresses DETACH, so server never sends DETACHED + detach_future = asyncio.ensure_future(channel.detach()) + + # Channel should enter DETACHING + await await_channel_state(channel, ChannelState.DETACHING, PENDING_TIMEOUT) + + # Wait for the channel to revert to ATTACHED after realtimeRequestTimeout + await await_channel_state(channel, ChannelState.ATTACHED, CHANNEL_TIMEOUT) + + # The detach() call should have failed with a timeout error + with pytest.raises(AblyException) as excinfo: + await asyncio.wait_for(detach_future, PENDING_TIMEOUT) + + # Channel reverted to ATTACHED (previous state) + assert channel.state == ChannelState.ATTACHED + + # Error indicates timeout + assert excinfo.value is not None + + # State sequence: DETACHING -> ATTACHED (revert) + assert contains_in_order(channel_state_changes, [ChannelState.DETACHING, ChannelState.ATTACHED]) + + # Connection remains CONNECTED + assert client.connection.state == ConnectionState.CONNECTED + + +# UTS: realtime/proxy/RTL13a/unsolicited-detach-reattach-0 +async def test_rtl13a_unsolicited_detach_reattach(realtime_sandbox, proxy_session): + channel_name = f'test-RTL13a-{random_id()}' + + session = await proxy_session(rules=[]) + + client = proxied_client(realtime_sandbox.key_str, session) + channel = client.channels.get(channel_name) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + await channel.attach() + assert channel.state == ChannelState.ATTACHED + + # Record channel state changes from this point + channel_state_changes = state_recorder(channel) + + # Inject an unsolicited DETACHED message with error via imperative action + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': { + 'action': 13, + 'channel': channel_name, + 'error': {'code': 90198, 'statusCode': 500, 'message': 'Channel detached by server'}, + }, + }) + + # Channel should transition ATTACHING (reattach) -> ATTACHED (reattach succeeds). + # The channel is attached when this wait begins, so it is the recorded sequence that + # is waited on rather than the state itself. + await wall_clock_poll_until( + lambda: contains_in_order(channel_state_changes, [ChannelState.ATTACHING, ChannelState.ATTACHED]), + CHANNEL_TIMEOUT, + 'the channel to re-attach after the injected DETACHED', + 0.2) + + # Channel re-attached successfully + assert channel.state == ChannelState.ATTACHED + + # State sequence: ATTACHING (with error from DETACHED) -> ATTACHED + assert contains_in_order(channel_state_changes, [ChannelState.ATTACHING, ChannelState.ATTACHED]) + + # Connection remains CONNECTED throughout + assert client.connection.state == ConnectionState.CONNECTED + + # Proxy log shows the re-attach ATTACH message from the client: at least 2 ATTACH + # frames, the initial attach and the reattach after the injected DETACHED + log = await session.get_log() + assert len(attach_frames(log, channel_name)) >= 2 + + +# UTS: realtime/proxy/RTL14/channel-error-goes-failed-1 +async def test_rtl14_channel_error_goes_failed(realtime_sandbox, proxy_session): + channel_name = f'test-RTL14-{random_id()}' + + session = await proxy_session(rules=[]) + + client = proxied_client(realtime_sandbox.key_str, session) + channel = client.channels.get(channel_name) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + await channel.attach() + assert channel.state == ChannelState.ATTACHED + + # Record channel state changes from this point + channel_state_changes = state_recorder(channel) + + # Inject a channel-scoped ERROR message via imperative action + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': { + 'action': 9, + 'channel': channel_name, + 'error': {'code': 40160, 'statusCode': 403, 'message': 'Not permitted'}, + }, + }) + + await await_channel_state(channel, ChannelState.FAILED, FAILED_TIMEOUT) + + # Channel transitioned to FAILED + assert channel.state == ChannelState.FAILED + + # errorReason is set from the injected ERROR + assert channel.error_reason is not None + assert channel.error_reason.code == 40160 + assert channel.error_reason.status_code == 403 + assert 'Not permitted' in channel.error_reason.message + + # State change event shows ATTACHED -> FAILED + assert contains_in_order(channel_state_changes, [ChannelState.FAILED]) + assert len(channel_state_changes) == 1 + + # Connection remains CONNECTED (channel-scoped ERROR does not close connection) + assert client.connection.state == ConnectionState.CONNECTED + + +# UTS: realtime/proxy/RTL12/attached-non-resumed-update-0 +async def test_rtl12_attached_non_resumed_update(realtime_sandbox, proxy_session): + channel_name = f'test-RTL12-{random_id()}' + + session = await proxy_session(rules=[]) + + client = proxied_client(realtime_sandbox.key_str, session) + channel = client.channels.get(channel_name) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + await channel.attach() + assert channel.state == ChannelState.ATTACHED + + # Listen for both 'update' and 'attached' events. `EventEmitter` keys its listener + # registry on the function, so the two events take a function each. + update_events = [] + attached_events = [] + + def on_update(change): + update_events.append(change) + + def on_attached(change): + attached_events.append(change) + + channel.on('update', on_update) + channel.on('attached', on_attached) + + # Inject an ATTACHED message with resumed=false and an error via imperative action + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': { + 'action': 11, + 'channel': channel_name, + 'flags': 0, + 'error': {'code': 91001, 'statusCode': 500, 'message': 'Continuity lost'}, + }, + }) + + # Wait for the update event to be emitted + await wall_clock_poll_until( + lambda: len(update_events) >= 1, FAILED_TIMEOUT, 'the channel to emit UPDATE', 0.2) + + # Channel emitted an UPDATE event + assert len(update_events) == 1 + + # The ChannelStateChange has correct fields + assert update_events[0].current == ChannelState.ATTACHED + assert update_events[0].previous == ChannelState.ATTACHED + assert update_events[0].resumed is False + assert update_events[0].reason is not None + assert update_events[0].reason.code == 91001 + assert update_events[0].reason.status_code == 500 + assert 'Continuity lost' in update_events[0].reason.message + + # No 'attached' event was emitted (RTL2g) + assert len(attached_events) == 0 + + # Channel state remains ATTACHED + assert channel.state == ChannelState.ATTACHED + + # Connection remains CONNECTED + assert client.connection.state == ConnectionState.CONNECTED + + +# UTS: realtime/proxy/RTL3d/channels-reattach-on-reconnect-0 +async def test_rtl3d_channels_reattach_on_reconnect(realtime_sandbox, proxy_session): + channel_a_name = f'test-RTL3d-a-{random_id()}' + channel_b_name = f'test-RTL3d-b-{random_id()}' + + session = await proxy_session(rules=[]) + + client = proxied_client(realtime_sandbox.key_str, session) + channel_a = client.channels.get(channel_a_name) + channel_b = client.channels.get(channel_b_name) + + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + await channel_a.attach() + await channel_b.attach() + assert channel_a.state == ChannelState.ATTACHED + assert channel_b.state == ChannelState.ATTACHED + + # Record channel state changes from this point + channel_a_state_changes = state_recorder(channel_a) + channel_b_state_changes = state_recorder(channel_b) + + # The connection retries a drop from CONNECTED at once, so DISCONNECTED is recorded + # rather than waited on: by the time a wait could be registered the connection may + # already be CONNECTING again. + connection_state_changes = state_recorder(client.connection) + + # Trigger disconnect via imperative action (close the WebSocket) + await session.trigger_action({'type': 'close'}) + + # Wait for connection to reach DISCONNECTED + await wall_clock_poll_until( + lambda: ConnectionState.DISCONNECTED in connection_state_changes, + FAILED_TIMEOUT, 'the connection to report DISCONNECTED', 0.2) + + # Wait for connection to recover to CONNECTED + await await_connection_state(client, ConnectionState.CONNECTED, 30.0) + + # Wait for both channels to re-attach. Both are attached when this wait begins, so + # it is their recorded sequences that are waited on. + reattached = [ChannelState.ATTACHING, ChannelState.ATTACHED] + await wall_clock_poll_until( + lambda: contains_in_order(channel_a_state_changes, reattached), + CHANNEL_TIMEOUT, 'channel a to re-attach', 0.2) + await wall_clock_poll_until( + lambda: contains_in_order(channel_b_state_changes, reattached), + CHANNEL_TIMEOUT, 'channel b to re-attach', 0.2) + + # Both channels end in ATTACHED state + assert channel_a.state == ChannelState.ATTACHED + assert channel_b.state == ChannelState.ATTACHED + + # Both channels transitioned through ATTACHING -> ATTACHED after reconnection + assert contains_in_order(channel_a_state_changes, reattached) + assert contains_in_order(channel_b_state_changes, reattached) + + # Connection is CONNECTED + assert client.connection.state == ConnectionState.CONNECTED + + # Proxy log shows ATTACH messages for both channels on the second WS connection: at + # least 2 each, the initial attach and the reattach after reconnection + log = await session.get_log() + assert len(attach_frames(log, channel_a_name)) >= 2 + assert len(attach_frames(log, channel_b_name)) >= 2 diff --git a/test/uts/realtime/integration/proxy/conftest.py b/test/uts/realtime/integration/proxy/conftest.py new file mode 100644 index 00000000..1d564201 --- /dev/null +++ b/test/uts/realtime/integration/proxy/conftest.py @@ -0,0 +1,80 @@ +"""Fixtures the proxy integration specifications share. + +The specifications in this package run their traffic through `uts-proxy`, so +each of them opens with a proxy session and closes it again afterwards. The +`realtime_sandbox` app they connect to comes from the parent package, which +provisions it once for the whole realtime integration tier. + +`test/uts/helpers/proxy.py` describes the proxy itself and the two environment +variables that change how it is obtained and started. +""" + +import os + +import pytest +import pytest_asyncio + +from test.uts.helpers.proxy import create_proxy_session, ensure_proxy, stop_proxy + +# What a test in this package gets, in seconds, in place of the 120 the parent +# package gives the rest of the tier. A proxy test spends its budget on things +# the other integration tests do not: the first one to run waits for the +# binary to be downloaded on a cold cache and for the control process to come +# up, and a specification that provokes a timeout deliberately sits through +# a twenty-second delay before the request it is measuring even fails. +SUITE_TIMEOUT = 300 + +__package_dir = os.path.dirname(os.path.abspath(__file__)) + + +def pytest_collection_modifyitems(items): + # The parent package marks everything beneath it, this package included, + # with its own shorter timeout, and pytest-timeout reads the marker + # closest to the test — the first of the item's own markers. Putting this + # one at the front rather than on the end is what makes it the one read, + # whichever order the two hooks happen to run in. + for item in items: + if os.path.abspath(str(item.fspath)).startswith(__package_dir + os.sep): + item.add_marker(pytest.mark.timeout(SUITE_TIMEOUT), append=False) + + +@pytest_asyncio.fixture(scope='session') +async def proxy_control(): + """The running `uts-proxy` control API, shared by every test in the package. + + One control process serves any number of sessions, each on a port of its + own, so it is started once and reaped when the run ends. Asking for this + fixture is what guarantees a proxy is up; `proxy_session` asks for it, so + a test that opens sessions does not have to. + """ + await ensure_proxy() + yield + stop_proxy() + + +@pytest_asyncio.fixture +async def proxy_session(proxy_control): + """Opens proxy sessions, and closes every one of them when the test ends. + + This is the specifications' `create_proxy_session(...)` together with + their `AFTER EACH TEST: IF session IS NOT null: session.close()`. A test + calls it as it would the function: + + session = await proxy_session(rules=[...]) + + and leaves the closing alone. Each session holds a port and an event log + on the proxy until it is closed, so a test that fails part way through — + which, in a suite about faults, is the case worth planning for — still + gives them back. + """ + sessions = [] + + async def open_session(**options): + session = await create_proxy_session(**options) + sessions.append(session) + return session + + yield open_session + + for session in reversed(sessions): + await session.close() diff --git a/test/uts/realtime/integration/proxy/connection_open_failures_test.py b/test/uts/realtime/integration/proxy/connection_open_failures_test.py new file mode 100644 index 00000000..58206871 --- /dev/null +++ b/test/uts/realtime/integration/proxy/connection_open_failures_test.py @@ -0,0 +1,344 @@ +"""Derived from uts/realtime/integration/proxy/connection_open_failures.md in ably/specification. + +Spec points: RTN14a, RTN14b, RTN14c, RTN14d, RTN14g + +The five faults here are the ones a connection can meet before it is ever open: a +fatal ERROR in place of CONNECTED, a token error in place of CONNECTED, a refused +WebSocket, and a CONNECTED that never arrives. Each is injected by a `uts-proxy` +rule carrying `times: 1`, so the attempt that follows the fault reaches the sandbox +unaltered and the test can tell a client that gave up from one that retried. + +`test/uts/realtime/unit/connection/connection_open_failures_test.py` covers the same +five spec points against `MockWebSocket`, and is the place where the state machine +itself is pinned — it drives the transition and retry timers through a `FakeClock`, +answers the connection attempt synthetically, and so can assert on timings that real +network traffic would make flaky. What these tests add is the transport underneath: +the SDK opens a real WebSocket, the frames it reads are the sandbox's own with one +substituted, and the retry it makes is a second connection the proxy's event log +records. A fault the unit tier can only describe — a refused TCP connection, a token +renewed against the real `/keys/…/requestToken` — is here exercised end to end. + +`endpoint='localhost'` names both hosts and disables the fallbacks (REC2c2), so every +attempt arrives at the one session port and appears in the one event log. A realtime +connection carries its credentials in the WebSocket's query string rather than in an +Authorization header, so `key=` works over the session's plain `ws://` where a REST +client's basic auth would be refused; RTN14b is the exception and authenticates with a +callback, because renewing a token is what it is about. +""" + +from ably import AblyRest +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client +from test.uts.helpers.deviations import deviation +from test.uts.helpers.mock_websocket import contains_in_order +from test.uts.helpers.sandbox import SANDBOX_ENDPOINT + +# The specification's `AWAIT_STATE ... WITH timeout`, in seconds: 15 for a +# connection that is meant to fail, 30 for one that has to fail and then succeed. +FAILURE_TIMEOUT = 15 +RECOVERY_TIMEOUT = 30 + +# The specification's `realtimeRequestTimeout: 3000` and +# `disconnectedRetryTimeout: 2000`. Both are milliseconds in ably-python too. +REALTIME_REQUEST_TIMEOUT_MS = 3000 +DISCONNECTED_RETRY_TIMEOUT_MS = 2000 + + +def connection_id(client): + """The specification's `client.connection.id`. + + `Connection` exposes no `id`; the connection manager holds it, and clears it + on the terminal states. See the house ruling on missing accessors in + `test/uts/deviations.md`. + """ + return client.connection.connection_manager.connection_id + + +def connection_key(client): + """The specification's `client.connection.key`. + + `Connection` exposes no `key` either. It arrives in `connectionDetails`, and + the whole `ConnectionDetails` is `None` until a CONNECTED has been received. + """ + details = client.connection.connection_details + return details.connection_key if details is not None else None + + +def record_states(client): + """The specification's `state_changes`, filled from `client.connection.on(...)`. + + Registered before `connect()` in every test here, since the first transition a + specification names is the CONNECTING that `connect()` itself causes. + """ + states = [] + + def on_change(change): + states.append(change.current) + + client.connection.on(on_change) + return states + + +def ws_connects(log): + """The `ws_connect` events the proxy recorded, in the order it accepted them. + + One per attempt the client made, each carrying the `queryParams` the SDK put + in the WebSocket URL. + """ + return [event for event in log if event['type'] == 'ws_connect'] + + +def token_auth_callback(api_key): + """The specification's `request_token_from_sandbox(api_key, params)`. + + The client under test points at the proxy, where a rule is waiting for the + first CONNECTED. A token requested through that client would go out over the + session's plain HTTP, where basic auth is refused (RSC18), and would add + traffic to the log beside the connections the test counts. So the callback + builds a Rest client of its own aimed straight at the sandbox and closes it + again; the token arrives over a connection the proxy never sees. + """ + async def auth_callback(params): + inner_rest = AblyRest(key=api_key, endpoint=SANDBOX_ENDPOINT) + try: + return await inner_rest.auth.request_token() + finally: + await inner_rest.close() + + return auth_callback + + +# UTS: realtime/proxy/RTN14a/fatal-connect-error-0 +async def test_rtn14a_fatal_connect_error(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': 'CONNECTED'}, + 'action': { + 'type': 'replace', + 'message': { + 'action': 9, + 'error': {'code': 40005, 'statusCode': 400, 'message': 'Invalid key'}, + }, + }, + 'times': 1, + 'comment': 'RTN14a: Replace CONNECTED with fatal ERROR', + }]) + + client = sandbox_realtime_client( + key=realtime_sandbox.key_str, + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + ) + state_changes = record_states(client) + + client.connect() + + await await_connection_state(client, ConnectionState.FAILED, timeout=FAILURE_TIMEOUT) + + # Connection transitioned to FAILED + assert client.connection.state == ConnectionState.FAILED + + # Error reason is set from the injected ERROR message + assert client.connection.error_reason is not None + assert client.connection.error_reason.code == 40005 + assert client.connection.error_reason.status_code == 400 + + # State sequence includes CONNECTING -> FAILED + assert contains_in_order(state_changes, [ConnectionState.CONNECTING, ConnectionState.FAILED]) + + # Connection ID/key not set (never received real CONNECTED) + assert connection_id(client) is None + assert connection_key(client) is None + + +# UTS: realtime/proxy/RTN14b/token-error-renew-reconnect-0 +@deviation +async def test_rtn14b_token_error_renew_reconnect(realtime_sandbox, proxy_session): + auth_callback_calls = [] + request_token = token_auth_callback(realtime_sandbox.key_str) + + async def auth_callback(params): + auth_callback_calls.append(params) + return await request_token(params) + + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': 'CONNECTED'}, + 'action': { + 'type': 'replace', + 'message': { + 'action': 9, + 'error': {'code': 40142, 'statusCode': 401, 'message': 'Token expired'}, + }, + }, + 'times': 1, + 'comment': 'RTN14b: Token error on first connect, renewal should succeed', + }]) + + client = sandbox_realtime_client( + auth_callback=auth_callback, + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + ) + state_changes = record_states(client) + + client.connect() + + await await_connection_state(client, ConnectionState.CONNECTED, timeout=RECOVERY_TIMEOUT) + + # Successfully connected after token renewal + assert client.connection.state == ConnectionState.CONNECTED + + # Connection properties are set (from the real CONNECTED on second attempt) + assert connection_id(client) is not None + assert connection_key(client) is not None + + # authCallback was called at least twice (initial token + renewal) + assert len(auth_callback_calls) >= 2 + + # State sequence shows the SDK went through CONNECTING, then back to CONNECTING after + # the error, and finally reached CONNECTED + assert contains_in_order(state_changes, [ConnectionState.CONNECTING, ConnectionState.CONNECTED]) + + # Proxy event log shows two WebSocket connections + log = await session.get_log() + assert len(ws_connects(log)) >= 2 + + # No residual error reason on successful connection + assert client.connection.error_reason is None + + +# UTS: realtime/proxy/RTN14d/retry-after-refused-0 +async def test_rtn14d_retry_after_refused(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_connect', 'count': 1}, + 'action': {'type': 'refuse_connection'}, + 'times': 1, + 'comment': 'RTN14d: Refuse first WebSocket connection', + }]) + + client = sandbox_realtime_client( + key=realtime_sandbox.key_str, + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + disconnected_retry_timeout=DISCONNECTED_RETRY_TIMEOUT_MS, + ) + state_changes = record_states(client) + + client.connect() + + await await_connection_state(client, ConnectionState.CONNECTED, timeout=RECOVERY_TIMEOUT) + + # Successfully connected after retry + assert client.connection.state == ConnectionState.CONNECTED + + # Connection properties are set + assert connection_id(client) is not None + assert connection_key(client) is not None + + # State sequence shows CONNECTING -> DISCONNECTED -> CONNECTING -> CONNECTED + assert contains_in_order(state_changes, [ + ConnectionState.CONNECTING, + ConnectionState.DISCONNECTED, + ConnectionState.CONNECTING, + ConnectionState.CONNECTED, + ]) + + # Proxy event log shows two WebSocket connection attempts + log = await session.get_log() + assert len(ws_connects(log)) >= 2 + + +# UTS: realtime/proxy/RTN14g/server-error-causes-failed-0 +async def test_rtn14g_server_error_causes_failed(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': 'CONNECTED'}, + 'action': { + 'type': 'replace', + 'message': { + 'action': 9, + 'error': {'code': 50000, 'statusCode': 500, 'message': 'Internal server error'}, + }, + }, + 'times': 1, + 'comment': 'RTN14g: Connection-level ERROR (server error) during open', + }]) + + client = sandbox_realtime_client( + key=realtime_sandbox.key_str, + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + ) + state_changes = record_states(client) + + client.connect() + + await await_connection_state(client, ConnectionState.FAILED, timeout=FAILURE_TIMEOUT) + + # Connection transitioned to FAILED + assert client.connection.state == ConnectionState.FAILED + + # Error reason is set from the injected ERROR message + assert client.connection.error_reason is not None + assert client.connection.error_reason.code == 50000 + assert client.connection.error_reason.status_code == 500 + assert client.connection.error_reason.message == 'Internal server error' + + # State sequence includes CONNECTING -> FAILED + assert contains_in_order(state_changes, [ConnectionState.CONNECTING, ConnectionState.FAILED]) + + # Connection ID/key not set + assert connection_id(client) is None + assert connection_key(client) is None + + +# UTS: realtime/proxy/RTN14c/connection-timeout-0 +async def test_rtn14c_connection_timeout(realtime_sandbox, proxy_session): + # No `times`, so every CONNECTED is suppressed: the WebSocket opens and the + # sandbox answers, but nothing the SDK would read as an open connection + # reaches it, and only its own timeout can end the attempt. + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': 'CONNECTED'}, + 'action': {'type': 'suppress'}, + 'comment': 'RTN14c: Suppress CONNECTED to force timeout', + }]) + + client = sandbox_realtime_client( + key=realtime_sandbox.key_str, + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + realtime_request_timeout=REALTIME_REQUEST_TIMEOUT_MS, + ) + state_changes = record_states(client) + + client.connect() + + await await_connection_state(client, ConnectionState.DISCONNECTED, timeout=FAILURE_TIMEOUT) + + # Connection timed out and transitioned to DISCONNECTED + assert client.connection.state == ConnectionState.DISCONNECTED + + # Error reason indicates timeout + assert client.connection.error_reason is not None + assert ('timeout' in client.connection.error_reason.message + or client.connection.error_reason.code in (50003, 80003)) + + # State sequence includes CONNECTING -> DISCONNECTED + assert contains_in_order(state_changes, [ConnectionState.CONNECTING, ConnectionState.DISCONNECTED]) + + # Connection ID/key not set (CONNECTED was never received) + assert connection_id(client) is None + assert connection_key(client) is None diff --git a/test/uts/realtime/integration/proxy/connection_resume_test.py b/test/uts/realtime/integration/proxy/connection_resume_test.py new file mode 100644 index 00000000..ae1bc783 --- /dev/null +++ b/test/uts/realtime/integration/proxy/connection_resume_test.py @@ -0,0 +1,884 @@ +"""Derived from uts/realtime/integration/proxy/connection_resume.md in ably/specification. + +Spec points: RTN14h, RTN15a, RTN15b, RTN15c6, RTN15c7, RTN15h1, RTN15h3, RTN15j, RTN16d, +RTN16l, RTN19a, RTN19a2 + +Every test here opens a `uts-proxy` session against the sandbox, points a realtime client +at it, and then takes the transport away — with a close frame, with a bare TCP FIN, with a +DISCONNECTED carrying an error the proxy invented, or with a CONNECTED the proxy rewrote. +What each one is about is what the client does next: whether it reconnects, whether the +reconnection carries a `resume` query parameter, and what identity and error it ends up +with. The proxy's event log is the second witness throughout — the SDK's own state answers +half of each question and the log answers the other half. + +**The connection's identity is read through internal members.** `Connection` exposes +`state`, `error_reason`, `connection_details` and `connection_manager` and nothing else, +so the specification's `connection.id`, `connection.key` and `connection.createRecoveryKey()` +have no public spelling here. `connection_id`, `connection_key` and `recovery_key` below +are the readers the house ruling in `test/uts/deviations.md` calls for, defined once so the +adaptation is in one place. The proxy log is often the better witness anyway: the `resume` +query parameter *is* the connection key the client held when it reconnected. + +**DISCONNECTED is transient.** RTN15a has the client retry a drop from CONNECTED +immediately — `loop.call_soon`, not a timer — so DISCONNECTED is gone within a millisecond +or two of being entered and `AWAIT_STATE disconnected` on the live state would be satisfied +by luck or not at all. Every test therefore registers a recorder on the connection before +it connects and waits on the recorded list, which is also what the specifications that read +`state_changes` want. `await_recorded_state(states, state, count)` is this file's +`AWAIT_STATE`. + +The client options are the proxy tier's: `endpoint='localhost'` and `port` aim the client +at the session, `tls=False` because the session speaks plain WebSocket, and +`use_binary_protocol=False`. Authentication is the specification's `authCallback` returning +a locally signed Ably JWT, which reaches no network and so puts nothing in the event log +that a test counts. Cleanup is left to the fixtures — `proxy_session` closes every session +it opened and `test/uts/conftest.py` closes every client — except in RTN16d, where closing +the first client is a step of the scenario rather than tidying up. +""" + +import asyncio +import json + +from ably.realtime.channel import ChannelState +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import sandbox_realtime_client, sandbox_rest_client, wall_clock_poll_until +from test.uts.helpers.deviations import deviation +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id + +# The waits the specification's `Integration Test Notes` section gives each step, in +# seconds: auth and transport setup through the proxy, a reconnection including the SDK's +# own retry, an injected DISCONNECTED arriving 1 s after the proxy's rule fires, and a +# transition to FAILED. +CONNECT_TIMEOUT = 15.0 +RECONNECT_TIMEOUT = 15.0 +DISCONNECT_TIMEOUT = 10.0 +FAILED_TIMEOUT = 15.0 +SUSPENDED_TIMEOUT = 15.0 + +# How often the recorded state list is re-read while waiting. The specifications' own +# polling interval is half a second, which is longer than the whole DISCONNECTED → +# CONNECTING → CONNECTED sequence takes; the list is append-only so nothing is missed +# either way, but a short interval keeps a test that waits on three states in turn from +# spending more time asleep than the scenario takes. +STATE_POLL_INTERVAL = 0.05 + +# The action numbers `uts/docs/proxy.md` tabulates, as they appear in a logged frame's +# decoded `message`. +ACK_ACTION = 1 +MESSAGE_ACTION = 15 + + +def jwt_auth_callback(api_key): + """The specification's `authCallback`, returning an Ably JWT for `api_key`. + + The JWT is signed here rather than requested from the sandbox, so authenticating the + client under test costs no traffic through the session and leaves the event log holding + only what the scenario put there. + """ + key_name = extract_key_name(api_key) + key_secret = extract_key_secret(api_key) + + async def auth_callback(params): + return generate_jwt(key_name, key_secret) + + return auth_callback + + +def proxy_realtime_client(session, api_key, **kwargs): + """The specification's `Realtime(options: ClientOptions(...))` for a proxy session.""" + return sandbox_realtime_client( + auth_callback=jwt_auth_callback(api_key), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + **kwargs, + ) + + +def connection_id(client): + """The specification's `connection.id`, which has no public accessor.""" + return client.connection.connection_manager.connection_id + + +def connection_key(client): + """The specification's `connection.key`, which has no public accessor. + + `connection_details` is `None` whenever the key would be, so the two are null together + exactly as the specification expects of `Connection#key`. + """ + details = client.connection.connection_details + return details.connection_key if details is not None else None + + +def recovery_key(client): + """The specification's `connection.createRecoveryKey()`, which the SDK does not define. + + RTN16i makes a recovery key the JSON serialisation of the connection key, the current + `msgSerial` and the `channelSerial` of every channel that is attached or attaching. All + three are held internally, so they are read from there and assembled here. + """ + manager = client.connection.connection_manager + channel_serials = {} + for channel in client.channels: + if channel.state in (ChannelState.ATTACHED, ChannelState.ATTACHING): + channel_serials[channel.name] = channel._RealtimeChannel__channel_serial + return json.dumps({ + 'connectionKey': connection_key(client), + 'msgSerial': manager.msg_serial, + 'channelSerials': channel_serials, + }) + + +def record_states(emitter): + """Records every state `emitter` enters from now on, as the specifications' `state_changes`. + + Returned as a plain list of states in the order they were entered, which is what + `CONTAINS`, `CONTAINS_IN_ORDER` and `indexOf` in the assertions read. + """ + states = [] + + def on_state_change(change): + states.append(change.current) + + emitter.on(on_state_change) + return states + + +async def await_recorded_state(states, state, count=1, timeout=CONNECT_TIMEOUT): + """This file's `AWAIT_STATE`: waits until `states` holds `count` entries of `state`. + + Waiting on the recorded list rather than on the live state is what makes a transient + state waitable, and `count` distinguishes the reconnection a test is waiting for from + the CONNECTED the client already reached once. A timeout names the states that were + recorded instead, which is the first thing worth knowing when one of these fails. + """ + try: + await wall_clock_poll_until( + lambda: states.count(state) >= count, + timeout=timeout, + description=f'{count} state change(s) to {state.value}', + interval=STATE_POLL_INTERVAL, + ) + except AssertionError as error: + recorded = [recorded_state.value for recorded_state in states] + raise AssertionError(f'{error}; the states recorded were {recorded}') from None + + +def contains_in_order(states, expected): + """Whether `expected` appears in `states` in order, as the specifications' `CONTAINS_IN_ORDER`.""" + remaining = list(expected) + for state in states: + if remaining and state == remaining[0]: + remaining.pop(0) + return not remaining + + +def ws_connects(log): + """The `ws_connect` events the proxy recorded, in the order the client made them.""" + return [event for event in log if event['type'] == 'ws_connect'] + + +def resume_param(ws_connect): + """The `resume` query parameter of a `ws_connect` event, or `None` where it carried none.""" + return (ws_connect.get('queryParams') or {}).get('resume') + + +def recover_param(ws_connect): + """The `recover` query parameter of a `ws_connect` event, or `None` where it carried none.""" + return (ws_connect.get('queryParams') or {}).get('recover') + + +def frames(log, direction, action): + """The logged WebSocket frames of one action travelling one way. + + A frame is a `ws_frame` event carrying `direction` and the decoded `message`, whose + `action` is an integer. The specification's `ws_frame_to_server` and its `action == + "MESSAGE"` name the same two things the log spells this way. + """ + return [event for event in log + if event['type'] == 'ws_frame' + and event.get('direction') == direction + and (event.get('message') or {}).get('action') == action] + + +# UTS: realtime/proxy/RTN15a/disconnect-triggers-resume-0 +async def test_rtn15a_disconnect_triggers_resume(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'delay_after_ws_connect', 'delayMs': 1000}, + 'action': {'type': 'close'}, + 'times': 1, + 'comment': 'RTN15a: Close WebSocket after 1s to trigger unexpected disconnect', + }]) + + client = proxy_realtime_client(session, realtime_sandbox.key_str) + + # Register state listener BEFORE connecting so we capture all state transitions + state_changes = record_states(client.connection) + + client.connect() + + # Wait for first connected (rule fires after 1s, then proxy closes connection) + # SDK should reconnect and resume + await await_recorded_state(state_changes, ConnectionState.CONNECTED, count=2, timeout=30.0) + + # State changes should include: connecting, connected, disconnected, connecting, connected + disconnected_index = state_changes.index(ConnectionState.DISCONNECTED) + assert disconnected_index >= 0 + + # After the disconnected, there should be another connecting and connected + post_disconnect_connecting = state_changes.index(ConnectionState.CONNECTING, disconnected_index) + assert post_disconnect_connecting > disconnected_index + + assert contains_in_order(state_changes, [ + ConnectionState.DISCONNECTED, + ConnectionState.CONNECTING, + ConnectionState.CONNECTED, + ]) + + # Verify resume was attempted via proxy log + log = await session.get_log() + connects = ws_connects(log) + assert len(connects) >= 2 + + # Second WebSocket connection should include resume query parameter + assert resume_param(connects[1]) is not None + + +# UTS: realtime/proxy/RTN15a/tcp-close-triggers-resume-1 +async def test_rtn15a_tcp_close_triggers_resume(realtime_sandbox, proxy_session): + # `disconnect` drops the TCP connection without a WebSocket close frame, where the test + # above sends one. The client detects the FIN and reports DISCONNECTED just as quickly. + session = await proxy_session(rules=[{ + 'match': {'type': 'delay_after_ws_connect', 'delayMs': 1000}, + 'action': {'type': 'disconnect'}, + 'times': 1, + 'comment': 'RTN15a: Close TCP (no close frame) after 1s to trigger unexpected disconnect', + }]) + + client = proxy_realtime_client(session, realtime_sandbox.key_str) + + state_changes = record_states(client.connection) + + client.connect() + + await await_recorded_state(state_changes, ConnectionState.CONNECTED, count=2, timeout=30.0) + + disconnected_index = state_changes.index(ConnectionState.DISCONNECTED) + assert disconnected_index >= 0 + + post_disconnect_connecting = state_changes.index(ConnectionState.CONNECTING, disconnected_index) + assert post_disconnect_connecting > disconnected_index + + assert contains_in_order(state_changes, [ + ConnectionState.DISCONNECTED, + ConnectionState.CONNECTING, + ConnectionState.CONNECTED, + ]) + + log = await session.get_log() + connects = ws_connects(log) + assert len(connects) >= 2 + + assert resume_param(connects[1]) is not None + + +# UTS: realtime/proxy/RTN15b/resume-preserves-connid-0 +async def test_rtn15b_resume_preserves_connid(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'delay_after_ws_connect', 'delayMs': 1000}, + 'action': {'type': 'close'}, + 'times': 1, + 'comment': 'RTN15b/c6: Close WebSocket after 1s to trigger resume', + }]) + + client = proxy_realtime_client(session, realtime_sandbox.key_str) + state_changes = record_states(client.connection) + + client.connect() + await await_recorded_state(state_changes, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # Record connection identity before disconnect + original_connection_id = connection_id(client) + original_connection_key = connection_key(client) + assert original_connection_id is not None + assert original_connection_key is not None + + # Proxy closes connection after 1s; wait for disconnected then reconnected + await await_recorded_state(state_changes, ConnectionState.DISCONNECTED, timeout=DISCONNECT_TIMEOUT) + + # Wait for SDK to resume + await await_recorded_state(state_changes, ConnectionState.CONNECTED, count=2, timeout=RECONNECT_TIMEOUT) + + # RTN15c6: Connection ID is preserved (successful resume) + assert connection_id(client) == original_connection_id + + # RTN15b: Second ws_connect URL includes resume={connectionKey} + log = await session.get_log() + connects = ws_connects(log) + assert len(connects) >= 2 + assert resume_param(connects[1]) == original_connection_key + + # No error reason on successful resume + assert client.connection.error_reason is None + + +# UTS: realtime/proxy/RTN15c7/failed-resume-new-connid-0 +async def test_rtn15c7_failed_resume_new_connid(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[ + { + 'match': {'type': 'delay_after_ws_connect', 'delayMs': 1000}, + 'action': {'type': 'close'}, + 'times': 1, + 'comment': 'RTN15c7: Close WebSocket after 1s to trigger resume attempt', + }, + { + 'match': {'type': 'ws_frame_to_client', 'action': 'CONNECTED', 'count': 2}, + 'action': { + 'type': 'replace', + 'message': { + 'action': 4, + 'connectionId': 'proxy-injected-new-id', + 'connectionKey': 'proxy-injected-new-key', + 'connectionDetails': { + 'connectionKey': 'proxy-injected-new-key', + 'clientId': None, + 'maxMessageSize': 65536, + 'maxInboundRate': 250, + 'maxOutboundRate': 100, + 'maxFrameSize': 524288, + 'serverId': 'test-server', + 'connectionStateTtl': 120000, + 'maxIdleInterval': 15000, + }, + 'error': { + 'code': 80008, + 'statusCode': 400, + 'message': 'Unable to recover connection', + }, + }, + }, + 'times': 1, + 'comment': 'RTN15c7: Replace 2nd CONNECTED with failed resume (different connectionId + error 80008)', + }, + ]) + + client = proxy_realtime_client(session, realtime_sandbox.key_str) + state_changes = record_states(client.connection) + + # Connect through proxy -- first CONNECTED passes through normally + client.connect() + await await_recorded_state(state_changes, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # Record original identity + original_connection_id = connection_id(client) + assert original_connection_id is not None + assert original_connection_id != 'proxy-injected-new-id' + + # Proxy closes connection after 1s; the SDK reconnects, but the proxy replaces the + # CONNECTED response with a new connectionId, so the SDK reaches CONNECTED with the + # new identity. + await await_recorded_state(state_changes, ConnectionState.DISCONNECTED, timeout=DISCONNECT_TIMEOUT) + await await_recorded_state(state_changes, ConnectionState.CONNECTED, count=2, timeout=RECONNECT_TIMEOUT) + + # RTN15c7: Connection ID changed (resume failed, got new connection) + assert connection_id(client) == 'proxy-injected-new-id' + assert connection_id(client) != original_connection_id + + # Connection key updated to the new one + assert connection_key(client) == 'proxy-injected-new-key' + + # Error reason is set indicating why resume failed + assert client.connection.error_reason is not None + assert client.connection.error_reason.code == 80008 + + # Connection is still CONNECTED (not FAILED -- the server gave a new connection) + assert client.connection.state == ConnectionState.CONNECTED + + # Verify resume was attempted in the proxy log + log = await session.get_log() + connects = ws_connects(log) + assert len(connects) >= 2 + assert resume_param(connects[1]) is not None + + +# UTS: realtime/proxy/RTN15h1/token-error-nonrenewable-failed-0 +async def test_rtn15h1_token_error_nonrenewable_failed(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'delay_after_ws_connect', 'delayMs': 1000}, + 'action': { + 'type': 'inject_to_client_and_close', + 'message': { + 'action': 6, + 'error': { + 'code': 40142, + 'statusCode': 401, + 'message': 'Token expired', + }, + }, + }, + 'times': 1, + 'comment': 'RTN15h1: Inject DISCONNECTED with token error (40142) after 1s', + }]) + + # Obtain a real token from the sandbox so the initial connection succeeds. The Rest + # client asking for it is aimed straight at the sandbox, so the request does not cross + # the session and leaves nothing in its event log. + rest = sandbox_rest_client(realtime_sandbox.key_str) + token_details = await rest.auth.request_token() + + # Use the token string directly -- no key, no authCallback. This makes the token + # non-renewable. + client = sandbox_realtime_client( + token=token_details.token, + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + ) + connect_states = record_states(client.connection) + + # Connect through proxy -- initial connection succeeds with the real token + client.connect() + await await_recorded_state(connect_states, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # Record state changes + state_changes = record_states(client.connection) + + # After 1s the proxy injects DISCONNECTED with 40142 and closes the socket. + # The SDK has a non-renewable token, so it cannot renew -> FAILED. + await await_recorded_state(state_changes, ConnectionState.FAILED, timeout=FAILED_TIMEOUT) + + # RTN15h1: Ended in FAILED state + assert client.connection.state == ConnectionState.FAILED + + # Error reason reflects the token error. The SDK detects it has no means to renew and + # substitutes 40171 for the injected 40142, which is what the specification asks for. + error = client.connection.error_reason + assert error is not None + assert error.code == 40171 + + # UTS SPEC ERROR: the specification asserts statusCode 401 here, citing ably-js. ably-js + # pairs 40171 with statusCode 403 (`src/common/lib/client/auth.ts`, the ErrorInfo thrown + # when authOptions offer no way to request a token), and so does ably-python + # (`ably/rest/auth.py:200`). 403 is what both libraries report. + assert error.status_code == 403 + + # State changes should show the transition to FAILED + # (may pass through DISCONNECTED briefly before FAILED) + assert ConnectionState.FAILED in state_changes + + +# UTS: realtime/proxy/RTN15h3/non-token-error-reconnects-0 +@deviation +async def test_rtn15h3_non_token_error_reconnects(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'delay_after_ws_connect', 'delayMs': 1000}, + 'action': { + 'type': 'inject_to_client_and_close', + 'message': { + 'action': 6, + 'error': { + 'code': 80003, + 'statusCode': 500, + 'message': 'Service temporarily unavailable', + }, + }, + }, + 'times': 1, + 'comment': 'RTN15h3: Inject DISCONNECTED with non-token error (80003) after 1s, once only', + }]) + + client = proxy_realtime_client(session, realtime_sandbox.key_str) + connect_states = record_states(client.connection) + + client.connect() + await await_recorded_state(connect_states, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # Record state changes + state_changes = record_states(client.connection) + + # After 1s the proxy injects DISCONNECTED with non-token error and closes. + # The rule fires once, so the reconnection attempt passes through to the real server. + + # Wait for DISCONNECTED (from the injected message) + await await_recorded_state(state_changes, ConnectionState.DISCONNECTED, timeout=DISCONNECT_TIMEOUT) + + # SDK should automatically reconnect + await await_recorded_state(state_changes, ConnectionState.CONNECTED, timeout=RECONNECT_TIMEOUT) + + # RTN15h3: SDK reconnected successfully (not FAILED) + assert client.connection.state == ConnectionState.CONNECTED + + # State changes should show: disconnected -> connecting -> connected + assert contains_in_order(state_changes, [ + ConnectionState.DISCONNECTED, + ConnectionState.CONNECTING, + ConnectionState.CONNECTED, + ]) + + # Verify resume was attempted + log = await session.get_log() + connects = ws_connects(log) + assert len(connects) >= 2 + assert resume_param(connects[1]) is not None + + # No error reason after successful reconnection + assert client.connection.error_reason is None + + +# UTS: realtime/proxy/RTN15j/fatal-error-established-conn-0 +@deviation +async def test_rtn15j_fatal_error_established_conn(realtime_sandbox, proxy_session): + # No rules: the ERROR is injected imperatively once the connection and both channels + # are up. + session = await proxy_session(rules=[]) + + client = proxy_realtime_client(session, realtime_sandbox.key_str) + connect_states = record_states(client.connection) + + client.connect() + await await_recorded_state(connect_states, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # Attach two channels in parallel + channel_a = client.channels.get(f'fatal-error-a-{random_id()}') + channel_b = client.channels.get(f'fatal-error-b-{random_id()}') + await asyncio.gather(channel_a.attach(), channel_b.attach()) + + # Record state changes + connection_state_changes = record_states(client.connection) + channel_a_state_changes = record_states(channel_a) + channel_b_state_changes = record_states(channel_b) + + # Inject a connection-level ERROR via proxy imperative action + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': { + 'action': 9, + 'error': { + 'code': 50000, + 'statusCode': 500, + 'message': 'Internal server error', + }, + }, + }) + + # SDK should transition to FAILED + await await_recorded_state(connection_state_changes, ConnectionState.FAILED, timeout=FAILED_TIMEOUT) + + # RTN15j: Connection is in FAILED state + assert client.connection.state == ConnectionState.FAILED + + # Connection errorReason has the injected error + assert client.connection.error_reason is not None + assert client.connection.error_reason.code == 50000 + assert client.connection.error_reason.status_code == 500 + + # Both channels transitioned to FAILED + assert channel_a.state == ChannelState.FAILED + assert channel_b.state == ChannelState.FAILED + + # Channel errors match the connection error + assert channel_a.error_reason is not None + assert channel_a.error_reason.code == 50000 + assert channel_b.error_reason is not None + assert channel_b.error_reason.code == 50000 + + # State change sequences + assert ConnectionState.FAILED in connection_state_changes + assert ChannelState.FAILED in channel_a_state_changes + assert ChannelState.FAILED in channel_b_state_changes + + # No reconnection attempted -- only the original ws_connect in the proxy log + log = await session.get_log() + assert len(ws_connects(log)) == 1 + + +# UTS: realtime/proxy/RTN14h/resume-after-ttl-expiry-0 +@deviation +async def test_rtn14h_resume_after_ttl_expiry(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[ + { + 'match': {'type': 'ws_frame_to_client', 'action': 'CONNECTED', 'count': 1}, + 'action': { + 'type': 'replace', + 'message': { + 'action': 4, + 'connectionId': 'proxy-ttl-test-id', + 'connectionKey': '__PASSTHROUGH__', + 'connectionDetails': { + 'connectionKey': '__PASSTHROUGH__', + 'clientId': None, + 'maxMessageSize': 65536, + 'maxInboundRate': 250, + 'maxOutboundRate': 100, + 'maxFrameSize': 524288, + 'serverId': 'test-server', + 'connectionStateTtl': 2000, + 'maxIdleInterval': 15000, + }, + }, + }, + 'times': 1, + 'comment': 'RTN14h: Replace 1st CONNECTED to set short connectionStateTtl (2s) and known connectionId', + }, + { + 'match': {'type': 'delay_after_ws_connect', 'delayMs': 1000}, + 'action': {'type': 'close'}, + 'times': 1, + 'comment': 'RTN14h: Close connection after 1s -- client enters DISCONNECTED with 2s TTL', + }, + { + 'match': {'type': 'ws_connect', 'count': 2}, + 'action': {'type': 'refuse_connection'}, + 'times': 1, + 'comment': 'RTN14h: Refuse 2nd ws_connect -- keeps client disconnected until TTL expires', + }, + ]) + + # A short `suspended_retry_timeout` so the test does not wait long after SUSPENDED. + client = proxy_realtime_client(session, realtime_sandbox.key_str, suspended_retry_timeout=1000) + state_changes = record_states(client.connection) + + # Connect through proxy -- first CONNECTED is replaced with short TTL and known connectionId + client.connect() + await await_recorded_state(state_changes, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # Verify proxy-injected connectionId + assert connection_id(client) == 'proxy-ttl-test-id' + + # T=1s: proxy closes connection -> DISCONNECTED + # T=1-3s: retry attempt is refused -> stays DISCONNECTED + # T=3s: connectionStateTtl(2s) expires -> SUSPENDED + # T=4s: suspendedRetryTimeout(1s) fires -> ws_connect that still attempts a resume + # (RTN14h); the server has discarded the state, so it responds with a new + # connectionId -> CONNECTED + await await_recorded_state(state_changes, ConnectionState.SUSPENDED, timeout=SUSPENDED_TIMEOUT) + + # After suspended, SDK makes a fresh connection + await await_recorded_state( + state_changes, ConnectionState.CONNECTED, count=2, timeout=RECONNECT_TIMEOUT) + + # The server discarded the connection state, so the resume failed server-side and the + # connection ID changed. + assert connection_id(client) != 'proxy-ttl-test-id' + + # Verify the proxy log shows at least 3 ws_connects + log = await session.get_log() + connects = ws_connects(log) + assert len(connects) >= 3 + + # First ws_connect: initial -- no resume + assert resume_param(connects[0]) is None + + # RTN14h: the reconnection made after the TTL expired and the connection became + # suspended still attempts a resume, so it carries the resume query param. + assert resume_param(connects[-1]) is not None + + +# UTS: realtime/proxy/RTN19a/unacked-resent-on-resume-0 +async def test_rtn19a_unacked_resent_on_resume(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': 'ACK'}, + 'action': {'type': 'suppress'}, + 'times': 1, + 'comment': 'RTN19a: Suppress the first ACK so the SDK has a pending unacked message', + }]) + + client = proxy_realtime_client(session, realtime_sandbox.key_str) + state_changes = record_states(client.connection) + + client.connect() + await await_recorded_state(state_changes, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # One sandbox app serves the whole realtime integration tier, so the specification's + # fixed channel name takes a suffix of its own. + channel = client.channels.get(f'test-resend-unacked-{random_id()}') + await channel.attach() + assert channel.state == ChannelState.ATTACHED + + # Start a publish -- do NOT await it yet. The message is sent to the server, but the + # ACK is suppressed by the proxy rule. + publish_future = asyncio.ensure_future(channel.publish('event', 'test-data')) + + # Poll the proxy log until we can confirm both: + # (a) the MESSAGE frame has been sent client->server (action==15) + # (b) the ACK frame has been suppressed server->client (action==1 with ruleMatched) + # This avoids a fixed sleep and ensures the disconnect fires at the right moment. + async def message_sent_and_ack_suppressed(): + log = await session.get_log() + message_sent = bool(frames(log, 'client_to_server', MESSAGE_ACTION)) + ack_suppressed = any(event.get('ruleMatched') for event in frames(log, 'server_to_client', ACK_ACTION)) + return message_sent and ack_suppressed + + await wall_clock_poll_until( + message_sent_and_ack_suppressed, + timeout=DISCONNECT_TIMEOUT, + description='the MESSAGE to be sent and its ACK to be suppressed') + + # Close the connection -- the SDK has an unacked message pending + await session.trigger_action({'type': 'close'}) + + # SDK reconnects and resumes (the ACK suppression rule already fired once, so the + # reconnected session passes ACKs through normally) + await await_recorded_state( + state_changes, ConnectionState.CONNECTED, count=2, timeout=RECONNECT_TIMEOUT) + + # Now await the publish -- it should complete successfully after the message is resent + # on the new transport and ACKed. The publish completed: no exception raised. + await asyncio.wait_for(asyncio.shield(publish_future), RECONNECT_TIMEOUT) + assert publish_future.done() + assert publish_future.exception() is None + + # Verify resume occurred + log = await session.get_log() + connects = ws_connects(log) + assert len(connects) >= 2 + assert resume_param(connects[1]) is not None + + # RTN19a: The MESSAGE frame was sent on both transports (original + resend) + message_frames = frames(log, 'client_to_server', MESSAGE_ACTION) + assert len(message_frames) >= 2 + + # RTN19a2: On successful resume, the resent message has the same msgSerial + assert message_frames[0]['message']['msgSerial'] == message_frames[1]['message']['msgSerial'] + + +# UTS: realtime/proxy/RTN16d/recovery-preserves-connid-0 +@deviation +async def test_rtn16d_recovery_preserves_connid(realtime_sandbox, proxy_session): + # A session each: the first establishes the connection whose recovery key is taken, the + # second carries the recovering client so its `recover` query parameter can be read off + # a log of its own. + session_1 = await proxy_session(rules=[]) + session_2 = await proxy_session(rules=[]) + + client_1 = proxy_realtime_client(session_1, realtime_sandbox.key_str) + client_1_states = record_states(client_1.connection) + + # --- Phase 1: Obtain recovery key from first client --- + + client_1.connect() + await await_recorded_state(client_1_states, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + original_connection_id = connection_id(client_1) + original_connection_key = connection_key(client_1) + assert original_connection_id is not None + + # Attach a channel so it appears in the recovery key + channel_1 = client_1.channels.get(f'recovery-test-{random_id()}') + await channel_1.attach() + assert channel_1.state == ChannelState.ATTACHED + + # Get the recovery key + key = recovery_key(client_1) + assert key is not None + + # Close the first client's transport WITHOUT closing the Ably connection gracefully, so + # that the server keeps the connection state alive for recovery. + await session_1.trigger_action({'type': 'close'}) + + # Wait for the client to detect the disconnect + await await_recorded_state(client_1_states, ConnectionState.DISCONNECTED, timeout=DISCONNECT_TIMEOUT) + + # Close client_1 without allowing it to reconnect + await client_1.close() + assert client_1.connection.state == ConnectionState.CLOSED + await session_1.close() + + # --- Phase 2: Recover using the recovery key --- + + client_2 = proxy_realtime_client(session_2, realtime_sandbox.key_str, recover=key) + client_2_states = record_states(client_2.connection) + + client_2.connect() + await await_recorded_state(client_2_states, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # RTN16d: Connection ID is preserved (same as original connection) + assert connection_id(client_2) == original_connection_id + + # RTN16d: Connection key is updated (new key from server) + assert connection_key(client_2) is not None + assert connection_key(client_2) != original_connection_key + + # RTN16k: Verify the recover query parameter was sent via proxy log + log = await session_2.get_log() + connects = ws_connects(log) + assert len(connects) >= 1 + assert recover_param(connects[0]) == original_connection_key + + # No resume param (this is recovery, not resume) + assert resume_param(connects[0]) is None + + # No error on successful recovery + assert client_2.connection.error_reason is None + + +# UTS: realtime/proxy/RTN16l/recovery-failure-fresh-conn-0 +@deviation +async def test_rtn16l_recovery_failure_fresh_conn(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'ws_frame_to_client', 'action': 'CONNECTED', 'count': 1}, + 'action': { + 'type': 'replace', + 'message': { + 'action': 4, + 'connectionId': 'recovery-failed-new-id', + 'connectionKey': 'recovery-failed-new-key', + 'connectionDetails': { + 'connectionKey': 'recovery-failed-new-key', + 'clientId': None, + 'maxMessageSize': 65536, + 'maxInboundRate': 250, + 'maxOutboundRate': 100, + 'maxFrameSize': 524288, + 'serverId': 'test-server', + 'connectionStateTtl': 120000, + 'maxIdleInterval': 15000, + }, + 'error': { + 'code': 80008, + 'statusCode': 400, + 'message': 'Unable to recover connection', + }, + }, + }, + 'times': 1, + 'comment': 'RTN16l: Replace CONNECTED with recovery failure (new connectionId + error 80008)', + }]) + + # A fabricated recovery key. The connectionKey does not need to be valid, since the + # proxy replaces the server's response anyway. + fabricated_recovery_key = json.dumps({ + 'connectionKey': 'stale-old-key', + 'msgSerial': 99, + 'channelSerials': { + 'old-channel': 'old-serial', + }, + }) + + client = proxy_realtime_client(session, realtime_sandbox.key_str, recover=fabricated_recovery_key) + state_changes = record_states(client.connection) + + # Connect with the fabricated recovery key + client.connect() + await await_recorded_state(state_changes, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # RTN16l + RTN15c7: Connection got a new ID (recovery failed) + assert connection_id(client) == 'recovery-failed-new-id' + assert connection_key(client) == 'recovery-failed-new-key' + + # RTN15c7: Error is set on the connection indicating recovery failure + assert client.connection.error_reason is not None + assert client.connection.error_reason.code == 80008 + + # Connection is still CONNECTED (not FAILED -- the server gave a new connection) + assert client.connection.state == ConnectionState.CONNECTED + + # Verify the recover param was sent via proxy log + log = await session.get_log() + connects = ws_connects(log) + assert len(connects) >= 1 + assert recover_param(connects[0]) == 'stale-old-key' diff --git a/test/uts/realtime/integration/proxy/heartbeat_test.py b/test/uts/realtime/integration/proxy/heartbeat_test.py new file mode 100644 index 00000000..b9602be6 --- /dev/null +++ b/test/uts/realtime/integration/proxy/heartbeat_test.py @@ -0,0 +1,153 @@ +"""Derived from uts/realtime/integration/proxy/heartbeat.md in ably/specification. + +Spec points: RTN23a + +RTN23a says a transport that has heard nothing for `maxIdleInterval + +realtimeRequestTimeout` is to be treated as dropped. This test does not wait that +interval out: the rule is `delay_after_ws_connect` at 2000 ms followed by `close`, +so the proxy sends a WebSocket close frame two seconds into the connection and the +SDK reacts to the close rather than to an expired idle timer. The sandbox advertises +`maxIdleInterval: 15000` in the CONNECTED frame — read off `connectionDetails` in +the proxy event log while this was derived — so the close lands well inside the +window the idle timer would have measured, and the reconnection the assertions look +for can only be the close frame's doing. That is also why the session needs no +`timeout_ms` of its own: the whole test runs in three or four seconds, nowhere near +the harness's 120-second idle limit. + +`test/uts/realtime/unit/connection/heartbeat_test.py` covers RTN23a — and RTN23b, +RTN23c and RTN23c1 alongside it — against `MockWebSocket`, and is where the idle +timer itself is pinned: it scales `maxIdleInterval` down to 200 ms and shows that a +HEARTBEAT, a MESSAGE or an ACK each reset the timer while silence does not. None of +that is observable here, where the interval is the server's and the timer never +expires. What this test adds is the recovery: a real transport goes away mid- +connection, and the SDK opens a second real WebSocket carrying the first +connection's key as its `resume` parameter, which the proxy records. + +The `times: 1` on the rule is load-bearing — the second connection has no close +waiting for it, so the reconnection settles instead of cycling. +""" + +from ably.realtime.connection import ConnectionState +from test.uts.helpers.client import await_connection_state, sandbox_realtime_client, wall_clock_poll_until +from test.uts.helpers.mock_websocket import contains_in_order +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt + +# The specification's `AWAIT_STATE ... WITH timeout`, in seconds. +CONNECT_TIMEOUT = 15 +RECONNECT_TIMEOUT = 30 + +# The specification's `delayMs: 2000`. +CLOSE_DELAY_MS = 2000 + +# The specification's `state_changes CONTAINS_IN_ORDER [...]`: one connection lost +# and replaced. +FULL_CYCLE = [ + ConnectionState.CONNECTING, + ConnectionState.CONNECTED, + ConnectionState.DISCONNECTED, + ConnectionState.CONNECTING, + ConnectionState.CONNECTED, +] + + +def connection_id(client): + """The specification's `client.connection.id`, which `Connection` does not expose. + + See the house ruling on missing accessors in `test/uts/deviations.md`. + """ + return client.connection.connection_manager.connection_id + + +def connection_key(client): + """The specification's `client.connection.key`, which arrives in `connectionDetails`.""" + details = client.connection.connection_details + return details.connection_key if details is not None else None + + +def record_states(client): + """The specification's `state_changes`, filled from `client.connection.on(...)`.""" + states = [] + + def on_change(change): + states.append(change.current) + + client.connection.on(on_change) + return states + + +def jwt_auth_callback(api_key): + """The specification's `authCallback` returning `generateJWT({keyName, keySecret})`. + + A JWT is signed from the two halves of the sandbox key, so the callback reaches + no network at all and adds nothing to the session's event log. + """ + key_name = extract_key_name(api_key) + key_secret = extract_key_secret(api_key) + + async def auth_callback(params): + return generate_jwt(key_name, key_secret) + + return auth_callback + + +# UTS: realtime/proxy/RTN23a/heartbeat-starvation-reconnect-0 +async def test_rtn23a_heartbeat_starvation_reconnect(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'delay_after_ws_connect', 'delayMs': CLOSE_DELAY_MS}, + 'action': {'type': 'close'}, + 'times': 1, + 'comment': 'RTN23a: Close WebSocket after 2s to simulate transport failure', + }]) + + client = sandbox_realtime_client( + auth_callback=jwt_auth_callback(realtime_sandbox.key_str), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + ) + state_changes = record_states(client) + + client.connect() + + await await_connection_state(client, ConnectionState.CONNECTED, timeout=CONNECT_TIMEOUT) + + # Capture connection details from the first connection + first_connection_id = connection_id(client) + first_connection_key = connection_key(client) + assert first_connection_id is not None + + # RTN15a has the SDK retry a drop from CONNECTED on the next loop iteration + # rather than after a timer, so DISCONNECTED is gone again before anything can + # observe the connection resting in it. Both waits below are therefore made + # against the recorded sequence, which keeps every state the connection passed + # through, rather than against the state the connection currently holds. + await wall_clock_poll_until( + lambda: ConnectionState.DISCONNECTED in state_changes, + timeout=CONNECT_TIMEOUT, + description='the connection to report DISCONNECTED') + + await wall_clock_poll_until( + lambda: contains_in_order(state_changes, FULL_CYCLE), + timeout=RECONNECT_TIMEOUT, + description='the connection to be re-established') + + # Connection is re-established with new connection details + assert client.connection.state == ConnectionState.CONNECTED + assert connection_id(client) is not None + assert connection_key(client) is not None + + # State sequence shows: connected -> disconnected -> reconnecting -> connected + assert contains_in_order(state_changes, FULL_CYCLE) + + # Proxy event log confirms two WebSocket connections + log = await session.get_log() + ws_connects = [event for event in log if event['type'] == 'ws_connect'] + assert len(ws_connects) >= 2 + + # Second connection should include resume parameter (RTN15c) + assert ws_connects[1]['queryParams'].get('resume') is not None + # The resume parameter is the first connection's key, which is how the key the + # specification reads off `connection.key` is observable at all here + assert ws_connects[1]['queryParams']['resume'] == first_connection_key diff --git a/test/uts/realtime/integration/proxy/presence_reentry_test.py b/test/uts/realtime/integration/proxy/presence_reentry_test.py new file mode 100644 index 00000000..9e20a4c0 --- /dev/null +++ b/test/uts/realtime/integration/proxy/presence_reentry_test.py @@ -0,0 +1,253 @@ +"""Derived from uts/realtime/integration/proxy/presence_reentry.md in ably/specification. + +Spec points: RTP17i, RTP17g + +A member entered on a connection belongs to that connection, so re-entry is what keeps +it present once the channel attaches afresh. Both tests enter one member through a +`uts-proxy` session and then read the wire: a re-entry is a PRESENCE frame +(`action == 14`) from the client carrying an ENTER for the member, and the event log +records every one of them. Nothing here reads the SDK's internal presence map, and +neither test uses a second observer client — the server does not broadcast a re-entry +whose member, as far as it is concerned, never left. + +`test/uts/realtime/unit/presence/realtime_presence_reentry_test.py` covers RTP17 against +a mock, where the reconnection is a dropped transport and a server that plays the +member back. These two differ in what triggers the re-attach and in what is measured: +the first injects an ATTACHED onto a channel that is already attached, which the mock +tier does not exercise, and the second drives a real WebSocket close from the proxy so +that the re-attach and the re-entry both complete against the sandbox. Where the unit +tests assert on the frames a mock server captured, these assert on the proxy's log. + +Test 27 is gated: ably-python re-enters from `RealtimePresence.on_attached`, which runs +only when the channel transitions into ATTACHED, so an ATTACHED arriving on an +already-attached channel takes the RTL12 update path and re-enters nothing. Test 28 +goes through a genuine ATTACHING and passes. +""" + +from datetime import datetime + +from ably.realtime.connection import ConnectionState +from ably.types.channelstate import ChannelState +from test.uts.helpers.client import ( + await_channel_state, + await_connection_state, + sandbox_realtime_client, + wall_clock_poll_until, +) +from test.uts.helpers.deviations import deviation +from test.uts.helpers.sandbox import extract_key_name, extract_key_secret, generate_jwt, random_id + +# The specification's `AWAIT_STATE ... WITH timeout` and `POLL_UNTIL` values, as +# wall-clock seconds, and the interval it polls the event log on. +CONNECT_TIMEOUT = 15.0 +DISCONNECT_TIMEOUT = 10.0 +CHANNEL_TIMEOUT = 15.0 +POLL_TIMEOUT = 10.0 +POLL_INTERVAL = 0.2 + +# PRESENCE and ENTER, as the event log reports the `action` of a protocol message and of +# the presence messages inside it. +PRESENCE_ACTION = 14 +ENTER_ACTION = 2 + + +def jwt_auth_callback(api_key, client_id): + """The specification's `authCallback`, signing an Ably JWT carrying `clientId`. + + Presence needs an identity, and the specification gives the client one through the + JWT's `clientId` claim rather than through `ClientOptions.clientId`, so that the + identity the member is entered under is the one the token grants. + """ + async def auth_callback(params): + return generate_jwt( + extract_key_name(api_key), extract_key_secret(api_key), client_id=client_id) + + return auth_callback + + +def proxied_client(api_key, session, client_id): + """The `ClientOptions` both tests build their presence member with.""" + return sandbox_realtime_client( + auth_callback=jwt_auth_callback(api_key, client_id), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False) + + +def presence_frames(log): + """The PRESENCE frames the proxy recorded from the client.""" + return [event for event in log + if event['type'] == 'ws_frame' + and event.get('direction') == 'client_to_server' + and (event.get('message') or {}).get('action') == PRESENCE_ACTION] + + +def event_time(event): + """An event's `timestamp`, as a comparable value. + + The proxy timestamps an event as RFC 3339 with however many fractional digits the + value needs, so `'...:24.46228Z'` and `'...:24.9Z'` do not order correctly as + strings. The fraction is padded out to microseconds and parsed. + """ + stamp = event['timestamp'].rstrip('Z') + seconds, _, fraction = stamp.partition('.') + return datetime.strptime(f'{seconds}.{fraction[:6].ljust(6, "0")}', '%Y-%m-%dT%H:%M:%S.%f') + + +# UTS: realtime/proxy/RTP17i/reenter-on-non-resumed-0 +@deviation +async def test_rtp17i_reenter_on_non_resumed(realtime_sandbox, proxy_session): + channel_name = f'test-rtp17i-{random_id()}' + + session = await proxy_session(rules=[]) + + client = proxied_client(realtime_sandbox.key_str, session, 'client-a') + channel = client.channels.get(channel_name) + + # Phase 1 -- Establish real presence state + client.connect() + await await_connection_state(client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + await channel.attach() + await channel.presence.enter(data='hello') + + # Phase 2 -- Count PRESENCE frames in the log before injection + log_before = await session.get_log() + presence_frames_before = len(presence_frames(log_before)) + + # Phase 3 -- Inject ATTACHED with resumed=false (flags=0). This triggers RTP17i + # re-entry without needing an actual disconnect. + await session.trigger_action({ + 'type': 'inject_to_client', + 'message': { + 'action': 11, + 'channel': channel_name, + 'flags': 0, + 'error': {'code': 91001, 'statusCode': 500, 'message': 'Continuity lost'}, + }, + }) + + # Phase 4 -- Poll until a new PRESENCE frame appears in the log + async def a_new_presence_frame(): + return len(presence_frames(await session.get_log())) > presence_frames_before + + await wall_clock_poll_until( + a_new_presence_frame, POLL_TIMEOUT, 'a re-enter PRESENCE frame', POLL_INTERVAL) + + log_after = await session.get_log() + all_presence_frames = presence_frames(log_after) + + # At least one new PRESENCE frame was sent after the injection + assert len(all_presence_frames) > presence_frames_before + + # The last (most recent) re-enter frame should contain the presence data + reenter_frame = all_presence_frames[-1] + assert reenter_frame['message'].get('presence') is not None + assert len(reenter_frame['message']['presence']) >= 1 + + # RTP17g: re-enter uses stored clientId, data, and ENTER action + reenter_msg = reenter_frame['message']['presence'][0] + assert reenter_msg['clientId'] == 'client-a' + assert reenter_msg['data'] == 'hello' + assert reenter_msg['action'] == ENTER_ACTION + + # Channel should still be attached and connection still connected + assert channel.state == ChannelState.ATTACHED + assert client.connection.state == ConnectionState.CONNECTED + + +# UTS: realtime/proxy/RTP17i/reenter-after-disconnect-1 +async def test_rtp17i_reenter_after_disconnect(realtime_sandbox, proxy_session): + channel_name = f'test-rtp17i-real-{random_id()}' + + session = await proxy_session(rules=[ + { + 'match': {'type': 'delay_after_ws_connect', 'delayMs': 3000}, + 'action': {'type': 'close'}, + 'times': 1, + 'comment': 'RTP17i: Close WebSocket after 3s to trigger reconnect', + }, + { + 'match': {'type': 'ws_frame_to_client', 'action': 'ATTACHED', + 'channel': channel_name, 'count': 2}, + 'action': { + 'type': 'replace', + 'message': { + 'action': 11, + 'channel': channel_name, + 'flags': 0, + 'error': {'code': 91001, 'statusCode': 500, 'message': 'Continuity lost'}, + }, + }, + 'times': 1, + 'comment': 'RTP17i: Replace 2nd ATTACHED with non-resumed to trigger re-entry', + }, + ]) + + client_a = proxied_client(realtime_sandbox.key_str, session, 'client-a') + channel_a = client_a.channels.get(channel_name) + + # The proxy closes the WebSocket 3 seconds after it opens, and the client retries at + # once, so the connection states are recorded from before the connection is made + # rather than waited on one at a time. + connection_state_changes = [] + + def record(change): + connection_state_changes.append(change.current) + + client_a.connection.on(record) + + # Phase 1 -- Establish presence before the proxy closes the connection + client_a.connect() + await await_connection_state(client_a, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + await channel_a.attach() + await channel_a.presence.enter(data='hello') + + # Phase 2 -- Wait for the temporal trigger to fire (at T+3s) and for reconnect + await wall_clock_poll_until( + lambda: ConnectionState.DISCONNECTED in connection_state_changes, + DISCONNECT_TIMEOUT, 'the connection to report DISCONNECTED', POLL_INTERVAL) + await await_connection_state(client_a, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + # Wait for the channel to reattach (the 2nd ATTACHED is replaced with non-resumed) + await await_channel_state(channel_a, ChannelState.ATTACHED, CHANNEL_TIMEOUT) + + # Phase 3 -- Poll until a PRESENCE frame appears in the log after the 2nd ws_connect + async def presence_after_reconnect(): + log = await session.get_log() + ws_connects = [event for event in log if event['type'] == 'ws_connect'] + if len(ws_connects) < 2: + return None + second_connect_time = event_time(ws_connects[1]) + return [event for event in presence_frames(log) + if event_time(event) > second_connect_time] or None + + await wall_clock_poll_until( + presence_after_reconnect, POLL_TIMEOUT, 'a re-enter PRESENCE frame', POLL_INTERVAL) + + log = await session.get_log() + ws_connects = [event for event in log if event['type'] == 'ws_connect'] + second_connect_time = event_time(ws_connects[1]) + + reenter_frames = [event for event in presence_frames(log) + if event_time(event) > second_connect_time] + + assert len(reenter_frames) >= 1 + + # Check the first re-enter frame + reenter_frame = reenter_frames[0] + assert reenter_frame['message'].get('presence') is not None + assert len(reenter_frame['message']['presence']) >= 1 + + # RTP17g: re-enter uses stored clientId, data, and ENTER action + reenter_msg = reenter_frame['message']['presence'][0] + assert reenter_msg['clientId'] == 'client-a' + assert reenter_msg['data'] == 'hello' + assert reenter_msg['action'] == ENTER_ACTION + + # Channel is still attached and connection is still connected + assert channel_a.state == ChannelState.ATTACHED + assert client_a.connection.state == ConnectionState.CONNECTED diff --git a/test/uts/realtime/integration/proxy/rest_faults_test.py b/test/uts/realtime/integration/proxy/rest_faults_test.py new file mode 100644 index 00000000..fa8d2c1a --- /dev/null +++ b/test/uts/realtime/integration/proxy/rest_faults_test.py @@ -0,0 +1,243 @@ +"""Derived from uts/realtime/integration/proxy/rest_faults.md in ably/specification. + +Spec points: RSC10, RSC15m, REC2c2, RTL6 + +These are the HTTP faults a realtime client's REST side meets, so two of the three tests +drive a `Rest` client through the session and the third drives a realtime and a REST +client through the same one. `endpoint='localhost'` disables the fallback hosts by +itself (REC2c2), so a request the proxy faults is not retried anywhere else and the +event log holds exactly the attempts the SDK made. + +Authentication is a callback in every test, because `tls=False` makes the session plain +HTTP and RSC18 has the SDK refuse basic auth over it with 40103 before a request is +written. RSC10 and RSC15m take a token from an inner `Rest` client aimed straight at the +sandbox, so the token request is neither consumed by the waiting rule nor counted among +the requests a test asserts on; RTL6 signs an Ably JWT locally and makes no request at +all. `test/uts/rest/integration/proxy/rest_fallback_test.py` carries the same two +patterns for the REST tier. + +`http_response` events carry a `status` and no path, so "the injected response fired" is +read off the responses in the order the proxy sent them. + +There is no `## Protocol Variants` section, so these run against JSON only, which the +proxy tier requires in any case. +""" + +import pytest + +from ably import AblyRest +from ably.realtime.connection import ConnectionState +from ably.types.channelstate import ChannelState +from ably.util.exceptions import AblyException +from test.uts.helpers.client import ( + await_channel_state, + await_connection_state, + sandbox_realtime_client, + sandbox_rest_client, + wall_clock_poll_until, +) +from test.uts.helpers.sandbox import ( + SANDBOX_ENDPOINT, + extract_key_name, + extract_key_secret, + generate_jwt, + random_id, +) + +# The specification's `AWAIT_STATE ... WITH timeout` and `pollUntil` values, as +# wall-clock seconds. +CONNECT_TIMEOUT = 15.0 +ATTACH_TIMEOUT = 10.0 +HISTORY_TIMEOUT = 10.0 +HISTORY_INTERVAL = 0.5 + + +def token_auth_callback(api_key, invocations=None): + """The specification's `request_token_from_sandbox`, as an `authCallback`. + + Each invocation builds a `Rest` client of its own pointed straight at the sandbox, + asks it for a token and closes it, so the token arrives over a connection the proxy + never sees. A token requested through the client under test would be consumed by the + rule waiting to fault the first matching request, and would show up in the event log + beside the requests a test counts. + + `invocations` is the specification's `auth_callback_count`, appended to once per + call where a test counts renewals. + """ + async def auth_callback(params): + if invocations is not None: + invocations.append(params) + inner_rest = AblyRest(key=api_key, endpoint=SANDBOX_ENDPOINT) + try: + return await inner_rest.auth.request_token() + finally: + await inner_rest.close() + + return auth_callback + + +def jwt_auth_callback(api_key): + """The specification's JWT `authCallback`, which makes no request of its own.""" + async def auth_callback(params): + return generate_jwt(extract_key_name(api_key), extract_key_secret(api_key)) + + return auth_callback + + +def http_requests(log, path): + """The `http_request` events the proxy recorded for `path`.""" + return [event for event in log if event['type'] == 'http_request' and path in event['path']] + + +def http_responses(log): + """The `http_response` events the proxy recorded, in the order it sent them.""" + return [event for event in log if event['type'] == 'http_response'] + + +# UTS: realtime/proxy/RSC10/token-renewal-on-401-0 +async def test_rsc10_token_renewal_on_401(realtime_sandbox, proxy_session): + # Track authCallback invocations + auth_callback_invocations = [] + + session = await proxy_session(rules=[{ + 'match': {'type': 'http_request', 'pathContains': '/channels/'}, + 'action': { + 'type': 'http_respond', + 'status': 401, + 'body': {'error': {'code': 40142, 'statusCode': 401, 'message': 'Token expired'}}, + }, + 'times': 1, + 'comment': 'RSC10: Return 401 on first channel request, then passthrough', + }]) + + client = sandbox_rest_client( + auth_callback=token_auth_callback(realtime_sandbox.key_str, auth_callback_invocations), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + ) + + channel_name = f'test-RSC10-token-renewal-{random_id()}' + channel = client.channels.get(channel_name) + + # Publish a message -- first request gets 401, SDK renews token, retries. + # The publish completed successfully: no error raised. + await channel.publish('test-event', 'hello') + + # authCallback was called at least twice (initial token + renewal after 401) + assert len(auth_callback_invocations) >= 2 + + # Proxy event log shows two HTTP requests to the channel endpoint + log = await session.get_log() + assert len(http_requests(log, '/channels/')) >= 2 + + # First request was intercepted (got 401), second request passed through (got 2xx) + responses = http_responses(log) + assert responses[0]['status'] == 401 + assert responses[1]['status'] in (200, 201) + + +# UTS: realtime/proxy/RSC15m/http-503-no-fallback-0 +async def test_rsc15m_http_503_no_fallback(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'http_request', 'pathContains': '/channels/'}, + 'action': { + 'type': 'http_respond', + 'status': 503, + 'body': {'error': {'code': 50300, 'statusCode': 503, + 'message': 'Service temporarily unavailable'}}, + }, + 'times': 1, + 'comment': 'RSC15m: Return 503 on first channel request', + }]) + + # No fallback_hosts -- endpoint='localhost' disables fallback (REC2c2) + client = sandbox_rest_client( + auth_callback=token_auth_callback(realtime_sandbox.key_str), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + ) + + channel_name = f'test-RSC15m-503-error-{random_id()}' + channel = client.channels.get(channel_name) + + # Try to publish a message -- should fail with 503 error + with pytest.raises(AblyException) as excinfo: + await channel.publish('test-event', 'hello') + + # The error propagates to the caller with the correct error code + assert excinfo.value.code == 50300 + assert excinfo.value.status_code == 503 + + # Proxy event log shows only one HTTP request to the channel endpoint + # (no fallback attempts since endpoint='localhost' disables fallback hosts) + log = await session.get_log() + assert len(http_requests(log, '/channels/')) == 1 + + +# UTS: realtime/proxy/RTL6/publish-history-through-proxy-0 +async def test_rtl6_publish_history_through_proxy(realtime_sandbox, proxy_session): + # A session with no rules: pure passthrough, so what this test shows is that the + # proxy forwards WebSocket and HTTP traffic without interfering with either. + session = await proxy_session(rules=[]) + + realtime_client = sandbox_realtime_client( + auth_callback=jwt_auth_callback(realtime_sandbox.key_str), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + auto_connect=False, + ) + + rest_client = sandbox_rest_client( + auth_callback=jwt_auth_callback(realtime_sandbox.key_str), + endpoint='localhost', + port=session.proxy_port, + tls=False, + use_binary_protocol=False, + ) + + channel_name = f'test-RTL6-publish-history-{random_id()}' + realtime_channel = realtime_client.channels.get(channel_name) + rest_channel = rest_client.channels.get(channel_name) + + # Connect Realtime client through proxy and wait until connected + realtime_client.connect() + await await_connection_state(realtime_client, ConnectionState.CONNECTED, CONNECT_TIMEOUT) + + await realtime_channel.attach() + await await_channel_state(realtime_channel, ChannelState.ATTACHED, ATTACH_TIMEOUT) + + await realtime_channel.publish('test-msg', 'hello world') + + # Poll history via REST until the published message appears. History is eventually + # consistent so a single immediate read may return nothing, and a `PaginatedResult` + # is truthy whether or not it holds anything, so the condition answers None until + # the page has an item. + async def published_history(): + page = await rest_channel.history() + return page if len(page.items) > 0 else None + + history = await wall_clock_poll_until( + published_history, HISTORY_TIMEOUT, 'the published message to reach history', + HISTORY_INTERVAL) + + # History contains the published message + assert len(history.items) >= 1 + + published_msg = next((m for m in history.items if m.name == 'test-msg'), None) + assert published_msg is not None + assert published_msg.data == 'hello world' + + # Proxy event log shows both WebSocket and HTTP traffic + log = await session.get_log() + + # At least one WebSocket connection was made (Realtime client) + assert len([event for event in log if event['type'] == 'ws_connect']) >= 1 + + # At least one HTTP request was made (REST history call) + assert len([event for event in log if event['type'] == 'http_request']) >= 1 From 8d9ccb32d9b1727d4726d17a79db144afdd2342b Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 25 Sep 2026 11:26:35 +0100 Subject: [PATCH 4/5] test: close the connection where the presence specification closes it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `presence.md` closes its realtime client between the presence operations and the REST read in both RSP4 and RSP4b2. The close synthesizes a LEAVE, which becomes the newest event and so the one a backwards read lands on, and the server gives that LEAVE the member's last data — so both specifications' assertions hold with the close where they put it. Co-Authored-By: Claude Opus 5 (1M context) --- test/uts/rest/integration/presence_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/uts/rest/integration/presence_test.py b/test/uts/rest/integration/presence_test.py index 68a69cef..53a6d5dc 100644 --- a/test/uts/rest/integration/presence_test.py +++ b/test/uts/rest/integration/presence_test.py @@ -156,8 +156,7 @@ async def test_rsp4_history_returns_events(sandbox, use_binary_protocol): await realtime_channel.presence.enter('entered') await realtime_channel.presence.update('updated') await realtime_channel.presence.leave('left') - # NOTE: the spec closes the realtime client here. The suite's autouse - # teardown closes every client it built, after the assertions have run. + await realtime.close() rest_channel = client.channels.get(channel_name) @@ -214,6 +213,10 @@ async def test_rsp4b2_history_direction_forwards(sandbox, use_binary_protocol): await realtime_channel.presence.enter('first') await realtime_channel.presence.update('second') await realtime_channel.presence.update('third') + # The close synthesizes a LEAVE, which becomes the newest event and so the one + # the backwards read below lands on. The server gives it the member's last data, + # so the assertion holds either way; see deviations.md. + await realtime.close() rest_channel = client.channels.get(channel_name) await wall_clock_poll_until( From 80e0ade7d5974fee462434314bb9a5c3d8a296a6 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Fri, 25 Sep 2026 11:36:49 +0100 Subject: [PATCH 5/5] docs: record the realtime integration tier and file its faults upstream The four `uts/rest/integration` specification faults are filed as ably/specification#547 to #550 and the eight `uts/realtime/integration` ones as #551 to #554, each cross-referenced from the entry that found it. The realtime integration tier adds four SDK root causes and extends four that the unit tiers already recorded. None of the eight specification faults is gated: every one is a heading, a fixture or a label, so the derived test keeps the corrected fixture and passes. Co-Authored-By: Claude Opus 5 (1M context) --- .claude/skills/uts-to-python/SKILL.md | 120 ++++- test/uts/README.md | 62 ++- test/uts/deviations.md | 608 +++++++++++++++++++++----- 3 files changed, 663 insertions(+), 127 deletions(-) diff --git a/.claude/skills/uts-to-python/SKILL.md b/.claude/skills/uts-to-python/SKILL.md index 1dbeb50e..fa396e17 100644 --- a/.claude/skills/uts-to-python/SKILL.md +++ b/.claude/skills/uts-to-python/SKILL.md @@ -32,12 +32,15 @@ and `uts/rest/integration/history.md` becomes `test/uts/rest/integration/history Every directory needs an `__init__.py`, as `test` is a package. There are two kinds of tier. `rest/unit` and `realtime/unit` serve every request from a -mock and reach no network; `rest/integration` runs against the real Ably sandbox and has -no mock at all. See **The integration tier** below. +mock and reach no network; `rest/integration` and `realtime/integration` run against the +real Ably sandbox and have no mock at all. See **The integration tier** below. `test/uts/rest/unit/time_test.py` is the reference example for REST unit, -`test/uts/realtime/unit/connection/auto_connect_test.py` for realtime, and -`test/uts/rest/integration/history_test.py` for integration. Follow their shape. +`test/uts/realtime/unit/connection/auto_connect_test.py` for realtime unit, +`test/uts/rest/integration/history_test.py` for REST integration and +`test/uts/realtime/integration/channels/channel_publish_test.py` for realtime +integration — two clients, protocol variants, a state wait and a `connection_id` reader, +which is most of what a realtime integration module needs. Follow their shape. ## Anatomy of a derived test @@ -287,8 +290,10 @@ All in `test.uts.helpers.clock`. ## The integration tier -`uts/rest/integration/.md` becomes `test/uts/rest/integration/_test.py` and -runs against the real Ably sandbox — twelve specifications, 84 tests, one of them the +`uts/rest/integration/.md` becomes `test/uts/rest/integration/_test.py`, and +`uts/realtime/integration/.md` becomes +`test/uts/realtime/integration/_test.py`. Both run against the real Ably sandbox — +twelve REST specifications, 84 tests, and twenty realtime ones, 73 tests, each tier with a proxy package the section below covers. There is no mock and no `test_options`: the client reaches the network. `test/uts/README.md` covers the same ground for someone reading the suite; this is what someone writing a test needs. @@ -298,9 +303,11 @@ What differs from the mock-backed tiers: - **Nothing sits in front of the client.** No `install_mock`, no captured request to assert on, no way to make the server answer a chosen way. A spec point that can only be shown through a stubbed response belongs in the unit tier. -- **One sandbox app serves the whole session**, standing in for `BEFORE ALL TESTS`. - So every channel name, client id and device id takes a `random_id()` suffix, and - anything a test registers it removes in a `finally`. +- **One sandbox app serves each tier**, standing in for `BEFORE ALL TESTS` — `sandbox` + for REST, `realtime_sandbox` for realtime, provisioned separately so that neither + tier's channels, presence members or devices are visible to the other. Within a tier + the app is shared, so every channel name, client id and device id takes a + `random_id()` suffix, and anything a test registers it removes in a `finally`. - **Waits are wall-clock**, the inverse of the unit tier's rule. See Timers below. - **The per-test timeout is 120 seconds**, set by the package's own `conftest.py`, not the 30 `pyproject.toml` gives the rest of the suite. @@ -308,8 +315,10 @@ What differs from the mock-backed tiers: | Name | Is | |---|---| | `sandbox` fixture, in `rest/integration/conftest.py` | a specification's `app_config`. Session-scoped: provisioned once from the vendored `assets/test-app-setup.json` and deleted afterwards | +| `realtime_sandbox` fixture, in `realtime/integration/conftest.py` | the same for the realtime tier, and a separate app. Same `key(i)`, `key_str` and `app_id` | | `sandbox.key(i)` | `app_config.keys[i]`, carrying `key_str`, `key_name`, `key_secret` and `capability`. The index means what it means in a spec — 0 full access, 1 push admin, 2 per-channel capabilities, 3 subscribe-only, 4 revocable tokens. `sandbox.key_str` (the full-access key) and `sandbox.app_id` are shorthands | -| `use_binary_protocol` fixture | runs the test once per protocol. **Only a spec carrying a `## Protocol Variants` section takes it** — `publish`, `history`, `presence`, `batch_presence`, `mutable_messages`. A test that does not take it runs json only, which is the clients' default here | +| `use_binary_protocol` fixture | runs the test once per protocol; each tier's `conftest.py` defines its own. **Only a spec carrying a `## Protocol Variants` section takes it** — in REST `publish`, `history`, `presence`, `batch_presence`, `mutable_messages`; in realtime `channel_history`, `channels/channel_publish`, `delta_decoding`, `mutable_messages`, `presence_lifecycle`. A test that does not take it runs json only, which is the clients' default here | +| `await_connection_state(client, state, timeout=5)`, `await_channel_state(channel, state, timeout=5)` | the specifications' `AWAIT_STATE`. Pass `timeout=10` in this tier: five is the budget a mock-backed test needs, and `channel_history_test.md` spells out ten for a connect that opens a real socket | | `sandbox_rest_client(key=None, **kwargs)` | `Rest(ClientOptions(key: api_key, endpoint: "nonprod:sandbox"))`. Registered for the same teardown as `rest_client`. Leave `key` out and pass `token=`, `auth_callback=` or `auth_url=` where the spec authenticates some other way | | `sandbox_realtime_client(key=None, **kwargs)` | the same for realtime, for the REST specs that need presence members or presence history a connection has to produce. Unlike `realtime_client` it keeps `auto_connect` and the fallback hosts at the **library** defaults | | `wall_clock_poll_until(condition, timeout=10.0, description='condition', interval=0.5)` | this tier's `poll_until`. Sleeps `interval` between attempts, takes a sync or async condition, and **returns whatever the condition answered with**, so a condition that fetches a page saves fetching it again | @@ -340,7 +349,8 @@ async def test_rsl2a_history_returns_messages(sandbox, use_binary_protocol): ## The proxy tier `uts/rest/integration/proxy/.md` becomes -`test/uts/rest/integration/proxy/_test.py`, and routes its traffic through +`test/uts/rest/integration/proxy/_test.py` and `uts/realtime/integration/proxy/.md` +becomes `test/uts/realtime/integration/proxy/_test.py`, and each routes its traffic through [ably/uts-proxy](https://github.com/ably/uts-proxy) on the way to the sandbox. The proxy binds a port per session, takes plain HTTP on it, speaks TLS onwards, applies the session's rules and records what crosses it. `uts/docs/proxy.md` governs the tier and @@ -353,7 +363,19 @@ What differs from the rest of the integration tier: `port=session.proxy_port`, `tls=False`, `use_binary_protocol=False`. `endpoint` and `port` set the primary host, and `fallback_hosts=['localhost']` sets the fallback to the same session, so both attempts land in one event log. Leave `fallback_hosts` out - where the spec does: `endpoint='localhost'` disables fallbacks by itself (REC2c2). + where the spec does: `endpoint='localhost'` disables fallbacks by itself (REC2c2). A + realtime client takes the same four and adds `auto_connect=False`, so that a state + recorder can be registered before the connection opens. +- **The event log's field names are the proxy's, not the specification's.** `ws_connect` + carries `queryParams`. `ws_frame` carries `direction` — `server_to_client` or + `client_to_server` — a `message` whose `action` is an **integer**, and `ruleMatched` + holding the rule's `comment` string verbatim. `ws_disconnect` carries `initiator`. A + `replace`d frame is logged as the frame the server sent, not as the replacement, and a + `suppress`ed frame is logged too, so both are still countable. An imperative + `trigger_action` appears as an `action` event followed by the `ws_frame` it produced. + A specification writing `e.type == "ws_frame_to_server"` or `action == "MESSAGE"` is + reading fields that do not exist: derive against the real names, through a filter + defined once at the top of the file, and record the drift in `deviations.md`. - **Authentication is a callback.** A plain connection cannot carry basic auth, so every client takes `auth_callback=`; see the traps below. - **A fault is a rule, and everything else passes through.** `times: 1` faults the @@ -735,6 +757,57 @@ Established against the real proxy and the real sandbox while deriving unit is wrong. The test is written as the specification has it and gated with `@deviation`; `test/uts/deviations.md` carries the entry. +## Traps found while deriving the realtime integration tier + +Each was established against the sandbox, or against the sandbox behind `uts-proxy`. + +- **`RealtimeChannel.publish()` takes name and data positionally only.** + `publish(name='x', data='y')` raises `ValueError: publish() expects either (name, data) + or a message object or array of messages` before anything reaches the server + (`ably/realtime/channel.py:394`), where `RestChannel.publish()` accepts the keyword + form. The specifications write the keyword form throughout, so every realtime publish + is translated positionally. +- **A binary payload comes back as `bytearray`, not `bytes`**, under both protocols. It + compares equal to the `bytes` that was published, so only the type assertion is + affected: `ASSERT data IS Binary` has to read `isinstance(data, (bytes, bytearray))`. +- **DISCONNECTED is transient after a drop from CONNECTED.** The retry is a + `loop.call_soon`, not a timer, so DISCONNECTED and CONNECTING land in the same + millisecond and `await_connection_state(client, DISCONNECTED)` is a coin toss. Register + a `connection.on(...)` recorder before connecting and wait on the recorded list, which + is also what a specification reading `state_changes` wants. +- **An `AWAIT_STATE` for a state the channel already holds asserts nothing** — it returns + at once. `channel_faults.md`'s RTL13a and RTL3d both re-attach an already-attached + channel, so both are taken on the recorded sequence instead: ATTACHING followed by + ATTACHED, which is what their `CONTAINS_IN_ORDER` assertion checks anyway. +- **`refuse_connection` through the proxy is a caught failure**, unlike the mock tier's + `respond_with_refused`. The proxy answers the upgrade with HTTP 502, `websockets` + raises, and the SDK reports `40000/400 'Error opening websocket connection: server + rejected WebSocket connection: HTTP 502'` at once. No timeout shortening is needed. +- **`realtime_request_timeout` is milliseconds at the client option**, and `Timer` + divides by 1000, so a specification's `realtimeRequestTimeout: 3000` maps straight + across. That is the opposite of `http_request_timeout`, which the REST proxy tier found + is applied as seconds; the defect does not generalise, so do not carry it over. +- **The proxy's event timestamps are RFC 3339 with a variable number of fractional + digits.** Comparing them as strings is wrong, and `datetime.fromisoformat` will not + take the trailing `Z` on Python 3.9. Order the log by position rather than by timestamp + wherever that will do. +- **A locally signed Ably JWT is the cheapest credential for a proxy test.** It costs no + round trip, so nothing extra lands in the event log beside the frames a test counts. + `test/uts/helpers/sandbox.py`'s `generate_jwt` signs one, and each proxy module wraps it + in a small `jwt_auth_callback` of its own. +- **`wall_clock_poll_until`'s `description` is evaluated eagerly**, so it cannot report + state a test accumulated while waiting. `connection_resume_test.py` wraps it to re-raise + with the recorded states appended, which is what makes its timeouts self-explanatory; + copy that wherever the wait is on a state machine. +- **Server-initiated reauth is fully implemented.** Injecting `{'action': 17}` on a live + connection re-invokes the authCallback, leaves the state CONNECTED and the connectionId + unchanged, and the sandbox answers with a second CONNECTED — frame actions `[4, 17, 4]`. + A test written expecting a disturbance will not find one. +- **`close` and `disconnect` are indistinguishable to the client.** Both give + CONNECTING → CONNECTED → DISCONNECTED → CONNECTING → CONNECTED in about 1.2 s. + `disconnect` leaves `error_reason` set to "no close frame received or sent" and `close` + leaves it `None`, which is the only way to tell them apart from inside the SDK. + ## Timers Three regimes; pick by tier. @@ -752,12 +825,16 @@ installs a `FakeClock` on it. Still prefer a short real interval through a clien for `FakeClock` only for `connection_state_ttl`, which no option sets and whose default costs 120 real seconds. See the fake-time section of `test/uts/deviations.md`. -**REST integration.** Real time, deliberately. There is no seam to install in front of -a server the tests exist to talk to, and shortening a timeout would only make the tier -flaky. Poll with `wall_clock_poll_until` rather than sleeping a guess. +**Integration, REST and realtime.** Real time, deliberately. There is no seam to install +in front of a server the tests exist to talk to, and shortening a timeout would only make +the tier flaky. Poll with `wall_clock_poll_until` rather than sleeping a guess, and wait on +states with `await_connection_state` / `await_channel_state`. Where a specification sets +`realtimeRequestTimeout`, `disconnectedRetryTimeout` or `suspendedRetryTimeout`, pass it +through as the client option of the same name: those drive real timers against a real +server, and they are milliseconds on both sides. -The pytest timeout is 30 seconds for the suite, 120 for `rest/integration` and 300 for -`rest/integration/proxy`, so keep waits well under whichever applies. +The pytest timeout is 30 seconds for the suite, 120 for each integration tier and 300 for +the `proxy` package inside each, so keep waits well under whichever applies. ## Deviations @@ -808,7 +885,7 @@ the reasoning. The next reader will otherwise reach the same first conclusion. ```bash uv run --frozen --extra crypto --extra dev ruff check ably/ test/ uv run --frozen --extra crypto --extra dev pytest test/uts/rest/unit test/uts/realtime/unit test/uts/helpers -q -uv run --frozen --extra crypto --extra dev pytest test/uts/rest/integration -q +uv run --frozen --extra crypto --extra dev pytest test/uts/rest/integration test/uts/realtime/integration -q RUN_DEVIATIONS=1 uv run --frozen --extra crypto --extra dev pytest test/uts -q ``` @@ -816,9 +893,10 @@ RUN_DEVIATIONS=1 uv run --frozen --extra crypto --extra dev pytest test/uts -q environment's cutoff — and `--extra dev` carries pytest. Line length is 115. If `uv.lock` changes, `git checkout -- uv.lock`. -The second command is the offline tiers, which need no network. The third provisions a -sandbox app and does; `pytest test/uts -q` runs both together, so run the tiers -separately when only one is in question. All three must pass. +The second command is the offline tiers, which need no network. The third is the two +integration tiers, each of which provisions a sandbox app of its own and does need the +network; `pytest test/uts -q` runs everything together, so run the tiers separately when +only one is in question. All three must pass. The fourth is the check that the deviations record is still true, across both tiers: **every gated test must fail when enabled**, so gated + unimplementable under it must diff --git a/test/uts/README.md b/test/uts/README.md index d3fea133..c91bf62e 100644 --- a/test/uts/README.md +++ b/test/uts/README.md @@ -192,6 +192,64 @@ Tests in this package are given 300 seconds each: a cold cache downloads the bin before the first of them runs, and a specification that provokes a timeout sits through the delay it asked the proxy for. +`realtime/integration/` runs against the same sandbox over a real WebSocket — twenty +specifications, thirteen of them straight to the sandbox and seven under `proxy/`. It +provisions an app of its own, which arrives as the `realtime_sandbox` fixture, so that a +realtime test entering presence or publishing to a channel cannot be seen by a REST test +reading the same channel name. The app carries the same `key(i)`, `key_str` and `app_id` +members the `sandbox` fixture does. + +```python +async def test_rtl7a_subscribe_all_messages(realtime_sandbox): + client = sandbox_realtime_client(realtime_sandbox.key_str) + channel = client.channels.get('test-rtl7a-' + random_id()) +``` + +Five specifications carry a `## Protocol Variants` section — `channel_history`, +`channels/channel_publish`, `delta_decoding`, `mutable_messages` and `presence_lifecycle` — +and take the tier's own `use_binary_protocol` fixture, passing it to every client they +build. The other fifteen are json only. + +The connections are real, so the waits are wall-clock here too. +`await_connection_state(client, state, timeout)` and `await_channel_state(channel, state, +timeout)` are the specifications' `AWAIT_STATE`, and ten seconds is the figure the +specifications give for reaching CONNECTED over a network, against the five those helpers +default to for a mock. A state the client passes through in a millisecond cannot be waited +for after the fact — DISCONNECTED after a drop from CONNECTED is one, the retry being a +`loop.call_soon` — so a test that needs it registers a `connection.on(...)` recorder before +connecting and waits on the recorded list. + +Tests here are given 120 seconds each, from the package's own `conftest.py`, as in the REST +integration tier. + +`realtime/integration/proxy/` puts the same pinned `uts-proxy` between the client and the +sandbox, with the same `proxy_control` and `proxy_session` fixtures and the same two +environment variables, `UTS_PROXY_LOCAL_PATH` and `UTS_PROXY_CONTROL_URL`, that +[helpers/proxy.py](helpers/proxy.py) documents. What these specifications fault is the +WebSocket rather than an HTTP request — a frame suppressed, replaced or injected, a socket +closed, an upgrade refused — and the event log is read for the frames that crossed: + +```python +async def test_rtn15a_disconnect_triggers_resume(realtime_sandbox, proxy_session): + session = await proxy_session(rules=[{ + 'match': {'type': 'delay_after_ws_connect', 'delayMs': 1000}, + 'action': {'type': 'close'}, + 'times': 1, + }]) + client = sandbox_realtime_client( + auth_callback=jwt_auth_callback(realtime_sandbox.key_str), + endpoint='localhost', port=session.proxy_port, tls=False, + use_binary_protocol=False, auto_connect=False) +``` + +`endpoint='localhost'` disables the fallback hosts by itself (REC2c2), so every attempt +lands on the one session port and appears in the one event log. A realtime connection +carries its credentials in the WebSocket's query string, so a plain `key=` does work over +the session; the modules here sign an Ably JWT locally instead, through a file-local +`jwt_auth_callback` built on `generate_jwt`, because it costs no round trip and so adds +nothing to the log a test is counting. Tests in this package are given 300 seconds each, +as in the REST proxy package. + ## Running ``` @@ -204,10 +262,12 @@ The offline tiers alone, which need no network: uv run --frozen --extra crypto --extra dev pytest test/uts/rest/unit test/uts/realtime/unit test/uts/helpers -q ``` -The integration tier alone, which provisions a sandbox app and needs network access: +Either integration tier alone, each of which provisions a sandbox app and needs network +access: ``` uv run --frozen --extra crypto --extra dev pytest test/uts/rest/integration -q +uv run --frozen --extra crypto --extra dev pytest test/uts/realtime/integration -q ``` `--frozen` is required: without it dependency resolution reaches past the diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 1637afed..11cd3e43 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -24,33 +24,34 @@ One Test ID can become more than one derived test: five Test IDs in `rest/unit` `error_types_test.py`, `fallback_test.py`, `rest_client_test.py` (two) and `paginated_result_test.py` — assert several independent things under a single id, and the derivation writes a function for each rather than one function with an unrelated -second half. That turns 1059 Test IDs into 1068 derived tests. Going the other way, one +second half. That turns 1132 Test IDs into 1141 derived tests. Going the other way, one derived test can become more than one case: five of the twelve `rest/integration` -specifications carry a `## Protocol Variants` section and run every one of their tests -twice, once per protocol, and nine `rest/unit` tests are parametrized over a table of -fixtures the specification gives inline. That turns 1068 derived tests into 1139 pytest -cases. +specifications and five of the twenty `realtime/integration` ones carry a `## Protocol +Variants` section and run every one of their tests twice, once per protocol, and nine +`rest/unit` tests are parametrized over a table of fixtures the specification gives +inline. That turns 1141 derived tests into 1234 pytest cases. -Of **1059 Test IDs, derived as 1068 tests and run as 1139 pytest cases**: 841 Test IDs -(850 tests, 917 cases) pass, 203 (203 tests, 207 cases) are gated behind +Of **1132 Test IDs, derived as 1141 tests and run as 1234 pytest cases**: 905 Test IDs +(914 tests, 1002 cases) pass, 212 (212 tests, 217 cases) are gated behind `RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are disjoint: two Test IDs, and one parametrized test, have a gated part and a passing part, and are counted with the gated. Every gated test has been confirmed to fail when enabled, so none of them passes under both behaviours. 494 of the Test IDs come from -`uts/rest/unit` (503 tests, 536 cases), 481 from `uts/realtime/unit` (481, 481) and 84 -from `uts/rest/integration` (84, 122), 8 of those (8, 8) from the `proxy` package within -it; of the gated Test IDs 122 are REST and 81 realtime, which is 126 REST cases and 81 -realtime. +`uts/rest/unit` (503 tests, 536 cases), 481 from `uts/realtime/unit` (481, 481), 84 +from `uts/rest/integration` (84, 122) and 73 from `uts/realtime/integration` (73, 95); 8 +of the REST integration ids (8, 8) and 30 of the realtime ones (30, 30) come from the +`proxy` package within each. Of the gated Test IDs 122 are REST and 90 realtime, which is +126 REST cases and 91 realtime. A further 122 pytest cases under `helpers/` cover the mock infrastructure itself and are not derived from a specification. -The 193 gated Test IDs that record SDK non-compliance — 193 tests, 197 cases — reduce to -**67 distinct root causes**, 27 on the REST side and 40 on the realtime side. Three further +The 202 gated Test IDs that record SDK non-compliance — 202 tests, 207 cases — reduce to +**71 distinct root causes**, 27 on the REST side and 44 on the realtime side. Three further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a), has nothing to assert against (the timezone split on synthesized LEAVE timestamps), or is worked around in the setup of every test that would otherwise trip over it (`enterClient` on an anonymous connection), so the file -carries **70 SDK root causes** in all. The remaining 10 gated Test IDs are +carries **74 SDK root causes** in all. The remaining 10 gated Test IDs are specification faults, and reduce to 7. Entries closed by a fix are removed rather than kept as history; `git log` holds that. @@ -95,7 +96,9 @@ assertion it carries still stands. Those tests keep the corrected fixture (or th corrected label in a comment), pass, and carry a `# UTS SPEC ERROR:` comment at the site. The entries below cover both kinds and say which applies. Almost every realtime fault is of the second kind, which is why only one realtime test is gated as a spec -error while fourteen realtime entries appear below. +error while fourteen `realtime/unit` entries appear below. The eight +`realtime/integration` faults are of that kind without exception, so none of them is +gated either. The three sections that follow this one record SDK behaviour rather than specification faults. @@ -119,18 +122,28 @@ Raised upstream: | [#544](https://github.com/ably/specification/issues/544) | Fixtures that cannot produce the condition they describe | | [#545](https://github.com/ably/specification/issues/545) | Fixtures written against mock methods the contract does not define | | [#546](https://github.com/ably/specification/issues/546) | Connection setups crediting a key-authenticated client with an initial token request | +| [#547](https://github.com/ably/specification/issues/547) | A device identity token hard-coded to a literal the server rejects | +| [#548](https://github.com/ably/specification/issues/548) | A restricted-key test closing the connection that owns the presence member it asserts on | +| [#549](https://github.com/ably/specification/issues/549) | A time-range test that passes when the range is ignored, under three rotated section labels | +| [#550](https://github.com/ably/specification/issues/550) | Housekeeping in the integration tier: two short headers, a JWT fixture, an empty presence array | +| [#551](https://github.com/ably/specification/issues/551) | Three sections whose heading or setup contradicts the steps and assertions below it | +| [#552](https://github.com/ably/specification/issues/552) | `proxy/connection_resume.md`: a status code neither SDK returns, a proxy substitution that does not exist, and event-log fields the proxy does not emit | +| [#553](https://github.com/ably/specification/issues/553) | A heartbeat-starvation test that closes the socket thirteen seconds inside the idle window | +| [#554](https://github.com/ably/specification/issues/554) | Two sections provoking one server response, leaving the revoked-key point uncovered | `#527` also carries a comment on the realtime wire-format assertions, `#532` one on the same housekeeping categories in `realtime/unit`, and [#466](https://github.com/ably/specification/issues/466) — which is not ours — one on the RSA4c3 contradiction, since that issue is what decides it. +`#547` to `#550` are the `uts/rest/integration` faults and `#551` to `#554` the +`uts/realtime/integration` ones. Each of them is of the second kind — a fixture, a setup +step or a header label — so the derived test keeps the corrected fixture and passes, and +none of them is among the ten gated above. + Not every entry has an issue of its own: the URL-safe base64 alphabet is recorded below -and not filed, because ably-python's own encoding settles the tests either way. Neither -are the faults in `uts/rest/integration`, which have no issue numbers against them. Each -of those is of the second kind — a fixture, a setup step or a header label — so the -derived test keeps the corrected fixture and passes, and none of them is among the ten -gated above. Line references in these entries are against `ably/specification@d9a04ca`. +and not filed, because ably-python's own encoding settles the tests either way. Line +references in these entries are against `ably/specification@d9a04ca`. ### `/time` is stubbed as an object rather than an array @@ -510,6 +523,7 @@ the path, or drop the reference and keep the inline cases as the definition. ### `push_channels.md` hard-codes a device identity token the server rejects **Spec points:** RSH7a, RSH7c, `rest/integration/RSH7a/subscribe-unsubscribe-device-0`. +Filed as [#547](https://github.com/ably/specification/issues/547). The setup's own comment says "The deviceIdentityToken is obtained from the registration response", and the pseudocode immediately beneath it writes @@ -531,38 +545,43 @@ ordinary credentials. ### Closing the realtime client destroys the presence the following REST read is about **Spec points:** RSC24 and BGF2 (`batch_presence.md`, -`rest/integration/RSC24/restricted-key-channel-failure-1`); RSP4b2 (`presence.md`, -`rest/integration/RSP4b2/history-direction-forwards-0`). - -Two specifications put `AWAIT realtime.close()` between the presence operations that -create their fixture and the REST read that asserts on it, and in both the close is what -breaks the assertion. +`rest/integration/RSC24/restricted-key-channel-failure-1`). Filed as +[#548](https://github.com/ably/specification/issues/548). `batch_presence.md`'s restricted-key test enters `member-1` on the allowed channel and -`member-2` on the denied one, closes the realtime client, and then requires -`success.presence.length == 1` with `success.presence[0].clientId == "member-1"`. A -presence member belongs to the connection that entered it, so closing the connection takes -it away. Measured against the sandbox: after the close, `GET +`member-2` on the denied one, puts `AWAIT realtime.close()` between that fixture and the +REST read, and then requires `success.presence.length == 1` with +`success.presence[0].clientId == "member-1"`. A presence member belongs to the connection +that entered it, so closing the connection takes it away. Measured against the sandbox, +three runs, each querying before the close and at +0, +1 and +3 seconds after: before the +close the allowed channel carries `presence: ["member-1"]`; after it, `GET /presence?channels=channel6,denied-…` answers `{"successCount": 1, "failureCount": 1, "results": [{"channel": "channel6"}, {…"error": {"code": 40160, "statusCode": 401}}]}` -every time, the allowed channel carrying no `presence` at all. The other two tests in the -same file get it right and say so — "Keep realtime open during the REST query so the -presence member persists on the server." - -`presence.md`'s RSP4b2 closes the connection while the member is still present, which -produces a LEAVE carrying no data, and then reads -`history(direction: "backwards").items[0].data == "third"` — which now reads the -synthesized LEAVE rather than the last update. - -In both places the close belongs in cleanup, as the sibling tests put it. The derived -tests omit it and leave the suite's autouse teardown to close every client a test built, -which is what the specifications' own cleanup steps amount to; the five `presence.md` -tests that generate presence events do the same. Every assertion is the specifications', +every time, the allowed channel carrying no `presence` key at all. Not a race. The other +two tests in the same file get it right and say so — "Keep realtime open during the REST +query so the presence member persists on the server." + +The close belongs in cleanup, as the sibling tests put it. The derived test omits it and +leaves the suite's autouse teardown to close every client the test built, which is what +the specification's own cleanup step amounts to. Every assertion is the specification's, unchanged. +`presence.md`'s RSP4b2 puts the same `AWAIT realtime.close()` in the same place and is +**not** affected, which is worth recording because it looks as though it should be. The +close does add a synthesized LEAVE as the newest presence event, so +`history(direction: "backwards").items[0]` is that LEAVE rather than the last update — but +the server gives the synthesized LEAVE the member's last data, so the assertion the +specification writes, `items[0].data == "third"`, still holds. Measured: before the close +the backwards page is `[(4, "third"), (4, "second"), (2, "first")]`, and after it +`[(3, "third"), (4, "third"), (4, "second"), (2, "first")]`, unchanged at +0.5, +1.5 and ++3 seconds. The derived test keeps the close. What the assertion cannot see is that it is +reading a LEAVE at all; `items[0].action` would pin the intent, and that is a remark on +[#548](https://github.com/ably/specification/issues/548) rather than a change asked for. + ### RSL2b3's assertions cannot detect an ignored time range **Spec point:** RSL2b3, `history.md`, `rest/integration/RSL2b3/history-time-range-0`. +Filed as [#549](https://github.com/ably/specification/issues/549). The test publishes two "early" messages, waits 2 ms, publishes two "late" ones, computes a boundary from the server-assigned timestamps, and queries twice — once from before the @@ -586,6 +605,150 @@ depends on: with both batches inside one millisecond there is no side of the bou put them on, and the test should fail on the stated premise rather than on an exclusion that cannot hold. It passes. +The same section is also filed under the wrong point, as are its two siblings. +`features.md` has RSL2b1 as `start` and `end`, RSL2b2 as `direction` and RSL2b3 as +`limit`; `history.md` heads them `RSL2b1 - History direction forwards`, `RSL2b2 - History +limit parameter` and `RSL2b3 - History time range parameters`, which rotates all three by +one. The sibling `presence.md` files the identical RSP4b family correctly, so it is +`history.md` alone. The Test IDs carry the specification's labels, so the derived tests +are named for the rotated points rather than the real ones. + +### `connection_lifecycle_test.md`'s RTN4b fixture contradicts its own first assertion + +**Spec point:** RTN4b, `realtime/integration/RTN4b/successful-connection-0`. + +The setup builds `Realtime(key, endpoint)` and leaves `autoConnect` at the library +default; the first Test Step then asserts `connection.state == initialized`. RTN3 makes +that default **true**, so there is no moment at which both hold. ably-python's constructor +calls `request_state(CONNECTING, force=True)` synchronously — measured, the state is +already CONNECTING when the constructor returns. The sibling RTN11 test in the same file +does set `autoConnect: false`, which is what this one wants too, so the derived test adds +`auto_connect=False` and keeps every assertion. + +Filed as [#551](https://github.com/ably/specification/issues/551). + +### `auth.md`'s RSA7 mismatched-clientId test contradicts its own assertions + +**Spec point:** RSA7, `realtime/integration/RSA7/mismatched-clientid-fails-1`. + +Test Steps says `EXPECT THROW creating Realtime(options: …)`. The Assertions block +immediately below it says "the key assertion is that the connection enters FAILED state +with error code 40102". Both cannot hold: a constructor has no token to compare a clientId +against, so nothing is knowable until the server answers. Measured: construction does not +raise — the client comes back INITIALIZED — and the connection reaches FAILED with +40102/401 "invalid clientId for credentials" about fifteen seconds later. The derived test +drops the throw and makes the specification's own named assertion. The fifteen seconds are +not the server's: they are the spurious `disconnected_retry_timeout` recorded under +`#### A JWT string with a matching clientId is rejected 40102` below, which is why the +test waits twenty seconds rather than the default ten. + +Filed as [#551](https://github.com/ably/specification/issues/551). + +### `channel_attach_test.md`'s RTL14 heading and prose contradict its own test steps + +**Spec point:** RTL14, `realtime/integration/RTL14/insufficient-capability-failed-0`. + +The heading reads "Insufficient capability causes channel FAILED" and the prose has the +server "responds with a channel-scoped ERROR and the channel transitions to FAILED". The +Test Steps then say "Attach succeeds (subscribe-only key can attach to any channel)" and +assert `channel.state == ATTACHED` outright, and the Assertions read the publish error +and the connection state without looking at the channel again. Measured with `keys[3]`, `{"*": ["subscribe"]}`: the attach reaches +ATTACHED, the publish raises 40160/401 "Unable to publish a message due to lacking the +required 'publish' capability", and the connection is still CONNECTED. The steps are what +the server does, so the steps are what is derived, and the heading is the fault. + +Filed as [#551](https://github.com/ably/specification/issues/551). + +### `proxy/connection_resume.md`'s RTN15h1 asserts a 401 where the status code is 403 + +**Spec point:** RTN15h1, `realtime/proxy/RTN15h1/token-error-nonrenewable-failed-0`. + +The section asserts `errorReason.statusCode == 401` beside `code == 40171`, and its own +note says it follows ably-js in expecting 40171. ably-js throws that `ErrorInfo` with +`statusCode: 403` (`src/common/lib/client/auth.ts`, the "Need a new token, but authOptions +does not include any way to request one" branch), and ably-python raises +`AblyAuthException(msg, 403, 40171)` at `ably/rest/auth.py:200`. Measured end to end +through the proxy: FAILED, 40171, 403. The derived test asserts 403 and passes. The +realtime unit tier arrives at the same 40171 by a different route — see +`### A token error with no means to renew reports the renewal failure, not the server's error`. + +Filed as [#552](https://github.com/ably/specification/issues/552). + +### `proxy/connection_resume.md`'s RTN14h asks the proxy for a substitution it does not make + +**Spec point:** RTN14h, `realtime/proxy/RTN14h/resume-after-ttl-expiry-0`. + +The rule replaces the first CONNECTED and writes `"connectionKey": "__PASSTHROUGH__"` in +both the frame and its `connectionDetails`, intending the proxy to fill in the key the +server issued. uts-proxy v0.3.0 has no such sentinel and passes the literal through. +Measured: the client took `__PASSTHROUGH__` as its connection key, reconnected with +`?resume=__PASSTHROUGH__`, and the sandbox answered `{'code': 80018, 'message': 'invalid +connection key: __PASSTHROUGH__'}`. The rule is kept verbatim, because the test is gated +on the TTL defect before the connection key matters. Either the proxy grows the +substitution or the specification stops asking for it. + +Filed as [#552](https://github.com/ably/specification/issues/552). + +### `proxy/connection_resume.md`'s RTN19a reads log fields the proxy does not emit + +**Spec point:** RTN19a, `realtime/proxy/RTN19a/unacked-resent-on-resume-0`. + +It filters the event log on `e.type == "ws_frame_to_server"` and `e.message.action == +"MESSAGE"`. The proxy emits `type: "ws_frame"` with the direction in a separate +`direction` field, and `action` as the protocol integer. The same drift is already +recorded above for `connection_recovery_test.md`. The derived test reads the real field +names through a `frames(log, direction, action)` helper defined at the top of the file, +and asserts exactly what the specification asserts. + +Filed as [#552](https://github.com/ably/specification/issues/552). + +### `proxy/heartbeat.md` does not exercise the spec point it is filed under + +**Spec point:** RTN23a, `realtime/proxy/RTN23a/heartbeat-starvation-reconnect-0`. + +The file is titled "Heartbeat starvation causes disconnect and reconnect" and quotes +RTN23a — "if no activity is received for `maxIdleInterval + realtimeRequestTimeout`, the +transport should be disconnected". Its rule is `delay_after_ws_connect: 2000` followed by +`close`, so the proxy sends a WebSocket close frame two seconds into the connection. +Measured: the sandbox advertises `maxIdleInterval: 15000` in CONNECTED, so the close lands +thirteen seconds inside the window the idle timer would have measured, and the +disconnection observed is the close frame's doing. The specification's own Integration Test +Notes admit as much. Every assertion is sound for what the test does do, and the derived +test makes all of them; the fault is that the spec point is unexercised at this tier. +Exercising it wants a `suppress_onwards` rule, a wait past twenty-five seconds, and a +session `timeoutMs` long enough to survive the idle. uts-proxy's own API reference gives +that pairing as its worked "Heartbeat starvation" example. + +Two smaller notes on the same file. It describes the connection as "re-established with +**new** connection details" where the resume in fact succeeds and the connectionId is +unchanged — measured identical across both connections; the assertions require only +non-null, so they hold either way. And it captures `first_connection_key` and never uses +it: the derived test spends it on +`ws_connects[1]['queryParams']['resume'] == first_connection_key`, which is measured true +and is the only place the connection key is observable in the scenario. + +Filed as [#553](https://github.com/ably/specification/issues/553). + +### `connection_failures_test.md`'s RTN14a and RTN14g are the same provocation + +**Spec points:** RTN14a (`realtime/integration/RTN14a/invalid-key-failed-0`) and RTN14g +(`realtime/integration/RTN14g/revoked-key-failed-0`), +`connection/connection_failures_test.md`. + +They are presented as "invalid API key" and "revoked key / deleted app", but both fixtures +name an application that does not exist and the sandbox answers both identically: +40101/401 "unable to handle request; no application id found in request". Both sets of +assertions admit that code — RTN14a as one of 40005 or 40101, RTN14g as anything outside +the token-error range 40140–40149 — so both derived tests pass, and the two differ in what +they assert about one shared server response rather than in the response they provoke. +RTN14g is not exercising a revoked key. Doing so wants a key that exists and has been +revoked, which neither fixture nor the app-provisioning section produces: +`ably-common/test-resources/test-app-setup.json`'s only revocation affordance is +`{"revocableTokens": true}`, which revokes tokens and answers 40141 — inside the +40140–40149 range RTN14g excludes. + +Filed as [#554](https://github.com/ably/specification/issues/554). + ### Smaller faults | Spec | Fault | @@ -599,9 +762,10 @@ that cannot hold. It passes. | `fallback.md` | REC3a, REC3b and REC3 drive a Realtime client but sit in `rest/unit` | | `message_encoding.md`, `msgpack_interop.md`, `annotations.md` | Six sections carry no Test ID; ids were inferred by sibling convention | | `publish.md`, `rest_presence.md`, `message_encoding.md`, `history.md`, `idempotency.md` | All point at `/Users/paddy/data/worknew/dev/dart-experiments/...` for the mock contract | -| `publish.md` (integration) | The `Spec points:` header reads RSL1d, RSL1l1, RSL1m4, RSL1n, and the file carries a fifth section, `## RSL1k5 - Idempotent publish with client-supplied IDs`, with its own Test ID. The section is sound; only the header is short. Same housekeeping class as [#532](https://github.com/ably/specification/issues/532) | -| `auth.md` (integration) | RSC10's expired-JWT fixture is `generate_jwt(expires_at: now() - 5_seconds)`, naming `exp` and leaving `iat` open. Ably reads a JWT's lifetime as `exp - iat` and rejects a negative one with 400/40003 "Invalid value for ttl" before it considers expiry, so `iat` at now produces a token that fails the wrong way and never reaches the 40140–40149 renewal path the test is about. Backdating `iat` past `exp` gives the already-expired token the test wants, answered 401/40142. An SDK signing its own Ably JWT has to choose, so the fixture should say which | -| `batch_presence.md` | BGR2 says a channel with no members "returns a success result with an empty `presence` array", and the unit tier's mocks all send `'presence': []`. The server sends no `presence` key at all, so an implementation has to default the field for the assertion to hold. The derived test asserts the specification's `length == 0`, with the wire shape in a comment | +| `publish.md` (integration) | The `Spec points:` header reads RSL1d, RSL1l1, RSL1m4, RSL1n, and the file carries a fifth section, `## RSL1k5 - Idempotent publish with client-supplied IDs`, with its own Test ID. The section is sound; only the header is short. `auth.md` (integration) has the same shape: its header reads RSA4, RSA8 and it carries `## RSC10` with its own Test ID. Same housekeeping class as [#532](https://github.com/ably/specification/issues/532); filed as [#550](https://github.com/ably/specification/issues/550) | +| `auth.md` (integration) | RSC10's expired-JWT fixture is `generate_jwt(expires_at: now() - 5_seconds)`, naming `exp` and leaving `iat` open. Ably reads a JWT's lifetime as `exp - iat` and rejects a negative one with 400/40003 "Invalid value for ttl" before it considers expiry, so `iat` at now produces a token that fails the wrong way and never reaches the 40140–40149 renewal path the test is about. Backdating `iat` past `exp` gives the already-expired token the test wants, answered 401/40142. An SDK signing its own Ably JWT has to choose, so the fixture should say which. Filed as [#550](https://github.com/ably/specification/issues/550) | +| `batch_presence.md` | The restricted-key setup's comment reads "only has access to \"batch-allowed\" channel" while the setup fixes `allowed_channel = "channel6"`; `batch-allowed` appears nowhere in the file. Filed with [#548](https://github.com/ably/specification/issues/548), whose fix replaces the same lines | +| `batch_presence.md` | BGR2 says a channel with no members "returns a success result with an empty `presence` array", and the unit tier's mocks all send `'presence': []`. The server sends no `presence` key at all, so an implementation has to default the field for the assertion to hold. The derived test asserts the specification's `length == 0`, with the wire shape in a comment. Filed as [#550](https://github.com/ably/specification/issues/550) | ## Failing Tests @@ -614,10 +778,11 @@ Nothing to fix here, only something to build. Each row is one feature, and the c the number of gated Test IDs that fall with it, with the pytest case count beside it where the two differ. -Four of these rows are gated at both tiers. `batchPresence`, `Auth#revokeTokens`, the +Five of these rows are gated at both tiers. `batchPresence`, `Auth#revokeTokens`, the `PushChannel` surface and the `clientId` filter on `RestPresence#get` each carry -`uts/rest/integration` tests as well as unit ones, written against the spelling the unit -tier already gates on, so both tiers go green together when the API lands. Those +`uts/rest/integration` tests as well as unit ones, and connection recovery carries two +`uts/realtime/integration` ones; all are written against the spelling the unit tier +already gates on, so both tiers go green together when the API lands. Those integration tests do all their real work first — the sandbox app, the channels, the presence members entered over a realtime connection, the registered device and the issued token are all real, and each test reaches the missing call before it fails, so the @@ -625,14 +790,16 @@ assertions either side of it are known to hold against real server responses. Th `batch_presence` and `push_channels` files were additionally run against throwaway shims — a `batch_presence` forwarding to `GET /presence`, and a `PushChannel` posting and deleting `/push/channelSubscriptions` with `X-Ably-DeviceToken` — and pass in full -against them. +against them. The two recovery tests are a different shape: there is no missing call for +them to reach, so each runs end to end against the sandbox through `uts-proxy` and fails +on the `recover` parameter the connection never sends. | Spec points | Missing | Test IDs | |---|---|---| | RSC22, RSC24, BSP2, BPR2, BPF2, BAR2, BGR2, BGF2 | `batchPublish` and `batchPresence`, and all six result types. `grep -rn batch ably/` finds nothing | 44 (47 cases) | | RSA17, RSA17b–g, BAR2, TRS2, TRF2 | `Auth#revokeTokens`, `TokenRevocationTargetSpecifier`, `BatchResult`. Gated against `auth.revoke_tokens(targets, issued_before=, allow_reauth_margin=)` returning `success_count` / `failure_count` / `results`, with `target` / `issued_before` / `applies_at` / `error` per result. RSA17d is the one case that needs no server at all — a token-authenticated client must refuse locally with 40162/401 — so it can be satisfied before any of the wire work | 21 | | RSH7, RSH7a–e, RSH6, RSH8 | `PushChannel`: `channel.push`, `client.device`, `LocalDevice`. The push *admin* surface (RSH1) does exist | 12 | -| RTN16, RTN16f–k, RTC1c (TO3i) | Connection recovery, entire. `recover` is in the `Options` signature, stored, and given a property and a setter (`options.py:30,111,193,196`), and read nowhere. No `Connection#createRecoveryKey`, no `recover` connect parameter, no recovery-key decoding | 6 | +| RTN16, RTN16f–k, RTC1c (TO3i) | Connection recovery, entire. `recover` is in the `Options` signature, stored, and given a property and a setter (`options.py:30,111,193,196`), and read nowhere. No `Connection#createRecoveryKey`, no `recover` connect parameter, no recovery-key decoding. Measured through the proxy: a client built with a valid `recover=` opened a `ws_connect` whose query parameters were `{'accessToken': …, 'v': '5'}` — no `recover` — and was given a fresh `connectionId`. RTN16l is otherwise fully compliant, taking the proxy's `recovery-failed-new-id`, `recovery-failed-new-key` and error 80008 and staying CONNECTED; only the absent parameter fails it | 8 | | RTL22, RTL22a–d, MFI1, MFI2a–e | `MessageFilter`. `RealtimeChannel.subscribe` (`channel.py:262-273`) accepts only a `str` or a callable, and there is no filter type of any shape to spell. Each test builds its filter through the module's `message_filter()` helper, which is the one place to repoint when the type lands | 5 | | RTS5, RTS5a, RTS5a1, RTS5a2, DO2a | Derived channels: `DeriveOptions` and `Channels.getDerived`. `grep -r derive ably/` is empty. Each test imports `DeriveOptions` inside its body so the module still loads | 5 | | RTB1, RTB1a, RTB1b | Retry backoff, jitter and `retryIn`. Retry timers schedule the flat configured timeout (`connectionmanager.py:753`, `channel.py:866-871`); `grep` for jitter/backoff/retry_in returns nothing, and neither `ConnectionStateChange` nor `ChannelStateChange` carries `retryIn` | 4 | @@ -650,13 +817,13 @@ against them. | RTL10b | `untilAttach` on `RealtimeChannel#history`. The realtime channel does not override `history`, so the call lands on `Channel.history`, which takes only `direction`, `limit`, `start` and `end`. No `fromSerial` is ever sent, and the attach serial the channel does record is private and read nowhere | 1 | | TB3 | `ChannelOptions.withCipherKey`. The nearest equivalent, `ably.util.crypto.get_default_params({'key': key})`, is not reachable from `ChannelOptions` | 1 | | RTL2i, TH6 | `hasBacklog` on `ChannelStateChange`. `Flag.HAS_BACKLOG` is defined (`types/flags.py:7`) but `_on_message` reads only RESUMED and HAS_PRESENCE (`channel.py:715-721`). See the UTS Spec Error above: the features spec makes this optional | 1 | -| RTP6b | Subscribing to an *array* of presence actions. The list reaches `EventEmitter.on` and pyee uses the event as a dict key, so `presence.subscribe([ENTER, LEAVE], listener)` raises `TypeError: unhashable type: 'list'`. A fix has to fan the list out into one registration per action, and `unsubscribe` with it | 1 | +| RTP6b | Subscribing to an *array* of presence actions. The list reaches `EventEmitter.on` and pyee uses the event as a dict key, so `presence.subscribe([ENTER, LEAVE], listener)` raises `TypeError: unhashable type: 'list'`. A fix has to fan the list out into one registration per action, and `unsubscribe` with it. `realtime/integration/presence_lifecycle_test.py::test_rtp4_bulk_enter_observed` adapts instead of gating: it registers the one listener once for `'enter'` and once for `'present'`, which is what the array form means, and the counted set is identical | 1 | | RSL1i | REST publish never calls `validate_message_size`. The helper exists and is correct, but only `ably/realtime/channel.py:423` calls it, so an oversized REST publish goes out | 1 | | TP5 | `size` on `PresenceMessage`. The related `maxMessageSize` gap is adapted rather than gated, below; `features.md` TM6 has no UTS test | 1 | ### Connection -#### A connection-level ERROR bypasses everything that matters on a failure — 4 tests +#### A connection-level ERROR bypasses everything that matters on a failure — 5 tests **Spec points:** RTL3a, RTN7e. @@ -678,12 +845,38 @@ incompatible `clientId` (`:422`) and the two authorize failures (`:483`, `:487`) is specific to the ERROR path. Two batches found the two halves independently; **it is one issue, not two.** +The same shape holds against the real server. An ERROR 50000/500 injected onto an +established connection through `uts-proxy` took the connection to FAILED with the right +`error_reason` and made no further `ws_connect`, while both attached channels stayed +**ATTACHED with `error_reason is None`** and emitted no state change — the first +observation of the channel half against the sandbox rather than a mock. + +The left-running timers are observable too, on any connection the server fails with an +ERROR. A client whose configured `clientId` does not match its token reaches FAILED on the +server's 40102/401 and then emits a **further** state change about ten seconds afterwards: +`DISCONNECTED, 50003/504 "Connection cancelled due to request timeout"`. The CONNECTING +transition timer started at `:579` is still running, and `on_transition_timer_expire` +(`:711-718`) calls `notify_state` when it fires. Measured sequence: + +``` +connecting → disconnected 80019/401 'Client configured authentication provider request failed' +connecting → failed 40102/401 'invalid clientId for credentials' + → disconnected 50003/504 'Connection cancelled due to request timeout' +``` + +No test of the suite asserts that last transition, because each asserts within its own +wait, but anyone writing a test that sits on a FAILED connection for longer than +`realtimeRequestTimeout` will meet it. + **Tests affected:** `test_rtl3a_failed_attached_to_failed` and `test_rtl3a_failed_attaching_to_failed` fail on `assert channel.state == ChannelState.FAILED` with `attached` and `attaching`; `test_rtn7e_pending_fail_failed` and `test_rtn7e_error_represents_reason` fail with `asyncio.TimeoutError` from the bounded await on the publish. `test_rtl3a_other_states_unaffected` passes, but only because nothing happens at all; it becomes a real test of RTL3a once this is fixed. +`realtime/integration/proxy/connection_resume_test.py::test_rtn15j_fatal_error_established_conn` +is the fifth, and the one that reaches the defect through a real transport: it fails on the +channel state its two attached channels never leave. Note that `client.connection.error_reason` *is* populated correctly with the ERROR's 80019/400, so the reason RTN7e asks for is in hand at the point a fix would need it. The @@ -725,7 +918,7 @@ and FAILED — closes both. **Status:** open bug. -#### A DISCONNECTED carrying a 5xx with no fallback hosts stalls the connection — 1 test +#### A DISCONNECTED carrying a 5xx with no fallback hosts stalls the connection — 2 tests **Spec point:** RTN15h3. @@ -743,7 +936,14 @@ the empty list these unit tests use — is stranded. The specification's own fix `code: 80003, statusCode: 503`. **Tests affected:** `test_rtn15h3_non_token_error_resume` — -`Timed out waiting for connection state connecting; it was connected`. +`Timed out waiting for connection state connecting; it was connected` — and +`realtime/integration/proxy/connection_resume_test.py::test_rtn15h3_non_token_error_reconnects`, +which reaches the same place through a real transport. Measured there: an injected +`{action: 6, error: {code: 80003, statusCode: 500}}` followed by a socket close produces +**no state change at all**, CONNECTED being held for the whole ten-second wait with +`error_reason is None`, while the proxy log records `ws_disconnect initiator=proxy`. What +empties `__fallback_hosts` in that test is `endpoint='localhost'` (REC2c2), so the +stranding is reached from an ordinary client configuration rather than from a test fixture. **Status:** open bug. This is the most serious connection-level defect found. @@ -771,6 +971,44 @@ lists among the errors that must set it. **Status:** open bug, and a one-line fix: pass `reason=exception` into the `ConnectionStateChange`. +#### `errorReason` is not cleared by a successful reconnect — 1 test + +**Spec points:** RTN14b (`realtime/proxy/RTN14b/token-error-renew-reconnect-0`), and RTN25 +(`realtime/unit/RTN25/error-reason-cleared-on-connect-4`). + +`Connection._on_state_update` assigns `__error_reason` only when the incoming change carries +a reason, and the only place that clears it is `Connection.connect()` — which an automatic +retry, driven through `ConnectionManager.request_state`, does not go through. The connection +manager tells the same story: `on_token_error` (`connectionmanager.py:458`) assigns +`self.__error_reason`, and the only other writer, `enact_state_change` (`:168-169`), assigns +it only when `reason` is truthy. A successful CONNECTED carries no reason, so nothing +overwrites it and nothing resets it, and the error that caused the drop is still readable +once the connection is back. + +RTN14b is what makes this a defect rather than a choice. It requires that after the SDK +meets a 40142 while opening a connection, renews its token and reaches CONNECTED, +`connection.errorReason` is null, and offers no alternative reading. Measured through the +proxy: `state=connected`, states `[CONNECTING, DISCONNECTED, CONNECTING, CONNECTED]`, +`auth_calls=2`, `ws_connects=2`, and `error_reason: code=40142 status=401 'Token expired'` +still in place at the end. + +RTN25 is the nuance, and it stands. Its own test names `errorReason IS null` as the primary +assertion while explicitly sanctioning the alternative — "errorReason is kept but clearly +not relevant to current state (Implementation-specific behavior)" — and `features.md` RTN25 +says only when `errorReason` is *set*, never when it is cleared. So +`test_rtn25_error_reason_cleared_on_connect` asserts the retained error, the +specification's option B, with option A in a comment, and is an adapted test rather than a +gated one. What settles the question is the second specification requiring the clearing +that the first merely permitted. + +**Tests affected:** +`realtime/integration/proxy/connection_open_failures_test.py::test_rtn14b_token_error_renew_reconnect` +— `assert AblyAuthException() is None`. + +**Status:** open bug. Clearing `__error_reason` on entry to CONNECTED closes it, and turns +the adapted RTN25 unit test round to the specification's option A, so that test's +adaptation goes with the fix. + #### `ping()` rejects DISCONNECTED instead of deferring, and charges the wait to the caller — 3 tests **Spec points:** RTN13b, RTN13c, RTN13d. @@ -971,7 +1209,7 @@ DETACH-message count. **Status:** open bug. -#### A decode error other than 40018 has no channel-level handling — 1 test +#### A decode error other than 40018 has no channel-level handling — 2 tests, 3 cases **Spec point:** PC3. @@ -989,8 +1227,28 @@ reported on it. Two causes, both verified: "Message processing error … Skip messages" and skips the batch silently, with no state change and no `error_reason`. +The second cause is what the integration tier reaches, because the server's deltas are +real: the first message on the channel is sent whole, so the decoding context is populated +and the reference check passes, and the **second** message is a genuine vcdiff delta. +`Message.from_encoded_array` then raises `AblyException(…, 40019)` from +`ably/types/mixins.py:81-83` as it should, `RealtimeChannel._on_message` +(`ably/realtime/channel.py:738-749`) takes the generic `else`, and the batch is logged and +dropped. `from_encoded_array` rolls the decoding context back as it goes, so the *next* +delta's `extras.delta.from` no longer matches `context.last_message_id`, that raises 40018, +and the channel goes round the RTL18 recovery instead. Measured: the channel stays ATTACHED +throughout and ends carrying 40018 rather than the 40019 PC3 asks for. + +``` +ERROR ably.types.mixins: Message cannot be decoded as no VCDiff decoder available +ERROR ably.realtime.channel: Message processing error 40019 40019 VCDiff decoder not available. Skip messages +ERROR ably.realtime.channel: VCDiff decode failure: 40018 400 Delta message decode failure - previous message not available +``` + **Tests affected:** `test_pc3_no_plugin_fails` — "Timed out waiting until the channel fails -for want of a vcdiff decoder". +for want of a vcdiff decoder" — and +`realtime/integration/delta_decoding_test.py::test_pc3_no_plugin_causes_failed`, which is +one Test ID run once per protocol and fails both times with +`Timed out waiting for channel state failed; it was attached`. **Status:** open bug. The delta-reference check needs to run after the decoder-availability check, and a decode error needs to fail the channel whatever its code. @@ -1141,6 +1399,33 @@ on `resumed`, with `auto-reenter failed: 40160 401 Presence denied` in the log. **Status:** open bug. `AblyException` already carries a `cause`, so the fix is local to this one method. +#### RTP17i re-entry never runs on a channel that is already ATTACHED — 1 test + +**Spec points:** RTP17i, RTP17g. + +RTP17i requires automatic re-entry whenever a channel receives an ATTACHED +ProtocolMessage, except where the channel is already attached **and** the RESUMED flag is +set. `RealtimePresence.on_attached()`, which carries out the RTP17g re-entry, is reached +only from `RealtimeChannel._notify_state()`, and `_on_message` +(`ably/realtime/channel.py:723-728`) sends an ATTACHED received while already ATTACHED +down the RTL12 branch, which emits an `update` event and never calls `_notify_state`. The +re-entry path is therefore unreachable on an attached channel whatever the RESUMED flag +says. Measured through `uts-proxy`: injecting +`{action: 11, flags: 0, error: {code: 91001}}` onto an attached channel holding one entered +member produces no further client→server PRESENCE frame at all — the proxy log shows the +injected frame arriving and the channel staying ATTACHED. + +This is invisible to the unit tier, whose RTP17i cases all drive a dropped transport, so +the channel passes through ATTACHING first and `_notify_state` runs. + +**Tests affected:** +`realtime/integration/proxy/presence_reentry_test.py::test_rtp17i_reenter_on_non_resumed` — +`Timed out after 10.0s waiting for a re-enter PRESENCE frame`. Its sibling +`test_rtp17i_reenter_after_disconnect` passes, for exactly that reason. + +**Status:** open bug. A fix has to reach the re-entry from the RTL12 branch too, gated on +the RESUMED flag being clear. + #### A new sync sequence does not discard the in-flight one — no test **Spec point:** RTP18a. @@ -1183,6 +1468,43 @@ two aware `datetime.now(timezone.utc)` readings and passes. ### Auth +#### A JWT string with a matching clientId is rejected 40102 — 1 test + +**Spec point:** RSA7, `realtime/integration/RSA7/matching-clientid-succeeds-0`. + +RSA7 requires that a token whose `clientId` equals the client's configured `clientId` is +accepted. `Auth._ensure_valid_auth_credentials` calls +`self._configure_client_id(self.__token_details.client_id)` after every token fetch +(`ably/rest/auth.py:126`). Where an `auth_callback` returns a **JWT string**, +`request_token` wraps it as `TokenDetails(token=)` (`auth.py:213`) without parsing the +JWT, so `token_details.client_id` is `None`. `_configure_client_id(None)` then finds +`None != 'test-client-…'` and raises `IncompatibleClientIdException(…, 400, 40102)` +(`auth.py:345`), although the JWT's own `x-ably-clientId` claim is exactly the configured +id. Measured against the sandbox, one auth-callback invocation: + +``` +initialized→connecting None +connecting→disconnected AblyAuthException 80019/401 'Client configured authentication provider request failed' +disconnected→connecting None +connecting→connected None +``` + +The connection does arrive in the end, because `__token_details` is assigned before the +raise: the retry takes the cached-token branch and skips `_configure_client_id` altogether. +The observable cost is a spurious failed attempt and a full `disconnected_retry_timeout` — +fifteen seconds — on a clientId that matched. + +**Tests affected:** +`realtime/integration/auth_test.py::test_rsa7_matching_clientid_succeeds` — +`AssertionError: Timed out waiting for connection state connected; it was disconnected`. +The sibling `test_rsa7_mismatched_clientid_fails` is not gated and pays the same fifteen +seconds before the server's genuine rejection arrives, which is why it waits twenty +seconds rather than the default ten. + +**Status:** open bug. A fix either reads the `x-ably-clientId` claim when wrapping a bare +token string, or leaves `_configure_client_id` alone where the fetched token carries no +clientId of its own. + #### An authCallback error is always rewritten as 401/40170, so RSA4d is unreachable — 4 tests **Spec points:** RSA4d, RSA4d1. @@ -1351,7 +1673,11 @@ suite carries the same workaround for the same reason, commented "Use wildcard a enterClient" (`test/ably/realtime/realtimepresence_test.py:394-396`). It is setup rather than subject: those tests are about `batchPresence`, and nothing they assert depends on how the members got there. A fix would let the setup halves be written exactly as the -specification writes them. +specification writes them. Both `realtime/integration` presence specifications carry the +same workaround for the same reason — `presence_lifecycle_test.py::test_rtp4_bulk_enter_observed` +and `presence/presence_sync_test.py::test_rtp2_sync_multiple_members` each build the client +that enters members on behalf of others with `client_id='*'` — which makes five tests +across two tiers that would otherwise be written as their specifications write them. This is distinct from the wildcard-clientId contradiction recorded under UTS Spec Errors, which is about a client that *does* configure `clientId: "*"`. @@ -1451,7 +1777,7 @@ through an internal object to get at a value the specification makes public. | Spec points | Missing accessor | What the test reads instead | Tests | |---|---|---|---| -| RTN3, RTN8, RTN8a, RTN8b, RTN8d, RTN9, RTN9a, RTN9b, RTN9d | `Connection#id` and `Connection#key`. `Connection` exposes `state`, `error_reason`, `connection_manager` and `connection_details` only | `connection.connection_manager.connection_id` and `connection.connection_details.connection_key`, which is `None` whenever the key would be. Each file defines `connection_id(client)` / `connection_key(client)` at the top | 8 in `connection_id_key_test.py`, plus ~11 across the auth, failures and liveness suites | +| RTN3, RTN8, RTN8a, RTN8b, RTN8d, RTN9, RTN9a, RTN9b, RTN9d | `Connection#id` and `Connection#key`. `Connection` exposes `state`, `error_reason`, `connection_manager` and `connection_details` only | `connection.connection_manager.connection_id` and `connection.connection_details.connection_key`, which is `None` whenever the key would be. Each file defines `connection_id(client)` / `connection_key(client)` at the top | 8 in `connection_id_key_test.py`, plus ~11 across the auth, failures and liveness suites, and eight modules of `realtime/integration` — `auth_test.py`, `auth/token_request_test.py`, `channels/channel_publish_test.py`, `connection_lifecycle_test.py` and four under `proxy/` — each defining the same file-local readers, `connection_resume_test.py` adding a `recovery_key(client)` for `createRecoveryKey()` | | RTL15 | `RealtimeChannel#properties`, a `ChannelProperties` holding `attachSerial` and `channelSerial`. There is no `properties` attribute and no such type | the name-mangled `__attach_serial` and `__channel_serial` (`channel.py:66-67`), through `attach_serial(channel)` / `channel_serial(channel)` defined in the file | 10 in `channel_properties_test.py`, of which 4 are gated for behaviour above | | RTN26, RTN26a, RTN26b | `Connection#whenState(state, listener)` — the SDK has `Connection._when_state(state)`, private, returning an awaitable | the awaitable, driven as a task through a `when_state(connection, state)` helper. Both branches are correct: already in the state it resolves with `None`, otherwise it is a `once` registration that resolves for the first entry only | 6 in `when_state_test.py` | | RTL5, RTL2, RTL2d, RTL2g, RTL12, TH5 | `ChannelStateChange#event`, and a `ChannelEvent` type. `ChannelStateChange` is `(previous, current, resumed, reason)` | the event is the key a listener is registered against, so each test registers on `ChannelState.ATTACHING` / `ATTACHED` / `'update'` and receiving the change at all *is* the `event` assertion | 7 across `channel_state_events_test.py` and `channel_detach_test.py` | @@ -1614,6 +1940,19 @@ then advances the `FakeClock` to the 120000 default instead, either directly or `advance_to_connection_state`. Because the clock is notional this costs nothing in wall time: the three connection tests take 0.06 s, 0.09 s and 0.08 s. Each says so in a comment. +The integration tier has no clock to advance, so there the same root cause is a gated +failure rather than an adaptation. +`realtime/integration/proxy/connection_resume_test.py::test_rtn14h_resume_after_ttl_expiry` +has `uts-proxy` replace the first CONNECTED with one carrying `connectionStateTtl: 2000`, +drops the socket and refuses the retry. Measured: the replacement did reach +`connection_details.connection_state_ttl == 2000`, and SUSPENDED still did not arrive +within 45 s — the client sat DISCONNECTED and reconnected on the fifteen-second +`disconnected_retry_timeout`. Honouring the TTL will not on its own turn that test green. +`#### The connection id, key and details are cleared on SUSPENDED` means a suspended client +stops sending `resume`, which is precisely what RTN14h asserts it still does, and +`"connectionKey": "__PASSTHROUGH__"` in the fixture is not substituted by uts-proxy v0.3.0 +either — see the UTS Spec Error above. The three want closing together. + **Status:** open bug. It is the reason the fake clock exists in this suite at all; see the fake-time section at the end of this file. @@ -1688,26 +2027,6 @@ POSTed to `/keys/{keyName}/revokeTokens` directly, reproduced three times. RSA4a does not say which state a token error observed mid-connection should be reported in, and other SDKs may report DISCONNECTED first and fail afterwards. -### `errorReason` is not cleared by a successful reconnect - -**Spec point:** RTN25, `realtime/unit/RTN25/error-reason-cleared-on-connect-4`. - -`Connection._on_state_update` assigns `__error_reason` only when the incoming change carries -a reason, and the only place that clears it is `Connection.connect()` — which an automatic -retry, driven through `ConnectionManager.request_state`, does not go through. So a -DISCONNECTED error is still readable after the connection comes back. - -The test's primary assertion is `errorReason IS null`, while explicitly sanctioning the -alternative, "errorReason is kept but clearly not relevant to current state -(Implementation-specific behavior)". `features.md` RTN25 says only when `errorReason` is -*set*, never when it is cleared, so neither reading is wrong. - -**Tests affected:** `test_rtn25_error_reason_cleared_on_connect` asserts the retained error — -the specification's option B — with option A in a comment. - -**Status:** intentional, and permitted. Worth raising against the UTS spec instead, which -should pick one reading: a test that accepts either provides no signal. - ### A pending `attach()` resolves, rather than failing, when the connection closes **Spec points:** RTL3b, RTL4d. @@ -2095,8 +2414,11 @@ all — no state change, no retry, and the client goes on believing it is connec socket the server has closed. `on_disconnected` (`connectionmanager.py:437-450`) routes 500–504 to the fallback path and, with an empty fallback list, logs and falls out of the `if`/`elif` chain without calling `notify_state`. Any client with a custom endpoint, a local -cluster, or `fallback_hosts=[]` is affected, and there is no way back. +cluster, or `fallback_hosts=[]` is affected, and there is no way back. Confirmed against +the sandbox through `uts-proxy`, where the stall is reached from `endpoint='localhost'` +alone: CONNECTED held for the whole wait, `error_reason is None`, no retry. `test/uts/realtime/unit/connection/connection_failures_test.py -k rtn15h3` +`test/uts/realtime/integration/proxy/connection_resume_test.py -k rtn15h3` **1.2 A connection-level ERROR skips every failure action.** RTL3a, RTN7e. `on_error` ends with `enact_state_change(FAILED, …)` (`connectionmanager.py:477`) instead of @@ -2106,8 +2428,13 @@ ATTACHED or ATTACHING with a null `error_reason` and no state change; a pending never returns; pending publishes never resolve **or** reject; the transition and suspend timers keep running. Every other route to FAILED goes through `notify_state` and is correct, so the fix is one line. Two batches found the two halves independently; it is one issue. +Confirmed end to end against the sandbox: an injected ERROR 50000/500 left both attached +channels ATTACHED with a null `error_reason` and no state change, and the left-running +transition timer surfaces as a spurious `DISCONNECTED 50003/504` about ten seconds after a +connection fails. `test/uts/realtime/unit/channels/channel_connection_state_test.py -k rtl3a` `test/uts/realtime/unit/channels/channel_publish_pending_test.py -k rtn7e` +`test/uts/realtime/integration/proxy/connection_resume_test.py -k rtn15j` **1.3 `detach()` never returns when the connection is not CONNECTED.** RTL5l. RTL5l requires an immediate transition to DETACHED. `detach()` requests DETACHING, @@ -2377,7 +2704,7 @@ Each row is one feature and one issue. None is a bug in existing code. | Feature | Spec points | Tests | Reproduction (`-k` against `test/uts/realtime/unit/`) | |---|---|---|---| -| Connection recovery, entire — `createRecoveryKey`, the `recover` connect parameter, recovery-key decoding. `recover` is stored on `Options` and read nowhere | RTN16, RTN16f–k, RTC1c | 6 | `connection/connection_recovery_test.py`, `client/realtime_client_test.py -k rtc1c` | +| Connection recovery, entire — `createRecoveryKey`, the `recover` connect parameter, recovery-key decoding. `recover` is stored on `Options` and read nowhere. Measured through the proxy: a client given a valid `recover=` sends no `recover` query parameter and is issued a fresh `connectionId` | RTN16, RTN16f–k, RTC1c | 8 | `connection/connection_recovery_test.py`, `client/realtime_client_test.py -k rtc1c`, and `test/uts/realtime/integration/proxy/connection_resume_test.py -k "rtn16d or rtn16l"` | | `MessageFilter` and filtered subscriptions | RTL22, RTL22a–d, MFI1, MFI2a–e | 5 | `channels/channel_subscribe_test.py -k rtl22` | | Derived channels — `DeriveOptions`, `Channels.getDerived` | RTS5, RTS5a, RTS5a1, RTS5a2, DO2a | 5 | `channels/channel_options_test.py -k "rts5 or do2a"` | | Retry backoff, jitter and `retryIn` on both state-change types | RTB1, RTB1a, RTB1b | 4 | `connection/backoff_jitter_test.py` | @@ -2408,12 +2735,20 @@ none of these shows up as a failure — which is why they are easy to lose. ### From the integration tier -The five tiers above classify the realtime derivation; the REST unit derivation's +The five tiers above classify the realtime unit derivation; the REST unit derivation's candidates went upstream as [#709](https://github.com/ably/ably-python/issues/709)–[#712](https://github.com/ably/ably-python/issues/712). -Three further defects came out of `rest/integration`, and none of them is filed. The -first two are tier 2 by the ranking above — an error where there should be none. The -third is tier 1 for a client that configures the option it concerns, since the call does -not come back when the caller asked for it to. +Six further defects came out of the two integration tiers, and none of them is filed. I.1 +and I.2 are tier 2 by the ranking above — an error where there should be none — and I.3 is +tier 1 for a client that configures the option it concerns, since the call does not come +back when the caller asked for it to. I.4 and I.6 are tier 2 as well; I.5 is tier 3, a +presence member silently leaving the set with nothing to report it. + +The realtime integration tier also put four defects the unit tier had already recorded in +front of the real server, and those extend the entries above rather than opening issues of +their own: the RTN15h3 stall (1.1), the connection-level ERROR (1.2), the unused +`connectionStateTtl`, and connection recovery (tier 4). A fifth, the missing channel-level +handling for a decode error other than 40018, is reached there through a genuine +server-sent delta rather than through the specification's unreachable fixture. **I.1 `Rest#request` never renews an expired token.** RSC10, RSC19. Every other REST operation renews and retries on a 40140–40149; `Rest#request` returns the 401 to the @@ -2448,6 +2783,37 @@ no timeout and no fallback at all. Distinct from bounding a single socket read rather than the request; a fix wants to settle both. `test/uts/rest/integration/proxy/rest_fallback_test.py -k rsc15l2` +**I.4 A JWT string whose clientId matches the configured one is rejected 40102.** RSA7. +An `auth_callback` returning an Ably JWT gets wrapped as `TokenDetails(token=)` +without the JWT being parsed (`ably/rest/auth.py:213`), so `token_details.client_id` is +`None`; `_configure_client_id(None)` (`:126`, raising at `:345`) then reads that as a +clientId change and raises 40102 although the JWT's `x-ably-clientId` claim is the +configured id exactly. The connection recovers on the retry, which takes the cached-token +branch, so the cost is a spurious failed attempt and a full `disconnected_retry_timeout` — +fifteen seconds — on every such client. Affects anyone whose auth provider returns a JWT +and who also configures `clientId`, which is the ordinary shape of a JWT deployment. +`test/uts/realtime/integration/auth_test.py -k rsa7_matching` + +**I.5 RTP17i re-entry never runs on a channel that is already ATTACHED.** RTP17i, RTP17g. +`RealtimePresence.on_attached()` is reached only from `RealtimeChannel._notify_state()`, and +an ATTACHED arriving on an already-attached channel takes the RTL12 branch +(`ably/realtime/channel.py:723-728`), which emits `update` and never calls `_notify_state`. +So a server that re-attaches a channel without the RESUMED flag — the case RTP17i exists +for — gets no re-entry, and the member is gone from the presence set with nothing raised +anywhere. The unit tier cannot see it, because its RTP17i cases all pass through ATTACHING. +Adjacent to [#658](https://github.com/ably/ably-python/issues/658), which is the other half +of RTP17 automatic re-entry. +`test/uts/realtime/integration/proxy/presence_reentry_test.py -k rtp17i_reenter_on_non_resumed` + +**I.6 `errorReason` survives a successful reconnect, which RTN14b forbids.** RTN14b, RTN25. +Nothing clears `Connection#errorReason` on entry to CONNECTED: `enact_state_change` +(`connectionmanager.py:168-169`) writes it only when the state change carries a reason, and +a successful CONNECTED carries none. After the SDK meets a 40142 opening a connection, +renews its token and connects, the 40142 is still there. RTN25 permits either reading and +the unit tier adapts to that; RTN14b does not, which is what makes this a defect rather +than a choice. +`test/uts/realtime/integration/proxy/connection_open_failures_test.py -k rtn14b` + ## How the specifications are adopted here Choices about the approach, as against the behaviour recorded above. @@ -2702,23 +3068,29 @@ outright. Only wrong behaviour is gated. -### The integration tier runs against one provisioned sandbox app, once per protocol +### Each integration tier runs against a provisioned sandbox app of its own, once per protocol -`uts/rest/integration` is the first tier with a server behind it, and three harness -choices follow from that. +`uts/rest/integration` and `uts/realtime/integration` are the tiers with a server behind +them, and three harness choices follow from that. -The app is provisioned once for the whole tier and deleted at the end, which is the +Each tier provisions one app for itself and deletes it at the end, which is the specifications' `BEFORE ALL TESTS` / `AFTER ALL TESTS` pair; a fresh app per test would -make the tier several times slower and invite the sandbox's rate limiting. -`sandbox.key(0)` is the full-access key the specifications call `api_key`, and the other -indices are the capabilities each specification's app-provisioning section names. +make the tiers several times slower and invite the sandbox's rate limiting. The REST tier's +app arrives as the `sandbox` fixture and the realtime tier's as `realtime_sandbox`, each +provisioned and named separately, so a realtime test entering presence or publishing to a +channel cannot be seen by a REST test reading the same channel name. `key(0)` is the +full-access key the specifications call `api_key`, and the other indices are the +capabilities each specification's app-provisioning section names. A specification carrying a `## Protocol Variants` section runs each of its Test IDs twice, through a `use_binary_protocol` fixture parametrized `[False, True]` with the ids `json` -and `msgpack`. Five of the twelve specifications carry that section, so 38 of the 84 -integration Test IDs are two pytest cases each. The seven that do not are json only and -take the default `sandbox_rest_client` applies. This is why the counts in this file give -Test IDs and cases separately. +and `msgpack`; each tier defines its own. Five of the twelve REST specifications carry that +section, so 38 of the 84 REST integration Test IDs are two pytest cases each, and five of +the twenty realtime ones do — `channel_history_test.md`, `channels/channel_publish_test.md`, +`delta_decoding_test.md`, `mutable_messages_test.md` and `presence_lifecycle_test.md` — so +22 of the 73 realtime Test IDs are. The rest are json only and take the default their +tier's `sandbox_rest_client` or `sandbox_realtime_client` applies. This is why the counts +in this file give Test IDs and cases separately. An autouse fixture closes every client a test built, whether or not its assertions held (`test/uts/conftest.py`, `close_open_clients`). The specifications write @@ -2729,8 +3101,11 @@ above. Tests omit the inline close and leave it to teardown. ### The proxy package runs against a pinned proxy, with a session per test `uts/docs/proxy.md` puts `ably/uts-proxy` between the client and the sandbox for the -specifications under `rest/integration/proxy`, and three harness choices follow from -having to supply the proxy itself. +specifications under `rest/integration/proxy` and `realtime/integration/proxy`, and three +harness choices follow from having to supply the proxy itself. The two packages share +`test/uts/helpers/proxy.py` and each repeats the `proxy_control` / `proxy_session` pair, +including the `append=False` on the timeout marker that keeps the package's 300 seconds +ahead of its parent's 120. The release is pinned and verified rather than built or assumed present. The archive for the machine is downloaded on first use, checked against the sha256 the release publishes, @@ -2751,7 +3126,20 @@ twenty-second delay before it reads anything; the package's per-test pytest time wait on. Every client in the package authenticates through an `authCallback` whose own request goes straight to the sandbox: the session speaks plain HTTP, RSC18 refuses basic auth over it, and a token request routed through the session would be counted by the -assertions that count requests. +assertions that count requests. A realtime connection carries its credentials in the +WebSocket's query string rather than in an Authorization header, so a `key=` does reach the +session over plain `ws://`; the realtime modules still prefer a locally signed Ably JWT +wherever the scenario is not about authentication, because signing one costs no round trip +and so leaves nothing in the event log beside the frames a test counts. + +A realtime client reaches its session exactly as a REST one does — `endpoint='localhost'`, +`port=session.proxy_port`, `tls=False`, `use_binary_protocol=False` — and that rests on +`WebSocketTransport` interpolating `Defaults.get_port(self.options)` into the URL it opens, +so that the `port` and `tlsPort` client options (TO3k4, TO3k5) the REST layer honours reach +the websocket too. Without it a realtime client can be pointed at no port but the default, +and the realtime half of this tier cannot run at all. That is a fix in the SDK rather than +a deviation, so it has no entry above — entries closed by a fix are removed rather than +kept as history — but it is what the tier stands on. ### A hedged integration setup is provisioned so its guarded assertions bite @@ -2792,7 +3180,17 @@ reported as a defect and then refuted. The integration round found four gaps the tier had already recorded, which extend the rows they belong to rather than opening new ones: `Auth#revokeTokens`, `Rest#batchPresence`, the `PushChannel` surface and the `clientId` filter on `RestPresence#get`. The revoked-token 40171 it observed is the -same root cause as RTN15h1's, and sits in that entry. +same root cause as RTN15h1's, and sits in that entry. The realtime integration round did +the same for five more: the RTN15h3 stall, the connection-level ERROR bypass, the unused +`connectionStateTtl`, connection recovery, and the missing channel-level handling for a +decode error other than 40018. + +A verdict can change the same way, when a second specification reaches a behaviour the +first was content with. `errorReason` surviving a successful reconnect is permitted by +RTN25, whose test names either reading, and forbidden by RTN14b, which names one; so the +entry sits under *Failing Tests* and the RTN25 unit test that asserts the retained error +stays an adaptation. Where two specifications differ in strength, the entry follows the +stronger. So the per-area files are merged into this file and deleted, and the comments in the tests that pointed at them point here instead. @@ -2805,13 +3203,13 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 206 failures and 15 skips with the variable set, and -221 skips and 1032 passes without it, the 1032 being 910 derived cases and 122 +it. As of this writing that is 217 failures and 15 skips with the variable set, and +232 skips and 1124 passes without it, the 1124 being 1002 derived cases and 122 `helpers/` ones. The other two counts are measured from the source rather than from a run. The number of -**derived tests** is the number of `# UTS:` comments, 1060. The number of **Test IDs** is -the number of *distinct* ids in them, 1051 — not the same figure, because five ids in +**derived tests** is the number of `# UTS:` comments, 1141. The number of **Test IDs** is +the number of *distinct* ids in them, 1132 — not the same figure, because five ids in `rest/unit` are carried by more than one test function. Counting the comments and calling the result Test IDs is the easy mistake here, and it overstates the specification coverage by nine.