diff --git a/app/config/locale/translations/ar.json b/app/config/locale/translations/ar.json index df077c8685b..2ac479d72e2 100644 --- a/app/config/locale/translations/ar.json +++ b/app/config/locale/translations/ar.json @@ -16,8 +16,8 @@ "emails.magicSession.signature": "فريق {{project}}", "emails.recovery.subject": "تغيير كلمة السر", "emails.recovery.hello": "أهلا {{user}}،", - "emails.recovery.body": "برجاء اتباع الراط التالي لتغيير كلمة السر الخاصة بـ{{project}}", - "emails.recovery.footer": "لولم تطلب تغيير كلمة السر، يمكنك تجاهل هذه الرسالة", + "emails.recovery.body": "برجاء اتباع الرابط التالي لتغيير كلمة السر الخاصة بـ{{project}}", + "emails.recovery.footer": "لو لم تطلب تغيير كلمة السر، يمكنك تجاهل هذه الرسالة", "emails.recovery.thanks": "شكرا،", "emails.recovery.buttonText": "إعادة تعيين كلمة المرور", "emails.recovery.signature": "فريق {{project}}", diff --git a/app/config/oAuthProviders.php b/app/config/oAuthProviders.php index c6fe102c54a..4de082b6740 100644 --- a/app/config/oAuthProviders.php +++ b/app/config/oAuthProviders.php @@ -478,6 +478,17 @@ 'mock' => false, 'class' => 'Appwrite\\Auth\\OAuth2\\Twitch', ], + 'webflow' => [ + 'name' => 'Webflow', + 'developers' => 'https://developers.webflow.com/data/reference/oauth-app', + 'icon' => 'icon-webflow', + 'enabled' => true, + 'sandbox' => false, + 'form' => false, + 'beta' => false, + 'mock' => false, + 'class' => 'Appwrite\\Auth\\OAuth2\\Webflow', + ], 'wordpress' => [ 'name' => 'WordPress', 'developers' => 'https://developer.wordpress.com/docs/oauth2/', diff --git a/app/controllers/api/account.php b/app/controllers/api/account.php index ec54f2ad854..1967024c479 100644 --- a/app/controllers/api/account.php +++ b/app/controllers/api/account.php @@ -600,22 +600,21 @@ ->inject('response') ->inject('targetUser') ->inject('locale') - ->inject('store') - ->inject('proofForToken') - ->action(function (Response $response, User $targetUser, Locale $locale, Store $store, ProofsToken $proofForToken) { + ->inject('session') + ->action(function (Response $response, User $targetUser, Locale $locale, ?Document $current) { $sessions = $targetUser->getAttribute('sessions', []); // While impersonating, the request runs on the impersonator's session, so none of // the target's sessions is marked current. - $current = $targetUser->sessionVerify($store->getProperty('secret', ''), $proofForToken); + $currentId = $current?->getId(); foreach ($sessions as $key => $session) { /** @var Document $session */ $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session->setAttribute('countryName', $countryName); - $session->setAttribute('current', ($current == $session->getId()) ? true : false); + $session->setAttribute('current', $currentId === $session->getId()); $session->setAttribute('secret', $session->getAttribute('secret', '')); $sessions[$key] = $session; @@ -734,16 +733,15 @@ ->inject('response') ->inject('targetUser') ->inject('locale') - ->inject('store') - ->inject('proofForToken') - ->action(function (?string $sessionId, Response $response, User $targetUser, Locale $locale, Store $store, ProofsToken $proofForToken) { + ->inject('session') + ->action(function (?string $sessionId, Response $response, User $targetUser, Locale $locale, ?Document $current) { $sessions = $targetUser->getAttribute('sessions', []); // While impersonating, the request runs on the impersonator's session, so 'current' // resolves against none of the target's sessions and this throws. That matches the // sessions list, which marks none of them current for the same reason. $sessionId = ($sessionId === 'current') - ? $targetUser->sessionVerify($store->getProperty('secret', ''), $proofForToken) + ? $current?->getId() : $sessionId; foreach ($sessions as $session) { @@ -752,7 +750,7 @@ $countryName = $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown')); $session - ->setAttribute('current', ($proofForToken->verify($store->getProperty('secret', ''), $session->getAttribute('secret')))) + ->setAttribute('current', $session->getId() === $current?->getId()) ->setAttribute('countryName', $countryName) ->setAttribute('secret', $session->getAttribute('secret', '')) ; @@ -801,11 +799,12 @@ ->inject('proofForToken') ->inject('domainVerification') ->inject('cookieDomain') - ->action(function (?string $sessionId, ?\DateTime $requestTimestamp, Request $request, Response $response, User $user, Database $dbForProject, Locale $locale, Event $queueForEvents, DeletePublisher $publisherForDeletes, Store $store, ProofsToken $proofForToken, bool $domainVerification, ?string $cookieDomain) { + ->inject('session') + ->action(function (?string $sessionId, ?\DateTime $requestTimestamp, Request $request, Response $response, User $user, Database $dbForProject, Locale $locale, Event $queueForEvents, DeletePublisher $publisherForDeletes, Store $store, ProofsToken $proofForToken, bool $domainVerification, ?string $cookieDomain, ?Document $current) { $protocol = $request->getProtocol(); $sessionId = ($sessionId === 'current') - ? $user->sessionVerify($store->getProperty('secret', ''), $proofForToken) + ? $current?->getId() : $sessionId; $sessions = $user->getAttribute('sessions', []); @@ -820,13 +819,13 @@ unset($sessions[$key]); - $session->setAttribute('current', false); + $session->setAttribute('current', $session->getId() === $current?->getId()); - if ($proofForToken->verify($store->getProperty('secret', ''), $session->getAttribute('secret'))) { // If current session delete the cookies too - $session - ->setAttribute('current', true) - ->setAttribute('countryName', $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown'))); + if ($session->getAttribute('current')) { + $session->setAttribute('countryName', $locale->getText('countries.' . strtolower($session->getAttribute('countryCode')), $locale->getText('locale.country.unknown'))); + } + if ($proofForToken->verify($store->getProperty('secret', ''), $session->getAttribute('secret'))) { // If current session delete the cookies too if (!$domainVerification) { $response->addHeader('X-Fallback-Cookies', \json_encode([])); } @@ -885,12 +884,11 @@ ->inject('dbForProject') ->inject('project') ->inject('queueForEvents') - ->inject('store') - ->inject('proofForToken') - ->action(function (?string $sessionId, Response $response, User $user, Database $dbForProject, Document $project, Event $queueForEvents, Store $store, ProofsToken $proofForToken) { + ->inject('session') + ->action(function (?string $sessionId, Response $response, User $user, Database $dbForProject, Document $project, Event $queueForEvents, ?Document $current) { $sessionId = ($sessionId === 'current') - ? $user->sessionVerify($store->getProperty('secret', ''), $proofForToken) + ? $current?->getId() : $sessionId; $sessions = $user->getAttribute('sessions', []); @@ -1627,15 +1625,17 @@ if (!empty($state['failure']) && !$redirectValidator->isValid($state['failure'])) { throw new Exception(Exception::PROJECT_INVALID_FAILURE_URL); } - // The default relays live on the console host; the same path on any other allowed host is a customer page + // The default relays live on the console host; the same path on any other allowed host is a customer page. + // Native apps skip the relay: its JavaScript redirect into the app is late or dropped on slow in-app browsers. $consoleHostname = \parse_url($platform['consoleUrl'] ?? '', PHP_URL_HOST); + $nativeCallback = ['scheme' => 'appwrite-callback-' . $project->getId()]; $failure = []; if (!empty($state['failure'])) { $failure = URLParser::parse($state['failure']); } - $failureRedirect = (function (string $type, ?string $message = null, ?int $code = null, ?\Throwable $previous = null, array $params = []) use ($failure, $response, $project, $oauthDefaultFailure, $consoleHostname) { + $failureRedirect = (function (string $type, ?string $message = null, ?int $code = null, ?\Throwable $previous = null, array $params = []) use ($failure, $response, $project, $oauthDefaultFailure, $consoleHostname, $nativeCallback) { $exception = new Exception($type, $message, $code, $previous, params: $params); if (!empty($failure)) { $query = URLParser::parseQuery($failure['query']); @@ -1644,10 +1644,9 @@ 'type' => $exception->getType(), 'code' => !\is_null($code) ? $code : $exception->getCode(), ]); - // Mirror success path: default OAuth failure relay needs project to deep-link - // back into the native app via appwrite-callback-{project}:// if ($failure['host'] === $consoleHostname && $failure['path'] === $oauthDefaultFailure) { $query['project'] = $project->getId(); + $failure = $nativeCallback; } $failure['query'] = URLParser::unparseQuery($query); $response->redirect(URLParser::unparse($failure), 301); @@ -2234,6 +2233,7 @@ $query['domain'] = $cookieDomain; $query['key'] = $store->getKey(); $query['secret'] = $encoded; + $state['success'] = $nativeCallback; } $response @@ -3508,11 +3508,10 @@ ->inject('dbForProject') ->inject('queueForEvents') ->inject('hooks') - ->inject('store') ->inject('proofForPassword') - ->inject('proofForToken') ->inject('pwnedPasswords') - ->action(function (string $password, string $oldPassword, Response $response, User $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks, Store $store, ProofsPassword $proofForPassword, ProofsToken $proofForToken, PasswordPwned $pwnedPasswords) { + ->inject('session') + ->action(function (string $password, string $oldPassword, Response $response, User $user, Document $project, Database $dbForProject, Event $queueForEvents, Hooks $hooks, ProofsPassword $proofForPassword, PasswordPwned $pwnedPasswords, ?Document $current) { $userProofForPassword = ProofsPassword::createHash($user->getAttribute('hash'), $user->getAttribute('hashOptions')); // Check old password only if its an existing user. if (!empty($user->getAttribute('passwordUpdate')) && !$userProofForPassword->verify($oldPassword, $user->getAttribute('password'))) { // Double check user password @@ -3562,13 +3561,11 @@ $sessions = $user->getAttribute('sessions', []); - $current = $user->sessionVerify($store->getProperty('secret', ''), $proofForToken); - $invalidate = $project->getAttribute('auths', default: [])['invalidateSessions'] ?? false; - if ($invalidate && !empty($current)) { + if ($invalidate && $current !== null) { foreach ($sessions as $session) { /** @var Document $session */ - if ($session->getId() !== $current) { + if ($session->getId() !== $current->getId()) { $dbForProject->deleteDocument('sessions', $session->getId()); } } @@ -5263,10 +5260,9 @@ ->inject('request') ->inject('response') ->inject('dbForProject') - ->inject('store') - ->inject('proofForToken') ->inject('authorization') - ->action(function (string $targetId, string $identifier, string $providerId, Event $queueForEvents, User $user, Request $request, Response $response, Database $dbForProject, Store $store, ProofsToken $proofForToken, Authorization $authorization) { + ->inject('session') + ->action(function (string $targetId, string $identifier, string $providerId, Event $queueForEvents, User $user, Request $request, Response $response, Database $dbForProject, Authorization $authorization, ?Document $current) { $targetId = $targetId == 'unique()' ? ID::unique() : $targetId; $provider = $authorization->skip(fn () => $dbForProject->getDocument('providers', $providerId)); @@ -5282,8 +5278,7 @@ $device = $detector->getDevice(); - $sessionId = $user->sessionVerify($store->getProperty('secret', ''), $proofForToken); - $session = $dbForProject->getDocument('sessions', $sessionId); + $session = $dbForProject->getDocument('sessions', $current?->getId() ?? ''); $name = "{$device['deviceBrand']} {$device['deviceModel']}"; // A session is one device install holding one push token per provider. Re-registering a rotated diff --git a/app/init/constants.php b/app/init/constants.php index f51b57ffc92..67b1efeb3ae 100644 --- a/app/init/constants.php +++ b/app/init/constants.php @@ -41,6 +41,7 @@ const APP_LIMIT_ARRAY_LABELS_SIZE = 1000; // Default maximum of how many labels elements can there be in API parameter that expects array value const APP_LIMIT_ARRAY_SCOPES_SIZE = 200; // Default maximum of how many scope elements can there be in API parameter that expects array value const APP_LIMIT_ARRAY_ELEMENT_SIZE = 4096; // Default maximum length of element in array parameter represented by maximum URL length. +const APP_LIMIT_ROLE_LENGTH = 81; // Maximum length of a team role: `project--` is 9 template characters around two 36-character IDs const APP_LIMIT_SUBQUERY = 1000; const APP_LIMIT_SUBSCRIBERS_SUBQUERY = 25; diff --git a/app/init/models.php b/app/init/models.php index 2eb36ce9644..b8d71d9a724 100644 --- a/app/init/models.php +++ b/app/init/models.php @@ -157,6 +157,7 @@ use Appwrite\Utopia\Response\Model\OAuth2TikTok; use Appwrite\Utopia\Response\Model\OAuth2Tradeshift; use Appwrite\Utopia\Response\Model\OAuth2Twitch; +use Appwrite\Utopia\Response\Model\OAuth2Webflow; use Appwrite\Utopia\Response\Model\OAuth2WordPress; use Appwrite\Utopia\Response\Model\OAuth2X; use Appwrite\Utopia\Response\Model\OAuth2Yahoo; @@ -435,6 +436,7 @@ Response::setModel(new OAuth2Yandex()); Response::setModel(new OAuth2X()); Response::setModel(new OAuth2WordPress()); +Response::setModel(new OAuth2Webflow()); Response::setModel(new OAuth2Twitch()); Response::setModel(new OAuth2Stripe()); Response::setModel(new OAuth2Spotify()); diff --git a/app/init/resources/request.php b/app/init/resources/request.php index 86d10eb0cb6..09acccdcc92 100644 --- a/app/init/resources/request.php +++ b/app/init/resources/request.php @@ -647,7 +647,7 @@ return $project; }, ['dbForPlatform', 'request', 'console', 'authorization', 'utopia', 'projectIdFromPath']); - $context->set('session', function (User $user, Store $store, Token $proofForToken) { + $context->set('session', function (User $user, Store $store, Token $proofForToken, Request $request) { if ($user->isEmpty()) { return; } @@ -655,6 +655,21 @@ $sessions = $user->getAttribute('sessions', []); $sessionId = $user->sessionVerify($store->getProperty('secret', ''), $proofForToken); + $authJWT = $request->getHeaderLine('x-appwrite-jwt', ''); + if (! $sessionId && ! empty($authJWT)) { + $jwt = new JWT(System::getEnv('_APP_OPENSSL_KEY_V1'), 'HS256', 3600, 0); + try { + $payload = $jwt->decode($authJWT); + } catch (JWTException) { + return; + } + + $jwtSessionId = $payload['sessionId'] ?? ''; + if (($payload['userId'] ?? '') === $user->getId() && ! empty($jwtSessionId) && $user->sessionActive($jwtSessionId)) { + $sessionId = $jwtSessionId; + } + } + if (! $sessionId) { return; } @@ -666,7 +681,7 @@ } return; - }, ['user', 'store', 'proofForToken']); + }, ['user', 'store', 'proofForToken', 'request']); $context->set('pwnedPasswords', function (Cache $cache) { // Nothing is asked until an operator points this at a service diff --git a/composer.json b/composer.json index 159b0f1ee74..7fc947aa7c7 100644 --- a/composer.json +++ b/composer.json @@ -26,6 +26,7 @@ "psr-4": { "Appwrite\\": "src/Appwrite", "Executor\\": "src/Executor", + "Utopia\\Abuse\\": "packages/abuse/src", "Utopia\\Agents\\": "packages/agents/src", "Utopia\\Audit\\": "packages/audit/src", "Utopia\\Auth\\": "packages/auth/src", @@ -49,6 +50,7 @@ "Utopia\\Lock\\": "packages/lock/src", "Utopia\\Messaging\\": "packages/messaging/src", "Utopia\\Mqtt\\": "packages/mqtt/src", + "Utopia\\NATS\\": "packages/nats/src", "Utopia\\OpenAPI\\": "packages/openapi/src", "Utopia\\Platform\\": "packages/platform/src", "Utopia\\Pools\\": "packages/pools/src", @@ -78,6 +80,7 @@ "Tests\\E2E\\": "tests/e2e", "Tests\\Unit\\": "tests/unit", "Appwrite\\Tests\\": "tests/extensions", + "Utopia\\Abuse\\Tests\\": "packages/abuse/tests", "Utopia\\Agents\\Tests\\": "packages/agents/tests", "Utopia\\Audit\\Tests\\": "packages/audit/tests", "Utopia\\Auth\\Tests\\": "packages/auth/tests", @@ -102,6 +105,7 @@ "Utopia\\Lock\\Tests\\": "packages/lock/tests", "Utopia\\Messaging\\Tests\\": "packages/messaging/tests", "Utopia\\Mqtt\\Tests\\": "packages/mqtt/tests", + "Utopia\\NATS\\Tests\\": "packages/nats/tests", "Utopia\\OpenAPI\\Tests\\": "packages/openapi/tests", "Utopia\\Platform\\Tests\\": "packages/platform/tests", "Utopia\\Pools\\Tests\\": "packages/pools/tests", @@ -144,9 +148,8 @@ "ext-sockets": "*", "appwrite/php-runtimes": "0.20.*", "appwrite/php-clamav": "2.0.*", - "utopia-php/abuse": "2.0.*", "utopia-php/config": "1.*", - "utopia-php/database": "^7.3.11", + "utopia-php/database": "^7.4.0", "utopia-php/migration": "^2.0.0", "mustangostang/spyc": "0.6.*", "dragonmantank/cron-expression": "3.4.*", @@ -164,7 +167,8 @@ "psr/http-factory": "^1.0", "psr/http-client": "^1.0", "psr/http-message": "^2.0", - "psr/container": "^2.0" + "psr/container": "^2.0", + "appwrite/appwrite": "^27.1" }, "require-dev": { "ext-fileinfo": "*", @@ -193,6 +197,7 @@ } }, "replace": { + "utopia-php/abuse": "*", "utopia-php/agents": "*", "utopia-php/audit": "*", "utopia-php/auth": "*", @@ -217,6 +222,7 @@ "utopia-php/lock": "*", "utopia-php/messaging": "*", "utopia-php/mqtt": "*", + "utopia-php/nats": "*", "utopia-php/openapi": "*", "utopia-php/platform": "*", "utopia-php/pools": "*", diff --git a/composer.lock b/composer.lock index e8f89b76bef..e174f1d49f8 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "8a44d1fb1f4674bf3d8b83f4b9c978b3", + "content-hash": "dd4622072c49f8b71053c2866ca7fc92", "packages": [ { "name": "adhocore/jwt", @@ -2778,59 +2778,6 @@ }, "time": "2025-06-29T15:42:06+00:00" }, - { - "name": "utopia-php/abuse", - "version": "2.0.1", - "source": { - "type": "git", - "url": "https://github.com/utopia-php/abuse.git", - "reference": "821a1884c8067736e1a27d9fc70bdb2cc84d7146" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/utopia-php/abuse/zipball/821a1884c8067736e1a27d9fc70bdb2cc84d7146", - "reference": "821a1884c8067736e1a27d9fc70bdb2cc84d7146", - "shasum": "" - }, - "require": { - "appwrite/appwrite": "^27.1", - "ext-curl": "*", - "ext-pdo": "*", - "ext-redis": "*", - "php": ">=8.4.1", - "utopia-php/database": "^7.0.0", - "utopia-php/pools": "2.*" - }, - "require-dev": { - "laravel/pint": "1.*", - "phpbench/phpbench": "1.*", - "phpstan/phpstan": "1.*", - "phpunit/phpunit": "9.*" - }, - "type": "library", - "autoload": { - "psr-4": { - "Utopia\\Abuse\\": "src/Abuse" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "description": "A simple abuse library to manage application usage limits", - "keywords": [ - "Abuse", - "framework", - "php", - "upf", - "utopia" - ], - "support": { - "issues": "https://github.com/utopia-php/abuse/issues", - "source": "https://github.com/utopia-php/abuse/tree/2.0.1" - }, - "time": "2026-08-13T11:44:26+00:00" - }, { "name": "utopia-php/config", "version": "1.0.0", @@ -2880,16 +2827,16 @@ }, { "name": "utopia-php/database", - "version": "7.3.11", + "version": "7.4.0", "source": { "type": "git", "url": "https://github.com/utopia-php/database.git", - "reference": "e45195ffb7019e70b88cf7d33814ecafd3fa2866" + "reference": "1c99c2179d13f79476a51dd3b73ecc0025058d14" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/database/zipball/e45195ffb7019e70b88cf7d33814ecafd3fa2866", - "reference": "e45195ffb7019e70b88cf7d33814ecafd3fa2866", + "url": "https://api.github.com/repos/utopia-php/database/zipball/1c99c2179d13f79476a51dd3b73ecc0025058d14", + "reference": "1c99c2179d13f79476a51dd3b73ecc0025058d14", "shasum": "" }, "require": { @@ -2934,22 +2881,22 @@ ], "support": { "issues": "https://github.com/utopia-php/database/issues", - "source": "https://github.com/utopia-php/database/tree/7.3.11" + "source": "https://github.com/utopia-php/database/tree/7.4.0" }, - "time": "2026-09-17T08:18:45+00:00" + "time": "2026-09-29T10:10:07+00:00" }, { "name": "utopia-php/migration", - "version": "2.0.7", + "version": "2.0.8", "source": { "type": "git", "url": "https://github.com/utopia-php/migration.git", - "reference": "a702ce80d0073983df6d51769d82be006a69fa1f" + "reference": "1cacecb310251f7a7abf15009680a2f5b8991f80" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/utopia-php/migration/zipball/a702ce80d0073983df6d51769d82be006a69fa1f", - "reference": "a702ce80d0073983df6d51769d82be006a69fa1f", + "url": "https://api.github.com/repos/utopia-php/migration/zipball/1cacecb310251f7a7abf15009680a2f5b8991f80", + "reference": "1cacecb310251f7a7abf15009680a2f5b8991f80", "shasum": "" }, "require": { @@ -2989,9 +2936,9 @@ ], "support": { "issues": "https://github.com/utopia-php/migration/issues", - "source": "https://github.com/utopia-php/migration/tree/2.0.7" + "source": "https://github.com/utopia-php/migration/tree/2.0.8" }, - "time": "2026-09-08T12:03:52+00:00" + "time": "2026-09-29T10:25:37+00:00" }, { "name": "utopia-php/mongo", diff --git a/docker-compose.yml b/docker-compose.yml index e551fef2634..b98bcf341d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -270,7 +270,7 @@ services: max-file: "5" max-size: 10m container_name: appwrite-console - image: appwrite/new:1.1.180-self-hosted + image: appwrite/new:1.2.5-self-hosted restart: unless-stopped networks: - appwrite diff --git a/docs/sdks/flutter/GETTING_STARTED.md b/docs/sdks/flutter/GETTING_STARTED.md index 75cf9de1afc..6d419c5dd1d 100644 --- a/docs/sdks/flutter/GETTING_STARTED.md +++ b/docs/sdks/flutter/GETTING_STARTED.md @@ -7,6 +7,8 @@ From the options, choose to add a new **Flutter** platform and add your app cred If you are building your Flutter application for multiple devices, you have to follow this process for each different device. +> On Android, iOS and macOS, call `createOAuth2Session` without the `success` and `failure` URLs. The SDK only returns to your app through the `appwrite-callback-[PROJECT_ID]` scheme, and only the default redirect carries the new session with it. A custom URL, such as an https App Link, makes the call fail (for example with `PlatformException(CANCELED)`) even when the login itself succeeded. Await the returned `Future` and navigate from your Dart code instead. + ### Android For **Android** first add your app name and package name, Your package name is generally the **applicationId** in your app-level build.gradle file. By registering your new app platform, you are allowing your app to communicate with the Appwrite API. diff --git a/packages/abuse/.env.example b/packages/abuse/.env.example new file mode 100644 index 00000000000..013b71041f3 --- /dev/null +++ b/packages/abuse/.env.example @@ -0,0 +1,3 @@ +APPWRITE_ENDPOINT= +APPWRITE_PROJECT_ID= +APPWRITE_API_KEY= \ No newline at end of file diff --git a/packages/abuse/.github/workflows/mirror.yml b/packages/abuse/.github/workflows/mirror.yml new file mode 100644 index 00000000000..6b646ca6610 --- /dev/null +++ b/packages/abuse/.github/workflows/mirror.yml @@ -0,0 +1,17 @@ +name: Mirror + +on: + pull_request_target: + types: [opened] + issues: + types: [opened] + +permissions: + issues: write + pull-requests: write + +jobs: + redirect: + uses: appwrite/appwrite/.github/workflows/mirror-redirect.yml@main + with: + package: abuse diff --git a/packages/abuse/.gitignore b/packages/abuse/.gitignore new file mode 100755 index 00000000000..6963341a694 --- /dev/null +++ b/packages/abuse/.gitignore @@ -0,0 +1,5 @@ +/vendor/ +/.idea/ +.env +.phpunit.cache +composer.lock diff --git a/packages/abuse/CODE_OF_CONDUCT.md b/packages/abuse/CODE_OF_CONDUCT.md new file mode 100644 index 00000000000..2dec654fbbc --- /dev/null +++ b/packages/abuse/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity, expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +- Using welcoming and inclusive language +- Being respectful of differing viewpoints and experiences +- Gracefully accepting constructive criticism +- Focusing on what is best for the community +- Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +- The use of sexualized language or imagery and unwelcome sexual attention or + advances +- Trolling, insulting/derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or electronic + address, without explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at team@appwrite.io. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/packages/abuse/LICENSE.md b/packages/abuse/LICENSE.md new file mode 100755 index 00000000000..27feb0854cd --- /dev/null +++ b/packages/abuse/LICENSE.md @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 Eldad Fux + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/packages/abuse/README.md b/packages/abuse/README.md new file mode 100644 index 00000000000..efa3542663d --- /dev/null +++ b/packages/abuse/README.md @@ -0,0 +1,135 @@ +# Utopia Abuse + +> [!IMPORTANT] +> This repository is a read-only mirror of [`packages/abuse`](https://github.com/appwrite/appwrite/tree/main/packages/abuse) in [appwrite/appwrite](https://github.com/appwrite/appwrite). Development happens there — please open issues and pull requests against appwrite/appwrite. + +![Total Downloads](https://img.shields.io/packagist/dt/utopia-php/abuse.svg) +[![Discord](https://img.shields.io/discord/564160730845151244)](https://appwrite.io/discord) + +Utopia framework abuse library is simple and lite library for managing application usage limits. This library is aiming to be as simple and easy to learn and use. This library is maintained by the [Appwrite team](https://appwrite.io). + +Although this library is part of the [Utopia Framework](https://github.com/utopia-php/framework) project it is dependency free, and can be used as standalone with any other PHP project or framework. + +## Getting Started + +Install using composer: + +```bash +composer require utopia-php/abuse +``` + +**Time Limit Abuse** + +The time limit abuse allow each key (action) to be performed [X] times in given time frame. +This adapter uses a MySQL / MariaDB to store usage attempts. Before using it, call `$adapter->setup()` once to create the collection it stores attempts in. + +### Database adapter + +```php + 3, // Seconds + PDO::ATTR_PERSISTENT => true, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_EMULATE_PREPARES => true, + PDO::ATTR_STRINGIFY_FETCHES => true, +]); + +$db = new Database(new MySQL($pdo), new Cache(new NoCache())); +$db->setNamespace('namespace'); + +// Limit login attempts to 10 time in 5 minutes time frame +$adapter = new TimeLimit('login-attempt-from-{{ip}}', 10, (60 * 5), $db); + +$adapter->setup(); //setup database as required +$adapter->setParam('{{ip}}', '127.0.0.1') +; + +$abuse = new Abuse($adapter); + +// Use vars to resolve adapter key + +if($abuse->check()) { + throw new Exception('Service was abused!'); // throw error and return X-Rate limit headers here +} +``` + +### Appwrite TablesDB adapter + +```php +setEndpoint('[YOUR_ENDPOINT]') + ->setProject('[YOUR_PROJECT_ID]') + ->setKey('[YOUR_API_KEY]'); +$databaseId = 'abuse'; + +// Limit login attempts to 10 time in 5 minutes time frame +$adapter = new TablesDBAdapter('login-attempt-from-{{ip}}', 10, (60 * 5), $client, $databaseId); + +$adapter->setup(); //setup database as required +$adapter->setParam('{{ip}}', '127.0.0.1'); + +$abuse = new Abuse($adapter); + +// Use vars to resolve adapter key + +if($abuse->check()) { + throw new Exception('Service was abused!'); // throw error and return X-Rate limit headers here +} +``` + +**ReCaptcha Abuse** + +The ReCaptcha abuse controller is using Google ReCaptcha service to detect when service is being abused by bots. +To use this adapter you need to create an API key from the Google ReCaptcha service [admin console](https://www.google.com/recaptcha/admin). + +```php +check()) { + throw new Exception('Service was abused!'); // throw error and return X-Rate limit headers here +} +``` + +*Notice: The code above is for example purpose only. It is always recommended to validate user input before using it in your code. If you are using a load balancer or any proxy server you might need to get user IP from the HTTP_X_FORWARDE‌​D_FOR header.* + +## System Requirements + +Utopia Framework requires PHP 8.0 or later. We recommend using the latest PHP version whenever possible. + +## Copyright and license + +The MIT License (MIT) [http://www.opensource.org/licenses/mit-license.php](http://www.opensource.org/licenses/mit-license.php) diff --git a/packages/abuse/composer.json b/packages/abuse/composer.json new file mode 100644 index 00000000000..cb78a60e73f --- /dev/null +++ b/packages/abuse/composer.json @@ -0,0 +1,47 @@ +{ + "name": "utopia-php/abuse", + "description": "A simple abuse library to manage application usage limits", + "type": "library", + "keywords": [ + "php", + "framework", + "upf", + "utopia", + "abuse" + ], + "license": "MIT", + "minimum-stability": "stable", + "autoload": { + "psr-4": { + "Utopia\\Abuse\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\Abuse\\Tests\\": "tests/" + } + }, + "scripts": { + "test": "phpunit --testsuite unit", + "test:e2e": "phpunit --testsuite e2e", + "bench": "vendor/bin/phpbench run --report=aggregate" + }, + "require": { + "php": ">=8.4.1", + "ext-pdo": "*", + "ext-curl": "*", + "ext-redis": "*", + "utopia-php/database": "^7.0.0", + "utopia-php/pools": "2.*", + "appwrite/appwrite": "^27.1" + }, + "require-dev": { + "phpbench/phpbench": "1.*" + }, + "config": { + "allow-plugins": { + "php-http/discovery": true, + "tbachert/spi": true + } + } +} diff --git a/packages/abuse/docker-compose.yml b/packages/abuse/docker-compose.yml new file mode 100644 index 00000000000..9c6f60bc3b4 --- /dev/null +++ b/packages/abuse/docker-compose.yml @@ -0,0 +1,40 @@ +name: utopia-abuse + +services: + mysql: + image: mysql:8 + environment: + MYSQL_ROOT_PASSWORD: password + ports: + - "13308:3306" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-ppassword"] + interval: 2s + timeout: 3s + retries: 60 + + redis: + image: redis:7-alpine + command: ["redis-server", "--save", "", "--appendonly", "no"] + ports: + - "16386:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 1s + timeout: 1s + retries: 30 + + # The cluster advertises the addresses clients dial, so its published ports + # match the ones inside the container. + redis-cluster: + image: grokzen/redis-cluster:7.0.10 + environment: + IP: 127.0.0.1 + INITIAL_PORT: 17010 + ports: + - "17010-17015:17010-17015" + healthcheck: + test: ["CMD-SHELL", "redis-cli -p 17010 cluster info | grep -q cluster_state:ok"] + interval: 2s + timeout: 3s + retries: 60 diff --git a/packages/abuse/phpbench.json b/packages/abuse/phpbench.json new file mode 100644 index 00000000000..f09975f9c4a --- /dev/null +++ b/packages/abuse/phpbench.json @@ -0,0 +1,6 @@ +{ + "$schema": "vendor/phpbench/phpbench/phpbench.schema.json", + "runner.bootstrap": "vendor/autoload.php", + "runner.path": "tests/bench", + "runner.file_pattern": "*.php" +} \ No newline at end of file diff --git a/packages/abuse/phpstan-baseline.neon b/packages/abuse/phpstan-baseline.neon new file mode 100644 index 00000000000..9546ee2864f --- /dev/null +++ b/packages/abuse/phpstan-baseline.neon @@ -0,0 +1,265 @@ +parameters: + ignoreErrors: + - + message: '#^Parameter \#3 \$value of function curl_setopt expects bool, int given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/ReCaptcha.php + + - + message: '#^Method Utopia\\Abuse\\Adapters\\SlidingWindow\\RedisCluster\:\:getLogs\(\) should return array\ but returns array\\.$#' + identifier: return.type + count: 1 + path: src/Adapters/SlidingWindow/RedisCluster.php + + - + message: '#^Parameter \#1 \$array of function sort contains unresolvable type\.$#' + identifier: argument.unresolvableType + count: 1 + path: src/Adapters/SlidingWindow/RedisCluster.php + + - + message: '#^Parameter \#1 \$keys of function array_combine expects an array of values castable to string, list\ given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/SlidingWindow/RedisCluster.php + + - + message: '#^Parameter \#1 \$keys of function array_combine expects array\, list\ given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/SlidingWindow/RedisCluster.php + + - + message: '#^Parameter \#2 \.\.\.\$arrays of function array_merge expects array, array\|true given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/SlidingWindow/RedisCluster.php + + - + message: '#^Call to function is_array\(\) with array will always evaluate to true\.$#' + identifier: function.alreadyNarrowedType + count: 1 + path: src/Adapters/SlidingWindow/RedisPool.php + + - + message: '#^Call to function is_array\(\) with array\ will always evaluate to true\.$#' + identifier: function.alreadyNarrowedType + count: 1 + path: src/Adapters/SlidingWindow/RedisPool.php + + - + message: '#^Method Utopia\\Abuse\\Adapters\\SlidingWindow\\RedisPool\:\:getRedisClusterLogs\(\) should return array\ but returns array\\.$#' + identifier: return.type + count: 1 + path: src/Adapters/SlidingWindow/RedisPool.php + + - + message: '#^Only iterables can be unpacked, array\|true given in argument \#2\.$#' + identifier: argument.unpackNonIterable + count: 1 + path: src/Adapters/SlidingWindow/RedisPool.php + + - + message: '#^Parameter \#1 \$array of function sort contains unresolvable type\.$#' + identifier: argument.unresolvableType + count: 1 + path: src/Adapters/SlidingWindow/RedisPool.php + + - + message: '#^Parameter \#1 \$keys of function array_combine expects an array of values castable to string, list\ given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/SlidingWindow/RedisPool.php + + - + message: '#^Parameter \#1 \$keys of function array_combine expects array\, list\ given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/SlidingWindow/RedisPool.php + + - + message: '#^Method Utopia\\Abuse\\Adapters\\TimeLimit\\RedisCluster\:\:getLogs\(\) should return array\ but returns array\\.$#' + identifier: return.type + count: 1 + path: src/Adapters/TimeLimit/RedisCluster.php + + - + message: '#^Parameter \#1 \$array of function sort contains unresolvable type\.$#' + identifier: argument.unresolvableType + count: 1 + path: src/Adapters/TimeLimit/RedisCluster.php + + - + message: '#^Parameter \#1 \$keys of function array_combine expects an array of values castable to string, list\ given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/TimeLimit/RedisCluster.php + + - + message: '#^Parameter \#1 \$keys of function array_combine expects array\, list\ given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/TimeLimit/RedisCluster.php + + - + message: '#^Parameter \#2 \.\.\.\$arrays of function array_merge expects array, array\|true given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/TimeLimit/RedisCluster.php + + - + message: '#^Call to function is_array\(\) with array will always evaluate to true\.$#' + identifier: function.alreadyNarrowedType + count: 1 + path: src/Adapters/TimeLimit/RedisPool.php + + - + message: '#^Call to function is_array\(\) with array\ will always evaluate to true\.$#' + identifier: function.alreadyNarrowedType + count: 1 + path: src/Adapters/TimeLimit/RedisPool.php + + - + message: '#^Method Utopia\\Abuse\\Adapters\\TimeLimit\\RedisPool\:\:getRedisClusterLogs\(\) should return array\ but returns array\\.$#' + identifier: return.type + count: 1 + path: src/Adapters/TimeLimit/RedisPool.php + + - + message: '#^Only iterables can be unpacked, array\|true given in argument \#2\.$#' + identifier: argument.unpackNonIterable + count: 1 + path: src/Adapters/TimeLimit/RedisPool.php + + - + message: '#^Parameter \#1 \$array of function sort contains unresolvable type\.$#' + identifier: argument.unresolvableType + count: 1 + path: src/Adapters/TimeLimit/RedisPool.php + + - + message: '#^Parameter \#1 \$keys of function array_combine expects an array of values castable to string, list\ given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/TimeLimit/RedisPool.php + + - + message: '#^Parameter \#1 \$keys of function array_combine expects array\, list\ given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/TimeLimit/RedisPool.php + + - + message: '#^Method Utopia\\Abuse\\Adapters\\TokenBucket\\RedisCluster\:\:getLogs\(\) should return array\ but returns array\\.$#' + identifier: return.type + count: 1 + path: src/Adapters/TokenBucket/RedisCluster.php + + - + message: '#^Parameter \#1 \$array of function sort contains unresolvable type\.$#' + identifier: argument.unresolvableType + count: 1 + path: src/Adapters/TokenBucket/RedisCluster.php + + - + message: '#^Parameter \#1 \$key of method RedisCluster\:\:hGetAll\(\) expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/TokenBucket/RedisCluster.php + + - + message: '#^Parameter \#2 \.\.\.\$arrays of function array_merge expects array, array\|true given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/TokenBucket/RedisCluster.php + + - + message: '#^Possibly invalid array key type mixed\.$#' + identifier: offsetAccess.invalidOffset + count: 1 + path: src/Adapters/TokenBucket/RedisCluster.php + + - + message: '#^Method Utopia\\Abuse\\Adapters\\TokenBucket\\RedisPool\:\:getRedisClusterLogs\(\) should return array\ but returns array\\.$#' + identifier: return.type + count: 1 + path: src/Adapters/TokenBucket/RedisPool.php + + - + message: '#^Only iterables can be unpacked, array\|true given in argument \#2\.$#' + identifier: argument.unpackNonIterable + count: 1 + path: src/Adapters/TokenBucket/RedisPool.php + + - + message: '#^Parameter \#1 \$array of function sort contains unresolvable type\.$#' + identifier: argument.unresolvableType + count: 1 + path: src/Adapters/TokenBucket/RedisPool.php + + - + message: '#^Parameter \#1 \$key of method RedisCluster\:\:hGetAll\(\) expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/Adapters/TokenBucket/RedisPool.php + + - + message: '#^Possibly invalid array key type mixed\.$#' + identifier: offsetAccess.invalidOffset + count: 1 + path: src/Adapters/TokenBucket/RedisPool.php + + - + message: '#^Method Utopia\\Abuse\\Tests\\E2E\\Appwrite\\TablesDBTest\:\:columnsByKey\(\) should return array\\> but returns array\\>\.$#' + identifier: return.type + count: 1 + path: tests/E2E/Appwrite/TablesDBTest.php + + - + message: '#^Call to function is_int\(\) with int will always evaluate to true\.$#' + identifier: function.alreadyNarrowedType + count: 1 + path: tests/E2E/Base.php + + - + message: '#^Argument of an invalid type array\|true supplied for foreach, only iterables are supported\.$#' + identifier: foreach.nonIterable + count: 1 + path: tests/E2E/RedisPoolClusterTest.php + + - + message: '#^Instanceof between RedisCluster and RedisCluster will always evaluate to true\.$#' + identifier: instanceof.alwaysTrue + count: 1 + path: tests/E2E/RedisPoolClusterTest.php + + - + message: '#^Parameter \#1 \$key of method RedisCluster\:\:del\(\) expects int\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/E2E/RedisPoolClusterTest.php + + - + message: '#^Instanceof between Redis and Redis will always evaluate to true\.$#' + identifier: instanceof.alwaysTrue + count: 1 + path: tests/E2E/RedisPoolTest.php + + - + message: '#^Instanceof between Redis and Redis will always evaluate to true\.$#' + identifier: instanceof.alwaysTrue + count: 1 + path: tests/E2E/SlidingWindow/RedisPoolTest.php + + - + message: '#^Call to function is_int\(\) with int will always evaluate to true\.$#' + identifier: function.alreadyNarrowedType + count: 1 + path: tests/E2E/TokenBucket/Base.php + + - + message: '#^Instanceof between Redis and Redis will always evaluate to true\.$#' + identifier: instanceof.alwaysTrue + count: 1 + path: tests/E2E/TokenBucket/RedisPoolTest.php diff --git a/packages/abuse/phpstan.neon b/packages/abuse/phpstan.neon new file mode 100644 index 00000000000..210b39af9ee --- /dev/null +++ b/packages/abuse/phpstan.neon @@ -0,0 +1,10 @@ +includes: + - phpstan-baseline.neon + +parameters: + level: max + paths: + - src + - tests + excludePaths: + - tests/bench diff --git a/packages/abuse/phpunit.xml b/packages/abuse/phpunit.xml new file mode 100755 index 00000000000..4c54595ff57 --- /dev/null +++ b/packages/abuse/phpunit.xml @@ -0,0 +1,22 @@ + + + + + tests + tests/E2E + tests/bench + + + tests/E2E + + + + + src + + + diff --git a/packages/abuse/rector.php b/packages/abuse/rector.php new file mode 100644 index 00000000000..c9819ac6ced --- /dev/null +++ b/packages/abuse/rector.php @@ -0,0 +1,38 @@ +withPaths([ + __DIR__ . '/src', + __DIR__ . '/tests', + ]) + ->withPhpSets() + ->withPreparedSets( + typeDeclarations: true, + ) + // Absorbing moves code: keep src exactly as released. + ->withSkip([ + // Rector's PHP 8.4 printer drops the parentheses in `(new \DateTime())->` + // whenever it reprints these files. + __DIR__ . '/src/Adapters/TimeLimit/Appwrite/TablesDB.php', + __DIR__ . '/src/Adapters/TimeLimit/Database.php', + AddArrayFunctionClosureParamTypeRector::class => [__DIR__ . '/src'], + AddArrowFunctionReturnTypeRector::class => [__DIR__ . '/src'], + AddClosureVoidReturnTypeWhereNoReturnRector::class => [__DIR__ . '/src'], + ClassPropertyAssignToConstructorPromotionRector::class => [__DIR__ . '/src'], + ClosureReturnTypeRector::class => [__DIR__ . '/src'], + ClosureToArrowFunctionRector::class => [__DIR__ . '/src'], + NullCoalescingOperatorRector::class => [__DIR__ . '/src'], + RemoveUnusedVariableInCatchRector::class => [__DIR__ . '/src'], + ]); diff --git a/packages/abuse/src/Abuse.php b/packages/abuse/src/Abuse.php new file mode 100644 index 00000000000..cdb6f9c54b2 --- /dev/null +++ b/packages/abuse/src/Abuse.php @@ -0,0 +1,68 @@ +adapter = $adapter; + } + + /** + * Check + * + * Checks if request is considered abuse or not + * + * @return bool + */ + public function check(): bool + { + return $this->adapter->check(); + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + return $this->adapter->getLogs($offset, $limit); + } + + /** + * Delete all logs older than $timestamp + * + * @param int $timestamp + * @return bool + */ + public function cleanup(int $timestamp): bool + { + return $this->adapter->cleanup($timestamp); + } + + /** + * Reset + * + * Reset the count to 0 for the current adapter + * + * @return void + */ + public function reset(): void + { + $this->adapter->reset(); + } +} diff --git a/packages/abuse/src/Adapter.php b/packages/abuse/src/Adapter.php new file mode 100644 index 00000000000..b511d0319b2 --- /dev/null +++ b/packages/abuse/src/Adapter.php @@ -0,0 +1,95 @@ + + */ + protected array $params = []; + + /** + * @var string + */ + protected string $key = ''; + + /** + * Check + * + * Checks if number of counts is bigger or smaller than current limit + * + * @return bool + */ + abstract public function check(): bool; + + /** + * Set Param + * + * Set custom param for key pattern parsing + * + * @param string $key + * @param string $value + * @return $this + */ + public function setParam(string $key, string $value): self + { + $this->params[$key] = $value; + + return $this; + } + + /** + * Get Params + * + * Return array of all key params + * + * @return array + */ + protected function getParams(): array + { + return $this->params; + } + + /** + * Parse key with all custom attached params + * + * @return string + */ + protected function parseKey(): string + { + foreach ($this->getParams() as $key => $value) { + $this->key = \str_replace($key, $value, $this->key); + } + + return $this->key; + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + abstract public function getLogs(?int $offset = null, ?int $limit = 25): array; + + /** + * Delete all logs older than $datetime + * + * @param int $timestamp + * @return bool + */ + abstract public function cleanup(int $timestamp): bool; + + /** + * Reset + * + * Reset the count to 0 + * + * @return void + */ + abstract public function reset(): void; +} diff --git a/packages/abuse/src/Adapters/ReCaptcha.php b/packages/abuse/src/Adapters/ReCaptcha.php new file mode 100644 index 00000000000..7e1f67c7570 --- /dev/null +++ b/packages/abuse/src/Adapters/ReCaptcha.php @@ -0,0 +1,132 @@ +secret = $secret; + $this->response = $response; + $this->remoteIP = $remoteIP; + } + + /** + * Check + * + * Check if user is human or not, compared to score + * + * @param float $score + * @return bool + */ + public function check(float $score = 0.5): bool + { + $url = 'https://www.google.com/recaptcha/api/siteverify'; + $fields = [ + 'secret' => \urlencode($this->secret), + 'response' => \urlencode($this->response), + 'remoteip' => \urlencode($this->remoteIP), + ]; + + //open connection + $ch = \curl_init(); + + //set the url, number of POST vars, POST data + \curl_setopt($ch, CURLOPT_URL, $url); + \curl_setopt($ch, CURLOPT_POST, \count($fields)); + \curl_setopt($ch, CURLOPT_POSTFIELDS, \http_build_query($fields)); + \curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); + + //execute post + /** @var array $result */ + $result = \json_decode((string) \curl_exec($ch), true); + + if ($result['success'] && $result['score'] >= $score) { + return true; + } else { + return false; + } + } + + /** + * Delete logs older than $timestamp + * + * @param int $timestamp + * @return bool + * + * @throws Exception + */ + public function cleanup(int $timestamp): bool + { + throw new Exception('Method not supported'); + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + * + * @throws Exception + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + throw new Exception('Method not supported'); + } + + /** + * Reset + * + * Reset is not applicable for ReCaptcha adapter + * + * @return void + * + * @throws Exception + */ + public function reset(): void + { + throw new Exception('Method not supported'); + } +} diff --git a/packages/abuse/src/Adapters/SlidingWindow.php b/packages/abuse/src/Adapters/SlidingWindow.php new file mode 100644 index 00000000000..a488db643da --- /dev/null +++ b/packages/abuse/src/Adapters/SlidingWindow.php @@ -0,0 +1,100 @@ +limit - ($this->count($this->parseKey(), $this->timestamp) + 1); + + return (0 > $left) ? 0 : $left; + } + + /** + * Limit + * + * Return the limit integer + * + * @return int + */ + public function limit(): int + { + return $this->limit; + } + + /** + * Time + * + * Return the timestamp + * + * @return int + */ + public function time(): int + { + return $this->timestamp; + } + + /** + * Reset + * + * Clear the counters for the current key so the limit starts fresh. + * Implementations must clear both the current and previous window buckets. + * + * @return void + * + * @throws \Exception + */ + abstract public function reset(): void; +} diff --git a/packages/abuse/src/Adapters/SlidingWindow/None.php b/packages/abuse/src/Adapters/SlidingWindow/None.php new file mode 100644 index 00000000000..74f9554d072 --- /dev/null +++ b/packages/abuse/src/Adapters/SlidingWindow/None.php @@ -0,0 +1,62 @@ +key = $key; + $this->limit = $limit; + $now = \time(); + $this->timestamp = (int) ($now - ($now % $windowSize)); + } + + protected function count(string $key, int $timestamp): int + { + return 0; + } + + public function check(): bool + { + return false; + } + + public function reset(): void + { + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + return []; + } + + /** + * Delete all logs older than $timestamp + * + * @param int $timestamp + * @return bool + */ + public function cleanup(int $timestamp): bool + { + return true; + } +} diff --git a/packages/abuse/src/Adapters/SlidingWindow/Redis.php b/packages/abuse/src/Adapters/SlidingWindow/Redis.php new file mode 100644 index 00000000000..af22065c7ed --- /dev/null +++ b/packages/abuse/src/Adapters/SlidingWindow/Redis.php @@ -0,0 +1,94 @@ += $windowSize so the + * previous window's bucket survives long enough to be weighted + * @param \Redis $redis Redis connection used for storage + */ + public function __construct(protected string $key, protected int $limit, int $windowSize, int $ttl, protected \Redis $redis) + { + $this->initWindow($windowSize, $ttl); + } + + /** + * @param string $script + * @param list $keys + * @param list $argv + * @return mixed + * + * @throws \RedisException + */ + protected function eval(string $script, array $keys, array $argv): mixed + { + return $this->redis->eval($script, [...$keys, ...$argv], \count($keys)); + } + + /** + * @param string $key + * @return mixed + * + * @throws \RedisException + */ + protected function get(string $key): mixed + { + return $this->redis->get($key); + } + + /** + * @param string ...$keys + * @return void + * + * @throws \RedisException + */ + protected function delete(string ...$keys): void + { + $this->redis->del(...$keys); + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + $offset = $offset ?? 0; + $limit = $limit ?? 25; + + $cursor = null; + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + do { + $keys = $this->redis->scan($cursor, $pattern, 100); + if ($keys !== false) { + \array_push($matches, ...$keys); + } + } while ($cursor > 0); + + \sort($matches); + $matches = \array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + $logs = []; + foreach ($matches as $key) { + $logs[$key] = $this->redis->get($key); + } + + return $logs; + } +} diff --git a/packages/abuse/src/Adapters/SlidingWindow/RedisBase.php b/packages/abuse/src/Adapters/SlidingWindow/RedisBase.php new file mode 100644 index 00000000000..39b357e0796 --- /dev/null +++ b/packages/abuse/src/Adapters/SlidingWindow/RedisBase.php @@ -0,0 +1,253 @@ += max_requests then + return { 0, 0, math.floor(estimated) } + end + + local new_count = redis.call('INCR', current_key) + redis.call('EXPIRE', current_key, ttl) + + local new_estimate = weighted_prev + new_count + local remaining = math.max(0, math.floor(max_requests - new_estimate)) + return { 1, remaining, math.floor(new_estimate) } + LUA; + + /** + * @var int + */ + protected int $windowSize; + + /** + * @var int + */ + protected int $ttl; + + /** + * Run a Lua script against the storage backend. + * + * @param string $script + * @param list $keys + * @param list $argv + * @return mixed the raw script result + */ + abstract protected function eval(string $script, array $keys, array $argv): mixed; + + /** + * Get the raw value stored at $key (null/false when missing). + * + * @param string $key + * @return mixed + */ + abstract protected function get(string $key): mixed; + + /** + * Delete the given keys. + * + * @param string ...$keys + * @return void + */ + abstract protected function delete(string ...$keys): void; + + /** + * Validate and store the window configuration. The window itself is derived + * live in window(); here we only seed $timestamp so the inherited property is + * initialised before remaining()/time() read it. + * + * @param int $windowSize + * @param int $ttl + * @return void + */ + protected function initWindow(int $windowSize, int $ttl): void + { + if ($windowSize <= 0) { + throw new \InvalidArgumentException('windowSize must be greater than 0'); + } + + // The previous bucket keeps contributing (weighted) throughout the whole + // current window, and its ttl is set from its last write - which in the + // worst case is at the very start of its own window. It therefore needs to + // survive up to two full windows, so ttl must be >= 2 * windowSize. + if ($ttl < $windowSize * 2) { + throw new \InvalidArgumentException('ttl must be at least twice the windowSize so the previous window bucket outlives the current window'); + } + + $this->windowSize = $windowSize; + $this->ttl = $ttl; + [$this->timestamp] = $this->window(); + } + + /** + * Compute the live window from the current time. + * + * @return array{0:int,1:float} [window start timestamp, elapsed fraction in [0,1)] + */ + private function window(): array + { + $now = \time(); + $timestamp = (int)($now - ($now % $this->windowSize)); // start of the current window + + return [$timestamp, ($now - $timestamp) / $this->windowSize]; + } + + /** + * Build a bucket key. The hash tag around $key forces the current and previous + * window buckets into the same cluster slot, so the multi-key Lua script and + * reset() do not raise CROSSSLOT on a Redis Cluster (harmless on single Redis). + * + * @param string $key + * @param int $timestamp + * @return string + */ + protected function bucketKey(string $key, int $timestamp): string + { + return self::NAMESPACE . '__{' . $key . '}__' . $timestamp; + } + + /** + * Time + * + * Start timestamp of the current window, recomputed from the clock. + * + * @return int + */ + public function time(): int + { + [$this->timestamp] = $this->window(); + + return $this->timestamp; + } + + /** + * Check + * + * @return bool + * + * @throws \Throwable + */ + public function check(): bool + { + if ($this->limit === 0) { + return false; + } + + $key = $this->parseKey(); + [$timestamp, $elapsed] = $this->window(); + $this->timestamp = $timestamp; + + /** @var array{0:int,1:int,2:int} $result */ + $result = $this->eval( + self::LIMIT_CHECK_SCRIPT, + [ + $this->bucketKey($key, $timestamp), // KEYS[1] current bucket + $this->bucketKey($key, $timestamp - $this->windowSize), // KEYS[2] previous bucket + ], + [ + $this->limit, // ARGV[1] max_requests + $elapsed, // ARGV[2] elapsed fraction + $this->ttl, // ARGV[3] ttl seconds + ], + ); + + [$allowed] = $result; + + return $allowed === 0; + } + + /** + * Count + * + * Read-only weighted estimate of hits in the current sliding window + * (current bucket + weighted previous bucket). Used by remaining(). + * + * @param string $key + * @param int $timestamp + * @return int + */ + protected function count(string $key, int $timestamp): int + { + if (0 == $this->limit) { + return 0; + } + + [$windowStart, $elapsed] = $this->window(); + $this->timestamp = $windowStart; + + $currentRaw = $this->get($this->bucketKey($key, $windowStart)); + $previousRaw = $this->get($this->bucketKey($key, $windowStart - $this->windowSize)); + + $current = \is_numeric($currentRaw) ? (int) $currentRaw : 0; + $previous = \is_numeric($previousRaw) ? (int) $previousRaw : 0; + + return (int) \floor($current + $previous * (1 - $elapsed)); + } + + /** + * Reset + * + * Clear both the current and previous window buckets so the limit starts fresh. + * + * @return void + */ + public function reset(): void + { + $key = $this->parseKey(); + [$windowStart] = $this->window(); + $this->timestamp = $windowStart; + + $this->delete( + $this->bucketKey($key, $windowStart), + $this->bucketKey($key, $windowStart - $this->windowSize), + ); + } + + /** + * No need for manual cleanup - Redis TTL handles this automatically + * + * @param int $timestamp + * @return bool + */ + public function cleanup(int $timestamp): bool + { + return true; + } +} diff --git a/packages/abuse/src/Adapters/SlidingWindow/RedisCluster.php b/packages/abuse/src/Adapters/SlidingWindow/RedisCluster.php new file mode 100644 index 00000000000..6e519a8a6d4 --- /dev/null +++ b/packages/abuse/src/Adapters/SlidingWindow/RedisCluster.php @@ -0,0 +1,91 @@ += $windowSize so the + * previous window's bucket survives long enough to be weighted + * @param \RedisCluster $redis Redis Cluster connection used for storage + */ + public function __construct(protected string $key, protected int $limit, int $windowSize, int $ttl, protected \RedisCluster $redis) + { + $this->initWindow($windowSize, $ttl); + } + + /** + * @param string $script + * @param list $keys + * @param list $argv + * @return mixed + * + * @throws \RedisClusterException + */ + protected function eval(string $script, array $keys, array $argv): mixed + { + return $this->redis->eval($script, [...$keys, ...$argv], \count($keys)); + } + + /** + * @param string $key + * @return mixed + * + * @throws \RedisClusterException + */ + protected function get(string $key): mixed + { + return $this->redis->get($key); + } + + /** + * @param string ...$keys + * @return void + * + * @throws \RedisClusterException + */ + protected function delete(string ...$keys): void + { + $this->redis->del(...$keys); + } + + /** + * Get abuse logs with cursor-based pagination across masters + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = 0, ?int $limit = 25): array + { + $offset = $offset ?? 0; + $limit = $limit ?? 25; + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + foreach ($this->redis->_masters() as $master) { + $cursor = null; + do { + /** @phpstan-ignore-next-line */ + $keys = $this->redis->scan($cursor, $master, $pattern, 100); + if ($keys !== false) { + $matches = array_merge($matches, $keys); + } + } while ($cursor > 0 && count($matches) < $offset + $limit); + } + + sort($matches); + $matches = array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + $values = $this->redis->mget($matches); + + return array_combine($matches, $values); + } +} diff --git a/packages/abuse/src/Adapters/SlidingWindow/RedisPool.php b/packages/abuse/src/Adapters/SlidingWindow/RedisPool.php new file mode 100644 index 00000000000..fd0729a7bba --- /dev/null +++ b/packages/abuse/src/Adapters/SlidingWindow/RedisPool.php @@ -0,0 +1,148 @@ += $windowSize so the + * previous window's bucket survives long enough to be weighted + * @param UtopiaPool<\Redis>|UtopiaPool<\RedisCluster> $pool Pool yielding a Redis or RedisCluster connection + */ + public function __construct( + protected string $key, + protected int $limit, + int $windowSize, + int $ttl, + protected UtopiaPool $pool + ) { + $this->initWindow($windowSize, $ttl); + } + + /** + * @param string $script + * @param list $keys + * @param list $argv + * @return mixed + */ + protected function eval(string $script, array $keys, array $argv): mixed + { + return $this->pool->use(fn (\Redis|\RedisCluster $redis): mixed => $redis->eval($script, [...$keys, ...$argv], \count($keys))); + } + + /** + * @param string $key + * @return mixed + */ + protected function get(string $key): mixed + { + return $this->pool->use(fn (\Redis|\RedisCluster $redis): mixed => $redis->get($key)); + } + + /** + * @param string ...$keys + * @return void + */ + protected function delete(string ...$keys): void + { + $this->pool->use(function (\Redis|\RedisCluster $redis) use ($keys): void { + $redis->del(...$keys); + }); + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + $offset = $offset ?? 0; + $limit = $limit ?? 25; + + /** @var array $result */ + $result = $this->pool->use(function (\Redis|\RedisCluster $redis) use ($offset, $limit): array { + if ($redis instanceof \RedisCluster) { + return $this->getRedisClusterLogs($redis, $offset, $limit); + } + + $cursor = null; + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + do { + $keys = $redis->scan($cursor, $pattern, 100); + if ($keys !== false) { + \array_push($matches, ...$keys); + } + } while ($cursor > 0); + + \sort($matches); + $matches = \array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + $logs = []; + foreach ($matches as $key) { + $logs[$key] = $redis->get($key); + } + + return $logs; + }); + + return $result; + } + + /** + * @param \RedisCluster $redis + * @param int $offset + * @param int $limit + * @return array + */ + private function getRedisClusterLogs(\RedisCluster $redis, int $offset, int $limit): array + { + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + foreach ($redis->_masters() as $master) { + $cursor = null; + do { + /** @phpstan-ignore-next-line */ + $keys = $redis->scan($cursor, $master, $pattern, 100); + if ($keys !== false) { + \array_push($matches, ...$keys); + } + } while ($cursor > 0); + } + + \sort($matches); + $matches = \array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + $values = $redis->mget($matches); + if (!\is_array($values)) { + return []; + } + + $logs = \array_combine($matches, $values); + if (!\is_array($logs)) { + return []; + } + + return $logs; + } +} diff --git a/packages/abuse/src/Adapters/TimeLimit.php b/packages/abuse/src/Adapters/TimeLimit.php new file mode 100644 index 00000000000..073c8f3dff8 --- /dev/null +++ b/packages/abuse/src/Adapters/TimeLimit.php @@ -0,0 +1,121 @@ +limit) { + return false; + } + + $key = $this->parseKey(); + + if ($this->limit > $this->count($key, $this->timestamp)) { + $this->hit($key, $this->timestamp); + + return false; + } + + return true; + } + + /** + * Remaining + * + * Returns the number of current remaining counts + * + * @return int + * + * @throws \Exception + */ + public function remaining(): int + { + $left = $this->limit - ($this->count($this->parseKey(), $this->timestamp) + 1); // Add one because we need to say how many left not how many done + + return (0 > $left) ? 0 : $left; + } + + /** + * Limit + * + * Return the limit integer + * + * @return int + */ + public function limit(): int + { + return $this->limit; + } + + /** + * Time + * + * Return the timestamp + * + * @return int + */ + public function time(): int + { + return $this->timestamp; + } + + /** + * Reset + * + * Reset the count to 0 for the current key and timestamp + * + * @return void + * + * @throws \Exception + */ + public function reset(): void + { + $this->set($this->parseKey(), $this->timestamp, 0); + } +} diff --git a/packages/abuse/src/Adapters/TimeLimit/Appwrite/TablesDB.php b/packages/abuse/src/Adapters/TimeLimit/Appwrite/TablesDB.php new file mode 100644 index 00000000000..fcd2563b1df --- /dev/null +++ b/packages/abuse/src/Adapters/TimeLimit/Appwrite/TablesDB.php @@ -0,0 +1,450 @@ +key = $key; + $now = \time(); + $this->timestamp = (int)($now - ($now % $seconds)); + $this->limit = $limit; + $this->tablesDB = new TablesDBService($client); + $this->databaseId = $databaseId; + } + + /** + * @throws \Exception + */ + public function setup(): void + { + if ($this->isSetupComplete()) { + return; + } + + $this->createDatabase(); + + if (! $this->createTable()) { + // The table is left over from a setup that did not run to completion, + // so some of its columns or indexes may be missing. Inline definitions + // only apply while the table is being created, so add them one by one. + $this->createColumns(); + $this->waitForResourcesReady('columns'); + $this->createIndexes(); + $this->waitForResourcesReady('indexes'); + } + + $this->createLockTable(); + } + + protected function isSetupComplete(): bool + { + try { + $this->tablesDB->getTable($this->databaseId, self::TABLE_LOCK); + return true; + } catch (\Throwable $err) { + return false; + } + } + + protected function createDatabase(): void + { + $this->executeWithSilentError( + fn () => $this->tablesDB->create($this->databaseId, self::DATABASE_NAME), + 'database_already_exists' + ); + } + + /** + * Create the abuse table along with its columns and indexes in one request. + * + * Inline columns and indexes are created synchronously and come back + * available, so there is nothing to poll for afterwards. + * + * @return bool false when the table already existed + */ + protected function createTable(): bool + { + return $this->executeWithSilentError( + fn () => $this->tablesDB->createTable( + $this->databaseId, + self::TABLE_ID, + self::TABLE_NAME, + columns: $this->columnDefinitions(), + indexes: $this->indexDefinitions(), + ), + 'table_already_exists' + ); + } + + /** + * Columns sent inline when the table is created. + * + * createColumns() repairs a table that already exists from the same list. + * + * @return array + */ + protected function columnDefinitions(): array + { + return [ + ['key' => 'key', 'type' => 'string', 'size' => 255, 'required' => true], + ['key' => 'time', 'type' => 'datetime', 'required' => true], + ['key' => 'count', 'type' => 'integer', 'required' => true, 'min' => 0, 'max' => PHP_INT_MAX], + ]; + } + + /** + * Indexes sent inline when the table is created. + * + * createIndexes() repairs a table that already exists from the same list. + * + * An inline definition names its columns under 'attributes', even though + * the index that comes back reports them under 'columns'. + * + * @return array}> + */ + protected function indexDefinitions(): array + { + return [ + ['key' => 'unique1', 'type' => (string) TablesDBIndexType::UNIQUE(), 'attributes' => ['key', 'time']], + ['key' => 'index2', 'type' => (string) TablesDBIndexType::KEY(), 'attributes' => ['time']], + ]; + } + + /** + * Add the columns to a table that already exists, one endpoint per type. + */ + protected function createColumns(): void + { + foreach ($this->columnDefinitions() as $column) { + $key = $column['key']; + $required = $column['required']; + + $createColumnFunction = match ($column['type']) { + 'string' => fn () => $this->tablesDB->createStringColumn($this->databaseId, self::TABLE_ID, $key, $column['size'] ?? 0, $required), + 'datetime' => fn () => $this->tablesDB->createDatetimeColumn($this->databaseId, self::TABLE_ID, $key, $required), + 'integer' => fn () => $this->tablesDB->createIntegerColumn($this->databaseId, self::TABLE_ID, $key, $required, $column['min'] ?? null, $column['max'] ?? null), + default => throw new \Exception("No endpoint for column '{$key}'."), + }; + + $this->executeWithSilentError($createColumnFunction, 'column_already_exists'); + } + } + + /** + * Add the indexes to a table that already exists. + */ + protected function createIndexes(): void + { + foreach ($this->indexDefinitions() as $index) { + $this->executeWithSilentError( + fn () => $this->tablesDB->createIndex( + $this->databaseId, + self::TABLE_ID, + $index['key'], + TablesDBIndexType::from($index['type']), + $index['attributes'], + ), + 'index_already_exists' + ); + } + } + + protected function waitForResourcesReady(string $resourceType): void + { + $attempts = 0; + $maxAttempts = 15; + + while ($attempts < $maxAttempts) { + $attempts++; + + $resources = $resourceType === 'columns' + ? $this->tablesDB->listColumns($this->databaseId, self::TABLE_ID, [Query::notEqual('status', 'available'), Query::limit(1)])->columns + : $this->tablesDB->listIndexes($this->databaseId, self::TABLE_ID, [Query::notEqual('status', 'available'), Query::limit(1)])->indexes; + + $resources = \array_filter($resources, fn ($resource) => $this->resourceStatus($resource) !== 'available'); + + if (\count($resources) === 0) { + return; + } + + \sleep(1); + } + + throw new \Exception("Failed to setup {$resourceType}."); + } + + /** + * Read the status off a listed column or index. + * + * A listed column arrives as the raw payload, since the SDK has no single + * model to hydrate the union of column types into, while a listed index + * arrives as a ColumnIndex. Accept either shape. + */ + protected function resourceStatus(mixed $resource): string + { + $status = null; + + if (\is_array($resource)) { + $status = $resource['status'] ?? null; + } elseif (\is_object($resource) && \property_exists($resource, 'status')) { + $status = $resource->status; + } + + return \is_scalar($status) || $status instanceof \Stringable ? (string) $status : ''; + } + + protected function createLockTable(): void + { + $this->executeWithSilentError( + fn () => $this->tablesDB->createTable($this->databaseId, self::TABLE_LOCK, name: self::TABLE_LOCK), + 'table_already_exists' + ); + } + + /** + * @return bool false when the call failed with the tolerated error + */ + protected function executeWithSilentError(callable $callback, string $allowedErrorType): bool + { + try { + $callback(); + + return true; + } catch (AppwriteException $err) { + if ($err->getType() !== $allowedErrorType) { + throw $err; + } + + return false; + } + } + + /** + * Check + * + * Checks if number of counts is bigger or smaller than current limit + * + * @param string $key + * @param int $timestamp + * @return int + * + * @throws \Exception + */ + protected function count(string $key, int $timestamp): int + { + if (0 == $this->limit) { // No limit no point for counting + return 0; + } + + if (! \is_null($this->count)) { // Get fetched result + return $this->count; + } + + $timestamp = $this->toDateTime($timestamp); + + $rows = $this->tablesDB->listRows($this->databaseId, self::TABLE_ID, [ + Query::equal('key', [$key]), + Query::equal('time', [$timestamp]), + ])->rows; + + $this->count = 0; + + if (\count($rows) === 1) { // Unique Index + $count = $rows[0]->data['count'] ?? 0; + if (\is_numeric($count)) { + $this->count = intval($count); + } + } + + return $this->count; + } + + /** + * @param string $key + * @param int $timestamp + * @return void + * + * @throws \Throwable + */ + protected function hit(string $key, int $timestamp): void + { + if (0 == $this->limit) { // No limit no point for counting + return; + } + + $timestamp = $this->toDateTime($timestamp); + + $rows = $this->tablesDB->listRows($this->databaseId, self::TABLE_ID, [ + Query::equal('key', [$key]), + Query::equal('time', [$timestamp]), + ])->rows; + $row = $rows[0] ?? null; + + if (\is_null($row)) { + $data = [ + 'key' => $key, + 'time' => $timestamp, + 'count' => 1, + ]; + + try { + $this->tablesDB->createRow($this->databaseId, self::TABLE_ID, ID::unique(), $data); + } catch (AppwriteException $err) { + if ($err->getType() !== 'row_already_exists') { + throw $err; + } + + $rows = $this->tablesDB->listRows($this->databaseId, self::TABLE_ID, [ + Query::equal('key', [$key]), + Query::equal('time', [$timestamp]), + ])->rows; + + $row = $rows[0] ?? null; + + if (!is_null($row)) { + $count = $row->data['count'] ?? 0; + if (\is_numeric($count)) { + $this->count = intval($count); + } + + $this->tablesDB->incrementRowColumn($this->databaseId, self::TABLE_ID, $row->id, 'count', 1); + } else { + throw new \Exception('Document Not Found'); + } + } + } else { + $this->tablesDB->incrementRowColumn($this->databaseId, self::TABLE_ID, $row->id, 'count', 1); + } + + $this->count++; + } + + /** + * @param string $key + * @param int $timestamp + * @param int $value + * @return void + * + * @throws \Throwable + */ + protected function set(string $key, int $timestamp, int $value): void + { + $timestamp = $this->toDateTime($timestamp); + + $rows = $this->tablesDB->listRows($this->databaseId, self::TABLE_ID, [ + Query::equal('key', [$key]), + Query::equal('time', [$timestamp]), + ])->rows; + $row = $rows[0] ?? null; + + if (\is_null($row)) { + $data = [ + 'key' => $key, + 'time' => $timestamp, + 'count' => $value, + ]; + + try { + $this->tablesDB->createRow($this->databaseId, self::TABLE_ID, ID::unique(), $data); + } catch (AppwriteException $err) { + if ($err->getType() !== 'row_already_exists') { + throw $err; + } + + $rows = $this->tablesDB->listRows($this->databaseId, self::TABLE_ID, [ + Query::equal('key', [$key]), + Query::equal('time', [$timestamp]), + ])->rows; + + $row = $rows[0] ?? null; + + if (!is_null($row)) { + $this->tablesDB->updateRow($this->databaseId, self::TABLE_ID, $row->id, ['count' => $value]); + } else { + throw new \Exception('Unable to find abuse tracking row after race condition handling'); + } + } + } else { + $this->tablesDB->updateRow($this->databaseId, self::TABLE_ID, $row->id, ['count' => $value]); + } + + $this->count = $value; + } + + /** + * Get abuse logs + * + * Return logs with an optional offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + * + * @throws \Exception + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + $queries = []; + + $queries[] = Query::orderDesc(''); + + if (! \is_null($offset)) { + $queries[] = Query::offset($offset); + } + if (! \is_null($limit)) { + $queries[] = Query::limit($limit); + } + + $rows = $this->tablesDB->listRows($this->databaseId, self::TABLE_ID, $queries)->rows; + + return \array_map(fn (Row $row) => new Document($row->toArray()), $rows); + } + + /** + * Delete logs older than $timestamp seconds + * + * @param int $timestamp + * @return bool + * + * @throws \Exception + */ + public function cleanup(int $timestamp): bool + { + $timestamp = $this->toDateTime($timestamp); + + do { + $response = $this->tablesDB->deleteRows($this->databaseId, self::TABLE_ID, [ + Query::lessThan('time', $timestamp), + ]); + } while ($response->total > 0); + + return true; + } + + protected function toDateTime(int $timestamp): string + { + return (new \DateTime())->setTimestamp($timestamp)->format('Y-m-d H:i:s.v'); + } +} diff --git a/packages/abuse/src/Adapters/TimeLimit/Database.php b/packages/abuse/src/Adapters/TimeLimit/Database.php new file mode 100644 index 00000000000..cf9c30cf618 --- /dev/null +++ b/packages/abuse/src/Adapters/TimeLimit/Database.php @@ -0,0 +1,332 @@ + 'key', + 'type' => UtopiaDB::VAR_STRING, + 'size' => UtopiaDB::LENGTH_KEY, + 'required' => true, + 'signed' => true, + 'array' => false, + 'filters' => [], + ], [ + '$id' => 'time', + 'type' => UtopiaDB::VAR_DATETIME, + 'size' => 0, + 'required' => true, + 'signed' => false, + 'array' => false, + 'filters' => ['datetime'], + ], [ + '$id' => 'count', + 'type' => UtopiaDB::VAR_INTEGER, + 'size' => 11, + 'required' => true, + 'signed' => false, + 'array' => false, + 'filters' => [], + ], + ]; + + public const INDEXES = [ + [ + '$id' => 'unique1', + 'type' => UtopiaDB::INDEX_UNIQUE, + 'attributes' => ['key', 'time'], + 'lengths' => [], + 'orders' => [], + ], [ + '$id' => 'index2', + 'type' => UtopiaDB::INDEX_KEY, + 'attributes' => ['time'], + 'lengths' => [], + 'orders' => [], + ], + ]; + + /** + * @var UtopiaDB + */ + protected UtopiaDB $db; + + /** + * @var int|null + */ + protected ?int $count = null; + + /** + * @param string $key + * @param int $seconds + * @param int $limit + * @param UtopiaDB $db + */ + public function __construct(string $key, int $limit, int $seconds, UtopiaDB $db) + { + $this->key = $key; + $now = \time(); + $this->timestamp = (int)($now - ($now % $seconds)); + $this->limit = $limit; + $this->db = $db; + } + + /** + * @throws Duplicate + * @throws \Exception + */ + public function setup(): void + { + if (! $this->db->exists($this->db->getDatabase())) { + throw new \Exception('You need to create database before running timelimit setup'); + } + + $attributes = \array_map(function ($attribute) { + return new Document($attribute); + }, self::ATTRIBUTES); + + $indexes = \array_map(function ($index) { + return new Document($index); + }, self::INDEXES); + + try { + $this->db->createCollection( + self::COLLECTION, + $attributes, + $indexes + ); + } catch (Duplicate) { + // Collection already exists + } + } + + /** + * Check + * + * Checks if number of counts is bigger or smaller than current limit + * + * @param string $key + * @param int $timestamp + * @return int + * + * @throws \Exception + */ + protected function count(string $key, int $timestamp): int + { + if (0 == $this->limit) { // No limit no point for counting + return 0; + } + + if (! \is_null($this->count)) { // Get fetched result + return $this->count; + } + + $timestamp = $this->toDateTime($timestamp); + + /** @var array $result */ + $result = $this->db->getAuthorization()->skip(function () use ($key, $timestamp) { + return $this->db->find(self::COLLECTION, [ + Query::equal('key', [$key]), + Query::equal('time', [$timestamp]), + ]); + }); + + $this->count = 0; + + if (\count($result) === 1) { // Unique Index + $count = $result[0]->getAttribute('count', 0); + if (\is_numeric($count)) { + $this->count = intval($count); + } + } + + return $this->count; + } + + /** + * @param string $key + * @param int $timestamp + * @return void + * + * @throws AuthorizationException|Structure|\Exception|\Throwable + */ + protected function hit(string $key, int $timestamp): void + { + if (0 == $this->limit) { // No limit no point for counting + return; + } + + $timestamp = $this->toDateTime($timestamp); + $this->db->getAuthorization()->skip(function () use ($timestamp, $key) { + $data = $this->db->findOne(self::COLLECTION, [ + Query::equal('key', [$key]), + Query::equal('time', [$timestamp]), + ]); + + if ($data->isEmpty()) { + $data = [ + '$permissions' => [], + 'key' => $key, + 'time' => $timestamp, + 'count' => 1, + '$collection' => self::COLLECTION, + ]; + + try { + $this->db->createDocument(self::COLLECTION, new Document($data)); + } catch (Duplicate $e) { + // Duplicate in case of race condition + $data = $this->db->findOne(self::COLLECTION, [ + Query::equal('key', [$key]), + Query::equal('time', [$timestamp]), + ]); + + if (!$data->isEmpty()) { + $count = $data->getAttribute('count', 0); + if (\is_numeric($count)) { + $this->count = intval($count); + } + $this->db->increaseDocumentAttribute(self::COLLECTION, $data->getId(), 'count'); + } else { + throw new \Exception('Document Not Found'); + } + } + } else { + /** @var Document $data */ + $this->db->increaseDocumentAttribute(self::COLLECTION, $data->getId(), 'count'); + } + }); + + $this->count++; + } + + /** + * @param string $key + * @param int $timestamp + * @param int $value + * @return void + * + * @throws AuthorizationException|Structure|\Exception|\Throwable + */ + protected function set(string $key, int $timestamp, int $value): void + { + $timestamp = $this->toDateTime($timestamp); + $this->db->getAuthorization()->skip(function () use ($timestamp, $key, $value) { + $data = $this->db->findOne(self::COLLECTION, [ + Query::equal('key', [$key]), + Query::equal('time', [$timestamp]), + ]); + + if ($data->isEmpty()) { + $data = [ + '$permissions' => [], + 'key' => $key, + 'time' => $timestamp, + 'count' => $value, + '$collection' => self::COLLECTION, + ]; + + try { + $this->db->createDocument(self::COLLECTION, new Document($data)); + } catch (Duplicate $e) { + // Duplicate in case of race condition - update existing document + $data = $this->db->findOne(self::COLLECTION, [ + Query::equal('key', [$key]), + Query::equal('time', [$timestamp]), + ]); + + if (!$data->isEmpty()) { + /** @var Document $data */ + $this->db->updateDocument(self::COLLECTION, $data->getId(), new Document([ + 'count' => $value, + ])); + } else { + throw new \Exception('Unable to find abuse tracking document after race condition handling'); + } + } + } else { + /** @var Document $data */ + $this->db->updateDocument(self::COLLECTION, $data->getId(), new Document([ + 'count' => $value, + ])); + } + }); + + $this->count = $value; + } + + /** + * Get abuse logs + * + * Return logs with an optional offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + * + * @throws \Exception + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + /** @var array $results */ + $results = $this->db->getAuthorization()->skip(function () use ($offset, $limit) { + $queries = []; + $queries[] = Query::orderDesc(''); + + if (! \is_null($offset)) { + $queries[] = Query::offset($offset); + } + if (! \is_null($limit)) { + $queries[] = Query::limit($limit); + } + + return $this->db->find(self::COLLECTION, $queries); + }); + + return $results; + } + + /** + * Delete logs older than $timestamp seconds + * + * @param int $timestamp + * @return bool + * + * @throws AuthorizationException|\Exception + */ + public function cleanup(int $timestamp): bool + { + $timestamp = $this->toDateTime($timestamp); + $this->db->getAuthorization()->skip(function () use ($timestamp) { + do { + $documents = $this->db->find(self::COLLECTION, [ + Query::lessThan('time', $timestamp), + ]); + + foreach ($documents as $document) { + $this->db->deleteDocument(self::COLLECTION, $document->getId()); + } + } while (! empty($documents)); + }); + + return true; + } + + protected function toDateTime(int $timestamp): string + { + return DateTime::format((new \DateTime())->setTimestamp($timestamp)); + } +} diff --git a/packages/abuse/src/Adapters/TimeLimit/None.php b/packages/abuse/src/Adapters/TimeLimit/None.php new file mode 100644 index 00000000000..37629440e9b --- /dev/null +++ b/packages/abuse/src/Adapters/TimeLimit/None.php @@ -0,0 +1,60 @@ +key = $key; + $this->ttl = $seconds; + $now = \time(); + $this->timestamp = (int) ($now - ($now % $seconds)); + $this->limit = $limit; + } + + protected function count(string $key, int $timestamp): int + { + return 0; + } + + protected function hit(string $key, int $timestamp): void + { + } + + protected function set(string $key, int $timestamp, int $value): void + { + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + return []; + } + + /** + * Delete all logs older than $timestamp + * + * @param int $timestamp + * @return bool + */ + public function cleanup(int $timestamp): bool + { + return true; + } +} diff --git a/packages/abuse/src/Adapters/TimeLimit/Redis.php b/packages/abuse/src/Adapters/TimeLimit/Redis.php new file mode 100644 index 00000000000..2ffb02090ba --- /dev/null +++ b/packages/abuse/src/Adapters/TimeLimit/Redis.php @@ -0,0 +1,135 @@ +redis = $redis; + $this->key = $key; + $this->ttl = $seconds; + $now = \time(); + $this->timestamp = (int)($now - ($now % $seconds)); + $this->limit = $limit; + } + + /** + * Undocumented function + * + * @param string $key + * @param int $timestamp + * @return integer + */ + protected function count(string $key, int $timestamp): int + { + if (0 == $this->limit) { // No limit no point for counting + return 0; + } + + if (! \is_null($this->count)) { // Get fetched result + return $this->count; + } + + /** @var string $count */ + $count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $timestamp); + if (!$count) { + $this->count = 0; + } else { + $this->count = intval($count); + } + + return $this->count; + } + + /** + * @param string $key + * @param int $timestamp + * @return void + * + */ + protected function hit(string $key, int $timestamp): void + { + if (0 == $this->limit) { // No limit no point for counting + return; + } + + $key = self::NAMESPACE . '__' . $key . '__' . $timestamp; + $this->redis->multi() + ->incr($key) + ->expire($key, $this->ttl) + ->exec(); + + $this->count = ($this->count ?? 0) + 1; + } + + /** + * Set count for a key at specific timestamp + * + * @param string $key + * @param int $timestamp + * @param int $value + * @return void + */ + protected function set(string $key, int $timestamp, int $value): void + { + $key = self::NAMESPACE . '__' . $key . '__' . $timestamp; + $this->redis->multi() + ->set($key, (string)$value) + ->expire($key, $this->ttl) + ->exec(); + + $this->count = $value; + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + // TODO limit potential is SCAN but needs cursor no offset + $cursor = null; + $keys = $this->redis->scan($cursor, self::NAMESPACE . '__*', $limit); + if (!$keys) { + return []; + } + + $logs = []; + foreach ($keys as $key) { + $logs[$key] = $this->redis->get($key); + } + return $logs; + } + + /** + * Delete all logs older than $timestamp + * + * @param int $timestamp + * @return bool + */ + public function cleanup(int $timestamp): bool + { + // No need for manual cleanup - Redis TTL handles this automatically + return true; + } +} diff --git a/packages/abuse/src/Adapters/TimeLimit/RedisCluster.php b/packages/abuse/src/Adapters/TimeLimit/RedisCluster.php new file mode 100644 index 00000000000..731415e9ae0 --- /dev/null +++ b/packages/abuse/src/Adapters/TimeLimit/RedisCluster.php @@ -0,0 +1,154 @@ +redis = $redis; + $this->key = $key; + $this->ttl = $seconds; + $now = \time(); + $this->timestamp = (int)($now - ($now % $seconds)); + $this->limit = $limit; + } + + /** + * Get count for a key at specific timestamp + * + * @param string $key + * @param int $timestamp + * @return integer + */ + protected function count(string $key, int $timestamp): int + { + if (0 == $this->limit) { // No limit no point for counting + return 0; + } + + if (! \is_null($this->count)) { // Get fetched result + return $this->count; + } + + /** @var string|false $count */ + $count = $this->redis->get(self::NAMESPACE . '__'. $key .'__'. $timestamp); + if ($count === false) { + $this->count = 0; + } else { + $this->count = intval($count); + } + + return $this->count; + } + + /** + * Record a hit for a key at specific timestamp + * + * @param string $key + * @param int $timestamp + * @return void + */ + protected function hit(string $key, int $timestamp): void + { + if (0 == $this->limit) { // No limit no point for counting + return; + } + + $key = self::NAMESPACE . '__'. $key .'__'. $timestamp; + + $this->redis->multi(); + $this->redis->incr($key); + $this->redis->expire($key, $this->ttl); + $this->redis->exec(); + + $this->count = ($this->count ?? 0) + 1; + } + + /** + * Set count for a key at specific timestamp + * + * @param string $key + * @param int $timestamp + * @param int $value + * @return void + */ + protected function set(string $key, int $timestamp, int $value): void + { + + $key = self::NAMESPACE . '__' . $key . '__' . $timestamp; + + $this->redis->multi(); + $this->redis->set($key, (string)$value); + $this->redis->expire($key, $this->ttl); + $this->redis->exec(); + + $this->count = $value; + } + + /** + * Get abuse logs with proper cursor-based pagination + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = 0, ?int $limit = 25): array + { + $offset = $offset ?? 0; + $limit = $limit ?? 25; + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + // Get all keys from each master + foreach ($this->redis->_masters() as $master) { + $cursor = null; + do { + /** @phpstan-ignore-next-line */ + $keys = $this->redis->scan($cursor, $master, $pattern, 100); + if ($keys !== false) { + $matches = array_merge($matches, $keys); + } + } while ($cursor > 0 && count($matches) < $offset + $limit); + } + + // Sort to ensure consistent ordering + sort($matches); + + // Apply offset and limit + $matches = array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + // Batch fetch values using mget + $values = $this->redis->mget($matches); + return array_combine($matches, $values); + } + + /** + * No need for manual cleanup - using Redis TTL + * + * @param int $timestamp + * @return bool + */ + public function cleanup(int $timestamp): bool + { + return true; + } +} diff --git a/packages/abuse/src/Adapters/TimeLimit/RedisPool.php b/packages/abuse/src/Adapters/TimeLimit/RedisPool.php new file mode 100644 index 00000000000..a7a4cc87f20 --- /dev/null +++ b/packages/abuse/src/Adapters/TimeLimit/RedisPool.php @@ -0,0 +1,215 @@ +|UtopiaPool<\RedisCluster> $pool + */ + public function __construct( + string $key, + int $limit, + int $seconds, + protected UtopiaPool $pool + ) { + $this->key = $key; + $this->ttl = $seconds; + $now = \time(); + $this->timestamp = (int) ($now - ($now % $seconds)); + $this->limit = $limit; + } + + protected function count(string $key, int $timestamp): int + { + if (0 == $this->limit) { + return 0; + } + + if (!\is_null($this->count)) { + return $this->count; + } + + /** @var int $count */ + $count = $this->pool->use(function (\Redis|\RedisCluster $redis) use ($key, $timestamp): int { + $count = $redis->get(Redis::NAMESPACE . '__' . $key . '__' . $timestamp); + + return \is_numeric($count) ? (int) $count : 0; + }); + + $this->count = $count; + + return $this->count; + } + + protected function hit(string $key, int $timestamp): void + { + if (0 == $this->limit) { + return; + } + + $ttl = $this->ttl; + $key = Redis::NAMESPACE . '__' . $key . '__' . $timestamp; + + $this->pool->use(function (\Redis|\RedisCluster $redis) use ($key, $ttl): void { + $redis->multi(); + try { + $redis->incr($key); + $redis->expire($key, $ttl); + $result = $redis->exec(); + } catch (Throwable $th) { + $this->discard($redis); + throw $th; + } + + if (!\is_array($result) || \in_array(false, $result, true)) { + $this->discard($redis); + throw new RuntimeException('Redis transaction failed.'); + } + }); + + $this->count = ($this->count ?? 0) + 1; + } + + protected function set(string $key, int $timestamp, int $value): void + { + $ttl = $this->ttl; + $key = Redis::NAMESPACE . '__' . $key . '__' . $timestamp; + + $this->pool->use(function (\Redis|\RedisCluster $redis) use ($key, $ttl, $value): void { + $redis->multi(); + try { + $redis->set($key, (string) $value); + $redis->expire($key, $ttl); + $result = $redis->exec(); + } catch (Throwable $th) { + $this->discard($redis); + throw $th; + } + + if (!\is_array($result) || \in_array(false, $result, true)) { + $this->discard($redis); + throw new RuntimeException('Redis transaction failed.'); + } + }); + + $this->count = $value; + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + $offset = $offset ?? 0; + $limit = $limit ?? 25; + + /** @var array $result */ + $result = $this->pool->use(function (\Redis|\RedisCluster $redis) use ($offset, $limit): array { + if ($redis instanceof \RedisCluster) { + return $this->getRedisClusterLogs($redis, $offset, $limit); + } + + $cursor = null; + $matches = []; + $pattern = Redis::NAMESPACE . '__*'; + + do { + $keys = $redis->scan($cursor, $pattern, 100); + if ($keys !== false) { + \array_push($matches, ...$keys); + } + } while ($cursor > 0); + + \sort($matches); + $matches = \array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + $logs = []; + foreach ($matches as $key) { + $logs[$key] = $redis->get($key); + } + + return $logs; + }); + + return $result; + } + + /** + * Delete all logs older than $timestamp + * + * @param int $timestamp + * @return bool + */ + public function cleanup(int $timestamp): bool + { + return true; + } + + private function discard(\Redis|\RedisCluster $redis): void + { + try { + $redis->discard(); + } catch (Throwable) { + } + } + + /** + * @return array + */ + private function getRedisClusterLogs(\RedisCluster $redis, int $offset, int $limit): array + { + $matches = []; + $pattern = Redis::NAMESPACE . '__*'; + + foreach ($redis->_masters() as $master) { + $cursor = null; + do { + /** @phpstan-ignore-next-line */ + $keys = $redis->scan($cursor, $master, $pattern, 100); + if ($keys !== false) { + \array_push($matches, ...$keys); + } + } while ($cursor > 0); + } + + \sort($matches); + $matches = \array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + $values = $redis->mget($matches); + if (!\is_array($values)) { + return []; + } + + $logs = \array_combine($matches, $values); + if (!\is_array($logs)) { + return []; + } + + return $logs; + } +} diff --git a/packages/abuse/src/Adapters/TokenBucket.php b/packages/abuse/src/Adapters/TokenBucket.php new file mode 100644 index 00000000000..19d50d30352 --- /dev/null +++ b/packages/abuse/src/Adapters/TokenBucket.php @@ -0,0 +1,102 @@ +tokens - ($this->count($this->parseKey(), $this->timestamp) + 1); + + return (0 > $left) ? 0 : $left; + } + + /** + * Limit + * + * Return the bucket capacity + * + * @return int + */ + public function limit(): int + { + return $this->tokens; + } + + /** + * Time + * + * Return the timestamp + * + * @return int + */ + public function time(): int + { + return $this->timestamp; + } + + /** + * Reset + * + * Clear the bucket for the current key so it starts full again. + * + * @return void + * + * @throws \Exception + */ + abstract public function reset(): void; +} diff --git a/packages/abuse/src/Adapters/TokenBucket/None.php b/packages/abuse/src/Adapters/TokenBucket/None.php new file mode 100644 index 00000000000..cdec3544cc1 --- /dev/null +++ b/packages/abuse/src/Adapters/TokenBucket/None.php @@ -0,0 +1,57 @@ +key = $key; + $this->tokens = $tokens; + $this->timestamp = \time(); + } + + protected function count(string $key, int $timestamp): int + { + return 0; + } + + public function check(): bool + { + return false; + } + + public function reset(): void + { + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + return []; + } + + /** + * Delete all logs older than $timestamp + * + * @param int $timestamp + * @return bool + */ + public function cleanup(int $timestamp): bool + { + return true; + } +} diff --git a/packages/abuse/src/Adapters/TokenBucket/Redis.php b/packages/abuse/src/Adapters/TokenBucket/Redis.php new file mode 100644 index 00000000000..58e24026d5e --- /dev/null +++ b/packages/abuse/src/Adapters/TokenBucket/Redis.php @@ -0,0 +1,81 @@ +initBucket($refillRate); + } + + /** + * @param string $script + * @param list $keys + * @param list $argv + * @return mixed + * + * @throws \RedisException + */ + protected function eval(string $script, array $keys, array $argv): mixed + { + return $this->redis->eval($script, [...$keys, ...$argv], \count($keys)); + } + + /** + * @param string ...$keys + * @return void + * + * @throws \RedisException + */ + protected function delete(string ...$keys): void + { + $this->redis->del(...$keys); + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + $offset = $offset ?? 0; + $limit = $limit ?? 25; + + $cursor = null; + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + do { + $keys = $this->redis->scan($cursor, $pattern, 100); + if ($keys !== false) { + \array_push($matches, ...$keys); + } + } while ($cursor > 0); + + \sort($matches); + $matches = \array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + $logs = []; + foreach ($matches as $key) { + $logs[$key] = $this->redis->hGetAll($key); + } + + return $logs; + } +} diff --git a/packages/abuse/src/Adapters/TokenBucket/RedisBase.php b/packages/abuse/src/Adapters/TokenBucket/RedisBase.php new file mode 100644 index 00000000000..6fa8ec9cdfa --- /dev/null +++ b/packages/abuse/src/Adapters/TokenBucket/RedisBase.php @@ -0,0 +1,228 @@ += 1 then + tokens = tokens - 1 + allowed = 1 + end + + redis.call('HSET', key, 'tokens', tostring(tokens), 'last_refill', tostring(now)) + redis.call('EXPIRE', key, math.ceil(max_tokens / refill_rate) + 1) + + return { allowed, tostring(tokens) } + LUA; + + /** + * Read-only token estimate: refills the bucket for the elapsed time without + * consuming anything or writing back. Used by remaining(). + * + * KEYS[1] bucket hash key. + * ARGV[1] max_tokens, ARGV[2] refill_rate, ARGV[3] now. + * + * Returns the available token balance as a string. + */ + protected const string TOKENS_SCRIPT = <<<'LUA' + local key = KEYS[1] + local max_tokens = tonumber(ARGV[1]) + local refill_rate = tonumber(ARGV[2]) + local now = tonumber(ARGV[3]) + + local data = redis.call('HMGET', key, 'tokens', 'last_refill') + local tokens = tonumber(data[1]) or max_tokens + local last_refill = tonumber(data[2]) or now + + local elapsed = now - last_refill + if elapsed < 0 then elapsed = 0 end + tokens = math.min(max_tokens, tokens + elapsed * refill_rate) + + return tostring(tokens) + LUA; + + /** + * Tokens refilled per second. + * + * @var float + */ + protected float $refillRate; + + /** + * Run a Lua script against the storage backend. + * + * @param string $script + * @param list $keys + * @param list $argv + * @return mixed the raw script result + */ + abstract protected function eval(string $script, array $keys, array $argv): mixed; + + /** + * Delete the given keys. + * + * @param string ...$keys + * @return void + */ + abstract protected function delete(string ...$keys): void; + + /** + * Validate and store the bucket configuration. + * + * @param float $refillRate + * @return void + */ + protected function initBucket(float $refillRate): void + { + if ($refillRate <= 0) { + throw new \InvalidArgumentException('refillRate must be greater than 0'); + } + + $this->refillRate = $refillRate; + $this->timestamp = \time(); + } + + /** + * Build the bucket hash key for a given abuse key. + * + * @param string $key + * @return string + */ + protected function bucketKey(string $key): string + { + return self::NAMESPACE . '__' . $key; + } + + /** + * Check + * + * @return bool + * + * @throws \Throwable + */ + public function check(): bool + { + if ($this->tokens === 0) { + return false; + } + + $key = $this->parseKey(); + $this->timestamp = \time(); + + /** @var array{0:int,1:string} $result */ + $result = $this->eval( + self::LIMIT_CHECK_SCRIPT, + [ + $this->bucketKey($key), // KEYS[1] bucket hash + ], + [ + $this->tokens, // ARGV[1] max_tokens + $this->refillRate, // ARGV[2] refill_rate + \microtime(true), // ARGV[3] now (fractional seconds) + ], + ); + + [$allowed] = $result; + + return (int) $allowed === 0; + } + + /** + * Count + * + * Read-only estimate of the tokens already consumed from the bucket + * (capacity minus the tokens available after refilling). Used by remaining(). + * The bucket refills continuously, so this always reads a fresh estimate + * rather than reusing a cached value that would go stale as tokens refill. + * + * @param string $key + * @param int $timestamp + * @return int + */ + protected function count(string $key, int $timestamp): int + { + if ($this->tokens === 0) { + return 0; + } + + $this->timestamp = \time(); + + $raw = $this->eval( + self::TOKENS_SCRIPT, + [ + $this->bucketKey($key), + ], + [ + $this->tokens, + $this->refillRate, + \microtime(true), + ], + ); + + $balance = \is_numeric($raw) ? (float) $raw : (float) $this->tokens; + + return $this->tokens - (int) \floor($balance); + } + + /** + * Reset + * + * Drop the bucket state so the next request sees a full bucket. + * + * @return void + */ + public function reset(): void + { + $this->delete($this->bucketKey($this->parseKey())); + } + + /** + * No need for manual cleanup - Redis TTL handles this automatically + * + * @param int $timestamp + * @return bool + */ + public function cleanup(int $timestamp): bool + { + return true; + } +} diff --git a/packages/abuse/src/Adapters/TokenBucket/RedisCluster.php b/packages/abuse/src/Adapters/TokenBucket/RedisCluster.php new file mode 100644 index 00000000000..beb86ec0a1c --- /dev/null +++ b/packages/abuse/src/Adapters/TokenBucket/RedisCluster.php @@ -0,0 +1,81 @@ +initBucket($refillRate); + } + + /** + * @param string $script + * @param list $keys + * @param list $argv + * @return mixed + * + * @throws \RedisClusterException + */ + protected function eval(string $script, array $keys, array $argv): mixed + { + return $this->redis->eval($script, [...$keys, ...$argv], \count($keys)); + } + + /** + * @param string ...$keys + * @return void + * + * @throws \RedisClusterException + */ + protected function delete(string ...$keys): void + { + $this->redis->del(...$keys); + } + + /** + * Get abuse logs with cursor-based pagination across masters + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = 0, ?int $limit = 25): array + { + $offset = $offset ?? 0; + $limit = $limit ?? 25; + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + foreach ($this->redis->_masters() as $master) { + $cursor = null; + do { + /** @phpstan-ignore-next-line */ + $keys = $this->redis->scan($cursor, $master, $pattern, 100); + if ($keys !== false) { + $matches = array_merge($matches, $keys); + } + } while ($cursor > 0); + } + + sort($matches); + $matches = array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + $logs = []; + foreach ($matches as $key) { + $logs[$key] = $this->redis->hGetAll($key); + } + + return $logs; + } +} diff --git a/packages/abuse/src/Adapters/TokenBucket/RedisPool.php b/packages/abuse/src/Adapters/TokenBucket/RedisPool.php new file mode 100644 index 00000000000..34cfec93c27 --- /dev/null +++ b/packages/abuse/src/Adapters/TokenBucket/RedisPool.php @@ -0,0 +1,131 @@ +|UtopiaPool<\RedisCluster> $pool Pool yielding a Redis or RedisCluster connection + */ + public function __construct( + protected string $key, + protected int $tokens, + float $refillRate, + protected UtopiaPool $pool + ) { + $this->initBucket($refillRate); + } + + /** + * @param string $script + * @param list $keys + * @param list $argv + * @return mixed + */ + protected function eval(string $script, array $keys, array $argv): mixed + { + return $this->pool->use(fn (\Redis|\RedisCluster $redis): mixed => $redis->eval($script, [...$keys, ...$argv], \count($keys))); + } + + /** + * @param string ...$keys + * @return void + */ + protected function delete(string ...$keys): void + { + $this->pool->use(function (\Redis|\RedisCluster $redis) use ($keys): void { + $redis->del(...$keys); + }); + } + + /** + * Get abuse logs + * + * Return logs with an offset and limit + * + * @param int|null $offset + * @param int|null $limit + * @return array + */ + public function getLogs(?int $offset = null, ?int $limit = 25): array + { + $offset = $offset ?? 0; + $limit = $limit ?? 25; + + /** @var array $result */ + $result = $this->pool->use(function (\Redis|\RedisCluster $redis) use ($offset, $limit): array { + if ($redis instanceof \RedisCluster) { + return $this->getRedisClusterLogs($redis, $offset, $limit); + } + + $cursor = null; + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + do { + $keys = $redis->scan($cursor, $pattern, 100); + if ($keys !== false) { + \array_push($matches, ...$keys); + } + } while ($cursor > 0); + + \sort($matches); + $matches = \array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + $logs = []; + foreach ($matches as $key) { + $logs[$key] = $redis->hGetAll($key); + } + + return $logs; + }); + + return $result; + } + + /** + * @param \RedisCluster $redis + * @param int $offset + * @param int $limit + * @return array + */ + private function getRedisClusterLogs(\RedisCluster $redis, int $offset, int $limit): array + { + $matches = []; + $pattern = self::NAMESPACE . '__*'; + + foreach ($redis->_masters() as $master) { + $cursor = null; + do { + /** @phpstan-ignore-next-line */ + $keys = $redis->scan($cursor, $master, $pattern, 100); + if ($keys !== false) { + \array_push($matches, ...$keys); + } + } while ($cursor > 0); + } + + \sort($matches); + $matches = \array_slice($matches, $offset, $limit); + + if (empty($matches)) { + return []; + } + + $logs = []; + foreach ($matches as $key) { + $logs[$key] = $redis->hGetAll($key); + } + + return $logs; + } +} diff --git a/packages/abuse/tests/E2E/Appwrite/TablesDBTest.php b/packages/abuse/tests/E2E/Appwrite/TablesDBTest.php new file mode 100755 index 00000000000..1d9c023ba4e --- /dev/null +++ b/packages/abuse/tests/E2E/Appwrite/TablesDBTest.php @@ -0,0 +1,184 @@ +setEndpoint(\getenv('APPWRITE_ENDPOINT') ?: '') + ->setProject(\getenv('APPWRITE_PROJECT_ID') ?: '') + ->setKey(\getenv('APPWRITE_API_KEY') ?: ''); + + $adapter = new TablesDB('', 1, 1, self::$client, self::$databaseId); + $adapter->setup(); + } + + public function getAdapter(string $key, int $limit, int $seconds): TimeLimit + { + return new TablesDB($key, $limit, $seconds, self::$client, self::$databaseId); + } + + /** + * The schema is sent inline with the table, so assert it lands exactly as + * the dedicated per-column endpoints would have created it. + */ + public function testSetupCreatesSchema(): void + { + $tablesDB = new TablesDBService(self::$client); + + $columns = $this->columnsByKey($tablesDB->listColumns(self::$databaseId, TablesDB::TABLE_ID)->columns); + + $this->assertCount(3, $columns); + + $this->assertSame('string', $columns['key']['type']); + $this->assertSame(255, $columns['key']['size']); + $this->assertTrue($columns['key']['required']); + + $this->assertSame('datetime', $columns['time']['type']); + $this->assertTrue($columns['time']['required']); + + $this->assertSame('integer', $columns['count']['type']); + $this->assertTrue($columns['count']['required']); + $this->assertEquals(0, $columns['count']['min']); + $this->assertEquals(PHP_INT_MAX, $columns['count']['max']); + + $indexes = $this->indexesByKey($tablesDB->listIndexes(self::$databaseId, TablesDB::TABLE_ID)->indexes); + + $this->assertCount(2, $indexes); + + $this->assertSame('unique', $indexes['unique1']->type); + $this->assertSame(['key', 'time'], $indexes['unique1']->columns); + + $this->assertSame('key', $indexes['index2']->type); + $this->assertSame(['time'], $indexes['index2']->columns); + } + + /** + * A table left behind by a setup that did not run to completion is missing + * its columns and indexes, and they can no longer be sent inline. Setup has + * to fill them in one by one instead. + */ + public function testSetupRepairsPartiallyCreatedTable(): void + { + $databaseId = 'abuse-cicd-repair-' . \uniqid(); + $tablesDB = new TablesDBService(self::$client); + + $tablesDB->create($databaseId, TablesDB::DATABASE_NAME); + + try { + $tablesDB->createTable($databaseId, TablesDB::TABLE_ID, TablesDB::TABLE_NAME); + + $adapter = new TablesDB('repair-{{ip}}', 2, 60, self::$client, $databaseId); + $adapter->setup(); + + $columns = $this->columnsByKey($tablesDB->listColumns($databaseId, TablesDB::TABLE_ID)->columns); + $indexes = $this->indexesByKey($tablesDB->listIndexes($databaseId, TablesDB::TABLE_ID)->indexes); + + $this->assertCount(3, $columns); + $this->assertArrayHasKey('key', $columns); + $this->assertArrayHasKey('time', $columns); + $this->assertArrayHasKey('count', $columns); + + $this->assertCount(2, $indexes); + $this->assertArrayHasKey('unique1', $indexes); + $this->assertArrayHasKey('index2', $indexes); + + $adapter->setParam('{{ip}}', '0.0.0.20'); + $abuse = new Abuse($adapter); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), true); + } finally { + $tablesDB->delete($databaseId); + } + } + + /** + * Setup runs on every boot, so it has to be a no-op once the table is there. + */ + public function testSetupIsIdempotent(): void + { + $adapter = new TablesDB('', 1, 1, self::$client, self::$databaseId); + $adapter->setup(); + + $tablesDB = new TablesDBService(self::$client); + + $this->assertCount(3, $tablesDB->listColumns(self::$databaseId, TablesDB::TABLE_ID)->columns); + $this->assertCount(2, $tablesDB->listIndexes(self::$databaseId, TablesDB::TABLE_ID)->indexes); + } + + /** + * A listed column arrives as the raw payload: the SDK has no single model + * to hydrate the union of column types into. + * + * @param array $columns + * @return array> + */ + private function columnsByKey(array $columns): array + { + $byKey = []; + + foreach ($columns as $column) { + $this->assertIsArray($column); + $this->assertSame('available', $column['status']); + + $key = $column['key']; + $this->assertIsString($key); + + $byKey[$key] = $column; + } + + return $byKey; + } + + /** + * A listed index, unlike a column, arrives hydrated. + * + * @param array $indexes + * @return array + */ + private function indexesByKey(array $indexes): array + { + $byKey = []; + + foreach ($indexes as $index) { + $this->assertInstanceOf(ColumnIndex::class, $index); + $this->assertSame('available', $index->status); + + $byKey[$index->key] = $index; + } + + return $byKey; + } + + public static function tearDownAfterClass(): void + { + } +} diff --git a/packages/abuse/tests/E2E/Base.php b/packages/abuse/tests/E2E/Base.php new file mode 100644 index 00000000000..4390245b304 --- /dev/null +++ b/packages/abuse/tests/E2E/Base.php @@ -0,0 +1,138 @@ +getAdapter('static-key', 2, 1); + $abuse = new Abuse($adapter); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), true); + } + + /** + * Test a dynamic key with a limit of 2 requests per second + */ + public function testDynamicKey(): void + { + $adapter = $this->getAdapter('dynamic-key-{{ip}}', 2, 1); + $adapter->setParam('{{ip}}', '0.0.0.10'); + $abuse = new Abuse($adapter); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), true); + } + + /** + * Test a dynamic key with 2 params + */ + public function testDynamicKeyWith2Params(): void + { + $adapter = $this->getAdapter('two-params-{{ip}}-{{email}}', 2, 1); + $adapter->setParam('{{ip}}', '0.0.0.10'); + $adapter->setParam('{{email}}', 'test@test.com'); + $abuse = new Abuse($adapter); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), true); + } + + /** + * Test a dynamic key with higher request rate like 10 requests per second + */ + public function testDynamicKeyFastRequests(): void + { + $adapter = $this->getAdapter('fast-requests-{{ip}}', 10, 1); + $adapter->setParam('{{ip}}', '0.0.0.10'); + $abuse = new Abuse($adapter); + for ($i = 0; $i < 10; $i++) { + $this->assertSame($abuse->check(), false); + } + $this->assertSame($abuse->check(), true); + } + + /** + * Test that the limit is reset after the time limit + */ + public function testLimitReset(): void + { + $adapter = $this->getAdapter('limit-reset-{{ip}}', 10, 2); + $adapter->setParam('{{ip}}', '127.0.0.1'); + $abuse = new Abuse($adapter); + for ($i = 0; $i < 10; $i++) { + $this->assertSame($abuse->check(), false); + } + $this->assertSame($abuse->check(), true); + + // Wait for the limit to reset + sleep(2); + + /** Seems to be a bug in the code where if use the same adapter, it caches the result of the previous check */ + $adapter = $this->getAdapter('limit-reset-{{ip}}', 10, 1); + $adapter->setParam('{{ip}}', '127.0.0.1'); + $abuse = new Abuse($adapter); + $this->assertSame($abuse->check(), false); + } + + /** + * Verify that the time format is correct + */ + public function testTimeFormat(): void + { + $now = time(); + $adapter = $this->getAdapter('', 1, 1); + $this->assertSame($adapter->time(), $now); + $this->assertSame(true, \is_int($adapter->time())); + } + + /** + * Test the reset functionality + */ + public function testReset(): void + { + $adapter = $this->getAdapter('reset-test-{{ip}}', 5, 600); + $adapter->setParam('{{ip}}', '192.168.1.1'); + $abuse = new Abuse($adapter); + + // 5 OK, 6th has limit + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), true); + + // Reset the count + $abuse->reset(); + + // Should be 5 more OK, then 6th limit + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), false); + $this->assertSame($abuse->check(), true); + + // TO be sure, lets do bunch of requests with resets + // All should pass successfully + $adapter = $this->getAdapter('reset-test-{{ip}}', 2, 600); + $adapter->setParam('{{ip}}', '192.168.1.2'); + $abuse = new Abuse($adapter); + for ($i = 0; $i < 15; $i++) { + $this->assertSame($abuse->check(), false); + $abuse->reset(); + } + } +} diff --git a/packages/abuse/tests/E2E/DatabaseTest.php b/packages/abuse/tests/E2E/DatabaseTest.php new file mode 100755 index 00000000000..82cad70549e --- /dev/null +++ b/packages/abuse/tests/E2E/DatabaseTest.php @@ -0,0 +1,59 @@ +setDatabase('utopiaTests'); + $db->setNamespace('namespace'); + + $adapter = new AdapterDatabase('', 1, 1, $db); + if (!$db->exists('utopiaTests')) { + $db->create(); + $adapter->setup(); + } + + return $db; + } + + public function getAdapter(string $key, int $limit, int $seconds): TimeLimit + { + return new AdapterDatabase($key, $limit, $seconds, self::$db); + } + + public static function tearDownAfterClass(): void + { + if (isset(self::$db)) { + self::$db->delete(); + } + } +} diff --git a/packages/abuse/tests/E2E/RedisClusterTest.php b/packages/abuse/tests/E2E/RedisClusterTest.php new file mode 100644 index 00000000000..dbcda580c52 --- /dev/null +++ b/packages/abuse/tests/E2E/RedisClusterTest.php @@ -0,0 +1,43 @@ +close(); + } + } +} diff --git a/packages/abuse/tests/E2E/RedisPoolClusterTest.php b/packages/abuse/tests/E2E/RedisPoolClusterTest.php new file mode 100644 index 00000000000..b394941ec92 --- /dev/null +++ b/packages/abuse/tests/E2E/RedisPoolClusterTest.php @@ -0,0 +1,104 @@ +|null + */ + protected static ?Pool $pool = null; + + public static function setUpBeforeClass(): void + { + if (isset(self::$pool)) { + return; + } + + self::$pool = new Pool(new Stack(), 'abuse-redis-cluster', 2, fn (): \RedisCluster => new \RedisCluster(null, Services::CLUSTER_SEEDS), timeout: 0.0); + } + + public function getAdapter(string $key, int $limit, int $seconds): TimeLimit + { + $pool = self::$pool; + $this->assertInstanceOf(Pool::class, $pool); + + /** @var Pool<\RedisCluster> $pool */ + return new AdapterRedisPool('redis-cluster-pool-' . $key, $limit, $seconds, $pool); + } + + public function testGetLogsSupportsNullableLimit(): void + { + $adapter = $this->getAdapter('logs-null-limit', 1, 60); + $abuse = new \Utopia\Abuse\Abuse($adapter); + + $this->assertSame(false, $abuse->check()); + $this->assertNotEmpty($adapter->getLogs(null, null)); + } + + public function testGetLogsAppliesOffset(): void + { + $this->clearRedisClusterPoolLogs(); + $adapter = $this->getAdapter('logs-offset', 1, 60); + + $this->setRedisClusterPoolLog('a', '1'); + $this->setRedisClusterPoolLog('b', '2'); + $this->setRedisClusterPoolLog('c', '3'); + + $logs = $adapter->getLogs(1, 1); + + $this->assertSame(['abuse__redis-cluster-pool-logs-offset-b__1' => '2'], $logs); + } + + public static function tearDownAfterClass(): void + { + if (!isset(self::$pool)) { + return; + } + + self::$pool->use(function (mixed $redis): void { + if ($redis instanceof \RedisCluster) { + $redis->close(); + } + }); + self::$pool = null; + } + + private function clearRedisClusterPoolLogs(): void + { + $pool = self::$pool; + $this->assertInstanceOf(Pool::class, $pool); + + $pool->use(function (\RedisCluster $redis): void { + foreach ($redis->_masters() as $master) { + $cursor = null; + do { + /** @phpstan-ignore-next-line */ + $keys = $redis->scan($cursor, $master, 'abuse__*', 100); + if ($keys === false) { + continue; + } + + foreach ($keys as $key) { + $redis->del($key); + } + } while ($cursor > 0); + } + }); + } + + private function setRedisClusterPoolLog(string $key, string $value): void + { + $pool = self::$pool; + $this->assertInstanceOf(Pool::class, $pool); + + $pool->use(function (\RedisCluster $redis) use ($key, $value): void { + $redis->set('abuse__redis-cluster-pool-logs-offset-' . $key . '__1', $value); + }); + } +} diff --git a/packages/abuse/tests/E2E/RedisPoolTest.php b/packages/abuse/tests/E2E/RedisPoolTest.php new file mode 100644 index 00000000000..7fd4e57ea88 --- /dev/null +++ b/packages/abuse/tests/E2E/RedisPoolTest.php @@ -0,0 +1,106 @@ +|null + */ + protected static ?Pool $pool = null; + + public static function setUpBeforeClass(): void + { + if (isset(self::$pool)) { + return; + } + + self::$pool = new Pool(new Stack(), 'abuse-redis', 2, function (): \Redis { + $redis = new \Redis(); + $redis->connect(Services::HOST, Services::REDIS_PORT); + + return $redis; + }, timeout: 0.0); + } + + public function getAdapter(string $key, int $limit, int $seconds): TimeLimit + { + $pool = self::$pool; + $this->assertInstanceOf(Pool::class, $pool); + + /** @var Pool<\Redis> $pool */ + return new AdapterRedisPool('redis-pool-' . $key, $limit, $seconds, $pool); + } + + public function testGetLogsSupportsNullableLimit(): void + { + $adapter = $this->getAdapter('logs-null-limit', 1, 60); + $abuse = new \Utopia\Abuse\Abuse($adapter); + + $this->assertSame(false, $abuse->check()); + $this->assertNotEmpty($adapter->getLogs(null, null)); + } + + public function testGetLogsAppliesOffset(): void + { + $this->clearRedisPoolLogs(); + $adapter = $this->getAdapter('logs-offset', 1, 60); + + $this->setRedisPoolLog('a', '1'); + $this->setRedisPoolLog('b', '2'); + $this->setRedisPoolLog('c', '3'); + + $logs = $adapter->getLogs(1, 1); + + $this->assertSame(['abuse__redis-pool-logs-offset-b__1' => '2'], $logs); + } + + public static function tearDownAfterClass(): void + { + if (!isset(self::$pool)) { + return; + } + + self::$pool->use(function (mixed $redis): void { + if ($redis instanceof \Redis) { + $redis->close(); + } + }); + self::$pool = null; + } + + private function clearRedisPoolLogs(): void + { + $pool = self::$pool; + $this->assertInstanceOf(Pool::class, $pool); + + $pool->use(function (\Redis $redis): void { + $cursor = null; + do { + $keys = $redis->scan($cursor, 'abuse__*', 100); + if ($keys === false) { + continue; + } + + foreach ($keys as $key) { + $redis->del($key); + } + } while ($cursor > 0); + }); + } + + private function setRedisPoolLog(string $key, string $value): void + { + $pool = self::$pool; + $this->assertInstanceOf(Pool::class, $pool); + + $pool->use(function (\Redis $redis) use ($key, $value): void { + $redis->set('abuse__redis-pool-logs-offset-' . $key . '__1', $value); + }); + } +} diff --git a/packages/abuse/tests/E2E/RedisTest.php b/packages/abuse/tests/E2E/RedisTest.php new file mode 100644 index 00000000000..fe8d0a7cc3a --- /dev/null +++ b/packages/abuse/tests/E2E/RedisTest.php @@ -0,0 +1,46 @@ +connect(Services::HOST, Services::REDIS_PORT); + return $redis; + } + + public function getAdapter(string $key, int $limit, int $seconds): TimeLimit + { + return new AdapterRedis($key, $limit, $seconds, self::$redis); + } + + /** + * Clean up Redis connection after all tests + */ + public static function tearDownAfterClass(): void + { + if (isset(self::$redis)) { + self::$redis->close(); + } + } +} diff --git a/packages/abuse/tests/E2E/Services.php b/packages/abuse/tests/E2E/Services.php new file mode 100644 index 00000000000..0b8375e8fa7 --- /dev/null +++ b/packages/abuse/tests/E2E/Services.php @@ -0,0 +1,18 @@ +getAdapter('sw-static-key', 2, 1, 2); + $abuse = new Abuse($adapter); + $this->assertSame(false, $abuse->check()); + $this->assertSame(false, $abuse->check()); + $this->assertSame(true, $abuse->check()); + } + + /** + * Test a dynamic key with a limit of 2 requests per window + */ + public function testDynamicKey(): void + { + $adapter = $this->getAdapter('sw-dynamic-key-{{ip}}', 2, 1, 2); + $adapter->setParam('{{ip}}', '0.0.0.10'); + $abuse = new Abuse($adapter); + $this->assertSame(false, $abuse->check()); + $this->assertSame(false, $abuse->check()); + $this->assertSame(true, $abuse->check()); + } + + /** + * Test a dynamic key with 2 params + */ + public function testDynamicKeyWith2Params(): void + { + $adapter = $this->getAdapter('sw-two-params-{{ip}}-{{email}}', 2, 1, 2); + $adapter->setParam('{{ip}}', '0.0.0.10'); + $adapter->setParam('{{email}}', 'test@test.com'); + $abuse = new Abuse($adapter); + $this->assertSame(false, $abuse->check()); + $this->assertSame(false, $abuse->check()); + $this->assertSame(true, $abuse->check()); + } + + /** + * Test a higher request rate like 10 requests per window + */ + public function testFastRequests(): void + { + $adapter = $this->getAdapter('sw-fast-requests-{{ip}}', 10, 1, 2); + $adapter->setParam('{{ip}}', '0.0.0.11'); + $abuse = new Abuse($adapter); + for ($i = 0; $i < 10; $i++) { + $this->assertSame(false, $abuse->check()); + } + $this->assertSame(true, $abuse->check()); + } + + /** + * Test that remaining reports the correct number of allowed requests + */ + public function testRemaining(): void + { + $adapter = $this->getAdapter('sw-remaining-{{ip}}', 3, 60, 120); + $adapter->setParam('{{ip}}', '0.0.0.12'); + $abuse = new Abuse($adapter); + + $this->assertSame(2, $adapter->remaining()); // nothing counted yet: limit - (0 + 1) + $this->assertSame(false, $abuse->check()); // 1 used + $this->assertSame(1, $adapter->remaining()); + $this->assertSame(false, $abuse->check()); // 2 used + $this->assertSame(0, $adapter->remaining()); + } + + /** + * Test that the window resets once both buckets expire + */ + public function testWindowExpiry(): void + { + $adapter = $this->getAdapter('sw-window-expiry-{{ip}}', 3, 1, 2); + $adapter->setParam('{{ip}}', '127.0.0.1'); + $abuse = new Abuse($adapter); + for ($i = 0; $i < 3; $i++) { + $this->assertSame(false, $abuse->check()); + } + $this->assertSame(true, $abuse->check()); + + // Wait for both the current and previous buckets (ttl = 2) to expire + sleep(3); + + // A fresh adapter recomputes the window; the old buckets are gone + $adapter = $this->getAdapter('sw-window-expiry-{{ip}}', 3, 1, 2); + $adapter->setParam('{{ip}}', '127.0.0.1'); + $abuse = new Abuse($adapter); + $this->assertSame(false, $abuse->check()); + } + + /** + * Verify that time() returns the aligned window start as an int + */ + public function testTimeFormat(): void + { + $windowSize = 1; + $now = \time(); + $adapter = $this->getAdapter('sw-time', 1, $windowSize, 2); + $this->assertSame((int)($now - ($now % $windowSize)), $adapter->time()); + $this->assertSame(true, \is_int($adapter->time())); + } + + /** + * Test the reset functionality clears both buckets + */ + public function testReset(): void + { + $adapter = $this->getAdapter('sw-reset-test-{{ip}}', 5, 600, 1200); + $adapter->setParam('{{ip}}', '192.168.1.1'); + $abuse = new Abuse($adapter); + + // 5 OK, 6th limited + for ($i = 0; $i < 5; $i++) { + $this->assertSame(false, $abuse->check()); + } + $this->assertSame(true, $abuse->check()); + + // Reset clears the counters + $abuse->reset(); + + // 5 more OK, then limited again + for ($i = 0; $i < 5; $i++) { + $this->assertSame(false, $abuse->check()); + } + $this->assertSame(true, $abuse->check()); + } + + /** + * Test that a ttl smaller than the window size is rejected + */ + public function testTtlGuard(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->getAdapter('sw-guard', 1, 10, 5); + } + + /** + * Test that limit 0 means unlimited + */ + public function testUnlimited(): void + { + $adapter = $this->getAdapter('sw-unlimited', 0, 1, 2); + $abuse = new Abuse($adapter); + for ($i = 0; $i < 20; $i++) { + $this->assertSame(false, $abuse->check()); + } + } +} diff --git a/packages/abuse/tests/E2E/SlidingWindow/RedisClusterTest.php b/packages/abuse/tests/E2E/SlidingWindow/RedisClusterTest.php new file mode 100644 index 00000000000..fe1554072bd --- /dev/null +++ b/packages/abuse/tests/E2E/SlidingWindow/RedisClusterTest.php @@ -0,0 +1,41 @@ +close(); + } + } +} diff --git a/packages/abuse/tests/E2E/SlidingWindow/RedisPoolTest.php b/packages/abuse/tests/E2E/SlidingWindow/RedisPoolTest.php new file mode 100644 index 00000000000..54cb9acb727 --- /dev/null +++ b/packages/abuse/tests/E2E/SlidingWindow/RedisPoolTest.php @@ -0,0 +1,54 @@ +|null + */ + protected static ?Pool $pool = null; + + public static function setUpBeforeClass(): void + { + if (isset(self::$pool)) { + return; + } + + self::$pool = new Pool(new Stack(), 'abuse-sw-redis', 2, function (): \Redis { + $redis = new \Redis(); + $redis->connect(Services::HOST, Services::REDIS_PORT); + + return $redis; + }, timeout: 0.0); + } + + public function getAdapter(string $key, int $limit, int $windowSize, int $ttl): SlidingWindow + { + $pool = self::$pool; + $this->assertInstanceOf(Pool::class, $pool); + + /** @var Pool<\Redis> $pool */ + return new AdapterRedisPool('sw-pool-' . $key, $limit, $windowSize, $ttl, $pool); + } + + public static function tearDownAfterClass(): void + { + if (!isset(self::$pool)) { + return; + } + + self::$pool->use(function (mixed $redis): void { + if ($redis instanceof \Redis) { + $redis->close(); + } + }); + self::$pool = null; + } +} diff --git a/packages/abuse/tests/E2E/SlidingWindow/RedisTest.php b/packages/abuse/tests/E2E/SlidingWindow/RedisTest.php new file mode 100644 index 00000000000..ba8008edb22 --- /dev/null +++ b/packages/abuse/tests/E2E/SlidingWindow/RedisTest.php @@ -0,0 +1,44 @@ +connect(Services::HOST, Services::REDIS_PORT); + + return $redis; + } + + public function getAdapter(string $key, int $limit, int $windowSize, int $ttl): SlidingWindow + { + return new AdapterRedis($key, $limit, $windowSize, $ttl, self::$redis); + } + + public static function tearDownAfterClass(): void + { + if (isset(self::$redis)) { + self::$redis->close(); + } + } +} diff --git a/packages/abuse/tests/E2E/TokenBucket/Base.php b/packages/abuse/tests/E2E/TokenBucket/Base.php new file mode 100644 index 00000000000..2f840349899 --- /dev/null +++ b/packages/abuse/tests/E2E/TokenBucket/Base.php @@ -0,0 +1,160 @@ +getAdapter('tb-static-key', 2, 0.001); + $abuse = new Abuse($adapter); + $this->assertSame(false, $abuse->check()); + $this->assertSame(false, $abuse->check()); + $this->assertSame(true, $abuse->check()); + } + + /** + * Test a dynamic key with a capacity of 2 tokens + */ + public function testDynamicKey(): void + { + $adapter = $this->getAdapter('tb-dynamic-key-{{ip}}', 2, 0.001); + $adapter->setParam('{{ip}}', '0.0.0.10'); + $abuse = new Abuse($adapter); + $this->assertSame(false, $abuse->check()); + $this->assertSame(false, $abuse->check()); + $this->assertSame(true, $abuse->check()); + } + + /** + * Test a dynamic key with 2 params + */ + public function testDynamicKeyWith2Params(): void + { + $adapter = $this->getAdapter('tb-two-params-{{ip}}-{{email}}', 2, 0.001); + $adapter->setParam('{{ip}}', '0.0.0.10'); + $adapter->setParam('{{email}}', 'test@test.com'); + $abuse = new Abuse($adapter); + $this->assertSame(false, $abuse->check()); + $this->assertSame(false, $abuse->check()); + $this->assertSame(true, $abuse->check()); + } + + /** + * Test that a full bucket allows a burst up to its capacity + */ + public function testBurst(): void + { + $adapter = $this->getAdapter('tb-burst-{{ip}}', 10, 0.001); + $adapter->setParam('{{ip}}', '0.0.0.11'); + $abuse = new Abuse($adapter); + for ($i = 0; $i < 10; $i++) { + $this->assertSame(false, $abuse->check()); + } + $this->assertSame(true, $abuse->check()); + } + + /** + * Test that remaining reports the tokens still available + */ + public function testRemaining(): void + { + $adapter = $this->getAdapter('tb-remaining-{{ip}}', 3, 0.001); + $adapter->setParam('{{ip}}', '0.0.0.12'); + $abuse = new Abuse($adapter); + + $this->assertSame(2, $adapter->remaining()); // full bucket: limit - (0 + 1) + $this->assertSame(false, $abuse->check()); // 1 consumed + $this->assertSame(1, $adapter->remaining()); + $this->assertSame(false, $abuse->check()); // 2 consumed + $this->assertSame(0, $adapter->remaining()); + } + + /** + * Test that tokens refill over time + */ + public function testRefill(): void + { + // 1 token/sec, capacity 1: consume it, then a refill lets one more through + $adapter = $this->getAdapter('tb-refill-{{ip}}', 1, 1.0); + $adapter->setParam('{{ip}}', '0.0.0.13'); + $abuse = new Abuse($adapter); + + $this->assertSame(false, $abuse->check()); // consume the only token + $this->assertSame(true, $abuse->check()); // empty, throttled + + sleep(2); // refill ~2 tokens (capped at capacity 1) + + $this->assertSame(false, $abuse->check()); // refilled, allowed again + } + + /** + * Verify that time() returns the current time as an int + */ + public function testTimeFormat(): void + { + $adapter = $this->getAdapter('tb-time', 1, 1.0); + $this->assertSame(true, \is_int($adapter->time())); + } + + /** + * Test the reset functionality refills the bucket + */ + public function testReset(): void + { + $adapter = $this->getAdapter('tb-reset-test-{{ip}}', 5, 0.001); + $adapter->setParam('{{ip}}', '192.168.1.1'); + $abuse = new Abuse($adapter); + + // 5 OK, 6th limited + for ($i = 0; $i < 5; $i++) { + $this->assertSame(false, $abuse->check()); + } + $this->assertSame(true, $abuse->check()); + + // Reset refills the bucket + $abuse->reset(); + + // 5 more OK, then limited again + for ($i = 0; $i < 5; $i++) { + $this->assertSame(false, $abuse->check()); + } + $this->assertSame(true, $abuse->check()); + } + + /** + * Test that a non-positive refill rate is rejected + */ + public function testRefillRateGuard(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->getAdapter('tb-guard', 1, 0.0); + } + + /** + * Test that limit 0 means unlimited + */ + public function testUnlimited(): void + { + $adapter = $this->getAdapter('tb-unlimited', 0, 1.0); + $abuse = new Abuse($adapter); + for ($i = 0; $i < 20; $i++) { + $this->assertSame(false, $abuse->check()); + } + } +} diff --git a/packages/abuse/tests/E2E/TokenBucket/RedisClusterTest.php b/packages/abuse/tests/E2E/TokenBucket/RedisClusterTest.php new file mode 100644 index 00000000000..6fd8779f238 --- /dev/null +++ b/packages/abuse/tests/E2E/TokenBucket/RedisClusterTest.php @@ -0,0 +1,41 @@ +close(); + } + } +} diff --git a/packages/abuse/tests/E2E/TokenBucket/RedisPoolTest.php b/packages/abuse/tests/E2E/TokenBucket/RedisPoolTest.php new file mode 100644 index 00000000000..84fd48d53e0 --- /dev/null +++ b/packages/abuse/tests/E2E/TokenBucket/RedisPoolTest.php @@ -0,0 +1,54 @@ +|null + */ + protected static ?Pool $pool = null; + + public static function setUpBeforeClass(): void + { + if (isset(self::$pool)) { + return; + } + + self::$pool = new Pool(new Stack(), 'abuse-tb-redis', 2, function (): \Redis { + $redis = new \Redis(); + $redis->connect(Services::HOST, Services::REDIS_PORT); + + return $redis; + }, timeout: 0.0); + } + + public function getAdapter(string $key, int $tokens, float $refillRate): TokenBucket + { + $pool = self::$pool; + $this->assertInstanceOf(Pool::class, $pool); + + /** @var Pool<\Redis> $pool */ + return new AdapterRedisPool('tb-pool-' . $key, $tokens, $refillRate, $pool); + } + + public static function tearDownAfterClass(): void + { + if (!isset(self::$pool)) { + return; + } + + self::$pool->use(function (mixed $redis): void { + if ($redis instanceof \Redis) { + $redis->close(); + } + }); + self::$pool = null; + } +} diff --git a/packages/abuse/tests/E2E/TokenBucket/RedisTest.php b/packages/abuse/tests/E2E/TokenBucket/RedisTest.php new file mode 100644 index 00000000000..51fc968b100 --- /dev/null +++ b/packages/abuse/tests/E2E/TokenBucket/RedisTest.php @@ -0,0 +1,44 @@ +connect(Services::HOST, Services::REDIS_PORT); + + return $redis; + } + + public function getAdapter(string $key, int $tokens, float $refillRate): TokenBucket + { + return new AdapterRedis($key, $tokens, $refillRate, self::$redis); + } + + public static function tearDownAfterClass(): void + { + if (isset(self::$redis)) { + self::$redis->close(); + } + } +} diff --git a/packages/abuse/tests/NoneTest.php b/packages/abuse/tests/NoneTest.php new file mode 100644 index 00000000000..fa833fa99dd --- /dev/null +++ b/packages/abuse/tests/NoneTest.php @@ -0,0 +1,38 @@ +assertSame(false, $abuse->check()); + $this->assertSame(false, $abuse->check()); + $this->assertSame(false, $abuse->check()); + } + + public function testReturnsNoLogsAndCleanupSucceeds(): void + { + $adapter = new None('none-key', 1, 60); + + $this->assertSame([], $adapter->getLogs()); + $this->assertSame(true, $adapter->cleanup(time())); + } + + public function testResetIsNoop(): void + { + $adapter = new None('none-key', 1, 60); + $abuse = new Abuse($adapter); + + $abuse->reset(); + + $this->assertSame(false, $abuse->check()); + } +} diff --git a/packages/abuse/tests/bench/Base.php b/packages/abuse/tests/bench/Base.php new file mode 100644 index 00000000000..9299d189958 --- /dev/null +++ b/packages/abuse/tests/bench/Base.php @@ -0,0 +1,32 @@ +adapter->setParam('{{ip}}', $ip); + $this->abuse->check(); + } +} diff --git a/packages/abuse/tests/bench/Database.php b/packages/abuse/tests/bench/Database.php new file mode 100644 index 00000000000..1dab385717f --- /dev/null +++ b/packages/abuse/tests/bench/Database.php @@ -0,0 +1,50 @@ +setDatabase('utopiaTests'); + $db->setNamespace('namespace'); + $this->db = $db; + + $adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 60 * 5, $db); + if (!$db->exists('utopiaTests')) { + $db->create(); + $adapter->setup(); + } + $this->adapter = $adapter; + $this->abuse = new Abuse($this->adapter); + } +} diff --git a/packages/abuse/tests/bench/Redis.php b/packages/abuse/tests/bench/Redis.php new file mode 100644 index 00000000000..d2dae253943 --- /dev/null +++ b/packages/abuse/tests/bench/Redis.php @@ -0,0 +1,24 @@ +redis = new Client(); + $this->redis->connect(Services::HOST, Services::REDIS_PORT); + $this->adapter = new TimeLimit('login-attempt-from-{{ip}}', 3, 60 * 5, $this->redis); + $this->abuse = new Abuse($this->adapter); + } +} diff --git a/packages/abuse/tests/bench/RedisCluster.php b/packages/abuse/tests/bench/RedisCluster.php new file mode 100644 index 00000000000..badb048ae4d --- /dev/null +++ b/packages/abuse/tests/bench/RedisCluster.php @@ -0,0 +1,23 @@ +redis = new Client(null, Services::CLUSTER_SEEDS); + $this->adapter = new RedisClusterAdapter('login-attempt-from-{{ip}}', 3, 60 * 5, $this->redis); + $this->abuse = new Abuse($this->adapter); + } +} diff --git a/packages/detector/src/Detection/Framework/JS.php b/packages/detector/src/Detection/Framework/JS.php index c7452c21d5f..f07b21efe42 100644 --- a/packages/detector/src/Detection/Framework/JS.php +++ b/packages/detector/src/Detection/Framework/JS.php @@ -19,6 +19,11 @@ public function getPackages(): array */ public function getFiles(): array { - return ['package.json']; + // Do not score a bare package.json. Every JS framework inherits this + // base, so treating the lockfile manifest as a hit forces a multi-way + // tie (all score 1) that collapses to Angular via fewest-parents — + // even for lodash-only or tooling-only repos. Frameworks must match + // their own config files and/or dependency packages instead. + return []; } } diff --git a/packages/detector/src/Detection/Framework/TanStackStart.php b/packages/detector/src/Detection/Framework/TanStackStart.php index 74c1af06873..72bea17ee54 100644 --- a/packages/detector/src/Detection/Framework/TanStackStart.php +++ b/packages/detector/src/Detection/Framework/TanStackStart.php @@ -62,6 +62,11 @@ public function getAdapter(string $configContent): string { $stripped = \preg_replace('/(?assertSame($framework, $detection->getName(), $assertion); } + /** + * A package.json with no framework dependencies must not resolve to Angular + * (or any other framework) via a multi-way package.json path-match tie. + * + * @see https://github.com/appwrite/appwrite/issues/13911 + */ + public function testFrameworkDetectionReturnsNullWithoutFrameworkSignals(): void + { + $cases = [ + '{"dependencies":{"lodash":"^4"}}', + '{"devDependencies":{"turbo":"^2"}}', + '{}', + ]; + + foreach ($cases as $packageJson) { + $detector = new Framework('npm'); + $detector + ->addOption(new Analog()) + ->addOption(new Angular()) + ->addOption(new Astro()) + ->addOption(new Flutter()) + ->addOption(new Lynx()) + ->addOption(new NextJs()) + ->addOption(new Nuxt()) + ->addOption(new React()) + ->addOption(new ReactNative()) + ->addOption(new Remix()) + ->addOption(new Svelte()) + ->addOption(new SvelteKit()) + ->addOption(new TanStackStart()) + ->addOption(new Vue()); + + $detector->addInput('package.json', Framework::INPUT_FILE); + $detector->addInput($packageJson, Framework::INPUT_PACKAGES); + + $this->assertNull( + $detector->detect(), + "Expected null for package.json content: {$packageJson}" + ); + } + } + public function testTanStackStartAdapterDetection(): void { $fw = new TanStackStart(); @@ -800,6 +843,9 @@ public function testTanStackStartAdapterDetection(): void $this->assertSame('ssr', $fw->getAdapter('export default defineConfig({ plugins: [tanstackStart({ "prerender": false })] })')); $this->assertSame('ssr', $fw->getAdapter('// prerender: true' . "\n" . 'export default defineConfig({})')); $this->assertSame('static', $fw->getAdapter('server: { url: "https://example.com" },' . "\n" . 'prerender: { routes: [\'/\'] }')); + $this->assertSame('ssr', $fw->getAdapter('import { nitro } from \'nitro/vite\'' . "\n" . 'export default defineConfig({ plugins: [tanstackStart(), nitro({ prerender: { routes: [\'/\'], crawlLinks: true } })] })')); + $this->assertSame('ssr', $fw->getAdapter('import { nitroV2Plugin } from \'@tanstack/nitro-v2-vite-plugin\'' . "\n" . 'export default defineConfig({ plugins: [tanstackStart({ prerender: { enabled: true } }), nitroV2Plugin()] })')); + $this->assertSame('static', $fw->getAdapter('// import { nitro } from \'nitro/vite\'' . "\n" . 'export default defineConfig({ plugins: [tanstackStart({ prerender: { enabled: true } })] })')); $this->assertNotEmpty($fw->getConfigFiles()); } diff --git a/packages/nats/.github/workflows/mirror.yml b/packages/nats/.github/workflows/mirror.yml new file mode 100644 index 00000000000..167c0f5f1bb --- /dev/null +++ b/packages/nats/.github/workflows/mirror.yml @@ -0,0 +1,17 @@ +name: Mirror + +on: + pull_request_target: + types: [opened] + issues: + types: [opened] + +permissions: + issues: write + pull-requests: write + +jobs: + redirect: + uses: appwrite/appwrite/.github/workflows/mirror-redirect.yml@main + with: + package: nats diff --git a/packages/nats/.gitignore b/packages/nats/.gitignore new file mode 100644 index 00000000000..734074d299a --- /dev/null +++ b/packages/nats/.gitignore @@ -0,0 +1,14 @@ +/vendor/ +composer.phar +.phpunit.result.cache +.phpunit.cache/ +*.cache +.idea/ +.vscode/ +*.swp +*.swo +*~ +.DS_Store +Thumbs.db +.env +composer.lock diff --git a/packages/nats/README.md b/packages/nats/README.md new file mode 100644 index 00000000000..5636afa4d15 --- /dev/null +++ b/packages/nats/README.md @@ -0,0 +1,397 @@ +# nats.php + +> [!IMPORTANT] +> This repository is a read-only mirror of [`packages/nats`](https://github.com/appwrite/appwrite/tree/main/packages/nats) in [appwrite/appwrite](https://github.com/appwrite/appwrite). Development happens there — please open issues and pull requests against appwrite/appwrite. + +A modern PHP client for [NATS](https://nats.io) messaging system with JetStream and Key-Value store support. + +## Requirements + +- PHP 8.3+ +- `ext-json` +- `ext-sodium` (optional, for NKey/JWT authentication) + +## Installation + +```bash +composer require utopia-php/nats +``` + +## Quick start + +```php +use Utopia\NATS\Connection; + +$conn = Connection::connect('nats://127.0.0.1:4222'); + +// Publish +$conn->publish('greet.world', 'Hello, World!'); + +// Subscribe +$conn->subscribe('greet.*', function ($msg) { + echo "Received: {$msg->data}\n"; +}); + +// Process messages +$conn->wait(); +``` + +## Core NATS + +### Connecting + +```php +use Utopia\NATS\Connection; +use Utopia\NATS\ConnectionOptions; + +// Simple +$conn = Connection::connect('nats://127.0.0.1:4222'); + +// With options +$conn = Connection::connect(new ConnectionOptions( + servers: ['nats://host1:4222', 'nats://host2:4222'], + name: 'my-service', + user: 'alice', + pass: 'secret', + connectTimeout: 5.0, + allowReconnect: true, + maxReconnectAttempts: 60, +)); + +// From URL with credentials +$conn = Connection::connect('nats://user:pass@127.0.0.1:4222'); +``` + +### Publishing + +```php +use Utopia\NATS\Headers; + +// Simple publish +$conn->publish('orders.new', '{"id": 1}'); + +// Publish with headers +$headers = new Headers(); +$headers->set('Content-Type', 'application/json'); +$headers->set('X-Trace-Id', 'abc-123'); +$conn->publish('orders.new', '{"id": 1}', headers: $headers); +``` + +### Subscribing + +```php +// Async with callback +$sub = $conn->subscribe('orders.*', function ($msg) { + echo "{$msg->subject}: {$msg->data}\n"; +}); + +// Sync +$sub = $conn->subscribe('orders.new'); +$msg = $sub->nextMessage(timeout: 5.0); + +// Wildcards +$conn->subscribe('events.>', fn($msg) => handle($msg)); // multi-level +$conn->subscribe('orders.*', fn($msg) => handle($msg)); // single-level + +// Queue groups (load-balanced) +$conn->queueSubscribe('tasks', 'workers', function ($msg) { + processTask($msg->data); +}); + +// Unsubscribe +$sub->unsubscribe(); + +// Auto-unsubscribe after N messages +$conn->unsubscribe($sub, maxMessages: 10); +``` + +### Request-reply + +```php +// Responder +$conn->subscribe('math.double', function ($msg) use ($conn) { + $value = (int) $msg->data; + $conn->publish($msg->replyTo, (string) ($value * 2)); +}); + +// Requester +$response = $conn->request('math.double', '21', timeout: 2.0); +echo $response->data; // "42" +``` + +### Connection management + +```php +$conn->flush(); // Ensure all messages are sent +$conn->drain(); // Gracefully close (flush + unsubscribe) +$conn->close(); // Immediate close + +$conn->isConnected(); // Check status +$conn->getServerInfo()->version; // Server info +``` + +## JetStream + +### Streams + +```php +use Utopia\NATS\JetStream\StreamConfig; +use Utopia\NATS\JetStream\StorageType; +use Utopia\NATS\JetStream\RetentionPolicy; + +$js = $conn->jetStream(); + +// Create a stream +$stream = $js->createOrUpdateStream(new StreamConfig( + name: 'ORDERS', + subjects: ['orders.>'], + storage: StorageType::File, + retention: RetentionPolicy::Limits, + maxAge: 86400.0, // 1 day in seconds + replicas: 1, +)); + +// Stream info +$info = $stream->info(refresh: true); +echo "Messages: {$info->state->messages}\n"; + +// List streams +$names = $js->getStreamNames(); + +// Delete +$stream->delete(); +``` + +### Publishing with acknowledgment + +```php +$ack = $js->publish('orders.new', '{"id": 1}'); +echo "Stream: {$ack->stream}, Seq: {$ack->sequence}\n"; + +// With deduplication +$ack = $js->publish('orders.new', '{"id": 1}', msgId: 'order-1'); + +// With expected sequence (optimistic concurrency) +$ack = $js->publish('orders.new', $data, expectedLastSeq: 42); +``` + +### Publishing a batch + +`publish()` waits for each acknowledgment before it sends the next message, so a +batch of `N` messages costs `N` round trips. `publishMany()` writes a window of +messages first and reads their acknowledgments afterwards, which costs one round +trip per window. Each message keeps its own acknowledgment, so deduplication and +per-message errors work the same way. + +```php +$acks = $js->publishMany([ + ['subject' => 'orders.new', 'data' => '{"id": 1}', 'msgId' => 'order-1'], + ['subject' => 'orders.new', 'data' => '{"id": 2}', 'msgId' => 'order-2'], +]); + +// The acknowledgments come back in the order the messages were given. +echo "Seq: {$acks[0]->sequence}\n"; +``` + +The `window` argument sets how many messages stay in flight before their +acknowledgments are collected. A message the server rejects throws, as it does on +`publish()`. + +### Consumers + +```php +use Utopia\NATS\JetStream\ConsumerConfig; +use Utopia\NATS\JetStream\AckPolicy; +use Utopia\NATS\JetStream\DeliverPolicy; + +// Create a pull consumer +$consumer = $js->createConsumer('ORDERS', new ConsumerConfig( + name: 'order-processor', + durableName: 'order-processor', + ackPolicy: AckPolicy::Explicit, + filterSubject: 'orders.>', + deliverPolicy: DeliverPolicy::All, +)); + +// Fetch a batch of messages +$batch = $consumer->fetch(batch: 10, timeout: 5.0); +foreach ($batch as $msg) { + echo "Processing: {$msg->getData()}\n"; + + $msg->ack(); // Acknowledge + // $msg->nak(); // Negative ack (redeliver) + // $msg->term(); // Terminate (no redeliver) + // $msg->inProgress(); // Extend ack deadline +} + +// Fetch single message +$msg = $consumer->next(timeout: 5.0); + +// Message metadata +$meta = $msg->metadata(); +echo "Stream seq: {$meta->streamSequence}\n"; +echo "Deliveries: {$meta->numDelivered}\n"; +``` + +## Key-value store + +```php +use Utopia\NATS\KeyValue\KeyValueConfig; +use Utopia\NATS\JetStream\StorageType; + +$js = $conn->jetStream(); + +// Create a KV bucket +$kv = $js->createKeyValue(new KeyValueConfig( + bucket: 'config', + history: 5, + ttl: 3600.0, // 1 hour + storage: StorageType::File, +)); + +// Put +$revision = $kv->put('app.name', 'My Service'); + +// Get +$entry = $kv->get('app.name'); +echo "{$entry->key} = {$entry->value} (rev: {$entry->revision})\n"; + +// Create (fails if key exists) +$revision = $kv->create('app.version', '1.0.0'); + +// Update with CAS (compare-and-swap) +$revision = $kv->update('app.version', '1.1.0', revision: $revision); + +// Delete / Purge +$kv->delete('app.name'); +$kv->purge('app.version'); + +// List keys +$keys = $kv->keys(); + +// Bucket status +$status = $kv->status(); +echo "Values: {$status->values}, Bytes: {$status->bytes}\n"; +``` + +## Authentication + +```php +use Utopia\NATS\ConnectionOptions; + +// User/Password +$conn = Connection::connect(new ConnectionOptions( + servers: 'nats://127.0.0.1:4222', + user: 'alice', + pass: 'secret', +)); + +// Token +$conn = Connection::connect(new ConnectionOptions( + servers: 'nats://127.0.0.1:4222', + token: 'my-token', +)); + +// NKey (requires ext-sodium) +$conn = Connection::connect(new ConnectionOptions( + servers: 'nats://127.0.0.1:4222', + nkey: 'UABC...', + nkeySeed: 'SUABC...', +)); + +// JWT Credentials file (requires ext-sodium) +$conn = Connection::connect(new ConnectionOptions( + servers: 'nats://127.0.0.1:4222', + credentialsFile: '/path/to/user.creds', +)); +``` + +## TLS + +```php +use Utopia\NATS\ConnectionOptions; + +// TLS with system CA +$conn = Connection::connect(new ConnectionOptions( + servers: 'tls://nats.example.com:4222', +)); + +// TLS with custom CA and client certificates (mTLS) +$conn = Connection::connect(new ConnectionOptions( + servers: 'nats://127.0.0.1:4222', + tls: true, + tlsCaFile: '/path/to/ca.pem', + tlsCertFile: '/path/to/client-cert.pem', + tlsKeyFile: '/path/to/client-key.pem', +)); +``` + +## Event callbacks + +```php +$conn = Connection::connect(new ConnectionOptions( + servers: 'nats://127.0.0.1:4222', + onDisconnect: function () { + echo "Disconnected!\n"; + }, + onReconnect: function () { + echo "Reconnected!\n"; + }, + onClose: function () { + echo "Connection closed.\n"; + }, + onError: function ($e) { + echo "Error: {$e->getMessage()}\n"; + }, +)); +``` + +## Testing + +```bash +# Unit tests +composer test + +# E2E tests (against the NATS servers in docker-compose.yml) +docker compose up -d --wait +composer test:e2e +docker compose down -v + +# With custom NATS URL +NATS_URL=nats://host:4222 composer test:e2e +``` + +## Batched requests + +`Connection::requestBatch()` sends independent requests together and delivers each outcome to `reply` as it arrives. The callback receives the original input index and either a `Message` or a `Throwable`. One missing reply does not delay successful replies or discard their outcomes. + +```php +$connection->requestBatch( + requests: [ + new Request(subject: 'service.first', data: 'one'), + new Request(subject: 'service.second', data: 'two', headers: $headers), + ], + reply: function (int $index, Message|Throwable $result): void { + // Handle this request's reply or failure. + }, + timeout: 5.0, +); +``` + +`Request` has read-only typed fields; both `request()` and `requestBatch()` use its subject validation and the same request lifecycle. The method returns `void`; collect results in the callback if you need an array. Completion order can differ from input order. The timeout is one response deadline for the whole group, starting after the write, rather than a separate wait per reply. Callback execution counts toward that deadline. Empty input performs no I/O. Invalid input throws before any request is published; request headers and payload limits follow `request()`. + +If the callback throws, collection stops immediately, pending state is cleaned up, and the exception propagates. Requests already sent are not cancelled or replayed. Later replies can still arrive. Use one owner for reading the connection; nested reads from the callback throw `LogicException`. + +This differs from `requestMany()`, which sends one request and gathers several responses. Ambiguous writes are not replayed on reconnect. Use `JetStream::ackBatch()` to confirm a selected list of `JetStreamMessage` instances. JetStream constructs each acknowledgement; its callback receives the original index and `null` on server confirmation or a `Throwable` on failure. The consumer's configured acknowledgement policy still applies. Use `AckPolicy::Explicit` to leave messages outside the list unacknowledged. With `AckPolicy::All`, acknowledging a later message also acknowledges earlier messages, including those outside the list. + +```php +$jetStream->ackBatch($messages, function (int $index, ?Throwable $error): void { + // Null means this message's acknowledgement was confirmed. +}); +``` + +Single `request()` calls retain one retry for an explicit stale-connection rejection. Batch requests do not retry. Neither path replays an ambiguous write. + +## License + +Apache-2.0 diff --git a/packages/nats/composer.json b/packages/nats/composer.json new file mode 100644 index 00000000000..df21463b659 --- /dev/null +++ b/packages/nats/composer.json @@ -0,0 +1,41 @@ +{ + "name": "utopia-php/nats", + "description": "Modern PHP client for NATS messaging system", + "type": "library", + "license": "Apache-2.0", + "keywords": [ + "nats", + "messaging", + "pubsub", + "jetstream", + "queue" + ], + "require": { + "php": ">=8.3", + "ext-json": "*" + }, + "suggest": { + "ext-sodium": "Required for NKey and JWT credential authentication", + "ext-swoole": "Required for the coroutine-native SwooleTransport" + }, + "autoload": { + "psr-4": { + "Utopia\\NATS\\": "src/" + } + }, + "autoload-dev": { + "psr-4": { + "Utopia\\NATS\\Tests\\": "tests/" + } + }, + "scripts": { + "test": "phpunit --testsuite unit", + "test:e2e": "phpunit --testsuite e2e" + }, + "config": { + "optimize-autoloader": true, + "sort-packages": true + }, + "minimum-stability": "stable", + "prefer-stable": true +} diff --git a/packages/nats/docker-compose.yml b/packages/nats/docker-compose.yml new file mode 100644 index 00000000000..8159350dbc6 --- /dev/null +++ b/packages/nats/docker-compose.yml @@ -0,0 +1,40 @@ +name: utopia-nats + +services: + nats: + image: nats:2.11-alpine + command: ["-js", "-m", "8222"] + ports: + - "14222:4222" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8222/healthz"] + interval: 2s + timeout: 2s + retries: 30 + + nats-tls: + image: nats:2.11-alpine + command: ["-c", "/nats.conf"] + volumes: + - ./tests/fixtures/certs:/certs:ro + - ./tests/fixtures/nats-tls.conf:/nats.conf:ro + ports: + - "14223:4222" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8222/healthz"] + interval: 2s + timeout: 2s + retries: 30 + + nats-ws: + image: nats:2.11-alpine + command: ["-c", "/nats.conf"] + volumes: + - ./tests/fixtures/nats-ws.conf:/nats.conf:ro + ports: + - "14224:8080" + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:8222/healthz"] + interval: 2s + timeout: 2s + retries: 30 diff --git a/packages/nats/examples/jetstream.php b/packages/nats/examples/jetstream.php new file mode 100644 index 00000000000..fb8860d667f --- /dev/null +++ b/packages/nats/examples/jetstream.php @@ -0,0 +1,54 @@ +jetStream(); + +echo "Connected to NATS with JetStream\n"; + +// Create a stream +$stream = $js->createOrUpdateStream(new StreamConfig( + name: 'ORDERS', + subjects: ['orders.>'], +)); +echo "Stream ORDERS created\n"; + +// Publish messages +for ($i = 1; $i <= 5; $i++) { + $ack = $js->publish('orders.new', json_encode(['id' => $i, 'item' => "Item {$i}"])); + echo "Published order {$i}, stream seq: {$ack->sequence}\n"; +} + +// Create a pull consumer +$consumer = $js->createConsumer('ORDERS', new ConsumerConfig( + name: 'order-processor', + durableName: 'order-processor', + ackPolicy: AckPolicy::Explicit, + filterSubject: 'orders.>', +)); +echo "Consumer created: {$consumer->getName()}\n"; + +// Fetch messages +$batch = $consumer->fetch(5, 5.0); +echo "Fetched {$batch->count()} messages\n"; + +foreach ($batch as $msg) { + $data = json_decode($msg->getData(), true); + echo "Processing order: {$data['id']} - {$data['item']}\n"; + $msg->ack(); +} + +// Clean up +$stream->delete(); +echo "Stream deleted\n"; + +$conn->close(); +echo "Done\n"; diff --git a/packages/nats/examples/publish.php b/packages/nats/examples/publish.php new file mode 100644 index 00000000000..356ce08dd83 --- /dev/null +++ b/packages/nats/examples/publish.php @@ -0,0 +1,26 @@ +publish('greet.world', 'Hello, World!'); +echo "Published to greet.world\n"; + +// Publish with headers +$headers = new \Utopia\NATS\Headers(); +$headers->set('Content-Type', 'application/json'); +$conn->publish('events.user', '{"action":"login","user":"alice"}', headers: $headers); +echo "Published to events.user with headers\n"; + +$conn->flush(); +$conn->close(); + +echo "Done\n"; diff --git a/packages/nats/examples/request_reply.php b/packages/nats/examples/request_reply.php new file mode 100644 index 00000000000..c84345e52af --- /dev/null +++ b/packages/nats/examples/request_reply.php @@ -0,0 +1,27 @@ +subscribe('echo', function (Message $msg) use ($conn) { + echo "Service received: {$msg->data}\n"; + if ($msg->replyTo !== null) { + $conn->publish($msg->replyTo, 'Echo: ' . $msg->data); + } +}); + +// Send a request +$response = $conn->request('echo', 'Hello, NATS!', 2.0); +echo "Got response: {$response->data}\n"; + +$conn->close(); +echo "Done\n"; diff --git a/packages/nats/examples/subscribe.php b/packages/nats/examples/subscribe.php new file mode 100644 index 00000000000..b9994b3cbd7 --- /dev/null +++ b/packages/nats/examples/subscribe.php @@ -0,0 +1,28 @@ +subscribe('greet.*', function (Message $msg) { + echo "Received on {$msg->subject}: {$msg->data}\n"; + + if ($msg->headers !== null) { + foreach ($msg->headers->all() as $name => $values) { + echo " Header {$name}: " . implode(', ', $values) . "\n"; + } + } +}); + +echo "Subscribed to greet.*, waiting for messages... (Ctrl+C to quit)\n"; + +// Process messages forever +$conn->wait(); diff --git a/packages/nats/phpstan-baseline.neon b/packages/nats/phpstan-baseline.neon new file mode 100644 index 00000000000..7f57a8f903d --- /dev/null +++ b/packages/nats/phpstan-baseline.neon @@ -0,0 +1,2047 @@ +parameters: + ignoreErrors: + - + message: '#^Parameter \#1 \$codepoint of function chr expects int\<0, 255\>, int given\.$#' + identifier: argument.type + count: 1 + path: src/Auth/NKeyAuth.php + + - + message: '#^Parameter \#1 \$seed of function sodium_crypto_sign_seed_keypair expects non\-empty\-string, string given\.$#' + identifier: argument.type + count: 2 + path: src/Auth/NKeyAuth.php + + - + message: '#^Cannot cast mixed to string\.$#' + identifier: cast.string + count: 2 + path: src/Connection.php + + - + message: '#^Method Utopia\\NATS\\Connection\:\:buildConnectPayload\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/Connection.php + + - + message: '#^Method Utopia\\NATS\\Connection\:\:handleMessage\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/Connection.php + + - + message: '#^Parameter \#1 \$data of method Utopia\\NATS\\Connection\:\:handleMessage\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/Connection.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\ServerInfo\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/Connection.php + + - + message: '#^Parameter \#1 \$raw of static method Utopia\\NATS\\Headers\:\:fromWire\(\) expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/Connection.php + + - + message: '#^Parameter \#1 \$url of method Utopia\\NATS\\Connection\:\:normalizeUrl\(\) expects string, mixed given\.$#' + identifier: argument.type + count: 2 + path: src/Connection.php + + - + message: '#^Parameter \$data of class Utopia\\NATS\\Message constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/Connection.php + + - + message: '#^Parameter \$replyTo of class Utopia\\NATS\\Message constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/Connection.php + + - + message: '#^Parameter \$sid of class Utopia\\NATS\\Message constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/Connection.php + + - + message: '#^Parameter \$subject of class Utopia\\NATS\\Message constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/Connection.php + + - + message: '#^Possibly invalid array key type mixed\.$#' + identifier: offsetAccess.invalidOffset + count: 2 + path: src/Connection.php + + - + message: '#^Property Utopia\\NATS\\Connection\:\:\$subscriptions \(array\\) does not accept array\\.$#' + identifier: assign.propertyType + count: 1 + path: src/Connection.php + + - + message: '#^Property Utopia\\NATS\\Connection\:\:\$transport \(Utopia\\NATS\\Transport\\Transport\) does not accept mixed\.$#' + identifier: assign.propertyType + count: 1 + path: src/Connection.php + + - + message: '#^Class Utopia\\NATS\\Headers implements generic interface IteratorAggregate but does not specify its types\: TKey, TValue$#' + identifier: missingType.generics + count: 1 + path: src/Headers.php + + - + message: '#^Method Utopia\\NATS\\Headers\:\:getIterator\(\) return type with generic class ArrayIterator does not specify its types\: TKey, TValue$#' + identifier: missingType.generics + count: 1 + path: src/Headers.php + + - + message: '#^Cannot access offset ''errors'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Cannot access offset ''total'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\AccountInfo\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Parameter \$apiErrors of class Utopia\\NATS\\JetStream\\AccountInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Parameter \$apiTotal of class Utopia\\NATS\\JetStream\\AccountInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Parameter \$consumers of class Utopia\\NATS\\JetStream\\AccountInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Parameter \$domain of class Utopia\\NATS\\JetStream\\AccountInfo constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Parameter \$limits of class Utopia\\NATS\\JetStream\\AccountInfo constructor expects array\, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Parameter \$memory of class Utopia\\NATS\\JetStream\\AccountInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Parameter \$raw of class Utopia\\NATS\\JetStream\\AccountInfo constructor expects array\, array given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Parameter \$storage of class Utopia\\NATS\\JetStream\\AccountInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Parameter \$streams of class Utopia\\NATS\\JetStream\\AccountInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/AccountInfo.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\ApiError\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/ApiError.php + + - + message: '#^Parameter \$code of class Utopia\\NATS\\JetStream\\ApiError constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ApiError.php + + - + message: '#^Parameter \$description of class Utopia\\NATS\\JetStream\\ApiError constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ApiError.php + + - + message: '#^Parameter \$errCode of class Utopia\\NATS\\JetStream\\ApiError constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ApiError.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\ConsumerInfo\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/Consumer.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\JetStream\:\:checkError\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/Consumer.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\ConsumerConfig\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\ConsumerConfig\:\:toArray\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \#1 \$callback of function array_map expects \(callable\(mixed\)\: mixed\)\|null, Closure\(int\)\: float given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \#1 \$nanos of static method Utopia\\NATS\\JetStream\\StreamConfig\:\:nanosToSeconds\(\) expects int, mixed given\.$#' + identifier: argument.type + count: 3 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \#1 \$value of static method Utopia\\NATS\\JetStream\\AckPolicy\:\:tryFrom\(\) expects int\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \#1 \$value of static method Utopia\\NATS\\JetStream\\DeliverPolicy\:\:tryFrom\(\) expects int\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \#1 \$value of static method Utopia\\NATS\\JetStream\\ReplayPolicy\:\:tryFrom\(\) expects int\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \#2 \$array of function array_map expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$backoff of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects list\\|null, array\\|null given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$deliverGroup of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$deliverSubject of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$description of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$durableName of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$filterSubject of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$filterSubjects of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects list\\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$flowControl of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$maxAckPending of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$maxBatch of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$maxBytes of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$maxDeliver of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$maxWaiting of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$memStorage of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$metadata of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects array\\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$name of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$numReplicas of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$optStartSeq of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Parameter \$optStartTime of class Utopia\\NATS\\JetStream\\ConsumerConfig constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerConfig.php + + - + message: '#^Cannot access offset ''name'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Cannot cast mixed to string\.$#' + identifier: cast.string + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\ConsumerInfo\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\ConsumerConfig\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\SequenceInfo\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 2 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Parameter \$created of class Utopia\\NATS\\JetStream\\ConsumerInfo constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Parameter \$name of class Utopia\\NATS\\JetStream\\ConsumerInfo constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Parameter \$numAckPending of class Utopia\\NATS\\JetStream\\ConsumerInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Parameter \$numPending of class Utopia\\NATS\\JetStream\\ConsumerInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Parameter \$numRedelivered of class Utopia\\NATS\\JetStream\\ConsumerInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Parameter \$numWaiting of class Utopia\\NATS\\JetStream\\ConsumerInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Parameter \$pushBound of class Utopia\\NATS\\JetStream\\ConsumerInfo constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Parameter \$streamName of class Utopia\\NATS\\JetStream\\ConsumerInfo constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerInfo.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\ConsumerLimits\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/ConsumerLimits.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\ConsumerLimits\:\:toArray\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/ConsumerLimits.php + + - + message: '#^Parameter \#1 \$nanos of static method Utopia\\NATS\\JetStream\\StreamConfig\:\:nanosToSeconds\(\) expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerLimits.php + + - + message: '#^Parameter \$maxAckPending of class Utopia\\NATS\\JetStream\\ConsumerLimits constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ConsumerLimits.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\ExternalStream\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/ExternalStream.php + + - + message: '#^Parameter \$api of class Utopia\\NATS\\JetStream\\ExternalStream constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ExternalStream.php + + - + message: '#^Parameter \$deliver of class Utopia\\NATS\\JetStream\\ExternalStream constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/ExternalStream.php + + - + message: '#^Argument of an invalid type mixed supplied for foreach, only iterables are supported\.$#' + identifier: foreach.nonIterable + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\JetStream\:\:apiRequest\(\) has parameter \$payload with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\JetStream\:\:apiRequest\(\) should return array\ but returns mixed\.$#' + identifier: return.type + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\JetStream\:\:checkError\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\JetStream\:\:getConsumerNames\(\) should return list\ but returns mixed\.$#' + identifier: return.type + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\JetStream\:\:getStreamNames\(\) should return list\ but returns mixed\.$#' + identifier: return.type + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\JetStream\:\:listStreams\(\) should return list\ but returns array\\.$#' + identifier: return.type + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Parameter \#1 \$callback of function array_map expects \(callable\(mixed\)\: mixed\)\|null, Closure\(array\)\: Utopia\\NATS\\JetStream\\StreamInfo given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\ApiError\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\ConsumerInfo\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\JetStream\:\:checkError\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 3 + path: src/JetStream/JetStream.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\PubAck\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 2 + path: src/JetStream/JetStream.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\StreamMessage\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 2 + path: src/JetStream/JetStream.php + + - + message: '#^Parameter \#2 \$array of function array_map expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/JetStream.php + + - + message: '#^Class Utopia\\NATS\\JetStream\\MessageBatch implements generic interface IteratorAggregate but does not specify its types\: TKey, TValue$#' + identifier: missingType.generics + count: 1 + path: src/JetStream/MessageBatch.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\MessageBatch\:\:getIterator\(\) return type with generic class ArrayIterator does not specify its types\: TKey, TValue$#' + identifier: missingType.generics + count: 1 + path: src/JetStream/MessageBatch.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\Placement\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/Placement.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\Placement\:\:toArray\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/Placement.php + + - + message: '#^Parameter \$cluster of class Utopia\\NATS\\JetStream\\Placement constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/Placement.php + + - + message: '#^Parameter \$tags of class Utopia\\NATS\\JetStream\\Placement constructor expects list\\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/Placement.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\PubAck\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/PubAck.php + + - + message: '#^Parameter \$domain of class Utopia\\NATS\\JetStream\\PubAck constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/PubAck.php + + - + message: '#^Parameter \$duplicate of class Utopia\\NATS\\JetStream\\PubAck constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/PubAck.php + + - + message: '#^Parameter \$sequence of class Utopia\\NATS\\JetStream\\PubAck constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/PubAck.php + + - + message: '#^Parameter \$stream of class Utopia\\NATS\\JetStream\\PubAck constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/PubAck.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\Republish\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/Republish.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\Republish\:\:toArray\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/Republish.php + + - + message: '#^Parameter \$destination of class Utopia\\NATS\\JetStream\\Republish constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/Republish.php + + - + message: '#^Parameter \$headersOnly of class Utopia\\NATS\\JetStream\\Republish constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/Republish.php + + - + message: '#^Parameter \$source of class Utopia\\NATS\\JetStream\\Republish constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/Republish.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\SequenceInfo\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/SequenceInfo.php + + - + message: '#^Parameter \$consumerSeq of class Utopia\\NATS\\JetStream\\SequenceInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/SequenceInfo.php + + - + message: '#^Parameter \$lastActive of class Utopia\\NATS\\JetStream\\SequenceInfo constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/SequenceInfo.php + + - + message: '#^Parameter \$streamSeq of class Utopia\\NATS\\JetStream\\SequenceInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/SequenceInfo.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\StreamConfig\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\StreamConfig\:\:toArray\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#1 \$callback of function array_map expects \(callable\(mixed\)\: mixed\)\|null, Closure\(array\)\: Utopia\\NATS\\JetStream\\StreamSource given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\ConsumerLimits\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\Placement\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\Republish\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\StreamSource\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\SubjectTransform\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#1 \$nanos of static method Utopia\\NATS\\JetStream\\StreamConfig\:\:nanosToSeconds\(\) expects int, mixed given\.$#' + identifier: argument.type + count: 3 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#1 \$value of static method Utopia\\NATS\\JetStream\\DiscardPolicy\:\:tryFrom\(\) expects int\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#1 \$value of static method Utopia\\NATS\\JetStream\\RetentionPolicy\:\:tryFrom\(\) expects int\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#1 \$value of static method Utopia\\NATS\\JetStream\\StorageType\:\:tryFrom\(\) expects int\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \#2 \$array of function array_map expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$allowDirect of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$allowMsgTtl of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$allowRollup of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$compression of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$denyDelete of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$denyPurge of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$description of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$firstSeq of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$maxBytes of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$maxConsumers of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$maxMsgSize of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$maxMsgs of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$maxMsgsPerSubject of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$metadata of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects array\\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$mirrorDirect of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$name of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$noAck of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$replicas of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$sealed of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$sources of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects list\\|null, array\\|null given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Parameter \$subjects of class Utopia\\NATS\\JetStream\\StreamConfig constructor expects list\, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamConfig.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\StreamInfo\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/StreamInfo.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\StreamConfig\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamInfo.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\StreamState\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamInfo.php + + - + message: '#^Parameter \$created of class Utopia\\NATS\\JetStream\\StreamInfo constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamInfo.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\StreamMessage\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/StreamMessage.php + + - + message: '#^Parameter \#1 \$string of function base64_decode expects string, mixed given\.$#' + identifier: argument.type + count: 2 + path: src/JetStream/StreamMessage.php + + - + message: '#^Parameter \$sequence of class Utopia\\NATS\\JetStream\\StreamMessage constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamMessage.php + + - + message: '#^Parameter \$subject of class Utopia\\NATS\\JetStream\\StreamMessage constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamMessage.php + + - + message: '#^Parameter \$time of class Utopia\\NATS\\JetStream\\StreamMessage constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamMessage.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\StreamSource\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/StreamSource.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\StreamSource\:\:toArray\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/StreamSource.php + + - + message: '#^Parameter \#1 \$callback of function array_map expects \(callable\(mixed\)\: mixed\)\|null, Closure\(array\)\: Utopia\\NATS\\JetStream\\SubjectTransform given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamSource.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\ExternalStream\:\:fromArray\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamSource.php + + - + message: '#^Parameter \#2 \$array of function array_map expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamSource.php + + - + message: '#^Parameter \$filterSubject of class Utopia\\NATS\\JetStream\\StreamSource constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamSource.php + + - + message: '#^Parameter \$name of class Utopia\\NATS\\JetStream\\StreamSource constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamSource.php + + - + message: '#^Parameter \$optStartSeq of class Utopia\\NATS\\JetStream\\StreamSource constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamSource.php + + - + message: '#^Parameter \$optStartTime of class Utopia\\NATS\\JetStream\\StreamSource constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamSource.php + + - + message: '#^Parameter \$subjectTransforms of class Utopia\\NATS\\JetStream\\StreamSource constructor expects list\\|null, array\\|null given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamSource.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\StreamState\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/StreamState.php + + - + message: '#^Parameter \$bytes of class Utopia\\NATS\\JetStream\\StreamState constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamState.php + + - + message: '#^Parameter \$consumerCount of class Utopia\\NATS\\JetStream\\StreamState constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamState.php + + - + message: '#^Parameter \$firstSeq of class Utopia\\NATS\\JetStream\\StreamState constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamState.php + + - + message: '#^Parameter \$firstTs of class Utopia\\NATS\\JetStream\\StreamState constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamState.php + + - + message: '#^Parameter \$lastSeq of class Utopia\\NATS\\JetStream\\StreamState constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamState.php + + - + message: '#^Parameter \$lastTs of class Utopia\\NATS\\JetStream\\StreamState constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamState.php + + - + message: '#^Parameter \$messages of class Utopia\\NATS\\JetStream\\StreamState constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamState.php + + - + message: '#^Parameter \$numDeleted of class Utopia\\NATS\\JetStream\\StreamState constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamState.php + + - + message: '#^Parameter \$numSubjects of class Utopia\\NATS\\JetStream\\StreamState constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/StreamState.php + + - + message: '#^Method Utopia\\NATS\\JetStream\\SubjectTransform\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/JetStream/SubjectTransform.php + + - + message: '#^Parameter \$destination of class Utopia\\NATS\\JetStream\\SubjectTransform constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/SubjectTransform.php + + - + message: '#^Parameter \$source of class Utopia\\NATS\\JetStream\\SubjectTransform constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/JetStream/SubjectTransform.php + + - + message: '#^Argument of an invalid type mixed supplied for foreach, only iterables are supported\.$#' + identifier: foreach.nonIterable + count: 1 + path: src/KeyValue/KeyValue.php + + - + message: '#^Cannot access offset ''message'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/KeyValue/KeyValue.php + + - + message: '#^Cannot access offset ''num_pending'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/KeyValue/KeyValue.php + + - + message: '#^Cannot access offset ''state'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/KeyValue/KeyValue.php + + - + message: '#^Cannot access offset ''subjects'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/KeyValue/KeyValue.php + + - + message: '#^Cannot call method ack\(\) on mixed\.$#' + identifier: method.nonObject + count: 1 + path: src/KeyValue/KeyValue.php + + - + message: '#^Cannot cast mixed to int\.$#' + identifier: cast.int + count: 2 + path: src/KeyValue/KeyValue.php + + - + message: '#^Cannot cast mixed to string\.$#' + identifier: cast.string + count: 3 + path: src/KeyValue/KeyValue.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\JetStream\:\:checkError\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 4 + path: src/KeyValue/KeyValue.php + + - + message: '#^Parameter \#2 \$msg of method Utopia\\NATS\\KeyValue\\KeyValue\:\:entryFromDelivered\(\) expects Utopia\\NATS\\JetStream\\JetStreamMessage, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/KeyValue/KeyValue.php + + - + message: '#^Parameter \$created of class Utopia\\NATS\\KeyValue\\KeyValueEntry constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/KeyValue/KeyValue.php + + - + message: '#^Cannot cast mixed to string\.$#' + identifier: cast.string + count: 2 + path: src/ObjectStore/ObjectLink.php + + - + message: '#^Cannot access offset ''link'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/ObjectStore/ObjectMeta.php + + - + message: '#^Cannot cast mixed to int\.$#' + identifier: cast.int + count: 2 + path: src/ObjectStore/ObjectMeta.php + + - + message: '#^Cannot cast mixed to string\.$#' + identifier: cast.string + count: 7 + path: src/ObjectStore/ObjectMeta.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\ObjectStore\\ObjectLink\:\:fromArray\(\) expects array\, array\ given\.$#' + identifier: argument.type + count: 1 + path: src/ObjectStore/ObjectMeta.php + + - + message: '#^Parameter \$metadata of class Utopia\\NATS\\ObjectStore\\ObjectMeta constructor expects array\\|null, array\\|null given\.$#' + identifier: argument.type + count: 1 + path: src/ObjectStore/ObjectMeta.php + + - + message: '#^Binary operation "\.\=" between string and mixed results in an error\.$#' + identifier: assignOp.invalid + count: 1 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Cannot access offset ''config'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Cannot access offset ''data'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Cannot access offset ''error'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Cannot access offset ''message'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Cannot access offset ''seq'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Cannot call method ack\(\) on mixed\.$#' + identifier: method.nonObject + count: 2 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Cannot call method getData\(\) on mixed\.$#' + identifier: method.nonObject + count: 2 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Cannot cast mixed to int\.$#' + identifier: cast.int + count: 1 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Cannot cast mixed to string\.$#' + identifier: cast.string + count: 2 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\JetStream\\JetStream\:\:checkError\(\) expects array, mixed given\.$#' + identifier: argument.type + count: 3 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Parameter \#1 \$data of static method Utopia\\NATS\\ObjectStore\\ObjectMeta\:\:fromArray\(\) expects array\, array\ given\.$#' + identifier: argument.type + count: 3 + path: src/ObjectStore/ObjectStore.php + + - + message: '#^Method Utopia\\NATS\\Protocol\\Writer\:\:connect\(\) has parameter \$options with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/Protocol/Writer.php + + - + message: '#^Method Utopia\\NATS\\ServerInfo\:\:__construct\(\) has parameter \$connectUrls with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/ServerInfo.php + + - + message: '#^Method Utopia\\NATS\\ServerInfo\:\:fromArray\(\) has parameter \$data with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$authRequired of class Utopia\\NATS\\ServerInfo constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$clientId of class Utopia\\NATS\\ServerInfo constructor expects int\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$clientIp of class Utopia\\NATS\\ServerInfo constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$connectUrls of class Utopia\\NATS\\ServerInfo constructor expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$headersSupported of class Utopia\\NATS\\ServerInfo constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$host of class Utopia\\NATS\\ServerInfo constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$jetstream of class Utopia\\NATS\\ServerInfo constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$maxPayload of class Utopia\\NATS\\ServerInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$nonce of class Utopia\\NATS\\ServerInfo constructor expects string\|null, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$port of class Utopia\\NATS\\ServerInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$proto of class Utopia\\NATS\\ServerInfo constructor expects int, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$serverId of class Utopia\\NATS\\ServerInfo constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$serverName of class Utopia\\NATS\\ServerInfo constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$tlsAvailable of class Utopia\\NATS\\ServerInfo constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$tlsRequired of class Utopia\\NATS\\ServerInfo constructor expects bool, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Parameter \$version of class Utopia\\NATS\\ServerInfo constructor expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/ServerInfo.php + + - + message: '#^Cannot access property \$data on mixed\.$#' + identifier: property.nonObject + count: 1 + path: src/Subscription.php + + - + message: '#^Cannot cast mixed to string\.$#' + identifier: cast.string + count: 1 + path: src/Subscription.php + + - + message: '#^Method Utopia\\NATS\\Subscription\:\:nextMessage\(\) should return Utopia\\NATS\\Message\|null but returns mixed\.$#' + identifier: return.type + count: 1 + path: src/Subscription.php + + - + message: '#^Property Utopia\\NATS\\Subscription\:\:\$pendingMessages with generic class SplQueue does not specify its types\: TValue$#' + identifier: missingType.generics + count: 1 + path: src/Subscription.php + + - + message: '#^Binary operation "\+\=" between float\|int and mixed results in an error\.$#' + identifier: assignOp.invalid + count: 1 + path: src/Transport/SwooleTransport.php + + - + message: '#^Binary operation "\.\=" between string and mixed results in an error\.$#' + identifier: assignOp.invalid + count: 1 + path: src/Transport/SwooleTransport.php + + - + message: '#^Method Utopia\\NATS\\Transport\\SwooleTransport\:\:upgradeTls\(\) has parameter \$options with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/Transport/SwooleTransport.php + + - + message: '#^Parameter \#1 \$options of method Utopia\\NATS\\Transport\\SwooleTransport\:\:buildTlsSettings\(\) expects array\, array given\.$#' + identifier: argument.type + count: 1 + path: src/Transport/SwooleTransport.php + + - + message: '#^Part \$client\-\>errCode \(mixed\) of encapsed string cannot be cast to string\.$#' + identifier: encapsedStringPart.nonString + count: 3 + path: src/Transport/SwooleTransport.php + + - + message: '#^Part \$client\-\>errMsg \(mixed\) of encapsed string cannot be cast to string\.$#' + identifier: encapsedStringPart.nonString + count: 3 + path: src/Transport/SwooleTransport.php + + - + message: '#^Argument of an invalid type mixed supplied for foreach, only iterables are supported\.$#' + identifier: foreach.nonIterable + count: 1 + path: src/Transport/TcpTransport.php + + - + message: '#^Method Utopia\\NATS\\Transport\\TcpTransport\:\:upgradeTls\(\) has parameter \$options with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/Transport/TcpTransport.php + + - + message: '#^Parameter \#2 \$length of function fread expects int\<1, max\>, int given\.$#' + identifier: argument.type + count: 1 + path: src/Transport/TcpTransport.php + + - + message: '#^Parameter \#3 \$optionname of function stream_context_set_option expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: src/Transport/TcpTransport.php + + - + message: '#^Method Utopia\\NATS\\Transport\\TlsTransport\:\:__construct\(\) has parameter \$tlsOptions with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/Transport/TlsTransport.php + + - + message: '#^Method Utopia\\NATS\\Transport\\TlsTransport\:\:buildSslOptions\(\) return type has no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/Transport/TlsTransport.php + + - + message: '#^Method Utopia\\NATS\\Transport\\TlsTransport\:\:upgradeTls\(\) has parameter \$options with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/Transport/TlsTransport.php + + - + message: '#^Parameter \#2 \$length of function fread expects int\<1, max\>, int given\.$#' + identifier: argument.type + count: 1 + path: src/Transport/TlsTransport.php + + - + message: '#^Method Utopia\\NATS\\Transport\\Transport\:\:upgradeTls\(\) has parameter \$options with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/Transport/Transport.php + + - + message: '#^Method Utopia\\NATS\\Transport\\WebSocketTransport\:\:upgradeTls\(\) has parameter \$options with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: src/Transport/WebSocketTransport.php + + - + message: '#^Parameter \#1 \$codepoint of function chr expects int\<0, 255\>, int given\.$#' + identifier: argument.type + count: 1 + path: src/Transport/WebSocketTransport.php + + - + message: '#^Parameter \#2 \$length of function fread expects int\<1, max\>, int given\.$#' + identifier: argument.type + count: 1 + path: src/Transport/WebSocketTransport.php + + - + message: '#^Parameter \#1 \$input of method Utopia\\NATS\\Tests\\Auth\\NKeyAuthTest\:\:base32Decode\(\) expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/Auth/NKeyAuthTest.php + + - + message: '#^Parameter \#1 \$signature of function sodium_crypto_sign_verify_detached expects non\-empty\-string, string given\.$#' + identifier: argument.type + count: 1 + path: tests/Auth/NKeyAuthTest.php + + - + message: '#^Call to an undefined method object\:\:isConnected\(\)\.$#' + identifier: method.notFound + count: 1 + path: tests/ConnectionDeathDetectionTest.php + + - + message: '#^Call to an undefined method object\:\:read\(\)\.$#' + identifier: method.notFound + count: 1 + path: tests/ConnectionDeathDetectionTest.php + + - + message: '#^Call to an undefined method object\:\:write\(\)\.$#' + identifier: method.notFound + count: 2 + path: tests/ConnectionDeathDetectionTest.php + + - + message: '#^Generator expects key type int\<0, max\>, int given\.$#' + identifier: generator.keyType + count: 26 + path: tests/ConnectionDeathDetectionTest.php + + - + message: '#^Parameter \#1 \$servers of class Utopia\\NATS\\ConnectionOptions constructor expects list\\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/ConnectionDeathDetectionTest.php + + - + message: '#^Parameter \#1 \$stream of function fclose expects resource, resource\|false given\.$#' + identifier: argument.type + count: 1 + path: tests/ConnectionDeathDetectionTest.php + + - + message: '#^Method Utopia\\NATS\\Tests\\ConnectionProtocolTest\:\:connect\(\) has parameter \$extra with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: tests/ConnectionProtocolTest.php + + - + message: '#^Parameter \#1 \$servers of class Utopia\\NATS\\ConnectionOptions constructor expects list\\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/ConnectionProtocolTest.php + + - + message: '#^Method Utopia\\NATS\\Tests\\ConnectionTlsAndAuthTest\:\:connect\(\) has parameter \$extra with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: tests/ConnectionTlsAndAuthTest.php + + - + message: '#^Parameter \#1 \$servers of class Utopia\\NATS\\ConnectionOptions constructor expects list\\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/ConnectionTlsAndAuthTest.php + + - + message: '#^Offset 1 might not exist on array\{\}\|array\{non\-falsy\-string, numeric\-string\}\.$#' + identifier: offsetAccess.notFound + count: 1 + path: tests/ConsumerFetchLifecycleTest.php + + - + message: '#^Parameter \#1 \$servers of class Utopia\\NATS\\ConnectionOptions constructor expects list\\|string, mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/ConsumerFetchLifecycleTest.php + + - + message: '#^Cannot call method getData\(\) on mixed\.$#' + identifier: method.nonObject + count: 2 + path: tests/ConsumerPullRequestTest.php + + - + message: '#^Cannot cast mixed to string\.$#' + identifier: cast.string + count: 1 + path: tests/ConsumerPullRequestTest.php + + - + message: '#^Cannot call method ack\(\) on mixed\.$#' + identifier: method.nonObject + count: 1 + path: tests/E2E/ConsumerNoWaitTest.php + + - + message: '#^Cannot call method getData\(\) on mixed\.$#' + identifier: method.nonObject + count: 1 + path: tests/E2E/ConsumerNoWaitTest.php + + - + message: '#^Cannot use array destructuring on mixed\.$#' + identifier: offsetAccess.nonArray + count: 1 + path: tests/E2E/ObjectStoreExtraTest.php + + - + message: '#^Cannot use array destructuring on mixed\.$#' + identifier: offsetAccess.nonArray + count: 1 + path: tests/E2E/ObjectStoreTest.php + + - + message: '#^Cannot cast mixed to int\.$#' + identifier: cast.int + count: 1 + path: tests/E2E/OrderedConsumerTest.php + + - + message: '#^Cannot access property \$data on mixed\.$#' + identifier: property.nonObject + count: 1 + path: tests/E2E/PubSubTest.php + + - + message: '#^Cannot access property \$headers on mixed\.$#' + identifier: property.nonObject + count: 3 + path: tests/E2E/PubSubTest.php + + - + message: '#^Cannot access property \$subject on mixed\.$#' + identifier: property.nonObject + count: 1 + path: tests/E2E/PubSubTest.php + + - + message: '#^Cannot call method get\(\) on mixed\.$#' + identifier: method.nonObject + count: 2 + path: tests/E2E/PubSubTest.php + + - + message: '#^Cannot access property \$replyTo on mixed\.$#' + identifier: property.nonObject + count: 2 + path: tests/E2E/RequestManyTest.php + + - + message: '#^Parameter \#1 \$subject of method Utopia\\NATS\\Connection\:\:publish\(\) expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/E2E/RequestManyTest.php + + - + message: '#^Binary operation "\." between ''echo\: '' and mixed results in an error\.$#' + identifier: binaryOp.invalid + count: 1 + path: tests/E2E/RequestReplyTest.php + + - + message: '#^Cannot access property \$data on mixed\.$#' + identifier: property.nonObject + count: 1 + path: tests/E2E/RequestReplyTest.php + + - + message: '#^Cannot access property \$replyTo on mixed\.$#' + identifier: property.nonObject + count: 2 + path: tests/E2E/RequestReplyTest.php + + - + message: '#^Parameter \#1 \$subject of method Utopia\\NATS\\Connection\:\:publish\(\) expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/E2E/RequestReplyTest.php + + - + message: '#^Access to an undefined property Throwable\|Utopia\\NATS\\Message\:\:\$data\.$#' + identifier: property.notFound + count: 2 + path: tests/E2E/RequestsTest.php + + - + message: '#^Cannot call method get\(\) on Utopia\\NATS\\Headers\|null\.$#' + identifier: method.nonObject + count: 1 + path: tests/E2E/RequestsTest.php + + - + message: '#^Parameter \#1 \$subject of method Utopia\\NATS\\Connection\:\:publish\(\) expects string, string\|null given\.$#' + identifier: argument.type + count: 1 + path: tests/E2E/RequestsTest.php + + - + message: '#^Argument of an invalid type mixed supplied for foreach, only iterables are supported\.$#' + identifier: foreach.nonIterable + count: 3 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Cannot access offset ''endpoints'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 3 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Cannot access offset ''env'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Cannot access offset ''metadata'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 3 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Cannot access offset ''name'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 3 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Cannot access offset ''num_errors'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Cannot access offset ''num_requests'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Cannot access offset ''owner'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Cannot access offset ''queue_group'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 2 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Cannot access offset ''subject'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Cannot access offset ''visibility'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Possibly invalid array key type mixed\.$#' + identifier: offsetAccess.invalidOffset + count: 3 + path: tests/E2E/ServiceExtrasTest.php + + - + message: '#^Argument of an invalid type mixed supplied for foreach, only iterables are supported\.$#' + identifier: foreach.nonIterable + count: 1 + path: tests/E2E/ServiceTest.php + + - + message: '#^Cannot access offset ''description'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/E2E/ServiceTest.php + + - + message: '#^Cannot access offset ''endpoints'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 2 + path: tests/E2E/ServiceTest.php + + - + message: '#^Cannot access offset ''id'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/E2E/ServiceTest.php + + - + message: '#^Cannot access offset ''name'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 2 + path: tests/E2E/ServiceTest.php + + - + message: '#^Cannot access offset ''num_errors'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 2 + path: tests/E2E/ServiceTest.php + + - + message: '#^Cannot access offset ''num_requests'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 2 + path: tests/E2E/ServiceTest.php + + - + message: '#^Cannot access offset ''processing_time'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/E2E/ServiceTest.php + + - + message: '#^Cannot access offset ''type'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 3 + path: tests/E2E/ServiceTest.php + + - + message: '#^Cannot access offset ''version'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/E2E/ServiceTest.php + + - + message: '#^Parameter \#1 \$callback of function array_map expects \(callable\(mixed\)\: mixed\)\|null, Closure\(array\)\: mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/E2E/ServiceTest.php + + - + message: '#^Parameter \#2 \$array of function array_map expects array, mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/E2E/ServiceTest.php + + - + message: '#^Parameter \#2 \$array of method PHPUnit\\Framework\\Assert\:\:assertArrayHasKey\(\) expects array\\|ArrayAccess\<\(int\|string\), covariant mixed\>, mixed given\.$#' + identifier: argument.type + count: 2 + path: tests/E2E/ServiceTest.php + + - + message: '#^Possibly invalid array key type mixed\.$#' + identifier: offsetAccess.invalidOffset + count: 1 + path: tests/E2E/ServiceTest.php + + - + message: '#^Binary operation "\." between ''pong\:'' and mixed results in an error\.$#' + identifier: binaryOp.invalid + count: 1 + path: tests/E2E/WebSocketTest.php + + - + message: '#^Cannot access property \$data on mixed\.$#' + identifier: property.nonObject + count: 1 + path: tests/E2E/WebSocketTest.php + + - + message: '#^Cannot access property \$replyTo on mixed\.$#' + identifier: property.nonObject + count: 1 + path: tests/E2E/WebSocketTest.php + + - + message: '#^Parameter \#1 \$subject of method Utopia\\NATS\\Connection\:\:publish\(\) expects string, mixed given\.$#' + identifier: argument.type + count: 1 + path: tests/E2E/WebSocketTest.php + + - + message: '#^Cannot access offset ''payload'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/ParserFrameIntegrityTest.php + + - + message: '#^Cannot access offset ''subject'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/ParserFrameIntegrityTest.php + + - + message: '#^Cannot access offset ''headers'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/Protocol/ParserTest.php + + - + message: '#^Cannot access offset ''payload'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 6 + path: tests/Protocol/ParserTest.php + + - + message: '#^Cannot access offset ''replyTo'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 3 + path: tests/Protocol/ParserTest.php + + - + message: '#^Cannot access offset ''server_id'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/Protocol/ParserTest.php + + - + message: '#^Cannot access offset ''sid'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 2 + path: tests/Protocol/ParserTest.php + + - + message: '#^Cannot access offset ''subject'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 3 + path: tests/Protocol/ParserTest.php + + - + message: '#^Cannot access offset ''version'' on mixed\.$#' + identifier: offsetAccess.nonOffsetAccessible + count: 1 + path: tests/Protocol/ParserTest.php + + - + message: '#^Method Utopia\\NATS\\Transport\\Transport@anonymous/tests/Protocol/ParserTest\.php\:17\:\:upgradeTls\(\) has parameter \$options with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: tests/Protocol/ParserTest.php + + - + message: '#^Access to an undefined property Throwable\|Utopia\\NATS\\Message\:\:\$data\.$#' + identifier: property.notFound + count: 4 + path: tests/RequestsTest.php + + - + message: '#^Cannot call method isConnected\(\) on Utopia\\NATS\\Connection\|null\.$#' + identifier: method.nonObject + count: 1 + path: tests/RequestsTest.php + + - + message: '#^Method Utopia\\NATS\\Tests\\RequestsTest\:\:bufferedReplies\(\) return type has no value type specified in iterable type iterable\.$#' + identifier: missingType.iterableValue + count: 1 + path: tests/RequestsTest.php + + - + message: '#^Method Utopia\\NATS\\Tests\\RequestsTest\:\:invalid\(\) return type has no value type specified in iterable type iterable\.$#' + identifier: missingType.iterableValue + count: 1 + path: tests/RequestsTest.php + + - + message: '#^Method Utopia\\NATS\\Tests\\RequestsTest\:\:keepalive\(\) return type has no value type specified in iterable type iterable\.$#' + identifier: missingType.iterableValue + count: 1 + path: tests/RequestsTest.php + + - + message: '#^Method Utopia\\NATS\\Tests\\RequestsTest\:\:reconnect\(\) return type has no value type specified in iterable type iterable\.$#' + identifier: missingType.iterableValue + count: 1 + path: tests/RequestsTest.php + + - + message: '#^Method Utopia\\NATS\\Tests\\RequestsTest\:\:testInvalidInputDoesNotWriteOrInvokeCallbacks\(\) has parameter \$requests with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: tests/RequestsTest.php + + - + message: '#^Offset int\<0, max\>\|null might not exist on list\\.$#' + identifier: offsetAccess.notFound + count: 1 + path: tests/RequestsTest.php + + - + message: '#^Parameter \#1 \$requests of method Utopia\\NATS\\Connection\:\:requestBatch\(\) expects list\, array given\.$#' + identifier: argument.type + count: 1 + path: tests/RequestsTest.php + + - + message: '#^Possibly invalid array key type int\<0, max\>\|null\.$#' + identifier: offsetAccess.invalidOffset + count: 2 + path: tests/RequestsTest.php + + - + message: '#^Trying to invoke Closure\|null but it might not be a callable\.$#' + identifier: callable.nonCallable + count: 1 + path: tests/RequestsTest.php + + - + message: '#^Method Utopia\\NATS\\Tests\\Support\\FakeTransport\:\:connectPayload\(\) should return array\ but returns mixed\.$#' + identifier: return.type + count: 1 + path: tests/Support/FakeTransport.php + + - + message: '#^Method Utopia\\NATS\\Tests\\Support\\FakeTransport\:\:upgradeTls\(\) has parameter \$options with no value type specified in iterable type array\.$#' + identifier: missingType.iterableValue + count: 1 + path: tests/Support/FakeTransport.php + + - + message: '#^Property Utopia\\NATS\\Tests\\Support\\FakeTransport\:\:\$tlsUpgrades \(list\\>\) does not accept non\-empty\-list\\.$#' + identifier: assign.propertyType + count: 1 + path: tests/Support/FakeTransport.php diff --git a/packages/nats/phpstan.neon b/packages/nats/phpstan.neon new file mode 100644 index 00000000000..bfd97a685a5 --- /dev/null +++ b/packages/nats/phpstan.neon @@ -0,0 +1,9 @@ +includes: + - phpstan-baseline.neon + +parameters: + level: max + treatPhpDocTypesAsCertain: false + paths: + - src + - tests diff --git a/packages/nats/phpunit.xml b/packages/nats/phpunit.xml new file mode 100644 index 00000000000..bc0ac92eada --- /dev/null +++ b/packages/nats/phpunit.xml @@ -0,0 +1,26 @@ + + + + + tests + tests/E2E + + + tests/E2E + + + + + src + + + + + + diff --git a/packages/nats/rector.php b/packages/nats/rector.php new file mode 100644 index 00000000000..e912ff108d0 --- /dev/null +++ b/packages/nats/rector.php @@ -0,0 +1,29 @@ +withPaths([ + __DIR__ . '/src', + __DIR__ . '/tests', + ]) + ->withPhpSets() + ->withPreparedSets( + typeDeclarations: true, + ) + // Absorbing moves code: keep the public surface (property declarations, + // readonly-ness) exactly as released. + ->withSkip([ + ClassPropertyAssignToConstructorPromotionRector::class, + ReadOnlyPropertyRector::class, + ReadOnlyClassRector::class, + AddTypeToConstRector::class, + AddArrowFunctionReturnTypeRector::class, + ]); diff --git a/packages/nats/src/Auth/Authenticator.php b/packages/nats/src/Auth/Authenticator.php new file mode 100644 index 00000000000..b9ddacbd475 --- /dev/null +++ b/packages/nats/src/Auth/Authenticator.php @@ -0,0 +1,16 @@ + + */ + public function authenticate(?string $nonce = null): array; +} diff --git a/packages/nats/src/Auth/CredentialsAuth.php b/packages/nats/src/Auth/CredentialsAuth.php new file mode 100644 index 00000000000..2e1e0e9afac --- /dev/null +++ b/packages/nats/src/Auth/CredentialsAuth.php @@ -0,0 +1,67 @@ +extractBetween($contents, '-----BEGIN NATS USER JWT-----', '------END NATS USER JWT------'); + $seed = $this->extractBetween($contents, '-----BEGIN USER NKEY SEED-----', '------END USER NKEY SEED------'); + + if ($jwt === null) { + throw new AuthenticationException('No JWT found in credentials file'); + } + if ($seed === null) { + throw new AuthenticationException('No NKey seed found in credentials file'); + } + + $this->jwt = $jwt; + // NKeyAuth derives the real public key from the seed when none is given, + // so authenticate() below sends the correct nkey rather than an empty one. + $this->nkeyAuth = new NKeyAuth('', $seed); + } + + public function authenticate(?string $nonce = null): array + { + $nkeyFields = $this->nkeyAuth->authenticate($nonce); + + return [ + 'jwt' => $this->jwt, + 'nkey' => $nkeyFields['nkey'] ?? '', + 'sig' => $nkeyFields['sig'] ?? '', + ]; + } + + private function extractBetween(string $content, string $begin, string $end): ?string + { + $startPos = strpos($content, $begin); + if ($startPos === false) { + return null; + } + + $startPos += \strlen($begin); + $endPos = strpos($content, $end, $startPos); + if ($endPos === false) { + return null; + } + + return trim(substr($content, $startPos, $endPos - $startPos)); + } +} diff --git a/packages/nats/src/Auth/NKeyAuth.php b/packages/nats/src/Auth/NKeyAuth.php new file mode 100644 index 00000000000..31a76eb4dc1 --- /dev/null +++ b/packages/nats/src/Auth/NKeyAuth.php @@ -0,0 +1,158 @@ +decodeSeed($this->seed); + $keyPair = sodium_crypto_sign_seed_keypair($rawSeed); + $secretKey = sodium_crypto_sign_secretkey($keyPair); + $signature = sodium_crypto_sign_detached($nonce, $secretKey); + + return [ + 'nkey' => $this->publicKey(), + 'sig' => $this->base32Encode($signature), + ]; + } + + /** + * The public NKey. When constructed without an explicit public key (e.g. from + * a credentials file), it is derived from the seed so the server receives the + * real ed25519 public key rather than an empty string. + */ + public function publicKey(): string + { + if ($this->publicKey !== '') { + return $this->publicKey; + } + + return $this->derivePublicKey(); + } + + private function derivePublicKey(): string + { + $decoded = $this->base32Decode($this->seed); + if (\strlen($decoded) < 4) { + throw new AuthenticationException('Invalid NKey seed'); + } + + // The two-byte seed prefix encodes both the seed marker and the public + // key's role byte (see nkeys EncodeSeed): b1 = SEED | (role >> 5), + // b2 = (role & 31) << 3. Recover the role so the derived public key + // carries the correct prefix (e.g. 'U' for a user). + $b1 = \ord($decoded[0]); + $b2 = \ord($decoded[1]); + $role = (($b1 & 7) << 5) | (($b2 >> 3) & 31); + + $rawSeed = substr($decoded, 2, -2); + $keyPair = sodium_crypto_sign_seed_keypair($rawSeed); + $publicKey = sodium_crypto_sign_publickey($keyPair); + + return $this->encodePublicKey($role, $publicKey); + } + + private function encodePublicKey(int $role, string $publicKey): string + { + $raw = \chr($role) . $publicKey; + $crc = $this->crc16($raw); + // CRC is appended little-endian, matching the nkeys encoding. + $raw .= \chr($crc & 0xFF) . \chr(($crc >> 8) & 0xFF); + + return $this->base32Encode($raw); + } + + /** + * CRC-16/XMODEM (poly 0x1021, init 0x0000), as used by NATS nkeys. + */ + private function crc16(string $data): int + { + $crc = 0; + for ($i = 0, $len = \strlen($data); $i < $len; $i++) { + $crc ^= \ord($data[$i]) << 8; + for ($j = 0; $j < 8; $j++) { + $crc = ($crc & 0x8000) !== 0 ? (($crc << 1) ^ 0x1021) & 0xFFFF : ($crc << 1) & 0xFFFF; + } + } + + return $crc & 0xFFFF; + } + + private function decodeSeed(string $seed): string + { + $decoded = $this->base32Decode($seed); + if (\strlen($decoded) < 4) { + throw new AuthenticationException('Invalid NKey seed'); + } + + // Remove the 2-byte prefix and 2-byte CRC suffix + return substr($decoded, 2, -2); + } + + private function base32Decode(string $input): string + { + $alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'; + $input = strtoupper(rtrim($input, '=')); + $output = ''; + $buffer = 0; + $bitsLeft = 0; + + for ($i = 0, $len = \strlen($input); $i < $len; $i++) { + $val = strpos($alphabet, $input[$i]); + if ($val === false) { + throw new AuthenticationException('Invalid base32 character in NKey seed'); + } + $buffer = ($buffer << 5) | $val; + $bitsLeft += 5; + + if ($bitsLeft >= 8) { + $bitsLeft -= 8; + $output .= \chr(($buffer >> $bitsLeft) & 0xFF); + } + } + + return $output; + } + + private function base32Encode(string $input): string + { + $alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567'; + $output = ''; + $buffer = 0; + $bitsLeft = 0; + + for ($i = 0, $len = \strlen($input); $i < $len; $i++) { + $buffer = ($buffer << 8) | \ord($input[$i]); + $bitsLeft += 8; + + while ($bitsLeft >= 5) { + $bitsLeft -= 5; + $output .= $alphabet[($buffer >> $bitsLeft) & 0x1F]; + } + } + + if ($bitsLeft > 0) { + $output .= $alphabet[($buffer << (5 - $bitsLeft)) & 0x1F]; + } + + return $output; + } +} diff --git a/packages/nats/src/Auth/NoAuth.php b/packages/nats/src/Auth/NoAuth.php new file mode 100644 index 00000000000..f2f578b0332 --- /dev/null +++ b/packages/nats/src/Auth/NoAuth.php @@ -0,0 +1,13 @@ + $this->token, + ]; + } +} diff --git a/packages/nats/src/Auth/UserPassAuth.php b/packages/nats/src/Auth/UserPassAuth.php new file mode 100644 index 00000000000..682ef51c90e --- /dev/null +++ b/packages/nats/src/Auth/UserPassAuth.php @@ -0,0 +1,22 @@ + $this->user, + 'pass' => $this->pass, + ]; + } +} diff --git a/packages/nats/src/Connection.php b/packages/nats/src/Connection.php new file mode 100644 index 00000000000..00abb83f7dd --- /dev/null +++ b/packages/nats/src/Connection.php @@ -0,0 +1,1314 @@ + + */ + private const array CLOSING_ERRORS = [ + 'stale connection', + 'slow consumer', + 'maximum payload', + 'invalid client protocol', + 'authorization violation', + 'authentication timeout', + 'authentication expired', + 'secure connection - tls required', + 'maximum connections exceeded', + 'maximum control line exceeded', + 'unknown protocol operation', + 'parser error', + ]; + + /** + * The subset of {@see self::CLOSING_ERRORS} a reconnect cannot fix, so the + * connection is marked dead rather than rebuilt in a loop. + * + * @var list + */ + private const array UNRECOVERABLE_ERRORS = [ + 'authorization violation', + 'authentication timeout', + 'authentication expired', + 'secure connection - tls required', + 'maximum connections exceeded', + 'maximum control line exceeded', + 'unknown protocol operation', + 'parser error', + ]; + + private Transport $transport; + private Parser $parser; + private readonly Writer $writer; + private Authenticator $auth; + private ServerInfo $serverInfo; + private ConnectionOptions $options; + + /** @var array */ + private array $subscriptions = []; + private int $nextSid = 1; + private string $status = self::STATUS_DISCONNECTED; + private int $outstandingPings = 0; + private float $lastPingTime = 0.0; + + // Mux inbox for request-reply + private ?Subscription $inboxSub = null; + private string $inboxPrefix = ''; + private bool $collecting = false; + + // Reconnection + /** @var list */ + private array $serverPool = []; + private string $currentServer = ''; + /** @var list */ + private array $pendingBuffer = []; + private int $pendingBufferBytes = 0; + + private function __construct() + { + $this->writer = new Writer(); + } + + /** + * Connect to a NATS server. + * + * @param string|list|ConnectionOptions $urlOrOptions + */ + public static function connect( + string|array|ConnectionOptions $urlOrOptions = 'nats://127.0.0.1:4222', + ?ConnectionOptions $options = null, + ): self { + if ($urlOrOptions instanceof ConnectionOptions) { + $options = $urlOrOptions; + } elseif (!$options instanceof \Utopia\NATS\ConnectionOptions) { + $options = new ConnectionOptions(servers: $urlOrOptions); + } + + $conn = new self(); + $conn->options = $options; + $conn->auth = $conn->resolveAuthenticator($options); + $conn->serverPool = $conn->buildServerPool($options); + $conn->doConnect(); + + return $conn; + } + + public function publish(string $subject, string $data = '', ?string $replyTo = null, ?Headers $headers = null): void + { + $this->ensureConnected(); + + $this->send($this->encode($subject, $data, $replyTo, $headers)); + } + + private function encode(string $subject, string $data, ?string $replyTo, ?Headers $headers): string + { + $hasHeaders = $headers instanceof \Utopia\NATS\Headers && $headers->all() !== []; + + if ($hasHeaders && isset($this->serverInfo) && !$this->serverInfo->headersSupported) { + throw new ProtocolException('Server does not support message headers'); + } + + $headerWire = $hasHeaders ? $headers->toWire() : ''; + // The header block counts against the server's max payload budget. + $wireSize = \strlen($headerWire) + \strlen($data); + if (isset($this->serverInfo) && $wireSize > $this->serverInfo->maxPayload) { + throw new MaxPayloadException( + "Payload size {$wireSize} exceeds server maximum of {$this->serverInfo->maxPayload}", + ); + } + + return $hasHeaders + ? $this->writer->hpub($subject, $headerWire, $data, $replyTo) + : $this->writer->pub($subject, $data, $replyTo); + + } + + public function subscribe(string $subject, ?\Closure $callback = null, ?string $queue = null): Subscription + { + $this->ensureConnected(); + + $sid = (string) $this->nextSid++; + $slow = $this->options->onSlowConsumer; + $sub = new Subscription( + $sid, + $subject, + $queue, + $callback instanceof \Closure ? fn (Message $message) => $this->notify($callback, $message) : null, + $this->options->subPendingMsgsLimit, + $this->options->subPendingBytesLimit, + $slow instanceof \Closure ? fn (Subscription $subscription) => $this->notify($slow, $subscription) : null, + ); + $sub->setConnection($this); + + $this->subscriptions[$sid] = $sub; + $this->send($this->writer->sub($subject, $sid, $queue)); + + return $sub; + } + + public function queueSubscribe(string $subject, string $queue, ?\Closure $callback = null): Subscription + { + return $this->subscribe($subject, $callback, $queue); + } + + public function unsubscribe(Subscription $sub, ?int $maxMessages = null): void + { + if ($maxMessages !== null) { + $sub->setMaxMessages($sub->getReceived() + $maxMessages); + $this->send($this->writer->unsub($sub->sid, $maxMessages)); + } else { + $sub->setInactive(); + unset($this->subscriptions[$sub->sid]); + $this->send($this->writer->unsub($sub->sid)); + } + } + + public function request(string $subject, string $data = '', ?float $timeout = null, ?Headers $headers = null): Message + { + $request = new Request($subject, $data, $headers); + for ($attempt = 0; ; $attempt++) { + $result = null; + $this->requestBatch([$request], static function (int $index, Message|\Throwable $response) use (&$result): void { + $result = $response; + }, $timeout); + // Only the single-request API retries a server's explicit stale rejection. + // Ambiguous writes and other errors are never replayed. + if ($attempt === 0 && $result instanceof ProtocolException + && str_contains(strtolower($result->getMessage()), 'stale connection') + && $this->options->allowReconnect && $this->status === self::STATUS_DISCONNECTED) { + $this->attemptReconnect(); + continue; + } + if ($result instanceof \Throwable) { + throw $result; + } + return $result ?? throw new \LogicException('Request completed without an outcome'); + } + } + + /** + * Send independent requests together. Deliver indexed outcomes as replies arrive. + * One response deadline starts after writing; ambiguous writes are never replayed. + * Callback exceptions abort collection, not already-sent remote work. + * + * @param list $requests + * @param \Closure(int, Message|\Throwable): void $reply + */ + public function requestBatch(array $requests, \Closure $reply, ?float $timeout = null): void + { + $this->assertReadable(); + $timeout ??= $this->options->requestTimeout; + if (!is_finite($timeout) || $timeout <= 0 || !array_is_list($requests)) { + throw new \InvalidArgumentException('Requests must be a list with a finite positive timeout'); + } + foreach ($requests as $request) { + if (!$request instanceof Request) { + throw new \InvalidArgumentException('Expected Request objects'); + } + } + if ($requests === []) { + return; + } + $pending = []; + $publishing = $written = false; + $this->collecting = true; + try { + try { + $this->ensureConnected(); + $this->ensureInboxSub(); + $wire = ''; + foreach ($requests as $index => $request) { + $token = Inbox::generateId(); + $pending[$token] = $index; + $wire .= $this->encode($request->subject, $request->data, $this->inboxPrefix . '.' . $token, $request->headers); + } + $publishing = true; + $this->transport->write($wire); + $written = true; + $deadline = hrtime(true) / 1e9 + $timeout; + while ($pending !== []) { + $remaining = $deadline - hrtime(true) / 1e9; + if ($remaining <= 0) { + throw new TimeoutException("Requests timed out after {$timeout}s"); + } + $this->checkPings(); + $message = $this->readMessage($remaining, reconnect: false); + $token = $message instanceof Message ? $this->extractInboxToken($message->subject) : null; + if ($token !== null && isset($pending[$token])) { + $index = $pending[$token]; + unset($pending[$token]); + $response = $message->headers?->getStatus() === '503' + ? new NatsException('No responders for request') : $message; + $this->notify($reply, $index, $response); + } + } + } catch (NatsException $error) { + if ($written && $error instanceof TimeoutException) { + $error = new TimeoutException("Request timed out after {$timeout}s", previous: $error); + } + if ($error instanceof ConnectionException && (!$written || !$error instanceof TimeoutException)) { + $this->recycleDeadConnection(false); + } + if (!$publishing) { + throw $error; + } + foreach ($pending as $index) { + $this->notify($reply, $index, $error); + } + } + } catch (CallbackException $error) { + throw $error->cause; + } finally { + $this->collecting = false; + } + } + + private function assertReadable(): void + { + if ($this->collecting) { + throw new \LogicException('Cannot read the connection while collecting requests'); + } + } + + /** Preserve exception provenance across protocol dispatch and application callbacks. */ + private function notify(\Closure $callback, mixed ...$arguments): void + { + try { + $callback(...$arguments); + } catch (\Throwable $error) { + throw $this->collecting && !$error instanceof CallbackException ? new CallbackException($error) : $error; + } + } + + /** + * Scatter-gather request: publish once and collect every reply until a stop + * condition is met (ADR-47). A 503 "no responders" reply means zero + * responders and yields an empty list. + * + * @param array{max?: int, timeout?: float, stall?: float} $opts + * max stop after this many replies + * timeout overall deadline in seconds (defaults to the request timeout) + * stall stop when no new reply arrives within this many seconds + * @return list + */ + public function requestMany(string $subject, string $data = '', array $opts = []): array + { + $this->assertReadable(); + $this->ensureConnected(); + + $max = $opts['max'] ?? null; + $timeout = $opts['timeout'] ?? $this->options->requestTimeout; + $stall = $opts['stall'] ?? null; + + // Dedicated inbox subscription so replies never touch the mux inbox used + // by the single-reply request(). + $inbox = Inbox::create($this->options->inboxPrefix); + $sub = $this->subscribe($inbox); + $this->publish($subject, $data, $inbox); + + $deadline = microtime(true) + $timeout; + $messages = []; + + while ($max === null || \count($messages) < $max) { + $remaining = $deadline - microtime(true); + if ($remaining <= 0) { + break; + } + + // A stall window caps how long we wait for the next reply; whichever + // of stall/overall-deadline is sooner bounds this iteration. + $wait = $stall !== null ? min($remaining, $stall) : $remaining; + + $msg = $sub->nextMessage($wait); + if (!$msg instanceof Message) { + // Overall deadline or stall window elapsed with no new reply. + break; + } + + // 503 no-responders: zero responders, return whatever we have (none). + if ($msg->headers instanceof Headers && $msg->headers->getStatus() === '503') { + break; + } + + $messages[] = $msg; + } + + $this->unsubscribe($sub); + + return $messages; + } + + public function newInbox(): string + { + return Inbox::create($this->options->inboxPrefix); + } + + /** + * Drive connection maintenance without consuming a message: send the + * keepalive PING when it is due, and recycle the connection when the server + * has stopped answering. A holder that keeps a connection open without + * reading from it -- a pooled publisher, say -- must call this on an + * interval shorter than the server's ping deadline, because nothing else + * in this class runs on its own. + * + * This reads the socket, so the caller must hold the connection + * exclusively for the call. Pool maintenance satisfies that by sweeping + * only resources that are idle; a coroutine must never tick a connection + * another coroutine is inside a call on. + */ + public function tick(): void + { + $this->assertReadable(); + $this->checkPings(); + $this->collectPongs(); + } + + /** + * Consume the PONGs owed for the keepalive PINGs this connection has sent. + * + * checkPings() only writes: it is the read path that clears + * $outstandingPings, and a holder that never reads has none. Ticking such a + * connection would therefore march it to maxPingsOut and declare a + * perfectly healthy socket stale -- the keepalive would cause the outage it + * exists to prevent. Reading here closes that loop. + * + * Bounded, and only while a PONG is actually owed, so maintenance can never + * turn into a receive loop on a connection carrying subscription traffic. + */ + private function collectPongs(): void + { + // readMessage() rather than processMessage(): the latter checks the + // keepalive on entry, so draining through it would send a fresh PING + // for every PONG collected and the outstanding count could never fall. + for ($read = 0; $this->outstandingPings > 0 && $read < self::TICK_MAX_READS; $read++) { + $this->readMessage(self::TICK_READ_TIMEOUT); + } + } + + /** + * Read and dispatch one server message. + */ + public function processMessage(?float $timeout = null): ?Message + { + $this->assertReadable(); + $this->checkPings(); + + return $this->readMessage($timeout); + } + + /** + * The read half of {@see self::processMessage()}, without the keepalive + * check, so maintenance can drain the socket without writing to it. + */ + private function readMessage(?float $timeout = null, bool $reconnect = true): ?Message + { + try { + [$op, $data] = $this->parser->next($timeout); + } catch (TimeoutException $error) { + if (!$reconnect) { + throw $error; + } + return null; + } catch (ConnectionException $e) { + if ($reconnect && $this->options->allowReconnect && $this->status !== self::STATUS_CLOSED) { + $this->attemptReconnect(); + return null; + } + throw $e; + } + + return $this->dispatchOp($op, $data); + } + + /** + * Process messages in a loop. + * + * @param int $count Number of messages to process (0 = forever) + */ + public function wait(int $count = 0, ?float $timeout = null): void + { + $processed = 0; + $deadline = $timeout !== null ? microtime(true) + $timeout : null; + + while ($count === 0 || $processed < $count) { + $remaining = $deadline !== null ? $deadline - microtime(true) : null; + if ($remaining !== null && $remaining <= 0) { + return; + } + + $msg = $this->processMessage($remaining); + if ($msg instanceof \Utopia\NATS\Message) { + $processed++; + } + } + } + + public function jetStream(?string $domain = null, ?string $apiPrefix = null): JetStream\JetStream + { + return new JetStream\JetStream($this, $domain, $apiPrefix); + } + + public function flush(?float $timeout = null): void + { + $this->assertReadable(); + $this->ensureConnected(); + $timeout ??= $this->options->connectTimeout; + + $this->send($this->writer->ping()); + $deadline = microtime(true) + $timeout; + + while (true) { + $remaining = $deadline - microtime(true); + if ($remaining <= 0) { + throw new TimeoutException('Flush timed out'); + } + + [$op, $data] = $this->parser->next($remaining); + if ($op === ServerOp::Pong) { + $this->outstandingPings = 0; + return; + } + $this->dispatchOp($op, $data); + } + } + + public function drain(?float $timeout = null): void + { + $this->assertReadable(); + if ($this->status !== self::STATUS_CONNECTED) { + return; + } + + $this->status = self::STATUS_DRAINING; + $timeout ??= $this->options->drainTimeout; + + // Unsub all subscriptions, then send a PING. The server processes the + // UNSUBs and flushes any already-queued messages ahead of the PONG, so + // receiving that PONG is a deterministic barrier: everything the server + // had for us has arrived, and nothing new will. This replaces the old + // pure-timeout drain. + foreach ($this->subscriptions as $sub) { + $this->send($this->writer->unsub($sub->sid)); + } + $this->send($this->writer->ping()); + + $deadline = microtime(true) + $timeout; + while (true) { + $remaining = $deadline - microtime(true); + if ($remaining <= 0) { + break; + } + + try { + [$op, $data] = $this->parser->next($remaining); + } catch (TimeoutException|ConnectionException) { + break; + } + + if ($op === ServerOp::Pong) { + break; + } + + $this->dispatchOp($op, $data); + } + + $this->close(); + } + + public function close(): void + { + if ($this->status === self::STATUS_CLOSED) { + return; + } + + $previousStatus = $this->status; + $this->status = self::STATUS_CLOSED; + + foreach ($this->subscriptions as $sub) { + $sub->setInactive(); + } + $this->subscriptions = []; + + if (isset($this->transport)) { + $this->transport->close(); + } + + if ($previousStatus === self::STATUS_CONNECTED && $this->options->onClose instanceof \Closure) { + $this->notify($this->options->onClose); + } + } + + public function isConnected(): bool + { + return $this->status === self::STATUS_CONNECTED; + } + + public function isClosed(): bool + { + return $this->status === self::STATUS_CLOSED; + } + + public function isReconnecting(): bool + { + return $this->status === self::STATUS_RECONNECTING; + } + + public function getServerInfo(): ServerInfo + { + return $this->serverInfo; + } + + public function getOptions(): ConnectionOptions + { + return $this->options; + } + + public function getStatus(): string + { + return $this->status; + } + + // --- Internal --- + + private function doConnect(): void + { + $this->status = self::STATUS_CONNECTING; + $lastError = null; + + foreach ($this->serverPool as $url) { + try { + $this->connectToServer($url); + $this->status = self::STATUS_CONNECTED; + return; + } catch (\Throwable $e) { + $lastError = $e; + continue; + } + } + + $this->status = self::STATUS_DISCONNECTED; + throw new ConnectionException( + 'Failed to connect to any NATS server', + previous: $lastError, + ); + } + + private function connectToServer(string $url): void + { + $parsed = $this->parseUrl($url); + $host = $parsed['host']; + $port = $parsed['port']; + $scheme = $parsed['scheme']; + + // Create transport + if ($this->options->transportFactory instanceof \Closure) { + $this->transport = ($this->options->transportFactory)($scheme); + } elseif ($scheme === 'ws' || $scheme === 'wss') { + $this->transport = new WebSocketTransport($scheme === 'wss', $this->tlsOptions()); + } elseif ($scheme === 'tls' || $this->options->tls) { + $this->transport = new TlsTransport($this->tlsOptions()); + } else { + $this->transport = new TcpTransport(); + } + + $this->transport->connect($host, $port, $this->options->connectTimeout); + $this->parser = new Parser($this->transport); + + // Read INFO + [$op, $data] = $this->parser->next($this->options->connectTimeout); + if ($op !== ServerOp::Info) { + throw new ProtocolException("Expected INFO, got {$op->value}"); + } + $this->serverInfo = ServerInfo::fromArray($data); + + // Merge connect_urls into server pool + foreach ($this->serverInfo->connectUrls as $connectUrl) { + if (!\in_array($connectUrl, $this->serverPool, true)) { + $this->serverPool[] = $this->normalizeUrl($connectUrl); + } + } + + // TLS upgrade over a plaintext TCP connection: when the server requires + // TLS, or when the caller opted in and the server advertises it is + // available. A TlsTransport / custom transport handles its own TLS. + if ($this->transport instanceof TcpTransport && $scheme !== 'tls') { + $wantsUpgrade = $this->serverInfo->tlsRequired + || ($this->options->tls && $this->serverInfo->tlsAvailable); + if ($wantsUpgrade) { + $this->transport->upgradeTls($this->tlsOptions()); + } + } + + // Send CONNECT + $connectPayload = $this->buildConnectPayload(); + $this->transport->write($this->writer->connect($connectPayload)); + + // Send PING and wait for PONG to confirm connection + $this->transport->write($this->writer->ping()); + + // Read until we get PONG (skip +OK if verbose) + $deadline = microtime(true) + $this->options->connectTimeout; + while (true) { + $remaining = $deadline - microtime(true); + if ($remaining <= 0) { + throw new TimeoutException('Connection handshake timed out'); + } + + [$op, $data] = $this->parser->next($remaining); + + if ($op === ServerOp::Pong) { + break; + } + if ($op === ServerOp::Err) { + $this->transport->close(); + $errMsg = \is_string($data) ? $data : 'Unknown error'; + if (stripos($errMsg, 'authorization') !== false || stripos($errMsg, 'authentication') !== false) { + throw new AuthenticationException("Server error: {$errMsg}"); + } + throw new ConnectionException("Server error: {$errMsg}"); + } + // Skip +OK + } + + $this->lastPingTime = microtime(true); + $this->currentServer = $url; + } + + private function buildConnectPayload(): array + { + $headersSupported = $this->serverInfo->headersSupported; + + $payload = [ + 'verbose' => $this->options->verbose, + 'pedantic' => $this->options->pedantic, + 'lang' => self::CLIENT_LANG, + 'version' => self::CLIENT_VERSION, + 'protocol' => 1, + 'echo' => $this->options->echo, + // Only negotiate headers (and the header-based no_responders reply) + // when the server advertises support for them. + 'headers' => $headersSupported, + 'no_responders' => $headersSupported, + ]; + + if ($this->options->name !== '') { + $payload['name'] = $this->options->name; + } + + $authFields = $this->auth->authenticate($this->serverInfo->nonce); + $payload = array_merge($payload, $authFields); + + // Dynamic providers are resolved on every (re)connect so refreshed + // tokens/JWTs take effect without rebuilding the connection. + if ($this->options->tokenProvider instanceof \Closure) { + $payload['auth_token'] = (string) ($this->options->tokenProvider)(); + } + if ($this->options->jwtProvider instanceof \Closure) { + $payload['jwt'] = (string) ($this->options->jwtProvider)(); + } + + return $payload; + } + + private function dispatchOp(ServerOp $op, mixed $data): ?Message + { + return match ($op) { + ServerOp::Msg, ServerOp::HMsg => $this->handleMessage($data), + ServerOp::Ping => $this->handlePing(), + ServerOp::Pong => $this->handlePong(), + ServerOp::Err => $this->handleError($data), + ServerOp::Ok => null, + ServerOp::Info => $this->handleInfo($data), + }; + } + + private function handleMessage(array $data): Message + { + $headers = null; + if (isset($data['headers'])) { + $headers = Headers::fromWire($data['headers']); + } + + $msg = new Message( + subject: $data['subject'], + data: $data['payload'], + replyTo: $data['replyTo'], + headers: $headers, + sid: $data['sid'], + ); + + // Private inbox replies belong to the active request reader, never a backlog. + if ($this->inboxSub instanceof Subscription && $data['sid'] === $this->inboxSub->sid) { + return $msg; + } + + // Dispatch to subscription + $sub = $this->subscriptions[$data['sid']] ?? null; + if ($sub !== null && $sub->isActive()) { + $sub->deliver($msg); + + // Clean up auto-unsubscribed subscriptions + if (!$sub->isActive()) { + unset($this->subscriptions[$data['sid']]); + } + } + + return $msg; + } + + private function handlePing(): null + { + $this->send($this->writer->pong()); + return null; + } + + private function handlePong(): null + { + $this->outstandingPings = max(0, $this->outstandingPings - 1); + return null; + } + + private function handleError(mixed $data): never + { + $message = \is_string($data) ? $data : 'Unknown server error'; + $error = self::mapServerError($message); + + // Closing errors must make the connection unusable even when onError throws. + if (self::closesConnection($message)) { + $this->recycleDeadConnection(!$this->collecting && self::reconnectsAfter($message)); + } + + if ($this->options->onError instanceof \Closure) { + $this->notify($this->options->onError, new NatsException($message)); + } + + throw $error; + } + + /** + * Recycle a connection the server has closed under us: reconnect when that + * can help, otherwise mark it dead so ensureConnected() refuses the next + * call. A reconnect that exhausts its attempts leaves the status at + * disconnected, and the -ERR that got us here is the more useful thing to + * report, so its ConnectionException is deliberately swallowed. + * + * Marking it dead is the important half. An error we decline to reconnect + * after is still an error the server closed the socket for, and leaving the + * status at connected would put us back in the state this whole path + * exists to remove: a publish that writes into a dead socket and returns + * success. + */ + private function recycleDeadConnection(bool $mayReconnect = true): void + { + if ($mayReconnect && $this->options->allowReconnect && $this->status !== self::STATUS_CLOSED) { + try { + $this->attemptReconnect(); + } catch (ConnectionException) { + // Status is disconnected; handleError() still throws the -ERR. + } + + return; + } + + $this->status = self::STATUS_DISCONNECTED; + + if (isset($this->transport)) { + $this->transport->close(); + } + } + + /** + * Whether NATS closes the connection after this -ERR. Purely a statement + * about the socket, deliberately separate from whether reconnecting is + * worth doing -- see {@see self::reconnectsAfter()}. + * + * The two were one predicate at first, which quietly recreated the bug this + * class is fixing: an error excluded from reconnect was also excluded from + * being recorded as closed, so an authorization failure left the status at + * connected over a socket the server had already dropped, and the next + * publish() wrote into it and returned success. + * + * Pure so the classification can be unit tested without a live connection. + */ + public static function closesConnection(string $message): bool + { + $lower = strtolower($message); + + foreach (self::CLOSING_ERRORS as $needle) { + if (str_contains($lower, $needle)) { + return true; + } + } + + return false; + } + + /** + * Whether rebuilding the connection can plausibly succeed after this -ERR. + * + * Only asked of errors that closed the connection. False for the ones a + * retry cannot fix -- credentials the server just rejected, a TLS + * requirement we are not meeting, a server at its connection ceiling, or + * protocol this client got wrong -- because reconnecting there turns a hard + * failure into a hot loop against a server that will refuse us identically + * every time. Those connections are marked dead instead, which surfaces the + * failure to the caller rather than hiding it behind a retry storm. + */ + public static function reconnectsAfter(string $message): bool + { + $lower = strtolower($message); + + foreach (self::UNRECOVERABLE_ERRORS as $needle) { + if (str_contains($lower, $needle)) { + return false; + } + } + + return true; + } + + /** + * Map a server -ERR string to a typed exception (ADR-7). Pure so the mapping + * can be unit tested without a live connection. + */ + public static function mapServerError(string $message): NatsException + { + $lower = strtolower($message); + + return match (true) { + str_contains($lower, 'permissions violation') => new PermissionException($message), + str_contains($lower, 'authorization violation'), + str_contains($lower, 'authentication expired'), + str_contains($lower, 'authorization'), + str_contains($lower, 'authentication') => new AuthenticationException($message), + str_contains($lower, 'maximum payload') => new MaxPayloadException($message), + default => new ProtocolException("Server error: {$message}"), + }; + } + + private function handleInfo(mixed $data): null + { + if (\is_array($data)) { + $this->serverInfo = ServerInfo::fromArray($data); + + // Async INFO may advertise additional cluster members; fold any new + // ones into the pool so failover has somewhere to go. + foreach ($this->serverInfo->connectUrls as $connectUrl) { + $normalized = $this->normalizeUrl($connectUrl); + if (!\in_array($normalized, $this->serverPool, true)) { + $this->serverPool[] = $normalized; + } + } + + // Lame-duck mode (ADR-5): the server is draining and will close the + // connection. Notify the caller and move to a different server if we + // know of one. + if (($data['ldm'] ?? false) === true) { + $this->handleLameDuck(); + } + } + return null; + } + + private function handleLameDuck(): void + { + if ($this->options->onLameDuck instanceof \Closure) { + $this->notify($this->options->onLameDuck); + } + + $others = array_values(array_filter( + $this->serverPool, + fn (string $url): bool => $url !== $this->currentServer, + )); + + // Only proactively reconnect when a different server is available; + // otherwise ride out the current connection until it is closed. + if ($others !== [] && !$this->collecting && $this->options->allowReconnect) { + $this->serverPool = [...$others, $this->currentServer]; + $this->attemptReconnect(); + } + } + + /** + * Emit the keepalive PING when one is due, and act on a spent budget first. + * + * Only called from paths that also read the socket -- {@see + * self::processMessage()}, {@see self::requestBatch()} and {@see self::tick()} -- because a PING is only + * half a keepalive. It is the read that clears $outstandingPings, so a + * caller that sends without reading marches the count to maxPingsOut and + * declares a healthy connection stale. See {@see self::ensureConnected()} + * for the write path, which takes the verdict without the emission. + */ + private function checkPings(): void + { + // Drain received operations before sending another probe or judging its reply. + if ($this->status !== self::STATUS_CONNECTED || $this->parser->hasBufferedData()) { + return; + } + + $now = microtime(true); + if (($now - $this->lastPingTime) < $this->options->pingInterval) { + return; + } + + if ($this->checkStale()) { + return; + } + + try { + $this->send($this->writer->ping()); + $this->outstandingPings++; + $this->lastPingTime = $now; + } catch (ConnectionException $error) { + if ($this->collecting) { + throw $error; + } + if ($this->options->allowReconnect) { + $this->attemptReconnect(); + } + } + } + + /** + * The staleness verdict on its own: the server has left this many PINGs + * unanswered, so the connection is gone. Reconnects where that is allowed + * and raises otherwise, and reports whether it acted so callers can stop. + * + * Split out of checkPings() so the write path can reach the verdict without + * the emission that goes with it. + */ + private function checkStale(): bool + { + // An owed PONG may already be buffered, including between request batches. + if ($this->status !== self::STATUS_CONNECTED || $this->parser->hasBufferedData()) { + return false; + } + + if ($this->outstandingPings < $this->options->maxPingsOut) { + return false; + } + + if (!$this->collecting && $this->options->allowReconnect) { + $this->attemptReconnect(); + + return true; + } + + throw new ConnectionException('Stale connection: too many outstanding pings'); + } + + private function attemptReconnect(): void + { + if ($this->status === self::STATUS_CLOSED || $this->status === self::STATUS_RECONNECTING) { + return; + } + + $this->status = self::STATUS_RECONNECTING; + + if ($this->options->onDisconnect instanceof \Closure) { + $this->notify($this->options->onDisconnect); + } + + if (isset($this->transport)) { + $this->transport->close(); + } + + for ($attempt = 0; $attempt < $this->options->maxReconnectAttempts; $attempt++) { + // Exponential backoff (capped) plus jitter before reconnecting. + if ($attempt > 0) { + $backoff = self::reconnectBackoff( + $attempt, + $this->options->reconnectWait, + $this->options->maxReconnectWait, + ); + $wait = $backoff + (lcg_value() * $this->options->reconnectJitter); + usleep((int) ($wait * 1_000_000)); + } + + foreach ($this->serverPool as $url) { + try { + $this->connectToServer($url); + $this->status = self::STATUS_CONNECTED; + $this->outstandingPings = 0; + + // Re-subscribe all active subscriptions + foreach ($this->subscriptions as $sub) { + if ($sub->isActive()) { + $this->send($this->writer->sub($sub->subject, $sub->sid, $sub->queue)); + } + } + + // Flush any buffered publishes + $buffered = $this->pendingBuffer; + $this->pendingBuffer = []; + $this->pendingBufferBytes = 0; + foreach ($buffered as $cmd) { + $this->send($cmd); + } + + if ($this->options->onReconnect instanceof \Closure) { + $this->notify($this->options->onReconnect); + } + + return; + } catch (CallbackException $error) { + throw $error; + } catch (\Throwable) { + continue; + } + } + } + + $this->status = self::STATUS_DISCONNECTED; + throw new ConnectionException('Failed to reconnect to any NATS server'); + } + + private function ensureConnected(): void + { + if ($this->status !== self::STATUS_CONNECTED && $this->status !== self::STATUS_DRAINING) { + throw new ConnectionException("Not connected (status: {$this->status})"); + } + + // Take the keepalive verdict before the caller writes rather than + // after: a connection that has already outlived the server's ping + // deadline is recycled here, so the write that follows lands on a live + // socket or raises. + // + // The verdict only, never the PING. checkPings() increments + // $outstandingPings and it is the read path that clears it, so emitting + // here would make every write-only caller -- a pooled publisher between + // maintenance sweeps, most of all -- accumulate one unanswered PING per + // interval and reconnect a perfectly healthy socket on a timer. A + // connection that is actively publishing is not idle, either: the + // server sees its traffic, so there is nothing for a PING from here to + // keep alive. Emission belongs to the paths that also read -- see + // {@see self::checkPings()}. + $this->checkStale(); + } + + private function send(string $data): void + { + if ($this->status === self::STATUS_RECONNECTING) { + $this->bufferPending($data); + return; + } + + try { + $this->transport->write($data); + } catch (ConnectionException $e) { + if (!$this->collecting && $this->options->allowReconnect && $this->status !== self::STATUS_CLOSED) { + $this->bufferPending($data); + $this->attemptReconnect(); + return; + } + $this->recycleDeadConnection(false); + throw $e; + } + } + + /** + * Buffer a command while reconnecting, enforcing the reconnect buffer cap. + * Commands that would exceed the cap are dropped (and reported via onError) + * rather than growing the buffer without bound. + */ + private function bufferPending(string $data): void + { + if (!self::reconnectBufferAccepts($this->pendingBufferBytes, \strlen($data), $this->options->reconnectBufSize)) { + if ($this->options->onError instanceof \Closure) { + $this->notify($this->options->onError, new NatsException('Reconnect buffer full; dropping pending message')); + } + return; + } + + $this->pendingBuffer[] = $data; + $this->pendingBufferBytes += \strlen($data); + } + + /** + * Exponential reconnect backoff (in seconds), capped. Attempt 0 waits 0s + * (the first reconnect is immediate); subsequent attempts grow by $factor. + */ + public static function reconnectBackoff(int $attempt, float $base, float $cap, float $factor = 2.0): float + { + if ($attempt <= 0) { + return 0.0; + } + + $delay = $base * ($factor ** ($attempt - 1)); + + return min($delay, $cap); + } + + /** + * Whether $incomingBytes may be appended to the reconnect buffer without + * exceeding $cap. A non-positive cap disables buffering entirely. + */ + public static function reconnectBufferAccepts(int $currentBytes, int $incomingBytes, int $cap): bool + { + if ($cap <= 0) { + return false; + } + + return ($currentBytes + $incomingBytes) <= $cap; + } + + private function ensureInboxSub(): void + { + if ($this->inboxSub instanceof \Utopia\NATS\Subscription) { + return; + } + + $this->inboxPrefix = $this->options->inboxPrefix . '.' . Inbox::generateId(); + $this->inboxSub = $this->subscribe($this->inboxPrefix . '.*'); + } + + private function extractInboxToken(string $subject): ?string + { + if (!str_starts_with($subject, $this->inboxPrefix . '.')) { + return null; + } + + return substr($subject, \strlen($this->inboxPrefix) + 1); + } + + private function resolveAuthenticator(ConnectionOptions $options): Authenticator + { + if ($options->credentialsFile !== null) { + return new CredentialsAuth($options->credentialsFile); + } + + if ($options->nkey !== null && $options->nkeySeed !== null) { + return new NKeyAuth($options->nkey, $options->nkeySeed); + } + + if ($options->token !== null) { + return new TokenAuth($options->token); + } + + if ($options->user !== null && $options->pass !== null) { + return new UserPassAuth($options->user, $options->pass); + } + + // Check URL for user info + foreach ($options->servers as $url) { + $parsed = parse_url($url); + if (isset($parsed['user'])) { + $user = rawurldecode($parsed['user']); + $pass = isset($parsed['pass']) ? rawurldecode($parsed['pass']) : ''; + + if ($pass !== '') { + return new UserPassAuth($user, $pass); + } + return new TokenAuth($user); + } + } + + return new NoAuth(); + } + + /** + * @return list + */ + private function buildServerPool(ConnectionOptions $options): array + { + $servers = array_map($this->normalizeUrl(...), $options->servers); + + if (!$options->noRandomize && \count($servers) > 1) { + shuffle($servers); + } + + return $servers; + } + + /** + * TLS context options shared by the initial TLS connect and STARTTLS upgrade. + * + * @return array + */ + private function tlsOptions(): array + { + $opts = [ + 'cafile' => $this->options->tlsCaFile, + 'local_cert' => $this->options->tlsCertFile, + 'local_pk' => $this->options->tlsKeyFile, + 'verify_peer' => $this->options->tlsVerify, + 'verify_peer_name' => $this->options->tlsVerify, + ]; + + if ($this->options->tlsServerName !== null) { + $opts['peer_name'] = $this->options->tlsServerName; + } + + return $opts; + } + + private function normalizeUrl(string $url): string + { + if (!preg_match('#^(nats|tls|ws|wss)://#', $url)) { + return 'nats://' . $url; + } + return $url; + } + + /** + * @return array{scheme: string, host: string, port: int} + */ + private function parseUrl(string $url): array + { + $parsed = parse_url($url); + if ($parsed === false) { + throw new ConnectionException("Invalid server URL: {$url}"); + } + + return [ + 'scheme' => $parsed['scheme'] ?? 'nats', + 'host' => $parsed['host'] ?? '127.0.0.1', + 'port' => $parsed['port'] ?? 4222, + ]; + } +} diff --git a/packages/nats/src/ConnectionOptions.php b/packages/nats/src/ConnectionOptions.php new file mode 100644 index 00000000000..56719d0c40a --- /dev/null +++ b/packages/nats/src/ConnectionOptions.php @@ -0,0 +1,74 @@ + */ + public readonly array $servers; + + /** + * @param string|list $servers + */ + public function __construct( + string|array $servers = 'nats://127.0.0.1:4222', + public readonly string $name = '', + // Auth + public readonly ?string $user = null, + public readonly ?string $pass = null, + public readonly ?string $token = null, + public readonly ?string $nkey = null, + public readonly ?string $nkeySeed = null, + public readonly ?string $credentialsFile = null, + // TLS + public readonly bool $tls = false, + public readonly ?string $tlsCaFile = null, + public readonly ?string $tlsCertFile = null, + public readonly ?string $tlsKeyFile = null, + public readonly bool $tlsVerify = true, + public readonly ?string $tlsServerName = null, + // Dynamic auth providers, resolved on every (re)connect so tokens can refresh. + public readonly ?\Closure $tokenProvider = null, + public readonly ?\Closure $jwtProvider = null, + // Reconnection + public readonly bool $allowReconnect = true, + public readonly int $maxReconnectAttempts = 60, + public readonly float $reconnectWait = 2.0, + public readonly float $maxReconnectWait = 8.0, + public readonly float $reconnectJitter = 0.1, + // Max bytes buffered for pending publishes while reconnecting; excess is dropped. + public readonly int $reconnectBufSize = 8_388_608, + // Slow-consumer limits per subscription (pending messages / bytes). + public readonly int $subPendingMsgsLimit = 65536, + public readonly int $subPendingBytesLimit = 67_108_864, + // Timeouts + public readonly float $connectTimeout = 2.0, + public readonly float $requestTimeout = 5.0, + public readonly float $drainTimeout = 30.0, + // PING/PONG + public readonly float $pingInterval = 120.0, + public readonly int $maxPingsOut = 2, + // Misc + public readonly bool $verbose = false, + public readonly bool $pedantic = false, + public readonly bool $echo = true, + public readonly bool $noRandomize = false, + public readonly string $inboxPrefix = '_INBOX', + // Callbacks + public readonly ?\Closure $onDisconnect = null, + public readonly ?\Closure $onReconnect = null, + public readonly ?\Closure $onClose = null, + public readonly ?\Closure $onError = null, + // Fired with the Subscription when it exceeds its pending limits. + public readonly ?\Closure $onSlowConsumer = null, + // Fired when the server signals lame-duck mode (async INFO with "ldm": true). + public readonly ?\Closure $onLameDuck = null, + // Transport: fn(string $scheme): Transport. Defaults to the stream-based + // Tcp/Tls transports; inject to use a coroutine-native transport (e.g. Swoole). + public readonly ?\Closure $transportFactory = null, + ) { + $this->servers = \is_string($servers) ? [$servers] : $servers; + } +} diff --git a/packages/nats/src/Exception/AuthenticationException.php b/packages/nats/src/Exception/AuthenticationException.php new file mode 100644 index 00000000000..a171a94f31e --- /dev/null +++ b/packages/nats/src/Exception/AuthenticationException.php @@ -0,0 +1,9 @@ +> */ + private array $headers = []; + private string $status = ''; + private string $description = ''; + + public function set(string $name, string $value): self + { + $this->headers[$name] = [$value]; + return $this; + } + + public function add(string $name, string $value): self + { + $this->headers[$name][] = $value; + return $this; + } + + public function get(string $name): ?string + { + return $this->headers[$name][0] ?? null; + } + + /** @return list */ + public function getAll(string $name): array + { + return $this->headers[$name] ?? []; + } + + public function has(string $name): bool + { + return isset($this->headers[$name]); + } + + public function delete(string $name): self + { + unset($this->headers[$name]); + return $this; + } + + /** @return array> */ + public function all(): array + { + return $this->headers; + } + + public function getStatus(): string + { + return $this->status; + } + + public function setStatus(string $status, string $description = ''): self + { + $this->status = $status; + $this->description = $description; + return $this; + } + + public function getDescription(): string + { + return $this->description; + } + + public function toWire(): string + { + $result = 'NATS/1.0'; + if ($this->status !== '') { + $result .= ' ' . $this->status; + if ($this->description !== '') { + $result .= ' ' . $this->description; + } + } + $result .= "\r\n"; + + foreach ($this->headers as $name => $values) { + foreach ($values as $value) { + $result .= "{$name}: {$value}\r\n"; + } + } + + return $result . "\r\n"; + } + + public static function fromWire(string $raw): self + { + $headers = new self(); + + // explode() never returns an empty array -- on '' it returns [''] -- so + // testing the result could not reject an empty block, and one fell through + // to the version check below and was reported as an invalid version. + if ($raw === '') { + throw new ProtocolException('Empty header block'); + } + + $lines = explode("\r\n", $raw); + + // Parse status line: "NATS/1.0" or "NATS/1.0 503" or "NATS/1.0 503 No Responders" + $statusLine = array_shift($lines); + if (!str_starts_with($statusLine, 'NATS/1.0')) { + throw new ProtocolException("Invalid header version: {$statusLine}"); + } + + $remainder = trim(substr($statusLine, 8)); + if ($remainder !== '') { + $spacePos = strpos($remainder, ' '); + if ($spacePos !== false) { + $headers->status = substr($remainder, 0, $spacePos); + $headers->description = substr($remainder, $spacePos + 1); + } else { + $headers->status = $remainder; + } + } + + // Parse header lines + foreach ($lines as $line) { + if ($line === '') { + continue; + } + + $colonPos = strpos($line, ':'); + if ($colonPos === false) { + continue; + } + + $name = substr($line, 0, $colonPos); + $value = ltrim(substr($line, $colonPos + 1)); + $headers->headers[$name][] = $value; + } + + return $headers; + } + + public function getIterator(): \ArrayIterator + { + return new \ArrayIterator($this->headers); + } + + public function count(): int + { + return \count($this->headers); + } +} diff --git a/packages/nats/src/Inbox.php b/packages/nats/src/Inbox.php new file mode 100644 index 00000000000..97c7c3ce85d --- /dev/null +++ b/packages/nats/src/Inbox.php @@ -0,0 +1,29 @@ +getMessage(), previous: $cause); + } +} diff --git a/packages/nats/src/JetStream/AccountInfo.php b/packages/nats/src/JetStream/AccountInfo.php new file mode 100644 index 00000000000..a67871ca770 --- /dev/null +++ b/packages/nats/src/JetStream/AccountInfo.php @@ -0,0 +1,40 @@ + $limits + * @param array $raw + */ + public function __construct( + public readonly int $memory, + public readonly int $storage, + public readonly int $streams, + public readonly int $consumers, + public readonly array $limits, + public readonly int $apiTotal, + public readonly int $apiErrors, + public readonly ?string $domain = null, + public readonly array $raw = [], + ) { + } + + public static function fromArray(array $data): self + { + return new self( + memory: $data['memory'] ?? 0, + storage: $data['storage'] ?? 0, + streams: $data['streams'] ?? 0, + consumers: $data['consumers'] ?? 0, + limits: $data['limits'] ?? [], + apiTotal: $data['api']['total'] ?? 0, + apiErrors: $data['api']['errors'] ?? 0, + domain: $data['domain'] ?? null, + raw: $data, + ); + } +} diff --git a/packages/nats/src/JetStream/AckPolicy.php b/packages/nats/src/JetStream/AckPolicy.php new file mode 100644 index 00000000000..caba9de82f0 --- /dev/null +++ b/packages/nats/src/JetStream/AckPolicy.php @@ -0,0 +1,12 @@ +apiPrefix}.CONSUMER.MSG.NEXT.{$this->stream}.{$this->getName()}"; + + $request = ['batch' => $batch]; + if ($noWait) { + // No 'expires' alongside it. A pull request that carries an expiry + // waits the whole window and then answers 408 Request Timeout, so + // no_wait does nothing at all; sent on its own it comes back 404 No + // Messages immediately, which is the only reason to ask for it. A + // caller polling an empty consumer therefore paid the full timeout + // per call -- 0.25s of poll is a ceiling of four calls a second. + $request['no_wait'] = true; + } else { + // Expire the server's pull request just before the client stops + // waiting. On an identical deadline a message the server dispatches + // at the boundary is dropped here while the server has already + // counted the delivery, burning an attempt against maxDeliver for + // nothing. + $serverExpiry = $timeout - min(self::FETCH_EXPIRY_MARGIN, $timeout * 0.1); + $request['expires'] = StreamConfig::secondsToNanos($serverExpiry); + } + if ($maxBytes !== null) { + $request['max_bytes'] = $maxBytes; + } + + $payload = json_encode($request, JSON_THROW_ON_ERROR); + + $inbox = $this->conn->newInbox(); + $sub = $this->conn->subscribe($inbox); + + $messageBatch = new MessageBatch($this->conn); + + try { + $this->conn->publish($requestSubject, $payload, $inbox); + $this->collectBatch($sub, $messageBatch, $batch, $timeout); + } finally { + // Outside a finally this leaked the inbox subscription on any throw, + // and attemptReconnect() re-subscribes every leaked sid on each + // reconnect -- so the leak compounds across a reconnect storm. + $sub->unsubscribe(); + } + + return $messageBatch; + } + + /** + * Gather up to $batch messages for a pull request, stopping on the server's + * terminal status messages or when the client deadline passes. + */ + private function collectBatch( + Subscription $sub, + MessageBatch $messageBatch, + int $batch, + float $timeout, + ): void { + $deadline = microtime(true) + $timeout; + + while (\count($messageBatch) < $batch) { + $remaining = $deadline - microtime(true); + if ($remaining <= 0) { + break; + } + + $msg = $sub->nextMessage($remaining); + if (!$msg instanceof \Utopia\NATS\Message) { + break; + } + + if ($msg->headers instanceof \Utopia\NATS\Headers) { + $status = $msg->headers->getStatus(); + + // 100 = flow control / idle heartbeat: keep-alive, not data, and + // the only status that means "carry on waiting". + if ($status === '100') { + if ($msg->replyTo !== null && $msg->replyTo !== '') { + $this->conn->publish($msg->replyTo, ''); + } + continue; + } + + // Every other status ends this pull request. Naming the expected + // codes (404/408/409) and falling through on the rest handed the + // unnamed ones back as data: a 503 No Responders -- which the + // server sends while a consumer is still being created or a raft + // leader is moving -- became a message with an empty body, and a + // caller that json_decode()s the payload got null and died on it. + // A status frame carries no payload whatever its number, so it can + // never be a message. + if ($status !== '') { + break; + } + } + + $messageBatch->addMessage($msg); + } + } + + /** + * Fetch the next single message. + */ + public function next(?float $timeout = null): ?JetStreamMessage + { + $batch = $this->fetch(1, $timeout); + $messages = $batch->getMessages(); + return $messages[0] ?? null; + } + + public function info(bool $refresh = false): ConsumerInfo + { + if ($refresh) { + $subject = "{$this->apiPrefix}.CONSUMER.INFO.{$this->stream}.{$this->getName()}"; + try { + $response = $this->conn->request($subject); + $data = json_decode($response->data, true, 512, JSON_THROW_ON_ERROR); + JetStream::checkError($data); + $this->info = ConsumerInfo::fromArray($data); + } catch (TimeoutException) { + // Return cached info on timeout + } + } + return $this->info; + } + + public function getName(): string + { + return $this->info->name; + } + + public function getStream(): string + { + return $this->stream; + } +} diff --git a/packages/nats/src/JetStream/ConsumerConfig.php b/packages/nats/src/JetStream/ConsumerConfig.php new file mode 100644 index 00000000000..39342bfdf9f --- /dev/null +++ b/packages/nats/src/JetStream/ConsumerConfig.php @@ -0,0 +1,157 @@ +|null $filterSubjects + * @param array|null $metadata Arbitrary key-value metadata (ADR-33) + * @param list|null $backoff Redelivery delays in seconds, one per attempt + * (the last entry repeats). Server rules, not sanitized here: when set + * together with $ackWait the first entry must equal $ackWait, and + * $maxDeliver must exceed the number of entries. + */ + public function __construct( + public readonly ?string $name = null, + public readonly ?string $durableName = null, + public readonly ?string $description = null, + public readonly DeliverPolicy $deliverPolicy = DeliverPolicy::All, + public readonly AckPolicy $ackPolicy = AckPolicy::Explicit, + public readonly ?float $ackWait = null, + public readonly ?int $maxDeliver = null, + public readonly ?string $filterSubject = null, + public readonly ?array $filterSubjects = null, + public readonly ReplayPolicy $replayPolicy = ReplayPolicy::Instant, + public readonly ?int $maxWaiting = null, + public readonly ?int $maxAckPending = null, + public readonly ?float $inactiveThreshold = null, + public readonly ?int $optStartSeq = null, + public readonly ?string $optStartTime = null, + public readonly ?int $maxBatch = null, + public readonly ?int $maxBytes = null, + public readonly bool $memStorage = false, + public readonly ?int $numReplicas = null, + public readonly ?string $deliverSubject = null, + public readonly ?string $deliverGroup = null, + public readonly bool $flowControl = false, + public readonly ?float $idleHeartbeat = null, + public readonly ?array $metadata = null, + public readonly ?array $backoff = null, + ) { + } + + public function toArray(): array + { + $data = [ + 'deliver_policy' => $this->deliverPolicy->value, + 'ack_policy' => $this->ackPolicy->value, + 'replay_policy' => $this->replayPolicy->value, + ]; + + if ($this->name !== null) { + $data['name'] = $this->name; + } + if ($this->durableName !== null) { + $data['durable_name'] = $this->durableName; + } + if ($this->description !== null) { + $data['description'] = $this->description; + } + if ($this->ackWait !== null) { + $data['ack_wait'] = StreamConfig::secondsToNanos($this->ackWait); + } + if ($this->maxDeliver !== null) { + $data['max_deliver'] = $this->maxDeliver; + } + if ($this->filterSubject !== null) { + $data['filter_subject'] = $this->filterSubject; + } + if ($this->filterSubjects !== null) { + $data['filter_subjects'] = $this->filterSubjects; + } + if ($this->maxWaiting !== null) { + $data['max_waiting'] = $this->maxWaiting; + } + if ($this->maxAckPending !== null) { + $data['max_ack_pending'] = $this->maxAckPending; + } + if ($this->inactiveThreshold !== null) { + $data['inactive_threshold'] = StreamConfig::secondsToNanos($this->inactiveThreshold); + } + if ($this->optStartSeq !== null) { + $data['opt_start_seq'] = $this->optStartSeq; + } + if ($this->optStartTime !== null) { + $data['opt_start_time'] = $this->optStartTime; + } + if ($this->maxBatch !== null) { + $data['max_batch'] = $this->maxBatch; + } + if ($this->maxBytes !== null) { + $data['max_bytes'] = $this->maxBytes; + } + if ($this->memStorage) { + $data['mem_storage'] = true; + } + if ($this->numReplicas !== null) { + $data['num_replicas'] = $this->numReplicas; + } + if ($this->deliverSubject !== null) { + $data['deliver_subject'] = $this->deliverSubject; + } + if ($this->deliverGroup !== null) { + $data['deliver_group'] = $this->deliverGroup; + } + if ($this->flowControl) { + $data['flow_control'] = true; + } + if ($this->idleHeartbeat !== null) { + $data['idle_heartbeat'] = StreamConfig::secondsToNanos($this->idleHeartbeat); + } + if ($this->metadata !== null) { + $data['metadata'] = $this->metadata; + } + if ($this->backoff !== null) { + $data['backoff'] = array_map(StreamConfig::secondsToNanos(...), $this->backoff); + } + + return $data; + } + + public static function fromArray(array $data): self + { + return new self( + name: $data['name'] ?? null, + durableName: $data['durable_name'] ?? null, + description: $data['description'] ?? null, + deliverPolicy: DeliverPolicy::tryFrom($data['deliver_policy'] ?? '') ?? DeliverPolicy::All, + ackPolicy: AckPolicy::tryFrom($data['ack_policy'] ?? '') ?? AckPolicy::Explicit, + ackWait: isset($data['ack_wait']) ? StreamConfig::nanosToSeconds($data['ack_wait']) : null, + maxDeliver: $data['max_deliver'] ?? null, + filterSubject: $data['filter_subject'] ?? null, + filterSubjects: $data['filter_subjects'] ?? null, + replayPolicy: ReplayPolicy::tryFrom($data['replay_policy'] ?? '') ?? ReplayPolicy::Instant, + maxWaiting: $data['max_waiting'] ?? null, + maxAckPending: $data['max_ack_pending'] ?? null, + inactiveThreshold: isset($data['inactive_threshold']) ? StreamConfig::nanosToSeconds($data['inactive_threshold']) : null, + optStartSeq: $data['opt_start_seq'] ?? null, + optStartTime: $data['opt_start_time'] ?? null, + maxBatch: $data['max_batch'] ?? null, + maxBytes: $data['max_bytes'] ?? null, + memStorage: $data['mem_storage'] ?? false, + numReplicas: $data['num_replicas'] ?? null, + deliverSubject: $data['deliver_subject'] ?? null, + deliverGroup: $data['deliver_group'] ?? null, + flowControl: $data['flow_control'] ?? false, + idleHeartbeat: isset($data['idle_heartbeat']) ? StreamConfig::nanosToSeconds($data['idle_heartbeat']) : null, + metadata: $data['metadata'] ?? null, + backoff: isset($data['backoff']) ? array_map(StreamConfig::nanosToSeconds(...), $data['backoff']) : null, + ); + } +} diff --git a/packages/nats/src/JetStream/ConsumerInfo.php b/packages/nats/src/JetStream/ConsumerInfo.php new file mode 100644 index 00000000000..fbaf5cdeead --- /dev/null +++ b/packages/nats/src/JetStream/ConsumerInfo.php @@ -0,0 +1,49 @@ +|null $metadata Consumer metadata (ADR-33) + */ + public function __construct( + public readonly string $streamName, + public readonly string $name, + public readonly ConsumerConfig $config, + public readonly string $created, + public readonly int $numAckPending, + public readonly int $numRedelivered, + public readonly int $numWaiting, + public readonly int $numPending, + public readonly SequenceInfo $delivered, + public readonly SequenceInfo $ackFloor, + public readonly bool $pushBound = false, + public readonly ?string $cluster = null, + public readonly ?array $metadata = null, + ) { + } + + public static function fromArray(array $data): self + { + $config = ConsumerConfig::fromArray($data['config'] ?? []); + + return new self( + streamName: $data['stream_name'] ?? '', + name: $data['name'] ?? '', + config: $config, + created: $data['created'] ?? '', + numAckPending: $data['num_ack_pending'] ?? 0, + numRedelivered: $data['num_redelivered'] ?? 0, + numWaiting: $data['num_waiting'] ?? 0, + numPending: $data['num_pending'] ?? 0, + delivered: SequenceInfo::fromArray($data['delivered'] ?? []), + ackFloor: SequenceInfo::fromArray($data['ack_floor'] ?? []), + pushBound: $data['push_bound'] ?? false, + cluster: isset($data['cluster']['name']) ? (string) $data['cluster']['name'] : null, + metadata: $config->metadata, + ); + } +} diff --git a/packages/nats/src/JetStream/ConsumerLimits.php b/packages/nats/src/JetStream/ConsumerLimits.php new file mode 100644 index 00000000000..b24bdd74752 --- /dev/null +++ b/packages/nats/src/JetStream/ConsumerLimits.php @@ -0,0 +1,40 @@ +inactiveThreshold !== null) { + $data['inactive_threshold'] = StreamConfig::secondsToNanos($this->inactiveThreshold); + } + if ($this->maxAckPending !== null) { + $data['max_ack_pending'] = $this->maxAckPending; + } + + return $data; + } + + public static function fromArray(array $data): self + { + return new self( + inactiveThreshold: isset($data['inactive_threshold']) + ? StreamConfig::nanosToSeconds($data['inactive_threshold']) + : null, + maxAckPending: $data['max_ack_pending'] ?? null, + ); + } +} diff --git a/packages/nats/src/JetStream/DeliverPolicy.php b/packages/nats/src/JetStream/DeliverPolicy.php new file mode 100644 index 00000000000..7fe33d73488 --- /dev/null +++ b/packages/nats/src/JetStream/DeliverPolicy.php @@ -0,0 +1,15 @@ + + */ + public function toArray(): array + { + $data = ['api' => $this->api]; + if ($this->deliver !== null) { + $data['deliver'] = $this->deliver; + } + + return $data; + } + + public static function fromArray(array $data): self + { + return new self( + api: $data['api'] ?? '', + deliver: $data['deliver'] ?? null, + ); + } +} diff --git a/packages/nats/src/JetStream/JetStream.php b/packages/nats/src/JetStream/JetStream.php new file mode 100644 index 00000000000..781c1195cd8 --- /dev/null +++ b/packages/nats/src/JetStream/JetStream.php @@ -0,0 +1,514 @@ +apiPrefix = $apiPrefix; + } elseif ($domain !== null) { + $this->apiPrefix = "\$JS.{$domain}.API"; + } else { + $this->apiPrefix = '$JS.API'; + } + } + + /** + * Confirm acknowledgements for the selected messages using the consumer policy. + * AckPolicy::Explicit is required to leave unselected messages unacknowledged. + * With AckPolicy::All, acknowledging a later message also acknowledges earlier ones. + * @param list $messages + * @param \Closure(int, ?\Throwable): void $confirmed Null means server-confirmed. + */ + public function ackBatch(array $messages, \Closure $confirmed, ?float $timeout = null): void + { + $requests = array_map(static fn (JetStreamMessage $message): Request => new Request( + subject: $message->message->replyTo ?? throw new \RuntimeException('Cannot acknowledge: message has no reply subject'), + ), $messages); + $this->conn->requestBatch($requests, static function (int $index, Message|\Throwable $result) use ($confirmed): void { + $confirmed($index, $result instanceof \Throwable ? $result : null); + }, $timeout ?? 5.0); + } + + // --- Stream Management --- + + public function createStream(StreamConfig $config): Stream + { + $data = $this->apiRequest("STREAM.CREATE.{$config->name}", $config->toArray()); + return new Stream($this, StreamInfo::fromArray($data)); + } + + public function updateStream(StreamConfig $config): Stream + { + $data = $this->apiRequest("STREAM.UPDATE.{$config->name}", $config->toArray()); + return new Stream($this, StreamInfo::fromArray($data)); + } + + public function createOrUpdateStream(StreamConfig $config): Stream + { + try { + return $this->updateStream($config); + } catch (JetStreamException $e) { + if ($e->apiError instanceof \Utopia\NATS\JetStream\ApiError && $e->apiError->code === 404) { + return $this->createStream($config); + } + throw $e; + } + } + + public function deleteStream(string $name): void + { + $this->apiRequest("STREAM.DELETE.{$name}"); + } + + public function getStream(string $name): Stream + { + $info = $this->getStreamInfo($name); + return new Stream($this, $info); + } + + public function getStreamInfo(string $name): StreamInfo + { + $data = $this->apiRequest("STREAM.INFO.{$name}"); + return StreamInfo::fromArray($data); + } + + /** @return list */ + public function getStreamNames(?string $subject = null): array + { + $payload = $subject !== null ? ['subject' => $subject] : null; + $data = $this->apiRequest('STREAM.NAMES', $payload); + return $data['streams'] ?? []; + } + + /** @return list */ + public function listStreams(?string $subject = null): array + { + $payload = $subject !== null ? ['subject' => $subject] : null; + $data = $this->apiRequest('STREAM.LIST', $payload); + return array_map( + StreamInfo::fromArray(...), + $data['streams'] ?? [], + ); + } + + public function purgeStream(string $name, ?string $subject = null): void + { + $payload = $subject !== null ? ['filter' => $subject] : null; + $this->apiRequest("STREAM.PURGE.{$name}", $payload); + } + + // --- Consumer Management --- + + public function createConsumer(string $stream, ConsumerConfig $config): Consumer + { + $consumerName = $config->name ?? $config->durableName; + + $subject = $consumerName !== null ? "CONSUMER.CREATE.{$stream}.{$consumerName}" : "CONSUMER.CREATE.{$stream}"; + + $payload = [ + 'stream_name' => $stream, + 'config' => $config->toArray(), + ]; + + $data = $this->apiRequest($subject, $payload); + return new Consumer($this->conn, $stream, ConsumerInfo::fromArray($data), $this->apiPrefix); + } + + public function updateConsumer(string $stream, ConsumerConfig $config): Consumer + { + return $this->createConsumer($stream, $config); + } + + public function deleteConsumer(string $stream, string $consumer): void + { + $this->apiRequest("CONSUMER.DELETE.{$stream}.{$consumer}"); + } + + public function getConsumer(string $stream, string $consumer): Consumer + { + $data = $this->apiRequest("CONSUMER.INFO.{$stream}.{$consumer}"); + return new Consumer($this->conn, $stream, ConsumerInfo::fromArray($data), $this->apiPrefix); + } + + /** @return list */ + public function getConsumerNames(string $stream): array + { + $data = $this->apiRequest("CONSUMER.NAMES.{$stream}"); + return $data['consumers'] ?? []; + } + + /** + * List consumers of a stream as typed ConsumerInfo objects, following the + * API's offset/total paging until every consumer has been retrieved. + * + * @return list + */ + public function getConsumers(string $stream): array + { + $consumers = []; + $offset = 0; + + do { + $data = $this->apiRequest("CONSUMER.LIST.{$stream}", ['offset' => $offset]); + $page = $data['consumers'] ?? []; + foreach ($page as $entry) { + $consumers[] = ConsumerInfo::fromArray($entry); + } + $total = $data['total'] ?? \count($consumers); + $offset = \count($consumers); + } while ($page !== [] && $offset < $total); + + return $consumers; + } + + /** + * Create a push consumer and deliver its messages to the callback. + * + * If the config has no deliver subject, an inbox is generated. The callback + * receives a JetStreamMessage and may ack it. Drive delivery with + * Connection::wait(). + */ + public function pushSubscribe(string $stream, ConsumerConfig $config, \Closure $callback): PushSubscription + { + if ($config->deliverSubject === null) { + $config = ConsumerConfig::fromArray( + ['deliver_subject' => $this->conn->newInbox()] + $config->toArray(), + ); + } + + $consumer = $this->createConsumer($stream, $config); + + return new PushSubscription($this->conn, $consumer->info(), $callback); + } + + /** + * Create an ordered (ephemeral, in-order, auto-healing) push consumer. + */ + public function orderedConsumer(string $stream, DeliverPolicy $deliverPolicy = DeliverPolicy::All, ?string $filterSubject = null, float $idleHeartbeat = 5.0): OrderedConsumer + { + return new OrderedConsumer($this->conn, $this, $stream, $deliverPolicy, $filterSubject, $idleHeartbeat); + } + + // --- Stream Message Operations --- + + public function getMessage(string $stream, int $seq): StreamMessage + { + $data = $this->apiRequest("STREAM.MSG.GET.{$stream}", ['seq' => $seq]); + return StreamMessage::fromArray($data['message'] ?? []); + } + + public function getLastMessage(string $stream, string $subject): StreamMessage + { + $data = $this->apiRequest("STREAM.MSG.GET.{$stream}", ['last_by_subj' => $subject]); + return StreamMessage::fromArray($data['message'] ?? []); + } + + public function deleteMessage(string $stream, int $seq, bool $noErase = false): void + { + $payload = ['seq' => $seq]; + if ($noErase) { + $payload['no_erase'] = true; + } + $this->apiRequest("STREAM.MSG.DELETE.{$stream}", $payload); + } + + // --- Publishing --- + + /** + * @param int|string|null $ttl Per-message TTL (ADR-43). An int is interpreted as + * seconds; a string is passed through as a duration + * (e.g. "1s", "never"). Sets the Nats-TTL header. + * @param int $retryOnNoResponders Number of extra attempts if the publish is met + * with no responders / 503 (ADR-22). 0 keeps the + * default single-attempt behavior. + */ + public function publish(string $subject, string $data = '', ?Headers $headers = null, ?string $msgId = null, ?string $expectedLastMsgId = null, ?int $expectedLastSeq = null, ?int $expectedLastSubjectSeq = null, ?string $expectedStream = null, int|string|null $ttl = null, int $retryOnNoResponders = 0): PubAck + { + $useHeaders = $this->publishHeaders( + $headers, + $msgId, + $expectedLastMsgId, + $expectedLastSeq, + $expectedLastSubjectSeq, + $expectedStream, + $ttl, + ); + + $attempt = 0; + while (true) { + try { + $response = $this->conn->request($subject, $data, headers: $useHeaders); + break; + } catch (\Utopia\NATS\Exception\NatsException $e) { + if ($attempt >= $retryOnNoResponders || $e->getMessage() !== 'No responders for request') { + throw $e; + } + $attempt++; + usleep(50_000 * $attempt); + } + } + + $responseData = json_decode($response->data, true, 512, JSON_THROW_ON_ERROR); + self::checkError($responseData); + + return PubAck::fromArray($responseData); + } + + /** + * Publish many messages and collect an acknowledgment for each. + * + * publish() pays a round trip per message: it writes one PUB, then blocks on the + * reply before writing the next, so a batch of N messages costs N round trips. + * This writes a window of PUBs first and reads their acknowledgments afterwards, + * which costs one round trip per window. Each message still carries its own reply + * subject and still gets its own acknowledgment, so per-message deduplication and + * per-message errors behave exactly as they do on the single publish. + * + * @param list $messages + * @param float|null $timeout Deadline for one window's acknowledgments, in seconds. + * Defaults to the connection's request timeout. + * @param int $window How many messages may be in flight before their acknowledgments + * are collected. This bounds the reply burst the server sends back + * and keeps it under the subscription's pending limit. + * @return list One entry per message, in the order the messages were given. + */ + public function publishMany(array $messages, ?float $timeout = null, int $window = 256): array + { + if ($messages === []) { + return []; + } + + if ($window < 1) { + throw new \InvalidArgumentException("Publish window must be at least 1, got {$window}"); + } + + $acks = []; + + // Preserved keys carry the caller's index through the windows, so an + // acknowledgment can be placed back where its message came from. + foreach (array_chunk($messages, $window, true) as $chunk) { + foreach ($this->publishWindow($chunk, $timeout) as $index => $ack) { + $acks[$index] = $ack; + } + } + + ksort($acks); + + return array_values($acks); + } + + /** + * Write one window of PUBs, then read back an acknowledgment for every one of them. + * + * @param array> $chunk keyed by the caller's message index + * @return array keyed by the same index + */ + private function publishWindow(array $chunk, ?float $timeout): array + { + $options = $this->conn->getOptions(); + + // Every message replies to its own subject beneath a shared prefix. The server + // acknowledges in whatever order it durably stores the messages, so the trailing + // token is what ties an acknowledgment back to the message that asked for it. + $inbox = Inbox::create($options->inboxPrefix); + $sub = $this->conn->subscribe($inbox . '.*'); + + try { + foreach ($chunk as $index => $message) { + /** @var array{subject: string, data?: string, headers?: Headers, msgId?: string, expectedLastMsgId?: string, expectedLastSeq?: int, expectedLastSubjectSeq?: int, expectedStream?: string, ttl?: int|string} $message */ + $this->conn->publish( + $message['subject'], + $message['data'] ?? '', + $inbox . '.' . $index, + $this->publishHeaders( + $message['headers'] ?? null, + $message['msgId'] ?? null, + $message['expectedLastMsgId'] ?? null, + $message['expectedLastSeq'] ?? null, + $message['expectedLastSubjectSeq'] ?? null, + $message['expectedStream'] ?? null, + $message['ttl'] ?? null, + ), + ); + } + + $deadline = microtime(true) + ($timeout ?? $options->requestTimeout); + $acks = []; + + while (\count($acks) < \count($chunk)) { + $remaining = $deadline - microtime(true); + if ($remaining <= 0) { + break; + } + + $reply = $sub->nextMessage($remaining); + if (!$reply instanceof Message) { + break; + } + + // 503 with no body: JetStream is not answering on this account at all, + // which is the same condition publish() surfaces as "No responders". + if ($reply->headers instanceof Headers && $reply->headers->getStatus() === '503') { + throw new NatsException('No responders for request'); + } + + $responseData = json_decode($reply->data, true, 512, JSON_THROW_ON_ERROR); + self::checkError($responseData); + + $acks[(int) substr($reply->subject, (int) strrpos($reply->subject, '.') + 1)] = PubAck::fromArray($responseData); + } + + if (\count($acks) < \count($chunk)) { + throw new TimeoutException(\sprintf( + 'Timed out waiting for JetStream publish acknowledgments: received %d of %d', + \count($acks), + \count($chunk), + )); + } + + return $acks; + } finally { + $this->conn->unsubscribe($sub); + } + } + + /** + * Assemble the JetStream publish headers, so the single and batched publish paths + * cannot drift on a header name or on how a value is written to the wire. + */ + private function publishHeaders( + ?Headers $headers, + ?string $msgId, + ?string $expectedLastMsgId, + ?int $expectedLastSeq, + ?int $expectedLastSubjectSeq, + ?string $expectedStream, + int|string|null $ttl, + ): ?Headers { + $headers ??= new Headers(); + + if ($msgId !== null) { + $headers->set('Nats-Msg-Id', $msgId); + } + if ($expectedLastMsgId !== null) { + $headers->set('Nats-Expected-Last-Msg-Id', $expectedLastMsgId); + } + if ($expectedLastSeq !== null) { + $headers->set('Nats-Expected-Last-Sequence', (string) $expectedLastSeq); + } + if ($expectedLastSubjectSeq !== null) { + $headers->set('Nats-Expected-Last-Subject-Sequence', (string) $expectedLastSubjectSeq); + } + if ($expectedStream !== null) { + $headers->set('Nats-Expected-Stream', $expectedStream); + } + if ($ttl !== null) { + $headers->set('Nats-TTL', \is_int($ttl) ? "{$ttl}s" : $ttl); + } + + return \count($headers) > 0 ? $headers : null; + } + + // --- Key-Value --- + + public function createKeyValue(KeyValueConfig $config): KeyValue + { + $streamConfig = $config->toStreamConfig(); + $this->createOrUpdateStream($streamConfig); + return new KeyValue($this->conn, $this, $config->bucket); + } + + public function getKeyValue(string $bucket): KeyValue + { + // Verify the KV stream exists + $this->getStreamInfo("KV_{$bucket}"); + return new KeyValue($this->conn, $this, $bucket); + } + + public function deleteKeyValue(string $bucket): void + { + $this->deleteStream("KV_{$bucket}"); + } + + // --- Object Store --- + + public function getObjectStore(string $bucket): \Utopia\NATS\ObjectStore\ObjectStore + { + // Verify the object-store stream exists. + $this->getStreamInfo("OBJ_{$bucket}"); + return new \Utopia\NATS\ObjectStore\ObjectStore($this->conn, $this, $bucket); + } + + public function deleteObjectStore(string $bucket): void + { + $this->deleteStream("OBJ_{$bucket}"); + } + + // --- Account Info --- + + public function accountInfo(): AccountInfo + { + return AccountInfo::fromArray($this->apiRequest('INFO')); + } + + // --- Internal --- + + /** + * @return array + */ + private function apiRequest(string $subject, ?array $payload = null, ?float $timeout = null): array + { + $fullSubject = "{$this->apiPrefix}.{$subject}"; + $body = $payload !== null ? json_encode($payload, JSON_THROW_ON_ERROR) : ''; + + $response = $this->conn->request($fullSubject, $body, $timeout); + $data = json_decode($response->data, true, 512, JSON_THROW_ON_ERROR); + + self::checkError($data); + + return $data; + } + + /** + * @throws JetStreamException + */ + public static function checkError(array $data): void + { + if (isset($data['error'])) { + $error = ApiError::fromArray($data['error']); + throw new JetStreamException( + $error->description, + $error->code, + apiError: $error, + ); + } + } +} diff --git a/packages/nats/src/JetStream/JetStreamMessage.php b/packages/nats/src/JetStream/JetStreamMessage.php new file mode 100644 index 00000000000..3ddcdfa50a4 --- /dev/null +++ b/packages/nats/src/JetStream/JetStreamMessage.php @@ -0,0 +1,98 @@ +respond(''); + } + + /** + * Acknowledge and wait for the server to confirm the ack was persisted. + * Unlike ack(), this blocks until confirmation or throws on timeout. + */ + public function ackSync(?float $timeout = null): void + { + if ($this->message->replyTo === null) { + throw new \RuntimeException('Cannot acknowledge: message has no reply subject'); + } + + $this->conn->request($this->message->replyTo, '', $timeout ?? 5.0); + } + + public function nak(?float $delay = null): void + { + if ($delay !== null) { + $nanos = StreamConfig::secondsToNanos($delay); + $this->respond("-NAK {\"delay\":{$nanos}}"); + } else { + $this->respond('-NAK'); + } + } + + public function inProgress(): void + { + $this->respond('+WPI'); + } + + public function term(?string $reason = null): void + { + if ($reason !== null) { + $this->respond("+TERM {$reason}"); + } else { + $this->respond('+TERM'); + } + } + + public function metadata(): MsgMetadata + { + if (!$this->metadata instanceof \Utopia\NATS\JetStream\MsgMetadata) { + if ($this->message->replyTo === null) { + throw new \RuntimeException('Message has no reply subject for metadata parsing'); + } + $this->metadata = MsgMetadata::fromReplySubject($this->message->replyTo); + } + + return $this->metadata; + } + + public function getData(): string + { + return $this->message->data; + } + + public function getSubject(): string + { + return $this->message->subject; + } + + public function getHeaders(): ?Headers + { + return $this->message->headers; + } + + private function respond(string $data): void + { + if ($this->message->replyTo === null) { + throw new \RuntimeException('Cannot acknowledge: message has no reply subject'); + } + + $this->conn->publish($this->message->replyTo, $data); + } +} diff --git a/packages/nats/src/JetStream/MessageBatch.php b/packages/nats/src/JetStream/MessageBatch.php new file mode 100644 index 00000000000..64fa2f8d31f --- /dev/null +++ b/packages/nats/src/JetStream/MessageBatch.php @@ -0,0 +1,40 @@ + */ + private array $messages = []; + + public function __construct( + private readonly Connection $conn, + ) { + } + + public function addMessage(Message $msg): void + { + $this->messages[] = new JetStreamMessage($this->conn, $msg); + } + + /** @return list */ + public function getMessages(): array + { + return $this->messages; + } + + public function getIterator(): \ArrayIterator + { + return new \ArrayIterator($this->messages); + } + + public function count(): int + { + return \count($this->messages); + } +} diff --git a/packages/nats/src/JetStream/MsgMetadata.php b/packages/nats/src/JetStream/MsgMetadata.php new file mode 100644 index 00000000000..2f026a69a27 --- /dev/null +++ b/packages/nats/src/JetStream/MsgMetadata.php @@ -0,0 +1,60 @@ +...... + * Or with domain: $JS.ACK......... + */ + public static function fromReplySubject(string $reply): self + { + $parts = explode('.', $reply); + + // Standard format: $JS.ACK....... + if (\count($parts) >= 9 && $parts[0] === '$JS' && $parts[1] === 'ACK') { + if (\count($parts) === 9) { + return new self( + stream: $parts[2], + consumer: $parts[3], + numDelivered: (int) $parts[4], + streamSequence: (int) $parts[5], + consumerSequence: (int) $parts[6], + timestamp: $parts[7], + numPending: (int) $parts[8], + ); + } + + // Domain format has extra parts + if (\count($parts) >= 11) { + return new self( + stream: $parts[4], + consumer: $parts[5], + numDelivered: (int) $parts[6], + streamSequence: (int) $parts[7], + consumerSequence: (int) $parts[8], + timestamp: $parts[9], + numPending: (int) $parts[10], + domain: $parts[2], + ); + } + } + + throw new \InvalidArgumentException("Cannot parse JetStream reply subject: {$reply}"); + } +} diff --git a/packages/nats/src/JetStream/OrderedConsumer.php b/packages/nats/src/JetStream/OrderedConsumer.php new file mode 100644 index 00000000000..244bc67b081 --- /dev/null +++ b/packages/nats/src/JetStream/OrderedConsumer.php @@ -0,0 +1,130 @@ +create(null); + } + + /** + * Return the next message in order, or null on timeout. Flow-control and + * heartbeat frames are handled internally; on a detected gap the consumer + * is recreated and iteration continues seamlessly. + */ + public function next(?float $timeout = null): ?JetStreamMessage + { + $timeout ??= 5.0; + $deadline = microtime(true) + $timeout; + + while (true) { + $remaining = $deadline - microtime(true); + if ($remaining <= 0) { + return null; + } + + $msg = $this->sub->nextMessage($remaining); + if (!$msg instanceof \Utopia\NATS\Message) { + return null; + } + + if (PushSubscription::handleControl($this->conn, $msg)) { + // A heartbeat reports the last consumer sequence the server has + // delivered; if it is ahead of us we missed messages. + $lastConsumer = $msg->headers?->get('Nats-Last-Consumer'); + if ($lastConsumer !== null && (int) $lastConsumer > $this->expectedConsumerSeq - 1) { + $this->reset($this->lastStreamSeq + 1); + } + continue; + } + + $jsMsg = new JetStreamMessage($this->conn, $msg); + $meta = $jsMsg->metadata(); + + if ($meta->consumerSequence !== $this->expectedConsumerSeq) { + // Gap: recreate from the message after the last good one. + $this->reset($this->lastStreamSeq + 1); + continue; + } + + $this->expectedConsumerSeq++; + $this->lastStreamSeq = $meta->streamSequence; + + return $jsMsg; + } + } + + public function getConsumerName(): string + { + return $this->info->name; + } + + public function stop(): void + { + $this->teardown(); + } + + private function reset(?int $startSeq): void + { + $this->teardown(); + $this->create($startSeq); + } + + private function teardown(): void + { + $this->sub->unsubscribe(); + try { + $this->js->deleteConsumer($this->stream, $this->info->name); + } catch (\Throwable) { + // Ephemeral consumer will expire on its own. + } + } + + private function create(?int $startSeq): void + { + // Subscribe before creating the consumer so no pushed message is missed. + $deliverSubject = $this->conn->newInbox(); + $this->sub = $this->conn->subscribe($deliverSubject); + + $config = new ConsumerConfig( + deliverPolicy: $startSeq !== null ? DeliverPolicy::ByStartSequence : $this->deliverPolicy, + ackPolicy: AckPolicy::None, + filterSubject: $this->filterSubject, + replayPolicy: ReplayPolicy::Instant, + inactiveThreshold: 30.0, + optStartSeq: $startSeq, + deliverSubject: $deliverSubject, + flowControl: true, + idleHeartbeat: $this->idleHeartbeat, + ); + + $this->info = $this->js->createConsumer($this->stream, $config)->info(); + $this->expectedConsumerSeq = 1; + } +} diff --git a/packages/nats/src/JetStream/Placement.php b/packages/nats/src/JetStream/Placement.php new file mode 100644 index 00000000000..09962dafce9 --- /dev/null +++ b/packages/nats/src/JetStream/Placement.php @@ -0,0 +1,38 @@ +|null $tags + */ + public function __construct( + public readonly ?string $cluster = null, + public readonly ?array $tags = null, + ) { + } + + public function toArray(): array + { + $data = []; + if ($this->cluster !== null) { + $data['cluster'] = $this->cluster; + } + if ($this->tags !== null) { + $data['tags'] = $this->tags; + } + + return $data; + } + + public static function fromArray(array $data): self + { + return new self( + cluster: $data['cluster'] ?? null, + tags: $data['tags'] ?? null, + ); + } +} diff --git a/packages/nats/src/JetStream/PubAck.php b/packages/nats/src/JetStream/PubAck.php new file mode 100644 index 00000000000..54c7c2add41 --- /dev/null +++ b/packages/nats/src/JetStream/PubAck.php @@ -0,0 +1,26 @@ +config->deliverSubject; + if ($deliverSubject === null) { + throw new \RuntimeException('Consumer has no deliver subject; not a push consumer'); + } + + $handler = function (Message $msg) use ($callback): void { + if (self::handleControl($this->conn, $msg)) { + return; + } + $callback(new JetStreamMessage($this->conn, $msg)); + }; + + $this->sub = $this->conn->subscribe($deliverSubject, $handler, $info->config->deliverGroup); + } + + /** + * Handle a JetStream control (status 100) message. Returns true when the + * message was a control frame and should not be surfaced as data. + */ + public static function handleControl(Connection $conn, Message $msg): bool + { + if (!$msg->headers instanceof \Utopia\NATS\Headers) { + return false; + } + if ($msg->headers->getStatus() !== '100') { + return false; + } + + // Flow control request: acknowledge by responding to the reply subject. + if ($msg->replyTo !== null && $msg->replyTo !== '') { + $conn->publish($msg->replyTo, ''); + return true; + } + + // Idle heartbeat: may carry a stalled indicator to nudge flow control. + $stalled = $msg->headers->get('Nats-Consumer-Stalled'); + if ($stalled !== null && $stalled !== '') { + $conn->publish($stalled, ''); + } + + return true; + } + + public function getSubscription(): Subscription + { + return $this->sub; + } + + public function getConsumerInfo(): ConsumerInfo + { + return $this->info; + } + + public function getConsumerName(): string + { + return $this->info->name; + } + + public function unsubscribe(): void + { + $this->sub->unsubscribe(); + } +} diff --git a/packages/nats/src/JetStream/ReplayPolicy.php b/packages/nats/src/JetStream/ReplayPolicy.php new file mode 100644 index 00000000000..3618124591a --- /dev/null +++ b/packages/nats/src/JetStream/ReplayPolicy.php @@ -0,0 +1,11 @@ + $this->source, + 'dest' => $this->destination, + ]; + if ($this->headersOnly) { + $data['headers_only'] = true; + } + + return $data; + } + + public static function fromArray(array $data): self + { + return new self( + source: $data['src'] ?? '', + destination: $data['dest'] ?? '', + headersOnly: $data['headers_only'] ?? false, + ); + } +} diff --git a/packages/nats/src/JetStream/RetentionPolicy.php b/packages/nats/src/JetStream/RetentionPolicy.php new file mode 100644 index 00000000000..146e43f122d --- /dev/null +++ b/packages/nats/src/JetStream/RetentionPolicy.php @@ -0,0 +1,12 @@ +info->config->name; + } + + public function getConfig(): StreamConfig + { + return $this->info->config; + } + + public function getState(): StreamState + { + return $this->info->state; + } + + public function info(bool $refresh = false): StreamInfo + { + if ($refresh) { + $this->info = $this->js->getStreamInfo($this->getName()); + } + return $this->info; + } + + public function createConsumer(ConsumerConfig $config): Consumer + { + return $this->js->createConsumer($this->getName(), $config); + } + + public function getConsumer(string $name): Consumer + { + return $this->js->getConsumer($this->getName(), $name); + } + + public function deleteConsumer(string $name): void + { + $this->js->deleteConsumer($this->getName(), $name); + } + + public function purge(?string $subject = null): void + { + $this->js->purgeStream($this->getName(), $subject); + } + + public function delete(): void + { + $this->js->deleteStream($this->getName()); + } +} diff --git a/packages/nats/src/JetStream/StreamConfig.php b/packages/nats/src/JetStream/StreamConfig.php new file mode 100644 index 00000000000..c2436d69d94 --- /dev/null +++ b/packages/nats/src/JetStream/StreamConfig.php @@ -0,0 +1,185 @@ + $subjects + * @param float|null $maxAge Max age in seconds (converted to nanoseconds for wire) + * @param float|null $duplicateWindow Duplicate window in seconds + * @param array|null $metadata Arbitrary key-value metadata (ADR-33) + * @param list|null $sources + * @param string|null $compression Compression algorithm: "none" or "s2" + * @param float|null $subjectDeleteMarkerTtl Delete-marker TTL in seconds (ADR-43) + */ + public function __construct( + public readonly string $name, + public readonly array $subjects = [], + public readonly ?string $description = null, + public readonly RetentionPolicy $retention = RetentionPolicy::Limits, + public readonly int $maxConsumers = -1, + public readonly int $maxMsgs = -1, + public readonly int $maxBytes = -1, + public readonly int $maxMsgsPerSubject = -1, + public readonly ?int $maxMsgSize = null, + public readonly ?float $maxAge = null, + public readonly StorageType $storage = StorageType::File, + public readonly int $replicas = 1, + public readonly DiscardPolicy $discard = DiscardPolicy::Old, + public readonly bool $noAck = false, + public readonly ?float $duplicateWindow = null, + public readonly bool $allowDirect = false, + public readonly bool $mirrorDirect = false, + public readonly bool $sealed = false, + public readonly bool $denyDelete = false, + public readonly bool $denyPurge = false, + public readonly bool $allowRollup = false, + public readonly ?array $metadata = null, + public readonly ?StreamSource $mirror = null, + public readonly ?array $sources = null, + public readonly ?Republish $republish = null, + public readonly ?SubjectTransform $subjectTransform = null, + public readonly ?Placement $placement = null, + public readonly ?string $compression = null, + public readonly ?int $firstSeq = null, + public readonly ?ConsumerLimits $consumerLimits = null, + public readonly bool $allowMsgTtl = false, + public readonly ?float $subjectDeleteMarkerTtl = null, + ) { + } + + public function toArray(): array + { + $data = [ + 'name' => $this->name, + 'retention' => $this->retention->value, + 'max_consumers' => $this->maxConsumers, + 'max_msgs' => $this->maxMsgs, + 'max_bytes' => $this->maxBytes, + 'max_msgs_per_subject' => $this->maxMsgsPerSubject, + 'storage' => $this->storage->value, + 'num_replicas' => $this->replicas, + 'discard' => $this->discard->value, + 'no_ack' => $this->noAck, + 'allow_direct' => $this->allowDirect, + 'mirror_direct' => $this->mirrorDirect, + 'sealed' => $this->sealed, + 'deny_delete' => $this->denyDelete, + 'deny_purge' => $this->denyPurge, + 'allow_rollup_hdrs' => $this->allowRollup, + ]; + + if ($this->subjects !== []) { + $data['subjects'] = $this->subjects; + } + if ($this->description !== null) { + $data['description'] = $this->description; + } + if ($this->maxMsgSize !== null) { + $data['max_msg_size'] = $this->maxMsgSize; + } + if ($this->maxAge !== null) { + $data['max_age'] = self::secondsToNanos($this->maxAge); + } + if ($this->duplicateWindow !== null) { + $data['duplicate_window'] = self::secondsToNanos($this->duplicateWindow); + } + if ($this->metadata !== null) { + $data['metadata'] = $this->metadata; + } + if ($this->mirror instanceof \Utopia\NATS\JetStream\StreamSource) { + $data['mirror'] = $this->mirror->toArray(); + } + if ($this->sources !== null) { + $data['sources'] = array_map( + static fn (StreamSource $s): array => $s->toArray(), + $this->sources, + ); + } + if ($this->republish instanceof \Utopia\NATS\JetStream\Republish) { + $data['republish'] = $this->republish->toArray(); + } + if ($this->subjectTransform instanceof \Utopia\NATS\JetStream\SubjectTransform) { + $data['subject_transform'] = $this->subjectTransform->toArray(); + } + if ($this->placement instanceof \Utopia\NATS\JetStream\Placement) { + $data['placement'] = $this->placement->toArray(); + } + if ($this->compression !== null) { + $data['compression'] = $this->compression; + } + if ($this->firstSeq !== null) { + $data['first_seq'] = $this->firstSeq; + } + if ($this->consumerLimits instanceof \Utopia\NATS\JetStream\ConsumerLimits) { + $data['consumer_limits'] = $this->consumerLimits->toArray(); + } + if ($this->allowMsgTtl) { + $data['allow_msg_ttl'] = true; + } + if ($this->subjectDeleteMarkerTtl !== null) { + $data['subject_delete_marker_ttl'] = self::secondsToNanos($this->subjectDeleteMarkerTtl); + } + + return $data; + } + + public static function fromArray(array $data): self + { + return new self( + name: $data['name'] ?? '', + subjects: $data['subjects'] ?? [], + description: $data['description'] ?? null, + retention: RetentionPolicy::tryFrom($data['retention'] ?? '') ?? RetentionPolicy::Limits, + maxConsumers: $data['max_consumers'] ?? -1, + maxMsgs: $data['max_msgs'] ?? -1, + maxBytes: $data['max_bytes'] ?? -1, + maxMsgsPerSubject: $data['max_msgs_per_subject'] ?? -1, + maxMsgSize: $data['max_msg_size'] ?? null, + maxAge: isset($data['max_age']) ? self::nanosToSeconds($data['max_age']) : null, + storage: StorageType::tryFrom($data['storage'] ?? '') ?? StorageType::File, + replicas: $data['num_replicas'] ?? 1, + discard: DiscardPolicy::tryFrom($data['discard'] ?? '') ?? DiscardPolicy::Old, + noAck: $data['no_ack'] ?? false, + duplicateWindow: isset($data['duplicate_window']) ? self::nanosToSeconds($data['duplicate_window']) : null, + allowDirect: $data['allow_direct'] ?? false, + mirrorDirect: $data['mirror_direct'] ?? false, + sealed: $data['sealed'] ?? false, + denyDelete: $data['deny_delete'] ?? false, + denyPurge: $data['deny_purge'] ?? false, + allowRollup: $data['allow_rollup_hdrs'] ?? false, + metadata: $data['metadata'] ?? null, + mirror: isset($data['mirror']) ? StreamSource::fromArray($data['mirror']) : null, + sources: isset($data['sources']) + ? array_map(StreamSource::fromArray(...), $data['sources']) + : null, + republish: isset($data['republish']) ? Republish::fromArray($data['republish']) : null, + subjectTransform: isset($data['subject_transform']) + ? SubjectTransform::fromArray($data['subject_transform']) + : null, + placement: isset($data['placement']) ? Placement::fromArray($data['placement']) : null, + compression: $data['compression'] ?? null, + firstSeq: $data['first_seq'] ?? null, + consumerLimits: isset($data['consumer_limits']) + ? ConsumerLimits::fromArray($data['consumer_limits']) + : null, + allowMsgTtl: $data['allow_msg_ttl'] ?? false, + subjectDeleteMarkerTtl: isset($data['subject_delete_marker_ttl']) + ? self::nanosToSeconds($data['subject_delete_marker_ttl']) + : null, + ); + } + + public static function secondsToNanos(float $seconds): int + { + return (int) ($seconds * 1_000_000_000); + } + + public static function nanosToSeconds(int $nanos): float + { + return $nanos / 1_000_000_000; + } +} diff --git a/packages/nats/src/JetStream/StreamInfo.php b/packages/nats/src/JetStream/StreamInfo.php new file mode 100644 index 00000000000..043ec24bce8 --- /dev/null +++ b/packages/nats/src/JetStream/StreamInfo.php @@ -0,0 +1,24 @@ +|null $subjectTransforms + */ + public function __construct( + public readonly string $name, + public readonly ?int $optStartSeq = null, + public readonly ?string $optStartTime = null, + public readonly ?string $filterSubject = null, + public readonly ?array $subjectTransforms = null, + public readonly ?ExternalStream $external = null, + ) { + } + + public function toArray(): array + { + $data = ['name' => $this->name]; + + if ($this->optStartSeq !== null) { + $data['opt_start_seq'] = $this->optStartSeq; + } + if ($this->optStartTime !== null) { + $data['opt_start_time'] = $this->optStartTime; + } + if ($this->filterSubject !== null) { + $data['filter_subject'] = $this->filterSubject; + } + if ($this->subjectTransforms !== null) { + $data['subject_transforms'] = array_map( + static fn (SubjectTransform $t): array => $t->toArray(), + $this->subjectTransforms, + ); + } + if ($this->external instanceof \Utopia\NATS\JetStream\ExternalStream) { + $data['external'] = $this->external->toArray(); + } + + return $data; + } + + public static function fromArray(array $data): self + { + return new self( + name: $data['name'] ?? '', + optStartSeq: $data['opt_start_seq'] ?? null, + optStartTime: $data['opt_start_time'] ?? null, + filterSubject: $data['filter_subject'] ?? null, + subjectTransforms: isset($data['subject_transforms']) + ? array_map(SubjectTransform::fromArray(...), $data['subject_transforms']) + : null, + external: isset($data['external']) ? ExternalStream::fromArray($data['external']) : null, + ); + } +} diff --git a/packages/nats/src/JetStream/StreamState.php b/packages/nats/src/JetStream/StreamState.php new file mode 100644 index 00000000000..abbcc9ba88d --- /dev/null +++ b/packages/nats/src/JetStream/StreamState.php @@ -0,0 +1,36 @@ + + */ + public function toArray(): array + { + return [ + 'src' => $this->source, + 'dest' => $this->destination, + ]; + } + + public static function fromArray(array $data): self + { + return new self( + source: $data['src'] ?? '', + destination: $data['dest'] ?? '', + ); + } +} diff --git a/packages/nats/src/KeyValue/KeyValue.php b/packages/nats/src/KeyValue/KeyValue.php new file mode 100644 index 00000000000..5f099d2f245 --- /dev/null +++ b/packages/nats/src/KeyValue/KeyValue.php @@ -0,0 +1,508 @@ +validateKey($key); + + $subject = "\$KV.{$this->bucket}.{$key}"; + + try { + $msg = $this->conn->request("\$JS.API.DIRECT.GET.KV_{$this->bucket}", json_encode([ + 'last_by_subj' => $subject, + ], JSON_THROW_ON_ERROR)); + } catch (\Throwable) { + throw new KeyValueException("Key not found: {$key}"); + } + + // Check for delete/purge markers + if ($msg->headers instanceof \Utopia\NATS\Headers) { + $op = $msg->headers->get('KV-Operation'); + if ($op === 'DEL' || $op === 'PURGE') { + throw new KeyValueException("Key not found: {$key}"); + } + } + + $revision = 0; + $created = null; + if ($msg->headers instanceof \Utopia\NATS\Headers) { + $seqStr = $msg->headers->get('Nats-Sequence'); + if ($seqStr !== null) { + $revision = (int) $seqStr; + } + $created = $msg->headers->get('Nats-Time-Stamp'); + } + + return new KeyValueEntry( + bucket: $this->bucket, + key: $key, + value: $msg->data, + revision: $revision, + created: $created, + operation: KeyValueOperation::Put, + ); + } + + /** + * Put a value, returning the revision number. + */ + public function put(string $key, string $value): int + { + $this->validateKey($key); + + $subject = "\$KV.{$this->bucket}.{$key}"; + $ack = $this->js->publish($subject, $value); + + return $ack->sequence; + } + + /** + * Create a key only if it does not already exist. + */ + public function create(string $key, string $value): int + { + $this->validateKey($key); + + $subject = "\$KV.{$this->bucket}.{$key}"; + $headers = new Headers(); + $headers->set('Nats-Expected-Last-Subject-Sequence', '0'); + + try { + $ack = $this->js->publish($subject, $value, $headers); + } catch (JetStreamException $e) { + // Only a rejected CAS publish means the key exists. Anything else -- + // a stale connection, a timeout, a permissions error -- must keep its + // own identity, because "you lost the race" is a verdict the caller + // acts on by not retrying. + if ($e->apiError?->errCode !== JetStream::ERR_WRONG_LAST_SEQUENCE) { + throw $e; + } + + throw new KeyValueException("Key already exists: {$key}", $e->getCode(), previous: $e); + } + + return $ack->sequence; + } + + /** + * Update a key only if the current revision matches (CAS). + */ + public function update(string $key, string $value, int $revision): int + { + $this->validateKey($key); + + $subject = "\$KV.{$this->bucket}.{$key}"; + + try { + $ack = $this->js->publish( + $subject, + $value, + expectedLastSubjectSeq: $revision, + ); + } catch (JetStreamException $e) { + if ($e->apiError?->errCode !== JetStream::ERR_WRONG_LAST_SEQUENCE) { + throw $e; + } + + throw new KeyValueException("Wrong last revision for key: {$key}", $e->getCode(), previous: $e); + } + + return $ack->sequence; + } + + public function delete(string $key): void + { + $this->validateKey($key); + + $subject = "\$KV.{$this->bucket}.{$key}"; + $headers = new Headers(); + $headers->set('KV-Operation', 'DEL'); + + $this->js->publish($subject, '', $headers); + } + + public function purge(string $key): void + { + $this->validateKey($key); + + $subject = "\$KV.{$this->bucket}.{$key}"; + $headers = new Headers(); + $headers->set('KV-Operation', 'PURGE'); + $headers->set('Nats-Rollup', 'sub'); + + $this->js->publish($subject, '', $headers); + } + + /** @return list */ + public function keys(): array + { + $streamName = "KV_{$this->bucket}"; + $subject = "\$KV.{$this->bucket}.>"; + + // Use stream subjects to get all keys + try { + $msg = $this->conn->request('$JS.API.STREAM.INFO.' . $streamName, json_encode([ + 'subjects_filter' => $subject, + ], JSON_THROW_ON_ERROR)); + + $data = json_decode($msg->data, true, 512, JSON_THROW_ON_ERROR); + JetStream::checkError($data); + + $keys = []; + $prefix = "\$KV.{$this->bucket}."; + $subjects = $data['state']['subjects'] ?? []; + foreach ($subjects as $subj => $count) { + if (str_starts_with((string) $subj, $prefix)) { + $keys[] = substr((string) $subj, \strlen($prefix)); + } + } + + return $keys; + } catch (\Throwable) { + return []; + } + } + + public function status(): KeyValueStatus + { + $info = $this->js->getStreamInfo("KV_{$this->bucket}"); + + return new KeyValueStatus( + bucket: $this->bucket, + values: $info->state->messages, + bytes: $info->state->bytes, + history: $info->config->maxMsgsPerSubject, + ttl: $info->config->maxAge, + streamInfo: $info, + ); + } + + /** + * Fetch a specific revision of a key by its stream sequence. + */ + public function getRevision(string $key, int $seq): KeyValueEntry + { + $this->validateKey($key); + + return $this->fetchStored(['seq' => $seq], $key); + } + + /** + * Return every stored revision for a key, oldest first. + * + * @return list + */ + public function history(string $key): array + { + $this->validateKey($key); + + $stream = "KV_{$this->bucket}"; + $subject = "\$KV.{$this->bucket}.{$key}"; + + $consumer = $this->js->createConsumer($stream, new ConsumerConfig( + deliverPolicy: DeliverPolicy::All, + ackPolicy: AckPolicy::Explicit, + filterSubject: $subject, + inactiveThreshold: 30.0, + )); + + try { + $entries = []; + foreach ($consumer->fetch(1024, 1.0) as $msg) { + $entries[] = $this->entryFromDelivered($key, $msg); + $msg->ack(); + } + + return $entries; + } finally { + try { + $this->js->deleteConsumer($stream, $consumer->getName()); + } catch (\Throwable) { + // Ephemeral consumer will expire on its own. + } + } + } + + /** + * Watch a key (or wildcard pattern) for updates. + * + * The returned subscription must be pumped by the connection + * (e.g. `$conn->wait()`), and unsubscribed when no longer needed. + * + * With no options, only new updates are delivered (the historical + * default). Options change what is delivered: + * - includeHistory: every stored revision is delivered first, then live updates. + * - updatesOnly: only updates from now on (equivalent to the default). + * - ignoreDeletes: DEL/PURGE marker entries are not passed to `$callback`. + * - metaOnly: entries carry headers/metadata only, with an empty value. + * + * When `$onInitDone` is provided it is invoked exactly once, right after the + * initial/historical set has been fully delivered and before any live update. + * If there is no history to replay it fires immediately. Every `$callback` + * invocation after `$onInitDone` is a live update. + * + * @param callable(KeyValueEntry): void $callback + * @param callable(): void|null $onInitDone + */ + public function watch( + string $keyPattern, + callable $callback, + ?KeyValueWatchOptions $options = null, + ?callable $onInitDone = null, + ): Subscription { + $options ??= new KeyValueWatchOptions(updatesOnly: true); + + $stream = "KV_{$this->bucket}"; + $filter = "\$KV.{$this->bucket}.{$keyPattern}"; + $deliverSubject = $this->conn->newInbox(); + + $deliverPolicy = match (true) { + $options->includeHistory => DeliverPolicy::All->value, + $options->updatesOnly => DeliverPolicy::New->value, + default => DeliverPolicy::LastPerSubject->value, + }; + + $config = [ + 'deliver_subject' => $deliverSubject, + 'deliver_policy' => $deliverPolicy, + 'ack_policy' => 'none', + 'filter_subject' => $filter, + 'inactive_threshold' => 30 * 1_000_000_000, + ]; + if ($options->metaOnly) { + $config['headers_only'] = true; + } + + $response = $this->conn->request("\$JS.API.CONSUMER.CREATE.{$stream}", json_encode([ + 'stream_name' => $stream, + 'config' => $config, + ], JSON_THROW_ON_ERROR)); + $data = json_decode($response->data, true, 512, JSON_THROW_ON_ERROR); + JetStream::checkError($data); + + $numPending = (int) ($data['num_pending'] ?? 0); + $delivered = 0; + $initSignaled = false; + + $sub = $this->conn->subscribe($deliverSubject, function (Message $msg) use ($callback, $options, $onInitDone, $numPending, &$delivered, &$initSignaled): void { + // Ignore JetStream idle heartbeats / flow control (100 status). + if ($msg->headers instanceof Headers && $msg->headers->getStatus() !== '') { + return; + } + + $delivered++; + $entry = $this->entryFromMessage($msg); + + $isMarker = $entry->operation === KeyValueOperation::Delete + || $entry->operation === KeyValueOperation::Purge; + if (!$options->ignoreDeletes || !$isMarker) { + $callback($entry); + } + + if (!$initSignaled && $delivered >= $numPending) { + $initSignaled = true; + if ($onInitDone !== null) { + $onInitDone(); + } + } + }); + + if ($numPending === 0 && !$initSignaled) { + $initSignaled = true; + if ($onInitDone !== null) { + $onInitDone(); + } + } + + return $sub; + } + + /** + * Remove DEL/PURGE tombstone markers: for every key whose latest entry is a + * delete/purge marker, purge that subject. + * + * @param float|null $threshold When set, only markers older than this many + * seconds are removed; newer markers are kept. + * Null removes all delete markers. + * @return int Number of keys whose tombstones were removed. + */ + public function purgeDeletes(?float $threshold = null): int + { + $purged = 0; + $now = microtime(true); + + foreach ($this->keys() as $key) { + $subject = "\$KV.{$this->bucket}.{$key}"; + + try { + $msg = $this->conn->request("\$JS.API.DIRECT.GET.KV_{$this->bucket}", json_encode([ + 'last_by_subj' => $subject, + ], JSON_THROW_ON_ERROR)); + } catch (\Throwable) { + continue; + } + + if (!$msg->headers instanceof Headers) { + continue; + } + + $op = $msg->headers->get('KV-Operation'); + if ($op !== 'DEL' && $op !== 'PURGE') { + continue; + } + + // keep=1 retains a not-yet-expired marker; keep=0 removes the subject entirely. + $keep = 0; + if ($threshold !== null) { + $ts = $msg->headers->get('Nats-Time-Stamp'); + $created = $ts !== null ? strtotime($ts) : false; + if ($created !== false && ($now - $created) < $threshold) { + $keep = 1; + } + } + + $response = $this->conn->request("\$JS.API.STREAM.PURGE.KV_{$this->bucket}", json_encode([ + 'filter' => $subject, + 'keep' => $keep, + ], JSON_THROW_ON_ERROR)); + $result = json_decode($response->data, true, 512, JSON_THROW_ON_ERROR); + JetStream::checkError($result); + + if ($keep === 0) { + $purged++; + } + } + + return $purged; + } + + public function getBucket(): string + { + return $this->bucket; + } + + private function entryFromDelivered(string $key, JetStreamMessage $msg): KeyValueEntry + { + return new KeyValueEntry( + bucket: $this->bucket, + key: $key, + value: $msg->getData(), + revision: $msg->metadata()->streamSequence, + created: $msg->metadata()->timestamp, + operation: $this->operationFromHeaders($msg->getHeaders()), + ); + } + + private function entryFromMessage(Message $msg): KeyValueEntry + { + $prefix = "\$KV.{$this->bucket}."; + $key = str_starts_with($msg->subject, $prefix) + ? substr($msg->subject, \strlen($prefix)) + : $msg->subject; + + $revision = 0; + $created = null; + if ($msg->replyTo !== null) { + $meta = MsgMetadata::fromReplySubject($msg->replyTo); + $revision = $meta->streamSequence; + $created = $meta->timestamp; + } + + return new KeyValueEntry( + bucket: $this->bucket, + key: $key, + value: $msg->data, + revision: $revision, + created: $created, + operation: $this->operationFromHeaders($msg->headers), + ); + } + + /** + * @param array $request + */ + private function fetchStored(array $request, string $key): KeyValueEntry + { + try { + $response = $this->conn->request( + "\$JS.API.STREAM.MSG.GET.KV_{$this->bucket}", + json_encode($request, JSON_THROW_ON_ERROR), + ); + } catch (\Throwable $e) { + throw new KeyValueException("Revision not found for key: {$key}", $e->getCode(), previous: $e); + } + + $data = json_decode($response->data, true, 512, JSON_THROW_ON_ERROR); + JetStream::checkError($data); + + $stored = $data['message'] ?? null; + if (!\is_array($stored)) { + throw new KeyValueException("Revision not found for key: {$key}"); + } + + // A sequence lookup is stream-global: reject a revision that belongs to a different key. + $expectedSubject = "\$KV.{$this->bucket}.{$key}"; + if (isset($stored['subject']) && $stored['subject'] !== $expectedSubject) { + throw new KeyValueException("Revision not found for key: {$key}"); + } + + $headers = null; + if (isset($stored['hdrs']) && \is_string($stored['hdrs'])) { + $headers = Headers::fromWire((string) base64_decode($stored['hdrs'], true)); + } + + return new KeyValueEntry( + bucket: $this->bucket, + key: $key, + value: isset($stored['data']) ? (string) base64_decode((string) $stored['data'], true) : '', + revision: (int) ($stored['seq'] ?? 0), + created: $stored['time'] ?? null, + operation: $this->operationFromHeaders($headers), + ); + } + + private function operationFromHeaders(?Headers $headers): KeyValueOperation + { + if (!$headers instanceof Headers) { + return KeyValueOperation::Put; + } + + return match ($headers->get('KV-Operation')) { + 'DEL' => KeyValueOperation::Delete, + 'PURGE' => KeyValueOperation::Purge, + default => KeyValueOperation::Put, + }; + } + + private function validateKey(string $key): void + { + if ($key === '' || str_contains($key, ' ') || str_contains($key, '>') || str_contains($key, '*')) { + throw new KeyValueException("Invalid key: {$key}"); + } + } +} diff --git a/packages/nats/src/KeyValue/KeyValueConfig.php b/packages/nats/src/KeyValue/KeyValueConfig.php new file mode 100644 index 00000000000..2edb3e29cef --- /dev/null +++ b/packages/nats/src/KeyValue/KeyValueConfig.php @@ -0,0 +1,47 @@ +bucket}", + subjects: ["\$KV.{$this->bucket}.>"], + description: $this->description, + retention: RetentionPolicy::Limits, + maxBytes: $this->maxBytes, + maxMsgsPerSubject: $this->history, + maxMsgSize: $this->maxValueSize > 0 ? $this->maxValueSize : null, + maxAge: $this->ttl, + storage: $this->storage, + replicas: $this->replicas, + discard: DiscardPolicy::New, + allowDirect: true, + allowRollup: true, + ); + } +} diff --git a/packages/nats/src/KeyValue/KeyValueEntry.php b/packages/nats/src/KeyValue/KeyValueEntry.php new file mode 100644 index 00000000000..26f04f608fa --- /dev/null +++ b/packages/nats/src/KeyValue/KeyValueEntry.php @@ -0,0 +1,18 @@ + + */ + public function toArray(): array + { + $data = ['bucket' => $this->bucket]; + if ($this->name !== null && $this->name !== '') { + $data['name'] = $this->name; + } + + return $data; + } + + /** + * @param array $data + */ + public static function fromArray(array $data): self + { + return new self( + bucket: (string) ($data['bucket'] ?? ''), + name: isset($data['name']) ? (string) $data['name'] : null, + ); + } +} diff --git a/packages/nats/src/ObjectStore/ObjectMeta.php b/packages/nats/src/ObjectStore/ObjectMeta.php new file mode 100644 index 00000000000..43b2ddba6b0 --- /dev/null +++ b/packages/nats/src/ObjectStore/ObjectMeta.php @@ -0,0 +1,89 @@ +|null $metadata + */ + public function __construct( + public readonly string $name, + public readonly string $bucket, + public readonly string $nuid, + public readonly int $size, + public readonly int $chunks, + public readonly string $digest, + public readonly ?string $description = null, + public readonly ?string $modified = null, + public readonly bool $deleted = false, + public readonly ?array $metadata = null, + public readonly ?ObjectLink $link = null, + ) { + } + + /** + * @return array + */ + public function toArray(): array + { + $data = [ + 'name' => $this->name, + 'bucket' => $this->bucket, + 'nuid' => $this->nuid, + 'size' => $this->size, + 'chunks' => $this->chunks, + 'digest' => $this->digest, + ]; + + if ($this->description !== null) { + $data['description'] = $this->description; + } + if ($this->modified !== null) { + $data['mtime'] = $this->modified; + } + if ($this->deleted) { + $data['deleted'] = true; + } + if ($this->metadata !== null && $this->metadata !== []) { + $data['metadata'] = $this->metadata; + } + if ($this->link instanceof ObjectLink) { + $data['options'] = ['link' => $this->link->toArray()]; + } + + return $data; + } + + /** + * @param array $data + */ + public static function fromArray(array $data): self + { + $metadata = null; + if (isset($data['metadata']) && \is_array($data['metadata'])) { + $metadata = array_map(static fn (mixed $v): string => (string) $v, $data['metadata']); + } + + $link = null; + if (isset($data['options']['link']) && \is_array($data['options']['link'])) { + $link = ObjectLink::fromArray($data['options']['link']); + } + + return new self( + name: (string) ($data['name'] ?? ''), + bucket: (string) ($data['bucket'] ?? ''), + nuid: (string) ($data['nuid'] ?? ''), + size: (int) ($data['size'] ?? 0), + chunks: (int) ($data['chunks'] ?? 0), + digest: (string) ($data['digest'] ?? ''), + description: isset($data['description']) ? (string) $data['description'] : null, + modified: isset($data['mtime']) ? (string) $data['mtime'] : null, + deleted: (bool) ($data['deleted'] ?? false), + metadata: $metadata, + link: $link, + ); + } +} diff --git a/packages/nats/src/ObjectStore/ObjectStore.php b/packages/nats/src/ObjectStore/ObjectStore.php new file mode 100644 index 00000000000..28dd457daf1 --- /dev/null +++ b/packages/nats/src/ObjectStore/ObjectStore.php @@ -0,0 +1,505 @@ +createOrUpdateStream($config->toStreamConfig()); + + return new self($conn, $js, $config->bucket); + } + + /** + * Store an object, chunking its data and writing a meta record. + */ + public function put(string $name, string $data): ObjectMeta + { + [$previous, $previousSeq] = $this->readMetaWithSeq($name); + + return $this->writeVersion($name, $data, $previous, $previousSeq); + } + + /** + * Write a new version of an object, expecting the meta subject to still be at + * $expectedSeq (0 = must not exist yet). Separated from put() so the optimistic + * concurrency path can be driven deterministically in tests. + */ + private function writeVersion(string $name, string $data, ?ObjectMeta $previous, int $expectedSeq): ObjectMeta + { + $nuid = strtoupper(bin2hex(random_bytes(12))); + $chunkSubject = "\$O.{$this->bucket}.C.{$nuid}"; + + $chunks = 0; + $length = \strlen($data); + for ($offset = 0; $offset < $length; $offset += self::CHUNK_SIZE) { + $this->js->publish($chunkSubject, substr($data, $offset, self::CHUNK_SIZE)); + $chunks++; + } + + $meta = new ObjectMeta( + name: $name, + bucket: $this->bucket, + nuid: $nuid, + size: $length, + chunks: $chunks, + digest: $this->digest($data), + modified: gmdate('Y-m-d\TH:i:s\Z'), + ); + + // Optimistic concurrency: the meta publish only succeeds if the meta subject's + // last sequence still matches what we read (0 means "must not exist yet"). If a + // concurrent or stale writer already advanced it, JetStream rejects the publish, + // we purge only the chunks THIS put wrote (never the previous NUID), and surface a + // clear conflict. This replaces the former last-writer-wins behaviour that could + // orphan the loser's chunks. + $headers = new Headers(); + $headers->set('Nats-Rollup', 'sub'); + try { + $this->js->publish( + $this->metaSubject($name), + json_encode($meta->toArray(), JSON_THROW_ON_ERROR), + $headers, + expectedLastSubjectSeq: $expectedSeq, + ); + } catch (JetStreamException $e) { + // Purge only when the server actually rejected the CAS publish. On a + // transport failure the meta publish is ambiguous -- it may well have + // landed -- and purging here would delete the chunks this put just + // wrote successfully. + if ($e->apiError?->errCode !== JetStream::ERR_WRONG_LAST_SEQUENCE) { + throw $e; + } + + $this->purgeChunks($nuid); + throw new ObjectStoreException("conflicting concurrent write for object: {$name}", $e->getCode(), previous: $e); + } + + // Reclaim chunks left behind by a prior version of this object. + if ($previous instanceof ObjectMeta && $previous->nuid !== '' && $previous->nuid !== $nuid) { + $this->purgeChunks($previous->nuid); + } + + return $meta; + } + + /** + * Retrieve an object's bytes, reassembling and verifying its chunks. + */ + public function get(string $name): string + { + $meta = $this->readMeta($name); + if (!$meta instanceof ObjectMeta || $meta->deleted) { + throw new \RuntimeException("Object not found: {$name}"); + } + + // A link transparently resolves to its target's bytes (same bucket). A + // bucket link has no target object, so it cannot be read as an object. + if ($meta->link instanceof ObjectLink) { + if ($meta->link->name === null || $meta->link->name === '') { + throw new ObjectStoreException("cannot get a bucket link: {$name}"); + } + + return $this->get($meta->link->name); + } + + $data = ''; + if ($meta->chunks > 0) { + $stream = "OBJ_{$this->bucket}"; + $subject = "\$O.{$this->bucket}.C.{$meta->nuid}"; + + $consumer = $this->js->createConsumer($stream, new ConsumerConfig( + deliverPolicy: DeliverPolicy::All, + ackPolicy: AckPolicy::Explicit, + filterSubject: $subject, + inactiveThreshold: 30.0, + )); + + try { + foreach ($consumer->fetch($meta->chunks, 10.0) as $msg) { + $data .= $msg->getData(); + $msg->ack(); + } + } finally { + try { + $this->js->deleteConsumer($stream, $consumer->getName()); + } catch (\Throwable) { + // Ephemeral consumer expires on its own. + } + } + } + + if (\strlen($data) !== $meta->size || $this->digest($data) !== $meta->digest) { + throw new \RuntimeException("Object integrity check failed: {$name}"); + } + + return $data; + } + + public function getMeta(string $name): ObjectMeta + { + $meta = $this->readMeta($name); + if (!$meta instanceof ObjectMeta || $meta->deleted) { + throw new \RuntimeException("Object not found: {$name}"); + } + + return $meta; + } + + public function delete(string $name): void + { + [$meta, $expectedSeq] = $this->readMetaWithSeq($name); + if (!$meta instanceof ObjectMeta || $meta->deleted) { + return; + } + + $this->deleteVersion($meta, $expectedSeq); + } + + /** + * Commit a deletion tombstone guarded by optimistic concurrency, then reclaim the + * object's chunks. The guarded publish goes FIRST: a concurrent put()/updateMeta() + * advances the meta subject, so this conflicts and throws before any chunk is purged, + * leaving the replacement version intact. Separated so the conflict path is testable. + */ + private function deleteVersion(ObjectMeta $meta, int $expectedSeq): void + { + // Write a deletion marker (tombstone) rather than purging the meta subject, + // so watchers observe the delete and get()/list() still treat it as gone. + // The rollup header collapses the subject to this single record. + $tombstone = new ObjectMeta( + name: $meta->name, + bucket: $meta->bucket, + nuid: '', + size: 0, + chunks: 0, + digest: '', + description: $meta->description, + modified: gmdate('Y-m-d\TH:i:s\Z'), + deleted: true, + ); + + $this->publishMeta($tombstone, $expectedSeq); + + // Only after the tombstone is committed do we reclaim the chunks. + if ($meta->nuid !== '') { + $this->purgeChunks($meta->nuid); + } + } + + /** + * List all (non-deleted) objects in the bucket. + * + * @return list + */ + public function list(): array + { + $stream = "OBJ_{$this->bucket}"; + $subject = "\$O.{$this->bucket}.M.>"; + + try { + $consumer = $this->js->createConsumer($stream, new ConsumerConfig( + deliverPolicy: DeliverPolicy::LastPerSubject, + ackPolicy: AckPolicy::Explicit, + filterSubject: $subject, + inactiveThreshold: 30.0, + )); + } catch (\Throwable) { + return []; + } + + try { + $objects = []; + foreach ($consumer->fetch(1024, 1.0) as $msg) { + $msg->ack(); + $decoded = json_decode((string) $msg->getData(), true, 512, JSON_THROW_ON_ERROR); + if (!\is_array($decoded)) { + continue; + } + $meta = ObjectMeta::fromArray($decoded); + if (!$meta->deleted) { + $objects[] = $meta; + } + } + + return $objects; + } finally { + try { + $this->js->deleteConsumer($stream, $consumer->getName()); + } catch (\Throwable) { + // Ephemeral consumer expires on its own. + } + } + } + + public function status(): StreamInfo + { + return $this->js->getStreamInfo("OBJ_{$this->bucket}"); + } + + /** + * Watch the bucket for object meta updates (puts and deletes), delivering an + * ObjectMeta to the callback for each change. + * + * Backed by an ephemeral push consumer on the meta subject created via the raw + * JetStream API. The returned subscription must be pumped by the connection + * (e.g. `$conn->wait()` / `$conn->processMessage()`) and unsubscribed when done. + * + * @param callable(ObjectMeta): void $callback + * @param bool $includeHistory deliver the current meta of every object first + */ + public function watch(callable $callback, bool $includeHistory = false): Subscription + { + $stream = "OBJ_{$this->bucket}"; + $filter = "\$O.{$this->bucket}.M.>"; + $deliverSubject = $this->conn->newInbox(); + + $payload = json_encode([ + 'stream_name' => $stream, + 'config' => [ + 'deliver_subject' => $deliverSubject, + 'deliver_policy' => $includeHistory ? 'last_per_subject' : 'new', + 'ack_policy' => 'none', + 'filter_subject' => $filter, + 'inactive_threshold' => 30 * 1_000_000_000, + ], + ], JSON_THROW_ON_ERROR); + + $response = $this->conn->request("\$JS.API.CONSUMER.CREATE.{$stream}", $payload); + $data = json_decode($response->data, true, 512, JSON_THROW_ON_ERROR); + JetStream::checkError($data); + + return $this->conn->subscribe($deliverSubject, function (Message $msg) use ($callback): void { + // Ignore JetStream idle heartbeats / flow control (status messages). + if ($msg->headers instanceof Headers && $msg->headers->getStatus() !== '') { + return; + } + + $decoded = json_decode($msg->data, true, 512, JSON_THROW_ON_ERROR); + if (!\is_array($decoded)) { + return; + } + + $callback(ObjectMeta::fromArray($decoded)); + }); + } + + /** + * Create a link under $linkName pointing to another object in this bucket. + * get($linkName) then transparently resolves to the target's bytes. + */ + public function addLink(string $linkName, string $targetObjectName): ObjectMeta + { + return $this->writeLink($linkName, new ObjectLink($this->bucket, $targetObjectName)); + } + + /** + * Create a link under $linkName pointing to an entire bucket. A bucket link + * cannot be read via get(); doing so throws. + */ + public function addBucketLink(string $linkName, string $targetBucket): ObjectMeta + { + return $this->writeLink($linkName, new ObjectLink($targetBucket)); + } + + /** + * Update mutable meta fields, preserving the stored bytes (nuid/size/chunks/digest). + * + * @param array|null $metadata + */ + public function updateMeta(string $name, ?string $description = null, ?array $metadata = null): ObjectMeta + { + [$previous, $expectedSeq] = $this->readMetaWithSeq($name); + if (!$previous instanceof ObjectMeta || $previous->deleted) { + throw new \RuntimeException("Object not found: {$name}"); + } + + $meta = new ObjectMeta( + name: $previous->name, + bucket: $previous->bucket, + nuid: $previous->nuid, + size: $previous->size, + chunks: $previous->chunks, + digest: $previous->digest, + description: $description ?? $previous->description, + modified: gmdate('Y-m-d\TH:i:s\Z'), + metadata: $metadata ?? $previous->metadata, + link: $previous->link, + ); + + $this->publishMeta($meta, $expectedSeq); + + return $meta; + } + + /** + * Seal the bucket, making it permanently read-only by updating the backing + * stream config. After sealing, the server rejects put()/delete(). + */ + public function seal(): void + { + $stream = "OBJ_{$this->bucket}"; + + $infoResponse = $this->conn->request("\$JS.API.STREAM.INFO.{$stream}"); + $info = json_decode($infoResponse->data, true, 512, JSON_THROW_ON_ERROR); + JetStream::checkError($info); + + $config = $info['config'] ?? []; + if (!\is_array($config)) { + throw new ObjectStoreException("cannot read stream config for bucket: {$this->bucket}"); + } + // Empty JSON objects in the info response (e.g. consumer_limits) decode to + // empty PHP arrays and would re-encode as [] — which the API rejects as + // invalid JSON for a struct field. Drop them; the server re-applies defaults. + $config = array_filter($config, static fn (mixed $v): bool => $v !== []); + $config['sealed'] = true; + + $updateResponse = $this->conn->request( + "\$JS.API.STREAM.UPDATE.{$stream}", + json_encode($config, JSON_THROW_ON_ERROR), + ); + $updated = json_decode($updateResponse->data, true, 512, JSON_THROW_ON_ERROR); + JetStream::checkError($updated); + } + + public function getBucket(): string + { + return $this->bucket; + } + + private function writeLink(string $linkName, ObjectLink $link): ObjectMeta + { + [$previous, $expectedSeq] = $this->readMetaWithSeq($linkName); + + $meta = new ObjectMeta( + name: $linkName, + bucket: $this->bucket, + nuid: '', + size: 0, + chunks: 0, + digest: '', + modified: gmdate('Y-m-d\TH:i:s\Z'), + link: $link, + ); + + $this->publishMeta($meta, $expectedSeq); + + // Reclaim chunks left behind if this name previously held a real object. + if ($previous instanceof ObjectMeta && $previous->nuid !== '') { + $this->purgeChunks($previous->nuid); + } + + return $meta; + } + + /** + * Publish a meta record with a rolling-up header under the optimistic-concurrency + * guard: the publish only succeeds if the meta subject is still at $expectedSeq. + */ + private function publishMeta(ObjectMeta $meta, int $expectedSeq): void + { + $headers = new Headers(); + $headers->set('Nats-Rollup', 'sub'); + + try { + $this->js->publish( + $this->metaSubject($meta->name), + json_encode($meta->toArray(), JSON_THROW_ON_ERROR), + $headers, + expectedLastSubjectSeq: $expectedSeq, + ); + } catch (JetStreamException $e) { + if ($e->apiError?->errCode !== JetStream::ERR_WRONG_LAST_SEQUENCE) { + throw $e; + } + + throw new ObjectStoreException("conflicting concurrent write for object: {$meta->name}", $e->getCode(), previous: $e); + } + } + + private function readMeta(string $name): ?ObjectMeta + { + return $this->readMetaWithSeq($name)[0]; + } + + /** + * Read the latest meta record for an object along with its stream sequence. + * The sequence is 0 when no meta record exists, which callers use as the + * expected-last-subject-sequence for an optimistic first write. + * + * @return array{0: ?ObjectMeta, 1: int} + */ + private function readMetaWithSeq(string $name): array + { + try { + $response = $this->conn->request( + "\$JS.API.STREAM.MSG.GET.OBJ_{$this->bucket}", + json_encode(['last_by_subj' => $this->metaSubject($name)], JSON_THROW_ON_ERROR), + ); + } catch (\Throwable) { + return [null, 0]; + } + + $data = json_decode($response->data, true, 512, JSON_THROW_ON_ERROR); + if (isset($data['error']) || !isset($data['message']['data'])) { + return [null, 0]; + } + + $decoded = json_decode((string) base64_decode((string) $data['message']['data'], true), true, 512, JSON_THROW_ON_ERROR); + if (!\is_array($decoded)) { + return [null, 0]; + } + + return [ObjectMeta::fromArray($decoded), (int) ($data['message']['seq'] ?? 0)]; + } + + private function purgeChunks(string $nuid): void + { + try { + $this->js->purgeStream("OBJ_{$this->bucket}", "\$O.{$this->bucket}.C.{$nuid}"); + } catch (\Throwable) { + // Best effort cleanup. + } + } + + private function metaSubject(string $name): string + { + $token = rtrim(strtr(base64_encode($name), '+/', '-_'), '='); + + return "\$O.{$this->bucket}.M.{$token}"; + } + + private function digest(string $data): string + { + $raw = hash('sha256', $data, true); + + return 'SHA-256=' . rtrim(strtr(base64_encode($raw), '+/', '-_'), '='); + } +} diff --git a/packages/nats/src/ObjectStore/ObjectStoreConfig.php b/packages/nats/src/ObjectStore/ObjectStoreConfig.php new file mode 100644 index 00000000000..0d1a5abc1ed --- /dev/null +++ b/packages/nats/src/ObjectStore/ObjectStoreConfig.php @@ -0,0 +1,46 @@ +bucket}", + subjects: [ + "\$O.{$this->bucket}.C.>", + "\$O.{$this->bucket}.M.>", + ], + description: $this->description, + retention: RetentionPolicy::Limits, + maxBytes: $this->maxBytes, + maxAge: $this->ttl, + storage: $this->storage, + replicas: $this->replicas, + discard: DiscardPolicy::New, + allowDirect: true, + allowRollup: true, + ); + } +} diff --git a/packages/nats/src/Protocol/Command.php b/packages/nats/src/Protocol/Command.php new file mode 100644 index 00000000000..e20646b062e --- /dev/null +++ b/packages/nats/src/Protocol/Command.php @@ -0,0 +1,16 @@ +buffer !== ''; + } + + /** + * Read and parse the next server operation. + * + * @return array{0: ServerOp, 1: mixed} Tuple of [operation, parsed data] + */ + public function next(?float $timeout = null): array + { + // Once a frame body has been partially consumed the buffer no longer + // starts on a frame boundary, so nothing after that point can be parsed. + if ($this->poisoned) { + throw new ConnectionException('Parser desynced from the stream; the connection must be rebuilt'); + } + + $line = $this->readLine($timeout); + $line = rtrim($line, "\r\n"); + + if ($line === '') { + throw new ProtocolException('Empty protocol line received'); + } + + // +OK + if ($line === '+OK') { + return [ServerOp::Ok, null]; + } + + // PING + if ($line === 'PING') { + return [ServerOp::Ping, null]; + } + + // PONG + if ($line === 'PONG') { + return [ServerOp::Pong, null]; + } + + // -ERR 'message' + if (str_starts_with($line, '-ERR')) { + $message = trim(substr($line, 4), " \t'\""); + return [ServerOp::Err, $message]; + } + + // INFO {json} + if (str_starts_with($line, 'INFO ')) { + $json = substr($line, 5); + $data = json_decode($json, true, 512, JSON_THROW_ON_ERROR); + return [ServerOp::Info, $data]; + } + + // MSG [reply-to] <#bytes> + if (str_starts_with($line, 'MSG ')) { + return $this->readFrame(fn (): array => $this->parseMsg(substr($line, 4), $timeout)); + } + + // HMSG [reply-to] <#header-bytes> <#total-bytes> + if (str_starts_with($line, 'HMSG ')) { + return $this->readFrame(fn (): array => $this->parseHmsg(substr($line, 5), $timeout)); + } + + throw new ProtocolException("Unknown protocol operation: {$line}"); + } + + /** + * Parse MSG: subject sid [reply-to] #bytes + * + * @return array{0: ServerOp, 1: array{subject: string, sid: string, replyTo: ?string, payload: string}} + */ + private function parseMsg(string $args, ?float $timeout = null): array + { + $parts = preg_split('/\s+/', trim($args)); + + if ($parts === false || \count($parts) < 3 || \count($parts) > 4) { + throw new ProtocolException("Invalid MSG line: MSG {$args}"); + } + + if (\count($parts) === 3) { + [$subject, $sid, $byteCount] = $parts; + $replyTo = null; + } else { + [$subject, $sid, $replyTo, $byteCount] = $parts; + } + + $bytes = (int) $byteCount; + $payload = $this->readExactly($bytes, $timeout); + // Consume trailing \r\n + $this->readExactly(2, $timeout); + + return [ServerOp::Msg, [ + 'subject' => $subject, + 'sid' => $sid, + 'replyTo' => $replyTo, + 'payload' => $payload, + 'headers' => null, + ]]; + } + + /** + * Parse HMSG: subject sid [reply-to] #header-bytes #total-bytes + * + * @return array{0: ServerOp, 1: array{subject: string, sid: string, replyTo: ?string, payload: string, headers: string}} + */ + private function parseHmsg(string $args, ?float $timeout = null): array + { + $parts = preg_split('/\s+/', trim($args)); + + if ($parts === false || \count($parts) < 4 || \count($parts) > 5) { + throw new ProtocolException("Invalid HMSG line: HMSG {$args}"); + } + + if (\count($parts) === 4) { + [$subject, $sid, $headerBytes, $totalBytes] = $parts; + $replyTo = null; + } else { + [$subject, $sid, $replyTo, $headerBytes, $totalBytes] = $parts; + } + + $hdrLen = (int) $headerBytes; + $totalLen = (int) $totalBytes; + $payloadLen = $totalLen - $hdrLen; + + if ($payloadLen < 0) { + throw new ProtocolException("Invalid HMSG byte counts: header={$hdrLen}, total={$totalLen}"); + } + + $headerBlock = $this->readExactly($hdrLen, $timeout); + $payload = $payloadLen > 0 ? $this->readExactly($payloadLen, $timeout) : ''; + // Consume trailing \r\n + $this->readExactly(2, $timeout); + + return [ServerOp::HMsg, [ + 'subject' => $subject, + 'sid' => $sid, + 'replyTo' => $replyTo, + 'payload' => $payload, + 'headers' => $headerBlock, + ]]; + } + + private function readLine(?float $timeout = null): string + { + // Check buffer for a complete line first + $pos = strpos($this->buffer, "\n"); + if ($pos !== false) { + $line = substr($this->buffer, 0, $pos + 1); + $this->buffer = substr($this->buffer, $pos + 1); + return $line; + } + + // Read from transport until we get a line + while (true) { + $data = $this->transport->read(65536, $timeout); + $this->buffer .= $data; + + $pos = strpos($this->buffer, "\n"); + if ($pos !== false) { + $line = substr($this->buffer, 0, $pos + 1); + $this->buffer = substr($this->buffer, $pos + 1); + return $line; + } + } + } + + /** + * Read exactly $bytes, passing the timeout explicitly on every transport + * read. Previously it passed none, so the read inherited whatever deadline + * the last readLine() happened to leave on the stream. The timeout applies + * per read rather than to the frame as a whole: a large payload arrives + * across several segments, and charging the whole body to one deadline + * would fail frames that are making steady progress. + */ + private function readExactly(int $bytes, ?float $timeout = null): string + { + while (\strlen($this->buffer) < $bytes) { + $data = $this->transport->read(max(65536, $bytes - \strlen($this->buffer)), $timeout); + if ($data === '') { + throw new ProtocolException('Unexpected end of data while reading payload'); + } + $this->buffer .= $data; + } + + $result = substr($this->buffer, 0, $bytes); + $this->buffer = substr($this->buffer, $bytes); + return $result; + } + + /** + * Run a frame-body read, turning any failure into a poisoned parser. + * + * The header line is already consumed by the time this runs, so a partial + * body read leaves the buffer mid-frame. Reporting that as a timeout would + * be indistinguishable from "nothing arrived" -- the caller would carry on + * against a stream whose next bytes are payload, not protocol. Raising + * ConnectionException instead routes it to the reconnect path, which builds + * a fresh parser. The dropped frame is often the PubAck itself. + * + * @param \Closure(): array{0: ServerOp, 1: mixed} $read + * @return array{0: ServerOp, 1: mixed} + */ + private function readFrame(\Closure $read): array + { + try { + return $read(); + } catch (\Throwable $e) { + $this->poisoned = true; + $this->buffer = ''; + + throw new ConnectionException("Failed mid-frame, parser desynced from the stream: {$e->getMessage()}", $e->getCode(), previous: $e); + } + } +} diff --git a/packages/nats/src/Protocol/ServerOp.php b/packages/nats/src/Protocol/ServerOp.php new file mode 100644 index 00000000000..9620a81c9ed --- /dev/null +++ b/packages/nats/src/Protocol/ServerOp.php @@ -0,0 +1,16 @@ +]/', $subject) + || str_starts_with($subject, '.') || str_ends_with($subject, '.') + || str_contains($subject, '..')) { + throw new \InvalidArgumentException('Invalid request subject'); + } + } +} diff --git a/packages/nats/src/ServerInfo.php b/packages/nats/src/ServerInfo.php new file mode 100644 index 00000000000..f591e5486ad --- /dev/null +++ b/packages/nats/src/ServerInfo.php @@ -0,0 +1,50 @@ + $metadata + */ + public function __construct( + public readonly string $name, + public readonly string $subject, + callable $handler, + public readonly string $queueGroup = 'q', + public readonly array $metadata = [], + ) { + $this->handler = $handler; + } + + /** + * @return array + */ + public function info(): array + { + return [ + 'name' => $this->name, + 'subject' => $this->subject, + 'queue_group' => $this->queueGroup, + 'metadata' => $this->metadata === [] ? new \stdClass() : $this->metadata, + ]; + } + + /** + * @return array + */ + public function stats(): array + { + $average = $this->numRequests > 0 + ? intdiv($this->processingTime, $this->numRequests) + : 0; + + return [ + 'name' => $this->name, + 'subject' => $this->subject, + 'queue_group' => $this->queueGroup, + 'metadata' => $this->metadata === [] ? new \stdClass() : $this->metadata, + 'num_requests' => $this->numRequests, + 'num_errors' => $this->numErrors, + 'last_error' => $this->lastError ?? '', + 'processing_time' => $this->processingTime, + 'average_processing_time' => $average, + ]; + } +} diff --git a/packages/nats/src/Services/Group.php b/packages/nats/src/Services/Group.php new file mode 100644 index 00000000000..00b58882633 --- /dev/null +++ b/packages/nats/src/Services/Group.php @@ -0,0 +1,60 @@ +service, + $this->prefix . '.' . $name, + $queueGroup ?? $this->queueGroup, + ); + } + + /** + * Register an endpoint under this group. The subject is prefixed with + * the group name; when $subject is omitted the endpoint name is used. + * + * @param callable(Message): string $handler + * @param array $metadata + */ + public function addEndpoint( + string $name, + callable $handler, + ?string $subject = null, + ?string $queueGroup = null, + array $metadata = [], + ): self { + $this->service->registerEndpoint( + $name, + $this->prefix . '.' . ($subject ?? $name), + $handler, + $queueGroup ?? $this->queueGroup, + $metadata, + ); + + return $this; + } +} diff --git a/packages/nats/src/Services/Service.php b/packages/nats/src/Services/Service.php new file mode 100644 index 00000000000..3c87adca90a --- /dev/null +++ b/packages/nats/src/Services/Service.php @@ -0,0 +1,261 @@ + */ + private array $endpoints = []; + + /** @var list */ + private array $subscriptions = []; + + private bool $running = false; + + /** + * @param array $metadata + */ + public function __construct( + private readonly Connection $conn, + private readonly string $name, + private readonly string $version, + private readonly string $description = '', + private readonly array $metadata = [], + ) { + $this->id = strtoupper(bin2hex(random_bytes(11))); + $this->started = gmdate('Y-m-d\TH:i:s\Z'); + } + + /** + * Register an endpoint handler. The handler receives the request + * Message and returns the reply payload as a string. + * + * @param callable(Message): string $handler + * @param array $metadata + */ + public function addEndpoint( + string $name, + string $subject, + callable $handler, + ?string $queueGroup = null, + array $metadata = [], + ): self { + $this->registerEndpoint($name, $subject, $handler, $queueGroup, $metadata); + + return $this; + } + + /** + * Create an endpoint group with a subject prefix and optional default + * queue group inherited by its endpoints and nested groups. + */ + public function addGroup(string $name, ?string $queueGroup = null): Group + { + return new Group($this, $name, $queueGroup); + } + + /** + * Register an endpoint on the service. Used by both the bare + * addEndpoint and the group handles. + * + * @param callable(Message): string $handler + * @param array $metadata + * + * @internal + */ + public function registerEndpoint( + string $name, + string $subject, + callable $handler, + ?string $queueGroup = null, + array $metadata = [], + ): void { + $this->endpoints[$name] = new Endpoint($name, $subject, $handler, $queueGroup ?? 'q', $metadata); + + if ($this->running) { + $this->subscribeEndpoint($this->endpoints[$name]); + } + } + + /** + * Subscribe all endpoint handlers and discovery responders. + * Non-blocking: the caller must pump the connection. + */ + public function start(): self + { + if ($this->running) { + return $this; + } + $this->running = true; + + foreach ($this->endpoints as $endpoint) { + $this->subscribeEndpoint($endpoint); + } + + $this->subscribeDiscovery('PING', fn (Message $msg) => $this->reply($msg, $this->pingResponse())); + $this->subscribeDiscovery('INFO', fn (Message $msg) => $this->reply($msg, $this->infoResponse())); + $this->subscribeDiscovery('STATS', fn (Message $msg) => $this->reply($msg, $this->statsResponse())); + + return $this; + } + + /** + * Start and then block, processing messages until stopped. + */ + public function run(): void + { + $this->start(); + $this->conn->wait(); + } + + public function stop(): void + { + foreach ($this->subscriptions as $sub) { + try { + $sub->unsubscribe(); + } catch (\Throwable) { + // ignore + } + } + $this->subscriptions = []; + $this->running = false; + } + + public function getId(): string + { + return $this->id; + } + + public function getName(): string + { + return $this->name; + } + + private function subscribeEndpoint(Endpoint $endpoint): void + { + $this->subscriptions[] = $this->conn->subscribe( + $endpoint->subject, + function (Message $msg) use ($endpoint): void { + $this->handleEndpoint($endpoint, $msg); + }, + $endpoint->queueGroup, + ); + } + + private function handleEndpoint(Endpoint $endpoint, Message $msg): void + { + $endpoint->numRequests++; + $start = hrtime(true); + + try { + $result = ($endpoint->handler)($msg); + $endpoint->processingTime += (int) (hrtime(true) - $start); + + if ($msg->replyTo !== null) { + $this->conn->publish($msg->replyTo, $result); + } + } catch (\Throwable $e) { + $endpoint->processingTime += (int) (hrtime(true) - $start); + $endpoint->numErrors++; + $endpoint->lastError = $e->getMessage(); + + if ($msg->replyTo !== null) { + $code = $e instanceof ServiceException ? $e->getErrorCode() : '500'; + $headers = new Headers(); + $headers->set('Nats-Service-Error', $e->getMessage()); + $headers->set('Nats-Service-Error-Code', $code); + $this->conn->publish($msg->replyTo, '', headers: $headers); + } + } + } + + private function subscribeDiscovery(string $verb, \Closure $callback): void + { + foreach ([ + self::API_PREFIX . ".{$verb}", + self::API_PREFIX . ".{$verb}.{$this->name}", + self::API_PREFIX . ".{$verb}.{$this->name}.{$this->id}", + ] as $subject) { + $this->subscriptions[] = $this->conn->subscribe($subject, $callback); + } + } + + private function reply(Message $msg, string $payload): void + { + if ($msg->replyTo !== null) { + $this->conn->publish($msg->replyTo, $payload); + } + } + + private function pingResponse(): string + { + return $this->encode([ + 'type' => self::TYPE_PREFIX . '.ping_response', + 'name' => $this->name, + 'id' => $this->id, + 'version' => $this->version, + 'metadata' => $this->metadataObject(), + ]); + } + + private function infoResponse(): string + { + return $this->encode([ + 'type' => self::TYPE_PREFIX . '.info_response', + 'name' => $this->name, + 'id' => $this->id, + 'version' => $this->version, + 'description' => $this->description, + 'metadata' => $this->metadataObject(), + 'endpoints' => array_map(fn (Endpoint $e): array => $e->info(), array_values($this->endpoints)), + ]); + } + + private function statsResponse(): string + { + return $this->encode([ + 'type' => self::TYPE_PREFIX . '.stats_response', + 'name' => $this->name, + 'id' => $this->id, + 'version' => $this->version, + 'started' => $this->started, + 'metadata' => $this->metadataObject(), + 'endpoints' => array_map(fn (Endpoint $e): array => $e->stats(), array_values($this->endpoints)), + ]); + } + + /** + * Encode service metadata as a JSON object (empty stays {} not []). + * + * @return array|\stdClass + */ + private function metadataObject(): array|\stdClass + { + return $this->metadata === [] ? new \stdClass() : $this->metadata; + } + + /** + * @param array $data + */ + private function encode(array $data): string + { + return json_encode($data, JSON_THROW_ON_ERROR); + } +} diff --git a/packages/nats/src/Services/ServiceException.php b/packages/nats/src/Services/ServiceException.php new file mode 100644 index 00000000000..d95041eae64 --- /dev/null +++ b/packages/nats/src/Services/ServiceException.php @@ -0,0 +1,24 @@ +errorCode; + } +} diff --git a/packages/nats/src/Subscription.php b/packages/nats/src/Subscription.php new file mode 100644 index 00000000000..9a4b4fde8c8 --- /dev/null +++ b/packages/nats/src/Subscription.php @@ -0,0 +1,148 @@ +pendingMessages = new \SplQueue(); + } + + public function setConnection(Connection $connection): void + { + $this->connection = $connection; + } + + public function nextMessage(?float $timeout = null): ?Message + { + $deadline = $timeout !== null ? microtime(true) + $timeout : null; + + while (true) { + // Drain anything already queued before blocking on the socket. + if (!$this->pendingMessages->isEmpty()) { + $msg = $this->pendingMessages->dequeue(); + $this->pendingBytes -= \strlen((string) $msg->data); + if ($this->pendingBytes < 0) { + $this->pendingBytes = 0; + } + $this->slowConsumerSignaled = false; + return $msg; + } + + if (!$this->active || !$this->connection instanceof \Utopia\NATS\Connection) { + return null; + } + + $remaining = $deadline !== null ? $deadline - microtime(true) : null; + if ($remaining !== null && $remaining <= 0) { + return null; + } + + $this->connection->processMessage($remaining); + } + } + + public function unsubscribe(?int $afterMessages = null): void + { + if ($this->connection instanceof \Utopia\NATS\Connection) { + $this->connection->unsubscribe($this, $afterMessages); + } + } + + public function isActive(): bool + { + return $this->active; + } + + public function deliver(Message $msg): void + { + if ($this->callback instanceof \Closure) { + $this->received++; + ($this->callback)($msg); + } else { + // Guard the pending queue against unbounded growth: a consumer that + // never drains its messages signals slow-consumer and the message is + // dropped rather than exhausting memory. + $msgBytes = \strlen($msg->data); + if ($this->pendingMessages->count() >= $this->pendingMsgsLimit + || ($this->pendingBytes + $msgBytes) > $this->pendingBytesLimit) { + $this->signalSlowConsumer(); + return; + } + + $this->received++; + $this->pendingBytes += $msgBytes; + $this->pendingMessages->enqueue($msg); + } + + if ($this->maxMessages !== null && $this->received >= $this->maxMessages) { + $this->active = false; + } + } + + public function getPendingCount(): int + { + return $this->pendingMessages->count(); + } + + public function getPendingBytes(): int + { + return $this->pendingBytes; + } + + private function signalSlowConsumer(): void + { + if ($this->slowConsumerSignaled) { + return; + } + + $this->slowConsumerSignaled = true; + + if ($this->onSlowConsumer instanceof \Closure) { + ($this->onSlowConsumer)($this); + } + } + + public function setMaxMessages(int $max): void + { + $this->maxMessages = $max; + if ($this->received >= $max) { + $this->active = false; + } + } + + public function setInactive(): void + { + $this->active = false; + } + + public function getReceived(): int + { + return $this->received; + } + + public function hasCallback(): bool + { + return $this->callback instanceof \Closure; + } +} diff --git a/packages/nats/src/Transport/SwooleTransport.php b/packages/nats/src/Transport/SwooleTransport.php new file mode 100644 index 00000000000..0454143f907 --- /dev/null +++ b/packages/nats/src/Transport/SwooleTransport.php @@ -0,0 +1,185 @@ + $tlsOptions */ + public function __construct( + private readonly bool $secure = false, + private readonly array $tlsOptions = [], + ) { + } + + public function connect(string $host, int $port, float $timeout): void + { + $flags = SWOOLE_SOCK_TCP | ($this->secure ? SWOOLE_SSL : 0); + $client = new Client($flags); + $client->set($this->buildSettings($timeout)); + + if (!$client->connect($host, $port, $timeout)) { + throw new ConnectionException("Failed to connect to {$host}:{$port}: [{$client->errCode}] {$client->errMsg}"); + } + + $this->client = $client; + } + + public function write(string $data): int + { + $client = $this->ensureConnected(); + $total = \strlen($data); + $written = 0; + + // Loop until every byte is on the wire: send() may accept a short write. + while ($written < $total) { + $chunk = $client->send($written === 0 ? $data : substr($data, $written)); + + if ($chunk === false || $chunk === 0) { + throw new ConnectionException("Failed to write to socket: [{$client->errCode}] {$client->errMsg}"); + } + + $written += $chunk; + } + + return $written; + } + + public function read(int $maxBytes, ?float $timeout = null): string + { + if ($this->buffer === '') { + $this->fill($timeout); + } + + $chunk = substr($this->buffer, 0, $maxBytes); + $this->buffer = substr($this->buffer, \strlen($chunk)); + + return $chunk; + } + + public function readLine(?float $timeout = null): string + { + $deadline = $timeout !== null ? microtime(true) + $timeout : null; + + while (($pos = strpos($this->buffer, "\n")) === false) { + $remaining = $deadline !== null ? max(0.0, $deadline - microtime(true)) : null; + $this->fill($remaining); + } + + $line = substr($this->buffer, 0, $pos + 1); + $this->buffer = substr($this->buffer, $pos + 1); + + return $line; + } + + public function upgradeTls(array $options): void + { + // Coroutine-native STARTTLS: handshake on the live connection, avoiding + // stream_socket_enable_crypto() (which misbehaves under Swoole hooks). + $client = $this->ensureConnected(); + $client->set($this->buildTlsSettings($options)); + + if (!$client->enableSSL()) { + throw new ConnectionException('Failed to upgrade connection to TLS'); + } + } + + public function isConnected(): bool + { + return $this->client instanceof \Swoole\Coroutine\Client && $this->client->isConnected(); + } + + public function close(): void + { + if ($this->client instanceof \Swoole\Coroutine\Client) { + $this->client->close(); + $this->client = null; + } + } + + private function fill(?float $timeout): void + { + $client = $this->ensureConnected(); + $data = $client->recv($timeout ?? -1); + + if ($data === '') { + throw new ConnectionException('Connection closed by server'); + } + + if ($data === false) { + // ponytail: Swoole sets errCode 110 (ETIMEDOUT) on a recv timeout; anything else is a hard error. + if ($client->errCode === 110) { + throw new TimeoutException('Read timed out'); + } + throw new ConnectionException("Failed to read from socket: [{$client->errCode}] {$client->errMsg}"); + } + + $this->buffer .= $data; + } + + private function ensureConnected(): Client + { + if (!$this->client instanceof \Swoole\Coroutine\Client) { + throw new ConnectionException('Not connected'); + } + + return $this->client; + } + + /** @return array */ + private function buildSettings(float $timeout): array + { + $settings = ['timeout' => $timeout]; + + if ($this->secure) { + $settings += $this->buildTlsSettings($this->tlsOptions); + } + + return $settings; + } + + /** + * @param array $options + * @return array + */ + private function buildTlsSettings(array $options): array + { + $settings = []; + + if (!empty($options['cafile'])) { + $settings['ssl_cafile'] = $options['cafile']; + } + if (!empty($options['local_cert'])) { + $settings['ssl_cert_file'] = $options['local_cert']; + } + if (!empty($options['local_pk'])) { + $settings['ssl_key_file'] = $options['local_pk']; + } + if (!empty($options['peer_name'])) { + $settings['ssl_host_name'] = $options['peer_name']; + } + $settings['ssl_verify_peer'] = $options['verify_peer'] ?? true; + + return $settings; + } +} diff --git a/packages/nats/src/Transport/TcpTransport.php b/packages/nats/src/Transport/TcpTransport.php new file mode 100644 index 00000000000..03d80b34086 --- /dev/null +++ b/packages/nats/src/Transport/TcpTransport.php @@ -0,0 +1,222 @@ + ['tcp_nodelay' => true]]), + ); + + if ($stream === false) { + throw new ConnectionException("Failed to connect to {$address}: [{$errno}] {$errstr}"); + } + + stream_set_blocking($stream, true); + $this->setTimeout($stream, $timeout); + $this->stream = $stream; + } + + public function write(string $data): int + { + $this->ensureConnected(); + $total = \strlen($data); + $written = 0; + + // Loop until every byte is on the wire: fwrite may perform a short write. + // Re-resolved on every pass rather than captured once. fwrite() is a + // yield point under Swoole's stream hooks, so a close() can land between + // two iterations of a short write -- and the next fwrite() on the + // handle the first pass captured raises TypeError, an \Error, which is + // precisely what the reconnect paths in Connection do not catch. The + // classification helpers below were hardened for this; the write itself + // was still holding the stale handle. + while ($written < $total) { + $chunk = @fwrite($this->ensureConnected(), substr($data, $written)); + + if ($chunk === false || $chunk === 0) { + if ($this->isTimedOut()) { + throw new TimeoutException('Write timed out'); + } + if ($this->isAtEof()) { + throw new ConnectionException('Connection closed by server'); + } + throw new ConnectionException('Failed to write to socket'); + } + + $written += $chunk; + } + + return $written; + } + + public function read(int $maxBytes, ?float $timeout = null): string + { + $stream = $this->ensureConnected(); + + if ($timeout !== null) { + $this->setTimeout($stream, $timeout); + } + + $data = @fread($stream, $maxBytes); + + if ($data === false) { + if ($this->isTimedOut()) { + throw new TimeoutException('Read timed out'); + } + throw new ConnectionException('Failed to read from socket'); + } + + if ($data === '' && $this->isAtEof()) { + throw new ConnectionException('Connection closed by server'); + } + + return $data; + } + + public function readLine(?float $timeout = null): string + { + $stream = $this->ensureConnected(); + + if ($timeout !== null) { + $this->setTimeout($stream, $timeout); + } + + $line = @fgets($stream); + + if ($line === false) { + if ($this->isTimedOut()) { + throw new TimeoutException('Read timed out'); + } + if ($this->isAtEof()) { + throw new ConnectionException('Connection closed by server'); + } + throw new ConnectionException('Failed to read line from socket'); + } + + return $line; + } + + public function upgradeTls(array $options): void + { + $stream = $this->ensureConnected(); + + $contextOptions = ['ssl' => []]; + + if (isset($options['cafile'])) { + $contextOptions['ssl']['cafile'] = $options['cafile']; + } + if (isset($options['local_cert'])) { + $contextOptions['ssl']['local_cert'] = $options['local_cert']; + } + if (isset($options['local_pk'])) { + $contextOptions['ssl']['local_pk'] = $options['local_pk']; + } + if (isset($options['peer_name'])) { + $contextOptions['ssl']['peer_name'] = $options['peer_name']; + } + $contextOptions['ssl']['verify_peer'] = $options['verify_peer'] ?? true; + $contextOptions['ssl']['verify_peer_name'] = $options['verify_peer_name'] ?? true; + + $context = stream_context_get_options($stream); + $merged = array_merge_recursive($context, $contextOptions); + foreach ($merged as $wrapper => $opts) { + foreach ($opts as $key => $value) { + stream_context_set_option($stream, $wrapper, $key, \is_array($value) ? end($value) : $value); + } + } + + $result = @stream_socket_enable_crypto($stream, true, STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT); + + if ($result !== true) { + throw new ConnectionException('Failed to upgrade connection to TLS'); + } + } + + public function isConnected(): bool + { + return \is_resource($this->stream) && !feof($this->stream); + } + + public function close(): void + { + if ($this->stream !== null) { + @fclose($this->stream); + $this->stream = null; + } + } + + /** @return resource */ + private function ensureConnected() + { + // A null stream means close() ran; a non-resource stream means it ran + // concurrently and left a closed handle behind. Both are "not + // connected", and the second must be caught here because every stream + // function raises TypeError on a closed resource -- an \Error, which + // the reconnect paths in Connection do not catch. + if (!\is_resource($this->stream)) { + throw new ConnectionException('Not connected'); + } + + return $this->stream; + } + + /** @param resource $stream */ + private function setTimeout($stream, float $timeout): void + { + $seconds = (int) $timeout; + $microseconds = (int) (($timeout - $seconds) * 1_000_000); + stream_set_timeout($stream, $seconds, $microseconds); + } + + /** + * Whether the stream hit its timeout. Re-reads the property instead of + * trusting the caller's copy: a concurrent close() invalidates the resource + * the caller captured, and stream_get_meta_data() would raise TypeError on + * it. A stream that is gone did not time out -- it is EOF. + */ + private function isTimedOut(): bool + { + if (!\is_resource($this->stream)) { + return false; + } + + return stream_get_meta_data($this->stream)['timed_out']; + } + + /** + * Whether the stream is at end of file. A closed or vanished stream counts + * as EOF so callers report "connection closed" rather than raising. + */ + private function isAtEof(): bool + { + if (!\is_resource($this->stream)) { + return true; + } + + return feof($this->stream); + } +} diff --git a/packages/nats/src/Transport/TlsTransport.php b/packages/nats/src/Transport/TlsTransport.php new file mode 100644 index 00000000000..c9ee802a19d --- /dev/null +++ b/packages/nats/src/Transport/TlsTransport.php @@ -0,0 +1,212 @@ + $this->buildSslOptions()]); + + $stream = @stream_socket_client( + $address, + $errno, + $errstr, + $timeout, + STREAM_CLIENT_CONNECT, + $context, + ); + + if ($stream === false) { + throw new ConnectionException("Failed to connect to {$address}: [{$errno}] {$errstr}"); + } + + stream_set_blocking($stream, true); + $seconds = (int) $timeout; + $microseconds = (int) (($timeout - $seconds) * 1_000_000); + stream_set_timeout($stream, $seconds, $microseconds); + + $this->stream = $stream; + } + + public function write(string $data): int + { + $this->ensureConnected(); + $total = \strlen($data); + $written = 0; + + // Loop until every byte is on the wire: fwrite may perform a short write. + // Re-resolved on every pass rather than captured once. fwrite() is a + // yield point under Swoole's stream hooks, so a close() can land between + // two iterations of a short write -- and the next fwrite() on the + // handle the first pass captured raises TypeError, an \Error, which is + // precisely what the reconnect paths in Connection do not catch. The + // classification helpers below were hardened for this; the write itself + // was still holding the stale handle. + while ($written < $total) { + $chunk = @fwrite($this->ensureConnected(), substr($data, $written)); + + if ($chunk === false || $chunk === 0) { + if ($this->isTimedOut()) { + throw new TimeoutException('Write timed out'); + } + if ($this->isAtEof()) { + throw new ConnectionException('Connection closed by server'); + } + throw new ConnectionException('Failed to write to TLS socket'); + } + + $written += $chunk; + } + + return $written; + } + + public function read(int $maxBytes, ?float $timeout = null): string + { + $stream = $this->ensureConnected(); + + if ($timeout !== null) { + $seconds = (int) $timeout; + $microseconds = (int) (($timeout - $seconds) * 1_000_000); + stream_set_timeout($stream, $seconds, $microseconds); + } + + $data = @fread($stream, $maxBytes); + + if ($data === false) { + if ($this->isTimedOut()) { + throw new TimeoutException('Read timed out'); + } + throw new ConnectionException('Failed to read from TLS socket'); + } + + if ($data === '' && $this->isAtEof()) { + throw new ConnectionException('Connection closed by server'); + } + + return $data; + } + + public function readLine(?float $timeout = null): string + { + $stream = $this->ensureConnected(); + + if ($timeout !== null) { + $seconds = (int) $timeout; + $microseconds = (int) (($timeout - $seconds) * 1_000_000); + stream_set_timeout($stream, $seconds, $microseconds); + } + + $line = @fgets($stream); + + if ($line === false) { + if ($this->isTimedOut()) { + throw new TimeoutException('Read timed out'); + } + if ($this->isAtEof()) { + throw new ConnectionException('Connection closed by server'); + } + throw new ConnectionException('Failed to read line from TLS socket'); + } + + return $line; + } + + public function upgradeTls(array $options): void + { + // Already TLS, nothing to do + } + + public function isConnected(): bool + { + return \is_resource($this->stream) && !feof($this->stream); + } + + public function close(): void + { + if ($this->stream !== null) { + @fclose($this->stream); + $this->stream = null; + } + } + + private function buildSslOptions(): array + { + $opts = [ + 'verify_peer' => $this->tlsOptions['verify_peer'] ?? true, + 'verify_peer_name' => $this->tlsOptions['verify_peer_name'] ?? true, + ]; + + if (isset($this->tlsOptions['cafile'])) { + $opts['cafile'] = $this->tlsOptions['cafile']; + } + if (isset($this->tlsOptions['local_cert'])) { + $opts['local_cert'] = $this->tlsOptions['local_cert']; + } + if (isset($this->tlsOptions['local_pk'])) { + $opts['local_pk'] = $this->tlsOptions['local_pk']; + } + if (isset($this->tlsOptions['peer_name'])) { + $opts['peer_name'] = $this->tlsOptions['peer_name']; + } + + return $opts; + } + + /** @return resource */ + private function ensureConnected() + { + // See TcpTransport::ensureConnected(): a concurrently closed stream is + // still a resource-typed property but raises TypeError on use, and that + // \Error bypasses the reconnect paths in Connection. + if (!\is_resource($this->stream)) { + throw new ConnectionException('Not connected'); + } + + return $this->stream; + } + + /** + * Whether the stream hit its timeout, re-reading the property rather than + * trusting a caller's copy that a concurrent close() may have invalidated. + */ + private function isTimedOut(): bool + { + if (!\is_resource($this->stream)) { + return false; + } + + return stream_get_meta_data($this->stream)['timed_out']; + } + + /** + * Whether the stream is at end of file. A closed or vanished stream counts + * as EOF so callers report "connection closed" rather than raising. + */ + private function isAtEof(): bool + { + if (!\is_resource($this->stream)) { + return true; + } + + return feof($this->stream); + } +} diff --git a/packages/nats/src/Transport/Transport.php b/packages/nats/src/Transport/Transport.php new file mode 100644 index 00000000000..bb899206c34 --- /dev/null +++ b/packages/nats/src/Transport/Transport.php @@ -0,0 +1,22 @@ + $tlsOptions ssl context options when $secure + * @param string $path HTTP request path for the upgrade (NATS default "/") + */ + public function __construct( + private readonly bool $secure = false, + private readonly array $tlsOptions = [], + private readonly string $path = '/', + ) { + } + + public function connect(string $host, int $port, float $timeout): void + { + $scheme = $this->secure ? 'tls' : 'tcp'; + $context = stream_context_create($this->secure ? ['ssl' => $this->tlsOptions] : []); + + $errno = 0; + $errstr = ''; + $stream = @stream_socket_client( + "{$scheme}://{$host}:{$port}", + $errno, + $errstr, + $timeout, + STREAM_CLIENT_CONNECT, + $context, + ); + + if ($stream === false) { + throw new ConnectionException("Failed to connect to {$host}:{$port}: [{$errno}] {$errstr}"); + } + + stream_set_blocking($stream, true); + $this->setTimeout($stream, $timeout); + $this->stream = $stream; + + $this->handshake($host, $port); + } + + public function write(string $data): int + { + $this->writeFrame($data); + + return \strlen($data); + } + + public function read(int $maxBytes, ?float $timeout = null): string + { + if ($timeout !== null) { + $this->setTimeout($this->ensureConnected(), $timeout); + } + + while ($this->buffer === '') { + $this->readFrame(); + } + + $chunk = substr($this->buffer, 0, $maxBytes); + $this->buffer = substr($this->buffer, \strlen($chunk)); + + return $chunk; + } + + public function readLine(?float $timeout = null): string + { + if ($timeout !== null) { + $this->setTimeout($this->ensureConnected(), $timeout); + } + + while (($pos = strpos($this->buffer, "\n")) === false) { + $this->readFrame(); + } + + $line = substr($this->buffer, 0, $pos + 1); + $this->buffer = substr($this->buffer, $pos + 1); + + return $line; + } + + public function upgradeTls(array $options): void + { + // wss:// negotiates TLS at connect; there is no in-band STARTTLS for WebSocket. + throw new ConnectionException('STARTTLS is not supported over WebSocket; use wss://'); + } + + public function isConnected(): bool + { + return \is_resource($this->stream) && !feof($this->stream); + } + + public function close(): void + { + if ($this->stream !== null) { + @fclose($this->stream); + $this->stream = null; + } + } + + private function handshake(string $host, int $port): void + { + $key = base64_encode(random_bytes(16)); + $request = "GET {$this->path} HTTP/1.1\r\n" + . "Host: {$host}:{$port}\r\n" + . "Upgrade: websocket\r\n" + . "Connection: Upgrade\r\n" + . "Sec-WebSocket-Key: {$key}\r\n" + . "Sec-WebSocket-Version: 13\r\n\r\n"; + $this->rawWrite($request); + + // Read the HTTP response headers up to the blank line, keeping any trailing + // bytes (the first WS frame) — they must not be swallowed. + $response = ''; + while (!str_contains($response, "\r\n\r\n")) { + $response .= $this->rawRead(1); + } + + if (!preg_match('#^HTTP/1\.1 101#i', $response)) { + $status = strtok($response, "\r\n"); + throw new ConnectionException("WebSocket upgrade failed: {$status}"); + } + + $expected = base64_encode(sha1($key . '258EAFA5-E914-47DA-95CA-C5AB0DC85B11', true)); + if (!preg_match('#Sec-WebSocket-Accept:\s*(.+?)\r\n#i', $response, $m) || trim($m[1]) !== $expected) { + throw new ConnectionException('WebSocket upgrade failed: bad Sec-WebSocket-Accept'); + } + } + + /** + * Read one WebSocket frame, handling control frames, and append data payloads + * (text/binary/continuation) to the buffer. + */ + private function readFrame(): void + { + $header = $this->rawRead(2); + $b0 = \ord($header[0]); + $b1 = \ord($header[1]); + + $opcode = $b0 & 0x0F; + $masked = ($b1 & 0x80) !== 0; + $len = $b1 & 0x7F; + + if ($len === 126) { + $ext = $this->rawRead(2); + $len = (\ord($ext[0]) << 8) | \ord($ext[1]); + } elseif ($len === 127) { + $ext = $this->rawRead(8); + $len = 0; + for ($i = 0; $i < 8; $i++) { + $len = ($len << 8) | \ord($ext[$i]); + } + } + + $maskKey = $masked ? $this->rawRead(4) : ''; + $payload = $len > 0 ? $this->rawRead($len) : ''; + + if ($masked && $payload !== '') { + $unmasked = ''; + for ($i = 0, $n = \strlen($payload); $i < $n; $i++) { + $unmasked .= $payload[$i] ^ $maskKey[$i % 4]; + } + $payload = $unmasked; + } + + switch ($opcode) { + case 0x0: // continuation + case 0x1: // text + case 0x2: // binary + $this->buffer .= $payload; + return; + case 0x8: // close + throw new ConnectionException('WebSocket closed by server'); + case 0x9: // ping -> pong + $this->writeFrame($payload, 0xA); + return; + case 0xA: // pong + return; + default: + throw new ConnectionException("Unexpected WebSocket opcode: {$opcode}"); + } + } + + private function writeFrame(string $payload, int $opcode = 0x2): void + { + $len = \strlen($payload); + $frame = \chr(0x80 | $opcode); // FIN + opcode + + if ($len < 126) { + $frame .= \chr(0x80 | $len); + } elseif ($len <= 0xFFFF) { + $frame .= \chr(0x80 | 126) . pack('n', $len); + } else { + $frame .= \chr(0x80 | 127) . pack('J', $len); + } + + $maskKey = random_bytes(4); + $frame .= $maskKey; + + $masked = ''; + for ($i = 0; $i < $len; $i++) { + $masked .= $payload[$i] ^ $maskKey[$i % 4]; + } + + $this->rawWrite($frame . $masked); + } + + private function rawWrite(string $data): void + { + $this->ensureConnected(); + $total = \strlen($data); + $written = 0; + + // Re-resolved on every pass rather than captured once. fwrite() is a + // yield point under Swoole's stream hooks, so a close() can land between + // two iterations of a short write -- and the next fwrite() on the + // handle the first pass captured raises TypeError, an \Error, which is + // precisely what the reconnect paths in Connection do not catch. The + // classification helpers below were hardened for this; the write itself + // was still holding the stale handle. + while ($written < $total) { + $chunk = @fwrite($this->ensureConnected(), substr($data, $written)); + if ($chunk === false || $chunk === 0) { + throw new ConnectionException('Failed to write to WebSocket'); + } + $written += $chunk; + } + } + + private function rawRead(int $length): string + { + $this->ensureConnected(); + $data = ''; + + // Re-resolved per pass for the same reason as rawWrite(): fread() yields, + // so a partial read spans a window in which close() can invalidate the + // handle this loop would otherwise keep using. + while (\strlen($data) < $length) { + $chunk = @fread($this->ensureConnected(), $length - \strlen($data)); + if ($chunk === false || $chunk === '') { + if ($this->isTimedOut()) { + throw new TimeoutException('Read timed out'); + } + throw new ConnectionException('Connection closed by server'); + } + $data .= $chunk; + } + + return $data; + } + + /** @return resource */ + private function ensureConnected() + { + // See TcpTransport::ensureConnected(): a concurrently closed stream is + // still a resource-typed property but raises TypeError on use, and that + // \Error bypasses the reconnect paths in Connection. + if (!\is_resource($this->stream)) { + throw new ConnectionException('Not connected'); + } + + return $this->stream; + } + + /** + * Whether the stream hit its timeout, re-reading the property rather than + * trusting a caller's copy that a concurrent close() may have invalidated. + */ + private function isTimedOut(): bool + { + if (!\is_resource($this->stream)) { + return false; + } + + return stream_get_meta_data($this->stream)['timed_out']; + } + + /** @param resource $stream */ + private function setTimeout($stream, float $timeout): void + { + $seconds = (int) $timeout; + $microseconds = (int) (($timeout - $seconds) * 1_000_000); + stream_set_timeout($stream, $seconds, $microseconds); + } +} diff --git a/packages/nats/tests/Auth/NKeyAuthTest.php b/packages/nats/tests/Auth/NKeyAuthTest.php new file mode 100644 index 00000000000..67cb2dabc73 --- /dev/null +++ b/packages/nats/tests/Auth/NKeyAuthTest.php @@ -0,0 +1,133 @@ +markTestSkipped('sodium extension required'); + } + } + + public function testDerivesPublicKeyMatchingSodium(): void + { + $rawSeed = random_bytes(32); + $keyPair = sodium_crypto_sign_seed_keypair($rawSeed); + $publicRaw = sodium_crypto_sign_publickey($keyPair); + + $seedString = $this->encodeUserSeed($rawSeed); + $auth = new NKeyAuth('', $seedString); + + $nkey = $auth->publicKey(); + + // A user public NKey is 'U' + base32(1 prefix byte + 32-byte key + 2 CRC). + $this->assertSame('U', $nkey[0]); + $this->assertSame(56, \strlen($nkey)); + + $decoded = $this->base32Decode($nkey); + $this->assertSame(35, \strlen($decoded)); + $this->assertSame(160, \ord($decoded[0]), 'user role prefix byte'); + // The 32-byte payload must be exactly sodium's public key. + $this->assertSame($publicRaw, substr($decoded, 1, 32)); + } + + public function testAuthenticateSignatureVerifiesAgainstDerivedKey(): void + { + $rawSeed = random_bytes(32); + $keyPair = sodium_crypto_sign_seed_keypair($rawSeed); + $publicRaw = sodium_crypto_sign_publickey($keyPair); + + $auth = new NKeyAuth('', $this->encodeUserSeed($rawSeed)); + $nonce = 'server-nonce-' . bin2hex(random_bytes(8)); + + $result = $auth->authenticate($nonce); + + $this->assertSame($auth->publicKey(), $result['nkey']); + $this->assertNotSame('', $result['nkey']); + + $signature = $this->base32Decode($result['sig']); + $this->assertTrue( + sodium_crypto_sign_verify_detached($signature, $nonce, $publicRaw), + 'signature must verify against the derived public key', + ); + } + + public function testExplicitPublicKeyIsPreserved(): void + { + $rawSeed = random_bytes(32); + $auth = new NKeyAuth('UEXPLICITKEY', $this->encodeUserSeed($rawSeed)); + + $this->assertSame('UEXPLICITKEY', $auth->publicKey()); + } + + /** + * Encode a raw 32-byte seed as a user NKey seed string. The seed's own CRC is + * never validated by the client, so a zero CRC is fine here. + */ + private function encodeUserSeed(string $rawSeed): string + { + $userRole = 160; // PrefixByteUser + $seedMarker = 144; // PrefixByteSeed + $b1 = $seedMarker | ($userRole >> 5); + $b2 = ($userRole & 31) << 3; + + $raw = \chr($b1) . \chr($b2) . $rawSeed . "\x00\x00"; + + return $this->base32Encode($raw); + } + + private function base32Encode(string $input): string + { + $output = ''; + $buffer = 0; + $bitsLeft = 0; + + for ($i = 0, $len = \strlen($input); $i < $len; $i++) { + $buffer = ($buffer << 8) | \ord($input[$i]); + $bitsLeft += 8; + while ($bitsLeft >= 5) { + $bitsLeft -= 5; + $output .= self::ALPHABET[($buffer >> $bitsLeft) & 0x1F]; + } + } + + if ($bitsLeft > 0) { + $output .= self::ALPHABET[($buffer << (5 - $bitsLeft)) & 0x1F]; + } + + return $output; + } + + private function base32Decode(string $input): string + { + $output = ''; + $buffer = 0; + $bitsLeft = 0; + + for ($i = 0, $len = \strlen($input); $i < $len; $i++) { + $val = strpos(self::ALPHABET, $input[$i]); + $buffer = ($buffer << 5) | $val; + $bitsLeft += 5; + if ($bitsLeft >= 8) { + $bitsLeft -= 8; + $output .= \chr(($buffer >> $bitsLeft) & 0xFF); + } + } + + return $output; + } +} diff --git a/packages/nats/tests/ConnectionDeathDetectionTest.php b/packages/nats/tests/ConnectionDeathDetectionTest.php new file mode 100644 index 00000000000..d6584f32070 --- /dev/null +++ b/packages/nats/tests/ConnectionDeathDetectionTest.php @@ -0,0 +1,615 @@ + $extra */ + private function connect(FakeTransport $fake, array $extra = []): Connection + { + return Connection::connect(new ConnectionOptions(...array_merge([ + 'servers' => 'nats://127.0.0.1:4222', + 'transportFactory' => fn (string $scheme): FakeTransport => $fake, + // Keep reconnects instant and free of jitter so the assertions below + // are about behaviour rather than timing. + 'reconnectWait' => 0.0, + 'reconnectJitter' => 0.0, + ], $extra))); + } + + // --- -ERR classification (pure) --- + /** + * @return \Iterator, array{string, bool}> + */ + public static function serverErrorProvider(): \Iterator + { + yield ['Stale Connection', true]; + yield ['stale connection', true]; + yield ['Slow Consumer', true]; + yield ['Maximum Payload Exceeded', true]; + yield ['Invalid Client Protocol', true]; + // Closing too, even though reconnecting after them is refused: whether + // the socket is gone and whether a retry can help are separate + // questions. See reconnectsAfterProvider(). + yield ['Authorization Violation', true]; + yield ['Authentication Expired', true]; + yield ['Secure Connection - TLS Required', true]; + yield ['Maximum Connections Exceeded', true]; + yield ['Unknown Protocol Operation', true]; + // Genuinely not connection-closing. + yield ["Permissions Violation for Publish to 'foo'", false]; + yield ['Maximum Subscriptions Exceeded', false]; + yield ['Invalid Subject', false]; + yield ['', false]; + } + + #[DataProvider('serverErrorProvider')] + public function testClosesConnectionClassifiesServerErrors(string $message, bool $expected): void + { + $this->assertSame($expected, Connection::closesConnection($message)); + } + + /** + * @return \Iterator, array{string, bool}> + */ + public static function reconnectsAfterProvider(): \Iterator + { + // Transient or infrastructural: a fresh connection is a real fix. + yield ['Stale Connection', true]; + yield ['Slow Consumer', true]; + yield ['Maximum Payload Exceeded', true]; + // Nothing a retry can change -- reconnecting would hot-loop against a + // server that refuses us identically every time. + yield ['Authorization Violation', false]; + yield ['Authentication Timeout', false]; + yield ['Authentication Expired', false]; + yield ['Secure Connection - TLS Required', false]; + yield ['Maximum Connections Exceeded', false]; + yield ['Unknown Protocol Operation', false]; + } + + #[DataProvider('reconnectsAfterProvider')] + public function testReconnectsAfterSeparatesRecoverableErrors(string $message, bool $expected): void + { + $this->assertSame($expected, Connection::reconnectsAfter($message)); + } + + // --- Finding: reconnect on a connection-closing -ERR --- + + public function testConnectionClosingErrorTriggersReconnect(): void + { + $reconnected = false; + $fake = new FakeTransport(); + $conn = $this->connect($fake, [ + 'onReconnect' => function () use (&$reconnected): void { + $reconnected = true; + }, + ]); + + $fake->pushInbound("-ERR 'Stale Connection'\r\n"); + + // The error still surfaces to the caller, but the connection underneath + // has been rebuilt rather than left dead-but-"connected". + try { + $conn->processMessage(1.0); + $this->fail('Expected the -ERR to surface'); + } catch (ProtocolException) { + // expected + } + + $this->assertTrue($reconnected, 'A connection-closing -ERR must reconnect'); + $this->assertTrue($conn->isConnected()); + + $conn->close(); + } + + public function testConnectionClosingErrorWithoutReconnectMarksConnectionDead(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake, ['allowReconnect' => false]); + + $fake->pushInbound("-ERR 'Stale Connection'\r\n"); + + try { + $conn->processMessage(1.0); + $this->fail('Expected the -ERR to surface'); + } catch (ProtocolException) { + // expected + } + + // The whole point: status no longer claims a usable connection, so the + // next publish raises instead of writing into a dead socket. + $this->assertFalse($conn->isConnected()); + $this->expectException(ConnectionException::class); + $conn->publish('foo', 'bar'); + } + + /** + * The regression the split predicates exist for. + * + * An authorization failure closes the connection server-side but must not + * be reconnected. When one predicate answered both questions, "do not + * reconnect" also meant "do not record it as closed", so the status stayed + * connected over a socket the server had already dropped -- and the next + * publish() wrote into it and returned success. Exactly the silent loss the + * -ERR handling was added to remove, on a different error. + */ + public function testUnrecoverableClosingErrorMarksConnectionDeadWithoutReconnecting(): void + { + $reconnected = false; + $fake = new FakeTransport(); + $conn = $this->connect($fake, [ + 'onReconnect' => function () use (&$reconnected): void { + $reconnected = true; + }, + ]); + + $fake->pushInbound("-ERR 'Authorization Violation'\r\n"); + + try { + $conn->processMessage(1.0); + $this->fail('Expected the -ERR to surface'); + } catch (AuthenticationException) { + // expected + } + + $this->assertFalse($reconnected, 'Credentials the server just rejected must not be retried'); + $this->assertFalse($conn->isConnected(), 'The socket is gone; the status must say so'); + + // The half that matters: the next write raises instead of vanishing. + $this->expectException(ConnectionException::class); + $conn->publish('foo', 'bar'); + } + + public function testNonClosingErrorLeavesConnectionIntact(): void + { + $reconnected = false; + $fake = new FakeTransport(); + $conn = $this->connect($fake, [ + 'onReconnect' => function () use (&$reconnected): void { + $reconnected = true; + }, + ]); + + $fake->pushInbound("-ERR 'Permissions Violation for Publish to \"foo\"'\r\n"); + + try { + $conn->processMessage(1.0); + $this->fail('Expected the -ERR to surface'); + } catch (PermissionException) { + // expected + } + + $this->assertFalse($reconnected, 'A permissions error must not recycle the connection'); + $this->assertTrue($conn->isConnected()); + + $conn->close(); + } + + // --- Finding: keepalive independent of the caller --- + + public function testTickSendsKeepalivePingWhenDue(): void + { + $fake = new FakeTransport(); + // pingInterval 0 makes the keepalive due on every check. + $conn = $this->connect($fake, ['pingInterval' => 0.0]); + + $before = $fake->written; + $conn->tick(); + + $this->assertStringContainsString( + "PING\r\n", + substr($fake->written, \strlen($before)), + 'tick() must drive the keepalive without a message being read', + ); + + $conn->close(); + } + + public function testTickRecyclesAConnectionThatStoppedAnsweringPings(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake, [ + 'pingInterval' => 0.0, + 'maxPingsOut' => 1, + ]); + + // Counted off the wire rather than from a callback: a recycled + // connection re-runs the handshake, so a fresh CONNECT is the effect + // worth asserting on. + $handshakes = fn (): int => substr_count($fake->written, 'CONNECT '); + $this->assertSame(1, $handshakes()); + + // The handshake needed its PONG; from here the server goes silent. + $fake->answerPings = false; + + // First tick sends the PING and collects no answer; the second sees the + // outstanding ping budget exhausted and recycles. + $conn->tick(); + $this->assertSame(1, $handshakes(), 'One unanswered ping is still within budget'); + + // The same silent server serves the reconnect, so it cannot complete + // either -- which is the honest outcome. A connection that cannot be + // rebuilt must raise rather than report itself healthy. + try { + $conn->tick(); + $this->fail('tick() must not return normally once the connection is dead'); + } catch (ConnectionException) { + // expected + } + + $this->assertGreaterThan( + 1, + $handshakes(), + 'An unanswered keepalive must recycle the connection', + ); + + $conn->close(); + } + + /** + * A publisher must not be able to out-ping itself. + * + * A PING is only half a keepalive: the read path is what clears the + * outstanding count, and a caller that only writes never reads. Emitting + * from the write path therefore accumulated one unanswered PING per + * interval on every write-only connection until the budget was spent, and + * then declared a perfectly healthy socket stale -- the keepalive causing + * the outage it exists to prevent, on a connection the server was watching + * publish the whole time. + * + * The server here answers nothing, standing in for a connection with no + * reader. What is asserted is that publishing alone neither emits a + * keepalive nor rebuilds the connection. + */ + public function testPublishingDoesNotAccumulateUnansweredPings(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake, [ + 'pingInterval' => 0.0, + 'maxPingsOut' => 1, + ]); + + $handshakes = fn (): int => substr_count($fake->written, 'CONNECT '); + $this->assertSame(1, $handshakes()); + + // Nothing will answer from here, exactly as on a connection whose + // holder only ever writes to it. + $fake->answerPings = false; + $before = \strlen($fake->written); + + for ($i = 0; $i < 5; $i++) { + $conn->publish('foo', 'bar'); + } + + $written = substr($fake->written, $before); + + $this->assertStringNotContainsString( + "PING\r\n", + $written, + 'The write path must take the keepalive verdict without emitting a PING nothing will collect', + ); + $this->assertSame( + 1, + $handshakes(), + 'A busy publisher must not reconnect: the server can see its traffic, so the socket is not idle', + ); + $this->assertTrue($conn->isConnected()); + + $conn->close(); + } + + + public function testTickCollectsItsOwnPongsSoAHealthyConnectionSurvives(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake, [ + 'pingInterval' => 0.0, + 'maxPingsOut' => 2, + ]); + + $handshakes = fn (): int => substr_count($fake->written, 'CONNECT '); + + // checkPings() only writes. Nothing else clears the outstanding count + // on a connection whose holder never reads -- a pooled publisher -- so + // without collecting its own PONGs the keepalive would march to + // maxPingsOut and declare a live server stale. Far more ticks than the + // budget, against a server answering every one of them. + for ($i = 0; $i < 20; $i++) { + $conn->tick(); + } + + $this->assertSame( + 1, + $handshakes(), + 'A server answering every ping must never be recycled, however long the connection idles', + ); + + // And it is still usable rather than merely un-recycled. + $conn->publish('subject', 'payload'); + $this->assertStringContainsString('PUB subject', $fake->written); + + $conn->close(); + } + + /** + * The write path takes the keepalive verdict before writing -- and only the + * verdict. + * + * This test used to assert that publish() emitted a PING ahead of the PUB, + * which is what the keepalive looked like when it was one method. But a + * PING is only half a keepalive: the read path clears the outstanding + * count, and a caller that only writes never reads, so emitting here made + * every write-only connection accumulate one unanswered PING per interval + * until the budget was spent and a healthy socket was declared stale. A + * connection that is actively publishing is not idle either -- the server + * can see its traffic -- so there was nothing for that PING to keep alive. + * + * What has to survive is the ordering intent: a connection already known to + * be dead must be recycled *before* the payload is written, so a publish + * never lands on a socket we have already given up on. That is asserted + * here, and {@see self::testPublishingDoesNotAccumulateUnansweredPings()} + * pins the half that was removed. + */ + public function testPublishTakesTheKeepaliveVerdictBeforeWriting(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake, [ + 'pingInterval' => 0.0, + 'maxPingsOut' => 1, + ]); + + $handshakes = fn (): int => substr_count($fake->written, 'CONNECT '); + + // Spend the budget from the read side, which is where it is spent in + // practice: a PING the now-silent server will never answer. + $fake->answerPings = false; + $conn->tick(); + $this->assertSame(1, $handshakes()); + + $offset = \strlen($fake->written); + + try { + $conn->publish('foo', 'bar'); + $this->fail('A publish onto a spent keepalive budget must not report success'); + } catch (ConnectionException) { + // The reconnect is served by the same silent server, so it cannot + // complete -- which is the honest outcome. + } + + $this->assertGreaterThan( + 1, + $handshakes(), + 'The write path must act on a connection whose keepalive budget is spent', + ); + $this->assertStringNotContainsString( + 'PUB foo', + substr($fake->written, $offset), + 'The payload must not reach a socket the keepalive had already condemned', + ); + } + + // --- Finding: transports must not raise TypeError on a closed stream --- + /** + * @return \Iterator, array{class-string}> + */ + public static function streamTransportProvider(): \Iterator + { + yield [TcpTransport::class]; + yield [TlsTransport::class]; + yield [WebSocketTransport::class]; + } + + /** + * A close() concurrent with a yielded fread/fwrite leaves a closed resource + * behind. Every stream function raises TypeError on one, and TypeError is an + * \Error -- so it bypasses the ConnectionException catches that drive + * reconnection. The transports must report it as a connection failure. + * + * @param class-string $class + */ + #[DataProvider('streamTransportProvider')] + public function testClosedStreamRaisesConnectionExceptionNotTypeError(string $class): void + { + $transport = $this->transportWithClosedStream($class); + + $this->expectException(ConnectionException::class); + $transport->write("PING\r\n"); + } + + /** + * @param class-string $class + */ + #[DataProvider('streamTransportProvider')] + public function testClosedStreamReadsRaiseConnectionExceptionNotTypeError(string $class): void + { + $transport = $this->transportWithClosedStream($class); + + $this->expectException(ConnectionException::class); + $transport->read(64); + } + + /** + * @param class-string $class + */ + #[DataProvider('streamTransportProvider')] + public function testClosedStreamIsNotReportedAsConnected(string $class): void + { + $transport = $this->transportWithClosedStream($class); + + // Previously feof() on the closed resource raised TypeError here. + $this->assertFalse($transport->isConnected()); + } + + /** + * The same defect, one call deeper: mid-loop rather than on entry. + * + * fwrite() performs short writes, so write() loops -- and under Swoole's + * stream hooks each fwrite() is a yield point, which makes the gap between + * two iterations a window a close() can land in. The loop captured its + * handle once before the first pass, so the next iteration wrote to the + * stale one and raised TypeError: an \Error, so it bypassed the + * ConnectionException catches that drive reconnection, exactly as the + * on-entry case did before it was fixed. + * + * A stream wrapper supplies the short write and the seam in one: PHP calls + * stream_write() repeatedly inside a single fwrite(), so accepting one byte + * and then refusing makes fwrite() report a genuine partial write, and the + * callback fired on the way past swaps the property for the closed handle a + * concurrent close() would leave behind. + * + * Both versions raise ConnectionException here, so the message is what + * tells them apart -- and it is the real distinction, not an incidental + * one. A loop that re-reads the property reports the connection gone; one + * writing to the copy it captured gets as far as the socket and blames the + * write. Before the fix there was no third outcome only because this test + * cannot run PHP's stream functions against a genuinely closed resource + * without them raising TypeError, which is the production symptom. + * + * @param class-string $class + */ + #[DataProvider('streamTransportProvider')] + public function testStreamClosedMidWriteRaisesConnectionExceptionNotTypeError(string $class): void + { + $transport = new $class(); + $property = new \ReflectionProperty($class, 'stream'); + + $stream = fopen('utopia-shortwrite://mid-loop', 'r+'); + $this->assertIsResource($stream); + $property->setValue($transport, $stream); + + ShortWriteStream::$afterFirstWrite = function () use ($transport, $property): void { + $dead = fopen('php://temp', 'r+'); + fclose($dead); + $property->setValue($transport, $dead); + }; + + try { + $transport->write("PING\r\n"); + $this->fail('A write onto a stream closed mid-loop must not report success'); + } catch (ConnectionException $e) { + $this->assertStringContainsString( + 'Not connected', + $e->getMessage(), + 'The loop must re-read the stream property, not write to the handle it captured', + ); + } finally { + ShortWriteStream::$afterFirstWrite = null; + } + } + + /** + * Build a transport holding a stream that has been closed underneath it, + * standing in for a close() that landed during a yielded socket call. + * + * @param class-string $class + */ + private function transportWithClosedStream(string $class): object + { + $transport = new $class(); + + $stream = fopen('php://temp', 'r+'); + $this->assertIsResource($stream); + + $property = new \ReflectionProperty($class, 'stream'); + $property->setValue($transport, $stream); + + fclose($stream); + + return $transport; + } +} + +/** + * A stream that accepts one byte per write, so a transport's short-write loop is + * guaranteed more than one pass, and that runs a callback after the first of + * them -- the seam a concurrent close() would land in. + */ +final class ShortWriteStream +{ + /** Fired once, after the first write, to stand in for the racing close(). */ + public static ?\Closure $afterFirstWrite = null; + + /** @var resource|null Set by PHP for stream context; unused here. */ + public $context; + + private int $writes = 0; + + public function stream_open(): bool + { + return true; + } + + public function stream_write(string $data): int + { + $this->writes++; + + // PHP drives stream_write() in a loop inside one fwrite() call until + // everything is accepted, so accepting a byte and then refusing is what + // makes fwrite() itself return short -- which is the only thing the + // transport's own loop responds to. + if ($this->writes > 1) { + return 0; + } + + if (self::$afterFirstWrite instanceof \Closure) { + $callback = self::$afterFirstWrite; + self::$afterFirstWrite = null; + $callback(); + } + + return min(1, \strlen($data)); + } + + public function stream_eof(): bool + { + return false; + } + + /** @return array */ + public function stream_stat(): array + { + return []; + } + + // PHP passes three arguments; none of them matter to a stream that only has + // to accept a timeout being set on it. + public function stream_set_option(): bool + { + return true; + } +} diff --git a/packages/nats/tests/ConnectionExtrasTest.php b/packages/nats/tests/ConnectionExtrasTest.php new file mode 100644 index 00000000000..d0ad2ced286 --- /dev/null +++ b/packages/nats/tests/ConnectionExtrasTest.php @@ -0,0 +1,113 @@ +assertInstanceOf( + AuthenticationException::class, + Connection::mapServerError('Authorization Violation'), + ); + } + + public function testUserAuthenticationExpiredMapsToAuthenticationException(): void + { + $this->assertInstanceOf( + AuthenticationException::class, + Connection::mapServerError('User Authentication Expired'), + ); + } + + public function testMaximumPayloadMapsToMaxPayloadException(): void + { + $this->assertInstanceOf( + MaxPayloadException::class, + Connection::mapServerError('Maximum Payload Exceeded'), + ); + } + + public function testPermissionsViolationForSubscriptionMapsToPermissionException(): void + { + $this->assertInstanceOf( + PermissionException::class, + Connection::mapServerError("Permissions Violation for Subscription to 'foo.bar'"), + ); + } + + public function testPermissionsViolationForPublishMapsToPermissionException(): void + { + $this->assertInstanceOf( + PermissionException::class, + Connection::mapServerError("Permissions Violation for Publish to 'foo.bar'"), + ); + } + + public function testUnknownErrorMapsToProtocolException(): void + { + $this->assertInstanceOf( + ProtocolException::class, + Connection::mapServerError('some unexpected error'), + ); + } + + public function testLameDuckInfoInvokesCallback(): void + { + $fired = false; + $fake = new FakeTransport(); + $conn = Connection::connect(new ConnectionOptions( + servers: 'nats://127.0.0.1:4222', + onLameDuck: function () use (&$fired): void { + $fired = true; + }, + transportFactory: fn (string $scheme): FakeTransport => $fake, + )); + + // Server signals lame-duck mode via an asynchronous INFO. With only one + // known server there is nowhere to fail over, so the callback fires and + // the connection is left intact. + $fake->pushInbound('INFO {"server_id":"FAKE","ldm":true}' . "\r\n"); + $conn->processMessage(1.0); + + $this->assertTrue($fired); + $this->assertFalse($conn->isReconnecting()); + + $conn->close(); + } + + public function testNonLameDuckInfoDoesNotInvokeCallback(): void + { + $fired = false; + $fake = new FakeTransport(); + $conn = Connection::connect(new ConnectionOptions( + servers: 'nats://127.0.0.1:4222', + onLameDuck: function () use (&$fired): void { + $fired = true; + }, + transportFactory: fn (string $scheme): FakeTransport => $fake, + )); + + $fake->pushInbound('INFO {"server_id":"FAKE","ldm":false}' . "\r\n"); + $conn->processMessage(1.0); + + $this->assertFalse($fired); + + $conn->close(); + } +} diff --git a/packages/nats/tests/ConnectionProtocolTest.php b/packages/nats/tests/ConnectionProtocolTest.php new file mode 100644 index 00000000000..faa0269395c --- /dev/null +++ b/packages/nats/tests/ConnectionProtocolTest.php @@ -0,0 +1,114 @@ + 'nats://127.0.0.1:4222', + 'transportFactory' => fn (string $scheme): FakeTransport => $fake, + ], $extra); + + return Connection::connect(new ConnectionOptions(...$args)); + } + + public function testConnectNegotiatesHeadersWhenSupported(): void + { + $fake = new FakeTransport(['headers' => true]); + $conn = $this->connect($fake); + + $payload = $fake->connectPayload(); + $this->assertTrue($payload['headers']); + $this->assertTrue($payload['no_responders']); + + $conn->close(); + } + + public function testConnectDisablesHeadersWhenNotSupported(): void + { + $fake = new FakeTransport(['headers' => false]); + $conn = $this->connect($fake); + + $payload = $fake->connectPayload(); + $this->assertFalse($payload['headers']); + $this->assertFalse($payload['no_responders']); + + $conn->close(); + } + + public function testPublishWithHeadersUsesHpub(): void + { + $fake = new FakeTransport(['headers' => true]); + $conn = $this->connect($fake); + + $headers = new Headers(); + $headers->set('X-Key', 'value'); + $conn->publish('subj', 'hello', null, $headers); + + $this->assertStringContainsString('HPUB subj', $fake->written); + + $conn->close(); + } + + public function testPublishWithHeadersRejectedWhenServerLacksSupport(): void + { + $fake = new FakeTransport(['headers' => false]); + $conn = $this->connect($fake); + + $headers = new Headers(); + $headers->set('X-Key', 'value'); + + $this->expectException(ProtocolException::class); + try { + $conn->publish('subj', 'hello', null, $headers); + } finally { + $conn->close(); + } + } + + public function testMaxPayloadIncludesHeaderBytes(): void + { + $fake = new FakeTransport(['headers' => true, 'max_payload' => 40]); + $conn = $this->connect($fake); + + $headers = new Headers(); + $headers->set('X-Long-Header', 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'); + + // Body alone (5 bytes) is well under 40, but header block + body exceeds it. + $this->assertGreaterThan(40, \strlen($headers->toWire()) + 5); + + $this->expectException(MaxPayloadException::class); + try { + $conn->publish('subj', 'hello', null, $headers); + } finally { + $conn->close(); + } + } + + public function testTlsAvailableIsParsed(): void + { + $fake = new FakeTransport(['tls_available' => true, 'tls_required' => false]); + $conn = $this->connect($fake); + + $this->assertTrue($conn->getServerInfo()->tlsAvailable); + $this->assertFalse($conn->getServerInfo()->tlsRequired); + + $conn->close(); + } +} diff --git a/packages/nats/tests/ConnectionTlsAndAuthTest.php b/packages/nats/tests/ConnectionTlsAndAuthTest.php new file mode 100644 index 00000000000..0be5769b400 --- /dev/null +++ b/packages/nats/tests/ConnectionTlsAndAuthTest.php @@ -0,0 +1,100 @@ + 'nats://127.0.0.1:4222', + 'transportFactory' => fn (string $scheme): FakeTransport => $fake, + ], $extra); + + return Connection::connect(new ConnectionOptions(...$args)); + } + + public function testTlsOptionDefaults(): void + { + $options = new ConnectionOptions(); + $this->assertTrue($options->tlsVerify); + $this->assertNull($options->tlsServerName); + } + + public function testConnectionTlsOptionsMapping(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake, [ + 'tlsVerify' => false, + 'tlsServerName' => 'example.com', + ]); + + $method = new \ReflectionMethod(Connection::class, 'tlsOptions'); + /** @var array $tls */ + $tls = $method->invoke($conn); + + $this->assertFalse($tls['verify_peer']); + $this->assertFalse($tls['verify_peer_name']); + $this->assertSame('example.com', $tls['peer_name']); + + $conn->close(); + } + + public function testTlsTransportAppliesVerifyAndSni(): void + { + $transport = new TlsTransport([ + 'verify_peer' => false, + 'verify_peer_name' => false, + 'peer_name' => 'example.com', + ]); + + $method = new \ReflectionMethod(TlsTransport::class, 'buildSslOptions'); + /** @var array $ssl */ + $ssl = $method->invoke($transport); + + $this->assertFalse($ssl['verify_peer']); + $this->assertFalse($ssl['verify_peer_name']); + $this->assertSame('example.com', $ssl['peer_name']); + } + + public function testTokenProviderResolvedAtConnect(): void + { + $calls = 0; + $fake = new FakeTransport(); + $conn = $this->connect($fake, [ + 'tokenProvider' => function () use (&$calls): string { + $calls++; + return 'token-' . $calls; + }, + ]); + + $this->assertSame(1, $calls, 'token provider invoked once at connect'); + $this->assertSame('token-1', $fake->connectPayload()['auth_token']); + + $conn->close(); + } + + public function testJwtProviderResolvedAtConnect(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake, [ + 'jwtProvider' => fn (): string => 'my.jwt.token', + ]); + + $this->assertSame('my.jwt.token', $fake->connectPayload()['jwt']); + + $conn->close(); + } +} diff --git a/packages/nats/tests/ConsumerFetchLifecycleTest.php b/packages/nats/tests/ConsumerFetchLifecycleTest.php new file mode 100644 index 00000000000..ccec0109b94 --- /dev/null +++ b/packages/nats/tests/ConsumerFetchLifecycleTest.php @@ -0,0 +1,138 @@ + $extra */ + private function connect(FakeTransport $fake, array $extra = []): Connection + { + return Connection::connect(new ConnectionOptions(...array_merge([ + 'servers' => 'nats://127.0.0.1:4222', + 'allowReconnect' => false, + 'transportFactory' => fn (string $scheme): FakeTransport => $fake, + ], $extra))); + } + + private function consumer(Connection $conn): Consumer + { + return new Consumer($conn, 'STREAM', ConsumerInfo::fromArray([ + 'stream_name' => 'STREAM', + 'name' => 'durable', + ])); + } + + /** + * @return array + */ + private function subscriptions(Connection $conn): array + { + $property = new \ReflectionProperty(Connection::class, 'subscriptions'); + + /** @var array $subs */ + $subs = $property->getValue($conn); + + return $subs; + } + + public function testFetchReleasesItsInboxSubscription(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake); + $consumer = $this->consumer($conn); + + $before = \count($this->subscriptions($conn)); + $consumer->fetch(1, 0.02); + + $this->assertCount( + $before, + $this->subscriptions($conn), + 'The inbox subscription must be released on the success path', + ); + } + + public function testFetchReleasesItsInboxSubscriptionWhenItThrows(): void + { + // A tiny max_payload makes the pull request itself unpublishable, so the + // throw lands after the inbox subscription is already open. + $fake = new FakeTransport(['max_payload' => 4]); + $conn = $this->connect($fake); + $consumer = $this->consumer($conn); + + $before = \count($this->subscriptions($conn)); + + try { + $consumer->fetch(1, 0.02); + $this->fail('Expected the oversized pull request to raise'); + } catch (MaxPayloadException) { + // expected + } + + $this->assertCount( + $before, + $this->subscriptions($conn), + 'A throw must not leak the inbox subscription', + ); + } + + public function testRepeatedFailedFetchesDoNotAccumulateSubscriptions(): void + { + $fake = new FakeTransport(['max_payload' => 4]); + $conn = $this->connect($fake); + $consumer = $this->consumer($conn); + + $before = \count($this->subscriptions($conn)); + + for ($i = 0; $i < 5; $i++) { + try { + $consumer->fetch(1, 0.02); + } catch (MaxPayloadException) { + // expected + } + } + + $this->assertCount( + $before, + $this->subscriptions($conn), + 'Subscription count must stay flat across repeated failures', + ); + } + + public function testServerPullExpiryLandsBeforeTheClientDeadline(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake); + $consumer = $this->consumer($conn); + + $timeout = 0.05; + $consumer->fetch(1, $timeout); + + $this->assertMatchesRegularExpression('/\{"batch":1,"expires":\d+\}/', $fake->written); + preg_match('/\{"batch":1,"expires":(\d+)\}/', $fake->written, $matches); + $expires = (int) $matches[1]; + + $this->assertGreaterThan(0, $expires, 'The server still needs a usable window'); + $this->assertLessThan( + StreamConfig::secondsToNanos($timeout), + $expires, + 'A message dispatched at a shared boundary is dropped client-side while the server counts the delivery', + ); + } +} diff --git a/packages/nats/tests/ConsumerPullRequestTest.php b/packages/nats/tests/ConsumerPullRequestTest.php new file mode 100644 index 00000000000..62f7c614dfb --- /dev/null +++ b/packages/nats/tests/ConsumerPullRequestTest.php @@ -0,0 +1,221 @@ + $fake, + )); + } + + private function consumer(Connection $conn): Consumer + { + return new Consumer($conn, 'STREAM', ConsumerInfo::fromArray([ + 'stream_name' => 'STREAM', + 'name' => 'durable', + ])); + } + + /** + * The JSON body of the pull request the client published. + * + * @return array + */ + private function pullRequest(FakeTransport $fake): array + { + $matched = preg_match( + '#PUB \$JS\.API\.CONSUMER\.MSG\.NEXT\.STREAM\.durable \S+ \d+\r\n(\{.*?\})\r\n#s', + $fake->written, + $m, + ); + $this->assertSame(1, $matched, 'No pull request was published'); + + /** @var array $decoded */ + $decoded = json_decode($m[1], true, 512, JSON_THROW_ON_ERROR); + + return $decoded; + } + + /** + * The sid fetch() is about to give its inbox subscription. + * + * Inbound frames are routed by sid, and a reply has to be queued on the fake + * before the synchronous fetch() starts reading -- by which point the inbox + * subject it generated is not observable yet. The sid is, because it comes + * from a counter. + */ + private function pendingSid(Connection $conn): string + { + return (string) (new \ReflectionProperty(Connection::class, 'nextSid'))->getValue($conn); + } + + /** Queue a status frame on the inbox fetch() is about to open. */ + private function pushStatus(FakeTransport $fake, string $sid, string $status, string $replyTo = ''): void + { + $block = "NATS/1.0 {$status}\r\n\r\n"; + $length = \strlen($block); + $reply = $replyTo === '' ? '' : " {$replyTo}"; + + $fake->pushInbound("HMSG _INBOX.scripted {$sid}{$reply} {$length} {$length}\r\n{$block}\r\n"); + } + + public function testNoWaitPullRequestCarriesNoExpiry(): void + { + // The server honours the expiry over no_wait: given both, it waits out + // the window and answers 408 instead of answering 404 at once, which + // makes a no_wait poll cost the full timeout it was meant to avoid. + $fake = new FakeTransport(); + $consumer = $this->consumer($this->connect($fake)); + + $consumer->fetch(1, 0.02, true); + + $request = $this->pullRequest($fake); + + $this->assertTrue($request['no_wait'] ?? null); + $this->assertArrayNotHasKey('expires', $request); + } + + public function testWaitingPullRequestStillCarriesAnExpiry(): void + { + // The other half of the same decision: without no_wait the server needs + // the expiry, or it holds the request for its own default. It lands + // just inside the client deadline -- see + // ConsumerFetchLifecycleTest::testServerPullExpiryLandsBeforeTheClientDeadline + // for why the two must not share a boundary. + $fake = new FakeTransport(); + $consumer = $this->consumer($this->connect($fake)); + + $consumer->fetch(1, 0.02); + + $request = $this->pullRequest($fake); + + $this->assertArrayNotHasKey('no_wait', $request); + $this->assertSame(18_000_000, $request['expires']); + } + + public function testBatchSizeAndMaxBytesSurviveTheNoWaitPath(): void + { + $fake = new FakeTransport(); + $consumer = $this->consumer($this->connect($fake)); + + $consumer->fetch(7, 0.02, true, 4096); + + $request = $this->pullRequest($fake); + + $this->assertSame(7, $request['batch']); + $this->assertSame(4096, $request['max_bytes']); + $this->assertTrue($request['no_wait'] ?? null); + $this->assertArrayNotHasKey('expires', $request); + } + + /** + * A 503 is what the server sends when the consumer's API subject has no + * responder at that instant -- a consumer still being created, or a raft + * leader moving. It was not among the codes fetch() named, so it fell + * through and was returned as a message with an empty body. + */ + public function testNoRespondersStatusIsNotReturnedAsAMessage(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake); + $consumer = $this->consumer($conn); + + $this->pushStatus($fake, $this->pendingSid($conn), '503 No Responders'); + + $batch = $consumer->fetch(1, 0.02, true); + + $this->assertCount(0, $batch, 'A 503 status frame is not a message'); + } + + /** + * The same for a status nobody has enumerated. The rule has to be "a status + * frame is not data", not a longer list of numbers, or the next code the + * server adds is returned as a job all over again. + */ + public function testAnUnrecognisedStatusIsNotReturnedAsAMessage(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake); + $consumer = $this->consumer($conn); + + $this->pushStatus($fake, $this->pendingSid($conn), '599 Something New'); + + $batch = $consumer->fetch(1, 0.02, true); + + $this->assertCount(0, $batch); + } + + public function testNoMessagesStatusEndsTheFetchWithoutAMessage(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake); + $consumer = $this->consumer($conn); + + $this->pushStatus($fake, $this->pendingSid($conn), '404 No Messages'); + + $batch = $consumer->fetch(1, 0.02, true); + + $this->assertCount(0, $batch); + } + + /** + * 100 is the one status that means carry on: it is a keep-alive, so it must + * not end the fetch, and a real message behind it still has to arrive. + */ + public function testFlowControlStatusIsAcknowledgedAndDoesNotEndTheFetch(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake); + $consumer = $this->consumer($conn); + + $sid = $this->pendingSid($conn); + $this->pushStatus($fake, $sid, '100 FlowControl Request', '_FC.reply'); + $fake->pushInbound("MSG _INBOX.scripted {$sid} 5\r\nhello\r\n"); + + $batch = $consumer->fetch(1, 0.5, true); + + $this->assertCount(1, $batch, 'A keep-alive must not end the fetch'); + foreach ($batch as $message) { + $this->assertSame('hello', $message->getData()); + } + $this->assertStringContainsString('PUB _FC.reply', $fake->written, 'The keep-alive is answered'); + } + + public function testAMessageWithoutAStatusIsStillDelivered(): void + { + $fake = new FakeTransport(); + $conn = $this->connect($fake); + $consumer = $this->consumer($conn); + + $fake->pushInbound("MSG _INBOX.scripted {$this->pendingSid($conn)} 7\r\npayload\r\n"); + + $batch = $consumer->fetch(1, 0.5, true); + + $this->assertCount(1, $batch); + foreach ($batch as $message) { + $this->assertSame('payload', $message->getData()); + } + } +} diff --git a/packages/nats/tests/E2E/ConnectionTest.php b/packages/nats/tests/E2E/ConnectionTest.php new file mode 100644 index 00000000000..12d348be2f1 --- /dev/null +++ b/packages/nats/tests/E2E/ConnectionTest.php @@ -0,0 +1,57 @@ +getServerUrl()); + $this->assertTrue($conn->isConnected()); + $this->assertFalse($conn->isClosed()); + + $info = $conn->getServerInfo(); + $this->assertNotEmpty($info->serverId); + $this->assertNotEmpty($info->version); + + $conn->close(); + $this->assertTrue($conn->isClosed()); + $this->assertFalse($conn->isConnected()); + } + + public function testConnectWithOptions(): void + { + $conn = Connection::connect(new ConnectionOptions( + servers: $this->getServerUrl(), + name: 'test-client', + verbose: false, + )); + + $this->assertTrue($conn->isConnected()); + $conn->close(); + } + + public function testFlush(): void + { + $conn = Connection::connect($this->getServerUrl()); + $conn->flush(); + $this->assertTrue($conn->isConnected()); + $conn->close(); + } +} diff --git a/packages/nats/tests/E2E/ConsumerNoWaitTest.php b/packages/nats/tests/E2E/ConsumerNoWaitTest.php new file mode 100644 index 00000000000..1a310d39d6e --- /dev/null +++ b/packages/nats/tests/E2E/ConsumerNoWaitTest.php @@ -0,0 +1,152 @@ + */ + private array $streams = []; + + private function getServerUrl(): string + { + return getenv('NATS_URL') ?: 'nats://127.0.0.1:4222'; + } + + protected function setUp(): void + { + $this->conn = Connection::connect($this->getServerUrl()); + $this->js = $this->conn->jetStream(); + } + + protected function tearDown(): void + { + foreach ($this->streams as $stream) { + try { + $this->js->deleteStream($stream); + } catch (\Throwable) { + // already gone + } + } + $this->conn->close(); + } + + /** @return array{0: string, 1: string} stream name and subject */ + private function createStream(): array + { + $name = 'NW_' . uniqid(); + $subject = "nowait.{$name}"; + + $this->js->createStream(new StreamConfig( + name: $name, + subjects: [$subject], + storage: StorageType::Memory, + )); + $this->streams[] = $name; + + return [$name, $subject]; + } + + public function testNoWaitFetchOnAnEmptyConsumerReturnsWithoutWaitingOutTheTimeout(): void + { + [$stream, $subject] = $this->createStream(); + $consumer = $this->js->createConsumer($stream, new ConsumerConfig( + durableName: 'nowait', + ackPolicy: AckPolicy::Explicit, + filterSubject: $subject, + )); + + // A deliberately long timeout, so "returned promptly" cannot be confused + // with "the timeout happened to be short". + $timeout = 2.0; + $started = microtime(true); + $batch = $consumer->fetch(1, $timeout, true); + $elapsed = microtime(true) - $started; + + $this->assertCount(0, $batch, 'The consumer is empty, so nothing comes back'); + $this->assertLessThan( + $timeout / 4, + $elapsed, + \sprintf( + 'A no_wait fetch on an empty consumer took %.3fs of a %.1fs timeout; it is asking the ' + . 'server to answer immediately, so it must not track the timeout', + $elapsed, + $timeout, + ), + ); + } + + public function testNoWaitFetchStillReturnsAMessageThatIsWaiting(): void + { + [$stream, $subject] = $this->createStream(); + $consumer = $this->js->createConsumer($stream, new ConsumerConfig( + durableName: 'nowait', + ackPolicy: AckPolicy::Explicit, + filterSubject: $subject, + )); + + $this->js->publish($subject, 'payload'); + + $batch = $consumer->fetch(1, 2.0, true); + + $this->assertCount(1, $batch, 'Answering immediately must not mean answering empty'); + foreach ($batch as $message) { + $this->assertSame('payload', $message->getData()); + $message->ack(); + } + } + + public function testWaitingFetchStillHonoursItsTimeoutOnAnEmptyConsumer(): void + { + [$stream, $subject] = $this->createStream(); + $consumer = $this->js->createConsumer($stream, new ConsumerConfig( + durableName: 'waiting', + ackPolicy: AckPolicy::Explicit, + filterSubject: $subject, + )); + + // The complement of the first test: without no_wait the caller asked to + // be held, and dropping the expiry from that request would leave the + // server holding it for its own default instead of the one given. + $timeout = 0.5; + $started = microtime(true); + $batch = $consumer->fetch(1, $timeout, false); + $elapsed = microtime(true) - $started; + + $this->assertCount(0, $batch); + $this->assertGreaterThanOrEqual( + $timeout * 0.5, + $elapsed, + 'A waiting fetch must actually wait', + ); + $this->assertLessThan( + $timeout * 3, + $elapsed, + 'A waiting fetch must return near its own timeout, not the server default', + ); + } +} diff --git a/packages/nats/tests/E2E/HardeningTest.php b/packages/nats/tests/E2E/HardeningTest.php new file mode 100644 index 00000000000..e02e1064f1f --- /dev/null +++ b/packages/nats/tests/E2E/HardeningTest.php @@ -0,0 +1,80 @@ +getServerUrl(), + drainTimeout: 20.0, + )); + + $subject = 'test.drain.' . uniqid(); + $received = 0; + $conn->subscribe($subject, function () use (&$received): void { + $received++; + }); + + $count = 25; + for ($i = 0; $i < $count; $i++) { + $conn->publish($subject, "msg-{$i}"); + } + + $start = microtime(true); + $conn->drain(); + $elapsed = microtime(true) - $start; + + $this->assertSame($count, $received, 'all pending messages drained'); + $this->assertTrue($conn->isClosed()); + $this->assertLessThan(10.0, $elapsed, 'drain completed on the PONG barrier, not the timeout'); + } + + public function testServerInfoTlsAvailableParsed(): void + { + $conn = Connection::connect($this->getServerUrl()); + $info = $conn->getServerInfo(); + + // Plaintext dev server: tls_available is parsed and reflects that TLS is + // neither available nor required on this connection. + $this->assertFalse($info->tlsAvailable); + $this->assertFalse($info->tlsRequired); + + $conn->close(); + } + + public function testTokenProviderInvokedAtConnect(): void + { + $calls = 0; + $conn = Connection::connect(new ConnectionOptions( + servers: $this->getServerUrl(), + tokenProvider: function () use (&$calls): string { + $calls++; + return 'dynamic-token'; + }, + )); + + $this->assertTrue($conn->isConnected()); + $this->assertSame(1, $calls, 'token provider resolved during the connect handshake'); + + $conn->close(); + } +} diff --git a/packages/nats/tests/E2E/JetStreamExtraTest.php b/packages/nats/tests/E2E/JetStreamExtraTest.php new file mode 100644 index 00000000000..cbf6039a739 --- /dev/null +++ b/packages/nats/tests/E2E/JetStreamExtraTest.php @@ -0,0 +1,238 @@ + */ + private array $streams = []; + + private function getServerUrl(): string + { + return getenv('NATS_URL') ?: 'nats://127.0.0.1:4222'; + } + + protected function setUp(): void + { + $this->conn = Connection::connect($this->getServerUrl()); + $this->js = $this->conn->jetStream(); + } + + protected function tearDown(): void + { + foreach ($this->streams as $stream) { + try { + $this->js->deleteStream($stream); + } catch (\Throwable) { + // already gone + } + } + $this->conn->close(); + } + + private function createStream(string $subject): string + { + $name = 'JSX_' . uniqid(); + $this->js->createStream(new StreamConfig( + name: $name, + subjects: [$subject], + storage: StorageType::Memory, + )); + $this->streams[] = $name; + return $name; + } + + public function testPushConsumerDeliversAndAcks(): void + { + $id = uniqid(); + $subject = "push.{$id}"; + $stream = $this->createStream($subject); + + /** @var list $received */ + $received = []; + $push = $this->js->pushSubscribe( + $stream, + new ConsumerConfig(ackPolicy: AckPolicy::Explicit), + function (JetStreamMessage $msg) use (&$received): void { + $received[] = $msg; + $msg->ack(); + }, + ); + + $this->js->publish($subject, 'hello-push'); + + $deadline = microtime(true) + 3.0; + while ($received === [] && microtime(true) < $deadline) { + $this->conn->processMessage(0.5); + } + + $this->assertCount(1, $received); + $this->assertSame('hello-push', $received[0]->getData()); + $this->assertSame($subject, $received[0]->getSubject()); + + // The ack (fire-and-forget) should clear the pending count shortly. + $ackPending = 1; + $deadline = microtime(true) + 3.0; + while ($ackPending !== 0 && microtime(true) < $deadline) { + $ackPending = $this->js->getConsumer($stream, $push->getConsumerName())->info()->numAckPending; + if ($ackPending !== 0) { + usleep(100_000); + } + } + $this->assertSame(0, $ackPending); + + $push->unsubscribe(); + } + + public function testOrderedConsumerPreservesSequence(): void + { + $id = uniqid(); + $subject = "ordered.{$id}"; + $stream = $this->createStream($subject); + + $count = 25; + for ($i = 1; $i <= $count; $i++) { + $this->js->publish($subject, "msg-{$i}"); + } + + $ordered = $this->js->orderedConsumer($stream); + + $streamSeqs = []; + $consumerSeqs = []; + for ($i = 0; $i < $count; $i++) { + $msg = $ordered->next(3.0); + $this->assertInstanceOf(JetStreamMessage::class, $msg); + $meta = $msg->metadata(); + $streamSeqs[] = $meta->streamSequence; + $consumerSeqs[] = $meta->consumerSequence; + } + + $this->assertSame(range(1, $count), $streamSeqs, 'stream sequences must be continuous and in order'); + $this->assertSame(range(1, $count), $consumerSeqs, 'consumer delivery sequences must be continuous'); + + $ordered->stop(); + } + + public function testAckSyncConfirmsAndPreventsRedelivery(): void + { + $id = uniqid(); + $subject = "acksync.{$id}"; + $stream = $this->createStream($subject); + + $this->js->publish($subject, 'ack-me'); + + $consumer = $this->js->createConsumer($stream, new ConsumerConfig( + durableName: 'c_' . $id, + ackPolicy: AckPolicy::Explicit, + ackWait: 1.0, + )); + + $batch = $consumer->fetch(1, 2.0); + $messages = $batch->getMessages(); + $this->assertCount(1, $messages); + + $start = microtime(true); + $messages[0]->ackSync(2.0); + $elapsed = microtime(true) - $start; + $this->assertLessThan(2.0, $elapsed, 'ackSync must return before its timeout'); + + // Wait past ack_wait, then confirm the message is not redelivered. + usleep(1_300_000); + $again = $consumer->fetch(1, 1.0); + $this->assertCount(0, $again->getMessages(), 'acked message must not be redelivered'); + } + + public function testAckBatchConfirmsOnlySelectedMessages(): void + { + $id = uniqid(); + $subject = "ackbatch.{$id}"; + $stream = $this->createStream($subject); + foreach (['first', 'unfinished', 'third'] as $data) { + $this->js->publish($subject, $data); + } + $consumer = $this->js->createConsumer($stream, new ConsumerConfig( + durableName: 'c_' . $id, + ackPolicy: AckPolicy::Explicit, + ackWait: 0.2, + )); + $messages = $consumer->fetch(3, 2.0)->getMessages(); + $this->assertCount(3, $messages); + $confirmed = []; + $this->js->ackBatch([$messages[0], $messages[2]], static function (int $index, ?\Throwable $error) use (&$confirmed): void { + $confirmed[$index] = $error; + }); + ksort($confirmed); + $this->assertSame([null, null], $confirmed); + usleep(300_000); + $remaining = $consumer->fetch(3, 0.5, noWait: true)->getMessages(); + $this->assertCount(1, $remaining); + $this->assertSame('unfinished', $remaining[0]->getData()); + $remaining[0]->ackSync(); + } + + public function testMetadataStreamOpsAndNumPending(): void + { + $id = uniqid(); + $subject = "meta.{$id}"; + $stream = $this->createStream($subject); + + $seqs = []; + for ($i = 1; $i <= 3; $i++) { + $ack = $this->js->publish($subject, "payload-{$i}"); + $seqs[] = $ack->sequence; + } + $this->assertSame([1, 2, 3], $seqs, 'PubAck sequence numbers must be populated'); + + // getMessage by sequence returns the exact payload/subject. + $msg = $this->js->getMessage($stream, 2); + $this->assertSame('payload-2', $msg->data); + $this->assertSame($subject, $msg->subject); + $this->assertSame(2, $msg->sequence); + + // Create a consumer and consume 2 of 3 without acking. + $consumer = $this->js->createConsumer($stream, new ConsumerConfig( + durableName: 'm_' . $id, + ackPolicy: AckPolicy::Explicit, + ackWait: 30.0, + )); + + $before = $consumer->info(true); + $this->assertSame(3, $before->numPending, 'all messages pending before consumption'); + + $batch = $consumer->fetch(2, 2.0); + $this->assertCount(2, $batch->getMessages()); + + $after = $consumer->info(true); + $this->assertSame(2, $after->numAckPending, 'delivered-but-unacked count'); + $this->assertSame(1, $after->numPending, 'one message still awaiting delivery'); + $this->assertSame(2, $after->delivered->consumerSeq, 'expanded delivered metadata populated'); + + // no_wait fetch returns immediately with the remaining message. + $remaining = $consumer->fetch(10, 2.0, noWait: true); + $this->assertCount(1, $remaining->getMessages()); + + // deleteMessage removes it from the stream. + $this->js->deleteMessage($stream, 1); + $this->expectException(JetStreamException::class); + $this->js->getMessage($stream, 1); + } +} diff --git a/packages/nats/tests/E2E/JetStreamParityTest.php b/packages/nats/tests/E2E/JetStreamParityTest.php new file mode 100644 index 00000000000..c554a3c02ed --- /dev/null +++ b/packages/nats/tests/E2E/JetStreamParityTest.php @@ -0,0 +1,259 @@ + */ + private array $streams = []; + + private function getServerUrl(): string + { + return getenv('NATS_URL') ?: 'nats://127.0.0.1:4222'; + } + + protected function setUp(): void + { + $this->conn = Connection::connect($this->getServerUrl()); + $this->js = $this->conn->jetStream(); + } + + protected function tearDown(): void + { + foreach ($this->streams as $stream) { + try { + $this->js->deleteStream($stream); + } catch (\Throwable) { + // already gone + } + } + $this->conn->close(); + } + + private function track(string $name): string + { + $this->streams[] = $name; + return $name; + } + + public function testStreamMetadataRepublishAndSubjectTransform(): void + { + $id = uniqid(); + $name = $this->track("PARITY_MRT_{$id}"); + + $stream = $this->js->createStream(new StreamConfig( + name: $name, + subjects: ["rp.{$id}.>"], + storage: StorageType::Memory, + metadata: ['env' => 'test', 'team' => 'core'], + republish: new Republish(source: "rp.{$id}.>", destination: "pub.{$id}.>", headersOnly: false), + subjectTransform: new SubjectTransform(source: "rp.{$id}.>", destination: "rp.{$id}.>"), + )); + + $config = $stream->info()->config; + + $this->assertIsArray($config->metadata); + $this->assertSame('test', $config->metadata['env'] ?? null); + $this->assertSame('core', $config->metadata['team'] ?? null); + + $this->assertInstanceOf(Republish::class, $config->republish); + $this->assertSame("rp.{$id}.>", $config->republish->source); + $this->assertSame("pub.{$id}.>", $config->republish->destination); + + $this->assertInstanceOf(SubjectTransform::class, $config->subjectTransform); + $this->assertSame("rp.{$id}.>", $config->subjectTransform->source); + + // Republish must forward stored messages to the destination subject. + $sub = $this->conn->subscribe("pub.{$id}.>"); + $this->js->publish("rp.{$id}.one", 'hello-republish'); + + $received = $sub->nextMessage(3.0); + $this->assertInstanceOf(\Utopia\NATS\Message::class, $received, 'republish should forward the message'); + $this->assertSame('hello-republish', $received->data); + $sub->unsubscribe(); + } + + public function testMirrorAndSourceReflectAndReplicate(): void + { + $id = uniqid(); + $origin = $this->track("PARITY_ORIG_{$id}"); + $this->js->createStream(new StreamConfig( + name: $origin, + subjects: ["ev.{$id}.>"], + storage: StorageType::Memory, + )); + + for ($i = 1; $i <= 5; $i++) { + $this->js->publish("ev.{$id}.a", "m-{$i}"); + } + + // Mirror. + $mirrorName = $this->track("PARITY_MIR_{$id}"); + $mirror = $this->js->createStream(new StreamConfig( + name: $mirrorName, + storage: StorageType::Memory, + mirror: new StreamSource(name: $origin), + )); + $this->assertInstanceOf(StreamSource::class, $mirror->info()->config->mirror); + $this->assertSame($origin, $mirror->info()->config->mirror->name); + + // Source. + $sourceName = $this->track("PARITY_SRC_{$id}"); + $sourced = $this->js->createStream(new StreamConfig( + name: $sourceName, + storage: StorageType::Memory, + sources: [new StreamSource(name: $origin)], + )); + $sources = $sourced->info()->config->sources; + $this->assertIsArray($sources); + $this->assertCount(1, $sources); + $this->assertSame($origin, $sources[0]->name); + + // Both should replicate the 5 origin messages. + $mirrorMsgs = $this->waitForMessages($mirrorName, 5); + $sourceMsgs = $this->waitForMessages($sourceName, 5); + $this->assertSame(5, $mirrorMsgs, 'mirror must replicate all origin messages'); + $this->assertSame(5, $sourceMsgs, 'source must replicate all origin messages'); + } + + private function waitForMessages(string $stream, int $expected): int + { + $count = 0; + $deadline = microtime(true) + 5.0; + while ($count < $expected && microtime(true) < $deadline) { + $count = $this->js->getStreamInfo($stream)->state->messages; + if ($count < $expected) { + usleep(100_000); + } + } + return $count; + } + + public function testConsumerMetadataIsSurfaced(): void + { + $id = uniqid(); + $name = $this->track("PARITY_CM_{$id}"); + $this->js->createStream(new StreamConfig( + name: $name, + subjects: ["cm.{$id}.>"], + storage: StorageType::Memory, + )); + + $consumer = $this->js->createConsumer($name, new ConsumerConfig( + durableName: "dur_{$id}", + metadata: ['owner' => 'billing', 'tier' => 'gold'], + )); + + $info = $consumer->info(); + $this->assertIsArray($info->metadata); + $this->assertSame('billing', $info->metadata['owner'] ?? null); + $this->assertSame('gold', $info->metadata['tier'] ?? null); + $this->assertSame('billing', $info->config->metadata['owner'] ?? null); + } + + public function testGetConsumersReturnsTypedInfos(): void + { + $id = uniqid(); + $name = $this->track("PARITY_CL_{$id}"); + $this->js->createStream(new StreamConfig( + name: $name, + subjects: ["cl.{$id}.>"], + storage: StorageType::Memory, + )); + + $this->js->createConsumer($name, new ConsumerConfig(durableName: "one_{$id}")); + $this->js->createConsumer($name, new ConsumerConfig(durableName: "two_{$id}")); + + $consumers = $this->js->getConsumers($name); + $this->assertCount(2, $consumers); + foreach ($consumers as $c) { + $this->assertInstanceOf(ConsumerInfo::class, $c); + $this->assertSame($name, $c->streamName); + } + + $names = array_map(static fn (ConsumerInfo $c): string => $c->name, $consumers); + sort($names); + $this->assertSame(["one_{$id}", "two_{$id}"], $names); + } + + public function testPerMessageTtl(): void + { + $id = uniqid(); + $name = $this->track("PARITY_TTL_{$id}"); + + $config = new StreamConfig( + name: $name, + subjects: ["ttl.{$id}.>"], + storage: StorageType::Memory, + allowMsgTtl: true, + subjectDeleteMarkerTtl: 2.0, + ); + + // Client serialization must carry the ADR-43 keys regardless of server support. + $wire = $config->toArray(); + $this->assertTrue($wire['allow_msg_ttl']); + $this->assertSame(StreamConfig::secondsToNanos(2.0), $wire['subject_delete_marker_ttl']); + + $this->js->createStream($config); + + // Publish with a per-message TTL; the Nats-TTL header must be stored. + $ack = $this->js->publish("ttl.{$id}.a", 'expires-soon', ttl: 2); + $this->assertSame(1, $ack->sequence); + + $stored = $this->js->getMessage($name, 1); + $this->assertInstanceOf(\Utopia\NATS\Headers::class, $stored->headers); + $this->assertSame('2s', $stored->headers->get('Nats-TTL')); + + // The message must actually expire (server is NATS 2.11+, per docker-compose). + $expired = false; + $deadline = microtime(true) + 6.0; + while (microtime(true) < $deadline) { + if ($this->js->getStreamInfo($name)->state->messages === 0) { + $expired = true; + break; + } + usleep(200_000); + } + $this->assertTrue($expired, 'message with TTL must expire'); + } + + public function testAccountInfoReturnsTypedObject(): void + { + $id = uniqid(); + $name = $this->track("PARITY_ACC_{$id}"); + $this->js->createStream(new StreamConfig( + name: $name, + subjects: ["acc.{$id}.>"], + storage: StorageType::Memory, + )); + + $info = $this->js->accountInfo(); + $this->assertInstanceOf(AccountInfo::class, $info); + $this->assertGreaterThanOrEqual(1, $info->streams); + $this->assertGreaterThanOrEqual(0, $info->memory); + $this->assertGreaterThanOrEqual(0, $info->apiTotal); + $this->assertArrayHasKey('memory', $info->raw); + } +} diff --git a/packages/nats/tests/E2E/JetStreamPublishManyTest.php b/packages/nats/tests/E2E/JetStreamPublishManyTest.php new file mode 100644 index 00000000000..4f995a1efc7 --- /dev/null +++ b/packages/nats/tests/E2E/JetStreamPublishManyTest.php @@ -0,0 +1,159 @@ +conn = Connection::connect(getenv('NATS_URL') ?: 'nats://127.0.0.1:4222'); + $this->js = $this->conn->jetStream(); + + try { + $this->js->deleteStream(self::STREAM); + } catch (\Throwable) { + // not there yet + } + + $this->js->createStream(new StreamConfig( + name: self::STREAM, + subjects: ['publishmany.>'], + duplicateWindow: 120, + )); + } + + protected function tearDown(): void + { + try { + $this->js->deleteStream(self::STREAM); + } catch (\Throwable) { + // already gone + } + $this->conn->close(); + } + + public function testEmptyBatchPublishesNothing(): void + { + $this->assertSame([], $this->js->publishMany([])); + $this->assertSame(0, $this->js->getStreamInfo(self::STREAM)->state->messages); + } + + public function testRejectsAWindowBelowOne(): void + { + $this->expectException(\InvalidArgumentException::class); + $this->js->publishMany([['subject' => 'publishmany.a']], window: 0); + } + + /** + * The acknowledgments come back in whatever order the server stored the + * messages, so this reads each message back out of the stream at the + * sequence its acknowledgment reported. A batch that returned the right + * count while mixing up which acknowledgment belonged to which message + * fails here and passes on a count assertion alone. + */ + public function testEachAcknowledgementBelongsToItsOwnMessage(): void + { + $messages = []; + for ($i = 0; $i < 50; $i++) { + $messages[] = ['subject' => 'publishmany.a', 'data' => "payload-{$i}"]; + } + + // A window smaller than the batch so the collection runs more than once. + $acks = $this->js->publishMany($messages, window: 8); + + $this->assertCount(50, $acks); + + foreach ($acks as $index => $ack) { + $this->assertSame(self::STREAM, $ack->stream); + $this->assertSame( + "payload-{$index}", + $this->js->getMessage(self::STREAM, $ack->sequence)->data, + "acknowledgment {$index} points at sequence {$ack->sequence}, which holds another message", + ); + } + } + + public function testRepublishedMessageIdsComeBackAsDuplicates(): void + { + $messages = []; + for ($i = 0; $i < 10; $i++) { + $messages[] = ['subject' => 'publishmany.a', 'data' => "payload-{$i}", 'msgId' => "id-{$i}"]; + } + + $first = $this->js->publishMany($messages); + foreach ($first as $ack) { + $this->assertFalse($ack->duplicate); + } + + $second = $this->js->publishMany($messages); + foreach ($second as $ack) { + $this->assertTrue($ack->duplicate); + } + + // The duplicates collapsed rather than double-delivering. + $this->assertSame(10, $this->js->getStreamInfo(self::STREAM)->state->messages); + } + + public function testHeadersReachTheStream(): void + { + $headers = new Headers(); + $headers->set('X-Trace', 'abc123'); + + $acks = $this->js->publishMany([ + ['subject' => 'publishmany.a', 'data' => 'with-headers', 'headers' => $headers], + ]); + + $stored = $this->js->getMessage(self::STREAM, $acks[0]->sequence); + $this->assertSame('abc123', $stored->headers?->get('X-Trace')); + } + + /** + * A server-side rejection of one message has to reach the caller. The batch + * writes every message before reading any acknowledgment, so the rejection + * arrives while other messages are still in flight. + */ + public function testARejectedMessageThrows(): void + { + $messages = [ + ['subject' => 'publishmany.a', 'data' => 'fine'], + ['subject' => 'publishmany.a', 'data' => 'rejected', 'expectedStream' => 'NO_SUCH_STREAM'], + ['subject' => 'publishmany.a', 'data' => 'also-fine'], + ]; + + $this->expectException(JetStreamException::class); + $this->js->publishMany($messages); + } + + public function testTheConnectionStaysUsableAfterARejection(): void + { + try { + $this->js->publishMany([ + ['subject' => 'publishmany.a', 'data' => 'rejected', 'expectedStream' => 'NO_SUCH_STREAM'], + ]); + $this->fail('expected the rejection to throw'); + } catch (JetStreamException) { + // the reply subscription is torn down on the way out + } + + $acks = $this->js->publishMany([['subject' => 'publishmany.a', 'data' => 'after']]); + $this->assertSame('after', $this->js->getMessage(self::STREAM, $acks[0]->sequence)->data); + } +} diff --git a/packages/nats/tests/E2E/KeyValueWatchOptionsTest.php b/packages/nats/tests/E2E/KeyValueWatchOptionsTest.php new file mode 100644 index 00000000000..0b451c6e967 --- /dev/null +++ b/packages/nats/tests/E2E/KeyValueWatchOptionsTest.php @@ -0,0 +1,201 @@ +conn = Connection::connect($url); + $this->js = $this->conn->jetStream(); + $this->bucket = 'kvwo_' . uniqid(); + $this->kv = $this->js->createKeyValue(new KeyValueConfig( + bucket: $this->bucket, + history: 10, + )); + } + + protected function tearDown(): void + { + try { + $this->js->deleteKeyValue($this->bucket); + } catch (\Throwable) { + // ignore + } + $this->conn->close(); + } + + private function pumpUntil(callable $done, float $seconds = 3.0): void + { + $deadline = microtime(true) + $seconds; + while (!$done() && microtime(true) < $deadline) { + $this->conn->processMessage(0.2); + } + } + + public function testWatchIncludeHistoryThenLiveUpdates(): void + { + $this->kv->put('k', 'v1'); + $this->kv->put('k', 'v2'); + $this->kv->put('k', 'v3'); + $this->kv->delete('k'); + + /** @var list $received */ + $received = []; + $initDone = false; + $countAtInit = null; + + $sub = $this->kv->watch( + 'k', + function (KeyValueEntry $entry) use (&$received): void { + $received[] = $entry; + }, + new KeyValueWatchOptions(includeHistory: true), + function () use (&$initDone, &$received, &$countAtInit): void { + $initDone = true; + $countAtInit = \count($received); + }, + ); + $this->conn->flush(); + + $this->pumpUntil(function () use (&$initDone): bool { + return $initDone; + }); + + $this->assertTrue($initDone, 'onInitDone should fire after historical replay'); + $this->assertSame(4, $countAtInit, 'all four historical entries delivered before init-done'); + + $values = array_map(static fn (KeyValueEntry $e): string => $e->value, \array_slice($received, 0, 3)); + $this->assertSame(['v1', 'v2', 'v3'], $values); + $this->assertSame(KeyValueOperation::Delete, $received[3]->operation); + + // Live update after the historical set. + $this->kv->put('k', 'v4'); + $this->pumpUntil(function () use (&$received): bool { + return \count($received) >= 5; + }); + $sub->unsubscribe(); + + $this->assertGreaterThanOrEqual(5, \count($received)); + $this->assertSame('v4', $received[4]->value); + $this->assertSame(KeyValueOperation::Put, $received[4]->operation); + } + + public function testWatchIgnoreDeletesSkipsMarkers(): void + { + $this->kv->put('k', 'v1'); + $this->kv->delete('k'); + + $ops = []; + $initDone = false; + + $sub = $this->kv->watch( + 'k', + function ($entry) use (&$ops): void { + $ops[] = $entry->operation; + }, + new KeyValueWatchOptions(includeHistory: true, ignoreDeletes: true), + function () use (&$initDone): void { + $initDone = true; + }, + ); + $this->conn->flush(); + + $this->pumpUntil(function () use (&$initDone): bool { + return $initDone; + }); + $sub->unsubscribe(); + + $this->assertTrue($initDone); + $this->assertContains(KeyValueOperation::Put, $ops); + $this->assertNotContains(KeyValueOperation::Delete, $ops); + $this->assertNotContains(KeyValueOperation::Purge, $ops); + } + + public function testWatchMetaOnlyDeliversNoValue(): void + { + $this->kv->put('k', 'a-real-value'); + + $entries = []; + $initDone = false; + + $sub = $this->kv->watch( + 'k', + function ($entry) use (&$entries): void { + $entries[] = $entry; + }, + new KeyValueWatchOptions(includeHistory: true, metaOnly: true), + function () use (&$initDone): void { + $initDone = true; + }, + ); + $this->conn->flush(); + + $this->pumpUntil(function () use (&$initDone): bool { + return $initDone; + }); + $sub->unsubscribe(); + + $this->assertCount(1, $entries); + $this->assertSame('k', $entries[0]->key); + $this->assertSame('', $entries[0]->value, 'metaOnly entries carry no value body'); + $this->assertGreaterThan(0, $entries[0]->revision); + } + + public function testPurgeDeletesRemovesTombstones(): void + { + $this->kv->put('a', '1'); + $this->kv->put('b', '2'); + $this->kv->put('c', '3'); + $this->kv->delete('a'); + $this->kv->purge('b'); + // 'c' stays live. + + // Markers keep 'a' and 'b' present before purging. + $keysBefore = $this->kv->keys(); + sort($keysBefore); + $this->assertSame(['a', 'b', 'c'], $keysBefore); + $valuesBefore = $this->kv->status()->values; + + $removed = $this->kv->purgeDeletes(); + + $this->assertSame(2, $removed); + + $keysAfter = $this->kv->keys(); + sort($keysAfter); + $this->assertSame(['c'], $keysAfter); + + $valuesAfter = $this->kv->status()->values; + $this->assertLessThan($valuesBefore, $valuesAfter); + $this->assertSame(1, $valuesAfter); + } + + public function testPurgeDeletesRespectsThreshold(): void + { + $this->kv->put('a', '1'); + $this->kv->delete('a'); + + // Marker was just created; "older than 60s" keeps it. + $removed = $this->kv->purgeDeletes(60.0); + + $this->assertSame(0, $removed); + $this->assertContains('a', $this->kv->keys()); + } +} diff --git a/packages/nats/tests/E2E/KeyValueWatchTest.php b/packages/nats/tests/E2E/KeyValueWatchTest.php new file mode 100644 index 00000000000..46b326c3fc9 --- /dev/null +++ b/packages/nats/tests/E2E/KeyValueWatchTest.php @@ -0,0 +1,137 @@ +conn = Connection::connect($url); + $this->js = $this->conn->jetStream(); + $this->bucket = 'kvw_' . uniqid(); + $this->kv = $this->js->createKeyValue(new KeyValueConfig( + bucket: $this->bucket, + history: 10, + )); + } + + protected function tearDown(): void + { + try { + $this->js->deleteKeyValue($this->bucket); + } catch (\Throwable) { + // ignore + } + $this->conn->close(); + } + + public function testHistoryReturnsAllRevisionsInOrder(): void + { + $r1 = $this->kv->put('color', 'red'); + $r2 = $this->kv->put('color', 'green'); + $r3 = $this->kv->put('color', 'blue'); + $this->kv->delete('color'); + + $history = $this->kv->history('color'); + + $this->assertCount(4, $history); + $this->assertSame('red', $history[0]->value); + $this->assertSame('green', $history[1]->value); + $this->assertSame('blue', $history[2]->value); + + $this->assertSame($r1, $history[0]->revision); + $this->assertSame($r2, $history[1]->revision); + $this->assertSame($r3, $history[2]->revision); + + // Revisions strictly increasing. + $this->assertTrue($r1 < $r2 && $r2 < $r3); + + $this->assertSame(KeyValueOperation::Put, $history[0]->operation); + $this->assertSame(KeyValueOperation::Delete, $history[3]->operation); + } + + public function testGetRevisionFetchesSpecificSeq(): void + { + $r1 = $this->kv->put('name', 'alice'); + $this->kv->put('name', 'bob'); + + $entry = $this->kv->getRevision('name', $r1); + + $this->assertSame('alice', $entry->value); + $this->assertSame($r1, $entry->revision); + $this->assertSame('name', $entry->key); + } + + public function testGetRevisionRejectsSeqFromAnotherKey(): void + { + // 'alpha' revision seq must not resolve when requested under 'beta'. + $alphaSeq = $this->kv->put('alpha', 'a-value'); + $this->kv->put('beta', 'b-value'); + + $this->expectException(KeyValueException::class); + $this->kv->getRevision('beta', $alphaSeq); + } + + public function testWatchFiresCallbackOnPut(): void + { + $received = []; + $sub = $this->kv->watch('greeting', function ($entry) use (&$received): void { + $received[] = $entry; + }); + + // Ensure the consumer/subscription is established before producing. + $this->conn->flush(); + + $this->kv->put('greeting', 'hello'); + + $deadline = microtime(true) + 3.0; + while ($received === [] && microtime(true) < $deadline) { + $this->conn->processMessage(0.2); + } + + $sub->unsubscribe(); + + $this->assertCount(1, $received); + $this->assertSame('greeting', $received[0]->key); + $this->assertSame('hello', $received[0]->value); + $this->assertSame(KeyValueOperation::Put, $received[0]->operation); + } + + public function testWatchFiresCallbackOnDelete(): void + { + $this->kv->put('flag', 'on'); + + $ops = []; + $sub = $this->kv->watch('flag', function ($entry) use (&$ops): void { + $ops[] = $entry->operation; + }); + $this->conn->flush(); + + $this->kv->delete('flag'); + + $deadline = microtime(true) + 3.0; + while ($ops === [] && microtime(true) < $deadline) { + $this->conn->processMessage(0.2); + } + + $sub->unsubscribe(); + + $this->assertContains(KeyValueOperation::Delete, $ops); + } +} diff --git a/packages/nats/tests/E2E/ObjectStoreExtraTest.php b/packages/nats/tests/E2E/ObjectStoreExtraTest.php new file mode 100644 index 00000000000..6dd2e21d709 --- /dev/null +++ b/packages/nats/tests/E2E/ObjectStoreExtraTest.php @@ -0,0 +1,180 @@ +conn = Connection::connect($url); + $this->js = $this->conn->jetStream(); + $this->bucket = 'objx_' . uniqid(); + $this->store = ObjectStore::createOrUpdate($this->conn, $this->js, new ObjectStoreConfig( + bucket: $this->bucket, + )); + } + + protected function tearDown(): void + { + try { + $this->js->deleteStream("OBJ_{$this->bucket}"); + } catch (\Throwable) { + // ignore + } + $this->conn->close(); + } + + public function testWatchFiresOnPutAndDelete(): void + { + /** @var list $events */ + $events = []; + $sub = $this->store->watch(function (ObjectMeta $meta) use (&$events): void { + $events[] = $meta; + }); + + // Ensure the consumer/subscription is established before producing. + $this->conn->flush(); + + $this->store->put('watched.txt', 'hello'); + $this->pumpUntil(fn (): bool => $events !== []); + + $this->assertCount(1, $events); + $this->assertSame('watched.txt', $events[0]->name); + $this->assertFalse($events[0]->deleted); + + $this->store->delete('watched.txt'); + $this->pumpUntil(fn (): bool => \count($events) >= 2); + + $sub->unsubscribe(); + + $this->assertCount(2, $events); + $this->assertSame('watched.txt', $events[1]->name); + $this->assertTrue($events[1]->deleted); + } + + public function testWatchIncludeHistoryDeliversCurrentMetaFirst(): void + { + $this->store->put('existing.txt', 'already here'); + + /** @var list $events */ + $events = []; + $sub = $this->store->watch(function (ObjectMeta $meta) use (&$events): void { + $events[] = $meta; + }, includeHistory: true); + + $this->conn->flush(); + $this->pumpUntil(fn (): bool => $events !== []); + + $sub->unsubscribe(); + + $this->assertNotEmpty($events); + $this->assertSame('existing.txt', $events[0]->name); + } + + public function testAddLinkResolvesToTargetBytes(): void + { + $this->store->put('target.bin', 'the real payload'); + + $linkMeta = $this->store->addLink('alias.bin', 'target.bin'); + $this->assertInstanceOf(\Utopia\NATS\ObjectStore\ObjectLink::class, $linkMeta->link); + $this->assertSame('target.bin', $linkMeta->link->name); + + // get() on the link transparently returns the target's bytes. + $this->assertSame('the real payload', $this->store->get('alias.bin')); + } + + public function testBucketLinkCannotBeRead(): void + { + $this->store->addBucketLink('otherbucket', 'SOME_OTHER_BUCKET'); + + $this->expectException(ObjectStoreException::class); + $this->store->get('otherbucket'); + } + + public function testUpdateMetaChangesDescriptionAndKeepsBytes(): void + { + $payload = 'immutable bytes'; + $this->store->put('doc.txt', $payload); + + $updated = $this->store->updateMeta('doc.txt', description: 'a helpful description'); + $this->assertSame('a helpful description', $updated->description); + + // Meta round-trips the new description... + $this->assertSame('a helpful description', $this->store->getMeta('doc.txt')->description); + + // ...and the bytes are untouched. + $this->assertSame($payload, $this->store->get('doc.txt')); + } + + public function testSealThenPutThrows(): void + { + $this->store->put('before.txt', 'written before seal'); + + $this->store->seal(); + + $threw = false; + try { + $this->store->put('after.txt', 'should be rejected'); + } catch (\Throwable) { + $threw = true; + } + + $this->assertTrue($threw, 'put() after seal() should fail because the stream is sealed'); + + // Data written before sealing is still readable. + $this->assertSame('written before seal', $this->store->get('before.txt')); + } + + /** + * @param callable(): bool $done + */ + private function pumpUntil(callable $done, float $timeout = 3.0): void + { + $deadline = microtime(true) + $timeout; + while (!$done() && microtime(true) < $deadline) { + $this->conn->processMessage(0.2); + } + } + + public function testDeleteConflictLeavesReplacementIntact(): void + { + // A stale delete must not corrupt a concurrent replacement: it should conflict + // on the guarded tombstone publish BEFORE purging any chunks. + $this->store->put('doc', 'v1'); + + $readMeta = new \ReflectionMethod($this->store, 'readMetaWithSeq'); + [$staleMeta, $staleSeq] = $readMeta->invoke($this->store, 'doc'); + + // A writer replaces the object, advancing the meta subject past $staleSeq. + $this->store->put('doc', 'v2'); + + // Replay the stale delete: it expects $staleSeq but the subject moved on. + $deleteVersion = new \ReflectionMethod($this->store, 'deleteVersion'); + $conflicted = false; + try { + $deleteVersion->invoke($this->store, $staleMeta, $staleSeq); + } catch (ObjectStoreException) { + $conflicted = true; + } + + $this->assertTrue($conflicted, 'stale delete should have conflicted'); + // The replacement is intact — its chunks were never purged by the stale delete. + $this->assertSame('v2', $this->store->get('doc')); + } +} diff --git a/packages/nats/tests/E2E/ObjectStoreTest.php b/packages/nats/tests/E2E/ObjectStoreTest.php new file mode 100644 index 00000000000..2f18042d137 --- /dev/null +++ b/packages/nats/tests/E2E/ObjectStoreTest.php @@ -0,0 +1,151 @@ +conn = Connection::connect($url); + $this->js = $this->conn->jetStream(); + $this->bucket = 'obj_' . uniqid(); + $this->store = ObjectStore::createOrUpdate($this->conn, $this->js, new ObjectStoreConfig( + bucket: $this->bucket, + )); + } + + protected function tearDown(): void + { + try { + $this->js->deleteStream("OBJ_{$this->bucket}"); + } catch (\Throwable) { + // ignore + } + $this->conn->close(); + } + + public function testRoundTripMultiChunkPayload(): void + { + // ~300KB => spans multiple 128KB chunks. + $payload = random_bytes(300 * 1024); + + $putMeta = $this->store->put('big.bin', $payload); + $this->assertGreaterThan(1, $putMeta->chunks); + $this->assertSame(\strlen($payload), $putMeta->size); + + $fetched = $this->store->get('big.bin'); + $this->assertSame($payload, $fetched); + + $meta = $this->store->getMeta('big.bin'); + $this->assertSame($putMeta->digest, $meta->digest); + + // Digest matches an independent computation. + $expectedDigest = 'SHA-256=' . rtrim(strtr(base64_encode(hash('sha256', $payload, true)), '+/', '-_'), '='); + $this->assertSame($expectedDigest, $meta->digest); + } + + public function testSmallObjectRoundTrip(): void + { + $this->store->put('hello.txt', 'hello world'); + $this->assertSame('hello world', $this->store->get('hello.txt')); + } + + public function testListReturnsStoredObjects(): void + { + $this->store->put('a.txt', 'aaa'); + $this->store->put('b.txt', 'bbb'); + + $names = array_map(fn (\Utopia\NATS\ObjectStore\ObjectMeta $m): string => $m->name, $this->store->list()); + sort($names); + + $this->assertSame(['a.txt', 'b.txt'], $names); + } + + public function testDeleteRemovesObject(): void + { + $this->store->put('temp.txt', 'gone soon'); + $this->assertSame('gone soon', $this->store->get('temp.txt')); + + $this->store->delete('temp.txt'); + + $this->expectException(\RuntimeException::class); + $this->store->get('temp.txt'); + } + + public function testOverwriteReplacesData(): void + { + $this->store->put('file', 'version-one'); + $this->store->put('file', 'version-two-longer'); + + $this->assertSame('version-two-longer', $this->store->get('file')); + $this->assertCount(1, $this->store->list()); + } + + public function testStatusReportsBucketStream(): void + { + $this->store->put('x.txt', 'data'); + $info = $this->store->status(); + $this->assertSame("OBJ_{$this->bucket}", $info->config->name); + $this->assertGreaterThan(0, $info->state->messages); + } + + public function testSingleWriterOverwriteReclaimsOldChunks(): void + { + // Each version is a single chunk. After overwrite, only the new chunk + // plus one rolled-up meta record should remain (2 messages total); + // a leftover previous chunk would push the count to 3. + $this->store->put('reclaim.bin', 'version-one'); + $this->store->put('reclaim.bin', 'version-two'); + + $this->assertSame('version-two', $this->store->get('reclaim.bin')); + $this->assertSame(2, $this->store->status()->state->messages); + } + + public function testConcurrentOverwriteConflictsInsteadOfOrphaning(): void + { + // Deterministically drive the optimistic-concurrency conflict rather than + // racing two real writers (which serialises on a fast host). Capture the meta + // sequence a stale writer would hold, let a second write advance the subject, + // then replay the stale write via the seq-aware seam and assert it conflicts + // and cleans up its own chunks. + $this->store->put('conflict.bin', 'v1'); + + $readMeta = new \ReflectionMethod($this->store, 'readMetaWithSeq'); + [$stalePrev, $staleSeq] = $readMeta->invoke($this->store, 'conflict.bin'); + + // A second writer wins, advancing the meta subject past $staleSeq. + $this->store->put('conflict.bin', 'v2'); + + // Replay the stale write: it expects $staleSeq but the subject moved on. + $writeVersion = new \ReflectionMethod($this->store, 'writeVersion'); + $conflicted = false; + try { + $writeVersion->invoke($this->store, 'conflict.bin', 'v3-stale', $stalePrev, $staleSeq); + } catch (ObjectStoreException) { + $conflicted = true; + } + + $this->assertTrue($conflicted, 'stale write should have conflicted'); + + // Winner intact, and no orphaned chunks: only v2's single chunk plus the + // rolled-up meta remain (the stale attempt purged its own chunk). + $this->assertSame('v2', $this->store->get('conflict.bin')); + $this->assertCount(1, $this->store->list()); + $this->assertSame(2, $this->store->status()->state->messages); + } +} diff --git a/packages/nats/tests/E2E/OrderedConsumerTest.php b/packages/nats/tests/E2E/OrderedConsumerTest.php new file mode 100644 index 00000000000..e45a099e417 --- /dev/null +++ b/packages/nats/tests/E2E/OrderedConsumerTest.php @@ -0,0 +1,142 @@ + teardown() + create() and recreate the + * ephemeral consumer from the message after the last good one. Iteration must + * recover and still yield every message exactly once, in stream order, with no + * loss and no duplicate handed to the caller past the reset. + * + * If the reset logic is broken (e.g. it recreated from the wrong stream + * sequence, or did not recreate at all) this test fails: the collected stream + * sequences would show a gap or a duplicate. + * + * Requires a JetStream-enabled server (NATS_URL). + */ +final class OrderedConsumerTest extends TestCase +{ + private Connection $conn; + private JetStream $js; + /** @var list */ + private array $streams = []; + + private function getServerUrl(): string + { + return getenv('NATS_URL') ?: 'nats://127.0.0.1:4222'; + } + + protected function setUp(): void + { + $this->conn = Connection::connect($this->getServerUrl()); + $this->js = $this->conn->jetStream(); + } + + protected function tearDown(): void + { + foreach ($this->streams as $stream) { + try { + $this->js->deleteStream($stream); + } catch (\Throwable) { + // already gone + } + } + $this->conn->close(); + } + + private function createStream(string $subject): string + { + $name = 'OC_' . uniqid(); + $this->js->createStream(new StreamConfig( + name: $name, + subjects: [$subject], + storage: StorageType::Memory, + )); + $this->streams[] = $name; + return $name; + } + + private function setExpectedConsumerSeq(OrderedConsumer $ordered, int $value): void + { + $ref = new \ReflectionProperty(OrderedConsumer::class, 'expectedConsumerSeq'); + $ref->setValue($ordered, $value); + } + + private function getExpectedConsumerSeq(OrderedConsumer $ordered): int + { + $ref = new \ReflectionProperty(OrderedConsumer::class, 'expectedConsumerSeq'); + return (int) $ref->getValue($ordered); + } + + public function testResetRecreatesAndRecoversInOrder(): void + { + $id = uniqid(); + $subject = "orderedreset.{$id}"; + $stream = $this->createStream($subject); + + $count = 20; + for ($i = 1; $i <= $count; $i++) { + $this->js->publish($subject, "msg-{$i}"); + } + + $ordered = $this->js->orderedConsumer($stream); + + // Consume the first half normally. These stream sequences are 1..half. + $half = 10; + $streamSeqs = []; + for ($i = 0; $i < $half; $i++) { + $msg = $ordered->next(3.0); + $this->assertInstanceOf(JetStreamMessage::class, $msg); + $streamSeqs[] = $msg->metadata()->streamSequence; + } + $this->assertSame(range(1, $half), $streamSeqs, 'first half must arrive in order before the reset'); + + $consumerBefore = $ordered->getConsumerName(); + + // Simulate a missed delivery: advance the expected consumer sequence by + // one so the very next pushed message (whose consumer sequence is what + // we would otherwise expect) reads as a gap and forces a reset. + $expected = $this->getExpectedConsumerSeq($ordered); + $this->setExpectedConsumerSeq($ordered, $expected + 1); + + // Drain the rest. The first next() here observes the gap, triggers + // reset($lastStreamSeq + 1) and recreates the consumer from the message + // after the last good one (stream seq $half + 1), then returns it. + for ($i = 0; $i < $count - $half; $i++) { + $msg = $ordered->next(3.0); + $this->assertInstanceOf(JetStreamMessage::class, $msg); + $streamSeqs[] = $msg->metadata()->streamSequence; + } + + $consumerAfter = $ordered->getConsumerName(); + + // The reset must have torn down and recreated the ephemeral consumer. + $this->assertNotSame($consumerBefore, $consumerAfter, 'reset must recreate the ephemeral consumer'); + + // Every message delivered to the caller exactly once, in stream order, + // with no loss and no duplicate across the reset boundary. + $this->assertSame(range(1, $count), $streamSeqs, 'all messages must be delivered once, in order, across the reset'); + + // No further messages remain. + $this->assertNotInstanceOf(\Utopia\NATS\JetStream\JetStreamMessage::class, $ordered->next(0.5), 'no extra or duplicate messages after recovery'); + + $ordered->stop(); + } +} diff --git a/packages/nats/tests/E2E/PubSubTest.php b/packages/nats/tests/E2E/PubSubTest.php new file mode 100644 index 00000000000..d2fa5105a6f --- /dev/null +++ b/packages/nats/tests/E2E/PubSubTest.php @@ -0,0 +1,130 @@ +getServerUrl()); + + $received = null; + $sub = $conn->subscribe('test.pubsub', function ($msg) use (&$received): void { + $received = $msg; + }); + + $conn->publish('test.pubsub', 'hello'); + $conn->processMessage(1.0); + + $this->assertNotNull($received); + $this->assertSame('test.pubsub', $received->subject); + $this->assertSame('hello', $received->data); + + $sub->unsubscribe(); + $conn->close(); + } + + public function testSyncSubscribe(): void + { + $conn = Connection::connect($this->getServerUrl()); + + $sub = $conn->subscribe('test.sync'); + + $conn->publish('test.sync', 'sync-message'); + $msg = $sub->nextMessage(1.0); + + $this->assertInstanceOf(\Utopia\NATS\Message::class, $msg); + $this->assertSame('sync-message', $msg->data); + + $sub->unsubscribe(); + $conn->close(); + } + + public function testWildcard(): void + { + $conn = Connection::connect($this->getServerUrl()); + + $messages = []; + $sub = $conn->subscribe('test.wild.*', function ($msg) use (&$messages): void { + $messages[] = $msg; + }); + + $conn->publish('test.wild.one', 'first'); + $conn->publish('test.wild.two', 'second'); + $conn->processMessage(1.0); + $conn->processMessage(1.0); + + $this->assertCount(2, $messages); + + $sub->unsubscribe(); + $conn->close(); + } + + public function testQueueSubscribe(): void + { + $conn = Connection::connect($this->getServerUrl()); + + $count1 = 0; + $count2 = 0; + + $sub1 = $conn->queueSubscribe('test.queue', 'workers', function () use (&$count1): void { + $count1++; + }); + $sub2 = $conn->queueSubscribe('test.queue', 'workers', function () use (&$count2): void { + $count2++; + }); + + for ($i = 0; $i < 10; $i++) { + $conn->publish('test.queue', "msg-{$i}"); + } + + for ($i = 0; $i < 10; $i++) { + $conn->processMessage(1.0); + } + + $this->assertSame(10, $count1 + $count2); + + $sub1->unsubscribe(); + $sub2->unsubscribe(); + $conn->close(); + } + + public function testPubSubWithHeaders(): void + { + $conn = Connection::connect($this->getServerUrl()); + + $received = null; + $sub = $conn->subscribe('test.headers', function ($msg) use (&$received): void { + $received = $msg; + }); + + $headers = new Headers(); + $headers->set('X-Custom', 'test-value'); + $headers->set('Content-Type', 'text/plain'); + + $conn->publish('test.headers', 'with-headers', headers: $headers); + $conn->processMessage(1.0); + + $this->assertNotNull($received); + $this->assertNotNull($received->headers); + $this->assertSame('test-value', $received->headers->get('X-Custom')); + $this->assertSame('text/plain', $received->headers->get('Content-Type')); + + $sub->unsubscribe(); + $conn->close(); + } +} diff --git a/packages/nats/tests/E2E/RequestManyTest.php b/packages/nats/tests/E2E/RequestManyTest.php new file mode 100644 index 00000000000..74fd1184238 --- /dev/null +++ b/packages/nats/tests/E2E/RequestManyTest.php @@ -0,0 +1,121 @@ +subscribe($subject, function ($msg) use ($conn, $tag): void { + if ($msg->replyTo !== null) { + $conn->publish($msg->replyTo, $tag); + } + }); + } + } + + public function testCollectsAllResponders(): void + { + $conn = Connection::connect($this->getServerUrl()); + $subject = 'test.rm.all.' . uniqid(); + $this->attachResponders($conn, $subject); + + $replies = $conn->requestMany($subject, 'ping', ['timeout' => 2.0]); + + $this->assertCount(3, $replies); + $bodies = array_map(fn (\Utopia\NATS\Message $m): string => $m->data, $replies); + sort($bodies); + $this->assertSame(['r1', 'r2', 'r3'], $bodies); + + $conn->close(); + } + + public function testMaxStopsEarly(): void + { + $conn = Connection::connect($this->getServerUrl()); + $subject = 'test.rm.max.' . uniqid(); + $this->attachResponders($conn, $subject); + + $replies = $conn->requestMany($subject, 'ping', ['max' => 2, 'timeout' => 2.0]); + + $this->assertCount(2, $replies); + + $conn->close(); + } + + public function testTimeoutBoundsWhenMaxUnreached(): void + { + $conn = Connection::connect($this->getServerUrl()); + $subject = 'test.rm.timeout.' . uniqid(); + $this->attachResponders($conn, $subject); + + // Ask for more replies than exist: the overall timeout is the stop + // condition, and we still collect the three that did answer. + $start = microtime(true); + $replies = $conn->requestMany($subject, 'ping', ['max' => 10, 'timeout' => 0.8]); + $elapsed = microtime(true) - $start; + + $this->assertCount(3, $replies); + $this->assertGreaterThanOrEqual(0.7, $elapsed); + + $conn->close(); + } + + public function testStallStopsBeforeTimeout(): void + { + $conn = Connection::connect($this->getServerUrl()); + $subject = 'test.rm.stall.' . uniqid(); + $this->attachResponders($conn, $subject); + + // Three fast replies then silence. A short stall window returns well + // before the generous overall timeout would. + // + // The overall timeout is wide and the bound is set between the two, so + // this measures which of them stopped collection rather than how fast + // the runner is. The responders share the requester's connection, so + // the single loop has to carry the request out and three replies back + // through nextMessage() rounds of one stall window each; on a loaded + // runner that is comfortably over a second, while a stall that never + // fired would take the full ten. + $start = microtime(true); + $replies = $conn->requestMany($subject, 'ping', ['timeout' => 10.0, 'stall' => 0.3]); + $elapsed = microtime(true) - $start; + + $this->assertCount(3, $replies); + $this->assertLessThan(5.0, $elapsed); + + $conn->close(); + } + + public function testEmptyWhenNoResponders(): void + { + $conn = Connection::connect($this->getServerUrl()); + $subject = 'test.rm.none.' . uniqid(); + + // No subscribers: the server's 503 no-responders reply means zero + // responders, so requestMany yields an empty list. + $replies = $conn->requestMany($subject, 'ping', ['timeout' => 2.0]); + + $this->assertSame([], $replies); + + $conn->close(); + } +} diff --git a/packages/nats/tests/E2E/RequestReplyTest.php b/packages/nats/tests/E2E/RequestReplyTest.php new file mode 100644 index 00000000000..0cf72e92056 --- /dev/null +++ b/packages/nats/tests/E2E/RequestReplyTest.php @@ -0,0 +1,67 @@ +getServerUrl()); + + // Set up responder + $sub = $conn->subscribe('test.echo', function ($msg) use ($conn): void { + if ($msg->replyTo !== null) { + $conn->publish($msg->replyTo, 'echo: ' . $msg->data); + } + }); + + $response = $conn->request('test.echo', 'hello', 2.0); + $this->assertSame('echo: hello', $response->data); + + $sub->unsubscribe(); + $conn->close(); + } + + public function testRequestTimeout(): void + { + $conn = Connection::connect($this->getServerUrl()); + + // Subscribe a silent responder so the server sees interest and does not + // short-circuit with a "no responders" reply — the request must hang + // until it times out. + $conn->subscribe('test.silent', fn ($msg) => null); + + $this->expectException(TimeoutException::class); + $conn->request('test.silent', 'hello', 0.5); + + $conn->close(); + } + + public function testRequestNoResponders(): void + { + $conn = Connection::connect($this->getServerUrl()); + + // With no subscriber on the subject the server replies immediately with + // a 503 "no responders" status rather than letting the request hang. + $this->expectException(NatsException::class); + $this->expectExceptionMessage('No responders for request'); + $conn->request('test.no-responder-' . uniqid(), 'hello', 2.0); + + $conn->close(); + } +} diff --git a/packages/nats/tests/E2E/RequestsTest.php b/packages/nats/tests/E2E/RequestsTest.php new file mode 100644 index 00000000000..bbedf36b159 --- /dev/null +++ b/packages/nats/tests/E2E/RequestsTest.php @@ -0,0 +1,46 @@ +set('Trace', 'example'); + $received = []; + $connection->subscribe($subject, static function (Message $message) use ($connection, &$received): void { + $received[] = $message; + $connection->publish($message->replyTo, $message->data); + }); + $connection->flush(); + $results = []; + $connection->requestBatch([ + new Request(subject: $subject, data: 'first', headers: $headers), + new Request(subject: $subject . '.missing'), + new Request(subject: $subject, data: 'third'), + ], static function (int $index, Message|\Throwable $result) use (&$results): void { + $results[$index] = $result; + }); + $this->assertCount(3, $results); + $this->assertSame('first', $results[0]->data); + $this->assertInstanceOf(NatsException::class, $results[1]); + $this->assertSame('third', $results[2]->data); + $this->assertSame('example', $received[0]->headers->get('Trace')); + } finally { + $connection->close(); + } + } +} diff --git a/packages/nats/tests/E2E/ServiceExtrasTest.php b/packages/nats/tests/E2E/ServiceExtrasTest.php new file mode 100644 index 00000000000..4213ab640ef --- /dev/null +++ b/packages/nats/tests/E2E/ServiceExtrasTest.php @@ -0,0 +1,156 @@ +conn = Connection::connect($url); + $this->name = 'svcx_' . uniqid(); + + $this->service = new Service( + $this->conn, + $this->name, + '1.0.0', + 'Extras service', + ['owner' => 'levi', 'env' => 'test'], + ); + } + + protected function tearDown(): void + { + $this->service->stop(); + $this->conn->close(); + } + + public function testGroupedEndpointIsReachableAtPrefixedSubject(): void + { + // Root group carries the unique service name so subjects don't collide. + $group = $this->service->addGroup($this->name)->addGroup('math'); + $group->addEndpoint('add', fn (Message $msg): string => 'sum:' . $msg->data); + + // Nested group prefixes cumulatively. + $sub = $group->addGroup('trig'); + $sub->addEndpoint('sin', fn (Message $msg): string => 'sin:' . $msg->data); + + $this->service->start(); + + $r1 = $this->conn->request("{$this->name}.math.add", '2', 2.0); + $this->assertSame('sum:2', $r1->data); + + $r2 = $this->conn->request("{$this->name}.math.trig.sin", '0', 2.0); + $this->assertSame('sin:0', $r2->data); + } + + public function testCustomErrorCodeSurfacesInHeadersAndCountsError(): void + { + $this->service->addEndpoint('fail', "{$this->name}.fail", function (Message $msg): string { + throw new ServiceException('418', 'I am a teapot'); + }); + $this->service->start(); + + $response = $this->conn->request("{$this->name}.fail", 'x', 2.0); + + $this->assertInstanceOf(\Utopia\NATS\Headers::class, $response->headers); + $this->assertSame('418', $response->headers->get('Nats-Service-Error-Code')); + $this->assertSame('I am a teapot', $response->headers->get('Nats-Service-Error')); + + // Error count is tracked in stats. + $stats = json_decode($this->conn->request("\$SRV.STATS.{$this->name}", '', 2.0)->data, true); + $byName = []; + foreach ($stats['endpoints'] as $ep) { + $byName[$ep['name']] = $ep; + } + $this->assertSame(1, $byName['fail']['num_requests']); + $this->assertSame(1, $byName['fail']['num_errors']); + } + + public function testInfoIncludesEndpointsWithSubjectsAndMetadata(): void + { + $group = $this->service->addGroup($this->name)->addGroup('v1'); + $group->addEndpoint( + 'status', + fn (Message $msg): string => 'ok', + null, + null, + ['visibility' => 'public'], + ); + $this->service->start(); + + $info = json_decode($this->conn->request("\$SRV.INFO.{$this->name}", '', 2.0)->data, true); + + // Service-level metadata. + $this->assertSame('levi', $info['metadata']['owner']); + $this->assertSame('test', $info['metadata']['env']); + + // Endpoint appears with its prefixed subject and its own metadata. + $byName = []; + foreach ($info['endpoints'] as $ep) { + $byName[$ep['name']] = $ep; + } + $this->assertSame("{$this->name}.v1.status", $byName['status']['subject']); + $this->assertSame('public', $byName['status']['metadata']['visibility']); + } + + public function testCustomQueueGroupIsHonored(): void + { + $queue = 'workers_' . uniqid(); + + // Group-level queue group is inherited by its endpoints. + $group = $this->service->addGroup('jobs', $queue); + $group->addEndpoint('run', fn (Message $msg): string => 'done'); + + // Per-endpoint override on the bare service. + $this->service->addEndpoint( + 'direct', + "{$this->name}.direct", + fn (Message $msg): string => 'direct', + $queue, + ); + $this->service->start(); + + $info = json_decode($this->conn->request("\$SRV.INFO.{$this->name}", '', 2.0)->data, true); + $byName = []; + foreach ($info['endpoints'] as $ep) { + $byName[$ep['name']] = $ep; + } + $this->assertSame($queue, $byName['run']['queue_group']); + $this->assertSame($queue, $byName['direct']['queue_group']); + } + + public function testQueueGroupLoadBalancesAcrossInstances(): void + { + $queue = 'lb_' . uniqid(); + $subject = "{$this->name}.work"; + + // Two endpoints on the same subject + queue group behave as two queue + // members: NATS delivers each request to only one of them. + $this->service->addEndpoint('a', $subject, fn (Message $msg): string => 'a', $queue); + $this->service->addEndpoint('b', $subject, fn (Message $msg): string => 'b', $queue); + $this->service->start(); + + $seen = ['a' => 0, 'b' => 0]; + for ($i = 0; $i < 30; $i++) { + $reply = $this->conn->request($subject, (string) $i, 2.0)->data; + $seen[$reply]++; + } + + $this->assertSame(30, $seen['a'] + $seen['b']); + $this->assertGreaterThan(0, $seen['a']); + $this->assertGreaterThan(0, $seen['b']); + } +} diff --git a/packages/nats/tests/E2E/ServiceTest.php b/packages/nats/tests/E2E/ServiceTest.php new file mode 100644 index 00000000000..b7b3544ead2 --- /dev/null +++ b/packages/nats/tests/E2E/ServiceTest.php @@ -0,0 +1,94 @@ +conn = Connection::connect($url); + $this->name = 'svc_' . uniqid(); + $this->echoSubject = "{$this->name}.echo"; + + $this->service = new Service($this->conn, $this->name, '1.2.3', 'Test service'); + $this->service->addEndpoint('echo', $this->echoSubject, fn (Message $msg): string => 'echo:' . $msg->data); + $this->service->addEndpoint('boom', "{$this->name}.boom", function (Message $msg): string { + throw new \RuntimeException('kaboom'); + }); + $this->service->start(); + } + + protected function tearDown(): void + { + $this->service->stop(); + $this->conn->close(); + } + + public function testEndpointHandlesRequest(): void + { + $response = $this->conn->request($this->echoSubject, 'hello', 2.0); + $this->assertSame('echo:hello', $response->data); + } + + public function testPingReturnsServiceIdentity(): void + { + $response = $this->conn->request('$SRV.PING', '', 2.0); + $data = json_decode($response->data, true); + + $this->assertSame('io.nats.micro.v1.ping_response', $data['type']); + $this->assertSame($this->name, $data['name']); + $this->assertSame('1.2.3', $data['version']); + $this->assertArrayHasKey('id', $data); + $this->assertSame($this->service->getId(), $data['id']); + } + + public function testInfoListsEndpoints(): void + { + $response = $this->conn->request("\$SRV.INFO.{$this->name}", '', 2.0); + $data = json_decode($response->data, true); + + $this->assertSame('io.nats.micro.v1.info_response', $data['type']); + $this->assertSame('Test service', $data['description']); + + $subjects = array_map(fn (array $e) => $e['subject'], $data['endpoints']); + $this->assertContains($this->echoSubject, $subjects); + } + + public function testStatsTrackRequestAndErrorCounts(): void + { + // Drive one successful request and one failing request. + $this->conn->request($this->echoSubject, 'a', 2.0); + $this->conn->request($this->echoSubject, 'b', 2.0); + $this->conn->request("{$this->name}.boom", 'x', 2.0); + + $response = $this->conn->request("\$SRV.STATS.{$this->name}", '', 2.0); + $data = json_decode($response->data, true); + + $this->assertSame('io.nats.micro.v1.stats_response', $data['type']); + $this->assertArrayHasKey('started', $data); + + $byName = []; + foreach ($data['endpoints'] as $ep) { + $byName[$ep['name']] = $ep; + } + + $this->assertSame(2, $byName['echo']['num_requests']); + $this->assertSame(0, $byName['echo']['num_errors']); + $this->assertSame(1, $byName['boom']['num_requests']); + $this->assertSame(1, $byName['boom']['num_errors']); + $this->assertGreaterThanOrEqual(0, $byName['echo']['processing_time']); + } +} diff --git a/packages/nats/tests/E2E/TlsTest.php b/packages/nats/tests/E2E/TlsTest.php new file mode 100644 index 00000000000..2b0846720ef --- /dev/null +++ b/packages/nats/tests/E2E/TlsTest.php @@ -0,0 +1,70 @@ +url = getenv('NATS_TLS_URL') ?: 'tls://127.0.0.1:14223'; + $this->certs = __DIR__ . '/../fixtures/certs'; + + // Skip when the TLS server isn't running (e.g. local runs without the + // nats-tls compose service); CI brings it up so the tests execute there. + $host = parse_url($this->url, PHP_URL_HOST) ?: '127.0.0.1'; + $port = parse_url($this->url, PHP_URL_PORT) ?: 14223; + $probe = @fsockopen($host, (int) $port, $errno, $errstr, 1.0); + if ($probe === false) { + $this->markTestSkipped("TLS server not reachable at {$this->url}"); + } + fclose($probe); + } + + public function testMutualTlsConnectAndRoundTrip(): void + { + $conn = Connection::connect(new ConnectionOptions( + servers: $this->url, + tlsCaFile: "{$this->certs}/ca.pem", + tlsCertFile: "{$this->certs}/client-cert.pem", + tlsKeyFile: "{$this->certs}/client-key.pem", + )); + + $this->assertTrue($conn->isConnected()); + + $sub = $conn->subscribe('tls.echo'); + $conn->publish('tls.echo', 'secure-hello'); + $msg = $sub->nextMessage(2.0); + + $this->assertInstanceOf(\Utopia\NATS\Message::class, $msg); + $this->assertSame('secure-hello', $msg->data); + + $conn->close(); + } + + public function testConnectWithoutClientCertIsRejected(): void + { + // Server requires a client certificate (verify: true); connecting with only + // the CA must fail the TLS handshake. + $this->expectException(ConnectionException::class); + + Connection::connect(new ConnectionOptions( + servers: $this->url, + tlsCaFile: "{$this->certs}/ca.pem", + )); + } +} diff --git a/packages/nats/tests/E2E/WebSocketTest.php b/packages/nats/tests/E2E/WebSocketTest.php new file mode 100644 index 00000000000..2a7fbefbbfd --- /dev/null +++ b/packages/nats/tests/E2E/WebSocketTest.php @@ -0,0 +1,68 @@ +url = getenv('NATS_WS_URL') ?: 'ws://127.0.0.1:14224'; + + $host = parse_url($this->url, PHP_URL_HOST) ?: '127.0.0.1'; + $port = parse_url($this->url, PHP_URL_PORT) ?: 14224; + $probe = @fsockopen($host, (int) $port, $errno, $errstr, 1.0); + if ($probe === false) { + $this->markTestSkipped("WebSocket server not reachable at {$this->url}"); + } + fclose($probe); + } + + public function testConnectAndRoundTripOverWebSocket(): void + { + $conn = Connection::connect(new ConnectionOptions( + servers: $this->url, + transportFactory: fn (string $scheme): WebSocketTransport => new WebSocketTransport(secure: $scheme === 'wss'), + )); + + $this->assertTrue($conn->isConnected()); + + $sub = $conn->subscribe('ws.echo'); + $conn->publish('ws.echo', 'hello-over-ws'); + $msg = $sub->nextMessage(2.0); + + $this->assertInstanceOf(\Utopia\NATS\Message::class, $msg); + $this->assertSame('hello-over-ws', $msg->data); + + $conn->close(); + } + + public function testRequestReplyOverWebSocket(): void + { + $conn = Connection::connect(new ConnectionOptions( + servers: $this->url, + transportFactory: fn (string $scheme): WebSocketTransport => new WebSocketTransport(secure: $scheme === 'wss'), + )); + + $conn->subscribe('ws.service', function ($msg) use ($conn): void { + $conn->publish($msg->replyTo, 'pong:' . $msg->data); + }); + + $reply = $conn->request('ws.service', 'ping', 2.0); + $this->assertSame('pong:ping', $reply->data); + + $conn->close(); + } +} diff --git a/packages/nats/tests/HeadersTest.php b/packages/nats/tests/HeadersTest.php new file mode 100644 index 00000000000..a800892f51c --- /dev/null +++ b/packages/nats/tests/HeadersTest.php @@ -0,0 +1,124 @@ +set('X-Foo', 'bar'); + $this->assertSame('bar', $h->get('X-Foo')); + } + + public function testAddMultipleValues(): void + { + $h = new Headers(); + $h->add('X-Multi', 'a'); + $h->add('X-Multi', 'b'); + $this->assertSame('a', $h->get('X-Multi')); + $this->assertSame(['a', 'b'], $h->getAll('X-Multi')); + } + + public function testSetOverwrites(): void + { + $h = new Headers(); + $h->add('X-Key', 'a'); + $h->add('X-Key', 'b'); + $h->set('X-Key', 'c'); + $this->assertSame(['c'], $h->getAll('X-Key')); + } + + public function testHas(): void + { + $h = new Headers(); + $this->assertFalse($h->has('X-Missing')); + $h->set('X-Present', 'yes'); + $this->assertTrue($h->has('X-Present')); + } + + public function testDelete(): void + { + $h = new Headers(); + $h->set('X-Del', 'val'); + $h->delete('X-Del'); + $this->assertFalse($h->has('X-Del')); + } + + public function testCount(): void + { + $h = new Headers(); + $this->assertCount(0, $h); + $h->set('A', '1'); + $h->set('B', '2'); + $this->assertCount(2, $h); + } + + public function testToWire(): void + { + $h = new Headers(); + $h->set('X-Key', 'value'); + $h->set('Content-Type', 'text/plain'); + + $wire = $h->toWire(); + $this->assertStringStartsWith("NATS/1.0\r\n", $wire); + $this->assertStringContainsString("X-Key: value\r\n", $wire); + $this->assertStringContainsString("Content-Type: text/plain\r\n", $wire); + $this->assertStringEndsWith("\r\n\r\n", $wire); + } + + public function testToWireWithStatus(): void + { + $h = new Headers(); + $h->setStatus('503', 'No Responders'); + + $wire = $h->toWire(); + $this->assertStringStartsWith("NATS/1.0 503 No Responders\r\n", $wire); + } + + public function testFromWire(): void + { + $wire = "NATS/1.0\r\nX-Key: value\r\nAnother: test\r\n\r\n"; + $h = Headers::fromWire($wire); + + $this->assertSame('value', $h->get('X-Key')); + $this->assertSame('test', $h->get('Another')); + $this->assertSame('', $h->getStatus()); + } + + public function testFromWireWithStatus(): void + { + $wire = "NATS/1.0 503 No Responders\r\n\r\n"; + $h = Headers::fromWire($wire); + + $this->assertSame('503', $h->getStatus()); + $this->assertSame('No Responders', $h->getDescription()); + } + + public function testFromWireWithStatusOnly(): void + { + $wire = "NATS/1.0 408\r\n\r\n"; + $h = Headers::fromWire($wire); + + $this->assertSame('408', $h->getStatus()); + $this->assertSame('', $h->getDescription()); + } + + public function testRoundTrip(): void + { + $original = new Headers(); + $original->set('Nats-Msg-Id', 'abc-123'); + $original->set('X-Custom', 'test'); + + $wire = $original->toWire(); + $parsed = Headers::fromWire($wire); + + $this->assertSame('abc-123', $parsed->get('Nats-Msg-Id')); + $this->assertSame('test', $parsed->get('X-Custom')); + } +} diff --git a/packages/nats/tests/InboxTest.php b/packages/nats/tests/InboxTest.php new file mode 100644 index 00000000000..145a084c546 --- /dev/null +++ b/packages/nats/tests/InboxTest.php @@ -0,0 +1,38 @@ +assertStringStartsWith('_INBOX.', $inbox); + $this->assertSame(29, \strlen($inbox)); // "_INBOX." (7) + 22 chars + } + + public function testCreateWithCustomPrefix(): void + { + $inbox = Inbox::create('MY_INBOX'); + $this->assertStringStartsWith('MY_INBOX.', $inbox); + } + + public function testCreateUnique(): void + { + $inbox1 = Inbox::create(); + $inbox2 = Inbox::create(); + $this->assertNotSame($inbox1, $inbox2); + } + + public function testGenerateId(): void + { + $id = Inbox::generateId(); + $this->assertSame(22, \strlen($id)); + $this->assertMatchesRegularExpression('/^[0-9A-Za-z]+$/', $id); + } +} diff --git a/packages/nats/tests/JetStream/ConfigTest.php b/packages/nats/tests/JetStream/ConfigTest.php new file mode 100644 index 00000000000..15dfbe4c3bd --- /dev/null +++ b/packages/nats/tests/JetStream/ConfigTest.php @@ -0,0 +1,102 @@ +toArray(); + $this->assertSame('deliver.here', $arr['deliver_subject']); + $this->assertSame('group-a', $arr['deliver_group']); + $this->assertTrue($arr['flow_control']); + $this->assertSame(2_000_000_000, $arr['idle_heartbeat']); + } + + public function testConsumerConfigRoundTrip(): void + { + $config = ConsumerConfig::fromArray([ + 'deliver_subject' => 'x.y', + 'flow_control' => true, + 'idle_heartbeat' => 5_000_000_000, + ]); + + $this->assertSame('x.y', $config->deliverSubject); + $this->assertTrue($config->flowControl); + $this->assertEqualsWithDelta(5.0, $config->idleHeartbeat, PHP_FLOAT_EPSILON); + } + + public function testConsumerInfoPopulatesSequences(): void + { + $info = ConsumerInfo::fromArray([ + 'stream_name' => 'S', + 'name' => 'C', + 'num_pending' => 7, + 'num_ack_pending' => 3, + 'delivered' => ['consumer_seq' => 4, 'stream_seq' => 10], + 'ack_floor' => ['consumer_seq' => 1, 'stream_seq' => 7], + ]); + + $this->assertSame(7, $info->numPending); + $this->assertSame(3, $info->numAckPending); + $this->assertInstanceOf(SequenceInfo::class, $info->delivered); + $this->assertSame(4, $info->delivered->consumerSeq); + $this->assertSame(10, $info->delivered->streamSeq); + $this->assertSame(7, $info->ackFloor->streamSeq); + } + + public function testBackoffSerializesToNanos(): void + { + $config = new ConsumerConfig( + ackWait: 10.0, + maxDeliver: 5, + backoff: [10.0, 30.0, 120.0], + ); + + $arr = $config->toArray(); + $this->assertSame([10_000_000_000, 30_000_000_000, 120_000_000_000], $arr['backoff']); + $this->assertSame(10_000_000_000, $arr['ack_wait']); + } + + public function testBackoffOmittedWhenUnset(): void + { + $this->assertArrayNotHasKey('backoff', (new ConsumerConfig())->toArray()); + } + + public function testBackoffRoundTrip(): void + { + $config = ConsumerConfig::fromArray([ + 'backoff' => [5_000_000_000, 60_000_000_000], + ]); + + $this->assertSame([5.0, 60.0], $config->backoff); + } + + public function testStreamMessageDecodesBase64Payload(): void + { + $msg = StreamMessage::fromArray([ + 'subject' => 'foo.bar', + 'seq' => 42, + 'data' => base64_encode('the-payload'), + ]); + + $this->assertSame('foo.bar', $msg->subject); + $this->assertSame(42, $msg->sequence); + $this->assertSame('the-payload', $msg->data); + } +} diff --git a/packages/nats/tests/ParserFrameIntegrityTest.php b/packages/nats/tests/ParserFrameIntegrityTest.php new file mode 100644 index 00000000000..f20a756e98b --- /dev/null +++ b/packages/nats/tests/ParserFrameIntegrityTest.php @@ -0,0 +1,85 @@ +pushInbound("MSG foo 1 5\r\nhello\r\n"); + + [$op, $data] = $parser->next(1.0); + + $this->assertSame(ServerOp::Msg, $op); + $this->assertSame('hello', $data['payload']); + $this->assertSame('foo', $data['subject']); + } + + public function testMidFrameTimeoutIsNotReportedAsATimeout(): void + { + $fake = new FakeTransport(); + $parser = new Parser($fake); + // Header announces five payload bytes that never arrive. + $fake->pushInbound("MSG foo 1 5\r\n"); + + try { + $parser->next(0.01); + $this->fail('Expected the mid-frame failure to raise'); + } catch (TimeoutException) { + $this->fail('A mid-frame failure must not surface as a timeout: the caller would read it as "nothing arrived" and carry on against a desynced stream'); + } catch (ConnectionException $e) { + $this->assertStringContainsString('desynced', $e->getMessage()); + } + } + + public function testPoisonedParserRefusesEvenValidFollowingData(): void + { + $fake = new FakeTransport(); + $parser = new Parser($fake); + $fake->pushInbound("MSG foo 1 5\r\n"); + + try { + $parser->next(0.01); + } catch (ConnectionException) { + // expected; the parser is now poisoned + } + + // Whatever arrives next cannot be trusted to start on a frame boundary, + // so the parser must keep refusing until the connection is rebuilt. + $fake->pushInbound("PING\r\n"); + + $this->expectException(ConnectionException::class); + $parser->next(1.0); + } + + public function testMidFrameFailureSurfacesTheOriginalCause(): void + { + $fake = new FakeTransport(); + $parser = new Parser($fake); + $fake->pushInbound("HMSG foo 1 12 20\r\n"); + + try { + $parser->next(0.01); + $this->fail('Expected the mid-frame failure to raise'); + } catch (ConnectionException $e) { + $this->assertInstanceOf(TimeoutException::class, $e->getPrevious()); + } + } +} diff --git a/packages/nats/tests/Protocol/ParserTest.php b/packages/nats/tests/Protocol/ParserTest.php new file mode 100644 index 00000000000..7c52373a81b --- /dev/null +++ b/packages/nats/tests/Protocol/ParserTest.php @@ -0,0 +1,185 @@ +pos >= \strlen($this->data)) { + return ''; + } + $chunk = substr($this->data, $this->pos, $maxBytes); + $this->pos += \strlen($chunk); + return $chunk; + } + + public function readLine(?float $timeout = null): string + { + $nlPos = strpos($this->data, "\n", $this->pos); + if ($nlPos === false) { + return ''; + } + $line = substr($this->data, $this->pos, $nlPos - $this->pos + 1); + $this->pos = $nlPos + 1; + return $line; + } + + public function upgradeTls(array $options): void + { + } + public function isConnected(): bool + { + return true; + } + public function close(): void + { + } + }; + + return new Parser($transport); + } + + public function testParseInfo(): void + { + $parser = $this->createParser("INFO {\"server_id\":\"test\",\"version\":\"2.10.0\"}\r\n"); + [$op, $data] = $parser->next(); + $this->assertSame(ServerOp::Info, $op); + $this->assertSame('test', $data['server_id']); + $this->assertSame('2.10.0', $data['version']); + } + + public function testParsePing(): void + { + $parser = $this->createParser("PING\r\n"); + [$op, $data] = $parser->next(); + $this->assertSame(ServerOp::Ping, $op); + $this->assertNull($data); + } + + public function testParsePong(): void + { + $parser = $this->createParser("PONG\r\n"); + [$op, $data] = $parser->next(); + $this->assertSame(ServerOp::Pong, $op); + } + + public function testParseOk(): void + { + $parser = $this->createParser("+OK\r\n"); + [$op, $data] = $parser->next(); + $this->assertSame(ServerOp::Ok, $op); + } + + public function testParseErr(): void + { + $parser = $this->createParser("-ERR 'Authorization Violation'\r\n"); + [$op, $data] = $parser->next(); + $this->assertSame(ServerOp::Err, $op); + $this->assertSame('Authorization Violation', $data); + } + + public function testParseMsg(): void + { + $parser = $this->createParser("MSG foo.bar 1 5\r\nhello\r\n"); + [$op, $data] = $parser->next(); + $this->assertSame(ServerOp::Msg, $op); + $this->assertSame('foo.bar', $data['subject']); + $this->assertSame('1', $data['sid']); + $this->assertNull($data['replyTo']); + $this->assertSame('hello', $data['payload']); + } + + public function testParseMsgWithReply(): void + { + $parser = $this->createParser("MSG foo.bar 1 _INBOX.xyz 5\r\nhello\r\n"); + [$op, $data] = $parser->next(); + $this->assertSame(ServerOp::Msg, $op); + $this->assertSame('foo.bar', $data['subject']); + $this->assertSame('1', $data['sid']); + $this->assertSame('_INBOX.xyz', $data['replyTo']); + $this->assertSame('hello', $data['payload']); + } + + public function testParseMsgEmpty(): void + { + $parser = $this->createParser("MSG foo 1 0\r\n\r\n"); + [$op, $data] = $parser->next(); + $this->assertSame(ServerOp::Msg, $op); + $this->assertSame('', $data['payload']); + } + + public function testParseHmsg(): void + { + $headers = "NATS/1.0\r\nX-Test: value\r\n\r\n"; + $headerLen = \strlen($headers); + $payload = 'hello'; + $totalLen = $headerLen + \strlen($payload); + $parser = $this->createParser("HMSG foo.bar 1 {$headerLen} {$totalLen}\r\n{$headers}{$payload}\r\n"); + [$op, $data] = $parser->next(); + $this->assertSame(ServerOp::HMsg, $op); + $this->assertSame('foo.bar', $data['subject']); + $this->assertSame($headers, $data['headers']); + $this->assertSame('hello', $data['payload']); + } + + public function testParseHmsgWithReply(): void + { + $headers = "NATS/1.0\r\n\r\n"; + $headerLen = \strlen($headers); + $totalLen = $headerLen + 3; + $parser = $this->createParser("HMSG foo 1 reply {$headerLen} {$totalLen}\r\n{$headers}bar\r\n"); + [$op, $data] = $parser->next(); + $this->assertSame(ServerOp::HMsg, $op); + $this->assertSame('reply', $data['replyTo']); + $this->assertSame('bar', $data['payload']); + } + + public function testParseMultipleOps(): void + { + $parser = $this->createParser("PING\r\n+OK\r\nMSG foo 1 3\r\nabc\r\n"); + + [$op1,] = $parser->next(); + $this->assertSame(ServerOp::Ping, $op1); + + [$op2,] = $parser->next(); + $this->assertSame(ServerOp::Ok, $op2); + + [$op3, $data3] = $parser->next(); + $this->assertSame(ServerOp::Msg, $op3); + $this->assertSame('abc', $data3['payload']); + } + + public function testParseUnknownOpThrows(): void + { + $parser = $this->createParser("UNKNOWN command\r\n"); + $this->expectException(ProtocolException::class); + $parser->next(); + } +} diff --git a/packages/nats/tests/Protocol/WriterTest.php b/packages/nats/tests/Protocol/WriterTest.php new file mode 100644 index 00000000000..61b3b3e9c67 --- /dev/null +++ b/packages/nats/tests/Protocol/WriterTest.php @@ -0,0 +1,94 @@ +writer = new Writer(); + } + + public function testConnect(): void + { + $result = $this->writer->connect(['verbose' => false, 'lang' => 'php']); + $this->assertSame("CONNECT {\"verbose\":false,\"lang\":\"php\"}\r\n", $result); + } + + public function testPub(): void + { + $result = $this->writer->pub('foo.bar', 'hello'); + $this->assertSame("PUB foo.bar 5\r\nhello\r\n", $result); + } + + public function testPubEmpty(): void + { + $result = $this->writer->pub('foo', ''); + $this->assertSame("PUB foo 0\r\n\r\n", $result); + } + + public function testPubWithReply(): void + { + $result = $this->writer->pub('foo', 'world', '_INBOX.abc'); + $this->assertSame("PUB foo _INBOX.abc 5\r\nworld\r\n", $result); + } + + public function testHpub(): void + { + $headers = "NATS/1.0\r\nX-Key: value\r\n\r\n"; + $result = $this->writer->hpub('foo', $headers, 'hello'); + $headerLen = \strlen($headers); + $totalLen = $headerLen + 5; + $this->assertSame("HPUB foo {$headerLen} {$totalLen}\r\n{$headers}hello\r\n", $result); + } + + public function testHpubWithReply(): void + { + $headers = "NATS/1.0\r\n\r\n"; + $result = $this->writer->hpub('foo', $headers, 'data', 'reply'); + $headerLen = \strlen($headers); + $totalLen = $headerLen + 4; + $this->assertSame("HPUB foo reply {$headerLen} {$totalLen}\r\n{$headers}data\r\n", $result); + } + + public function testSub(): void + { + $result = $this->writer->sub('foo.>', '1'); + $this->assertSame("SUB foo.> 1\r\n", $result); + } + + public function testSubWithQueue(): void + { + $result = $this->writer->sub('foo', '5', 'workers'); + $this->assertSame("SUB foo workers 5\r\n", $result); + } + + public function testUnsub(): void + { + $result = $this->writer->unsub('3'); + $this->assertSame("UNSUB 3\r\n", $result); + } + + public function testUnsubWithMax(): void + { + $result = $this->writer->unsub('3', 10); + $this->assertSame("UNSUB 3 10\r\n", $result); + } + + public function testPing(): void + { + $this->assertSame("PING\r\n", $this->writer->ping()); + } + + public function testPong(): void + { + $this->assertSame("PONG\r\n", $this->writer->pong()); + } +} diff --git a/packages/nats/tests/ReconnectBackoffTest.php b/packages/nats/tests/ReconnectBackoffTest.php new file mode 100644 index 00000000000..dc6c5f237f9 --- /dev/null +++ b/packages/nats/tests/ReconnectBackoffTest.php @@ -0,0 +1,62 @@ +assertEqualsWithDelta(0.0, Connection::reconnectBackoff(0, 2.0, 30.0), PHP_FLOAT_EPSILON); + } + + public function testBackoffGrowsExponentially(): void + { + // base 2.0, factor 2.0: 2, 4, 8, 16 ... + $this->assertEqualsWithDelta(2.0, Connection::reconnectBackoff(1, 2.0, 100.0), PHP_FLOAT_EPSILON); + $this->assertEqualsWithDelta(4.0, Connection::reconnectBackoff(2, 2.0, 100.0), PHP_FLOAT_EPSILON); + $this->assertEqualsWithDelta(8.0, Connection::reconnectBackoff(3, 2.0, 100.0), PHP_FLOAT_EPSILON); + $this->assertEqualsWithDelta(16.0, Connection::reconnectBackoff(4, 2.0, 100.0), PHP_FLOAT_EPSILON); + } + + public function testBackoffIsCapped(): void + { + $cap = 8.0; + for ($attempt = 1; $attempt <= 20; $attempt++) { + $this->assertLessThanOrEqual($cap, Connection::reconnectBackoff($attempt, 2.0, $cap)); + } + + // Well past the cap it stays pinned. + $this->assertSame($cap, Connection::reconnectBackoff(10, 2.0, $cap)); + } + + public function testCustomFactor(): void + { + $this->assertEqualsWithDelta(1.0, Connection::reconnectBackoff(1, 1.0, 100.0, 3.0), PHP_FLOAT_EPSILON); + $this->assertEqualsWithDelta(3.0, Connection::reconnectBackoff(2, 1.0, 100.0, 3.0), PHP_FLOAT_EPSILON); + $this->assertEqualsWithDelta(9.0, Connection::reconnectBackoff(3, 1.0, 100.0, 3.0), PHP_FLOAT_EPSILON); + } + + public function testBufferAcceptsUntilCap(): void + { + $cap = 100; + + $this->assertTrue(Connection::reconnectBufferAccepts(0, 50, $cap)); + $this->assertTrue(Connection::reconnectBufferAccepts(50, 50, $cap), 'exactly at cap fits'); + $this->assertFalse(Connection::reconnectBufferAccepts(50, 51, $cap), 'one byte over cap'); + $this->assertFalse(Connection::reconnectBufferAccepts(100, 1, $cap)); + } + + public function testZeroCapDisablesBuffering(): void + { + $this->assertFalse(Connection::reconnectBufferAccepts(0, 1, 0)); + $this->assertFalse(Connection::reconnectBufferAccepts(0, 1, -5)); + } +} diff --git a/packages/nats/tests/RequestsTest.php b/packages/nats/tests/RequestsTest.php new file mode 100644 index 00000000000..eab46743072 --- /dev/null +++ b/packages/nats/tests/RequestsTest.php @@ -0,0 +1,448 @@ + $fake)); + $fake->onWrite = static function (string $wire, FakeTransport $fake): void { + if (str_starts_with($wire, 'PUB ')) { + $fake->pushInbound("PING\r\n"); + } elseif ($wire === "PONG\r\n") { + throw new ConnectionException('PONG write failed'); + } + }; + $results = []; + $connection->requestBatch([new Request(subject: 'one'), new Request(subject: 'two')], static function (int $index, Message|\Throwable $result) use (&$results): void { + $results[$index] = $result; + }); + $this->assertCount(2, $results); + $this->assertInstanceOf(ConnectionException::class, $results[0]); + $this->assertSame($results[0], $results[1]); + $this->assertFalse($connection->isConnected()); + $connection->close(); + } + + #[DataProvider('reconnect')] + public function testClosingErrorDisconnectsBeforeThrowingCallback(bool $reconnect): void + { + $fake = new FakeTransport(); + $failure = new ConnectionException('Application callback failed'); + $connection = null; + $connection = Connection::connect(new ConnectionOptions( + allowReconnect: $reconnect, + onError: function () use (&$connection, $failure): never { + $this->assertFalse($connection->isConnected()); + throw $failure; + }, + transportFactory: fn (): FakeTransport => $fake, + )); + $fake->onWrite = static function (string $wire, FakeTransport $fake): void { + if (str_starts_with($wire, 'PUB ')) { + $fake->pushInbound("-ERR 'Stale Connection'\r\n"); + } + }; + $callbacks = 0; + try { + $connection->requestBatch([new Request('one'), new Request('two')], static function () use (&$callbacks): void { + $callbacks++; + }); + $this->fail('Expected the callback exception'); + } catch (\Throwable $error) { + $this->assertSame($failure, $error); + } + $this->assertFalse($connection->isConnected()); + $this->assertSame(0, $callbacks); + $this->assertSame(1, substr_count($fake->written, 'PUB one ')); + $this->assertSame(1, substr_count($fake->written, 'PUB two ')); + $connection->close(); + } + + public static function bufferedReplies(): iterable + { + yield [false, false]; + yield [false, true]; + yield [true, false]; + yield [true, true]; + } + + #[DataProvider('bufferedReplies')] + public function testBufferedPongAndRepliesAreReadBeforeKeepalive(bool $reconnect, bool $pongLast): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions( + allowReconnect: $reconnect, + pingInterval: 0.0, + maxPingsOut: 1, + transportFactory: fn (): FakeTransport => $fake, + )); + $fake->answerPings = false; + $subjects = []; + $fake->onWrite = static function (string $wire, FakeTransport $fake) use (&$subjects, $pongLast): void { + if (str_starts_with($wire, 'PUB ')) { + preg_match_all('/PUB work ([^ ]+) 0\r\n\r\n/', $wire, $matches); + $subjects = $matches[1]; + } elseif ($wire === "PING\r\n") { + // One transport read buffers +OK, the PONG, and both replies. + $fake->pushInbound($pongLast ? "+OK\r\n" : "+OK\r\nPONG\r\n"); + foreach ($subjects as $subject) { + $fake->pushInbound("MSG {$subject} 1 2\r\nok\r\n"); + } + if ($pongLast) { + $fake->pushInbound("PONG\r\n"); + } + } + }; + $results = []; + $connection->requestBatch([new Request('work'), new Request('work')], static function (int $index, Message|\Throwable $result) use (&$results): void { + $results[$index] = $result; + }); + $this->assertCount(2, $results); + foreach ($results as $result) { + $this->assertInstanceOf(Message::class, $result); + $this->assertSame('ok', $result->data); + } + $this->assertTrue($connection->isConnected()); + $this->assertSame(2, substr_count($fake->written, 'PUB work ')); + $this->assertSame('ok', $connection->request('work')->data); + $connection->close(); + } + + public static function keepalive(): iterable + { + yield 'request, silent peer' => [false, false]; + yield 'batch, silent peer' => [true, false]; + yield 'request, failed ping' => [false, true]; + yield 'batch, failed ping' => [true, true]; + } + + #[DataProvider('keepalive')] + public function testKeepaliveFailureDoesNotReplayRequests(bool $batch, bool $failedWrite): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions( + allowReconnect: true, + maxReconnectAttempts: 1, + pingInterval: 0.0, + maxPingsOut: 1, + transportFactory: fn (): FakeTransport => $fake, + )); + $fake->answerPings = false; + $before = substr_count($fake->written, "PING\r\n"); + if ($failedWrite) { + $fake->onWrite = static function (string $wire): void { + if ($wire === "PING\r\n") { + throw new ConnectionException('PING write failed'); + } + }; + } + $outcomes = []; + for ($attempt = 0; $attempt < ($failedWrite ? 1 : 2); $attempt++) { + try { + if ($batch) { + $connection->requestBatch([new Request('work')], static function (int $index, Message|\Throwable $result) use (&$outcomes): void { + $outcomes[] = $result; + }, 0.01); + } else { + $connection->request('work', timeout: 0.01); + } + } catch (ConnectionException $error) { + $outcomes[] = $error; + } + } + $this->assertCount($failedWrite ? 1 : 2, $outcomes); + $this->assertInstanceOf(ConnectionException::class, $outcomes[array_key_last($outcomes)]); + $this->assertNotInstanceOf(TimeoutException::class, $outcomes[array_key_last($outcomes)]); + $this->assertSame($before + 1, substr_count($fake->written, "PING\r\n")); + $this->assertSame(1, substr_count($fake->written, 'PUB work ')); + $this->assertFalse($connection->isConnected()); + $connection->close(); + } + + public function testPipelinesRequestsAndRetainsOutOfOrderPartialReplies(): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions(transportFactory: fn (): FakeTransport => $fake)); + $groups = []; + $fake->onWrite = static function (string $wire, FakeTransport $fake) use (&$groups): void { + preg_match_all('/PUB work\.(\d+) ([^ ]+) 0\r\n\r\n/', $wire, $matches, PREG_SET_ORDER); + if ($matches === []) { + return; + } + $groups[] = \count($matches); + foreach (array_reverse($matches) as $match) { + if ($match[1] !== '2') { + $fake->pushInbound("MSG {$match[2]} 1 1\r\n{$match[1]}\r\n"); + } + } + }; + $results = []; + $connection->requestBatch([ + new Request(subject: 'work.1'), new Request(subject: 'work.2'), new Request(subject: 'work.3'), + ], static function (int $index, Message|\Throwable $result) use (&$results): void { + $results[$index] = $result; + }, 0.01); + $this->assertSame([3], $groups, 'all requests are sent before waiting for replies'); + $this->assertSame('1', $results[0]->data); + $this->assertInstanceOf(TimeoutException::class, $results[1]); + $this->assertSame('3', $results[2]->data); + $this->assertSame([2, 0, 1], array_keys($results), 'successful replies arrive before the missing reply times out'); + $this->assertSame('1', $connection->request('work.1')->data); + $connection->close(); + } + + public function testSubscriptionCallbackExceptionIsNotATransportFailure(): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions(transportFactory: fn (): FakeTransport => $fake)); + $failure = new ConnectionException('Application callback failed'); + $connection->subscribe('events', static fn () => throw $failure); + $fake->onWrite = static function (string $wire, FakeTransport $fake): void { + if (str_starts_with($wire, 'PUB ')) { + $fake->pushInbound("MSG events 1 1\r\nx\r\n"); + } + }; + $called = false; + try { + $connection->requestBatch([new Request('work')], static function () use (&$called): void { + $called = true; + }); + self::fail('Application exception must propagate'); + } catch (ConnectionException $error) { + $this->assertSame($failure, $error); + } + $this->assertFalse($called); + $this->assertTrue($connection->isConnected()); + $connection->close(); + } + + public function testSingleAndBatchRequestsShareSubjectValidation(): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions(transportFactory: fn (): FakeTransport => $fake)); + $before = $fake->written; + $refused = 0; + foreach (['', 'bad subject', 'work.*', 'work..x'] as $subject) { + foreach ([fn (): \Utopia\NATS\Request => new Request($subject), fn (): \Utopia\NATS\Message => $connection->request($subject)] as $create) { + try { + $create(); + self::fail('Invalid subject must be refused'); + } catch (\InvalidArgumentException) { + $refused++; + } + } + } + $this->assertSame(8, $refused); + $this->assertSame($before, $fake->written); + $connection->close(); + } + + public function testAmbiguousWriteIsNotReplayedAndDisconnects(): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions(transportFactory: fn (): FakeTransport => $fake)); + $writes = 0; + $fake->onWrite = static function (string $wire) use (&$writes): void { + if (str_starts_with($wire, 'PUB ')) { + $writes++; + throw new ConnectionException('Lost connection after write'); + } + }; + $results = []; + $connection->requestBatch([new Request(subject: 'first'), new Request(subject: 'second')], static function (int $index, Message|\Throwable $result) use (&$results): void { + $results[$index] = $result; + }); + $this->assertSame(1, $writes); + $this->assertInstanceOf(ConnectionException::class, $results[0]); + $this->assertInstanceOf(ConnectionException::class, $results[1]); + $this->assertFalse($connection->isConnected()); + $connection->close(); + } + + public function testCallbackFailureAbortsAndLeavesConnectionUsable(): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions(subPendingMsgsLimit: 1, onSlowConsumer: static fn () => self::fail('Late replies must not accumulate'), transportFactory: fn (): FakeTransport => $fake)); + $this->respond($fake); + $seen = []; + // Even transport-shaped callback errors must not become request failures. + $failure = new ConnectionException('Callback failed'); + try { + $connection->requestBatch(array_fill(0, 4, new Request(subject: 'work')), static function (int $index, Message|\Throwable $result) use (&$seen, $failure): never { + $seen[$index] = $result; + throw $failure; + }); + self::fail('Callback error must be surfaced'); + } catch (ConnectionException $error) { + $this->assertSame($failure, $error); + } + $this->assertSame([0], array_keys($seen)); + $this->assertSame('ok', $connection->request('work')->data); + $connection->close(); + } + + public function testHeadersAndNoRespondersRetainIndependentOutcomes(): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions(transportFactory: fn (): FakeTransport => $fake)); + $observed = ''; + $fake->onWrite = static function (string $wire, FakeTransport $fake) use (&$observed): void { + if (preg_match('/HPUB work ([^ ]+) (\d+) (\d+)\r\n/', $wire, $match)) { + $observed = $wire; + $fake->pushInbound("MSG {$match[1]} 1 2\r\nok\r\n"); + } + if (preg_match('/PUB missing ([^ ]+) 0\r\n/', $wire, $match)) { + $header = "NATS/1.0 503\r\n\r\n"; + $length = \strlen($header); + $fake->pushInbound("HMSG {$match[1]} 1 {$length} {$length}\r\n{$header}\r\n"); + } + }; + $headers = new Headers(); + $headers->set('Trace', 'example'); + $results = []; + $connection->requestBatch([ + new Request(subject: 'work', data: 'payload', headers: $headers), + new Request(subject: 'missing'), + ], static function (int $index, Message|\Throwable $result) use (&$results): void { + $results[$index] = $result; + }); + $this->assertStringContainsString("Trace: example\r\n", $observed); + $this->assertStringContainsString("\r\npayload\r\n", $observed); + $this->assertSame('ok', $results[0]->data); + $this->assertInstanceOf(NatsException::class, $results[1]); + $connection->close(); + } + + public static function invalid(): iterable + { + yield 'array instead of Request' => [[['subject' => 'work']], 1.0]; + yield 'not a list' => [['key' => new Request('work')], 1.0]; + yield 'timeout' => [[new Request('work')], 0.0]; + yield 'infinite timeout' => [[new Request('work')], INF]; + yield 'invalid later request' => [[new Request('work'), null], 1.0]; + } + + #[DataProvider('invalid')] + public function testInvalidInputDoesNotWriteOrInvokeCallbacks(array $requests, float $timeout): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions(transportFactory: fn (): FakeTransport => $fake)); + $before = $fake->written; + $called = false; + try { + $connection->requestBatch($requests, static function () use (&$called): void { + $called = true; + }, $timeout); + self::fail('Invalid input must throw'); + } catch (\InvalidArgumentException) { + } + $this->assertSame($before, $fake->written); + $this->assertFalse($called); + $connection->close(); + } + + public function testEmptyInputNeedsNoConnection(): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions(transportFactory: fn (): FakeTransport => $fake)); + $connection->close(); + $before = $fake->written; + $connection->requestBatch([], static fn () => self::fail('Empty batch must not invoke callback')); + $this->assertSame($before, $fake->written); + } + + public function testOversizedLaterRequestDoesNotPublishEarlierRequests(): void + { + $fake = new FakeTransport(['max_payload' => 16]); + $connection = Connection::connect(new ConnectionOptions(transportFactory: fn (): FakeTransport => $fake)); + try { + $connection->requestBatch([ + new Request(subject: 'work'), new Request(subject: 'work', data: str_repeat('x', 17)), + ], static fn () => self::fail('Validation must not invoke callback')); + self::fail('Payload validation must throw'); + } catch (MaxPayloadException) { + } + $this->assertStringNotContainsString('PUB ', $fake->written); + $this->respond($fake); + $this->assertSame('ok', $connection->request('work')->data); + $connection->close(); + } + + public function testCallbackCannotReadConnectionReentrantly(): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions(transportFactory: fn (): FakeTransport => $fake)); + $this->respond($fake); + $refused = 0; + $connection->requestBatch([new Request(subject: 'work')], function () use ($connection, &$refused): void { + foreach ([fn (): \Utopia\NATS\Message => $connection->request('work'), $connection->processMessage(...), fn (): array => $connection->requestMany('work'), fn () => $connection->wait(1), $connection->flush(...), $connection->tick(...), $connection->drain(...), fn () => $connection->requestBatch([], static fn () => null)] as $read) { + try { + $read(); + self::fail('Nested read must be refused'); + } catch (\LogicException) { + $refused++; + } + } + }); + $this->assertSame(8, $refused); + $this->assertSame('ok', $connection->request('work')->data); + $connection->close(); + } + + public function testSharedDeadlineStartsAfterWriteAndIncludesCallbacks(): void + { + $fake = new FakeTransport(); + $connection = Connection::connect(new ConnectionOptions(transportFactory: fn (): FakeTransport => $fake)); + $this->respond($fake); + $respond = $fake->onWrite; + $fake->onWrite = static function (string $wire, FakeTransport $fake) use ($respond): void { + usleep(30_000); + $respond($wire, $fake); + }; + $results = []; + $connection->requestBatch(array_fill(0, 2, new Request(subject: 'work')), static function (int $index, Message|\Throwable $result) use (&$results): void { + $results[$index] = $result; + if ($index === 0) { + usleep(30_000); + } + }, 0.01); + $this->assertSame('ok', $results[0]->data, 'writing does not consume the response timeout'); + $this->assertInstanceOf(TimeoutException::class, $results[1], 'callbacks do not reset the shared deadline'); + $connection->close(); + } + + private function respond(FakeTransport $fake): void + { + $fake->onWrite = static function (string $wire, FakeTransport $fake): void { + preg_match_all('/PUB work ([^ ]+) 0\r\n\r\n/', $wire, $matches, PREG_SET_ORDER); + foreach ($matches as $match) { + $fake->pushInbound("MSG {$match[1]} 1 2\r\nok\r\n"); + } + }; + } +} diff --git a/packages/nats/tests/SubscriptionSlowConsumerTest.php b/packages/nats/tests/SubscriptionSlowConsumerTest.php new file mode 100644 index 00000000000..17c8911501e --- /dev/null +++ b/packages/nats/tests/SubscriptionSlowConsumerTest.php @@ -0,0 +1,111 @@ +sid; + }, + ); + + for ($i = 0; $i < 10; $i++) { + $sub->deliver(new Message('foo', 'x')); + } + + $this->assertNotEmpty($signaled, 'slow consumer callback fired'); + $this->assertSame('1', $signaled[0]); + // Queue is bounded at the limit rather than holding all 10 messages. + $this->assertSame(3, $sub->getPendingCount()); + } + + public function testExceedingByteLimitSignals(): void + { + $fired = 0; + $sub = new Subscription( + sid: '2', + subject: 'foo', + pendingMsgsLimit: 1_000_000, + pendingBytesLimit: 10, + onSlowConsumer: function () use (&$fired): void { + $fired++; + }, + ); + + $sub->deliver(new Message('foo', str_repeat('a', 6))); + $this->assertSame(0, $fired); + $this->assertSame(6, $sub->getPendingBytes()); + + // Next 6 bytes would push past the 10-byte cap. + $sub->deliver(new Message('foo', str_repeat('a', 6))); + $this->assertSame(1, $fired); + $this->assertSame(6, $sub->getPendingBytes(), 'over-limit message dropped'); + $this->assertSame(1, $sub->getPendingCount()); + } + + public function testDrainingResetsSignalAndBytes(): void + { + $fired = 0; + $sub = new Subscription( + sid: '3', + subject: 'foo', + pendingMsgsLimit: 2, + pendingBytesLimit: 1_000_000, + onSlowConsumer: function () use (&$fired): void { + $fired++; + }, + ); + + $sub->deliver(new Message('foo', 'aa')); + $sub->deliver(new Message('foo', 'bb')); + $sub->deliver(new Message('foo', 'cc')); // dropped + signal + $this->assertSame(1, $fired); + + // Drain one, freeing a slot; the signal latch resets so a later overflow re-fires. + $sub->nextMessage(0.0); + $this->assertSame(1, $sub->getPendingCount()); + + $sub->deliver(new Message('foo', 'dd')); // back to limit + $sub->deliver(new Message('foo', 'ee')); // dropped + signal again + $this->assertSame(2, $fired); + } + + public function testCallbackSubscriptionsNeverQueue(): void + { + $received = 0; + $sub = new Subscription( + sid: '4', + subject: 'foo', + callback: function () use (&$received): void { + $received++; + }, + pendingMsgsLimit: 1, + pendingBytesLimit: 1, + ); + + for ($i = 0; $i < 5; $i++) { + $sub->deliver(new Message('foo', 'payload')); + } + + $this->assertSame(5, $received); + $this->assertSame(0, $sub->getPendingCount()); + } +} diff --git a/packages/nats/tests/Support/FakeTransport.php b/packages/nats/tests/Support/FakeTransport.php new file mode 100644 index 00000000000..9d2f5e447f6 --- /dev/null +++ b/packages/nats/tests/Support/FakeTransport.php @@ -0,0 +1,146 @@ + */ + public array $writes = []; + /** @var list> */ + public array $tlsUpgrades = []; + + /** + * Whether the server answers PINGs. Set false to emulate a server that has + * gone away without closing the socket -- the case the stale-connection + * budget exists for. + */ + public bool $answerPings = true; + + private string $inbound = ''; + private bool $connected = false; + + /** @param array $info Fields merged into the served INFO. */ + public function __construct(private readonly array $info = []) + { + } + + public function connect(string $host, int $port, float $timeout): void + { + $this->connected = true; + $this->inbound .= $this->infoLine(); + } + + public function write(string $data): int + { + $this->written .= $data; + $this->writes[] = $data; + if ($this->onWrite instanceof \Closure) { + ($this->onWrite)($data, $this); + } + + // Answer PINGs so the handshake / flush / drain barrier completes. + if ($this->answerPings) { + $pings = substr_count($data, "PING\r\n"); + for ($i = 0; $i < $pings; $i++) { + $this->inbound .= "PONG\r\n"; + } + } + + return \strlen($data); + } + + public function read(int $maxBytes, ?float $timeout = null): string + { + if ($this->inbound === '') { + throw new TimeoutException('No inbound data'); + } + + $chunk = substr($this->inbound, 0, $maxBytes); + $this->inbound = substr($this->inbound, \strlen($chunk)); + + return $chunk; + } + + public function readLine(?float $timeout = null): string + { + $pos = strpos($this->inbound, "\n"); + if ($pos === false) { + throw new TimeoutException('No inbound line'); + } + + $line = substr($this->inbound, 0, $pos + 1); + $this->inbound = substr($this->inbound, $pos + 1); + + return $line; + } + + public function upgradeTls(array $options): void + { + $this->tlsUpgrades[] = $options; + } + + public function isConnected(): bool + { + return $this->connected; + } + + public function close(): void + { + $this->connected = false; + } + + // --- Test helpers --- + + public function pushInbound(string $data): void + { + $this->inbound .= $data; + } + + /** + * Decode the CONNECT payload the client sent during the handshake. + * + * @return array + */ + public function connectPayload(): array + { + if (!preg_match('/CONNECT (\{.*?\})\r\n/', $this->written, $m)) { + throw new ConnectionException('No CONNECT sent'); + } + + return json_decode($m[1], true, 512, JSON_THROW_ON_ERROR); + } + + private function infoLine(): string + { + $info = array_merge([ + 'server_id' => 'FAKE', + 'server_name' => 'fake', + 'version' => '2.10.0', + 'proto' => 1, + 'host' => '127.0.0.1', + 'port' => 4222, + 'headers' => true, + 'auth_required' => false, + 'tls_required' => false, + 'tls_available' => false, + 'max_payload' => 1048576, + 'jetstream' => true, + ], $this->info); + + return 'INFO ' . json_encode($info, JSON_THROW_ON_ERROR) . "\r\n"; + } +} diff --git a/packages/nats/tests/Support/PartialWriteStream.php b/packages/nats/tests/Support/PartialWriteStream.php new file mode 100644 index 00000000000..dcc87a8bfd9 --- /dev/null +++ b/packages/nats/tests/Support/PartialWriteStream.php @@ -0,0 +1,44 @@ +markTestSkipped('Reading a socket option back needs ext-sockets'); + } + + $server = @stream_socket_server('tcp://127.0.0.1:0', $errno, $errstr); + if ($server === false) { + $this->markTestSkipped("Could not open a loopback listener: [{$errno}] {$errstr}"); + } + + $this->server = $server; + $this->address = (string) stream_socket_get_name($server, false); + } + + protected function tearDown(): void + { + if (\is_resource($this->server)) { + fclose($this->server); + } + } + + public function testConnectedSocketHasNagleDisabled(): void + { + [$host, $port] = explode(':', $this->address); + + $transport = new TcpTransport(); + $transport->connect($host, (int) $port, 2.0); + + try { + $this->assertTrue($transport->isConnected()); + + $property = new \ReflectionProperty(TcpTransport::class, 'stream'); + $stream = $property->getValue($transport); + $this->assertIsResource($stream); + + // socket_import_stream shares the underlying descriptor, so this reads + // the option actually set on the connected socket rather than a copy of + // whatever the context asked for. + $socket = socket_import_stream($stream); + if ($socket === false) { + $this->markTestSkipped('This platform cannot import a stream as a socket'); + } + + // Asserted as "not off" rather than a literal: the value read back is + // platform-specific (1 on Linux, 4 on macOS) while 0 is off everywhere. + $this->assertNotSame( + 0, + socket_get_option($socket, SOL_TCP, TCP_NODELAY), + 'TCP_NODELAY must be set, or every request-reply stalls on the peer delayed ACK', + ); + } finally { + $transport->close(); + } + } +} diff --git a/packages/nats/tests/Transport/TcpTransportWriteTest.php b/packages/nats/tests/Transport/TcpTransportWriteTest.php new file mode 100644 index 00000000000..2264a64d0b3 --- /dev/null +++ b/packages/nats/tests/Transport/TcpTransportWriteTest.php @@ -0,0 +1,55 @@ +assertNotFalse($stream); + + $transport = new TcpTransport(); + $prop = new \ReflectionProperty(TcpTransport::class, 'stream'); + $prop->setValue($transport, $stream); + + $data = 'HELLO NATS WORLD'; // 16 bytes, chunk size 3 + $written = $transport->write($data); + + $this->assertSame(\strlen($data), $written); + $this->assertSame($data, PartialWriteStream::$buffer, 'all bytes reached the sink'); + // 16 bytes at 3 bytes/call => ceil(16/3) = 6 fwrite calls. + $this->assertSame(6, PartialWriteStream::$writeCalls); + + fclose($stream); + } +} diff --git a/packages/nats/tests/WriteFailureSemanticsTest.php b/packages/nats/tests/WriteFailureSemanticsTest.php new file mode 100644 index 00000000000..5f8527baeb1 --- /dev/null +++ b/packages/nats/tests/WriteFailureSemanticsTest.php @@ -0,0 +1,92 @@ +publish() waits for a + * PubAck that never arrives and raises TimeoutException. + */ + private function unresponsiveConnection(FakeTransport $fake): Connection + { + return Connection::connect(new ConnectionOptions( + servers: 'nats://127.0.0.1:4222', + allowReconnect: false, + requestTimeout: 0.02, + transportFactory: fn (string $scheme): FakeTransport => $fake, + )); + } + + public function testKeyValueCreatePropagatesATransportFailure(): void + { + $conn = $this->unresponsiveConnection(new FakeTransport()); + $kv = new KeyValue($conn, $conn->jetStream(), 'BUCKET'); + + // Previously rewritten to KeyValueException('Key already exists'), which + // tells the caller the write is settled when in fact it is unknown. + $this->expectException(TimeoutException::class); + $kv->create('key', 'value'); + } + + public function testKeyValueUpdatePropagatesATransportFailure(): void + { + $conn = $this->unresponsiveConnection(new FakeTransport()); + $kv = new KeyValue($conn, $conn->jetStream(), 'BUCKET'); + + // Previously rewritten to KeyValueException('Wrong last revision'). + $this->expectException(TimeoutException::class); + $kv->update('key', 'value', 7); + } + + public function testKeyValueTransportFailureIsNotAKeyValueException(): void + { + $conn = $this->unresponsiveConnection(new FakeTransport()); + $kv = new KeyValue($conn, $conn->jetStream(), 'BUCKET'); + + try { + $kv->create('key', 'value'); + $this->fail('Expected the transport failure to raise'); + } catch (KeyValueException $e) { + $this->fail('A transport failure must not be reported as a CAS verdict: ' . $e->getMessage()); + } catch (TimeoutException $e) { + $this->assertStringContainsString('Request timed out', $e->getMessage()); + } + } + + public function testObjectStorePutPropagatesATransportFailure(): void + { + $conn = $this->unresponsiveConnection(new FakeTransport()); + $store = new ObjectStore($conn, $conn->jetStream(), 'BUCKET'); + + // An empty object writes no chunks, so the first publish attempted is the + // guarded meta publish -- the path that used to purge on any failure. + try { + $store->put('object', ''); + $this->fail('Expected the transport failure to raise'); + } catch (ObjectStoreException $e) { + $this->fail('A transport failure must not be reported as a conflict, and must not purge chunks: ' . $e->getMessage()); + } catch (TimeoutException $e) { + $this->assertStringContainsString('Request timed out', $e->getMessage()); + } + } +} diff --git a/packages/nats/tests/fixtures/certs/ca-key.pem b/packages/nats/tests/fixtures/certs/ca-key.pem new file mode 100644 index 00000000000..a9362fb9e73 --- /dev/null +++ b/packages/nats/tests/fixtures/certs/ca-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQC2iVEZqAA73ET3 +csEhFSwnyMidhZzoU4tGUAY8cPAVpDDVuryH5ZHjHrQkARXQ2zsMILgijPHcii8x +rgXEf7Hd+pGTM08fzy2Ffr82UpyP5HoMqZ3Xdrpb2+j/7VJBhF85YbFDFDFd/LJ6 +jljLyUT390zfh1wt6VE0BM4g+9/TMBKPYbh49dGInUYwoqqDznbkHJc0VoGRaBQK +BDkHPVujb+frRmiEk9jNuoZ6t20Ch8LTJnUmWLEcCOGbQNQLhVPofDd+Hzs4xY2M +PKiaVEIlEUeSqgtkkZCwUfPBuB2RiJeI5GZkgbTwpWKS4lI96Knkg4amZkrXoZaP +h6yK1uNhAgMBAAECggEABcO3uZ1c8dxWmpCXhB7yDQjsj6TyO+1ffvH0sHX0mbUH +YciK6akYqVkv4S8zltaa5RrSFdfo+h5XDTTss7RAqWefWfcLLvr7pLVIU+6EnpAI +QVwd7aRZ5aAVRKJ5sDNTpTFDnYBcFWwZ4Crx/PA3QF7L9lyE+beI5qFRjShx0hkr +Ceup5G2zMA1UZqpSr+tg8mKr6AFem6pegiwr5V57hSxjPDmjsF0sc1nzoSnXH/3s +PrtPFSobiEXywJp7KWPHL8uAHErAvyUgqpd/NpZE+oMSxcGhl2P5Q2lq6etweDX3 +dxg3U8Bt+TvUcmX02AtOo5or2M+0KEcYjnSXSNKmkQKBgQD/hN3boQhov8ULdOEg +Crwbxm4DCJwSv6ds1XBZmwd0vTt7pwZ9dzgIPddr9qfZJYh0R6t9JZBRz3eAcqPp +m8ra46OBKncxUTXPeQeWvEabP5rjThJ8m4cJM6ACaT5gvGolLwjyFwEQjDxye91Y +/CaLZbyc1ngp7A2KBWK0s2vJJQKBgQC24Ue7BeiXcmcektkp4ZUyXkE9oHV1/TZY +OScOmyAQQTY6TUtunkJwjcadnVF4LVl9YlilkOahtyvzC9iwEe/A77vnYB/5LXZz +RsQ0cwqyI1fOAQGu5rLK2epY1WbDwuHgwjWyZlp0m0VQIPKDgKhD/CQfAG57oSrO +34O06hqSjQKBgDnZ5Tkyrmq48G31mnrcs1Y+iO3a5ys/hTKY6IUBGqENrwgB+vAx +K9iB1+IzyDDb1JcvGt6MwK4lINNOIVpk0XrZi01VRgM+dcZxioxQOFQyBGluZbUc +qWgl7hZdqafRQN8GVSoqly0d8xmgfa8F99wbvqZr0Gz9attS+NQcw0yZAoGAMM+o +KY7GKcXrKAnh1H/VJoBpwV0WfMt8ucdnWUqmIDCHnMAMphWvfgMZQ/A9UmooPIMH +TgmCxR7yqAg9aaEf8MsLyBGr9wHfX50/uRALhshQIze5j0kb53CKT/OFXjKhvy+W +zWrsMbtPoLdqgxbP0nV13qBq3KyDlHur15m0RfECgYAF0xX0AbcHMslV7ajPhgk1 +DjowCEv+e+68oOVKIfaBJxUMf/hbbJhILZcyBovHhWKJEt4zwTZ7ilb/c3HrslD6 ++UnkCXg3gvqZcn9s8w/8kSbAdXAloXhiVvR8qTyGaTsJic60dcV4zH/wELp6S0mF +zW5ZxeTugAgtjo2D4FKlTw== +-----END PRIVATE KEY----- diff --git a/packages/nats/tests/fixtures/certs/ca.pem b/packages/nats/tests/fixtures/certs/ca.pem new file mode 100644 index 00000000000..29a63f97739 --- /dev/null +++ b/packages/nats/tests/fixtures/certs/ca.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDETCCAfmgAwIBAgIUcSUzcxuJw5B9bBcUM9oU65+A/tgwDQYJKoZIhvcNAQEL +BQAwFzEVMBMGA1UEAwwMbmF0cy10ZXN0LWNhMCAXDTI2MDgxMTExMTQzOFoYDzIx +MjYwNzE4MTExNDM4WjAXMRUwEwYDVQQDDAxuYXRzLXRlc3QtY2EwggEiMA0GCSqG +SIb3DQEBAQUAA4IBDwAwggEKAoIBAQC2iVEZqAA73ET3csEhFSwnyMidhZzoU4tG +UAY8cPAVpDDVuryH5ZHjHrQkARXQ2zsMILgijPHcii8xrgXEf7Hd+pGTM08fzy2F +fr82UpyP5HoMqZ3Xdrpb2+j/7VJBhF85YbFDFDFd/LJ6jljLyUT390zfh1wt6VE0 +BM4g+9/TMBKPYbh49dGInUYwoqqDznbkHJc0VoGRaBQKBDkHPVujb+frRmiEk9jN +uoZ6t20Ch8LTJnUmWLEcCOGbQNQLhVPofDd+Hzs4xY2MPKiaVEIlEUeSqgtkkZCw +UfPBuB2RiJeI5GZkgbTwpWKS4lI96Knkg4amZkrXoZaPh6yK1uNhAgMBAAGjUzBR +MB0GA1UdDgQWBBTpbolZGopxa06csvvhcRaymzmDNzAfBgNVHSMEGDAWgBTpbolZ +Gopxa06csvvhcRaymzmDNzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA +A4IBAQARVLyEyJllnDp4rs4y3Y93z9znBI+fX/DJE23X0DWXZYHdG2hD/RXnDIAb +vFDJX2Jq9lmXsKTWq1l4jA3hlYcJVJSBqRhm03DdG4y2o9sHeudMYUsU3CigfzGb +0w/pk5/6O+GbqtLTrU3m0umMS40osxM4FpAp1hl+GG98Lrx56N6MuTaOrq6Tf6ar +SwcChWxsMY8/0r/w10CtK3scRESEfJffmE6Q6fgjLmhtwcbdnMGh80btlrhsvrrR +MOIHpBMbVsush5wEDwMqGY5u4gmhswc9rm9S8nwf4O21PU43QXvJjKkmUrvnWyJg +iU9afTPKr3wn9kGjyLZ/6QxGAGWa +-----END CERTIFICATE----- diff --git a/packages/nats/tests/fixtures/certs/client-cert.pem b/packages/nats/tests/fixtures/certs/client-cert.pem new file mode 100644 index 00000000000..d80350555a7 --- /dev/null +++ b/packages/nats/tests/fixtures/certs/client-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDGTCCAgGgAwIBAgIUeaIyHLsYIv+B4URfEjrWzO4PeiwwDQYJKoZIhvcNAQEL +BQAwFzEVMBMGA1UEAwwMbmF0cy10ZXN0LWNhMCAXDTI2MDgxMTExMTQzOFoYDzIx +MjYwNzE4MTExNDM4WjAbMRkwFwYDVQQDDBBuYXRzLXRlc3QtY2xpZW50MIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxAj/FIOX9XjANFJad0V4+DsCWdRt +Y9jjg9S3UfQT5dVjmrCnjp3ON6MsrTDi1jNT1PxXNkiF6VsORSG54NAgVOg0N835 +dcHn14iPxudsbkKptd55AUKsHNYJPdZkkFEWPgeLn/FKxYtaW1m0AQxha5HoNZtR +bIFrk6Rl6K09GDme+Fa+f3Dr4kf11hK40dYJN+JVkpyGEZ1ynm/cvP7ZFKkxPxwT +yEpF3QHpf33RXEpvMTJ6GIolcZntlo1bS7WhLau/jkFo6gcUWDKl1RnOV4Lasg3+ +5zdEe8sPCQGPtRGnms3gf4k/LcZnChh6tc8X8XcdrZiMKNdLCfJWJjOu3wIDAQAB +o1cwVTATBgNVHSUEDDAKBggrBgEFBQcDAjAdBgNVHQ4EFgQU45iN6bX5AQpApk16 +qvd67P9sMEQwHwYDVR0jBBgwFoAU6W6JWRqKcWtOnLL74XEWsps5gzcwDQYJKoZI +hvcNAQELBQADggEBADZWxjitWyI5Vt8cEUWupA6BdBFDlLbJuxiIVwUmyH4RqWTl +0jh5dxlQdGdDPCzkhKHzLOcNLmRNkISPqzcszqlquAm+aFygdWROLpwBUb73kgdZ +WrocrpWRsBhfNqr2mQEk+p33qHi0D52894gUbI+woLlwNa28z3tx9nttGs3gHF6o +86gQRIjQW8c4PcnFrol9z2FX31aOHgDHHuWcbGLFxEycDOn7nSLAex/uITr/1Ens +yU5G6mRlaezXdPIPUghxUgCCLh2pUl3znKURx9AhT+RVV/Oo4hY19buN0C4AJ/km +t2YEpH+Vk84wmLXdSKQwpko1qlbvQ/agPhTS2sU= +-----END CERTIFICATE----- diff --git a/packages/nats/tests/fixtures/certs/client-key.pem b/packages/nats/tests/fixtures/certs/client-key.pem new file mode 100644 index 00000000000..716a0630970 --- /dev/null +++ b/packages/nats/tests/fixtures/certs/client-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDECP8Ug5f1eMA0 +Ulp3RXj4OwJZ1G1j2OOD1LdR9BPl1WOasKeOnc43oyytMOLWM1PU/Fc2SIXpWw5F +Ibng0CBU6DQ3zfl1wefXiI/G52xuQqm13nkBQqwc1gk91mSQURY+B4uf8UrFi1pb +WbQBDGFrkeg1m1FsgWuTpGXorT0YOZ74Vr5/cOviR/XWErjR1gk34lWSnIYRnXKe +b9y8/tkUqTE/HBPISkXdAel/fdFcSm8xMnoYiiVxme2WjVtLtaEtq7+OQWjqBxRY +MqXVGc5XgtqyDf7nN0R7yw8JAY+1EaeazeB/iT8txmcKGHq1zxfxdx2tmIwo10sJ +8lYmM67fAgMBAAECggEAOZGixzV19MKHP1ZH9urSAaPiptq5XS8ecWKQi2KwTTiD +TbQ/IKS7IJ+syReoJdtg7RVFz00w9jylTiv/WEQGmqiuXQ0CC1DscuFJx9HCiAbP +q/cvde1MehNpF13UnoURkRnKKShFGwo3eMg5Em2+Vka8+LEqEkFXhD4tJ01ouan+ +qLJ9XI45S72tVvRAqx7zaIlmXzVUevDiRgyXnz589BguRbKex9Dgnz1oqNSULQbS +t4Jp6m5ohjh2ChN30eYR48DB3+7UlS0ZX3jVC5JLaThxE4VZBSeUrZjQ51L0Q3kp +IIADiN5A4L4P9DawqU11MrG52G/5+6eAF5rUluA8MQKBgQDkq5K5Dz6ju0q5xcbd +DcC9tVbuXVdHVOP8N4eKjVo8HYwEIbWPgTvnORrThN+sqZgMqXMagE9zKVY7mJFd +u+fte/PYTtSWdxbhUZD5JB6ywcz2HRZL3bPx6QYDzRwxlcfqltVoN+BTG/Ki5yzx +ndXFhNSWZIcycB8pMkm9SnIMowKBgQDbduqD0JxnJSrRUwWa6pUrD6mPDRqhnd+f +HmBbmxBupE+BDHgYSlwEj6hN5SjI0Rc2oYmQNqwJ7jodYYn/kLxwf2CEg91zt0JD +FiNpAhNNqHDhheeY7tlOMdypHtjzF8suIrwpGQ3pBg5jxuRXUdVtjqyhYXcMQjyX +cHQbYFWclQKBgA6XfDde56pfCzerjTMR65ivd6qiPi2JFztC9905g+aOJB8nX4SW +xJHOrQ2OTw+fitpT/cpVIbzBiIUpCS2oDfazkC3faeVoGAeswsmDKopRn5ic5o1U +ymTHb+DSN9haLHWlxlxuJFxpTiRjCBUq8zc1cLNgGth3GyXv1ZDErV5tAoGAX37G +GudaNxOYJw9uH/O1RtCA4ms8TFjq5k0Mdq/pnYtHwQqQ81jf42gzPvnZHlDFPhNa +TXNMtuF/TiuUJa1wIa97aw4J/YEfhxuJ+vZZOfBydd+TYBuGWR0fPEPc+SJAWenp +YCSzxdLJzibMUyabrRlpCqvjZmUoAml/TC5F8mUCgYEA2tZaJVS8v7v/mXDC/BsD +pFtGhMDNQugDw0eiFKd/LMh++7cCEMP1+MyJfbs5pm8ImuOx4Zpj1YUw3SPQil1j +BPcN6mez/uFh+IrPLl5AUeuUft/Cgdl96Qgfw7j4bvVVjYeDdQlDwIsYTIffWpBl +1X95UoWygyZCP50IcQbBDB8= +-----END PRIVATE KEY----- diff --git a/packages/nats/tests/fixtures/certs/server-cert.pem b/packages/nats/tests/fixtures/certs/server-cert.pem new file mode 100644 index 00000000000..eefc08d7c04 --- /dev/null +++ b/packages/nats/tests/fixtures/certs/server-cert.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDLjCCAhagAwIBAgIUeaIyHLsYIv+B4URfEjrWzO4PeiswDQYJKoZIhvcNAQEL +BQAwFzEVMBMGA1UEAwwMbmF0cy10ZXN0LWNhMCAXDTI2MDgxMTExMTQzOFoYDzIx +MjYwNzE4MTExNDM4WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQDHjdlPKZGVaVk3i3d+r3vxciLPJI33nSt30dV8 +vLOrobmqSrgBWOtEOjqLcmZ+F/DWcFPb+/xCzmOIjUPS5GgxZWqiAvFKRPNvtDlP +T+d4+MIONbnj39iMYa+0w5vnHhH07FYXqLET9/1pua29+iHQULSFPcA/yNVGLITG +nHEIHnPD18ku25ekt9jEzZ+IJvGPSuV6P0LMbFvj4k8F1LI5A0rJwNC/ROhKYLlT +CjF9Mp1hTunxAzNQ/kdFdXRcTxk1q84/jp3pIFFMolYqXRHohc+xDzH6QQ8YJ9uT +x5pBj9HXXvwQiUP3hhCfBklTME/xwrZwo3EwY7yNGUfAcx6jAgMBAAGjczBxMBoG +A1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATATBgNVHSUEDDAKBggrBgEFBQcDATAd +BgNVHQ4EFgQUYGODh+L1Ax1Xl8e4rkHz/tJ5JtUwHwYDVR0jBBgwFoAU6W6JWRqK +cWtOnLL74XEWsps5gzcwDQYJKoZIhvcNAQELBQADggEBALQsDa4B9evIZMKpCvP3 +Gfe/iyrzWXuDVvkFUe0IAIVKP075diRTVzlI/nzu4urRxEE4G/NGsASi5AsLrS6Q +TQZc/A8nK7lCGS2pX9HLwb7QLZptdCSQoowm1IYZygv9oTMEKB2nEzFYu/FXQzkN +mktTDC1eGmGNuHKAjuffRvKcrFR+LJryPcAUy9hVS59tQd1ZflwoeOnsXGljYFbw +0aE2ov8TGy+vmB0zJswFZ9Kqxtrj7GeJ4spC+Sfy7w+ASurz33SdlLi5CLNlBeqy +I3I/Hcm7di0nxNCnh1nFfdRsggRELrMO42UXTnR++tjUWntxNxdgStnlt7FAhteR +t+o= +-----END CERTIFICATE----- diff --git a/packages/nats/tests/fixtures/certs/server-key.pem b/packages/nats/tests/fixtures/certs/server-key.pem new file mode 100644 index 00000000000..70892854586 --- /dev/null +++ b/packages/nats/tests/fixtures/certs/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDHjdlPKZGVaVk3 +i3d+r3vxciLPJI33nSt30dV8vLOrobmqSrgBWOtEOjqLcmZ+F/DWcFPb+/xCzmOI +jUPS5GgxZWqiAvFKRPNvtDlPT+d4+MIONbnj39iMYa+0w5vnHhH07FYXqLET9/1p +ua29+iHQULSFPcA/yNVGLITGnHEIHnPD18ku25ekt9jEzZ+IJvGPSuV6P0LMbFvj +4k8F1LI5A0rJwNC/ROhKYLlTCjF9Mp1hTunxAzNQ/kdFdXRcTxk1q84/jp3pIFFM +olYqXRHohc+xDzH6QQ8YJ9uTx5pBj9HXXvwQiUP3hhCfBklTME/xwrZwo3EwY7yN +GUfAcx6jAgMBAAECggEAAPRfQT8wKJhP6nGa2TrKq32ByB7jJPeWASU6UbXvYBdO +dY+U8MC1o2Zp6EfcUKcZqlTwOcGrU8B7/me6ltQr4WycRXkhIVtSFMs2aPg3Yb9C +4mGEaaAk/4rqFcHHyaO9rz/AK63NZwyNJEfUmm+LYN0599czR7rUtli5K+AMtF66 +v61ETN3JECJCEMsDM1UhiOBTMK+R8fDSA0ozlRzx4QFhHOe6qRWQX3bwDcrSw1sM +f99VGoq7jnGuJB31LGYA2gj07/HmFnG2smffo7jS05RY4RAbmStd7In3fFULcrfZ +UDdWbB4Vi0nefSwYB+vKBaiqIhbAPgHluLBHZeh2TQKBgQD52p2pDfR64CKxq0d/ +nRJxNWgyacpMAdARW3ilJPtBf6xHwIxYnPcClXpvWKWb/YKPoA9lWqbLq9xQWLkO +i70B0h2AhpH+BW95mRR5YxPO8sCp4pY1eGdttGSYVUGxTzcp+A+WUO3SE0RYnL0t +YPhfsb6MHSLRtQoELcN6cdgkvQKBgQDMdnvhJIHpzBP4Gq9eGvztxJCUBnH/w9xu +X2iK/9t9A3A6vWQvnDNkqZ87F3tnNypAcpWv+BOl+Ds1jyfzM6RCov9sbMUR7nf5 +fgKY7HiVg7bkJQqdjoke0Ay8Q2XVis438mBuxIBdlLXbFqHzg09B84NQf7FZC101 +PXJPzux23wKBgCbrwD2Sd/AgNCdbGBMwA6xw9l9MDYBTk/bwOdX38c/dNJgkzXgM +U4oqgN0RgRLQpeL7uwo92dfGWNaHS6SRbgr3kTLxQ5K/hAL8Lq1OGsrVE+Ai9OU1 +cV8sh7MXLNmTDnPSOZNf3fwoli/7kpicD3TQum8XJtJmEj5eZqIPU+09AoGAby8H +Elk95Bs7yLEKKAxY+hrn2/bdSw4A8mwjd2LGwmZ6Qxb2QgWY1OaAVivgyTHwYfdt +erFWTex8lhVwzgmJC0uDxjwntiaNSzjAxNPihruHWuzhpuf5nK3aHw4vdD31Aetl +kRZ+D2VstuTJ0JfZI639/GpT0G4jk24EmW2WnNUCgYBW/7TbSYEUhq4fHG8RBLli +lhznY1agzTVBu/V9uU1iQpUJ4ZN3+XlLNyakzZ72Lp+6XfYplgFGQqy8P+rsU72o +IbZvxkfgbLwpZQ95yMgThpYAJAMDVjNaEkDDwHGX0vbOdmPQxuBaqqsEkMDwXKJ3 +zt+/r86x0UCZsz39IIfK9A== +-----END PRIVATE KEY----- diff --git a/packages/nats/tests/fixtures/nats-tls.conf b/packages/nats/tests/fixtures/nats-tls.conf new file mode 100644 index 00000000000..16e8e1d44c5 --- /dev/null +++ b/packages/nats/tests/fixtures/nats-tls.conf @@ -0,0 +1,14 @@ +# NATS server with mutual TLS, used by the TLS e2e tests. +jetstream {} +http: 8222 + +tls { + cert_file: "/certs/server-cert.pem" + key_file: "/certs/server-key.pem" + ca_file: "/certs/ca.pem" + verify: true + timeout: 5 + # TLS-first: negotiate TLS at connect (matches the client's tls:// path) instead of + # the plaintext-INFO-then-STARTTLS-upgrade default, which the stream transport can't do. + handshake_first: true +} diff --git a/packages/nats/tests/fixtures/nats-ws.conf b/packages/nats/tests/fixtures/nats-ws.conf new file mode 100644 index 00000000000..6be50e922ec --- /dev/null +++ b/packages/nats/tests/fixtures/nats-ws.conf @@ -0,0 +1,8 @@ +# NATS server with WebSocket enabled, used by the WebSocket transport e2e test. +jetstream {} +http: 8222 + +websocket { + port: 8080 + no_tls: true +} diff --git a/packages/queue/README.md b/packages/queue/README.md index 1f4a854113c..24c80d8f803 100644 --- a/packages/queue/README.md +++ b/packages/queue/README.md @@ -94,7 +94,9 @@ $broker = new Nats( $broker->publish(new Queue('my-queue'), ['type' => 'test_number', 'value' => 123]); ``` -Each queue is a WorkQueue-retention stream (a message is removed once acknowledged) with a companion dead stream. `commit()` acknowledges a message, `reject()` schedules redelivery until `maxDeliver` and then dead-letters — unless the handler declared the failure permanent, which dead-letters it at once — `retry()` re-drives the dead stream onto the queue, and `getQueueSize()` reports pending (consumer `num_pending`) or failed (dead stream) counts. `reap()` is a no-op — redelivery after `ackWait` reclaims jobs stranded by a dead worker. Requires [`utopia-php/nats`](https://github.com/utopia-php/nats). +Each queue is a WorkQueue-retention stream (a message is removed once acknowledged) with a companion dead stream. `commit()` acknowledges a message, `reject()` schedules redelivery until `maxDeliver` and then dead-letters — unless the handler declared the failure permanent, which dead-letters it at once — `retry()` re-drives the dead stream onto the queue, and `getQueueSize()` reports pending (consumer `num_pending`) or failed (dead stream) counts. `reap()` is a no-op — redelivery after `ackWait` reclaims jobs stranded by a dead worker. + +The server-side consumer is created with `max_deliver` one above `maxDeliver`, so reading it off `/jsz` shows the extra delivery. A message whose every attempt died unacknowledged, with no `reject()`, arrives once more and `receive()` dead-letters it without running a handler. At exactly `maxDeliver`, JetStream would retire it silently. It would stay on the work stream, counted as neither pending nor in flight, and only reach the dead stream if a broker happened to be subscribed when the server announced it. The consumer carries a `utopia_queue_spare_delivery` metadata key saying so. A broker under `Provisioning::Require`, which never writes configuration, reads that key to find the spare delivery, and reports through `onError` if it adopts a consumer provisioned without one. Requires [`utopia-php/nats`](https://github.com/utopia-php/nats). ### Shaping a queue diff --git a/packages/queue/phpstan-baseline.neon b/packages/queue/phpstan-baseline.neon index 114818064b0..dfccceffbec 100644 --- a/packages/queue/phpstan-baseline.neon +++ b/packages/queue/phpstan-baseline.neon @@ -984,12 +984,6 @@ parameters: count: 2 path: tests/E2E/NatsBrokerTest.php - - - message: '#^Method Utopia\\NATS\\Transport\\Transport@anonymous/tests/E2E/NatsBrokerTest\.php\:863\:\:upgradeTls\(\) has parameter \$options with no value type specified in iterable type array\.$#' - identifier: missingType.iterableValue - count: 1 - path: tests/E2E/NatsBrokerTest.php - - message: '#^Method Utopia\\Queue\\Tests\\E2E\\NatsBrokerTest\:\:failedAcknowledgement\(\) return type has no value type specified in iterable type iterable\.$#' identifier: missingType.iterableValue diff --git a/packages/queue/src/Broker/Nats.php b/packages/queue/src/Broker/Nats.php index fa70461b4c0..eeb8c5058d7 100644 --- a/packages/queue/src/Broker/Nats.php +++ b/packages/queue/src/Broker/Nats.php @@ -96,6 +96,10 @@ class Nats implements Synchronous, Consumer, Bounded private const int MAX_STREAM_NAME = 255; private const string METADATA_IDENTITY = 'utopia_queue_identity'; + // Consumer-metadata key marking a work consumer provisioned with one delivery past + // maxDeliver, so a broker adopting it can tell that spare from a larger budget. + private const string METADATA_SPARE_DELIVERY = 'utopia_queue_spare_delivery'; + // Retry budget for first-time provisioning of a replicated stream. The window // doubles per attempt because a fixed one re-synchronises the losers: every process // that lost the first race waits the same interval and collides again. See ensure(). @@ -120,6 +124,13 @@ class Nats implements Synchronous, Consumer, Bounded /** @var array max-deliveries advisory subscription per queue */ private array $advisories = []; + /** + * @var array per queue, the delivery that is dead-lettered on + * arrival -- the server's last, when it allows one past maxDeliver; null when + * the consumer allows none, and exhaustion falls to the advisory alone + */ + private array $spareDelivery = []; + /** Publishes the stream recognised as duplicates of an id it already held. */ private int $duplicates = 0; @@ -724,7 +735,32 @@ private function pull(Queue $queue, int $timeout, int $max): array $messages = []; + $spare = $this->spareDelivery[$key] ?? null; + foreach ($deliveries as $jsMessage) { + // The server allows one delivery past maxDeliver (see provision()), and it + // only reaches here when no attempt before it ended in reject() -- a worker + // died holding each one. Dead-lettered on this delivery rather than by the + // advisory, which nobody receives while no broker is subscribed. + if ($spare !== null && $jsMessage->metadata()->numDelivered >= $spare) { + // The consumer is shared: a broker with a larger maxDeliver may have + // reprovisioned it since this one cached its spare. Parking on a stale + // threshold would take attempts the server still allows, so re-read it. + // If the read fails, the cached one stands: this may be the server's last + // delivery, and a throw here would leave it, and every message behind it + // in the batch, unacknowledged -- stranded, if no advisory is caught. + try { + $spare = $this->spareDelivery[$key] = $this->spareOf($this->consumers[$key]->info(true)->config); + } catch (\Throwable $error) { + $this->report($error); + } + if ($spare !== null && $jsMessage->metadata()->numDelivered >= $spare) { + $this->park($queue, $jsMessage, 'max deliveries exceeded'); + + continue; + } + } + try { $data = $this->codec->decode($jsMessage->getData()); } catch (\Throwable) { @@ -735,7 +771,7 @@ private function pull(Queue $queue, int $timeout, int $max): array // Parked rather than thrown: in a batch a throw here would leave // every message behind it unregistered and unacknowledged, each // burning an attempt and a maxAckPending slot for a full ackWait. - $this->park($queue, $jsMessage); + $this->park($queue, $jsMessage, 'payload could not be decoded'); continue; } @@ -753,7 +789,8 @@ private function pull(Queue $queue, int $timeout, int $max): array } /** - * Set aside a message no codec on this worker can read. + * Set aside a message no handler should see: one no codec on this worker can + * read, or one past its last attempt. * * Straight to the dead stream and terminated, rather than NAK'd: every * redelivery would fail the same way, and unacknowledged it would hold a @@ -766,7 +803,7 @@ private function pull(Queue $queue, int $timeout, int $max): array * redelivery and ends on the dead stream anyway, where dropping the ack * first would lose it outright. */ - private function park(Queue $queue, JetStreamMessage $jsMessage): void + private function park(Queue $queue, JetStreamMessage $jsMessage, string $reason): void { try { // The message's own Content-Type, not this codec's: the bytes go over @@ -779,7 +816,7 @@ private function park(Queue $queue, JetStreamMessage $jsMessage): void } $this->js()->publish($this->deadSubject($queue), $jsMessage->getData(), headers: $headers); - $jsMessage->term('payload could not be decoded'); + $jsMessage->term($reason); } catch (\Throwable $error) { $this->report($error); } @@ -876,6 +913,7 @@ public function reconnect(): bool $this->commandsConsumers = []; $this->consumers = []; $this->advisories = []; + $this->spareDelivery = []; $this->provisioned = []; $this->inFlight = []; @@ -1288,20 +1326,25 @@ private function provision(Queue $queue, string $key): void durableName: self::CONSUMER_WORK, ackPolicy: AckPolicy::Explicit, ackWait: $this->ackWait, - maxDeliver: $this->maxDeliver, + // One more than the broker's own budget. reject() dead-letters on the + // maxDeliver-th failure itself; the spare delivery is for the message whose + // every attempt died unacknowledged, which pull() dead-letters on arrival. + // At exactly maxDeliver the server would retire it silently instead, left + // on the work stream where nothing delivers or counts it. + maxDeliver: $this->maxDeliver + 1, filterSubject: $this->workSubject($queue), maxWaiting: $this->maxWaiting, maxAckPending: $this->maxAckPending, inactiveThreshold: $this->inactiveThreshold, + metadata: [self::METADATA_SPARE_DELIVERY => '1'], backoff: $this->backoff, )); - // Best-effort terminal dead-lettering for the crash-loop case: a worker that - // dies (never reject()s) is redelivered by AckWait until maxDeliver, after which - // JetStream stops delivering and emits this advisory. We drain it in receive() - // and move the stuck message to the dead stream. Caveat: core - // advisories are ephemeral, so a message that exhausts while no broker is - // subscribed stays as pending backlog (still visible) rather than dead-lettered. + // The fallback for the spare delivery above dying too: JetStream then stops + // delivering and emits this advisory, which receive() drains to move the + // message to the dead stream. Only a fallback, because core advisories are + // ephemeral: one emitted while no broker is subscribed is lost, and the + // message stays on the work stream reading as neither pending nor in flight. // The queue group is what keeps this to one dead-letter copy. A plain // subscription delivers the advisory to every worker process, and each // of them then publishes its own copy of the exhausted message onto the @@ -1313,6 +1356,7 @@ private function provision(Queue $queue, string $key): void queue: self::ADVISORY_GROUP, ); + $this->spareDelivery[$key] = $this->maxDeliver + 1; $this->provisioned[$key] = true; } @@ -1348,6 +1392,18 @@ private function adopt(Queue $queue, string $key): void $this->consumers[$key] = $this->adoptConsumer($queue, self::CONSUMER_WORK); + // The owner's spare delivery, not this broker's: it exists only if the owner + // provisioned one, which it marks in the consumer's metadata. A consumer from + // before the spare was introduced allows none, and nothing here may change that, + // so it is reported instead -- a message it exhausts while no broker is + // subscribed stays on the work stream until its owner reprovisions. + $config = $this->consumers[$key]->info()->config; + $this->spareDelivery[$key] = $this->spareOf($config); + $allowed = $config->maxDeliver ?? -1; + if ($this->spareDelivery[$key] === null && $allowed >= 1) { + $this->report(new \RuntimeException('NATS consumer "' . self::CONSUMER_WORK . "\" on stream \"{$this->workStream($queue)}\" allows no delivery past max_deliver {$allowed}, so a message exhausted while no broker is subscribed is left on the work stream; reprovision it from the queue's owner.")); + } + // Same advisory subscription provision() takes: a core subscription carries no // configuration, and a broker consuming a pre-provisioned queue still owes its // exhausted messages a dead letter. @@ -1359,6 +1415,18 @@ private function adopt(Queue $queue, string $key): void $this->provisioned[$key] = true; } + /** + * The delivery a work consumer's configuration spares for dead-lettering: its last, + * when the metadata marks one past maxDeliver; null when it spares none. A limit + * below 1 is unlimited, so nothing ever exhausts it. + */ + private function spareOf(ConsumerConfig $config): ?int + { + $allowed = $config->maxDeliver ?? -1; + + return isset($config->metadata[self::METADATA_SPARE_DELIVERY]) && $allowed >= 1 ? $allowed : null; + } + /** Resolve an existing durable consumer, refusing rather than creating it. */ private function adoptConsumer(Queue $queue, string $durable): NatsConsumer { diff --git a/packages/queue/tests/E2E/NatsBrokerTest.php b/packages/queue/tests/E2E/NatsBrokerTest.php index ad4f1586a24..aa692cd4ebb 100644 --- a/packages/queue/tests/E2E/NatsBrokerTest.php +++ b/packages/queue/tests/E2E/NatsBrokerTest.php @@ -9,6 +9,8 @@ use Utopia\NATS\Connection; use Utopia\NATS\ConnectionOptions; use Utopia\NATS\Exception\ConnectionException; +use Utopia\NATS\Exception\NatsException; +use Utopia\NATS\JetStream\ConsumerConfig; use Utopia\NATS\JetStream\DiscardPolicy; use Utopia\NATS\JetStream\StorageType; use Utopia\NATS\Transport\TcpTransport; @@ -605,6 +607,35 @@ public function testCrashLoopedMessageIsTerminallyDeadLettered(): void $broker->close(); } + public function testAMessageExhaustedWhileNoBrokerListensStillReachesTheDeadStream(): void + { + // The max-deliveries advisory is core NATS: nobody subscribed, nobody gets it. + // A worker that dies holding a message's last delivery -- a rollout, a crash + // loop -- used to leave it on the work stream past maxDeliver, where nothing + // delivers it, getQueueSize() reads 0 and it never expires. Seen on staging: + // 110 edge and 35 webhook messages, stranded for over a day. + $url = getenv('NATS_URL') ?: 'nats://127.0.0.1:14225'; + $queue = new Queue('t_' . substr(md5(uniqid('', true)), 0, 8)); + + $dying = new Nats(Connection::connect($url), ackWait: 1.0, maxDeliver: 2); + $dying->publish($queue, ['poison' => true]); + $this->assertCount(1, $dying->receive($queue, 2)); + $this->assertCount(1, $dying->receive($queue, 2)); // redelivered after ackWait: the last, never acked + $dying->close(); + // Not a wait for an event: the gap itself is the scenario. The last delivery's + // ackWait has to lapse while nothing is subscribed, or the advisory is caught. + sleep(2); + + $fresh = new Nats(Connection::connect($url), ackWait: 1.0, maxDeliver: 2); + $this->assertSame([], $fresh->receive($queue, 2), 'an exhausted message must not reach a handler'); + + $js = Connection::connect($url)->jetStream(); + $this->assertSame(0, $js->getStreamInfo('Q_' . strtoupper($queue->name))->state->messages, 'work stream holds nothing'); + $this->assertSame(1, $js->getStreamInfo('Q_' . strtoupper($queue->name) . '_DEAD')->state->messages, 'the message is on the dead stream'); + + $fresh->close(); + } + /** * getQueueSize() must be safe to call while another coroutine is blocked in * receive() on the SAME broker — the shape the Swoole worker runs, where the @@ -908,6 +939,7 @@ public function readLine(?float $timeout = null): string return $this->inner->readLine($timeout); } + /** @param array $options */ public function upgradeTls(array $options): void { $this->inner->upgradeTls($options); @@ -1549,6 +1581,143 @@ public function testRequireLeavesStreamAndConsumerConfigUntouched(): void $owner->close(); } + public function testARequireBrokerDeadLettersOnTheOwnersSpareDelivery(): void + { + // A Require broker writes no configuration, so the spare delivery it can use is + // the one the owner provisioned -- read from the server, not from its own knobs. + $url = getenv('NATS_URL') ?: 'nats://127.0.0.1:14225'; + $queue = new Queue('t_' . substr(md5(uniqid('', true)), 0, 8)); + + $owner = new Nats(Connection::connect($url), ackWait: 1.0, maxDeliver: 2); + $owner->publish($queue, ['poison' => true]); + $this->assertCount(1, $owner->receive($queue, 2)); + $this->assertCount(1, $owner->receive($queue, 2)); + $owner->close(); + sleep(2); // the last delivery's ackWait lapses with nothing subscribed + + $adopter = new Nats(Connection::connect($url), ackWait: 1.0, maxDeliver: 9, provisioning: Provisioning::Require); + $this->assertSame([], $adopter->receive($queue, 2), 'an exhausted message must not reach a handler'); + + $js = Connection::connect($url)->jetStream(); + $this->assertSame(0, $js->getStreamInfo('Q_' . strtoupper($queue->name))->state->messages, 'work stream holds nothing'); + $this->assertSame(1, $js->getStreamInfo('Q_' . strtoupper($queue->name) . '_DEAD')->state->messages, 'the message is on the dead stream'); + + $adopter->close(); + } + + public function testARequireBrokerReportsAConsumerWithoutASpareDelivery(): void + { + // A consumer provisioned before the spare delivery existed, which a Require + // broker may not change: it says so rather than leaving strandings unexplained. + $url = getenv('NATS_URL') ?: 'nats://127.0.0.1:14225'; + $queue = new Queue('t_' . substr(md5(uniqid('', true)), 0, 8)); + $owner = new Nats(Connection::connect($url), maxDeliver: 2); + $owner->publish($queue, ['task' => 'a']); + $owner->close(); + + $js = Connection::connect($url)->jetStream(); + $stream = 'Q_' . strtoupper($queue->name); + $legacy = $js->getConsumer($stream, 'worker')->info(true)->config->toArray(); + // What a pre-spare broker provisioned: no extra delivery, and no marker for one. + $legacy['max_deliver'] = $maxDeliver = 2; + $this->assertIsArray($legacy['metadata']); + unset($legacy['metadata']['utopia_queue_spare_delivery']); + $js->updateConsumer($stream, ConsumerConfig::fromArray($legacy)); + + $reported = []; + $adopter = new Nats( + Connection::connect($url), + maxDeliver: 2, + onError: static function (\Throwable $error) use (&$reported): void { + $reported[] = $error->getMessage(); + }, + provisioning: Provisioning::Require, + ); + $adopter->receive($queue, 1); + + $this->assertCount(1, $reported, 'the missing spare delivery is reported'); + $this->assertSame($maxDeliver, $js->getConsumer($stream, 'worker')->info(true)->config->maxDeliver, 'and the consumer left as it was found'); + + $adopter->close(); + } + + public function testASpareDeliveryRaisedByAnotherBrokerIsNotParkedEarly(): void + { + // The work consumer is shared. A broker that cached a spare delivery of 3 must + // not park the 3rd delivery once a broker with a larger budget has reprovisioned + // the consumer to allow more. + $url = getenv('NATS_URL') ?: 'nats://127.0.0.1:14225'; + $queue = new Queue('t_' . substr(md5(uniqid('', true)), 0, 8)); + + $small = new Nats(Connection::connect($url), ackWait: 1.0, maxDeliver: 2); + $small->publish($queue, ['task' => 'a']); + $this->assertCount(1, $small->receive($queue, 2)); + $this->assertCount(1, $small->receive($queue, 2)); + + $large = new Nats(Connection::connect($url), ackWait: 1.0, maxDeliver: 5); + $this->assertCount(1, $large->receive($queue, 2), 'the 3rd delivery, within the new budget'); + + $this->assertCount(1, $small->receive($queue, 2), 'the 4th delivery reaches a handler, not the dead stream'); + + $small->close(); + $large->close(); + } + + public function testASpareDeliveryIsParkedWhenTheThresholdCannotBeReRead(): void + { + // Re-reading the consumer before parking is a request, and it can fail. The + // delivery in hand may be the server's last, so it is parked on the cached + // threshold rather than dropped unacknowledged with the rest of its batch. + $url = getenv('NATS_URL') ?: 'nats://127.0.0.1:14225'; + $queue = new Queue('t_' . substr(md5(uniqid('', true)), 0, 8)); + + $dying = new Nats(Connection::connect($url), ackWait: 1.0, maxDeliver: 2); + $dying->publish($queue, ['poison' => true]); + $this->assertCount(1, $dying->receive($queue, 2)); + $this->assertCount(1, $dying->receive($queue, 2)); + $dying->close(); + + $transport = new TcpTransport(); + $fault = $this->createStub(Transport::class); + foreach (['connect', 'read', 'readLine', 'upgradeTls', 'isConnected', 'close'] as $method) { + $fault->method($method)->willReturnCallback($transport->$method(...)); + } + $state = new class () { + public bool $armed = false; + }; + $failure = new NatsException('consumer info unavailable'); + $fault->method('write')->willReturnCallback(static function (string $data) use ($transport, $state, $failure): int { + if ($state->armed && str_contains($data, 'CONSUMER.INFO')) { + $state->armed = false; + throw $failure; + } + return $transport->write($data); + }); + $reported = []; + $fresh = new Nats( + static fn (): Connection => Connection::connect(new ConnectionOptions( + servers: $url, + transportFactory: static fn (): Transport => $fault, + )), + ackWait: 1.0, + maxDeliver: 2, + onError: static function (\Throwable $error) use (&$reported): void { + $reported[] = $error; + }, + ); + $state->armed = true; // provisioning sends no CONSUMER.INFO, so the re-read is the first + + $this->assertSame([], $fresh->receive($queue, 3), 'an exhausted message must not reach a handler'); + $this->assertFalse($state->armed, 'the re-read failed'); + $this->assertSame([$failure], $reported, 'and was reported'); + + $js = Connection::connect($url)->jetStream(); + $this->assertSame(0, $js->getStreamInfo('Q_' . strtoupper($queue->name))->state->messages, 'work stream holds nothing'); + $this->assertSame(1, $js->getStreamInfo('Q_' . strtoupper($queue->name) . '_DEAD')->state->messages, 'the message is on the dead stream'); + + $fresh->close(); + } + // JetStream updates a consumer's num_pending asynchronously after the publish ack. private function pendingSettled(int $expected): int { diff --git a/packages/storage/src/Device/Local.php b/packages/storage/src/Device/Local.php index 70cb90a70d1..68f22f95483 100644 --- a/packages/storage/src/Device/Local.php +++ b/packages/storage/src/Device/Local.php @@ -221,6 +221,12 @@ public function abort(string $path, string $uploadId = ''): bool if (! file_exists(\dirname($tmp))) { // Checks if directory path to file exists throw new NotFoundException('File doesn\'t exist: ' . \dirname($path)); } + + // Chunks that were already joined into the file leave no directory behind. + if (! file_exists($tmp)) { + return ! file_exists($path); + } + $files = $this->scanDirectory($tmp); foreach ($files as $file) { diff --git a/packages/storage/src/Device/S3/RetryStrategy.php b/packages/storage/src/Device/S3/RetryStrategy.php index 6711fdfd9e8..848141fb967 100644 --- a/packages/storage/src/Device/S3/RetryStrategy.php +++ b/packages/storage/src/Device/S3/RetryStrategy.php @@ -9,18 +9,24 @@ use Psr\Http\Message\RequestInterface; use Psr\Http\Message\ResponseInterface; use Utopia\Client\Decorator\Retry\Strategy; +use Utopia\Client\Exception\ConnectionException; +use Utopia\Client\Exception\DnsException; +use Utopia\Client\Exception\TimeoutException; +use Utopia\Client\Exception\TlsException; +use Utopia\Psr7\Header; +use Utopia\Psr7\Method; /** - * Retry strategy for transient S3 rate-limiting errors (e.g. SlowDown, - * ServiceUnavailable), for use with the `utopia-php/client` Retry decorator. + * Retry strategy for transient S3 failures, for use with the + * `utopia-php/client` Retry decorator. * - * The XML body is parsed first so that specific S3 error codes are detected - * regardless of HTTP status: a 503/429 carrying a parseable but non-transient - * error code is not retried, while unparseable 429/503 responses fall back to - * status-code detection. - * - * Waits use exponential backoff with full jitter so a fleet throttled at the - * same moment does not retry in lockstep. + * A rejected request (throttled, or never sent) is always retried. One that may + * have been applied (internal error, dropped or timed-out connection) is only + * retried when replaying it is harmless: not a POST other than + * CompleteMultipartUpload, and not a conditional write, whose condition the + * first attempt may already have changed. The XML error code wins over the HTTP + * status. Waits use exponential backoff with full jitter, or a numeric + * Retry-After. * @see \Utopia\Storage\Tests\Device\S3\RetryStrategyTest */ final readonly class RetryStrategy implements Strategy @@ -28,12 +34,26 @@ /** * @var array */ - private const array TRANSIENT_ERROR_CODES = ['SlowDown', 'ServiceUnavailable', 'Throttling', 'RequestThrottled']; + private const array REJECTED_ERROR_CODES = ['SlowDown', 'ServiceUnavailable', 'Throttling', 'RequestThrottled']; + + /** + * @var array + */ + private const array UNKNOWN_ERROR_CODES = ['InternalError']; + + /** + * @var array + */ + private const array REJECTED_STATUS_CODES = [429, 503]; /** * @var array */ - private const array TRANSIENT_STATUS_CODES = [429, 503]; + private const array UNKNOWN_STATUS_CODES = [500, 502, 504]; + + private const string REJECTED = 'rejected'; + + private const string UNKNOWN = 'unknown'; private Closure $randomizer; @@ -54,37 +74,84 @@ public function __construct( public function delay(RequestInterface $request, int $attempt, ?ResponseInterface $response, ?ClientExceptionInterface $error): ?float { - if ($attempt > $this->retries || ! $response instanceof ResponseInterface) { + if ($attempt > $this->retries) { return null; } - if (! $this->isTransient($response)) { + $outcome = $response instanceof ResponseInterface + ? $this->classifyResponse($response) + : $this->classifyError($error); + + if ($outcome === null || ($outcome === self::UNKNOWN && ! $this->isReplayable($request))) { return null; } - return ($this->randomizer)() * min($this->maxDelay, $this->delay * 2 ** ($attempt - 1)); + return $this->retryAfter($response) ?? ($this->randomizer)() * min($this->maxDelay, $this->delay * 2 ** ($attempt - 1)); } - private function isTransient(ResponseInterface $response): bool + private function classifyResponse(ResponseInterface $response): ?string { $body = (string) $response->getBody(); $trimmed = ltrim($body); if (str_starts_with($trimmed, 'Code ?? ''); - if (\in_array($code, self::TRANSIENT_ERROR_CODES, true)) { - return true; - } - // Successfully parsed XML with a non-transient error code — do not retry. - if ($code !== '') { - return false; - } + $code = $xml === false ? '' : (string) ($xml->Code ?? ''); + if ($code !== '') { + return match (true) { + \in_array($code, self::REJECTED_ERROR_CODES, true) => self::REJECTED, + \in_array($code, self::UNKNOWN_ERROR_CODES, true) => self::UNKNOWN, + default => null, + }; } } - // Fall back to HTTP status code for responses that cannot be parsed as XML. - return \in_array($response->getStatusCode(), self::TRANSIENT_STATUS_CODES, true); + return match (true) { + \in_array($response->getStatusCode(), self::REJECTED_STATUS_CODES, true) => self::REJECTED, + \in_array($response->getStatusCode(), self::UNKNOWN_STATUS_CODES, true) => self::UNKNOWN, + default => null, + }; + } + + private function classifyError(?ClientExceptionInterface $error): ?string + { + // Nothing was sent: cURL and Swoole report a refused connection differently. + $refused = [\CURLE_COULDNT_CONNECT, \defined('SOCKET_ECONNREFUSED') ? \SOCKET_ECONNREFUSED : 111]; + if ($error instanceof DnsException || ($error instanceof ConnectionException && \in_array($error->getCode(), $refused, true))) { + return self::REJECTED; + } + + if ($error instanceof TlsException) { + return null; + } + + return $error instanceof ConnectionException || $error instanceof TimeoutException ? self::UNKNOWN : null; + } + + private function isReplayable(RequestInterface $request): bool + { + if ($request->getMethod() === Method::PUT) { + return ! $request->hasHeader(Header::IF_MATCH) && ! $request->hasHeader(Header::IF_NONE_MATCH); + } + + if ($request->getMethod() !== Method::POST) { + return true; + } + + // Of the POSTs, only CompleteMultipartUpload is safe: a landed completion leaves the object in place. + parse_str($request->getUri()->getQuery(), $query); + + return \array_key_exists('uploadId', $query); + } + + private function retryAfter(?ResponseInterface $response): ?float + { + $value = $response?->getHeaderLine(Header::RETRY_AFTER) ?? ''; + + if (! is_numeric($value)) { + return null; + } + + return min($this->maxDelay, max(0.0, (float) $value)); } } diff --git a/packages/storage/tests/Device/S3/RetryStrategyTest.php b/packages/storage/tests/Device/S3/RetryStrategyTest.php index 32319ad6520..46fdb92ec1e 100644 --- a/packages/storage/tests/Device/S3/RetryStrategyTest.php +++ b/packages/storage/tests/Device/S3/RetryStrategyTest.php @@ -5,7 +5,10 @@ namespace Utopia\Storage\Tests\Device\S3; use PHPUnit\Framework\TestCase; -use Utopia\Client\Exception\NetworkException; +use Utopia\Client\Exception\ConnectionException; +use Utopia\Client\Exception\DnsException; +use Utopia\Client\Exception\TimeoutException; +use Utopia\Client\Exception\TlsException; use Utopia\Psr7\Request; use Utopia\Psr7\Response; use Utopia\Psr7\Stream; @@ -14,9 +17,14 @@ final class RetryStrategyTest extends TestCase { - private function request(): Request + private function request(string $method = 'PUT', string $query = ''): Request { - return new Request('PUT', Uri::parse('https://s3.example.com/root/file.txt')); + return new Request($method, Uri::parse('https://s3.example.com/root/file.txt' . ($query === '' ? '' : '?' . $query))); + } + + private function internalError(): Response + { + return $this->response(500, 'InternalErrorinternal incident'); } private function response(int $status, string $body = ''): Response @@ -57,18 +65,74 @@ public function testStatusFallbackIsTransient(): void { $strategy = new RetryStrategy(); - $this->assertNotNull($strategy->delay($this->request(), 1, $this->response(429), null)); - $this->assertNotNull($strategy->delay($this->request(), 1, $this->response(503), null)); + foreach ([429, 500, 502, 503, 504] as $status) { + $this->assertNotNull($strategy->delay($this->request(), 1, $this->response($status), null), "HTTP {$status}"); + } + $this->assertNull($strategy->delay($this->request(), 1, $this->response(501), null)); + } + + public function testInternalErrorIsRetried(): void + { + $strategy = new RetryStrategy(); + + $this->assertNotNull($strategy->delay($this->request('PUT', 'partNumber=2&uploadId=upload-1'), 1, $this->internalError(), null)); + $this->assertNotNull($strategy->delay($this->request('POST', 'uploadId=upload-1'), 1, $this->internalError(), null), 'completion'); + } + + public function testMaybeAppliedMultipartCreationIsNotRetried(): void + { + $strategy = new RetryStrategy(); + $create = $this->request('POST', 'uploads='); + + $this->assertNull($strategy->delay($create, 1, $this->internalError(), null)); + $this->assertNull($strategy->delay($create, 1, null, new TimeoutException($create, 'Operation timed out', \CURLE_OPERATION_TIMEDOUT))); + $this->assertNotNull($strategy->delay($create, 1, $this->response(503), null), 'throttled, so never applied'); + } + + public function testMaybeAppliedBatchDeleteIsNotRetried(): void + { + // A replay could delete an object written under a listed key since the first attempt. + $strategy = new RetryStrategy(); + $delete = $this->request('POST', 'delete='); + + $this->assertNull($strategy->delay($delete, 1, $this->internalError(), null)); + $this->assertNull($strategy->delay($delete, 1, null, new TimeoutException($delete, 'Operation timed out', \CURLE_OPERATION_TIMEDOUT))); + $this->assertNotNull($strategy->delay($delete, 1, $this->response(503), null), 'throttled, so never applied'); + } + + public function testMaybeAppliedConditionalWriteIsNotRetried(): void + { + $strategy = new RetryStrategy(); + + foreach (['If-None-Match' => '*', 'If-Match' => '"etag"'] as $header => $value) { + $write = $this->request('PUT')->withHeader($header, $value); + + $this->assertNull($strategy->delay($write, 1, $this->internalError(), null), $header); + $this->assertNull($strategy->delay($write, 1, null, new ConnectionException($write, 'Connection reset by peer', \CURLE_RECV_ERROR)), $header); + $this->assertNotNull($strategy->delay($write, 1, $this->response(503), null), $header . ' throttled, so never applied'); + } + + $read = $this->request('GET')->withHeader('If-Match', '"etag"'); + $this->assertNotNull($strategy->delay($read, 1, $this->internalError(), null), 'a conditional read is harmless to repeat'); } /** XML error code takes precedence over HTTP status — 503 with non-transient XML must not be retried. */ public function test503WithNonTransientXmlIsNotRetried(): void { - $body = 'InternalErrorInternal server error.'; + $body = 'AccessDeniedAccess denied.'; $this->assertNull(new RetryStrategy()->delay($this->request(), 1, $this->response(503, $body), null)); } + public function testRetryAfterIsHonouredWithinTheCeiling(): void + { + $strategy = new RetryStrategy(delay: 0.5, maxDelay: 5.0, randomizer: static fn (): float => 1.0); + + $this->assertEqualsWithDelta(2.0, $strategy->delay($this->request(), 1, $this->response(503)->withHeader('Retry-After', '2'), null), PHP_FLOAT_EPSILON); + $this->assertEqualsWithDelta(5.0, $strategy->delay($this->request(), 1, $this->response(503)->withHeader('Retry-After', '120'), null), PHP_FLOAT_EPSILON); + $this->assertEqualsWithDelta(0.5, $strategy->delay($this->request(), 1, $this->response(503)->withHeader('Retry-After', 'Wed, 21 Oct 2026 07:28:00 GMT'), null), PHP_FLOAT_EPSILON, 'an HTTP-date falls back to backoff'); + } + public function testRetriesAreCapped(): void { $strategy = new RetryStrategy(retries: 2); @@ -79,11 +143,48 @@ public function testRetriesAreCapped(): void $this->assertNull($strategy->delay($this->request(), 3, $response, null)); } - public function testTransportErrorsAreNotRetried(): void + public function testFailuresBeforeSendingAreRetriedForEveryRequest(): void + { + $strategy = new RetryStrategy(); + $create = $this->request('POST', 'uploads='); + + $this->assertNotNull($strategy->delay($create, 1, null, new ConnectionException($create, 'Could not connect to server', \CURLE_COULDNT_CONNECT)), 'cURL'); + $this->assertNotNull($strategy->delay($create, 1, null, new ConnectionException($create, 'Connection refused', \defined('SOCKET_ECONNREFUSED') ? \SOCKET_ECONNREFUSED : 111)), 'Swoole'); + $this->assertNotNull($strategy->delay($create, 1, null, new DnsException($create, 'Could not resolve host', \CURLE_COULDNT_RESOLVE_HOST))); + } + + public function testDroppedOrTimedOutPutIsRetried(): void + { + $strategy = new RetryStrategy(); + $put = $this->request('PUT'); + + $this->assertNotNull($strategy->delay($put, 1, null, new ConnectionException($put, 'Connection reset by peer', \CURLE_RECV_ERROR))); + $this->assertNotNull($strategy->delay($put, 1, null, new TimeoutException($put, 'Operation timed out', \CURLE_OPERATION_TIMEDOUT))); + } + + public function testTimedOutCompletionIsRetried(): void + { + // A completion that did land leaves the object in place, which finalize() accepts. + $complete = $this->request('POST', 'uploadId=upload-1'); + + $this->assertNotNull(new RetryStrategy()->delay($complete, 1, null, new TimeoutException($complete, 'Operation timed out', \CURLE_OPERATION_TIMEDOUT))); + } + + public function testTlsFailureIsNotRetried(): void + { + $put = $this->request('PUT'); + + $this->assertNull(new RetryStrategy()->delay($put, 1, null, new TlsException($put, 'SSL connect error', \CURLE_SSL_CONNECT_ERROR))); + } + + public function testTransportRetriesAreCapped(): void { - $error = new NetworkException($this->request(), 'Connection reset'); + $put = $this->request('PUT'); + $error = new TimeoutException($put, 'Operation timed out', \CURLE_OPERATION_TIMEDOUT); + $strategy = new RetryStrategy(retries: 1); - $this->assertNull(new RetryStrategy()->delay($this->request(), 1, null, $error)); + $this->assertNotNull($strategy->delay($put, 1, null, $error)); + $this->assertNull($strategy->delay($put, 2, null, $error)); } public function testReadingTheBodyLeavesItReadable(): void diff --git a/packages/storage/tests/Device/S3Test.php b/packages/storage/tests/Device/S3Test.php index ab4eaa5a26a..c429e2108d4 100644 --- a/packages/storage/tests/Device/S3Test.php +++ b/packages/storage/tests/Device/S3Test.php @@ -12,6 +12,7 @@ use Utopia\Client\Adapter; use Utopia\Client\Decorator\Retry; use Utopia\Client\Exception\NetworkException; +use Utopia\Client\Exception\TimeoutException; use Utopia\Client\Redirect; use Utopia\Client\Tls; use Utopia\Psr7\Request; @@ -395,6 +396,79 @@ public function testTransientErrorRetriesAreExhausted(): void $this->assertCount(4, $client->requests); } + private function internalIncident(): Response + { + $body = 'InternalErrorinternal incident'; + + return new Response(500, body: new Stream($body))->withHeader('content-type', 'application/xml'); + } + + public function testMultipartUploadSurvivesInternalErrors(): void + { + $client = new ScriptedClient([ + new Response(200, body: new Stream('upload-1'))->withHeader('content-type', 'application/xml'), + new Response(200)->withHeader('etag', '"etag-1"'), + $this->internalIncident(), + new Response(200)->withHeader('etag', '"etag-2"'), + $this->internalIncident(), + new Response(200, body: new Stream('"etag-final"'))->withHeader('content-type', 'application/xml'), + ]); + $device = $this->device($client); + + $metadata = []; + $device->prepare('/root/archive.tar.gz', 'application/gzip', 2, $metadata); + $device->upload(new Stream('first'), '/root/archive.tar.gz', 'application/gzip', 1, 2, $metadata); + + // The last chunk completes the upload. + $this->assertSame(2, $device->upload(new Stream('second'), '/root/archive.tar.gz', 'application/gzip', 2, 2, $metadata)); + $this->assertCount(6, $client->requests); + $this->assertSame('second', (string) $client->requests[3]->getBody()); + } + + public function testConditionalWriteIsNotReplayedAfterAnInternalError(): void + { + // The first attempt may have written the object, so a replay would fail its own If-None-Match. + $client = new ScriptedClient([$this->internalIncident()]); + + try { + $this->device($client)->create('/root/file.txt', new Stream('Hello World'), 'text/plain'); + self::fail('Expected the internal error to surface'); + } catch (RemoteException $e) { + $this->assertSame('InternalError', $e->errorCode); + } + + $this->assertCount(1, $client->requests); + } + + public function testTimedOutPartIsReplayed(): void + { + $client = new ScriptedClient([ + new TimeoutException(new Request('PUT', Uri::parse('https://s3.example.com/root/file.txt')), 'Operation timed out', \CURLE_OPERATION_TIMEDOUT), + new Response(200)->withHeader('etag', '"abc"'), + ]); + + $this->assertSame('abc', $this->device($client)->write('/root/file.txt', new Stream('Hello World'), 'text/plain')); + $this->assertCount(2, $client->requests); + $this->assertSame('Hello World', (string) $client->requests[1]->getBody()); + } + + public function testTimedOutMultipartCreationIsNotReplayed(): void + { + $client = new ScriptedClient([ + new TimeoutException(new Request('POST', Uri::parse('https://s3.example.com/root/file.txt')), 'Operation timed out', \CURLE_OPERATION_TIMEDOUT), + ]); + + $metadata = []; + try { + $this->device($client)->prepare('/root/file.txt', 'text/plain', 2, $metadata); + self::fail('Expected the timeout to surface'); + } catch (TransportException) { + } + + $this->assertCount(1, $client->requests); + $this->assertArrayNotHasKey('uploadId', $metadata); + } + public function testNoSuchKeyBecomesNotFoundException(): void { $body = 'NoSuchKeyThe specified key does not exist.'; @@ -487,6 +561,37 @@ public function testCopyLargeObjectUsesMultipartServerSideCopy(): void $this->assertStringContainsString('etag-2', $s3->completedBody); } + /** + * Nothing tells a copy whose replayed completion finds the upload gone apart from + * one that was aborted, so it is reported as failed: repeating a copy is safe, + * while a false success would let move() delete its source. + */ + public function testLargeCopyWithAnUnprovenCompletionFails(): void + { + $xml = static fn (string $body): Response => new Response(200, body: new Stream('' . $body))->withHeader('content-type', 'application/xml'); + $client = new ScriptedClient([ + new Response(200)->withHeader('content-length', '6442450944'), + $xml('upload-1'), + $xml('"etag-1"'), + $xml('"etag-2"'), + new TimeoutException(new Request('POST', Uri::parse('https://s3.example.com/root/b.bin')), 'Operation timed out', \CURLE_OPERATION_TIMEDOUT), + new Response(404, body: new Stream('NoSuchUploadThe specified upload does not exist.')), + new Response(204), + ]); + $device = new S3( + root: '/root', + accessKey: 'test-key', + secretKey: 'test-secret', + host: 'https://s3.example.com', + region: 'us-east-1', + client: new Retry($client, new RetryStrategy(delay: 0.0)), + bucket: 'my-bucket', + ); + + $this->expectException(NotFoundException::class); + $device->copy('/root/a.bin', '/root/b.bin'); + } + public function testCopyWithoutBucketFallsBackToStreaming(): void { $this->s3->objectExists = true; diff --git a/rfc/monorepo.md b/rfc/monorepo.md index 9e16bff5630..f7fdbb3092b 100644 --- a/rfc/monorepo.md +++ b/rfc/monorepo.md @@ -18,7 +18,7 @@ Every `utopia-php/*` library Appwrite depends on moves into this repository unde - Rewriting library APIs. Absorption moves code; API changes are separate work with their own releases. - Folding libraries into `src/Appwrite/`. Generic code stays generic and lives in `packages/`. `src/Utopia/` is gone: `Bus`, its only occupant, became `packages/bus` with a mirror like every other package. -- Moving the four monorepo packages Appwrite does not use. `fastly` is archived, `nats` and `replication` go to `appwrite/cloud`, `reputation` is undecided (Cloud or here). +- Moving the three monorepo packages Appwrite does not use. `fastly` and `replication` are archived, `reputation` is undecided (Cloud or here). `nats` was planned for `appwrite/cloud` but lives in `packages/nats`: its real consumer is `queue`'s NATS JetStream broker (`Broker\Nats`), and Cloud takes it through `server-ce`. - Adopting `utopia-php/config` 2.x. See [Version gaps](#version-gaps). ## Current state @@ -263,7 +263,7 @@ Exit: `composer.lock` contains no `utopia-php/*` package. ### Phase 7. Retire the old homes and move Cloud -- Archive `utopia-php/monorepo` with a README pointer here. Move `nats` and `replication` to `appwrite/cloud`; archive `fastly`; place `reputation` per the phase 0 decision. +- Archive `utopia-php/monorepo` with a README pointer here. Archive `fastly` and `replication` (`nats` already lives in `packages/`); place `reputation` per the phase 0 decision. - `appwrite/cloud` removes its `utopia-php/*` requirements and takes the classes through `server-ce`'s autoloader. Its stale pins (`validators ^0.5`, `span 3.0`, `usage 0.14`, `audit ^3`, `query 0.1`) disappear with them; that upgrade is Cloud's own PR series and is not blocked by anything here. ### Phase 8. Harvest @@ -272,6 +272,7 @@ Exit: `composer.lock` contains no `utopia-php/*` package. - Collapse `||` compatibility constraints in package manifests to single ranges once every sibling is on the current major. - Delete duplicated test helpers (`tests/extensions/Queue/InMemoryConnection.php` versus the queue package's own fakes) and every Appwrite-side workaround that existed only because a library fix was waiting on a release. - Burn down every `packages/*/phpstan-baseline.neon` a package arrives with (abuse's Redis cluster log adapters need one under PHPStan 2). + - `abuse`: 44 findings (its standalone repository analysed it at level max under PHPStan 1): 35 in `src`, `array|true` `scan()` and `_masters()` replies merged, sorted and combined into log maps in the `RedisCluster` and `RedisPool` adapters of all three strategies, plus an integer passed to `curl_setopt()` in `ReCaptcha`; 9 in its e2e tests, always-true `instanceof` and `is_int()` checks, the cluster `scan()` reply iterated unnarrowed, and a column shape in `TablesDBTest`. - `audit`: 5 findings: `Log::getData()` returning the decoded `mixed` array against its `array` docblock, Pint's `simplified_null_return` turning the untyped `SQL::getAttribute()`'s `return null;` into `return;`, and the batch fixtures in its e2e tests typed as plain arrays against `logBatch()`'s event shape. - `auth`: 33 findings (it had no PHPStan config of its own): `mixed` out-parameters and results from `openssl_pkey_export()`, `openssl_pkey_get_details()` and `openssl_sign()` in the asymmetric issuer and verifier, integer arithmetic in the PHPass encoder, and array shapes in `AuthorizationDetails` and `ResourceIndicators`, plus decoded-claim arithmetic in its tests. - `cache`: 25 findings (level 5 in the monorepo): `mixed` from the Memcached and Hazelcast server stats and the `RedisCluster` node addresses, values passed to `Envelope::encode()` untyped, and casts of Redis replies in its multiplexing and leasable e2e tests. @@ -286,6 +287,7 @@ Exit: `composer.lock` contains no `utopia-php/*` package. - `emails`: 25 findings, all in `src` (its standalone repository analysed it at level 4): unvalued `array` types on the canonical providers' domain lists and `Email`'s parts and domain caches, and concatenation of `mixed` parts in `Email`. - `http`: 394 findings (level 7 in the monorepo): 81 in `src`, casts and offset access on `mixed` request globals, Swoole server stats and the `__utopia__` coroutine context in the FPM and Swoole adapters, and the `mixed` param and injection definitions in `Http`; 313 in its tests, mostly calls on nullable `?Request`, `?Response`, `?Route` and `?Http` fixtures in `RequestTest`, `HttpTest` and `RouteTest`. - `mqtt`: 82 findings (its own repository analysed it at level max under PHPStan 1): `chr()` arguments not narrowed to `int<0, 255>` and casts from `mixed` in the packet codecs and `Property`, untyped Swoole client and request fields in `Client` and the Swoole adapter, and loosely typed data providers and e2e assertions in its tests. + - `nats`: 445 findings (level 5 in the monorepo): 292 in `src`, nearly all arguments, offset access and casts on the decoded `mixed` JSON in the `fromArray()` builders of the JetStream, KeyValue and ObjectStore configs and infos (`StreamConfig`, `ConsumerConfig`), `ServerInfo` and `Connection`; 153 in its tests, generator key types and decoded protocol frames in `ConnectionDeathDetectionTest`, `ParserTest` and `RequestsTest`, and decoded service replies in the `ServiceTest` and `ServiceExtrasTest` e2e tests. - `openapi`: 166 findings (level 5 in the monorepo), nearly all offset access on the decoded `mixed` document in its readers. - `platform`: 182 findings (level 5 in the monorepo): 146 in `src`, nearly all `mixed` values read from the untyped `array` param, option and label definitions and worker params in `Platform` and passed on to `Hook`, `Http`, `CLI` and the queue `Server`, plus unvalued `array` types in `Action` and `Module`; 36 in its tests, calls on the nullable `?Http` and `?Service` fixtures in `HttpServicesTest` and `WorkerServicesTest` and untyped `$response` parameters in the test actions. - `queue`: 296 findings (level 5 in the monorepo): 162 in `src`, unvalued `array` payloads and returns across `Connection`, `Message` and the brokers, and `mixed` Redis replies and decoded jobs in `Connection\Redis`, `Connection\RedisCluster`, `Broker\Redis`, `Broker\Pool` and `Server`; 134 in its tests, loosely typed connection fakes, the Swoole restart and proxy fixture servers, and decoded NATS and Redis payloads in its e2e tests. diff --git a/src/Appwrite/Auth/OAuth2/Google.php b/src/Appwrite/Auth/OAuth2/Google.php index 3347956efa7..be2e62c1d8d 100644 --- a/src/Appwrite/Auth/OAuth2/Google.php +++ b/src/Appwrite/Auth/OAuth2/Google.php @@ -67,7 +67,7 @@ public function getLoginURL(): string protected function getTokens(string $code): array { if (empty($this->tokens)) { - $this->tokens = \json_decode($this->request( + $this->tokens = $this->parseTokens($this->request( 'POST', 'https://oauth2.googleapis.com/token?' . \http_build_query([ 'code' => $code, @@ -77,7 +77,7 @@ protected function getTokens(string $code): array 'scope' => null, 'grant_type' => 'authorization_code' ]) - ), true); + )); } return $this->tokens; @@ -90,7 +90,7 @@ protected function getTokens(string $code): array */ public function refreshTokens(string $refreshToken): array { - $this->tokens = \json_decode($this->request( + $this->tokens = $this->parseTokens($this->request( 'POST', 'https://oauth2.googleapis.com/token?' . \http_build_query([ 'refresh_token' => $refreshToken, @@ -98,7 +98,7 @@ public function refreshTokens(string $refreshToken): array 'client_secret' => $this->getClientSecret(), 'grant_type' => 'refresh_token' ]) - ), true); + )); if (empty($this->tokens['refresh_token'])) { $this->tokens['refresh_token'] = $refreshToken; @@ -107,6 +107,24 @@ public function refreshTokens(string $refreshToken): array return $this->tokens; } + /** + * @param string $response + * + * @return array + * + * @throws Exception + */ + private function parseTokens(string $response): array + { + $tokens = \json_decode($response, true); + + if (!\is_array($tokens) || empty($tokens['access_token'])) { + throw new Exception($response, 424); + } + + return $tokens; + } + /** * @param string $accessToken * @@ -184,7 +202,7 @@ protected function getUser(string $accessToken): array { if (empty($this->user)) { $user = $this->request('GET', 'https://www.googleapis.com/oauth2/v3/userinfo?access_token=' . \urlencode($accessToken)); - $this->user = \json_decode($user, true); + $this->user = \json_decode($user, true) ?? []; } return $this->user; diff --git a/src/Appwrite/Auth/OAuth2/Webflow.php b/src/Appwrite/Auth/OAuth2/Webflow.php new file mode 100644 index 00000000000..f999d2120c5 --- /dev/null +++ b/src/Appwrite/Auth/OAuth2/Webflow.php @@ -0,0 +1,191 @@ + 'code', + 'client_id' => $this->appID, + 'redirect_uri' => $this->callback, + 'scope' => \implode(' ', $this->getScopes()), + 'state' => \json_encode($this->state) + ]); + } + + /** + * @param string $code + * + * @return array + */ + protected function getTokens(string $code): array + { + if (empty($this->tokens)) { + $this->tokens = \json_decode($this->request( + 'POST', + 'https://api.webflow.com/oauth/access_token', + ['Content-Type: application/x-www-form-urlencoded'], + \http_build_query([ + 'grant_type' => 'authorization_code', + 'client_id' => $this->appID, + 'client_secret' => $this->appSecret, + 'redirect_uri' => $this->callback, + 'code' => $code + ]) + ), true); + } + + return $this->tokens; + } + + /** + * Webflow exposes no refresh grant, so there is nothing to exchange. + * + * @param string $refreshToken + * + * @return array + */ + public function refreshTokens(string $refreshToken): array + { + $this->tokens['refresh_token'] = $refreshToken; + + return $this->tokens; + } + + /** + * @param string $accessToken + * + * @return string + */ + public function getUserID(string $accessToken): string + { + $user = $this->getUser($accessToken); + + return $user['id'] ?? ''; + } + + /** + * @param string $accessToken + * + * @return string + */ + public function getUserEmail(string $accessToken): string + { + $user = $this->getUser($accessToken); + + return $user['email'] ?? ''; + } + + /** + * @param string $accessToken + * + * @return bool + */ + public function isEmailVerified(string $accessToken): bool + { + // Provider exposes no email verification signal, so treat as unverified until one is confirmed + return false; + } + + /** + * @param string $accessToken + * + * @return string + */ + public function getUserName(string $accessToken): string + { + $user = $this->getUser($accessToken); + + return \trim(($user['firstName'] ?? '') . ' ' . ($user['lastName'] ?? '')); + } + + /** + * @param string $accessToken + * + * @return array + */ + protected function getUser(string $accessToken): array + { + if (empty($this->user)) { + $this->user = \json_decode($this->request( + 'GET', + $this->resourceEndpoint, + ['Authorization: Bearer ' . $accessToken] + ), true); + } + + return $this->user; + } + + public function verifyCredentials(): void + { + $response = (new Client(new CurlAdapter())) + ->withTimeout(15) + ->withFollowRedirects(maxHops: 5) + ->sendRequest((new RequestFactory())->form( + Method::POST, + 'https://api.webflow.com/oauth/access_token', + [ + 'grant_type' => 'authorization_code', + 'client_id' => $this->appID, + 'client_secret' => $this->appSecret, + 'redirect_uri' => 'https://invalid.appwrite.callback/intentionally-invalid', + 'code' => 'intentionally-invalid-code', + ], + )); + + $json = \json_decode((string) $response->getBody(), true); + + if (isset($json['error']) && $json['error'] === 'invalid_client') { + throw new \Exception('Webflow application with the provided Client ID and/or Client Secret is invalid.'); + } + + // We still expect an error, like invalid_grant or invalid_request, + // but that indicates valid credentials + } +} diff --git a/src/Appwrite/Functions/Validator/Headers.php b/src/Appwrite/Functions/Validator/Headers.php index a18660fa2dc..ba21089ec0f 100644 --- a/src/Appwrite/Functions/Validator/Headers.php +++ b/src/Appwrite/Functions/Validator/Headers.php @@ -24,7 +24,7 @@ public function __construct(protected bool $allowEmpty = true, protected int $ma */ public function getDescription(): string { - return 'Invalid headers: Alphanumeric characters or hyphens only, cannot start with "x-appwrite", maximum ' . $this->maxKeys . ' keys, and total size ' . $this->maxSize . '.'; + return 'Headers must contain at most ' . $this->maxKeys . ' keys and ' . $this->maxSize . ' bytes in total. Valid key chars are a-z, A-Z, 0-9, and hyphen. Keys can\'t start or end with a hyphen, or start with "x-appwrite". Values can\'t be arrays or objects.'; } /** @@ -55,6 +55,11 @@ public function isValid($value): bool return false; } + // Reject array and object values + if (!\is_scalar($val) && !\is_null($val)) { + return false; + } + $length = \strlen($key); if ($length === 0) { return false; diff --git a/src/Appwrite/Messaging/Adapter/Push/Appwrite.php b/src/Appwrite/Messaging/Adapter/Push/Appwrite.php index 00730b59424..036cbcb14f0 100644 --- a/src/Appwrite/Messaging/Adapter/Push/Appwrite.php +++ b/src/Appwrite/Messaging/Adapter/Push/Appwrite.php @@ -238,6 +238,10 @@ private function buildPayload(PushMessage $message): string { $envelope = []; + if ($this->messageId !== '') { + $envelope['messageId'] = $this->messageId; + } + if ($message->getTitle() !== null) { $envelope['notification']['title'] = $message->getTitle(); } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php index cc9319a0f4f..6c3279c83b8 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/Databases/Collections/Documents/Delete.php @@ -4,6 +4,7 @@ use Appwrite\Databases\TransactionState; use Appwrite\Event\Event; +use Appwrite\Event\Realtime; use Appwrite\Extend\Exception; use Appwrite\SDK\AuthType; use Appwrite\SDK\ContentType; @@ -13,6 +14,7 @@ use Appwrite\Usage\Context; use Appwrite\Utopia\Database\Documents\User; use Appwrite\Utopia\Response as UtopiaResponse; +use Utopia\Console; use Utopia\Database\Database; use Utopia\Database\Document; use Utopia\Database\Exception\Conflict as ConflictException; @@ -82,6 +84,7 @@ public function __construct() ->inject('dbForProject') ->inject('getDatabasesDB') ->inject('queueForEvents') + ->inject('queueForRealtime') ->inject('usage') ->inject('transactionState') ->inject('plan') @@ -100,6 +103,7 @@ public function action( Database $dbForProject, callable $getDatabasesDB, Event $queueForEvents, + Realtime $queueForRealtime, Context $usage, TransactionState $transactionState, array $plan, @@ -192,6 +196,12 @@ public function action( return; } + // The database fires an update for each related document the delete changed + $related = []; + $dbForDatabases->on(Database::EVENT_DOCUMENT_UPDATE, 'relationship-delete', function (string $event, Document $document) use (&$related) { + $related[] = $document; + }); + try { $dbForDatabases->withRequestTimestamp($requestTimestamp, function () use ($dbForDatabases, $database, $collection, $documentId) { $dbForDatabases->deleteDocument( @@ -203,6 +213,8 @@ public function action( throw new Exception($this->getConflictException()); } catch (RestrictedException) { throw new Exception($this->getRestrictedException()); + } finally { + $dbForDatabases->on(Database::EVENT_DOCUMENT_UPDATE, 'relationship-delete', null); } $collectionsCache = []; @@ -242,6 +254,104 @@ public function action( ->setContext($this->getCollectionsEventsContext(), $collection) ->setPayload($response->output($document, $this->getResponseModel()), sensitive: $relationships); + // The delete already happened, so failing to notify related documents must not fail it + try { + $this->triggerRelationshipUpdates( + database: $database, + collection: $collection, + related: $related, + dbForProject: $dbForProject, + queueForEvents: $queueForEvents, + queueForRealtime: $queueForRealtime, + response: $response, + authorization: $authorization + ); + } catch (\Throwable $e) { + Console::warning('Failed to publish relationship updates: ' . $e->getMessage()); + } + $response->noContent(); } + + /** + * Publish an update for each related document the delete changed + * + * @param array $related + */ + private function triggerRelationshipUpdates( + Document $database, + Document $collection, + array $related, + Database $dbForProject, + Event $queueForEvents, + Realtime $queueForRealtime, + UtopiaResponse $response, + Authorization $authorization + ): void { + if (empty($related)) { + return; + } + + // Related documents carry the internal collection id, so map it back through the two-way relationships + $relatedCollections = []; + foreach ($collection->getAttribute('attributes', []) as $attribute) { + if ( + $attribute->getAttribute('type') !== Database::VAR_RELATIONSHIP + || !$attribute->getAttribute('twoWay') + ) { + continue; + } + + $relatedCollection = $authorization->skip(fn () => $dbForProject->getDocument( + 'database_' . $database->getSequence(), + $attribute->getAttribute('relatedCollection') + )); + if ($relatedCollection->isEmpty()) { + continue; + } + + $relatedCollections['database_' . $database->getSequence() . '_collection_' . $relatedCollection->getSequence()] = $relatedCollection; + } + + $collectionsCache = []; + + foreach ($related as $peer) { + $relatedCollection = $relatedCollections[$peer->getCollection()] ?? null; + if ($relatedCollection === null) { + continue; + } + + $sensitive = \array_map( + fn (Document $attr) => $attr->getAttribute('key'), + \array_filter( + $relatedCollection->getAttribute('attributes', []), + fn (Document $attr) => $attr->getAttribute('type') === Database::VAR_RELATIONSHIP + ) + ); + + $this->processDocument( + database: $database, + collection: $relatedCollection, + document: $peer, + dbForProject: $dbForProject, + collectionsCache: $collectionsCache, + authorization: $authorization + ); + + // Clone so the delete event stays intact for the shutdown hook. + $event = clone $queueForEvents; + $event->reset() + ->setEvent('databases.[databaseId].collections.[collectionId].documents.[documentId].update') + ->setParam('databaseId', $database->getId()) + ->setParam('collectionId', $relatedCollection->getId()) + ->setParam('tableId', $relatedCollection->getId()) + ->setParam('documentId', $peer->getId()) + ->setParam('rowId', $peer->getId()) + ->setContext($this->getCollectionsEventsContext(), $relatedCollection) + // Filter through the model directly; output() would replace the audited response payload. + ->setPayload($response->getModel($this->getResponseModel())->filter($peer)->getArrayCopy(), sensitive: $sensitive); + + $queueForRealtime->from($event)->trigger(); + } + } } diff --git a/src/Appwrite/Platform/Modules/Databases/Http/DocumentsDB/Collections/Documents/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/DocumentsDB/Collections/Documents/Delete.php index 9efbd942a39..2dcc363664f 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/DocumentsDB/Collections/Documents/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/DocumentsDB/Collections/Documents/Delete.php @@ -68,6 +68,7 @@ public function __construct() ->inject('dbForProject') ->inject('getDatabasesDB') ->inject('queueForEvents') + ->inject('queueForRealtime') ->inject('usage') ->inject('transactionState') ->inject('plan') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php index 12b8ba510c3..cd35555eaa7 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/TablesDB/Tables/Rows/Delete.php @@ -70,6 +70,7 @@ public function __construct() ->inject('dbForProject') ->inject('getDatabasesDB') ->inject('queueForEvents') + ->inject('queueForRealtime') ->inject('usage') ->inject('transactionState') ->inject('plan') diff --git a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Collections/Documents/Delete.php b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Collections/Documents/Delete.php index c6857c2ac6e..cd9df1fc22c 100644 --- a/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Collections/Documents/Delete.php +++ b/src/Appwrite/Platform/Modules/Databases/Http/VectorsDB/Collections/Documents/Delete.php @@ -68,6 +68,7 @@ public function __construct() ->inject('dbForProject') ->inject('getDatabasesDB') ->inject('queueForEvents') + ->inject('queueForRealtime') ->inject('usage') ->inject('transactionState') ->inject('plan') diff --git a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php index e66054101c4..c3bca83671b 100644 --- a/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php +++ b/src/Appwrite/Platform/Modules/Functions/Http/Executions/Create.php @@ -166,7 +166,7 @@ public function action( // 'headers' validator $validator = new Headers(); if (!$validator->isValid($headers)) { - throw new Exception($validator->getDescription(), 400); + throw new Exception(Exception::GENERAL_ARGUMENT_INVALID, $validator->getDescription()); } /* @var Document $function */ diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Base.php b/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Base.php index 036dbf185a8..23e9304cd94 100644 --- a/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Base.php +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Base.php @@ -375,6 +375,7 @@ public static function getProviderActions(): array 'yandex' => Yandex\Update::class, 'x' => X\Update::class, 'wordpress' => WordPress\Update::class, + 'webflow' => Webflow\Update::class, 'twitch' => Twitch\Update::class, 'stripe' => Stripe\Update::class, 'spotify' => Spotify\Update::class, diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Get.php b/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Get.php index 2392734b6b2..d545355fb26 100644 --- a/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Get.php +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Get.php @@ -58,6 +58,7 @@ public function __construct() Response::MODEL_OAUTH2_YANDEX, Response::MODEL_OAUTH2_X, Response::MODEL_OAUTH2_WORDPRESS, + Response::MODEL_OAUTH2_WEBFLOW, Response::MODEL_OAUTH2_TWITCH, Response::MODEL_OAUTH2_STRIPE, Response::MODEL_OAUTH2_SPOTIFY, diff --git a/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Webflow/Update.php b/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Webflow/Update.php new file mode 100644 index 00000000000..6484ad0f5d6 --- /dev/null +++ b/src/Appwrite/Platform/Modules/Project/Http/Project/OAuth2/Webflow/Update.php @@ -0,0 +1,55 @@ +addAction(UpdateOAuth2Yandex::getName(), new UpdateOAuth2Yandex()); $this->addAction(UpdateOAuth2X::getName(), new UpdateOAuth2X()); $this->addAction(UpdateOAuth2WordPress::getName(), new UpdateOAuth2WordPress()); + $this->addAction(UpdateOAuth2Webflow::getName(), new UpdateOAuth2Webflow()); $this->addAction(UpdateOAuth2Twitch::getName(), new UpdateOAuth2Twitch()); $this->addAction(UpdateOAuth2Stripe::getName(), new UpdateOAuth2Stripe()); $this->addAction(UpdateOAuth2Spotify::getName(), new UpdateOAuth2Spotify()); diff --git a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php index 61e5bc64028..43f457db5fc 100644 --- a/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php +++ b/src/Appwrite/Platform/Modules/Storage/Http/Buckets/Files/Create.php @@ -3,6 +3,7 @@ namespace Appwrite\Platform\Modules\Storage\Http\Buckets\Files; use Appwrite\ClamAV\Network; +use Appwrite\ClamAV\ScanResult; use Appwrite\Event\Event; use Appwrite\Extend\Exception; use Appwrite\OpenSSL\OpenSSL; @@ -370,16 +371,27 @@ public function action( (int) System::getEnv('_APP_STORAGE_ANTIVIRUS_PORT', 3310) ); - $scan = $antivirus->scanInStream($path); + try { + $scan = $antivirus->scanInStream($path); + } catch (\RuntimeException $e) { + $scan = ScanResult::failed($e->getMessage()); + } + + if (!$scan->isClean()) { + // A pending record would be left pointing at chunks that are already joined. Keep it + // while the joined file is still on disk, so the removal can be retried by deleting the file. + $removed = $deviceForFiles->delete($path); + + if ($removed && !$file->isEmpty() && !$authorization->skip(fn () => $dbForProject->deleteDocument('bucket_' . $bucket->getSequence(), $fileId))) { + throw new Exception(Exception::GENERAL_SERVER_ERROR, 'Failed to remove file from DB'); + } + } if ($scan->isInfected()) { - $deviceForFiles->delete($path); throw new Exception(Exception::STORAGE_INVALID_FILE); } if ($scan->hasFailed()) { - // The finalized upload has no completed file record yet. - $deviceForFiles->delete($path); throw new Exception( Exception::GENERAL_SERVER_ERROR, 'Unable to scan the uploaded file: ' . $scan->getReply() diff --git a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php index d4aee458e76..824880c0d45 100644 --- a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php +++ b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Create.php @@ -80,7 +80,7 @@ public function __construct() ->param('email', '', new EmailValidator(), 'Email of the new team member.', true) ->param('userId', '', new UID(), 'ID of the user to be added to a team.', true) ->param('phone', '', new Phone(), 'Phone number. Format this number with a leading \'+\' and a country code, e.g., +16175551212.', true) - ->param('roles', [], new ArrayList(new Key(maxLength: 81), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of strings. Use this param to set the user roles in the team. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 81 characters long.', false, ['project'], example: '["editor"]') // For project-specific permissions, roles will be in the format `project--`. Template takes 9 characters, `projectId` and `role` can be upto 36 characters. In total, 81 characters. + ->param('roles', [], new ArrayList(new Key(maxLength: APP_LIMIT_ROLE_LENGTH), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of strings. Use this param to set the user roles in the team. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each ' . APP_LIMIT_ROLE_LENGTH . ' characters long.', false, ['project'], example: '["editor"]') ->param('url', '', fn ($redirectValidator) => $redirectValidator, 'URL to redirect the user back to your app from the invitation email. This parameter is not required when an API key is supplied. Only URLs from hostnames in your project platform list are allowed. This requirement helps to prevent an [open redirect](https://cheatsheetseries.owasp.org/cheatsheets/Unvalidated_Redirects_and_Forwards_Cheat_Sheet.html) attack against your project API.', true, ['redirectValidator']) // TODO add our own built-in confirm page ->param('name', '', new Text(128), 'Name of the new team member. Max length: 128 chars.', true) ->inject('response') diff --git a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Update.php b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Update.php index 6bb0a720696..4e72cea4fbd 100644 --- a/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Update.php +++ b/src/Appwrite/Platform/Modules/Teams/Http/Memberships/Update.php @@ -55,7 +55,7 @@ public function __construct() )) ->param('teamId', '', new UID(), 'Team ID.') ->param('membershipId', '', new UID(), 'Membership ID.') - ->param('roles', [], new ArrayList(new Key(maxLength: 81), APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of strings. Use this param to set the user\'s roles in the team. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 81 characters long.', false, ['project'], example: '["editor"]') // For project-specific permissions, roles will be in the format `project--`. Template takes 9 characters, `projectId` and `role` can be upto 36 characters. In total, 81 characters. + ->param('roles', [], new ArrayList(new Key(maxLength: APP_LIMIT_ROLE_LENGTH), APP_LIMIT_ARRAY_PARAMS_SIZE), 'An array of strings. Use this param to set the user\'s roles in the team. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each ' . APP_LIMIT_ROLE_LENGTH . ' characters long.', false, ['project'], example: '["editor"]') ->inject('request') ->inject('response') ->inject('user') diff --git a/src/Appwrite/Platform/Modules/Teams/Http/Teams/Create.php b/src/Appwrite/Platform/Modules/Teams/Http/Teams/Create.php index 72c8add2969..df998f66449 100644 --- a/src/Appwrite/Platform/Modules/Teams/Http/Teams/Create.php +++ b/src/Appwrite/Platform/Modules/Teams/Http/Teams/Create.php @@ -62,7 +62,7 @@ public function __construct() )) ->param('teamId', '', new CustomId(), 'Team ID. Choose a custom ID or generate a random ID with `ID.unique()`. Valid chars are a-z, A-Z, 0-9, period, hyphen, and underscore. Can\'t start with a special char. Max length is 36 chars.') ->param('name', null, new Text(128), 'Team name. Max length: 128 chars.') - ->param('roles', ['owner'], new ArrayList(new Key(), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of strings. Use this param to set the roles in the team for the user who created it. The default role is **owner**. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each 32 characters long.', true) + ->param('roles', ['owner'], new ArrayList(new Key(maxLength: APP_LIMIT_ROLE_LENGTH), APP_LIMIT_ARRAY_PARAMS_SIZE), 'Array of strings. Use this param to set the roles in the team for the user who created it. The default role is **owner**. A role can be any string. Learn more about [roles and permissions](https://appwrite.io/docs/permissions). Maximum of ' . APP_LIMIT_ARRAY_PARAMS_SIZE . ' roles are allowed, each ' . APP_LIMIT_ROLE_LENGTH . ' characters long.', true) ->inject('response') ->inject('user') ->inject('dbForProject') diff --git a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php index 55e0479a697..52d55eb0229 100644 --- a/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php +++ b/src/Appwrite/Platform/Modules/VCS/Http/GitHub/Deployment.php @@ -103,6 +103,13 @@ protected function createGitDeployments( Span::add("{$logBase}.build.skipped", 'true'); continue; } + + // Stale repository rows can outlive a disconnect, so only build for the repository the resource still links to. + if ($resource->getAttribute('repositoryId', '') !== $repositoryId) { + Span::add("{$logBase}.build.skipped.reason", 'repository not connected'); + Span::add("{$logBase}.build.skipped", 'true'); + continue; + } $resourceInternalId = $resource->getSequence(); $validator = new Contains(VCS_DEPLOYMENT_SKIP_PATTERNS); diff --git a/src/Appwrite/Utopia/Database/Validator/Queries/Messages.php b/src/Appwrite/Utopia/Database/Validator/Queries/Messages.php index 10032a87455..c3fa2317f74 100644 --- a/src/Appwrite/Utopia/Database/Validator/Queries/Messages.php +++ b/src/Appwrite/Utopia/Database/Validator/Queries/Messages.php @@ -11,6 +11,8 @@ class Messages extends Base 'status', 'description', 'providerType', + 'users', + 'targets', ]; /** diff --git a/src/Appwrite/Utopia/Response.php b/src/Appwrite/Utopia/Response.php index 60985c8145d..f88775cc64f 100644 --- a/src/Appwrite/Utopia/Response.php +++ b/src/Appwrite/Utopia/Response.php @@ -303,6 +303,7 @@ class Response extends SwooleResponse public const MODEL_OAUTH2_YANDEX = 'oAuth2Yandex'; public const MODEL_OAUTH2_X = 'oAuth2X'; public const MODEL_OAUTH2_WORDPRESS = 'oAuth2WordPress'; + public const MODEL_OAUTH2_WEBFLOW = 'oAuth2Webflow'; public const MODEL_OAUTH2_TWITCH = 'oAuth2Twitch'; public const MODEL_OAUTH2_STRIPE = 'oAuth2Stripe'; public const MODEL_OAUTH2_SPOTIFY = 'oAuth2Spotify'; diff --git a/src/Appwrite/Utopia/Response/Model/OAuth2ProviderList.php b/src/Appwrite/Utopia/Response/Model/OAuth2ProviderList.php index 927efd6b830..c8c46d2fd8b 100644 --- a/src/Appwrite/Utopia/Response/Model/OAuth2ProviderList.php +++ b/src/Appwrite/Utopia/Response/Model/OAuth2ProviderList.php @@ -33,6 +33,7 @@ public function __construct() Response::MODEL_OAUTH2_YANDEX, Response::MODEL_OAUTH2_X, Response::MODEL_OAUTH2_WORDPRESS, + Response::MODEL_OAUTH2_WEBFLOW, Response::MODEL_OAUTH2_TWITCH, Response::MODEL_OAUTH2_STRIPE, Response::MODEL_OAUTH2_SPOTIFY, diff --git a/src/Appwrite/Utopia/Response/Model/OAuth2Webflow.php b/src/Appwrite/Utopia/Response/Model/OAuth2Webflow.php new file mode 100644 index 00000000000..3e21c3e55a5 --- /dev/null +++ b/src/Appwrite/Utopia/Response/Model/OAuth2Webflow.php @@ -0,0 +1,47 @@ + 'webflow', + ]; + + public function getProviderLabel(): string + { + return 'Webflow'; + } + + public function getClientIdExample(): string + { + return '8bb20000000000000000000000000000000000000000000000000000000040dd'; + } + + public function getClientSecretExample(): string + { + return '59bf00000000000000000000000000000000000000000000000000000000fe59'; + } + + /** + * Get Name + * + * @return string + */ + public function getName(): string + { + return 'OAuth2Webflow'; + } + + /** + * Get Type + * + * @return string + */ + public function getType(): string + { + return Response::MODEL_OAUTH2_WEBFLOW; + } +} diff --git a/tests/e2e/Services/Account/AccountCustomClientTest.php b/tests/e2e/Services/Account/AccountCustomClientTest.php index d1d0a00a425..9750f32b7fb 100644 --- a/tests/e2e/Services/Account/AccountCustomClientTest.php +++ b/tests/e2e/Services/Account/AccountCustomClientTest.php @@ -2535,7 +2535,10 @@ public function testOAuth2TokenSessionProviderAccessToken(): void * app's success URL. The headers (a session cookie) ride along on every * hop, like a browser would send them. Returns the final redirect response. */ - private function followMockOAuth2Flow(string $path, array $headers = []): array + private function followMockOAuth2Flow(string $path, array $headers = [], array $params = [ + 'success' => 'http://localhost/v1/mock/tests/general/oauth2/success', + 'failure' => 'http://localhost/v1/mock/tests/general/oauth2/failure', + ]): array { $projectId = $this->getProject()['$id']; @@ -2543,10 +2546,7 @@ private function followMockOAuth2Flow(string $path, array $headers = []): array 'origin' => 'http://localhost', 'content-type' => 'application/json', 'x-appwrite-project' => $projectId, - ], $headers), [ - 'success' => 'http://localhost/v1/mock/tests/general/oauth2/success', - 'failure' => 'http://localhost/v1/mock/tests/general/oauth2/failure', - ], followRedirects: false); + ], $headers), $params, followRedirects: false); $this->assertEquals(301, $response['headers']['status-code']); @@ -3443,9 +3443,52 @@ public function testOAuthUnverifiedEmailCannotLinkToExistingAccount(): void } /** - * Default OAuth failure relay pages need `project` so native apps can deep-link via - * appwrite-callback-{project}://. Without it the UI shows "Missing redirect URL" - * instead of the real OAuth error. + * The default OAuth success URL redirects straight to appwrite-callback-{project}://, + * carrying the session the native SDKs store. + */ + public function testOAuthDefaultSuccessRedirectsToApp(): void + { + $provider = 'mock'; + $projectId = $this->getProject()['$id']; + + $response = $this->client->call(Client::METHOD_PATCH, '/projects/' . $projectId . '/oauth2', array_merge([ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => 'console', + 'cookie' => 'a_session_console=' . $this->getRoot()['session'], + ]), [ + 'provider' => $provider, + 'appId' => '1', + 'secret' => '123456', + 'enabled' => true, + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + + // Omit success and failure so Appwrite uses the default relay URLs + $response = $this->followMockOAuth2Flow('/account/sessions/oauth2/' . $provider, params: []); + + $location = $response['headers']['location']; + $this->assertStringStartsWith('appwrite-callback-' . $projectId . '://?', $location); + + // parse_url() rejects a scheme with no host, so read the query directly + $query = []; + \parse_str(\explode('?', $location, 2)[1], $query); + + // Native SDKs store the handoff as their session cookie + $response = $this->client->call(Client::METHOD_GET, '/account', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => ($query['key'] ?? '') . '=' . ($query['secret'] ?? ''), + ]); + + $this->assertEquals(200, $response['headers']['status-code']); + } + + /** + * The default OAuth failure URL redirects straight to appwrite-callback-{project}://, + * carrying `project` and the real OAuth error. */ public function testOAuthDefaultFailureRedirectIncludesProject(): void { @@ -3503,11 +3546,11 @@ public function testOAuthDefaultFailureRedirectIncludesProject(): void $this->assertEquals(301, $response['headers']['status-code']); $location = $response['headers']['location']; - $path = \parse_url($location, PHP_URL_PATH); - $query = []; - \parse_str((string) \parse_url($location, PHP_URL_QUERY), $query); + $this->assertStringStartsWith('appwrite-callback-' . $projectId . '://?', $location); - $this->assertEquals('/auth/oauth2/failure', $path); + // parse_url() rejects a scheme with no host, so read the query directly + $query = []; + \parse_str(\explode('?', $location, 2)[1], $query); $this->assertEquals($projectId, $query['project'] ?? null); $this->assertNotEmpty($query['error'] ?? null); @@ -5452,6 +5495,223 @@ public function testMFAAuthenticatorCompletesEmailPasswordSession(): void $this->assertEquals($data['id'], $account['body']['$id']); } + protected function createJWTHeaders(string $session): array + { + $projectId = $this->getProject()['$id']; + + $response = $this->client->call(Client::METHOD_POST, '/account/jwts', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + return [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-jwt' => $response['body']['jwt'], + ]; + } + + protected function createSessionCookie(string $email, string $password): array + { + $projectId = $this->getProject()['$id']; + + $response = $this->client->call(Client::METHOD_POST, '/account/sessions/email', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], [ + 'email' => $email, + 'password' => $password, + ]); + $this->assertEquals(201, $response['headers']['status-code']); + + return [ + 'id' => $response['body']['$id'], + 'secret' => $response['cookies']['a_session_' . $projectId], + 'headers' => [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => 'a_session_' . $projectId . '=' . $response['cookies']['a_session_' . $projectId], + ], + ]; + } + + public function testMFARecencyCheckUnderJWT(): void + { + $data = $this->createFreshAccountWithSession(); + $projectId = $this->getProject()['$id']; + + $headers = [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => 'a_session_' . $projectId . '=' . $data['session'], + ]; + + $recoveryCodes = $this->client->call(Client::METHOD_POST, '/account/mfa/recovery-codes', $headers); + $this->assertEquals(201, $recoveryCodes['headers']['status-code']); + + $cookieAttempt = $this->client->call(Client::METHOD_PATCH, '/account/mfa/recovery-codes', $headers); + $this->assertEquals(401, $cookieAttempt['headers']['status-code']); + $this->assertEquals('user_challenge_required', $cookieAttempt['body']['type']); + + $jwtAttempt = $this->client->call(Client::METHOD_PATCH, '/account/mfa/recovery-codes', $this->createJWTHeaders($data['session'])); + $this->assertEquals(401, $jwtAttempt['headers']['status-code']); + $this->assertEquals('user_challenge_required', $jwtAttempt['body']['type']); + } + + public function testMFAFactorCountUnderJWT(): void + { + $data = $this->createFreshAccountWithSession(); + $projectId = $this->getProject()['$id']; + + $headers = [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => 'a_session_' . $projectId . '=' . $data['session'], + ]; + + $sessionB = $this->createSessionCookie($data['email'], $data['password']); + + // Issued before MFA is enabled, since session B could not create a JWT afterwards + $jwtHeaders = $this->createJWTHeaders($sessionB['secret']); + + $authenticator = $this->client->call(Client::METHOD_POST, '/account/mfa/authenticators/totp', $headers); + $this->assertEquals(200, $authenticator['headers']['status-code']); + + $totp = \OTPHP\TOTP::create($authenticator['body']['secret']); + if ($totp->expiresIn() <= 5) { + $this->getNextTOTP($totp, $totp->now()); + } + $verification = $this->client->call(Client::METHOD_PUT, '/account/mfa/authenticators/totp', $headers, [ + 'otp' => $totp->now(), + ]); + $this->assertEquals(200, $verification['headers']['status-code']); + + $mfa = $this->client->call(Client::METHOD_PATCH, '/account/mfa', $headers, ['mfa' => true]); + $this->assertEquals(200, $mfa['headers']['status-code']); + + $cookieAttempt = $this->client->call(Client::METHOD_GET, '/account', $sessionB['headers']); + $this->assertEquals(401, $cookieAttempt['headers']['status-code']); + $this->assertEquals('user_more_factors_required', $cookieAttempt['body']['type']); + + $jwtAttempt = $this->client->call(Client::METHOD_GET, '/account', $jwtHeaders); + $this->assertEquals(401, $jwtAttempt['headers']['status-code']); + $this->assertEquals('user_more_factors_required', $jwtAttempt['body']['type']); + } + + public function testGetSessionCurrentUnderJWT(): void + { + $data = $this->createFreshAccountWithSession(); + + $response = $this->client->call(Client::METHOD_GET, '/account/sessions/current', $this->createJWTHeaders($data['session'])); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($data['sessionId'], $response['body']['$id']); + } + + public function testUpdateSessionCurrentUnderJWT(): void + { + $data = $this->createFreshAccountWithSession(); + + $response = $this->client->call(Client::METHOD_PATCH, '/account/sessions/current', $this->createJWTHeaders($data['session'])); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals($data['sessionId'], $response['body']['$id']); + } + + public function testDeleteSessionCurrentUnderJWT(): void + { + $data = $this->createFreshAccountWithSession(); + $projectId = $this->getProject()['$id']; + + $sessionB = $this->createSessionCookie($data['email'], $data['password']); + + $delete = $this->client->call(Client::METHOD_DELETE, '/account/sessions/current', $this->createJWTHeaders($data['session'])); + $this->assertEquals(204, $delete['headers']['status-code']); + + $checkA = $this->client->call(Client::METHOD_GET, '/account', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => 'a_session_' . $projectId . '=' . $data['session'], + ]); + $this->assertEquals(401, $checkA['headers']['status-code']); + + $checkB = $this->client->call(Client::METHOD_GET, '/account', $sessionB['headers']); + $this->assertEquals(200, $checkB['headers']['status-code']); + } + + public function testListSessionsCurrentUnderJWT(): void + { + $data = $this->createFreshAccountWithSession(); + + $sessionB = $this->createSessionCookie($data['email'], $data['password']); + + $response = $this->client->call(Client::METHOD_GET, '/account/sessions', $this->createJWTHeaders($data['session'])); + $this->assertEquals(200, $response['headers']['status-code']); + $this->assertEquals(2, $response['body']['total']); + + $current = []; + foreach ($response['body']['sessions'] as $session) { + $current[$session['$id']] = $session['current']; + } + + $this->assertTrue($current[$data['sessionId']]); + $this->assertFalse($current[$sessionB['id']]); + } + + public function testUpdatePasswordInvalidatesOtherSessionsUnderJWT(): void + { + $data = $this->createFreshAccountWithSession(); + $this->updateProjectinvalidateSessionsProperty(true); + + $sessionB = $this->createSessionCookie($data['email'], $data['password']); + $jwtHeaders = $this->createJWTHeaders($data['session']); + + $response = $this->client->call(Client::METHOD_PATCH, '/account/password', $jwtHeaders, [ + 'password' => 'new-password', + 'oldPassword' => $data['password'], + ]); + $this->assertEquals(200, $response['headers']['status-code']); + + $checkB = $this->client->call(Client::METHOD_GET, '/account', $sessionB['headers']); + $this->assertEquals(401, $checkB['headers']['status-code']); + + $checkJWT = $this->client->call(Client::METHOD_GET, '/account', $jwtHeaders); + $this->assertEquals(200, $checkJWT['headers']['status-code']); + } + + public function testCreatePushTargetReplacesRotatedTokenUnderJWT(): void + { + $data = $this->createFreshAccountWithSession(); + $jwtHeaders = $this->createJWTHeaders($data['session']); + + $first = $this->client->call(Client::METHOD_POST, '/account/targets/push', $jwtHeaders, [ + 'targetId' => ID::unique(), + 'identifier' => 'jwt-identifier-before-rotation-' . $data['id'], + ]); + $this->assertEquals(201, $first['headers']['status-code']); + + $second = $this->client->call(Client::METHOD_POST, '/account/targets/push', $jwtHeaders, [ + 'targetId' => ID::unique(), + 'identifier' => 'jwt-identifier-after-rotation-' . $data['id'], + ]); + $this->assertEquals(201, $second['headers']['status-code']); + $this->assertEquals($first['body']['$id'], $second['body']['$id']); + + $account = $this->client->call(Client::METHOD_GET, '/account', $jwtHeaders); + $this->assertEquals(200, $account['headers']['status-code']); + + $identifiers = \array_column($account['body']['targets'], 'identifier'); + $this->assertContains('jwt-identifier-after-rotation-' . $data['id'], $identifiers); + $this->assertNotContains('jwt-identifier-before-rotation-' . $data['id'], $identifiers); + } + public function testRefreshEmailPasswordSession(): void { $email = uniqid() . 'user@localhost.test'; diff --git a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php index 95c36625151..24b2b765832 100644 --- a/tests/e2e/Services/Functions/FunctionsCustomServerTest.php +++ b/tests/e2e/Services/Functions/FunctionsCustomServerTest.php @@ -2140,6 +2140,27 @@ public function testCreateExecution(): void ], $this->getHeaders()), []); $this->assertEquals(204, $execution['headers']['status-code']); }, 10000, 500); + + /** + * Test for FAILURE + */ + $execution = $this->createExecution($data['functionId'], [ + 'headers' => [ + 'X-Test' => ['bad'], + ], + ]); + + $this->assertEquals(400, $execution['headers']['status-code']); + $this->assertEquals('general_argument_invalid', $execution['body']['type']); + + $execution = $this->createExecution($data['functionId'], [ + 'headers' => [ + 'bad/name' => 'value', + ], + ]); + + $this->assertEquals(400, $execution['headers']['status-code']); + $this->assertEquals('general_argument_invalid', $execution['body']['type']); } finally { $this->cleanupFunction($functionId); } diff --git a/tests/e2e/Services/Mqtt/MqttServerTest.php b/tests/e2e/Services/Mqtt/MqttServerTest.php index 4dbfe7cfbfa..ff48fa68d5e 100644 --- a/tests/e2e/Services/Mqtt/MqttServerTest.php +++ b/tests/e2e/Services/Mqtt/MqttServerTest.php @@ -416,7 +416,7 @@ private function setupPushTopic(array $server, string $userId, string $name, int * @param array $server * @param array $data */ - private function publishCampaign(array $server, string $topicId, string $title, string $body, array $data = [], array $extra = []): void + private function publishCampaign(array $server, string $topicId, string $title, string $body, array $data = [], array $extra = []): string { $push = $this->client->call(Client::METHOD_POST, '/messaging/messages/push', $server, \array_merge([ 'messageId' => ID::unique(), @@ -432,6 +432,8 @@ private function publishCampaign(array $server, string $topicId, string $title, $message = $this->client->call(Client::METHOD_GET, '/messaging/messages/' . $messageId, $server); $this->assertContains($message['body']['status'], [MessageStatus::SENT, MessageStatus::FAILED]); }, 30000, 500); + + return $messageId; } /** @@ -501,7 +503,7 @@ public function testCampaignFansOutToMqttSubscriber(): void $subscriber->subscribe([$topicName]); try { - $this->publishCampaign($server, $topicId, 'Match update', 'India needs 12 off 6', ['matchId' => '42']); + $messageId = $this->publishCampaign($server, $topicId, 'Match update', 'India needs 12 off 6', ['matchId' => '42']); $received = $subscriber->consume(limit: 1, timeout: 20.0); } finally { $subscriber->disconnect(); @@ -516,6 +518,7 @@ public function testCampaignFansOutToMqttSubscriber(): void $this->assertEquals('Match update', $payload['notification']['title']); $this->assertEquals('India needs 12 off 6', $payload['notification']['body']); $this->assertEquals(['matchId' => '42'], $payload['data']); + $this->assertSame($messageId, $payload['messageId']); } public function testCampaignCarriesChannelIdToSubscriber(): void diff --git a/tests/e2e/Services/ProjectWebhooks/WebhooksCustomClientTest.php b/tests/e2e/Services/ProjectWebhooks/WebhooksCustomClientTest.php index 7ed5edfad19..5dd9c6b119d 100644 --- a/tests/e2e/Services/ProjectWebhooks/WebhooksCustomClientTest.php +++ b/tests/e2e/Services/ProjectWebhooks/WebhooksCustomClientTest.php @@ -355,6 +355,33 @@ public function testDeleteAccountSession(): void $this->assertEquals(true, $webhook['data']['current']); } + public function testDeleteAccountSessionCurrentUnderJWT(): void + { + $data = $this->setupAccountWithSession(); + $projectId = $this->getProject()['$id']; + + $jwt = $this->client->call(Client::METHOD_POST, '/account/jwts', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'cookie' => 'a_session_' . $projectId . '=' . $data['session'], + ]); + $this->assertEquals(201, $jwt['headers']['status-code']); + + $response = $this->client->call(Client::METHOD_DELETE, '/account/sessions/current', [ + 'origin' => 'http://localhost', + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-jwt' => $jwt['body']['jwt'], + ]); + $this->assertEquals(204, $response['headers']['status-code']); + + $webhook = $this->getLastRequest($this->webhookEventProbe("users.{$data['id']}.sessions.{$data['sessionId']}.delete")); + $this->assertEquals($data['sessionId'], $webhook['data']['$id']); + $this->assertEquals(true, $webhook['data']['current']); + $this->assertIsString($webhook['data']['countryName']); + } + public function testDeleteAccountSessions(): void { // Set up account with session diff --git a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php index 736664c6a8e..80eff2a4088 100644 --- a/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php +++ b/tests/e2e/Services/Realtime/RealtimeCustomClientTest.php @@ -4195,6 +4195,347 @@ public function testRelationshipPayloadHidesRelatedDoc() $client->close(); } + public function testChannelDatabaseRelationshipDelete(): void + { + if (!$this->getSupportForRelationships()) { + $this->expectNotToPerformAssertions(); + return; + } + + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + $database = $this->client->call(Client::METHOD_POST, '/databases', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'Relationship Delete DB', + ]); + $databaseId = $database['body']['$id']; + + $collections = []; + foreach (['parent', 'child'] as $side) { + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => $side, + 'permissions' => [Permission::create(Role::any())], + 'documentSecurity' => true, + ]); + $collections[$side] = $collection['body']['$id']; + } + + $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collections['child'] . '/attributes/string', $headers, [ + 'key' => 'name', + 'size' => 256, + 'required' => false, + ]); + + $this->assertEventually(function () use ($databaseId, $collections, $headers) { + $attribute = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collections['child'] . '/attributes/name', $headers); + $this->assertEquals('available', $attribute['body']['status']); + }, 30000, 250); + + /** + * Test for SUCCESS + * + * Every successful delete notifies the surviving side, whatever onDelete is. + */ + $cases = [ + ['type' => 'oneToMany', 'onDelete' => 'setNull', 'deleted' => 'child'], + ['type' => 'oneToMany', 'onDelete' => 'restrict', 'deleted' => 'child'], + ['type' => 'oneToMany', 'onDelete' => 'cascade', 'deleted' => 'child'], + ['type' => 'oneToOne', 'onDelete' => 'setNull', 'deleted' => 'parent'], + ['type' => 'manyToOne', 'onDelete' => 'restrict', 'deleted' => 'parent'], + ['type' => 'manyToMany', 'onDelete' => 'cascade', 'deleted' => 'child'], + ]; + + foreach ($cases as $index => $case) { + $key = 'children' . $index; + $twoWayKey = 'parent' . $index; + + $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collections['parent'] . '/attributes/relationship', $headers, [ + 'relatedCollectionId' => $collections['child'], + 'type' => $case['type'], + 'twoWay' => true, + 'key' => $key, + 'twoWayKey' => $twoWayKey, + 'onDelete' => $case['onDelete'], + ]); + + $this->assertEventually(function () use ($databaseId, $collections, $headers, $key) { + $attribute = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collections['parent'] . '/attributes/' . $key, $headers); + $this->assertEquals('available', $attribute['body']['status']); + }, 30000, 250); + + $permissions = [ + Permission::read(Role::any()), + Permission::delete(Role::any()), + ]; + + $child = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collections['child'] . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => ['name' => 'child'], + 'permissions' => $permissions, + ]); + $childId = $child['body']['$id']; + + $parent = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collections['parent'] . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => [$key => \in_array($case['type'], ['oneToMany', 'manyToMany']) ? [$childId] : $childId], + 'permissions' => $permissions, + ]); + $this->assertEquals(201, $parent['headers']['status-code']); + + $ids = ['parent' => $parent['body']['$id'], 'child' => $childId]; + $survivor = $case['deleted'] === 'child' ? 'parent' : 'child'; + + $channel = 'databases.' . $databaseId . '.collections.' . $collections[$survivor] . '.documents.' . $ids[$survivor]; + + $client = $this->getWebsocket([$channel], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]); + $client->receive(); + + $response = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $collections[$case['deleted']] . '/documents/' . $ids[$case['deleted']], array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders())); + $this->assertEquals(204, $response['headers']['status-code']); + + $event = $this->receiveUntilEvent( + $client, + fn (array $message): bool => \in_array($channel . '.update', $message['data']['events'] ?? [], true), + timeoutMs: 10000 + ); + + $this->assertEquals($ids[$survivor], $event['data']['payload']['$id']); + $this->assertArrayNotHasKey($survivor === 'parent' ? $key : $twoWayKey, $event['data']['payload']); + + $client->close(); + } + } + + public function testChannelDatabaseRelationshipDeletePermissions(): void + { + if (!$this->getSupportForRelationships()) { + $this->expectNotToPerformAssertions(); + return; + } + + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + $database = $this->client->call(Client::METHOD_POST, '/databases', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'Relationship Delete Permissions DB', + ]); + $databaseId = $database['body']['$id']; + + $collections = []; + foreach (['parent', 'child'] as $side) { + $collection = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections', $headers, [ + 'collectionId' => ID::unique(), + 'name' => $side, + 'permissions' => [Permission::create(Role::any())], + 'documentSecurity' => true, + ]); + $collections[$side] = $collection['body']['$id']; + } + + $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collections['child'] . '/attributes/string', $headers, [ + 'key' => 'name', + 'size' => 256, + 'required' => false, + ]); + + $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collections['parent'] . '/attributes/relationship', $headers, [ + 'relatedCollectionId' => $collections['child'], + 'type' => 'oneToMany', + 'twoWay' => true, + 'key' => 'children', + 'twoWayKey' => 'parent', + 'onDelete' => 'setNull', + ]); + + $this->assertEventually(function () use ($databaseId, $collections, $headers) { + $name = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collections['child'] . '/attributes/name', $headers); + $this->assertEquals('available', $name['body']['status']); + + $children = $this->client->call(Client::METHOD_GET, '/databases/' . $databaseId . '/collections/' . $collections['parent'] . '/attributes/children', $headers); + $this->assertEquals('available', $children['body']['status']); + }, 30000, 250); + + $child = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collections['child'] . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => ['name' => 'child'], + 'permissions' => [ + Permission::read(Role::any()), + Permission::delete(Role::any()), + ], + ]); + $childId = $child['body']['$id']; + + $parent = $this->client->call(Client::METHOD_POST, '/databases/' . $databaseId . '/collections/' . $collections['parent'] . '/documents', $headers, [ + 'documentId' => ID::unique(), + 'data' => ['children' => [$childId]], + 'permissions' => [Permission::read(Role::user($user['$id']))], + ]); + $parentId = $parent['body']['$id']; + + $channel = 'databases.' . $databaseId . '.collections.' . $collections['parent'] . '.documents.' . $parentId; + + $owner = $this->getWebsocket([$channel], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]); + $owner->receive(); + + $guest = $this->getWebsocket([$channel], [ + 'origin' => 'http://localhost', + ]); + $guest->receive(); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_DELETE, '/databases/' . $databaseId . '/collections/' . $collections['child'] . '/documents/' . $childId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders())); + $this->assertEquals(204, $response['headers']['status-code']); + + $this->receiveUntilEvent( + $owner, + fn (array $message): bool => \in_array($channel . '.update', $message['data']['events'] ?? [], true), + timeoutMs: 10000 + ); + + /** + * Test for FAILURE + */ + try { + $guest->receive(); + $this->fail('Guest should not receive an update for a parent it cannot read'); + } catch (TimeoutException) { + $this->addToAssertionCount(1); + } + + $owner->close(); + $guest->close(); + } + + public function testChannelTablesDBRelationshipDelete(): void + { + if (!$this->getSupportForRelationships()) { + $this->expectNotToPerformAssertions(); + return; + } + + $user = $this->getUser(); + $session = $user['session'] ?? ''; + $projectId = $this->getProject()['$id']; + $headers = [ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + 'x-appwrite-key' => $this->getProject()['apiKey'], + ]; + + $database = $this->client->call(Client::METHOD_POST, '/tablesdb', $headers, [ + 'databaseId' => ID::unique(), + 'name' => 'Relationship Delete TablesDB', + ]); + $databaseId = $database['body']['$id']; + + $tables = []; + foreach (['parent', 'child'] as $side) { + $table = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables', $headers, [ + 'tableId' => ID::unique(), + 'name' => $side, + 'permissions' => [Permission::create(Role::any())], + 'rowSecurity' => true, + ]); + $tables[$side] = $table['body']['$id']; + } + + $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tables['child'] . '/columns/string', $headers, [ + 'key' => 'name', + 'size' => 256, + 'required' => false, + ]); + + $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tables['parent'] . '/columns/relationship', $headers, [ + 'relatedTableId' => $tables['child'], + 'type' => 'oneToMany', + 'twoWay' => true, + 'key' => 'children', + 'twoWayKey' => 'parent', + 'onDelete' => 'setNull', + ]); + + $this->assertEventually(function () use ($databaseId, $tables, $headers) { + $name = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tables['child'] . '/columns/name', $headers); + $this->assertEquals('available', $name['body']['status']); + + $children = $this->client->call(Client::METHOD_GET, '/tablesdb/' . $databaseId . '/tables/' . $tables['parent'] . '/columns/children', $headers); + $this->assertEquals('available', $children['body']['status']); + }, 30000, 250); + + $permissions = [ + Permission::read(Role::any()), + Permission::delete(Role::any()), + ]; + + $child = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tables['child'] . '/rows', $headers, [ + 'rowId' => ID::unique(), + 'data' => ['name' => 'child'], + 'permissions' => $permissions, + ]); + $childId = $child['body']['$id']; + + $parent = $this->client->call(Client::METHOD_POST, '/tablesdb/' . $databaseId . '/tables/' . $tables['parent'] . '/rows', $headers, [ + 'rowId' => ID::unique(), + 'data' => ['children' => [$childId]], + 'permissions' => $permissions, + ]); + $parentId = $parent['body']['$id']; + + $channel = 'databases.' . $databaseId . '.tables.' . $tables['parent'] . '.rows.' . $parentId; + + $client = $this->getWebsocket([$channel], [ + 'origin' => 'http://localhost', + 'cookie' => 'a_session_' . $projectId . '=' . $session, + ]); + $client->receive(); + + /** + * Test for SUCCESS + */ + $response = $this->client->call(Client::METHOD_DELETE, '/tablesdb/' . $databaseId . '/tables/' . $tables['child'] . '/rows/' . $childId, array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $projectId, + ], $this->getHeaders())); + $this->assertEquals(204, $response['headers']['status-code']); + + $this->receiveUntilEvent( + $client, + fn (array $message): bool => \in_array($channel . '.update', $message['data']['events'] ?? [], true), + timeoutMs: 10000 + ); + + $client->close(); + } + /** * Simulate concurrent realtime traffic using Swoole coroutines. * Opens multiple websocket clients concurrently, then performs create/update/delete ops. diff --git a/tests/e2e/Services/Teams/TeamsBase.php b/tests/e2e/Services/Teams/TeamsBase.php index 413438cd824..d0e0548ab87 100644 --- a/tests/e2e/Services/Teams/TeamsBase.php +++ b/tests/e2e/Services/Teams/TeamsBase.php @@ -166,6 +166,29 @@ public function testCreateTeam(): void $this->assertIsInt($response3['body']['total']); $this->assertEquals(true, $dateValidator->isValid($response3['body']['$createdAt'])); + // A project-scoped role is longer than a bare ID + $role = 'project-' . ID::unique() . '-developer'; + $response4 = $this->client->call(Client::METHOD_POST, '/teams', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders()), [ + 'teamId' => ID::unique(), + 'name' => 'Aston Villa', + 'roles' => [$role], + ]); + + $this->assertEquals(201, $response4['headers']['status-code']); + + // An API key creates the team without a creator membership to read the role from + if ($this->getSide() !== 'server') { + $memberships = $this->client->call(Client::METHOD_GET, '/teams/' . $response4['body']['$id'] . '/memberships', array_merge([ + 'content-type' => 'application/json', + 'x-appwrite-project' => $this->getProject()['$id'], + ], $this->getHeaders())); + + $this->assertContains($role, $memberships['body']['memberships'][0]['roles']); + } + /** * Test for FAILURE */ diff --git a/tests/unit/Appwrite/Messaging/Adapter/Push/AppwriteTest.php b/tests/unit/Appwrite/Messaging/Adapter/Push/AppwriteTest.php index 493a1bc9b48..44a9c88147b 100644 --- a/tests/unit/Appwrite/Messaging/Adapter/Push/AppwriteTest.php +++ b/tests/unit/Appwrite/Messaging/Adapter/Push/AppwriteTest.php @@ -228,6 +228,7 @@ public function testPayloadEnvelopeShape(): void $this->assertSame('India vs Australia', $payload['notification']['body']); $this->assertSame(['matchId' => '42'], $payload['data']); $this->assertSame('high', $payload['priority']); + $this->assertSame('msg-1', $payload['messageId']); } public function testSequenceAdvancesPerPublish(): void diff --git a/tests/unit/Functions/Validator/HeadersTest.php b/tests/unit/Functions/Validator/HeadersTest.php index 9cc48c9bb34..22bf601791d 100644 --- a/tests/unit/Functions/Validator/HeadersTest.php +++ b/tests/unit/Functions/Validator/HeadersTest.php @@ -83,6 +83,26 @@ public function testValues(): void ]; $this->assertTrue($this->object->isValid($headers)); + $headers = [ + 'X-Header' => ['bad'], + ]; + $this->assertFalse($this->object->isValid($headers)); + + $headers = [ + 'X-Header' => 123, + ]; + $this->assertTrue($this->object->isValid($headers)); + + $headers = [ + 'X-Header' => true, + ]; + $this->assertTrue($this->object->isValid($headers)); + + $headers = [ + 'bad/name' => 'value', + ]; + $this->assertFalse($this->object->isValid($headers)); + $headers = [ true => 'value', ];