From 6ed4e4abd150d932d1a33c64104f4e711812f6ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?An=C4=B1lcan=20=C3=87ak=C4=B1r?= Date: Sat, 26 Sep 2026 03:06:54 +0300 Subject: [PATCH 1/2] feat: add the sync skeleton, Str.ascii and Str.squish, and AppLifecycle.states Add SyncFeed, SyncLedger, SyncBookmarks, SyncPushRow, SyncReport and CreateSyncCursorsTable: a client-side push-then-pull sync skeleton over a documented wire protocol, with ledger writes issued in a SAVEPOINT so they nest under a caller's own transaction. Add Str.ascii (Latin-only diacritic fold, case preserved) and Str.squish (Laravel semantics), and route the User-Agent app-name folding through Str.ascii instead of its own logic. Add AppLifecycle.states(), a lazy lifecycle stream. Update exports, doc/digging-deeper/sync.md, the helpers doc, the README row, CHANGELOG under Unreleased, the magic-framework skill references (including the starter guest claim marked unreleased) and the SKILL.md stamp. --- CHANGELOG.md | 4 + README.md | 1 + doc/contributing/documentation.md | 3 +- doc/digging-deeper/helpers.md | 32 +- doc/digging-deeper/sync.md | 136 +++++++ lib/magic.dart | 6 + lib/src/network/network_service_provider.dart | 103 +---- lib/src/support/app_lifecycle.dart | 53 +++ lib/src/support/str.dart | 140 +++++++ lib/src/sync/create_sync_cursors_table.dart | 34 ++ lib/src/sync/sync_feed.dart | 356 ++++++++++++++++++ lib/src/sync/sync_ledger.dart | 107 ++++++ skills/magic-framework/SKILL.md | 4 +- .../references/plugin-starter.md | 23 +- .../references/secondary-systems.md | 43 +++ test/support/app_lifecycle_test.dart | 76 ++++ test/support/exports_test.dart | 27 +- test/support/str_fold_parity_test.dart | 173 +++++++++ test/support/str_test.dart | 44 +++ test/sync/sync_feed_test.dart | 250 ++++++++++++ test/sync/sync_ledger_test.dart | 167 ++++++++ 21 files changed, 1685 insertions(+), 97 deletions(-) create mode 100644 doc/digging-deeper/sync.md create mode 100644 lib/src/support/app_lifecycle.dart create mode 100644 lib/src/sync/create_sync_cursors_table.dart create mode 100644 lib/src/sync/sync_feed.dart create mode 100644 lib/src/sync/sync_ledger.dart create mode 100644 test/support/app_lifecycle_test.dart create mode 100644 test/support/str_fold_parity_test.dart create mode 100644 test/sync/sync_feed_test.dart create mode 100644 test/sync/sync_ledger_test.dart diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e944c26..51e8c43f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,10 @@ All notable changes to this project will be documented in this file. - **`RefetchesOnMount` and `SubmitsOnce`, two view-layer mixins closing gaps in magic's singleton-controller and async-submit model.** `RefetchesOnMount`, mixed onto a `MagicStatefulViewState`, fire-and-forget refetches a controller's data on every mount (not only the first, since controllers fire `onInit` once per instance for the app's lifetime); `SubmitsOnce`, mixed onto a form's `State`, guards a submit handler against a second tap while the first write is in flight, resetting in a `finally` so a throwing submit re-arms the button. (`lib/src/ui/refetches_on_mount.dart`, `lib/src/ui/submits_once.dart`, `doc/basics/views.md`, `skills/magic-framework/`) - **`Env.filled(key, fallback)` and `Env.getOrFail(key)`, guards for a value a blank silently corrupts.** `Env.get`/`env()` only fall back when a key is entirely absent, so a present-but-blank or quote-only value silently resolved to `''` and has shipped as a blank browser tab title and a link pointing at a path with no origin. `Env.filled` treats absent, blank, and quote-only the same way, stripping one wrapping quote pair and surrounding whitespace from a present value. `Env.getOrFail` mirrors Laravel's `Env::getOrFail`, throwing a `StateError` only when the key is entirely missing. (`lib/src/foundation/env.dart`, `doc/getting-started/configuration.md`, `skills/magic-framework/`) - **`MagicTest.loadTranslations(locale, {directory})`, a real translation catalogue for a widget test.** Reads `/.json` off disk, flattens it with the app's own flatten rule (`JsonAssetLoader.flatten`, now public), and installs it before awaiting the translator's load, so `trans()` resolves real catalogue strings in a test instead of rendering the raw dotted key. (`lib/src/testing/magic_test.dart`, `doc/testing/getting-started.md`, `skills/magic-framework/`) +- **`SyncFeed`, `SyncLedger` and `CreateSyncCursorsTable`, a push-then-pull sync skeleton over one REST resource.** `SyncFeed.run({scope, account})` sends everything a subclass's `pending()` reports written since this device's own push mark (`POST '$resource/sync'`, batched at `batchSize`), then walks pull pages (`GET resource`, up to `maxPages`) handing each row to the subclass's `adoptRow()`, and never throws: an exception surfaces as a `SyncReport.failure` string, logged via `Log.error`. Only the push mark advances locally; the pull cursor is the server's opaque text handed straight back, because a row adopted from the server carries the originating device's own clock. `SyncLedger` is the bookmark store behind it (`sync_cursors`, upserted by delete-then-insert inside a `SAVEPOINT` so it nests inside a caller's own transaction), and `CreateSyncCursorsTable` creates that table; magic has no migration discovery, so an app lists it in its own `Migrator().run([...])` call. Scope derivation, any additional salt, and when a feed runs are left to the app. (`lib/src/sync/sync_feed.dart`, `lib/src/sync/sync_ledger.dart`, `lib/src/sync/create_sync_cursors_table.dart`, `doc/digging-deeper/sync.md`, `skills/magic-framework/`) +- **`Str.ascii(value)` and `Str.squish(value)`, two more of Laravel's `Str` helpers ported.** `ascii` folds Latin letters (accents, the Romanian comma-below letters, `ẞ`) to their plain ASCII base for a search key, Latin only, diverging from Laravel's `Str::ascii`, which transliterates every script it has a table for and would otherwise collapse a non-Latin word to `?` or a phonetic guess. `squish` trims and collapses every run of whitespace to one space, matching Laravel's `Str::squish`, over Dart's `\s` class plus two Hangul filler code points a rendered blank can carry without registering as whitespace. (`lib/src/support/str.dart`, `doc/digging-deeper/helpers.md`, `skills/magic-framework/`) +- **The default `User-Agent`'s app-name folding (`NetworkServiceProvider`, via `Str.ascii`) now also folds `ș`, `ț`, `ẞ`, and the Angstrom sign U+212B to their plain ASCII base.** They used to fall through `Str.ascii`'s table untouched and were then dropped by the printable-ASCII filter that follows it. +- **`AppLifecycle.states()`, the app lifecycle as a `Stream` for a reader built before a `WidgetsBinding` necessarily exists.** A dependency constructed inside a service provider's `register()` runs before the app has bound anything, so reaching for `WidgetsBinding.instance` there throws; `states()` defers that lookup to the moment a listener subscribes, adding its own observer to the binding on `listen` and removing it on `cancel`, so nothing outlives its reader. Prefer Flutter's own `AppLifecycleListener` once a binding is guaranteed to exist. (`lib/src/support/app_lifecycle.dart`, `doc/digging-deeper/helpers.md`, `skills/magic-framework/`) ### Changed diff --git a/README.md b/README.md index ca2a779a..1dc11001 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ The same Facades, the same Eloquent syntax, the same Service Provider lifecycle | 🌍 | **Localization** | JSON-based i18n with `:attribute` placeholders. | | 🎨 | **Wind UI** | Built-in [Wind](https://wind.fluttersdk.com) Tailwind-syntax styling with `className` strings. | | 📡 | **Broadcasting** | Laravel Echo equivalent: real-time WebSocket channels via the `Echo` facade with presence support and `Echo.fake()`. | +| 🔄 | **Offline Sync** | `SyncFeed` runs a push-then-pull loop over any REST resource, bookmarked by `SyncLedger`'s `sync_cursors` table (`CreateSyncCursorsTable`). | | 🧪 | **Testing** | First-class fakes: `Http.fake()`, `Auth.fake()`, `Cache.fake()`, `Vault.fake()`, `Log.fake()`, `Echo.fake()`. No mockito needed. | | 🧰 | **Magic CLI** | Artisan-style scaffolding via `dart run magic:artisan make:model`, `make:controller`, 15 generators, plus `make:component` for design-first component workflows and `design:sync` / `design:lint` to drive the Wind theme from a `DESIGN.md`. | diff --git a/doc/contributing/documentation.md b/doc/contributing/documentation.md index 3ced06a2..9d320abe 100644 --- a/doc/contributing/documentation.md +++ b/doc/contributing/documentation.md @@ -55,7 +55,7 @@ Pages live under `doc/` in one of these subdirectories. Do not create new subdir | `security/` | Authentication, authorization, encryption, vault | | `database/` | Getting started with the DB facade, migrations, seeding | | `eloquent/` | Eloquent ORM getting started, mutators, serialization | -| `digging-deeper/` | Broadcasting, cache, events, validation, localization, logging, file storage, file picker, launch, session, encryption, carbon | +| `digging-deeper/` | Broadcasting, cache, events, validation, localization, logging, file storage, file picker, launch, session, encryption, carbon, helpers, sync | | `testing/` | Getting started with testing, HTTP tests, database testing, facade fakes | | `packages/` | Magic CLI, devtools (dusk + telescope), and other first-party integrations | | `contributing/` | Contribution guide (code) and this authoring guide (docs) | @@ -110,6 +110,7 @@ When a facade or framework feature changes, apply this checklist before marking | `Schema` facade / migrations | `doc/database/migrations.md` | | `Session` facade | `doc/digging-deeper/session.md` | | `Storage` facade | `doc/digging-deeper/file-storage.md` | +| `SyncFeed` / `SyncLedger` / sync skeleton | `doc/digging-deeper/sync.md` | | `Vault` facade | `doc/security/vault.md` | | Eloquent model / ORM | `doc/eloquent/getting-started.md` | | Service providers / container | `doc/architecture/service-container.md` and `doc/getting-started/service-providers.md` | diff --git a/doc/digging-deeper/helpers.md b/doc/digging-deeper/helpers.md index 23b3c9c5..2efc843d 100644 --- a/doc/digging-deeper/helpers.md +++ b/doc/digging-deeper/helpers.md @@ -1,12 +1,13 @@ # Helpers -`Str`, `Number`, and `Arr` are static namespace helpers modelled on Laravel's Support layer, and `Cast` is magic's own, together covering locale-aware casing, locale-aware number formatting, dot-path map access, and defensive type reading for loosely-typed wire data. +`Str`, `Number`, and `Arr` are static namespace helpers modelled on Laravel's Support layer, and `Cast` is magic's own, together covering locale-aware casing, locale-aware number formatting, dot-path map access, and defensive type reading for loosely-typed wire data. `AppLifecycle` is a small, unrelated static namespace for reading the app lifecycle as a stream from code that runs before a `WidgetsBinding` necessarily exists. - [Str](#str) - [Number](#number) - [Arr](#arr) - [Cast](#cast) - [Composing Arr and Cast](#composing-arr-and-cast) +- [AppLifecycle](#applifecycle) ## Str @@ -35,6 +36,19 @@ Str.unwrap('"x', '"'); // 'x', prefix-only match still strips Str.unwrap('[value]', '[', ']'); // 'value' ``` +`Str.ascii(value)` folds Latin-1 Supplement and Latin Extended-A letters, the Romanian comma-below letters `Ș ș Ț ț`, `ẞ`, and the Angstrom sign U+212B to their plain ASCII base, for building a search key. Combining marks (U+0300-U+036F) are always dropped, whatever letter they decorate. A Latin letter outside that coverage (Vietnamese, the rest of Latin Extended-B/Additional) and every other script pass through untouched; magic folds Latin only, unlike Laravel's `Str::ascii`, which transliterates every script it has a table for and would otherwise collapse a non-Latin word to `?` or a phonetic guess. + +```dart +Str.ascii('çalışan izleyiciler'); // 'calisan izleyiciler' +Str.ascii('Ångström'); // 'Angstrom' +``` + +`Str.squish(value)` trims `value` and collapses every run of whitespace to one space, mirroring Laravel's `Str::squish`. The whitespace class is Dart's `\s` plus the two Hangul filler code points (U+3164, U+1160) a rendered blank can carry without registering as `\s`; both ends of `value` are checked against the same class, so a boundary and a middle occurrence of the same code point fold the same way. + +```dart +Str.squish(' hello world '); // 'hello world' +``` + ## Number @@ -126,3 +140,19 @@ Cast.idOrNull(true); // null final priority = Cast.intOr(Arr.get(payload, 'meta.priority'), 0); final label = Cast.stringOrNull(Arr.get(payload, 'meta.label')); ``` + + +## AppLifecycle + +`AppLifecycle.states()` exposes the app lifecycle as a `Stream`, for a reader constructed before a `WidgetsBinding` necessarily exists. A dependency built inside a service provider's `register()` runs before the app has bound anything, so reaching for `WidgetsBinding.instance` at construction time throws; `AppLifecycle.states()` defers that lookup to the moment a listener actually subscribes. + +```dart +final subscription = AppLifecycle.states().listen((state) { + if (state == AppLifecycleState.paused) Log.info('app paused'); +}); + +// Later, when done: +await subscription.cancel(); +``` + +Each subscription owns its own observer: it is added to the binding on `listen` and removed on `cancel`, so nothing outlives its reader and nothing before the first `listen` touches the binding at all. Prefer Flutter's own `AppLifecycleListener` when the reader is a widget-lifetime object; reach for `AppLifecycle.states()` only when construction has to happen before a binding is guaranteed to exist. diff --git a/doc/digging-deeper/sync.md b/doc/digging-deeper/sync.md new file mode 100644 index 00000000..0f683204 --- /dev/null +++ b/doc/digging-deeper/sync.md @@ -0,0 +1,136 @@ +# Sync + +The sync skeleton is a push-then-pull loop over one resource, for an app that keeps a local store consistent with a server across many devices. + +- [Introduction](#introduction) +- [The Wire Protocol](#the-wire-protocol) +- [Writing a Feed](#writing-a-feed) +- [Registering the Ledger's Table](#registering-the-ledgers-table) +- [The Two-Clock Rule](#the-two-clock-rule) +- [The Savepoint Note](#the-savepoint-note) +- [What Stays App-Side](#what-stays-app-side) + + +## Introduction + +`SyncFeed` runs one resource's sync: it pushes everything this device wrote since its last push, then, only when that push had no failure, pulls everything the server has past this device's last pull. It never throws, and answers a `SyncReport` (`pushed`, `adopted`, `failure`), logged through `Log.error` on the way out rather than propagated, because a sync is a convenience layered on top of a local store that already works without it. + +```dart +final SyncReport report = await ItemsSyncFeed().run(scope: 'team-42', account: userId); + +if (!report.complete) { + Log.warning('items sync did not finish: ${report.failure}'); +} +``` + +`SyncLedger` is the bookkeeping `SyncFeed` reads and writes between runs, backed by the `sync_cursors` table `CreateSyncCursorsTable` creates. + + +## The Wire Protocol + +**Push.** `POST '$resource/sync'` with `{envelopeKey: [row.data, ...]}`, batched at `SyncFeed.batchSize` (default 500) rows per request. The push response is read the same way a pull page is (`{"data": [...]}`), so a write this device lost to a conflicting write elsewhere adopts the winner immediately, without waiting for the next pull. + +**Pull.** `GET resource` with query `{scope, cursor}`, `cursor` omitted when there is none yet. The pull response is `{"data": [...], "cursor": , "has_more": bool}`. `cursor` is taken from the body rather than derived from the last row: the pair it encodes is the server's business, and a client that built one would be guessing at a tie-break it cannot see. A run walks pages until `has_more` is false or `SyncFeed.maxPages` (default 100) is reached. + +Push runs first, and the pull runs only after a push with no failure: a failed push returns before the pull ever starts. The push makes the server's answer to the pull already reflect this device's own writes, so one round leaves the device consistent. Pulling first would leave a device that had just resolved a conflict holding a stale view until the next run. + + +## Writing a Feed + +A feed subclasses `SyncFeed` and supplies three things: where its bookmarks live (`feed`), which endpoint it talks to (`resource`, `envelopeKey`), and how to read and write its own rows (`pending`, `adoptRow`). + +```dart +import 'package:magic/magic.dart'; + +class ItemsSyncFeed extends SyncFeed { + @override + String get feed => 'items'; + + @override + String get resource => 'items'; + + @override + String get envelopeKey => 'items'; + + @override + Future> pending({ + required String account, + required int sinceMillis, + required String scope, + }) async { + final List> rows = DB.select( + 'SELECT * FROM items WHERE scope = ? AND updated_at_ms > ? ORDER BY updated_at_ms ASC', + [scope, sinceMillis], + ); + + return [ + for (final row in rows) + (data: row, mark: row['updated_at_ms'] as int), + ]; + } + + @override + Future adoptRow({ + required String account, + required Map row, + }) async { + final int? remoteMark = Cast.intOrNull(row['updated_at_ms']); + if (remoteMark == null) return false; + + final List> existing = DB.select( + 'SELECT updated_at_ms FROM items WHERE id = ?', + [row['id']], + ); + + if (existing.isNotEmpty && (existing.single['updated_at_ms'] as int) >= remoteMark) { + return false; + } + + DB.statement( + 'INSERT OR REPLACE INTO items (id, scope, updated_at_ms, payload) VALUES (?, ?, ?, ?)', + [row['id'], row['scope'], remoteMark, row['payload']], + ); + + return true; + } +} +``` + +`pending` answers rows oldest first: the push advances its mark to the last row of each batch it sends, so a batch that did not end on its newest row would move the mark past rows it never sent. `adoptRow` answers `false` for a row this client cannot read (a malformed payload) rather than throwing, so one bad row does not stop the rest of a page from landing. Use `Cast.intOrNull`/`doubleOrNull`/`boolOrNull` inside `adoptRow` when the wire value's exact numeric type (`int` vs `double` on web) is not guaranteed. + + +## Registering the Ledger's Table + +`CreateSyncCursorsTable` creates the `sync_cursors` table `SyncLedger` reads and writes by default. Magic has no migration discovery, so list it explicitly alongside the app's own migrations: + +```dart +await Migrator().run([ + CreateUsersTable(), + CreateSyncCursorsTable(), +]); +``` + +The table carries five columns (`scope`, `feed`, `account`, `pull_cursor`, `push_mark`) with no `id()`, no `timestamps()`, and no unique index: `SyncLedger` upserts by delete-then-insert, so an app that already keeps a table of this shape can point a custom `SyncLedger(table: '...')` at it and skip this migration. + + +## The Two-Clock Rule + +A `SyncFeed` run tracks two clocks, and only one of them advances locally. The push mark is this device's own `updated_at`, epoch millis, deciding which rows are worth sending next time. The pull cursor is the server's, opaque text handed straight back unread. + +Only the push mark advances here; the pull cursor never substitutes for it. A row adopted from the server carries the ORIGINATING device's clock, so advancing this device's push mark to that value would step over a local row written earlier and never sent. The cost is small: an adopted row is echoed back to the server exactly once, on the next run, where the server's own `>=` comparison rejects it and the mark then covers it. + + +## The Savepoint Note + +`SyncLedger.write` issues its delete-then-insert inside a `SAVEPOINT`/`RELEASE` pair rather than `DB.transaction`, because a bare `BEGIN` does not nest and a feed's own sync run may already be executing inside a caller's outer transaction. A savepoint does nest: opened with no transaction open it behaves like one, and opened inside an existing transaction it joins that transaction and rolls back with it. + +That means a caller whose outer transaction fails after this savepoint released loses the bookmark write too. The next run simply resends the tail the server's own `>=` check already absorbs: one extra round trip, never a duplicate write. + + +## What Stays App-Side + +The skeleton is deliberately narrow. Three things stay the app's own responsibility: + +- **Scope derivation.** `scope` is an opaque string `SyncFeed.run` passes straight through to the ledger and the wire calls; how an app derives it (a team id, a workspace id) is not this package's concern. +- **Salt.** Any value mixed into a feed's own local identifiers or cache keys beyond `scope` and `account` is the app's own. +- **Orchestration.** When and how often a feed runs (on a timer, on reconnect, on app resume) is left to the app; `SyncFeed.run` answers one call, it does not schedule itself. diff --git a/lib/magic.dart b/lib/magic.dart index a987a010..5081971f 100644 --- a/lib/magic.dart +++ b/lib/magic.dart @@ -54,6 +54,12 @@ export 'src/support/number.dart'; export 'src/support/str.dart'; export 'src/support/cast.dart'; export 'src/support/arr.dart'; +export 'src/support/app_lifecycle.dart'; + +// Sync +export 'src/sync/sync_feed.dart'; +export 'src/sync/sync_ledger.dart'; +export 'src/sync/create_sync_cursors_table.dart'; // Routing export 'src/routing/magic_platform_page.dart'; diff --git a/lib/src/network/network_service_provider.dart b/lib/src/network/network_service_provider.dart index 40101ebc..f719bb96 100644 --- a/lib/src/network/network_service_provider.dart +++ b/lib/src/network/network_service_provider.dart @@ -4,6 +4,7 @@ import 'package:flutter/foundation.dart' import '../facades/config.dart'; import '../network/drivers/dio_network_driver.dart'; import '../support/service_provider.dart'; +import '../support/str.dart'; /// The Network Service Provider. /// @@ -82,13 +83,14 @@ class NetworkServiceProvider extends ServiceProvider { /// connect the two. Measured against the same `HttpClient` path Dio's IO /// adapter uses: `Invalid HTTP header field value`. /// - /// Accented Latin letters are folded to their base letter rather than - /// dropped, because dropping leaves a mangled word where the adopter would - /// have picked a plain-ASCII name. The table covers every letter in Latin-1 - /// Supplement and Latin Extended-A, so Turkish, German, French, Spanish, - /// Nordic, Polish, Czech and Dutch names survive legibly, and a test walks - /// both ranges so the claim checks itself. A script with no Latin base (CJK, - /// Arabic, Cyrillic) has nothing to fold to and is dropped. + /// [Str.ascii] folds accented Latin letters to their base letter rather + /// than dropping them, because dropping leaves a mangled word where the + /// adopter would have picked a plain-ASCII name. It covers every letter in + /// Latin-1 Supplement and Latin Extended-A, so Turkish, German, French, + /// Spanish, Nordic, Polish, Czech and Dutch names survive legibly, and a + /// test walks both ranges so the claim checks itself. A script with no + /// Latin base (CJK, Arabic, Cyrillic) has nothing to fold to and is left + /// as-is by [Str.ascii], then dropped below by the printable-ASCII filter. /// /// Everything still outside printable ASCII goes, which also closes the /// injection shape: a name carrying a carriage return or newline cannot split @@ -99,15 +101,10 @@ class NetworkServiceProvider extends ServiceProvider { static String _headerSafeAppName(String name) { final StringBuffer folded = StringBuffer(); - for (final int rune in name.runes) { - final String? base = _latinFolding[rune]; - - if (base != null) { - folded.write(base); - continue; - } - - // Printable ASCII only: 0x20 (space) through 0x7E (tilde). + for (final int rune in Str.ascii(name).runes) { + // Printable ASCII only: 0x20 (space) through 0x7E (tilde). [Str.ascii] + // folds Latin diacritics but leaves non-Latin scripts and control code + // points in place, so this filter still has to run. if (rune >= 0x20 && rune <= 0x7E) folded.write(String.fromCharCode(rune)); } @@ -119,80 +116,6 @@ class NetworkServiceProvider extends ServiceProvider { return cleaned.isEmpty ? _fallbackAppName : cleaned; } - /// Accented Latin letters to their base letter, keyed by rune. - /// - /// Built from grouped strings rather than entry by entry, so the coverage of - /// each base letter is readable at a glance and a missing accent is visible - /// rather than buried in sixty lines of map literal. - static final Map _latinFolding = _buildFolding({ - 'A': 'ÀÁÂÃÄÅĀĂĄ', - 'a': 'àáâãäåāăąª', - 'C': 'ÇĆĈĊČ', - 'c': 'çćĉċč', - 'D': 'ÐĎĐ', - 'd': 'ðďđ', - 'E': 'ÈÉÊËĒĔĖĘĚ', - 'e': 'èéêëēĕėęě', - 'G': 'ĜĞĠĢ', - 'g': 'ĝğġģ', - 'H': 'ĤĦ', - 'h': 'ĥħ', - 'I': 'ÌÍÎÏĨĪĬĮİ', - 'i': 'ìíîïĩīĭįı', - 'J': 'Ĵ', - 'j': 'ĵ', - 'K': 'Ķ', - 'k': 'ķĸ', - 'L': 'ĹĻĽĿŁ', - 'l': 'ĺļľŀł', - 'N': 'ÑŃŅŇŊ', - 'n': 'ñńņňʼnŋ', - 'O': 'ÒÓÔÕÖØŌŎŐ', - 'o': 'òóôõöøōŏőº', - 'R': 'ŔŖŘ', - 'r': 'ŕŗř', - 'S': 'ŚŜŞŠ', - 's': 'śŝşšſ', - 'T': 'ŢŤŦ', - 't': 'ţťŧ', - 'U': 'ÙÚÛÜŨŪŬŮŰŲ', - 'u': 'ùúûüũūŭůűųµ', - 'W': 'Ŵ', - 'w': 'ŵ', - 'Y': 'ÝŶŸ', - 'y': 'ýÿŷ', - 'Z': 'ŹŻŽ', - 'z': 'źżž', - 'AE': 'ÆǼ', - 'ae': 'æǽ', - 'OE': 'Œ', - 'oe': 'œ', - 'ss': 'ß', - 'TH': 'Þ', - 'th': 'þ', - // The two ligatures, the only entries whose base is two letters and so the - // only ones that cannot join a group above. The kra, the eng and the long s - // were missing for the same reason and are folded into `k`, `N`/`n` and `s` - // rather than added here: a repeated key in a Dart map literal takes the - // LAST value, so a fresh `'n': 'ŋ'` would have silently replaced the five - // accented n's above it. - 'IJ': 'IJ', - 'ij': 'ij', - }); - - /// Inverts the grouped folding table into a rune-keyed lookup. - static Map _buildFolding(Map groups) { - final Map table = {}; - - groups.forEach((String base, String accented) { - for (final int rune in accented.runes) { - table[rune] = base; - } - }); - - return table; - } - /// The platform name a server can read, in the casing Apple and Google use. static String _platformName() { return switch (defaultTargetPlatform) { diff --git a/lib/src/support/app_lifecycle.dart b/lib/src/support/app_lifecycle.dart new file mode 100644 index 00000000..4259fabc --- /dev/null +++ b/lib/src/support/app_lifecycle.dart @@ -0,0 +1,53 @@ +import 'dart:async'; + +import 'package:flutter/widgets.dart'; + +/// Forwards every [WidgetsBinding] lifecycle callback to one stream listener. +/// +/// Private, and the only instances are the ones [AppLifecycle.states] holds: +/// the binding keeps its observers for the process's life, so an object that +/// added itself and never came off is a leak that keeps answering. +class _LifecycleObserver with WidgetsBindingObserver { + _LifecycleObserver(this._report); + + final void Function(AppLifecycleState state) _report; + + @override + void didChangeAppLifecycleState(AppLifecycleState state) => _report(state); +} + +/// The app lifecycle exposed as a stream, for readers constructed before a +/// [WidgetsBinding] necessarily exists. +/// +/// A dependency built inside a service provider's `register()` runs before +/// the app has bound anything, so reaching for [WidgetsBinding.instance] at +/// construction time throws. [states] defers that lookup to the moment a +/// listener actually subscribes, and each subscription owns its own observer: +/// the observer goes on the binding at `listen` and comes off at `cancel`, so +/// nothing outlives its reader and nothing before the first listen touches +/// the binding at all. +/// +/// Prefer Flutter's own [AppLifecycleListener] when the reader is a +/// widget-lifetime object: it binds to [WidgetsBinding] at construction, +/// which is exactly right once a binding is guaranteed to exist, and it +/// carries the metrics/exit-request callbacks this class deliberately does +/// not. Reach for [AppLifecycle.states] only when construction has to happen +/// before that guarantee holds. +abstract final class AppLifecycle { + /// The app lifecycle as a stream, reaching [WidgetsBinding] only on listen. + /// + /// Built with [Stream.multi] rather than a [StreamController] of this + /// method's own: a controller built here has nobody to close it, so its + /// observer would leave the binding only if some caller remembered to close + /// a sink it never saw. This shape puts the removal on the cancel, which + /// each listener's own teardown performs. + static Stream states() => Stream.multi(( + MultiStreamController listener, + ) { + final _LifecycleObserver observer = _LifecycleObserver(listener.add); + + WidgetsBinding.instance.addObserver(observer); + + listener.onCancel = () => WidgetsBinding.instance.removeObserver(observer); + }); +} diff --git a/lib/src/support/str.dart b/lib/src/support/str.dart index e4295b6c..2e0b4916 100644 --- a/lib/src/support/str.dart +++ b/lib/src/support/str.dart @@ -100,4 +100,144 @@ abstract final class Str { return result; } + + /// [value] with every Latin letter [_latinFolding] covers (Latin-1 + /// Supplement, Latin Extended-A, the Romanian comma-below letters `Ș ș Ț + /// ț`, `ẞ`, and the Angstrom sign U+212B) folded to its plain ASCII base, + /// case PRESERVED; every other rune left untouched, including a Latin + /// letter outside that coverage (Vietnamese and the rest of Latin + /// Extended-B/Additional) and every other script (Cyrillic, Greek, + /// Arabic, CJK, ...). Combining marks (U+0300-U+036F) are always dropped, + /// whatever letter they decorate, so decomposed non-Latin text loses its + /// marks here too. + /// + /// Diverges from Laravel's `Str::ascii`, which transliterates every + /// script it has a table for: magic folds Latin only, so a search key + /// built from [ascii] keeps a non-Latin word searchable by its own exact + /// letters instead of collapsing it to `?` or a phonetic guess. + static String ascii(String value) { + final StringBuffer folded = StringBuffer(); + + for (final int rune in value.runes) { + if (rune >= 0x300 && rune <= 0x36F) continue; + + folded.write(_latinFolding[rune] ?? String.fromCharCode(rune)); + } + + return folded.toString(); + } + + /// [value] trimmed, with every run of whitespace collapsed to one space. + /// + /// Mirrors Laravel's `Str::squish`: the whitespace class is Dart's `\s` + /// (every Unicode space separator and line terminator) plus the two + /// Hangul filler code points (U+3164, U+1160) a rendered blank can carry + /// without registering as `\s`. + /// + /// Trims with the same regex class rather than `String.trim()`, which + /// follows Unicode's broader `White_Space` property and additionally + /// strips U+0085 (NEL): a boundary and a middle occurrence of the same + /// code point must fold the same way, or the two ends of [value] disagree + /// about what counts as whitespace. + static String squish(String value) { + return value + .replaceAll(_leadingOrTrailingSquishable, '') + .replaceAll(_squishable, ' '); + } + + /// The whitespace class [squish] collapses. + static final RegExp _squishable = RegExp('[\\s\u3164\u1160]+'); + + /// [_squishable]'s pattern anchored to either end, for trimming. + static final RegExp _leadingOrTrailingSquishable = RegExp( + '^[\\s\u3164\u1160]+|[\\s\u3164\u1160]+\$', + ); + + /// Accented Latin letters (and a handful of Latin letters with no accent + /// but no ASCII base of their own) to their plain letter, keyed by rune, + /// case preserved. + /// + /// Built from grouped strings rather than entry by entry, so the coverage + /// of each base letter is readable at a glance and a missing accent is + /// visible rather than buried in sixty lines of map literal. Covers every + /// letter in Latin-1 Supplement and Latin Extended-A, plus the Romanian + /// letters with a comma below (`Ș ș Ț ț`, Latin Extended-B) and `ẞ` + /// (U+1E9E, capital sharp s). The Angstrom sign (U+212B) is added below + /// rather than into the `A` group: a font renders it identically to the + /// `Å` (U+00C5) already there, and the two glyphs side by side in one + /// string would read as an accidental duplicate. + static final Map _latinFolding = + _buildFolding({ + 'A': 'ÀÁÂÃÄÅĀĂĄ', + 'a': 'àáâãäåāăąª', + 'C': 'ÇĆĈĊČ', + 'c': 'çćĉċč', + 'D': 'ÐĎĐ', + 'd': 'ðďđ', + 'E': 'ÈÉÊËĒĔĖĘĚ', + 'e': 'èéêëēĕėęě', + 'G': 'ĜĞĠĢ', + 'g': 'ĝğġģ', + 'H': 'ĤĦ', + 'h': 'ĥħ', + 'I': 'ÌÍÎÏĨĪĬĮİ', + 'i': 'ìíîïĩīĭįı', + 'J': 'Ĵ', + 'j': 'ĵ', + 'K': 'Ķ', + 'k': 'ķĸ', + 'L': 'ĹĻĽĿŁ', + 'l': 'ĺļľŀł', + 'N': 'ÑŃŅŇŊ', + 'n': 'ñńņňʼnŋ', + 'O': 'ÒÓÔÕÖØŌŎŐ', + 'o': 'òóôõöøōŏőº', + 'R': 'ŔŖŘ', + 'r': 'ŕŗř', + 'S': 'ŚŜŞŠȘ', + 's': 'śŝşšſș', + 'T': 'ŢŤŦȚ', + 't': 'ţťŧț', + 'U': 'ÙÚÛÜŨŪŬŮŰŲ', + 'u': 'ùúûüũūŭůűųµ', + 'W': 'Ŵ', + 'w': 'ŵ', + 'Y': 'ÝŶŸ', + 'y': 'ýÿŷ', + 'Z': 'ŹŻŽ', + 'z': 'źżž', + 'AE': 'ÆǼ', + 'ae': 'æǽ', + 'OE': 'Œ', + 'oe': 'œ', + 'SS': 'ẞ', + 'ss': 'ß', + 'TH': 'Þ', + 'th': 'þ', + // The two ligatures, the only entries whose base is two letters and so + // the only ones that cannot join a group above. The kra, the eng and + // the long s were missing for the same reason and are folded into `k`, + // `N`/`n` and `s` rather than added here: a repeated key in a Dart map + // literal takes the LAST value, so a fresh `'n': 'ŋ'` would have + // silently replaced the five accented n's above it. + 'IJ': 'IJ', + 'ij': 'ij', + }) + // The Angstrom sign (U+212B): canonically equivalent to Å but a + // distinct code point, kept out of the `A` group string above (see + // the doc comment) because a font renders the two identically. + ..[0x212B] = 'A'; + + /// Inverts the grouped folding table into a rune-keyed lookup. + static Map _buildFolding(Map groups) { + final Map table = {}; + + groups.forEach((String base, String accented) { + for (final int rune in accented.runes) { + table[rune] = base; + } + }); + + return table; + } } diff --git a/lib/src/sync/create_sync_cursors_table.dart b/lib/src/sync/create_sync_cursors_table.dart new file mode 100644 index 00000000..b8652cb1 --- /dev/null +++ b/lib/src/sync/create_sync_cursors_table.dart @@ -0,0 +1,34 @@ +import '../database/migrations/migration.dart'; +import '../database/schema/blueprint.dart'; +import '../facades/schema.dart'; + +/// Creates the `sync_cursors` table `SyncLedger` reads and writes. +/// +/// No `id()`, no `timestamps()`, no unique index: `SyncLedger` needs only the +/// five columns and upserts by delete-then-insert, so an app that already +/// keeps a table of this shape can point `SyncLedger` at it and skip this +/// migration. +/// +/// Magic has no migration discovery (`Migrator.run`): an app that wants +/// this table lists it explicitly in its own `Migrator().run([...])` call +/// alongside its other migrations. +class CreateSyncCursorsTable extends Migration { + @override + String get name => '2026_09_26_000000_create_sync_cursors_table'; + + @override + void up() { + Schema.create('sync_cursors', (Blueprint table) { + table.string('scope'); + table.string('feed'); + table.string('account'); + table.string('pull_cursor').nullable(); + table.integer('push_mark'); + }); + } + + @override + void down() { + Schema.dropIfExists('sync_cursors'); + } +} diff --git a/lib/src/sync/sync_feed.dart b/lib/src/sync/sync_feed.dart new file mode 100644 index 00000000..2639f305 --- /dev/null +++ b/lib/src/sync/sync_feed.dart @@ -0,0 +1,356 @@ +import 'package:flutter/foundation.dart'; + +import '../facades/http.dart'; +import '../facades/log.dart'; +import '../network/magic_response.dart'; +import 'sync_ledger.dart'; + +/// One local row a push owes the server: the wire payload and the local +/// clock that decides how far the push mark may advance. +/// +/// The two are separate because they answer different questions. [data] is +/// whatever the feed's own endpoint validates, and [mark] is always this +/// device's epoch millis for the row, which is the unit a feed's own +/// `*ChangedSince` reader compares against. +typedef SyncPushRow = ({Map data, int mark}); + +/// What one sync run did, in the terms a caller can act on. +class SyncReport { + /// Creates a report. [SyncFeed.run] builds one per run; an app's own + /// wrapper may build one for a run it declined to start. + const SyncReport({required this.pushed, required this.adopted, this.failure}); + + /// Rows this device sent that the server accepted as the newer state. + final int pushed; + + /// Rows this device adopted from another device. + final int adopted; + + /// Why the run stopped early, or null when it finished. + /// + /// A sync is a background convenience, so a failure is reported and never + /// thrown. + final String? failure; + + /// Whether the run finished both halves. + bool get complete => failure == null; +} + +/// The push-then-pull skeleton every feed's sync runs, over one resource. +/// +/// ### Wire protocol +/// +/// Push request: `POST '$resource/sync'` with `{envelopeKey: [row.data, ...]}`. +/// Push response: whatever the server answers, read the same way a pull page +/// is (`{"data": [...]}`) so a write this device lost adopts the winner here +/// rather than waiting for the next pull. +/// +/// Pull request: `GET resource` with query `{scope, cursor}`, `cursor` +/// omitted when there is none yet. +/// Pull response: `{"data": [...], "cursor": , "has_more": +/// bool}`. `cursor` is taken from the body rather than derived from the last +/// row: the pair it encodes is the server's business, and a client that +/// built one would be guessing at a tie-break it cannot see. +/// +/// ### Two clocks, and only one of them advances here +/// +/// The push mark is a local `updated_at`, this device's own epoch millis, +/// and decides which rows are worth sending. The pull cursor is the +/// server's, opaque text handed straight back. Only the push advances the +/// mark, never the pull: a row adopted from the server carries the +/// ORIGINATING device's clock, so advancing the mark to it would step over a +/// local row written earlier and never sent. The cost is that an adopted row +/// is echoed back to the server exactly once, on the next run, where the +/// server's own `>=` rejects it and the mark then covers it. +/// +/// ### Push first, then pull +/// +/// The pull runs only after a push with no failure. The push makes the +/// server's answer to the pull already reflect this device's writes, so +/// one round leaves the device consistent; pulling first would leave a +/// device that had just resolved a conflict holding a stale view until the +/// next run. A failed push returns before the pull ever starts, since a +/// pull cursor advanced on top of an unsent push would let the pull outrun +/// what this device still owes the server. +abstract class SyncFeed { + /// Creates the sync over [ledger]. + const SyncFeed({this.ledger = const SyncLedger()}); + + /// Where this feed's bookmarks live. + @protected + final SyncLedger ledger; + + /// The `sync_cursors.feed` literal this feed's bookmarks live under. + @protected + String get feed; + + /// The API path, under whatever prefix the network driver's base URL + /// carries. A missing leading slash is normalised by [run]; declare it + /// with or without one. + @protected + String get resource; + + /// The key the push body wraps its rows in. + @protected + String get envelopeKey; + + /// How many rows one push batch carries. + int get batchSize => 500; + + /// How many pull pages one run will walk before giving up. + int get maxPages => 100; + + /// Everything written locally after [sinceMillis], oldest first, as wire + /// rows. + /// + /// Oldest first is required rather than tidy: the push advances its mark + /// to the last row of each batch it sends, so a batch that did not end on + /// its newest row would move the mark past rows it never sent. + @protected + Future> pending({ + required String account, + required int sinceMillis, + required String scope, + }); + + /// Writes one row the server answered with, and says whether it was newer + /// than what this device already held. + /// + /// A row this client cannot read answers false rather than throwing: one + /// such row must not stop the rest of a page from landing. + @protected + Future adoptRow({ + required String account, + required Map row, + }); + + /// Runs both halves for [scope] and [account]. + /// + /// Never throws. An exception from [pending], [adoptRow] or the ledger + /// arrives as a [SyncReport] carrying [SyncReport.failure] instead, + /// logged with [Log.error]: every feed here is a convenience on top of a + /// local store that already works without it. + Future run({ + required String scope, + required String account, + }) async { + try { + // The raw `Http` methods hand the path to the driver untouched, and a + // base URL with no trailing slash plus a resource with no leading one + // joins into one word (`/api/v1` + `items` is `/api/v1items`, a path + // that does not exist); prefixing it here once covers both halves. + final String path = resource.startsWith('/') ? resource : '/$resource'; + + final SyncBookmarks bookmarks = ledger.read(scope: scope, feed: feed); + + final _PushResult push = await _push( + path: path, + scope: scope, + account: account, + mark: bookmarks.pushMark, + ); + + if (push.failure != null) { + // The mark still advances over whatever landed before the failure, + // so a retry resends the tail rather than the whole store. The pull + // cursor is left exactly as it was: nothing was pulled this run. + await ledger.write( + scope: scope, + feed: feed, + account: account, + pullCursor: bookmarks.pullCursor, + pushMark: push.mark, + ); + + return SyncReport(pushed: push.sent, adopted: 0, failure: push.failure); + } + + final _PullResult pull = await _pull( + path: path, + scope: scope, + account: account, + cursor: bookmarks.pullCursor, + ); + + await ledger.write( + scope: scope, + feed: feed, + account: account, + pullCursor: pull.cursor, + pushMark: push.mark, + ); + + return SyncReport( + pushed: push.sent, + adopted: pull.adopted, + failure: pull.failure, + ); + } catch (error, stackTrace) { + Log.error('sync run failed for feed "$feed"', { + 'error': error.toString(), + 'stackTrace': stackTrace.toString(), + }); + + return SyncReport(pushed: 0, adopted: 0, failure: 'sync failed: $error'); + } + } + + /// Sends everything written locally since [mark], in [batchSize] slices. + /// + /// An empty set sends nothing rather than an empty batch: every one of + /// these endpoints validates its array as non-empty, so an empty push + /// would be a 422 and not a no-op. + Future<_PushResult> _push({ + required String path, + required String scope, + required String account, + required int mark, + }) async { + final List changed = await pending( + account: account, + sinceMillis: mark, + scope: scope, + ); + + int reached = mark; + int sent = 0; + + for (int start = 0; start < changed.length; start += batchSize) { + final int end = start + batchSize > changed.length + ? changed.length + : start + batchSize; + final List slice = changed.sublist(start, end); + + final MagicResponse response = await Http.post( + '$path/sync', + data: { + envelopeKey: slice.map((SyncPushRow row) => row.data).toList(), + }, + ); + + if (!response.successful) { + return _PushResult( + sent: sent, + mark: reached, + failure: _failureOf(response, 'push'), + ); + } + + // Only past a 2xx: the batch is all or nothing on the server, so a + // mark advanced on a refused batch would lose every row in it. + reached = slice.last.mark; + sent += slice.length; + + await _adopt(account: account, body: response.data); + } + + return _PushResult(sent: sent, mark: reached, failure: null); + } + + /// Walks every page the server has past [cursor], writing each row. + Future<_PullResult> _pull({ + required String path, + required String scope, + required String account, + required String? cursor, + }) async { + String? at = cursor; + int adopted = 0; + + for (int page = 0; page < maxPages; page++) { + final MagicResponse response = await Http.get( + path, + query: {'scope': scope, 'cursor': ?at}, + ); + + if (!response.successful) { + return _PullResult( + cursor: at, + adopted: adopted, + failure: _failureOf(response, 'pull'), + ); + } + + final Map? body = response.data is Map + ? response.data as Map + : null; + + if (body == null) { + return _PullResult( + cursor: at, + adopted: adopted, + failure: 'pull answered a body that is not an object', + ); + } + + adopted += await _adopt(account: account, body: body); + + final Object? next = body['cursor']; + at = next is String ? next : at; + + if (body['has_more'] != true) { + return _PullResult(cursor: at, adopted: adopted, failure: null); + } + } + + return _PullResult( + cursor: at, + adopted: adopted, + failure: 'pull did not finish in $maxPages pages', + ); + } + + /// Writes every row in a `{"data": [...]}` body, answering how many were + /// newer than what this device already held. + Future _adopt({required String account, required Object? body}) async { + if (body is! Map) return 0; + + final Object? rows = body['data']; + + if (rows is! List) return 0; + + int adopted = 0; + + for (final Object? row in rows) { + if (row is! Map) continue; + + if (await adoptRow(account: account, row: row)) adopted++; + } + + return adopted; + } + + /// A failure line carrying the status and nothing from the body. + /// + /// The body is not interpolated on purpose: a failure string is the kind + /// of value that ends up in a log, and a 4xx from a proxy in front of this + /// API can echo the request. + String _failureOf(MagicResponse response, String half) => + '$half failed with HTTP ${response.statusCode}'; +} + +/// What one push half produced. +class _PushResult { + const _PushResult({ + required this.sent, + required this.mark, + required this.failure, + }); + + final int sent; + final int mark; + final String? failure; +} + +/// What one pull half produced. +class _PullResult { + const _PullResult({ + required this.cursor, + required this.adopted, + required this.failure, + }); + + final String? cursor; + final int adopted; + final String? failure; +} diff --git a/lib/src/sync/sync_ledger.dart b/lib/src/sync/sync_ledger.dart new file mode 100644 index 00000000..00b914ca --- /dev/null +++ b/lib/src/sync/sync_ledger.dart @@ -0,0 +1,107 @@ +import '../database/database_manager.dart'; +import '../facades/db.dart'; + +/// Where a `(scope, feed)` pair had got to in each sync direction. +/// +/// The two fields are two different clocks and neither stands in for the +/// other. [pullCursor] is whatever the server handed back last time, kept as +/// the opaque text it arrived as. [pushMark] is this device's own epoch +/// millis for the newest row it has already sent, in the unit a feed's own +/// `*ChangedSince` reader compares against. +typedef SyncBookmarks = ({String? pullCursor, int pushMark}); + +/// Reads and writes one feed's sync bookmarks. +/// +/// [table] carries more than one feed under a shared scope, so every read and +/// write here is keyed by `(scope, feed)` together; reading by [table] and +/// `scope` alone would answer an arbitrary row the moment a second feed wrote +/// under it. +class SyncLedger { + /// Creates a ledger over [table]. + /// + /// The default is the table `CreateSyncCursorsTable` creates. An app with + /// its own table of the same five columns passes its name instead. + const SyncLedger({this.table = 'sync_cursors'}); + + /// The table both [read] and [write] operate on. + final String table; + + /// The savepoint [write] wraps its delete-then-insert in. + static const String _savepoint = 'magic_sync_ledger_write'; + + /// Answers the bookmarks [scope] and [feed] had got to. + /// + /// An unknown pair answers `(pullCursor: null, pushMark: 0)`: a null cursor + /// tells the next pull to ask for everything, and a zero mark tells the + /// next push the same in the other direction, because no local write ever + /// carries a non-positive epoch. + SyncBookmarks read({required String scope, required String feed}) { + final List> rows = DB.select( + 'SELECT pull_cursor, push_mark FROM $table WHERE scope = ? AND feed = ?', + [scope, feed], + ); + + if (rows.isEmpty) return (pullCursor: null, pushMark: 0); + + final Map row = rows.single; + + return ( + pullCursor: row['pull_cursor'] as String?, + pushMark: row['push_mark'] as int, + ); + } + + /// Records both bookmarks for [scope] and [feed], replacing any row that + /// pair already held. + /// + /// [table] carries no unique constraint, so a plain `INSERT` would + /// accumulate rows and a later [read] would pick an arbitrary one among + /// them; the delete-then-insert here is what makes this an upsert. + /// + /// Issued through [DB.statement] inside a `SAVEPOINT`/`RELEASE` rather than + /// [DB.transaction], because a `BEGIN` does not nest (`DB.transaction`) and + /// a feed's own sync run may already be executing inside a caller's own + /// transaction. + /// A savepoint does nest: opened with no transaction open it behaves like + /// one, and opened inside an existing transaction it JOINS that + /// transaction and is rolled back with it. So a caller whose outer + /// transaction fails after this savepoint released loses the bookmark + /// write too, and the next run resends the tail the server's own `>=` + /// already absorbs: one extra round trip, never a duplicate write. + Future write({ + required String scope, + required String feed, + required String account, + required String? pullCursor, + required int pushMark, + }) async { + DB.statement('SAVEPOINT $_savepoint'); + + try { + DB.statement('DELETE FROM $table WHERE scope = ? AND feed = ?', [ + scope, + feed, + ]); + DB.statement( + 'INSERT INTO $table (scope, feed, account, pull_cursor, push_mark) ' + 'VALUES (?, ?, ?, ?, ?)', + [scope, feed, account, pullCursor, pushMark], + ); + + DB.statement('RELEASE $_savepoint'); + } catch (_) { + // `ROLLBACK TO` leaves the savepoint in place; the `RELEASE` after it + // is what actually discards it. Guarded on `autocommit` because SQLite + // itself already rolls back the whole transaction on some errors + // (SQLITE_FULL, IOERR, NOMEM), taking this savepoint with it; issuing + // `ROLLBACK TO` against a savepoint that no longer exists throws "no + // such savepoint" and replaces the original error with that one. + if (!DatabaseManager().connection.autocommit) { + DB.statement('ROLLBACK TO $_savepoint'); + DB.statement('RELEASE $_savepoint'); + } + + rethrow; + } + } +} diff --git a/skills/magic-framework/SKILL.md b/skills/magic-framework/SKILL.md index f1bfd239..31d16903 100644 --- a/skills/magic-framework/SKILL.md +++ b/skills/magic-framework/SKILL.md @@ -2,10 +2,10 @@ name: magic-framework description: "Write correct, idiomatic code in a Flutter app that depends on the `magic` framework (Laravel-inspired: IoC container, 18 facades, Eloquent-style ORM, service providers, reactive controllers, GoRouter routing, validation, auth, broadcasting). Use whenever code imports `package:magic/magic.dart` or `package:magic/testing.dart`, or the work touches Magic.init, MagicApp, a facade (Auth/Http/Cache/DB/Echo/Event/Gate/Config/Lang/Launch/Log/Pick/MagicRoute/Schema/Session/Storage/Vault/Crypt), a Model, MagicController, a MagicView, MagicFormData, FormRequest, a ServiceProvider, a migration, or the artisan make:* CLI. UI styling is Wind (separate wind-ui skill). Do NOT use for plain Flutter or Wind-only work with no magic import." when_to_use: "Use proactively when editing or scaffolding a magic app: Magic.init / a facade / a Model / a MagicController or MagicView / a form (MagicFormData, FormRequest, Validator) / a ServiceProvider / a route or MagicMiddleware / a migration / MagicStateMixin + RxStatus + fetchList / Session flash + old() + trans() / testing with MagicTest + Http.fake/Auth.fake / the artisan make:* CLI / the magic_deeplink, magic_notifications, magic_social_auth, magic_starter, magic_payments, or magic_devtools plugins. Trigger even when the user does not say the word 'magic'. Do NOT trigger for plain Flutter or Wind-only UI with no package:magic import." -version: 0.1.48 +version: 0.1.49 --- - + # Magic Framework diff --git a/skills/magic-framework/references/plugin-starter.md b/skills/magic-framework/references/plugin-starter.md index 71738cde..dffe0281 100644 --- a/skills/magic-framework/references/plugin-starter.md +++ b/skills/magic-framework/references/plugin-starter.md @@ -1,4 +1,4 @@ - + # magic_starter Plugin @@ -19,6 +19,7 @@ Versions left the alpha rail at 0.0.27: `0.0.1-alpha.26` is followed by `0.0.27` - [Plan upgrade wall](#plan-upgrade-wall) - [Page geometry](#page-geometry) - [Controllers](#controllers) +- [Guest Claim](#guest-claim) - [Layouts & Notification Integration](#layouts--notification-integration) - [Gate Abilities](#gate-abilities) - [Gotchas](#gotchas) @@ -688,6 +689,26 @@ await MagicStarterAuthController.instance.logout(); The preference matrix is `NotificationPreferencesController` in `magic_notifications` now; see `plugin-notifications.md`. +## Guest Claim + +**Unreleased (next release): not shipped by the `magic_starter v0.0.36` stamped above.** + +With `features.guest_auth` on, `MagicStarterGuestClaim` moves what a guest accumulated onto the account they sign in to next, wired by `MagicStarterServiceProvider` to magic's auth events. `GuestClaimOutcome` is `none` (nothing settled: signed out, still the guest, no record, a keychain refusal, or an unreachable API; every one of these keeps the record for the next attempt), `claimed` (the server accepted the claim), `refused` (the server's 422, final, no retry), or `promoted` (the signed-in account IS the recorded guest, promoted in place by registration; nothing to move). + +1. **Guest sign-in** (`AuthLogin` for a user whose `is_guest` is `true`): the guest's bearer token and user id are written to `Vault` under `MagicStarterGuestClaim.tokenKey` (`guest_claim_token`) and `MagicStarterGuestClaim.userKey` (`guest_claim_user`), awaited. +2. **Sign-in or restore of a real account** (`AuthLogin` for a non-guest, or `AuthRestored`): `MagicStarterGuestClaim.instance.claimIfPending()` runs unawaited, posting `POST /auth/guest/claim` with `{'guest_token': ...}` authenticated as the target account. A claim already in flight hands every caller the same future. +3. **Sign-out** (`AuthLogout`): `MagicStarterGuestClaim.forget()` deletes the record before `Auth.logout()` returns, so the next person on the device cannot claim the previous viewer's rows. + +Every outcome but `none` reaches the host through `onGuestClaimed`, set via `MagicStarter.bootstrap(onGuestClaimed:)` or `MagicStarter.useGuestClaimed(callback)`: + +```dart +MagicStarter.useGuestClaimed((outcome) async { + if (outcome == GuestClaimOutcome.claimed) await Library.refresh(); +}); +``` + +The claim does not ride the `Http` facade or its interceptors: it posts on a bare `DioNetworkDriver` built straight from `network.drivers.api`, since its body is the guest's still-live bearer token and the facade driver is what `magic_devtools` records request bodies from in debug/profile builds. Full contract: `doc/basics/authentication.md#guest-claim` (magic_starter's own doc). + ## Layouts & Notification Integration The app layout (`layout.app`) auto-manages notification polling: diff --git a/skills/magic-framework/references/secondary-systems.md b/skills/magic-framework/references/secondary-systems.md index 306b4440..980484f4 100644 --- a/skills/magic-framework/references/secondary-systems.md +++ b/skills/magic-framework/references/secondary-systems.md @@ -18,6 +18,7 @@ Complete reference for Magic framework utility systems: Cache, Events, Logging, - [Launch (URL Launcher)](#launch-url-launcher) - [Pick (File & Image Selection)](#pick-file--image-selection) - [Broadcasting](#broadcasting) +- [Sync](#sync) - [Key Gotchas](#key-gotchas) ## Support Helpers (Number, Str, Arr, Cast) @@ -45,6 +46,8 @@ Locale-aware casing. `String.toUpperCase()`/`toLowerCase()` get Turkish/Azerbaij | `Str.upper(value, {locale})` / `Str.lower(value, {locale})` | Dotted-i aware casing. `İ` maps to a plain `i` in EVERY locale (not only tr/az) to avoid the web's combining-dot lowercase. | | `Str.initials(value, {limit, capitalize, locale})` | First letter of each whitespace-separated word; `limit` keeps only the first N words. | | `Str.unwrap(value, before, [after])` | Strips `before` from the start and `after` (default `before`) from the end, each checked/stripped independently (Laravel's `Str::unwrap`); a prefix-only match (`'"x'`) still loses the leading quote. | +| `Str.ascii(value)` | Folds Latin-1 Supplement, Latin Extended-A, the Romanian comma-below letters, `ẞ`, and U+212B to their plain ASCII base, for a search key. A Latin letter outside that coverage (Vietnamese, ...) and every other script pass through untouched; combining marks (U+0300-U+036F) are always dropped. Diverging from Laravel's `Str::ascii`, which transliterates every script it has a table for. | +| `Str.squish(value)` | Trims and collapses every run of whitespace to one space (Laravel's `Str::squish`), over Dart's `\s` class plus two Hangul filler code points. | ### Arr @@ -80,6 +83,16 @@ Total, throw-free readers for a loosely-typed wire value (a nested-map field, no | `Env.filled(key, fallback)` | Treats absent, blank, AND quote-only the same way, all resolving to `fallback`. Strips one wrapping quote pair + surrounding whitespace from a present value (an inner apostrophe survives). Use for anything that becomes a URL, a title, or a link. | | `Env.getOrFail(key)` | Throws `StateError` only when `key` is entirely absent; still returns `''` for a present-but-empty value. | +### AppLifecycle + +`AppLifecycle.states()` (`lib/src/support/app_lifecycle.dart`) answers a `Stream`, for a reader constructed before a `WidgetsBinding` necessarily exists (a service provider's `register()`, for instance, where `WidgetsBinding.instance` throws). Each subscription adds its own observer on `listen` and removes it on `cancel`; nothing before the first `listen` touches the binding. Prefer Flutter's own `AppLifecycleListener` for a widget-lifetime reader. + +```dart +final subscription = AppLifecycle.states().listen((state) { + if (state == AppLifecycleState.paused) Log.info('app paused'); +}); +``` + ## Cache System The Cache system provides a unified key-value caching API with TTL (time-to-live) support. Backed by the `CacheManager` and resolved via the `Cache` facade. @@ -1211,6 +1224,36 @@ Echo.onReconnect.listen((_) { BroadcastManager.extend('pusher', (config) => PusherBroadcastDriver(config)); ``` +## Sync + +`SyncFeed` (`lib/src/sync/sync_feed.dart`) runs a push-then-pull skeleton over one REST resource: push everything written locally since this device's own mark (`POST '$resource/sync'`, batched at `batchSize`, default 500), then pull every page past the server's own cursor (`GET resource`, up to `maxPages`, default 100), never throwing (an exception becomes `SyncReport.failure`, logged via `Log.error`). + +A subclass supplies `feed` (the ledger key), `resource`/`envelopeKey` (the wire endpoint), `pending({account, sinceMillis, scope})` (rows to push, oldest first), and `adoptRow({account, row})` (write one pulled row, answering whether it was newer). Use `Cast.intOrNull`/`doubleOrNull`/`boolOrNull` inside `adoptRow` for a numeric/boolean field whose wire type is not guaranteed (web's `int`/`double` share one float). + +```dart +class ItemsSyncFeed extends SyncFeed { + @override String get feed => 'items'; + @override String get resource => 'items'; + @override String get envelopeKey => 'items'; + + @override + Future> pending({required String account, required int sinceMillis, required String scope}) async { + // return locally-written rows newer than sinceMillis, oldest first + } + + @override + Future adoptRow({required String account, required Map row}) async { + // write the row locally, return true when it was newer than what was held + } +} + +final SyncReport report = await ItemsSyncFeed().run(scope: 'team-42', account: userId); +``` + +**Two clocks, only one advances locally.** The push mark is this device's own `updated_at` epoch millis; the pull cursor is the server's opaque text, read and rewritten unread. A row adopted from a pull carries the originating device's clock, so the push mark never advances to it; the row is simply re-sent once and rejected by the server's own `>=` check. + +`SyncLedger` (`lib/src/sync/sync_ledger.dart`) is the bookmark store behind `SyncFeed.run`: `read`/`write` over a `(scope, feed)` pair, upserted by delete-then-insert inside a `SAVEPOINT`/`RELEASE` (not `DB.transaction`, since `BEGIN` does not nest and a feed may already run inside a caller's own transaction; a savepoint does). `CreateSyncCursorsTable` (`lib/src/sync/create_sync_cursors_table.dart`) creates the `sync_cursors` table it reads; magic has no migration discovery, so list it in the app's own `Migrator().run([...])` call. Scope derivation, salt, and run scheduling stay app-side. Full reference: `doc/digging-deeper/sync.md`. + ## Key Gotchas - **Cache**: `remember()` returns cached value directly (not awaited) if it exists; only awaits callback on miss. diff --git a/test/support/app_lifecycle_test.dart b/test/support/app_lifecycle_test.dart new file mode 100644 index 00000000..a39c029e --- /dev/null +++ b/test/support/app_lifecycle_test.dart @@ -0,0 +1,76 @@ +import 'dart:async'; + +import 'package:flutter/widgets.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:magic/src/support/app_lifecycle.dart'; + +void main() { + // Runs first and deliberately never initialises a binding: a provider's + // register() constructs a service before any binding exists, and + // AppLifecycle.states() has to survive that unlisted-to. + test('states() does not throw before any binding is initialised', () { + expect(AppLifecycle.states, returnsNormally); + }); + + group('once a binding exists', () { + setUpAll(TestWidgetsFlutterBinding.ensureInitialized); + + test('a listener receives the state the binding reports', () async { + final List received = []; + final StreamSubscription subscription = + AppLifecycle.states().listen(received.add); + addTearDown(subscription.cancel); + + TestWidgetsFlutterBinding.instance.handleAppLifecycleStateChanged( + AppLifecycleState.paused, + ); + await Future.delayed(Duration.zero); + + expect(received, [AppLifecycleState.paused]); + }); + + test('cancelling the subscription stops further delivery', () async { + final List received = []; + final StreamSubscription subscription = + AppLifecycle.states().listen(received.add); + + TestWidgetsFlutterBinding.instance.handleAppLifecycleStateChanged( + AppLifecycleState.paused, + ); + await Future.delayed(Duration.zero); + await subscription.cancel(); + + TestWidgetsFlutterBinding.instance.handleAppLifecycleStateChanged( + AppLifecycleState.resumed, + ); + await Future.delayed(Duration.zero); + + expect(received, [AppLifecycleState.paused]); + }); + + test('a second listen receives its own next state', () async { + final List first = []; + final List second = []; + + final StreamSubscription firstSubscription = + AppLifecycle.states().listen(first.add); + TestWidgetsFlutterBinding.instance.handleAppLifecycleStateChanged( + AppLifecycleState.paused, + ); + await Future.delayed(Duration.zero); + await firstSubscription.cancel(); + + final StreamSubscription secondSubscription = + AppLifecycle.states().listen(second.add); + addTearDown(secondSubscription.cancel); + + TestWidgetsFlutterBinding.instance.handleAppLifecycleStateChanged( + AppLifecycleState.resumed, + ); + await Future.delayed(Duration.zero); + + expect(first, [AppLifecycleState.paused]); + expect(second, [AppLifecycleState.resumed]); + }); + }); +} diff --git a/test/support/exports_test.dart b/test/support/exports_test.dart index db3b459e..bab570be 100644 --- a/test/support/exports_test.dart +++ b/test/support/exports_test.dart @@ -3,8 +3,10 @@ import 'package:magic/magic.dart'; /// Pins that the new Support surface (Number, Str, Cast, Arr), the two UI /// mixins (RefetchesOnMount, SubmitsOnce), CollapsesIndexedErrorKeys, -/// Env.filled and Carbon.shortDiffForHumans all reach a consumer through -/// `package:magic/magic.dart` alone, not only through `package:magic/src/...`. +/// Env.filled, Carbon.shortDiffForHumans, the sync skeleton (SyncFeed, +/// SyncLedger, CreateSyncCursorsTable), Str.ascii/squish, and AppLifecycle +/// all reach a consumer through `package:magic/magic.dart` alone, not only +/// through `package:magic/src/...`. void main() { setUp(() { MagicApp.reset(); @@ -46,4 +48,25 @@ void main() { expect(event.shortDiffForHumans(reference), '14m ago'); }); + + test( + 'SyncFeed, SyncLedger, CreateSyncCursorsTable resolve through the public barrel', + () { + expect(SyncFeed, isNotNull); + expect(const SyncLedger(), isNotNull); + expect(CreateSyncCursorsTable(), isNotNull); + + final report = SyncReport(pushed: 1, adopted: 2); + expect(report.complete, isTrue); + }, + ); + + test('Str.ascii and Str.squish resolve through the public barrel', () { + expect(Str.ascii('çalışan'), 'calisan'); + expect(Str.squish(' a b '), 'a b'); + }); + + test('AppLifecycle.states resolves through the public barrel', () { + expect(AppLifecycle.states(), isNotNull); + }); } diff --git a/test/support/str_fold_parity_test.dart b/test/support/str_fold_parity_test.dart new file mode 100644 index 00000000..f89fcf25 --- /dev/null +++ b/test/support/str_fold_parity_test.dart @@ -0,0 +1,173 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:magic/src/support/str.dart'; + +/// A verbatim copy of the app-side search fold that [Str.ascii] + +/// [Str.squish] were extracted from, kept here only as the parity oracle: an +/// app that stored search keys folded by it must be able to tell exactly +/// which inputs now fold differently. The copy is intentionally NOT +/// refactored to match this file's own style, so a diff against the original +/// stays trivial. +abstract final class _OldFold { + static final RegExp _turkishI = RegExp('[İIı]'); + + static const Map _accented = { + 'a': 'àáâãäåāăą', + 'ae': 'æ', + 'c': 'çćĉċč', + 'd': 'ďđ', + 'e': 'èéêëēĕėęě', + 'g': 'ĝğġģ', + 'h': 'ĥħ', + 'i': 'ìíîïĩīĭį', + 'j': 'ĵ', + 'k': 'ķ', + 'l': 'ĺļľŀł', + 'n': 'ñńņň', + 'o': 'òóôõöøōŏő', + 'oe': 'œ', + 'r': 'ŕŗř', + 's': 'śŝşšș', + 'ss': 'ß', + 't': 'ţťŧț', + 'u': 'ùúûüũūŭůűų', + 'w': 'ŵ', + 'y': 'ýÿŷ', + 'z': 'źżž', + }; + + static final Map _plain = { + for (final MapEntry group in _accented.entries) + for (final int rune in group.value.runes) rune: group.key, + }; + + static String fold(String value) { + final String lower = value.replaceAll(_turkishI, 'i').toLowerCase(); + final StringBuffer folded = StringBuffer(); + bool pendingSpace = false; + + for (final int rune in lower.runes) { + if (_isSpace(rune)) { + pendingSpace = folded.isNotEmpty; + continue; + } + + // U+0300 to U+036F: a decomposed accent, or the dot `toLowerCase` hangs + // on an `i` it was handed as `İ` from somewhere other than [_turkishI]. + if (rune >= 0x300 && rune <= 0x36F) continue; + + if (pendingSpace) { + folded.write(' '); + pendingSpace = false; + } + + folded.write(_plain[rune] ?? String.fromCharCode(rune)); + } + + return folded.toString(); + } + + static bool _isSpace(int rune) => + rune == 0x20 || rune == 0xA0 || (rune >= 0x09 && rune <= 0x0D); +} + +/// Whitespace-class code points where [Str.ascii] and [Str.squish] together +/// treat the rune as space (Dart's `\s` regex class, plus the two Hangul +/// filler additions [Str.squish] documents) while [_OldFold]'s narrow +/// `_isSpace` (0x20, 0xA0, 0x09-0x0D only, built for a search index over a +/// Turkish/Western-European catalogue) leaves it as a literal, lowercased +/// character. Intended: `Str` uses Unicode's actual space-separator and +/// line-terminator set, matching Laravel's own `squish`. +const Set _whitespaceDivergence = { + 0x1680, + 0x2000, + 0x2001, + 0x2002, + 0x2003, + 0x2004, + 0x2005, + 0x2006, + 0x2007, + 0x2008, + 0x2009, + 0x200A, + 0x2028, + 0x2029, + 0x202F, + 0x205F, + 0x3000, + 0xFEFF, + 0x3164, + 0x1160, +}; + +/// Letters only `Str`'s own folding table (`_latinFolding` in +/// `lib/src/support/str.dart`) carries; [_OldFold]'s `_accented` table, built +/// for a Turkish/Western-European catalogue, has no entry for any of them and +/// leaves each as a lowercased literal. +const Set _extraLetterDivergence = { + 0x00AA, // ª feminine ordinal indicator + 0x00B5, // µ micro sign + 0x00BA, // º masculine ordinal indicator + 0x00D0, // Ð capital eth + 0x00DE, // Þ capital thorn + 0x00F0, // ð small eth + 0x00FE, // þ small thorn + 0x0132, // IJ capital ligature IJ + 0x0133, // ij small ligature ij + 0x0138, // ĸ small kra + 0x0149, // ʼn small n preceded by apostrophe + 0x014A, // Ŋ capital eng + 0x014B, // ŋ small eng + 0x017F, // ſ small long s + 0x01FC, // Ǽ capital AE with acute + 0x01FD, // ǽ small ae with acute +}; + +/// The union of both allow-listed divergence classes: a rune here is exempt +/// from the parity assertion, everything else must match [_OldFold] exactly. +final Set _allowedDivergence = _whitespaceDivergence.union( + _extraLetterDivergence, +); + +void main() { + test('Str.squish(Str.ascii(s)).toLowerCase() matches the old fold ' + 'for every BMP code point, outside the allow-listed divergences', () { + final List unexpected = []; + + // A plain loop over the whole BMP rather than per-codepoint `test()` + // registration, so 65 thousand-odd cases stay one fast assertion + // instead of 65 thousand slow ones. + for (int rune = 0x0000; rune <= 0xFFFF; rune++) { + // Lone surrogate halves are not valid standalone code points. + if (rune >= 0xD800 && rune <= 0xDFFF) continue; + + final String c = String.fromCharCode(rune); + + for (final String subject in [c, 'a${c}b', ' $c ']) { + final String actual = Str.squish(Str.ascii(subject)).toLowerCase(); + final String expected = _OldFold.fold(subject); + + if (actual == expected) continue; + if (_allowedDivergence.contains(rune)) continue; + + unexpected.add( + 'U+${rune.toRadixString(16).toUpperCase().padLeft(4, '0')} ' + '(subject: ${subject == c + ? 'alone' + : subject == 'a${c}b' + ? 'a_b' + : '_ _'}): ' + 'got `$actual`, old fold gave `$expected`', + ); + } + } + + expect( + unexpected, + isEmpty, + reason: + 'divergence outside the two allow-listed classes ' + '(unexpected count: ${unexpected.length})', + ); + }); +} diff --git a/test/support/str_test.dart b/test/support/str_test.dart index fb27da1e..a20dd867 100644 --- a/test/support/str_test.dart +++ b/test/support/str_test.dart @@ -56,6 +56,50 @@ void main() { expect(Str.lower('IŞIK', locale: 'tr-TR'), 'ışık'); }); + group('Str.ascii', () { + test('folds Latin diacritics while preserving case', () { + expect(Str.ascii('Çağrı İşık Şeyma'), 'Cagri Isik Seyma'); + }); + + test('folds ß to ss and its capital ẞ to SS', () { + expect(Str.ascii('Straße'), 'Strasse'); + expect(Str.ascii('ẞ'), 'SS'); + }); + + test('folds the Romanian s/t-with-comma letters (U+0219/U+021B)', () { + expect(Str.ascii('Ștefan'), 'Stefan'); + }); + + test('leaves non-Latin scripts untouched', () { + expect(Str.ascii('Москва'), 'Москва'); + }); + + test('folds the Turkish i family, matching Str.lower/Str.upper naming', () { + expect(Str.ascii('İ'), 'I'); + expect(Str.ascii('ı'), 'i'); + }); + + test('folds the ligatures and the Angstrom sign to a plain letter', () { + expect(Str.ascii('æ'), 'ae'); + expect(Str.ascii('œ'), 'oe'); + expect(Str.ascii('Å'), 'A'); + }); + + test('strips a combining mark that survived without composing', () { + expect(Str.ascii('é'), 'e'); + }); + }); + + group('Str.squish', () { + test('trims and collapses runs of whitespace to one space', () { + expect(Str.squish(' a \t\n b '), 'a b'); + }); + + test('collapses the two Hangul filler code points as whitespace too', () { + expect(Str.squish('aㅤᅠb'), 'a b'); + }); + }); + group('Str.unwrap', () { test('strips a matching before/after pair', () { expect(Str.unwrap('"x"', '"'), 'x'); diff --git a/test/sync/sync_feed_test.dart b/test/sync/sync_feed_test.dart new file mode 100644 index 00000000..ca84481d --- /dev/null +++ b/test/sync/sync_feed_test.dart @@ -0,0 +1,250 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:magic/magic.dart'; +import 'package:sqlite3/sqlite3.dart'; + +/// A minimal [SyncFeed] whose pending rows, adoption outcome, batch size and +/// page bound are all handed in by the test, so one class covers every QA +/// scenario instead of a subclass per test. +class _TestFeed extends SyncFeed { + _TestFeed({ + required this.resource, + this.batchSize = 500, + this.maxPages = 100, + this.pendingRows = const [], + Future Function({ + required String account, + required Map row, + })? + onAdoptRow, + }) : _onAdoptRow = onAdoptRow; + + @override + final String resource; + + @override + String get envelopeKey => 'rows'; + + @override + final int batchSize; + + @override + final int maxPages; + + /// The rows [pending] answers, unconditionally of the arguments it is + /// called with: no scenario here needs a real `sinceMillis` filter. + final List pendingRows; + + final Future Function({ + required String account, + required Map row, + })? + _onAdoptRow; + + /// Every row [adoptRow] was asked to write, in call order. + final List> adopted = >[]; + + @override + String get feed => 'test-feed'; + + @override + Future> pending({ + required String account, + required int sinceMillis, + required String scope, + }) async => pendingRows; + + @override + Future adoptRow({ + required String account, + required Map row, + }) async { + adopted.add(row); + if (_onAdoptRow != null) return _onAdoptRow(account: account, row: row); + return true; + } +} + +void main() { + setUp(() { + MagicApp.reset(); + Magic.flush(); + Magic.singleton('log', () => LogManager()); + DatabaseManager().setConnection(sqlite3.openInMemory()); + CreateSyncCursorsTable().up(); + }); + + tearDown(DatabaseManager().dispose); + + test( + "a 2-batch push where the second batch answers 500 advances the stored mark " + "to the first batch's last mark and reports the failure", + () async { + int postCalls = 0; + + Http.fake((MagicRequest request) { + if (request.method == 'POST') { + postCalls++; + if (postCalls == 1) return Http.response({'data': []}, 200); + return Http.response({'message': 'server error'}, 500); + } + + return Http.response({'data': [], 'has_more': false}, 200); + }); + + final _TestFeed feed = _TestFeed( + resource: 'items', + batchSize: 2, + pendingRows: const [ + (data: {'id': 1}, mark: 100), + (data: {'id': 2}, mark: 200), + (data: {'id': 3}, mark: 300), + ], + ); + + final SyncReport report = await feed.run( + scope: 'scope-a', + account: 'acc-1', + ); + + expect(report.complete, isFalse); + expect(report.failure, contains('push failed with HTTP 500')); + expect(report.pushed, 2); + + final SyncBookmarks bookmarks = const SyncLedger().read( + scope: 'scope-a', + feed: 'test-feed', + ); + expect( + bookmarks.pushMark, + 200, + reason: 'only the first (succeeded) batch may advance the mark', + ); + expect( + bookmarks.pullCursor, + isNull, + reason: 'a failed push never reaches the pull half', + ); + }, + ); + + test("a pull over 3 pages stores the last page's cursor", () async { + int getCalls = 0; + + Http.fake((MagicRequest request) { + if (request.method == 'GET') { + getCalls++; + final bool last = getCalls == 3; + return Http.response({ + 'data': [], + 'cursor': 'page-$getCalls', + 'has_more': !last, + }, 200); + } + + return Http.response({'data': []}, 200); + }); + + final _TestFeed feed = _TestFeed(resource: 'items'); + + final SyncReport report = await feed.run( + scope: 'scope-b', + account: 'acc-1', + ); + + expect(report.complete, isTrue); + expect(getCalls, 3); + + final SyncBookmarks bookmarks = const SyncLedger().read( + scope: 'scope-b', + feed: 'test-feed', + ); + expect(bookmarks.pullCursor, 'page-3'); + }); + + test('a pull that never ends stops at maxPages with a failure', () async { + Http.fake((MagicRequest request) { + if (request.method == 'GET') { + return Http.response({ + 'data': [], + 'cursor': 'same', + 'has_more': true, + }, 200); + } + + return Http.response({'data': []}, 200); + }); + + final _TestFeed feed = _TestFeed(resource: 'items', maxPages: 3); + + final SyncReport report = await feed.run( + scope: 'scope-c', + account: 'acc-1', + ); + + expect(report.complete, isFalse); + expect(report.failure, contains('3 pages')); + }); + + test( + 'an adoptRow that throws yields a failure report, not a throw', + () async { + Http.fake((MagicRequest request) { + if (request.method == 'GET') { + return Http.response({ + 'data': [ + {'id': 1}, + ], + 'has_more': false, + }, 200); + } + + return Http.response({'data': []}, 200); + }); + + final _TestFeed feed = _TestFeed( + resource: 'items', + onAdoptRow: + ({ + required String account, + required Map row, + }) async { + throw StateError('cannot adopt this row'); + }, + ); + + final SyncReport report = await feed.run( + scope: 'scope-d', + account: 'acc-1', + ); + + expect(report.complete, isFalse); + expect(report.failure, isNotNull); + }, + ); + + test( + 'a feed whose resource is "items" requests /items/sync and /items', + () async { + final FakeNetworkDriver fake = Http.fake( + (MagicRequest request) => + Http.response({'data': [], 'has_more': false}, 200), + ); + + final _TestFeed feed = _TestFeed( + resource: 'items', + pendingRows: const [ + (data: {'id': 1}, mark: 1), + ], + ); + + await feed.run(scope: 'scope-e', account: 'acc-1'); + + fake.assertSent( + (MagicRequest r) => r.method == 'POST' && r.url == '/items/sync', + ); + fake.assertSent( + (MagicRequest r) => r.method == 'GET' && r.url == '/items', + ); + }, + ); +} diff --git a/test/sync/sync_ledger_test.dart b/test/sync/sync_ledger_test.dart new file mode 100644 index 00000000..cda3587f --- /dev/null +++ b/test/sync/sync_ledger_test.dart @@ -0,0 +1,167 @@ +import 'package:flutter_test/flutter_test.dart'; +import 'package:magic/magic.dart'; +import 'package:sqlite3/sqlite3.dart'; + +void main() { + setUp(() { + MagicApp.reset(); + Magic.flush(); + DatabaseManager().setConnection(sqlite3.openInMemory()); + }); + + tearDown(DatabaseManager().dispose); + + group('SyncLedger', () { + test('read answers the empty bookmarks for an unknown pair', () { + CreateSyncCursorsTable().up(); + + const SyncLedger ledger = SyncLedger(); + final SyncBookmarks bookmarks = ledger.read( + scope: 'scope-a', + feed: 'resume', + ); + + expect(bookmarks.pullCursor, isNull); + expect(bookmarks.pushMark, 0); + }); + + test('write replaces the row for the same scope and feed', () async { + CreateSyncCursorsTable().up(); + + const SyncLedger ledger = SyncLedger(); + await ledger.write( + scope: 's', + feed: 'resume', + account: 'a', + pullCursor: 'c1', + pushMark: 10, + ); + await ledger.write( + scope: 's', + feed: 'resume', + account: 'a', + pullCursor: 'c2', + pushMark: 20, + ); + + final SyncBookmarks bookmarks = ledger.read(scope: 's', feed: 'resume'); + + expect(bookmarks.pullCursor, 'c2'); + expect(bookmarks.pushMark, 20); + expect(DB.select('SELECT * FROM sync_cursors'), hasLength(1)); + }); + + test('two feeds under one scope keep separate rows', () async { + CreateSyncCursorsTable().up(); + + const SyncLedger ledger = SyncLedger(); + await ledger.write( + scope: 's', + feed: 'resume', + account: 'a', + pullCursor: 'c1', + pushMark: 1, + ); + await ledger.write( + scope: 's', + feed: 'favourites', + account: 'a', + pullCursor: 'c2', + pushMark: 2, + ); + + expect(ledger.read(scope: 's', feed: 'resume').pushMark, 1); + expect(ledger.read(scope: 's', feed: 'favourites').pushMark, 2); + }); + + test('write inside a transaction that rolls back leaves no row', () async { + CreateSyncCursorsTable().up(); + + const SyncLedger ledger = SyncLedger(); + + await expectLater( + DB.transaction(() async { + await ledger.write( + scope: 's', + feed: 'resume', + account: 'a', + pullCursor: 'c', + pushMark: 1, + ); + throw StateError('rollback'); + }), + throwsA(isA()), + ); + + expect(DB.select('SELECT * FROM sync_cursors'), isEmpty); + }); + + test( + 'write inside a transaction that commits leaves exactly one row', + () async { + CreateSyncCursorsTable().up(); + + const SyncLedger ledger = SyncLedger(); + + await DB.transaction(() async { + await ledger.write( + scope: 's', + feed: 'resume', + account: 'a', + pullCursor: 'c', + pushMark: 1, + ); + }); + + expect(DB.select('SELECT * FROM sync_cursors'), hasLength(1)); + }, + ); + + test('CreateSyncCursorsTable then SyncLedger round-trips', () async { + CreateSyncCursorsTable().up(); + + const SyncLedger ledger = SyncLedger(); + await ledger.write( + scope: 's', + feed: 'resume', + account: 'a', + pullCursor: 'server-cursor', + pushMark: 42, + ); + + final SyncBookmarks bookmarks = ledger.read(scope: 's', feed: 'resume'); + + expect(bookmarks.pullCursor, 'server-cursor'); + expect(bookmarks.pushMark, 42); + }); + + test( + 'SyncLedger also works on an app-owned table whose feed column was added later', + () async { + DB.statement(''' + CREATE TABLE IF NOT EXISTS sync_cursors ( + scope TEXT NOT NULL, + feed TEXT NOT NULL DEFAULT '', + account TEXT NOT NULL, + pull_cursor TEXT, + push_mark INTEGER NOT NULL + ) + '''); + + const SyncLedger ledger = SyncLedger(); + await ledger.write( + scope: 's', + feed: 'resume', + account: 'a', + pullCursor: 'c', + pushMark: 5, + ); + + final SyncBookmarks bookmarks = ledger.read(scope: 's', feed: 'resume'); + + expect(bookmarks.pullCursor, 'c'); + expect(bookmarks.pushMark, 5); + }, + ); + }); +} From 787edd7a539c1bdd8b5afc2ea95d7a9be71a9286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?An=C4=B1lcan=20=C3=87ak=C4=B1r?= Date: Sat, 26 Sep 2026 03:22:02 +0300 Subject: [PATCH 2/2] fix: never step the push mark past rows that share its value After a successful push batch, SyncFeed.run advanced the mark to slice.last.mark unconditionally. Marks are a local clock, not a row id, so a bulk write can stamp many rows with the same mark; when that run straddles a batch boundary, advancing to it makes the next run's sinceMillis filter (a strict >) skip the still-unsent rows at that mark forever. _push now advances only to the greatest mark in the batch that is strictly below a shared value on the next unsent row, or leaves the mark where it was when every row in the batch carries it. The rows the mark stops short of are resent next run, absorbed by the server's own >= as a duplicate rather than lost. Also: a throw from pending, adoptRow or the ledger after a batch landed used to report pushed: 0. run() now tracks accepted rows in a variable outside the try and reports that instead. --- CHANGELOG.md | 2 +- lib/src/sync/sync_feed.dart | 64 +++++++++++++++-- test/sync/sync_feed_test.dart | 125 +++++++++++++++++++++++++++++++++- 3 files changed, 183 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 51e8c43f..9cd4b879 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ All notable changes to this project will be documented in this file. - **`RefetchesOnMount` and `SubmitsOnce`, two view-layer mixins closing gaps in magic's singleton-controller and async-submit model.** `RefetchesOnMount`, mixed onto a `MagicStatefulViewState`, fire-and-forget refetches a controller's data on every mount (not only the first, since controllers fire `onInit` once per instance for the app's lifetime); `SubmitsOnce`, mixed onto a form's `State`, guards a submit handler against a second tap while the first write is in flight, resetting in a `finally` so a throwing submit re-arms the button. (`lib/src/ui/refetches_on_mount.dart`, `lib/src/ui/submits_once.dart`, `doc/basics/views.md`, `skills/magic-framework/`) - **`Env.filled(key, fallback)` and `Env.getOrFail(key)`, guards for a value a blank silently corrupts.** `Env.get`/`env()` only fall back when a key is entirely absent, so a present-but-blank or quote-only value silently resolved to `''` and has shipped as a blank browser tab title and a link pointing at a path with no origin. `Env.filled` treats absent, blank, and quote-only the same way, stripping one wrapping quote pair and surrounding whitespace from a present value. `Env.getOrFail` mirrors Laravel's `Env::getOrFail`, throwing a `StateError` only when the key is entirely missing. (`lib/src/foundation/env.dart`, `doc/getting-started/configuration.md`, `skills/magic-framework/`) - **`MagicTest.loadTranslations(locale, {directory})`, a real translation catalogue for a widget test.** Reads `/.json` off disk, flattens it with the app's own flatten rule (`JsonAssetLoader.flatten`, now public), and installs it before awaiting the translator's load, so `trans()` resolves real catalogue strings in a test instead of rendering the raw dotted key. (`lib/src/testing/magic_test.dart`, `doc/testing/getting-started.md`, `skills/magic-framework/`) -- **`SyncFeed`, `SyncLedger` and `CreateSyncCursorsTable`, a push-then-pull sync skeleton over one REST resource.** `SyncFeed.run({scope, account})` sends everything a subclass's `pending()` reports written since this device's own push mark (`POST '$resource/sync'`, batched at `batchSize`), then walks pull pages (`GET resource`, up to `maxPages`) handing each row to the subclass's `adoptRow()`, and never throws: an exception surfaces as a `SyncReport.failure` string, logged via `Log.error`. Only the push mark advances locally; the pull cursor is the server's opaque text handed straight back, because a row adopted from the server carries the originating device's own clock. `SyncLedger` is the bookmark store behind it (`sync_cursors`, upserted by delete-then-insert inside a `SAVEPOINT` so it nests inside a caller's own transaction), and `CreateSyncCursorsTable` creates that table; magic has no migration discovery, so an app lists it in its own `Migrator().run([...])` call. Scope derivation, any additional salt, and when a feed runs are left to the app. (`lib/src/sync/sync_feed.dart`, `lib/src/sync/sync_ledger.dart`, `lib/src/sync/create_sync_cursors_table.dart`, `doc/digging-deeper/sync.md`, `skills/magic-framework/`) +- **`SyncFeed`, `SyncLedger` and `CreateSyncCursorsTable`, a push-then-pull sync skeleton over one REST resource.** `SyncFeed.run({scope, account})` sends everything a subclass's `pending()` reports written since this device's own push mark (`POST '$resource/sync'`, batched at `batchSize`), then walks pull pages (`GET resource`, up to `maxPages`) handing each row to the subclass's `adoptRow()`, and never throws: an exception surfaces as a `SyncReport.failure` string, logged via `Log.error`. Only the push mark advances locally, and only past a shared mark once every row carrying it has been sent (marks need not be unique, and a batch boundary may land in the middle of a run of rows that share one); a run halted by a throw after a batch landed reports that batch's rows as `pushed` instead of zero. The pull cursor is the server's opaque text handed straight back, because a row adopted from the server carries the originating device's own clock. `SyncLedger` is the bookmark store behind it (`sync_cursors`, upserted by delete-then-insert inside a `SAVEPOINT` so it nests inside a caller's own transaction), and `CreateSyncCursorsTable` creates that table; magic has no migration discovery, so an app lists it in its own `Migrator().run([...])` call. Scope derivation, any additional salt, and when a feed runs are left to the app. (`lib/src/sync/sync_feed.dart`, `lib/src/sync/sync_ledger.dart`, `lib/src/sync/create_sync_cursors_table.dart`, `doc/digging-deeper/sync.md`, `skills/magic-framework/`) - **`Str.ascii(value)` and `Str.squish(value)`, two more of Laravel's `Str` helpers ported.** `ascii` folds Latin letters (accents, the Romanian comma-below letters, `ẞ`) to their plain ASCII base for a search key, Latin only, diverging from Laravel's `Str::ascii`, which transliterates every script it has a table for and would otherwise collapse a non-Latin word to `?` or a phonetic guess. `squish` trims and collapses every run of whitespace to one space, matching Laravel's `Str::squish`, over Dart's `\s` class plus two Hangul filler code points a rendered blank can carry without registering as whitespace. (`lib/src/support/str.dart`, `doc/digging-deeper/helpers.md`, `skills/magic-framework/`) - **The default `User-Agent`'s app-name folding (`NetworkServiceProvider`, via `Str.ascii`) now also folds `ș`, `ț`, `ẞ`, and the Angstrom sign U+212B to their plain ASCII base.** They used to fall through `Str.ascii`'s table untouched and were then dropped by the printable-ASCII filter that follows it. - **`AppLifecycle.states()`, the app lifecycle as a `Stream` for a reader built before a `WidgetsBinding` necessarily exists.** A dependency constructed inside a service provider's `register()` runs before the app has bound anything, so reaching for `WidgetsBinding.instance` there throws; `states()` defers that lookup to the moment a listener subscribes, adding its own observer to the binding on `listen` and removing it on `cancel`, so nothing outlives its reader. Prefer Flutter's own `AppLifecycleListener` once a binding is guaranteed to exist. (`lib/src/support/app_lifecycle.dart`, `doc/digging-deeper/helpers.md`, `skills/magic-framework/`) diff --git a/lib/src/sync/sync_feed.dart b/lib/src/sync/sync_feed.dart index 2639f305..65b73782 100644 --- a/lib/src/sync/sync_feed.dart +++ b/lib/src/sync/sync_feed.dart @@ -105,7 +105,10 @@ abstract class SyncFeed { /// /// Oldest first is required rather than tidy: the push advances its mark /// to the last row of each batch it sends, so a batch that did not end on - /// its newest row would move the mark past rows it never sent. + /// its newest row would move the mark past rows it never sent. Marks need + /// not be unique across rows, so a run of rows sharing one mark may + /// straddle a batch boundary; [_markAfterBatch] is what keeps that case + /// from losing the rows on the far side of the boundary. @protected Future> pending({ required String account, @@ -134,6 +137,12 @@ abstract class SyncFeed { required String scope, required String account, }) async { + // Tracks rows the server has already accepted across every batch, so a + // throw from `pending`, `adoptRow` or the ledger after some batch landed + // still reports what actually got through instead of claiming nothing + // did. + int pushedSoFar = 0; + try { // The raw `Http` methods hand the path to the driver untouched, and a // base URL with no trailing slash plus a resource with no leading one @@ -148,6 +157,7 @@ abstract class SyncFeed { scope: scope, account: account, mark: bookmarks.pushMark, + onBatchAccepted: (int sent) => pushedSoFar += sent, ); if (push.failure != null) { @@ -191,7 +201,11 @@ abstract class SyncFeed { 'stackTrace': stackTrace.toString(), }); - return SyncReport(pushed: 0, adopted: 0, failure: 'sync failed: $error'); + return SyncReport( + pushed: pushedSoFar, + adopted: 0, + failure: 'sync failed: $error', + ); } } @@ -205,6 +219,7 @@ abstract class SyncFeed { required String scope, required String account, required int mark, + required void Function(int sent) onBatchAccepted, }) async { final List changed = await pending( account: account, @@ -237,9 +252,17 @@ abstract class SyncFeed { } // Only past a 2xx: the batch is all or nothing on the server, so a - // mark advanced on a refused batch would lose every row in it. - reached = slice.last.mark; + // mark advanced on a refused batch would lose every row in it. The + // mark itself may still fall short of this slice's last row; see + // _markAfterBatch. + reached = _markAfterBatch( + slice: slice, + changed: changed, + end: end, + previousMark: reached, + ); sent += slice.length; + onBatchAccepted(slice.length); await _adopt(account: account, body: response.data); } @@ -247,6 +270,39 @@ abstract class SyncFeed { return _PushResult(sent: sent, mark: reached, failure: null); } + /// The mark a batch that just landed may safely advance to. + /// + /// A mark is this device's local clock, not a row id, so it need not be + /// unique: a bulk write can stamp many rows with the same value. When the + /// next unsent row ([changed] at [end]) shares [slice]'s last mark, that + /// value has not finished sending yet, and advancing to it would make the + /// next run's `sinceMillis` filter (a strict `>`, per [pending]'s doc) + /// skip the rows still waiting at that mark forever. This falls back to + /// the greatest mark in [slice] strictly below the shared value, or to + /// [previousMark] when every row in [slice] carries it. Either way, the + /// rows the mark stops short of are resent on the next run, absorbed by + /// the server's own `>=` as a duplicate rather than lost. + int _markAfterBatch({ + required List slice, + required List changed, + required int end, + required int previousMark, + }) { + final int lastMark = slice.last.mark; + + if (end >= changed.length || changed[end].mark > lastMark) return lastMark; + + int? safe; + + for (final SyncPushRow row in slice) { + if (row.mark < lastMark && (safe == null || row.mark > safe)) { + safe = row.mark; + } + } + + return safe ?? previousMark; + } + /// Walks every page the server has past [cursor], writing each row. Future<_PullResult> _pull({ required String path, diff --git a/test/sync/sync_feed_test.dart b/test/sync/sync_feed_test.dart index ca84481d..e85cb2ea 100644 --- a/test/sync/sync_feed_test.dart +++ b/test/sync/sync_feed_test.dart @@ -11,6 +11,7 @@ class _TestFeed extends SyncFeed { this.batchSize = 500, this.maxPages = 100, this.pendingRows = const [], + this.filterPendingBySinceMillis = false, Future Function({ required String account, required Map row, @@ -30,10 +31,15 @@ class _TestFeed extends SyncFeed { @override final int maxPages; - /// The rows [pending] answers, unconditionally of the arguments it is - /// called with: no scenario here needs a real `sinceMillis` filter. + /// The rows [pending] answers. final List pendingRows; + /// Whether [pending] filters [pendingRows] by `sinceMillis` (a strict `>`, + /// matching the documented reader). False keeps the old behaviour of + /// answering [pendingRows] unconditionally, which most scenarios here do + /// not need a real filter for; the mark-boundary scenarios do. + final bool filterPendingBySinceMillis; + final Future Function({ required String account, required Map row, @@ -51,7 +57,9 @@ class _TestFeed extends SyncFeed { required String account, required int sinceMillis, required String scope, - }) async => pendingRows; + }) async => filterPendingBySinceMillis + ? pendingRows.where((SyncPushRow row) => row.mark > sinceMillis).toList() + : pendingRows; @override Future adoptRow({ @@ -127,6 +135,117 @@ void main() { }, ); + test('rows that share a mark across a batch boundary are resent rather than ' + 'lost: the mark never steps past the shared value until every row at it ' + 'is sent', () async { + int postCalls = 0; + + Http.fake((MagicRequest request) { + if (request.method == 'POST') { + postCalls++; + if (postCalls == 1) return Http.response({'data': []}, 200); + return Http.response({'message': 'server error'}, 500); + } + + return Http.response({'data': [], 'has_more': false}, 200); + }); + + final List rows = [ + for (int i = 0; i < 600; i++) (data: {'id': i}, mark: 1000), + ]; + + final _TestFeed feed = _TestFeed( + resource: 'items', + batchSize: 500, + pendingRows: rows, + filterPendingBySinceMillis: true, + ); + + final SyncReport firstRun = await feed.run( + scope: 'scope-f', + account: 'acc-1', + ); + + expect(firstRun.complete, isFalse); + expect(firstRun.pushed, 500); + + final SyncBookmarks afterFirst = const SyncLedger().read( + scope: 'scope-f', + feed: 'test-feed', + ); + expect( + afterFirst.pushMark, + lessThan(1000), + reason: + 'every row in the accepted batch shares mark 1000 with the ' + 'unsent 100, so the mark must not reach 1000 yet', + ); + + // The server now accepts everything, including the batch it refused + // the first time. + Http.fake((MagicRequest request) { + if (request.method == 'POST') { + return Http.response({'data': []}, 200); + } + + return Http.response({'data': [], 'has_more': false}, 200); + }); + + final SyncReport secondRun = await feed.run( + scope: 'scope-f', + account: 'acc-1', + ); + + expect(secondRun.complete, isTrue); + expect( + secondRun.pushed, + greaterThanOrEqualTo(100), + reason: + 'the 100 rows the mark stopped short of must be resent, the ' + 'first 500 may be resent alongside them', + ); + }); + + test('an adoptRow that throws after a batch was accepted reports the rows ' + 'that batch already pushed, not zero', () async { + Http.fake((MagicRequest request) { + if (request.method == 'POST') { + return Http.response({ + 'data': [ + {'id': 1}, + ], + }, 200); + } + + return Http.response({'data': [], 'has_more': false}, 200); + }); + + final _TestFeed feed = _TestFeed( + resource: 'items', + batchSize: 2, + pendingRows: const [ + (data: {'id': 1}, mark: 100), + (data: {'id': 2}, mark: 200), + ], + onAdoptRow: + ({required String account, required Map row}) async { + throw StateError('cannot adopt this row'); + }, + ); + + final SyncReport report = await feed.run( + scope: 'scope-g', + account: 'acc-1', + ); + + expect(report.complete, isFalse); + expect( + report.pushed, + 2, + reason: 'the batch that threw was already accepted by the server', + ); + }); + test("a pull over 3 pages stores the last page's cursor", () async { int getCalls = 0;