From d5e9f4011cc0964f67d4713f4da0c711c8129846 Mon Sep 17 00:00:00 2001 From: Ty J Everett Date: Sun, 2 Aug 2026 19:03:31 -0700 Subject: [PATCH 1/4] perf(wallet): accelerate createAction funding pipeline --- docs/packages/sdk/bsv-sdk.md | 2 +- docs/packages/wallet/wallet-toolbox-client.md | 2 +- docs/packages/wallet/wallet-toolbox-mobile.md | 2 +- docs/packages/wallet/wallet-toolbox.md | 2 +- docs/reference/package-api-migrations.md | 28 +- docs/reference/stack-facts.md | 8 +- governance/package-release-notes.json | 18 +- governance/repository-health/baselines.json | 8 +- packages/sdk/CHANGELOG.md | 4 + .../sdk/benchmarks/beef-sort-cache-bench.js | 15 + packages/sdk/browser-budget.json | 2 +- packages/sdk/docs/performance.md | 18 + packages/sdk/package.json | 2 +- packages/sdk/scripts/run-benchmarks.js | 7 + packages/sdk/src/transaction/Beef.ts | 77 +- .../sdk/src/transaction/__tests/Beef.test.ts | 29 + packages/wallet/wallet-toolbox/CHANGELOG.md | 9 + packages/wallet/wallet-toolbox/README.md | 32 + .../create-action-funding.bench.test.ts | 107 +++ .../wallet/wallet-toolbox/client/package.json | 2 +- .../wallet/wallet-toolbox/mobile/package.json | 2 +- packages/wallet/wallet-toolbox/package.json | 3 +- packages/wallet/wallet-toolbox/src/Wallet.ts | 18 +- .../wallet-toolbox/src/storage/StorageIdb.ts | 108 ++- .../wallet-toolbox/src/storage/StorageKnex.ts | 68 +- .../src/storage/StorageProvider.ts | 46 +- .../src/storage/__test/StorageIdb.test.ts | 78 +- .../__test/createActionPerformance.test.ts | 196 +++++ .../__test/getBeefForTransaction.test.ts | 18 + .../wallet-toolbox/src/storage/idbHelpers.ts | 2 + .../src/storage/methods/actionBatch.ts | 24 +- .../storage/methods/availableManagedChange.ts | 30 + .../src/storage/methods/createAction.ts | 708 ++++++++++++++---- .../storage/methods/getBeefForTransaction.ts | 27 +- .../src/storage/schema/KnexMigrations.ts | 20 + .../src/storage/schema/StorageIdbSchema.ts | 2 + .../test/Wallet/action/actionBatch.test.ts | 3 + .../test/storage/KnexMigrations.test.ts | 37 + 38 files changed, 1521 insertions(+), 243 deletions(-) create mode 100644 packages/sdk/benchmarks/beef-sort-cache-bench.js create mode 100644 packages/wallet/wallet-toolbox/benchmarks/create-action-funding.bench.test.ts create mode 100644 packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts create mode 100644 packages/wallet/wallet-toolbox/src/storage/methods/availableManagedChange.ts diff --git a/docs/packages/sdk/bsv-sdk.md b/docs/packages/sdk/bsv-sdk.md index 265790cdf..306afee2a 100644 --- a/docs/packages/sdk/bsv-sdk.md +++ b/docs/packages/sdk/bsv-sdk.md @@ -3,7 +3,7 @@ id: bsv-sdk title: '@bsv/sdk' kind: package domain: sdk -version: '2.2.17' +version: '2.2.18' npm: '@bsv/sdk' last_updated: '2026-07-31' last_verified: '2026-07-31' diff --git a/docs/packages/wallet/wallet-toolbox-client.md b/docs/packages/wallet/wallet-toolbox-client.md index 5445252db..09de505ab 100644 --- a/docs/packages/wallet/wallet-toolbox-client.md +++ b/docs/packages/wallet/wallet-toolbox-client.md @@ -3,7 +3,7 @@ id: pkg-wallet-toolbox-client title: '@bsv/wallet-toolbox-client' kind: package domain: wallet -version: '2.4.21' +version: '2.4.22' last_updated: '2026-07-31' last_verified: '2026-07-31' review_cadence_days: 30 diff --git a/docs/packages/wallet/wallet-toolbox-mobile.md b/docs/packages/wallet/wallet-toolbox-mobile.md index 746ab5c4b..9861dcd2f 100644 --- a/docs/packages/wallet/wallet-toolbox-mobile.md +++ b/docs/packages/wallet/wallet-toolbox-mobile.md @@ -3,7 +3,7 @@ id: pkg-wallet-toolbox-mobile title: '@bsv/wallet-toolbox-mobile' kind: package domain: wallet -version: '2.4.21' +version: '2.4.22' last_updated: '2026-07-31' last_verified: '2026-07-31' review_cadence_days: 30 diff --git a/docs/packages/wallet/wallet-toolbox.md b/docs/packages/wallet/wallet-toolbox.md index 973372357..3b8824cb5 100644 --- a/docs/packages/wallet/wallet-toolbox.md +++ b/docs/packages/wallet/wallet-toolbox.md @@ -4,7 +4,7 @@ title: '@bsv/wallet-toolbox' kind: package domain: wallet npm: '@bsv/wallet-toolbox' -version: '2.4.21' +version: '2.4.22' last_updated: '2026-07-31' last_verified: '2026-07-31' review_cadence_days: 30 diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index e8d8b1f4d..165937212 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -3,8 +3,8 @@ id: package-api-migrations title: 'Package API, Declarations, and Migration Ledger' kind: reference version: '1.0.0' -last_updated: '2026-07-31' -last_verified: '2026-07-31' +last_updated: '2026-08-02' +last_verified: '2026-08-02' review_cadence_days: 30 status: stable tags: [reference, packages, api, declarations, migrations, release-notes] @@ -45,16 +45,16 @@ and clean-consumer tests remain the executable type authority. | `@bsv/overlay-topics` | `1.6.1` | `1.6.8` | patch | [API and usage](../packages/overlays/overlay-topics.md) | No consumer migration is required; topic IDs, lookup contracts, and persisted formats are unchanged. | | `@bsv/paymail` | `2.4.2` | `2.4.6` | patch | [API and usage](../packages/messaging/paymail.md) | Existing Paymail client APIs and protocol semantics are retained. Consumers provide one Express 4.18 or 5 runtime and matching type graph; browser bundles continue to exclude the server router implementation. | | `@bsv/payment-express-middleware` | `2.1.1` | `2.1.5` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No consumer migration is required; legacy x-bsv-payment JSON behavior remains supported, and Express 4 and 5 applications use their own peer-provided Express installation. | -| `@bsv/sdk` | `2.2.0` | `2.2.17` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | The 2.x wire encodings and supported imports are unchanged. Peer listeners may continue returning void and may now return Promise; rejected or malformed authenticated HTTP responses reject and clean up the owning request, and pending authenticated responses now time out after 30 seconds. | +| `@bsv/sdk` | `2.2.0` | `2.2.18` | patch | [API and usage](../packages/sdk/bsv-sdk.md) | No consumer migration is required. BEEF ordering, validation results, serialized bytes, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged. | | `@bsv/simple` | `0.4.1` | `0.4.8` | patch | [API and usage](../packages/helpers/simple.md) | No consumer migration is required; the browser and server entry points remain compatible. | | `@bsv/templates` | `1.9.1` | `1.9.6` | patch | [API and usage](../packages/helpers/templates.md) | No consumer migration is required; template APIs, supported imports, and generated script semantics are unchanged. | | `@bsv/teranode-listener` | `1.1.1` | `1.1.4` | patch | [API and usage](../packages/network/teranode-listener.md) | No consumer migration is required; listener APIs, topics, and network configuration are unchanged. | | `@bsv/verifast` | `0.3.0` | `0.3.4` | patch | [API and usage](../packages/sdk/verifast.md) | No consumer migration is required; exports, verification behavior, worker protocols, package paths, and runtime defaults are unchanged. | | `@bsv/wallet-helper` | `0.1.1` | `0.1.6` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No consumer migration is required; fluent builder APIs and transaction semantics are unchanged. | | `@bsv/wallet-relay` | `0.2.2` | `0.3.4` | minor | [API and usage](../packages/wallet/wallet-relay.md) | QRPairingCode now renders a native button and accepts button wrapper attributes. Existing className, style, data, and ARIA props continue to work; update div-specific wrapper selectors or explicitly typed div event handlers. Express integrations now use the host application's matching Express runtime and type graph. | -| `@bsv/wallet-toolbox` | `2.4.4` | `2.4.21` | patch | [API and usage](../packages/wallet/wallet-toolbox.md) | No consumer migration is required; BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, persisted schemas, provider behavior, and the 2.x wallet and storage interfaces remain unchanged. | -| `@bsv/wallet-toolbox-client` | `2.4.4` | `2.4.21` | patch | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No consumer migration is required; BRC-103/104, AuthFetch, browser entry points, and remote storage contracts remain unchanged. | -| `@bsv/wallet-toolbox-mobile` | `2.4.4` | `2.4.21` | patch | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No consumer migration is required; BRC-103/104, AuthFetch, React Native, mobile bridge, and remote storage contracts remain unchanged. | +| `@bsv/wallet-toolbox` | `2.4.4` | `2.4.22` | patch | [API and usage](../packages/wallet/wallet-toolbox.md) | No consumer migration is required. SQLite/MySQL and IndexedDB indexes are added by the existing migration paths; wallet data and results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are unchanged. | +| `@bsv/wallet-toolbox-client` | `2.4.4` | `2.4.22` | patch | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No consumer migration is required; IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged. | +| `@bsv/wallet-toolbox-mobile` | `2.4.4` | `2.4.22` | patch | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No consumer migration is required; BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged. | | `create-bsv-app` | `1.0.2` | `1.0.4` | patch | [API and usage](../packages/helpers/create-bsv-app.md) | No consumer migration is required; generated application structure and CLI behavior are unchanged. | `none` means the source manifest matches the recorded npm baseline. Any other @@ -316,8 +316,8 @@ CLI entry points: `{"fund-metanet":"./dist/index.mjs"}`. - Package documentation: [docs/packages/sdk/bsv-sdk.md](../packages/sdk/bsv-sdk.md) - Source: [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) -- Release note: Accumulates security and correctness hardening, transaction and action-batch performance work, strict package contracts, safer text and telemetry handling, behavior-preserving maintainability remediation across cryptographic, transaction, remittance, lookup, script-interpreter, broadcaster, and wallet-wire paths, and hardens BRC-103 message, handshake, response, async listener, timeout, and pending-request boundaries. -- Migration: The 2.x wire encodings and supported imports are unchanged. Peer listeners may continue returning void and may now return Promise; rejected or malformed authenticated HTTP responses reject and clean up the owning request, and pending authenticated responses now time out after 30 seconds. +- Release note: Accumulates security and correctness hardening, transaction and action-batch performance work, strict package contracts, safer text and telemetry handling, and caches unchanged BEEF dependency-sort results so repeated wallet known-txid preparation avoids a full topological sort. +- Migration: No consumer migration is required. BEEF ordering, validation results, serialized bytes, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------------------------- | ---------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | @@ -477,8 +477,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox.md](../packages/wallet/wallet-toolbox.md) - Source: [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) -- Release note: Adds atomic action-batch transport, storage and proof resilience, security hardening, compatible runtime dependency maintenance, strict package contracts, broad maintainability remediation, safe telemetry, and restores remote-storage compatibility by retaining trace correlation locally instead of adding unsupported AuthFetch headers. -- Migration: No consumer migration is required; BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, persisted schemas, provider behavior, and the 2.x wallet and storage interfaces remain unchanged. +- Release note: Adds fee-aware read-only legacy createAction planning, one-transaction funding claims, exact reservation filtering, Knex and IndexedDB funding indexes, faster known-txid and proof handling, and privacy-safe timings for every material funding and BEEF phase. +- Migration: No consumer migration is required. SQLite/MySQL and IndexedDB indexes are added by the existing migration paths; wallet data and results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ---------------------------------------------------- | -------------------------- | @@ -491,8 +491,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-client.md](../packages/wallet/wallet-toolbox-client.md) - Source: [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) -- Release note: Carries the lockstep Wallet Toolbox browser candidate with safe local trace correlation and no unsupported authenticated request headers. -- Migration: No consumer migration is required; BRC-103/104, AuthFetch, browser entry points, and remote storage contracts remain unchanged. +- Release note: Carries the lockstep browser build with read-only funding planning, atomic claims, IndexedDB funding indexes, BEEF-history caching, and expanded privacy-safe createAction timings. +- Migration: No consumer migration is required; IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | @@ -503,8 +503,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-mobile.md](../packages/wallet/wallet-toolbox-mobile.md) - Source: [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) -- Release note: Carries the lockstep Wallet Toolbox mobile candidate with safe local trace correlation and no unsupported authenticated request headers. -- Migration: No consumer migration is required; BRC-103/104, AuthFetch, React Native, mobile bridge, and remote storage contracts remain unchanged. +- Release note: Carries the lockstep mobile build with read-only funding planning, atomic claims, faster BEEF-history handling, and expanded privacy-safe createAction timings. +- Migration: No consumer migration is required; BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 4d390bd80..7ba1b58e7 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -60,14 +60,14 @@ authorized release action. | overlays | `@bsv/overlay-discovery-services` | `2.1.6` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) | | overlays | `@bsv/overlay-express` | `2.4.9` | node-library | node-cjs, node-esm | node | `>=22` | [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) | | overlays | `@bsv/overlay-topics` | `1.6.8` | node-library | node-esm | node | `>=22` | [packages/overlays/topics](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/topics) | -| sdk | `@bsv/sdk` | `2.2.17` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | +| sdk | `@bsv/sdk` | `2.2.18` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global | browser, node, umd | `>=22` | [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) | | sdk | `@bsv/verifast` | `0.3.4` | wasm-library | browser-bundler, browser-esm, node-cjs, node-esm, umd-global, wasm-worker | browser, node, umd, wasm, worker | `>=22` | [packages/verifast](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/verifast) | | wallet | `@bsv/btms` | `1.1.4` | node-library | node-cjs, node-esm | node | `>=22` | [packages/wallet/btms](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms) | | wallet | `@bsv/btms-permission-module` | `1.1.3` | node-library | node-esm | node | `>=22` | [packages/wallet/btms-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms-permission-module) | | wallet | `@bsv/wallet-relay` | `0.3.4` | cli-library | browser-bundler, browser-esm, cli, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/ts-wallet-relay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/ts-wallet-relay) | -| wallet | `@bsv/wallet-toolbox` | `2.4.21` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | -| wallet | `@bsv/wallet-toolbox-client` | `2.4.21` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | -| wallet | `@bsv/wallet-toolbox-mobile` | `2.4.21` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | +| wallet | `@bsv/wallet-toolbox` | `2.4.22` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | +| wallet | `@bsv/wallet-toolbox-client` | `2.4.22` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | +| wallet | `@bsv/wallet-toolbox-mobile` | `2.4.22` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | ## Standalone infrastructure manifests diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index 63273bd57..e1116418f 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "lastReviewed": "2026-07-31", + "lastReviewed": "2026-08-02", "owner": "ts-stack-maintainers", "entries": [ { @@ -147,8 +147,8 @@ "name": "@bsv/sdk", "publishedVersion": "2.2.0", "releaseType": "patch", - "summary": "Accumulates security and correctness hardening, transaction and action-batch performance work, strict package contracts, safer text and telemetry handling, behavior-preserving maintainability remediation across cryptographic, transaction, remittance, lookup, script-interpreter, broadcaster, and wallet-wire paths, and hardens BRC-103 message, handshake, response, async listener, timeout, and pending-request boundaries.", - "migration": "The 2.x wire encodings and supported imports are unchanged. Peer listeners may continue returning void and may now return Promise; rejected or malformed authenticated HTTP responses reject and clean up the owning request, and pending authenticated responses now time out after 30 seconds." + "summary": "Accumulates security and correctness hardening, transaction and action-batch performance work, strict package contracts, safer text and telemetry handling, and caches unchanged BEEF dependency-sort results so repeated wallet known-txid preparation avoids a full topological sort.", + "migration": "No consumer migration is required. BEEF ordering, validation results, serialized bytes, SDK 2.x wire encodings, BRC-103/104 behavior, and supported imports are unchanged." }, { "name": "@bsv/simple", @@ -196,22 +196,22 @@ "name": "@bsv/wallet-toolbox", "publishedVersion": "2.4.4", "releaseType": "patch", - "summary": "Adds atomic action-batch transport, storage and proof resilience, security hardening, compatible runtime dependency maintenance, strict package contracts, broad maintainability remediation, safe telemetry, and restores remote-storage compatibility by retaining trace correlation locally instead of adding unsupported AuthFetch headers.", - "migration": "No consumer migration is required; BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, persisted schemas, provider behavior, and the 2.x wallet and storage interfaces remain unchanged." + "summary": "Adds fee-aware read-only legacy createAction planning, one-transaction funding claims, exact reservation filtering, Knex and IndexedDB funding indexes, faster known-txid and proof handling, and privacy-safe timings for every material funding and BEEF phase.", + "migration": "No consumer migration is required. SQLite/MySQL and IndexedDB indexes are added by the existing migration paths; wallet data and results, BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, provider calls, and wallet wire behavior are unchanged." }, { "name": "@bsv/wallet-toolbox-client", "publishedVersion": "2.4.4", "releaseType": "patch", - "summary": "Carries the lockstep Wallet Toolbox browser candidate with safe local trace correlation and no unsupported authenticated request headers.", - "migration": "No consumer migration is required; BRC-103/104, AuthFetch, browser entry points, and remote storage contracts remain unchanged." + "summary": "Carries the lockstep browser build with read-only funding planning, atomic claims, IndexedDB funding indexes, BEEF-history caching, and expanded privacy-safe createAction timings.", + "migration": "No consumer migration is required; IndexedDB upgrades automatically, and BRC-103/104, AuthFetch, browser entry points, JSON-RPC, and remote storage contracts remain unchanged." }, { "name": "@bsv/wallet-toolbox-mobile", "publishedVersion": "2.4.4", "releaseType": "patch", - "summary": "Carries the lockstep Wallet Toolbox mobile candidate with safe local trace correlation and no unsupported authenticated request headers.", - "migration": "No consumer migration is required; BRC-103/104, AuthFetch, React Native, mobile bridge, and remote storage contracts remain unchanged." + "summary": "Carries the lockstep mobile build with read-only funding planning, atomic claims, faster BEEF-history handling, and expanded privacy-safe createAction timings.", + "migration": "No consumer migration is required; BRC-103/104, AuthFetch, React Native, the mobile bridge, JSON-RPC, and remote storage contracts remain unchanged." }, { "name": "create-bsv-app", diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index e8ed51102..8ac5de719 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -320,13 +320,13 @@ "@bsv/overlay-discovery-services": "2.1.6", "@bsv/overlay-express": "2.4.9", "@bsv/overlay-topics": "1.6.8", - "@bsv/sdk": "2.2.17", + "@bsv/sdk": "2.2.18", "@bsv/verifast": "0.3.4", "@bsv/btms": "1.1.4", "@bsv/btms-permission-module": "1.1.3", "@bsv/wallet-relay": "0.3.4", - "@bsv/wallet-toolbox-client": "2.4.21", - "@bsv/wallet-toolbox-mobile": "2.4.21", - "@bsv/wallet-toolbox": "2.4.21" + "@bsv/wallet-toolbox-client": "2.4.22", + "@bsv/wallet-toolbox-mobile": "2.4.22", + "@bsv/wallet-toolbox": "2.4.22" } } diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index c18fa348c..2c46df11c 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -230,6 +230,10 @@ All notable changes to this project will be documented in this file. The format ### Changed +- Cache BEEF dependency-sort results until transaction or proof state changes, + so repeated wallet known-txid preparation is proportional to the returned + list instead of re-running the complete topological sort. BEEF ordering, + validation results, and serialized bytes are unchanged. - Point SDK contributors and AI agents to the canonical stack-level contribution and quality policy without changing package behavior. - Harden UTF-8, binary JSON, secret-redaction, cryptographic compatibility, and diff --git a/packages/sdk/benchmarks/beef-sort-cache-bench.js b/packages/sdk/benchmarks/beef-sort-cache-bench.js new file mode 100644 index 000000000..d43b6e85c --- /dev/null +++ b/packages/sdk/benchmarks/beef-sort-cache-bench.js @@ -0,0 +1,15 @@ +import Beef from '../dist/esm/src/transaction/Beef.js' +import { runBenchmark } from './lib/benchmark-runner.js' + +const transactionCount = Number.parseInt(process.env.BEEF_SORT_TXS ?? '20000', 10) + +const beef = new Beef() +for (let index = 0; index < transactionCount; index++) { + beef.mergeTxidOnly(index.toString(16).padStart(64, '0')) +} +beef.sortTxs() + +await runBenchmark(`Beef.sortTxs unchanged (${transactionCount} txs)`, () => beef.sortTxs(), { + minSampleMs: 100, + samples: 7 +}) diff --git a/packages/sdk/browser-budget.json b/packages/sdk/browser-budget.json index d1b07ffad..7db2cf106 100644 --- a/packages/sdk/browser-budget.json +++ b/packages/sdk/browser-budget.json @@ -30,7 +30,7 @@ "path": "dist/umd/bundle.js", "global": "bsv", "maximumBytes": { - "raw": 545000, + "raw": 547000, "gzip": 162000, "brotli": 136000 } diff --git a/packages/sdk/docs/performance.md b/packages/sdk/docs/performance.md index cdbbd8d42..7c79f50fb 100644 --- a/packages/sdk/docs/performance.md +++ b/packages/sdk/docs/performance.md @@ -1,5 +1,23 @@ # Performance Benchmarks +## Repeated BEEF dependency sorting (August 2026) + +Wallet transaction creation can ask the same unchanged BEEF for its dependency +order more than once while preparing known transaction IDs, merging proofs, and +serializing the final result. The mutation-aware sort cache retains the public +ordering while avoiding another complete graph partition and topological sort. + +```bash +BEEF_SORT_TXS=20000 node benchmarks/beef-sort-cache-bench.js +``` + +Against unmodified commit `c212b5ee7`, seven same-host samples on Apple Silicon +with Node.js v24.18.0 produced a 235.86 ms median for 50 unchanged sorts. The +optimized build took 9.72 ms after validating directly reachable dependency +state, a 24.3x improvement. The cold sort remained about +7–8 ms; transaction or proof mutation invalidates the cache, and callers receive +new result arrays so consumer mutation cannot alter the cached order. + ## Transaction and BEEF pipeline (July 2026) The transaction pipeline benchmark uses built ESM artifacts and is intentionally runnable against another build through `SDK_DIST_ROOT`. Measurements below were taken on the same Apple Silicon host with Node.js v25.9.0. Each comparison uses identical generated transactions and benchmark code. diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 16c565839..cf1eee307 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/sdk", - "version": "2.2.17", + "version": "2.2.18", "sideEffects": false, "engines": { "node": ">=22" diff --git a/packages/sdk/scripts/run-benchmarks.js b/packages/sdk/scripts/run-benchmarks.js index 500dc41d4..86b80403a 100644 --- a/packages/sdk/scripts/run-benchmarks.js +++ b/packages/sdk/scripts/run-benchmarks.js @@ -55,6 +55,13 @@ const BENCHMARKS = [ commands: [ { script: 'benchmarks/atomic-beef-bench.js', args: [] } ] + }, + { + id: 'beef-sort-cache', + label: 'BEEF Sort Cache', + commands: [ + { script: 'benchmarks/beef-sort-cache-bench.js', args: [] } + ] } ] diff --git a/packages/sdk/src/transaction/Beef.ts b/packages/sdk/src/transaction/Beef.ts index 35dba47c2..81bb38ae2 100644 --- a/packages/sdk/src/transaction/Beef.ts +++ b/packages/sdk/src/transaction/Beef.ts @@ -43,6 +43,22 @@ interface BeefBumpSerializationState { levels: BeefBumpLevelSerializationState[] } +interface BeefSortResult { + missingInputs: string[] + notValid: string[] + valid: string[] + withMissingInputs: string[] + txidOnly: string[] +} + +interface BeefSortTxState { + ref: BeefTx + txid: string + bumpIndex?: number + isTxidOnly: boolean + inputTxids: string[] +} + /* * BEEF standard: BRC-62: Background Evaluation Extended Format (BEEF) Transactions * https://github.com/bsv-blockchain/BRCs/blob/master/transactions/0062.md @@ -118,6 +134,8 @@ export class Beef { private bumpState?: BeefBumpSerializationState[] private needsSort: boolean = true + private sortResultCache?: BeefSortResult + private sortTxState?: BeefSortTxState[] constructor(version: number = BEEF_V2) { this.version = version @@ -235,6 +253,9 @@ export class Beef { } if (!this.bumpStateMatches()) { this.invalidateSerializationCaches() + this.sortResultCache = undefined + this.sortTxState = undefined + this.needsSort = true this.invalidateBumpIndexes() this.captureBumpState() } @@ -256,6 +277,8 @@ export class Beef { private markMutated(requiresSort: boolean = true): void { this.invalidateSerializationCaches() + this.sortResultCache = undefined + this.sortTxState = undefined if (requiresSort) { this.needsSort = true } @@ -274,6 +297,8 @@ export class Beef { for (const tx of this.txs) changed = tx.syncRawTxFromTransaction() || changed if (changed) { this.invalidateSerializationCaches() + this.sortResultCache = undefined + this.sortTxState = undefined this.needsSort = true this.rebuildTxIndexes() } @@ -1154,6 +1179,12 @@ export class Beef { txidOnly: string[] } { this.synchronizeNestedTransactionMutations() + this.synchronizeNestedBumpMutations() + if (this.sortResultCache != null && this.sortTxStateMatches()) { + return this.cloneSortResult(this.sortResultCache) + } + this.sortResultCache = undefined + this.sortTxState = undefined // Hashtable of valid txids (with proof or all inputs chain to proof) const validTxids: Record = {} @@ -1184,13 +1215,55 @@ export class Beef { this.invalidateSerializationCaches() this.rebuildTxIndexes() - return { + const sortResult: BeefSortResult = { missingInputs: Object.keys(missingInputs), notValid: txsNotValid.map(tx => tx.txid), valid: Object.keys(validTxids), withMissingInputs: txsMissingInputs.map(tx => tx.txid), txidOnly: txidOnly.map(tx => tx.txid) } + this.sortResultCache = sortResult + this.captureSortTxState() + return this.cloneSortResult(sortResult) + } + + private captureSortTxState(): void { + this.sortTxState = this.txs.map(tx => ({ + ref: tx, + txid: tx.txid, + bumpIndex: tx.bumpIndex, + isTxidOnly: tx.isTxidOnly, + inputTxids: [...tx.inputTxids] + })) + } + + private sortTxStateMatches(): boolean { + if (this.sortTxState?.length !== this.txs.length) return false + for (let index = 0; index < this.txs.length; index++) { + const tx = this.txs[index] + const state = this.sortTxState[index] + if ( + state.ref !== tx || + state.txid !== tx.txid || + state.bumpIndex !== tx.bumpIndex || + state.isTxidOnly !== tx.isTxidOnly || + state.inputTxids.length !== tx.inputTxids.length + ) return false + for (let inputIndex = 0; inputIndex < tx.inputTxids.length; inputIndex++) { + if (state.inputTxids[inputIndex] !== tx.inputTxids[inputIndex]) return false + } + } + return true + } + + private cloneSortResult(result: BeefSortResult): BeefSortResult { + return { + missingInputs: [...result.missingInputs], + notValid: [...result.notValid], + valid: [...result.valid], + withMissingInputs: [...result.withMissingInputs], + txidOnly: [...result.txidOnly] + } } /** @@ -1364,6 +1437,8 @@ export class Beef { c.bumpIndexByKey = undefined c.bumpIndexByTxid = undefined c.needsSort = this.needsSort + c.sortResultCache = this.sortResultCache == null ? undefined : this.cloneSortResult(this.sortResultCache) + if (c.sortResultCache != null) c.captureSortTxState() c.hexCache = this.hexCache c.rawBytesCache = this.rawBytesCache if (c.rawBytesCache != null) c.captureSerializationState() diff --git a/packages/sdk/src/transaction/__tests/Beef.test.ts b/packages/sdk/src/transaction/__tests/Beef.test.ts index ada98691c..7283e6cf9 100644 --- a/packages/sdk/src/transaction/__tests/Beef.test.ts +++ b/packages/sdk/src/transaction/__tests/Beef.test.ts @@ -521,6 +521,35 @@ describe('Beef tests', () => { } }) + test('sortTxs reuses an immutable result until BEEF contents change', () => { + const beef = new Beef() + beef.mergeTxidOnly('11'.repeat(32)) + const partition = jest.spyOn(beef as any, 'partitionTxs') + + const first = beef.sortTxs() + first.valid.length = 0 + const second = beef.sortTxs() + + expect(partition).toHaveBeenCalledTimes(1) + expect(second.valid).toEqual(['11'.repeat(32)]) + + beef.mergeTxidOnly('22'.repeat(32)) + expect(beef.sortTxs().valid).toEqual(['11'.repeat(32), '22'.repeat(32)]) + expect(partition).toHaveBeenCalledTimes(2) + }) + + test('sortTxs invalidates its cache after directly reachable dependency state changes', () => { + const beef = new Beef() + beef.mergeTxidOnly('11'.repeat(32)) + beef.sortTxs() + + beef.txs.push(BeefTx.fromTxid('22'.repeat(32))) + expect(beef.sortTxs().valid).toEqual(['11'.repeat(32), '22'.repeat(32)]) + + beef.txs[0].inputTxids.push('33'.repeat(32)) + expect(beef.sortTxs().missingInputs).toContain('33'.repeat(32)) + }) + test('10_deserialize beef with extra leaves', async () => { const b58Beef = b58Beef10 const beef = Beef.fromBinary(fromBase58(b58Beef)) diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index bf8ceafa4..b6665e6ee 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -6,6 +6,15 @@ attention to changes that materially alter behavior or extend functionality. ## wallet-toolbox (unreleased) +- Plan legacy `createAction` funding against the exact unreserved managed-change + set before persistence, claim the selected inputs atomically in one storage + transaction, and fail economically impossible fragmented wallets before + inserting a transaction row. Add targeted Knex and IndexedDB indexes and + remove IndexedDB's per-output transaction-status lookup. +- Avoid building full known-txid indexes on the common single-proof path and + report privacy-safe createAction candidate, funding, proof-fetch, BEEF merge, + trim, serialization, and known-history timings. BRC-103/104, AuthFetch, Auth + Express Middleware, AuthSocket, JSON-RPC, and wallet wire behavior are unchanged. - Keep remote-storage trace correlation inside the telemetry sink instead of adding unsupported headers to AuthFetch requests. BRC-103/104, AuthFetch, Auth Express Middleware, AuthSocket, JSON-RPC, and storage wire behavior are diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index f3d364a56..e377a8ab3 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -85,6 +85,38 @@ See [In-memory action batch planning](./docs/action-batch-planning.md) for capability-negotiated `noSend` planning, compact manifests, compressed binary pack transport, atomic commit, compatibility behavior, and retained benchmarks. +### `createAction` performance telemetry + +With the optional SDK telemetry sink enabled, legacy `createAction` reports +bounded-cardinality spans for input validation, record/output persistence, +funding candidate selection, fee-aware planning, atomic input claiming, input +assembly, proof fetch, BEEF merge, and final trim/serialization. +Only counts, byte sizes, fee totals, retry counts, and durations are reported; +transaction IDs, scripts, payloads, keys, and identities are not attributes. + +The planner uses the same exact / least-over / largest-under selection policy +as the historical allocator, but proves economic sufficiency before writing a +transaction and claims every selected input in one database transaction. Knex +storage automatically adds a composite funding-selection index on migration; +IndexedDB schema version 3 adds corresponding user/basket and outpoint indexes +and resolves transaction-status eligibility in one indexed pass. + +The retained fragmented-funding benchmark is runnable with: + +```bash +pnpm bench:create-action-funding +``` + +Against unmodified commit `c212b5ee7`, a representative 102-input SQLite plan +fell from 622 queries, 102 database transactions, and 107.3 ms to 17 queries, +one transaction, and 8.8 ms. Query and transaction counts remain flat when the +selected input count grows; networked database deployments should benefit most. + +Trace context remains local to the telemetry carrier and sink. Wallet Toolbox +does not add telemetry headers to AuthFetch, so BRC-103/104, Auth Express +Middleware, AuthSocket, JSON-RPC, and mixed-version remote storage behavior are +unchanged. + The codebase has detailed JSDoc annotations throughout — these will surface inline in editors like VS Code. ### Horizontal Storage scaling diff --git a/packages/wallet/wallet-toolbox/benchmarks/create-action-funding.bench.test.ts b/packages/wallet/wallet-toolbox/benchmarks/create-action-funding.bench.test.ts new file mode 100644 index 000000000..3917e5c7e --- /dev/null +++ b/packages/wallet/wallet-toolbox/benchmarks/create-action-funding.bench.test.ts @@ -0,0 +1,107 @@ +import { Validation } from '@bsv/sdk' +import { performance } from 'node:perf_hooks' +import { _tu } from '../test/utils/TestUtilsWalletStorage' +import { managedChangeOutputFields } from '../src/storage/methods/managedChange' +import { TableOutput, TableOutputBasket, TableTransaction } from '../src/storage/schema/tables' + +interface FundingMeasurement { + candidateCount: number + selectedInputCount: number + elapsedMs: number + queryCount: number + databaseTransactions: number +} + +async function measureFunding (candidateCount: number, outputSatoshis: number): Promise { + const ctx = await _tu.createLegacyWalletSQLiteCopy(`createActionFundingBench-${candidateCount}`, 'legacy') + try { + ctx.activeStorage.feeModel = { model: 'sat/kb', value: 100 } + const basket = (await ctx.activeStorage.findOutputBaskets({ + partial: { userId: ctx.userId, name: 'default' } + }))[0] as TableOutputBasket + await ctx.activeStorage.updateOutputBasket(basket.basketId, { + numberOfDesiredUTXOs: 0, + minimumDesiredUTXOValue: 1 + }) + const existing = await ctx.activeStorage.findOutputs({ + partial: { userId: ctx.userId, basketId: basket.basketId }, + noScript: true + }) + for (const output of existing) { + if (output.spendable) await ctx.activeStorage.updateOutput(output.outputId, { spendable: false }) + } + const source = (await ctx.activeStorage.findTransactions({ + partial: { userId: ctx.userId }, + status: ['completed'], + noRawTx: true + }))[0] as TableTransaction + const lockingScript = [0x76, 0xa9, 0x14, ...Array(20).fill(0x11), 0x88, 0xac] + for (let index = 0; index < candidateCount; index++) { + const now = new Date() + const output: TableOutput = { + outputId: 0, + userId: ctx.userId, + transactionId: source.transactionId, + basketId: basket.basketId, + spendable: true, + satoshis: 1_000, + vout: 10_000 + index, + txid: source.txid, + lockingScript, + scriptLength: lockingScript.length, + derivationPrefix: 'funding-benchmark-prefix', + derivationSuffix: `funding-benchmark-${index}`, + outputDescription: 'fragmented funding benchmark candidate', + ...managedChangeOutputFields, + created_at: now, + updated_at: now + } + await ctx.activeStorage.insertOutput(output) + } + + let queryCount = 0 + let databaseTransactions = 0 + const countQuery = (query: { sql?: string }): void => { + queryCount++ + if (/^begin\b/i.test(query.sql?.trim() ?? '')) databaseTransactions++ + } + ctx.activeStorage.knex.on('query', countQuery) + const args = Validation.validateCreateActionArgs({ + outputs: [{ + satoshis: outputSatoshis, + lockingScript: '51', + outputDescription: 'funding benchmark output' + }], + description: 'createAction fragmented funding benchmark', + options: { noSend: true, randomizeOutputs: false, returnTXIDOnly: true } + }) + const start = performance.now() + const result = await ctx.activeStorage.createAction({ userId: ctx.userId }, args) + const elapsedMs = performance.now() - start + ctx.activeStorage.knex.off('query', countQuery) + return { + candidateCount, + selectedInputCount: result.inputs.length, + elapsedMs, + queryCount, + databaseTransactions + } + } finally { + await ctx.wallet.destroy() + } +} + +describe('createAction fragmented funding benchmark', () => { + jest.setTimeout(120000) + + test('records representative fragmented funding latency and storage work', async () => { + const measurements = [ + await measureFunding(20, 5_000), + await measureFunding(147, 100_000) + ] + expect(measurements.every(measurement => measurement.selectedInputCount > 4)).toBe(true) + expect(measurements.every(measurement => measurement.databaseTransactions === 1)).toBe(true) + expect(measurements.every(measurement => measurement.queryCount <= 20)).toBe(true) + process.stdout.write(`${JSON.stringify({ measurements }, null, 2)}\n`) + }) +}) diff --git a/packages/wallet/wallet-toolbox/client/package.json b/packages/wallet/wallet-toolbox/client/package.json index 6ffc7412b..d9eed7c21 100644 --- a/packages/wallet/wallet-toolbox/client/package.json +++ b/packages/wallet/wallet-toolbox/client/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-client", - "version": "2.4.21", + "version": "2.4.22", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/mobile/package.json b/packages/wallet/wallet-toolbox/mobile/package.json index 40547fbc9..c130a0e83 100644 --- a/packages/wallet/wallet-toolbox/mobile/package.json +++ b/packages/wallet/wallet-toolbox/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-mobile", - "version": "2.4.21", + "version": "2.4.22", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/package.json b/packages/wallet/wallet-toolbox/package.json index 1c2fbe854..a095b9b53 100644 --- a/packages/wallet/wallet-toolbox/package.json +++ b/packages/wallet/wallet-toolbox/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox", - "version": "2.4.21", + "version": "2.4.22", "sideEffects": false, "type": "commonjs", "engines": { @@ -59,6 +59,7 @@ "operator:build": "pnpm build && tsc --project operator/tsconfig.json --pretty false", "operator": "pnpm operator:build && node .operator-build/cli.js", "bench:action-batch": "pnpm build && jest --runInBand --runTestsByPath benchmarks/action-batch.bench.test.ts --testPathIgnorePatterns=man.test.ts", + "bench:create-action-funding": "pnpm build && jest --runInBand --runTestsByPath benchmarks/create-action-funding.bench.test.ts --testPathIgnorePatterns=man.test.ts", "format:check": "pnpm --workspace-root exec prettier --check \"packages/wallet/wallet-toolbox/{README.md,jest.config.cjs,package.json,tsconfig*.json}\"", "lint": "oxlint src test benchmarks examples operator --deny-warnings", "lint:ci": "pnpm lint", diff --git a/packages/wallet/wallet-toolbox/src/Wallet.ts b/packages/wallet/wallet-toolbox/src/Wallet.ts index 35047233d..4e2b753f4 100644 --- a/packages/wallet/wallet-toolbox/src/Wallet.ts +++ b/packages/wallet/wallet-toolbox/src/Wallet.ts @@ -961,7 +961,23 @@ export class Wallet implements WalletInterface, ProtoWallet { args.options ??= {} args.options.trustSelf ||= this.trustSelf if (this.autoKnownTxids && args.options.knownTxids == null) { - args.options.knownTxids = this.getKnownTxids(args.options.knownTxids) + if (this.telemetry.enabled) { + args.options.knownTxids = this.telemetry.withSpan( + 'wallet.create_action.prepare_known_txids', + { + component: 'wallet-toolbox', + carrier: args, + attributes: { 'beef.tx_count': this.beef.txs.length } + }, + span => { + const knownTxids = this.getKnownTxids(args.options?.knownTxids) + span.end({ attributes: { 'beef.known_txid_count': knownTxids.length } }) + return knownTxids + } + ) + } else { + args.options.knownTxids = this.getKnownTxids(args.options.knownTxids) + } } const { auth, vargs } = this.validateAuthAndArgs(args, Validation.validateCreateActionArgs, logger) diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts b/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts index 4ac28a1ed..e4e54dc57 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts @@ -203,10 +203,17 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider async initDB(storageName?: string, storageIdentityKey?: string): Promise> { const chain = this.chain const maxOutputScript = 1024 - const db = await openDB(this.dbName, 2, { - upgrade(db) { + const db = await openDB(this.dbName, 3, { + upgrade(db, _oldVersion, _newVersion, transaction) { upgradeAllStoresV1(db) upgradeActionBatchStoresV2(db) + const outputs = transaction.objectStore('outputs') + if (!outputs.indexNames.contains('userId_basketId')) { + outputs.createIndex('userId_basketId', ['userId', 'basketId']) + } + if (!outputs.indexNames.contains('txid_vout_userId')) { + outputs.createIndex('txid_vout_userId', ['txid', 'vout', 'userId'], { unique: true }) + } if (!db.objectStoreNames.contains('settings')) { if (storageName == null || storageName === '' || storageIdentityKey == null || storageIdentityKey === '') { throw new WERR_INVALID_OPERATION('migrate must be called before first access') @@ -415,11 +422,73 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider txStatus, noScript: true } - let count = 0 + const outputIds: number[] = [] await this.filterOutputs(args, r => { - if (isAutoSpendableChangeOutput(r)) count++ + if (isAutoSpendableChangeOutput(r)) outputIds.push(r.outputId) }) - return count + const reserved = await this.findReservedActionBatchOutputIds(outputIds) + return outputIds.length - reserved.length + } + + override async findTransactionStatusesByIds( + userId: number, + transactionIds: number[], + trx?: TrxToken + ): Promise> { + const statuses = new Map() + if (transactionIds.length === 0) return statuses + const dbTrx = this.toDbTrx(['transactions'], 'readonly', trx) + const store = dbTrx.objectStore('transactions') + for (const transactionId of new Set(transactionIds)) { + const transaction = await store.get(transactionId) + if (transaction?.userId === userId) statuses.set(transactionId, transaction.status) + } + if (trx == null) await dbTrx.done + return statuses + } + + private async findOutputsByOutpointsInternal( + userId: number, + outpoints: Array<{ txid: string; vout: number }>, + trx?: TrxToken, + noScript = false + ): Promise> { + const byOutpoint: Record = {} + if (outpoints.length === 0) return byOutpoint + const dbTrx = this.toDbTrx( + noScript ? ['outputs'] : ['outputs', 'proven_txs', 'proven_tx_reqs'], + 'readonly', + trx + ) + const index = dbTrx.objectStore('outputs').index('txid_vout_userId') + const unique = [...new Map(outpoints.map(outpoint => [`${outpoint.txid}.${outpoint.vout}`, outpoint])).values()] + const rows = await Promise.all(unique.map(async outpoint => + await index.get([outpoint.txid, outpoint.vout, userId]) + )) + for (const row of rows) { + if (row == null) continue + if (!noScript) await this.validateOutputScript(row, dbTrx) + byOutpoint[`${String(row.txid)}.${row.vout}`] = this.validateEntity(row) + } + if (trx == null) await dbTrx.done + return byOutpoint + } + + override async findOutputsByOutpoints( + userId: number, + outpoints: Array<{ txid: string; vout: number }>, + trx?: TrxToken + ): Promise> { + return await this.findOutputsByOutpointsInternal(userId, outpoints, trx) + } + + override async findOutputsByOutpointsForUpdate( + userId: number, + outpoints: Array<{ txid: string; vout: number }>, + trx: TrxToken, + noScript = false + ): Promise> { + return await this.findOutputsByOutpointsInternal(userId, outpoints, trx, noScript) } async findCertificatesAuth(auth: AuthId, args: FindCertificatesArgs): Promise { @@ -1546,6 +1615,14 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider .index('transactionId_vout_userId') .openCursor([partial.transactionId, partial.vout, partial.userId], direction) } + if (partial?.txid != null && partial.txid !== '' && partial?.vout !== undefined) { + return store + .index('txid_vout_userId') + .openCursor([partial.txid, partial.vout, partial.userId], direction) + } + if (partial?.basketId !== undefined) { + return store.index('userId_basketId').openCursor([partial.userId, partial.basketId], direction) + } return store.index('userId').openCursor(partial.userId, direction) } if (partial?.transactionId !== undefined) @@ -1574,6 +1651,18 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider const dbTrx = this.toDbTrx(stores, 'readonly', args.trx) const direction: IDBCursorDirection = args.orderDescending === true ? 'prev' : 'next' const store = dbTrx.objectStore('outputs') + let validTransactionIds: Set | undefined + if (args.txStatus != null) { + validTransactionIds = new Set() + const transactions = dbTrx.objectStore('transactions') + for (const status of args.txStatus) { + const index = args.partial.userId === undefined + ? transactions.index('status') + : transactions.index('status_userId') + const key = args.partial.userId === undefined ? status : [status, args.partial.userId] + for (const transactionId of await index.getAllKeys(key)) validTransactionIds.add(Number(transactionId)) + } + } const cursor = await this.openOutputsCursor(store, args.partial, direction) await scanCursor( cursor, @@ -1582,14 +1671,7 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider args.paged?.limit, async r => { if (!matchesOutputPartial(r, args.partial)) return false - if (args.txStatus !== undefined) { - const txCount = await this.countTransactions({ - partial: { transactionId: r.transactionId }, - status: args.txStatus, - trx: dbTrx - }) - if (txCount === 0) return false - } + if (validTransactionIds != null && !validTransactionIds.has(r.transactionId)) return false if ( tagIds != null && tagIds.length > 0 && diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts b/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts index fff93a762..b2fcea845 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts @@ -1408,11 +1408,50 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide .whereNot('o.derivationPrefix', '') .whereNotNull('o.derivationSuffix') .whereNot('o.derivationSuffix', '') + .whereNotExists(function () { + void this.select(1) + .from('action_batch_outputs as abo') + .whereRaw('abo.outputId = o.outputId') + }) .whereIn('t.status', status) const count = await this.getCount(q) return count } + override async findAvailableManagedChangeInputs ( + userId: number, + basketId: number, + excludeSending: boolean, + trx?: TrxToken + ): Promise { + const statuses: TransactionStatus[] = ['completed', 'unproven'] + if (!excludeSending) statuses.push('sending') + const rows = await this.toDb(trx)('outputs as o') + .join('transactions as t', 'o.transactionId', 't.transactionId') + .where({ + 'o.userId': userId, + 'o.basketId': basketId, + 'o.spendable': true, + 'o.type': managedChangeOutputFields.type, + 'o.change': managedChangeOutputFields.change, + 'o.providedBy': managedChangeOutputFields.providedBy, + 'o.purpose': managedChangeOutputFields.purpose + }) + .whereNull('o.spentBy') + .whereNotNull('o.derivationPrefix') + .whereNot('o.derivationPrefix', '') + .whereNotNull('o.derivationSuffix') + .whereNot('o.derivationSuffix', '') + .whereNotExists(function () { + void this.select(1) + .from('action_batch_outputs as abo') + .whereRaw('abo.outputId = o.outputId') + }) + .whereIn('t.status', statuses) + .select(outputColumnsWithoutLockingScript.map(column => `o.${column}`)) + return this.validateEntities(rows, undefined, ['spendable', 'change']) + } + override async findOutputsByIds (outputIds: number[], trx?: TrxToken): Promise> { const byId: Record = {} if (outputIds.length < 1) return byId @@ -1455,7 +1494,8 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide override async findOutputsByOutpointsForUpdate ( userId: number, outpoints: Array<{ txid: string, vout: number }>, - trx: TrxToken + trx: TrxToken, + noScript = false ): Promise> { const byOutpoint: Record = {} if (outpoints.length < 1) return byOutpoint @@ -1468,12 +1508,27 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide .forUpdate() const filteredRows = rows.filter(r => outpointSet.has(`${String(r.txid)}.${String(r.vout)}`)) for (const row of this.validateEntities(filteredRows, undefined, ['spendable', 'change'])) { - await this.validateOutputScript(row, trx) + if (!noScript) await this.validateOutputScript(row, trx) byOutpoint[`${String(row.txid)}.${String(row.vout)}`] = row } return byOutpoint } + override async findTransactionStatusesByIds ( + userId: number, + transactionIds: number[], + trx?: TrxToken + ): Promise> { + const statuses = new Map() + if (transactionIds.length === 0) return statuses + const rows = await this.toDb(trx)>('transactions') + .where('userId', userId) + .whereIn('transactionId', [...new Set(transactionIds)]) + .select('transactionId', 'status') + for (const row of rows) statuses.set(row.transactionId, row.status) + return statuses + } + override async findOrInsertOutputBasketsBulk ( userId: number, names: string[], @@ -1643,6 +1698,15 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide return r } + override async markChangeInputsSpent (outputIds: number[], transactionId: number, trx: TrxToken): Promise { + if (outputIds.length === 0) return 0 + return await this.toDb(trx)('outputs') + .whereIn('outputId', outputIds) + .where('spendable', true) + .whereNull('spentBy') + .update({ spendable: false, spentBy: transactionId }) + } + /** Convert null→undefined and Buffer→number[] on a retrieved entity in-place. */ private deserialiseFromKnex(entity: T): void { for (const key of Object.keys(entity as object)) { diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts index 9cbe62305..5b0efc3bb 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts @@ -108,6 +108,7 @@ import { putActionBatchBlob as putBatchBlob, putActionBatchPack as putBatchPack } from './methods/actionBatchBlobs' +import { availableManagedChange } from './methods/availableManagedChange' export abstract class StorageProvider extends StorageReaderWriter implements WalletStorageProvider { isDirty = false @@ -157,6 +158,42 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal transactionId: number ): Promise + /** Mark a planned set of change inputs spent within the caller's transaction. */ + async markChangeInputsSpent(outputIds: number[], transactionId: number, trx: TrxToken): Promise { + let updated = 0 + const current = await this.findOutputsByIds(outputIds, trx) + for (const outputId of outputIds) { + const output = current[outputId] + if (output == null || !output.spendable || output.spentBy != null) continue + updated += await this.updateOutput(outputId, { spendable: false, spentBy: transactionId }, trx) + } + return updated + } + + /** Return unreserved wallet-managed outputs eligible for automatic funding. */ + async findAvailableManagedChangeInputs( + userId: number, + basketId: number, + excludeSending: boolean, + trx?: TrxToken + ): Promise { + return await availableManagedChange(this, userId, basketId, excludeSending, trx) + } + + /** Read the current status of a set of source transactions without loading raw transaction bytes. */ + async findTransactionStatusesByIds( + userId: number, + transactionIds: number[], + trx?: TrxToken + ): Promise> { + const statuses = new Map() + for (const transactionId of new Set(transactionIds)) { + const transaction = await this.findTransactionById(transactionId, trx, true) + if (transaction?.userId === userId) statuses.set(transactionId, transaction.status) + } + return statuses + } + abstract getProvenOrRawTx(txid: string, trx?: TrxToken): Promise abstract getRawTxOfKnownValidTransaction( txid?: string, @@ -321,9 +358,14 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal async findOutputsByOutpointsForUpdate( userId: number, outpoints: Array<{ txid: string; vout: number }>, - trx: TrxToken + trx: TrxToken, + noScript = false ): Promise> { - return await this.findOutputsByOutpoints(userId, outpoints, trx) + const byOutpoint = await this.findOutputsByOutpoints(userId, outpoints, trx) + // Backends that cannot skip hydration remain correct; optimized backends may + // use noScript to keep raw-transaction I/O outside the write lock. + void noScript + return byOutpoint } async findOrInsertOutputBasketsBulk( diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts index 978effad8..6554b2af5 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts @@ -6,6 +6,8 @@ import { StorageIdb } from '../StorageIdb' import { StorageProvider, StorageProviderOptions } from '../StorageProvider' import { TableOutput, TableOutputBasket, TableTransaction, TableUser } from '../schema/tables' import { TableActionBatch } from '../schema/tables/TableActionBatch' +import { StorageIdbSchema } from '../schema/StorageIdbSchema' +import { openDB } from 'idb' import 'fake-indexeddb/auto' describe('StorageIdb tests', () => { @@ -18,8 +20,82 @@ describe('StorageIdb tests', () => { try { const r = await storage.migrate(`storageIdbTest-${Date.now()}`, '42'.repeat(32)) const db = storage.db - expect(r).toBe('2') + expect(r).toBe('3') expect(db).toBeTruthy() + expect(db?.transaction('outputs').objectStore('outputs').indexNames.contains('userId_basketId')).toBe(true) + expect(db?.transaction('outputs').objectStore('outputs').indexNames.contains('txid_vout_userId')).toBe(true) + expect(db?.transaction('certificates').objectStore('certificates') + .indexNames.contains('userId_basketId')).toBe(false) + } finally { + await resetStorage(storage) + } + }) + + test('upgrades a version 2 outputs store with the funding compound index', async () => { + const options: StorageProviderOptions = StorageProvider.createStorageBaseOptions('main') + const storage = new StorageIdb(options) + storage.dbName = `storageIdbUpgrade-${randomUUID()}` + const legacy = await openDB(storage.dbName, 2, { + upgrade (db) { + const outputs = db.createObjectStore('outputs', { keyPath: 'outputId', autoIncrement: true }) + outputs.createIndex('userId', 'userId') + outputs.createIndex('transactionId', 'transactionId') + outputs.createIndex('basketId', 'basketId') + outputs.createIndex('spentBy', 'spentBy') + outputs.createIndex('transactionId_vout_userId', ['transactionId', 'vout', 'userId'], { unique: true }) + } + }) + legacy.close() + + try { + const upgraded = await storage.initDB('version 2 upgrade test', '42'.repeat(32)) + expect(upgraded.version).toBe(3) + expect(upgraded.transaction('outputs').objectStore('outputs') + .indexNames.contains('userId_basketId')).toBe(true) + expect(upgraded.transaction('outputs').objectStore('outputs') + .indexNames.contains('txid_vout_userId')).toBe(true) + upgraded.close() + } finally { + await resetStorage(storage) + } + }) + + test('funding candidates exclude invalid source statuses and active batch reservations', async () => { + const storage = await makeStorage() + try { + const userId = await insertUser(storage) + const basketId = await insertBasket(storage, userId) + const availableTxId = await insertTransaction(storage, userId, { + status: 'completed', txid: '11'.repeat(32) + }) + const reservedTxId = await insertTransaction(storage, userId, { + status: 'completed', txid: '12'.repeat(32) + }) + const failedTxId = await insertTransaction(storage, userId, { + status: 'failed', txid: '13'.repeat(32) + }) + const availableOutputId = await insertOutput(storage, userId, availableTxId, basketId, { + txid: '11'.repeat(32), satoshis: 100 + }) + const reservedOutputId = await insertOutput(storage, userId, reservedTxId, basketId, { + txid: '12'.repeat(32), satoshis: 200 + }) + await insertOutput(storage, userId, failedTxId, basketId, { + txid: '13'.repeat(32), satoshis: 300 + }) + const batch = makeActionBatch(userId, 'funding-candidate-reservation') + await storage.insertActionBatch(batch) + const now = new Date() + await storage.reserveActionBatchOutputs([{ + actionBatchId: batch.actionBatchId, + outputId: reservedOutputId, + created_at: now, + updated_at: now + }]) + + const available = await storage.findAvailableManagedChangeInputs(userId, basketId, true) + expect(available.map(output => output.outputId)).toEqual([availableOutputId]) + await expect(storage.countChangeInputs(userId, basketId, true)).resolves.toBe(1) } finally { await resetStorage(storage) } diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts new file mode 100644 index 000000000..1cd90cc77 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts @@ -0,0 +1,196 @@ +import { TelemetryEvent, Validation } from '@bsv/sdk' +import { _tu, TestWalletNoSetup } from '../../../test/utils/TestUtilsWalletStorage' +import { StorageKnex } from '../StorageKnex' +import { TableOutput, TableOutputBasket, TableTransaction } from '../schema/tables' +import { managedChangeOutputFields } from '../methods/managedChange' + +describe('createAction funding performance', () => { + jest.setTimeout(120000) + let ctx: TestWalletNoSetup + + beforeEach(async () => { + ctx = await _tu.createLegacyWalletSQLiteCopy( + expect.getState().currentTestName ?? 'createActionPerformance', + 'legacy' + ) + ctx.activeStorage.feeModel = { model: 'sat/kb', value: 100 } + }) + + afterEach(async () => { + await ctx.wallet.destroy() + }) + + test('claims a fragmented funding plan in one database transaction', async () => { + await replaceFundingCandidates(20, 1_000) + let databaseTransactions = 0 + const countTransactions = (query: { sql?: string }): void => { + if (/^begin\b/i.test(query.sql?.trim() ?? '')) databaseTransactions++ + } + ctx.activeStorage.knex.on('query', countTransactions) + const markSpent = jest.spyOn(ctx.activeStorage, 'markChangeInputsSpent') + const readStatuses = jest.spyOn(ctx.activeStorage, 'findTransactionStatusesByIds') + try { + const result = await ctx.activeStorage.createAction( + { userId: ctx.userId }, + actionArgs(5_000) + ) + + expect(result.inputs.length).toBeGreaterThan(4) + expect(markSpent).toHaveBeenCalledTimes(1) + expect(markSpent.mock.calls[0][0]).toHaveLength(result.inputs.length) + expect(readStatuses).toHaveBeenCalledTimes(1) + expect(readStatuses.mock.calls[0][1]).toHaveLength(1) + expect(databaseTransactions).toBe(1) + } finally { + ctx.activeStorage.knex.off('query', countTransactions) + } + }) + + test('hydrates offloaded funding scripts once per source and outside the locked lookup', async () => { + await replaceFundingCandidates(20, 1_000, true) + const lockedLookup = jest.spyOn(ctx.activeStorage, 'findOutputsByOutpointsForUpdate') + const getRawTx = jest.spyOn(ctx.activeStorage, 'getRawTxOfKnownValidTransaction') + + const result = await ctx.activeStorage.createAction({ userId: ctx.userId }, actionArgs(5_000)) + + expect(result.inputs.length).toBeGreaterThan(4) + expect(lockedLookup).toHaveBeenCalledTimes(1) + expect(lockedLookup.mock.calls[0][3]).toBe(true) + expect(getRawTx).toHaveBeenCalledTimes(1) + expect(getRawTx.mock.calls[0]).toEqual([expect.any(String)]) + }) + + test('rolls back a concurrent bulk-claim conflict before replanning', async () => { + await replaceFundingCandidates(20, 1_000) + const original = ctx.activeStorage.markChangeInputsSpent.bind(ctx.activeStorage) + const markSpent = jest.spyOn(ctx.activeStorage, 'markChangeInputsSpent') + .mockResolvedValueOnce(0) + .mockImplementation(original) + + const result = await ctx.activeStorage.createAction({ userId: ctx.userId }, actionArgs(5_000)) + + expect(result.inputs.length).toBeGreaterThan(4) + expect(markSpent).toHaveBeenCalledTimes(2) + const spent = (await ctx.activeStorage.findOutputs({ + partial: { userId: ctx.userId }, + noScript: true + })).filter(output => output.vout >= 10_000 && output.spentBy != null) + expect(spent).toHaveLength(result.inputs.length) + }) + + test('rejects economically insufficient dust before writing a transaction row', async () => { + await replaceFundingCandidates(147, 1) + const insertTransaction = jest.spyOn(ctx.activeStorage, 'insertTransaction') + + await expect(ctx.activeStorage.createAction( + { userId: ctx.userId }, + actionArgs(1_000) + )).rejects.toMatchObject({ code: 'WERR_INSUFFICIENT_FUNDS' }) + + expect(insertTransaction).not.toHaveBeenCalled() + }) + + test('reports funding and BEEF phases with bounded cardinality attributes', async () => { + await replaceFundingCandidates(1, 5_000) + const events: TelemetryEvent[] = [] + const storage = new StorageKnex({ + ...StorageKnex.defaultOptions(), + chain: ctx.chain, + knex: ctx.activeStorage.knex, + feeModel: ctx.activeStorage.feeModel, + telemetry: { sink: { capture: event => events.push(event) } } + }) + await storage.makeAvailable() + + await storage.createAction({ userId: ctx.userId }, actionArgs(1_000, false)) + + const byName = new Map(events.map(event => [event.name, event])) + expect(byName.get('wallet.storage.create_action')).toMatchObject({ spanStatus: 'ok' }) + expect(byName.get('wallet.storage.create_action.validate')?.attributes).toMatchObject({ + 'action.validated_input_count': 0, + 'action.no_send_change_input_count': 0 + }) + expect(byName.get('wallet.storage.create_action.create_record')?.attributes).toMatchObject({ + 'action.transaction_record_created': true + }) + expect(byName.get('wallet.storage.create_action.funding_candidates')?.attributes).toMatchObject({ + 'funding.candidate_count': 1, + 'funding.candidate_satoshis': 5_000 + }) + expect(byName.get('wallet.storage.create_action.funding_claim')?.attributes).toMatchObject({ + 'funding.claim_retry_count': 0, + 'funding.source_transaction_count': 1, + 'funding.hydrated_script_count': 0, + 'funding.script_source_transaction_count': 0 + }) + expect(byName.get('wallet.storage.create_action.beef_fetch')?.attributes).toMatchObject({ + 'beef.allocated_change_count': 1, + 'beef.known_txid_count': 0 + }) + expect(byName.get('wallet.storage.create_action.persist_outputs')?.attributes).toMatchObject({ + 'action.persisted_output_count': expect.any(Number) + }) + expect(byName.get('wallet.storage.create_action.assemble_inputs')?.attributes).toMatchObject({ + 'action.result_input_count': 1 + }) + const serialized = JSON.stringify(events) + expect(serialized).not.toContain(ctx.identityKey) + expect(serialized).not.toContain('lockingScript') + }) + + function actionArgs (satoshis: number, returnTXIDOnly = true): Validation.ValidCreateActionArgs { + return Validation.validateCreateActionArgs({ + outputs: [{ satoshis, lockingScript: '51', outputDescription: 'performance test output' }], + description: 'createAction funding performance test', + options: { noSend: true, randomizeOutputs: false, returnTXIDOnly } + }) + } + + async function replaceFundingCandidates (count: number, satoshis: number, offloadScript = false): Promise { + const basket = (await ctx.activeStorage.findOutputBaskets({ + partial: { userId: ctx.userId, name: 'default' } + }))[0] as TableOutputBasket + await ctx.activeStorage.updateOutputBasket(basket.basketId, { + numberOfDesiredUTXOs: 0, + minimumDesiredUTXOValue: 1 + }) + const existing = await ctx.activeStorage.findOutputs({ + partial: { userId: ctx.userId, basketId: basket.basketId }, + noScript: true + }) + for (const output of existing) { + if (output.spendable) await ctx.activeStorage.updateOutput(output.outputId, { spendable: false }) + } + const source = (await ctx.activeStorage.findTransactions({ + partial: { userId: ctx.userId }, + status: ['completed'], + noRawTx: true + }))[0] as TableTransaction + expect(source?.txid).toBeDefined() + const lockingScript = [0x76, 0xa9, 0x14, ...Array(20).fill(0x11), 0x88, 0xac] + for (let index = 0; index < count; index++) { + const now = new Date() + const output: TableOutput = { + outputId: 0, + userId: ctx.userId, + transactionId: source.transactionId, + basketId: basket.basketId, + spendable: true, + spentBy: undefined, + satoshis, + vout: 10_000 + index, + txid: source.txid, + lockingScript: offloadScript ? undefined : lockingScript, + scriptLength: offloadScript ? 1 : lockingScript.length, + scriptOffset: offloadScript ? 1 : undefined, + derivationPrefix: 'funding-performance-prefix', + derivationSuffix: `funding-performance-${index}`, + outputDescription: 'fragmented funding candidate', + ...managedChangeOutputFields, + created_at: now, + updated_at: now + } + await ctx.activeStorage.insertOutput(output) + } + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts index 1e9f02821..4613235c6 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts @@ -89,6 +89,24 @@ describe('getBeefForTransaction tests', () => { expect(fresh.findTxid(txid)).toBeDefined() }) + test('does not index an entire known-txid history for a single known root', async () => { + const storage = new ProtoStorage('main') + const txid = '12'.repeat(32) + const knownTxids = Array.from({ length: 10_000 }, (_, index) => index.toString(16).padStart(64, '0')) + knownTxids.push(txid) + Object.defineProperty(knownTxids, Symbol.iterator, { + value: () => { throw new Error('the common single-root path must not iterate the full history') } + }) + + const beef = await storage.getBeefForTransaction(txid, { + ignoreStorage: true, + ignoreServices: true, + knownTxids + }) + + expect(beef.findTxid(txid)?.isTxidOnly).toBe(true) + }) + test('uses storage BEEF and forwards proof-validation policy', async () => { const storage = new ProtoStorage('main') const txid = '22'.repeat(32) diff --git a/packages/wallet/wallet-toolbox/src/storage/idbHelpers.ts b/packages/wallet/wallet-toolbox/src/storage/idbHelpers.ts index 1ccea39b0..c75d8820a 100644 --- a/packages/wallet/wallet-toolbox/src/storage/idbHelpers.ts +++ b/packages/wallet/wallet-toolbox/src/storage/idbHelpers.ts @@ -376,6 +376,8 @@ export function upgradeCommissions (db: IDBPDatabase): void { export function upgradeOutputs (db: IDBPDatabase): void { const store = db.createObjectStore('outputs', { keyPath: 'outputId', autoIncrement: true }) store.createIndex('userId', 'userId') + store.createIndex('userId_basketId', ['userId', 'basketId']) + store.createIndex('txid_vout_userId', ['txid', 'vout', 'userId'], { unique: true }) store.createIndex('transactionId', 'transactionId') store.createIndex('basketId', 'basketId') store.createIndex('spentBy', 'spentBy') diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/actionBatch.ts b/packages/wallet/wallet-toolbox/src/storage/methods/actionBatch.ts index c2d7d3080..63bd55927 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/actionBatch.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/actionBatch.ts @@ -125,24 +125,6 @@ export async function cleanupExpiredActionBatches (storage: StorageProvider): Pr return released } -async function availableManagedChange ( - storage: StorageProvider, - userId: number, - basketId: number, - excludeSending: boolean, - trx?: TrxToken -): Promise { - const statuses: TransactionStatus[] = ['completed', 'unproven'] - if (!excludeSending) statuses.push('sending') - const outputs = (await storage.findOutputs({ - partial: { userId, basketId, spendable: true }, - txStatus: statuses, - trx - })).filter(isAutoSpendableChangeOutput) - const reserved = new Set(await storage.findReservedActionBatchOutputIds(outputs.map(o => o.outputId), trx)) - return outputs.filter(output => output.spentBy == null && !reserved.has(output.outputId)) -} - function sourceOutputFromBeef ( beef: Beef, outpoint: { txid: string, vout: number } @@ -386,8 +368,8 @@ export async function beginActionBatch ( ) const noSendChange = await resolveNoSendChangeOutputs(storage, userId, args.firstAction) const fixedOutputIds = new Set([...explicit.outputs, ...noSendChange.outputs].map(output => output.outputId)) - const available = (await availableManagedChange( - storage, userId, changeBasket.basketId, !args.firstAction.isDelayed + const available = (await storage.findAvailableManagedChangeInputs( + userId, changeBasket.basketId, !args.firstAction.isDelayed )).filter(output => !fixedOutputIds.has(output.outputId)) const target = estimateFirstActionTarget( storage, @@ -448,7 +430,7 @@ export async function extendActionBatch ( const batch = requireLiveBatch(await storage.findActionBatch(userId, args.batchId)) const basket = verifyOne(await storage.findOutputBaskets({ partial: { userId, name: 'default' } })) const alreadyReserved = await storage.findActionBatchOutputIds(batch.actionBatchId) - const available = await availableManagedChange(storage, userId, basket.basketId, false) + const available = await storage.findAvailableManagedChangeInputs(userId, basket.basketId, false) if (!Number.isSafeInteger(args.requestedOutputs) || args.requestedOutputs < 0) { throw new WERR_INVALID_PARAMETER('requestedOutputs', 'non-negative safe integer') } diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/availableManagedChange.ts b/packages/wallet/wallet-toolbox/src/storage/methods/availableManagedChange.ts new file mode 100644 index 000000000..beb91fe08 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/methods/availableManagedChange.ts @@ -0,0 +1,30 @@ +import type { TransactionStatus } from '../../sdk/types' +import type { TrxToken } from '../../sdk/WalletStorage.interfaces' +import type { StorageProvider } from '../StorageProvider' +import type { TableOutput } from '../schema/tables/TableOutput' +import { isAutoSpendableChangeOutput, managedChangeOutputFields } from './managedChange' + +/** + * Return the exact set of wallet-managed outputs currently eligible for + * automatic funding. Keeping this predicate shared prevents the planner, + * allocator, action-batch reservations, and availability count from drifting. + */ +export async function availableManagedChange ( + storage: StorageProvider, + userId: number, + basketId: number, + excludeSending: boolean, + trx?: TrxToken +): Promise { + const statuses: TransactionStatus[] = ['completed', 'unproven'] + if (!excludeSending) statuses.push('sending') + const outputs = (await storage.findOutputs({ + partial: { userId, basketId, spendable: true, ...managedChangeOutputFields }, + txStatus: statuses, + noScript: true, + trx + })).filter(isAutoSpendableChangeOutput) + if (outputs.length === 0) return outputs + const reserved = new Set(await storage.findReservedActionBatchOutputIds(outputs.map(output => output.outputId), trx)) + return outputs.filter(output => !reserved.has(output.outputId)) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts b/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts index d7bd3c62a..2c0988776 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts @@ -1,8 +1,9 @@ -import { Beef, OriginatorDomainNameStringUnder250Bytes, Random, Script, Utils, Validation } from '@bsv/sdk' +import { Beef, OriginatorDomainNameStringUnder250Bytes, Random, Script, TelemetrySpan, Utils, Validation } from '@bsv/sdk' import { generateChangeSdk, GenerateChangeSdkChangeInput, GenerateChangeSdkParams, + GenerateChangeSdkResult, maxPossibleSatoshis } from './generateChange' import { StorageProvider, validateStorageFeeModel } from '../StorageProvider' @@ -16,8 +17,8 @@ import { StorageProvidedBy } from '../../sdk/WalletStorage.interfaces' import { - WERR_INSUFFICIENT_FUNDS, WERR_INTERNAL, + WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER, WERR_REVIEW_ACTIONS } from '../../sdk/WERR_errors' @@ -38,10 +39,10 @@ import { TableTransaction } from '../schema/tables/TableTransaction' import { EntityProvenTx } from '../schema/entities/EntityProvenTx' import { throwDummyReviewActions } from '../../Wallet' import { createStorageServiceChargeScript } from './offsetKey' -import { transactionSize } from './utils' import { WalletError } from '../../sdk' import { isAutoSpendableChangeOutput } from './managedChange' -import { randomizeOutputVouts as randomizePlannedOutputVouts } from './actionPlanning' +import { randomizeOutputVouts as randomizePlannedOutputVouts, selectCanonicalChange } from './actionPlanning' +import { TransactionStatus } from '../../sdk/types' let disableDoubleSpendCheckForTest = true export function setDisableDoubleSpendCheckForTest(v: boolean) { @@ -53,6 +54,40 @@ export async function createAction( auth: AuthId, vargs: Validation.ValidCreateActionArgs, _originator?: OriginatorDomainNameStringUnder250Bytes +): Promise { + if (!storage.telemetry.enabled) return await createActionCore(storage, auth, vargs) + return await storage.telemetry.withSpan( + 'wallet.storage.create_action', + { + component: 'wallet-storage', + carrier: vargs, + attributes: { + 'action.fixed_input_count': vargs.inputs.length, + 'action.fixed_output_count': vargs.outputs.length, + 'action.known_txid_count': vargs.options.knownTxids?.length ?? 0, + 'action.is_delayed': vargs.isDelayed, + 'action.is_no_send': vargs.isNoSend + } + }, + async span => { + const result = await createActionCore(storage, auth, vargs, span) + span.end({ + attributes: { + 'action.result_input_count': result.inputs.length, + 'action.result_output_count': result.outputs.length, + 'action.input_beef_bytes': result.inputBeef?.length ?? 0 + } + }) + return result + } + ) +} + +async function createActionCore( + storage: StorageProvider, + auth: AuthId, + vargs: Validation.ValidCreateActionArgs, + parent?: TelemetrySpan ): Promise { const logger = vargs.logger logger?.group('storage createAction') @@ -80,38 +115,80 @@ export async function createAction( * - Create result inputs with source locking scripts * - Create result outputs with new locking scripts. * - Create and return result. - */ + */ const userId = auth.userId! - const { storageBeef, beef, xinputs } = await validateRequiredInputs(storage, userId, vargs) - logger?.log('validated required inputs') - const xoutputs = validateRequiredOutputs(storage, userId, vargs) - logger?.log('validated required outputs') - - const changeBasketName = 'default' - const changeBasket = verifyOne( - await storage.findOutputBaskets({ - partial: { userId, name: changeBasketName } - }), - `Invalid outputGeneration basket "${changeBasketName}"` + const validated = await traceStorageStep( + storage, + 'wallet.storage.create_action.validate', + parent, + { + 'action.fixed_input_count': vargs.inputs.length, + 'action.fixed_output_count': vargs.outputs.length + }, + async span => { + const requiredInputs = await validateRequiredInputs(storage, userId, vargs) + logger?.log('validated required inputs') + const xoutputs = validateRequiredOutputs(storage, userId, vargs) + logger?.log('validated required outputs') + + const changeBasketName = 'default' + const changeBasket = verifyOne( + await storage.findOutputBaskets({ + partial: { userId, name: changeBasketName } + }), + `Invalid outputGeneration basket "${changeBasketName}"` + ) + logger?.log('found change basket') + + const noSendChangeIn = await validateNoSendChange(storage, userId, vargs, changeBasket) + logger?.log('validated noSendChange') + span?.end({ + attributes: { + 'action.validated_input_count': requiredInputs.xinputs.length, + 'action.validated_output_count': xoutputs.length, + 'action.no_send_change_input_count': noSendChangeIn.length, + 'action.validated_beef_tx_count': requiredInputs.beef.txs.length + } + }) + return { ...requiredInputs, xoutputs, changeBasket, noSendChangeIn } + } ) - logger?.log('found change basket') - - const noSendChangeIn = await validateNoSendChange(storage, userId, vargs, changeBasket) - logger?.log('validated noSendChange') - - const availableChangeCount = await storage.countChangeInputs(userId, changeBasket.basketId, !vargs.isDelayed) - logger?.log(`counted change inputs ${availableChangeCount}`) + const { storageBeef, beef, xinputs, xoutputs, changeBasket, noSendChangeIn } = validated const feeModel = validateStorageFeeModel(storage.feeModel) logger?.log(`validated fee model ${JSON.stringify(feeModel)}`) - await preflightInsufficientFundsFastPath(vargs, xinputs, xoutputs, noSendChangeIn, availableChangeCount, feeModel) - logger?.log('passed insufficient-funds preflight') + const initialFundingPlan = await prepareFundingPlan( + storage, + userId, + vargs, + xinputs, + xoutputs, + changeBasket, + noSendChangeIn, + feeModel, + parent + ) + logger?.log(`planned funding from ${initialFundingPlan.availableChangeCount} change inputs`) let newTx: TableTransaction | undefined try { - newTx = await createNewTxRecord(storage, userId, vargs, storageBeef) + const storageBeefBytes = storageBeef.toBinary() + newTx = await traceStorageStep( + storage, + 'wallet.storage.create_action.create_record', + parent, + { + 'action.label_count': vargs.labels.length, + 'action.storage_beef_bytes': storageBeefBytes.length + }, + async span => { + const transaction = await createNewTxRecord(storage, userId, vargs, storageBeefBytes) + span?.end({ attributes: { 'action.transaction_record_created': true } }) + return transaction + } + ) logger?.log('created new transaction record') const ctx: CreateTransactionSdkContext = { @@ -119,13 +196,12 @@ export async function createAction( xoutputs, changeBasket, noSendChangeIn, - availableChangeCount, feeModel, transactionId: newTx.transactionId } const { allocatedChange, changeOutputs, derivationPrefix, maxPossibleSatoshisAdjustment } = - await fundNewTransactionSdk(storage, userId, vargs, ctx) + await fundNewTransactionSdk(storage, userId, vargs, ctx, initialFundingPlan, parent) logger?.log('funded new transaction') if (maxPossibleSatoshisAdjustment != null) { @@ -138,15 +214,41 @@ export async function createAction( // The satoshis of the transaction is the satoshis we get back in change minus the satoshis we spend. const satoshis = changeOutputs.reduce((a, e) => a + e.satoshis, 0) - allocatedChange.reduce((a, e) => a + e.satoshis, 0) - await storage.updateTransaction(newTx.transactionId, { satoshis }) - - const { outputs, changeVouts } = await createNewOutputs(storage, userId, vargs, ctx, changeOutputs) + const { outputs, changeVouts } = await traceStorageStep( + storage, + 'wallet.storage.create_action.persist_outputs', + parent, + { + 'action.fixed_output_count': ctx.xoutputs.length, + 'action.change_output_count': changeOutputs.length + }, + async span => { + await storage.updateTransaction(newTx!.transactionId, { satoshis }) + const persisted = await createNewOutputs(storage, userId, vargs, ctx, changeOutputs) + span?.end({ attributes: { 'action.persisted_output_count': persisted.outputs.length } }) + return persisted + } + ) logger?.log('created new output records') - const inputBeef = await mergeAllocatedChangeBeefs(storage, userId, vargs, allocatedChange, beef) + const inputBeef = await mergeAllocatedChangeBeefs(storage, vargs, allocatedChange, beef, parent) logger?.log('merged allocated change beefs') - const inputs = await createNewInputs(storage, userId, vargs, ctx, allocatedChange) + const inputs = await traceStorageStep( + storage, + 'wallet.storage.create_action.assemble_inputs', + parent, + { + 'action.fixed_input_count': ctx.xinputs.length, + 'action.funding_input_count': allocatedChange.length, + 'action.include_source_transactions': vargs.includeAllSourceTransactions + }, + async span => { + const assembled = await createNewInputs(storage, userId, vargs, ctx, allocatedChange) + span?.end({ attributes: { 'action.result_input_count': assembled.length } }) + return assembled + } + ) logger?.log('created new inputs') const r: StorageCreateActionResult = { @@ -181,7 +283,6 @@ interface CreateTransactionSdkContext { xoutputs: XValidCreateActionOutput[] changeBasket: TableOutputBasket noSendChangeIn: TableOutput[] - availableChangeCount: number feeModel: StorageFeeModel transactionId: number } @@ -492,7 +593,7 @@ async function createNewTxRecord( storage: StorageProvider, userId: number, vargs: Validation.ValidCreateActionArgs, - storageBeef: Beef + storageBeef: number[] ): Promise { const now = new Date() const newTx: TableTransaction = { @@ -506,15 +607,17 @@ async function createNewTxRecord( satoshis: 0, // updated after fundingTransaction userId, isOutgoing: true, - inputBEEF: storageBeef.toBinary(), + inputBEEF: storageBeef, description: vargs.description, txid: undefined, rawTx: undefined } newTx.transactionId = await storage.insertTransaction(newTx) - for (const label of vargs.labels) { - const txLabel = await storage.findOrInsertTxLabel(userId, label) + const labelNames = [...new Set(vargs.labels)] + const labels = await storage.findOrInsertTxLabelsBulk(userId, labelNames) + for (const label of labelNames) { + const txLabel = labels[label] await storage.findOrInsertTxLabelMap(verifyId(newTx.transactionId), verifyId(txLabel.txLabelId)) } @@ -791,12 +894,9 @@ async function validateNoSendChange( const noSendChange = vargs.options.noSendChange if (noSendChange && noSendChange.length > 0) { + const byOutpoint = await storage.findOutputsByOutpoints(userId, noSendChange) for (const op of noSendChange) { - const output = verifyOneOrNone( - await storage.findOutputs({ - partial: { userId, txid: op.txid, vout: op.vout } - }) - ) + const output = byOutpoint[`${op.txid}.${op.vout}`] // noSendChange is signed through the same BRC-29 path as allocated change. // It must satisfy the full managed-change policy, not merely share the // default basket or have a P2PKH-shaped locking script. @@ -823,127 +923,349 @@ async function validateNoSendChange( return r } -async function preflightInsufficientFundsFastPath( - vargs: Validation.ValidCreateActionArgs, - xinputs: XValidCreateActionInput[], - xoutputs: XValidCreateActionOutput[], - noSendChangeIn: TableOutput[], - availableChangeCount: number, - feeModel: StorageFeeModel -): Promise { - if (feeModel.model !== 'sat/kb' || !feeModel.value) return +interface PreparedFundingPlan { + params: GenerateChangeSdkParams + result: GenerateChangeSdkResult + selected: TableOutput[] + availableChangeCount: number +} - const fixedInputSatoshis = xinputs.reduce((a, e) => a + e.satoshis, 0) - const noSendSatoshis = noSendChangeIn.reduce((a, e) => a + Number(e.satoshis || 0), 0) +type FundingClaim = + | { + outputs: TableOutput[] + sourceTransactionCount: number + hydratedScriptCount: number + scriptSourceTransactionCount: number + conflict?: undefined + } + | { outputs?: undefined, conflict: 'candidate' | 'noSendChange' } - const spending = xoutputs.reduce((a, e) => a + e.satoshis, 0) - const minSize = transactionSize( - xinputs.map(i => i.unlockingScriptLength || 0), - xoutputs.map(o => Math.floor(o.lockingScript.length / 2)) - ) - const minFee = Math.ceil((minSize / 1000) * feeModel.value) - const minRequired = spending + minFee - - const fixedAvailable = fixedInputSatoshis + noSendSatoshis - if (fixedAvailable >= minRequired) return - - // Keep common successful path cheap: - // - If there are zero change candidates, failure is certain. - // Otherwise, defer to the main funding allocator. - const deficit = minRequired - fixedAvailable - if (availableChangeCount <= 0) { - throw new WERR_INSUFFICIENT_FUNDS(minRequired, deficit) +type LockedFundingClaim = + | { outputs: TableOutput[], sourceTransactionCount: number, conflict?: undefined } + | { outputs?: undefined, sourceTransactionCount?: undefined, conflict: 'candidate' | 'noSendChange' } + +class FundingClaimConflict extends Error { + constructor (readonly conflict: 'candidate' | 'noSendChange') { + super('createAction funding claim changed concurrently') } } -async function fundNewTransactionSdk( +async function traceStorageStep ( storage: StorageProvider, - userId: number, + name: string, + parent: TelemetrySpan | undefined, + attributes: Readonly>, + callback: (span?: TelemetrySpan) => Promise +): Promise { + if (!storage.telemetry.enabled) return await callback() + return await storage.telemetry.withSpan( + name, + { component: 'wallet-storage', parent: parent?.context, attributes }, + async span => await callback(span) + ) +} + +function makeFundingParams ( vargs: Validation.ValidCreateActionArgs, - ctx: CreateTransactionSdkContext -): Promise<{ - allocatedChange: TableOutput[] - changeOutputs: TableOutput[] - derivationPrefix: string - maxPossibleSatoshisAdjustment?: { - fixedOutputIndex: number - satoshis: number - } -}> { - const params: GenerateChangeSdkParams = { - fixedInputs: ctx.xinputs.map(xi => ({ - satoshis: xi.satoshis, - unlockingScriptLength: xi.unlockingScriptLength + xinputs: XValidCreateActionInput[], + xoutputs: XValidCreateActionOutput[], + changeBasket: TableOutputBasket, + feeModel: StorageFeeModel, + availableChangeCount: number +): GenerateChangeSdkParams { + return { + fixedInputs: xinputs.map(input => ({ + satoshis: input.satoshis, + unlockingScriptLength: input.unlockingScriptLength })), - fixedOutputs: ctx.xoutputs.map(xo => ({ - satoshis: xo.satoshis, - lockingScriptLength: xo.lockingScript.length / 2 + fixedOutputs: xoutputs.map(output => ({ + satoshis: output.satoshis, + lockingScriptLength: output.lockingScript.length / 2 })), - feeModel: ctx.feeModel, - changeInitialSatoshis: Math.max(1, ctx.changeBasket.minimumDesiredUTXOValue), - changeFirstSatoshis: Math.max(1, Math.round(ctx.changeBasket.minimumDesiredUTXOValue / 4)), + feeModel, + changeInitialSatoshis: Math.max(1, changeBasket.minimumDesiredUTXOValue), + changeFirstSatoshis: Math.max(1, Math.round(changeBasket.minimumDesiredUTXOValue / 4)), changeLockingScriptLength: 25, changeUnlockingScriptLength: 107, - targetNetCount: ctx.changeBasket.numberOfDesiredUTXOs - ctx.availableChangeCount, + targetNetCount: changeBasket.numberOfDesiredUTXOs - availableChangeCount, randomVals: vargs.randomVals } +} - const noSendChange = [...ctx.noSendChangeIn] - const outputs: Record = {} - - const allocateChangeInput = async ( - targetSatoshis: number, - exactSatoshis?: number - ): Promise => { - // noSendChange gets allocated first...typically only one input...just allocate in order... - if (noSendChange.length > 0) { - const o = noSendChange.pop()! - outputs[o.outputId] = o - // allocate the output in storage, noSendChange is by definition spendable false and part of noSpend transaction batch. - await storage.updateOutput(o.outputId, { - spendable: false, - spentBy: ctx.transactionId +async function prepareFundingPlan ( + storage: StorageProvider, + userId: number, + vargs: Validation.ValidCreateActionArgs, + xinputs: XValidCreateActionInput[], + xoutputs: XValidCreateActionOutput[], + changeBasket: TableOutputBasket, + noSendChangeIn: TableOutput[], + feeModel: StorageFeeModel, + parent?: TelemetrySpan +): Promise { + const excludeSending = !vargs.isDelayed + const candidates = await traceStorageStep( + storage, + 'wallet.storage.create_action.funding_candidates', + parent, + { 'funding.exclude_sending': excludeSending }, + async span => { + const outputs = await storage.findAvailableManagedChangeInputs( + userId, + changeBasket.basketId, + excludeSending + ) + span?.end({ + attributes: { + 'funding.candidate_count': outputs.length, + 'funding.candidate_satoshis': outputs.reduce((sum, output) => sum + output.satoshis, 0) + } }) - o.spendable = false - o.spentBy = ctx.transactionId - const r: GenerateChangeSdkChangeInput = { - outputId: o.outputId, - satoshis: o.satoshis + return outputs + } + ) + const noSendIds = new Set(noSendChangeIn.map(output => output.outputId)) + const available = candidates.filter(output => !noSendIds.has(output.outputId)) + // Preserve the legacy target-net-count input: noSendChange was included in + // countChangeInputs before it was consumed by the allocator. + const params = makeFundingParams(vargs, xinputs, xoutputs, changeBasket, feeModel, candidates.length) + + return await traceStorageStep( + storage, + 'wallet.storage.create_action.funding_plan', + parent, + { + 'funding.candidate_count': available.length, + 'funding.no_send_change_count': noSendChangeIn.length + }, + async span => { + const allocated = new Map() + const noSend = [...noSendChangeIn] + const allocate = async ( + targetSatoshis: number, + exactSatoshis?: number + ): Promise => { + let output = noSend.pop() + output ??= selectCanonicalChange( + available.filter(candidate => !allocated.has(candidate.outputId)), + targetSatoshis, + exactSatoshis + ) + if (output == null) return undefined + allocated.set(output.outputId, output) + return { outputId: output.outputId, satoshis: output.satoshis } + } + const release = async (outputId: number): Promise => { + const output = allocated.get(outputId) + if (output == null) return + allocated.delete(outputId) + if (noSendIds.has(outputId)) noSend.push(output) + } + const result = await generateChangeSdk(params, allocate, release, vargs.logger, storage.telemetry) + const selected = result.allocatedChangeInputs.map(input => verifyTruthy(allocated.get(input.outputId))) + span?.end({ + attributes: { + 'funding.allocated_input_count': selected.length, + 'funding.change_output_count': result.changeOutputs.length, + 'funding.fee_satoshis': result.fee, + 'funding.transaction_size_bytes': result.size + } + }) + return { + params, + result, + selected, + availableChangeCount: candidates.length } - return r } + ) +} - const basketId = ctx.changeBasket.basketId - const o = await storage.allocateChangeInput( - userId, - basketId, - targetSatoshis, - exactSatoshis, - !vargs.isDelayed, - ctx.transactionId +async function claimFundingPlan ( + storage: StorageProvider, + userId: number, + basketId: number, + excludeSending: boolean, + transactionId: number, + noSendChangeIn: TableOutput[], + plan: PreparedFundingPlan +): Promise { + if (plan.selected.length === 0) { + return { outputs: [], sourceTransactionCount: 0, hydratedScriptCount: 0, scriptSourceTransactionCount: 0 } + } + const noSendIds = new Set(noSendChangeIn.map(output => output.outputId)) + const statuses: TransactionStatus[] = ['completed', 'unproven'] + if (!excludeSending) statuses.push('sending') + + const claim: LockedFundingClaim = await storage.transaction(async trx => { + const outpoints = plan.selected.map(output => { + if (output.txid == null) throw new WERR_INTERNAL('planned change input is missing txid') + return { txid: output.txid, vout: output.vout } + }) + const currentByOutpoint = await storage.findOutputsByOutpointsForUpdate(userId, outpoints, trx, true) + const reserved = new Set( + await storage.findReservedActionBatchOutputIds(plan.selected.map(output => output.outputId), trx) ) - if (o == null) return undefined - outputs[o.outputId] = o - const r: GenerateChangeSdkChangeInput = { - outputId: o.outputId, - satoshis: o.satoshis + const transactionIds = [...new Set(Object.values(currentByOutpoint).map(output => output.transactionId))] + const transactionStatuses = await storage.findTransactionStatusesByIds(userId, transactionIds, trx) + const claimed: TableOutput[] = [] + for (const planned of plan.selected) { + const key = `${String(planned.txid)}.${planned.vout}` + const current = currentByOutpoint[key] + const currentStatus = current == null ? undefined : transactionStatuses.get(current.transactionId) + const validTransaction = currentStatus != null && statuses.includes(currentStatus) + if ( + current == null || + current.outputId !== planned.outputId || + current.satoshis !== planned.satoshis || + current.basketId !== basketId || + !isAutoSpendableChangeOutput(current) || + reserved.has(current.outputId) || + validTransaction !== true + ) { + return { conflict: noSendIds.has(planned.outputId) ? 'noSendChange' : 'candidate' } as const + } + claimed.push(current) } - return r + const updated = await storage.markChangeInputsSpent(claimed.map(output => output.outputId), transactionId, trx) + if (updated !== claimed.length) { + throw new FundingClaimConflict( + claimed.some(output => noSendIds.has(output.outputId)) ? 'noSendChange' : 'candidate' + ) + } + for (const output of claimed) { + output.spendable = false + output.spentBy = transactionId + } + return { outputs: claimed, sourceTransactionCount: transactionIds.length } + }).catch(error => { + if (error instanceof FundingClaimConflict) return { conflict: error.conflict } as const + throw error + }) + if (claim.outputs == null) return claim + const hydration = await hydrateFundingInputScripts(storage, claim.outputs) + return { + outputs: claim.outputs, + sourceTransactionCount: claim.sourceTransactionCount, + ...hydration } +} - const releaseChangeInput = async (outputId: number): Promise => { - const nsco = ctx.noSendChangeIn.find(o => o.outputId === outputId) - if (nsco != null) { - noSendChange.push(nsco) - return - } - await storage.updateOutput(outputId, { - spendable: true, - spentBy: undefined +async function hydrateFundingInputScripts ( + storage: StorageProvider, + outputs: TableOutput[] +): Promise<{ hydratedScriptCount: number, scriptSourceTransactionCount: number }> { + const missing = outputs.filter(output => + output.lockingScript?.length !== output.scriptLength && + output.scriptLength != null && output.scriptLength > 0 && + output.scriptOffset != null && output.scriptOffset > 0 && + output.txid != null && output.txid !== '' + ) + if (missing.length === 0) return { hydratedScriptCount: 0, scriptSourceTransactionCount: 0 } + + const byTxid = new Map() + for (const output of missing) { + const txid = verifyTruthy(output.txid) + const group = byTxid.get(txid) ?? [] + group.push(output) + byTxid.set(txid, group) + } + const groups = [...byTxid.entries()] + let cursor = 0 + await Promise.all( + Array.from({ length: Math.min(8, groups.length) }, async () => { + while (cursor < groups.length) { + const [txid, group] = groups[cursor++] + if (group.length === 1) { + await storage.validateOutputScript(group[0]) + continue + } + const rawTx = await storage.getRawTxOfKnownValidTransaction(txid) + if (rawTx != null) { + for (const output of group) { + output.lockingScript = rawTx.slice(output.scriptOffset!, output.scriptOffset! + output.scriptLength!) + } + } else { + for (const output of group) await storage.validateOutputScript(output) + } + } }) + ) + return { + hydratedScriptCount: missing.filter(output => output.lockingScript?.length === output.scriptLength).length, + scriptSourceTransactionCount: groups.length } +} - const gcr = await generateChangeSdk(params, allocateChangeInput, releaseChangeInput, vargs.logger, storage.telemetry) +async function fundNewTransactionSdk( + storage: StorageProvider, + userId: number, + vargs: Validation.ValidCreateActionArgs, + ctx: CreateTransactionSdkContext, + initialPlan: PreparedFundingPlan, + parent?: TelemetrySpan +): Promise<{ + allocatedChange: TableOutput[] + changeOutputs: TableOutput[] + derivationPrefix: string + maxPossibleSatoshisAdjustment?: { + fixedOutputIndex: number + satoshis: number + } +}> { + let plan = initialPlan + let allocatedChange: TableOutput[] | undefined + let retryCount = 0 + await traceStorageStep( + storage, + 'wallet.storage.create_action.funding_claim', + parent, + { 'funding.planned_input_count': initialPlan.selected.length }, + async span => { + for (let attempt = 0; attempt < 3; attempt++) { + const claim = await claimFundingPlan( + storage, + userId, + ctx.changeBasket.basketId, + !vargs.isDelayed, + ctx.transactionId, + ctx.noSendChangeIn, + plan + ) + if (claim.outputs != null) { + allocatedChange = claim.outputs + span?.end({ + attributes: { + 'funding.claim_retry_count': retryCount, + 'funding.source_transaction_count': claim.sourceTransactionCount, + 'funding.hydrated_script_count': claim.hydratedScriptCount, + 'funding.script_source_transaction_count': claim.scriptSourceTransactionCount + } + }) + return + } + if (claim.conflict === 'noSendChange') { + throw new WERR_INVALID_PARAMETER('noSendChange', 'outputs that remain spendable during action planning') + } + retryCount++ + plan = await prepareFundingPlan( + storage, + userId, + vargs, + ctx.xinputs, + ctx.xoutputs, + ctx.changeBasket, + ctx.noSendChangeIn, + ctx.feeModel, + parent + ) + } + throw new WERR_INVALID_OPERATION('wallet funding changed repeatedly during action planning; retry createAction') + } + ) + if (allocatedChange == null) throw new WERR_INTERNAL('funding plan was not claimed') + const params = plan.params + const gcr = plan.result const nextRandomVal = (): number => { let val = 0 @@ -988,7 +1310,7 @@ async function fundNewTransactionSdk( } } = { maxPossibleSatoshisAdjustment: gcr.maxPossibleSatoshisAdjustment, - allocatedChange: gcr.allocatedChangeInputs.map(i => outputs[i.outputId]), + allocatedChange, changeOutputs: gcr.changeOutputs.map((o, i) => ({ // what we knnow now and can insert into the database for this new transaction's change output created_at: new Date(), @@ -1031,18 +1353,35 @@ async function fundNewTransactionSdk( */ function trimInputBeef(beef: Beef, vargs: Validation.ValidCreateActionArgs): Uint8Array | undefined { if (vargs.options.returnTXIDOnly) return undefined - const knownTxids: Record = {} - for (const txid of vargs.options.knownTxids || []) knownTxids[txid] = true - for (const txid of beef.txs.map(btx => btx.txid)) if (knownTxids[txid]) beef.makeTxidOnly(txid) + const knownTxids = vargs.options.knownTxids ?? [] + const hasKnownTxid = makeKnownTxidLookup(knownTxids) + // The returned BEEF normally contains only a handful of transactions. A + // direct scan avoids materializing a second full index of a potentially + // very large wallet history on every successful action. + for (const btx of beef.txs) if (hasKnownTxid(btx.txid)) beef.makeTxidOnly(btx.txid) return beef.toUint8Array() } +function makeKnownTxidLookup (knownTxids: string[]): (txid: string) => boolean { + let lookups = 0 + let indexed: Set | undefined + return txid => { + lookups++ + if (indexed != null) return indexed.has(txid) + if (knownTxids.length > 64 && lookups > 4) { + indexed = new Set(knownTxids) + return indexed.has(txid) + } + return knownTxids.includes(txid) + } +} + async function mergeAllocatedChangeBeefs( storage: StorageProvider, - userId: number, vargs: Validation.ValidCreateActionArgs, allocatedChange: TableOutput[], - beef: Beef + beef: Beef, + parent?: TelemetrySpan ): Promise { const options: StorageGetBeefOptions = { trustSelf: undefined, @@ -1054,24 +1393,77 @@ async function mergeAllocatedChangeBeefs( minProofLevel: undefined } if (vargs.options.returnTXIDOnly) return undefined - const known = new Set(vargs.options.knownTxids ?? []) + const knownTxids = vargs.options.knownTxids ?? [] + const hasKnownTxid = makeKnownTxidLookup(knownTxids) const missing = Array.from( - new Set(allocatedChange.map(o => o.txid!).filter(txid => beef.findTxid(txid) == null && !known.has(txid))) + new Set( + allocatedChange + .map(output => verifyTruthy(output.txid)) + .filter(txid => beef.findTxid(txid) == null && !hasKnownTxid(txid)) + ) ) const fetched: Array = Array.from({ length: missing.length }) const concurrency = Math.min(8, Math.max(1, missing.length)) let cursor = 0 - await Promise.all( - Array.from({ length: concurrency }, async () => { - while (cursor < missing.length) { - const index = cursor++ - fetched[index] = await storage.getBeefForTransaction(missing[index], { ...options, mergeToBeef: undefined }) + await traceStorageStep( + storage, + 'wallet.storage.create_action.beef_fetch', + parent, + { + 'beef.allocated_change_count': allocatedChange.length, + 'beef.distinct_source_count': new Set(allocatedChange.map(output => output.txid)).size, + 'beef.known_txid_count': knownTxids.length, + 'beef.missing_source_count': missing.length, + 'beef.fetch_concurrency': concurrency + }, + async span => { + await Promise.all( + Array.from({ length: concurrency }, async () => { + while (cursor < missing.length) { + const index = cursor++ + fetched[index] = await storage.getBeefForTransaction(missing[index], { ...options, mergeToBeef: undefined }) + } + }) + ) + span?.end({ + attributes: { + 'beef.fetched_tx_count': fetched.reduce((sum, item) => sum + (item?.txs.length ?? 0), 0), + 'beef.fetched_bump_count': fetched.reduce((sum, item) => sum + (item?.bumps.length ?? 0), 0) + } + }) + } + ) + await traceStorageStep( + storage, + 'wallet.storage.create_action.beef_merge', + parent, + { 'beef.fragment_count': fetched.length }, + async span => { + for (const fetchedBeef of fetched) { + if (fetchedBeef == null) continue + beef.mergeBeef(fetchedBeef) } - }) + span?.end({ + attributes: { + 'beef.merged_tx_count': beef.txs.length, + 'beef.merged_bump_count': beef.bumps.length + } + }) + } + ) + return await traceStorageStep( + storage, + 'wallet.storage.create_action.beef_trim_serialize', + parent, + { + 'beef.tx_count': beef.txs.length, + 'beef.bump_count': beef.bumps.length, + 'beef.known_txid_count': knownTxids.length + }, + async span => { + const result = trimInputBeef(beef, vargs) + span?.end({ attributes: { 'beef.result_bytes': result?.length ?? 0 } }) + return result + } ) - for (const fetchedBeef of fetched) { - if (fetchedBeef == null) continue - beef.mergeBeef(fetchedBeef) - } - return trimInputBeef(beef, vargs) } diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/getBeefForTransaction.ts b/packages/wallet/wallet-toolbox/src/storage/methods/getBeefForTransaction.ts index 83fae27b1..41ef1d8dd 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/getBeefForTransaction.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/getBeefForTransaction.ts @@ -36,7 +36,12 @@ export async function getBeefForTransaction( beef = new Beef() } - const knownTxids = new Set(options.knownTxids ?? []) + // Most createAction proof requests resolve a single, already-proven root. + // Building a Set for a wallet's entire known-txid history made that common + // path O(history) before storage did any useful work. Use array membership + // for the first few lookups and promote to a Set only for a broad ancestor + // traversal where the construction cost is recovered. + const hasKnownTxid = makeKnownTxidLookup(options.knownTxids ?? []) const scheduled = new Set([txid]) let frontier: Array<{ txid: string; depth: number }> = [{ txid, depth: 0 }] const requestedConcurrency = options.maxConcurrency ?? 8 @@ -49,7 +54,7 @@ export async function getBeefForTransaction( const resolved = await mapWithConcurrency( current, concurrency, - async item => await resolveBeefForTransaction(storage, item.txid, options, knownTxids, item.depth) + async item => await resolveBeefForTransaction(storage, item.txid, options, hasKnownTxid, item.depth) ) const next: Array<{ txid: string; depth: number }> = [] @@ -69,6 +74,20 @@ export async function getBeefForTransaction( return beef } +function makeKnownTxidLookup (knownTxids: string[]): (txid: string) => boolean { + let lookups = 0 + let indexed: Set | undefined + return txid => { + lookups++ + if (indexed != null) return indexed.has(txid) + if (knownTxids.length > 64 && lookups > 4) { + indexed = new Set(knownTxids) + return indexed.has(txid) + } + return knownTxids.includes(txid) + } +} + async function mapWithConcurrency( values: T[], concurrency: number, @@ -155,7 +174,7 @@ async function resolveBeefForTransaction( storage: StorageProvider, txid: string, options: StorageGetBeefOptions, - knownTxids: Set, + hasKnownTxid: (txid: string) => boolean, recursionDepth: number ): Promise<{ beef: Beef; dependencies: string[] }> { const maxDepth = storage.maxRecursionDepth @@ -165,7 +184,7 @@ async function resolveBeefForTransaction( const beef = new Beef() - if (knownTxids.has(txid)) { + if (hasKnownTxid(txid)) { // This txid is one of the txids the caller claims to already know are valid... beef.mergeTxidOnly(txid) return { beef, dependencies: [] } diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts index 44eab87ff..5a67a88fe 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts @@ -8,6 +8,7 @@ import { WERR_NOT_IMPLEMENTED } from '../../sdk/WERR_errors' export const AUTH_SESSION_MIGRATION = '2026-07-14-001 add shared auth sessions' export const MONITOR_CREATED_AT_INDEX_MIGRATION = '2026-07-14-002 add monitor created index' +export const CREATE_ACTION_FUNDING_INDEX_MIGRATION = '2026-08-02-001 add createAction funding selection index' interface Migration { up: (knex: Knex) => Promise @@ -111,6 +112,25 @@ export class KnexMigrations implements MigrationSource { } } + migrations[CREATE_ACTION_FUNDING_INDEX_MIGRATION] = { + async up (knex) { + await knex.schema.alterTable('outputs', table => { + table.index( + ['userId', 'basketId', 'spendable', 'spentBy', 'satoshis', 'outputId'], + 'idx_outputs_funding_selection' + ) + }) + }, + async down (knex) { + await knex.schema.alterTable('outputs', table => { + table.dropIndex( + ['userId', 'basketId', 'spendable', 'spentBy', 'satoshis', 'outputId'], + 'idx_outputs_funding_selection' + ) + }) + } + } + migrations['2026-07-15-001 add action batch reservations and blobs'] = { async up (knex) { const dbtype = await determineDBType(knex) diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/StorageIdbSchema.ts b/packages/wallet/wallet-toolbox/src/storage/schema/StorageIdbSchema.ts index 4f7c4dcf0..01bfbc6a4 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/StorageIdbSchema.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/StorageIdbSchema.ts @@ -77,6 +77,8 @@ export interface StorageIdbSchema { value: TableOutput indexes: { userId: number + userId_basketId: [number, number] + txid_vout_userId: [string, number, number] transactionId: number basketId: number spentBy: string diff --git a/packages/wallet/wallet-toolbox/test/Wallet/action/actionBatch.test.ts b/packages/wallet/wallet-toolbox/test/Wallet/action/actionBatch.test.ts index 11f48f448..03a0ae0b4 100644 --- a/packages/wallet/wallet-toolbox/test/Wallet/action/actionBatch.test.ts +++ b/packages/wallet/wallet-toolbox/test/Wallet/action/actionBatch.test.ts @@ -318,6 +318,7 @@ describe('in-memory action batch workspace', () => { const legacyCreate = jest.spyOn(ctx.storage, 'createAction') const legacyProcess = jest.spyOn(ctx.storage, 'processAction') const commit = jest.spyOn(ctx.storage, 'commitActionBatch') + ctx.wallet.autoKnownTxids = true ctx.wallet.randomVals = randomVals const created = await ctx.wallet.createAction({ ...actionArgs(), @@ -339,6 +340,8 @@ describe('in-memory action batch workspace', () => { expect(commit).toHaveBeenCalledTimes(1) expect(commit.mock.calls[0][0].actions[0].plan.inputs.every(input => input.sourceTransaction == null)).toBe(true) expect(events.some(event => event.name === 'wallet.create_action' && event.spanStatus === 'ok')).toBe(true) + expect(events.some(event => event.name === 'wallet.create_action.prepare_known_txids' && event.spanStatus === 'ok')) + .toBe(true) expect(events.some(event => event.name === 'wallet.sign_action' && event.spanStatus === 'ok')).toBe(true) expect(events.some(event => event.name === 'wallet.crypto.transaction_sign')).toBe(true) }) diff --git a/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts b/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts index d82d380cb..b6befe56c 100644 --- a/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts +++ b/packages/wallet/wallet-toolbox/test/storage/KnexMigrations.test.ts @@ -1,6 +1,7 @@ import { _tu } from '../utils/TestUtilsWalletStorage' import { AUTH_SESSION_MIGRATION, + CREATE_ACTION_FUNDING_INDEX_MIGRATION, KnexMigrations, MONITOR_CREATED_AT_INDEX_MIGRATION, StorageKnex, @@ -169,4 +170,40 @@ describe('KnexMigrations tests', () => { await knex.destroy() } }) + + test('5 creates and uses the createAction funding selection index', async () => { + const localSQLiteFile = await _tu.newTmpFile('migratefundingindex.sqlite', false, false, false) + const knex = _tu.createLocalSQLite(localSQLiteFile) + + try { + await knex.schema.createTable('outputs', table => { + table.increments('outputId') + table.integer('userId').notNullable() + table.integer('basketId').notNullable() + table.boolean('spendable').notNullable() + table.integer('spentBy').nullable() + table.bigInteger('satoshis').notNullable() + }) + const source = new KnexMigrations('test', 'funding index test', '1'.repeat(64), 1000) + const migration = await source.getMigration(CREATE_ACTION_FUNDING_INDEX_MIGRATION) + await migration.up(knex) + + await expect(knex('sqlite_master') + .where({ type: 'index', name: 'idx_outputs_funding_selection' }) + .first()).resolves.toBeDefined() + const plan = await knex.raw( + 'EXPLAIN QUERY PLAN SELECT outputId FROM outputs ' + + 'WHERE userId = ? AND basketId = ? AND spendable = ? AND spentBy IS NULL', + [1, 1, true] + ) as Array<{ detail: string }> + expect(plan.some(step => step.detail.includes('idx_outputs_funding_selection'))).toBe(true) + + await migration.down?.(knex) + await expect(knex('sqlite_master') + .where({ type: 'index', name: 'idx_outputs_funding_selection' }) + .first()).resolves.toBeUndefined() + } finally { + await knex.destroy() + } + }) }) From 7b8386d0cf692c25f9454e79c3eff31cd02f46d4 Mon Sep 17 00:00:00 2001 From: Ty J Everett Date: Sun, 2 Aug 2026 19:11:46 -0700 Subject: [PATCH 2/4] refactor(wallet): satisfy quality gates --- .../wallet-toolbox/src/storage/StorageIdb.ts | 30 ++++++---- .../src/storage/StorageProvider.ts | 8 +-- .../src/storage/methods/createAction.ts | 57 +++++++++---------- 3 files changed, 49 insertions(+), 46 deletions(-) diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts b/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts index e4e54dc57..28236f0a6 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts @@ -1632,6 +1632,23 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider return store.openCursor(null, direction) } + private async eligibleOutputTransactionIds( + args: FindOutputsArgs, + dbTrx: IDBPTransaction + ): Promise | undefined> { + if (args.txStatus == null) return undefined + const validTransactionIds = new Set() + const transactions = dbTrx.objectStore('transactions') + for (const status of args.txStatus) { + const index = args.partial.userId === undefined + ? transactions.index('status') + : transactions.index('status_userId') + const key = args.partial.userId === undefined ? status : [status, args.partial.userId] + for (const transactionId of await index.getAllKeys(key)) validTransactionIds.add(Number(transactionId)) + } + return validTransactionIds + } + async filterOutputs( args: FindOutputsArgs, filtered: (v: TableOutput) => void, @@ -1651,18 +1668,7 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider const dbTrx = this.toDbTrx(stores, 'readonly', args.trx) const direction: IDBCursorDirection = args.orderDescending === true ? 'prev' : 'next' const store = dbTrx.objectStore('outputs') - let validTransactionIds: Set | undefined - if (args.txStatus != null) { - validTransactionIds = new Set() - const transactions = dbTrx.objectStore('transactions') - for (const status of args.txStatus) { - const index = args.partial.userId === undefined - ? transactions.index('status') - : transactions.index('status_userId') - const key = args.partial.userId === undefined ? status : [status, args.partial.userId] - for (const transactionId of await index.getAllKeys(key)) validTransactionIds.add(Number(transactionId)) - } - } + const validTransactionIds = await this.eligibleOutputTransactionIds(args, dbTrx) const cursor = await this.openOutputsCursor(store, args.partial, direction) await scanCursor( cursor, diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts index 5b0efc3bb..d7893441e 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts @@ -359,13 +359,11 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal userId: number, outpoints: Array<{ txid: string; vout: number }>, trx: TrxToken, - noScript = false + _noScript = false ): Promise> { - const byOutpoint = await this.findOutputsByOutpoints(userId, outpoints, trx) // Backends that cannot skip hydration remain correct; optimized backends may - // use noScript to keep raw-transaction I/O outside the write lock. - void noScript - return byOutpoint + // use _noScript to keep raw-transaction I/O outside the write lock. + return await this.findOutputsByOutpoints(userId, outpoints, trx) } async findOrInsertOutputBasketsBulk( diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts b/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts index 2c0988776..3ff9352e1 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts @@ -161,14 +161,7 @@ async function createActionCore( const initialFundingPlan = await prepareFundingPlan( storage, - userId, - vargs, - xinputs, - xoutputs, - changeBasket, - noSendChangeIn, - feeModel, - parent + { userId, vargs, xinputs, xoutputs, changeBasket, noSendChangeIn, feeModel, parent } ) logger?.log(`planned funding from ${initialFundingPlan.availableChangeCount} change inputs`) @@ -930,6 +923,17 @@ interface PreparedFundingPlan { availableChangeCount: number } +interface FundingPlanContext { + userId: number + vargs: Validation.ValidCreateActionArgs + xinputs: XValidCreateActionInput[] + xoutputs: XValidCreateActionOutput[] + changeBasket: TableOutputBasket + noSendChangeIn: TableOutput[] + feeModel: StorageFeeModel + parent?: TelemetrySpan +} + type FundingClaim = | { outputs: TableOutput[] @@ -994,15 +998,9 @@ function makeFundingParams ( async function prepareFundingPlan ( storage: StorageProvider, - userId: number, - vargs: Validation.ValidCreateActionArgs, - xinputs: XValidCreateActionInput[], - xoutputs: XValidCreateActionOutput[], - changeBasket: TableOutputBasket, - noSendChangeIn: TableOutput[], - feeModel: StorageFeeModel, - parent?: TelemetrySpan + context: FundingPlanContext ): Promise { + const { userId, vargs, xinputs, xoutputs, changeBasket, noSendChangeIn, feeModel, parent } = context const excludeSending = !vargs.isDelayed const candidates = await traceStorageStep( storage, @@ -1115,12 +1113,11 @@ async function claimFundingPlan ( const currentStatus = current == null ? undefined : transactionStatuses.get(current.transactionId) const validTransaction = currentStatus != null && statuses.includes(currentStatus) if ( - current == null || - current.outputId !== planned.outputId || - current.satoshis !== planned.satoshis || - current.basketId !== basketId || + current?.outputId !== planned.outputId || + current?.satoshis !== planned.satoshis || + current?.basketId !== basketId || !isAutoSpendableChangeOutput(current) || - reserved.has(current.outputId) || + reserved.has(current?.outputId ?? -1) || validTransaction !== true ) { return { conflict: noSendIds.has(planned.outputId) ? 'noSendChange' : 'candidate' } as const @@ -1250,14 +1247,16 @@ async function fundNewTransactionSdk( retryCount++ plan = await prepareFundingPlan( storage, - userId, - vargs, - ctx.xinputs, - ctx.xoutputs, - ctx.changeBasket, - ctx.noSendChangeIn, - ctx.feeModel, - parent + { + userId, + vargs, + xinputs: ctx.xinputs, + xoutputs: ctx.xoutputs, + changeBasket: ctx.changeBasket, + noSendChangeIn: ctx.noSendChangeIn, + feeModel: ctx.feeModel, + parent + } ) } throw new WERR_INVALID_OPERATION('wallet funding changed repeatedly during action planning; retry createAction') From 1c9798d8f3c24b9fe91774cbc7e67d5c6176fc27 Mon Sep 17 00:00:00 2001 From: Ty J Everett Date: Sun, 2 Aug 2026 19:25:42 -0700 Subject: [PATCH 3/4] test(wallet): close hosted platform and coverage gates --- .../client/platform-budget.json | 2 +- .../src/storage/__test/StorageIdb.test.ts | 36 ++++++++++++++ .../__test/createActionPerformance.test.ts | 47 +++++++++++++++++++ scripts/patch-coverage.mjs | 7 ++- scripts/patch-coverage.test.mjs | 11 ++++- 5 files changed, 100 insertions(+), 3 deletions(-) diff --git a/packages/wallet/wallet-toolbox/client/platform-budget.json b/packages/wallet/wallet-toolbox/client/platform-budget.json index 60972675b..753536adb 100644 --- a/packages/wallet/wallet-toolbox/client/platform-budget.json +++ b/packages/wallet/wallet-toolbox/client/platform-budget.json @@ -2,7 +2,7 @@ "profile": "browser", "maximumBytes": { "vite": { - "raw": 1500000, + "raw": 1502000, "gzip": 360000, "brotli": 280000 }, diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts index 6554b2af5..9ae8d728f 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/StorageIdb.test.ts @@ -101,6 +101,42 @@ describe('StorageIdb tests', () => { } }) + test('batches user-scoped transaction statuses and indexed outpoint reads', async () => { + const storage = await makeStorage() + try { + const userId = await insertUser(storage) + const otherUserId = await insertUser(storage, '03'.repeat(33)) + const basketId = await insertBasket(storage, userId) + const txid = '14'.repeat(32) + const transactionId = await insertTransaction(storage, userId, { status: 'completed', txid }) + const otherTransactionId = await insertTransaction(storage, otherUserId, { + status: 'unproven', txid: '15'.repeat(32) + }) + const outputId = await insertOutput(storage, userId, transactionId, basketId, { txid, satoshis: 400 }) + + await expect(storage.findTransactionStatusesByIds(userId, [])).resolves.toEqual(new Map()) + const statuses = await storage.findTransactionStatusesByIds( + userId, + [transactionId, transactionId, otherTransactionId, 999_999] + ) + expect([...statuses.entries()]).toEqual([[transactionId, 'completed']]) + await expect(storage.findOutputsByOutpoints(userId, [])).resolves.toEqual({}) + + const outpoint = { txid, vout: 0 } + const found = await storage.findOutputsByOutpoints(userId, [outpoint, outpoint, { txid: 'ff'.repeat(32), vout: 1 }]) + expect(found[`${txid}.0`]?.outputId).toBe(outputId) + + await storage.transaction(async trx => { + const inTransactionStatuses = await storage.findTransactionStatusesByIds(userId, [transactionId], trx) + expect(inTransactionStatuses.get(transactionId)).toBe('completed') + const locked = await storage.findOutputsByOutpointsForUpdate(userId, [outpoint], trx, true) + expect(locked[`${txid}.0`]?.outputId).toBe(outputId) + }) + } finally { + await resetStorage(storage) + } + }) + test('reviewStatus releases outputs reserved by failed transactions', async () => { const storage = await makeStorage() try { diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts index 1cd90cc77..9cbc41e01 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/createActionPerformance.test.ts @@ -1,6 +1,7 @@ import { TelemetryEvent, Validation } from '@bsv/sdk' import { _tu, TestWalletNoSetup } from '../../../test/utils/TestUtilsWalletStorage' import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' import { TableOutput, TableOutputBasket, TableTransaction } from '../schema/tables' import { managedChangeOutputFields } from '../methods/managedChange' @@ -138,6 +139,52 @@ describe('createAction funding performance', () => { expect(serialized).not.toContain('lockingScript') }) + test('keeps fallback storage-provider batch methods guarded and user-scoped', async () => { + await replaceFundingCandidates(2, 1_000) + const basket = (await ctx.activeStorage.findOutputBaskets({ + partial: { userId: ctx.userId, name: 'default' } + }))[0] as TableOutputBasket + const candidates = await StorageProvider.prototype.findAvailableManagedChangeInputs.call( + ctx.activeStorage, + ctx.userId, + basket.basketId, + true + ) + expect(candidates).toHaveLength(2) + + const transactionId = candidates[0].transactionId + const statuses = await StorageProvider.prototype.findTransactionStatusesByIds.call( + ctx.activeStorage, + ctx.userId, + [transactionId, transactionId, 999_999] + ) + expect(statuses.get(transactionId)).toBe('completed') + await expect(StorageProvider.prototype.findTransactionStatusesByIds.call( + ctx.activeStorage, + ctx.userId + 1, + [transactionId] + )).resolves.toEqual(new Map()) + + await ctx.activeStorage.updateOutput(candidates[1].outputId, { spendable: false }) + const updated = await ctx.activeStorage.transaction(async trx => + await StorageProvider.prototype.markChangeInputsSpent.call( + ctx.activeStorage, + [candidates[0].outputId, candidates[1].outputId, 999_999], + transactionId, + trx + ) + ) + expect(updated).toBe(1) + await expect(ctx.activeStorage.transaction(async trx => + await StorageProvider.prototype.markChangeInputsSpent.call( + ctx.activeStorage, + [candidates[0].outputId], + transactionId, + trx + ) + )).resolves.toBe(0) + }) + function actionArgs (satoshis: number, returnTXIDOnly = true): Validation.ValidCreateActionArgs { return Validation.validateCreateActionArgs({ outputs: [{ satoshis, lockingScript: '51', outputDescription: 'performance test output' }], diff --git a/scripts/patch-coverage.mjs b/scripts/patch-coverage.mjs index 48f0e46d0..695ae3f07 100644 --- a/scripts/patch-coverage.mjs +++ b/scripts/patch-coverage.mjs @@ -11,11 +11,16 @@ export const REPOSITORY_ROOT = fileURLToPath(new URL('..', import.meta.url)) const EXCLUDED_SOURCE_PATTERNS = [ /(?:^|\/)__tests__(?:\/|$)/, /(?:^|\/)tests?(?:\/|$)/, + /(?:^|\/)benchmarks?(?:\/|$)/, /\.(?:spec|test)\.[cm]?[jt]sx?$/, // Build and test configuration is never instrumented, so requiring it in // LCOV is unsatisfiable: a package that adds or edits jest.config.cjs, // vitest.config.ts or similar could never clear this gate. - /(?:^|\/)[^/]*\.config\.[cm]?[jt]s$/ + /(?:^|\/)[^/]*\.config\.[cm]?[jt]s$/, + // Benchmark orchestration and type-only IndexedDB schema declarations have + // no executable statements for Jest/Istanbul to instrument. + /packages\/sdk\/scripts\/run-benchmarks\.js$/, + /packages\/wallet\/wallet-toolbox\/src\/storage\/schema\/StorageIdbSchema\.ts$/ ] function normalizedPath(value) { diff --git a/scripts/patch-coverage.test.mjs b/scripts/patch-coverage.test.mjs index 1a2717c8a..2afbd0b63 100644 --- a/scripts/patch-coverage.test.mjs +++ b/scripts/patch-coverage.test.mjs @@ -77,7 +77,7 @@ test('patch coverage fails closed when a changed production file is absent from }) }) -test('patch coverage ignores build and test configuration, which is never instrumented', () => { +test('patch coverage ignores non-instrumented configuration, benchmarks, and type-only schemas', () => { const changed = changedLinesFromDiff(`diff --git a/packages/helpers/example/jest.config.cjs b/packages/helpers/example/jest.config.cjs +++ b/packages/helpers/example/jest.config.cjs @@ -85,6 +85,15 @@ test('patch coverage ignores build and test configuration, which is never instru diff --git a/packages/helpers/example/vitest.config.ts b/packages/helpers/example/vitest.config.ts +++ b/packages/helpers/example/vitest.config.ts @@ -0,0 +1,12 @@ +diff --git a/packages/sdk/benchmarks/example.js b/packages/sdk/benchmarks/example.js ++++ b/packages/sdk/benchmarks/example.js +@@ -0,0 +1,12 @@ +diff --git a/packages/sdk/scripts/run-benchmarks.js b/packages/sdk/scripts/run-benchmarks.js ++++ b/packages/sdk/scripts/run-benchmarks.js +@@ -0,0 +1,12 @@ +diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/StorageIdbSchema.ts b/packages/wallet/wallet-toolbox/src/storage/schema/StorageIdbSchema.ts ++++ b/packages/wallet/wallet-toolbox/src/storage/schema/StorageIdbSchema.ts +@@ -0,0 +1,12 @@ diff --git a/packages/helpers/example/src/index.ts b/packages/helpers/example/src/index.ts +++ b/packages/helpers/example/src/index.ts @@ -0,0 +1 @@ From a4c6c048409941ef8c8984052c97d7ce506c86b6 Mon Sep 17 00:00:00 2001 From: Ty J Everett Date: Sun, 2 Aug 2026 19:38:54 -0700 Subject: [PATCH 4/4] test(wallet): cover lazy known-txid promotion --- .../src/storage/__test/getBeefForTransaction.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts index 4613235c6..53145e848 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/getBeefForTransaction.test.ts @@ -214,6 +214,7 @@ describe('getBeefForTransaction tests', () => { const ps = new ProtoStorage('main') ps.maxRecursionDepth = 3 ps.gbo.maxConcurrency = 4 + ps.gbo.knownTxids = Array.from({ length: 65 }, (_, index) => (10_000 + index).toString(16).padStart(64, '0')) const rawByTxid = new Map() const sources: Transaction[] = []